kopia lustrzana https://github.com/micropython/micropython
extmod/vfs_reader: Add file ioctl to set read buffer size.
Can be used to speed up importing a file from a vfs based filesystem. Signed-off-by: Andrew Leech <andrew.leech@planetinnovation.com.au>pull/12836/head
rodzic
dff293840e
commit
4cf741062b
|
@ -26,6 +26,7 @@
|
||||||
|
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
|
||||||
#include "py/runtime.h"
|
#include "py/runtime.h"
|
||||||
#include "py/stream.h"
|
#include "py/stream.h"
|
||||||
|
@ -34,33 +35,39 @@
|
||||||
|
|
||||||
#if MICROPY_READER_VFS
|
#if MICROPY_READER_VFS
|
||||||
|
|
||||||
|
#ifndef MICROPY_READER_VFS_DEFAULT_BUFFER_SIZE
|
||||||
|
#define MICROPY_READER_VFS_DEFAULT_BUFFER_SIZE (2 * MICROPY_BYTES_PER_GC_BLOCK - offsetof(mp_reader_vfs_t, buf))
|
||||||
|
#endif
|
||||||
|
#define MICROPY_READER_VFS_MIN_BUFFER_SIZE (MICROPY_BYTES_PER_GC_BLOCK - offsetof(mp_reader_vfs_t, buf))
|
||||||
|
#define MICROPY_READER_VFS_MAX_BUFFER_SIZE (255)
|
||||||
|
|
||||||
typedef struct _mp_reader_vfs_t {
|
typedef struct _mp_reader_vfs_t {
|
||||||
mp_obj_t file;
|
mp_obj_t file;
|
||||||
uint16_t len;
|
uint8_t bufpos;
|
||||||
uint16_t pos;
|
uint8_t buflen;
|
||||||
byte buf[24];
|
uint8_t bufsize;
|
||||||
|
byte buf[];
|
||||||
} mp_reader_vfs_t;
|
} mp_reader_vfs_t;
|
||||||
|
|
||||||
STATIC mp_uint_t mp_reader_vfs_readbyte(void *data) {
|
STATIC mp_uint_t mp_reader_vfs_readbyte(void *data) {
|
||||||
mp_reader_vfs_t *reader = (mp_reader_vfs_t *)data;
|
mp_reader_vfs_t *reader = (mp_reader_vfs_t *)data;
|
||||||
if (reader->pos >= reader->len) {
|
if (reader->bufpos >= reader->buflen) {
|
||||||
if (reader->len < sizeof(reader->buf)) {
|
if (reader->buflen < reader->bufsize) {
|
||||||
return MP_READER_EOF;
|
return MP_READER_EOF;
|
||||||
} else {
|
} else {
|
||||||
int errcode;
|
int errcode;
|
||||||
reader->len = mp_stream_rw(reader->file, reader->buf, sizeof(reader->buf),
|
reader->buflen = mp_stream_rw(reader->file, reader->buf, reader->bufsize, &errcode, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE);
|
||||||
&errcode, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE);
|
|
||||||
if (errcode != 0) {
|
if (errcode != 0) {
|
||||||
// TODO handle errors properly
|
// TODO handle errors properly
|
||||||
return MP_READER_EOF;
|
return MP_READER_EOF;
|
||||||
}
|
}
|
||||||
if (reader->len == 0) {
|
if (reader->buflen == 0) {
|
||||||
return MP_READER_EOF;
|
return MP_READER_EOF;
|
||||||
}
|
}
|
||||||
reader->pos = 0;
|
reader->bufpos = 0;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return reader->buf[reader->pos++];
|
return reader->buf[reader->bufpos++];
|
||||||
}
|
}
|
||||||
|
|
||||||
STATIC void mp_reader_vfs_close(void *data) {
|
STATIC void mp_reader_vfs_close(void *data) {
|
||||||
|
@ -70,18 +77,31 @@ STATIC void mp_reader_vfs_close(void *data) {
|
||||||
}
|
}
|
||||||
|
|
||||||
void mp_reader_new_file(mp_reader_t *reader, qstr filename) {
|
void mp_reader_new_file(mp_reader_t *reader, qstr filename) {
|
||||||
mp_reader_vfs_t *rf = m_new_obj(mp_reader_vfs_t);
|
|
||||||
mp_obj_t args[2] = {
|
mp_obj_t args[2] = {
|
||||||
MP_OBJ_NEW_QSTR(filename),
|
MP_OBJ_NEW_QSTR(filename),
|
||||||
MP_OBJ_NEW_QSTR(MP_QSTR_rb),
|
MP_OBJ_NEW_QSTR(MP_QSTR_rb),
|
||||||
};
|
};
|
||||||
rf->file = mp_vfs_open(MP_ARRAY_SIZE(args), &args[0], (mp_map_t *)&mp_const_empty_map);
|
mp_obj_t file = mp_vfs_open(MP_ARRAY_SIZE(args), &args[0], (mp_map_t *)&mp_const_empty_map);
|
||||||
int errcode;
|
|
||||||
rf->len = mp_stream_rw(rf->file, rf->buf, sizeof(rf->buf), &errcode, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE);
|
const mp_stream_p_t *stream_p = mp_get_stream(file);
|
||||||
|
int errcode = 0;
|
||||||
|
mp_uint_t bufsize = stream_p->ioctl(file, MP_STREAM_GET_BUFFER_SIZE, 0, &errcode);
|
||||||
|
if (bufsize == MP_STREAM_ERROR || bufsize == 0) {
|
||||||
|
// bufsize == 0 is included here to support mpremote v1.21 and older where mount file ioctl
|
||||||
|
// returned 0 by default.
|
||||||
|
bufsize = MICROPY_READER_VFS_DEFAULT_BUFFER_SIZE;
|
||||||
|
} else {
|
||||||
|
bufsize = MIN(MICROPY_READER_VFS_MAX_BUFFER_SIZE, MAX(MICROPY_READER_VFS_MIN_BUFFER_SIZE, bufsize));
|
||||||
|
}
|
||||||
|
|
||||||
|
mp_reader_vfs_t *rf = m_new_obj_var(mp_reader_vfs_t, buf, byte, bufsize);
|
||||||
|
rf->file = file;
|
||||||
|
rf->bufsize = bufsize;
|
||||||
|
rf->buflen = mp_stream_rw(rf->file, rf->buf, rf->bufsize, &errcode, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE);
|
||||||
if (errcode != 0) {
|
if (errcode != 0) {
|
||||||
mp_raise_OSError(errcode);
|
mp_raise_OSError(errcode);
|
||||||
}
|
}
|
||||||
rf->pos = 0;
|
rf->bufpos = 0;
|
||||||
reader->data = rf;
|
reader->data = rf;
|
||||||
reader->readbyte = mp_reader_vfs_readbyte;
|
reader->readbyte = mp_reader_vfs_readbyte;
|
||||||
reader->close = mp_reader_vfs_close;
|
reader->close = mp_reader_vfs_close;
|
||||||
|
|
|
@ -43,6 +43,7 @@
|
||||||
#define MP_STREAM_GET_DATA_OPTS (8) // Get data/message options
|
#define MP_STREAM_GET_DATA_OPTS (8) // Get data/message options
|
||||||
#define MP_STREAM_SET_DATA_OPTS (9) // Set data/message options
|
#define MP_STREAM_SET_DATA_OPTS (9) // Set data/message options
|
||||||
#define MP_STREAM_GET_FILENO (10) // Get fileno of underlying file
|
#define MP_STREAM_GET_FILENO (10) // Get fileno of underlying file
|
||||||
|
#define MP_STREAM_GET_BUFFER_SIZE (11) // Get preferred buffer size for file
|
||||||
|
|
||||||
// These poll ioctl values are compatible with Linux
|
// These poll ioctl values are compatible with Linux
|
||||||
#define MP_STREAM_POLL_RD (0x0001)
|
#define MP_STREAM_POLL_RD (0x0001)
|
||||||
|
|
|
@ -16,6 +16,8 @@ except (ImportError, AttributeError):
|
||||||
|
|
||||||
|
|
||||||
class UserFile(io.IOBase):
|
class UserFile(io.IOBase):
|
||||||
|
buffer_size = 16
|
||||||
|
|
||||||
def __init__(self, mode, data):
|
def __init__(self, mode, data):
|
||||||
assert isinstance(data, bytes)
|
assert isinstance(data, bytes)
|
||||||
self.is_text = mode.find("b") == -1
|
self.is_text = mode.find("b") == -1
|
||||||
|
@ -39,7 +41,11 @@ class UserFile(io.IOBase):
|
||||||
|
|
||||||
def ioctl(self, req, arg):
|
def ioctl(self, req, arg):
|
||||||
print("ioctl", req, arg)
|
print("ioctl", req, arg)
|
||||||
|
if req == 4: # MP_STREAM_CLOSE
|
||||||
return 0
|
return 0
|
||||||
|
if req == 11: # MP_STREAM_GET_BUFFER_SIZE
|
||||||
|
return UserFile.buffer_size
|
||||||
|
return -1
|
||||||
|
|
||||||
|
|
||||||
class UserFS:
|
class UserFS:
|
||||||
|
@ -70,6 +76,8 @@ user_files = {
|
||||||
"/usermod2.py": b"print('in usermod2')",
|
"/usermod2.py": b"print('in usermod2')",
|
||||||
"/usermod3.py": b"syntax error",
|
"/usermod3.py": b"syntax error",
|
||||||
"/usermod4.mpy": b"syntax error",
|
"/usermod4.mpy": b"syntax error",
|
||||||
|
"/usermod5.py": b"print('in usermod5')",
|
||||||
|
"/usermod6.py": b"print('in usermod6')",
|
||||||
}
|
}
|
||||||
os.mount(UserFS(user_files), "/userfs")
|
os.mount(UserFS(user_files), "/userfs")
|
||||||
|
|
||||||
|
@ -93,6 +101,14 @@ try:
|
||||||
except ValueError:
|
except ValueError:
|
||||||
print("ValueError in usermod4")
|
print("ValueError in usermod4")
|
||||||
|
|
||||||
|
# Test an import with largest buffer size
|
||||||
|
UserFile.buffer_size = 255
|
||||||
|
import usermod5
|
||||||
|
|
||||||
|
# Test an import with over-size buffer size (should be safely limited internally)
|
||||||
|
UserFile.buffer_size = 1024
|
||||||
|
import usermod6
|
||||||
|
|
||||||
# unmount and undo path addition
|
# unmount and undo path addition
|
||||||
os.umount("/userfs")
|
os.umount("/userfs")
|
||||||
sys.path.pop()
|
sys.path.pop()
|
||||||
|
|
|
@ -3,21 +3,37 @@ some data in a text file
|
||||||
stat /usermod1
|
stat /usermod1
|
||||||
stat /usermod1.py
|
stat /usermod1.py
|
||||||
open /usermod1.py rb
|
open /usermod1.py rb
|
||||||
|
ioctl 11 0
|
||||||
ioctl 4 0
|
ioctl 4 0
|
||||||
in usermod1
|
in usermod1
|
||||||
stat /usermod2
|
stat /usermod2
|
||||||
stat /usermod2.py
|
stat /usermod2.py
|
||||||
open /usermod2.py rb
|
open /usermod2.py rb
|
||||||
|
ioctl 11 0
|
||||||
ioctl 4 0
|
ioctl 4 0
|
||||||
in usermod2
|
in usermod2
|
||||||
stat /usermod3
|
stat /usermod3
|
||||||
stat /usermod3.py
|
stat /usermod3.py
|
||||||
open /usermod3.py rb
|
open /usermod3.py rb
|
||||||
|
ioctl 11 0
|
||||||
ioctl 4 0
|
ioctl 4 0
|
||||||
SyntaxError in usermod3
|
SyntaxError in usermod3
|
||||||
stat /usermod4
|
stat /usermod4
|
||||||
stat /usermod4.py
|
stat /usermod4.py
|
||||||
stat /usermod4.mpy
|
stat /usermod4.mpy
|
||||||
open /usermod4.mpy rb
|
open /usermod4.mpy rb
|
||||||
|
ioctl 11 0
|
||||||
ioctl 4 0
|
ioctl 4 0
|
||||||
ValueError in usermod4
|
ValueError in usermod4
|
||||||
|
stat /usermod5
|
||||||
|
stat /usermod5.py
|
||||||
|
open /usermod5.py rb
|
||||||
|
ioctl 11 0
|
||||||
|
ioctl 4 0
|
||||||
|
in usermod5
|
||||||
|
stat /usermod6
|
||||||
|
stat /usermod6.py
|
||||||
|
open /usermod6.py rb
|
||||||
|
ioctl 11 0
|
||||||
|
ioctl 4 0
|
||||||
|
in usermod6
|
||||||
|
|
Ładowanie…
Reference in New Issue