From 78c5d2e8dd7df548707cffd86852ff9b654348b8 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 22 Feb 2024 11:22:13 +1100 Subject: [PATCH 01/15] py/reader: Change close method to ioctl. To allow more flexibility in the future. Signed-off-by: Damien George --- extmod/vfs_reader.c | 16 +++++++++---- ports/esp8266/lexerstr32.c | 11 ++++++--- ports/nrf/modules/os/microbitfs.c | 23 +++++++++++------- py/lexer.c | 2 +- py/persistentcode.c | 7 +++++- py/reader.c | 40 ++++++++++++++++++++----------- py/reader.h | 20 +++++++++++----- shared/runtime/pyexec.c | 29 +++++++++++++--------- 8 files changed, 99 insertions(+), 49 deletions(-) diff --git a/extmod/vfs_reader.c b/extmod/vfs_reader.c index 80d0fa6344..cecef5dc96 100644 --- a/extmod/vfs_reader.c +++ b/extmod/vfs_reader.c @@ -49,7 +49,7 @@ typedef struct _mp_reader_vfs_t { byte buf[]; } mp_reader_vfs_t; -static mp_uint_t mp_reader_vfs_readbyte(void *data) { +static uintptr_t mp_reader_vfs_readbyte(void *data) { mp_reader_vfs_t *reader = (mp_reader_vfs_t *)data; if (reader->bufpos >= reader->buflen) { if (reader->buflen < reader->bufsize) { @@ -70,10 +70,16 @@ static mp_uint_t mp_reader_vfs_readbyte(void *data) { return reader->buf[reader->bufpos++]; } -static void mp_reader_vfs_close(void *data) { +static intptr_t mp_reader_vfs_ioctl(void *data, uintptr_t request, uintptr_t arg) { mp_reader_vfs_t *reader = (mp_reader_vfs_t *)data; - mp_stream_close(reader->file); - m_del_obj(mp_reader_vfs_t, reader); + + if (request == MP_READER_CLOSE) { + mp_stream_close(reader->file); + m_del_obj(mp_reader_vfs_t, reader); + return 0; + } + + return -MP_EINVAL; } void mp_reader_new_file(mp_reader_t *reader, qstr filename) { @@ -104,7 +110,7 @@ void mp_reader_new_file(mp_reader_t *reader, qstr filename) { rf->bufpos = 0; reader->data = rf; reader->readbyte = mp_reader_vfs_readbyte; - reader->close = mp_reader_vfs_close; + reader->ioctl = mp_reader_vfs_ioctl; } #endif // MICROPY_READER_VFS diff --git a/ports/esp8266/lexerstr32.c b/ports/esp8266/lexerstr32.c index 51d5658e43..d51c15e7c8 100644 --- a/ports/esp8266/lexerstr32.c +++ b/ports/esp8266/lexerstr32.c @@ -26,6 +26,7 @@ */ #include "py/lexer.h" +#include "py/mperrno.h" #if MICROPY_ENABLE_COMPILER @@ -35,7 +36,7 @@ typedef struct _mp_lexer_str32_buf_t { uint8_t byte_off; } mp_lexer_str32_buf_t; -static mp_uint_t str32_buf_next_byte(void *sb_in) { +static uintptr_t str32_buf_next_byte(void *sb_in) { mp_lexer_str32_buf_t *sb = (mp_lexer_str32_buf_t *)sb_in; byte c = sb->val & 0xff; if (c == 0) { @@ -52,9 +53,13 @@ static mp_uint_t str32_buf_next_byte(void *sb_in) { return c; } -static void str32_buf_free(void *sb_in) { +static intptr_t str32_buf_free(void *sb_in, uintptr_t request, uintptr_t arg) { mp_lexer_str32_buf_t *sb = (mp_lexer_str32_buf_t *)sb_in; - m_del_obj(mp_lexer_str32_buf_t, sb); + if (request == MP_READER_CLOSE) { + m_del_obj(mp_lexer_str32_buf_t, sb); + return 0; + } + return -MP_EINVAL; } mp_lexer_t *mp_lexer_new_from_str32(qstr src_name, const char *str, mp_uint_t len, mp_uint_t free_len) { diff --git a/ports/nrf/modules/os/microbitfs.c b/ports/nrf/modules/os/microbitfs.c index f549e6f32e..d8ea5115c9 100644 --- a/ports/nrf/modules/os/microbitfs.c +++ b/ports/nrf/modules/os/microbitfs.c @@ -458,11 +458,18 @@ static mp_uint_t microbit_file_write(mp_obj_t obj, const void *buf, mp_uint_t si return size; } -static void microbit_file_close(file_descriptor_obj *fd) { - if (fd->writable) { - flash_write_byte((uint32_t)&(file_system_chunks[fd->start_chunk].header.end_offset), fd->seek_offset); +static intptr_t microbit_file_ioctl(void *fd_in, uintptr_t request, uintptr_t arg) { + file_descriptor_obj *fd = fd_in; + + if (request == MP_READER_CLOSE) { + if (fd->writable) { + flash_write_byte((uint32_t)&(file_system_chunks[fd->start_chunk].header.end_offset), fd->seek_offset); + } + fd->open = false; + return 0; } - fd->open = false; + + return -MP_EINVAL; } static mp_obj_t microbit_file_list(void) { @@ -495,7 +502,7 @@ static mp_obj_t microbit_file_size(mp_obj_t filename) { return mp_obj_new_int(len); } -static mp_uint_t file_read_byte(file_descriptor_obj *fd) { +static uintptr_t file_read_byte(file_descriptor_obj *fd) { if (file_system_chunks[fd->seek_chunk].next_chunk == UNUSED_CHUNK) { uint8_t end_offset = file_system_chunks[fd->start_chunk].header.end_offset; if (end_offset == UNUSED_CHUNK || fd->seek_offset == end_offset) { @@ -516,8 +523,8 @@ mp_lexer_t *os_mbfs_new_reader(const char *filename) { } mp_reader_t reader; reader.data = fd; - reader.readbyte = (mp_uint_t(*)(void*))file_read_byte; - reader.close = (void(*)(void*))microbit_file_close; // no-op + reader.readbyte = (uintptr_t(*)(void*))file_read_byte; + reader.close = microbit_file_ioctl; return mp_lexer_new(qstr_from_str(filename), reader); } @@ -538,7 +545,7 @@ static MP_DEFINE_CONST_FUN_OBJ_1(os_mbfs_file_name_obj, os_mbfs_file_name); static mp_obj_t os_mbfs_file_close(mp_obj_t self) { file_descriptor_obj *fd = (file_descriptor_obj*)self; - microbit_file_close(fd); + microbit_file_ioctl(fd, MP_READER_CLOSE, 0); return mp_const_none; } static MP_DEFINE_CONST_FUN_OBJ_1(os_mbfs_file_close_obj, os_mbfs_file_close); diff --git a/py/lexer.c b/py/lexer.c index bff8e63765..1f4a614c5a 100644 --- a/py/lexer.c +++ b/py/lexer.c @@ -908,7 +908,7 @@ mp_lexer_t *mp_lexer_new_from_fd(qstr filename, int fd, bool close_fd) { void mp_lexer_free(mp_lexer_t *lex) { if (lex) { - lex->reader.close(lex->reader.data); + lex->reader.ioctl(lex->reader.data, MP_READER_CLOSE, 0); vstr_clear(&lex->vstr); #if MICROPY_PY_FSTRINGS vstr_clear(&lex->fstring_args); diff --git a/py/persistentcode.c b/py/persistentcode.c index 5088c05f03..208231530c 100644 --- a/py/persistentcode.c +++ b/py/persistentcode.c @@ -393,9 +393,14 @@ static mp_raw_code_t *load_raw_code(mp_reader_t *reader, mp_module_context_t *co return rc; } +static void reader_close(void *reader_in) { + mp_reader_t *reader = reader_in; + reader->ioctl(reader->data, MP_READER_CLOSE, 0); +} + void mp_raw_code_load(mp_reader_t *reader, mp_compiled_module_t *cm) { // Set exception handler to close the reader if an exception is raised. - MP_DEFINE_NLR_JUMP_CALLBACK_FUNCTION_1(ctx, reader->close, reader->data); + MP_DEFINE_NLR_JUMP_CALLBACK_FUNCTION_1(ctx, reader_close, reader); nlr_push_jump_callback(&ctx.callback, mp_call_function_1_from_nlr_jump_callback); byte header[4]; diff --git a/py/reader.c b/py/reader.c index 151e04cac2..8232de55ca 100644 --- a/py/reader.c +++ b/py/reader.c @@ -39,7 +39,7 @@ typedef struct _mp_reader_mem_t { const byte *end; } mp_reader_mem_t; -static mp_uint_t mp_reader_mem_readbyte(void *data) { +static uintptr_t mp_reader_mem_readbyte(void *data) { mp_reader_mem_t *reader = (mp_reader_mem_t *)data; if (reader->cur < reader->end) { return *reader->cur++; @@ -48,12 +48,18 @@ static mp_uint_t mp_reader_mem_readbyte(void *data) { } } -static void mp_reader_mem_close(void *data) { +static intptr_t mp_reader_mem_ioctl(void *data, uintptr_t request, uintptr_t arg) { mp_reader_mem_t *reader = (mp_reader_mem_t *)data; - if (reader->free_len > 0) { - m_del(char, (char *)reader->beg, reader->free_len); + + if (request == MP_READER_CLOSE) { + if (reader->free_len > 0) { + m_del(char, (char *)reader->beg, reader->free_len); + } + m_del_obj(mp_reader_mem_t, reader); + return 0; } - m_del_obj(mp_reader_mem_t, reader); + + return -MP_EINVAL; } void mp_reader_new_mem(mp_reader_t *reader, const byte *buf, size_t len, size_t free_len) { @@ -64,7 +70,7 @@ void mp_reader_new_mem(mp_reader_t *reader, const byte *buf, size_t len, size_t rm->end = buf + len; reader->data = rm; reader->readbyte = mp_reader_mem_readbyte; - reader->close = mp_reader_mem_close; + reader->ioctl = mp_reader_mem_ioctl; } #if MICROPY_READER_POSIX @@ -81,7 +87,7 @@ typedef struct _mp_reader_posix_t { byte buf[20]; } mp_reader_posix_t; -static mp_uint_t mp_reader_posix_readbyte(void *data) { +static uintptr_t mp_reader_posix_readbyte(void *data) { mp_reader_posix_t *reader = (mp_reader_posix_t *)data; if (reader->pos >= reader->len) { if (reader->len == 0) { @@ -101,14 +107,20 @@ static mp_uint_t mp_reader_posix_readbyte(void *data) { return reader->buf[reader->pos++]; } -static void mp_reader_posix_close(void *data) { +static intptr_t mp_reader_posix_ioctl(void *data, uintptr_t request, uintptr_t arg) { mp_reader_posix_t *reader = (mp_reader_posix_t *)data; - if (reader->close_fd) { - MP_THREAD_GIL_EXIT(); - close(reader->fd); - MP_THREAD_GIL_ENTER(); + + if (request == MP_READER_CLOSE) { + if (reader->close_fd) { + MP_THREAD_GIL_EXIT(); + close(reader->fd); + MP_THREAD_GIL_ENTER(); + } + m_del_obj(mp_reader_posix_t, reader); + return 0; } - m_del_obj(mp_reader_posix_t, reader); + + return -MP_EINVAL; } void mp_reader_new_file_from_fd(mp_reader_t *reader, int fd, bool close_fd) { @@ -129,7 +141,7 @@ void mp_reader_new_file_from_fd(mp_reader_t *reader, int fd, bool close_fd) { rp->pos = 0; reader->data = rp; reader->readbyte = mp_reader_posix_readbyte; - reader->close = mp_reader_posix_close; + reader->ioctl = mp_reader_posix_ioctl; } #if !MICROPY_VFS_POSIX diff --git a/py/reader.h b/py/reader.h index 5cb1e67966..7d763e4ff6 100644 --- a/py/reader.h +++ b/py/reader.h @@ -28,15 +28,23 @@ #include "py/obj.h" -// the readbyte function must return the next byte in the input stream -// it must return MP_READER_EOF if end of stream -// it can be called again after returning MP_READER_EOF, and in that case must return MP_READER_EOF -#define MP_READER_EOF ((mp_uint_t)(-1)) +#define MP_READER_EOF ((uintptr_t)(-1)) + +// Reader ioctl request codes. +#define MP_READER_CLOSE (1) typedef struct _mp_reader_t { + // Pointer to the context of this reader, passed as the first argument to the methods below. void *data; - mp_uint_t (*readbyte)(void *data); - void (*close)(void *data); + + // The readbyte function must return the next byte in the input stream. + // It must return MP_READER_EOF if end of stream. It can be called again after returning + // MP_READER_EOF, and in that case must return MP_READER_EOF. + uintptr_t (*readbyte)(void *data); + + // Ioctl method for performing various control actions. + // On error it must return a negative errno value. + intptr_t (*ioctl)(void *data, uintptr_t request, uintptr_t arg); } mp_reader_t; void mp_reader_new_mem(mp_reader_t *reader, const byte *buf, size_t len, size_t free_len); diff --git a/shared/runtime/pyexec.c b/shared/runtime/pyexec.c index a305e6a5df..144184705c 100644 --- a/shared/runtime/pyexec.c +++ b/shared/runtime/pyexec.c @@ -34,6 +34,7 @@ #include "py/repl.h" #include "py/gc.h" #include "py/frozenmod.h" +#include "py/mperrno.h" #include "py/mphal.h" #if MICROPY_HW_ENABLE_USB #include "irq.h" @@ -135,7 +136,7 @@ static int parse_compile_execute(const void *source, mp_parse_input_kind_t input if (exec_flags & EXEC_FLAG_SOURCE_IS_READER) { const mp_reader_t *reader = source; - reader->close(reader->data); + reader->ioctl(reader->data, MP_READER_CLOSE, 0); } // print EOF after normal output @@ -201,7 +202,7 @@ typedef struct _mp_reader_stdin_t { uint16_t window_remain; } mp_reader_stdin_t; -static mp_uint_t mp_reader_stdin_readbyte(void *data) { +static uintptr_t mp_reader_stdin_readbyte(void *data) { mp_reader_stdin_t *reader = (mp_reader_stdin_t *)data; if (reader->eof) { @@ -233,18 +234,24 @@ static mp_uint_t mp_reader_stdin_readbyte(void *data) { return c; } -static void mp_reader_stdin_close(void *data) { +static intptr_t mp_reader_stdin_ioctl(void *data, uintptr_t request, uintptr_t arg) { mp_reader_stdin_t *reader = (mp_reader_stdin_t *)data; - if (!reader->eof) { - reader->eof = true; - mp_hal_stdout_tx_strn("\x04", 1); // indicate end to host - for (;;) { - int c = mp_hal_stdin_rx_chr(); - if (c == CHAR_CTRL_C || c == CHAR_CTRL_D) { - break; + + if (request == MP_READER_CLOSE) { + if (!reader->eof) { + reader->eof = true; + mp_hal_stdout_tx_strn("\x04", 1); // indicate end to host + for (;;) { + int c = mp_hal_stdin_rx_chr(); + if (c == CHAR_CTRL_C || c == CHAR_CTRL_D) { + break; + } } } + return 0; } + + return -MP_EINVAL; } static void mp_reader_new_stdin(mp_reader_t *reader, mp_reader_stdin_t *reader_stdin, uint16_t buf_max) { @@ -260,7 +267,7 @@ static void mp_reader_new_stdin(mp_reader_t *reader, mp_reader_stdin_t *reader_s reader_stdin->window_remain = window; reader->data = reader_stdin; reader->readbyte = mp_reader_stdin_readbyte; - reader->close = mp_reader_stdin_close; + reader->ioctl = mp_reader_stdin_ioctl; } static int do_reader_stdin(int c) { From 7f67ee7ccc06e1bb618cac2308cb01d1221347d9 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 22 Feb 2024 11:36:47 +1100 Subject: [PATCH 02/15] py/reader: Add MP_READER_MEMMAP ioctl code. Signed-off-by: Damien George --- py/reader.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/py/reader.h b/py/reader.h index 7d763e4ff6..26fb92d243 100644 --- a/py/reader.h +++ b/py/reader.h @@ -32,6 +32,13 @@ // Reader ioctl request codes. #define MP_READER_CLOSE (1) +#define MP_READER_MEMMAP (2) + +// Used as arg for MP_READER_MEMMAP ioctl request. +typedef struct _mp_reader_ioctl_memmap_t { + size_t len; + const uint8_t *ptr; +} mp_reader_ioctl_memmap_t; typedef struct _mp_reader_t { // Pointer to the context of this reader, passed as the first argument to the methods below. From 22f3f683a6df293b5e40e0dabd341eb9eb8ac291 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 22 Feb 2024 18:41:52 +1100 Subject: [PATCH 03/15] py/reader: Make mem reader support MP_READER_MEMMAP. Signed-off-by: Damien George --- py/reader.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/py/reader.c b/py/reader.c index 8232de55ca..6cffe1dffc 100644 --- a/py/reader.c +++ b/py/reader.c @@ -52,11 +52,16 @@ static intptr_t mp_reader_mem_ioctl(void *data, uintptr_t request, uintptr_t arg mp_reader_mem_t *reader = (mp_reader_mem_t *)data; if (request == MP_READER_CLOSE) { - if (reader->free_len > 0) { + if (reader->free_len > 0 && reader->free_len != (size_t)-1) { m_del(char, (char *)reader->beg, reader->free_len); } m_del_obj(mp_reader_mem_t, reader); return 0; + } else if (request == MP_READER_MEMMAP && reader->free_len == (size_t)-1) { + mp_reader_ioctl_memmap_t *memmap = (mp_reader_ioctl_memmap_t *)arg; + memmap->ptr = reader->cur; + reader->cur += memmap->len; + return 0; } return -MP_EINVAL; From b7030fa1b889047c2a55d47352cb886a2d8fa2cc Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 22 Feb 2024 16:56:37 +1100 Subject: [PATCH 04/15] py/stream: Add MP_STREAM_INIT_READER ioctl value. Signed-off-by: Damien George --- py/stream.h | 1 + 1 file changed, 1 insertion(+) diff --git a/py/stream.h b/py/stream.h index e6e6f283df..ba41681dda 100644 --- a/py/stream.h +++ b/py/stream.h @@ -44,6 +44,7 @@ #define MP_STREAM_SET_DATA_OPTS (9) // Set data/message options #define MP_STREAM_GET_FILENO (10) // Get fileno of underlying file #define MP_STREAM_GET_BUFFER_SIZE (11) // Get preferred buffer size for file +#define MP_STREAM_INIT_READER (12) // Initialise a mp_reader_t for this stream // These poll ioctl values are compatible with Linux #define MP_STREAM_POLL_RD (0x0001) From fbcb45ba0baba9c3f1af43cd22d78fa7e4fc6eae Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 4 Mar 2022 10:33:01 +1100 Subject: [PATCH 05/15] extmod/vfs: Support querying the VFS mount table via uos.mount(path). Signed-off-by: Damien George --- extmod/vfs.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/extmod/vfs.c b/extmod/vfs.c index ae09b8afe8..afc1a71f0e 100644 --- a/extmod/vfs.c +++ b/extmod/vfs.c @@ -208,6 +208,17 @@ mp_obj_t mp_vfs_mount(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args { MP_QSTR_mkfs, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_rom_obj = MP_ROM_FALSE} }, }; + if (n_args == 1) { + // query mount table + const char *path = mp_obj_str_get_str(pos_args[0]); + for (mp_vfs_mount_t *vfs = MP_STATE_VM(vfs_mount_table); vfs != NULL; vfs = vfs->next) { + if (strncmp(path, vfs->str, vfs->len) == 0 && path[vfs->len] == '\0') { + return vfs->obj; + } + } + return mp_const_none; + } + // parse args mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)]; mp_arg_parse_all(n_args - 2, pos_args + 2, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args); @@ -262,7 +273,7 @@ mp_obj_t mp_vfs_mount(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args return mp_const_none; } -MP_DEFINE_CONST_FUN_OBJ_KW(mp_vfs_mount_obj, 2, mp_vfs_mount); +MP_DEFINE_CONST_FUN_OBJ_KW(mp_vfs_mount_obj, 1, mp_vfs_mount); mp_obj_t mp_vfs_umount(mp_obj_t mnt_in) { // remove vfs from the mount table From 3e78b1fdd92ee891ba4f6f249ce399f37eca0b87 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 4 Mar 2022 10:40:15 +1100 Subject: [PATCH 06/15] py/qstr: Add qstr_from_strn_static. Signed-off-by: Damien George --- py/qstr.c | 20 ++++++++++++++++++-- py/qstr.h | 1 + 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/py/qstr.c b/py/qstr.c index 3720932250..f5c1b113df 100644 --- a/py/qstr.c +++ b/py/qstr.c @@ -309,7 +309,7 @@ qstr qstr_from_str(const char *str) { return qstr_from_strn(str, strlen(str)); } -qstr qstr_from_strn(const char *str, size_t len) { +static qstr qstr_from_strn_helper(const char *str, size_t len, bool is_static) { QSTR_ENTER(); qstr q = qstr_find_strn(str, len); if (q == 0) { @@ -321,6 +321,11 @@ qstr qstr_from_strn(const char *str, size_t len) { mp_raise_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("name too long")); } + if (is_static) { + assert(str[len] == '\0'); + goto add; + } + // compute number of bytes needed to intern this string size_t n_bytes = len + 1; @@ -364,12 +369,23 @@ qstr qstr_from_strn(const char *str, size_t len) { // store the interned strings' data memcpy(q_ptr, str, len); q_ptr[len] = '\0'; - q = qstr_add(len, q_ptr); + str = q_ptr; + + add: + q = qstr_add(len, str); } QSTR_EXIT(); return q; } +qstr qstr_from_strn(const char *str, size_t len) { + return qstr_from_strn_helper(str, len, false); +} + +qstr qstr_from_strn_static(const char *str, size_t len) { + return qstr_from_strn_helper(str, len, true); +} + mp_uint_t qstr_hash(qstr q) { const qstr_pool_t *pool = find_qstr(&q); #if MICROPY_QSTR_BYTES_IN_HASH diff --git a/py/qstr.h b/py/qstr.h index 58ce285fd2..69d52e85fe 100644 --- a/py/qstr.h +++ b/py/qstr.h @@ -101,6 +101,7 @@ qstr qstr_find_strn(const char *str, size_t str_len); // returns MP_QSTRnull if qstr qstr_from_str(const char *str); qstr qstr_from_strn(const char *str, size_t len); +qstr qstr_from_strn_static(const char *str, size_t len); mp_uint_t qstr_hash(qstr q); const char *qstr_str(qstr q); From d15c1e0e17a54126f33790602101fa9ff0885a33 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 4 Mar 2022 10:52:35 +1100 Subject: [PATCH 07/15] extmod/vfs_map: Add VfsMap filesystem object. Signed-off-by: Damien George --- extmod/extmod.cmake | 2 + extmod/extmod.mk | 2 + extmod/modvfs.c | 4 + extmod/vfs_map.c | 325 ++++++++++++++++++++++++++++++++++++++++++ extmod/vfs_map.h | 39 +++++ extmod/vfs_map_file.c | 202 ++++++++++++++++++++++++++ 6 files changed, 574 insertions(+) create mode 100644 extmod/vfs_map.c create mode 100644 extmod/vfs_map.h create mode 100644 extmod/vfs_map_file.c diff --git a/extmod/extmod.cmake b/extmod/extmod.cmake index 53c7740132..5cb63fb908 100644 --- a/extmod/extmod.cmake +++ b/extmod/extmod.cmake @@ -56,6 +56,8 @@ set(MICROPY_SOURCE_EXTMOD ${MICROPY_EXTMOD_DIR}/vfs_fat_diskio.c ${MICROPY_EXTMOD_DIR}/vfs_fat_file.c ${MICROPY_EXTMOD_DIR}/vfs_lfs.c + ${MICROPY_EXTMOD_DIR}/vfs_map.c + ${MICROPY_EXTMOD_DIR}/vfs_map_file.c ${MICROPY_EXTMOD_DIR}/vfs_posix.c ${MICROPY_EXTMOD_DIR}/vfs_posix_file.c ${MICROPY_EXTMOD_DIR}/vfs_reader.c diff --git a/extmod/extmod.mk b/extmod/extmod.mk index f0428bcd05..36ac82744a 100644 --- a/extmod/extmod.mk +++ b/extmod/extmod.mk @@ -55,6 +55,8 @@ SRC_EXTMOD_C += \ extmod/vfs_fat_diskio.c \ extmod/vfs_fat_file.c \ extmod/vfs_lfs.c \ + extmod/vfs_map.c \ + extmod/vfs_map_file.c \ extmod/vfs_posix.c \ extmod/vfs_posix_file.c \ extmod/vfs_reader.c \ diff --git a/extmod/modvfs.c b/extmod/modvfs.c index 32dc0e5d08..7f083e7589 100644 --- a/extmod/modvfs.c +++ b/extmod/modvfs.c @@ -31,6 +31,7 @@ #include "extmod/vfs.h" #include "extmod/vfs_fat.h" #include "extmod/vfs_lfs.h" +#include "extmod/vfs_map.h" #include "extmod/vfs_posix.h" #if !MICROPY_VFS @@ -51,6 +52,9 @@ static const mp_rom_map_elem_t vfs_module_globals_table[] = { #if MICROPY_VFS_LFS2 { MP_ROM_QSTR(MP_QSTR_VfsLfs2), MP_ROM_PTR(&mp_type_vfs_lfs2) }, #endif + #if MICROPY_VFS_MAP + { MP_ROM_QSTR(MP_QSTR_VfsMap), MP_ROM_PTR(&mp_type_vfs_map) }, + #endif #if MICROPY_VFS_POSIX { MP_ROM_QSTR(MP_QSTR_VfsPosix), MP_ROM_PTR(&mp_type_vfs_posix) }, #endif diff --git a/extmod/vfs_map.c b/extmod/vfs_map.c new file mode 100644 index 0000000000..b9e48f7541 --- /dev/null +++ b/extmod/vfs_map.c @@ -0,0 +1,325 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2022 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include + +#include "py/runtime.h" +#include "py/mperrno.h" +#include "extmod/vfs.h" +#include "extmod/vfs_map.h" + +#if MICROPY_VFS_MAP + +#define MAGIC_LEN (2) +#define MAGIC_BYTE0 ('M') +#define MAGIC_BYTE1 ('F') + +#define GET_LE16(p) ((p)[0] | (p)[1] << 8) +#define GET_LE32(p) ((p)[0] | (p)[1] << 8 | (p)[2] << 16 | (p)[3] << 24) + +struct _mp_obj_vfs_map_t { + mp_obj_base_t base; + mp_obj_t memory; + mp_obj_t device; + const uint8_t *filesystem; +}; + +mp_import_stat_t mp_vfs_map_search_filesystem(mp_obj_vfs_map_t *self, const char *path, size_t *size_out, const uint8_t **data_out) { + if (!(self->filesystem[0] == MAGIC_BYTE0 && self->filesystem[1] == MAGIC_BYTE1)) { + return MP_IMPORT_STAT_NO_EXIST; + } + if (path[0] == '/') { + ++path; + } + size_t path_len = strlen(path); + const uint8_t *fs = self->filesystem + MAGIC_LEN; + for (;;) { + uint16_t nlen = GET_LE16(fs); + fs += 2; + if (nlen == 0) { + return MP_IMPORT_STAT_NO_EXIST; + } + uint32_t flen; + mp_import_stat_t type; + if (nlen & 0x8000) { + // A directory. + nlen &= 0x7fff; + flen = 0; + type = MP_IMPORT_STAT_DIR; + } else { + // A file. + flen = GET_LE32(fs); + fs += 4; + type = MP_IMPORT_STAT_FILE; + } + if (path_len == nlen && memcmp(path, fs, path_len) == 0) { + if (size_out != NULL) { + *size_out = flen; + *data_out = fs + nlen; + } + return type; + } + fs += nlen + flen; + } +} + +static inline const char *vfs_map_get_path_str(mp_obj_vfs_map_t *self, mp_obj_t path) { + return mp_obj_str_get_str(path); +} + +static mp_obj_t vfs_map_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { + mp_arg_check_num(n_args, n_kw, 1, 2, false); + + mp_obj_vfs_map_t *self = m_new_obj(mp_obj_vfs_map_t); + self->base.type = type; + self->memory = args[0]; + self->device = n_args == 1 ? mp_const_none : args[1]; + + mp_buffer_info_t bufinfo; + if (mp_get_buffer(self->memory, &bufinfo, MP_BUFFER_READ)) { + self->filesystem = bufinfo.buf; + } else { + self->filesystem = (const uint8_t *)(uintptr_t)mp_obj_get_int(self->memory); + } + + return MP_OBJ_FROM_PTR(self); +} + +static mp_obj_t vfs_map_mount(mp_obj_t self_in, mp_obj_t readonly, mp_obj_t mkfs) { + (void)self_in; + (void)readonly; + if (mp_obj_is_true(mkfs)) { + mp_raise_OSError(MP_EPERM); + } + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_3(vfs_map_mount_obj, vfs_map_mount); + +static mp_obj_t vfs_map_umount(mp_obj_t self_in) { + (void)self_in; + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_1(vfs_map_umount_obj, vfs_map_umount); + +// mp_vfs_map_file_open is implemented in vfs_map_file.c. +static MP_DEFINE_CONST_FUN_OBJ_3(vfs_map_open_obj, mp_vfs_map_file_open); + +static mp_obj_t vfs_map_chdir(mp_obj_t self_in, mp_obj_t path_in) { + (void)self_in; + (void)path_in; + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_2(vfs_map_chdir_obj, vfs_map_chdir); + +typedef struct _vfs_map_ilistdir_it_t { + mp_obj_base_t base; + mp_fun_1_t iternext; + bool is_str; + const char *path; + const uint8_t *index; +} vfs_map_ilistdir_it_t; + +static mp_obj_t vfs_map_ilistdir_it_iternext(mp_obj_t self_in) { + vfs_map_ilistdir_it_t *self = MP_OBJ_TO_PTR(self_in); + + if (self->index == NULL) { + return MP_OBJ_STOP_ITERATION; + } + + const char *path = self->path; + if (path[0] == '/' || (path[0] == '.' && path[1] == '\0')) { + ++path; + } + size_t path_len = strlen(path); + + for (;;) { + uint16_t nlen = GET_LE16(self->index); + self->index += 2; + if (nlen == 0) { + self->index = NULL; + return MP_OBJ_STOP_ITERATION; + } + + uint32_t flen; + uint32_t type; + if (nlen & 0x8000) { + // dir + nlen &= 0x7fff; + flen = 0; + type = MP_S_IFDIR; + } else { + // file + flen = GET_LE32(self->index); + self->index += 4; + type = MP_S_IFREG; + } + + const uint8_t *nstr = self->index; + self->index += nlen + flen; + + if ((path_len == 0 && memchr(nstr, '/', nlen) == NULL) + || (path_len < nlen && nstr[path_len] == '/' && memcmp(path, nstr, path_len) == 0 && memchr(nstr + path_len + 1, '/', nlen - path_len - 1) == NULL)) { + // Make 4-tuple with info about this entry: (name, attr, inode, size) + mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(4, NULL)); + + if (path_len > 0) { + nstr += path_len + 1; + nlen -= path_len + 1; + } + + if (self->is_str) { + t->items[0] = mp_obj_new_str((const char *)nstr, nlen); + } else { + t->items[0] = mp_obj_new_bytes(nstr, nlen); + } + + t->items[1] = MP_OBJ_NEW_SMALL_INT(type); + t->items[2] = MP_OBJ_NEW_SMALL_INT(0); + t->items[3] = mp_obj_new_int(flen); + + return MP_OBJ_FROM_PTR(t); + } + } +} + +static mp_obj_t vfs_map_ilistdir(mp_obj_t self_in, mp_obj_t path_in) { + mp_obj_vfs_map_t *self = MP_OBJ_TO_PTR(self_in); + vfs_map_ilistdir_it_t *iter = m_new_obj(vfs_map_ilistdir_it_t); + iter->base.type = &mp_type_polymorph_iter; + iter->iternext = vfs_map_ilistdir_it_iternext; + iter->is_str = mp_obj_get_type(path_in) == &mp_type_str; + iter->path = mp_obj_str_get_str(path_in); + if (!(self->filesystem[0] == MAGIC_BYTE0 && self->filesystem[1] == MAGIC_BYTE1)) { + mp_raise_OSError(MP_ENOENT); + } + if (iter->path[0] == '\0' + || (iter->path[0] == '.' && iter->path[1] == '\0') + || (iter->path[0] == '/' && iter->path[1] == '\0')) { + // pass + } else if (mp_vfs_map_search_filesystem(self, iter->path, NULL, NULL) != MP_IMPORT_STAT_DIR) { + mp_raise_OSError(MP_ENOENT); + } + iter->index = self->filesystem + MAGIC_LEN; + return MP_OBJ_FROM_PTR(iter); +} +static MP_DEFINE_CONST_FUN_OBJ_2(vfs_map_ilistdir_obj, vfs_map_ilistdir); + +static mp_obj_t vfs_map_stat(mp_obj_t self_in, mp_obj_t path_in) { + mp_obj_vfs_map_t *self = MP_OBJ_TO_PTR(self_in); + const char *path = vfs_map_get_path_str(self, path_in); + size_t file_size; + const uint8_t *file_data; + mp_import_stat_t stat = mp_vfs_map_search_filesystem(self, path, &file_size, &file_data); + if (stat == MP_IMPORT_STAT_NO_EXIST) { + mp_raise_OSError(MP_ENOENT); + } + mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(10, NULL)); + t->items[0] = MP_OBJ_NEW_SMALL_INT(stat == MP_IMPORT_STAT_FILE ? MP_S_IFREG : MP_S_IFDIR); // st_mode + t->items[1] = MP_OBJ_NEW_SMALL_INT(0); // st_ino + t->items[2] = MP_OBJ_NEW_SMALL_INT(0); // st_dev + t->items[3] = MP_OBJ_NEW_SMALL_INT(0); // st_nlink + t->items[4] = MP_OBJ_NEW_SMALL_INT(0); // st_uid + t->items[5] = MP_OBJ_NEW_SMALL_INT(0); // st_gid + t->items[6] = MP_OBJ_NEW_SMALL_INT(file_size); // st_size + t->items[7] = MP_OBJ_NEW_SMALL_INT(0); // st_atime + t->items[8] = MP_OBJ_NEW_SMALL_INT(0); // st_mtime + t->items[9] = MP_OBJ_NEW_SMALL_INT(0); // st_ctime + return MP_OBJ_FROM_PTR(t); +} +static MP_DEFINE_CONST_FUN_OBJ_2(vfs_map_stat_obj, vfs_map_stat); + +static mp_obj_t vfs_map_statvfs(mp_obj_t self_in, mp_obj_t path_in) { + (void)self_in; + (void)path_in; + mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(10, NULL)); + t->items[0] = MP_OBJ_NEW_SMALL_INT(0); // f_bsize + t->items[1] = MP_OBJ_NEW_SMALL_INT(0); // f_frsize + t->items[2] = MP_OBJ_NEW_SMALL_INT(0); // f_blocks + t->items[3] = MP_OBJ_NEW_SMALL_INT(0); // f_bfree + t->items[4] = MP_OBJ_NEW_SMALL_INT(0); // f_bavail + t->items[5] = MP_OBJ_NEW_SMALL_INT(0); // f_files + t->items[6] = MP_OBJ_NEW_SMALL_INT(0); // f_ffree + t->items[7] = MP_OBJ_NEW_SMALL_INT(0); // f_favail + t->items[8] = MP_OBJ_NEW_SMALL_INT(0); // f_flags + t->items[9] = MP_OBJ_NEW_SMALL_INT(65535); // f_namemax + return MP_OBJ_FROM_PTR(t); +} +static MP_DEFINE_CONST_FUN_OBJ_2(vfs_map_statvfs_obj, vfs_map_statvfs); + +static mp_obj_t vfs_map_device(mp_obj_t self_in) { + mp_obj_vfs_map_t *self = MP_OBJ_TO_PTR(self_in); + return self->device; +} +static MP_DEFINE_CONST_FUN_OBJ_1(vfs_map_device_obj, vfs_map_device); + +static const mp_rom_map_elem_t vfs_map_locals_dict_table[] = { + { MP_ROM_QSTR(MP_QSTR_mount), MP_ROM_PTR(&vfs_map_mount_obj) }, + { MP_ROM_QSTR(MP_QSTR_umount), MP_ROM_PTR(&vfs_map_umount_obj) }, + { MP_ROM_QSTR(MP_QSTR_open), MP_ROM_PTR(&vfs_map_open_obj) }, + + { MP_ROM_QSTR(MP_QSTR_chdir), MP_ROM_PTR(&vfs_map_chdir_obj) }, + { MP_ROM_QSTR(MP_QSTR_ilistdir), MP_ROM_PTR(&vfs_map_ilistdir_obj) }, + { MP_ROM_QSTR(MP_QSTR_stat), MP_ROM_PTR(&vfs_map_stat_obj) }, + { MP_ROM_QSTR(MP_QSTR_statvfs), MP_ROM_PTR(&vfs_map_statvfs_obj) }, + + { MP_ROM_QSTR(MP_QSTR_device), MP_ROM_PTR(&vfs_map_device_obj) }, +}; +static MP_DEFINE_CONST_DICT(vfs_map_locals_dict, vfs_map_locals_dict_table); + +static mp_import_stat_t mp_vfs_map_import_stat(void *self_in, const char *path) { + mp_obj_vfs_map_t *self = self_in; + return mp_vfs_map_search_filesystem(self, path, NULL, NULL); +} + +static size_t mp_vfs_map_memmap(void *self_in, const char *path, const void **ptr_out) { + mp_obj_vfs_map_t *self = self_in; + size_t size; + const uint8_t *data; + mp_import_stat_t stat = mp_vfs_map_search_filesystem(self, path, &size, &data); + if (stat == MP_IMPORT_STAT_FILE) { + *ptr_out = data; + return size; + } else { + *ptr_out = NULL; + return 0; + } +} + +static const mp_vfs_proto_t vfs_map_proto = { + .import_stat = mp_vfs_map_import_stat, + .memmap = mp_vfs_map_memmap, +}; + +MP_DEFINE_CONST_OBJ_TYPE( + mp_type_vfs_map, + MP_QSTR_VfsMap, + MP_TYPE_FLAG_NONE, + make_new, vfs_map_make_new, + protocol, &vfs_map_proto, + locals_dict, &vfs_map_locals_dict + ); + +#endif // MICROPY_VFS_MAP diff --git a/extmod/vfs_map.h b/extmod/vfs_map.h new file mode 100644 index 0000000000..19f7956349 --- /dev/null +++ b/extmod/vfs_map.h @@ -0,0 +1,39 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2022 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#ifndef MICROPY_INCLUDED_EXTMOD_VFS_MAP_H +#define MICROPY_INCLUDED_EXTMOD_VFS_MAP_H + +#include "py/builtin.h" +#include "py/obj.h" + +typedef struct _mp_obj_vfs_map_t mp_obj_vfs_map_t; + +extern const mp_obj_type_t mp_type_vfs_map; + +mp_import_stat_t mp_vfs_map_search_filesystem(mp_obj_vfs_map_t *self, const char *path, size_t *size_out, const uint8_t **data_out); +mp_obj_t mp_vfs_map_file_open(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mode_in); + +#endif // MICROPY_INCLUDED_EXTMOD_VFS_MAP_H diff --git a/extmod/vfs_map_file.c b/extmod/vfs_map_file.c new file mode 100644 index 0000000000..896b642f62 --- /dev/null +++ b/extmod/vfs_map_file.c @@ -0,0 +1,202 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2022 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include + +#include "py/reader.h" +#include "py/runtime.h" +#include "py/stream.h" +#include "extmod/vfs_map.h" + +#if MICROPY_VFS_MAP + +typedef struct _mp_obj_vfs_map_file_t { + mp_obj_base_t base; + size_t file_size; + size_t file_offset; + const uint8_t *file_data; +} mp_obj_vfs_map_file_t; + +static const mp_obj_type_t mp_type_vfs_map_fileio; +static const mp_obj_type_t mp_type_vfs_map_textio; + +static uintptr_t mp_vfs_map_reader_readbyte(void *data) { + mp_obj_vfs_map_file_t *self = data; + if (self->file_offset >= self->file_size) { + return MP_READER_EOF; + } + return self->file_data[self->file_offset++]; +} + +static intptr_t mp_vfs_map_reader_ioctl(void *data, uintptr_t request, uintptr_t arg) { + mp_obj_vfs_map_file_t *self = data; + + if (request == MP_READER_CLOSE) { + return 0; + } else if (request == MP_READER_MEMMAP) { + mp_reader_ioctl_memmap_t *memmap = (mp_reader_ioctl_memmap_t *)arg; + memmap->ptr = self->file_data + self->file_offset; + self->file_offset += memmap->len; + return 0; + } + + return -MP_EINVAL; +} + +mp_obj_t mp_vfs_map_file_open(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mode_in) { + mp_obj_vfs_map_t *self = MP_OBJ_TO_PTR(self_in); + + const char *mode_s = mp_obj_str_get_str(mode_in); + const mp_obj_type_t *type = &mp_type_vfs_map_textio; + while (*mode_s) { + switch (*mode_s++) { + case 'r': + break; + case 'w': + case 'a': + case '+': + mp_raise_OSError(MP_EROFS); + case 'b': + type = &mp_type_vfs_map_fileio; + break; + case 't': + type = &mp_type_vfs_map_textio; + break; + } + } + + mp_obj_vfs_map_file_t *o = m_new_obj(mp_obj_vfs_map_file_t); + o->base.type = type; + o->file_offset = 0; + + const char *path = mp_obj_str_get_str(path_in); + mp_import_stat_t stat = mp_vfs_map_search_filesystem(self, path, &o->file_size, &o->file_data); + if (stat == MP_IMPORT_STAT_NO_EXIST) { + mp_raise_OSError(MP_ENOENT); + } + + return MP_OBJ_FROM_PTR(o); +} + +static mp_obj_t vfs_map_file___exit__(size_t n_args, const mp_obj_t *args) { + (void)n_args; + return mp_stream_close(args[0]); +} +static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(vfs_map_file___exit___obj, 4, 4, vfs_map_file___exit__); + +static mp_uint_t vfs_map_file_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *errcode) { + mp_obj_vfs_map_file_t *self = MP_OBJ_TO_PTR(o_in); + size_t remain = self->file_size - self->file_offset; + if (size > remain) { + size = remain; + } + memcpy(buf, self->file_data + self->file_offset, size); + self->file_offset += size; + return size; +} + +static mp_uint_t vfs_map_file_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, int *errcode) { + mp_obj_vfs_map_file_t *self = MP_OBJ_TO_PTR(o_in); + + switch (request) { + case MP_STREAM_FLUSH: + return 0; + case MP_STREAM_SEEK: { + struct mp_stream_seek_t *s = (struct mp_stream_seek_t *)arg; + if (s->whence == 0) { // SEEK_SET + self->file_offset = MIN(self->file_size, (size_t)s->offset); + } else if (s->whence == 1) { // SEEK_CUR + self->file_offset = MIN(self->file_size, self->file_offset + s->offset); + } else { // SEEK_END + self->file_offset = self->file_size; + } + s->offset = self->file_offset; + return 0; + } + case MP_STREAM_CLOSE: + return 0; + case MP_STREAM_POLL: { + mp_uint_t ret = 0; + if (arg & MP_STREAM_POLL_RD) { + ret |= MP_STREAM_POLL_RD; + } + return ret; + } + case MP_STREAM_INIT_READER: { + mp_reader_t *reader = (mp_reader_t *)arg; + reader->data = self; + reader->readbyte = mp_vfs_map_reader_readbyte; + reader->ioctl = mp_vfs_map_reader_ioctl; + return 0; + } + default: + *errcode = MP_EINVAL; + return MP_STREAM_ERROR; + } +} + +static const mp_rom_map_elem_t vfs_map_rawfile_locals_dict_table[] = { + { MP_ROM_QSTR(MP_QSTR_read), MP_ROM_PTR(&mp_stream_read_obj) }, + { MP_ROM_QSTR(MP_QSTR_readinto), MP_ROM_PTR(&mp_stream_readinto_obj) }, + { MP_ROM_QSTR(MP_QSTR_readline), MP_ROM_PTR(&mp_stream_unbuffered_readline_obj) }, + { MP_ROM_QSTR(MP_QSTR_readlines), MP_ROM_PTR(&mp_stream_unbuffered_readlines_obj) }, + { MP_ROM_QSTR(MP_QSTR_seek), MP_ROM_PTR(&mp_stream_seek_obj) }, + { MP_ROM_QSTR(MP_QSTR_tell), MP_ROM_PTR(&mp_stream_tell_obj) }, + { MP_ROM_QSTR(MP_QSTR_flush), MP_ROM_PTR(&mp_stream_flush_obj) }, + { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_stream_close_obj) }, + { MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&mp_identity_obj) }, + { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&vfs_map_file___exit___obj) }, +}; +static MP_DEFINE_CONST_DICT(vfs_map_rawfile_locals_dict, vfs_map_rawfile_locals_dict_table); + +static const mp_stream_p_t vfs_map_fileio_stream_p = { + .read = vfs_map_file_read, + .ioctl = vfs_map_file_ioctl, +}; + +static MP_DEFINE_CONST_OBJ_TYPE( + mp_type_vfs_map_fileio, + MP_QSTR_FileIO, + MP_TYPE_FLAG_ITER_IS_STREAM, + protocol, &vfs_map_fileio_stream_p, + locals_dict, &vfs_map_rawfile_locals_dict + ); + +static const mp_stream_p_t vfs_map_textio_stream_p = { + .read = vfs_map_file_read, + .ioctl = vfs_map_file_ioctl, + .is_text = true, +}; + +static MP_DEFINE_CONST_OBJ_TYPE( + mp_type_vfs_map_textio, + MP_QSTR_TextIOWrapper, + MP_TYPE_FLAG_ITER_IS_STREAM, + protocol, &vfs_map_textio_stream_p, + locals_dict, &vfs_map_rawfile_locals_dict + ); + +#endif // MICROPY_VFS_MAP From c34bfed356d4b2bd9e591d7bae20001e24f47dae Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 4 Mar 2022 10:54:05 +1100 Subject: [PATCH 08/15] py/persistentcode: Support loading from VfsMap. Signed-off-by: Damien George --- extmod/vfs.h | 1 + extmod/vfs_reader.c | 30 ++++++++++++++++++++ py/persistentcode.c | 68 +++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 93 insertions(+), 6 deletions(-) diff --git a/extmod/vfs.h b/extmod/vfs.h index f577d3e337..2b05a6bdfd 100644 --- a/extmod/vfs.h +++ b/extmod/vfs.h @@ -55,6 +55,7 @@ // At the moment the VFS protocol just has import_stat, but could be extended to other methods typedef struct _mp_vfs_proto_t { mp_import_stat_t (*import_stat)(void *self, const char *path); + size_t (*memmap)(void *self, const char *path, const void **ptr_out); } mp_vfs_proto_t; typedef struct _mp_vfs_blockdev_t { diff --git a/extmod/vfs_reader.c b/extmod/vfs_reader.c index cecef5dc96..4eb2f596cf 100644 --- a/extmod/vfs_reader.c +++ b/extmod/vfs_reader.c @@ -83,6 +83,26 @@ static intptr_t mp_reader_vfs_ioctl(void *data, uintptr_t request, uintptr_t arg } void mp_reader_new_file(mp_reader_t *reader, qstr filename) { + #if MICROPY_VFS_MAP + const char *path_out; + mp_vfs_mount_t *vfs = mp_vfs_lookup_path(qstr_str(filename), &path_out); + if (!(vfs == MP_VFS_NONE || vfs == MP_VFS_ROOT)) { + // If the mounted object has the VFS protocol, call its memmap helper. + const mp_obj_type_t *type = mp_obj_get_type(vfs->obj); + if (MP_OBJ_TYPE_HAS_SLOT(type, protocol)) { + const mp_vfs_proto_t *proto = MP_OBJ_TYPE_GET_SLOT(type, protocol); + if (proto->memmap != NULL) { + const void *data; + size_t size = proto->memmap(MP_OBJ_TO_PTR(vfs->obj), path_out, &data); + if (data != NULL) { + mp_reader_new_mem(reader, data, size, (size_t)-1); + return; + } + } + } + } + #endif + mp_obj_t args[2] = { MP_OBJ_NEW_QSTR(filename), MP_OBJ_NEW_QSTR(MP_QSTR_rb), @@ -91,6 +111,15 @@ void mp_reader_new_file(mp_reader_t *reader, qstr filename) { const mp_stream_p_t *stream_p = mp_get_stream(file); int errcode = 0; + + #if 0 && MICROPY_VFS_MAP + // Check if the stream can initialising a reader itself. + mp_uint_t reader_ret = stream_p->ioctl(file, MP_STREAM_INIT_READER, (uintptr_t)reader, &errcode); + if (reader_ret == 0) { + return; + } + #endif + mp_uint_t bufsize = stream_p->ioctl(file, MP_STREAM_GET_BUFFER_SIZE, 0, &errcode); if (bufsize == MP_STREAM_ERROR || bufsize == 0) { // bufsize == 0 is included here to support mpremote v1.21 and older where mount file ioctl @@ -104,6 +133,7 @@ void mp_reader_new_file(mp_reader_t *reader, qstr filename) { rf->file = file; rf->bufsize = bufsize; rf->buflen = mp_stream_rw(rf->file, rf->buf, rf->bufsize, &errcode, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE); + if (errcode != 0) { mp_raise_OSError(errcode); } diff --git a/py/persistentcode.c b/py/persistentcode.c index 208231530c..b9976a4b65 100644 --- a/py/persistentcode.c +++ b/py/persistentcode.c @@ -160,6 +160,18 @@ static size_t read_uint(mp_reader_t *reader) { return unum; } +#if MICROPY_VFS_MAP +static inline const uint8_t *map_try_read_bytes(mp_reader_t *reader, size_t len) { + mp_reader_ioctl_memmap_t memmap; + memmap.len = len; + intptr_t ret = reader->ioctl(reader->data, MP_READER_MEMMAP, (uintptr_t)&memmap); + if (ret < 0) { + return NULL; + } + return memmap.ptr; +} +#endif + static qstr load_qstr(mp_reader_t *reader) { size_t len = read_uint(reader); if (len & 1) { @@ -167,6 +179,12 @@ static qstr load_qstr(mp_reader_t *reader) { return len >> 1; } len >>= 1; + #if MICROPY_VFS_MAP + const uint8_t *memmap = map_try_read_bytes(reader, len + 1); + if (memmap != NULL) { + return qstr_from_strn_static((const char *)memmap, len); + } + #endif char *str = m_new(char, len); read_bytes(reader, (byte *)str, len); read_byte(reader); // read and discard null terminator @@ -175,6 +193,22 @@ static qstr load_qstr(mp_reader_t *reader) { return qst; } +#if MICROPY_VFS_MAP +static mp_obj_t mp_obj_new_str_static(const mp_obj_type_t *type, const byte *data, size_t len) { + if (type == &mp_type_str) { + qstr q = qstr_find_strn((const char *)data, len); + if (q != MP_QSTRnull) { + return MP_OBJ_NEW_QSTR(q); + } + } + mp_obj_str_t *o = m_new_obj(mp_obj_str_t); + o->base.type = type; + o->len = len; + o->hash = qstr_compute_hash(data, len); + o->data = data; + return MP_OBJ_FROM_PTR(o); +} +#endif static mp_obj_t load_obj(mp_reader_t *reader) { byte obj_type = read_byte(reader); #if MICROPY_EMIT_MACHINE_CODE @@ -192,6 +226,7 @@ static mp_obj_t load_obj(mp_reader_t *reader) { return MP_OBJ_FROM_PTR(&mp_const_ellipsis_obj); } else { size_t len = read_uint(reader); + if (len == 0 && obj_type == MP_PERSISTENT_OBJ_BYTES) { read_byte(reader); // skip null terminator return mp_const_empty_bytes; @@ -202,11 +237,25 @@ static mp_obj_t load_obj(mp_reader_t *reader) { } return MP_OBJ_FROM_PTR(tuple); } + + + const uint8_t *memmap = NULL; vstr_t vstr; - vstr_init_len(&vstr, len); - read_bytes(reader, (byte *)vstr.buf, len); + #if MICROPY_VFS_MAP + memmap = map_try_read_bytes(reader, len); + #endif + if (memmap == NULL) { + vstr_init_len(&vstr, len); + read_bytes(reader, (byte *)vstr.buf, len); + } if (obj_type == MP_PERSISTENT_OBJ_STR || obj_type == MP_PERSISTENT_OBJ_BYTES) { read_byte(reader); // skip null terminator + #if MICROPY_VFS_MAP + if (memmap != NULL) { + const mp_obj_type_t *t = obj_type == MP_PERSISTENT_OBJ_STR ? &mp_type_str : &mp_type_bytes; + return mp_obj_new_str_static(t, memmap, len); + } + #endif if (obj_type == MP_PERSISTENT_OBJ_STR) { return mp_obj_new_str_from_utf8_vstr(&vstr); } else { @@ -243,10 +292,17 @@ static mp_raw_code_t *load_raw_code(mp_reader_t *reader, mp_module_context_t *co #endif if (kind == MP_CODE_BYTECODE) { - // Allocate memory for the bytecode - fun_data = m_new(uint8_t, fun_data_len); - // Load bytecode - read_bytes(reader, fun_data, fun_data_len); + #if MICROPY_VFS_MAP + // Try to reference memory-mapped data for the bytecode. + fun_data = (uint8_t *)map_try_read_bytes(reader, fun_data_len); + #endif + + if (fun_data == NULL) { + // Allocate memory for the bytecode. + fun_data = m_new(uint8_t, fun_data_len); + // Load bytecode. + read_bytes(reader, fun_data, fun_data_len); + } #if MICROPY_EMIT_MACHINE_CODE } else { From 240b9d07e0944366718ab65624d54669712a0fab Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 4 Mar 2022 10:56:27 +1100 Subject: [PATCH 09/15] unix: Enable VfsMap on dev and coverage variants. Signed-off-by: Damien George --- ports/unix/variants/mpconfigvariant_common.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ports/unix/variants/mpconfigvariant_common.h b/ports/unix/variants/mpconfigvariant_common.h index 981babca9f..979282800a 100644 --- a/ports/unix/variants/mpconfigvariant_common.h +++ b/ports/unix/variants/mpconfigvariant_common.h @@ -117,3 +117,5 @@ #define MICROPY_PY_MACHINE (1) #define MICROPY_PY_MACHINE_PULSE (1) #define MICROPY_PY_MACHINE_PIN_BASE (1) + +#define MICROPY_VFS_MAP (1) From c31503c20338a16dbcd68c32cb21c9bba747cb16 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 4 Mar 2022 10:56:50 +1100 Subject: [PATCH 10/15] stm32: Enable VfsMap. Signed-off-by: Damien George --- ports/stm32/boardctrl.h | 4 + .../boards/NUCLEO_F091RC/mpconfigboard.mk | 2 +- ports/stm32/boards/PYBD_SF2/f722_qspi.ld | 10 +- ports/stm32/boards/PYBD_SF6/f767.ld | 6 +- ports/stm32/boards/_boot.py | 8 ++ ports/stm32/boards/manifest.py | 1 + ports/stm32/boards/manifest_minimal.py | 1 + ports/stm32/boards/stm32f091xc.ld | 6 +- ports/stm32/boards/stm32f405.ld | 6 +- ports/stm32/mpconfigport.h | 2 + ports/stm32/qstrdefsport.h | 1 + ports/stm32/storage.c | 107 ++++++++++++++++++ 12 files changed, 149 insertions(+), 5 deletions(-) create mode 100644 ports/stm32/boards/_boot.py create mode 100644 ports/stm32/boards/manifest_minimal.py diff --git a/ports/stm32/boardctrl.h b/ports/stm32/boardctrl.h index 8f4ce30eff..893e68a72c 100644 --- a/ports/stm32/boardctrl.h +++ b/ports/stm32/boardctrl.h @@ -61,6 +61,10 @@ #define MICROPY_BOARD_TOP_SOFT_RESET_LOOP boardctrl_top_soft_reset_loop #endif +#ifndef MICROPY_BOARD_FROZEN_BOOT_FILE +#define MICROPY_BOARD_FROZEN_BOOT_FILE "_boot.py" +#endif + #ifndef MICROPY_BOARD_RUN_BOOT_PY #define MICROPY_BOARD_RUN_BOOT_PY boardctrl_run_boot_py #endif diff --git a/ports/stm32/boards/NUCLEO_F091RC/mpconfigboard.mk b/ports/stm32/boards/NUCLEO_F091RC/mpconfigboard.mk index bb7142d1b3..142f11a345 100644 --- a/ports/stm32/boards/NUCLEO_F091RC/mpconfigboard.mk +++ b/ports/stm32/boards/NUCLEO_F091RC/mpconfigboard.mk @@ -8,7 +8,7 @@ MICROPY_VFS_FAT = 0 MICROPY_VFS_LFS1 ?= 1 # Don't include default frozen modules because MCU is tight on flash space -FROZEN_MANIFEST ?= +FROZEN_MANIFEST ?= boards/manifest_minimal.py # LTO reduces final binary size, may be slower to build depending on gcc version and hardware LTO ?= 1 diff --git a/ports/stm32/boards/PYBD_SF2/f722_qspi.ld b/ports/stm32/boards/PYBD_SF2/f722_qspi.ld index 6ce4c74c3f..cf99091eef 100644 --- a/ports/stm32/boards/PYBD_SF2/f722_qspi.ld +++ b/ports/stm32/boards/PYBD_SF2/f722_qspi.ld @@ -20,7 +20,8 @@ MEMORY { FLASH (rx) : ORIGIN = 0x08000000, LENGTH = 512K FLASH_ISR (rx) : ORIGIN = 0x08000000, LENGTH = 32K /* sectors 0,1 */ - FLASH_APP (rx) : ORIGIN = 0x08008000, LENGTH = 480K /* sectors 2-7 */ + FLASH_APP (rx) : ORIGIN = 0x08008000, LENGTH = 352K /* sectors 2-6 */ + FLASH_MAPFS (rx): ORIGIN = 0x08060000, LENGTH = 128K /* sector 7 */ FLASH_EXT (rx) : ORIGIN = 0x90000000, LENGTH = 2048K /* external QSPI */ RAM (rwx) : ORIGIN = 0x20000000, LENGTH = 256K /* DTCM+SRAM1+SRAM2 */ } @@ -40,6 +41,9 @@ _ram_end = ORIGIN(RAM) + LENGTH(RAM); _heap_start = _ebss; /* heap starts just after statically allocated memory */ _heap_end = _sstack; +_micropy_hw_mapfs_start = ORIGIN(FLASH_MAPFS); +_micropy_hw_mapfs_end = ORIGIN(FLASH_MAPFS) + LENGTH(FLASH_MAPFS); + /* Define output sections */ SECTIONS { @@ -51,6 +55,10 @@ SECTIONS *lib/mynewt-nimble/*(.text* .rodata*) *lib/cyw43-driver/*(.rodata.w4343*_combined) *drivers/cyw43/*(.rodata.cyw43_btfw_*) + *lib/oofatfs/*(.text* .rodata*) + *lib/littlefs/*(.text* .rodata*) + *lib/lwip/*(.text* .rodata*) + *extmod/*(.text* .rodata*) . = ALIGN(4); } >FLASH_EXT } diff --git a/ports/stm32/boards/PYBD_SF6/f767.ld b/ports/stm32/boards/PYBD_SF6/f767.ld index 167d2c6a1e..eec2301e9a 100644 --- a/ports/stm32/boards/PYBD_SF6/f767.ld +++ b/ports/stm32/boards/PYBD_SF6/f767.ld @@ -18,7 +18,8 @@ MEMORY { FLASH (rx) : ORIGIN = 0x08000000, LENGTH = 2048K FLASH_ISR (rx) : ORIGIN = 0x08000000, LENGTH = 32K /* sector 0, 32K */ - FLASH_APP (rx) : ORIGIN = 0x08008000, LENGTH = 2016K /* sectors 1-11 3x32K 1*128K 7*256K */ + FLASH_APP (rx) : ORIGIN = 0x08008000, LENGTH = 1504K /* sectors 1-11 3x32K 1*128K 7*256K */ + FLASH_MAPFS (rx): ORIGIN = 0x08180000, LENGTH = 512K FLASH_EXT (rx) : ORIGIN = 0x90000000, LENGTH = 2048K /* external QSPI */ RAM (rwx) : ORIGIN = 0x20000000, LENGTH = 512K /* DTCM=128k, SRAM1=368K, SRAM2=16K */ } @@ -38,4 +39,7 @@ _ram_end = ORIGIN(RAM) + LENGTH(RAM); _heap_start = _ebss; /* heap starts just after statically allocated memory */ _heap_end = _sstack; +_micropy_hw_mapfs_start = ORIGIN(FLASH_MAPFS); +_micropy_hw_mapfs_end = ORIGIN(FLASH_MAPFS) + LENGTH(FLASH_MAPFS); + INCLUDE common_bl.ld diff --git a/ports/stm32/boards/_boot.py b/ports/stm32/boards/_boot.py new file mode 100644 index 0000000000..6fd26f8377 --- /dev/null +++ b/ports/stm32/boards/_boot.py @@ -0,0 +1,8 @@ +import vfs, sys, pyb + + +mapfs = pyb.Flash("mapfs") +vfs.mount(vfs.VfsMap(mapfs.ioctl(0x100, 0), mapfs), "/mapfs") +sys.path.insert(0, "/mapfs") + +del vfs, sys, pyb, mapfs diff --git a/ports/stm32/boards/manifest.py b/ports/stm32/boards/manifest.py index 4f38179713..7e4d182135 100644 --- a/ports/stm32/boards/manifest.py +++ b/ports/stm32/boards/manifest.py @@ -1,3 +1,4 @@ +freeze(".", "_boot.py", opt=3) include("$(MPY_DIR)/extmod/asyncio") require("dht") diff --git a/ports/stm32/boards/manifest_minimal.py b/ports/stm32/boards/manifest_minimal.py new file mode 100644 index 0000000000..ec2408e107 --- /dev/null +++ b/ports/stm32/boards/manifest_minimal.py @@ -0,0 +1 @@ +freeze(".", "_boot.py", opt=3) diff --git a/ports/stm32/boards/stm32f091xc.ld b/ports/stm32/boards/stm32f091xc.ld index a8f0d3316e..56a34701bf 100644 --- a/ports/stm32/boards/stm32f091xc.ld +++ b/ports/stm32/boards/stm32f091xc.ld @@ -5,7 +5,8 @@ /* Specify the memory areas */ MEMORY { - FLASH (rx) : ORIGIN = 0x08000000, LENGTH = 236K /* sectors 0-117 */ + FLASH (rx) : ORIGIN = 0x08000000, LENGTH = 226K /* sectors 0-112 */ + FLASH_MAPFS (rx): ORIGIN = 0x08036000, LENGTH = 10K /* sectors 113-117 */ FLASH_FS (r) : ORIGIN = 0x0803B000, LENGTH = 20K /* sectors 118-127 */ RAM (xrw) : ORIGIN = 0x20000000, LENGTH = 32K } @@ -32,3 +33,6 @@ _heap_end = _sstack; _micropy_hw_internal_flash_storage_start = ORIGIN(FLASH_FS); _micropy_hw_internal_flash_storage_end = ORIGIN(FLASH_FS) + LENGTH(FLASH_FS); + +_micropy_hw_mapfs_start = ORIGIN(FLASH_MAPFS); +_micropy_hw_mapfs_end = ORIGIN(FLASH_MAPFS) + LENGTH(FLASH_MAPFS); diff --git a/ports/stm32/boards/stm32f405.ld b/ports/stm32/boards/stm32f405.ld index 6658c1e991..8dcf292d74 100644 --- a/ports/stm32/boards/stm32f405.ld +++ b/ports/stm32/boards/stm32f405.ld @@ -8,7 +8,8 @@ MEMORY FLASH (rx) : ORIGIN = 0x08000000, LENGTH = 1024K /* entire flash */ FLASH_ISR (rx) : ORIGIN = 0x08000000, LENGTH = 16K /* sector 0 */ FLASH_FS (rx) : ORIGIN = 0x08004000, LENGTH = 112K /* sectors 1,2,3,4 are for filesystem */ - FLASH_TEXT (rx) : ORIGIN = 0x08020000, LENGTH = 896K /* sectors 5,6,7,8,9,10,11 */ + FLASH_TEXT (rx) : ORIGIN = 0x08020000, LENGTH = 640K /* sectors 5,6,7,8,9 */ + FLASH_MAPFS (rx): ORIGIN = 0x080c0000, LENGTH = 256K /* sectors 10,11 */ CCMRAM (xrw) : ORIGIN = 0x10000000, LENGTH = 64K RAM (xrw) : ORIGIN = 0x20000000, LENGTH = 128K } @@ -33,3 +34,6 @@ _micropy_hw_internal_flash_storage_ram_cache_start = ORIGIN(CCMRAM); _micropy_hw_internal_flash_storage_ram_cache_end = ORIGIN(CCMRAM) + LENGTH(CCMRAM); _micropy_hw_internal_flash_storage_start = ORIGIN(FLASH_FS); _micropy_hw_internal_flash_storage_end = ORIGIN(FLASH_FS) + LENGTH(FLASH_FS); + +_micropy_hw_mapfs_start = ORIGIN(FLASH_MAPFS); +_micropy_hw_mapfs_end = ORIGIN(FLASH_MAPFS) + LENGTH(FLASH_MAPFS); diff --git a/ports/stm32/mpconfigport.h b/ports/stm32/mpconfigport.h index 300ad086bf..3fb45d5a7a 100644 --- a/ports/stm32/mpconfigport.h +++ b/ports/stm32/mpconfigport.h @@ -154,6 +154,8 @@ #define MICROPY_PY_ONEWIRE (1) #endif +#define MICROPY_VFS_MAP (1) + // fatfs configuration used in ffconf.h #define MICROPY_FATFS_ENABLE_LFN (1) #define MICROPY_FATFS_LFN_CODE_PAGE 437 /* 1=SFN/ANSI 437=LFN/U.S.(OEM) */ diff --git a/ports/stm32/qstrdefsport.h b/ports/stm32/qstrdefsport.h index d44388ebd2..e70a306e08 100644 --- a/ports/stm32/qstrdefsport.h +++ b/ports/stm32/qstrdefsport.h @@ -32,6 +32,7 @@ Q(/flash) Q(/flash/lib) Q(/sd) Q(/sd/lib) +Q(/mapfs) // For os.sep Q(/) diff --git a/ports/stm32/storage.c b/ports/stm32/storage.c index a6594fd4d9..452b24d50a 100644 --- a/ports/stm32/storage.c +++ b/ports/stm32/storage.c @@ -259,6 +259,8 @@ int storage_readblocks_ext(uint8_t *dest, uint32_t block, uint32_t offset, uint3 } #endif +static struct _pyb_mapfs_obj_t pyb_mapfs_obj; + typedef struct _pyb_flash_obj_t { mp_obj_base_t base; uint32_t start; // in bytes @@ -289,6 +291,11 @@ static mp_obj_t pyb_flash_make_new(const mp_obj_type_t *type, size_t n_args, siz { MP_QSTR_start, MP_ARG_KW_ONLY | MP_ARG_INT, {.u_int = -1} }, { MP_QSTR_len, MP_ARG_KW_ONLY | MP_ARG_INT, {.u_int = -1} }, }; + + if (n_args == 1 && mp_obj_is_str(all_args[0]) && mp_obj_str_get_qstr(all_args[0]) == MP_QSTR_mapfs) { + return MP_OBJ_FROM_PTR(&pyb_mapfs_obj); + } + mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)]; mp_arg_parse_all_kw_array(n_args, n_kw, all_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args); @@ -475,3 +482,103 @@ void pyb_flash_init_vfs(fs_user_mount_t *vfs) { } #endif + +/******************************************************************************/ +// mapfs partition +// +// TODO: this doesn't properly implement the block device protocol because it +// has potentially large sectors but pretends they are small. +// writing to the start of a sector will erase that whole sector first. + +#include "flash.h" + +#define MICROPY_HW_MAPFS_BASE (uintptr_t)(&_micropy_hw_mapfs_start) +#define MICROPY_HW_MAPFS_BYTES (uintptr_t)(&_micropy_hw_mapfs_end - &_micropy_hw_mapfs_start) +#define MAPFS_BLOCK_SIZE_BYTES (256) // can be anything bigger than alignment requirements of hardware writes + +typedef struct _pyb_mapfs_obj_t { + mp_obj_base_t base; + // uint32_t flash_base; + // uint32_t flash_size; +} pyb_mapfs_obj_t; + +extern uint8_t _micropy_hw_mapfs_start; +extern uint8_t _micropy_hw_mapfs_end; + +static const mp_obj_type_t pyb_mapfs_type; + +static pyb_mapfs_obj_t pyb_mapfs_obj = { + .base = { &pyb_mapfs_type }, + // .flash_base = MICROPY_HW_MAPFS_BASE, + // .flash_size = MICROPY_HW_MAPFS_BYTES, +}; + +static mp_obj_t pyb_mapfs_readblocks(size_t n_args, const mp_obj_t *args) { + // pyb_mapfs_obj_t *self = MP_OBJ_TO_PTR(args[0]); + uint32_t offset = mp_obj_get_int(args[1]) * MAPFS_BLOCK_SIZE_BYTES; + mp_buffer_info_t bufinfo; + mp_get_buffer_raise(args[2], &bufinfo, MP_BUFFER_WRITE); + memcpy(bufinfo.buf, (void *)(MICROPY_HW_MAPFS_BASE + offset), bufinfo.len); + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(pyb_mapfs_readblocks_obj, 3, 3, pyb_mapfs_readblocks); + +static mp_obj_t pyb_mapfs_writeblocks(size_t n_args, const mp_obj_t *args) { + // pyb_mapfs_obj_t *self = MP_OBJ_TO_PTR(args[0]); + uint32_t offset = mp_obj_get_int(args[1]) * MAPFS_BLOCK_SIZE_BYTES; + mp_buffer_info_t bufinfo; + mp_get_buffer_raise(args[2], &bufinfo, MP_BUFFER_READ); + uint32_t dest = MICROPY_HW_MAPFS_BASE + offset; + + // Compute start and end address of the sector being written. + uint32_t sector_size = 0; + uint32_t sector_start = 0; + int ret = flash_get_sector_info(dest, §or_start, §or_size); + if (ret < 0) { + return MP_OBJ_NEW_SMALL_INT(ret); + } + + // Erase sector if writing to the start of it. + if (dest == sector_start) { + ret = flash_erase(sector_start); + if (ret != 0) { + return MP_OBJ_NEW_SMALL_INT(ret); + } + } + + flash_write(dest, bufinfo.buf, bufinfo.len / 4); + // TODO check return value + + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(pyb_mapfs_writeblocks_obj, 3, 4, pyb_mapfs_writeblocks); + +static mp_obj_t pyb_mapfs_ioctl(mp_obj_t self_in, mp_obj_t cmd_in, mp_obj_t arg_in) { + // pyb_mapfs_obj_t *self = MP_OBJ_TO_PTR(self_in); + mp_int_t cmd = mp_obj_get_int(cmd_in); + switch (cmd) { + case MP_BLOCKDEV_IOCTL_BLOCK_COUNT: + return MP_OBJ_NEW_SMALL_INT(MICROPY_HW_MAPFS_BYTES / MAPFS_BLOCK_SIZE_BYTES); + case MP_BLOCKDEV_IOCTL_BLOCK_SIZE: + return MP_OBJ_NEW_SMALL_INT(MAPFS_BLOCK_SIZE_BYTES); + case 0x100: // mmap + return mp_obj_new_int(MICROPY_HW_MAPFS_BASE); + default: + return mp_const_none; + } +} +static MP_DEFINE_CONST_FUN_OBJ_3(pyb_mapfs_ioctl_obj, pyb_mapfs_ioctl); + +static const mp_rom_map_elem_t pyb_mapfs_locals_dict_table[] = { + { MP_ROM_QSTR(MP_QSTR_readblocks), MP_ROM_PTR(&pyb_mapfs_readblocks_obj) }, + { MP_ROM_QSTR(MP_QSTR_writeblocks), MP_ROM_PTR(&pyb_mapfs_writeblocks_obj) }, + { MP_ROM_QSTR(MP_QSTR_ioctl), MP_ROM_PTR(&pyb_mapfs_ioctl_obj) }, +}; +static MP_DEFINE_CONST_DICT(pyb_mapfs_locals_dict, pyb_mapfs_locals_dict_table); + +static MP_DEFINE_CONST_OBJ_TYPE( + pyb_mapfs_type, + MP_QSTR_Flash, + MP_TYPE_FLAG_NONE, + locals_dict, &pyb_mapfs_locals_dict + ); From 56d038088ff31b195a0c4e539ffccb05f0da97b2 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 4 Mar 2022 10:57:38 +1100 Subject: [PATCH 11/15] rp2: Enable VfsMap. Signed-off-by: Damien George --- ports/rp2/modules/_boot.py | 8 +++++++- ports/rp2/mpconfigport.h | 1 + ports/rp2/rp2_flash.c | 21 +++++++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/ports/rp2/modules/_boot.py b/ports/rp2/modules/_boot.py index 71cb28d682..743bb24962 100644 --- a/ports/rp2/modules/_boot.py +++ b/ports/rp2/modules/_boot.py @@ -1,7 +1,13 @@ import vfs +import sys import machine, rp2 +mapfs = rp2.Flash("mapfs") +vfs.mount(vfs.VfsMap(mapfs.ioctl(0x100, 0), mapfs), "/mapfs") +sys.path.insert(0, "/mapfs") + + # Try to mount the filesystem, and format the flash if it doesn't exist. # Note: the flash requires the programming size to be aligned to 256 bytes. bdev = rp2.Flash() @@ -12,4 +18,4 @@ except: fs = vfs.VfsLfs2(bdev, progsize=256) vfs.mount(fs, "/") -del vfs, bdev, fs +del vfs, sys, bdev, fs, mapfs diff --git a/ports/rp2/mpconfigport.h b/ports/rp2/mpconfigport.h index 8f073002e8..fdedf6d22c 100644 --- a/ports/rp2/mpconfigport.h +++ b/ports/rp2/mpconfigport.h @@ -146,6 +146,7 @@ #define MICROPY_VFS (1) #define MICROPY_VFS_LFS2 (1) #define MICROPY_VFS_FAT (1) +#define MICROPY_VFS_MAP (1) #define MICROPY_SSL_MBEDTLS (1) #define MICROPY_PY_LWIP_SOCK_RAW (MICROPY_PY_LWIP) diff --git a/ports/rp2/rp2_flash.c b/ports/rp2/rp2_flash.c index c1acb54e75..45cf027a24 100644 --- a/ports/rp2/rp2_flash.c +++ b/ports/rp2/rp2_flash.c @@ -28,6 +28,7 @@ #include "py/mphal.h" #include "py/runtime.h" +#include "py/mperrno.h" #include "extmod/vfs.h" #include "modrp2.h" #include "hardware/flash.h" @@ -44,6 +45,9 @@ static_assert(MICROPY_HW_FLASH_STORAGE_BYTES % 4096 == 0, "Flash storage size mu #define MICROPY_HW_FLASH_STORAGE_BASE (PICO_FLASH_SIZE_BYTES - MICROPY_HW_FLASH_STORAGE_BYTES) #endif +#define MICROPY_HW_MAPFS_BASE (512 * 1024) // leave 512k for firmware... +#define MICROPY_HW_MAPFS_BYTES (MICROPY_HW_FLASH_STORAGE_BASE - MICROPY_HW_MAPFS_BASE) + static_assert(MICROPY_HW_FLASH_STORAGE_BYTES <= PICO_FLASH_SIZE_BYTES, "MICROPY_HW_FLASH_STORAGE_BYTES too big"); static_assert(MICROPY_HW_FLASH_STORAGE_BASE + MICROPY_HW_FLASH_STORAGE_BYTES <= PICO_FLASH_SIZE_BYTES, "MICROPY_HW_FLASH_STORAGE_BYTES too big"); @@ -53,6 +57,12 @@ typedef struct _rp2_flash_obj_t { uint32_t flash_size; } rp2_flash_obj_t; +static rp2_flash_obj_t rp2_flash_mapfs_obj = { + .base = { &rp2_flash_type }, + .flash_base = MICROPY_HW_MAPFS_BASE, + .flash_size = MICROPY_HW_MAPFS_BYTES, +}; + static rp2_flash_obj_t rp2_flash_obj = { .base = { &rp2_flash_type }, .flash_base = MICROPY_HW_FLASH_STORAGE_BASE, @@ -87,6 +97,15 @@ static void end_critical_flash_section(uint32_t state) { } static mp_obj_t rp2_flash_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *all_args) { + if (n_args == 1 && n_kw == 0 && mp_obj_is_str(all_args[0])) { + if (mp_obj_str_get_qstr(all_args[0]) == MP_QSTR_mapfs) { + if (MICROPY_HW_MAPFS_BYTES <= 0) { + mp_raise_OSError(MP_ENODEV); + } + return MP_OBJ_FROM_PTR(&rp2_flash_mapfs_obj); + } + } + // Parse arguments enum { ARG_start, ARG_len }; static const mp_arg_t allowed_args[] = { @@ -190,6 +209,8 @@ static mp_obj_t rp2_flash_ioctl(mp_obj_t self_in, mp_obj_t cmd_in, mp_obj_t arg_ // TODO check return value return MP_OBJ_NEW_SMALL_INT(0); } + case 0x100: // mmap + return mp_obj_new_int(XIP_BASE + self->flash_base); default: return mp_const_none; } From 6e663a4afebb338c3ab243cf936791eb20ad6654 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 4 Mar 2022 10:57:46 +1100 Subject: [PATCH 12/15] tools/mpremote: Add deploy-mapfs. Signed-off-by: Damien George --- tools/mpremote/mpremote/commands.py | 44 +++++++++++ tools/mpremote/mpremote/main.py | 11 +++ tools/mpremote/mpremote/mapfs.py | 110 ++++++++++++++++++++++++++++ 3 files changed, 165 insertions(+) create mode 100644 tools/mpremote/mpremote/mapfs.py diff --git a/tools/mpremote/mpremote/commands.py b/tools/mpremote/mpremote/commands.py index aae612765b..8aa8378a1e 100644 --- a/tools/mpremote/mpremote/commands.py +++ b/tools/mpremote/mpremote/commands.py @@ -6,6 +6,7 @@ import serial.tools.list_ports from .transport import TransportError from .transport_serial import SerialTransport, stdout_write_bytes +from .mapfs import make_mapfs class CommandError(Exception): @@ -259,3 +260,46 @@ def do_rtc(state, args): _do_execbuffer(state, "import machine; machine.RTC().datetime({})".format(timetuple), True) else: _do_execbuffer(state, "import machine; print(machine.RTC().datetime())", True) + + +def do_deploy_mapfs(state, args): + state.ensure_raw_repl() + state.did_action() + + # Detect the mapfs and get its associated device. + state.transport.exec("import vfs; mapfs = vfs.mount('/mapfs')") + if state.transport.eval("mapfs") == b"None": + print("/mapfs does not exist on device") + sys.exit(1) + if state.transport.eval("mapfs.device()") == b"None": + print("/mapfs does not have an associated device") + sys.exit(1) + state.transport.exec("dev=mapfs.device()") + block_count = int(str(state.transport.eval("dev.ioctl(4,0)"), "ascii")) + block_size = int(str(state.transport.eval("dev.ioctl(5,0)"), "ascii")) + print( + f"/mapfs is a block device of size {block_count}*{block_size}={block_count * block_size} bytes" + ) + + # Create the mapfs filesystem. + mapfs = make_mapfs(args.path[0], "/") + print(f"Image size is {len(mapfs)} bytes") + + if len(mapfs) > block_count * block_size: + print("/mapfs is too small for image") + sys.exit(1) + + # Deploy the mapfs filesystem to the device. + state.transport.exec(f"buf=bytearray({block_size})") + for block in range(0, (len(mapfs) + block_size - 1) // block_size): + mapfs_block = mapfs[block * block_size : (block + 1) * block_size] + off = 0 + while off < len(mapfs_block): + l = min(len(mapfs_block) - off, 256) + state.transport.exec(f"buf[{off}:{off+l}]=" + repr(mapfs_block[off : off + l])) + off += l + print(f"\rWriting block {block}", end="") + state.transport.exec(f"dev.writeblocks({block},buf)") + + print() + print("Image deployed") diff --git a/tools/mpremote/mpremote/main.py b/tools/mpremote/mpremote/main.py index eeb9cbd989..3a86c83054 100644 --- a/tools/mpremote/mpremote/main.py +++ b/tools/mpremote/mpremote/main.py @@ -36,6 +36,7 @@ from .commands import ( do_resume, do_rtc, do_soft_reset, + do_deploy_mapfs, ) from .mip import do_mip from .repl import do_repl @@ -219,6 +220,12 @@ def argparse_mip(): return cmd_parser +def argparse_deploy_mapfs(): + cmd_parser = argparse.ArgumentParser(description="deploy a directory to /mapfs on the device") + cmd_parser.add_argument("path", nargs=1, help="path to directory to deploy") + return cmd_parser + + def argparse_none(description): return lambda: argparse.ArgumentParser(description=description) @@ -293,6 +300,10 @@ _COMMANDS = { do_version, argparse_none("print version and exit"), ), + "deploy-mapfs": ( + do_deploy_mapfs, + argparse_deploy_mapfs, + ), } # Additional commands aliases. diff --git a/tools/mpremote/mpremote/mapfs.py b/tools/mpremote/mpremote/mapfs.py new file mode 100644 index 0000000000..3ea5bb35b8 --- /dev/null +++ b/tools/mpremote/mpremote/mapfs.py @@ -0,0 +1,110 @@ +# MIT license; Copyright (c) 2022 Damien P. George + +import struct, sys, os + + +class VfsMapWriter: + MAGIC = b"MF" + + def __init__(self): + self.filename = None + self.data = bytearray() + self.data += VfsMapWriter.MAGIC + + def finalise(self): + self.data += b"\x00\x00" + + def open(self, filename, attr): + assert self.filename is None + assert attr == "wb" + self.filename = filename + self.filedata = b"" + return self + + def mkdir(self, dir): + assert self.filename is None + self.data += struct.pack("".format(sys.argv[0]), file=sys.stderr) + sys.exit(1) + + # Parse arguments. + dir = sys.argv[1] + + mapfs = make_mapfs(dir, "/") + + # Save the block device data. + output = dir.rstrip("/") + ".mapfs" + print("Writing filesystem image to {}".format(output)) + with open(output, "wb") as f: + f.write(mapfs) + + +if __name__ == "__main__": + main() From e26ff73fef4c5348991f5e775dbe0e6bed999b05 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 3 Mar 2022 18:05:06 +1100 Subject: [PATCH 13/15] esp32/esp32_partition: Add Partition.mmap() method. Signed-off-by: Damien George --- ports/esp32/esp32_partition.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ports/esp32/esp32_partition.c b/ports/esp32/esp32_partition.c index e5d4809620..b7e09ef823 100644 --- a/ports/esp32/esp32_partition.c +++ b/ports/esp32/esp32_partition.c @@ -265,6 +265,16 @@ static MP_DEFINE_CONST_FUN_OBJ_1(esp32_partition_mark_app_valid_cancel_rollback_ static MP_DEFINE_CONST_CLASSMETHOD_OBJ(esp32_partition_mark_app_valid_cancel_rollback_obj, MP_ROM_PTR(&esp32_partition_mark_app_valid_cancel_rollback_fun_obj)); +static mp_obj_t esp32_partition_mmap(mp_obj_t self_in) { + esp32_partition_obj_t *self = MP_OBJ_TO_PTR(self_in); + const void *ptr; + esp_partition_mmap_handle_t handle; + // TODO need to esp_partition_munmap(handle) on soft reset + check_esp_err(esp_partition_mmap(self->part, 0, self->part->size, ESP_PARTITION_MMAP_DATA, &ptr, &handle)); + return mp_obj_new_memoryview('B', self->part->size, (void *)ptr); +} +static MP_DEFINE_CONST_FUN_OBJ_1(esp32_partition_mmap_obj, esp32_partition_mmap); + static const mp_rom_map_elem_t esp32_partition_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_find), MP_ROM_PTR(&esp32_partition_find_obj) }, @@ -276,6 +286,7 @@ static const mp_rom_map_elem_t esp32_partition_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_set_boot), MP_ROM_PTR(&esp32_partition_set_boot_obj) }, { MP_ROM_QSTR(MP_QSTR_mark_app_valid_cancel_rollback), MP_ROM_PTR(&esp32_partition_mark_app_valid_cancel_rollback_obj) }, { MP_ROM_QSTR(MP_QSTR_get_next_update), MP_ROM_PTR(&esp32_partition_get_next_update_obj) }, + { MP_ROM_QSTR(MP_QSTR_mmap), MP_ROM_PTR(&esp32_partition_mmap_obj) }, { MP_ROM_QSTR(MP_QSTR_BOOT), MP_ROM_INT(ESP32_PARTITION_BOOT) }, { MP_ROM_QSTR(MP_QSTR_RUNNING), MP_ROM_INT(ESP32_PARTITION_RUNNING) }, From 8477e891e29080d8d96c9e5fc6af4479a60e28fb Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 3 Mar 2022 18:05:25 +1100 Subject: [PATCH 14/15] esp32/partitions.csv: Reserve 128K for mapfs. Signed-off-by: Damien George --- ports/esp32/partitions-4MiB.csv | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ports/esp32/partitions-4MiB.csv b/ports/esp32/partitions-4MiB.csv index 53f0f16744..399d29f9d4 100644 --- a/ports/esp32/partitions-4MiB.csv +++ b/ports/esp32/partitions-4MiB.csv @@ -3,5 +3,6 @@ # Name, Type, SubType, Offset, Size, Flags nvs, data, nvs, 0x9000, 0x6000, phy_init, data, phy, 0xf000, 0x1000, -factory, app, factory, 0x10000, 0x1F0000, +factory, app, factory, 0x10000, 0x1D0000, +mapfs, data, 0x8f, 0x1E0000, 0x20000, vfs, data, fat, 0x200000, 0x200000, From 303502936357469ad00a5d2a18879fc2831a5677 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 3 Mar 2022 18:11:09 +1100 Subject: [PATCH 15/15] esp32: Enable VfsMap. Signed-off-by: Damien George --- ports/esp32/modules/_boot.py | 7 +++++++ ports/esp32/mpconfigport.h | 1 + 2 files changed, 8 insertions(+) diff --git a/ports/esp32/modules/_boot.py b/ports/esp32/modules/_boot.py index 96af581f38..6ff771425b 100644 --- a/ports/esp32/modules/_boot.py +++ b/ports/esp32/modules/_boot.py @@ -1,7 +1,14 @@ import gc import vfs +from esp32 import Partition from flashbdev import bdev +if mapfs := Partition.find(Partition.TYPE_DATA, label="mapfs"): + import sys + + sys.path.insert(0, "/mapfs") +del mapfs + try: if bdev: vfs.mount(bdev, "/") diff --git a/ports/esp32/mpconfigport.h b/ports/esp32/mpconfigport.h index acf065384b..e2004f463b 100644 --- a/ports/esp32/mpconfigport.h +++ b/ports/esp32/mpconfigport.h @@ -71,6 +71,7 @@ #define MICROPY_USE_INTERNAL_PRINTF (0) // ESP32 SDK requires its own printf #define MICROPY_SCHEDULER_DEPTH (8) #define MICROPY_VFS (1) +#define MICROPY_VFS_MAP (1) // control over Python builtins #define MICROPY_PY_STR_BYTES_CMP_WARN (1)