From f76cf2940288f699b8c6346beabdf4de65ddfa31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Neusch=C3=A4fer?= Date: Sat, 20 Apr 2024 19:36:03 +0200 Subject: [PATCH 01/18] github/workflows: Update coverage workflow to codecov-action@v4. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes: https://github.com/micropython/micropython/issues/14340 Signed-off-by: J. Neuschäfer --- .github/workflows/ports_unix.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ports_unix.yml b/.github/workflows/ports_unix.yml index 2367eddbe6..4473847db6 100644 --- a/.github/workflows/ports_unix.yml +++ b/.github/workflows/ports_unix.yml @@ -88,10 +88,11 @@ jobs: (cd ports/unix && gcov -o build-coverage/py ../../py/*.c || true) (cd ports/unix && gcov -o build-coverage/extmod ../../extmod/*.c || true) - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: fail_ci_if_error: true verbose: true + token: ${{ secrets.CODECOV_TOKEN }} - name: Print failures if: failure() run: tests/run-tests.py --print-failures From 19844b4983066c5771cb60a341c3fce52bca2a87 Mon Sep 17 00:00:00 2001 From: Simon Wood Date: Thu, 28 Mar 2024 09:27:25 -0600 Subject: [PATCH 02/18] rp2/modmachine: Prevent lock-up when lightsleep() called within thread. When `lightsleep()` is called from within a thread the interrupts may not be enabled on current core, and thus the call to `lightsleep()` never completes. Fixes issue #14092. Signed-off-by: Simon Wood --- ports/rp2/modmachine.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/ports/rp2/modmachine.c b/ports/rp2/modmachine.c index 6189e7dc53..5551bf723a 100644 --- a/ports/rp2/modmachine.c +++ b/ports/rp2/modmachine.c @@ -127,10 +127,10 @@ static void mp_machine_lightsleep(size_t n_args, const mp_obj_t *args) { const uint32_t xosc_hz = XOSC_MHZ * 1000000; - uint32_t my_interrupts = save_and_disable_interrupts(); + uint32_t my_interrupts = mp_thread_begin_atomic_section(); #if MICROPY_PY_NETWORK_CYW43 if (cyw43_has_pending && cyw43_poll != NULL) { - restore_interrupts(my_interrupts); + mp_thread_end_atomic_section(my_interrupts); return; } #endif @@ -165,8 +165,15 @@ static void mp_machine_lightsleep(size_t n_args, const mp_obj_t *args) { } else { uint32_t sleep_en0 = clocks_hw->sleep_en0; uint32_t sleep_en1 = clocks_hw->sleep_en1; + bool timer3_enabled = irq_is_enabled(3); + clocks_hw->sleep_en0 = CLOCKS_SLEEP_EN0_CLK_RTC_RTC_BITS; if (use_timer_alarm) { + // Make sure ALARM3/IRQ3 is enabled on _this_ core + timer_hw->inte |= 1 << 3; + if (!timer3_enabled) { + irq_set_enabled(3, true); + } // Use timer alarm to wake. clocks_hw->sleep_en1 = CLOCKS_SLEEP_EN1_CLK_SYS_TIMER_BITS; timer_hw->alarm[3] = timer_hw->timerawl + delay_ms * 1000; @@ -177,6 +184,9 @@ static void mp_machine_lightsleep(size_t n_args, const mp_obj_t *args) { scb_hw->scr |= M0PLUS_SCR_SLEEPDEEP_BITS; __wfi(); scb_hw->scr &= ~M0PLUS_SCR_SLEEPDEEP_BITS; + if (!timer3_enabled) { + irq_set_enabled(3, false); + } clocks_hw->sleep_en0 = sleep_en0; clocks_hw->sleep_en1 = sleep_en1; } @@ -186,7 +196,7 @@ static void mp_machine_lightsleep(size_t n_args, const mp_obj_t *args) { // Bring back all clocks. clocks_init(); - restore_interrupts(my_interrupts); + mp_thread_end_atomic_section(my_interrupts); } NORETURN static void mp_machine_deepsleep(size_t n_args, const mp_obj_t *args) { From 3129b69e0f990d9f9bf8cc73d7650ddd8a5e06a3 Mon Sep 17 00:00:00 2001 From: "Michiel W. Beijen" Date: Tue, 2 Apr 2024 21:58:29 +0200 Subject: [PATCH 03/18] rp2/README: Fix typo, improve sentence about building with other boards. Signed-off-by: Michiel W. Beijen --- ports/rp2/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ports/rp2/README.md b/ports/rp2/README.md index e22fb3093d..c2f3771cd3 100644 --- a/ports/rp2/README.md +++ b/ports/rp2/README.md @@ -37,7 +37,7 @@ You can also build the standard CMake way. The final firmware is found in the top-level of the CMake build directory (`build` by default) and is called `firmware.uf2`. -If you are using a different board other than a Rasoberry Pi Pico, then you should +If you are using a board other than a Raspberry Pi Pico, you should pass the board name to the build; e.g. for Raspberry Pi Pico W: $ make BOARD=RPI_PICO_W submodules From 40f7e9ce20e845ce2372347e5b5fd4dbff69ef6e Mon Sep 17 00:00:00 2001 From: stijn Date: Thu, 18 Apr 2024 12:01:45 +0200 Subject: [PATCH 04/18] py/objfun: Fix C++ compatibility with casting in inline functions. Explicit casts are needed. Fixes recent changes from 648a7578da21cc7ddb4046fc59891144e797b983 and 9400229766624e80db6a6f95af287a5542dc1b43. Signed-off-by: stijn --- py/objfun.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/py/objfun.h b/py/objfun.h index af7c334858..f16ef76b80 100644 --- a/py/objfun.h +++ b/py/objfun.h @@ -56,14 +56,14 @@ void mp_obj_fun_bc_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest); #if MICROPY_EMIT_NATIVE static inline mp_obj_t mp_obj_new_fun_native(const mp_obj_t *def_args, const void *fun_data, const mp_module_context_t *mc, struct _mp_raw_code_t *const *child_table) { - mp_obj_fun_bc_t *o = MP_OBJ_TO_PTR(mp_obj_new_fun_bc(def_args, (const byte *)fun_data, mc, child_table)); + mp_obj_fun_bc_t *o = (mp_obj_fun_bc_t *)MP_OBJ_TO_PTR(mp_obj_new_fun_bc(def_args, (const byte *)fun_data, mc, child_table)); o->base.type = &mp_type_fun_native; return MP_OBJ_FROM_PTR(o); } static inline mp_obj_t mp_obj_new_fun_viper(const void *fun_data, const mp_module_context_t *mc, struct _mp_raw_code_t *const *child_table) { mp_obj_fun_bc_t *o = mp_obj_malloc(mp_obj_fun_bc_t, &mp_type_fun_viper); - o->bytecode = fun_data; + o->bytecode = (const byte *)fun_data; o->context = mc; o->child_table = child_table; return MP_OBJ_FROM_PTR(o); @@ -101,9 +101,9 @@ static inline void *mp_obj_fun_native_get_generator_resume(const mp_obj_fun_bc_t #if MICROPY_EMIT_INLINE_ASM static inline mp_obj_t mp_obj_new_fun_asm(size_t n_args, const void *fun_data, mp_uint_t type_sig) { - mp_obj_fun_asm_t *o = mp_obj_malloc(mp_obj_fun_asm_t, &mp_type_fun_asm); + mp_obj_fun_asm_t *o = (mp_obj_fun_asm_t *)mp_obj_malloc(mp_obj_fun_asm_t, &mp_type_fun_asm); o->n_args = n_args; - o->fun_data = fun_data; + o->fun_data = (const byte *)fun_data; o->type_sig = type_sig; return MP_OBJ_FROM_PTR(o); } From ce491ab0d168a8278062e1fc7ebed3ca47ab89d2 Mon Sep 17 00:00:00 2001 From: Vonasmic Date: Wed, 3 Apr 2024 10:35:58 +0200 Subject: [PATCH 05/18] py/obj: Fix initialiser order in MP_DEFINE_CONST_OBJ_TYPE_NARGS_ macros. This commit swaps the order of the `flags` and `name` struct initialisers for `mp_obj_type_t`, to fix an incompatibility with C++. The original order of the initialiser didn't match the definition of the type, and although that's still legal C, it's not legal C++. Signed-off-by: Vonasmic --- py/obj.h | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/py/obj.h b/py/obj.h index 9f2bb46e41..86952565c5 100644 --- a/py/obj.h +++ b/py/obj.h @@ -753,20 +753,20 @@ typedef struct _mp_obj_full_type_t { // Do not use these directly, instead use MP_DEFINE_CONST_OBJ_TYPE. // Generated with: // for i in range(13): -// print(f"#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_{i}(_struct_type, _typename, _name, _flags{''.join(f', f{j+1}, v{j+1}' for j in range(i))}) const _struct_type _typename = {{ .base = {{ &mp_type_type }}, .name = _name, .flags = _flags{''.join(f', .slot_index_##f{j+1} = {j+1}' for j in range(i))}{', .slots = { ' + ''.join(f'v{j+1}, ' for j in range(i)) + '}' if i else '' } }}") -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_0(_struct_type, _typename, _name, _flags) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_1(_struct_type, _typename, _name, _flags, f1, v1) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slots = { v1, } } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_2(_struct_type, _typename, _name, _flags, f1, v1, f2, v2) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slots = { v1, v2, } } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_3(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slots = { v1, v2, v3, } } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_4(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slots = { v1, v2, v3, v4, } } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_5(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slots = { v1, v2, v3, v4, v5, } } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_6(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slots = { v1, v2, v3, v4, v5, v6, } } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_7(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slots = { v1, v2, v3, v4, v5, v6, v7, } } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_8(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, } } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_9(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, } } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_10(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, } } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_11(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10, f11, v11) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slot_index_##f11 = 11, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, } } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_12(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10, f11, v11, f12, v12) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slot_index_##f11 = 11, .slot_index_##f12 = 12, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, } } +// print(f"#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_{i}(_struct_type, _typename, _name, _flags{''.join(f', f{j+1}, v{j+1}' for j in range(i))}) const _struct_type _typename = {{ .base = {{ &mp_type_type }}, .flags = _flags, .name = _name{''.join(f', .slot_index_##f{j+1} = {j+1}' for j in range(i))}{', .slots = { ' + ''.join(f'v{j+1}, ' for j in range(i)) + '}' if i else '' } }}") +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_0(_struct_type, _typename, _name, _flags) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_1(_struct_type, _typename, _name, _flags, f1, v1) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slots = { v1, } } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_2(_struct_type, _typename, _name, _flags, f1, v1, f2, v2) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slots = { v1, v2, } } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_3(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slots = { v1, v2, v3, } } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_4(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slots = { v1, v2, v3, v4, } } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_5(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slots = { v1, v2, v3, v4, v5, } } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_6(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slots = { v1, v2, v3, v4, v5, v6, } } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_7(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slots = { v1, v2, v3, v4, v5, v6, v7, } } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_8(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, } } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_9(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, } } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_10(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, } } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_11(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10, f11, v11) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slot_index_##f11 = 11, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, } } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_12(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10, f11, v11, f12, v12) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slot_index_##f11 = 11, .slot_index_##f12 = 12, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, } } // Because the mp_obj_type_t instances are in (zero-initialised) ROM, we take // slot_index_foo=0 to mean that the slot is unset. This also simplifies checking From 4bed614e707c0644c06e117f848fa12605c711cd Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Tue, 13 Feb 2024 09:24:36 +1100 Subject: [PATCH 06/18] py/objarray: Fix use-after-free if extending a bytearray from itself. Two cases, one assigning to a slice. Closes https://github.com/micropython/micropython/issues/13283 Second is extending a slice from itself, similar logic. In both cases the problem occurs when m_renew causes realloc to move the buffer, leaving a dangling pointer behind. There are more complex and hard to fix cases when either argument is a memoryview into the buffer, currently resizing to a new address breaks memoryviews into that object. Reproducing this bug and confirming the fix was done by running the unix port under valgrind with GC-aware extensions. Note in default configurations with GIL this bug exists but has no impact (the free buffer won't be reused while the function is still executing, and is no longer referenced after it returns). Signed-off-by: Angus Gratton --- py/objarray.c | 20 ++++++++++++++++---- tests/basics/bytearray_add.py | 9 ++++++++- tests/basics/bytearray_add_self.py | 8 ++++++++ tests/basics/bytearray_add_self.py.exp | 1 + tests/basics/bytearray_slice_assign.py | 18 ++++++++++++------ 5 files changed, 45 insertions(+), 11 deletions(-) create mode 100644 tests/basics/bytearray_add_self.py create mode 100644 tests/basics/bytearray_add_self.py.exp diff --git a/py/objarray.c b/py/objarray.c index 1fff234822..803af2cd27 100644 --- a/py/objarray.c +++ b/py/objarray.c @@ -424,6 +424,13 @@ static mp_obj_t array_extend(mp_obj_t self_in, mp_obj_t arg_in) { if (self->free < len) { self->items = m_renew(byte, self->items, (self->len + self->free) * sz, (self->len + len) * sz); self->free = 0; + + if (self_in == arg_in) { + // Get arg_bufinfo again in case self->items has moved + // + // (Note not possible to handle case that arg_in is a memoryview into self) + mp_get_buffer_raise(arg_in, &arg_bufinfo, MP_BUFFER_READ); + } } else { self->free -= len; } @@ -456,7 +463,8 @@ static mp_obj_t array_subscr(mp_obj_t self_in, mp_obj_t index_in, mp_obj_t value #if MICROPY_PY_ARRAY_SLICE_ASSIGN // Assign size_t src_len; - void *src_items; + uint8_t *src_items; + size_t src_offs = 0; size_t item_sz = mp_binary_get_size('@', o->typecode & TYPECODE_MASK, NULL); if (mp_obj_is_obj(value) && MP_OBJ_TYPE_GET_SLOT_OR_NULL(((mp_obj_base_t *)MP_OBJ_TO_PTR(value))->type, subscr) == array_subscr) { // value is array, bytearray or memoryview @@ -469,7 +477,7 @@ static mp_obj_t array_subscr(mp_obj_t self_in, mp_obj_t index_in, mp_obj_t value src_items = src_slice->items; #if MICROPY_PY_BUILTINS_MEMORYVIEW if (mp_obj_is_type(value, &mp_type_memoryview)) { - src_items = (uint8_t *)src_items + (src_slice->memview_offset * item_sz); + src_offs = src_slice->memview_offset * item_sz; } #endif } else if (mp_obj_is_type(value, &mp_type_bytes)) { @@ -504,13 +512,17 @@ static mp_obj_t array_subscr(mp_obj_t self_in, mp_obj_t index_in, mp_obj_t value // TODO: alloc policy; at the moment we go conservative o->items = m_renew(byte, o->items, (o->len + o->free) * item_sz, (o->len + len_adj) * item_sz); o->free = len_adj; + // m_renew may have moved o->items + if (src_items == dest_items) { + src_items = o->items; + } dest_items = o->items; } mp_seq_replace_slice_grow_inplace(dest_items, o->len, - slice.start, slice.stop, src_items, src_len, len_adj, item_sz); + slice.start, slice.stop, src_items + src_offs, src_len, len_adj, item_sz); } else { mp_seq_replace_slice_no_grow(dest_items, o->len, - slice.start, slice.stop, src_items, src_len, item_sz); + slice.start, slice.stop, src_items + src_offs, src_len, item_sz); // Clear "freed" elements at the end of list // TODO: This is actually only needed for typecode=='O' mp_seq_clear(dest_items, o->len + len_adj, o->len, item_sz); diff --git a/tests/basics/bytearray_add.py b/tests/basics/bytearray_add.py index a7e2b57374..1f30a3b740 100644 --- a/tests/basics/bytearray_add.py +++ b/tests/basics/bytearray_add.py @@ -15,4 +15,11 @@ print(b) # this inplace add tests the code when the buffer doesn't need to be increased b = bytearray() -b += b'' +b += b"" + +# extend a bytearray from itself +b = bytearray(b"abcdefgh") +for _ in range(4): + c = bytearray(b) # extra allocation, as above + b.extend(b) +print(b) diff --git a/tests/basics/bytearray_add_self.py b/tests/basics/bytearray_add_self.py new file mode 100644 index 0000000000..94ae8689fd --- /dev/null +++ b/tests/basics/bytearray_add_self.py @@ -0,0 +1,8 @@ +# add a bytearray to itself +# This is not supported by CPython as of 3.11.18. + +b = bytearray(b"123456789") +for _ in range(4): + c = bytearray(b) # extra allocation increases chance 'b' has to relocate + b += b +print(b) diff --git a/tests/basics/bytearray_add_self.py.exp b/tests/basics/bytearray_add_self.py.exp new file mode 100644 index 0000000000..5ef948157a --- /dev/null +++ b/tests/basics/bytearray_add_self.py.exp @@ -0,0 +1 @@ +bytearray(b'123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789') diff --git a/tests/basics/bytearray_slice_assign.py b/tests/basics/bytearray_slice_assign.py index fa7878e10d..4de0819042 100644 --- a/tests/basics/bytearray_slice_assign.py +++ b/tests/basics/bytearray_slice_assign.py @@ -18,7 +18,7 @@ l = bytearray(x) l[1:3] = bytearray() print(l) l = bytearray(x) -#del l[1:3] +# del l[1:3] print(l) l = bytearray(x) @@ -28,7 +28,7 @@ l = bytearray(x) l[:3] = bytearray() print(l) l = bytearray(x) -#del l[:3] +# del l[:3] print(l) l = bytearray(x) @@ -38,7 +38,7 @@ l = bytearray(x) l[:-3] = bytearray() print(l) l = bytearray(x) -#del l[:-3] +# del l[:-3] print(l) # slice assignment that extends the array @@ -61,8 +61,14 @@ b[1:1] = b"12345" print(b) # Growth of bytearray via slice extension -b = bytearray(b'12345678') -b.append(57) # expand and add a bit of unused space at end of the bytearray +b = bytearray(b"12345678") +b.append(57) # expand and add a bit of unused space at end of the bytearray for i in range(400): - b[-1:] = b'ab' # grow slowly into the unused space + b[-1:] = b"ab" # grow slowly into the unused space +print(len(b), b) + +# Growth of bytearray via slice extension from itself +b = bytearray(b"1234567") +for i in range(3): + b[-1:] = b print(len(b), b) From 6877987002684465d30b349759285648824553da Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Wed, 27 Mar 2024 17:25:25 +1100 Subject: [PATCH 07/18] tests/cpydiff: Add a note about risk of resizing memoryview targets. This a stop-gap until there is a proper fix for this. This work was funded through GitHub Sponsors. Signed-off-by: Angus Gratton --- tests/cpydiff/types_memoryview_invalid.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 tests/cpydiff/types_memoryview_invalid.py diff --git a/tests/cpydiff/types_memoryview_invalid.py b/tests/cpydiff/types_memoryview_invalid.py new file mode 100644 index 0000000000..c995a2e899 --- /dev/null +++ b/tests/cpydiff/types_memoryview_invalid.py @@ -0,0 +1,12 @@ +""" +categories: Types,memoryview +description: memoryview can become invalid if its target is resized +cause: CPython prevents a ``bytearray`` or ``io.bytesIO`` object from changing size while there is a ``memoryview`` object that references it. MicroPython requires the programmer to manually ensure that an object is not resized while any ``memoryview`` references it. + +In the worst case scenario, resizing an object which is the target of a memoryview can cause the memoryview(s) to reference invalid freed memory (a use-after-free bug) and corrupt the MicroPython runtime. +workaround: Do not change the size of any ``bytearray`` or ``io.bytesIO`` object that has a ``memoryview`` assigned to it. +""" +b = bytearray(b"abcdefg") +m = memoryview(b) +b.extend(b"hijklmnop") +print(b, bytes(m)) From 49ce7a607517dc0152a4f8768f5d03b3e5613e02 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 22 Apr 2024 12:35:31 +1000 Subject: [PATCH 08/18] github/workflows: Run code size workflow on shared or port code changes. To get more insight to firmware size changes when code changes. Signed-off-by: Damien George --- .github/workflows/code_size.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/code_size.yml b/.github/workflows/code_size.yml index a65a00201a..1ef43d1fd3 100644 --- a/.github/workflows/code_size.yml +++ b/.github/workflows/code_size.yml @@ -8,9 +8,15 @@ on: - 'tools/**' - 'py/**' - 'extmod/**' + - 'shared/**' - 'lib/**' - 'ports/bare-arm/**' + - 'ports/mimxrt/**' - 'ports/minimal/**' + - 'ports/rp2/**' + - 'ports/samd/**' + - 'ports/stm32/**' + - 'ports/unix/**' concurrency: group: ${{ github.workflow }}-${{ github.ref }} From 45848f77cac07dac1c77680041862ba444c1f3a3 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 24 Apr 2024 13:00:20 +1000 Subject: [PATCH 09/18] webassembly/api: Fix waiting for Emscripten module to be loaded. In modularize mode, the `_createMicroPythonModule()` constructor must be await'ed on, before `Module` is ready to use. Signed-off-by: Damien George --- ports/webassembly/api.js | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/ports/webassembly/api.js b/ports/webassembly/api.js index 7d1832af4f..2459075fc5 100644 --- a/ports/webassembly/api.js +++ b/ports/webassembly/api.js @@ -38,7 +38,7 @@ export async function loadMicroPython(options) { { heapsize: 1024 * 1024, linebuffer: true }, options, ); - const Module = {}; + let Module = {}; Module.locateFile = (path, scriptDirectory) => url || scriptDirectory + path; Module._textDecoder = new TextDecoder(); @@ -83,11 +83,7 @@ export async function loadMicroPython(options) { Module.stderr = (c) => stderr(new Uint8Array([c])); } } - const moduleLoaded = new Promise((r) => { - Module.postRun = r; - }); - _createMicroPythonModule(Module); - await moduleLoaded; + Module = await _createMicroPythonModule(Module); globalThis.Module = Module; proxy_js_init(); const pyimport = (name) => { From 9c7f0659e2a6db37125fa092ebe5150125eb14b3 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 17 Apr 2024 14:25:48 +1000 Subject: [PATCH 10/18] webassembly/api: Allocate code data on C heap when running Python code. Otherwise Emscripten allocates it on the Emscripten C stack, which will overflow for large amounts of code. Fixes issue #14307. Signed-off-by: Damien George --- ports/webassembly/api.js | 16 ++++++++++++---- ports/webassembly/main.c | 8 ++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/ports/webassembly/api.js b/ports/webassembly/api.js index 2459075fc5..2f2ad95221 100644 --- a/ports/webassembly/api.js +++ b/ports/webassembly/api.js @@ -127,23 +127,31 @@ export async function loadMicroPython(options) { }, pyimport: pyimport, runPython(code) { + const len = Module.lengthBytesUTF8(code); + const buf = Module._malloc(len + 1); + Module.stringToUTF8(code, buf, len + 1); const value = Module._malloc(3 * 4); Module.ccall( "mp_js_do_exec", "number", - ["string", "pointer"], - [code, value], + ["pointer", "number", "pointer"], + [buf, len, value], ); + Module._free(buf); return proxy_convert_mp_to_js_obj_jsside_with_free(value); }, runPythonAsync(code) { + const len = Module.lengthBytesUTF8(code); + const buf = Module._malloc(len + 1); + Module.stringToUTF8(code, buf, len + 1); const value = Module._malloc(3 * 4); Module.ccall( "mp_js_do_exec_async", "number", - ["string", "pointer"], - [code, value], + ["pointer", "number", "pointer"], + [buf, len, value], ); + Module._free(buf); return proxy_convert_mp_to_js_obj_jsside_with_free(value); }, replInit() { diff --git a/ports/webassembly/main.c b/ports/webassembly/main.c index 441c6d67e3..23d12b6dbb 100644 --- a/ports/webassembly/main.c +++ b/ports/webassembly/main.c @@ -104,7 +104,7 @@ void mp_js_do_import(const char *name, uint32_t *out) { } } -void mp_js_do_exec(const char *src, uint32_t *out) { +void mp_js_do_exec(const char *src, size_t len, uint32_t *out) { // Collect at the top-level, where there are no root pointers from stack/registers. gc_collect_start(); gc_collect_end(); @@ -112,7 +112,7 @@ void mp_js_do_exec(const char *src, uint32_t *out) { mp_parse_input_kind_t input_kind = MP_PARSE_FILE_INPUT; nlr_buf_t nlr; if (nlr_push(&nlr) == 0) { - mp_lexer_t *lex = mp_lexer_new_from_str_len_dedent(MP_QSTR__lt_stdin_gt_, src, strlen(src), 0); + mp_lexer_t *lex = mp_lexer_new_from_str_len_dedent(MP_QSTR__lt_stdin_gt_, src, len, 0); qstr source_name = lex->source_name; mp_parse_tree_t parse_tree = mp_parse(lex, input_kind); mp_obj_t module_fun = mp_compile(&parse_tree, source_name, false); @@ -125,9 +125,9 @@ void mp_js_do_exec(const char *src, uint32_t *out) { } } -void mp_js_do_exec_async(const char *src, uint32_t *out) { +void mp_js_do_exec_async(const char *src, size_t len, uint32_t *out) { mp_compile_allow_top_level_await = true; - mp_js_do_exec(src, out); + mp_js_do_exec(src, len, out); mp_compile_allow_top_level_await = false; } From 4c3f5f552b15f74984e906db799812df6b3e9555 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 18 Apr 2024 17:19:56 +1000 Subject: [PATCH 11/18] webassembly/objjsproxy: Fix handling of thrown value into JS generator. Signed-off-by: Damien George --- ports/webassembly/objjsproxy.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ports/webassembly/objjsproxy.c b/ports/webassembly/objjsproxy.c index 098f4e75f5..65c806536d 100644 --- a/ports/webassembly/objjsproxy.c +++ b/ports/webassembly/objjsproxy.c @@ -346,6 +346,12 @@ typedef struct _jsproxy_gen_t { mp_vm_return_kind_t jsproxy_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_obj_t throw_value, mp_obj_t *ret_val) { jsproxy_gen_t *self = MP_OBJ_TO_PTR(self_in); + + if (throw_value) { + *ret_val = throw_value; + return MP_VM_RETURN_EXCEPTION; + } + switch (self->state) { case JSOBJ_GEN_STATE_WAITING: self->state = JSOBJ_GEN_STATE_COMPLETED; From 92b3b69648304b292fb59e734fc9a61a188ba419 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 18 Apr 2024 17:20:28 +1000 Subject: [PATCH 12/18] webassembly/proxy_c: Fix proxy then reject handling. An exception on the Python side should be passed to the Promise reject callback on the JavaScript side. Signed-off-by: Damien George --- ports/webassembly/proxy_c.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/ports/webassembly/proxy_c.c b/ports/webassembly/proxy_c.c index 8d137f6271..fc863c06be 100644 --- a/ports/webassembly/proxy_c.c +++ b/ports/webassembly/proxy_c.c @@ -296,10 +296,11 @@ EM_JS(void, js_then_resolve, (uint32_t * ret_value, uint32_t * resolve, uint32_t resolve_js(ret_value_js); }); -EM_JS(void, js_then_reject, (uint32_t * resolve, uint32_t * reject), { +EM_JS(void, js_then_reject, (uint32_t * ret_value, uint32_t * resolve, uint32_t * reject), { + const ret_value_js = proxy_convert_mp_to_js_obj_jsside(ret_value); const resolve_js = proxy_convert_mp_to_js_obj_jsside(resolve); const reject_js = proxy_convert_mp_to_js_obj_jsside(reject); - reject_js(null); + reject_js(ret_value_js); }); // *FORMAT-OFF* @@ -335,10 +336,12 @@ static mp_obj_t proxy_resume_execute(mp_obj_t self_in, mp_obj_t value, mp_obj_t uint32_t out[PVN]; js_then_continue(ref, out_py_resume, out_resolve, out_reject, out); return proxy_convert_js_to_mp_obj_cside(out); - } else { - // MP_VM_RETURN_EXCEPTION; - js_then_reject(out_resolve, out_reject); - nlr_raise(ret_value); + } else { // ret_kind == MP_VM_RETURN_EXCEPTION; + // Pass the exception through as an object to reject the promise (don't raise/throw it). + uint32_t out_ret_value[PVN]; + proxy_convert_mp_to_js_obj_cside(ret_value, out_ret_value); + js_then_reject(out_ret_value, out_resolve, out_reject); + return mp_const_none; } } From d998ca78c8abb975ba00a658544a1b72575f9732 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 18 Apr 2024 17:20:57 +1000 Subject: [PATCH 13/18] webassembly/proxy_c: Fix then-continue to convert reason to throw value. When a Promise is rejected on the JavaScript side, the reject reason should be thrown into the encapsulating generator on the Python side. Signed-off-by: Damien George --- ports/webassembly/proxy_c.c | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/ports/webassembly/proxy_c.c b/ports/webassembly/proxy_c.c index fc863c06be..ebf670b56e 100644 --- a/ports/webassembly/proxy_c.c +++ b/ports/webassembly/proxy_c.c @@ -308,14 +308,34 @@ EM_JS(void, js_then_continue, (int jsref, uint32_t * py_resume, uint32_t * resol const py_resume_js = proxy_convert_mp_to_js_obj_jsside(py_resume); const resolve_js = proxy_convert_mp_to_js_obj_jsside(resolve); const reject_js = proxy_convert_mp_to_js_obj_jsside(reject); - const ret = proxy_js_ref[jsref].then((x) => {py_resume_js(x, resolve_js, reject_js);}, reject_js); + const ret = proxy_js_ref[jsref].then( + (result) => { + // The Promise is fulfilled on the JavaScript side. Take the result and + // send it to the encapsulating generator on the Python side, so it + // becomes the result of the "yield from" that deferred to this Promise. + py_resume_js(result, null, resolve_js, reject_js); + }, + (reason) => { + // The Promise is rejected on the JavaScript side. Take the reason and + // throw it into the encapsulating generator on the Python side. + py_resume_js(null, reason, resolve_js, reject_js); + }, + ); proxy_convert_js_to_mp_obj_jsside(ret, out); }); // *FORMAT-ON* -static mp_obj_t proxy_resume_execute(mp_obj_t self_in, mp_obj_t value, mp_obj_t resolve, mp_obj_t reject) { +static mp_obj_t proxy_resume_execute(mp_obj_t self_in, mp_obj_t send_value, mp_obj_t throw_value, mp_obj_t resolve, mp_obj_t reject) { + if (throw_value != MP_OBJ_NULL && throw_value != mp_const_none) { + if (send_value == mp_const_none) { + send_value = MP_OBJ_NULL; + } + } else { + throw_value = MP_OBJ_NULL; + } + mp_obj_t ret_value; - mp_vm_return_kind_t ret_kind = mp_resume(self_in, value, MP_OBJ_NULL, &ret_value); + mp_vm_return_kind_t ret_kind = mp_resume(self_in, send_value, throw_value, &ret_value); uint32_t out_resolve[PVN]; uint32_t out_reject[PVN]; @@ -346,9 +366,9 @@ static mp_obj_t proxy_resume_execute(mp_obj_t self_in, mp_obj_t value, mp_obj_t } static mp_obj_t resume_fun(size_t n_args, const mp_obj_t *args) { - return proxy_resume_execute(args[0], args[1], args[2], args[3]); + return proxy_resume_execute(args[0], args[1], args[2], args[3], args[4]); } -static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(resume_obj, 4, 4, resume_fun); +static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(resume_obj, 5, 5, resume_fun); void proxy_c_to_js_resume(uint32_t c_ref, uint32_t *args) { nlr_buf_t nlr; @@ -356,7 +376,7 @@ void proxy_c_to_js_resume(uint32_t c_ref, uint32_t *args) { mp_obj_t obj = proxy_c_get_obj(c_ref); mp_obj_t resolve = proxy_convert_js_to_mp_obj_cside(args + 1 * 3); mp_obj_t reject = proxy_convert_js_to_mp_obj_cside(args + 2 * 3); - mp_obj_t ret = proxy_resume_execute(obj, mp_const_none, resolve, reject); + mp_obj_t ret = proxy_resume_execute(obj, mp_const_none, mp_const_none, resolve, reject); nlr_pop(); return proxy_convert_mp_to_js_obj_cside(ret, args); } else { From 967ad38ac7e09270e9f5734c59baebf2d1eb3723 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 19 Apr 2024 12:14:49 +1000 Subject: [PATCH 14/18] extmod/modasyncio: Make mp_asyncio_context variable public. So it can be accessed by a port if needed, for example to see if asyncio has been imported. Signed-off-by: Damien George --- extmod/modasyncio.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/extmod/modasyncio.c b/extmod/modasyncio.c index 61dd707223..4f82370d23 100644 --- a/extmod/modasyncio.c +++ b/extmod/modasyncio.c @@ -156,7 +156,7 @@ static MP_DEFINE_CONST_OBJ_TYPE( // Task class // This is the core asyncio context with cur_task, _task_queue and CancelledError. -static mp_obj_t asyncio_context = MP_OBJ_NULL; +mp_obj_t mp_asyncio_context = MP_OBJ_NULL; static mp_obj_t task_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { mp_arg_check_num(n_args, n_kw, 1, 2, false); @@ -168,7 +168,7 @@ static mp_obj_t task_make_new(const mp_obj_type_t *type, size_t n_args, size_t n self->state = TASK_STATE_RUNNING_NOT_WAITED_ON; self->ph_key = MP_OBJ_NEW_SMALL_INT(0); if (n_args == 2) { - asyncio_context = args[1]; + mp_asyncio_context = args[1]; } return MP_OBJ_FROM_PTR(self); } @@ -186,7 +186,7 @@ static mp_obj_t task_cancel(mp_obj_t self_in) { return mp_const_false; } // Can't cancel self (not supported yet). - mp_obj_t cur_task = mp_obj_dict_get(asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_cur_task)); + mp_obj_t cur_task = mp_obj_dict_get(mp_asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_cur_task)); if (self_in == cur_task) { mp_raise_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("can't cancel self")); } @@ -195,7 +195,7 @@ static mp_obj_t task_cancel(mp_obj_t self_in) { self = MP_OBJ_TO_PTR(self->data); } - mp_obj_t _task_queue = mp_obj_dict_get(asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR__task_queue)); + mp_obj_t _task_queue = mp_obj_dict_get(mp_asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR__task_queue)); // Reschedule Task as a cancelled task. mp_obj_t dest[3]; @@ -218,7 +218,7 @@ static mp_obj_t task_cancel(mp_obj_t self_in) { task_queue_push(2, dest); } - self->data = mp_obj_dict_get(asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_CancelledError)); + self->data = mp_obj_dict_get(mp_asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_CancelledError)); return mp_const_true; } @@ -278,7 +278,7 @@ static mp_obj_t task_iternext(mp_obj_t self_in) { nlr_raise(self->data); } else { // Put calling task on waiting queue. - mp_obj_t cur_task = mp_obj_dict_get(asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_cur_task)); + mp_obj_t cur_task = mp_obj_dict_get(mp_asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_cur_task)); mp_obj_t args[2] = { self->state, cur_task }; task_queue_push(2, args); // Set calling task's data to this task that it waits on, to double-link it. From 84d6f8e8cb993e82f03e209ffd49d5c44fc780e0 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 19 Apr 2024 12:16:27 +1000 Subject: [PATCH 15/18] webassembly/modjsffi: Add jsffi.async_timeout_ms. This function exposes `setTimeout()` as an async function. Signed-off-by: Damien George --- ports/webassembly/modjsffi.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/ports/webassembly/modjsffi.c b/ports/webassembly/modjsffi.c index d4e61e368f..cb2f578f8f 100644 --- a/ports/webassembly/modjsffi.c +++ b/ports/webassembly/modjsffi.c @@ -61,12 +61,27 @@ static mp_obj_t mp_jsffi_to_js(mp_obj_t arg) { } static MP_DEFINE_CONST_FUN_OBJ_1(mp_jsffi_to_js_obj, mp_jsffi_to_js); +// *FORMAT-OFF* +EM_JS(void, promise_with_timeout_ms, (double ms, uint32_t * out), { + const ret = new Promise((resolve) => setTimeout(resolve, ms)); + proxy_convert_js_to_mp_obj_jsside(ret, out); +}); +// *FORMAT-ON* + +static mp_obj_t mp_jsffi_async_timeout_ms(mp_obj_t arg) { + uint32_t out[PVN]; + promise_with_timeout_ms(mp_obj_get_float_to_d(arg), out); + return proxy_convert_js_to_mp_obj_cside(out); +} +static MP_DEFINE_CONST_FUN_OBJ_1(mp_jsffi_async_timeout_ms_obj, mp_jsffi_async_timeout_ms); + static const mp_rom_map_elem_t mp_module_jsffi_globals_table[] = { { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_jsffi) }, { MP_ROM_QSTR(MP_QSTR_JsProxy), MP_ROM_PTR(&mp_type_jsproxy) }, { MP_ROM_QSTR(MP_QSTR_create_proxy), MP_ROM_PTR(&mp_jsffi_create_proxy_obj) }, { MP_ROM_QSTR(MP_QSTR_to_js), MP_ROM_PTR(&mp_jsffi_to_js_obj) }, + { MP_ROM_QSTR(MP_QSTR_async_timeout_ms), MP_ROM_PTR(&mp_jsffi_async_timeout_ms_obj) }, }; static MP_DEFINE_CONST_DICT(mp_module_jsffi_globals, mp_module_jsffi_globals_table); From 8a3546b3bd71dbc6d79900afbe58767e09b82c3e Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 19 Apr 2024 12:17:11 +1000 Subject: [PATCH 16/18] webassembly: Add JavaScript-based asyncio support. This commit adds a significant portion of the existing MicroPython asyncio module to the webassembly port, using parts of the existing asyncio code and some custom JavaScript parts. The key difference to the standard asyncio is that this version uses the JavaScript runtime to do the actual scheduling and waiting on events, eg Promise fulfillment, timeouts, fetching URLs. This implementation does not include asyncio.run(). Instead one just uses asyncio.create_task(..) to start tasks and then returns to the JavaScript. Then JavaScript will run the tasks. The implementation here tries to reuse as much existing asyncio code as possible, and gets all the semantics correct for things like cancellation and asyncio.wait_for. An alternative approach would reimplement Task, Event, etc using JavaScript Promise's. That approach is very difficult to get right when trying to implement cancellation (because it's not possible to cancel a JavaScript Promise). Signed-off-by: Damien George --- ports/webassembly/Makefile | 3 + ports/webassembly/asyncio/__init__.py | 9 + ports/webassembly/asyncio/core.py | 249 ++++++++++++++++++ ports/webassembly/objjsproxy.c | 20 ++ ports/webassembly/variants/manifest.py | 24 ++ .../webassembly/variants/pyscript/manifest.py | 2 + .../ports/webassembly/asyncio_create_task.mjs | 44 ++++ .../webassembly/asyncio_create_task.mjs.exp | 14 + tests/ports/webassembly/asyncio_sleep.mjs | 25 ++ tests/ports/webassembly/asyncio_sleep.mjs.exp | 6 + tests/run-tests.py | 11 + 11 files changed, 407 insertions(+) create mode 100644 ports/webassembly/asyncio/__init__.py create mode 100644 ports/webassembly/asyncio/core.py create mode 100644 ports/webassembly/variants/manifest.py create mode 100644 tests/ports/webassembly/asyncio_create_task.mjs create mode 100644 tests/ports/webassembly/asyncio_create_task.mjs.exp create mode 100644 tests/ports/webassembly/asyncio_sleep.mjs create mode 100644 tests/ports/webassembly/asyncio_sleep.mjs.exp diff --git a/ports/webassembly/Makefile b/ports/webassembly/Makefile index d0a8aa9924..93b92ef583 100644 --- a/ports/webassembly/Makefile +++ b/ports/webassembly/Makefile @@ -22,6 +22,9 @@ BUILD ?= build-$(VARIANT) include ../../py/mkenv.mk include $(VARIANT_DIR)/mpconfigvariant.mk +# Use the default frozen manifest, variants may override this. +FROZEN_MANIFEST ?= variants/manifest.py + # Qstr definitions (must come before including py.mk). QSTR_DEFS = qstrdefsport.h diff --git a/ports/webassembly/asyncio/__init__.py b/ports/webassembly/asyncio/__init__.py new file mode 100644 index 0000000000..ba1ca63514 --- /dev/null +++ b/ports/webassembly/asyncio/__init__.py @@ -0,0 +1,9 @@ +# MicroPython asyncio module, for use with webassembly port +# MIT license; Copyright (c) 2024 Damien P. George + +from .core import * +from .funcs import wait_for, wait_for_ms, gather +from .event import Event +from .lock import Lock + +__version__ = (3, 0, 0) diff --git a/ports/webassembly/asyncio/core.py b/ports/webassembly/asyncio/core.py new file mode 100644 index 0000000000..a128bb6055 --- /dev/null +++ b/ports/webassembly/asyncio/core.py @@ -0,0 +1,249 @@ +# MicroPython asyncio module, for use with webassembly port +# MIT license; Copyright (c) 2019-2024 Damien P. George + +from time import ticks_ms as ticks, ticks_diff, ticks_add +import sys, js, jsffi + +# Import TaskQueue and Task from built-in C code. +from _asyncio import TaskQueue, Task + + +################################################################################ +# Exceptions + + +class CancelledError(BaseException): + pass + + +class TimeoutError(Exception): + pass + + +# Used when calling Loop.call_exception_handler. +_exc_context = {"message": "Task exception wasn't retrieved", "exception": None, "future": None} + + +################################################################################ +# Sleep functions + + +# "Yield" once, then raise StopIteration +class SingletonGenerator: + def __init__(self): + self.state = None + self.exc = StopIteration() + + def __iter__(self): + return self + + def __next__(self): + if self.state is not None: + _task_queue.push(cur_task, self.state) + self.state = None + return None + else: + self.exc.__traceback__ = None + raise self.exc + + +# Pause task execution for the given time (integer in milliseconds, uPy extension) +# Use a SingletonGenerator to do it without allocating on the heap +def sleep_ms(t, sgen=SingletonGenerator()): + if cur_task is None: + # Support top-level asyncio.sleep, via a JavaScript Promise. + return jsffi.async_timeout_ms(t) + assert sgen.state is None + sgen.state = ticks_add(ticks(), max(0, t)) + return sgen + + +# Pause task execution for the given time (in seconds) +def sleep(t): + return sleep_ms(int(t * 1000)) + + +################################################################################ +# Main run loop + +asyncio_timer = None + + +class ThenableEvent: + def __init__(self, thenable): + self.result = None # Result of the thenable + self.waiting = None # Task waiting on completion of this thenable + thenable.then(self.set) + + def set(self, value): + # Thenable/Promise is fulfilled, set result and schedule any waiting task. + self.result = value + if self.waiting: + _task_queue.push(self.waiting) + self.waiting = None + _schedule_run_iter(0) + + def remove(self, task): + self.waiting = None + + # async + def wait(self): + # Set the calling task as the task waiting on this thenable. + self.waiting = cur_task + # Set calling task's data to this object so it can be removed if needed. + cur_task.data = self + # Wait for the thenable to fulfill. + yield + # Return the result of the thenable. + return self.result + + +# Ensure the awaitable is a task +def _promote_to_task(aw): + return aw if isinstance(aw, Task) else create_task(aw) + + +def _schedule_run_iter(dt): + global asyncio_timer + if asyncio_timer is not None: + js.clearTimeout(asyncio_timer) + asyncio_timer = js.setTimeout(_run_iter, dt) + + +def _run_iter(): + global cur_task + excs_all = (CancelledError, Exception) # To prevent heap allocation in loop + excs_stop = (CancelledError, StopIteration) # To prevent heap allocation in loop + while True: + # Wait until the head of _task_queue is ready to run + t = _task_queue.peek() + if t: + # A task waiting on _task_queue; "ph_key" is time to schedule task at + dt = max(0, ticks_diff(t.ph_key, ticks())) + else: + # No tasks can be woken so finished running + cur_task = None + return + + if dt > 0: + # schedule to call again later + cur_task = None + _schedule_run_iter(dt) + return + + # Get next task to run and continue it + t = _task_queue.pop() + cur_task = t + try: + # Continue running the coroutine, it's responsible for rescheduling itself + exc = t.data + if not exc: + t.coro.send(None) + else: + # If the task is finished and on the run queue and gets here, then it + # had an exception and was not await'ed on. Throwing into it now will + # raise StopIteration and the code below will catch this and run the + # call_exception_handler function. + t.data = None + t.coro.throw(exc) + except excs_all as er: + # Check the task is not on any event queue + assert t.data is None + # This task is done. + if t.state: + # Task was running but is now finished. + waiting = False + if t.state is True: + # "None" indicates that the task is complete and not await'ed on (yet). + t.state = None + elif callable(t.state): + # The task has a callback registered to be called on completion. + t.state(t, er) + t.state = False + waiting = True + else: + # Schedule any other tasks waiting on the completion of this task. + while t.state.peek(): + _task_queue.push(t.state.pop()) + waiting = True + # "False" indicates that the task is complete and has been await'ed on. + t.state = False + if not waiting and not isinstance(er, excs_stop): + # An exception ended this detached task, so queue it for later + # execution to handle the uncaught exception if no other task retrieves + # the exception in the meantime (this is handled by Task.throw). + _task_queue.push(t) + # Save return value of coro to pass up to caller. + t.data = er + elif t.state is None: + # Task is already finished and nothing await'ed on the task, + # so call the exception handler. + + # Save exception raised by the coro for later use. + t.data = exc + + # Create exception context and call the exception handler. + _exc_context["exception"] = exc + _exc_context["future"] = t + Loop.call_exception_handler(_exc_context) + + +# Create and schedule a new task from a coroutine. +def create_task(coro): + if not hasattr(coro, "send"): + raise TypeError("coroutine expected") + t = Task(coro, globals()) + _task_queue.push(t) + _schedule_run_iter(0) + return t + + +################################################################################ +# Event loop wrapper + + +cur_task = None + + +class Loop: + _exc_handler = None + + def create_task(coro): + return create_task(coro) + + def close(): + pass + + def set_exception_handler(handler): + Loop._exc_handler = handler + + def get_exception_handler(): + return Loop._exc_handler + + def default_exception_handler(loop, context): + print(context["message"], file=sys.stderr) + print("future:", context["future"], "coro=", context["future"].coro, file=sys.stderr) + sys.print_exception(context["exception"], sys.stderr) + + def call_exception_handler(context): + (Loop._exc_handler or Loop.default_exception_handler)(Loop, context) + + +def get_event_loop(): + return Loop + + +def current_task(): + if cur_task is None: + raise RuntimeError("no running event loop") + return cur_task + + +def new_event_loop(): + global _task_queue + _task_queue = TaskQueue() # TaskQueue of Task instances. + return Loop + + +# Initialise default event loop. +new_event_loop() diff --git a/ports/webassembly/objjsproxy.c b/ports/webassembly/objjsproxy.c index 65c806536d..15fbb57523 100644 --- a/ports/webassembly/objjsproxy.c +++ b/ports/webassembly/objjsproxy.c @@ -474,9 +474,29 @@ static mp_obj_t jsproxy_new_gen(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf) { /******************************************************************************/ +#if MICROPY_PY_ASYNCIO +extern mp_obj_t mp_asyncio_context; +#endif + static mp_obj_t jsproxy_getiter(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf) { mp_obj_jsproxy_t *self = MP_OBJ_TO_PTR(self_in); if (has_attr(self->ref, "then")) { + #if MICROPY_PY_ASYNCIO + // When asyncio is running and the caller here is a task, wrap the JavaScript + // thenable in a ThenableEvent, and get the task to wait on that event. This + // decouples the task from the thenable and allows cancelling the task. + if (mp_asyncio_context != MP_OBJ_NULL) { + mp_obj_t cur_task = mp_obj_dict_get(mp_asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_cur_task)); + if (cur_task != mp_const_none) { + mp_obj_t thenable_event_class = mp_obj_dict_get(mp_asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_ThenableEvent)); + mp_obj_t thenable_event = mp_call_function_1(thenable_event_class, self_in); + mp_obj_t dest[2]; + mp_load_method(thenable_event, MP_QSTR_wait, dest); + mp_obj_t wait_gen = mp_call_method_n_kw(0, 0, dest); + return mp_getiter(wait_gen, iter_buf); + } + } + #endif return jsproxy_new_gen(self_in, iter_buf); } else { return jsproxy_new_it(self_in, iter_buf); diff --git a/ports/webassembly/variants/manifest.py b/ports/webassembly/variants/manifest.py new file mode 100644 index 0000000000..b2ee8cd640 --- /dev/null +++ b/ports/webassembly/variants/manifest.py @@ -0,0 +1,24 @@ +# The asyncio package is built from the standard implementation but with the +# core scheduler replaced with a custom scheduler that uses the JavaScript +# runtime (with setTimeout an Promise's) to contrtol the scheduling. + +package( + "asyncio", + ( + "event.py", + "funcs.py", + "lock.py", + ), + base_path="$(MPY_DIR)/extmod", + opt=3, +) + +package( + "asyncio", + ( + "__init__.py", + "core.py", + ), + base_path="$(PORT_DIR)", + opt=3, +) diff --git a/ports/webassembly/variants/pyscript/manifest.py b/ports/webassembly/variants/pyscript/manifest.py index 0646e1d897..db088e70d0 100644 --- a/ports/webassembly/variants/pyscript/manifest.py +++ b/ports/webassembly/variants/pyscript/manifest.py @@ -1,3 +1,5 @@ +include("$(PORT_DIR)/variants/manifest.py") + require("abc") require("base64") require("collections") diff --git a/tests/ports/webassembly/asyncio_create_task.mjs b/tests/ports/webassembly/asyncio_create_task.mjs new file mode 100644 index 0000000000..e388ade4f1 --- /dev/null +++ b/tests/ports/webassembly/asyncio_create_task.mjs @@ -0,0 +1,44 @@ +// Test asyncio.create_task(), and tasks waiting on a Promise. + +const mp = await (await import(process.argv[2])).loadMicroPython(); + +globalThis.p0 = new Promise((resolve, reject) => { + resolve(123); +}); + +globalThis.p1 = new Promise((resolve, reject) => { + setTimeout(() => { + console.log("setTimeout resolved"); + resolve(456); + }, 200); +}); + +mp.runPython(` +import js +import asyncio + +async def task(id, promise): + print("task start", id) + print("task await", id, await promise) + print("task await", id, await promise) + print("task end", id) + +print("start") +t1 = asyncio.create_task(task(1, js.p0)) +t2 = asyncio.create_task(task(2, js.p1)) +print("t1", t1.done(), t2.done()) +print("end") +`); + +// Wait for p1 to fulfill so t2 can continue. +await globalThis.p1; + +// Wait a little longer so t2 can complete. +await new Promise((resolve, reject) => { + setTimeout(resolve, 10); +}); + +mp.runPython(` +print("restart") +print("t1", t1.done(), t2.done()) +`); diff --git a/tests/ports/webassembly/asyncio_create_task.mjs.exp b/tests/ports/webassembly/asyncio_create_task.mjs.exp new file mode 100644 index 0000000000..c1958bba59 --- /dev/null +++ b/tests/ports/webassembly/asyncio_create_task.mjs.exp @@ -0,0 +1,14 @@ +start +t1 False False +end +task start 1 +task start 2 +task await 1 123 +task await 1 123 +task end 1 +setTimeout resolved +task await 2 456 +task await 2 456 +task end 2 +restart +t1 True True diff --git a/tests/ports/webassembly/asyncio_sleep.mjs b/tests/ports/webassembly/asyncio_sleep.mjs new file mode 100644 index 0000000000..74d22ee1f8 --- /dev/null +++ b/tests/ports/webassembly/asyncio_sleep.mjs @@ -0,0 +1,25 @@ +// Test asyncio.sleep(), both at the top level and within a task. + +const mp = await (await import(process.argv[2])).loadMicroPython(); + +await mp.runPythonAsync(` +import time +import asyncio + +print("main start") +t0 = time.time() +await asyncio.sleep(0.25) +dt = time.time() - t0 +print(0.2 <= dt <= 0.3) + +async def task(): + print("task start") + t0 = time.time() + await asyncio.sleep(0.25) + dt = time.time() - t0 + print(0.2 <= dt <= 0.3) + print("task end") + +asyncio.create_task(task()) +print("main end") +`); diff --git a/tests/ports/webassembly/asyncio_sleep.mjs.exp b/tests/ports/webassembly/asyncio_sleep.mjs.exp new file mode 100644 index 0000000000..619ba175f6 --- /dev/null +++ b/tests/ports/webassembly/asyncio_sleep.mjs.exp @@ -0,0 +1,6 @@ +main start +True +main end +task start +True +task end diff --git a/tests/run-tests.py b/tests/run-tests.py index 4f55cdd398..8acdcd2b36 100755 --- a/tests/run-tests.py +++ b/tests/run-tests.py @@ -681,6 +681,17 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): elif args.target == "webassembly": skip_tests.add("basics/string_format_modulo.py") # can't print nulls to stdout skip_tests.add("basics/string_strip.py") # can't print nulls to stdout + skip_tests.add("extmod/asyncio_basic2.py") + skip_tests.add("extmod/asyncio_cancel_self.py") + skip_tests.add("extmod/asyncio_current_task.py") + skip_tests.add("extmod/asyncio_exception.py") + skip_tests.add("extmod/asyncio_gather_finished_early.py") + skip_tests.add("extmod/asyncio_get_event_loop.py") + skip_tests.add("extmod/asyncio_heaplock.py") + skip_tests.add("extmod/asyncio_loop_stop.py") + skip_tests.add("extmod/asyncio_new_event_loop.py") + skip_tests.add("extmod/asyncio_threadsafeflag.py") + skip_tests.add("extmod/asyncio_wait_for_fwd.py") skip_tests.add("extmod/binascii_a2b_base64.py") skip_tests.add("extmod/re_stack_overflow.py") skip_tests.add("extmod/time_res.py") From 49af8cad49415521813bda6d756dbe50a125f472 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 18 Apr 2024 16:36:34 +1000 Subject: [PATCH 17/18] webassembly/api: Inject asyncio.run if needed by the script. This allows a simple way to run the existing asyncio tests under the webassembly port, which doesn't support `asyncio.run()`. Signed-off-by: Damien George --- ports/webassembly/api.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ports/webassembly/api.js b/ports/webassembly/api.js index 2f2ad95221..2be82e8907 100644 --- a/ports/webassembly/api.js +++ b/ports/webassembly/api.js @@ -228,6 +228,16 @@ async function runCLI() { } }); } else { + // If the script to run ends with a running of the asyncio main loop, then inject + // a simple `asyncio.run` hook that starts the main task. This is primarily to + // support running the standard asyncio tests. + if (contents.endsWith("asyncio.run(main())\n")) { + const asyncio = mp.pyimport("asyncio"); + asyncio.run = async (task) => { + await asyncio.create_task(task); + }; + } + try { mp.runPython(contents); } catch (error) { From 7b050b366b7dacfb43779c51702a892d8f1873d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Neusch=C3=A4fer?= Date: Sun, 2 Apr 2023 19:58:42 +0200 Subject: [PATCH 18/18] py/nlrthumb: Make non-Thumb2 long-jump workaround opt-in. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Although the original motivation given for the workaround[1] is correct, nlr.o and nlrthumb.o are linked with a small enough distance that the problem does not occur, and the workaround isn't necessary. The distance between the b instruction and its target (nlr_push_tail) is just 64 bytes[2], well within the ±2046 byte range addressable by an unconditional branch instruction in Thumb mode. The workaround induces a relocation in the text section (textrel), which isn't supported everywhere, notably not on musl-libc[3], where it causes a crash on start-up. With the workaround removed, micropython works on an ARMv5T Linux system built with musl-libc. This commit changes nlrthumb.c to use a direct jump by default, but leaves the long jump workaround as an option for those cases where it's actually needed. [1]: commit dd376a239dc4f47b0ee7866810fcda151f3cf6dd Author: Damien George Date: Fri Sep 1 15:25:29 2017 +1000 py/nlrthumb: Get working again on standard Thumb arch (ie not Thumb2). "b" on Thumb might not be long enough for the jump to nlr_push_tail so it must be done indirectly. [2]: Excerpt from objdump -d micropython: 000095c4 : 95c4: b510 push {r4, lr} 95c6: 0004 movs r4, r0 95c8: f02d fd42 bl 37050 95cc: 6943 ldr r3, [r0, #20] 95ce: 6023 str r3, [r4, #0] 95d0: 6144 str r4, [r0, #20] 95d2: 2000 movs r0, #0 95d4: bd10 pop {r4, pc} 000095d6 : 95d6: b510 push {r4, lr} 95d8: f02d fd3a bl 37050 95dc: 6943 ldr r3, [r0, #20] 95de: 681b ldr r3, [r3, #0] 95e0: 6143 str r3, [r0, #20] 95e2: bd10 pop {r4, pc} 000095e4 : 95e4: 60c4 str r4, [r0, #12] 95e6: 6105 str r5, [r0, #16] 95e8: 6146 str r6, [r0, #20] 95ea: 6187 str r7, [r0, #24] 95ec: 4641 mov r1, r8 95ee: 61c1 str r1, [r0, #28] 95f0: 4649 mov r1, r9 95f2: 6201 str r1, [r0, #32] 95f4: 4651 mov r1, sl 95f6: 6241 str r1, [r0, #36] @ 0x24 95f8: 4659 mov r1, fp 95fa: 6281 str r1, [r0, #40] @ 0x28 95fc: 4669 mov r1, sp 95fe: 62c1 str r1, [r0, #44] @ 0x2c 9600: 4671 mov r1, lr 9602: 6081 str r1, [r0, #8] 9604: e7de b.n 95c4 [3]: https://www.openwall.com/lists/musl/2020/09/25/4 Signed-off-by: J. Neuschäfer --- py/mpconfig.h | 6 ++++++ py/nlrthumb.c | 10 +++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/py/mpconfig.h b/py/mpconfig.h index af2480266b..1aa3e06994 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -587,6 +587,12 @@ /*****************************************************************************/ /* Python internal features */ +// Use a special long jump in nlrthumb.c, which may be necessary if nlr.o and +// nlrthumb.o are linked far apart from each other. +#ifndef MICROPY_NLR_THUMB_USE_LONG_JUMP +#define MICROPY_NLR_THUMB_USE_LONG_JUMP (0) +#endif + // Whether to enable import of external modules // When disabled, only importing of built-in modules is supported // When enabled, a port must implement mp_import_stat (among other things) diff --git a/py/nlrthumb.c b/py/nlrthumb.c index a22c5df5b9..e7b24f242b 100644 --- a/py/nlrthumb.c +++ b/py/nlrthumb.c @@ -38,6 +38,14 @@ __attribute__((naked)) unsigned int nlr_push(nlr_buf_t *nlr) { + // If you get a linker error here, indicating that a relocation doesn't + // fit, try the following (in that order): + // + // 1. Ensure that nlr.o nlrthumb.o are linked closely together, i.e. + // there aren't too many other files between them in the linker list + // (PY_CORE_O_BASENAME in py/py.mk) + // 2. Set -DMICROPY_NLR_THUMB_USE_LONG_JUMP=1 during the build + // __asm volatile ( "str r4, [r0, #12] \n" // store r4 into nlr_buf "str r5, [r0, #16] \n" // store r5 into nlr_buf @@ -71,7 +79,7 @@ __attribute__((naked)) unsigned int nlr_push(nlr_buf_t *nlr) { "str lr, [r0, #8] \n" // store lr into nlr_buf #endif - #if !defined(__thumb2__) + #if MICROPY_NLR_THUMB_USE_LONG_JUMP "ldr r1, nlr_push_tail_var \n" "bx r1 \n" // do the rest in C ".align 2 \n"