diff --git a/.github/workflows/code_size.yml b/.github/workflows/code_size.yml index a65a00201a..1ef43d1fd3 100644 --- a/.github/workflows/code_size.yml +++ b/.github/workflows/code_size.yml @@ -8,9 +8,15 @@ on: - 'tools/**' - 'py/**' - 'extmod/**' + - 'shared/**' - 'lib/**' - 'ports/bare-arm/**' + - 'ports/mimxrt/**' - 'ports/minimal/**' + - 'ports/rp2/**' + - 'ports/samd/**' + - 'ports/stm32/**' + - 'ports/unix/**' concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/ports_unix.yml b/.github/workflows/ports_unix.yml index 2367eddbe6..4473847db6 100644 --- a/.github/workflows/ports_unix.yml +++ b/.github/workflows/ports_unix.yml @@ -88,10 +88,11 @@ jobs: (cd ports/unix && gcov -o build-coverage/py ../../py/*.c || true) (cd ports/unix && gcov -o build-coverage/extmod ../../extmod/*.c || true) - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: fail_ci_if_error: true verbose: true + token: ${{ secrets.CODECOV_TOKEN }} - name: Print failures if: failure() run: tests/run-tests.py --print-failures diff --git a/extmod/modasyncio.c b/extmod/modasyncio.c index 61dd707223..4f82370d23 100644 --- a/extmod/modasyncio.c +++ b/extmod/modasyncio.c @@ -156,7 +156,7 @@ static MP_DEFINE_CONST_OBJ_TYPE( // Task class // This is the core asyncio context with cur_task, _task_queue and CancelledError. -static mp_obj_t asyncio_context = MP_OBJ_NULL; +mp_obj_t mp_asyncio_context = MP_OBJ_NULL; static mp_obj_t task_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { mp_arg_check_num(n_args, n_kw, 1, 2, false); @@ -168,7 +168,7 @@ static mp_obj_t task_make_new(const mp_obj_type_t *type, size_t n_args, size_t n self->state = TASK_STATE_RUNNING_NOT_WAITED_ON; self->ph_key = MP_OBJ_NEW_SMALL_INT(0); if (n_args == 2) { - asyncio_context = args[1]; + mp_asyncio_context = args[1]; } return MP_OBJ_FROM_PTR(self); } @@ -186,7 +186,7 @@ static mp_obj_t task_cancel(mp_obj_t self_in) { return mp_const_false; } // Can't cancel self (not supported yet). - mp_obj_t cur_task = mp_obj_dict_get(asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_cur_task)); + mp_obj_t cur_task = mp_obj_dict_get(mp_asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_cur_task)); if (self_in == cur_task) { mp_raise_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("can't cancel self")); } @@ -195,7 +195,7 @@ static mp_obj_t task_cancel(mp_obj_t self_in) { self = MP_OBJ_TO_PTR(self->data); } - mp_obj_t _task_queue = mp_obj_dict_get(asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR__task_queue)); + mp_obj_t _task_queue = mp_obj_dict_get(mp_asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR__task_queue)); // Reschedule Task as a cancelled task. mp_obj_t dest[3]; @@ -218,7 +218,7 @@ static mp_obj_t task_cancel(mp_obj_t self_in) { task_queue_push(2, dest); } - self->data = mp_obj_dict_get(asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_CancelledError)); + self->data = mp_obj_dict_get(mp_asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_CancelledError)); return mp_const_true; } @@ -278,7 +278,7 @@ static mp_obj_t task_iternext(mp_obj_t self_in) { nlr_raise(self->data); } else { // Put calling task on waiting queue. - mp_obj_t cur_task = mp_obj_dict_get(asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_cur_task)); + mp_obj_t cur_task = mp_obj_dict_get(mp_asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_cur_task)); mp_obj_t args[2] = { self->state, cur_task }; task_queue_push(2, args); // Set calling task's data to this task that it waits on, to double-link it. diff --git a/ports/rp2/README.md b/ports/rp2/README.md index e22fb3093d..c2f3771cd3 100644 --- a/ports/rp2/README.md +++ b/ports/rp2/README.md @@ -37,7 +37,7 @@ You can also build the standard CMake way. The final firmware is found in the top-level of the CMake build directory (`build` by default) and is called `firmware.uf2`. -If you are using a different board other than a Rasoberry Pi Pico, then you should +If you are using a board other than a Raspberry Pi Pico, you should pass the board name to the build; e.g. for Raspberry Pi Pico W: $ make BOARD=RPI_PICO_W submodules diff --git a/ports/rp2/modmachine.c b/ports/rp2/modmachine.c index 6189e7dc53..5551bf723a 100644 --- a/ports/rp2/modmachine.c +++ b/ports/rp2/modmachine.c @@ -127,10 +127,10 @@ static void mp_machine_lightsleep(size_t n_args, const mp_obj_t *args) { const uint32_t xosc_hz = XOSC_MHZ * 1000000; - uint32_t my_interrupts = save_and_disable_interrupts(); + uint32_t my_interrupts = mp_thread_begin_atomic_section(); #if MICROPY_PY_NETWORK_CYW43 if (cyw43_has_pending && cyw43_poll != NULL) { - restore_interrupts(my_interrupts); + mp_thread_end_atomic_section(my_interrupts); return; } #endif @@ -165,8 +165,15 @@ static void mp_machine_lightsleep(size_t n_args, const mp_obj_t *args) { } else { uint32_t sleep_en0 = clocks_hw->sleep_en0; uint32_t sleep_en1 = clocks_hw->sleep_en1; + bool timer3_enabled = irq_is_enabled(3); + clocks_hw->sleep_en0 = CLOCKS_SLEEP_EN0_CLK_RTC_RTC_BITS; if (use_timer_alarm) { + // Make sure ALARM3/IRQ3 is enabled on _this_ core + timer_hw->inte |= 1 << 3; + if (!timer3_enabled) { + irq_set_enabled(3, true); + } // Use timer alarm to wake. clocks_hw->sleep_en1 = CLOCKS_SLEEP_EN1_CLK_SYS_TIMER_BITS; timer_hw->alarm[3] = timer_hw->timerawl + delay_ms * 1000; @@ -177,6 +184,9 @@ static void mp_machine_lightsleep(size_t n_args, const mp_obj_t *args) { scb_hw->scr |= M0PLUS_SCR_SLEEPDEEP_BITS; __wfi(); scb_hw->scr &= ~M0PLUS_SCR_SLEEPDEEP_BITS; + if (!timer3_enabled) { + irq_set_enabled(3, false); + } clocks_hw->sleep_en0 = sleep_en0; clocks_hw->sleep_en1 = sleep_en1; } @@ -186,7 +196,7 @@ static void mp_machine_lightsleep(size_t n_args, const mp_obj_t *args) { // Bring back all clocks. clocks_init(); - restore_interrupts(my_interrupts); + mp_thread_end_atomic_section(my_interrupts); } NORETURN static void mp_machine_deepsleep(size_t n_args, const mp_obj_t *args) { diff --git a/ports/webassembly/Makefile b/ports/webassembly/Makefile index d0a8aa9924..93b92ef583 100644 --- a/ports/webassembly/Makefile +++ b/ports/webassembly/Makefile @@ -22,6 +22,9 @@ BUILD ?= build-$(VARIANT) include ../../py/mkenv.mk include $(VARIANT_DIR)/mpconfigvariant.mk +# Use the default frozen manifest, variants may override this. +FROZEN_MANIFEST ?= variants/manifest.py + # Qstr definitions (must come before including py.mk). QSTR_DEFS = qstrdefsport.h diff --git a/ports/webassembly/api.js b/ports/webassembly/api.js index 7d1832af4f..2be82e8907 100644 --- a/ports/webassembly/api.js +++ b/ports/webassembly/api.js @@ -38,7 +38,7 @@ export async function loadMicroPython(options) { { heapsize: 1024 * 1024, linebuffer: true }, options, ); - const Module = {}; + let Module = {}; Module.locateFile = (path, scriptDirectory) => url || scriptDirectory + path; Module._textDecoder = new TextDecoder(); @@ -83,11 +83,7 @@ export async function loadMicroPython(options) { Module.stderr = (c) => stderr(new Uint8Array([c])); } } - const moduleLoaded = new Promise((r) => { - Module.postRun = r; - }); - _createMicroPythonModule(Module); - await moduleLoaded; + Module = await _createMicroPythonModule(Module); globalThis.Module = Module; proxy_js_init(); const pyimport = (name) => { @@ -131,23 +127,31 @@ export async function loadMicroPython(options) { }, pyimport: pyimport, runPython(code) { + const len = Module.lengthBytesUTF8(code); + const buf = Module._malloc(len + 1); + Module.stringToUTF8(code, buf, len + 1); const value = Module._malloc(3 * 4); Module.ccall( "mp_js_do_exec", "number", - ["string", "pointer"], - [code, value], + ["pointer", "number", "pointer"], + [buf, len, value], ); + Module._free(buf); return proxy_convert_mp_to_js_obj_jsside_with_free(value); }, runPythonAsync(code) { + const len = Module.lengthBytesUTF8(code); + const buf = Module._malloc(len + 1); + Module.stringToUTF8(code, buf, len + 1); const value = Module._malloc(3 * 4); Module.ccall( "mp_js_do_exec_async", "number", - ["string", "pointer"], - [code, value], + ["pointer", "number", "pointer"], + [buf, len, value], ); + Module._free(buf); return proxy_convert_mp_to_js_obj_jsside_with_free(value); }, replInit() { @@ -224,6 +228,16 @@ async function runCLI() { } }); } else { + // If the script to run ends with a running of the asyncio main loop, then inject + // a simple `asyncio.run` hook that starts the main task. This is primarily to + // support running the standard asyncio tests. + if (contents.endsWith("asyncio.run(main())\n")) { + const asyncio = mp.pyimport("asyncio"); + asyncio.run = async (task) => { + await asyncio.create_task(task); + }; + } + try { mp.runPython(contents); } catch (error) { diff --git a/ports/webassembly/asyncio/__init__.py b/ports/webassembly/asyncio/__init__.py new file mode 100644 index 0000000000..ba1ca63514 --- /dev/null +++ b/ports/webassembly/asyncio/__init__.py @@ -0,0 +1,9 @@ +# MicroPython asyncio module, for use with webassembly port +# MIT license; Copyright (c) 2024 Damien P. George + +from .core import * +from .funcs import wait_for, wait_for_ms, gather +from .event import Event +from .lock import Lock + +__version__ = (3, 0, 0) diff --git a/ports/webassembly/asyncio/core.py b/ports/webassembly/asyncio/core.py new file mode 100644 index 0000000000..a128bb6055 --- /dev/null +++ b/ports/webassembly/asyncio/core.py @@ -0,0 +1,249 @@ +# MicroPython asyncio module, for use with webassembly port +# MIT license; Copyright (c) 2019-2024 Damien P. George + +from time import ticks_ms as ticks, ticks_diff, ticks_add +import sys, js, jsffi + +# Import TaskQueue and Task from built-in C code. +from _asyncio import TaskQueue, Task + + +################################################################################ +# Exceptions + + +class CancelledError(BaseException): + pass + + +class TimeoutError(Exception): + pass + + +# Used when calling Loop.call_exception_handler. +_exc_context = {"message": "Task exception wasn't retrieved", "exception": None, "future": None} + + +################################################################################ +# Sleep functions + + +# "Yield" once, then raise StopIteration +class SingletonGenerator: + def __init__(self): + self.state = None + self.exc = StopIteration() + + def __iter__(self): + return self + + def __next__(self): + if self.state is not None: + _task_queue.push(cur_task, self.state) + self.state = None + return None + else: + self.exc.__traceback__ = None + raise self.exc + + +# Pause task execution for the given time (integer in milliseconds, uPy extension) +# Use a SingletonGenerator to do it without allocating on the heap +def sleep_ms(t, sgen=SingletonGenerator()): + if cur_task is None: + # Support top-level asyncio.sleep, via a JavaScript Promise. + return jsffi.async_timeout_ms(t) + assert sgen.state is None + sgen.state = ticks_add(ticks(), max(0, t)) + return sgen + + +# Pause task execution for the given time (in seconds) +def sleep(t): + return sleep_ms(int(t * 1000)) + + +################################################################################ +# Main run loop + +asyncio_timer = None + + +class ThenableEvent: + def __init__(self, thenable): + self.result = None # Result of the thenable + self.waiting = None # Task waiting on completion of this thenable + thenable.then(self.set) + + def set(self, value): + # Thenable/Promise is fulfilled, set result and schedule any waiting task. + self.result = value + if self.waiting: + _task_queue.push(self.waiting) + self.waiting = None + _schedule_run_iter(0) + + def remove(self, task): + self.waiting = None + + # async + def wait(self): + # Set the calling task as the task waiting on this thenable. + self.waiting = cur_task + # Set calling task's data to this object so it can be removed if needed. + cur_task.data = self + # Wait for the thenable to fulfill. + yield + # Return the result of the thenable. + return self.result + + +# Ensure the awaitable is a task +def _promote_to_task(aw): + return aw if isinstance(aw, Task) else create_task(aw) + + +def _schedule_run_iter(dt): + global asyncio_timer + if asyncio_timer is not None: + js.clearTimeout(asyncio_timer) + asyncio_timer = js.setTimeout(_run_iter, dt) + + +def _run_iter(): + global cur_task + excs_all = (CancelledError, Exception) # To prevent heap allocation in loop + excs_stop = (CancelledError, StopIteration) # To prevent heap allocation in loop + while True: + # Wait until the head of _task_queue is ready to run + t = _task_queue.peek() + if t: + # A task waiting on _task_queue; "ph_key" is time to schedule task at + dt = max(0, ticks_diff(t.ph_key, ticks())) + else: + # No tasks can be woken so finished running + cur_task = None + return + + if dt > 0: + # schedule to call again later + cur_task = None + _schedule_run_iter(dt) + return + + # Get next task to run and continue it + t = _task_queue.pop() + cur_task = t + try: + # Continue running the coroutine, it's responsible for rescheduling itself + exc = t.data + if not exc: + t.coro.send(None) + else: + # If the task is finished and on the run queue and gets here, then it + # had an exception and was not await'ed on. Throwing into it now will + # raise StopIteration and the code below will catch this and run the + # call_exception_handler function. + t.data = None + t.coro.throw(exc) + except excs_all as er: + # Check the task is not on any event queue + assert t.data is None + # This task is done. + if t.state: + # Task was running but is now finished. + waiting = False + if t.state is True: + # "None" indicates that the task is complete and not await'ed on (yet). + t.state = None + elif callable(t.state): + # The task has a callback registered to be called on completion. + t.state(t, er) + t.state = False + waiting = True + else: + # Schedule any other tasks waiting on the completion of this task. + while t.state.peek(): + _task_queue.push(t.state.pop()) + waiting = True + # "False" indicates that the task is complete and has been await'ed on. + t.state = False + if not waiting and not isinstance(er, excs_stop): + # An exception ended this detached task, so queue it for later + # execution to handle the uncaught exception if no other task retrieves + # the exception in the meantime (this is handled by Task.throw). + _task_queue.push(t) + # Save return value of coro to pass up to caller. + t.data = er + elif t.state is None: + # Task is already finished and nothing await'ed on the task, + # so call the exception handler. + + # Save exception raised by the coro for later use. + t.data = exc + + # Create exception context and call the exception handler. + _exc_context["exception"] = exc + _exc_context["future"] = t + Loop.call_exception_handler(_exc_context) + + +# Create and schedule a new task from a coroutine. +def create_task(coro): + if not hasattr(coro, "send"): + raise TypeError("coroutine expected") + t = Task(coro, globals()) + _task_queue.push(t) + _schedule_run_iter(0) + return t + + +################################################################################ +# Event loop wrapper + + +cur_task = None + + +class Loop: + _exc_handler = None + + def create_task(coro): + return create_task(coro) + + def close(): + pass + + def set_exception_handler(handler): + Loop._exc_handler = handler + + def get_exception_handler(): + return Loop._exc_handler + + def default_exception_handler(loop, context): + print(context["message"], file=sys.stderr) + print("future:", context["future"], "coro=", context["future"].coro, file=sys.stderr) + sys.print_exception(context["exception"], sys.stderr) + + def call_exception_handler(context): + (Loop._exc_handler or Loop.default_exception_handler)(Loop, context) + + +def get_event_loop(): + return Loop + + +def current_task(): + if cur_task is None: + raise RuntimeError("no running event loop") + return cur_task + + +def new_event_loop(): + global _task_queue + _task_queue = TaskQueue() # TaskQueue of Task instances. + return Loop + + +# Initialise default event loop. +new_event_loop() diff --git a/ports/webassembly/main.c b/ports/webassembly/main.c index 441c6d67e3..23d12b6dbb 100644 --- a/ports/webassembly/main.c +++ b/ports/webassembly/main.c @@ -104,7 +104,7 @@ void mp_js_do_import(const char *name, uint32_t *out) { } } -void mp_js_do_exec(const char *src, uint32_t *out) { +void mp_js_do_exec(const char *src, size_t len, uint32_t *out) { // Collect at the top-level, where there are no root pointers from stack/registers. gc_collect_start(); gc_collect_end(); @@ -112,7 +112,7 @@ void mp_js_do_exec(const char *src, uint32_t *out) { mp_parse_input_kind_t input_kind = MP_PARSE_FILE_INPUT; nlr_buf_t nlr; if (nlr_push(&nlr) == 0) { - mp_lexer_t *lex = mp_lexer_new_from_str_len_dedent(MP_QSTR__lt_stdin_gt_, src, strlen(src), 0); + mp_lexer_t *lex = mp_lexer_new_from_str_len_dedent(MP_QSTR__lt_stdin_gt_, src, len, 0); qstr source_name = lex->source_name; mp_parse_tree_t parse_tree = mp_parse(lex, input_kind); mp_obj_t module_fun = mp_compile(&parse_tree, source_name, false); @@ -125,9 +125,9 @@ void mp_js_do_exec(const char *src, uint32_t *out) { } } -void mp_js_do_exec_async(const char *src, uint32_t *out) { +void mp_js_do_exec_async(const char *src, size_t len, uint32_t *out) { mp_compile_allow_top_level_await = true; - mp_js_do_exec(src, out); + mp_js_do_exec(src, len, out); mp_compile_allow_top_level_await = false; } diff --git a/ports/webassembly/modjsffi.c b/ports/webassembly/modjsffi.c index d4e61e368f..cb2f578f8f 100644 --- a/ports/webassembly/modjsffi.c +++ b/ports/webassembly/modjsffi.c @@ -61,12 +61,27 @@ static mp_obj_t mp_jsffi_to_js(mp_obj_t arg) { } static MP_DEFINE_CONST_FUN_OBJ_1(mp_jsffi_to_js_obj, mp_jsffi_to_js); +// *FORMAT-OFF* +EM_JS(void, promise_with_timeout_ms, (double ms, uint32_t * out), { + const ret = new Promise((resolve) => setTimeout(resolve, ms)); + proxy_convert_js_to_mp_obj_jsside(ret, out); +}); +// *FORMAT-ON* + +static mp_obj_t mp_jsffi_async_timeout_ms(mp_obj_t arg) { + uint32_t out[PVN]; + promise_with_timeout_ms(mp_obj_get_float_to_d(arg), out); + return proxy_convert_js_to_mp_obj_cside(out); +} +static MP_DEFINE_CONST_FUN_OBJ_1(mp_jsffi_async_timeout_ms_obj, mp_jsffi_async_timeout_ms); + static const mp_rom_map_elem_t mp_module_jsffi_globals_table[] = { { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_jsffi) }, { MP_ROM_QSTR(MP_QSTR_JsProxy), MP_ROM_PTR(&mp_type_jsproxy) }, { MP_ROM_QSTR(MP_QSTR_create_proxy), MP_ROM_PTR(&mp_jsffi_create_proxy_obj) }, { MP_ROM_QSTR(MP_QSTR_to_js), MP_ROM_PTR(&mp_jsffi_to_js_obj) }, + { MP_ROM_QSTR(MP_QSTR_async_timeout_ms), MP_ROM_PTR(&mp_jsffi_async_timeout_ms_obj) }, }; static MP_DEFINE_CONST_DICT(mp_module_jsffi_globals, mp_module_jsffi_globals_table); diff --git a/ports/webassembly/objjsproxy.c b/ports/webassembly/objjsproxy.c index 098f4e75f5..15fbb57523 100644 --- a/ports/webassembly/objjsproxy.c +++ b/ports/webassembly/objjsproxy.c @@ -346,6 +346,12 @@ typedef struct _jsproxy_gen_t { mp_vm_return_kind_t jsproxy_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_obj_t throw_value, mp_obj_t *ret_val) { jsproxy_gen_t *self = MP_OBJ_TO_PTR(self_in); + + if (throw_value) { + *ret_val = throw_value; + return MP_VM_RETURN_EXCEPTION; + } + switch (self->state) { case JSOBJ_GEN_STATE_WAITING: self->state = JSOBJ_GEN_STATE_COMPLETED; @@ -468,9 +474,29 @@ static mp_obj_t jsproxy_new_gen(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf) { /******************************************************************************/ +#if MICROPY_PY_ASYNCIO +extern mp_obj_t mp_asyncio_context; +#endif + static mp_obj_t jsproxy_getiter(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf) { mp_obj_jsproxy_t *self = MP_OBJ_TO_PTR(self_in); if (has_attr(self->ref, "then")) { + #if MICROPY_PY_ASYNCIO + // When asyncio is running and the caller here is a task, wrap the JavaScript + // thenable in a ThenableEvent, and get the task to wait on that event. This + // decouples the task from the thenable and allows cancelling the task. + if (mp_asyncio_context != MP_OBJ_NULL) { + mp_obj_t cur_task = mp_obj_dict_get(mp_asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_cur_task)); + if (cur_task != mp_const_none) { + mp_obj_t thenable_event_class = mp_obj_dict_get(mp_asyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_ThenableEvent)); + mp_obj_t thenable_event = mp_call_function_1(thenable_event_class, self_in); + mp_obj_t dest[2]; + mp_load_method(thenable_event, MP_QSTR_wait, dest); + mp_obj_t wait_gen = mp_call_method_n_kw(0, 0, dest); + return mp_getiter(wait_gen, iter_buf); + } + } + #endif return jsproxy_new_gen(self_in, iter_buf); } else { return jsproxy_new_it(self_in, iter_buf); diff --git a/ports/webassembly/proxy_c.c b/ports/webassembly/proxy_c.c index 8d137f6271..ebf670b56e 100644 --- a/ports/webassembly/proxy_c.c +++ b/ports/webassembly/proxy_c.c @@ -296,10 +296,11 @@ EM_JS(void, js_then_resolve, (uint32_t * ret_value, uint32_t * resolve, uint32_t resolve_js(ret_value_js); }); -EM_JS(void, js_then_reject, (uint32_t * resolve, uint32_t * reject), { +EM_JS(void, js_then_reject, (uint32_t * ret_value, uint32_t * resolve, uint32_t * reject), { + const ret_value_js = proxy_convert_mp_to_js_obj_jsside(ret_value); const resolve_js = proxy_convert_mp_to_js_obj_jsside(resolve); const reject_js = proxy_convert_mp_to_js_obj_jsside(reject); - reject_js(null); + reject_js(ret_value_js); }); // *FORMAT-OFF* @@ -307,14 +308,34 @@ EM_JS(void, js_then_continue, (int jsref, uint32_t * py_resume, uint32_t * resol const py_resume_js = proxy_convert_mp_to_js_obj_jsside(py_resume); const resolve_js = proxy_convert_mp_to_js_obj_jsside(resolve); const reject_js = proxy_convert_mp_to_js_obj_jsside(reject); - const ret = proxy_js_ref[jsref].then((x) => {py_resume_js(x, resolve_js, reject_js);}, reject_js); + const ret = proxy_js_ref[jsref].then( + (result) => { + // The Promise is fulfilled on the JavaScript side. Take the result and + // send it to the encapsulating generator on the Python side, so it + // becomes the result of the "yield from" that deferred to this Promise. + py_resume_js(result, null, resolve_js, reject_js); + }, + (reason) => { + // The Promise is rejected on the JavaScript side. Take the reason and + // throw it into the encapsulating generator on the Python side. + py_resume_js(null, reason, resolve_js, reject_js); + }, + ); proxy_convert_js_to_mp_obj_jsside(ret, out); }); // *FORMAT-ON* -static mp_obj_t proxy_resume_execute(mp_obj_t self_in, mp_obj_t value, mp_obj_t resolve, mp_obj_t reject) { +static mp_obj_t proxy_resume_execute(mp_obj_t self_in, mp_obj_t send_value, mp_obj_t throw_value, mp_obj_t resolve, mp_obj_t reject) { + if (throw_value != MP_OBJ_NULL && throw_value != mp_const_none) { + if (send_value == mp_const_none) { + send_value = MP_OBJ_NULL; + } + } else { + throw_value = MP_OBJ_NULL; + } + mp_obj_t ret_value; - mp_vm_return_kind_t ret_kind = mp_resume(self_in, value, MP_OBJ_NULL, &ret_value); + mp_vm_return_kind_t ret_kind = mp_resume(self_in, send_value, throw_value, &ret_value); uint32_t out_resolve[PVN]; uint32_t out_reject[PVN]; @@ -335,17 +356,19 @@ static mp_obj_t proxy_resume_execute(mp_obj_t self_in, mp_obj_t value, mp_obj_t uint32_t out[PVN]; js_then_continue(ref, out_py_resume, out_resolve, out_reject, out); return proxy_convert_js_to_mp_obj_cside(out); - } else { - // MP_VM_RETURN_EXCEPTION; - js_then_reject(out_resolve, out_reject); - nlr_raise(ret_value); + } else { // ret_kind == MP_VM_RETURN_EXCEPTION; + // Pass the exception through as an object to reject the promise (don't raise/throw it). + uint32_t out_ret_value[PVN]; + proxy_convert_mp_to_js_obj_cside(ret_value, out_ret_value); + js_then_reject(out_ret_value, out_resolve, out_reject); + return mp_const_none; } } static mp_obj_t resume_fun(size_t n_args, const mp_obj_t *args) { - return proxy_resume_execute(args[0], args[1], args[2], args[3]); + return proxy_resume_execute(args[0], args[1], args[2], args[3], args[4]); } -static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(resume_obj, 4, 4, resume_fun); +static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(resume_obj, 5, 5, resume_fun); void proxy_c_to_js_resume(uint32_t c_ref, uint32_t *args) { nlr_buf_t nlr; @@ -353,7 +376,7 @@ void proxy_c_to_js_resume(uint32_t c_ref, uint32_t *args) { mp_obj_t obj = proxy_c_get_obj(c_ref); mp_obj_t resolve = proxy_convert_js_to_mp_obj_cside(args + 1 * 3); mp_obj_t reject = proxy_convert_js_to_mp_obj_cside(args + 2 * 3); - mp_obj_t ret = proxy_resume_execute(obj, mp_const_none, resolve, reject); + mp_obj_t ret = proxy_resume_execute(obj, mp_const_none, mp_const_none, resolve, reject); nlr_pop(); return proxy_convert_mp_to_js_obj_cside(ret, args); } else { diff --git a/ports/webassembly/variants/manifest.py b/ports/webassembly/variants/manifest.py new file mode 100644 index 0000000000..b2ee8cd640 --- /dev/null +++ b/ports/webassembly/variants/manifest.py @@ -0,0 +1,24 @@ +# The asyncio package is built from the standard implementation but with the +# core scheduler replaced with a custom scheduler that uses the JavaScript +# runtime (with setTimeout an Promise's) to contrtol the scheduling. + +package( + "asyncio", + ( + "event.py", + "funcs.py", + "lock.py", + ), + base_path="$(MPY_DIR)/extmod", + opt=3, +) + +package( + "asyncio", + ( + "__init__.py", + "core.py", + ), + base_path="$(PORT_DIR)", + opt=3, +) diff --git a/ports/webassembly/variants/pyscript/manifest.py b/ports/webassembly/variants/pyscript/manifest.py index 0646e1d897..db088e70d0 100644 --- a/ports/webassembly/variants/pyscript/manifest.py +++ b/ports/webassembly/variants/pyscript/manifest.py @@ -1,3 +1,5 @@ +include("$(PORT_DIR)/variants/manifest.py") + require("abc") require("base64") require("collections") diff --git a/py/mpconfig.h b/py/mpconfig.h index af2480266b..1aa3e06994 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -587,6 +587,12 @@ /*****************************************************************************/ /* Python internal features */ +// Use a special long jump in nlrthumb.c, which may be necessary if nlr.o and +// nlrthumb.o are linked far apart from each other. +#ifndef MICROPY_NLR_THUMB_USE_LONG_JUMP +#define MICROPY_NLR_THUMB_USE_LONG_JUMP (0) +#endif + // Whether to enable import of external modules // When disabled, only importing of built-in modules is supported // When enabled, a port must implement mp_import_stat (among other things) diff --git a/py/nlrthumb.c b/py/nlrthumb.c index a22c5df5b9..e7b24f242b 100644 --- a/py/nlrthumb.c +++ b/py/nlrthumb.c @@ -38,6 +38,14 @@ __attribute__((naked)) unsigned int nlr_push(nlr_buf_t *nlr) { + // If you get a linker error here, indicating that a relocation doesn't + // fit, try the following (in that order): + // + // 1. Ensure that nlr.o nlrthumb.o are linked closely together, i.e. + // there aren't too many other files between them in the linker list + // (PY_CORE_O_BASENAME in py/py.mk) + // 2. Set -DMICROPY_NLR_THUMB_USE_LONG_JUMP=1 during the build + // __asm volatile ( "str r4, [r0, #12] \n" // store r4 into nlr_buf "str r5, [r0, #16] \n" // store r5 into nlr_buf @@ -71,7 +79,7 @@ __attribute__((naked)) unsigned int nlr_push(nlr_buf_t *nlr) { "str lr, [r0, #8] \n" // store lr into nlr_buf #endif - #if !defined(__thumb2__) + #if MICROPY_NLR_THUMB_USE_LONG_JUMP "ldr r1, nlr_push_tail_var \n" "bx r1 \n" // do the rest in C ".align 2 \n" diff --git a/py/obj.h b/py/obj.h index 9f2bb46e41..86952565c5 100644 --- a/py/obj.h +++ b/py/obj.h @@ -753,20 +753,20 @@ typedef struct _mp_obj_full_type_t { // Do not use these directly, instead use MP_DEFINE_CONST_OBJ_TYPE. // Generated with: // for i in range(13): -// print(f"#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_{i}(_struct_type, _typename, _name, _flags{''.join(f', f{j+1}, v{j+1}' for j in range(i))}) const _struct_type _typename = {{ .base = {{ &mp_type_type }}, .name = _name, .flags = _flags{''.join(f', .slot_index_##f{j+1} = {j+1}' for j in range(i))}{', .slots = { ' + ''.join(f'v{j+1}, ' for j in range(i)) + '}' if i else '' } }}") -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_0(_struct_type, _typename, _name, _flags) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_1(_struct_type, _typename, _name, _flags, f1, v1) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slots = { v1, } } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_2(_struct_type, _typename, _name, _flags, f1, v1, f2, v2) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slots = { v1, v2, } } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_3(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slots = { v1, v2, v3, } } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_4(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slots = { v1, v2, v3, v4, } } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_5(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slots = { v1, v2, v3, v4, v5, } } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_6(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slots = { v1, v2, v3, v4, v5, v6, } } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_7(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slots = { v1, v2, v3, v4, v5, v6, v7, } } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_8(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, } } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_9(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, } } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_10(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, } } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_11(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10, f11, v11) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slot_index_##f11 = 11, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, } } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_12(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10, f11, v11, f12, v12) const _struct_type _typename = { .base = { &mp_type_type }, .name = _name, .flags = _flags, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slot_index_##f11 = 11, .slot_index_##f12 = 12, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, } } +// print(f"#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_{i}(_struct_type, _typename, _name, _flags{''.join(f', f{j+1}, v{j+1}' for j in range(i))}) const _struct_type _typename = {{ .base = {{ &mp_type_type }}, .flags = _flags, .name = _name{''.join(f', .slot_index_##f{j+1} = {j+1}' for j in range(i))}{', .slots = { ' + ''.join(f'v{j+1}, ' for j in range(i)) + '}' if i else '' } }}") +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_0(_struct_type, _typename, _name, _flags) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_1(_struct_type, _typename, _name, _flags, f1, v1) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slots = { v1, } } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_2(_struct_type, _typename, _name, _flags, f1, v1, f2, v2) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slots = { v1, v2, } } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_3(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slots = { v1, v2, v3, } } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_4(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slots = { v1, v2, v3, v4, } } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_5(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slots = { v1, v2, v3, v4, v5, } } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_6(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slots = { v1, v2, v3, v4, v5, v6, } } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_7(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slots = { v1, v2, v3, v4, v5, v6, v7, } } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_8(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, } } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_9(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, } } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_10(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, } } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_11(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10, f11, v11) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slot_index_##f11 = 11, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, } } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_12(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10, f11, v11, f12, v12) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slot_index_##f11 = 11, .slot_index_##f12 = 12, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, } } // Because the mp_obj_type_t instances are in (zero-initialised) ROM, we take // slot_index_foo=0 to mean that the slot is unset. This also simplifies checking diff --git a/py/objarray.c b/py/objarray.c index 1fff234822..803af2cd27 100644 --- a/py/objarray.c +++ b/py/objarray.c @@ -424,6 +424,13 @@ static mp_obj_t array_extend(mp_obj_t self_in, mp_obj_t arg_in) { if (self->free < len) { self->items = m_renew(byte, self->items, (self->len + self->free) * sz, (self->len + len) * sz); self->free = 0; + + if (self_in == arg_in) { + // Get arg_bufinfo again in case self->items has moved + // + // (Note not possible to handle case that arg_in is a memoryview into self) + mp_get_buffer_raise(arg_in, &arg_bufinfo, MP_BUFFER_READ); + } } else { self->free -= len; } @@ -456,7 +463,8 @@ static mp_obj_t array_subscr(mp_obj_t self_in, mp_obj_t index_in, mp_obj_t value #if MICROPY_PY_ARRAY_SLICE_ASSIGN // Assign size_t src_len; - void *src_items; + uint8_t *src_items; + size_t src_offs = 0; size_t item_sz = mp_binary_get_size('@', o->typecode & TYPECODE_MASK, NULL); if (mp_obj_is_obj(value) && MP_OBJ_TYPE_GET_SLOT_OR_NULL(((mp_obj_base_t *)MP_OBJ_TO_PTR(value))->type, subscr) == array_subscr) { // value is array, bytearray or memoryview @@ -469,7 +477,7 @@ static mp_obj_t array_subscr(mp_obj_t self_in, mp_obj_t index_in, mp_obj_t value src_items = src_slice->items; #if MICROPY_PY_BUILTINS_MEMORYVIEW if (mp_obj_is_type(value, &mp_type_memoryview)) { - src_items = (uint8_t *)src_items + (src_slice->memview_offset * item_sz); + src_offs = src_slice->memview_offset * item_sz; } #endif } else if (mp_obj_is_type(value, &mp_type_bytes)) { @@ -504,13 +512,17 @@ static mp_obj_t array_subscr(mp_obj_t self_in, mp_obj_t index_in, mp_obj_t value // TODO: alloc policy; at the moment we go conservative o->items = m_renew(byte, o->items, (o->len + o->free) * item_sz, (o->len + len_adj) * item_sz); o->free = len_adj; + // m_renew may have moved o->items + if (src_items == dest_items) { + src_items = o->items; + } dest_items = o->items; } mp_seq_replace_slice_grow_inplace(dest_items, o->len, - slice.start, slice.stop, src_items, src_len, len_adj, item_sz); + slice.start, slice.stop, src_items + src_offs, src_len, len_adj, item_sz); } else { mp_seq_replace_slice_no_grow(dest_items, o->len, - slice.start, slice.stop, src_items, src_len, item_sz); + slice.start, slice.stop, src_items + src_offs, src_len, item_sz); // Clear "freed" elements at the end of list // TODO: This is actually only needed for typecode=='O' mp_seq_clear(dest_items, o->len + len_adj, o->len, item_sz); diff --git a/py/objfun.h b/py/objfun.h index af7c334858..f16ef76b80 100644 --- a/py/objfun.h +++ b/py/objfun.h @@ -56,14 +56,14 @@ void mp_obj_fun_bc_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest); #if MICROPY_EMIT_NATIVE static inline mp_obj_t mp_obj_new_fun_native(const mp_obj_t *def_args, const void *fun_data, const mp_module_context_t *mc, struct _mp_raw_code_t *const *child_table) { - mp_obj_fun_bc_t *o = MP_OBJ_TO_PTR(mp_obj_new_fun_bc(def_args, (const byte *)fun_data, mc, child_table)); + mp_obj_fun_bc_t *o = (mp_obj_fun_bc_t *)MP_OBJ_TO_PTR(mp_obj_new_fun_bc(def_args, (const byte *)fun_data, mc, child_table)); o->base.type = &mp_type_fun_native; return MP_OBJ_FROM_PTR(o); } static inline mp_obj_t mp_obj_new_fun_viper(const void *fun_data, const mp_module_context_t *mc, struct _mp_raw_code_t *const *child_table) { mp_obj_fun_bc_t *o = mp_obj_malloc(mp_obj_fun_bc_t, &mp_type_fun_viper); - o->bytecode = fun_data; + o->bytecode = (const byte *)fun_data; o->context = mc; o->child_table = child_table; return MP_OBJ_FROM_PTR(o); @@ -101,9 +101,9 @@ static inline void *mp_obj_fun_native_get_generator_resume(const mp_obj_fun_bc_t #if MICROPY_EMIT_INLINE_ASM static inline mp_obj_t mp_obj_new_fun_asm(size_t n_args, const void *fun_data, mp_uint_t type_sig) { - mp_obj_fun_asm_t *o = mp_obj_malloc(mp_obj_fun_asm_t, &mp_type_fun_asm); + mp_obj_fun_asm_t *o = (mp_obj_fun_asm_t *)mp_obj_malloc(mp_obj_fun_asm_t, &mp_type_fun_asm); o->n_args = n_args; - o->fun_data = fun_data; + o->fun_data = (const byte *)fun_data; o->type_sig = type_sig; return MP_OBJ_FROM_PTR(o); } diff --git a/tests/basics/bytearray_add.py b/tests/basics/bytearray_add.py index a7e2b57374..1f30a3b740 100644 --- a/tests/basics/bytearray_add.py +++ b/tests/basics/bytearray_add.py @@ -15,4 +15,11 @@ print(b) # this inplace add tests the code when the buffer doesn't need to be increased b = bytearray() -b += b'' +b += b"" + +# extend a bytearray from itself +b = bytearray(b"abcdefgh") +for _ in range(4): + c = bytearray(b) # extra allocation, as above + b.extend(b) +print(b) diff --git a/tests/basics/bytearray_add_self.py b/tests/basics/bytearray_add_self.py new file mode 100644 index 0000000000..94ae8689fd --- /dev/null +++ b/tests/basics/bytearray_add_self.py @@ -0,0 +1,8 @@ +# add a bytearray to itself +# This is not supported by CPython as of 3.11.18. + +b = bytearray(b"123456789") +for _ in range(4): + c = bytearray(b) # extra allocation increases chance 'b' has to relocate + b += b +print(b) diff --git a/tests/basics/bytearray_add_self.py.exp b/tests/basics/bytearray_add_self.py.exp new file mode 100644 index 0000000000..5ef948157a --- /dev/null +++ b/tests/basics/bytearray_add_self.py.exp @@ -0,0 +1 @@ +bytearray(b'123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789') diff --git a/tests/basics/bytearray_slice_assign.py b/tests/basics/bytearray_slice_assign.py index fa7878e10d..4de0819042 100644 --- a/tests/basics/bytearray_slice_assign.py +++ b/tests/basics/bytearray_slice_assign.py @@ -18,7 +18,7 @@ l = bytearray(x) l[1:3] = bytearray() print(l) l = bytearray(x) -#del l[1:3] +# del l[1:3] print(l) l = bytearray(x) @@ -28,7 +28,7 @@ l = bytearray(x) l[:3] = bytearray() print(l) l = bytearray(x) -#del l[:3] +# del l[:3] print(l) l = bytearray(x) @@ -38,7 +38,7 @@ l = bytearray(x) l[:-3] = bytearray() print(l) l = bytearray(x) -#del l[:-3] +# del l[:-3] print(l) # slice assignment that extends the array @@ -61,8 +61,14 @@ b[1:1] = b"12345" print(b) # Growth of bytearray via slice extension -b = bytearray(b'12345678') -b.append(57) # expand and add a bit of unused space at end of the bytearray +b = bytearray(b"12345678") +b.append(57) # expand and add a bit of unused space at end of the bytearray for i in range(400): - b[-1:] = b'ab' # grow slowly into the unused space + b[-1:] = b"ab" # grow slowly into the unused space +print(len(b), b) + +# Growth of bytearray via slice extension from itself +b = bytearray(b"1234567") +for i in range(3): + b[-1:] = b print(len(b), b) diff --git a/tests/cpydiff/types_memoryview_invalid.py b/tests/cpydiff/types_memoryview_invalid.py new file mode 100644 index 0000000000..c995a2e899 --- /dev/null +++ b/tests/cpydiff/types_memoryview_invalid.py @@ -0,0 +1,12 @@ +""" +categories: Types,memoryview +description: memoryview can become invalid if its target is resized +cause: CPython prevents a ``bytearray`` or ``io.bytesIO`` object from changing size while there is a ``memoryview`` object that references it. MicroPython requires the programmer to manually ensure that an object is not resized while any ``memoryview`` references it. + +In the worst case scenario, resizing an object which is the target of a memoryview can cause the memoryview(s) to reference invalid freed memory (a use-after-free bug) and corrupt the MicroPython runtime. +workaround: Do not change the size of any ``bytearray`` or ``io.bytesIO`` object that has a ``memoryview`` assigned to it. +""" +b = bytearray(b"abcdefg") +m = memoryview(b) +b.extend(b"hijklmnop") +print(b, bytes(m)) diff --git a/tests/ports/webassembly/asyncio_create_task.mjs b/tests/ports/webassembly/asyncio_create_task.mjs new file mode 100644 index 0000000000..e388ade4f1 --- /dev/null +++ b/tests/ports/webassembly/asyncio_create_task.mjs @@ -0,0 +1,44 @@ +// Test asyncio.create_task(), and tasks waiting on a Promise. + +const mp = await (await import(process.argv[2])).loadMicroPython(); + +globalThis.p0 = new Promise((resolve, reject) => { + resolve(123); +}); + +globalThis.p1 = new Promise((resolve, reject) => { + setTimeout(() => { + console.log("setTimeout resolved"); + resolve(456); + }, 200); +}); + +mp.runPython(` +import js +import asyncio + +async def task(id, promise): + print("task start", id) + print("task await", id, await promise) + print("task await", id, await promise) + print("task end", id) + +print("start") +t1 = asyncio.create_task(task(1, js.p0)) +t2 = asyncio.create_task(task(2, js.p1)) +print("t1", t1.done(), t2.done()) +print("end") +`); + +// Wait for p1 to fulfill so t2 can continue. +await globalThis.p1; + +// Wait a little longer so t2 can complete. +await new Promise((resolve, reject) => { + setTimeout(resolve, 10); +}); + +mp.runPython(` +print("restart") +print("t1", t1.done(), t2.done()) +`); diff --git a/tests/ports/webassembly/asyncio_create_task.mjs.exp b/tests/ports/webassembly/asyncio_create_task.mjs.exp new file mode 100644 index 0000000000..c1958bba59 --- /dev/null +++ b/tests/ports/webassembly/asyncio_create_task.mjs.exp @@ -0,0 +1,14 @@ +start +t1 False False +end +task start 1 +task start 2 +task await 1 123 +task await 1 123 +task end 1 +setTimeout resolved +task await 2 456 +task await 2 456 +task end 2 +restart +t1 True True diff --git a/tests/ports/webassembly/asyncio_sleep.mjs b/tests/ports/webassembly/asyncio_sleep.mjs new file mode 100644 index 0000000000..74d22ee1f8 --- /dev/null +++ b/tests/ports/webassembly/asyncio_sleep.mjs @@ -0,0 +1,25 @@ +// Test asyncio.sleep(), both at the top level and within a task. + +const mp = await (await import(process.argv[2])).loadMicroPython(); + +await mp.runPythonAsync(` +import time +import asyncio + +print("main start") +t0 = time.time() +await asyncio.sleep(0.25) +dt = time.time() - t0 +print(0.2 <= dt <= 0.3) + +async def task(): + print("task start") + t0 = time.time() + await asyncio.sleep(0.25) + dt = time.time() - t0 + print(0.2 <= dt <= 0.3) + print("task end") + +asyncio.create_task(task()) +print("main end") +`); diff --git a/tests/ports/webassembly/asyncio_sleep.mjs.exp b/tests/ports/webassembly/asyncio_sleep.mjs.exp new file mode 100644 index 0000000000..619ba175f6 --- /dev/null +++ b/tests/ports/webassembly/asyncio_sleep.mjs.exp @@ -0,0 +1,6 @@ +main start +True +main end +task start +True +task end diff --git a/tests/run-tests.py b/tests/run-tests.py index 4f55cdd398..8acdcd2b36 100755 --- a/tests/run-tests.py +++ b/tests/run-tests.py @@ -681,6 +681,17 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): elif args.target == "webassembly": skip_tests.add("basics/string_format_modulo.py") # can't print nulls to stdout skip_tests.add("basics/string_strip.py") # can't print nulls to stdout + skip_tests.add("extmod/asyncio_basic2.py") + skip_tests.add("extmod/asyncio_cancel_self.py") + skip_tests.add("extmod/asyncio_current_task.py") + skip_tests.add("extmod/asyncio_exception.py") + skip_tests.add("extmod/asyncio_gather_finished_early.py") + skip_tests.add("extmod/asyncio_get_event_loop.py") + skip_tests.add("extmod/asyncio_heaplock.py") + skip_tests.add("extmod/asyncio_loop_stop.py") + skip_tests.add("extmod/asyncio_new_event_loop.py") + skip_tests.add("extmod/asyncio_threadsafeflag.py") + skip_tests.add("extmod/asyncio_wait_for_fwd.py") skip_tests.add("extmod/binascii_a2b_base64.py") skip_tests.add("extmod/re_stack_overflow.py") skip_tests.add("extmod/time_res.py")