webassembly: Implement runPythonAsync() for top-level async code.

With this commit, `interpreter.runPythonAsync(code)` can now be used to run
Python code that uses `await` at the top level.  That will yield up to
JavaScript and produce a thenable, which the JavaScript runtime can then
resume.  Also implemented is the ability for Python code to await on
JavaScript promises/thenables.  For example, outer JavaScript code can
await on `runPythonAsync(code)` which then runs Python code that does
`await js.fetch(url)`.  The entire chain of calls will be suspended until
the fetch completes.

Signed-off-by: Damien George <damien@micropython.org>

ports/webassembly/Makefile

@@ -47,6 +47,7 @@ CFLAGS += $(INC)
 
 EXPORTED_FUNCTIONS_EXTRA += ,\
 	_mp_js_do_exec,\
+	_mp_js_do_exec_async,\
 	_mp_js_do_import,\
 	_mp_js_register_js_module,\
 	_proxy_c_init,\
@@ -58,6 +59,7 @@ EXPORTED_FUNCTIONS_EXTRA += ,\
 	_proxy_c_to_js_get_type,\
 	_proxy_c_to_js_has_attr,\
 	_proxy_c_to_js_lookup_attr,\
+	_proxy_c_to_js_resume,\
 	_proxy_c_to_js_store_attr,\
 	_proxy_convert_mp_to_js_obj_cside
 

ports/webassembly/api.js

@@ -140,6 +140,16 @@ export async function loadMicroPython(options) {
             );
             return proxy_convert_mp_to_js_obj_jsside_with_free(value);
         },
+        runPythonAsync(code) {
+            const value = Module._malloc(3 * 4);
+            Module.ccall(
+                "mp_js_do_exec_async",
+                "number",
+                ["string", "pointer"],
+                [code, value],
+            );
+            return proxy_convert_mp_to_js_obj_jsside_with_free(value);
+        },
     };
 }
 

ports/webassembly/main.c

@@ -169,6 +169,12 @@ void mp_js_do_exec(const char *src, uint32_t *out) {
     }
 }
 
+void mp_js_do_exec_async(const char *src, uint32_t *out) {
+    mp_compile_allow_top_level_await = true;
+    mp_js_do_exec(src, out);
+    mp_compile_allow_top_level_await = false;
+}
+
 #if MICROPY_GC_SPLIT_HEAP_AUTO
 
 // The largest new region that is available to become Python heap.

ports/webassembly/mpconfigport.h

@@ -39,6 +39,7 @@
 #endif
 
 #define MICROPY_ALLOC_PATH_MAX      (256)
+#define MICROPY_COMP_ALLOW_TOP_LEVEL_AWAIT (1)
 #define MICROPY_READER_VFS          (MICROPY_VFS)
 #define MICROPY_ENABLE_GC           (1)
 #define MICROPY_ENABLE_PYSTACK      (1)

ports/webassembly/objjsproxy.c

@@ -32,6 +32,16 @@
 #include "py/runtime.h"
 #include "proxy_c.h"
 
+EM_JS(bool, has_attr, (int jsref, const char *str), {
+    const base = proxy_js_ref[jsref];
+    const attr = UTF8ToString(str);
+    if (attr in base) {
+        return true;
+    } else {
+        return false;
+    }
+});
+
 // *FORMAT-OFF*
 EM_JS(bool, lookup_attr, (int jsref, const char *str, uint32_t * out), {
     const base = proxy_js_ref[jsref];
@@ -299,18 +309,165 @@ static mp_obj_t jsproxy_it_iternext(mp_obj_t self_in) {
     }
 }
 
-static mp_obj_t jsproxy_getiter(mp_obj_t o_in, mp_obj_iter_buf_t *iter_buf) {
+static mp_obj_t jsproxy_new_it(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf) {
     assert(sizeof(jsproxy_it_t) <= sizeof(mp_obj_iter_buf_t));
+    mp_obj_jsproxy_t *self = MP_OBJ_TO_PTR(self_in);
     jsproxy_it_t *o = (jsproxy_it_t *)iter_buf;
     o->base.type = &mp_type_polymorph_iter;
     o->iternext = jsproxy_it_iternext;
-    o->ref = mp_obj_jsproxy_get_ref(o_in);
+    o->ref = self->ref;
     o->cur = 0;
-    o->len = js_get_len(o->ref);
+    o->len = js_get_len(self->ref);
     return MP_OBJ_FROM_PTR(o);
 }
 
 /******************************************************************************/
+// jsproxy generator
+
+enum {
+    JSOBJ_GEN_STATE_WAITING,
+    JSOBJ_GEN_STATE_COMPLETED,
+    JSOBJ_GEN_STATE_EXHAUSTED,
+};
+
+typedef struct _jsproxy_gen_t {
+    mp_obj_base_t base;
+    mp_obj_t thenable;
+    int state;
+} jsproxy_gen_t;
+
+mp_vm_return_kind_t jsproxy_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_obj_t throw_value, mp_obj_t *ret_val) {
+    jsproxy_gen_t *self = MP_OBJ_TO_PTR(self_in);
+    switch (self->state) {
+        case JSOBJ_GEN_STATE_WAITING:
+            self->state = JSOBJ_GEN_STATE_COMPLETED;
+            *ret_val = self->thenable;
+            return MP_VM_RETURN_YIELD;
+
+        case JSOBJ_GEN_STATE_COMPLETED:
+            self->state = JSOBJ_GEN_STATE_EXHAUSTED;
+            *ret_val = send_value;
+            return MP_VM_RETURN_NORMAL;
+
+        case JSOBJ_GEN_STATE_EXHAUSTED:
+        default:
+            // Trying to resume an already stopped generator.
+            // This is an optimised "raise StopIteration(None)".
+            *ret_val = mp_const_none;
+            return MP_VM_RETURN_NORMAL;
+    }
+}
+
+static mp_obj_t jsproxy_gen_resume_and_raise(mp_obj_t self_in, mp_obj_t send_value, mp_obj_t throw_value, bool raise_stop_iteration) {
+    mp_obj_t ret;
+    switch (jsproxy_gen_resume(self_in, send_value, throw_value, &ret)) {
+        case MP_VM_RETURN_NORMAL:
+        default:
+            // A normal return is a StopIteration, either raise it or return
+            // MP_OBJ_STOP_ITERATION as an optimisation.
+            if (ret == mp_const_none) {
+                ret = MP_OBJ_NULL;
+            }
+            if (raise_stop_iteration) {
+                mp_raise_StopIteration(ret);
+            } else {
+                return mp_make_stop_iteration(ret);
+            }
+
+        case MP_VM_RETURN_YIELD:
+            return ret;
+
+        case MP_VM_RETURN_EXCEPTION:
+            nlr_raise(ret);
+    }
+}
+
+static mp_obj_t jsproxy_gen_instance_iternext(mp_obj_t self_in) {
+    return jsproxy_gen_resume_and_raise(self_in, mp_const_none, MP_OBJ_NULL, false);
+}
+
+static mp_obj_t jsproxy_gen_instance_send(mp_obj_t self_in, mp_obj_t send_value) {
+    return jsproxy_gen_resume_and_raise(self_in, send_value, MP_OBJ_NULL, true);
+}
+static MP_DEFINE_CONST_FUN_OBJ_2(jsproxy_gen_instance_send_obj, jsproxy_gen_instance_send);
+
+static mp_obj_t jsproxy_gen_instance_throw(size_t n_args, const mp_obj_t *args) {
+    // The signature of this function is: throw(type[, value[, traceback]])
+    // CPython will pass all given arguments through the call chain and process them
+    // at the point they are used (native generators will handle them differently to
+    // user-defined generators with a throw() method).  To save passing multiple
+    // values, MicroPython instead does partial processing here to reduce it down to
+    // one argument and passes that through:
+    // - if only args[1] is given, or args[2] is given but is None, args[1] is
+    //   passed through (in the standard case it is an exception class or instance)
+    // - if args[2] is given and not None it is passed through (in the standard
+    //   case it would be an exception instance and args[1] its corresponding class)
+    // - args[3] is always ignored
+
+    mp_obj_t exc = args[1];
+    if (n_args > 2 && args[2] != mp_const_none) {
+        exc = args[2];
+    }
+
+    return jsproxy_gen_resume_and_raise(args[0], mp_const_none, exc, true);
+}
+static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(jsproxy_gen_instance_throw_obj, 2, 4, jsproxy_gen_instance_throw);
+
+static mp_obj_t jsproxy_gen_instance_close(mp_obj_t self_in) {
+    mp_obj_t ret;
+    switch (jsproxy_gen_resume(self_in, mp_const_none, MP_OBJ_FROM_PTR(&mp_const_GeneratorExit_obj), &ret)) {
+        case MP_VM_RETURN_YIELD:
+            mp_raise_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("generator ignored GeneratorExit"));
+
+        // Swallow GeneratorExit (== successful close), and re-raise any other
+        case MP_VM_RETURN_EXCEPTION:
+            // ret should always be an instance of an exception class
+            if (mp_obj_is_subclass_fast(MP_OBJ_FROM_PTR(mp_obj_get_type(ret)), MP_OBJ_FROM_PTR(&mp_type_GeneratorExit))) {
+                return mp_const_none;
+            }
+            nlr_raise(ret);
+
+        default:
+            // The only choice left is MP_VM_RETURN_NORMAL which is successful close
+            return mp_const_none;
+    }
+}
+static MP_DEFINE_CONST_FUN_OBJ_1(jsproxy_gen_instance_close_obj, jsproxy_gen_instance_close);
+
+static const mp_rom_map_elem_t jsproxy_gen_instance_locals_dict_table[] = {
+    { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&jsproxy_gen_instance_close_obj) },
+    { MP_ROM_QSTR(MP_QSTR_send), MP_ROM_PTR(&jsproxy_gen_instance_send_obj) },
+    { MP_ROM_QSTR(MP_QSTR_throw), MP_ROM_PTR(&jsproxy_gen_instance_throw_obj) },
+};
+static MP_DEFINE_CONST_DICT(jsproxy_gen_instance_locals_dict, jsproxy_gen_instance_locals_dict_table);
+
+MP_DEFINE_CONST_OBJ_TYPE(
+    mp_type_jsproxy_gen,
+    MP_QSTR_generator,
+    MP_TYPE_FLAG_ITER_IS_ITERNEXT,
+    iter, jsproxy_gen_instance_iternext,
+    locals_dict, &jsproxy_gen_instance_locals_dict
+    );
+
+static mp_obj_t jsproxy_new_gen(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf) {
+    assert(sizeof(jsproxy_gen_t) <= sizeof(mp_obj_iter_buf_t));
+    jsproxy_gen_t *o = (jsproxy_gen_t *)iter_buf;
+    o->base.type = &mp_type_jsproxy_gen;
+    o->thenable = self_in;
+    o->state = JSOBJ_GEN_STATE_WAITING;
+    return MP_OBJ_FROM_PTR(o);
+}
+
+/******************************************************************************/
+
+static mp_obj_t jsproxy_getiter(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf) {
+    mp_obj_jsproxy_t *self = MP_OBJ_TO_PTR(self_in);
+    if (has_attr(self->ref, "then")) {
+        return jsproxy_new_gen(self_in, iter_buf);
+    } else {
+        return jsproxy_new_it(self_in, iter_buf);
+    }
+}
 
 MP_DEFINE_CONST_OBJ_TYPE(
     mp_type_jsproxy,

ports/webassembly/objpyproxy.js

@@ -159,6 +159,9 @@ const py_proxy_handler = {
         if (prop === "_ref") {
             return target._ref;
         }
+        if (prop === "then") {
+            return null;
+        }
         const value = Module._malloc(3 * 4);
         Module.ccall(
             "proxy_c_to_js_lookup_attr",
@@ -189,3 +192,23 @@ const py_proxy_handler = {
         );
     },
 };
+
+// PyProxy of a Python generator, that implements the thenable interface.
+class PyProxyThenable {
+    constructor(ref) {
+        this._ref = ref;
+    }
+
+    then(resolve, reject) {
+        const values = Module._malloc(3 * 3 * 4);
+        proxy_convert_js_to_mp_obj_jsside(resolve, values + 3 * 4);
+        proxy_convert_js_to_mp_obj_jsside(reject, values + 2 * 3 * 4);
+        Module.ccall(
+            "proxy_c_to_js_resume",
+            "null",
+            ["number", "pointer"],
+            [this._ref, values],
+        );
+        return proxy_convert_mp_to_js_obj_jsside_with_free(values);
+    }
+}

ports/webassembly/proxy_c.c

@@ -27,6 +27,7 @@
 #include <stdlib.h>
 #include <string.h>
 
+#include "emscripten.h"
 #include "py/builtin.h"
 #include "py/runtime.h"
 #include "proxy_c.h"
@@ -42,8 +43,9 @@ enum {
     PROXY_KIND_MP_FLOAT = 4,
     PROXY_KIND_MP_STR = 5,
     PROXY_KIND_MP_CALLABLE = 6,
-    PROXY_KIND_MP_OBJECT = 7,
-    PROXY_KIND_MP_JSPROXY = 8,
+    PROXY_KIND_MP_GENERATOR = 7,
+    PROXY_KIND_MP_OBJECT = 8,
+    PROXY_KIND_MP_JSPROXY = 9,
 };
 
 enum {
@@ -115,6 +117,8 @@ void proxy_convert_mp_to_js_obj_cside(mp_obj_t obj, uint32_t *out) {
     } else {
         if (mp_obj_is_callable(obj)) {
             kind = PROXY_KIND_MP_CALLABLE;
+        } else if (mp_obj_is_type(obj, &mp_type_gen_instance)) {
+            kind = PROXY_KIND_MP_GENERATOR;
         } else {
             kind = PROXY_KIND_MP_OBJECT;
         }
@@ -279,3 +283,78 @@ void proxy_c_to_js_get_dict(uint32_t c_ref, uint32_t *out) {
     out[0] = map->alloc;
     out[1] = (uintptr_t)map->table;
 }
+
+/******************************************************************************/
+// Bridge Python generator to JavaScript thenable.
+
+static const mp_obj_fun_builtin_var_t resume_obj;
+
+EM_JS(void, js_then_resolve, (uint32_t * resolve, uint32_t * reject), {
+    const resolve_js = proxy_convert_mp_to_js_obj_jsside(resolve);
+    const reject_js = proxy_convert_mp_to_js_obj_jsside(reject);
+    resolve_js(null);
+});
+
+EM_JS(void, js_then_reject, (uint32_t * resolve, uint32_t * reject), {
+    const resolve_js = proxy_convert_mp_to_js_obj_jsside(resolve);
+    const reject_js = proxy_convert_mp_to_js_obj_jsside(reject);
+    reject_js(null);
+});
+
+// *FORMAT-OFF*
+EM_JS(void, js_then_continue, (int jsref, uint32_t * py_resume, uint32_t * resolve, uint32_t * reject, uint32_t * out), {
+    const py_resume_js = proxy_convert_mp_to_js_obj_jsside(py_resume);
+    const resolve_js = proxy_convert_mp_to_js_obj_jsside(resolve);
+    const reject_js = proxy_convert_mp_to_js_obj_jsside(reject);
+    const ret = proxy_js_ref[jsref].then((x) => {py_resume_js(x, resolve_js, reject_js);}, reject_js);
+    proxy_convert_js_to_mp_obj_jsside(ret, out);
+});
+// *FORMAT-ON*
+
+static mp_obj_t proxy_resume_execute(mp_obj_t self_in, mp_obj_t value, mp_obj_t resolve, mp_obj_t reject) {
+    mp_obj_t ret_value;
+    mp_vm_return_kind_t ret_kind = mp_resume(self_in, value, MP_OBJ_NULL, &ret_value);
+
+    uint32_t out_resolve[PVN];
+    uint32_t out_reject[PVN];
+    proxy_convert_mp_to_js_obj_cside(resolve, out_resolve);
+    proxy_convert_mp_to_js_obj_cside(reject, out_reject);
+
+    if (ret_kind == MP_VM_RETURN_NORMAL) {
+        js_then_resolve(out_resolve, out_reject);
+        return mp_const_none;
+    } else if (ret_kind == MP_VM_RETURN_YIELD) {
+        // ret_value should be a JS thenable
+        mp_obj_t py_resume = mp_obj_new_bound_meth(MP_OBJ_FROM_PTR(&resume_obj), self_in);
+        int ref = mp_obj_jsproxy_get_ref(ret_value);
+        uint32_t out_py_resume[PVN];
+        proxy_convert_mp_to_js_obj_cside(py_resume, out_py_resume);
+        uint32_t out[PVN];
+        js_then_continue(ref, out_py_resume, out_resolve, out_reject, out);
+        return proxy_convert_js_to_mp_obj_cside(out);
+    } else {
+        // MP_VM_RETURN_EXCEPTION;
+        js_then_reject(out_resolve, out_reject);
+        nlr_raise(ret_value);
+    }
+}
+
+static mp_obj_t resume_fun(size_t n_args, const mp_obj_t *args) {
+    return proxy_resume_execute(args[0], args[1], args[2], args[3]);
+}
+static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(resume_obj, 4, 4, resume_fun);
+
+void proxy_c_to_js_resume(uint32_t c_ref, uint32_t *args) {
+    nlr_buf_t nlr;
+    if (nlr_push(&nlr) == 0) {
+        mp_obj_t obj = proxy_c_get_obj(c_ref);
+        mp_obj_t resolve = proxy_convert_js_to_mp_obj_cside(args + 1 * 3);
+        mp_obj_t reject = proxy_convert_js_to_mp_obj_cside(args + 2 * 3);
+        mp_obj_t ret = proxy_resume_execute(obj, mp_const_none, resolve, reject);
+        nlr_pop();
+        return proxy_convert_mp_to_js_obj_cside(ret, args);
+    } else {
+        // uncaught exception
+        return proxy_convert_mp_to_js_exc_cside(nlr.ret_val, args);
+    }
+}

ports/webassembly/proxy_js.js

@@ -34,8 +34,9 @@ const PROXY_KIND_MP_INT = 3;
 const PROXY_KIND_MP_FLOAT = 4;
 const PROXY_KIND_MP_STR = 5;
 const PROXY_KIND_MP_CALLABLE = 6;
-const PROXY_KIND_MP_OBJECT = 7;
-const PROXY_KIND_MP_JSPROXY = 8;
+const PROXY_KIND_MP_GENERATOR = 7;
+const PROXY_KIND_MP_OBJECT = 8;
+const PROXY_KIND_MP_JSPROXY = 9;
 
 const PROXY_KIND_JS_NULL = 1;
 const PROXY_KIND_JS_BOOLEAN = 2;
@@ -122,6 +123,9 @@ function proxy_convert_js_to_mp_obj_jsside(js_obj, out) {
     } else if (js_obj instanceof PyProxy) {
         kind = PROXY_KIND_JS_PYPROXY;
         Module.setValue(out + 4, js_obj._ref, "i32");
+    } else if (js_obj instanceof PyProxyThenable) {
+        kind = PROXY_KIND_JS_PYPROXY;
+        Module.setValue(out + 4, js_obj._ref, "i32");
     } else {
         kind = PROXY_KIND_JS_OBJECT;
         const id = proxy_js_ref.length;
@@ -193,6 +197,8 @@ function proxy_convert_mp_to_js_obj_jsside(value) {
             obj = (...args) => {
                 return proxy_call_python(id, args);
             };
+        } else if (kind === PROXY_KIND_MP_GENERATOR) {
+            obj = new PyProxyThenable(id);
         } else {
             // PROXY_KIND_MP_OBJECT
             const target = new PyProxy(id);