From 6ea0e928d8aced4f8ce5ab451105c199092eb6df Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Fri, 11 Apr 2014 20:36:08 +0300 Subject: [PATCH 1/7] Revert "makeqstrdata.py: Add support for conditionally defined qstrs." This reverts commit acb133d1b1a68847bd85c545312c3e221a6f7c0b. Conditionals will be suported using C preprocessor. --- py/makeqstrdata.py | 23 ++--------------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py index e60f000440..7413365712 100644 --- a/py/makeqstrdata.py +++ b/py/makeqstrdata.py @@ -29,7 +29,6 @@ def do_work(infiles): for infile in infiles: with open(infile, 'rt') as f: line_number = 0 - conditional = None for line in f: line_number += 1 line = line.strip() @@ -38,18 +37,6 @@ def do_work(infiles): if len(line) == 0 or line.startswith('//'): continue - if line[0] == '#': - if conditional == "": - assert line == "#endif" - conditional = None - else: - assert conditional is None - conditional = line - continue - - if conditional == "": - assert False, "#endif expected before '%s'" % line - # verify line is of the correct form match = re.match(r'Q\((.+)\)$', line) if not match: @@ -65,21 +52,15 @@ def do_work(infiles): continue # add the qstr to the list, with order number to retain original order in file - qstrs[ident] = (len(qstrs), ident, qstr, conditional) - if conditional is not None: - conditional = "" + qstrs[ident] = (len(qstrs), ident, qstr) # process the qstrs, printing out the generated C header file print('// This file was automatically generated by makeqstrdata.py') print('') - for order, ident, qstr, conditional in sorted(qstrs.values(), key=lambda x: x[0]): + for order, ident, qstr in sorted(qstrs.values(), key=lambda x: x[0]): qhash = compute_hash(qstr) qlen = len(qstr) - if conditional: - print(conditional) print('Q({}, (const byte*)"\\x{:02x}\\x{:02x}\\x{:02x}\\x{:02x}" "{}")'.format(ident, qhash & 0xff, (qhash >> 8) & 0xff, qlen & 0xff, (qlen >> 8) & 0xff, qstr)) - if conditional: - print('#endif') return True From a925cb54f1b3490a9e082816eb785750c4b25324 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Fri, 11 Apr 2014 20:50:15 +0300 Subject: [PATCH 2/7] py: Preprocess qstrdefs.h before feeding to makeqstrdata.py. This is alternative implementation of supporting conditionals in qstrdefs.h, hard to say if it's much cleaner than munging #ifdef's in Python code... --- py/makeqstrdata.py | 9 +++++++++ py/py.mk | 5 +++-- py/qstrdefs.h | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py index 7413365712..81b0035451 100644 --- a/py/makeqstrdata.py +++ b/py/makeqstrdata.py @@ -26,6 +26,7 @@ def compute_hash(qstr): def do_work(infiles): # read the qstrs in from the input files qstrs = {} + cpp_header_blocks = 3 for infile in infiles: with open(infile, 'rt') as f: line_number = 0 @@ -37,6 +38,14 @@ def do_work(infiles): if len(line) == 0 or line.startswith('//'): continue + # We'll have 3 line-number lines for py/qstrdefs.h - initial, leaving it to + # go into other headers, and returning to it. + if line.startswith('# ') and 'py/qstrdefs.h' in line: + cpp_header_blocks -= 1 + continue + if cpp_header_blocks != 0: + continue + # verify line is of the correct form match = re.match(r'Q\((.+)\)$', line) if not match: diff --git a/py/py.mk b/py/py.mk index ecc4a6a0dc..23ba9ebe74 100644 --- a/py/py.mk +++ b/py/py.mk @@ -104,9 +104,10 @@ $(PY_BUILD)/py-version.h: FORCE # Adding an order only dependency on $(PY_BUILD) causes $(PY_BUILD) to get # created before we run the script to generate the .h $(PY_BUILD)/qstrdefs.generated.h: | $(PY_BUILD)/ -$(PY_BUILD)/qstrdefs.generated.h: $(PY_QSTR_DEFS) $(QSTR_DEFS) $(PY_SRC)/makeqstrdata.py +$(PY_BUILD)/qstrdefs.generated.h: $(PY_QSTR_DEFS) $(QSTR_DEFS) $(PY_SRC)/makeqstrdata.py mpconfigport.h $(PY_SRC)/mpconfig.h $(ECHO) "makeqstrdata $(PY_QSTR_DEFS) $(QSTR_DEFS)" - $(Q)$(PYTHON) $(PY_SRC)/makeqstrdata.py $(PY_QSTR_DEFS) $(QSTR_DEFS) > $@ + $(CPP) $(CFLAGS) $(PY_QSTR_DEFS) -o $(PY_BUILD)/qstrdefs.preprocessed.h + $(Q)$(PYTHON) $(PY_SRC)/makeqstrdata.py $(PY_BUILD)/qstrdefs.preprocessed.h $(QSTR_DEFS) > $@ # We don't know which source files actually need the generated.h (since # it is #included from str.h). The compiler generated dependencies will cause diff --git a/py/qstrdefs.h b/py/qstrdefs.h index 17948434b4..eb72fd265f 100644 --- a/py/qstrdefs.h +++ b/py/qstrdefs.h @@ -1,6 +1,6 @@ +#include "mpconfig.h" // All the qstr definitions in this file are available as constants. // That is, they are in ROM and you can reference them simply as MP_QSTR_xxxx. -// TODO make it so we can use #defines here to select only those words that will be used Q(__build_class__) Q(__class__) From e0813290980deeefd6a104193aece6b944945f0c Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Fri, 11 Apr 2014 23:08:29 +0300 Subject: [PATCH 3/7] builtinimport: Elaborate debug output support. --- py/builtinimport.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/py/builtinimport.c b/py/builtinimport.c index 5102a9bdec..f7c5e4a1eb 100644 --- a/py/builtinimport.c +++ b/py/builtinimport.c @@ -19,6 +19,13 @@ #include "runtime.h" #include "builtin.h" +#if 0 // print debugging info +#define DEBUG_PRINT (1) +#define DEBUG_printf DEBUG_printf +#else // don't print debugging info +#define DEBUG_printf(...) (void)0 +#endif + #define PATH_SEP_CHAR '/' mp_obj_t mp_sys_path; @@ -129,14 +136,14 @@ void do_load(mp_obj_t module_obj, vstr_t *file) { } mp_obj_t mp_builtin___import__(uint n_args, mp_obj_t *args) { - /* - printf("import:\n"); +#if DEBUG_PRINT + printf("__import__:\n"); for (int i = 0; i < n_args; i++) { printf(" "); mp_obj_print(args[i], PRINT_REPR); printf("\n"); } - */ +#endif mp_obj_t fromtuple = mp_const_none; int level = 0; @@ -158,6 +165,7 @@ mp_obj_t mp_builtin___import__(uint n_args, mp_obj_t *args) { // check if module already exists mp_obj_t module_obj = mp_module_get(mp_obj_str_get_qstr(args[0])); if (module_obj != MP_OBJ_NULL) { + DEBUG_printf("Module already loaded\n"); // If it's not a package, return module right away char *p = strchr(mod_str, '.'); if (p == NULL) { @@ -171,6 +179,7 @@ mp_obj_t mp_builtin___import__(uint n_args, mp_obj_t *args) { qstr pkg_name = qstr_from_strn(mod_str, p - mod_str); return mp_module_get(pkg_name); } + DEBUG_printf("Module not yet loaded\n"); uint last = 0; VSTR_FIXED(path, MICROPY_PATH_MAX) @@ -182,6 +191,7 @@ mp_obj_t mp_builtin___import__(uint n_args, mp_obj_t *args) { if (i == mod_len || mod_str[i] == '.') { // create a qstr for the module name up to this depth qstr mod_name = qstr_from_strn(mod_str, i); + DEBUG_printf("Processing module: %s\n", qstr_str(mod_name)); // find the file corresponding to the module name mp_import_stat_t stat; @@ -207,6 +217,7 @@ mp_obj_t mp_builtin___import__(uint n_args, mp_obj_t *args) { module_obj = mp_obj_new_module(mod_name); if (stat == MP_IMPORT_STAT_DIR) { + DEBUG_printf("%s is dir\n", vstr_str(&path)); vstr_add_char(&path, PATH_SEP_CHAR); vstr_add_str(&path, "__init__.py"); if (mp_import_stat(vstr_str(&path)) != MP_IMPORT_STAT_FILE) { From 13d52df4c54cc6360a5c9c0bdbf7a3abb45c22d0 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Fri, 11 Apr 2014 23:25:35 +0300 Subject: [PATCH 4/7] builtinimport: Set __path__ attribute on packages. Per https://docs.python.org/3.3/reference/import.html , this is the way to tell module from package: "Specifically, any module that contains a __path__ attribute is considered a package." And it for sure will be needed to implement relative imports. --- py/builtinimport.c | 3 +++ py/qstrdefs.h | 1 + 2 files changed, 4 insertions(+) diff --git a/py/builtinimport.c b/py/builtinimport.c index f7c5e4a1eb..501ced7644 100644 --- a/py/builtinimport.c +++ b/py/builtinimport.c @@ -228,6 +228,9 @@ mp_obj_t mp_builtin___import__(uint n_args, mp_obj_t *args) { } do_load(module_obj, &path); vstr_cut_tail_bytes(&path, sizeof("/__init__.py") - 1); // cut off /__init__.py + // https://docs.python.org/3.3/reference/import.html + // "Specifically, any module that contains a __path__ attribute is considered a package." + mp_store_attr(module_obj, MP_QSTR___path__, mp_obj_new_str((byte*)vstr_str(&path), vstr_len(&path), false)); } else { // MP_IMPORT_STAT_FILE do_load(module_obj, &path); // TODO: We cannot just break here, at the very least, we must execute diff --git a/py/qstrdefs.h b/py/qstrdefs.h index eb72fd265f..d52b870e65 100644 --- a/py/qstrdefs.h +++ b/py/qstrdefs.h @@ -13,6 +13,7 @@ Q(__module__) Q(__name__) Q(__next__) Q(__qualname__) +Q(__path__) Q(__repl_print__) Q(__bool__) From af620abcb576c80553ac8b8bbdfd5ba66c091e34 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 12 Apr 2014 00:15:19 +0300 Subject: [PATCH 5/7] py: Implement "from pkg import mod" variant of import. --- py/runtime.c | 44 ++++++++++++++++++++++++++++++++----- tests/basics/import-pkg3.py | 6 +++++ 2 files changed, 44 insertions(+), 6 deletions(-) create mode 100644 tests/basics/import-pkg3.py diff --git a/py/runtime.c b/py/runtime.c index 499905a0fa..4793f054a8 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -1043,13 +1043,45 @@ mp_obj_t mp_import_name(qstr name, mp_obj_t fromlist, mp_obj_t level) { mp_obj_t mp_import_from(mp_obj_t module, qstr name) { DEBUG_printf("import from %p %s\n", module, qstr_str(name)); - mp_obj_t x = mp_load_attr(module, name); - /* TODO convert AttributeError to ImportError - if (fail) { - (ImportError, "cannot import name %s", qstr_str(name), NULL) + mp_obj_t dest[2]; + + mp_load_method_maybe(module, name, dest); + + if (dest[1] != MP_OBJ_NULL) { + // Hopefully we can't import bound method from an object +import_error: + nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_ImportError, "Cannot import name '%s'", qstr_str(name))); } - */ - return x; + + if (dest[0] != MP_OBJ_NULL) { + return dest[0]; + } + + // See if it's a package, then can try FS import + mp_load_method_maybe(module, MP_QSTR___path__, dest); + if (dest[0] == MP_OBJ_NULL) { + goto import_error; + } + + mp_load_method_maybe(module, MP_QSTR___name__, dest); + uint pkg_name_len; + const char *pkg_name = mp_obj_str_get_data(dest[0], &pkg_name_len); + + char dot_name[pkg_name_len + 1 + qstr_len(name)]; + memcpy(dot_name, pkg_name, pkg_name_len); + dot_name[pkg_name_len] = '.'; + memcpy(dot_name + pkg_name_len + 1, qstr_str(name), qstr_len(name)); + qstr dot_name_q = qstr_from_strn(dot_name, sizeof(dot_name)); + + mp_obj_t args[5]; + args[0] = MP_OBJ_NEW_QSTR(dot_name_q); + args[1] = mp_const_none; // TODO should be globals + args[2] = mp_const_none; // TODO should be locals + args[3] = mp_const_true; // Pass sentinel "non empty" value to force returning of leaf module + args[4] = 0; + + // TODO lookup __import__ and call that instead of going straight to builtin implementation + return mp_builtin___import__(5, args); } void mp_import_all(mp_obj_t module) { diff --git a/tests/basics/import-pkg3.py b/tests/basics/import-pkg3.py new file mode 100644 index 0000000000..0ee885b220 --- /dev/null +++ b/tests/basics/import-pkg3.py @@ -0,0 +1,6 @@ +from pkg import mod + +print(mod.foo()) + +import pkg.mod +print(mod is pkg.mod) From 00a9d138b2e66c9e144d30f52ed9dfe8863a6143 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 12 Apr 2014 00:32:38 +0300 Subject: [PATCH 6/7] showbc: Dump LOAD_NULL. --- py/showbc.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/py/showbc.c b/py/showbc.c index c1e420f433..8d6e867b14 100644 --- a/py/showbc.c +++ b/py/showbc.c @@ -106,6 +106,10 @@ void mp_byte_code_print(const byte *ip, int len) { printf("LOAD_CONST_STRING %s", qstr_str(qstr)); break; + case MP_BC_LOAD_NULL: + printf("LOAD_NULL"); + break; + case MP_BC_LOAD_FAST_0: printf("LOAD_FAST_0"); break; From b9b1c00c8af9e4d654aed30f8f98875657f14891 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 12 Apr 2014 00:34:57 +0300 Subject: [PATCH 7/7] showbs: Dump LOAD_CONST_BYTES. --- py/showbc.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/py/showbc.c b/py/showbc.c index 8d6e867b14..08dec1e2bf 100644 --- a/py/showbc.c +++ b/py/showbc.c @@ -101,6 +101,11 @@ void mp_byte_code_print(const byte *ip, int len) { printf("LOAD_CONST_ID %s", qstr_str(qstr)); break; + case MP_BC_LOAD_CONST_BYTES: + DECODE_QSTR; + printf("LOAD_CONST_BYTES %s", qstr_str(qstr)); + break; + case MP_BC_LOAD_CONST_STRING: DECODE_QSTR; printf("LOAD_CONST_STRING %s", qstr_str(qstr));