diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py index 6a671e0867..7952e248a5 100644 --- a/py/makeqstrdata.py +++ b/py/makeqstrdata.py @@ -23,6 +23,7 @@ codepoint2name[ord('}')] = 'brace_close' codepoint2name[ord('*')] = 'star' # this must match the equivalent function in qstr.c +# Note that this hashes the UTF-8 encoded data bytes. def compute_hash(qstr): hash = 5381 for char in qstr: @@ -58,7 +59,8 @@ def do_work(infiles): for order, ident, qstr in sorted(qstrs.values(), key=lambda x: x[0]): qhash = compute_hash(qstr) qlen = len(qstr) - print('Q({}, (const byte*)"\\x{:02x}\\x{:02x}\\x{:02x}\\x{:02x}\\1" "{}")'.format(ident, qhash & 0xff, (qhash >> 8) & 0xff, qlen & 0xff, (qlen >> 8) & 0xff, qstr)) + qchlen = len(qstr.decode("utf-8")) + print('Q({}, (const byte*)"\\x{:02x}\\x{:02x}\\x{:02x}\\x{:02x}\\x{:02x}\\x{:02x}\\1" "{}")'.format(ident, qhash & 0xff, (qhash >> 8) & 0xff, qlen & 0xff, (qlen >> 8) & 0xff, qchlen & 0xff, (qchlen >> 8) & 0xff, qstr)) return True diff --git a/py/objstr.c b/py/objstr.c index dcaad1482b..857028c593 100644 --- a/py/objstr.c +++ b/py/objstr.c @@ -54,7 +54,7 @@ const mp_obj_t mp_const_empty_bytes; // use this macro to extract the string data, lengths, and flags // NOTE: Currently buggy as regards qstr, which doesn't record a charlen -#define GET_STR_INFO(str_obj_in, str_data, str_len, str_charlen, str_flags) const byte *str_data; uint str_len, str_charlen = -1; char str_flags; if (MP_OBJ_IS_QSTR(str_obj_in)) { str_data = qstr_data(MP_OBJ_QSTR_VALUE(str_obj_in), &str_len, &str_flags); } else { str_len = ((mp_obj_str_t*)str_obj_in)->len; str_charlen = ((mp_obj_str_t*)str_obj_in)->charlen; str_data = ((mp_obj_str_t*)str_obj_in)->data; str_flags = ((mp_obj_str_t*)str_obj_in)->flags; } +#define GET_STR_INFO(str_obj_in, str_data, str_len, str_charlen, str_flags) const byte *str_data; uint str_len, str_charlen; char str_flags; if (MP_OBJ_IS_QSTR(str_obj_in)) { str_data = qstr_data(MP_OBJ_QSTR_VALUE(str_obj_in), &str_len, &str_flags); str_charlen = qstr_charlen(MP_OBJ_QSTR_VALUE(str_obj_in)); } else { str_len = ((mp_obj_str_t*)str_obj_in)->len; str_charlen = ((mp_obj_str_t*)str_obj_in)->charlen; str_data = ((mp_obj_str_t*)str_obj_in)->data; str_flags = ((mp_obj_str_t*)str_obj_in)->flags; } // don't use this macro, it's only for conversions #define GET_STR_DATA_LEN(str_obj_in, str_data, str_len) GET_STR_DATA_LEN_FLAGS(str_obj_in, str_data, str_len, str_data ## _flags); assert(str_data ## _flags == 1); @@ -372,17 +372,6 @@ STATIC mp_obj_t str_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) { return mp_obj_new_str_of_type(type, self_data + slice.start, slice.stop - slice.start); } #endif - if (self_charlen == (uint)-1) - { - // HACK: Since qstr doesn't yet retain character length, count it up now. - // This allows tests to pass, but it's stupidly inefficient. - // (It's also safe. If charlen just happens to be (uint)-1, it won't - // break anything, it'll just recalculate it here.) - const byte *endptr, *top = self_data + self_len; - self_charlen = 0; - for (endptr = self_data; endptr < top; ++endptr) - if ((*endptr & 0xC0) != 0x80) ++self_charlen; - } uint index_val = mp_get_index(type, self_charlen, index, false); if (type == &mp_type_bytes) { return MP_OBJ_NEW_SMALL_INT((mp_small_int_t)self_data[index_val]); @@ -1833,18 +1822,7 @@ uint mp_obj_str_get_hash(mp_obj_t self_in) { uint mp_obj_str_get_len(mp_obj_t self_in) { // TODO This has a double check for the type, one in obj.c and one here if (MP_OBJ_IS_STR(self_in) || MP_OBJ_IS_TYPE(self_in, &mp_type_bytes)) { - GET_STR_INFO(self_in, self_data, self_len, self_charlen, self_flags); - if (self_charlen == (uint)-1) - { - // HACK: Since qstr doesn't yet retain character length, count it up now. - // This allows tests to pass, but it's stupidly inefficient. - // (It's also safe. If charlen just happens to be (uint)-1, it won't - // break anything, it'll just recalculate it here.) - const byte *endptr, *top = self_data + self_len; - self_charlen = 0; - for (endptr = self_data; endptr < top; ++endptr) - if ((*endptr & 0xC0) != 0x80) ++self_charlen; - } + GET_STR_INFO(self_in, self_data, self_len, self_charlen, self_flags); (void)self_data; return self_charlen; } else { bad_implicit_conversion(self_in); diff --git a/py/qstr.c b/py/qstr.c index 3566f28cb3..4013a67f79 100644 --- a/py/qstr.c +++ b/py/qstr.c @@ -46,17 +46,20 @@ // For now we use very simple encoding, just to get the framework correct: // - hash is 2 bytes (see function below) // - length is 2 bytes +// - character length is 2 bytes // - flags byte // - data follows // - \0 terminated (for now, so they can be printed using printf) #define Q_GET_HASH(q) ((q)[0] | ((q)[1] << 8)) -#define Q_GET_ALLOC(q) (4 + Q_GET_LENGTH(q) + 1) +#define Q_GET_ALLOC(q) (7 + Q_GET_LENGTH(q) + 1) #define Q_GET_LENGTH(q) ((q)[2] | ((q)[3] << 8)) -#define Q_GET_FLAGS(q) ((q)[4]) -#define Q_GET_DATA(q) ((q) + 5) +#define Q_GET_CHARLEN(q) ((q)[4] | ((q)[5] << 8)) +#define Q_GET_FLAGS(q) ((q)[6]) +#define Q_GET_DATA(q) ((q) + 7) // this must match the equivalent function in makeqstrdata.py +// Note that this hashes the UTF-8 encoded data bytes. machine_uint_t qstr_compute_hash(const byte *data, uint len) { // djb2 algorithm; see http://www.cse.yorku.ca/~oz/hash.html machine_uint_t hash = 5381; @@ -85,8 +88,8 @@ const static qstr_pool_t const_pool = { 10, // set so that the first dynamically allocated pool is twice this size; must be <= the len (just below) MP_QSTR_number_of, // corresponds to number of strings in array just below { - (const byte*) "\0\0\0\0\0", // invalid/no qstr has empty data - (const byte*) "\0\0\0\0\1", // empty qstr + (const byte*) "\0\0\0\0\0\0\0", // invalid/no qstr has empty data + (const byte*) "\0\0\0\0\0\0\1", // empty qstr #define Q(id, str) str, #include "genhdr/qstrdefs.generated.h" #undef Q @@ -157,14 +160,19 @@ qstr qstr_from_strn(const char *str, uint len) { qstr q = qstr_find_strn(str, len); if (q == 0) { machine_uint_t hash = qstr_compute_hash((const byte*)str, len); - byte *q_ptr = m_new(byte, 4 + len + 1); + byte *q_ptr = m_new(byte, 7 + len + 1); + uint charlen = 0; + for (const char *s = str; s < str + len; ++s) + if ((*s & 0xC0) != 0x80) ++charlen; q_ptr[0] = hash; q_ptr[1] = hash >> 8; q_ptr[2] = len; q_ptr[3] = len >> 8; - q_ptr[4] = 1; - memcpy(q_ptr + 5, str, len); - q_ptr[5 + len] = '\0'; + q_ptr[4] = charlen; + q_ptr[5] = charlen >> 8; + q_ptr[6] = 1; + memcpy(q_ptr + 7, str, len); + q_ptr[7 + len] = '\0'; q = qstr_add(q_ptr); } return q; @@ -172,7 +180,7 @@ qstr qstr_from_strn(const char *str, uint len) { byte *qstr_build_start(uint len, byte **q_ptr) { assert(len <= 65535); - *q_ptr = m_new(byte, 4 + len + 1); + *q_ptr = m_new(byte, 7 + len + 1); (*q_ptr)[2] = len; (*q_ptr)[3] = len >> 8; return Q_GET_DATA(*q_ptr); @@ -182,11 +190,17 @@ qstr qstr_build_end(byte *q_ptr) { qstr q = qstr_find_strn((const char*)Q_GET_DATA(q_ptr), Q_GET_LENGTH(q_ptr)); if (q == 0) { machine_uint_t len = Q_GET_LENGTH(q_ptr); - machine_uint_t hash = qstr_compute_hash(Q_GET_DATA(q_ptr), len); + const byte *str = Q_GET_DATA(q_ptr); + machine_uint_t hash = qstr_compute_hash(str, len); q_ptr[0] = hash; q_ptr[1] = hash >> 8; - q_ptr[4] = 1; - q_ptr[5 + len] = '\0'; + uint charlen = 0; + for (const byte *s = str; s < str + len; ++s) + if ((*s & 0xC0) != 0x80) ++charlen; + q_ptr[4] = charlen; + q_ptr[5] = charlen >> 8; + q_ptr[6] = 1; + q_ptr[7 + len] = '\0'; q = qstr_add(q_ptr); } else { m_del(byte, q_ptr, Q_GET_ALLOC(q_ptr)); @@ -203,6 +217,11 @@ uint qstr_len(qstr q) { return Q_GET_LENGTH(qd); } +uint qstr_charlen(qstr q) { + const byte *qd = find_qstr(q); + return Q_GET_CHARLEN(qd); +} + // XXX to remove! const char *qstr_str(qstr q) { const byte *qd = find_qstr(q); diff --git a/py/qstr.h b/py/qstr.h index 2d2687fede..1f641832f3 100644 --- a/py/qstr.h +++ b/py/qstr.h @@ -59,6 +59,7 @@ qstr qstr_build_end(byte *q_ptr); machine_uint_t qstr_hash(qstr q); const char* qstr_str(qstr q); uint qstr_len(qstr q); +uint qstr_charlen(qstr q); const byte* qstr_data(qstr q, uint *len, char *flags); void qstr_pool_info(uint *n_pool, uint *n_qstr, uint *n_str_data_bytes, uint *n_total_bytes);