mirror
/
micropython
kopia lustrzana https://github.com/micropython/micropython
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Add character length to qstr

pull/671/head
Chris Angelico 2014-06-07 05:09:35 +10:00
rodzic 6df1b946fb
commit 231031ac5f
4 zmienionych plików z 38 dodań i 38 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

4
py/makeqstrdata.py
Wyświetl plik

@ -23,6 +23,7 @@ codepoint2name[ord('}')] = 'brace_close'
codepoint2name[ord('*')] = 'star'
# this must match the equivalent function in qstr.c
# Note that this hashes the UTF-8 encoded data bytes.
def compute_hash(qstr):
hash = 5381
for char in qstr:
@ -58,7 +59,8 @@ def do_work(infiles):
for order, ident, qstr in sorted(qstrs.values(), key=lambda x: x[0]):
qhash = compute_hash(qstr)
qlen = len(qstr)
print('Q({}, (const byte*)"\\x{:02x}\\x{:02x}\\x{:02x}\\x{:02x}\\1" "{}")'.format(ident, qhash & 0xff, (qhash >> 8) & 0xff, qlen & 0xff, (qlen >> 8) & 0xff, qstr))
qchlen = len(qstr.decode("utf-8"))
print('Q({}, (const byte*)"\\x{:02x}\\x{:02x}\\x{:02x}\\x{:02x}\\x{:02x}\\x{:02x}\\1" "{}")'.format(ident, qhash & 0xff, (qhash >> 8) & 0xff, qlen & 0xff, (qlen >> 8) & 0xff, qchlen & 0xff, (qchlen >> 8) & 0xff, qstr))
return True

26
py/objstr.c
Wyświetl plik

@ -54,7 +54,7 @@ const mp_obj_t mp_const_empty_bytes;
// use this macro to extract the string data, lengths, and flags
// NOTE: Currently buggy as regards qstr, which doesn't record a charlen
#define GET_STR_INFO(str_obj_in, str_data, str_len, str_charlen, str_flags) const byte *str_data; uint str_len, str_charlen = -1; char str_flags; if (MP_OBJ_IS_QSTR(str_obj_in)) { str_data = qstr_data(MP_OBJ_QSTR_VALUE(str_obj_in), &str_len, &str_flags); } else { str_len = ((mp_obj_str_t*)str_obj_in)->len; str_charlen = ((mp_obj_str_t*)str_obj_in)->charlen; str_data = ((mp_obj_str_t*)str_obj_in)->data; str_flags = ((mp_obj_str_t*)str_obj_in)->flags; }
#define GET_STR_INFO(str_obj_in, str_data, str_len, str_charlen, str_flags) const byte *str_data; uint str_len, str_charlen; char str_flags; if (MP_OBJ_IS_QSTR(str_obj_in)) { str_data = qstr_data(MP_OBJ_QSTR_VALUE(str_obj_in), &str_len, &str_flags); str_charlen = qstr_charlen(MP_OBJ_QSTR_VALUE(str_obj_in)); } else { str_len = ((mp_obj_str_t*)str_obj_in)->len; str_charlen = ((mp_obj_str_t*)str_obj_in)->charlen; str_data = ((mp_obj_str_t*)str_obj_in)->data; str_flags = ((mp_obj_str_t*)str_obj_in)->flags; }
// don't use this macro, it's only for conversions
#define GET_STR_DATA_LEN(str_obj_in, str_data, str_len) GET_STR_DATA_LEN_FLAGS(str_obj_in, str_data, str_len, str_data ## _flags); assert(str_data ## _flags == 1);
@ -372,17 +372,6 @@ STATIC mp_obj_t str_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) {
return mp_obj_new_str_of_type(type, self_data + slice.start, slice.stop - slice.start);
}
#endif
if (self_charlen == (uint)-1)
{
// HACK: Since qstr doesn't yet retain character length, count it up now.
// This allows tests to pass, but it's stupidly inefficient.
// (It's also safe. If charlen just happens to be (uint)-1, it won't
// break anything, it'll just recalculate it here.)
const byte *endptr, *top = self_data + self_len;
self_charlen = 0;
for (endptr = self_data; endptr < top; ++endptr)
if ((*endptr & 0xC0) != 0x80) ++self_charlen;
}
uint index_val = mp_get_index(type, self_charlen, index, false);
if (type == &mp_type_bytes) {
return MP_OBJ_NEW_SMALL_INT((mp_small_int_t)self_data[index_val]);
@ -1833,18 +1822,7 @@ uint mp_obj_str_get_hash(mp_obj_t self_in) {
uint mp_obj_str_get_len(mp_obj_t self_in) {
// TODO This has a double check for the type, one in obj.c and one here
if (MP_OBJ_IS_STR(self_in) || MP_OBJ_IS_TYPE(self_in, &mp_type_bytes)) {
GET_STR_INFO(self_in, self_data, self_len, self_charlen, self_flags);
if (self_charlen == (uint)-1)
{
// HACK: Since qstr doesn't yet retain character length, count it up now.
// This allows tests to pass, but it's stupidly inefficient.
// (It's also safe. If charlen just happens to be (uint)-1, it won't
// break anything, it'll just recalculate it here.)
const byte *endptr, *top = self_data + self_len;
self_charlen = 0;
for (endptr = self_data; endptr < top; ++endptr)
if ((*endptr & 0xC0) != 0x80) ++self_charlen;
}
GET_STR_INFO(self_in, self_data, self_len, self_charlen, self_flags); (void)self_data;
return self_charlen;
} else {
bad_implicit_conversion(self_in);

45
py/qstr.c
Wyświetl plik

@ -46,17 +46,20 @@
// For now we use very simple encoding, just to get the framework correct:
// - hash is 2 bytes (see function below)
// - length is 2 bytes
// - character length is 2 bytes
// - flags byte
// - data follows
// - \0 terminated (for now, so they can be printed using printf)
#define Q_GET_HASH(q) ((q)[0] | ((q)[1] << 8))
#define Q_GET_ALLOC(q) (4 + Q_GET_LENGTH(q) + 1)
#define Q_GET_ALLOC(q) (7 + Q_GET_LENGTH(q) + 1)
#define Q_GET_LENGTH(q) ((q)[2] | ((q)[3] << 8))
#define Q_GET_FLAGS(q) ((q)[4])
#define Q_GET_DATA(q) ((q) + 5)
#define Q_GET_CHARLEN(q) ((q)[4] | ((q)[5] << 8))
#define Q_GET_FLAGS(q) ((q)[6])
#define Q_GET_DATA(q) ((q) + 7)
// this must match the equivalent function in makeqstrdata.py
// Note that this hashes the UTF-8 encoded data bytes.
machine_uint_t qstr_compute_hash(const byte *data, uint len) {
// djb2 algorithm; see http://www.cse.yorku.ca/~oz/hash.html
machine_uint_t hash = 5381;
@ -85,8 +88,8 @@ const static qstr_pool_t const_pool = {
10, // set so that the first dynamically allocated pool is twice this size; must be <= the len (just below)
MP_QSTR_number_of, // corresponds to number of strings in array just below
{
(const byte*) "\0\0\0\0\0", // invalid/no qstr has empty data
(const byte*) "\0\0\0\0\1", // empty qstr
(const byte*) "\0\0\0\0\0\0\0", // invalid/no qstr has empty data
(const byte*) "\0\0\0\0\0\0\1", // empty qstr
#define Q(id, str) str,
#include "genhdr/qstrdefs.generated.h"
#undef Q
@ -157,14 +160,19 @@ qstr qstr_from_strn(const char *str, uint len) {
qstr q = qstr_find_strn(str, len);
if (q == 0) {
machine_uint_t hash = qstr_compute_hash((const byte*)str, len);
byte *q_ptr = m_new(byte, 4 + len + 1);
byte *q_ptr = m_new(byte, 7 + len + 1);
uint charlen = 0;
for (const char *s = str; s < str + len; ++s)
if ((*s & 0xC0) != 0x80) ++charlen;
q_ptr[0] = hash;
q_ptr[1] = hash >> 8;
q_ptr[2] = len;
q_ptr[3] = len >> 8;
q_ptr[4] = 1;
memcpy(q_ptr + 5, str, len);
q_ptr[5 + len] = '\0';
q_ptr[4] = charlen;
q_ptr[5] = charlen >> 8;
q_ptr[6] = 1;
memcpy(q_ptr + 7, str, len);
q_ptr[7 + len] = '\0';
q = qstr_add(q_ptr);
}
return q;
@ -172,7 +180,7 @@ qstr qstr_from_strn(const char *str, uint len) {
byte *qstr_build_start(uint len, byte **q_ptr) {
assert(len <= 65535);
*q_ptr = m_new(byte, 4 + len + 1);
*q_ptr = m_new(byte, 7 + len + 1);
(*q_ptr)[2] = len;
(*q_ptr)[3] = len >> 8;
return Q_GET_DATA(*q_ptr);
@ -182,11 +190,17 @@ qstr qstr_build_end(byte *q_ptr) {
qstr q = qstr_find_strn((const char*)Q_GET_DATA(q_ptr), Q_GET_LENGTH(q_ptr));
if (q == 0) {
machine_uint_t len = Q_GET_LENGTH(q_ptr);
machine_uint_t hash = qstr_compute_hash(Q_GET_DATA(q_ptr), len);
const byte *str = Q_GET_DATA(q_ptr);
machine_uint_t hash = qstr_compute_hash(str, len);
q_ptr[0] = hash;
q_ptr[1] = hash >> 8;
q_ptr[4] = 1;
q_ptr[5 + len] = '\0';
uint charlen = 0;
for (const byte *s = str; s < str + len; ++s)
if ((*s & 0xC0) != 0x80) ++charlen;
q_ptr[4] = charlen;
q_ptr[5] = charlen >> 8;
q_ptr[6] = 1;
q_ptr[7 + len] = '\0';
q = qstr_add(q_ptr);
} else {
m_del(byte, q_ptr, Q_GET_ALLOC(q_ptr));
@ -203,6 +217,11 @@ uint qstr_len(qstr q) {
return Q_GET_LENGTH(qd);
}
uint qstr_charlen(qstr q) {
const byte *qd = find_qstr(q);
return Q_GET_CHARLEN(qd);
}
// XXX to remove!
const char *qstr_str(qstr q) {
const byte *qd = find_qstr(q);

1
py/qstr.h
Wyświetl plik

@ -59,6 +59,7 @@ qstr qstr_build_end(byte *q_ptr);
machine_uint_t qstr_hash(qstr q);
const char* qstr_str(qstr q);
uint qstr_len(qstr q);
uint qstr_charlen(qstr q);
const byte* qstr_data(qstr q, uint *len, char *flags);
void qstr_pool_info(uint *n_pool, uint *n_qstr, uint *n_str_data_bytes, uint *n_total_bytes);