py: Enable struct/binary-helper to parse q and Q sized ints.

Addresses issue #848.

py/binary.c

@@ -34,6 +34,7 @@
 #include "misc.h"
 #include "qstr.h"
 #include "obj.h"
+#include "smallint.h"
 #include "binary.h"
 
 // Helpers to work with binary-encoded data
@@ -136,7 +137,10 @@ mp_obj_t mp_binary_get_val_array(char typecode, void *p, int index) {
     return MP_OBJ_NEW_SMALL_INT(val);
 }
 
-mp_int_t mp_binary_get_int(mp_uint_t size, bool is_signed, bool big_endian, byte *p) {
+// The long long type is guaranteed to hold at least 64 bits, and size is at
+// most 8 (for q and Q), so we will always be able to parse the given data
+// and fit it into a long long.
+long long mp_binary_get_int(mp_uint_t size, bool is_signed, bool big_endian, byte *p) {
     int delta;
     if (!big_endian) {
         delta = -1;
@@ -145,7 +149,7 @@ mp_int_t mp_binary_get_int(mp_uint_t size, bool is_signed, bool big_endian, byte
         delta = 1;
     }
 
-    mp_int_t val = 0;
+    long long val = 0;
     if (is_signed && *p & 0x80) {
         val = -1;
     }
@@ -175,16 +179,25 @@ mp_obj_t mp_binary_get_val(char struct_type, char val_type, byte **ptr) {
     }
     *ptr = p + size;
 
-    mp_int_t val = mp_binary_get_int(size, is_signed(val_type), (struct_type == '>'), p);
+    long long val = mp_binary_get_int(size, is_signed(val_type), (struct_type == '>'), p);
 
     if (val_type == 'O') {
-        return (mp_obj_t)val;
+        return (mp_obj_t)(mp_uint_t)val;
     } else if (val_type == 'S') {
-        return mp_obj_new_str((char*)val, strlen((char*)val), false);
+        const char *s_val = (const char*)(mp_uint_t)val;
+        return mp_obj_new_str(s_val, strlen(s_val), false);
     } else if (is_signed(val_type)) {
-        return mp_obj_new_int(val);
+        if ((long long)MP_SMALL_INT_MIN <= val && val <= (long long)MP_SMALL_INT_MAX) {
+            return mp_obj_new_int((mp_int_t)val);
+        } else {
+            return mp_obj_new_int_from_ll(val);
+        }
     } else {
-        return mp_obj_new_int_from_uint(val);
+        if ((unsigned long long)val <= (unsigned long long)MP_SMALL_INT_MAX) {
+            return mp_obj_new_int_from_uint((mp_uint_t)val);
+        } else {
+            return mp_obj_new_int_from_ull(val);
+        }
     }
 }
 

py/binary.h

@@ -34,5 +34,5 @@ void mp_binary_set_val_array(char typecode, void *p, int index, mp_obj_t val_in)
 void mp_binary_set_val_array_from_int(char typecode, void *p, int index, mp_int_t val);
 mp_obj_t mp_binary_get_val(char struct_type, char val_type, byte **ptr);
 void mp_binary_set_val(char struct_type, char val_type, mp_obj_t val_in, byte **ptr);
-mp_int_t mp_binary_get_int(mp_uint_t size, bool is_signed, bool big_endian, byte *p);
+long long mp_binary_get_int(mp_uint_t size, bool is_signed, bool big_endian, byte *p);
 void mp_binary_set_int(mp_uint_t val_sz, bool big_endian, byte *p, byte *val_ptr);

py/mpz.c

@@ -589,9 +589,9 @@ mpz_t *mpz_from_int(mp_int_t val) {
     return z;
 }
 
-mpz_t *mpz_from_ll(long long val) {
+mpz_t *mpz_from_ll(long long val, bool is_signed) {
     mpz_t *z = mpz_zero();
-    mpz_set_from_ll(z, val);
+    mpz_set_from_ll(z, val, is_signed);
     return z;
 }
 
@@ -668,11 +668,11 @@ void mpz_set_from_int(mpz_t *z, mp_int_t val) {
     }
 }
 
-void mpz_set_from_ll(mpz_t *z, long long val) {
+void mpz_set_from_ll(mpz_t *z, long long val, bool is_signed) {
     mpz_need_dig(z, MPZ_NUM_DIG_FOR_LL);
 
     unsigned long long uval;
-    if (val < 0) {
+    if (is_signed && val < 0) {
         z->neg = 1;
         uval = -val;
     } else {

py/mpz.h

@@ -71,7 +71,7 @@ void mpz_deinit(mpz_t *z);
 
 mpz_t *mpz_zero();
 mpz_t *mpz_from_int(mp_int_t i);
-mpz_t *mpz_from_ll(long long i);
+mpz_t *mpz_from_ll(long long i, bool is_signed);
 mpz_t *mpz_from_str(const char *str, mp_uint_t len, bool neg, mp_uint_t base);
 void mpz_free(mpz_t *z);
 
@@ -79,7 +79,7 @@ mpz_t *mpz_clone(const mpz_t *src);
 
 void mpz_set(mpz_t *dest, const mpz_t *src);
 void mpz_set_from_int(mpz_t *z, mp_int_t src);
-void mpz_set_from_ll(mpz_t *z, long long i);
+void mpz_set_from_ll(mpz_t *z, long long i, bool is_signed);
 mp_uint_t mpz_set_from_str(mpz_t *z, const char *str, mp_uint_t len, bool neg, mp_uint_t base);
 
 bool mpz_is_zero(const mpz_t *z);

py/obj.h

@@ -369,6 +369,7 @@ mp_obj_t mp_obj_new_int(mp_int_t value);
 mp_obj_t mp_obj_new_int_from_uint(mp_uint_t value);
 mp_obj_t mp_obj_new_int_from_str_len(const char **str, mp_uint_t len, bool neg, mp_uint_t base);
 mp_obj_t mp_obj_new_int_from_ll(long long val); // this must return a multi-precision integer object (or raise an overflow exception)
+mp_obj_t mp_obj_new_int_from_ull(unsigned long long val); // this must return a multi-precision integer object (or raise an overflow exception)
 mp_obj_t mp_obj_new_str(const char* data, mp_uint_t len, bool make_qstr_if_not_already);
 mp_obj_t mp_obj_new_bytes(const byte* data, mp_uint_t len);
 #if MICROPY_PY_BUILTINS_FLOAT

py/objint.c

@@ -246,6 +246,12 @@ mp_obj_t mp_obj_new_int_from_ll(long long val) {
     return mp_const_none;
 }
 
+// This is called when an integer larger than a SMALL_INT is needed (although val might still fit in a SMALL_INT)
+mp_obj_t mp_obj_new_int_from_ull(unsigned long long val) {
+    nlr_raise(mp_obj_new_exception_msg(&mp_type_OverflowError, "small int overflow"));
+    return mp_const_none;
+}
+
 mp_obj_t mp_obj_new_int_from_uint(mp_uint_t value) {
     // SMALL_INT accepts only signed numbers, of one bit less size
     // then word size, which totals 2 bits less for unsigned numbers.

py/objint_longlong.c

@@ -178,7 +178,16 @@ mp_obj_t mp_obj_new_int_from_ll(long long val) {
     return o;
 }
 
-mp_obj_t mp_obj_new_int_from_str_len(const char **str, uint len, bool neg, uint base) {
+mp_obj_t mp_obj_new_int_from_ull(unsigned long long val) {
+    // TODO raise an exception if the unsigned long long won't fit
+    assert(val >> (sizeof(unsigned long long) * 8 - 1) == 0);
+    mp_obj_int_t *o = m_new_obj(mp_obj_int_t);
+    o->base.type = &mp_type_int;
+    o->val = val;
+    return o;
+}
+
+mp_obj_t mp_obj_new_int_from_str_len(const char **str, mp_uint_t len, bool neg, mp_uint_t base) {
     // TODO this does not honor the given length of the string, but it all cases it should anyway be null terminated
     // TODO check overflow
     mp_obj_int_t *o = m_new_obj(mp_obj_int_t);

py/objint_mpz.c

@@ -279,7 +279,13 @@ mp_obj_t mp_obj_new_int(mp_int_t value) {
 
 mp_obj_t mp_obj_new_int_from_ll(long long val) {
     mp_obj_int_t *o = mp_obj_int_new_mpz();
-    mpz_set_from_ll(&o->mpz, val);
+    mpz_set_from_ll(&o->mpz, val, true);
+    return o;
+}
+
+mp_obj_t mp_obj_new_int_from_ull(unsigned long long val) {
+    mp_obj_int_t *o = mp_obj_int_new_mpz();
+    mpz_set_from_ll(&o->mpz, val, false);
     return o;
 }