2014-05-03 22:27:38 +00:00
|
|
|
/*
|
2017-06-30 07:22:17 +00:00
|
|
|
* This file is part of the MicroPython project, http://micropython.org/
|
2014-05-03 22:27:38 +00:00
|
|
|
*
|
|
|
|
* The MIT License (MIT)
|
|
|
|
*
|
|
|
|
* Copyright (c) 2013, 2014 Damien P. George
|
|
|
|
*
|
|
|
|
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
|
|
* of this software and associated documentation files (the "Software"), to deal
|
|
|
|
* in the Software without restriction, including without limitation the rights
|
|
|
|
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
|
|
* copies of the Software, and to permit persons to whom the Software is
|
|
|
|
* furnished to do so, subject to the following conditions:
|
|
|
|
*
|
|
|
|
* The above copyright notice and this permission notice shall be included in
|
|
|
|
* all copies or substantial portions of the Software.
|
|
|
|
*
|
|
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
|
|
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
|
|
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
|
|
* THE SOFTWARE.
|
|
|
|
*/
|
|
|
|
|
2013-10-04 18:53:11 +00:00
|
|
|
#include <stdint.h>
|
|
|
|
|
2015-01-01 20:27:54 +00:00
|
|
|
#include "py/unicode.h"
|
2013-10-04 18:53:11 +00:00
|
|
|
|
|
|
|
// attribute flags
|
|
|
|
#define FL_PRINT (0x01)
|
|
|
|
#define FL_SPACE (0x02)
|
|
|
|
#define FL_DIGIT (0x04)
|
|
|
|
#define FL_ALPHA (0x08)
|
|
|
|
#define FL_UPPER (0x10)
|
|
|
|
#define FL_LOWER (0x20)
|
2015-05-19 23:41:07 +00:00
|
|
|
#define FL_XDIGIT (0x40)
|
2013-10-04 18:53:11 +00:00
|
|
|
|
|
|
|
// shorthand character attributes
|
|
|
|
#define AT_PR (FL_PRINT)
|
|
|
|
#define AT_SP (FL_SPACE | FL_PRINT)
|
2015-05-19 23:41:07 +00:00
|
|
|
#define AT_DI (FL_DIGIT | FL_PRINT | FL_XDIGIT)
|
2013-10-04 18:53:11 +00:00
|
|
|
#define AT_AL (FL_ALPHA | FL_PRINT)
|
|
|
|
#define AT_UP (FL_UPPER | FL_ALPHA | FL_PRINT)
|
|
|
|
#define AT_LO (FL_LOWER | FL_ALPHA | FL_PRINT)
|
2015-05-19 23:41:07 +00:00
|
|
|
#define AT_UX (FL_UPPER | FL_ALPHA | FL_PRINT | FL_XDIGIT)
|
|
|
|
#define AT_LX (FL_LOWER | FL_ALPHA | FL_PRINT | FL_XDIGIT)
|
2013-10-04 18:53:11 +00:00
|
|
|
|
|
|
|
// table of attributes for ascii characters
|
2014-02-12 16:31:30 +00:00
|
|
|
STATIC const uint8_t attr[] = {
|
2013-10-04 18:53:11 +00:00
|
|
|
0, 0, 0, 0, 0, 0, 0, 0,
|
2014-04-10 10:39:36 +00:00
|
|
|
0, AT_SP, AT_SP, AT_SP, AT_SP, AT_SP, 0, 0,
|
2013-10-04 18:53:11 +00:00
|
|
|
0, 0, 0, 0, 0, 0, 0, 0,
|
|
|
|
0, 0, 0, 0, 0, 0, 0, 0,
|
|
|
|
AT_SP, AT_PR, AT_PR, AT_PR, AT_PR, AT_PR, AT_PR, AT_PR,
|
|
|
|
AT_PR, AT_PR, AT_PR, AT_PR, AT_PR, AT_PR, AT_PR, AT_PR,
|
|
|
|
AT_DI, AT_DI, AT_DI, AT_DI, AT_DI, AT_DI, AT_DI, AT_DI,
|
|
|
|
AT_DI, AT_DI, AT_PR, AT_PR, AT_PR, AT_PR, AT_PR, AT_PR,
|
2015-05-19 23:41:07 +00:00
|
|
|
AT_PR, AT_UX, AT_UX, AT_UX, AT_UX, AT_UX, AT_UX, AT_UP,
|
2013-10-04 18:53:11 +00:00
|
|
|
AT_UP, AT_UP, AT_UP, AT_UP, AT_UP, AT_UP, AT_UP, AT_UP,
|
|
|
|
AT_UP, AT_UP, AT_UP, AT_UP, AT_UP, AT_UP, AT_UP, AT_UP,
|
|
|
|
AT_UP, AT_UP, AT_UP, AT_PR, AT_PR, AT_PR, AT_PR, AT_PR,
|
2015-05-19 23:41:07 +00:00
|
|
|
AT_PR, AT_LX, AT_LX, AT_LX, AT_LX, AT_LX, AT_LX, AT_LO,
|
2013-10-04 18:53:11 +00:00
|
|
|
AT_LO, AT_LO, AT_LO, AT_LO, AT_LO, AT_LO, AT_LO, AT_LO,
|
|
|
|
AT_LO, AT_LO, AT_LO, AT_LO, AT_LO, AT_LO, AT_LO, AT_LO,
|
|
|
|
AT_LO, AT_LO, AT_LO, AT_PR, AT_PR, AT_PR, AT_PR, 0
|
|
|
|
};
|
|
|
|
|
2014-06-14 03:39:20 +00:00
|
|
|
#if MICROPY_PY_BUILTINS_STR_UNICODE
|
2018-02-14 07:19:22 +00:00
|
|
|
|
|
|
|
unichar utf8_get_char(const byte *s) {
|
2014-06-03 19:28:12 +00:00
|
|
|
unichar ord = *s++;
|
|
|
|
if (!UTF8_IS_NONASCII(ord)) return ord;
|
|
|
|
ord &= 0x7F;
|
|
|
|
for (unichar mask = 0x40; ord & mask; mask >>= 1) {
|
|
|
|
ord &= ~mask;
|
|
|
|
}
|
|
|
|
while (UTF8_IS_CONT(*s)) {
|
|
|
|
ord = (ord << 6) | (*s++ & 0x3F);
|
|
|
|
}
|
|
|
|
return ord;
|
|
|
|
}
|
|
|
|
|
2014-06-14 03:39:20 +00:00
|
|
|
const byte *utf8_next_char(const byte *s) {
|
2014-06-03 19:28:12 +00:00
|
|
|
++s;
|
|
|
|
while (UTF8_IS_CONT(*s)) {
|
|
|
|
++s;
|
|
|
|
}
|
2014-06-14 03:39:20 +00:00
|
|
|
return s;
|
2013-10-04 18:53:11 +00:00
|
|
|
}
|
|
|
|
|
2014-12-10 22:37:07 +00:00
|
|
|
mp_uint_t utf8_ptr_to_index(const byte *s, const byte *ptr) {
|
2014-07-03 12:25:24 +00:00
|
|
|
mp_uint_t i = 0;
|
2014-06-14 00:16:17 +00:00
|
|
|
while (ptr > s) {
|
|
|
|
if (!UTF8_IS_CONT(*--ptr)) {
|
|
|
|
i++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return i;
|
|
|
|
}
|
|
|
|
|
2018-02-14 07:19:22 +00:00
|
|
|
size_t utf8_charlen(const byte *str, size_t len) {
|
|
|
|
size_t charlen = 0;
|
|
|
|
for (const byte *top = str + len; str < top; ++str) {
|
2014-06-03 19:28:12 +00:00
|
|
|
if (!UTF8_IS_CONT(*str)) {
|
|
|
|
++charlen;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return charlen;
|
2013-10-04 18:53:11 +00:00
|
|
|
}
|
|
|
|
|
2018-02-14 07:19:22 +00:00
|
|
|
#endif
|
|
|
|
|
2014-06-03 19:28:12 +00:00
|
|
|
// Be aware: These unichar_is* functions are actually ASCII-only!
|
2013-12-30 18:23:50 +00:00
|
|
|
bool unichar_isspace(unichar c) {
|
2013-10-04 18:53:11 +00:00
|
|
|
return c < 128 && (attr[c] & FL_SPACE) != 0;
|
|
|
|
}
|
|
|
|
|
2013-12-30 18:23:50 +00:00
|
|
|
bool unichar_isalpha(unichar c) {
|
2013-10-04 18:53:11 +00:00
|
|
|
return c < 128 && (attr[c] & FL_ALPHA) != 0;
|
|
|
|
}
|
|
|
|
|
2016-12-28 06:50:10 +00:00
|
|
|
/* unused
|
2013-12-30 18:23:50 +00:00
|
|
|
bool unichar_isprint(unichar c) {
|
2013-10-04 18:53:11 +00:00
|
|
|
return c < 128 && (attr[c] & FL_PRINT) != 0;
|
|
|
|
}
|
2016-12-28 06:50:10 +00:00
|
|
|
*/
|
2013-10-04 18:53:11 +00:00
|
|
|
|
2013-12-30 18:23:50 +00:00
|
|
|
bool unichar_isdigit(unichar c) {
|
2013-10-04 18:53:11 +00:00
|
|
|
return c < 128 && (attr[c] & FL_DIGIT) != 0;
|
|
|
|
}
|
|
|
|
|
2014-01-22 20:40:02 +00:00
|
|
|
bool unichar_isxdigit(unichar c) {
|
2015-05-19 23:41:07 +00:00
|
|
|
return c < 128 && (attr[c] & FL_XDIGIT) != 0;
|
2014-01-22 20:40:02 +00:00
|
|
|
}
|
|
|
|
|
2016-02-16 13:36:18 +00:00
|
|
|
bool unichar_isident(unichar c) {
|
|
|
|
return c < 128 && ((attr[c] & (FL_ALPHA | FL_DIGIT)) != 0 || c == '_');
|
2013-10-04 18:53:11 +00:00
|
|
|
}
|
|
|
|
|
2014-05-11 16:53:11 +00:00
|
|
|
bool unichar_isupper(unichar c) {
|
2013-10-04 18:53:11 +00:00
|
|
|
return c < 128 && (attr[c] & FL_UPPER) != 0;
|
|
|
|
}
|
|
|
|
|
2014-05-11 16:53:11 +00:00
|
|
|
bool unichar_islower(unichar c) {
|
2013-10-04 18:53:11 +00:00
|
|
|
return c < 128 && (attr[c] & FL_LOWER) != 0;
|
|
|
|
}
|
2014-05-10 16:47:41 +00:00
|
|
|
|
|
|
|
unichar unichar_tolower(unichar c) {
|
2014-05-11 16:53:11 +00:00
|
|
|
if (unichar_isupper(c)) {
|
2014-05-10 16:47:41 +00:00
|
|
|
return c + 0x20;
|
|
|
|
}
|
|
|
|
return c;
|
|
|
|
}
|
|
|
|
|
|
|
|
unichar unichar_toupper(unichar c) {
|
2014-05-11 16:53:11 +00:00
|
|
|
if (unichar_islower(c)) {
|
2014-05-10 16:47:41 +00:00
|
|
|
return c - 0x20;
|
|
|
|
}
|
|
|
|
return c;
|
|
|
|
}
|
2015-05-18 21:41:25 +00:00
|
|
|
|
|
|
|
mp_uint_t unichar_xdigit_value(unichar c) {
|
|
|
|
// c is assumed to be hex digit
|
|
|
|
mp_uint_t n = c - '0';
|
|
|
|
if (n > 9) {
|
|
|
|
n &= ~('a' - 'A');
|
|
|
|
n -= ('A' - ('9' + 1));
|
|
|
|
}
|
|
|
|
return n;
|
|
|
|
}
|
2017-06-24 00:38:32 +00:00
|
|
|
|
2018-02-14 07:19:22 +00:00
|
|
|
#if MICROPY_PY_BUILTINS_STR_UNICODE
|
|
|
|
|
2017-06-24 00:38:32 +00:00
|
|
|
bool utf8_check(const byte *p, size_t len) {
|
|
|
|
uint8_t need = 0;
|
|
|
|
const byte *end = p + len;
|
|
|
|
for (; p < end; p++) {
|
|
|
|
byte c = *p;
|
|
|
|
if (need) {
|
|
|
|
if (c >= 0x80) {
|
|
|
|
need--;
|
|
|
|
} else {
|
|
|
|
// mismatch
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
if (c >= 0xc0) {
|
|
|
|
if (c >= 0xf8) {
|
|
|
|
// mismatch
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
need = (0xe5 >> ((c >> 3) & 0x6)) & 3;
|
|
|
|
} else if (c >= 0x80) {
|
|
|
|
// mismatch
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return need == 0; // no pending fragments allowed
|
|
|
|
}
|
2018-02-14 07:19:22 +00:00
|
|
|
|
|
|
|
#endif
|