From 9193f89296d6757955db7ebdfd4ca332f0b60f71 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 8 Jan 2014 15:28:26 +0000 Subject: [PATCH] Move lexerstr to main py directory (everyone uses it). --- py/lexer.h | 2 ++ py/lexerstr.c | 35 +++++++++++++++++++++++++++++++++++ py/lexerunix.c | 36 ++---------------------------------- py/lexerunix.h | 1 - py/py.mk | 1 + stm/Makefile | 1 - stm/lexerfatfs.h | 2 +- stm/lexerstr.c | 28 ---------------------------- stm/lexerstr.h | 8 -------- stm/main.c | 7 ++----- teensy/Makefile | 2 +- teensy/main.c | 1 - 12 files changed, 44 insertions(+), 80 deletions(-) create mode 100644 py/lexerstr.c delete mode 100644 stm/lexerstr.c delete mode 100644 stm/lexerstr.h diff --git a/py/lexer.h b/py/lexer.h index 3cb48ce9e1..9dfcb128c5 100644 --- a/py/lexer.h +++ b/py/lexer.h @@ -127,6 +127,8 @@ void mp_token_show_error_prefix(const mp_token_t *tok); bool mp_token_show_error(const mp_token_t *tok, const char *msg); mp_lexer_t *mp_lexer_new(const char *src_name, void *stream_data, mp_lexer_stream_next_char_t stream_next_char, mp_lexer_stream_close_t stream_close); +mp_lexer_t *mp_lexer_new_from_str_len(const char *src_name, const char *str, uint len, uint free_len); + void mp_lexer_free(mp_lexer_t *lex); void mp_lexer_to_next(mp_lexer_t *lex); const mp_token_t *mp_lexer_cur(const mp_lexer_t *lex); diff --git a/py/lexerstr.c b/py/lexerstr.c new file mode 100644 index 0000000000..b8594f4205 --- /dev/null +++ b/py/lexerstr.c @@ -0,0 +1,35 @@ +#include +#include + +#include "misc.h" +#include "lexer.h" + +typedef struct _mp_lexer_str_buf_t { + uint free_len; // if > 0, src_beg will be freed when done by: m_free(src_beg, free_len) + const char *src_beg; // beginning of source + const char *src_cur; // current location in source + const char *src_end; // end (exclusive) of source +} mp_lexer_str_buf_t; + +static unichar str_buf_next_char(mp_lexer_str_buf_t *sb) { + if (sb->src_cur < sb->src_end) { + return *sb->src_cur++; + } else { + return MP_LEXER_CHAR_EOF; + } +} + +static void str_buf_free(mp_lexer_str_buf_t *sb) { + if (sb->free_len > 0) { + m_free((char*)sb->src_beg, sb->free_len); + } +} + +mp_lexer_t *mp_lexer_new_from_str_len(const char *src_name, const char *str, uint len, uint free_len) { + mp_lexer_str_buf_t *sb = m_new_obj(mp_lexer_str_buf_t); + sb->free_len = free_len; + sb->src_beg = str; + sb->src_cur = str; + sb->src_end = str + len; + return mp_lexer_new(src_name, sb, (mp_lexer_stream_next_char_t)str_buf_next_char, (mp_lexer_stream_close_t)str_buf_free); +} diff --git a/py/lexerunix.c b/py/lexerunix.c index 5336610bae..225ed20a7f 100644 --- a/py/lexerunix.c +++ b/py/lexerunix.c @@ -6,42 +6,10 @@ #include "misc.h" #include "mpconfig.h" #include "lexer.h" +#include "lexerunix.h" #if MICROPY_ENABLE_LEXER_UNIX -typedef struct _str_buf_t { - bool free; // free src_beg when done - const char *src_beg; // beginning of source - const char *src_cur; // current location in source - const char *src_end; // end (exclusive) of source -} str_buf_t; - -unichar str_buf_next_char(str_buf_t *sb) { - if (sb->src_cur < sb->src_end) { - return *sb->src_cur++; - } else { - return MP_LEXER_CHAR_EOF; - } -} - -void str_buf_free(str_buf_t *sb) { - if (sb) { - if (sb->free) { - m_del(char, (char*)sb->src_beg, 0 /* unknown size of src_beg */); - } - m_del_obj(str_buf_t, sb); - } -} - -mp_lexer_t *mp_lexer_new_from_str_len(const char *src_name, const char *str, uint len, bool free_str) { - str_buf_t *sb = m_new(str_buf_t, 1); - sb->free = free_str; - sb->src_beg = str; - sb->src_cur = str; - sb->src_end = str + len; - return mp_lexer_new(src_name, sb, (mp_lexer_stream_next_char_t)str_buf_next_char, (mp_lexer_stream_close_t)str_buf_free); -} - mp_lexer_t *mp_lexer_new_from_file(const char *filename) { int fd = open(filename, O_RDONLY); if (fd < 0) { @@ -59,7 +27,7 @@ mp_lexer_t *mp_lexer_new_from_file(const char *filename) { return NULL; } - return mp_lexer_new_from_str_len(filename, data, size, true); + return mp_lexer_new_from_str_len(filename, data, size, size); } /******************************************************************************/ diff --git a/py/lexerunix.h b/py/lexerunix.h index b422a43062..3451c1c1ce 100644 --- a/py/lexerunix.h +++ b/py/lexerunix.h @@ -1,4 +1,3 @@ -mp_lexer_t *mp_lexer_new_from_str_len(const char *src_name, const char *str, uint len, bool free_str); mp_lexer_t *mp_lexer_new_from_file(const char *filename); void mp_import_set_directory(const char *dir); diff --git a/py/py.mk b/py/py.mk index 3ed8a3e3d7..0502dc0ffd 100644 --- a/py/py.mk +++ b/py/py.mk @@ -29,6 +29,7 @@ PY_O_BASENAME = \ vstr.o \ unicode.o \ lexer.o \ + lexerstr.o \ lexerunix.o \ parse.o \ scope.o \ diff --git a/stm/Makefile b/stm/Makefile index 478441e04b..725eb4c30a 100644 --- a/stm/Makefile +++ b/stm/Makefile @@ -30,7 +30,6 @@ SRC_C = \ string0.c \ malloc0.c \ systick.c \ - lexerstr.c \ lexerfatfs.c \ led.c \ lcd.c \ diff --git a/stm/lexerfatfs.h b/stm/lexerfatfs.h index 2b41ed136b..f1fe5ad485 100644 --- a/stm/lexerfatfs.h +++ b/stm/lexerfatfs.h @@ -1,4 +1,4 @@ -typedef struct _py_lexer_file_buf_t { +typedef struct _mp_lexer_file_buf_t { FIL fp; char buf[20]; uint16_t len; diff --git a/stm/lexerstr.c b/stm/lexerstr.c deleted file mode 100644 index cc6be1d1ed..0000000000 --- a/stm/lexerstr.c +++ /dev/null @@ -1,28 +0,0 @@ -#include -#include - -#include "misc.h" -#include "lexer.h" -#include "lexerstr.h" - -unichar str_buf_next_char(mp_lexer_str_buf_t *sb) { - if (sb->src_cur < sb->src_end) { - return *sb->src_cur++; - } else { - return MP_LEXER_CHAR_EOF; - } -} - -void str_buf_free(mp_lexer_str_buf_t *sb) { - if (sb->free) { - m_del(char, (char*)sb->src_beg, 0 /* don't know allocated size of src */); - } -} - -mp_lexer_t *mp_lexer_new_from_str_len(const char *src_name, const char *str, uint len, bool free_str, mp_lexer_str_buf_t *sb) { - sb->free = free_str; - sb->src_beg = str; - sb->src_cur = str; - sb->src_end = str + len; - return mp_lexer_new(src_name, sb, (mp_lexer_stream_next_char_t)str_buf_next_char, (mp_lexer_stream_close_t)str_buf_free); -} diff --git a/stm/lexerstr.h b/stm/lexerstr.h deleted file mode 100644 index 961a70ada4..0000000000 --- a/stm/lexerstr.h +++ /dev/null @@ -1,8 +0,0 @@ -typedef struct _py_lexer_str_buf_t { - bool free; // free src_beg when done - const char *src_beg; // beginning of source - const char *src_cur; // current location in source - const char *src_end; // end (exclusive) of source -} mp_lexer_str_buf_t; - -mp_lexer_t *mp_lexer_new_from_str_len(const char *src_name, const char *str, uint len, bool free_str, mp_lexer_str_buf_t *sb); diff --git a/stm/main.c b/stm/main.c index a2f3bc7f03..0ad638cde1 100644 --- a/stm/main.c +++ b/stm/main.c @@ -19,7 +19,6 @@ #include "nlr.h" #include "misc.h" #include "lexer.h" -#include "lexerstr.h" #include "lexerfatfs.h" #include "parse.h" #include "obj.h" @@ -432,8 +431,7 @@ void do_repl(void) { } } - mp_lexer_str_buf_t sb; - mp_lexer_t *lex = mp_lexer_new_from_str_len("", vstr_str(&line), vstr_len(&line), false, &sb); + mp_lexer_t *lex = mp_lexer_new_from_str_len("", vstr_str(&line), vstr_len(&line), 0); mp_parse_node_t pn = mp_parse(lex, MP_PARSE_SINGLE_INPUT); mp_lexer_free(lex); @@ -1059,8 +1057,7 @@ soft_reset: " pass\n" "f()\n"; - mp_lexer_str_buf_t mp_lexer_str_buf; - mp_lexer_t *lex = mp_lexer_new_from_str_len("", pysrc, strlen(pysrc), false, &mp_lexer_str_buf); + mp_lexer_t *lex = mp_lexer_new_from_str_len("", pysrc, strlen(pysrc), 0); // nalloc=1740;6340;6836 -> 140;4600;496 bytes for lexer, parser, compiler printf("lex; al=%u\n", m_get_total_bytes_allocated()); diff --git a/teensy/Makefile b/teensy/Makefile index f86585c658..0e42867641 100644 --- a/teensy/Makefile +++ b/teensy/Makefile @@ -29,7 +29,6 @@ SRC_C = \ usb.c \ STM_SRC_C = \ - lexerstr.c \ malloc0.c \ printf.c \ string0.c \ @@ -45,6 +44,7 @@ PY_O = \ vstr.o \ unicode.o \ lexer.o \ + lexerstr.o \ parse.o \ scope.o \ compile.o \ diff --git a/teensy/main.c b/teensy/main.c index 4844941e70..ce6aa029da 100644 --- a/teensy/main.c +++ b/teensy/main.c @@ -8,7 +8,6 @@ #include "mpconfig.h" #include "mpqstr.h" #include "lexer.h" -#include "../stm/lexerstr.h" #include "parse.h" #include "obj.h" #include "compile.h"