From ad1606ae1d994de09ad64afc8eb00ae45a1f3592 Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Mon, 22 Mar 2021 21:05:22 +0100 Subject: [PATCH 1/6] tools: spiffsgen.py: esp8266 compatibility options 1. Implement --aligned-obj-ix-tables which is used by default on the ESP8266 in NodeMCU and Arduino. 2. Introduce --no-magic and --no-magic-len to allow disabling options --use-magic-len and --use-magic. As these have been declared with default=True and action='store_true', they couldn't be disabled otherwise. Closes https://github.com/espressif/esp-idf/issues/6717 --- components/spiffs/spiffsgen.py | 54 +++++++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/components/spiffs/spiffsgen.py b/components/spiffs/spiffsgen.py index 56018ceac4..a93c3465cc 100755 --- a/components/spiffs/spiffsgen.py +++ b/components/spiffs/spiffsgen.py @@ -44,7 +44,8 @@ SPIFFS_BLOCK_IX_LEN = 2 # spiffs_block_ix class SpiffsBuildConfig(): def __init__(self, page_size, page_ix_len, block_size, block_ix_len, meta_len, obj_name_len, obj_id_len, - span_ix_len, packed, aligned, endianness, use_magic, use_magic_len): + span_ix_len, packed, aligned, endianness, use_magic, use_magic_len, + aligned_obj_ix_tables): if block_size % page_size != 0: raise RuntimeError('block size should be a multiple of page size') @@ -61,6 +62,7 @@ class SpiffsBuildConfig(): self.endianness = endianness self.use_magic = use_magic self.use_magic_len = use_magic_len + self.aligned_obj_ix_tables = aligned_obj_ix_tables self.PAGES_PER_BLOCK = self.block_size // self.page_size self.OBJ_LU_PAGES_PER_BLOCK = int(math.ceil(self.block_size / self.page_size * self.obj_id_len / self.page_size)) @@ -78,7 +80,14 @@ class SpiffsBuildConfig(): self.OBJ_INDEX_PAGES_HEADER_LEN = (self.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED + SPIFFS_PH_IX_SIZE_LEN + SPIFFS_PH_IX_OBJ_TYPE_LEN + self.obj_name_len + self.meta_len) - self.OBJ_INDEX_PAGES_OBJ_IDS_HEAD_LIM = (self.page_size - self.OBJ_INDEX_PAGES_HEADER_LEN) // self.block_ix_len + if aligned_obj_ix_tables: + self.OBJ_INDEX_PAGES_HEADER_LEN_ALIGNED = (self.OBJ_INDEX_PAGES_HEADER_LEN + SPIFFS_PAGE_IX_LEN - 1) & ~(SPIFFS_PAGE_IX_LEN - 1) + self.OBJ_INDEX_PAGES_HEADER_LEN_ALIGNED_PAD = self.OBJ_INDEX_PAGES_HEADER_LEN_ALIGNED - self.OBJ_INDEX_PAGES_HEADER_LEN + else: + self.OBJ_INDEX_PAGES_HEADER_LEN_ALIGNED = self.OBJ_INDEX_PAGES_HEADER_LEN + self.OBJ_INDEX_PAGES_HEADER_LEN_ALIGNED_PAD = 0 + + self.OBJ_INDEX_PAGES_OBJ_IDS_HEAD_LIM = (self.page_size - self.OBJ_INDEX_PAGES_HEADER_LEN_ALIGNED) // self.block_ix_len self.OBJ_INDEX_PAGES_OBJ_IDS_LIM = (self.page_size - self.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED) / self.block_ix_len @@ -217,7 +226,10 @@ class SpiffsObjIndexPage(SpiffsPage): self.size, SPIFFS_TYPE_FILE) - img += self.name.encode() + (b'\x00' * ((self.build_config.obj_name_len - len(self.name)) + self.build_config.meta_len)) + img += self.name.encode() + (b'\x00' * ( + (self.build_config.obj_name_len - len(self.name)) + + self.build_config.meta_len + + self.build_config.OBJ_INDEX_PAGES_HEADER_LEN_ALIGNED_PAD)) # Finally, add the page index of daa pages for page in self.pages: @@ -490,24 +502,38 @@ def main(): default=4) parser.add_argument('--use-magic', + dest='use_magic', help='Use magic number to create an identifiable SPIFFS image. Specify if CONFIG_SPIFFS_USE_MAGIC.', - action='store_true', - default=True) + action='store_true') + + parser.add_argument('--no-magic', + dest='use_magic', + help='Inverse of --use-magic', + action='store_false') + + parser.add_argument('--use-magic-len', + dest='use_magic_len', + help='Use position in memory to create different magic numbers for each block. Specify if CONFIG_SPIFFS_USE_MAGIC_LENGTH.', + action='store_true') + + parser.add_argument('--no-magic-len', + dest='use_magic_len', + help='Inverse of --use-magic-len', + action='store_false') parser.add_argument('--follow-symlinks', help='Take into account symbolic links during partition image creation.', - action='store_true', - default=False) - - parser.add_argument('--use-magic-len', - help='Use position in memory to create different magic numbers for each block. Specify if CONFIG_SPIFFS_USE_MAGIC_LENGTH.', - action='store_true', - default=True) + action='store_true') parser.add_argument('--big-endian', help='Specify if the target architecture is big-endian. If not specified, little-endian is assumed.', + action='store_true') + + parser.add_argument('--aligned-obj-ix-tables', action='store_true', - default=False) + help='Use aligned object index tables. Specify if SPIFFS_ALIGNED_OBJECT_INDEX_TABLES is set.') + + parser.set_defaults(use_magic=True, use_magic_len=True) args = parser.parse_args() @@ -520,7 +546,7 @@ def main(): args.block_size, SPIFFS_BLOCK_IX_LEN, args.meta_len, args.obj_name_len, SPIFFS_OBJ_ID_LEN, SPIFFS_SPAN_IX_LEN, True, True, 'big' if args.big_endian else 'little', - args.use_magic, args.use_magic_len) + args.use_magic, args.use_magic_len, args.aligned_obj_ix_tables) spiffs = SpiffsFS(image_size, spiffs_build_default) From 930ee51b8fe1d7764a4c643fa1d987f2569479b2 Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Mon, 22 Mar 2021 21:11:36 +0100 Subject: [PATCH 2/6] tools: spiffsgen.py: avoid reallocating byte array for each new block On large filesystems (~15 MB), this reduces execution time from 11s to 0.3s. --- components/spiffs/spiffsgen.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/components/spiffs/spiffsgen.py b/components/spiffs/spiffsgen.py index a93c3465cc..d308310eaf 100755 --- a/components/spiffs/spiffsgen.py +++ b/components/spiffs/spiffsgen.py @@ -448,19 +448,21 @@ class SpiffsFS(): def to_binary(self): img = b'' + all_blocks = [] for block in self.blocks: - img += block.to_binary(self.blocks_lim) + all_blocks.append(block.to_binary(self.blocks_lim)) bix = len(self.blocks) if self.build_config.use_magic: # Create empty blocks with magic numbers while self.remaining_blocks > 0: block = SpiffsBlock(bix, self.blocks_lim, self.build_config) - img += block.to_binary(self.blocks_lim) + all_blocks.append(block.to_binary(self.blocks_lim)) self.remaining_blocks -= 1 bix += 1 else: # Just fill remaining spaces FF's - img += '\xFF' * (self.img_size - len(img)) + all_blocks.append(b'\xFF' * (self.img_size - len(img))) + img += b''.join([blk for blk in all_blocks]) return img From 9f20eeb1c029d0b688e692bd3c6e7b1d4277a4f8 Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Mon, 22 Mar 2021 22:06:01 +0100 Subject: [PATCH 3/6] tools: spiffsgen.py: add type annotations --- components/spiffs/spiffsgen.py | 136 +++++++++++++++++++-------------- tools/ci/mypy_ignore_list.txt | 1 - 2 files changed, 80 insertions(+), 57 deletions(-) diff --git a/components/spiffs/spiffsgen.py b/components/spiffs/spiffsgen.py index d308310eaf..6a65a288dc 100755 --- a/components/spiffs/spiffsgen.py +++ b/components/spiffs/spiffsgen.py @@ -19,13 +19,21 @@ from __future__ import division, print_function import argparse -import ctypes import io import math import os import struct import sys +try: + import typing + + TSP = typing.TypeVar('TSP', bound='SpiffsObjPageWithIdx') + ObjIdsItem = typing.Tuple[int, typing.Type[TSP]] +except ImportError: + pass + + SPIFFS_PH_FLAG_USED_FINAL_INDEX = 0xF8 SPIFFS_PH_FLAG_USED_FINAL = 0xFC @@ -42,10 +50,22 @@ SPIFFS_BLOCK_IX_LEN = 2 # spiffs_block_ix class SpiffsBuildConfig(): - def __init__(self, page_size, page_ix_len, block_size, - block_ix_len, meta_len, obj_name_len, obj_id_len, - span_ix_len, packed, aligned, endianness, use_magic, use_magic_len, - aligned_obj_ix_tables): + def __init__(self, + page_size, # type: int + page_ix_len, # type: int + block_size, # type: int + block_ix_len, # type: int + meta_len, # type: int + obj_name_len, # type: int + obj_id_len, # type: int + span_ix_len, # type: int + packed, # type: bool + aligned, # type: bool + endianness, # type: str + use_magic, # type: bool + use_magic_len, # type: bool + aligned_obj_ix_tables # type: bool + ): if block_size % page_size != 0: raise RuntimeError('block size should be a multiple of page size') @@ -88,11 +108,11 @@ class SpiffsBuildConfig(): self.OBJ_INDEX_PAGES_HEADER_LEN_ALIGNED_PAD = 0 self.OBJ_INDEX_PAGES_OBJ_IDS_HEAD_LIM = (self.page_size - self.OBJ_INDEX_PAGES_HEADER_LEN_ALIGNED) // self.block_ix_len - self.OBJ_INDEX_PAGES_OBJ_IDS_LIM = (self.page_size - self.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED) / self.block_ix_len + self.OBJ_INDEX_PAGES_OBJ_IDS_LIM = (self.page_size - self.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED) // self.block_ix_len class SpiffsFullError(RuntimeError): - def __init__(self, message=None): + def __init__(self, message=None): # type: (typing.Optional[str]) -> None super(SpiffsFullError, self).__init__(message) @@ -109,35 +129,39 @@ class SpiffsPage(): 8: 'Q' } - _type_dict = { - 1: ctypes.c_ubyte, - 2: ctypes.c_ushort, - 4: ctypes.c_uint, - 8: ctypes.c_ulonglong - } - - def __init__(self, bix, build_config): + def __init__(self, bix, build_config): # type: (int, SpiffsBuildConfig) -> None self.build_config = build_config self.bix = bix + def to_binary(self): # type: () -> bytes + # Consider rewriting this using ABC + raise NotImplementedError() + + +class SpiffsObjPageWithIdx(SpiffsPage): + def __init__(self, obj_id, build_config): # type: (int, SpiffsBuildConfig) -> None + super(SpiffsObjPageWithIdx, self).__init__(0, build_config) + self.obj_id = obj_id + class SpiffsObjLuPage(SpiffsPage): - def __init__(self, bix, build_config): + def __init__(self, bix, build_config): # type: (int, SpiffsBuildConfig) -> None SpiffsPage.__init__(self, bix, build_config) self.obj_ids_limit = self.build_config.OBJ_LU_PAGES_OBJ_IDS_LIM - self.obj_ids = list() + self.obj_ids = list() # type: typing.List[ObjIdsItem] - def _calc_magic(self, blocks_lim): + def _calc_magic(self, blocks_lim): # type: (int) -> int # Calculate the magic value mirrorring computation done by the macro SPIFFS_MAGIC defined in # spiffs_nucleus.h magic = 0x20140529 ^ self.build_config.page_size if self.build_config.use_magic_len: magic = magic ^ (blocks_lim - self.bix) - magic = SpiffsPage._type_dict[self.build_config.obj_id_len](magic) - return magic.value + # narrow the result to build_config.obj_id_len bytes + mask = (2 << (8 * self.build_config.obj_id_len)) - 1 + return magic & mask - def register_page(self, page): + def register_page(self, page): # type: (TSP) -> None if not self.obj_ids_limit > 0: raise SpiffsFullError() @@ -145,8 +169,7 @@ class SpiffsObjLuPage(SpiffsPage): self.obj_ids.append(obj_id) self.obj_ids_limit -= 1 - def to_binary(self): - global test + def to_binary(self): # type: () -> bytes img = b'' for (obj_id, page_type) in self.obj_ids: @@ -161,7 +184,7 @@ class SpiffsObjLuPage(SpiffsPage): return img - def magicfy(self, blocks_lim): + def magicfy(self, blocks_lim): # type: (int) -> None # Only use magic value if no valid obj id has been written to the spot, which is the # spot taken up by the last obj id on last lookup page. The parent is responsible # for determining which is the last lookup page and calling this function. @@ -172,7 +195,7 @@ class SpiffsObjLuPage(SpiffsPage): 4: 0xFFFFFFFF, 8: 0xFFFFFFFFFFFFFFFF } - if (remaining >= 2): + if remaining >= 2: for i in range(remaining): if i == remaining - 2: self.obj_ids.append((self._calc_magic(blocks_lim), SpiffsObjDataPage)) @@ -182,10 +205,10 @@ class SpiffsObjLuPage(SpiffsPage): self.obj_ids_limit -= 1 -class SpiffsObjIndexPage(SpiffsPage): - def __init__(self, obj_id, span_ix, size, name, build_config): - SpiffsPage.__init__(self, 0, build_config) - self.obj_id = obj_id +class SpiffsObjIndexPage(SpiffsObjPageWithIdx): + def __init__(self, obj_id, span_ix, size, name, build_config + ): # type: (int, int, int, str, SpiffsBuildConfig) -> None + super(SpiffsObjIndexPage, self).__init__(obj_id, build_config) self.span_ix = span_ix self.name = name self.size = size @@ -195,16 +218,16 @@ class SpiffsObjIndexPage(SpiffsPage): else: self.pages_lim = self.build_config.OBJ_INDEX_PAGES_OBJ_IDS_LIM - self.pages = list() + self.pages = list() # type: typing.List[int] - def register_page(self, page): + def register_page(self, page): # type: (SpiffsObjDataPage) -> None if not self.pages_lim > 0: raise SpiffsFullError self.pages.append(page.offset) self.pages_lim -= 1 - def to_binary(self): + def to_binary(self): # type: () -> bytes obj_id = self.obj_id ^ (1 << ((self.build_config.obj_id_len * 8) - 1)) img = struct.pack(SpiffsPage._endianness_dict[self.build_config.endianness] + SpiffsPage._len_dict[self.build_config.obj_id_len] + @@ -244,15 +267,15 @@ class SpiffsObjIndexPage(SpiffsPage): return img -class SpiffsObjDataPage(SpiffsPage): - def __init__(self, offset, obj_id, span_ix, contents, build_config): - SpiffsPage.__init__(self, 0, build_config) - self.obj_id = obj_id +class SpiffsObjDataPage(SpiffsObjPageWithIdx): + def __init__(self, offset, obj_id, span_ix, contents, build_config + ): # type: (int, int, int, bytes, SpiffsBuildConfig) -> None + super(SpiffsObjDataPage, self).__init__(obj_id, build_config) self.span_ix = span_ix self.contents = contents self.offset = offset - def to_binary(self): + def to_binary(self): # type: () -> bytes img = struct.pack(SpiffsPage._endianness_dict[self.build_config.endianness] + SpiffsPage._len_dict[self.build_config.obj_id_len] + SpiffsPage._len_dict[self.build_config.span_ix_len] + @@ -271,17 +294,17 @@ class SpiffsObjDataPage(SpiffsPage): class SpiffsBlock(): - def _reset(self): + def _reset(self): # type: () -> None self.cur_obj_index_span_ix = 0 self.cur_obj_data_span_ix = 0 self.cur_obj_id = 0 - self.cur_obj_idx_page = None + self.cur_obj_idx_page = None # type: typing.Optional[SpiffsObjIndexPage] - def __init__(self, bix, blocks_lim, build_config): + def __init__(self, bix, blocks_lim, build_config): # type: (int, int, SpiffsBuildConfig) -> None self.build_config = build_config self.offset = bix * self.build_config.block_size self.remaining_pages = self.build_config.OBJ_USABLE_PAGES_PER_BLOCK - self.pages = list() + self.pages = list() # type: typing.List[SpiffsPage] self.bix = bix lu_pages = list() @@ -296,8 +319,9 @@ class SpiffsBlock(): self._reset() - def _register_page(self, page): + def _register_page(self, page): # type: (TSP) -> None if isinstance(page, SpiffsObjDataPage): + assert self.cur_obj_idx_page is not None self.cur_obj_idx_page.register_page(page) # can raise SpiffsFullError try: @@ -313,7 +337,8 @@ class SpiffsBlock(): self.pages.append(page) - def begin_obj(self, obj_id, size, name, obj_index_span_ix=0, obj_data_span_ix=0): + def begin_obj(self, obj_id, size, name, obj_index_span_ix=0, obj_data_span_ix=0 + ): # type: (int, int, str, int, int) -> None if not self.remaining_pages > 0: raise SpiffsFullError() self._reset() @@ -330,7 +355,7 @@ class SpiffsBlock(): self.remaining_pages -= 1 self.cur_obj_index_span_ix += 1 - def update_obj(self, contents): + def update_obj(self, contents): # type: (bytes) -> None if not self.remaining_pages > 0: raise SpiffsFullError() page = SpiffsObjDataPage(self.offset + (len(self.pages) * self.build_config.page_size), @@ -341,18 +366,19 @@ class SpiffsBlock(): self.cur_obj_data_span_ix += 1 self.remaining_pages -= 1 - def end_obj(self): + def end_obj(self): # type: () -> None self._reset() - def is_full(self): + def is_full(self): # type: () -> bool return self.remaining_pages <= 0 - def to_binary(self, blocks_lim): + def to_binary(self, blocks_lim): # type: (int) -> bytes img = b'' if self.build_config.use_magic: for (idx, page) in enumerate(self.pages): if idx == self.build_config.OBJ_LU_PAGES_PER_BLOCK - 1: + assert isinstance(page, SpiffsObjLuPage) page.magicfy(blocks_lim) img += page.to_binary() else: @@ -366,19 +392,19 @@ class SpiffsBlock(): class SpiffsFS(): - def __init__(self, img_size, build_config): + def __init__(self, img_size, build_config): # type: (int, SpiffsBuildConfig) -> None if img_size % build_config.block_size != 0: raise RuntimeError('image size should be a multiple of block size') self.img_size = img_size self.build_config = build_config - self.blocks = list() + self.blocks = list() # type: typing.List[SpiffsBlock] self.blocks_lim = self.img_size // self.build_config.block_size self.remaining_blocks = self.blocks_lim self.cur_obj_id = 1 # starting object id - def _create_block(self): + def _create_block(self): # type: () -> SpiffsBlock if self.is_full(): raise SpiffsFullError('the image size has been exceeded') @@ -387,12 +413,10 @@ class SpiffsFS(): self.remaining_blocks -= 1 return block - def is_full(self): + def is_full(self): # type: () -> bool return self.remaining_blocks <= 0 - def create_file(self, img_path, file_path): - contents = None - + def create_file(self, img_path, file_path): # type: (str, str) -> None if len(img_path) > self.build_config.obj_name_len: raise RuntimeError("object name '%s' too long" % img_path) @@ -446,7 +470,7 @@ class SpiffsFS(): self.cur_obj_id += 1 - def to_binary(self): + def to_binary(self): # type: () -> bytes img = b'' all_blocks = [] for block in self.blocks: @@ -466,7 +490,7 @@ class SpiffsFS(): return img -def main(): +def main(): # type: () -> None if sys.version_info[0] < 3: print('WARNING: Support for Python 2 is deprecated and will be removed in future versions.', file=sys.stderr) elif sys.version_info[0] == 3 and sys.version_info[1] < 6: diff --git a/tools/ci/mypy_ignore_list.txt b/tools/ci/mypy_ignore_list.txt index 2a0ff979da..3ff4733a05 100644 --- a/tools/ci/mypy_ignore_list.txt +++ b/tools/ci/mypy_ignore_list.txt @@ -19,7 +19,6 @@ components/protocomm/python/constants_pb2.py components/protocomm/python/sec0_pb2.py components/protocomm/python/sec1_pb2.py components/protocomm/python/session_pb2.py -components/spiffs/spiffsgen.py components/ulp/esp32ulp_mapgen.py components/wifi_provisioning/python/wifi_config_pb2.py components/wifi_provisioning/python/wifi_constants_pb2.py From a9b81341caa5c5143bac0458da5ca5e8557995ad Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Wed, 14 Apr 2021 18:42:56 +0200 Subject: [PATCH 4/6] tools: spiffsgen.py: make default arguments meaningful MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without this, defaults of inverse options (--no-magic-len) were based on the 'dest' value. In this case, dest='use_magic_len’, and the default value is True. Which is confusing, because both —use-magic-len and --no-magic-len show the same default value. This adds a custom help formatter class which doesn’t add default to the option help text if the help string already includes it. --- components/spiffs/spiffsgen.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/components/spiffs/spiffsgen.py b/components/spiffs/spiffsgen.py index 6a65a288dc..672e98f4a8 100755 --- a/components/spiffs/spiffsgen.py +++ b/components/spiffs/spiffsgen.py @@ -490,13 +490,31 @@ class SpiffsFS(): return img +class CustomHelpFormatter(argparse.HelpFormatter): + """ + Similar to argparse.ArgumentDefaultsHelpFormatter, except it + doesn't add the default value if "(default:" is already present. + This helps in the case of options with action="store_false", like + --no-magic or --no-magic-len. + """ + def _get_help_string(self, action): # type: (argparse.Action) -> str + if action.help is None: + return '' + if '%(default)' not in action.help and '(default:' not in action.help: + if action.default is not argparse.SUPPRESS: + defaulting_nargs = [argparse.OPTIONAL, argparse.ZERO_OR_MORE] + if action.option_strings or action.nargs in defaulting_nargs: + return action.help + ' (default: %(default)s)' + return action.help + + def main(): # type: () -> None if sys.version_info[0] < 3: print('WARNING: Support for Python 2 is deprecated and will be removed in future versions.', file=sys.stderr) elif sys.version_info[0] == 3 and sys.version_info[1] < 6: print('WARNING: Python 3 versions older than 3.6 are not supported.', file=sys.stderr) parser = argparse.ArgumentParser(description='SPIFFS Image Generator', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + formatter_class=CustomHelpFormatter) parser.add_argument('image_size', help='Size of the created image') @@ -534,7 +552,7 @@ def main(): # type: () -> None parser.add_argument('--no-magic', dest='use_magic', - help='Inverse of --use-magic', + help='Inverse of --use-magic (default: --use-magic is enabled)', action='store_false') parser.add_argument('--use-magic-len', @@ -544,7 +562,7 @@ def main(): # type: () -> None parser.add_argument('--no-magic-len', dest='use_magic_len', - help='Inverse of --use-magic-len', + help='Inverse of --use-magic-len (default: --use-magic-len is enabled)', action='store_false') parser.add_argument('--follow-symlinks', From 9af485307efcb1007a63f4aa957e4bec3a9dd38e Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Wed, 14 Apr 2021 18:43:26 +0200 Subject: [PATCH 5/6] tools: spiffsgen.py: minor lint fixes Not squashing these since they should have gone into the commit before adding type hints. --- components/spiffs/spiffsgen.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/components/spiffs/spiffsgen.py b/components/spiffs/spiffsgen.py index 672e98f4a8..c1eb259cc8 100755 --- a/components/spiffs/spiffsgen.py +++ b/components/spiffs/spiffsgen.py @@ -112,8 +112,7 @@ class SpiffsBuildConfig(): class SpiffsFullError(RuntimeError): - def __init__(self, message=None): # type: (typing.Optional[str]) -> None - super(SpiffsFullError, self).__init__(message) + pass class SpiffsPage(): @@ -134,7 +133,6 @@ class SpiffsPage(): self.bix = bix def to_binary(self): # type: () -> bytes - # Consider rewriting this using ABC raise NotImplementedError() @@ -143,6 +141,9 @@ class SpiffsObjPageWithIdx(SpiffsPage): super(SpiffsObjPageWithIdx, self).__init__(0, build_config) self.obj_id = obj_id + def to_binary(self): # type: () -> bytes + raise NotImplementedError() + class SpiffsObjLuPage(SpiffsPage): def __init__(self, bix, build_config): # type: (int, SpiffsBuildConfig) -> None @@ -152,7 +153,7 @@ class SpiffsObjLuPage(SpiffsPage): self.obj_ids = list() # type: typing.List[ObjIdsItem] def _calc_magic(self, blocks_lim): # type: (int) -> int - # Calculate the magic value mirrorring computation done by the macro SPIFFS_MAGIC defined in + # Calculate the magic value mirroring computation done by the macro SPIFFS_MAGIC defined in # spiffs_nucleus.h magic = 0x20140529 ^ self.build_config.page_size if self.build_config.use_magic_len: @@ -300,7 +301,7 @@ class SpiffsBlock(): self.cur_obj_id = 0 self.cur_obj_idx_page = None # type: typing.Optional[SpiffsObjIndexPage] - def __init__(self, bix, blocks_lim, build_config): # type: (int, int, SpiffsBuildConfig) -> None + def __init__(self, bix, build_config): # type: (int, SpiffsBuildConfig) -> None self.build_config = build_config self.offset = bix * self.build_config.block_size self.remaining_pages = self.build_config.OBJ_USABLE_PAGES_PER_BLOCK @@ -408,7 +409,7 @@ class SpiffsFS(): if self.is_full(): raise SpiffsFullError('the image size has been exceeded') - block = SpiffsBlock(len(self.blocks), self.blocks_lim, self.build_config) + block = SpiffsBlock(len(self.blocks), self.build_config) self.blocks.append(block) self.remaining_blocks -= 1 return block @@ -479,7 +480,7 @@ class SpiffsFS(): if self.build_config.use_magic: # Create empty blocks with magic numbers while self.remaining_blocks > 0: - block = SpiffsBlock(bix, self.blocks_lim, self.build_config) + block = SpiffsBlock(bix, self.build_config) all_blocks.append(block.to_binary(self.blocks_lim)) self.remaining_blocks -= 1 bix += 1 From 0bd9f6fe12e1e1d950091aebecbe13568229b59b Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Fri, 30 Apr 2021 18:37:15 +0200 Subject: [PATCH 6/6] tools: spiffsgen: fix length error, add test case --- .gitlab/ci/host-test.yml | 2 + components/spiffs/spiffsgen.py | 10 +-- components/spiffs/test_spiffsgen/__init__.py | 0 .../spiffs/test_spiffsgen/test_spiffsgen.py | 68 +++++++++++++++++++ tools/ci/executable-list.txt | 1 + 5 files changed, 76 insertions(+), 5 deletions(-) create mode 100644 components/spiffs/test_spiffsgen/__init__.py create mode 100755 components/spiffs/test_spiffsgen/test_spiffsgen.py diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index 2408d3e26a..7a2f978dcc 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -110,6 +110,8 @@ test_spiffs_on_host: script: - cd components/spiffs/test_spiffs_host/ - make test + - cd ../test_spiffsgen + - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_spiffsgen.py test_multi_heap_on_host: extends: .host_test_template diff --git a/components/spiffs/spiffsgen.py b/components/spiffs/spiffsgen.py index c1eb259cc8..10884a87b2 100755 --- a/components/spiffs/spiffsgen.py +++ b/components/spiffs/spiffsgen.py @@ -49,7 +49,7 @@ SPIFFS_PAGE_IX_LEN = 2 # spiffs_page_ix SPIFFS_BLOCK_IX_LEN = 2 # spiffs_block_ix -class SpiffsBuildConfig(): +class SpiffsBuildConfig(object): def __init__(self, page_size, # type: int page_ix_len, # type: int @@ -115,7 +115,7 @@ class SpiffsFullError(RuntimeError): pass -class SpiffsPage(): +class SpiffsPage(object): _endianness_dict = { 'little': '<', 'big': '>' @@ -294,7 +294,7 @@ class SpiffsObjDataPage(SpiffsObjPageWithIdx): return img -class SpiffsBlock(): +class SpiffsBlock(object): def _reset(self): # type: () -> None self.cur_obj_index_span_ix = 0 self.cur_obj_data_span_ix = 0 @@ -392,7 +392,7 @@ class SpiffsBlock(): return img -class SpiffsFS(): +class SpiffsFS(object): def __init__(self, img_size, build_config): # type: (int, SpiffsBuildConfig) -> None if img_size % build_config.block_size != 0: raise RuntimeError('image size should be a multiple of block size') @@ -486,7 +486,7 @@ class SpiffsFS(): bix += 1 else: # Just fill remaining spaces FF's - all_blocks.append(b'\xFF' * (self.img_size - len(img))) + all_blocks.append(b'\xFF' * (self.img_size - len(all_blocks) * self.build_config.block_size)) img += b''.join([blk for blk in all_blocks]) return img diff --git a/components/spiffs/test_spiffsgen/__init__.py b/components/spiffs/test_spiffsgen/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/components/spiffs/test_spiffsgen/test_spiffsgen.py b/components/spiffs/test_spiffsgen/test_spiffsgen.py new file mode 100755 index 0000000000..df6a5f7745 --- /dev/null +++ b/components/spiffs/test_spiffsgen/test_spiffsgen.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python +import os +import sys +import unittest + +try: + import typing +except ImportError: + pass + +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) +try: + import spiffsgen +except ImportError: + raise + + +class SpiffsgenTest(unittest.TestCase): + def test_configs(self): # type: () -> None + """Run spiffsgen with different configs, and check that + an image is generated (there is no exception), and the image size + is as expected. + """ + default_config = dict( + page_size=256, + page_ix_len=spiffsgen.SPIFFS_PAGE_IX_LEN, + block_size=4096, + block_ix_len=spiffsgen.SPIFFS_BLOCK_IX_LEN, + meta_len=4, + obj_name_len=32, + obj_id_len=spiffsgen.SPIFFS_BLOCK_IX_LEN, + span_ix_len=spiffsgen.SPIFFS_SPAN_IX_LEN, + packed=True, + aligned=True, + endianness='little', + use_magic=True, + use_magic_len=True, + aligned_obj_ix_tables=False + ) + + def make_config(**kwargs): # type: (typing.Any) -> spiffsgen.SpiffsBuildConfig + """Return SpiffsBuildConfig object with configuration set + by default_config plus any options overridden in kwargs. + """ + new_config = dict(default_config) + new_config.update(**kwargs) + return spiffsgen.SpiffsBuildConfig(**new_config) + + configs = [ + make_config(), + make_config(use_magic_len=False, use_magic=False, aligned_obj_ix_tables=True), + make_config(meta_len=4, obj_name_len=16), + make_config(block_size=8192), + make_config(page_size=512) + ] + + image_size = 64 * 1024 + for config in configs: + spiffs = spiffsgen.SpiffsFS(image_size, config) + spiffs.create_file('/test', __file__) + image = spiffs.to_binary() + self.assertEqual(len(image), image_size) + # Note: it would be nice to compile spiffs for host with the given + # config, and verify that the image is parsed correctly. + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index 5e293e3287..a94b5cde5f 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -17,6 +17,7 @@ components/partition_table/parttool.py components/partition_table/test_gen_esp32part_host/check_sizes_test.py components/partition_table/test_gen_esp32part_host/gen_esp32part_tests.py components/spiffs/spiffsgen.py +components/spiffs/test_spiffsgen/test_spiffsgen.py components/ulp/esp32ulp_mapgen.py docs/build_docs.py docs/check_lang_folder_sync.sh