diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index 2408d3e26a..7a2f978dcc 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -110,6 +110,8 @@ test_spiffs_on_host: script: - cd components/spiffs/test_spiffs_host/ - make test + - cd ../test_spiffsgen + - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_spiffsgen.py test_multi_heap_on_host: extends: .host_test_template diff --git a/components/spiffs/spiffsgen.py b/components/spiffs/spiffsgen.py index 56018ceac4..10884a87b2 100755 --- a/components/spiffs/spiffsgen.py +++ b/components/spiffs/spiffsgen.py @@ -19,13 +19,21 @@ from __future__ import division, print_function import argparse -import ctypes import io import math import os import struct import sys +try: + import typing + + TSP = typing.TypeVar('TSP', bound='SpiffsObjPageWithIdx') + ObjIdsItem = typing.Tuple[int, typing.Type[TSP]] +except ImportError: + pass + + SPIFFS_PH_FLAG_USED_FINAL_INDEX = 0xF8 SPIFFS_PH_FLAG_USED_FINAL = 0xFC @@ -41,10 +49,23 @@ SPIFFS_PAGE_IX_LEN = 2 # spiffs_page_ix SPIFFS_BLOCK_IX_LEN = 2 # spiffs_block_ix -class SpiffsBuildConfig(): - def __init__(self, page_size, page_ix_len, block_size, - block_ix_len, meta_len, obj_name_len, obj_id_len, - span_ix_len, packed, aligned, endianness, use_magic, use_magic_len): +class SpiffsBuildConfig(object): + def __init__(self, + page_size, # type: int + page_ix_len, # type: int + block_size, # type: int + block_ix_len, # type: int + meta_len, # type: int + obj_name_len, # type: int + obj_id_len, # type: int + span_ix_len, # type: int + packed, # type: bool + aligned, # type: bool + endianness, # type: str + use_magic, # type: bool + use_magic_len, # type: bool + aligned_obj_ix_tables # type: bool + ): if block_size % page_size != 0: raise RuntimeError('block size should be a multiple of page size') @@ -61,6 +82,7 @@ class SpiffsBuildConfig(): self.endianness = endianness self.use_magic = use_magic self.use_magic_len = use_magic_len + self.aligned_obj_ix_tables = aligned_obj_ix_tables self.PAGES_PER_BLOCK = self.block_size // self.page_size self.OBJ_LU_PAGES_PER_BLOCK = int(math.ceil(self.block_size / self.page_size * self.obj_id_len / self.page_size)) @@ -78,16 +100,22 @@ class SpiffsBuildConfig(): self.OBJ_INDEX_PAGES_HEADER_LEN = (self.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED + SPIFFS_PH_IX_SIZE_LEN + SPIFFS_PH_IX_OBJ_TYPE_LEN + self.obj_name_len + self.meta_len) - self.OBJ_INDEX_PAGES_OBJ_IDS_HEAD_LIM = (self.page_size - self.OBJ_INDEX_PAGES_HEADER_LEN) // self.block_ix_len - self.OBJ_INDEX_PAGES_OBJ_IDS_LIM = (self.page_size - self.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED) / self.block_ix_len + if aligned_obj_ix_tables: + self.OBJ_INDEX_PAGES_HEADER_LEN_ALIGNED = (self.OBJ_INDEX_PAGES_HEADER_LEN + SPIFFS_PAGE_IX_LEN - 1) & ~(SPIFFS_PAGE_IX_LEN - 1) + self.OBJ_INDEX_PAGES_HEADER_LEN_ALIGNED_PAD = self.OBJ_INDEX_PAGES_HEADER_LEN_ALIGNED - self.OBJ_INDEX_PAGES_HEADER_LEN + else: + self.OBJ_INDEX_PAGES_HEADER_LEN_ALIGNED = self.OBJ_INDEX_PAGES_HEADER_LEN + self.OBJ_INDEX_PAGES_HEADER_LEN_ALIGNED_PAD = 0 + + self.OBJ_INDEX_PAGES_OBJ_IDS_HEAD_LIM = (self.page_size - self.OBJ_INDEX_PAGES_HEADER_LEN_ALIGNED) // self.block_ix_len + self.OBJ_INDEX_PAGES_OBJ_IDS_LIM = (self.page_size - self.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED) // self.block_ix_len class SpiffsFullError(RuntimeError): - def __init__(self, message=None): - super(SpiffsFullError, self).__init__(message) + pass -class SpiffsPage(): +class SpiffsPage(object): _endianness_dict = { 'little': '<', 'big': '>' @@ -100,35 +128,41 @@ class SpiffsPage(): 8: 'Q' } - _type_dict = { - 1: ctypes.c_ubyte, - 2: ctypes.c_ushort, - 4: ctypes.c_uint, - 8: ctypes.c_ulonglong - } - - def __init__(self, bix, build_config): + def __init__(self, bix, build_config): # type: (int, SpiffsBuildConfig) -> None self.build_config = build_config self.bix = bix + def to_binary(self): # type: () -> bytes + raise NotImplementedError() + + +class SpiffsObjPageWithIdx(SpiffsPage): + def __init__(self, obj_id, build_config): # type: (int, SpiffsBuildConfig) -> None + super(SpiffsObjPageWithIdx, self).__init__(0, build_config) + self.obj_id = obj_id + + def to_binary(self): # type: () -> bytes + raise NotImplementedError() + class SpiffsObjLuPage(SpiffsPage): - def __init__(self, bix, build_config): + def __init__(self, bix, build_config): # type: (int, SpiffsBuildConfig) -> None SpiffsPage.__init__(self, bix, build_config) self.obj_ids_limit = self.build_config.OBJ_LU_PAGES_OBJ_IDS_LIM - self.obj_ids = list() + self.obj_ids = list() # type: typing.List[ObjIdsItem] - def _calc_magic(self, blocks_lim): - # Calculate the magic value mirrorring computation done by the macro SPIFFS_MAGIC defined in + def _calc_magic(self, blocks_lim): # type: (int) -> int + # Calculate the magic value mirroring computation done by the macro SPIFFS_MAGIC defined in # spiffs_nucleus.h magic = 0x20140529 ^ self.build_config.page_size if self.build_config.use_magic_len: magic = magic ^ (blocks_lim - self.bix) - magic = SpiffsPage._type_dict[self.build_config.obj_id_len](magic) - return magic.value + # narrow the result to build_config.obj_id_len bytes + mask = (2 << (8 * self.build_config.obj_id_len)) - 1 + return magic & mask - def register_page(self, page): + def register_page(self, page): # type: (TSP) -> None if not self.obj_ids_limit > 0: raise SpiffsFullError() @@ -136,8 +170,7 @@ class SpiffsObjLuPage(SpiffsPage): self.obj_ids.append(obj_id) self.obj_ids_limit -= 1 - def to_binary(self): - global test + def to_binary(self): # type: () -> bytes img = b'' for (obj_id, page_type) in self.obj_ids: @@ -152,7 +185,7 @@ class SpiffsObjLuPage(SpiffsPage): return img - def magicfy(self, blocks_lim): + def magicfy(self, blocks_lim): # type: (int) -> None # Only use magic value if no valid obj id has been written to the spot, which is the # spot taken up by the last obj id on last lookup page. The parent is responsible # for determining which is the last lookup page and calling this function. @@ -163,7 +196,7 @@ class SpiffsObjLuPage(SpiffsPage): 4: 0xFFFFFFFF, 8: 0xFFFFFFFFFFFFFFFF } - if (remaining >= 2): + if remaining >= 2: for i in range(remaining): if i == remaining - 2: self.obj_ids.append((self._calc_magic(blocks_lim), SpiffsObjDataPage)) @@ -173,10 +206,10 @@ class SpiffsObjLuPage(SpiffsPage): self.obj_ids_limit -= 1 -class SpiffsObjIndexPage(SpiffsPage): - def __init__(self, obj_id, span_ix, size, name, build_config): - SpiffsPage.__init__(self, 0, build_config) - self.obj_id = obj_id +class SpiffsObjIndexPage(SpiffsObjPageWithIdx): + def __init__(self, obj_id, span_ix, size, name, build_config + ): # type: (int, int, int, str, SpiffsBuildConfig) -> None + super(SpiffsObjIndexPage, self).__init__(obj_id, build_config) self.span_ix = span_ix self.name = name self.size = size @@ -186,16 +219,16 @@ class SpiffsObjIndexPage(SpiffsPage): else: self.pages_lim = self.build_config.OBJ_INDEX_PAGES_OBJ_IDS_LIM - self.pages = list() + self.pages = list() # type: typing.List[int] - def register_page(self, page): + def register_page(self, page): # type: (SpiffsObjDataPage) -> None if not self.pages_lim > 0: raise SpiffsFullError self.pages.append(page.offset) self.pages_lim -= 1 - def to_binary(self): + def to_binary(self): # type: () -> bytes obj_id = self.obj_id ^ (1 << ((self.build_config.obj_id_len * 8) - 1)) img = struct.pack(SpiffsPage._endianness_dict[self.build_config.endianness] + SpiffsPage._len_dict[self.build_config.obj_id_len] + @@ -217,7 +250,10 @@ class SpiffsObjIndexPage(SpiffsPage): self.size, SPIFFS_TYPE_FILE) - img += self.name.encode() + (b'\x00' * ((self.build_config.obj_name_len - len(self.name)) + self.build_config.meta_len)) + img += self.name.encode() + (b'\x00' * ( + (self.build_config.obj_name_len - len(self.name)) + + self.build_config.meta_len + + self.build_config.OBJ_INDEX_PAGES_HEADER_LEN_ALIGNED_PAD)) # Finally, add the page index of daa pages for page in self.pages: @@ -232,15 +268,15 @@ class SpiffsObjIndexPage(SpiffsPage): return img -class SpiffsObjDataPage(SpiffsPage): - def __init__(self, offset, obj_id, span_ix, contents, build_config): - SpiffsPage.__init__(self, 0, build_config) - self.obj_id = obj_id +class SpiffsObjDataPage(SpiffsObjPageWithIdx): + def __init__(self, offset, obj_id, span_ix, contents, build_config + ): # type: (int, int, int, bytes, SpiffsBuildConfig) -> None + super(SpiffsObjDataPage, self).__init__(obj_id, build_config) self.span_ix = span_ix self.contents = contents self.offset = offset - def to_binary(self): + def to_binary(self): # type: () -> bytes img = struct.pack(SpiffsPage._endianness_dict[self.build_config.endianness] + SpiffsPage._len_dict[self.build_config.obj_id_len] + SpiffsPage._len_dict[self.build_config.span_ix_len] + @@ -258,18 +294,18 @@ class SpiffsObjDataPage(SpiffsPage): return img -class SpiffsBlock(): - def _reset(self): +class SpiffsBlock(object): + def _reset(self): # type: () -> None self.cur_obj_index_span_ix = 0 self.cur_obj_data_span_ix = 0 self.cur_obj_id = 0 - self.cur_obj_idx_page = None + self.cur_obj_idx_page = None # type: typing.Optional[SpiffsObjIndexPage] - def __init__(self, bix, blocks_lim, build_config): + def __init__(self, bix, build_config): # type: (int, SpiffsBuildConfig) -> None self.build_config = build_config self.offset = bix * self.build_config.block_size self.remaining_pages = self.build_config.OBJ_USABLE_PAGES_PER_BLOCK - self.pages = list() + self.pages = list() # type: typing.List[SpiffsPage] self.bix = bix lu_pages = list() @@ -284,8 +320,9 @@ class SpiffsBlock(): self._reset() - def _register_page(self, page): + def _register_page(self, page): # type: (TSP) -> None if isinstance(page, SpiffsObjDataPage): + assert self.cur_obj_idx_page is not None self.cur_obj_idx_page.register_page(page) # can raise SpiffsFullError try: @@ -301,7 +338,8 @@ class SpiffsBlock(): self.pages.append(page) - def begin_obj(self, obj_id, size, name, obj_index_span_ix=0, obj_data_span_ix=0): + def begin_obj(self, obj_id, size, name, obj_index_span_ix=0, obj_data_span_ix=0 + ): # type: (int, int, str, int, int) -> None if not self.remaining_pages > 0: raise SpiffsFullError() self._reset() @@ -318,7 +356,7 @@ class SpiffsBlock(): self.remaining_pages -= 1 self.cur_obj_index_span_ix += 1 - def update_obj(self, contents): + def update_obj(self, contents): # type: (bytes) -> None if not self.remaining_pages > 0: raise SpiffsFullError() page = SpiffsObjDataPage(self.offset + (len(self.pages) * self.build_config.page_size), @@ -329,18 +367,19 @@ class SpiffsBlock(): self.cur_obj_data_span_ix += 1 self.remaining_pages -= 1 - def end_obj(self): + def end_obj(self): # type: () -> None self._reset() - def is_full(self): + def is_full(self): # type: () -> bool return self.remaining_pages <= 0 - def to_binary(self, blocks_lim): + def to_binary(self, blocks_lim): # type: (int) -> bytes img = b'' if self.build_config.use_magic: for (idx, page) in enumerate(self.pages): if idx == self.build_config.OBJ_LU_PAGES_PER_BLOCK - 1: + assert isinstance(page, SpiffsObjLuPage) page.magicfy(blocks_lim) img += page.to_binary() else: @@ -353,34 +392,32 @@ class SpiffsBlock(): return img -class SpiffsFS(): - def __init__(self, img_size, build_config): +class SpiffsFS(object): + def __init__(self, img_size, build_config): # type: (int, SpiffsBuildConfig) -> None if img_size % build_config.block_size != 0: raise RuntimeError('image size should be a multiple of block size') self.img_size = img_size self.build_config = build_config - self.blocks = list() + self.blocks = list() # type: typing.List[SpiffsBlock] self.blocks_lim = self.img_size // self.build_config.block_size self.remaining_blocks = self.blocks_lim self.cur_obj_id = 1 # starting object id - def _create_block(self): + def _create_block(self): # type: () -> SpiffsBlock if self.is_full(): raise SpiffsFullError('the image size has been exceeded') - block = SpiffsBlock(len(self.blocks), self.blocks_lim, self.build_config) + block = SpiffsBlock(len(self.blocks), self.build_config) self.blocks.append(block) self.remaining_blocks -= 1 return block - def is_full(self): + def is_full(self): # type: () -> bool return self.remaining_blocks <= 0 - def create_file(self, img_path, file_path): - contents = None - + def create_file(self, img_path, file_path): # type: (str, str) -> None if len(img_path) > self.build_config.obj_name_len: raise RuntimeError("object name '%s' too long" % img_path) @@ -434,31 +471,51 @@ class SpiffsFS(): self.cur_obj_id += 1 - def to_binary(self): + def to_binary(self): # type: () -> bytes img = b'' + all_blocks = [] for block in self.blocks: - img += block.to_binary(self.blocks_lim) + all_blocks.append(block.to_binary(self.blocks_lim)) bix = len(self.blocks) if self.build_config.use_magic: # Create empty blocks with magic numbers while self.remaining_blocks > 0: - block = SpiffsBlock(bix, self.blocks_lim, self.build_config) - img += block.to_binary(self.blocks_lim) + block = SpiffsBlock(bix, self.build_config) + all_blocks.append(block.to_binary(self.blocks_lim)) self.remaining_blocks -= 1 bix += 1 else: # Just fill remaining spaces FF's - img += '\xFF' * (self.img_size - len(img)) + all_blocks.append(b'\xFF' * (self.img_size - len(all_blocks) * self.build_config.block_size)) + img += b''.join([blk for blk in all_blocks]) return img -def main(): +class CustomHelpFormatter(argparse.HelpFormatter): + """ + Similar to argparse.ArgumentDefaultsHelpFormatter, except it + doesn't add the default value if "(default:" is already present. + This helps in the case of options with action="store_false", like + --no-magic or --no-magic-len. + """ + def _get_help_string(self, action): # type: (argparse.Action) -> str + if action.help is None: + return '' + if '%(default)' not in action.help and '(default:' not in action.help: + if action.default is not argparse.SUPPRESS: + defaulting_nargs = [argparse.OPTIONAL, argparse.ZERO_OR_MORE] + if action.option_strings or action.nargs in defaulting_nargs: + return action.help + ' (default: %(default)s)' + return action.help + + +def main(): # type: () -> None if sys.version_info[0] < 3: print('WARNING: Support for Python 2 is deprecated and will be removed in future versions.', file=sys.stderr) elif sys.version_info[0] == 3 and sys.version_info[1] < 6: print('WARNING: Python 3 versions older than 3.6 are not supported.', file=sys.stderr) parser = argparse.ArgumentParser(description='SPIFFS Image Generator', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + formatter_class=CustomHelpFormatter) parser.add_argument('image_size', help='Size of the created image') @@ -490,24 +547,38 @@ def main(): default=4) parser.add_argument('--use-magic', + dest='use_magic', help='Use magic number to create an identifiable SPIFFS image. Specify if CONFIG_SPIFFS_USE_MAGIC.', - action='store_true', - default=True) + action='store_true') + + parser.add_argument('--no-magic', + dest='use_magic', + help='Inverse of --use-magic (default: --use-magic is enabled)', + action='store_false') + + parser.add_argument('--use-magic-len', + dest='use_magic_len', + help='Use position in memory to create different magic numbers for each block. Specify if CONFIG_SPIFFS_USE_MAGIC_LENGTH.', + action='store_true') + + parser.add_argument('--no-magic-len', + dest='use_magic_len', + help='Inverse of --use-magic-len (default: --use-magic-len is enabled)', + action='store_false') parser.add_argument('--follow-symlinks', help='Take into account symbolic links during partition image creation.', - action='store_true', - default=False) - - parser.add_argument('--use-magic-len', - help='Use position in memory to create different magic numbers for each block. Specify if CONFIG_SPIFFS_USE_MAGIC_LENGTH.', - action='store_true', - default=True) + action='store_true') parser.add_argument('--big-endian', help='Specify if the target architecture is big-endian. If not specified, little-endian is assumed.', + action='store_true') + + parser.add_argument('--aligned-obj-ix-tables', action='store_true', - default=False) + help='Use aligned object index tables. Specify if SPIFFS_ALIGNED_OBJECT_INDEX_TABLES is set.') + + parser.set_defaults(use_magic=True, use_magic_len=True) args = parser.parse_args() @@ -520,7 +591,7 @@ def main(): args.block_size, SPIFFS_BLOCK_IX_LEN, args.meta_len, args.obj_name_len, SPIFFS_OBJ_ID_LEN, SPIFFS_SPAN_IX_LEN, True, True, 'big' if args.big_endian else 'little', - args.use_magic, args.use_magic_len) + args.use_magic, args.use_magic_len, args.aligned_obj_ix_tables) spiffs = SpiffsFS(image_size, spiffs_build_default) diff --git a/components/spiffs/test_spiffsgen/__init__.py b/components/spiffs/test_spiffsgen/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/components/spiffs/test_spiffsgen/test_spiffsgen.py b/components/spiffs/test_spiffsgen/test_spiffsgen.py new file mode 100755 index 0000000000..df6a5f7745 --- /dev/null +++ b/components/spiffs/test_spiffsgen/test_spiffsgen.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python +import os +import sys +import unittest + +try: + import typing +except ImportError: + pass + +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) +try: + import spiffsgen +except ImportError: + raise + + +class SpiffsgenTest(unittest.TestCase): + def test_configs(self): # type: () -> None + """Run spiffsgen with different configs, and check that + an image is generated (there is no exception), and the image size + is as expected. + """ + default_config = dict( + page_size=256, + page_ix_len=spiffsgen.SPIFFS_PAGE_IX_LEN, + block_size=4096, + block_ix_len=spiffsgen.SPIFFS_BLOCK_IX_LEN, + meta_len=4, + obj_name_len=32, + obj_id_len=spiffsgen.SPIFFS_BLOCK_IX_LEN, + span_ix_len=spiffsgen.SPIFFS_SPAN_IX_LEN, + packed=True, + aligned=True, + endianness='little', + use_magic=True, + use_magic_len=True, + aligned_obj_ix_tables=False + ) + + def make_config(**kwargs): # type: (typing.Any) -> spiffsgen.SpiffsBuildConfig + """Return SpiffsBuildConfig object with configuration set + by default_config plus any options overridden in kwargs. + """ + new_config = dict(default_config) + new_config.update(**kwargs) + return spiffsgen.SpiffsBuildConfig(**new_config) + + configs = [ + make_config(), + make_config(use_magic_len=False, use_magic=False, aligned_obj_ix_tables=True), + make_config(meta_len=4, obj_name_len=16), + make_config(block_size=8192), + make_config(page_size=512) + ] + + image_size = 64 * 1024 + for config in configs: + spiffs = spiffsgen.SpiffsFS(image_size, config) + spiffs.create_file('/test', __file__) + image = spiffs.to_binary() + self.assertEqual(len(image), image_size) + # Note: it would be nice to compile spiffs for host with the given + # config, and verify that the image is parsed correctly. + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index 5e293e3287..a94b5cde5f 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -17,6 +17,7 @@ components/partition_table/parttool.py components/partition_table/test_gen_esp32part_host/check_sizes_test.py components/partition_table/test_gen_esp32part_host/gen_esp32part_tests.py components/spiffs/spiffsgen.py +components/spiffs/test_spiffsgen/test_spiffsgen.py components/ulp/esp32ulp_mapgen.py docs/build_docs.py docs/check_lang_folder_sync.sh diff --git a/tools/ci/mypy_ignore_list.txt b/tools/ci/mypy_ignore_list.txt index 2182f435fc..06eecb9cdd 100644 --- a/tools/ci/mypy_ignore_list.txt +++ b/tools/ci/mypy_ignore_list.txt @@ -19,7 +19,6 @@ components/protocomm/python/constants_pb2.py components/protocomm/python/sec0_pb2.py components/protocomm/python/sec1_pb2.py components/protocomm/python/session_pb2.py -components/spiffs/spiffsgen.py components/ulp/esp32ulp_mapgen.py components/wifi_provisioning/python/wifi_config_pb2.py components/wifi_provisioning/python/wifi_constants_pb2.py