diff --git a/components/app_update/CMakeLists.txt b/components/app_update/CMakeLists.txt index be5abbffb4..68ab01c4f9 100644 --- a/components/app_update/CMakeLists.txt +++ b/components/app_update/CMakeLists.txt @@ -33,10 +33,8 @@ if(NOT BOOTLOADER_BUILD) idf_build_get_property(idf_path IDF_PATH) idf_build_get_property(python PYTHON) add_custom_command(OUTPUT ${blank_otadata_file} - COMMAND ${python} ${idf_path}/components/partition_table/parttool.py - --partition-type data --partition-subtype ota -q - --partition-table-file ${PARTITION_CSV_PATH} generate_blank_partition_file - --output ${blank_otadata_file}) + COMMAND ${python} ${idf_path}/components/partition_table/gen_empty_partition.py + ${otadata_size} ${blank_otadata_file}) add_custom_target(blank_ota_data ALL DEPENDS ${blank_otadata_file}) add_dependencies(app blank_ota_data) @@ -44,10 +42,16 @@ if(NOT BOOTLOADER_BUILD) set(otatool_py ${python} ${COMPONENT_DIR}/otatool.py) add_custom_target(read_otadata DEPENDS "${PARTITION_CSV_PATH}" - COMMAND ${otatool_py} --partition-table-file ${PARTITION_CSV_PATH} read_otadata) + COMMAND ${otatool_py} + --partition-table-file ${PARTITION_CSV_PATH} + --partition-table-offset ${PARTITION_TABLE_OFFSET} + read_otadata) add_custom_target(erase_otadata DEPENDS "${PARTITION_CSV_PATH}" - COMMAND ${otatool_py} --partition-table-file ${PARTITION_CSV_PATH} erase_otadata) + COMMAND ${otatool_py} + --partition-table-file ${PARTITION_CSV_PATH} + --partition-table-offset ${PARTITION_TABLE_OFFSET} + erase_otadata) esptool_py_flash_project_args(otadata ${otadata_offset} "${blank_otadata_file}" FLASH_IN_PROJECT) endif() diff --git a/components/app_update/Makefile.projbuild b/components/app_update/Makefile.projbuild index 435d7535ac..a3f2f68423 100644 --- a/components/app_update/Makefile.projbuild +++ b/components/app_update/Makefile.projbuild @@ -17,8 +17,7 @@ endif $(BLANK_OTA_DATA_FILE): partition_table_get_info $(PARTITION_TABLE_CSV_PATH) | check_python_dependencies $(shell if [ "$(OTA_DATA_OFFSET)" != "" ] && [ "$(OTA_DATA_SIZE)" != "" ]; then \ - $(PARTTOOL_PY) --partition-type data --partition-subtype ota --partition-table-file $(PARTITION_TABLE_CSV_PATH) \ - -q generate_blank_partition_file --output $(BLANK_OTA_DATA_FILE); \ + $(PYTHON) $(IDF_PATH)/components/partition_table/gen_empty_partition.py $(OTA_DATA_SIZE) $(BLANK_OTA_DATA_FILE); \ fi; ) $(eval BLANK_OTA_DATA_FILE = $(shell if [ "$(OTA_DATA_OFFSET)" != "" ] && [ "$(OTA_DATA_SIZE)" != "" ]; then \ echo $(BLANK_OTA_DATA_FILE); else echo " "; fi) ) @@ -30,10 +29,14 @@ blank_ota_data: $(BLANK_OTA_DATA_FILE) ESPTOOL_ALL_FLASH_ARGS += $(OTA_DATA_OFFSET) $(BLANK_OTA_DATA_FILE) erase_otadata: $(PARTITION_TABLE_CSV_PATH) partition_table_get_info | check_python_dependencies - $(OTATOOL_PY) --partition-table-file $(PARTITION_TABLE_CSV_PATH) erase_otadata + $(OTATOOL_PY) --partition-table-file $(PARTITION_TABLE_CSV_PATH) \ + --partition-table-offset $(PARTITION_TABLE_OFFSET) \ + erase_otadata read_otadata: $(PARTITION_TABLE_CSV_PATH) partition_table_get_info | check_python_dependencies - $(OTATOOL_PY) --partition-table-file $(PARTITION_TABLE_CSV_PATH) read_otadata + $(OTATOOL_PY) --partition-table-file $(PARTITION_TABLE_CSV_PATH) \ + --partition-table-offset $(partition_table_offset) \ + read_otadata erase_ota: erase_otadata @echo "WARNING: erase_ota is deprecated. Use erase_otadata instead." diff --git a/components/app_update/otatool.py b/components/app_update/otatool.py index aead479d92..2cfecde18b 100755 --- a/components/app_update/otatool.py +++ b/components/app_update/otatool.py @@ -21,16 +21,20 @@ import argparse import os import sys import binascii -import subprocess import tempfile import collections import struct -__version__ = '1.0' +try: + from parttool import PartitionName, PartitionType, ParttoolTarget, PARTITION_TABLE_OFFSET +except ImportError: + COMPONENTS_PATH = os.path.expandvars(os.path.join("$IDF_PATH", "components")) + PARTTOOL_DIR = os.path.join(COMPONENTS_PATH, "partition_table") -IDF_COMPONENTS_PATH = os.path.expandvars(os.path.join("$IDF_PATH", "components")) + sys.path.append(PARTTOOL_DIR) + from parttool import PartitionName, PartitionType, ParttoolTarget, PARTITION_TABLE_OFFSET -PARTTOOL_PY = os.path.join(IDF_COMPONENTS_PATH, "partition_table", "parttool.py") +__version__ = '2.0' SPI_FLASH_SEC_SIZE = 0x2000 @@ -42,121 +46,69 @@ def status(msg): print(msg) -def _invoke_parttool(parttool_args, args, output=False, partition=None): - invoke_args = [] +class OtatoolTarget(): - if partition: - invoke_args += [sys.executable, PARTTOOL_PY] + partition - else: - invoke_args += [sys.executable, PARTTOOL_PY, "--partition-type", "data", "--partition-subtype", "ota"] + OTADATA_PARTITION = PartitionType("data", "ota") - if quiet: - invoke_args += ["-q"] + def __init__(self, port=None, partition_table_offset=PARTITION_TABLE_OFFSET, partition_table_file=None, spi_flash_sec_size=SPI_FLASH_SEC_SIZE): + self.target = ParttoolTarget(port, partition_table_offset, partition_table_file) + self.spi_flash_sec_size = spi_flash_sec_size - if args.port != "": - invoke_args += ["--port", args.port] + temp_file = tempfile.NamedTemporaryFile(delete=False) + temp_file.close() + try: + self.target.read_partition(OtatoolTarget.OTADATA_PARTITION, temp_file.name) + with open(temp_file.name, "rb") as f: + self.otadata = f.read() + except Exception: + self.otadata = None + finally: + os.unlink(temp_file.name) - if args.partition_table_file: - invoke_args += ["--partition-table-file", args.partition_table_file] + def _check_otadata_partition(self): + if not self.otadata: + raise Exception("No otadata partition found") - if args.partition_table_offset: - invoke_args += ["--partition-table-offset", args.partition_table_offset] + def erase_otadata(self): + self._check_otadata_partition() + self.target.erase_partition(OtatoolTarget.OTADATA_PARTITION) - invoke_args += parttool_args + def _get_otadata_info(self): + info = [] - if output: - return subprocess.check_output(invoke_args) - else: - return subprocess.check_call(invoke_args) + otadata_info = collections.namedtuple("otadata_info", "seq crc") + for i in range(2): + start = i * (self.spi_flash_sec_size >> 1) -def _get_otadata_contents(args, check=True): - global quiet + seq = bytearray(self.otadata[start:start + 4]) + crc = bytearray(self.otadata[start + 28:start + 32]) - if check: - check_args = ["get_partition_info", "--info", "offset", "size"] + seq = struct.unpack('>I', seq) + crc = struct.unpack('>I', crc) - quiet = True - output = _invoke_parttool(check_args, args, True).split(b" ") - quiet = args.quiet + info.append(otadata_info(seq[0], crc[0])) - if not output: - raise RuntimeError("No ota_data partition found") + return info - with tempfile.NamedTemporaryFile(delete=False) as f: - f_name = f.name + def _get_partition_id_from_ota_id(self, ota_id): + if isinstance(ota_id, int): + return PartitionType("app", "ota_" + str(ota_id)) + else: + return PartitionName(ota_id) - try: - invoke_args = ["read_partition", "--output", f_name] - _invoke_parttool(invoke_args, args) - with open(f_name, "rb") as f: - contents = f.read() - finally: - os.unlink(f_name) + def switch_ota_partition(self, ota_id): + self._check_otadata_partition() - return contents + sys.path.append(PARTTOOL_DIR) + import gen_esp32part as gen - -def _get_otadata_status(otadata_contents): - status = [] - - otadata_status = collections.namedtuple("otadata_status", "seq crc") - - for i in range(2): - start = i * (SPI_FLASH_SEC_SIZE >> 1) - - seq = bytearray(otadata_contents[start:start + 4]) - crc = bytearray(otadata_contents[start + 28:start + 32]) - - seq = struct.unpack('>I', seq) - crc = struct.unpack('>I', crc) - - status.append(otadata_status(seq[0], crc[0])) - - return status - - -def read_otadata(args): - status("Reading ota_data partition contents...") - otadata_info = _get_otadata_contents(args) - otadata_info = _get_otadata_status(otadata_info) - - print(otadata_info) - - print("\t\t{:11}\t{:8s}|\t{:8s}\t{:8s}".format("OTA_SEQ", "CRC", "OTA_SEQ", "CRC")) - print("Firmware: 0x{:8x} \t 0x{:8x} |\t0x{:8x} \t 0x{:8x}".format(otadata_info[0].seq, otadata_info[0].crc, - otadata_info[1].seq, otadata_info[1].crc)) - - -def erase_otadata(args): - status("Erasing ota_data partition contents...") - _invoke_parttool(["erase_partition"], args) - status("Erased ota_data partition contents") - - -def switch_otadata(args): - sys.path.append(os.path.join(IDF_COMPONENTS_PATH, "partition_table")) - import gen_esp32part as gen - - with tempfile.NamedTemporaryFile(delete=False) as f: - f_name = f.name - - try: - def is_otadata_status_valid(status): + def is_otadata_info_valid(status): seq = status.seq % (1 << 32) crc = hex(binascii.crc32(struct.pack("I", seq), 0xFFFFFFFF) % (1 << 32)) return seq < (int('0xFFFFFFFF', 16) % (1 << 32)) and status.crc == crc - status("Looking for ota app partitions...") - - # In order to get the number of ota app partitions, we need the partition table - partition_table = None - invoke_args = ["get_partition_info", "--table", f_name] - - _invoke_parttool(invoke_args, args) - - partition_table = open(f_name, "rb").read() - partition_table = gen.PartitionTable.from_binary(partition_table) + partition_table = self.target.partition_table ota_partitions = list() @@ -171,39 +123,36 @@ def switch_otadata(args): ota_partitions = sorted(ota_partitions, key=lambda p: p.subtype) if not ota_partitions: - raise RuntimeError("No ota app partitions found") - - status("Verifying partition to switch to exists...") + raise Exception("No ota app partitions found") # Look for the app partition to switch to ota_partition_next = None try: - if args.name: - ota_partition_next = filter(lambda p: p.name == args.name, ota_partitions) + if isinstance(ota_id, int): + ota_partition_next = filter(lambda p: p.subtype - gen.MIN_PARTITION_SUBTYPE_APP_OTA == ota_id, ota_partitions) else: - ota_partition_next = filter(lambda p: p.subtype - gen.MIN_PARTITION_SUBTYPE_APP_OTA == args.slot, ota_partitions) + ota_partition_next = filter(lambda p: p.name == ota_id, ota_partitions) ota_partition_next = list(ota_partition_next)[0] except IndexError: - raise RuntimeError("Partition to switch to not found") + raise Exception("Partition to switch to not found") - otadata_contents = _get_otadata_contents(args) - otadata_status = _get_otadata_status(otadata_contents) + otadata_info = self._get_otadata_info() # Find the copy to base the computation for ota sequence number on otadata_compute_base = -1 # Both are valid, take the max as computation base - if is_otadata_status_valid(otadata_status[0]) and is_otadata_status_valid(otadata_status[1]): - if otadata_status[0].seq >= otadata_status[1].seq: + if is_otadata_info_valid(otadata_info[0]) and is_otadata_info_valid(otadata_info[1]): + if otadata_info[0].seq >= otadata_info[1].seq: otadata_compute_base = 0 else: otadata_compute_base = 1 # Only one copy is valid, use that - elif is_otadata_status_valid(otadata_status[0]): + elif is_otadata_info_valid(otadata_info[0]): otadata_compute_base = 0 - elif is_otadata_status_valid(otadata_status[1]): + elif is_otadata_info_valid(otadata_info[1]): otadata_compute_base = 1 # Both are invalid (could be initial state - all 0xFF's) else: @@ -216,7 +165,7 @@ def switch_otadata(args): # Find the next ota sequence number if otadata_compute_base == 0 or otadata_compute_base == 1: - base_seq = otadata_status[otadata_compute_base].seq % (1 << 32) + base_seq = otadata_info[otadata_compute_base].seq % (1 << 32) i = 0 while base_seq > target_seq % ota_partitions_num + i * ota_partitions_num: @@ -231,47 +180,68 @@ def switch_otadata(args): ota_seq_crc_next = binascii.crc32(ota_seq_next, 0xFFFFFFFF) % (1 << 32) ota_seq_crc_next = struct.pack("I", ota_seq_crc_next) - with open(f_name, "wb") as otadata_next_file: - start = (1 if otadata_compute_base == 0 else 0) * (SPI_FLASH_SEC_SIZE >> 1) + temp_file = tempfile.NamedTemporaryFile(delete=False) + temp_file.close() - otadata_next_file.write(otadata_contents) + try: + with open(temp_file.name, "wb") as otadata_next_file: + start = (1 if otadata_compute_base == 0 else 0) * (self.spi_flash_sec_size >> 1) - otadata_next_file.seek(start) - otadata_next_file.write(ota_seq_next) + otadata_next_file.write(self.otadata) - otadata_next_file.seek(start + 28) - otadata_next_file.write(ota_seq_crc_next) + otadata_next_file.seek(start) + otadata_next_file.write(ota_seq_next) - otadata_next_file.flush() + otadata_next_file.seek(start + 28) + otadata_next_file.write(ota_seq_crc_next) - _invoke_parttool(["write_partition", "--input", f_name], args) - status("Updated ota_data partition") - finally: - os.unlink(f_name) + otadata_next_file.flush() + + self.target.write_partition(OtatoolTarget.OTADATA_PARTITION, temp_file.name) + finally: + os.unlink(temp_file.name) + + def read_ota_partition(self, ota_id, output): + self.target.read_partition(self._get_partition_id_from_ota_id(ota_id), output) + + def write_ota_partition(self, ota_id, input): + self.target.write_partition(self._get_partition_id_from_ota_id(ota_id), input) + + def erase_ota_partition(self, ota_id): + self.target.erase_partition(self._get_partition_id_from_ota_id(ota_id)) -def _get_partition_specifier(args): - if args.name: - return ["--partition-name", args.name] - else: - return ["--partition-type", "app", "--partition-subtype", "ota_" + str(args.slot)] +def _read_otadata(target): + target._check_otadata_partition() + + otadata_info = target._get_otadata_info(target.otadata) + + print("\t\t{:11}\t{:8s}|\t{:8s}\t{:8s}".format("OTA_SEQ", "CRC", "OTA_SEQ", "CRC")) + print("Firmware: 0x{:8x} \t 0x{:8x} |\t0x{:8x} \t 0x{:8x}".format(otadata_info[0].seq, otadata_info[0].crc, + otadata_info[1].seq, otadata_info[1].crc)) -def read_ota_partition(args): - invoke_args = ["read_partition", "--output", args.output] - _invoke_parttool(invoke_args, args, partition=_get_partition_specifier(args)) - status("Read ota partition contents to file {}".format(args.output)) +def _erase_otadata(target): + target.erase_otadata() + status("Erased ota_data partition contents") -def write_ota_partition(args): - invoke_args = ["write_partition", "--input", args.input] - _invoke_parttool(invoke_args, args, partition=_get_partition_specifier(args)) - status("Written contents of file {} to ota partition".format(args.input)) +def _switch_ota_partition(target, ota_id): + target.switch_ota_partition(ota_id) -def erase_ota_partition(args): - invoke_args = ["erase_partition"] - _invoke_parttool(invoke_args, args, partition=_get_partition_specifier(args)) +def _read_ota_partition(target, ota_id, output): + target.read_ota_partition(ota_id, output) + status("Read ota partition contents to file {}".format(output)) + + +def _write_ota_partition(target, ota_id, input): + target.write_ota_partition(ota_id, input) + status("Written contents of file {} to ota partition".format(input)) + + +def _erase_ota_partition(target, ota_id): + target.erase_ota_partition(ota_id) status("Erased contents of ota partition") @@ -284,17 +254,20 @@ def main(): # There are two possible sources for the partition table: a device attached to the host # or a partition table CSV/binary file. These sources are mutually exclusive. - partition_table_info_source_args = parser.add_mutually_exclusive_group() + parser.add_argument("--port", "-p", help="port where the device to read the partition table from is attached") - partition_table_info_source_args.add_argument("--port", "-p", help="port where the device to read the partition table from is attached", default="") - partition_table_info_source_args.add_argument("--partition-table-file", "-f", help="file (CSV/binary) to read the partition table from", default="") + parser.add_argument("--partition-table-offset", "-o", help="offset to read the partition table from", type=str) - parser.add_argument("--partition-table-offset", "-o", help="offset to read the partition table from", default="0x8000") + parser.add_argument("--partition-table-file", "-f", help="file (CSV/binary) to read the partition table from; \ + overrides device attached to specified port as the partition table source when defined") subparsers = parser.add_subparsers(dest="operation", help="run otatool -h for additional help") + spi_flash_sec_size = argparse.ArgumentParser(add_help=False) + spi_flash_sec_size.add_argument("--spi-flash-sec-size", help="value of SPI_FLASH_SEC_SIZE macro", type=str) + # Specify the supported operations - subparsers.add_parser("read_otadata", help="read otadata partition") + subparsers.add_parser("read_otadata", help="read otadata partition", parents=[spi_flash_sec_size]) subparsers.add_parser("erase_otadata", help="erase otadata partition") slot_or_name_parser = argparse.ArgumentParser(add_help=False) @@ -302,7 +275,7 @@ def main(): slot_or_name_parser_args.add_argument("--slot", help="slot number of the ota partition", type=int) slot_or_name_parser_args.add_argument("--name", help="name of the ota partition") - subparsers.add_parser("switch_otadata", help="switch otadata partition", parents=[slot_or_name_parser]) + subparsers.add_parser("switch_ota_partition", help="switch otadata partition", parents=[slot_or_name_parser, spi_flash_sec_size]) read_ota_partition_subparser = subparsers.add_parser("read_ota_partition", help="read contents of an ota partition", parents=[slot_or_name_parser]) read_ota_partition_subparser.add_argument("--output", help="file to write the contents of the ota partition to") @@ -322,17 +295,69 @@ def main(): parser.print_help() sys.exit(1) - # Else execute the operation - operation_func = globals()[args.operation] + target_args = {} + + if args.port: + target_args["port"] = args.port + + if args.partition_table_file: + target_args["partition_table_file"] = args.partition_table_file + + if args.partition_table_offset: + target_args["partition_table_offset"] = int(args.partition_table_offset, 0) + + try: + if args.spi_flash_sec_size: + target_args["spi_flash_sec_size"] = int(args.spi_flash_sec_size, 0) + except AttributeError: + pass + + target = OtatoolTarget(**target_args) + + # Create the operation table and execute the operation + common_args = {'target':target} + + ota_id = [] + + try: + if args.name is not None: + ota_id = ["name"] + else: + if args.slot is not None: + ota_id = ["slot"] + except AttributeError: + pass + + otatool_ops = { + 'read_otadata':(_read_otadata, []), + 'erase_otadata':(_erase_otadata, []), + 'switch_ota_partition':(_switch_ota_partition, ota_id), + 'read_ota_partition':(_read_ota_partition, ["output"] + ota_id), + 'write_ota_partition':(_write_ota_partition, ["input"] + ota_id), + 'erase_ota_partition':(_erase_ota_partition, ota_id) + } + + (op, op_args) = otatool_ops[args.operation] + + for op_arg in op_args: + common_args.update({op_arg:vars(args)[op_arg]}) + + try: + common_args['ota_id'] = common_args.pop('name') + except KeyError: + try: + common_args['ota_id'] = common_args.pop('slot') + except KeyError: + pass if quiet: # If exceptions occur, suppress and exit quietly try: - operation_func(args) + op(**common_args) except Exception: sys.exit(2) else: - operation_func(args) + op(**common_args) if __name__ == '__main__': diff --git a/components/partition_table/Makefile.projbuild b/components/partition_table/Makefile.projbuild index 1785ad811e..6b28b093f5 100644 --- a/components/partition_table/Makefile.projbuild +++ b/components/partition_table/Makefile.projbuild @@ -63,16 +63,21 @@ $(PARTITION_TABLE_BIN_UNSIGNED): $(PARTITION_TABLE_CSV_PATH) $(SDKCONFIG_MAKEFIL all_binaries: $(PARTITION_TABLE_BIN) partition_table_get_info check_table_contents partition_table_get_info: $(PARTITION_TABLE_BIN) - $(eval PHY_DATA_OFFSET:=$(shell $(GET_PART_INFO) --partition-type data --partition-subtype phy \ - --partition-table-file $(PARTITION_TABLE_BIN) get_partition_info --info offset)) - $(eval APP_OFFSET:=$(shell $(GET_PART_INFO) --partition-boot-default \ - --partition-table-file $(PARTITION_TABLE_BIN) get_partition_info --info offset)) - $(eval OTA_DATA_OFFSET:=$(shell $(GET_PART_INFO) --partition-type data --partition-subtype ota \ - --partition-table-file $(PARTITION_TABLE_BIN) get_partition_info --info offset)) - $(eval OTA_DATA_SIZE:=$(shell $(GET_PART_INFO) --partition-type data --partition-subtype ota \ - --partition-table-file $(PARTITION_TABLE_BIN) get_partition_info --info size)) - $(eval FACTORY_OFFSET:=$(shell $(GET_PART_INFO) --partition-type app --partition-subtype factory \ - --partition-table-file $(PARTITION_TABLE_BIN) get_partition_info --info offset)) + $(eval PHY_DATA_OFFSET:=$(shell $(GET_PART_INFO) --partition-table-file $(PARTITION_TABLE_BIN) \ + --partition-table-offset $(PARTITION_TABLE_OFFSET) \ + get_partition_info --partition-type data --partition-subtype phy --info offset)) + $(eval APP_OFFSET:=$(shell $(GET_PART_INFO) --partition-table-file $(PARTITION_TABLE_BIN) \ + --partition-table-offset $(PARTITION_TABLE_OFFSET) \ + get_partition_info --partition-boot-default --info offset)) + $(eval OTA_DATA_OFFSET:=$(shell $(GET_PART_INFO) --partition-table-file $(PARTITION_TABLE_BIN) \ + --partition-table-offset $(PARTITION_TABLE_OFFSET) \ + get_partition_info --partition-type data --partition-subtype ota --info offset)) + $(eval OTA_DATA_SIZE:=$(shell $(GET_PART_INFO) --partition-table-file $(PARTITION_TABLE_BIN) \ + --partition-table-offset $(PARTITION_TABLE_OFFSET) \ + get_partition_info --partition-type data --partition-subtype ota --info size)) + $(eval FACTORY_OFFSET:=$(shell $(GET_PART_INFO) --partition-table-file $(PARTITION_TABLE_BIN) \ + --partition-table-offset $(PARTITION_TABLE_OFFSET) \ + get_partition_info --partition-type app --partition-subtype factory --info offset)) export APP_OFFSET export PHY_DATA_OFFSET diff --git a/components/partition_table/gen_empty_partition.py b/components/partition_table/gen_empty_partition.py new file mode 100644 index 0000000000..f65f74d706 --- /dev/null +++ b/components/partition_table/gen_empty_partition.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +# +# generates an empty binary file +# +# This tool generates an empty binary file of the required size. +# +# Copyright 2018 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import print_function, division +from __future__ import unicode_literals +import argparse +import sys + +__version__ = '1.0' + +quiet = False + + +def generate_blanked_file(size, output_path): + output = b"\xFF" * size + try: + stdout_binary = sys.stdout.buffer # Python 3 + except AttributeError: + stdout_binary = sys.stdout + with stdout_binary if output_path == '-' else open(output_path, 'wb') as f: + f.write(output) + + +def main(): + parser = argparse.ArgumentParser(description='Generates an empty binary file of the required size.') + parser.add_argument('size', help='Size of generated the file', type=str) + + parser.add_argument('output', help='Path for binary file.', nargs='?', default='-') + args = parser.parse_args() + + size = int(args.size, 0) + if size > 0: + generate_blanked_file(size, args.output) + return 0 + + +class InputError(RuntimeError): + def __init__(self, e): + super(InputError, self).__init__(e) + + +if __name__ == '__main__': + try: + r = main() + sys.exit(r) + except InputError as e: + print(e, file=sys.stderr) + sys.exit(2) diff --git a/components/partition_table/parttool.py b/components/partition_table/parttool.py index 14fcdf4fe7..f2c005a10c 100755 --- a/components/partition_table/parttool.py +++ b/components/partition_table/parttool.py @@ -24,193 +24,165 @@ import subprocess import tempfile import gen_esp32part as gen -__version__ = '1.0' -IDF_COMPONENTS_PATH = os.path.expandvars(os.path.join("$IDF_PATH", "components")) +__version__ = '2.0' + +COMPONENTS_PATH = os.path.expandvars(os.path.join("$IDF_PATH", "components")) +ESPTOOL_PY = os.path.join(COMPONENTS_PATH, "esptool_py", "esptool", "esptool.py") + +PARTITION_TABLE_OFFSET = 0x8000 -ESPTOOL_PY = os.path.join(IDF_COMPONENTS_PATH, "esptool_py", "esptool", "esptool.py") quiet = False def status(msg): - """ Print status message to stderr """ if not quiet: print(msg) -def _invoke_esptool(esptool_args, args): - m_esptool_args = [sys.executable, ESPTOOL_PY] +class _PartitionId(): - if args.port != "": - m_esptool_args.extend(["--port", args.port]) - - m_esptool_args.extend(esptool_args) - - if quiet: - with open(os.devnull, "w") as fnull: - subprocess.check_call(m_esptool_args, stdout=fnull, stderr=fnull) - else: - subprocess.check_call(m_esptool_args) + def __init__(self, name=None, type=None, subtype=None): + self.name = name + self.type = type + self.subtype = subtype -def _get_partition_table(args): - partition_table = None +class PartitionName(_PartitionId): - gen.offset_part_table = int(args.partition_table_offset, 0) - - if args.partition_table_file: - status("Reading partition table from partition table file...") - - try: - with open(args.partition_table_file, "rb") as partition_table_file: - partition_table = gen.PartitionTable.from_binary(partition_table_file.read()) - status("Partition table read from binary file {}".format(partition_table_file.name)) - except (gen.InputError, TypeError): - with open(args.partition_table_file, "r") as partition_table_file: - partition_table_file.seek(0) - partition_table = gen.PartitionTable.from_csv(partition_table_file.read()) - status("Partition table read from CSV file {}".format(partition_table_file.name)) - else: - port_info = (" on port " + args.port if args.port else "") - status("Reading partition table from device{}...".format(port_info)) - - f_name = None - with tempfile.NamedTemporaryFile(delete=False) as f: - f_name = f.name - - try: - invoke_args = ["read_flash", str(gen.offset_part_table), str(gen.MAX_PARTITION_LENGTH), f_name] - _invoke_esptool(invoke_args, args) - with open(f_name, "rb") as f: - partition_table = gen.PartitionTable.from_binary(f.read()) - status("Partition table read from device" + port_info) - finally: - os.unlink(f_name) - - return partition_table + def __init__(self, name): + _PartitionId.__init__(self, name=name) -def _get_partition(args): - partition_table = _get_partition_table(args) +class PartitionType(_PartitionId): - partition = None - - if args.partition_name: - partition = partition_table.find_by_name(args.partition_name) - elif args.partition_type and args.partition_subtype: - partition = partition_table.find_by_type(args.partition_type, args.partition_subtype) - elif args.partition_boot_default: - search = ["factory"] + ["ota_{}".format(d) for d in range(16)] - for subtype in search: - partition = partition_table.find_by_type("app", subtype) - if partition is not None: - break - else: - raise RuntimeError("Invalid partition selection arguments. Specify --partition-name OR \ - --partition-type and --partition-subtype OR --partition--boot-default.") - - if partition: - status("Found partition {}".format(str(partition))) - - return partition + def __init__(self, type, subtype): + _PartitionId.__init__(self, type=type, subtype=subtype) -def _get_and_check_partition(args): - partition = None - - partition = _get_partition(args) - - if not partition: - raise RuntimeError("Unable to find specified partition.") - - return partition +PARTITION_BOOT_DEFAULT = _PartitionId() -def write_partition(args): - erase_partition(args) +class ParttoolTarget(): - partition = _get_and_check_partition(args) + def __init__(self, port=None, partition_table_offset=PARTITION_TABLE_OFFSET, partition_table_file=None): + self.port = port - status("Checking input file size...") + gen.offset_part_table = partition_table_offset - with open(args.input, "rb") as input_file: - content_len = len(input_file.read()) - - if content_len != partition.size: - status("File size (0x{:x}) does not match partition size (0x{:x})".format(content_len, partition.size)) + if partition_table_file: + try: + with open(partition_table_file, "rb") as f: + partition_table = gen.PartitionTable.from_binary(f.read()) + except (gen.InputError, IOError, TypeError): + with open(partition_table_file, "r") as f: + f.seek(0) + partition_table = gen.PartitionTable.from_csv(f.read()) else: - status("File size matches partition size (0x{:x})".format(partition.size)) - - _invoke_esptool(["write_flash", str(partition.offset), args.input], args) - - status("Written contents of file '{}' to device at offset 0x{:x}".format(args.input, partition.offset)) - - -def read_partition(args): - partition = _get_and_check_partition(args) - _invoke_esptool(["read_flash", str(partition.offset), str(partition.size), args.output], args) - status("Read partition contents from device at offset 0x{:x} to file '{}'".format(partition.offset, args.output)) - - -def erase_partition(args): - partition = _get_and_check_partition(args) - _invoke_esptool(["erase_region", str(partition.offset), str(partition.size)], args) - status("Erased partition at offset 0x{:x} on device".format(partition.offset)) - - -def get_partition_info(args): - partition = None - - if args.table: - partition_table = _get_partition_table(args) - - if args.table.endswith(".csv"): - partition_table = partition_table.to_csv() - else: - partition_table = partition_table.to_binary() - - with open(args.table, "wb") as table_file: - table_file.write(partition_table) - status("Partition table written to " + table_file.name) - else: - partition = _get_partition(args) - - if partition: - info_dict = { - "offset": '0x{:x}'.format(partition.offset), - "size": '0x{:x}'.format(partition.size) - } - - infos = [] + temp_file = tempfile.NamedTemporaryFile(delete=False) + temp_file.close() try: - for info in args.info: - infos += [info_dict[info]] - except KeyError: - raise RuntimeError("Request for unknown partition info {}".format(info)) + self._call_esptool(["read_flash", str(partition_table_offset), str(gen.MAX_PARTITION_LENGTH), temp_file.name]) + with open(temp_file.name, "rb") as f: + partition_table = gen.PartitionTable.from_binary(f.read()) + finally: + os.unlink(temp_file.name) - status("Requested partition information [{}]:".format(", ".join(args.info))) - print(" ".join(infos)) - else: - status("Partition not found") + self.partition_table = partition_table + + def _call_esptool(self, args, out=None): + esptool_args = [sys.executable, ESPTOOL_PY] + + if self.port: + esptool_args += ["--port", self.port] + + esptool_args += args + + with open(os.devnull, "w") as null_file: + subprocess.check_call(esptool_args, stdout=null_file, stderr=null_file) + + def get_partition_info(self, partition_id): + partition = None + + if partition_id.name: + partition = self.partition_table.find_by_name(partition_id.name) + elif partition_id.type and partition_id.subtype: + partition = self.partition_table.find_by_type(partition_id.type, partition_id.subtype) + else: # default boot partition + search = ["factory"] + ["ota_{}".format(d) for d in range(16)] + for subtype in search: + partition = self.partition_table.find_by_type("app", subtype) + if partition: + break + + if not partition: + raise Exception("Partition does not exist") + + return partition + + def erase_partition(self, partition_id): + partition = self.get_partition_info(partition_id) + self._call_esptool(["erase_region", str(partition.offset), str(partition.size)]) + + def read_partition(self, partition_id, output): + partition = self.get_partition_info(partition_id) + self._call_esptool(["read_flash", str(partition.offset), str(partition.size), output]) + + def write_partition(self, partition_id, input): + self.erase_partition(partition_id) + + partition = self.get_partition_info(partition_id) + + with open(input, "rb") as input_file: + content_len = len(input_file.read()) + + if content_len > partition.size: + raise Exception("Input file size exceeds partition size") + + self._call_esptool(["write_flash", str(partition.offset), input]) -def generate_blank_partition_file(args): - output = None - stdout_binary = None +def _write_partition(target, partition_id, input): + target.write_partition(partition_id, input) + partition = target.get_partition_info(partition_id) + status("Written contents of file '{}' at offset 0x{:x}".format(input, partition.offset)) - partition = _get_and_check_partition(args) - output = b"\xFF" * partition.size + +def _read_partition(target, partition_id, output): + target.read_partition(partition_id, output) + partition = target.get_partition_info(partition_id) + status("Read partition '{}' contents from device at offset 0x{:x} to file '{}'" + .format(partition.name, partition.offset, output)) + + +def _erase_partition(target, partition_id): + target.erase_partition(partition_id) + partition = target.get_partition_info(partition_id) + status("Erased partition '{}' at offset 0x{:x}".format(partition.name, partition.offset)) + + +def _get_partition_info(target, partition_id, info): + try: + partition = target.get_partition_info(partition_id) + except Exception: + return + + info_dict = { + "offset": '0x{:x}'.format(partition.offset), + "size": '0x{:x}'.format(partition.size) + } + + infos = [] try: - stdout_binary = sys.stdout.buffer # Python 3 - except AttributeError: - stdout_binary = sys.stdout + for i in info: + infos += [info_dict[i]] + except KeyError: + raise RuntimeError("Request for unknown partition info {}".format(i)) - with stdout_binary if args.output == "" else open(args.output, 'wb') as f: - f.write(output) - status("Blank partition file '{}' generated".format(args.output)) + print(" ".join(infos)) def main(): @@ -220,48 +192,45 @@ def main(): parser.add_argument("--quiet", "-q", help="suppress stderr messages", action="store_true") - # There are two possible sources for the partition table: a device attached to the host - # or a partition table CSV/binary file. These sources are mutually exclusive. - partition_table_info_source_args = parser.add_mutually_exclusive_group() + # By default the device attached to the specified port is queried for the partition table. If a partition table file + # is specified, that is used instead. + parser.add_argument("--port", "-p", help="port where the target device of the command is connected to; the partition table is sourced from this device \ + when the partition table file is not defined") - partition_table_info_source_args.add_argument("--port", "-p", help="port where the device to read the partition table from is attached", default="") - partition_table_info_source_args.add_argument("--partition-table-file", "-f", help="file (CSV/binary) to read the partition table from") + parser.add_argument("--partition-table-offset", "-o", help="offset to read the partition table from", type=str) + parser.add_argument("--partition-table-file", "-f", help="file (CSV/binary) to read the partition table from; \ + overrides device attached to specified port as the partition table source when defined") - parser.add_argument("--partition-table-offset", "-o", help="offset to read the partition table from", default="0x8000") + partition_selection_parser = argparse.ArgumentParser(add_help=False) # Specify what partition to perform the operation on. This can either be specified using the # partition name or the first partition that matches the specified type/subtype - partition_selection_args = parser.add_mutually_exclusive_group() + partition_selection_args = partition_selection_parser.add_mutually_exclusive_group() partition_selection_args.add_argument("--partition-name", "-n", help="name of the partition") partition_selection_args.add_argument("--partition-type", "-t", help="type of the partition") partition_selection_args.add_argument('--partition-boot-default', "-d", help='select the default boot partition \ using the same fallback logic as the IDF bootloader', action="store_true") - parser.add_argument("--partition-subtype", "-s", help="subtype of the partition") + partition_selection_parser.add_argument("--partition-subtype", "-s", help="subtype of the partition") subparsers = parser.add_subparsers(dest="operation", help="run parttool -h for additional help") # Specify the supported operations - read_part_subparser = subparsers.add_parser("read_partition", help="read partition from device and dump contents into a file") + read_part_subparser = subparsers.add_parser("read_partition", help="read partition from device and dump contents into a file", + parents=[partition_selection_parser]) read_part_subparser.add_argument("--output", help="file to dump the read partition contents to") - write_part_subparser = subparsers.add_parser("write_partition", help="write contents of a binary file to partition on device") + write_part_subparser = subparsers.add_parser("write_partition", help="write contents of a binary file to partition on device", + parents=[partition_selection_parser]) write_part_subparser.add_argument("--input", help="file whose contents are to be written to the partition offset") - subparsers.add_parser("erase_partition", help="erase the contents of a partition on the device") + subparsers.add_parser("erase_partition", help="erase the contents of a partition on the device", parents=[partition_selection_parser]) - print_partition_info_subparser = subparsers.add_parser("get_partition_info", help="get partition information") - print_partition_info_subparser_info_type = print_partition_info_subparser.add_mutually_exclusive_group() - print_partition_info_subparser_info_type.add_argument("--info", help="type of partition information to get", nargs="+") - print_partition_info_subparser_info_type.add_argument("--table", help="dump the partition table to a file") - - generate_blank_subparser = subparsers.add_parser("generate_blank_partition_file", help="generate a blank (all 0xFF) partition file of \ - the specified partition that can be flashed to the device") - generate_blank_subparser.add_argument("--output", help="blank partition file filename") + print_partition_info_subparser = subparsers.add_parser("get_partition_info", help="get partition information", parents=[partition_selection_parser]) + print_partition_info_subparser.add_argument("--info", help="type of partition information to get", nargs="+") args = parser.parse_args() - quiet = args.quiet # No operation specified, display help and exit @@ -270,17 +239,55 @@ def main(): parser.print_help() sys.exit(1) - # Else execute the operation - operation_func = globals()[args.operation] + # Prepare the partition to perform operation on + if args.partition_name: + partition_id = PartitionName(args.partition_name) + elif args.partition_type: + if not args.partition_subtype: + raise RuntimeError("--partition-subtype should be defined when --partition-type is defined") + partition_id = PartitionType(args.partition_type, args.partition_subtype) + elif args.partition_boot_default: + partition_id = PARTITION_BOOT_DEFAULT + else: + raise RuntimeError("Partition to operate on should be defined using --partition-name OR \ + partition-type,--partition-subtype OR partition-boot-default") + + # Prepare the device to perform operation on + target_args = {} + + if args.port: + target_args["port"] = args.port + + if args.partition_table_file: + target_args["partition_table_file"] = args.partition_table_file + + if args.partition_table_offset: + target_args["partition_table_offset"] = int(args.partition_table_offset, 0) + + target = ParttoolTarget(**target_args) + + # Create the operation table and execute the operation + common_args = {'target':target, 'partition_id':partition_id} + parttool_ops = { + 'erase_partition':(_erase_partition, []), + 'read_partition':(_read_partition, ["output"]), + 'write_partition':(_write_partition, ["input"]), + 'get_partition_info':(_get_partition_info, ["info"]) + } + + (op, op_args) = parttool_ops[args.operation] + + for op_arg in op_args: + common_args.update({op_arg:vars(args)[op_arg]}) if quiet: # If exceptions occur, suppress and exit quietly try: - operation_func(args) + op(**common_args) except Exception: sys.exit(2) else: - operation_func(args) + op(**common_args) if __name__ == '__main__': diff --git a/components/partition_table/project_include.cmake b/components/partition_table/project_include.cmake index d8e31c7d1d..9f0dc93854 100644 --- a/components/partition_table/project_include.cmake +++ b/components/partition_table/project_include.cmake @@ -39,7 +39,7 @@ function(partition_table_get_partition_info result get_part_info_args part_info) ${idf_path}/components/partition_table/parttool.py -q --partition-table-offset ${PARTITION_TABLE_OFFSET} --partition-table-file ${PARTITION_CSV_PATH} - ${get_part_info_args} get_partition_info --info ${part_info} + get_partition_info ${get_part_info_args} --info ${part_info} OUTPUT_VARIABLE info RESULT_VARIABLE exit_code OUTPUT_STRIP_TRAILING_WHITESPACE) diff --git a/components/partition_table/test_gen_esp32part_host/gen_esp32part_tests.py b/components/partition_table/test_gen_esp32part_host/gen_esp32part_tests.py index 989d4c4758..17f16c5b46 100755 --- a/components/partition_table/test_gen_esp32part_host/gen_esp32part_tests.py +++ b/components/partition_table/test_gen_esp32part_host/gen_esp32part_tests.py @@ -403,13 +403,13 @@ app,app, factory, 32K, 1M class PartToolTests(Py23TestCase): - def _run_parttool(self, csvcontents, args, info): + def _run_parttool(self, csvcontents, args): csvpath = tempfile.mktemp() with open(csvpath, "w") as f: f.write(csvcontents) try: - output = subprocess.check_output([sys.executable, "../parttool.py"] + args.split(" ") - + ["--partition-table-file", csvpath, "get_partition_info", "--info", info], + output = subprocess.check_output([sys.executable, "../parttool.py", "-q", "--partition-table-file", + csvpath, "get_partition_info"] + args, stderr=subprocess.STDOUT) self.assertNotIn(b"WARNING", output) m = re.search(b"0x[0-9a-fA-F]+", output) @@ -425,17 +425,17 @@ phy_init, data, phy, 0xf000, 0x1000 factory, app, factory, 0x10000, 1M """ - def rpt(args, info): - return self._run_parttool(csv, args, info) + def rpt(args): + return self._run_parttool(csv, args) self.assertEqual( - rpt("--partition-type=data --partition-subtype=nvs -q", "offset"), b"0x9000") + rpt(["--partition-type", "data", "--partition-subtype", "nvs", "--info", "offset"]), b"0x9000") self.assertEqual( - rpt("--partition-type=data --partition-subtype=nvs -q", "size"), b"0x4000") + rpt(["--partition-type", "data", "--partition-subtype", "nvs", "--info", "size"]), b"0x4000") self.assertEqual( - rpt("--partition-name=otadata -q", "offset"), b"0xd000") + rpt(["--partition-name", "otadata", "--info", "offset"]), b"0xd000") self.assertEqual( - rpt("--partition-boot-default -q", "offset"), b"0x10000") + rpt(["--partition-boot-default", "--info", "offset"]), b"0x10000") def test_fallback(self): csv = """ @@ -446,16 +446,16 @@ ota_0, app, ota_0, 0x30000, 1M ota_1, app, ota_1, , 1M """ - def rpt(args, info): - return self._run_parttool(csv, args, info) + def rpt(args): + return self._run_parttool(csv, args) self.assertEqual( - rpt("--partition-type=app --partition-subtype=ota_1 -q", "offset"), b"0x130000") + rpt(["--partition-type", "app", "--partition-subtype", "ota_1", "--info", "offset"]), b"0x130000") self.assertEqual( - rpt("--partition-boot-default -q", "offset"), b"0x30000") # ota_0 + rpt(["--partition-boot-default", "--info", "offset"]), b"0x30000") # ota_0 csv_mod = csv.replace("ota_0", "ota_2") self.assertEqual( - self._run_parttool(csv_mod, "--partition-boot-default -q", "offset"), + self._run_parttool(csv_mod, ["--partition-boot-default", "--info", "offset"]), b"0x130000") # now default is ota_1 diff --git a/components/spiffs/Makefile.projbuild b/components/spiffs/Makefile.projbuild index 2fab90b172..13b2223a9a 100644 --- a/components/spiffs/Makefile.projbuild +++ b/components/spiffs/Makefile.projbuild @@ -21,9 +21,9 @@ define spiffs_create_partition_image $(1)_bin: $(PARTITION_TABLE_BIN) | check_python_dependencies - partition_size=`$(GET_PART_INFO) --partition-name $(1) \ + partition_size=`$(GET_PART_INFO) \ --partition-table-file $(PARTITION_TABLE_BIN) \ - get_partition_info --info size`; \ + get_partition_info --partition-name $(1) --info size`; \ $(PYTHON) $(SPIFFSGEN_PY) $$$$partition_size $(2) $(BUILD_DIR_BASE)/$(1).bin \ --page-size=$(CONFIG_SPIFFS_PAGE_SIZE) \ --obj-name-len=$(CONFIG_SPIFFS_OBJ_NAME_LEN) \ @@ -41,5 +41,5 @@ endif endef ESPTOOL_ALL_FLASH_ARGS += $(foreach partition,$(SPIFFSGEN_FLASH_IN_PROJECT), \ -$(shell $(GET_PART_INFO) --partition-name $(partition) \ ---partition-table-file $(PARTITION_TABLE_BIN) get_partition_info --info offset) $(BUILD_DIR_BASE)/$(partition).bin) \ No newline at end of file +$(shell $(GET_PART_INFO) --partition-table-file $(PARTITION_TABLE_BIN) \ +get_partition_info --partition-name $(partition) --info offset) $(BUILD_DIR_BASE)/$(partition).bin) \ No newline at end of file diff --git a/docs/en/api-guides/partition-tables.rst b/docs/en/api-guides/partition-tables.rst index f538a507b9..ecf2367925 100644 --- a/docs/en/api-guides/partition-tables.rst +++ b/docs/en/api-guides/partition-tables.rst @@ -169,4 +169,100 @@ A manual flashing command is also printed as part of ``make partition_table``. Note that updating the partition table doesn't erase data that may have been stored according to the old partition table. You can use ``make erase_flash`` (or ``esptool.py erase_flash``) to erase the entire flash contents. +Partition Tool (parttool.py) +---------------------------- + +The component `partition_table` provides a tool :component_file:`parttool.py` for performing partition-related operations on a target device. The following operations can be performed using the tool: + + - reading a partition and saving the contents to a file (read_partition) + - writing the contents of a file to a partition (write_partition) + - erasing a partition (erase_partition) + - retrieving info such as offset and size of a given partition (get_partition_info) + +The tool can either be imported and used from another Python script or invoked from shell script for users wanting to perform operation programmatically. This is facilitated by the tool's Python API +and command-line interface, respectively. + +Python API +~~~~~~~~~~~ + +Before anything else, make sure that the `parttool` module is imported. + +.. code-block:: python + + import sys + import os + + idf_path = os.environ["IDF_PATH"] # get value of IDF_PATH from environment + parttool_dir = os.path.join(idf_path, "components", "partition_table") # parttool.py lives in $IDF_PATH/components/partition_table + + sys.path.append(parttool_dir) # this enables Python to find parttool module + from parttool import * # import all names inside parttool module + +The starting point for using the tool's Python API to do is create a `ParttoolTarget` object: + +.. code-block:: python + + # Create a partool.py target device connected on serial port /dev/ttyUSB1 + target = ParttoolTarget("/dev/ttyUSB1") + +The created object can now be used to perform operations on the target device: + +.. code-block:: python + + # Erase partition with name 'storage' + target.erase_partition(PartitionName("storage")) + + # Read partition with type 'data' and subtype 'spiffs' and save to file 'spiffs.bin' + target.read_partition(PartitionType("data", "spiffs"), "spiffs.bin") + + # Write to partition 'factory' the contents of a file named 'factory.bin' + target.write_partition(PartitionName("factory"), "factory.bin") + + # Print the size of default boot partition + storage = target.get_partition_info(PARTITION_BOOT_DEFAULT) + print(storage.size) + +The partition to operate on is specified using `PartitionName` or `PartitionType` or PARTITION_BOOT_DEFAULT. As the name implies, these can be used to refer +to partitions of a particular name, type-subtype combination, or the default boot partition. + +More information on the Python API is available in the docstrings for the tool. + +Command-line Interface +~~~~~~~~~~~~~~~~~~~~~~ + +The command-line interface of `parttool.py` has the following structure: + +.. code-block:: bash + + parttool.py [command-args] [subcommand] [subcommand-args] + + - command-args - These are arguments that are needed for executing the main command (parttool.py), mostly pertaining to the target device + - subcommand - This is the operation to be performed + - subcommand-args - These are arguments that are specific to the chosen operation + +.. code-block:: bash + + # Erase partition with name 'storage' + parttool.py --port "/dev/ttyUSB1" erase_partition --partition-name=storage + + # Read partition with type 'data' and subtype 'spiffs' and save to file 'spiffs.bin' + parttool.py --port "/dev/ttyUSB1" read_partition --partition-type=data --partition-subtype=spiffs "spiffs.bin" + + # Write to partition 'factory' the contents of a file named 'factory.bin' + parttool.py --port "/dev/ttyUSB1" write_partition --partition-name=factory "factory.bin" + + # Print the size of default boot partition + parttool.py --port "/dev/ttyUSB1" get_partition_info --partition-boot-default --info size + +More information can be obtained by specifying `--help` as argument: + +.. code-block:: bash + + # Display possible subcommands and show main command argument descriptions + parttool.py --help + + # Show descriptions for specific subcommand arguments + parttool.py [subcommand] --help + + .. _secure boot: security/secure-boot.rst diff --git a/docs/en/api-reference/system/ota.rst b/docs/en/api-reference/system/ota.rst index 50938a3776..d83a38dab3 100644 --- a/docs/en/api-reference/system/ota.rst +++ b/docs/en/api-reference/system/ota.rst @@ -199,6 +199,104 @@ Secure OTA Updates Without Secure boot The verification of signed OTA updates can be performed even without enabling hardware secure boot. For doing so, refer :ref:`signed-app-verify` + +OTA Tool (otatool.py) +--------------------- + +The component `app_update` provides a tool :component_file:`otatool.py` for performing OTA partition-related operations on a target device. The following operations can be performed using the tool: + + - read contents of otadata partition (read_otadata) + - erase otadata partition, effectively resetting device to factory app (erase_otadata) + - switch OTA partitions (switch_ota_partition) + - erasing OTA partition (erase_ota_partition) + - write to OTA partition (write_ota_partition) + - read contents of OTA partition (read_ota_partition) + +The tool can either be imported and used from another Python script or invoked from shell script for users wanting to perform operation programmatically. This is facilitated by the tool's Python API +and command-line interface, respectively. + +Python API +^^^^^^^^^^ + +Before anything else, make sure that the `otatool` module is imported. + +.. code-block:: python + + import sys + import os + + idf_path = os.environ["IDF_PATH"] # get value of IDF_PATH from environment + otatool_dir = os.path.join(idf_path, "components", "app_update") # otatool.py lives in $IDF_PATH/components/app_update + + sys.path.append(otatool_dir) # this enables Python to find otatool module + from otatool import * # import all names inside otatool module + +The starting point for using the tool's Python API to do is create a `OtatoolTarget` object: + +.. code-block:: python + + # Create a partool.py target device connected on serial port /dev/ttyUSB1 + target = OtatoolTarget("/dev/ttyUSB1") + +The created object can now be used to perform operations on the target device: + +.. code-block:: python + + # Erase otadata, reseting the device to factory app + target.erase_otadata() + + # Erase contents of OTA app slot 0 + target.erase_ota_partition(0) + + # Switch boot partition to that of app slot 1 + target.switch_ota_partition(1) + + # Read OTA partition 'ota_3' and save contents to a file named 'ota_3.bin' + target.read_ota_partition("ota_3", "ota_3.bin") + +The OTA partition to operate on is specified using either the app slot number or the partition name. + +More information on the Python API is available in the docstrings for the tool. + +Command-line Interface +^^^^^^^^^^^^^^^^^^^^^^ + +The command-line interface of `otatool.py` has the following structure: + +.. code-block:: bash + + otatool.py [command-args] [subcommand] [subcommand-args] + + - command-args - these are arguments that are needed for executing the main command (parttool.py), mostly pertaining to the target device + - subcommand - this is the operation to be performed + - subcommand-args - these are arguments that are specific to the chosen operation + +.. code-block:: bash + + # Erase otadata, resetting the device to factory app + otatool.py --port "/dev/ttyUSB1" erase_otadata + + # Erase contents of OTA app slot 0 + otatool.py --port "/dev/ttyUSB1" erase_ota_partition --slot 0 + + # Switch boot partition to that of app slot 1 + otatool.py --port "/dev/ttyUSB1" switch_ota_partition --slot 1 + + # Read OTA partition 'ota_3' and save contents to a file named 'ota_3.bin' + otatool.py --port "/dev/ttyUSB1" read_ota_partition --name=ota_3 + + +More information can be obtained by specifying `--help` as argument: + +.. code-block:: bash + + # Display possible subcommands and show main command argument descriptions + otatool.py --help + + # Show descriptions for specific subcommand arguments + otatool.py [subcommand] --help + + See also -------- diff --git a/examples/storage/parttool/README.md b/examples/storage/parttool/README.md index b84203e03f..8b888fcfdd 100644 --- a/examples/storage/parttool/README.md +++ b/examples/storage/parttool/README.md @@ -4,10 +4,11 @@ This example demonstrates common operations the partitions tool [parttool.py](.. - reading, writing and erasing partitions, - retrieving info on a certain partition, -- dumping the entire partition table, and -- generating a blank partition file. +- dumping the entire partition table -Users taking a look at this example should focus on the contents of the python script [parttool_example.py](parttool_example.py). The script contains programmatic invocations of [parttool.py](../../../components/partition_table/parttool.py) in Python for the operations mentioned above; and can serve as a guide for users wanting to do the same in their applications. +Users taking a look at this example should focus on the contents of the Python script [parttool_example.py](parttool_example.py) or shell script [parttool_example.sh](parttool_example.sh). The scripts contain +programmatic invocation of the tool's functions via the Python API and command-line interface, respectively. Note +that on Windows, the shell script example requires a POSIX-compatible environment via MSYS2/Git Bash/WSL etc. The example performs the operations mentioned above in a straightforward manner: it performs writes to partitions and then verifies correct content by reading it back. For partitions, contents are compared to the originally written file. For the partition table, contents are verified against the partition table CSV @@ -17,50 +18,54 @@ file. An erased partition's contents is compared to a generated blank file. ### Build and Flash -Before running the example script [parttool_example.py](parttool_example.py), it is necessary to build and flash the firmware using the usual means: +Before running either of the example scripts, it is necessary to build and flash the firmware using the usual means: +Make: ```bash -# If using Make make build flash +``` -# If using CMake +CMake: +```bash idf.py build flash ``` ### Running [parttool_example.py](parttool_example.py) -The example can be executed by running the script [parttool_example.py](parttool_example.py). Either run it directly using - -```bash -./parttool_example.py -``` - -or run it using +The example can be executed by running the script [parttool_example.py](parttool_example.py) or [parttool_example.sh](parttool_example.sh). +Python script: ```bash python parttool_example.py ``` +Shell script: +``` +./parttool_example.sh +``` + The script searches for valid target devices connected to the host and performs the operations on the first one it finds. To perform the operations on a specific device, specify the port it is attached to during script invocation: +Python script: ```bash -# The target device is attached to /dev/ttyUSB2, for example python parttool_example.py --port /dev/ttyUSB2 ``` +Shell script: +``` +./parttool_example.sh /dev/ttyUSB2 +``` + ## Example output Running the script produces the following output: ``` Checking if device app binary matches built binary -Checking if device partition table matches partition table csv -Retrieving data partition offset and size Found data partition at offset 0x110000 with size 0x10000 Writing to data partition Reading data partition Erasing data partition -Generating blank data partition file Reading data partition Partition tool operations performed successfully! diff --git a/examples/storage/parttool/parttool_example.py b/examples/storage/parttool/parttool_example.py index 24b8fcd18f..bf2c6f7cdf 100755 --- a/examples/storage/parttool/parttool_example.py +++ b/examples/storage/parttool/parttool_example.py @@ -18,19 +18,12 @@ # limitations under the License. import os import sys -import subprocess import argparse -IDF_PATH = os.path.expandvars("$IDF_PATH") - -PARTTOOL_PY = os.path.join(IDF_PATH, "components", "partition_table", "parttool.py") - -PARTITION_TABLE_OFFSET = 0x8000 - -INVOKE_ARGS = [sys.executable, PARTTOOL_PY, "-q", "--partition-table-offset", str(PARTITION_TABLE_OFFSET)] +PARTITION_TABLE_DIR = os.path.join("components", "partition_table", "") -def sized_file_compare(file1, file2): +def assert_file_same(file1, file2, err): with open(file1, "rb") as f1: with open(file2, "rb") as f2: f1 = f1.read() @@ -41,121 +34,17 @@ def sized_file_compare(file1, file2): else: f1 = f1[:len(f2)] - return f1 == f2 - - -def check(condition, message): - if not condition: - print("Error: " + message) - sys.exit(1) - - -def write_data_partition(size): - print("Writing to data partition") - with open("write.bin", "wb") as f: - # Create a file to write to the data partition with randomly generated content - f.write(os.urandom(int(size, 16))) - - # Invokes the command - # - # parttool.py --partition-table-offset 0x8000 -q --partition-name storage write_partition --input write.bin - # - # to write the contents of a file to a partition in the device. - invoke_args = INVOKE_ARGS + ["--partition-name", "storage", "write_partition", "--input", f.name] - subprocess.check_call(invoke_args) - return f.name - - -def read_data_partition(): - print("Reading data partition") - # Invokes the command - # - # parttool.py --partition-table-offset 0x8000 -q --partition-name storage read_partition --output read.bin - # - # to read the contents of a partition in the device, which is then written to a file. - f = "read.bin" - invoke_args = INVOKE_ARGS + ["--partition-name", "storage", "read_partition", "--output", f] - subprocess.check_call(invoke_args) - return f - - -def get_data_partition_info(): - print("Retrieving data partition offset and size") - # Invokes the command - # - # parttool.py --partition-table-offset 0x8000 -q --partition-name storage get_partition_info --info offset size - # - # to get the offset and size of a partition named 'storage'. - invoke_args = INVOKE_ARGS + ["--partition-name", "storage", "get_partition_info", "--info", "offset", "size"] - - (offset, size) = subprocess.check_output(invoke_args).strip().split(b" ") - return (offset, size) - - -def check_app(args): - print("Checking if device app binary matches built binary") - # Invokes the command - # - # parttool.py --partition-table-offset 0x8000 --partition-type app --partition-subtype factory read_partition --output app.bin" - # - # to read the app binary and write it to a file. The read app binary is compared to the built binary in the build folder. - invoke_args = INVOKE_ARGS + ["--partition-type", "app", "--partition-subtype", "factory", "read_partition", "--output", "app.bin"] - subprocess.check_call(invoke_args) - - app_same = sized_file_compare("app.bin", args.binary) - check(app_same, "Device app binary does not match built binary") - - -def check_partition_table(): - sys.path.append(os.path.join(IDF_PATH, "components", "partition_table")) - import gen_esp32part as gen - - print("Checking if device partition table matches partition table csv") - # Invokes the command - # - # parttool.py --partition-table-offset 0x8000 get_partition_info --table table.bin - # - # to read the device partition table and write it to a file. The read partition table is compared to - # the partition table csv. - invoke_args = INVOKE_ARGS + ["get_partition_info", "--table", "table.bin"] - subprocess.check_call(invoke_args) - - with open("table.bin", "rb") as read: - partition_table_csv = os.path.join(IDF_PATH, "examples", "storage", "parttool", "partitions_example.csv") - with open(partition_table_csv, "r") as csv: - read = gen.PartitionTable.from_binary(read.read()) - csv = gen.PartitionTable.from_csv(csv.read()) - check(read == csv, "Device partition table does not match csv partition table") - - -def erase_data_partition(): - print("Erasing data partition") - # Invokes the command - # - # parttool.py --partition-table-offset 0x8000 --partition-name storage erase_partition - # - # to erase the 'storage' partition. - invoke_args = INVOKE_ARGS + ["--partition-name", "storage", "erase_partition"] - subprocess.check_call(invoke_args) - - -def generate_blank_data_file(): - print("Generating blank data partition file") - - # Invokes the command - # - # parttool.py --partition-table-offset 0x8000 --partition-name storage generate_blank_partition_file --output blank.bin - # - # to generate a blank partition file and write it to a file. The blank partition file has the same size as the - # 'storage' partition. - f = "blank.bin" - invoke_args = INVOKE_ARGS + ["--partition-name", "storage", "generate_blank_partition_file", "--output", f] - subprocess.check_call(invoke_args) - return f + if not f1 == f2: + raise Exception(err) def main(): - global INVOKE_ARGS + COMPONENTS_PATH = os.path.expandvars(os.path.join("$IDF_PATH", "components")) + PARTTOOL_DIR = os.path.join(COMPONENTS_PATH, "partition_table") + + sys.path.append(PARTTOOL_DIR) + from parttool import PartitionName, PartitionType, ParttoolTarget + from gen_empty_partition import generate_blanked_file parser = argparse.ArgumentParser("ESP-IDF Partitions Tool Example") @@ -164,43 +53,53 @@ def main(): args = parser.parse_args() - if args.port: - INVOKE_ARGS += ["--port", args.port] + target = ParttoolTarget(args.port) - # Before proceeding, do checks to verify whether the app and partition table in the device matches the built binary and - # the generated partition table during build - check_app(args) - check_partition_table() + # Read app partition and save the contents to a file. The app partition is identified + # using type-subtype combination + print("Checking if device app binary matches built binary") + factory = PartitionType("app", "factory") + target.read_partition(factory, "app.bin") + assert_file_same(args.binary, "app.bin", "Device app binary does not match built binary") - # Get the offset and size of the data partition - (offset, size) = get_data_partition_info() + # Retrieve info on data storage partition, this time identifying it by name. + storage = PartitionName("storage") + storage_info = target.get_partition_info(storage) + print("Found data partition at offset 0x{:x} with size 0x{:x}".format(storage_info.offset, storage_info.size)) - print("Found data partition at offset %s with size %s" % (offset, size)) + # Create a file whose contents will be written to the storage partition + with open("write.bin", "wb") as f: + # Create a file to write to the data partition with randomly generated content + f.write(os.urandom(storage_info.size)) - # Write a generated file of random bytes to the found data partition - written = write_data_partition(size) + # Write the contents of the created file to storage partition + print("Writing to data partition") + target.write_partition(storage, "write.bin") - # Read back the contents of the data partition - read = read_data_partition() + # Read back the contents of the storage partition + print("Reading data partition") + target.read_partition(storage, "read.bin") - # Compare the written and read back data - data_same = sized_file_compare(read, written) - check(data_same, "Read contents of the data partition does not match written data") + assert_file_same("write.bin", "read.bin", "Read contents of storage partition does not match source file contents") - # Erase the data partition - erase_data_partition() + # Erase contents of the storage partition + print("Erasing data partition") + target.erase_partition(storage) - # Read back the erase data partition, which should be all 0xFF's after erasure - read = read_data_partition() + # Read back the erased data partition + print("Reading data partition") + target.read_partition(storage, "read.bin") - # Generate blank partition file (all 0xFF's) - blank = generate_blank_data_file() + # Generate a file of all 0xFF + generate_blanked_file(storage_info.size, "blank.bin") - # Verify that the partition has been erased by comparing the contents to the generated blank file - data_same = sized_file_compare(read, blank) - check(data_same, "Erased data partition contents does not match blank partition file") + assert_file_same("blank.bin", "read.bin", "Contents of storage partition not fully erased") + # Example end and cleanup print("\nPartition tool operations performed successfully!") + clean_files = ["app.bin", "read.bin", "blank.bin", "write.bin"] + for clean_file in clean_files: + os.unlink(clean_file) if __name__ == '__main__': diff --git a/examples/storage/parttool/parttool_example.sh b/examples/storage/parttool/parttool_example.sh new file mode 100644 index 0000000000..888c914686 --- /dev/null +++ b/examples/storage/parttool/parttool_example.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# +# Demonstrates command-line interface of Partition Tool, parttool.py +# +# +# $1 - serial port where target device to operate on is connnected to, by default the first found valid serial port +# $2 - path to this example's built binary file (parttool.bin), by default $PWD/build/parttool.bin +PORT=$1 +PARTTOOL_PY="python $IDF_PATH/components/partition_table/parttool.py -q" + +if [[ "$PORT" != "" ]]; then + PARTTOOL_PY="$PARTTOOL_PY --port $PORT" +fi + +GEN_EMPTY_PARTITION_PY="python $IDF_PATH/components/partition_table/gen_empty_partition.py" + +BINARY=$2 + +if [[ "$BINARY" == "" ]]; then + BINARY=build/parttool.bin +fi + +function assert_file_same() +{ + sz_a=$(stat -c %s $1) + sz_b=$(stat -c %s $2) + sz=$((sz_a < sz_b ? sz_a : sz_b)) + res=$(cmp -s -n $sz $1 $2) || + (echo "!!!!!!!!!!!!!!!!!!!" + echo "FAILURE: $3" + echo "!!!!!!!!!!!!!!!!!!!") +} + +# Read app partition and save the contents to a file. The app partition is identified +# using type-subtype combination +echo "Checking if device app binary matches built binary" +$PARTTOOL_PY read_partition --partition-type=app --partition-subtype=factory --output=app.bin +assert_file_same app.bin $BINARY "Device app binary does not match built binary" + +# Retrieve info on data storage partition, this time identifying it by name. +offset=$($PARTTOOL_PY get_partition_info --partition-name=storage --info offset) +size=$($PARTTOOL_PY get_partition_info --partition-name=storage --info size) +echo "Found data partition at offset $offset with size $size" + +# Create a file whose contents will be written to the storage partition +head -c $(($size)) /dev/urandom > write.bin + +# Write the contents of the created file to storage partition +echo "Writing to data partition" +$PARTTOOL_PY write_partition --partition-name=storage --input write.bin + +# Read back the contents of the storage partition +echo "Reading data partition" +$PARTTOOL_PY read_partition --partition-name=storage --output read.bin + +assert_file_same write.bin read.bin "Read contents of storage partition does not match source file contents" + +# Erase contents of the storage partition +echo "Erasing data partition" +$PARTTOOL_PY erase_partition --partition-name=storage + +# Read back the erased data partition +echo "Reading data partition" +$PARTTOOL_PY read_partition --partition-name=storage --output read.bin + +# Generate a file of all 0xFF +$GEN_EMPTY_PARTITION_PY $(($size)) blank.bin + +assert_file_same read.bin blank.bin "Contents of storage partition not fully erased" + +# Example end and cleanup +printf "\nPartition tool operations performed successfully\n" +rm -rf app.bin read.bin blank.bin write.bin \ No newline at end of file diff --git a/examples/system/ota/otatool/README.md b/examples/system/ota/otatool/README.md index c14e1cfb0d..9b52e8952c 100644 --- a/examples/system/ota/otatool/README.md +++ b/examples/system/ota/otatool/README.md @@ -6,7 +6,9 @@ This example demonstrates common operations the OTA tool [otatool.py](../../../c - switching boot partitions, and - switching to factory partition. -Users taking a look at this example should focus on the contents of the python script [otatool_example.py](otatool_example.py). The script contains programmatic invocations of the tool [otatool.py](../../../components/app_update/otatool.py) in Python for the operations mentioned above; and can serve as a guide for users wanting to do the same in their applications. +Users taking a look at this example should focus on the contents of the Python script [otatool_example.py](otatool_example.py) or shell script [otatool_example.sh](otatool_example.sh). The scripts contain +programmatic invocation of the tool's functions via the Python API and command-line interface, respectively. Note +that on Windows, the shell script example requires a POSIX-compatible environment via MSYS2/Git Bash/WSL etc. The built application in this example outputs the currently running partition, whose output is used to verify if the tool switched OTA partitions succesfully. The built application binary is written to all OTA partitions at the start of the example to be able to determine the running @@ -16,38 +18,46 @@ partition for all switches performed. ### Build and Flash -Before running the example script [otatool_example.py](otatool_example.py), it is necessary to build and flash the firmware using the usual means: +Before running either of the example scripts, it is necessary to build and flash the firmware using the usual means: +Make: ```bash -# If using Make make build flash +``` -# If using CMake +CMake: +```bash idf.py build flash ``` ### Running [otatool_example.py](otatool_example.py) -The example can be executed by running the script [otatool_example.py](otatool_example.py). Either run it directly using - -```bash -./otatool_example.py -``` - -or run it using +The example can be executed by running the script [otatool_example.py](otatool_example.py) or [otatool_example.sh](otatool_example.sh). +Python script: ```bash python otatool_example.py ``` -The script searches for valid target devices connected to the host and performs the operations on the first one it finds. This could present problems if there -are multiple viable target devices attached to the host. To perform the operations on a specific device, specify the port it is attached to during script invocation: +Shell script: +``` +./otatool_example.sh +``` +The script searches for valid target devices connected to the host and performs the operations on the first one it finds. This could present problems if there +are multiple viable target devices attached to the host. To perform the operations on a specific device, specify the port it is attached to during script invocation ("/dev/ttyUSB2" for example): + +Python script: ```bash -# The target device is attached to /dev/ttyUSB2, for example python otatool_example.py --port /dev/ttyUSB2 ``` + +Shell script: +``` +./otatool_example.sh /dev/ttyUSB2 +``` + ## Example output Running the script produces the following output: @@ -55,16 +65,13 @@ Running the script produces the following output: ``` Writing factory firmware to ota_0 Writing factory firmware to ota_1 -Checking written firmware to ota_0 and ota_1 match factory firmware -Switching to ota partition name factory -Switching to ota partition name factory -Switching to ota partition slot 0 -Switching to ota partition name ota_1 -Switching to ota partition slot 1 -Switching to ota partition name ota_0 -Switching to ota partition slot 0 -Switching to ota partition name factory -Switching to ota partition slot 1 +Switching to factory app +Switching to OTA slot 0 +Switching to OTA slot 1 (twice in a row) +Switching to OTA slot 0 (twice in a row) +Switching to factory app +Switching to OTA slot 1 + +Partition tool operations performed successfully -OTA tool operations executed successfully! ``` diff --git a/examples/system/ota/otatool/get_running_partition.py b/examples/system/ota/otatool/get_running_partition.py new file mode 100644 index 0000000000..b91d71a6d4 --- /dev/null +++ b/examples/system/ota/otatool/get_running_partition.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python +# +# Demonstrates the use of otatool.py, a tool for performing ota partition level +# operations. +# +# Copyright 2018 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import sys +import serial +import subprocess +import re +import argparse + +from subprocess import CalledProcessError + + +def get_running_partition(port=None): + # Monitor the serial output of target device. The firmware outputs the currently + # running partition + + IDF_PATH = os.path.expandvars("$IDF_PATH") + sys.path.append(os.path.join(IDF_PATH, 'components', 'esptool_py', 'esptool')) + import esptool + + ESPTOOL_PY = os.path.join(IDF_PATH, "components", "esptool_py", "esptool", "esptool.py") + + baud = os.environ.get("ESPTOOL_BAUD", esptool.ESPLoader.ESP_ROM_BAUD) + + if not port: + error_message = "Unable to obtain default target device port.\nSerial log:\n\n" + try: + # Check what esptool.py finds on what port the device is connected to + output = subprocess.check_output([sys.executable, ESPTOOL_PY, "chip_id"]) # may raise CalledProcessError + pattern = r"Serial port ([\S]+)" + pattern = re.compile(pattern.encode()) + + port = re.search(pattern, output).group(1) # may raise AttributeError + except CalledProcessError as e: + raise Exception(error_message + e.output) + except AttributeError: + raise Exception(error_message + output) + + serial_instance = serial.serial_for_url(port.decode("utf-8"), baud, do_not_open=True) + + serial_instance.dtr = False + serial_instance.rts = False + + serial_instance.rts = True + serial_instance.open() + serial_instance.rts = False + + # Read until example end and find the currently running partition string + content = serial_instance.read_until(b"Example end") + pattern = re.compile(b"Running partition: ([a-z0-9_]+)") + running = re.search(pattern, content).group(1) + + return running.decode("utf-8") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--port", default=None) + args = parser.parse_args() + + try: + res = get_running_partition(args.port) + except Exception as e: + print(e.message) + sys.exit(1) + + print(res) + + +if __name__ == "__main__": + main() diff --git a/examples/system/ota/otatool/otatool_example.py b/examples/system/ota/otatool/otatool_example.py index 17ed0cdb9e..b2a464eaec 100755 --- a/examples/system/ota/otatool/otatool_example.py +++ b/examples/system/ota/otatool/otatool_example.py @@ -18,20 +18,12 @@ # limitations under the License. import os import sys -import subprocess import argparse -import serial -import re -IDF_PATH = os.path.expandvars("$IDF_PATH") - -OTATOOL_PY = os.path.join(IDF_PATH, "components", "app_update", "otatool.py") -ESPTOOL_PY = os.path.join(IDF_PATH, "components", "esptool_py", "esptool", "esptool.py") - -INVOKE_ARGS = [sys.executable, OTATOOL_PY, "-q"] +from get_running_partition import get_running_partition -def sized_file_compare(file1, file2): +def assert_file_same(file1, file2, err): with open(file1, "rb") as f1: with open(file2, "rb") as f2: f1 = f1.read() @@ -42,122 +34,22 @@ def sized_file_compare(file1, file2): else: f1 = f1[:len(f2)] - return f1 == f2 + if not f1 == f2: + raise Exception(err) -def check(condition, message): - if not condition: - print("Error: " + message) - sys.exit(1) - - -def flash_example_firmware_to_ota_partitions(args): - # Invokes the command - # - # otatool.py -q write_ota_partition --slot or - # otatool.py -q write_ota_partition --name - # - # to write the contents of a file to the specified ota partition (either using name or the slot number) - print("Writing factory firmware to ota_0") - invoke_args = INVOKE_ARGS + ["write_ota_partition", "--slot", "0", "--input", args.binary] - subprocess.check_call(invoke_args) - - print("Writing factory firmware to ota_1") - invoke_args = INVOKE_ARGS + ["write_ota_partition", "--name", "ota_1", "--input", args.binary] - subprocess.check_call(invoke_args) - - # Verify that the contents of the two ota slots are the same as that of the factory partition - print("Checking written firmware to ota_0 and ota_1 match factory firmware") - - # Invokes the command - # - # otatool.py -q read_ota_partition --slot or - # otatool.py -q read_ota_partition --name - # - # to read the contents of a specified ota partition (either using name or the slot number) and write to a file - invoke_args = INVOKE_ARGS + ["read_ota_partition", "--slot", "0", "--output", "app_0.bin"] - subprocess.check_call(invoke_args) - - invoke_args = INVOKE_ARGS + ["read_ota_partition", "--name", "ota_1", "--output", "app_1.bin"] - subprocess.check_call(invoke_args) - - ota_same = sized_file_compare("app_0.bin", args.binary) - check(ota_same, "Slot 0 app does not match factory app") - - ota_same = sized_file_compare("app_1.bin", args.binary) - check(ota_same, "Slot 1 app does not match factory app") - - -def check_running_ota_partition(expected, port=None): - # Monitor the serial output of target device. The firmware outputs the currently - # running partition. It should match the partition the otatool switched to. - - if expected == 0 or expected == "ota_0": - expected = b"ota_0" - elif expected == 1 or expected == "ota_1": - expected = b"ota_1" - else: - expected = b"factory" - - sys.path.append(os.path.join(IDF_PATH, 'components', 'esptool_py', 'esptool')) - import esptool - - baud = os.environ.get("ESPTOOL_BAUD", esptool.ESPLoader.ESP_ROM_BAUD) - - if not port: - # Check what esptool.py finds on what port the device is connected to - output = subprocess.check_output([sys.executable, ESPTOOL_PY, "chip_id"]) - pattern = r"Serial port ([\S]+)" - pattern = re.compile(pattern.encode()) - port = re.search(pattern, output).group(1) - - serial_instance = serial.serial_for_url(port.decode("utf-8"), baud, do_not_open=True) - - serial_instance.dtr = False - serial_instance.rts = False - - serial_instance.rts = True - serial_instance.open() - serial_instance.rts = False - - # Read until example end and find the currently running partition string - content = serial_instance.read_until(b"Example end") - pattern = re.compile(b"Running partition: ([a-z0-9_]+)") - running = re.search(pattern, content).group(1) - - check(expected == running, "Running partition %s does not match expected %s" % (running, expected)) - - -def switch_partition(part, port): - if isinstance(part, int): - spec = "slot" - else: - spec = "name" - - print("Switching to ota partition %s %s" % (spec, str(part))) - - if str(part) == "factory": - # Invokes the command - # - # otatool.py -q erase_otadata - # - # to erase the otadata partition, effectively setting boot firmware to - # factory - subprocess.check_call(INVOKE_ARGS + ["erase_otadata"]) - else: - # Invokes the command - # - # otatool.py -q switch_otadata --slot or - # otatool.py -q switch_otadata --name - # - # to switch to the indicated ota partition (either using name or the slot number) - subprocess.check_call(INVOKE_ARGS + ["switch_otadata", "--" + spec, str(part)]) - - check_running_ota_partition(part, port) +def assert_running_partition(expected, port=None): + running = get_running_partition(port) + if running != expected: + raise Exception("Running partition %s does not match expected %s" % (running, expected)) def main(): - global INVOKE_ARGS + COMPONENTS_PATH = os.path.expandvars(os.path.join("$IDF_PATH", "components")) + OTATOOL_DIR = os.path.join(COMPONENTS_PATH, "app_update") + + sys.path.append(OTATOOL_DIR) + from otatool import OtatoolTarget parser = argparse.ArgumentParser("ESP-IDF OTA Tool Example") @@ -165,29 +57,61 @@ def main(): parser.add_argument("--binary", "-b", help="path to built example binary", default=os.path.join("build", "otatool.bin")) args = parser.parse_args() - if args.port: - INVOKE_ARGS += ["--port", args.port] + target = OtatoolTarget(args.port) - # Flash the factory firmware to all ota partitions - flash_example_firmware_to_ota_partitions(args) + print("Writing factory firmware to ota_0") + target.write_ota_partition(0, args.binary) - # Perform switching ota partitions - switch_partition("factory", args.port) - switch_partition("factory", args.port) # check switching to factory partition twice in a row + print("Writing factory firmware to ota_1") + target.write_ota_partition("ota_1", args.binary) - switch_partition(0, args.port) + # Verify that the contents of the two ota slots are the same as that of the factory partition + print("Checking written firmware to ota_0 and ota_1 match factory firmware") + target.read_ota_partition("ota_0", "app0.bin") + target.read_ota_partition(1, "app1.bin") - switch_partition("ota_1", args.port) - switch_partition(1, args.port) # check switching to ota_1 partition twice in a row + assert_file_same("app0.bin", args.binary, "Slot 0 app does not match factory app") + assert_file_same("app1.bin", args.binary, "Slot 1 app does not match factory app") - switch_partition("ota_0", args.port) - switch_partition(0, args.port) # check switching to ota_0 partition twice in a row + # Switch to factory app + print("Switching to factory app") + target.erase_otadata() + assert_running_partition("factory") - switch_partition("factory", args.port) + # Switch to slot 0 + print("Switching to OTA slot 0") + target.switch_ota_partition(0) + assert_running_partition("ota_0") - switch_partition(1, args.port) # check switching to ota_1 partition from factory + # Switch to slot 1 twice in a row + print("Switching to OTA slot 1 (twice in a row)") + target.switch_ota_partition(1) + assert_running_partition("ota_1") + target.switch_ota_partition("ota_1") + assert_running_partition("ota_1") + # Switch to slot 0 twice in a row + print("Switching to OTA slot 0 (twice in a row)") + target.switch_ota_partition(0) + assert_running_partition("ota_0") + target.switch_ota_partition("ota_0") + assert_running_partition("ota_0") + + # Switch to factory app + print("Switching to factory app") + target.erase_otadata() + assert_running_partition("factory") + + # Switch to slot 1 + print("Switching to OTA slot 1") + target.switch_ota_partition(1) + assert_running_partition("ota_1") + + # Example end and cleanup print("\nOTA tool operations executed successfully!") + clean_files = ["app0.bin", "app1.bin"] + for clean_file in clean_files: + os.unlink(clean_file) if __name__ == '__main__': diff --git a/examples/system/ota/otatool/otatool_example.sh b/examples/system/ota/otatool/otatool_example.sh new file mode 100644 index 0000000000..bcf8456dcc --- /dev/null +++ b/examples/system/ota/otatool/otatool_example.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# +# Demonstrates command-line interface of OTA Partitions Tool, otatool.py +# +# +# $1 - serial port where target device to operate on is connnected to, by default the first found valid serial port +# $2 - path to this example's built binary file (parttool.bin), by default $PWD/build/otatool.bin + +PORT=$1 +OTATOOL_PY="python $IDF_PATH/components/app_update/otatool.py -q" + +if [[ "$PORT" != "" ]]; then + OTATOOL_PY="$OTATOOL_PY --port $PORT" +fi + +BINARY=$2 + +if [[ "$BINARY" == "" ]]; then + BINARY=build/otatool.bin +fi + +function assert_file_same() +{ + sz_a=$(stat -c %s $1) + sz_b=$(stat -c %s $2) + sz=$((sz_a < sz_b ? sz_a : sz_b)) + res=$(cmp -s -n $sz $1 $2) || + (echo "!!!!!!!!!!!!!!!!!!!" + echo "FAILURE: $3" + echo "!!!!!!!!!!!!!!!!!!!") +} + +function assert_running_partition() +{ + running=$(python get_running_partition.py) + if [[ "$running" != "$1" ]]; then + echo "!!!!!!!!!!!!!!!!!!!" + echo "FAILURE: Running partition '$running' does not match expected '$1'" + echo "!!!!!!!!!!!!!!!!!!!" + exit 1 + fi +} + +# Flash the example firmware to OTA partitions. The first write uses slot number to identify OTA +# partition, the second one uses the name. +echo "Writing factory firmware to ota_0" +$OTATOOL_PY write_ota_partition --slot 0 --input $BINARY + +echo "Writing factory firmware to ota_1" +$OTATOOL_PY write_ota_partition --name ota_1 --input $BINARY + +# Read back the written firmware +$OTATOOL_PY read_ota_partition --name ota_0 --output app0.bin +$OTATOOL_PY read_ota_partition --slot 1 --output app1.bin + +assert_file_same $BINARY app0.bin "Slot 0 app does not match factory app" +assert_file_same $BINARY app1.bin "Slot 1 app does not match factory app" + +# Switch to factory app +echo "Switching to factory app" +$OTATOOL_PY erase_otadata +assert_running_partition factory + +# Switch to slot 0 +echo "Switching to OTA slot 0" +$OTATOOL_PY switch_ota_partition --slot 0 +assert_running_partition ota_0 + +# Switch to slot 1 twice in a row +echo "Switching to OTA slot 1 (twice in a row)" +$OTATOOL_PY switch_ota_partition --slot 1 +assert_running_partition ota_1 +$OTATOOL_PY switch_ota_partition --name ota_1 +assert_running_partition ota_1 + +# Switch to slot 0 twice in a row +echo "Switching to OTA slot 0 (twice in a row)" +$OTATOOL_PY switch_ota_partition --slot 0 +assert_running_partition ota_0 +$OTATOOL_PY switch_ota_partition --name ota_0 +assert_running_partition ota_0 + +# Switch to factory app +echo "Switching to factory app" +$OTATOOL_PY erase_otadata +assert_running_partition factory + +# Switch to slot 1 +echo "Switching to OTA slot 1" +$OTATOOL_PY switch_ota_partition --slot 1 +assert_running_partition ota_1 + +# Example end and cleanup +printf "\nPartition tool operations performed successfully\n" +rm -rf app0.bin app1.bin \ No newline at end of file diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index 7cfadade67..dcb4610b77 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -7,6 +7,7 @@ components/espcoredump/test/test_espcoredump.sh components/heap/test_multi_heap_host/test_all_configs.sh components/idf_test/unit_test/TestCaseScript/IDFUnitTest/__init__.py components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py +components/partition_table/gen_empty_partition.py components/partition_table/gen_esp32part.py components/partition_table/parttool.py components/partition_table/test_gen_esp32part_host/gen_esp32part_tests.py @@ -21,7 +22,10 @@ examples/build_system/cmake/idf_as_lib/build.sh examples/build_system/cmake/idf_as_lib/run-esp32.sh examples/build_system/cmake/idf_as_lib/run.sh examples/storage/parttool/parttool_example.py +examples/storage/parttool/parttool_example.sh +examples/system/ota/otatool/get_running_partition.py examples/system/ota/otatool/otatool_example.py +examples/system/ota/otatool/otatool_example.sh tools/check_kconfigs.py tools/check_python_dependencies.py tools/ci/apply_bot_filter.py