From cd6056c9cde2326237814a9251356844776c40ba Mon Sep 17 00:00:00 2001 From: "Michael (XIAO Xufeng)" Date: Mon, 23 May 2022 15:06:46 +0800 Subject: [PATCH] ci: extract ElfUnitTestParser allowing resolve elf offline --- tools/unit-test-app/README.md | 4 +- .../unit-test-app/tools/ElfUnitTestParser.py | 83 +++++++++++++++++ tools/unit-test-app/tools/UnitTestParser.py | 92 ++++++++----------- 3 files changed, 122 insertions(+), 57 deletions(-) create mode 100644 tools/unit-test-app/tools/ElfUnitTestParser.py diff --git a/tools/unit-test-app/README.md b/tools/unit-test-app/README.md index cbb7bfd3de..a1999fd698 100644 --- a/tools/unit-test-app/README.md +++ b/tools/unit-test-app/README.md @@ -13,6 +13,7 @@ ESP-IDF unit tests are run using Unit Test App. The app can be built with the un * `idf.py -T -T ... build` with `component` set to names of the components to be included in the test app. Or `idf.py -T all build` to build the test app with all the tests for components having `test` subdirectory. * Follow the printed instructions to flash, or run `idf.py -p PORT flash`. * Unit test have a few preset sdkconfigs. It provides command `idf.py ut-clean-config_name` and `idf.py ut-build-config_name` (where `config_name` is the file name under `unit-test-app/configs` folder) to build with preset configs. For example, you can use `idf.py -T all ut-build-default` to build with config file `unit-test-app/configs/default`. Built binary for this config will be copied to `unit-test-app/output/config_name` folder. +* You may extract the test cases presented in the built elf file by calling `ElfUnitTestParser.py `. # Flash Size @@ -38,7 +39,7 @@ Unit test uses 3 stages in CI: `build`, `assign_test`, `unit_test`. ### Build Stage: -`build_esp_idf_tests` job will build all UT configs and parse test cases form built elf files. Built binary (`tools/unit-test-app/output`) and parsed cases (`components/idf_test/unit_test/TestCaseAll.yml`) will be saved as artifacts. +`build_esp_idf_tests` job will build all UT configs and run script `UnitTestParser.py` to parse test cases form built elf files. Built binary (`tools/unit-test-app/output`) and parsed cases (`components/idf_test/unit_test/TestCaseAll.yml`) will be saved as artifacts. When we add new test case, it will construct a structure to save case data during build. We'll parse the test case from this structure. The description (defined in test case: `TEST_CASE("name", "description")`) is used to extend test case definition. The format of test description is a list of tags: @@ -117,6 +118,7 @@ If you want to reproduce locally, you need to: * You can refer to [unit test document](https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/unit-tests.html#running-unit-tests) to run test manually. * Or, you can use `tools/unit-test-app/unit_test.py` to run the test cases (see below) +# Testing and debugging on local machine ## Running unit tests on local machine by `unit_test.py` First, install Python dependencies and export the Python path where the IDF CI Python modules are found: diff --git a/tools/unit-test-app/tools/ElfUnitTestParser.py b/tools/unit-test-app/tools/ElfUnitTestParser.py new file mode 100644 index 0000000000..1e23552a7e --- /dev/null +++ b/tools/unit-test-app/tools/ElfUnitTestParser.py @@ -0,0 +1,83 @@ +# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +import argparse +import os +import subprocess +import sys +from typing import Dict, List + +import yaml + +try: + import CreateSectionTable +except ImportError: + sys.path.append(os.path.expandvars(os.path.join('$IDF_PATH', 'tools', 'unit-test-app', 'tools'))) + import CreateSectionTable + + +def get_target_objdump(idf_target: str) -> str: + toolchain_for_target = { + 'esp32': 'xtensa-esp32-elf-', + 'esp32s2': 'xtensa-esp32s2-elf-', + 'esp32s3': 'xtensa-esp32s3-elf-', + 'esp32c2': 'riscv32-esp-elf-', + 'esp32c3': 'riscv32-esp-elf-', + } + return toolchain_for_target.get(idf_target, '') + 'objdump' + + +def parse_elf_test_cases(elf_file: str, idf_target: str) -> List[Dict]: + objdump = get_target_objdump(idf_target) + + try: + subprocess.check_output('{} -s {} > section_table.tmp'.format(objdump, elf_file), shell=True) + table = CreateSectionTable.SectionTable('section_table.tmp') + except subprocess.CalledProcessError: + raise Exception('Can\'t resolve elf file. File not found.') + finally: + os.remove('section_table.tmp') + + bin_test_cases = [] + try: + subprocess.check_output('{} -t {} | grep test_desc > case_address.tmp'.format(objdump, elf_file), + shell=True) + + with open('case_address.tmp', 'rb') as input_f: + for line in input_f: + # process symbol table like: "3ffb4310 l O .dram0.data 00000018 test_desc_33$5010" + sections = line.split() + test_addr = int(sections[0], 16) + section = sections[3] + + name_addr = table.get_unsigned_int(section, test_addr, 4) + desc_addr = table.get_unsigned_int(section, test_addr + 4, 4) + tc = { + 'name': table.get_string('any', name_addr), + 'desc': table.get_string('any', desc_addr), + 'function_count': table.get_unsigned_int(section, test_addr + 20, 4), + } + bin_test_cases.append(tc) + except subprocess.CalledProcessError: + raise Exception('Test cases not found') + finally: + os.remove('case_address.tmp') + + return bin_test_cases + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('elf_file', help='Elf file to parse') + parser.add_argument('-t', '--idf_target', + type=str, default=os.environ.get('IDF_TARGET', ''), + help='Target of the elf, e.g. esp32s2') + parser.add_argument('-o', '--output_file', + type=str, default='elf_test_cases.yml', + help='Target of the elf, e.g. esp32s2') + args = parser.parse_args() + + assert args.idf_target + + test_cases = parse_elf_test_cases(args.elf_file, args.idf_target) + with open(args.output_file, 'w') as out_file: + yaml.dump(test_cases, out_file, default_flow_style=False) diff --git a/tools/unit-test-app/tools/UnitTestParser.py b/tools/unit-test-app/tools/UnitTestParser.py index 3515b263e3..b73e7eb529 100644 --- a/tools/unit-test-app/tools/UnitTestParser.py +++ b/tools/unit-test-app/tools/UnitTestParser.py @@ -4,10 +4,9 @@ import argparse import os import re import shutil -import subprocess +import sys from copy import deepcopy -import CreateSectionTable import yaml try: @@ -15,6 +14,13 @@ try: except ImportError: from yaml import Loader as Loader # type: ignore +try: + from ElfUnitTestParser import parse_elf_test_cases +except ImportError: + sys.path.append(os.path.expandvars(os.path.join('$IDF_PATH', 'tools', 'unit-test-app', 'tools'))) + from ElfUnitTestParser import parse_elf_test_cases + + TEST_CASE_PATTERN = { 'initial condition': 'UTINIT1', 'chip_target': 'esp32', @@ -50,12 +56,6 @@ class Parser(object): ELF_FILE = 'unit-test-app.elf' SDKCONFIG_FILE = 'sdkconfig' STRIP_CONFIG_PATTERN = re.compile(r'(.+?)(_\d+)?$') - TOOLCHAIN_FOR_TARGET = { - 'esp32': 'xtensa-esp32-elf-', - 'esp32s2': 'xtensa-esp32s2-elf-', - 'esp32s3': 'xtensa-esp32s3-elf-', - 'esp32c3': 'riscv32-esp-elf-', - } def __init__(self, binary_folder, node_index): idf_path = os.getenv('IDF_PATH') @@ -67,7 +67,6 @@ class Parser(object): self.idf_target = idf_target self.node_index = node_index self.ut_bin_folder = binary_folder - self.objdump = Parser.TOOLCHAIN_FOR_TARGET.get(idf_target, '') + 'objdump' self.tag_def = yaml.load(open(os.path.join(idf_path, self.TAG_DEF_FILE), 'r'), Loader=Loader) self.module_map = yaml.load(open(os.path.join(idf_path, self.MODULE_DEF_FILE), 'r'), Loader=Loader) self.config_dependencies = yaml.load(open(os.path.join(idf_path, self.CONFIG_DEPENDENCY_FILE), 'r'), @@ -89,62 +88,43 @@ class Parser(object): test_groups = self.get_test_groups(os.path.join(configs_folder, config_name)) elf_file = os.path.join(config_output_folder, self.ELF_FILE) - subprocess.check_output('{} -t {} | grep test_desc > case_address.tmp'.format(self.objdump, elf_file), - shell=True) - subprocess.check_output('{} -s {} > section_table.tmp'.format(self.objdump, elf_file), shell=True) + bin_test_cases = parse_elf_test_cases(elf_file, self.idf_target) - table = CreateSectionTable.SectionTable('section_table.tmp') test_cases = [] + for bin_tc in bin_test_cases: + # we could split cases of same config into multiple binaries as we have limited rom space + # we should regard those configs like `default` and `default_2` as the same config + match = self.STRIP_CONFIG_PATTERN.match(config_name) + stripped_config_name = match.group(1) - # we could split cases of same config into multiple binaries as we have limited rom space - # we should regard those configs like `default` and `default_2` as the same config - match = self.STRIP_CONFIG_PATTERN.match(config_name) - stripped_config_name = match.group(1) + tc = self.parse_one_test_case(bin_tc['name'], bin_tc['desc'], config_name, stripped_config_name, tags) - with open('case_address.tmp', 'rb') as f: - for line in f: - # process symbol table like: "3ffb4310 l O .dram0.data 00000018 test_desc_33$5010" - line = line.split() - test_addr = int(line[0], 16) - section = line[3] + # check if duplicated case names + # we need to use it to select case, + # if duplicated IDs, Unity could select incorrect case to run + # and we need to check all cases no matter if it's going te be executed by CI + # also add app_name here, we allow same case for different apps + if (tc['summary'] + stripped_config_name) in self.test_case_names: + self.parsing_errors.append('{} ({}): duplicated test case ID: {}'.format(stripped_config_name, config_name, tc['summary'])) + else: + self.test_case_names.add(tc['summary'] + stripped_config_name) - name_addr = table.get_unsigned_int(section, test_addr, 4) - desc_addr = table.get_unsigned_int(section, test_addr + 4, 4) - function_count = table.get_unsigned_int(section, test_addr + 20, 4) - name = table.get_string('any', name_addr) - desc = table.get_string('any', desc_addr) + test_group_included = True + if test_groups is not None and tc['group'] not in test_groups: + test_group_included = False - tc = self.parse_one_test_case(name, desc, config_name, stripped_config_name, tags) - - # check if duplicated case names - # we need to use it to select case, - # if duplicated IDs, Unity could select incorrect case to run - # and we need to check all cases no matter if it's going te be executed by CI - # also add app_name here, we allow same case for different apps - if (tc['summary'] + stripped_config_name) in self.test_case_names: - self.parsing_errors.append('{} ({}): duplicated test case ID: {}'.format(stripped_config_name, config_name, tc['summary'])) + if tc['CI ready'] == 'Yes' and test_group_included: + # update test env list and the cases of same env list + if tc['test environment'] in self.test_env_tags: + self.test_env_tags[tc['test environment']].append(tc['ID']) else: - self.test_case_names.add(tc['summary'] + stripped_config_name) + self.test_env_tags.update({tc['test environment']: [tc['ID']]}) - test_group_included = True - if test_groups is not None and tc['group'] not in test_groups: - test_group_included = False + if bin_tc['function_count'] > 1: + tc.update({'child case num': bin_tc['function_count']}) - if tc['CI ready'] == 'Yes' and test_group_included: - # update test env list and the cases of same env list - if tc['test environment'] in self.test_env_tags: - self.test_env_tags[tc['test environment']].append(tc['ID']) - else: - self.test_env_tags.update({tc['test environment']: [tc['ID']]}) - - if function_count > 1: - tc.update({'child case num': function_count}) - - # only add cases need to be executed - test_cases.append(tc) - - os.remove('section_table.tmp') - os.remove('case_address.tmp') + # only add cases need to be executed + test_cases.append(tc) return test_cases