kopia lustrzana https://github.com/espressif/esp-idf
Merge branch 'feat/elf_unit_test_parser' into 'master'
ci: extract ElfUnitTestParser allowing resolve elf offline See merge request espressif/esp-idf!18205pull/9068/head
Michael (XIAO Xufeng)
commit
9f5c03dc67
|
|
@ -13,6 +13,7 @@ ESP-IDF unit tests are run using Unit Test App. The app can be built with the un
|
|||
* `idf.py -T <component> -T <component> ... build` with `component` set to names of the components to be included in the test app. Or `idf.py -T all build` to build the test app with all the tests for components having `test` subdirectory.
|
||||
* Follow the printed instructions to flash, or run `idf.py -p PORT flash`.
|
||||
* Unit test have a few preset sdkconfigs. It provides command `idf.py ut-clean-config_name` and `idf.py ut-build-config_name` (where `config_name` is the file name under `unit-test-app/configs` folder) to build with preset configs. For example, you can use `idf.py -T all ut-build-default` to build with config file `unit-test-app/configs/default`. Built binary for this config will be copied to `unit-test-app/output/config_name` folder.
|
||||
* You may extract the test cases presented in the built elf file by calling `ElfUnitTestParser.py <your_elf>`.
|
||||
|
||||
# Flash Size
|
||||
|
||||
|
|
@ -38,7 +39,7 @@ Unit test uses 3 stages in CI: `build`, `assign_test`, `unit_test`.
|
|||
|
||||
### Build Stage:
|
||||
|
||||
`build_esp_idf_tests` job will build all UT configs and parse test cases form built elf files. Built binary (`tools/unit-test-app/output`) and parsed cases (`components/idf_test/unit_test/TestCaseAll.yml`) will be saved as artifacts.
|
||||
`build_esp_idf_tests` job will build all UT configs and run script `UnitTestParser.py` to parse test cases form built elf files. Built binary (`tools/unit-test-app/output`) and parsed cases (`components/idf_test/unit_test/TestCaseAll.yml`) will be saved as artifacts.
|
||||
|
||||
When we add new test case, it will construct a structure to save case data during build. We'll parse the test case from this structure. The description (defined in test case: `TEST_CASE("name", "description")`) is used to extend test case definition. The format of test description is a list of tags:
|
||||
|
||||
|
|
@ -117,6 +118,7 @@ If you want to reproduce locally, you need to:
|
|||
* You can refer to [unit test document](https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/unit-tests.html#running-unit-tests) to run test manually.
|
||||
* Or, you can use `tools/unit-test-app/unit_test.py` to run the test cases (see below)
|
||||
|
||||
# Testing and debugging on local machine
|
||||
## Running unit tests on local machine by `unit_test.py`
|
||||
|
||||
First, install Python dependencies and export the Python path where the IDF CI Python modules are found:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,83 @@
|
|||
# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
import argparse
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Dict, List
|
||||
|
||||
import yaml
|
||||
|
||||
try:
|
||||
import CreateSectionTable
|
||||
except ImportError:
|
||||
sys.path.append(os.path.expandvars(os.path.join('$IDF_PATH', 'tools', 'unit-test-app', 'tools')))
|
||||
import CreateSectionTable
|
||||
|
||||
|
||||
def get_target_objdump(idf_target: str) -> str:
|
||||
toolchain_for_target = {
|
||||
'esp32': 'xtensa-esp32-elf-',
|
||||
'esp32s2': 'xtensa-esp32s2-elf-',
|
||||
'esp32s3': 'xtensa-esp32s3-elf-',
|
||||
'esp32c2': 'riscv32-esp-elf-',
|
||||
'esp32c3': 'riscv32-esp-elf-',
|
||||
}
|
||||
return toolchain_for_target.get(idf_target, '') + 'objdump'
|
||||
|
||||
|
||||
def parse_elf_test_cases(elf_file: str, idf_target: str) -> List[Dict]:
|
||||
objdump = get_target_objdump(idf_target)
|
||||
|
||||
try:
|
||||
subprocess.check_output('{} -s {} > section_table.tmp'.format(objdump, elf_file), shell=True)
|
||||
table = CreateSectionTable.SectionTable('section_table.tmp')
|
||||
except subprocess.CalledProcessError:
|
||||
raise Exception('Can\'t resolve elf file. File not found.')
|
||||
finally:
|
||||
os.remove('section_table.tmp')
|
||||
|
||||
bin_test_cases = []
|
||||
try:
|
||||
subprocess.check_output('{} -t {} | grep test_desc > case_address.tmp'.format(objdump, elf_file),
|
||||
shell=True)
|
||||
|
||||
with open('case_address.tmp', 'rb') as input_f:
|
||||
for line in input_f:
|
||||
# process symbol table like: "3ffb4310 l O .dram0.data 00000018 test_desc_33$5010"
|
||||
sections = line.split()
|
||||
test_addr = int(sections[0], 16)
|
||||
section = sections[3]
|
||||
|
||||
name_addr = table.get_unsigned_int(section, test_addr, 4)
|
||||
desc_addr = table.get_unsigned_int(section, test_addr + 4, 4)
|
||||
tc = {
|
||||
'name': table.get_string('any', name_addr),
|
||||
'desc': table.get_string('any', desc_addr),
|
||||
'function_count': table.get_unsigned_int(section, test_addr + 20, 4),
|
||||
}
|
||||
bin_test_cases.append(tc)
|
||||
except subprocess.CalledProcessError:
|
||||
raise Exception('Test cases not found')
|
||||
finally:
|
||||
os.remove('case_address.tmp')
|
||||
|
||||
return bin_test_cases
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('elf_file', help='Elf file to parse')
|
||||
parser.add_argument('-t', '--idf_target',
|
||||
type=str, default=os.environ.get('IDF_TARGET', ''),
|
||||
help='Target of the elf, e.g. esp32s2')
|
||||
parser.add_argument('-o', '--output_file',
|
||||
type=str, default='elf_test_cases.yml',
|
||||
help='Target of the elf, e.g. esp32s2')
|
||||
args = parser.parse_args()
|
||||
|
||||
assert args.idf_target
|
||||
|
||||
test_cases = parse_elf_test_cases(args.elf_file, args.idf_target)
|
||||
with open(args.output_file, 'w') as out_file:
|
||||
yaml.dump(test_cases, out_file, default_flow_style=False)
|
||||
|
|
@ -4,10 +4,9 @@ import argparse
|
|||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from copy import deepcopy
|
||||
|
||||
import CreateSectionTable
|
||||
import yaml
|
||||
|
||||
try:
|
||||
|
|
@ -15,6 +14,13 @@ try:
|
|||
except ImportError:
|
||||
from yaml import Loader as Loader # type: ignore
|
||||
|
||||
try:
|
||||
from ElfUnitTestParser import parse_elf_test_cases
|
||||
except ImportError:
|
||||
sys.path.append(os.path.expandvars(os.path.join('$IDF_PATH', 'tools', 'unit-test-app', 'tools')))
|
||||
from ElfUnitTestParser import parse_elf_test_cases
|
||||
|
||||
|
||||
TEST_CASE_PATTERN = {
|
||||
'initial condition': 'UTINIT1',
|
||||
'chip_target': 'esp32',
|
||||
|
|
@ -50,12 +56,6 @@ class Parser(object):
|
|||
ELF_FILE = 'unit-test-app.elf'
|
||||
SDKCONFIG_FILE = 'sdkconfig'
|
||||
STRIP_CONFIG_PATTERN = re.compile(r'(.+?)(_\d+)?$')
|
||||
TOOLCHAIN_FOR_TARGET = {
|
||||
'esp32': 'xtensa-esp32-elf-',
|
||||
'esp32s2': 'xtensa-esp32s2-elf-',
|
||||
'esp32s3': 'xtensa-esp32s3-elf-',
|
||||
'esp32c3': 'riscv32-esp-elf-',
|
||||
}
|
||||
|
||||
def __init__(self, binary_folder, node_index):
|
||||
idf_path = os.getenv('IDF_PATH')
|
||||
|
|
@ -67,7 +67,6 @@ class Parser(object):
|
|||
self.idf_target = idf_target
|
||||
self.node_index = node_index
|
||||
self.ut_bin_folder = binary_folder
|
||||
self.objdump = Parser.TOOLCHAIN_FOR_TARGET.get(idf_target, '') + 'objdump'
|
||||
self.tag_def = yaml.load(open(os.path.join(idf_path, self.TAG_DEF_FILE), 'r'), Loader=Loader)
|
||||
self.module_map = yaml.load(open(os.path.join(idf_path, self.MODULE_DEF_FILE), 'r'), Loader=Loader)
|
||||
self.config_dependencies = yaml.load(open(os.path.join(idf_path, self.CONFIG_DEPENDENCY_FILE), 'r'),
|
||||
|
|
@ -89,62 +88,43 @@ class Parser(object):
|
|||
test_groups = self.get_test_groups(os.path.join(configs_folder, config_name))
|
||||
|
||||
elf_file = os.path.join(config_output_folder, self.ELF_FILE)
|
||||
subprocess.check_output('{} -t {} | grep test_desc > case_address.tmp'.format(self.objdump, elf_file),
|
||||
shell=True)
|
||||
subprocess.check_output('{} -s {} > section_table.tmp'.format(self.objdump, elf_file), shell=True)
|
||||
bin_test_cases = parse_elf_test_cases(elf_file, self.idf_target)
|
||||
|
||||
table = CreateSectionTable.SectionTable('section_table.tmp')
|
||||
test_cases = []
|
||||
for bin_tc in bin_test_cases:
|
||||
# we could split cases of same config into multiple binaries as we have limited rom space
|
||||
# we should regard those configs like `default` and `default_2` as the same config
|
||||
match = self.STRIP_CONFIG_PATTERN.match(config_name)
|
||||
stripped_config_name = match.group(1)
|
||||
|
||||
# we could split cases of same config into multiple binaries as we have limited rom space
|
||||
# we should regard those configs like `default` and `default_2` as the same config
|
||||
match = self.STRIP_CONFIG_PATTERN.match(config_name)
|
||||
stripped_config_name = match.group(1)
|
||||
tc = self.parse_one_test_case(bin_tc['name'], bin_tc['desc'], config_name, stripped_config_name, tags)
|
||||
|
||||
with open('case_address.tmp', 'rb') as f:
|
||||
for line in f:
|
||||
# process symbol table like: "3ffb4310 l O .dram0.data 00000018 test_desc_33$5010"
|
||||
line = line.split()
|
||||
test_addr = int(line[0], 16)
|
||||
section = line[3]
|
||||
# check if duplicated case names
|
||||
# we need to use it to select case,
|
||||
# if duplicated IDs, Unity could select incorrect case to run
|
||||
# and we need to check all cases no matter if it's going te be executed by CI
|
||||
# also add app_name here, we allow same case for different apps
|
||||
if (tc['summary'] + stripped_config_name) in self.test_case_names:
|
||||
self.parsing_errors.append('{} ({}): duplicated test case ID: {}'.format(stripped_config_name, config_name, tc['summary']))
|
||||
else:
|
||||
self.test_case_names.add(tc['summary'] + stripped_config_name)
|
||||
|
||||
name_addr = table.get_unsigned_int(section, test_addr, 4)
|
||||
desc_addr = table.get_unsigned_int(section, test_addr + 4, 4)
|
||||
function_count = table.get_unsigned_int(section, test_addr + 20, 4)
|
||||
name = table.get_string('any', name_addr)
|
||||
desc = table.get_string('any', desc_addr)
|
||||
test_group_included = True
|
||||
if test_groups is not None and tc['group'] not in test_groups:
|
||||
test_group_included = False
|
||||
|
||||
tc = self.parse_one_test_case(name, desc, config_name, stripped_config_name, tags)
|
||||
|
||||
# check if duplicated case names
|
||||
# we need to use it to select case,
|
||||
# if duplicated IDs, Unity could select incorrect case to run
|
||||
# and we need to check all cases no matter if it's going te be executed by CI
|
||||
# also add app_name here, we allow same case for different apps
|
||||
if (tc['summary'] + stripped_config_name) in self.test_case_names:
|
||||
self.parsing_errors.append('{} ({}): duplicated test case ID: {}'.format(stripped_config_name, config_name, tc['summary']))
|
||||
if tc['CI ready'] == 'Yes' and test_group_included:
|
||||
# update test env list and the cases of same env list
|
||||
if tc['test environment'] in self.test_env_tags:
|
||||
self.test_env_tags[tc['test environment']].append(tc['ID'])
|
||||
else:
|
||||
self.test_case_names.add(tc['summary'] + stripped_config_name)
|
||||
self.test_env_tags.update({tc['test environment']: [tc['ID']]})
|
||||
|
||||
test_group_included = True
|
||||
if test_groups is not None and tc['group'] not in test_groups:
|
||||
test_group_included = False
|
||||
if bin_tc['function_count'] > 1:
|
||||
tc.update({'child case num': bin_tc['function_count']})
|
||||
|
||||
if tc['CI ready'] == 'Yes' and test_group_included:
|
||||
# update test env list and the cases of same env list
|
||||
if tc['test environment'] in self.test_env_tags:
|
||||
self.test_env_tags[tc['test environment']].append(tc['ID'])
|
||||
else:
|
||||
self.test_env_tags.update({tc['test environment']: [tc['ID']]})
|
||||
|
||||
if function_count > 1:
|
||||
tc.update({'child case num': function_count})
|
||||
|
||||
# only add cases need to be executed
|
||||
test_cases.append(tc)
|
||||
|
||||
os.remove('section_table.tmp')
|
||||
os.remove('case_address.tmp')
|
||||
# only add cases need to be executed
|
||||
test_cases.append(tc)
|
||||
|
||||
return test_cases
|
||||
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue