Merge branch 'bugfix/fix_multi_dut_testcases_report' into 'master'

ci(pytest): Add functionality to merge JUnit files and collect real failure cases...

Closes RDT-495

See merge request espressif/esp-idf!24632
pull/11945/head^2
Fu Hanxi 2023-07-26 14:41:21 +08:00
commit 28167ea5a3
1 zmienionych plików z 64 dodań i 18 usunięć

Wyświetl plik

@ -1,4 +1,4 @@
# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2021-2023 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
# pylint: disable=W0621 # redefined-outer-name
@ -22,7 +22,7 @@ import sys
import xml.etree.ElementTree as ET
from datetime import datetime
from fnmatch import fnmatch
from typing import Callable, List, Optional, Tuple
from typing import Callable, Dict, List, Optional, Tuple
import pytest
from _pytest.config import Config, ExitCode
@ -139,6 +139,8 @@ ENV_MARKERS = {
'sdio_master_slave': 'Test sdio multi board.',
}
SUB_JUNIT_FILENAME = 'dut.xml'
##################
# Help Functions #
@ -215,6 +217,49 @@ def get_target_marker_from_expr(markexpr: str) -> str:
raise ValueError('Please specify one target marker via "--target [TARGET]" or via "-m [TARGET]"')
def merge_junit_files(junit_files: List[str], target_path: str) -> Optional[ET.Element]:
merged_testsuite: ET.Element = ET.Element('testsuite')
testcases: Dict[str, ET.Element] = {}
if len(junit_files) == 0:
return None
if len(junit_files) == 1:
return ET.parse(junit_files[0]).getroot()
for junit in junit_files:
logging.info(f'Merging {junit} to {target_path}')
tree: ET.ElementTree = ET.parse(junit)
testsuite: ET.Element = tree.getroot()
for testcase in testsuite.findall('testcase'):
name: str = testcase.get('name') if testcase.get('name') else '' # type: ignore
if name not in testcases:
testcases[name] = testcase
merged_testsuite.append(testcase)
continue
existing_testcase = testcases[name]
for element_name in ['failure', 'error']:
for element in testcase.findall(element_name):
existing_element = existing_testcase.find(element_name)
if existing_element is None:
existing_testcase.append(element)
else:
existing_element.attrib.setdefault('message', '') # type: ignore
existing_element.attrib['message'] += '. ' + element.get('message', '') # type: ignore
os.remove(junit)
merged_testsuite.set('tests', str(len(merged_testsuite.findall('testcase'))))
merged_testsuite.set('failures', str(len(merged_testsuite.findall('.//testcase/failure'))))
merged_testsuite.set('errors', str(len(merged_testsuite.findall('.//testcase/error'))))
merged_testsuite.set('skipped', str(len(merged_testsuite.findall('.//testcase/skipped'))))
return merged_testsuite
############
# Fixtures #
############
@ -448,13 +493,13 @@ def pytest_addoption(parser: pytest.Parser) -> None:
'--app-info-basedir',
default=IDF_PATH,
help='app info base directory. specify this value when you\'re building under a '
'different IDF_PATH. (Default: $IDF_PATH)',
'different IDF_PATH. (Default: $IDF_PATH)',
)
idf_group.addoption(
'--app-info-filepattern',
help='glob pattern to specify the files that include built app info generated by '
'`idf-build-apps --collect-app-info ...`. will not raise ValueError when binary '
'paths not exist in local file system if not listed recorded in the app info.',
'`idf-build-apps --collect-app-info ...`. will not raise ValueError when binary '
'paths not exist in local file system if not listed recorded in the app info.',
)
@ -688,22 +733,23 @@ class IdfPytestEmbedded:
failed_sub_cases = []
target = item.funcargs['target']
config = item.funcargs['config']
for junit in junits:
xml = ET.parse(junit)
testcases = xml.findall('.//testcase')
for case in testcases:
# modify the junit files
new_case_name = format_case_id(target, config, case.attrib['name'])
case.attrib['name'] = new_case_name
if 'file' in case.attrib:
case.attrib['file'] = case.attrib['file'].replace('/IDF/', '') # our unity test framework
merged_dut_junit_filepath = os.path.join(tempdir, SUB_JUNIT_FILENAME)
merged_testsuite = merge_junit_files(junit_files=junits, target_path=merged_dut_junit_filepath)
# collect real failure cases
if case.find('failure') is not None:
failed_sub_cases.append(new_case_name)
if merged_testsuite is None:
return
xml.write(junit)
for testcase in merged_testsuite.findall('testcase'):
new_case_name: str = format_case_id(target, config, testcase.attrib['name'])
testcase.attrib['name'] = new_case_name
if 'file' in testcase.attrib:
testcase.attrib['file'] = testcase.attrib['file'].replace('/IDF/', '') # Our unity test framework
# Collect real failure cases
if testcase.find('failure') is not None:
failed_sub_cases.append(new_case_name)
merged_tree: ET.ElementTree = ET.ElementTree(merged_testsuite)
merged_tree.write(merged_dut_junit_filepath)
item.stash[_item_failed_cases_key] = failed_sub_cases
def pytest_sessionfinish(self, session: Session, exitstatus: int) -> None: