kopia lustrzana https://github.com/espressif/esp-idf
ci(pytest): differentiate `temp_skip` and `temp_skip_ci` markers
rodzic
52e6fefb3d
commit
08e68c192a
114
conftest.py
114
conftest.py
|
@ -35,9 +35,11 @@ from pytest_embedded.utils import find_by_suffix
|
||||||
from pytest_embedded_idf.dut import IdfDut
|
from pytest_embedded_idf.dut import IdfDut
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
from idf_ci_utils import to_list
|
||||||
from idf_unity_tester import CaseTester
|
from idf_unity_tester import CaseTester
|
||||||
except ImportError:
|
except ImportError:
|
||||||
sys.path.append(os.path.join(os.path.dirname(__file__), 'tools', 'ci'))
|
sys.path.append(os.path.join(os.path.dirname(__file__), 'tools', 'ci'))
|
||||||
|
from idf_ci_utils import to_list
|
||||||
from idf_unity_tester import CaseTester
|
from idf_unity_tester import CaseTester
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
@ -65,10 +67,11 @@ SPECIAL_MARKERS = {
|
||||||
'supported_targets': "support all officially announced supported targets ('esp32', 'esp32s2', 'esp32c3', 'esp32s3', 'esp32c2', 'esp32c6')",
|
'supported_targets': "support all officially announced supported targets ('esp32', 'esp32s2', 'esp32c3', 'esp32s3', 'esp32c2', 'esp32c6')",
|
||||||
'preview_targets': "support all preview targets ('esp32h4')",
|
'preview_targets': "support all preview targets ('esp32h4')",
|
||||||
'all_targets': 'support all targets, including supported ones and preview ones',
|
'all_targets': 'support all targets, including supported ones and preview ones',
|
||||||
'temp_skip_ci': 'temp skip ci tests for specified targets, can only work with `supported_targets`, `preview_targets`, `all_targets`',
|
'temp_skip_ci': 'temp skip tests for specified targets only in ci',
|
||||||
|
'temp_skip': 'temp skip tests for specified targets both in ci and locally',
|
||||||
'nightly_run': 'tests should be executed as part of the nightly trigger pipeline',
|
'nightly_run': 'tests should be executed as part of the nightly trigger pipeline',
|
||||||
'host_test': 'tests which should not be built at the build stage, and instead built in host_test stage.',
|
'host_test': 'tests which should not be built at the build stage, and instead built in host_test stage',
|
||||||
'qemu': 'build and test using qemu-system-xtensa, not real target.',
|
'qemu': 'build and test using qemu-system-xtensa, not real target',
|
||||||
}
|
}
|
||||||
|
|
||||||
ENV_MARKERS = {
|
ENV_MARKERS = {
|
||||||
|
@ -133,9 +136,57 @@ def item_marker_names(item: Item) -> List[str]:
|
||||||
return [marker.name for marker in item.iter_markers()]
|
return [marker.name for marker in item.iter_markers()]
|
||||||
|
|
||||||
|
|
||||||
def get_target_marker(markexpr: str) -> str:
|
def item_target_marker_names(item: Item) -> List[str]:
|
||||||
|
res = set()
|
||||||
|
for marker in item.iter_markers():
|
||||||
|
if marker.name in TARGET_MARKERS:
|
||||||
|
res.add(marker.name)
|
||||||
|
|
||||||
|
return sorted(res)
|
||||||
|
|
||||||
|
|
||||||
|
def item_env_marker_names(item: Item) -> List[str]:
|
||||||
|
res = set()
|
||||||
|
for marker in item.iter_markers():
|
||||||
|
if marker.name in ENV_MARKERS:
|
||||||
|
res.add(marker.name)
|
||||||
|
|
||||||
|
return sorted(res)
|
||||||
|
|
||||||
|
|
||||||
|
def item_skip_targets(item: Item) -> List[str]:
|
||||||
|
def _get_temp_markers_disabled_targets(marker_name: str) -> List[str]:
|
||||||
|
temp_marker = item.get_closest_marker(marker_name)
|
||||||
|
|
||||||
|
if not temp_marker:
|
||||||
|
return []
|
||||||
|
|
||||||
|
# temp markers should always use keyword arguments `targets` and `reason`
|
||||||
|
if not temp_marker.kwargs.get('targets') or not temp_marker.kwargs.get('reason'):
|
||||||
|
raise ValueError(
|
||||||
|
f'`{marker_name}` should always use keyword arguments `targets` and `reason`. '
|
||||||
|
f'For example: '
|
||||||
|
f'`@pytest.mark.{marker_name}(targets=["esp32"], reason="IDF-xxxx, will fix it ASAP")`'
|
||||||
|
)
|
||||||
|
|
||||||
|
return to_list(temp_marker.kwargs['targets']) # type: ignore
|
||||||
|
|
||||||
|
temp_skip_ci_targets = _get_temp_markers_disabled_targets('temp_skip_ci')
|
||||||
|
temp_skip_targets = _get_temp_markers_disabled_targets('temp_skip')
|
||||||
|
|
||||||
|
# in CI we skip the union of `temp_skip` and `temp_skip_ci`
|
||||||
|
if os.getenv('CI_JOB_ID'):
|
||||||
|
skip_targets = list(set(temp_skip_ci_targets).union(set(temp_skip_targets)))
|
||||||
|
else: # we use `temp_skip` locally
|
||||||
|
skip_targets = temp_skip_targets
|
||||||
|
|
||||||
|
return skip_targets
|
||||||
|
|
||||||
|
|
||||||
|
def get_target_marker_from_expr(markexpr: str) -> str:
|
||||||
candidates = set()
|
candidates = set()
|
||||||
# we use `-m "esp32 and generic"` in our CI to filter the test cases
|
# we use `-m "esp32 and generic"` in our CI to filter the test cases
|
||||||
|
# this doesn't cover all use cases, but fit what we do in CI.
|
||||||
for marker in markexpr.split('and'):
|
for marker in markexpr.split('and'):
|
||||||
marker = marker.strip()
|
marker = marker.strip()
|
||||||
if marker in TARGET_MARKERS:
|
if marker in TARGET_MARKERS:
|
||||||
|
@ -283,7 +334,7 @@ def pytest_configure(config: Config) -> None:
|
||||||
break
|
break
|
||||||
|
|
||||||
if not target: # also could specify through markexpr via "-m"
|
if not target: # also could specify through markexpr via "-m"
|
||||||
target = get_target_marker(config.getoption('markexpr') or '')
|
target = get_target_marker_from_expr(config.getoption('markexpr') or '')
|
||||||
|
|
||||||
config.stash[_idf_pytest_embedded_key] = IdfPytestEmbedded(
|
config.stash[_idf_pytest_embedded_key] = IdfPytestEmbedded(
|
||||||
target=target,
|
target=target,
|
||||||
|
@ -292,7 +343,7 @@ def pytest_configure(config: Config) -> None:
|
||||||
)
|
)
|
||||||
config.pluginmanager.register(config.stash[_idf_pytest_embedded_key])
|
config.pluginmanager.register(config.stash[_idf_pytest_embedded_key])
|
||||||
|
|
||||||
for name, description in (TARGET_MARKERS | ENV_MARKERS | SPECIAL_MARKERS).items():
|
for name, description in {**TARGET_MARKERS, **ENV_MARKERS, **SPECIAL_MARKERS}.items():
|
||||||
config.addinivalue_line('markers', f'{name}: {description}')
|
config.addinivalue_line('markers', f'{name}: {description}')
|
||||||
|
|
||||||
|
|
||||||
|
@ -306,12 +357,12 @@ def pytest_unconfigure(config: Config) -> None:
|
||||||
class IdfPytestEmbedded:
|
class IdfPytestEmbedded:
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
target: Optional[str] = None,
|
target: str,
|
||||||
sdkconfig: Optional[str] = None,
|
sdkconfig: Optional[str] = None,
|
||||||
known_failure_cases_file: Optional[str] = None,
|
known_failure_cases_file: Optional[str] = None,
|
||||||
):
|
):
|
||||||
# CLI options to filter the test cases
|
# CLI options to filter the test cases
|
||||||
self.target = target
|
self.target = target.lower()
|
||||||
self.sdkconfig = sdkconfig
|
self.sdkconfig = sdkconfig
|
||||||
self.known_failure_patterns = self._parse_known_failure_cases_file(known_failure_cases_file)
|
self.known_failure_patterns = self._parse_known_failure_cases_file(known_failure_cases_file)
|
||||||
|
|
||||||
|
@ -351,9 +402,8 @@ class IdfPytestEmbedded:
|
||||||
|
|
||||||
@pytest.hookimpl(tryfirst=True)
|
@pytest.hookimpl(tryfirst=True)
|
||||||
def pytest_sessionstart(self, session: Session) -> None:
|
def pytest_sessionstart(self, session: Session) -> None:
|
||||||
if self.target:
|
# same behavior for vanilla pytest-embedded '--target'
|
||||||
self.target = self.target.lower()
|
session.config.option.target = self.target
|
||||||
session.config.option.target = self.target
|
|
||||||
|
|
||||||
@pytest.hookimpl(tryfirst=True)
|
@pytest.hookimpl(tryfirst=True)
|
||||||
def pytest_collection_modifyitems(self, items: List[Function]) -> None:
|
def pytest_collection_modifyitems(self, items: List[Function]) -> None:
|
||||||
|
@ -375,42 +425,17 @@ class IdfPytestEmbedded:
|
||||||
|
|
||||||
# add markers for special markers
|
# add markers for special markers
|
||||||
for item in items:
|
for item in items:
|
||||||
skip_ci_marker = item.get_closest_marker('temp_skip_ci')
|
|
||||||
skip_ci_targets: List[str] = []
|
|
||||||
if skip_ci_marker:
|
|
||||||
# `temp_skip_ci` should always use keyword arguments `targets` and `reason`
|
|
||||||
if not skip_ci_marker.kwargs.get('targets') or not skip_ci_marker.kwargs.get('reason'):
|
|
||||||
raise ValueError(
|
|
||||||
f'`temp_skip_ci` should always use keyword arguments `targets` and `reason`. '
|
|
||||||
f'For example: '
|
|
||||||
f'`@pytest.mark.temp_skip_ci(targets=["esp32"], reason="IDF-xxxx, will fix it ASAP")`'
|
|
||||||
)
|
|
||||||
|
|
||||||
skip_ci_targets = skip_ci_marker.kwargs['targets']
|
|
||||||
|
|
||||||
if 'supported_targets' in item.keywords:
|
if 'supported_targets' in item.keywords:
|
||||||
for _target in SUPPORTED_TARGETS:
|
for _target in SUPPORTED_TARGETS:
|
||||||
if _target not in skip_ci_targets:
|
item.add_marker(_target)
|
||||||
item.add_marker(_target)
|
|
||||||
if 'preview_targets' in item.keywords:
|
if 'preview_targets' in item.keywords:
|
||||||
for _target in PREVIEW_TARGETS:
|
for _target in PREVIEW_TARGETS:
|
||||||
if _target not in skip_ci_targets:
|
item.add_marker(_target)
|
||||||
item.add_marker(_target)
|
|
||||||
if 'all_targets' in item.keywords:
|
if 'all_targets' in item.keywords:
|
||||||
for _target in [*SUPPORTED_TARGETS, *PREVIEW_TARGETS]:
|
for _target in [*SUPPORTED_TARGETS, *PREVIEW_TARGETS]:
|
||||||
if _target not in skip_ci_targets:
|
item.add_marker(_target)
|
||||||
item.add_marker(_target)
|
|
||||||
|
|
||||||
# `temp_skip_ci(targets=...)` can't work with specified single target
|
# add 'xtal_40mhz' tag as a default tag for esp32c2 target
|
||||||
for skip_ci_target in skip_ci_targets:
|
|
||||||
if skip_ci_target in item.keywords:
|
|
||||||
raise ValueError(
|
|
||||||
'`skip_ci_targets` can only work with '
|
|
||||||
'`supported_targets`, `preview_targets`, `all_targets` markers'
|
|
||||||
)
|
|
||||||
|
|
||||||
# add 'xtal_40mhz' tag as a default tag for esp32c2 target
|
|
||||||
for item in items:
|
|
||||||
# only add this marker for esp32c2 cases
|
# only add this marker for esp32c2 cases
|
||||||
if (
|
if (
|
||||||
self.target == 'esp32c2'
|
self.target == 'esp32c2'
|
||||||
|
@ -428,9 +453,12 @@ class IdfPytestEmbedded:
|
||||||
else:
|
else:
|
||||||
items[:] = [item for item in items if 'nightly_run' not in item_marker_names(item)]
|
items[:] = [item for item in items if 'nightly_run' not in item_marker_names(item)]
|
||||||
|
|
||||||
# filter all the test cases with "--target"
|
# filter all the test cases with target and skip_targets
|
||||||
if self.target:
|
items[:] = [
|
||||||
items[:] = [item for item in items if self.target in item_marker_names(item)]
|
item
|
||||||
|
for item in items
|
||||||
|
if self.target in item_marker_names(item) and self.target not in item_skip_targets(item)
|
||||||
|
]
|
||||||
|
|
||||||
# filter all the test cases with cli option "config"
|
# filter all the test cases with cli option "config"
|
||||||
if self.sdkconfig:
|
if self.sdkconfig:
|
||||||
|
|
|
@ -412,6 +412,9 @@ def sort_yaml(files: List[str]) -> None:
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
if 'CI_JOB_ID' not in os.environ:
|
||||||
|
os.environ['CI_JOB_ID'] = 'fake' # this is a CI script
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(description='ESP-IDF apps build/test checker')
|
parser = argparse.ArgumentParser(description='ESP-IDF apps build/test checker')
|
||||||
action = parser.add_subparsers(dest='action')
|
action = parser.add_subparsers(dest='action')
|
||||||
|
|
||||||
|
|
|
@ -251,14 +251,12 @@ def get_pytest_cases(
|
||||||
cases = []
|
cases = []
|
||||||
for target in targets:
|
for target in targets:
|
||||||
collector = PytestCollectPlugin(target)
|
collector = PytestCollectPlugin(target)
|
||||||
if marker_expr:
|
|
||||||
_marker_expr = f'{target} and ({marker_expr})'
|
|
||||||
else:
|
|
||||||
_marker_expr = target # target is also a marker
|
|
||||||
|
|
||||||
with io.StringIO() as buf:
|
with io.StringIO() as buf:
|
||||||
with redirect_stdout(buf):
|
with redirect_stdout(buf):
|
||||||
cmd = ['--collect-only', *get_pytest_files(paths), '-q', '-m', _marker_expr]
|
cmd = ['--collect-only', *get_pytest_files(paths), '--target', target, '-q']
|
||||||
|
if marker_expr:
|
||||||
|
cmd.extend(['-m', marker_expr])
|
||||||
if filter_expr:
|
if filter_expr:
|
||||||
cmd.extend(['-k', filter_expr])
|
cmd.extend(['-k', filter_expr])
|
||||||
res = pytest.main(cmd, plugins=[collector])
|
res = pytest.main(cmd, plugins=[collector])
|
||||||
|
|
Ładowanie…
Reference in New Issue