ci: use tags as markers

pull/9179/head
Fu Hanxi 2022-05-22 00:38:17 +08:00
rodzic 3e78898d2c
commit 7e0bb1dabd
4 zmienionych plików z 118 dodań i 128 usunięć

Wyświetl plik

@ -3,7 +3,6 @@
stage: target_test
timeout: 1 hour
extends: .before_script_pytest
tags: [$TARGET, $ENV_MARKER]
artifacts:
when: always
paths:
@ -14,7 +13,11 @@
expire_in: 1 week
script:
- retry_failed git clone $KNOWN_FAILURE_CASES_REPO known_failure_cases
- pytest $TEST_DIR --target $TARGET -m $ENV_MARKER --junitxml=XUNIT_RESULT.xml --known-failure-cases-file known_failure_cases/known_failure_cases.txt
# using runner tags as markers to filter the test cases
# Runner tags are comma separated, replace the comma with " and " for markers
- job_tags=$(python tools/ci/python_packages/gitlab_api.py get_job_tags $CI_PROJECT_ID --job_id $CI_JOB_ID)
- markers=$(echo $job_tags | sed -e "s/,/ and /g")
- run_cmd pytest $TEST_DIR -m \"${markers}\" --junitxml=XUNIT_RESULT.xml --known-failure-cases-file known_failure_cases/known_failure_cases.txt
.pytest_examples_dir_template:
extends: .pytest_template
@ -27,9 +30,7 @@ example_test_pytest_esp32_generic:
- .rules:test:example_test-esp32
needs:
- build_pytest_examples_esp32
variables:
TARGET: ESP32
ENV_MARKER: generic
tags: [ esp32, generic ]
example_test_pytest_esp32_ir_transceiver:
extends:
@ -37,9 +38,7 @@ example_test_pytest_esp32_ir_transceiver:
- .rules:test:example_test-esp32
needs:
- build_pytest_examples_esp32
variables:
TARGET: ESP32
ENV_MARKER: ir_transceiver
tags: [ esp32, ir_transceiver ]
example_test_pytest_esp32s2_generic:
extends:
@ -47,9 +46,7 @@ example_test_pytest_esp32s2_generic:
- .rules:test:example_test-esp32s2
needs:
- build_pytest_examples_esp32s2
variables:
TARGET: ESP32S2
ENV_MARKER: generic
tags: [ esp32s2, generic ]
example_test_pytest_esp32s3_generic:
extends:
@ -57,9 +54,7 @@ example_test_pytest_esp32s3_generic:
- .rules:test:example_test-esp32s3
needs:
- build_pytest_examples_esp32s3
variables:
TARGET: ESP32S3
ENV_MARKER: generic
tags: [ esp32s3, generic ]
example_test_pytest_esp32c2_generic:
extends:
@ -67,9 +62,7 @@ example_test_pytest_esp32c2_generic:
- .rules:test:example_test-esp32c2
needs:
- build_pytest_examples_esp32c2
variables:
TARGET: ESP32C2
ENV_MARKER: generic
tags: [ esp32c2, generic ]
example_test_pytest_esp32c3_generic:
extends:
@ -77,9 +70,7 @@ example_test_pytest_esp32c3_generic:
- .rules:test:example_test-esp32c3
needs:
- build_pytest_examples_esp32c3
variables:
TARGET: ESP32C3
ENV_MARKER: generic
tags: [ esp32c3, generic ]
example_test_pytest_esp32c3_flash_suspend:
extends:
@ -87,9 +78,7 @@ example_test_pytest_esp32c3_flash_suspend:
- .rules:test:example_test-esp32c3
needs:
- build_pytest_examples_esp32c3
variables:
TARGET: ESP32C3
ENV_MARKER: flash_suspend
tags: [ esp32c3, flash_suspend ]
example_test_pytest_esp32_ethernet_ota:
extends:
@ -97,9 +86,7 @@ example_test_pytest_esp32_ethernet_ota:
- .rules:test:example_test-esp32
needs:
- build_pytest_examples_esp32
variables:
TARGET: ESP32
ENV_MARKER: ethernet_ota
tags: [ esp32, ethernet_ota ]
example_test_pytest_esp32_wifi_ota:
extends:
@ -107,9 +94,7 @@ example_test_pytest_esp32_wifi_ota:
- .rules:test:example_test-esp32
needs:
- build_pytest_examples_esp32
variables:
TARGET: ESP32
ENV_MARKER: wifi_ota
tags: [ esp32, wifi_ota ]
example_test_pytest_esp32_flash_encryption_ota:
extends:
@ -117,9 +102,7 @@ example_test_pytest_esp32_flash_encryption_ota:
- .rules:test:example_test-esp32
needs:
- build_pytest_examples_esp32
variables:
TARGET: ESP32
ENV_MARKER: flash_encryption_ota
tags: [ esp32, flash_encryption_ota ]
example_test_pytest_esp32c3_flash_encryption_wifi_ota:
extends:
@ -127,9 +110,7 @@ example_test_pytest_esp32c3_flash_encryption_wifi_ota:
- .rules:test:example_test-esp32c3
needs:
- build_pytest_examples_esp32c3
variables:
TARGET: ESP32C3
ENV_MARKER: flash_encryption_wifi_ota
tags: [ esp32c3, flash_encryption_wifi_ota ]
example_test_pytest_esp32_ethernet:
extends:
@ -137,9 +118,7 @@ example_test_pytest_esp32_ethernet:
- .rules:test:example_test-esp32
needs:
- build_pytest_examples_esp32
variables:
TARGET: ESP32
ENV_MARKER: ethernet
tags: [ esp32, ethernet]
example_test_pytest_esp32_8mb_flash:
extends:
@ -147,9 +126,7 @@ example_test_pytest_esp32_8mb_flash:
- .rules:test:example_test-esp32
needs:
- build_pytest_examples_esp32
variables:
TARGET: ESP32
ENV_MARKER: ethernet_flash_8m
tags: [ esp32, ethernet_flash_8m ]
example_test_pytest_esp32_wifi:
extends:
@ -157,9 +134,7 @@ example_test_pytest_esp32_wifi:
- .rules:test:example_test-esp32
needs:
- build_pytest_examples_esp32
variables:
TARGET: ESP32
ENV_MARKER: wifi
tags: [ esp32, wifi ]
example_test_pytest_esp32_wifi_bt:
extends:
@ -167,9 +142,7 @@ example_test_pytest_esp32_wifi_bt:
- .rules:test:example_test-esp32
needs:
- build_pytest_examples_esp32
variables:
TARGET: ESP32
ENV_MARKER: wifi_bt
tags: [ esp32, wifi_bt ]
example_test_pytest_esp32_ethernet_ip101:
extends:
@ -177,9 +150,7 @@ example_test_pytest_esp32_ethernet_ip101:
- .rules:test:example_test-esp32
needs:
- build_pytest_examples_esp32
variables:
TARGET: ESP32
ENV_MARKER: ip101
tags: [ esp32, ip101 ]
example_test_pytest_esp32_flash_encryption:
extends:
@ -187,9 +158,7 @@ example_test_pytest_esp32_flash_encryption:
- .rules:test:example_test-esp32
needs:
- build_pytest_examples_esp32
variables:
TARGET: ESP32
ENV_MARKER: flash_encryption
tags: [ esp32, flash_encryption ]
example_test_pytest_esp32_multi_dut_generic:
extends:
@ -197,9 +166,7 @@ example_test_pytest_esp32_multi_dut_generic:
- .rules:test:example_test-esp32
needs:
- build_pytest_examples_esp32
variables:
TARGET: ESP32
ENV_MARKER: multi_dut_generic
tags: [ esp32, multi_dut_generic ]
example_test_pytest_esp32c3_flash_encryption:
extends:
@ -207,9 +174,7 @@ example_test_pytest_esp32c3_flash_encryption:
- .rules:test:example_test-esp32c3
needs:
- build_pytest_examples_esp32c3
variables:
TARGET: ESP32C3
ENV_MARKER: flash_encryption
tags: [ esp32c3, flash_encryption ]
example_test_pytest_esp32s2_deepsleep: # Temp tag, will be removed IDF-5213
extends:
@ -217,9 +182,7 @@ example_test_pytest_esp32s2_deepsleep: # Temp tag, will be removed IDF-5213
- .rules:test:example_test-esp32s2
needs:
- build_pytest_examples_esp32s2
variables:
TARGET: ESP32S2
ENV_MARKER: deepsleep_temp_tag
tags: [ esp32s2, deepsleep_temp_tag ]
example_test_pytest_esp32s3_deepsleep: # Temp tag, will be removed IDF-5213
extends:
@ -227,9 +190,7 @@ example_test_pytest_esp32s3_deepsleep: # Temp tag, will be removed IDF-5213
- .rules:test:example_test-esp32s3
needs:
- build_pytest_examples_esp32s3
variables:
TARGET: ESP32S3
ENV_MARKER: deepsleep_temp_tag
tags: [esp32s3, deepsleep_temp_tag ]
.pytest_components_dir_template:
extends: .pytest_template
@ -242,9 +203,7 @@ component_ut_pytest_esp32_generic:
- .rules:test:component_ut-esp32
needs:
- build_pytest_components_esp32
variables:
TARGET: ESP32
ENV_MARKER: generic
tags: [ esp32, generic ]
component_ut_pytest_esp32_ip101:
extends:
@ -252,9 +211,7 @@ component_ut_pytest_esp32_ip101:
- .rules:test:component_ut-esp32
needs:
- build_pytest_components_esp32
variables:
TARGET: ESP32
ENV_MARKER: ip101
tags: [ esp32, ip101 ]
component_ut_pytest_esp32_lan8720:
extends:
@ -262,9 +219,7 @@ component_ut_pytest_esp32_lan8720:
- .rules:labels-protected:lan8720 # FIXME: IDFCI-1176
needs:
- build_pytest_components_esp32
variables:
TARGET: ESP32
ENV_MARKER: lan8720
tags: [ esp32, lan8720 ]
component_ut_pytest_esp32s2_generic:
extends:
@ -272,9 +227,7 @@ component_ut_pytest_esp32s2_generic:
- .rules:test:component_ut-esp32s2
needs:
- build_pytest_components_esp32s2
variables:
TARGET: ESP32S2
ENV_MARKER: generic
tags: [ esp32s2, generic ]
component_ut_pytest_esp32s3_generic:
extends:
@ -282,9 +235,7 @@ component_ut_pytest_esp32s3_generic:
- .rules:test:component_ut-esp32s3
needs:
- build_pytest_components_esp32s3
variables:
TARGET: ESP32S3
ENV_MARKER: generic
tags: [ esp32s3, generic ]
component_ut_pytest_esp32s3_octal_psram:
extends:
@ -292,9 +243,7 @@ component_ut_pytest_esp32s3_octal_psram:
- .rules:test:component_ut-esp32s3
needs:
- build_pytest_components_esp32s3
variables:
TARGET: ESP32S3
ENV_MARKER: octal_psram
tags: [ esp32s3, octal_psram ]
component_ut_pytest_esp32c2_generic:
extends:
@ -302,9 +251,7 @@ component_ut_pytest_esp32c2_generic:
- .rules:test:component_ut-esp32c2
needs:
- build_pytest_components_esp32c2
variables:
TARGET: ESP32C2
ENV_MARKER: generic
tags: [ esp32c2, generic ]
component_ut_pytest_esp32c3_generic:
extends:
@ -312,9 +259,7 @@ component_ut_pytest_esp32c3_generic:
- .rules:test:component_ut-esp32c3
needs:
- build_pytest_components_esp32c3
variables:
TARGET: ESP32C3
ENV_MARKER: generic
tags: [ esp32c3, generic ]
.pytest_test_apps_dir_template:
extends: .pytest_template
@ -327,9 +272,8 @@ test_app_test_pytest_esp32_generic:
- .rules:test:custom_test-esp32
needs:
- build_pytest_test_apps_esp32
tags: [ esp32, generic ]
variables:
TARGET: ESP32
ENV_MARKER: generic
SETUP_TOOLS: "1" # need gdb
test_app_test_pytest_esp32s2_generic:
@ -338,9 +282,8 @@ test_app_test_pytest_esp32s2_generic:
- .rules:test:custom_test-esp32s2
needs:
- build_pytest_test_apps_esp32s2
tags: [ esp32s2, generic ]
variables:
TARGET: ESP32S2
ENV_MARKER: generic
SETUP_TOOLS: "1" # need gdb
test_app_test_pytest_esp32s3_generic:
@ -349,9 +292,7 @@ test_app_test_pytest_esp32s3_generic:
- .rules:test:custom_test-esp32s3
needs:
- build_pytest_test_apps_esp32s3
variables:
TARGET: ESP32S3
ENV_MARKER: generic
tags: [ esp32s3, generic ]
test_app_test_pytest_esp32c2_generic:
extends:
@ -359,9 +300,7 @@ test_app_test_pytest_esp32c2_generic:
- .rules:test:custom_test-esp32c2
needs:
- build_pytest_test_apps_esp32c2
variables:
TARGET: ESP32C2
ENV_MARKER: generic
tags: [ esp32c2, generic ]
test_app_test_pytest_esp32c3_generic:
extends:
@ -369,9 +308,7 @@ test_app_test_pytest_esp32c3_generic:
- .rules:test:custom_test-esp32c3
needs:
- build_pytest_test_apps_esp32c3
variables:
TARGET: ESP32C3
ENV_MARKER: generic
tags: [ esp32c3, generic ]
test_app_test_pytest_esp32s2_usb_host:
extends:
@ -379,9 +316,7 @@ test_app_test_pytest_esp32s2_usb_host:
- .rules:test:custom_test-esp32s2
needs:
- build_pytest_test_apps_esp32s2
variables:
TARGET: ESP32S2
ENV_MARKER: usb_host
tags: [ esp32s2, usb_host ]
# for parallel jobs, CI_JOB_NAME will be "job_name index/total" (for example, "IT_001 1/2")
# we need to convert to pattern "job_name_index.yml"

Wyświetl plik

@ -42,10 +42,7 @@ DEFAULT_SDKCONFIG = 'default'
# Help Functions #
##################
def is_target_marker(marker: str) -> bool:
if marker.startswith('esp32'):
return True
if marker.startswith('esp8'):
if marker.startswith('esp32') or marker.startswith('esp8') or marker == 'linux':
return True
return False
@ -59,6 +56,26 @@ def item_marker_names(item: Item) -> List[str]:
return [marker.name for marker in item.iter_markers()]
def get_target_marker(markexpr: str) -> str:
candidates = set()
# we use `-m "esp32 and generic"` in our CI to filter the test cases
for marker in markexpr.split('and'):
marker = marker.strip()
if is_target_marker(marker):
candidates.add(marker)
if len(candidates) > 1:
raise ValueError(
f'Specified more than one target markers: {candidates}. Please specify no more than one.'
)
elif len(candidates) == 1:
return candidates.pop()
else:
raise ValueError(
'Please specify one target marker via "--target [TARGET]" or via "-m [TARGET]"'
)
############
# Fixtures #
############
@ -79,15 +96,19 @@ def session_tempdir() -> str:
def log_minimum_free_heap_size(dut: IdfDut, config: str) -> Callable[..., None]:
def real_func() -> None:
res = dut.expect(r'Minimum free heap size: (\d+) bytes')
logging.info('\n------ heap size info ------\n'
'[app_name] {}\n'
'[config_name] {}\n'
'[target] {}\n'
'[minimum_free_heap_size] {} Bytes\n'
'------ heap size end ------'.format(os.path.basename(dut.app.app_path),
config,
dut.target,
res.group(1).decode('utf8')))
logging.info(
'\n------ heap size info ------\n'
'[app_name] {}\n'
'[config_name] {}\n'
'[target] {}\n'
'[minimum_free_heap_size] {} Bytes\n'
'------ heap size end ------'.format(
os.path.basename(dut.app.app_path),
config,
dut.target,
res.group(1).decode('utf8'),
)
)
return real_func
@ -183,8 +204,20 @@ _idf_pytest_embedded_key = pytest.StashKey['IdfPytestEmbedded']
def pytest_configure(config: Config) -> None:
# cli option "--target"
target = config.getoption('target') or ''
help_commands = ['--help', '--fixtures', '--markers', '--version']
for cmd in help_commands:
if cmd in config.invocation_params.args:
target = 'unneeded'
break
if not target: # also could specify through markexpr via "-m"
target = get_target_marker(config.getoption('markexpr') or '')
config.stash[_idf_pytest_embedded_key] = IdfPytestEmbedded(
target=config.getoption('target'),
target=target,
sdkconfig=config.getoption('sdkconfig'),
known_failure_cases_file=config.getoption('known_failure_cases_file'),
)
@ -218,7 +251,11 @@ class IdfPytestEmbedded:
@property
def failed_cases(self) -> List[str]:
return [case for case, is_known, is_xfail in self._failed_cases if not is_known and not is_xfail]
return [
case
for case, is_known, is_xfail in self._failed_cases
if not is_known and not is_xfail
]
@property
def known_failure_cases(self) -> List[str]:
@ -268,13 +305,13 @@ class IdfPytestEmbedded:
# add markers for special markers
for item in items:
if 'supported_targets' in item_marker_names(item):
if 'supported_targets' in item.keywords:
for _target in SUPPORTED_TARGETS:
item.add_marker(_target)
if 'preview_targets' in item_marker_names(item):
if 'preview_targets' in item.keywords:
for _target in PREVIEW_TARGETS:
item.add_marker(_target)
if 'all_targets' in item_marker_names(item):
if 'all_targets' in item.keywords:
for _target in [*SUPPORTED_TARGETS, *PREVIEW_TARGETS]:
item.add_marker(_target)

Wyświetl plik

@ -204,6 +204,16 @@ class Gitlab(object):
return os.path.join(os.path.realpath(destination), root_name)
def get_job_tags(self, job_id: int) -> str:
"""
Get tags of a job
:param job_id: job id
:return: comma-separated tags of the job
"""
job = self.project.jobs.get(job_id)
return ','.join(job.tag_list)
def main() -> None:
parser = argparse.ArgumentParser()
@ -231,6 +241,9 @@ def main() -> None:
elif args.action == 'get_project_id':
ret = gitlab_inst.get_project_id(args.project_name)
print('project id: {}'.format(ret))
elif args.action == 'get_job_tags':
ret = gitlab_inst.get_job_tags(args.job_id)
print(ret)
if __name__ == '__main__':

Wyświetl plik

@ -62,17 +62,22 @@ function warning() {
}
function run_cmd() {
local args=( "$@" )
local cmd="${args[@]}"
local start=$(date +%s)
eval "$@"
info "\$ ${cmd}"
eval "${cmd}"
local ret=$?
local end=$(date +%s)
local duration=$((end - start))
local runtime=$((end-start))
if [[ $ret -eq 0 ]]; then
info "(\$ $*) succeeded in ${duration} seconds."
info "==> '\$ ${cmd}' succeeded in ${runtime} seconds."
return 0
else
error "(\$ $*) failed in ${duration} seconds."
echo
error "==> '\$ ${cmd}' failed (${ret}) in ${runtime} seconds."
return $ret
fi
}