From c5fd72ad139ee3f763537e3cd26cd6a00449a8d3 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Tue, 8 Mar 2022 11:15:26 +0800 Subject: [PATCH 1/4] ci: install pytest-embedded related packages from pypi --- .gitlab-ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4e625eaa0c..d88d48394c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -73,7 +73,7 @@ variables: TEST_ENV_CONFIG_REPO: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/ci-test-runner-configs.git" CI_AUTO_TEST_SCRIPT_REPO_URL: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/auto_test_script.git" CI_AUTO_TEST_SCRIPT_REPO_BRANCH: "ci/v4.1" - PYTEST_EMBEDDED_TAG: "v0.6.0rc0" + PYTEST_EMBEDDED_VERSION: "0.6.0" # cache python dependencies PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" @@ -169,11 +169,11 @@ before_script: - fetch_submodules - *download_test_python_contraint_file - $IDF_PATH/tools/idf_tools.py install-python-env - - cd /tmp - - retry_failed git clone --depth 1 --branch $PYTEST_EMBEDDED_TAG https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/idf/pytest-embedded.git - - cd pytest-embedded && bash foreach.sh install - - pip install pytest-rerunfailures - - pip install scapy + - pip install + "pytest-embedded-serial-esp~=$PYTEST_EMBEDDED_VERSION" + "pytest-embedded-idf~=$PYTEST_EMBEDDED_VERSION" + pytest-rerunfailures + scapy - cd $IDF_PATH default: From 1c3628d631e31441cff793275360efbe38289279 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Thu, 3 Mar 2022 16:05:28 +0800 Subject: [PATCH 2/4] ci: refactor a few test scripts to pytest --- .../peripherals/i2c_wifi/app_test.py | 18 ---------- .../peripherals/i2c_wifi/pytest_i2c_wifi.py | 11 +++++++ .../phy/phy_multi_init_data_test/app_test.py | 33 ------------------- .../pytest_phy_multi_init_data.py | 19 +++++++++++ 4 files changed, 30 insertions(+), 51 deletions(-) delete mode 100644 tools/test_apps/peripherals/i2c_wifi/app_test.py create mode 100644 tools/test_apps/peripherals/i2c_wifi/pytest_i2c_wifi.py delete mode 100644 tools/test_apps/phy/phy_multi_init_data_test/app_test.py create mode 100644 tools/test_apps/phy/phy_multi_init_data_test/pytest_phy_multi_init_data.py diff --git a/tools/test_apps/peripherals/i2c_wifi/app_test.py b/tools/test_apps/peripherals/i2c_wifi/app_test.py deleted file mode 100644 index 395613a53e..0000000000 --- a/tools/test_apps/peripherals/i2c_wifi/app_test.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python -import tiny_test_fw -import ttfw_idf - - -@ttfw_idf.idf_custom_test(env_tag='Example_GENERIC') -def test_startup(env, _): # type: (tiny_test_fw.Env.Env, None) -> None - # Only test for master usage. - dut = env.get_dut('i2c_wifi', 'tools/test_apps/peripherals/i2c_wifi') - dut.start_app() - - dut.expect('I2C-WIFI test success') - - env.close_dut(dut.name) - - -if __name__ == '__main__': - test_startup() diff --git a/tools/test_apps/peripherals/i2c_wifi/pytest_i2c_wifi.py b/tools/test_apps/peripherals/i2c_wifi/pytest_i2c_wifi.py new file mode 100644 index 0000000000..a678c703fd --- /dev/null +++ b/tools/test_apps/peripherals/i2c_wifi/pytest_i2c_wifi.py @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: CC0-1.0 + +import pytest +from pytest_embedded_idf.dut import IdfDut + + +@pytest.mark.supported_targets +@pytest.mark.generic +def test_startup(dut: IdfDut) -> None: + dut.expect_exact('I2C-WIFI test success') diff --git a/tools/test_apps/phy/phy_multi_init_data_test/app_test.py b/tools/test_apps/phy/phy_multi_init_data_test/app_test.py deleted file mode 100644 index dbe0d75346..0000000000 --- a/tools/test_apps/phy/phy_multi_init_data_test/app_test.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python -# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import glob -import os - -import tiny_test_fw -import ttfw_idf -from tiny_test_fw import Utility - - -@ttfw_idf.idf_custom_test(env_tag='Example_GENERIC', group='test-apps') -def test_phy_multi_init_data_bin(env, _): - # type: (tiny_test_fw.Env.Env, None) -> None - config_files = glob.glob(os.path.join(os.path.dirname(__file__), 'sdkconfig.ci.*')) - config_names = [os.path.basename(s).replace('sdkconfig.ci.', '') for s in config_files] - for name in config_names: - dut = env.get_dut('phy_multi_init_data_test', 'tools/test_apps/phy/phy_multi_init_data_test',app_config_name=name) - dut.start_app() - - if 'CONFIG_ESP_PHY_MULTIPLE_INIT_DATA_BIN_EMBED' in dut.app.get_sdkconfig().keys(): - Utility.console_log('multi init data bin embed test') - dut.expect('load embedded multi phy init data') - else: - Utility.console_log('multi init data bin test') - dut.expect('Support multiple PHY init data bins') - - dut.expect('wifi_init finished') - Utility.console_log('Test Success') - - -if __name__ == '__main__': - test_phy_multi_init_data_bin() diff --git a/tools/test_apps/phy/phy_multi_init_data_test/pytest_phy_multi_init_data.py b/tools/test_apps/phy/phy_multi_init_data_test/pytest_phy_multi_init_data.py new file mode 100644 index 0000000000..9f9c914542 --- /dev/null +++ b/tools/test_apps/phy/phy_multi_init_data_test/pytest_phy_multi_init_data.py @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: CC0-1.0 + +import pytest +from pytest_embedded_idf.dut import IdfDut + + +@pytest.mark.supported_targets +@pytest.mark.generic +@pytest.mark.parametrize('config', [ + 'phy_multiple_init_data', + 'phy_multiple_init_data_embed', +], indirect=True) +def test_phy_multi_init_data_bin(dut: IdfDut, config: str) -> None: + if config == 'phy_multiple_init_data': + dut.expect_exact('Support multiple PHY init data bins') + else: + dut.expect_exact('load embedded multi phy init data') + dut.expect_exact('wifi_init finished') From d261840dbdfe1e5bf5534baf56a507385ff6b0c3 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Tue, 8 Mar 2022 11:37:42 +0800 Subject: [PATCH 3/4] ci: pytest idf ci could accept "--target" as uppercase --- conftest.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/conftest.py b/conftest.py index 1bf5b55643..effd5f779d 100644 --- a/conftest.py +++ b/conftest.py @@ -22,6 +22,7 @@ from typing import Callable, List, Optional import pytest from _pytest.config import Config from _pytest.fixtures import FixtureRequest +from _pytest.main import Session from _pytest.nodes import Item from _pytest.python import Function from pytest_embedded.plugin import parse_configuration @@ -95,7 +96,7 @@ def build_dir( """ param_or_cli: str = getattr( request, 'param', None - ) or request.config.option.__dict__.get('build_dir') + ) or request.config.getoption('build_dir') if param_or_cli is not None: # respect the param and the cli return param_or_cli @@ -146,6 +147,12 @@ def pytest_addoption(parser: pytest.Parser) -> None: ) +@pytest.hookimpl(tryfirst=True) +def pytest_sessionstart(session: Session) -> None: + if session.config.option.target: + session.config.option.target = session.config.getoption('target').lower() + + @pytest.hookimpl(tryfirst=True) def pytest_collection_modifyitems(config: Config, items: List[Function]) -> None: target = config.getoption('target', None) # use the `build` dir From 851d832eb63d47e7f615b09c6feedf854f8e448f Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Tue, 8 Mar 2022 11:36:57 +0800 Subject: [PATCH 4/4] ci: enable test_apps s3 c3 tests Refactor the tags a bit to make the tags could reuse the env markers. They suppose to be the same thing. --- .gitlab/ci/build.yml | 30 ++++-- .gitlab/ci/target-test.yml | 91 +++++++------------ tools/ci/check_copyright_ignore.txt | 1 - .../tiny_test_fw/Utility/CIAssignTest.py | 18 +--- 4 files changed, 62 insertions(+), 78 deletions(-) diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index 952ad484d1..9006db3b41 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -93,6 +93,16 @@ build_pytest_components_esp32c3: script: - run_cmd python tools/ci/build_pytest_apps.py components --target esp32c3 --size-info $SIZE_INFO_LOCATION -vv +build_non_test_components_apps: + extends: + - .build_template + - .build_test_apps_template + - .rules:build:component_ut + variables: + IDF_TARGET: all + TEST_PREFIX: component_ut + TEST_RELATIVE_DIR: component_ut + build_pytest_test_apps_esp32: extends: - .build_pytest_template @@ -107,15 +117,19 @@ build_pytest_test_apps_esp32s2: script: - run_cmd python tools/ci/build_pytest_apps.py tools/test_apps --target esp32s2 --size-info $SIZE_INFO_LOCATION -vv -build_non_test_components_apps: +build_pytest_test_apps_esp32s3: extends: - - .build_template - - .build_test_apps_template - - .rules:build:component_ut - variables: - IDF_TARGET: all - TEST_PREFIX: component_ut - TEST_RELATIVE_DIR: component_ut + - .build_pytest_template + - .rules:build:custom_test-esp32s3 + script: + - run_cmd python tools/ci/build_pytest_apps.py tools/test_apps --target esp32s3 --size-info $SIZE_INFO_LOCATION -vv + +build_pytest_test_apps_esp32c3: + extends: + - .build_pytest_template + - .rules:build:custom_test-esp32c3 + script: + - run_cmd python tools/ci/build_pytest_apps.py tools/test_apps --target esp32c3 --size-info $SIZE_INFO_LOCATION -vv .build_template_app_template: extends: .build_template diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml index 216be7f9ee..c8bc9a7460 100644 --- a/.gitlab/ci/target-test.yml +++ b/.gitlab/ci/target-test.yml @@ -3,6 +3,7 @@ stage: target_test timeout: 1 hour extends: .before_script_pytest + tags: [$TARGET, $ENV_MARKER] artifacts: when: always paths: @@ -26,11 +27,8 @@ example_test_pytest_esp32_generic: needs: - build_pytest_examples_esp32 variables: - TARGET: esp32 + TARGET: ESP32 ENV_MARKER: generic - tags: # in gitlab 14.1 or later, we can use `parallel: matrix` with the `tags` keyword. https://docs.gitlab.com/ee/ci/jobs/job_control.html#run-a-matrix-of-parallel-trigger-jobs - - ESP32 - - Example_GENERIC example_test_pytest_esp32s2_generic: extends: @@ -39,11 +37,8 @@ example_test_pytest_esp32s2_generic: needs: - build_pytest_examples_esp32s2 variables: - TARGET: esp32s2 + TARGET: ESP32S2 ENV_MARKER: generic - tags: - - ESP32S2 - - Example_GENERIC example_test_pytest_esp32s3_generic: extends: @@ -52,11 +47,8 @@ example_test_pytest_esp32s3_generic: needs: - build_pytest_examples_esp32s3 variables: - TARGET: esp32s3 + TARGET: ESP32S3 ENV_MARKER: generic - tags: - - ESP32S3 - - Example_GENERIC example_test_pytest_esp32c3_generic: extends: @@ -65,11 +57,8 @@ example_test_pytest_esp32c3_generic: needs: - build_pytest_examples_esp32c3 variables: - TARGET: esp32c3 + TARGET: ESP32C3 ENV_MARKER: generic - tags: - - ESP32C3 - - Example_GENERIC example_test_pytest_esp32c3_flash_suspend: extends: @@ -78,11 +67,8 @@ example_test_pytest_esp32c3_flash_suspend: needs: - build_pytest_examples_esp32c3 variables: - TARGET: esp32c3 + TARGET: ESP32C3 ENV_MARKER: flash_suspend - tags: - - ESP32C3_IDF - - UT_T1_Flash_Suspend .pytest_components_dir_template: extends: .pytest_template @@ -96,11 +82,8 @@ component_ut_pytest_esp32_generic: needs: - build_pytest_components_esp32 variables: - TARGET: esp32 + TARGET: ESP32 ENV_MARKER: generic - tags: - - ESP32 - - COMPONENT_UT_GENERIC component_ut_pytest_esp32_ip101: extends: @@ -109,11 +92,8 @@ component_ut_pytest_esp32_ip101: needs: - build_pytest_components_esp32 variables: - TARGET: esp32 + TARGET: ESP32 ENV_MARKER: ip101 - tags: - - ESP32 - - COMPONENT_UT_IP101 component_ut_pytest_esp32_lan8720: extends: @@ -122,11 +102,8 @@ component_ut_pytest_esp32_lan8720: needs: - build_pytest_components_esp32 variables: - TARGET: esp32 + TARGET: ESP32 ENV_MARKER: lan8720 - tags: - - ESP32 - - COMPONENT_UT_LAN8720 component_ut_pytest_esp32s2_generic: extends: @@ -135,11 +112,8 @@ component_ut_pytest_esp32s2_generic: needs: - build_pytest_components_esp32s2 variables: - TARGET: esp32s2 + TARGET: ESP32S2 ENV_MARKER: generic - tags: - - ESP32S2 - - COMPONENT_UT_GENERIC component_ut_pytest_esp32s3_generic: extends: @@ -148,11 +122,8 @@ component_ut_pytest_esp32s3_generic: needs: - build_pytest_components_esp32s3 variables: - TARGET: esp32s3 + TARGET: ESP32S3 ENV_MARKER: generic - tags: - - ESP32S3 - - COMPONENT_UT_GENERIC component_ut_pytest_esp32s3_octal_psram: extends: @@ -161,11 +132,8 @@ component_ut_pytest_esp32s3_octal_psram: needs: - build_pytest_components_esp32s3 variables: - TARGET: esp32s3 + TARGET: ESP32S3 ENV_MARKER: octal_psram - tags: - - ESP32S3 - - MSPI_F8R8 component_ut_pytest_esp32c3_generic: extends: @@ -174,11 +142,8 @@ component_ut_pytest_esp32c3_generic: needs: - build_pytest_components_esp32c3 variables: - TARGET: esp32c3 + TARGET: ESP32C3 ENV_MARKER: generic - tags: - - ESP32C3 - - COMPONENT_UT_GENERIC .pytest_test_apps_dir_template: extends: .pytest_template @@ -192,12 +157,9 @@ test_app_test_pytest_esp32_generic: needs: - build_pytest_test_apps_esp32 variables: - TARGET: esp32 + TARGET: ESP32 ENV_MARKER: generic SETUP_TOOLS: "1" # need gdb - tags: - - ESP32 - - Example_GENERIC test_app_test_pytest_esp32s2_generic: extends: @@ -206,12 +168,29 @@ test_app_test_pytest_esp32s2_generic: needs: - build_pytest_test_apps_esp32s2 variables: - TARGET: esp32s2 + TARGET: ESP32S2 ENV_MARKER: generic SETUP_TOOLS: "1" # need gdb - tags: - - ESP32S2 - - Example_GENERIC + +test_app_test_pytest_esp32s3_generic: + extends: + - .pytest_test_apps_dir_template + - .rules:test:custom_test-esp32s3 + needs: + - build_pytest_test_apps_esp32s3 + variables: + TARGET: ESP32S3 + ENV_MARKER: generic + +test_app_test_pytest_esp32c3_generic: + extends: + - .pytest_test_apps_dir_template + - .rules:test:custom_test-esp32c3 + needs: + - build_pytest_test_apps_esp32c3 + variables: + TARGET: ESP32C3 + ENV_MARKER: generic # for parallel jobs, CI_JOB_NAME will be "job_name index/total" (for example, "IT_001 1/2") # we need to convert to pattern "job_name_index.yml" diff --git a/tools/ci/check_copyright_ignore.txt b/tools/ci/check_copyright_ignore.txt index 7afc65abf3..85bb6dc1ee 100644 --- a/tools/ci/check_copyright_ignore.txt +++ b/tools/ci/check_copyright_ignore.txt @@ -2503,7 +2503,6 @@ tools/ci/python_packages/tiny_test_fw/DUT.py tools/ci/python_packages/tiny_test_fw/Env.py tools/ci/python_packages/tiny_test_fw/EnvConfig.py tools/ci/python_packages/tiny_test_fw/TinyFW.py -tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py tools/ci/python_packages/tiny_test_fw/Utility/CaseConfig.py tools/ci/python_packages/tiny_test_fw/Utility/GitlabCIJob.py tools/ci/python_packages/tiny_test_fw/Utility/TestCase.py diff --git a/tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py b/tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py index 491f5c0475..0cad445f50 100644 --- a/tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py +++ b/tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py @@ -1,16 +1,5 @@ -# Copyright 2015-2017 Espressif Systems (Shanghai) PTE LTD -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http:#www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 """ Common logic to assign test cases to CI jobs. @@ -187,8 +176,11 @@ class AssignTest(object): job_list = list() for job_name in ci_config: + if 'pytest' in job_name: + continue if self.CI_TEST_JOB_PATTERN.search(job_name) is not None: job_list.extend(self._handle_parallel_attribute(job_name, ci_config[job_name])) + job_list.sort(key=lambda x: x['name']) return job_list