Merge branch 'ci/refactor_test_apps_fuhanxi' into 'master'

ci: enable test_apps s3 c3 tests Closes IDFCI-1124 See merge request espressif/esp-idf!17397
2022-03-10 17:40:32 +08:00 · 2022-03-10 17:40:32 +08:00 · 97846e883b
commit 97846e883b
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -73,7 +73,7 @@ variables:
  TEST_ENV_CONFIG_REPO: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/ci-test-runner-configs.git"
  CI_AUTO_TEST_SCRIPT_REPO_URL: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/auto_test_script.git"
  CI_AUTO_TEST_SCRIPT_REPO_BRANCH: "ci/v4.1"
-  PYTEST_EMBEDDED_TAG: "v0.6.0rc0"
+  PYTEST_EMBEDDED_VERSION: "0.6.0"

  # cache python dependencies
  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
@ -169,11 +169,11 @@ before_script:
    - fetch_submodules
    - *download_test_python_contraint_file
    - $IDF_PATH/tools/idf_tools.py install-python-env
-    - cd /tmp
-    - retry_failed git clone --depth 1 --branch $PYTEST_EMBEDDED_TAG https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/idf/pytest-embedded.git
-    - cd pytest-embedded && bash foreach.sh install
-    - pip install pytest-rerunfailures
-    - pip install scapy
+    - pip install
+      "pytest-embedded-serial-esp~=$PYTEST_EMBEDDED_VERSION"
+      "pytest-embedded-idf~=$PYTEST_EMBEDDED_VERSION"
+      pytest-rerunfailures
+      scapy
    - cd $IDF_PATH

 default:
--- a/.gitlab/ci/build.yml
+++ b/.gitlab/ci/build.yml
@ -93,6 +93,16 @@ build_pytest_components_esp32c3:
  script:
    - run_cmd python tools/ci/build_pytest_apps.py components --target esp32c3 --size-info $SIZE_INFO_LOCATION -vv

+build_non_test_components_apps:
+  extends:
+    - .build_template
+    - .build_test_apps_template
+    - .rules:build:component_ut
+  variables:
+    IDF_TARGET: all
+    TEST_PREFIX: component_ut
+    TEST_RELATIVE_DIR: component_ut
+
 build_pytest_test_apps_esp32:
  extends:
    - .build_pytest_template
@ -107,15 +117,19 @@ build_pytest_test_apps_esp32s2:
  script:
    - run_cmd python tools/ci/build_pytest_apps.py tools/test_apps --target esp32s2 --size-info $SIZE_INFO_LOCATION -vv

-build_non_test_components_apps:
+build_pytest_test_apps_esp32s3:
  extends:
-    - .build_template
-    - .build_test_apps_template
-    - .rules:build:component_ut
-  variables:
-    IDF_TARGET: all
-    TEST_PREFIX: component_ut
-    TEST_RELATIVE_DIR: component_ut
+    - .build_pytest_template
+    - .rules:build:custom_test-esp32s3
+  script:
+    - run_cmd python tools/ci/build_pytest_apps.py tools/test_apps --target esp32s3 --size-info $SIZE_INFO_LOCATION -vv
+
+build_pytest_test_apps_esp32c3:
+  extends:
+    - .build_pytest_template
+    - .rules:build:custom_test-esp32c3
+  script:
+    - run_cmd python tools/ci/build_pytest_apps.py tools/test_apps --target esp32c3 --size-info $SIZE_INFO_LOCATION -vv

 .build_template_app_template:
  extends: .build_template
--- a/.gitlab/ci/target-test.yml
+++ b/.gitlab/ci/target-test.yml
@ -3,6 +3,7 @@
  stage: target_test
  timeout: 1 hour
  extends: .before_script_pytest
+  tags: [$TARGET, $ENV_MARKER]
  artifacts:
    when: always
    paths:
@ -26,11 +27,8 @@ example_test_pytest_esp32_generic:
  needs:
    - build_pytest_examples_esp32
  variables:
-    TARGET: esp32
+    TARGET: ESP32
    ENV_MARKER: generic
-  tags: # in gitlab 14.1 or later, we can use `parallel: matrix` with the `tags` keyword. https://docs.gitlab.com/ee/ci/jobs/job_control.html#run-a-matrix-of-parallel-trigger-jobs
-    - ESP32
-    - Example_GENERIC

 example_test_pytest_esp32s2_generic:
  extends:
@ -39,11 +37,8 @@ example_test_pytest_esp32s2_generic:
  needs:
    - build_pytest_examples_esp32s2
  variables:
-    TARGET: esp32s2
+    TARGET: ESP32S2
    ENV_MARKER: generic
-  tags:
-    - ESP32S2
-    - Example_GENERIC

 example_test_pytest_esp32s3_generic:
  extends:
@ -52,11 +47,8 @@ example_test_pytest_esp32s3_generic:
  needs:
    - build_pytest_examples_esp32s3
  variables:
-    TARGET: esp32s3
+    TARGET: ESP32S3
    ENV_MARKER: generic
-  tags:
-    - ESP32S3
-    - Example_GENERIC

 example_test_pytest_esp32c3_generic:
  extends:
@ -65,11 +57,8 @@ example_test_pytest_esp32c3_generic:
  needs:
    - build_pytest_examples_esp32c3
  variables:
-    TARGET: esp32c3
+    TARGET: ESP32C3
    ENV_MARKER: generic
-  tags:
-    - ESP32C3
-    - Example_GENERIC

 example_test_pytest_esp32c3_flash_suspend:
  extends:
@ -78,11 +67,8 @@ example_test_pytest_esp32c3_flash_suspend:
  needs:
    - build_pytest_examples_esp32c3
  variables:
-    TARGET: esp32c3
+    TARGET: ESP32C3
    ENV_MARKER: flash_suspend
-  tags:
-    - ESP32C3_IDF
-    - UT_T1_Flash_Suspend

 .pytest_components_dir_template:
  extends: .pytest_template
@ -96,11 +82,8 @@ component_ut_pytest_esp32_generic:
  needs:
    - build_pytest_components_esp32
  variables:
-    TARGET: esp32
+    TARGET: ESP32
    ENV_MARKER: generic
-  tags:
-    - ESP32
-    - COMPONENT_UT_GENERIC

 component_ut_pytest_esp32_ip101:
  extends:
@ -109,11 +92,8 @@ component_ut_pytest_esp32_ip101:
  needs:
    - build_pytest_components_esp32
  variables:
-    TARGET: esp32
+    TARGET: ESP32
    ENV_MARKER: ip101
-  tags:
-    - ESP32
-    - COMPONENT_UT_IP101

 component_ut_pytest_esp32_lan8720:
  extends:
@ -122,11 +102,8 @@ component_ut_pytest_esp32_lan8720:
  needs:
    - build_pytest_components_esp32
  variables:
-    TARGET: esp32
+    TARGET: ESP32
    ENV_MARKER: lan8720
-  tags:
-    - ESP32
-    - COMPONENT_UT_LAN8720

 component_ut_pytest_esp32s2_generic:
  extends:
@ -135,11 +112,8 @@ component_ut_pytest_esp32s2_generic:
  needs:
    - build_pytest_components_esp32s2
  variables:
-    TARGET: esp32s2
+    TARGET: ESP32S2
    ENV_MARKER: generic
-  tags:
-    - ESP32S2
-    - COMPONENT_UT_GENERIC

 component_ut_pytest_esp32s3_generic:
  extends:
@ -148,11 +122,8 @@ component_ut_pytest_esp32s3_generic:
  needs:
    - build_pytest_components_esp32s3
  variables:
-    TARGET: esp32s3
+    TARGET: ESP32S3
    ENV_MARKER: generic
-  tags:
-    - ESP32S3
-    - COMPONENT_UT_GENERIC

 component_ut_pytest_esp32s3_octal_psram:
  extends:
@ -161,11 +132,8 @@ component_ut_pytest_esp32s3_octal_psram:
  needs:
    - build_pytest_components_esp32s3
  variables:
-    TARGET: esp32s3
+    TARGET: ESP32S3
    ENV_MARKER: octal_psram
-  tags:
-    - ESP32S3
-    - MSPI_F8R8

 component_ut_pytest_esp32c3_generic:
  extends:
@ -174,11 +142,8 @@ component_ut_pytest_esp32c3_generic:
  needs:
    - build_pytest_components_esp32c3
  variables:
-    TARGET: esp32c3
+    TARGET: ESP32C3
    ENV_MARKER: generic
-  tags:
-    - ESP32C3
-    - COMPONENT_UT_GENERIC

 .pytest_test_apps_dir_template:
  extends: .pytest_template
@ -192,12 +157,9 @@ test_app_test_pytest_esp32_generic:
  needs:
    - build_pytest_test_apps_esp32
  variables:
-    TARGET: esp32
+    TARGET: ESP32
    ENV_MARKER: generic
    SETUP_TOOLS: "1"  # need gdb
-  tags:
-    - ESP32
-    - Example_GENERIC

 test_app_test_pytest_esp32s2_generic:
  extends:
@ -206,12 +168,29 @@ test_app_test_pytest_esp32s2_generic:
  needs:
    - build_pytest_test_apps_esp32s2
  variables:
-    TARGET: esp32s2
+    TARGET: ESP32S2
    ENV_MARKER: generic
    SETUP_TOOLS: "1"  # need gdb
-  tags:
-    - ESP32S2
-    - Example_GENERIC
+
+test_app_test_pytest_esp32s3_generic:
+  extends:
+    - .pytest_test_apps_dir_template
+    - .rules:test:custom_test-esp32s3
+  needs:
+    - build_pytest_test_apps_esp32s3
+  variables:
+    TARGET: ESP32S3
+    ENV_MARKER: generic
+
+test_app_test_pytest_esp32c3_generic:
+  extends:
+    - .pytest_test_apps_dir_template
+    - .rules:test:custom_test-esp32c3
+  needs:
+    - build_pytest_test_apps_esp32c3
+  variables:
+    TARGET: ESP32C3
+    ENV_MARKER: generic

 # for parallel jobs, CI_JOB_NAME will be "job_name index/total" (for example, "IT_001 1/2")
 # we need to convert to pattern "job_name_index.yml"
--- a/conftest.py
+++ b/conftest.py
@ -22,6 +22,7 @@ from typing import Callable, List, Optional
 import pytest
 from _pytest.config import Config
 from _pytest.fixtures import FixtureRequest
+from _pytest.main import Session
 from _pytest.nodes import Item
 from _pytest.python import Function
 from pytest_embedded.plugin import parse_configuration
@ -95,7 +96,7 @@ def build_dir(
    """
    param_or_cli: str = getattr(
        request, 'param', None
-    ) or request.config.option.__dict__.get('build_dir')
+    ) or request.config.getoption('build_dir')
    if param_or_cli is not None:  # respect the param and the cli
        return param_or_cli

@ -146,6 +147,12 @@ def pytest_addoption(parser: pytest.Parser) -> None:
    )


+@pytest.hookimpl(tryfirst=True)
+def pytest_sessionstart(session: Session) -> None:
+    if session.config.option.target:
+        session.config.option.target = session.config.getoption('target').lower()
+
+
@pytest.hookimpl(tryfirst=True)
 def pytest_collection_modifyitems(config: Config, items: List[Function]) -> None:
    target = config.getoption('target', None)  # use the `build` dir
--- a/tools/ci/check_copyright_ignore.txt
+++ b/tools/ci/check_copyright_ignore.txt
@ -2502,7 +2502,6 @@ tools/ci/python_packages/tiny_test_fw/DUT.py
 tools/ci/python_packages/tiny_test_fw/Env.py
 tools/ci/python_packages/tiny_test_fw/EnvConfig.py
 tools/ci/python_packages/tiny_test_fw/TinyFW.py
-tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py
 tools/ci/python_packages/tiny_test_fw/Utility/CaseConfig.py
 tools/ci/python_packages/tiny_test_fw/Utility/GitlabCIJob.py
 tools/ci/python_packages/tiny_test_fw/Utility/TestCase.py
--- a/tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py
+++ b/tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py
@ -1,16 +1,5 @@
-# Copyright 2015-2017 Espressif Systems (Shanghai) PTE LTD
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http:#www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0

 """
 Common logic to assign test cases to CI jobs.
@ -187,8 +176,11 @@ class AssignTest(object):

        job_list = list()
        for job_name in ci_config:
+            if 'pytest' in job_name:
+                continue
            if self.CI_TEST_JOB_PATTERN.search(job_name) is not None:
                job_list.extend(self._handle_parallel_attribute(job_name, ci_config[job_name]))
+
        job_list.sort(key=lambda x: x['name'])
        return job_list

--- a/tools/test_apps/peripherals/i2c_wifi/app_test.py
+++ b/tools/test_apps/peripherals/i2c_wifi/app_test.py
@ -1,18 +0,0 @@
-#!/usr/bin/env python
-import tiny_test_fw
-import ttfw_idf
-
-
-@ttfw_idf.idf_custom_test(env_tag='Example_GENERIC')
-def test_startup(env, _):  # type: (tiny_test_fw.Env.Env, None) -> None
-    # Only test for master usage.
-    dut = env.get_dut('i2c_wifi', 'tools/test_apps/peripherals/i2c_wifi')
-    dut.start_app()
-
-    dut.expect('I2C-WIFI test success')
-
-    env.close_dut(dut.name)
-
-
-if __name__ == '__main__':
-    test_startup()
--- a/tools/test_apps/peripherals/i2c_wifi/pytest_i2c_wifi.py
+++ b/tools/test_apps/peripherals/i2c_wifi/pytest_i2c_wifi.py
@ -0,0 +1,11 @@
+# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: CC0-1.0
+
+import pytest
+from pytest_embedded_idf.dut import IdfDut
+
+
+@pytest.mark.supported_targets
+@pytest.mark.generic
+def test_startup(dut: IdfDut) -> None:
+    dut.expect_exact('I2C-WIFI test success')
--- a/tools/test_apps/phy/phy_multi_init_data_test/app_test.py
+++ b/tools/test_apps/phy/phy_multi_init_data_test/app_test.py
@ -1,33 +0,0 @@
-#!/usr/bin/env python
-# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
-# SPDX-License-Identifier: Apache-2.0
-import glob
-import os
-
-import tiny_test_fw
-import ttfw_idf
-from tiny_test_fw import Utility
-
-
-@ttfw_idf.idf_custom_test(env_tag='Example_GENERIC', group='test-apps')
-def test_phy_multi_init_data_bin(env, _):
-    # type: (tiny_test_fw.Env.Env, None) -> None
-    config_files = glob.glob(os.path.join(os.path.dirname(__file__), 'sdkconfig.ci.*'))
-    config_names = [os.path.basename(s).replace('sdkconfig.ci.', '') for s in config_files]
-    for name in config_names:
-        dut = env.get_dut('phy_multi_init_data_test', 'tools/test_apps/phy/phy_multi_init_data_test',app_config_name=name)
-        dut.start_app()
-
-        if 'CONFIG_ESP_PHY_MULTIPLE_INIT_DATA_BIN_EMBED' in dut.app.get_sdkconfig().keys():
-            Utility.console_log('multi init data bin embed test')
-            dut.expect('load embedded multi phy init data')
-        else:
-            Utility.console_log('multi init data bin test')
-            dut.expect('Support multiple PHY init data bins')
-
-        dut.expect('wifi_init finished')
-        Utility.console_log('Test Success')
-
-
-if __name__ == '__main__':
-    test_phy_multi_init_data_bin()
--- a/tools/test_apps/phy/phy_multi_init_data_test/pytest_phy_multi_init_data.py
+++ b/tools/test_apps/phy/phy_multi_init_data_test/pytest_phy_multi_init_data.py
@ -0,0 +1,19 @@
+# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: CC0-1.0
+
+import pytest
+from pytest_embedded_idf.dut import IdfDut
+
+
+@pytest.mark.supported_targets
+@pytest.mark.generic
+@pytest.mark.parametrize('config', [
+    'phy_multiple_init_data',
+    'phy_multiple_init_data_embed',
+], indirect=True)
+def test_phy_multi_init_data_bin(dut: IdfDut, config: str) -> None:
+    if config == 'phy_multiple_init_data':
+        dut.expect_exact('Support multiple PHY init data bins')
+    else:
+        dut.expect_exact('load embedded multi phy init data')
+    dut.expect_exact('wifi_init finished')