From 692deac5aefeeb068d9c80baa0e08dd92d211495 Mon Sep 17 00:00:00 2001 From: David Cermak Date: Mon, 27 Jan 2020 12:12:49 +0100 Subject: [PATCH] CI: assign and target-test stages updated to run test-apps in the ci --- tools/ci/build_test_apps.sh | 10 +++- tools/ci/config/assign-test.yml | 5 +- tools/ci/config/build.yml | 5 +- tools/ci/config/target-test.yml | 25 +++++----- .../ttfw_idf/CIAssignExampleTest.py | 47 ++++++++++------- .../ttfw_idf/CIAssignUnitTest.py | 2 +- tools/ci/python_packages/ttfw_idf/IDFApp.py | 50 +++++++------------ tools/ci/python_packages/ttfw_idf/__init__.py | 9 ++-- .../startup/{test.py => app_test.py} | 20 ++------ 9 files changed, 85 insertions(+), 88 deletions(-) rename tools/test_apps/startup/{test.py => app_test.py} (56%) diff --git a/tools/ci/build_test_apps.sh b/tools/ci/build_test_apps.sh index f014e6219b..d8498fe638 100755 --- a/tools/ci/build_test_apps.sh +++ b/tools/ci/build_test_apps.sh @@ -57,7 +57,7 @@ echo "build_examples running for target $IDF_TARGET" cd ${IDF_PATH} -# This part of the script produces the same result for all the example build jobs. It may be moved to a separate stage +# This part of the script produces the same result for all the test app build jobs. It may be moved to a separate stage # (pre-build) later, then the build jobs will receive ${BUILD_LIST_JSON} file as an artifact. # If changing the work-dir or build-dir, remember to update the "artifacts" in gitlab-ci configs, and IDFApp.py. @@ -68,7 +68,8 @@ ${IDF_PATH}/tools/find_apps.py tools/test_apps \ --build-system cmake \ --target ${IDF_TARGET} \ --recursive \ - --build-dir "\${IDF_PATH}/${BUILD_PATH}/@f/@w/@t/build" \ + --work-dir "${BUILD_PATH}/@f/@w/@t" \ + --build-dir build \ --build-log "${LOG_PATH}/@f.txt" \ --output ${ALL_BUILD_LIST_JSON} \ --config 'sdkconfig.ci=default' \ @@ -79,6 +80,11 @@ ${IDF_PATH}/tools/find_apps.py tools/test_apps \ # 1. If sdkconfig.ci exists, use it build the example with configuration name "default" # 2. If sdkconfig.ci.* exists, use it to build the "*" configuration # 3. If none of the above exist, build the default configuration under the name "default" +# --work-dir and --build-log above uses "placeholders" @x: +# - @f: full path to the test with slashes replaced with underscores +# - @w: wildcard used as config name +# - @t: target name +# so the workdir .../@f/@w/@t would expand to e.g. tools_test_apps_system_startup/default/esp32 # The part below is where the actual builds happen diff --git a/tools/ci/config/assign-test.yml b/tools/ci/config/assign-test.yml index 8978f7d871..4666bd9b62 100644 --- a/tools/ci/config/assign-test.yml +++ b/tools/ci/config/assign-test.yml @@ -9,7 +9,6 @@ assign_test: dependencies: - build_ssc_esp32 - build_esp_idf_tests_cmake - - build_test_apps_esp32 variables: SUBMODULES_TO_FETCH: "components/esptool_py/esptool" EXAMPLE_CONFIG_OUTPUT_PATH: "$CI_PROJECT_DIR/examples/test_configs" @@ -22,6 +21,7 @@ assign_test: - $EXAMPLE_CONFIG_OUTPUT_PATH - $TEST_APP_CONFIG_OUTPUT_PATH - build_examples/artifact_index.json + - build_test_apps/artifact_index.json expire_in: 1 week only: variables: @@ -29,11 +29,12 @@ assign_test: - $BOT_LABEL_UNIT_TEST - $BOT_LABEL_INTEGRATION_TEST - $BOT_LABEL_EXAMPLE_TEST + - $BOT_LABEL_CUSTOM_TEST script: # assign example tests - python tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py $IDF_PATH/examples $CI_TARGET_TEST_CONFIG_FILE $EXAMPLE_CONFIG_OUTPUT_PATH # assign test apps - - python tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py --job-prefix test_app_test_ $IDF_PATH/tools/test_apps $CI_TARGET_TEST_CONFIG_FILE $TEST_APP_CONFIG_OUTPUT_PATH + - python tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py --custom-group test-apps --job-prefix test_app_test_ $IDF_PATH/tools/test_apps $CI_TARGET_TEST_CONFIG_FILE $TEST_APP_CONFIG_OUTPUT_PATH # assign unit test cases - python tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py $UNIT_TEST_CASE_FILE $CI_TARGET_TEST_CONFIG_FILE $IDF_PATH/components/idf_test/unit_test/CIConfigs # clone test script to assign tests diff --git a/tools/ci/config/build.yml b/tools/ci/config/build.yml index f0da827cf1..e5f9c48b1d 100644 --- a/tools/ci/config/build.yml +++ b/tools/ci/config/build.yml @@ -186,8 +186,7 @@ build_examples_cmake_esp32s2: .build_test_apps: &build_test_apps extends: .build_template - parallel: 2 - stage: pre_build + stage: build artifacts: when: always paths: @@ -225,7 +224,7 @@ build_test_apps_esp32: build_test_apps_esp32s2: extends: .build_test_apps variables: - IDF_TARGET: esp32s2beta + IDF_TARGET: esp32s2 # If you want to add new build example jobs, please add it into dependencies of `.example_test_template` diff --git a/tools/ci/config/target-test.yml b/tools/ci/config/target-test.yml index 672c63399f..3384d3cd06 100644 --- a/tools/ci/config/target-test.yml +++ b/tools/ci/config/target-test.yml @@ -85,27 +85,26 @@ .test_app_template: extends: .example_test_template - stage: pre_target_test + stage: target_test dependencies: - assign_test - - build_test_apps_esp32 + only: + refs: + - master + - /^release\/v/ + - /^v\d+\.\d+(\.\d+)?($|-)/ + - triggers + - schedules + variables: + - $BOT_TRIGGER_WITH_LABEL == null + - $BOT_LABEL_CUSTOM_TEST + - $BOT_LABEL_EXAMPLE_TEST variables: TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw" TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/test_apps" CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/tools/test_apps/test_configs" LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS" ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml" - script: - - *define_config_file_name - # first test if config file exists, if not exist, exit 0 - - test -e $CONFIG_FILE || exit 0 - # clone test env configs - - git clone $TEST_ENV_CONFIG_REPOSITORY - - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs - - cd $TEST_FW_PATH - # run test - - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE - .unit_test_template: extends: .example_test_template diff --git a/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py b/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py index 0949c7e869..d887ae1d0e 100644 --- a/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py +++ b/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py @@ -25,25 +25,34 @@ import json import gitlab_api from tiny_test_fw.Utility import CIAssignTest - -EXAMPLE_BUILD_JOB_NAMES = ["build_examples_cmake_esp32", "build_examples_cmake_esp32s2"] IDF_PATH_FROM_ENV = os.getenv("IDF_PATH") -if IDF_PATH_FROM_ENV: - ARTIFACT_INDEX_FILE = os.path.join(IDF_PATH_FROM_ENV, - "build_examples", "artifact_index.json") -else: - ARTIFACT_INDEX_FILE = "artifact_index.json" class ExampleGroup(CIAssignTest.Group): SORT_KEYS = CI_JOB_MATCH_KEYS = ["env_tag", "chip"] + BUILD_LOCAL_DIR = "build_examples" + BUILD_JOB_NAMES = ["build_examples_cmake_esp32", "build_examples_cmake_esp32s2"] + + +class TestAppsGroup(ExampleGroup): + BUILD_LOCAL_DIR = "build_test_apps" + BUILD_JOB_NAMES = ["build_test_apps_esp32", "build_test_apps_esp32s2"] class CIExampleAssignTest(CIAssignTest.AssignTest): CI_TEST_JOB_PATTERN = re.compile(r"^example_test_.+") -def create_artifact_index_file(project_id=None, pipeline_id=None): +def get_artifact_index_file(case_group=ExampleGroup): + if IDF_PATH_FROM_ENV: + artifact_index_file = os.path.join(IDF_PATH_FROM_ENV, + case_group.BUILD_LOCAL_DIR, "artifact_index.json") + else: + artifact_index_file = "artifact_index.json" + return artifact_index_file + + +def create_artifact_index_file(project_id=None, pipeline_id=None, case_group=ExampleGroup): if project_id is None: project_id = os.getenv("CI_PROJECT_ID") if pipeline_id is None: @@ -52,9 +61,10 @@ def create_artifact_index_file(project_id=None, pipeline_id=None): artifact_index_list = [] def format_build_log_path(): - return "build_examples/list_job_{}.json".format(job_info["parallel_num"]) + parallel = job_info["parallel_num"] # Could be None if "parallel_num" not defined for the job + return "{}/list_job_{}.json".format(case_group.BUILD_LOCAL_DIR, parallel or 1) - for build_job_name in EXAMPLE_BUILD_JOB_NAMES: + for build_job_name in case_group.BUILD_JOB_NAMES: job_info_list = gitlab_inst.find_job_id(build_job_name, pipeline_id=pipeline_id) for job_info in job_info_list: raw_data = gitlab_inst.download_artifact(job_info["id"], [format_build_log_path()])[0] @@ -62,13 +72,14 @@ def create_artifact_index_file(project_id=None, pipeline_id=None): for build_info in build_info_list: build_info["ci_job_id"] = job_info["id"] artifact_index_list.append(build_info) + artifact_index_file = get_artifact_index_file(case_group=case_group) try: - os.makedirs(os.path.dirname(ARTIFACT_INDEX_FILE)) + os.makedirs(os.path.dirname(artifact_index_file)) except OSError: # already created pass - with open(ARTIFACT_INDEX_FILE, "w") as f: + with open(artifact_index_file, "w") as f: json.dump(artifact_index_list, f) @@ -86,16 +97,18 @@ if __name__ == '__main__': help="prefix of the test job name in CI yml file") parser.add_argument("--test-case-file-pattern", help="file name pattern used to find Python test case files") + parser.add_argument('--custom-group', + help='select custom-group for the test cases, if other than ExampleTest', + choices=['example','test-apps'], default='example') + args = parser.parse_args() if args.job_prefix: CIExampleAssignTest.CI_TEST_JOB_PATTERN = re.compile(r"^{}.+".format(args.job_prefix)) - assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file, case_group=ExampleGroup) - if args.test_case_file_pattern: - assign_test.test_case_file_pattern = args.test_case_file_pattern + case_group = ExampleGroup if args.custom_group == 'example' else TestAppsGroup - assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file, case_group=ExampleGroup) + assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file, case_group=case_group) assign_test.assign_cases() assign_test.output_configs(args.output_path) - create_artifact_index_file() + create_artifact_index_file(case_group=case_group) diff --git a/tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py b/tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py index 7b7fcb1b56..fc1de8e4cd 100644 --- a/tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py +++ b/tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py @@ -139,7 +139,7 @@ class UnitTestAssignTest(CIAssignTest.AssignTest): def __init__(self, test_case_path, ci_config_file): CIAssignTest.AssignTest.__init__(self, test_case_path, ci_config_file, case_group=Group) - def _search_cases(self, test_case_path, case_filter=None): + def _search_cases(self, test_case_path, case_filter=None, test_case_file_pattern=None): """ For unit test case, we don't search for test functions. The unit test cases is stored in a yaml file which is created in job build-idf-test. diff --git a/tools/ci/python_packages/ttfw_idf/IDFApp.py b/tools/ci/python_packages/ttfw_idf/IDFApp.py index 92376740e5..897fc0e399 100644 --- a/tools/ci/python_packages/ttfw_idf/IDFApp.py +++ b/tools/ci/python_packages/ttfw_idf/IDFApp.py @@ -310,7 +310,7 @@ class Example(IDFApp): """ return [os.path.join(self.binary_path, "..", "sdkconfig")] - def _try_get_binary_from_local_fs(self, app_path, config_name=None, target=None): + def _try_get_binary_from_local_fs(self, app_path, config_name=None, target=None, local_build_dir="build_examples"): # build folder of example path path = os.path.join(self.idf_path, app_path, "build") if os.path.exists(path): @@ -327,7 +327,7 @@ class Example(IDFApp): # (see tools/ci/build_examples_cmake.sh) # For example: $IDF_PATH/build_examples/examples_get-started_blink/default/esp32 app_path_underscored = app_path.replace(os.path.sep, "_") - example_path = os.path.join(self.idf_path, "build_examples") + example_path = os.path.join(self.idf_path, local_build_dir) for dirpath in os.listdir(example_path): if os.path.basename(dirpath) == app_path_underscored: path = os.path.join(example_path, dirpath, config_name, target, "build") @@ -341,7 +341,8 @@ class Example(IDFApp): if path: return path else: - artifacts = Artifacts(self.idf_path, CIAssignExampleTest.ARTIFACT_INDEX_FILE, + artifacts = Artifacts(self.idf_path, + CIAssignExampleTest.get_artifact_index_file(case_group=CIAssignExampleTest.ExampleGroup), app_path, config_name, target) path = artifacts.download_artifacts() if path: @@ -369,7 +370,8 @@ class LoadableElfExample(Example): if path: return path else: - artifacts = Artifacts(self.idf_path, CIAssignExampleTest.ARTIFACT_INDEX_FILE, + artifacts = Artifacts(self.idf_path, + CIAssignExampleTest.get_artifact_index_file(case_group=CIAssignExampleTest.ExampleGroup), app_path, config_name, target) path = artifacts.download_artifact_files(self.app_files) if path: @@ -402,34 +404,20 @@ class UT(IDFApp): raise OSError("Failed to get unit-test-app binary path") -class TestApp(IDFApp): - def _get_sdkconfig_paths(self): - """ - overrides the parent method to provide exact path of sdkconfig for example tests - """ - return [os.path.join(self.binary_path, "..", "sdkconfig")] - - def get_binary_path(self, app_path, config_name=None): - # local build folder - path = os.path.join(self.idf_path, app_path, "build") - if os.path.exists(path): +class TestApp(Example): + def get_binary_path(self, app_path, config_name=None, target=None): + path = self._try_get_binary_from_local_fs(app_path, config_name, target, local_build_dir="build_test_apps") + if path: return path - - if not config_name: - config_name = "default" - - # Search for CI build folders. - # Path format: $IDF_PATH/build_test_apps/app_path_with_underscores/config/target - # (see tools/ci/build_test_apps.sh) - # For example: $IDF_PATH/build_test_apps/startup/default/esp32 - app_path_underscored = app_path.replace(os.path.sep, "_") - build_root = os.path.join(self.idf_path, "build_test_apps") - for dirpath in os.listdir(build_root): - if os.path.basename(dirpath) == app_path_underscored: - path = os.path.join(build_root, dirpath, config_name, self.target, "build") - return path - - raise OSError("Failed to find test app binary") + else: + artifacts = Artifacts(self.idf_path, + CIAssignExampleTest.get_artifact_index_file(case_group=CIAssignExampleTest.TestAppsGroup), + app_path, config_name, target) + path = artifacts.download_artifacts() + if path: + return os.path.join(self.idf_path, path) + else: + raise OSError("Failed to find example binary") class SSC(IDFApp): diff --git a/tools/ci/python_packages/ttfw_idf/__init__.py b/tools/ci/python_packages/ttfw_idf/__init__.py index 46328c1c54..66e12ee489 100644 --- a/tools/ci/python_packages/ttfw_idf/__init__.py +++ b/tools/ci/python_packages/ttfw_idf/__init__.py @@ -88,10 +88,11 @@ def idf_unit_test(app=UT, dut=IDFDUT, chip="ESP32", module="unit-test", executio return test -def idf_test_app_test(app=TestApp, dut=IDFDUT, chip="ESP32", module="misc", execution_time=1, - level="integration", erase_nvs=True, **kwargs): +def idf_custom_test(app=TestApp, dut=IDFDUT, chip="ESP32", module="misc", execution_time=1, + level="integration", erase_nvs=True, config_name=None, group="test-apps", **kwargs): + """ - decorator for testing idf unit tests (with default values for some keyword args). + decorator for idf custom tests (with default values for some keyword args). :param app: test application class :param dut: dut class @@ -100,6 +101,8 @@ def idf_test_app_test(app=TestApp, dut=IDFDUT, chip="ESP32", module="misc", exec :param execution_time: execution time in minutes, int :param level: test level, could be used to filter test cases, string :param erase_nvs: if need to erase_nvs in DUT.start_app() + :param config_name: if specified, name of the app configuration + :param group: identifier to group custom tests (unused for now, defaults to "test-apps") :param kwargs: other keyword args :return: test method """ diff --git a/tools/test_apps/startup/test.py b/tools/test_apps/startup/app_test.py similarity index 56% rename from tools/test_apps/startup/test.py rename to tools/test_apps/startup/app_test.py index 75d067d6be..f6c1a779b3 100644 --- a/tools/test_apps/startup/test.py +++ b/tools/test_apps/startup/app_test.py @@ -1,27 +1,15 @@ #!/usr/bin/env python -import re import os -import sys import glob - -try: - import IDF -except ImportError: - # This environment variable is expected on the host machine - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - - import IDF - -import Utility +import ttfw_idf +from tiny_test_fw import Utility -@IDF.idf_test_app_test(env_tag="test_jtag_arm") +@ttfw_idf.idf_test_app_test(env_tag="test_jtag_arm") def test_startup(env, extra_data): config_files = glob.glob(os.path.join(os.path.dirname(__file__), "sdkconfig.ci.*")) - config_names = [s.replace("sdkconfig.ci.", "") for s in config_files] + config_names = [os.path.basename(s).replace("sdkconfig.ci.", "") for s in config_files] for name in config_names: Utility.console_log("Checking config \"{}\"... ".format(name), end="") dut = env.get_dut("startup", "tools/test_apps/startup", app_config_name=name)