From bfa96bc5976417f2f9cc67dccda32956264b4f25 Mon Sep 17 00:00:00 2001 From: Chen Yudong Date: Wed, 14 Sep 2022 02:43:40 +0800 Subject: [PATCH] CI: optimize ATS ci flow --- .gitlab/ci/assign-test.yml | 9 +++--- .gitlab/ci/target-test.yml | 22 ++++++------- tools/ci/checkout_project_ref.py | 55 ++++++++++++++++---------------- 3 files changed, 43 insertions(+), 43 deletions(-) diff --git a/.gitlab/ci/assign-test.yml b/.gitlab/ci/assign-test.yml index a775bf3d4e..fd616f929a 100644 --- a/.gitlab/ci/assign-test.yml +++ b/.gitlab/ci/assign-test.yml @@ -100,12 +100,13 @@ assign_integration_test: INTEGRATION_TEST_CASE_PATH: "${CI_PROJECT_DIR}/auto_test_script/TestCaseFiles" ASSIGN_TEST_CASE_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIAssignTestCases.py" PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages + GIT_LFS_SKIP_SMUDGE: 1 script: - add_gitlab_ssh_keys # clone test script to assign tests - # can not retry if downing git lfs files failed, so using empty_branch first. - - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch - - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH} - - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only + - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} auto_test_script + - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script + - cd auto_test_script + - ./tools/ci/setup_idfci.sh # assign integration test cases - python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b ${BUILD_DIR} -o $TEST_DIR/test_configs diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml index a3fefcdd34..190d4a732e 100644 --- a/.gitlab/ci/target-test.yml +++ b/.gitlab/ci/target-test.yml @@ -1129,8 +1129,8 @@ UT_S3_FLASH: CI_RUNNER_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIRunner.py" PREPARE_TEST_BIN_SCRIPT: "${CI_PROJECT_DIR}/tools/ci/integration_test/prepare_test_bins.py" PYTHONPATH: "${CI_PROJECT_DIR}/auto_test_script/packages:${CI_PROJECT_DIR}/tools/ci/python_packages:${PYTHONPATH}" - SETUP_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/tools/ci/setup_idfci.sh" INITIAL_CONDITION_RETRY_COUNT: "1" + GIT_LFS_SKIP_SMUDGE: 1 script: - *define_config_file_name # first test if config file exists, if not exist, exit 0 @@ -1140,11 +1140,11 @@ UT_S3_FLASH: - retry_failed git clone $TEST_ENV_CONFIG_REPO - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs # clone test bench - # can not retry if downing git lfs files failed, so using empty_branch first. - - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch - - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH} - - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only - - if [ -f ${SETUP_SCRIPT} ]; then ${SETUP_SCRIPT}; fi + - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} auto_test_script + - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script + - cd auto_test_script + - ./tools/ci/setup_idfci.sh + # Merge known issues - cat ${KNOWN_ISSUE_FILE} >> ${TEST_CASE_FILE_PATH}/KnownIssues # run test - python ${PREPARE_TEST_BIN_SCRIPT} $CONFIG_FILE @@ -1176,13 +1176,11 @@ nvs_compatible_test: - retry_failed git clone $TEST_ENV_CONFIG_REPO - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs # clone test bench - # can not retry if downing git lfs files failed, so using empty_branch first. - - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch - - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH} - - if [ -f ${SETUP_SCRIPT} ]; then ${SETUP_SCRIPT}; fi - - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only - # prepare nvs bins + - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} auto_test_script + - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script - cd auto_test_script + - ./tools/ci/setup_idfci.sh + # prepare nvs bins - ./tools/prepare_nvs_bin.sh # run test - python ${PREPARE_TEST_BIN_SCRIPT} $CONFIG_FILE diff --git a/tools/ci/checkout_project_ref.py b/tools/ci/checkout_project_ref.py index 10af27c760..4a987c2eee 100755 --- a/tools/ci/checkout_project_ref.py +++ b/tools/ci/checkout_project_ref.py @@ -10,10 +10,25 @@ import json import os import re import subprocess -from typing import List +from typing import List, Tuple IDF_GIT_DESCRIBE_PATTERN = re.compile(r'^v(\d)\.(\d)') -RETRY_COUNT = 3 + + +def _idf_version_from_cmake() -> Tuple[int, int]: + version_path = os.path.join(os.environ['IDF_PATH'], 'tools/cmake/version.cmake') + regex = re.compile(r'^\s*set\s*\(\s*IDF_VERSION_([A-Z]{5})\s+(\d+)') + try: + ver = {} + with open(version_path) as f: + for line in f: + m = regex.match(line) + if m: + ver[m.group(1)] = m.group(2) + return (int(ver['MAJOR']), int(ver['MINOR'])) + except (KeyError, OSError): + print('WARNING: Cannot find ESP-IDF version in version.cmake') + return (0, 0) def get_customized_project_revision(proj_name: str) -> str: @@ -47,20 +62,12 @@ def target_branch_candidates(proj_name: str) -> List: candidates.insert(0, customized_candidate) # branch name read from IDF - try: - git_describe = subprocess.check_output(['git', 'describe', 'HEAD']) - match = IDF_GIT_DESCRIBE_PATTERN.search(git_describe.decode()) - if match: - major_revision = match.group(1) - minor_revision = match.group(2) - # release branch - candidates.append('release/v{}.{}'.format(major_revision, minor_revision)) - # branch to match all major branches, like v3.x or v3 - candidates.append('release/v{}.x'.format(major_revision)) - candidates.append('release/v{}'.format(major_revision)) - except subprocess.CalledProcessError: - # this should not happen as IDF should have describe message - pass + major_revision, minor_revision = _idf_version_from_cmake() + # release branch + candidates.append('release/v{}.{}'.format(major_revision, minor_revision)) + # branch to match all major branches, like v3.x or v3 + candidates.append('release/v{}.x'.format(major_revision)) + candidates.append('release/v{}'.format(major_revision)) return [c for c in candidates if c] # filter out null value @@ -103,16 +110,10 @@ if __name__ == '__main__': continue if ref_to_use: - for _ in range(RETRY_COUNT): - # Add retry for projects with git-lfs - try: - subprocess.check_call(['git', 'checkout', '-f', ref_to_use], stdout=subprocess.PIPE) # not print the stdout - print('CI using ref {} for project {}'.format(ref_to_use, args.project)) - break - except subprocess.CalledProcessError: - pass - else: - print('Failed to use ref {} for project {}'.format(ref_to_use, args.project)) - exit(1) + try: + subprocess.check_call(['git', 'checkout', '-f', ref_to_use], stdout=subprocess.PIPE) # not print the stdout + print('CI using ref {} for project {}'.format(ref_to_use, args.project)) + except subprocess.CalledProcessError: + pass else: print('using default branch')