From 9e8315d9b3ac8a74db68fab74d80da877b54e548 Mon Sep 17 00:00:00 2001 From: Chen Yu Dong Date: Fri, 30 Oct 2020 14:39:42 +0800 Subject: [PATCH] ci: limit switching branches for auto_test_script --- tools/ci/checkout_project_ref.py | 60 ++++++++++++++++++++++++-------- tools/ci/config/assign-test.yml | 2 +- tools/ci/config/target-test.yml | 4 +-- 3 files changed, 48 insertions(+), 18 deletions(-) diff --git a/tools/ci/checkout_project_ref.py b/tools/ci/checkout_project_ref.py index d9d040254f..175c6fe295 100755 --- a/tools/ci/checkout_project_ref.py +++ b/tools/ci/checkout_project_ref.py @@ -11,6 +11,22 @@ import re IDF_GIT_DESCRIBE_PATTERN = re.compile(r"^v(\d)\.(\d)") +RETRY_COUNT = 3 + + +def get_customized_project_revision(proj_name): + """ + get customized project revision defined in bot message + """ + revision = "" + customized_project_revisions = os.getenv("BOT_CUSTOMIZED_REVISION") + if customized_project_revisions: + customized_project_revisions = json.loads(customized_project_revisions) + try: + revision = customized_project_revisions[proj_name.lower()] + except (KeyError, TypeError): + pass + return revision def target_branch_candidates(proj_name): @@ -23,16 +39,11 @@ def target_branch_candidates(proj_name): # CI_MERGE_REQUEST_TARGET_BRANCH_NAME os.getenv("CI_MERGE_REQUEST_TARGET_BRANCH_NAME"), ] - # revision defined in bot message - customized_project_revisions = os.getenv("BOT_CUSTOMIZED_REVISION") - if customized_project_revisions: - customized_project_revisions = json.loads(customized_project_revisions) - try: - ref_to_use = customized_project_revisions[proj_name.lower()] + customized_candidate = get_customized_project_revision(proj_name) + if customized_candidate: # highest priority, insert to head of list - candidates.insert(0, ref_to_use) - except (KeyError, TypeError): - pass + candidates.insert(0, customized_candidate) + # branch name read from IDF try: git_describe = subprocess.check_output(["git", "describe", "HEAD"]) @@ -58,20 +69,39 @@ if __name__ == "__main__": help="the name of project") parser.add_argument("project_relative_path", help="relative path of project to IDF repository directory") + parser.add_argument('--customized_only', action='store_true', + help="Only to find customized revision") args = parser.parse_args() - candidate_branches = target_branch_candidates(args.project) + if args.customized_only: + customized_revision = get_customized_project_revision(args.project) + candidate_branches = [customized_revision] if customized_revision else [] + else: + candidate_branches = target_branch_candidates(args.project) # change to project dir for checkout os.chdir(args.project_relative_path) + ref_to_use = "" for candidate in candidate_branches: - try: - subprocess.check_call(["git", "checkout", "-f", candidate], stdout=subprocess.PIPE, stderr=subprocess.PIPE) # not print the stdout nor stderr - print("CI using ref {} for project {}".format(candidate, args.project)) + # check if candidate branch exists + branch_match = subprocess.check_output(["git", "branch", "-a", "--list", "origin/" + candidate]) + if branch_match: + ref_to_use = candidate break - except subprocess.CalledProcessError: - pass + + if ref_to_use: + for _ in range(RETRY_COUNT): + # Add retry for projects with git-lfs + try: + subprocess.check_call(["git", "checkout", "-f", ref_to_use], stdout=subprocess.PIPE) # not print the stdout + print("CI using ref {} for project {}".format(ref_to_use, args.project)) + break + except subprocess.CalledProcessError: + pass + else: + print("Failed to use ref {} for project {}".format(ref_to_use, args.project)) + exit(1) else: print("using default branch") diff --git a/tools/ci/config/assign-test.yml b/tools/ci/config/assign-test.yml index bb1b8920c9..ce69bffad0 100644 --- a/tools/ci/config/assign-test.yml +++ b/tools/ci/config/assign-test.yml @@ -44,7 +44,7 @@ assign_test: # can not retry if downing git lfs files failed, so using empty_branch first. - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH} - - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script + - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only # assign integration test cases - python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b $IDF_PATH/SSC/ssc_bin -o $INTEGRATION_CONFIG_OUTPUT_PATH diff --git a/tools/ci/config/target-test.yml b/tools/ci/config/target-test.yml index 4b8cb9830a..613e178757 100644 --- a/tools/ci/config/target-test.yml +++ b/tools/ci/config/target-test.yml @@ -110,7 +110,7 @@ # can not retry if downing git lfs files failed, so using empty_branch first. - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH} - - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script + - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only - cat ${KNOWN_ISSUE_FILE} >> ${TEST_CASE_FILE_PATH}/KnownIssues # run test - python ${CI_RUNNER_SCRIPT} -l "$LOG_PATH/$JOB_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH @@ -560,7 +560,7 @@ nvs_compatible_test: # can not retry if downing git lfs files failed, so using empty_branch first. - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH} - - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script + - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only # prepare nvs bins - cd auto_test_script - ./tools/prepare_nvs_bin.sh