From 4d4b0b646dde1dae39aad88e2e1d90dc7ee682d8 Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Thu, 8 Jun 2017 11:55:27 +0800 Subject: [PATCH 1/4] CI: check if internal submodule synced to GH See MR !834 --- .gitlab-ci.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e674e5a625..b7dc3b66e5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -348,6 +348,21 @@ check_commit_msg: # commit start with "WIP: " need to be squashed before merge - 'git log --pretty=%s master..${CI_COMMIT_REF_NAME} | grep "^WIP: " || exit 0 && exit 1' +check_submodule_sync: + stage: deploy + image: $CI_DOCKER_REGISTRY/esp32-ci-env + tags: + - build + except: + - master + - /^release\/v/ + - /^v\d+\.\d+(\.\d+)?($|-)/ + before_script: + - echo "do not use gitlab submodule repository" + script: + # check if all submodules are correctly synced to public repostory + - git submodule update --init --recursive + assign_test: <<: *build_template stage: assign_test From 3f8e58a63795c69fd6517d91be0ee3a3c0fb0ac3 Mon Sep 17 00:00:00 2001 From: Anton Maklakov Date: Fri, 9 Jun 2017 19:05:17 +0800 Subject: [PATCH 2/4] CI: Put in order the job dependences --- .gitlab-ci.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b7dc3b66e5..56b604a824 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -171,6 +171,7 @@ test_nvs_on_host: image: $CI_DOCKER_REGISTRY/esp32-ci-env tags: - nvs_host_test + dependencies: [] script: - cd components/nvs_flash/test_nvs_host - make test @@ -180,6 +181,7 @@ test_partition_table_on_host: image: $CI_DOCKER_REGISTRY/esp32-ci-env tags: - build + dependencies: [] script: - cd components/partition_table/test_gen_esp32part_host - ./gen_esp32part_tests.py @@ -192,6 +194,7 @@ test_wl_on_host: artifacts: paths: - components/wear_levelling/test_wl_host/coverage_report.zip + dependencies: [] script: - cd components/wear_levelling/test_wl_host - make test @@ -201,6 +204,7 @@ test_build_system: image: $CI_DOCKER_REGISTRY/esp32-ci-env tags: - build_test + dependencies: [] script: - ./make/test_configure_ci_environment.sh - ./make/test_build_system.sh @@ -220,6 +224,8 @@ test_report: TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test" REPORT_PATH: "$CI_PROJECT_DIR/CI_Test_Report" MODULE_UPDATE_FILE: "$CI_PROJECT_DIR/tools/unit-test-app/tools/ModuleDefinition.yml" + #dependencies: + #We need all UT* and IT* artifacts except for only a few other artifacts: when: always paths: @@ -268,6 +274,7 @@ push_master_to_github: - /^release\/v/ - /^v\d+\.\d+(\.\d+)?($|-)/ when: on_success + dependencies: [] variables: GIT_STRATEGY: clone GITHUB_PUSH_REFS: refs/remotes/origin/release refs/remotes/origin/master @@ -298,6 +305,8 @@ deploy_docs: - /^release\/v/ - /^v\d+\.\d+(\.\d+)?($|-)/ - triggers + dependencies: + - build_docs script: - mkdir -p ~/.ssh - chmod 700 ~/.ssh @@ -341,6 +350,7 @@ check_commit_msg: - master - /^release\/v/ - /^v\d+\.\d+(\.\d+)?($|-)/ + dependencies: [] before_script: - echo "skip update submodule" script: @@ -357,6 +367,7 @@ check_submodule_sync: - master - /^release\/v/ - /^v\d+\.\d+(\.\d+)?($|-)/ + dependencies: [] before_script: - echo "do not use gitlab submodule repository" script: @@ -366,6 +377,9 @@ check_submodule_sync: assign_test: <<: *build_template stage: assign_test + dependencies: + - build_esp_idf_tests + - build_ssc artifacts: paths: - test_bins From e334b7267e7e917ed41c8e708f62d1622670e242 Mon Sep 17 00:00:00 2001 From: Anton Maklakov Date: Fri, 9 Jun 2017 18:14:29 +0800 Subject: [PATCH 3/4] CI: Minor changes in code formatting and cleanup --- .gitlab-ci.yml | 84 ++++++++++++++++++++++---------------------------- 1 file changed, 36 insertions(+), 48 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 56b604a824..0760e41a79 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -36,12 +36,10 @@ build_template_app: image: $CI_DOCKER_REGISTRY/esp32-ci-env tags: - build - variables: GIT_STRATEGY: clone BATCH_BUILD: "1" IDF_CI_BUILD: "1" - script: - git clone https://github.com/espressif/esp-idf-template.git - cd esp-idf-template @@ -66,7 +64,6 @@ build_template_app: image: $CI_DOCKER_REGISTRY/esp32-ci-env tags: - build - variables: GIT_STRATEGY: clone BATCH_BUILD: "1" @@ -76,9 +73,8 @@ build_ssc: <<: *build_template artifacts: paths: - - ./SSC/ssc_bin + - SSC/ssc_bin expire_in: 6 mos - script: - git clone $SSC_REPOSITORY - cd SSC @@ -98,15 +94,14 @@ build_esp_idf_tests: <<: *build_template artifacts: paths: - - ./tools/unit-test-app/build/*.bin - - ./tools/unit-test-app/build/*.elf - - ./tools/unit-test-app/build/*.map - - ./tools/unit-test-app/build/download.config - - ./tools/unit-test-app/build/bootloader/*.bin - - ./components/idf_test/unit_test/TestCaseAll.yml - - ./components/idf_test/unit_test/CIConfigs/*.yml + - tools/unit-test-app/build/*.bin + - tools/unit-test-app/build/*.elf + - tools/unit-test-app/build/*.map + - tools/unit-test-app/build/download.config + - tools/unit-test-app/build/bootloader/*.bin + - components/idf_test/unit_test/TestCaseAll.yml + - components/idf_test/unit_test/CIConfigs/*.yml expire_in: 6 mos - script: - cd tools/unit-test-app - make TESTS_ALL=1 @@ -154,6 +149,10 @@ build_docs: image: $CI_DOCKER_REGISTRY/esp32-ci-env tags: - build_docs + artifacts: + paths: + - docs/_build/html + expire_in: 1 mos script: - cd docs - doxygen @@ -161,10 +160,6 @@ build_docs: - test -n $IS_PRIVATE && test $(cat doxygen-warning-log.txt | wc -l) -eq 0 || ( echo "Doxygen pass had some warnings:" && cat doxygen-warning-log.txt && false ) - make gh-linkcheck - make html - artifacts: - paths: - - docs/_build/html - expire_in: 1 mos test_nvs_on_host: stage: test @@ -250,7 +245,7 @@ test_report: - python CITestReport.py -l $LOG_PATH -t $TEST_CASE_FILE_PATH -p $REPORT_PATH -r $RESULT_PATH -a $ARTIFACTS_PATH -m $MODULE_UPDATE_FILE || TEST_RESULT=Fail # commit to CI-test-result project - git clone $GITLAB_SSH_SERVER/qa/CI-test-result.git - - rm -rf CI-test-result/RawData/$RESULT_PATH + - rm -rf "CI-test-result/RawData/$RESULT_PATH" - cp -R $CI_PROJECT_NAME CI-test-result/RawData - cd CI-test-result # config git user @@ -329,6 +324,10 @@ check_doc_links: only: # can only be triggered - triggers + artifacts: + paths: + - docs/_build/linkcheck + expire_in: 1 mos script: # must be triggered with CHECK_LINKS=Yes, otherwise exit without test - test "$CHECK_LINKS" = "Yes" || exit 0 @@ -336,10 +335,6 @@ check_doc_links: - test "${CI_COMMIT_REF_NAME}" = "master" || exit 0 - cd docs - make linkcheck - artifacts: - paths: - - docs/_build/linkcheck - expire_in: 1 mos check_commit_msg: stage: deploy @@ -386,7 +381,6 @@ assign_test: - components/idf_test/*/CIConfigs - components/idf_test/*/TC.sqlite expire_in: 1 mos - script: # first move test bins together: test_bins/CHIP_SDK/TestApp/bin_files - mkdir -p test_bins/ESP32_IDF/UT @@ -402,6 +396,8 @@ assign_test: - python CIAssignTestCases.py -t $IDF_PATH/components/idf_test/integration_test -c $IDF_PATH/.gitlab-ci.yml -b $IDF_PATH/test_bins .test_template: &test_template + before_script: + - echo "Skip cloning submodule here" stage: test when: on_success only: @@ -410,10 +406,13 @@ assign_test: - /^v\d+\.\d+(\.\d+)?($|-)/ - triggers allow_failure: true - dependencies: - assign_test - + artifacts: + when: always + paths: + - $LOG_PATH + expire_in: 6 mos variables: # set git strategy to fetch so we can get esptool without update submodule GIT_STRATEGY: fetch @@ -422,21 +421,11 @@ assign_test: TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test/integration_test" MODULE_UPDATE_FILE: "$CI_PROJECT_DIR/components/idf_test/ModuleDefinition.yml" CONFIG_FILE: "$CI_PROJECT_DIR/components/idf_test/integration_test/CIConfigs/$CI_JOB_NAME.yml" - - artifacts: - when: always - paths: - - $LOG_PATH - expire_in: 6 mos - - before_script: - - echo "Skip cloning submodule here" - script: # first test if config file exists, if not exist, exit 0 - test -e $CONFIG_FILE || exit 0 - # remove artifacts from last stage (UT logs) - - rm -rf $LOG_PATH + # remove artifacts from the 'unit_test' stage + - rm -rf "$LOG_PATH" # add gitlab ssh key - mkdir -p ~/.ssh - chmod 700 ~/.ssh @@ -458,7 +447,6 @@ assign_test: <<: *test_template allow_failure: false stage: unit_test - variables: GIT_STRATEGY: fetch LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF" @@ -473,14 +461,14 @@ nvs_compatible_test: when: always paths: - $LOG_PATH - - $IDF_PATH/nvs_wifi.bin + - nvs_wifi.bin expire_in: 6 mos tags: - ESP32_IDF - NVS_Compatible script: - # remove artifacts from last stage (UT logs) - - rm -rf $LOG_PATH + # remove artifacts from the 'unit_test' stage + - rm -rf "$LOG_PATH" # add gitlab ssh key - mkdir -p ~/.ssh - chmod 700 ~/.ssh @@ -534,43 +522,43 @@ IT_001_02: tags: - ESP32_IDF - SSC_T1_1 - + IT_001_03: <<: *test_template tags: - ESP32_IDF - SSC_T1_1 - + IT_001_04: <<: *test_template tags: - ESP32_IDF - SSC_T1_1 - + IT_001_05: <<: *test_template tags: - ESP32_IDF - SSC_T1_1 - + IT_001_06: <<: *test_template tags: - ESP32_IDF - SSC_T1_1 - + IT_001_07: <<: *test_template tags: - ESP32_IDF - SSC_T1_1 - + IT_001_08: <<: *test_template tags: - ESP32_IDF - SSC_T1_1 - + IT_002_01: <<: *test_template tags: From 14125f336177aad42eae836e411354fb5c99fee2 Mon Sep 17 00:00:00 2001 From: Anton Maklakov Date: Mon, 12 Jun 2017 15:22:09 +0800 Subject: [PATCH 4/4] CI: Add CI options to use more attempts in some transfer situations --- .gitlab-ci.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0760e41a79..03c34057f0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,11 +7,19 @@ stages: - deploy variables: -# system environment +# System environment + # Common parameters for the 'make' during CI tests MAKEFLAGS: "-j5 --no-keep-going" +# GitLab-CI environment + + # more attempts for more robust + GET_SOURCES_ATTEMPTS: "10" + ARTIFACT_DOWNLOAD_ATTEMPTS: "10" + # IDF environment + IDF_PATH: "$CI_PROJECT_DIR" before_script: