From a57b0446b6a7ba9560503f61f4f7d1b8f88b5bff Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Wed, 3 Aug 2022 11:56:35 +0800 Subject: [PATCH 1/4] ci: improve requirements ci/ttfw files --- tools/requirements/requirements.ci.txt | 1 + tools/requirements/requirements.ttfw.txt | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/tools/requirements/requirements.ci.txt b/tools/requirements/requirements.ci.txt index b183644fec..0d4c00f282 100644 --- a/tools/requirements/requirements.ci.txt +++ b/tools/requirements/requirements.ci.txt @@ -4,6 +4,7 @@ # ci coverage idf-build-apps +jsonschema junit_xml python-gitlab pyyaml diff --git a/tools/requirements/requirements.ttfw.txt b/tools/requirements/requirements.ttfw.txt index ed7cb2187f..e100531d57 100644 --- a/tools/requirements/requirements.ttfw.txt +++ b/tools/requirements/requirements.ttfw.txt @@ -33,3 +33,7 @@ SimpleWebSocketServer # py_debug_backend debug_backend + +# examples/protocols/mqtt +# tools/test_apps/protocols/mqtt +paho-mqtt From da5a5b52c860f7febd84bd4c0c075005bcccd8b2 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Thu, 28 Jul 2022 15:32:13 +0800 Subject: [PATCH 2/4] ci: simplify the python env to idf venv only --- .gitlab-ci.yml | 170 +++++++++++-------------- .gitlab/ci/README.md | 2 +- .gitlab/ci/assign-test.yml | 2 + .gitlab/ci/deploy.yml | 1 - .gitlab/ci/docs.yml | 12 +- .gitlab/ci/host-test.yml | 8 +- .gitlab/ci/pre_check.yml | 44 +++---- .gitlab/ci/static-code-analysis.yml | 5 +- .gitlab/ci/target-test.yml | 6 +- .gitlab/ci/upload_cache.yml | 13 +- tools/ci/exclude_check_tools_files.txt | 1 - tools/ci/setup_python.sh | 50 -------- 12 files changed, 109 insertions(+), 205 deletions(-) delete mode 100644 tools/ci/setup_python.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9f58e89df0..ed7a85d60c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -57,20 +57,21 @@ variables: PYTHON_VER: 3.7.10 CLANG_TIDY_RUNNER_PROJ: 2107 # idf/clang-tidy-runner - IDF_BUILD_APPS_PROJ: 2818 # fuhanxi/idf-build-apps + IDF_BUILD_APPS_PROJ: 2818 # espressif/idf-build-apps # Docker images BOT_DOCKER_IMAGE_TAG: ":latest" - ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env-v5.0:2-3" - ESP_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-env-v5.0:3" - AFL_FUZZER_TEST_IMAGE: "$CI_DOCKER_REGISTRY/afl-fuzzer-test-v5.0:2-1" - CLANG_STATIC_ANALYSIS_IMAGE: "${CI_DOCKER_REGISTRY}/clang-static-analysis-v5.0:2-1" - TARGET_TEST_ENV_IMAGE: "$CI_DOCKER_REGISTRY/target-test-env-v5.0:2" - QEMU_IMAGE: "${CI_DOCKER_REGISTRY}/qemu-v5.0:2-20220802" + ESP_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-env-v5.0:4" + AFL_FUZZER_TEST_IMAGE: "$CI_DOCKER_REGISTRY/afl-fuzzer-test-v5.0:4-2" + CLANG_STATIC_ANALYSIS_IMAGE: "${CI_DOCKER_REGISTRY}/clang-static-analysis-v5.0:4-2" + ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env-v5.0:4-4" + QEMU_IMAGE: "${CI_DOCKER_REGISTRY}/qemu-v5.0:4-20220802" + TARGET_TEST_ENV_IMAGE: "$CI_DOCKER_REGISTRY/target-test-env-v5.0:3" SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:3" - LINUX_SHELL_IMAGE: "${CI_DOCKER_REGISTRY}/linux-shells-v5.0:2" + + PRE_COMMIT_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-pre-commit:1" # target test config file, used by assign test job CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/.gitlab/ci/target-test.yml" @@ -110,130 +111,101 @@ cache: - .cache/submodule_archives policy: pull -.setup_tools_unless_target_test: &setup_tools_unless_target_test | +.common_before_scripts: &common-before_scripts | + source tools/ci/utils.sh + is_based_on_commits $REQUIRED_ANCESTOR_COMMITS + if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then - export IDF_MIRROR_PREFIX_MAP= - fi - if [[ "$SETUP_TOOLS" == "1" || "$CI_JOB_STAGE" != "target_test" ]]; then - tools/idf_tools.py --non-interactive install ${SETUP_TOOLS_LIST:-} && eval "$(tools/idf_tools.py --non-interactive export)" || exit 1 - if [[ ! -z "$OOCD_DISTRO_URL" && "$CI_JOB_STAGE" == "target_test" ]]; then - echo "Using custom OpenOCD from ${OOCD_DISTRO_URL}" - wget $OOCD_DISTRO_URL - ARCH_NAME=$(basename $OOCD_DISTRO_URL) - tar -x -f $ARCH_NAME - export OPENOCD_SCRIPTS=$PWD/openocd-esp32/share/openocd/scripts - export PATH=$PWD/openocd-esp32/bin:$PATH - fi + export IDF_MIRROR_PREFIX_MAP= fi -.download_test_python_contraint_file: &download_test_python_contraint_file | + if echo "$CI_MERGE_REQUEST_LABELS" | egrep "^([^,\n\r]+,)*include_nightly_run(,[^,\n\r]+)*$"; then + export INCLUDE_NIGHTLY_RUN="1" + fi + + # configure cmake related flags + source tools/ci/configure_ci_environment.sh + + # add extra python packages + export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/esp_app_trace:$IDF_PATH/components/partition_table:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH" + +.setup_tools_and_idf_python_venv: &setup_tools_and_idf_python_venv | + # must use after setup_tools_except_target_test + # otherwise the export.sh won't work properly + + # download constraint file for dev if [[ -n "$CI_PYTHON_CONSTRAINT_BRANCH" ]]; then wget -O /tmp/constraint.txt --header="Authorization:Bearer ${ESPCI_TOKEN}" ${GITLAB_HTTP_SERVER}/api/v4/projects/2581/repository/files/${CI_PYTHON_CONSTRAINT_FILE}/raw?ref=${CI_PYTHON_CONSTRAINT_BRANCH} mkdir -p ~/.espressif mv /tmp/constraint.txt ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE} fi -.download_test_python_tool: &download_test_python_tool | + # Mirror + if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then + export IDF_MIRROR_PREFIX_MAP= + fi + + # install latest python packages + # target test jobs + if [[ "${CI_JOB_STAGE}" == "target_test" ]]; then + # ttfw jobs + if ! echo "${CI_JOB_NAME}" | egrep ".+_pytest_.+"; then + run_cmd bash install.sh --enable-ci --enable-ttfw + else + run_cmd bash install.sh --enable-ci --enable-pytest + fi + elif [[ "${CI_JOB_STAGE}" == "build_doc" ]]; then + run_cmd bash install.sh --enable-ci --enable-docs + elif [[ "${CI_JOB_STAGE}" == "build" ]]; then + run_cmd bash install.sh --enable-ci --enable-pytest + else + run_cmd bash install.sh --enable-ci + fi + + source ./export.sh + + # Custom OpenOCD + if [[ ! -z "$OOCD_DISTRO_URL" && "$CI_JOB_STAGE" == "target_test" ]]; then + echo "Using custom OpenOCD from ${OOCD_DISTRO_URL}" + wget $OOCD_DISTRO_URL + ARCH_NAME=$(basename $OOCD_DISTRO_URL) + tar -x -f $ARCH_NAME + export OPENOCD_SCRIPTS=$PWD/openocd-esp32/share/openocd/scripts + export PATH=$PWD/openocd-esp32/bin:$PATH + fi + if [[ -n "$CI_PYTHON_TOOL_REPO" ]]; then git clone --quiet --depth=1 -b ${CI_PYTHON_TOOL_BRANCH} https://gitlab-ci-token:${ESPCI_TOKEN}@${GITLAB_HTTPS_HOST}/espressif/${CI_PYTHON_TOOL_REPO}.git pip install ./${CI_PYTHON_TOOL_REPO} rm -rf ${CI_PYTHON_TOOL_REPO} fi -.set_include_nightly_run: &set_include_nightly_run | - if echo "$CI_MERGE_REQUEST_LABELS" | egrep "^([^,\n\r]+,)*include_nightly_run(,[^,\n\r]+)*$"; then - export INCLUDE_NIGHTLY_RUN="1" - fi - before_script: - - source tools/ci/utils.sh - - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS - - *set_include_nightly_run - - source tools/ci/setup_python.sh + - *common-before_scripts + - *setup_tools_and_idf_python_venv - add_gitlab_ssh_keys - - source tools/ci/configure_ci_environment.sh - - *setup_tools_unless_target_test - fetch_submodules - - *download_test_python_contraint_file - - $IDF_PATH/tools/idf_tools.py install-python-env - # install esptool globally, TODO: remove this, IDFCI-1207 - - pip install esptool -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE} - - *download_test_python_tool - -# used for check scripts which we want to run unconditionally -.before_script_no_sync_submodule: - before_script: - - echo "Not setting up GitLab key, not fetching submodules" - - source tools/ci/utils.sh - - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS - - *set_include_nightly_run - - source tools/ci/setup_python.sh - - source tools/ci/configure_ci_environment.sh - - *download_test_python_contraint_file - - $IDF_PATH/tools/idf_tools.py install-python-env .before_script_minimal: before_script: - - echo "Only load utils.sh" - - source tools/ci/utils.sh - - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS - - *set_include_nightly_run - -.before_script_integration_test: - before_script: - # needn't install idf python env - - source tools/ci/utils.sh - - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS - - add_gitlab_ssh_keys + - *common-before_scripts .before_script_macos: before_script: - - source tools/ci/utils.sh - - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS - - *set_include_nightly_run - - export IDF_TOOLS_PATH="${HOME}/.espressif_runner_${CI_RUNNER_ID}_${CI_CONCURRENT_ID}" - - *download_test_python_contraint_file - - $IDF_PATH/tools/idf_tools.py install-python-env + - *common-before_scripts # On macOS, these tools need to be installed + - export IDF_TOOLS_PATH="${HOME}/.espressif_runner_${CI_RUNNER_ID}_${CI_CONCURRENT_ID}" - $IDF_PATH/tools/idf_tools.py --non-interactive install cmake ninja # This adds tools (compilers) and the version-specific Python environment to PATH - - *setup_tools_unless_target_test - # Install packages required by CI scripts into IDF Python environment - - pip install -r $IDF_PATH/tools/requirements/requirements.ci.txt - - source tools/ci/configure_ci_environment.sh - # Part of tools/ci/setup_python.sh; we don't use pyenv on macOS, so can't run the rest of the script. - - export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH" + - *setup_tools_and_idf_python_venv - fetch_submodules -.before_script_pytest: - before_script: - - source tools/ci/utils.sh - - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS - - *set_include_nightly_run - - source tools/ci/setup_python.sh - - add_gitlab_ssh_keys - - source tools/ci/configure_ci_environment.sh - - *setup_tools_unless_target_test - - fetch_submodules - - *download_test_python_contraint_file - # only need ci and pytest related packages, ttfw requirements would be mocked if not installed - - $IDF_PATH/tools/idf_tools.py install-python-env --features pytest,ci - - eval "$($IDF_PATH/tools/idf_tools.py export)" # use idf venv instead - .before_script_build_jobs: before_script: - - source tools/ci/utils.sh - - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS - - *set_include_nightly_run - - source tools/ci/setup_python.sh + - *common-before_scripts + - *setup_tools_and_idf_python_venv - add_gitlab_ssh_keys - - source tools/ci/configure_ci_environment.sh - - *setup_tools_unless_target_test - fetch_submodules - - *download_test_python_contraint_file - # only need ci and pytest related packages, ttfw requirements would be mocked if not installed - - $IDF_PATH/tools/idf_tools.py install-python-env --features pytest,ci - - eval "$($IDF_PATH/tools/idf_tools.py export)" # use idf venv instead - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS} - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS} diff --git a/.gitlab/ci/README.md b/.gitlab/ci/README.md index 5c34d36fd6..8a8051e904 100644 --- a/.gitlab/ci/README.md +++ b/.gitlab/ci/README.md @@ -116,7 +116,7 @@ curl -X POST --header "PRIVATE-TOKEN: [YOUR PERSONAL ACCESS TOKEN]" [GITLAB_SERV ```yaml check_docs_lang_sync: extends: - - .pre_check_job_template + - .pre_check_template - .rules:build:docs script: - cd docs diff --git a/.gitlab/ci/assign-test.yml b/.gitlab/ci/assign-test.yml index ea342e59ad..a775bf3d4e 100644 --- a/.gitlab/ci/assign-test.yml +++ b/.gitlab/ci/assign-test.yml @@ -85,6 +85,7 @@ assign_integration_test: extends: - .assign_test_template - .rules:test:integration_test + - .before_script_minimal image: ${CI_INTEGRATION_TEST_ENV_IMAGE} needs: - build_ssc_esp32 @@ -100,6 +101,7 @@ assign_integration_test: ASSIGN_TEST_CASE_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIAssignTestCases.py" PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages script: + - add_gitlab_ssh_keys # clone test script to assign tests # can not retry if downing git lfs files failed, so using empty_branch first. - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch diff --git a/.gitlab/ci/deploy.yml b/.gitlab/ci/deploy.yml index bd0eec0678..0136f5f582 100644 --- a/.gitlab/ci/deploy.yml +++ b/.gitlab/ci/deploy.yml @@ -1,5 +1,4 @@ .deploy_job_template: - extends: .before_script_no_sync_submodule stage: deploy image: $ESP_ENV_IMAGE tags: diff --git a/.gitlab/ci/docs.yml b/.gitlab/ci/docs.yml index c781c77a52..88bfa3ac48 100644 --- a/.gitlab/ci/docs.yml +++ b/.gitlab/ci/docs.yml @@ -41,7 +41,7 @@ # stage: pre_check check_readme_links: extends: - - .pre_check_job_template + - .pre_check_template tags: ["build", "amd64", "internet"] allow_failure: true script: @@ -49,7 +49,7 @@ check_readme_links: check_docs_lang_sync: extends: - - .pre_check_job_template + - .pre_check_template - .doc-rules:build:docs script: - cd docs @@ -62,7 +62,6 @@ check_docs_lang_sync: dependencies: [] script: - cd docs - - pip install -U -r $IDF_PATH/tools/requirements/requirements.docs.txt - build-docs -t $DOCTGT -bs $DOC_BUILDERS -l $DOCLANG build parallel: matrix: @@ -72,11 +71,10 @@ check_docs_lang_sync: check_docs_gh_links: image: $ESP_IDF_DOC_ENV_IMAGE extends: - - .pre_check_job_template + - .pre_check_template - .doc-rules:build:docs script: - cd docs - - pip install -U -r $IDF_PATH/tools/requirements/requirements.docs.txt - build-docs gh-linkcheck # stage: build_doc @@ -148,8 +146,6 @@ build_docs_pdf: DOC_BUILDERS: "latex" .deploy_docs_template: - extends: - - .before_script_no_sync_submodule image: $ESP_IDF_DOC_ENV_IMAGE variables: DOCS_BUILD_DIR: "${IDF_PATH}/docs/_build/" @@ -161,7 +157,6 @@ build_docs_pdf: script: - add_doc_server_ssh_keys $DOCS_DEPLOY_PRIVATEKEY $DOCS_DEPLOY_SERVER $DOCS_DEPLOY_SERVER_USER - export GIT_VER=$(git describe --always ${PIPELINE_COMMIT_SHA} --) - - pip install -U -r $IDF_PATH/tools/requirements/requirements.docs.txt - deploy-docs # stage: test_deploy @@ -227,5 +222,4 @@ check_doc_links: allow_failure: true script: - cd docs - - pip install -U -r $IDF_PATH/tools/requirements/requirements.docs.txt - build-docs -t $DOCTGT -l $DOCLANG linkcheck diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index 9bc91765b6..2de277fbd5 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -316,8 +316,8 @@ test_mkuf2: - ./test_mkuf2.py test_autocomplete: - extends: .host_test_template - image: $LINUX_SHELL_IMAGE + extends: + - .host_test_template artifacts: when: on_failure paths: @@ -327,8 +327,8 @@ test_autocomplete: - ${IDF_PATH}/tools/ci/test_autocomplete.py test_detect_python: - extends: .host_test_template - image: $LINUX_SHELL_IMAGE + extends: + - .host_test_template script: - cd ${IDF_PATH} - shellcheck -s sh tools/detect_python.sh diff --git a/.gitlab/ci/pre_check.yml b/.gitlab/ci/pre_check.yml index db724d6228..b341cae523 100644 --- a/.gitlab/ci/pre_check.yml +++ b/.gitlab/ci/pre_check.yml @@ -1,21 +1,15 @@ -.pre_check_base_template: +.pre_check_template: stage: pre_check image: $ESP_ENV_IMAGE tags: - host_test dependencies: [] -.pre_check_job_template: - extends: - - .pre_check_base_template - - .before_script_no_sync_submodule - .check_pre_commit_template: - extends: .pre_check_job_template - image: "$CI_DOCKER_REGISTRY/esp-idf-pre-commit:1" - before_script: - - source tools/ci/utils.sh - - export PYTHONPATH="$CI_PROJECT_DIR/tools:$CI_PROJECT_DIR/tools/ci/python_packages:$PYTHONPATH" + extends: + - .pre_check_template + - .before_script_minimal + image: $PRE_COMMIT_IMAGE check_pre_commit_master_release: extends: @@ -35,21 +29,21 @@ check_version: # Don't run this for feature/bugfix branches, so that it is possible to modify # esp_idf_version.h in a branch before tagging the next version. extends: - - .pre_check_job_template + - .pre_check_template - .rules:protected script: - export IDF_PATH=$PWD - tools/ci/check_idf_version.sh check_rom_api_header: - extends: .pre_check_job_template + extends: .pre_check_template script: - tools/ci/check_examples_rom_header.sh - tools/ci/check_api_violation.sh check_python_style: extends: - - .pre_check_base_template + - .pre_check_template - .rules:patterns:python-files artifacts: when: on_failure @@ -60,7 +54,7 @@ check_python_style: - python -m flake8 --config=$IDF_PATH/.flake8 --output-file=flake8_output.txt --tee --benchmark $IDF_PATH test_check_kconfigs: - extends: .pre_check_job_template + extends: .pre_check_template artifacts: when: on_failure paths: @@ -75,7 +69,7 @@ test_check_kconfigs: - python ${IDF_PATH}/tools/ci/test_check_kconfigs.py check_blobs: - extends: .pre_check_base_template + extends: .pre_check_template tags: - build variables: @@ -90,7 +84,7 @@ check_blobs: - bash $IDF_PATH/tools/ci/check_blobs.sh check_fuzzer_compilation: - extends: .pre_check_base_template + extends: .pre_check_template image: $AFL_FUZZER_TEST_IMAGE script: - cd ${IDF_PATH}/components/lwip/test_afl_host @@ -100,7 +94,7 @@ check_fuzzer_compilation: check_public_headers: extends: - - .pre_check_base_template + - .pre_check_template - .rules:build tags: - build @@ -113,7 +107,7 @@ check_public_headers: check_chip_support_components: extends: - - .pre_check_base_template + - .pre_check_template - .rules:build tags: - build @@ -130,7 +124,7 @@ check_chip_support_components: check_esp_err_to_name: extends: - - .pre_check_base_template + - .pre_check_template - .rules:build tags: - build @@ -146,7 +140,7 @@ check_esp_err_to_name: check_esp_system: extends: - - .pre_check_base_template + - .pre_check_template - .rules:build tags: - build @@ -157,19 +151,19 @@ check_esp_system: # the version returned by 'git describe' check_version_tag: extends: - - .pre_check_job_template + - .pre_check_template - .rules:tag:release script: - (git cat-file -t $CI_COMMIT_REF_NAME | grep tag) || (echo "ESP-IDF versions must be annotated tags." && exit 1) check_artifacts_expire_time: - extends: .pre_check_job_template + extends: .pre_check_template script: # check if we have set expire time for all artifacts - python tools/ci/check_artifacts_expire_time.py check_commit_msg: - extends: .pre_check_job_template + extends: .pre_check_template script: - git status - git log -n10 --oneline ${PIPELINE_COMMIT_SHA} @@ -178,7 +172,7 @@ check_commit_msg: check_test_scripts_build_test_rules: extends: - - .pre_check_job_template + - .pre_check_template - .before_script_build_jobs script: - python tools/ci/check_build_test_rules.py check-test-scripts examples/ tools/test_apps components diff --git a/.gitlab/ci/static-code-analysis.yml b/.gitlab/ci/static-code-analysis.yml index 23de4fb9be..55ee95cf3e 100644 --- a/.gitlab/ci/static-code-analysis.yml +++ b/.gitlab/ci/static-code-analysis.yml @@ -1,7 +1,7 @@ # pre_check stage clang_tidy_check: extends: - - .pre_check_base_template + - .pre_check_template - .rules:patterns:clang_tidy image: ${CLANG_STATIC_ANALYSIS_IMAGE} artifacts: @@ -14,7 +14,6 @@ clang_tidy_check: RULES_FILE: ${CI_PROJECT_DIR}/tools/ci/static-analysis-rules.yml OUTPUT_DIR: ${CI_PROJECT_DIR}/clang_tidy_reports script: - - python -m pip install -U pip - internal_pip_install $CLANG_TIDY_RUNNER_PROJ pyclang - export PATH=$PATH:$(python -c "import sys; print(sys.executable.rsplit('/', 1)[0])") - dirs=$(cat ${CLANG_TIDY_DIRS_TXT} | while read line; do echo ${CI_PROJECT_DIR}/${line}; done | xargs) @@ -26,7 +25,7 @@ clang_tidy_check: check_pylint: extends: - - .pre_check_base_template + - .pre_check_template - .rules:patterns:python-files - .before_script_minimal image: $SONARQUBE_SCANNER_IMAGE diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml index 9a3c9c4854..2db8218cfb 100644 --- a/.gitlab/ci/target-test.yml +++ b/.gitlab/ci/target-test.yml @@ -15,7 +15,6 @@ .pytest_template: extends: - .target_test_template - - .before_script_pytest artifacts: when: always paths: @@ -1032,8 +1031,7 @@ UT_S3_FLASH: extends: - .target_test_job_template - .rules:test:integration_test - # needn't install idf python env - - .before_script_integration_test + - .before_script_minimal image: ${CI_INTEGRATION_TEST_ENV_IMAGE} cache: [] needs: # the assign already needs all the build jobs @@ -1054,6 +1052,7 @@ UT_S3_FLASH: # first test if config file exists, if not exist, exit 0 - test -e $CONFIG_FILE || exit 0 # clone local test env configs + - add_gitlab_ssh_keys - retry_failed git clone $TEST_ENV_CONFIG_REPO - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs # clone test bench @@ -1089,6 +1088,7 @@ nvs_compatible_test: # first test if config file exists, if not exist, exit 0 - test -e $CONFIG_FILE || exit 0 # clone local test env configs + - add_gitlab_ssh_keys - retry_failed git clone $TEST_ENV_CONFIG_REPO - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs # clone test bench diff --git a/.gitlab/ci/upload_cache.yml b/.gitlab/ci/upload_cache.yml index b2ddb81bc4..8a512f8c75 100644 --- a/.gitlab/ci/upload_cache.yml +++ b/.gitlab/ci/upload_cache.yml @@ -9,6 +9,7 @@ upload-pip-cache: extends: - .upload_cache_template + - .before_script_minimal - .rules:patterns:python-cache tags: - $GEO @@ -18,13 +19,9 @@ upload-pip-cache: paths: - .cache/pip policy: push - before_script: [] script: - - source tools/ci/utils.sh - - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS - - source tools/ci/setup_python.sh - rm -rf .cache/pip # clear old packages - - $IDF_PATH/tools/idf_tools.py install-python-env --features pytest,ci + - bash install.sh --enable-ci --enable-pytest parallel: matrix: - GEO: [ 'shiny', 'brew' ] @@ -32,6 +29,7 @@ upload-pip-cache: upload-submodules-cache: extends: - .upload_cache_template + - .before_script_minimal - .rules:patterns:submodule tags: - $GEO @@ -41,12 +39,9 @@ upload-submodules-cache: paths: - .cache/submodule_archives policy: push - before_script: [] script: - - source tools/ci/utils.sh - - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS - - source tools/ci/setup_python.sh - rm -rf .cache/submodule_archives # clear old submodule archives + - add_gitlab_ssh_keys - fetch_submodules parallel: matrix: diff --git a/tools/ci/exclude_check_tools_files.txt b/tools/ci/exclude_check_tools_files.txt index 168a8ee00d..ceaa7f173e 100644 --- a/tools/ci/exclude_check_tools_files.txt +++ b/tools/ci/exclude_check_tools_files.txt @@ -29,7 +29,6 @@ tools/ci/mypy_ignore_list.txt tools/ci/normalize_clangtidy_path.py tools/ci/push_to_github.sh tools/ci/python_packages/wifi_tools.py -tools/ci/setup_python.sh tools/ci/utils.sh tools/eclipse-code-style.xml tools/format-minimal.sh diff --git a/tools/ci/setup_python.sh b/tools/ci/setup_python.sh deleted file mode 100644 index c85924e087..0000000000 --- a/tools/ci/setup_python.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env bash - -if [ -z ${PYTHON_VER+x} ]; then - # Use this version of the Python interpreter if it was not defined before. - # 3.6.13 is the default python3 interpreter in esp32-ci-env - # Jobs which doesn't support this version should define PYTHON_VER themselves - PYTHON_VER=3.7.10 -fi - -if [ -f /opt/pyenv/activate ]; -then - source /opt/pyenv/activate - pyenv global $PYTHON_VER || { - echo 'Python' $PYTHON_VER 'is not installed.' - INSTALLED_PY_VERS=$(pyenv versions --bare) - - while [ ${#PYTHON_VER} -gt 0 ] - do - echo 'Tring to locate a match for' $PYTHON_VER - - for ver in ${INSTALLED_PY_VERS[@]} - do - if [[ $ver == $PYTHON_VER* ]]; - then - pyenv global $ver - break 2 - fi - done - - # Removing last character and trying to find some match. - # For example, if 3.4.8 was selected but isn't installed then it will try to - # find some other installed 3.4.X version, and then some 3.X.X version. - PYTHON_VER=${PYTHON_VER: : -1} - done - } - python --version || { - echo 'No matching Python interpreter is found!' - exit 1 - } -elif command -v python -V 1>/dev/null 2>&1; -then - python --version - echo 'No /opt/pyenv/activate exists and Python from path is used.' -else - echo 'No /opt/pyenv/activate exists and no Python interpreter is found!' - exit 1 -fi - -# add esp-idf local package path to PYTHONPATH so it can be imported directly -export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/components/partition_table:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH" From b88c6f09519f2684b920930c898404cd6685e763 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Tue, 16 Aug 2022 11:19:17 +0800 Subject: [PATCH 3/4] ci: test_idf_tools use system python --- .gitlab/ci/host-test.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index 2de277fbd5..e4a6d055a0 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -204,19 +204,24 @@ test_idf_py: - cd ${IDF_PATH}/tools/test_idf_py - ./test_idf_py.py +# Test for create virtualenv. It must be invoked from Python, not from virtualenv. +# Use docker image system python without any extra dependencies test_idf_tools: - extends: .host_test_template + extends: + - .host_test_template + - .before_script_minimal artifacts: when: on_failure paths: - tools/tools.new.json expire_in: 1 week + image: + name: $ESP_ENV_IMAGE + entrypoint: [""] # use system python3. no extra pip package installed script: - # Remove Xtensa and ULP toolchains from the PATH, tests will expect a clean environment - - export PATH=$(p=$(echo $PATH | tr ":" "\n" | grep -v "/root/.espressif/tools\|/opt/espressif" | tr "\n" ":"); echo ${p%:}) - cd ${IDF_PATH}/tools/test_idf_tools - - ./test_idf_tools.py - # Test for create virtualenv. It must be invoked from Python, not from virtualenv. + - python3 -m pip install jsonschema + - python3 ./test_idf_tools.py - python3 ./test_idf_tools_python_env.py .test_efuse_table_on_host_template: From 382fd801c7a1ef8b08c858dc858037a0359c9921 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Tue, 16 Aug 2022 18:56:08 +0800 Subject: [PATCH 4/4] ci: remove check_python_style since flake8 is included in pre-commit --- .gitlab/ci/pre_check.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.gitlab/ci/pre_check.yml b/.gitlab/ci/pre_check.yml index b341cae523..dd0180d60f 100644 --- a/.gitlab/ci/pre_check.yml +++ b/.gitlab/ci/pre_check.yml @@ -41,18 +41,6 @@ check_rom_api_header: - tools/ci/check_examples_rom_header.sh - tools/ci/check_api_violation.sh -check_python_style: - extends: - - .pre_check_template - - .rules:patterns:python-files - artifacts: - when: on_failure - paths: - - flake8_output.txt - expire_in: 1 week - script: - - python -m flake8 --config=$IDF_PATH/.flake8 --output-file=flake8_output.txt --tee --benchmark $IDF_PATH - test_check_kconfigs: extends: .pre_check_template artifacts: