diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c699b55956..a32a4e2de4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -118,6 +118,7 @@ before_script: include: - '/tools/ci/config/rules.yml' - '/tools/ci/config/docs.yml' + - '/tools/ci/config/static-code-analysis.yml' - '/tools/ci/config/pre_check.yml' - '/tools/ci/config/build.yml' - '/tools/ci/config/assign-test.yml' diff --git a/tools/ci/config/build.yml b/tools/ci/config/build.yml index aa3961c6f1..0537bfa83f 100644 --- a/tools/ci/config/build.yml +++ b/tools/ci/config/build.yml @@ -387,92 +387,3 @@ build_template_app: - job: fast_template_app artifacts: false extends: .build_template_app_template - -# Sonarqube related jobs put here for this reason: -# Here we have two jobs. code_quality_check and code_quality_report. -# -# code_quality_check will analyze the code changes between your MR and -# code repo stored in sonarqube server. The analysis result is only shown in -# the comments under this MR and won't be transferred to the server. -# -# code_quality_report will analyze and transfer both of the newly added code -# and the analysis result to the server. -# -# Put in the front to ensure that the newly merged code can be stored in -# sonarqube server ASAP, in order to avoid reporting unrelated code issues -.sonar_scan_template: - stage: build - image: - name: $CI_DOCKER_REGISTRY/sonarqube-scanner:2 - before_script: - - source tools/ci/utils.sh - - export PYTHONPATH="$CI_PROJECT_DIR/tools:$CI_PROJECT_DIR/tools/ci/python_packages:$PYTHONPATH" - - fetch_submodules - # Exclude the submodules, all paths ends with /** - - export SUBMODULES=$(get_all_submodules) - # get all exclude paths specified in tools/ci/sonar_exclude_list.txt | ignore lines start with # | xargs | replace all to - - export CUSTOM_EXCLUDES=$(cat $CI_PROJECT_DIR/tools/ci/sonar_exclude_list.txt | grep -v '^#' | xargs | sed -e 's/ /,/g') - # Exclude the report dir - - export EXCLUSIONS="$SUBMODULES,$REPORT_DIR/**,docs/_static/**,**/*.png,**/*.jpg" - - python $NORMALIZE_CLANGTIDY_PY $CI_PROJECT_DIR/$REPORT_DIR/warnings.txt $CI_PROJECT_DIR/$REPORT_DIR/clang_tidy_report.txt $CI_PROJECT_DIR - variables: - GIT_DEPTH: 0 - NORMALIZE_CLANGTIDY_PY: $CI_PROJECT_DIR/tools/ci/normalize_clangtidy_path.py - REPORT_DIR: examples/get-started/hello_world/tidybuild/report - tags: - - host_test - dependencies: # set dependencies to null to avoid missing artifacts issue - needs: - - clang_tidy_check_regular - -code_quality_check: - extends: - - .sonar_scan_template - - .rules:trigger - allow_failure: true - script: - - export CI_MR_IID=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py id ${CI_COMMIT_BRANCH}) - - export CI_MR_COMMITS=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py commits ${CI_COMMIT_BRANCH} | tr '\n' ',') - # test if this branch have merge request, if not, exit 0 - - test -n "$CI_MR_IID" || exit 0 - - test -n "$CI_MR_COMMITS" || exit 0 - - sonar-scanner - -Dsonar.analysis.mode=preview - -Dsonar.host.url=$SONAR_HOST_URL - -Dsonar.login=$SONAR_LOGIN - -Dsonar.sources=$CI_PROJECT_DIR - -Dsonar.sourceEncoding=UTF-8 - -Dsonar.projectKey=esp-idf - -Dsonar.projectBaseDir=$CI_PROJECT_DIR - -Dsonar.exclusions=$EXCLUSIONS - -Dsonar.gitlab.project_id=$CI_PROJECT_ID - -Dsonar.gitlab.commit_sha=$CI_MR_COMMITS - -Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME - -Dsonar.gitlab.failure_notification_mode=exit-code - -Dsonar.cxx.clangtidy.reportPath=$REPORT_DIR/clang_tidy_report.txt - -Dsonar.cxx.includeDirectories=components,/usr/include - -Dsonar.python.pylint_config=.pylintrc - -Dsonar.gitlab.ci_merge_request_iid=$CI_MR_IID - -Dsonar.gitlab.merge_request_discussion=true - -Dsonar.branch.name=$CI_COMMIT_REF_NAME - -code_quality_report: - extends: - - .sonar_scan_template - - .rules:protected-schedule - script: - - sonar-scanner - -Dsonar.host.url=$SONAR_HOST_URL - -Dsonar.login=$SONAR_LOGIN - -Dsonar.sources=$CI_PROJECT_DIR - -Dsonar.sourceEncoding=UTF-8 - -Dsonar.projectKey=esp-idf - -Dsonar.projectBaseDir=$CI_PROJECT_DIR - -Dsonar.exclusions=$EXCLUSIONS - -Dsonar.gitlab.project_id=$CI_PROJECT_ID - -Dsonar.gitlab.commit_sha=$CI_COMMIT_SHA - -Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME - -Dsonar.cxx.clangtidy.reportPath=$REPORT_DIR/clang_tidy_report.txt - -Dsonar.cxx.includeDirectories=components,/usr/include - -Dsonar.python.pylint_config=.pylintrc - -Dsonar.branch.name=$CI_COMMIT_REF_NAME diff --git a/tools/ci/config/deploy.yml b/tools/ci/config/deploy.yml index ddf2b9bb84..e709dcb72b 100644 --- a/tools/ci/config/deploy.yml +++ b/tools/ci/config/deploy.yml @@ -4,42 +4,6 @@ tags: - deploy -.clang_tidy_deploy_template: - extends: .deploy_job_template - tags: - - deploy - - shiny - script: - - add_doc_server_ssh_keys $DOCS_DEPLOY_KEY $DOCS_SERVER $DOCS_SERVER_USER - - export GIT_VER=$(git describe --always) - - cd $IDF_PATH/examples/get-started/hello_world/tidybuild - - mv report $GIT_VER - - tar czvf $GIT_VER.tar.gz $GIT_VER - - export STATIC_REPORT_PATH="web/static_analysis/esp-idf/" - - ssh $DOCS_SERVER -x "mkdir -p $STATIC_REPORT_PATH/clang-tidy" - - scp $GIT_VER.tar.gz $DOCS_SERVER:$STATIC_REPORT_PATH/clang-tidy - - ssh $DOCS_SERVER -x "cd $STATIC_REPORT_PATH/clang-tidy && tar xzvf $GIT_VER.tar.gz && rm -f latest && ln -s $GIT_VER latest" - # add link to view the report - - echo "[static analysis][clang tidy] $CI_DOCKER_REGISTRY/static_analysis/esp-idf/clang-tidy/${GIT_VER}/index.html" - - test ! -e ${GIT_VER}/FAILED_RULES || { echo 'Failed static analysis rules!'; cat ${GIT_VER}/FAILED_RULES; exit 1; } - -clang_tidy_deploy: - extends: .clang_tidy_deploy_template - # Override default stage to happen before the post_check - stage: test_deploy - needs: - - clang_tidy_check - - clang_tidy_check_all - variables: - BOT_NEEDS_TRIGGER_BY_NAME: 1 - -clang_tidy_deploy_regular: - extends: - - .clang_tidy_deploy_template - - .rules:labels:static_analysis-only - needs: - - clang_tidy_check_regular - push_to_github: extends: - .deploy_job_template diff --git a/tools/ci/config/pre_check.yml b/tools/ci/config/pre_check.yml index c9690dcf23..3b68f1531c 100644 --- a/tools/ci/config/pre_check.yml +++ b/tools/ci/config/pre_check.yml @@ -61,7 +61,9 @@ check_rom_api_header: - tools/ci/check_rom_apis.sh check_python_style: - extends: .pre_check_base_template + extends: + - .pre_check_base_template + - .rules:patterns:python-files artifacts: when: on_failure paths: @@ -137,37 +139,6 @@ scan_tests: - python $CI_SCAN_TESTS_PY test_apps $TEST_APPS_TEST_DIR -c $TEST_CONFIG_FILE -o $TEST_APPS_OUTPUT_DIR - python $CI_SCAN_TESTS_PY component_ut $COMPONENT_UT_DIRS --exclude $COMPONENT_UT_EXCLUDES -c $TEST_CONFIG_FILE -o $COMPONENT_UT_OUTPUT_DIR -.clang_tidy_check_template: - extends: .pre_check_base_template - image: ${CI_DOCKER_REGISTRY}/clang-static-analysis - artifacts: - reports: - junit: $IDF_PATH/output.xml - when: always - paths: - - $IDF_PATH/examples/get-started/hello_world/tidybuild/report/* - expire_in: 1 day - script: - - retry_failed git clone $IDF_ANALYSIS_UTILS static_analysis_utils && cd static_analysis_utils - # Setup parameters of triggered/regular job - - export TRIGGERED_RELATIVE=${BOT_LABEL_STATIC_ANALYSIS-} && export TRIGGERED_ABSOLUTE=${BOT_LABEL_STATIC_ANALYSIS_ALL-} && export TARGET_BRANCH=${BOT_CUSTOMIZED_REVISION-} - - ./analyze.sh $IDF_PATH/examples/get-started/hello_world/ $IDF_PATH/tools/ci/static-analysis-rules.yml $IDF_PATH/output.xml - -clang_tidy_check: - extends: .clang_tidy_check_template - variables: - BOT_NEEDS_TRIGGER_BY_NAME: 1 - BOT_LABEL_STATIC_ANALYSIS: 1 - -clang_tidy_check_regular: - extends: .clang_tidy_check_template - -clang_tidy_check_all: - extends: .clang_tidy_check_template - variables: - BOT_NEEDS_TRIGGER_BY_NAME: 1 - BOT_LABEL_STATIC_ANALYSIS_ALL: 1 - # For release tag pipelines only, make sure the tag was created with 'git tag -a' so it will update # the version returned by 'git describe' check_version_tag: diff --git a/tools/ci/config/rules.yml b/tools/ci/config/rules.yml index 62e05c82b5..3956b03eab 100644 --- a/tools/ci/config/rules.yml +++ b/tools/ci/config/rules.yml @@ -8,6 +8,19 @@ - "components/**/sdkconfig*" - "CONTRIBUTING.rst" +.patterns-c-files: &patterns-c-files + - "**/*.{c,C}" + - "**/*.{h,H}" + +.patterns-python-files: &patterns-python-files + - "**/*.py" + +.patterns-static-code-analysis: &patterns-static-code-analysis + # - "tools/ci/config/static-code-analysis.yml" # FIXME: after debug + - "**/*.{c,C}" + - "**/*.{h,H}" + - "**/*.py" + # if anchors .if-ref-master: &if-ref-master if: '$CI_COMMIT_REF_NAME == "master"' @@ -75,9 +88,6 @@ .if-label-nvs_coverage: &if-label-nvs_coverage if: '$BOT_LABEL_NVS_COVERAGE' -.if-label-static_analysis: &if-label-static_analysis - if: '$BOT_LABEL_STATIC_ANALYSIS || $BOT_LABEL_STATIC_ANALYSIS_ALL' - .if-label-iperf_stress_test: &if-label-iperf_stress_test if: '$BOT_LABEL_IPERF_STRESS_TEST' @@ -128,10 +138,6 @@ - <<: *if-schedule when: always -.rules:labels:static_analysis-only: - rules: - - <<: *if-label-static_analysis - .rules:labels:build: rules: - <<: *if-protected-no_label @@ -151,6 +157,28 @@ - <<: *if-dev-push changes: *patterns-docs +.rules:patterns:clang_tidy: + rules: + - <<: *if-protected-no_label + - <<: *if-dev-push + changes: *patterns-c-files + +.rules:patterns:clang_tidy-preview: + rules: + - <<: *if-dev-push + changes: *patterns-c-files + +.rules:patterns:python-files: + rules: + - <<: *if-protected-no_label + - <<: *if-dev-push + changes: *patterns-python-files + +.rules:patterns:static-code-analysis-preview: + rules: + - <<: *if-dev-push + changes: *patterns-static-code-analysis + .rules:labels:weekend_test-only: rules: - <<: *if-label-weekend_test diff --git a/tools/ci/config/static-code-analysis.yml b/tools/ci/config/static-code-analysis.yml new file mode 100644 index 0000000000..844bad0689 --- /dev/null +++ b/tools/ci/config/static-code-analysis.yml @@ -0,0 +1,163 @@ +.clang_tidy_check_template: + extends: + - .pre_check_base_template + - .rules:patterns:clang_tidy + image: ${CI_DOCKER_REGISTRY}/clang-static-analysis + artifacts: + reports: + junit: $IDF_PATH/output.xml + when: always + paths: + - $IDF_PATH/examples/get-started/hello_world/tidybuild/report/* + expire_in: 1 day + script: + - retry_failed git clone $IDF_ANALYSIS_UTILS static_analysis_utils && cd static_analysis_utils + # Setup parameters of triggered/regular job + - export TARGET_BRANCH=${BOT_CUSTOMIZED_REVISION-} + - ./analyze.sh $IDF_PATH/examples/get-started/hello_world/ $IDF_PATH/tools/ci/static-analysis-rules.yml $IDF_PATH/output.xml + +# pre_check stage +clang_tidy_check: + extends: .clang_tidy_check_template + variables: + BOT_NEEDS_TRIGGER_BY_NAME: 1 + TRIGGERED_RELATIVE: 1 + +clang_tidy_check_regular: + extends: + - .clang_tidy_check_template + - .rules:patterns:clang_tidy-preview + +clang_tidy_check_all: + extends: .clang_tidy_check_template + variables: + BOT_NEEDS_TRIGGER_BY_NAME: 1 + TRIGGERED_ABSOLUTE: 1 + +# build stage +# Sonarqube related jobs put here for this reason: +# Here we have two jobs. code_quality_check and code_quality_report. +# +# code_quality_check will analyze the code changes between your MR and +# code repo stored in sonarqube server. The analysis result is only shown in +# the comments under this MR and won't be transferred to the server. +# +# code_quality_report will analyze and transfer both of the newly added code +# and the analysis result to the server. +# +# Put in the front to ensure that the newly merged code can be stored in +# sonarqube server ASAP, in order to avoid reporting unrelated code issues +.sonar_scan_template: + stage: build + image: + name: $CI_DOCKER_REGISTRY/sonarqube-scanner:2 + before_script: + - source tools/ci/utils.sh + - export PYTHONPATH="$CI_PROJECT_DIR/tools:$CI_PROJECT_DIR/tools/ci/python_packages:$PYTHONPATH" + - fetch_submodules + # Exclude the submodules, all paths ends with /** + - export SUBMODULES=$(get_all_submodules) + # get all exclude paths specified in tools/ci/sonar_exclude_list.txt | ignore lines start with # | xargs | replace all to + - export CUSTOM_EXCLUDES=$(cat $CI_PROJECT_DIR/tools/ci/sonar_exclude_list.txt | grep -v '^#' | xargs | sed -e 's/ /,/g') + # Exclude the report dir + - export EXCLUSIONS="$SUBMODULES,$REPORT_DIR/**,docs/_static/**,**/*.png,**/*.jpg" + - python $NORMALIZE_CLANGTIDY_PY $CI_PROJECT_DIR/$REPORT_DIR/warnings.txt $CI_PROJECT_DIR/$REPORT_DIR/clang_tidy_report.txt $CI_PROJECT_DIR + variables: + GIT_DEPTH: 0 + NORMALIZE_CLANGTIDY_PY: $CI_PROJECT_DIR/tools/ci/normalize_clangtidy_path.py + REPORT_DIR: examples/get-started/hello_world/tidybuild/report + tags: + - host_test + dependencies: # Here is not a hard dependency relationship, could be skipped when only python files changed. so we do not use "needs" here. + - clang_tidy_check_regular + +code_quality_check: + extends: + - .sonar_scan_template + - .rules:patterns:static-code-analysis-preview + allow_failure: true + script: + - export CI_MR_IID=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py id ${CI_COMMIT_BRANCH}) + - export CI_MR_COMMITS=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py commits ${CI_COMMIT_BRANCH} | tr '\n' ',') + # test if this branch have merge request, if not, exit 0 + - test -n "$CI_MR_IID" || exit 0 + - test -n "$CI_MR_COMMITS" || exit 0 + - sonar-scanner + -Dsonar.analysis.mode=preview + -Dsonar.host.url=$SONAR_HOST_URL + -Dsonar.login=$SONAR_LOGIN + -Dsonar.sources=$CI_PROJECT_DIR + -Dsonar.sourceEncoding=UTF-8 + -Dsonar.projectKey=esp-idf + -Dsonar.projectBaseDir=$CI_PROJECT_DIR + -Dsonar.exclusions=$EXCLUSIONS + -Dsonar.gitlab.project_id=$CI_PROJECT_ID + -Dsonar.gitlab.commit_sha=$CI_MR_COMMITS + -Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME + -Dsonar.gitlab.failure_notification_mode=exit-code + -Dsonar.cxx.clangtidy.reportPath=$REPORT_DIR/clang_tidy_report.txt + -Dsonar.cxx.includeDirectories=components,/usr/include + -Dsonar.python.pylint_config=.pylintrc + -Dsonar.gitlab.ci_merge_request_iid=$CI_MR_IID + -Dsonar.gitlab.merge_request_discussion=true + -Dsonar.branch.name=$CI_COMMIT_REF_NAME + +code_quality_report: + extends: + - .sonar_scan_template + - .rules:protected-schedule + script: + - sonar-scanner + -Dsonar.host.url=$SONAR_HOST_URL + -Dsonar.login=$SONAR_LOGIN + -Dsonar.sources=$CI_PROJECT_DIR + -Dsonar.sourceEncoding=UTF-8 + -Dsonar.projectKey=esp-idf + -Dsonar.projectBaseDir=$CI_PROJECT_DIR + -Dsonar.exclusions=$EXCLUSIONS + -Dsonar.gitlab.project_id=$CI_PROJECT_ID + -Dsonar.gitlab.commit_sha=$CI_COMMIT_SHA + -Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME + -Dsonar.cxx.clangtidy.reportPath=$REPORT_DIR/clang_tidy_report.txt + -Dsonar.cxx.includeDirectories=components,/usr/include + -Dsonar.python.pylint_config=.pylintrc + -Dsonar.branch.name=$CI_COMMIT_REF_NAME + +# deploy stage +.clang_tidy_deploy_template: + extends: + - .deploy_job_template + - .rules:patterns:clang_tidy + tags: + - deploy + - shiny + script: + - add_doc_server_ssh_keys $DOCS_DEPLOY_KEY $DOCS_SERVER $DOCS_SERVER_USER + - export GIT_VER=$(git describe --always) + - cd $IDF_PATH/examples/get-started/hello_world/tidybuild + - mv report $GIT_VER + - tar czvf $GIT_VER.tar.gz $GIT_VER + - export STATIC_REPORT_PATH="web/static_analysis/esp-idf/" + - ssh $DOCS_SERVER -x "mkdir -p $STATIC_REPORT_PATH/clang-tidy" + - scp $GIT_VER.tar.gz $DOCS_SERVER:$STATIC_REPORT_PATH/clang-tidy + - ssh $DOCS_SERVER -x "cd $STATIC_REPORT_PATH/clang-tidy && tar xzvf $GIT_VER.tar.gz && rm -f latest && ln -s $GIT_VER latest" + # add link to view the report + - echo "[static analysis][clang tidy] $CI_DOCKER_REGISTRY/static_analysis/esp-idf/clang-tidy/${GIT_VER}/index.html" + - test ! -e ${GIT_VER}/FAILED_RULES || { echo 'Failed static analysis rules!'; cat ${GIT_VER}/FAILED_RULES; exit 1; } + +clang_tidy_deploy: + extends: .clang_tidy_deploy_template + # Override default stage to happen before the post_check + stage: test_deploy + needs: + - clang_tidy_check + - clang_tidy_check_all + variables: + BOT_NEEDS_TRIGGER_BY_NAME: 1 + +clang_tidy_deploy_regular: + extends: + - .clang_tidy_deploy_template + - .rules:patterns:clang_tidy-preview + needs: + - clang_tidy_check_regular diff --git a/tools/ci/normalize_clangtidy_path.py b/tools/ci/normalize_clangtidy_path.py index 9f7a26df5a..efbae31dc0 100755 --- a/tools/ci/normalize_clangtidy_path.py +++ b/tools/ci/normalize_clangtidy_path.py @@ -1,12 +1,17 @@ #!/usr/bin/env python import argparse import re -from os.path import dirname, join, normpath, relpath +from os.path import join, normpath, dirname, relpath, exists CLANG_TIDY_REGEX = re.compile(r'(.+|[a-zA-Z]:\\\\.+):([0-9]+):([0-9]+): ([^:]+): (.+)') def normalize_clang_tidy_path(file_path, output_path, base_dir): + if not exists(file_path): + print('Skipping normalizing. This could only happen when skipping clang-tidy check ' + 'because of no c file modified. Please double check') + return + with open(output_path, 'w') as fw: for line in open(file_path): result = CLANG_TIDY_REGEX.match(line)