From 30f098cd4956115466a686602be708f8ae186fe7 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Sat, 29 Jan 2022 12:09:04 +0800 Subject: [PATCH] ci: add build_non_test_component_apps --- .gitlab/ci/build.yml | 9 ++++++++ .gitlab/ci/pre_check.yml | 4 ++++ .../python_packages/ttfw_idf/CIScanTests.py | 23 +++++++++++++++++++ tools/ci/utils.sh | 7 ++++++ tools/find_build_apps/cmake.py | 1 + 5 files changed, 44 insertions(+) diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index 6bd7d56360..60d5544376 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -93,6 +93,15 @@ build_pytest_components_esp32c3: script: - run_cmd python tools/ci/build_pytest_apps.py components --target esp32c3 --size-info $SIZE_INFO_LOCATION -vv +build_non_test_components_apps: + extends: + - .build_template + - .build_test_apps_template + variables: + IDF_TARGET: all + TEST_PREFIX: component_ut + TEST_RELATIVE_DIR: component_ut + .build_template_app_template: extends: .build_template variables: diff --git a/.gitlab/ci/pre_check.yml b/.gitlab/ci/pre_check.yml index d2408fadd9..abab8b035f 100644 --- a/.gitlab/ci/pre_check.yml +++ b/.gitlab/ci/pre_check.yml @@ -151,11 +151,13 @@ scan_tests: paths: - $EXAMPLE_TEST_OUTPUT_DIR - $TEST_APPS_OUTPUT_DIR + - $COMPONENT_UT_OUTPUT_DIR variables: EXAMPLE_TEST_DIR: ${CI_PROJECT_DIR}/examples EXAMPLE_TEST_OUTPUT_DIR: ${CI_PROJECT_DIR}/examples/test_configs TEST_APPS_TEST_DIR: ${CI_PROJECT_DIR}/tools/test_apps TEST_APPS_OUTPUT_DIR: ${CI_PROJECT_DIR}/tools/test_apps/test_configs + COMPONENT_UT_OUTPUT_DIR: ${CI_PROJECT_DIR}/component_ut/test_configs CI_SCAN_TESTS_PY: ${CI_PROJECT_DIR}/tools/ci/python_packages/ttfw_idf/CIScanTests.py EXTRA_TEST_DIRS: >- examples/bluetooth/esp_ble_mesh/ble_mesh_console @@ -164,6 +166,8 @@ scan_tests: script: - run_cmd python $CI_SCAN_TESTS_PY example_test $EXAMPLE_TEST_DIR -b cmake --exclude examples/build_system/idf_as_lib -c $CI_TARGET_TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR --extra_test_dirs $EXTRA_TEST_DIRS - run_cmd python $CI_SCAN_TESTS_PY test_apps $TEST_APPS_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_APPS_OUTPUT_DIR + - set_component_ut_vars + - run_cmd python $CI_SCAN_TESTS_PY component_ut $COMPONENT_UT_DIRS --exclude $COMPONENT_UT_EXCLUDES -c $CI_TARGET_TEST_CONFIG_FILE -o $COMPONENT_UT_OUTPUT_DIR --combine-all-targets --except-targets linux # For release tag pipelines only, make sure the tag was created with 'git tag -a' so it will update # the version returned by 'git describe' diff --git a/tools/ci/python_packages/ttfw_idf/CIScanTests.py b/tools/ci/python_packages/ttfw_idf/CIScanTests.py index 5ad3563528..20ed6655c0 100644 --- a/tools/ci/python_packages/ttfw_idf/CIScanTests.py +++ b/tools/ci/python_packages/ttfw_idf/CIScanTests.py @@ -97,6 +97,10 @@ def main(): # type: () -> None help='add this flag to preserve artifacts for all apps') parser.add_argument('--build-all', action='store_true', help='add this flag to build all apps') + parser.add_argument('--combine-all-targets', action='store_true', + help='add this flag to combine all target jsons into one') + parser.add_argument('--except-targets', nargs='+', + help='only useful when "--combine-all-targets". Specified targets would be skipped.') args = parser.parse_args() build_test_case_apps, build_standalone_apps = _judge_build_or_not(args.test_type, args.build_all) @@ -170,6 +174,7 @@ def main(): # type: () -> None else: scan_info_dict[target]['standalone_apps'] = set() test_case_apps_preserve_default = True if build_system == 'cmake' else False + output_files = [] for target in SUPPORTED_TARGETS: # get pytest apps paths pytest_app_paths = set() @@ -201,6 +206,24 @@ def main(): # type: () -> None with open(output_path, 'w') as fw: fw.writelines([json.dumps(app) + '\n' for app in apps]) + if args.combine_all_targets: + if (args.except_targets and target not in [t.lower() for t in args.except_targets]) \ + or (not args.except_targets): + output_files.append(output_path) + build_items_total_count += len(build_items) + else: + print(f'skipping combining target {target}') + + if args.combine_all_targets: + scan_all_json = os.path.join(args.output_path, f'scan_all_{build_system}.json') + lines = [] + for file in output_files: + with open(file) as fr: + lines.extend([line for line in fr.readlines() if line.strip()]) + with open(scan_all_json, 'w') as fw: + fw.writelines(lines) + print(f'combined into file: {scan_all_json}') + if __name__ == '__main__': main() diff --git a/tools/ci/utils.sh b/tools/ci/utils.sh index 42b2a83373..426758a957 100644 --- a/tools/ci/utils.sh +++ b/tools/ci/utils.sh @@ -42,6 +42,13 @@ function get_all_submodules() { git config --file .gitmodules --get-regexp path | awk '{ print $2 }' | sed -e 's|$|/**|' | xargs | sed -e 's/ /,/g' } +function set_component_ut_vars() { + local exclude_list_fp="${IDF_PATH}/tools/ci/component_ut_excludes.txt" + export COMPONENT_UT_DIRS=$(find components/ -name test_apps -type d | xargs) + export COMPONENT_UT_EXCLUDES=$([ -r $exclude_list_fp ] && cat $exclude_list_fp | xargs) + echo "exported variables COMPONENT_UT_DIRS, COMPONENT_UT_EXCLUDES" +} + function error() { printf "\033[0;31m%s\n\033[0m" "${1}" >&2 } diff --git a/tools/find_build_apps/cmake.py b/tools/find_build_apps/cmake.py index 610c1bc4c0..5527dc8648 100644 --- a/tools/find_build_apps/cmake.py +++ b/tools/find_build_apps/cmake.py @@ -63,6 +63,7 @@ class CMakeBuildSystem(BuildSystem): build_stderr = log_file try: + os.environ['IDF_TARGET'] = build_item.target subprocess.check_call(args, stdout=build_stdout, stderr=build_stderr) except subprocess.CalledProcessError as e: raise BuildError('Build failed with exit code {}'.format(e.returncode))