# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 """This file is used for generating the child pipeline for build jobs.""" import argparse import os import typing as t import __init__ # noqa: F401 # inject the system path import yaml from dynamic_pipelines.constants import (DEFAULT_APPS_BUILD_PER_JOB, DEFAULT_BUILD_CHILD_PIPELINE_FILEPATH, DEFAULT_TEST_PATHS, NON_TEST_RELATED_APPS_FILENAME, NON_TEST_RELATED_BUILD_JOB_NAME, TEST_RELATED_APPS_FILENAME, TEST_RELATED_BUILD_JOB_NAME) from dynamic_pipelines.models import BuildJob, EmptyJob from dynamic_pipelines.utils import dump_jobs_to_yaml from idf_build_apps.utils import semicolon_separated_str_to_list from idf_ci.app import dump_apps_to_txt from idf_ci_utils import IDF_PATH from idf_pytest.constants import DEFAULT_CONFIG_RULES_STR, DEFAULT_FULL_BUILD_TEST_FILEPATTERNS, CollectMode from idf_pytest.script import get_all_apps def main(arguments: argparse.Namespace) -> None: # load from default build test rules config file extra_default_build_targets: t.List[str] = [] if arguments.default_build_test_rules: with open(arguments.default_build_test_rules) as fr: configs = yaml.safe_load(fr) if configs: extra_default_build_targets = configs.get('extra_default_build_targets') or [] build_jobs = [] ########################################### # special case with -k, ignore other args # ########################################### if arguments.filter_expr: # build only test related apps test_related_apps, _ = get_all_apps( arguments.paths, target=CollectMode.ALL, config_rules_str=DEFAULT_CONFIG_RULES_STR, filter_expr=arguments.filter_expr, marker_expr='not host_test', extra_default_build_targets=extra_default_build_targets, ) dump_apps_to_txt(sorted(test_related_apps), TEST_RELATED_APPS_FILENAME) print(f'Generate test related apps file {TEST_RELATED_APPS_FILENAME} with {len(test_related_apps)} apps') test_apps_build_job = BuildJob( name=TEST_RELATED_BUILD_JOB_NAME, parallel=len(test_related_apps) // DEFAULT_APPS_BUILD_PER_JOB + 1, variables={ 'APP_LIST_FILE': TEST_RELATED_APPS_FILENAME, }, ) build_jobs.append(test_apps_build_job) else: ############# # all cases # ############# test_related_apps, non_test_related_apps = get_all_apps( arguments.paths, CollectMode.ALL, marker_expr='not host_test', config_rules_str=DEFAULT_CONFIG_RULES_STR, extra_default_build_targets=extra_default_build_targets, modified_components=arguments.modified_components, modified_files=arguments.modified_files, ignore_app_dependencies_filepatterns=arguments.ignore_app_dependencies_filepatterns, ) dump_apps_to_txt(sorted(test_related_apps), TEST_RELATED_APPS_FILENAME) print(f'Generate test related apps file {TEST_RELATED_APPS_FILENAME} with {len(test_related_apps)} apps') dump_apps_to_txt(sorted(non_test_related_apps), NON_TEST_RELATED_APPS_FILENAME) print( f'Generate non-test related apps file {NON_TEST_RELATED_APPS_FILENAME} with {len(non_test_related_apps)} apps' ) if test_related_apps: test_apps_build_job = BuildJob( name=TEST_RELATED_BUILD_JOB_NAME, parallel=len(test_related_apps) // DEFAULT_APPS_BUILD_PER_JOB + 1, variables={ 'APP_LIST_FILE': TEST_RELATED_APPS_FILENAME, }, ) build_jobs.append(test_apps_build_job) if non_test_related_apps: non_test_apps_build_job = BuildJob( name=NON_TEST_RELATED_BUILD_JOB_NAME, parallel=len(non_test_related_apps) // DEFAULT_APPS_BUILD_PER_JOB + 1, variables={ 'APP_LIST_FILE': NON_TEST_RELATED_APPS_FILENAME, }, ) build_jobs.append(non_test_apps_build_job) # check if there's no jobs if not build_jobs: print('No apps need to be built. Create one empty job instead') build_jobs.append(EmptyJob()) extra_include_yml = [] else: extra_include_yml = ['tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml'] dump_jobs_to_yaml(build_jobs, arguments.yaml_output, extra_include_yml) print(f'Generate child pipeline yaml file {arguments.yaml_output} with {sum(j.parallel for j in build_jobs)} jobs') if __name__ == '__main__': parser = argparse.ArgumentParser( description='Generate build child pipeline', formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) parser.add_argument( '-o', '--yaml-output', default=DEFAULT_BUILD_CHILD_PIPELINE_FILEPATH, help='Output YAML path', ) parser.add_argument( '-p', '--paths', nargs='+', default=DEFAULT_TEST_PATHS, help='Paths to the apps to build.', ) parser.add_argument( '-k', '--filter-expr', help='only build tests matching given filter expression. For example: -k "test_hello_world". Works only' 'for pytest', ) parser.add_argument( '--default-build-test-rules', default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'), help='default build test rules config file', ) parser.add_argument( '--modified-components', type=semicolon_separated_str_to_list, help='semicolon-separated string which specifies the modified components. ' 'app with `depends_components` set in the corresponding manifest files would only be built ' 'if depends on any of the specified components. ' 'If set to "", the value would be considered as None. ' 'If set to ";", the value would be considered as an empty list', ) parser.add_argument( '--modified-files', type=semicolon_separated_str_to_list, help='semicolon-separated string which specifies the modified files. ' 'app with `depends_filepatterns` set in the corresponding manifest files would only be built ' 'if any of the specified file pattern matches any of the specified modified files. ' 'If set to "", the value would be considered as None. ' 'If set to ";", the value would be considered as an empty list', ) parser.add_argument( '-if', '--ignore-app-dependencies-filepatterns', type=semicolon_separated_str_to_list, help='semicolon-separated string which specifies the file patterns used for ' 'ignoring checking the app dependencies. ' 'The `depends_components` and `depends_filepatterns` set in the manifest files will be ignored ' 'when any of the specified file patterns matches any of the modified files. ' 'Must be used together with --modified-files. ' 'If set to "", the value would be considered as None. ' 'If set to ";", the value would be considered as an empty list', ) args = parser.parse_args() if os.getenv('IS_MR_PIPELINE') == '0' or os.getenv('BUILD_AND_TEST_ALL_APPS') == '1': print('Build and run all test cases, and compile all cmake apps') args.modified_components = None args.modified_files = None args.ignore_app_dependencies_filepatterns = None elif args.filter_expr is not None: print('Build and run only test cases matching "%s"' % args.filter_expr) args.modified_components = None args.modified_files = None args.ignore_app_dependencies_filepatterns = None else: print('Build and run only test cases matching the modified components and files') if args.modified_files and not args.ignore_app_dependencies_filepatterns: # setting default values args.ignore_app_dependencies_filepatterns = DEFAULT_FULL_BUILD_TEST_FILEPATTERNS main(args)