diff --git a/tools/ci/config/target-test.yml b/tools/ci/config/target-test.yml index ded90292d6..0a3fb1ef55 100644 --- a/tools/ci/config/target-test.yml +++ b/tools/ci/config/target-test.yml @@ -21,7 +21,6 @@ - $BOT_LABEL_EXAMPLE_TEST dependencies: - assign_test - - build_examples_cmake_esp32 artifacts: when: always paths: @@ -34,6 +33,7 @@ CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/examples/test_configs" LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS" ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml" + GIT_SUBMODULE_STRATEGY: none script: - *define_config_file_name # first test if config file exists, if not exist, exit 0 diff --git a/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py b/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py index 85beb1f086..ed27c58eb3 100644 --- a/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py +++ b/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py @@ -18,14 +18,23 @@ Command line tool to assign example tests to CI test jobs. # TODO: Need to handle running examples on different chips import os -import sys import re import argparse +import json import gitlab_api from tiny_test_fw.Utility import CIAssignTest +EXAMPLE_BUILD_JOB_NAMES = ["build_examples_cmake_esp32", "build_examples_cmake_esp32s2"] +IDF_PATH_FROM_ENV = os.getenv("IDF_PATH") +if IDF_PATH_FROM_ENV: + ARTIFACT_INDEX_FILE = os.path.join(IDF_PATH_FROM_ENV, + "build_examples", "artifact_index.json") +else: + ARTIFACT_INDEX_FILE = "artifact_index.json" + + class ExampleGroup(CIAssignTest.Group): SORT_KEYS = CI_JOB_MATCH_KEYS = ["env_tag", "chip"] @@ -34,15 +43,33 @@ class CIExampleAssignTest(CIAssignTest.AssignTest): CI_TEST_JOB_PATTERN = re.compile(r"^example_test_.+") -class ArtifactFile(object): - def __init__(self, project_id, job_name, artifact_file_path): - self.gitlab_api = gitlab_api.Gitlab(project_id) +def create_artifact_index_file(project_id=None, pipeline_id=None): + if project_id is None: + project_id = os.getenv("CI_PROJECT_ID") + if pipeline_id is None: + pipeline_id = os.getenv("CI_PIPELINE_ID") + gitlab_inst = gitlab_api.Gitlab(project_id) + artifact_index_list = [] - def process(self): + def format_build_log_path(): + return "build_examples/list_job_{}.json".format(job_info["parallel_num"]) + + for build_job_name in EXAMPLE_BUILD_JOB_NAMES: + job_info_list = gitlab_inst.find_job_id(build_job_name, pipeline_id=pipeline_id) + for job_info in job_info_list: + raw_data = gitlab_inst.download_artifact(job_info["id"], [format_build_log_path()])[0] + build_info_list = [json.loads(line) for line in raw_data.splitlines()] + for build_info in build_info_list: + build_info["ci_job_id"] = job_info["id"] + artifact_index_list.append(build_info) + try: + os.makedirs(os.path.dirname(ARTIFACT_INDEX_FILE)) + except OSError: + # already created pass - def output(self): - pass + with open(ARTIFACT_INDEX_FILE, "w") as f: + json.dump(artifact_index_list, f) if __name__ == '__main__': @@ -53,8 +80,11 @@ if __name__ == '__main__': help="gitlab ci config file") parser.add_argument("output_path", help="output path of config files") + parser.add_argument("--pipeline_id", "-p", type=int, default=None, + help="pipeline_id") args = parser.parse_args() assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file, case_group=ExampleGroup) assign_test.assign_cases() assign_test.output_configs(args.output_path) + create_artifact_index_file() diff --git a/tools/ci/python_packages/ttfw_idf/IDFApp.py b/tools/ci/python_packages/ttfw_idf/IDFApp.py index 385af49c2c..c17072ca82 100644 --- a/tools/ci/python_packages/ttfw_idf/IDFApp.py +++ b/tools/ci/python_packages/ttfw_idf/IDFApp.py @@ -17,7 +17,121 @@ import subprocess import os import json + from tiny_test_fw import App +from . import CIAssignExampleTest + +try: + import gitlab_api +except ImportError: + gitlab_api = None + + +def parse_flash_settings(path): + file_name = os.path.basename(path) + if file_name == "flasher_args.json": + # CMake version using build metadata file + with open(path, "r") as f: + args = json.load(f) + flash_files = [(offs, binary) for (offs, binary) in args["flash_files"].items() if offs != ""] + flash_settings = args["flash_settings"] + app_name = os.path.splitext(args["app"]["file"])[0] + else: + # GNU Make version uses download.config arguments file + with open(path, "r") as f: + args = f.readlines()[-1].split(" ") + flash_files = [] + flash_settings = {} + for idx in range(0, len(args), 2): # process arguments in pairs + if args[idx].startswith("--"): + # strip the -- from the command line argument + flash_settings[args[idx][2:]] = args[idx + 1] + else: + # offs, filename + flash_files.append((args[idx], args[idx + 1])) + # we can only guess app name in download.config. + for p in flash_files: + if not os.path.dirname(p[1]) and "partition" not in p[1]: + # app bin usually in the same dir with download.config and it's not partition table + app_name = os.path.splitext(p[1])[0] + break + else: + app_name = None + return flash_files, flash_settings, app_name + + +class Artifacts(object): + def __init__(self, dest_root_path, artifact_index_file, app_path, config_name, target): + assert gitlab_api + # at least one of app_path or config_name is not None. otherwise we can't match artifact + assert app_path or config_name + assert os.path.exists(artifact_index_file) + self.gitlab_inst = gitlab_api.Gitlab(os.getenv("CI_PROJECT_ID")) + self.dest_root_path = dest_root_path + with open(artifact_index_file, "r") as f: + artifact_index = json.load(f) + self.artifact_info = self._find_artifact(artifact_index, app_path, config_name, target) + + @staticmethod + def _find_artifact(artifact_index, app_path, config_name, target): + for artifact_info in artifact_index: + match_result = True + if app_path: + match_result = app_path in artifact_info["app_dir"] + if config_name: + match_result = match_result and config_name == artifact_info["config"] + if target: + match_result = match_result and target == artifact_info["target"] + if match_result: + ret = artifact_info + break + else: + ret = None + return ret + + def download_artifacts(self): + if self.artifact_info: + base_path = os.path.join(self.artifact_info["work_dir"], self.artifact_info["build_dir"]) + job_id = self.artifact_info["ci_job_id"] + + # 1. download flash args file + if self.artifact_info["build_system"] == "cmake": + flash_arg_file = os.path.join(base_path, "flasher_args.json") + else: + flash_arg_file = os.path.join(base_path, "download.config") + + self.gitlab_inst.download_artifact(job_id, [flash_arg_file], self.dest_root_path) + + # 2. download all binary files + flash_files, flash_settings, app_name = parse_flash_settings(os.path.join(self.dest_root_path, + flash_arg_file)) + artifact_files = [os.path.join(base_path, p[1]) for p in flash_files] + artifact_files.append(os.path.join(base_path, app_name + ".elf")) + + self.gitlab_inst.download_artifact(job_id, artifact_files, self.dest_root_path) + + # 3. download sdkconfig file + self.gitlab_inst.download_artifact(job_id, [os.path.join(os.path.dirname(base_path), "sdkconfig")], + self.dest_root_path) + else: + base_path = None + return base_path + + def download_artifact_files(self, file_names): + if self.artifact_info: + base_path = os.path.join(self.artifact_info["work_dir"], self.artifact_info["build_dir"]) + job_id = self.artifact_info["ci_job_id"] + + # download all binary files + artifact_files = [os.path.join(base_path, fn) for fn in file_names] + self.gitlab_inst.download_artifact(job_id, artifact_files, self.dest_root_path) + + # download sdkconfig file + self.gitlab_inst.download_artifact(job_id, [os.path.join(os.path.dirname(base_path), "sdkconfig")], + self.dest_root_path) + else: + base_path = None + return base_path class IDFApp(App.BaseApp): @@ -34,7 +148,7 @@ class IDFApp(App.BaseApp): self.config_name = config_name self.target = target self.idf_path = self.get_sdk_path() - self.binary_path = self.get_binary_path(app_path, config_name) + self.binary_path = self.get_binary_path(app_path, config_name, target) self.elf_file = self._get_elf_file_path(self.binary_path) assert os.path.exists(self.binary_path) if self.IDF_DOWNLOAD_CONFIG_FILE not in os.listdir(self.binary_path): @@ -52,6 +166,7 @@ class IDFApp(App.BaseApp): @classmethod def get_sdk_path(cls): + # type: () -> str idf_path = os.getenv("IDF_PATH") assert idf_path assert os.path.exists(idf_path) @@ -69,7 +184,6 @@ class IDFApp(App.BaseApp): """ reads sdkconfig and returns a dictionary with all configuredvariables - :param sdkconfig_file: location of sdkconfig :raise: AssertionError: if sdkconfig file does not exist in defined paths """ d = {} @@ -86,14 +200,16 @@ class IDFApp(App.BaseApp): d[configs[0]] = configs[1].rstrip() return d - def get_binary_path(self, app_path, config_name=None): + def get_binary_path(self, app_path, config_name=None, target=None): + # type: (str, str, str) -> str """ get binary path according to input app_path. subclass must overwrite this method. :param app_path: path of application - :param config_name: name of the application build config + :param config_name: name of the application build config. Will match any config if None + :param target: target name. Will match for target if None :return: abs app binary path """ pass @@ -120,24 +236,12 @@ class IDFApp(App.BaseApp): if self.IDF_FLASH_ARGS_FILE in os.listdir(self.binary_path): # CMake version using build metadata file - with open(os.path.join(self.binary_path, self.IDF_FLASH_ARGS_FILE), "r") as f: - args = json.load(f) - flash_files = [(offs,file) for (offs,file) in args["flash_files"].items() if offs != ""] - flash_settings = args["flash_settings"] + path = os.path.join(self.binary_path, self.IDF_FLASH_ARGS_FILE) else: # GNU Make version uses download.config arguments file - with open(os.path.join(self.binary_path, self.IDF_DOWNLOAD_CONFIG_FILE), "r") as f: - args = f.readlines()[-1].split(" ") - flash_files = [] - flash_settings = {} - for idx in range(0, len(args), 2): # process arguments in pairs - if args[idx].startswith("--"): - # strip the -- from the command line argument - flash_settings[args[idx][2:]] = args[idx + 1] - else: - # offs, filename - flash_files.append((args[idx], args[idx + 1])) + path = os.path.join(self.binary_path, self.IDF_DOWNLOAD_CONFIG_FILE) + flash_files, flash_settings, app_name = parse_flash_settings(path) # The build metadata file does not currently have details, which files should be encrypted and which not. # Assume that all files should be encrypted if flash encryption is enabled in development mode. sdkconfig_dict = self.get_sdkconfig() @@ -146,7 +250,7 @@ class IDFApp(App.BaseApp): # make file offsets into integers, make paths absolute flash_files = [(int(offs, 0), os.path.join(self.binary_path, path.strip())) for (offs, path) in flash_files] - return (flash_files, flash_settings) + return flash_files, flash_settings def _parse_partition_table(self): """ @@ -206,7 +310,7 @@ class Example(IDFApp): """ return [os.path.join(self.binary_path, "..", "sdkconfig")] - def get_binary_path(self, app_path, config_name=None): + def _try_get_binary_from_local_fs(self, app_path, config_name=None, target=None): # build folder of example path path = os.path.join(self.idf_path, app_path, "build") if os.path.exists(path): @@ -223,19 +327,28 @@ class Example(IDFApp): example_path = os.path.join(self.idf_path, "build_examples") for dirpath in os.listdir(example_path): if os.path.basename(dirpath) == app_path_underscored: - path = os.path.join(example_path, dirpath, config_name, self.target, "build") - return path + path = os.path.join(example_path, dirpath, config_name, target, "build") + if os.path.exists(path): + return path + else: + return None - raise OSError("Failed to find example binary") + def get_binary_path(self, app_path, config_name=None, target=None): + path = self._try_get_binary_from_local_fs(app_path, config_name, target) + if path: + return path + else: + artifacts = Artifacts(self.idf_path, CIAssignExampleTest.ARTIFACT_INDEX_FILE, + app_path, config_name, target) + path = artifacts.download_artifacts() + if path: + return os.path.join(self.idf_path, path) + else: + raise OSError("Failed to find example binary") class UT(IDFApp): - def get_binary_path(self, app_path, config_name=None): - """ - :param app_path: app path - :param config_name: config name - :return: binary path - """ + def get_binary_path(self, app_path, config_name=None, target=None): if not config_name: config_name = "default" @@ -259,12 +372,12 @@ class UT(IDFApp): class SSC(IDFApp): - def get_binary_path(self, app_path, config_name=None): + def get_binary_path(self, app_path, config_name=None, target=None): # TODO: to implement SSC get binary path return app_path class AT(IDFApp): - def get_binary_path(self, app_path, config_name=None): + def get_binary_path(self, app_path, config_name=None, target=None): # TODO: to implement AT get binary path return app_path