diff --git a/tools/ci/config/target-test.yml b/tools/ci/config/target-test.yml index baa30916f1..0ec079ee4d 100644 --- a/tools/ci/config/target-test.yml +++ b/tools/ci/config/target-test.yml @@ -21,7 +21,6 @@ - $BOT_LABEL_EXAMPLE_TEST dependencies: - assign_test - - build_examples_cmake_esp32 artifacts: when: always paths: @@ -64,8 +63,6 @@ - $BOT_LABEL_EXAMPLE_TEST dependencies: - assign_test - - build_examples_make - - build_examples_cmake_esp32 artifacts: when: always paths: @@ -282,7 +279,6 @@ example_test_010: example_test_011: extends: .example_debug_template - parallel: 4 tags: - ESP32 - Example_T2_RS485 diff --git a/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py b/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py index 85beb1f086..ed27c58eb3 100644 --- a/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py +++ b/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py @@ -18,14 +18,23 @@ Command line tool to assign example tests to CI test jobs. # TODO: Need to handle running examples on different chips import os -import sys import re import argparse +import json import gitlab_api from tiny_test_fw.Utility import CIAssignTest +EXAMPLE_BUILD_JOB_NAMES = ["build_examples_cmake_esp32", "build_examples_cmake_esp32s2"] +IDF_PATH_FROM_ENV = os.getenv("IDF_PATH") +if IDF_PATH_FROM_ENV: + ARTIFACT_INDEX_FILE = os.path.join(IDF_PATH_FROM_ENV, + "build_examples", "artifact_index.json") +else: + ARTIFACT_INDEX_FILE = "artifact_index.json" + + class ExampleGroup(CIAssignTest.Group): SORT_KEYS = CI_JOB_MATCH_KEYS = ["env_tag", "chip"] @@ -34,15 +43,33 @@ class CIExampleAssignTest(CIAssignTest.AssignTest): CI_TEST_JOB_PATTERN = re.compile(r"^example_test_.+") -class ArtifactFile(object): - def __init__(self, project_id, job_name, artifact_file_path): - self.gitlab_api = gitlab_api.Gitlab(project_id) +def create_artifact_index_file(project_id=None, pipeline_id=None): + if project_id is None: + project_id = os.getenv("CI_PROJECT_ID") + if pipeline_id is None: + pipeline_id = os.getenv("CI_PIPELINE_ID") + gitlab_inst = gitlab_api.Gitlab(project_id) + artifact_index_list = [] - def process(self): + def format_build_log_path(): + return "build_examples/list_job_{}.json".format(job_info["parallel_num"]) + + for build_job_name in EXAMPLE_BUILD_JOB_NAMES: + job_info_list = gitlab_inst.find_job_id(build_job_name, pipeline_id=pipeline_id) + for job_info in job_info_list: + raw_data = gitlab_inst.download_artifact(job_info["id"], [format_build_log_path()])[0] + build_info_list = [json.loads(line) for line in raw_data.splitlines()] + for build_info in build_info_list: + build_info["ci_job_id"] = job_info["id"] + artifact_index_list.append(build_info) + try: + os.makedirs(os.path.dirname(ARTIFACT_INDEX_FILE)) + except OSError: + # already created pass - def output(self): - pass + with open(ARTIFACT_INDEX_FILE, "w") as f: + json.dump(artifact_index_list, f) if __name__ == '__main__': @@ -53,8 +80,11 @@ if __name__ == '__main__': help="gitlab ci config file") parser.add_argument("output_path", help="output path of config files") + parser.add_argument("--pipeline_id", "-p", type=int, default=None, + help="pipeline_id") args = parser.parse_args() assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file, case_group=ExampleGroup) assign_test.assign_cases() assign_test.output_configs(args.output_path) + create_artifact_index_file() diff --git a/tools/ci/python_packages/ttfw_idf/IDFApp.py b/tools/ci/python_packages/ttfw_idf/IDFApp.py index eff8c14584..e90418caa0 100644 --- a/tools/ci/python_packages/ttfw_idf/IDFApp.py +++ b/tools/ci/python_packages/ttfw_idf/IDFApp.py @@ -17,7 +17,102 @@ import subprocess import os import json + from tiny_test_fw import App +from . import CIAssignExampleTest + +try: + import gitlab_api +except ImportError: + gitlab_api = None + + +def parse_flash_settings(path): + file_name = os.path.basename(path) + if file_name == "flasher_args.json": + # CMake version using build metadata file + with open(path, "r") as f: + args = json.load(f) + flash_files = [(offs, binary) for (offs, binary) in args["flash_files"].items() if offs != ""] + flash_settings = args["flash_settings"] + else: + # GNU Make version uses download.config arguments file + with open(path, "r") as f: + args = f.readlines()[-1].split(" ") + flash_files = [] + flash_settings = {} + for idx in range(0, len(args), 2): # process arguments in pairs + if args[idx].startswith("--"): + # strip the -- from the command line argument + flash_settings[args[idx][2:]] = args[idx + 1] + else: + # offs, filename + flash_files.append((args[idx], args[idx + 1])) + return flash_files, flash_settings + + +class Artifacts(object): + def __init__(self, dest_root_path): + assert gitlab_api + self.gitlab_inst = gitlab_api.Gitlab(os.getenv("CI_PROJECT_ID")) + self.dest_root_path = dest_root_path + + @staticmethod + def _find_artifact(artifact_index, app_path, config_name, target): + for artifact_info in artifact_index: + match_result = True + if app_path: + match_result = app_path in artifact_info["app_dir"] + if config_name: + match_result = match_result and config_name == artifact_info["config"] + if target: + match_result = match_result and target == artifact_info["target"] + if match_result: + ret = artifact_info + break + else: + ret = None + return ret + + def download_artifact(self, artifact_index_file, app_path, config_name, target): + # at least one of app_path or config_name is not None. otherwise we can't match artifact + assert app_path or config_name + assert os.path.exists(artifact_index_file) + with open(artifact_index_file, "r") as f: + artifact_index = json.load(f) + + artifact_info = self._find_artifact(artifact_index, app_path, config_name, target) + + if artifact_info: + base_path = os.path.join(artifact_info["work_dir"], artifact_info["build_dir"]) + job_id = artifact_info["ci_job_id"] + + # 1. download flash args file + if artifact_info["build_system"] == "cmake": + flash_arg_file = os.path.join(base_path, "flasher_args.json") + else: + flash_arg_file = os.path.join(base_path, "download.config") + + self.gitlab_inst.download_artifact(job_id, [flash_arg_file], self.dest_root_path) + + # 2. download all binary files + flash_files, flash_settings = parse_flash_settings(os.path.join(self.dest_root_path, flash_arg_file)) + artifact_files = [] + for p in flash_files: + artifact_files.append(os.path.join(base_path, p[1])) + if not os.path.dirname(p[1]): + # find app bin and also download elf + elf_file = os.path.splitext(p[1])[0] + ".elf" + artifact_files.append(os.path.join(base_path, elf_file)) + + self.gitlab_inst.download_artifact(job_id, artifact_files, self.dest_root_path) + + # 3. download sdkconfig file + self.gitlab_inst.download_artifact(job_id, [os.path.join(os.path.dirname(base_path), "sdkconfig")], + self.dest_root_path) + else: + base_path = None + return base_path class IDFApp(App.BaseApp): @@ -34,7 +129,7 @@ class IDFApp(App.BaseApp): self.config_name = config_name self.target = target self.idf_path = self.get_sdk_path() - self.binary_path = self.get_binary_path(app_path, config_name) + self.binary_path = self.get_binary_path(app_path, config_name, target) self.elf_file = self._get_elf_file_path(self.binary_path) assert os.path.exists(self.binary_path) sdkconfig_dict = self.get_sdkconfig() @@ -72,7 +167,6 @@ class IDFApp(App.BaseApp): """ reads sdkconfig and returns a dictionary with all configuredvariables - :param sdkconfig_file: location of sdkconfig :raise: AssertionError: if sdkconfig file does not exist in defined paths """ d = {} @@ -89,14 +183,15 @@ class IDFApp(App.BaseApp): d[configs[0]] = configs[1].rstrip() return d - def get_binary_path(self, app_path, config_name=None): + def get_binary_path(self, app_path, config_name=None, target=None): """ get binary path according to input app_path. subclass must overwrite this method. :param app_path: path of application - :param config_name: name of the application build config + :param config_name: name of the application build config. Will match any config if None + :param target: target name. Will match for target if None :return: abs app binary path """ pass @@ -123,24 +218,12 @@ class IDFApp(App.BaseApp): if self.IDF_FLASH_ARGS_FILE in os.listdir(self.binary_path): # CMake version using build metadata file - with open(os.path.join(self.binary_path, self.IDF_FLASH_ARGS_FILE), "r") as f: - args = json.load(f) - flash_files = [(offs,file) for (offs,file) in args["flash_files"].items() if offs != ""] - flash_settings = args["flash_settings"] + path = os.path.join(self.binary_path, self.IDF_FLASH_ARGS_FILE) else: # GNU Make version uses download.config arguments file - with open(os.path.join(self.binary_path, self.IDF_DOWNLOAD_CONFIG_FILE), "r") as f: - args = f.readlines()[-1].split(" ") - flash_files = [] - flash_settings = {} - for idx in range(0, len(args), 2): # process arguments in pairs - if args[idx].startswith("--"): - # strip the -- from the command line argument - flash_settings[args[idx][2:]] = args[idx + 1] - else: - # offs, filename - flash_files.append((args[idx], args[idx + 1])) + path = os.path.join(self.binary_path, self.IDF_DOWNLOAD_CONFIG_FILE) + flash_files, flash_settings = parse_flash_settings(path) # The build metadata file does not currently have details, which files should be encrypted and which not. # Assume that all files should be encrypted if flash encryption is enabled in development mode. sdkconfig_dict = self.get_sdkconfig() @@ -149,7 +232,7 @@ class IDFApp(App.BaseApp): # make file offsets into integers, make paths absolute flash_files = [(int(offs, 0), os.path.join(self.binary_path, path.strip())) for (offs, path) in flash_files] - return (flash_files, flash_settings) + return flash_files, flash_settings def _parse_partition_table(self): """ @@ -209,7 +292,7 @@ class Example(IDFApp): """ return [os.path.join(self.binary_path, "..", "sdkconfig")] - def get_binary_path(self, app_path, config_name=None): + def get_binary_path(self, app_path, config_name=None, target=None): # build folder of example path path = os.path.join(self.idf_path, app_path, "build") if os.path.exists(path): @@ -227,18 +310,23 @@ class Example(IDFApp): for dirpath in os.listdir(example_path): if os.path.basename(dirpath) == app_path_underscored: path = os.path.join(example_path, dirpath, config_name, self.target, "build") - return path + if os.path.exists(path): + return path + else: + # app path exists, but config name not exists. try to download artifacts. + break - raise OSError("Failed to find example binary") + artifacts = Artifacts(self.idf_path) + path = artifacts.download_artifact(CIAssignExampleTest.ARTIFACT_INDEX_FILE, + app_path, config_name, target) + if path: + return os.path.join(self.idf_path, path) + else: + raise OSError("Failed to find example binary") class UT(IDFApp): - def get_binary_path(self, app_path, config_name=None): - """ - :param app_path: app path - :param config_name: config name - :return: binary path - """ + def get_binary_path(self, app_path, config_name=None, target=None): if not config_name: config_name = "default" @@ -262,12 +350,12 @@ class UT(IDFApp): class SSC(IDFApp): - def get_binary_path(self, app_path, config_name=None): + def get_binary_path(self, app_path, config_name=None, target=None): # TODO: to implement SSC get binary path return app_path class AT(IDFApp): - def get_binary_path(self, app_path, config_name=None): + def get_binary_path(self, app_path, config_name=None, target=None): # TODO: to implement AT get binary path return app_path diff --git a/tools/ci/python_packages/ttfw_idf/__init__.py b/tools/ci/python_packages/ttfw_idf/__init__.py index d7aed38571..4262a26434 100644 --- a/tools/ci/python_packages/ttfw_idf/__init__.py +++ b/tools/ci/python_packages/ttfw_idf/__init__.py @@ -16,7 +16,7 @@ import re from tiny_test_fw import TinyFW, Utility from IDFApp import IDFApp, Example, UT -from IDFDUT import IDFDUT, ESP32DUT, ESP32S2DUT, ESP8266DUT +from IDFDUT import IDFDUT, ESP32DUT, ESP32S2DUT, ESP8266DUT # noqa: export DUTs for users def format_case_id(chip, case_name):