kopia lustrzana https://github.com/espressif/esp-idf
CI: assign and target-test stages updated to run test-apps in the ci
David Cermak
bot
rodzic
e63764b468
commit
692deac5ae
|
|
@ -57,7 +57,7 @@ echo "build_examples running for target $IDF_TARGET"
|
|||
|
||||
cd ${IDF_PATH}
|
||||
|
||||
# This part of the script produces the same result for all the example build jobs. It may be moved to a separate stage
|
||||
# This part of the script produces the same result for all the test app build jobs. It may be moved to a separate stage
|
||||
# (pre-build) later, then the build jobs will receive ${BUILD_LIST_JSON} file as an artifact.
|
||||
|
||||
# If changing the work-dir or build-dir, remember to update the "artifacts" in gitlab-ci configs, and IDFApp.py.
|
||||
|
|
@ -68,7 +68,8 @@ ${IDF_PATH}/tools/find_apps.py tools/test_apps \
|
|||
--build-system cmake \
|
||||
--target ${IDF_TARGET} \
|
||||
--recursive \
|
||||
--build-dir "\${IDF_PATH}/${BUILD_PATH}/@f/@w/@t/build" \
|
||||
--work-dir "${BUILD_PATH}/@f/@w/@t" \
|
||||
--build-dir build \
|
||||
--build-log "${LOG_PATH}/@f.txt" \
|
||||
--output ${ALL_BUILD_LIST_JSON} \
|
||||
--config 'sdkconfig.ci=default' \
|
||||
|
|
@ -79,6 +80,11 @@ ${IDF_PATH}/tools/find_apps.py tools/test_apps \
|
|||
# 1. If sdkconfig.ci exists, use it build the example with configuration name "default"
|
||||
# 2. If sdkconfig.ci.* exists, use it to build the "*" configuration
|
||||
# 3. If none of the above exist, build the default configuration under the name "default"
|
||||
# --work-dir and --build-log above uses "placeholders" @x:
|
||||
# - @f: full path to the test with slashes replaced with underscores
|
||||
# - @w: wildcard used as config name
|
||||
# - @t: target name
|
||||
# so the workdir .../@f/@w/@t would expand to e.g. tools_test_apps_system_startup/default/esp32
|
||||
|
||||
# The part below is where the actual builds happen
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@ assign_test:
|
|||
dependencies:
|
||||
- build_ssc_esp32
|
||||
- build_esp_idf_tests_cmake
|
||||
- build_test_apps_esp32
|
||||
variables:
|
||||
SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
|
||||
EXAMPLE_CONFIG_OUTPUT_PATH: "$CI_PROJECT_DIR/examples/test_configs"
|
||||
|
|
@ -22,6 +21,7 @@ assign_test:
|
|||
- $EXAMPLE_CONFIG_OUTPUT_PATH
|
||||
- $TEST_APP_CONFIG_OUTPUT_PATH
|
||||
- build_examples/artifact_index.json
|
||||
- build_test_apps/artifact_index.json
|
||||
expire_in: 1 week
|
||||
only:
|
||||
variables:
|
||||
|
|
@ -29,11 +29,12 @@ assign_test:
|
|||
- $BOT_LABEL_UNIT_TEST
|
||||
- $BOT_LABEL_INTEGRATION_TEST
|
||||
- $BOT_LABEL_EXAMPLE_TEST
|
||||
- $BOT_LABEL_CUSTOM_TEST
|
||||
script:
|
||||
# assign example tests
|
||||
- python tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py $IDF_PATH/examples $CI_TARGET_TEST_CONFIG_FILE $EXAMPLE_CONFIG_OUTPUT_PATH
|
||||
# assign test apps
|
||||
- python tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py --job-prefix test_app_test_ $IDF_PATH/tools/test_apps $CI_TARGET_TEST_CONFIG_FILE $TEST_APP_CONFIG_OUTPUT_PATH
|
||||
- python tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py --custom-group test-apps --job-prefix test_app_test_ $IDF_PATH/tools/test_apps $CI_TARGET_TEST_CONFIG_FILE $TEST_APP_CONFIG_OUTPUT_PATH
|
||||
# assign unit test cases
|
||||
- python tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py $UNIT_TEST_CASE_FILE $CI_TARGET_TEST_CONFIG_FILE $IDF_PATH/components/idf_test/unit_test/CIConfigs
|
||||
# clone test script to assign tests
|
||||
|
|
|
|||
|
|
@ -186,8 +186,7 @@ build_examples_cmake_esp32s2:
|
|||
|
||||
.build_test_apps: &build_test_apps
|
||||
extends: .build_template
|
||||
parallel: 2
|
||||
stage: pre_build
|
||||
stage: build
|
||||
artifacts:
|
||||
when: always
|
||||
paths:
|
||||
|
|
@ -225,7 +224,7 @@ build_test_apps_esp32:
|
|||
build_test_apps_esp32s2:
|
||||
extends: .build_test_apps
|
||||
variables:
|
||||
IDF_TARGET: esp32s2beta
|
||||
IDF_TARGET: esp32s2
|
||||
|
||||
|
||||
# If you want to add new build example jobs, please add it into dependencies of `.example_test_template`
|
||||
|
|
|
|||
|
|
@ -85,27 +85,26 @@
|
|||
|
||||
.test_app_template:
|
||||
extends: .example_test_template
|
||||
stage: pre_target_test
|
||||
stage: target_test
|
||||
dependencies:
|
||||
- assign_test
|
||||
- build_test_apps_esp32
|
||||
only:
|
||||
refs:
|
||||
- master
|
||||
- /^release\/v/
|
||||
- /^v\d+\.\d+(\.\d+)?($|-)/
|
||||
- triggers
|
||||
- schedules
|
||||
variables:
|
||||
- $BOT_TRIGGER_WITH_LABEL == null
|
||||
- $BOT_LABEL_CUSTOM_TEST
|
||||
- $BOT_LABEL_EXAMPLE_TEST
|
||||
variables:
|
||||
TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
|
||||
TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/test_apps"
|
||||
CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/tools/test_apps/test_configs"
|
||||
LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
|
||||
ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"
|
||||
script:
|
||||
- *define_config_file_name
|
||||
# first test if config file exists, if not exist, exit 0
|
||||
- test -e $CONFIG_FILE || exit 0
|
||||
# clone test env configs
|
||||
- git clone $TEST_ENV_CONFIG_REPOSITORY
|
||||
- python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
|
||||
- cd $TEST_FW_PATH
|
||||
# run test
|
||||
- python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE
|
||||
|
||||
|
||||
.unit_test_template:
|
||||
extends: .example_test_template
|
||||
|
|
|
|||
|
|
@ -25,25 +25,34 @@ import json
|
|||
import gitlab_api
|
||||
from tiny_test_fw.Utility import CIAssignTest
|
||||
|
||||
|
||||
EXAMPLE_BUILD_JOB_NAMES = ["build_examples_cmake_esp32", "build_examples_cmake_esp32s2"]
|
||||
IDF_PATH_FROM_ENV = os.getenv("IDF_PATH")
|
||||
if IDF_PATH_FROM_ENV:
|
||||
ARTIFACT_INDEX_FILE = os.path.join(IDF_PATH_FROM_ENV,
|
||||
"build_examples", "artifact_index.json")
|
||||
else:
|
||||
ARTIFACT_INDEX_FILE = "artifact_index.json"
|
||||
|
||||
|
||||
class ExampleGroup(CIAssignTest.Group):
|
||||
SORT_KEYS = CI_JOB_MATCH_KEYS = ["env_tag", "chip"]
|
||||
BUILD_LOCAL_DIR = "build_examples"
|
||||
BUILD_JOB_NAMES = ["build_examples_cmake_esp32", "build_examples_cmake_esp32s2"]
|
||||
|
||||
|
||||
class TestAppsGroup(ExampleGroup):
|
||||
BUILD_LOCAL_DIR = "build_test_apps"
|
||||
BUILD_JOB_NAMES = ["build_test_apps_esp32", "build_test_apps_esp32s2"]
|
||||
|
||||
|
||||
class CIExampleAssignTest(CIAssignTest.AssignTest):
|
||||
CI_TEST_JOB_PATTERN = re.compile(r"^example_test_.+")
|
||||
|
||||
|
||||
def create_artifact_index_file(project_id=None, pipeline_id=None):
|
||||
def get_artifact_index_file(case_group=ExampleGroup):
|
||||
if IDF_PATH_FROM_ENV:
|
||||
artifact_index_file = os.path.join(IDF_PATH_FROM_ENV,
|
||||
case_group.BUILD_LOCAL_DIR, "artifact_index.json")
|
||||
else:
|
||||
artifact_index_file = "artifact_index.json"
|
||||
return artifact_index_file
|
||||
|
||||
|
||||
def create_artifact_index_file(project_id=None, pipeline_id=None, case_group=ExampleGroup):
|
||||
if project_id is None:
|
||||
project_id = os.getenv("CI_PROJECT_ID")
|
||||
if pipeline_id is None:
|
||||
|
|
@ -52,9 +61,10 @@ def create_artifact_index_file(project_id=None, pipeline_id=None):
|
|||
artifact_index_list = []
|
||||
|
||||
def format_build_log_path():
|
||||
return "build_examples/list_job_{}.json".format(job_info["parallel_num"])
|
||||
parallel = job_info["parallel_num"] # Could be None if "parallel_num" not defined for the job
|
||||
return "{}/list_job_{}.json".format(case_group.BUILD_LOCAL_DIR, parallel or 1)
|
||||
|
||||
for build_job_name in EXAMPLE_BUILD_JOB_NAMES:
|
||||
for build_job_name in case_group.BUILD_JOB_NAMES:
|
||||
job_info_list = gitlab_inst.find_job_id(build_job_name, pipeline_id=pipeline_id)
|
||||
for job_info in job_info_list:
|
||||
raw_data = gitlab_inst.download_artifact(job_info["id"], [format_build_log_path()])[0]
|
||||
|
|
@ -62,13 +72,14 @@ def create_artifact_index_file(project_id=None, pipeline_id=None):
|
|||
for build_info in build_info_list:
|
||||
build_info["ci_job_id"] = job_info["id"]
|
||||
artifact_index_list.append(build_info)
|
||||
artifact_index_file = get_artifact_index_file(case_group=case_group)
|
||||
try:
|
||||
os.makedirs(os.path.dirname(ARTIFACT_INDEX_FILE))
|
||||
os.makedirs(os.path.dirname(artifact_index_file))
|
||||
except OSError:
|
||||
# already created
|
||||
pass
|
||||
|
||||
with open(ARTIFACT_INDEX_FILE, "w") as f:
|
||||
with open(artifact_index_file, "w") as f:
|
||||
json.dump(artifact_index_list, f)
|
||||
|
||||
|
||||
|
|
@ -86,16 +97,18 @@ if __name__ == '__main__':
|
|||
help="prefix of the test job name in CI yml file")
|
||||
parser.add_argument("--test-case-file-pattern",
|
||||
help="file name pattern used to find Python test case files")
|
||||
parser.add_argument('--custom-group',
|
||||
help='select custom-group for the test cases, if other than ExampleTest',
|
||||
choices=['example','test-apps'], default='example')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.job_prefix:
|
||||
CIExampleAssignTest.CI_TEST_JOB_PATTERN = re.compile(r"^{}.+".format(args.job_prefix))
|
||||
|
||||
assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file, case_group=ExampleGroup)
|
||||
if args.test_case_file_pattern:
|
||||
assign_test.test_case_file_pattern = args.test_case_file_pattern
|
||||
case_group = ExampleGroup if args.custom_group == 'example' else TestAppsGroup
|
||||
|
||||
assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file, case_group=ExampleGroup)
|
||||
assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file, case_group=case_group)
|
||||
assign_test.assign_cases()
|
||||
assign_test.output_configs(args.output_path)
|
||||
create_artifact_index_file()
|
||||
create_artifact_index_file(case_group=case_group)
|
||||
|
|
|
|||
|
|
@ -139,7 +139,7 @@ class UnitTestAssignTest(CIAssignTest.AssignTest):
|
|||
def __init__(self, test_case_path, ci_config_file):
|
||||
CIAssignTest.AssignTest.__init__(self, test_case_path, ci_config_file, case_group=Group)
|
||||
|
||||
def _search_cases(self, test_case_path, case_filter=None):
|
||||
def _search_cases(self, test_case_path, case_filter=None, test_case_file_pattern=None):
|
||||
"""
|
||||
For unit test case, we don't search for test functions.
|
||||
The unit test cases is stored in a yaml file which is created in job build-idf-test.
|
||||
|
|
|
|||
|
|
@ -310,7 +310,7 @@ class Example(IDFApp):
|
|||
"""
|
||||
return [os.path.join(self.binary_path, "..", "sdkconfig")]
|
||||
|
||||
def _try_get_binary_from_local_fs(self, app_path, config_name=None, target=None):
|
||||
def _try_get_binary_from_local_fs(self, app_path, config_name=None, target=None, local_build_dir="build_examples"):
|
||||
# build folder of example path
|
||||
path = os.path.join(self.idf_path, app_path, "build")
|
||||
if os.path.exists(path):
|
||||
|
|
@ -327,7 +327,7 @@ class Example(IDFApp):
|
|||
# (see tools/ci/build_examples_cmake.sh)
|
||||
# For example: $IDF_PATH/build_examples/examples_get-started_blink/default/esp32
|
||||
app_path_underscored = app_path.replace(os.path.sep, "_")
|
||||
example_path = os.path.join(self.idf_path, "build_examples")
|
||||
example_path = os.path.join(self.idf_path, local_build_dir)
|
||||
for dirpath in os.listdir(example_path):
|
||||
if os.path.basename(dirpath) == app_path_underscored:
|
||||
path = os.path.join(example_path, dirpath, config_name, target, "build")
|
||||
|
|
@ -341,7 +341,8 @@ class Example(IDFApp):
|
|||
if path:
|
||||
return path
|
||||
else:
|
||||
artifacts = Artifacts(self.idf_path, CIAssignExampleTest.ARTIFACT_INDEX_FILE,
|
||||
artifacts = Artifacts(self.idf_path,
|
||||
CIAssignExampleTest.get_artifact_index_file(case_group=CIAssignExampleTest.ExampleGroup),
|
||||
app_path, config_name, target)
|
||||
path = artifacts.download_artifacts()
|
||||
if path:
|
||||
|
|
@ -369,7 +370,8 @@ class LoadableElfExample(Example):
|
|||
if path:
|
||||
return path
|
||||
else:
|
||||
artifacts = Artifacts(self.idf_path, CIAssignExampleTest.ARTIFACT_INDEX_FILE,
|
||||
artifacts = Artifacts(self.idf_path,
|
||||
CIAssignExampleTest.get_artifact_index_file(case_group=CIAssignExampleTest.ExampleGroup),
|
||||
app_path, config_name, target)
|
||||
path = artifacts.download_artifact_files(self.app_files)
|
||||
if path:
|
||||
|
|
@ -402,34 +404,20 @@ class UT(IDFApp):
|
|||
raise OSError("Failed to get unit-test-app binary path")
|
||||
|
||||
|
||||
class TestApp(IDFApp):
|
||||
def _get_sdkconfig_paths(self):
|
||||
"""
|
||||
overrides the parent method to provide exact path of sdkconfig for example tests
|
||||
"""
|
||||
return [os.path.join(self.binary_path, "..", "sdkconfig")]
|
||||
|
||||
def get_binary_path(self, app_path, config_name=None):
|
||||
# local build folder
|
||||
path = os.path.join(self.idf_path, app_path, "build")
|
||||
if os.path.exists(path):
|
||||
class TestApp(Example):
|
||||
def get_binary_path(self, app_path, config_name=None, target=None):
|
||||
path = self._try_get_binary_from_local_fs(app_path, config_name, target, local_build_dir="build_test_apps")
|
||||
if path:
|
||||
return path
|
||||
|
||||
if not config_name:
|
||||
config_name = "default"
|
||||
|
||||
# Search for CI build folders.
|
||||
# Path format: $IDF_PATH/build_test_apps/app_path_with_underscores/config/target
|
||||
# (see tools/ci/build_test_apps.sh)
|
||||
# For example: $IDF_PATH/build_test_apps/startup/default/esp32
|
||||
app_path_underscored = app_path.replace(os.path.sep, "_")
|
||||
build_root = os.path.join(self.idf_path, "build_test_apps")
|
||||
for dirpath in os.listdir(build_root):
|
||||
if os.path.basename(dirpath) == app_path_underscored:
|
||||
path = os.path.join(build_root, dirpath, config_name, self.target, "build")
|
||||
return path
|
||||
|
||||
raise OSError("Failed to find test app binary")
|
||||
else:
|
||||
artifacts = Artifacts(self.idf_path,
|
||||
CIAssignExampleTest.get_artifact_index_file(case_group=CIAssignExampleTest.TestAppsGroup),
|
||||
app_path, config_name, target)
|
||||
path = artifacts.download_artifacts()
|
||||
if path:
|
||||
return os.path.join(self.idf_path, path)
|
||||
else:
|
||||
raise OSError("Failed to find example binary")
|
||||
|
||||
|
||||
class SSC(IDFApp):
|
||||
|
|
|
|||
|
|
@ -88,10 +88,11 @@ def idf_unit_test(app=UT, dut=IDFDUT, chip="ESP32", module="unit-test", executio
|
|||
return test
|
||||
|
||||
|
||||
def idf_test_app_test(app=TestApp, dut=IDFDUT, chip="ESP32", module="misc", execution_time=1,
|
||||
level="integration", erase_nvs=True, **kwargs):
|
||||
def idf_custom_test(app=TestApp, dut=IDFDUT, chip="ESP32", module="misc", execution_time=1,
|
||||
level="integration", erase_nvs=True, config_name=None, group="test-apps", **kwargs):
|
||||
|
||||
"""
|
||||
decorator for testing idf unit tests (with default values for some keyword args).
|
||||
decorator for idf custom tests (with default values for some keyword args).
|
||||
|
||||
:param app: test application class
|
||||
:param dut: dut class
|
||||
|
|
@ -100,6 +101,8 @@ def idf_test_app_test(app=TestApp, dut=IDFDUT, chip="ESP32", module="misc", exec
|
|||
:param execution_time: execution time in minutes, int
|
||||
:param level: test level, could be used to filter test cases, string
|
||||
:param erase_nvs: if need to erase_nvs in DUT.start_app()
|
||||
:param config_name: if specified, name of the app configuration
|
||||
:param group: identifier to group custom tests (unused for now, defaults to "test-apps")
|
||||
:param kwargs: other keyword args
|
||||
:return: test method
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1,27 +1,15 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
import re
|
||||
import os
|
||||
import sys
|
||||
import glob
|
||||
|
||||
try:
|
||||
import IDF
|
||||
except ImportError:
|
||||
# This environment variable is expected on the host machine
|
||||
test_fw_path = os.getenv("TEST_FW_PATH")
|
||||
if test_fw_path and test_fw_path not in sys.path:
|
||||
sys.path.insert(0, test_fw_path)
|
||||
|
||||
import IDF
|
||||
|
||||
import Utility
|
||||
import ttfw_idf
|
||||
from tiny_test_fw import Utility
|
||||
|
||||
|
||||
@IDF.idf_test_app_test(env_tag="test_jtag_arm")
|
||||
@ttfw_idf.idf_test_app_test(env_tag="test_jtag_arm")
|
||||
def test_startup(env, extra_data):
|
||||
config_files = glob.glob(os.path.join(os.path.dirname(__file__), "sdkconfig.ci.*"))
|
||||
config_names = [s.replace("sdkconfig.ci.", "") for s in config_files]
|
||||
config_names = [os.path.basename(s).replace("sdkconfig.ci.", "") for s in config_files]
|
||||
for name in config_names:
|
||||
Utility.console_log("Checking config \"{}\"... ".format(name), end="")
|
||||
dut = env.get_dut("startup", "tools/test_apps/startup", app_config_name=name)
|
||||
Ładowanie…
Reference in New Issue