kopia lustrzana https://github.com/espressif/esp-idf
Merge branch 'feature/test_apps_v2' into 'master'
test apps (2nd attempt) Closes IDF-641 See merge request espressif/esp-idf!7084pull/2405/merge
commit
3017bfb8e3
|
@ -0,0 +1,102 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Build test apps
|
||||
#
|
||||
# Runs as part of CI process.
|
||||
#
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Safety settings (see https://gist.github.com/ilg-ul/383869cbb01f61a51c4d).
|
||||
|
||||
if [[ ! -z ${DEBUG_SHELL} ]]
|
||||
then
|
||||
set -x # Activate the expand mode if DEBUG is anything but empty.
|
||||
fi
|
||||
|
||||
set -o errexit # Exit if command failed.
|
||||
set -o pipefail # Exit if pipe failed.
|
||||
|
||||
export PATH="$IDF_PATH/tools/ci:$IDF_PATH/tools:$PATH"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
die() {
|
||||
echo "${1:-"Unknown Error"}" 1>&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
[ -z ${IDF_PATH} ] && die "IDF_PATH is not set"
|
||||
[ -z ${LOG_PATH} ] && die "LOG_PATH is not set"
|
||||
[ -z ${BUILD_PATH} ] && die "BUILD_PATH is not set"
|
||||
[ -z ${IDF_TARGET} ] && die "IDF_TARGET is not set"
|
||||
[ -d ${LOG_PATH} ] || mkdir -p ${LOG_PATH}
|
||||
[ -d ${BUILD_PATH} ] || mkdir -p ${BUILD_PATH}
|
||||
|
||||
if [ -z ${CI_NODE_TOTAL} ]; then
|
||||
CI_NODE_TOTAL=1
|
||||
echo "Assuming CI_NODE_TOTAL=${CI_NODE_TOTAL}"
|
||||
fi
|
||||
if [ -z ${CI_NODE_INDEX} ]; then
|
||||
# Gitlab uses a 1-based index
|
||||
CI_NODE_INDEX=1
|
||||
echo "Assuming CI_NODE_INDEX=${CI_NODE_INDEX}"
|
||||
fi
|
||||
|
||||
|
||||
set -o nounset # Exit if variable not set.
|
||||
|
||||
# Convert LOG_PATH to relative, to make the json file less verbose.
|
||||
LOG_PATH=$(realpath --relative-to ${IDF_PATH} ${LOG_PATH})
|
||||
BUILD_PATH=$(realpath --relative-to ${IDF_PATH} ${BUILD_PATH})
|
||||
|
||||
ALL_BUILD_LIST_JSON="${BUILD_PATH}/list.json"
|
||||
JOB_BUILD_LIST_JSON="${BUILD_PATH}/list_job_${CI_NODE_INDEX}.json"
|
||||
mkdir -p "${BUILD_PATH}/example_builds"
|
||||
|
||||
echo "build_examples running for target $IDF_TARGET"
|
||||
|
||||
cd ${IDF_PATH}
|
||||
|
||||
# This part of the script produces the same result for all the test app build jobs. It may be moved to a separate stage
|
||||
# (pre-build) later, then the build jobs will receive ${BUILD_LIST_JSON} file as an artifact.
|
||||
|
||||
# If changing the work-dir or build-dir, remember to update the "artifacts" in gitlab-ci configs, and IDFApp.py.
|
||||
|
||||
${IDF_PATH}/tools/find_apps.py tools/test_apps \
|
||||
-vv \
|
||||
--format json \
|
||||
--build-system cmake \
|
||||
--target ${IDF_TARGET} \
|
||||
--recursive \
|
||||
--work-dir "${BUILD_PATH}/@f/@w/@t" \
|
||||
--build-dir build \
|
||||
--build-log "${LOG_PATH}/@f.txt" \
|
||||
--output ${ALL_BUILD_LIST_JSON} \
|
||||
--config 'sdkconfig.ci=default' \
|
||||
--config 'sdkconfig.ci.*=' \
|
||||
--config '=default' \
|
||||
|
||||
# --config rules above explained:
|
||||
# 1. If sdkconfig.ci exists, use it build the example with configuration name "default"
|
||||
# 2. If sdkconfig.ci.* exists, use it to build the "*" configuration
|
||||
# 3. If none of the above exist, build the default configuration under the name "default"
|
||||
# --work-dir and --build-log above uses "placeholders" @x:
|
||||
# - @f: full path to the test with slashes replaced with underscores
|
||||
# - @w: wildcard used as config name
|
||||
# - @t: target name
|
||||
# so the workdir .../@f/@w/@t would expand to e.g. tools_test_apps_system_startup/default/esp32
|
||||
|
||||
# The part below is where the actual builds happen
|
||||
|
||||
${IDF_PATH}/tools/build_apps.py \
|
||||
-vv \
|
||||
--format json \
|
||||
--keep-going \
|
||||
--parallel-count ${CI_NODE_TOTAL} \
|
||||
--parallel-index ${CI_NODE_INDEX} \
|
||||
--output-build-list ${JOB_BUILD_LIST_JSON} \
|
||||
${ALL_BUILD_LIST_JSON}\
|
||||
|
||||
|
||||
# Check for build warnings
|
||||
${IDF_PATH}/tools/ci/check_build_warnings.py -vv ${JOB_BUILD_LIST_JSON}
|
|
@ -5,20 +5,23 @@ assign_test:
|
|||
image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
|
||||
stage: assign_test
|
||||
# gitlab ci do not support match job with RegEx or wildcard now in dependencies.
|
||||
# we have a lot build example jobs. now we don't use dependencies, just download all artificats of build stage.
|
||||
# we have a lot build example jobs. now we don't use dependencies, just download all artifacts of build stage.
|
||||
dependencies:
|
||||
- build_ssc_esp32
|
||||
- build_esp_idf_tests_cmake
|
||||
variables:
|
||||
SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
|
||||
EXAMPLE_CONFIG_OUTPUT_PATH: "$CI_PROJECT_DIR/examples/test_configs"
|
||||
TEST_APP_CONFIG_OUTPUT_PATH: "$CI_PROJECT_DIR/tools/test_apps/test_configs"
|
||||
UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test/TestCaseAll.yml"
|
||||
artifacts:
|
||||
paths:
|
||||
- components/idf_test/*/CIConfigs
|
||||
- components/idf_test/*/TC.sqlite
|
||||
- $EXAMPLE_CONFIG_OUTPUT_PATH
|
||||
- $TEST_APP_CONFIG_OUTPUT_PATH
|
||||
- build_examples/artifact_index.json
|
||||
- build_test_apps/artifact_index.json
|
||||
expire_in: 1 week
|
||||
only:
|
||||
variables:
|
||||
|
@ -26,9 +29,12 @@ assign_test:
|
|||
- $BOT_LABEL_UNIT_TEST
|
||||
- $BOT_LABEL_INTEGRATION_TEST
|
||||
- $BOT_LABEL_EXAMPLE_TEST
|
||||
- $BOT_LABEL_CUSTOM_TEST
|
||||
script:
|
||||
# assign example tests
|
||||
- python tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py $IDF_PATH/examples $CI_TARGET_TEST_CONFIG_FILE $EXAMPLE_CONFIG_OUTPUT_PATH
|
||||
# assign test apps
|
||||
- python tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py --custom-group test-apps --job-prefix test_app_test_ $IDF_PATH/tools/test_apps $CI_TARGET_TEST_CONFIG_FILE $TEST_APP_CONFIG_OUTPUT_PATH
|
||||
# assign unit test cases
|
||||
- python tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py $UNIT_TEST_CASE_FILE $CI_TARGET_TEST_CONFIG_FILE $IDF_PATH/components/idf_test/unit_test/CIConfigs
|
||||
# clone test script to assign tests
|
||||
|
|
|
@ -184,6 +184,49 @@ build_examples_cmake_esp32s2:
|
|||
variables:
|
||||
IDF_TARGET: esp32s2
|
||||
|
||||
.build_test_apps: &build_test_apps
|
||||
extends: .build_template
|
||||
stage: build
|
||||
artifacts:
|
||||
when: always
|
||||
paths:
|
||||
- build_test_apps/list.json
|
||||
- build_test_apps/list_job_*.json
|
||||
- build_test_apps/*/*/*/build/*.bin
|
||||
- build_test_apps/*/*/*/sdkconfig
|
||||
- build_test_apps/*/*/*/build/*.elf
|
||||
- build_test_apps/*/*/*/build/*.map
|
||||
- build_test_apps/*/*/*/build/flasher_args.json
|
||||
- build_test_apps/*/*/*/build/bootloader/*.bin
|
||||
- build_test_apps/*/*/*/build/partition_table/*.bin
|
||||
- $LOG_PATH
|
||||
expire_in: 3 days
|
||||
variables:
|
||||
LOG_PATH: "$CI_PROJECT_DIR/log_test_apps"
|
||||
BUILD_PATH: "$CI_PROJECT_DIR/build_test_apps"
|
||||
only:
|
||||
variables:
|
||||
- $BOT_TRIGGER_WITH_LABEL == null
|
||||
- $BOT_LABEL_BUILD
|
||||
- $BOT_LABEL_INTEGRATION_TEST
|
||||
- $BOT_LABEL_REGULAR_TEST
|
||||
- $BOT_LABEL_WEEKEND_TEST
|
||||
script:
|
||||
- mkdir -p ${BUILD_PATH}
|
||||
- mkdir -p ${LOG_PATH}
|
||||
- ${IDF_PATH}/tools/ci/build_test_apps.sh
|
||||
|
||||
build_test_apps_esp32:
|
||||
extends: .build_test_apps
|
||||
variables:
|
||||
IDF_TARGET: esp32
|
||||
|
||||
build_test_apps_esp32s2:
|
||||
extends: .build_test_apps
|
||||
variables:
|
||||
IDF_TARGET: esp32s2
|
||||
|
||||
|
||||
# If you want to add new build example jobs, please add it into dependencies of `.example_test_template`
|
||||
|
||||
.build_docs_template: &build_docs_template
|
||||
|
|
|
@ -83,6 +83,29 @@
|
|||
# run test
|
||||
- python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE
|
||||
|
||||
.test_app_template:
|
||||
extends: .example_test_template
|
||||
stage: target_test
|
||||
dependencies:
|
||||
- assign_test
|
||||
only:
|
||||
refs:
|
||||
- master
|
||||
- /^release\/v/
|
||||
- /^v\d+\.\d+(\.\d+)?($|-)/
|
||||
- triggers
|
||||
- schedules
|
||||
variables:
|
||||
- $BOT_TRIGGER_WITH_LABEL == null
|
||||
- $BOT_LABEL_CUSTOM_TEST
|
||||
- $BOT_LABEL_EXAMPLE_TEST
|
||||
variables:
|
||||
TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
|
||||
TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/test_apps"
|
||||
CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/tools/test_apps/test_configs"
|
||||
LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
|
||||
ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"
|
||||
|
||||
.unit_test_template:
|
||||
extends: .example_test_template
|
||||
stage: target_test
|
||||
|
@ -279,6 +302,12 @@ example_test_010:
|
|||
- ESP32
|
||||
- Example_ExtFlash
|
||||
|
||||
test_app_test_001:
|
||||
extends: .test_app_template
|
||||
tags:
|
||||
- ESP32
|
||||
- test_jtag_arm
|
||||
|
||||
example_test_011:
|
||||
extends: .example_debug_template
|
||||
tags:
|
||||
|
|
|
@ -32,6 +32,7 @@ tools/check_python_dependencies.py
|
|||
tools/ci/apply_bot_filter.py
|
||||
tools/ci/build_examples.sh
|
||||
tools/ci/build_examples_cmake.sh
|
||||
tools/ci/build_test_apps.sh
|
||||
tools/ci/check-executable.sh
|
||||
tools/ci/check-line-endings.sh
|
||||
tools/ci/check_build_warnings.py
|
||||
|
|
|
@ -148,6 +148,7 @@ class AssignTest(object):
|
|||
|
||||
def __init__(self, test_case_path, ci_config_file, case_group=Group):
|
||||
self.test_case_path = test_case_path
|
||||
self.test_case_file_pattern = None
|
||||
self.test_cases = []
|
||||
self.jobs = self._parse_gitlab_ci_config(ci_config_file)
|
||||
self.case_group = case_group
|
||||
|
@ -177,7 +178,7 @@ class AssignTest(object):
|
|||
job_list.sort(key=lambda x: x["name"])
|
||||
return job_list
|
||||
|
||||
def _search_cases(self, test_case_path, case_filter=None):
|
||||
def _search_cases(self, test_case_path, case_filter=None, test_case_file_pattern=None):
|
||||
"""
|
||||
:param test_case_path: path contains test case folder
|
||||
:param case_filter: filter for test cases. the filter to use is default filter updated with case_filter param.
|
||||
|
@ -186,7 +187,7 @@ class AssignTest(object):
|
|||
_case_filter = self.DEFAULT_FILTER.copy()
|
||||
if case_filter:
|
||||
_case_filter.update(case_filter)
|
||||
test_methods = SearchCases.Search.search_test_cases(test_case_path)
|
||||
test_methods = SearchCases.Search.search_test_cases(test_case_path, test_case_file_pattern)
|
||||
return CaseConfig.filter_test_cases(test_methods, _case_filter)
|
||||
|
||||
def _group_cases(self):
|
||||
|
@ -276,7 +277,7 @@ class AssignTest(object):
|
|||
failed_to_assign = []
|
||||
assigned_groups = []
|
||||
case_filter = self._apply_bot_filter()
|
||||
self.test_cases = self._search_cases(self.test_case_path, case_filter)
|
||||
self.test_cases = self._search_cases(self.test_case_path, case_filter, self.test_case_file_pattern)
|
||||
self._apply_bot_test_count()
|
||||
test_groups = self._group_cases()
|
||||
|
||||
|
|
|
@ -93,14 +93,14 @@ class Search(object):
|
|||
return replicated_cases
|
||||
|
||||
@classmethod
|
||||
def search_test_cases(cls, test_case):
|
||||
def search_test_cases(cls, test_case, test_case_file_pattern=None):
|
||||
"""
|
||||
search all test cases from a folder or file, and then do case replicate.
|
||||
|
||||
:param test_case: test case file(s) path
|
||||
:return: a list of replicated test methods
|
||||
"""
|
||||
test_case_files = cls._search_test_case_files(test_case, cls.TEST_CASE_FILE_PATTERN)
|
||||
test_case_files = cls._search_test_case_files(test_case, test_case_file_pattern or cls.TEST_CASE_FILE_PATTERN)
|
||||
test_cases = []
|
||||
for test_case_file in test_case_files:
|
||||
test_cases += cls._search_cases_from_file(test_case_file)
|
||||
|
|
|
@ -25,25 +25,34 @@ import json
|
|||
import gitlab_api
|
||||
from tiny_test_fw.Utility import CIAssignTest
|
||||
|
||||
|
||||
EXAMPLE_BUILD_JOB_NAMES = ["build_examples_cmake_esp32", "build_examples_cmake_esp32s2"]
|
||||
IDF_PATH_FROM_ENV = os.getenv("IDF_PATH")
|
||||
if IDF_PATH_FROM_ENV:
|
||||
ARTIFACT_INDEX_FILE = os.path.join(IDF_PATH_FROM_ENV,
|
||||
"build_examples", "artifact_index.json")
|
||||
else:
|
||||
ARTIFACT_INDEX_FILE = "artifact_index.json"
|
||||
|
||||
|
||||
class ExampleGroup(CIAssignTest.Group):
|
||||
SORT_KEYS = CI_JOB_MATCH_KEYS = ["env_tag", "chip"]
|
||||
BUILD_LOCAL_DIR = "build_examples"
|
||||
BUILD_JOB_NAMES = ["build_examples_cmake_esp32", "build_examples_cmake_esp32s2"]
|
||||
|
||||
|
||||
class TestAppsGroup(ExampleGroup):
|
||||
BUILD_LOCAL_DIR = "build_test_apps"
|
||||
BUILD_JOB_NAMES = ["build_test_apps_esp32", "build_test_apps_esp32s2"]
|
||||
|
||||
|
||||
class CIExampleAssignTest(CIAssignTest.AssignTest):
|
||||
CI_TEST_JOB_PATTERN = re.compile(r"^example_test_.+")
|
||||
|
||||
|
||||
def create_artifact_index_file(project_id=None, pipeline_id=None):
|
||||
def get_artifact_index_file(case_group=ExampleGroup):
|
||||
if IDF_PATH_FROM_ENV:
|
||||
artifact_index_file = os.path.join(IDF_PATH_FROM_ENV,
|
||||
case_group.BUILD_LOCAL_DIR, "artifact_index.json")
|
||||
else:
|
||||
artifact_index_file = "artifact_index.json"
|
||||
return artifact_index_file
|
||||
|
||||
|
||||
def create_artifact_index_file(project_id=None, pipeline_id=None, case_group=ExampleGroup):
|
||||
if project_id is None:
|
||||
project_id = os.getenv("CI_PROJECT_ID")
|
||||
if pipeline_id is None:
|
||||
|
@ -52,9 +61,10 @@ def create_artifact_index_file(project_id=None, pipeline_id=None):
|
|||
artifact_index_list = []
|
||||
|
||||
def format_build_log_path():
|
||||
return "build_examples/list_job_{}.json".format(job_info["parallel_num"])
|
||||
parallel = job_info["parallel_num"] # Could be None if "parallel_num" not defined for the job
|
||||
return "{}/list_job_{}.json".format(case_group.BUILD_LOCAL_DIR, parallel or 1)
|
||||
|
||||
for build_job_name in EXAMPLE_BUILD_JOB_NAMES:
|
||||
for build_job_name in case_group.BUILD_JOB_NAMES:
|
||||
job_info_list = gitlab_inst.find_job_id(build_job_name, pipeline_id=pipeline_id)
|
||||
for job_info in job_info_list:
|
||||
raw_data = gitlab_inst.download_artifact(job_info["id"], [format_build_log_path()])[0]
|
||||
|
@ -62,13 +72,14 @@ def create_artifact_index_file(project_id=None, pipeline_id=None):
|
|||
for build_info in build_info_list:
|
||||
build_info["ci_job_id"] = job_info["id"]
|
||||
artifact_index_list.append(build_info)
|
||||
artifact_index_file = get_artifact_index_file(case_group=case_group)
|
||||
try:
|
||||
os.makedirs(os.path.dirname(ARTIFACT_INDEX_FILE))
|
||||
os.makedirs(os.path.dirname(artifact_index_file))
|
||||
except OSError:
|
||||
# already created
|
||||
pass
|
||||
|
||||
with open(ARTIFACT_INDEX_FILE, "w") as f:
|
||||
with open(artifact_index_file, "w") as f:
|
||||
json.dump(artifact_index_list, f)
|
||||
|
||||
|
||||
|
@ -82,9 +93,22 @@ if __name__ == '__main__':
|
|||
help="output path of config files")
|
||||
parser.add_argument("--pipeline_id", "-p", type=int, default=None,
|
||||
help="pipeline_id")
|
||||
parser.add_argument("--job-prefix",
|
||||
help="prefix of the test job name in CI yml file")
|
||||
parser.add_argument("--test-case-file-pattern",
|
||||
help="file name pattern used to find Python test case files")
|
||||
parser.add_argument('--custom-group',
|
||||
help='select custom-group for the test cases, if other than ExampleTest',
|
||||
choices=['example','test-apps'], default='example')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file, case_group=ExampleGroup)
|
||||
if args.job_prefix:
|
||||
CIExampleAssignTest.CI_TEST_JOB_PATTERN = re.compile(r"^{}.+".format(args.job_prefix))
|
||||
|
||||
case_group = ExampleGroup if args.custom_group == 'example' else TestAppsGroup
|
||||
|
||||
assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file, case_group=case_group)
|
||||
assign_test.assign_cases()
|
||||
assign_test.output_configs(args.output_path)
|
||||
create_artifact_index_file()
|
||||
create_artifact_index_file(case_group=case_group)
|
||||
|
|
|
@ -139,7 +139,7 @@ class UnitTestAssignTest(CIAssignTest.AssignTest):
|
|||
def __init__(self, test_case_path, ci_config_file):
|
||||
CIAssignTest.AssignTest.__init__(self, test_case_path, ci_config_file, case_group=Group)
|
||||
|
||||
def _search_cases(self, test_case_path, case_filter=None):
|
||||
def _search_cases(self, test_case_path, case_filter=None, test_case_file_pattern=None):
|
||||
"""
|
||||
For unit test case, we don't search for test functions.
|
||||
The unit test cases is stored in a yaml file which is created in job build-idf-test.
|
||||
|
|
|
@ -310,7 +310,7 @@ class Example(IDFApp):
|
|||
"""
|
||||
return [os.path.join(self.binary_path, "..", "sdkconfig")]
|
||||
|
||||
def _try_get_binary_from_local_fs(self, app_path, config_name=None, target=None):
|
||||
def _try_get_binary_from_local_fs(self, app_path, config_name=None, target=None, local_build_dir="build_examples"):
|
||||
# build folder of example path
|
||||
path = os.path.join(self.idf_path, app_path, "build")
|
||||
if os.path.exists(path):
|
||||
|
@ -327,7 +327,7 @@ class Example(IDFApp):
|
|||
# (see tools/ci/build_examples_cmake.sh)
|
||||
# For example: $IDF_PATH/build_examples/examples_get-started_blink/default/esp32
|
||||
app_path_underscored = app_path.replace(os.path.sep, "_")
|
||||
example_path = os.path.join(self.idf_path, "build_examples")
|
||||
example_path = os.path.join(self.idf_path, local_build_dir)
|
||||
for dirpath in os.listdir(example_path):
|
||||
if os.path.basename(dirpath) == app_path_underscored:
|
||||
path = os.path.join(example_path, dirpath, config_name, target, "build")
|
||||
|
@ -341,7 +341,8 @@ class Example(IDFApp):
|
|||
if path:
|
||||
return path
|
||||
else:
|
||||
artifacts = Artifacts(self.idf_path, CIAssignExampleTest.ARTIFACT_INDEX_FILE,
|
||||
artifacts = Artifacts(self.idf_path,
|
||||
CIAssignExampleTest.get_artifact_index_file(case_group=CIAssignExampleTest.ExampleGroup),
|
||||
app_path, config_name, target)
|
||||
path = artifacts.download_artifacts()
|
||||
if path:
|
||||
|
@ -369,7 +370,8 @@ class LoadableElfExample(Example):
|
|||
if path:
|
||||
return path
|
||||
else:
|
||||
artifacts = Artifacts(self.idf_path, CIAssignExampleTest.ARTIFACT_INDEX_FILE,
|
||||
artifacts = Artifacts(self.idf_path,
|
||||
CIAssignExampleTest.get_artifact_index_file(case_group=CIAssignExampleTest.ExampleGroup),
|
||||
app_path, config_name, target)
|
||||
path = artifacts.download_artifact_files(self.app_files)
|
||||
if path:
|
||||
|
@ -402,6 +404,22 @@ class UT(IDFApp):
|
|||
raise OSError("Failed to get unit-test-app binary path")
|
||||
|
||||
|
||||
class TestApp(Example):
|
||||
def get_binary_path(self, app_path, config_name=None, target=None):
|
||||
path = self._try_get_binary_from_local_fs(app_path, config_name, target, local_build_dir="build_test_apps")
|
||||
if path:
|
||||
return path
|
||||
else:
|
||||
artifacts = Artifacts(self.idf_path,
|
||||
CIAssignExampleTest.get_artifact_index_file(case_group=CIAssignExampleTest.TestAppsGroup),
|
||||
app_path, config_name, target)
|
||||
path = artifacts.download_artifacts()
|
||||
if path:
|
||||
return os.path.join(self.idf_path, path)
|
||||
else:
|
||||
raise OSError("Failed to find example binary")
|
||||
|
||||
|
||||
class SSC(IDFApp):
|
||||
def get_binary_path(self, app_path, config_name=None, target=None):
|
||||
# TODO: to implement SSC get binary path
|
||||
|
|
|
@ -15,7 +15,7 @@ import os
|
|||
import re
|
||||
|
||||
from tiny_test_fw import TinyFW, Utility
|
||||
from .IDFApp import IDFApp, Example, LoadableElfExample, UT # noqa: export all Apps for users
|
||||
from .IDFApp import IDFApp, Example, LoadableElfExample, UT, TestApp # noqa: export all Apps for users
|
||||
from .IDFDUT import IDFDUT, ESP32DUT, ESP32S2DUT, ESP8266DUT, ESP32QEMUDUT # noqa: export DUTs for users
|
||||
|
||||
|
||||
|
@ -88,6 +88,41 @@ def idf_unit_test(app=UT, dut=IDFDUT, chip="ESP32", module="unit-test", executio
|
|||
return test
|
||||
|
||||
|
||||
def idf_custom_test(app=TestApp, dut=IDFDUT, chip="ESP32", module="misc", execution_time=1,
|
||||
level="integration", erase_nvs=True, config_name=None, group="test-apps", **kwargs):
|
||||
|
||||
"""
|
||||
decorator for idf custom tests (with default values for some keyword args).
|
||||
|
||||
:param app: test application class
|
||||
:param dut: dut class
|
||||
:param chip: chip supported, string or tuple
|
||||
:param module: module, string
|
||||
:param execution_time: execution time in minutes, int
|
||||
:param level: test level, could be used to filter test cases, string
|
||||
:param erase_nvs: if need to erase_nvs in DUT.start_app()
|
||||
:param config_name: if specified, name of the app configuration
|
||||
:param group: identifier to group custom tests (unused for now, defaults to "test-apps")
|
||||
:param kwargs: other keyword args
|
||||
:return: test method
|
||||
"""
|
||||
try:
|
||||
# try to config the default behavior of erase nvs
|
||||
dut.ERASE_NVS = erase_nvs
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
original_method = TinyFW.test_method(app=app, dut=dut, chip=chip, module=module,
|
||||
execution_time=execution_time, level=level, **kwargs)
|
||||
|
||||
def test(func):
|
||||
test_func = original_method(func)
|
||||
test_func.case_info["ID"] = format_case_id(chip, test_func.case_info["name"])
|
||||
return test_func
|
||||
|
||||
return test
|
||||
|
||||
|
||||
def log_performance(item, value):
|
||||
"""
|
||||
do print performance with pre-defined format to console
|
||||
|
|
|
@ -0,0 +1,32 @@
|
|||
# Test Apps
|
||||
|
||||
This directory contains a set of ESP-IDF projects to be used as tests only, which aim to exercise various
|
||||
configuration of components to check completely arbitrary functionality should it be building only, executing under
|
||||
various conditions or combination with other components, including custom test frameworks.
|
||||
|
||||
The test apps are not intended to demonstrate the ESP-IDF functionality in any way.
|
||||
|
||||
# Test Apps projects
|
||||
|
||||
Test applications are treated the same way as ESP-IDF examples, so each project directory shall contain
|
||||
* Build recipe in cmake and the main component with app sources
|
||||
* Configuration files
|
||||
- `sdkconfig.ci` - Default configuration for the project
|
||||
- `sdkconfig.ci.<CONFIG>` - Other configurations, where `<CONFIG>` indicates name of the configuration
|
||||
* Test executor in `ttfw_idf` format if the project is intended to also run tests (otherwise the example is build only)
|
||||
- test file in the project dir must end with `_test.py`, by should be named `app_test.py`
|
||||
- test cases shall be decorated with `@ttfw_idf.idf_custom_test(env_tag="...")`
|
||||
|
||||
# Test Apps layout
|
||||
|
||||
The test apps should be grouped into subdirectories by category. Categories are:
|
||||
* `protocols` contains test of protocol interactions.
|
||||
* `network` contains system network tests
|
||||
* `system` contains tests on the internal chip features, debugging and development tools.
|
||||
|
||||
# Test Apps local execution
|
||||
|
||||
* Append relevant `sdkconfig.ci.<CONFIG>` to the sdkconfig for the configuration under test
|
||||
* Run `idf.py menuconfig` to configure local project attributes
|
||||
* Run `idf.py build` to build the test app
|
||||
* Run `python app_test.py` to run the test locally
|
|
@ -0,0 +1,6 @@
|
|||
# The following lines of boilerplate have to be in your project's
|
||||
# CMakeLists in this exact order for cmake to work correctly
|
||||
cmake_minimum_required(VERSION 3.5)
|
||||
|
||||
include($ENV{IDF_PATH}/tools/cmake/project.cmake)
|
||||
project(test_startup)
|
|
@ -0,0 +1,4 @@
|
|||
This project tests if the app can start up in a certain configuration.
|
||||
To add new configuration, create one more sdkconfig.ci.NAME file in this directory.
|
||||
|
||||
If you need to test for anything other than app starting up, create another test project.
|
|
@ -0,0 +1,23 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
import os
|
||||
import glob
|
||||
import ttfw_idf
|
||||
from tiny_test_fw import Utility
|
||||
|
||||
|
||||
@ttfw_idf.idf_custom_test(env_tag="test_jtag_arm", group="test-apps")
|
||||
def test_startup(env, extra_data):
|
||||
config_files = glob.glob(os.path.join(os.path.dirname(__file__), "sdkconfig.ci.*"))
|
||||
config_names = [os.path.basename(s).replace("sdkconfig.ci.", "") for s in config_files]
|
||||
for name in config_names:
|
||||
Utility.console_log("Checking config \"{}\"... ".format(name), end="")
|
||||
dut = env.get_dut("startup", "tools/test_apps/system/startup", app_config_name=name)
|
||||
dut.start_app()
|
||||
dut.expect("app_main running")
|
||||
env.close_dut(dut.name)
|
||||
Utility.console_log("done")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
test_startup()
|
|
@ -0,0 +1,2 @@
|
|||
idf_component_register(SRCS "test_startup_main.c"
|
||||
INCLUDE_DIRS ".")
|
|
@ -0,0 +1,6 @@
|
|||
#include <stdio.h>
|
||||
|
||||
void app_main(void)
|
||||
{
|
||||
printf("app_main running\n");
|
||||
}
|
|
@ -0,0 +1,2 @@
|
|||
CONFIG_ESPTOOLPY_FLASHFREQ_80M=y
|
||||
CONFIG_ESPTOOLPY_FLASHMODE_QIO=y
|
Ładowanie…
Reference in New Issue