diff --git a/.gitignore b/.gitignore index 24639f73f9..84039cf144 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,8 @@ tools/unit-test-app/output # IDF monitor test tools/test_idf_monitor/outputs +TEST_LOGS + # AWS IoT Examples require device-specific certs/keys examples/protocols/aws_iot/*/main/certs/*.pem.* diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index 2626fc0741..0b88e669aa 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -37,3 +37,4 @@ tools/kconfig/mconf tools/windows/eclipse_make.sh tools/test_idf_monitor/run_test_idf_monitor.py tools/mass_mfg/mfg_gen.py +tools/unit-test-app/unit_test.py diff --git a/tools/unit-test-app/README.md b/tools/unit-test-app/README.md index 7c06ad7a69..358992f3e3 100644 --- a/tools/unit-test-app/README.md +++ b/tools/unit-test-app/README.md @@ -106,6 +106,28 @@ If you want to reproduce locally, you need to: * You can refer to [unit test document](https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/unit-tests.html#running-unit-tests) to run test manually. * Or, you can use `tools/unit-test-app/unit_test.py` to run the test cases: * read document of tiny-test-fw, set correct `TEST_FW_PATH` and `IDF_PATH` - * modify `unit_test.py`, pass the test cases need to test as parameter (refer to test function doc string for supported parameter format) to test functions. - * use `python unit_test.py` to run test + * run `unit_test.py` (see examples below) * You can also use `tools/tiny-test-fw/Runner.py` to run test cases (it will be the same as what Runner do). Please use `python Runner.py -c $CONFIG_FILE $IDF_PATH/tools/unit-test-app` command, where `CONFIG_FILE` is a YAML file with same name with CI job in `components/idf_test/unit_test/CIConfigs` (artifacts, need to be download from `assign_test` job). + +## Running unit tests on local machine by `unit_test.py` + +A couple of examples follow for running unit tests on local machine. + +```bash +# run a simple unit test +./unit_test.py "UART can do select()" +# repeat the tests two times +./unit_test.py -r 2 "UART can do select()" +# use custom environment config file +./unit_test.py -e /tmp/EnvConfigTemplate.yml "UART can do select()" +# use custom application binary +./unit_test.py -b /tmp/app.bin "UART can do select()" +# run a list of unit tests +./unit_test.py "UART can do select()" "concurent selects work" +# add some options for unit tests +./unit_test.py "UART can do select()",timeout:10 "concurent selects work",config:release,env_tag:UT_T2_1 +# run a multi stage test (type of test and child case numbers are autodetected) +./unit_test.py "check a time after wakeup from deep sleep" +# run a list of different unit tests (one simple and one multi stage test) +./unit_test.py "concurent selects work" "NOINIT attributes behavior" +``` diff --git a/tools/unit-test-app/components/unity/unity_platform.c b/tools/unit-test-app/components/unity/unity_platform.c index fa1adf6333..6beb85cb9d 100644 --- a/tools/unit-test-app/components/unity/unity_platform.c +++ b/tools/unit-test-app/components/unity/unity_platform.c @@ -309,6 +309,7 @@ static int print_test_menu(void) } } } + printf("\nEnter test for running.\n"); /* unit_test.py needs it for finding the end of test menu */ return test_counter; } diff --git a/tools/unit-test-app/unit_test.py b/tools/unit-test-app/unit_test.py old mode 100644 new mode 100755 index 4ce625af33..c1e6f96d14 --- a/tools/unit-test-app/unit_test.py +++ b/tools/unit-test-app/unit_test.py @@ -1,3 +1,19 @@ +#!/usr/bin/env python +# +# Copyright 2018 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """ Test script for unit test case. """ @@ -6,6 +22,7 @@ import re import os import sys import time +import argparse import threading @@ -18,6 +35,7 @@ if test_fw_path and test_fw_path not in sys.path: import TinyFW import IDF import Utility +import Env from DUT import ExpectTimeout from IDF.IDFApp import UT @@ -27,9 +45,16 @@ RESET_PATTERN = re.compile(r"(ets [\w]{3}\s+[\d]{1,2} [\d]{4} [\d]{2}:[\d]{2}:[\ EXCEPTION_PATTERN = re.compile(r"(Guru Meditation Error: Core\s+\d panic'ed \([\w].*?\))") ABORT_PATTERN = re.compile(r"(abort\(\) was called at PC 0x[a-eA-E\d]{8} on core \d)") FINISH_PATTERN = re.compile(r"1 Tests (\d) Failures (\d) Ignored") +END_LIST_STR = r'\r?\nEnter test for running' +TEST_PATTERN = re.compile(r'\((\d+)\)\s+"([^"]+)" ([^\r]+)\r?\n(' + END_LIST_STR + r')?') +TEST_SUBMENU_PATTERN = re.compile(r'\s+\((\d+)\)\s+"[^"]+"\r?\n(?=(?=\()|(' + END_LIST_STR + r'))') + +SIMPLE_TEST_ID = 0 +MULTI_STAGE_ID = 1 +MULTI_DEVICE_ID = 2 STARTUP_TIMEOUT=10 - +DEFAULT_TIMEOUT=20 def format_test_case_config(test_case_data): """ @@ -104,6 +129,15 @@ def format_test_case_config(test_case_data): return case_config +def replace_app_bin(dut, name, new_app_bin): + if new_app_bin is None: + return + search_pattern = '/{}.bin'.format(name) + for i, config in enumerate(dut.download_config): + if config.endswith(search_pattern): + dut.download_config[i] = new_app_bin + Utility.console_log("The replaced application binary is {}".format(new_app_bin), "O") + break @IDF.idf_unit_test(env_tag="UT_T1_1") def run_unit_test_cases(env, extra_data): @@ -132,6 +166,8 @@ def run_unit_test_cases(env, extra_data): for ut_config in case_config: Utility.console_log("Running unit test for config: " + ut_config, "O") dut = env.get_dut("unit-test-app", app_path=ut_config) + if len(case_config[ut_config]) > 0: + replace_app_bin(dut, "unit-test-app", case_config[ut_config][0].get('app_bin')) dut.start_app() for one_case in case_config[ut_config]: @@ -306,17 +342,18 @@ def get_case_info(one_case): return parent_case, child_case_num -def get_dut(duts, env, name, ut_config): +def get_dut(duts, env, name, ut_config, app_bin=None): if name in duts: dut = duts[name] else: dut = env.get_dut(name, app_path=ut_config) duts[name] = dut + replace_app_bin(dut, "unit-test-app", app_bin) dut.start_app() return dut -def case_run(duts, ut_config, env, one_case, failed_cases): +def case_run(duts, ut_config, env, one_case, failed_cases, app_bin): lock = threading.RLock() threads = [] send_signal_list = [] @@ -327,7 +364,7 @@ def case_run(duts, ut_config, env, one_case, failed_cases): THREAD_TERMINATE_FLAG = False for i in range(case_num): - dut = get_dut(duts, env, "dut%d" % i, ut_config) + dut = get_dut(duts, env, "dut%d" % i, ut_config, app_bin) threads.append(Handler(dut, send_signal_list, lock, parent_case, i, one_case["timeout"])) for thread in threads: @@ -374,7 +411,7 @@ def run_multiple_devices_cases(env, extra_data): for ut_config in case_config: Utility.console_log("Running unit test for config: " + ut_config, "O") for one_case in case_config[ut_config]: - case_run(DUTS, ut_config, env, one_case, failed_cases) + case_run(DUTS, ut_config, env, one_case, failed_cases, one_case.get('app_bin')) if failed_cases: Utility.console_log("Failed Cases:", color="red") @@ -405,6 +442,8 @@ def run_multiple_stage_cases(env, extra_data): for ut_config in case_config: Utility.console_log("Running unit test for config: " + ut_config, "O") dut = env.get_dut("unit-test-app", app_path=ut_config) + if len(case_config[ut_config]) > 0: + replace_app_bin(dut, "unit-test-app", case_config[ut_config][0].get('app_bin')) dut.start_app() for one_case in case_config[ut_config]: @@ -512,9 +551,132 @@ def run_multiple_stage_cases(env, extra_data): Utility.console_log("\t" + _case_name, color="red") raise AssertionError("Unit Test Failed") +def detect_update_unit_test_info(env, extra_data, app_bin): + + case_config = format_test_case_config(extra_data) + + for ut_config in case_config: + dut = env.get_dut("unit-test-app", app_path=ut_config) + replace_app_bin(dut, "unit-test-app", app_bin) + dut.start_app() + + dut.write("-", flush=False) + dut.expect_any(UT_APP_BOOT_UP_DONE, "0 Tests 0 Failures 0 Ignored", timeout=STARTUP_TIMEOUT) + + # get the list of test cases + dut.write("") + dut.expect("Here's the test menu, pick your combo:", timeout=DEFAULT_TIMEOUT) + + def find_update_dic(name, t, timeout, child_case_num=None): + for dic in extra_data: + if dic['name'] == name: + dic['type'] = t + if 'timeout' not in dic: + dic['timeout'] = timeout + if child_case_num: + dic['child case num'] = child_case_num + + try: + while True: + data = dut.expect(TEST_PATTERN, timeout=DEFAULT_TIMEOUT) + test_case_name = data[1] + m = re.search(r'\[timeout=(\d+)\]', data[2]) + if m: + timeout = int(m.group(1)) + else: + timeout = 30 + m = re.search(r'\[multi_stage\]', data[2]) + if m: + test_case_type = MULTI_STAGE_ID + else: + m = re.search(r'\[multi_device\]', data[2]) + if m: + test_case_type = MULTI_DEVICE_ID + else: + test_case_type = SIMPLE_TEST_ID + find_update_dic(test_case_name, test_case_type, timeout) + if data[3] and re.search(END_LIST_STR, data[3]): + break + continue + # find the last submenu item + data = dut.expect(TEST_SUBMENU_PATTERN, timeout=DEFAULT_TIMEOUT) + find_update_dic(test_case_name, test_case_type, timeout, child_case_num=int(data[0])) + if data[1] and re.search(END_LIST_STR, data[1]): + break + # check if the unit test case names are correct, i.e. they could be found in the device + for dic in extra_data: + if 'type' not in dic: + raise ValueError("Unit test \"{}\" doesn't exist in the flashed device!".format(dic.get('name'))) + except ExpectTimeout: + Utility.console_log("Timeout during getting the test list", color="red") + finally: + dut.close() + + # These options are the same for all configs, therefore there is no need to continue + break if __name__ == '__main__': - run_multiple_devices_cases(extra_data={"name": "gpio master/slave test example", - "child case num": 2, - "config": "release", - "env_tag": "UT_T2_1"}) + parser = argparse.ArgumentParser() + parser.add_argument( + '--repeat', '-r', + help='Number of repetitions for the test(s). Default is 1.', + type=int, + default=1 + ) + parser.add_argument("--env_config_file", "-e", + help="test env config file", + default=None + ) + parser.add_argument("--app_bin", "-b", + help="application binary file for flashing the chip", + default=None + ) + parser.add_argument( + 'test', + help='Comma separated list of