From 783cb692e8da551afaee1e58d0ad678b4186fc98 Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Thu, 13 Dec 2018 16:58:34 +0800 Subject: [PATCH 1/3] unit-test-app: support passing parameter with signals --- docs/en/api-guides/unit-tests.rst | 26 +++++-- docs/zh_CN/api-guides/unit-tests.rst | 23 ++++-- .../test_utils/include/test_utils.h | 60 ++++++++++++++-- .../components/test_utils/test_utils.c | 70 +++++++++++++------ tools/unit-test-app/unit_test.py | 24 ++++--- 5 files changed, 160 insertions(+), 43 deletions(-) diff --git a/docs/en/api-guides/unit-tests.rst b/docs/en/api-guides/unit-tests.rst index b9cfaa39da..577bdd5881 100644 --- a/docs/en/api-guides/unit-tests.rst +++ b/docs/en/api-guides/unit-tests.rst @@ -75,13 +75,27 @@ As the secnario in the above example, slave should get GPIO level after master s DUT1 (master) console:: Waiting for signal: [output high level]! - Please press "Enter" key to once any board send this signal. + Please press "Enter" key once any board send this signal. DUT2 (slave) console:: Send signal: [output high level]! -Once the signal is set from DUT2, you need to press "Enter" on DUT1, then DUT1 unblocks from ``unity_wait_for_signal`` and starts to change GPIO level. +Once the signal is sent from DUT2, you need to press "Enter" on DUT1, then DUT1 unblocks from ``unity_wait_for_signal`` and starts to change GPIO level. + +Signals can also be used to pass parameters between multiple devices. For example, DUT1 want to know the MAC address of DUT2, so it can connect to DUT2. +In this case, ``unity_wait_for_signal_param`` and ``unity_send_signal_param`` can be used: + +DUT1 console:: + + Waiting for signal: [dut2 mac address]! + Please input parameter value from any board send this signal and press "Enter" key. + +DUT2 console:: + + Send signal: [dut2 mac address][10:20:30:40:50:60]! + +Once the signal is sent from DUT2, you need to input ``10:20:30:40:50:60`` on DUT1 and press "Enter". Then DUT1 will get the MAC address string of DUT2 and unblocks from ``unity_wait_for_signal_param``, start to connect to DUT2. Add multiple stages test cases @@ -165,13 +179,13 @@ Normal case will print the case name and description. Master slave cases will al Test cases can be run by inputting one of the following: -- Test case name in quotation marks to run a single test case +- Test case name in quotation marks (for example, ``"esp_ota_begin() verifies arguments"``) to run a single test case. -- Test case index to run a single test case +- Test case index (for example, ``1``) to run a single test case. -- Module name in square brackets to run all test cases for a specific module +- Module name in square brackets (for example, ``[cxx]``) to run all test cases for a specific module. -- An asterisk to run all test cases +- An asterisk (``*``) to run all test cases ``[multi_device]`` and ``[multi_stage]`` tags tell the test runner whether a test case is a multiple devices or multiple stages test case. These tags are automatically added by ```TEST_CASE_MULTIPLE_STAGES`` and ``TEST_CASE_MULTIPLE_DEVICES`` macros. diff --git a/docs/zh_CN/api-guides/unit-tests.rst b/docs/zh_CN/api-guides/unit-tests.rst index 1e78cd98a7..7924fc6beb 100644 --- a/docs/zh_CN/api-guides/unit-tests.rst +++ b/docs/zh_CN/api-guides/unit-tests.rst @@ -109,6 +109,21 @@ DUT2（slave）终端： 一旦 DUT2 发送了该信号，您需要在 DUT2 的终端输入回车，然后 DUT1 会从 ``unity_wait_for_signal`` 函数中解除阻塞，并开始更改 GPIO 的电平。 +信号也可以用来在不同 DUT 之间传递参数。例如，DUT1 希望能够拿到 DUT2 的 MAC 地址，来进行蓝牙连接。 +这时，我们可以使用 ``unity_wait_for_signal_param`` 以及 ``unity_send_signal_param``。 + +DUT1 终端:: + + Waiting for signal: [dut2 mac address]! + Please input parameter value from any board send this signal and press "Enter" key. + + +DUT2 终端:: + + Send signal: [dut2 mac address][10:20:30:40:50:60]! + +一旦 DUT2 发送信号，您需要在 DUT1 输入 ``10:20:30:40:50:60`` 及回车，然后 DUT1 会从 ``unity_wait_for_signal_param`` 中获取到蓝牙地址的字符串，并解除阻塞开始蓝牙连接。 + 添加多阶段测试用例 ------------------ @@ -203,13 +218,13 @@ DUT2（slave）终端： 可以输入以下任意一项来运行测试用例： -- 引号中的测试用例的名字，运行单个测试用例。 +- 引号中的测试用例的名字（例如 ``"esp_ota_begin() verifies arguments"``），运行单个测试用例。 -- 测试用例的序号，运行单个测试用例。 +- 测试用例的序号（例如 ``1``），运行单个测试用例。 -- 方括号中的模块名字，运行指定模块所有的测试用例。 +- 方括号中的模块名字（例如 ``[cxx]``），运行指定模块所有的测试用例。 -- 星号，运行所有测试用例。 +- 星号 (``*``)，运行所有测试用例。 ``[multi_device]`` 和 ``[multi_stage]`` 标签告诉测试运行者该用例是多设备测试还是多阶段测试。这些标签由 diff --git a/tools/unit-test-app/components/test_utils/include/test_utils.h b/tools/unit-test-app/components/test_utils/include/test_utils.h index a4b7eccb7b..67c7e0aa47 100644 --- a/tools/unit-test-app/components/test_utils/include/test_utils.h +++ b/tools/unit-test-app/components/test_utils/include/test_utils.h @@ -107,9 +107,8 @@ void unity_reset_leak_checks(void); */ void test_case_uses_tcpip(void); - /** - * @brief wait for signals. + * @brief wait for signals with parameters. * * for multiple devices test cases, DUT might need to wait for other DUTs before continue testing. * As all DUTs are independent, need user (or test script) interaction to make test synchronized. @@ -136,13 +135,64 @@ void test_case_uses_tcpip(void); * * Then we press Enter key on DUT1's console, DUT1 starts to read input and then test success. * + * Another example, we have 2 DUTs in multiple devices test, and DUT1 need to get DUT2's mac address to perform BT connection. + * DUT1 should call `unity_wait_for_signal_param("dut2 mac address", mac, 19);` to wait for DUT2's mac address. + * DUT2 should call `unity_send_signal_param("dut2 mac address", "10:20:30:40:50:60");` to send to DUT1 its mac address. + * According to the console logs: + * + * DUT1 console: + * + * ``` + * Waiting for signal: [dut2 mac address]! + * Please input parameter value from any board send this signal and press "Enter" key. + * ``` + * + * DUT2 console: + * + * ``` + * Send signal: [dut2 mac address][10:20:30:40:50:60]! + * ``` + * * @param signal_name signal name which DUT expected to wait before proceed testing + * @param parameter_buf buffer to receive parameter + * @param buf_len length of parameter_buf. + * Currently we have a limitation that it will write 1 extra byte at the end of string. + * We need to use a buffer with 2 bytes longer than actual string length. */ -void unity_wait_for_signal(const char* signal_name); +void unity_wait_for_signal_param(const char* signal_name, char *parameter_buf, uint8_t buf_len); /** - * @brief DUT send signal. + * @brief wait for signals. + * + * @param signal_name signal name which DUT expected to wait before proceed testing + */ +static inline void unity_wait_for_signal(const char* signal_name) +{ + unity_wait_for_signal_param(signal_name, NULL, 0); +} + +/** + * @brief DUT send signal and pass parameter to other devices. + * + * @param signal_name signal name which DUT send once it finished preparing. + * @param parameter a string to let remote device to receive. + */ +void unity_send_signal_param(const char* signal_name, const char *parameter); + +/** + * @brief DUT send signal with parameter. * * @param signal_name signal name which DUT send once it finished preparing. */ -void unity_send_signal(const char* signal_name); +static inline void unity_send_signal(const char* signal_name) +{ + unity_send_signal_param(signal_name, NULL); +} + +/** + * @brief convert mac string to mac address + * + * @param mac_str MAC address string with format "xx:xx:xx:xx:xx:xx" + * @param[out] mac_addr store converted MAC address + */ +bool unity_util_convert_mac_from_string(const char* mac_str, uint8_t *mac_addr); diff --git a/tools/unit-test-app/components/test_utils/test_utils.c b/tools/unit-test-app/components/test_utils/test_utils.c index 36aae4c291..e42843f76c 100644 --- a/tools/unit-test-app/components/test_utils/test_utils.c +++ b/tools/unit-test-app/components/test_utils/test_utils.c @@ -31,20 +31,6 @@ const esp_partition_t *get_test_data_partition() return result; } -// wait user to send "Enter" key -static void wait_user_control() -{ - char sign[5] = {0}; - while(strlen(sign) == 0) - { - /* Flush anything already in the RX buffer */ - while(uart_rx_one_char((uint8_t *) sign) == OK) { - } - /* Read line */ - UartRxString((uint8_t*) sign, sizeof(sign) - 1); - } -} - void test_case_uses_tcpip() { // Can be called more than once, does nothing on subsequent calls @@ -71,16 +57,60 @@ void test_case_uses_tcpip() unity_reset_leak_checks(); } +// wait user to send "Enter" key or input parameter +static void wait_user_control(char* parameter_buf, uint8_t buf_len) +{ + char *buffer = parameter_buf; + char sign[5]; + uint8_t buffer_len = buf_len - 1; + + if (parameter_buf == NULL) { + buffer = sign; + buffer_len = sizeof(sign) - 1; + } + // workaround that unity_gets (UartRxString) will not set '\0' correctly + bzero(buffer, buffer_len); + + unity_gets(buffer, buffer_len); +} + // signal functions, used for sync between unity DUTs for multiple devices cases -void unity_wait_for_signal(const char* signal_name) +void unity_wait_for_signal_param(const char* signal_name, char* parameter_buf, uint8_t buf_len) { - printf("Waiting for signal: [%s]!\n" - "Please press \"Enter\" key to once any board send this signal.\n", signal_name); - wait_user_control(); + printf("Waiting for signal: [%s]!\n", signal_name); + if (parameter_buf == NULL) { + printf("Please press \"Enter\" key once any board send this signal.\n"); + } else { + printf("Please input parameter value from any board send this signal and press \"Enter\" key.\n"); + } + wait_user_control(parameter_buf, buf_len); } -void unity_send_signal(const char* signal_name) +void unity_send_signal_param(const char* signal_name, const char *parameter) { - printf("Send signal: [%s]!\n", signal_name); + if (parameter == NULL) { + printf("Send signal: [%s]!\n", signal_name); + } else { + printf("Send signal: [%s][%s]!\n", signal_name, parameter); + } +} + +bool unity_util_convert_mac_from_string(const char* mac_str, uint8_t *mac_addr) +{ + uint8_t loop = 0; + uint8_t tmp = 0; + const char *start; + char *stop; + + for (loop = 0; loop < 6; loop++) { + start = mac_str + loop * 3; + tmp = strtol(start, &stop, 16); + if (stop - start == 2 && (*stop == ':' || (*stop == 0 && loop == 5))) { + mac_addr[loop] = tmp; + } else { + return false; + } + } + return true; } diff --git a/tools/unit-test-app/unit_test.py b/tools/unit-test-app/unit_test.py index 407f97fe44..4f4a814822 100755 --- a/tools/unit-test-app/unit_test.py +++ b/tools/unit-test-app/unit_test.py @@ -300,8 +300,8 @@ def run_unit_test_cases(env, extra_data): class Handler(threading.Thread): - WAIT_SIGNAL_PATTERN = re.compile(r'Waiting for signal: \[(.+)\]!') - SEND_SIGNAL_PATTERN = re.compile(r'Send signal: \[(.+)\]!') + WAIT_SIGNAL_PATTERN = re.compile(r'Waiting for signal: \[(.+)]!') + SEND_SIGNAL_PATTERN = re.compile(r'Send signal: \[([^]]+)](\[([^]]+)])?!') FINISH_PATTERN = re.compile(r"1 Tests (\d) Failures (\d) Ignored") def __init__(self, dut, sent_signal_list, lock, parent_case_name, child_case_index, timeout): @@ -348,15 +348,23 @@ class Handler(threading.Thread): Utility.console_log("Timeout in device for function: %s" % self.child_case_name, color="orange") break with self.lock: - if expected_signal in self.sent_signal_list: - self.dut.write(" ") - self.sent_signal_list.remove(expected_signal) - break - time.sleep(0.01) + for sent_signal in self.sent_signal_list: + if expected_signal == sent_signal["name"]: + self.dut.write(sent_signal["parameter"]) + self.sent_signal_list.remove(sent_signal) + break + else: + time.sleep(0.01) + continue + break def device_send_action(data): with self.lock: - self.sent_signal_list.append(data[0].encode('utf-8')) + self.sent_signal_list.append({ + "name": data[0].encode('utf-8'), + "parameter": "" if data[2] is None else data[2].encode('utf-8') + # no parameter means we only write EOL to DUT + }) def handle_device_test_finish(data): """ test finished without reset """ From 7c592073518404a66e5eeb4a7b59f58ab500e3a5 Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Wed, 19 Dec 2018 15:48:58 +0800 Subject: [PATCH 2/3] test: add test to check wifi conneciton between sta and softap --- components/esp32/test/test_wifi.c | 133 +++++++++++++++++++++++++++++- 1 file changed, 129 insertions(+), 4 deletions(-) diff --git a/components/esp32/test/test_wifi.c b/components/esp32/test/test_wifi.c index 833d5bbcaf..08996d9de3 100644 --- a/components/esp32/test/test_wifi.c +++ b/components/esp32/test/test_wifi.c @@ -1,6 +1,7 @@ /* Tests for the Wi-Fi */ +#include "string.h" #include "esp_system.h" #include "unity.h" #include "esp_system.h" @@ -11,12 +12,22 @@ #include "nvs_flash.h" #include "test_utils.h" #include "freertos/task.h" +#include "freertos/event_groups.h" static const char* TAG = "test_wifi"; #define DEFAULT_SSID "TEST_SSID" #define DEFAULT_PWD "TEST_PASS" +#define GOT_IP_EVENT 0x00000001 +#define DISCONNECT_EVENT 0x00000002 + +#define EVENT_HANDLER_FLAG_DO_NOT_AUTO_RECONNECT 0x00000001 + +static uint32_t wifi_event_handler_flag; + +static EventGroupHandle_t wifi_events; + static esp_err_t event_handler(void *ctx, system_event_t *event) { printf("ev_handle_called.\n"); @@ -30,10 +41,18 @@ static esp_err_t event_handler(void *ctx, system_event_t *event) ESP_LOGI(TAG, "SYSTEM_EVENT_STA_GOT_IP"); ESP_LOGI(TAG, "got ip:%s\n", ip4addr_ntoa(&event->event_info.got_ip.ip_info.ip)); + if (wifi_events) { + xEventGroupSetBits(wifi_events, GOT_IP_EVENT); + } break; case SYSTEM_EVENT_STA_DISCONNECTED: ESP_LOGI(TAG, "SYSTEM_EVENT_STA_DISCONNECTED"); - TEST_ESP_OK(esp_wifi_connect()); + if (! (EVENT_HANDLER_FLAG_DO_NOT_AUTO_RECONNECT & wifi_event_handler_flag) ) { + TEST_ESP_OK(esp_wifi_connect()); + } + if (wifi_events) { + xEventGroupSetBits(wifi_events, DISCONNECT_EVENT); + } break; default: break; @@ -129,8 +148,8 @@ static void start_wifi_as_softap(void) cfg.nvs_enable = false; wifi_config_t w_config = { - .ap.ssid = "default_ssid", - .ap.password = "default_password", + .ap.ssid = DEFAULT_SSID, + .ap.password = DEFAULT_PWD, .ap.ssid_len = 0, .ap.channel = 1, .ap.authmode = WIFI_AUTH_WPA2_PSK, @@ -139,21 +158,60 @@ static void start_wifi_as_softap(void) .ap.beacon_interval = 100, }; + TEST_ESP_OK(esp_event_loop_init(event_handler, NULL)); + + // can't deinit event loop, need to reset leak check + unity_reset_leak_checks(); + + if (wifi_events == NULL) { + wifi_events = xEventGroupCreate(); + } + TEST_ESP_OK(esp_wifi_init(&cfg)); TEST_ESP_OK(esp_wifi_set_mode(WIFI_MODE_AP)); TEST_ESP_OK(esp_wifi_set_config(WIFI_IF_AP, &w_config)); TEST_ESP_OK(esp_wifi_start()); +} + +static void start_wifi_as_sta(void) +{ + wifi_init_config_t cfg = WIFI_INIT_CONFIG_DEFAULT(); + cfg.nvs_enable = false; + + // do not auto connect + wifi_event_handler_flag |= EVENT_HANDLER_FLAG_DO_NOT_AUTO_RECONNECT; + TEST_ESP_OK(esp_event_loop_init(event_handler, NULL)); + + // can't deinit event loop, need to reset leak check + unity_reset_leak_checks(); + + if (wifi_events == NULL) { + wifi_events = xEventGroupCreate(); + } else { + xEventGroupClearBits(wifi_events, 0x00ffffff); + } + + TEST_ESP_OK(esp_wifi_init(&cfg)); + TEST_ESP_OK(esp_wifi_set_mode(WIFI_MODE_STA)); + TEST_ESP_OK(esp_wifi_start()); } static void stop_wifi(void) { + printf("stop wifi\n"); TEST_ESP_OK(esp_wifi_stop()); TEST_ESP_OK(esp_wifi_deinit()); + if (wifi_events) { + vEventGroupDelete(wifi_events); + wifi_events = NULL; + } + vTaskDelay(1000/portTICK_PERIOD_MS); } static void receive_ds2ds_packet(void) { + test_case_uses_tcpip(); start_wifi_as_softap(); unity_wait_for_signal("sender ready"); unity_send_signal("receiver ready"); @@ -161,7 +219,6 @@ static void receive_ds2ds_packet(void) // wait for sender to send packets vTaskDelay(1000/portTICK_PERIOD_MS); stop_wifi(); - } static const char ds2ds_pdu[] = { @@ -174,6 +231,7 @@ static const char ds2ds_pdu[] = { static void send_ds2ds_packet(void) { + test_case_uses_tcpip(); start_wifi_as_softap(); unity_send_signal("sender ready"); unity_wait_for_signal("receiver ready"); @@ -187,3 +245,70 @@ static void send_ds2ds_packet(void) } TEST_CASE_MULTIPLE_DEVICES("receive ds2ds packet without exception", "[wifi][test_env=UT_T2_1]", receive_ds2ds_packet, send_ds2ds_packet); + +static void wifi_connect_by_bssid(uint8_t *bssid) +{ + EventBits_t bits; + + wifi_config_t w_config = { + .sta.ssid = DEFAULT_SSID, + .sta.password = DEFAULT_PWD, + .sta.bssid_set = true, + }; + + memcpy(w_config.sta.bssid, bssid, 6); + + TEST_ESP_OK(esp_wifi_set_config(WIFI_IF_STA, &w_config)); + TEST_ESP_OK(esp_wifi_connect()); + bits = xEventGroupWaitBits(wifi_events, GOT_IP_EVENT, 1, 0, 5000/portTICK_RATE_MS); + TEST_ASSERT(bits == GOT_IP_EVENT); +} + +static void test_wifi_connection_sta(void) +{ + char mac_str[19]; + uint8_t mac[6]; + EventBits_t bits; + + test_case_uses_tcpip(); + + start_wifi_as_sta(); + + unity_wait_for_signal_param("SoftAP mac", mac_str, 19); + + TEST_ASSERT_TRUE(unity_util_convert_mac_from_string(mac_str, mac)); + + wifi_connect_by_bssid(mac); + + unity_send_signal("STA connected"); + + bits = xEventGroupWaitBits(wifi_events, DISCONNECT_EVENT, 1, 0, 60000 / portTICK_RATE_MS); + // disconnect event not triggered + printf("wait finish\n"); + TEST_ASSERT(bits == 0); + + stop_wifi(); +} + +static void test_wifi_connection_softap(void) +{ + char mac_str[19] = {0}; + uint8_t mac[6]; + + test_case_uses_tcpip(); + + start_wifi_as_softap(); + + TEST_ESP_OK(esp_wifi_get_mac(ESP_IF_WIFI_AP, mac)); + sprintf(mac_str, MACSTR, MAC2STR(mac)); + + unity_send_signal_param("SoftAP mac", mac_str); + + unity_wait_for_signal("STA connected"); + + vTaskDelay(60000 / portTICK_PERIOD_MS); + + stop_wifi(); +} + +TEST_CASE_MULTIPLE_DEVICES("test wifi retain connection for 60s", "[wifi][test_env=UT_T2_1][timeout=90]", test_wifi_connection_sta, test_wifi_connection_softap); From 431653bba87e69ba3d854ef34d450263ee1921db Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Thu, 20 Dec 2018 17:00:12 +0800 Subject: [PATCH 3/3] unit-test: fix 2 issues in unit test script: 1. Gitlab will only regard failure (not error) as failed case. Use log failure instead of log error, to display all error msg in test result 2. fix failed case not recognized by test script when failed during reset DUT --- tools/unit-test-app/unit_test.py | 40 +++++++++++++++++++------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/tools/unit-test-app/unit_test.py b/tools/unit-test-app/unit_test.py index 4f4a814822..86e352d7f3 100755 --- a/tools/unit-test-app/unit_test.py +++ b/tools/unit-test-app/unit_test.py @@ -64,7 +64,11 @@ MULTI_DEVICE_ID = 2 DEFAULT_TIMEOUT = 20 DUT_STARTUP_CHECK_RETRY_COUNT = 5 -TEST_HISTROY_CHECK_TIMEOUT = 1 +TEST_HISTORY_CHECK_TIMEOUT = 1 + + +class TestCaseFailed(AssertionError): + pass def format_test_case_config(test_case_data): @@ -163,7 +167,7 @@ def reset_dut(dut): for _ in range(DUT_STARTUP_CHECK_RETRY_COUNT): dut.write("-") try: - dut.expect("0 Tests 0 Failures 0 Ignored", timeout=TEST_HISTROY_CHECK_TIMEOUT) + dut.expect("0 Tests 0 Failures 0 Ignored", timeout=TEST_HISTORY_CHECK_TIMEOUT) break except ExpectTimeout: pass @@ -171,7 +175,7 @@ def reset_dut(dut): raise AssertionError("Reset {} ({}) failed!".format(dut.name, dut.port)) -def run_one_normal_case(dut, one_case, junit_test_case, failed_cases): +def run_one_normal_case(dut, one_case, junit_test_case): reset_dut(dut) @@ -194,9 +198,9 @@ def run_one_normal_case(dut, one_case, junit_test_case, failed_cases): if result: Utility.console_log("Success: " + one_case["name"], color="green") else: - failed_cases.append(one_case["name"]) Utility.console_log("Failed: " + one_case["name"], color="red") junit_test_case.add_failure_info(output) + raise TestCaseFailed() def handle_exception_reset(data): """ @@ -242,7 +246,7 @@ def run_one_normal_case(dut, one_case, junit_test_case, failed_cases): timeout=one_case["timeout"]) except ExpectTimeout: Utility.console_log("Timeout in expect", color="orange") - junit_test_case.add_error_info("timeout") + junit_test_case.add_failure_info("timeout") one_case_finish(False) break @@ -284,10 +288,12 @@ def run_unit_test_cases(env, extra_data): # create junit report test case junit_test_case = TinyFW.JunitReport.create_test_case("[{}] {}".format(ut_config, one_case["name"])) try: - run_one_normal_case(dut, one_case, junit_test_case, failed_cases) - TinyFW.JunitReport.test_case_finish(junit_test_case) + run_one_normal_case(dut, one_case, junit_test_case) + except TestCaseFailed: + failed_cases.append(one_case["name"]) except Exception as e: - junit_test_case.add_error_info("Unexpected exception: " + str(e)) + junit_test_case.add_failure_info("Unexpected exception: " + str(e)) + finally: TinyFW.JunitReport.test_case_finish(junit_test_case) # raise exception if any case fails @@ -476,9 +482,9 @@ def run_multiple_devices_cases(env, extra_data): try: run_one_multiple_devices_case(duts, ut_config, env, one_case, failed_cases, one_case.get('app_bin'), junit_test_case) - TinyFW.JunitReport.test_case_finish(junit_test_case) except Exception as e: - junit_test_case.add_error_info("Unexpected exception: " + str(e)) + junit_test_case.add_failure_info("Unexpected exception: " + str(e)) + finally: TinyFW.JunitReport.test_case_finish(junit_test_case) if failed_cases: @@ -488,7 +494,7 @@ def run_multiple_devices_cases(env, extra_data): raise AssertionError("Unit Test Failed") -def run_one_multiple_stage_case(dut, one_case, failed_cases, junit_test_case): +def run_one_multiple_stage_case(dut, one_case, junit_test_case): reset_dut(dut) dut.start_capture_raw_data() @@ -523,7 +529,7 @@ def run_one_multiple_stage_case(dut, one_case, failed_cases, junit_test_case): err_msg = "Reset Check Failed: \r\n\tExpected: {}\r\n\tGet: {}".format(one_case["reset"], exception_reset_list) Utility.console_log(err_msg, color="orange") - junit_test_case.add_error_info(err_msg) + junit_test_case.add_failure_info(err_msg) else: # we allow omit reset in multi stage cases result = True @@ -538,9 +544,9 @@ def run_one_multiple_stage_case(dut, one_case, failed_cases, junit_test_case): if result: Utility.console_log("Success: " + one_case["name"], color="green") else: - failed_cases.append(one_case["name"]) Utility.console_log("Failed: " + one_case["name"], color="red") junit_test_case.add_failure_info(output) + raise TestCaseFailed() stage_finish.append("break") def handle_exception_reset(data): @@ -620,10 +626,12 @@ def run_multiple_stage_cases(env, extra_data): for one_case in case_config[ut_config]: junit_test_case = TinyFW.JunitReport.create_test_case("[{}] {}".format(ut_config, one_case["name"])) try: - run_one_multiple_stage_case(dut, one_case, failed_cases, junit_test_case) - TinyFW.JunitReport.test_case_finish(junit_test_case) + run_one_multiple_stage_case(dut, one_case, junit_test_case) + except TestCaseFailed: + failed_cases.append(one_case["name"]) except Exception as e: - junit_test_case.add_error_info("Unexpected exception: " + str(e)) + junit_test_case.add_failure_info("Unexpected exception: " + str(e)) + finally: TinyFW.JunitReport.test_case_finish(junit_test_case) # raise exception if any case fails