From 9acd3b785664552e87b48ae5b0160cf5693c3809 Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Tue, 19 Sep 2023 17:18:39 +0200 Subject: [PATCH] feat(perfmon): migrate the tests from unit-test-app --- components/perfmon/test/CMakeLists.txt | 10 - components/perfmon/test/test_perfmon_ansi.c | 184 ------------------ .../perfmon/test_apps/.build-test-rules.yml | 6 + components/perfmon/test_apps/CMakeLists.txt | 9 + components/perfmon/test_apps/README.md | 19 ++ .../perfmon/test_apps/main/CMakeLists.txt | 4 + .../perfmon/test_apps/main/test_perfmon.c | 178 +++++++++++++++++ .../test_apps/main/test_perfmon_main.c | 48 +++++ .../perfmon/test_apps/pytest_perfmon_ut.py | 13 ++ .../perfmon/test_apps/sdkconfig.defaults | 1 + 10 files changed, 278 insertions(+), 194 deletions(-) delete mode 100644 components/perfmon/test/CMakeLists.txt delete mode 100644 components/perfmon/test/test_perfmon_ansi.c create mode 100644 components/perfmon/test_apps/.build-test-rules.yml create mode 100644 components/perfmon/test_apps/CMakeLists.txt create mode 100644 components/perfmon/test_apps/README.md create mode 100644 components/perfmon/test_apps/main/CMakeLists.txt create mode 100644 components/perfmon/test_apps/main/test_perfmon.c create mode 100644 components/perfmon/test_apps/main/test_perfmon_main.c create mode 100644 components/perfmon/test_apps/pytest_perfmon_ut.py create mode 100644 components/perfmon/test_apps/sdkconfig.defaults diff --git a/components/perfmon/test/CMakeLists.txt b/components/perfmon/test/CMakeLists.txt deleted file mode 100644 index c4a984da44..0000000000 --- a/components/perfmon/test/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -if(CONFIG_IDF_TARGET_ARCH_XTENSA) - list(APPEND src_dirs .) -endif() - -idf_component_register(SRC_DIRS ${src_dirs} - PRIV_INCLUDE_DIRS "." - PRIV_REQUIRES cmock xtensa perfmon) -if(CONFIG_IDF_TARGET_ARCH_XTENSA) - target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format") -endif() diff --git a/components/perfmon/test/test_perfmon_ansi.c b/components/perfmon/test/test_perfmon_ansi.c deleted file mode 100644 index f8f09022b8..0000000000 --- a/components/perfmon/test/test_perfmon_ansi.c +++ /dev/null @@ -1,184 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2018-2022 Espressif Systems (Shanghai) CO LTD - * - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include "unity.h" -#include "esp_log.h" -#include - -#include "perfmon.h" - -// These includes required only for the tests -#include "xtensa-debug-module.h" -#include "eri.h" - -static const char *TAG = "perfmon"; - - -TEST_CASE("Perfomance counter dump", "[perfmon]") -{ - - xtensa_perfmon_dump(); - xtensa_perfmon_stop(); - xtensa_perfmon_dump(); - xtensa_perfmon_init(0, 0, 0xffff, 0, 6); - xtensa_perfmon_dump(); - xtensa_perfmon_reset(0); - xtensa_perfmon_start(); - int pm_data[10]; - for (int i = 0 ; i < 10 ; i++) { - if (i == 4) { - xtensa_perfmon_reset(0); - xtensa_perfmon_start(); - } - if (i == 6) { - xtensa_perfmon_stop(); - } - if (i == 8) { - xtensa_perfmon_start(); - } - pm_data[i] = eri_read(ERI_PERFMON_PM0); - } - for (int i = 0 ; i < 10 ; i++) { - ESP_LOGI(TAG, "pm_data[%i]= %08x", i, pm_data[i]); - } - if (pm_data[4] > pm_data[3]) { - ESP_LOGE(TAG, "The functions xtensa_perfmon_reset and xtensa_perfmon_start are not working correct."); - ESP_LOGW(TAG, "pm_data[3]= %i, must be > pm_data[4]= %i", pm_data[3], pm_data[4]); - TEST_ESP_OK(ESP_FAIL); - } - if ( pm_data[6] != pm_data[7]) { - ESP_LOGE(TAG, "The xtensa_perfmon_stop functions is not working correct."); - ESP_LOGW(TAG, "pm_data[6]= %i, must be == pm_data[7]= %i", pm_data[6], pm_data[7]); - TEST_ESP_OK(ESP_FAIL); - } - if ( pm_data[7] == pm_data[8]) { - ESP_LOGE(TAG, "The xtensa_perfmon_start functions is not working correct."); - ESP_LOGW(TAG, "pm_data[7]= %i, must be < pm_data[8]= %i", pm_data[7], pm_data[8]); - TEST_ESP_OK(ESP_FAIL); - } - - xtensa_perfmon_stop(); -} - -static void test_call(void* params) -{ - for (int i = 0 ; i < 1000 ; i++) { - __asm__ __volatile__(" nop"); - } -} - -static bool callback_called = false; -static int callback_call_count = 0; -static void test_callback(void *params, uint32_t select, uint32_t mask, uint32_t value) -{ - ESP_LOGI("test", "test_callback select = %i, mask = %i, value = %i", select, mask, value); - callback_called = true; - callback_call_count++; -} - -TEST_CASE("Performacnce test callback", "[perfmon]") -{ - ESP_LOGI(TAG, "Initialize performance structure"); - xtensa_perfmon_config_t pm_config = {}; - pm_config.counters_size = sizeof(xtensa_perfmon_select_mask_all) / sizeof(uint32_t) / 2; - pm_config.select_mask = xtensa_perfmon_select_mask_all; - pm_config.repeat_count = 200; - pm_config.max_deviation = 1; - pm_config.call_function = test_call; - pm_config.callback = test_callback; - pm_config.callback_params = stdout; - pm_config.tracelevel = -1; // Trace all events - callback_called = false; - callback_call_count = 0; - xtensa_perfmon_exec(&pm_config); - - ESP_LOGI(TAG, "Callback count = %i", callback_call_count); - if (callback_call_count != pm_config.counters_size) { - ESP_LOGE(TAG, "The callback count is not correct."); - ESP_LOGW(TAG, "callback_call_count= %i, must be == pm_config.counters_size= %i", callback_call_count, pm_config.counters_size); - TEST_ESP_OK(ESP_FAIL); - } - if (ESP_OK != xtensa_perfmon_overflow(0)) - { - ESP_LOGE(TAG, "Perfmon 0 overflow detected!"); - TEST_ESP_OK(ESP_FAIL); - } - if (ESP_OK != xtensa_perfmon_overflow(1)) - { - ESP_LOGE(TAG, "Perfmon 1 overflow detected!"); - TEST_ESP_OK(ESP_FAIL); - } - if (false == callback_called) { - TEST_ESP_OK(ESP_FAIL); - } -} - -static void exec_callback(void *params) -{ - for (int i = 0 ; i < 100 ; i++) { - __asm__ __volatile__(" nop"); - } -} - -static const uint32_t test_dsp_table[] = { - XTPERF_CNT_CYCLES, XTPERF_MASK_CYCLES, // total cycles - XTPERF_CNT_INSN, XTPERF_MASK_INSN_ALL, // total instructions - XTPERF_CNT_D_LOAD_U1, XTPERF_MASK_D_LOAD_LOCAL_MEM, // Mem read - XTPERF_CNT_D_STORE_U1, XTPERF_MASK_D_STORE_LOCAL_MEM, // Mem write - XTPERF_CNT_BUBBLES, XTPERF_MASK_BUBBLES_ALL &(~XTPERF_MASK_BUBBLES_R_HOLD_REG_DEP), // wait for other reasons - XTPERF_CNT_BUBBLES, XTPERF_MASK_BUBBLES_R_HOLD_REG_DEP, // Wait for register dependency - XTPERF_CNT_OVERFLOW, XTPERF_MASK_OVERFLOW, // Last test cycle -}; - -TEST_CASE("Performance test for Empty callback", "[perfmon]") -{ - for (int i = 5 ; i < 10 ; i++) { - exec_callback(NULL); - ESP_LOGD(TAG, "Empty call passed."); - } - - ESP_LOGI(TAG, "Start first test"); - xtensa_perfmon_config_t pm_config = {}; - pm_config.counters_size = sizeof(xtensa_perfmon_select_mask_all) / sizeof(uint32_t) / 2; - pm_config.select_mask = xtensa_perfmon_select_mask_all; - pm_config.repeat_count = 200; - pm_config.max_deviation = 1; - pm_config.call_function = exec_callback; - pm_config.callback = xtensa_perfmon_view_cb; - pm_config.callback_params = stdout; - pm_config.tracelevel = -1; - - xtensa_perfmon_exec(&pm_config); - callback_call_count = 0; - ESP_LOGI(TAG, "Start second test"); - pm_config.counters_size = sizeof(test_dsp_table) / sizeof(uint32_t) / 2; - pm_config.select_mask = test_dsp_table; - pm_config.repeat_count = 200; - pm_config.max_deviation = 1; - pm_config.call_function = exec_callback; - pm_config.callback = xtensa_perfmon_view_cb; - pm_config.callback_params = stdout; - pm_config.tracelevel = -1; - - xtensa_perfmon_exec(&pm_config); - callback_call_count = 0; - ESP_LOGI(TAG, "Start third test"); - pm_config.counters_size = sizeof(test_dsp_table) / sizeof(uint32_t) / 2; - pm_config.select_mask = test_dsp_table; - pm_config.repeat_count = 200; - pm_config.max_deviation = 1; - pm_config.call_function = exec_callback; - pm_config.callback = test_callback; - pm_config.callback_params = stdout; - pm_config.tracelevel = -1; - - xtensa_perfmon_exec(&pm_config); - if (callback_call_count != pm_config.counters_size) { - TEST_ESP_OK(ESP_FAIL); - } - ESP_LOGI(TAG, "All tests passed."); -} diff --git a/components/perfmon/test_apps/.build-test-rules.yml b/components/perfmon/test_apps/.build-test-rules.yml new file mode 100644 index 0000000000..f89ac8c269 --- /dev/null +++ b/components/perfmon/test_apps/.build-test-rules.yml @@ -0,0 +1,6 @@ +# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps + +components/perfmon/test_apps: + enable: + - if: IDF_TARGET in ["esp32", "esp32s2", "esp32s3"] + reason: Perfmon is only supported on Xtensa diff --git a/components/perfmon/test_apps/CMakeLists.txt b/components/perfmon/test_apps/CMakeLists.txt new file mode 100644 index 0000000000..b9d294924a --- /dev/null +++ b/components/perfmon/test_apps/CMakeLists.txt @@ -0,0 +1,9 @@ +cmake_minimum_required(VERSION 3.16) + +include($ENV{IDF_PATH}/tools/cmake/project.cmake) +set(COMPONENTS main) +list(PREPEND SDKCONFIG_DEFAULTS + "$ENV{IDF_PATH}/tools/test_apps/configs/sdkconfig.debug_helpers" + "sdkconfig.defaults") + +project(perfmon_test) diff --git a/components/perfmon/test_apps/README.md b/components/perfmon/test_apps/README.md new file mode 100644 index 0000000000..b6a2613b40 --- /dev/null +++ b/components/perfmon/test_apps/README.md @@ -0,0 +1,19 @@ +| Supported Targets | ESP32 | ESP32-S2 | ESP32-S3 | +| ----------------- | ----- | -------- | -------- | + +# Perfmon test + +To build and run this test app, using esp32s3 target for example: + +```bash +idf.py set-target esp32s3 +idf.py build flash monitor +``` + +To run tests using pytest: + +```bash +idf.py set-target esp32s3 +idf.py build +pytest --target=esp32s3 +``` diff --git a/components/perfmon/test_apps/main/CMakeLists.txt b/components/perfmon/test_apps/main/CMakeLists.txt new file mode 100644 index 0000000000..eb7cda692d --- /dev/null +++ b/components/perfmon/test_apps/main/CMakeLists.txt @@ -0,0 +1,4 @@ +idf_component_register(SRCS "test_perfmon_main.c" "test_perfmon.c" + INCLUDE_DIRS "." + PRIV_REQUIRES perfmon xtensa unity + WHOLE_ARCHIVE) diff --git a/components/perfmon/test_apps/main/test_perfmon.c b/components/perfmon/test_apps/main/test_perfmon.c new file mode 100644 index 0000000000..f061365c10 --- /dev/null +++ b/components/perfmon/test_apps/main/test_perfmon.c @@ -0,0 +1,178 @@ +/* + * SPDX-FileCopyrightText: 2018-2023 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include "esp_log.h" +#include "perfmon.h" +#include "unity.h" + +#include "xtensa-debug-module.h" +#include "eri.h" +#include "xtensa_perfmon_access.h" + +static const char *TAG = "perfmon"; + +static void delay(void) +{ + for (int i = 0 ; i < 1000 ; i++) { + __asm__ __volatile__("nop"); + } +} + +TEST_CASE("Start/stop/reset sanity check", "[perfmon]") +{ + xtensa_perfmon_stop(); + xtensa_perfmon_init(0, 0, 0xffff, 0, 6); + + xtensa_perfmon_reset(0); + delay(); + uint32_t count_0 = eri_read(ERI_PERFMON_PM0); + TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, count_0, "Counter should be 0 after reset"); + + xtensa_perfmon_start(); + delay(); + uint32_t count_1 = eri_read(ERI_PERFMON_PM0); + TEST_ASSERT_GREATER_THAN_UINT32_MESSAGE(0, count_1, "Counter should not be 0 after start"); + + xtensa_perfmon_stop(); + uint32_t count_2 = eri_read(ERI_PERFMON_PM0); + delay(); + uint32_t count_3 = eri_read(ERI_PERFMON_PM0); + TEST_ASSERT_EQUAL_UINT32_MESSAGE(count_2, count_3, "Counter should not change after stop"); + + xtensa_perfmon_reset(0); + xtensa_perfmon_start(); + delay(); + delay(); + uint32_t count_4 = eri_read(ERI_PERFMON_PM0); + TEST_ASSERT_GREATER_THAN_UINT32_MESSAGE(count_1, count_4, "Counter should be greater when delay is longer"); + + xtensa_perfmon_stop(); +} + +static void test_call(void *params) +{ + delay(); +} + +static void test_callback(void *params, uint32_t select, uint32_t mask, uint32_t value) +{ + int *call_count = (int *)params; + ESP_LOGI(TAG, "%s: select=%" PRIu32 ", mask=%" PRIx32 ", value=%" PRIu32, + __func__, select, mask, value); + (*call_count)++; +} + +TEST_CASE("xtensa_perfmon_exec custom callback", "[perfmon]") +{ + int num_counters = sizeof(xtensa_perfmon_select_mask_all) / sizeof(xtensa_perfmon_select_mask_all[0]) / 2; + int callback_call_count = 0; + xtensa_perfmon_config_t pm_config = { + .counters_size = num_counters, + .select_mask = xtensa_perfmon_select_mask_all, + .repeat_count = 200, + .max_deviation = 1, + .call_function = test_call, + .call_params = NULL, + .callback = test_callback, + .callback_params = &callback_call_count, + .tracelevel = -1 + }; + TEST_ESP_OK(xtensa_perfmon_exec(&pm_config)); + + TEST_ASSERT_NOT_EQUAL_MESSAGE(0, callback_call_count, "Callback should be called at least once"); + // Performance counters should not have overflow status set + TEST_ESP_OK(xtensa_perfmon_overflow(0)); + TEST_ESP_OK(xtensa_perfmon_overflow(1)); + TEST_ASSERT_EQUAL_MESSAGE(num_counters, callback_call_count, + "Callback should be called once for every counter"); +} + +TEST_CASE("xtensa_perfmon_view_cb test", "[perfmon]") +{ + const uint32_t test_table[] = { + XTPERF_CNT_CYCLES, XTPERF_MASK_CYCLES, // total cycles + XTPERF_CNT_INSN, XTPERF_MASK_INSN_ALL, // total instructions + XTPERF_CNT_D_LOAD_U1, XTPERF_MASK_D_LOAD_LOCAL_MEM, // Mem read + XTPERF_CNT_D_STORE_U1, XTPERF_MASK_D_STORE_LOCAL_MEM, // Mem write + XTPERF_CNT_BUBBLES, XTPERF_MASK_BUBBLES_ALL &(~XTPERF_MASK_BUBBLES_R_HOLD_REG_DEP), // wait for other reasons + XTPERF_CNT_BUBBLES, XTPERF_MASK_BUBBLES_R_HOLD_REG_DEP, // Wait for register dependency + XTPERF_CNT_OVERFLOW, XTPERF_MASK_OVERFLOW, // Last test cycle + }; + int num_counters = sizeof(test_table) / sizeof(test_table[0]) / 2; + + // We will collect the output of xtensa_perfmon_view_cb in a string + // and check that the output matches the counters table above. + char *out_str = NULL; + size_t out_len = 0; + FILE *out_stream = open_memstream(&out_str, &out_len); + + xtensa_perfmon_config_t pm_config = { + .counters_size = num_counters, + .select_mask = test_table, + .repeat_count = 200, + .max_deviation = 1, + .call_function = test_call, + .call_params = NULL, + .callback = xtensa_perfmon_view_cb, + .callback_params = out_stream, + .tracelevel = -1, + }; + + TEST_ESP_OK(xtensa_perfmon_exec(&pm_config)); + fclose(out_stream); + + TEST_ASSERT_MESSAGE(strlen(out_str) > 0, "xtensa_perfmon_view_cb should print something"); + // Check that performance counters defined in test_table are present in the output: + const char *p = out_str; + + // 1. XTPERF_CNT_CYCLES + p = strstr(p, "Value ="); + TEST_ASSERT_NOT_NULL(p); + p = strstr(p, "Counts cycles."); + TEST_ASSERT_NOT_NULL(p); + + // 2. XTPERF_CNT_INSN + p = strstr(p, "Value ="); + TEST_ASSERT_NOT_NULL(p); + p = strstr(p, "Successfully Retired Instructions."); + + // 3. XTPERF_CNT_D_LOAD_U1 + p = strstr(p, "Value ="); + TEST_ASSERT_NOT_NULL(p); + p = strstr(p, "Load Instruction (Data Memory)."); + + // 4. XTPERF_CNT_D_STORE_U1 + p = strstr(p, "Value ="); + TEST_ASSERT_NOT_NULL(p); + p = strstr(p, "Store Instruction (Data Memory)."); + TEST_ASSERT_NOT_NULL(p); + + // 5. XTPERF_CNT_BUBBLES + p = strstr(p, "Value ="); + TEST_ASSERT_NOT_NULL(p); + p = strstr(p, "Hold and Other Bubble cycles."); + TEST_ASSERT_NOT_NULL(p); + p = strstr(p, "CTI bubble (e.g. branch delay slot)"); + + // 6. XTPERF_CNT_BUBBLES (with a different mask) + p = strstr(p, "Value ="); + TEST_ASSERT_NOT_NULL(p); + p = strstr(p, "Hold and Other Bubble cycles."); + TEST_ASSERT_NOT_NULL(p); + p = strstr(p, "R hold caused by register dependency"); + + // 7. XTPERF_CNT_OVERFLOW + p = strstr(p, "Value ="); + TEST_ASSERT_NOT_NULL(p); + p = strstr(p, "Overflow counter"); + TEST_ASSERT_NOT_NULL(p); + + free(out_str); +} diff --git a/components/perfmon/test_apps/main/test_perfmon_main.c b/components/perfmon/test_apps/main/test_perfmon_main.c new file mode 100644 index 0000000000..656bf5596c --- /dev/null +++ b/components/perfmon/test_apps/main/test_perfmon_main.c @@ -0,0 +1,48 @@ +/* + * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "unity.h" +#include "unity_test_runner.h" +#include "esp_heap_caps.h" + +#define TEST_MEMORY_LEAK_THRESHOLD_DEFAULT 0 +static int leak_threshold = TEST_MEMORY_LEAK_THRESHOLD_DEFAULT; +void set_leak_threshold(int threshold) +{ + leak_threshold = threshold; +} + +static size_t before_free_8bit; +static size_t before_free_32bit; + +static void check_leak(size_t before_free, size_t after_free, const char *type) +{ + ssize_t delta = after_free - before_free; + printf("MALLOC_CAP_%s: Before %u bytes free, After %u bytes free (delta %d)\n", type, before_free, after_free, delta); + TEST_ASSERT_MESSAGE(delta >= leak_threshold, "memory leak"); +} + +void setUp(void) +{ + before_free_8bit = heap_caps_get_free_size(MALLOC_CAP_8BIT); + before_free_32bit = heap_caps_get_free_size(MALLOC_CAP_32BIT); +} + +void tearDown(void) +{ + size_t after_free_8bit = heap_caps_get_free_size(MALLOC_CAP_8BIT); + size_t after_free_32bit = heap_caps_get_free_size(MALLOC_CAP_32BIT); + check_leak(before_free_8bit, after_free_8bit, "8BIT"); + check_leak(before_free_32bit, after_free_32bit, "32BIT"); + + leak_threshold = TEST_MEMORY_LEAK_THRESHOLD_DEFAULT; +} + +void app_main(void) +{ + printf("Running perfmon component tests\n"); + unity_run_menu(); +} diff --git a/components/perfmon/test_apps/pytest_perfmon_ut.py b/components/perfmon/test_apps/pytest_perfmon_ut.py new file mode 100644 index 0000000000..32adeee597 --- /dev/null +++ b/components/perfmon/test_apps/pytest_perfmon_ut.py @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: CC0-1.0 + +import pytest +from pytest_embedded import Dut + + +@pytest.mark.generic +@pytest.mark.esp32 +@pytest.mark.esp32s2 +@pytest.mark.esp32s3 +def test_perfmon_ut(dut: Dut) -> None: + dut.run_all_single_board_cases() diff --git a/components/perfmon/test_apps/sdkconfig.defaults b/components/perfmon/test_apps/sdkconfig.defaults new file mode 100644 index 0000000000..7b569fa075 --- /dev/null +++ b/components/perfmon/test_apps/sdkconfig.defaults @@ -0,0 +1 @@ +CONFIG_ESP_TASK_WDT_CHECK_IDLE_TASK_CPU0=n