feat(perfmon): migrate the tests from unit-test-app

components/perfmon/test/CMakeLists.txt

@@ -1,10 +0,0 @@
-if(CONFIG_IDF_TARGET_ARCH_XTENSA)
-    list(APPEND src_dirs .)
-endif()
-
-idf_component_register(SRC_DIRS ${src_dirs}
-                    PRIV_INCLUDE_DIRS "."
-                    PRIV_REQUIRES cmock xtensa perfmon)
-if(CONFIG_IDF_TARGET_ARCH_XTENSA)
-    target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")
-endif()

components/perfmon/test/test_perfmon_ansi.c

@@ -1,184 +0,0 @@
-/*
- * SPDX-FileCopyrightText: 2018-2022 Espressif Systems (Shanghai) CO LTD
- *
- * SPDX-License-Identifier: Apache-2.0
- */
-
-#include <string.h>
-#include "unity.h"
-#include "esp_log.h"
-#include <stdlib.h>
-
-#include "perfmon.h"
-
-// These includes required only for the tests
-#include "xtensa-debug-module.h"
-#include "eri.h"
-
-static const char *TAG = "perfmon";
-
-
-TEST_CASE("Perfomance counter dump", "[perfmon]")
-{
-
-    xtensa_perfmon_dump();
-    xtensa_perfmon_stop();
-    xtensa_perfmon_dump();
-    xtensa_perfmon_init(0, 0, 0xffff, 0, 6);
-    xtensa_perfmon_dump();
-    xtensa_perfmon_reset(0);
-    xtensa_perfmon_start();
-    int pm_data[10];
-    for (int i = 0 ; i < 10 ; i++) {
-        if (i == 4) {
-            xtensa_perfmon_reset(0);
-            xtensa_perfmon_start();
-        }
-        if (i == 6) {
-            xtensa_perfmon_stop();
-        }
-        if (i == 8) {
-            xtensa_perfmon_start();
-        }
-        pm_data[i] = eri_read(ERI_PERFMON_PM0);
-    }
-    for (int i = 0 ; i < 10 ; i++) {
-        ESP_LOGI(TAG, "pm_data[%i]= %08x", i, pm_data[i]);
-    }
-    if (pm_data[4] > pm_data[3]) {
-        ESP_LOGE(TAG, "The functions xtensa_perfmon_reset and xtensa_perfmon_start are not working correct.");
-        ESP_LOGW(TAG, "pm_data[3]= %i, must be > pm_data[4]= %i", pm_data[3], pm_data[4]);
-        TEST_ESP_OK(ESP_FAIL);
-    }
-    if ( pm_data[6] !=  pm_data[7]) {
-        ESP_LOGE(TAG, "The xtensa_perfmon_stop functions is not working correct.");
-        ESP_LOGW(TAG, "pm_data[6]= %i, must be == pm_data[7]= %i", pm_data[6], pm_data[7]);
-        TEST_ESP_OK(ESP_FAIL);
-    }
-    if ( pm_data[7] ==  pm_data[8]) {
-        ESP_LOGE(TAG, "The xtensa_perfmon_start functions is not working correct.");
-        ESP_LOGW(TAG, "pm_data[7]= %i, must be < pm_data[8]= %i", pm_data[7], pm_data[8]);
-        TEST_ESP_OK(ESP_FAIL);
-    }
-
-    xtensa_perfmon_stop();
-}
-
-static void test_call(void* params)
-{
-    for (int i = 0 ; i < 1000 ; i++) {
-        __asm__ __volatile__("   nop");
-    }
-}
-
-static bool callback_called = false;
-static int callback_call_count = 0;
-static void test_callback(void *params, uint32_t select, uint32_t mask, uint32_t value)
-{
-    ESP_LOGI("test", "test_callback select = %i,  mask = %i, value = %i", select, mask, value);
-    callback_called = true;
-    callback_call_count++;
-}
-
-TEST_CASE("Performacnce test callback", "[perfmon]")
-{
-    ESP_LOGI(TAG, "Initialize performance structure");
-    xtensa_perfmon_config_t pm_config = {};
-    pm_config.counters_size = sizeof(xtensa_perfmon_select_mask_all) / sizeof(uint32_t) / 2;
-    pm_config.select_mask = xtensa_perfmon_select_mask_all;
-    pm_config.repeat_count = 200;
-    pm_config.max_deviation = 1;
-    pm_config.call_function = test_call;
-    pm_config.callback = test_callback;
-    pm_config.callback_params = stdout;
-    pm_config.tracelevel = -1; // Trace all events
-    callback_called = false;
-    callback_call_count = 0;
-    xtensa_perfmon_exec(&pm_config);
-
-    ESP_LOGI(TAG, "Callback count = %i", callback_call_count);
-    if (callback_call_count != pm_config.counters_size) {
-        ESP_LOGE(TAG, "The callback count is not correct.");
-        ESP_LOGW(TAG, "callback_call_count= %i, must be == pm_config.counters_size= %i", callback_call_count, pm_config.counters_size);
-        TEST_ESP_OK(ESP_FAIL);
-    }
-    if (ESP_OK != xtensa_perfmon_overflow(0))
-    {
-        ESP_LOGE(TAG, "Perfmon 0 overflow detected!");
-        TEST_ESP_OK(ESP_FAIL);
-    }
-    if (ESP_OK != xtensa_perfmon_overflow(1))
-    {
-        ESP_LOGE(TAG, "Perfmon 1 overflow detected!");
-        TEST_ESP_OK(ESP_FAIL);
-    }
-    if (false == callback_called) {
-        TEST_ESP_OK(ESP_FAIL);
-    }
-}
-
-static void exec_callback(void *params)
-{
-    for (int i = 0 ; i < 100 ; i++) {
-        __asm__ __volatile__("   nop");
-    }
-}
-
-static const uint32_t test_dsp_table[] = {
-    XTPERF_CNT_CYCLES, XTPERF_MASK_CYCLES, // total cycles
-    XTPERF_CNT_INSN, XTPERF_MASK_INSN_ALL, // total instructions
-    XTPERF_CNT_D_LOAD_U1, XTPERF_MASK_D_LOAD_LOCAL_MEM, // Mem read
-    XTPERF_CNT_D_STORE_U1, XTPERF_MASK_D_STORE_LOCAL_MEM, // Mem write
-    XTPERF_CNT_BUBBLES, XTPERF_MASK_BUBBLES_ALL &(~XTPERF_MASK_BUBBLES_R_HOLD_REG_DEP),  // wait for other reasons
-    XTPERF_CNT_BUBBLES, XTPERF_MASK_BUBBLES_R_HOLD_REG_DEP,           // Wait for register dependency
-    XTPERF_CNT_OVERFLOW, XTPERF_MASK_OVERFLOW,               // Last test cycle
-};
-
-TEST_CASE("Performance test for Empty callback", "[perfmon]")
-{
-    for (int i = 5 ; i < 10 ; i++) {
-        exec_callback(NULL);
-        ESP_LOGD(TAG, "Empty call passed.");
-    }
-
-    ESP_LOGI(TAG, "Start first test");
-    xtensa_perfmon_config_t pm_config = {};
-    pm_config.counters_size = sizeof(xtensa_perfmon_select_mask_all) / sizeof(uint32_t) / 2;
-    pm_config.select_mask = xtensa_perfmon_select_mask_all;
-    pm_config.repeat_count = 200;
-    pm_config.max_deviation = 1;
-    pm_config.call_function = exec_callback;
-    pm_config.callback = xtensa_perfmon_view_cb;
-    pm_config.callback_params = stdout;
-    pm_config.tracelevel = -1;
-
-    xtensa_perfmon_exec(&pm_config);
-    callback_call_count = 0;
-    ESP_LOGI(TAG, "Start second test");
-    pm_config.counters_size = sizeof(test_dsp_table) / sizeof(uint32_t) / 2;
-    pm_config.select_mask = test_dsp_table;
-    pm_config.repeat_count = 200;
-    pm_config.max_deviation = 1;
-    pm_config.call_function = exec_callback;
-    pm_config.callback = xtensa_perfmon_view_cb;
-    pm_config.callback_params = stdout;
-    pm_config.tracelevel = -1;
-
-    xtensa_perfmon_exec(&pm_config);
-    callback_call_count = 0;
-    ESP_LOGI(TAG, "Start third test");
-    pm_config.counters_size = sizeof(test_dsp_table) / sizeof(uint32_t) / 2;
-    pm_config.select_mask = test_dsp_table;
-    pm_config.repeat_count = 200;
-    pm_config.max_deviation = 1;
-    pm_config.call_function = exec_callback;
-    pm_config.callback = test_callback;
-    pm_config.callback_params = stdout;
-    pm_config.tracelevel = -1;
-
-    xtensa_perfmon_exec(&pm_config);
-    if (callback_call_count != pm_config.counters_size) {
-        TEST_ESP_OK(ESP_FAIL);
-    }
-    ESP_LOGI(TAG, "All tests passed.");
-}

components/perfmon/test_apps/.build-test-rules.yml

@@ -0,0 +1,6 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+components/perfmon/test_apps:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32s2", "esp32s3"]
+      reason: Perfmon is only supported on Xtensa

components/perfmon/test_apps/CMakeLists.txt

@@ -0,0 +1,9 @@
+cmake_minimum_required(VERSION 3.16)
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+set(COMPONENTS main)
+list(PREPEND SDKCONFIG_DEFAULTS
+    "$ENV{IDF_PATH}/tools/test_apps/configs/sdkconfig.debug_helpers"
+    "sdkconfig.defaults")
+
+project(perfmon_test)

components/perfmon/test_apps/README.md

@@ -0,0 +1,19 @@
+| Supported Targets | ESP32 | ESP32-S2 | ESP32-S3 |
+| ----------------- | ----- | -------- | -------- |
+
+# Perfmon test
+
+To build and run this test app, using esp32s3 target for example:
+
+```bash
+idf.py set-target esp32s3
+idf.py build flash monitor
+```
+
+To run tests using pytest:
+
+```bash
+idf.py set-target esp32s3
+idf.py build
+pytest --target=esp32s3
+```

components/perfmon/test_apps/main/CMakeLists.txt

@@ -0,0 +1,4 @@
+idf_component_register(SRCS "test_perfmon_main.c" "test_perfmon.c"
+                    INCLUDE_DIRS "."
+                    PRIV_REQUIRES perfmon xtensa unity
+                    WHOLE_ARCHIVE)

components/perfmon/test_apps/main/test_perfmon.c

@@ -0,0 +1,178 @@
+/*
+ * SPDX-FileCopyrightText: 2018-2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include <stddef.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include "esp_log.h"
+#include "perfmon.h"
+#include "unity.h"
+
+#include "xtensa-debug-module.h"
+#include "eri.h"
+#include "xtensa_perfmon_access.h"
+
+static const char *TAG = "perfmon";
+
+static void delay(void)
+{
+    for (int i = 0 ; i < 1000 ; i++) {
+        __asm__ __volatile__("nop");
+    }
+}
+
+TEST_CASE("Start/stop/reset sanity check", "[perfmon]")
+{
+    xtensa_perfmon_stop();
+    xtensa_perfmon_init(0, 0, 0xffff, 0, 6);
+
+    xtensa_perfmon_reset(0);
+    delay();
+    uint32_t count_0 = eri_read(ERI_PERFMON_PM0);
+    TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, count_0, "Counter should be 0 after reset");
+
+    xtensa_perfmon_start();
+    delay();
+    uint32_t count_1 = eri_read(ERI_PERFMON_PM0);
+    TEST_ASSERT_GREATER_THAN_UINT32_MESSAGE(0, count_1, "Counter should not be 0 after start");
+
+    xtensa_perfmon_stop();
+    uint32_t count_2 = eri_read(ERI_PERFMON_PM0);
+    delay();
+    uint32_t count_3 = eri_read(ERI_PERFMON_PM0);
+    TEST_ASSERT_EQUAL_UINT32_MESSAGE(count_2, count_3, "Counter should not change after stop");
+
+    xtensa_perfmon_reset(0);
+    xtensa_perfmon_start();
+    delay();
+    delay();
+    uint32_t count_4 = eri_read(ERI_PERFMON_PM0);
+    TEST_ASSERT_GREATER_THAN_UINT32_MESSAGE(count_1, count_4, "Counter should be greater when delay is longer");
+
+    xtensa_perfmon_stop();
+}
+
+static void test_call(void *params)
+{
+    delay();
+}
+
+static void test_callback(void *params, uint32_t select, uint32_t mask, uint32_t value)
+{
+    int *call_count = (int *)params;
+    ESP_LOGI(TAG, "%s: select=%" PRIu32 ",  mask=%" PRIx32 ", value=%" PRIu32,
+             __func__, select, mask, value);
+    (*call_count)++;
+}
+
+TEST_CASE("xtensa_perfmon_exec custom callback", "[perfmon]")
+{
+    int num_counters = sizeof(xtensa_perfmon_select_mask_all) / sizeof(xtensa_perfmon_select_mask_all[0]) / 2;
+    int callback_call_count = 0;
+    xtensa_perfmon_config_t pm_config = {
+        .counters_size = num_counters,
+        .select_mask = xtensa_perfmon_select_mask_all,
+        .repeat_count = 200,
+        .max_deviation = 1,
+        .call_function = test_call,
+        .call_params = NULL,
+        .callback = test_callback,
+        .callback_params = &callback_call_count,
+        .tracelevel = -1
+    };
+    TEST_ESP_OK(xtensa_perfmon_exec(&pm_config));
+
+    TEST_ASSERT_NOT_EQUAL_MESSAGE(0, callback_call_count, "Callback should be called at least once");
+    // Performance counters should not have overflow status set
+    TEST_ESP_OK(xtensa_perfmon_overflow(0));
+    TEST_ESP_OK(xtensa_perfmon_overflow(1));
+    TEST_ASSERT_EQUAL_MESSAGE(num_counters, callback_call_count,
+                              "Callback should be called once for every counter");
+}
+
+TEST_CASE("xtensa_perfmon_view_cb test", "[perfmon]")
+{
+    const uint32_t test_table[] = {
+        XTPERF_CNT_CYCLES, XTPERF_MASK_CYCLES, // total cycles
+        XTPERF_CNT_INSN, XTPERF_MASK_INSN_ALL, // total instructions
+        XTPERF_CNT_D_LOAD_U1, XTPERF_MASK_D_LOAD_LOCAL_MEM, // Mem read
+        XTPERF_CNT_D_STORE_U1, XTPERF_MASK_D_STORE_LOCAL_MEM, // Mem write
+        XTPERF_CNT_BUBBLES, XTPERF_MASK_BUBBLES_ALL &(~XTPERF_MASK_BUBBLES_R_HOLD_REG_DEP),  // wait for other reasons
+        XTPERF_CNT_BUBBLES, XTPERF_MASK_BUBBLES_R_HOLD_REG_DEP,           // Wait for register dependency
+        XTPERF_CNT_OVERFLOW, XTPERF_MASK_OVERFLOW,               // Last test cycle
+    };
+    int num_counters = sizeof(test_table) / sizeof(test_table[0]) / 2;
+
+    // We will collect the output of xtensa_perfmon_view_cb in a string
+    // and check that the output matches the counters table above.
+    char *out_str = NULL;
+    size_t out_len = 0;
+    FILE *out_stream = open_memstream(&out_str, &out_len);
+
+    xtensa_perfmon_config_t pm_config = {
+        .counters_size = num_counters,
+        .select_mask = test_table,
+        .repeat_count = 200,
+        .max_deviation = 1,
+        .call_function = test_call,
+        .call_params = NULL,
+        .callback = xtensa_perfmon_view_cb,
+        .callback_params = out_stream,
+        .tracelevel = -1,
+    };
+
+    TEST_ESP_OK(xtensa_perfmon_exec(&pm_config));
+    fclose(out_stream);
+
+    TEST_ASSERT_MESSAGE(strlen(out_str) > 0, "xtensa_perfmon_view_cb should print something");
+    // Check that performance counters defined in test_table are present in the output:
+    const char *p = out_str;
+
+    // 1. XTPERF_CNT_CYCLES
+    p = strstr(p, "Value =");
+    TEST_ASSERT_NOT_NULL(p);
+    p = strstr(p, "Counts cycles.");
+    TEST_ASSERT_NOT_NULL(p);
+
+    // 2. XTPERF_CNT_INSN
+    p = strstr(p, "Value =");
+    TEST_ASSERT_NOT_NULL(p);
+    p = strstr(p, "Successfully Retired Instructions.");
+
+    // 3. XTPERF_CNT_D_LOAD_U1
+    p = strstr(p, "Value =");
+    TEST_ASSERT_NOT_NULL(p);
+    p = strstr(p, "Load Instruction (Data Memory).");
+
+    // 4. XTPERF_CNT_D_STORE_U1
+    p = strstr(p, "Value =");
+    TEST_ASSERT_NOT_NULL(p);
+    p = strstr(p, "Store Instruction (Data Memory).");
+    TEST_ASSERT_NOT_NULL(p);
+
+    // 5. XTPERF_CNT_BUBBLES
+    p = strstr(p, "Value =");
+    TEST_ASSERT_NOT_NULL(p);
+    p = strstr(p, "Hold and Other Bubble cycles.");
+    TEST_ASSERT_NOT_NULL(p);
+    p = strstr(p, "CTI bubble (e.g. branch delay slot)");
+
+    // 6. XTPERF_CNT_BUBBLES (with a different mask)
+    p = strstr(p, "Value =");
+    TEST_ASSERT_NOT_NULL(p);
+    p = strstr(p, "Hold and Other Bubble cycles.");
+    TEST_ASSERT_NOT_NULL(p);
+    p = strstr(p, "R hold caused by register dependency");
+
+    // 7. XTPERF_CNT_OVERFLOW
+    p = strstr(p, "Value =");
+    TEST_ASSERT_NOT_NULL(p);
+    p = strstr(p, "Overflow counter");
+    TEST_ASSERT_NOT_NULL(p);
+
+    free(out_str);
+}

components/perfmon/test_apps/main/test_perfmon_main.c

@@ -0,0 +1,48 @@
+/*
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include "unity.h"
+#include "unity_test_runner.h"
+#include "esp_heap_caps.h"
+
+#define TEST_MEMORY_LEAK_THRESHOLD_DEFAULT 0
+static int leak_threshold = TEST_MEMORY_LEAK_THRESHOLD_DEFAULT;
+void set_leak_threshold(int threshold)
+{
+    leak_threshold = threshold;
+}
+
+static size_t before_free_8bit;
+static size_t before_free_32bit;
+
+static void check_leak(size_t before_free, size_t after_free, const char *type)
+{
+    ssize_t delta = after_free - before_free;
+    printf("MALLOC_CAP_%s: Before %u bytes free, After %u bytes free (delta %d)\n", type, before_free, after_free, delta);
+    TEST_ASSERT_MESSAGE(delta >= leak_threshold, "memory leak");
+}
+
+void setUp(void)
+{
+    before_free_8bit = heap_caps_get_free_size(MALLOC_CAP_8BIT);
+    before_free_32bit = heap_caps_get_free_size(MALLOC_CAP_32BIT);
+}
+
+void tearDown(void)
+{
+    size_t after_free_8bit = heap_caps_get_free_size(MALLOC_CAP_8BIT);
+    size_t after_free_32bit = heap_caps_get_free_size(MALLOC_CAP_32BIT);
+    check_leak(before_free_8bit, after_free_8bit, "8BIT");
+    check_leak(before_free_32bit, after_free_32bit, "32BIT");
+
+    leak_threshold = TEST_MEMORY_LEAK_THRESHOLD_DEFAULT;
+}
+
+void app_main(void)
+{
+    printf("Running perfmon component tests\n");
+    unity_run_menu();
+}

components/perfmon/test_apps/pytest_perfmon_ut.py

@@ -0,0 +1,13 @@
+# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: CC0-1.0
+
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.generic
+@pytest.mark.esp32
+@pytest.mark.esp32s2
+@pytest.mark.esp32s3
+def test_perfmon_ut(dut: Dut) -> None:
+    dut.run_all_single_board_cases()

components/perfmon/test_apps/sdkconfig.defaults

@@ -0,0 +1 @@
+CONFIG_ESP_TASK_WDT_CHECK_IDLE_TASK_CPU0=n