wear_levelling: move tests from unit-test-app to a component test app

* Migrate test cases from IDF test runner to Unity fixture.
* Tighten heap checks in the test case a bit.
* Run formatting script on test_wl.c
* NEW: Define 4 test configurations, including configs with 512 byte
  sector size. Previously these configs weren't tested in CI.
* NEW: The test app only runs for ESP32 and ESP32-C3 (one chip for
  each architecture). The component is pretty high level so we don't
  need to test it for each chip. This reduces the load on CI.

components/wear_levelling/test/component.mk

@@ -1,2 +0,0 @@
-COMPONENT_ADD_LDFLAGS = -Wl,--whole-archive -l$(COMPONENT_NAME) -Wl,--no-whole-archive
-COMPONENT_EMBED_FILES := test_partition_v1.bin

components/wear_levelling/test_apps/CMakeLists.txt

@@ -0,0 +1,8 @@
+# This is the project CMakeLists.txt file for the test subproject
+cmake_minimum_required(VERSION 3.5)
+
+# "Trim" the build. Include the minimal set of components, main, and anything it depends on.
+set(COMPONENTS main)
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+project(wear_levelling_test)

components/wear_levelling/test_apps/README.md

@@ -0,0 +1,33 @@
+| Supported Targets | ESP32 | ESP32-C3 |
+| ----------------- | ----- | -------- |
+
+This is a test app for wear_levelling component. This app is for internal use.
+In CI, it is sufficient to run this test for one chip of each architecture.
+
+# Building
+Several configurations are provided as `sdkconfig.ci.XXX` and serve as a template.
+
+## Example with configuration "4k" for target ESP32
+
+```bash
+rm -rf sdkconfig build
+idf.py -DIDF_TARGET=esp32 -DSDKCONFIG_DEFAULTS="sdkconfig.defaults;sdkconfig.ci.4k" build
+```
+
+# Running
+
+To run locally:
+
+```bash
+idf.py flash monitor
+```
+
+The tests will be executed and the summary will be printed:
+
+```
+-----------------------
+4 Tests 0 Failures 0 Ignored 
+OK
+```
+
+Note, when the Python test script is executed in internal CI, it will test each configuration one by one. When executing this script locally, it will use whichever binary is already built and available in `build` directory.

components/wear_levelling/test_apps/component_ut_test.py

@@ -0,0 +1,28 @@
+# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+import glob
+import os
+
+import ttfw_idf
+from tiny_test_fw import Utility
+
+
+@ttfw_idf.idf_component_unit_test(env_tag='COMPONENT_UT_GENERIC', target=['esp32', 'esp32c3'])
+def test_component_ut_wear_levelling(env, _):   # type: (ttfw_idf.TinyFW.Env, None) -> None
+    # Get the names of all configs (sdkconfig.ci.* files)
+    config_files = glob.glob(os.path.join(os.path.dirname(__file__), 'sdkconfig.ci.*'))
+    config_names = [os.path.basename(s).replace('sdkconfig.ci.', '') for s in config_files]
+
+    # Run test once with binaries built for each config
+    for name in config_names:
+        Utility.console_log("Checking config \"{}\"... ".format(name), end='')
+        dut = env.get_dut('wear_levelling', 'components/wear_levelling/test_apps', app_config_name=name)
+        dut.start_app()
+        stdout = dut.expect('Tests finished', full_stdout=True, timeout=30)
+        ttfw_idf.ComponentUTResult.parse_result(stdout)
+        env.close_dut(dut.name)
+        Utility.console_log('done')
+
+
+if __name__ == '__main__':
+    test_component_ut_wear_levelling()

components/wear_levelling/test_apps/main/CMakeLists.txt

@@ -1,5 +1,5 @@
-idf_component_register(SRC_DIRS .
+idf_component_register(SRCS test_wl.c
                        PRIV_INCLUDE_DIRS .
-                       PRIV_REQUIRES cmock test_utils wear_levelling
+                       PRIV_REQUIRES wear_levelling unity
                        EMBED_FILES test_partition_v1.bin
                       )

components/wear_levelling/test_apps/main/test_wl.c

@@ -1,44 +1,56 @@
-#include "sdkconfig.h"
+/*
+ * SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 #include <string.h>
 #include "unity.h"
+#include "unity_fixture.h"
 #include "wear_levelling.h"
-#include "test_utils.h"
 #include "freertos/FreeRTOS.h"
-#include "freertos/portable.h"
 #include "freertos/task.h"
 #include "freertos/semphr.h"
-#if CONFIG_IDF_TARGET_ESP32
-#include "esp32/clk.h"
-#elif CONFIG_IDF_TARGET_ESP32S2
-#include "esp32s2/clk.h"
-#elif CONFIG_IDF_TARGET_ESP32S3
-#include "esp32s3/clk.h"
-#elif CONFIG_IDF_TARGET_ESP32C3
-#include "esp32c3/clk.h"
-#endif
+#include "esp_private/esp_clk.h"
 #include "soc/cpu.h"
-#include "esp_rom_sys.h"
+#include "sdkconfig.h"
 
-TEST_CASE("wl_unmount doesn't leak memory", "[wear_levelling]")
+
+TEST_GROUP(wear_levelling);
+
+TEST_SETUP(wear_levelling)
+{
+}
+
+TEST_TEAR_DOWN(wear_levelling)
+{
+}
+
+static const esp_partition_t *get_test_data_partition(void)
+{
+    const esp_partition_t *result = esp_partition_find_first(ESP_PARTITION_TYPE_DATA,
+                                    ESP_PARTITION_SUBTYPE_ANY, "flash_test");
+    TEST_ASSERT_NOT_NULL(result); /* means partition table set wrong */
+    return result;
+}
+
+TEST(wear_levelling, wl_unmount_doesnt_leak_memory)
 {
     const esp_partition_t *partition = get_test_data_partition();
     wl_handle_t handle;
-    // dummy unmount is needed to initialize static lock in WL
-    wl_unmount(WL_INVALID_HANDLE);
+    // mount and unmount once to initialize static locks
+    TEST_ESP_OK(wl_mount(partition, &handle));
+    wl_unmount(handle);
+
+    // test that we didn't leak any memory on the next init/deinit
     size_t size_before = xPortGetFreeHeapSize();
     TEST_ESP_OK(wl_mount(partition, &handle));
     wl_unmount(handle);
     size_t size_after = xPortGetFreeHeapSize();
 
-    // Original code:
-    //TEST_ASSERT_EQUAL_HEX32(size_before, size_after);
-    // Workaround for problem with heap size calculation:
-    ptrdiff_t stack_diff = size_before - size_after;
-    stack_diff = abs(stack_diff);
-    if (stack_diff > 8) TEST_ASSERT_EQUAL(0, stack_diff);
+    TEST_ASSERT_EQUAL(size_before, size_after);
 }
 
-TEST_CASE("wl_mount check partition parameters", "[wear_levelling][ignore]")
+TEST(wear_levelling, wl_mount_checks_partition_params)
 {
     const esp_partition_t *test_partition = get_test_data_partition();
     esp_partition_t fake_partition;
@@ -49,19 +61,13 @@ TEST_CASE("wl_mount check partition parameters", "[wear_levelling][ignore]")
 
     esp_partition_erase_range(test_partition, 0, test_partition->size);
     // test small partition: result should be error
-    for (int i=0 ; i< 5 ; i++)
-    {
-        fake_partition.size = SPI_FLASH_SEC_SIZE*(i);
+    for (int i = 0; i < 5; i++) {
+        fake_partition.size = SPI_FLASH_SEC_SIZE * (i);
         size_before = xPortGetFreeHeapSize();
         TEST_ESP_ERR(ESP_ERR_INVALID_ARG, wl_mount(&fake_partition, &handle));
+        // test that we didn't leak any memory
         size_after = xPortGetFreeHeapSize();
-
-        // Original code:
-        //TEST_ASSERT_EQUAL_HEX32(size_before, size_after);
-        // Workaround for problem with heap size calculation:
-        ptrdiff_t stack_diff = size_before - size_after;
-        stack_diff = abs(stack_diff);
-        if (stack_diff > 8) TEST_ASSERT_EQUAL(0, stack_diff);
+        TEST_ASSERT_EQUAL_HEX32(size_before, size_after);
     }
 
     // test minimum size partition: result should be OK
@@ -69,15 +75,10 @@ TEST_CASE("wl_mount check partition parameters", "[wear_levelling][ignore]")
     size_before = xPortGetFreeHeapSize();
     TEST_ESP_OK(wl_mount(&fake_partition, &handle));
     wl_unmount(handle);
-    printf("Test done\n");
-    size_after = xPortGetFreeHeapSize();
 
-    // Original code:
-    //TEST_ASSERT_EQUAL_HEX32(size_before, size_after);
-    // Workaround for problem with heap size calculation:
-    ptrdiff_t stack_diff = size_before - size_after;
-    stack_diff = abs(stack_diff);
-    if (stack_diff > 8) TEST_ASSERT_EQUAL(0, stack_diff);
+    // test that we didn't leak any memory
+    size_after = xPortGetFreeHeapSize();
+    TEST_ASSERT_EQUAL_HEX32(size_before, size_after);
 }
 
 typedef struct {
@@ -100,9 +101,9 @@ typedef struct {
             .handle = handle_ \
         }
 
-static void read_write_task(void* param)
+static void read_write_task(void *param)
 {
-    read_write_test_arg_t* args = (read_write_test_arg_t*) param;
+    read_write_test_arg_t *args = (read_write_test_arg_t *) param;
     esp_err_t err;
     srand(args->seed);
     for (size_t i = 0; i < args->word_count; ++i) {
@@ -117,7 +118,7 @@ static void read_write_task(void* param)
             uint32_t rval;
             err = wl_read(args->handle, args->offset + i * sizeof(rval), &rval, sizeof(rval));
             if (err != ESP_OK || rval != val) {
-                esp_rom_printf("E: i=%d, cnt=%d rval=%d val=%d\n\n", i, args->word_count, rval, val);
+                printf("E: i=%d, cnt=%d rval=%d val=%d\n\n", i, args->word_count, rval, val);
                 args->result = ESP_FAIL;
                 goto done;
             }
@@ -131,7 +132,7 @@ done:
     vTaskDelete(NULL);
 }
 
-TEST_CASE("multiple tasks can access wl handle simultaneously", "[wear_levelling]")
+TEST(wear_levelling, multiple_tasks_single_handle)
 {
     const esp_partition_t *partition = get_test_data_partition();
     wl_handle_t handle;
@@ -139,8 +140,8 @@ TEST_CASE("multiple tasks can access wl handle simultaneously", "[wear_levelling
 
     size_t sector_size = wl_sector_size(handle);
     TEST_ESP_OK(wl_erase_range(handle, 0, sector_size * 8));
-    read_write_test_arg_t args1 = READ_WRITE_TEST_ARG_INIT(0, 1, handle, sector_size/sizeof(uint32_t));
-    read_write_test_arg_t args2 = READ_WRITE_TEST_ARG_INIT(sector_size, 2, handle, sector_size/sizeof(uint32_t));
+    read_write_test_arg_t args1 = READ_WRITE_TEST_ARG_INIT(0, 1, handle, sector_size / sizeof(uint32_t));
+    read_write_test_arg_t args2 = READ_WRITE_TEST_ARG_INIT(sector_size, 2, handle, sector_size / sizeof(uint32_t));
     const size_t stack_size = 8192;
 
     printf("writing 1 and 2\n");
@@ -158,8 +159,8 @@ TEST_CASE("multiple tasks can access wl handle simultaneously", "[wear_levelling
 
     args1.write = false;
     args2.write = false;
-    read_write_test_arg_t args3 = READ_WRITE_TEST_ARG_INIT(2 * sector_size, 3, handle, sector_size/sizeof(uint32_t));
-    read_write_test_arg_t args4 = READ_WRITE_TEST_ARG_INIT(3 * sector_size, 4, handle, sector_size/sizeof(uint32_t));
+    read_write_test_arg_t args3 = READ_WRITE_TEST_ARG_INIT(2 * sector_size, 3, handle, sector_size / sizeof(uint32_t));
+    read_write_test_arg_t args4 = READ_WRITE_TEST_ARG_INIT(3 * sector_size, 4, handle, sector_size / sizeof(uint32_t));
 
     printf("reading 1 and 2, writing 3 and 4\n");
     xTaskCreatePinnedToCore(&read_write_task, "rw3", stack_size, &args3, 3, NULL, cpuid_1);
@@ -189,14 +190,14 @@ TEST_CASE("multiple tasks can access wl handle simultaneously", "[wear_levelling
 
 #define TEST_SECTORS_COUNT 8
 
-static void check_mem_data(wl_handle_t handle, uint32_t init_val, uint32_t* buff)
+static void check_mem_data(wl_handle_t handle, uint32_t init_val, uint32_t *buff)
 {
     size_t sector_size = wl_sector_size(handle);
 
-    for (int m=0 ; m < TEST_SECTORS_COUNT ; m++) {
+    for (int m = 0; m < TEST_SECTORS_COUNT; m++) {
         TEST_ESP_OK(wl_read(handle, sector_size * m, buff, sector_size));
-        for (int i=0 ; i< sector_size/sizeof(uint32_t) ; i++) {
-            uint32_t compare_val = init_val + i +  m*sector_size;
+        for (int i = 0; i < sector_size / sizeof(uint32_t); i++) {
+            uint32_t compare_val = init_val + i +  m * sector_size;
             TEST_ASSERT_EQUAL( buff[i], compare_val);
         }
     }
@@ -207,13 +208,13 @@ static void check_mem_data(wl_handle_t handle, uint32_t init_val, uint32_t* buff
 // And then write one sector many times.
 // A data in other secors should be the same.
 // We do this also with unmount
-TEST_CASE("multiple write is correct", "[wear_levelling]")
+TEST(wear_levelling, write_doesnt_touch_other_sectors)
 {
     const esp_partition_t *partition = get_test_data_partition();
     esp_partition_t fake_partition;
     memcpy(&fake_partition, partition, sizeof(fake_partition));
 
-    fake_partition.size = SPI_FLASH_SEC_SIZE*(4 + TEST_SECTORS_COUNT);
+    fake_partition.size = SPI_FLASH_SEC_SIZE * (4 + TEST_SECTORS_COUNT);
 
     wl_handle_t handle;
     TEST_ESP_OK(wl_mount(&fake_partition, &handle));
@@ -226,13 +227,13 @@ TEST_CASE("multiple write is correct", "[wear_levelling]")
     // Set initial random value
     uint32_t init_val = rand();
 
-    uint32_t* buff = (uint32_t*)malloc(sector_size);
-    for (int m=0 ; m < TEST_SECTORS_COUNT ; m++) {
-        for (int i=0 ; i< sector_size/sizeof(uint32_t) ; i++) {
-            buff[i] = init_val + i +  m*sector_size;
+    uint32_t *buff = (uint32_t *)malloc(sector_size);
+    for (int m = 0; m < TEST_SECTORS_COUNT; m++) {
+        for (int i = 0; i < sector_size / sizeof(uint32_t); i++) {
+            buff[i] = init_val + i +  m * sector_size;
         }
-        TEST_ESP_OK(wl_erase_range(handle, sector_size*m, sector_size));
-        TEST_ESP_OK(wl_write(handle, sector_size*m, buff, sector_size));
+        TEST_ESP_OK(wl_erase_range(handle, sector_size * m, sector_size));
+        TEST_ESP_OK(wl_write(handle, sector_size * m, buff, sector_size));
     }
 
     check_mem_data(handle, init_val, buff);
@@ -241,13 +242,13 @@ TEST_CASE("multiple write is correct", "[wear_levelling]")
     start = cpu_hal_get_cycle_count();
 
 
-    for (int m=0 ; m< 100000 ; m++) {
+    for (int m = 0; m < 100000; m++) {
         uint32_t sector = m % TEST_SECTORS_COUNT;
-        for (int i=0 ; i< sector_size/sizeof(uint32_t) ; i++) {
-            buff[i] = init_val + i +  sector*sector_size;
+        for (int i = 0; i < sector_size / sizeof(uint32_t); i++) {
+            buff[i] = init_val + i +  sector * sector_size;
         }
-        TEST_ESP_OK(wl_erase_range(handle, sector_size*sector, sector_size));
-        TEST_ESP_OK(wl_write(handle, sector_size*sector, buff, sector_size));
+        TEST_ESP_OK(wl_erase_range(handle, sector_size * sector, sector_size));
+        TEST_ESP_OK(wl_write(handle, sector_size * sector, buff, sector_size));
         check_mem_data(handle, init_val, buff);
 
         uint32_t end;
@@ -263,6 +264,9 @@ TEST_CASE("multiple write is correct", "[wear_levelling]")
     wl_unmount(handle);
 }
 
+
+#if CONFIG_WL_SECTOR_SIZE_4096
+// This test runs for 4k sector size only, since the original (version 1) partition binary is generated this way
 extern const uint8_t test_partition_v1_bin_start[] asm("_binary_test_partition_v1_bin_start");
 extern const uint8_t test_partition_v1_bin_end[]   asm("_binary_test_partition_v1_bin_end");
 
@@ -270,15 +274,13 @@ extern const uint8_t test_partition_v1_bin_end[]   asm("_binary_test_partition_v
 
 // We write to partition prepared image with V1
 // Then we convert image to new version and verifying the data
-
-TEST_CASE("Version update test", "[wear_levelling]")
+TEST(wear_levelling, version_update)
 {
     const esp_partition_t *partition = get_test_data_partition();
     esp_partition_t fake_partition;
     memcpy(&fake_partition, partition, sizeof(fake_partition));
 
-    if (partition->encrypted)
-    {
+    if (partition->encrypted) {
         printf("Update from V1 to V2 will not work.\n");
         return;
     }
@@ -289,23 +291,21 @@ TEST_CASE("Version update test", "[wear_levelling]")
     esp_partition_erase_range(&fake_partition, 0, fake_partition.size);
 
     esp_partition_write(&fake_partition, 0, test_partition_v1_bin_start,  fake_partition.size);
-    for (int i=0 ; i< 3 ; i++)
-    {
+    for (int i = 0; i < 3; i++) {
         printf("Pass %i\n", i);
         wl_handle_t handle;
         TEST_ESP_OK(wl_mount(&fake_partition, &handle));
         size_t sector_size = wl_sector_size(handle);
-        uint32_t* buff = (uint32_t*)malloc(sector_size);
+        uint32_t *buff = (uint32_t *)malloc(sector_size);
 
         uint32_t init_val = COMPARE_START_CONST;
-        int test_count = fake_partition.size/sector_size - 4;
+        int test_count = fake_partition.size / sector_size - 4;
 
-        for (int m=0 ; m <  test_count; m++) {
+        for (int m = 0; m <  test_count; m++) {
             TEST_ESP_OK(wl_read(handle, sector_size * m, buff, sector_size));
-            for (int i=0 ; i< sector_size/sizeof(uint32_t) ; i++) {
-                uint32_t compare_val = init_val + i +  m*sector_size;
-                if (buff[i] != compare_val)
-                {
+            for (int i = 0; i < sector_size / sizeof(uint32_t); i++) {
+                uint32_t compare_val = init_val + i +  m * sector_size;
+                if (buff[i] != compare_val) {
                     printf("error compare: 0x%08x != 0x%08x \n", buff[i], compare_val);
                 }
                 TEST_ASSERT_EQUAL( buff[i], compare_val);
@@ -315,3 +315,21 @@ TEST_CASE("Version update test", "[wear_levelling]")
         wl_unmount(handle);
     }
 }
+#endif // CONFIG_WL_SECTOR_SIZE_4096
+
+TEST_GROUP_RUNNER(wear_levelling)
+{
+    RUN_TEST_CASE(wear_levelling, wl_unmount_doesnt_leak_memory)
+    RUN_TEST_CASE(wear_levelling, wl_mount_checks_partition_params)
+    RUN_TEST_CASE(wear_levelling, multiple_tasks_single_handle)
+    RUN_TEST_CASE(wear_levelling, write_doesnt_touch_other_sectors)
+
+#if CONFIG_WL_SECTOR_SIZE_4096
+    RUN_TEST_CASE(wear_levelling, version_update)
+#endif
+}
+
+void app_main(void)
+{
+    UNITY_MAIN(wear_levelling);
+}

components/wear_levelling/test_apps/partitions.csv

@@ -0,0 +1,5 @@
+# Name,     Type, SubType, Offset,   Size, Flags
+# Note: if you have increased the bootloader size, make sure to update the offsets to avoid overlap
+nvs,        data, nvs,      0x9000,  0x6000,
+factory,    0,    0,        0x10000, 1M
+flash_test, data, fat,      ,        528K

components/wear_levelling/test_apps/sdkconfig.ci.4k

@@ -0,0 +1 @@
+CONFIG_WL_SECTOR_SIZE_4096=y

components/wear_levelling/test_apps/sdkconfig.ci.512perf

@@ -0,0 +1,2 @@
+CONFIG_WL_SECTOR_SIZE_512=y
+CONFIG_WL_SECTOR_MODE_PERF=y

components/wear_levelling/test_apps/sdkconfig.ci.512safe

@@ -0,0 +1,2 @@
+CONFIG_WL_SECTOR_SIZE_512=y
+CONFIG_WL_SECTOR_MODE_SAFE=y

components/wear_levelling/test_apps/sdkconfig.ci.release

@@ -0,0 +1 @@
+CONFIG_COMPILER_OPTIMIZATION_SIZE=y

components/wear_levelling/test_apps/sdkconfig.defaults

@@ -0,0 +1,15 @@
+# General options for additional checks
+CONFIG_HEAP_POISONING_COMPREHENSIVE=y
+CONFIG_COMPILER_WARN_WRITE_STRINGS=y
+CONFIG_BOOTLOADER_LOG_LEVEL_WARN=y
+CONFIG_FREERTOS_WATCHPOINT_END_OF_STACK=y
+CONFIG_COMPILER_STACK_CHECK_MODE_STRONG=y
+CONFIG_COMPILER_STACK_CHECK=y
+
+# Enable Unity fixture support
+CONFIG_UNITY_ENABLE_FIXTURE=y
+CONFIG_UNITY_ENABLE_IDF_TEST_RUNNER=n
+
+# Custom partition table for this test app
+CONFIG_PARTITION_TABLE_CUSTOM=y
+CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv"

tools/ci/check_copyright_ignore.txt

@@ -2565,7 +2565,6 @@ components/wear_levelling/private_include/WL_Ext_Perf.h
 components/wear_levelling/private_include/WL_Ext_Safe.h
 components/wear_levelling/private_include/WL_Flash.h
 components/wear_levelling/private_include/WL_State.h
-components/wear_levelling/test/test_wl.c
 components/wear_levelling/test_wl_host/esp_error_check_stub.cpp
 components/wear_levelling/test_wl_host/main.cpp
 components/wear_levelling/test_wl_host/sdkconfig/sdkconfig.h