From 25603522e8d20ce6068f4bae7387316c8b744b2c Mon Sep 17 00:00:00 2001 From: gaoxiaojie Date: Fri, 21 Jul 2023 09:57:32 +0800 Subject: [PATCH] fix(esp32s3): patch Cache_WriteBack_Addr api Need to ensure that the cacheline being written back will not be accessed during the write back process. --- components/esp_rom/CMakeLists.txt | 4 + components/esp_rom/component.mk | 4 +- components/esp_rom/esp32s3/esp_rom_caps.h | 1 + components/esp_rom/esp32s3/ld/esp32s3.rom.ld | 2 +- components/esp_rom/patches/esp_rom_cache.c | 71 +++++++++ .../patches/esp_rom_cache_writeback_esp32s3.S | 135 ++++++++++++++++++ 6 files changed, 215 insertions(+), 2 deletions(-) create mode 100644 components/esp_rom/patches/esp_rom_cache_writeback_esp32s3.S diff --git a/components/esp_rom/CMakeLists.txt b/components/esp_rom/CMakeLists.txt index 991522eb4c..36cd409550 100644 --- a/components/esp_rom/CMakeLists.txt +++ b/components/esp_rom/CMakeLists.txt @@ -30,6 +30,10 @@ if(target STREQUAL "esp32s2" OR target STREQUAL "esp32s3") list(APPEND sources "patches/esp_rom_cache.c") endif() +if(target STREQUAL "esp32s3") + list(APPEND sources "patches/esp_rom_cache_writeback_esp32s3.S") +endif() + idf_component_register(SRCS ${sources} INCLUDE_DIRS ${include_dirs} PRIV_REQUIRES ${private_required_comp}) diff --git a/components/esp_rom/component.mk b/components/esp_rom/component.mk index f2cbf5e500..161c8ef480 100644 --- a/components/esp_rom/component.mk +++ b/components/esp_rom/component.mk @@ -1,8 +1,10 @@ COMPONENT_ADD_INCLUDEDIRS := include esp32 include/esp32 COMPONENT_SRCDIRS := patches . +COMPONENT_OBJEXCLUDE := patches/esp_rom_cache_writeback_esp32s3.o + ifdef IS_BOOTLOADER_BUILD -COMPONENT_OBJEXCLUDE := patches/esp_rom_longjmp.o +COMPONENT_OBJEXCLUDE += patches/esp_rom_longjmp.o endif #Linker scripts used to link the final application. diff --git a/components/esp_rom/esp32s3/esp_rom_caps.h b/components/esp_rom/esp32s3/esp_rom_caps.h index 8c814c713a..a47c2a1f87 100644 --- a/components/esp_rom/esp32s3/esp_rom_caps.h +++ b/components/esp_rom/esp32s3/esp_rom_caps.h @@ -25,3 +25,4 @@ #define ESP_ROM_HAS_ETS_PRINTF_BUG (1) // ROM has ets_printf bug when disable the ROM log either by eFuse or RTC storage register #define ESP_ROM_HAS_FLASH_COUNT_PAGES_BUG (1) // ROM api Cache_Count_Flash_Pages will return unexpected value #define ESP_ROM_HAS_CACHE_SUSPEND_WAITI_BUG (1) // ROM api Cache_Suspend_I/DCache and Cache_Freeze_I/DCache_Enable does not waiti +#define ESP_ROM_HAS_CACHE_WRITEBACK_BUG (1) // ROM api Cache_WriteBack_Addr access cacheline being writen back may cause cache hit with wrong value. diff --git a/components/esp_rom/esp32s3/ld/esp32s3.rom.ld b/components/esp_rom/esp32s3/ld/esp32s3.rom.ld index c6f1e4dd1d..6df6dd22cc 100644 --- a/components/esp_rom/esp32s3/ld/esp32s3.rom.ld +++ b/components/esp_rom/esp32s3/ld/esp32s3.rom.ld @@ -381,7 +381,7 @@ PROVIDE( Cache_WriteBack_Items = 0x40001698 ); PROVIDE( Cache_Op_Addr = 0x400016a4 ); PROVIDE( Cache_Invalidate_Addr = 0x400016b0 ); PROVIDE( Cache_Clean_Addr = 0x400016bc ); -PROVIDE( Cache_WriteBack_Addr = 0x400016c8 ); +PROVIDE( rom_Cache_WriteBack_Addr = 0x400016c8 ); PROVIDE( Cache_Invalidate_ICache_All = 0x400016d4 ); PROVIDE( Cache_Invalidate_DCache_All = 0x400016e0 ); PROVIDE( Cache_Clean_All = 0x400016ec ); diff --git a/components/esp_rom/patches/esp_rom_cache.c b/components/esp_rom/patches/esp_rom_cache.c index 038d695e98..f9f2e37344 100644 --- a/components/esp_rom/patches/esp_rom_cache.c +++ b/components/esp_rom/patches/esp_rom_cache.c @@ -10,10 +10,14 @@ #include "esp_attr.h" #include "esp_rom_caps.h" #include "soc/extmem_reg.h" +#include "xtensa/xtruntime.h" #if CONFIG_IDF_TARGET_ESP32S3 #include "esp32s3/rom/cache.h" #endif +#define ALIGN_UP(addr, align) (((addr) + (align)-1) & ~((align)-1)) +#define ALIGN_DOWN(addr, align) ((addr) & ~((align) - 1)) + // this api is renamed for patch extern uint32_t rom_Cache_Count_Flash_Pages(uint32_t bus, uint32_t * page0_mapped); IRAM_ATTR uint32_t Cache_Count_Flash_Pages(uint32_t bus, uint32_t * page0_mapped) @@ -83,4 +87,71 @@ IRAM_ATTR void Cache_Freeze_DCache_Enable(cache_freeze_mode_t mode) } extern void Cache_Freeze_DCache_Enable(cache_freeze_mode_t mode); #endif + +#if ESP_ROM_HAS_CACHE_WRITEBACK_BUG +/* Defined in esp_rom_cache_writeback_esp32s3.S */ +extern void cache_writeback_items_freeze(uint32_t addr, uint32_t items); +// renamed for patch +extern int rom_Cache_WriteBack_Addr(uint32_t addr, uint32_t size); +IRAM_ATTR int Cache_WriteBack_Addr(uint32_t addr, uint32_t size) +{ + /* Do special processing for unaligned memory at the start and end of the cache writeback memory. + * 1. Disable the interrupt to prevent the current CPU accessing the same cacheline. + * 2. Enable dcache freeze to prevent the another CPU accessing the same cacheline. + */ + uint32_t irq_status; + uint32_t start_len, end_len; + uint32_t start, end; + uint32_t dcache_line_size; + uint32_t autoload; + int ret = 0; + start = addr; + end = addr + size; + dcache_line_size = Cache_Get_DCache_Line_Size(); + + if (size == 0) { + return 0; + } + + /*the start address is unaligned*/ + if (start & (dcache_line_size -1)) { + addr = ALIGN_UP(start, dcache_line_size); + start_len = addr - start; + size = (size < start_len) ? 0 : (size - start_len); + + /*writeback start unaligned mem, one cacheline*/ + irq_status = XTOS_SET_INTLEVEL(XCHAL_NMILEVEL);//mask all interrupts + cache_writeback_items_freeze(start, 1); + XTOS_RESTORE_INTLEVEL(irq_status); + + if (size == 0) { + return 0; + } + } + + /*the end address is unaligned*/ + if (end & (dcache_line_size -1)) { + end = ALIGN_DOWN(end, dcache_line_size); + end_len = addr + size - end; + size = (size - end_len); + + /*writeback end unaligned mem, one cacheline*/ + irq_status = XTOS_SET_INTLEVEL(XCHAL_NMILEVEL);//mask all interrupts + cache_writeback_items_freeze(end, 1); + XTOS_RESTORE_INTLEVEL(irq_status); + + if (size == 0) { + return 0; + } + } + + /*suspend autoload, avoid load cachelines being written back*/ + autoload = Cache_Suspend_DCache_Autoload(); + ret = rom_Cache_WriteBack_Addr(addr, size); + Cache_Resume_DCache_Autoload(autoload); + + return ret; +} +extern int Cache_WriteBack_Addr(uint32_t addr, uint32_t size); +#endif //ESP_ROM_HAS_CACHE_WRITEBACK_BUG #endif diff --git a/components/esp_rom/patches/esp_rom_cache_writeback_esp32s3.S b/components/esp_rom/patches/esp_rom_cache_writeback_esp32s3.S new file mode 100644 index 0000000000..13f78820e5 --- /dev/null +++ b/components/esp_rom/patches/esp_rom_cache_writeback_esp32s3.S @@ -0,0 +1,135 @@ +/* + * SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "sdkconfig.h" +#include "esp_bit_defs.h" +#include "soc/extmem_reg.h" + +#if CONFIG_IDF_TARGET_ESP32S3 +/** + * @brief Write back the cache items of DCache, enable cache freeze during writeback. + * Operation will be done CACHE_LINE_SIZE aligned. + * If the region is not in DCache addr room, nothing will be done. + * Please do not call this function in your SDK application. + * @param uint32_t addr: start address to write back + * @param uint32_t items: cache lines to invalidate, items * cache_line_size should + * not exceed the bus address size(4MB) + * + * void cache_writeback_items_freeze(uint32_t addr, uint32_t items) +*/ + +/******************************************************************************* + +This function is a cache write-back function that works around the following +hardware errata on the ESP32-S3: + +- Core X manually triggers (via the EXTMEM_DCACHE_SYNC_CTRL_REG register) the +write-back of one or more cache lines. +- While the write-back is in progress, there are two scenarios that may cause +cache hit error. + - Core X enters the interrupt handler and access the same cache line + being written back. + - Core Y access the same cache line being written back. + +To workaround this errata, the following steps must be taken when manually +triggering a cache write-back: + +- Core X must disable interrupts so that it cannot be preempted +- Core X must freeze the cache (via the EXTMEM_DCACHE_FREEZE_REG register) to +prevent Core Y from accessing the same cache lines that are about to be written +back. +- Core X now triggers the cache write-back. During the write-back... + - If Core Y attempts the access any address in the cache region, Core Y will + busy wait until the cache is unfrozen. + - Core X must ensure that it does not access any address in the cache region, + otherwise Core X will busy wait thus causing a deadlock. +- After the write-back is complete, Core X unfreezes the cache, and reenables +interrupts. + +Notes: + +- Please do not modify this function, it must strictly follow the current execution +sequence, otherwise it may cause unexpected errors. +- This function is written in assmebly to ensure that the function itself never +accesses any cache address while the cache is frozen. Unexpected cache access +could occur if... + - the function triggers an window overflow onto a stack placed in PSRAM. + Thus, we only use two window panes (a0 to a8) in this function and trigger + all window overflows before freezing the cache. + - the function accesses literals/read-only variables placed in Flash. + +*******************************************************************************/ + + .section .iram1,"ax" + .align 4 + /* + Create dedicated literal pool for this function. Mostly used to store out + of range movi transformations. + */ + .literal_position + .global cache_writeback_items_freeze + .type cache_writeback_items_freeze, @function +cache_writeback_items_freeze: + entry sp, 32 + + /* REG_WRITE(EXTMEM_DCACHE_SYNC_ADDR_REG, addr); */ + movi a4, EXTMEM_DCACHE_SYNC_ADDR_REG + s32i a2, a4, 0 + /* REG_WRITE(EXTMEM_DCACHE_SYNC_SIZE_REG, items); */ + movi a4, EXTMEM_DCACHE_SYNC_SIZE_REG + s32i a3, a4, 0 + memw /* About to freeze the cache. Ensure all previous memory R/W are completed */ + + movi a2, EXTMEM_DCACHE_FREEZE_REG + movi a3, EXTMEM_DCACHE_SYNC_CTRL_REG + + /* + REG_CLR_BIT(EXTMEM_DCACHE_FREEZE_REG, EXTMEM_DCACHE_FREEZE_MODE); + REG_SET_BIT(EXTMEM_DCACHE_FREEZE_REG, EXTMEM_DCACHE_FREEZE_ENA); + */ + l32i a4, a2, 0 /* a4 = *(EXTMEM_DCACHE_FREEZE_REG) */ + movi a5, ~(EXTMEM_DCACHE_FREEZE_MODE_M) + and a4, a4, a5 + movi a5, EXTMEM_DCACHE_FREEZE_ENA_M + or a4, a4, a5 + s32i a4, a2, 0 /* *(EXTMEM_DCACHE_FREEZE_REG) = a4 */ + + /* while (!REG_GET_BIT(EXTMEM_DCACHE_FREEZE_REG, EXTMEM_DCACHE_FREEZE_DONE)); */ + movi a5, EXTMEM_DCACHE_FREEZE_DONE_M +_wait_freeze_done: + l32i a4, a2, 0 /* a4 = *(EXTMEM_DCACHE_FREEZE_REG) */ + memw + bnone a4, a5, _wait_freeze_done + + /* REG_SET_BIT(EXTMEM_DCACHE_SYNC_CTRL_REG, EXTMEM_DCACHE_WRITEBACK_ENA); */ + l32i a4, a3, 0 /* a4 = *(EXTMEM_DCACHE_SYNC_CTRL_REG) */ + movi a5, EXTMEM_DCACHE_WRITEBACK_ENA_M + or a4, a4, a5 + s32i a4, a3, 0 /* *(EXTMEM_DCACHE_SYNC_CTRL_REG) = a4 */ + + /* while(!REG_GET_BIT(EXTMEM_DCACHE_SYNC_CTRL_REG, EXTMEM_DCACHE_SYNC_DONE)); */ + movi a5, EXTMEM_DCACHE_SYNC_DONE_M +_wait_writeback_done: + l32i a4, a3, 0 /* a4 = *(EXTMEM_DCACHE_SYNC_CTRL_REG) */ + memw + bnone a4, a5, _wait_writeback_done + + /* REG_CLR_BIT(EXTMEM_DCACHE_FREEZE_REG, EXTMEM_DCACHE_FREEZE_ENA); */ + l32i a4, a2, 0 /* a4 = *(EXTMEM_DCACHE_FREEZE_REG) */ + movi a5, ~(EXTMEM_DCACHE_FREEZE_ENA_M) + and a4, a4, a5 + s32i a4, a2, 0 /* *(EXTMEM_DCACHE_FREEZE_REG) = a4 */ + + /* while (REG_GET_BIT(EXTMEM_DCACHE_FREEZE_REG, EXTMEM_DCACHE_FREEZE_DONE)); */ + movi a5, EXTMEM_DCACHE_FREEZE_DONE_M +_wait_unfreeze_done: + l32i a4, a2, 0 /* a4 = *(EXTMEM_DCACHE_FREEZE_REG) */ + memw + bany a4, a5, _wait_unfreeze_done + + retw + .size cache_writeback_items_freeze, . - cache_writeback_items_freeze +#endif /* CONFIG_IDF_TARGET_ESP32S3 */