diff --git a/components/spi_flash/cache_utils.c b/components/spi_flash/cache_utils.c
index 4a92c51f75..f8afdccf9c 100644
--- a/components/spi_flash/cache_utils.c
+++ b/components/spi_flash/cache_utils.c
@@ -80,6 +80,17 @@ static volatile bool s_flash_op_complete = false;
 static volatile int s_flash_op_cpu = -1;
 #endif
 
+static inline bool esp_task_stack_is_sane_cache_disabled(void)
+{
+    const void *sp = (const void *)esp_cpu_get_sp();
+
+    return esp_ptr_in_dram(sp)
+#if CONFIG_ESP_SYSTEM_ALLOW_RTC_FAST_MEM_AS_HEAP
+        || esp_ptr_in_rtc_dram_fast(sp)
+#endif
+    ;
+}
+
 void spi_flash_init_lock(void)
 {
     s_flash_op_mutex = xSemaphoreCreateRecursiveMutex();
@@ -127,7 +138,7 @@ void IRAM_ATTR spi_flash_op_block_func(void *arg)
 
 void IRAM_ATTR spi_flash_disable_interrupts_caches_and_other_cpu(void)
 {
-    assert(esp_ptr_in_dram((const void *)esp_cpu_get_sp()));
+    assert(esp_task_stack_is_sane_cache_disabled());
 
     spi_flash_op_lock();