diff --git a/components/esp32/dport_access.c b/components/esp32/dport_access.c
index 04494ca672..238e0e61ce 100644
--- a/components/esp32/dport_access.c
+++ b/components/esp32/dport_access.c
@@ -165,6 +165,9 @@ static void dport_access_init_core(void *arg)
     dport_access_end[core_id] = 0;
     dport_core_state[core_id] = DPORT_CORE_STATE_RUNNING;
 
+    /* If this fails then the minimum stack size for this config is too close to running out */
+    assert(uxTaskGetStackHighWaterMark(NULL) > 128);
+
     vTaskDelete(NULL);
 }
 #endif
diff --git a/components/freertos/xtensa/include/freertos/FreeRTOSConfig.h b/components/freertos/xtensa/include/freertos/FreeRTOSConfig.h
index 282ff0e547..b8a2fa2fc1 100644
--- a/components/freertos/xtensa/include/freertos/FreeRTOSConfig.h
+++ b/components/freertos/xtensa/include/freertos/FreeRTOSConfig.h
@@ -181,13 +181,37 @@ int xt_clock_freq(void) __attribute__((deprecated));
 #define configMAX_PRIORITIES			( 25 )
 #endif
 
-#ifndef CONFIG_APPTRACE_ENABLE
-#define configMINIMAL_STACK_SIZE		768
+/* Various things that impact minimum stack sizes */
+
+/* Higher stack checker modes cause overhead on each function call */
+#if CONFIG_STACK_CHECK_ALL || CONFIG_STACK_CHECK_STRONG
+#define configSTACK_OVERHEAD_CHECKER 256
 #else
-/* apptrace module requires at least 2KB of stack per task */
-#define configMINIMAL_STACK_SIZE		2048
+#define configSTACK_OVERHEAD_CHECKER 0
 #endif
 
+/* with optimizations disabled, scheduler uses additional stack */
+#if CONFIG_COMPILER_OPTIMIZATION_NONE
+#define configSTACK_OVERHEAD_OPTIMIZATION 320
+#else
+#define configSTACK_OVERHEAD_OPTIMIZATION 0
+#endif
+
+/* apptrace mdule increases minimum stack usage */
+#if CONFIG_APPTRACE_ENABLE
+#define configSTACK_OVERHEAD_APPTRACE 1280
+#else
+#define configSTACK_OVERHEAD_APPTRACE 0
+#endif
+
+#define configSTACK_OVERHEAD_TOTAL (                                    \
+                                    configSTACK_OVERHEAD_CHECKER +      \
+                                    configSTACK_OVERHEAD_OPTIMIZATION + \
+                                    configSTACK_OVERHEAD_APPTRACE       \
+                                                                        )
+
+#define configMINIMAL_STACK_SIZE  (768 + configSTACK_OVERHEAD_TOTAL)
+
 #ifndef configIDLE_TASK_STACK_SIZE
 #define configIDLE_TASK_STACK_SIZE CONFIG_FREERTOS_IDLE_TASK_STACKSIZE
 #endif