diff --git a/components/freertos/Kconfig b/components/freertos/Kconfig index 86544ab915..abc8ff218f 100644 --- a/components/freertos/Kconfig +++ b/components/freertos/Kconfig @@ -389,35 +389,6 @@ menu "FreeRTOS" FreeRTOS will enter light sleep mode if no tasks need to run for this number of ticks. - menuconfig FREERTOS_DEBUG_INTERNALS - bool "Debug FreeRTOS internals" - default n - help - Enable this option to show the menu with internal FreeRTOS debugging features. - This option does not change any code by itself, it just shows/hides some options. - - if FREERTOS_DEBUG_INTERNALS - - config FREERTOS_PORTMUX_DEBUG - bool "Debug portMUX portENTER_CRITICAL/portEXIT_CRITICAL" - depends on FREERTOS_DEBUG_INTERNALS - default n - help - If enabled, debug information (including integrity checks) will be printed - to UART for the port-specific MUX implementation. - - if !FREERTOS_UNICORE - config FREERTOS_PORTMUX_DEBUG_RECURSIVE - bool "Debug portMUX Recursion" - depends on FREERTOS_PORTMUX_DEBUG - default n - help - If enabled, additional debug information will be printed for recursive - portMUX usage. - endif #FREERTOS_UNICORE - - endif # FREERTOS_DEBUG_INTERNALS - config FREERTOS_TASK_FUNCTION_WRAPPER bool "Enclose all task functions in a wrapper function" depends on COMPILER_OPTIMIZATION_DEFAULT diff --git a/components/freertos/include/freertos/portable.h b/components/freertos/include/freertos/portable.h index 7fab985277..b87495841c 100644 --- a/components/freertos/include/freertos/portable.h +++ b/components/freertos/include/freertos/portable.h @@ -179,12 +179,6 @@ void vPortYieldOtherCore( BaseType_t coreid) PRIVILEGED_FUNCTION; */ void vPortSetStackWatchpoint( void* pxStackStart ); -/* - * Returns true if the current core is in ISR context; low prio ISR, med prio ISR or timer tick ISR. High prio ISRs - * aren't detected here, but they normally cannot call C code, so that should not be an issue anyway. - */ -BaseType_t xPortInIsrContext(void); - /* * This function will be called in High prio ISRs. Returns true if the current core was in ISR context * before calling into high prio ISR context. @@ -239,7 +233,12 @@ static inline bool IRAM_ATTR xPortCanYield(void) } #endif -void uxPortCompareSetExtram(volatile uint32_t *addr, uint32_t compare, uint32_t *set); +static inline void uxPortCompareSetExtram(volatile uint32_t *addr, uint32_t compare, uint32_t *set) +{ +#if defined(CONFIG_ESP32_SPIRAM_SUPPORT) + compare_and_set_extram(addr, compare, set); +#endif +} #endif /* PORTABLE_H */ diff --git a/components/freertos/include/freertos/portmacro.h b/components/freertos/include/freertos/portmacro.h index f544c2d768..1ba84d68ab 100644 --- a/components/freertos/include/freertos/portmacro.h +++ b/components/freertos/include/freertos/portmacro.h @@ -82,8 +82,7 @@ extern "C" { #include #include "esp_private/crosscore_int.h" #include "esp_timer.h" /* required for FreeRTOS run time stats */ - - +#include "soc/spinlock.h" #include #include "sdkconfig.h" @@ -133,57 +132,24 @@ typedef unsigned portBASE_TYPE UBaseType_t; #include "sdkconfig.h" #include "esp_attr.h" -/* "mux" data structure (spinlock) */ -typedef struct { - /* owner field values: - * 0 - Uninitialized (invalid) - * portMUX_FREE_VAL - Mux is free, can be locked by either CPU - * CORE_ID_REGVAL_PRO / CORE_ID_REGVAL_APP - Mux is locked to the particular core - * - * Note that for performance reasons we use the full Xtensa CORE ID values - * (CORE_ID_REGVAL_PRO, CORE_ID_REGVAL_APP) and not the 0,1 values which are used in most - * other FreeRTOS code. - * - * Any value other than portMUX_FREE_VAL, CORE_ID_REGVAL_PRO, CORE_ID_REGVAL_APP indicates corruption - */ - uint32_t owner; - /* count field: - * If mux is unlocked, count should be zero. - * If mux is locked, count is non-zero & represents the number of recursive locks on the mux. - */ - uint32_t count; -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG - const char *lastLockedFn; - int lastLockedLine; -#endif -} portMUX_TYPE; +static inline uint32_t xPortGetCoreID(void); -#define portMUX_FREE_VAL 0xB33FFFFF +// Critical section management. NW-TODO: replace XTOS_SET_INTLEVEL with more efficient version, if any? +// These cannot be nested. They should be used with a lot of care and cannot be called from interrupt level. +// +// Only applies to one CPU. See notes above & below for reasons not to use these. +#define portDISABLE_INTERRUPTS() do { XTOS_SET_INTLEVEL(XCHAL_EXCM_LEVEL); portbenchmarkINTERRUPT_DISABLE(); } while (0) +#define portENABLE_INTERRUPTS() do { portbenchmarkINTERRUPT_RESTORE(0); XTOS_SET_INTLEVEL(0); } while (0) -/* Special constants for vPortCPUAcquireMutexTimeout() */ -#define portMUX_NO_TIMEOUT (-1) /* When passed for 'timeout_cycles', spin forever if necessary */ -#define portMUX_TRY_LOCK 0 /* Try to acquire the spinlock a single time only */ - -// Keep this in sync with the portMUX_TYPE struct definition please. -#ifndef CONFIG_FREERTOS_PORTMUX_DEBUG -#define portMUX_INITIALIZER_UNLOCKED { \ - .owner = portMUX_FREE_VAL, \ - .count = 0, \ - } -#else -#define portMUX_INITIALIZER_UNLOCKED { \ - .owner = portMUX_FREE_VAL, \ - .count = 0, \ - .lastLockedFn = "(never locked)", \ - .lastLockedLine = -1 \ - } -#endif - - -#define portASSERT_IF_IN_ISR() vPortAssertIfInISR() -void vPortAssertIfInISR(void); - -#define portCRITICAL_NESTING_IN_TCB 1 +// Cleaner solution allows nested interrupts disabling and restoring via local registers or stack. +// They can be called from interrupts too. +// WARNING: Only applies to current CPU. See notes above. +static inline unsigned portENTER_CRITICAL_NESTED(void) { + unsigned state = XTOS_SET_INTLEVEL(XCHAL_EXCM_LEVEL); + portbenchmarkINTERRUPT_DISABLE(); + return state; +} +#define portEXIT_CRITICAL_NESTED(state) do { portbenchmarkINTERRUPT_RESTORE(state); XTOS_RESTORE_JUST_INTLEVEL(state); } while (0) /* Modifications to portENTER_CRITICAL. @@ -211,125 +177,119 @@ Remark: For the ESP32, portENTER_CRITICAL and portENTER_CRITICAL_ISR both alias that either function can be called both from ISR as well as task context. This is not standard FreeRTOS behaviour; please keep this in mind if you need any compatibility with other FreeRTOS implementations. */ -void vPortCPUInitializeMutex(portMUX_TYPE *mux); -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG -void vPortCPUAcquireMutex(portMUX_TYPE *mux, const char *function, int line); -bool vPortCPUAcquireMutexTimeout(portMUX_TYPE *mux, int timeout_cycles, const char *function, int line); -void vPortCPUReleaseMutex(portMUX_TYPE *mux, const char *function, int line); +/* "mux" data structure (spinlock) */ +typedef struct { + spinlock_t spinlock; +} portMUX_TYPE; -void vTaskEnterCritical( portMUX_TYPE *mux, const char *function, int line ); -void vTaskExitCritical( portMUX_TYPE *mux, const char *function, int line ); +#define portMUX_FREE_VAL SPINLOCK_FREE +#define portMUX_NO_TIMEOUT SPINLOCK_WAIT_FOREVER /* When passed for 'timeout_cycles', spin forever if necessary */ +#define portMUX_TRY_LOCK SPINLOCK_NO_WAIT /* Try to acquire the spinlock a single time only */ +#define portMUX_INITIALIZER_UNLOCKED {.spinlock=SPINLOCK_INITIALIZER} -#ifdef CONFIG_FREERTOS_CHECK_PORT_CRITICAL_COMPLIANCE -/* Calling port*_CRITICAL from ISR context would cause an assert failure. - * If the parent function is called from both ISR and Non-ISR context then call port*_CRITICAL_SAFE - */ -#define portENTER_CRITICAL(mux) do { \ - if(!xPortInIsrContext()) { \ - vTaskEnterCritical(mux, __FUNCTION__, __LINE__); \ - } else { \ - ets_printf("%s:%d (%s)- port*_CRITICAL called from ISR context!\n", __FILE__, __LINE__, \ - __FUNCTION__); \ - abort(); \ - } \ - } while(0) +#define portASSERT_IF_IN_ISR() vPortAssertIfInISR() +void vPortAssertIfInISR(void); -#define portEXIT_CRITICAL(mux) do { \ - if(!xPortInIsrContext()) { \ - vTaskExitCritical(mux, __FUNCTION__, __LINE__); \ - } else { \ - ets_printf("%s:%d (%s)- port*_CRITICAL called from ISR context!\n", __FILE__, __LINE__, \ - __FUNCTION__); \ - abort(); \ - } \ - } while(0) -#else -#define portENTER_CRITICAL(mux) vTaskEnterCritical(mux, __FUNCTION__, __LINE__) -#define portEXIT_CRITICAL(mux) vTaskExitCritical(mux, __FUNCTION__, __LINE__) -#endif -#define portENTER_CRITICAL_ISR(mux) vTaskEnterCritical(mux, __FUNCTION__, __LINE__) -#define portEXIT_CRITICAL_ISR(mux) vTaskExitCritical(mux, __FUNCTION__, __LINE__) -#else -void vTaskExitCritical( portMUX_TYPE *mux ); -void vTaskEnterCritical( portMUX_TYPE *mux ); -void vPortCPUAcquireMutex(portMUX_TYPE *mux); +#define portCRITICAL_NESTING_IN_TCB 0 -/** @brief Acquire a portmux spinlock with a timeout - * - * @param mux Pointer to portmux to acquire. - * @param timeout_cycles Timeout to spin, in CPU cycles. Pass portMUX_NO_TIMEOUT to wait forever, - * portMUX_TRY_LOCK to try a single time to acquire the lock. - * - * @return true if mutex is successfully acquired, false on timeout. - */ -bool vPortCPUAcquireMutexTimeout(portMUX_TYPE *mux, int timeout_cycles); -void vPortCPUReleaseMutex(portMUX_TYPE *mux); - -#ifdef CONFIG_FREERTOS_CHECK_PORT_CRITICAL_COMPLIANCE -/* Calling port*_CRITICAL from ISR context would cause an assert failure. - * If the parent function is called from both ISR and Non-ISR context then call port*_CRITICAL_SAFE - */ -#define portENTER_CRITICAL(mux) do { \ - if(!xPortInIsrContext()) { \ - vTaskEnterCritical(mux); \ - } else { \ - ets_printf("%s:%d (%s)- port*_CRITICAL called from ISR context!\n", __FILE__, __LINE__, \ - __FUNCTION__); \ - abort(); \ - } \ - } while(0) - -#define portEXIT_CRITICAL(mux) do { \ - if(!xPortInIsrContext()) { \ - vTaskExitCritical(mux); \ - } else { \ - ets_printf("%s:%d (%s)- port*_CRITICAL called from ISR context!\n", __FILE__, __LINE__, \ - __FUNCTION__); \ - abort(); \ - } \ - } while(0) -#else -#define portENTER_CRITICAL(mux) vTaskEnterCritical(mux) -#define portEXIT_CRITICAL(mux) vTaskExitCritical(mux) -#endif -#define portENTER_CRITICAL_ISR(mux) vTaskEnterCritical(mux) -#define portEXIT_CRITICAL_ISR(mux) vTaskExitCritical(mux) -#endif - -#define portENTER_CRITICAL_SAFE(mux) do { \ - if (xPortInIsrContext()) { \ - portENTER_CRITICAL_ISR(mux); \ - } else { \ - portENTER_CRITICAL(mux); \ - } \ - } while(0) - -#define portEXIT_CRITICAL_SAFE(mux) do { \ - if (xPortInIsrContext()) { \ - portEXIT_CRITICAL_ISR(mux); \ - } else { \ - portEXIT_CRITICAL(mux); \ - } \ - } while(0) - - -// Critical section management. NW-TODO: replace XTOS_SET_INTLEVEL with more efficient version, if any? -// These cannot be nested. They should be used with a lot of care and cannot be called from interrupt level. -// -// Only applies to one CPU. See notes above & below for reasons not to use these. -#define portDISABLE_INTERRUPTS() do { XTOS_SET_INTLEVEL(XCHAL_EXCM_LEVEL); portbenchmarkINTERRUPT_DISABLE(); } while (0) -#define portENABLE_INTERRUPTS() do { portbenchmarkINTERRUPT_RESTORE(0); XTOS_SET_INTLEVEL(0); } while (0) - -// Cleaner solution allows nested interrupts disabling and restoring via local registers or stack. -// They can be called from interrupts too. -// WARNING: Only applies to current CPU. See notes above. -static inline unsigned portENTER_CRITICAL_NESTED(void) { - unsigned state = XTOS_SET_INTLEVEL(XCHAL_EXCM_LEVEL); - portbenchmarkINTERRUPT_DISABLE(); - return state; +static inline void __attribute__((always_inline)) vPortCPUInitializeMutex(portMUX_TYPE *mux) +{ + spinlock_initialize(&mux->spinlock); +} + +static inline void __attribute__((always_inline)) vPortCPUAcquireMutex(portMUX_TYPE *mux) +{ + spinlock_acquire(&mux->spinlock, portMUX_NO_TIMEOUT); +} + +static inline bool __attribute__((always_inline)) vPortCPUAcquireMutexTimeout(portMUX_TYPE *mux, int timeout) +{ + return (spinlock_acquire(&mux->spinlock, timeout)); +} + +static inline void __attribute__((always_inline)) vPortCPUReleaseMutex(portMUX_TYPE *mux) +{ + spinlock_release(&mux->spinlock); +} + +void vPortEnterCritical(portMUX_TYPE *mux); +void vPortExitCritical(portMUX_TYPE *mux); + +/* + * Returns true if the current core is in ISR context; low prio ISR, med prio ISR or timer tick ISR. High prio ISRs + * aren't detected here, but they normally cannot call C code, so that should not be an issue anyway. + */ +BaseType_t xPortInIsrContext(void); + +static inline void __attribute__((always_inline)) vPortEnterCriticalCompliance(portMUX_TYPE *mux) +{ + if(!xPortInIsrContext()) { + vPortEnterCritical(mux); + } else { + ets_printf("%s:%d (%s)- port*_CRITICAL called from ISR context!\n", __FILE__, __LINE__, + __FUNCTION__); + abort(); + } +} + +static inline void __attribute__((always_inline)) vPortExitCriticalCompliance(portMUX_TYPE *mux) +{ + if(!xPortInIsrContext()) { + vPortExitCritical(mux); + } else { + ets_printf("%s:%d (%s)- port*_CRITICAL called from ISR context!\n", __FILE__, __LINE__, + __FUNCTION__); + abort(); + } +} + +#ifdef CONFIG_FREERTOS_CHECK_PORT_CRITICAL_COMPLIANCE +/* Calling port*_CRITICAL from ISR context would cause an assert failure. + * If the parent function is called from both ISR and Non-ISR context then call port*_CRITICAL_SAFE + */ +#define portENTER_CRITICAL(mux) vPortEnterCriticalCompliance(mux) +#define portEXIT_CRITICAL(mux) vPortExitCriticalCompliance(mux) +#else +#define portENTER_CRITICAL(mux) vPortEnterCritical(mux) +#define portEXIT_CRITICAL(mux) vPortExitCritical(mux) +#endif + +#define portENTER_CRITICAL_ISR(mux) vPortEnterCritical(mux) +#define portEXIT_CRITICAL_ISR(mux) vPortExitCritical(mux) + +static inline void __attribute__((always_inline)) vPortEnterCriticalSafe(portMUX_TYPE *mux) +{ + if (xPortInIsrContext()) { + portENTER_CRITICAL_ISR(mux); + } else { + portENTER_CRITICAL(mux); + } +} + +static inline void __attribute__((always_inline)) vPortExitCriticalSafe(portMUX_TYPE *mux) +{ + if (xPortInIsrContext()) { + portEXIT_CRITICAL_ISR(mux); + } else { + portEXIT_CRITICAL(mux); + } +} + +#define portENTER_CRITICAL_SAFE(mux) vPortEnterCriticalSafe(mux) +#define portEXIT_CRITICAL_SAFE(mux) vPortExitCriticalSafe(mux) +/* + * Wrapper for the Xtensa compare-and-set instruction. This subroutine will atomically compare + * *addr to 'compare'. If *addr == compare, *addr is set to *set. *set is updated with the previous + * value of *addr (either 'compare' or some other value.) + * + * Warning: From the ISA docs: in some (unspecified) cases, the s32c1i instruction may return the + * *bitwise inverse* of the old mem if the mem wasn't written. This doesn't seem to happen on the + * ESP32 (portMUX assertions would fail). + */ +static inline void __attribute__((always_inline)) uxPortCompareSet(volatile uint32_t *addr, uint32_t compare, uint32_t *set) { + compare_and_set_native(addr, compare, set); } -#define portEXIT_CRITICAL_NESTED(state) do { portbenchmarkINTERRUPT_RESTORE(state); XTOS_RESTORE_JUST_INTLEVEL(state); } while (0) // These FreeRTOS versions are similar to the nested versions above #define portSET_INTERRUPT_MASK_FROM_ISR() portENTER_CRITICAL_NESTED() @@ -343,43 +303,6 @@ static inline unsigned portENTER_CRITICAL_NESTED(void) { #define pvPortMallocTcbMem(size) heap_caps_malloc(size, portTcbMemoryCaps) #define pvPortMallocStackMem(size) heap_caps_malloc(size, portStackMemoryCaps) -/* - * Wrapper for the Xtensa compare-and-set instruction. This subroutine will atomically compare - * *addr to 'compare'. If *addr == compare, *addr is set to *set. *set is updated with the previous - * value of *addr (either 'compare' or some other value.) - * - * Warning: From the ISA docs: in some (unspecified) cases, the s32c1i instruction may return the - * *bitwise inverse* of the old mem if the mem wasn't written. This doesn't seem to happen on the - * ESP32 (portMUX assertions would fail). - */ -static inline void uxPortCompareSet(volatile uint32_t *addr, uint32_t compare, uint32_t *set) { -#if XCHAL_HAVE_S32C1I - __asm__ __volatile__ ( - "WSR %2,SCOMPARE1 \n" - "S32C1I %0, %1, 0 \n" - :"=r"(*set) - :"r"(addr), "r"(compare), "0"(*set) - ); -#else - // No S32C1I, so do this by disabling and re-enabling interrupts (slower) - uint32_t intlevel, old_value; - __asm__ __volatile__ ("rsil %0, " XTSTR(XCHAL_EXCM_LEVEL) "\n" - : "=r"(intlevel)); - - old_value = *addr; - if (old_value == compare) { - *addr = *set; - } - - __asm__ __volatile__ ("memw \n" - "wsr %0, ps\n" - :: "r"(intlevel)); - - *set = old_value; -#endif -} - - /*-----------------------------------------------------------*/ /* Architecture specifics. */ @@ -400,14 +323,12 @@ static inline void uxPortCompareSet(volatile uint32_t *addr, uint32_t compare, u #endif - /* Kernel utilities. */ void vPortYield( void ); void _frxt_setup_switch( void ); #define portYIELD() vPortYield() #define portYIELD_FROM_ISR() {traceISR_EXIT_TO_SCHEDULER(); _frxt_setup_switch();} -static inline uint32_t xPortGetCoreID(void); /* Yielding within an API call (when interrupts are off), means the yield should be delayed until interrupts are re-enabled. diff --git a/components/freertos/port.c b/components/freertos/port.c index 0fb5a97a95..0ae24f9817 100644 --- a/components/freertos/port.c +++ b/components/freertos/port.c @@ -127,10 +127,10 @@ extern void _xt_coproc_init(void); _Static_assert(tskNO_AFFINITY == CONFIG_FREERTOS_NO_AFFINITY, "incorrect tskNO_AFFINITY value"); /*-----------------------------------------------------------*/ - unsigned port_xSchedulerRunning[portNUM_PROCESSORS] = {0}; // Duplicate of inaccessible xSchedulerRunning; needed at startup to avoid counting nesting unsigned port_interruptNesting[portNUM_PROCESSORS] = {0}; // Interrupt nesting level. Increased/decreased in portasm.c, _frxt_int_enter/_frxt_int_exit - +BaseType_t port_uxCriticalNesting[portNUM_PROCESSORS] = {0}; +BaseType_t port_uxOldInterruptState[portNUM_PROCESSORS] = {0}; /*-----------------------------------------------------------*/ // User exception dispatcher when exiting @@ -355,74 +355,6 @@ void vPortAssertIfInISR(void) configASSERT(xPortInIsrContext()); } -/* - * For kernel use: Initialize a per-CPU mux. Mux will be initialized unlocked. - */ -void vPortCPUInitializeMutex(portMUX_TYPE *mux) { - -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG - ets_printf("Initializing mux %p\n", mux); - mux->lastLockedFn="(never locked)"; - mux->lastLockedLine=-1; -#endif - mux->owner=portMUX_FREE_VAL; - mux->count=0; -} - -#include "portmux_impl.h" - -/* - * For kernel use: Acquire a per-CPU mux. Spinlocks, so don't hold on to these muxes for too long. - */ -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG -void vPortCPUAcquireMutex(portMUX_TYPE *mux, const char *fnName, int line) { - unsigned int irqStatus = portENTER_CRITICAL_NESTED(); - vPortCPUAcquireMutexIntsDisabled(mux, portMUX_NO_TIMEOUT, fnName, line); - portEXIT_CRITICAL_NESTED(irqStatus); -} - -bool vPortCPUAcquireMutexTimeout(portMUX_TYPE *mux, int timeout_cycles, const char *fnName, int line) { - unsigned int irqStatus = portENTER_CRITICAL_NESTED(); - bool result = vPortCPUAcquireMutexIntsDisabled(mux, timeout_cycles, fnName, line); - portEXIT_CRITICAL_NESTED(irqStatus); - return result; -} - -#else -void vPortCPUAcquireMutex(portMUX_TYPE *mux) { - unsigned int irqStatus = portENTER_CRITICAL_NESTED(); - vPortCPUAcquireMutexIntsDisabled(mux, portMUX_NO_TIMEOUT); - portEXIT_CRITICAL_NESTED(irqStatus); -} - -bool vPortCPUAcquireMutexTimeout(portMUX_TYPE *mux, int timeout_cycles) { - unsigned int irqStatus = portENTER_CRITICAL_NESTED(); - bool result = vPortCPUAcquireMutexIntsDisabled(mux, timeout_cycles); - portEXIT_CRITICAL_NESTED(irqStatus); - return result; -} -#endif - - -/* - * For kernel use: Release a per-CPU mux - * - * Mux must be already locked by this core - */ -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG -void vPortCPUReleaseMutex(portMUX_TYPE *mux, const char *fnName, int line) { - unsigned int irqStatus = portENTER_CRITICAL_NESTED(); - vPortCPUReleaseMutexIntsDisabled(mux, fnName, line); - portEXIT_CRITICAL_NESTED(irqStatus); -} -#else -void vPortCPUReleaseMutex(portMUX_TYPE *mux) { - unsigned int irqStatus = portENTER_CRITICAL_NESTED(); - vPortCPUReleaseMutexIntsDisabled(mux); - portEXIT_CRITICAL_NESTED(irqStatus); -} -#endif - void vPortSetStackWatchpoint( void* pxStackStart ) { //Set watchpoint 1 to watch the last 32 bytes of the stack. //Unfortunately, the Xtensa watchpoints can't set a watchpoint on a random [base - base+n] region because @@ -435,39 +367,45 @@ void vPortSetStackWatchpoint( void* pxStackStart ) { esp_set_watchpoint(1, (char*)addr, 32, ESP_WATCHPOINT_STORE); } -#if defined(CONFIG_SPIRAM) -/* - * Compare & set (S32C1) does not work in external RAM. Instead, this routine uses a mux (in internal memory) to fake it. - */ -static portMUX_TYPE extram_mux = portMUX_INITIALIZER_UNLOCKED; - -void uxPortCompareSetExtram(volatile uint32_t *addr, uint32_t compare, uint32_t *set) { - uint32_t prev; - - uint32_t oldlevel = portENTER_CRITICAL_NESTED(); - -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG - vPortCPUAcquireMutexIntsDisabled(&extram_mux, portMUX_NO_TIMEOUT, __FUNCTION__, __LINE__); -#else - vPortCPUAcquireMutexIntsDisabled(&extram_mux, portMUX_NO_TIMEOUT); -#endif - prev=*addr; - if (prev==compare) { - *addr=*set; - } - *set=prev; -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG - vPortCPUReleaseMutexIntsDisabled(&extram_mux, __FUNCTION__, __LINE__); -#else - vPortCPUReleaseMutexIntsDisabled(&extram_mux); -#endif - - portEXIT_CRITICAL_NESTED(oldlevel); -} -#endif //defined(CONFIG_SPIRAM) - - - uint32_t xPortGetTickRateHz(void) { return (uint32_t)configTICK_RATE_HZ; } + +void __attribute__((optimize("-O3"))) vPortEnterCritical(portMUX_TYPE *mux) +{ + BaseType_t oldInterruptLevel = portENTER_CRITICAL_NESTED(); + /* Interrupts may already be disabled (because we're doing this recursively) + * but we can't get the interrupt level after + * vPortCPUAquireMutex, because it also may mess with interrupts. + * Get it here first, then later figure out if we're nesting + * and save for real there. + */ + vPortCPUAcquireMutex( mux ); + BaseType_t coreID = xPortGetCoreID(); + BaseType_t newNesting = port_uxCriticalNesting[coreID] + 1; + port_uxCriticalNesting[coreID] = newNesting; + + if( newNesting == 1 ) + { + //This is the first time we get called. Save original interrupt level. + port_uxOldInterruptState[coreID] = oldInterruptLevel; + } +} + +void __attribute__((optimize("-O3"))) vPortExitCritical(portMUX_TYPE *mux) +{ + vPortCPUReleaseMutex( mux ); + BaseType_t coreID = xPortGetCoreID(); + BaseType_t nesting = port_uxCriticalNesting[coreID]; + + if(nesting > 0U) + { + nesting--; + port_uxCriticalNesting[coreID] = nesting; + + if( nesting == 0U ) + { + portEXIT_CRITICAL_NESTED(port_uxOldInterruptState[coreID]); + } + } +} diff --git a/components/freertos/portmux_impl.h b/components/freertos/portmux_impl.h deleted file mode 100644 index af170bb3ea..0000000000 --- a/components/freertos/portmux_impl.h +++ /dev/null @@ -1,118 +0,0 @@ -/* - Copyright (C) 2016-2017 Espressif Shanghai PTE LTD - Copyright (C) 2015 Real Time Engineers Ltd. - - All rights reserved - - FreeRTOS is free software; you can redistribute it and/or modify it under - the terms of the GNU General Public License (version 2) as published by the - Free Software Foundation >>!AND MODIFIED BY!<< the FreeRTOS exception. - - *************************************************************************** - >>! NOTE: The modification to the GPL is included to allow you to !<< - >>! distribute a combined work that includes FreeRTOS without being !<< - >>! obliged to provide the source code for proprietary components !<< - >>! outside of the FreeRTOS kernel. !<< - *************************************************************************** - - FreeRTOS is distributed in the hope that it will be useful, but WITHOUT ANY - WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS - FOR A PARTICULAR PURPOSE. Full license text is available on the following - link: http://www.freertos.org/a00114.html -*/ - -/* This header exists for performance reasons, in order to inline the - implementation of vPortCPUAcquireMutexIntsDisabled and - vPortCPUReleaseMutexIntsDisabled into the - vTaskEnterCritical/vTaskExitCritical functions in task.c as well as the - vPortCPUAcquireMutex/vPortCPUReleaseMutex implementations. - - Normally this kind of performance hack is over the top, but - vTaskEnterCritical/vTaskExitCritical is called a great - deal by FreeRTOS internals. - - It should be #included by freertos port.c or tasks.c, in esp-idf. - - The way it works is that it essentially uses portmux_impl.inc.h as a - generator template of sorts. When no external memory is used, this - template is only used to generate the vPortCPUAcquireMutexIntsDisabledInternal - and vPortCPUReleaseMutexIntsDisabledInternal functions, which use S32C1 to - do an atomic compare & swap. When external memory is used the functions - vPortCPUAcquireMutexIntsDisabledExtram and vPortCPUReleaseMutexIntsDisabledExtram - are also generated, which use uxPortCompareSetExtram to fake the S32C1 instruction. - The wrapper functions vPortCPUAcquireMutexIntsDisabled and - vPortCPUReleaseMutexIntsDisabled will then use the appropriate function to do the - actual lock/unlock. -*/ -#include "soc/cpu.h" -#include "portable.h" -#include "soc/soc_memory_layout.h" - -/* XOR one core ID with this value to get the other core ID */ -#define CORE_ID_REGVAL_XOR_SWAP (CORE_ID_REGVAL_PRO ^ CORE_ID_REGVAL_APP) - - - - -//Define the mux routines for use with muxes in internal RAM -#define PORTMUX_AQUIRE_MUX_FN_NAME vPortCPUAcquireMutexIntsDisabledInternal -#define PORTMUX_RELEASE_MUX_FN_NAME vPortCPUReleaseMutexIntsDisabledInternal -#define PORTMUX_COMPARE_SET_FN_NAME uxPortCompareSet -#include "portmux_impl.inc.h" -#undef PORTMUX_AQUIRE_MUX_FN_NAME -#undef PORTMUX_RELEASE_MUX_FN_NAME -#undef PORTMUX_COMPARE_SET_FN_NAME - - -#if defined(CONFIG_SPIRAM) - -#define PORTMUX_AQUIRE_MUX_FN_NAME vPortCPUAcquireMutexIntsDisabledExtram -#define PORTMUX_RELEASE_MUX_FN_NAME vPortCPUReleaseMutexIntsDisabledExtram -#define PORTMUX_COMPARE_SET_FN_NAME uxPortCompareSetExtram -#include "portmux_impl.inc.h" -#undef PORTMUX_AQUIRE_MUX_FN_NAME -#undef PORTMUX_RELEASE_MUX_FN_NAME -#undef PORTMUX_COMPARE_SET_FN_NAME - -#endif - - -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG -#define PORTMUX_AQUIRE_MUX_FN_ARGS portMUX_TYPE *mux, int timeout_cycles, const char *fnName, int line -#define PORTMUX_RELEASE_MUX_FN_ARGS portMUX_TYPE *mux, const char *fnName, int line -#define PORTMUX_AQUIRE_MUX_FN_CALL_ARGS(x) x, timeout_cycles, fnName, line -#define PORTMUX_RELEASE_MUX_FN_CALL_ARGS(x) x, fnName, line -#else -#define PORTMUX_AQUIRE_MUX_FN_ARGS portMUX_TYPE *mux, int timeout_cycles -#define PORTMUX_RELEASE_MUX_FN_ARGS portMUX_TYPE *mux -#define PORTMUX_AQUIRE_MUX_FN_CALL_ARGS(x) x, timeout_cycles -#define PORTMUX_RELEASE_MUX_FN_CALL_ARGS(x) x -#endif - - -static inline bool __attribute__((always_inline)) vPortCPUAcquireMutexIntsDisabled(PORTMUX_AQUIRE_MUX_FN_ARGS) { -#if !defined(CONFIG_FREERTOS_UNICORE) -#if defined(CONFIG_SPIRAM) - if (esp_ptr_external_ram(mux)) { - return vPortCPUAcquireMutexIntsDisabledExtram(PORTMUX_AQUIRE_MUX_FN_CALL_ARGS(mux)); - } -#endif - return vPortCPUAcquireMutexIntsDisabledInternal(PORTMUX_AQUIRE_MUX_FN_CALL_ARGS(mux)); -#else - return true; -#endif -} - - -static inline void vPortCPUReleaseMutexIntsDisabled(PORTMUX_RELEASE_MUX_FN_ARGS) { -#if !defined(CONFIG_FREERTOS_UNICORE) -#if defined(CONFIG_SPIRAM) - if (esp_ptr_external_ram(mux)) { - vPortCPUReleaseMutexIntsDisabledExtram(PORTMUX_RELEASE_MUX_FN_CALL_ARGS(mux)); - return; - } -#endif - vPortCPUReleaseMutexIntsDisabledInternal(PORTMUX_RELEASE_MUX_FN_CALL_ARGS(mux)); -#endif -} - diff --git a/components/freertos/portmux_impl.inc.h b/components/freertos/portmux_impl.inc.h deleted file mode 100644 index 908fec1539..0000000000 --- a/components/freertos/portmux_impl.inc.h +++ /dev/null @@ -1,169 +0,0 @@ -/* - Copyright (C) 2016-2017 Espressif Shanghai PTE LTD - Copyright (C) 2015 Real Time Engineers Ltd. - - All rights reserved - - FreeRTOS is free software; you can redistribute it and/or modify it under - the terms of the GNU General Public License (version 2) as published by the - Free Software Foundation >>!AND MODIFIED BY!<< the FreeRTOS exception. - - *************************************************************************** - >>! NOTE: The modification to the GPL is included to allow you to !<< - >>! distribute a combined work that includes FreeRTOS without being !<< - >>! obliged to provide the source code for proprietary components !<< - >>! outside of the FreeRTOS kernel. !<< - *************************************************************************** - - FreeRTOS is distributed in the hope that it will be useful, but WITHOUT ANY - WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS - FOR A PARTICULAR PURPOSE. Full license text is available on the following - link: http://www.freertos.org/a00114.html -*/ - - -/* - Warning: funky preprocessor hackery ahead. Including these headers will generate two - functions, which names are defined by the preprocessor macros - PORTMUX_AQUIRE_MUX_FN_NAME and PORTMUX_RELEASE_MUX_FN_NAME. In order to do the compare - and exchange function, they will use whatever PORTMUX_COMPARE_SET_FN_NAME resolves to. - - In some scenarios, this header is included *twice* in portmux_impl.h: one time - for the 'normal' mux code which uses a compare&exchange routine, another time - to generate code for a second set of these routines that use a second mux - (in internal ram) to fake a compare&exchange on a variable in external memory. -*/ - - - -static inline bool __attribute__((always_inline)) -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG -PORTMUX_AQUIRE_MUX_FN_NAME(portMUX_TYPE *mux, int timeout_cycles, const char *fnName, int line) { -#else -PORTMUX_AQUIRE_MUX_FN_NAME(portMUX_TYPE *mux, int timeout_cycles) { -#endif - - -#if !CONFIG_FREERTOS_UNICORE - uint32_t res; - portBASE_TYPE coreID, otherCoreID; - uint32_t ccount_start; - bool set_timeout = timeout_cycles > portMUX_NO_TIMEOUT; -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG - if (!set_timeout) { - timeout_cycles = 10000; // Always set a timeout in debug mode - set_timeout = true; - } -#endif - if (set_timeout) { // Timeout - RSR(CCOUNT, ccount_start); - } - -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG - uint32_t owner = mux->owner; - if (owner != portMUX_FREE_VAL && owner != CORE_ID_REGVAL_PRO && owner != CORE_ID_REGVAL_APP) { - ets_printf("ERROR: vPortCPUAcquireMutex: mux %p is uninitialized (0x%X)! Called from %s line %d.\n", mux, owner, fnName, line); - mux->owner=portMUX_FREE_VAL; - } -#endif - - /* Spin until we own the core */ - - RSR(PRID, coreID); - /* Note: coreID is the full 32 bit core ID (CORE_ID_REGVAL_PRO/CORE_ID_REGVAL_APP), - not the 0/1 value returned by xPortGetCoreID() - */ - otherCoreID = CORE_ID_REGVAL_XOR_SWAP ^ coreID; - do { - /* mux->owner should be one of portMUX_FREE_VAL, CORE_ID_REGVAL_PRO, - CORE_ID_REGVAL_APP: - - - If portMUX_FREE_VAL, we want to atomically set to 'coreID'. - - If "our" coreID, we can drop through immediately. - - If "otherCoreID", we spin here. - */ - res = coreID; - PORTMUX_COMPARE_SET_FN_NAME(&mux->owner, portMUX_FREE_VAL, &res); - - if (res != otherCoreID) { - break; // mux->owner is "our" coreID - } - - if (set_timeout) { - uint32_t ccount_now; - RSR(CCOUNT, ccount_now); - if (ccount_now - ccount_start > (unsigned)timeout_cycles) { -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG - ets_printf("Timeout on mux! last non-recursive lock %s line %d, curr %s line %d\n", mux->lastLockedFn, mux->lastLockedLine, fnName, line); - ets_printf("Owner 0x%x count %d\n", mux->owner, mux->count); -#endif - return false; - } - } - } while (1); - - assert(res == coreID || res == portMUX_FREE_VAL); /* any other value implies memory corruption or uninitialized mux */ - assert((res == portMUX_FREE_VAL) == (mux->count == 0)); /* we're first to lock iff count is zero */ - assert(mux->count < 0xFF); /* Bad count value implies memory corruption */ - - /* now we own it, we can increment the refcount */ - mux->count++; - - -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG - if (res==portMUX_FREE_VAL) { //initial lock - mux->lastLockedFn=fnName; - mux->lastLockedLine=line; - } else { - ets_printf("Recursive lock: count=%d last non-recursive lock %s line %d, curr %s line %d\n", mux->count-1, - mux->lastLockedFn, mux->lastLockedLine, fnName, line); - } -#endif /* CONFIG_FREERTOS_PORTMUX_DEBUG */ -#endif /* CONFIG_FREERTOS_UNICORE */ - return true; -} - -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG -static inline void PORTMUX_RELEASE_MUX_FN_NAME(portMUX_TYPE *mux, const char *fnName, int line) { -#else -static inline void PORTMUX_RELEASE_MUX_FN_NAME(portMUX_TYPE *mux) { -#endif - - -#if !CONFIG_FREERTOS_UNICORE - portBASE_TYPE coreID; -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG - const char *lastLockedFn=mux->lastLockedFn; - int lastLockedLine=mux->lastLockedLine; - mux->lastLockedFn=fnName; - mux->lastLockedLine=line; - uint32_t owner = mux->owner; - if (owner != portMUX_FREE_VAL && owner != CORE_ID_REGVAL_PRO && owner != CORE_ID_REGVAL_APP) { - ets_printf("ERROR: vPortCPUReleaseMutex: mux %p is invalid (0x%x)!\n", mux, mux->owner); - } -#endif - -#if CONFIG_FREERTOS_PORTMUX_DEBUG || !defined(NDEBUG) - RSR(PRID, coreID); -#endif - -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG - if (coreID != mux->owner) { - ets_printf("ERROR: vPortCPUReleaseMutex: mux %p was already unlocked!\n", mux); - ets_printf("Last non-recursive unlock %s line %d, curr unlock %s line %d\n", lastLockedFn, lastLockedLine, fnName, line); - } -#endif - - assert(coreID == mux->owner); // This is a mutex we didn't lock, or it's corrupt - - mux->count--; - if(mux->count == 0) { - mux->owner = portMUX_FREE_VAL; - } else { - assert(mux->count < 0x100); // Indicates memory corruption -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG_RECURSIVE - ets_printf("Recursive unlock: count=%d last locked %s line %d, curr %s line %d\n", mux->count, lastLockedFn, lastLockedLine, fnName, line); -#endif - } -#endif //!CONFIG_FREERTOS_UNICORE -} diff --git a/components/freertos/tasks.c b/components/freertos/tasks.c index f34227d3ba..c057cb4c0c 100644 --- a/components/freertos/tasks.c +++ b/components/freertos/tasks.c @@ -2776,11 +2776,7 @@ void vTaskSwitchContext( void ) swapping that out here. Instead, we're going to do the work here ourselves. Because interrupts are already disabled, we only need to acquire the mutex. */ -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG - vPortCPUAcquireMutex( &xTaskQueueMutex, __FUNCTION__, __LINE__ ); -#else vPortCPUAcquireMutex( &xTaskQueueMutex ); -#endif #if !configUSE_PORT_OPTIMISED_TASK_SELECTION unsigned portBASE_TYPE foundNonExecutingWaiter = pdFALSE, ableToSchedule = pdFALSE, resetListHead; @@ -2881,11 +2877,7 @@ void vTaskSwitchContext( void ) //Exit critical region manually as well: release the mux now, interrupts will be re-enabled when we //exit the function. -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG - vPortCPUReleaseMutex( &xTaskQueueMutex, __FUNCTION__, __LINE__ ); -#else vPortCPUReleaseMutex( &xTaskQueueMutex ); -#endif #if CONFIG_FREERTOS_WATCHPOINT_END_OF_STACK @@ -4189,13 +4181,7 @@ For ESP32 FreeRTOS, vTaskEnterCritical implements both portENTER_CRITICAL and po #if ( portCRITICAL_NESTING_IN_TCB == 1 ) -#include "portmux_impl.h" - -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG - void vTaskEnterCritical( portMUX_TYPE *mux, const char *function, int line ) -#else void vTaskEnterCritical( portMUX_TYPE *mux ) -#endif { BaseType_t oldInterruptLevel=0; BaseType_t schedulerRunning = xSchedulerRunning; @@ -4206,13 +4192,9 @@ For ESP32 FreeRTOS, vTaskEnterCritical implements both portENTER_CRITICAL and po //and save for real there. oldInterruptLevel=portENTER_CRITICAL_NESTED(); } -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG - vPortCPUAcquireMutexIntsDisabled( mux, portMUX_NO_TIMEOUT, function, line ); -#else - vPortCPUAcquireMutexIntsDisabled( mux, portMUX_NO_TIMEOUT ); -#endif - if(schedulerRunning != pdFALSE) + vPortCPUAcquireMutex( mux ); + if( schedulerRunning != pdFALSE ) { TCB_t *tcb = pxCurrentTCB[xPortGetCoreID()]; BaseType_t newNesting = tcb->uxCriticalNesting + 1; @@ -4253,25 +4235,16 @@ For ESP32 FreeRTOS, vTaskEnterCritical implements both portENTER_CRITICAL and po #endif /* portCRITICAL_NESTING_IN_TCB */ /*-----------------------------------------------------------*/ - - /* For ESP32 FreeRTOS, vTaskExitCritical implements both portEXIT_CRITICAL and portEXIT_CRITICAL_ISR. */ #if ( portCRITICAL_NESTING_IN_TCB == 1 ) -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG - void vTaskExitCritical( portMUX_TYPE *mux, const char *function, int line ) -#else void vTaskExitCritical( portMUX_TYPE *mux ) -#endif { -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG - vPortCPUReleaseMutexIntsDisabled( mux, function, line ); -#else - vPortCPUReleaseMutexIntsDisabled( mux ); -#endif - if(xSchedulerRunning != pdFALSE) + + vPortCPUReleaseMutex( mux ); + if( xSchedulerRunning != pdFALSE ) { TCB_t *tcb = pxCurrentTCB[xPortGetCoreID()]; BaseType_t nesting = tcb->uxCriticalNesting; diff --git a/components/soc/CMakeLists.txt b/components/soc/CMakeLists.txt index 4386c7277f..07e3042328 100644 --- a/components/soc/CMakeLists.txt +++ b/components/soc/CMakeLists.txt @@ -35,6 +35,7 @@ list(APPEND srcs "src/hal/uart_hal_iram.c" "src/hal/spi_flash_hal.c" "src/hal/spi_flash_hal_iram.c" + "src/compare_set.c" ) if(IDF_TARGET STREQUAL "esp32") diff --git a/components/soc/include/soc/compare_set.h b/components/soc/include/soc/compare_set.h new file mode 100644 index 0000000000..626752ad51 --- /dev/null +++ b/components/soc/include/soc/compare_set.h @@ -0,0 +1,56 @@ +// Copyright 2015-2019 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __COMPARE_SET_H +#define __COMPARE_SET_H + +#include +#include +#include "soc/cpu.h" +#include "soc/soc_memory_layout.h" +#include "xtensa/xtruntime.h" + + +static inline void __attribute__((always_inline)) compare_and_set_native(volatile uint32_t *addr, uint32_t compare, uint32_t *set) +{ +#if (XCHAL_HAVE_S32C1I > 0) + __asm__ __volatile__ ( + "WSR %2,SCOMPARE1 \n" + "S32C1I %0, %1, 0 \n" + :"=r"(*set) + :"r"(addr), "r"(compare), "0"(*set) + ); +#else + // No S32C1I, so do this by disabling and re-enabling interrupts (slower) + uint32_t intlevel, old_value; + __asm__ __volatile__ ("rsil %0, " XTSTR(XCHAL_EXCM_LEVEL) "\n" + : "=r"(intlevel)); + + old_value = *addr; + if (old_value == compare) { + *addr = *set; + } + + __asm__ __volatile__ ("memw \n" + "wsr %0, ps\n" + :: "r"(intlevel)); + + *set = old_value; +#endif +} + + +void compare_and_set_extram(volatile uint32_t *addr, uint32_t compare, uint32_t *set); + +#endif \ No newline at end of file diff --git a/components/soc/include/soc/spinlock.h b/components/soc/include/soc/spinlock.h new file mode 100644 index 0000000000..a19b41e25e --- /dev/null +++ b/components/soc/include/soc/spinlock.h @@ -0,0 +1,153 @@ +// Copyright 2015-2019 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#ifndef __SOC_SPINLOCK_H +#define __SOC_SPINLOCK_H + +#include +#include +#include "soc/cpu.h" +#include "soc/soc_memory_layout.h" +#include "soc/compare_set.h" +#include "xtensa/xtruntime.h" + + +#define SPINLOCK_FREE 0xB33FFFFF +#define SPINLOCK_WAIT_FOREVER (-1) +#define SPINLOCK_NO_WAIT 0 +#define SPINLOCK_INITIALIZER {.owner = SPINLOCK_FREE,.count = 0} +#define CORE_ID_REGVAL_XOR_SWAP (0xCDCD ^ 0xABAB) + +typedef struct { + uint32_t owner; + uint32_t count; +}spinlock_t; + +/** + * @brief Initialize a lock to its default state - unlocked + * @param lock - spinlock object to initialize + */ +static inline void __attribute__((always_inline)) spinlock_initialize(spinlock_t *lock) +{ + assert(lock); + +#if !CONFIG_FREERTOS_UNICORE + lock->owner = SPINLOCK_FREE; + lock->count = 0; +#endif +} + +/** + * @brief Top level spinlock acquire function, spins until get the lock + * @param lock - target spinlock object + * @param timeout - cycles to wait, passing SPINLOCK_WAIT_FOREVER blocs indefinitely + */ +static inline bool __attribute__((always_inline)) spinlock_acquire(spinlock_t *lock, int32_t timeout) +{ +#if !CONFIG_FREERTOS_UNICORE + uint32_t result; + uint32_t irq_status; + uint32_t ccount_start; + uint32_t core_id, other_core_id; + + assert(lock); + irq_status = XTOS_SET_INTLEVEL(XCHAL_EXCM_LEVEL); + + if(timeout != SPINLOCK_WAIT_FOREVER){ + RSR(CCOUNT, ccount_start); + } + + /*spin until we own a core */ + RSR(PRID, core_id); + + /* Note: coreID is the full 32 bit core ID (CORE_ID_REGVAL_PRO/CORE_ID_REGVAL_APP) */ + + other_core_id = CORE_ID_REGVAL_XOR_SWAP ^ core_id; + do { + + /* lock->owner should be one of SPINLOCK_FREE, CORE_ID_REGVAL_PRO, + * CORE_ID_REGVAL_APP: + * - If SPINLOCK_FREE, we want to atomically set to 'core_id'. + * - If "our" core_id, we can drop through immediately. + * - If "other_core_id", we spin here. + */ + result = core_id; + +#if defined(CONFIG_ESP32_SPIRAM_SUPPORT) + if (esp_ptr_external_ram(lock)) { + compare_and_set_extram(&lock->owner, SPINLOCK_FREE, &result); + } else { +#endif + compare_and_set_native(&lock->owner, SPINLOCK_FREE, &result); +#if defined(CONFIG_ESP32_SPIRAM_SUPPORT) + } +#endif + if(result != other_core_id) { + break; + } + + if (timeout != SPINLOCK_WAIT_FOREVER) { + uint32_t ccount_now; + RSR(CCOUNT, ccount_now); + if (ccount_now - ccount_start > (unsigned)timeout) { + XTOS_RESTORE_INTLEVEL(irq_status); + return false; + } + } + }while(1); + + /* any other value implies memory corruption or uninitialized mux */ + assert(result == core_id || result == SPINLOCK_FREE); + assert((result == SPINLOCK_FREE) == (lock->count == 0)); /* we're first to lock iff count is zero */ + assert(lock->count < 0xFF); /* Bad count value implies memory corruption */ + + lock->count++; + XTOS_RESTORE_INTLEVEL(irq_status); + return true; + +#else + return true; +#endif + +} + +/** + * @brief Top level spinlock unlock function, unlocks a previously locked spinlock + * @param lock - target, locked before, spinlock object + */ +static inline void __attribute__((always_inline)) spinlock_release(spinlock_t *lock) +{ + +#if !CONFIG_FREERTOS_UNICORE + uint32_t irq_status; + uint32_t core_id; + + assert(lock); + irq_status = XTOS_SET_INTLEVEL(XCHAL_EXCM_LEVEL); + + RSR(PRID, core_id); + assert(core_id == lock->owner); // This is a mutex we didn't lock, or it's corrupt + lock->count--; + + if(!lock->count) { + lock->owner = SPINLOCK_FREE; + } else { + assert(lock->count < 0x100); // Indicates memory corruption + } + + XTOS_RESTORE_INTLEVEL(irq_status); +#endif +} + +#endif + diff --git a/components/soc/src/compare_set.c b/components/soc/src/compare_set.c new file mode 100644 index 0000000000..1a396c0adf --- /dev/null +++ b/components/soc/src/compare_set.c @@ -0,0 +1,40 @@ +// Copyright 2015-2019 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "soc/compare_set.h" +#include "soc/spinlock.h" + +static spinlock_t global_extram_lock = SPINLOCK_INITIALIZER; + +void compare_and_set_extram(volatile uint32_t *addr, uint32_t compare, uint32_t *set) +{ + uint32_t intlevel, old_value; + __asm__ __volatile__ ("rsil %0, " XTSTR(XCHAL_EXCM_LEVEL) "\n" + : "=r"(intlevel)); + + spinlock_acquire(&global_extram_lock, SPINLOCK_WAIT_FOREVER); + + old_value = *addr; + if (old_value == compare) { + *addr = *set; + } + + spinlock_release(&global_extram_lock); + + __asm__ __volatile__ ("memw \n" + "wsr %0, ps\n" + :: "r"(intlevel)); + + *set = old_value; +} + diff --git a/docs/en/api-guides/freertos-smp.rst b/docs/en/api-guides/freertos-smp.rst index 558f43ecde..f4ffb81676 100644 --- a/docs/en/api-guides/freertos-smp.rst +++ b/docs/en/api-guides/freertos-smp.rst @@ -356,14 +356,15 @@ a valid protection method against simultaneous access to shared data as it leaves the other core free to access the data even if the current core has disabled its own interrupts. -For this reason, ESP-IDF FreeRTOS implements critical sections using mutexes, -and calls to enter or exit a critical must provide a mutex that is associated -with a shared resource requiring access protection. When entering a critical -section in ESP-IDF FreeRTOS, the calling core will disable its scheduler and -interrupts similar to the vanilla FreeRTOS implementation. However, the calling -core will also take the mutex whilst the other core is left unaffected during -the critical section. If the other core attempts to take the same mutex, it -will spin until the mutex is released. Therefore, the ESP-IDF FreeRTOS +For this reason, ESP-IDF FreeRTOS implements critical sections using special mutexes, +referred by portMUX_Type objects on top of specific ESP32 spinlock component +and calls to enter or exit a critical must provide a spinlock object that +is associated with a shared resource requiring access protection. +When entering a critical section in ESP-IDF FreeRTOS, the calling core will disable +its scheduler and interrupts similar to the vanilla FreeRTOS implementation. However, +the calling core will also take the locks whilst the other core is left unaffected during +the critical section. If the other core attempts to take the spinlock, it +will spin until the lock is released. Therefore, the ESP-IDF FreeRTOS implementation of critical sections allows a core to have protected access to a shared resource without disabling the other core. The other core will only be affected if it tries to concurrently access the same resource. @@ -389,7 +390,7 @@ and :component_file:`freertos/task.c` It should be noted that when modifying vanilla FreeRTOS code to be ESP-IDF FreeRTOS compatible, it is trivial to modify the type of critical section called as they are all defined to call the same function. As long as the same -mutex is provided upon entering and exiting, the type of call should not +spinlock is provided upon entering and exiting, the type of call should not matter.