diff --git a/components/mbedtls/port/esp_ds/esp_rsa_sign_alt.c b/components/mbedtls/port/esp_ds/esp_rsa_sign_alt.c
index f3d6fdf574..2512627cb9 100644
--- a/components/mbedtls/port/esp_ds/esp_rsa_sign_alt.c
+++ b/components/mbedtls/port/esp_ds/esp_rsa_sign_alt.c
@@ -104,8 +104,10 @@ esp_err_t esp_ds_init_data_ctx(esp_ds_data_ctx_t *ds_data)
 
 void esp_ds_release_ds_lock(void)
 {
-    /* Give back the semaphore (DS lock) */
-    xSemaphoreGive(s_ds_lock);
+    if (xSemaphoreGetMutexHolder(s_ds_lock) == xTaskGetCurrentTaskHandle()) {
+        /* Give back the semaphore (DS lock) */
+        xSemaphoreGive(s_ds_lock);
+    }
 }
 
 size_t esp_ds_get_keylen(void *ctx)