diff --git a/components/mbedtls/port/dynamic/esp_mbedtls_dynamic_impl.c b/components/mbedtls/port/dynamic/esp_mbedtls_dynamic_impl.c
index e7f9a4a401..b8620e05f6 100644
--- a/components/mbedtls/port/dynamic/esp_mbedtls_dynamic_impl.c
+++ b/components/mbedtls/port/dynamic/esp_mbedtls_dynamic_impl.c
@@ -416,7 +416,16 @@ int esp_mbedtls_free_rx_buffer(mbedtls_ssl_context *ssl)
     /**
      * The previous processing is just skipped, so "ssl->in_msglen = 0"
      */
-    if (!ssl->in_msgtype) {
+    if (!ssl->in_msgtype
+#if defined(MBEDTLS_SSL_SRV_C)
+        /**
+         * The ssl server read ClientHello manually without mbedtls_ssl_read_record(), so in_msgtype is not set and is zero.
+         * ClientHello has been processed and rx buffer should be freed.
+         * After processing ClientHello, the ssl state has been changed to MBEDTLS_SSL_SERVER_HELLO.
+         */
+        && !(ssl->conf->endpoint == MBEDTLS_SSL_IS_SERVER && ssl->state == MBEDTLS_SSL_SERVER_HELLO)
+#endif
+    ) {
         goto exit;
     }