diff --git a/components/driver/usb_serial_jtag.c b/components/driver/usb_serial_jtag.c
index cdefb31469..94879aa117 100644
--- a/components/driver/usb_serial_jtag.c
+++ b/components/driver/usb_serial_jtag.c
@@ -8,6 +8,7 @@
 #include <stdbool.h>
 #include "esp_log.h"
 #include "hal/usb_serial_jtag_ll.h"
+#include "hal/usb_phy_ll.h"
 #include "freertos/FreeRTOS.h"
 #include "freertos/semphr.h"
 #include "freertos/ringbuf.h"
@@ -112,6 +113,9 @@ esp_err_t usb_serial_jtag_driver_install(usb_serial_jtag_driver_config_t *usb_se
         goto _exit;
     }
 
+    // Configure PHY
+    usb_phy_ll_int_jtag_enable(&USB_SERIAL_JTAG);
+
     usb_serial_jtag_ll_clr_intsts_mask(USB_SERIAL_JTAG_INTR_SERIAL_IN_EMPTY|
                                          USB_SERIAL_JTAG_INTR_SERIAL_OUT_RECV_PKT);
     usb_serial_jtag_ll_ena_intr_mask(USB_SERIAL_JTAG_INTR_SERIAL_IN_EMPTY|
diff --git a/components/hal/esp32c3/include/hal/gpio_ll.h b/components/hal/esp32c3/include/hal/gpio_ll.h
index 2a71eb55ab..bf53824875 100644
--- a/components/hal/esp32c3/include/hal/gpio_ll.h
+++ b/components/hal/esp32c3/include/hal/gpio_ll.h
@@ -51,6 +51,12 @@ static inline void gpio_ll_pullup_en(gpio_dev_t *hw, uint32_t gpio_num)
   */
 static inline void gpio_ll_pullup_dis(gpio_dev_t *hw, uint32_t gpio_num)
 {
+    // The pull-up value of the USB pins are controlled by the pins’ pull-up value together with USB pull-up value
+    // USB DP pin is default to PU enabled
+    if (gpio_num == USB_DP_GPIO_NUM) {
+        SET_PERI_REG_MASK(USB_SERIAL_JTAG_CONF0_REG, USB_SERIAL_JTAG_PAD_PULL_OVERRIDE);
+        CLEAR_PERI_REG_MASK(USB_SERIAL_JTAG_CONF0_REG, USB_SERIAL_JTAG_DP_PULLUP);
+    }
     REG_CLR_BIT(GPIO_PIN_MUX_REG[gpio_num], FUN_PU);
 }
 
diff --git a/components/hal/esp32c3/include/hal/usb_phy_ll.h b/components/hal/esp32c3/include/hal/usb_phy_ll.h
new file mode 100644
index 0000000000..40d4cd1800
--- /dev/null
+++ b/components/hal/esp32c3/include/hal/usb_phy_ll.h
@@ -0,0 +1,34 @@
+/*
+ * SPDX-FileCopyrightText: 2015-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#pragma once
+
+#include "soc/usb_serial_jtag_struct.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * @brief Configures the internal PHY for USB_Serial_JTAG
+ *
+ * @param hw Start address of the USB Serial_JTAG registers
+ */
+static inline void usb_phy_ll_int_jtag_enable(usb_serial_jtag_dev_t *hw)
+{
+    // USB_Serial_JTAG use internal PHY
+    hw->conf0.phy_sel = 0;
+    // Disable software control USB D+ D- pullup pulldown (Device FS: dp_pullup = 1)
+    hw->conf0.pad_pull_override = 0;
+    // Enable USB D+ pullup
+    hw->conf0.dp_pullup = 1;
+    // Enable USB pad function
+    hw->conf0.usb_pad_enable = 1;
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/components/hal/esp32h2/include/hal/usb_phy_ll.h b/components/hal/esp32h2/include/hal/usb_phy_ll.h
new file mode 100644
index 0000000000..40d4cd1800
--- /dev/null
+++ b/components/hal/esp32h2/include/hal/usb_phy_ll.h
@@ -0,0 +1,34 @@
+/*
+ * SPDX-FileCopyrightText: 2015-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#pragma once
+
+#include "soc/usb_serial_jtag_struct.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * @brief Configures the internal PHY for USB_Serial_JTAG
+ *
+ * @param hw Start address of the USB Serial_JTAG registers
+ */
+static inline void usb_phy_ll_int_jtag_enable(usb_serial_jtag_dev_t *hw)
+{
+    // USB_Serial_JTAG use internal PHY
+    hw->conf0.phy_sel = 0;
+    // Disable software control USB D+ D- pullup pulldown (Device FS: dp_pullup = 1)
+    hw->conf0.pad_pull_override = 0;
+    // Enable USB D+ pullup
+    hw->conf0.dp_pullup = 1;
+    // Enable USB pad function
+    hw->conf0.usb_pad_enable = 1;
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/components/hal/esp32h2/include/rev1/hal/gpio_ll.h b/components/hal/esp32h2/include/rev1/hal/gpio_ll.h
index 64274e845c..bb7376aae5 100644
--- a/components/hal/esp32h2/include/rev1/hal/gpio_ll.h
+++ b/components/hal/esp32h2/include/rev1/hal/gpio_ll.h
@@ -51,6 +51,12 @@ static inline void gpio_ll_pullup_en(gpio_dev_t *hw, uint32_t gpio_num)
   */
 static inline void gpio_ll_pullup_dis(gpio_dev_t *hw, uint32_t gpio_num)
 {
+    // The pull-up value of the USB pins are controlled by the pins’ pull-up value together with USB pull-up value
+    // USB DP pin is default to PU enabled
+    if (gpio_num == USB_DP_GPIO_NUM) {
+        SET_PERI_REG_MASK(USB_SERIAL_JTAG_CONF0_REG, USB_SERIAL_JTAG_PAD_PULL_OVERRIDE);
+        CLEAR_PERI_REG_MASK(USB_SERIAL_JTAG_CONF0_REG, USB_SERIAL_JTAG_DP_PULLUP);
+    }
     REG_CLR_BIT(GPIO_PIN_MUX_REG[gpio_num], FUN_PU);
 }
 
diff --git a/components/hal/esp32h2/include/rev2/hal/gpio_ll.h b/components/hal/esp32h2/include/rev2/hal/gpio_ll.h
index f414512cb4..7975f502a2 100644
--- a/components/hal/esp32h2/include/rev2/hal/gpio_ll.h
+++ b/components/hal/esp32h2/include/rev2/hal/gpio_ll.h
@@ -51,6 +51,12 @@ static inline void gpio_ll_pullup_en(gpio_dev_t *hw, uint32_t gpio_num)
   */
 static inline void gpio_ll_pullup_dis(gpio_dev_t *hw, uint32_t gpio_num)
 {
+    // The pull-up value of the USB pins are controlled by the pins’ pull-up value together with USB pull-up value
+    // USB DP pin is default to PU enabled
+    if (gpio_num == USB_DP_GPIO_NUM) {
+        SET_PERI_REG_MASK(USB_SERIAL_JTAG_CONF0_REG, USB_SERIAL_JTAG_PAD_PULL_OVERRIDE);
+        CLEAR_PERI_REG_MASK(USB_SERIAL_JTAG_CONF0_REG, USB_SERIAL_JTAG_DP_PULLUP);
+    }
     REG_CLR_BIT(GPIO_PIN_MUX_REG[gpio_num], FUN_PU);
 }
 
diff --git a/components/hal/esp32s3/include/hal/gpio_ll.h b/components/hal/esp32s3/include/hal/gpio_ll.h
index e98e0395a3..5fde3624f3 100644
--- a/components/hal/esp32s3/include/hal/gpio_ll.h
+++ b/components/hal/esp32s3/include/hal/gpio_ll.h
@@ -52,6 +52,14 @@ static inline void gpio_ll_pullup_en(gpio_dev_t *hw, uint32_t gpio_num)
   */
 static inline void gpio_ll_pullup_dis(gpio_dev_t *hw, uint32_t gpio_num)
 {
+    // The pull-up value of the USB pins are controlled by the pins’ pull-up value together with USB pull-up value
+    // USB DP pin is default to PU enabled
+    // Note that from esp32s3 ECO1, USB_EXCHG_PINS feature has been supported. If this efuse is burnt, the gpio pin
+    // which should be checked is USB_DM_GPIO_NUM instead.
+    if (gpio_num == USB_DP_GPIO_NUM) {
+        SET_PERI_REG_MASK(USB_SERIAL_JTAG_CONF0_REG, USB_SERIAL_JTAG_PAD_PULL_OVERRIDE);
+        CLEAR_PERI_REG_MASK(USB_SERIAL_JTAG_CONF0_REG, USB_SERIAL_JTAG_DP_PULLUP);
+    }
     REG_CLR_BIT(GPIO_PIN_MUX_REG[gpio_num], FUN_PU);
 }
 
diff --git a/components/hal/esp32s3/include/hal/rtc_io_ll.h b/components/hal/esp32s3/include/hal/rtc_io_ll.h
index ff3a314fd5..3516b791af 100644
--- a/components/hal/esp32s3/include/hal/rtc_io_ll.h
+++ b/components/hal/esp32s3/include/hal/rtc_io_ll.h
@@ -17,6 +17,8 @@
 #include "soc/rtc_io_struct.h"
 #include "hal/rtc_io_types.h"
 #include "hal/gpio_types.h"
+#include "soc/io_mux_reg.h"
+#include "soc/usb_serial_jtag_reg.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -181,6 +183,14 @@ static inline void rtcio_ll_pullup_enable(int rtcio_num)
  */
 static inline void rtcio_ll_pullup_disable(int rtcio_num)
 {
+    // The pull-up value of the USB pins are controlled by the pins’ pull-up value together with USB pull-up value
+    // USB DP pin is default to PU enabled
+    // Note that from esp32s3 ECO1, USB_EXCHG_PINS feature has been supported. If this efuse is burnt, the gpio pin
+    // which should be checked is USB_DM_GPIO_NUM instead.
+    if (rtcio_num == USB_DP_GPIO_NUM) {
+        SET_PERI_REG_MASK(USB_SERIAL_JTAG_CONF0_REG, USB_SERIAL_JTAG_PAD_PULL_OVERRIDE);
+        CLEAR_PERI_REG_MASK(USB_SERIAL_JTAG_CONF0_REG, USB_SERIAL_JTAG_DP_PULLUP);
+    }
     if (rtc_io_desc[rtcio_num].pullup) {
         CLEAR_PERI_REG_MASK(rtc_io_desc[rtcio_num].reg, rtc_io_desc[rtcio_num].pullup);
     }
diff --git a/components/hal/esp32s3/include/hal/usb_phy_ll.h b/components/hal/esp32s3/include/hal/usb_phy_ll.h
index fc665552d8..acf6eed102 100644
--- a/components/hal/esp32s3/include/hal/usb_phy_ll.h
+++ b/components/hal/esp32s3/include/hal/usb_phy_ll.h
@@ -59,6 +59,8 @@ static inline void usb_phy_ll_int_jtag_enable(usb_serial_jtag_dev_t *hw)
     hw->conf0.phy_sel = 0;
     // Disable software control USB D+ D- pullup pulldown (Device FS: dp_pullup = 1)
     hw->conf0.pad_pull_override = 0;
+    // Enable USB D+ pullup
+    hw->conf0.dp_pullup = 1;
     // Enable USB pad function
     hw->conf0.usb_pad_enable = 1;
     // phy_sel is controlled by the following register value