diff --git a/components/esp_hw_support/port/esp32c5/rtc_clk.c b/components/esp_hw_support/port/esp32c5/rtc_clk.c
index dc9bb274d1..7be9200dc6 100644
--- a/components/esp_hw_support/port/esp32c5/rtc_clk.c
+++ b/components/esp_hw_support/port/esp32c5/rtc_clk.c
@@ -15,6 +15,7 @@
 #include "esp_private/rtc_clk.h"
 #include "esp_hw_log.h"
 #include "esp_rom_sys.h"
+#include "esp_sleep.h"
 #include "hal/clk_tree_ll.h"
 #include "hal/regi2c_ctrl_ll.h"
 #include "hal/gpio_ll.h"
@@ -97,6 +98,13 @@ void rtc_clk_slow_src_set(soc_rtc_slow_clk_src_t clk_src)
 {
     clk_ll_rtc_slow_set_src(clk_src);
     esp_rom_delay_us(SOC_DELAY_RTC_SLOW_CLK_SWITCH);
+#ifndef BOOTLOADER_BUILD
+    if ((clk_src == SOC_RTC_SLOW_CLK_SRC_XTAL32K) || (clk_src == SOC_RTC_SLOW_CLK_SRC_OSC_SLOW)) {
+        esp_sleep_pd_config(ESP_PD_DOMAIN_XTAL32K, ESP_PD_OPTION_ON);
+    } else {
+        esp_sleep_pd_config(ESP_PD_DOMAIN_XTAL32K, ESP_PD_OPTION_AUTO);
+    }
+#endif
 }
 
 soc_rtc_slow_clk_src_t rtc_clk_slow_src_get(void)
diff --git a/components/esp_hw_support/port/esp32c6/rtc_clk.c b/components/esp_hw_support/port/esp32c6/rtc_clk.c
index bdb38b696b..a293f3d29b 100644
--- a/components/esp_hw_support/port/esp32c6/rtc_clk.c
+++ b/components/esp_hw_support/port/esp32c6/rtc_clk.c
@@ -15,6 +15,7 @@
 #include "esp_private/rtc_clk.h"
 #include "esp_hw_log.h"
 #include "esp_rom_sys.h"
+#include "esp_sleep.h"
 #include "hal/clk_tree_ll.h"
 #include "hal/regi2c_ctrl_ll.h"
 #include "hal/gpio_ll.h"
@@ -105,6 +106,18 @@ void rtc_clk_slow_src_set(soc_rtc_slow_clk_src_t clk_src)
 {
     clk_ll_rtc_slow_set_src(clk_src);
     esp_rom_delay_us(SOC_DELAY_RTC_SLOW_CLK_SWITCH);
+#ifndef BOOTLOADER_BUILD
+    if ((clk_src == SOC_RTC_SLOW_CLK_SRC_XTAL32K) || (clk_src == SOC_RTC_SLOW_CLK_SRC_OSC_SLOW)) {
+        esp_sleep_pd_config(ESP_PD_DOMAIN_XTAL32K, ESP_PD_OPTION_ON);
+    } else {
+        esp_sleep_pd_config(ESP_PD_DOMAIN_XTAL32K, ESP_PD_OPTION_AUTO);
+    }
+    if (clk_src == SOC_RTC_SLOW_CLK_SRC_RC32K) {
+        esp_sleep_pd_config(ESP_PD_DOMAIN_RC32K, ESP_PD_OPTION_ON);
+    } else {
+        esp_sleep_pd_config(ESP_PD_DOMAIN_RC32K, ESP_PD_OPTION_AUTO);
+    }
+#endif
 }
 
 soc_rtc_slow_clk_src_t rtc_clk_slow_src_get(void)
diff --git a/components/esp_hw_support/port/esp32c61/rtc_clk.c b/components/esp_hw_support/port/esp32c61/rtc_clk.c
index 2578b14900..04363f56b4 100644
--- a/components/esp_hw_support/port/esp32c61/rtc_clk.c
+++ b/components/esp_hw_support/port/esp32c61/rtc_clk.c
@@ -15,6 +15,7 @@
 #include "esp_private/rtc_clk.h"
 #include "esp_hw_log.h"
 #include "esp_rom_sys.h"
+#include "esp_sleep.h"
 #include "hal/clk_tree_ll.h"
 #include "hal/regi2c_ctrl_ll.h"
 #include "hal/gpio_ll.h"
@@ -95,6 +96,13 @@ void rtc_clk_slow_src_set(soc_rtc_slow_clk_src_t clk_src)
 {
     clk_ll_rtc_slow_set_src(clk_src);
     esp_rom_delay_us(SOC_DELAY_RTC_SLOW_CLK_SWITCH);
+#ifndef BOOTLOADER_BUILD
+    if ((clk_src == SOC_RTC_SLOW_CLK_SRC_XTAL32K) || (clk_src == SOC_RTC_SLOW_CLK_SRC_OSC_SLOW)) {
+        esp_sleep_pd_config(ESP_PD_DOMAIN_XTAL32K, ESP_PD_OPTION_ON);
+    } else {
+        esp_sleep_pd_config(ESP_PD_DOMAIN_XTAL32K, ESP_PD_OPTION_AUTO);
+    }
+#endif
 }
 
 soc_rtc_slow_clk_src_t rtc_clk_slow_src_get(void)
diff --git a/components/esp_hw_support/port/esp32h2/rtc_clk.c b/components/esp_hw_support/port/esp32h2/rtc_clk.c
index 63e10b3fe2..3960eb5d9e 100644
--- a/components/esp_hw_support/port/esp32h2/rtc_clk.c
+++ b/components/esp_hw_support/port/esp32h2/rtc_clk.c
@@ -15,6 +15,7 @@
 #include "esp_private/rtc_clk.h"
 #include "esp_hw_log.h"
 #include "esp_rom_sys.h"
+#include "esp_sleep.h"
 #include "hal/clk_tree_ll.h"
 #include "hal/regi2c_ctrl_ll.h"
 #include "hal/gpio_ll.h"
@@ -121,6 +122,13 @@ void rtc_clk_slow_src_set(soc_rtc_slow_clk_src_t clk_src)
 {
     clk_ll_rtc_slow_set_src(clk_src);
     esp_rom_delay_us(SOC_DELAY_RTC_SLOW_CLK_SWITCH);
+#ifndef BOOTLOADER_BUILD
+    if ((clk_src == SOC_RTC_SLOW_CLK_SRC_XTAL32K) || (clk_src == SOC_RTC_SLOW_CLK_SRC_OSC_SLOW)) {
+        esp_sleep_pd_config(ESP_PD_DOMAIN_XTAL32K, ESP_PD_OPTION_ON);
+    } else {
+        esp_sleep_pd_config(ESP_PD_DOMAIN_XTAL32K, ESP_PD_OPTION_AUTO);
+    }
+#endif
 }
 
 soc_rtc_slow_clk_src_t rtc_clk_slow_src_get(void)
diff --git a/components/esp_hw_support/port/esp32h21/rtc_clk.c b/components/esp_hw_support/port/esp32h21/rtc_clk.c
index 4183989c9c..7cfe686943 100644
--- a/components/esp_hw_support/port/esp32h21/rtc_clk.c
+++ b/components/esp_hw_support/port/esp32h21/rtc_clk.c
@@ -15,6 +15,7 @@
 #include "esp_private/rtc_clk.h"
 #include "esp_hw_log.h"
 #include "esp_rom_sys.h"
+#include "esp_sleep.h"
 #include "hal/clk_tree_ll.h"
 #include "hal/regi2c_ctrl_ll.h"
 #include "hal/gpio_ll.h"
@@ -96,6 +97,13 @@ void rtc_clk_slow_src_set(soc_rtc_slow_clk_src_t clk_src)
 {
     clk_ll_rtc_slow_set_src(clk_src);
     esp_rom_delay_us(SOC_DELAY_RTC_SLOW_CLK_SWITCH);
+#ifndef BOOTLOADER_BUILD
+    if ((clk_src == SOC_RTC_SLOW_CLK_SRC_XTAL32K) || (clk_src == SOC_RTC_SLOW_CLK_SRC_OSC_SLOW)) {
+        esp_sleep_pd_config(ESP_PD_DOMAIN_XTAL32K, ESP_PD_OPTION_ON);
+    } else {
+        esp_sleep_pd_config(ESP_PD_DOMAIN_XTAL32K, ESP_PD_OPTION_AUTO);
+    }
+#endif
 }
 
 soc_rtc_slow_clk_src_t rtc_clk_slow_src_get(void)
diff --git a/components/esp_hw_support/port/esp32h4/rtc_clk.c b/components/esp_hw_support/port/esp32h4/rtc_clk.c
index 1b241b140d..d28b821589 100644
--- a/components/esp_hw_support/port/esp32h4/rtc_clk.c
+++ b/components/esp_hw_support/port/esp32h4/rtc_clk.c
@@ -15,6 +15,7 @@
 #include "esp_private/rtc_clk.h"
 #include "esp_hw_log.h"
 #include "esp_rom_sys.h"
+#include "esp_sleep.h"
 #include "hal/clk_tree_ll.h"
 #include "hal/gpio_ll.h"
 #include "hal/regi2c_ctrl_ll.h"
@@ -95,6 +96,13 @@ void rtc_clk_slow_src_set(soc_rtc_slow_clk_src_t clk_src)
 {
     clk_ll_rtc_slow_set_src(clk_src);
     esp_rom_delay_us(SOC_DELAY_RTC_SLOW_CLK_SWITCH);
+#ifndef BOOTLOADER_BUILD
+    if ((clk_src == SOC_RTC_SLOW_CLK_SRC_XTAL32K) || (clk_src == SOC_RTC_SLOW_CLK_SRC_OSC_SLOW)) {
+        esp_sleep_pd_config(ESP_PD_DOMAIN_XTAL32K, ESP_PD_OPTION_ON);
+    } else {
+        esp_sleep_pd_config(ESP_PD_DOMAIN_XTAL32K, ESP_PD_OPTION_AUTO);
+    }
+#endif
 }
 
 soc_rtc_slow_clk_src_t rtc_clk_slow_src_get(void)
diff --git a/components/esp_hw_support/port/esp32p4/rtc_clk.c b/components/esp_hw_support/port/esp32p4/rtc_clk.c
index 24188e1c50..eaa3276e35 100644
--- a/components/esp_hw_support/port/esp32p4/rtc_clk.c
+++ b/components/esp_hw_support/port/esp32p4/rtc_clk.c
@@ -16,6 +16,7 @@
 #include "esp_attr.h"
 #include "esp_hw_log.h"
 #include "esp_rom_sys.h"
+#include "esp_sleep.h"
 #include "hal/clk_tree_ll.h"
 #include "hal/regi2c_ctrl_ll.h"
 #include "hal/gpio_ll.h"
@@ -99,6 +100,18 @@ void rtc_clk_slow_src_set(soc_rtc_slow_clk_src_t clk_src)
 {
     clk_ll_rtc_slow_set_src(clk_src);
     esp_rom_delay_us(SOC_DELAY_RTC_SLOW_CLK_SWITCH);
+#ifndef BOOTLOADER_BUILD
+    if (clk_src == SOC_RTC_SLOW_CLK_SRC_XTAL32K) {
+        esp_sleep_pd_config(ESP_PD_DOMAIN_XTAL32K, ESP_PD_OPTION_ON);
+    } else {
+        esp_sleep_pd_config(ESP_PD_DOMAIN_XTAL32K, ESP_PD_OPTION_AUTO);
+    }
+    if (clk_src == SOC_RTC_SLOW_CLK_SRC_RC32K) {
+        esp_sleep_pd_config(ESP_PD_DOMAIN_RC32K, ESP_PD_OPTION_ON);
+    } else {
+        esp_sleep_pd_config(ESP_PD_DOMAIN_RC32K, ESP_PD_OPTION_AUTO);
+    }
+#endif
 }
 
 soc_rtc_slow_clk_src_t rtc_clk_slow_src_get(void)
diff --git a/components/esp_system/port/soc/esp32c6/clk.c b/components/esp_system/port/soc/esp32c6/clk.c
index 0e20e4ab2d..6ebb6809e0 100644
--- a/components/esp_system/port/soc/esp32c6/clk.c
+++ b/components/esp_system/port/soc/esp32c6/clk.c
@@ -216,6 +216,7 @@ void rtc_clk_select_rtc_slow_clk(void)
  */
 __attribute__((weak)) void esp_perip_clk_init(void)
 {
+#if CONFIG_ESP_WIFI_ENABLED
     /* During system initialization, the low-power clock source of the modem
      * (WiFi, BLE or Coexist) follows the configuration of the slow clock source
      * of the system. If the WiFi, BLE or Coexist module needs a higher
@@ -231,6 +232,7 @@ __attribute__((weak)) void esp_perip_clk_init(void)
                                                                         : (rtc_slow_clk_src == SOC_RTC_SLOW_CLK_SRC_OSC_SLOW) ? MODEM_CLOCK_LPCLK_SRC_EXT32K   \
                                                                         : MODEM_CLOCK_LPCLK_SRC_RC_SLOW);
     modem_clock_select_lp_clock_source(PERIPH_WIFI_MODULE, modem_lpclk_src, 0);
+#endif
 
     soc_reset_reason_t rst_reason = esp_rom_get_reset_reason(0);
     if ((rst_reason != RESET_REASON_CPU0_MWDT0) && (rst_reason != RESET_REASON_CPU0_MWDT1)      \
diff --git a/components/esp_system/port/soc/esp32c61/clk.c b/components/esp_system/port/soc/esp32c61/clk.c
index 906ead4ecf..9a45b79a61 100644
--- a/components/esp_system/port/soc/esp32c61/clk.c
+++ b/components/esp_system/port/soc/esp32c61/clk.c
@@ -181,7 +181,7 @@ void rtc_clk_select_rtc_slow_clk(void)
  */
 __attribute__((weak)) void esp_perip_clk_init(void)
 {
-#if SOC_MODEM_CLOCK_SUPPORTED
+#if CONFIG_ESP_WIFI_ENABLED
     /* During system initialization, the low-power clock source of the modem
      * (WiFi, BLE or Coexist) follows the configuration of the slow clock source
      * of the system. If the WiFi, BLE or Coexist module needs a higher
diff --git a/components/esp_system/port/soc/esp32h2/clk.c b/components/esp_system/port/soc/esp32h2/clk.c
index 203bbba9ff..eb1a28da19 100644
--- a/components/esp_system/port/soc/esp32h2/clk.c
+++ b/components/esp_system/port/soc/esp32h2/clk.c
@@ -13,6 +13,7 @@
 #include "esp_log.h"
 #include "esp_cpu.h"
 #include "esp_clk_internal.h"
+#include "esp_sleep.h"
 #include "esp32h2/rom/ets_sys.h"
 #include "esp32h2/rom/uart.h"
 #include "soc/soc.h"
@@ -48,7 +49,6 @@
 #include "esp_private/esp_pmu.h"
 #include "esp_rom_serial_output.h"
 #include "esp_rom_sys.h"
-#include "esp_sleep.h"
 
 /* Number of cycles to wait from the 32k XTAL oscillator to consider it running.
  * Larger values increase startup delay. Smaller values may cause false positive
@@ -220,13 +220,6 @@ void rtc_clk_select_rtc_slow_clk(void)
  */
 __attribute__((weak)) void esp_perip_clk_init(void)
 {
-    soc_rtc_slow_clk_src_t rtc_slow_clk_src = rtc_clk_slow_src_get();
-    esp_sleep_pd_domain_t pu_domain = (esp_sleep_pd_domain_t)(\
-                                                              (rtc_slow_clk_src == SOC_RTC_SLOW_CLK_SRC_XTAL32K) ? ESP_PD_DOMAIN_XTAL32K \
-                                                              : (rtc_slow_clk_src == SOC_RTC_SLOW_CLK_SRC_RC32K) ? ESP_PD_DOMAIN_RC32K \
-                                                              : ESP_PD_DOMAIN_MAX);
-    esp_sleep_pd_config(pu_domain, ESP_PD_OPTION_ON);
-
     soc_reset_reason_t rst_reason = esp_rom_get_reset_reason(0);
     if ((rst_reason != RESET_REASON_CPU0_MWDT0) && (rst_reason != RESET_REASON_CPU0_MWDT1)      \
             && (rst_reason != RESET_REASON_CPU0_SW) && (rst_reason != RESET_REASON_CPU0_RTC_WDT)    \
diff --git a/components/esp_system/port/soc/esp32h21/clk.c b/components/esp_system/port/soc/esp32h21/clk.c
index 8692ea17f4..5b383d089d 100644
--- a/components/esp_system/port/soc/esp32h21/clk.c
+++ b/components/esp_system/port/soc/esp32h21/clk.c
@@ -25,7 +25,6 @@
 #include "esp_private/esp_pmu.h"
 #include "esp_rom_serial_output.h"
 #include "esp_rom_sys.h"
-#include "esp_sleep.h"
 
 /* Number of cycles to wait from the 32k XTAL oscillator to consider it running.
  * Larger values increase startup delay. Smaller values may cause false positive
@@ -185,10 +184,5 @@ void rtc_clk_select_rtc_slow_clk(void)
  */
 __attribute__((weak)) void esp_perip_clk_init(void)
 {
-    soc_rtc_slow_clk_src_t rtc_slow_clk_src = rtc_clk_slow_src_get();
-    esp_sleep_pd_domain_t pu_domain = (esp_sleep_pd_domain_t)(\
-                                                              (rtc_slow_clk_src == SOC_RTC_SLOW_CLK_SRC_XTAL32K) ? ESP_PD_DOMAIN_XTAL32K \
-                                                              : ESP_PD_DOMAIN_MAX);
-    esp_sleep_pd_config(pu_domain, ESP_PD_OPTION_ON);
     ESP_EARLY_LOGW(TAG, "esp_perip_clk_init() has not been implemented yet");
 }
diff --git a/components/esp_system/port/soc/esp32h4/clk.c b/components/esp_system/port/soc/esp32h4/clk.c
index 3e4a969229..ed7938156c 100644
--- a/components/esp_system/port/soc/esp32h4/clk.c
+++ b/components/esp_system/port/soc/esp32h4/clk.c
@@ -182,9 +182,5 @@ void rtc_clk_select_rtc_slow_clk(void)
  */
 __attribute__((weak)) void esp_perip_clk_init(void)
 {
-    soc_rtc_slow_clk_src_t rtc_slow_clk_src = rtc_clk_slow_src_get();
-    esp_sleep_pd_domain_t pu_domain = (esp_sleep_pd_domain_t)(\
-                                                              (rtc_slow_clk_src == SOC_RTC_SLOW_CLK_SRC_XTAL32K) ? ESP_PD_DOMAIN_XTAL32K \
-                                                              : ESP_PD_DOMAIN_MAX);
-    esp_sleep_pd_config(pu_domain, ESP_PD_OPTION_ON);
+    ESP_EARLY_LOGW(TAG, "esp_perip_clk_init() has not been implemented yet");
 }
diff --git a/components/esp_system/port/soc/esp32p4/clk.c b/components/esp_system/port/soc/esp32p4/clk.c
index d41a7843ad..d8b8cfc070 100644
--- a/components/esp_system/port/soc/esp32p4/clk.c
+++ b/components/esp_system/port/soc/esp32p4/clk.c
@@ -229,25 +229,6 @@ void rtc_clk_select_rtc_slow_clk(void)
  */
 __attribute__((weak)) void esp_perip_clk_init(void)
 {
-    soc_rtc_slow_clk_src_t rtc_slow_clk_src = rtc_clk_slow_src_get();
-
-    if (rtc_slow_clk_src == SOC_RTC_SLOW_CLK_SRC_RC_SLOW) {
-        esp_sleep_pd_config(ESP_PD_DOMAIN_XTAL32K, ESP_PD_OPTION_AUTO);
-        REG_CLR_BIT(LP_CLKRST_HP_CLK_CTRL_REG, LP_CLKRST_HP_XTAL_32K_CLK_EN);
-        REG_CLR_BIT(LP_CLKRST_HP_CLK_CTRL_REG, LP_CLKRST_HP_RC_32K_CLK_EN);
-        esp_sleep_pd_config(ESP_PD_DOMAIN_RC32K, ESP_PD_OPTION_AUTO);
-    } else if (rtc_slow_clk_src == SOC_RTC_SLOW_CLK_SRC_XTAL32K) {
-        // RC slow (150K) always ON
-        esp_sleep_pd_config(ESP_PD_DOMAIN_RC32K, ESP_PD_OPTION_AUTO);
-        REG_CLR_BIT(LP_CLKRST_HP_CLK_CTRL_REG, LP_CLKRST_HP_RC_32K_CLK_EN);
-        esp_sleep_pd_config(ESP_PD_DOMAIN_XTAL32K, ESP_PD_OPTION_ON);
-    } else if (rtc_slow_clk_src == SOC_RTC_SLOW_CLK_SRC_RC32K) {
-        // RC slow (150K) always ON
-        esp_sleep_pd_config(ESP_PD_DOMAIN_XTAL32K, ESP_PD_OPTION_AUTO);
-        REG_CLR_BIT(LP_CLKRST_HP_CLK_CTRL_REG, LP_CLKRST_HP_XTAL_32K_CLK_EN);
-        esp_sleep_pd_config(ESP_PD_DOMAIN_RC32K, ESP_PD_OPTION_ON);
-    }
-
     soc_reset_reason_t rst_reason = esp_rom_get_reset_reason(0);
     // HP modules related clock control
     if ((rst_reason == RESET_REASON_CHIP_POWER_ON) || (rst_reason == RESET_REASON_CORE_PMU_PWR_DOWN)
diff --git a/components/hal/esp32p4/include/hal/clk_tree_ll.h b/components/hal/esp32p4/include/hal/clk_tree_ll.h
index a3ec789860..0950a644f5 100644
--- a/components/hal/esp32p4/include/hal/clk_tree_ll.h
+++ b/components/hal/esp32p4/include/hal/clk_tree_ll.h
@@ -181,6 +181,7 @@ static inline __attribute__((always_inline)) void clk_ll_xtal32k_enable(clk_ll_x
     LP_AON_CLKRST.xtal32k.dbuf_xtal32k = cfg.dbuf;
     // Enable xtal32k xpd
     SET_PERI_REG_MASK(PMU_HP_SLEEP_LP_CK_POWER_REG, PMU_HP_SLEEP_XPD_XTAL32K);
+    REG_SET_BIT(LP_CLKRST_HP_CLK_CTRL_REG, LP_CLKRST_HP_XTAL_32K_CLK_EN);
 }
 
 /**
@@ -188,6 +189,7 @@ static inline __attribute__((always_inline)) void clk_ll_xtal32k_enable(clk_ll_x
  */
 static inline __attribute__((always_inline)) void clk_ll_xtal32k_disable(void)
 {
+    REG_CLR_BIT(LP_CLKRST_HP_CLK_CTRL_REG, LP_CLKRST_HP_XTAL_32K_CLK_EN);
     // Disable xtal32k xpd
     CLEAR_PERI_REG_MASK(PMU_HP_SLEEP_LP_CK_POWER_REG, PMU_HP_SLEEP_XPD_XTAL32K);
 }
@@ -207,6 +209,7 @@ static inline __attribute__((always_inline)) bool clk_ll_xtal32k_is_enabled(void
  */
 static inline __attribute__((always_inline)) void clk_ll_rc32k_enable(void)
 {
+    REG_SET_BIT(LP_CLKRST_HP_CLK_CTRL_REG, LP_CLKRST_HP_RC_32K_CLK_EN);
     // Enable rc32k xpd status
     SET_PERI_REG_MASK(PMU_HP_SLEEP_LP_CK_POWER_REG, PMU_HP_SLEEP_XPD_RC32K);
 }
@@ -218,6 +221,7 @@ static inline __attribute__((always_inline)) void clk_ll_rc32k_disable(void)
 {
     // Disable rc32k xpd status
     CLEAR_PERI_REG_MASK(PMU_HP_SLEEP_LP_CK_POWER_REG, PMU_HP_SLEEP_XPD_RC32K);
+    REG_CLR_BIT(LP_CLKRST_HP_CLK_CTRL_REG, LP_CLKRST_HP_RC_32K_CLK_EN);
 }
 
 /**