diff --git a/components/esp_hw_support/lowpower/port/esp32c5/sleep_cpu.c b/components/esp_hw_support/lowpower/port/esp32c5/sleep_cpu.c index 6c874503f1..34cd70abb4 100644 --- a/components/esp_hw_support/lowpower/port/esp32c5/sleep_cpu.c +++ b/components/esp_hw_support/lowpower/port/esp32c5/sleep_cpu.c @@ -50,29 +50,25 @@ static DRAM_ATTR sleep_cpu_retention_t s_cpu_retention; extern RvCoreCriticalSleepFrame *rv_core_critical_regs_frame; -FORCE_INLINE_ATTR uint32_t save_mstatus_and_disable_global_int(void) +FORCE_INLINE_ATTR void save_csr_disable_global_int(uint32_t *mstatus_val, uint32_t *mintthresh_val) { - return RV_READ_MSTATUS_AND_DISABLE_INTR(); +#if __riscv_zcmp && SOC_CPU_ZCMP_WORKAROUND + *mintthresh_val = rv_utils_set_intlevel_regval(0xff); +#else + (void) mintthresh_val; +#endif + *mstatus_val = RV_READ_MSTATUS_AND_DISABLE_INTR(); } -FORCE_INLINE_ATTR void restore_mstatus(uint32_t mstatus_val) +FORCE_INLINE_ATTR void restore_csr_enable_global_int(uint32_t mstatus_val, uint32_t mintthresh_val) { RV_WRITE_CSR(mstatus, mstatus_val); -} - #if __riscv_zcmp && SOC_CPU_ZCMP_WORKAROUND -FORCE_INLINE_ATTR uint32_t save_mintthresh_and_disable_global_int(void) -{ - /* Due to the reason described in IDF-14279, when mie is set to 0, mintthresh needs to be set to 0xff. */ - // TODO: IDF-14279 DIG-661 - return RV_READ_MINTTHRESH_AND_DISABLE_INTR(); -} - -FORCE_INLINE_ATTR void restore_mintthresh(uint32_t mintthresh_val) -{ - RV_RESTORE_MINTTHRESH(mintthresh_val); -} + rv_utils_restore_intlevel_regval(mintthresh_val); +#else + (void) mintthresh_val; #endif +} static IRAM_ATTR RvCoreNonCriticalSleepFrame * rv_core_noncritical_regs_save(void) { @@ -304,11 +300,10 @@ static IRAM_ATTR esp_err_t do_cpu_retention(sleep_cpu_entry_cb_t goto_sleep, esp_err_t IRAM_ATTR esp_sleep_cpu_retention(uint32_t (*goto_sleep)(uint32_t, uint32_t, uint32_t, bool), uint32_t wakeup_opt, uint32_t reject_opt, uint32_t lslp_mem_inf_fpu, bool dslp) { + uint32_t mstatus = 0; + uint32_t mintthresh = 0; esp_sleep_execute_event_callbacks(SLEEP_EVENT_SW_CPU_TO_MEM_START, (void *)0); - uint32_t mstatus = save_mstatus_and_disable_global_int(); -#if __riscv_zcmp && SOC_CPU_ZCMP_WORKAROUND - uint32_t mintthresh = save_mintthresh_and_disable_global_int(); -#endif + save_csr_disable_global_int(&mstatus, &mintthresh); cpu_domain_dev_regs_save(s_cpu_retention.retent.clic_frame); cpu_domain_dev_regs_save(s_cpu_retention.retent.clint_frame); @@ -331,10 +326,7 @@ esp_err_t IRAM_ATTR esp_sleep_cpu_retention(uint32_t (*goto_sleep)(uint32_t, uin cpu_domain_dev_regs_restore(s_cpu_retention.retent.clint_frame); cpu_domain_dev_regs_restore(s_cpu_retention.retent.clic_frame); } -#if __riscv_zcmp && SOC_CPU_ZCMP_WORKAROUND - restore_mintthresh(mintthresh); -#endif - restore_mstatus(mstatus); + restore_csr_enable_global_int(mstatus, mintthresh); return err; } diff --git a/components/esp_hw_support/lowpower/port/esp32c61/sleep_cpu.c b/components/esp_hw_support/lowpower/port/esp32c61/sleep_cpu.c index f03bef5c4e..e6b8377bef 100644 --- a/components/esp_hw_support/lowpower/port/esp32c61/sleep_cpu.c +++ b/components/esp_hw_support/lowpower/port/esp32c61/sleep_cpu.c @@ -50,14 +50,24 @@ static DRAM_ATTR sleep_cpu_retention_t s_cpu_retention; extern RvCoreCriticalSleepFrame *rv_core_critical_regs_frame; -FORCE_INLINE_ATTR uint32_t save_mstatus_and_disable_global_int(void) +FORCE_INLINE_ATTR void save_csr_disable_global_int(uint32_t *mstatus_val, uint32_t *mintthresh_val) { - return RV_READ_MSTATUS_AND_DISABLE_INTR(); +#if __riscv_zcmp && SOC_CPU_ZCMP_WORKAROUND + *mintthresh_val = rv_utils_set_intlevel_regval(0xff); +#else + (void) mintthresh_val; +#endif + *mstatus_val = RV_READ_MSTATUS_AND_DISABLE_INTR(); } -FORCE_INLINE_ATTR void restore_mstatus(uint32_t mstatus_val) +FORCE_INLINE_ATTR void restore_csr_enable_global_int(uint32_t mstatus_val, uint32_t mintthresh_val) { RV_WRITE_CSR(mstatus, mstatus_val); +#if __riscv_zcmp && SOC_CPU_ZCMP_WORKAROUND + rv_utils_restore_intlevel_regval(mintthresh_val); +#else + (void) mintthresh_val; +#endif } #if __riscv_zcmp && SOC_CPU_ZCMP_WORKAROUND @@ -304,11 +314,10 @@ static IRAM_ATTR esp_err_t do_cpu_retention(sleep_cpu_entry_cb_t goto_sleep, esp_err_t IRAM_ATTR esp_sleep_cpu_retention(uint32_t (*goto_sleep)(uint32_t, uint32_t, uint32_t, bool), uint32_t wakeup_opt, uint32_t reject_opt, uint32_t lslp_mem_inf_fpu, bool dslp) { + uint32_t mstatus = 0; + uint32_t mintthresh = 0; esp_sleep_execute_event_callbacks(SLEEP_EVENT_SW_CPU_TO_MEM_START, (void *)0); - uint32_t mstatus = save_mstatus_and_disable_global_int(); -#if __riscv_zcmp && SOC_CPU_ZCMP_WORKAROUND - uint32_t mintthresh = save_mintthresh_and_disable_global_int(); -#endif + save_csr_disable_global_int(&mstatus, &mintthresh); cpu_domain_dev_regs_save(s_cpu_retention.retent.clic_frame); cpu_domain_dev_regs_save(s_cpu_retention.retent.clint_frame); @@ -331,10 +340,7 @@ esp_err_t IRAM_ATTR esp_sleep_cpu_retention(uint32_t (*goto_sleep)(uint32_t, uin cpu_domain_dev_regs_restore(s_cpu_retention.retent.clint_frame); cpu_domain_dev_regs_restore(s_cpu_retention.retent.clic_frame); } -#if __riscv_zcmp && SOC_CPU_ZCMP_WORKAROUND - restore_mintthresh(mintthresh); -#endif - restore_mstatus(mstatus); + restore_csr_enable_global_int(mstatus, mintthresh); return err; } diff --git a/components/esp_hw_support/lowpower/port/esp32h4/sleep_cpu.c b/components/esp_hw_support/lowpower/port/esp32h4/sleep_cpu.c index 21d62d79ee..6b3749a890 100644 --- a/components/esp_hw_support/lowpower/port/esp32h4/sleep_cpu.c +++ b/components/esp_hw_support/lowpower/port/esp32h4/sleep_cpu.c @@ -64,29 +64,25 @@ static DRAM_ATTR __attribute__((unused)) sleep_cpu_retention_t s_cpu_retention; extern RvCoreCriticalSleepFrame *rv_core_critical_regs_frame[portNUM_PROCESSORS]; -FORCE_INLINE_ATTR uint32_t save_mstatus_and_disable_global_int(void) +FORCE_INLINE_ATTR void save_csr_disable_global_int(uint32_t *mstatus_val, uint32_t *mintthresh_val) { - return RV_READ_MSTATUS_AND_DISABLE_INTR(); +#if __riscv_zcmp && SOC_CPU_ZCMP_WORKAROUND + *mintthresh_val = rv_utils_set_intlevel_regval(0xff); +#else + (void) mintthresh_val; +#endif + *mstatus_val = RV_READ_MSTATUS_AND_DISABLE_INTR(); } -FORCE_INLINE_ATTR void restore_mstatus(uint32_t mstatus_val) +FORCE_INLINE_ATTR void restore_csr_enable_global_int(uint32_t mstatus_val, uint32_t mintthresh_val) { RV_WRITE_CSR(mstatus, mstatus_val); -} - #if __riscv_zcmp && SOC_CPU_ZCMP_WORKAROUND -FORCE_INLINE_ATTR uint32_t save_mintthresh_and_disable_global_int(void) -{ - /* Due to the reason described in IDF-14279, when mie is set to 0, mintthresh needs to be set to 0xff. */ - // TODO: IDF-14279 DIG-661 - return RV_READ_MINTTHRESH_AND_DISABLE_INTR(); -} - -FORCE_INLINE_ATTR void restore_mintthresh(uint32_t mintthresh_val) -{ - RV_RESTORE_MINTTHRESH(mintthresh_val); -} + rv_utils_restore_intlevel_regval(mintthresh_val); +#else + (void) mintthresh_val; #endif +} static IRAM_ATTR RvCoreNonCriticalSleepFrame * rv_core_noncritical_regs_save(void) { @@ -347,15 +343,14 @@ typedef uint32_t (* sleep_cpu_entry_cb_t)(uint32_t, uint32_t, uint32_t, bool); static IRAM_ATTR esp_err_t do_cpu_retention(sleep_cpu_entry_cb_t goto_sleep, uint32_t wakeup_opt, uint32_t reject_opt, uint32_t lslp_mem_inf_fpu, bool dslp) { + uint32_t mstatus = 0; + uint32_t mintthresh = 0; __attribute__((unused)) uint8_t core_id = esp_cpu_get_core_id(); bool reject = false; RvCoreCriticalSleepFrame * frame = s_cpu_retention.retent.critical_frame[core_id]; /* mstatus is core privated CSR, do it near the core critical regs restore */ - uint32_t mstatus = save_mstatus_and_disable_global_int(); -#if __riscv_zcmp && SOC_CPU_ZCMP_WORKAROUND - uint32_t mintthresh = save_mintthresh_and_disable_global_int(); -#endif + save_csr_disable_global_int(&mstatus, &mintthresh); s_fpu_saved[core_id] = xPortFPUContextIsDirty(core_id); if (s_fpu_saved[core_id]) { rv_core_fpu_save(frame); @@ -387,10 +382,7 @@ static IRAM_ATTR esp_err_t do_cpu_retention(sleep_cpu_entry_cb_t goto_sleep, if (s_fpu_saved[core_id]) { rv_core_fpu_restore(frame); } -#if __riscv_zcmp && SOC_CPU_ZCMP_WORKAROUND - restore_mintthresh(mintthresh); -#endif - restore_mstatus(mstatus); + restore_csr_enable_global_int(mstatus, mintthresh); return reject ? reject : pmu_sleep_finish(dslp); } @@ -512,11 +504,10 @@ static IRAM_ATTR void smp_core_do_retention(void) ESP_COMPILER_DIAGNOSTIC_POP("-Wanalyzer-infinite-loop") if (!smp_skip_retention) { + uint32_t mstatus = 0; + uint32_t mintthresh = 0; atomic_store(&s_smp_retention_state[core_id], SMP_BACKUP_START); - uint32_t mstatus = save_mstatus_and_disable_global_int(); -#if __riscv_zcmp && SOC_CPU_ZCMP_WORKAROUND - uint32_t mintthresh = save_mintthresh_and_disable_global_int(); -#endif + save_csr_disable_global_int(&mstatus, &mintthresh); RvCoreCriticalSleepFrame *frame_critical = s_cpu_retention.retent.critical_frame[core_id]; s_fpu_saved[core_id] = xPortFPUContextIsDirty(core_id); if (s_fpu_saved[core_id]) { @@ -550,10 +541,7 @@ static IRAM_ATTR void smp_core_do_retention(void) if (s_fpu_saved[core_id]) { rv_core_fpu_restore(frame_critical); } -#if __riscv_zcmp && SOC_CPU_ZCMP_WORKAROUND - restore_mintthresh(mintthresh); -#endif - restore_mstatus(mstatus); + restore_csr_enable_global_int(mstatus, mintthresh); atomic_store(&s_smp_retention_state[core_id], SMP_RESTORE_DONE); } } diff --git a/components/esp_hw_support/lowpower/port/esp32p4/sleep_cpu.c b/components/esp_hw_support/lowpower/port/esp32p4/sleep_cpu.c index abac8eb4b0..affb48d347 100644 --- a/components/esp_hw_support/lowpower/port/esp32p4/sleep_cpu.c +++ b/components/esp_hw_support/lowpower/port/esp32p4/sleep_cpu.c @@ -52,14 +52,24 @@ static SPM_DRAM_ATTR __attribute__((unused)) sleep_cpu_retention_t s_cpu_retenti extern RvCoreCriticalSleepFrame *rv_core_critical_regs_frame[portNUM_PROCESSORS]; -FORCE_INLINE_ATTR uint32_t save_mstatus_and_disable_global_int(void) +FORCE_INLINE_ATTR void save_csr_disable_global_int(uint32_t *mstatus_val, uint32_t *mintthresh_val) { - return RV_READ_MSTATUS_AND_DISABLE_INTR(); +#if __riscv_zcmp && SOC_CPU_ZCMP_WORKAROUND + *mintthresh_val = rv_utils_set_intlevel_regval(0xff); +#else + (void) mintthresh_val; +#endif + *mstatus_val = RV_READ_MSTATUS_AND_DISABLE_INTR(); } -FORCE_INLINE_ATTR void restore_mstatus(uint32_t mstatus_val) +FORCE_INLINE_ATTR void restore_csr_enable_global_int(uint32_t mstatus_val, uint32_t mintthresh_val) { RV_WRITE_CSR(mstatus, mstatus_val); +#if __riscv_zcmp && SOC_CPU_ZCMP_WORKAROUND + rv_utils_restore_intlevel_regval(mintthresh_val); +#else + (void) mintthresh_val; +#endif } static SPM_IRAM_ATTR RvCoreNonCriticalSleepFrame * rv_core_noncritical_regs_save(void) @@ -263,7 +273,9 @@ static SPM_IRAM_ATTR esp_err_t do_cpu_retention(sleep_cpu_entry_cb_t goto_sleep, bool reject = false; RvCoreCriticalSleepFrame *frame = s_cpu_retention.retent.critical_frame[core_id]; /* mstatus is core privated CSR, do it near the core critical regs restore */ - uint32_t mstatus = save_mstatus_and_disable_global_int(); + uint32_t mstatus = 0; + uint32_t mintthresh = 0; + save_csr_disable_global_int(&mstatus, &mintthresh); s_fpu_saved[core_id] = xPortFPUContextIsDirty(core_id); if (s_fpu_saved[core_id]) { rv_core_fpu_save(frame); @@ -284,7 +296,7 @@ static SPM_IRAM_ATTR esp_err_t do_cpu_retention(sleep_cpu_entry_cb_t goto_sleep, } #endif - reject = (*goto_sleep)(wakeup_opt, reject_opt, lslp_mem_inf_fpu, dslp); + reject = (*goto_sleep)(wakeup_opt, reject_opt, lslp_mem_inf_fpu, dslp); } #if CONFIG_PM_CHECK_SLEEP_RETENTION_FRAME else { @@ -294,7 +306,7 @@ static SPM_IRAM_ATTR esp_err_t do_cpu_retention(sleep_cpu_entry_cb_t goto_sleep, if (s_fpu_saved[core_id]) { rv_core_fpu_restore(frame); } - restore_mstatus(mstatus); + restore_csr_enable_global_int(mstatus, mintthresh); return reject ? reject : pmu_sleep_finish(dslp); } @@ -410,11 +422,13 @@ static SPM_IRAM_ATTR void smp_core_do_retention(void) ESP_COMPILER_DIAGNOSTIC_POP("-Wanalyzer-infinite-loop") if (!smp_skip_retention) { + uint32_t mstatus = 0; + uint32_t mintthresh = 0; atomic_store(&s_smp_retention_state[core_id], SMP_BACKUP_START); rv_core_noncritical_regs_save(); cpu_domain_dev_regs_save(s_cpu_retention.retent.clic_frame[core_id]); RvCoreCriticalSleepFrame *frame_critical = s_cpu_retention.retent.critical_frame[core_id]; - uint32_t mstatus = save_mstatus_and_disable_global_int(); + save_csr_disable_global_int(&mstatus, &mintthresh); s_fpu_saved[core_id] = xPortFPUContextIsDirty(core_id); if (s_fpu_saved[core_id]) { rv_core_fpu_save(frame_critical); @@ -440,7 +454,7 @@ static SPM_IRAM_ATTR void smp_core_do_retention(void) if (s_fpu_saved[core_id]) { rv_core_fpu_restore(frame_critical); } - restore_mstatus(mstatus); + restore_csr_enable_global_int(mstatus, mintthresh); cpu_domain_dev_regs_restore(s_cpu_retention.retent.clic_frame[core_id]); rv_core_noncritical_regs_restore(); atomic_store(&s_smp_retention_state[core_id], SMP_RESTORE_DONE); diff --git a/components/riscv/include/riscv/csr.h b/components/riscv/include/riscv/csr.h index 04c72e1a88..91d7722933 100644 --- a/components/riscv/include/riscv/csr.h +++ b/components/riscv/include/riscv/csr.h @@ -239,15 +239,6 @@ extern "C" { #define RV_READ_MSTATUS_AND_DISABLE_INTR() ({ unsigned long __tmp; \ asm volatile ("csrrci %0, mstatus, 0x8" : "=r"(__tmp)); __tmp; }) -#if __riscv_zcmp && SOC_CPU_ZCMP_WORKAROUND -#define RV_READ_MINTTHRESH_AND_DISABLE_INTR() ({ unsigned long __tmp; \ - asm volatile ( \ - "li t0, 0xff\n\t" \ - "csrrw %0, %1, t0" : "=r"(__tmp) : "i"(MINTTHRESH_CSR) : "t0", "memory"); __tmp; }) - -#define RV_RESTORE_MINTTHRESH(val) \ - asm volatile ("csrw %0, %1" :: "i"(MINTTHRESH_CSR), "r"(val) : "memory") -#endif #define _CSR_STRINGIFY(REG) #REG /* needed so the 'reg' argument can be a macro or a register name */ diff --git a/components/riscv/include/riscv/rv_utils.h b/components/riscv/include/riscv/rv_utils.h index 59bd582120..7b9b6ddfbd 100644 --- a/components/riscv/include/riscv/rv_utils.h +++ b/components/riscv/include/riscv/rv_utils.h @@ -17,6 +17,10 @@ #include "riscv/csr_dsp.h" #include "sdkconfig.h" +#if __riscv_zcmp && SOC_CPU_ZCMP_WORKAROUND +#include "esp_private/interrupt_clic.h" +#endif + #if CONFIG_SECURE_ENABLE_TEE && !NON_OS_BUILD #include "secure_service_num.h" #endif @@ -167,15 +171,50 @@ FORCE_INLINE_ATTR void rv_utils_set_xtvec(uint32_t xtvec_val) // ------------------ Interrupt Control -------------------- +#if __riscv_zcmp && SOC_CPU_ZCMP_WORKAROUND + +extern uint32_t g_xintthresh[SOC_CPU_CORES_NUM]; + +FORCE_INLINE_ATTR void rv_utils_xintthres_raise(void) +{ + /** + * Make sure NOT to use `g_xintthresh[rv_utils_get_core_id()] = rv_utils_set_intlevel_regval(0xff);` + * since that statement would let the compiler first calculate the offset in `g_xintthresh` array + * before setting the interrupt threshold, which may lead to a race condition if an interrupt occurs in between. + */ + uint32_t threshold = rv_utils_set_intlevel_regval(0xff); + g_xintthresh[rv_utils_get_core_id()] = threshold; +} + +FORCE_INLINE_ATTR void rv_utils_xintthres_lower(void) +{ + rv_utils_restore_intlevel_regval(g_xintthresh[rv_utils_get_core_id()]); +} + +#else + +FORCE_INLINE_ATTR void rv_utils_xintthres_raise(void) +{ +} + +FORCE_INLINE_ATTR void rv_utils_xintthres_lower(void) +{ +} + +#endif // __riscv_zcmp && SOC_CPU_ZCMP_WORKAROUND + + FORCE_INLINE_ATTR void rv_utils_intr_enable(uint32_t intr_mask) { #if CONFIG_SECURE_ENABLE_TEE && !NON_OS_BUILD esp_tee_intr_sec_srv_cb(2, SS_RV_UTILS_INTR_ENABLE, intr_mask); #else // Disable all interrupts to make updating of the interrupt mask atomic. + rv_utils_xintthres_raise(); unsigned old_mstatus = RV_CLEAR_CSR(mstatus, MSTATUS_MIE); esprv_int_enable(intr_mask); RV_SET_CSR(mstatus, old_mstatus & MSTATUS_MIE); + rv_utils_xintthres_lower(); #endif } @@ -185,9 +224,11 @@ FORCE_INLINE_ATTR void rv_utils_intr_disable(uint32_t intr_mask) esp_tee_intr_sec_srv_cb(2, SS_RV_UTILS_INTR_DISABLE, intr_mask); #else // Disable all interrupts to make updating of the interrupt mask atomic. + rv_utils_xintthres_raise(); unsigned old_mstatus = RV_CLEAR_CSR(mstatus, MSTATUS_MIE); esprv_int_disable(intr_mask); RV_SET_CSR(mstatus, old_mstatus & MSTATUS_MIE); + rv_utils_xintthres_lower(); #endif } @@ -198,10 +239,12 @@ FORCE_INLINE_ATTR void rv_utils_intr_global_enable(void) #else RV_SET_CSR(mstatus, MSTATUS_MIE); #endif + rv_utils_xintthres_lower(); } FORCE_INLINE_ATTR void rv_utils_intr_global_disable(void) { + rv_utils_xintthres_raise(); #if CONFIG_SECURE_ENABLE_TEE if (IS_PRV_M_MODE()) { RV_CLEAR_CSR(mstatus, MSTATUS_MIE); diff --git a/components/riscv/interrupt_clic.c b/components/riscv/interrupt_clic.c index 95d48a3cc4..160ebde58d 100644 --- a/components/riscv/interrupt_clic.c +++ b/components/riscv/interrupt_clic.c @@ -7,7 +7,16 @@ #include "riscv/rv_utils.h" #include "hal/interrupt_clic_ll.h" #include "esp_private/interrupt_clic.h" +#include "esp_attr.h" +#if __riscv_zcmp && SOC_CPU_ZCMP_WORKAROUND +/* Due to a hardware bug, the interrupt threshold must be saved before disabling the interrupts. + * Make sure the array is always accessible. */ +#if CONFIG_IDF_TARGET_ESP32P4 +SPM_DRAM_ATTR +#endif +uint32_t g_xintthresh[SOC_CPU_CORES_NUM]; +#endif void intr_matrix_route(int intr_src, int intr_num) { diff --git a/components/riscv/vectors.S b/components/riscv/vectors.S index 88d446746d..ce1326cde8 100644 --- a/components/riscv/vectors.S +++ b/components/riscv/vectors.S @@ -184,8 +184,8 @@ .macro mintthresh_csr_disable reg #if __riscv_zcmp && SOC_CPU_ZCMP_WORKAROUND /* Workaround for triggering an interrupt even when mstatus.mie is 0, when cm.push is called. */ - li t0, 0xff - csrrw \reg, MINTTHRESH_CSR, t0 + li \reg, 0xff + csrrw \reg, MINTTHRESH_CSR, \reg #endif .endm