Skip to content

GitLab

Commit b3e2268e authored by danh-arm's avatar danh-arm
Browse files

Merge pull request #59 from athoelke/review-barriers-v2 

Correct usage of data and instruction barriers v2
parents aefebcc3 8cec598b
Show whitespace changes
Inline Side-by-side
bl1/aarch64/bl1_arch_setup.c
View file @ b3e2268e
...@@ -44,6 +44,7 @@ void bl1_arch_setup(void) ...@@ -44,6 +44,7 @@ void bl1_arch_setup(void)
tmp_reg |= (SCTLR_A_BIT | SCTLR_SA_BIT); tmp_reg |= (SCTLR_A_BIT | SCTLR_SA_BIT);
tmp_reg &= ~SCTLR_EE_BIT; tmp_reg &= ~SCTLR_EE_BIT;
write_sctlr_el3(tmp_reg); write_sctlr_el3(tmp_reg);
isb();
/* /*
* Enable HVCs, route FIQs to EL3, set the next EL to be AArch64, route * Enable HVCs, route FIQs to EL3, set the next EL to be AArch64, route
......
bl1/aarch64/bl1_entrypoint.S
View file @ b3e2268e
...@@ -86,7 +86,6 @@ func bl1_entrypoint ...@@ -86,7 +86,6 @@ func bl1_entrypoint
mrs x0, sctlr_el3 mrs x0, sctlr_el3
orr x0, x0, #SCTLR_I_BIT orr x0, x0, #SCTLR_I_BIT
msr sctlr_el3, x0 msr sctlr_el3, x0
isb isb
_wait_for_entrypoint: _wait_for_entrypoint:
......
bl1/aarch64/bl1_exceptions.S
View file @ b3e2268e
...@@ -221,6 +221,7 @@ func process_exception ...@@ -221,6 +221,7 @@ func process_exception
bl read_sctlr_el3 bl read_sctlr_el3
bic x0, x0, x1 bic x0, x0, x1
bl write_sctlr_el3 bl write_sctlr_el3
isb
mov x0, #DCCISW mov x0, #DCCISW
bl dcsw_op_all bl dcsw_op_all
bl tlbialle3 bl tlbialle3
......
bl2/aarch64/bl2_entrypoint.S
View file @ b3e2268e
...@@ -73,7 +73,6 @@ func bl2_entrypoint ...@@ -73,7 +73,6 @@ func bl2_entrypoint
mrs x0, sctlr_el1 mrs x0, sctlr_el1
orr x0, x0, #SCTLR_I_BIT orr x0, x0, #SCTLR_I_BIT
msr sctlr_el1, x0 msr sctlr_el1, x0
isb isb
/* --------------------------------------------- /* ---------------------------------------------
......
bl31/aarch64/bl31_entrypoint.S
View file @ b3e2268e
...@@ -89,7 +89,6 @@ func bl31_entrypoint ...@@ -89,7 +89,6 @@ func bl31_entrypoint
mrs x1, sctlr_el3 mrs x1, sctlr_el3
orr x1, x1, #SCTLR_I_BIT orr x1, x1, #SCTLR_I_BIT
msr sctlr_el3, x1 msr sctlr_el3, x1
isb isb
/* --------------------------------------------- /* ---------------------------------------------
......
bl31/bl31_main.c
View file @ b3e2268e
...@@ -100,6 +100,7 @@ void bl31_main(void) ...@@ -100,6 +100,7 @@ void bl31_main(void)
assert(cm_get_context(mpidr, NON_SECURE)); assert(cm_get_context(mpidr, NON_SECURE));
cm_set_next_eret_context(NON_SECURE); cm_set_next_eret_context(NON_SECURE);
write_vbar_el3((uint64_t) runtime_exceptions); write_vbar_el3((uint64_t) runtime_exceptions);
isb();
next_image_type = NON_SECURE; next_image_type = NON_SECURE;
/* /*
......
drivers/arm/gic/aarch64/gic_v3_sysregs.S
View file @ b3e2268e
...@@ -67,23 +67,19 @@ func read_icc_sre_el3 ...@@ -67,23 +67,19 @@ func read_icc_sre_el3
func write_icc_sre_el1 func write_icc_sre_el1
msr ICC_SRE_EL1, x0 msr ICC_SRE_EL1, x0
isb
ret ret
func write_icc_sre_el2 func write_icc_sre_el2
msr ICC_SRE_EL2, x0 msr ICC_SRE_EL2, x0
isb
ret ret
func write_icc_sre_el3 func write_icc_sre_el3
msr ICC_SRE_EL3, x0 msr ICC_SRE_EL3, x0
isb
ret ret
func write_icc_pmr_el1 func write_icc_pmr_el1
msr ICC_PMR_EL1, x0 msr ICC_PMR_EL1, x0
isb
ret ret
lib/aarch64/cache_helpers.S
View file @ b3e2268e
...@@ -46,57 +46,41 @@ ...@@ -46,57 +46,41 @@
func dcisw func dcisw
dc isw, x0 dc isw, x0
dsb sy
isb
ret ret
func dccisw func dccisw
dc cisw, x0 dc cisw, x0
dsb sy
isb
ret ret
func dccsw func dccsw
dc csw, x0 dc csw, x0
dsb sy
isb
ret ret
func dccvac func dccvac
dc cvac, x0 dc cvac, x0
dsb sy
isb
ret ret
func dcivac func dcivac
dc ivac, x0 dc ivac, x0
dsb sy
isb
ret ret
func dccivac func dccivac
dc civac, x0 dc civac, x0
dsb sy
isb
ret ret
func dccvau func dccvau
dc cvau, x0 dc cvau, x0
dsb sy
isb
ret ret
func dczva func dczva
dc zva, x0 dc zva, x0
dsb sy
isb
ret ret
......
lib/aarch64/cpu_helpers.S
View file @ b3e2268e
...@@ -52,5 +52,6 @@ smp_setup_begin: ...@@ -52,5 +52,6 @@ smp_setup_begin:
bl read_cpuectlr bl read_cpuectlr
orr x0, x0, #CPUECTLR_SMP_BIT orr x0, x0, #CPUECTLR_SMP_BIT
bl write_cpuectlr bl write_cpuectlr
isb
smp_setup_end: smp_setup_end:
ret x19 ret x19
lib/aarch64/misc_helpers.S
View file @ b3e2268e
...@@ -187,19 +187,16 @@ func write_spsr ...@@ -187,19 +187,16 @@ func write_spsr
func write_spsr_el1 func write_spsr_el1
msr spsr_el1, x0 msr spsr_el1, x0
isb
ret ret
func write_spsr_el2 func write_spsr_el2
msr spsr_el2, x0 msr spsr_el2, x0
isb
ret ret
func write_spsr_el3 func write_spsr_el3
msr spsr_el3, x0 msr spsr_el3, x0
isb
ret ret
...@@ -240,19 +237,16 @@ func write_elr ...@@ -240,19 +237,16 @@ func write_elr
func write_elr_el1 func write_elr_el1
msr elr_el1, x0 msr elr_el1, x0
isb
ret ret
func write_elr_el2 func write_elr_el2
msr elr_el2, x0 msr elr_el2, x0
isb
ret ret
func write_elr_el3 func write_elr_el3
msr elr_el3, x0 msr elr_el3, x0
isb
ret ret
......
lib/aarch64/sysreg_helpers.S
View file @ b3e2268e
...@@ -201,19 +201,16 @@ func read_vbar_el3 ...@@ -201,19 +201,16 @@ func read_vbar_el3
func write_vbar_el1 func write_vbar_el1
msr vbar_el1, x0 msr vbar_el1, x0
isb
ret ret
func write_vbar_el2 func write_vbar_el2
msr vbar_el2, x0 msr vbar_el2, x0
isb
ret ret
func write_vbar_el3 func write_vbar_el3
msr vbar_el3, x0 msr vbar_el3, x0
isb
ret ret
...@@ -238,19 +235,16 @@ func read_afsr0_el3 ...@@ -238,19 +235,16 @@ func read_afsr0_el3
func write_afsr0_el1 func write_afsr0_el1
msr afsr0_el1, x0 msr afsr0_el1, x0
isb
ret ret
func write_afsr0_el2 func write_afsr0_el2
msr afsr0_el2, x0 msr afsr0_el2, x0
isb
ret ret
func write_afsr0_el3 func write_afsr0_el3
msr afsr0_el3, x0 msr afsr0_el3, x0
isb
ret ret
...@@ -275,19 +269,16 @@ func read_far_el3 ...@@ -275,19 +269,16 @@ func read_far_el3
func write_far_el1 func write_far_el1
msr far_el1, x0 msr far_el1, x0
isb
ret ret
func write_far_el2 func write_far_el2
msr far_el2, x0 msr far_el2, x0
isb
ret ret
func write_far_el3 func write_far_el3
msr far_el3, x0 msr far_el3, x0
isb
ret ret
...@@ -312,19 +303,16 @@ func read_mair_el3 ...@@ -312,19 +303,16 @@ func read_mair_el3
func write_mair_el1 func write_mair_el1
msr mair_el1, x0 msr mair_el1, x0
isb
ret ret
func write_mair_el2 func write_mair_el2
msr mair_el2, x0 msr mair_el2, x0
isb
ret ret
func write_mair_el3 func write_mair_el3
msr mair_el3, x0 msr mair_el3, x0
isb
ret ret
...@@ -349,19 +337,16 @@ func read_amair_el3 ...@@ -349,19 +337,16 @@ func read_amair_el3
func write_amair_el1 func write_amair_el1
msr amair_el1, x0 msr amair_el1, x0
isb
ret ret
func write_amair_el2 func write_amair_el2
msr amair_el2, x0 msr amair_el2, x0
isb
ret ret
func write_amair_el3 func write_amair_el3
msr amair_el3, x0 msr amair_el3, x0
isb
ret ret
...@@ -405,19 +390,16 @@ func read_rmr_el3 ...@@ -405,19 +390,16 @@ func read_rmr_el3
func write_rmr_el1 func write_rmr_el1
msr rmr_el1, x0 msr rmr_el1, x0
isb
ret ret
func write_rmr_el2 func write_rmr_el2
msr rmr_el2, x0 msr rmr_el2, x0
isb
ret ret
func write_rmr_el3 func write_rmr_el3
msr rmr_el3, x0 msr rmr_el3, x0
isb
ret ret
...@@ -442,19 +424,16 @@ func read_afsr1_el3 ...@@ -442,19 +424,16 @@ func read_afsr1_el3
func write_afsr1_el1 func write_afsr1_el1
msr afsr1_el1, x0 msr afsr1_el1, x0
isb
ret ret
func write_afsr1_el2 func write_afsr1_el2
msr afsr1_el2, x0 msr afsr1_el2, x0
isb
ret ret
func write_afsr1_el3 func write_afsr1_el3
msr afsr1_el3, x0 msr afsr1_el3, x0
isb
ret ret
...@@ -479,22 +458,16 @@ func read_sctlr_el3 ...@@ -479,22 +458,16 @@ func read_sctlr_el3
func write_sctlr_el1 func write_sctlr_el1
msr sctlr_el1, x0 msr sctlr_el1, x0
dsb sy
isb
ret ret
func write_sctlr_el2 func write_sctlr_el2
msr sctlr_el2, x0 msr sctlr_el2, x0
dsb sy
isb
ret ret
func write_sctlr_el3 func write_sctlr_el3
msr sctlr_el3, x0 msr sctlr_el3, x0
dsb sy
isb
ret ret
...@@ -519,22 +492,16 @@ func read_actlr_el3 ...@@ -519,22 +492,16 @@ func read_actlr_el3
func write_actlr_el1 func write_actlr_el1
msr actlr_el1, x0 msr actlr_el1, x0
dsb sy
isb
ret ret
func write_actlr_el2 func write_actlr_el2
msr actlr_el2, x0 msr actlr_el2, x0
dsb sy
isb
ret ret
func write_actlr_el3 func write_actlr_el3
msr actlr_el3, x0 msr actlr_el3, x0
dsb sy
isb
ret ret
...@@ -559,22 +526,16 @@ func read_esr_el3 ...@@ -559,22 +526,16 @@ func read_esr_el3
func write_esr_el1 func write_esr_el1
msr esr_el1, x0 msr esr_el1, x0
dsb sy
isb
ret ret
func write_esr_el2 func write_esr_el2
msr esr_el2, x0 msr esr_el2, x0
dsb sy
isb
ret ret
func write_esr_el3 func write_esr_el3
msr esr_el3, x0 msr esr_el3, x0
dsb sy
isb
ret ret
...@@ -599,22 +560,16 @@ func read_tcr_el3 ...@@ -599,22 +560,16 @@ func read_tcr_el3
func write_tcr_el1 func write_tcr_el1
msr tcr_el1, x0 msr tcr_el1, x0
dsb sy
isb
ret ret
func write_tcr_el2 func write_tcr_el2
msr tcr_el2, x0 msr tcr_el2, x0
dsb sy
isb
ret ret
func write_tcr_el3 func write_tcr_el3
msr tcr_el3, x0 msr tcr_el3, x0
dsb sy
isb
ret ret
...@@ -643,15 +598,11 @@ func write_cptr_el1 ...@@ -643,15 +598,11 @@ func write_cptr_el1
func write_cptr_el2 func write_cptr_el2
msr cptr_el2, x0 msr cptr_el2, x0
dsb sy
isb
ret ret
func write_cptr_el3 func write_cptr_el3
msr cptr_el3, x0 msr cptr_el3, x0
dsb sy
isb
ret ret
...@@ -676,19 +627,16 @@ func read_ttbr0_el3 ...@@ -676,19 +627,16 @@ func read_ttbr0_el3
func write_ttbr0_el1 func write_ttbr0_el1
msr ttbr0_el1, x0 msr ttbr0_el1, x0
isb
ret ret
func write_ttbr0_el2 func write_ttbr0_el2
msr ttbr0_el2, x0 msr ttbr0_el2, x0
isb
ret ret
func write_ttbr0_el3 func write_ttbr0_el3
msr ttbr0_el3, x0 msr ttbr0_el3, x0
isb
ret ret
...@@ -711,7 +659,6 @@ func read_ttbr1_el3 ...@@ -711,7 +659,6 @@ func read_ttbr1_el3
func write_ttbr1_el1 func write_ttbr1_el1
msr ttbr1_el1, x0 msr ttbr1_el1, x0
isb
ret ret
...@@ -730,8 +677,6 @@ func read_hcr ...@@ -730,8 +677,6 @@ func read_hcr
func write_hcr func write_hcr
msr hcr_el2, x0 msr hcr_el2, x0
dsb sy
isb
ret ret
...@@ -762,8 +707,6 @@ func read_cpuectlr ...@@ -762,8 +707,6 @@ func read_cpuectlr
func write_cpuectlr func write_cpuectlr
msr CPUECTLR_EL1, x0 msr CPUECTLR_EL1, x0
dsb sy
isb
ret ret
...@@ -789,8 +732,6 @@ func write_cntfrq ...@@ -789,8 +732,6 @@ func write_cntfrq
func write_scr func write_scr
msr scr_el3, x0 msr scr_el3, x0
dsb sy
isb
ret ret
...@@ -818,6 +759,7 @@ func enable_vfp ...@@ -818,6 +759,7 @@ func enable_vfp
mov x1, #AARCH64_CPTR_TFP mov x1, #AARCH64_CPTR_TFP
bic x0, x0, x1 bic x0, x0, x1
msr cptr_el3, x0 msr cptr_el3, x0
isb
ret ret
......
lib/aarch64/tlb_helpers.S
View file @ b3e2268e
...@@ -41,47 +41,33 @@ ...@@ -41,47 +41,33 @@
func tlbialle1 func tlbialle1
tlbi alle1 tlbi alle1
dsb sy
isb
ret ret
func tlbialle1is func tlbialle1is
tlbi alle1is tlbi alle1is
dsb sy
isb
ret ret
func tlbialle2 func tlbialle2
tlbi alle2 tlbi alle2
dsb sy
isb
ret ret
func tlbialle2is func tlbialle2is
tlbi alle2is tlbi alle2is
dsb sy
isb
ret ret
func tlbialle3 func tlbialle3
tlbi alle3 tlbi alle3
dsb sy
isb
ret ret
func tlbialle3is func tlbialle3is
tlbi alle3is tlbi alle3is
dsb sy
isb
ret ret
func tlbivmalle1 func tlbivmalle1
tlbi vmalle1 tlbi vmalle1
dsb sy
isb
ret ret
plat/fvp/aarch64/plat_common.c
View file @ b3e2268e
...@@ -69,6 +69,8 @@ void enable_mmu() ...@@ -69,6 +69,8 @@ void enable_mmu()
ttbr = (unsigned long) l1_xlation_table; ttbr = (unsigned long) l1_xlation_table;
if (GET_EL(current_el) == MODE_EL3) { if (GET_EL(current_el) == MODE_EL3) {
assert((read_sctlr_el3() & SCTLR_M_BIT) == 0);
write_mair_el3(mair); write_mair_el3(mair);
tcr |= TCR_EL3_RES1; tcr |= TCR_EL3_RES1;
/* Invalidate EL3 TLBs */ /* Invalidate EL3 TLBs */
...@@ -77,11 +79,19 @@ void enable_mmu() ...@@ -77,11 +79,19 @@ void enable_mmu()
write_tcr_el3(tcr); write_tcr_el3(tcr);
write_ttbr0_el3(ttbr); write_ttbr0_el3(ttbr);
/* ensure all translation table writes have drained into memory,
* the TLB invalidation is complete, and translation register
* writes are committed before enabling the MMU
*/
dsb();
isb();
sctlr = read_sctlr_el3(); sctlr = read_sctlr_el3();
sctlr |= SCTLR_WXN_BIT | SCTLR_M_BIT | SCTLR_I_BIT; sctlr |= SCTLR_WXN_BIT | SCTLR_M_BIT | SCTLR_I_BIT;
sctlr |= SCTLR_A_BIT | SCTLR_C_BIT; sctlr |= SCTLR_A_BIT | SCTLR_C_BIT;
write_sctlr_el3(sctlr); write_sctlr_el3(sctlr);
} else { } else {
assert((read_sctlr_el1() & SCTLR_M_BIT) == 0);
write_mair_el1(mair); write_mair_el1(mair);
/* Invalidate EL1 TLBs */ /* Invalidate EL1 TLBs */
...@@ -90,11 +100,20 @@ void enable_mmu() ...@@ -90,11 +100,20 @@ void enable_mmu()
write_tcr_el1(tcr); write_tcr_el1(tcr);
write_ttbr0_el1(ttbr); write_ttbr0_el1(ttbr);
/* ensure all translation table writes have drained into memory,
* the TLB invalidation is complete, and translation register
* writes are committed before enabling the MMU
*/
dsb();
isb();
sctlr = read_sctlr_el1(); sctlr = read_sctlr_el1();
sctlr |= SCTLR_WXN_BIT | SCTLR_M_BIT | SCTLR_I_BIT; sctlr |= SCTLR_WXN_BIT | SCTLR_M_BIT | SCTLR_I_BIT;
sctlr |= SCTLR_A_BIT | SCTLR_C_BIT; sctlr |= SCTLR_A_BIT | SCTLR_C_BIT;
write_sctlr_el1(sctlr); write_sctlr_el1(sctlr);
} }
/* ensure the MMU enable takes effect immediately */
isb();
return; return;
} }
...@@ -113,6 +132,8 @@ void disable_mmu(void) ...@@ -113,6 +132,8 @@ void disable_mmu(void)
sctlr = sctlr & ~(SCTLR_M_BIT | SCTLR_C_BIT); sctlr = sctlr & ~(SCTLR_M_BIT | SCTLR_C_BIT);
write_sctlr_el1(sctlr); write_sctlr_el1(sctlr);
} }
/* ensure the MMU disable takes effect immediately */
isb();
/* Flush the caches */ /* Flush the caches */
dcsw_op_all(DCCISW); dcsw_op_all(DCCISW);
......
plat/fvp/plat_gic.c
View file @ b3e2268e
...@@ -86,6 +86,7 @@ void gicv3_cpuif_setup(void) ...@@ -86,6 +86,7 @@ void gicv3_cpuif_setup(void)
*/ */
scr_val = read_scr(); scr_val = read_scr();
write_scr(scr_val | SCR_NS_BIT); write_scr(scr_val | SCR_NS_BIT);
isb(); /* ensure NS=1 takes effect before accessing ICC_SRE_EL2 */
/* /*
* By default EL2 and NS-EL1 software should be able to enable GICv3 * By default EL2 and NS-EL1 software should be able to enable GICv3
...@@ -103,9 +104,11 @@ void gicv3_cpuif_setup(void) ...@@ -103,9 +104,11 @@ void gicv3_cpuif_setup(void)
write_icc_sre_el2(val | ICC_SRE_EN | ICC_SRE_SRE); write_icc_sre_el2(val | ICC_SRE_EN | ICC_SRE_SRE);
write_icc_pmr_el1(GIC_PRI_MASK); write_icc_pmr_el1(GIC_PRI_MASK);
isb(); /* commite ICC_* changes before setting NS=0 */
/* Restore SCR_EL3 */ /* Restore SCR_EL3 */
write_scr(scr_val); write_scr(scr_val);
isb(); /* ensure NS=0 takes effect immediately */
} }
/******************************************************************************* /*******************************************************************************
......
plat/fvp/plat_pm.c
View file @ b3e2268e
...@@ -54,7 +54,11 @@ int fvp_affinst_standby(unsigned int power_state) ...@@ -54,7 +54,11 @@ int fvp_affinst_standby(unsigned int power_state)
if (target_afflvl != MPIDR_AFFLVL0) if (target_afflvl != MPIDR_AFFLVL0)
return PSCI_E_INVALID_PARAMS; return PSCI_E_INVALID_PARAMS;
/* Enter standby state */ /*
* Enter standby state
* dsb is good practice before using wfi to enter low power states
*/
dsb();
wfi(); wfi();
return PSCI_E_SUCCESS; return PSCI_E_SUCCESS;
......
services/std_svc/psci/psci_afflvl_off.c
View file @ b3e2268e
...@@ -82,6 +82,7 @@ static int psci_afflvl0_off(unsigned long mpidr, aff_map_node_t *cpu_node) ...@@ -82,6 +82,7 @@ static int psci_afflvl0_off(unsigned long mpidr, aff_map_node_t *cpu_node)
sctlr = read_sctlr_el3(); sctlr = read_sctlr_el3();
sctlr &= ~SCTLR_C_BIT; sctlr &= ~SCTLR_C_BIT;
write_sctlr_el3(sctlr); write_sctlr_el3(sctlr);
isb(); /* ensure MMU disable takes immediate effect */
/* /*
* CAUTION: This flush to the level of unification makes an assumption * CAUTION: This flush to the level of unification makes an assumption
......
services/std_svc/psci/psci_afflvl_suspend.c
View file @ b3e2268e
...@@ -198,6 +198,7 @@ static int psci_afflvl0_suspend(unsigned long mpidr, ...@@ -198,6 +198,7 @@ static int psci_afflvl0_suspend(unsigned long mpidr,
sctlr = read_sctlr_el3(); sctlr = read_sctlr_el3();
sctlr &= ~SCTLR_C_BIT; sctlr &= ~SCTLR_C_BIT;
write_sctlr_el3(sctlr); write_sctlr_el3(sctlr);
isb(); /* ensure MMU disable takes immediate effect */
/* /*
* CAUTION: This flush to the level of unification makes an assumption * CAUTION: This flush to the level of unification makes an assumption
......
services/std_svc/psci/psci_entry.S
View file @ b3e2268e
...@@ -75,7 +75,6 @@ psci_aff_common_finish_entry: ...@@ -75,7 +75,6 @@ psci_aff_common_finish_entry:
* --------------------------------------------- * ---------------------------------------------
*/ */
msr spsel, #0 msr spsel, #0
isb
bl read_mpidr bl read_mpidr
mov x19, x0 mov x19, x0
...@@ -158,7 +157,7 @@ func __psci_cpu_suspend ...@@ -158,7 +157,7 @@ func __psci_cpu_suspend
ret ret
func final_wfi func final_wfi
dsb sy dsb sy // ensure write buffer empty
wfi wfi
wfi_spill: wfi_spill:
b wfi_spill b wfi_spill
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment