Merge "Implement workaround for AT speculative behaviour" into integration

Makefile

@@ -891,6 +891,7 @@ $(eval $(call assert_boolean,BL2_INV_DCACHE))
 $(eval $(call assert_boolean,USE_SPINLOCK_CAS))
 $(eval $(call assert_boolean,ENCRYPT_BL31))
 $(eval $(call assert_boolean,ENCRYPT_BL32))
+$(eval $(call assert_boolean,ERRATA_SPECULATIVE_AT))
 
 $(eval $(call assert_numeric,ARM_ARCH_MAJOR))
 $(eval $(call assert_numeric,ARM_ARCH_MINOR))
@@ -967,6 +968,7 @@ $(eval $(call add_define,BL2_AT_EL3))
 $(eval $(call add_define,BL2_IN_XIP_MEM))
 $(eval $(call add_define,BL2_INV_DCACHE))
 $(eval $(call add_define,USE_SPINLOCK_CAS))
+$(eval $(call add_define,ERRATA_SPECULATIVE_AT))
 
 ifeq (${SANITIZE_UB},trap)
         $(eval $(call add_define,MONITOR_TRAPS))

docs/getting_started/build-options.rst

@@ -673,6 +673,29 @@ Common build options
    default value of this flag is ``no``. Note this option must be enabled only
    for ARM architecture greater than Armv8.5-A.
 
+-  ``ERRATA_SPECULATIVE_AT``: This flag enables/disables page table walk during
+   context restore as speculative AT instructions using an out-of-context
+   translation regime could cause subsequent requests to generate an incorrect
+   translation.
+   System registers are not updated during context save, hence this workaround
+   need not be applied in the context save path.
+
+   This boolean option enables errata for all below CPUs.
+
+   +---------+--------------+
+   | Errata  |      CPU     |
+   +=========+==============+
+   | 1165522 |  Cortex-A76  |
+   +---------+--------------+
+   | 1319367 |  Cortex-A72  |
+   +---------+--------------+
+   | 1319537 |  Cortex-A57  |
+   +---------+--------------+
+   | 1530923 |  Cortex-A55  |
+   +---------+--------------+
+   | 1530924 |  Cortex-A53  |
+   +---------+--------------+
+
 GICv3 driver options
 --------------------
 

include/arch/aarch64/arch.h

@@ -381,6 +381,7 @@
 /* HCR definitions */
 #define HCR_API_BIT		(ULL(1) << 41)
 #define HCR_APK_BIT		(ULL(1) << 40)
+#define HCR_E2H_BIT		(ULL(1) << 34)
 #define HCR_TGE_BIT		(ULL(1) << 27)
 #define HCR_RW_SHIFT		U(31)
 #define HCR_RW_BIT		(ULL(1) << HCR_RW_SHIFT)

lib/el3_runtime/aarch64/context.S

@@ -234,6 +234,21 @@ endfunc el2_sysregs_context_save
  */
 func el2_sysregs_context_restore
 
+#if ERRATA_SPECULATIVE_AT
+/* Clear EPD0 and EPD1 bit and M bit to disable PTW */
+	mrs	x9, hcr_el2
+	tst	x9, #HCR_E2H_BIT
+	bne	1f
+	mrs	x9, tcr_el2
+	orr	x9, x9, #TCR_EPD0_BIT
+	orr	x9, x9, #TCR_EPD1_BIT
+	msr	tcr_el2, x9
+1:	mrs	x9, sctlr_el2
+	bic	x9, x9, #SCTLR_M_BIT
+	msr	sctlr_el2, x9
+	isb
+#endif
+
 	ldp	x9, x10, [x0, #CTX_ACTLR_EL2]
 	msr	actlr_el2, x9
 	msr	afsr0_el2, x10
@@ -282,17 +297,15 @@ func el2_sysregs_context_restore
 	msr	mair_el2, x15
 	msr	mdcr_el2, x16
 
-	ldp	x17, x9, [x0, #CTX_PMSCR_EL2]
+	ldr	x17, [x0, #CTX_PMSCR_EL2]
 	msr	PMSCR_EL2, x17
-	msr	sctlr_el2, x9
 
 	ldp	x10, x11, [x0, #CTX_SPSR_EL2]
 	msr	spsr_el2, x10
 	msr	sp_el2, x11
 
-	ldp	x12, x13, [x0, #CTX_TCR_EL2]
-	msr	tcr_el2, x12
-	msr	tpidr_el2, x13
+	ldr	x12, [x0, #CTX_TPIDR_EL2]
+	msr	tpidr_el2, x12
 
 	ldp	x14, x15, [x0, #CTX_TTBR0_EL2]
 	msr	ttbr0_el2, x14
@@ -404,6 +417,19 @@ func el2_sysregs_context_restore
 	msr	scxtnum_el2, x9
 #endif
 
+#if ERRATA_SPECULATIVE_AT
+/*
+ * Make sure all registers are stored successfully except
+ * SCTLR_EL2 and TCR_EL2
+ */
+	isb
+#endif
+
+	ldr	x9, [x0, #CTX_SCTLR_EL2]
+	msr	sctlr_el2, x9
+	ldr	x9, [x0, #CTX_TCR_EL2]
+	msr	tcr_el2, x9
+
 	ret
 endfunc el2_sysregs_context_restore
 
@@ -515,12 +541,22 @@ endfunc el1_sysregs_context_save
  */
 func el1_sysregs_context_restore
 
+#if ERRATA_SPECULATIVE_AT
+	mrs	x9, tcr_el1
+	orr	x9, x9, #TCR_EPD0_BIT
+	orr	x9, x9, #TCR_EPD1_BIT
+	msr	tcr_el1, x9
+	mrs	x9, sctlr_el1
+	bic	x9, x9, #SCTLR_M_BIT
+	msr	sctlr_el1, x9
+	isb
+#endif
+
 	ldp	x9, x10, [x0, #CTX_SPSR_EL1]
 	msr	spsr_el1, x9
 	msr	elr_el1, x10
 
-	ldp	x15, x16, [x0, #CTX_SCTLR_EL1]
-	msr	sctlr_el1, x15
+	ldr	x16, [x0, #CTX_ACTLR_EL1]
 	msr	actlr_el1, x16
 
 	ldp	x17, x9, [x0, #CTX_CPACR_EL1]
@@ -539,9 +575,8 @@ func el1_sysregs_context_restore
 	msr	mair_el1, x14
 	msr	amair_el1, x15
 
-	ldp	x16, x17, [x0, #CTX_TCR_EL1]
-	msr	tcr_el1, x16
-	msr	tpidr_el1, x17
+	ldr	x16,[x0, #CTX_TPIDR_EL1]
+	msr	tpidr_el1, x16
 
 	ldp	x9, x10, [x0, #CTX_TPIDR_EL0]
 	msr	tpidr_el0, x9
@@ -597,6 +632,19 @@ func el1_sysregs_context_restore
 	msr	GCR_EL1, x14
 #endif
 
+#if ERRATA_SPECULATIVE_AT
+/*
+ * Make sure all registers are stored successfully except
+ * SCTLR_EL1 and TCR_EL1
+ */
+	isb
+#endif
+
+	ldr	x9, [x0, #CTX_SCTLR_EL1]
+	msr	sctlr_el1, x9
+	ldr	x9, [x0, #CTX_TCR_EL1]
+	msr	tcr_el1, x9
+
 	/* No explict ISB required here as ERET covers it */
 	ret
 endfunc el1_sysregs_context_restore

make_helpers/defaults.mk

@@ -293,3 +293,6 @@ CTX_INCLUDE_EL2_REGS		:= 0
 # than Armv8.5-A
 # By default it is set to "no"
 SUPPORT_STACK_MEMTAG		:= no
+
+# Select workaround for AT speculative behaviour.
+ERRATA_SPECULATIVE_AT           := 0