Apply workaround for errata 813419 of Cortex-A57

TLBI instructions for EL3 won't have the desired effect under specific
circumstances in Cortex-A57 r0p0. The workaround is to execute DSB and
TLBI twice each time.

Even though this errata is only needed in r0p0, the current errata
framework is not prepared to apply run-time workarounds. The current one
is always applied if compiled in, regardless of the CPU or its revision.

This errata has been enabled for Juno.

The `DSB` instruction used when initializing the translation tables has
been changed to `DSB ISH` as an optimization and to be consistent with
the barriers used for the workaround.

Change-Id: Ifc1d70b79cb5e0d87e90d88d376a59385667d338
Signed-off-by: Antonio Nino Diaz <antonio.ninodiaz@arm.com>

docs/cpu-specific-build-macros.md

@@ -65,6 +65,9 @@ For Cortex-A57, following errata build flags are defined :
 *   `ERRATA_A57_806969`: This applies errata 806969 workaround to Cortex-A57
      CPU. This needs to be enabled only for revision r0p0 of the CPU.
 
+*   `ERRATA_A57_813419`: This applies errata 813419 workaround to Cortex-A57
+     CPU. This needs to be enabled only for revision r0p0 of the CPU.
+
 *   `ERRATA_A57_813420`: This applies errata 813420 workaround to Cortex-A57
      CPU. This needs to be enabled only for revision r0p0 of the CPU.
 

include/lib/aarch64/arch_helpers.h

@@ -116,20 +116,57 @@ static inline void _op ## _type(uint64_t v)		\
 /*******************************************************************************
  * TLB maintenance accessor prototypes
  ******************************************************************************/
+
+#if ERRATA_A57_813419
+/*
+ * Define function for TLBI instruction with type specifier that implements
+ * the workaround for errata 813419 of Cortex-A57.
+ */
+#define DEFINE_TLBIOP_ERRATA_A57_813419_TYPE_FUNC(_type)\
+static inline void tlbi ## _type(void)			\
+{							\
+	__asm__("tlbi " #_type "\n"			\
+		"dsb ish\n"				\
+		"tlbi " #_type);			\
+}
+
+/*
+ * Define function for TLBI instruction with register parameter that implements
+ * the workaround for errata 813419 of Cortex-A57.
+ */
+#define DEFINE_TLBIOP_ERRATA_A57_813419_TYPE_PARAM_FUNC(_type)	\
+static inline void tlbi ## _type(uint64_t v)			\
+{								\
+	__asm__("tlbi " #_type ", %0\n"				\
+		"dsb ish\n"					\
+		"tlbi " #_type ", %0" : : "r" (v));		\
+}
+#endif /* ERRATA_A57_813419 */
+
 DEFINE_SYSOP_TYPE_FUNC(tlbi, alle1)
 DEFINE_SYSOP_TYPE_FUNC(tlbi, alle1is)
 DEFINE_SYSOP_TYPE_FUNC(tlbi, alle2)
 DEFINE_SYSOP_TYPE_FUNC(tlbi, alle2is)
+#if ERRATA_A57_813419
+DEFINE_TLBIOP_ERRATA_A57_813419_TYPE_FUNC(alle3)
+DEFINE_TLBIOP_ERRATA_A57_813419_TYPE_FUNC(alle3is)
+#else
 DEFINE_SYSOP_TYPE_FUNC(tlbi, alle3)
 DEFINE_SYSOP_TYPE_FUNC(tlbi, alle3is)
+#endif
 DEFINE_SYSOP_TYPE_FUNC(tlbi, vmalle1)
 
 DEFINE_SYSOP_TYPE_PARAM_FUNC(tlbi, vaae1is)
 DEFINE_SYSOP_TYPE_PARAM_FUNC(tlbi, vaale1is)
 DEFINE_SYSOP_TYPE_PARAM_FUNC(tlbi, vae2is)
 DEFINE_SYSOP_TYPE_PARAM_FUNC(tlbi, vale2is)
+#if ERRATA_A57_813419
+DEFINE_TLBIOP_ERRATA_A57_813419_TYPE_PARAM_FUNC(vae3is)
+DEFINE_TLBIOP_ERRATA_A57_813419_TYPE_PARAM_FUNC(vale3is)
+#else
 DEFINE_SYSOP_TYPE_PARAM_FUNC(tlbi, vae3is)
 DEFINE_SYSOP_TYPE_PARAM_FUNC(tlbi, vale3is)
+#endif
 
 /*******************************************************************************
  * Cache maintenance accessor prototypes

lib/cpus/aarch64/cortex_a57.S

@@ -114,6 +114,21 @@ func check_errata_806969
 	b	cpu_rev_var_ls
 endfunc check_errata_806969
 
+	/* ---------------------------------------------------
+	 * Errata Workaround for Cortex A57 Errata #813419.
+	 * This applies only to revision r0p0 of Cortex A57.
+	 * ---------------------------------------------------
+	 */
+func check_errata_813419
+	/*
+	 * Even though this is only needed for revision r0p0, it
+	 * is always applied due to limitations of the current
+	 * errata framework.
+	 */
+	mov	x0, #ERRATA_APPLIES
+	ret
+endfunc check_errata_813419
+
 	/* ---------------------------------------------------
 	 * Errata Workaround for Cortex A57 Errata #813420.
 	 * This applies only to revision r0p0 of Cortex A57.
@@ -482,6 +497,7 @@ func cortex_a57_errata_report
 	 * checking functions of each errata.
 	 */
 	report_errata ERRATA_A57_806969, cortex_a57, 806969
+	report_errata ERRATA_A57_813419, cortex_a57, 813419
 	report_errata ERRATA_A57_813420, cortex_a57, 813420
 	report_errata A57_DISABLE_NON_TEMPORAL_HINT, cortex_a57, \
 		disable_ldnp_overread

lib/cpus/cpu-ops.mk

@@ -70,6 +70,10 @@ ERRATA_A53_836870	?=0
 # only to revision r0p0 of the Cortex A57 cpu.
 ERRATA_A57_806969	?=0
 
+# Flag to apply erratum 813419 workaround during reset. This erratum applies
+# only to revision r0p0 of the Cortex A57 cpu.
+ERRATA_A57_813419	?=0
+
 # Flag to apply erratum 813420  workaround during reset. This erratum applies
 # only to revision r0p0 of the Cortex A57 cpu.
 ERRATA_A57_813420	?=0
@@ -106,6 +110,10 @@ $(eval $(call add_define,ERRATA_A53_836870))
 $(eval $(call assert_boolean,ERRATA_A57_806969))
 $(eval $(call add_define,ERRATA_A57_806969))
 
+# Process ERRATA_A57_813419 flag
+$(eval $(call assert_boolean,ERRATA_A57_813419))
+$(eval $(call add_define,ERRATA_A57_813419))
+
 # Process ERRATA_A57_813420 flag
 $(eval $(call assert_boolean,ERRATA_A57_813420))
 $(eval $(call add_define,ERRATA_A57_813420))

lib/xlat_tables/aarch64/xlat_tables.c

@@ -208,7 +208,7 @@ void init_xlat_tables(void)
 		/* into memory, the TLB invalidation is complete, */	\
 		/* and translation register writes are committed */	\
 		/* before enabling the MMU */				\
-		dsb();							\
+		dsbish();						\
 		isb();							\
 									\
 		sctlr = read_sctlr_el##_el();				\

lib/xlat_tables_v2/aarch64/xlat_tables_arch.c

@@ -217,7 +217,7 @@ void init_xlat_tables_arch(unsigned long long max_pa)
 		/* into memory, the TLB invalidation is complete, */	\
 		/* and translation register writes are committed */	\
 		/* before enabling the MMU */				\
-		dsb();							\
+		dsbish();						\
 		isb();							\
 									\
 		sctlr = read_sctlr_el##_el();				\

plat/arm/board/juno/platform.mk

@@ -69,6 +69,7 @@ BL31_SOURCES		+=	lib/cpus/aarch64/cortex_a53.S		\
 
 # Enable workarounds for selected Cortex-A57 erratas.
 ERRATA_A57_806969		:=	0
+ERRATA_A57_813419		:=	1
 ERRATA_A57_813420		:=	1
 
 # Enable option to skip L1 data cache flush during the Cortex-A57 cluster