Merge changes from topic "at_errata_fix" into integration

* changes:
  doc: Update description for AT speculative workaround
  lib/cpus: Report AT speculative erratum workaround
  Add wrapper for AT instruction

common/backtrace/backtrace.c

@@ -70,7 +70,7 @@ static bool is_address_readable(uintptr_t addr)
 	} else if (el == 2U) {
 		ats1e2r(addr);
 	} else {
-		ats1e1r(addr);
+		AT(ats1e1r, addr);
 	}
 
 	isb();

docs/design/cpu-specific-build-macros.rst

@@ -127,6 +127,9 @@ For Cortex-A53, the following errata build flags are defined :
    Earlier revisions of the CPU have other errata which require the same
    workaround in software, so they should be covered anyway.
 
+-  ``ERRATA_A53_1530924``: This applies errata 1530924 workaround to all
+   revisions of Cortex-A53 CPU.
+
 For Cortex-A55, the following errata build flags are defined :
 
 -  ``ERRATA_A55_768277``: This applies errata 768277 workaround to Cortex-A55
@@ -147,6 +150,9 @@ For Cortex-A55, the following errata build flags are defined :
 -  ``ERRATA_A55_1221012``: This applies errata 1221012 workaround to Cortex-A55
    CPU. This needs to be enabled only for revision <= r1p0 of the CPU.
 
+-  ``ERRATA_A55_1530923``: This applies errata 1530923 workaround to all
+   revisions of Cortex-A55 CPU.
+
 For Cortex-A57, the following errata build flags are defined :
 
 -  ``ERRATA_A57_806969``: This applies errata 806969 workaround to Cortex-A57
@@ -182,12 +188,17 @@ For Cortex-A57, the following errata build flags are defined :
 -  ``ERRATA_A57_859972``: This applies errata 859972 workaround to Cortex-A57
    CPU. This needs to be enabled only for revision <= r1p3 of the CPU.
 
+-  ``ERRATA_A57_1319537``: This applies errata 1319537 workaround to all
+   revisions of Cortex-A57 CPU.
 
 For Cortex-A72, the following errata build flags are defined :
 
 -  ``ERRATA_A72_859971``: This applies errata 859971 workaround to Cortex-A72
    CPU. This needs to be enabled only for revision <= r0p3 of the CPU.
 
+-  ``ERRATA_A72_1319367``: This applies errata 1319367 workaround to all
+   revisions of Cortex-A72 CPU.
+
 For Cortex-A73, the following errata build flags are defined :
 
 -  ``ERRATA_A73_852427``: This applies errata 852427 workaround to Cortex-A73
@@ -233,6 +244,11 @@ For Cortex-A76, the following errata build flags are defined :
 -  ``ERRATA_A76_1800710``: This applies errata 1800710 workaround to Cortex-A76
    CPU. This needs to be enabled only for revision <= r4p0 of the CPU.
 
+-  ``ERRATA_A76_1165522``: This applies errata 1165522 workaround to all
+   revisions of Cortex-A76 CPU. This errata is fixed in r3p0 but due to
+   limitation of errata framework this errata is applied to all revisions
+   of Cortex-A76 CPU.
+
 For Cortex-A77, the following errata build flags are defined :
 
 -  ``ERRATA_A77_1800714``: This applies errata 1800714 workaround to Cortex-A77

docs/getting_started/build-options.rst

@@ -694,28 +694,36 @@ Common build options
    default value of this flag is ``no``. Note this option must be enabled only
    for ARM architecture greater than Armv8.5-A.
 
--  ``ERRATA_SPECULATIVE_AT``: This flag enables/disables page table walk during
-   context restore as speculative AT instructions using an out-of-context
-   translation regime could cause subsequent requests to generate an incorrect
-   translation.
-   System registers are not updated during context save, hence this workaround
-   need not be applied in the context save path.
+-  ``ERRATA_SPECULATIVE_AT``: This flag determines whether to enable ``AT``
+   speculative errata workaround or not. It accepts 2 values: ``1`` and ``0``.
+   The default value of this flag is ``0``.
+
+   ``AT`` speculative errata workaround disables stage1 page table walk for
+   lower ELs (EL1 and EL0) in EL3 so that ``AT`` speculative fetch at any point
+   produces either the correct result or failure without TLB allocation.
 
    This boolean option enables errata for all below CPUs.
 
-   +---------+--------------+
-   | Errata  |      CPU     |
-   +=========+==============+
-   | 1165522 |  Cortex-A76  |
-   +---------+--------------+
-   | 1319367 |  Cortex-A72  |
-   +---------+--------------+
-   | 1319537 |  Cortex-A57  |
-   +---------+--------------+
-   | 1530923 |  Cortex-A55  |
-   +---------+--------------+
-   | 1530924 |  Cortex-A53  |
-   +---------+--------------+
+   +---------+--------------+-------------------------+
+   | Errata  |      CPU     |     Workaround Define   |
+   +=========+==============+=========================+
+   | 1165522 |  Cortex-A76  |  ``ERRATA_A76_1165522`` |
+   +---------+--------------+-------------------------+
+   | 1319367 |  Cortex-A72  |  ``ERRATA_A72_1319367`` |
+   +---------+--------------+-------------------------+
+   | 1319537 |  Cortex-A57  |  ``ERRATA_A57_1319537`` |
+   +---------+--------------+-------------------------+
+   | 1530923 |  Cortex-A55  |  ``ERRATA_A55_1530923`` |
+   +---------+--------------+-------------------------+
+   | 1530924 |  Cortex-A53  |  ``ERRATA_A53_1530924`` |
+   +---------+--------------+-------------------------+
+
+   .. note::
+      This option is enabled by build only if platform sets any of above defines
+      mentioned in ’Workaround Define' column in the table.
+      If this option is enabled for the EL3 software then EL2 software also must
+      implement this workaround due to the behaviour of the errata mentioned
+      in new SDEN document which will get published soon.
 
 - ``RAS_TRAP_LOWER_EL_ERR_ACCESS``: This flag enables/disables the SCR_EL3.TERR
   bit, to trap access to the RAS ERR and RAS ERX registers from lower ELs.

docs/glossary.rst

@@ -18,6 +18,9 @@ You can find additional definitions in the `Arm Glossary`_.
    API
       Application Programming Interface
 
+   AT
+      Address Translation
+
    BTI
       Branch Target Identification. An Armv8.5 extension providing additional
       control flow integrity around indirect branches and their targets.

include/arch/aarch64/arch_helpers.h

@@ -590,4 +590,24 @@ static inline uint64_t el_implemented(unsigned int el)
 #define read_clusterpwrdn()	read_clusterpwrdn_el1()
 #define write_clusterpwrdn(_v)	write_clusterpwrdn_el1(_v)
 
+#if ERRATA_SPECULATIVE_AT
+/*
+ * Assuming SCTLR.M bit is already enabled
+ * 1. Enable page table walk by clearing TCR_EL1.EPDx bits
+ * 2. Execute AT instruction for lower EL1/0
+ * 3. Disable page table walk by setting TCR_EL1.EPDx bits
+ */
+#define AT(_at_inst, _va)	\
+{	\
+	assert((read_sctlr_el1() & SCTLR_M_BIT) != 0ULL);	\
+	write_tcr_el1(read_tcr_el1() & ~(TCR_EPD0_BIT | TCR_EPD1_BIT));	\
+	isb();	\
+	_at_inst(_va);	\
+	write_tcr_el1(read_tcr_el1() | (TCR_EPD0_BIT | TCR_EPD1_BIT));	\
+	isb();	\
+}
+#else
+#define AT(_at_inst, _va)	_at_inst(_va);
+#endif
+
 #endif /* ARCH_HELPERS_H */

lib/cpus/aarch64/cortex_a53.S

 /*
- * Copyright (c) 2014-2019, ARM Limited and Contributors. All rights reserved.
+ * Copyright (c) 2014-2020, ARM Limited and Contributors. All rights reserved.
  *
  * SPDX-License-Identifier: BSD-3-Clause
  */
@@ -239,6 +239,20 @@ exit_check_errata_843419:
 	ret
 endfunc check_errata_843419
 
+	/* --------------------------------------------------
+	 * Errata workaround for Cortex A53 Errata #1530924.
+	 * This applies to all revisions of Cortex A53.
+	 * --------------------------------------------------
+	 */
+func check_errata_1530924
+#if ERRATA_A53_1530924
+	mov	x0, #ERRATA_APPLIES
+#else
+	mov	x0, #ERRATA_MISSING
+#endif
+	ret
+endfunc check_errata_1530924
+
 	/* -------------------------------------------------
 	 * The CPU Ops reset function for Cortex-A53.
 	 * Shall clobber: x0-x19
@@ -359,6 +373,7 @@ func cortex_a53_errata_report
 	report_errata ERRATA_A53_836870, cortex_a53, disable_non_temporal_hint
 	report_errata ERRATA_A53_843419, cortex_a53, 843419
 	report_errata ERRATA_A53_855873, cortex_a53, 855873
+	report_errata ERRATA_A53_1530924, cortex_a53, 1530924
 
 	ldp	x8, x30, [sp], #16
 	ret

lib/cpus/aarch64/cortex_a55.S

 /*
- * Copyright (c) 2017-2019, ARM Limited and Contributors. All rights reserved.
+ * Copyright (c) 2017-2020, ARM Limited and Contributors. All rights reserved.
  *
  * SPDX-License-Identifier: BSD-3-Clause
  */
@@ -222,6 +222,20 @@ func check_errata_1221012
 	b	cpu_rev_var_ls
 endfunc check_errata_1221012
 
+	/* --------------------------------------------------
+	 * Errata workaround for Cortex A55 Errata #1530923.
+	 * This applies to all revisions of Cortex A55.
+	 * --------------------------------------------------
+	 */
+func check_errata_1530923
+#if ERRATA_A55_1530923
+	mov	x0, #ERRATA_APPLIES
+#else
+	mov	x0, #ERRATA_MISSING
+#endif
+	ret
+endfunc check_errata_1530923
+
 func cortex_a55_reset_func
 	mov	x19, x30
 
@@ -306,6 +320,7 @@ func cortex_a55_errata_report
 	report_errata ERRATA_A55_846532, cortex_a55, 846532
 	report_errata ERRATA_A55_903758, cortex_a55, 903758
 	report_errata ERRATA_A55_1221012, cortex_a55, 1221012
+	report_errata ERRATA_A55_1530923, cortex_a55, 1530923
 
 	ldp	x8, x30, [sp], #16
 	ret

lib/cpus/aarch64/cortex_a57.S

 /*
- * Copyright (c) 2014-2019, ARM Limited and Contributors. All rights reserved.
+ * Copyright (c) 2014-2020, ARM Limited and Contributors. All rights reserved.
  * Copyright (c) 2020, NVIDIA Corporation. All rights reserved.
  *
  * SPDX-License-Identifier: BSD-3-Clause
@@ -396,6 +396,20 @@ func check_errata_cve_2018_3639
 	ret
 endfunc check_errata_cve_2018_3639
 
+	/* --------------------------------------------------
+	 * Errata workaround for Cortex A57 Errata #1319537.
+	 * This applies to all revisions of Cortex A57.
+	 * --------------------------------------------------
+	 */
+func check_errata_1319537
+#if ERRATA_A57_1319537
+	mov	x0, #ERRATA_APPLIES
+#else
+	mov	x0, #ERRATA_MISSING
+#endif
+	ret
+endfunc check_errata_1319537
+
 	/* -------------------------------------------------
 	 * The CPU Ops reset function for Cortex-A57.
 	 * Shall clobber: x0-x19
@@ -613,6 +627,7 @@ func cortex_a57_errata_report
 	report_errata ERRATA_A57_829520, cortex_a57, 829520
 	report_errata ERRATA_A57_833471, cortex_a57, 833471
 	report_errata ERRATA_A57_859972, cortex_a57, 859972
+	report_errata ERRATA_A57_1319537, cortex_a57, 1319537
 	report_errata WORKAROUND_CVE_2017_5715, cortex_a57, cve_2017_5715
 	report_errata WORKAROUND_CVE_2018_3639, cortex_a57, cve_2018_3639
 

lib/cpus/aarch64/cortex_a72.S

 /*
- * Copyright (c) 2015-2018, ARM Limited and Contributors. All rights reserved.
+ * Copyright (c) 2015-2020, ARM Limited and Contributors. All rights reserved.
  *
  * SPDX-License-Identifier: BSD-3-Clause
  */
@@ -119,6 +119,20 @@ func check_errata_cve_2018_3639
 	ret
 endfunc check_errata_cve_2018_3639
 
+	/* --------------------------------------------------
+	 * Errata workaround for Cortex A72 Errata #1319367.
+	 * This applies to all revisions of Cortex A72.
+	 * --------------------------------------------------
+	 */
+func check_errata_1319367
+#if ERRATA_A72_1319367
+	mov	x0, #ERRATA_APPLIES
+#else
+	mov	x0, #ERRATA_MISSING
+#endif
+	ret
+endfunc check_errata_1319367
+
 	/* -------------------------------------------------
 	 * The CPU Ops reset function for Cortex-A72.
 	 * -------------------------------------------------
@@ -282,6 +296,7 @@ func cortex_a72_errata_report
 	 * checking functions of each errata.
 	 */
 	report_errata ERRATA_A72_859971, cortex_a72, 859971
+	report_errata ERRATA_A72_1319367, cortex_a72, 1319367
 	report_errata WORKAROUND_CVE_2017_5715, cortex_a72, cve_2017_5715
 	report_errata WORKAROUND_CVE_2018_3639, cortex_a72, cve_2018_3639
 

lib/cpus/aarch64/cortex_a76.S

@@ -465,6 +465,23 @@ func cortex_a76_disable_wa_cve_2018_3639
 	ret
 endfunc cortex_a76_disable_wa_cve_2018_3639
 
+	/* --------------------------------------------------------------
+	 * Errata Workaround for Cortex A76 Errata #1165522.
+	 * This applies only to revisions <= r3p0 of Cortex A76.
+	 * Due to the nature of the errata it is applied unconditionally
+	 * when built in, report it as applicable in this case
+	 * --------------------------------------------------------------
+	 */
+func check_errata_1165522
+#if ERRATA_A76_1165522
+	mov	x0, #ERRATA_APPLIES
+	ret
+#else
+	mov	x1, #0x30
+	b	cpu_rev_var_ls
+#endif
+endfunc check_errata_1165522
+
 	/* -------------------------------------------------
 	 * The CPU Ops reset function for Cortex-A76.
 	 * Shall clobber: x0-x19
@@ -597,6 +614,7 @@ func cortex_a76_errata_report
 	report_errata ERRATA_A76_1286807, cortex_a76, 1286807
 	report_errata ERRATA_A76_1791580, cortex_a76, 1791580
 	report_errata ERRATA_A76_1800710, cortex_a76, 1800710
+	report_errata ERRATA_A76_1165522, cortex_a76, 1165522
 	report_errata WORKAROUND_CVE_2018_3639, cortex_a76, cve_2018_3639
 	report_errata ERRATA_DSU_798953, cortex_a76, dsu_798953
 	report_errata ERRATA_DSU_936184, cortex_a76, dsu_936184

lib/cpus/cpu-ops.mk

@@ -130,6 +130,10 @@ ERRATA_A53_843419	?=0
 # of by the rich OS.
 ERRATA_A53_855873	?=0
 
+# Flag to apply erratum 1530924 workaround during reset. This erratum applies
+# to all revisions of Cortex A53 cpu.
+ERRATA_A53_1530924	?=0
+
 # Flag to apply erratum 768277 workaround during reset. This erratum applies
 # only to revision r0p0 of the Cortex A55 cpu.
 ERRATA_A55_768277	?=0
@@ -154,6 +158,10 @@ ERRATA_A55_903758	?=0
 # only to revision <= r1p0 of the Cortex A55 cpu.
 ERRATA_A55_1221012	?=0
 
+# Flag to apply erratum 1530923 workaround during reset. This erratum applies
+# to all revisions of Cortex A55 cpu.
+ERRATA_A55_1530923	?=0
+
 # Flag to apply erratum 806969 workaround during reset. This erratum applies
 # only to revision r0p0 of the Cortex A57 cpu.
 ERRATA_A57_806969	?=0
@@ -198,10 +206,18 @@ ERRATA_A57_833471	?=0
 # only to revision <= r1p3 of the Cortex A57 cpu.
 ERRATA_A57_859972	?=0
 
+# Flag to apply erratum 1319537 workaround during reset. This erratum applies
+# to all revisions of Cortex A57 cpu.
+ERRATA_A57_1319537	?=0
+
 # Flag to apply erratum 855971 workaround during reset. This erratum applies
 # only to revision <= r0p3 of the Cortex A72 cpu.
 ERRATA_A72_859971	?=0
 
+# Flag to apply erratum 1319367 workaround during reset. This erratum applies
+# to all revisions of Cortex A72 cpu.
+ERRATA_A72_1319367	?=0
+
 # Flag to apply erratum 852427 workaround during reset. This erratum applies
 # only to revision r0p0 of the Cortex A73 cpu.
 ERRATA_A73_852427	?=0
@@ -258,6 +274,10 @@ ERRATA_A76_1791580	?=0
 # only to revision <= r4p0 of the Cortex A76 cpu.
 ERRATA_A76_1800710	?=0
 
+# Flag to apply erratum 1165522 workaround during reset. This erratum applies
+# to all revisions of Cortex A76 cpu.
+ERRATA_A76_1165522	?=0
+
 # Flag to apply erratum 1800714 workaround during reset. This erratum applies
 # only to revision <= r1p1 of the Cortex A77 cpu.
 ERRATA_A77_1800714	?=0
@@ -379,6 +399,10 @@ $(eval $(call add_define,ERRATA_A53_843419))
 $(eval $(call assert_boolean,ERRATA_A53_855873))
 $(eval $(call add_define,ERRATA_A53_855873))
 
+# Process ERRATA_A53_1530924 flag
+$(eval $(call assert_boolean,ERRATA_A53_1530924))
+$(eval $(call add_define,ERRATA_A53_1530924))
+
 # Process ERRATA_A55_768277 flag
 $(eval $(call assert_boolean,ERRATA_A55_768277))
 $(eval $(call add_define,ERRATA_A55_768277))
@@ -403,6 +427,10 @@ $(eval $(call add_define,ERRATA_A55_903758))
 $(eval $(call assert_boolean,ERRATA_A55_1221012))
 $(eval $(call add_define,ERRATA_A55_1221012))
 
+# Process ERRATA_A55_1530923 flag
+$(eval $(call assert_boolean,ERRATA_A55_1530923))
+$(eval $(call add_define,ERRATA_A55_1530923))
+
 # Process ERRATA_A57_806969 flag
 $(eval $(call assert_boolean,ERRATA_A57_806969))
 $(eval $(call add_define,ERRATA_A57_806969))
@@ -447,10 +475,18 @@ $(eval $(call add_define,ERRATA_A57_833471))
 $(eval $(call assert_boolean,ERRATA_A57_859972))
 $(eval $(call add_define,ERRATA_A57_859972))
 
+# Process ERRATA_A57_1319537 flag
+$(eval $(call assert_boolean,ERRATA_A57_1319537))
+$(eval $(call add_define,ERRATA_A57_1319537))
+
 # Process ERRATA_A72_859971 flag
 $(eval $(call assert_boolean,ERRATA_A72_859971))
 $(eval $(call add_define,ERRATA_A72_859971))
 
+# Process ERRATA_A72_1319367 flag
+$(eval $(call assert_boolean,ERRATA_A72_1319367))
+$(eval $(call add_define,ERRATA_A72_1319367))
+
 # Process ERRATA_A73_852427 flag
 $(eval $(call assert_boolean,ERRATA_A73_852427))
 $(eval $(call add_define,ERRATA_A73_852427))
@@ -507,6 +543,10 @@ $(eval $(call add_define,ERRATA_A76_1791580))
 $(eval $(call assert_boolean,ERRATA_A76_1800710))
 $(eval $(call add_define,ERRATA_A76_1800710))
 
+# Process ERRATA_A76_1165522 flag
+$(eval $(call assert_boolean,ERRATA_A76_1165522))
+$(eval $(call add_define,ERRATA_A76_1165522))
+
 # Process ERRATA_A77_1800714 flag
 $(eval $(call assert_boolean,ERRATA_A77_1800714))
 $(eval $(call add_define,ERRATA_A77_1800714))
@@ -580,3 +620,10 @@ ifneq (${ERRATA_A53_835769},0)
 TF_CFLAGS_aarch64	+= -mfix-cortex-a53-835769
 TF_LDFLAGS_aarch64	+= --fix-cortex-a53-835769
 endif
+
+ifneq ($(filter 1,${ERRATA_A53_1530924} ${ERRATA_A55_1530923}	\
+	${ERRATA_A57_1319537} ${ERRATA_A72_1319367} ${ERRATA_A76_1165522}),)
+ERRATA_SPECULATIVE_AT	:= 1
+else
+ERRATA_SPECULATIVE_AT	:= 0
+endif

plat/arm/common/arm_common.c

@@ -216,7 +216,7 @@ int plat_sdei_validate_entry_point(uintptr_t ep, unsigned int client_mode)
 		 * Translate entry point to Physical Address using the EL1&0
 		 * translation regime, including stage 2.
 		 */
-		ats12e1r(ep);
+		AT(ats12e1r, ep);
 	}
 	isb();
 	par = read_par_el1();

services/spd/tlkd/tlkd_common.c

@@ -38,16 +38,16 @@ uint64_t tlkd_va_translate(uintptr_t va, int type)
 	int at = type & AT_MASK;
 	switch (at) {
 	case 0:
-		ats12e1r(va);
+		AT(ats12e1r, va);
 		break;
 	case 1:
-		ats12e1w(va);
+		AT(ats12e1w, va);
 		break;
 	case 2:
-		ats12e0r(va);
+		AT(ats12e0r, va);
 		break;
 	case 3:
-		ats12e0w(va);
+		AT(ats12e0w, va);
 		break;
 	default:
 		assert(0); /* Unreachable */