diff --git a/Makefile b/Makefile index 017fc659e02f19c9b2a5f9fc17e42b6904f4ac3e..fbc1cc7d09582a9dd062a69175045fbf8e769989 100644 --- a/Makefile +++ b/Makefile @@ -913,6 +913,7 @@ $(eval $(call assert_booleans,\ ENABLE_RUNTIME_INSTRUMENTATION \ ENABLE_SPE_FOR_LOWER_ELS \ ENABLE_SVE_FOR_NS \ + ENABLE_SVE_FOR_SWD \ ERROR_DEPRECATED \ FAULT_INJECTION_SUPPORT \ GENERATE_COT \ @@ -1006,6 +1007,7 @@ $(eval $(call add_defines,\ ENABLE_RUNTIME_INSTRUMENTATION \ ENABLE_SPE_FOR_LOWER_ELS \ ENABLE_SVE_FOR_NS \ + ENABLE_SVE_FOR_SWD \ ENCRYPT_BL31 \ ENCRYPT_BL32 \ ERROR_DEPRECATED \ diff --git a/docs/getting_started/build-options.rst b/docs/getting_started/build-options.rst index 99fc21db329f25993f3506a521d3f1a3ff79e71e..584430432348db2d7e6e0edaec009b58e1e5c750 100644 --- a/docs/getting_started/build-options.rst +++ b/docs/getting_started/build-options.rst @@ -278,7 +278,8 @@ Common build options - ``ENABLE_SVE_FOR_NS``: Boolean option to enable Scalable Vector Extension (SVE) for the Non-secure world only. SVE is an optional architectural feature for AArch64. Note that when SVE is enabled for the Non-secure world, access - to SIMD and floating-point functionality from the Secure world is disabled. + to SIMD and floating-point functionality from the Secure world is disabled by + default and controlled with ENABLE_SVE_FOR_SWD. This is to avoid corruption of the Non-secure world data in the Z-registers which are aliased by the SIMD and FP registers. The build option is not compatible with the ``CTX_INCLUDE_FPREGS`` build option, and will raise an @@ -286,6 +287,11 @@ Common build options 1. The default is 1 but is automatically disabled when the target architecture is AArch32. +- ``ENABLE_SVE_FOR_SWD``: Boolean option to enable SVE for the Secure world. + SVE is an optional architectural feature for AArch64. Note that this option + requires ENABLE_SVE_FOR_NS to be enabled. The default is 0 and it is + automatically disabled when the target architecture is AArch32. + - ``ENABLE_STACK_PROTECTOR``: String option to enable the stack protection checks in GCC. Allowed values are "all", "strong", "default" and "none". The default value is set to "none". "strong" is the recommended stack protection diff --git a/include/arch/aarch64/arch.h b/include/arch/aarch64/arch.h index 9ef9c2689496db994bcaeada0ed981ad4ffc941a..3383a3bf467cd86b06b251859d11f45529d6809e 100644 --- a/include/arch/aarch64/arch.h +++ b/include/arch/aarch64/arch.h @@ -170,6 +170,7 @@ #define ID_AA64PFR0_GIC_MASK ULL(0xf) #define ID_AA64PFR0_SVE_SHIFT U(32) #define ID_AA64PFR0_SVE_MASK ULL(0xf) +#define ID_AA64PFR0_SVE_LENGTH U(4) #define ID_AA64PFR0_SEL2_SHIFT U(36) #define ID_AA64PFR0_SEL2_MASK ULL(0xf) #define ID_AA64PFR0_MPAM_SHIFT U(40) @@ -529,7 +530,7 @@ #define TTA_BIT (U(1) << 20) #define TFP_BIT (U(1) << 10) #define CPTR_EZ_BIT (U(1) << 8) -#define CPTR_EL3_RESET_VAL U(0x0) +#define CPTR_EL3_RESET_VAL (TCPAC_BIT | TAM_BIT | TTA_BIT | TFP_BIT & ~(CPTR_EZ_BIT)) /* CPTR_EL2 definitions */ #define CPTR_EL2_RES1 ((U(1) << 13) | (U(1) << 12) | (U(0x3ff))) diff --git a/include/arch/aarch64/el3_common_macros.S b/include/arch/aarch64/el3_common_macros.S index b610b37bd141426171d0c8754f97e7bd0beed216..9734335755eb1db072b483ab420b34b87f8ca850 100644 --- a/include/arch/aarch64/el3_common_macros.S +++ b/include/arch/aarch64/el3_common_macros.S @@ -185,7 +185,14 @@ * CPTR_EL3.TFP: Set to zero so that accesses to the V- or Z- registers * by Advanced SIMD, floating-point or SVE instructions (if implemented) * do not trap to EL3. + * + * CPTR_EL3.TAM: Set to one so that Activity Monitor access is + * trapped to EL3 by default. + * + * CPTR_EL3.EZ: Set to zero so that all SVE functionality is trapped + * to EL3 by default. */ + mov_imm x0, (CPTR_EL3_RESET_VAL & ~(TCPAC_BIT | TTA_BIT | TFP_BIT)) msr cptr_el3, x0 diff --git a/include/lib/el3_runtime/aarch64/context.h b/include/lib/el3_runtime/aarch64/context.h index 9d9f9d3325557ee3253058baf0944477339a3b22..d449a65ed3cc5dfd4d0d4c04b4630d82004e6e27 100644 --- a/include/lib/el3_runtime/aarch64/context.h +++ b/include/lib/el3_runtime/aarch64/context.h @@ -61,7 +61,9 @@ #define CTX_ELR_EL3 U(0x20) #define CTX_PMCR_EL0 U(0x28) #define CTX_IS_IN_EL3 U(0x30) -#define CTX_EL3STATE_END U(0x40) /* Align to the next 16 byte boundary */ +#define CTX_CPTR_EL3 U(0x38) +#define CTX_ZCR_EL3 U(0x40) +#define CTX_EL3STATE_END U(0x50) /* Align to the next 16 byte boundary */ /******************************************************************************* * Constants that allow assembler code to access members of and the diff --git a/include/lib/extensions/sve.h b/include/lib/extensions/sve.h index 83df1775e57b9eb4ef4028249b22129304875f67..c85e08c9f73f5b5438e00443fedab083285ccbe3 100644 --- a/include/lib/extensions/sve.h +++ b/include/lib/extensions/sve.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2018, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2017-2021, ARM Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ @@ -7,9 +7,8 @@ #ifndef SVE_H #define SVE_H -#include +#include -bool sve_supported(void); -void sve_enable(bool el2_unused); +void sve_enable(cpu_context_t *context); #endif /* SVE_H */ diff --git a/lib/el3_runtime/aarch64/context.S b/lib/el3_runtime/aarch64/context.S index 0ec9ffd5d6a77a2e335b1e42e872b0114af99aba..d610fd49234ef37ea2104cac3d9210c3a3d8f079 100644 --- a/lib/el3_runtime/aarch64/context.S +++ b/lib/el3_runtime/aarch64/context.S @@ -901,6 +901,29 @@ func el3_exit msr spsr_el3, x16 msr elr_el3, x17 +#if IMAGE_BL31 + /* ---------------------------------------------------------- + * Restore CPTR_EL3, ZCR_EL3 for SVE support. + * If SVE is not supported - skip the restoration. + * ZCR is only restored if SVE is supported and enabled. + * Synchronization is required before zcr_el3 is addressed. + * ---------------------------------------------------------- + */ + mrs x17, id_aa64pfr0_el1 + ubfx x17, x17, ID_AA64PFR0_SVE_SHIFT, ID_AA64PFR0_SVE_LENGTH + cbz x17, sve_not_enabled + + ldp x19, x20, [sp, #CTX_EL3STATE_OFFSET + CTX_CPTR_EL3] + msr cptr_el3, x19 + + ands x19, x19, #CPTR_EZ_BIT + beq sve_not_enabled + + isb + msr S3_6_C1_C2_0, x20 /* zcr_el3 */ +sve_not_enabled: +#endif + #if IMAGE_BL31 && DYNAMIC_WORKAROUND_CVE_2018_3639 /* ---------------------------------------------------------- * Restore mitigation state as it was on entry to EL3 diff --git a/lib/el3_runtime/aarch64/context_mgmt.c b/lib/el3_runtime/aarch64/context_mgmt.c index 96023b694180b79c8d0786f4298e75d6f416467d..7a25151c587041bc476d7a67d36dc6082c6fa71d 100644 --- a/lib/el3_runtime/aarch64/context_mgmt.c +++ b/lib/el3_runtime/aarch64/context_mgmt.c @@ -178,6 +178,18 @@ void cm_setup_context(cpu_context_t *ctx, const entry_point_info_t *ep) * indicated by the interrupt routing model for BL31. */ scr_el3 |= get_scr_el3_from_routing_model(security_state); + +#if ENABLE_SVE_FOR_NS + if (security_state == NON_SECURE) { + sve_enable(ctx); + } +#endif +#if ENABLE_SVE_FOR_SWD + if (security_state == SECURE) { + sve_enable(ctx); + } +#endif + #endif /* @@ -334,10 +346,6 @@ static void enable_extensions_nonsecure(bool el2_unused) amu_enable(el2_unused); #endif -#if ENABLE_SVE_FOR_NS - sve_enable(el2_unused); -#endif - #if ENABLE_MPAM_FOR_LOWER_ELS mpam_enable(el2_unused); #endif diff --git a/lib/extensions/sve/sve.c b/lib/extensions/sve/sve.c index fa4ac77582976ad5efdc7d8a88a178a42f1d1752..7043cc220dfc9575226b607d046270b2390e9623 100644 --- a/lib/extensions/sve/sve.c +++ b/lib/extensions/sve/sve.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2018, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2017-2021, ARM Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ @@ -11,7 +11,13 @@ #include #include -bool sve_supported(void) +/* + * Converts SVE vector size restriction in bytes to LEN according to ZCR_EL3 documentation. + * VECTOR_SIZE = (LEN+1) * 128 + */ +#define CONVERT_SVE_LENGTH(x) (((x / 128) - 1)) + +static bool sve_supported(void) { uint64_t features; @@ -19,113 +25,19 @@ bool sve_supported(void) return (features & ID_AA64PFR0_SVE_MASK) == 1U; } -static void *disable_sve_hook(const void *arg) +void sve_enable(cpu_context_t *context) { - uint64_t cptr; - - if (!sve_supported()) - return (void *)-1; - - /* - * Disable SVE, SIMD and FP access for the Secure world. - * As the SIMD/FP registers are part of the SVE Z-registers, any - * use of SIMD/FP functionality will corrupt the SVE registers. - * Therefore it is necessary to prevent use of SIMD/FP support - * in the Secure world as well as SVE functionality. - */ - cptr = read_cptr_el3(); - cptr = (cptr | TFP_BIT) & ~(CPTR_EZ_BIT); - write_cptr_el3(cptr); - - /* - * No explicit ISB required here as ERET to switch to Secure - * world covers it - */ - return (void *)0; -} - -static void *enable_sve_hook(const void *arg) -{ - uint64_t cptr; - - if (!sve_supported()) - return (void *)-1; - - /* - * Enable SVE, SIMD and FP access for the Non-secure world. - */ - cptr = read_cptr_el3(); - cptr = (cptr | CPTR_EZ_BIT) & ~(TFP_BIT); - write_cptr_el3(cptr); - - /* - * No explicit ISB required here as ERET to switch to Non-secure - * world covers it - */ - return (void *)0; -} - -void sve_enable(bool el2_unused) -{ - uint64_t cptr; - - if (!sve_supported()) + if (!sve_supported()) { return; + } -#if CTX_INCLUDE_FPREGS - /* - * CTX_INCLUDE_FPREGS is not supported on SVE enabled systems. - */ - assert(0); -#endif - /* - * Update CPTR_EL3 to enable access to SVE functionality for the - * Non-secure world. - * NOTE - assumed that CPTR_EL3.TFP is set to allow access to - * the SIMD, floating-point and SVE support. - * - * CPTR_EL3.EZ: Set to 1 to enable access to SVE functionality - * in the Non-secure world. - */ - cptr = read_cptr_el3(); - cptr |= CPTR_EZ_BIT; - write_cptr_el3(cptr); - - /* - * Need explicit ISB here to guarantee that update to ZCR_ELx - * and CPTR_EL2.TZ do not result in trap to EL3. - */ - isb(); - - /* - * Ensure lower ELs have access to full vector length. - */ - write_zcr_el3(ZCR_EL3_LEN_MASK); + u_register_t cptr_el3 = read_cptr_el3(); - if (el2_unused) { - /* - * Update CPTR_EL2 to enable access to SVE functionality - * for Non-secure world, EL2 and Non-secure EL1 and EL0. - * NOTE - assumed that CPTR_EL2.TFP is set to allow - * access to the SIMD, floating-point and SVE support. - * - * CPTR_EL2.TZ: Set to 0 to enable access to SVE support - * for EL2 and Non-secure EL1 and EL0. - */ - cptr = read_cptr_el2(); - cptr &= ~(CPTR_EL2_TZ_BIT); - write_cptr_el2(cptr); + /* Enable access to SVE functionality for all ELs. */ + cptr_el3 = (cptr_el3 | CPTR_EZ_BIT) & ~(TFP_BIT); + write_ctx_reg(get_el3state_ctx(context), CTX_CPTR_EL3, cptr_el3); - /* - * Ensure lower ELs have access to full vector length. - */ - write_zcr_el2(ZCR_EL2_LEN_MASK); - } - /* - * No explicit ISB required here as ERET to switch to - * Non-secure world covers it. - */ + /* Restrict maximum SVE vector length (SVE_VECTOR_LENGTH+1) * 128. */ + write_ctx_reg(get_el3state_ctx(context), CTX_ZCR_EL3, + (ZCR_EL3_LEN_MASK & CONVERT_SVE_LENGTH(512))); } - -SUBSCRIBE_TO_EVENT(cm_exited_normal_world, disable_sve_hook); -SUBSCRIBE_TO_EVENT(cm_entering_normal_world, enable_sve_hook); diff --git a/make_helpers/defaults.mk b/make_helpers/defaults.mk index 043312372015801de20af29c162ce1254ec27962..b2d1ee26740d6668a46ae116362ea2c18d4b8b1a 100644 --- a/make_helpers/defaults.mk +++ b/make_helpers/defaults.mk @@ -299,13 +299,15 @@ CTX_INCLUDE_MTE_REGS := 0 ENABLE_AMU := 0 AMU_RESTRICT_COUNTERS := 0 -# By default, enable Scalable Vector Extension if implemented for Non-secure +# By default, enable Scalable Vector Extension if implemented only for Non-secure # lower ELs # Note SVE is only supported on AArch64 - therefore do not enable in AArch32 ifneq (${ARCH},aarch32) ENABLE_SVE_FOR_NS := 1 + ENABLE_SVE_FOR_SWD := 0 else override ENABLE_SVE_FOR_NS := 0 + override ENABLE_SVE_FOR_SWD := 0 endif SANITIZE_UB := off