Merge pull request #1603 from antonio-nino-diaz-arm/db/reclaim-init

 Reclaim BL31 initialization code memory for runtime data

Makefile

@@ -634,6 +634,7 @@ $(eval $(call add_define,PSCI_EXTENDED_STATE_ID))
 $(eval $(call add_define,RAS_EXTENSION))
 $(eval $(call add_define,RESET_TO_BL31))
 $(eval $(call add_define,SEPARATE_CODE_AND_RODATA))
+$(eval $(call add_define,RECLAIM_INIT_CODE))
 $(eval $(call add_define,SMCCC_MAJOR_VERSION))
 $(eval $(call add_define,SPD_${SPD}))
 $(eval $(call add_define,SPIN_ON_BL1_EXIT))

bl31/bl31_main.c

@@ -57,7 +57,7 @@ uintptr_t get_arm_std_svc_args(unsigned int svc_mask)
 /*******************************************************************************
  * Simple function to initialise all BL31 helper libraries.
  ******************************************************************************/
-void bl31_lib_init(void)
+void __init bl31_lib_init(void)
 {
 	cm_init();
 }
@@ -149,7 +149,7 @@ uint32_t bl31_get_next_image_type(void)
  * This function programs EL3 registers and performs other setup to enable entry
  * into the next image after BL31 at the next ERET.
  ******************************************************************************/
-void bl31_prepare_next_image_entry(void)
+void __init bl31_prepare_next_image_entry(void)
 {
 	entry_point_info_t *next_image_info;
 	uint32_t image_type;

bl31/ehf.c

@@ -451,7 +451,7 @@ static uint64_t ehf_el3_interrupt_handler(uint32_t id, uint32_t flags,
 /*
  * Initialize the EL3 exception handling.
  */
-void ehf_init(void)
+void __init ehf_init(void)
 {
 	unsigned int flags = 0;
 	int ret __unused;

common/runtime_svc.c

@@ -93,7 +93,7 @@ static int32_t validate_rt_svc_desc(const rt_svc_desc_t *desc)
  * The unique oen is used as an index into the 'rt_svc_descs_indices' array.
  * The index of the runtime service descriptor is stored at this index.
  ******************************************************************************/
-void runtime_svc_init(void)
+void __init runtime_svc_init(void)
 {
 	int rc = 0;
 	unsigned int index, start_idx, end_idx;

docs/firmware-design.rst

@@ -2336,6 +2336,29 @@ implement:
 
    SUBSCRIBE_TO_EVENT(foo, foo_handler);
 
+
+Reclaiming the BL31 initialization code
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A significant amount of the code used for the initialization of BL31 is never
+needed again after boot time. In order to reduce the runtime memory
+footprint, the memory used for this code can be reclaimed after initialization
+has finished and be used for runtime data.
+
+The build option ``RECLAIM_INIT_CODE`` can be set to mark this boot time code
+with a ``.text.init.*`` attribute which can be filtered and placed suitably
+within the BL image for later reclaimation by the platform. The platform can
+specify the fiter and the memory region for this init section in BL31 via the
+plat.ld.S linker script. For example, on the FVP, this section is placed
+overlapping the secondary CPU stacks so that after the cold boot is done, this
+memory can be reclaimed for the stacks. The init memory section is initially
+mapped with ``RO``, ``EXECUTE`` attributes. After BL31 initilization has
+completed, the FVP changes the attributes of this section to ``RW``,
+``EXECUTE_NEVER`` allowing it to be used for runtime data. The memory attributes
+are changed within the ``bl31_plat_runtime_setup`` platform hook. The init
+section section can be reclaimed for any data which is accessed after cold
+boot initialization and it is upto the platform to make the decision.
+
 Performance Measurement Framework
 ---------------------------------
 

drivers/arm/cci/cci.c

@@ -107,7 +107,8 @@ static int get_slave_ports(unsigned int part_num)
 }
 #endif /* ENABLE_ASSERTIONS */
 
-void cci_init(uintptr_t base, const int *map, unsigned int num_cci_masters)
+void __init cci_init(uintptr_t base, const int *map,
+				unsigned int num_cci_masters)
 {
 	assert(map != NULL);
 	assert(base != 0U);

drivers/arm/ccn/ccn.c

 /*
- * Copyright (c) 2015-2017, ARM Limited and Contributors. All rights reserved.
+ * Copyright (c) 2015-2018, ARM Limited and Contributors. All rights reserved.
  *
  * SPDX-License-Identifier: BSD-3-Clause
  */
@@ -167,7 +167,7 @@ static unsigned int ccn_get_rn_master_info(uintptr_t periphbase,
  * It compares this with the information provided by the platform to determine
  * the validity of the latter.
  ******************************************************************************/
-static void ccn_validate_plat_params(const ccn_desc_t *plat_desc)
+static void __init ccn_validate_plat_params(const ccn_desc_t *plat_desc)
 {
 	unsigned int master_id, num_rn_masters;
 	rn_info_t info = { {0} };
@@ -208,7 +208,7 @@ static void ccn_validate_plat_params(const ccn_desc_t *plat_desc)
  * simultaneous CCN operations at runtime (only BL31) to add and remove Request
  * nodes from coherency.
  ******************************************************************************/
-void ccn_init(const ccn_desc_t *plat_desc)
+void __init ccn_init(const ccn_desc_t *plat_desc)
 {
 #if ENABLE_ASSERTIONS
 	ccn_validate_plat_params(plat_desc);

drivers/arm/gic/v3/gicv3_main.c

@@ -55,7 +55,7 @@ static spinlock_t gic_lock;
  * This function initialises the ARM GICv3 driver in EL3 with provided platform
  * inputs.
  ******************************************************************************/
-void gicv3_driver_init(const gicv3_driver_data_t *plat_driver_data)
+void __init gicv3_driver_init(const gicv3_driver_data_t *plat_driver_data)
 {
 	unsigned int gic_version;
 
@@ -129,7 +129,7 @@ void gicv3_driver_init(const gicv3_driver_data_t *plat_driver_data)
  * This function initialises the GIC distributor interface based upon the data
  * provided by the platform while initialising the driver.
  ******************************************************************************/
-void gicv3_distif_init(void)
+void __init gicv3_distif_init(void)
 {
 	unsigned int bitmap = 0;
 

drivers/arm/smmu/smmu_v3.c

@@ -4,21 +4,22 @@
  * SPDX-License-Identifier: BSD-3-Clause
  */
 
+#include <cdefs.h>
 #include <mmio.h>
 #include <smmu_v3.h>
 #include <stdbool.h>
 
-static inline uint32_t smmuv3_read_s_idr1(uintptr_t base)
+static inline uint32_t __init smmuv3_read_s_idr1(uintptr_t base)
 {
 	return mmio_read_32(base + SMMU_S_IDR1);
 }
 
-static inline uint32_t smmuv3_read_s_init(uintptr_t base)
+static inline uint32_t __init smmuv3_read_s_init(uintptr_t base)
 {
 	return mmio_read_32(base + SMMU_S_INIT);
 }
 
-static inline void smmuv3_write_s_init(uintptr_t base, uint32_t value)
+static inline void __init smmuv3_write_s_init(uintptr_t base, uint32_t value)
 {
 	mmio_write_32(base + SMMU_S_INIT, value);
 }
@@ -34,7 +35,7 @@ static inline bool smmuv3_inval_pending(uintptr_t base)
  *
  * Returns 0 on success, and -1 on failure.
  */
-int smmuv3_init(uintptr_t smmu_base)
+int __init smmuv3_init(uintptr_t smmu_base)
 {
 	uint32_t idr1_reg;
 

include/lib/libc/cdefs.h

@@ -14,6 +14,15 @@
 #define __unused	__attribute__((__unused__))
 #define __aligned(x)	__attribute__((__aligned__(x)))
 #define __section(x)	__attribute__((__section__(x)))
+#if RECLAIM_INIT_CODE
+/*
+ * Add each function to a section that is unique so the functions can still
+ * be garbage collected
+ */
+#define __init		__section(".text.init." __FILE__ "." __XSTRING(__LINE__))
+#else
+#define __init
+#endif
 
 #define __printflike(fmtarg, firstvararg) \
 		__attribute__((__format__ (__printf__, fmtarg, firstvararg)))

include/plat/arm/common/arm_def.h

@@ -280,7 +280,7 @@
  * The max number of regions like RO(code), coherent and data required by
  * different BL stages which need to be mapped in the MMU.
  */
-# define ARM_BL_REGIONS			4
+#define ARM_BL_REGIONS			5
 
 #define MAX_MMAP_REGIONS		(PLAT_ARM_MMAP_ENTRIES +	\
 					 ARM_BL_REGIONS)

include/plat/arm/common/arm_reclaim_init.ld.S

+/*
+ * Copyright (c) 2017-2018, ARM Limited and Contributors. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+#ifndef ARM_RECLAIM_INIT_LD_S
+#define ARM_RECLAIM_INIT_LD_S
+
+SECTIONS
+{
+        .init __STACKS_START__ : {
+            . = . + PLATFORM_STACK_SIZE;
+            . = ALIGN(PAGE_SIZE);
+            __INIT_CODE_START__ = .;
+            /*
+             * Exclude PSCI initialization functions to ensure the init section
+             * does not become larger than the overlaid stack region
+             */
+            *(EXCLUDE_FILE (*psci_setup.o).text.init*)
+            __INIT_CODE_UNALIGNED__ = .;
+            .  = ALIGN(PAGE_SIZE);
+            __INIT_CODE_END__ = .;
+        } >RAM
+
+#ifdef BL31_PROGBITS_LIMIT
+    ASSERT(__INIT_CODE_END__ <= BL31_PROGBITS_LIMIT,
+            "BL31 init has exceeded progbits limit.")
+#endif
+
+#if RECLAIM_INIT_CODE
+    ASSERT(__INIT_CODE_END__ <= __STACKS_END__,
+        "Init code ends past the end of the stacks")
+#endif
+}
+
+#endif /* ARM_RECLAIM_INIT_LD_S */

include/plat/arm/common/arm_common.ld.S → include/plat/arm/common/arm_tzc_dram.ld.S

@@ -3,8 +3,8 @@
  *
  * SPDX-License-Identifier: BSD-3-Clause
  */
-#ifndef __ARM_COMMON_LD_S__
-#define __ARM_COMMON_LD_S__
+#ifndef ARM_TZC_DRAM_LD_S__
+#define ARM_TZC_DRAM_LD_S__
 
 #include <xlat_tables_defs.h>
 
@@ -27,4 +27,4 @@ SECTIONS
 	} >EL3_SEC_DRAM
 }
 
-#endif /* __ARM_COMMON_LD_S__ */
+#endif /* ARM_TZC_DRAM_LD_S__ */

include/plat/arm/common/plat_arm.h

@@ -220,6 +220,12 @@ void arm_bl2_dyn_cfg_init(void);
 void arm_bl1_set_mbedtls_heap(void);
 int arm_get_mbedtls_heap(void **heap_addr, size_t *heap_size);
 
+/*
+ * Free the memory storing initialization code only used during an images boot
+ * time so it can be reclaimed for runtime data
+ */
+void arm_free_init_memory(void);
+
 /*
  * Mandatory functions required in ARM standard platforms
  */

lib/el3_runtime/aarch64/context_mgmt.c

@@ -36,7 +36,7 @@
  * which will used for programming an entry into a lower EL. The same context
  * will used to save state upon exception entry from that EL.
  ******************************************************************************/
-void cm_init(void)
+void __init cm_init(void)
 {
 	/*
 	 * The context management library has only global data to intialize, but

lib/extensions/ras/ras_common.c

@@ -128,7 +128,7 @@ static int ras_interrupt_handler(uint32_t intr_raw, uint32_t flags,
 	return 0;
 }
 
-void ras_init(void)
+void __init ras_init(void)
 {
 #if ENABLE_ASSERTIONS
 	/* Check RAS interrupts are sorted */

lib/psci/psci_common.c

@@ -216,7 +216,7 @@ static void psci_set_req_local_pwr_state(unsigned int pwrlvl,
 /******************************************************************************
  * This function initializes the psci_req_local_pwr_states.
  *****************************************************************************/
-void psci_init_req_local_pwr_states(void)
+void __init psci_init_req_local_pwr_states(void)
 {
 	/* Initialize the requested state of all non CPU power domains as OFF */
 	unsigned int pwrlvl;

lib/psci/psci_setup.c

@@ -32,7 +32,7 @@ unsigned int psci_caps;
  * Function which initializes the 'psci_non_cpu_pd_nodes' or the
  * 'psci_cpu_pd_nodes' corresponding to the power level.
  ******************************************************************************/
-static void psci_init_pwr_domain_node(unsigned char node_idx,
+static void __init psci_init_pwr_domain_node(unsigned char node_idx,
 					unsigned int parent_idx,
 					unsigned char level)
 {
@@ -80,7 +80,7 @@ static void psci_init_pwr_domain_node(unsigned char node_idx,
  * mapping of the CPUs to indices via plat_core_pos_by_mpidr() and
  * plat_my_core_pos() APIs.
  *******************************************************************************/
-static void psci_update_pwrlvl_limits(void)
+static void __init psci_update_pwrlvl_limits(void)
 {
 	int j, cpu_idx;
 	unsigned int nodes_idx[PLAT_MAX_PWR_LVL] = {0};
@@ -107,7 +107,7 @@ static void psci_update_pwrlvl_limits(void)
  * informs the number of root power domains. The parent nodes of the root nodes
  * will point to an invalid entry(-1).
  ******************************************************************************/
-static void populate_power_domain_tree(const unsigned char *topology)
+static void __init populate_power_domain_tree(const unsigned char *topology)
 {
 	unsigned int i, j = 0U, num_nodes_at_lvl = 1U, num_nodes_at_next_lvl;
 	unsigned int node_index = 0U, num_children;
@@ -184,7 +184,7 @@ static void populate_power_domain_tree(const unsigned char *topology)
  * |   CPU 0   |   CPU 1   |   CPU 2   |   CPU 3  |
  * ------------------------------------------------
  ******************************************************************************/
-int psci_setup(const psci_lib_args_t *lib_args)
+int __init psci_setup(const psci_lib_args_t *lib_args)
 {
 	const unsigned char *topology_tree;
 

lib/xlat_tables/xlat_tables_common.c

@@ -176,7 +176,7 @@ void mmap_add(const mmap_region_t *mm)
 {
 	const mmap_region_t *mm_cursor = mm;
 
-	while (mm_cursor->size != 0U) {
+	while (mm_cursor->attr != 0U) {
 		mmap_add_region(mm_cursor->base_pa, mm_cursor->base_va,
 				mm_cursor->size, mm_cursor->attr);
 		mm_cursor++;

lib/xlat_tables_v2/xlat_tables_context.c

@@ -56,7 +56,7 @@ int mmap_remove_dynamic_region(uintptr_t base_va, size_t size)
 
 #endif /* PLAT_XLAT_TABLES_DYNAMIC */
 
-void init_xlat_tables(void)
+void __init init_xlat_tables(void)
 {
 	assert(tf_xlat_ctx.xlat_regime == EL_REGIME_INVALID);