n1sdp: update platform macros for dual-chip setup

N1SDP supports multichip configuration wherein n1sdp boards are
connected over high speed coherent CCIX link  for now only dual-chip is
supported.

A single instance of TF-A runs on master chip which should be aware of
slave chip's CPU and memory topology.

This patch updates platform macros to include remote chip's information
and also ensures that a single version of firmware works for both single
and dual-chip setup.

Change-Id: I75799fd46dc10527aa99585226099d836c21da70
Signed-off-by: Manish Pandey <manish.pandey2@arm.com>

plat/arm/board/n1sdp/aarch64/n1sdp_helper.S

@@ -17,19 +17,20 @@
 	 * unsigned int plat_arm_calc_core_pos(u_register_t mpidr)
 	 *
 	 * Helper function to calculate the core position.
-	 * (ClusterId * N1SDP_MAX_CPUS_PER_CLUSTER * N1SDP_MAX_PE_PER_CPU) +
-	 * (CPUId * N1SDP_MAX_PE_PER_CPU) +
-	 * ThreadId
+	 * ((ChipId * N1SDP_MAX_CLUSTERS_PER_CHIP + ClusterId) *
+	 * N1SDP_MAX_CPUS_PER_CLUSTER * N1SDP_MAX_PE_PER_CPU) +
+	 * (CPUId * N1SDP_MAX_PE_PER_CPU) + ThreadId
 	 *
 	 * which can be simplified as:
 	 *
-	 * ((ClusterId * N1SDP_MAX_CPUS_PER_CLUSTER + CPUId) *
-	 * N1SDP_MAX_PE_PER_CPU) + ThreadId
+	 * (((ChipId * N1SDP_MAX_CLUSTERS_PER_CHIP + ClusterId) *
+	 * N1SDP_MAX_CPUS_PER_CLUSTER + CPUId) * N1SDP_MAX_PE_PER_CPU) +
+	 * ThreadId
 	 * ------------------------------------------------------
 	 */
 
 func plat_arm_calc_core_pos
-	mov	x3, x0
+	mov	x4, x0
 
 	/*
 	 * The MT bit in MPIDR is always set for n1sdp and the
@@ -37,15 +38,18 @@ func plat_arm_calc_core_pos
 	 */
 
 	/* Extract individual affinity fields from MPIDR */
-	ubfx	x0, x3, #MPIDR_AFF0_SHIFT, #MPIDR_AFFINITY_BITS
-	ubfx	x1, x3, #MPIDR_AFF1_SHIFT, #MPIDR_AFFINITY_BITS
-	ubfx	x2, x3, #MPIDR_AFF2_SHIFT, #MPIDR_AFFINITY_BITS
+	ubfx	x0, x4, #MPIDR_AFF0_SHIFT, #MPIDR_AFFINITY_BITS
+	ubfx	x1, x4, #MPIDR_AFF1_SHIFT, #MPIDR_AFFINITY_BITS
+	ubfx	x2, x4, #MPIDR_AFF2_SHIFT, #MPIDR_AFFINITY_BITS
+	ubfx	x3, x4, #MPIDR_AFF3_SHIFT, #MPIDR_AFFINITY_BITS
 
 	/* Compute linear position */
+	mov	x4, #N1SDP_MAX_CLUSTERS_PER_CHIP
+	madd	x2, x3, x4, x2
 	mov	x4, #N1SDP_MAX_CPUS_PER_CLUSTER
 	madd	x1, x2, x4, x1
-	mov	x5, #N1SDP_MAX_PE_PER_CPU
-	madd	x0, x1, x5, x0
+	mov	x4, #N1SDP_MAX_PE_PER_CPU
+	madd	x0, x1, x4, x0
 	ret
 endfunc plat_arm_calc_core_pos
 

plat/arm/board/n1sdp/include/platform_def.h

@@ -27,16 +27,27 @@
 #define PLAT_ARM_DRAM2_BASE			ULL(0x8080000000)
 #define PLAT_ARM_DRAM2_SIZE			ULL(0xF80000000)
 
+/* N1SDP remote chip at 4 TB offset */
+#define PLAT_ARM_REMOTE_CHIP_OFFSET		(ULL(1) << 42)
+
+#define N1SDP_REMOTE_DRAM1_BASE			ARM_DRAM1_BASE + \
+						PLAT_ARM_REMOTE_CHIP_OFFSET
+#define N1SDP_REMOTE_DRAM1_SIZE			ARM_DRAM1_SIZE
+
+#define N1SDP_REMOTE_DRAM2_BASE			PLAT_ARM_DRAM2_BASE + \
+						PLAT_ARM_REMOTE_CHIP_OFFSET
+#define N1SDP_REMOTE_DRAM2_SIZE			PLAT_ARM_DRAM2_SIZE
+
 /*
  * N1SDP platform supports RDIMMs with ECC capability. To use the ECC
  * capability, the entire DDR memory space has to be zeroed out before
- * enabling the ECC bits in DMC620. The access the complete DDR memory
- * space the physical & virtual address space limits are extended to
- * 40-bits.
+ * enabling the ECC bits in DMC620. To access the complete DDR memory
+ * along with remote chip's DDR memory, which is at 4 TB offset, physical
+ * and virtual address space limits are extended to 43-bits.
  */
 #ifdef __aarch64__
-#define PLAT_PHY_ADDR_SPACE_SIZE		(1ULL << 40)
-#define PLAT_VIRT_ADDR_SPACE_SIZE		(1ULL << 40)
+#define PLAT_PHY_ADDR_SPACE_SIZE		(1ULL << 43)
+#define PLAT_VIRT_ADDR_SPACE_SIZE		(1ULL << 43)
 #else
 #define PLAT_PHY_ADDR_SPACE_SIZE		(1ULL << 32)
 #define PLAT_VIRT_ADDR_SPACE_SIZE		(1ULL << 32)
@@ -51,34 +62,36 @@
 #define PLAT_ARM_TRUSTED_SRAM_SIZE		0x00080000	/* 512 KB */
 #define PLAT_ARM_MAX_BL31_SIZE			0X20000
 
-
 /*******************************************************************************
  * N1SDP topology related constants
  ******************************************************************************/
-#define N1SDP_MAX_CPUS_PER_CLUSTER		2
-#define PLAT_ARM_CLUSTER_COUNT			2
-#define N1SDP_MAX_PE_PER_CPU			1
-
-#define PLATFORM_CORE_COUNT			(PLAT_ARM_CLUSTER_COUNT *	\
+#define N1SDP_MAX_CPUS_PER_CLUSTER		U(2)
+#define PLAT_ARM_CLUSTER_COUNT			U(2)
+#define PLAT_N1SDP_CHIP_COUNT			U(2)
+#define N1SDP_MAX_CLUSTERS_PER_CHIP		U(2)
+#define N1SDP_MAX_PE_PER_CPU			U(1)
+
+#define PLATFORM_CORE_COUNT			(PLAT_N1SDP_CHIP_COUNT *	\
+						PLAT_ARM_CLUSTER_COUNT *	\
 						N1SDP_MAX_CPUS_PER_CLUSTER *	\
 						N1SDP_MAX_PE_PER_CPU)
 
 /* System power domain level */
-#define CSS_SYSTEM_PWR_DMN_LVL			ARM_PWR_LVL2
+#define CSS_SYSTEM_PWR_DMN_LVL			ARM_PWR_LVL3
 
 /*
  * PLAT_ARM_MMAP_ENTRIES depends on the number of entries in the
  * plat_arm_mmap array defined for each BL stage.
  */
-#define PLAT_ARM_MMAP_ENTRIES			6
-#define MAX_XLAT_TABLES				7
+#define PLAT_ARM_MMAP_ENTRIES			9
+#define MAX_XLAT_TABLES				10
 
 #define PLATFORM_STACK_SIZE			0x400
 
 #define PLAT_ARM_NSTIMER_FRAME_ID		0
 #define PLAT_CSS_MHU_BASE			0x45000000
 #define PLAT_MHUV2_BASE				PLAT_CSS_MHU_BASE
-#define PLAT_MAX_PWR_LVL			1
+#define PLAT_MAX_PWR_LVL			2
 
 #define PLAT_ARM_G1S_IRQS			ARM_G1S_IRQS,			\
 						CSS_IRQ_MHU
@@ -88,8 +101,12 @@
 #define PLAT_ARM_G0_IRQ_PROPS(grp)		ARM_G0_IRQ_PROPS(grp)
 
 
-#define N1SDP_DEVICE_BASE			(0x08000000)
-#define N1SDP_DEVICE_SIZE			(0x48000000)
+#define N1SDP_DEVICE_BASE			ULL(0x08000000)
+#define N1SDP_DEVICE_SIZE			ULL(0x48000000)
+#define N1SDP_REMOTE_DEVICE_BASE		N1SDP_DEVICE_BASE + \
+						PLAT_ARM_REMOTE_CHIP_OFFSET
+#define N1SDP_REMOTE_DEVICE_SIZE		N1SDP_DEVICE_SIZE
+
 #define N1SDP_MAP_DEVICE		MAP_REGION_FLAT(	\
 					N1SDP_DEVICE_BASE,	\
 					N1SDP_DEVICE_SIZE,	\
@@ -100,6 +117,21 @@
 					ARM_DRAM1_SIZE,		\
 					MT_MEMORY | MT_RW | MT_NS)
 
+#define N1SDP_MAP_REMOTE_DEVICE		MAP_REGION_FLAT(		\
+					N1SDP_REMOTE_DEVICE_BASE,	\
+					N1SDP_REMOTE_DEVICE_SIZE,	\
+					MT_DEVICE | MT_RW | MT_SECURE)
+
+#define N1SDP_MAP_REMOTE_DRAM1		MAP_REGION_FLAT(		\
+					N1SDP_REMOTE_DRAM1_BASE,	\
+					N1SDP_REMOTE_DRAM1_SIZE,	\
+					MT_MEMORY | MT_RW | MT_NS)
+
+#define N1SDP_MAP_REMOTE_DRAM2		MAP_REGION_FLAT(		\
+					N1SDP_REMOTE_DRAM2_BASE,	\
+					N1SDP_REMOTE_DRAM2_SIZE,	\
+					MT_MEMORY | MT_RW | MT_NS)
+
 /* GIC related constants */
 #define PLAT_ARM_GICD_BASE			0x30000000
 #define PLAT_ARM_GICC_BASE			0x2C000000

plat/arm/board/n1sdp/n1sdp_bl31_setup.c

@@ -74,7 +74,7 @@ const plat_psci_ops_t *plat_arm_psci_override_pm_ops(plat_psci_ops_t *ops)
  * from IOFPGA-DDR3 memory to main DDR4 memory.
  */
 
-void dmc_ecc_setup(uint32_t ddr_size_gb)
+void dmc_ecc_setup(uint8_t ddr_size_gb)
 {
 	uint64_t dram2_size;
 
@@ -101,6 +101,38 @@ void dmc_ecc_setup(uint32_t ddr_size_gb)
 	mmio_write_32(N1SDP_DMC1_MEMC_CMD_REG, N1SDP_DMC_MEMC_CMD_READY);
 }
 
+void remote_dmc_ecc_setup(uint8_t remote_ddr_size)
+{
+	uint64_t remote_dram2_size;
+
+	remote_dram2_size = (remote_ddr_size * 1024UL * 1024UL * 1024UL) -
+				N1SDP_REMOTE_DRAM1_SIZE;
+	/* multichip setup */
+	INFO("Zeroing remote DDR memories\n");
+	zero_normalmem((void *)N1SDP_REMOTE_DRAM1_BASE,
+			N1SDP_REMOTE_DRAM1_SIZE);
+	flush_dcache_range(N1SDP_REMOTE_DRAM1_BASE, N1SDP_REMOTE_DRAM1_SIZE);
+	zero_normalmem((void *)N1SDP_REMOTE_DRAM2_BASE, remote_dram2_size);
+	flush_dcache_range(N1SDP_REMOTE_DRAM2_BASE, remote_dram2_size);
+
+	INFO("Enabling ECC on remote DMCs\n");
+	/* Set DMCs to CONFIG state before writing ERR0CTLR0 register */
+	mmio_write_32(N1SDP_REMOTE_DMC0_MEMC_CMD_REG,
+			N1SDP_DMC_MEMC_CMD_CONFIG);
+	mmio_write_32(N1SDP_REMOTE_DMC1_MEMC_CMD_REG,
+			N1SDP_DMC_MEMC_CMD_CONFIG);
+
+	/* Enable ECC in DMCs */
+	mmio_setbits_32(N1SDP_REMOTE_DMC0_ERR0CTLR0_REG,
+			N1SDP_DMC_ERR0CTLR0_ECC_EN);
+	mmio_setbits_32(N1SDP_REMOTE_DMC1_ERR0CTLR0_REG,
+			N1SDP_DMC_ERR0CTLR0_ECC_EN);
+
+	/* Set DMCs to READY state */
+	mmio_write_32(N1SDP_REMOTE_DMC0_MEMC_CMD_REG, N1SDP_DMC_MEMC_CMD_READY);
+	mmio_write_32(N1SDP_REMOTE_DMC1_MEMC_CMD_REG, N1SDP_DMC_MEMC_CMD_READY);
+}
+
 void copy_bl33(uint32_t src, uint32_t dst, uint32_t size)
 {
 	uint32_t i;
@@ -151,6 +183,10 @@ void bl31_platform_setup(void)
 
 	dmc_ecc_setup(plat_info.local_ddr_size);
 
+	/* Check if remote memory is present */
+	if ((plat_info.multichip_mode) && (plat_info.remote_ddr_size != 0))
+		remote_dmc_ecc_setup(plat_info.remote_ddr_size);
+
 	ret = sds_struct_read(N1SDP_SDS_BL33_INFO_STRUCT_ID,
 				N1SDP_SDS_BL33_INFO_OFFSET,
 				&bl33_info,

plat/arm/board/n1sdp/n1sdp_def.h

@@ -35,6 +35,18 @@
 #define N1SDP_DMC0_ERR0CTLR0_REG		0x4E000708
 #define N1SDP_DMC1_ERR0CTLR0_REG		0x4E100708
 
+/* Remote DMC memory command registers */
+#define N1SDP_REMOTE_DMC0_MEMC_CMD_REG		PLAT_ARM_REMOTE_CHIP_OFFSET +\
+							N1SDP_DMC0_MEMC_CMD_REG
+#define N1SDP_REMOTE_DMC1_MEMC_CMD_REG		PLAT_ARM_REMOTE_CHIP_OFFSET +\
+							N1SDP_DMC1_MEMC_CMD_REG
+
+/* Remote DMC ERR0CTLR0 registers */
+#define N1SDP_REMOTE_DMC0_ERR0CTLR0_REG		PLAT_ARM_REMOTE_CHIP_OFFSET +\
+							N1SDP_DMC0_ERR0CTLR0_REG
+#define N1SDP_REMOTE_DMC1_ERR0CTLR0_REG		PLAT_ARM_REMOTE_CHIP_OFFSET +\
+							N1SDP_DMC1_ERR0CTLR0_REG
+
 /* DMC memory commands */
 #define N1SDP_DMC_MEMC_CMD_CONFIG		0
 #define N1SDP_DMC_MEMC_CMD_READY		3

plat/arm/board/n1sdp/n1sdp_plat.c

 /*
- * Copyright (c) 2018, ARM Limited and Contributors. All rights reserved.
+ * Copyright (c) 2018-2019, ARM Limited and Contributors. All rights reserved.
  *
  * SPDX-License-Identifier: BSD-3-Clause
  */
@@ -25,6 +25,9 @@ const mmap_region_t plat_arm_mmap[] = {
 	N1SDP_MAP_NS_SRAM,
 	ARM_MAP_DRAM1,
 	ARM_MAP_DRAM2,
+	N1SDP_MAP_REMOTE_DEVICE,
+	N1SDP_MAP_REMOTE_DRAM1,
+	N1SDP_MAP_REMOTE_DRAM2,
 	{0}
 };
 

plat/arm/board/n1sdp/n1sdp_topology.c

 /*
- * Copyright (c) 2018, ARM Limited and Contributors. All rights reserved.
+ * Copyright (c) 2018-2019, ARM Limited and Contributors. All rights reserved.
  *
  * SPDX-License-Identifier: BSD-3-Clause
  */
@@ -19,7 +19,11 @@ typedef struct n1sdp_topology {
  * indices returned by plat_core_pos_by_mpidr().
  */
 const unsigned char n1sdp_pd_tree_desc[] = {
+	PLAT_N1SDP_CHIP_COUNT,
 	PLAT_ARM_CLUSTER_COUNT,
+	PLAT_ARM_CLUSTER_COUNT,
+	N1SDP_MAX_CPUS_PER_CLUSTER,
+	N1SDP_MAX_CPUS_PER_CLUSTER,
 	N1SDP_MAX_CPUS_PER_CLUSTER,
 	N1SDP_MAX_CPUS_PER_CLUSTER
 };
@@ -52,4 +56,4 @@ unsigned int plat_arm_get_cluster_core_count(u_register_t mpidr)
  * to the SCMI power domain ID implemented by SCP.
  ******************************************************************************/
 const uint32_t plat_css_core_pos_to_scmi_dmn_id_map[PLATFORM_CORE_COUNT] = {
-	0, 1, 2, 3};
+	0, 1, 2, 3, 4, 5, 6, 7};