nonlpae_tables.c

/*
 * Copyright (c) 2016-2017, Linaro Limited. All rights reserved.
 * Copyright (c) 2014-2020, Arm Limited. All rights reserved.
 * Copyright (c) 2014, STMicroelectronics International N.V.
 * All rights reserved.
 *
 * SPDX-License-Identifier: BSD-3-Clause
 */

#include <assert.h>
#include <stdio.h>
#include <string.h>

#include <platform_def.h>

#include <arch.h>
#include <arch_helpers.h>
#include <common/debug.h>
#include <lib/cassert.h>
#include <lib/utils.h>
#include <lib/xlat_tables/xlat_tables.h>

#include "../xlat_tables_private.h"

#ifdef ARMV7_SUPPORTS_LARGE_PAGE_ADDRESSING
#error "ARMV7_SUPPORTS_LARGE_PAGE_ADDRESSING flag is set. \
This module is to be used when LPAE is not supported"
#endif

CASSERT(PLAT_VIRT_ADDR_SPACE_SIZE == (1ULL << 32), invalid_vaddr_space_size);
CASSERT(PLAT_PHY_ADDR_SPACE_SIZE == (1ULL << 32), invalid_paddr_space_size);

#define MMU32B_UNSET_DESC	~0UL
#define MMU32B_INVALID_DESC	0UL

#define MT_UNKNOWN	~0U

/*
 * MMU related values
 */

/* Sharable */
#define MMU32B_TTB_S           (1U << 1)

/* Not Outer Sharable */
#define MMU32B_TTB_NOS         (1U << 5)

/* Normal memory, Inner Non-cacheable */
#define MMU32B_TTB_IRGN_NC     0U

/* Normal memory, Inner Write-Back Write-Allocate Cacheable */
#define MMU32B_TTB_IRGN_WBWA   (1U << 6)

/* Normal memory, Inner Write-Through Cacheable */
#define MMU32B_TTB_IRGN_WT     1U

/* Normal memory, Inner Write-Back no Write-Allocate Cacheable */
#define MMU32B_TTB_IRGN_WB     (1U | (1U << 6))

/* Normal memory, Outer Write-Back Write-Allocate Cacheable */
#define MMU32B_TTB_RNG_WBWA    (1U << 3)

#define MMU32B_DEFAULT_ATTRS \
		(MMU32B_TTB_S | MMU32B_TTB_NOS | \
		 MMU32B_TTB_IRGN_WBWA | MMU32B_TTB_RNG_WBWA)

/* armv7 memory mapping attributes: section mapping */
#define SECTION_SECURE			(0U << 19)
#define SECTION_NOTSECURE		(1U << 19)
#define SECTION_SHARED			(1U << 16)
#define SECTION_NOTGLOBAL		(1U << 17)
#define SECTION_ACCESS_FLAG		(1U << 10)
#define SECTION_UNPRIV			(1U << 11)
#define SECTION_RO			(1U << 15)
#define SECTION_TEX(tex)		((((tex) >> 2) << 12) | \
					((((tex) >> 1) & 0x1) << 3) | \
					(((tex) & 0x1) << 2))
#define SECTION_DEVICE			SECTION_TEX(MMU32B_ATTR_DEVICE_INDEX)
#define SECTION_NORMAL			SECTION_TEX(MMU32B_ATTR_DEVICE_INDEX)
#define SECTION_NORMAL_CACHED		\
				SECTION_TEX(MMU32B_ATTR_IWBWA_OWBWA_INDEX)

#define SECTION_XN			(1U << 4)
#define SECTION_PXN			(1U << 0)
#define SECTION_SECTION			(2U << 0)

#define SECTION_PT_NOTSECURE		(1U << 3)
#define SECTION_PT_PT			(1U << 0)

#define SMALL_PAGE_SMALL_PAGE		(1U << 1)
#define SMALL_PAGE_SHARED		(1U << 10)
#define SMALL_PAGE_NOTGLOBAL		(1U << 11)
#define SMALL_PAGE_TEX(tex)		((((tex) >> 2) << 6) | \
					((((tex) >> 1) & 0x1) << 3) | \
					(((tex) & 0x1) << 2))
#define SMALL_PAGE_DEVICE		\
				SMALL_PAGE_TEX(MMU32B_ATTR_DEVICE_INDEX)
#define SMALL_PAGE_NORMAL		\
				SMALL_PAGE_TEX(MMU32B_ATTR_DEVICE_INDEX)
#define SMALL_PAGE_NORMAL_CACHED	\
				SMALL_PAGE_TEX(MMU32B_ATTR_IWBWA_OWBWA_INDEX)
#define SMALL_PAGE_ACCESS_FLAG		(1U << 4)
#define SMALL_PAGE_UNPRIV		(1U << 5)
#define SMALL_PAGE_RO			(1U << 9)
#define SMALL_PAGE_XN			(1U << 0)

/* The TEX, C and B bits concatenated */
#define MMU32B_ATTR_DEVICE_INDEX	0U
#define MMU32B_ATTR_IWBWA_OWBWA_INDEX	1U

#define MMU32B_PRRR_IDX(idx, tr, nos)	(((tr) << (2 * (idx))) | \
					 ((uint32_t)(nos) << ((idx) + 24)))
#define MMU32B_NMRR_IDX(idx, ir, or)	(((ir) << (2 * (idx))) | \
					 ((uint32_t)(or) << (2 * (idx) + 16)))
#define MMU32B_PRRR_DS0			(1U << 16)
#define MMU32B_PRRR_DS1			(1U << 17)
#define MMU32B_PRRR_NS0			(1U << 18)
#define MMU32B_PRRR_NS1			(1U << 19)

#define DACR_DOMAIN(num, perm)		((perm) << ((num) * 2))
#define DACR_DOMAIN_PERM_NO_ACCESS	0U
#define DACR_DOMAIN_PERM_CLIENT		1U
#define DACR_DOMAIN_PERM_MANAGER	3U

#define NUM_1MB_IN_4GB		(1UL << 12)
#define NUM_4K_IN_1MB		(1UL << 8)

#define ONE_MB_SHIFT		20

/* mmu 32b integration */
#define MMU32B_L1_TABLE_SIZE		(NUM_1MB_IN_4GB * 4)
#define MMU32B_L2_TABLE_SIZE		(NUM_4K_IN_1MB * 4)
#define MMU32B_L1_TABLE_ALIGN		(1U << 14)
#define MMU32B_L2_TABLE_ALIGN		(1U << 10)

static unsigned int next_xlat;
static unsigned long long xlat_max_pa;
static uintptr_t xlat_max_va;

static uint32_t mmu_l1_base[NUM_1MB_IN_4GB]
	__aligned(MMU32B_L1_TABLE_ALIGN) __attribute__((section("xlat_table")));

static uint32_t mmu_l2_base[MAX_XLAT_TABLES][NUM_4K_IN_1MB]
	__aligned(MMU32B_L2_TABLE_ALIGN) __attribute__((section("xlat_table")));

/*
 * Array of all memory regions stored in order of ascending base address.
 * The list is terminated by the first entry with size == 0.
 */
static mmap_region_t mmap[MAX_MMAP_REGIONS + 1];

void print_mmap(void)
{
#if LOG_LEVEL >= LOG_LEVEL_VERBOSE
	mmap_region_t *mm = mmap;

	printf("init xlat - l1:%p  l2:%p (%d)\n",
		    (void *)mmu_l1_base, (void *)mmu_l2_base, MAX_XLAT_TABLES);
	printf("mmap:\n");
	while (mm->size) {
		printf(" VA:%p  PA:0x%llx  size:0x%zx  attr:0x%x\n",
				(void *)mm->base_va, mm->base_pa,
				mm->size, mm->attr);
		++mm;
	};
	printf("\n");
#endif
}

void mmap_add(const mmap_region_t *mm)
{
	const mmap_region_t *mm_cursor = mm;

	while ((mm_cursor->size != 0U) || (mm_cursor->attr != 0U)) {
		mmap_add_region(mm_cursor->base_pa, mm_cursor->base_va,
				mm_cursor->size, mm_cursor->attr);
		mm_cursor++;
	}
}

void mmap_add_region(unsigned long long base_pa, uintptr_t base_va,
		     size_t size, unsigned int attr)
{
	mmap_region_t *mm = mmap;
	const mmap_region_t *mm_last = mm + ARRAY_SIZE(mmap) - 1U;
	unsigned long long end_pa = base_pa + size - 1U;
	uintptr_t end_va = base_va + size - 1U;

	assert(IS_PAGE_ALIGNED(base_pa));
	assert(IS_PAGE_ALIGNED(base_va));
	assert(IS_PAGE_ALIGNED(size));

	if (size == 0U) {
		return;
	}

	assert(base_pa < end_pa); /* Check for overflows */
	assert(base_va < end_va);

	assert((base_va + (uintptr_t)size - (uintptr_t)1) <=
					(PLAT_VIRT_ADDR_SPACE_SIZE - 1U));
	assert((base_pa + (unsigned long long)size - 1ULL) <=
					(PLAT_PHY_ADDR_SPACE_SIZE - 1U));

#if ENABLE_ASSERTIONS

	/* Check for PAs and VAs overlaps with all other regions */
	for (mm = mmap; mm->size; ++mm) {

		uintptr_t mm_end_va = mm->base_va + mm->size - 1U;

		/*
		 * Check if one of the regions is completely inside the other
		 * one.
		 */
		bool fully_overlapped_va =
			((base_va >= mm->base_va) && (end_va <= mm_end_va)) ||
			((mm->base_va >= base_va) && (mm_end_va <= end_va));

		/*
		 * Full VA overlaps are only allowed if both regions are
		 * identity mapped (zero offset) or have the same VA to PA
		 * offset. Also, make sure that it's not the exact same area.
		 */
		if (fully_overlapped_va) {
			assert((mm->base_va - mm->base_pa) ==
			       (base_va - base_pa));
			assert((base_va != mm->base_va) || (size != mm->size));
		} else {
			/*
			 * If the regions do not have fully overlapping VAs,
			 * then they must have fully separated VAs and PAs.
			 * Partial overlaps are not allowed
			 */

			unsigned long long mm_end_pa =
						     mm->base_pa + mm->size - 1;

			bool separated_pa = (end_pa < mm->base_pa) ||
				(base_pa > mm_end_pa);
			bool separated_va = (end_va < mm->base_va) ||
				(base_va > mm_end_va);

			assert(separated_va && separated_pa);
		}
	}

	mm = mmap; /* Restore pointer to the start of the array */

#endif /* ENABLE_ASSERTIONS */

	/* Find correct place in mmap to insert new region */
	while ((mm->base_va < base_va) && (mm->size != 0U)) {
		++mm;
	}

	/*
	 * If a section is contained inside another one with the same base
	 * address, it must be placed after the one it is contained in:
	 *
	 * 1st |-----------------------|
	 * 2nd |------------|
	 * 3rd |------|
	 *
	 * This is required for mmap_region_attr() to get the attributes of the
	 * small region correctly.
	 */
	while ((mm->base_va == base_va) && (mm->size > size)) {
		++mm;
	}

	/* Make room for new region by moving other regions up by one place */
	(void)memmove(mm + 1, mm, (uintptr_t)mm_last - (uintptr_t)mm);

	/* Check we haven't lost the empty sentinal from the end of the array */
	assert(mm_last->size == 0U);

	mm->base_pa = base_pa;
	mm->base_va = base_va;
	mm->size = size;
	mm->attr = attr;

	if (end_pa > xlat_max_pa) {
		xlat_max_pa = end_pa;
	}
	if (end_va > xlat_max_va) {
		xlat_max_va = end_va;
	}
}

/* map all memory as shared/global/domain0/no-usr access */
static uint32_t mmap_desc(unsigned attr, unsigned int addr_pa,
		unsigned int level)
{
	uint32_t desc;

	switch (level) {
	case 1U:
		assert((addr_pa & (MMU32B_L1_TABLE_ALIGN - 1)) == 0U);

		desc = SECTION_SECTION | SECTION_SHARED;

		desc |= (attr & MT_NS) != 0U ? SECTION_NOTSECURE : 0U;

		desc |= SECTION_ACCESS_FLAG;
		desc |= (attr & MT_RW) != 0U ? 0U : SECTION_RO;

		desc |= (attr & MT_MEMORY) != 0U ?
			SECTION_NORMAL_CACHED : SECTION_DEVICE;

		if (((attr & MT_RW) != 0U) || ((attr & MT_MEMORY) == 0U)) {
			desc |= SECTION_XN;
		}
		break;
	case 2U:
		assert((addr_pa & (MMU32B_L2_TABLE_ALIGN - 1)) == 0U);

		desc = SMALL_PAGE_SMALL_PAGE | SMALL_PAGE_SHARED;

		desc |= SMALL_PAGE_ACCESS_FLAG;
		desc |= (attr & MT_RW) != 0U ? 0U : SMALL_PAGE_RO;

		desc |= (attr & MT_MEMORY) != 0U ?
			SMALL_PAGE_NORMAL_CACHED : SMALL_PAGE_DEVICE;

		if (((attr & MT_RW) != 0U) || ((attr & MT_MEMORY) == 0U)) {
			desc |= SMALL_PAGE_XN;
		}
		break;
	default:
		panic();
	}
#if LOG_LEVEL >= LOG_LEVEL_VERBOSE
	/* dump only the non-lpae level 2 tables */
	if (level == 2U) {
		printf(attr & MT_MEMORY ? "MEM" : "dev");
		printf(attr & MT_RW ? "-rw" : "-RO");
		printf(attr & MT_NS ? "-NS" : "-S");
	}
#endif
	return desc | addr_pa;
}

static unsigned int mmap_region_attr(const mmap_region_t *mm, uintptr_t base_va,
				     size_t size, unsigned int *attr)
{
	/* Don't assume that the area is contained in the first region */
	unsigned int ret = MT_UNKNOWN;

	/*
	 * Get attributes from last (innermost) region that contains the
	 * requested area. Don't stop as soon as one region doesn't contain it
	 * because there may be other internal regions that contain this area:
	 *
	 * |-----------------------------1-----------------------------|
	 * |----2----|     |-------3-------|    |----5----|
	 *                   |--4--|
	 *
	 *                   |---| <- Area we want the attributes of.
	 *
	 * In this example, the area is contained in regions 1, 3 and 4 but not
	 * in region 2. The loop shouldn't stop at region 2 as inner regions
	 * have priority over outer regions, it should stop at region 5.
	 */
	for ( ; ; ++mm) {

		if (mm->size == 0U) {
			return ret; /* Reached end of list */
		}

		if (mm->base_va > (base_va + size - 1U)) {
			return ret; /* Next region is after area so end */
		}

		if ((mm->base_va + mm->size - 1U) < base_va) {
			continue; /* Next region has already been overtaken */
		}

		if ((ret == 0U) && (mm->attr == *attr)) {
			continue; /* Region doesn't override attribs so skip */
		}

		if ((mm->base_va > base_va) ||
			((mm->base_va + mm->size - 1U) <
					(base_va + size - 1U))) {
			return MT_UNKNOWN; /* Region doesn't fully cover area */
		}

		*attr = mm->attr;
		ret = 0U;
	}
}

static mmap_region_t *init_xlation_table_inner(mmap_region_t *mm,
						unsigned int base_va,
						uint32_t *table,
						unsigned int level)
{
	unsigned int level_size_shift = (level == 1U) ?
					ONE_MB_SHIFT : FOUR_KB_SHIFT;
	unsigned int level_size = 1U << level_size_shift;
	unsigned int level_index_mask = (level == 1U) ?
					(NUM_1MB_IN_4GB - 1) << ONE_MB_SHIFT :
					(NUM_4K_IN_1MB - 1) << FOUR_KB_SHIFT;

	assert((level == 1U) || (level == 2U));

	VERBOSE("init xlat table at %p (level%1u)\n", (void *)table, level);

	do  {
		uint32_t desc = MMU32B_UNSET_DESC;

		if (mm->base_va + mm->size <= base_va) {
			/* Area now after the region so skip it */
			++mm;
			continue;
		}
#if LOG_LEVEL >= LOG_LEVEL_VERBOSE
		/* dump only non-lpae level 2 tables content */
		if (level == 2U) {
			printf("      0x%lx %x " + 6 - 2 * level,
						base_va, level_size);
		}
#endif
		if (mm->base_va >= base_va + level_size) {
			/* Next region is after area so nothing to map yet */
			desc = MMU32B_INVALID_DESC;
		} else if ((mm->base_va <= base_va) &&
				(mm->base_va + mm->size) >=
				(base_va + level_size)) {
			/* Next region covers all of area */
			unsigned int attr = mm->attr;
			unsigned int r = mmap_region_attr(mm, base_va,
							  level_size, &attr);

			if (r == 0U) {
				desc = mmap_desc(attr,
					base_va - mm->base_va + mm->base_pa,
					level);
			}
		}

		if (desc == MMU32B_UNSET_DESC) {
			uintptr_t xlat_table;

			/*
			 * Area not covered by a region so need finer table
			 * Reuse next level table if any (assert attrib matching).
			 * Otherwise allocate a xlat table.
			 */
			if (*table) {
				assert((*table & 3) == SECTION_PT_PT);
				assert(((*table & SECTION_PT_NOTSECURE) == 0U)
						== ((mm->attr & MT_NS) == 0U));

				xlat_table = (*table) &
						~(MMU32B_L1_TABLE_ALIGN - 1);
				desc = *table;
			} else {
				xlat_table = (uintptr_t)mmu_l2_base +
					next_xlat * MMU32B_L2_TABLE_SIZE;
				next_xlat++;
				assert(next_xlat <= MAX_XLAT_TABLES);
				(void)memset((char *)xlat_table, 0,
					MMU32B_L2_TABLE_SIZE);

				desc = xlat_table | SECTION_PT_PT;
				desc |= (mm->attr & MT_NS) != 0U ?
					SECTION_PT_NOTSECURE : 0;
			}
			/* Recurse to fill in new table */
			mm = init_xlation_table_inner(mm, base_va,
						(uint32_t *)xlat_table,
						level + 1);
		}
#if LOG_LEVEL >= LOG_LEVEL_VERBOSE
		/* dump only non-lpae level 2 tables content */
		if (level == 2U) {
			printf("\n");
		}
#endif
		*table++ = desc;
		base_va += level_size;
	} while ((mm->size != 0U) && ((base_va & level_index_mask) != 0U));

	return mm;
}

void init_xlat_tables(void)
{
	print_mmap();

	assert(((unsigned int)mmu_l1_base & (MMU32B_L1_TABLE_ALIGN - 1)) == 0U);
	assert(((unsigned int)mmu_l2_base & (MMU32B_L2_TABLE_ALIGN - 1)) == 0U);

	(void)memset(mmu_l1_base, 0, MMU32B_L1_TABLE_SIZE);

	init_xlation_table_inner(mmap, 0, (uint32_t *)mmu_l1_base, 1);

	VERBOSE("init xlat - max_va=%p, max_pa=%llx\n",
			(void *)xlat_max_va, xlat_max_pa);
	assert(xlat_max_pa <= (PLAT_VIRT_ADDR_SPACE_SIZE - 1));
}

/*******************************************************************************
 * Function for enabling the MMU in Secure PL1, assuming that the
 * page-tables have already been created.
 ******************************************************************************/
void enable_mmu_svc_mon(unsigned int flags)
{
	unsigned int prrr;
	unsigned int nmrr;
	unsigned int sctlr;

	assert(IS_IN_SECURE());
	assert((read_sctlr() & SCTLR_M_BIT) == 0U);

	/* Enable Access flag (simplified access permissions) and TEX remap */
	write_sctlr(read_sctlr() | SCTLR_AFE_BIT | SCTLR_TRE_BIT);

	prrr = MMU32B_PRRR_IDX(MMU32B_ATTR_DEVICE_INDEX, 1, 0) \
			| MMU32B_PRRR_IDX(MMU32B_ATTR_IWBWA_OWBWA_INDEX, 2, 1);
	nmrr = MMU32B_NMRR_IDX(MMU32B_ATTR_DEVICE_INDEX, 0, 0) \
			| MMU32B_NMRR_IDX(MMU32B_ATTR_IWBWA_OWBWA_INDEX, 1, 1);

	prrr |= MMU32B_PRRR_NS1 | MMU32B_PRRR_DS1;

	write_prrr(prrr);
	write_nmrr(nmrr);

	/* Program Domain access control register: domain 0 only */
	write_dacr(DACR_DOMAIN(0, DACR_DOMAIN_PERM_CLIENT));

	/* Invalidate TLBs at the current exception level */
	tlbiall();

	/* set MMU base xlat table entry (use only TTBR0) */
	write_ttbr0((uint32_t)mmu_l1_base | MMU32B_DEFAULT_ATTRS);
	write_ttbr1(0U);

	/*
	 * Ensure all translation table writes have drained
	 * into memory, the TLB invalidation is complete,
	 * and translation register writes are committed
	 * before enabling the MMU
	 */
	dsb();
	isb();

	sctlr = read_sctlr();
	sctlr |= SCTLR_M_BIT;
#ifdef ARMV7_SUPPORTS_VIRTUALIZATION
	sctlr |= SCTLR_WXN_BIT;
#endif

	if ((flags & DISABLE_DCACHE) != 0U) {
		sctlr &= ~SCTLR_C_BIT;
	} else {
		sctlr |= SCTLR_C_BIT;
	}

	write_sctlr(sctlr);

	/* Ensure the MMU enable takes effect immediately */
	isb();
}