/* * Copyright (c) 2017-2018, ARM Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include "xlat_tables_private.h" /* * Each platform can define the size of its physical and virtual address spaces. * If the platform hasn't defined one or both of them, default to * ADDR_SPACE_SIZE. The latter is deprecated, though. */ #if ERROR_DEPRECATED # ifdef ADDR_SPACE_SIZE # error "ADDR_SPACE_SIZE is deprecated. Use PLAT_xxx_ADDR_SPACE_SIZE instead." # endif #elif defined(ADDR_SPACE_SIZE) # ifndef PLAT_PHY_ADDR_SPACE_SIZE # define PLAT_PHY_ADDR_SPACE_SIZE ADDR_SPACE_SIZE # endif # ifndef PLAT_VIRT_ADDR_SPACE_SIZE # define PLAT_VIRT_ADDR_SPACE_SIZE ADDR_SPACE_SIZE # endif #endif /* * Allocate and initialise the default translation context for the BL image * currently executing. */ REGISTER_XLAT_CONTEXT(tf, MAX_MMAP_REGIONS, MAX_XLAT_TABLES, PLAT_VIRT_ADDR_SPACE_SIZE, PLAT_PHY_ADDR_SPACE_SIZE); #if PLAT_XLAT_TABLES_DYNAMIC /* * The following functions assume that they will be called using subtables only. * The base table can't be unmapped, so it is not needed to do any special * handling for it. */ /* * Returns the index of the array corresponding to the specified translation * table. */ static int xlat_table_get_index(xlat_ctx_t *ctx, const uint64_t *table) { for (unsigned int i = 0; i < ctx->tables_num; i++) if (ctx->tables[i] == table) return i; /* * Maybe we were asked to get the index of the base level table, which * should never happen. */ assert(0); return -1; } /* Returns a pointer to an empty translation table. */ static uint64_t *xlat_table_get_empty(xlat_ctx_t *ctx) { for (unsigned int i = 0; i < ctx->tables_num; i++) if (ctx->tables_mapped_regions[i] == 0) return ctx->tables[i]; return NULL; } /* Increments region count for a given table. */ static void xlat_table_inc_regions_count(xlat_ctx_t *ctx, const uint64_t *table) { ctx->tables_mapped_regions[xlat_table_get_index(ctx, table)]++; } /* Decrements region count for a given table. */ static void xlat_table_dec_regions_count(xlat_ctx_t *ctx, const uint64_t *table) { ctx->tables_mapped_regions[xlat_table_get_index(ctx, table)]--; } /* Returns 0 if the speficied table isn't empty, otherwise 1. */ static int xlat_table_is_empty(xlat_ctx_t *ctx, const uint64_t *table) { return !ctx->tables_mapped_regions[xlat_table_get_index(ctx, table)]; } #else /* PLAT_XLAT_TABLES_DYNAMIC */ /* Returns a pointer to the first empty translation table. */ static uint64_t *xlat_table_get_empty(xlat_ctx_t *ctx) { assert(ctx->next_table < ctx->tables_num); return ctx->tables[ctx->next_table++]; } #endif /* PLAT_XLAT_TABLES_DYNAMIC */ /* * Returns a block/page table descriptor for the given level and attributes. */ static uint64_t xlat_desc(const xlat_ctx_t *ctx, uint32_t attr, unsigned long long addr_pa, int level) { uint64_t desc; int mem_type; /* Make sure that the granularity is fine enough to map this address. */ assert((addr_pa & XLAT_BLOCK_MASK(level)) == 0); desc = addr_pa; /* * There are different translation table descriptors for level 3 and the * rest. */ desc |= (level == XLAT_TABLE_LEVEL_MAX) ? PAGE_DESC : BLOCK_DESC; /* * Always set the access flag, as TF doesn't manage access flag faults. * Deduce other fields of the descriptor based on the MT_NS and MT_RW * memory region attributes. */ desc |= LOWER_ATTRS(ACCESS_FLAG); desc |= (attr & MT_NS) ? LOWER_ATTRS(NS) : 0; desc |= (attr & MT_RW) ? LOWER_ATTRS(AP_RW) : LOWER_ATTRS(AP_RO); /* * Do not allow unprivileged access when the mapping is for a privileged * EL. For translation regimes that do not have mappings for access for * lower exception levels, set AP[2] to AP_NO_ACCESS_UNPRIVILEGED. */ if (ctx->xlat_regime == EL1_EL0_REGIME) { if (attr & MT_USER) { /* EL0 mapping requested, so we give User access */ desc |= LOWER_ATTRS(AP_ACCESS_UNPRIVILEGED); } else { /* EL1 mapping requested, no User access granted */ desc |= LOWER_ATTRS(AP_NO_ACCESS_UNPRIVILEGED); } } else { assert(ctx->xlat_regime == EL3_REGIME); desc |= LOWER_ATTRS(AP_ONE_VA_RANGE_RES1); } /* * Deduce shareability domain and executability of the memory region * from the memory type of the attributes (MT_TYPE). * * Data accesses to device memory and non-cacheable normal memory are * coherent for all observers in the system, and correspondingly are * always treated as being Outer Shareable. Therefore, for these 2 types * of memory, it is not strictly needed to set the shareability field * in the translation tables. */ mem_type = MT_TYPE(attr); if (mem_type == MT_DEVICE) { desc |= LOWER_ATTRS(ATTR_DEVICE_INDEX | OSH); /* * Always map device memory as execute-never. * This is to avoid the possibility of a speculative instruction * fetch, which could be an issue if this memory region * corresponds to a read-sensitive peripheral. */ desc |= xlat_arch_regime_get_xn_desc(ctx->xlat_regime); } else { /* Normal memory */ /* * Always map read-write normal memory as execute-never. * (Trusted Firmware doesn't self-modify its code, therefore * R/W memory is reserved for data storage, which must not be * executable.) * Note that setting the XN bit here is for consistency only. * The function that enables the MMU sets the SCTLR_ELx.WXN bit, * which makes any writable memory region to be treated as * execute-never, regardless of the value of the XN bit in the * translation table. * * For read-only memory, rely on the MT_EXECUTE/MT_EXECUTE_NEVER * attribute to figure out the value of the XN bit. The actual * XN bit(s) to set in the descriptor depends on the context's * translation regime and the policy applied in * xlat_arch_regime_get_xn_desc(). */ if ((attr & MT_RW) || (attr & MT_EXECUTE_NEVER)) { desc |= xlat_arch_regime_get_xn_desc(ctx->xlat_regime); } if (mem_type == MT_MEMORY) { desc |= LOWER_ATTRS(ATTR_IWBWA_OWBWA_NTR_INDEX | ISH); } else { assert(mem_type == MT_NON_CACHEABLE); desc |= LOWER_ATTRS(ATTR_NON_CACHEABLE_INDEX | OSH); } } return desc; } /* * Enumeration of actions that can be made when mapping table entries depending * on the previous value in that entry and information about the region being * mapped. */ typedef enum { /* Do nothing */ ACTION_NONE, /* Write a block (or page, if in level 3) entry. */ ACTION_WRITE_BLOCK_ENTRY, /* * Create a new table and write a table entry pointing to it. Recurse * into it for further processing. */ ACTION_CREATE_NEW_TABLE, /* * There is a table descriptor in this entry, read it and recurse into * that table for further processing. */ ACTION_RECURSE_INTO_TABLE, } action_t; #if PLAT_XLAT_TABLES_DYNAMIC /* * Recursive function that writes to the translation tables and unmaps the * specified region. */ static void xlat_tables_unmap_region(xlat_ctx_t *ctx, mmap_region_t *mm, const uintptr_t table_base_va, uint64_t *const table_base, const int table_entries, const unsigned int level) { assert(level >= ctx->base_level && level <= XLAT_TABLE_LEVEL_MAX); uint64_t *subtable; uint64_t desc; uintptr_t table_idx_va; uintptr_t table_idx_end_va; /* End VA of this entry */ uintptr_t region_end_va = mm->base_va + mm->size - 1; int table_idx; if (mm->base_va > table_base_va) { /* Find the first index of the table affected by the region. */ table_idx_va = mm->base_va & ~XLAT_BLOCK_MASK(level); table_idx = (table_idx_va - table_base_va) >> XLAT_ADDR_SHIFT(level); assert(table_idx < table_entries); } else { /* Start from the beginning of the table. */ table_idx_va = table_base_va; table_idx = 0; } while (table_idx < table_entries) { table_idx_end_va = table_idx_va + XLAT_BLOCK_SIZE(level) - 1; desc = table_base[table_idx]; uint64_t desc_type = desc & DESC_MASK; action_t action = ACTION_NONE; if ((mm->base_va <= table_idx_va) && (region_end_va >= table_idx_end_va)) { /* Region covers all block */ if (level == 3) { /* * Last level, only page descriptors allowed, * erase it. */ assert(desc_type == PAGE_DESC); action = ACTION_WRITE_BLOCK_ENTRY; } else { /* * Other levels can have table descriptors. If * so, recurse into it and erase descriptors * inside it as needed. If there is a block * descriptor, just erase it. If an invalid * descriptor is found, this table isn't * actually mapped, which shouldn't happen. */ if (desc_type == TABLE_DESC) { action = ACTION_RECURSE_INTO_TABLE; } else { assert(desc_type == BLOCK_DESC); action = ACTION_WRITE_BLOCK_ENTRY; } } } else if ((mm->base_va <= table_idx_end_va) || (region_end_va >= table_idx_va)) { /* * Region partially covers block. * * It can't happen in level 3. * * There must be a table descriptor here, if not there * was a problem when mapping the region. */ assert(level < 3); assert(desc_type == TABLE_DESC); action = ACTION_RECURSE_INTO_TABLE; } if (action == ACTION_WRITE_BLOCK_ENTRY) { table_base[table_idx] = INVALID_DESC; xlat_arch_tlbi_va_regime(table_idx_va, ctx->xlat_regime); } else if (action == ACTION_RECURSE_INTO_TABLE) { subtable = (uint64_t *)(uintptr_t)(desc & TABLE_ADDR_MASK); /* Recurse to write into subtable */ xlat_tables_unmap_region(ctx, mm, table_idx_va, subtable, XLAT_TABLE_ENTRIES, level + 1); /* * If the subtable is now empty, remove its reference. */ if (xlat_table_is_empty(ctx, subtable)) { table_base[table_idx] = INVALID_DESC; xlat_arch_tlbi_va_regime(table_idx_va, ctx->xlat_regime); } } else { assert(action == ACTION_NONE); } table_idx++; table_idx_va += XLAT_BLOCK_SIZE(level); /* If reached the end of the region, exit */ if (region_end_va <= table_idx_va) break; } if (level > ctx->base_level) xlat_table_dec_regions_count(ctx, table_base); } #endif /* PLAT_XLAT_TABLES_DYNAMIC */ /* * From the given arguments, it decides which action to take when mapping the * specified region. */ static action_t xlat_tables_map_region_action(const mmap_region_t *mm, const int desc_type, const unsigned long long dest_pa, const uintptr_t table_entry_base_va, const unsigned int level) { uintptr_t mm_end_va = mm->base_va + mm->size - 1; uintptr_t table_entry_end_va = table_entry_base_va + XLAT_BLOCK_SIZE(level) - 1; /* * The descriptor types allowed depend on the current table level. */ if ((mm->base_va <= table_entry_base_va) && (mm_end_va >= table_entry_end_va)) { /* * Table entry is covered by region * -------------------------------- * * This means that this table entry can describe the whole * translation with this granularity in principle. */ if (level == 3) { /* * Last level, only page descriptors are allowed. */ if (desc_type == PAGE_DESC) { /* * There's another region mapped here, don't * overwrite. */ return ACTION_NONE; } else { assert(desc_type == INVALID_DESC); return ACTION_WRITE_BLOCK_ENTRY; } } else { /* * Other levels. Table descriptors are allowed. Block * descriptors too, but they have some limitations. */ if (desc_type == TABLE_DESC) { /* There's already a table, recurse into it. */ return ACTION_RECURSE_INTO_TABLE; } else if (desc_type == INVALID_DESC) { /* * There's nothing mapped here, create a new * entry. * * Check if the destination granularity allows * us to use a block descriptor or we need a * finer table for it. * * Also, check if the current level allows block * descriptors. If not, create a table instead. */ if ((dest_pa & XLAT_BLOCK_MASK(level)) || (level < MIN_LVL_BLOCK_DESC) || (mm->granularity < XLAT_BLOCK_SIZE(level))) return ACTION_CREATE_NEW_TABLE; else return ACTION_WRITE_BLOCK_ENTRY; } else { /* * There's another region mapped here, don't * overwrite. */ assert(desc_type == BLOCK_DESC); return ACTION_NONE; } } } else if ((mm->base_va <= table_entry_end_va) || (mm_end_va >= table_entry_base_va)) { /* * Region partially covers table entry * ----------------------------------- * * This means that this table entry can't describe the whole * translation, a finer table is needed. * There cannot be partial block overlaps in level 3. If that * happens, some of the preliminary checks when adding the * mmap region failed to detect that PA and VA must at least be * aligned to PAGE_SIZE. */ assert(level < 3); if (desc_type == INVALID_DESC) { /* * The block is not fully covered by the region. Create * a new table, recurse into it and try to map the * region with finer granularity. */ return ACTION_CREATE_NEW_TABLE; } else { assert(desc_type == TABLE_DESC); /* * The block is not fully covered by the region, but * there is already a table here. Recurse into it and * try to map with finer granularity. * * PAGE_DESC for level 3 has the same value as * TABLE_DESC, but this code can't run on a level 3 * table because there can't be overlaps in level 3. */ return ACTION_RECURSE_INTO_TABLE; } } /* * This table entry is outside of the region specified in the arguments, * don't write anything to it. */ return ACTION_NONE; } /* * Recursive function that writes to the translation tables and maps the * specified region. On success, it returns the VA of the last byte that was * succesfully mapped. On error, it returns the VA of the next entry that * should have been mapped. */ static uintptr_t xlat_tables_map_region(xlat_ctx_t *ctx, mmap_region_t *mm, const uintptr_t table_base_va, uint64_t *const table_base, const int table_entries, const unsigned int level) { assert(level >= ctx->base_level && level <= XLAT_TABLE_LEVEL_MAX); uintptr_t mm_end_va = mm->base_va + mm->size - 1; uintptr_t table_idx_va; unsigned long long table_idx_pa; uint64_t *subtable; uint64_t desc; int table_idx; if (mm->base_va > table_base_va) { /* Find the first index of the table affected by the region. */ table_idx_va = mm->base_va & ~XLAT_BLOCK_MASK(level); table_idx = (table_idx_va - table_base_va) >> XLAT_ADDR_SHIFT(level); assert(table_idx < table_entries); } else { /* Start from the beginning of the table. */ table_idx_va = table_base_va; table_idx = 0; } #if PLAT_XLAT_TABLES_DYNAMIC if (level > ctx->base_level) xlat_table_inc_regions_count(ctx, table_base); #endif while (table_idx < table_entries) { desc = table_base[table_idx]; table_idx_pa = mm->base_pa + table_idx_va - mm->base_va; action_t action = xlat_tables_map_region_action(mm, desc & DESC_MASK, table_idx_pa, table_idx_va, level); if (action == ACTION_WRITE_BLOCK_ENTRY) { table_base[table_idx] = xlat_desc(ctx, (uint32_t)mm->attr, table_idx_pa, level); } else if (action == ACTION_CREATE_NEW_TABLE) { subtable = xlat_table_get_empty(ctx); if (subtable == NULL) { /* Not enough free tables to map this region */ return table_idx_va; } /* Point to new subtable from this one. */ table_base[table_idx] = TABLE_DESC | (unsigned long)subtable; /* Recurse to write into subtable */ uintptr_t end_va = xlat_tables_map_region(ctx, mm, table_idx_va, subtable, XLAT_TABLE_ENTRIES, level + 1); if (end_va != table_idx_va + XLAT_BLOCK_SIZE(level) - 1) return end_va; } else if (action == ACTION_RECURSE_INTO_TABLE) { subtable = (uint64_t *)(uintptr_t)(desc & TABLE_ADDR_MASK); /* Recurse to write into subtable */ uintptr_t end_va = xlat_tables_map_region(ctx, mm, table_idx_va, subtable, XLAT_TABLE_ENTRIES, level + 1); if (end_va != table_idx_va + XLAT_BLOCK_SIZE(level) - 1) return end_va; } else { assert(action == ACTION_NONE); } table_idx++; table_idx_va += XLAT_BLOCK_SIZE(level); /* If reached the end of the region, exit */ if (mm_end_va <= table_idx_va) break; } return table_idx_va - 1; } void print_mmap(mmap_region_t *const mmap) { #if LOG_LEVEL >= LOG_LEVEL_VERBOSE tf_printf("mmap:\n"); mmap_region_t *mm = mmap; while (mm->size) { tf_printf(" VA:%p PA:0x%llx size:0x%zx attr:0x%x", (void *)mm->base_va, mm->base_pa, mm->size, mm->attr); tf_printf(" granularity:0x%zx\n", mm->granularity); ++mm; }; tf_printf("\n"); #endif } /* * Function that verifies that a region can be mapped. * Returns: * 0: Success, the mapping is allowed. * EINVAL: Invalid values were used as arguments. * ERANGE: The memory limits were surpassed. * ENOMEM: There is not enough memory in the mmap array. * EPERM: Region overlaps another one in an invalid way. */ static int mmap_add_region_check(xlat_ctx_t *ctx, const mmap_region_t *mm) { unsigned long long base_pa = mm->base_pa; uintptr_t base_va = mm->base_va; size_t size = mm->size; size_t granularity = mm->granularity; unsigned long long end_pa = base_pa + size - 1; uintptr_t end_va = base_va + size - 1; if (!IS_PAGE_ALIGNED(base_pa) || !IS_PAGE_ALIGNED(base_va) || !IS_PAGE_ALIGNED(size)) return -EINVAL; if ((granularity != XLAT_BLOCK_SIZE(1)) && (granularity != XLAT_BLOCK_SIZE(2)) && (granularity != XLAT_BLOCK_SIZE(3))) { return -EINVAL; } /* Check for overflows */ if ((base_pa > end_pa) || (base_va > end_va)) return -ERANGE; if ((base_va + (uintptr_t)size - (uintptr_t)1) > ctx->va_max_address) return -ERANGE; if ((base_pa + (unsigned long long)size - 1ULL) > ctx->pa_max_address) return -ERANGE; /* Check that there is space in the ctx->mmap array */ if (ctx->mmap[ctx->mmap_num - 1].size != 0) return -ENOMEM; /* Check for PAs and VAs overlaps with all other regions */ for (mmap_region_t *mm_cursor = ctx->mmap; mm_cursor->size; ++mm_cursor) { uintptr_t mm_cursor_end_va = mm_cursor->base_va + mm_cursor->size - 1; /* * Check if one of the regions is completely inside the other * one. */ int fully_overlapped_va = ((base_va >= mm_cursor->base_va) && (end_va <= mm_cursor_end_va)) || ((mm_cursor->base_va >= base_va) && (mm_cursor_end_va <= end_va)); /* * Full VA overlaps are only allowed if both regions are * identity mapped (zero offset) or have the same VA to PA * offset. Also, make sure that it's not the exact same area. * This can only be done with static regions. */ if (fully_overlapped_va) { #if PLAT_XLAT_TABLES_DYNAMIC if ((mm->attr & MT_DYNAMIC) || (mm_cursor->attr & MT_DYNAMIC)) return -EPERM; #endif /* PLAT_XLAT_TABLES_DYNAMIC */ if ((mm_cursor->base_va - mm_cursor->base_pa) != (base_va - base_pa)) return -EPERM; if ((base_va == mm_cursor->base_va) && (size == mm_cursor->size)) return -EPERM; } else { /* * If the regions do not have fully overlapping VAs, * then they must have fully separated VAs and PAs. * Partial overlaps are not allowed */ unsigned long long mm_cursor_end_pa = mm_cursor->base_pa + mm_cursor->size - 1; int separated_pa = (end_pa < mm_cursor->base_pa) || (base_pa > mm_cursor_end_pa); int separated_va = (end_va < mm_cursor->base_va) || (base_va > mm_cursor_end_va); if (!(separated_va && separated_pa)) return -EPERM; } } return 0; } void mmap_add_region_ctx(xlat_ctx_t *ctx, const mmap_region_t *mm) { mmap_region_t *mm_cursor = ctx->mmap; const mmap_region_t *mm_end = ctx->mmap + ctx->mmap_num; mmap_region_t *mm_last; unsigned long long end_pa = mm->base_pa + mm->size - 1; uintptr_t end_va = mm->base_va + mm->size - 1; int ret; /* Ignore empty regions */ if (!mm->size) return; /* Static regions must be added before initializing the xlat tables. */ assert(!ctx->initialized); ret = mmap_add_region_check(ctx, mm); if (ret != 0) { ERROR("mmap_add_region_check() failed. error %d\n", ret); assert(0); return; } /* * Find correct place in mmap to insert new region. * * 1 - Lower region VA end first. * 2 - Smaller region size first. * * VA 0 0xFF * * 1st |------| * 2nd |------------| * 3rd |------| * 4th |---| * 5th |---| * 6th |----------| * 7th |-------------------------------------| * * This is required for overlapping regions only. It simplifies adding * regions with the loop in xlat_tables_init_internal because the outer * ones won't overwrite block or page descriptors of regions added * previously. * * Overlapping is only allowed for static regions. */ while ((mm_cursor->base_va + mm_cursor->size - 1) < end_va && mm_cursor->size) ++mm_cursor; while ((mm_cursor->base_va + mm_cursor->size - 1 == end_va) && (mm_cursor->size < mm->size)) ++mm_cursor; /* * Find the last entry marker in the mmap */ mm_last = ctx->mmap; while ((mm_last->size != 0U) && (mm_last < mm_end)) { ++mm_last; } /* * Check if we have enough space in the memory mapping table. * This shouldn't happen as we have checked in mmap_add_region_check * that there is free space. */ assert(mm_last->size == 0U); /* Make room for new region by moving other regions up by one place */ memmove(mm_cursor + 1, mm_cursor, (uintptr_t)mm_last - (uintptr_t)mm_cursor); /* * Check we haven't lost the empty sentinel from the end of the array. * This shouldn't happen as we have checked in mmap_add_region_check * that there is free space. */ assert(mm_end->size == 0U); *mm_cursor = *mm; if (end_pa > ctx->max_pa) ctx->max_pa = end_pa; if (end_va > ctx->max_va) ctx->max_va = end_va; } void mmap_add_region(unsigned long long base_pa, uintptr_t base_va, size_t size, mmap_attr_t attr) { mmap_region_t mm = MAP_REGION(base_pa, base_va, size, attr); mmap_add_region_ctx(&tf_xlat_ctx, &mm); } void mmap_add_ctx(xlat_ctx_t *ctx, const mmap_region_t *mm) { while (mm->size) { mmap_add_region_ctx(ctx, mm); mm++; } } void mmap_add(const mmap_region_t *mm) { mmap_add_ctx(&tf_xlat_ctx, mm); } #if PLAT_XLAT_TABLES_DYNAMIC int mmap_add_dynamic_region_ctx(xlat_ctx_t *ctx, mmap_region_t *mm) { mmap_region_t *mm_cursor = ctx->mmap; mmap_region_t *mm_last = mm_cursor + ctx->mmap_num; unsigned long long end_pa = mm->base_pa + mm->size - 1; uintptr_t end_va = mm->base_va + mm->size - 1; int ret; /* Nothing to do */ if (!mm->size) return 0; /* Now this region is a dynamic one */ mm->attr |= MT_DYNAMIC; ret = mmap_add_region_check(ctx, mm); if (ret != 0) return ret; /* * Find the adequate entry in the mmap array in the same way done for * static regions in mmap_add_region_ctx(). */ while ((mm_cursor->base_va + mm_cursor->size - 1) < end_va && mm_cursor->size) ++mm_cursor; while ((mm_cursor->base_va + mm_cursor->size - 1 == end_va) && (mm_cursor->size < mm->size)) ++mm_cursor; /* Make room for new region by moving other regions up by one place */ memmove(mm_cursor + 1, mm_cursor, (uintptr_t)mm_last - (uintptr_t)mm_cursor); /* * Check we haven't lost the empty sentinal from the end of the array. * This shouldn't happen as we have checked in mmap_add_region_check * that there is free space. */ assert(mm_last->size == 0); *mm_cursor = *mm; /* * Update the translation tables if the xlat tables are initialized. If * not, this region will be mapped when they are initialized. */ if (ctx->initialized) { uintptr_t end_va = xlat_tables_map_region(ctx, mm_cursor, 0, ctx->base_table, ctx->base_table_entries, ctx->base_level); /* Failed to map, remove mmap entry, unmap and return error. */ if (end_va != mm_cursor->base_va + mm_cursor->size - 1) { memmove(mm_cursor, mm_cursor + 1, (uintptr_t)mm_last - (uintptr_t)mm_cursor); /* * Check if the mapping function actually managed to map * anything. If not, just return now. */ if (mm->base_va >= end_va) return -ENOMEM; /* * Something went wrong after mapping some table * entries, undo every change done up to this point. */ mmap_region_t unmap_mm = { .base_pa = 0, .base_va = mm->base_va, .size = end_va - mm->base_va, .attr = 0 }; xlat_tables_unmap_region(ctx, &unmap_mm, 0, ctx->base_table, ctx->base_table_entries, ctx->base_level); return -ENOMEM; } /* * Make sure that all entries are written to the memory. There * is no need to invalidate entries when mapping dynamic regions * because new table/block/page descriptors only replace old * invalid descriptors, that aren't TLB cached. */ dsbishst(); } if (end_pa > ctx->max_pa) ctx->max_pa = end_pa; if (end_va > ctx->max_va) ctx->max_va = end_va; return 0; } int mmap_add_dynamic_region(unsigned long long base_pa, uintptr_t base_va, size_t size, mmap_attr_t attr) { mmap_region_t mm = MAP_REGION(base_pa, base_va, size, attr); return mmap_add_dynamic_region_ctx(&tf_xlat_ctx, &mm); } /* * Removes the region with given base Virtual Address and size from the given * context. * * Returns: * 0: Success. * EINVAL: Invalid values were used as arguments (region not found). * EPERM: Tried to remove a static region. */ int mmap_remove_dynamic_region_ctx(xlat_ctx_t *ctx, uintptr_t base_va, size_t size) { mmap_region_t *mm = ctx->mmap; mmap_region_t *mm_last = mm + ctx->mmap_num; int update_max_va_needed = 0; int update_max_pa_needed = 0; /* Check sanity of mmap array. */ assert(mm[ctx->mmap_num].size == 0); while (mm->size) { if ((mm->base_va == base_va) && (mm->size == size)) break; ++mm; } /* Check that the region was found */ if (mm->size == 0) return -EINVAL; /* If the region is static it can't be removed */ if (!(mm->attr & MT_DYNAMIC)) return -EPERM; /* Check if this region is using the top VAs or PAs. */ if ((mm->base_va + mm->size - 1) == ctx->max_va) update_max_va_needed = 1; if ((mm->base_pa + mm->size - 1) == ctx->max_pa) update_max_pa_needed = 1; /* Update the translation tables if needed */ if (ctx->initialized) { xlat_tables_unmap_region(ctx, mm, 0, ctx->base_table, ctx->base_table_entries, ctx->base_level); xlat_arch_tlbi_va_sync(); } /* Remove this region by moving the rest down by one place. */ memmove(mm, mm + 1, (uintptr_t)mm_last - (uintptr_t)mm); /* Check if we need to update the max VAs and PAs */ if (update_max_va_needed) { ctx->max_va = 0; mm = ctx->mmap; while (mm->size) { if ((mm->base_va + mm->size - 1) > ctx->max_va) ctx->max_va = mm->base_va + mm->size - 1; ++mm; } } if (update_max_pa_needed) { ctx->max_pa = 0; mm = ctx->mmap; while (mm->size) { if ((mm->base_pa + mm->size - 1) > ctx->max_pa) ctx->max_pa = mm->base_pa + mm->size - 1; ++mm; } } return 0; } int mmap_remove_dynamic_region(uintptr_t base_va, size_t size) { return mmap_remove_dynamic_region_ctx(&tf_xlat_ctx, base_va, size); } #endif /* PLAT_XLAT_TABLES_DYNAMIC */ #if LOG_LEVEL >= LOG_LEVEL_VERBOSE /* Print the attributes of the specified block descriptor. */ static void xlat_desc_print(const xlat_ctx_t *ctx, uint64_t desc) { int mem_type_index = ATTR_INDEX_GET(desc); xlat_regime_t xlat_regime = ctx->xlat_regime; if (mem_type_index == ATTR_IWBWA_OWBWA_NTR_INDEX) { tf_printf("MEM"); } else if (mem_type_index == ATTR_NON_CACHEABLE_INDEX) { tf_printf("NC"); } else { assert(mem_type_index == ATTR_DEVICE_INDEX); tf_printf("DEV"); } const char *priv_str = "(PRIV)"; const char *user_str = "(USER)"; /* * Showing Privileged vs Unprivileged only makes sense for EL1&0 * mappings */ const char *ro_str = "-RO"; const char *rw_str = "-RW"; const char *no_access_str = "-NOACCESS"; if (xlat_regime == EL3_REGIME) { /* For EL3, the AP[2] bit is all what matters */ tf_printf((desc & LOWER_ATTRS(AP_RO)) ? ro_str : rw_str); } else { const char *ap_str = (desc & LOWER_ATTRS(AP_RO)) ? ro_str : rw_str; tf_printf(ap_str); tf_printf(priv_str); /* * EL0 can only have the same permissions as EL1 or no * permissions at all. */ tf_printf((desc & LOWER_ATTRS(AP_ACCESS_UNPRIVILEGED)) ? ap_str : no_access_str); tf_printf(user_str); } const char *xn_str = "-XN"; const char *exec_str = "-EXEC"; if (xlat_regime == EL3_REGIME) { /* For EL3, the XN bit is all what matters */ tf_printf(LOWER_ATTRS(XN) & desc ? xn_str : exec_str); } else { /* For EL0 and EL1, we need to know who has which rights */ tf_printf(LOWER_ATTRS(PXN) & desc ? xn_str : exec_str); tf_printf(priv_str); tf_printf(LOWER_ATTRS(UXN) & desc ? xn_str : exec_str); tf_printf(user_str); } tf_printf(LOWER_ATTRS(NS) & desc ? "-NS" : "-S"); } static const char * const level_spacers[] = { "[LV0] ", " [LV1] ", " [LV2] ", " [LV3] " }; static const char *invalid_descriptors_ommited = "%s(%d invalid descriptors omitted)\n"; /* * Recursive function that reads the translation tables passed as an argument * and prints their status. */ static void xlat_tables_print_internal(xlat_ctx_t *ctx, const uintptr_t table_base_va, uint64_t *const table_base, const int table_entries, const unsigned int level) { assert(level <= XLAT_TABLE_LEVEL_MAX); uint64_t desc; uintptr_t table_idx_va = table_base_va; int table_idx = 0; size_t level_size = XLAT_BLOCK_SIZE(level); /* * Keep track of how many invalid descriptors are counted in a row. * Whenever multiple invalid descriptors are found, only the first one * is printed, and a line is added to inform about how many descriptors * have been omitted. */ int invalid_row_count = 0; while (table_idx < table_entries) { desc = table_base[table_idx]; if ((desc & DESC_MASK) == INVALID_DESC) { if (invalid_row_count == 0) { tf_printf("%sVA:%p size:0x%zx\n", level_spacers[level], (void *)table_idx_va, level_size); } invalid_row_count++; } else { if (invalid_row_count > 1) { tf_printf(invalid_descriptors_ommited, level_spacers[level], invalid_row_count - 1); } invalid_row_count = 0; /* * Check if this is a table or a block. Tables are only * allowed in levels other than 3, but DESC_PAGE has the * same value as DESC_TABLE, so we need to check. */ if (((desc & DESC_MASK) == TABLE_DESC) && (level < XLAT_TABLE_LEVEL_MAX)) { /* * Do not print any PA for a table descriptor, * as it doesn't directly map physical memory * but instead points to the next translation * table in the translation table walk. */ tf_printf("%sVA:%p size:0x%zx\n", level_spacers[level], (void *)table_idx_va, level_size); uintptr_t addr_inner = desc & TABLE_ADDR_MASK; xlat_tables_print_internal(ctx, table_idx_va, (uint64_t *)addr_inner, XLAT_TABLE_ENTRIES, level + 1); } else { tf_printf("%sVA:%p PA:0x%llx size:0x%zx ", level_spacers[level], (void *)table_idx_va, (unsigned long long)(desc & TABLE_ADDR_MASK), level_size); xlat_desc_print(ctx, desc); tf_printf("\n"); } } table_idx++; table_idx_va += level_size; } if (invalid_row_count > 1) { tf_printf(invalid_descriptors_ommited, level_spacers[level], invalid_row_count - 1); } } #endif /* LOG_LEVEL >= LOG_LEVEL_VERBOSE */ void xlat_tables_print(xlat_ctx_t *ctx) { #if LOG_LEVEL >= LOG_LEVEL_VERBOSE const char *xlat_regime_str; if (ctx->xlat_regime == EL1_EL0_REGIME) { xlat_regime_str = "1&0"; } else { assert(ctx->xlat_regime == EL3_REGIME); xlat_regime_str = "3"; } VERBOSE("Translation tables state:\n"); VERBOSE(" Xlat regime: EL%s\n", xlat_regime_str); VERBOSE(" Max allowed PA: 0x%llx\n", ctx->pa_max_address); VERBOSE(" Max allowed VA: %p\n", (void *) ctx->va_max_address); VERBOSE(" Max mapped PA: 0x%llx\n", ctx->max_pa); VERBOSE(" Max mapped VA: %p\n", (void *) ctx->max_va); VERBOSE(" Initial lookup level: %i\n", ctx->base_level); VERBOSE(" Entries @initial lookup level: %i\n", ctx->base_table_entries); int used_page_tables; #if PLAT_XLAT_TABLES_DYNAMIC used_page_tables = 0; for (unsigned int i = 0; i < ctx->tables_num; ++i) { if (ctx->tables_mapped_regions[i] != 0) ++used_page_tables; } #else used_page_tables = ctx->next_table; #endif VERBOSE(" Used %i sub-tables out of %i (spare: %i)\n", used_page_tables, ctx->tables_num, ctx->tables_num - used_page_tables); xlat_tables_print_internal(ctx, 0, ctx->base_table, ctx->base_table_entries, ctx->base_level); #endif /* LOG_LEVEL >= LOG_LEVEL_VERBOSE */ } void init_xlat_tables_ctx(xlat_ctx_t *ctx) { assert(ctx != NULL); assert(!ctx->initialized); assert(ctx->xlat_regime == EL3_REGIME || ctx->xlat_regime == EL1_EL0_REGIME); assert(!is_mmu_enabled_ctx(ctx)); mmap_region_t *mm = ctx->mmap; print_mmap(mm); /* All tables must be zeroed before mapping any region. */ for (unsigned int i = 0; i < ctx->base_table_entries; i++) ctx->base_table[i] = INVALID_DESC; for (unsigned int j = 0; j < ctx->tables_num; j++) { #if PLAT_XLAT_TABLES_DYNAMIC ctx->tables_mapped_regions[j] = 0; #endif for (unsigned int i = 0; i < XLAT_TABLE_ENTRIES; i++) ctx->tables[j][i] = INVALID_DESC; } while (mm->size) { uintptr_t end_va = xlat_tables_map_region(ctx, mm, 0, ctx->base_table, ctx->base_table_entries, ctx->base_level); if (end_va != mm->base_va + mm->size - 1) { ERROR("Not enough memory to map region:\n" " VA:%p PA:0x%llx size:0x%zx attr:0x%x\n", (void *)mm->base_va, mm->base_pa, mm->size, mm->attr); panic(); } mm++; } assert(ctx->pa_max_address <= xlat_arch_get_max_supported_pa()); assert(ctx->max_va <= ctx->va_max_address); assert(ctx->max_pa <= ctx->pa_max_address); ctx->initialized = 1; xlat_tables_print(ctx); } void init_xlat_tables(void) { init_xlat_tables_ctx(&tf_xlat_ctx); } /* * If dynamic allocation of new regions is disabled then by the time we call the * function enabling the MMU, we'll have registered all the memory regions to * map for the system's lifetime. Therefore, at this point we know the maximum * physical address that will ever be mapped. * * If dynamic allocation is enabled then we can't make any such assumption * because the maximum physical address could get pushed while adding a new * region. Therefore, in this case we have to assume that the whole address * space size might be mapped. */ #ifdef PLAT_XLAT_TABLES_DYNAMIC #define MAX_PHYS_ADDR tf_xlat_ctx.pa_max_address #else #define MAX_PHYS_ADDR tf_xlat_ctx.max_pa #endif #ifdef AARCH32 void enable_mmu_secure(unsigned int flags) { enable_mmu_arch(flags, tf_xlat_ctx.base_table, MAX_PHYS_ADDR, tf_xlat_ctx.va_max_address); } #else void enable_mmu_el1(unsigned int flags) { enable_mmu_arch(flags, tf_xlat_ctx.base_table, MAX_PHYS_ADDR, tf_xlat_ctx.va_max_address); } void enable_mmu_el3(unsigned int flags) { enable_mmu_arch(flags, tf_xlat_ctx.base_table, MAX_PHYS_ADDR, tf_xlat_ctx.va_max_address); } #endif /* AARCH32 */ /* * Do a translation table walk to find the block or page descriptor that maps * virtual_addr. * * On success, return the address of the descriptor within the translation * table. Its lookup level is stored in '*out_level'. * On error, return NULL. * * xlat_table_base * Base address for the initial lookup level. * xlat_table_base_entries * Number of entries in the translation table for the initial lookup level. * virt_addr_space_size * Size in bytes of the virtual address space. */ static uint64_t *find_xlat_table_entry(uintptr_t virtual_addr, void *xlat_table_base, int xlat_table_base_entries, unsigned long long virt_addr_space_size, int *out_level) { unsigned int start_level; uint64_t *table; int entries; VERBOSE("%s(%p)\n", __func__, (void *)virtual_addr); start_level = GET_XLAT_TABLE_LEVEL_BASE(virt_addr_space_size); VERBOSE("Starting translation table walk from level %i\n", start_level); table = xlat_table_base; entries = xlat_table_base_entries; for (unsigned int level = start_level; level <= XLAT_TABLE_LEVEL_MAX; ++level) { int idx; uint64_t desc; uint64_t desc_type; VERBOSE("Table address: %p\n", (void *)table); idx = XLAT_TABLE_IDX(virtual_addr, level); VERBOSE("Index into level %i table: %i\n", level, idx); if (idx >= entries) { VERBOSE("Invalid address\n"); return NULL; } desc = table[idx]; desc_type = desc & DESC_MASK; VERBOSE("Descriptor at level %i: 0x%llx\n", level, (unsigned long long)desc); if (desc_type == INVALID_DESC) { VERBOSE("Invalid entry (memory not mapped)\n"); return NULL; } if (level == XLAT_TABLE_LEVEL_MAX) { /* * There can't be table entries at the final lookup * level. */ assert(desc_type == PAGE_DESC); VERBOSE("Descriptor mapping a memory page (size: 0x%llx)\n", (unsigned long long)XLAT_BLOCK_SIZE(XLAT_TABLE_LEVEL_MAX)); *out_level = level; return &table[idx]; } if (desc_type == BLOCK_DESC) { VERBOSE("Descriptor mapping a memory block (size: 0x%llx)\n", (unsigned long long)XLAT_BLOCK_SIZE(level)); *out_level = level; return &table[idx]; } assert(desc_type == TABLE_DESC); VERBOSE("Table descriptor, continuing xlat table walk...\n"); table = (uint64_t *)(uintptr_t)(desc & TABLE_ADDR_MASK); entries = XLAT_TABLE_ENTRIES; } /* * This shouldn't be reached, the translation table walk should end at * most at level XLAT_TABLE_LEVEL_MAX and return from inside the loop. */ assert(0); return NULL; } static int get_mem_attributes_internal(const xlat_ctx_t *ctx, uintptr_t base_va, uint32_t *attributes, uint64_t **table_entry, unsigned long long *addr_pa, int *table_level) { uint64_t *entry; uint64_t desc; int level; unsigned long long virt_addr_space_size; /* * Sanity-check arguments. */ assert(ctx != NULL); assert(ctx->initialized); assert(ctx->xlat_regime == EL1_EL0_REGIME || ctx->xlat_regime == EL3_REGIME); virt_addr_space_size = (unsigned long long)ctx->va_max_address + 1; assert(virt_addr_space_size > 0); entry = find_xlat_table_entry(base_va, ctx->base_table, ctx->base_table_entries, virt_addr_space_size, &level); if (entry == NULL) { WARN("Address %p is not mapped.\n", (void *)base_va); return -EINVAL; } if (addr_pa != NULL) { *addr_pa = *entry & TABLE_ADDR_MASK; } if (table_entry != NULL) { *table_entry = entry; } if (table_level != NULL) { *table_level = level; } desc = *entry; #if LOG_LEVEL >= LOG_LEVEL_VERBOSE VERBOSE("Attributes: "); xlat_desc_print(ctx, desc); tf_printf("\n"); #endif /* LOG_LEVEL >= LOG_LEVEL_VERBOSE */ assert(attributes != NULL); *attributes = 0; int attr_index = (desc >> ATTR_INDEX_SHIFT) & ATTR_INDEX_MASK; if (attr_index == ATTR_IWBWA_OWBWA_NTR_INDEX) { *attributes |= MT_MEMORY; } else if (attr_index == ATTR_NON_CACHEABLE_INDEX) { *attributes |= MT_NON_CACHEABLE; } else { assert(attr_index == ATTR_DEVICE_INDEX); *attributes |= MT_DEVICE; } int ap2_bit = (desc >> AP2_SHIFT) & 1; if (ap2_bit == AP2_RW) *attributes |= MT_RW; if (ctx->xlat_regime == EL1_EL0_REGIME) { int ap1_bit = (desc >> AP1_SHIFT) & 1; if (ap1_bit == AP1_ACCESS_UNPRIVILEGED) *attributes |= MT_USER; } int ns_bit = (desc >> NS_SHIFT) & 1; if (ns_bit == 1) *attributes |= MT_NS; uint64_t xn_mask = xlat_arch_regime_get_xn_desc(ctx->xlat_regime); if ((desc & xn_mask) == xn_mask) { *attributes |= MT_EXECUTE_NEVER; } else { assert((desc & xn_mask) == 0); } return 0; } int get_mem_attributes(const xlat_ctx_t *ctx, uintptr_t base_va, uint32_t *attributes) { return get_mem_attributes_internal(ctx, base_va, attributes, NULL, NULL, NULL); } int change_mem_attributes(xlat_ctx_t *ctx, uintptr_t base_va, size_t size, uint32_t attr) { /* Note: This implementation isn't optimized. */ assert(ctx != NULL); assert(ctx->initialized); unsigned long long virt_addr_space_size = (unsigned long long)ctx->va_max_address + 1; assert(virt_addr_space_size > 0); if (!IS_PAGE_ALIGNED(base_va)) { WARN("%s: Address %p is not aligned on a page boundary.\n", __func__, (void *)base_va); return -EINVAL; } if (size == 0) { WARN("%s: Size is 0.\n", __func__); return -EINVAL; } if ((size % PAGE_SIZE) != 0) { WARN("%s: Size 0x%zx is not a multiple of a page size.\n", __func__, size); return -EINVAL; } if (((attr & MT_EXECUTE_NEVER) == 0) && ((attr & MT_RW) != 0)) { WARN("%s() doesn't allow to remap memory as read-write and executable.\n", __func__); return -EINVAL; } int pages_count = size / PAGE_SIZE; VERBOSE("Changing memory attributes of %i pages starting from address %p...\n", pages_count, (void *)base_va); uintptr_t base_va_original = base_va; /* * Sanity checks. */ for (int i = 0; i < pages_count; ++i) { uint64_t *entry; uint64_t desc; int level; entry = find_xlat_table_entry(base_va, ctx->base_table, ctx->base_table_entries, virt_addr_space_size, &level); if (entry == NULL) { WARN("Address %p is not mapped.\n", (void *)base_va); return -EINVAL; } desc = *entry; /* * Check that all the required pages are mapped at page * granularity. */ if (((desc & DESC_MASK) != PAGE_DESC) || (level != XLAT_TABLE_LEVEL_MAX)) { WARN("Address %p is not mapped at the right granularity.\n", (void *)base_va); WARN("Granularity is 0x%llx, should be 0x%x.\n", (unsigned long long)XLAT_BLOCK_SIZE(level), PAGE_SIZE); return -EINVAL; } /* * If the region type is device, it shouldn't be executable. */ int attr_index = (desc >> ATTR_INDEX_SHIFT) & ATTR_INDEX_MASK; if (attr_index == ATTR_DEVICE_INDEX) { if ((attr & MT_EXECUTE_NEVER) == 0) { WARN("Setting device memory as executable at address %p.", (void *)base_va); return -EINVAL; } } base_va += PAGE_SIZE; } /* Restore original value. */ base_va = base_va_original; VERBOSE("%s: All pages are mapped, now changing their attributes...\n", __func__); for (int i = 0; i < pages_count; ++i) { uint32_t old_attr, new_attr; uint64_t *entry; int level; unsigned long long addr_pa; get_mem_attributes_internal(ctx, base_va, &old_attr, &entry, &addr_pa, &level); VERBOSE("Old attributes: 0x%x\n", old_attr); /* * From attr, only MT_RO/MT_RW, MT_EXECUTE/MT_EXECUTE_NEVER and * MT_USER/MT_PRIVILEGED are taken into account. Any other * information is ignored. */ /* Clean the old attributes so that they can be rebuilt. */ new_attr = old_attr & ~(MT_RW|MT_EXECUTE_NEVER|MT_USER); /* * Update attributes, but filter out the ones this function * isn't allowed to change. */ new_attr |= attr & (MT_RW|MT_EXECUTE_NEVER|MT_USER); VERBOSE("New attributes: 0x%x\n", new_attr); /* * The break-before-make sequence requires writing an invalid * descriptor and making sure that the system sees the change * before writing the new descriptor. */ *entry = INVALID_DESC; /* Invalidate any cached copy of this mapping in the TLBs. */ xlat_arch_tlbi_va_regime(base_va, ctx->xlat_regime); /* Ensure completion of the invalidation. */ xlat_arch_tlbi_va_sync(); /* Write new descriptor */ *entry = xlat_desc(ctx, new_attr, addr_pa, level); base_va += PAGE_SIZE; } /* Ensure that the last descriptor writen is seen by the system. */ dsbish(); return 0; }