OrgZephyr/zephyr
OrgZephyr
/
zephyr
mirror of https://github.com/zephyrproject-rtos/zephyr.git
1
0
Fork 0
Code Issues Releases Wiki Activity
zephyr/arch/arm/core/mmu/arm_mmu.c

1101 lines
40 KiB
C
Raw Blame History

/*
* ARMv7 MMU support
*
* This implementation supports the Short-descriptor translation
* table format. The standard page size is 4 kB, 1 MB sections
* are only used for mapping the code and data of the Zephyr image.
* Secure mode and PL1 is always assumed. LPAE and PXN extensions
* as well as TEX remapping are not supported. The AP[2:1] plus
* Access flag permissions model is used, as the AP[2:0] model is
* deprecated. As the AP[2:1] model can only disable write access,
* the read permission flag is always implied.
*
* Reference documentation:
* ARM Architecture Reference Manual, ARMv7-A and ARMv7-R edition,
* ARM document ID DDI0406C Rev. d, March 2018
*
* Copyright (c) 2021 Weidmueller Interface GmbH & Co. KG
* SPDX-License-Identifier: Apache-2.0
*/
#include <zephyr/device.h>
#include <zephyr/init.h>
#include <zephyr/kernel.h>
#include <zephyr/linker/linker-defs.h>
#include <zephyr/logging/log.h>
#include <zephyr/sys/__assert.h>
#include <zephyr/sys/util.h>
#include <zephyr/kernel/mm.h>
#include <zephyr/sys/barrier.h>
#include <cmsis_core.h>
#include <zephyr/arch/arm/mmu/arm_mmu.h>
#include "arm_mmu_priv.h"
LOG_MODULE_DECLARE(os, CONFIG_KERNEL_LOG_LEVEL);
/* Level 1 page table: always required, must be 16k-aligned */
static struct arm_mmu_l1_page_table
l1_page_table __aligned(KB(16)) = {0};
/*
* Array of level 2 page tables with 4k granularity:
* each table covers a range of 1 MB, the number of L2 tables
* is configurable.
*/
static struct arm_mmu_l2_page_table
l2_page_tables[CONFIG_ARM_MMU_NUM_L2_TABLES] __aligned(KB(1)) = {0};
/*
* For each level 2 page table, a separate dataset tracks
* if the respective table is in use, if so, to which 1 MB
* virtual address range it is assigned, and how many entries,
* each mapping a 4 kB page, it currently contains.
*/
static struct arm_mmu_l2_page_table_status
l2_page_tables_status[CONFIG_ARM_MMU_NUM_L2_TABLES] = {0};
/* Available L2 tables count & next free index for an L2 table request */
static uint32_t arm_mmu_l2_tables_free = CONFIG_ARM_MMU_NUM_L2_TABLES;
static uint32_t arm_mmu_l2_next_free_table;
/*
* Static definition of all code & data memory regions of the
* current Zephyr image. This information must be available &
* processed upon MMU initialization.
*/
static const struct arm_mmu_flat_range mmu_zephyr_ranges[] = {
/*
* Mark the zephyr execution regions (data, bss, noinit, etc.)
* cacheable, read / write and non-executable
*/
{ .name = "zephyr_data",
.start = (uint32_t)_image_ram_start,
.end = (uint32_t)_image_ram_end,
.attrs = MT_NORMAL | MATTR_SHARED |
MPERM_R | MPERM_W |
MATTR_CACHE_OUTER_WB_WA | MATTR_CACHE_INNER_WB_WA},
/* Mark text segment cacheable, read only and executable */
{ .name = "zephyr_code",
.start = (uint32_t)__text_region_start,
.end = (uint32_t)__text_region_end,
.attrs = MT_NORMAL | MATTR_SHARED |
/* The code needs to have write permission in order for
* software breakpoints (which modify instructions) to work
*/
#if defined(CONFIG_GDBSTUB)
MPERM_R | MPERM_X | MPERM_W |
#else
MPERM_R | MPERM_X |
#endif
MATTR_CACHE_OUTER_WB_nWA | MATTR_CACHE_INNER_WB_nWA |
MATTR_MAY_MAP_L1_SECTION},
/* Mark rodata segment cacheable, read only and non-executable */
{ .name = "zephyr_rodata",
.start = (uint32_t)__rodata_region_start,
.end = (uint32_t)__rodata_region_end,
.attrs = MT_NORMAL | MATTR_SHARED |
MPERM_R |
MATTR_CACHE_OUTER_WB_nWA | MATTR_CACHE_INNER_WB_nWA |
MATTR_MAY_MAP_L1_SECTION},
#ifdef CONFIG_NOCACHE_MEMORY
/* Mark nocache segment read / write and non-executable */
{ .name = "nocache",
.start = (uint32_t)_nocache_ram_start,
.end = (uint32_t)_nocache_ram_end,
.attrs = MT_STRONGLY_ORDERED |
MPERM_R | MPERM_W},
#endif
};
static void arm_mmu_l2_map_page(uint32_t va, uint32_t pa,
struct arm_mmu_perms_attrs perms_attrs);
/**
* @brief Invalidates the TLB
* Helper function which invalidates the entire TLB. This action
* is performed whenever the MMU is (re-)enabled or changes to the
* page tables are made at run-time, as the TLB might contain entries
* which are no longer valid once the changes are applied.
*/
static void invalidate_tlb_all(void)
{
__set_TLBIALL(0); /* 0 = opc2 = invalidate entire TLB */
barrier_dsync_fence_full();
barrier_isync_fence_full();
}
/**
* @brief Returns a free level 2 page table
* Initializes and returns the next free L2 page table whenever
* a page is to be mapped in a 1 MB virtual address range that
* is not yet covered by a level 2 page table.
*
* @param va 32-bit virtual address to be mapped.
* @retval pointer to the L2 table now assigned to the 1 MB
* address range the target virtual address is in.
*/
static struct arm_mmu_l2_page_table *arm_mmu_assign_l2_table(uint32_t va)
{
struct arm_mmu_l2_page_table *l2_page_table;
__ASSERT(arm_mmu_l2_tables_free > 0,
"Cannot set up L2 page table for VA 0x%08X: "
"no more free L2 page tables available\n",
va);
__ASSERT(l2_page_tables_status[arm_mmu_l2_next_free_table].entries == 0,
"Cannot set up L2 page table for VA 0x%08X: "
"expected empty L2 table at index [%u], but the "
"entries value is %u\n",
va, arm_mmu_l2_next_free_table,
l2_page_tables_status[arm_mmu_l2_next_free_table].entries);
/*
* Store in the status dataset of the L2 table to be returned
* which 1 MB virtual address range it is being assigned to.
* Set the current page table entry count to 0.
*/
l2_page_tables_status[arm_mmu_l2_next_free_table].l1_index =
((va >> ARM_MMU_PTE_L1_INDEX_PA_SHIFT) & ARM_MMU_PTE_L1_INDEX_MASK);
l2_page_tables_status[arm_mmu_l2_next_free_table].entries = 0;
l2_page_table = &l2_page_tables[arm_mmu_l2_next_free_table];
/*
* Decrement the available L2 page table count. As long as at
* least one more L2 table is available afterwards, update the
* L2 next free table index. If we're about to return the last
* available L2 table, calculating a next free table index is
* impossible.
*/
--arm_mmu_l2_tables_free;
if (arm_mmu_l2_tables_free > 0) {
do {
arm_mmu_l2_next_free_table = (arm_mmu_l2_next_free_table + 1) %
CONFIG_ARM_MMU_NUM_L2_TABLES;
} while (l2_page_tables_status[arm_mmu_l2_next_free_table].entries != 0);
}
return l2_page_table;
}
/**
* @brief Releases a level 2 page table
* Releases a level 2 page table, marking it as no longer in use.
* From that point on, it can be re-used for mappings in another
* 1 MB virtual address range. This function is called whenever
* it is determined during an unmap call at run-time that the page
* table entry count in the respective page table has reached 0.
*
* @param l2_page_table Pointer to L2 page table to be released.
*/
static void arm_mmu_release_l2_table(struct arm_mmu_l2_page_table *l2_page_table)
{
uint32_t l2_page_table_index = ARM_MMU_L2_PT_INDEX(l2_page_table);
l2_page_tables_status[l2_page_table_index].l1_index = 0;
if (arm_mmu_l2_tables_free == 0) {
arm_mmu_l2_next_free_table = l2_page_table_index;
}
++arm_mmu_l2_tables_free;
}
/**
* @brief Increments the page table entry counter of a L2 page table
* Increments the page table entry counter of a level 2 page table.
* Contains a check to ensure that no attempts are made to set up
* more page table entries than the table can hold.
*
* @param l2_page_table Pointer to the L2 page table whose entry
* counter shall be incremented.
*/
static void arm_mmu_inc_l2_table_entries(struct arm_mmu_l2_page_table *l2_page_table)
{
uint32_t l2_page_table_index = ARM_MMU_L2_PT_INDEX(l2_page_table);
__ASSERT(l2_page_tables_status[l2_page_table_index].entries < ARM_MMU_PT_L2_NUM_ENTRIES,
"Cannot increment entry count of the L2 page table at index "
"[%u] / addr %p / ref L1[%u]: maximum entry count already reached",
l2_page_table_index, l2_page_table,
l2_page_tables_status[l2_page_table_index].l1_index);
++l2_page_tables_status[l2_page_table_index].entries;
}
/**
* @brief Decrements the page table entry counter of a L2 page table
* Decrements the page table entry counter of a level 2 page table.
* Contains a check to ensure that no attempts are made to remove
* entries from the respective table that aren't actually there.
*
* @param l2_page_table Pointer to the L2 page table whose entry
* counter shall be decremented.
*/
static void arm_mmu_dec_l2_table_entries(struct arm_mmu_l2_page_table *l2_page_table)
{
uint32_t l2_page_table_index = ARM_MMU_L2_PT_INDEX(l2_page_table);
__ASSERT(l2_page_tables_status[l2_page_table_index].entries > 0,
"Cannot decrement entry count of the L2 page table at index "
"[%u] / addr %p / ref L1[%u]: entry count is already zero",
l2_page_table_index, l2_page_table,
l2_page_tables_status[l2_page_table_index].l1_index);
if (--l2_page_tables_status[l2_page_table_index].entries == 0) {
arm_mmu_release_l2_table(l2_page_table);
}
}
/**
* @brief Converts memory attributes and permissions to MMU format
* Converts memory attributes and permissions as used in the boot-
* time memory mapping configuration data array (MT_..., MATTR_...,
* MPERM_...) to the equivalent bit (field) values used in the MMU's
* L1 and L2 page table entries. Contains plausibility checks.
*
* @param attrs type/attribute/permissions flags word obtained from
* an entry of the mmu_config mapping data array.
* @retval A struct containing the information from the input flags
* word converted to the bits / bit fields used in L1 and
* L2 page table entries.
*/
static struct arm_mmu_perms_attrs arm_mmu_convert_attr_flags(uint32_t attrs)
{
struct arm_mmu_perms_attrs perms_attrs = {0};
__ASSERT(((attrs & MT_MASK) > 0),
"Cannot convert attrs word to PTE control bits: no "
"memory type specified");
__ASSERT(!((attrs & MPERM_W) && !(attrs & MPERM_R)),
"attrs must not define write permission without read "
"permission");
__ASSERT(!((attrs & MPERM_W) && (attrs & MPERM_X)),
"attrs must not define executable memory with write "
"permission");
/*
* The translation of the memory type / permissions / attributes
* flags in the attrs word to the TEX, C, B, S and AP bits of the
* target PTE is based on the reference manual:
* TEX, C, B, S: Table B3-10, chap. B3.8.2, p. B3-1363f.
* AP : Table B3-6, chap. B3.7.1, p. B3-1353.
* Device / strongly ordered memory is always assigned to a domain
* other than that used for normal memory. Assuming that userspace
* support utilizing the MMU is eventually implemented, a single
* modification of the DACR register when entering/leaving unprivi-
* leged mode could be used in order to enable/disable all device
* memory access without having to modify any PTs/PTEs.
*/
if (attrs & MT_STRONGLY_ORDERED) {
/* Strongly ordered is always shareable, S bit is ignored */
perms_attrs.tex = 0;
perms_attrs.cacheable = 0;
perms_attrs.bufferable = 0;
perms_attrs.shared = 0;
perms_attrs.domain = ARM_MMU_DOMAIN_DEVICE;
} else if (attrs & MT_DEVICE) {
/*
* Shareability of device memory is determined by TEX, C, B.
* The S bit is ignored. C is always 0 for device memory.
*/
perms_attrs.shared = 0;
perms_attrs.cacheable = 0;
perms_attrs.domain = ARM_MMU_DOMAIN_DEVICE;
if (attrs & MATTR_SHARED) {
perms_attrs.tex = 0;
perms_attrs.bufferable = 1;
} else {
perms_attrs.tex = 2;
perms_attrs.bufferable = 0;
}
} else if (attrs & MT_NORMAL) {
/*
* TEX[2] is always 1. TEX[1:0] contain the outer cache attri-
* butes encoding, C and B contain the inner cache attributes
* encoding.
*/
perms_attrs.tex |= ARM_MMU_TEX2_CACHEABLE_MEMORY;
perms_attrs.domain = ARM_MMU_DOMAIN_OS;
/* For normal memory, shareability depends on the S bit */
if (attrs & MATTR_SHARED) {
perms_attrs.shared = 1;
}
if (attrs & MATTR_CACHE_OUTER_WB_WA) {
perms_attrs.tex |= ARM_MMU_TEX_CACHE_ATTRS_WB_WA;
} else if (attrs & MATTR_CACHE_OUTER_WT_nWA) {
perms_attrs.tex |= ARM_MMU_TEX_CACHE_ATTRS_WT_nWA;
} else if (attrs & MATTR_CACHE_OUTER_WB_nWA) {
perms_attrs.tex |= ARM_MMU_TEX_CACHE_ATTRS_WB_nWA;
}
if (attrs & MATTR_CACHE_INNER_WB_WA) {
perms_attrs.cacheable = ARM_MMU_C_CACHE_ATTRS_WB_WA;
perms_attrs.bufferable = ARM_MMU_B_CACHE_ATTRS_WB_WA;
} else if (attrs & MATTR_CACHE_INNER_WT_nWA) {
perms_attrs.cacheable = ARM_MMU_C_CACHE_ATTRS_WT_nWA;
perms_attrs.bufferable = ARM_MMU_B_CACHE_ATTRS_WT_nWA;
} else if (attrs & MATTR_CACHE_INNER_WB_nWA) {
perms_attrs.cacheable = ARM_MMU_C_CACHE_ATTRS_WB_nWA;
perms_attrs.bufferable = ARM_MMU_B_CACHE_ATTRS_WB_nWA;
}
}
if (attrs & MATTR_NON_SECURE) {
perms_attrs.non_sec = 1;
}
if (attrs & MATTR_NON_GLOBAL) {
perms_attrs.not_global = 1;
}
/*
* Up next is the consideration of the case that a PTE shall be configured
* for a page that shall not be accessible at all (e.g. guard pages), and
* therefore has neither read nor write permissions. In the AP[2:1] access
* permission specification model, the only way to indicate this is to
* actually mask out the PTE's identifier bits, as otherwise, read permission
* is always granted for any valid PTE, it can't be revoked explicitly,
* unlike the write permission.
*/
if (!((attrs & MPERM_R) || (attrs & MPERM_W))) {
perms_attrs.id_mask = 0x0;
} else {
perms_attrs.id_mask = 0x3;
}
if (!(attrs & MPERM_W)) {
perms_attrs.acc_perms |= ARM_MMU_PERMS_AP2_DISABLE_WR;
}
if (attrs & MPERM_UNPRIVILEGED) {
perms_attrs.acc_perms |= ARM_MMU_PERMS_AP1_ENABLE_PL0;
}
if (!(attrs & MPERM_X)) {
perms_attrs.exec_never = 1;
}
return perms_attrs;
}
/**
* @brief Maps a 1 MB memory range via a level 1 page table entry
* Maps a 1 MB memory range using a level 1 page table entry of type
* 'section'. This type of entry saves a level 2 page table, but has
* two pre-conditions: the memory area to be mapped must contain at
* least 1 MB of contiguous memory, starting at an address with suit-
* able alignment. This mapping method should only be used for map-
* pings for which it is unlikely that the attributes of those mappings
* will mappings will change at run-time (e.g. code sections will al-
* ways be read-only and executable). Should the case occur that the
* permissions or attributes of a subset of a 1 MB section entry shall
* be re-configured at run-time, a L1 section entry will be broken
* down into 4k segments using a L2 table with identical attributes
* before any modifications are performed for the subset of the affec-
* ted 1 MB range. This comes with an undeterministic performance
* penalty at the time of re-configuration, therefore, any mappings
* for which L1 section entries are a valid option, shall be marked in
* their declaration with the MATTR_MAY_MAP_L1_SECTION flag.
*
* @param va 32-bit target virtual address to be mapped.
* @param pa 32-bit physical address to be mapped.
* @param perms_attrs Permission and attribute bits in the format
* used in the MMU's L1 page table entries.
*/
static void arm_mmu_l1_map_section(uint32_t va, uint32_t pa,
struct arm_mmu_perms_attrs perms_attrs)
{
uint32_t l1_index = (va >> ARM_MMU_PTE_L1_INDEX_PA_SHIFT) &
ARM_MMU_PTE_L1_INDEX_MASK;
__ASSERT(l1_page_table.entries[l1_index].undefined.id == ARM_MMU_PTE_ID_INVALID,
"Unexpected non-zero L1 PTE ID %u for VA 0x%08X / PA 0x%08X",
l1_page_table.entries[l1_index].undefined.id,
va, pa);
l1_page_table.entries[l1_index].l1_section_1m.id =
(ARM_MMU_PTE_ID_SECTION & perms_attrs.id_mask);
l1_page_table.entries[l1_index].l1_section_1m.bufferable = perms_attrs.bufferable;
l1_page_table.entries[l1_index].l1_section_1m.cacheable = perms_attrs.cacheable;
l1_page_table.entries[l1_index].l1_section_1m.exec_never = perms_attrs.exec_never;
l1_page_table.entries[l1_index].l1_section_1m.domain = perms_attrs.domain;
l1_page_table.entries[l1_index].l1_section_1m.impl_def = 0;
l1_page_table.entries[l1_index].l1_section_1m.acc_perms10 =
((perms_attrs.acc_perms & 0x1) << 1) | 0x1;
l1_page_table.entries[l1_index].l1_section_1m.tex = perms_attrs.tex;
l1_page_table.entries[l1_index].l1_section_1m.acc_perms2 =
(perms_attrs.acc_perms >> 1) & 0x1;
l1_page_table.entries[l1_index].l1_section_1m.shared = perms_attrs.shared;
l1_page_table.entries[l1_index].l1_section_1m.not_global = perms_attrs.not_global;
l1_page_table.entries[l1_index].l1_section_1m.zero = 0;
l1_page_table.entries[l1_index].l1_section_1m.non_sec = perms_attrs.non_sec;
l1_page_table.entries[l1_index].l1_section_1m.base_address =
(pa >> ARM_MMU_PTE_L1_INDEX_PA_SHIFT);
}
/**
* @brief Converts a L1 1 MB section mapping to a full L2 table
* When this function is called, something has happened that shouldn't
* happen for the sake of run-time performance and determinism: the
* attributes and/or permissions of a subset of a 1 MB memory range
* currently represented by a level 1 page table entry of type 'section'
* shall be modified so that they differ from the rest of the 1 MB
* range's attributes/permissions. Therefore, the single L1 page table
* entry has to be broken down to the full 256 4k-wide entries of a
* L2 page table with identical properties so that afterwards, the
* modification of the subset can be performed with a 4k granularity.
* The risk at this point is that all L2 tables are already in use,
* which will result in an assertion failure in the first contained
* #arm_mmu_l2_map_page() call.
* @warning While the conversion is being performed, interrupts are
* locked globally and the MMU is disabled (the required
* Zephyr code & data are still accessible in this state as
* those are identity mapped). Expect non-deterministic be-
* haviour / interrupt latencies while the conversion is in
* progress!
*
* @param va 32-bit virtual address within the 1 MB range that shall
* be converted from L1 1 MB section mapping to L2 4 kB page
* mappings.
* @param l2_page_table Pointer to an empty L2 page table allocated
* for the purpose of replacing the L1 section
* mapping.
*/
static void arm_mmu_remap_l1_section_to_l2_table(uint32_t va,
struct arm_mmu_l2_page_table *l2_page_table)
{
struct arm_mmu_perms_attrs perms_attrs = {0};
uint32_t l1_index = (va >> ARM_MMU_PTE_L1_INDEX_PA_SHIFT) &
ARM_MMU_PTE_L1_INDEX_MASK;
uint32_t rem_size = MB(1);
uint32_t reg_val;
int lock_key;
/*
* Extract the permissions and attributes from the current 1 MB section entry.
* This data will be carried over to the resulting L2 page table.
*/
perms_attrs.acc_perms = (l1_page_table.entries[l1_index].l1_section_1m.acc_perms2 << 1) |
((l1_page_table.entries[l1_index].l1_section_1m.acc_perms10 >> 1) & 0x1);
perms_attrs.bufferable = l1_page_table.entries[l1_index].l1_section_1m.bufferable;
perms_attrs.cacheable = l1_page_table.entries[l1_index].l1_section_1m.cacheable;
perms_attrs.domain = l1_page_table.entries[l1_index].l1_section_1m.domain;
perms_attrs.id_mask = (l1_page_table.entries[l1_index].l1_section_1m.id ==
ARM_MMU_PTE_ID_INVALID) ? 0x0 : 0x3;
perms_attrs.not_global = l1_page_table.entries[l1_index].l1_section_1m.not_global;
perms_attrs.non_sec = l1_page_table.entries[l1_index].l1_section_1m.non_sec;
perms_attrs.shared = l1_page_table.entries[l1_index].l1_section_1m.shared;
perms_attrs.tex = l1_page_table.entries[l1_index].l1_section_1m.tex;
perms_attrs.exec_never = l1_page_table.entries[l1_index].l1_section_1m.exec_never;
/*
* Disable interrupts - no interrupts shall occur before the L2 table has
* been set up in place of the former L1 section entry.
*/
lock_key = arch_irq_lock();
/*
* Disable the MMU. The L1 PTE array and the L2 PT array may actually be
* covered by the L1 PTE we're about to replace, so access to this data
* must remain functional during the entire remap process. Yet, the only
* memory areas for which L1 1 MB section entries are even considered are
* those belonging to the Zephyr image. Those areas are *always* identity
* mapped, so the MMU can be turned off and the relevant data will still
* be available.
*/
reg_val = __get_SCTLR();
__set_SCTLR(reg_val & (~ARM_MMU_SCTLR_MMU_ENABLE_BIT));
/*
* Clear the entire L1 PTE & re-configure it as a L2 PT reference
* -> already sets the correct values for: zero0, zero1, impl_def.
*/
l1_page_table.entries[l1_index].word = 0;
l1_page_table.entries[l1_index].l2_page_table_ref.id = ARM_MMU_PTE_ID_L2_PT;
l1_page_table.entries[l1_index].l2_page_table_ref.domain = perms_attrs.domain;
l1_page_table.entries[l1_index].l2_page_table_ref.non_sec = perms_attrs.non_sec;
l1_page_table.entries[l1_index].l2_page_table_ref.l2_page_table_address =
(((uint32_t)l2_page_table >> ARM_MMU_PT_L2_ADDR_SHIFT) &
ARM_MMU_PT_L2_ADDR_MASK);
/* Align the target VA to the base address of the section we're converting */
va &= ~(MB(1) - 1);
while (rem_size > 0) {
arm_mmu_l2_map_page(va, va, perms_attrs);
rem_size -= KB(4);
va += KB(4);
}
/* Remap complete, re-enable the MMU, unlock the interrupts. */
invalidate_tlb_all();
__set_SCTLR(reg_val);
arch_irq_unlock(lock_key);
}
/**
* @brief Maps a 4 kB memory page using a L2 page table entry
* Maps a single 4 kB page of memory from the specified physical
* address to the specified virtual address, using the provided
* attributes and permissions which have already been converted
* from the system's format provided to arch_mem_map() to the
* bits / bit masks used in the L2 page table entry.
*
* @param va 32-bit target virtual address.
* @param pa 32-bit physical address.
* @param perms_attrs Permission and attribute bits in the format
* used in the MMU's L2 page table entries.
*/
static void arm_mmu_l2_map_page(uint32_t va, uint32_t pa,
struct arm_mmu_perms_attrs perms_attrs)
{
struct arm_mmu_l2_page_table *l2_page_table = NULL;
uint32_t l1_index = (va >> ARM_MMU_PTE_L1_INDEX_PA_SHIFT) &
ARM_MMU_PTE_L1_INDEX_MASK;
uint32_t l2_index = (va >> ARM_MMU_PTE_L2_INDEX_PA_SHIFT) &
ARM_MMU_PTE_L2_INDEX_MASK;
/*
* Use the calculated L1 index in order to determine if a L2 page
* table is required in order to complete the current mapping.
* -> See below for an explanation of the possible scenarios.
*/
if (l1_page_table.entries[l1_index].undefined.id == ARM_MMU_PTE_ID_INVALID ||
(l1_page_table.entries[l1_index].undefined.id & ARM_MMU_PTE_ID_SECTION) != 0) {
l2_page_table = arm_mmu_assign_l2_table(pa);
__ASSERT(l2_page_table != NULL,
"Unexpected L2 page table NULL pointer for VA 0x%08X",
va);
}
/*
* Check what is currently present at the corresponding L1 table entry.
* The following scenarios are possible:
* 1) The L1 PTE's ID bits are zero, as is the rest of the entry.
* In this case, the L1 PTE is currently unused. A new L2 PT to
* refer to in this entry has already been allocated above.
* 2) The L1 PTE's ID bits indicate a L2 PT reference entry (01).
* The corresponding L2 PT's address will be resolved using this
* entry.
* 3) The L1 PTE's ID bits may or may not be zero, and the rest of
* the descriptor contains some non-zero data. This always indicates
* an existing 1 MB section entry in this place. Checking only the
* ID bits wouldn't be enough, as the only way to indicate a section
* with neither R nor W permissions is to set the ID bits to 00 in
* the AP[2:1] permissions model. As we're now about to map a single
* page overlapping with the 1 MB section, the section has to be
* converted into a L2 table. Afterwards, the current page mapping
* can be added/modified.
*/
if (l1_page_table.entries[l1_index].word == 0) {
/* The matching L1 PT entry is currently unused */
l1_page_table.entries[l1_index].l2_page_table_ref.id = ARM_MMU_PTE_ID_L2_PT;
l1_page_table.entries[l1_index].l2_page_table_ref.zero0 = 0;
l1_page_table.entries[l1_index].l2_page_table_ref.zero1 = 0;
l1_page_table.entries[l1_index].l2_page_table_ref.impl_def = 0;
l1_page_table.entries[l1_index].l2_page_table_ref.domain = 0; /* TODO */
l1_page_table.entries[l1_index].l2_page_table_ref.non_sec =
perms_attrs.non_sec;
l1_page_table.entries[l1_index].l2_page_table_ref.l2_page_table_address =
(((uint32_t)l2_page_table >> ARM_MMU_PT_L2_ADDR_SHIFT) &
ARM_MMU_PT_L2_ADDR_MASK);
} else if (l1_page_table.entries[l1_index].undefined.id == ARM_MMU_PTE_ID_L2_PT) {
/* The matching L1 PT entry already points to a L2 PT */
l2_page_table = (struct arm_mmu_l2_page_table *)
((l1_page_table.entries[l1_index].word &
(ARM_MMU_PT_L2_ADDR_MASK << ARM_MMU_PT_L2_ADDR_SHIFT)));
/*
* The only configuration bit contained in the L2 PT entry is the
* NS bit. Set it according to the attributes passed to this function,
* warn if there is a mismatch between the current page's NS attribute
* value and the value currently contained in the L2 PT entry.
*/
if (l1_page_table.entries[l1_index].l2_page_table_ref.non_sec !=
perms_attrs.non_sec) {
LOG_WRN("NS bit mismatch in L2 PT reference at L1 index [%u], "
"re-configuring from %u to %u",
l1_index,
l1_page_table.entries[l1_index].l2_page_table_ref.non_sec,
perms_attrs.non_sec);
l1_page_table.entries[l1_index].l2_page_table_ref.non_sec =
perms_attrs.non_sec;
}
} else if (l1_page_table.entries[l1_index].undefined.reserved != 0) {
/*
* The matching L1 PT entry currently holds a 1 MB section entry
* in order to save a L2 table (as it's neither completely blank
* nor a L2 PT reference), but now we have to map an overlapping
* 4 kB page, so the section entry must be converted to a L2 table
* first before the individual L2 entry for the page to be mapped is
* accessed. A blank L2 PT has already been assigned above.
*/
arm_mmu_remap_l1_section_to_l2_table(va, l2_page_table);
}
/*
* If the matching L2 PTE is blank, increment the number of used entries
* in the L2 table. If the L2 PTE already contains some data, we're re-
* placing the entry's data instead, the used entry count remains unchanged.
* Once again, checking the ID bits might be misleading if the PTE declares
* a page which has neither R nor W permissions.
*/
if (l2_page_table->entries[l2_index].word == 0) {
arm_mmu_inc_l2_table_entries(l2_page_table);
}
l2_page_table->entries[l2_index].l2_page_4k.id =
(ARM_MMU_PTE_ID_SMALL_PAGE & perms_attrs.id_mask);
l2_page_table->entries[l2_index].l2_page_4k.id |= perms_attrs.exec_never; /* XN in [0] */
l2_page_table->entries[l2_index].l2_page_4k.bufferable = perms_attrs.bufferable;
l2_page_table->entries[l2_index].l2_page_4k.cacheable = perms_attrs.cacheable;
l2_page_table->entries[l2_index].l2_page_4k.acc_perms10 =
((perms_attrs.acc_perms & 0x1) << 1) | 0x1;
l2_page_table->entries[l2_index].l2_page_4k.tex = perms_attrs.tex;
l2_page_table->entries[l2_index].l2_page_4k.acc_perms2 =
((perms_attrs.acc_perms >> 1) & 0x1);
l2_page_table->entries[l2_index].l2_page_4k.shared = perms_attrs.shared;
l2_page_table->entries[l2_index].l2_page_4k.not_global = perms_attrs.not_global;
l2_page_table->entries[l2_index].l2_page_4k.pa_base =
((pa >> ARM_MMU_PTE_L2_SMALL_PAGE_ADDR_SHIFT) &
ARM_MMU_PTE_L2_SMALL_PAGE_ADDR_MASK);
}
/**
* @brief Unmaps a 4 kB memory page by clearing its L2 page table entry
* Unmaps a single 4 kB page of memory from the specified virtual
* address by clearing its respective L2 page table entry.
*
* @param va 32-bit virtual address to be unmapped.
*/
static void arm_mmu_l2_unmap_page(uint32_t va)
{
struct arm_mmu_l2_page_table *l2_page_table;
uint32_t l1_index = (va >> ARM_MMU_PTE_L1_INDEX_PA_SHIFT) &
ARM_MMU_PTE_L1_INDEX_MASK;
uint32_t l2_index = (va >> ARM_MMU_PTE_L2_INDEX_PA_SHIFT) &
ARM_MMU_PTE_L2_INDEX_MASK;
if (l1_page_table.entries[l1_index].undefined.id != ARM_MMU_PTE_ID_L2_PT) {
/*
* No L2 PT currently exists for the given VA - this should be
* tolerated without an error, just as in the case that while
* a L2 PT exists, the corresponding PTE is blank - see explanation
* below, the same applies here.
*/
return;
}
l2_page_table = (struct arm_mmu_l2_page_table *)
((l1_page_table.entries[l1_index].word &
(ARM_MMU_PT_L2_ADDR_MASK << ARM_MMU_PT_L2_ADDR_SHIFT)));
if (l2_page_table->entries[l2_index].word == 0) {
/*
* We're supposed to unmap a page at the given VA, but there currently
* isn't anything mapped at this address, the L2 PTE is blank.
* -> This is normal if a memory area is being mapped via k_mem_map,
* which contains two calls to arch_mem_unmap (which effectively end up
* here) in order to unmap the leading and trailing guard pages.
* Therefore, it has to be expected that unmap calls are made for unmapped
* memory which hasn't been in use before.
* -> Just return, don't decrement the entry counter of the corresponding
* L2 page table, as we're not actually clearing any PTEs.
*/
return;
}
if ((l2_page_table->entries[l2_index].undefined.id & ARM_MMU_PTE_ID_SMALL_PAGE) !=
ARM_MMU_PTE_ID_SMALL_PAGE) {
LOG_ERR("Cannot unmap virtual memory at 0x%08X: invalid "
"page table entry type in level 2 page table at "
"L1 index [%u], L2 index [%u]", va, l1_index, l2_index);
return;
}
l2_page_table->entries[l2_index].word = 0;
arm_mmu_dec_l2_table_entries(l2_page_table);
}
/**
* @brief MMU boot-time initialization function
* Initializes the MMU at boot time. Sets up the page tables and
* applies any specified memory mappings for either the different
* sections of the Zephyr binary image, or for device memory as
* specified at the SoC level.
*
* @retval Always 0, errors are handled by assertions.
*/
int z_arm_mmu_init(void)
{
uint32_t mem_range;
uint32_t pa;
uint32_t va;
uint32_t attrs;
uint32_t pt_attrs = 0;
uint32_t rem_size;
uint32_t reg_val = 0;
struct arm_mmu_perms_attrs perms_attrs;
__ASSERT(KB(4) == CONFIG_MMU_PAGE_SIZE,
"MMU_PAGE_SIZE value %u is invalid, only 4 kB pages are supported\n",
CONFIG_MMU_PAGE_SIZE);
/* Set up the memory regions pre-defined by the image */
for (mem_range = 0; mem_range < ARRAY_SIZE(mmu_zephyr_ranges); mem_range++) {
pa = mmu_zephyr_ranges[mem_range].start;
rem_size = mmu_zephyr_ranges[mem_range].end - pa;
attrs = mmu_zephyr_ranges[mem_range].attrs;
perms_attrs = arm_mmu_convert_attr_flags(attrs);
/*
* Check if the L1 page table is within the region currently
* being mapped. If so, store the permissions and attributes
* of the current section. This information is required when
* writing to the TTBR0 register.
*/
if (((uint32_t)&l1_page_table >= pa) &&
((uint32_t)&l1_page_table < (pa + rem_size))) {
pt_attrs = attrs;
}
while (rem_size > 0) {
if (rem_size >= MB(1) && (pa & 0xFFFFF) == 0 &&
(attrs & MATTR_MAY_MAP_L1_SECTION)) {
/*
* Remaining area size > 1 MB & matching alignment
* -> map a 1 MB section instead of individual 4 kB
* pages with identical configuration.
*/
arm_mmu_l1_map_section(pa, pa, perms_attrs);
rem_size -= MB(1);
pa += MB(1);
} else {
arm_mmu_l2_map_page(pa, pa, perms_attrs);
rem_size -= (rem_size >= KB(4)) ? KB(4) : rem_size;
pa += KB(4);
}
}
}
/* Set up the memory regions defined at the SoC level */
for (mem_range = 0; mem_range < mmu_config.num_regions; mem_range++) {
pa = (uint32_t)(mmu_config.mmu_regions[mem_range].base_pa);
va = (uint32_t)(mmu_config.mmu_regions[mem_range].base_va);
rem_size = (uint32_t)(mmu_config.mmu_regions[mem_range].size);
attrs = mmu_config.mmu_regions[mem_range].attrs;
perms_attrs = arm_mmu_convert_attr_flags(attrs);
while (rem_size > 0) {
arm_mmu_l2_map_page(va, pa, perms_attrs);
rem_size -= (rem_size >= KB(4)) ? KB(4) : rem_size;
va += KB(4);
pa += KB(4);
}
}
/* Clear TTBR1 */
__asm__ __volatile__("mcr p15, 0, %0, c2, c0, 1" : : "r"(reg_val));
/* Write TTBCR: EAE, security not yet relevant, N[2:0] = 0 */
__asm__ __volatile__("mcr p15, 0, %0, c2, c0, 2"
: : "r"(reg_val));
/* Write TTBR0 */
reg_val = ((uint32_t)&l1_page_table.entries[0] & ~0x3FFF);
/*
* Set IRGN, RGN, S in TTBR0 based on the configuration of the
* memory area the actual page tables are located in.
*/
if (pt_attrs & MATTR_SHARED) {
reg_val |= ARM_MMU_TTBR_SHAREABLE_BIT;
}
if (pt_attrs & MATTR_CACHE_OUTER_WB_WA) {
reg_val |= (ARM_MMU_TTBR_RGN_OUTER_WB_WA_CACHEABLE <<
ARM_MMU_TTBR_RGN_SHIFT);
} else if (pt_attrs & MATTR_CACHE_OUTER_WT_nWA) {
reg_val |= (ARM_MMU_TTBR_RGN_OUTER_WT_CACHEABLE <<
ARM_MMU_TTBR_RGN_SHIFT);
} else if (pt_attrs & MATTR_CACHE_OUTER_WB_nWA) {
reg_val |= (ARM_MMU_TTBR_RGN_OUTER_WB_nWA_CACHEABLE <<
ARM_MMU_TTBR_RGN_SHIFT);
}
if (pt_attrs & MATTR_CACHE_INNER_WB_WA) {
reg_val |= ARM_MMU_TTBR_IRGN0_BIT_MP_EXT_ONLY;
} else if (pt_attrs & MATTR_CACHE_INNER_WT_nWA) {
reg_val |= ARM_MMU_TTBR_IRGN1_BIT_MP_EXT_ONLY;
} else if (pt_attrs & MATTR_CACHE_INNER_WB_nWA) {
reg_val |= ARM_MMU_TTBR_IRGN0_BIT_MP_EXT_ONLY;
reg_val |= ARM_MMU_TTBR_IRGN1_BIT_MP_EXT_ONLY;
}
__set_TTBR0(reg_val);
/* Write DACR -> all domains to client = 01b. */
reg_val = ARM_MMU_DACR_ALL_DOMAINS_CLIENT;
__set_DACR(reg_val);
invalidate_tlb_all();
/* Enable the MMU and Cache in SCTLR */
reg_val = __get_SCTLR();
reg_val |= ARM_MMU_SCTLR_AFE_BIT;
reg_val |= ARM_MMU_SCTLR_ICACHE_ENABLE_BIT;
reg_val |= ARM_MMU_SCTLR_DCACHE_ENABLE_BIT;
reg_val |= ARM_MMU_SCTLR_MMU_ENABLE_BIT;
__set_SCTLR(reg_val);
return 0;
}
/**
* @brief ARMv7-specific implementation of memory mapping at run-time
* Maps memory according to the parameters provided by the caller
* at run-time.
*
* @param virt 32-bit target virtual address.
* @param phys 32-bit physical address.
* @param size Size (in bytes) of the memory area to map.
* @param flags Memory attributes & permissions. Comp. K_MEM_...
* flags in kernel/mm.h.
* @retval 0 on success, -EINVAL if an invalid parameter is detected.
*/
static int __arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flags)
{
uint32_t va = (uint32_t)virt;
uint32_t pa = (uint32_t)phys;
uint32_t rem_size = (uint32_t)size;
uint32_t conv_flags = MPERM_R;
struct arm_mmu_perms_attrs perms_attrs;
int key;
if (size == 0) {
LOG_ERR("Cannot map physical memory at 0x%08X: invalid "
"zero size", (uint32_t)phys);
return -EINVAL;
}
switch (flags & K_MEM_CACHE_MASK) {
case K_MEM_CACHE_NONE:
default:
conv_flags |= MT_DEVICE;
break;
case K_MEM_CACHE_WB:
conv_flags |= MT_NORMAL;
conv_flags |= MATTR_SHARED;
if (flags & K_MEM_PERM_RW) {
conv_flags |= MATTR_CACHE_OUTER_WB_WA;
conv_flags |= MATTR_CACHE_INNER_WB_WA;
} else {
conv_flags |= MATTR_CACHE_OUTER_WB_nWA;
conv_flags |= MATTR_CACHE_INNER_WB_nWA;
}
break;
case K_MEM_CACHE_WT:
conv_flags |= MT_NORMAL;
conv_flags |= MATTR_SHARED;
conv_flags |= MATTR_CACHE_OUTER_WT_nWA;
conv_flags |= MATTR_CACHE_INNER_WT_nWA;
break;
}
if (flags & K_MEM_PERM_RW) {
conv_flags |= MPERM_W;
}
if (flags & K_MEM_PERM_EXEC) {
conv_flags |= MPERM_X;
}
perms_attrs = arm_mmu_convert_attr_flags(conv_flags);
key = arch_irq_lock();
while (rem_size > 0) {
arm_mmu_l2_map_page(va, pa, perms_attrs);
rem_size -= (rem_size >= KB(4)) ? KB(4) : rem_size;
va += KB(4);
pa += KB(4);
}
arch_irq_unlock(key);
return 0;
}
/**
* @brief Arch-specific wrapper function for memory mapping at run-time
* Maps memory according to the parameters provided by the caller
* at run-time. This function wraps the ARMv7 MMU specific implementation
* #__arch_mem_map() for the upper layers of the memory management.
* If the map operation fails, a kernel panic will be triggered.
*
* @param virt 32-bit target virtual address.
* @param phys 32-bit physical address.
* @param size Size (in bytes) of the memory area to map.
* @param flags Memory attributes & permissions. Comp. K_MEM_...
* flags in kernel/mm.h.
*/
void arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flags)
{
int ret = __arch_mem_map(virt, phys, size, flags);
if (ret) {
LOG_ERR("__arch_mem_map() returned %d", ret);
k_panic();
} else {
invalidate_tlb_all();
}
}
/**
* @brief ARMv7-specific implementation of memory unmapping at run-time
* Unmaps memory according to the parameters provided by the caller
* at run-time.
*
* @param addr 32-bit virtual address to unmap.
* @param size Size (in bytes) of the memory area to unmap.
* @retval 0 on success, -EINVAL if an invalid parameter is detected.
*/
static int __arch_mem_unmap(void *addr, size_t size)
{
uint32_t va = (uint32_t)addr;
uint32_t rem_size = (uint32_t)size;
int key;
if (addr == NULL) {
LOG_ERR("Cannot unmap virtual memory: invalid NULL pointer");
return -EINVAL;
}
if (size == 0) {
LOG_ERR("Cannot unmap virtual memory at 0x%08X: invalid "
"zero size", (uint32_t)addr);
return -EINVAL;
}
key = arch_irq_lock();
while (rem_size > 0) {
arm_mmu_l2_unmap_page(va);
rem_size -= (rem_size >= KB(4)) ? KB(4) : rem_size;
va += KB(4);
}
arch_irq_unlock(key);
return 0;
}
/**
* @brief Arch-specific wrapper function for memory unmapping at run-time
* Unmaps memory according to the parameters provided by the caller
* at run-time. This function wraps the ARMv7 MMU specific implementation
* #__arch_mem_unmap() for the upper layers of the memory management.
*
* @param addr 32-bit virtual address to unmap.
* @param size Size (in bytes) of the memory area to unmap.
*/
void arch_mem_unmap(void *addr, size_t size)
{
int ret = __arch_mem_unmap(addr, size);
if (ret) {
LOG_ERR("__arch_mem_unmap() returned %d", ret);
} else {
invalidate_tlb_all();
}
}
/**
* @brief Arch-specific virtual-to-physical address resolver function
* ARMv7 MMU specific implementation of a function that resolves the
* physical address corresponding to the given virtual address.
*
* @param virt 32-bit target virtual address to resolve.
* @param phys Pointer to a variable to which the resolved physical
* address will be written. May be NULL if this information
* is not actually required by the caller.
* @retval 0 if the physical address corresponding to the specified
* virtual address could be resolved successfully, -EFAULT
* if the specified virtual address is not currently mapped.
*/
int arch_page_phys_get(void *virt, uintptr_t *phys)
{
uint32_t l1_index = ((uint32_t)virt >> ARM_MMU_PTE_L1_INDEX_PA_SHIFT) &
ARM_MMU_PTE_L1_INDEX_MASK;
uint32_t l2_index = ((uint32_t)virt >> ARM_MMU_PTE_L2_INDEX_PA_SHIFT) &
ARM_MMU_PTE_L2_INDEX_MASK;
struct arm_mmu_l2_page_table *l2_page_table;
uint32_t pa_resolved = 0;
uint32_t l2_pt_resolved;
int rc = 0;
int key;
key = arch_irq_lock();
if (l1_page_table.entries[l1_index].undefined.id == ARM_MMU_PTE_ID_SECTION) {
/*
* If the virtual address points to a level 1 PTE whose ID bits
* identify it as a 1 MB section entry rather than a level 2 PT
* entry, the given VA belongs to a memory region used by the
* Zephyr image itself - it is only for those static regions that
* L1 Section entries are used to save L2 tables if a sufficient-
* ly large block of memory is specified. The memory regions be-
* longing to the Zephyr image are identity mapped -> just return
* the value of the VA as the value of the PA.
*/
pa_resolved = (uint32_t)virt;
} else if (l1_page_table.entries[l1_index].undefined.id == ARM_MMU_PTE_ID_L2_PT) {
/*
* The VA points to a level 1 PTE which re-directs to a level 2
* PT. -> Assemble the level 2 PT pointer and resolve the PA for
* the specified VA from there.
*/
l2_pt_resolved =
l1_page_table.entries[l1_index].l2_page_table_ref.l2_page_table_address;
l2_pt_resolved <<= ARM_MMU_PT_L2_ADDR_SHIFT;
l2_page_table = (struct arm_mmu_l2_page_table *)l2_pt_resolved;
/*
* Check if the PTE for the specified VA is actually in use before
* assembling & returning the corresponding PA. k_mem_unmap will
* call this function for the leading & trailing guard pages when
* unmapping a VA. As those guard pages were explicitly unmapped
* when the VA was originally mapped, their L2 PTEs will be empty.
* In that case, the return code of this function must not be 0.
*/
if (l2_page_table->entries[l2_index].word == 0) {
rc = -EFAULT;
}
pa_resolved = l2_page_table->entries[l2_index].l2_page_4k.pa_base;
pa_resolved <<= ARM_MMU_PTE_L2_SMALL_PAGE_ADDR_SHIFT;
pa_resolved |= ((uint32_t)virt & ARM_MMU_ADDR_BELOW_PAGE_GRAN_MASK);
} else {
/* The level 1 PTE is invalid -> the specified VA is not mapped */
rc = -EFAULT;
}
arch_irq_unlock(key);
if (phys) {
*phys = (uintptr_t)pa_resolved;
}
return rc;
}
Reference in New Issue View Git Blame Copy Permalink