libs/rt-thread-official
libs/rt-thread-official
4
0
Fork 0
mirror of https://github.com/RT-Thread/rt-thread.git synced 2025-02-01 19:50:47 +08:00
Code Issues Releases Wiki Activity
rt-thread-official/libcpu/aarch64/common/mmu.c
Shell fe2b124345 feat: arm64 ASID support 
Support for ARM64 ASID to enhance virtual memory management efficiency
by reducing the need for TLB flushes during address space switches.
These changes improve performance especially for multi-process systems.

Changes:
- Added `ARCH_USING_ASID` configuration in `libcpu/aarch64/Kconfig`.
- Defined ASID-related constants in `mmu.h`.
- Updated `TLBI_ARG` macro to include ASID manipulation.
- Implemented ASID allocation mechanism with spinlock synchronization.
- Enhanced TLB invalidation to support ASID-specific operations.
- Modified `rt_hw_aspace_switch` to use ASIDs when switching address spaces.
- Adjusted debug logging and function documentation to reflect ASID usage.
- Refactored AArch64 MMU and TLB handling for ASID integration.

Signed-off-by: Shell <smokewood@qq.com>
2024-11-24 13:44:34 -05:00

941 lines
25 KiB
C
Raw Blame History

/*
* Copyright (c) 2006-2023, RT-Thread Development Team
*
* SPDX-License-Identifier: Apache-2.0
*
* Change Logs:
* Date Author Notes
* 2012-01-10 bernard porting to AM1808
* 2021-11-28 GuEe-GUI first version
* 2022-12-10 WangXiaoyao porting to MM
* 2024-07-08 Shell added support for ASID
*/
#define DBG_TAG "hw.mmu"
#define DBG_LVL DBG_INFO
#include <rtdbg.h>
#include <rthw.h>
#include <rtthread.h>
#include <stddef.h>
#include <stdint.h>
#include <string.h>
#define __MMU_INTERNAL
#include "mm_aspace.h"
#include "mm_page.h"
#include "mmu.h"
#include "tlb.h"
#include "ioremap.h"
#ifdef RT_USING_SMART
#include <lwp_mm.h>
#endif
#define TCR_CONFIG_TBI0 rt_hw_mmu_config_tbi(0)
#define TCR_CONFIG_TBI1 rt_hw_mmu_config_tbi(1)
#define MMU_LEVEL_MASK 0x1ffUL
#define MMU_LEVEL_SHIFT 9
#define MMU_ADDRESS_BITS 39
#define MMU_ADDRESS_MASK 0x0000fffffffff000UL
#define MMU_ATTRIB_MASK 0xfff0000000000ffcUL
#define MMU_TYPE_MASK 3UL
#define MMU_TYPE_USED 1UL
#define MMU_TYPE_BLOCK 1UL
#define MMU_TYPE_TABLE 3UL
#define MMU_TYPE_PAGE 3UL
#define MMU_TBL_BLOCK_2M_LEVEL 2
#define MMU_TBL_PAGE_4k_LEVEL 3
#define MMU_TBL_LEVEL_NR 4
/* restrict virtual address on usage of RT_NULL */
#ifndef KERNEL_VADDR_START
#define KERNEL_VADDR_START 0x1000
#endif
volatile unsigned long MMUTable[512] __attribute__((aligned(4 * 1024)));
struct mmu_level_info
{
unsigned long *pos;
void *page;
};
static void _kenrel_unmap_4K(unsigned long *lv0_tbl, void *v_addr)
{
int level;
unsigned long va = (unsigned long)v_addr;
unsigned long *cur_lv_tbl = lv0_tbl;
unsigned long page;
unsigned long off;
struct mmu_level_info level_info[4];
int ref;
int level_shift = MMU_ADDRESS_BITS;
unsigned long *pos;
rt_memset(level_info, 0, sizeof level_info);
for (level = 0; level < MMU_TBL_LEVEL_NR; level++)
{
off = (va >> level_shift);
off &= MMU_LEVEL_MASK;
page = cur_lv_tbl[off];
if (!(page & MMU_TYPE_USED))
{
break;
}
if ((page & MMU_TYPE_MASK) == MMU_TYPE_BLOCK)
{
break;
}
/* next table entry in current level */
level_info[level].pos = cur_lv_tbl + off;
cur_lv_tbl = (unsigned long *)(page & MMU_ADDRESS_MASK);
cur_lv_tbl = (unsigned long *)((unsigned long)cur_lv_tbl - PV_OFFSET);
level_info[level].page = cur_lv_tbl;
level_shift -= MMU_LEVEL_SHIFT;
}
level = MMU_TBL_PAGE_4k_LEVEL;
pos = level_info[level].pos;
if (pos)
{
*pos = (unsigned long)RT_NULL;
rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, pos, sizeof(void *));
}
level--;
while (level >= 0)
{
pos = level_info[level].pos;
if (pos)
{
void *cur_page = level_info[level].page;
ref = rt_page_ref_get(cur_page, 0);
if (ref == 1)
{
*pos = (unsigned long)RT_NULL;
rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, pos, sizeof(void *));
}
rt_pages_free(cur_page, 0);
}
else
{
break;
}
level--;
}
return;
}
static int _kernel_map_4K(unsigned long *lv0_tbl, void *vaddr, void *paddr, unsigned long attr)
{
int ret = 0;
int level;
unsigned long *cur_lv_tbl = lv0_tbl;
unsigned long page;
unsigned long off;
rt_ubase_t va = (rt_ubase_t)vaddr;
rt_ubase_t pa = (rt_ubase_t)paddr;
int level_shift = MMU_ADDRESS_BITS;
if (va & ARCH_PAGE_MASK)
{
return MMU_MAP_ERROR_VANOTALIGN;
}
if (pa & ARCH_PAGE_MASK)
{
return MMU_MAP_ERROR_PANOTALIGN;
}
for (level = 0; level < MMU_TBL_PAGE_4k_LEVEL; level++)
{
off = (va >> level_shift);
off &= MMU_LEVEL_MASK;
if (!(cur_lv_tbl[off] & MMU_TYPE_USED))
{
page = (unsigned long)rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE);
if (!page)
{
ret = MMU_MAP_ERROR_NOPAGE;
goto err;
}
rt_memset((void *)page, 0, ARCH_PAGE_SIZE);
rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, (void *)page, ARCH_PAGE_SIZE);
cur_lv_tbl[off] = (page + PV_OFFSET) | MMU_TYPE_TABLE;
rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off, sizeof(void *));
}
else
{
page = cur_lv_tbl[off];
page &= MMU_ADDRESS_MASK;
/* page to va */
page -= PV_OFFSET;
rt_page_ref_inc((void *)page, 0);
}
page = cur_lv_tbl[off];
if ((page & MMU_TYPE_MASK) == MMU_TYPE_BLOCK)
{
/* is block! error! */
ret = MMU_MAP_ERROR_CONFLICT;
goto err;
}
cur_lv_tbl = (unsigned long *)(page & MMU_ADDRESS_MASK);
cur_lv_tbl = (unsigned long *)((unsigned long)cur_lv_tbl - PV_OFFSET);
level_shift -= MMU_LEVEL_SHIFT;
}
/* now is level page */
attr &= MMU_ATTRIB_MASK;
pa |= (attr | MMU_TYPE_PAGE); /* page */
off = (va >> ARCH_PAGE_SHIFT);
off &= MMU_LEVEL_MASK;
cur_lv_tbl[off] = pa; /* page */
rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off, sizeof(void *));
return ret;
err:
_kenrel_unmap_4K(lv0_tbl, (void *)va);
return ret;
}
static int _kernel_map_2M(unsigned long *lv0_tbl, void *vaddr, void *paddr, unsigned long attr)
{
int ret = 0;
int level;
unsigned long *cur_lv_tbl = lv0_tbl;
unsigned long page;
unsigned long off;
unsigned long va = (unsigned long)vaddr;
unsigned long pa = (unsigned long)paddr;
int level_shift = MMU_ADDRESS_BITS;
if (va & ARCH_SECTION_MASK)
{
return MMU_MAP_ERROR_VANOTALIGN;
}
if (pa & ARCH_PAGE_MASK)
{
return MMU_MAP_ERROR_PANOTALIGN;
}
for (level = 0; level < MMU_TBL_BLOCK_2M_LEVEL; level++)
{
off = (va >> level_shift);
off &= MMU_LEVEL_MASK;
if (!(cur_lv_tbl[off] & MMU_TYPE_USED))
{
page = (unsigned long)rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE);
if (!page)
{
ret = MMU_MAP_ERROR_NOPAGE;
goto err;
}
rt_memset((char *)page, 0, ARCH_PAGE_SIZE);
rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, (void *)page, ARCH_PAGE_SIZE);
cur_lv_tbl[off] = (page + PV_OFFSET) | MMU_TYPE_TABLE;
rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off, sizeof(void *));
}
else
{
page = cur_lv_tbl[off];
page &= MMU_ADDRESS_MASK;
/* page to va */
page -= PV_OFFSET;
rt_page_ref_inc((void *)page, 0);
}
page = cur_lv_tbl[off];
if ((page & MMU_TYPE_MASK) == MMU_TYPE_BLOCK)
{
/* is block! error! */
ret = MMU_MAP_ERROR_CONFLICT;
goto err;
}
cur_lv_tbl = (unsigned long *)(page & MMU_ADDRESS_MASK);
cur_lv_tbl = (unsigned long *)((unsigned long)cur_lv_tbl - PV_OFFSET);
level_shift -= MMU_LEVEL_SHIFT;
}
/* now is level page */
attr &= MMU_ATTRIB_MASK;
pa |= (attr | MMU_TYPE_BLOCK); /* block */
off = (va >> ARCH_SECTION_SHIFT);
off &= MMU_LEVEL_MASK;
cur_lv_tbl[off] = pa;
rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off, sizeof(void *));
return ret;
err:
_kenrel_unmap_4K(lv0_tbl, (void *)va);
return ret;
}
void *rt_hw_mmu_map(rt_aspace_t aspace, void *v_addr, void *p_addr, size_t size,
size_t attr)
{
int ret = -1;
void *unmap_va = v_addr;
size_t npages;
size_t stride;
int (*mapper)(unsigned long *lv0_tbl, void *vaddr, void *paddr, unsigned long attr);
if (((rt_ubase_t)v_addr & ARCH_SECTION_MASK) || (size & ARCH_SECTION_MASK))
{
/* legacy 4k mapping */
npages = size >> ARCH_PAGE_SHIFT;
stride = ARCH_PAGE_SIZE;
mapper = _kernel_map_4K;
}
else
{
/* 2m huge page */
npages = size >> ARCH_SECTION_SHIFT;
stride = ARCH_SECTION_SIZE;
mapper = _kernel_map_2M;
}
while (npages--)
{
MM_PGTBL_LOCK(aspace);
ret = mapper(aspace->page_table, v_addr, p_addr, attr);
MM_PGTBL_UNLOCK(aspace);
if (ret != 0)
{
/* other types of return value are taken as programming error */
RT_ASSERT(ret == MMU_MAP_ERROR_NOPAGE);
/* error, undo map */
while (unmap_va != v_addr)
{
MM_PGTBL_LOCK(aspace);
_kenrel_unmap_4K(aspace->page_table, (void *)unmap_va);
MM_PGTBL_UNLOCK(aspace);
unmap_va = (char *)unmap_va + stride;
}
break;
}
v_addr = (char *)v_addr + stride;
p_addr = (char *)p_addr + stride;
}
if (ret == 0)
{
return unmap_va;
}
return NULL;
}
void rt_hw_mmu_unmap(rt_aspace_t aspace, void *v_addr, size_t size)
{
// caller guarantee that v_addr & size are page aligned
size_t npages = size >> ARCH_PAGE_SHIFT;
if (!aspace->page_table)
{
return;
}
while (npages--)
{
MM_PGTBL_LOCK(aspace);
if (rt_hw_mmu_v2p(aspace, v_addr) != ARCH_MAP_FAILED)
_kenrel_unmap_4K(aspace->page_table, v_addr);
MM_PGTBL_UNLOCK(aspace);
v_addr = (char *)v_addr + ARCH_PAGE_SIZE;
}
}
#ifdef ARCH_USING_ASID
/**
* the asid is to identified specialized address space on TLB.
* In the best case, each address space has its own exclusive asid. However,
* ARM only guarantee with 8 bits of ID space, which give us only 254(except
* the reserved 1 ASID for kernel).
*/
static rt_spinlock_t _asid_lock = RT_SPINLOCK_INIT;
rt_uint16_t _aspace_get_asid(rt_aspace_t aspace)
{
static rt_uint16_t _asid_pool = 0;
rt_uint16_t asid_to, asid_from;
rt_ubase_t ttbr0_from;
asid_to = aspace->asid;
if (asid_to == 0)
{
rt_spin_lock(&_asid_lock);
#define MAX_ASID (1ul << MMU_SUPPORTED_ASID_BITS)
if (_asid_pool && _asid_pool < MAX_ASID)
{
asid_to = ++_asid_pool;
LOG_D("Allocated ASID %d to PID %d(aspace %p)", asid_to, lwp_self()->pid, aspace);
}
else
{
asid_to = _asid_pool = 1;
LOG_D("Overflowed ASID %d to PID %d(aspace %p)", asid_to, lwp_self()->pid, aspace);
}
rt_spin_unlock(&_asid_lock);
aspace->asid = asid_to;
rt_hw_tlb_invalidate_aspace(aspace);
}
__asm__ volatile("mrs %0, ttbr0_el1" :"=r"(ttbr0_from));
asid_from = ttbr0_from >> MMU_ASID_SHIFT;
if (asid_from == asid_to)
{
LOG_D("Conflict ASID. from %d, to %d", asid_from, asid_to);
rt_hw_tlb_invalidate_aspace(aspace);
}
else
{
LOG_D("ASID switched. from %d, to %d", asid_from, asid_to);
}
return asid_to;
}
#else
rt_uint16_t _aspace_get_asid(rt_aspace_t aspace)
{
rt_hw_tlb_invalidate_all();
return 0;
}
#endif /* ARCH_USING_ASID */
#define CREATE_TTBR0(pgtbl, asid) ((rt_ubase_t)(pgtbl) | (rt_ubase_t)(asid) << MMU_ASID_SHIFT)
void rt_hw_aspace_switch(rt_aspace_t aspace)
{
if (aspace != &rt_kernel_space)
{
rt_ubase_t ttbr0;
void *pgtbl = aspace->page_table;
pgtbl = rt_kmem_v2p(pgtbl);
ttbr0 = CREATE_TTBR0(pgtbl, _aspace_get_asid(aspace));
__asm__ volatile("msr ttbr0_el1, %0" ::"r"(ttbr0));
__asm__ volatile("isb" ::: "memory");
}
}
void rt_hw_mmu_ktbl_set(unsigned long tbl)
{
#ifdef RT_USING_SMART
tbl += PV_OFFSET;
__asm__ volatile("msr TTBR1_EL1, %0\n dsb sy\nisb" ::"r"(tbl) : "memory");
#else
__asm__ volatile("msr TTBR0_EL1, %0\n dsb sy\nisb" ::"r"(tbl) : "memory");
#endif
__asm__ volatile("tlbi vmalle1\n dsb sy\nisb" ::: "memory");
__asm__ volatile("ic ialluis\n dsb sy\nisb" ::: "memory");
}
/**
* @brief setup Page Table for kernel space. It's a fixed map
* and all mappings cannot be changed after initialization.
*
* Memory region in struct mem_desc must be page aligned,
* otherwise is a failure and no report will be
* returned.
*
* @param mmu_info
* @param mdesc
* @param desc_nr
*/
void rt_hw_mmu_setup(rt_aspace_t aspace, struct mem_desc *mdesc, int desc_nr)
{
void *err;
for (size_t i = 0; i < desc_nr; i++)
{
size_t attr;
switch (mdesc->attr)
{
case NORMAL_MEM:
attr = MMU_MAP_K_RWCB;
break;
case NORMAL_NOCACHE_MEM:
attr = MMU_MAP_K_RWCB;
break;
case DEVICE_MEM:
attr = MMU_MAP_K_DEVICE;
break;
default:
attr = MMU_MAP_K_DEVICE;
}
struct rt_mm_va_hint hint = {.flags = MMF_MAP_FIXED,
.limit_start = aspace->start,
.limit_range_size = aspace->size,
.map_size = mdesc->vaddr_end -
mdesc->vaddr_start + 1,
.prefer = (void *)mdesc->vaddr_start};
if (mdesc->paddr_start == (rt_size_t)ARCH_MAP_FAILED)
mdesc->paddr_start = mdesc->vaddr_start + PV_OFFSET;
int retval;
retval = rt_aspace_map_phy_static(aspace, &mdesc->varea, &hint, attr,
mdesc->paddr_start >> MM_PAGE_SHIFT, &err);
if (retval)
{
LOG_E("%s: map failed with code %d", __FUNCTION__, retval);
RT_ASSERT(0);
}
mdesc++;
}
rt_hw_mmu_ktbl_set((unsigned long)rt_kernel_space.page_table);
rt_page_cleanup();
}
static void _init_region(void *vaddr, size_t size)
{
rt_ioremap_start = vaddr;
rt_ioremap_size = size;
rt_mpr_start = (char *)rt_ioremap_start - rt_mpr_size;
}
/**
* This function will initialize rt_mmu_info structure.
*
* @param mmu_info rt_mmu_info structure
* @param v_address virtual address
* @param size map size
* @param vtable mmu table
* @param pv_off pv offset in kernel space
*
* @return 0 on successful and -1 for fail
*/
int rt_hw_mmu_map_init(rt_aspace_t aspace, void *v_address, size_t size,
size_t *vtable, size_t pv_off)
{
size_t va_s, va_e;
if (!aspace || !vtable)
{
return -1;
}
va_s = (size_t)v_address;
va_e = (size_t)v_address + size - 1;
if (va_e < va_s)
{
return -1;
}
va_s >>= ARCH_SECTION_SHIFT;
va_e >>= ARCH_SECTION_SHIFT;
if (va_s == 0)
{
return -1;
}
rt_aspace_init(aspace, (void *)KERNEL_VADDR_START, 0 - KERNEL_VADDR_START,
vtable);
_init_region(v_address, size);
return 0;
}
rt_weak long rt_hw_mmu_config_tbi(int tbi_index)
{
return 0;
}
/************ setting el1 mmu register**************
MAIR_EL1
index 0 : memory outer writeback, write/read alloc
index 1 : memory nocache
index 2 : device nGnRnE
*****************************************************/
void mmu_tcr_init(void)
{
unsigned long val64;
unsigned long pa_range;
val64 = 0x00447fUL;
__asm__ volatile("msr MAIR_EL1, %0\n dsb sy\n" ::"r"(val64));
__asm__ volatile ("mrs %0, ID_AA64MMFR0_EL1":"=r"(val64));
pa_range = val64 & 0xf; /* PARange */
/* TCR_EL1 */
val64 = (16UL << 0) /* t0sz 48bit */
| (0x0UL << 6) /* reserved */
| (0x0UL << 7) /* epd0 */
| (0x3UL << 8) /* t0 wb cacheable */
| (0x3UL << 10) /* inner shareable */
| (0x2UL << 12) /* t0 outer shareable */
| (0x0UL << 14) /* t0 4K */
| (16UL << 16) /* t1sz 48bit */
| (0x0UL << 22) /* define asid use ttbr0.asid */
| (0x0UL << 23) /* epd1 */
| (0x3UL << 24) /* t1 inner wb cacheable */
| (0x3UL << 26) /* t1 outer wb cacheable */
| (0x2UL << 28) /* t1 outer shareable */
| (0x2UL << 30) /* t1 4k */
| (pa_range << 32) /* PA range */
| (0x0UL << 35) /* reserved */
| (0x1UL << 36) /* as: 0:8bit 1:16bit */
| (TCR_CONFIG_TBI0 << 37) /* tbi0 */
| (TCR_CONFIG_TBI1 << 38); /* tbi1 */
__asm__ volatile("msr TCR_EL1, %0\n" ::"r"(val64));
}
struct page_table
{
unsigned long page[512];
};
/* */
static struct page_table* __init_page_array;
static unsigned long __page_off = 0UL;
unsigned long get_ttbrn_base(void)
{
return (unsigned long) __init_page_array;
}
void set_free_page(void *page_array)
{
__init_page_array = page_array;
}
unsigned long get_free_page(void)
{
return (unsigned long) (__init_page_array[__page_off++].page);
}
static int _map_single_page_2M(unsigned long *lv0_tbl, unsigned long va,
unsigned long pa, unsigned long attr,
rt_bool_t flush)
{
int level;
unsigned long *cur_lv_tbl = lv0_tbl;
unsigned long page;
unsigned long off;
int level_shift = MMU_ADDRESS_BITS;
if (va & ARCH_SECTION_MASK)
{
return MMU_MAP_ERROR_VANOTALIGN;
}
if (pa & ARCH_PAGE_MASK)
{
return MMU_MAP_ERROR_PANOTALIGN;
}
for (level = 0; level < MMU_TBL_BLOCK_2M_LEVEL; level++)
{
off = (va >> level_shift);
off &= MMU_LEVEL_MASK;
if (!(cur_lv_tbl[off] & MMU_TYPE_USED))
{
page = get_free_page();
if (!page)
{
return MMU_MAP_ERROR_NOPAGE;
}
rt_memset((char *)page, 0, ARCH_PAGE_SIZE);
cur_lv_tbl[off] = page | MMU_TYPE_TABLE;
if (flush)
{
rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off, sizeof(void *));
}
}
page = cur_lv_tbl[off];
if ((page & MMU_TYPE_MASK) == MMU_TYPE_BLOCK)
{
/* is block! error! */
return MMU_MAP_ERROR_CONFLICT;
}
cur_lv_tbl = (unsigned long *)(page & MMU_ADDRESS_MASK);
level_shift -= MMU_LEVEL_SHIFT;
}
attr &= MMU_ATTRIB_MASK;
pa |= (attr | MMU_TYPE_BLOCK); /* block */
off = (va >> ARCH_SECTION_SHIFT);
off &= MMU_LEVEL_MASK;
cur_lv_tbl[off] = pa;
if (flush)
{
rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off, sizeof(void *));
}
return 0;
}
void *rt_hw_mmu_tbl_get(void)
{
uintptr_t tbl;
__asm__ volatile("MRS %0, TTBR0_EL1" : "=r"(tbl));
return rt_kmem_p2v((void *)(tbl & ((1ul << 48) - 2)));
}
void *rt_ioremap_early(void *paddr, size_t size)
{
volatile size_t count;
rt_ubase_t base;
static void *tbl = RT_NULL;
if (!size)
{
return RT_NULL;
}
if (!tbl)
{
tbl = rt_hw_mmu_tbl_get();
}
/* get the total size required including overhead for alignment */
count = (size + ((rt_ubase_t)paddr & ARCH_SECTION_MASK)
+ ARCH_SECTION_MASK) >> ARCH_SECTION_SHIFT;
base = (rt_ubase_t)paddr & (~ARCH_SECTION_MASK);
while (count --> 0)
{
if (_map_single_page_2M(tbl, base, base, MMU_MAP_K_DEVICE, RT_TRUE))
{
return RT_NULL;
}
base += ARCH_SECTION_SIZE;
}
return paddr;
}
static int _init_map_2M(unsigned long *lv0_tbl, unsigned long va,
unsigned long pa, unsigned long count,
unsigned long attr)
{
unsigned long i;
int ret;
if (va & ARCH_SECTION_MASK)
{
return -1;
}
if (pa & ARCH_SECTION_MASK)
{
return -1;
}
for (i = 0; i < count; i++)
{
ret = _map_single_page_2M(lv0_tbl, va, pa, attr, RT_FALSE);
va += ARCH_SECTION_SIZE;
pa += ARCH_SECTION_SIZE;
if (ret != 0)
{
return ret;
}
}
return 0;
}
static unsigned long *_query(rt_aspace_t aspace, void *vaddr, int *plvl_shf)
{
int level;
unsigned long va = (unsigned long)vaddr;
unsigned long *cur_lv_tbl;
unsigned long page;
unsigned long off;
int level_shift = MMU_ADDRESS_BITS;
cur_lv_tbl = aspace->page_table;
RT_ASSERT(cur_lv_tbl);
for (level = 0; level < MMU_TBL_PAGE_4k_LEVEL; level++)
{
off = (va >> level_shift);
off &= MMU_LEVEL_MASK;
if (!(cur_lv_tbl[off] & MMU_TYPE_USED))
{
*plvl_shf = level_shift;
return (void *)0;
}
page = cur_lv_tbl[off];
if ((page & MMU_TYPE_MASK) == MMU_TYPE_BLOCK)
{
*plvl_shf = level_shift;
return &cur_lv_tbl[off];
}
cur_lv_tbl = (unsigned long *)(page & MMU_ADDRESS_MASK);
cur_lv_tbl = (unsigned long *)((unsigned long)cur_lv_tbl - PV_OFFSET);
level_shift -= MMU_LEVEL_SHIFT;
}
/* now is level MMU_TBL_PAGE_4k_LEVEL */
off = (va >> ARCH_PAGE_SHIFT);
off &= MMU_LEVEL_MASK;
page = cur_lv_tbl[off];
*plvl_shf = level_shift;
if (!(page & MMU_TYPE_USED))
{
return (void *)0;
}
return &cur_lv_tbl[off];
}
void *rt_hw_mmu_v2p(rt_aspace_t aspace, void *v_addr)
{
int level_shift;
unsigned long paddr;
if (aspace == &rt_kernel_space)
{
paddr = (unsigned long)rt_hw_mmu_kernel_v2p(v_addr);
}
else
{
unsigned long *pte = _query(aspace, v_addr, &level_shift);
if (pte)
{
paddr = *pte & MMU_ADDRESS_MASK;
paddr |= (rt_ubase_t)v_addr & ((1ul << level_shift) - 1);
}
else
{
paddr = (unsigned long)ARCH_MAP_FAILED;
}
}
return (void *)paddr;
}
static int _noncache(rt_ubase_t *pte)
{
int err = 0;
const rt_ubase_t idx_shift = 2;
const rt_ubase_t idx_mask = 0x7 << idx_shift;
rt_ubase_t entry = *pte;
if ((entry & idx_mask) == (NORMAL_MEM << idx_shift))
{
*pte = (entry & ~idx_mask) | (NORMAL_NOCACHE_MEM << idx_shift);
}
else
{
// do not support other type to be noncache
err = -RT_ENOSYS;
}
return err;
}
static int _cache(rt_ubase_t *pte)
{
int err = 0;
const rt_ubase_t idx_shift = 2;
const rt_ubase_t idx_mask = 0x7 << idx_shift;
rt_ubase_t entry = *pte;
if ((entry & idx_mask) == (NORMAL_NOCACHE_MEM << idx_shift))
{
*pte = (entry & ~idx_mask) | (NORMAL_MEM << idx_shift);
}
else
{
// do not support other type to be cache
err = -RT_ENOSYS;
}
return err;
}
static int (*control_handler[MMU_CNTL_DUMMY_END])(rt_ubase_t *pte) = {
[MMU_CNTL_CACHE] = _cache,
[MMU_CNTL_NONCACHE] = _noncache,
};
int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size,
enum rt_mmu_cntl cmd)
{
int level_shift;
int err = -RT_EINVAL;
rt_ubase_t vstart = (rt_ubase_t)vaddr;
rt_ubase_t vend = vstart + size;
int (*handler)(rt_ubase_t * pte);
if (cmd >= 0 && cmd < MMU_CNTL_DUMMY_END)
{
handler = control_handler[cmd];
while (vstart < vend)
{
rt_ubase_t *pte = _query(aspace, (void *)vstart, &level_shift);
rt_ubase_t range_end = vstart + (1ul << level_shift);
RT_ASSERT(range_end <= vend);
if (pte)
{
err = handler(pte);
RT_ASSERT(err == RT_EOK);
}
vstart = range_end;
}
}
else
{
err = -RT_ENOSYS;
}
return err;
}
void rt_hw_mem_setup_early(unsigned long *tbl0, unsigned long *tbl1,
unsigned long size, unsigned long pv_off)
{
int ret;
unsigned long count = (size + ARCH_SECTION_MASK) >> ARCH_SECTION_SHIFT;
unsigned long normal_attr = MMU_MAP_K_RWCB;
extern unsigned char _start;
unsigned long va = (unsigned long) &_start - pv_off;
va = RT_ALIGN_DOWN(va, 0x200000);
/* setup pv off */
rt_kmem_pvoff_set(pv_off);
/* clean the first two pages */
rt_memset((char *)tbl0, 0, ARCH_PAGE_SIZE);
rt_memset((char *)tbl1, 0, ARCH_PAGE_SIZE);
ret = _init_map_2M(tbl1, va, va + pv_off, count, normal_attr);
if (ret != 0)
{
while (1);
}
ret = _init_map_2M(tbl0, va + pv_off, va + pv_off, count, normal_attr);
if (ret != 0)
{
while (1);
}
}
void *rt_hw_mmu_pgtbl_create(void)
{
size_t *mmu_table;
mmu_table = (size_t *)rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE);
if (!mmu_table)
{
return RT_NULL;
}
memset(mmu_table, 0, ARCH_PAGE_SIZE);
rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_table, ARCH_PAGE_SIZE);
return mmu_table;
}
void rt_hw_mmu_pgtbl_delete(void *pgtbl)
{
rt_pages_free(pgtbl, 0);
}
View Git Blame Copy Permalink