rt-thread/libcpu/aarch64/common/mmu.c

1032 lines
25 KiB
C
Raw Normal View History

/*
* Copyright (c) 2006-2018, RT-Thread Development Team
*
2020-01-10 10:38:21 +08:00
* SPDX-License-Identifier: Apache-2.0
*
* Change Logs:
2022-01-07 13:49:06 +08:00
* Date Author Notes
* 2012-01-10 bernard porting to AM1808
*/
2020-02-20 15:42:10 +08:00
2022-01-07 13:49:06 +08:00
#include <rtthread.h>
#include <rthw.h>
#include <board.h>
2020-02-20 15:42:10 +08:00
#include "mmu.h"
#ifdef RT_USING_SMART
#include <lwp_mm.h>
#include <page.h>
#endif
#define MMU_LEVEL_MASK 0x1ffUL
#define MMU_LEVEL_SHIFT 9
#define MMU_ADDRESS_BITS 39
#define MMU_ADDRESS_MASK 0x0000fffffffff000UL
#define MMU_ATTRIB_MASK 0xfff0000000000ffcUL
#define MMU_TYPE_MASK 3UL
#define MMU_TYPE_USED 1UL
#define MMU_TYPE_BLOCK 1UL
#define MMU_TYPE_TABLE 3UL
#define MMU_TYPE_PAGE 3UL
#define MMU_TBL_BLOCK_2M_LEVEL 2
#define MMU_TBL_PAGE_4k_LEVEL 3
#define MMU_TBL_LEVEL_NR 4
void *_rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void *v_addr);
struct page_table
{
unsigned long page[512];
};
#ifndef RT_USING_SMART
2022-01-07 13:49:06 +08:00
#define MMU_TBL_PAGE_NR_MAX 32
2020-03-17 13:45:13 +08:00
static rt_mutex_t mm_lock = RT_NULL;
void rt_mm_lock(void)
{
if (rt_thread_self())
{
if (!mm_lock)
{
mm_lock = rt_mutex_create("mm_lock", RT_IPC_FLAG_FIFO);
}
if (mm_lock)
{
rt_mutex_take(mm_lock, RT_WAITING_FOREVER);
}
}
}
void rt_mm_unlock(void)
2020-03-17 13:45:13 +08:00
{
if (rt_thread_self())
{
if (mm_lock)
{
rt_mutex_release(mm_lock);
}
}
}
2020-03-17 13:45:13 +08:00
static volatile struct page_table MMUPage[MMU_TBL_PAGE_NR_MAX] __attribute__((aligned(4096)));
#define rt_page_ref_inc(...)
unsigned long rt_pages_alloc(rt_size_t size_bits)
2020-02-20 15:42:10 +08:00
{
2022-01-07 13:49:06 +08:00
static unsigned long i = 0;
2020-02-20 15:42:10 +08:00
2022-01-07 13:49:06 +08:00
if (i >= MMU_TBL_PAGE_NR_MAX)
{
return RT_NULL;
}
2020-02-20 15:42:10 +08:00
2022-01-07 13:49:06 +08:00
++i;
2020-03-17 13:45:13 +08:00
return (unsigned long)&MMUPage[i - 1].page;
}
#endif
static struct page_table *__init_page_array;
static unsigned long __page_off = 0UL;
unsigned long get_free_page(void)
{
if (!__init_page_array)
{
unsigned long temp_page_start;
asm volatile("mov %0, sp":"=r"(temp_page_start));
__init_page_array = (struct page_table *)(temp_page_start & ~(ARCH_SECTION_MASK));
__page_off = 2; /* 0, 1 for ttbr0, ttrb1 */
}
__page_off++;
return (unsigned long)(__init_page_array[__page_off - 1].page);
2020-02-20 15:42:10 +08:00
}
void mmu_memset(char *dst, char v, size_t len)
{
while (len--)
{
*dst++ = v;
}
}
static int _map_single_page_2M(unsigned long *lv0_tbl, unsigned long va, unsigned long pa, unsigned long attr)
{
2020-02-20 15:42:10 +08:00
int level;
unsigned long *cur_lv_tbl = lv0_tbl;
2020-02-20 15:42:10 +08:00
unsigned long page;
unsigned long off;
2022-01-07 13:49:06 +08:00
int level_shift = MMU_ADDRESS_BITS;
2020-02-20 15:42:10 +08:00
2022-01-07 13:49:06 +08:00
if (va & ARCH_SECTION_MASK)
{
2020-02-20 15:42:10 +08:00
return MMU_MAP_ERROR_VANOTALIGN;
}
2022-01-07 13:49:06 +08:00
if (pa & ARCH_SECTION_MASK)
{
2020-02-20 15:42:10 +08:00
return MMU_MAP_ERROR_PANOTALIGN;
}
for (level = 0; level < MMU_TBL_BLOCK_2M_LEVEL; level++)
{
2020-02-20 15:42:10 +08:00
off = (va >> level_shift);
off &= MMU_LEVEL_MASK;
2022-01-07 13:49:06 +08:00
if (!(cur_lv_tbl[off] & MMU_TYPE_USED))
{
page = get_free_page();
2021-03-27 17:51:56 +08:00
if (!page)
{
2020-02-20 15:42:10 +08:00
return MMU_MAP_ERROR_NOPAGE;
}
mmu_memset((char *)page, 0, ARCH_PAGE_SIZE);
cur_lv_tbl[off] = page | MMU_TYPE_TABLE;
}
page = cur_lv_tbl[off];
if ((page & MMU_TYPE_MASK) == MMU_TYPE_BLOCK)
{
/* is block! error! */
return MMU_MAP_ERROR_CONFLICT;
}
cur_lv_tbl = (unsigned long *)(page & MMU_ADDRESS_MASK);
level_shift -= MMU_LEVEL_SHIFT;
}
attr &= MMU_ATTRIB_MASK;
pa |= (attr | MMU_TYPE_BLOCK); /* block */
off = (va >> ARCH_SECTION_SHIFT);
off &= MMU_LEVEL_MASK;
cur_lv_tbl[off] = pa;
return 0;
}
int armv8_init_map_2M(unsigned long *lv0_tbl, unsigned long va, unsigned long pa, unsigned long count, unsigned long attr)
{
unsigned long i;
int ret;
if (va & ARCH_SECTION_MASK)
{
return -1;
}
if (pa & ARCH_SECTION_MASK)
{
return -1;
}
for (i = 0; i < count; i++)
{
ret = _map_single_page_2M(lv0_tbl, va, pa, attr);
va += ARCH_SECTION_SIZE;
pa += ARCH_SECTION_SIZE;
if (ret != 0)
{
return ret;
}
}
return 0;
}
static int _kenrel_map_2M(unsigned long *lv0_tbl, unsigned long va, unsigned long pa, unsigned long attr)
{
int level;
unsigned long *cur_lv_tbl = lv0_tbl;
unsigned long page;
unsigned long off;
int level_shift = MMU_ADDRESS_BITS;
2022-01-07 13:49:06 +08:00
if (va & ARCH_SECTION_MASK)
{
return MMU_MAP_ERROR_VANOTALIGN;
}
if (pa & ARCH_SECTION_MASK)
{
return MMU_MAP_ERROR_PANOTALIGN;
}
for (level = 0; level < MMU_TBL_BLOCK_2M_LEVEL; level++)
{
off = (va >> level_shift);
off &= MMU_LEVEL_MASK;
if (!(cur_lv_tbl[off] & MMU_TYPE_USED))
{
page = (unsigned long)rt_pages_alloc(0);
if (!page)
{
return MMU_MAP_ERROR_NOPAGE;
}
2022-01-07 13:49:06 +08:00
rt_memset((char *)page, 0, ARCH_PAGE_SIZE);
rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, (void *)page, ARCH_PAGE_SIZE);
cur_lv_tbl[off] = (page + PV_OFFSET) | MMU_TYPE_TABLE;
2022-01-07 13:49:06 +08:00
rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off, sizeof(void *));
}
else
{
page = cur_lv_tbl[off];
page &= MMU_ADDRESS_MASK;
/* page to va */
page -= PV_OFFSET;
rt_page_ref_inc((void *)page, 0);
}
2020-02-20 15:42:10 +08:00
page = cur_lv_tbl[off];
2022-01-07 13:49:06 +08:00
if ((page & MMU_TYPE_MASK) == MMU_TYPE_BLOCK)
{
2022-01-07 13:49:06 +08:00
/* is block! error! */
2020-02-20 15:42:10 +08:00
return MMU_MAP_ERROR_CONFLICT;
}
2022-01-07 13:49:06 +08:00
cur_lv_tbl = (unsigned long *)(page & MMU_ADDRESS_MASK);
cur_lv_tbl = (unsigned long *)((unsigned long)cur_lv_tbl - PV_OFFSET);
2022-01-07 13:49:06 +08:00
level_shift -= MMU_LEVEL_SHIFT;
}
2022-01-07 13:49:06 +08:00
attr &= MMU_ATTRIB_MASK;
pa |= (attr | MMU_TYPE_BLOCK); /* block */
2022-01-07 13:49:06 +08:00
off = (va >> ARCH_SECTION_SHIFT);
2020-02-20 15:42:10 +08:00
off &= MMU_LEVEL_MASK;
cur_lv_tbl[off] = pa;
2022-01-07 13:49:06 +08:00
rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off, sizeof(void *));
2020-02-20 15:42:10 +08:00
return 0;
}
struct mmu_level_info
{
unsigned long *pos;
void *page;
};
#ifdef RT_USING_LWP
static void _kenrel_unmap_4K(unsigned long *lv0_tbl, void *v_addr)
{
int level;
unsigned long va = (unsigned long)v_addr;
unsigned long *cur_lv_tbl = lv0_tbl;
unsigned long page;
unsigned long off;
struct mmu_level_info level_info[4];
int ref;
int level_shift = MMU_ADDRESS_BITS;
unsigned long *pos;
rt_memset(level_info, 0, sizeof level_info);
for (level = 0; level < MMU_TBL_LEVEL_NR; level++)
{
off = (va >> level_shift);
off &= MMU_LEVEL_MASK;
page = cur_lv_tbl[off];
if (!(page & MMU_TYPE_USED))
{
break;
}
if ((page & MMU_TYPE_MASK) == MMU_TYPE_BLOCK)
{
break;
}
level_info[level].pos = cur_lv_tbl + off;
cur_lv_tbl = (unsigned long *)(page & MMU_ADDRESS_MASK);
cur_lv_tbl = (unsigned long *)((unsigned long)cur_lv_tbl - PV_OFFSET);
level_info[level].page = cur_lv_tbl;
level_shift -= MMU_LEVEL_SHIFT;
}
level = MMU_TBL_PAGE_4k_LEVEL;
pos = level_info[level].pos;
if (pos)
2022-01-07 13:49:06 +08:00
{
*pos = (unsigned long)RT_NULL;
rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, pos, sizeof(void *));
2022-01-07 13:49:06 +08:00
}
level--;
while (level >= 0)
2022-01-07 13:49:06 +08:00
{
pos = level_info[level].pos;
if (pos)
{
void *cur_page = level_info[level].page;
ref = rt_page_ref_get(cur_page, 0);
if (ref == 1)
{
*pos = (unsigned long)RT_NULL;
rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, pos, sizeof(void *));
}
rt_pages_free(cur_page, 0);
}
level--;
2022-01-07 13:49:06 +08:00
}
asm volatile("tlbi vae1, %0\ndsb sy"::"r"(v_addr):"memory");
return;
}
static int _kenrel_map_4K(unsigned long *lv0_tbl, unsigned long va, unsigned long pa, unsigned long attr)
{
int ret = 0;
int level;
unsigned long *cur_lv_tbl = lv0_tbl;
unsigned long page;
unsigned long off;
int level_shift = MMU_ADDRESS_BITS;
if (va & ARCH_PAGE_MASK)
{
return MMU_MAP_ERROR_VANOTALIGN;
2020-02-20 15:42:10 +08:00
}
if (pa & ARCH_PAGE_MASK)
2020-02-20 15:42:10 +08:00
{
return MMU_MAP_ERROR_PANOTALIGN;
2020-02-20 15:42:10 +08:00
}
for (level = 0; level < MMU_TBL_PAGE_4k_LEVEL; level++)
{
off = (va >> level_shift);
off &= MMU_LEVEL_MASK;
if (!(cur_lv_tbl[off] & MMU_TYPE_USED))
{
page = (unsigned long)rt_pages_alloc(0);
if (!page)
{
ret = MMU_MAP_ERROR_NOPAGE;
goto err;
}
rt_memset((void *)page, 0, ARCH_PAGE_SIZE);
rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, (void *)page, ARCH_PAGE_SIZE);
cur_lv_tbl[off] = (page + PV_OFFSET) | MMU_TYPE_TABLE;
rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off, sizeof(void *));
}
else
{
page = cur_lv_tbl[off];
page &= MMU_ADDRESS_MASK;
/* page to va */
page -= PV_OFFSET;
rt_page_ref_inc((void *)page, 0);
}
page = cur_lv_tbl[off];
if ((page & MMU_TYPE_MASK) == MMU_TYPE_BLOCK)
{
/* is block! error! */
ret = MMU_MAP_ERROR_CONFLICT;
goto err;
}
cur_lv_tbl = (unsigned long *)(page & MMU_ADDRESS_MASK);
cur_lv_tbl = (unsigned long *)((unsigned long)cur_lv_tbl - PV_OFFSET);
level_shift -= MMU_LEVEL_SHIFT;
}
/* now is level page */
attr &= MMU_ATTRIB_MASK;
pa |= (attr | MMU_TYPE_PAGE); /* page */
off = (va >> ARCH_PAGE_SHIFT);
off &= MMU_LEVEL_MASK;
cur_lv_tbl[off] = pa; /* page */
rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off, sizeof(void *));
return ret;
err:
_kenrel_unmap_4K(lv0_tbl, (void *)va);
return ret;
}
#endif
2022-01-07 13:49:06 +08:00
static int _kernel_map_fixed(unsigned long *lv0_tbl, unsigned long va, unsigned long pa, unsigned long count, unsigned long attr)
{
unsigned long i;
int ret;
unsigned long _attr = MMU_MAP_CUSTOM(MMU_AP_KAUN, attr);
2022-01-07 13:49:06 +08:00
if (va & ARCH_SECTION_MASK)
{
return -1;
}
if (pa & ARCH_SECTION_MASK)
{
return -1;
}
2020-02-20 15:42:10 +08:00
for (i = 0; i < count; i++)
{
ret = _kenrel_map_2M(lv0_tbl, va, pa, _attr);
va += ARCH_SECTION_SIZE;
pa += ARCH_SECTION_SIZE;
2020-02-20 15:42:10 +08:00
if (ret != 0)
{
return ret;
}
}
return 0;
}
/************ setting el1 mmu register**************
MAIR_EL1
index 0 : memory outer writeback, write/read alloc
index 1 : memory nocache
index 2 : device nGnRnE
*****************************************************/
void mmu_tcr_init(void)
{
unsigned long val64;
val64 = 0x00447fUL;
__asm__ volatile("msr MAIR_EL1, %0\n dsb sy\n"::"r"(val64));
/* TCR_EL1 */
val64 = (16UL << 0) /* t0sz 48bit */
| (0x0UL << 6) /* reserved */
| (0x0UL << 7) /* epd0 */
| (0x3UL << 8) /* t0 wb cacheable */
| (0x3UL << 10) /* inner shareable */
| (0x2UL << 12) /* t0 outer shareable */
| (0x0UL << 14) /* t0 4K */
| (16UL << 16) /* t1sz 48bit */
| (0x0UL << 22) /* define asid use ttbr0.asid */
| (0x0UL << 23) /* epd1 */
| (0x3UL << 24) /* t1 inner wb cacheable */
| (0x3UL << 26) /* t1 outer wb cacheable */
| (0x2UL << 28) /* t1 outer shareable */
| (0x2UL << 30) /* t1 4k */
| (0x1UL << 32) /* 001b 64GB PA */
| (0x0UL << 35) /* reserved */
| (0x1UL << 36) /* as: 0:8bit 1:16bit */
| (0x0UL << 37) /* tbi0 */
| (0x0UL << 38); /* tbi1 */
__asm__ volatile("msr TCR_EL1, %0\n"::"r"(val64));
}
/* dump 2nd level page table */
void rt_hw_cpu_dump_page_table_2nd(rt_uint32_t *ptb)
{
}
void rt_hw_cpu_dump_page_table(rt_uint32_t *ptb)
{
}
volatile unsigned long MMUTable[512] __attribute__((aligned(4 * 1024)));
void rt_hw_mmu_setmtt(unsigned long vaddrStart,
unsigned long vaddrEnd,
unsigned long paddrStart,
unsigned long attr)
{
unsigned long count;
if (vaddrStart & ARCH_SECTION_MASK)
{
while (1);
}
if (paddrStart & ARCH_SECTION_MASK)
{
while (1);
}
if (vaddrStart > vaddrEnd)
{
while (1);
}
count = vaddrEnd + 1;
if (count & ARCH_SECTION_MASK)
{
while (1);
}
count -= vaddrStart;
if (count == 0)
{
while (1);
}
count >>= ARCH_SECTION_SHIFT;
_kernel_map_fixed((unsigned long *)MMUTable, vaddrStart, paddrStart, count, attr);
}
void rt_hw_mmu_ktbl_set(unsigned long tbl)
{
#ifdef RT_USING_SMART
tbl += PV_OFFSET;
__asm__ volatile("msr TTBR1_EL1, %0\n dsb sy\nisb"::"r"(tbl):"memory");
#else
__asm__ volatile("msr TTBR0_EL1, %0\n dsb sy\nisb"::"r"(tbl):"memory");
#endif
__asm__ volatile("tlbi vmalle1\n dsb sy\nisb":::"memory");
__asm__ volatile("ic ialluis\n dsb sy\nisb":::"memory");
}
void rt_hw_mmu_setup(struct mem_desc *mdesc, int desc_nr)
{
/* set page table */
for (; desc_nr > 0; desc_nr--)
{
rt_hw_mmu_setmtt(mdesc->vaddr_start, mdesc->vaddr_end,
mdesc->paddr_start, mdesc->attr);
mdesc++;
}
rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, (void *)MMUTable, sizeof MMUTable);
rt_hw_mmu_ktbl_set((unsigned long)MMUTable);
}
/**
* This function will initialize rt_mmu_info structure.
*
* @param mmu_info rt_mmu_info structure
* @param v_address virtual address
* @param size map size
* @param vtable mmu table
* @param pv_off pv offset in kernel space
*
* @return 0 on successful and -1 for fail
*/
int rt_hw_mmu_map_init(rt_mmu_info *mmu_info, void *v_address, size_t size, size_t *vtable, size_t pv_off)
{
size_t va_s, va_e;
if (!mmu_info || !vtable)
{
return -1;
}
va_s = (size_t)v_address;
va_e = (size_t)v_address + size - 1;
if (va_e < va_s)
{
return -1;
}
va_s >>= ARCH_SECTION_SHIFT;
va_e >>= ARCH_SECTION_SHIFT;
if (va_s == 0)
{
return -1;
}
mmu_info->vtable = vtable;
mmu_info->vstart = va_s;
mmu_info->vend = va_e;
mmu_info->pv_off = pv_off;
return 0;
}
int rt_hw_mmu_ioremap_init(rt_mmu_info *mmu_info, void *v_address, size_t size)
{
return 0;
}
static size_t find_vaddr(rt_mmu_info *mmu_info, int pages)
{
size_t loop_pages;
size_t va;
size_t find_va = 0;
int n = 0;
size_t i;
if (!pages)
{
return 0;
}
if (!mmu_info)
{
return 0;
}
loop_pages = mmu_info->vend - mmu_info->vstart + 1;
loop_pages <<= (ARCH_SECTION_SHIFT - ARCH_PAGE_SHIFT);
va = mmu_info->vstart;
va <<= ARCH_SECTION_SHIFT;
for (i = 0; i < loop_pages; i++, va += ARCH_PAGE_SIZE)
{
if (_rt_hw_mmu_v2p(mmu_info, (void *)va))
{
n = 0;
find_va = 0;
continue;
}
if (!find_va)
{
find_va = va;
}
n++;
if (n >= pages)
{
return find_va;
}
}
return 0;
}
#ifdef RT_USING_SMART
static int check_vaddr(rt_mmu_info *mmu_info, void *va, int pages)
{
size_t loop_va;
if (!pages)
{
return -1;
}
if (!mmu_info)
{
return -1;
}
loop_va = ((size_t)va >> ARCH_SECTION_SHIFT);
if (loop_va < mmu_info->vstart || loop_va > mmu_info->vend)
{
return -1;
}
loop_va += ((pages << ARCH_PAGE_SHIFT) >> ARCH_SECTION_SHIFT);
if (loop_va < mmu_info->vstart || loop_va > mmu_info->vend + 1)
{
return -1;
}
loop_va = (size_t)va & ~ARCH_PAGE_MASK;
while (pages--)
{
if (_rt_hw_mmu_v2p(mmu_info, (void *)loop_va))
{
return -1;
}
loop_va += ARCH_PAGE_SIZE;
}
return 0;
}
static void __rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void *v_addr, size_t npages)
{
size_t loop_va = (size_t)v_addr & ~ARCH_PAGE_MASK;
if (!mmu_info || !mmu_info->vtable)
{
return;
}
while (npages--)
{
_kenrel_unmap_4K(mmu_info->vtable, (void *)loop_va);
loop_va += ARCH_PAGE_SIZE;
}
}
static int __rt_hw_mmu_map(rt_mmu_info *mmu_info, void *v_addr, void *p_addr, size_t npages, size_t attr)
{
int ret = -1;
size_t loop_va = (size_t)v_addr & ~ARCH_PAGE_MASK;
size_t loop_pa = (size_t)p_addr & ~ARCH_PAGE_MASK;
size_t unmap_va = loop_va;
2020-02-20 15:42:10 +08:00
if (mmu_info)
{
while (npages--)
{
ret = _kenrel_map_4K(mmu_info->vtable, loop_va, loop_pa, attr);
if (ret != 0)
{
/* error, undo map */
while (unmap_va != loop_va)
{
_kenrel_unmap_4K(mmu_info->vtable, (void *)unmap_va);
unmap_va += ARCH_PAGE_SIZE;
}
break;
}
loop_va += ARCH_PAGE_SIZE;
loop_pa += ARCH_PAGE_SIZE;
}
}
2022-01-07 13:49:06 +08:00
return ret;
}
#endif
static void rt_hw_cpu_tlb_invalidate(void)
{
__asm__ volatile("tlbi vmalle1\n dsb sy\n isb sy\n");
}
#ifdef RT_USING_SMART
void *_rt_hw_mmu_map(rt_mmu_info *mmu_info, void *v_addr, void *p_addr, size_t size, size_t attr)
{
size_t pa_s, pa_e;
size_t vaddr;
int pages;
int ret;
if (!size)
{
return 0;
}
pa_s = (size_t)p_addr;
pa_e = (size_t)p_addr + size - 1;
pa_s >>= ARCH_PAGE_SHIFT;
pa_e >>= ARCH_PAGE_SHIFT;
pages = pa_e - pa_s + 1;
if (v_addr)
{
vaddr = (size_t)v_addr;
pa_s = (size_t)p_addr;
if ((vaddr & ARCH_PAGE_MASK) != (pa_s & ARCH_PAGE_MASK))
{
return 0;
}
vaddr &= ~ARCH_PAGE_MASK;
if (check_vaddr(mmu_info, (void *)vaddr, pages) != 0)
{
return 0;
}
}
else
{
vaddr = find_vaddr(mmu_info, pages);
}
if (vaddr)
{
rt_enter_critical();
ret = __rt_hw_mmu_map(mmu_info, (void *)vaddr, p_addr, pages, attr);
if (ret == 0)
{
rt_hw_cpu_tlb_invalidate();
rt_exit_critical();
return (void *)(vaddr + ((size_t)p_addr & ARCH_PAGE_MASK));
}
rt_exit_critical();
}
return 0;
}
#else
void *_rt_hw_mmu_map(rt_mmu_info *mmu_info, void* p_addr, size_t size, size_t attr)
{
return p_addr;
}
#endif
#ifdef RT_USING_SMART
static int __rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void *v_addr, size_t npages, size_t attr)
{
size_t loop_va = (size_t)v_addr & ~ARCH_PAGE_MASK;
size_t loop_pa;
if (!mmu_info)
{
return -1;
}
while (npages--)
{
loop_pa = (size_t)rt_pages_alloc(0);
if (!loop_pa)
{
goto err;
}
loop_pa += mmu_info->pv_off;
_kenrel_map_4K(mmu_info->vtable, loop_va, loop_pa, attr);
loop_va += ARCH_PAGE_SIZE;
}
return 0;
err:
{
/* error, unmap and quit */
int i;
void *va, *pa;
va = (void *)((size_t)v_addr & ~ARCH_PAGE_MASK);
for (i = 0; i < npages; i++)
{
pa = rt_hw_mmu_v2p(mmu_info, va);
pa = (void *)((char *)pa - mmu_info->pv_off);
rt_pages_free(pa, 0);
va = (void *)((char *)va + ARCH_PAGE_SIZE);
}
__rt_hw_mmu_unmap(mmu_info, v_addr, npages);
return -1;
}
}
void *_rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void *v_addr, size_t size, size_t attr)
{
size_t vaddr;
size_t offset;
int pages;
int ret;
if (!size)
{
return 0;
}
offset = (size_t)v_addr & ARCH_PAGE_MASK;
size += (offset + ARCH_PAGE_SIZE - 1);
pages = (size >> ARCH_PAGE_SHIFT);
if (v_addr)
{
vaddr = (size_t)v_addr;
vaddr &= ~ARCH_PAGE_MASK;
if (check_vaddr(mmu_info, (void *)vaddr, pages) != 0)
{
return 0;
}
}
else
{
vaddr = find_vaddr(mmu_info, pages);
}
if (vaddr)
{
rt_enter_critical();
ret = __rt_hw_mmu_map_auto(mmu_info, (void *)vaddr, pages, attr);
if (ret == 0)
{
rt_hw_cpu_tlb_invalidate();
rt_exit_critical();
return (void *)((char *)vaddr + offset);
}
rt_exit_critical();
}
return 0;
}
void _rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void *v_addr, size_t size)
{
size_t va_s, va_e;
int pages;
va_s = (size_t)v_addr;
va_e = (size_t)v_addr + size - 1;
va_s >>= ARCH_PAGE_SHIFT;
va_e >>= ARCH_PAGE_SHIFT;
pages = va_e - va_s + 1;
rt_enter_critical();
__rt_hw_mmu_unmap(mmu_info, v_addr, pages);
rt_hw_cpu_tlb_invalidate();
rt_exit_critical();
}
void *rt_hw_mmu_map(rt_mmu_info *mmu_info, void *v_addr, void *p_addr, size_t size, size_t attr)
{
void *ret;
rt_mm_lock();
ret = _rt_hw_mmu_map(mmu_info, v_addr, p_addr, size, attr);
rt_mm_unlock();
return ret;
}
void *rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void *v_addr, size_t size, size_t attr)
{
void *ret;
rt_mm_lock();
ret = _rt_hw_mmu_map_auto(mmu_info, v_addr, size, attr);
rt_mm_unlock();
return ret;
}
void rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void *v_addr, size_t size)
{
rt_mm_lock();
_rt_hw_mmu_unmap(mmu_info, v_addr, size);
rt_mm_unlock();
}
#else
#include <cache.h>
2022-01-07 13:49:06 +08:00
void rt_hw_init_mmu_table(struct mem_desc *mdesc, rt_size_t desc_nr)
{
2022-01-07 13:49:06 +08:00
rt_memset((void *)MMUTable, 0, sizeof(MMUTable));
rt_memset((void *)MMUPage, 0, sizeof(MMUPage));
/* set page table */
for (; desc_nr > 0; --desc_nr)
{
2022-01-07 13:49:06 +08:00
rt_hw_mmu_setmtt(mdesc->vaddr_start, mdesc->vaddr_end, mdesc->paddr_start, mdesc->attr);
++mdesc;
}
2020-01-10 10:38:21 +08:00
rt_hw_cpu_dcache_clean((void *)MMUTable, sizeof(MMUTable));
2020-01-10 10:38:21 +08:00
}
2022-01-07 13:49:06 +08:00
void rt_hw_mmu_init(void)
{
2022-01-07 13:49:06 +08:00
unsigned long reg_val;
2022-01-07 13:49:06 +08:00
reg_val = 0x00447fUL;
__asm__ volatile("msr mair_el1, %0"::"r"(reg_val));
2022-01-07 13:49:06 +08:00
rt_hw_isb();
2020-02-20 15:42:10 +08:00
2022-01-07 13:49:06 +08:00
reg_val = (16UL << 0) /* t0sz 48bit */
| (0UL << 6) /* reserved */
| (0UL << 7) /* epd0 */
| (3UL << 8) /* t0 wb cacheable */
| (3UL << 10) /* inner shareable */
| (2UL << 12) /* t0 outer shareable */
| (0UL << 14) /* t0 4K */
| (16UL << 16) /* t1sz 48bit */
| (0UL << 22) /* define asid use ttbr0.asid */
| (0UL << 23) /* epd1 */
| (3UL << 24) /* t1 inner wb cacheable */
| (3UL << 26) /* t1 outer wb cacheable */
| (2UL << 28) /* t1 outer shareable */
| (2UL << 30) /* t1 4k */
| (1UL << 32) /* 001b 64GB PA */
| (0UL << 35) /* reserved */
| (1UL << 36) /* as: 0:8bit 1:16bit */
| (0UL << 37) /* tbi0 */
| (0UL << 38); /* tbi1 */
__asm__ volatile("msr tcr_el1, %0"::"r"(reg_val));
2020-02-20 15:42:10 +08:00
2022-01-07 13:49:06 +08:00
rt_hw_isb();
2020-03-17 13:45:13 +08:00
2022-01-07 13:49:06 +08:00
__asm__ volatile ("mrs %0, sctlr_el1":"=r"(reg_val));
2020-03-17 13:45:13 +08:00
2022-01-07 13:49:06 +08:00
reg_val |= 1 << 2; /* enable dcache */
reg_val |= 1 << 0; /* enable mmu */
2020-03-17 13:45:13 +08:00
2022-01-07 13:49:06 +08:00
__asm__ volatile (
"msr ttbr0_el1, %0\n\r"
"msr sctlr_el1, %1\n\r"
"dsb sy\n\r"
"isb sy\n\r"
::"r"(MMUTable), "r"(reg_val) :"memory");
2020-03-17 13:45:13 +08:00
rt_hw_cpu_tlb_invalidate();
2020-03-17 13:45:13 +08:00
}
#endif
2020-03-17 13:45:13 +08:00
void *_rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void *v_addr)
2020-03-17 13:45:13 +08:00
{
int level;
unsigned long va = (unsigned long)v_addr;
unsigned long pa;
unsigned long *cur_lv_tbl;
unsigned long page;
unsigned long off;
unsigned long off_addr;
int level_shift = MMU_ADDRESS_BITS;
2020-03-17 13:45:13 +08:00
if (!mmu_info)
{
return (void *)0;
}
cur_lv_tbl = mmu_info->vtable;
for (level = 0; level < MMU_TBL_PAGE_4k_LEVEL; level++)
{
off = (va >> level_shift);
off &= MMU_LEVEL_MASK;
if (!(cur_lv_tbl[off] & MMU_TYPE_USED))
{
return (void *)0;
}
page = cur_lv_tbl[off];
if ((page & MMU_TYPE_MASK) == MMU_TYPE_BLOCK)
{
off_addr = va & ((1UL << level_shift) - 1);
pa = (page & MMU_ADDRESS_MASK);
pa += off_addr;
return (void *)pa;
}
cur_lv_tbl = (unsigned long *)(page & MMU_ADDRESS_MASK);
cur_lv_tbl = (unsigned long *)((unsigned long)cur_lv_tbl - PV_OFFSET);
level_shift -= MMU_LEVEL_SHIFT;
}
/* now is level MMU_TBL_PAGE_4k_LEVEL */
off = (va >> ARCH_PAGE_SHIFT);
off &= MMU_LEVEL_MASK;
page = cur_lv_tbl[off];
if (!(page & MMU_TYPE_USED))
{
return (void *)0;
}
pa = (page & MMU_ADDRESS_MASK);
pa += (va & ARCH_PAGE_MASK);
return (void *)pa;
}
2020-03-17 13:45:13 +08:00
void *rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void *v_addr)
{
void *ret;
2020-03-17 13:45:13 +08:00
rt_mm_lock();
ret = _rt_hw_mmu_v2p(mmu_info, v_addr);
rt_mm_unlock();
2022-01-07 13:49:06 +08:00
return ret;
2021-03-27 17:51:56 +08:00
}
void rt_hw_mmu_setup_early(unsigned long *tbl0, unsigned long *tbl1, unsigned long size, unsigned long pv_off)
{
int ret;
unsigned long va = KERNEL_VADDR_START;
unsigned long count = (size + ARCH_SECTION_MASK) >> ARCH_SECTION_SHIFT;
unsigned long normal_attr = MMU_MAP_CUSTOM(MMU_AP_KAUN, NORMAL_MEM);
/* clean the first two pages */
mmu_memset((char *)tbl0, 0, ARCH_PAGE_SIZE);
mmu_memset((char *)tbl1, 0, ARCH_PAGE_SIZE);
ret = armv8_init_map_2M(tbl1, va, va + pv_off, count, normal_attr);
if (ret != 0)
{
while (1);
}
ret = armv8_init_map_2M(tbl0, va + pv_off, va + pv_off, count, normal_attr);
if (ret != 0)
{
while (1);
}
}