rt-thread/libcpu/aarch64/common/mmu.c

368 lines
8.4 KiB
C
Raw Normal View History

/*
2021-03-27 17:51:56 +08:00
* Copyright (c) 2006-2021, RT-Thread Development Team
*
2020-01-10 10:38:21 +08:00
* SPDX-License-Identifier: Apache-2.0
*
* Change Logs:
2020-02-20 15:42:10 +08:00
* Date Author Notes
* 2020-02-20 bigmagic first version
*/
2020-02-20 15:42:10 +08:00
#include <mmu.h>
#include <stddef.h>
2020-03-17 13:45:13 +08:00
#include <rthw.h>
2020-02-20 15:42:10 +08:00
#define TTBR_CNP 1
typedef unsigned long int uint64_t;
2020-02-20 15:42:10 +08:00
static unsigned long main_tbl[512 * 20] __attribute__((aligned (4096)));
#define IS_ALIGNED(x, a) (((x) & ((typeof(x))(a) - 1)) == 0)
2020-02-20 15:42:10 +08:00
#define PMD_TYPE_SECT (1 << 0)
2020-02-20 15:42:10 +08:00
#define PMD_TYPE_TABLE (3 << 0)
2020-02-20 15:42:10 +08:00
#define PTE_TYPE_PAGE (3 << 0)
2020-02-20 15:42:10 +08:00
#define BITS_PER_VA 39
2020-02-20 15:42:10 +08:00
/* Granule size of 4KB is being used */
#define GRANULE_SIZE_SHIFT 12
#define GRANULE_SIZE (1 << GRANULE_SIZE_SHIFT)
#define XLAT_ADDR_MASK ((1UL << BITS_PER_VA) - GRANULE_SIZE)
#define PMD_TYPE_MASK (3 << 0)
2020-02-20 15:42:10 +08:00
int free_idx = 1;
2020-03-17 13:45:13 +08:00
void __asm_invalidate_icache_all(void);
void __asm_flush_dcache_all(void);
int __asm_flush_l3_cache(void);
void __asm_flush_dcache_range(unsigned long long start, unsigned long long end);
void __asm_invalidate_dcache_all(void);
void __asm_invalidate_icache_all(void);
2020-02-20 15:42:10 +08:00
void mmu_memset(char *dst, char v, size_t len)
{
while (len--)
{
2020-02-20 15:42:10 +08:00
*dst++ = v;
}
}
2020-02-20 15:42:10 +08:00
static unsigned long __page_off = 0;
2021-03-27 17:51:56 +08:00
static unsigned long get_free_page(void)
{
2020-02-20 15:42:10 +08:00
__page_off += 512;
return (unsigned long)(main_tbl + __page_off);
}
2020-03-17 13:45:13 +08:00
static inline unsigned int get_sctlr(void)
{
unsigned int val;
asm volatile("mrs %0, sctlr_el1" : "=r" (val) : : "cc");
return val;
}
static inline void set_sctlr(unsigned int val)
{
asm volatile("msr sctlr_el1, %0" : : "r" (val) : "cc");
asm volatile("isb");
}
2020-02-20 15:42:10 +08:00
void mmu_init(void)
{
unsigned long val64;
unsigned long val32;
2020-02-20 15:42:10 +08:00
val64 = 0x007f6eUL;
__asm__ volatile("msr MAIR_EL1, %0\n dsb sy\n"::"r"(val64));
__asm__ volatile("mrs %0, MAIR_EL1\n dsb sy\n":"=r"(val64));
//TCR_EL1
val32 = (16UL << 0)//48bit
| (0x0UL << 6)
| (0x0UL << 7)
| (0x3UL << 8)
2020-02-20 15:42:10 +08:00
| (0x3UL << 10)//Inner Shareable
| (0x2UL << 12)
| (0x0UL << 14)//4K
| (0x0UL << 16)
| (0x0UL << 22)
| (0x1UL << 23)
| (0x2UL << 30)
| (0x1UL << 32)
| (0x0UL << 35)
| (0x0UL << 36)
| (0x0UL << 37)
| (0x0UL << 38);
__asm__ volatile("msr TCR_EL1, %0\n"::"r"(val32));
__asm__ volatile("mrs %0, TCR_EL1\n":"=r"(val32));
__asm__ volatile("msr TTBR0_EL1, %0\n dsb sy\n"::"r"(main_tbl));
__asm__ volatile("mrs %0, TTBR0_EL1\n dsb sy\n":"=r"(val64));
mmu_memset((char *)main_tbl, 0, 4096);
}
void mmu_enable(void)
{
unsigned long val64;
unsigned long val32;
__asm__ volatile("mrs %0, SCTLR_EL1\n":"=r"(val64));
val64 &= ~0x1000; //disable I
__asm__ volatile("dmb sy\n msr SCTLR_EL1, %0\n isb sy\n"::"r"(val64));
__asm__ volatile("IC IALLUIS\n dsb sy\n isb sy\n");
__asm__ volatile("tlbi vmalle1\n dsb sy\n isb sy\n");
//SCTLR_EL1, turn on mmu
__asm__ volatile("mrs %0, SCTLR_EL1\n":"=r"(val32));
val32 |= 0x1005; //enable mmu, I C M
__asm__ volatile("dmb sy\n msr SCTLR_EL1, %0\nisb sy\n"::"r"(val32));
2020-03-17 13:45:13 +08:00
rt_hw_icache_enable();
rt_hw_dcache_enable();
2020-02-20 15:42:10 +08:00
}
2021-03-27 17:51:56 +08:00
static int map_single_page_2M(unsigned long* lv0_tbl, unsigned long va, unsigned long pa, unsigned long attr)
{
2020-02-20 15:42:10 +08:00
int level;
unsigned long* cur_lv_tbl = lv0_tbl;
unsigned long page;
unsigned long off;
int level_shift = 39;
2021-03-27 17:51:56 +08:00
if (va & (0x200000UL - 1))
{
2020-02-20 15:42:10 +08:00
return MMU_MAP_ERROR_VANOTALIGN;
}
2021-03-27 17:51:56 +08:00
if (pa & (0x200000UL - 1))
{
2020-02-20 15:42:10 +08:00
return MMU_MAP_ERROR_PANOTALIGN;
}
2021-03-27 17:51:56 +08:00
for (level = 0; level < 2; level++)
{
2020-02-20 15:42:10 +08:00
off = (va >> level_shift);
off &= MMU_LEVEL_MASK;
2021-03-27 17:51:56 +08:00
if ((cur_lv_tbl[off] & 1) == 0)
{
2020-02-20 15:42:10 +08:00
page = get_free_page();
2021-03-27 17:51:56 +08:00
if (!page)
{
2020-02-20 15:42:10 +08:00
return MMU_MAP_ERROR_NOPAGE;
}
mmu_memset((char *)page, 0, 4096);
cur_lv_tbl[off] = page | 0x3UL;
}
2020-02-20 15:42:10 +08:00
page = cur_lv_tbl[off];
2021-03-27 17:51:56 +08:00
if (!(page & 0x2))
{
2020-02-20 15:42:10 +08:00
//is block! error!
return MMU_MAP_ERROR_CONFLICT;
}
2020-02-20 15:42:10 +08:00
cur_lv_tbl = (unsigned long*)(page & 0x0000fffffffff000UL);
level_shift -= 9;
}
2020-02-20 15:42:10 +08:00
attr &= 0xfff0000000000ffcUL;
pa |= (attr | 0x1UL); //block
off = (va >> 21);
off &= MMU_LEVEL_MASK;
cur_lv_tbl[off] = pa;
return 0;
}
2020-02-20 15:42:10 +08:00
int armv8_map_2M(unsigned long va, unsigned long pa, int count, unsigned long attr)
{
int i;
2020-02-20 15:42:10 +08:00
int ret;
2020-02-20 15:42:10 +08:00
if (va & (0x200000 - 1))
{
2020-02-20 15:42:10 +08:00
return -1;
}
if (pa & (0x200000 - 1))
{
return -1;
}
for (i = 0; i < count; i++)
{
ret = map_single_page_2M((unsigned long *)main_tbl, va, pa, attr);
va += 0x200000;
pa += 0x200000;
if (ret != 0)
{
2020-02-20 15:42:10 +08:00
return ret;
}
}
2020-02-20 15:42:10 +08:00
return 0;
}
static void set_table(uint64_t *pt, uint64_t *table_addr)
{
uint64_t val;
val = (0x3UL | (uint64_t)table_addr);
*pt = val;
}
2020-02-20 15:42:10 +08:00
void mmu_memset2(unsigned char *dst, char v, int len)
{
2020-02-20 15:42:10 +08:00
while (len--)
{
2020-02-20 15:42:10 +08:00
*dst++ = v;
}
}
2020-02-20 15:42:10 +08:00
static uint64_t *create_table(void)
{
uint64_t *new_table = (uint64_t *)((unsigned char *)&main_tbl[0] + free_idx * 4096); //+ free_idx * GRANULE_SIZE;
/* Mark all entries as invalid */
mmu_memset2((unsigned char *)new_table, 0, 4096);
free_idx++;
return new_table;
2020-01-10 10:38:21 +08:00
}
2020-02-20 15:42:10 +08:00
static int pte_type(uint64_t *pte)
2020-01-10 10:38:21 +08:00
{
return *pte & PMD_TYPE_MASK;
2020-01-10 10:38:21 +08:00
}
2020-02-20 15:42:10 +08:00
static int level2shift(int level)
2020-01-10 10:38:21 +08:00
{
/* Page is 12 bits wide, every level translates 9 bits */
return (12 + 9 * (3 - level));
}
2020-02-20 15:42:10 +08:00
static uint64_t *get_level_table(uint64_t *pte)
{
uint64_t *table = (uint64_t *)(*pte & XLAT_ADDR_MASK);
2021-03-27 17:51:56 +08:00
if (pte_type(pte) != PMD_TYPE_TABLE)
{
table = create_table();
set_table(pte, table);
}
return table;
2020-02-20 15:42:10 +08:00
}
2020-02-20 15:42:10 +08:00
static void map_region(uint64_t virt, uint64_t phys, uint64_t size, uint64_t attr)
{
uint64_t block_size = 0;
uint64_t block_shift = 0;
uint64_t *pte;
uint64_t idx = 0;
uint64_t addr = 0;
uint64_t *table = 0;
int level = 0;
addr = virt;
2021-03-27 17:51:56 +08:00
while (size)
{
table = &main_tbl[0];
2021-03-27 17:51:56 +08:00
for (level = 0; level < 4; level++)
{
block_shift = level2shift(level);
2020-02-20 15:42:10 +08:00
idx = addr >> block_shift;
idx = idx%512;
2020-02-20 15:42:10 +08:00
block_size = (uint64_t)(1L << block_shift);
pte = table + idx;
2021-03-27 17:51:56 +08:00
if (size >= block_size && IS_ALIGNED(addr, block_size))
{
2020-02-20 15:42:10 +08:00
attr &= 0xfff0000000000ffcUL;
if(level != 3)
{
*pte = phys | (attr | 0x1UL);
}
else
{
*pte = phys | (attr | 0x3UL);
}
addr += block_size;
phys += block_size;
size -= block_size;
break;
}
table = get_level_table(pte);
}
}
}
2020-02-20 15:42:10 +08:00
void armv8_map(unsigned long va, unsigned long pa, unsigned long size, unsigned long attr)
{
map_region(va, pa, size, attr);
}
2020-03-17 13:45:13 +08:00
void rt_hw_dcache_enable(void)
{
2021-03-27 17:51:56 +08:00
if (!(get_sctlr() & CR_M))
2020-03-17 13:45:13 +08:00
{
rt_kprintf("please init mmu!\n");
}
else
{
set_sctlr(get_sctlr() | CR_C);
}
}
void rt_hw_dcache_flush_all(void)
{
int ret;
__asm_flush_dcache_all();
ret = __asm_flush_l3_cache();
if (ret)
{
rt_kprintf("flushing dcache returns 0x%x\n", ret);
}
else
{
rt_kprintf("flushing dcache successfully.\n");
}
}
void rt_hw_dcache_flush_range(unsigned long start_addr, unsigned long size)
{
__asm_flush_dcache_range(start_addr, start_addr + size);
}
void rt_hw_dcache_invalidate_range(unsigned long start_addr,unsigned long size)
{
__asm_flush_dcache_range(start_addr, start_addr + size);
}
void rt_hw_dcache_invalidate_all(void)
{
__asm_invalidate_dcache_all();
}
void rt_hw_dcache_disable(void)
{
/* if cache isn't enabled no need to disable */
if(!(get_sctlr() & CR_C))
{
rt_kprintf("need enable cache!\n");
return;
}
set_sctlr(get_sctlr() & ~CR_C);
}
//icache
void rt_hw_icache_enable(void)
{
__asm_invalidate_icache_all();
set_sctlr(get_sctlr() | CR_I);
}
void rt_hw_icache_invalidate_all(void)
{
__asm_invalidate_icache_all();
}
void rt_hw_icache_disable(void)
{
set_sctlr(get_sctlr() & ~CR_I);
2021-03-27 17:51:56 +08:00
}