[components/mm] support for scalable memory management (#7277)
* [mm/page] multi-list page manager [mm/page] page debugger [libcpu/aarch64] hugepage support * [quality] remove void-arith * [format] remove kasan codes
This commit is contained in:
parent
90c69a0588
commit
470454d201
|
@ -11,6 +11,7 @@
|
|||
#include <rthw.h>
|
||||
#include <rtthread.h>
|
||||
#include <cpuport.h>
|
||||
#include <mm_aspace.h>
|
||||
|
||||
#ifdef RT_USING_VIRTIO_NET
|
||||
|
||||
|
@ -106,7 +107,7 @@ static struct pbuf *virtio_net_rx(rt_device_t dev)
|
|||
#ifdef RT_USING_SMP
|
||||
level = rt_spin_lock_irqsave(&virtio_dev->spinlock);
|
||||
#endif
|
||||
rt_memcpy(p->payload, (void *)VIRTIO_PA2VA(queue_rx->desc[id].addr), len);
|
||||
rt_memcpy(p->payload, (void *)queue_rx->desc[id].addr - PV_OFFSET, len);
|
||||
|
||||
queue_rx->used_idx++;
|
||||
|
||||
|
|
|
@ -26,7 +26,7 @@ int arch_user_space_init(struct rt_lwp *lwp)
|
|||
{
|
||||
size_t *mmu_table;
|
||||
|
||||
mmu_table = (size_t *)rt_pages_alloc(0);
|
||||
mmu_table = (size_t *)rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE);
|
||||
if (!mmu_table)
|
||||
{
|
||||
return -RT_ENOMEM;
|
||||
|
|
|
@ -91,7 +91,7 @@ int arch_user_space_init(struct rt_lwp *lwp)
|
|||
{
|
||||
rt_ubase_t *mmu_table;
|
||||
|
||||
mmu_table = (rt_ubase_t *)rt_pages_alloc(0);
|
||||
mmu_table = (rt_ubase_t *)rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE);
|
||||
if (!mmu_table)
|
||||
{
|
||||
return -RT_ENOMEM;
|
||||
|
|
|
@ -82,7 +82,7 @@ int arch_user_space_init(struct rt_lwp *lwp)
|
|||
{
|
||||
rt_size_t *mmu_table;
|
||||
|
||||
mmu_table = (rt_size_t *)rt_pages_alloc(0);
|
||||
mmu_table = (rt_size_t *)rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE);
|
||||
if (!mmu_table)
|
||||
{
|
||||
return -1;
|
||||
|
|
|
@ -9,6 +9,7 @@
|
|||
* 2018-11-02 heyuanjie fix complie error in iar
|
||||
* 2021-02-03 lizhirui add 64-bit arch support and riscv64 arch support
|
||||
* 2021-08-26 linzhenxing add lwp_setcwd\lwp_getcwd
|
||||
* 2023-02-20 wangxiaoyao inv icache before new app startup
|
||||
*/
|
||||
|
||||
#include <rthw.h>
|
||||
|
@ -1097,10 +1098,22 @@ static void _lwp_thread_entry(void *parameter)
|
|||
icache_invalid_all();
|
||||
}
|
||||
|
||||
/**
|
||||
* without ASID support, it will be a special case when trying to run application
|
||||
* and exit multiple times and a same page frame allocated to it bound to
|
||||
* different text segment. Then we are in a situation where icache contains
|
||||
* out-of-dated data and must be handle by the running core itself.
|
||||
* with ASID support, this should be a rare case that ASID & page frame both
|
||||
* identical to previous running application.
|
||||
*
|
||||
* For a new application loaded into memory, icache are seen as empty. And there
|
||||
* should be nothing in the icache entry to match. So this icache invalidation
|
||||
* operation should have barely influence.
|
||||
*/
|
||||
rt_hw_icache_invalidate_all();
|
||||
|
||||
#ifdef ARCH_MM_MMU
|
||||
arch_start_umode(lwp->args, lwp->text_entry, (void *)USER_STACK_VEND, tid->stack_addr + tid->stack_size);
|
||||
arch_start_umode(lwp->args, lwp->text_entry, (void *)USER_STACK_VEND, (char *)tid->stack_addr + tid->stack_size);
|
||||
#else
|
||||
arch_start_umode(lwp->args, lwp->text_entry, lwp->data_entry, (void *)((uint32_t)lwp->data_entry + lwp->data_size));
|
||||
#endif /* ARCH_MM_MMU */
|
||||
|
|
|
@ -6,6 +6,7 @@
|
|||
* Change Logs:
|
||||
* Date Author Notes
|
||||
* 2019-10-12 Jesven first version
|
||||
* 2023-02-20 wangxiaoyao adapt to mm
|
||||
*/
|
||||
#include <rthw.h>
|
||||
#include <rtthread.h>
|
||||
|
@ -17,8 +18,6 @@
|
|||
|
||||
#include <lwp_user_mm.h>
|
||||
#include <mmu.h>
|
||||
#include <mm_aspace.h>
|
||||
#include <mm_flag.h>
|
||||
|
||||
/* the kernel structure to represent a share-memory */
|
||||
struct lwp_shm_struct
|
||||
|
@ -64,7 +63,7 @@ static void on_shm_page_fault(struct rt_varea *varea, struct rt_aspace_fault_msg
|
|||
|
||||
/* map all share page frames to user space in a time */
|
||||
void *page = (void *)shm->addr;
|
||||
void *pg_paddr = page + PV_OFFSET;
|
||||
void *pg_paddr = (char *)page + PV_OFFSET;
|
||||
err = rt_varea_map_range(varea, varea->start, pg_paddr, shm->size);
|
||||
|
||||
if (err == RT_EOK)
|
||||
|
@ -140,7 +139,7 @@ static int _lwp_shmget(size_t key, size_t size, int create)
|
|||
|
||||
/* allocate pages up to 2's exponent to cover the required size */
|
||||
bit = rt_page_bits(size);
|
||||
page_addr = rt_pages_alloc(bit); /* virtual address */
|
||||
page_addr = rt_pages_alloc_ext(bit, PAGE_ANY_AVAILABLE); /* virtual address */
|
||||
if (!page_addr)
|
||||
{
|
||||
goto err;
|
||||
|
|
|
@ -306,7 +306,7 @@ static void _crt_thread_entry(void *parameter)
|
|||
user_stack &= ~7; //align 8
|
||||
|
||||
#ifdef ARCH_MM_MMU
|
||||
arch_crt_start_umode(parameter, tid->user_entry, (void *)user_stack, tid->stack_addr + tid->stack_size);
|
||||
arch_crt_start_umode(parameter, tid->user_entry, (void *)user_stack, (char *)tid->stack_addr + tid->stack_size);
|
||||
#else
|
||||
set_user_context((void*)user_stack);
|
||||
arch_start_umode(parameter, tid->user_entry, ((struct rt_lwp *)tid->lwp)->data_entry, (void*)user_stack);
|
||||
|
@ -1861,7 +1861,7 @@ static char *_insert_args(int new_argc, char *new_argv[], struct lwp_args_info *
|
|||
{
|
||||
goto quit;
|
||||
}
|
||||
page = rt_pages_alloc(0); /* 1 page */
|
||||
page = rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE); /* 1 page */
|
||||
if (!page)
|
||||
{
|
||||
goto quit;
|
||||
|
@ -2065,7 +2065,7 @@ int load_ldso(struct rt_lwp *lwp, char *exec_name, char *const argv[], char *con
|
|||
}
|
||||
}
|
||||
|
||||
page = rt_pages_alloc(0); /* 1 page */
|
||||
page = rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE); /* 1 page */
|
||||
if (!page)
|
||||
{
|
||||
SET_ERRNO(ENOMEM);
|
||||
|
@ -2252,7 +2252,7 @@ sysret_t sys_execve(const char *path, char *const argv[], char *const envp[])
|
|||
SET_ERRNO(EINVAL);
|
||||
goto quit;
|
||||
}
|
||||
page = rt_pages_alloc(0); /* 1 page */
|
||||
page = rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE); /* 1 page */
|
||||
if (!page)
|
||||
{
|
||||
SET_ERRNO(ENOMEM);
|
||||
|
@ -2396,7 +2396,7 @@ sysret_t sys_execve(const char *path, char *const argv[], char *const envp[])
|
|||
arch_start_umode(lwp->args,
|
||||
lwp->text_entry,
|
||||
(void*)USER_STACK_VEND,
|
||||
thread->stack_addr + thread->stack_size);
|
||||
(char *)thread->stack_addr + thread->stack_size);
|
||||
/* never reach here */
|
||||
}
|
||||
return -EINVAL;
|
||||
|
|
|
@ -10,6 +10,7 @@
|
|||
* 2021-02-12 lizhirui add 64-bit support for lwp_brk
|
||||
* 2021-02-19 lizhirui add riscv64 support for lwp_user_accessable and lwp_get_from_user
|
||||
* 2021-06-07 lizhirui modify user space bound check
|
||||
* 2022-12-25 wangxiaoyao adapt to new mm
|
||||
*/
|
||||
|
||||
#include <rtthread.h>
|
||||
|
@ -122,7 +123,7 @@ static void _user_do_page_fault(struct rt_varea *varea,
|
|||
|
||||
if (lwp_objs->source)
|
||||
{
|
||||
void *paddr = rt_hw_mmu_v2p(lwp_objs->source, msg->fault_vaddr);
|
||||
char *paddr = rt_hw_mmu_v2p(lwp_objs->source, msg->fault_vaddr);
|
||||
if (paddr != ARCH_MAP_FAILED)
|
||||
{
|
||||
void *vaddr;
|
||||
|
@ -130,7 +131,7 @@ static void _user_do_page_fault(struct rt_varea *varea,
|
|||
|
||||
if (!(varea->flag & MMF_TEXT))
|
||||
{
|
||||
void *cp = rt_pages_alloc(0);
|
||||
void *cp = rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE);
|
||||
if (cp)
|
||||
{
|
||||
memcpy(cp, vaddr, ARCH_PAGE_SIZE);
|
||||
|
@ -220,9 +221,9 @@ int lwp_unmap_user(struct rt_lwp *lwp, void *va)
|
|||
static void _dup_varea(rt_varea_t varea, struct rt_lwp *src_lwp,
|
||||
rt_aspace_t dst)
|
||||
{
|
||||
void *vaddr = varea->start;
|
||||
void *vend = vaddr + varea->size;
|
||||
if (vaddr < (void *)USER_STACK_VSTART || vaddr >= (void *)USER_STACK_VEND)
|
||||
char *vaddr = varea->start;
|
||||
char *vend = vaddr + varea->size;
|
||||
if (vaddr < (char *)USER_STACK_VSTART || vaddr >= (char *)USER_STACK_VEND)
|
||||
{
|
||||
while (vaddr != vend)
|
||||
{
|
||||
|
@ -430,7 +431,7 @@ void *lwp_map_user_phy(struct rt_lwp *lwp, void *map_va, void *map_pa,
|
|||
size_t map_size, int cached)
|
||||
{
|
||||
int err;
|
||||
void *va;
|
||||
char *va;
|
||||
size_t offset = 0;
|
||||
|
||||
if (!map_size)
|
||||
|
@ -458,7 +459,7 @@ void *lwp_map_user_phy(struct rt_lwp *lwp, void *map_va, void *map_pa,
|
|||
rt_size_t attr = cached ? MMU_MAP_U_RWCB : MMU_MAP_U_RW;
|
||||
|
||||
err =
|
||||
rt_aspace_map_phy(lwp->aspace, &hint, attr, MM_PA_TO_OFF(map_pa), &va);
|
||||
rt_aspace_map_phy(lwp->aspace, &hint, attr, MM_PA_TO_OFF(map_pa), (void **)&va);
|
||||
if (err != RT_EOK)
|
||||
{
|
||||
va = RT_NULL;
|
||||
|
|
|
@ -78,7 +78,7 @@ static struct rt_varea *search(struct util_avl_root *root,
|
|||
{
|
||||
rt_varea_t varea = VAREA_ENTRY(node);
|
||||
int cmp = compare(range.start, range.end, varea->start,
|
||||
varea->start + varea->size - 1);
|
||||
(char *)varea->start + varea->size - 1);
|
||||
|
||||
if (cmp < 0)
|
||||
{
|
||||
|
@ -118,7 +118,7 @@ rt_varea_t _aspace_bst_search_exceed(struct rt_aspace *aspace, void *start)
|
|||
if (cmp < 0)
|
||||
{
|
||||
/* varae exceed start */
|
||||
ptrdiff_t off = va_s - start;
|
||||
ptrdiff_t off = (char *)va_s - (char *)start;
|
||||
if (off < min_off)
|
||||
{
|
||||
min_off = off;
|
||||
|
|
|
@ -33,12 +33,12 @@ enum ioremap_type
|
|||
|
||||
static void *_ioremap_type(void *paddr, size_t size, enum ioremap_type type)
|
||||
{
|
||||
void *v_addr = NULL;
|
||||
char *v_addr = NULL;
|
||||
size_t attr;
|
||||
size_t lo_off;
|
||||
int err;
|
||||
|
||||
lo_off = (uintptr_t)paddr & ARCH_PAGE_MASK;
|
||||
lo_off = (rt_ubase_t)paddr & ARCH_PAGE_MASK;
|
||||
|
||||
struct rt_mm_va_hint hint = {
|
||||
.prefer = RT_NULL,
|
||||
|
@ -62,7 +62,7 @@ static void *_ioremap_type(void *paddr, size_t size, enum ioremap_type type)
|
|||
default:
|
||||
return v_addr;
|
||||
}
|
||||
err = rt_aspace_map_phy(&rt_kernel_space, &hint, attr, MM_PA_TO_OFF(paddr), &v_addr);
|
||||
err = rt_aspace_map_phy(&rt_kernel_space, &hint, attr, MM_PA_TO_OFF(paddr), (void **)&v_addr);
|
||||
|
||||
if (err)
|
||||
{
|
||||
|
|
|
@ -134,24 +134,16 @@ static int _do_named_map(rt_aspace_t aspace, void *vaddr, rt_size_t length,
|
|||
int err = RT_EOK;
|
||||
|
||||
/* it's ensured by caller that (void*)end will not overflow */
|
||||
void *end = vaddr + length;
|
||||
void *phyaddr = (void *)(offset << MM_PAGE_SHIFT);
|
||||
while (vaddr != end)
|
||||
|
||||
void *ret = rt_hw_mmu_map(aspace, vaddr, phyaddr, length, attr);
|
||||
if (ret == RT_NULL)
|
||||
{
|
||||
/* TODO try to map with huge TLB, when flag & HUGEPAGE */
|
||||
rt_size_t pgsz = ARCH_PAGE_SIZE;
|
||||
void *ret = rt_hw_mmu_map(aspace, vaddr, phyaddr, pgsz, attr);
|
||||
if (ret == RT_NULL)
|
||||
{
|
||||
err = -RT_ERROR;
|
||||
break;
|
||||
}
|
||||
vaddr += pgsz;
|
||||
phyaddr += pgsz;
|
||||
err = -RT_ERROR;
|
||||
}
|
||||
|
||||
if (err == RT_EOK)
|
||||
rt_hw_tlb_invalidate_range(aspace, end - length, length, ARCH_PAGE_SIZE);
|
||||
rt_hw_tlb_invalidate_range(aspace, vaddr, length, ARCH_PAGE_SIZE);
|
||||
|
||||
return err;
|
||||
}
|
||||
|
@ -164,7 +156,7 @@ rt_inline void _do_page_fault(struct rt_aspace_fault_msg *msg, rt_size_t off,
|
|||
msg->fault_vaddr = vaddr;
|
||||
msg->fault_op = MM_FAULT_OP_READ;
|
||||
msg->fault_type = MM_FAULT_TYPE_PAGE_FAULT;
|
||||
msg->response.status = -1;
|
||||
msg->response.status = MM_FAULT_STATUS_UNRECOVERABLE;
|
||||
msg->response.vaddr = 0;
|
||||
msg->response.size = 0;
|
||||
|
||||
|
@ -180,9 +172,9 @@ int _varea_map_with_msg(rt_varea_t varea, struct rt_aspace_fault_msg *msg)
|
|||
* the page returned by handler is not checked
|
||||
* cause no much assumption can make on it
|
||||
*/
|
||||
void *store = msg->response.vaddr;
|
||||
char *store = msg->response.vaddr;
|
||||
rt_size_t store_sz = msg->response.size;
|
||||
if (msg->fault_vaddr + store_sz > varea->start + varea->size)
|
||||
if ((char *)msg->fault_vaddr + store_sz > (char *)varea->start + varea->size)
|
||||
{
|
||||
LOG_W("%s: too much (0x%lx) of buffer on vaddr %p is provided",
|
||||
__func__, store_sz, msg->fault_vaddr);
|
||||
|
@ -232,9 +224,9 @@ static int _do_prefetch(rt_aspace_t aspace, rt_varea_t varea, void *start,
|
|||
int err = RT_EOK;
|
||||
|
||||
/* it's ensured by caller that start & size ara page-aligned */
|
||||
void *end = start + size;
|
||||
void *vaddr = start;
|
||||
rt_size_t off = varea->offset + ((start - varea->start) >> ARCH_PAGE_SHIFT);
|
||||
char *end = (char *)start + size;
|
||||
char *vaddr = start;
|
||||
rt_size_t off = varea->offset + ((vaddr - (char *)varea->start) >> ARCH_PAGE_SHIFT);
|
||||
|
||||
while (vaddr != end)
|
||||
{
|
||||
|
@ -243,8 +235,10 @@ static int _do_prefetch(rt_aspace_t aspace, rt_varea_t varea, void *start,
|
|||
_do_page_fault(&msg, off, vaddr, varea->mem_obj, varea);
|
||||
|
||||
if (_varea_map_with_msg(varea, &msg))
|
||||
{
|
||||
err = -RT_ENOMEM;
|
||||
break;
|
||||
|
||||
}
|
||||
/**
|
||||
* It's hard to identify the mapping pattern on a customized handler
|
||||
* So we terminate the prefetch process on that case
|
||||
|
@ -386,7 +380,7 @@ rt_varea_t _varea_create(void *start, rt_size_t size)
|
|||
}
|
||||
|
||||
#define _IS_OVERFLOW(start, length) ((length) > (0ul - (uintptr_t)(start)))
|
||||
#define _IS_OVERSIZE(start, length, limit_s, limit_sz) (((length) + (rt_size_t)((start) - (limit_start))) > (limit_size))
|
||||
#define _IS_OVERSIZE(start, length, limit_s, limit_sz) (((length) + (rt_size_t)((char *)(start) - (char *)(limit_start))) > (limit_size))
|
||||
|
||||
static inline int _not_in_range(void *start, rt_size_t length,
|
||||
void *limit_start, rt_size_t limit_size)
|
||||
|
@ -449,6 +443,10 @@ int rt_aspace_map(rt_aspace_t aspace, void **addr, rt_size_t length,
|
|||
{
|
||||
rt_free(varea);
|
||||
}
|
||||
else
|
||||
{
|
||||
*addr = varea->start;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
|
@ -461,10 +459,7 @@ int rt_aspace_map(rt_aspace_t aspace, void **addr, rt_size_t length,
|
|||
{
|
||||
*addr = NULL;
|
||||
}
|
||||
else
|
||||
{
|
||||
*addr = varea->start;
|
||||
}
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
|
@ -642,7 +637,7 @@ int rt_aspace_unmap(rt_aspace_t aspace, void *addr)
|
|||
if (_not_in_range(addr, 1, aspace->start, aspace->size))
|
||||
{
|
||||
LOG_I("%s: %lx not in range of aspace[%lx:%lx]", __func__, addr,
|
||||
aspace->start, aspace->start + aspace->size);
|
||||
aspace->start, (char *)aspace->start + aspace->size);
|
||||
return -RT_EINVAL;
|
||||
}
|
||||
|
||||
|
@ -658,7 +653,7 @@ static inline void *_lower(void *a, void *b)
|
|||
|
||||
static inline void *_align(void *va, rt_ubase_t align_mask)
|
||||
{
|
||||
return (void *)((rt_ubase_t)(va + ~align_mask) & align_mask);
|
||||
return (void *)((rt_ubase_t)((char *)va + ~align_mask) & align_mask);
|
||||
}
|
||||
|
||||
static void *_ascending_search(rt_varea_t varea, rt_size_t req_size,
|
||||
|
@ -667,17 +662,17 @@ static void *_ascending_search(rt_varea_t varea, rt_size_t req_size,
|
|||
void *ret = RT_NULL;
|
||||
while (varea && varea->start < limit.end)
|
||||
{
|
||||
void *candidate = varea->start + varea->size;
|
||||
char *candidate = (char *)varea->start + varea->size;
|
||||
candidate = _align(candidate, align_mask);
|
||||
|
||||
if (candidate > limit.end || limit.end - candidate + 1 < req_size)
|
||||
if (candidate > (char *)limit.end || (char *)limit.end - candidate + 1 < req_size)
|
||||
break;
|
||||
|
||||
rt_varea_t nx_va = ASPACE_VAREA_NEXT(varea);
|
||||
if (nx_va)
|
||||
{
|
||||
rt_size_t gap_size =
|
||||
_lower(limit.end, nx_va->start - 1) - candidate + 1;
|
||||
(char *)_lower(limit.end, (char *)nx_va->start - 1) - candidate + 1;
|
||||
if (gap_size >= req_size)
|
||||
{
|
||||
ret = candidate;
|
||||
|
@ -703,15 +698,15 @@ static void *_find_head_and_asc_search(rt_aspace_t aspace, rt_size_t req_size,
|
|||
rt_varea_t varea = _aspace_bst_search_exceed(aspace, limit.start);
|
||||
if (varea)
|
||||
{
|
||||
void *candidate = _align(limit.start, align_mask);
|
||||
rt_size_t gap_size = varea->start - candidate;
|
||||
char *candidate = _align(limit.start, align_mask);
|
||||
rt_size_t gap_size = (char *)varea->start - candidate;
|
||||
if (gap_size >= req_size)
|
||||
{
|
||||
rt_varea_t former = _aspace_bst_search(aspace, limit.start);
|
||||
if (former)
|
||||
{
|
||||
candidate = _align(former->start + former->size, align_mask);
|
||||
gap_size = varea->start - candidate;
|
||||
candidate = _align((char *)former->start + former->size, align_mask);
|
||||
gap_size = (char *)varea->start - candidate;
|
||||
|
||||
if (gap_size >= req_size)
|
||||
va = candidate;
|
||||
|
@ -730,12 +725,12 @@ static void *_find_head_and_asc_search(rt_aspace_t aspace, rt_size_t req_size,
|
|||
}
|
||||
else
|
||||
{
|
||||
void *candidate;
|
||||
char *candidate;
|
||||
rt_size_t gap_size;
|
||||
|
||||
candidate = limit.start;
|
||||
candidate = _align(candidate, align_mask);
|
||||
gap_size = limit.end - candidate + 1;
|
||||
gap_size = (char *)limit.end - candidate + 1;
|
||||
|
||||
if (gap_size >= req_size)
|
||||
va = candidate;
|
||||
|
@ -750,7 +745,7 @@ static void *_find_free(rt_aspace_t aspace, void *prefer, rt_size_t req_size,
|
|||
{
|
||||
rt_varea_t varea = NULL;
|
||||
void *va = RT_NULL;
|
||||
struct _mm_range limit = {limit_start, limit_start + limit_size - 1};
|
||||
struct _mm_range limit = {limit_start, (char *)limit_start + limit_size - 1};
|
||||
|
||||
rt_ubase_t align_mask = ~0ul;
|
||||
if (flags & MMF_REQUEST_ALIGN)
|
||||
|
@ -762,7 +757,7 @@ static void *_find_free(rt_aspace_t aspace, void *prefer, rt_size_t req_size,
|
|||
{
|
||||
/* if prefer and free, just return the prefer region */
|
||||
prefer = _align(prefer, align_mask);
|
||||
struct _mm_range range = {prefer, prefer + req_size - 1};
|
||||
struct _mm_range range = {prefer, (char *)prefer + req_size - 1};
|
||||
varea = _aspace_bst_search_overlap(aspace, range);
|
||||
|
||||
if (!varea)
|
||||
|
@ -780,7 +775,7 @@ static void *_find_free(rt_aspace_t aspace, void *prefer, rt_size_t req_size,
|
|||
if (va == RT_NULL)
|
||||
{
|
||||
/* rewind to first range */
|
||||
limit.end = varea->start - 1;
|
||||
limit.end = (char *)varea->start - 1;
|
||||
va = _find_head_and_asc_search(aspace, req_size, align_mask,
|
||||
limit);
|
||||
}
|
||||
|
@ -798,7 +793,7 @@ int rt_aspace_load_page(rt_aspace_t aspace, void *addr, rt_size_t npage)
|
|||
{
|
||||
int err = RT_EOK;
|
||||
rt_varea_t varea;
|
||||
void *end = addr + (npage << ARCH_PAGE_SHIFT);
|
||||
char *end = (char *)addr + (npage << ARCH_PAGE_SHIFT);
|
||||
|
||||
WR_LOCK(aspace);
|
||||
varea = _aspace_bst_search(aspace, addr);
|
||||
|
@ -809,7 +804,7 @@ int rt_aspace_load_page(rt_aspace_t aspace, void *addr, rt_size_t npage)
|
|||
LOG_W("%s: varea not exist", __func__);
|
||||
err = -RT_ENOENT;
|
||||
}
|
||||
else if (addr >= end || (rt_size_t)addr & ARCH_PAGE_MASK ||
|
||||
else if ((char *)addr >= end || (rt_size_t)addr & ARCH_PAGE_MASK ||
|
||||
_not_in_range(addr, npage << ARCH_PAGE_SHIFT, varea->start,
|
||||
varea->size))
|
||||
{
|
||||
|
@ -938,12 +933,12 @@ static int _dump(rt_varea_t varea, void *arg)
|
|||
{
|
||||
if (varea->mem_obj && varea->mem_obj->get_name)
|
||||
{
|
||||
rt_kprintf("[%p - %p] %s\n", varea->start, varea->start + varea->size,
|
||||
rt_kprintf("[%p - %p] %s\n", varea->start, (char *)varea->start + varea->size,
|
||||
varea->mem_obj->get_name(varea));
|
||||
}
|
||||
else
|
||||
{
|
||||
rt_kprintf("[%p - %p] phy-map\n", varea->start, varea->start + varea->size);
|
||||
rt_kprintf("[%p - %p] phy-map\n", varea->start, (char *)varea->start + varea->size);
|
||||
rt_kprintf("\t\\_ paddr = %p\n", varea->offset << MM_PAGE_SHIFT);
|
||||
}
|
||||
return 0;
|
||||
|
|
|
@ -104,7 +104,7 @@ int rt_aspace_fault_try_fix(struct rt_aspace_fault_msg *msg)
|
|||
if (varea)
|
||||
{
|
||||
void *pa = rt_hw_mmu_v2p(aspace, msg->fault_vaddr);
|
||||
msg->off = (msg->fault_vaddr - varea->start) >> ARCH_PAGE_SHIFT;
|
||||
msg->off = ((char *)msg->fault_vaddr - (char *)varea->start) >> ARCH_PAGE_SHIFT;
|
||||
|
||||
/* permission checked by fault op */
|
||||
switch (msg->fault_op)
|
||||
|
|
|
@ -56,21 +56,21 @@ void rt_varea_pgmgr_pop_all(rt_varea_t varea)
|
|||
|
||||
void rt_varea_pgmgr_pop(rt_varea_t varea, void *vaddr, rt_size_t size)
|
||||
{
|
||||
void *vend = vaddr + size;
|
||||
void *vend = (char *)vaddr + size;
|
||||
while (vaddr != vend)
|
||||
{
|
||||
rt_page_t page = rt_page_addr2page(vaddr);
|
||||
page->pre->next = page->next;
|
||||
page->next->pre = page->pre;
|
||||
rt_pages_free(vaddr, 0);
|
||||
vaddr += ARCH_PAGE_SIZE;
|
||||
vaddr = (char *)vaddr + ARCH_PAGE_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
static void on_page_fault(struct rt_varea *varea, struct rt_aspace_fault_msg *msg)
|
||||
{
|
||||
void *page;
|
||||
page = rt_pages_alloc(0);
|
||||
page = rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE);
|
||||
|
||||
if (!page)
|
||||
{
|
||||
|
|
|
@ -8,6 +8,7 @@
|
|||
* 2019-11-01 Jesven The first version
|
||||
* 2022-12-13 WangXiaoyao Hot-pluggable, extensible
|
||||
* page management algorithm
|
||||
* 2023-02-20 WangXiaoyao Multi-list page-management
|
||||
*/
|
||||
#include <rtthread.h>
|
||||
|
||||
|
@ -39,7 +40,8 @@ static void *init_mpr_cont_start;
|
|||
|
||||
static struct rt_varea mpr_varea;
|
||||
|
||||
static struct rt_page *page_list[RT_PAGE_MAX_ORDER];
|
||||
static struct rt_page *page_list_low[RT_PAGE_MAX_ORDER];
|
||||
static struct rt_page *page_list_high[RT_PAGE_MAX_ORDER];
|
||||
|
||||
#define page_start ((rt_page_t)rt_mpr_start)
|
||||
|
||||
|
@ -61,21 +63,18 @@ static void hint_free(rt_mm_va_hint_t hint)
|
|||
|
||||
static void on_page_fault(struct rt_varea *varea, struct rt_aspace_fault_msg *msg)
|
||||
{
|
||||
void *init_start = (void *)init_mpr_align_start;
|
||||
void *init_end = (void *)init_mpr_align_end;
|
||||
if (msg->fault_vaddr < init_end && msg->fault_vaddr >= init_start)
|
||||
char *init_start = (void *)init_mpr_align_start;
|
||||
char *init_end = (void *)init_mpr_align_end;
|
||||
if ((char *)msg->fault_vaddr < init_end && (char *)msg->fault_vaddr >= init_start)
|
||||
{
|
||||
rt_size_t offset = msg->fault_vaddr - init_start;
|
||||
rt_size_t offset = (char *)msg->fault_vaddr - init_start;
|
||||
msg->response.status = MM_FAULT_STATUS_OK;
|
||||
msg->response.vaddr = init_mpr_cont_start + offset;
|
||||
msg->response.vaddr = (char *)init_mpr_cont_start + offset;
|
||||
msg->response.size = ARCH_PAGE_SIZE;
|
||||
}
|
||||
else
|
||||
{
|
||||
void *raw_page = rt_pages_alloc(0);
|
||||
msg->response.status = MM_FAULT_STATUS_OK;
|
||||
msg->response.vaddr = raw_page;
|
||||
msg->response.size = ARCH_PAGE_SIZE;
|
||||
rt_mm_dummy_mapper.on_page_fault(varea, msg);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -85,15 +84,123 @@ static struct rt_mem_obj mm_page_mapper = {
|
|||
.hint_free = hint_free,
|
||||
};
|
||||
|
||||
#ifdef RT_DEBUG_PAGE_LEAK
|
||||
static volatile int enable;
|
||||
static rt_page_t _trace_head;
|
||||
#define TRACE_ALLOC(pg, size) _trace_alloc(pg, __builtin_return_address(0), size)
|
||||
#define TRACE_FREE(pgaddr, size) _trace_free(pgaddr, __builtin_return_address(0), size)
|
||||
|
||||
void rt_page_leak_trace_start()
|
||||
{
|
||||
// TODO multicore safety
|
||||
_trace_head = NULL;
|
||||
enable = 1;
|
||||
}
|
||||
MSH_CMD_EXPORT(rt_page_leak_trace_start, start page leak tracer);
|
||||
|
||||
static void _collect()
|
||||
{
|
||||
rt_page_t page = _trace_head;
|
||||
if (!page)
|
||||
{
|
||||
rt_kputs("ok!\n");
|
||||
}
|
||||
|
||||
while (page)
|
||||
{
|
||||
rt_page_t next = page->next;
|
||||
void *pg_va = rt_page_page2addr(page);
|
||||
LOG_W("LEAK: %p, allocator: %p, size bits: %lx", pg_va, page->caller, page->trace_size);
|
||||
rt_pages_free(pg_va, page->trace_size);
|
||||
page = next;
|
||||
}
|
||||
}
|
||||
|
||||
void rt_page_leak_trace_stop()
|
||||
{
|
||||
// TODO multicore safety
|
||||
enable = 0;
|
||||
_collect();
|
||||
}
|
||||
MSH_CMD_EXPORT(rt_page_leak_trace_stop, stop page leak tracer);
|
||||
|
||||
static void _trace_alloc(rt_page_t page, void *caller, size_t size_bits)
|
||||
{
|
||||
if (enable)
|
||||
{
|
||||
page->caller = caller;
|
||||
page->trace_size = size_bits;
|
||||
page->tl_prev = NULL;
|
||||
page->tl_next = NULL;
|
||||
|
||||
if (_trace_head == NULL)
|
||||
{
|
||||
_trace_head = page;
|
||||
}
|
||||
else
|
||||
{
|
||||
_trace_head->tl_prev = page;
|
||||
page->tl_next = _trace_head;
|
||||
_trace_head = page;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void _report(rt_page_t page, size_bits, char *msg)
|
||||
{
|
||||
void *pg_va = rt_page_page2addr(page);
|
||||
LOG_W("%s: %p, allocator: %p, size bits: %lx", msg, pg_va, page->caller, page->trace_size);
|
||||
rt_kputs("backtrace\n");
|
||||
rt_hw_backtrace(0, 0);
|
||||
}
|
||||
|
||||
static void _trace_free(rt_page_t page, void *caller, size_t size_bits)
|
||||
{
|
||||
if (enable)
|
||||
{
|
||||
/* free after free */
|
||||
if (page->trace_size == 0xabadcafe)
|
||||
{
|
||||
_report("free after free")
|
||||
return ;
|
||||
}
|
||||
else if (page->trace_size != size_bits)
|
||||
{
|
||||
rt_kprintf("free with size bits %lx\n", size_bits);
|
||||
_report("incompatible size bits parameter");
|
||||
return ;
|
||||
}
|
||||
|
||||
if (page->ref_cnt == 1)
|
||||
{
|
||||
if (page->tl_prev)
|
||||
page->tl_prev->tl_next = page->tl_next;
|
||||
if (page->tl_next)
|
||||
page->tl_next->tl_prev = page->tl_prev;
|
||||
|
||||
if (page == _trace_head)
|
||||
_trace_head = page->next;
|
||||
|
||||
page->tl_prev = NULL;
|
||||
page->tl_next = NULL;
|
||||
page->trace_size = 0xabadcafe;
|
||||
}
|
||||
}
|
||||
}
|
||||
#else
|
||||
#define TRACE_ALLOC(x, y)
|
||||
#define TRACE_FREE(x, y)
|
||||
#endif
|
||||
|
||||
static inline void *page_to_addr(rt_page_t page)
|
||||
{
|
||||
return (void *)((page - page_start) << ARCH_PAGE_SHIFT) - PV_OFFSET;
|
||||
return (void *)(((page - page_start) << ARCH_PAGE_SHIFT) - PV_OFFSET);
|
||||
}
|
||||
|
||||
static inline rt_page_t addr_to_page(rt_page_t pg_start, void *addr)
|
||||
{
|
||||
addr += PV_OFFSET;
|
||||
return &pg_start[((uintptr_t)addr >> ARCH_PAGE_SHIFT)];
|
||||
addr = (char *)addr + PV_OFFSET;
|
||||
return &pg_start[((rt_ubase_t)addr >> ARCH_PAGE_SHIFT)];
|
||||
}
|
||||
|
||||
#define FLOOR(val, align) (((rt_size_t)(val) + (align)-1) & ~((align)-1))
|
||||
|
@ -143,7 +250,7 @@ void *rt_page_page2addr(struct rt_page *p)
|
|||
return page_to_addr(p);
|
||||
}
|
||||
|
||||
static inline struct rt_page *buddy_get(struct rt_page *p,
|
||||
static inline struct rt_page *_buddy_get(struct rt_page *p,
|
||||
rt_uint32_t size_bits)
|
||||
{
|
||||
rt_size_t addr;
|
||||
|
@ -153,7 +260,7 @@ static inline struct rt_page *buddy_get(struct rt_page *p,
|
|||
return rt_page_addr2page((void *)addr);
|
||||
}
|
||||
|
||||
static void page_remove(struct rt_page *p, rt_uint32_t size_bits)
|
||||
static void _page_remove(rt_page_t page_list[], struct rt_page *p, rt_uint32_t size_bits)
|
||||
{
|
||||
if (p->pre)
|
||||
{
|
||||
|
@ -172,7 +279,7 @@ static void page_remove(struct rt_page *p, rt_uint32_t size_bits)
|
|||
p->size_bits = ARCH_ADDRESS_WIDTH_BITS;
|
||||
}
|
||||
|
||||
static void page_insert(struct rt_page *p, rt_uint32_t size_bits)
|
||||
static void _page_insert(rt_page_t page_list[], struct rt_page *p, rt_uint32_t size_bits)
|
||||
{
|
||||
p->next = page_list[size_bits];
|
||||
if (p->next)
|
||||
|
@ -194,7 +301,7 @@ static void _pages_ref_inc(struct rt_page *p, rt_uint32_t size_bits)
|
|||
idx = idx & ~((1UL << size_bits) - 1);
|
||||
|
||||
page_head = page_start + idx;
|
||||
page_head = (void *)page_head + early_offset;
|
||||
page_head = (void *)((char *)page_head + early_offset);
|
||||
page_head->ref_cnt++;
|
||||
}
|
||||
|
||||
|
@ -211,13 +318,13 @@ static int _pages_ref_get(struct rt_page *p, rt_uint32_t size_bits)
|
|||
return page_head->ref_cnt;
|
||||
}
|
||||
|
||||
static int _pages_free(struct rt_page *p, rt_uint32_t size_bits)
|
||||
static int _pages_free(rt_page_t page_list[], struct rt_page *p, rt_uint32_t size_bits)
|
||||
{
|
||||
rt_uint32_t level = size_bits;
|
||||
struct rt_page *buddy;
|
||||
|
||||
RT_ASSERT(p >= page_start);
|
||||
RT_ASSERT((void *)p < rt_mpr_start + rt_mpr_size);
|
||||
RT_ASSERT((char *)p < (char *)rt_mpr_start + rt_mpr_size);
|
||||
RT_ASSERT(rt_kmem_v2p(p));
|
||||
RT_ASSERT(p->ref_cnt > 0);
|
||||
RT_ASSERT(p->size_bits == ARCH_ADDRESS_WIDTH_BITS);
|
||||
|
@ -231,10 +338,10 @@ static int _pages_free(struct rt_page *p, rt_uint32_t size_bits)
|
|||
|
||||
while (level < RT_PAGE_MAX_ORDER - 1)
|
||||
{
|
||||
buddy = buddy_get(p, level);
|
||||
buddy = _buddy_get(p, level);
|
||||
if (buddy && buddy->size_bits == level)
|
||||
{
|
||||
page_remove(buddy, level);
|
||||
_page_remove(page_list, buddy, level);
|
||||
p = (p < buddy) ? p : buddy;
|
||||
level++;
|
||||
}
|
||||
|
@ -243,18 +350,18 @@ static int _pages_free(struct rt_page *p, rt_uint32_t size_bits)
|
|||
break;
|
||||
}
|
||||
}
|
||||
page_insert(p, level);
|
||||
_page_insert(page_list, p, level);
|
||||
return 1;
|
||||
}
|
||||
|
||||
static struct rt_page *_pages_alloc(rt_uint32_t size_bits)
|
||||
static struct rt_page *_pages_alloc(rt_page_t page_list[], rt_uint32_t size_bits)
|
||||
{
|
||||
struct rt_page *p;
|
||||
|
||||
if (page_list[size_bits])
|
||||
{
|
||||
p = page_list[size_bits];
|
||||
page_remove(p, size_bits);
|
||||
_page_remove(page_list, p, size_bits);
|
||||
}
|
||||
else
|
||||
{
|
||||
|
@ -273,11 +380,11 @@ static struct rt_page *_pages_alloc(rt_uint32_t size_bits)
|
|||
}
|
||||
|
||||
p = page_list[level];
|
||||
page_remove(p, level);
|
||||
_page_remove(page_list, p, level);
|
||||
while (level > size_bits)
|
||||
{
|
||||
page_insert(p, level - 1);
|
||||
p = buddy_get(p, level - 1);
|
||||
_page_insert(page_list, p, level - 1);
|
||||
p = _buddy_get(p, level - 1);
|
||||
level--;
|
||||
}
|
||||
}
|
||||
|
@ -286,12 +393,12 @@ static struct rt_page *_pages_alloc(rt_uint32_t size_bits)
|
|||
return p;
|
||||
}
|
||||
|
||||
static void _early_page_remove(rt_page_t page, rt_uint32_t size_bits)
|
||||
static void _early_page_remove(rt_page_t page_list[], rt_page_t page, rt_uint32_t size_bits)
|
||||
{
|
||||
rt_page_t page_cont = (void *)page + early_offset;
|
||||
rt_page_t page_cont = (rt_page_t)((char *)page + early_offset);
|
||||
if (page_cont->pre)
|
||||
{
|
||||
rt_page_t pre_cont = (void *)page_cont->pre + early_offset;
|
||||
rt_page_t pre_cont = (rt_page_t)((char *)page_cont->pre + early_offset);
|
||||
pre_cont->next = page_cont->next;
|
||||
}
|
||||
else
|
||||
|
@ -301,23 +408,23 @@ static void _early_page_remove(rt_page_t page, rt_uint32_t size_bits)
|
|||
|
||||
if (page_cont->next)
|
||||
{
|
||||
rt_page_t next_cont = (void *)page_cont->next + early_offset;
|
||||
rt_page_t next_cont = (rt_page_t)((char *)page_cont->next + early_offset);
|
||||
next_cont->pre = page_cont->pre;
|
||||
}
|
||||
|
||||
page_cont->size_bits = ARCH_ADDRESS_WIDTH_BITS;
|
||||
}
|
||||
|
||||
static void _early_page_insert(rt_page_t page, int size_bits)
|
||||
static void _early_page_insert(rt_page_t page_list[], rt_page_t page, int size_bits)
|
||||
{
|
||||
RT_ASSERT((void *)page >= rt_mpr_start &&
|
||||
(void *)page - rt_mpr_start < +rt_mpr_size);
|
||||
rt_page_t page_cont = (void *)page + early_offset;
|
||||
((char *)page - (char *)rt_mpr_start) < rt_mpr_size);
|
||||
rt_page_t page_cont = (rt_page_t)((char *)page + early_offset);
|
||||
|
||||
page_cont->next = page_list[size_bits];
|
||||
if (page_cont->next)
|
||||
{
|
||||
rt_page_t next_cont = (void *)page_cont->next + early_offset;
|
||||
rt_page_t next_cont = (rt_page_t)((char *)page_cont->next + early_offset);
|
||||
next_cont->pre = page;
|
||||
}
|
||||
page_cont->pre = 0;
|
||||
|
@ -325,14 +432,14 @@ static void _early_page_insert(rt_page_t page, int size_bits)
|
|||
page_cont->size_bits = size_bits;
|
||||
}
|
||||
|
||||
static struct rt_page *_early_pages_alloc(rt_uint32_t size_bits)
|
||||
static struct rt_page *_early_pages_alloc(rt_page_t page_list[], rt_uint32_t size_bits)
|
||||
{
|
||||
struct rt_page *p;
|
||||
|
||||
if (page_list[size_bits])
|
||||
{
|
||||
p = page_list[size_bits];
|
||||
_early_page_remove(p, size_bits);
|
||||
_early_page_remove(page_list, p, size_bits);
|
||||
}
|
||||
else
|
||||
{
|
||||
|
@ -351,20 +458,35 @@ static struct rt_page *_early_pages_alloc(rt_uint32_t size_bits)
|
|||
}
|
||||
|
||||
p = page_list[level];
|
||||
_early_page_remove(p, level);
|
||||
_early_page_remove(page_list, p, level);
|
||||
while (level > size_bits)
|
||||
{
|
||||
_early_page_insert(p, level - 1);
|
||||
p = buddy_get(p, level - 1);
|
||||
_early_page_insert(page_list, p, level - 1);
|
||||
p = _buddy_get(p, level - 1);
|
||||
level--;
|
||||
}
|
||||
}
|
||||
rt_page_t page_cont = (void *)p + early_offset;
|
||||
rt_page_t page_cont = (rt_page_t)((char *)p + early_offset);
|
||||
page_cont->size_bits = ARCH_ADDRESS_WIDTH_BITS;
|
||||
page_cont->ref_cnt = 1;
|
||||
return p;
|
||||
}
|
||||
|
||||
static rt_page_t *_get_page_list(void *vaddr)
|
||||
{
|
||||
rt_ubase_t pa_int = (rt_ubase_t)vaddr + PV_OFFSET;
|
||||
rt_page_t *list;
|
||||
if (pa_int > UINT32_MAX)
|
||||
{
|
||||
list = page_list_high;
|
||||
}
|
||||
else
|
||||
{
|
||||
list = page_list_low;
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
int rt_page_ref_get(void *addr, rt_uint32_t size_bits)
|
||||
{
|
||||
struct rt_page *p;
|
||||
|
@ -389,27 +511,73 @@ void rt_page_ref_inc(void *addr, rt_uint32_t size_bits)
|
|||
rt_hw_interrupt_enable(level);
|
||||
}
|
||||
|
||||
static rt_page_t (*pages_alloc_handler)(rt_uint32_t size_bits);
|
||||
static rt_page_t (*pages_alloc_handler)(rt_page_t page_list[], rt_uint32_t size_bits);
|
||||
|
||||
void *rt_pages_alloc(rt_uint32_t size_bits)
|
||||
/* if not, we skip the finding on page_list_high */
|
||||
static size_t _high_page_configured = 0;
|
||||
|
||||
static rt_page_t *_flag_to_page_list(size_t flags)
|
||||
{
|
||||
rt_page_t *page_list;
|
||||
if (_high_page_configured && (flags & PAGE_ANY_AVAILABLE))
|
||||
{
|
||||
page_list = page_list_high;
|
||||
}
|
||||
else
|
||||
{
|
||||
page_list = page_list_low;
|
||||
}
|
||||
return page_list;
|
||||
}
|
||||
|
||||
static void *_do_pages_alloc(rt_uint32_t size_bits, size_t flags)
|
||||
{
|
||||
void *alloc_buf = RT_NULL;
|
||||
struct rt_page *p;
|
||||
rt_base_t level;
|
||||
rt_page_t *page_list = _flag_to_page_list(flags);
|
||||
|
||||
level = rt_hw_interrupt_disable();
|
||||
p = pages_alloc_handler(size_bits);
|
||||
p = pages_alloc_handler(page_list, size_bits);
|
||||
rt_hw_interrupt_enable(level);
|
||||
|
||||
if (!p && page_list != page_list_low)
|
||||
{
|
||||
/* fall back */
|
||||
page_list = page_list_low;
|
||||
|
||||
level = rt_hw_interrupt_disable();
|
||||
p = pages_alloc_handler(page_list, size_bits);
|
||||
rt_hw_interrupt_enable(level);
|
||||
}
|
||||
|
||||
if (p)
|
||||
{
|
||||
alloc_buf = page_to_addr(p);
|
||||
|
||||
#ifdef RT_DEBUG_PAGE_LEAK
|
||||
level = rt_hw_interrupt_disable();
|
||||
TRACE_ALLOC(p, size_bits);
|
||||
rt_hw_interrupt_enable(level);
|
||||
#endif
|
||||
}
|
||||
return alloc_buf;
|
||||
}
|
||||
|
||||
void *rt_pages_alloc(rt_uint32_t size_bits)
|
||||
{
|
||||
return _do_pages_alloc(size_bits, 0);
|
||||
}
|
||||
|
||||
void *rt_pages_alloc_ext(rt_uint32_t size_bits, size_t flags)
|
||||
{
|
||||
return _do_pages_alloc(size_bits, flags);
|
||||
}
|
||||
|
||||
int rt_pages_free(void *addr, rt_uint32_t size_bits)
|
||||
{
|
||||
struct rt_page *p;
|
||||
rt_page_t *page_list = _get_page_list(addr);
|
||||
int real_free = 0;
|
||||
|
||||
p = rt_page_addr2page(addr);
|
||||
|
@ -417,14 +585,18 @@ int rt_pages_free(void *addr, rt_uint32_t size_bits)
|
|||
{
|
||||
rt_base_t level;
|
||||
level = rt_hw_interrupt_disable();
|
||||
real_free = _pages_free(p, size_bits);
|
||||
real_free = _pages_free(page_list, p, size_bits);
|
||||
if (real_free)
|
||||
TRACE_FREE(p, size_bits);
|
||||
rt_hw_interrupt_enable(level);
|
||||
}
|
||||
|
||||
return real_free;
|
||||
}
|
||||
|
||||
void rt_page_list(void) __attribute__((alias("list_page")));
|
||||
|
||||
#warning TODO: improve list page
|
||||
void list_page(void)
|
||||
{
|
||||
int i;
|
||||
|
@ -435,7 +607,7 @@ void list_page(void)
|
|||
|
||||
for (i = 0; i < RT_PAGE_MAX_ORDER; i++)
|
||||
{
|
||||
struct rt_page *p = page_list[i];
|
||||
struct rt_page *p = page_list_low[i];
|
||||
|
||||
rt_kprintf("level %d ", i);
|
||||
|
||||
|
@ -447,6 +619,21 @@ void list_page(void)
|
|||
}
|
||||
rt_kprintf("\n");
|
||||
}
|
||||
for (i = 0; i < RT_PAGE_MAX_ORDER; i++)
|
||||
{
|
||||
struct rt_page *p = page_list_high[i];
|
||||
|
||||
rt_kprintf("level %d ", i);
|
||||
|
||||
while (p)
|
||||
{
|
||||
total += (1UL << i);
|
||||
rt_kprintf("[0x%08p]", rt_page_page2addr(p));
|
||||
p = p->next;
|
||||
}
|
||||
rt_kprintf("\n");
|
||||
}
|
||||
|
||||
rt_hw_interrupt_enable(level);
|
||||
rt_kprintf("free pages is 0x%08lx (%ld KB)\n", total, total * ARCH_PAGE_SIZE / 1024);
|
||||
rt_kprintf("-------------------------------\n");
|
||||
|
@ -462,7 +649,17 @@ void rt_page_get_info(rt_size_t *total_nr, rt_size_t *free_nr)
|
|||
level = rt_hw_interrupt_disable();
|
||||
for (i = 0; i < RT_PAGE_MAX_ORDER; i++)
|
||||
{
|
||||
struct rt_page *p = page_list[i];
|
||||
struct rt_page *p = page_list_low[i];
|
||||
|
||||
while (p)
|
||||
{
|
||||
total_free += (1UL << i);
|
||||
p = p->next;
|
||||
}
|
||||
}
|
||||
for (i = 0; i < RT_PAGE_MAX_ORDER; i++)
|
||||
{
|
||||
struct rt_page *p = page_list_high[i];
|
||||
|
||||
while (p)
|
||||
{
|
||||
|
@ -475,6 +672,62 @@ void rt_page_get_info(rt_size_t *total_nr, rt_size_t *free_nr)
|
|||
*free_nr = total_free;
|
||||
}
|
||||
|
||||
static void _install_page(rt_page_t mpr_head, rt_region_t region, void *insert_handler)
|
||||
{
|
||||
void (*insert)(rt_page_t *page_list, rt_page_t page, int size_bits) = insert_handler;
|
||||
rt_region_t shadow;
|
||||
shadow.start = region.start & ~shadow_mask;
|
||||
shadow.end = FLOOR(region.end, shadow_mask + 1);
|
||||
|
||||
if (shadow.end > UINT32_MAX)
|
||||
_high_page_configured = 1;
|
||||
|
||||
rt_page_t shad_head = addr_to_page(mpr_head, (void *)shadow.start);
|
||||
rt_page_t shad_tail = addr_to_page(mpr_head, (void *)shadow.end);
|
||||
rt_page_t head = addr_to_page(mpr_head, (void *)region.start);
|
||||
rt_page_t tail = addr_to_page(mpr_head, (void *)region.end);
|
||||
|
||||
/* mark shadow pages as illegal */
|
||||
for (rt_page_t iter = shad_head; iter < head; iter++)
|
||||
{
|
||||
iter->size_bits = ARCH_ADDRESS_WIDTH_BITS;
|
||||
}
|
||||
for (rt_page_t iter = tail; iter < shad_tail; iter++)
|
||||
{
|
||||
iter->size_bits = ARCH_ADDRESS_WIDTH_BITS;
|
||||
}
|
||||
|
||||
/* insert reserved pages to list */
|
||||
const int max_order = RT_PAGE_MAX_ORDER + ARCH_PAGE_SHIFT - 1;
|
||||
while (region.start != region.end)
|
||||
{
|
||||
struct rt_page *p;
|
||||
int align_bits;
|
||||
int size_bits;
|
||||
|
||||
size_bits =
|
||||
ARCH_ADDRESS_WIDTH_BITS - 1 - rt_hw_clz(region.end - region.start);
|
||||
align_bits = rt_hw_ctz(region.start);
|
||||
if (align_bits < size_bits)
|
||||
{
|
||||
size_bits = align_bits;
|
||||
}
|
||||
if (size_bits > max_order)
|
||||
{
|
||||
size_bits = max_order;
|
||||
}
|
||||
|
||||
p = addr_to_page(mpr_head, (void *)region.start);
|
||||
p->size_bits = ARCH_ADDRESS_WIDTH_BITS;
|
||||
p->ref_cnt = 0;
|
||||
|
||||
/* insert to list */
|
||||
rt_page_t *page_list = _get_page_list((void *)region.start);
|
||||
insert(page_list, (rt_page_t)((char *)p - early_offset), size_bits - ARCH_PAGE_SHIFT);
|
||||
region.start += (1UL << size_bits);
|
||||
}
|
||||
}
|
||||
|
||||
void rt_page_init(rt_region_t reg)
|
||||
{
|
||||
int i;
|
||||
|
@ -500,7 +753,8 @@ void rt_page_init(rt_region_t reg)
|
|||
/* init free list */
|
||||
for (i = 0; i < RT_PAGE_MAX_ORDER; i++)
|
||||
{
|
||||
page_list[i] = 0;
|
||||
page_list_low[i] = 0;
|
||||
page_list_high[i] = 0;
|
||||
}
|
||||
|
||||
/* map MPR area */
|
||||
|
@ -524,9 +778,9 @@ void rt_page_init(rt_region_t reg)
|
|||
rt_size_t init_mpr_npage = init_mpr_size >> ARCH_PAGE_SHIFT;
|
||||
|
||||
init_mpr_cont_start = (void *)reg.start;
|
||||
void *init_mpr_cont_end = init_mpr_cont_start + init_mpr_size;
|
||||
early_offset = init_mpr_cont_start - (void *)init_mpr_align_start;
|
||||
rt_page_t mpr_cont = rt_mpr_start + early_offset;
|
||||
rt_size_t init_mpr_cont_end = (rt_size_t)init_mpr_cont_start + init_mpr_size;
|
||||
early_offset = (rt_size_t)init_mpr_cont_start - init_mpr_align_start;
|
||||
rt_page_t mpr_cont = (void *)((char *)rt_mpr_start + early_offset);
|
||||
|
||||
/* mark init mpr pages as illegal */
|
||||
rt_page_t head_cont = addr_to_page(mpr_cont, (void *)reg.start);
|
||||
|
@ -536,48 +790,8 @@ void rt_page_init(rt_region_t reg)
|
|||
iter->size_bits = ARCH_ADDRESS_WIDTH_BITS;
|
||||
}
|
||||
|
||||
/* mark shadow pages as illegal */
|
||||
rt_page_t shad_head_cont = addr_to_page(mpr_cont, (void *)shadow.start);
|
||||
for (rt_page_t iter = shad_head_cont; iter < head_cont; iter++)
|
||||
{
|
||||
iter->size_bits = ARCH_ADDRESS_WIDTH_BITS;
|
||||
}
|
||||
rt_page_t shad_tail_cont = addr_to_page(mpr_cont, (void *)shadow.end);
|
||||
for (rt_page_t iter = tail_cont; iter < shad_tail_cont; iter++)
|
||||
{
|
||||
iter->size_bits = ARCH_ADDRESS_WIDTH_BITS;
|
||||
}
|
||||
|
||||
/* insert reserved pages to list */
|
||||
reg.start = (rt_size_t)init_mpr_cont_end;
|
||||
const int max_order = RT_PAGE_MAX_ORDER + ARCH_PAGE_SHIFT - 1;
|
||||
while (reg.start != reg.end)
|
||||
{
|
||||
struct rt_page *p;
|
||||
int align_bits;
|
||||
int size_bits;
|
||||
|
||||
size_bits =
|
||||
ARCH_ADDRESS_WIDTH_BITS - 1 - rt_hw_clz(reg.end - reg.start);
|
||||
align_bits = rt_hw_ctz(reg.start);
|
||||
if (align_bits < size_bits)
|
||||
{
|
||||
size_bits = align_bits;
|
||||
}
|
||||
if (size_bits > max_order)
|
||||
{
|
||||
size_bits = max_order;
|
||||
}
|
||||
|
||||
p = addr_to_page(mpr_cont, (void *)reg.start);
|
||||
p->size_bits = ARCH_ADDRESS_WIDTH_BITS;
|
||||
p->ref_cnt = 0;
|
||||
|
||||
/* insert to list */
|
||||
_early_page_insert((void *)p - early_offset,
|
||||
size_bits - ARCH_PAGE_SHIFT);
|
||||
reg.start += (1UL << size_bits);
|
||||
}
|
||||
reg.start = init_mpr_cont_end;
|
||||
_install_page(mpr_cont, reg, _early_page_insert);
|
||||
|
||||
pages_alloc_handler = _early_pages_alloc;
|
||||
/* doing the page table bushiness */
|
||||
|
@ -594,7 +808,7 @@ void rt_page_init(rt_region_t reg)
|
|||
static int _load_mpr_area(void *head, void *tail)
|
||||
{
|
||||
int err = 0;
|
||||
void *iter = (void *)((uintptr_t)head & ~ARCH_PAGE_MASK);
|
||||
char *iter = (char *)((rt_ubase_t)head & ~ARCH_PAGE_MASK);
|
||||
tail = (void *)FLOOR(tail, ARCH_PAGE_SIZE);
|
||||
|
||||
while (iter != tail)
|
||||
|
@ -630,19 +844,7 @@ int rt_page_install(rt_region_t region)
|
|||
|
||||
if (err == RT_EOK)
|
||||
{
|
||||
while (region.start != region.end)
|
||||
{
|
||||
struct rt_page *p;
|
||||
int size_bits;
|
||||
|
||||
size_bits = RT_PAGE_MAX_ORDER - 1;
|
||||
p = addr_to_page(page_start, (void *)region.start);
|
||||
p->size_bits = ARCH_ADDRESS_WIDTH_BITS;
|
||||
p->ref_cnt = 1;
|
||||
|
||||
_pages_free(p, size_bits);
|
||||
region.start += (1UL << (size_bits + ARCH_PAGE_SHIFT));
|
||||
}
|
||||
_install_page(rt_mpr_start, region, _page_insert);
|
||||
}
|
||||
}
|
||||
return err;
|
||||
|
|
|
@ -23,10 +23,35 @@
|
|||
union {struct {fields}; char _padding[GET_FLOOR(struct {fields})];};\
|
||||
} *rt_page_t
|
||||
|
||||
/**
|
||||
* @brief PAGE ALLOC FLAGS
|
||||
*
|
||||
* @info PAGE_ANY_AVAILABLE
|
||||
* page allocation default to use lower region, this behavior can change by setting
|
||||
* PAGE_ANY_AVAILABLE
|
||||
*/
|
||||
|
||||
#define PAGE_ANY_AVAILABLE 0x1ul
|
||||
|
||||
|
||||
#ifdef RT_DEBUG_PAGE_LEAK
|
||||
#define DEBUG_FIELD { \
|
||||
/* trace list */ \
|
||||
struct rt_page *tl_next; \
|
||||
struct rt_page *tl_prev; \
|
||||
void *caller; \
|
||||
size_t trace_size; \
|
||||
}
|
||||
#else
|
||||
#define DEBUG_FIELD
|
||||
#endif
|
||||
|
||||
DEF_PAGE_T(
|
||||
struct rt_page *next; /* same level next */
|
||||
struct rt_page *pre; /* same level pre */
|
||||
|
||||
DEBUG_FIELD
|
||||
|
||||
rt_uint32_t size_bits; /* if is ARCH_ADDRESS_WIDTH_BITS, means not free */
|
||||
rt_uint32_t ref_cnt; /* page group ref count */
|
||||
);
|
||||
|
@ -49,6 +74,8 @@ void rt_page_cleanup(void);
|
|||
|
||||
void *rt_pages_alloc(rt_uint32_t size_bits);
|
||||
|
||||
void *rt_pages_alloc_ext(rt_uint32_t size_bits, size_t flags);
|
||||
|
||||
void rt_page_ref_inc(void *addr, rt_uint32_t size_bits);
|
||||
|
||||
int rt_page_ref_get(void *addr, rt_uint32_t size_bits);
|
||||
|
@ -78,4 +105,8 @@ struct rt_page *rt_page_addr2page(void *addr);
|
|||
*/
|
||||
int rt_page_install(rt_region_t region);
|
||||
|
||||
void rt_page_leak_trace_start(void);
|
||||
|
||||
void rt_page_leak_trace_stop(void);
|
||||
|
||||
#endif /* __MM_PAGE_H__ */
|
||||
|
|
|
@ -23,7 +23,10 @@ void rt_hw_cpu_dcache_invalidate(void *start_addr, unsigned long size);
|
|||
|
||||
static inline void rt_hw_icache_invalidate_all(void)
|
||||
{
|
||||
__asm_invalidate_icache_all();
|
||||
/* wait for any modification complete */
|
||||
__asm__ volatile ("dsb ishst");
|
||||
__asm__ volatile ("ic iallu");
|
||||
__asm__ volatile ("isb");
|
||||
}
|
||||
|
||||
void rt_hw_cpu_icache_invalidate(void *addr, rt_size_t size);
|
||||
|
|
|
@ -132,7 +132,10 @@ static rt_uint64_t _read_be_number(void *start, int size)
|
|||
{
|
||||
rt_uint64_t buf = 0;
|
||||
for (; size > 0; size--)
|
||||
buf = (buf << 32) | fdt32_to_cpu(*(uint32_t *)start++);
|
||||
{
|
||||
buf = (buf << 32) | fdt32_to_cpu(*(uint32_t *)start);
|
||||
start = (uint32_t *)start + 1;
|
||||
}
|
||||
return buf;
|
||||
}
|
||||
|
||||
|
|
|
@ -1,13 +1,14 @@
|
|||
/*
|
||||
* Copyright (c) 2006-2018, RT-Thread Development Team
|
||||
* Copyright (c) 2006-2023, RT-Thread Development Team
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*
|
||||
* Change Logs:
|
||||
* Date Author Notes
|
||||
* 2012-01-10 bernard porting to AM1808
|
||||
* 2021-11-28 GuEe-GUI first version
|
||||
* 2022-12-10 WangXiaoyao porting to MM
|
||||
*/
|
||||
|
||||
#include <board.h>
|
||||
#include <rthw.h>
|
||||
#include <rtthread.h>
|
||||
|
@ -79,6 +80,7 @@ static void _kenrel_unmap_4K(unsigned long *lv0_tbl, void *v_addr)
|
|||
{
|
||||
break;
|
||||
}
|
||||
/* next table entry in current level */
|
||||
level_info[level].pos = cur_lv_tbl + off;
|
||||
cur_lv_tbl = (unsigned long *)(page & MMU_ADDRESS_MASK);
|
||||
cur_lv_tbl = (unsigned long *)((unsigned long)cur_lv_tbl - PV_OFFSET);
|
||||
|
@ -119,8 +121,7 @@ static void _kenrel_unmap_4K(unsigned long *lv0_tbl, void *v_addr)
|
|||
return;
|
||||
}
|
||||
|
||||
static int _kenrel_map_4K(unsigned long *lv0_tbl, void *vaddr, void *paddr,
|
||||
unsigned long attr)
|
||||
static int _kernel_map_4K(unsigned long *lv0_tbl, void *vaddr, void *paddr, unsigned long attr)
|
||||
{
|
||||
int ret = 0;
|
||||
int level;
|
||||
|
@ -145,7 +146,7 @@ static int _kenrel_map_4K(unsigned long *lv0_tbl, void *vaddr, void *paddr,
|
|||
off &= MMU_LEVEL_MASK;
|
||||
if (!(cur_lv_tbl[off] & MMU_TYPE_USED))
|
||||
{
|
||||
page = (unsigned long)rt_pages_alloc(0);
|
||||
page = (unsigned long)rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE);
|
||||
if (!page)
|
||||
{
|
||||
ret = MMU_MAP_ERROR_NOPAGE;
|
||||
|
@ -188,19 +189,104 @@ err:
|
|||
return ret;
|
||||
}
|
||||
|
||||
static int _kernel_map_2M(unsigned long *lv0_tbl, void *vaddr, void *paddr, unsigned long attr)
|
||||
{
|
||||
int ret = 0;
|
||||
int level;
|
||||
unsigned long *cur_lv_tbl = lv0_tbl;
|
||||
unsigned long page;
|
||||
unsigned long off;
|
||||
unsigned long va = (unsigned long)vaddr;
|
||||
unsigned long pa = (unsigned long)paddr;
|
||||
|
||||
int level_shift = MMU_ADDRESS_BITS;
|
||||
|
||||
if (va & ARCH_SECTION_MASK)
|
||||
{
|
||||
return MMU_MAP_ERROR_VANOTALIGN;
|
||||
}
|
||||
if (pa & ARCH_SECTION_MASK)
|
||||
{
|
||||
return MMU_MAP_ERROR_PANOTALIGN;
|
||||
}
|
||||
for (level = 0; level < MMU_TBL_BLOCK_2M_LEVEL; level++)
|
||||
{
|
||||
off = (va >> level_shift);
|
||||
off &= MMU_LEVEL_MASK;
|
||||
if (!(cur_lv_tbl[off] & MMU_TYPE_USED))
|
||||
{
|
||||
page = (unsigned long)rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE);
|
||||
if (!page)
|
||||
{
|
||||
ret = MMU_MAP_ERROR_NOPAGE;
|
||||
goto err;
|
||||
}
|
||||
rt_memset((char *)page, 0, ARCH_PAGE_SIZE);
|
||||
rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, (void *)page, ARCH_PAGE_SIZE);
|
||||
cur_lv_tbl[off] = (page + PV_OFFSET) | MMU_TYPE_TABLE;
|
||||
rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off, sizeof(void *));
|
||||
}
|
||||
else
|
||||
{
|
||||
page = cur_lv_tbl[off];
|
||||
page &= MMU_ADDRESS_MASK;
|
||||
/* page to va */
|
||||
page -= PV_OFFSET;
|
||||
rt_page_ref_inc((void *)page, 0);
|
||||
}
|
||||
page = cur_lv_tbl[off];
|
||||
if ((page & MMU_TYPE_MASK) == MMU_TYPE_BLOCK)
|
||||
{
|
||||
/* is block! error! */
|
||||
ret = MMU_MAP_ERROR_CONFLICT;
|
||||
goto err;
|
||||
}
|
||||
cur_lv_tbl = (unsigned long *)(page & MMU_ADDRESS_MASK);
|
||||
cur_lv_tbl = (unsigned long *)((unsigned long)cur_lv_tbl - PV_OFFSET);
|
||||
level_shift -= MMU_LEVEL_SHIFT;
|
||||
}
|
||||
/* now is level page */
|
||||
attr &= MMU_ATTRIB_MASK;
|
||||
pa |= (attr | MMU_TYPE_BLOCK); /* block */
|
||||
off = (va >> ARCH_SECTION_SHIFT);
|
||||
off &= MMU_LEVEL_MASK;
|
||||
cur_lv_tbl[off] = pa;
|
||||
rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off, sizeof(void *));
|
||||
return ret;
|
||||
err:
|
||||
_kenrel_unmap_4K(lv0_tbl, (void *)va);
|
||||
return ret;
|
||||
}
|
||||
|
||||
void *rt_hw_mmu_map(rt_aspace_t aspace, void *v_addr, void *p_addr, size_t size,
|
||||
size_t attr)
|
||||
{
|
||||
int ret = -1;
|
||||
|
||||
void *unmap_va = v_addr;
|
||||
size_t npages = size >> ARCH_PAGE_SHIFT;
|
||||
size_t npages;
|
||||
size_t stride;
|
||||
int (*mapper)(unsigned long *lv0_tbl, void *vaddr, void *paddr, unsigned long attr);
|
||||
|
||||
if (((rt_ubase_t)v_addr & ARCH_SECTION_MASK) || (size & ARCH_SECTION_MASK))
|
||||
{
|
||||
/* legacy 4k mapping */
|
||||
npages = size >> ARCH_PAGE_SHIFT;
|
||||
stride = ARCH_PAGE_SIZE;
|
||||
mapper = _kernel_map_4K;
|
||||
}
|
||||
else
|
||||
{
|
||||
/* 2m huge page */
|
||||
npages = size >> ARCH_SECTION_SHIFT;
|
||||
stride = ARCH_SECTION_SIZE;
|
||||
mapper = _kernel_map_2M;
|
||||
}
|
||||
|
||||
// TODO trying with HUGEPAGE here
|
||||
while (npages--)
|
||||
{
|
||||
MM_PGTBL_LOCK(aspace);
|
||||
ret = _kenrel_map_4K(aspace->page_table, v_addr, p_addr, attr);
|
||||
ret = mapper(aspace->page_table, v_addr, p_addr, attr);
|
||||
MM_PGTBL_UNLOCK(aspace);
|
||||
|
||||
if (ret != 0)
|
||||
|
@ -213,12 +299,12 @@ void *rt_hw_mmu_map(rt_aspace_t aspace, void *v_addr, void *p_addr, size_t size,
|
|||
MM_PGTBL_LOCK(aspace);
|
||||
_kenrel_unmap_4K(aspace->page_table, (void *)unmap_va);
|
||||
MM_PGTBL_UNLOCK(aspace);
|
||||
unmap_va += ARCH_PAGE_SIZE;
|
||||
unmap_va = (char *)unmap_va + stride;
|
||||
}
|
||||
break;
|
||||
}
|
||||
v_addr += ARCH_PAGE_SIZE;
|
||||
p_addr += ARCH_PAGE_SIZE;
|
||||
v_addr = (char *)v_addr + stride;
|
||||
p_addr = (char *)p_addr + stride;
|
||||
}
|
||||
|
||||
if (ret == 0)
|
||||
|
@ -244,7 +330,7 @@ void rt_hw_mmu_unmap(rt_aspace_t aspace, void *v_addr, size_t size)
|
|||
MM_PGTBL_LOCK(aspace);
|
||||
_kenrel_unmap_4K(aspace->page_table, v_addr);
|
||||
MM_PGTBL_UNLOCK(aspace);
|
||||
v_addr += ARCH_PAGE_SIZE;
|
||||
v_addr = (char *)v_addr + ARCH_PAGE_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -254,7 +340,7 @@ void rt_hw_aspace_switch(rt_aspace_t aspace)
|
|||
{
|
||||
void *pgtbl = aspace->page_table;
|
||||
pgtbl = rt_kmem_v2p(pgtbl);
|
||||
uintptr_t tcr;
|
||||
rt_ubase_t tcr;
|
||||
|
||||
__asm__ volatile("msr ttbr0_el1, %0" ::"r"(pgtbl) : "memory");
|
||||
|
||||
|
@ -337,20 +423,19 @@ void rt_hw_mmu_setup(rt_aspace_t aspace, struct mem_desc *mdesc, int desc_nr)
|
|||
rt_page_cleanup();
|
||||
}
|
||||
|
||||
|
||||
#ifdef RT_USING_SMART
|
||||
static inline void _init_region(void *vaddr, size_t size)
|
||||
static void _init_region(void *vaddr, size_t size)
|
||||
{
|
||||
rt_ioremap_start = vaddr;
|
||||
rt_ioremap_size = size;
|
||||
rt_mpr_start = rt_ioremap_start - rt_mpr_size;
|
||||
rt_mpr_start = (char *)rt_ioremap_start - rt_mpr_size;
|
||||
}
|
||||
#else
|
||||
|
||||
#define RTOS_VEND ((void *)0xfffffffff000UL)
|
||||
#define RTOS_VEND (0xfffffffff000UL)
|
||||
static inline void _init_region(void *vaddr, size_t size)
|
||||
{
|
||||
rt_mpr_start = RTOS_VEND - rt_mpr_size;
|
||||
rt_mpr_start = (void *)(RTOS_VEND - rt_mpr_size);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
@ -395,7 +480,7 @@ int rt_hw_mmu_map_init(rt_aspace_t aspace, void *v_address, size_t size,
|
|||
rt_aspace_init(aspace, (void *)KERNEL_VADDR_START, 0 - KERNEL_VADDR_START,
|
||||
vtable);
|
||||
#else
|
||||
rt_aspace_init(aspace, (void *)0x1000, RTOS_VEND - (void *)0x1000, vtable);
|
||||
rt_aspace_init(aspace, (void *)0x1000, RTOS_VEND - 0x1000ul, vtable);
|
||||
#endif
|
||||
|
||||
_init_region(v_address, size);
|
||||
|
@ -586,26 +671,35 @@ void *rt_hw_mmu_v2p(rt_aspace_t aspace, void *v_addr)
|
|||
{
|
||||
int level_shift;
|
||||
unsigned long paddr;
|
||||
unsigned long *pte = _query(aspace, v_addr, &level_shift);
|
||||
|
||||
if (pte)
|
||||
if (aspace == &rt_kernel_space)
|
||||
{
|
||||
paddr = *pte & MMU_ADDRESS_MASK;
|
||||
paddr |= (uintptr_t)v_addr & ((1ul << level_shift) - 1);
|
||||
paddr = (unsigned long)rt_hw_mmu_kernel_v2p(v_addr);
|
||||
}
|
||||
else
|
||||
{
|
||||
paddr = (unsigned long)ARCH_MAP_FAILED;
|
||||
unsigned long *pte = _query(aspace, v_addr, &level_shift);
|
||||
|
||||
if (pte)
|
||||
{
|
||||
paddr = *pte & MMU_ADDRESS_MASK;
|
||||
paddr |= (rt_ubase_t)v_addr & ((1ul << level_shift) - 1);
|
||||
}
|
||||
else
|
||||
{
|
||||
paddr = (unsigned long)ARCH_MAP_FAILED;
|
||||
}
|
||||
}
|
||||
|
||||
return (void *)paddr;
|
||||
}
|
||||
|
||||
static int _noncache(uintptr_t *pte)
|
||||
static int _noncache(rt_ubase_t *pte)
|
||||
{
|
||||
int err = 0;
|
||||
const uintptr_t idx_shift = 2;
|
||||
const uintptr_t idx_mask = 0x7 << idx_shift;
|
||||
uintptr_t entry = *pte;
|
||||
const rt_ubase_t idx_shift = 2;
|
||||
const rt_ubase_t idx_mask = 0x7 << idx_shift;
|
||||
rt_ubase_t entry = *pte;
|
||||
if ((entry & idx_mask) == (NORMAL_MEM << idx_shift))
|
||||
{
|
||||
*pte = (entry & ~idx_mask) | (NORMAL_NOCACHE_MEM << idx_shift);
|
||||
|
@ -618,12 +712,12 @@ static int _noncache(uintptr_t *pte)
|
|||
return err;
|
||||
}
|
||||
|
||||
static int _cache(uintptr_t *pte)
|
||||
static int _cache(rt_ubase_t *pte)
|
||||
{
|
||||
int err = 0;
|
||||
const uintptr_t idx_shift = 2;
|
||||
const uintptr_t idx_mask = 0x7 << idx_shift;
|
||||
uintptr_t entry = *pte;
|
||||
const rt_ubase_t idx_shift = 2;
|
||||
const rt_ubase_t idx_mask = 0x7 << idx_shift;
|
||||
rt_ubase_t entry = *pte;
|
||||
if ((entry & idx_mask) == (NORMAL_NOCACHE_MEM << idx_shift))
|
||||
{
|
||||
*pte = (entry & ~idx_mask) | (NORMAL_MEM << idx_shift);
|
||||
|
@ -636,7 +730,7 @@ static int _cache(uintptr_t *pte)
|
|||
return err;
|
||||
}
|
||||
|
||||
static int (*control_handler[MMU_CNTL_DUMMY_END])(uintptr_t *pte) = {
|
||||
static int (*control_handler[MMU_CNTL_DUMMY_END])(rt_ubase_t *pte) = {
|
||||
[MMU_CNTL_CACHE] = _cache,
|
||||
[MMU_CNTL_NONCACHE] = _noncache,
|
||||
};
|
||||
|
@ -646,17 +740,18 @@ int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size,
|
|||
{
|
||||
int level_shift;
|
||||
int err = -RT_EINVAL;
|
||||
void *vend = vaddr + size;
|
||||
rt_ubase_t vstart = (rt_ubase_t)vaddr;
|
||||
rt_ubase_t vend = vstart + size;
|
||||
|
||||
int (*handler)(uintptr_t * pte);
|
||||
int (*handler)(rt_ubase_t * pte);
|
||||
if (cmd >= 0 && cmd < MMU_CNTL_DUMMY_END)
|
||||
{
|
||||
handler = control_handler[cmd];
|
||||
|
||||
while (vaddr < vend)
|
||||
while (vstart < vend)
|
||||
{
|
||||
uintptr_t *pte = _query(aspace, vaddr, &level_shift);
|
||||
void *range_end = vaddr + (1ul << level_shift);
|
||||
rt_ubase_t *pte = _query(aspace, (void *)vstart, &level_shift);
|
||||
rt_ubase_t range_end = vstart + (1ul << level_shift);
|
||||
RT_ASSERT(range_end <= vend);
|
||||
|
||||
if (pte)
|
||||
|
@ -664,7 +759,7 @@ int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size,
|
|||
err = handler(pte);
|
||||
RT_ASSERT(err == RT_EOK);
|
||||
}
|
||||
vaddr = range_end;
|
||||
vstart = range_end;
|
||||
}
|
||||
}
|
||||
else
|
||||
|
|
|
@ -97,6 +97,28 @@ static inline void *rt_hw_mmu_tbl_get()
|
|||
return (void *)(tbl & ((1ul << 48) - 2));
|
||||
}
|
||||
|
||||
static inline void *rt_hw_mmu_kernel_v2p(void *v_addr)
|
||||
{
|
||||
rt_ubase_t par;
|
||||
void *paddr;
|
||||
asm volatile("at s1e1w, %0"::"r"(v_addr):"memory");
|
||||
asm volatile("mrs %0, par_el1":"=r"(par)::"memory");
|
||||
|
||||
if (par & 0x1)
|
||||
{
|
||||
paddr = ARCH_MAP_FAILED;
|
||||
}
|
||||
else
|
||||
{
|
||||
#define MMU_ADDRESS_MASK 0x0000fffffffff000UL
|
||||
par &= MMU_ADDRESS_MASK;
|
||||
par |= (rt_ubase_t)v_addr & ARCH_PAGE_MASK;
|
||||
paddr = (void *)par;
|
||||
}
|
||||
|
||||
return paddr;
|
||||
}
|
||||
|
||||
int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size,
|
||||
enum rt_mmu_cntl cmd);
|
||||
|
||||
|
|
26
src/Kconfig
26
src/Kconfig
|
@ -261,6 +261,12 @@ if RT_DEBUG
|
|||
int
|
||||
default 1 if RT_DEBUG_MEMHEAP_CONFIG
|
||||
|
||||
if ARCH_MM_MMU
|
||||
config RT_DEBUG_PAGE_LEAK
|
||||
bool "Enable page leaking tracer"
|
||||
default n
|
||||
endif
|
||||
|
||||
config RT_DEBUG_MODULE_CONFIG
|
||||
bool "Enable debugging of Application Module"
|
||||
default n
|
||||
|
@ -305,15 +311,17 @@ endmenu
|
|||
|
||||
menu "Memory Management"
|
||||
|
||||
config RT_PAGE_MAX_ORDER
|
||||
int "Max order of pages allocatable by page allocator"
|
||||
default 11
|
||||
help
|
||||
For example, A value of 11 means the maximum chunk of contiguous memory
|
||||
allocatable by page system is 2^(11 + ARCH_PAGE_BITS - 1) Bytes.
|
||||
Large memory requirement can consume all system resource, and should
|
||||
consider reserved memory instead to enhance system endurance.
|
||||
Max order should at least satisfied usage by huge page.
|
||||
if ARCH_MM_MMU
|
||||
config RT_PAGE_MAX_ORDER
|
||||
int "Max order of pages allocatable by page allocator"
|
||||
default 11
|
||||
help
|
||||
For example, A value of 11 means the maximum chunk of contiguous memory
|
||||
allocatable by page system is 2^(11 + ARCH_PAGE_BITS - 1) Bytes.
|
||||
Large memory requirement can consume all system resource, and should
|
||||
consider reserved memory instead to enhance system endurance.
|
||||
Max order should at least satisfied usage by huge page.
|
||||
endif
|
||||
|
||||
config RT_USING_MEMPOOL
|
||||
bool "Using memory pool"
|
||||
|
|
Loading…
Reference in New Issue