gh0s7/mckernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

OFP: temporary ANON mmap() rewrite

Browse Source
This commit is contained in:
Balazs Gerofi
2018-06-07 07:16:49 +09:00
parent 2f0c2aae9e
commit 09f63483cc
9 changed files with 239 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
kernel/include/process.h
View File

@ -32,6 +32,7 @@
#define VR_STACK 0x1
#define VR_RESERVED 0x2
#define VR_AP_USER 0x4
#define VR_PREALLOC 0x8
#define VR_IO_NOCACHE 0x100
#define VR_REMOTE 0x200
#define VR_WRITE_COMBINED 0x400
@ -387,6 +388,8 @@ struct vm_range {
int pgshift; /* page size. 0 means THP */
int padding;
void *private_data;
unsigned long lowest_accesed;
unsigned long faulted_size;
};
struct vm_range_numa_policy {

27
kernel/mem.c
View File

@ -698,6 +698,22 @@ static void *mckernel_allocate_aligned_pages_node(int npages, int p2align,
break;
}
else {
dkprintf("%s: couldn't fulfill user policy for"
" %d contiguous pages from node %d "
#ifdef IHK_RBTREE_ALLOCATOR
"(free pages left: %d)"
#endif
"\n",
__FUNCTION__,
npages,
numa_id
#ifdef IHK_RBTREE_ALLOCATOR
, memory_nodes[numa_id].nr_free_pages
#endif
);
//return NULL;
}
}
if (pa) break;
@ -719,8 +735,8 @@ static void *mckernel_allocate_aligned_pages_node(int npages, int p2align,
#ifdef PROFILE_ENABLE
profile_event_add(PROFILE_mpol_alloc_missed, npages * 4096);
#endif
dkprintf("%s: couldn't fulfill user policy for %d pages\n",
__FUNCTION__, npages);
dkprintf("%s: couldn't fulfill user policy for %d pages from node %d\n",
__FUNCTION__, npages, i);
}
distance_based:
@ -926,6 +942,8 @@ static void query_free_mem_interrupt_handler(void *priv)
/* Iterate memory allocators */
for (i = 0; i < ihk_mc_get_nr_numa_nodes(); ++i) {
#ifdef IHK_RBTREE_ALLOCATOR
kprintf("McKernel free pages in NUMA node %d: %d\n",
i, memory_nodes[i].nr_free_pages);
pages += memory_nodes[i].nr_free_pages;
#else
struct ihk_page_allocator_desc *pa_allocator;
@ -981,6 +999,8 @@ void coredump(struct thread *thread, void *regs)
struct coretable *coretable;
int chunks;
return;
#ifdef POSTK_DEBUG_ARCH_DEP_67 /* use limit corefile size. (temporarily fix.) */
if (thread->proc->rlimit[MCK_RLIMIT_CORE].rlim_cur == 0) {
return;
@ -1189,6 +1209,7 @@ static void page_fault_handler(void *fault_addr, uint64_t reason, void *regs)
if (!lptep || !pte_is_present(lptep)) {
kprintf("%s: ERROR: no mapping in Linux for: 0x%lx?\n",
__FUNCTION__, virt);
terminate(0, SIGKILL);
goto regular_handler;
}
@ -1210,7 +1231,7 @@ static void page_fault_handler(void *fault_addr, uint64_t reason, void *regs)
}
*ptep = *lptep;
kprintf("%s: Linux ioremap address 0x%lx -> 0x%lx "
dkprintf("%s: Linux ioremap address 0x%lx -> 0x%lx "
"mapped on demand\n",
__FUNCTION__, virt, phys);

175
kernel/process.c
View File

@ -1015,6 +1015,10 @@ int free_process_memory_range(struct process_vm *vm, struct vm_range *range)
if (vm->range_cache[i] == range)
vm->range_cache[i] = NULL;
}
if (range->flag & VR_STACK) {
kprintf("%s: VR_STACK faulted_size: %lu\n", __FUNCTION__, range->faulted_size);
}
kfree(range);
dkprintf("free_process_memory_range(%p,%lx-%lx): 0\n",
@ -1230,6 +1234,9 @@ int add_process_memory_range(struct process_vm *vm,
range->pgshift = pgshift;
range->private_data = NULL;
range->lowest_accesed = end;
range->faulted_size = 0;
rc = 0;
if (phys == NOPHYS) {
/* Nothing to map */
@ -1266,6 +1273,138 @@ int add_process_memory_range(struct process_vm *vm,
return rc;
}
/*
* Allocate and map physical memory,
* interpret NUMA policy.
* TODO: move out to a function..
*/
if (flag & VR_PREALLOC && phys == NOPHYS) {
#if 0
unsigned long addr = start;
enum ihk_mc_pt_attribute ptattr;
ptattr = arch_vrflag_to_ptattr(range->flag, PF_POPULATE, NULL);
unsigned long irqflags;
unsigned long len = 0;
void *frame = NULL;
int npages;
int p2align;
len = end - addr;
/* Figure out size */
if (len >= LARGE_PAGE_SIZE) {
p2align = LARGE_PAGE_P2ALIGN;
}
else {
p2align = PAGE_P2ALIGN;
}
npages = len >> PAGE_SHIFT;
frame = ihk_mc_alloc_aligned_pages_user(npages,
p2align,
IHK_MC_AP_NOWAIT | (range->flag & VR_AP_USER ? IHK_MC_AP_USER : 0),
-1);
if (!frame) {
kprintf("%s: error: out of memory\n", __FUNCTION__);
panic("panic");
return -ENOMEM;
}
irqflags = ihk_mc_spinlock_lock(&vm->page_table_lock);
rc = ihk_mc_pt_set_range(vm->address_space->page_table,
vm,
(void *)addr,
(void *)addr + len,
virt_to_phys(frame),
ptattr,
PAGE_SHIFT + p2align,
range);
if (rc) {
kprintf("%s: ERROR: mapping\n", __FUNCTION__);
ihk_mc_spinlock_unlock(&vm->page_table_lock, irqflags);
return -ENOMEM;
}
ihk_mc_spinlock_unlock(&vm->page_table_lock, irqflags);
memset(frame, 0, len);
addr += len;
#else
unsigned long addr = start;
enum ihk_mc_pt_attribute ptattr;
ptattr = arch_vrflag_to_ptattr(range->flag, PF_POPULATE, NULL);
while (addr < end) {
unsigned long irqflags;
unsigned long len = 0;
void *frame = NULL;
int npages;
int p2align;
len = end - addr;
/* Figure out size */
if (len >= LARGE_PAGE_SIZE) {
len = LARGE_PAGE_SIZE;
p2align = LARGE_PAGE_P2ALIGN;
}
else {
len = PAGE_SIZE;
p2align = PAGE_P2ALIGN;
}
npages = len >> PAGE_SHIFT;
#if 0
frame = ihk_mc_alloc_aligned_pages_node_user(npages,
p2align,
IHK_MC_AP_NOWAIT | (range->flag & VR_AP_USER ? IHK_MC_AP_USER : 0),
node, -1);
node = 1 - node;
#else
frame = ihk_mc_alloc_aligned_pages_user(npages,
p2align,
IHK_MC_AP_NOWAIT | (range->flag & VR_AP_USER ? IHK_MC_AP_USER : 0),
-1);
#endif
if (!frame) {
kprintf("%s: error: out of memory\n", __FUNCTION__);
return -ENOMEM;
}
irqflags = ihk_mc_spinlock_lock(&vm->page_table_lock);
rc = ihk_mc_pt_set_range(vm->address_space->page_table,
vm,
(void *)addr,
(void *)addr + len,
virt_to_phys(frame),
ptattr,
PAGE_SHIFT + p2align,
range);
if (rc) {
kprintf("%s: ERROR: mapping\n", __FUNCTION__);
ihk_mc_spinlock_unlock(&vm->page_table_lock, irqflags);
return -ENOMEM;
}
ihk_mc_spinlock_unlock(&vm->page_table_lock, irqflags);
memset(frame, 0, len);
addr += len;
}
#endif
dkprintf("%s: 0x%lx:%lu mapped\n",
__FUNCTION__,
start,
end - start);
}
/* Clear content! */
if (phys != NOPHYS && !(flag & (VR_REMOTE | VR_DEMAND_PAGING))
&& ((flag & VR_PROT_MASK) != VR_PROT_NONE)) {
@ -1784,6 +1923,22 @@ static int page_fault_process_memory_range(struct process_vm *vm, struct vm_rang
}
pgaddr = (void *)(fault_addr & ~(pgsize - 1));
}
if (pgsize > LARGE_PAGE_SIZE) {
dkprintf("%s: 0x%lx, pgsize: %lu\n",
__FUNCTION__, pgaddr, pgsize);
}
if (range->flag & VR_STACK) {
range->faulted_size += pgsize;
if (range->lowest_accesed > (unsigned long)pgaddr) {
dkprintf("%s: VR_STACK @ 0x%lx, pgsize: %lu, distance: %lu\n",
__FUNCTION__, pgaddr, pgsize, range->end - (unsigned long)pgaddr);
range->lowest_accesed = (unsigned long)pgaddr;
}
}
/*****/
dkprintf("%s: ptep=%lx,pte_is_null=%d,pte_is_fileoff=%d\n", __FUNCTION__, ptep, ptep ? pte_is_null(ptep) : -1, ptep ? pte_is_fileoff(ptep, pgsize) : -1);
if (!ptep || pte_is_null(ptep) || pte_is_fileoff(ptep, pgsize)) {
@ -2155,6 +2310,8 @@ int init_process_stack(struct thread *thread, struct program_load_desc *pn,
struct vm_range *range;
int stack_populated_size = 0;
int stack_align_padding = 0;
int p2align = LARGE_PAGE_P2ALIGN;
int pgshift = LARGE_PAGE_SHIFT;
/* Create stack range */
end = STACK_TOP(&thread->vm->region) & LARGE_PAGE_MASK;
@ -2177,18 +2334,27 @@ int init_process_stack(struct thread *thread, struct program_load_desc *pn,
else if (size < minsz) {
size = minsz;
}
#if 0
if (minsz >= GB_PAGE_SIZE) {
end = end & GB_PAGE_MASK;
p2align = GB_PAGE_P2ALIGN;
pgshift = GB_PAGE_SHIFT;
}
#endif
start = (end - size) & LARGE_PAGE_MASK;
/* Apply user allocation policy to stacks */
/* TODO: make threshold kernel or mcexec argument */
ap_flag = (size >= proc->mpol_threshold &&
!(proc->mpol_flags & MPOL_NO_STACK)) ? IHK_MC_AP_USER : 0;
dkprintf("%s: max size: %lu, mapped size: %lu %s\n",
__FUNCTION__, size, minsz,
kprintf("%s: stack: 0x%lx-0x%lx:%lu, mapped: %lu %s\n",
__FUNCTION__, start, end, size, minsz,
ap_flag ? "(IHK_MC_AP_USER)" : "");
stack = ihk_mc_alloc_aligned_pages_user(minsz >> PAGE_SHIFT,
LARGE_PAGE_P2ALIGN, IHK_MC_AP_NOWAIT | ap_flag, start);
p2align, IHK_MC_AP_NOWAIT | ap_flag, start);
if (!stack) {
kprintf("%s: error: couldn't allocate initial stack\n",
@ -2215,8 +2381,7 @@ int init_process_stack(struct thread *thread, struct program_load_desc *pn,
thread->vm, (void *)(end - minsz),
(void *)end, virt_to_phys(stack),
arch_vrflag_to_ptattr(vrflag, PF_POPULATE, NULL),
LARGE_PAGE_SHIFT, range
);
pgshift, range);
if (error) {
kprintf("init_process_stack:"

33
kernel/syscall.c
View File

@ -1242,6 +1242,8 @@ interrupt_syscall(struct thread *thread, int sig)
SYSCALL_DECLARE(exit_group)
{
dkprintf("sys_exit_group,pid=%d\n", cpu_local_var(current)->proc->pid);
dkprintf("%s: PID: %d, TID: %d\n", __FUNCTION__,
cpu_local_var(current)->proc->pid, cpu_local_var(current)->tid);
terminate((int)ihk_mc_syscall_arg0(ctx), 0);
return 0;
@ -1572,6 +1574,24 @@ do_mmap(const intptr_t addr0, const size_t len0, const int prot,
vrflags |= VR_AP_USER;
}
#if 1
if (len < (unsigned long)4*1024*1024*1024) {
phys = NOPHYS;
vrflags |= VR_PREALLOC;
}
else {
kprintf("%s: big ANON mapping!!: %lu\n", __FUNCTION__, len);
/* Give demand paging a chance */
vrflags |= VR_DEMAND_PAGING;
populated_mapping = 0;
error = zeroobj_create(&memobj);
if (error) {
ekprintf("%s: zeroobj_create failed, error: %d\n",
__FUNCTION__, error);
goto out;
}
}
#else
p = ihk_mc_alloc_aligned_pages_user(npages, p2align,
IHK_MC_AP_NOWAIT | ap_flag, addr0);
if (p == NULL) {
@ -1603,6 +1623,7 @@ do_mmap(const intptr_t addr0, const size_t len0, const int prot,
__FUNCTION__, addr, len, npages, p2align);
phys = virt_to_phys(p);
}
#endif
}
else if (flags & MAP_SHARED) {
dkprintf("%s: MAP_SHARED,flags=%x,len=%ld\n", __FUNCTION__, flags, len);
@ -5530,6 +5551,7 @@ do_exit(int code)
int sig = code & 255;
dkprintf("sys_exit,pid=%d\n", proc->pid);
dkprintf("%s: PID: %d, TID: %d\n", __FUNCTION__, proc->pid, thread->tid);
mcs_rwlock_reader_lock(&proc->threads_lock, &lock);
nproc = 0;
@ -7813,6 +7835,10 @@ SYSCALL_DECLARE(mremap)
uintptr_t lckstart = -1;
uintptr_t lckend = -1;
/* Not for lammps for now.. */
if (!strcmp("./lammps", thread->proc->saved_cmdline))
return -ENOSYS;
dkprintf("sys_mremap(%#lx,%#lx,%#lx,%#x,%#lx)\n",
oldaddr, oldsize0, newsize0, flags, newaddr);
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
@ -9535,6 +9561,10 @@ long syscall(int num, ihk_mc_user_context_t *ctx)
}
#endif // PROFILE_ENABLE
if (thread->proc->nohost) { // mcexec termination was detected
terminate(0, SIGKILL);
}
#if defined(POSTK_DEBUG_TEMP_FIX_60) && defined(POSTK_DEBUG_TEMP_FIX_56)
check_need_resched();
#elif defined(POSTK_DEBUG_TEMP_FIX_60) /* sched_yield called check_signal fix. */
@ -9562,9 +9592,6 @@ long syscall(int num, ihk_mc_user_context_t *ctx)
#endif // DISABLE_SCHED_YIELD
set_cputime(0);
if (thread->proc->nohost) { // mcexec termination was detected
terminate(0, SIGKILL);
}
//kprintf("syscall=%d returns %lx(%ld)\n", num, l, l);
return l;

5
kernel/user_exp_rcv.c
View File

@ -110,8 +110,11 @@ int hfi1_user_exp_rcv_setup(struct hfi1_filedata *fd, struct hfi1_tid_info *tinf
#if 0
/* Verify that access is OK for the user buffer */
if (access_ok(vm, VERIFY_WRITE, tinfo->vaddr, tinfo->length))
if (access_ok(vm, VERIFY_WRITE, tinfo->vaddr, tinfo->length)) {
kprintf("%s: access_ok() failed for 0x%lx:%lu\n",
__FUNCTION__, tinfo->vaddr, tinfo->length);
return -EFAULT;
}
#endif
vaddr_end = tinfo->vaddr + tinfo->length;