From c319fe08a49a820bca502553aa9b962637223447 Mon Sep 17 00:00:00 2001 From: "TOIDA,Suguru" Date: Fri, 30 Nov 2018 09:18:21 +0900 Subject: [PATCH] ContiguousPTE[7/12] modify ihk_mc_pt_set_range Change-Id: Ib38530ce64a01f21107e0a6a73de7c54f214eb5a --- arch/arm64/kernel/memory.c | 55 ++++++++++++++++++++++++++++++------ arch/x86_64/kernel/memory.c | 4 +-- arch/x86_64/kernel/syscall.c | 8 +++--- kernel/host.c | 2 +- kernel/process.c | 8 +++--- kernel/syscall.c | 3 +- kernel/xpmem.c | 2 +- lib/include/ihk/mm.h | 2 +- 8 files changed, 62 insertions(+), 22 deletions(-) diff --git a/arch/arm64/kernel/memory.c b/arch/arm64/kernel/memory.c index 5ac7ae70..159e9299 100644 --- a/arch/arm64/kernel/memory.c +++ b/arch/arm64/kernel/memory.c @@ -2274,6 +2274,7 @@ struct set_range_args { uintptr_t diff; struct process_vm *vm; struct vm_range *range; /* To find pages we don't need to call memory_stat_rss_add() */ + int overwrite; }; int set_range_middle(void *args0, pte_t *ptep, uintptr_t base, uintptr_t start, @@ -2289,7 +2290,7 @@ int set_range_l1(void *args0, pte_t *ptep, uintptr_t base, uintptr_t start, dkprintf("set_range_l1(%lx,%lx,%lx)\n", base, start, end); - if (!ptl1_null(ptep)) { + if (!args->overwrite && !ptl1_null(ptep)) { error = -EBUSY; ekprintf("set_range_l1(%lx,%lx,%lx):page exists. %d %lx\n", base, start, end, error, *ptep); @@ -2298,6 +2299,21 @@ int set_range_l1(void *args0, pte_t *ptep, uintptr_t base, uintptr_t start, } phys = args->phys + (base - start); + if (__page_offset(base, PTL1_CONT_SIZE) == 0) { //check head pte + uintptr_t next_addr = base + PTL1_CONT_SIZE; + + if (end < next_addr) { + next_addr = end; + } + + // set contiguous bit until the next head pte + // if phys is aligned and range does not end early. + if (__page_offset(phys | next_addr, PTL1_CONT_SIZE) == 0) { + args->attr[0] |= PTE_CONT; + } else { + args->attr[0] &= ~PTE_CONT; + } + } pte = phys | args->attr[0]; ptl1_set(ptep, pte); @@ -2334,10 +2350,11 @@ int set_range_middle(void *args0, pte_t *ptep, uintptr_t base, uintptr_t start, walk_pte_fn_t* callback; unsigned long pgsize; unsigned long pgshift; + unsigned long cont_pgsize; } table[] = { - {walk_pte_l1, set_range_l1, PTL2_SIZE, PTL2_SHIFT}, /*PTL2: second*/ - {walk_pte_l2, set_range_l2, PTL3_SIZE, PTL3_SHIFT}, /*PTL3: first*/ - {walk_pte_l3, set_range_l3, PTL4_SIZE, PTL4_SHIFT}, /*PTL4: zero*/ + {walk_pte_l1, set_range_l1, PTL2_SIZE, PTL2_SHIFT, PTL2_CONT_SIZE}, /*PTL2: second*/ + {walk_pte_l2, set_range_l2, PTL3_SIZE, PTL3_SHIFT, PTL3_CONT_SIZE}, /*PTL3: first*/ + {walk_pte_l3, set_range_l3, PTL4_SIZE, PTL4_SHIFT, PTL4_CONT_SIZE}, /*PTL4: zero*/ }; const struct table tbl = table[level-2]; @@ -2349,7 +2366,7 @@ int set_range_middle(void *args0, pte_t *ptep, uintptr_t base, uintptr_t start, dkprintf("set_range_middle(%lx,%lx,%lx,%d)\n", base, start, end, level); retry: - if (ptl_null(ptep, level)) { + if (ptl_null(ptep, level) || (args->overwrite && ptl_type_page(ptep, level))) { pte_t pte; uintptr_t phys; if (level == 2 || (level == 3 && FIRST_LEVEL_BLOCK_SUPPORT)) { @@ -2358,7 +2375,28 @@ retry: && (!args->pgshift || (args->pgshift == tbl.pgshift))) { phys = args->phys + (base - start); - ptl_set(ptep, phys | args->attr[level-1], level); + + //check head pte + if (__page_offset(base, tbl.cont_pgsize) == 0) { + uintptr_t next_addr = base + + tbl.cont_pgsize; + + if (end < next_addr) { + next_addr = end; + } + + // set contiguous bit until the + // next head pte if phys is aligned + // and range does not end early. + if (__page_offset(phys | next_addr, tbl.cont_pgsize) == 0) { + args->attr[level-1] |= PTE_CONT; + } else { + args->attr[level-1] &= ~PTE_CONT; + } + } + + ptl_set(ptep, phys | args->attr[level-1], + level); error = 0; dkprintf("set_range_middle(%lx,%lx,%lx,%d):" "large page. %d %lx\n", @@ -2422,7 +2460,7 @@ out: int ihk_mc_pt_set_range(page_table_t pt, struct process_vm *vm, void *start, void *end, uintptr_t phys, enum ihk_mc_pt_attribute attr, - int pgshift, struct vm_range *range) + int pgshift, struct vm_range *range, int overwrite) { const struct table { walk_pte_t* walk; @@ -2446,6 +2484,7 @@ int ihk_mc_pt_set_range(page_table_t pt, struct process_vm *vm, void *start, args.vm = vm; args.pgshift = pgshift; args.range = range; + args.overwrite = overwrite; // conversion switch (CONFIG_ARM64_PGTABLE_LEVELS) @@ -2698,7 +2737,7 @@ static int move_one_page(void *arg0, page_table_t pt, pte_t *ptep, attr = apte & ~PT_PHYSMASK; error = ihk_mc_pt_set_range(pt, args->vm, (void *)dest, - (void *)(dest + pgsize), phys, attr, pgshift, args->range); + (void *)(dest + pgsize), phys, attr, pgshift, args->range, 0); if (error) { kprintf("move_one_page(%p,%p,%p %#lx,%p,%d):" "set failed. %d\n", diff --git a/arch/x86_64/kernel/memory.c b/arch/x86_64/kernel/memory.c index 32225975..cf7cac4b 100644 --- a/arch/x86_64/kernel/memory.c +++ b/arch/x86_64/kernel/memory.c @@ -2239,7 +2239,7 @@ out: int ihk_mc_pt_set_range(page_table_t pt, struct process_vm *vm, void *start, void *end, uintptr_t phys, enum ihk_mc_pt_attribute attr, - int pgshift, struct vm_range *range) + int pgshift, struct vm_range *range, int overwrite) { int error; struct set_range_args args; @@ -2448,7 +2448,7 @@ static int move_one_page(void *arg0, page_table_t pt, pte_t *ptep, attr = apte & ~PT_PHYSMASK; error = ihk_mc_pt_set_range(pt, args->vm, (void *)dest, - (void *)(dest + pgsize), phys, attr, pgshift, args->range); + (void *)(dest + pgsize), phys, attr, pgshift, args->range, 0); if (error) { kprintf("move_one_page(%p,%p,%p %#lx,%p,%d):" "set failed. %d\n", diff --git a/arch/x86_64/kernel/syscall.c b/arch/x86_64/kernel/syscall.c index ad325d8f..3c1f61d0 100644 --- a/arch/x86_64/kernel/syscall.c +++ b/arch/x86_64/kernel/syscall.c @@ -2017,7 +2017,7 @@ int arch_map_vdso(struct process_vm *vm) s = vm->vdso_addr + (i * PAGE_SIZE); e = s + PAGE_SIZE; error = ihk_mc_pt_set_range(pt, vm, s, e, - vdso.vdso_physlist[i], attr, 0, range); + vdso.vdso_physlist[i], attr, 0, range, 0); if (error) { ekprintf("ihk_mc_pt_set_range failed. %d\n", error); goto out; @@ -2049,7 +2049,7 @@ int arch_map_vdso(struct process_vm *vm) e = s + PAGE_SIZE; attr = PTATTR_ACTIVE | PTATTR_USER | PTATTR_NO_EXECUTE; error = ihk_mc_pt_set_range(pt, vm, s, e, - vdso.vvar_phys, attr, 0, range); + vdso.vvar_phys, attr, 0, range, 0); if (error) { ekprintf("ihk_mc_pt_set_range failed. %d\n", error); goto out; @@ -2060,7 +2060,7 @@ int arch_map_vdso(struct process_vm *vm) e = s + PAGE_SIZE; attr = PTATTR_ACTIVE | PTATTR_USER | PTATTR_NO_EXECUTE | PTATTR_UNCACHABLE; error = ihk_mc_pt_set_range(pt, vm, s, e, - vdso.hpet_phys, attr, 0, range); + vdso.hpet_phys, attr, 0, range, 0); if (error) { ekprintf("ihk_mc_pt_set_range failed. %d\n", error); goto out; @@ -2071,7 +2071,7 @@ int arch_map_vdso(struct process_vm *vm) e = s + PAGE_SIZE; attr = PTATTR_ACTIVE | PTATTR_USER | PTATTR_NO_EXECUTE; error = ihk_mc_pt_set_range(pt, vm, s, e, - vdso.pvti_phys, attr, 0, range); + vdso.pvti_phys, attr, 0, range, 0); if (error) { ekprintf("ihk_mc_pt_set_range failed. %d\n", error); goto out; diff --git a/kernel/host.c b/kernel/host.c index 28e5700a..c5ce9d25 100644 --- a/kernel/host.c +++ b/kernel/host.c @@ -159,7 +159,7 @@ int prepare_process_ranges_args_envs(struct thread *thread, (void *)range->start, (void *)range->start + (range_npages * PAGE_SIZE), up, ptattr, - range->pgshift, range); + range->pgshift, range, 0); if (error) { kprintf("%s: ihk_mc_pt_set_range failed. %d\n", diff --git a/kernel/process.c b/kernel/process.c index 78037b71..75527e20 100644 --- a/kernel/process.c +++ b/kernel/process.c @@ -707,7 +707,7 @@ static int copy_user_pte(void *arg0, page_table_t src_pt, pte_t *src_ptep, void error = ihk_mc_pt_set_range(args->new_vm->address_space->page_table, args->new_vm, pgaddr, pgaddr + pgsize, phys, attr, - pgshift, args->range); + pgshift, args->range, 0); if (error) { args->fault_addr = (intptr_t)pgaddr; goto out; @@ -821,7 +821,7 @@ int update_process_page_table(struct process_vm *vm, flags = ihk_mc_spinlock_lock(&vm->page_table_lock); error = ihk_mc_pt_set_range(vm->address_space->page_table, vm, (void *)range->start, (void *)range->end, phys, attr, - range->pgshift, range); + range->pgshift, range, 0); if (error) { kprintf("update_process_page_table:ihk_mc_pt_set_range failed. %d\n", error); goto out; @@ -2016,7 +2016,7 @@ retry: else { error = ihk_mc_pt_set_range(vm->address_space->page_table, vm, pgaddr, pgaddr + pgsize, phys, - attr, range->pgshift, range); + attr, range->pgshift, range, 0); if (error) { kprintf("page_fault_process_memory_range(%p,%lx-%lx %lx,%lx,%lx):set_range failed. %d\n", vm, range->start, range->end, range->flag, fault_addr, reason, error); goto out; @@ -2291,7 +2291,7 @@ int init_process_stack(struct thread *thread, struct program_load_desc *pn, thread->vm, (void *)(end - minsz), (void *)end, virt_to_phys(stack), arch_vrflag_to_ptattr(vrflag, PF_POPULATE, NULL), - LARGE_PAGE_SHIFT, range + LARGE_PAGE_SHIFT, range, 0 ); if (error) { diff --git a/kernel/syscall.c b/kernel/syscall.c index 04286eba..5f4c2e9c 100644 --- a/kernel/syscall.c +++ b/kernel/syscall.c @@ -1938,7 +1938,8 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot, virt_to_phys(memobj->pages[i]), ptattr, PAGE_SHIFT, - range); + range, + 0); if (error) { kprintf("%s: ERROR: mapping %d page of pre-mapped file\n", __FUNCTION__, i); diff --git a/kernel/xpmem.c b/kernel/xpmem.c index 4de99832..72e07946 100644 --- a/kernel/xpmem.c +++ b/kernel/xpmem.c @@ -1957,7 +1957,7 @@ static int xpmem_remap_pte( else { ret = ihk_mc_pt_set_range(vm->address_space->page_table, vm, att_pgaddr, att_pgaddr + att_pgsize, seg_phys, att_attr, - vmr->pgshift, vmr); + vmr->pgshift, vmr, 0); if (ret) { ret = -EFAULT; ekprintf("%s: ERROR: ihk_mc_pt_set_range() failed %d\n", diff --git a/lib/include/ihk/mm.h b/lib/include/ihk/mm.h index 88cb5c20..d917a047 100644 --- a/lib/include/ihk/mm.h +++ b/lib/include/ihk/mm.h @@ -179,7 +179,7 @@ int ihk_mc_pt_change_attr_range(page_table_t pt, void *start, void *end, pte_t *ihk_mc_pt_lookup_pte(page_table_t pt, void *virt, int pgshift, void **pgbasep, size_t *pgsizep, int *p2alignp); int ihk_mc_pt_set_range(page_table_t pt, struct process_vm *vm, void *start, void *end, uintptr_t phys, enum ihk_mc_pt_attribute attr, - int pgshift, struct vm_range *range); + int pgshift, struct vm_range *range, int overwrite); int ihk_mc_pt_set_pte(page_table_t pt, pte_t *ptep, size_t pgsize, uintptr_t phys, enum ihk_mc_pt_attribute attr); int ihk_mc_pt_prepare_map(page_table_t pt, void *virt, unsigned long size, enum ihk_mc_pt_prepare_flag);