gh0s7/mckernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ContiguousPTE[7/12] modify ihk_mc_pt_set_range

Browse Source 
Change-Id: Ib38530ce64a01f21107e0a6a73de7c54f214eb5a
This commit is contained in:
TOIDA,Suguru
2018-11-30 09:18:21 +09:00
committed by Masamichi Takagi
parent 24d3da32ed
commit c319fe08a4
8 changed files with 62 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
arch/arm64/kernel/memory.c
View File

@ -2274,6 +2274,7 @@ struct set_range_args {
uintptr_t diff;
struct process_vm *vm;
struct vm_range *range; /* To find pages we don't need to call memory_stat_rss_add() */
int overwrite;
};
int set_range_middle(void *args0, pte_t *ptep, uintptr_t base, uintptr_t start,
@ -2289,7 +2290,7 @@ int set_range_l1(void *args0, pte_t *ptep, uintptr_t base, uintptr_t start,
dkprintf("set_range_l1(%lx,%lx,%lx)\n", base, start, end);
if (!ptl1_null(ptep)) {
if (!args->overwrite && !ptl1_null(ptep)) {
error = -EBUSY;
ekprintf("set_range_l1(%lx,%lx,%lx):page exists. %d %lx\n",
base, start, end, error, *ptep);
@ -2298,6 +2299,21 @@ int set_range_l1(void *args0, pte_t *ptep, uintptr_t base, uintptr_t start,
}
phys = args->phys + (base - start);
if (__page_offset(base, PTL1_CONT_SIZE) == 0) { //check head pte
uintptr_t next_addr = base + PTL1_CONT_SIZE;
if (end < next_addr) {
next_addr = end;
}
// set contiguous bit until the next head pte
// if phys is aligned and range does not end early.
if (__page_offset(phys | next_addr, PTL1_CONT_SIZE) == 0) {
args->attr[0] |= PTE_CONT;
} else {
args->attr[0] &= ~PTE_CONT;
}
}
pte = phys | args->attr[0];
ptl1_set(ptep, pte);
@ -2334,10 +2350,11 @@ int set_range_middle(void *args0, pte_t *ptep, uintptr_t base, uintptr_t start,
walk_pte_fn_t* callback;
unsigned long pgsize;
unsigned long pgshift;
unsigned long cont_pgsize;
} table[] = {
{walk_pte_l1, set_range_l1, PTL2_SIZE, PTL2_SHIFT}, /*PTL2: second*/
{walk_pte_l2, set_range_l2, PTL3_SIZE, PTL3_SHIFT}, /*PTL3: first*/
{walk_pte_l3, set_range_l3, PTL4_SIZE, PTL4_SHIFT}, /*PTL4: zero*/
{walk_pte_l1, set_range_l1, PTL2_SIZE, PTL2_SHIFT, PTL2_CONT_SIZE}, /*PTL2: second*/
{walk_pte_l2, set_range_l2, PTL3_SIZE, PTL3_SHIFT, PTL3_CONT_SIZE}, /*PTL3: first*/
{walk_pte_l3, set_range_l3, PTL4_SIZE, PTL4_SHIFT, PTL4_CONT_SIZE}, /*PTL4: zero*/
};
const struct table tbl = table[level-2];
@ -2349,7 +2366,7 @@ int set_range_middle(void *args0, pte_t *ptep, uintptr_t base, uintptr_t start,
dkprintf("set_range_middle(%lx,%lx,%lx,%d)\n", base, start, end, level);
retry:
if (ptl_null(ptep, level)) {
if (ptl_null(ptep, level) || (args->overwrite && ptl_type_page(ptep, level))) {
pte_t pte;
uintptr_t phys;
if (level == 2 || (level == 3 && FIRST_LEVEL_BLOCK_SUPPORT)) {
@ -2358,7 +2375,28 @@ retry:
&& (!args->pgshift
|| (args->pgshift == tbl.pgshift))) {
phys = args->phys + (base - start);
ptl_set(ptep, phys | args->attr[level-1], level);
//check head pte
if (__page_offset(base, tbl.cont_pgsize) == 0) {
uintptr_t next_addr = base +
tbl.cont_pgsize;
if (end < next_addr) {
next_addr = end;
}
// set contiguous bit until the
// next head pte if phys is aligned
// and range does not end early.
if (__page_offset(phys | next_addr, tbl.cont_pgsize) == 0) {
args->attr[level-1] |= PTE_CONT;
} else {
args->attr[level-1] &= ~PTE_CONT;
}
}
ptl_set(ptep, phys | args->attr[level-1],
level);
error = 0;
dkprintf("set_range_middle(%lx,%lx,%lx,%d):"
"large page. %d %lx\n",
@ -2422,7 +2460,7 @@ out:
int ihk_mc_pt_set_range(page_table_t pt, struct process_vm *vm, void *start,
void *end, uintptr_t phys, enum ihk_mc_pt_attribute attr,
int pgshift, struct vm_range *range)
int pgshift, struct vm_range *range, int overwrite)
{
const struct table {
walk_pte_t* walk;
@ -2446,6 +2484,7 @@ int ihk_mc_pt_set_range(page_table_t pt, struct process_vm *vm, void *start,
args.vm = vm;
args.pgshift = pgshift;
args.range = range;
args.overwrite = overwrite;
// conversion
switch (CONFIG_ARM64_PGTABLE_LEVELS)
@ -2698,7 +2737,7 @@ static int move_one_page(void *arg0, page_table_t pt, pte_t *ptep,
attr = apte & ~PT_PHYSMASK;
error = ihk_mc_pt_set_range(pt, args->vm, (void *)dest,
(void *)(dest + pgsize), phys, attr, pgshift, args->range);
(void *)(dest + pgsize), phys, attr, pgshift, args->range, 0);
if (error) {
kprintf("move_one_page(%p,%p,%p %#lx,%p,%d):"
"set failed. %d\n",

4
arch/x86_64/kernel/memory.c
View File

@ -2239,7 +2239,7 @@ out:
int ihk_mc_pt_set_range(page_table_t pt, struct process_vm *vm, void *start,
void *end, uintptr_t phys, enum ihk_mc_pt_attribute attr,
int pgshift, struct vm_range *range)
int pgshift, struct vm_range *range, int overwrite)
{
int error;
struct set_range_args args;
@ -2448,7 +2448,7 @@ static int move_one_page(void *arg0, page_table_t pt, pte_t *ptep,
attr = apte & ~PT_PHYSMASK;
error = ihk_mc_pt_set_range(pt, args->vm, (void *)dest,
(void *)(dest + pgsize), phys, attr, pgshift, args->range);
(void *)(dest + pgsize), phys, attr, pgshift, args->range, 0);
if (error) {
kprintf("move_one_page(%p,%p,%p %#lx,%p,%d):"
"set failed. %d\n",

8
arch/x86_64/kernel/syscall.c
View File

@ -2017,7 +2017,7 @@ int arch_map_vdso(struct process_vm *vm)
s = vm->vdso_addr + (i * PAGE_SIZE);
e = s + PAGE_SIZE;
error = ihk_mc_pt_set_range(pt, vm, s, e,
vdso.vdso_physlist[i], attr, 0, range);
vdso.vdso_physlist[i], attr, 0, range, 0);
if (error) {
ekprintf("ihk_mc_pt_set_range failed. %d\n", error);
goto out;
@ -2049,7 +2049,7 @@ int arch_map_vdso(struct process_vm *vm)
e = s + PAGE_SIZE;
attr = PTATTR_ACTIVE | PTATTR_USER | PTATTR_NO_EXECUTE;
error = ihk_mc_pt_set_range(pt, vm, s, e,
vdso.vvar_phys, attr, 0, range);
vdso.vvar_phys, attr, 0, range, 0);
if (error) {
ekprintf("ihk_mc_pt_set_range failed. %d\n", error);
goto out;
@ -2060,7 +2060,7 @@ int arch_map_vdso(struct process_vm *vm)
e = s + PAGE_SIZE;
attr = PTATTR_ACTIVE | PTATTR_USER | PTATTR_NO_EXECUTE | PTATTR_UNCACHABLE;
error = ihk_mc_pt_set_range(pt, vm, s, e,
vdso.hpet_phys, attr, 0, range);
vdso.hpet_phys, attr, 0, range, 0);
if (error) {
ekprintf("ihk_mc_pt_set_range failed. %d\n", error);
goto out;
@ -2071,7 +2071,7 @@ int arch_map_vdso(struct process_vm *vm)
e = s + PAGE_SIZE;
attr = PTATTR_ACTIVE | PTATTR_USER | PTATTR_NO_EXECUTE;
error = ihk_mc_pt_set_range(pt, vm, s, e,
vdso.pvti_phys, attr, 0, range);
vdso.pvti_phys, attr, 0, range, 0);
if (error) {
ekprintf("ihk_mc_pt_set_range failed. %d\n", error);
goto out;

2
kernel/host.c
View File

@ -159,7 +159,7 @@ int prepare_process_ranges_args_envs(struct thread *thread,
(void *)range->start,
(void *)range->start + (range_npages * PAGE_SIZE),
up, ptattr,
range->pgshift, range);
range->pgshift, range, 0);
if (error) {
kprintf("%s: ihk_mc_pt_set_range failed. %d\n",

8
kernel/process.c
View File

@ -707,7 +707,7 @@ static int copy_user_pte(void *arg0, page_table_t src_pt, pte_t *src_ptep, void
error = ihk_mc_pt_set_range(args->new_vm->address_space->page_table,
args->new_vm, pgaddr, pgaddr + pgsize, phys, attr,
pgshift, args->range);
pgshift, args->range, 0);
if (error) {
args->fault_addr = (intptr_t)pgaddr;
goto out;
@ -821,7 +821,7 @@ int update_process_page_table(struct process_vm *vm,
flags = ihk_mc_spinlock_lock(&vm->page_table_lock);
error = ihk_mc_pt_set_range(vm->address_space->page_table, vm,
(void *)range->start, (void *)range->end, phys, attr,
range->pgshift, range);
range->pgshift, range, 0);
if (error) {
kprintf("update_process_page_table:ihk_mc_pt_set_range failed. %d\n", error);
goto out;
@ -2016,7 +2016,7 @@ retry:
else {
error = ihk_mc_pt_set_range(vm->address_space->page_table, vm,
pgaddr, pgaddr + pgsize, phys,
attr, range->pgshift, range);
attr, range->pgshift, range, 0);
if (error) {
kprintf("page_fault_process_memory_range(%p,%lx-%lx %lx,%lx,%lx):set_range failed. %d\n", vm, range->start, range->end, range->flag, fault_addr, reason, error);
goto out;
@ -2291,7 +2291,7 @@ int init_process_stack(struct thread *thread, struct program_load_desc *pn,
thread->vm, (void *)(end - minsz),
(void *)end, virt_to_phys(stack),
arch_vrflag_to_ptattr(vrflag, PF_POPULATE, NULL),
LARGE_PAGE_SHIFT, range
LARGE_PAGE_SHIFT, range, 0
);
if (error) {

3
kernel/syscall.c
View File

@ -1938,7 +1938,8 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot,
virt_to_phys(memobj->pages[i]),
ptattr,
PAGE_SHIFT,
range);
range,
0);
if (error) {
kprintf("%s: ERROR: mapping %d page of pre-mapped file\n",
__FUNCTION__, i);

2
kernel/xpmem.c
View File

@ -1957,7 +1957,7 @@ static int xpmem_remap_pte(
else {
ret = ihk_mc_pt_set_range(vm->address_space->page_table, vm,
att_pgaddr, att_pgaddr + att_pgsize, seg_phys, att_attr,
vmr->pgshift, vmr);
vmr->pgshift, vmr, 0);
if (ret) {
ret = -EFAULT;
ekprintf("%s: ERROR: ihk_mc_pt_set_range() failed %d\n",

2
lib/include/ihk/mm.h
View File

@ -179,7 +179,7 @@ int ihk_mc_pt_change_attr_range(page_table_t pt, void *start, void *end,
pte_t *ihk_mc_pt_lookup_pte(page_table_t pt, void *virt, int pgshift, void **pgbasep, size_t *pgsizep, int *p2alignp);
int ihk_mc_pt_set_range(page_table_t pt, struct process_vm *vm, void *start,
void *end, uintptr_t phys, enum ihk_mc_pt_attribute attr,
int pgshift, struct vm_range *range);
int pgshift, struct vm_range *range, int overwrite);
int ihk_mc_pt_set_pte(page_table_t pt, pte_t *ptep, size_t pgsize, uintptr_t phys, enum ihk_mc_pt_attribute attr);
int ihk_mc_pt_prepare_map(page_table_t pt, void *virt, unsigned long size,
enum ihk_mc_pt_prepare_flag);