From 201f5ce500a6bce9d773af4aef32f937ca9a51a3 Mon Sep 17 00:00:00 2001 From: Balazs Gerofi Date: Wed, 27 May 2020 13:54:04 +0900 Subject: [PATCH] MM: straight mapping Change-Id: I70871f8c382fb00aa719ed501cc5de436d916d7f --- arch/arm64/kernel/memory.c | 8 ++ arch/x86_64/kernel/memory.c | 8 ++ executer/include/uprotocol.h | 2 + executer/kernel/mcctrl/syscall.c | 10 +- executer/user/mcexec.c | 25 +++- kernel/host.c | 3 + kernel/include/process.h | 9 ++ kernel/include/profile.h | 2 + kernel/include/syscall.h | 2 + kernel/process.c | 81 ++++++++++++ kernel/profile.c | 2 + kernel/syscall.c | 214 ++++++++++++++++++++++++++++++- kernel/xpmem.c | 40 ++++-- 13 files changed, 385 insertions(+), 21 deletions(-) diff --git a/arch/arm64/kernel/memory.c b/arch/arm64/kernel/memory.c index a84bc218..9d59849e 100644 --- a/arch/arm64/kernel/memory.c +++ b/arch/arm64/kernel/memory.c @@ -2344,6 +2344,14 @@ static int clear_range(struct page_table *pt, struct process_vm *vm, if (memobj && ((memobj->flags & MF_PREMAP))) { args.free_physical = 0; } + + if (vm->proc->straight_va && + (void *)start == vm->proc->straight_va && + (void *)end == (vm->proc->straight_va + + vm->proc->straight_len)) { + args.free_physical = 0; + } + args.memobj = memobj; args.vm = vm; diff --git a/arch/x86_64/kernel/memory.c b/arch/x86_64/kernel/memory.c index 6b34036b..bd99572d 100644 --- a/arch/x86_64/kernel/memory.c +++ b/arch/x86_64/kernel/memory.c @@ -1651,6 +1651,14 @@ static int clear_range(struct page_table *pt, struct process_vm *vm, if (memobj && ((memobj->flags & MF_PREMAP))) { args.free_physical = 0; } + + if (vm->proc->straight_va && + (void *)start == vm->proc->straight_va && + (void *)end == (vm->proc->straight_va + + vm->proc->straight_len)) { + args.free_physical = 0; + } + args.memobj = memobj; args.vm = vm; diff --git a/executer/include/uprotocol.h b/executer/include/uprotocol.h index a277162e..003caaa2 100644 --- a/executer/include/uprotocol.h +++ b/executer/include/uprotocol.h @@ -150,6 +150,8 @@ struct program_load_desc { int thp_disable; int uti_thread_rank; /* N-th clone() spawns a thread on Linux CPU */ int uti_use_last_cpu; /* Work-around not to share CPU with OpenMP thread */ + int straight_map; + size_t straight_map_threshold; int nr_processes; int process_rank; __cpu_set_unit cpu_set[PLD_CPU_SET_SIZE]; diff --git a/executer/kernel/mcctrl/syscall.c b/executer/kernel/mcctrl/syscall.c index 1ae300d1..5c5cb565 100644 --- a/executer/kernel/mcctrl/syscall.c +++ b/executer/kernel/mcctrl/syscall.c @@ -655,6 +655,9 @@ static int rus_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) goto put_and_out; } + // Force regular page size + pgsize = PAGE_SIZE; + rva = (unsigned long)addr & ~(pgsize - 1); rpa = rpa & ~(pgsize - 1); @@ -666,7 +669,8 @@ static int rus_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) /* LWK may hold large page based mappings that align rva outside * Linux' VMA, make sure we don't try to map to those pages */ - if (rva + (pix * PAGE_SIZE) < vma->vm_start) { + if (rva + (pix * PAGE_SIZE) < vma->vm_start || + rva + (pix * PAGE_SIZE) > vma->vm_end) { continue; } @@ -677,11 +681,11 @@ static int rus_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) if (error) { pr_err("%s: error inserting mapping for 0x%#lx " "(req: TID: %d, syscall: %lu) error: %d," - " vm_start: 0x%lx, vm_end: 0x%lx\n", + " vm_start: 0x%lx, vm_end: 0x%lx, pgsize: %lu, ind: %lu\n", __func__, (unsigned long)addr, packet.fault_tid, rsysnum, error, - vma->vm_start, vma->vm_end); + vma->vm_start, vma->vm_end, pgsize, pix); } } else diff --git a/executer/user/mcexec.c b/executer/user/mcexec.c index 1bbd4740..a210ee8c 100644 --- a/executer/user/mcexec.c +++ b/executer/user/mcexec.c @@ -187,6 +187,8 @@ static int mpol_no_stack = 0; static int mpol_no_bss = 0; static int mpol_shm_premap = 0; static int no_bind_ikc_map = 0; +static int straight_map = 0; +static unsigned long straight_map_threshold = (1024*1024); static unsigned long mpol_threshold = 0; static unsigned long heap_extension = -1; static int profile = 0; @@ -1674,6 +1676,18 @@ static struct option mcexec_options[] = { .flag = NULL, .val = 'M', }, + { + .name = "enable-straight-map", + .has_arg = no_argument, + .flag = &straight_map, + .val = 1, + }, + { + .name = "straight-map-threshold", + .has_arg = required_argument, + .flag = NULL, + .val = 'S', + }, { .name = "disable-sched-yield", .has_arg = no_argument, @@ -2095,10 +2109,10 @@ int main(int argc, char **argv) /* Parse options ("+" denotes stop at the first non-option) */ #ifdef ADD_ENVS_OPTION - while ((opt = getopt_long(argc, argv, "+c:n:t:M:h:e:s:m:u:", + while ((opt = getopt_long(argc, argv, "+c:n:t:M:h:e:s:m:u:S:", mcexec_options, NULL)) != -1) { #else /* ADD_ENVS_OPTION */ - while ((opt = getopt_long(argc, argv, "+c:n:t:M:h:s:m:u:", + while ((opt = getopt_long(argc, argv, "+c:n:t:M:h:s:m:u:S:", mcexec_options, NULL)) != -1) { #endif /* ADD_ENVS_OPTION */ switch (opt) { @@ -2140,6 +2154,10 @@ int main(int argc, char **argv) heap_extension = atobytes(optarg); break; + case 'S': + straight_map_threshold = atobytes(optarg); + break; + #ifdef ADD_ENVS_OPTION case 'e': add_env_list(&extra_env, optarg); @@ -2678,6 +2696,9 @@ int main(int argc, char **argv) desc->uti_use_last_cpu = uti_use_last_cpu; desc->thp_disable = get_thp_disable(); + desc->straight_map = straight_map; + desc->straight_map_threshold = straight_map_threshold; + /* user_start and user_end are set by this call */ if (ioctl(fd, MCEXEC_UP_PREPARE_IMAGE, (unsigned long)desc) != 0) { perror("prepare"); diff --git a/kernel/host.c b/kernel/host.c index 8c2caf14..4322d381 100644 --- a/kernel/host.c +++ b/kernel/host.c @@ -546,6 +546,9 @@ static int process_msg_prepare_process(unsigned long rphys) proc->uti_thread_rank = pn->uti_thread_rank; proc->uti_use_last_cpu = pn->uti_use_last_cpu; + proc->straight_map = pn->straight_map; + proc->straight_map_threshold = pn->straight_map_threshold; + #ifdef PROFILE_ENABLE proc->profile = pn->profile; thread->profile = pn->profile; diff --git a/kernel/include/process.h b/kernel/include/process.h index f958a390..89a4fbf7 100644 --- a/kernel/include/process.h +++ b/kernel/include/process.h @@ -390,6 +390,7 @@ struct vm_range { struct rb_node vm_rb_node; unsigned long start, end; unsigned long flag; + unsigned long straight_start; struct memobj *memobj; off_t objoff; int pgshift; /* page size. 0 means THP */ @@ -563,6 +564,9 @@ struct process { int clone_count; int thp_disable; + int straight_map; + size_t straight_map_threshold; + // perf_event int perf_status; #define PP_NONE 0 @@ -578,6 +582,11 @@ struct process { #endif // PROFILE_ENABLE int nr_processes; /* For partitioned execution */ int process_rank; /* Rank in partition */ + + void *straight_va; + size_t straight_len; + unsigned long straight_pa; + int coredump_barrier_count, coredump_barrier_count2; mcs_rwlock_lock_t coredump_lock; // lock for coredump }; diff --git a/kernel/include/profile.h b/kernel/include/profile.h index 21ac64c1..78ab15cd 100644 --- a/kernel/include/profile.h +++ b/kernel/include/profile.h @@ -40,6 +40,8 @@ enum profile_event_type { PROFILE_remote_page_fault, PROFILE_mpol_alloc_missed, PROFILE_mmap_anon_contig_phys, + PROFILE_mmap_anon_straight, + PROFILE_mmap_anon_not_straight, PROFILE_mmap_anon_no_contig_phys, PROFILE_mmap_regular_file, PROFILE_mmap_device_file, diff --git a/kernel/include/syscall.h b/kernel/include/syscall.h index 156aa3b7..f0a757c2 100644 --- a/kernel/include/syscall.h +++ b/kernel/include/syscall.h @@ -217,6 +217,8 @@ struct program_load_desc { int thp_disable; int uti_thread_rank; /* N-th clone() spawns a thread on Linux CPU */ int uti_use_last_cpu; /* Work-around not to share CPU with OpenMP thread */ + int straight_map; + size_t straight_map_threshold; int nr_processes; int process_rank; __cpu_set_unit cpu_set[PLD_CPU_SET_SIZE]; diff --git a/kernel/process.c b/kernel/process.c index 9f8f5947..02458678 100644 --- a/kernel/process.c +++ b/kernel/process.c @@ -945,6 +945,11 @@ int split_process_memory_range(struct process_vm *vm, struct vm_range *range, } newrange->start = addr; + newrange->straight_start = 0; + if (range->straight_start) { + newrange->straight_start = + range->straight_start + (addr - range->start); + } newrange->end = range->end; newrange->flag = range->flag; newrange->pgshift = range->pgshift; @@ -1045,6 +1050,11 @@ static int free_process_memory_range(struct process_vm *vm, start = range->start; end = range->end; + + /* No regular page table manipulation for straight mappings */ + if (range->straight_start || ((void *)start == vm->proc->straight_va)) + goto straight_out; + if (!(range->flag & (VR_REMOTE | VR_IO_NOCACHE | VR_RESERVED))) { neighbor = previous_process_memory_range(vm, range); pgsize = -1; @@ -1126,11 +1136,37 @@ static int free_process_memory_range(struct process_vm *vm, memobj_unref(range->memobj); } +straight_out: rb_erase(&range->vm_rb_node, &vm->vm_range_tree); for (i = 0; i < VM_RANGE_CACHE_SIZE; ++i) { if (vm->range_cache[i] == range) vm->range_cache[i] = NULL; } + + /* For straight ranges just free physical memory */ + if (range->straight_start) { + ihk_mc_free_pages(phys_to_virt(vm->proc->straight_pa + + (range->straight_start - (unsigned long)vm->proc->straight_va)), + (range->end - range->start) >> PAGE_SHIFT); + + dkprintf("%s: straight range 0x%lx @ straight 0x%lx physical memory freed\n", + __FUNCTION__, range->start, range->straight_start); + } + /* For the main straight mapping, free page tables */ + else if (range->start == (unsigned long)vm->proc->straight_va && + range->end == ((unsigned long)vm->proc->straight_va + + vm->proc->straight_len)) { + ihk_mc_spinlock_lock_noirq(&vm->page_table_lock); + error = ihk_mc_pt_clear_range(vm->address_space->page_table, vm, + (void *)start, (void *)end); + ihk_mc_spinlock_unlock_noirq(&vm->page_table_lock); + + dkprintf("%s: straight mapping 0x%lx unmapped\n", + __FUNCTION__, vm->proc->straight_va); + vm->proc->straight_va = 0; + vm->proc->straight_len = 0; + } + kfree(range); dkprintf("free_process_memory_range(%p,%lx-%lx): 0\n", @@ -1148,6 +1184,50 @@ int remove_process_memory_range(struct process_vm *vm, dkprintf("remove_process_memory_range(%p,%lx,%lx)\n", vm, start, end); + /* + * Convert to real virtual address for straight ranges, + * but not for the main straight mapping + */ + if (vm->proc->straight_va && + start >= (unsigned long)vm->proc->straight_va && + end <= ((unsigned long)vm->proc->straight_va + + vm->proc->straight_len) && + !(start == (unsigned long)vm->proc->straight_va && + end == ((unsigned long)vm->proc->straight_va + + vm->proc->straight_len))) { + struct vm_range *range_iter; + struct vm_range *range = NULL; + unsigned long len = end - start; + + range_iter = lookup_process_memory_range(vm, 0, -1); + + while (range_iter) { + if (range_iter->straight_start && + start >= range_iter->straight_start && + start < (range_iter->straight_start + + (range_iter->end - range_iter->start))) { + range = range_iter; + break; + } + + range_iter = next_process_memory_range(vm, range_iter); + } + + if (!range) { + kprintf("%s: WARNING: no straight mapping range found for 0x%lx\n", + __FUNCTION__, start); + return 0; + } + + dkprintf("%s: straight range converted from 0x%lx:%lu -> 0x%lx:%lu\n", + __FUNCTION__, + start, len, + range->start + (start - range->straight_start), len); + + start = range->start + (start - range->straight_start); + end = start + len; + } + next = lookup_process_memory_range(vm, start, end); while ((range = next) && range->start < end) { next = next_process_memory_range(vm, range); @@ -1345,6 +1425,7 @@ int add_process_memory_range(struct process_vm *vm, range->objoff = offset; range->pgshift = pgshift; range->private_data = NULL; + range->straight_start = 0; rc = 0; if (phys == NOPHYS) { diff --git a/kernel/profile.c b/kernel/profile.c index 01dab56a..cbab0a6d 100644 --- a/kernel/profile.c +++ b/kernel/profile.c @@ -63,6 +63,8 @@ char *profile_event_names[] = "remote_page_fault", "mpol_alloc_missed", "mmap_anon_contig_phys", + "|-------mmap_straight", + "|---mmap_not_straight", "mmap_anon_no_contig_phys", "mmap_regular_file", "mmap_device_file", diff --git a/kernel/syscall.c b/kernel/syscall.c index fdc6b185..69b6118b 100644 --- a/kernel/syscall.c +++ b/kernel/syscall.c @@ -1582,10 +1582,20 @@ int do_munmap(void *addr, size_t len, int holding_memory_range_lock) { int error; int ro_freed; + struct thread *thread = cpu_local_var(current); begin_free_pages_pending(); error = remove_process_memory_range(cpu_local_var(current)->vm, (intptr_t)addr, (intptr_t)addr+len, &ro_freed); + + /* No host involvement for straight mapping ranges */ + if (thread->proc->straight_va && + addr >= thread->proc->straight_va && + (addr + len) <= + (thread->proc->straight_va + thread->proc->straight_len)) { + goto out; + } + if (error || !ro_freed) { clear_host_pte((uintptr_t)addr, len, holding_memory_range_lock); } @@ -1596,6 +1606,8 @@ int do_munmap(void *addr, size_t len, int holding_memory_range_lock) /* through */ } } + +out: finish_free_pages_pending(); dkprintf("%s: 0x%lx:%lu, error: %ld\n", @@ -1667,6 +1679,7 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot, void *p = NULL; int vrflags; uintptr_t phys; + intptr_t straight_phys; struct memobj *memobj = NULL; int maxprot; int denied; @@ -1708,6 +1721,124 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot, flush_nfo_tlb(); + /* Initialize straight large memory mapping */ + if (proc->straight_map && !proc->straight_va) { + unsigned long straight_pa_start = 0xFFFFFFFFFFFFFFFF; + unsigned long straight_pa_end = 0; + int i; + int p2align = PAGE_P2ALIGN; + size_t psize = PAGE_SIZE; + unsigned long vrflags; + enum ihk_mc_pt_attribute ptattr; + struct vm_range *range; + + vrflags = PROT_TO_VR_FLAG(PROT_READ | PROT_WRITE); + vrflags |= VRFLAG_PROT_TO_MAXPROT(vrflags); + vrflags |= VR_DEMAND_PAGING; + + for (i = 0; i < ihk_mc_get_nr_memory_chunks(); ++i) { + unsigned long start, end; + + ihk_mc_get_memory_chunk(i, &start, &end, NULL); + + if (straight_pa_start > start) { + straight_pa_start = start; + } + + if (straight_pa_end < end) { + straight_pa_end = end; + } + } + + kprintf("%s: straight_pa_start: 0x%lx, straight_pa_end: 0x%lx\n", + __FUNCTION__, straight_pa_start, straight_pa_end); + + error = arch_get_smaller_page_size(NULL, + straight_pa_end - straight_pa_start, + &psize, &p2align); + + if (error) { + kprintf("%s: arch_get_smaller_page_size failed: %d\n", + __FUNCTION__, error); + goto straight_out; + } + //psize = PTL2_SIZE; + //p2align = PTL2_SHIFT - PTL1_SHIFT; + + // Force 512G page + //psize = (1UL << 39); + //p2align = 39 - PAGE_SHIFT; + + // Force 512MB page + psize = (1UL << 29); + p2align = 29 - PAGE_SHIFT; + + kprintf("%s: using page shift: %d, psize: %lu\n", + __FUNCTION__, p2align + PAGE_SHIFT, psize); + + straight_pa_start &= ~(psize - 1); + straight_pa_end = (straight_pa_end + psize - 1) & ~(psize - 1); + + kprintf("%s: aligned straight_pa_start: 0x%lx, straight_pa_end: 0x%lx\n", + __FUNCTION__, straight_pa_start, straight_pa_end); + + proc->straight_len = straight_pa_end - straight_pa_start; + error = search_free_space(proc->straight_len, + PAGE_SHIFT + p2align, (uintptr_t *)&proc->straight_va); + + if (error) { + kprintf("%s: search_free_space() failed: %d\n", + __FUNCTION__, error); + proc->straight_va = 0; + goto straight_out; + } + + dkprintf("%s: straight_va: 0x%lx to be used\n", + __FUNCTION__, proc->straight_va); + + if (add_process_memory_range(proc->vm, (unsigned long)proc->straight_va, + (unsigned long)proc->straight_va + proc->straight_len, + NOPHYS, vrflags, NULL, 0, + PAGE_SHIFT + p2align, &range) != 0) { + kprintf("%s: error: adding straight memory range \n", + __FUNCTION__); + proc->straight_va = 0; + goto straight_out; + } + + kprintf("%s: straight_va: 0x%lx, range->pgshift: %d, range OK\n", + __FUNCTION__, proc->straight_va, range->pgshift); + + ptattr = arch_vrflag_to_ptattr(range->flag, PF_POPULATE, NULL); + error = ihk_mc_pt_set_range(proc->vm->address_space->page_table, + proc->vm, + (void *)range->start, + (void *)range->end, + straight_pa_start, ptattr, + range->pgshift, + range, 0); + + if (error) { + kprintf("%s: ihk_mc_pt_set_range() failed: %d\n", + __FUNCTION__, error); + proc->straight_va = 0; + goto straight_out; + } + //ihk_mc_pt_print_pte(proc->vm->address_space->page_table, range->start); + + region->map_end = (unsigned long)proc->straight_va + proc->straight_len; + proc->straight_pa = straight_pa_start; + kprintf("%s: straight mapping: 0x%lx:%lu @ 0x%lx, " + "psize: %lu, straight_map_threshold: %lu\n", + __FUNCTION__, + proc->straight_va, + proc->straight_len, + proc->straight_pa, + psize, + proc->straight_map_threshold); + } +straight_out: + if (flags & MAP_HUGETLB) { pgshift = (flags >> MAP_HUGE_SHIFT) & 0x3F; if (!pgshift) { @@ -1735,6 +1866,15 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot, ihk_rwspinlock_write_lock_noirq(&thread->vm->memory_range_lock); + if ((flags & MAP_FIXED) && proc->straight_va && + ((void *)addr >= proc->straight_va) && + ((void *)addr + len) <= (proc->straight_va + proc->straight_len)) { + kprintf("%s: can't map MAP_FIXED into straight mapping\n", + __FUNCTION__); + error = -EINVAL; + goto out; + } + if (flags & MAP_FIXED) { /* clear specified address range */ error = do_munmap((void *)addr, len, 1/* holding memory_range_lock */); @@ -1791,6 +1931,7 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot, } phys = 0; + straight_phys = 0; off = 0; maxprot = PROT_READ | PROT_WRITE | PROT_EXEC; if (!(flags & MAP_ANONYMOUS)) { @@ -1978,6 +2119,31 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot, } vrflags |= VRFLAG_PROT_TO_MAXPROT(PROT_TO_VR_FLAG(maxprot)); + /* + * Large anonymous non-fix allocations are in straight mapping, + * pretend demand paging to avoid filling in PTEs + */ + if ((flags & MAP_ANONYMOUS) && proc->straight_map && + !(flags & MAP_FIXED) && phys) { + if (len >= proc->straight_map_threshold) { + dkprintf("%s: range 0x%lx:%lu will be straight, addding VR_DEMAND\n", + __FUNCTION__, addr, len); + vrflags |= VR_DEMAND_PAGING; + straight_phys = phys; + phys = 0; +#ifdef PROFILE_ENABLE + profile_event_add(PROFILE_mmap_anon_straight, len); +#endif // PROFILE_ENABLE + } + else { +#ifdef PROFILE_ENABLE + if (cpu_local_var(current)->profile) + kprintf("%s: contiguous but not straight? len: %lu\n", __func__, len); + profile_event_add(PROFILE_mmap_anon_not_straight, len); +#endif // PROFILE_ENABLE + } + } + error = add_process_memory_range(thread->vm, addr, addr+len, phys, vrflags, memobj, off, pgshift, &range); if (error) { @@ -1988,6 +2154,19 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot, goto out; } + /* Update straight mapping start address */ + if (straight_phys) { + extern int zero_at_free; + range->straight_start = + (unsigned long)proc->straight_va + + (straight_phys - proc->straight_pa); + dkprintf("%s: range 0x%lx:%lu is straight starting at 0x%lx\n", + __FUNCTION__, addr, len, range->straight_start); + if (!zero_at_free) { + memset((void *)phys_to_virt(straight_phys), 0, len); + } + } + /* Determine pre-populated size */ populate_len = memobj ? min(len, memobj->size) : len; @@ -2042,12 +2221,13 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot, ro_vma_mapped = 0; out: - if (ro_vma_mapped) { + if (ro_vma_mapped && !range->straight_start) { (void)set_host_vma(addr, len, PROT_READ | PROT_WRITE | PROT_EXEC, 1/* holding memory_range_lock */); } ihk_rwspinlock_write_unlock_noirq(&thread->vm->memory_range_lock); - if (!error && populated_mapping && !((vrflags & VR_PROT_MASK) == VR_PROT_NONE)) { + if (!error && populated_mapping && + !((vrflags & VR_PROT_MASK) == VR_PROT_NONE) && !range->straight_start) { error = populate_process_memory(thread->vm, (void *)addr, populate_len); @@ -2087,7 +2267,9 @@ out: addr, len, addr0, len0, prot, flags, fd, off0, error, addr); - return (!error)? addr: error; + return !error ? + (range->straight_start ? range->straight_start : addr) : + error; } SYSCALL_DECLARE(munmap) @@ -2167,6 +2349,16 @@ SYSCALL_DECLARE(mprotect) return 0; } + if (thread->proc->straight_va && + ((void *)start >= thread->proc->straight_va) && + (void *)end <= (thread->proc->straight_va + + thread->proc->straight_len)) { + kprintf("%s: ignored for straight mapping 0x%lx\n", + __FUNCTION__, start); + error = 0; + goto out_straight; + } + flush_nfo_tlb(); ihk_rwspinlock_write_lock_noirq(&thread->vm->memory_range_lock); @@ -2260,6 +2452,8 @@ out: } } ihk_rwspinlock_write_unlock_noirq(&thread->vm->memory_range_lock); + +out_straight: dkprintf("[%d]sys_mprotect(%lx,%lx,%x): %d\n", ihk_mc_get_processor_id(), start, len0, prot, error); return error; @@ -8730,6 +8924,15 @@ SYSCALL_DECLARE(mremap) dkprintf("sys_mremap(%#lx,%#lx,%#lx,%#x,%#lx)\n", oldaddr, oldsize0, newsize0, flags, newaddr); + + if (vm->proc->straight_va && + (void *)oldaddr >= vm->proc->straight_va && + (void *)oldaddr < vm->proc->straight_va + vm->proc->straight_len) { + kprintf("%s: reject for straight range 0x%lx\n", + __FUNCTION__, oldaddr); + return -EINVAL; + } + ihk_rwspinlock_write_lock_noirq(&vm->memory_range_lock); /* check arguments */ @@ -9133,6 +9336,11 @@ SYSCALL_DECLARE(mbind) __FUNCTION__, addr, len, mode, nodemask, flags); + /* No bind support for straight mapped processes */ + if (cpu_local_var(current)->proc->straight_va) { + return 0; + } + /* Validate arguments */ if (addr & ~PAGE_MASK) { return -EINVAL; diff --git a/kernel/xpmem.c b/kernel/xpmem.c index ca561129..b50e7c7b 100644 --- a/kernel/xpmem.c +++ b/kernel/xpmem.c @@ -1919,21 +1919,35 @@ static int xpmem_remap_pte( goto out; } - seg_pte = ihk_mc_pt_lookup_pte(seg_tg->vm->address_space->page_table, - (void *)seg_vaddr, seg_vmr->pgshift, &seg_pgaddr, &seg_pgsize, - &seg_p2align); - if (!seg_pte) { - ret = -EFAULT; - ekprintf("%s: ERROR: ihk_mc_pt_lookup_pte() failed\n", - __FUNCTION__); - goto out; + if (seg_tg->vm->proc->straight_va && + seg_vaddr >= (unsigned long)seg_tg->vm->proc->straight_va && + seg_vaddr < ((unsigned long)seg_tg->vm->proc->straight_va + + seg_tg->vm->proc->straight_len)) { + seg_phys = (((unsigned long)seg_vaddr & PAGE_MASK) - + (unsigned long)seg_tg->vm->proc->straight_va) + + seg_tg->vm->proc->straight_pa; + dkprintf("%s: 0x%lx in PID %d is straight -> phys: 0x%lx\n", + __func__, (unsigned long)seg_vaddr & PAGE_MASK, + seg_tg->tgid, seg_phys); } - XPMEM_DEBUG("seg_pte=0x%016lx, seg_pgaddr=0x%p, seg_pgsize=%lu, " - "seg_p2align=%d", - *seg_pte, seg_pgaddr, seg_pgsize, seg_p2align); + else { - seg_phys = pte_get_phys(seg_pte); - XPMEM_DEBUG("seg_phys=0x%lx", seg_phys); + seg_pte = ihk_mc_pt_lookup_pte(seg_tg->vm->address_space->page_table, + (void *)seg_vaddr, seg_vmr->pgshift, &seg_pgaddr, &seg_pgsize, + &seg_p2align); + if (!seg_pte) { + ret = -EFAULT; + ekprintf("%s: ERROR: ihk_mc_pt_lookup_pte() failed\n", + __FUNCTION__); + goto out; + } + XPMEM_DEBUG("seg_pte=0x%016lx, seg_pgaddr=0x%p, seg_pgsize=%lu, " + "seg_p2align=%d", + *seg_pte, seg_pgaddr, seg_pgsize, seg_p2align); + + seg_phys = pte_get_phys(seg_pte); + XPMEM_DEBUG("seg_phys=0x%lx", seg_phys); + } att_pte = ihk_mc_pt_lookup_pte(vm->address_space->page_table, (void *)vaddr, vmr->pgshift, &att_pgaddr, &att_pgsize,