gh0s7/mckernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

VM: use RW spinlock for vm_range_lock

Browse Source 
Change-Id: Id4654084207d55bf77cc9f8b42795e0f9873cfa0
This commit is contained in:
Balazs Gerofi
2020-04-30 13:32:32 +09:00
committed by Masamichi Takagi
parent 54b529c82d
commit d5aafca1ae
7 changed files with 94 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
arch/arm64/kernel/syscall.c
View File

@ -2151,7 +2151,7 @@ int do_process_vm_read_writev(int pid,
}
/* Check if parameters are okay */
ihk_mc_spinlock_lock_noirq(&lthread->vm->memory_range_lock);
ihk_rwspinlock_read_lock_noirq(&lthread->vm->memory_range_lock);
range = lookup_process_memory_range(lthread->vm,
(uintptr_t)local_iov,
@ -2173,7 +2173,7 @@ int do_process_vm_read_writev(int pid,
ret = 0;
arg_out:
ihk_mc_spinlock_unlock_noirq(&lthread->vm->memory_range_lock);
ihk_rwspinlock_read_unlock_noirq(&lthread->vm->memory_range_lock);
if (ret != 0) {
goto out;
@ -2242,7 +2242,7 @@ arg_out:
if (pli != li) {
struct vm_range *range;
ihk_mc_spinlock_lock_noirq(&lthread->vm->memory_range_lock);
ihk_rwspinlock_read_lock_noirq(&lthread->vm->memory_range_lock);
/* Is base valid? */
range = lookup_process_memory_range(lthread->vm,
@ -2272,7 +2272,7 @@ arg_out:
ret = 0;
pli_out:
ihk_mc_spinlock_unlock_noirq(&lthread->vm->memory_range_lock);
ihk_rwspinlock_read_unlock_noirq(&lthread->vm->memory_range_lock);
if (ret != 0) {
goto out;
@ -2285,7 +2285,7 @@ pli_out:
if (pri != ri) {
struct vm_range *range;
ihk_mc_spinlock_lock_noirq(&rvm->memory_range_lock);
ihk_rwspinlock_read_lock_noirq(&rvm->memory_range_lock);
/* Is base valid? */
range = lookup_process_memory_range(rvm,
@ -2315,7 +2315,7 @@ pli_out:
ret = 0;
pri_out:
ihk_mc_spinlock_unlock_noirq(&rvm->memory_range_lock);
ihk_rwspinlock_read_unlock_noirq(&rvm->memory_range_lock);
if (ret != 0) {
goto out;

12
arch/x86_64/kernel/syscall.c
View File

@ -2290,7 +2290,7 @@ int do_process_vm_read_writev(int pid,
}
/* Check if parameters are okay */
ihk_mc_spinlock_lock_noirq(&lthread->vm->memory_range_lock);
ihk_rwspinlock_read_lock_noirq(&lthread->vm->memory_range_lock);
range = lookup_process_memory_range(lthread->vm,
(uintptr_t)local_iov,
@ -2312,7 +2312,7 @@ int do_process_vm_read_writev(int pid,
ret = 0;
arg_out:
ihk_mc_spinlock_unlock_noirq(&lthread->vm->memory_range_lock);
ihk_rwspinlock_read_unlock_noirq(&lthread->vm->memory_range_lock);
if (ret != 0) {
goto out;
@ -2381,7 +2381,7 @@ arg_out:
if (pli != li) {
struct vm_range *range;
ihk_mc_spinlock_lock_noirq(&lthread->vm->memory_range_lock);
ihk_rwspinlock_read_lock_noirq(&lthread->vm->memory_range_lock);
/* Is base valid? */
range = lookup_process_memory_range(lthread->vm,
@ -2411,7 +2411,7 @@ arg_out:
ret = 0;
pli_out:
ihk_mc_spinlock_unlock_noirq(&lthread->vm->memory_range_lock);
ihk_rwspinlock_read_unlock_noirq(&lthread->vm->memory_range_lock);
if (ret != 0) {
goto out;
@ -2424,7 +2424,7 @@ pli_out:
if (pri != ri) {
struct vm_range *range;
ihk_mc_spinlock_lock_noirq(&rvm->memory_range_lock);
ihk_rwspinlock_read_lock_noirq(&rvm->memory_range_lock);
/* Is base valid? */
range = lookup_process_memory_range(rvm,
@ -2454,7 +2454,7 @@ pli_out:
ret = 0;
pri_out:
ihk_mc_spinlock_unlock_noirq(&rvm->memory_range_lock);
ihk_rwspinlock_read_unlock_noirq(&rvm->memory_range_lock);
if (ret != 0) {
goto out;

2
kernel/include/process.h
View File

@ -749,7 +749,7 @@ struct process_vm {
void *vvar_addr;
ihk_spinlock_t page_table_lock;
ihk_spinlock_t memory_range_lock;
ihk_rwspinlock_t memory_range_lock;
// to protect the followings:
// 1. addition of process "memory range" (extend_process_region, add_process_memory_range)
// 2. addition of process page table (allocate_pages, update_process_page_table)

26
kernel/process.c
View File

@ -243,7 +243,7 @@ static int
init_process_vm(struct process *owner, struct address_space *asp, struct process_vm *vm)
{
int i;
ihk_mc_spinlock_init(&vm->memory_range_lock);
ihk_rwspinlock_init(&vm->memory_range_lock);
ihk_mc_spinlock_init(&vm->page_table_lock);
ihk_atomic_set(&vm->refcount, 1);
@ -792,7 +792,7 @@ static int copy_user_ranges(struct process_vm *vm, struct process_vm *orgvm)
struct vm_range *last_insert;
struct copy_args args;
ihk_mc_spinlock_lock_noirq(&orgvm->memory_range_lock);
ihk_rwspinlock_read_lock_noirq(&orgvm->memory_range_lock);
/* Iterate original process' vm_range list and take a copy one-by-one */
last_insert = NULL;
@ -854,7 +854,7 @@ static int copy_user_ranges(struct process_vm *vm, struct process_vm *orgvm)
// memory_stat_rss_add() is called in child-node, i.e. copy_user_pte()
}
ihk_mc_spinlock_unlock_noirq(&orgvm->memory_range_lock);
ihk_rwspinlock_read_unlock_noirq(&orgvm->memory_range_lock);
return 0;
@ -881,7 +881,7 @@ err_rollback:
}
}
ihk_mc_spinlock_unlock_noirq(&orgvm->memory_range_lock);
ihk_rwspinlock_read_unlock_noirq(&orgvm->memory_range_lock);
return -1;
}
@ -2190,7 +2190,7 @@ static int do_page_fault_process_vm(struct process_vm *vm, void *fault_addr0, ui
if (thread->vm->is_memory_range_lock_taken == -1 ||
thread->vm->is_memory_range_lock_taken != ihk_mc_get_processor_id()) {
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_read_lock_noirq(&vm->memory_range_lock);
locked = 1;
} else {
dkprintf("%s: INFO: skip locking of memory_range_lock,pid=%d,tid=%d\n",
@ -2305,7 +2305,7 @@ static int do_page_fault_process_vm(struct process_vm *vm, void *fault_addr0, ui
error = 0;
out:
if (locked) {
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock);
}
dkprintf("[%d]do_page_fault_process_vm(%p,%lx,%lx): %d\n",
ihk_mc_get_processor_id(), vm, fault_addr0,
@ -2609,7 +2609,7 @@ void flush_process_memory(struct process_vm *vm)
int error;
dkprintf("flush_process_memory(%p)\n", vm);
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_write_lock_noirq(&vm->memory_range_lock);
/* Let concurrent page faults know the VM will be gone */
vm->exiting = 1;
while ((node = next)) {
@ -2627,7 +2627,7 @@ void flush_process_memory(struct process_vm *vm)
}
}
}
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock);
dkprintf("flush_process_memory(%p):\n", vm);
return;
}
@ -2642,7 +2642,7 @@ void free_process_memory_ranges(struct process_vm *vm)
return;
}
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_write_lock_noirq(&vm->memory_range_lock);
while ((node = next)) {
range = rb_entry(node, struct vm_range, vm_rb_node);
next = rb_next(node);
@ -2655,7 +2655,7 @@ void free_process_memory_ranges(struct process_vm *vm)
/* through */
}
}
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock);
}
static void free_thread_pages(struct thread *thread)
@ -2745,7 +2745,7 @@ free_all_process_memory_range(struct process_vm *vm)
struct rb_node *node, *next = rb_first(&vm->vm_range_tree);
int error;
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_write_lock_noirq(&vm->memory_range_lock);
while ((node = next)) {
range = rb_entry(node, struct vm_range, vm_rb_node);
next = rb_next(node);
@ -2758,7 +2758,7 @@ free_all_process_memory_range(struct process_vm *vm)
/* through */
}
}
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock);
}
void
@ -3189,7 +3189,7 @@ void sched_init(void)
&idle_thread->proc->children_list);
ihk_mc_init_context(&idle_thread->ctx, NULL, idle);
ihk_mc_spinlock_init(&idle_thread->vm->memory_range_lock);
ihk_rwspinlock_init(&idle_thread->vm->memory_range_lock);
idle_thread->vm->vm_range_tree = RB_ROOT;
idle_thread->vm->vm_range_numa_policy_tree = RB_ROOT;
idle_thread->proc->pid = 0;

14
kernel/procfs.c
View File

@ -459,7 +459,7 @@ static int _process_procfs_request(struct ikc_scd_packet *rpacket, int *result)
if (strcmp(p, "maps") == 0) {
struct vm_range *range;
if (!ihk_mc_spinlock_trylock_noirq(&vm->memory_range_lock)) {
if (!ihk_rwspinlock_read_trylock_noirq(&vm->memory_range_lock)) {
if (!result) {
if ((err = procfs_backlog(vm, rpacket))) {
goto err;
@ -505,14 +505,14 @@ static int _process_procfs_request(struct ikc_scd_packet *rpacket, int *result)
if (ans < 0 || ans > count ||
buf_add(&buf_top, &buf_cur, buf, ans) < 0) {
ihk_mc_spinlock_unlock_noirq(
ihk_rwspinlock_read_unlock_noirq(
&vm->memory_range_lock);
goto err;
}
range = next_process_memory_range(vm, range);
}
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock);
ans = 0;
goto end;
@ -535,7 +535,7 @@ static int _process_procfs_request(struct ikc_scd_packet *rpacket, int *result)
start = (offset / sizeof(uint64_t)) << PAGE_SHIFT;
end = start + ((count / sizeof(uint64_t)) << PAGE_SHIFT);
if (!ihk_mc_spinlock_trylock_noirq(&vm->memory_range_lock)) {
if (!ihk_rwspinlock_read_trylock_noirq(&vm->memory_range_lock)) {
if (!result) {
if ((err = procfs_backlog(vm, rpacket))) {
goto err;
@ -555,7 +555,7 @@ static int _process_procfs_request(struct ikc_scd_packet *rpacket, int *result)
++_buf;
}
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock);
dprintf("/proc/pagemap: 0x%lx - 0x%lx, count: %d\n",
start, end, count);
@ -587,7 +587,7 @@ static int _process_procfs_request(struct ikc_scd_packet *rpacket, int *result)
goto err;
}
if (!ihk_mc_spinlock_trylock_noirq(&vm->memory_range_lock)) {
if (!ihk_rwspinlock_read_trylock_noirq(&vm->memory_range_lock)) {
if (!result) {
if ((err = procfs_backlog(vm, rpacket))) {
goto err;
@ -604,7 +604,7 @@ static int _process_procfs_request(struct ikc_scd_packet *rpacket, int *result)
lockedsize += range->end - range->start;
range = next_process_memory_range(vm, range);
}
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock);
cpu_bitmask = &bitmasks[bitmasks_offset];
bitmasks_offset += bitmap_scnprintf(cpu_bitmask,

86
kernel/syscall.c
View File

@ -1710,7 +1710,7 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot,
p2align = PAGE_P2ALIGN;
}
ihk_mc_spinlock_lock_noirq(&thread->vm->memory_range_lock);
ihk_rwspinlock_write_lock_noirq(&thread->vm->memory_range_lock);
if (flags & MAP_FIXED) {
/* clear specified address range */
@ -2024,7 +2024,7 @@ out:
if (ro_vma_mapped) {
(void)set_host_vma(addr, len, PROT_READ | PROT_WRITE | PROT_EXEC, 1/* holding memory_range_lock */);
}
ihk_mc_spinlock_unlock_noirq(&thread->vm->memory_range_lock);
ihk_rwspinlock_write_unlock_noirq(&thread->vm->memory_range_lock);
if (!error && populated_mapping && !((vrflags & VR_PROT_MASK) == VR_PROT_NONE)) {
error = populate_process_memory(thread->vm,
@ -2092,9 +2092,9 @@ SYSCALL_DECLARE(munmap)
goto out;
}
ihk_mc_spinlock_lock_noirq(&thread->vm->memory_range_lock);
ihk_rwspinlock_write_lock_noirq(&thread->vm->memory_range_lock);
error = do_munmap((void *)addr, len, 1/* holding memory_range_lock */);
ihk_mc_spinlock_unlock_noirq(&thread->vm->memory_range_lock);
ihk_rwspinlock_write_unlock_noirq(&thread->vm->memory_range_lock);
out:
dkprintf("[%d]sys_munmap(%lx,%lx): %d\n",
@ -2148,7 +2148,7 @@ SYSCALL_DECLARE(mprotect)
flush_nfo_tlb();
ihk_mc_spinlock_lock_noirq(&thread->vm->memory_range_lock);
ihk_rwspinlock_write_lock_noirq(&thread->vm->memory_range_lock);
first = lookup_process_memory_range(thread->vm, start, start+PAGE_SIZE);
@ -2238,7 +2238,7 @@ out:
/* through */
}
}
ihk_mc_spinlock_unlock_noirq(&thread->vm->memory_range_lock);
ihk_rwspinlock_write_unlock_noirq(&thread->vm->memory_range_lock);
dkprintf("[%d]sys_mprotect(%lx,%lx,%x): %d\n",
ihk_mc_get_processor_id(), start, len0, prot, error);
return error;
@ -2281,11 +2281,11 @@ SYSCALL_DECLARE(brk)
vrflag |= VR_PRIVATE;
vrflag |= VRFLAG_PROT_TO_MAXPROT(vrflag);
old_brk_end_allocated = region->brk_end_allocated;
ihk_mc_spinlock_lock_noirq(&cpu_local_var(current)->vm->memory_range_lock);
ihk_rwspinlock_write_lock_noirq(&cpu_local_var(current)->vm->memory_range_lock);
region->brk_end_allocated =
extend_process_region(cpu_local_var(current)->vm,
region->brk_end_allocated, address, vrflag);
ihk_mc_spinlock_unlock_noirq(&cpu_local_var(current)->vm->memory_range_lock);
ihk_rwspinlock_write_unlock_noirq(&cpu_local_var(current)->vm->memory_range_lock);
if (old_brk_end_allocated == region->brk_end_allocated) {
r = old_brk_end_allocated;
@ -2510,7 +2510,7 @@ static void munmap_all(void)
size_t size;
int error;
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_write_lock_noirq(&vm->memory_range_lock);
next = lookup_process_memory_range(vm, 0, -1);
while ((range = next)) {
next = next_process_memory_range(vm, range);
@ -2524,7 +2524,7 @@ static void munmap_all(void)
/* through */
}
}
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock);
/* free vm_ranges which do_munmap() failed to remove. */
free_process_memory_ranges(thread->vm);
@ -2554,18 +2554,18 @@ static int do_execveat(ihk_mc_user_context_t *ctx, int dirfd,
struct process *proc = thread->proc;
int i;
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_read_lock_noirq(&vm->memory_range_lock);
range = lookup_process_memory_range(vm, (unsigned long)filename,
(unsigned long)filename+1);
if (range == NULL || !(range->flag & VR_PROT_READ)) {
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock);
kprintf("execve(): ERROR: filename is bad address\n");
return -EFAULT;
}
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock);
desc = ihk_mc_alloc_pages(4, IHK_MC_AP_NOWAIT);
if (!desc) {
@ -5143,10 +5143,10 @@ SYSCALL_DECLARE(mincore)
range = NULL;
up = vec;
for (addr = start; addr < end; addr += PAGE_SIZE) {
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_read_lock_noirq(&vm->memory_range_lock);
range = lookup_process_memory_range(vm, addr, addr+1);
if (!range) {
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock);
dkprintf("mincore(0x%lx,0x%lx,%p):lookup failed. ENOMEM\n",
start, len, vec);
return -ENOMEM;
@ -5168,7 +5168,7 @@ SYSCALL_DECLARE(mincore)
value = 0;
}
ihk_mc_spinlock_unlock_noirq(&vm->page_table_lock);
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock);
error = copy_to_user(up, &value, sizeof(value));
if (error) {
@ -5344,7 +5344,7 @@ SYSCALL_DECLARE(madvise)
goto out2;
}
ihk_mc_spinlock_lock_noirq(&thread->vm->memory_range_lock);
ihk_rwspinlock_write_lock_noirq(&thread->vm->memory_range_lock);
/* check contiguous map */
first = NULL;
range = NULL; /* for avoidance of warning */
@ -5515,7 +5515,7 @@ SYSCALL_DECLARE(madvise)
error = 0;
out:
ihk_mc_spinlock_unlock_noirq(&thread->vm->memory_range_lock);
ihk_rwspinlock_write_unlock_noirq(&thread->vm->memory_range_lock);
out2:
dkprintf("[%d]sys_madvise(%lx,%lx,%x): %d\n",
@ -5827,11 +5827,11 @@ SYSCALL_DECLARE(shmat)
return -EACCES;
}
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_write_lock_noirq(&vm->memory_range_lock);
if (addr) {
if (lookup_process_memory_range(vm, addr, addr+len)) {
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock);
shmobj_list_unlock();
memobj_unref(&obj->memobj);
dkprintf("shmat(%#x,%p,%#x):lookup_process_memory_range succeeded. -ENOMEM\n", shmid, shmaddr, shmflg);
@ -5841,7 +5841,7 @@ SYSCALL_DECLARE(shmat)
else {
error = search_free_space(len, obj->pgshift, &addr);
if (error) {
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock);
shmobj_list_unlock();
memobj_unref(&obj->memobj);
dkprintf("shmat(%#x,%p,%#x):search_free_space failed. %d\n", shmid, shmaddr, shmflg, error);
@ -5857,7 +5857,7 @@ SYSCALL_DECLARE(shmat)
if (!(prot & PROT_WRITE)) {
error = set_host_vma(addr, len, PROT_READ | PROT_EXEC, 1/* holding memory_range_lock */);
if (error) {
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock);
shmobj_list_unlock();
memobj_unref(&obj->memobj);
dkprintf("shmat(%#x,%p,%#x):set_host_vma failed. %d\n", shmid, shmaddr, shmflg, error);
@ -5872,13 +5872,13 @@ SYSCALL_DECLARE(shmat)
(void)set_host_vma(addr, len, PROT_READ | PROT_WRITE | PROT_EXEC, 1/* holding memory_range_lock */);
}
memobj_unref(&obj->memobj);
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock);
shmobj_list_unlock();
dkprintf("shmat(%#x,%p,%#x):add_process_memory_range failed. %d\n", shmid, shmaddr, shmflg, error);
return error;
}
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock);
shmobj_list_unlock();
dkprintf("shmat(%#x,%p,%#x): 0x%lx. %d\n", shmid, shmaddr, shmflg, addr);
@ -6165,23 +6165,23 @@ SYSCALL_DECLARE(shmdt)
int error;
dkprintf("shmdt(%p)\n", shmaddr);
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_write_lock_noirq(&vm->memory_range_lock);
range = lookup_process_memory_range(vm, (uintptr_t)shmaddr, (uintptr_t)shmaddr+1);
if (!range || (range->start != (uintptr_t)shmaddr) || !range->memobj
|| !(range->memobj->flags & MF_SHMDT_OK)) {
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock);
dkprintf("shmdt(%p): -EINVAL\n", shmaddr);
return -EINVAL;
}
error = do_munmap((void *)range->start, (range->end - range->start), 1/* holding memory_range_lock */);
if (error) {
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock);
dkprintf("shmdt(%p): %d\n", shmaddr, error);
return error;
}
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock);
dkprintf("shmdt(%p): 0\n", shmaddr);
return 0;
} /* sys_shmdt() */
@ -8094,7 +8094,7 @@ SYSCALL_DECLARE(mlock)
goto out2;
}
ihk_mc_spinlock_lock_noirq(&thread->vm->memory_range_lock);
ihk_rwspinlock_write_lock_noirq(&thread->vm->memory_range_lock);
/* check contiguous map */
first = NULL;
@ -8199,7 +8199,7 @@ SYSCALL_DECLARE(mlock)
error = 0;
out:
ihk_mc_spinlock_unlock_noirq(&thread->vm->memory_range_lock);
ihk_rwspinlock_write_unlock_noirq(&thread->vm->memory_range_lock);
if (!error) {
error = populate_process_memory(thread->vm, (void *)start, len);
@ -8269,7 +8269,7 @@ SYSCALL_DECLARE(munlock)
goto out2;
}
ihk_mc_spinlock_lock_noirq(&thread->vm->memory_range_lock);
ihk_rwspinlock_write_lock_noirq(&thread->vm->memory_range_lock);
/* check contiguous map */
first = NULL;
@ -8374,7 +8374,7 @@ SYSCALL_DECLARE(munlock)
error = 0;
out:
ihk_mc_spinlock_unlock_noirq(&thread->vm->memory_range_lock);
ihk_rwspinlock_write_unlock_noirq(&thread->vm->memory_range_lock);
out2:
dkprintf("[%d]sys_munlock(%lx,%lx): %d\n",
ihk_mc_get_processor_id(), start0, len0, error);
@ -8430,7 +8430,7 @@ SYSCALL_DECLARE(remap_file_pages)
dkprintf("sys_remap_file_pages(%#lx,%#lx,%#x,%#lx,%#x)\n",
start0, size, prot, pgoff, flags);
ihk_mc_spinlock_lock_noirq(&thread->vm->memory_range_lock);
ihk_rwspinlock_write_lock_noirq(&thread->vm->memory_range_lock);
#define PGOFF_LIMIT ((off_t)1 << ((8*sizeof(off_t) - 1) - PAGE_SHIFT))
if ((size <= 0) || (size & (PAGE_SIZE - 1)) || (prot != 0)
|| (PGOFF_LIMIT <= pgoff)
@ -8474,7 +8474,7 @@ SYSCALL_DECLARE(remap_file_pages)
}
error = 0;
out:
ihk_mc_spinlock_unlock_noirq(&thread->vm->memory_range_lock);
ihk_rwspinlock_write_unlock_noirq(&thread->vm->memory_range_lock);
if (need_populate
&& (er = populate_process_memory(
@ -8515,7 +8515,7 @@ SYSCALL_DECLARE(mremap)
dkprintf("sys_mremap(%#lx,%#lx,%#lx,%#x,%#lx)\n",
oldaddr, oldsize0, newsize0, flags, newaddr);
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_write_lock_noirq(&vm->memory_range_lock);
/* check arguments */
if ((oldaddr & ~PAGE_MASK)
@ -8734,7 +8734,7 @@ SYSCALL_DECLARE(mremap)
error = 0;
out:
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock);
if (!error && (lckstart < lckend)) {
error = populate_process_memory(thread->vm, (void *)lckstart, (lckend - lckstart));
if (error) {
@ -8769,7 +8769,7 @@ SYSCALL_DECLARE(msync)
uintptr_t e;
dkprintf("sys_msync(%#lx,%#lx,%#x)\n", start0, len0, flags);
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_read_lock_noirq(&vm->memory_range_lock);
if ((start0 & ~PAGE_MASK)
|| (flags & ~(MS_ASYNC|MS_INVALIDATE|MS_SYNC))
@ -8863,7 +8863,7 @@ SYSCALL_DECLARE(msync)
error = 0;
out:
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock);
dkprintf("sys_msync(%#lx,%#lx,%#x):%d\n", start0, len0, flags, error);
return error;
} /* sys_msync() */
@ -9044,7 +9044,7 @@ SYSCALL_DECLARE(mbind)
}
/* Validate address range */
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_write_lock_noirq(&vm->memory_range_lock);
range = lookup_process_memory_range(vm, addr, addr + len);
if (!range) {
@ -9182,7 +9182,7 @@ mbind_update_only:
error = 0;
unlock_out:
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock);
out:
return error;
} /* sys_mbind() */
@ -9412,18 +9412,18 @@ SYSCALL_DECLARE(get_mempolicy)
if (flags & MPOL_F_ADDR) {
struct vm_range *range;
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_read_lock_noirq(&vm->memory_range_lock);
range = lookup_process_memory_range(vm, addr, addr + 1);
if (!range) {
dkprintf("%s: ERROR: range is invalid\n", __FUNCTION__);
error = -EFAULT;
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock);
goto out;
}
range_policy = vm_range_policy_search(vm, addr);
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock);
}
/* Return policy */

36
kernel/xpmem.c
View File

@ -1130,7 +1130,7 @@ static int xpmem_attach(
if (flags & MAP_FIXED) {
struct vm_range *existing_vmr;
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_read_lock_noirq(&vm->memory_range_lock);
existing_vmr = lookup_process_memory_range(vm, vaddr,
vaddr + size);
@ -1140,13 +1140,13 @@ static int xpmem_attach(
existing_vmr)) {
if (xpmem_is_private_data(existing_vmr)) {
ret = -EINVAL;
ihk_mc_spinlock_unlock_noirq(
ihk_rwspinlock_read_unlock_noirq(
&vm->memory_range_lock);
goto out_2;
}
}
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock);
}
flags |= MAP_ANONYMOUS;
@ -1199,18 +1199,18 @@ static int xpmem_detach(
XPMEM_DEBUG("call: at_vaddr=0x%lx", at_vaddr);
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_write_lock_noirq(&vm->memory_range_lock);
range = lookup_process_memory_range(vm, at_vaddr, at_vaddr + 1);
if (!range || range->start > at_vaddr) {
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock);
return 0;
}
att = (struct xpmem_attachment *)range->private_data;
if (att == NULL) {
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock);
return -EINVAL;
}
@ -1220,7 +1220,7 @@ static int xpmem_detach(
if (att->flags & XPMEM_FLAG_DESTROYING) {
mcs_rwlock_writer_unlock(&att->at_lock, &at_lock);
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock);
xpmem_att_deref(att);
return 0;
}
@ -1233,7 +1233,7 @@ static int xpmem_detach(
att->flags &= ~XPMEM_FLAG_DESTROYING;
xpmem_ap_deref(ap);
mcs_rwlock_writer_unlock(&att->at_lock, &at_lock);
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock);
xpmem_att_deref(att);
return -EACCES;
}
@ -1253,7 +1253,7 @@ static int xpmem_detach(
ekprintf("%s: ERROR: xpmem_vm_munmap() failed %d\n",
__FUNCTION__, ret);
}
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock);
DBUG_ON(ret != 0);
att->flags &= ~XPMEM_FLAG_VALIDPTEs;
@ -1428,7 +1428,7 @@ static void xpmem_detach_att(
att->flags |= XPMEM_FLAG_DESTROYING;
vm = att->vm;
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_read_lock_noirq(&vm->memory_range_lock);
range = lookup_process_memory_range(vm,
att->at_vaddr, att->at_vaddr + 1);
@ -1438,7 +1438,7 @@ static void xpmem_detach_att(
list_del_init(&att->att_list);
ihk_mc_spinlock_unlock_noirq(&ap->lock);
mcs_rwlock_writer_unlock(&att->at_lock, &at_lock);
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock);
xpmem_att_destroyable(att);
XPMEM_DEBUG("return: range=%p");
return;
@ -1473,7 +1473,7 @@ static void xpmem_detach_att(
__FUNCTION__, ret);
}
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock);
xpmem_att_destroyable(att);
@ -1576,7 +1576,7 @@ static void xpmem_clear_PTEs_of_att(
XPMEM_DEBUG("call: att=0x%p, start=0x%lx, end=0x%lx",
att, start, end);
ihk_mc_spinlock_lock_noirq(&att->vm->memory_range_lock);
ihk_rwspinlock_read_lock_noirq(&att->vm->memory_range_lock);
mcs_rwlock_writer_lock(&att->at_lock, &at_lock);
if (att->flags & XPMEM_FLAG_VALIDPTEs) {
@ -1636,7 +1636,7 @@ static void xpmem_clear_PTEs_of_att(
}
out:
mcs_rwlock_writer_unlock(&att->at_lock, &at_lock);
ihk_mc_spinlock_unlock_noirq(&att->vm->memory_range_lock);
ihk_rwspinlock_read_unlock_noirq(&att->vm->memory_range_lock);
XPMEM_DEBUG("return: ");
}
@ -2009,7 +2009,7 @@ static int xpmem_remap_pte(
vmr, vaddr, reason, seg->segid, seg_vaddr);
if (is_remote_vm(seg_tg->vm)) {
ihk_mc_spinlock_lock_noirq(&seg_tg->vm->memory_range_lock);
ihk_rwspinlock_read_lock_noirq(&seg_tg->vm->memory_range_lock);
}
seg_vmr = lookup_process_memory_range(seg_tg->vm, seg_vaddr,
@ -2074,7 +2074,7 @@ static int xpmem_remap_pte(
out:
if (is_remote_vm(seg_tg->vm)) {
ihk_mc_spinlock_unlock_noirq(&seg_tg->vm->memory_range_lock);
ihk_rwspinlock_read_unlock_noirq(&seg_tg->vm->memory_range_lock);
}
XPMEM_DEBUG("return: ret=%d", ret);
@ -2149,7 +2149,7 @@ static int xpmem_pin_page(
XPMEM_DEBUG("call: tgid=%d, vaddr=0x%lx", tg->tgid, vaddr);
if (is_remote_vm(src_vm)) {
ihk_mc_spinlock_lock_noirq(&src_vm->memory_range_lock);
ihk_rwspinlock_read_lock_noirq(&src_vm->memory_range_lock);
}
range = lookup_process_memory_range(src_vm, vaddr, vaddr + 1);
@ -2167,7 +2167,7 @@ static int xpmem_pin_page(
ihk_atomic_inc(&tg->n_pinned);
out:
if (is_remote_vm(src_vm)) {
ihk_mc_spinlock_unlock_noirq(&src_vm->memory_range_lock);
ihk_rwspinlock_read_unlock_noirq(&src_vm->memory_range_lock);
}
XPMEM_DEBUG("return: ret=%d", ret);