From d5aafca1ae10aa17f927400cf6b2a7260374f8cb Mon Sep 17 00:00:00 2001 From: Balazs Gerofi Date: Thu, 30 Apr 2020 13:32:32 +0900 Subject: [PATCH] VM: use RW spinlock for vm_range_lock Change-Id: Id4654084207d55bf77cc9f8b42795e0f9873cfa0 --- arch/arm64/kernel/syscall.c | 12 ++--- arch/x86_64/kernel/syscall.c | 12 ++--- kernel/include/process.h | 2 +- kernel/process.c | 26 +++++------ kernel/procfs.c | 14 +++--- kernel/syscall.c | 86 ++++++++++++++++++------------------ kernel/xpmem.c | 36 +++++++-------- 7 files changed, 94 insertions(+), 94 deletions(-) diff --git a/arch/arm64/kernel/syscall.c b/arch/arm64/kernel/syscall.c index 73453fab..c494df1f 100644 --- a/arch/arm64/kernel/syscall.c +++ b/arch/arm64/kernel/syscall.c @@ -2151,7 +2151,7 @@ int do_process_vm_read_writev(int pid, } /* Check if parameters are okay */ - ihk_mc_spinlock_lock_noirq(<hread->vm->memory_range_lock); + ihk_rwspinlock_read_lock_noirq(<hread->vm->memory_range_lock); range = lookup_process_memory_range(lthread->vm, (uintptr_t)local_iov, @@ -2173,7 +2173,7 @@ int do_process_vm_read_writev(int pid, ret = 0; arg_out: - ihk_mc_spinlock_unlock_noirq(<hread->vm->memory_range_lock); + ihk_rwspinlock_read_unlock_noirq(<hread->vm->memory_range_lock); if (ret != 0) { goto out; @@ -2242,7 +2242,7 @@ arg_out: if (pli != li) { struct vm_range *range; - ihk_mc_spinlock_lock_noirq(<hread->vm->memory_range_lock); + ihk_rwspinlock_read_lock_noirq(<hread->vm->memory_range_lock); /* Is base valid? */ range = lookup_process_memory_range(lthread->vm, @@ -2272,7 +2272,7 @@ arg_out: ret = 0; pli_out: - ihk_mc_spinlock_unlock_noirq(<hread->vm->memory_range_lock); + ihk_rwspinlock_read_unlock_noirq(<hread->vm->memory_range_lock); if (ret != 0) { goto out; @@ -2285,7 +2285,7 @@ pli_out: if (pri != ri) { struct vm_range *range; - ihk_mc_spinlock_lock_noirq(&rvm->memory_range_lock); + ihk_rwspinlock_read_lock_noirq(&rvm->memory_range_lock); /* Is base valid? */ range = lookup_process_memory_range(rvm, @@ -2315,7 +2315,7 @@ pli_out: ret = 0; pri_out: - ihk_mc_spinlock_unlock_noirq(&rvm->memory_range_lock); + ihk_rwspinlock_read_unlock_noirq(&rvm->memory_range_lock); if (ret != 0) { goto out; diff --git a/arch/x86_64/kernel/syscall.c b/arch/x86_64/kernel/syscall.c index 33c69e18..588bebfe 100644 --- a/arch/x86_64/kernel/syscall.c +++ b/arch/x86_64/kernel/syscall.c @@ -2290,7 +2290,7 @@ int do_process_vm_read_writev(int pid, } /* Check if parameters are okay */ - ihk_mc_spinlock_lock_noirq(<hread->vm->memory_range_lock); + ihk_rwspinlock_read_lock_noirq(<hread->vm->memory_range_lock); range = lookup_process_memory_range(lthread->vm, (uintptr_t)local_iov, @@ -2312,7 +2312,7 @@ int do_process_vm_read_writev(int pid, ret = 0; arg_out: - ihk_mc_spinlock_unlock_noirq(<hread->vm->memory_range_lock); + ihk_rwspinlock_read_unlock_noirq(<hread->vm->memory_range_lock); if (ret != 0) { goto out; @@ -2381,7 +2381,7 @@ arg_out: if (pli != li) { struct vm_range *range; - ihk_mc_spinlock_lock_noirq(<hread->vm->memory_range_lock); + ihk_rwspinlock_read_lock_noirq(<hread->vm->memory_range_lock); /* Is base valid? */ range = lookup_process_memory_range(lthread->vm, @@ -2411,7 +2411,7 @@ arg_out: ret = 0; pli_out: - ihk_mc_spinlock_unlock_noirq(<hread->vm->memory_range_lock); + ihk_rwspinlock_read_unlock_noirq(<hread->vm->memory_range_lock); if (ret != 0) { goto out; @@ -2424,7 +2424,7 @@ pli_out: if (pri != ri) { struct vm_range *range; - ihk_mc_spinlock_lock_noirq(&rvm->memory_range_lock); + ihk_rwspinlock_read_lock_noirq(&rvm->memory_range_lock); /* Is base valid? */ range = lookup_process_memory_range(rvm, @@ -2454,7 +2454,7 @@ pli_out: ret = 0; pri_out: - ihk_mc_spinlock_unlock_noirq(&rvm->memory_range_lock); + ihk_rwspinlock_read_unlock_noirq(&rvm->memory_range_lock); if (ret != 0) { goto out; diff --git a/kernel/include/process.h b/kernel/include/process.h index 8b8c894a..31bac2d9 100644 --- a/kernel/include/process.h +++ b/kernel/include/process.h @@ -749,7 +749,7 @@ struct process_vm { void *vvar_addr; ihk_spinlock_t page_table_lock; - ihk_spinlock_t memory_range_lock; + ihk_rwspinlock_t memory_range_lock; // to protect the followings: // 1. addition of process "memory range" (extend_process_region, add_process_memory_range) // 2. addition of process page table (allocate_pages, update_process_page_table) diff --git a/kernel/process.c b/kernel/process.c index 0c144dd4..16247269 100644 --- a/kernel/process.c +++ b/kernel/process.c @@ -243,7 +243,7 @@ static int init_process_vm(struct process *owner, struct address_space *asp, struct process_vm *vm) { int i; - ihk_mc_spinlock_init(&vm->memory_range_lock); + ihk_rwspinlock_init(&vm->memory_range_lock); ihk_mc_spinlock_init(&vm->page_table_lock); ihk_atomic_set(&vm->refcount, 1); @@ -792,7 +792,7 @@ static int copy_user_ranges(struct process_vm *vm, struct process_vm *orgvm) struct vm_range *last_insert; struct copy_args args; - ihk_mc_spinlock_lock_noirq(&orgvm->memory_range_lock); + ihk_rwspinlock_read_lock_noirq(&orgvm->memory_range_lock); /* Iterate original process' vm_range list and take a copy one-by-one */ last_insert = NULL; @@ -854,7 +854,7 @@ static int copy_user_ranges(struct process_vm *vm, struct process_vm *orgvm) // memory_stat_rss_add() is called in child-node, i.e. copy_user_pte() } - ihk_mc_spinlock_unlock_noirq(&orgvm->memory_range_lock); + ihk_rwspinlock_read_unlock_noirq(&orgvm->memory_range_lock); return 0; @@ -881,7 +881,7 @@ err_rollback: } } - ihk_mc_spinlock_unlock_noirq(&orgvm->memory_range_lock); + ihk_rwspinlock_read_unlock_noirq(&orgvm->memory_range_lock); return -1; } @@ -2190,7 +2190,7 @@ static int do_page_fault_process_vm(struct process_vm *vm, void *fault_addr0, ui if (thread->vm->is_memory_range_lock_taken == -1 || thread->vm->is_memory_range_lock_taken != ihk_mc_get_processor_id()) { - ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_read_lock_noirq(&vm->memory_range_lock); locked = 1; } else { dkprintf("%s: INFO: skip locking of memory_range_lock,pid=%d,tid=%d\n", @@ -2305,7 +2305,7 @@ static int do_page_fault_process_vm(struct process_vm *vm, void *fault_addr0, ui error = 0; out: if (locked) { - ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock); } dkprintf("[%d]do_page_fault_process_vm(%p,%lx,%lx): %d\n", ihk_mc_get_processor_id(), vm, fault_addr0, @@ -2609,7 +2609,7 @@ void flush_process_memory(struct process_vm *vm) int error; dkprintf("flush_process_memory(%p)\n", vm); - ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_write_lock_noirq(&vm->memory_range_lock); /* Let concurrent page faults know the VM will be gone */ vm->exiting = 1; while ((node = next)) { @@ -2627,7 +2627,7 @@ void flush_process_memory(struct process_vm *vm) } } } - ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock); dkprintf("flush_process_memory(%p):\n", vm); return; } @@ -2642,7 +2642,7 @@ void free_process_memory_ranges(struct process_vm *vm) return; } - ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_write_lock_noirq(&vm->memory_range_lock); while ((node = next)) { range = rb_entry(node, struct vm_range, vm_rb_node); next = rb_next(node); @@ -2655,7 +2655,7 @@ void free_process_memory_ranges(struct process_vm *vm) /* through */ } } - ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock); } static void free_thread_pages(struct thread *thread) @@ -2745,7 +2745,7 @@ free_all_process_memory_range(struct process_vm *vm) struct rb_node *node, *next = rb_first(&vm->vm_range_tree); int error; - ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_write_lock_noirq(&vm->memory_range_lock); while ((node = next)) { range = rb_entry(node, struct vm_range, vm_rb_node); next = rb_next(node); @@ -2758,7 +2758,7 @@ free_all_process_memory_range(struct process_vm *vm) /* through */ } } - ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock); } void @@ -3189,7 +3189,7 @@ void sched_init(void) &idle_thread->proc->children_list); ihk_mc_init_context(&idle_thread->ctx, NULL, idle); - ihk_mc_spinlock_init(&idle_thread->vm->memory_range_lock); + ihk_rwspinlock_init(&idle_thread->vm->memory_range_lock); idle_thread->vm->vm_range_tree = RB_ROOT; idle_thread->vm->vm_range_numa_policy_tree = RB_ROOT; idle_thread->proc->pid = 0; diff --git a/kernel/procfs.c b/kernel/procfs.c index 0b1b1c80..a7297b38 100644 --- a/kernel/procfs.c +++ b/kernel/procfs.c @@ -459,7 +459,7 @@ static int _process_procfs_request(struct ikc_scd_packet *rpacket, int *result) if (strcmp(p, "maps") == 0) { struct vm_range *range; - if (!ihk_mc_spinlock_trylock_noirq(&vm->memory_range_lock)) { + if (!ihk_rwspinlock_read_trylock_noirq(&vm->memory_range_lock)) { if (!result) { if ((err = procfs_backlog(vm, rpacket))) { goto err; @@ -505,14 +505,14 @@ static int _process_procfs_request(struct ikc_scd_packet *rpacket, int *result) if (ans < 0 || ans > count || buf_add(&buf_top, &buf_cur, buf, ans) < 0) { - ihk_mc_spinlock_unlock_noirq( + ihk_rwspinlock_read_unlock_noirq( &vm->memory_range_lock); goto err; } range = next_process_memory_range(vm, range); } - ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock); ans = 0; goto end; @@ -535,7 +535,7 @@ static int _process_procfs_request(struct ikc_scd_packet *rpacket, int *result) start = (offset / sizeof(uint64_t)) << PAGE_SHIFT; end = start + ((count / sizeof(uint64_t)) << PAGE_SHIFT); - if (!ihk_mc_spinlock_trylock_noirq(&vm->memory_range_lock)) { + if (!ihk_rwspinlock_read_trylock_noirq(&vm->memory_range_lock)) { if (!result) { if ((err = procfs_backlog(vm, rpacket))) { goto err; @@ -555,7 +555,7 @@ static int _process_procfs_request(struct ikc_scd_packet *rpacket, int *result) ++_buf; } - ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock); dprintf("/proc/pagemap: 0x%lx - 0x%lx, count: %d\n", start, end, count); @@ -587,7 +587,7 @@ static int _process_procfs_request(struct ikc_scd_packet *rpacket, int *result) goto err; } - if (!ihk_mc_spinlock_trylock_noirq(&vm->memory_range_lock)) { + if (!ihk_rwspinlock_read_trylock_noirq(&vm->memory_range_lock)) { if (!result) { if ((err = procfs_backlog(vm, rpacket))) { goto err; @@ -604,7 +604,7 @@ static int _process_procfs_request(struct ikc_scd_packet *rpacket, int *result) lockedsize += range->end - range->start; range = next_process_memory_range(vm, range); } - ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock); cpu_bitmask = &bitmasks[bitmasks_offset]; bitmasks_offset += bitmap_scnprintf(cpu_bitmask, diff --git a/kernel/syscall.c b/kernel/syscall.c index d59d9955..eb39e71e 100644 --- a/kernel/syscall.c +++ b/kernel/syscall.c @@ -1710,7 +1710,7 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot, p2align = PAGE_P2ALIGN; } - ihk_mc_spinlock_lock_noirq(&thread->vm->memory_range_lock); + ihk_rwspinlock_write_lock_noirq(&thread->vm->memory_range_lock); if (flags & MAP_FIXED) { /* clear specified address range */ @@ -2024,7 +2024,7 @@ out: if (ro_vma_mapped) { (void)set_host_vma(addr, len, PROT_READ | PROT_WRITE | PROT_EXEC, 1/* holding memory_range_lock */); } - ihk_mc_spinlock_unlock_noirq(&thread->vm->memory_range_lock); + ihk_rwspinlock_write_unlock_noirq(&thread->vm->memory_range_lock); if (!error && populated_mapping && !((vrflags & VR_PROT_MASK) == VR_PROT_NONE)) { error = populate_process_memory(thread->vm, @@ -2092,9 +2092,9 @@ SYSCALL_DECLARE(munmap) goto out; } - ihk_mc_spinlock_lock_noirq(&thread->vm->memory_range_lock); + ihk_rwspinlock_write_lock_noirq(&thread->vm->memory_range_lock); error = do_munmap((void *)addr, len, 1/* holding memory_range_lock */); - ihk_mc_spinlock_unlock_noirq(&thread->vm->memory_range_lock); + ihk_rwspinlock_write_unlock_noirq(&thread->vm->memory_range_lock); out: dkprintf("[%d]sys_munmap(%lx,%lx): %d\n", @@ -2148,7 +2148,7 @@ SYSCALL_DECLARE(mprotect) flush_nfo_tlb(); - ihk_mc_spinlock_lock_noirq(&thread->vm->memory_range_lock); + ihk_rwspinlock_write_lock_noirq(&thread->vm->memory_range_lock); first = lookup_process_memory_range(thread->vm, start, start+PAGE_SIZE); @@ -2238,7 +2238,7 @@ out: /* through */ } } - ihk_mc_spinlock_unlock_noirq(&thread->vm->memory_range_lock); + ihk_rwspinlock_write_unlock_noirq(&thread->vm->memory_range_lock); dkprintf("[%d]sys_mprotect(%lx,%lx,%x): %d\n", ihk_mc_get_processor_id(), start, len0, prot, error); return error; @@ -2281,11 +2281,11 @@ SYSCALL_DECLARE(brk) vrflag |= VR_PRIVATE; vrflag |= VRFLAG_PROT_TO_MAXPROT(vrflag); old_brk_end_allocated = region->brk_end_allocated; - ihk_mc_spinlock_lock_noirq(&cpu_local_var(current)->vm->memory_range_lock); + ihk_rwspinlock_write_lock_noirq(&cpu_local_var(current)->vm->memory_range_lock); region->brk_end_allocated = extend_process_region(cpu_local_var(current)->vm, region->brk_end_allocated, address, vrflag); - ihk_mc_spinlock_unlock_noirq(&cpu_local_var(current)->vm->memory_range_lock); + ihk_rwspinlock_write_unlock_noirq(&cpu_local_var(current)->vm->memory_range_lock); if (old_brk_end_allocated == region->brk_end_allocated) { r = old_brk_end_allocated; @@ -2510,7 +2510,7 @@ static void munmap_all(void) size_t size; int error; - ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_write_lock_noirq(&vm->memory_range_lock); next = lookup_process_memory_range(vm, 0, -1); while ((range = next)) { next = next_process_memory_range(vm, range); @@ -2524,7 +2524,7 @@ static void munmap_all(void) /* through */ } } - ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock); /* free vm_ranges which do_munmap() failed to remove. */ free_process_memory_ranges(thread->vm); @@ -2554,18 +2554,18 @@ static int do_execveat(ihk_mc_user_context_t *ctx, int dirfd, struct process *proc = thread->proc; int i; - ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_read_lock_noirq(&vm->memory_range_lock); range = lookup_process_memory_range(vm, (unsigned long)filename, (unsigned long)filename+1); if (range == NULL || !(range->flag & VR_PROT_READ)) { - ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock); kprintf("execve(): ERROR: filename is bad address\n"); return -EFAULT; } - ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock); desc = ihk_mc_alloc_pages(4, IHK_MC_AP_NOWAIT); if (!desc) { @@ -5143,10 +5143,10 @@ SYSCALL_DECLARE(mincore) range = NULL; up = vec; for (addr = start; addr < end; addr += PAGE_SIZE) { - ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_read_lock_noirq(&vm->memory_range_lock); range = lookup_process_memory_range(vm, addr, addr+1); if (!range) { - ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock); dkprintf("mincore(0x%lx,0x%lx,%p):lookup failed. ENOMEM\n", start, len, vec); return -ENOMEM; @@ -5168,7 +5168,7 @@ SYSCALL_DECLARE(mincore) value = 0; } ihk_mc_spinlock_unlock_noirq(&vm->page_table_lock); - ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock); error = copy_to_user(up, &value, sizeof(value)); if (error) { @@ -5344,7 +5344,7 @@ SYSCALL_DECLARE(madvise) goto out2; } - ihk_mc_spinlock_lock_noirq(&thread->vm->memory_range_lock); + ihk_rwspinlock_write_lock_noirq(&thread->vm->memory_range_lock); /* check contiguous map */ first = NULL; range = NULL; /* for avoidance of warning */ @@ -5515,7 +5515,7 @@ SYSCALL_DECLARE(madvise) error = 0; out: - ihk_mc_spinlock_unlock_noirq(&thread->vm->memory_range_lock); + ihk_rwspinlock_write_unlock_noirq(&thread->vm->memory_range_lock); out2: dkprintf("[%d]sys_madvise(%lx,%lx,%x): %d\n", @@ -5827,11 +5827,11 @@ SYSCALL_DECLARE(shmat) return -EACCES; } - ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_write_lock_noirq(&vm->memory_range_lock); if (addr) { if (lookup_process_memory_range(vm, addr, addr+len)) { - ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock); shmobj_list_unlock(); memobj_unref(&obj->memobj); dkprintf("shmat(%#x,%p,%#x):lookup_process_memory_range succeeded. -ENOMEM\n", shmid, shmaddr, shmflg); @@ -5841,7 +5841,7 @@ SYSCALL_DECLARE(shmat) else { error = search_free_space(len, obj->pgshift, &addr); if (error) { - ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock); shmobj_list_unlock(); memobj_unref(&obj->memobj); dkprintf("shmat(%#x,%p,%#x):search_free_space failed. %d\n", shmid, shmaddr, shmflg, error); @@ -5857,7 +5857,7 @@ SYSCALL_DECLARE(shmat) if (!(prot & PROT_WRITE)) { error = set_host_vma(addr, len, PROT_READ | PROT_EXEC, 1/* holding memory_range_lock */); if (error) { - ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock); shmobj_list_unlock(); memobj_unref(&obj->memobj); dkprintf("shmat(%#x,%p,%#x):set_host_vma failed. %d\n", shmid, shmaddr, shmflg, error); @@ -5872,13 +5872,13 @@ SYSCALL_DECLARE(shmat) (void)set_host_vma(addr, len, PROT_READ | PROT_WRITE | PROT_EXEC, 1/* holding memory_range_lock */); } memobj_unref(&obj->memobj); - ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock); shmobj_list_unlock(); dkprintf("shmat(%#x,%p,%#x):add_process_memory_range failed. %d\n", shmid, shmaddr, shmflg, error); return error; } - ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock); shmobj_list_unlock(); dkprintf("shmat(%#x,%p,%#x): 0x%lx. %d\n", shmid, shmaddr, shmflg, addr); @@ -6165,23 +6165,23 @@ SYSCALL_DECLARE(shmdt) int error; dkprintf("shmdt(%p)\n", shmaddr); - ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_write_lock_noirq(&vm->memory_range_lock); range = lookup_process_memory_range(vm, (uintptr_t)shmaddr, (uintptr_t)shmaddr+1); if (!range || (range->start != (uintptr_t)shmaddr) || !range->memobj || !(range->memobj->flags & MF_SHMDT_OK)) { - ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock); dkprintf("shmdt(%p): -EINVAL\n", shmaddr); return -EINVAL; } error = do_munmap((void *)range->start, (range->end - range->start), 1/* holding memory_range_lock */); if (error) { - ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock); dkprintf("shmdt(%p): %d\n", shmaddr, error); return error; } - ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock); dkprintf("shmdt(%p): 0\n", shmaddr); return 0; } /* sys_shmdt() */ @@ -8094,7 +8094,7 @@ SYSCALL_DECLARE(mlock) goto out2; } - ihk_mc_spinlock_lock_noirq(&thread->vm->memory_range_lock); + ihk_rwspinlock_write_lock_noirq(&thread->vm->memory_range_lock); /* check contiguous map */ first = NULL; @@ -8199,7 +8199,7 @@ SYSCALL_DECLARE(mlock) error = 0; out: - ihk_mc_spinlock_unlock_noirq(&thread->vm->memory_range_lock); + ihk_rwspinlock_write_unlock_noirq(&thread->vm->memory_range_lock); if (!error) { error = populate_process_memory(thread->vm, (void *)start, len); @@ -8269,7 +8269,7 @@ SYSCALL_DECLARE(munlock) goto out2; } - ihk_mc_spinlock_lock_noirq(&thread->vm->memory_range_lock); + ihk_rwspinlock_write_lock_noirq(&thread->vm->memory_range_lock); /* check contiguous map */ first = NULL; @@ -8374,7 +8374,7 @@ SYSCALL_DECLARE(munlock) error = 0; out: - ihk_mc_spinlock_unlock_noirq(&thread->vm->memory_range_lock); + ihk_rwspinlock_write_unlock_noirq(&thread->vm->memory_range_lock); out2: dkprintf("[%d]sys_munlock(%lx,%lx): %d\n", ihk_mc_get_processor_id(), start0, len0, error); @@ -8430,7 +8430,7 @@ SYSCALL_DECLARE(remap_file_pages) dkprintf("sys_remap_file_pages(%#lx,%#lx,%#x,%#lx,%#x)\n", start0, size, prot, pgoff, flags); - ihk_mc_spinlock_lock_noirq(&thread->vm->memory_range_lock); + ihk_rwspinlock_write_lock_noirq(&thread->vm->memory_range_lock); #define PGOFF_LIMIT ((off_t)1 << ((8*sizeof(off_t) - 1) - PAGE_SHIFT)) if ((size <= 0) || (size & (PAGE_SIZE - 1)) || (prot != 0) || (PGOFF_LIMIT <= pgoff) @@ -8474,7 +8474,7 @@ SYSCALL_DECLARE(remap_file_pages) } error = 0; out: - ihk_mc_spinlock_unlock_noirq(&thread->vm->memory_range_lock); + ihk_rwspinlock_write_unlock_noirq(&thread->vm->memory_range_lock); if (need_populate && (er = populate_process_memory( @@ -8515,7 +8515,7 @@ SYSCALL_DECLARE(mremap) dkprintf("sys_mremap(%#lx,%#lx,%#lx,%#x,%#lx)\n", oldaddr, oldsize0, newsize0, flags, newaddr); - ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_write_lock_noirq(&vm->memory_range_lock); /* check arguments */ if ((oldaddr & ~PAGE_MASK) @@ -8734,7 +8734,7 @@ SYSCALL_DECLARE(mremap) error = 0; out: - ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock); if (!error && (lckstart < lckend)) { error = populate_process_memory(thread->vm, (void *)lckstart, (lckend - lckstart)); if (error) { @@ -8769,7 +8769,7 @@ SYSCALL_DECLARE(msync) uintptr_t e; dkprintf("sys_msync(%#lx,%#lx,%#x)\n", start0, len0, flags); - ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_read_lock_noirq(&vm->memory_range_lock); if ((start0 & ~PAGE_MASK) || (flags & ~(MS_ASYNC|MS_INVALIDATE|MS_SYNC)) @@ -8863,7 +8863,7 @@ SYSCALL_DECLARE(msync) error = 0; out: - ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock); dkprintf("sys_msync(%#lx,%#lx,%#x):%d\n", start0, len0, flags, error); return error; } /* sys_msync() */ @@ -9044,7 +9044,7 @@ SYSCALL_DECLARE(mbind) } /* Validate address range */ - ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_write_lock_noirq(&vm->memory_range_lock); range = lookup_process_memory_range(vm, addr, addr + len); if (!range) { @@ -9182,7 +9182,7 @@ mbind_update_only: error = 0; unlock_out: - ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock); out: return error; } /* sys_mbind() */ @@ -9412,18 +9412,18 @@ SYSCALL_DECLARE(get_mempolicy) if (flags & MPOL_F_ADDR) { struct vm_range *range; - ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_read_lock_noirq(&vm->memory_range_lock); range = lookup_process_memory_range(vm, addr, addr + 1); if (!range) { dkprintf("%s: ERROR: range is invalid\n", __FUNCTION__); error = -EFAULT; - ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock); goto out; } range_policy = vm_range_policy_search(vm, addr); - ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock); } /* Return policy */ diff --git a/kernel/xpmem.c b/kernel/xpmem.c index 5d32c196..fc437d3f 100644 --- a/kernel/xpmem.c +++ b/kernel/xpmem.c @@ -1130,7 +1130,7 @@ static int xpmem_attach( if (flags & MAP_FIXED) { struct vm_range *existing_vmr; - ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_read_lock_noirq(&vm->memory_range_lock); existing_vmr = lookup_process_memory_range(vm, vaddr, vaddr + size); @@ -1140,13 +1140,13 @@ static int xpmem_attach( existing_vmr)) { if (xpmem_is_private_data(existing_vmr)) { ret = -EINVAL; - ihk_mc_spinlock_unlock_noirq( + ihk_rwspinlock_read_unlock_noirq( &vm->memory_range_lock); goto out_2; } } - ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock); } flags |= MAP_ANONYMOUS; @@ -1199,18 +1199,18 @@ static int xpmem_detach( XPMEM_DEBUG("call: at_vaddr=0x%lx", at_vaddr); - ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_write_lock_noirq(&vm->memory_range_lock); range = lookup_process_memory_range(vm, at_vaddr, at_vaddr + 1); if (!range || range->start > at_vaddr) { - ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock); return 0; } att = (struct xpmem_attachment *)range->private_data; if (att == NULL) { - ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock); return -EINVAL; } @@ -1220,7 +1220,7 @@ static int xpmem_detach( if (att->flags & XPMEM_FLAG_DESTROYING) { mcs_rwlock_writer_unlock(&att->at_lock, &at_lock); - ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock); xpmem_att_deref(att); return 0; } @@ -1233,7 +1233,7 @@ static int xpmem_detach( att->flags &= ~XPMEM_FLAG_DESTROYING; xpmem_ap_deref(ap); mcs_rwlock_writer_unlock(&att->at_lock, &at_lock); - ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock); xpmem_att_deref(att); return -EACCES; } @@ -1253,7 +1253,7 @@ static int xpmem_detach( ekprintf("%s: ERROR: xpmem_vm_munmap() failed %d\n", __FUNCTION__, ret); } - ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock); DBUG_ON(ret != 0); att->flags &= ~XPMEM_FLAG_VALIDPTEs; @@ -1428,7 +1428,7 @@ static void xpmem_detach_att( att->flags |= XPMEM_FLAG_DESTROYING; vm = att->vm; - ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_read_lock_noirq(&vm->memory_range_lock); range = lookup_process_memory_range(vm, att->at_vaddr, att->at_vaddr + 1); @@ -1438,7 +1438,7 @@ static void xpmem_detach_att( list_del_init(&att->att_list); ihk_mc_spinlock_unlock_noirq(&ap->lock); mcs_rwlock_writer_unlock(&att->at_lock, &at_lock); - ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock); xpmem_att_destroyable(att); XPMEM_DEBUG("return: range=%p"); return; @@ -1473,7 +1473,7 @@ static void xpmem_detach_att( __FUNCTION__, ret); } - ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock); + ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock); xpmem_att_destroyable(att); @@ -1576,7 +1576,7 @@ static void xpmem_clear_PTEs_of_att( XPMEM_DEBUG("call: att=0x%p, start=0x%lx, end=0x%lx", att, start, end); - ihk_mc_spinlock_lock_noirq(&att->vm->memory_range_lock); + ihk_rwspinlock_read_lock_noirq(&att->vm->memory_range_lock); mcs_rwlock_writer_lock(&att->at_lock, &at_lock); if (att->flags & XPMEM_FLAG_VALIDPTEs) { @@ -1636,7 +1636,7 @@ static void xpmem_clear_PTEs_of_att( } out: mcs_rwlock_writer_unlock(&att->at_lock, &at_lock); - ihk_mc_spinlock_unlock_noirq(&att->vm->memory_range_lock); + ihk_rwspinlock_read_unlock_noirq(&att->vm->memory_range_lock); XPMEM_DEBUG("return: "); } @@ -2009,7 +2009,7 @@ static int xpmem_remap_pte( vmr, vaddr, reason, seg->segid, seg_vaddr); if (is_remote_vm(seg_tg->vm)) { - ihk_mc_spinlock_lock_noirq(&seg_tg->vm->memory_range_lock); + ihk_rwspinlock_read_lock_noirq(&seg_tg->vm->memory_range_lock); } seg_vmr = lookup_process_memory_range(seg_tg->vm, seg_vaddr, @@ -2074,7 +2074,7 @@ static int xpmem_remap_pte( out: if (is_remote_vm(seg_tg->vm)) { - ihk_mc_spinlock_unlock_noirq(&seg_tg->vm->memory_range_lock); + ihk_rwspinlock_read_unlock_noirq(&seg_tg->vm->memory_range_lock); } XPMEM_DEBUG("return: ret=%d", ret); @@ -2149,7 +2149,7 @@ static int xpmem_pin_page( XPMEM_DEBUG("call: tgid=%d, vaddr=0x%lx", tg->tgid, vaddr); if (is_remote_vm(src_vm)) { - ihk_mc_spinlock_lock_noirq(&src_vm->memory_range_lock); + ihk_rwspinlock_read_lock_noirq(&src_vm->memory_range_lock); } range = lookup_process_memory_range(src_vm, vaddr, vaddr + 1); @@ -2167,7 +2167,7 @@ static int xpmem_pin_page( ihk_atomic_inc(&tg->n_pinned); out: if (is_remote_vm(src_vm)) { - ihk_mc_spinlock_unlock_noirq(&src_vm->memory_range_lock); + ihk_rwspinlock_read_unlock_noirq(&src_vm->memory_range_lock); } XPMEM_DEBUG("return: ret=%d", ret);