diff --git a/kernel/include/xpmem_private.h b/kernel/include/xpmem_private.h
index 60a98dc0..c47498e3 100644
--- a/kernel/include/xpmem_private.h
+++ b/kernel/include/xpmem_private.h
@@ -331,6 +331,7 @@ static void xpmem_ap_deref(struct xpmem_access_permit *ap);
 static void xpmem_att_deref(struct xpmem_attachment *att);
 static int xpmem_validate_access(struct xpmem_access_permit *, off_t, size_t,
 	int, unsigned long *);
+static int is_remote_vm(struct process_vm *vm);
 
 /*
  * Inlines that mark an internal driver structure as being destroyable or not.
diff --git a/kernel/xpmem.c b/kernel/xpmem.c
index af17341a..d1d078a5 100644
--- a/kernel/xpmem.c
+++ b/kernel/xpmem.c
@@ -1119,16 +1119,18 @@ static int xpmem_attach(
 		existing_vmr = lookup_process_memory_range(vm, vaddr, 
 			vaddr + size);
 
-		ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock);
-
 		for (; existing_vmr && existing_vmr->start < vaddr + size;
 			existing_vmr = next_process_memory_range(vm, 
 			existing_vmr)) {
 			if (xpmem_is_private_data(existing_vmr)) {
 				ret = -EINVAL;
+				ihk_rwspinlock_read_unlock_noirq(
+					&vm->memory_range_lock);
 				goto out_2;
 			}
 		}
+
+		ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock);
 	}
 
 	flags |= MAP_ANONYMOUS;
@@ -1423,20 +1425,18 @@ static void xpmem_detach_att(
 
 	XPMEM_DEBUG("detaching att->vm=0x%p", (void *)att->vm);
 
-	vm = att->vm;
-
-	ihk_rwspinlock_read_lock_noirq(&vm->memory_range_lock);
-
 	mcs_rwlock_writer_lock(&att->at_lock, &at_lock);
 
 	if (att->flags & XPMEM_FLAG_DESTROYING) {
 		mcs_rwlock_writer_unlock(&att->at_lock, &at_lock);
-		ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock);
 		XPMEM_DEBUG("return: XPMEM_FLAG_DESTROYING");
 		return;
 	}
 	att->flags |= XPMEM_FLAG_DESTROYING;
 
+	vm = att->vm;
+	ihk_rwspinlock_read_lock_noirq(&vm->memory_range_lock);
+
 	range = lookup_process_memory_range(vm,
 		att->at_vaddr, att->at_vaddr + 1);
 
@@ -1676,10 +1676,8 @@ int xpmem_remove_process_memory_range(
 	mcs_rwlock_writer_lock(&att->at_lock, &at_lock);
 
 	if (att->flags & XPMEM_FLAG_DESTROYING) {
-		mcs_rwlock_writer_unlock(&att->at_lock, &at_lock);
-		xpmem_att_deref(att);
 		XPMEM_DEBUG("already cleaned up");
-		return 0;
+		goto out;
 	}
 
 	if (vmr->start == att->at_vaddr &&
@@ -1910,13 +1908,13 @@ static int xpmem_remap_pte(
 		"seg_vaddr=0x%lx", 
 		vmr, vaddr, reason, seg->segid, seg_vaddr);
 
-	ihk_rwspinlock_read_lock_noirq(&seg_tg->vm->memory_range_lock);
+	if (is_remote_vm(seg_tg->vm)) {
+		ihk_rwspinlock_read_lock_noirq(&seg_tg->vm->memory_range_lock);
+	}
 
 	seg_vmr = lookup_process_memory_range(seg_tg->vm, seg_vaddr, 
 		seg_vaddr + 1);
 
-	ihk_rwspinlock_read_unlock_noirq(&seg_tg->vm->memory_range_lock);
-
 	if (!seg_vmr) {
 		ret = -EFAULT;
 		ekprintf("%s: ERROR: lookup_process_memory_range() failed\n", 
@@ -1989,6 +1987,10 @@ static int xpmem_remap_pte(
 	}
 
 out:
+	if (is_remote_vm(seg_tg->vm)) {
+		ihk_rwspinlock_read_unlock_noirq(&seg_tg->vm->memory_range_lock);
+	}
+
 	XPMEM_DEBUG("return: ret=%d", ret);
 
 	return ret;
@@ -2336,3 +2338,14 @@ static int xpmem_validate_access(
 	return 0;
 }
 
+static int is_remote_vm(struct process_vm *vm)
+{
+	int ret = 0;
+
+	if (cpu_local_var(current)->proc->vm != vm) {
+		/* vm is not mine */
+		ret = 1;
+	}
+
+	return ret;
+}