gh0s7/mckernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

xpmem: replace mcs_rwlock with ihk_rwspinlock

Browse Source 
Change-Id: I2d29f37f590f55db009f522395ede4c6494f8c89
This commit is contained in:
Masamichi Takagi
2021-02-27 08:36:13 +09:00
committed by Masamichi Takagi
parent 865eb37b11
commit 3aaa5350f0
3 changed files with 39 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
kernel/include/xpmem.h
View File

@ -28,5 +28,17 @@ int xpmem_remove_process_memory_range(struct process_vm *vm,
int xpmem_fault_process_memory_range(struct process_vm *vm, int xpmem_fault_process_memory_range(struct process_vm *vm,
struct vm_range *vmr, unsigned long vaddr, uint64_t reason); struct vm_range *vmr, unsigned long vaddr, uint64_t reason);
struct xpmem_attachment {
ihk_rwspinlock_t at_lock; /* att lock */
unsigned long vaddr; /* starting address of seg attached */
unsigned long at_vaddr; /* address where seg is attached */
size_t at_size; /* size of seg attachment */
struct vm_range *at_vmr; /* vm_range where seg is attachment */
int flags; /* att attributes and state */
ihk_atomic_t refcnt; /* references to att */
struct xpmem_access_permit *ap; /* associated access permit */
struct list_head att_list; /* atts linked to access permit */
struct process_vm *vm; /* process_vm attached to */
};
#endif /* _XPMEM_H */ #endif /* _XPMEM_H */

13
kernel/include/xpmem_private.h
View File

@ -177,19 +177,6 @@ struct xpmem_access_permit {
struct list_head ap_hashlist; /* access permit hash list */ struct list_head ap_hashlist; /* access permit hash list */
}; };
struct xpmem_attachment {
mcs_rwlock_lock_t at_lock; /* att lock */
unsigned long vaddr; /* starting address of seg attached */
unsigned long at_vaddr; /* address where seg is attached */
size_t at_size; /* size of seg attachment */
struct vm_range *at_vmr; /* vm_range where seg is attachment */
volatile int flags; /* att attributes and state */
ihk_atomic_t refcnt; /* references to att */
struct xpmem_access_permit *ap; /* associated access permit */
struct list_head att_list; /* atts linked to access permit */
struct process_vm *vm; /* process_vm attached to */
};
struct xpmem_partition { struct xpmem_partition {
ihk_atomic_t n_opened; /* # of /dev/xpmem opened */ ihk_atomic_t n_opened; /* # of /dev/xpmem opened */
struct xpmem_hashlist tg_hashtable[]; /* locks + tg hash lists */ struct xpmem_hashlist tg_hashtable[]; /* locks + tg hash lists */

54
kernel/xpmem.c
View File

@ -1010,7 +1010,7 @@ static int xpmem_attach(
struct xpmem_access_permit *ap; struct xpmem_access_permit *ap;
struct xpmem_segment *seg; struct xpmem_segment *seg;
struct xpmem_attachment *att; struct xpmem_attachment *att;
struct mcs_rwlock_node_irqsave at_lock; unsigned long at_lock;
struct vm_range *vmr; struct vm_range *vmr;
struct process_vm *vm = cpu_local_var(current)->vm; struct process_vm *vm = cpu_local_var(current)->vm;
@ -1086,7 +1086,7 @@ static int xpmem_attach(
XPMEM_DEBUG("kmalloc(): att=0x%p", att); XPMEM_DEBUG("kmalloc(): att=0x%p", att);
memset(att, 0, sizeof(struct xpmem_attachment)); memset(att, 0, sizeof(struct xpmem_attachment));
mcs_rwlock_init(&att->at_lock); ihk_rwspinlock_init(&att->at_lock);
att->vaddr = seg_vaddr; att->vaddr = seg_vaddr;
att->at_size = size; att->at_size = size;
att->ap = ap; att->ap = ap;
@ -1096,7 +1096,7 @@ static int xpmem_attach(
xpmem_att_not_destroyable(att); xpmem_att_not_destroyable(att);
xpmem_att_ref(att); xpmem_att_ref(att);
mcs_rwlock_writer_lock(&att->at_lock, &at_lock); at_lock = ihk_rwspinlock_write_lock(&att->at_lock);
ihk_mc_spinlock_lock_noirq(&ap->lock); ihk_mc_spinlock_lock_noirq(&ap->lock);
list_add_tail(&att->att_list, &ap->att_list); list_add_tail(&att->att_list, &ap->att_list);
@ -1180,7 +1180,7 @@ out_2:
ihk_mc_spinlock_unlock_noirq(&ap->lock); ihk_mc_spinlock_unlock_noirq(&ap->lock);
xpmem_att_destroyable(att); xpmem_att_destroyable(att);
} }
mcs_rwlock_writer_unlock(&att->at_lock, &at_lock); ihk_rwspinlock_write_unlock(&att->at_lock, at_lock);
xpmem_att_deref(att); xpmem_att_deref(att);
out_1: out_1:
xpmem_ap_deref(ap); xpmem_ap_deref(ap);
@ -1200,7 +1200,7 @@ static int xpmem_detach(
int ret; int ret;
struct xpmem_access_permit *ap; struct xpmem_access_permit *ap;
struct xpmem_attachment *att; struct xpmem_attachment *att;
struct mcs_rwlock_node_irqsave at_lock; unsigned long at_lock;
struct vm_range *range; struct vm_range *range;
struct process_vm *vm = cpu_local_var(current)->vm; struct process_vm *vm = cpu_local_var(current)->vm;
@ -1223,10 +1223,10 @@ static int xpmem_detach(
xpmem_att_ref(att); xpmem_att_ref(att);
mcs_rwlock_writer_lock(&att->at_lock, &at_lock); at_lock = ihk_rwspinlock_write_lock(&att->at_lock);
if (att->flags & XPMEM_FLAG_DESTROYING) { if (att->flags & XPMEM_FLAG_DESTROYING) {
mcs_rwlock_writer_unlock(&att->at_lock, &at_lock); ihk_rwspinlock_write_unlock(&att->at_lock, at_lock);
ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock); ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock);
xpmem_att_deref(att); xpmem_att_deref(att);
return 0; return 0;
@ -1239,7 +1239,7 @@ static int xpmem_detach(
if (cpu_local_var(current)->proc->pid != ap->tg->tgid) { if (cpu_local_var(current)->proc->pid != ap->tg->tgid) {
att->flags &= ~XPMEM_FLAG_DESTROYING; att->flags &= ~XPMEM_FLAG_DESTROYING;
xpmem_ap_deref(ap); xpmem_ap_deref(ap);
mcs_rwlock_writer_unlock(&att->at_lock, &at_lock); ihk_rwspinlock_write_unlock(&att->at_lock, at_lock);
ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock); ihk_rwspinlock_write_unlock_noirq(&vm->memory_range_lock);
xpmem_att_deref(att); xpmem_att_deref(att);
return -EACCES; return -EACCES;
@ -1251,7 +1251,7 @@ static int xpmem_detach(
/* range->memobj is released in xpmem_vm_munmap() --> xpmem_remove_process_range() --> /* range->memobj is released in xpmem_vm_munmap() --> xpmem_remove_process_range() -->
xpmem_free_process_memory_range() */ xpmem_free_process_memory_range() */
mcs_rwlock_writer_unlock(&att->at_lock, &at_lock); ihk_rwspinlock_write_unlock(&att->at_lock, at_lock);
XPMEM_DEBUG("xpmem_vm_munmap(): start=0x%lx, len=0x%lx", XPMEM_DEBUG("xpmem_vm_munmap(): start=0x%lx, len=0x%lx",
range->start, att->at_size); range->start, att->at_size);
@ -1419,16 +1419,16 @@ static void xpmem_detach_att(
int ret; int ret;
struct vm_range *range; struct vm_range *range;
struct process_vm *vm; struct process_vm *vm;
struct mcs_rwlock_node_irqsave at_lock; unsigned long at_lock;
XPMEM_DEBUG("call: apid=0x%lx, att=0x%p", ap->apid, att); XPMEM_DEBUG("call: apid=0x%lx, att=0x%p", ap->apid, att);
XPMEM_DEBUG("detaching att->vm=0x%p", (void *)att->vm); XPMEM_DEBUG("detaching att->vm=0x%p", (void *)att->vm);
mcs_rwlock_writer_lock(&att->at_lock, &at_lock); at_lock = ihk_rwspinlock_write_lock(&att->at_lock);
if (att->flags & XPMEM_FLAG_DESTROYING) { if (att->flags & XPMEM_FLAG_DESTROYING) {
mcs_rwlock_writer_unlock(&att->at_lock, &at_lock); ihk_rwspinlock_write_unlock(&att->at_lock, at_lock);
XPMEM_DEBUG("return: XPMEM_FLAG_DESTROYING"); XPMEM_DEBUG("return: XPMEM_FLAG_DESTROYING");
return; return;
} }
@ -1444,7 +1444,7 @@ static void xpmem_detach_att(
ihk_mc_spinlock_lock_noirq(&ap->lock); ihk_mc_spinlock_lock_noirq(&ap->lock);
list_del_init(&att->att_list); list_del_init(&att->att_list);
ihk_mc_spinlock_unlock_noirq(&ap->lock); ihk_mc_spinlock_unlock_noirq(&ap->lock);
mcs_rwlock_writer_unlock(&att->at_lock, &at_lock); ihk_rwspinlock_write_unlock(&att->at_lock, at_lock);
ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock); ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock);
xpmem_att_destroyable(att); xpmem_att_destroyable(att);
XPMEM_DEBUG("return: range=%p"); XPMEM_DEBUG("return: range=%p");
@ -1470,7 +1470,7 @@ static void xpmem_detach_att(
list_del_init(&att->att_list); list_del_init(&att->att_list);
ihk_mc_spinlock_unlock_noirq(&ap->lock); ihk_mc_spinlock_unlock_noirq(&ap->lock);
mcs_rwlock_writer_unlock(&att->at_lock, &at_lock); ihk_rwspinlock_write_unlock(&att->at_lock, at_lock);
XPMEM_DEBUG("xpmem_vm_munmap(): start=0x%lx, len=0x%lx", XPMEM_DEBUG("xpmem_vm_munmap(): start=0x%lx, len=0x%lx",
range->start, att->at_size); range->start, att->at_size);
@ -1578,13 +1578,13 @@ static void xpmem_clear_PTEs_of_att(
unsigned long end) unsigned long end)
{ {
int ret; int ret;
struct mcs_rwlock_node_irqsave at_lock; unsigned long at_lock;
XPMEM_DEBUG("call: att=0x%p, start=0x%lx, end=0x%lx", XPMEM_DEBUG("call: att=0x%p, start=0x%lx, end=0x%lx",
att, start, end); att, start, end);
ihk_rwspinlock_read_lock_noirq(&att->vm->memory_range_lock); ihk_rwspinlock_read_lock_noirq(&att->vm->memory_range_lock);
mcs_rwlock_writer_lock(&att->at_lock, &at_lock); at_lock = ihk_rwspinlock_write_lock(&att->at_lock);
if (att->flags & XPMEM_FLAG_VALIDPTEs) { if (att->flags & XPMEM_FLAG_VALIDPTEs) {
struct vm_range *range; struct vm_range *range;
@ -1624,7 +1624,7 @@ static void xpmem_clear_PTEs_of_att(
goto out; goto out;
} }
mcs_rwlock_writer_unlock(&att->at_lock, &at_lock); ihk_rwspinlock_write_unlock(&att->at_lock, at_lock);
XPMEM_DEBUG( XPMEM_DEBUG(
"xpmem_vm_munmap(): start=0x%lx, len=0x%lx", "xpmem_vm_munmap(): start=0x%lx, len=0x%lx",
@ -1636,13 +1636,13 @@ static void xpmem_clear_PTEs_of_att(
__FUNCTION__, ret); __FUNCTION__, ret);
} }
mcs_rwlock_writer_lock(&att->at_lock, &at_lock); at_lock = ihk_rwspinlock_write_lock(&att->at_lock);
if (offset_start == 0 && att->at_size == invalidate_len) if (offset_start == 0 && att->at_size == invalidate_len)
att->flags &= ~XPMEM_FLAG_VALIDPTEs; att->flags &= ~XPMEM_FLAG_VALIDPTEs;
} }
out: out:
mcs_rwlock_writer_unlock(&att->at_lock, &at_lock); ihk_rwspinlock_write_unlock(&att->at_lock, at_lock);
ihk_rwspinlock_read_unlock_noirq(&att->vm->memory_range_lock); ihk_rwspinlock_read_unlock_noirq(&att->vm->memory_range_lock);
XPMEM_DEBUG("return: "); XPMEM_DEBUG("return: ");
@ -1657,7 +1657,7 @@ int xpmem_remove_process_memory_range(
u64 remaining_vaddr; u64 remaining_vaddr;
struct xpmem_access_permit *ap; struct xpmem_access_permit *ap;
struct xpmem_attachment *att; struct xpmem_attachment *att;
struct mcs_rwlock_node_irqsave at_lock; unsigned long at_lock;
XPMEM_DEBUG("call: vmr=0x%p, att=0x%p", vmr, vmr->private_data); XPMEM_DEBUG("call: vmr=0x%p, att=0x%p", vmr, vmr->private_data);
@ -1671,7 +1671,7 @@ int xpmem_remove_process_memory_range(
xpmem_att_ref(att); xpmem_att_ref(att);
mcs_rwlock_writer_lock(&att->at_lock, &at_lock); at_lock = ihk_rwspinlock_write_lock(&att->at_lock);
if (att->flags & XPMEM_FLAG_DESTROYING) { if (att->flags & XPMEM_FLAG_DESTROYING) {
XPMEM_DEBUG("already cleaned up"); XPMEM_DEBUG("already cleaned up");
@ -1747,7 +1747,7 @@ int xpmem_remove_process_memory_range(
*/ */
out: out:
mcs_rwlock_writer_unlock(&att->at_lock, &at_lock); ihk_rwspinlock_write_unlock(&att->at_lock, at_lock);
xpmem_att_deref(att); xpmem_att_deref(att);
@ -1773,7 +1773,7 @@ int xpmem_fault_process_memory_range(
struct xpmem_attachment *att; struct xpmem_attachment *att;
struct xpmem_segment *seg; struct xpmem_segment *seg;
size_t pgsize; size_t pgsize;
struct mcs_rwlock_node_irqsave at_lock = { 0 }; unsigned long at_lock;
int att_locked = 0; int att_locked = 0;
XPMEM_DEBUG("call: vmr=0x%p, vaddr=0x%lx, reason=0x%lx", XPMEM_DEBUG("call: vmr=0x%p, vaddr=0x%lx, reason=0x%lx",
@ -1804,15 +1804,15 @@ int xpmem_fault_process_memory_range(
seg_tg = seg->tg; seg_tg = seg->tg;
xpmem_tg_ref(seg_tg); xpmem_tg_ref(seg_tg);
at_lock = ihk_rwspinlock_write_lock(&att->at_lock);
att_locked = 1;
if ((seg->flags & XPMEM_FLAG_DESTROYING) || if ((seg->flags & XPMEM_FLAG_DESTROYING) ||
(seg_tg->flags & XPMEM_FLAG_DESTROYING)) { (seg_tg->flags & XPMEM_FLAG_DESTROYING)) {
ret = -ENOENT; ret = -ENOENT;
goto out_2; goto out_2;
} }
mcs_rwlock_writer_lock(&att->at_lock, &at_lock);
att_locked = 1;
if ((att->flags & XPMEM_FLAG_DESTROYING) || if ((att->flags & XPMEM_FLAG_DESTROYING) ||
(ap_tg->flags & XPMEM_FLAG_DESTROYING) || (ap_tg->flags & XPMEM_FLAG_DESTROYING) ||
(seg_tg->flags & XPMEM_FLAG_DESTROYING)) { (seg_tg->flags & XPMEM_FLAG_DESTROYING)) {
@ -1865,7 +1865,7 @@ out_2:
out_1: out_1:
if (att_locked) { if (att_locked) {
mcs_rwlock_writer_unlock(&att->at_lock, &at_lock); ihk_rwspinlock_write_unlock(&att->at_lock, at_lock);
} }
xpmem_tg_deref(seg_tg); xpmem_tg_deref(seg_tg);