gh0s7/mckernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Tofu: keep track of stags per memory range

Browse Source 
Change-Id: I033beaeee3b141dab4485dd3a2a3848eaa84e54e
This commit is contained in:
Balazs Gerofi
2020-12-15 14:43:12 +09:00
committed by Masamichi Takagi
parent 75694152f0
commit 0b82c8942b
4 changed files with 306 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
kernel/include/process.h
View File

@ -395,6 +395,9 @@ struct vm_range {
off_t objoff;
int pgshift; /* page size. 0 means THP */
int padding;
#ifdef ENABLE_TOFU
struct list_head tofu_stag_list;
#endif
void *private_data;
};
@ -764,6 +767,9 @@ struct thread {
};
#define VM_RANGE_CACHE_SIZE 4
#ifdef ENABLE_TOFU
#define TOFU_STAG_HASH_SIZE 4
#endif
struct process_vm {
struct address_space *address_space;
@ -796,6 +802,12 @@ struct process_vm {
struct vm_range *range_cache[VM_RANGE_CACHE_SIZE];
int range_cache_ind;
struct swapinfo *swapinfo;
#ifdef ENABLE_TOFU
/* Tofu STAG hash */
ihk_spinlock_t tofu_stag_lock;
struct list_head tofu_stag_hash[TOFU_STAG_HASH_SIZE];
#endif
};
static inline int has_cap_ipc_lock(struct thread *th)

57
kernel/process.c
View File

@ -36,6 +36,9 @@
#include <rusage_private.h>
#include <ihk/monitor.h>
#include <ihk/debug.h>
#ifdef ENABLE_TOFU
#include <tofu/tofu_stag_range.h>
#endif
//#define DEBUG_PRINT_PROCESS
@ -269,6 +272,12 @@ init_process_vm(struct process *owner, struct address_space *asp, struct process
}
vm->range_cache_ind = 0;
#ifdef ENABLE_TOFU
ihk_mc_spinlock_init(&vm->tofu_stag_lock);
for (i = 0; i < TOFU_STAG_HASH_SIZE; ++i) {
INIT_LIST_HEAD(&vm->tofu_stag_hash[i]);
}
#endif
return 0;
}
@ -955,6 +964,11 @@ int split_process_memory_range(struct process_vm *vm, struct vm_range *range,
newrange->pgshift = range->pgshift;
newrange->private_data = range->private_data;
#ifdef ENABLE_TOFU
/* TODO: figure out which entries to put on which list! */
INIT_LIST_HEAD(&newrange->tofu_stag_list);
#endif
if (range->memobj) {
memobj_ref(range->memobj);
newrange->memobj = range->memobj;
@ -1023,6 +1037,28 @@ int join_process_memory_range(struct process_vm *vm,
if (vm->range_cache[i] == merging)
vm->range_cache[i] = surviving;
}
#ifdef ENABLE_TOFU
/* Move Tofu stag range entries */
if (vm->proc->enable_tofu) {
struct tofu_stag_range *tsr, *next;
ihk_mc_spinlock_lock_noirq(&vm->tofu_stag_lock);
list_for_each_entry_safe(tsr, next,
&merging->tofu_stag_list, list) {
list_del(&tsr->list);
list_add_tail(&tsr->list, &surviving->tofu_stag_list);
dkprintf("%s: stag: %d @ %p:%lu moved in VM range merge\n",
__func__,
tsr->stag,
tsr->start,
(unsigned long)(tsr->end - tsr->start));
}
ihk_mc_spinlock_unlock_noirq(&vm->tofu_stag_lock);
}
#endif
kfree(merging);
error = 0;
@ -1137,6 +1173,24 @@ static int free_process_memory_range(struct process_vm *vm,
}
straight_out:
#ifdef ENABLE_TOFU
if (vm->proc->enable_tofu) {
int entries;
extern int tofu_stag_range_remove_overlapping(struct process_vm *vm,
struct vm_range *range);
entries = tofu_stag_range_remove_overlapping(vm, range);
if (entries > 0) {
kprintf("%s: removed %d Tofu stag entries for range 0x%lx:%lu\n",
__func__,
entries,
range->start,
range->end - range->start);
}
}
#endif
rb_erase(&range->vm_rb_node, &vm->vm_range_tree);
for (i = 0; i < VM_RANGE_CACHE_SIZE; ++i) {
if (vm->range_cache[i] == range)
@ -1428,6 +1482,9 @@ int add_process_memory_range(struct process_vm *vm,
range->pgshift = pgshift;
range->private_data = NULL;
range->straight_start = 0;
#ifdef ENABLE_TOFU
INIT_LIST_HEAD(&range->tofu_stag_list);
#endif
rc = 0;
if (phys == NOPHYS) {

9
kernel/syscall.c
View File

@ -1390,6 +1390,15 @@ void terminate(int rc, int sig)
mcs_rwlock_writer_unlock(&proc->threads_lock, &lock);
vm = proc->vm;
#ifdef ENABLE_TOFU
if (proc->enable_tofu) {
extern void tof_utofu_finalize();
tof_utofu_finalize();
}
#endif
free_all_process_memory_range(vm);
if (proc->saved_cmdline) {

230
kernel/tofu/tof_utofu_main.c
View File

@ -43,6 +43,121 @@ typedef void (*tof_core_signal_handler)(int, int, uint64_t, uint64_t);
#include <tofu/tofu_generated-tof_utofu_bg.h>
#include <tofu/tofu_generated-tof_utofu_mbpt.h>
#include <tofu/tofu_stag_range.h>
/*
* Tofu STAG regions list keeps track of stags in a given VM range..
* Per-process tree is protected by process' vm_range_lock.
*/
int tof_utofu_stag_range_insert(struct process_vm *vm,
struct vm_range *range,
uintptr_t start, uintptr_t end,
struct tof_utofu_cq *ucq, int stag)
{
struct tofu_stag_range *tsr = kmalloc(sizeof(*tsr), IHK_MC_AP_NOWAIT);
if (!tsr) {
kprintf("%s: error: allocating tofu_stag_range\n", __func__);
return -ENOMEM;
}
tsr->start = start;
tsr->end = end;
tsr->ucq = ucq;
tsr->stag = stag;
ihk_mc_spinlock_lock_noirq(&vm->tofu_stag_lock);
list_add_tail(&tsr->list, &range->tofu_stag_list);
list_add_tail(&tsr->hash, &vm->tofu_stag_hash[stag % TOFU_STAG_HASH_SIZE]);
ihk_mc_spinlock_unlock_noirq(&vm->tofu_stag_lock);
dkprintf("%s: stag: %d for TNI %d CQ %d @ %p:%lu\n",
__func__,
tsr->stag,
tsr->ucq->tni,
tsr->ucq->cqid,
tsr->start,
(unsigned long)(tsr->end - tsr->start));
return 0;
}
struct tofu_stag_range *tofu_stag_range_lookup_by_stag(struct process_vm *vm,
int stag)
{
struct tofu_stag_range *tsr;
struct tofu_stag_range *match = NULL;
ihk_mc_spinlock_lock_noirq(&vm->tofu_stag_lock);
list_for_each_entry(tsr,
&vm->tofu_stag_hash[stag % TOFU_STAG_HASH_SIZE], hash) {
if (tsr->stag == stag) {
match = tsr;
break;
}
}
ihk_mc_spinlock_unlock_noirq(&vm->tofu_stag_lock);
return match;
}
/* XXX: vm->tofu_stag_lock must be held */
void __tofu_stag_range_remove(struct process_vm *vm, struct tofu_stag_range *tsr)
{
dkprintf("%s: stag: %d for TNI %d CQ %d @ %p:%lu\n",
__func__,
tsr->stag,
tsr->ucq->tni,
tsr->ucq->cqid,
tsr->start,
(unsigned long)(tsr->end - tsr->start));
list_del(&tsr->list);
list_del(&tsr->hash);
kfree(tsr);
}
void tofu_stag_range_remove(struct process_vm *vm, struct tofu_stag_range *tsr)
{
ihk_mc_spinlock_lock_noirq(&vm->tofu_stag_lock);
__tofu_stag_range_remove(vm, tsr);
ihk_mc_spinlock_unlock_noirq(&vm->tofu_stag_lock);
}
static int tof_utofu_free_stag(struct tof_utofu_cq *ucq, int stag);
int tofu_stag_range_remove_overlapping(struct process_vm *vm,
struct vm_range *range)
{
struct tofu_stag_range *tsr, *next;
int entries = 0;
ihk_mc_spinlock_lock_noirq(&vm->tofu_stag_lock);
list_for_each_entry_safe(tsr, next,
&range->tofu_stag_list, list) {
dkprintf("%s: stag: %d @ %p:%lu\n",
__func__,
tsr->stag,
tsr->start,
(unsigned long)(tsr->end - tsr->start));
linux_spin_lock(&tsr->ucq->trans.mru_lock);
tof_utofu_free_stag(tsr->ucq, tsr->stag);
linux_spin_unlock(&tsr->ucq->trans.mru_lock);
__tofu_stag_range_remove(vm, tsr);
++entries;
}
ihk_mc_spinlock_unlock_noirq(&vm->tofu_stag_lock);
return entries;
}
#define TOF_UTOFU_VERSION TOF_UAPI_VERSION
#define TOF_UTOFU_NUM_STAG_NTYPES 3
#define TOF_UTOFU_NUM_STAG_BITS(size) ((size) + 13)
@ -1016,6 +1131,7 @@ static int tof_utofu_ioctl_alloc_stag(struct tof_utofu_device *dev, unsigned lon
size_t pgsz;
int ret = -ENOTSUPP;
unsigned long irqflags;
struct vm_range *range = NULL;
ucq = container_of(dev, struct tof_utofu_cq, common);
if(!ucq->common.enabled){
@ -1037,7 +1153,46 @@ static int tof_utofu_ioctl_alloc_stag(struct tof_utofu_device *dev, unsigned lon
}
readonly = (req.flags & 1) != 0;
ihk_rwspinlock_read_lock_noirq(&vm->memory_range_lock);
/* Assume smallest page size at first */
start = round_down((uintptr_t)req.va, PAGE_SIZE);
end = round_up((uintptr_t)req.va + req.len, PAGE_SIZE);
/* Find range, straight mapping special lookup */
if (vm->proc->straight_va &&
start >= (unsigned long)vm->proc->straight_va &&
end <= ((unsigned long)vm->proc->straight_va +
vm->proc->straight_len) &&
!(start == (unsigned long)vm->proc->straight_va &&
end == ((unsigned long)vm->proc->straight_va +
vm->proc->straight_len))) {
struct vm_range *range_iter;
range_iter = lookup_process_memory_range(vm, 0, -1);
while (range_iter) {
if (range_iter->straight_start &&
start >= range_iter->straight_start &&
start < (range_iter->straight_start +
(range_iter->end - range_iter->start))) {
range = range_iter;
break;
}
range_iter = next_process_memory_range(vm, range_iter);
}
}
else {
range = lookup_process_memory_range(vm, start, end);
}
if (!range) {
ret = -EINVAL;
goto unlock_out;
}
pgszbits = PAGE_SHIFT;
if (req.flags & TOF_UTOFU_ALLOC_STAG_LPG) {
ret = tof_utofu_get_pagesize_locked((uintptr_t)req.va,
@ -1113,6 +1268,12 @@ static int tof_utofu_ioctl_alloc_stag(struct tof_utofu_device *dev, unsigned lon
//up(&ucq->ucq_sem);
ihk_mc_spinlock_unlock_noirq(&tofu_tni_cq_lock[ucq->tni][ucq->cqid]);
if (ret == 0) {
tof_utofu_stag_range_insert(vm, range, start, end, ucq, req.stag);
}
unlock_out:
ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock);
if(ret == 0){
@ -1336,6 +1497,21 @@ static int tof_utofu_ioctl_free_stags(struct tof_utofu_device *dev, unsigned lon
linux_spin_lock_irqsave(&ucq->trans.mru_lock, irqflags);
ret = tof_utofu_free_stag(ucq, stags[i]);
linux_spin_unlock_irqrestore(&ucq->trans.mru_lock, irqflags);
{
struct tofu_stag_range *tsr;
tsr = tofu_stag_range_lookup_by_stag(
cpu_local_var(current)->vm, stags[i]);
if (tsr) {
tofu_stag_range_remove(cpu_local_var(current)->vm, tsr);
}
else {
kprintf("%s: no stag range object for %d??\n", __func__, stags[i]);
}
}
if(ret == 0){
stags[i] = -1;
}
@ -1378,9 +1554,11 @@ static int tof_utofu_ioctl_free_stags(struct tof_utofu_device *dev, unsigned lon
void tof_utofu_release_cq(void *pde_data)
{
struct tof_utofu_cq *ucq;
int stag;
//int stag;
struct tof_utofu_device *dev;
unsigned long irqflags;
struct process_vm *vm = cpu_local_var(current)->vm;
int do_free = 1;
dev = (struct tof_utofu_device *)pde_data;
ucq = container_of(dev, struct tof_utofu_cq, common);
@ -1388,13 +1566,43 @@ void tof_utofu_release_cq(void *pde_data)
if (!ucq->common.enabled) {
kprintf("%s: UCQ TNI %d, CQ %d is disabled\n",
__func__, ucq->tni, ucq->cqid);
return;
do_free = 0;
}
#if 0
for (stag = 0; stag < TOF_UTOFU_NUM_STAG(ucq->num_stag); stag++) {
linux_spin_lock_irqsave(&ucq->trans.mru_lock, irqflags);
tof_utofu_free_stag(ucq, stag);
linux_spin_unlock_irqrestore(&ucq->trans.mru_lock, irqflags);
#endif
{
int i;
struct tofu_stag_range *tsr, *next;
ihk_mc_spinlock_lock_noirq(&vm->tofu_stag_lock);
for (i = 0; i < TOFU_STAG_HASH_SIZE; ++i) {
list_for_each_entry_safe(tsr, next,
&vm->tofu_stag_hash[i], hash) {
if (tsr->ucq != ucq)
continue;
if (do_free) {
dkprintf("%s: removing stag %d for TNI %d CQ %d\n",
__func__, tsr->stag, ucq->tni, ucq->cqid);
linux_spin_lock_irqsave(&ucq->trans.mru_lock, irqflags);
tof_utofu_free_stag(tsr->ucq, tsr->stag);
linux_spin_unlock_irqrestore(&ucq->trans.mru_lock, irqflags);
}
else {
kprintf("%s: WARNING: could not free stag %d for TNI %d CQ %d (UCQ is disabled)\n",
__func__, tsr->stag, ucq->tni, ucq->cqid);
}
__tofu_stag_range_remove(vm, tsr);
}
}
ihk_mc_spinlock_unlock_noirq(&vm->tofu_stag_lock);
}
dkprintf("%s: UCQ (pde: %p) TNI %d, CQ %d\n",
@ -2131,6 +2339,24 @@ void tof_utofu_finalize(void)
{
struct tofu_globals *tg = ihk_mc_get_tofu_globals();
/* Could be called from idle.. */
if (cpu_local_var(current)->proc->enable_tofu) {
int i;
struct process_vm *vm = cpu_local_var(current)->vm;
struct tofu_stag_range *tsr, *next;
for (i = 0; i < TOFU_STAG_HASH_SIZE; ++i) {
list_for_each_entry_safe(tsr, next,
&vm->tofu_stag_hash[i], hash) {
dkprintf("%s: WARNING: stray stag %d for TNI %d CQ %d?\n",
__func__, tsr->stag, tsr->ucq->tni, tsr->ucq->cqid);
}
}
kprintf("%s: STAG processing done\n", __func__);
}
ihk_mc_clear_kernel_range((void *)tg->linux_vmalloc_start,
(void *)tg->linux_vmalloc_end);
}