diff --git a/kernel/include/process.h b/kernel/include/process.h index 1718836b..0faf40a0 100644 --- a/kernel/include/process.h +++ b/kernel/include/process.h @@ -395,6 +395,9 @@ struct vm_range { off_t objoff; int pgshift; /* page size. 0 means THP */ int padding; +#ifdef ENABLE_TOFU + struct list_head tofu_stag_list; +#endif void *private_data; }; @@ -764,6 +767,9 @@ struct thread { }; #define VM_RANGE_CACHE_SIZE 4 +#ifdef ENABLE_TOFU +#define TOFU_STAG_HASH_SIZE 4 +#endif struct process_vm { struct address_space *address_space; @@ -796,6 +802,12 @@ struct process_vm { struct vm_range *range_cache[VM_RANGE_CACHE_SIZE]; int range_cache_ind; struct swapinfo *swapinfo; + +#ifdef ENABLE_TOFU + /* Tofu STAG hash */ + ihk_spinlock_t tofu_stag_lock; + struct list_head tofu_stag_hash[TOFU_STAG_HASH_SIZE]; +#endif }; static inline int has_cap_ipc_lock(struct thread *th) diff --git a/kernel/process.c b/kernel/process.c index 44242d35..c09ad6c2 100644 --- a/kernel/process.c +++ b/kernel/process.c @@ -36,6 +36,9 @@ #include #include #include +#ifdef ENABLE_TOFU +#include +#endif //#define DEBUG_PRINT_PROCESS @@ -269,6 +272,12 @@ init_process_vm(struct process *owner, struct address_space *asp, struct process } vm->range_cache_ind = 0; +#ifdef ENABLE_TOFU + ihk_mc_spinlock_init(&vm->tofu_stag_lock); + for (i = 0; i < TOFU_STAG_HASH_SIZE; ++i) { + INIT_LIST_HEAD(&vm->tofu_stag_hash[i]); + } +#endif return 0; } @@ -955,6 +964,11 @@ int split_process_memory_range(struct process_vm *vm, struct vm_range *range, newrange->pgshift = range->pgshift; newrange->private_data = range->private_data; +#ifdef ENABLE_TOFU + /* TODO: figure out which entries to put on which list! */ + INIT_LIST_HEAD(&newrange->tofu_stag_list); +#endif + if (range->memobj) { memobj_ref(range->memobj); newrange->memobj = range->memobj; @@ -1023,6 +1037,28 @@ int join_process_memory_range(struct process_vm *vm, if (vm->range_cache[i] == merging) vm->range_cache[i] = surviving; } + +#ifdef ENABLE_TOFU + /* Move Tofu stag range entries */ + if (vm->proc->enable_tofu) { + struct tofu_stag_range *tsr, *next; + + ihk_mc_spinlock_lock_noirq(&vm->tofu_stag_lock); + list_for_each_entry_safe(tsr, next, + &merging->tofu_stag_list, list) { + + list_del(&tsr->list); + list_add_tail(&tsr->list, &surviving->tofu_stag_list); + dkprintf("%s: stag: %d @ %p:%lu moved in VM range merge\n", + __func__, + tsr->stag, + tsr->start, + (unsigned long)(tsr->end - tsr->start)); + } + ihk_mc_spinlock_unlock_noirq(&vm->tofu_stag_lock); + } +#endif + kfree(merging); error = 0; @@ -1137,6 +1173,24 @@ static int free_process_memory_range(struct process_vm *vm, } straight_out: + +#ifdef ENABLE_TOFU + if (vm->proc->enable_tofu) { + int entries; + extern int tofu_stag_range_remove_overlapping(struct process_vm *vm, + struct vm_range *range); + + entries = tofu_stag_range_remove_overlapping(vm, range); + if (entries > 0) { + kprintf("%s: removed %d Tofu stag entries for range 0x%lx:%lu\n", + __func__, + entries, + range->start, + range->end - range->start); + } + } +#endif + rb_erase(&range->vm_rb_node, &vm->vm_range_tree); for (i = 0; i < VM_RANGE_CACHE_SIZE; ++i) { if (vm->range_cache[i] == range) @@ -1428,6 +1482,9 @@ int add_process_memory_range(struct process_vm *vm, range->pgshift = pgshift; range->private_data = NULL; range->straight_start = 0; +#ifdef ENABLE_TOFU + INIT_LIST_HEAD(&range->tofu_stag_list); +#endif rc = 0; if (phys == NOPHYS) { diff --git a/kernel/syscall.c b/kernel/syscall.c index 3aeedb56..8a919e16 100644 --- a/kernel/syscall.c +++ b/kernel/syscall.c @@ -1390,6 +1390,15 @@ void terminate(int rc, int sig) mcs_rwlock_writer_unlock(&proc->threads_lock, &lock); vm = proc->vm; + +#ifdef ENABLE_TOFU + if (proc->enable_tofu) { + extern void tof_utofu_finalize(); + + tof_utofu_finalize(); + } +#endif + free_all_process_memory_range(vm); if (proc->saved_cmdline) { diff --git a/kernel/tofu/tof_utofu_main.c b/kernel/tofu/tof_utofu_main.c index 83128d73..1a6b0340 100644 --- a/kernel/tofu/tof_utofu_main.c +++ b/kernel/tofu/tof_utofu_main.c @@ -43,6 +43,121 @@ typedef void (*tof_core_signal_handler)(int, int, uint64_t, uint64_t); #include #include +#include + +/* + * Tofu STAG regions list keeps track of stags in a given VM range.. + * Per-process tree is protected by process' vm_range_lock. + */ +int tof_utofu_stag_range_insert(struct process_vm *vm, + struct vm_range *range, + uintptr_t start, uintptr_t end, + struct tof_utofu_cq *ucq, int stag) +{ + struct tofu_stag_range *tsr = kmalloc(sizeof(*tsr), IHK_MC_AP_NOWAIT); + + if (!tsr) { + kprintf("%s: error: allocating tofu_stag_range\n", __func__); + return -ENOMEM; + } + + tsr->start = start; + tsr->end = end; + tsr->ucq = ucq; + tsr->stag = stag; + + ihk_mc_spinlock_lock_noirq(&vm->tofu_stag_lock); + list_add_tail(&tsr->list, &range->tofu_stag_list); + list_add_tail(&tsr->hash, &vm->tofu_stag_hash[stag % TOFU_STAG_HASH_SIZE]); + ihk_mc_spinlock_unlock_noirq(&vm->tofu_stag_lock); + + dkprintf("%s: stag: %d for TNI %d CQ %d @ %p:%lu\n", + __func__, + tsr->stag, + tsr->ucq->tni, + tsr->ucq->cqid, + tsr->start, + (unsigned long)(tsr->end - tsr->start)); + + return 0; +} + +struct tofu_stag_range *tofu_stag_range_lookup_by_stag(struct process_vm *vm, + int stag) +{ + struct tofu_stag_range *tsr; + struct tofu_stag_range *match = NULL; + + ihk_mc_spinlock_lock_noirq(&vm->tofu_stag_lock); + list_for_each_entry(tsr, + &vm->tofu_stag_hash[stag % TOFU_STAG_HASH_SIZE], hash) { + if (tsr->stag == stag) { + match = tsr; + break; + } + } + ihk_mc_spinlock_unlock_noirq(&vm->tofu_stag_lock); + + return match; +} + +/* XXX: vm->tofu_stag_lock must be held */ +void __tofu_stag_range_remove(struct process_vm *vm, struct tofu_stag_range *tsr) +{ + dkprintf("%s: stag: %d for TNI %d CQ %d @ %p:%lu\n", + __func__, + tsr->stag, + tsr->ucq->tni, + tsr->ucq->cqid, + tsr->start, + (unsigned long)(tsr->end - tsr->start)); + + list_del(&tsr->list); + list_del(&tsr->hash); + kfree(tsr); +} + +void tofu_stag_range_remove(struct process_vm *vm, struct tofu_stag_range *tsr) +{ + ihk_mc_spinlock_lock_noirq(&vm->tofu_stag_lock); + __tofu_stag_range_remove(vm, tsr); + ihk_mc_spinlock_unlock_noirq(&vm->tofu_stag_lock); +} + +static int tof_utofu_free_stag(struct tof_utofu_cq *ucq, int stag); + +int tofu_stag_range_remove_overlapping(struct process_vm *vm, + struct vm_range *range) +{ + struct tofu_stag_range *tsr, *next; + int entries = 0; + + ihk_mc_spinlock_lock_noirq(&vm->tofu_stag_lock); + + list_for_each_entry_safe(tsr, next, + &range->tofu_stag_list, list) { + + dkprintf("%s: stag: %d @ %p:%lu\n", + __func__, + tsr->stag, + tsr->start, + (unsigned long)(tsr->end - tsr->start)); + + linux_spin_lock(&tsr->ucq->trans.mru_lock); + tof_utofu_free_stag(tsr->ucq, tsr->stag); + linux_spin_unlock(&tsr->ucq->trans.mru_lock); + + __tofu_stag_range_remove(vm, tsr); + ++entries; + } + + ihk_mc_spinlock_unlock_noirq(&vm->tofu_stag_lock); + + return entries; +} + + + #define TOF_UTOFU_VERSION TOF_UAPI_VERSION #define TOF_UTOFU_NUM_STAG_NTYPES 3 #define TOF_UTOFU_NUM_STAG_BITS(size) ((size) + 13) @@ -1016,6 +1131,7 @@ static int tof_utofu_ioctl_alloc_stag(struct tof_utofu_device *dev, unsigned lon size_t pgsz; int ret = -ENOTSUPP; unsigned long irqflags; + struct vm_range *range = NULL; ucq = container_of(dev, struct tof_utofu_cq, common); if(!ucq->common.enabled){ @@ -1037,7 +1153,46 @@ static int tof_utofu_ioctl_alloc_stag(struct tof_utofu_device *dev, unsigned lon } readonly = (req.flags & 1) != 0; + ihk_rwspinlock_read_lock_noirq(&vm->memory_range_lock); + + /* Assume smallest page size at first */ + start = round_down((uintptr_t)req.va, PAGE_SIZE); + end = round_up((uintptr_t)req.va + req.len, PAGE_SIZE); + + /* Find range, straight mapping special lookup */ + if (vm->proc->straight_va && + start >= (unsigned long)vm->proc->straight_va && + end <= ((unsigned long)vm->proc->straight_va + + vm->proc->straight_len) && + !(start == (unsigned long)vm->proc->straight_va && + end == ((unsigned long)vm->proc->straight_va + + vm->proc->straight_len))) { + struct vm_range *range_iter; + + range_iter = lookup_process_memory_range(vm, 0, -1); + + while (range_iter) { + if (range_iter->straight_start && + start >= range_iter->straight_start && + start < (range_iter->straight_start + + (range_iter->end - range_iter->start))) { + range = range_iter; + break; + } + + range_iter = next_process_memory_range(vm, range_iter); + } + } + else { + range = lookup_process_memory_range(vm, start, end); + } + + if (!range) { + ret = -EINVAL; + goto unlock_out; + } + pgszbits = PAGE_SHIFT; if (req.flags & TOF_UTOFU_ALLOC_STAG_LPG) { ret = tof_utofu_get_pagesize_locked((uintptr_t)req.va, @@ -1113,6 +1268,12 @@ static int tof_utofu_ioctl_alloc_stag(struct tof_utofu_device *dev, unsigned lon //up(&ucq->ucq_sem); ihk_mc_spinlock_unlock_noirq(&tofu_tni_cq_lock[ucq->tni][ucq->cqid]); + + if (ret == 0) { + tof_utofu_stag_range_insert(vm, range, start, end, ucq, req.stag); + } + +unlock_out: ihk_rwspinlock_read_unlock_noirq(&vm->memory_range_lock); if(ret == 0){ @@ -1336,6 +1497,21 @@ static int tof_utofu_ioctl_free_stags(struct tof_utofu_device *dev, unsigned lon linux_spin_lock_irqsave(&ucq->trans.mru_lock, irqflags); ret = tof_utofu_free_stag(ucq, stags[i]); linux_spin_unlock_irqrestore(&ucq->trans.mru_lock, irqflags); + + { + struct tofu_stag_range *tsr; + + tsr = tofu_stag_range_lookup_by_stag( + cpu_local_var(current)->vm, stags[i]); + + if (tsr) { + tofu_stag_range_remove(cpu_local_var(current)->vm, tsr); + } + else { + kprintf("%s: no stag range object for %d??\n", __func__, stags[i]); + } + } + if(ret == 0){ stags[i] = -1; } @@ -1378,9 +1554,11 @@ static int tof_utofu_ioctl_free_stags(struct tof_utofu_device *dev, unsigned lon void tof_utofu_release_cq(void *pde_data) { struct tof_utofu_cq *ucq; - int stag; + //int stag; struct tof_utofu_device *dev; unsigned long irqflags; + struct process_vm *vm = cpu_local_var(current)->vm; + int do_free = 1; dev = (struct tof_utofu_device *)pde_data; ucq = container_of(dev, struct tof_utofu_cq, common); @@ -1388,13 +1566,43 @@ void tof_utofu_release_cq(void *pde_data) if (!ucq->common.enabled) { kprintf("%s: UCQ TNI %d, CQ %d is disabled\n", __func__, ucq->tni, ucq->cqid); - return; + do_free = 0; } +#if 0 for (stag = 0; stag < TOF_UTOFU_NUM_STAG(ucq->num_stag); stag++) { linux_spin_lock_irqsave(&ucq->trans.mru_lock, irqflags); tof_utofu_free_stag(ucq, stag); linux_spin_unlock_irqrestore(&ucq->trans.mru_lock, irqflags); +#endif + { + int i; + struct tofu_stag_range *tsr, *next; + + ihk_mc_spinlock_lock_noirq(&vm->tofu_stag_lock); + for (i = 0; i < TOFU_STAG_HASH_SIZE; ++i) { + list_for_each_entry_safe(tsr, next, + &vm->tofu_stag_hash[i], hash) { + if (tsr->ucq != ucq) + continue; + + if (do_free) { + dkprintf("%s: removing stag %d for TNI %d CQ %d\n", + __func__, tsr->stag, ucq->tni, ucq->cqid); + + linux_spin_lock_irqsave(&ucq->trans.mru_lock, irqflags); + tof_utofu_free_stag(tsr->ucq, tsr->stag); + linux_spin_unlock_irqrestore(&ucq->trans.mru_lock, irqflags); + } + else { + kprintf("%s: WARNING: could not free stag %d for TNI %d CQ %d (UCQ is disabled)\n", + __func__, tsr->stag, ucq->tni, ucq->cqid); + } + + __tofu_stag_range_remove(vm, tsr); + } + } + ihk_mc_spinlock_unlock_noirq(&vm->tofu_stag_lock); } dkprintf("%s: UCQ (pde: %p) TNI %d, CQ %d\n", @@ -2131,6 +2339,24 @@ void tof_utofu_finalize(void) { struct tofu_globals *tg = ihk_mc_get_tofu_globals(); + /* Could be called from idle.. */ + if (cpu_local_var(current)->proc->enable_tofu) { + int i; + struct process_vm *vm = cpu_local_var(current)->vm; + struct tofu_stag_range *tsr, *next; + + for (i = 0; i < TOFU_STAG_HASH_SIZE; ++i) { + list_for_each_entry_safe(tsr, next, + &vm->tofu_stag_hash[i], hash) { + + dkprintf("%s: WARNING: stray stag %d for TNI %d CQ %d?\n", + __func__, tsr->stag, tsr->ucq->tni, tsr->ucq->cqid); + + } + } + kprintf("%s: STAG processing done\n", __func__); + } + ihk_mc_clear_kernel_range((void *)tg->linux_vmalloc_start, (void *)tg->linux_vmalloc_end); }