Tofu: manage stag ranges in VM range split and misc cleanup

Conflicts:
	kernel/process.c

Change-Id: I480850fe93a7963a5bd4d1687fb1e5c43f58057f

kernel/include/tofu/generate_headers.sh

@@ -6,10 +6,25 @@ CURRENT_DIR=`pwd`
 
 cd ${SCRIPT_DIR}
 
-DWARF_TOOL=~/src/mckernel-apollo+a64fx/mckernel/tools/dwarf-extract-struct/dwarf-extract-struct
+DWARF_TOOL=${SCRIPT_DIR}/../../../tools/dwarf-extract-struct/dwarf-extract-struct
+if [ ! -x ${DWARF_TOOL} ]; then
+	echo "error: couldn't find DWARF extractor executable (${DWARF_TOOL}), have you compiled it?"
+	cd -
+	exit 1
+fi
+
+echo "Looking for Tofu driver debug symbols..."
+if [ "`find /lib/modules/ -name "tof_module.tar.gz" | xargs -r ls -t | head -n 1 | wc -l`" == "0" ]; then
+	echo "error: couldn't find Tofu modules with debug symbols"
+	cd -
+	exit 1
+fi
+
+MODULE_TAR_GZ=`find /lib/modules/ -name "tof_module.tar.gz" | xargs ls -t | head -n 1`
+echo "Using Tofu driver debug symbols: ${MODULE_TAR_GZ}"
 
 KMODULE=tof_utofu.ko 
-if ! tar zxvf /lib/modules/`uname -r`+debug/extra/tof_module.tar.gz ${KMODULE} 2>&1 > /dev/null; then
+if ! tar zxvf ${MODULE_TAR_GZ} ${KMODULE} 2>&1 > /dev/null; then
 	echo "error: uncompressing kernel module with debug symbols"
 	cd -
 	exit 1
@@ -22,7 +37,7 @@ ${DWARF_TOOL} ${KMODULE} tof_utofu_bg common tni bgid bch | sed "s/struct FILL_I
 rm ${KMODULE}
 
 KMODULE=tof_core.ko 
-if ! tar zxvf /lib/modules/`uname -r`+debug/extra/tof_module.tar.gz ${KMODULE} 2>&1 > /dev/null; then
+if ! tar zxvf ${MODULE_TAR_GZ} ${KMODULE} 2>&1 > /dev/null; then
 	echo "error: uncompressing kernel module with debug symbols"
 	cd -
 	exit 1
@@ -33,4 +48,4 @@ ${DWARF_TOOL} ${KMODULE} tof_core_bg lock reg irq subnet gpid sighandler | sed "
 rm ${KMODULE}
 
 #cat tofu_generated*.h
-cd -
+cd - > /dev/null

kernel/process.c

@@ -965,8 +965,18 @@ int split_process_memory_range(struct process_vm *vm, struct vm_range *range,
 	newrange->private_data = range->private_data;
 
 #ifdef ENABLE_TOFU
-	/* TODO: figure out which entries to put on which list! */
 	INIT_LIST_HEAD(&newrange->tofu_stag_list);
+	{
+		extern int tofu_stag_split_vm_range_on_addr(struct process_vm *vm,
+				struct vm_range *range_low, struct vm_range *range_high,
+				uintptr_t addr);
+
+		int moved =
+			tofu_stag_split_vm_range_on_addr(vm, range, newrange, addr);
+		if (moved > 0) {
+			kprintf("%s: moved %d stag ranges\n", __func__, moved);
+		}
+	}
 #endif
 
 	if (range->memobj) {
@@ -1182,7 +1192,7 @@ straight_out:
 
 		entries = tofu_stag_range_remove_overlapping(vm, range);
 		if (entries > 0) {
-			kprintf("%s: removed %d Tofu stag entries for range 0x%lx:%lu\n",
+			dkprintf("%s: removed %d Tofu stag entries for range 0x%lx:%lu\n",
 				__func__,
 				entries,
 				range->start,

kernel/tofu/tof_utofu_main.c

@@ -160,6 +160,83 @@ int tofu_stag_range_remove_overlapping(struct process_vm *vm,
 	return entries;
 }
 
+void tofu_stag_range_remove_by_addr(struct process_vm *vm,
+	uintptr_t addr, size_t len)
+{
+	struct tofu_stag_range *tsr, *next;
+	int hash;
+
+	ihk_mc_spinlock_lock_noirq(&vm->tofu_stag_lock);
+	for (hash = 0; hash < TOFU_STAG_HASH_SIZE; ++hash) {
+		list_for_each_entry_safe(tsr, next,
+				&vm->tofu_stag_hash[hash], hash) {
+
+			if (tsr->start >= addr && tsr->end <= (addr + len)) {
+				linux_spin_lock(&tsr->ucq->trans.mru_lock);
+				tof_utofu_free_stag(tsr->ucq, tsr->stag);
+				linux_spin_unlock(&tsr->ucq->trans.mru_lock);
+
+				kprintf("%s: removed stag %d in %p:%lu\n",
+						__func__, tsr->stag, addr, len);
+				__tofu_stag_range_remove(vm, tsr);
+			}
+
+			{
+				uintptr_t max_start, min_end;
+
+				max_start = addr > tsr->start ? addr : tsr->start;
+				min_end = (addr + len) < tsr->end ? (addr + len) : tsr->end;
+
+				if ((tsr->start != 0 || vm->proc->status == PS_EXITED) &&
+						(max_start < min_end)) {
+					linux_spin_lock(&tsr->ucq->trans.mru_lock);
+					tof_utofu_free_stag(tsr->ucq, tsr->stag);
+					linux_spin_unlock(&tsr->ucq->trans.mru_lock);
+
+					kprintf("%s: removed stag %p:%lu (overlaps with range %p:%lu)\n",
+						__func__, tsr->start, (tsr->end - tsr->start), addr, len);
+					__tofu_stag_range_remove(vm, tsr);
+				}
+			}
+		}
+	}
+	ihk_mc_spinlock_unlock_noirq(&vm->tofu_stag_lock);
+}
+
+int tofu_stag_split_vm_range_on_addr(struct process_vm *vm,
+		struct vm_range *range_low, struct vm_range *range_high,
+		uintptr_t addr)
+{
+	struct tofu_stag_range *tsr, *next;
+	int moved = 0;
+
+	ihk_mc_spinlock_lock_noirq(&vm->tofu_stag_lock);
+
+	list_for_each_entry_safe(tsr, next,
+			&range_low->tofu_stag_list, list) {
+
+		if (tsr->start >= addr) {
+			list_del(&tsr->list);
+			list_add_tail(&tsr->list, &range_high->tofu_stag_list);
+			++moved;
+
+			kprintf("%s: stag: %d @ %p:%lu moved to high range..\n",
+					__func__,
+					tsr->stag,
+					tsr->start,
+					(unsigned long)(tsr->end - tsr->start));
+		}
+
+		if (tsr->start < addr && tsr->end > addr) {
+			kprintf("%s: WARNING: VM range split in middle of stag range..\n", __func__);
+		}
+	}
+
+	ihk_mc_spinlock_unlock_noirq(&vm->tofu_stag_lock);
+
+	return moved;
+}
+
 
 
 #define TOF_UTOFU_VERSION TOF_UAPI_VERSION
@@ -1463,7 +1540,8 @@ static int tof_utofu_free_stag(struct tof_utofu_cq *ucq, int stag){
 #endif // PROFILE_ENABLE
 	kref_put(&ucq->trans.mru[stag].mbpt->kref, tof_utofu_mbpt_release);
 	ucq->trans.mru[stag].mbpt = NULL;
-	dkprintf("%s: stag: %d deallocated\n", __func__, stag);
+	dkprintf("%s: TNI: %d, CQ: %d, STAG: %d deallocated\n",
+			__func__, ucq->tni, ucq->cqid, stag);
 #ifdef PROFILE_ENABLE
 	profile_event_add(PROFILE_tofu_stag_free_stag_dealloc, rdtsc() - ts_rolling);
 	profile_event_add(PROFILE_tofu_stag_free_stag, rdtsc() - ts);
@@ -1486,7 +1564,7 @@ static int tof_utofu_ioctl_free_stags(struct tof_utofu_device *dev, unsigned lon
 		return -EFAULT;
 	}
 	//tof_log_if("[IN] tni=%d cqid=%d num=%u stags=%p\n", ucq->tni, ucq->cqid, req.num, req.stags);
-	dkprintf("%: [IN] tni=%d cqid=%d num=%u stags=%p\n",
+	dkprintf("%s: [IN] tni=%d cqid=%d num=%u stags=%p\n",
 			__func__, ucq->tni, ucq->cqid, req.num, req.stags);
 
 	if(req.num > 1024 || req.stags == NULL){
@@ -1559,10 +1637,8 @@ static int tof_utofu_ioctl_free_stags(struct tof_utofu_device *dev, unsigned lon
 void tof_utofu_release_cq(void *pde_data)
 {
 	struct tof_utofu_cq *ucq;
-	//int stag;
 	struct tof_utofu_device *dev;
 	unsigned long irqflags;
-	struct process_vm *vm = cpu_local_var(current)->vm;
 	int do_free = 1;
 
 	dev = (struct tof_utofu_device *)pde_data;
@@ -1574,15 +1650,10 @@ void tof_utofu_release_cq(void *pde_data)
 		do_free = 0;
 	}
 
-#if 0
-	for (stag = 0; stag < TOF_UTOFU_NUM_STAG(ucq->num_stag); stag++) {
-		linux_spin_lock_irqsave(&ucq->trans.mru_lock, irqflags);
-		tof_utofu_free_stag(ucq, stag);
-		linux_spin_unlock_irqrestore(&ucq->trans.mru_lock, irqflags);
-#endif
 	{
 		int i;
 		struct tofu_stag_range *tsr, *next;
+		struct process_vm *vm = cpu_local_var(current)->vm;
 
 		ihk_mc_spinlock_lock_noirq(&vm->tofu_stag_lock);
 		for (i = 0; i < TOFU_STAG_HASH_SIZE; ++i) {
@@ -1610,6 +1681,17 @@ void tof_utofu_release_cq(void *pde_data)
 		ihk_mc_spinlock_unlock_noirq(&vm->tofu_stag_lock);
 	}
 
+	/* Loop through as well just to make sure everything is cleaned up */
+	if (do_free) {
+		int stag;
+
+		for (stag = 0; stag < TOF_UTOFU_NUM_STAG(ucq->num_stag); stag++) {
+			linux_spin_lock_irqsave(&ucq->trans.mru_lock, irqflags);
+			tof_utofu_free_stag(ucq, stag);
+			linux_spin_unlock_irqrestore(&ucq->trans.mru_lock, irqflags);
+		}
+	}
+
 	dkprintf("%s: UCQ (pde: %p) TNI %d, CQ %d\n",
 		__func__, pde_data, ucq->tni, ucq->cqid);
 }
@@ -2356,9 +2438,10 @@ void tof_utofu_finalize(void)
 			list_for_each_entry_safe(tsr, next,
 					&vm->tofu_stag_hash[i], hash) {
 
-				dkprintf("%s: WARNING: stray stag %d for TNI %d CQ %d?\n",
+				dkprintf("%s: WARNING: stray stag %d (%p:%lu) for TNI %d CQ %d?\n",
-						__func__, tsr->stag, tsr->ucq->tni, tsr->ucq->cqid);
+						__func__, tsr->stag,
-
+						tsr->start, tsr->end - tsr->start,
+						tsr->ucq->tni, tsr->ucq->cqid);
 			}
 		}
 		kprintf("%s: STAG processing done\n", __func__);