Tofu: support for barrier gate, kmalloc cache

Change-Id: I6f4cfec2ec404efd03b332fc3f449a775816230e

kernel/include/process.h

@@ -750,6 +750,9 @@ struct thread {
 	void *coredump_regs;
 	struct waitq coredump_wq;
 	int coredump_status;
+
+	/* Path of file being opened */
+	char *fd_path_in_open;
 };
 
 #define VM_RANGE_CACHE_SIZE	4

kernel/include/profile.h

@@ -45,6 +45,16 @@ enum profile_event_type {
 	PROFILE_mmap_anon_no_contig_phys,
 	PROFILE_mmap_regular_file,
 	PROFILE_mmap_device_file,
+	PROFILE_tofu_stag_alloc,
+	PROFILE_tofu_stag_alloc_new_steering,
+	PROFILE_tofu_stag_alloc_new_steering_alloc_mbpt,
+	PROFILE_tofu_stag_alloc_new_steering_update_mbpt,
+	PROFILE_tofu_stag_free_stags,
+	PROFILE_tofu_stag_free_stag,
+	PROFILE_tofu_stag_free_stag_pre,
+	PROFILE_tofu_stag_free_stag_cqflush,
+	PROFILE_tofu_stag_free_stag_dealloc,
+	PROFILE_tofu_stag_free_stag_dealloc_free_pages,
 	PROFILE_EVENT_MAX	/* Should be the last event type */
 };
 

kernel/include/tofu/generate_headers.sh

@@ -15,9 +15,10 @@ if ! tar zxvf /lib/modules/`uname -r`+debug/extra/tof_module.tar.gz ${KMODULE} 2
 	exit 1
 fi
 
-${DWARF_TOOL} ${KMODULE} tof_utofu_device enabled > tofu_generated-tof_utofu_device.h
+${DWARF_TOOL} ${KMODULE} tof_utofu_device enabled subnet gpid > tofu_generated-tof_utofu_device.h
 ${DWARF_TOOL} ${KMODULE} tof_utofu_cq common tni cqid trans steering mb num_stag | sed "s/struct FILL_IN_MANUALLY trans;/#include \"tof_utofu_cq_trans.h\"/g" > tofu_generated-tof_utofu_cq.h
 ${DWARF_TOOL} ${KMODULE} tof_utofu_mbpt ucq iova sg nsgents mbptstart pgsz kref > tofu_generated-tof_utofu_mbpt.h
+${DWARF_TOOL} ${KMODULE} tof_utofu_bg common tni bgid bch | sed "s/struct FILL_IN_MANUALLY bch;/#include \"tof_utofu_bg_bch.h\"/g" > tofu_generated-tof_utofu_bg.h
 rm ${KMODULE}
 
 KMODULE=tof_core.ko 
@@ -28,6 +29,7 @@ if ! tar zxvf /lib/modules/`uname -r`+debug/extra/tof_module.tar.gz ${KMODULE} 2
 fi
 
 ${DWARF_TOOL} ${KMODULE} tof_core_cq reg | sed "s/struct FILL_IN_MANUALLY reg;/#include \"tof_core_cq_reg.h\"/g" > tofu_generated-tof_core_cq.h 
+${DWARF_TOOL} ${KMODULE} tof_core_bg lock reg irq subnet gpid sighandler | sed "s/struct FILL_IN_MANUALLY reg;/#include \"tof_core_bg_reg.h\"/g" > tofu_generated-tof_core_bg.h 
 rm ${KMODULE}
 
 #cat tofu_generated*.h

kernel/include/tofu/tof_core_bg_reg.h

@@ -0,0 +1,4 @@
+	struct {
+		void *bgs;
+		void *bch;
+	} reg;

kernel/include/tofu/tof_utofu_bg_bch.h

@@ -0,0 +1,6 @@
+	struct {
+		bool enabled;
+		uint64_t bgmask[TOF_ICC_NTNIS];
+		uintptr_t iova;
+		void *kaddr;
+	} bch;

kernel/include/tofu/tofu_generated-tof_core_bg.h

@@ -0,0 +1,29 @@
+struct tof_core_bg {
+	union {
+		char whole_struct[120];
+		struct {
+			char padding0[0];
+			spinlock_t lock;
+		};
+		struct {
+			char padding1[8];
+			#include "tof_core_bg_reg.h"
+		};
+		struct {
+			char padding2[24];
+			struct tof_core_irq irq;
+		};
+		struct {
+			char padding3[88];
+			tof_core_signal_handler sighandler;
+		};
+		struct {
+			char padding4[104];
+			uint64_t subnet;
+		};
+		struct {
+			char padding5[112];
+			uint32_t gpid;
+		};
+	};
+};

kernel/include/tofu/tofu_generated-tof_utofu_bg.h

@@ -0,0 +1,21 @@
+struct tof_utofu_bg {
+	union {
+		char whole_struct[160];
+		struct {
+			char padding0[0];
+			struct tof_utofu_device common;
+		};
+		struct {
+			char padding1[80];
+			uint8_t tni;
+		};
+		struct {
+			char padding2[81];
+			uint8_t bgid;
+		};
+		struct {
+			char padding3[88];
+			#include "tof_utofu_bg_bch.h"
+		};
+	};
+};

kernel/include/tofu/tofu_generated-tof_utofu_device.h

@@ -5,5 +5,13 @@ struct tof_utofu_device {
 			char padding0[0];
 			bool enabled;
 		};
+		struct {
+			char padding1[12];
+			uint32_t gpid;
+		};
+		struct {
+			char padding2[24];
+			uint64_t subnet;
+		};
 	};
 };

kernel/include/tofu/tofu_stag_range.h

@@ -0,0 +1,15 @@
+#ifndef __TOFU_STAG_RANGE_HEADER__
+#define __TOFU_STAG_RANGE_HEADER__
+
+struct tof_utofu_cq;
+
+struct tofu_stag_range {
+	uintptr_t start, end;
+	int stag;
+	struct tof_utofu_cq *ucq;
+	struct list_head list; // per-vm_range list
+	struct list_head hash; // per-process stag hash
+};
+
+#endif // __TOFU_STAG_RANGE_HEADER__
+

kernel/profile.c

@@ -68,7 +68,16 @@ char *profile_event_names[] =
 	"mmap_anon_no_contig_phys",
 	"mmap_regular_file",
 	"mmap_device_file",
-	""
+	"tofu_stag_alloc ",
+	"|--new_steering ",
+	"   |-alloc_mbpt ",
+	"   |-update_mbpt",
+	"tofu_stag_free_stags",
+	"tofu_stag_free_stag",
+	"   |--------pre",
+	"   |----cqflush",
+	"   |----dealloc",
+	"      |---free_pages",
 };
 
 mcs_lock_node_t job_profile_lock = { 0 };

kernel/syscall.c

@@ -381,28 +381,30 @@ long do_syscall(struct syscall_request *req, int cpu)
 
 	rc = res.ret;
 
-	if (req->number == __NR_ioctl && rc == 0) {
+	if ((req->number == __NR_ioctl && rc == 0) ||
+			(req->number == __NR_openat && rc > 0)) {
+		int fd = req->number == __NR_ioctl ? req->args[0] : rc;
+		char *path = req->number == __NR_ioctl ?
+			thread->proc->fd_path[fd] : thread->fd_path_in_open;
+
 		if (cpu_local_var(current)->proc->enable_tofu &&
 				res.pde_data &&
-				!thread->proc->fd_pde_data[req->args[0]] &&
-				!strncmp(thread->proc->fd_path[req->args[0]],
-					"/proc/tofu/dev/", 15)) {
+				fd < MAX_FD_PDE &&
+				!thread->proc->fd_pde_data[fd] &&
+				!strncmp(path, "/proc/tofu/dev/", 15)) {
+			unsigned long irqstate;
 
-			if (req->args[0] < MAX_FD_PDE) {
-				unsigned long irqstate;
+			irqstate = ihk_mc_spinlock_lock(&thread->proc->mckfd_lock);
+			thread->proc->fd_pde_data[fd] = res.pde_data;
+			ihk_mc_spinlock_unlock(&thread->proc->mckfd_lock, irqstate);
 
-				irqstate = ihk_mc_spinlock_lock(&thread->proc->mckfd_lock);
-				thread->proc->fd_pde_data[req->args[0]] = res.pde_data;
-				ihk_mc_spinlock_unlock(&thread->proc->mckfd_lock, irqstate);
-
-				kprintf("%s: PID: %d, ioctl fd: %d, filename: "
-						"%s, pde_data: 0x%lx\n",
-						__FUNCTION__,
-						thread->proc->pid,
-						req->args[0],
-						thread->proc->fd_path[req->args[0]],
-						res.pde_data);
-			}
+			dkprintf("%s: PID: %d, ioctl fd: %d, filename: "
+					"%s, pde_data: 0x%lx\n",
+					__FUNCTION__,
+					thread->proc->pid,
+					fd,
+					path,
+					res.pde_data);
 		}
 	}
 
@@ -1292,16 +1294,20 @@ void terminate(int rc, int sig)
 	mcs_rwlock_writer_unlock_noirq(&proc->update_lock, &updatelock);
 
 #ifdef ENABLE_TOFU
-	/* Tofu: clean up stags, must be done before mcexec is gone */
+	/* Tofu: cleanup, must be done before mcexec is gone */
 	if (proc->enable_tofu) {
 		int fd;
 
 		for (fd = 0; fd < MAX_FD_PDE; ++fd) {
 			/* Tofu? */
 			if (proc->enable_tofu && proc->fd_pde_data[fd]) {
-				extern void tof_utofu_release_cq(void *pde_data);
+				extern void tof_utofu_release_fd(struct process *proc, int fd);
 
-				tof_utofu_release_cq(proc->fd_pde_data[fd]);
+				if (proc->fd_path[fd]) {
+					dkprintf("%s: -> tof_utofu_release_fd() @ fd: %d (%s)\n",
+							__func__, fd, proc->fd_path[fd]);
+				}
+				tof_utofu_release_fd(proc, fd);
 				proc->fd_pde_data[fd] = NULL;
 			}
 
@@ -3865,10 +3871,10 @@ SYSCALL_DECLARE(ioctl)
 	/* Tofu? */
 	if (proc->enable_tofu &&
 			fd < MAX_FD_PDE && thread->proc->fd_pde_data[fd]) {
-		extern long tof_utofu_unlocked_ioctl_cq(int fd,
+		extern long tof_utofu_unlocked_ioctl(int fd,
 				unsigned int cmd, unsigned long arg);
 
-		rc = tof_utofu_unlocked_ioctl_cq(fd,
+		rc = tof_utofu_unlocked_ioctl(fd,
 			ihk_mc_syscall_arg1(ctx),
 			ihk_mc_syscall_arg2(ctx));
 
@@ -3913,6 +3919,8 @@ SYSCALL_DECLARE(open)
 		goto out;
 	}
 
+	cpu_local_var(current)->fd_path_in_open = pathname;
+
 	dkprintf("open(): pathname=%s\n", pathname);
 	if (!strncmp(pathname, XPMEM_DEV_PATH, len)) {
 		rc = xpmem_open(pathname, flags, ctx);
@@ -3920,6 +3928,8 @@ SYSCALL_DECLARE(open)
 		rc = syscall_generic_forwarding(__NR_open, ctx);
 	}
 
+	cpu_local_var(current)->fd_path_in_open = NULL;
+
  out:
 	if (rc > 0 && rc < MAX_FD_PDE) {
 		cpu_local_var(current)->proc->fd_path[rc] = pathname;
@@ -3954,6 +3964,8 @@ SYSCALL_DECLARE(openat)
 		goto out;
 	}
 
+	cpu_local_var(current)->fd_path_in_open = pathname;
+
 	dkprintf("openat(): pathname=%s\n", pathname);
 	if (!strncmp(pathname, XPMEM_DEV_PATH, len)) {
 		rc = xpmem_openat(pathname, flags, ctx);
@@ -3961,6 +3973,8 @@ SYSCALL_DECLARE(openat)
 		rc = syscall_generic_forwarding(__NR_openat, ctx);
 	}
 
+	cpu_local_var(current)->fd_path_in_open = NULL;
+
 out:
 	if (rc > 0 && rc < MAX_FD_PDE) {
 		cpu_local_var(current)->proc->fd_path[rc] = pathname;
@@ -4021,9 +4035,13 @@ SYSCALL_DECLARE(close)
 	if (fd >= 0 && fd < MAX_FD_PDE) {
 		/* Tofu? */
 		if (thread->proc->fd_pde_data[fd]) {
-			extern void tof_utofu_release_cq(void *pde_data);
+			extern void tof_utofu_release_fd(struct process *proc, int fd);
 
-			tof_utofu_release_cq(thread->proc->fd_pde_data[fd]);
+			if (thread->proc->fd_path[fd]) {
+				dkprintf("%s: -> tof_utofu_release_fd() @ fd: %d (%s)\n",
+						__func__, fd, thread->proc->fd_path[fd]);
+			}
+			tof_utofu_release_fd(thread->proc, fd);
 			thread->proc->fd_pde_data[fd] = NULL;
 		}