release: 1.8.0: MAP_LOCKED and pre-populate PMIx shared memory PFNs

Change-Id: I171c87f0f49cf2f791693e397a1d94b1bc2d0440

.gitmodules

@@ -1,12 +1,12 @@
 [submodule "ihk"]
 	path = ihk
-	url = https://github.com/RIKEN-SysSoft/ihk.git
+	url = https://github.com/ihkmckernel/ihk.git
 [submodule "executer/user/lib/libdwarf/libdwarf"]
 	path = executer/user/lib/libdwarf/libdwarf
 	url = https://github.com/bgerofi/libdwarf.git
 [submodule "executer/user/lib/syscall_intercept"]
 	path = executer/user/lib/syscall_intercept
-	url = https://github.com/RIKEN-SysSoft/syscall_intercept.git
+	url = https://github.com/ihkmckernel/syscall_intercept.git
 [submodule "executer/user/lib/uti"]
 	path = executer/user/lib/uti
-	url = https://github.com/RIKEN-SysSoft/uti.git
+	url = https://github.com/ihkmckernel/uti.git

CMakeLists.txt

@@ -7,7 +7,7 @@ endif (NOT CMAKE_BUILD_TYPE)
 enable_language(C ASM)
 
 project(mckernel C ASM)
-set(MCKERNEL_VERSION "1.7.10")
+set(MCKERNEL_VERSION "1.8.0")
 
 # See "Fedora Packaging Guidelines -- Versioning"
 set(MCKERNEL_RELEASE "")

docs/NEWS.rst

@@ -1,3 +1,28 @@
+=============================================
+Version 1.8.0 (Mar 23, 2021)
+=============================================
+
+----------------------
+IHK major updates
+----------------------
+N/A
+
+------------------------
+IHK major bug fixes
+------------------------
+N/A
+
+----------------------
+McKernel major updates
+----------------------
+N/A
+
+------------------------
+McKernel major bug fixes
+------------------------
+#. profile: fix infinite recursion for allocation miss event
+#. Fugaku: MAP_LOCKED and pre-populate PMIx shared memory PFNs
+
 =============================================
 Version 1.7.10 (Mar 18, 2021)
 =============================================

kernel/devobj.c

@@ -136,6 +136,27 @@ int devobj_create(int fd, size_t len, off_t off, struct memobj **objp, int *maxp
 	error = 0;
 	*objp = to_memobj(obj);
 	*maxprotp = result.maxprot;
+
+#ifdef ENABLE_FUGAKU_HACKS
+	/* Pre-populate device file PFNs for PMIx shared mem */
+	if (!strncmp(obj->memobj.path,
+				"/var/opt/FJSVtcs/ple/daemonif", 29)) {
+		off_t offset;
+		uintptr_t phys;
+		unsigned long flag;
+
+		for (offset = 0; offset < obj->memobj.size; offset += PAGE_SIZE) {
+			if (devobj_get_page(&obj->memobj, offset, PAGE_P2ALIGN,
+						&phys, &flag, 0) < 0) {
+				kprintf("%s: WARNING: failed to populate offset %lu in %s\n",
+						__func__, offset, obj->memobj.path);
+			}
+		}
+		dkprintf("%s: pre-populated PFNs for %s, len: %lu\n",
+			__func__, obj->memobj.path, obj->memobj.size);
+	}
+#endif
+
 	obj = NULL;
 
 out:
@@ -200,6 +221,10 @@ static int devobj_get_page(struct memobj *memobj, off_t off, int p2align, uintpt
 	uintptr_t attr;
 	ihk_mc_user_context_t ctx;
 	int ix;
+	unsigned long irqstate;
+#ifdef ENABLE_FUGAKU_HACKS
+	int page_fault_attempts = 5;
+#endif
 
 	dkprintf("devobj_get_page(%p %lx,%lx,%d)\n", memobj, obj->handle, off, p2align);
 
@@ -214,8 +239,15 @@ static int devobj_get_page(struct memobj *memobj, off_t off, int p2align, uintpt
 #ifdef PROFILE_ENABLE
 	profile_event_add(PROFILE_page_fault_dev_file, PAGE_SIZE);
 #endif // PROFILE_ENABLE
+
+	irqstate = ihk_mc_spinlock_lock(&obj->pfn_table_lock);
 	pfn = obj->pfn_table[ix];
+	ihk_mc_spinlock_unlock(&obj->pfn_table_lock, irqstate);
+
 	if (!(pfn & PFN_VALID)) {
+#ifdef ENABLE_FUGAKU_HACKS
+pf_retry:
+#endif
 		ihk_mc_syscall_arg0(&ctx) = PAGER_REQ_PFN;
 		ihk_mc_syscall_arg1(&ctx) = obj->handle;
 		ihk_mc_syscall_arg2(&ctx) = off & ~(PAGE_SIZE - 1);
@@ -241,8 +273,24 @@ static int devobj_get_page(struct memobj *memobj, off_t off, int p2align, uintpt
 			pfn |= attr;
 			dkprintf("devobj_get_page(%p %lx,%lx,%d):PFN_PRESENT after %#lx\n", memobj, obj->handle, off, p2align, pfn);
 		}
+#ifdef ENABLE_FUGAKU_HACKS
+		else if (page_fault_attempts > 0) {
+			kprintf("%s(): va: 0x%lx !PFN_PRESENT for offset %lu in %s, "
+					"page_fault_attempts: %d\n",
+					__func__, virt_addr, off,
+					memobj->path ? memobj->path : "<unknown>",
+					page_fault_attempts);
+			--page_fault_attempts;
+			goto pf_retry;
+		}
+#endif
 
-		obj->pfn_table[ix] = pfn;
+		/* Update atomically if unset */
+		irqstate = ihk_mc_spinlock_lock(&obj->pfn_table_lock);
+		if (obj->pfn_table[ix] == 0) {
+			obj->pfn_table[ix] = pfn;
+		}
+		ihk_mc_spinlock_unlock(&obj->pfn_table_lock, irqstate);
 		// Don't call memory_stat_rss_add() because devobj related pages don't reside in main memory
 	}
 

kernel/profile.c

@@ -100,8 +100,13 @@ void profile_event_add(enum profile_event_type type, uint64_t tsc)
 		return;
 
 	if (!cpu_local_var(current)->profile_events) {
-		if (profile_alloc_events(cpu_local_var(current)) < 0)
+		if (type == PROFILE_mpol_alloc_missed) {
 			return;
+		}
+
+		if (profile_alloc_events(cpu_local_var(current)) < 0) {
+			return;
+		}
 	}
 
 	if (type < PROFILE_EVENT_MAX) {

kernel/syscall.c

@@ -2219,6 +2219,8 @@ straight_out:
 		}
 #endif // PROFILE_ENABLE
 		if (error == -ESRCH) {
+			int populate_flags = 0;
+
 			dkprintf("do_mmap:hit non VREG\n");
 			/*
 			 * XXX: temporary:
@@ -2230,8 +2232,21 @@ straight_out:
 				vrflags &= ~VR_MEMTYPE_MASK;
 				vrflags |= VR_MEMTYPE_UC;
 			}
+
+#ifdef ENABLE_FUGAKU_HACKS
+#ifdef ENABLE_TOFU
+			if (!strncmp("/var/opt/FJSVtcs/ple/daemonif/",
+						thread->proc->fd_path[fd], 30)) {
+				dkprintf("%s: MAP_POPULATE | MAP_LOCKED for %s\n",
+					__func__, thread->proc->fd_path[fd]);
+				populate_flags = (MAP_POPULATE | MAP_LOCKED);
+			}
+#endif
+#endif
+
 			error = devobj_create(fd, len, off, &memobj, &maxprot, 
-					prot, (flags & (MAP_POPULATE | MAP_LOCKED)));
+					prot,
+					populate_flags | (flags & (MAP_POPULATE | MAP_LOCKED)));
 
 			if (!error) {
 #ifdef PROFILE_ENABLE