From ba04c8a7b9b7e5b2c98cd19e1e71044df0660c57 Mon Sep 17 00:00:00 2001
From: Balazs Gerofi <bgerofi@riken.jp>
Date: Fri, 19 Mar 2021 19:11:30 +0900
Subject: [PATCH] Fugaku: MAP_LOCKED and pre-populate PMIx shared memory PFNs

Change-Id: I74a0d0e50af0b6c60a6f9a4389ef3ab0534deda2
---
 kernel/devobj.c  | 50 +++++++++++++++++++++++++++++++++++++++++++++++-
 kernel/syscall.c | 17 +++++++++++++++-
 2 files changed, 65 insertions(+), 2 deletions(-)

diff --git a/kernel/devobj.c b/kernel/devobj.c
index c423e451..a93b0247 100644
--- a/kernel/devobj.c
+++ b/kernel/devobj.c
@@ -136,6 +136,27 @@ int devobj_create(int fd, size_t len, off_t off, struct memobj **objp, int *maxp
 	error = 0;
 	*objp = to_memobj(obj);
 	*maxprotp = result.maxprot;
+
+#ifdef ENABLE_FUGAKU_HACKS
+	/* Pre-populate device file PFNs for PMIx shared mem */
+	if (!strncmp(obj->memobj.path,
+				"/var/opt/FJSVtcs/ple/daemonif", 29)) {
+		off_t offset;
+		uintptr_t phys;
+		unsigned long flag;
+
+		for (offset = 0; offset < obj->memobj.size; offset += PAGE_SIZE) {
+			if (devobj_get_page(&obj->memobj, offset, PAGE_P2ALIGN,
+						&phys, &flag, 0) < 0) {
+				kprintf("%s: WARNING: failed to populate offset %lu in %s\n",
+						__func__, offset, obj->memobj.path);
+			}
+		}
+		dkprintf("%s: pre-populated PFNs for %s, len: %lu\n",
+			__func__, obj->memobj.path, obj->memobj.size);
+	}
+#endif
+
 	obj = NULL;
 
 out:
@@ -200,6 +221,10 @@ static int devobj_get_page(struct memobj *memobj, off_t off, int p2align, uintpt
 	uintptr_t attr;
 	ihk_mc_user_context_t ctx;
 	int ix;
+	unsigned long irqstate;
+#ifdef ENABLE_FUGAKU_HACKS
+	int page_fault_attempts = 5;
+#endif
 
 	dkprintf("devobj_get_page(%p %lx,%lx,%d)\n", memobj, obj->handle, off, p2align);
 
@@ -214,8 +239,15 @@ static int devobj_get_page(struct memobj *memobj, off_t off, int p2align, uintpt
 #ifdef PROFILE_ENABLE
 	profile_event_add(PROFILE_page_fault_dev_file, PAGE_SIZE);
 #endif // PROFILE_ENABLE
+
+	irqstate = ihk_mc_spinlock_lock(&obj->pfn_table_lock);
 	pfn = obj->pfn_table[ix];
+	ihk_mc_spinlock_unlock(&obj->pfn_table_lock, irqstate);
+
 	if (!(pfn & PFN_VALID)) {
+#ifdef ENABLE_FUGAKU_HACKS
+pf_retry:
+#endif
 		ihk_mc_syscall_arg0(&ctx) = PAGER_REQ_PFN;
 		ihk_mc_syscall_arg1(&ctx) = obj->handle;
 		ihk_mc_syscall_arg2(&ctx) = off & ~(PAGE_SIZE - 1);
@@ -241,8 +273,24 @@ static int devobj_get_page(struct memobj *memobj, off_t off, int p2align, uintpt
 			pfn |= attr;
 			dkprintf("devobj_get_page(%p %lx,%lx,%d):PFN_PRESENT after %#lx\n", memobj, obj->handle, off, p2align, pfn);
 		}
+#ifdef ENABLE_FUGAKU_HACKS
+		else if (page_fault_attempts > 0) {
+			kprintf("%s(): va: 0x%lx !PFN_PRESENT for offset %lu in %s, "
+					"page_fault_attempts: %d\n",
+					__func__, virt_addr, off,
+					memobj->path ? memobj->path : "<unknown>",
+					page_fault_attempts);
+			--page_fault_attempts;
+			goto pf_retry;
+		}
+#endif
 
-		obj->pfn_table[ix] = pfn;
+		/* Update atomically if unset */
+		irqstate = ihk_mc_spinlock_lock(&obj->pfn_table_lock);
+		if (obj->pfn_table[ix] == 0) {
+			obj->pfn_table[ix] = pfn;
+		}
+		ihk_mc_spinlock_unlock(&obj->pfn_table_lock, irqstate);
 		// Don't call memory_stat_rss_add() because devobj related pages don't reside in main memory
 	}
 
diff --git a/kernel/syscall.c b/kernel/syscall.c
index 0bcc4478..72dfceb7 100644
--- a/kernel/syscall.c
+++ b/kernel/syscall.c
@@ -2219,6 +2219,8 @@ straight_out:
 		}
 #endif // PROFILE_ENABLE
 		if (error == -ESRCH) {
+			int populate_flags = 0;
+
 			dkprintf("do_mmap:hit non VREG\n");
 			/*
 			 * XXX: temporary:
@@ -2230,8 +2232,21 @@ straight_out:
 				vrflags &= ~VR_MEMTYPE_MASK;
 				vrflags |= VR_MEMTYPE_UC;
 			}
+
+#ifdef ENABLE_FUGAKU_HACKS
+#ifdef ENABLE_TOFU
+			if (!strncmp("/var/opt/FJSVtcs/ple/daemonif/",
+						thread->proc->fd_path[fd], 30)) {
+				dkprintf("%s: MAP_POPULATE | MAP_LOCKED for %s\n",
+					__func__, thread->proc->fd_path[fd]);
+				populate_flags = (MAP_POPULATE | MAP_LOCKED);
+			}
+#endif
+#endif
+
 			error = devobj_create(fd, len, off, &memobj, &maxprot, 
-					prot, (flags & (MAP_POPULATE | MAP_LOCKED)));
+					prot,
+					populate_flags | (flags & (MAP_POPULATE | MAP_LOCKED)));
 
 			if (!error) {
 #ifdef PROFILE_ENABLE