From 73e1a4f1f9bfb4a3b85a1b908811f62047872024 Mon Sep 17 00:00:00 2001
From: Balazs Gerofi <bgerofi@riken.jp>
Date: Fri, 14 Oct 2016 07:32:01 +0900
Subject: [PATCH] NUMA: fill in /sys/devices/system/cpu/nodeX properly and sync
 with boot script

---
 arch/x86/tools/mcreboot-smp-x86.sh.in |  9 ++++++-
 executer/kernel/mcctrl/sysfs_files.c  | 35 +++++++++++++++++++++++++++
 2 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/arch/x86/tools/mcreboot-smp-x86.sh.in b/arch/x86/tools/mcreboot-smp-x86.sh.in
index b80c9c87..9b8783a6 100644
--- a/arch/x86/tools/mcreboot-smp-x86.sh.in
+++ b/arch/x86/tools/mcreboot-smp-x86.sh.in
@@ -205,7 +205,7 @@ if [ "$enable_mcoverlay" == "yes" ]; then
 	if ! mount -t mcoverlay mcoverlay -o lowerdir=/proc/mcos0:/proc,upperdir=/tmp/mcos/mcos0_proc_upper,workdir=/tmp/mcos/mcos0_proc_work,nocopyupw,nofscheck /tmp/mcos/mcos0_proc; then echo "error: mount /tmp/mcos/mcos0_proc" >&2; exit 1; fi
 	mount --make-rprivate /proc
 
-	while [ ! -e /sys/devices/virtual/mcos/mcos0/sys/devices/system/node ]
+	while [ ! -e /sys/devices/virtual/mcos/mcos0/sys/setup_complete ]
 	do
 		sleep 1
 	done
@@ -214,9 +214,16 @@ if [ "$enable_mcoverlay" == "yes" ]; then
 	if [ ! -e /tmp/mcos/mcos0_sys_work ]; then mkdir -p /tmp/mcos/mcos0_sys_work; fi
 	if ! mount -t mcoverlay mcoverlay -o lowerdir=/sys/devices/virtual/mcos/mcos0/sys:/sys,upperdir=/tmp/mcos/mcos0_sys_upper,workdir=/tmp/mcos/mcos0_sys_work,nocopyupw,nofscheck /tmp/mcos/mcos0_sys; then echo "error: mount /tmp/mcos/mcos0_sys" >&2; exit 1; fi
 	mount --make-rprivate /sys
+	rm -rf /tmp/mcos/mcos0_sys/setup_complete
 	for cpuid in `find /sys/devices/system/cpu/* -maxdepth 0 -name "cpu[0123456789]*" -printf "%f "`; do
 		if [ ! -e "/sys/devices/virtual/mcos/mcos0/sys/devices/system/cpu/$cpuid" ]; then
 			rm -rf /tmp/mcos/mcos0_sys/devices/system/cpu/$cpuid
+		else
+			for nodeid in `find /sys/devices/system/cpu/$cpuid/* -maxdepth 0 -name "node[0123456789]*" -printf "%f "`; do
+				if [ ! -e "/sys/devices/virtual/mcos/mcos0/sys/devices/system/cpu/$cpuid/$nodeid" ]; then
+					rm -f /tmp/mcos/mcos0_sys/devices/system/cpu/$cpuid/$nodeid
+				fi
+			done
 		fi
 	done
 	for nodeid in `find /sys/devices/system/node/* -maxdepth 0 -name "node[0123456789]*" -printf "%f "`; do
diff --git a/executer/kernel/mcctrl/sysfs_files.c b/executer/kernel/mcctrl/sysfs_files.c
index 2f59e8e1..f6324dc2 100644
--- a/executer/kernel/mcctrl/sysfs_files.c
+++ b/executer/kernel/mcctrl/sysfs_files.c
@@ -538,6 +538,34 @@ static void setup_cpu_sysfs_files(struct mcctrl_usrdata *udp,
 	return;
 } /* setup_cpu_sysfs_files() */
 
+static void setup_cpus_sysfs_files_node_link(struct mcctrl_usrdata *udp)
+{
+	int error;
+	int cpu;
+	struct sysfs_handle handle;
+
+	for (cpu = 0; cpu < udp->cpu_info->n_cpus; ++cpu) {
+		int node = linux_numa_2_mckernel_numa(udp,
+				cpu_to_node(mckernel_cpu_2_linux_cpu(udp, cpu)));
+
+		error = sysfsm_lookupf(udp->os, &handle,
+				"/sys/devices/system/node/node%d", node);
+		if (error) {
+			panic("sysfsm_lookupf: node for CPU");
+		}
+
+		error = sysfsm_symlinkf(udp->os, handle,
+				"/sys/devices/system/cpu/cpu%d/node%d",
+				cpu, node);
+		if (error) {
+			panic("sysfsm_symlinkf(CPU in node)");
+		}
+	}
+
+	error = 0;
+	return;
+}
+
 static void setup_cpus_sysfs_files(struct mcctrl_usrdata *udp)
 {
 	int error;
@@ -1028,8 +1056,15 @@ void setup_sysfs_files(ihk_os_t os)
 	setup_local_snooping_files(os);
 	setup_cpus_sysfs_files(udp);
 	setup_node_files(udp);
+	setup_cpus_sysfs_files_node_link(udp);
 	setup_pci_files(udp);
 
+	/* Indicate sysfs files setup completion for boot script */
+	error = sysfsm_mkdirf(os, NULL, "/sys/setup_complete");
+	if (error) {
+		panic("sysfsm_mkdir(complete)");
+	}
+
 	return;
 } /* setup_files() */