partitioned execution: bind mcexec to corresponding NUMA node

executer/include/uprotocol.h

@@ -85,6 +85,7 @@ struct get_cpu_set_arg {
 	void *cpu_set;
 	size_t cpu_set_size;	// Size in bytes
 	int *target_core;
+	int *mcexec_linux_numa; // NUMA domain to bind mcexec to
 };
 
 #define PLD_CPU_SET_MAX_CPUS 1024

executer/kernel/mcctrl/control.c

@@ -493,6 +493,7 @@ static long mcexec_get_cpuset(ihk_os_t os, unsigned long arg)
 	struct cache_topology *cache_top;
 	int cpu, cpus_assigned, cpus_to_assign, cpu_prev;
 	int ret = 0;
+	int mcexec_linux_numa;
 	cpumask_t cpus_used;
 	cpumask_t cpus_to_use;
 	struct mcctrl_per_proc_data *ppd;
@@ -643,6 +644,7 @@ next_cpu:
 		goto put_and_unlock_out;
 	}
 
+	/* Copy IKC target core and mcexec Linux NUMA id */
 	cpu = cpumask_next(-1, &cpus_to_use);
 	if (copy_to_user(req.target_core, &cpu, sizeof(cpu))) {
 		printk("%s: error copying target core to user\n",
@@ -651,6 +653,15 @@ next_cpu:
 		goto put_and_unlock_out;
 	}
 
+	mcexec_linux_numa = cpu_to_node(mckernel_cpu_2_linux_cpu(udp, cpu));
+	if (copy_to_user(req.mcexec_linux_numa, &mcexec_linux_numa,
+				sizeof(mcexec_linux_numa))) {
+		printk("%s: error copying mcexec Linux NUMA id\n",
+				__FUNCTION__);
+		ret = -EINVAL;
+		goto put_and_unlock_out;
+	}
+
 	/* Save in per-process structure */
 	memcpy(&ppd->cpu_set, &cpus_to_use, sizeof(cpumask_t));
 	ppd->ikc_target_cpu = cpu;

executer/user/Makefile.in

@@ -11,7 +11,7 @@ IHKDIR ?= $(VPATH)/../../../ihk/linux/include/
 all: $(TARGET)
 
 mcexec: mcexec.c
-	$(CC) -I${KDIR} $(CFLAGS) $(EXTRA_CFLAGS) -fPIE -pie -lrt -pthread -o $@ $^ $(EXTRA_OBJS)
+	$(CC) -I${KDIR} $(CFLAGS) $(EXTRA_CFLAGS) -fPIE -pie -lrt -lnuma -pthread -o $@ $^ $(EXTRA_OBJS)
 
 eclair: eclair.c
 	$(CC) $(CFLAGS) -I${IHKDIR} -o $@ $^ $(LIBS)

executer/user/mcexec.c

@@ -66,6 +66,8 @@
 #include "../include/uprotocol.h"
 #include <getopt.h>
 #include "../config.h"
+#include <numa.h>
+#include <numaif.h>
 
 //#define DEBUG
 
@@ -1623,21 +1625,50 @@ int main(int argc, char **argv)
 	/* Partitioned execution, obtain CPU set */
 	if (nr_processes > 0) {
 		struct get_cpu_set_arg cpu_set_arg;
+		int mcexec_linux_numa = 0;
 
 		cpu_set_arg.cpu_set = (void *)&desc->cpu_set;
 		cpu_set_arg.cpu_set_size = sizeof(desc->cpu_set);
 		cpu_set_arg.nr_processes = nr_processes;
 		cpu_set_arg.target_core = &target_core;
-		
+		cpu_set_arg.mcexec_linux_numa = &mcexec_linux_numa;
+
 		if (ioctl(fd, MCEXEC_UP_GET_CPUSET, (void *)&cpu_set_arg) != 0) {
 			perror("getting CPU set for partitioned execution");
 			close(fd);
 			return 1;
 		}
-		
+
 		desc->cpu = target_core;
+
+		/* This call may not succeed, but that is fine */
+		if (numa_run_on_node(mcexec_linux_numa) < 0) {
+			__dprint("%s: WARNING: couldn't bind to NUMA %d\n",
+				__FUNCTION__, mcexec_linux_numa);
+		}
+#ifdef DEBUG
+		else {
+			cpu_set_t cpuset;
+			char affinity[BUFSIZ];
+
+			CPU_ZERO(&cpuset);
+			if ((sched_getaffinity(0, sizeof(cpu_set_t), &cpuset)) != 0) {
+				perror("Error sched_getaffinity");
+				exit(1);
+			}
+
+			affinity[0] = '\0';
+			for (i = 0; i < 512; i++) {
+				if (CPU_ISSET(i, &cpuset) == 1) {
+					sprintf(affinity, "%s %d", affinity, i);
+				}
+			}
+			__dprint("%s: PID: %d affinity: %s\n",
+					__FUNCTION__, getpid(), affinity);
+		}
+#endif
 	}
-	
+
 	if (ioctl(fd, MCEXEC_UP_PREPARE_IMAGE, (unsigned long)desc) != 0) {
 		perror("prepare");
 		close(fd);