From d5629606c5601bd461307c44318290e4da84251c Mon Sep 17 00:00:00 2001
From: Balazs Gerofi <bgerofi@riken.jp>
Date: Wed, 18 Oct 2017 16:39:48 +0900
Subject: [PATCH] mcexec: -m: interpret as numactl -m (i.e., MPOL_BIND)

Conflicts:
	executer/include/uprotocol.h
	executer/user/mcexec.c
	kernel/include/syscall.h
---
 executer/include/uprotocol.h |  1 +
 executer/user/mcexec.c       | 24 ++++++++++++++++++++++--
 kernel/host.c                | 21 +++++++++++++++++++++
 kernel/include/process.h     |  1 +
 kernel/include/syscall.h     |  1 +
 5 files changed, 46 insertions(+), 2 deletions(-)

diff --git a/executer/include/uprotocol.h b/executer/include/uprotocol.h
index e4150f82..5aebd6c2 100644
--- a/executer/include/uprotocol.h
+++ b/executer/include/uprotocol.h
@@ -141,6 +141,7 @@ struct program_load_desc {
 	unsigned long mpol_threshold;
 	unsigned long heap_extension;
 	long stack_premap;
+	unsigned long mpol_bind_mask;
 	int nr_processes;
 	char shell_path[SHELL_PATH_MAX_LEN];
 	__cpu_set_unit cpu_set[PLD_CPU_SET_SIZE];
diff --git a/executer/user/mcexec.c b/executer/user/mcexec.c
index 1d064d63..613390ad 100644
--- a/executer/user/mcexec.c
+++ b/executer/user/mcexec.c
@@ -217,6 +217,7 @@ static int disable_sched_yield = 0;
 static long stack_premap = (2ULL << 20);
 static long stack_max = -1;
 static struct rlimit rlim_stack;
+static char *mpol_bind_nodes = NULL;
 
 /* Partitioned execution (e.g., for MPI) */
 static int nr_processes = 0;
@@ -1869,9 +1870,9 @@ int main(int argc, char **argv)
 
 	/* Parse options ("+" denotes stop at the first non-option) */
 #ifdef ADD_ENVS_OPTION
-	while ((opt = getopt_long(argc, argv, "+c:n:t:M:h:e:s:", mcexec_options, NULL)) != -1) {
+	while ((opt = getopt_long(argc, argv, "+c:n:t:M:h:e:s:m:", mcexec_options, NULL)) != -1) {
 #else /* ADD_ENVS_OPTION */
-	while ((opt = getopt_long(argc, argv, "+c:n:t:M:h:s:", mcexec_options, NULL)) != -1) {
+	while ((opt = getopt_long(argc, argv, "+c:n:t:M:h:s:m:", mcexec_options, NULL)) != -1) {
 #endif /* ADD_ENVS_OPTION */
 		switch (opt) {
 			char *tmp;
@@ -1904,6 +1905,10 @@ int main(int argc, char **argv)
 				mpol_threshold = atobytes(optarg);
 				break;
 
+			case 'm':
+				mpol_bind_nodes = optarg;
+				break;
+
 			case 'h':
 				heap_extension = atobytes(optarg);
 				break;
@@ -2388,6 +2393,21 @@ int main(int argc, char **argv)
 	desc->mpol_threshold = mpol_threshold;
 	desc->heap_extension = heap_extension;
 
+	desc->mpol_bind_mask = 0;
+	if (mpol_bind_nodes) {
+		struct bitmask *bind_mask;
+		bind_mask = numa_parse_nodestring_all(mpol_bind_nodes);
+
+		if (bind_mask) {
+			int node;
+			for (node = 0; node <= numa_max_possible_node(); ++node) {
+				if (numa_bitmask_isbitset(bind_mask, node)) {
+					desc->mpol_bind_mask |= (1UL << node);
+				}
+			}
+		}
+	}
+
 	if (ioctl(fd, MCEXEC_UP_PREPARE_IMAGE, (unsigned long)desc) != 0) {
 		perror("prepare");
 		close(fd);
diff --git a/kernel/host.c b/kernel/host.c
index 8ebcc967..c9d2759f 100644
--- a/kernel/host.c
+++ b/kernel/host.c
@@ -476,6 +476,27 @@ static int process_msg_prepare_process(unsigned long rphys)
 	proc->mpol_threshold = pn->mpol_threshold;
 	proc->nr_processes = pn->nr_processes;
 	proc->heap_extension = pn->heap_extension;
+
+	/* Update NUMA binding policy if requested */
+	if (pn->mpol_bind_mask) {
+		int bit;
+
+		memset(&vm->numa_mask, 0, sizeof(vm->numa_mask));
+
+		for_each_set_bit(bit, &pn->mpol_bind_mask,
+				sizeof(pn->mpol_bind_mask) * BITS_PER_BYTE) {
+
+			if (bit >= ihk_mc_get_nr_numa_nodes()) {
+				kprintf("%s: error: NUMA id %d is larger than mask size!\n",
+						__FUNCTION__, bit);
+				return -EINVAL;
+			}
+
+			set_bit(bit, &vm->numa_mask[0]);
+		}
+		vm->numa_mem_policy = MPOL_BIND;
+	}
+
 #ifdef PROFILE_ENABLE
 	proc->profile = pn->profile;
 	thread->profile = pn->profile;
diff --git a/kernel/include/process.h b/kernel/include/process.h
index 87a7e059..1b7dbe56 100644
--- a/kernel/include/process.h
+++ b/kernel/include/process.h
@@ -553,6 +553,7 @@ struct process {
 	unsigned long mpol_flags;
 	size_t mpol_threshold;
 	unsigned long heap_extension;
+	unsigned long mpol_bind_mask;
 
 	// perf_event
 	int perf_status;
diff --git a/kernel/include/syscall.h b/kernel/include/syscall.h
index 0d64226f..d2b9044f 100644
--- a/kernel/include/syscall.h
+++ b/kernel/include/syscall.h
@@ -198,6 +198,7 @@ struct program_load_desc {
 	unsigned long mpol_threshold;
 	unsigned long heap_extension;
 	long stack_premap;
+	unsigned long mpol_bind_mask;
 	int nr_processes;
 	char shell_path[SHELL_PATH_MAX_LEN];
 	__cpu_set_unit cpu_set[PLD_CPU_SET_SIZE];