fix: memory leak due to forced termination during startup

Change-Id: Ide519f01702bfd17ae4576e04806b6d155ae846a
refs: #1397

executer/kernel/mcctrl/control.c

@@ -86,8 +86,19 @@ int syscall_backward(struct mcctrl_usrdata *, int, unsigned long, unsigned long,
                      unsigned long, unsigned long, unsigned long,
                      unsigned long, unsigned long *);
 
+struct mcos_handler_info {
+	int pid;
+	int cpu;
+	struct mcctrl_usrdata *ud;
+	struct file *file;
+	unsigned long user_start;
+	unsigned long user_end;
+	unsigned long prepare_thread;
+};
+
 static long mcexec_prepare_image(ihk_os_t os,
-                                 struct program_load_desc * __user udesc)
+				struct program_load_desc * __user udesc,
+				struct file *file)
 {
 	struct program_load_desc *desc = NULL;
 	struct program_load_desc *pdesc = NULL;
@@ -99,6 +110,7 @@ static long mcexec_prepare_image(ihk_os_t os,
 	struct mcctrl_per_proc_data *ppd = NULL;
 	int num_sections;
 	int free_ikc_pointers = 1;
+	struct mcos_handler_info *info;
 
 	if (!usrdata) {
 		pr_err("%s: error: mcctrl_usrdata not found\n", __func__);
@@ -121,6 +133,14 @@ static long mcexec_prepare_image(ihk_os_t os,
 		goto free_out;
 	}
 
+	info = ihk_os_get_mcos_private_data(file);
+	if (!info) {
+		ret = -EFAULT;
+		goto free_out;
+	}
+	/* To serialize SCD_MSG_SCHEDULE_PROCESS and SCD_MSG_CLEANUP_PROCESS */
+	info->cpu = desc->cpu;
+
 	ppd = mcctrl_get_per_proc_data(usrdata, desc->pid);
 	if (!ppd) {
 		printk("%s: ERROR: no per process data for PID %d\n",
@@ -192,6 +212,11 @@ static long mcexec_prepare_image(ihk_os_t os,
 		/* either send or remote prepare_process failed */
 		goto put_and_free_out;
 	}
+	/*
+	 * Used as SCD_MSG_CLEANUP_PROCESS target which isn't scheduled
+	 * with SCD_MSG_SCHEDULE_PROCESS
+	 */
+	info->prepare_thread = pdesc->rprocess;
 
 	/* Update rpgtable */
 	ppd->rpgtable = pdesc->rpgtable;
@@ -306,15 +331,6 @@ int mcexec_transfer_image(ihk_os_t os, struct remote_transfer *__user upt)
 #endif
 }
 
-struct mcos_handler_info {
-	int pid;
-	int cpu;
-	struct mcctrl_usrdata *ud;
-	struct file *file;
-	unsigned long user_start;
-	unsigned long user_end;
-};
-
 struct mcos_handler_info;
 static LIST_HEAD(host_threads); /* Used for FS switch */
 DEFINE_RWLOCK(host_thread_lock);
@@ -379,6 +395,7 @@ static void release_handler(ihk_os_t os, void *param)
 	memset(&isp, '\0', sizeof isp);
 	isp.msg = SCD_MSG_CLEANUP_PROCESS;
 	isp.pid = info->pid;
+	isp.arg = info->prepare_thread;
 
 	dprintk("%s: SCD_MSG_CLEANUP_PROCESS, info: %p, cpu: %d\n",
 			__FUNCTION__, info, info->cpu);
@@ -414,6 +431,7 @@ static long mcexec_start_image(ihk_os_t os,
 	struct mcctrl_channel *c;
 	struct mcctrl_usrdata *usrdata = ihk_host_os_get_usrdata(os);
 	struct mcos_handler_info *info;
+	struct mcos_handler_info *prev_info;
 	int ret = 0;
 
 	if (!usrdata) {
@@ -434,6 +452,7 @@ static long mcexec_start_image(ihk_os_t os,
 		goto out;
 	}
 
+	prev_info = ihk_os_get_mcos_private_data(file);
 	info = new_mcos_handler_info(os, file);
 	if (info == NULL) {
 		ret = -ENOMEM;
@@ -444,6 +463,7 @@ static long mcexec_start_image(ihk_os_t os,
 	info->cpu = desc->cpu;
 	info->user_start = desc->user_start;
 	info->user_end = desc->user_end;
+	info->prepare_thread = prev_info->prepare_thread;
 	ihk_os_register_release_handler(file, release_handler, info);
 	ihk_os_set_mcos_private_data(file, info);
 
@@ -460,8 +480,10 @@ static long mcexec_start_image(ihk_os_t os,
 	ret = mcctrl_ikc_send(os, desc->cpu, &isp);
 	if (ret < 0) {
 		printk("%s: error: sending IKC msg\n", __FUNCTION__);
+		goto out;
 	}
-
+	/* clear prepared thread struct */
+	info->prepare_thread = 0;
 out:
 	kfree(desc);
 	return ret;
@@ -3224,7 +3246,8 @@ long __mcctrl_control(ihk_os_t os, unsigned int req, unsigned long arg,
 	switch (req) {
 	case MCEXEC_UP_PREPARE_IMAGE:
 		return mcexec_prepare_image(os,
-		                            (struct program_load_desc *)arg);
+					(struct program_load_desc *)arg,
+					file);
 	case MCEXEC_UP_TRANSFER:
 		return mcexec_transfer_image(os, (struct remote_transfer *)arg);
 

kernel/host.c

@@ -598,7 +598,6 @@ static void syscall_channel_send(struct ihk_ikc_channel_desc *c,
 }
 
 extern unsigned long do_kill(struct thread *, int, int, int, struct siginfo *, int ptracecont);
-extern void terminate_host(int pid);
 extern void debug_log(long);
 
 void send_procfs_answer(struct ikc_scd_packet *packet, int err)
@@ -764,8 +763,9 @@ out_remote_pf:
 		break;
 
 	case SCD_MSG_CLEANUP_PROCESS:
-		dkprintf("SCD_MSG_CLEANUP_PROCESS pid=%d\n", packet->pid);
+		dkprintf("SCD_MSG_CLEANUP_PROCESS pid=%d, thread=0x%llx\n",
-		terminate_host(packet->pid);
+				packet->pid, packet->arg);
+		terminate_host(packet->pid, (struct thread *)packet->arg);
 		ret = 0;
 		break;
 

kernel/include/syscall.h

@@ -634,4 +634,5 @@ extern int (*linux_clock_gettime)(clockid_t clk_id, struct timespec *tp);
 #define COREDUMP_DESCHEDULED      1
 #define COREDUMP_TO_BE_WOKEN      2
 
+extern void terminate_host(int pid, struct thread *thread);
 #endif

kernel/process.c

@@ -73,7 +73,6 @@ static struct vm_range *vm_range_find(struct process_vm *vm,
 		unsigned long addr);
 static int copy_user_ranges(struct process_vm *vm, struct process_vm *orgvm);
 extern void __runq_add_proc(struct thread *proc, int cpu_id);
-extern void terminate_host(int pid);
 extern void lapic_timer_enable(unsigned int clocks);
 extern void lapic_timer_disable();
 extern int num_processors;
@@ -288,6 +287,8 @@ struct thread *create_thread(unsigned long user_pc,
 		return NULL;
 	memset(thread, 0, sizeof(struct thread));
 	ihk_atomic_set(&thread->refcount, 2);
+	INIT_LIST_HEAD(&thread->hash_list);
+	INIT_LIST_HEAD(&thread->siblings_list);
 	proc = kmalloc(sizeof(struct process), IHK_MC_AP_NOWAIT);
 	vm = kmalloc(sizeof(struct process_vm), IHK_MC_AP_NOWAIT);
 	asp = create_address_space(cpu_local_var(resource_set), 1);

kernel/syscall.c

@@ -1407,14 +1407,21 @@ void terminate(int rc, int sig)
 }
 
 void
-terminate_host(int pid)
+terminate_host(int pid, struct thread *thread)
 {
 	struct process *proc;
 	struct mcs_rwlock_node_irqsave lock;
 
 	proc = find_process(pid, &lock);
-	if(!proc)
+	if (!proc) {
+		if (thread) {
+			proc = thread->proc;
+			ihk_atomic_set(&thread->refcount, 1);
+			release_thread(thread);
+			release_process(proc);
+		}
 		return;
+	}
 
 	if (proc->nohost != 1) {
 		proc->nohost = 1;

test/mcexec_signalonboot/README

@@ -0,0 +1,43 @@
+==========
+How to run
+==========
+
+(1) cd <mckernel> && patch -p1 < <mckernel>/test/mcexec_signalonboot/signal_injection.patch
+(2) Build McKernel
+(3) cd <mckernel>/test/mcexec_signalonboot
+(4) bash ./run.sh
+
+============
+What to test
+============
+
+Testing memory leaks in program booting.
+Terminate mcexec at some timing to check for memory leaks.
+
+ID  TIMING                         SIGNAL
+------------------------------------------
+001 MCEXEC_UP_PREPARE_IMAGE:before SIGINT
+002 MCEXEC_UP_PREPARE_IMAGE:before SIGKILL
+003 MCEXEC_UP_PREPARE_IMAGE:before SIGTERM
+011 MCEXEC_UP_PREPARE_IMAGE:after  SIGINT
+012 MCEXEC_UP_PREPARE_IMAGE:after  SIGKILL
+013 MCEXEC_UP_PREPARE_IMAGE:after  SIGTERM
+101 MCEXEC_UP_TRANSFER:before      SIGINT
+102 MCEXEC_UP_TRANSFER:before      SIGKILL
+103 MCEXEC_UP_TRANSFER:before      SIGTERM
+111 MCEXEC_UP_TRANSFER:after       SIGINT
+112 MCEXEC_UP_TRANSFER:after       SIGKILL
+113 MCEXEC_UP_TRANSFER:after       SIGTERM
+201 init_sigaction:before          SIGINT
+202 init_sigaction:before          SIGKILL
+203 init_sigaction:before          SIGTERM
+211 init_sigaction:after           SIGINT
+212 init_sigaction:after           SIGKILL
+213 init_sigaction:after           SIGTERM
+301 MCEXEC_UP_START_IMAGE:before   SIGINT
+302 MCEXEC_UP_START_IMAGE:before   SIGKILL
+303 MCEXEC_UP_START_IMAGE:before   SIGTERM
+311 MCEXEC_UP_START_IMAGE:after    SIGINT
+312 MCEXEC_UP_START_IMAGE:after    SIGKILL
+313 MCEXEC_UP_START_IMAGE:after    SIGTERM
+------------------------------------------

test/mcexec_signalonboot/result.log

@@ -0,0 +1,27 @@
+[root@hostname mcexec_signalonboot]# sh run.sh
+mcstop+release.sh ... done
+mcreboot.sh -c 12-59 -m 512M@4 ... done
+001: OK
+002: OK
+003: OK
+011: OK
+012: OK
+013: OK
+101: OK
+102: OK
+103: OK
+111: OK
+112: OK
+113: OK
+201: OK
+202: OK
+203: OK
+211: OK
+212: OK
+213: OK
+301: OK
+302: OK
+303: OK
+311: OK
+312: OK
+313: OK

test/mcexec_signalonboot/run.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+# run.sh COPYRIGHT FUJITSU LIMITED 2019
+test_dir=$(dirname "${BASH_SOURCE[0]}")
+
+#
+# init
+#
+. "${test_dir}/../common.sh"
+SIGINT=2
+SIGKILL=9
+SIGTERM=15
+
+"$BIN/mcexec" -c 0 -- ls 2>&1 | grep -q "signal_injection.patch is applied."
+if [ $? -ne 0 ]; then
+    echo "signal_injection.patch is not been applied." >&2
+	exit 1
+fi
+sleep 1
+meminfo="/sys/devices/virtual/mcos/mcos0/sys/devices/system/node/node0/meminfo"
+exp_free_mem=`cat "$meminfo" | grep MemFree:`
+
+#
+# run
+#
+test_cases=`cat <<__EOF__
+001 MCEXEC_UP_PREPARE_IMAGE:before $SIGINT
+002 MCEXEC_UP_PREPARE_IMAGE:before $SIGKILL
+003 MCEXEC_UP_PREPARE_IMAGE:before $SIGTERM
+011 MCEXEC_UP_PREPARE_IMAGE:after $SIGINT
+012 MCEXEC_UP_PREPARE_IMAGE:after $SIGKILL
+013 MCEXEC_UP_PREPARE_IMAGE:after $SIGTERM
+101 MCEXEC_UP_TRANSFER:before $SIGINT
+102 MCEXEC_UP_TRANSFER:before $SIGKILL
+103 MCEXEC_UP_TRANSFER:before $SIGTERM
+111 MCEXEC_UP_TRANSFER:after $SIGINT
+112 MCEXEC_UP_TRANSFER:after $SIGKILL
+113 MCEXEC_UP_TRANSFER:after $SIGTERM
+201 init_sigaction:before $SIGINT
+202 init_sigaction:before $SIGKILL
+203 init_sigaction:before $SIGTERM
+211 init_sigaction:after $SIGINT
+212 init_sigaction:after $SIGKILL
+213 init_sigaction:after $SIGTERM
+301 MCEXEC_UP_START_IMAGE:before $SIGINT
+302 MCEXEC_UP_START_IMAGE:before $SIGKILL
+303 MCEXEC_UP_START_IMAGE:before $SIGTERM
+311 MCEXEC_UP_START_IMAGE:after $SIGINT
+312 MCEXEC_UP_START_IMAGE:after $SIGKILL
+313 MCEXEC_UP_START_IMAGE:after $SIGTERM
+__EOF__`
+
+IFS='
+'
+for tc in $test_cases
+do
+	no=`echo $tc | awk '{print $1}'`
+	opt_i=`echo $tc | awk '{print $2}'`
+	opt_s=`echo $tc | awk '{print $3}'`
+	echo -n "$no: "
+	bash -c "'$BIN/mcexec' -c 0 -- -i $opt_i -s $opt_s ls >/dev/null 2>&1" \
+		>/dev/null 2>&1
+	sleep 1
+
+	free_mem=`cat "$meminfo" | grep MemFree:`
+	if [ "$exp_free_mem" != "$free_mem" ]; then
+		echo "NG - detected memory leak."
+		echo "     before: ${exp_free_mem}"
+		echo "     after: ${free_mem}"
+		exp_free_mem=$free_mem
+	else
+		echo "OK"
+	fi
+done
+exit 0

test/mcexec_signalonboot/signal_injection.patch

@@ -0,0 +1,124 @@
+diff --git a/executer/user/mcexec.c b/executer/user/mcexec.c
+index 7f84284..3d65118 100644
+--- a/executer/user/mcexec.c
++++ b/executer/user/mcexec.c
+@@ -200,6 +200,8 @@ static char *mpol_bind_nodes = NULL;
+ static int uti_thread_rank = 0;
+ static int uti_use_last_cpu = 0;
+ static int enable_uti = 0;
++static const char *signal_injection = "";
++static int injection_signo = 0;
+ 
+ /* Partitioned execution (e.g., for MPI) */
+ static int nr_processes = 0;
+@@ -892,11 +894,20 @@ int transfer_image(int fd, struct program_load_desc *desc)
+ 			/* No more left to upload.. */
+ 			if (lr == 0 && flen == 0) break;
+ 
++			if (strcmp(signal_injection, "MCEXEC_UP_TRANSFER:before") == 0) {
++				printf("raise\n");
++				raise(injection_signo);
++			}
++			printf("ioctl(MCEXEC_UP_TRANSFER)\n");
+ 			if (ioctl(fd, MCEXEC_UP_TRANSFER,
+ 						(unsigned long)&pt)) {
+ 				perror("dma");
+ 				break;
+ 			}
++			if (strcmp(signal_injection, "MCEXEC_UP_TRANSFER:after") == 0) {
++				printf("raise\n");
++				raise(injection_signo);
++			}
+ 		}
+ 	}
+ 
+@@ -2092,6 +2103,8 @@ int main(int argc, char **argv)
+ 	__glob_argv = argv;
+ #endif
+ 
++	printf("signal_injection.patch is applied.\n");
++
+ 	page_size = sysconf(_SC_PAGESIZE);
+ 	page_mask = ~(page_size - 1);
+ 
+@@ -2237,6 +2250,21 @@ int main(int argc, char **argv)
+ 		}
+ 	}
+ 
++	while ((opt = getopt_long(argc, argv, "i:s:",
++			mcexec_options, NULL)) != -1) {
++		switch (opt) {
++		case 'i':
++			signal_injection = optarg;
++			break;
++		case 's':
++			injection_signo = strtol(optarg, NULL, 0);
++			break;
++		default: /* '?' */
++			print_usage(argv);
++			exit(EXIT_FAILURE);
++		}
++	}
++
+ 	if (heap_extension == -1) {
+ 		heap_extension = sysconf(_SC_PAGESIZE);
+ 	}
+@@ -2653,11 +2681,20 @@ int main(int argc, char **argv)
+ 	desc->thp_disable = get_thp_disable();
+ 
+ 	/* user_start and user_end are set by this call */
++	if (strcmp(signal_injection, "MCEXEC_UP_PREPARE_IMAGE:before") == 0) {
++		printf("raise\n");
++		raise(injection_signo);
++	}
++	printf("ioctl(MCEXEC_UP_PREPARE_IMAGE)\n");
+ 	if (ioctl(fd, MCEXEC_UP_PREPARE_IMAGE, (unsigned long)desc) != 0) {
+ 		perror("prepare");
+ 		close(fd);
+ 		return 1;
+ 	}
++	if (strcmp(signal_injection, "MCEXEC_UP_PREPARE_IMAGE:after") == 0) {
++		printf("raise\n");
++		raise(injection_signo);
++	}
+ 
+ 	print_desc(desc);
+ 	if (transfer_image(fd, desc) < 0) {
+@@ -2692,7 +2729,16 @@ int main(int argc, char **argv)
+ 	__dprintf("mccmd server initialized\n");
+ #endif
+ 
++	if (strcmp(signal_injection, "init_sigaction:before") == 0) {
++		printf("raise\n");
++		raise(injection_signo);
++	}
++	printf("init_sigaction\n");
+ 	init_sigaction();
++	if (strcmp(signal_injection, "init_sigaction:after") == 0) {
++		printf("raise\n");
++		raise(injection_signo);
++	}
+ 
+ 	/* Initialize watchdog thread which detects hang of McKernel */
+ 
+@@ -2721,11 +2767,20 @@ int main(int argc, char **argv)
+ 		return 1;
+ 	}
+ 
++	if (strcmp(signal_injection, "MCEXEC_UP_START_IMAGE:before") == 0) {
++		printf("raise\n");
++		raise(injection_signo);
++	}
++	printf("ioctl(MCEXEC_UP_START_IMAGE)\n");
+ 	if (ioctl(fd, MCEXEC_UP_START_IMAGE, (unsigned long)desc) != 0) {
+ 		perror("exec");
+ 		close(fd);
+ 		return 1;
+ 	}
++	if (strcmp(signal_injection, "MCEXEC_UP_START_IMAGE:after") == 0) {
++		printf("raise\n");
++		raise(injection_signo);
++	}
+ 
+ #if 1 /* debug : thread killed by exit_group() are still joinable? */
+ 	join_all_threads();