Wait for LWK to run at shutdown.

refs #898
refs #928

arch/x86_64/kernel/interrupt.S

@@ -145,6 +145,8 @@ nmi:
 		movq	%rsp,%gs:PANIC_REGS+0x08
 
 		movl	nmi_mode(%rip),%eax
+		cmp $3,%rax
+		je  4f
 		cmp	$1,%rax
 		je	1f
 		cmp	$2,%rax
@@ -199,9 +201,9 @@ nmi:
 		movl	%eax,%gs:PANIC_REGS+0xA0
 		movq	$1,%gs:PANICED
 		call	ihk_mc_query_mem_areas
-1:
+4:
 		hlt
-		jmp	1b
+		jmp	4b
 
 .globl x86_syscall
 x86_syscall:

executer/kernel/mcctrl/driver.c

@@ -27,6 +27,7 @@
 #include <linux/miscdevice.h>
 #include <linux/slab.h>
 #include <linux/device.h>
+#include <linux/delay.h>
 #include "mcctrl.h"
 #include <ihk/ihk_host_user.h>
 
@@ -169,6 +170,14 @@ error_cleanup_channels:
 int mcctrl_os_shutdown_notifier(int os_index)
 {
 	if (os[os_index]) {
+		/* Wait for os running */
+		if (ihk_os_wait_for_status(os[os_index], IHK_OS_STATUS_RUNNING, 0, 200) != 0) {
+			printk("IHK: OS does not become RUNNING in shutdown. Force shutdown.\n");
+			/* send nmi to force shutdown */
+			ihk_os_send_nmi(os[os_index], 3);
+			mdelay(200);
+		}
+
 		sysfsm_cleanup(os[os_index]);
 		free_topology_info(os[os_index]);
 		ihk_os_unregister_user_call_handlers(os[os_index], mcctrl_uc + os_index);

kernel/include/init.h

@@ -18,6 +18,7 @@ extern void mem_init(void);
 extern void ihk_ikc_master_init(void);
 extern void ap_init(void);
 extern void arch_ready(void);
+extern void done_init(void);
 extern void mc_ikc_test_init(void);
 extern void cpu_local_var_init(void);
 extern void kmalloc_init(void);

kernel/init.c

@@ -387,6 +387,7 @@ int main(void)
 
 	futex_init();
 
+	done_init();
 	kputs("IHK/McKernel booted.\n");
 
 #ifdef DCFA_KMOD

test/mng_mod/issues/898_928/CT_001.c

@@ -0,0 +1,177 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <ihklib.h>
+#include <sys/types.h>
+
+#define MCK_DIR "/home/satoken/ppos"
+static char prefix[256] = MCK_DIR;
+
+static char test_name[64] = "CT_001";
+
+#define OKNG(cond, ...)													\
+    do {                                                                \
+		if(cond) {                                                      \
+			printf("[OK] ");											\
+			printf(__VA_ARGS__);										\
+		} else {														\
+            printf("[NG] ");											\
+			printf(__VA_ARGS__);										\
+			char buf[65536];\
+			char cmd[256];\
+			sprintf(cmd, "%s/sbin/ihkosctl 0 kmsg", prefix);\
+			FILE* fp = popen(cmd, "r");		\
+			size_t nread = fread(buf, 1, sizeof(buf), fp);				\
+			buf[nread] = 0;												\
+			printf("%s", buf);											\
+			goto fn_fail;												\
+		}																\
+    } while(0)
+
+int main(int argc, char** argv) {
+    int ret = 0, status, ret_ihklib;
+	FILE *fp;
+	char buf[65536];
+	size_t nread;
+
+	char cmd[1024];
+	char fn[256];
+	char kargs[256];
+
+	int cpus[4] = {6, 7, 8, 9};
+	int num_cpus = 4;
+
+	struct ihk_mem_chunk mem_chunks[4];
+	int num_mem_chunks;
+
+	printf("*** %s start *************************\n", test_name);
+	/*--------------------------------------------
+	 * Preparing                                  
+	 *--------------------------------------------*/
+	sprintf(cmd, "%s/sbin/mcstop+release.sh", prefix);
+	status = system(cmd);
+
+	// ihk_os_destroy_pseudofs
+	ret_ihklib = ihk_os_destroy_pseudofs(0);
+	fp = popen("cat /proc/mounts | grep /tmp/mcos/mcos0_sys", "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+
+	sprintf(cmd, "insmod %s/kmod/ihk.ko", prefix);
+	status = system(cmd);
+
+	sprintf(cmd, "insmod %s/kmod/ihk-smp-x86_64.ko ihk_start_irq=240 ihk_ikc_irq_core=0", prefix);
+	status = system(cmd);
+
+	sprintf(cmd, "insmod %s/kmod/mcctrl.ko", prefix);
+	status = system(cmd);
+
+	/*--------------------------------------------
+	 * Test                                  
+	 *--------------------------------------------*/
+	// create 0
+    ret_ihklib = ihk_create_os(0);
+
+	// reserve cpus
+	ret_ihklib = ihk_reserve_cpu(0, cpus, num_cpus);
+	//OKNG(ret_ihklib == 0, "ihk_reserve_cpu\n");
+
+	// assign cpus
+    ret_ihklib = ihk_os_assign_cpu(0, cpus, num_cpus);
+    //OKNG(ret_ihklib == 0, "ihk_os_assign_cpu\n");
+
+	// reserve mem 128m@0,128m@1
+	num_mem_chunks = 2;
+	mem_chunks[0].size = 128*1024*1024ULL;
+	mem_chunks[0].numa_node_number = 0;
+	mem_chunks[1].size = 128*1024*1024ULL;
+	mem_chunks[1].numa_node_number = 1;
+    ret_ihklib = ihk_reserve_mem(0, mem_chunks, num_mem_chunks);
+    //OKNG(ret_ihklib == 0, "ihk_reserve_mem (2)\n");
+
+	// assign mem 128m@0,128m@1
+	num_mem_chunks = 2;
+	mem_chunks[0].size = 128*1024*1024ULL;
+	mem_chunks[0].numa_node_number = 0;
+	mem_chunks[1].size = 128*1024*1024ULL;
+	mem_chunks[1].numa_node_number = 1;
+    ret_ihklib = ihk_os_assign_mem(0, mem_chunks, num_mem_chunks);
+    //OKNG(ret_ihklib == 0, "ihk_os_assign_mem (2)\n");
+
+	// load
+	sprintf(fn, "%s/smp-x86/kernel/mckernel.img", prefix);
+    ret_ihklib = ihk_os_load(0, fn);
+	//OKNG(ret_ihklib == 0, "ihk_os_load\n");
+
+	// kargs
+	sprintf(kargs, "hidos ksyslogd=0");
+    ret_ihklib = ihk_os_kargs(0, kargs);
+	//OKNG(ret_ihklib == 0, "ihk_os_kargs\n");
+
+	// boot
+    ret_ihklib = ihk_os_boot(0);
+goto shutdown;
+	OKNG(ret_ihklib == 0, "ihk_os_boot\n");
+
+	/* Make sure that all initialization related transactions between McKernel and IHK finish
+	   sysfs_init(void) (in mckernel/kernel/sysfs.c)
+	   packet.msg = SCD_MSG_SYSFS_REQ_SETUP;
+	   sysfsm_work_main() in (mckernel/executer/kernel/mcctrl/sysfs.c)
+       sysfsm_req_setup
+       sysfsm_setup */
+	usleep(100*1000);
+
+	// create pseudofs
+	ret_ihklib = ihk_os_create_pseudofs(0);
+	fp = popen("cat /proc/mounts | grep /tmp/mcos/mcos0_sys", "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+	//OKNG(ret_ihklib == 0 &&
+	//	 strstr(buf, "/tmp/mcos/mcos0_sys") != NULL, "ihk_os_create_pseudofs()\n");
+
+	// mcexec
+	sprintf(cmd, "%s/bin/mcexec ls -l | grep Makefile", prefix);
+    fp = popen(cmd, "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+	OKNG(strstr(buf, "Makefile") != NULL, "mcexec\n");
+
+	// shutdown
+shutdown:
+    ret_ihklib = ihk_os_shutdown(0);
+	OKNG(ret_ihklib == 0, "shutdown immediately after boot returned 0\n");
+
+	// get status. Note that the smp_ihk_os_shutdown() transitions 
+	// smp-x86 status to BUILTIN_OS_STATUS_SHUTDOWN
+	// and smp_ihk_os_query_status() transitions os status to IHK_OS_STATUS_NOT_BOOTED.
+	ret_ihklib = ihk_os_get_status(0);
+	//OKNG(ret_ihklib == IHK_STATUS_SHUTDOWN ||
+	//	 ret_ihklib == IHK_STATUS_INACTIVE, "ihk_os_get_status (5) returned %d\n", ret_ihklib);
+
+destroy:
+    ret_ihklib = ihk_destroy_os(0, 0);
+	//OKNG(ret_ihklib == 0, "destroy immediately after boot\n");
+
+	sprintf(cmd, "rmmod %s/kmod/mcctrl.ko", prefix);
+	status = system(cmd);
+
+	// destroy pseudofs
+	ret_ihklib = ihk_os_destroy_pseudofs(0);
+	fp = popen("cat /proc/mounts | grep /tmp/mcos/mcos0_sys", "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+
+	sprintf(cmd, "rmmod %s/kmod/ihk-smp-x86_64.ko", prefix);
+	status = system(cmd);
+
+	sprintf(cmd, "rmmod %s/kmod/ihk.ko", prefix);
+	status = system(cmd);
+
+	printf("*** All tests finished\n\n");
+
+ fn_exit:
+    return ret;
+ fn_fail:
+    goto fn_exit;
+}

test/mng_mod/issues/898_928/CT_002.c

@@ -0,0 +1,177 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <ihklib.h>
+#include <sys/types.h>
+
+#define MCK_DIR "/home/satoken/ppos"
+static char prefix[256] = MCK_DIR;
+
+static char test_name[64] = "CT_002";
+
+#define OKNG(cond, ...)													\
+    do {                                                                \
+		if(cond) {                                                      \
+			printf("[OK] ");											\
+			printf(__VA_ARGS__);										\
+		} else {														\
+            printf("[NG] ");											\
+			printf(__VA_ARGS__);										\
+			char buf[65536];\
+			char cmd[256];\
+			sprintf(cmd, "%s/sbin/ihkosctl 0 kmsg", prefix);\
+			FILE* fp = popen(cmd, "r");		\
+			size_t nread = fread(buf, 1, sizeof(buf), fp);				\
+			buf[nread] = 0;												\
+			printf("%s", buf);											\
+			goto fn_fail;												\
+		}																\
+    } while(0)
+
+int main(int argc, char** argv) {
+    int ret = 0, status, ret_ihklib;
+	FILE *fp;
+	char buf[65536];
+	size_t nread;
+
+	char cmd[1024];
+	char fn[256];
+	char kargs[256];
+
+	int cpus[4] = {6, 7, 8, 9};
+	int num_cpus = 4;
+
+	struct ihk_mem_chunk mem_chunks[4];
+	int num_mem_chunks;
+
+	printf("*** %s start *************************\n", test_name);
+	/*--------------------------------------------
+	 * Preparing                                  
+	 *--------------------------------------------*/
+	sprintf(cmd, "%s/sbin/mcstop+release.sh", prefix);
+	status = system(cmd);
+
+	// ihk_os_destroy_pseudofs
+	ret_ihklib = ihk_os_destroy_pseudofs(0);
+	fp = popen("cat /proc/mounts | grep /tmp/mcos/mcos0_sys", "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+
+	sprintf(cmd, "insmod %s/kmod/ihk.ko", prefix);
+	status = system(cmd);
+
+	sprintf(cmd, "insmod %s/kmod/ihk-smp-x86_64.ko ihk_start_irq=240 ihk_ikc_irq_core=0", prefix);
+	status = system(cmd);
+
+	sprintf(cmd, "insmod %s/kmod/mcctrl.ko", prefix);
+	status = system(cmd);
+
+	/*--------------------------------------------
+	 * Test                                  
+	 *--------------------------------------------*/
+	// create 0
+    ret_ihklib = ihk_create_os(0);
+
+	// reserve cpus
+	ret_ihklib = ihk_reserve_cpu(0, cpus, num_cpus);
+	//OKNG(ret_ihklib == 0, "ihk_reserve_cpu\n");
+
+	// assign cpus
+    ret_ihklib = ihk_os_assign_cpu(0, cpus, num_cpus);
+    //OKNG(ret_ihklib == 0, "ihk_os_assign_cpu\n");
+
+	// reserve mem 128m@0,128m@1
+	num_mem_chunks = 2;
+	mem_chunks[0].size = 128*1024*1024ULL;
+	mem_chunks[0].numa_node_number = 0;
+	mem_chunks[1].size = 128*1024*1024ULL;
+	mem_chunks[1].numa_node_number = 1;
+    ret_ihklib = ihk_reserve_mem(0, mem_chunks, num_mem_chunks);
+    //OKNG(ret_ihklib == 0, "ihk_reserve_mem (2)\n");
+
+	// assign mem 128m@0,128m@1
+	num_mem_chunks = 2;
+	mem_chunks[0].size = 128*1024*1024ULL;
+	mem_chunks[0].numa_node_number = 0;
+	mem_chunks[1].size = 128*1024*1024ULL;
+	mem_chunks[1].numa_node_number = 1;
+    ret_ihklib = ihk_os_assign_mem(0, mem_chunks, num_mem_chunks);
+    //OKNG(ret_ihklib == 0, "ihk_os_assign_mem (2)\n");
+
+	// load
+	sprintf(fn, "%s/smp-x86/kernel/mckernel.img", prefix);
+    ret_ihklib = ihk_os_load(0, fn);
+	//OKNG(ret_ihklib == 0, "ihk_os_load\n");
+
+	// kargs
+	sprintf(kargs, "hidos ksyslogd=0");
+    ret_ihklib = ihk_os_kargs(0, kargs);
+	//OKNG(ret_ihklib == 0, "ihk_os_kargs\n");
+
+	// boot
+    ret_ihklib = ihk_os_boot(0);
+goto destroy;
+	OKNG(ret_ihklib == 0, "ihk_os_boot\n");
+
+	/* Make sure that all initialization related transactions between McKernel and IHK finish
+	   sysfs_init(void) (in mckernel/kernel/sysfs.c)
+	   packet.msg = SCD_MSG_SYSFS_REQ_SETUP;
+	   sysfsm_work_main() in (mckernel/executer/kernel/mcctrl/sysfs.c)
+       sysfsm_req_setup
+       sysfsm_setup */
+	usleep(100*1000);
+
+	// create pseudofs
+	ret_ihklib = ihk_os_create_pseudofs(0);
+	fp = popen("cat /proc/mounts | grep /tmp/mcos/mcos0_sys", "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+	//OKNG(ret_ihklib == 0 &&
+	//	 strstr(buf, "/tmp/mcos/mcos0_sys") != NULL, "ihk_os_create_pseudofs()\n");
+
+	// mcexec
+	sprintf(cmd, "%s/bin/mcexec ls -l | grep Makefile", prefix);
+    fp = popen(cmd, "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+	OKNG(strstr(buf, "Makefile") != NULL, "mcexec\n");
+
+	// shutdown
+shutdown:
+    ret_ihklib = ihk_os_shutdown(0);
+	//OKNG(ret_ihklib == 0, "shutdown immediately after boot\n");
+
+	// get status. Note that the smp_ihk_os_shutdown() transitions 
+	// smp-x86 status to BUILTIN_OS_STATUS_SHUTDOWN
+	// and smp_ihk_os_query_status() transitions os status to IHK_OS_STATUS_NOT_BOOTED.
+	ret_ihklib = ihk_os_get_status(0);
+	//OKNG(ret_ihklib == IHK_STATUS_SHUTDOWN ||
+	//	 ret_ihklib == IHK_STATUS_INACTIVE, "ihk_os_get_status (5) returned %d\n", ret_ihklib);
+
+destroy:
+    ret_ihklib = ihk_destroy_os(0, 0);
+	OKNG(ret_ihklib == 0, "destroy immediately after boot returned NOT 0\n");
+
+	sprintf(cmd, "rmmod %s/kmod/mcctrl.ko", prefix);
+	status = system(cmd);
+
+	// destroy pseudofs
+	ret_ihklib = ihk_os_destroy_pseudofs(0);
+	fp = popen("cat /proc/mounts | grep /tmp/mcos/mcos0_sys", "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+
+	sprintf(cmd, "rmmod %s/kmod/ihk-smp-x86_64.ko", prefix);
+	status = system(cmd);
+
+	sprintf(cmd, "rmmod %s/kmod/ihk.ko", prefix);
+	status = system(cmd);
+
+	printf("*** All tests finished\n\n");
+
+ fn_exit:
+    return ret;
+ fn_fail:
+    goto fn_exit;
+}

test/mng_mod/issues/898_928/CT_003.c

@@ -0,0 +1,176 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <ihklib.h>
+#include <sys/types.h>
+
+#define MCK_DIR "/home/satoken/ppos"
+static char prefix[256] = MCK_DIR;
+
+static char test_name[64] = "CT_003";
+
+#define OKNG(cond, ...)													\
+    do {                                                                \
+		if(cond) {                                                      \
+			printf("[OK] ");											\
+			printf(__VA_ARGS__);										\
+		} else {														\
+            printf("[NG] ");											\
+			printf(__VA_ARGS__);										\
+			char buf[65536];\
+			char cmd[256];\
+			sprintf(cmd, "%s/sbin/ihkosctl 0 kmsg", prefix);\
+			FILE* fp = popen(cmd, "r");		\
+			size_t nread = fread(buf, 1, sizeof(buf), fp);				\
+			buf[nread] = 0;												\
+			printf("%s", buf);											\
+			goto fn_fail;												\
+		}																\
+    } while(0)
+
+int main(int argc, char** argv) {
+    int ret = 0, status, ret_ihklib;
+	FILE *fp;
+	char buf[65536];
+	size_t nread;
+
+	char cmd[1024];
+	char fn[256];
+	char kargs[256];
+
+	int cpus[4] = {6, 7, 8, 9};
+	int num_cpus = 4;
+
+	struct ihk_mem_chunk mem_chunks[4];
+	int num_mem_chunks;
+
+	printf("*** %s start *************************\n", test_name);
+	/*--------------------------------------------
+	 * Preparing                                  
+	 *--------------------------------------------*/
+	sprintf(cmd, "%s/sbin/mcstop+release.sh", prefix);
+	status = system(cmd);
+
+	// ihk_os_destroy_pseudofs
+	ret_ihklib = ihk_os_destroy_pseudofs(0);
+	fp = popen("cat /proc/mounts | grep /tmp/mcos/mcos0_sys", "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+
+	sprintf(cmd, "insmod %s/kmod/ihk.ko", prefix);
+	status = system(cmd);
+
+	sprintf(cmd, "insmod %s/kmod/ihk-smp-x86_64.ko ihk_start_irq=240 ihk_ikc_irq_core=0", prefix);
+	status = system(cmd);
+
+	sprintf(cmd, "insmod %s/kmod/mcctrl.ko", prefix);
+	status = system(cmd);
+
+	/*--------------------------------------------
+	 * Test                                  
+	 *--------------------------------------------*/
+	// create 0
+    ret_ihklib = ihk_create_os(0);
+
+	// reserve cpus
+	ret_ihklib = ihk_reserve_cpu(0, cpus, num_cpus);
+	//OKNG(ret_ihklib == 0, "ihk_reserve_cpu\n");
+
+	// assign cpus
+    ret_ihklib = ihk_os_assign_cpu(0, cpus, num_cpus);
+    //OKNG(ret_ihklib == 0, "ihk_os_assign_cpu\n");
+
+	// reserve mem 128m@0,128m@1
+	num_mem_chunks = 2;
+	mem_chunks[0].size = 128*1024*1024ULL;
+	mem_chunks[0].numa_node_number = 0;
+	mem_chunks[1].size = 128*1024*1024ULL;
+	mem_chunks[1].numa_node_number = 1;
+    ret_ihklib = ihk_reserve_mem(0, mem_chunks, num_mem_chunks);
+    //OKNG(ret_ihklib == 0, "ihk_reserve_mem (2)\n");
+
+	// assign mem 128m@0,128m@1
+	num_mem_chunks = 2;
+	mem_chunks[0].size = 128*1024*1024ULL;
+	mem_chunks[0].numa_node_number = 0;
+	mem_chunks[1].size = 128*1024*1024ULL;
+	mem_chunks[1].numa_node_number = 1;
+    ret_ihklib = ihk_os_assign_mem(0, mem_chunks, num_mem_chunks);
+    //OKNG(ret_ihklib == 0, "ihk_os_assign_mem (2)\n");
+
+	// load
+	sprintf(fn, "%s/smp-x86/kernel/mckernel.img", prefix);
+    ret_ihklib = ihk_os_load(0, fn);
+	//OKNG(ret_ihklib == 0, "ihk_os_load\n");
+
+	// kargs
+	sprintf(kargs, "hidos ksyslogd=0");
+    ret_ihklib = ihk_os_kargs(0, kargs);
+	//OKNG(ret_ihklib == 0, "ihk_os_kargs\n");
+
+	// boot
+    ret_ihklib = ihk_os_boot(0);
+	OKNG(ret_ihklib == 0, "ihk_os_boot\n");
+
+	/* Make sure that all initialization related transactions between McKernel and IHK finish
+	   sysfs_init(void) (in mckernel/kernel/sysfs.c)
+	   packet.msg = SCD_MSG_SYSFS_REQ_SETUP;
+	   sysfsm_work_main() in (mckernel/executer/kernel/mcctrl/sysfs.c)
+       sysfsm_req_setup
+       sysfsm_setup */
+	usleep(100*1000);
+
+	// create pseudofs
+	ret_ihklib = ihk_os_create_pseudofs(0);
+	fp = popen("cat /proc/mounts | grep /tmp/mcos/mcos0_sys", "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+	//OKNG(ret_ihklib == 0 &&
+	//	 strstr(buf, "/tmp/mcos/mcos0_sys") != NULL, "ihk_os_create_pseudofs()\n");
+
+	// mcexec
+	sprintf(cmd, "%s/bin/mcexec ls -l | grep Makefile", prefix);
+    fp = popen(cmd, "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+	OKNG(strstr(buf, "Makefile") != NULL, "done mcexec\n");
+
+	// shutdown
+shutdown:
+    ret_ihklib = ihk_os_shutdown(0);
+	OKNG(ret_ihklib == 0, "shutdown after mcexec returned 0\n");
+
+	// get status. Note that the smp_ihk_os_shutdown() transitions 
+	// smp-x86 status to BUILTIN_OS_STATUS_SHUTDOWN
+	// and smp_ihk_os_query_status() transitions os status to IHK_OS_STATUS_NOT_BOOTED.
+	ret_ihklib = ihk_os_get_status(0);
+	//OKNG(ret_ihklib == IHK_STATUS_SHUTDOWN ||
+	//	 ret_ihklib == IHK_STATUS_INACTIVE, "ihk_os_get_status (5) returned %d\n", ret_ihklib);
+
+destroy:
+    ret_ihklib = ihk_destroy_os(0, 0);
+	//OKNG(ret_ihklib == 0, "destroy immediately after boot\n");
+
+	sprintf(cmd, "rmmod %s/kmod/mcctrl.ko", prefix);
+	status = system(cmd);
+
+	// destroy pseudofs
+	ret_ihklib = ihk_os_destroy_pseudofs(0);
+	fp = popen("cat /proc/mounts | grep /tmp/mcos/mcos0_sys", "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+
+	sprintf(cmd, "rmmod %s/kmod/ihk-smp-x86_64.ko", prefix);
+	status = system(cmd);
+
+	sprintf(cmd, "rmmod %s/kmod/ihk.ko", prefix);
+	status = system(cmd);
+
+	printf("*** All tests finished\n\n");
+
+ fn_exit:
+    return ret;
+ fn_fail:
+    goto fn_exit;
+}

test/mng_mod/issues/898_928/CT_004.c

@@ -0,0 +1,177 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <ihklib.h>
+#include <sys/types.h>
+
+#define MCK_DIR "/home/satoken/ppos"
+static char prefix[256] = MCK_DIR;
+
+static char test_name[64] = "CT_004";
+
+#define OKNG(cond, ...)													\
+    do {                                                                \
+		if(cond) {                                                      \
+			printf("[OK] ");											\
+			printf(__VA_ARGS__);										\
+		} else {														\
+            printf("[NG] ");											\
+			printf(__VA_ARGS__);										\
+			char buf[65536];\
+			char cmd[256];\
+			sprintf(cmd, "%s/sbin/ihkosctl 0 kmsg", prefix);\
+			FILE* fp = popen(cmd, "r");		\
+			size_t nread = fread(buf, 1, sizeof(buf), fp);				\
+			buf[nread] = 0;												\
+			printf("%s", buf);											\
+			goto fn_fail;												\
+		}																\
+    } while(0)
+
+int main(int argc, char** argv) {
+    int ret = 0, status, ret_ihklib;
+	FILE *fp;
+	char buf[65536];
+	size_t nread;
+
+	char cmd[1024];
+	char fn[256];
+	char kargs[256];
+
+	int cpus[4] = {6, 7, 8, 9};
+	int num_cpus = 4;
+
+	struct ihk_mem_chunk mem_chunks[4];
+	int num_mem_chunks;
+
+	printf("*** %s start *************************\n", test_name);
+	/*--------------------------------------------
+	 * Preparing                                  
+	 *--------------------------------------------*/
+	sprintf(cmd, "%s/sbin/mcstop+release.sh", prefix);
+	status = system(cmd);
+
+	// ihk_os_destroy_pseudofs
+	ret_ihklib = ihk_os_destroy_pseudofs(0);
+	fp = popen("cat /proc/mounts | grep /tmp/mcos/mcos0_sys", "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+
+	sprintf(cmd, "insmod %s/kmod/ihk.ko", prefix);
+	status = system(cmd);
+
+	sprintf(cmd, "insmod %s/kmod/ihk-smp-x86_64.ko ihk_start_irq=240 ihk_ikc_irq_core=0", prefix);
+	status = system(cmd);
+
+	sprintf(cmd, "insmod %s/kmod/mcctrl.ko", prefix);
+	status = system(cmd);
+
+	/*--------------------------------------------
+	 * Test                                  
+	 *--------------------------------------------*/
+	// create 0
+    ret_ihklib = ihk_create_os(0);
+
+	// reserve cpus
+	ret_ihklib = ihk_reserve_cpu(0, cpus, num_cpus);
+	//OKNG(ret_ihklib == 0, "ihk_reserve_cpu\n");
+
+	// assign cpus
+    ret_ihklib = ihk_os_assign_cpu(0, cpus, num_cpus);
+    //OKNG(ret_ihklib == 0, "ihk_os_assign_cpu\n");
+
+	// reserve mem 128m@0,128m@1
+	num_mem_chunks = 2;
+	mem_chunks[0].size = 128*1024*1024ULL;
+	mem_chunks[0].numa_node_number = 0;
+	mem_chunks[1].size = 128*1024*1024ULL;
+	mem_chunks[1].numa_node_number = 1;
+    ret_ihklib = ihk_reserve_mem(0, mem_chunks, num_mem_chunks);
+    //OKNG(ret_ihklib == 0, "ihk_reserve_mem (2)\n");
+
+	// assign mem 128m@0,128m@1
+	num_mem_chunks = 2;
+	mem_chunks[0].size = 128*1024*1024ULL;
+	mem_chunks[0].numa_node_number = 0;
+	mem_chunks[1].size = 128*1024*1024ULL;
+	mem_chunks[1].numa_node_number = 1;
+    ret_ihklib = ihk_os_assign_mem(0, mem_chunks, num_mem_chunks);
+    //OKNG(ret_ihklib == 0, "ihk_os_assign_mem (2)\n");
+
+	// load
+	sprintf(fn, "%s/smp-x86/kernel/mckernel.img", prefix);
+    ret_ihklib = ihk_os_load(0, fn);
+	//OKNG(ret_ihklib == 0, "ihk_os_load\n");
+
+	// kargs
+	sprintf(kargs, "hidos ksyslogd=0");
+    ret_ihklib = ihk_os_kargs(0, kargs);
+	//OKNG(ret_ihklib == 0, "ihk_os_kargs\n");
+
+	// boot
+    ret_ihklib = ihk_os_boot(0);
+	OKNG(ret_ihklib == 0, "ihk_os_boot\n");
+
+	/* Make sure that all initialization related transactions between McKernel and IHK finish
+	   sysfs_init(void) (in mckernel/kernel/sysfs.c)
+	   packet.msg = SCD_MSG_SYSFS_REQ_SETUP;
+	   sysfsm_work_main() in (mckernel/executer/kernel/mcctrl/sysfs.c)
+       sysfsm_req_setup
+       sysfsm_setup */
+	usleep(100*1000);
+
+	// create pseudofs
+	ret_ihklib = ihk_os_create_pseudofs(0);
+	fp = popen("cat /proc/mounts | grep /tmp/mcos/mcos0_sys", "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+	//OKNG(ret_ihklib == 0 &&
+	//	 strstr(buf, "/tmp/mcos/mcos0_sys") != NULL, "ihk_os_create_pseudofs()\n");
+
+	// mcexec
+	sprintf(cmd, "%s/bin/mcexec ls -l | grep Makefile", prefix);
+    fp = popen(cmd, "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+	OKNG(strstr(buf, "Makefile") != NULL, "do mcexec\n");
+goto destroy;
+
+	// shutdown
+shutdown:
+    ret_ihklib = ihk_os_shutdown(0);
+	//OKNG(ret_ihklib == 0, "shutdown after mcexec\n");
+
+	// get status. Note that the smp_ihk_os_shutdown() transitions 
+	// smp-x86 status to BUILTIN_OS_STATUS_SHUTDOWN
+	// and smp_ihk_os_query_status() transitions os status to IHK_OS_STATUS_NOT_BOOTED.
+	ret_ihklib = ihk_os_get_status(0);
+	//OKNG(ret_ihklib == IHK_STATUS_SHUTDOWN ||
+	//	 ret_ihklib == IHK_STATUS_INACTIVE, "ihk_os_get_status (5) returned %d\n", ret_ihklib);
+
+destroy:
+    ret_ihklib = ihk_destroy_os(0, 0);
+	OKNG(ret_ihklib == 0, "destroy after mcexec returned 0\n");
+
+	sprintf(cmd, "rmmod %s/kmod/mcctrl.ko", prefix);
+	status = system(cmd);
+
+	// destroy pseudofs
+	ret_ihklib = ihk_os_destroy_pseudofs(0);
+	fp = popen("cat /proc/mounts | grep /tmp/mcos/mcos0_sys", "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+
+	sprintf(cmd, "rmmod %s/kmod/ihk-smp-x86_64.ko", prefix);
+	status = system(cmd);
+
+	sprintf(cmd, "rmmod %s/kmod/ihk.ko", prefix);
+	status = system(cmd);
+
+	printf("*** All tests finished\n\n");
+
+ fn_exit:
+    return ret;
+ fn_fail:
+    goto fn_exit;
+}

test/mng_mod/issues/898_928/CT_005.c

@@ -0,0 +1,183 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <ihklib.h>
+#include <sys/types.h>
+
+#define MCK_DIR "/home/satoken/ppos"
+static char prefix[256] = MCK_DIR;
+
+static char test_name[64] = "CT_005";
+
+#define OKNG(cond, ...)													\
+    do {                                                                \
+		if(cond) {                                                      \
+			printf("[OK] ");											\
+			printf(__VA_ARGS__);										\
+		} else {														\
+            printf("[NG] ");											\
+			printf(__VA_ARGS__);										\
+			char buf[65536];\
+			char cmd[256];\
+			sprintf(cmd, "%s/sbin/ihkosctl 0 kmsg", prefix);\
+			FILE* fp = popen(cmd, "r");		\
+			size_t nread = fread(buf, 1, sizeof(buf), fp);				\
+			buf[nread] = 0;												\
+			printf("%s", buf);											\
+			goto fn_fail;												\
+		}																\
+    } while(0)
+
+int main(int argc, char** argv) {
+    int ret = 0, status, ret_ihklib, pid;
+	FILE *fp;
+	char buf[65536];
+	size_t nread;
+
+	char cmd[1024];
+	char fn[256];
+	char kargs[256];
+
+	int cpus[4] = {6, 7, 8, 9};
+	int num_cpus = 4;
+
+	struct ihk_mem_chunk mem_chunks[4];
+	int num_mem_chunks;
+
+	printf("*** %s start *************************\n", test_name);
+	/*--------------------------------------------
+	 * Preparing                                  
+	 *--------------------------------------------*/
+	sprintf(cmd, "%s/sbin/mcstop+release.sh", prefix);
+	status = system(cmd);
+
+	// ihk_os_destroy_pseudofs
+	ret_ihklib = ihk_os_destroy_pseudofs(0);
+	fp = popen("cat /proc/mounts | grep /tmp/mcos/mcos0_sys", "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+
+	sprintf(cmd, "insmod %s/kmod/ihk.ko", prefix);
+	status = system(cmd);
+
+	sprintf(cmd, "insmod %s/kmod/ihk-smp-x86_64.ko ihk_start_irq=240 ihk_ikc_irq_core=0", prefix);
+	status = system(cmd);
+
+	sprintf(cmd, "insmod %s/kmod/mcctrl.ko", prefix);
+	status = system(cmd);
+
+	/*--------------------------------------------
+	 * Test                                  
+	 *--------------------------------------------*/
+	// create 0
+    ret_ihklib = ihk_create_os(0);
+
+	// reserve cpus
+	ret_ihklib = ihk_reserve_cpu(0, cpus, num_cpus);
+	//OKNG(ret_ihklib == 0, "ihk_reserve_cpu\n");
+
+	// assign cpus
+    ret_ihklib = ihk_os_assign_cpu(0, cpus, num_cpus);
+    //OKNG(ret_ihklib == 0, "ihk_os_assign_cpu\n");
+
+	// reserve mem 128m@0,128m@1
+	num_mem_chunks = 2;
+	mem_chunks[0].size = 128*1024*1024ULL;
+	mem_chunks[0].numa_node_number = 0;
+	mem_chunks[1].size = 128*1024*1024ULL;
+	mem_chunks[1].numa_node_number = 1;
+    ret_ihklib = ihk_reserve_mem(0, mem_chunks, num_mem_chunks);
+    //OKNG(ret_ihklib == 0, "ihk_reserve_mem (2)\n");
+
+	// assign mem 128m@0,128m@1
+	num_mem_chunks = 2;
+	mem_chunks[0].size = 128*1024*1024ULL;
+	mem_chunks[0].numa_node_number = 0;
+	mem_chunks[1].size = 128*1024*1024ULL;
+	mem_chunks[1].numa_node_number = 1;
+    ret_ihklib = ihk_os_assign_mem(0, mem_chunks, num_mem_chunks);
+    //OKNG(ret_ihklib == 0, "ihk_os_assign_mem (2)\n");
+
+	// load
+	sprintf(fn, "%s/smp-x86/kernel/mckernel.img", prefix);
+    ret_ihklib = ihk_os_load(0, fn);
+	//OKNG(ret_ihklib == 0, "ihk_os_load\n");
+
+	// kargs
+	sprintf(kargs, "hidos ksyslogd=0");
+    ret_ihklib = ihk_os_kargs(0, kargs);
+	//OKNG(ret_ihklib == 0, "ihk_os_kargs\n");
+
+	// boot
+    ret_ihklib = ihk_os_boot(0);
+	OKNG(ret_ihklib == 0, "ihk_os_boot\n");
+
+	/* Make sure that all initialization related transactions between McKernel and IHK finish
+	   sysfs_init(void) (in mckernel/kernel/sysfs.c)
+	   packet.msg = SCD_MSG_SYSFS_REQ_SETUP;
+	   sysfsm_work_main() in (mckernel/executer/kernel/mcctrl/sysfs.c)
+       sysfsm_req_setup
+       sysfsm_setup */
+	usleep(100*1000);
+
+	// create pseudofs
+	ret_ihklib = ihk_os_create_pseudofs(0);
+	fp = popen("cat /proc/mounts | grep /tmp/mcos/mcos0_sys", "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+	//OKNG(ret_ihklib == 0 &&
+	//	 strstr(buf, "/tmp/mcos/mcos0_sys") != NULL, "ihk_os_create_pseudofs()\n");
+
+	// mcexec
+	pid = fork();
+	if (pid == 0) {
+		printf("  start long mcexec...\n");
+		sprintf(cmd, "%s/bin/mcexec sleep 5", prefix);
+    	fp = popen(cmd, "r");
+		nread = fread(buf, 1, sizeof(buf), fp);
+		return 0;
+	}
+	usleep(100*1000);
+
+	// shutdown
+shutdown:
+    ret_ihklib = ihk_os_shutdown(0);
+	OKNG(ret_ihklib == 0, "shutdown during mcexec returned 0\n");
+	printf(" (But, mcexec process remain due to #846)\n");
+goto done_test;
+
+	// get status. Note that the smp_ihk_os_shutdown() transitions 
+	// smp-x86 status to BUILTIN_OS_STATUS_SHUTDOWN
+	// and smp_ihk_os_query_status() transitions os status to IHK_OS_STATUS_NOT_BOOTED.
+	ret_ihklib = ihk_os_get_status(0);
+	//OKNG(ret_ihklib == IHK_STATUS_SHUTDOWN ||
+	//	 ret_ihklib == IHK_STATUS_INACTIVE, "ihk_os_get_status (5) returned %d\n", ret_ihklib);
+
+destroy:
+    ret_ihklib = ihk_destroy_os(0, 0);
+	//OKNG(ret_ihklib == 0, "destroy immediately after boot\n");
+
+	sprintf(cmd, "rmmod %s/kmod/mcctrl.ko", prefix);
+	status = system(cmd);
+
+	// destroy pseudofs
+	ret_ihklib = ihk_os_destroy_pseudofs(0);
+	fp = popen("cat /proc/mounts | grep /tmp/mcos/mcos0_sys", "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+
+	sprintf(cmd, "rmmod %s/kmod/ihk-smp-x86_64.ko", prefix);
+	status = system(cmd);
+
+	sprintf(cmd, "rmmod %s/kmod/ihk.ko", prefix);
+	status = system(cmd);
+
+ done_test:
+	printf("*** All tests finished\n\n");
+
+ fn_exit:
+    return ret;
+ fn_fail:
+    goto fn_exit;
+}

test/mng_mod/issues/898_928/CT_006.c

@@ -0,0 +1,187 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <ihklib.h>
+#include <sys/types.h>
+
+#define MCK_DIR "/home/satoken/ppos"
+static char prefix[256] = MCK_DIR;
+
+static char test_name[64] = "CT_006";
+
+#define OKNG(cond, ...)													\
+    do {                                                                \
+		if(cond) {                                                      \
+			printf("[OK] ");											\
+			printf(__VA_ARGS__);										\
+		} else {														\
+            printf("[NG] ");											\
+			printf(__VA_ARGS__);										\
+			char buf[65536];\
+			char cmd[256];\
+			sprintf(cmd, "%s/sbin/ihkosctl 0 kmsg", prefix);\
+			FILE* fp = popen(cmd, "r");		\
+			size_t nread = fread(buf, 1, sizeof(buf), fp);				\
+			buf[nread] = 0;												\
+			printf("%s", buf);											\
+			goto fn_fail;												\
+		}																\
+    } while(0)
+
+int main(int argc, char** argv) {
+    int ret = 0, status, ret_ihklib, pid;
+	FILE *fp;
+	char buf[65536];
+	size_t nread;
+
+	char cmd[1024];
+	char fn[256];
+	char kargs[256];
+
+	int cpus[4] = {6, 7, 8, 9};
+	int num_cpus = 4;
+
+	struct ihk_mem_chunk mem_chunks[4];
+	int num_mem_chunks;
+
+	printf("*** %s start *************************\n", test_name);
+	/*--------------------------------------------
+	 * Preparing                                  
+	 *--------------------------------------------*/
+	sprintf(cmd, "%s/sbin/mcstop+release.sh", prefix);
+	status = system(cmd);
+
+	// ihk_os_destroy_pseudofs
+	ret_ihklib = ihk_os_destroy_pseudofs(0);
+	fp = popen("cat /proc/mounts | grep /tmp/mcos/mcos0_sys", "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+
+	sprintf(cmd, "insmod %s/kmod/ihk.ko", prefix);
+	status = system(cmd);
+
+	sprintf(cmd, "insmod %s/kmod/ihk-smp-x86_64.ko ihk_start_irq=240 ihk_ikc_irq_core=0", prefix);
+	status = system(cmd);
+
+	sprintf(cmd, "insmod %s/kmod/mcctrl.ko", prefix);
+	status = system(cmd);
+
+	/*--------------------------------------------
+	 * Test                                  
+	 *--------------------------------------------*/
+	// create 0
+    ret_ihklib = ihk_create_os(0);
+
+	// reserve cpus
+	ret_ihklib = ihk_reserve_cpu(0, cpus, num_cpus);
+	//OKNG(ret_ihklib == 0, "ihk_reserve_cpu\n");
+
+	// assign cpus
+    ret_ihklib = ihk_os_assign_cpu(0, cpus, num_cpus);
+    //OKNG(ret_ihklib == 0, "ihk_os_assign_cpu\n");
+
+	// reserve mem 128m@0,128m@1
+	num_mem_chunks = 2;
+	mem_chunks[0].size = 128*1024*1024ULL;
+	mem_chunks[0].numa_node_number = 0;
+	mem_chunks[1].size = 128*1024*1024ULL;
+	mem_chunks[1].numa_node_number = 1;
+    ret_ihklib = ihk_reserve_mem(0, mem_chunks, num_mem_chunks);
+    //OKNG(ret_ihklib == 0, "ihk_reserve_mem (2)\n");
+
+	// assign mem 128m@0,128m@1
+	num_mem_chunks = 2;
+	mem_chunks[0].size = 128*1024*1024ULL;
+	mem_chunks[0].numa_node_number = 0;
+	mem_chunks[1].size = 128*1024*1024ULL;
+	mem_chunks[1].numa_node_number = 1;
+    ret_ihklib = ihk_os_assign_mem(0, mem_chunks, num_mem_chunks);
+    //OKNG(ret_ihklib == 0, "ihk_os_assign_mem (2)\n");
+
+	// load
+	sprintf(fn, "%s/smp-x86/kernel/mckernel.img", prefix);
+    ret_ihklib = ihk_os_load(0, fn);
+	//OKNG(ret_ihklib == 0, "ihk_os_load\n");
+
+	// kargs
+	sprintf(kargs, "hidos ksyslogd=0");
+    ret_ihklib = ihk_os_kargs(0, kargs);
+	//OKNG(ret_ihklib == 0, "ihk_os_kargs\n");
+
+	// boot
+    ret_ihklib = ihk_os_boot(0);
+	OKNG(ret_ihklib == 0, "ihk_os_boot\n");
+
+	/* Make sure that all initialization related transactions between McKernel and IHK finish
+	   sysfs_init(void) (in mckernel/kernel/sysfs.c)
+	   packet.msg = SCD_MSG_SYSFS_REQ_SETUP;
+	   sysfsm_work_main() in (mckernel/executer/kernel/mcctrl/sysfs.c)
+       sysfsm_req_setup
+       sysfsm_setup */
+	usleep(100*1000);
+
+	// create pseudofs
+	ret_ihklib = ihk_os_create_pseudofs(0);
+	fp = popen("cat /proc/mounts | grep /tmp/mcos/mcos0_sys", "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+	//OKNG(ret_ihklib == 0 &&
+	//	 strstr(buf, "/tmp/mcos/mcos0_sys") != NULL, "ihk_os_create_pseudofs()\n");
+
+	// mcexec
+	fflush(stdout);
+	pid = fork();
+	if (pid == 0) {
+		printf("  start long mcexec...\n");
+		sprintf(cmd, "%s/bin/mcexec sleep 5", prefix);
+    	fp = popen(cmd, "r");
+		nread = fread(buf, 1, sizeof(buf), fp);
+		return 0;
+	}
+	usleep(100*1000);
+goto destroy;
+
+	// shutdown
+shutdown:
+    ret_ihklib = ihk_os_shutdown(0);
+	OKNG(ret_ihklib == 0, "shutdown during mcexec\n");
+
+	// get status. Note that the smp_ihk_os_shutdown() transitions 
+	// smp-x86 status to BUILTIN_OS_STATUS_SHUTDOWN
+	// and smp_ihk_os_query_status() transitions os status to IHK_OS_STATUS_NOT_BOOTED.
+	ret_ihklib = ihk_os_get_status(0);
+	//OKNG(ret_ihklib == IHK_STATUS_SHUTDOWN ||
+	//	 ret_ihklib == IHK_STATUS_INACTIVE, "ihk_os_get_status (5) returned %d\n", ret_ihklib);
+
+destroy:
+    ret_ihklib = ihk_destroy_os(0, 0);
+	OKNG(ret_ihklib != 0, "destroy during mcexec returned NOT 0\n");
+
+	// wait for child process
+	waitpid(pid, &status, 0);
+	usleep(100*1000);
+
+	sprintf(cmd, "rmmod %s/kmod/mcctrl.ko", prefix);
+	status = system(cmd);
+
+	// destroy pseudofs
+	ret_ihklib = ihk_os_destroy_pseudofs(0);
+	fp = popen("cat /proc/mounts | grep /tmp/mcos/mcos0_sys", "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+
+	sprintf(cmd, "rmmod %s/kmod/ihk-smp-x86_64.ko", prefix);
+	status = system(cmd);
+
+	sprintf(cmd, "rmmod %s/kmod/ihk.ko", prefix);
+	status = system(cmd);
+
+ done_test:
+	printf("*** All tests finished\n\n");
+
+ fn_exit:
+    return ret;
+ fn_fail:
+    goto fn_exit;
+}

test/mng_mod/issues/898_928/CT_007.c

@@ -0,0 +1,177 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <ihklib.h>
+#include <sys/types.h>
+
+#define MCK_DIR "/home/satoken/ppos"
+static char prefix[256] = MCK_DIR;
+
+static char test_name[64] = "CT_007";
+
+#define OKNG(cond, ...)													\
+    do {                                                                \
+		if(cond) {                                                      \
+			printf("[OK] ");											\
+			printf(__VA_ARGS__);										\
+		} else {														\
+            printf("[NG] ");											\
+			printf(__VA_ARGS__);										\
+			char buf[65536];\
+			char cmd[256];\
+			sprintf(cmd, "%s/sbin/ihkosctl 0 kmsg", prefix);\
+			FILE* fp = popen(cmd, "r");		\
+			size_t nread = fread(buf, 1, sizeof(buf), fp);				\
+			buf[nread] = 0;												\
+			printf("%s", buf);											\
+			goto fn_fail;												\
+		}																\
+    } while(0)
+
+int main(int argc, char** argv) {
+    int ret = 0, status, ret_ihklib;
+	FILE *fp;
+	char buf[65536];
+	size_t nread;
+
+	char cmd[1024];
+	char fn[256];
+	char kargs[256];
+
+	int cpus[4] = {6, 7, 8, 9};
+	int num_cpus = 4;
+
+	struct ihk_mem_chunk mem_chunks[4];
+	int num_mem_chunks;
+
+	printf("*** %s start *************************\n", test_name);
+	/*--------------------------------------------
+	 * Preparing                                  
+	 *--------------------------------------------*/
+	sprintf(cmd, "%s/sbin/mcstop+release.sh", prefix);
+	status = system(cmd);
+
+	// ihk_os_destroy_pseudofs
+	ret_ihklib = ihk_os_destroy_pseudofs(0);
+	fp = popen("cat /proc/mounts | grep /tmp/mcos/mcos0_sys", "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+
+	sprintf(cmd, "insmod %s/kmod/ihk.ko", prefix);
+	status = system(cmd);
+
+	sprintf(cmd, "insmod %s/kmod/ihk-smp-x86_64.ko ihk_start_irq=240 ihk_ikc_irq_core=0", prefix);
+	status = system(cmd);
+
+	sprintf(cmd, "insmod %s/kmod/mcctrl.ko", prefix);
+	status = system(cmd);
+
+	/*--------------------------------------------
+	 * Test                                  
+	 *--------------------------------------------*/
+	// create 0
+    ret_ihklib = ihk_create_os(0);
+
+	// reserve cpus
+	ret_ihklib = ihk_reserve_cpu(0, cpus, num_cpus);
+	//OKNG(ret_ihklib == 0, "ihk_reserve_cpu\n");
+
+	// assign cpus
+    ret_ihklib = ihk_os_assign_cpu(0, cpus, num_cpus);
+    //OKNG(ret_ihklib == 0, "ihk_os_assign_cpu\n");
+
+	// reserve mem 128m@0,128m@1
+	num_mem_chunks = 2;
+	mem_chunks[0].size = 128*1024*1024ULL;
+	mem_chunks[0].numa_node_number = 0;
+	mem_chunks[1].size = 128*1024*1024ULL;
+	mem_chunks[1].numa_node_number = 1;
+    ret_ihklib = ihk_reserve_mem(0, mem_chunks, num_mem_chunks);
+    //OKNG(ret_ihklib == 0, "ihk_reserve_mem (2)\n");
+
+	// assign mem 128m@0,128m@1
+	num_mem_chunks = 2;
+	mem_chunks[0].size = 128*1024*1024ULL;
+	mem_chunks[0].numa_node_number = 0;
+	mem_chunks[1].size = 128*1024*1024ULL;
+	mem_chunks[1].numa_node_number = 1;
+    ret_ihklib = ihk_os_assign_mem(0, mem_chunks, num_mem_chunks);
+    //OKNG(ret_ihklib == 0, "ihk_os_assign_mem (2)\n");
+
+	// load
+	sprintf(fn, "%s/smp-x86/kernel/mckernel.img", prefix);
+    ret_ihklib = ihk_os_load(0, fn);
+goto shutdown;
+	//OKNG(ret_ihklib == 0, "ihk_os_load\n");
+
+	// kargs
+	sprintf(kargs, "hidos ksyslogd=0");
+    ret_ihklib = ihk_os_kargs(0, kargs);
+	//OKNG(ret_ihklib == 0, "ihk_os_kargs\n");
+
+	// boot
+    ret_ihklib = ihk_os_boot(0);
+	OKNG(ret_ihklib == 0, "ihk_os_boot\n");
+
+	/* Make sure that all initialization related transactions between McKernel and IHK finish
+	   sysfs_init(void) (in mckernel/kernel/sysfs.c)
+	   packet.msg = SCD_MSG_SYSFS_REQ_SETUP;
+	   sysfsm_work_main() in (mckernel/executer/kernel/mcctrl/sysfs.c)
+       sysfsm_req_setup
+       sysfsm_setup */
+	usleep(100*1000);
+
+	// create pseudofs
+	ret_ihklib = ihk_os_create_pseudofs(0);
+	fp = popen("cat /proc/mounts | grep /tmp/mcos/mcos0_sys", "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+	//OKNG(ret_ihklib == 0 &&
+	//	 strstr(buf, "/tmp/mcos/mcos0_sys") != NULL, "ihk_os_create_pseudofs()\n");
+
+	// mcexec
+	sprintf(cmd, "%s/bin/mcexec ls -l | grep Makefile", prefix);
+    fp = popen(cmd, "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+	OKNG(strstr(buf, "Makefile") != NULL, "mcexec\n");
+
+	// shutdown
+shutdown:
+    ret_ihklib = ihk_os_shutdown(0);
+	OKNG(ret_ihklib == 0, "shutdown before boot returned 0\n");
+
+	// get status. Note that the smp_ihk_os_shutdown() transitions 
+	// smp-x86 status to BUILTIN_OS_STATUS_SHUTDOWN
+	// and smp_ihk_os_query_status() transitions os status to IHK_OS_STATUS_NOT_BOOTED.
+	ret_ihklib = ihk_os_get_status(0);
+	//OKNG(ret_ihklib == IHK_STATUS_SHUTDOWN ||
+	//	 ret_ihklib == IHK_STATUS_INACTIVE, "ihk_os_get_status (5) returned %d\n", ret_ihklib);
+
+destroy:
+    ret_ihklib = ihk_destroy_os(0, 0);
+	//OKNG(ret_ihklib == 0, "destroy immediately after boot\n");
+
+	sprintf(cmd, "rmmod %s/kmod/mcctrl.ko", prefix);
+	status = system(cmd);
+
+	// destroy pseudofs
+	ret_ihklib = ihk_os_destroy_pseudofs(0);
+	fp = popen("cat /proc/mounts | grep /tmp/mcos/mcos0_sys", "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+
+	sprintf(cmd, "rmmod %s/kmod/ihk-smp-x86_64.ko", prefix);
+	status = system(cmd);
+
+	sprintf(cmd, "rmmod %s/kmod/ihk.ko", prefix);
+	status = system(cmd);
+
+	printf("*** All tests finished\n\n");
+
+ fn_exit:
+    return ret;
+ fn_fail:
+    goto fn_exit;
+}

test/mng_mod/issues/898_928/CT_008.c

@@ -0,0 +1,180 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <ihklib.h>
+#include <sys/types.h>
+
+#define MCK_DIR "/home/satoken/ppos"
+static char prefix[256] = MCK_DIR;
+
+static char test_name[64] = "CT_008";
+
+#define OKNG(cond, ...)													\
+    do {                                                                \
+		if(cond) {                                                      \
+			printf("[OK] ");											\
+			printf(__VA_ARGS__);										\
+		} else {														\
+            printf("[NG] ");											\
+			printf(__VA_ARGS__);										\
+			char buf[65536];\
+			char cmd[256];\
+			sprintf(cmd, "%s/sbin/ihkosctl 0 kmsg", prefix);\
+			FILE* fp = popen(cmd, "r");		\
+			size_t nread = fread(buf, 1, sizeof(buf), fp);				\
+			buf[nread] = 0;												\
+			printf("%s", buf);											\
+			goto fn_fail;												\
+		}																\
+    } while(0)
+
+int main(int argc, char** argv) {
+    int ret = 0, status, ret_ihklib;
+	FILE *fp;
+	char buf[65536];
+	size_t nread;
+
+	char cmd[1024];
+	char fn[256];
+	char kargs[256];
+
+	int cpus[4] = {6, 7, 8, 9};
+	int num_cpus = 4;
+
+	struct ihk_mem_chunk mem_chunks[4];
+	int num_mem_chunks;
+
+	printf("*** %s start *************************\n", test_name);
+	/*--------------------------------------------
+	 * Preparing                                  
+	 *--------------------------------------------*/
+	sprintf(cmd, "%s/sbin/mcstop+release.sh", prefix);
+	status = system(cmd);
+
+	// ihk_os_destroy_pseudofs
+	ret_ihklib = ihk_os_destroy_pseudofs(0);
+	fp = popen("cat /proc/mounts | grep /tmp/mcos/mcos0_sys", "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+
+	sprintf(cmd, "insmod %s/kmod/ihk.ko", prefix);
+	status = system(cmd);
+
+	sprintf(cmd, "insmod %s/kmod/ihk-smp-x86_64.ko ihk_start_irq=240 ihk_ikc_irq_core=0", prefix);
+	status = system(cmd);
+
+	sprintf(cmd, "insmod %s/kmod/mcctrl.ko", prefix);
+	status = system(cmd);
+
+	/*--------------------------------------------
+	 * Test                                  
+	 *--------------------------------------------*/
+	// create 0
+    ret_ihklib = ihk_create_os(0);
+
+	// reserve cpus
+	ret_ihklib = ihk_reserve_cpu(0, cpus, num_cpus);
+	//OKNG(ret_ihklib == 0, "ihk_reserve_cpu\n");
+
+	// assign cpus
+    ret_ihklib = ihk_os_assign_cpu(0, cpus, num_cpus);
+    //OKNG(ret_ihklib == 0, "ihk_os_assign_cpu\n");
+
+	// reserve mem 128m@0,128m@1
+	num_mem_chunks = 2;
+	mem_chunks[0].size = 128*1024*1024ULL;
+	mem_chunks[0].numa_node_number = 0;
+	mem_chunks[1].size = 128*1024*1024ULL;
+	mem_chunks[1].numa_node_number = 1;
+    ret_ihklib = ihk_reserve_mem(0, mem_chunks, num_mem_chunks);
+    //OKNG(ret_ihklib == 0, "ihk_reserve_mem (2)\n");
+
+	// assign mem 128m@0,128m@1
+	num_mem_chunks = 2;
+	mem_chunks[0].size = 128*1024*1024ULL;
+	mem_chunks[0].numa_node_number = 0;
+	mem_chunks[1].size = 128*1024*1024ULL;
+	mem_chunks[1].numa_node_number = 1;
+    ret_ihklib = ihk_os_assign_mem(0, mem_chunks, num_mem_chunks);
+    //OKNG(ret_ihklib == 0, "ihk_os_assign_mem (2)\n");
+
+	// load
+	sprintf(fn, "%s/smp-x86/kernel/mckernel.img", prefix);
+    ret_ihklib = ihk_os_load(0, fn);
+	//OKNG(ret_ihklib == 0, "ihk_os_load\n");
+
+	// kargs
+	sprintf(kargs, "hidos ksyslogd=0");
+    ret_ihklib = ihk_os_kargs(0, kargs);
+	//OKNG(ret_ihklib == 0, "ihk_os_kargs\n");
+
+	// boot
+    ret_ihklib = ihk_os_boot(0);
+	OKNG(ret_ihklib == 0, "ihk_os_boot\n");
+
+	/* Make sure that all initialization related transactions between McKernel and IHK finish
+	   sysfs_init(void) (in mckernel/kernel/sysfs.c)
+	   packet.msg = SCD_MSG_SYSFS_REQ_SETUP;
+	   sysfsm_work_main() in (mckernel/executer/kernel/mcctrl/sysfs.c)
+       sysfsm_req_setup
+       sysfsm_setup */
+	usleep(100*1000);
+
+	// create pseudofs
+	ret_ihklib = ihk_os_create_pseudofs(0);
+	fp = popen("cat /proc/mounts | grep /tmp/mcos/mcos0_sys", "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+	//OKNG(ret_ihklib == 0 &&
+	//	 strstr(buf, "/tmp/mcos/mcos0_sys") != NULL, "ihk_os_create_pseudofs()\n");
+
+	// mcexec
+	sprintf(cmd, "%s/bin/mcexec ls -l | grep Makefile", prefix);
+    fp = popen(cmd, "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+	OKNG(strstr(buf, "Makefile") != NULL, "do mcexec\n");
+
+	// shutdown
+shutdown:
+    ret_ihklib = ihk_os_shutdown(0);
+	OKNG(ret_ihklib == 0, "shutdown after mcexec returned 0\n");
+
+	// get status. Note that the smp_ihk_os_shutdown() transitions 
+	// smp-x86 status to BUILTIN_OS_STATUS_SHUTDOWN
+	// and smp_ihk_os_query_status() transitions os status to IHK_OS_STATUS_NOT_BOOTED.
+	ret_ihklib = ihk_os_get_status(0);
+	OKNG(ret_ihklib == IHK_STATUS_SHUTDOWN ||
+		 ret_ihklib == IHK_STATUS_INACTIVE, "ihk_os_get_status returned SHUTDOWN or INACTIVE\n");
+
+	// shutdown again
+    ret_ihklib = ihk_os_shutdown(0);
+	OKNG(ret_ihklib != 0, "shutdown after shutdown returned NOT 0\n");
+
+destroy:
+    ret_ihklib = ihk_destroy_os(0, 0);
+	//OKNG(ret_ihklib == 0, "destroy immediately after boot\n");
+
+	sprintf(cmd, "rmmod %s/kmod/mcctrl.ko", prefix);
+	status = system(cmd);
+
+	// destroy pseudofs
+	ret_ihklib = ihk_os_destroy_pseudofs(0);
+	fp = popen("cat /proc/mounts | grep /tmp/mcos/mcos0_sys", "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+
+	sprintf(cmd, "rmmod %s/kmod/ihk-smp-x86_64.ko", prefix);
+	status = system(cmd);
+
+	sprintf(cmd, "rmmod %s/kmod/ihk.ko", prefix);
+	status = system(cmd);
+
+	printf("*** All tests finished\n\n");
+
+ fn_exit:
+    return ret;
+ fn_fail:
+    goto fn_exit;
+}

test/mng_mod/issues/898_928/CT_009.c

@@ -0,0 +1,177 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <ihklib.h>
+#include <sys/types.h>
+
+#define MCK_DIR "/home/satoken/ppos"
+static char prefix[256] = MCK_DIR;
+
+static char test_name[64] = "CT_009";
+
+#define OKNG(cond, ...)													\
+    do {                                                                \
+		if(cond) {                                                      \
+			printf("[OK] ");											\
+			printf(__VA_ARGS__);										\
+		} else {														\
+            printf("[NG] ");											\
+			printf(__VA_ARGS__);										\
+			char buf[65536];\
+			char cmd[256];\
+			sprintf(cmd, "%s/sbin/ihkosctl 0 kmsg", prefix);\
+			FILE* fp = popen(cmd, "r");		\
+			size_t nread = fread(buf, 1, sizeof(buf), fp);				\
+			buf[nread] = 0;												\
+			printf("%s", buf);											\
+			goto fn_fail;												\
+		}																\
+    } while(0)
+
+int main(int argc, char** argv) {
+    int ret = 0, status, ret_ihklib;
+	FILE *fp;
+	char buf[65536];
+	size_t nread;
+
+	char cmd[1024];
+	char fn[256];
+	char kargs[256];
+
+	int cpus[4] = {6, 7, 8, 9};
+	int num_cpus = 4;
+
+	struct ihk_mem_chunk mem_chunks[4];
+	int num_mem_chunks;
+
+	printf("*** %s start *************************\n", test_name);
+	/*--------------------------------------------
+	 * Preparing                                  
+	 *--------------------------------------------*/
+	sprintf(cmd, "%s/sbin/mcstop+release.sh", prefix);
+	status = system(cmd);
+
+	// ihk_os_destroy_pseudofs
+	ret_ihklib = ihk_os_destroy_pseudofs(0);
+	fp = popen("cat /proc/mounts | grep /tmp/mcos/mcos0_sys", "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+
+	sprintf(cmd, "insmod %s/kmod/ihk.ko", prefix);
+	status = system(cmd);
+
+	sprintf(cmd, "insmod %s/kmod/ihk-smp-x86_64.ko ihk_start_irq=240 ihk_ikc_irq_core=0", prefix);
+	status = system(cmd);
+
+	sprintf(cmd, "insmod %s/kmod/mcctrl.ko", prefix);
+	status = system(cmd);
+
+	/*--------------------------------------------
+	 * Test                                  
+	 *--------------------------------------------*/
+	// create 0
+    ret_ihklib = ihk_create_os(0);
+
+	// reserve cpus
+	ret_ihklib = ihk_reserve_cpu(0, cpus, num_cpus);
+	//OKNG(ret_ihklib == 0, "ihk_reserve_cpu\n");
+
+	// assign cpus
+    ret_ihklib = ihk_os_assign_cpu(0, cpus, num_cpus);
+    //OKNG(ret_ihklib == 0, "ihk_os_assign_cpu\n");
+
+	// reserve mem 128m@0,128m@1
+	num_mem_chunks = 2;
+	mem_chunks[0].size = 128*1024*1024ULL;
+	mem_chunks[0].numa_node_number = 0;
+	mem_chunks[1].size = 128*1024*1024ULL;
+	mem_chunks[1].numa_node_number = 1;
+    ret_ihklib = ihk_reserve_mem(0, mem_chunks, num_mem_chunks);
+    //OKNG(ret_ihklib == 0, "ihk_reserve_mem (2)\n");
+
+	// assign mem 128m@0,128m@1
+	num_mem_chunks = 2;
+	mem_chunks[0].size = 128*1024*1024ULL;
+	mem_chunks[0].numa_node_number = 0;
+	mem_chunks[1].size = 128*1024*1024ULL;
+	mem_chunks[1].numa_node_number = 1;
+    ret_ihklib = ihk_os_assign_mem(0, mem_chunks, num_mem_chunks);
+    //OKNG(ret_ihklib == 0, "ihk_os_assign_mem (2)\n");
+
+	// load
+	sprintf(fn, "%s/smp-x86/kernel/mckernel.img", prefix);
+    ret_ihklib = ihk_os_load(0, fn);
+	//OKNG(ret_ihklib == 0, "ihk_os_load\n");
+
+	// kargs
+	sprintf(kargs, "hidos ksyslogd=0");
+    ret_ihklib = ihk_os_kargs(0, kargs);
+	//OKNG(ret_ihklib == 0, "ihk_os_kargs\n");
+
+	// boot
+    ret_ihklib = ihk_os_boot(0);
+goto shutdown;
+	OKNG(ret_ihklib == 0, "ihk_os_boot\n");
+
+	/* Make sure that all initialization related transactions between McKernel and IHK finish
+	   sysfs_init(void) (in mckernel/kernel/sysfs.c)
+	   packet.msg = SCD_MSG_SYSFS_REQ_SETUP;
+	   sysfsm_work_main() in (mckernel/executer/kernel/mcctrl/sysfs.c)
+       sysfsm_req_setup
+       sysfsm_setup */
+	usleep(100*1000);
+
+	// create pseudofs
+	ret_ihklib = ihk_os_create_pseudofs(0);
+	fp = popen("cat /proc/mounts | grep /tmp/mcos/mcos0_sys", "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+	//OKNG(ret_ihklib == 0 &&
+	//	 strstr(buf, "/tmp/mcos/mcos0_sys") != NULL, "ihk_os_create_pseudofs()\n");
+
+	// mcexec
+	sprintf(cmd, "%s/bin/mcexec ls -l | grep Makefile", prefix);
+    fp = popen(cmd, "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+	OKNG(strstr(buf, "Makefile") != NULL, "mcexec\n");
+
+	// shutdown
+shutdown:
+    ret_ihklib = ihk_os_shutdown(0);
+	OKNG(ret_ihklib == 0, "force shutdown immediately after boot returned 0\n");
+
+	// get status. Note that the smp_ihk_os_shutdown() transitions 
+	// smp-x86 status to BUILTIN_OS_STATUS_SHUTDOWN
+	// and smp_ihk_os_query_status() transitions os status to IHK_OS_STATUS_NOT_BOOTED.
+	ret_ihklib = ihk_os_get_status(0);
+	//OKNG(ret_ihklib == IHK_STATUS_SHUTDOWN ||
+	//	 ret_ihklib == IHK_STATUS_INACTIVE, "ihk_os_get_status (5) returned %d\n", ret_ihklib);
+
+destroy:
+    ret_ihklib = ihk_destroy_os(0, 0);
+	//OKNG(ret_ihklib == 0, "destroy immediately after boot\n");
+
+	sprintf(cmd, "rmmod %s/kmod/mcctrl.ko", prefix);
+	status = system(cmd);
+
+	// destroy pseudofs
+	ret_ihklib = ihk_os_destroy_pseudofs(0);
+	fp = popen("cat /proc/mounts | grep /tmp/mcos/mcos0_sys", "r");
+	nread = fread(buf, 1, sizeof(buf), fp);
+	buf[nread] = 0;
+
+	sprintf(cmd, "rmmod %s/kmod/ihk-smp-x86_64.ko", prefix);
+	status = system(cmd);
+
+	sprintf(cmd, "rmmod %s/kmod/ihk.ko", prefix);
+	status = system(cmd);
+
+	printf("*** All tests finished\n\n");
+
+ fn_exit:
+    return ret;
+ fn_fail:
+    goto fn_exit;
+}

test/mng_mod/issues/898_928/Makefile

@@ -0,0 +1,51 @@
+CC = gcc
+MCK_DIR=/home/satoken/ppos
+TARGET=CT_001 CT_002 CT_003 CT_004 CT_005 CT_006 CT_007 CT_008 CT_009
+
+CPPFLAGS = -I$(MCK_DIR)/include
+LDFLAGS = -L$(MCK_DIR)/lib -lihk -Wl,-rpath -Wl,$(MCK_DIR)/lib -lbfd
+
+all: $(TARGET)
+
+CT_001: CT_001.c
+	$(CC) -o $@ $(CPPFLAGS) $(LDFLAGS) $^
+
+CT_002: CT_002.c
+	$(CC) -o $@ $(CPPFLAGS) $(LDFLAGS) $^
+
+CT_003: CT_003.c
+	$(CC) -o $@ $(CPPFLAGS) $(LDFLAGS) $^
+
+CT_004: CT_004.c
+	$(CC) -o $@ $(CPPFLAGS) $(LDFLAGS) $^
+
+CT_005: CT_005.c
+	$(CC) -o $@ $(CPPFLAGS) $(LDFLAGS) $^
+
+CT_006: CT_006.c
+	$(CC) -o $@ $(CPPFLAGS) $(LDFLAGS) $^
+
+CT_007: CT_007.c
+	$(CC) -o $@ $(CPPFLAGS) $(LDFLAGS) $^
+
+CT_008: CT_008.c
+	$(CC) -o $@ $(CPPFLAGS) $(LDFLAGS) $^
+
+CT_009: CT_009.c
+	$(CC) -o $@ $(CPPFLAGS) $(LDFLAGS) $^
+
+
+test:: $(TARGET)
+	-sudo ./CT_001
+	-sudo ./CT_002
+	-sudo ./CT_003
+	-sudo ./CT_004
+	#-sudo ./CT_005
+	-sudo ./CT_006
+	-sudo ./CT_007
+	-sudo ./CT_008
+	-sudo ./CT_009
+
+clean:
+	rm -f $(TARGET)
+

test/mng_mod/issues/898_928/README

@@ -0,0 +1,93 @@
+【Issue#898,#928 動作確認】
+Issue#898, #928が解決され、既存機能に影響がないことをihklibテストスイートを用いた確認（2項目）と、
+McKernelの起動/終了の基本動作確認（9項目）の計11項目のテストによって確認した。
+
+①ihklibテストスイートを用いた確認
+ ・Issue#898
+ ihklibテストスイートのihklib001_lin.c を以下のように修正して1000回繰り返し実行し、
+ すべての実行においてテストをパスすることを確認した。
+  - McKernelのブート処理の直後にgoto文を追加し、シャットダウン処理の直前に移動する
+
+ ・Issue#928
+ ihklibテストスイートのihklib001_lin.c を以下のように修正して1000回繰り返し実行し、
+ すべての実行においてテストをパスすることを確認した。
+  - McKernelプロセス(mcexec)の実行直後にgoto文を追加し、シャットダウン処理の直前に移動する
+ 
+②McKernelの起動/終了の基本動作確認
+McKernelの状態と、終了処理(shutdown, destroy)の組み合わせで、
+9項目のテストを実施した。
+
+基本動作確認の詳細を以下に示す。
+
+1. ファイルの説明
+  CT_xxx.c              各テスト項目のテストプログラム
+  force_shutdown.patch  McKernelの状態がRUNNINGにならなかった場合に行う強制シャットダウン処理を
+                        発生させるパッチファイル
+  result.log            テストプログラムの実行結果
+
+2. テストの実行方法
+ 以下の手順でテストを実行する
+	1. Makefileの変数MCK_DIRの内容を、McKernelがインストールされているディレクトリに変更する
+	2. CT_xxx.c の定数MCK_DIRの内容を、McKernelがインストールされているディレクトリに変更する
+	3. sh make test を実行する
+
+3. テスト項目
+以下の条件でMcKernelの起動/終了が正常に行われることを確認する
+なお、McKernelの起動、終了の操作はihklibを用いて実施する
+
+CT_001:
+	1. McKernelを起動(ihk_os_boot)する
+	2. 起動の直後にMcKernelを終了(ihk_os_shutdown)する
+	⇒ ihk_os_shutdownが0を返す
+
+CT_002:
+	1. McKernelを起動(ihk_os_boot)する
+	2. 起動の直後に対象のIHK_OSインスタンスを破棄(ihk_os_destroy)する
+	⇒ ihk_os_destroyが0を返す
+
+CT_003:
+	1. McKernelを起動(ihk_os_boot)する
+	2. McKernelプロセスを実行する
+	3. McKernelプロセスの実行終了直後にMcKernelを終了(ihk_os_shutdown)する
+	⇒ ihk_os_shutdownが0を返す
+
+CT_004:
+	1. McKernelを起動(ihk_os_boot)する
+	2. McKernelプロセスを実行する
+	3. McKernelプロセスの実行終了直後に対象のIHK_OSインスタンスを破棄(ihk_os_destroy)する
+	⇒ ihk_os_destroyが0を返す
+
+CT_005:
+	1. McKernelを起動(ihk_os_boot)する
+	2. McKernelプロセスを実行する
+	3. McKernelプロセスの実行中にMcKernelを終了(ihk_os_shutdown)する
+	⇒ ihk_os_shutdownが0を返す
+
+CT_006:
+	1. McKernelを起動(ihk_os_boot)する
+	2. McKernelプロセスを実行する
+	3. McKernelプロセスの実行中に対象のIHK_OSインスタンスを破棄(ihk_os_destroy)する
+	⇒ ihk_os_destroyが0以外を返す (ref_count != 0)
+
+CT_007:
+	1. McKernelの起動処理の中の、カーネルイメージのロード(ihk_os_load)までを実施する
+	2. McKernelを終了(ihk_os_shutdown)する
+	⇒ ihk_os_shutdownが0を返す
+
+CT_008:
+	1. McKernelを起動(ihk_os_boot)する
+	2. McKernelプロセスを実行する
+	3. McKernelプロセスの実行終了直後にMcKernelを終了(ihk_os_shutdown)する
+	4. ihk_os_get_status がSHUTDOWNまたは、INACTIVEを返すことを確認する
+	5. 終了したMcKernelに対して、再度終了(ihk_os_shutdown)する
+	⇒ ihk_os_shutdownが0以外を返す
+
+CT_009:
+	前提：force_shutdown.patch を適用してMcKernelをビルドする
+	1. McKernelを起動(ihk_os_boot)する
+	2. 起動の直後にMcKernelを終了(ihk_os_shutdown)する
+	⇒ ihk_os_shutdownが0を返す
+
+4. 結果
+テストプログラムの実行結果はresult.log に出力される。
+上記9項目で[OK]が出力されていることを確認した。

test/mng_mod/issues/898_928/force_shutdown.patch

@@ -0,0 +1,20 @@
+diff --git a/executer/kernel/mcctrl/driver.c b/executer/kernel/mcctrl/driver.c
+index 390dac6..a87aa45 100644
+--- a/executer/kernel/mcctrl/driver.c
++++ b/executer/kernel/mcctrl/driver.c
+@@ -171,12 +171,12 @@ int mcctrl_os_shutdown_notifier(int os_index)
+ {
+ 	if (os[os_index]) {
+ 		/* Wait for os running */
+-		if (ihk_os_wait_for_status(os[os_index], IHK_OS_STATUS_RUNNING, 0, 200) != 0) {
+-			printk("IHK: OS does not become RUNNING in shutdown. Force shutdown.\n");
++//		if (ihk_os_wait_for_status(os[os_index], IHK_OS_STATUS_RUNNING, 0, 200) != 0) {
++//			printk("IHK: OS does not become RUNNING in shutdown. Force shutdown.\n");
+ 			/* send nmi to force shutdown */
+ 			ihk_os_send_nmi(os[os_index], 3);
+ 			mdelay(200);
+-		}
++//		}
+ 
+ 		sysfsm_cleanup(os[os_index]);
+ 		free_topology_info(os[os_index]);

test/mng_mod/issues/898_928/result.log

@@ -0,0 +1,58 @@
+sudo ./CT_001
+*** CT_001 start *************************
+[OK] shutdown immediately after boot returned 0
+*** All tests finished
+
+sudo ./CT_002
+*** CT_002 start *************************
+[OK] destroy immediately after boot returned NOT 0
+*** All tests finished
+
+sudo ./CT_003
+*** CT_003 start *************************
+[OK] ihk_os_boot
+[OK] done mcexec
+[OK] shutdown after mcexec returned 0
+*** All tests finished
+
+sudo ./CT_004
+*** CT_004 start *************************
+[OK] ihk_os_boot
+[OK] do mcexec
+[OK] destroy after mcexec returned 0
+*** All tests finished
+
+-sudo ./CT_005
+*** CT_005 start *************************
+[OK] ihk_os_boot
+  start long mcexec...
+[OK] shutdown during mcexec returned 0
+ (But, mcexec process remain due to #846)
+*** All tests finished
+
+sudo ./CT_006
+*** CT_006 start *************************
+[OK] ihk_os_boot
+  start long mcexec...
+[OK] destroy during mcexec returned NOT 0
+*** All tests finished
+
+sudo ./CT_007
+*** CT_007 start *************************
+[OK] shutdown before boot returned 0
+*** All tests finished
+
+sudo ./CT_008
+*** CT_008 start *************************
+[OK] ihk_os_boot
+[OK] do mcexec
+[OK] shutdown after mcexec returned 0
+[OK] ihk_os_get_status returned SHUTDOWN or INACTIVE
+[OK] shutdown after shutdown returned NOT 0
+*** All tests finished
+
+sudo ./CT_009
+*** CT_009 start *************************
+[OK] force shutdown immediately after boot returned 0
+*** All tests finished
+