diff --git a/arch/x86/kernel/cpu.c b/arch/x86/kernel/cpu.c
index 6166c171..3237125e 100644
--- a/arch/x86/kernel/cpu.c
+++ b/arch/x86/kernel/cpu.c
@@ -792,7 +792,7 @@ void init_cpu(void)
 	init_pat();
 }
 
-void setup_x86(void)
+void setup_x86_phase1(void)
 {
 	cpu_disable_interrupt();
 
@@ -801,7 +801,10 @@ void setup_x86(void)
 	init_gdt();
 
 	init_page_table();
+}
 
+void setup_x86_phase2(void)
+{
 	check_no_execute();
 
 	init_lapic_bsp();
diff --git a/arch/x86/kernel/include/arch/rusage.h b/arch/x86/kernel/include/arch/rusage.h
deleted file mode 100644
index dfb6e9af..00000000
--- a/arch/x86/kernel/include/arch/rusage.h
+++ /dev/null
@@ -1,58 +0,0 @@
-#ifndef ARCH_RUSAGE_H_INCLUDED
-#define ARCH_RUSAGE_H_INCLUDED
-
-#define DEBUG_RUSAGE
-
-#define IHK_OS_PGSIZE_4KB 0
-#define IHK_OS_PGSIZE_2MB 1
-#define IHK_OS_PGSIZE_1GB 2
-
-extern struct rusage_global *rusage;
-
-extern int sprintf(char * buf, const char *fmt, ...);
-
-#define DEBUG_ARCH_RUSAGE
-#ifdef DEBUG_ARCH_RUSAGE
-#define	dprintf(...)											\
-	do {														\
-		char msg[1024];											\
-		sprintf(msg, __VA_ARGS__);								\
-		kprintf("%s,%s", __FUNCTION__, msg);					\
-	} while (0);
-#define	eprintf(...)											\
-	do {														\
-		char msg[1024];											\
-		sprintf(msg, __VA_ARGS__);								\
-		kprintf("%s,%s", __FUNCTION__, msg);					\
-	} while (0);
-#else
-#define dprintf(...) do {  } while (0)
-#define	eprintf(...)											\
-	do {														\
-		char msg[1024];											\
-		sprintf(msg, __VA_ARGS__);								\
-		kprintf("%s,%s", __FUNCTION__, msg);					\
-	} while (0);
-#endif
-
-static inline int rusage_pgsize_to_pgtype(size_t pgsize)
-{
-	int ret = IHK_OS_PGSIZE_4KB;
-	switch (pgsize) {
-	case PTL1_SIZE:
-		ret = IHK_OS_PGSIZE_4KB;
-		break;
-	case PTL2_SIZE:
-		ret = IHK_OS_PGSIZE_2MB;
-		break;
-	case PTL3_SIZE:
-		ret = IHK_OS_PGSIZE_1GB;
-		break;
-	default:
-		eprintf("unknown pgsize=%ld\n", pgsize);
-		break;
-	}
-	return ret;
-}
-
-#endif /* !defined(ARCH_RUSAGE_H_INCLUDED) */
diff --git a/arch/x86/kernel/include/arch_rusage.h b/arch/x86/kernel/include/arch_rusage.h
new file mode 100644
index 00000000..134a2725
--- /dev/null
+++ b/arch/x86/kernel/include/arch_rusage.h
@@ -0,0 +1,32 @@
+#ifndef ARCH_RUSAGE_H_INCLUDED
+#define ARCH_RUSAGE_H_INCLUDED
+
+#define DEBUG_RUSAGE
+
+#define IHK_OS_PGSIZE_4KB 0
+#define IHK_OS_PGSIZE_2MB 1
+#define IHK_OS_PGSIZE_1GB 2
+
+extern struct rusage_global *rusage;
+
+static inline int rusage_pgsize_to_pgtype(size_t pgsize)
+{
+	int ret = IHK_OS_PGSIZE_4KB;
+	switch (pgsize) {
+	case PTL1_SIZE:
+		ret = IHK_OS_PGSIZE_4KB;
+		break;
+	case PTL2_SIZE:
+		ret = IHK_OS_PGSIZE_2MB;
+		break;
+	case PTL3_SIZE:
+		ret = IHK_OS_PGSIZE_1GB;
+		break;
+	default:
+		kprintf("%s: Error: Unknown pgsize=%ld\n", __FUNCTION__, pgsize);
+		break;
+	}
+	return ret;
+}
+
+#endif /* !defined(ARCH_RUSAGE_H_INCLUDED) */
diff --git a/arch/x86/tools/mcreboot-smp-x86.sh.in b/arch/x86/tools/mcreboot-smp-x86.sh.in
index 550baf0b..1738b19f 100644
--- a/arch/x86/tools/mcreboot-smp-x86.sh.in
+++ b/arch/x86/tools/mcreboot-smp-x86.sh.in
@@ -372,7 +372,7 @@ fi
 
 if [ "$ikc_map" != "" ]; then
 	# Specify IKC map
-	if ! ${SBINDIR}/ihkosctl 0 ikc_map ${ikc_map}; then
+	if ! ${SBINDIR}/ihkosctl 0 set ikc_map ${ikc_map}; then
 		echo "error: assign CPUs" >&2
 		error_exit "os_created"
 	fi
diff --git a/config.h.in b/config.h.in
index d7df6e9d..e3721b6c 100644
--- a/config.h.in
+++ b/config.h.in
@@ -1,5 +1,8 @@
 /* config.h.in.  Generated from configure.ac by autoheader.  */
 
+/* Path of install directory for binary */
+#undef BINDIR
+
 /* whether mcoverlayfs is enabled */
 #undef ENABLE_MCOVERLAYFS
 
@@ -18,6 +21,9 @@
 /* Define to 1 if you have the `bfd' library (-lbfd). */
 #undef HAVE_LIBBFD
 
+/* Define to 1 if you have the `iberty' library (-liberty). */
+#undef HAVE_LIBIBERTY
+
 /* Define to 1 if you have the <memory.h> header file. */
 #undef HAVE_MEMORY_H
 
@@ -72,6 +78,9 @@
 /* Define to address of kernel symbol vdso_pages, or 0 if exported */
 #undef MCCTRL_KSYM_vdso_pages
 
+/* Define to address of kernel symbol vdso_spec, or 0 if exported */
+#undef MCCTRL_KSYM_vdso_spec
+
 /* Define to address of kernel symbol vdso_start, or 0 if exported */
 #undef MCCTRL_KSYM_vdso_start
 
@@ -81,9 +90,6 @@
 /* Define to address of kernel symbol zap_page_range, or 0 if exported */
 #undef MCCTRL_KSYM_zap_page_range
 
-/* Define to address of kernel symbol vdso_spec, or 0 if exported. POSTK_DEBUG_ARCH_DEP_50 */
-#undef MCCTRL_KSYM_vdso_spec
-
 /* McKernel specific headers */
 #undef MCKERNEL_INCDIR
 
@@ -111,11 +117,8 @@
 /* Path of bind-mount source directory */
 #undef ROOTFSDIR
 
-/* Define to 1 if you have the ANSI C header files. */
-#undef STDC_HEADERS
-
-/* install directory for system binary. */
+/* Path of install directory for system binary */
 #undef SBINDIR
 
-/* install directory for binary. */
-#undef BINDIR
+/* Define to 1 if you have the ANSI C header files. */
+#undef STDC_HEADERS
diff --git a/configure b/configure
index 4f0bd389..93d955a3 100755
--- a/configure
+++ b/configure
@@ -628,6 +628,7 @@ IHK_RELEASE_DATE
 DCFA_VERSION
 MCKERNEL_VERSION
 IHK_VERSION
+ENABLE_QLMPI
 ENABLE_RUSAGE
 ENABLE_MCOVERLAYFS
 MANDIR
@@ -635,7 +636,9 @@ KERNDIR
 KMODDIR
 INCLUDEDIR
 ETCDIR
+INCDIR
 MCKERNEL_LIBDIR
+MCKERNEL_INCDIR
 SBINDIR
 BINDIR
 TARGET
@@ -643,6 +646,7 @@ UNAME_R
 KDIR
 ARCH
 XCC
+FGREP
 EGREP
 GREP
 CPP
@@ -694,6 +698,9 @@ SHELL'
 ac_subst_files=''
 ac_user_opts='
 enable_option_checking
+with_mpi
+with_mpi_include
+with_mpi_lib
 with_kernelsrc
 with_target
 with_system_map
@@ -701,6 +708,7 @@ enable_dcfa
 enable_memdump
 enable_mcoverlayfs
 enable_rusage
+enable_qlmpi
 with_uname_r
 '
       ac_precious_vars='build_alias
@@ -1325,10 +1333,16 @@ Optional Features:
   --enable-memdump        enable dumping memory and analyzing a dump
   --enable-mcoverlayfs    enable mcoverlayfs implementation
   --enable-rusage         enable rusage implementation
+  --enable-qlmpi          enable qlmpi implementation
 
 Optional Packages:
   --with-PACKAGE[=ARG]    use PACKAGE [ARG=yes]
   --without-PACKAGE       do not use PACKAGE (same as --with-PACKAGE=no)
+  --with-mpi=PATH         specify path where mpi include directory and lib
+                          directory can be found
+  --with-mpi-include=PATH specify path where mpi include directory can be
+                          found
+  --with-mpi-lib=PATH     specify path where mpi lib directory can be found
   --with-kernelsrc=path   Path to 'kernel src', default is
                           /lib/modules/uname_r/build
   --with-target={attached-mic | builtin-mic | builtin-x86 | smp-x86}
@@ -2061,6 +2075,10 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu
 
 
 
+
+
+
+
 IHK_VERSION=0.9.0
 MCKERNEL_VERSION=0.9.0
 DCFA_VERSION=0.9.0
@@ -3310,6 +3328,187 @@ if test "x$numa_lib_found" != "xyes"; then :
   as_fn_error $? "Unable to find NUMA library, missing numactl-devel?" "$LINENO" 5
 fi
 
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for fgrep" >&5
+$as_echo_n "checking for fgrep... " >&6; }
+if ${ac_cv_path_FGREP+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  if echo 'ab*c' | $GREP -F 'ab*c' >/dev/null 2>&1
+   then ac_cv_path_FGREP="$GREP -F"
+   else
+     if test -z "$FGREP"; then
+  ac_path_FGREP_found=false
+  # Loop through the user's path and test for each of PROGNAME-LIST
+  as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH$PATH_SEPARATOR/usr/xpg4/bin
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+    for ac_prog in fgrep; do
+    for ac_exec_ext in '' $ac_executable_extensions; do
+      ac_path_FGREP="$as_dir/$ac_prog$ac_exec_ext"
+      as_fn_executable_p "$ac_path_FGREP" || continue
+# Check for GNU ac_path_FGREP and select it if it is found.
+  # Check for GNU $ac_path_FGREP
+case `"$ac_path_FGREP" --version 2>&1` in
+*GNU*)
+  ac_cv_path_FGREP="$ac_path_FGREP" ac_path_FGREP_found=:;;
+*)
+  ac_count=0
+  $as_echo_n 0123456789 >"conftest.in"
+  while :
+  do
+    cat "conftest.in" "conftest.in" >"conftest.tmp"
+    mv "conftest.tmp" "conftest.in"
+    cp "conftest.in" "conftest.nl"
+    $as_echo 'FGREP' >> "conftest.nl"
+    "$ac_path_FGREP" FGREP < "conftest.nl" >"conftest.out" 2>/dev/null || break
+    diff "conftest.out" "conftest.nl" >/dev/null 2>&1 || break
+    as_fn_arith $ac_count + 1 && ac_count=$as_val
+    if test $ac_count -gt ${ac_path_FGREP_max-0}; then
+      # Best one so far, save it but keep looking for a better one
+      ac_cv_path_FGREP="$ac_path_FGREP"
+      ac_path_FGREP_max=$ac_count
+    fi
+    # 10*(2^10) chars as input seems more than enough
+    test $ac_count -gt 10 && break
+  done
+  rm -f conftest.in conftest.tmp conftest.nl conftest.out;;
+esac
+
+      $ac_path_FGREP_found && break 3
+    done
+  done
+  done
+IFS=$as_save_IFS
+  if test -z "$ac_cv_path_FGREP"; then
+    as_fn_error $? "no acceptable fgrep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" "$LINENO" 5
+  fi
+else
+  ac_cv_path_FGREP=$FGREP
+fi
+
+   fi
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_FGREP" >&5
+$as_echo "$ac_cv_path_FGREP" >&6; }
+ FGREP="$ac_cv_path_FGREP"
+
+
+
+
+# Check whether --with-mpi was given.
+if test "${with_mpi+set}" = set; then :
+  withval=$with_mpi; case "$withval" in #(
+  yes|no|'') :
+    { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: --without-mpi=PATH expects a valid PATH" >&5
+$as_echo "$as_me: WARNING: --without-mpi=PATH expects a valid PATH" >&2;}
+                          with_mpi="" ;; #(
+  *) :
+     ;;
+esac
+else
+  with_mpi=
+fi
+
+
+# Check whether --with-mpi-include was given.
+if test "${with_mpi_include+set}" = set; then :
+  withval=$with_mpi_include; case "$withval" in #(
+  yes|no|'') :
+    { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: --without-mpi-include=PATH expects a valid PATH" >&5
+$as_echo "$as_me: WARNING: --without-mpi-include=PATH expects a valid PATH" >&2;}
+                          with_mpi_include="" ;; #(
+  *) :
+     ;;
+esac
+fi
+
+
+# Check whether --with-mpi-lib was given.
+if test "${with_mpi_lib+set}" = set; then :
+  withval=$with_mpi_lib; case "$withval" in #(
+  yes|no|'') :
+    { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: --without-mpi-lib=PATH expects a valid PATH" >&5
+$as_echo "$as_me: WARNING: --without-mpi-lib=PATH expects a valid PATH" >&2;}
+                          with_mpi_lib="" ;; #(
+  *) :
+     ;;
+esac
+fi
+
+
+    # The args have been sanitized into empty/non-empty values above.
+    # Now append -I/-L args to CPPFLAGS/LDFLAGS, with more specific options
+    # taking priority
+
+    if test -n "${with_mpi_include}"; then :
+
+
+	if echo "$CPPFLAGS" | $FGREP -e "\<-I${with_mpi_include}\>" >/dev/null 2>&1; then :
+  echo "CPPFLAGS(='$CPPFLAGS') contains '-I${with_mpi_include}', not appending" >&5
+else
+  echo "CPPFLAGS(='$CPPFLAGS') does not contain '-I${with_mpi_include}', appending" >&5
+		CPPFLAGS="$CPPFLAGS -I${with_mpi_include}"
+
+fi
+
+else
+  if test -n "${with_mpi}"; then :
+
+
+	if echo "$CPPFLAGS" | $FGREP -e "\<-I${with_mpi}/include\>" >/dev/null 2>&1; then :
+  echo "CPPFLAGS(='$CPPFLAGS') contains '-I${with_mpi}/include', not appending" >&5
+else
+  echo "CPPFLAGS(='$CPPFLAGS') does not contain '-I${with_mpi}/include', appending" >&5
+		CPPFLAGS="$CPPFLAGS -I${with_mpi}/include"
+
+fi
+
+fi
+fi
+
+    if test -n "${with_mpi_lib}"; then :
+
+
+	if echo "$LDFLAGS" | $FGREP -e "\<-L${with_mpi_lib}\>" >/dev/null 2>&1; then :
+  echo "LDFLAGS(='$LDFLAGS') contains '-L${with_mpi_lib}', not appending" >&5
+else
+  echo "LDFLAGS(='$LDFLAGS') does not contain '-L${with_mpi_lib}', appending" >&5
+		LDFLAGS="$LDFLAGS -L${with_mpi_lib}"
+
+fi
+
+else
+  if test -n "${with_mpi}"; then :
+
+
+	if echo "$LDFLAGS" | $FGREP -e "\<-L${with_mpi}/lib\>" >/dev/null 2>&1; then :
+  echo "LDFLAGS(='$LDFLAGS') contains '-L${with_mpi}/lib', not appending" >&5
+else
+  echo "LDFLAGS(='$LDFLAGS') does not contain '-L${with_mpi}/lib', appending" >&5
+		LDFLAGS="$LDFLAGS -L${with_mpi}/lib"
+
+fi
+
+                  if test -d "${with_mpi}/lib64"; then :
+
+
+	if echo "$LDFLAGS" | $FGREP -e "\<-L${with_mpi}/lib64\>" >/dev/null 2>&1; then :
+  echo "LDFLAGS(='$LDFLAGS') contains '-L${with_mpi}/lib64', not appending" >&5
+else
+  echo "LDFLAGS(='$LDFLAGS') does not contain '-L${with_mpi}/lib64', appending" >&5
+		LDFLAGS="$LDFLAGS -L${with_mpi}/lib64"
+
+fi
+
+fi
+
+fi
+
+fi
+
+
 
 # Check whether --with-kernelsrc was given.
 if test "${with_kernelsrc+set}" = set; then :
@@ -3369,6 +3568,14 @@ else
 fi
 
 
+# Check whether --enable-qlmpi was given.
+if test "${enable_qlmpi+set}" = set; then :
+  enableval=$enable_qlmpi; ENABLE_QLMPI=$enableval
+else
+  ENABLE_QLMPI=no
+fi
+
+
 
 # Check whether --with-uname_r was given.
 if test "${with_uname_r+set}" = set; then :
@@ -3386,7 +3593,7 @@ esac
 
 case "X$WITH_KERNELSRC" in
     Xyes | Xno | X)
-	WITH_KERNELSRC='/lib/modules/`uname -r`/build'
+	WITH_KERNELSRC="/lib/modules/`uname -r`/build"
 	;;
     X/*)
 	;;
@@ -3407,8 +3614,21 @@ _ACEOF
 
 
 case $WITH_TARGET in
-    attached-mic|builtin-x86|smp-x86)
-	ARCH=`uname -m`
+# POSTK_DEBUG_ARCH_DEP_30, Expansion of the branch.
+#    attached-mic|builtin-x86|smp-x86)
+#	ARCH=`uname -m`
+#	AC_PROG_CC
+#	XCC=$CC
+#	;;
+    attached-mic|builtin-x86|smp-*)
+	case $WITH_TARGET in
+	    attached-mic|builtin-x86|smp-x86)
+		ARCH=`uname -m`
+		;;
+	    smp-arm64)
+		ARCH=arm64
+		;;
+	esac
 	ac_ext=c
 ac_cpp='$CPP $CPPFLAGS'
 ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
@@ -3991,6 +4211,48 @@ $as_echo "no" >&6; }
 fi
 
 
+	CC=$XCC
+	;;
+    smp-arm64)
+	ARCH=arm64
+	# Extract the first word of "${CROSS_COMPILE}gcc", so it can be a program name with args.
+set dummy ${CROSS_COMPILE}gcc; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_prog_XCC+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  if test -n "$XCC"; then
+  ac_cv_prog_XCC="$XCC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+    for ac_exec_ext in '' $ac_executable_extensions; do
+  if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_XCC="${CROSS_COMPILE}gcc"
+    $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+  done
+IFS=$as_save_IFS
+
+  test -z "$ac_cv_prog_XCC" && ac_cv_prog_XCC="no"
+fi
+fi
+XCC=$ac_cv_prog_XCC
+if test -n "$XCC"; then
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: $XCC" >&5
+$as_echo "$XCC" >&6; }
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
 	CC=$XCC
 	;;
     *)
@@ -4060,9 +4322,15 @@ case $WITH_TARGET in
 	if test "X$SBINDIR" = X; then
 		SBINDIR="$prefix/sbin"
 	fi
+	if test "X$MCKERNEL_INCDIR" = X; then
+		MCKERNEL_INCDIR="$prefix/include"
+	fi
         if test "X$MCKERNEL_LIBDIR" = X; then
                 MCKERNEL_LIBDIR="$prefix/lib"
         fi
+	if test "X$INCDIR" = X; then
+		INCDIR="$prefix/include"
+	fi
 	if test "X$ETCDIR" = X; then
 		ETCDIR="$prefix/etc"
 	fi
@@ -4076,6 +4344,35 @@ case $WITH_TARGET in
 		MANDIR="$prefix/smp-x86/man"
 	fi
 	;;
+    smp-arm64)
+	if test "X$KERNDIR" = X; then
+		KERNDIR="$prefix/smp-arm64/kernel"
+	fi
+	if test "X$BINDIR" = X; then
+		BINDIR="$prefix/bin"
+	fi
+	if test "X$SBINDIR" = X; then
+		SBINDIR="$prefix/sbin"
+	fi
+	if test "X$MCKERNEL_INCDIR" = X; then
+		MCKERNEL_INCDIR="$prefix/include"
+	fi
+        if test "X$MCKERNEL_LIBDIR" = X; then
+                MCKERNEL_LIBDIR="$prefix/lib"
+        fi
+	if test "X$INCDIR" = X; then
+		INCDIR="$prefix/include"
+	fi
+	if test "X$ETCDIR" = X; then
+		ETCDIR="$prefix/etc"
+	fi
+	if test "X$KMODDIR" = X; then
+		KMODDIR="$prefix/kmod"
+	fi
+	if test "X$MANDIR" = X; then
+		MANDIR="$prefix/smp-arm64/man"
+	fi
+	;;
     *)
 	as_fn_error $? "target $WITH_TARGET is unknwon" "$LINENO" 5
 	;;
@@ -4097,11 +4394,11 @@ esac
 
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for System.map" >&5
 $as_echo_n "checking for System.map... " >&6; }
-if test -f "$MCCTRL_LINUX_SYMTAB"; then
+if test -r "$MCCTRL_LINUX_SYMTAB"; then
 	MCCTRL_LINUX_SYMTAB="$MCCTRL_LINUX_SYMTAB"
-elif test -f "/boot/System.map-`uname -r`"; then
+elif test -r "/boot/System.map-`uname -r`"; then
 	MCCTRL_LINUX_SYMTAB="/boot/System.map-`uname -r`"
-elif test -f "$KDIR/System.map"; then
+elif test -r "$KDIR/System.map"; then
 	MCCTRL_LINUX_SYMTAB="$KDIR/System.map"
 fi
 
@@ -4376,6 +4673,32 @@ _ACEOF
 
   fi
 
+# POSTK_DEBUG_ARCH_DEP_50, add:find kernel symbol.
+
+  { $as_echo "$as_me:${as_lineno-$LINENO}: checking System.map for symbol vdso_spec" >&5
+$as_echo_n "checking System.map for symbol vdso_spec... " >&6; }
+  mcctrl_addr=`eval $MCCTRL_LINUX_SYMTAB_CMD | grep " vdso_spec\$" | cut -d\  -f1`
+  if test -z $mcctrl_addr; then
+    { $as_echo "$as_me:${as_lineno-$LINENO}: result: not found" >&5
+$as_echo "not found" >&6; }
+  else
+    mcctrl_result=$mcctrl_addr
+    mcctrl_addr="0x$mcctrl_addr"
+
+      if `eval $MCCTRL_LINUX_SYMTAB_CMD | grep " __ksymtab_vdso_spec\$" >/dev/null`; then
+        mcctrl_result="exported"
+        mcctrl_addr="0"
+      fi
+
+    { $as_echo "$as_me:${as_lineno-$LINENO}: result: $mcctrl_result" >&5
+$as_echo "$mcctrl_result" >&6; }
+
+cat >>confdefs.h <<_ACEOF
+#define MCCTRL_KSYM_vdso_spec $mcctrl_addr
+_ACEOF
+
+  fi
+
 
   { $as_echo "$as_me:${as_lineno-$LINENO}: checking System.map for symbol hv_clock" >&5
 $as_echo_n "checking System.map for symbol hv_clock... " >&6; }
@@ -4427,11 +4750,38 @@ _ACEOF
   fi
 
 
+  { $as_echo "$as_me:${as_lineno-$LINENO}: checking System.map for symbol walk_page_range" >&5
+$as_echo_n "checking System.map for symbol walk_page_range... " >&6; }
+  mcctrl_addr=`eval $MCCTRL_LINUX_SYMTAB_CMD | grep " walk_page_range\$" | cut -d\  -f1`
+  if test -z $mcctrl_addr; then
+    { $as_echo "$as_me:${as_lineno-$LINENO}: result: not found" >&5
+$as_echo "not found" >&6; }
+  else
+    mcctrl_result=$mcctrl_addr
+    mcctrl_addr="0x$mcctrl_addr"
+
+      if `eval $MCCTRL_LINUX_SYMTAB_CMD | grep " __ksymtab_walk_page_range\$" >/dev/null`; then
+        mcctrl_result="exported"
+        mcctrl_addr="0"
+      fi
+
+    { $as_echo "$as_me:${as_lineno-$LINENO}: result: $mcctrl_result" >&5
+$as_echo "$mcctrl_result" >&6; }
+
+cat >>confdefs.h <<_ACEOF
+#define MCCTRL_KSYM_walk_page_range $mcctrl_addr
+_ACEOF
+
+  fi
+
+
 case $ENABLE_MEMDUMP in
 	yes|no|auto)
 		;;
 	default)
-		if test "x$WITH_TARGET" = "xsmp-x86" ; then
+# POSTK_DEBUG_ARCH_DEP_30, Expansion of the branch.
+#		if test "x$WITH_TARGET" = "xsmp-x86" ; then
+		if test "x$WITH_TARGET" = "xsmp-x86" -o "x$WITH_TARGET" = "xsmp-arm64" ; then
 			ENABLE_MEMDUMP=auto
 		else
 			ENABLE_MEMDUMP=no
@@ -4444,6 +4794,54 @@ esac
 
 if test "x$ENABLE_MEMDUMP" != "xno" ; then
 	enableval=yes
+# POSTK_DEBUG_ARCH_DEP_32, AC_CHECK_LIB for libiberty
+	{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for hex_init in -liberty" >&5
+$as_echo_n "checking for hex_init in -liberty... " >&6; }
+if ${ac_cv_lib_iberty_hex_init+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-liberty  $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+char hex_init ();
+int
+main ()
+{
+return hex_init ();
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+  ac_cv_lib_iberty_hex_init=yes
+else
+  ac_cv_lib_iberty_hex_init=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_iberty_hex_init" >&5
+$as_echo "$ac_cv_lib_iberty_hex_init" >&6; }
+if test "x$ac_cv_lib_iberty_hex_init" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBIBERTY 1
+_ACEOF
+
+  LIBS="-liberty $LIBS"
+
+else
+  enableval=no
+fi
+
 	{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for bfd_init in -lbfd" >&5
 $as_echo_n "checking for bfd_init in -lbfd... " >&6; }
 if ${ac_cv_lib_bfd_bfd_init+:} false; then :
@@ -4530,6 +4928,17 @@ else
 $as_echo "$as_me: mcoverlayfs is disabled" >&6;}
 fi
 
+if test "x$ENABLE_QLMPI" = "xyes" ; then
+
+$as_echo "#define ENABLE_QLMPI 1" >>confdefs.h
+
+	{ $as_echo "$as_me:${as_lineno-$LINENO}: qlmpi is enabled" >&5
+$as_echo "$as_me: qlmpi is enabled" >&6;}
+else
+	{ $as_echo "$as_me:${as_lineno-$LINENO}: qlmpi is disabled" >&5
+$as_echo "$as_me: qlmpi is disabled" >&6;}
+fi
+
 case $ENABLE_RUSAGE in
 	yes|no)
 		;;
@@ -4552,6 +4961,14 @@ else
 $as_echo "$as_me: rusage is disabled" >&6;}
 fi
 
+if test "x$MCKERNEL_INCDIR" != "x" ; then
+
+cat >>confdefs.h <<_ACEOF
+#define MCKERNEL_INCDIR "$MCKERNEL_INCDIR"
+_ACEOF
+
+fi
+
 if test "x$MCKERNEL_LIBDIR" != "x" ; then
 
 cat >>confdefs.h <<_ACEOF
@@ -4561,6 +4978,20 @@ _ACEOF
 fi
 
 
+cat >>confdefs.h <<_ACEOF
+#define BINDIR "$BINDIR"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define SBINDIR "$SBINDIR"
+_ACEOF
+
+
+
+
+
+
 
 
 
@@ -4588,9 +5019,16 @@ fi
 
 ac_config_headers="$ac_config_headers config.h"
 
-ac_config_files="$ac_config_files Makefile executer/user/Makefile executer/user/arch/x86_64/Makefile executer/kernel/mcctrl/Makefile executer/kernel/mcctrl/arch/x86_64/Makefile executer/kernel/mcoverlayfs/Makefile executer/kernel/mcoverlayfs/linux-3.10.0-327.36.1.el7/Makefile executer/kernel/mcoverlayfs/linux-4.0.9/Makefile executer/kernel/mcoverlayfs/linux-4.6.7/Makefile kernel/Makefile kernel/Makefile.build arch/x86/tools/mcreboot-attached-mic.sh arch/x86/tools/mcshutdown-attached-mic.sh arch/x86/tools/mcreboot-builtin-x86.sh arch/x86/tools/mcreboot-smp-x86.sh arch/x86/tools/mcstop+release-smp-x86.sh arch/x86/tools/mcoverlay-destroy-smp-x86.sh arch/x86/tools/mcoverlay-create-smp-x86.sh arch/x86/tools/eclair-dump-backtrace.exp arch/x86/tools/mcshutdown-builtin-x86.sh arch/x86/tools/mcreboot.1:arch/x86/tools/mcreboot.1in arch/x86/tools/irqbalance_mck.service arch/x86/tools/irqbalance_mck.in"
+# POSTK_DEBUG_ARCH_DEP_37
+# AC_CONFIG_FILES arch dependfiles separate
+ac_config_files="$ac_config_files Makefile executer/user/Makefile executer/user/arch/$ARCH/Makefile executer/user/arch/x86_64/Makefile executer/user/vmcore2mckdump executer/kernel/mcctrl/Makefile executer/kernel/mcctrl/arch/$ARCH/Makefile executer/kernel/mcoverlayfs/Makefile executer/kernel/mcoverlayfs/linux-3.10.0-327.36.1.el7/Makefile executer/kernel/mcoverlayfs/linux-4.0.9/Makefile executer/kernel/mcoverlayfs/linux-4.6.7/Makefile executer/include/qlmpilib.h kernel/Makefile kernel/Makefile.build kernel/include/swapfmt.h arch/x86/tools/mcreboot-attached-mic.sh arch/x86/tools/mcshutdown-attached-mic.sh arch/x86/tools/mcreboot-builtin-x86.sh arch/x86/tools/mcreboot-smp-x86.sh arch/x86/tools/mcstop+release-smp-x86.sh arch/x86/tools/mcoverlay-destroy-smp-x86.sh arch/x86/tools/mcoverlay-create-smp-x86.sh arch/x86/tools/eclair-dump-backtrace.exp arch/x86/tools/mcshutdown-builtin-x86.sh arch/x86/tools/mcreboot.1:arch/x86/tools/mcreboot.1in arch/x86/tools/irqbalance_mck.service arch/x86/tools/irqbalance_mck.in"
 
 
+if test "$TARGET" = "smp-arm64"; then
+ac_config_files="$ac_config_files kernel/config/config.smp-arm64 arch/arm64/kernel/vdso/Makefile arch/arm64/kernel/Makefile.arch"
+
+fi
+
 if test "x$enable_dcfa" = xyes; then :
 
         ac_config_files="$ac_config_files kernel/Makefile.dcfa"
@@ -5289,15 +5727,19 @@ do
     "config.h") CONFIG_HEADERS="$CONFIG_HEADERS config.h" ;;
     "Makefile") CONFIG_FILES="$CONFIG_FILES Makefile" ;;
     "executer/user/Makefile") CONFIG_FILES="$CONFIG_FILES executer/user/Makefile" ;;
+    "executer/user/arch/$ARCH/Makefile") CONFIG_FILES="$CONFIG_FILES executer/user/arch/$ARCH/Makefile" ;;
     "executer/user/arch/x86_64/Makefile") CONFIG_FILES="$CONFIG_FILES executer/user/arch/x86_64/Makefile" ;;
+    "executer/user/vmcore2mckdump") CONFIG_FILES="$CONFIG_FILES executer/user/vmcore2mckdump" ;;
     "executer/kernel/mcctrl/Makefile") CONFIG_FILES="$CONFIG_FILES executer/kernel/mcctrl/Makefile" ;;
-    "executer/kernel/mcctrl/arch/x86_64/Makefile") CONFIG_FILES="$CONFIG_FILES executer/kernel/mcctrl/arch/x86_64/Makefile" ;;
+    "executer/kernel/mcctrl/arch/$ARCH/Makefile") CONFIG_FILES="$CONFIG_FILES executer/kernel/mcctrl/arch/$ARCH/Makefile" ;;
     "executer/kernel/mcoverlayfs/Makefile") CONFIG_FILES="$CONFIG_FILES executer/kernel/mcoverlayfs/Makefile" ;;
     "executer/kernel/mcoverlayfs/linux-3.10.0-327.36.1.el7/Makefile") CONFIG_FILES="$CONFIG_FILES executer/kernel/mcoverlayfs/linux-3.10.0-327.36.1.el7/Makefile" ;;
     "executer/kernel/mcoverlayfs/linux-4.0.9/Makefile") CONFIG_FILES="$CONFIG_FILES executer/kernel/mcoverlayfs/linux-4.0.9/Makefile" ;;
     "executer/kernel/mcoverlayfs/linux-4.6.7/Makefile") CONFIG_FILES="$CONFIG_FILES executer/kernel/mcoverlayfs/linux-4.6.7/Makefile" ;;
+    "executer/include/qlmpilib.h") CONFIG_FILES="$CONFIG_FILES executer/include/qlmpilib.h" ;;
     "kernel/Makefile") CONFIG_FILES="$CONFIG_FILES kernel/Makefile" ;;
     "kernel/Makefile.build") CONFIG_FILES="$CONFIG_FILES kernel/Makefile.build" ;;
+    "kernel/include/swapfmt.h") CONFIG_FILES="$CONFIG_FILES kernel/include/swapfmt.h" ;;
     "arch/x86/tools/mcreboot-attached-mic.sh") CONFIG_FILES="$CONFIG_FILES arch/x86/tools/mcreboot-attached-mic.sh" ;;
     "arch/x86/tools/mcshutdown-attached-mic.sh") CONFIG_FILES="$CONFIG_FILES arch/x86/tools/mcshutdown-attached-mic.sh" ;;
     "arch/x86/tools/mcreboot-builtin-x86.sh") CONFIG_FILES="$CONFIG_FILES arch/x86/tools/mcreboot-builtin-x86.sh" ;;
@@ -5310,6 +5752,9 @@ do
     "arch/x86/tools/mcreboot.1") CONFIG_FILES="$CONFIG_FILES arch/x86/tools/mcreboot.1:arch/x86/tools/mcreboot.1in" ;;
     "arch/x86/tools/irqbalance_mck.service") CONFIG_FILES="$CONFIG_FILES arch/x86/tools/irqbalance_mck.service" ;;
     "arch/x86/tools/irqbalance_mck.in") CONFIG_FILES="$CONFIG_FILES arch/x86/tools/irqbalance_mck.in" ;;
+    "kernel/config/config.smp-arm64") CONFIG_FILES="$CONFIG_FILES kernel/config/config.smp-arm64" ;;
+    "arch/arm64/kernel/vdso/Makefile") CONFIG_FILES="$CONFIG_FILES arch/arm64/kernel/vdso/Makefile" ;;
+    "arch/arm64/kernel/Makefile.arch") CONFIG_FILES="$CONFIG_FILES arch/arm64/kernel/Makefile.arch" ;;
     "kernel/Makefile.dcfa") CONFIG_FILES="$CONFIG_FILES kernel/Makefile.dcfa" ;;
 
   *) as_fn_error $? "invalid argument: \`$ac_config_target'" "$LINENO" 5;;
diff --git a/executer/kernel/mcctrl/control.c b/executer/kernel/mcctrl/control.c
index 44803be7..8546a3b0 100644
--- a/executer/kernel/mcctrl/control.c
+++ b/executer/kernel/mcctrl/control.c
@@ -42,11 +42,11 @@
 #include <linux/kallsyms.h>
 #include <linux/syscalls.h>
 #include <trace/events/sched.h>
-#include "../../../config.h"
+#include <config.h>
 #include "mcctrl.h"
 #include <ihk/ihk_host_user.h>
-#include <rusage.h>
 #include <ihklib_rusage.h>
+#include <rusage.h>
 
 //#define DEBUG
 
@@ -2197,9 +2197,9 @@ void mcctrl_perf_ack(ihk_os_t os, struct ikc_scd_packet *packet)
 }
 
 /* Compose LWK-specific rusage structure */
-long mcctrl_getrusage(ihk_os_t ihk_os, struct getrusage_desc *__user _desc)
+long mcctrl_getrusage(ihk_os_t ihk_os, struct mcctrl_ioctl_getrusage_desc *__user _desc)
 {
-	struct getrusage_desc desc;
+	struct mcctrl_ioctl_getrusage_desc desc;
 	struct rusage_global *rusage_global = ihk_os_get_rusage(ihk_os);
 	struct mckernel_rusage *rusage = NULL;
 	int ret = 0;
@@ -2207,7 +2207,7 @@ long mcctrl_getrusage(ihk_os_t ihk_os, struct getrusage_desc *__user _desc)
 	unsigned long ut;
 	unsigned long st;
 
-	ret = copy_from_user(&desc, _desc, sizeof(struct getrusage_desc));
+	ret = copy_from_user(&desc, _desc, sizeof(struct mcctrl_ioctl_getrusage_desc));
 	if (ret != 0) {
 		printk("%s: copy_from_user failed\n", __FUNCTION__);
 		goto out;
@@ -2872,7 +2872,7 @@ long __mcctrl_control(ihk_os_t os, unsigned int req, unsigned long arg,
 		return mcctrl_perf_destroy(os);
 
 	case IHK_OS_GETRUSAGE:
-		return mcctrl_getrusage(os, (struct getrusage_desc *)arg);
+		return mcctrl_getrusage(os, (struct mcctrl_ioctl_getrusage_desc *)arg);
 	}
 	return -EINVAL;
 }
diff --git a/executer/kernel/mcctrl/ikc.c b/executer/kernel/mcctrl/ikc.c
index 689d51da..5e750112 100644
--- a/executer/kernel/mcctrl/ikc.c
+++ b/executer/kernel/mcctrl/ikc.c
@@ -122,6 +122,7 @@ static int syscall_packet_handler(struct ihk_ikc_channel_desc *c,
 		break;
 
 	case SCD_MSG_EVENTFD:
+		dkprintf(KERN_EMERG "%s: SCD_MSG_EVENTFD,pisp->eventfd_type=%d\n", __FUNCTION__, pisp->eventfd_type);
 		mcctrl_eventfd(__os, pisp);
 		break;
 
@@ -403,5 +404,5 @@ void destroy_ikc_channels(ihk_os_t os)
 void
 mcctrl_eventfd(ihk_os_t os, struct ikc_scd_packet *pisp)
 {
-	ihk_os_eventfd(os, 0);
+	ihk_os_eventfd(os, pisp->eventfd_type);
 }
diff --git a/executer/kernel/mcctrl/mcctrl.h b/executer/kernel/mcctrl/mcctrl.h
index d38ebda3..1ef809f2 100644
--- a/executer/kernel/mcctrl/mcctrl.h
+++ b/executer/kernel/mcctrl/mcctrl.h
@@ -157,6 +157,11 @@ struct ikc_scd_packet {
 			enum mcctrl_os_cpu_operation op;
 			void *resp;
 		};
+
+		/* SCD_MSG_EVENTFD */
+		struct {
+			int eventfd_type;
+		};
 	};
 	char padding[12];
 };
@@ -490,7 +495,7 @@ struct ihk_perf_event_attr{
 	unsigned exclude_idle:1;
 };
 
-struct getrusage_desc {
+struct mcctrl_ioctl_getrusage_desc {
 	void* rusage;
 	size_t size_rusage;
 };
diff --git a/kernel/include/rusage.h b/kernel/include/rusage.h
index a20a13ea..97d9573e 100644
--- a/kernel/include/rusage.h
+++ b/kernel/include/rusage.h
@@ -3,6 +3,8 @@
 #ifndef __RUSAGE_H
 #define __RUSAGE_H
 
+//#define RUSAGE_DEBUG
+
 #define IHK_MAX_NUM_PGSIZES 4
 #define IHK_MAX_NUM_NUMA_NODES 1024
 #define IHK_MAX_NUM_CPUS 1024
@@ -13,21 +15,28 @@ struct rusage_percpu {
 };
 
 struct rusage_global {
+	/* Memory usage accounting */
 	long memory_stat_rss[IHK_MAX_NUM_PGSIZES];
 	long memory_stat_mapped_file[IHK_MAX_NUM_PGSIZES];
+	long rss_current; /* anon && user, used only for memory_max_usage */
 	unsigned long memory_max_usage;
 	unsigned long max_num_threads;
 	unsigned long num_threads;
-	long rss_current;
 	unsigned long memory_kmem_usage;
 	unsigned long memory_kmem_max_usage;
 	unsigned long memory_numa_stat[IHK_MAX_NUM_NUMA_NODES];
+
+	/* CPU usage accounting */
 	struct rusage_percpu cpu[IHK_MAX_NUM_CPUS]; /* clv[i].monitor = &cpu[i] */
 
+	/* OOM monitoring */
 	unsigned long total_memory;
 	unsigned long total_memory_usage;
 	unsigned long total_memory_max_usage;
-
+#ifdef RUSAGE_DEBUG
+	unsigned long total_memory_max_usage_old; /* debug */
+#endif
+	/* Used for translating results into struct mckernel_rusage */
 	unsigned long num_numa_nodes;
 	unsigned long num_processors;
 	unsigned long ns_per_tsc;
diff --git a/kernel/include/rusage_private.h b/kernel/include/rusage_private.h
index 12607779..4648515a 100644
--- a/kernel/include/rusage_private.h
+++ b/kernel/include/rusage_private.h
@@ -8,18 +8,25 @@
 #include <ihk/atomic.h>
 #include <memobj.h>
 #include <rusage.h>
-#include <arch/rusage.h>
+#include <ihk/ihk_monitor.h>
+#include <arch_rusage.h>
 
 #ifdef ENABLE_RUSAGE
 
-#define RUSAGE_MEM_LIMIT (2 * 1024 * 1024) // 2MB
+#define RUSAGE_OOM_MARGIN (2 * 1024 * 1024) // 2MB
 
-extern void eventfd();
+extern void eventfd(int type);
 
 static inline void
 rusage_total_memory_add(unsigned long size)
 {
+#ifdef RUSAGE_DEBUG
+	kprintf("%s: total_memory=%ld,size=%ld\n", __FUNCTION__, rusage->total_memory, size);
+#endif
 	rusage->total_memory += size;
+#ifdef RUSAGE_DEBUG
+	kprintf("%s: total_memory=%ld\n", __FUNCTION__, rusage->total_memory);
+#endif
 }
 
 static inline void
@@ -220,6 +227,22 @@ rusage_numa_sub(int numa_id, unsigned long size)
 	__sync_sub_and_fetch(rusage->memory_numa_stat + numa_id, size);
 }
 
+static inline int
+rusage_check_oom(int numa_id, unsigned long pages, int is_user)
+{
+	unsigned long size = pages * PAGE_SIZE;
+
+	if (rusage->total_memory_usage + size > rusage->total_memory - RUSAGE_OOM_MARGIN) {
+		kprintf("%s: memory used:%ld available:%ld\n", __FUNCTION__, rusage->total_memory_usage, rusage->total_memory);
+		eventfd(IHK_OS_EVENTFD_TYPE_OOM);
+		if (is_user) {
+			return -ENOMEM;
+		}
+	}
+
+	return 0;
+}
+
 static inline void
 rusage_page_add(int numa_id, unsigned long pages, int is_user)
 {
@@ -228,6 +251,12 @@ rusage_page_add(int numa_id, unsigned long pages, int is_user)
 	unsigned long oldval;
 	unsigned long retval;
 
+#ifdef RUSAGE_DEBUG
+	if (numa_id < 0 || numa_id >= rusage->num_numa_nodes) {
+		kprintf("%s: Error: invalid numa_id=%d\n", __FUNCTION__, numa_id);
+		return;
+	}
+#endif	
 	if (is_user)
 		rusage_numa_add(numa_id, size);
 	else
@@ -239,10 +268,12 @@ rusage_page_add(int numa_id, unsigned long pages, int is_user)
 		retval = __sync_val_compare_and_swap(&rusage->total_memory_max_usage,
 		                                     oldval, newval);
 		if (retval == oldval) {
-			if (rusage->total_memory - newval <
-			    RUSAGE_MEM_LIMIT) {
-				eventfd();
+#ifdef RUSAGE_DEBUG
+			if (rusage->total_memory_max_usage > rusage->total_memory_max_usage_old + (1 * (1ULL << 30))) {
+				kprintf("%s: max(%ld) > old + 1GB,numa_id=%d\n", __FUNCTION__, rusage->total_memory_max_usage, numa_id);
+				rusage->total_memory_max_usage_old = rusage->total_memory_max_usage;
 			}
+#endif
 			break;
 		}
 		oldval = retval;
@@ -253,7 +284,15 @@ static inline void
 rusage_page_sub(int numa_id, unsigned long pages, int is_user)
 {
 	unsigned long size = pages * PAGE_SIZE;
-
+#ifdef RUSAGE_DEBUG
+	if (numa_id < 0 || numa_id >= rusage->num_numa_nodes) {
+		kprintf("%s: Error: invalid numa_id=%d\n", __FUNCTION__, numa_id);
+		return;
+	}
+	if (rusage->total_memory_usage < size) {
+		kprintf("%s: Error, total_memory_usage=%ld,size=%ld\n", __FUNCTION__, rusage->total_memory_max_usage, size);
+	}
+#endif
 	__sync_sub_and_fetch(&rusage->total_memory_usage, size);
 
 	if (is_user)
@@ -343,9 +382,15 @@ rusage_numa_sub(int numa_id, unsigned long size)
 {
 }
 
+static inline int
+rusage_check_oom(int numa_id, unsigned long pages, int is_user)
+{
+	return 0;
+}
 static inline void
 rusage_page_add(int numa_id, unsigned long size, int is_user)
 {
+	return;
 }
 
 static inline void
diff --git a/kernel/include/syscall.h b/kernel/include/syscall.h
index 491caecd..89e73d51 100644
--- a/kernel/include/syscall.h
+++ b/kernel/include/syscall.h
@@ -268,6 +268,11 @@ struct ikc_scd_packet {
 			enum mcctrl_os_cpu_operation op;
 			void *resp;
 		};
+
+		/* SCD_MSG_EVENTFD */
+		struct {
+			int eventfd_type;
+		};
 	};
 	char padding[12];
 };
diff --git a/kernel/init.c b/kernel/init.c
index e43fc201..552c6447 100644
--- a/kernel/init.c
+++ b/kernel/init.c
@@ -31,7 +31,6 @@
 #include <cls.h>
 #include <syscall.h>
 #include <sysfs.h>
-#include <rusage_private.h>
 #include <ihk/monitor.h>
 
 //#define IOCTL_FUNC_EXTENSION
@@ -287,21 +286,6 @@ static void monitor_init()
 #endif /* POSTK_DEBUG_TEMP_FIX_73 */
 }
 
-static void rusage_init()
-{
-	int npages;
-	unsigned long phys;
-
-	npages = (sizeof(struct rusage_global) + PAGE_SIZE -1) >> PAGE_SHIFT;
-	rusage = ihk_mc_alloc_pages(npages, IHK_MC_AP_CRITICAL);
-	memset(rusage, 0, npages * PAGE_SIZE);
-	rusage->num_processors = num_processors;
-	rusage->num_numa_nodes = ihk_mc_get_nr_numa_nodes();
-	rusage->ns_per_tsc = ihk_mc_get_ns_per_tsc();
-	phys = virt_to_phys(rusage);
-	ihk_set_rusage(phys, sizeof(struct rusage_global));
-}
-
 int nmi_mode;
 
 static void nmi_init()
@@ -326,7 +310,6 @@ static void rest_init(void)
 #ifndef POSTK_DEBUG_TEMP_FIX_73 /* NULL access for *monitor fix */
 	monitor_init();
 #endif /* !POSTK_DEBUG_TEMP_FIX_73 */
-	rusage_init();
 	cpu_local_var_init();
 	nmi_init();
 	time_init();
diff --git a/kernel/mem.c b/kernel/mem.c
index e9e86406..1b9568e3 100644
--- a/kernel/mem.c
+++ b/kernel/mem.c
@@ -80,6 +80,8 @@ static void *___ihk_mc_alloc_aligned_pages_node(int npages,
 static void *___ihk_mc_alloc_pages(int npages, ihk_mc_ap_flag flag, int is_user);
 static void ___ihk_mc_free_pages(void *p, int npages, int is_user);
 
+extern unsigned long ihk_mc_get_ns_per_tsc(void);
+
 /*
  * Page allocator tracking routines
  */
@@ -571,14 +573,22 @@ static void *mckernel_allocate_aligned_pages_node(int npages, int p2align,
 	if (pref_node > -1 && pref_node < ihk_mc_get_nr_numa_nodes()) {
 #ifdef IHK_RBTREE_ALLOCATOR
 		{
-			pa = ihk_numa_alloc_pages(&memory_nodes[pref_node], npages, p2align);
+			if (rusage_check_oom(pref_node, npages, is_user) == -ENOMEM) {
+				pa = 0;
+			} else {
+				pa = ihk_numa_alloc_pages(&memory_nodes[pref_node], npages, p2align);
+			}
 #else
 		list_for_each_entry(pa_allocator,
 				&memory_nodes[pref_node].allocators, list) {
-			pa = ihk_pagealloc_alloc(pa_allocator, npages, p2align);
+			if (rusage_check_oom(pref_node, npages, is_user) == -ENOMEM) {
+				pa = 0;
+			} else {
+				pa = ihk_pagealloc_alloc(pa_allocator, npages, p2align);
+			}
 #endif
-
 			if (pa) {
+				rusage_page_add(pref_node, npages, is_user);
 				dkprintf("%s: explicit (node: %d) CPU @ node %d allocated "
 						"%d pages from node %d\n",
 						__FUNCTION__,
@@ -586,8 +596,6 @@ static void *mckernel_allocate_aligned_pages_node(int npages, int p2align,
 						ihk_mc_get_numa_id(),
 						npages, node);
 
-				rusage_page_add(pref_node, npages, is_user);
-
 				return phys_to_virt(pa);
 			}
 			else {
@@ -617,23 +625,30 @@ static void *mckernel_allocate_aligned_pages_node(int npages, int p2align,
 				numa_id = memory_nodes[node].nodes_by_distance[i].id;
 #ifdef IHK_RBTREE_ALLOCATOR
 				{
-					pa = ihk_numa_alloc_pages(&memory_nodes[memory_nodes[node].
-							nodes_by_distance[i].id], npages, p2align);
+					if (rusage_check_oom(numa_id, npages, is_user) == -ENOMEM) {
+						pa = 0;
+					} else {
+						pa = ihk_numa_alloc_pages(&memory_nodes[memory_nodes[node].
+																nodes_by_distance[i].id], npages, p2align);
+					}
 #else
 				list_for_each_entry(pa_allocator,
 						&memory_nodes[numa_id].allocators, list) {
-					pa = ihk_pagealloc_alloc(pa_allocator, npages, p2align);
+					if (rusage_check_oom(numa_id, npages, is_user) == -ENOMEM) {
+						pa = 0;
+					} else {
+						pa = ihk_pagealloc_alloc(pa_allocator, npages, p2align);
+					}
 #endif
 
 					if (pa) {
+						rusage_page_add(numa_id, npages, is_user);
 						dkprintf("%s: policy: CPU @ node %d allocated "
 								"%d pages from node %d\n",
 								__FUNCTION__,
 								ihk_mc_get_numa_id(),
 								npages, node);
 
-						rusage_page_add(numa_id, npages,
-						               is_user);
 
 						break;
 					}
@@ -674,22 +689,31 @@ distance_based:
 
 #ifdef IHK_RBTREE_ALLOCATOR
 		{
-			pa = ihk_numa_alloc_pages(&memory_nodes[memory_nodes[node].
-					nodes_by_distance[i].id], npages, p2align);
+			if (rusage_check_oom(numa_id, npages, is_user) == -ENOMEM) {
+				pa = 0;
+			} else {
+				pa = ihk_numa_alloc_pages(&memory_nodes[memory_nodes[node].
+														nodes_by_distance[i].id], npages, p2align);
+			}
 #else
 		list_for_each_entry(pa_allocator,
 		                    &memory_nodes[numa_id].allocators, list) {
-			pa = ihk_pagealloc_alloc(pa_allocator, npages, p2align);
+			if (rusage_check_oom(numa_id, npages, is_user) == -ENOMEM) {
+				pa = 0;
+			} else {
+				pa = ihk_pagealloc_alloc(pa_allocator, npages, p2align);
+			}
 #endif
 
+
 			if (pa) {
+				rusage_page_add(numa_id, npages, is_user);
 				dkprintf("%s: distance: CPU @ node %d allocated "
 						"%d pages from node %d\n",
 						__FUNCTION__,
 						ihk_mc_get_numa_id(),
 						npages,
 						memory_nodes[node].nodes_by_distance[i].id);
-				rusage_page_add(numa_id, npages, is_user);
 				break;
 			}
 		}
@@ -708,13 +732,22 @@ order_based:
 		numa_id = (node + i) % ihk_mc_get_nr_numa_nodes();
 #ifdef IHK_RBTREE_ALLOCATOR
 		{
-			pa = ihk_numa_alloc_pages(&memory_nodes[(node + i) %
-					ihk_mc_get_nr_numa_nodes()], npages, p2align);
+			if (rusage_check_oom(numa_id, npages, is_user) == -ENOMEM) {
+				pa = 0;
+			} else {
+				pa = ihk_numa_alloc_pages(&memory_nodes[(node + i) %
+														ihk_mc_get_nr_numa_nodes()], npages, p2align);
+			}
 #else
 		list_for_each_entry(pa_allocator,
 		                    &memory_nodes[numa_id].allocators, list) {
-			pa = ihk_pagealloc_alloc(pa_allocator, npages, p2align);
+			if (rusage_check_oom(numa_id, npages, is_user) == -ENOMEM) {
+				pa = 0;
+			} else {
+				pa = ihk_pagealloc_alloc(pa_allocator, npages, p2align);
+			}
 #endif
+
 			if (pa) {
 				rusage_page_add(numa_id, npages, is_user);
 				break;
@@ -730,6 +763,7 @@ order_based:
 	if(flag != IHK_MC_AP_NOWAIT)
 		panic("Not enough space\n");
 	*/
+	dkprintf("OOM\n", __FUNCTION__);
 	return NULL;
 }
 
@@ -1256,13 +1290,13 @@ static void numa_init(void)
 #endif
 
 #ifdef IHK_RBTREE_ALLOCATOR
-		dkprintf("Physical memory: 0x%lx - 0x%lx, %lu bytes, %d pages available @ NUMA: %d\n",
+		kprintf("Physical memory: 0x%lx - 0x%lx, %lu bytes, %d pages available @ NUMA: %d\n",
 				start, end,
 				end - start,
 				(end - start) >> PAGE_SHIFT,
 				numa_id);
 #else
-		dkprintf("Physical memory: 0x%lx - 0x%lx, %lu bytes, %d pages available @ NUMA: %d\n",
+		kprintf("Physical memory: 0x%lx - 0x%lx, %lu bytes, %d pages available @ NUMA: %d\n",
 				start, end,
 				ihk_pagealloc_count(allocator) * PAGE_SIZE,
 				ihk_pagealloc_count(allocator),
@@ -1659,6 +1693,22 @@ void ihk_mc_clean_micpa(void){
 }
 #endif
 
+static void rusage_init()
+{
+	int npages;
+	unsigned long phys;
+
+	npages = (sizeof(struct rusage_global) + PAGE_SIZE -1) >> PAGE_SHIFT;
+	rusage = ihk_mc_alloc_pages(npages, IHK_MC_AP_CRITICAL);
+	memset(rusage, 0, npages * PAGE_SIZE);
+	rusage->num_processors = num_processors;
+	rusage->num_numa_nodes = ihk_mc_get_nr_numa_nodes();
+	rusage->ns_per_tsc = ihk_mc_get_ns_per_tsc();
+	phys = virt_to_phys(rusage);
+	ihk_set_rusage(phys, sizeof(struct rusage_global));
+	dkprintf("%s: rusage->total_memory=%ld\n", __FUNCTION__, rusage->total_memory);
+}
+
 #ifdef POSTK_DEBUG_TEMP_FIX_73 /* NULL access for *monitor fix */
 extern void monitor_init(void);
 #endif /* POSTK_DEBUG_TEMP_FIX_73 */
@@ -1667,6 +1717,10 @@ void mem_init(void)
 #ifdef POSTK_DEBUG_TEMP_FIX_73 /* NULL access for *monitor fix */
 	monitor_init();
 #endif /* !POSTK_DEBUG_TEMP_FIX_73 */
+
+	/* It must precedes numa_init() because rusage->total_memory is initialized in numa_init() */
+	rusage_init();
+
 	/* Initialize NUMA information and memory allocator bitmaps */
 	numa_init();
 
diff --git a/kernel/process.c b/kernel/process.c
index 013bc289..e58026db 100644
--- a/kernel/process.c
+++ b/kernel/process.c
@@ -2638,6 +2638,19 @@ void release_thread(struct thread *thread)
 
 	release_process_vm(vm);
 	rusage_num_threads_dec();
+
+#ifdef RUSAGE_DEBUG
+	if (rusage->num_threads == 0) {
+		int i;
+		kprintf("total_memory_usage=%ld\n", rusage->total_memory_usage);
+		for(i = 0; i < IHK_MAX_NUM_PGSIZES; i++) {
+			kprintf("memory_stat_rss[%d]=%ld\n", i, rusage->memory_stat_rss[i]);
+		}
+		for(i = 0; i < IHK_MAX_NUM_PGSIZES; i++) {
+			kprintf("memory_stat_mapped_file[%d]=%ld\n", i, rusage->memory_stat_mapped_file[i]);
+		}
+	}
+#endif
 }
 
 void cpu_set(int cpu, cpu_set_t *cpu_set, ihk_spinlock_t *lock)
@@ -3397,6 +3410,18 @@ void runq_add_thread(struct thread *thread, int cpu_id)
 	procfs_create_thread(thread);
 
 	rusage_num_threads_inc();
+#ifdef RUSAGE_DEBUG
+	if (rusage->num_threads == 1) {
+		int i;
+		kprintf("total_memory_usage=%ld\n", rusage->total_memory_usage);
+		for(i = 0; i < IHK_MAX_NUM_PGSIZES; i++) {
+			kprintf("memory_stat_rss[%d]=%ld\n", i, rusage->memory_stat_rss[i]);
+		}
+		for(i = 0; i < IHK_MAX_NUM_PGSIZES; i++) {
+			kprintf("memory_stat_mapped_file[%d]=%ld\n", i, rusage->memory_stat_mapped_file[i]);
+		}
+	}
+#endif
 
 	/* Kick scheduler */
 #ifdef POSTK_DEBUG_ARCH_DEP_8 /* arch depend hide */
diff --git a/kernel/syscall.c b/kernel/syscall.c
index f85dd74a..c84ec0eb 100644
--- a/kernel/syscall.c
+++ b/kernel/syscall.c
@@ -1063,14 +1063,6 @@ void terminate(int rc, int sig)
 
 	dkprintf("terminate,pid=%d\n", proc->pid);
 
-	/* rusage debug */
-	for(i = 0; i < IHK_MAX_NUM_PGSIZES; i++) {
-		dkprintf("memory_stat_rss[%d]=%ld\n", i, rusage->memory_stat_rss[i]);
-	}
-	for(i = 0; i < IHK_MAX_NUM_PGSIZES; i++) {
-		dkprintf("memory_stat_mapped_file[%d]=%ld\n", i, rusage->memory_stat_mapped_file[i]);
-	}
-
 #ifdef DCFA_KMOD
 	do_mod_exit(rc);
 #endif
@@ -1147,8 +1139,7 @@ terminate_host(int pid)
 	do_kill(cpu_local_var(current), pid, -1, SIGKILL, NULL, 0);
 }
 
-void 
-eventfd()
+void eventfd(int type)
 {
 	struct ihk_ikc_channel_desc *syscall_channel;
 	struct ikc_scd_packet pckt;
@@ -1156,6 +1147,7 @@ eventfd()
 	syscall_channel = get_cpu_local_var(0)->ikc2linux;
 	memset(&pckt, '\0', sizeof pckt);
 	pckt.msg = SCD_MSG_EVENTFD;
+	pckt.eventfd_type = type;
 	ihk_ikc_send(syscall_channel, &pckt, 0);
 }
 
diff --git a/lib/abort.c b/lib/abort.c
index a3389bd6..1a8a8665 100644
--- a/lib/abort.c
+++ b/lib/abort.c
@@ -4,13 +4,15 @@
 #include <ihk/monitor.h>
 
 extern struct cpu_local_var *clv;
+extern void eventfd(int type);
 
 void panic(const char *msg)
 {
 	if (clv) {
 		struct ihk_os_cpu_monitor *monitor = cpu_local_var(monitor);
-
+		//kprintf("%s: calling eventfd\n", __FUNCTION__);
 		monitor->status = IHK_OS_MONITOR_PANIC;
+		eventfd(IHK_OS_EVENTFD_TYPE_STATUS);
 	}
 	cpu_disable_interrupt();
 
diff --git a/lib/include/ihk/debug.h b/lib/include/ihk/debug.h
index 33168930..73716601 100644
--- a/lib/include/ihk/debug.h
+++ b/lib/include/ihk/debug.h
@@ -33,15 +33,6 @@
 
 #endif /* POSTK_DEBUG_ARCH_DEP_9 */
 
-struct ihk_kmsg_buf {
-	int tail;
-	int len;
-	int head;
-	int mode;
-	ihk_spinlock_t lock;
-	char str[IHK_KMSG_SIZE - sizeof(int) * 4 - sizeof(ihk_spinlock_t)];
-};
-
 extern int kprintf(const char *format, ...);
 extern unsigned long kprintf_lock(void);
 extern void kprintf_unlock(unsigned long irqflags);