release: 1.7.8: fix ihklib/ihk_reserve_cpu when using krm

Change-Id: I57235d51f51ae7327cb08a9e3ae56be995157100

CMakeLists.txt

@@ -7,7 +7,7 @@ endif (NOT CMAKE_BUILD_TYPE)
 enable_language(C ASM)
 
 project(mckernel C ASM)
-set(MCKERNEL_VERSION "1.7.2")
+set(MCKERNEL_VERSION "1.7.8")
 
 # See "Fedora Packaging Guidelines -- Versioning"
 set(MCKERNEL_RELEASE "")
@@ -124,6 +124,14 @@ if(ENABLE_FUGAKU_DEBUG)
 	set(KBUILD_C_FLAGS "${KBUILD_C_FLAGS} -DENABLE_FUGAKU_DEBUG")
 endif()
 
+# redirect kernel messages to Linux's /dev/kmsg
+option(ENABLE_KMSG_REDIRECT "Redirect kernel message to Linux's /dev/kmsg" OFF)
+
+if(ENABLE_KMSG_REDIRECT)
+	add_definitions(-DENABLE_KMSG_REDIRECT)
+	set(KBUILD_C_FLAGS "${KBUILD_C_FLAGS} -DENABLE_KMSG_REDIRECT")
+endif()
+
 option(PROFILE_ENABLE "System call profile" ON)
 if(PROFILE_ENABLE)
 	add_definitions(-DPROFILE_ENABLE)
@@ -137,8 +145,7 @@ if (ENABLE_LINUX_WORK_IRQ_FOR_IKC)
 endif()
 
 if (BUILD_TARGET STREQUAL "smp-arm64")
-	execute_process(COMMAND bash -c "find -L ${CMAKE_CURRENT_SOURCE_DIR} -name \"*.[chS]\"|xargs -r grep -oE \"^#if.*def.*POSTK_DEBUG[^ ]*\" | cut -d':' -f2 | cut -d' ' -f2 | sort | uniq | awk '{print \"-D\"\$0 }' | xargs"
-		OUTPUT_VARIABLE POSTK_DEBUG_DEFINES OUTPUT_STRIP_TRAILING_WHITESPACE)
+	set(POSTK_DEBUG_DEFINES "-DPOSTK_DEBUG_ARCH_DEP_100 -DPOSTK_DEBUG_ARCH_DEP_42 -DPOSTK_DEBUG_TEMP_FIX_29 -DPOSTK_DEBUG_TEMP_FIX_49")
 	add_definitions("${POSTK_DEBUG_DEFINES}")
 	set(KBUILD_C_FLAGS "${KBUILD_C_FLAGS} ${POSTK_DEBUG_DEFINES}")
 
@@ -341,4 +348,5 @@ message("ENABLE_WERROR: ${ENABLE_WERROR}")
 message("ENABLE_UBSAN: ${ENABLE_UBSAN}")
 message("ENABLE_LINUX_WORK_IRQ_FOR_IKC: ${ENABLE_LINUX_WORK_IRQ_FOR_IKC}")
 message("ENABLE_PER_CPU_ALLOC_CACHE: ${ENABLE_PER_CPU_ALLOC_CACHE}")
+message("ENABLE_KMSG_REDIRECT: ${ENABLE_KMSG_REDIRECT}")
 message("-------------------------------")

arch/arm64/kernel/cpu.c

@@ -912,7 +912,6 @@ unsigned long cpu_enable_interrupt_save(void)
 	return flags;
 }
 
-#ifdef ENABLE_FUGAKU_HACKS
 int cpu_interrupt_disabled(void)
 {
 	unsigned long flags;
@@ -925,7 +924,6 @@ int cpu_interrupt_disabled(void)
 		: "memory");
 	return (flags == masked);
 }
-#endif
 
 #else /* defined(CONFIG_HAS_NMI) */
 
@@ -989,6 +987,18 @@ unsigned long cpu_enable_interrupt_save(void)
 		: "memory");
 	return flags;
 }
+
+int cpu_interrupt_disabled(void)
+{
+	unsigned long flags;
+
+	asm volatile(
+		"mrs    %0, daif	// arch_local_irq_save\n"
+		: "=r" (flags)
+		:
+		: "memory");
+	return !!(flags & 0x2);
+}
 #endif /* defined(CONFIG_HAS_NMI) */
 
 /* we not have "pause" instruction, instead "yield" instruction */

arch/x86_64/kernel/cpu.c

@@ -1273,6 +1273,15 @@ unsigned long cpu_enable_interrupt_save(void)
 	return flags;
 }
 
+int cpu_interrupt_disabled(void)
+{
+	unsigned long flags;
+
+	asm volatile("pushf; pop %0" : "=r"(flags) : : "memory", "cc");
+
+	return !(flags & 0x200);
+}
+
 /*@
   @ behavior valid_vector:
   @   assumes 32 <= vector <= 255;

docs/NEWS.rst

@@ -1,3 +1,246 @@
+=============================================
+Version 1.7.8 (Mar 12, 2021)
+=============================================
+
+----------------------
+IHK major updates
+----------------------
+N/A
+
+------------------------
+IHK major bug fixes
+------------------------
+#. ihklib: ihk_reserve_cpu: fix job cpu check when using krm
+
+----------------------
+McKernel major updates
+----------------------
+N/A
+
+------------------------
+McKernel major bug fixes
+------------------------
+N/A
+
+=============================================
+Version 1.7.7 (Mar 11, 2021)
+=============================================
+
+----------------------
+IHK major updates
+----------------------
+N/A
+
+------------------------
+IHK major bug fixes
+------------------------
+N/A
+
+----------------------
+McKernel major updates
+----------------------
+N/A
+
+------------------------
+McKernel major bug fixes
+------------------------
+#. mcexec: fput executable just after its contents is transferred
+#. spec: cmake-config cmake parameters
+
+=============================================
+Version 1.7.6 (Mar 11, 2021)
+=============================================
+
+----------------------
+IHK major updates
+----------------------
+N/A
+
+------------------------
+IHK major bug fixes
+------------------------
+#. ihklib: ihk_reserve_mem_conf*: apply change only to the next reservation
+
+----------------------
+McKernel major updates
+----------------------
+N/A
+
+------------------------
+McKernel major bug fixes
+------------------------
+N/A
+
+=============================================
+Version 1.7.5 (Mar 11, 2021)
+=============================================
+
+----------------------
+IHK major updates
+----------------------
+N/A
+
+------------------------
+IHK major bug fixes
+------------------------
+#. ihklib: fix cgroup cpuset.cpus/mems check when using krm
+#. ihklib: ihk_reserve_mem_conf_str: set default values to those not specified
+
+----------------------
+McKernel major updates
+----------------------
+N/A
+
+------------------------
+McKernel major bug fixes
+------------------------
+N/A
+
+=============================================
+Version 1.7.4 (Mar 7, 2021)
+=============================================
+
+----------------------
+IHK major updates
+----------------------
+N/A
+
+------------------------
+IHK major bug fixes
+------------------------
+N/A
+
+----------------------
+McKernel major updates
+----------------------
+N/A
+
+------------------------
+McKernel major bug fixes
+------------------------
+N/A
+
+=============================================
+Version 1.7.3 (Mar 5, 2021)
+=============================================
+
+----------------------
+IHK major updates
+----------------------
+N/A
+
+------------------------
+IHK major bug fixes
+------------------------
+N/A
+
+----------------------
+McKernel major updates
+----------------------
+N/A
+
+------------------------
+McKernel major bug fixes
+------------------------
+N/A
+
+=============================================
+Version 1.7.2 (Mar 5, 2021)
+=============================================
+
+----------------------
+IHK major updates
+----------------------
+#. ihklib: add *_str() functions for reserve, assign, IKC-map, kargs
+#. smp: make smp_call_func() arch independent
+
+------------------------
+IHK major bug fixes
+------------------------
+#. ihklib: ihk_reserve_mem: fix capped best-effort
+#. TO RESET: fake missing NUMA node pieces, 90% memory limit
+#. ihklib: ihk_reserve_mem_conf: range-check for IHK_RESERVE_MEM_MAX_SIZE_RATIO_ALL
+#. ihklib: ihk_os_kargs: check if "hidos" is included
+#. SMP: omit slab/slub shrink, use 95% limit by default
+#. check cpu / numa cgroup set by krm
+#. SMP: __ihk_smp_reserve_mem: add __GFP_COMP to __GFP_ATOMIC allocation
+#. ihk_register_device: record minor to IHK device object
+
+----------------------
+McKernel major updates
+----------------------
+#. mcexec: memory policy control by environmental variable
+#. mempolicy: Support MPOL_INTERLEAVE
+#. uti: futex call function in mcctrl
+#. uti: integrate libuti and redirect to mck/libuti.so
+#. uti: integrate syscall_intercept
+#. shmobj: support large page
+#. xpmem: support large page
+#. MM: handle zero_at_free in page faults
+
+------------------------
+McKernel major bug fixes
+------------------------
+#. TO RESET: stack changes
+#. Tofu: keep track of stags per memory range
+#. Tofu: match page sizes to MBPT and fault PTEs if not present
+#. Tofu: fix phys addr calculation for contiguous pages in MBPT/BCH update
+#. rus_vm_fault: vmf_insert_pfn: treat VM_FAULT_NOPAGE as success
+#. Tofu: mcctrl side MMU notifier and CQ/BCH cleanup
+#. copy_user_ranges: copy straight_start of struct vm_range
+#. mcctrl: abort on invalid addr in mcexec_transfer_image()
+#. mcctrl: fix access to uninitialized usrdata->cpu_topology_list
+#. mcexec: propagate error in __NR_gettid handler
+#. mcexec_transfer_image(): map exact size of remote memory (instead of forcing PAGE_SIZE)
+#. xpmem: fault stack area of remote process if VM range doesn't yet exist
+#. Tofu: fault stack area if VM range doesn't exist in STAG registration
+#. __mcctrl_os_read_write_cpu_register: fix timeout
+#. mbind: Use range_policy's numamask as priority on MPOL_BIND
+#. migrate: Don't migrate on in-kernel interrupt
+#. Send a signal to mcexec after switching to that process.
+#. uti: fix syscall response is mis-consumed by __do_in_kernel_irq_syscall
+#. uti: fix handling UTI_CPU_SET env
+#. do_execveat: kill instead of panic when init_process_stack fails
+#. remote_page_fault is handled by the offloaded thread.
+#. coredump: fix behavior when gencore fail
+#. xpmem: truncates the size of xpmem_attach at the page boundary (workaround for fjmpi)
+#. __mcctrl_os_read_write_cpu_register: spin timeout in mcctrl_ikc_send_wait()
+
+=============================================
+Version 1.7.1 (Dec 23, 2020)
+=============================================
+
+----------------------
+IHK major updates
+----------------------
+#. d5d5c23 Tofu: support for barrier gate
+#. Tofu: proper cleanup of premapped DMA regions
+#. Tofu: initial version
+#. SMP: try with GFP_ATOMIC as well in mem reserve
+
+------------------------
+IHK major bug fixes
+------------------------
+#. ihklib: ihk(_os)_query_{cpu,mem}: allow to pass empty array
+#. SMP: non compound page free and GFP_ATOMIC
+#. ihk_get_num_os_instances: don't open /dev/mcdN
+#. ihklib: ihk_create_os_str: fix variable prefix
+
+----------------------
+McKernel major updates
+----------------------
+#. stragiht map: creates a straight map covering the whole physical memory, and gives virtual address ranges out of it to maps to which physical pages are allocated on map
+#. free-time, lazy, potentially Linux-side page-zeroing
+#. Tofu built-in driver: supports memory registration and barrier gate setup
+#. kmalloc cache
+
+------------------------
+McKernel major bug fixes
+------------------------
+#. mmap: return -EINVAL for non-anonymous, MAP_HUGETLB map
+#. kernel: increase stack size
+#. Tofu: proper cleanup of device files when mcexec gets killed
+
 =============================================
 Version 1.7.0 (Nov 25, 2020)
 =============================================

docs/install.rst

@@ -29,7 +29,7 @@ Install the following packages to the build machine:
 
 ::
 
-   cmake kernel-devel binutils-devel systemd-devel numactl-devel gcc make nasm git libdwarf-devel
+   cmake kernel-devel binutils-devel systemd-devel numactl-devel gcc make nasm git libdwarf-devel capstone-devel
 
 When having access to repositories
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -46,17 +46,48 @@ On CentOS 8, enable the PowerTools repository:
 
    sudo dnf config-manager --set-enabled PowerTools
 
+Enable EPEL repository:
+
+::
+
+   sudo yum install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
+
 Install with yum:
 
 ::
 
-   sudo yum install cmake kernel-devel binutils-devel systemd-devel numactl-devel gcc make nasm git libdwarf-devel
+   sudo yum install cmake kernel-devel binutils-devel systemd-devel numactl-devel gcc make nasm git libdwarf-devel capstone-devel
 
 When not having access to repositories
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+``libdwarf-devel``
+""""""""""""""""""
+
 Ask the system administrator to install them. Note that ``libdwarf-devel`` is in the CodeReady Linux Builder repository on RHEL 8 or in the PowerTools repository on CentOS 8.
 
+``capstone-devel``
+""""""""""""""""""
+
+A. Ask the system administrator to install ``capstone-devel``. Note that it is in the EPEL repository.
+
+B. Download the rpm with the machine in which you are the administrator:
+
+::
+
+   sudo yum install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
+   sudo yum install yum-utils
+   yumdownloader capstone-devel
+
+And then install it to your home directory:
+
+::
+
+   cd $HOME/$(uname -p)
+   rpm2cpio capstone-devel-4.0.1-9.el8.aarch64.rpm | cpio -idv
+   sed -i 's#/usr/#'"$HOME"'/'"$(uname -p)"'/usr/#' $HOME/$(uname -p)/usr/lib64/pkgconfig/capstone.pc
+
+
 Clone, compile, install
 --------------------------
 
@@ -93,7 +124,12 @@ When not cross-compiling:
 
 ::
 
-   cmake -DCMAKE_INSTALL_PREFIX=${HOME}/ihk+mckernel ../mckernel
+   CMAKE_PREFIX_PATH=${HOME}/$(uname -p)/usr \
+     cmake -DCMAKE_INSTALL_PREFIX=${HOME}/ihk+mckernel \
+     -DENABLE_UTI=ON \
+     ../mckernel
+
+Note that ``CMAKE_PREFIX_PATH=${HOME}/$(uname -p)/usr`` is required only when ``capstone-devel`` is installed to your home directory.
 
 When cross-compiling:
 ~~~~~~~~~~~~~~~~~~~~~
@@ -105,10 +141,11 @@ When cross-compiling:
      -DKERNEL_DIR=<kernnel_dir> \
      -DBUILD_TARGET=smp-arm64 \
      -DCMAKE_TOOLCHAIN_FILE=../mckernel/cmake/cross-aarch64.cmake \
+     -DENABLE_UTI=ON \
      ../mckernel
 
 Install with cmake
-~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~
 
 Install with make:
 
@@ -129,22 +166,6 @@ Create the tarball and the spec file:
    make dist
    cp mckernel-<version>.tar.gz <rpmbuild>/SOURCES
 
-(optional) Edit the following line in ``scripts/mckernel.spec`` to change
-cmake options. For example:
-
-::
-
-   %cmake -DCMAKE_BUILD_TYPE=Release \
-	-DUNAME_R=%{kernel_version} \
-	-DKERNEL_DIR=%{kernel_dir} \
-	%{?cmake_libdir:-DCMAKE_INSTALL_LIBDIR=%{cmake_libdir}} \
-	%{?build_target:-DBUILD_TARGET=%{build_target}} \
-	%{?toolchain_file:-DCMAKE_TOOLCHAIN_FILE=%{toolchain_file}} \
-	-DENABLE_TOFU=ON -DENABLE_FUGAKU_HACKS=ON \
-	-DENABLE_KRM_WORKAROUND=OFF -DWITH_KRM=ON \
-	-DENABLE_FUGAKU_DEBUG=OFF -DENABLE_UTI=ON \
-	.
-
 Create the rpm package:
 
 When not cross-compiling:
@@ -191,10 +212,10 @@ Install the following packages to the compute nodes:
 
 ::
 
-   systemd-libs numactl-libs libdwarf
+   systemd-libs numactl-libs libdwarf capstone
 
 When having access to repositories
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 On RHEL 8, enable the CodeReady Linux Builder (CLB) repository:
 
@@ -208,13 +229,42 @@ On CentOS 8, enable the PowerTools repository:
 
    sudo dnf config-manager --set-enabled PowerTools
 
+Enable EPEL repository:
+
+::
+
+   sudo yum install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
+
 Install with yum:
 
 ::
 
-   sudo yum install systemd-libs numactl-libs libdwarf
+   sudo yum install systemd-libs numactl-libs libdwarf capstone
 
 When not having access to repositories
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+``libdwarf``
+""""""""""""
+
 Ask the system administrator to install them. Note that ``libdwarf`` is in the CodeReady Linux Builder repository on RHEL 8 or in the PowerTools repository on CentOS 8.
+
+``capstone``
+""""""""""""
+
+A. Ask the system administrator to install ``capstone``. Note that it is in the EPEL repository.
+
+B. Download the rpm with the machine in which you are the administrator:
+
+::
+
+   sudo yum install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
+   sudo yum install yum-utils
+   yumdownloader capstone
+
+and then install it to your home directory:
+
+::
+
+   cd $HOME/$(uname -p)
+   rpm2cpio capstone-4.0.1-9.el8.aarch64.rpm | cpio -idv

docs/operators.rst

@@ -7,12 +7,10 @@ This document will explain how to operate system with McKernel.
 Installation
 ============
 
-See `Quick Guide -- Installation <quick.html#installation>`__.
-
-.. include:: uti.rst
+Follow `Quick Guide -- Installation <quick.html#installation>`__.
 
 Boot and Shut-down
-=========================
+==================
 
 Related files
 -------------

docs/spec/ihk/ihk.tex

@@ -649,7 +649,9 @@ IHKはLinuxに以下の機能を提供する。
 \begin{tabular}[t]{@{}l@{}}
 {\quad} \texttt{int ihk\_reserve\_mem\_conf(int index, int key, void *value)}\\
 \end{tabular}
-\subsubsection*{説明}{\quad} \texttt{index}で指定されたIHKデバイスに対する\texttt{ihk\_reserve\_mem()}の動作を\texttt{key}と\texttt{value}のペアで指定したものに変更する。\texttt{value}は値へのポインタで指定する。\texttt{key}と\texttt{value}のペアの意味は以下のように定義される。
+\subsubsection*{説明}{\quad} \texttt{index}で指定されたIHKデバイスに対する\texttt{ihk\_reserve\_mem()}の動作を\texttt{key}と\texttt{value}のペアで指定したものに変更する。なお、設定は次の1回の予約に限り有効で、予約後にはデフォルト設定に戻る。
+
+\texttt{value}は値へのポインタで指定する。\texttt{key}と\texttt{value}のペアの意味は以下のように定義される。
 
 \subsubsection*{\texttt{IHK\_RESERVE\_MEM\_BALANCED\_\{ENABLE,BEST\_EFFORT,VARIANCE\_LIMIT\}}}
 \verb|IHK_RESERVE_MEM_BALANCED_ENABLE|（型は\verb|int|、デフォルトは0）が非ゼロの場合は、NUMAノードごとの予約サイズがNUMAノード間でなるべく均等になるように予約する。目的は、NUMAノードごとのメモリ空き容量にNUMAノード間でばらつきがあり、またそれらの空き容量が事前にわからないようなシステムで、合計予約サイズをより大きくすることである。ステップは以下の通り。
@@ -686,7 +688,7 @@ IHKはLinuxに以下の機能を提供する。
 
 \subsubsection{設定リストによるメモリ予約動作設定}
 \subsubsection*{書式}{\quad} \verb:int ihk_reserve_mem_conf_str(int dev_index, const char *envp, int num_env);:
-\subsubsection*{説明}{\quad} \verb:dev_index:で指定されたIHKデバイスに対し、\verb:envp:と\verb:num_env:で指定された文字列形式の設定リストに従ってメモリ予約の動作設定を行う。本関数は特権ユーザのみが呼び出せる。
+\subsubsection*{説明}{\quad} \verb:dev_index:で指定されたIHKデバイスに対し、\verb:envp:と\verb:num_env:で指定された文字列形式の設定リストに従ってメモリ予約の動作設定を行う。なお、設定は次の1回の予約に限り有効で、予約後にはデフォルト設定に戻る。本関数は特権ユーザのみが呼び出せる。
 
 \verb:envp:は\verb:NULL:文字で結合された\verb:num_env:個の設定文字列からなる。各設定文字列は\verb:"KEY=VAL":の形式を持つ。設定可能な項目は以下の通り。
 \begin{table}[!h]
@@ -702,7 +704,7 @@ IHKはLinuxに以下の機能を提供する。
 \end{tabular}
 \vspace{-0em}
 \end{table}
-\\なお、これ以外の設定は無視される。
+\\また、これら以外の設定項目は無視される。
 \FloatBarrier
 
 \subsubsection*{戻り値}

docs/users.rst

@@ -55,11 +55,11 @@ You need to insert ``mcexec`` into the command lines invoking the programs that
 Non-MPI programs
 ~~~~~~~~~~~~~~~~
 
-Insert ``mcexec`` before an executable:
+Insert ``mcexec`` before the command:
 
 ::
 
-   mcexec ./a.out
+   mcexec <command>
 
 MPI programs
 ~~~~~~~~~~~~
@@ -96,6 +96,19 @@ Fujitsu Technical Computing Suite.
    #PJM --mpi "proc=32"
    #PJM -L "node=8"
 
+(Advanced) When using Utility Thread offloading Interface (UTI)
+---------------------------------------------------------------
+
+UTI enables a runtime such as MPI runtime to spawn utility threads such
+as MPI asynchronous progress threads to Linux cores.
+
+Add ``--enable-uti`` option to ``mcexec``:
+
+::
+
+   mcexec --enable-uti <command>
+
+
 Limitations
 ===========
 

docs/uti.rst

@@ -1,94 +0,0 @@
-Advanced: Enable Utility Thread offloading Interface (UTI)
--------------------------------------------------------------
-
-UTI enables a runtime such as MPI runtime to spawn utility threads such
-as MPI asynchronous progress threads to Linux cores.
-
-Install ``capstone`` and ``capstone-devel``
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-When compute nodes don't have access to EPEL repository
-"""""""""""""""""""""""""""""""""""""""""""""""""""""""
-
-Install EPEL ``capstone`` and ``capstone-devel``:
-
-::
-
-   sudo yum install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
-   sudo yum install capstone capstone-devel
-
-
-When compute nodes don't have access to EPEL repository
-"""""""""""""""""""""""""""""""""""""""""""""""""""""""
-
-A. Ask the system administrator to install ``capstone`` and ``capstone-devel``. Note that it is in the EPEL repository.
-
-B. Download the rpm with the machine in which you are the administrator:
-
-::
-
-   sudo yum install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
-   sudo yum install yum-utils
-   yumdownloader capstone capstone-devel
-
-and then install it to your home directory of the login node:
-
-::
-
-   cd $HOME/$(uname -p)
-   rpm2cpio capstone-4.0.1-9.el8.aarch64.rpm | cpio -idv
-   rpm2cpio capstone-devel-4.0.1-9.el8.aarch64.rpm | cpio -idv
-   sed -i 's#/usr/#'"$HOME"'/'"$(uname -p)"'/usr/#' $HOME/$(uname -p)/usr/lib64/pkgconfig/capstone.pc
-
-
-Install McKernel
-~~~~~~~~~~~~~~~~~~~~
-
-``cmake`` with the additional options:
-
-::
-
-   cmake -DCMAKE_INSTALL_PREFIX=${HOME}/ihk+mckernel -DENABLE_UTI=ON $HOME/src/ihk+mckernel/mckernel
-   make -j install
-
-Run programs
-~~~~~~~~~~~~
-
-``mcexec`` with ``--enable-uti`` option:
-
-::
-
-   mcexec --enable-uti <command>
-
-(Optional) Install UTI for Linux
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-You can skip this step if you don't want to develop a run-time using UTI, or if it's already installed with, for example, Fujitsu Technical Computing Suite.
-
-Install by make
-"""""""""""""""
-
-.. code-block:: none
-
-   git clone https://github.com/RIKEN-SysSoft/uti.git
-   mkdir build && cd build
-   ../uti/configure --prefix=<uti-install> --with-rm=linux
-   make && make install
-
-Install by rpm
-""""""""""""""
-
-.. code-block:: none
-
-   git clone https://github.com/RIKEN-SysSoft/uti.git
-   mkdir build && cd build
-   ../uti/configure --prefix=<uti-install> --with-rm=linux
-   rm -f ~/rpmbuild/SOURCES/<version>.tar.gz
-   rpmbuild -ba ./scripts/uti.spec
-   rpm -Uvh uti-<version>-<release>-<arch>.rpm
-
-(Optional) Install UTI for McKernel
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-You can skip this step if you don't want to develop a run-time using UTI.
-Execute the commands above for installing UTI for Linux, with ``--with-rm=linux`` replaced with ``--with-rm=mckernel``.

executer/kernel/mcctrl/control.c

@@ -2010,7 +2010,7 @@ int mcexec_open_exec(ihk_os_t os, char * __user filename)
 	fullpath = d_path(&file->f_path, pathbuf, PATH_MAX);
 	if (IS_ERR(fullpath)) {
 		retval = PTR_ERR(fullpath);
-		goto out_free;
+		goto out_put_file;
 	}
 
 	mcef = kmalloc(sizeof(*mcef), GFP_KERNEL);

executer/user/mcexec.c

@@ -2419,6 +2419,7 @@ int main(int argc, char **argv)
 	}
 #endif // MCEXEC_BIND_MOUNT
 
+	/* fget executable as well */
 	if ((ret = load_elf_desc_shebang(argv[optind], &desc,
 					 &shebang_argv, 1 /* execvp */))) {
 		fprintf(stderr, "%s: could not load program: %s\n",
@@ -2860,6 +2861,14 @@ int main(int argc, char **argv)
 		fprintf(stderr, "error: transferring image\n");
 		return -1;
 	}
+
+	/* fput executable */
+	if ((ret = ioctl(fd, MCEXEC_UP_CLOSE_EXEC)) != 0) {
+		fprintf(stderr, "error: MCEXEC_UP_CLOSE_EXEC failed with %d\n",
+			ret);
+		return 1;
+	}
+
 	fflush(stdout);
 	fflush(stderr);
 	
@@ -4111,11 +4120,6 @@ int main_loop(struct thread_data_s *my_thread)
 			   It is done by not calling do_syscall_return(fd, cpu, 0, 0, 0, 0, 0);
 			   here and making McKernel side wait until release_handler() is called. */
 
-			/* Drop executable file */
-			if ((ret = ioctl(fd, MCEXEC_UP_CLOSE_EXEC)) != 0) {
-				fprintf(stderr, "WARNING: close_exec() couldn't find exec file?\n");
-			}
-
 			__dprintf("__NR_exit/__NR_exit_group: %ld (cpu_id: %d)\n",
 					w.sr.args[0], cpu);
 			if(w.sr.number == __NR_exit_group){
@@ -4308,15 +4312,6 @@ gettid_out:
 				__dprintf("pid(%d): signals and syscall threads OK\n", 
 						getpid());
 
-				/* Hold executable also in the child process */
-				if ((ret = ioctl(fd, MCEXEC_UP_OPEN_EXEC, exec_path)) 
-					!= 0) {
-					fprintf(stderr, "Error: open_exec() fails for %s: %d (fd: %d)\n", 
-							exec_path, ret, fd);
-					fs->status = -errno;
-					goto fork_child_sync_pipe;
-				}
-
 				/* Check if we need to limit number of threads in the pool */
 				if ((ret = ioctl(fd, MCEXEC_UP_GET_NUM_POOL_THREADS)) < 0) {
 					fprintf(stderr, "Error: obtaining thread pool count\n");
@@ -4472,6 +4467,7 @@ fork_err:
 					}
 					filename = pathbuf;
 
+					/* fget executable as well */
 					if ((ret = load_elf_desc_shebang(filename, &desc,
 									 &shebang_argv, 0)) != 0) {
 						goto return_execve1;
@@ -4569,6 +4565,13 @@ return_execve1:
 					}
 					__dprintf("%s", "execve(): image transferred\n");
 
+					/* fput executable */
+					if ((ret = ioctl(fd, MCEXEC_UP_CLOSE_EXEC)) != 0) {
+						fprintf(stderr, "error: MCEXEC_UP_CLOSE_EXEC failed with %d\n",
+							ret);
+						return 1;
+					}
+
 					if (close_cloexec_fds(fd) < 0) {
 						ret = EINVAL;
 						goto return_execve2;

kernel/debug.c

@@ -87,7 +87,7 @@ void kputs(char *buf)
 	debug_spin_unlock_irqrestore(&kmsg_buf->lock, flags_inner);
 	kprintf_unlock(flags_outer);
 
-	if (irqflags_can_interrupt(flags_outer) &&
+	if (!cpu_interrupt_disabled() &&
 			DEBUG_KMSG_USED > IHK_KMSG_HIGH_WATER_MARK) {
 		eventfd(IHK_OS_EVENTFD_TYPE_KMSG);
 		ihk_mc_delay_us(IHK_KMSG_NOTIFY_DELAY);
@@ -128,7 +128,7 @@ int __kprintf(const char *format, ...)
 	}
 
 	debug_spin_unlock_irqrestore(&kmsg_buf->lock, flags_inner);
-	if (irqflags_can_interrupt(flags_inner) &&
+	if (!cpu_interrupt_disabled() &&
 			DEBUG_KMSG_USED > IHK_KMSG_HIGH_WATER_MARK) {
 		eventfd(IHK_OS_EVENTFD_TYPE_KMSG);
 		ihk_mc_delay_us(IHK_KMSG_NOTIFY_DELAY);
@@ -171,7 +171,7 @@ int kprintf(const char *format, ...)
 	debug_spin_unlock_irqrestore(&kmsg_buf->lock, flags_inner);
 	kprintf_unlock(flags_outer);
 
-	if (irqflags_can_interrupt(flags_outer) &&
+	if (!cpu_interrupt_disabled() &&
 			DEBUG_KMSG_USED > IHK_KMSG_HIGH_WATER_MARK) {
 		eventfd(IHK_OS_EVENTFD_TYPE_KMSG);
 		ihk_mc_delay_us(IHK_KMSG_NOTIFY_DELAY);

lib/include/ihk/cpu.h

@@ -23,9 +23,7 @@ extern int num_processors;
 
 void cpu_enable_interrupt(void);
 void cpu_disable_interrupt(void);
-#ifdef ENABLE_FUGAKU_HACKS
 int cpu_interrupt_disabled(void);
-#endif
 void cpu_halt(void);
 #ifdef ENABLE_FUGAKU_HACKS
 void cpu_halt_panic(void);

scripts/mckernel.spec.in

@@ -15,6 +15,11 @@
 %{!?kernel_dir: %global kernel_dir /usr/src/kernels/%{kernel_version}}
 %define krequires %(echo %{kernel_version} | sed "s/.%{_target_cpu}$//")
 %define ktag %(echo %{krequires} | tr '-' '_' | sed -e 's/\.el[0-9_]*$//' | sed -e 's/\.\([a-zA-Z]\)/_\1/')
+%if "@ENABLE_UTI@" == "ON"
+%define enable_uti 1
+%else
+%define enable_uti 0
+%endif
 
 Name: mckernel
 Version: @MCKERNEL_VERSION@
@@ -77,9 +82,10 @@ pushd build
 	%{?cmake_libdir:-DCMAKE_INSTALL_LIBDIR=%{cmake_libdir}} \
 	%{?build_target:-DBUILD_TARGET=%{build_target}} \
 	%{?toolchain_file:-DCMAKE_TOOLCHAIN_FILE=%{toolchain_file}} \
-	-DENABLE_TOFU=ON -DENABLE_FUGAKU_HACKS=ON \
-	-DENABLE_KRM_WORKAROUND=OFF -DWITH_KRM=ON \
-	-DENABLE_FUGAKU_DEBUG=OFF -DENABLE_UTI=ON \
+	-DENABLE_TOFU=@ENABLE_TOFU@ -DENABLE_FUGAKU_HACKS=@ENABLE_FUGAKU_HACKS@ \
+	-DENABLE_KRM_WORKAROUND=@ENABLE_KRM_WORKAROUND@ -DWITH_KRM=@WITH_KRM@ \
+	-DENABLE_FUGAKU_DEBUG=@ENABLE_FUGAKU_DEBUG@ -DENABLE_UTI=@ENABLE_UTI@ \
+	-DENABLE_FJMPI_WORKAROUND=@ENABLE_FJMPI_WORKAROUND@ \
 	..
 %make_build
 popd
@@ -112,6 +118,7 @@ popd
 %{_libdir}/libsched_yield.so.1.0.0
 %{_libdir}/libsched_yield.so
 %{_libdir}/libldump2mcdump.so
+%if 0%{?enable_uti}
 %{_libdir}/libmck_syscall_intercept.so
 %{_libdir}/libsyscall_intercept.so.0.1.0
 %{_libdir}/libsyscall_intercept.so.0
@@ -119,6 +126,7 @@ popd
 %{_libdir}/mck/libuti.so.1.0.0
 %{_libdir}/mck/libuti.so.1
 %{_libdir}/mck/libuti.so
+%endif
 %{_sysconfdir}/irqbalance_mck.in
 %{_mandir}/man1/mcreboot.1.gz
 %{_mandir}/man1/ihkconfig.1.gz

test/issues/1507+1519/README

@@ -7,7 +7,7 @@ xpmemやshmobjを利用するライブラリの初期化および基本動作を
 
 □実行手順
 1.
-SSMで以下のとおりサンプルプログラムをビルドする。
+ログインノードで以下のとおりサンプルプログラムをビルドする。
 必要に応じて PATHを設定すること。
 
 tradモード:
@@ -18,7 +18,7 @@ $ mpifccpx -DMPI -DOPENMP -Kopenmp mpi+affinity.c -o mpi+llvmopenmp-affinity
 
 2.
 インタラクティブモードでジョブ実行を開始する。
-SSMで以下のように pjsubコマンドを発行する。
+ログインノードで以下のように pjsubコマンドを発行する。
 rscunitやrscgrp、jobenvは環境に応じて指定すること。
 
 $ pjsub --interact -L "rscunit=xxx,rscgrp=xxx,jobenv=xxx,node=1" --mpi "proc=2" --sparam wait-time=1000

test/uti/tofu/README

@@ -54,7 +54,8 @@ sudo ./ctrl 1 1 1 0 0 0 1 1 1
 
 (2) uti_perf
 
-progress threadを用いたtofu get通信速度を計測する。
+progress threadを用いたtofu get通信速度を測定する。
+測定結果は 10 ns 単位で出力される。
 オプションは以下のとおり。
 
 -a <x>,<y>,<z>,<a>,<b>,<c>
@@ -80,7 +81,7 @@ progress threadを用いたtofu get通信速度を計測する。
 
 -l <length>
   tofu get通信するデータサイズ(byte)を指定する。
-  (64 byte ～ 16*1024*1024 - 256 byte、既定値は 16*1024*1024 - 256 byte)
+  (16 Kbyte ～ 16 Mbyte - 256 byte、既定値は 16 Mbyte - 256 byte)
 
 -v
   デバッグ出力を有効にする。
@@ -110,3 +111,13 @@ progress threadを用いたtofu get通信速度を計測する。
 
 --recvusleep=<us>
   progress threadの受信完了を確認する間隔 (usec) を指定する。 (既定値は0)
+
+例えば、tofu座標が 0,0,0,0,0,0 (受信側) と 0,0,0,1,0,0 (送信側) の間で
+24プロセス生成して 16 Kbyte のデータを送信する測定を 100回繰り返す場合は
+次のように指定する。
+
+(受信側)
+./uti_perf -a 0,0,0,1,0,0 -n 100 -f 24 -l 16384 -r
+(送信側)
+./uti_perf -a 0,0,0,0,0,0 -n 100 -f 24 -l 16384
+