spec: prerelase 0.93 for testing ihk_create_os_str

Change-Id: Id31646c88da0640a3d58e7805fa61f0e0583ff1c

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "ihk"]
 	path = ihk
 	url = https://github.com/RIKEN-SysSoft/ihk.git
+[submodule "executer/user/lib/libdwarf/libdwarf"]
+	path = executer/user/lib/libdwarf/libdwarf
+	url = https://github.com/bgerofi/libdwarf.git

CMakeLists.txt

@@ -10,7 +10,7 @@ project(mckernel C ASM)
 set(MCKERNEL_VERSION "1.7.0")
 
 # See "Fedora Packaging Guidlines -- Versioning"
-set(MCKERNEL_RELEASE "0.2")
+set(MCKERNEL_RELEASE "0.93")
 
 set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake/modules)
 # for rpmbuild
@@ -34,6 +34,7 @@ include(GNUInstallDirs)
 include(CMakeParseArguments)
 include(Kbuild)
 include(CheckCCompilerFlag)
+include(AutoconfHelper)
 
 CHECK_C_COMPILER_FLAG(-Wno-implicit-fallthrough IMPLICIT_FALLTHROUGH)
 if(IMPLICIT_FALLTHROUGH)
@@ -119,9 +120,36 @@ find_package(PkgConfig REQUIRED)
 set(PKG_CONFIG_USE_CMAKE_PREFIX_PATH ON)
 
 find_library(LIBRT rt)
+if (NOT LIBRT)
+	message(FATAL_ERROR "error: couldn't find librt")
+endif()
 find_library(LIBNUMA numa)
+if (NOT LIBNUMA)
+	message(FATAL_ERROR "error: couldn't find libnuma")
+endif()
 find_library(LIBBFD bfd)
+if (NOT LIBBFD)
+	message(FATAL_ERROR "error: couldn't find libbfd")
+endif()
 find_library(LIBIBERTY iberty)
+if (NOT LIBIBERTY)
+	message(FATAL_ERROR "error: couldn't find libiberty")
+endif()
+
+find_library(LIBDWARF dwarf)
+
+if (NOT LIBDWARF)
+	if (CMAKE_CROSSCOMPILING)
+		message(FATAL_ERROR "Could not find libdwarf.so, install libdwarf-devel to ${CMAKE_FIND_ROOT_PATH}")
+	endif()
+	message("WARNING: libdwarf will be compiled locally")
+	enable_language(CXX)
+else()
+	# Note that libdwarf-devel provides /usr/include/libdwarf/dwarf.h
+	# but elfutils-devel provides /usr/include/dwarf.h
+	# while mcinspect.c performs "#include <dwarf.h>"
+	find_path(DWARF_H dwarf.h PATH_SUFFIXES libdwarf)
+endif()
 
 if (ENABLE_QLMPI)
 	find_package(MPI REQUIRED)
@@ -186,10 +214,14 @@ add_subdirectory(tools/crash)
 configure_file(scripts/mcreboot-smp.sh.in mcreboot.sh @ONLY)
 configure_file(scripts/mcstop+release-smp.sh.in mcstop+release.sh @ONLY)
 configure_file(scripts/mcreboot.1in mcreboot.1 @ONLY)
+configure_file(scripts/eclair-dump-backtrace.exp.in eclair-dump-backtrace.exp @ONLY)
 install(PROGRAMS
 		"${CMAKE_CURRENT_BINARY_DIR}/mcreboot.sh"
 		"${CMAKE_CURRENT_BINARY_DIR}/mcstop+release.sh"
 	DESTINATION "${CMAKE_INSTALL_SBINDIR}")
+install(PROGRAMS
+		"${CMAKE_CURRENT_BINARY_DIR}/eclair-dump-backtrace.exp"
+	DESTINATION "${CMAKE_INSTALL_BINDIR}")
 install(FILES "scripts/irqbalance_mck.in"
 	DESTINATION "${CMAKE_INSTALL_SYSCONFDIR}")
 install(FILES "${CMAKE_CURRENT_BINARY_DIR}/mcreboot.1"

NEWS.md

@@ -1,534 +0,0 @@
-=============================================
-What's new in version 1.7.0rc4 (Apr 15, 2020)
-=============================================
-
-----------------------
-McKernel major updates
-----------------------
-1. arm64: Contiguous PTE support
-2. arm64: Scalable Vector Extension (SVE) support
-3. arm64: PMU overflow interrupt support
-4. xpmem: Support large page attachment
-5. arm64 port: Direct access to Mckernel memory from Linux
-6. arm64 port: utility thread offloading, which spawns thread onto
-   Linux CPU
-7. eclair: support for live debug
-8. Crash utility extension
-9. Replace mcoverlayfs with a soft userspace overlay
-10. Build system is switched to cmake
-11. Core dump includes thread information
-
-------------------------
-McKernel major bug fixes
-------------------------
-1. shmobj: Fix rusage counting for large page
-2. mcctrl control: task start_time changed to u64 nsec
-3. mcctrl: add handling for one more level of page tables
-4. Add kernel argument to turn on/off time sharing
-5. flatten_string/process env: realign env and clear trailing bits
-6. madvise: Add MADV_HUGEPAGE support
-8. mcctrl: remove in-kernel calls to syscalls
-9. arch_cpu_read_write_register: error return fix.
-10. set_cputime(): interrupt enable/disable fix.
-11. set_mempolicy(): Add mode check.
-12. mbind(): Fix memory_range_lock deadlock.
-13. ihk_ikc_recv: Record channel to packet for release
-14. Add set_cputime() kernel to kernel case and mode enum.
-15. execve: Call preempt_enable() before error-exit
-16. memory/x86_64: fix linux safe_kernel_map
-17. do_kill(): fix pids table when nr of threads is larger than num_processors
-18. shmget: Use transparent huge pages when page size isn't specified
-19. prctl: Add support for PR_SET_THP_DISABLE and PR_GET_THP_DISABLE
-20. monitor_init: fix undetected hang on highest numbered core
-21. init_process_stack: change premapped stack size based on arch
-22. x86 syscalls: add a bunch of XXat() delegated syscalls
-23. do_pageout: fix direct kernel-user access
-24. stack: add hwcap auxval
-25. perf counters: add arch-specific perf counters
-26. Added check of nohost to terminate_host().
-27. kmalloc: Fix address order in free list
-28. sysfs: use nr_cpu_ids for cpumasks (fixes libnuma parsing error on ARM)
-29. monitor_init: Use ihk_mc_cpu_info()
-30. Fix ThunderX2 write-combined PTE flag insanity
-31. ARM: eliminate zero page mapping (i.e, init_low_area())
-32. eliminate futex_cmpxchg_enabled check (not used and dereffed a NULL pointer)
-33. page_table: Fix return value of lookup_pte when ptl4 is blank
-34. sysfs: add missing symlinks for cpu/node
-35. Make Linux handler run when mmap to procfs.
-36. Separate mmap area from program loading (relocation) area
-37. move rusage into kernel ELF image (avoid dynamic alloc before NUMA init)
-38. arm: turn off cpu on panic
-39. page fault handler: protect thread accesses
-40. Register PPD and release_handler at the same time.
-41. fix to missing exclusive processing between terminate() and
-    finalize_process().
-42. perfctr_stop: add flags to no 'disable_intens'
-43. fileobj, shmobj: free pages in object destructor (as opposed to page_unmap())
-44. clear_range_l1, clear_range_middle: Fix handling contiguous PTE
-45. do_mmap: don't pre-populate the whole file when asked for smaller segment
-46. invalidate_one_page: Support shmobj and contiguous PTE
-47. ubsan: fix undefined shifts
-48. x86: disable zero mapping and add a boot pt for ap trampoline
-49. rusage: Don't count PF_PATCH change
-50. Fixed time processing.
-51. copy_user_pte: vmap area not owned by McKernel
-52. gencore: Zero-clear ELF header and memory range table
-53. rpm: ignore CMakeCache.txt in dist and relax BuildRequires on cross build
-54. gencore: Allocate ELF header to heap instead of stack
-55. nanosleep: add cpu_pause() in spinwait loop
-56. init_process: add missing initializations to proc struct
-57. rus_vm_fault: always use a packet on the stack
-58. process stack: use PAGE_SIZE in aux vector
-59. copy_user_pte: base memobj copy on range & VR_PRIVATE
-60. arm64: ptrace: Fix overwriting 1st argument with return value
-61. page fault: use cow for private device mappings
-62. reproductible builds: remove most install paths in c code
-63. page fault: clear writable bit for non-dirtying access to shared ranges
-64. mcreboot/mcstop+release: support for regular user execution
-65. irqbalance_mck: replace extra service with service drop-in
-66. do_mmap: give addr argument a chance even if not MAP_FIXED
-67. x86: fix xchg() and cmpxchg() macros
-68. IHK: support for using Linux work IRQ as IKC interrupt (optional)
-69. MCS: fix ARM64 issue by using smp_XXX() functions (i.e., barrier()s)
-70. procfs: add number of threads to stat and status
-71. memory_range_lock: Fix deadlock in procfs/sysfs handler
-72. flush instruction cache at context switch time if necessary
-73. arm64: Fix PMU related functions
-74. page_fault_process_memory_range: Disable COW for VM region with zeroobj
-75. extend_process_region: Fall back to demand paging when not contiguous
-76. munmap: fix deadlock with remote pagefault on vm range lock
-77. procfs: if memory_range_lock fails, process later
-78. migrate-cpu: Prevent migration target from calling schedule() twice
-79. sched_request_migrate(): fix race condition between migration req and IRQs
-80. get_one_cpu_topology: Renumber core_id (physical core id)
-81. bb7e140 procfs cpuinfo: use sequence number as processor
-82. set_host_vma(): do NOT read protect Linux VMA
-
-===========================================
-What's new in V1.6.0 (Nov 11, 2018)
-===========================================
-
------------------------------------------------
-McKernel new features, improvements and changes
------------------------------------------------
-1. McKernel and Linux share one unified kernel virtual address space.
-   That is, McKernel sections resides in Linux sections spared for
-   modules.  In this way, Linux can access the McKernel kernel memory
-   area.
-2. hugetlbfs support
-3. IHK is now included as a git submodule
-4. Debug messages are turned on/off in per souce file basis at run-time.
-5. It's prohibited for McKernel to access physical memory ranges which
-   Linux didn't give to McKernel.
-6. UTI (capability to spawn a thread on Linux CPU) improvement:
-   * System calls issued from the thread are hooked by modifying
-     binary in memory.
-
----------------------------
-McKernel bug fixes (digest)
----------------------------
-#<num> below corresponds to the redmine issue number
-(https://postpeta.pccluster.org/redmine/).
-
-1. #926: shmget: Hide object with IPC_RMID from shmget
-2. #1028: init_process: Inherit parent cpu_set
-3. #995: Fix shebang recorded in argv[0]
-4. #1024: Fix VMAP virtual address leak
-5. #1109: init_process_stack: Support "ulimit -s unlimited"
-6. x86 mem init: do not map identity mapping
-7. mcexec_wait_syscall: requeue potential request on interrupted wait
-8. mcctrl_ikc_send_wait: fix interrupt with do_frees == NULL
-9. pager_req_read: handle short read
-10. kprintf: only call eventfd() if it is safe to interrupt
-11. process_procfs_request: Add Pid to /proc/<PID>/status
-12. terminate: fix oversubscribe hang when waiting for other threads on same CPU to die
-13. mcexec: Do not close fd returned to mckernel side
-14. #976: execve: Clear sigaltstack and fp_regs
-15. #1002: perf_event: Specify counter by bit_mask on start/stop
-16. #1027: schedule: Don't reschedule immediately when wake up on migrate
-17. #mcctrl: lookup unexported symbols at runtime
-18. __sched_wakeup_thread: Notify interrupt_exit() of re-schedule
-19. futex_wait_queue_me: Spin-sleep when timeout and idle_halt is specified
-20. #1167: ihk_os_getperfevent,setperfevent: Timeout IKC sent by mcctrl
-21. devobj: fix object size (POSTK_DEBUG_TEMP_FIX_36)
-22. mcctrl: remove rus page cache
-23. #1021: procfs: Support multiple reads of e.g. /proc/*/maps
-24. #1006: wait: Delay wake-up parent within switch context
-25. #1164: mem: Check if phys-mem is within the range of McKernel memory
-26. #1039: page_fault_process_memory_range: Remove ihk_mc_map_virtual for CoW of device map
-27. partitioned execution: pass process rank to LWK
-28. process/vm: implement access_ok()
-29. spinlock: rewrite spinlock to use Linux ticket head/tail format
-30. #986: Fix deadlock involving mmap_sem and memory_range_lock
-31. Prevent one CPU from getting chosen by concurrent forks
-32. #1009: check_signal: system call restart is done only once
-33. #1176: syscall: the signal received during system call processing is not processed.
-34. #1036 syscall_time: Handle by McKernel
-35. #1165 do_syscall: Delegate system calls to the mcexec with the same pid
-36. #1194 execve: Fix calling ptrace_report_signal after preemption is disabled
-37. #1005 coredump: Exclude special areas
-38. #1018 procfs: Fix pread/pwrite to procfs fail when specified size is bigger than 4MB
-39. #1180 sched_setaffinity: Check migration after decrementing in_interrupt
-40. #771, #1179, #1143 ptrace supports threads
-41. #1189 procfs/do_fork: wait until procfs entries are registered
-42. #1114 procfs: add '/proc/pid/stat' to mckernel side and fix its comm
-43. #1116 mcctrl procfs: check entry was returned before using it
-44. #1167 ihk_os_getperfevent,setperfevent: Return -ETIME when IKC timeouts
-45. mcexec/execve: fix shebangs handling
-46. procfs: handle 'comm' on mckernel side
-47. ihk_os_setperfevent: Return number of registered events
-48. mcexec: fix terminating zero after readlink()
-
-===========================================
-What's new in V1.5.1 (July 9, 2018)
-===========================================
-
------------------------------------------------
-McKernel new features, improvements and changes
------------------------------------------------
-1. Watchdog timer to detect hang of McKernel
-   mcexec prints out the following line to its stderr when a hang of
-   McKernel is detected.
-
-       mcexec detected hang of McKernel
-
-   The watchdog timer is enabled by passing -i <timeout_in_sec> option
-   to mcreboot.sh. <timeout_in_sec> specifies the interval of checking
-   if McKernel is alive.
-   Example: mcreboot.sh -i 600: Detect the hang with 10 minutes interval
-
-   The detailed step of the hang detection is as follows.
-   (1) mcexec acquires eventfd for notification from IHK and perform
-       epoll() on it.
-   (2) A daemon called ihkmond monitors the state of McKernel periodically
-       with the interval specified by the -i option. It judges that
-       McKernel is hanging and notifies mcexec by the eventfd if its
-       state hasn't changed since the last check.
-
-2. Documentation
-   man page: Installed directory is changed to <install_dir>/share/man
-
----------------------------
-McKernel bug fixes (digest)
----------------------------
-1.  #1146: pager_req_map(): do not take mmap_sem if not needed
-2.  #1135: prepare_process_ranges_args_envs(): fix saving cmdline
-3.  #1144: fileobj/devobj: record path name
-4.  #1145: fileobj: use MCS locks for per-file page hash
-5.  #1076: mcctrl: refactor prepare_image into new generic ikc send&wait
-6.  #1072: execve: fix execve with oversubscribing
-7.  #1132: execve: use thread variable instead of cpu_local_var(current)
-8.  #1117: mprotect: do not set page table writable for cow pages
-9.  #1143: syscall wait4: add _WALL (POSTK_DEBUG_ARCH_DEP_44)
-10. #1064: rusage: Fix initialization of rusage->num_processors
-11. #1133: pager_req_unmap: Put per-process data at exit
-12. #731: do_fork: Propagate error code returned by mcexec
-13. #1149: execve: Reinitialize vm_regions's map area on execve
-14. #1065: procfs: Show file names in /proc/<PID>/maps
-15. #1112: mremap: Fix type of size arguments (from ssize_t to size_t)
-16. #1121: sched_getaffinity: Check arguments in the same order as in Linux
-17. #1137: mmap, mremap: Check arguments in the same order as in Linux
-18. #1122: fix return value of sched_getaffinity
-19. #732: fix: /proc/<PID>/maps outputs a unnecessary NULL character
-
-===================================
-What's new in V1.5.0 (Apr 5, 2018)
-===================================
-
---------------------------------------
-McKernel new features and improvements
---------------------------------------
-1. Aid for Linux version migration: Detect /proc, /sys format change
-   between two kernel verions
-2. Swap out
-   * Only swap-out anonymous pages for now
-3. Improve support of /proc/maps
-4. mcstat: Linux tool to show resource usage
-
----------------------------
-McKernel bug fixes (digest)
----------------------------
-1. #727: execve: Fix memory leak when receiving SIGKILL
-2. #829: perf_event_open: Support PERF_TYPE_HARDWARE and PERF_TYPE_HW_CACHE
-3. #906: mcexec: Check return code of fork()
-4. #1038: mcexec: Timeout when incorrect value is given to -n option
-5. #943 #945 #946 #960 $961: mcexec: Support strace
-6. #1029: struct thread is not released with stress-test involving signal
-          and futex
-7. #863 #870: Respond immediately to terminating signal when
-              offloading system call
-8. #1119: translate_rva_to_rpa(): use 2MB blocks in 1GB pages on x86
-11. #898: Shutdown OS only after no in-flight IKC exist
-12. #882: release_handler: Destroy objects as the process which opened it
-13. #882: mcexec: Make child process exit if the parent is killed during
-          fork()
-14. #925: XPMEM: Don't destroy per-process object of the parent
-15. #885: ptrace: Support the case where a process attaches its child
-16. #1031: sigaction: Support SA_RESETHAND
-17. #923: rus_vm_fault: Return error when a thread not performing
-          system call offloading causes remote page fault
-18. #1032 #1033 #1034: getrusage: Fix ru_maxrss, RUSAGE_CHILDREN,
-                       ru_stime related bugs
-19. #1120: getrusage: Fix deadlock on thread->times_update
-20. #1123: Fix deadlock related to wait_queue_head_list_node
-21. #1124: Fix deadlock of calling terminate() from terminate()
-22. #1125: Fix deadlock related to thread status
-    * Related functions are: hold_thread(), do_kill() and terminate()
-23. #1126: uti: Fix uti thread on the McKernel side blocks others in do_syscall()
-24. #1066: procfs: Show Linux /proc/self/cgroup
-25. #1127: prepare_process_ranges_args_envs(): fix generating saved_cmdline to
-           avoid PF in strlen()
-26. #1128: ihk_mc_map/unmap_virtual(): do proper TLB invalidation
-27. #1043: terminate(): fix update_lock and threads_lock order to avoid deadlock
-28. #1129: mcreboot.sh: Save /proc/irq/*/smp_affinity to /tmp/mcreboot
-29. #1130: mcexec: drop READ_IMPLIES_EXEC from personality
-
---------------------
-McKernel workarounds
---------------------
-1. Forbid CPU oversubscription
-   * It can be turned on by mcreboot.sh -O option
-
-
-===================================
-What's new in V1.4.0 (Oct 30, 2017)
-===================================
-
------------------------------------------------------------
-Feature: Abstracted event type support in perf_event_open()
------------------------------------------------------------
-PERF_TYPE_HARDWARE and PERF_TYPE_CACHE types are supported.
-
-----------------------------------
-Clean-up: Direct user-space access
-----------------------------------
-Code lines using direct user-space access (e.g. passing user-space
-pointer to memcpy()) becomes more portable across processor
-architectures. The modification follows the following rules.
-
-1. Move the code section as it is to the architecture dependent
-   directory if it is a part of the critical-path.
-2. Otherwise, rewrite the code section by using the portable methods.
-   The methods include copy_from_user(), copy_to_user(),
-   pte_get_phys() and phys_to_virt().
-
---------------------------------
-Test: MPI and OpenMP micro-bench
---------------------------------
-The performance figures of MPI and OpenMP primitives are compared with
-those of Linux by using Intel MPI Benchmarks and EPCC OpenMP Micro
-Benchmark.
-
-
-===================================
-What's new in V1.3.0 (Sep 30, 2017)
-===================================
-
---------------------
-Feature: Kernel dump
---------------------
-1. A dump level of "only kernel memory" is added.
-
-The following two levels are available now:
-   0: Dump all
-  24: Dump only kernel memory
-
-The dump level can be set by -d option in ihkosctl or the argument
-for ihk_os_makedumpfile(), as shown in the following examples:
-
-   Command:		ihkosctl 0 dump -d 24
-   Function call:	ihk_os_makedumpfile(0, NULL, 24, 0);
-
-2. Dump file is created when Linux panics.
-
-The dump level can be set by dump_level kernel argument, as shown in the
-following example:
-
-   ihkosctl 0 kargs "hidos dump_level=24"
-
-The IHK dump function is registered to panic_notifier_list when creating
-/dev/mcdX and called when Linux panics.
-
------------------------------
-Feature: Quick Process Launch
------------------------------
-
-MPI process launch time and some of the initialization time can be
-reduced in application consisting of multiple MPI programs which are
-launched in turn in the job script.
-
-The following two steps should be performed to use this feature:
-1. Replace mpiexec with ql_mpiexec_start and add some lines for
-   ql_mpiexec_finalize in the job script
-2. Modify the app so that it can repeat calculations and wait for the
-   instructions from ql_mpiexec_{start,finalize} at the end of the
-   loop
-
-The first step is explained using an example. Assume the original job
-script looks like this:
-
-/* Execute ensamble simulation and then data assimilation, and repeat this
-   ten times */
-for i in {1..10}; do
-
-     /* Each ensamble simulation execution uses 100 nodes, launch ten of them
-        in parallel */
-    for j in {1..10}; do
-         mpiexec -n 100 -machinefile ./list1_$j p1.out a1 & pids[$i]=$!;
-    done
-
-    /* Wait until the ten ensamble simulation programs finish */
-    for j in {1..10}; do wait ${pids[$j]}; done
-
-    /* Launch one data assimilation program using 1000 nodes */
-    mpiexec -n 1000 -machinefile ./list2 p2.out a2
-done
-
-The job script should be modified like this:
-
-for i in {1..10}; do
-    for j in {1..10}; do
-        /*  Replace mpiexec with ql_mpiexec_start */
-        ql_mpiexec_start -n 100 -machinefile ./list1_$j p1.out a1 & pids[$j]=$!;
-    done
-
-    for j in {1..10}; do wait ${pids[$j]}; done
-
-    ql_mpiexec_start -n 1000 -machinefile ./list2 p2.out a2
-done
-
-/* p1.out and p2.out don't exit but are waiting for the next calculation.
-   So tell them to exit */
-for j in {1..10}; do
-   ql_mpiexec_finalize -machinefile ./list1_$i p1.out a1;
-done
-ql_mpiexec_finalize -machinefile ./list2 p2.out a2;
-
-
-The second step is explained using a pseudo-code.
-
-    MPI_Init();
-    Prepare data exchange with preceding / following MPI programs
-loop:
-    foreach Fortran module
-        Initialize data using command-line argments, parameter files,
-	environment variables
-    Input data from preceding MPI programs / Read snap-shot
-    Perform main calculation
-    Output data to following MPI programs / Write snap-shot
-    /* ql_client() waits for command of ql_mpiexec_{start,finish} */
-    if (ql_client() == QL_CONTINUE) { goto loop; }
-    MPI_Finalize();
-
-qlmpilib.h should be included in the code and libql{mpi,fort}.so
-should be linked to the executable file.
-
-
-========================
-Restrictions on McKernel
-========================
-
- 1. Pseudo devices such as /dev/mem and /dev/zero are not mmap()ed
-    correctly even if the mmap() returns a success. An access of their
-    mapping receives the SIGSEGV signal.
-
- 2. clone() supports only the following flags. All the other flags
-    cause clone() to return error or are simply ignored.
-
-    * CLONE_CHILD_CLEARTID
-    * CLONE_CHILD_SETTID
-    * CLONE_PARENT_SETTID
-    * CLONE_SETTLS
-    * CLONE_SIGHAND
-    * CLONE_VM
-
- 3. PAPI has the following restriction.
-
-    * Number of counters a user can use at the same time is up to the
-      number of the physical counters in the processor.
-
- 4. msync writes back only the modified pages mapped by the calling process.
-
- 5. The following syscalls always return the ENOSYS error.
-
-    * migrate_pages()
-    * move_pages()
-    * set_robust_list()
-
- 6. The following syscalls always return the EOPNOTSUPP error.
-
-    * arch_prctl(ARCH_SET_GS)
-    * signalfd()
-
- 7. signalfd4() returns a fd, but signal is not notified through the
-    fd.
-
- 8. set_rlimit sets the limit values but they are not enforced.
-
- 9. Address randomization is not supported.
-
-10. brk() extends the heap more than requestd when -h
-    (--extend-heap-by=)<step> option of mcexec is used with the value
-    larger than 4 KiB.  syscall_pwrite02 of LTP would fail for this
-    reason. This is because the test expects that the end of the heap
-    is set to the same address as the argument of sbrk() and expects a
-    segmentation violation occurs when it tries to access the memory
-    area right next to the boundary. However, the optimization sets
-    the end to a value larger than the requested. Therefore, the
-    expected segmentation violation doesn't occur.
-
-11. setpriority()/getpriority() won't work. They might set/get the
-    priority of a random mcexec thread. This is because there's no
-    fixed correspondence between a McKernel thread which issues the
-    system call and a mcexec thread which handles the offload request.
-
-12. mbind() can set the policy but it is not used when allocating
-    physical pages.
-
-13. MPOL_F_RELATIVE_NODES and MPOL_INTERLEAVE flags for
-    set_mempolicy()/mbind() are not supported.
-
-14. The MPOL_BIND policy for set_mempolicy()/mbind() works as the same
-    as the MPOL_PREFERRED policy. That is, the physical page allocator
-    doesn't give up the allocation when the specified nodes are
-    running out of pages but continues to search pages in the other
-    nodes.
-
-15. Kernel dump on Linux panic requires Linux kernel CentOS-7.4 and
-    later. In addition, crash_kexec_post_notifiers kernel argument
-    must be given to Linux kernel.
-
-16. setfsuid()/setfsgid() cannot change the id of the calling thread.
-    Instead, it changes that of the mcexec worker thread which takes
-    the system-call offload request.
-
-17. mmap (hugeTLBfs): The physical pages corresponding to a map are
-    released when no McKernel process exist. The next map gets fresh
-    physical pages.
-
-18. Sticky bit on executable file has no effect.
-
-19. Linux (RHEL-7 for x86_64) could hang when offlining CPUs in the
-    process of booting McKernel due to the Linux bug, found in
-    Linux-3.10 and fixed in the later version.  One way to circumvent
-    this is to always assign the same CPU set to McKernel.
-
-20. madvise:
-    * MADV_HWPOISON and MADV_SOFT_OFFLINE always returns -EPERM.
-    * MADV_MERGEABLE and MADV_UNMERGEABLE always returns -EINVAL.
-    * MADV_HUGEPAGE and MADV_NOHUGEPAGE on file map returns -EINVAL
-      (It succeeds on RHEL-8 for aarch64).
-
-21. brk() and mmap() doesn't report out-of-memory through its return
-    value. Instead, page-fault reports the error.
-
-22. Anonymous mmap pre-maps requested number of pages when contiguous
-    pages are available. Demand paging is used when not available.
-
-23. Mixing page sizes in anonymous shared mapping is not allowed. mmap
-    creates vm_range with one page size. And munmap or mremap that
-    needs the reduced page size changes the sizes of all the pages of
-    the vm_range.
-
-24. ihk_os_getperfevent() could time-out when invoked from Fujitsu TCS
-    (job-scheduler).

README.md

@@ -1,285 +0,0 @@
-![McKernel Logo](https://www.sys.r-ccs.riken.jp/members_files/bgerofi/mckernel-logo.png)
--------------------------
-
-IHK/McKernel is a light-weight multi-kernel operating system designed for high-end supercomputing. It runs Linux and McKernel, a light-weight kernel (LWK), side-by-side inside compute nodes and aims at the following:
-
-- Provide scalable and consistent execution of large-scale parallel scientific applications, but at the same time maintain the ability to rapidly adapt to new hardware features and emerging programming models
-- Provide efficient memory and device management so that resource contention and data movement are minimized at the system level
-- Eliminate OS noise by isolating OS services in Linux and provide jitter free execution on the LWK
-- Support the full POSIX/Linux APIs by selectively offloading (slow-path) system calls to Linux
-
-## Contents
-
-- [Background](#background-and-motivation)
-- [Architectural Overview](#architectural-overview)
-- [Installation](#installation)
-- [The Team](#the-team)
-
-## Background and Motivation
-
-With the growing complexity of high-end supercomputers, the current system software stack faces significant challenges as we move forward to exascale and beyond. The necessity to deal with extreme degree of parallelism, heterogeneous architectures, multiple levels of memory hierarchy, power constraints, etc., advocates operating systems that can rapidly adapt to new hardware requirements, and that can support novel programming paradigms and runtime systems. On the other hand, a new class of more dynamic and complex applications are also on the horizon, with an increasing demand for application constructs such as in-situ analysis, workflows, elaborate monitoring and performance tools. This complexity relies not only on the rich features of POSIX, but also on the Linux APIs (such as the */proc*, */sys* filesystems, etc.) in particular.
-
-
-##### Two Traditional HPC OS Approaches
-
-Traditionally, light-weight operating systems specialized for HPC followed two approaches to tackle scalable execution of large-scale applications. In the full weight kernel (FWK) approach, a full Linux environment is taken as the basis, and features that inhibit attaining HPC scalability are removed, i.e., making it light-weight. The pure light-weight kernel (LWK) approach, on the other hand, starts from scratch and effort is undertaken to add sufficient functionality so that it provides a familiar API, typically something close to that of a general purpose OS, while at the same time it retains the desired scalability and reliability attributes. Neither of these approaches yields a fully Linux compatible environment.
-
-
-##### The Multi-kernel Approach
-
-A hybrid approach recognized recently by the system software community is to run Linux simultaneously with a lightweight kernel on compute nodes and multiple research projects are now pursuing this direction. The basic idea is that simulations run on an HPC tailored lightweight kernel, ensuring the necessary isolation for noiseless execution of parallel applications, but Linux is leveraged so that the full POSIX API is supported. Additionally, the small code base of the LWK can also facilitate rapid prototyping for new, exotic hardware features. Nevertheless, the questions of how to share node resources between the two types of kernels, where do device drivers execute, how exactly do the two kernels interact with each other and to what extent are they integrated, remain subjects of ongoing debate.
-
-
-
-## Architectural Overview
-
-At the heart of the stack is a low-level software infrastructure called Interface for Heterogeneous Kernels (IHK). IHK is a general framework that provides capabilities for partitioning resources in a many-core environment (e.g.,CPU cores and physical memory) and it enables management of lightweight kernels. IHK can allocate and release host resources dynamically and no reboot of the host machine is required when altering configuration. IHK also provides a low-level inter-kernel messaging infrastructure, called the Inter-Kernel Communication (IKC) layer. An architectural overview of the main system components is shown below.
-
-
-![arch](https://www.sys.r-ccs.riken.jp/members_files/bgerofi/mckernel.png)
-
-
-McKernel is a lightweight kernel written from scratch. It is designed for HPC and is booted from IHK. McKernel retains a binary compatible ABI with Linux, however, it implements only a small set of performance sensitive system calls and the rest are offloaded to Linux. Specifically, McKernel has its own memory management, it supports processes and multi-threading with a simple round-robin cooperative (tick-less) scheduler, and it implements signaling. It also allows inter-process memory mappings and it provides interfaces to hardware performance counters.
-
-### Functionality
-
-An overview of some of the principal functionalities of the IHK/McKernel stack is provided below.
-
-#### System Call Offloading
-
-System call forwarding in McKernel is implemented as follows. When an offloaded system call occurs, McKernel marshals the system call number along with its arguments and sends a message to Linux via a dedicated IKC channel. The corresponding proxy process running on Linux is by default waiting for system call requests through an ioctl() call into IHK’s system call delegator kernel module. The delegator kernel module’s IKC interrupt handler wakes up the proxy process, which returns to userspace and simply invokes the requested system call. Once it obtains the return value, it instructs the delegator module to send the result back to McKernel, which subsequently passes the value to user-space.
-
-#### Unified Address Space
-
-The unified address space model in IHK/McKernel ensures that offloaded system calls can seamlessly resolve arguments even in case of pointers. This mechanism is depicted below and is implemented as follows.
-
-
-![unified_ap](https://www.sys.r-ccs.riken.jp/members_files/bgerofi/img/unified_address_space_en.png)
-
-
-First, the proxy process is compiled as a position independent binary, which enables us to map the code and data segments specific to the proxy process to an address range which is explicitly excluded from McKernel’s user space. The grey box on the right side of the figure demonstrates the excluded region. Second, the entire valid virtual address range of McKernel’s application user-space is covered by a special mapping in the proxy process for which we use a pseudo file mapping in Linux. This mapping is indicated by the blue box on the left side of the figure.
-
-
-## Installation
-
-For a smooth experience, we recommend the following combination of OS distributions and platforms:
-
-- CentOS 7.3+ running on Intel Xeon / Xeon Phi
-
-
-##### 1. Change SELinux settings
-Log in as the root and disable SELinux:
-
-~~~~
-vim /etc/selinux/config
-~~~~
-
-Change the file to SELINUX=disabled
-
-##### 2. Reboot the host machine
-~~~~
-sudo reboot
-~~~~
-
-##### 3. Prepare packages, kernel symbol table file
-You will need the following packages installed:
-
-~~~~
-sudo yum install cmake kernel-devel binutils-devel systemd-devel numactl-devel gcc make nasm git
-~~~~
-
-Grant read permission to the System.map file of your kernel version:
-
-~~~~
-sudo chmod a+r /boot/System.map-`uname -r`
-~~~~
-
-##### 4. Obtain sources and compile the kernel
-
-Clone the source code:
-
-~~~~
-mkdir -p ~/src/ihk+mckernel/
-cd ~/src/ihk+mckernel/
-git clone --recursive -b development https://github.com/RIKEN-SysSoft/mckernel.git
-~~~~
-
-(Optional) Checkout to the specific branch or version:
-
-~~~~
-cd mckernel
-git checkout <pathspec>
-git submodule update
-~~~~
-
-Foe example, if you want to try the development branch, use "development" as the pathspec. If you want to try the prerelease version 1.7.0-0.2, use "1.7.0-0.2".
-
-###### 4.1 Install with cmake
-
-Configure and compile:
-
-~~~~
-mkdir -p build && cd build
-cmake -DCMAKE_INSTALL_PREFIX=${HOME}/ihk+mckernel $HOME/src/ihk+mckernel/mckernel
-make -j install
-~~~~
-
-The IHK kernel modules and McKernel kernel image should be installed under the **ihk+mckernel** folder in your home directory.
-
-###### 4.2 Install with rpm
-
-Configure, compile and build rpm:
-
-~~~~
-mkdir -p build && cd build
-cmake $HOME/src/ihk+mckernel/mckernel
-make dist
-cp mckernel-<version>.tar.gz <rpmbuild>/SOURCES
-rpm -ba scripts/mckernel.spec
-sudo rpm -ivh <rpmbuild>/RPMS/<arch>/mckernel-<version>-<release>_<linux_kernel_ver>_<dist>.<arch>.rpm
-~~~~
-
-The IHK kernel modules and McKernel kernel image are installed under the system directory.
-
-##### 5. Boot McKernel
-
-A boot script called mcreboot.sh is provided under sbin in the install folder. To boot on logical CPU 1 with 512MB of memory, use the following invocation:
-
-~~~~
-export TOP=${HOME}/ihk+mckernel/
-cd ${TOP}
-sudo ./sbin/mcreboot.sh -c 1 -m 512m
-~~~~
-
-You should see something similar like this if you display the McKernel's kernel message log:
-
-
-~~~~
-./sbin/ihkosctl 0 kmsg
-
-IHK/McKernel started.
-[ -1]: no_execute_available: 1
-[ -1]: map_fixed: phys: 0xfee00000 => 0xffff860000009000 (1 pages)
-[ -1]: setup_x86 done.
-[ -1]: ns_per_tsc: 385
-[ -1]: KCommand Line: hidos    dump_level=24
-[ -1]: Physical memory: 0x1ad3000 - 0x21000000, 525520896 bytes, 128301 pages available @ NUMA: 0
-[ -1]: NUMA: 0, Linux NUMA: 0, type: 1, available bytes: 525520896, pages: 128301
-[ -1]: NUMA 0 distances: 0 (10),
-[ -1]: map_fixed: phys: 0x28000 => 0xffff86000000a000 (2 pages)
-[ -1]: Trampoline area: 0x28000
-[ -1]: map_fixed: phys: 0x0 => 0xffff86000000c000 (1 pages)
-[ -1]: # of cpus : 1
-[ -1]: locals = ffff880001af6000
-[  0]: BSP: 0 (HW ID: 1 @ NUMA 0)
-[  0]: BSP: booted 0 AP CPUs
-[  0]: Master channel init acked.
-[  0]: vdso is enabled
-IHK/McKernel booted.
-~~~~
-
-
-##### 5. Run a simple program on McKernel
-
-The mcexec command line tool (which is also the Linux proxy process) can be used for executing applications on McKernel:
-
-~~~~
-./bin/mcexec hostname
-centos-vm
-~~~~
-
-
-##### 6. Shutdown McKernel
-
-Finally, to shutdown McKernel and release CPU/memory resources back to Linux use the following command:
-
-~~~~
-sudo ./sbin/mcstop+release.sh
-~~~~
-
-##### 7. Advanced: Enable Utility Thread offloading Interface (UTI)
-
-UTI enables a runtime such as MPI runtime to spawn utility threads such as MPI asynchronous progress threads to Linux cores.
-
-1. Install capstone
-
-Install EPEL capstone-devel:
-
-~~~~
-sudo yum install epel-release
-sudo yum install capstone-devel
-~~~~
-
-2. Install syscall_intercept
-
-~~~~
-git clone https://github.com/RIKEN-SysSoft/syscall_intercept.git
-cmake ../arch/aarch64 -DCMAKE_INSTALL_PREFIX=<syscall-intercept-install> -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=gcc -DTREAT_WARNINGS_AS_ERRORS=OFF
-~~~~
-
-3. Install UTI for McKernel
-
-Install:
-
-~~~~
-git clone https://github.com/RIKEN-SysSoft/uti.git
-mkdir build && cd build
-../uti/configure --prefix=<mckernel-install> --with-rm=mckernel
-make && make install
-~~~~
-
-4. Install McKernel
-
-~~~~
-CMAKE_PREFIX_PATH=<syscall-intercept-install> cmake -DCMAKE_INSTALL_PREFIX=${HOME}/ihk+mckernel -DENABLE_UTI=ON $HOME/src/ihk+mckernel/mckernel
-~~~~
-
-5. Run executable
-
-~~~~
-mcexec --enable-uti <command>
-~~~~
-
-6. Install UTI for Linux for performance comparison
-
-Install by make:
-
-~~~~
-git clone https://github.com/RIKEN-SysSoft/uti.git
-mkdir build && cd build
-../uti/configure --prefix=<uti-install> --with-rm=linux
-make && make install
-~~~~
-
-Install by rpm:
-
-~~~~
-git clone https://github.com/RIKEN-SysSoft/uti.git
-mkdir build && cd build
-../uti/configure --prefix=<uti-install> --with-rm=linux
-rm -f ~/rpmbuild/SOURCES/<version>.tar.gz
-rpmbuild -ba ./scripts/uti.spec
-rpm -Uvh uti-<version>-<release>-<arch>.rpm
-~~~~
-
-## The Team
-
-The McKernel project was started at The University of Tokyo and currently it is mainly developed at RIKEN.
-Some of our collaborators include:
-
-- Hitachi
-- Fujitsu
-- CEA (France)
-- NEC
-
-
-## License
-
-McKernel is GPL licensed, as found in the LICENSE file.
-
-## Contact
-
-Please give your feedback to us via one of the following mailing lists. Subscription via [www.pccluster.org](http://www.pccluster.org/mailman/listinfo/mckernel-users) is needed.
-
-* English: mckernel-users@pccluster.org
-* Japanese: mckernel-users-jp@pccluster.org

README.rst

@@ -0,0 +1,23 @@
+.. figure:: docs/mckernel-logo.png
+
+IHK/McKernel is a light-weight multi-kernel operating system designed
+for high-end supercomputing. It runs Linux and McKernel, a light-weight
+kernel (LWK), side-by-side inside compute nodes and aims at the
+following:
+
+-  Provide scalable and consistent execution of large-scale parallel
+   scientific applications, but at the same time maintain the ability to
+   rapidly adapt to new hardware features and emerging programming
+   models
+-  Provide efficient memory and device management so that resource
+   contention and data movement are minimized at the system level
+-  Eliminate OS noise by isolating OS services in Linux and provide
+   jitter free execution on the LWK
+-  Support the full POSIX/Linux APIs by selectively offloading
+   (slow-path) system calls to Linux
+
+Documentation
+=============
+
+Documentation is available
+`here <https://ihkmckernel.readthedocs.io>`__.

arch/arm64/kernel/cpu.c

@@ -1497,6 +1497,19 @@ struct thread *arch_switch_context(struct thread *prev, struct thread *next)
 		}
 	}
 #endif /*ENABLE_PERF*/
+
+#ifdef PROFILE_ENABLE
+	if (prev && prev->profile && prev->profile_start_ts != 0) {
+		prev->profile_elapsed_ts +=
+			(rdtsc() - prev->profile_start_ts);
+		prev->profile_start_ts = 0;
+	}
+
+	if (next->profile && next->profile_start_ts == 0) {
+		next->profile_start_ts = rdtsc();
+	}
+#endif
+
 	if (likely(prev)) {
 		tls_thread_switch(prev, next);
 
@@ -1867,6 +1880,11 @@ int arch_cpu_read_write_register(
 		ret = -1;
 	}
 
+	dkprintf("%s: MCCTRL_OS_CPU_%s_REGISTER: reg: 0x%lx, val: 0x%lx\n",
+			__FUNCTION__,
+			(op == MCCTRL_OS_CPU_READ_REGISTER ? "READ" : "WRITE"),
+			desc->addr, desc->val);
+
 	return ret;
 }
 

arch/arm64/kernel/include/rlimit.h

@@ -85,7 +85,11 @@ enum __rlimit_resource
   __RLIMIT_RTPRIO = 14,
 #define RLIMIT_RTPRIO __RLIMIT_RTPRIO
 
-  __RLIMIT_NLIMITS = 15,
+  /* timeout for RT tasks in us */
+  __RLIMIT_RTTIME = 15,
+#define RLIMIT_RTTIME __RLIMIT_RTTIME
+
+  __RLIMIT_NLIMITS = 16,
   __RLIM_NLIMITS = __RLIMIT_NLIMITS
 #define RLIMIT_NLIMITS __RLIMIT_NLIMITS
 #define RLIM_NLIMITS __RLIM_NLIMITS

arch/arm64/kernel/include/syscall_list.h

@@ -118,6 +118,7 @@ SYSCALL_HANDLED(241, perf_event_open)
 SYSCALL_DELEGATED(241, perf_event_open)
 #endif // PERF_ENABLE
 SYSCALL_HANDLED(260, wait4)
+SYSCALL_HANDLED(261, prlimit64)
 SYSCALL_HANDLED(270, process_vm_readv)
 SYSCALL_HANDLED(271, process_vm_writev)
 SYSCALL_HANDLED(281, execveat)
@@ -143,3 +144,9 @@ SYSCALL_HANDLED(1045, signalfd)
 SYSCALL_DELEGATED(1049, stat)
 SYSCALL_DELEGATED(1060, getpgrp)
 SYSCALL_HANDLED(1062, time)
+SYSCALL_DELEGATED(1069, epoll_wait)
+
+/* Do not edit the lines including this comment and
+ * EOF just after it because those are used as a
+ * robust marker for the autotest patch.
+ */

arch/arm64/kernel/irq-gic-v3.c

@@ -310,6 +310,9 @@ void handle_interrupt_gicv3(struct pt_regs *regs)
 {
 	uint64_t irqnr;
 	const int from_user = interrupt_from_user(regs);
+	struct cpu_local_var *v = get_this_cpu_local_var();
+	//unsigned long irqflags;
+	int do_check = 0;
 
 	irqnr = gic_read_iar();
 	cpu_enable_nmi();
@@ -323,10 +326,18 @@ void handle_interrupt_gicv3(struct pt_regs *regs)
 	}
 	set_cputime(from_user ? CPUTIME_MODE_K2U : CPUTIME_MODE_K2K_OUT);
 
-	/* for migration by IPI */
-	if (get_this_cpu_local_var()->flags & CPU_FLAG_NEED_MIGRATE) {
-		schedule();
+	//irqflags = ihk_mc_spinlock_lock(&v->runq_lock);
+	/* For migration by IPI or by timesharing */
+	if (v->flags &
+	    (CPU_FLAG_NEED_MIGRATE | CPU_FLAG_NEED_RESCHED)) {
+		v->flags &= ~CPU_FLAG_NEED_RESCHED;
+		do_check = 1;
+	}
+	//ihk_mc_spinlock_unlock(&v->runq_lock, irqflags);
+
+	if (do_check) {
 		check_signal(0, regs, 0);
+		schedule();
 	}
 }
 

arch/arm64/kernel/mikc.c

@@ -19,7 +19,7 @@ int ihk_mc_ikc_init_first_local(struct ihk_ikc_channel_desc *channel,
 
 	memset(channel, 0, sizeof(struct ihk_ikc_channel_desc));
 
-	mikc_queue_pages = ((4 * num_processors * MASTER_IKCQ_PKTSIZE)
+	mikc_queue_pages = ((8 * num_processors * MASTER_IKCQ_PKTSIZE)
 			+ (PAGE_SIZE - 1)) / PAGE_SIZE;
 
 	/* Place both sides in this side */

arch/arm64/kernel/perfctr_armv8pmu.c

@@ -590,7 +590,7 @@ static inline void armv8pmu_write_counter(int idx, uint32_t value)
 		 * count using the lower 32bits and we want an interrupt when
 		 * it overflows.
 		 */
-		uint64_t value64 = 0xffffffff00000000ULL | value;
+		uint64_t value64 = (int32_t)value;
 
 		write_sysreg(value64, pmccntr_el0);
 	}

arch/arm64/kernel/syscall.c

@@ -15,7 +15,6 @@
 #include <limits.h>
 #include <uio.h>
 #include <syscall.h>
-#include <bitops.h>
 #include <rusage_private.h>
 #include <ihk/debug.h>
 
@@ -58,13 +57,34 @@ extern int num_processors;
 int obtain_clone_cpuid(cpu_set_t *cpu_set, int use_last)
 {
 	int min_queue_len = -1;
-	int cpu, min_cpu = -1, uti_cpu = -1;
-	unsigned long irqstate;
+	int cpu, min_cpu = -1;
+#if 0
+	int uti_cpu = -1;
+#endif
+	unsigned long irqstate = 0;
 
-	irqstate = ihk_mc_spinlock_lock(&runq_reservation_lock);
+	int start, end, step;
+
+	if (use_last) {
+		start = num_processors - 1;
+		end = -1;
+		step = -1;
+	}
+	else {
+		start = 0;
+		end = num_processors;
+		step = 1;
+	}
+
+	if (!cpu_local_var(current)->proc->nr_processes) {
+		irqstate = ihk_mc_spinlock_lock(&runq_reservation_lock);
+	}
+	else {
+		irqstate = cpu_disable_interrupt_save();
+	}
 
 	/* Find the first allowed core with the shortest run queue */
-	for (cpu = 0; cpu < num_processors; ++cpu) {
+	for (cpu = start; cpu != end; cpu += step) {
 		struct cpu_local_var *v;
 
 		if (!CPU_ISSET(cpu, cpu_set))
@@ -75,11 +95,14 @@ int obtain_clone_cpuid(cpu_set_t *cpu_set, int use_last)
 		dkprintf("%s: cpu=%d,runq_len=%d,runq_reserved=%d\n",
 			 __func__, cpu, v->runq_len, v->runq_reserved);
 		if (min_queue_len == -1 ||
-		    v->runq_len + v->runq_reserved < min_queue_len) {
-			min_queue_len = v->runq_len + v->runq_reserved;
+		    //v->runq_len + v->runq_reserved < min_queue_len) {
+		    v->runq_len < min_queue_len) {
+			//min_queue_len = v->runq_len + v->runq_reserved;
+			min_queue_len = v->runq_len;
 			min_cpu = cpu;
 		}
 
+#if 0
 		/* Record the last tie CPU */
 		if (min_cpu != cpu &&
 		    v->runq_len + v->runq_reserved == min_queue_len) {
@@ -88,14 +111,15 @@ int obtain_clone_cpuid(cpu_set_t *cpu_set, int use_last)
 		dkprintf("%s: cpu=%d,runq_len=%d,runq_reserved=%d,min_cpu=%d,uti_cpu=%d\n",
 			 __func__, cpu, v->runq_len, v->runq_reserved,
 			 min_cpu, uti_cpu);
+#else
 
 		ihk_mc_spinlock_unlock_noirq(&v->runq_lock);
-#if 0
 		if (min_queue_len == 0)
 			break;
 #endif
 	}
 
+#if 0
 	min_cpu = use_last ? uti_cpu : min_cpu;
 	if (min_cpu != -1) {
 		if (get_cpu_local_var(min_cpu)->status != CPU_STATUS_RESERVED)
@@ -104,22 +128,20 @@ int obtain_clone_cpuid(cpu_set_t *cpu_set, int use_last)
 		__sync_fetch_and_add(&get_cpu_local_var(min_cpu)->runq_reserved,
 				     1);
 	}
-	ihk_mc_spinlock_unlock(&runq_reservation_lock, irqstate);
+#else
+	__sync_fetch_and_add(&get_cpu_local_var(min_cpu)->runq_reserved, 1);
+#endif
+
+	if (!cpu_local_var(current)->proc->nr_processes) {
+		ihk_mc_spinlock_unlock(&runq_reservation_lock, irqstate);
+	}
+	else {
+		cpu_restore_interrupt(irqstate);
+	}
 
 	return min_cpu;
 }
 
-int
-arch_clear_host_user_space()
-{
-	struct thread *th = cpu_local_var(current);
-
-	/* XXX: might be unnecessary */
-	clear_host_pte(th->vm->region.user_start,
-			(th->vm->region.user_end - th->vm->region.user_start), 0);
-	return 0;
-}
-
 /* archtecture-depended syscall handlers */
 extern unsigned long do_fork(int clone_flags, unsigned long newsp,
 			     unsigned long parent_tidptr, unsigned long child_tidptr,
@@ -152,33 +174,6 @@ SYSCALL_DECLARE(clone)
 	return ret;
 }
 
-SYSCALL_DECLARE(rt_sigaction)
-{
-	int sig = ihk_mc_syscall_arg0(ctx);
-	const struct sigaction *act = (const struct sigaction *)ihk_mc_syscall_arg1(ctx);
-	struct sigaction *oact = (struct sigaction *)ihk_mc_syscall_arg2(ctx);
-	size_t sigsetsize = ihk_mc_syscall_arg3(ctx);
-	struct k_sigaction new_sa, old_sa;
-	int rc;
-
-	if (sigsetsize != sizeof(sigset_t))
-		return -EINVAL;
-
-	if(act)
-		if(copy_from_user(&new_sa.sa, act, sizeof new_sa.sa)){
-			goto fault;
-		}
-	rc = do_sigaction(sig, act? &new_sa: NULL, oact? &old_sa: NULL);
-	if(rc == 0 && oact)
-		if(copy_to_user(oact, &old_sa.sa, sizeof old_sa.sa)){
-			goto fault;
-		}
-
-	return rc;
-fault:
-	return -EFAULT;
-}
-
 SYSCALL_DECLARE(prctl)
 {
 	struct process *proc = cpu_local_var(current)->proc;
@@ -1335,70 +1330,6 @@ out:
 	return restart;
 }
 
-static struct sig_pending *
-getsigpending(struct thread *thread, int delflag){
-	struct list_head *head;
-	mcs_rwlock_lock_t *lock;
-	struct mcs_rwlock_node_irqsave mcs_rw_node;
-	struct sig_pending *next;
-	struct sig_pending *pending;
-	__sigset_t w;
-
-	w = thread->sigmask.__val[0];
-
-	lock = &thread->sigcommon->lock;
-	head = &thread->sigcommon->sigpending;
-	for(;;) {
-		if (delflag) {
-			mcs_rwlock_writer_lock(lock, &mcs_rw_node);
-		}
-		else {
-			mcs_rwlock_reader_lock(lock, &mcs_rw_node);
-		}
-
-		list_for_each_entry_safe(pending, next, head, list){
-			if(!(pending->sigmask.__val[0] & w)){
-				if(delflag) 
-					list_del(&pending->list);
-
-				if (delflag) {
-					mcs_rwlock_writer_unlock(lock, &mcs_rw_node);
-				}
-				else {
-					mcs_rwlock_reader_unlock(lock, &mcs_rw_node);
-				}
-				return pending;
-			}
-		}
-
-		if (delflag) {
-			mcs_rwlock_writer_unlock(lock, &mcs_rw_node);
-		}
-		else {
-			mcs_rwlock_reader_unlock(lock, &mcs_rw_node);
-		}
-
-		if(lock == &thread->sigpendinglock)
-			return NULL;
-
-		lock = &thread->sigpendinglock;
-		head = &thread->sigpending;
-	}
-
-	return NULL;
-}
-
-struct sig_pending *
-hassigpending(struct thread *thread)
-{
-	if (list_empty(&thread->sigpending) &&
-	    list_empty(&thread->sigcommon->sigpending)) {
-		return NULL;
-	}
-
-	return getsigpending(thread, 0);
-}
-
 int
 interrupt_from_user(void *regs0)
 {
@@ -1422,191 +1353,6 @@ void save_syscall_return_value(int num, unsigned long rc)
 	}
 }
 
-void
-check_signal(unsigned long rc, void *regs0, int num)
-{
-	__check_signal(rc, regs0, num, 0);
-}
-
-void
-check_signal_irq_disabled(unsigned long rc, void *regs0, int num)
-{
-	__check_signal(rc, regs0, num, 1);
-}
-
-static void
-__check_signal(unsigned long rc, void *regs0, int num, int irq_disabled)
-{
-	ihk_mc_user_context_t *regs = regs0;
-	struct thread *thread;
-	struct sig_pending *pending;
-	int	irqstate;
-
-	if(clv == NULL)
-		return;
-	thread = cpu_local_var(current);
-
-	if(thread == NULL || thread->proc->pid == 0){
-		struct thread *t;
-
-		irqstate = cpu_disable_interrupt_save();
-		ihk_mc_spinlock_lock_noirq(&(cpu_local_var(runq_lock)));
-		list_for_each_entry(t, &(cpu_local_var(runq)), sched_list){
-			if(t->proc->pid <= 0)
-				continue;
-			if(t->status == PS_INTERRUPTIBLE &&
-			   hassigpending(t)){
-				t->status = PS_RUNNING;
-				break;
-			}
-		}
-		ihk_mc_spinlock_unlock_noirq(&(cpu_local_var(runq_lock)));
-		cpu_restore_interrupt(irqstate);
-		goto out;
-	}
-
-	if(regs != NULL && !interrupt_from_user(regs)) {
-		goto out;
-	}
-
-	if (list_empty(&thread->sigpending) &&
-	    list_empty(&thread->sigcommon->sigpending)) {
-		goto out;
-	}
-
-	for(;;){
-		/* When this function called from check_signal_irq_disabled,
-		 * return with interrupt invalid.
-		 * This is to eliminate signal loss.
-		 */
-		if (irq_disabled == 1) {
-			irqstate = cpu_disable_interrupt_save();
-		}
-		pending = getsigpending(thread, 1);
-		if(!pending) {
-			dkprintf("check_signal,queue is empty\n");
-			goto out;
-		}
-		if (irq_disabled == 1) {
-			cpu_restore_interrupt(irqstate);
-		}
-		if (do_signal(rc, regs, thread, pending, num)) {
-			num = -1;
-		}
-	}
-
-out:
-	return;
-}
-
-static int
-check_sig_pending_thread(struct thread *thread)
-{
-	int found = 0;
-	struct list_head *head;
-	mcs_rwlock_lock_t *lock;
-	struct mcs_rwlock_node_irqsave mcs_rw_node;
-	struct sig_pending *next;
-	struct sig_pending *pending;
-	__sigset_t w;
-	__sigset_t x;
-	int sig = 0;
-	struct k_sigaction *k;
-	struct cpu_local_var *v;
-
-	v = get_this_cpu_local_var();
-	w = thread->sigmask.__val[0];
-
-	lock = &thread->sigcommon->lock;
-	head = &thread->sigcommon->sigpending;
-	for (;;) {
-		mcs_rwlock_reader_lock(lock, &mcs_rw_node);
-
-		list_for_each_entry_safe(pending, next, head, list) {
-			for (x = pending->sigmask.__val[0], sig = 0; x;
-			     sig++, x >>= 1)
-				;
-			k = thread->sigcommon->action + sig - 1;
-			if ((sig != SIGCHLD &&
-			     sig != SIGURG &&
-			     sig != SIGCONT) ||
-			    (k->sa.sa_handler != SIG_IGN &&
-			     k->sa.sa_handler != NULL)) {
-				if (!(pending->sigmask.__val[0] & w)) {
-					if (pending->interrupted == 0) {
-						pending->interrupted = 1;
-						found = 1;
-						if (sig != SIGCHLD &&
-						    sig != SIGURG &&
-						    sig != SIGCONT &&
-						    !k->sa.sa_handler) {
-							found = 2;
-							break;
-						}
-					}
-				}
-			}
-		}
-
-		mcs_rwlock_reader_unlock(lock, &mcs_rw_node);
-
-		if (found == 2) {
-			break;
-		}
-
-		if (lock == &thread->sigpendinglock) {
-			break;
-		}
-
-		lock = &thread->sigpendinglock;
-		head = &thread->sigpending;
-	}
-
-	if (found == 2) {
-		ihk_mc_spinlock_unlock(&v->runq_lock, v->runq_irqstate);
-		terminate_mcexec(0, sig);
-		return 1;
-	}
-	else if (found == 1) {
-		ihk_mc_spinlock_unlock(&v->runq_lock, v->runq_irqstate);
-		interrupt_syscall(thread, 0);
-		return 1;
-	}
-	return 0;
-}
-
-void
-check_sig_pending(void)
-{
-	struct thread *thread;
-	struct cpu_local_var *v;
-
-	if (clv == NULL)
-		return;
-
-	v = get_this_cpu_local_var();
-repeat:
-	v->runq_irqstate = ihk_mc_spinlock_lock(&v->runq_lock);
-	list_for_each_entry(thread, &(v->runq), sched_list) {
-
-		if (thread == NULL || thread == &cpu_local_var(idle)) {
-			continue;
-		}
-
-		if (thread->in_syscall_offload == 0) {
-			continue;
-		}
-
-		if (thread->proc->group_exit_status & 0x0000000100000000L) {
-			continue;
-		}
-
-		if (check_sig_pending_thread(thread))
-			goto repeat;
-	}
-	ihk_mc_spinlock_unlock(&v->runq_lock, v->runq_irqstate);
-}
-
 unsigned long
 do_kill(struct thread * thread, int pid, int tid, int sig, siginfo_t *info, int ptracecont)
 {
@@ -2043,7 +1789,7 @@ SYSCALL_DECLARE(mmap)
 		goto out;
 	}
 
-	addr = do_mmap(addr, len, prot, flags, fd, off0, 0, NULL);
+	addr = do_mmap(addr, len, prot, flags, fd, off0);
 
 	error = 0;
 out:
@@ -2137,7 +1883,9 @@ int do_process_vm_read_writev(int pid,
 	unsigned long lpage_left, rpage_left;
 	unsigned long lpsize, rpsize;
 	void *rva, *lva;
+#if 0
 	struct vm_range *range;
+#endif
 	struct mcs_rwlock_node_irqsave lock;
 	struct mcs_rwlock_node update_lock;
 
@@ -2150,8 +1898,9 @@ int do_process_vm_read_writev(int pid,
 		return -EINVAL;
 	}
 
+#if 0
 	/* Check if parameters are okay */
-	ihk_mc_spinlock_lock_noirq(&lthread->vm->memory_range_lock);
+	ihk_rwspinlock_read_lock_noirq(&lthread->vm->memory_range_lock);
 
 	range = lookup_process_memory_range(lthread->vm, 
 			(uintptr_t)local_iov, 
@@ -2173,11 +1922,12 @@ int do_process_vm_read_writev(int pid,
 
 	ret = 0;
 arg_out:
-	ihk_mc_spinlock_unlock_noirq(&lthread->vm->memory_range_lock);
+	ihk_rwspinlock_read_unlock_noirq(&lthread->vm->memory_range_lock);
 
 	if (ret != 0) {
 		goto out;
 	}
+#endif
 
 	for (li = 0; li < liovcnt; ++li) {
 		llen += local_iov[li].iov_len;
@@ -2242,7 +1992,7 @@ arg_out:
 		if (pli != li) {
 			struct vm_range *range;
 
-			ihk_mc_spinlock_lock_noirq(&lthread->vm->memory_range_lock);
+			ihk_rwspinlock_read_lock_noirq(&lthread->vm->memory_range_lock);
 
 			/* Is base valid? */
 			range = lookup_process_memory_range(lthread->vm,
@@ -2272,7 +2022,7 @@ arg_out:
 
 			ret = 0;
 pli_out:
-			ihk_mc_spinlock_unlock_noirq(&lthread->vm->memory_range_lock);
+			ihk_rwspinlock_read_unlock_noirq(&lthread->vm->memory_range_lock);
 
 			if (ret != 0) {
 				goto out;
@@ -2285,7 +2035,7 @@ pli_out:
 		if (pri != ri) {
 			struct vm_range *range;
 
-			ihk_mc_spinlock_lock_noirq(&rvm->memory_range_lock);
+			ihk_rwspinlock_read_lock_noirq(&rvm->memory_range_lock);
 
 			/* Is base valid? */
 			range = lookup_process_memory_range(rvm,
@@ -2315,7 +2065,7 @@ pli_out:
 
 			ret = 0;
 pri_out:
-			ihk_mc_spinlock_unlock_noirq(&rvm->memory_range_lock);
+			ihk_rwspinlock_read_unlock_noirq(&rvm->memory_range_lock);
 
 			if (ret != 0) {
 				goto out;

arch/arm64/kernel/vdso.c

@@ -155,7 +155,7 @@ int arch_map_vdso(struct process_vm *vm)
 	flag = VR_REMOTE | VR_PROT_READ;
 	flag |= VRFLAG_PROT_TO_MAXPROT(flag);
 	ret = add_process_memory_range(vm, start, end, vdso.vvar_phys, flag,
-				       NULL, 0, PAGE_SHIFT, NULL, &range);
+				       NULL, 0, PAGE_SHIFT, &range);
 	if (ret != 0){
 		dkprintf("ERROR: adding memory range for tod_data\n");
 		goto exit;
@@ -167,7 +167,7 @@ int arch_map_vdso(struct process_vm *vm)
 	flag = VR_REMOTE | VR_PROT_READ | VR_PROT_EXEC;
 	flag |= VRFLAG_PROT_TO_MAXPROT(flag);
 	ret = add_process_memory_range(vm, start, end, vdso.vdso_physlist[0], flag,
-				       NULL, 0, PAGE_SHIFT, NULL, &range);
+				       NULL, 0, PAGE_SHIFT, &range);
 	if (ret != 0) {
 		dkprintf("ERROR: adding memory range for vdso_text\n");
 

arch/x86_64/kernel/include/rlimit.h

@@ -84,7 +84,11 @@ enum __rlimit_resource
   __RLIMIT_RTPRIO = 14,
 #define RLIMIT_RTPRIO __RLIMIT_RTPRIO
 
-  __RLIMIT_NLIMITS = 15,
+  /* timeout for RT tasks in us */
+  __RLIMIT_RTTIME = 15,
+#define RLIMIT_RTTIME __RLIMIT_RTTIME
+
+  __RLIMIT_NLIMITS = 16,
   __RLIM_NLIMITS = __RLIMIT_NLIMITS
 #define RLIMIT_NLIMITS __RLIMIT_NLIMITS
 #define RLIM_NLIMITS __RLIM_NLIMITS

arch/x86_64/kernel/include/syscall_list.h

@@ -158,6 +158,7 @@ SYSCALL_HANDLED(289, signalfd4)
 #ifdef ENABLE_PERF
 SYSCALL_HANDLED(298, perf_event_open)
 #endif
+SYSCALL_HANDLED(302, prlimit64)
 #ifdef DCFA_KMOD
 SYSCALL_HANDLED(303, mod_call)
 #endif
@@ -180,4 +181,8 @@ SYSCALL_HANDLED(802, linux_mlock)
 SYSCALL_HANDLED(803, suspend_threads)
 SYSCALL_HANDLED(804, resume_threads)
 SYSCALL_HANDLED(811, linux_spawn)
-/**** End of File ****/
+
+/* Do not edit the lines including this comment and
+ * EOF just after it because those are used as a
+ * robust marker for the autotest patch.
+ */

arch/x86_64/kernel/syscall.c

@@ -31,7 +31,6 @@
 #include <page.h>
 #include <limits.h>
 #include <syscall.h>
-#include <bitops.h>
 #include <rusage_private.h>
 #include <ihk/debug.h>
 
@@ -148,44 +147,6 @@ int obtain_clone_cpuid(cpu_set_t *cpu_set, int use_last) {
     return min_cpu;
 }
 
-int
-arch_clear_host_user_space()
-{
-	struct thread *th = cpu_local_var(current);
-
-	/* XXX: might be unnecessary */
-	clear_host_pte(th->vm->region.user_start,
-			(th->vm->region.user_end - th->vm->region.user_start), 0);
-	return 0;
-}
-
-SYSCALL_DECLARE(rt_sigaction)
-{
-	int sig = ihk_mc_syscall_arg0(ctx);
-	const struct sigaction *act = (const struct sigaction *)ihk_mc_syscall_arg1(ctx);
-	struct sigaction *oact = (struct sigaction *)ihk_mc_syscall_arg2(ctx);
-	size_t sigsetsize = ihk_mc_syscall_arg3(ctx);
-	struct k_sigaction new_sa, old_sa;
-	int rc;
-
-	if (sigsetsize != sizeof(sigset_t))
-		return -EINVAL;
-
-	if(act)
-		if(copy_from_user(&new_sa.sa, act, sizeof new_sa.sa)){
-			goto fault;
-		}
-	rc = do_sigaction(sig, act? &new_sa: NULL, oact? &old_sa: NULL);
-	if(rc == 0 && oact)
-		if(copy_to_user(oact, &old_sa.sa, sizeof old_sa.sa)){
-			goto fault;
-		}
-
-	return rc;
-fault:
-	return -EFAULT;
-}
-
 SYSCALL_DECLARE(prctl)
 {
 	struct process *proc = cpu_local_var(current)->proc;
@@ -1040,82 +1001,6 @@ out:
 	return restart;
 }
 
-static struct sig_pending *
-getsigpending(struct thread *thread, int delflag){
-	struct list_head *head;
-	mcs_rwlock_lock_t *lock;
-	struct mcs_rwlock_node_irqsave mcs_rw_node;
-	struct sig_pending *next;
-	struct sig_pending *pending;
-	__sigset_t w;
-	__sigset_t x;
-	int sig;
-	struct k_sigaction *k;
-
-	w = thread->sigmask.__val[0];
-
-	lock = &thread->sigcommon->lock;
-	head = &thread->sigcommon->sigpending;
-	for(;;) {
-		if (delflag) {
-			mcs_rwlock_writer_lock(lock, &mcs_rw_node);
-		}
-		else {
-			mcs_rwlock_reader_lock(lock, &mcs_rw_node);
-		}
-
-		list_for_each_entry_safe(pending, next, head, list){
-			for(x = pending->sigmask.__val[0], sig = 0; x; sig++, x >>= 1);
-			k = thread->sigcommon->action + sig - 1;
-			if(delflag ||
-			   (sig != SIGCHLD &&
-			    sig != SIGURG &&
-			    sig != SIGCONT) ||
-			   (k->sa.sa_handler != (void *)1 &&
-			    k->sa.sa_handler != NULL)){
-				if(!(pending->sigmask.__val[0] & w)){
-					if(delflag) 
-						list_del(&pending->list);
-
-					if (delflag) {
-						mcs_rwlock_writer_unlock(lock, &mcs_rw_node);
-					}
-					else {
-						mcs_rwlock_reader_unlock(lock, &mcs_rw_node);
-					}
-					return pending;
-				}
-			}
-		}
-
-		if (delflag) {
-			mcs_rwlock_writer_unlock(lock, &mcs_rw_node);
-		}
-		else {
-			mcs_rwlock_reader_unlock(lock, &mcs_rw_node);
-		}
-
-		if(lock == &thread->sigpendinglock)
-			return NULL;
-
-		lock = &thread->sigpendinglock;
-		head = &thread->sigpending;
-	}
-
-	return NULL;
-}
-
-struct sig_pending *
-hassigpending(struct thread *thread)
-{
-	if (list_empty(&thread->sigpending) &&
-	    list_empty(&thread->sigcommon->sigpending)) {
-		return NULL;
-	}
-
-	return getsigpending(thread, 0);
-}
-
 int
 interrupt_from_user(void *regs0)
 {
@@ -1130,175 +1015,6 @@ void save_syscall_return_value(int num, unsigned long rc)
 	return;
 }
 
-/** \brief check arrived signals and processing
- *
- * @param rc    return value of syscall
- * @param regs0 context
- * @param num   syscall number (-1: Not called on exiting system call)
- */
-void
-check_signal(unsigned long rc, void *regs0, int num)
-{
-	struct x86_user_context *regs = regs0;
-	struct thread *thread;
-	struct sig_pending *pending;
-	int	irqstate;
-
-	if(clv == NULL)
-		return;
-	thread = cpu_local_var(current);
-
-	if(thread == NULL || thread == &cpu_local_var(idle)){
-		struct thread *t;
-
-		irqstate = cpu_disable_interrupt_save();
-		ihk_mc_spinlock_lock_noirq(&(cpu_local_var(runq_lock)));
-		list_for_each_entry(t, &(cpu_local_var(runq)), sched_list){
-			if(t == &cpu_local_var(idle))
-				continue;
-			if(t->status == PS_INTERRUPTIBLE &&
-			   hassigpending(t)){
-				t->status = PS_RUNNING;
-				break;
-			}
-		}
-		ihk_mc_spinlock_unlock_noirq(&(cpu_local_var(runq_lock)));
-		cpu_restore_interrupt(irqstate);
-		goto out;
-	}
-
-	if(regs != NULL && !interrupt_from_user(regs)) {
-		goto out;
-	}
-
-	if (list_empty(&thread->sigpending) &&
-	    list_empty(&thread->sigcommon->sigpending)) {
-		goto out;
-	}
-
-	for(;;){
-		pending = getsigpending(thread, 1);
-		if(!pending) {
-			dkprintf("check_signal,queue is empty\n");
-			goto out;
-		}
-
-		if (do_signal(rc, regs, thread, pending, num)) {
-			num = -1;
-		}
-	}
-
-out:
-	return;
-}
-
-static int
-check_sig_pending_thread(struct thread *thread)
-{
-	int found = 0;
-	struct list_head *head;
-	mcs_rwlock_lock_t *lock;
-	struct mcs_rwlock_node_irqsave mcs_rw_node;
-	struct sig_pending *next;
-	struct sig_pending *pending;
-	__sigset_t w;
-	__sigset_t x;
-	int sig = 0;
-	struct k_sigaction *k;
-	struct cpu_local_var *v;
-
-	v = get_this_cpu_local_var();
-	w = thread->sigmask.__val[0];
-
-	lock = &thread->sigcommon->lock;
-	head = &thread->sigcommon->sigpending;
-	for (;;) {
-		mcs_rwlock_reader_lock(lock, &mcs_rw_node);
-
-		list_for_each_entry_safe(pending, next, head, list){
-			for (x = pending->sigmask.__val[0], sig = 0; x;
-			     sig++, x >>= 1);
-			k = thread->sigcommon->action + sig - 1;
-			if ((sig != SIGCHLD &&
-			     sig != SIGURG &&
-			     sig != SIGCONT) ||
-			    (k->sa.sa_handler != (void *)1 &&
-			     k->sa.sa_handler != NULL)) {
-				if (!(pending->sigmask.__val[0] & w)) {
-					if (pending->interrupted == 0) {
-						pending->interrupted = 1;
-						found = 1;
-						if (sig != SIGCHLD &&
-						    sig != SIGURG &&
-						    sig != SIGCONT &&
-						    !k->sa.sa_handler) {
-							found = 2;
-							break;
-						}
-					}
-				}
-			}
-		}
-
-		mcs_rwlock_reader_unlock(lock, &mcs_rw_node);
-
-		if (found == 2) {
-			break;
-		}
-
-		if (lock == &thread->sigpendinglock) {
-			break;
-		}
-
-		lock = &thread->sigpendinglock;
-		head = &thread->sigpending;
-	}
-
-	if (found == 2) {
-		ihk_mc_spinlock_unlock(&v->runq_lock, v->runq_irqstate);
-		terminate_mcexec(0, sig);
-		return 1;
-	}
-	else if (found == 1) {
-		ihk_mc_spinlock_unlock(&v->runq_lock, v->runq_irqstate);
-		interrupt_syscall(thread, 0);
-		return 1;
-	}
-	return 0;
-}
-
-void
-check_sig_pending(void)
-{
-	struct thread *thread;
-	struct cpu_local_var *v;
-
-	if (clv == NULL)
-		return;
-
-	v = get_this_cpu_local_var();
-repeat:
-	v->runq_irqstate = ihk_mc_spinlock_lock(&v->runq_lock);
-	list_for_each_entry(thread, &(v->runq), sched_list) {
-
-		if (thread == NULL || thread == &cpu_local_var(idle)) {
-			continue;
-		}
-
-		if (thread->in_syscall_offload == 0) {
-			continue;
-		}
-
-		if (thread->proc->group_exit_status & 0x0000000100000000L) {
-			continue;
-		}
-
-		if (check_sig_pending_thread(thread))
-			goto repeat;
-	}
-	ihk_mc_spinlock_unlock(&v->runq_lock, v->runq_irqstate);
-}
-
 unsigned long
 do_kill(struct thread *thread, int pid, int tid, int sig, siginfo_t *info,
         int ptracecont)
@@ -1786,7 +1502,7 @@ recheck:
 		goto out;
 	}
 
-	addr = do_mmap(addr, len, prot, flags, fd, off0, 0, NULL);
+	addr = do_mmap(addr, len, prot, flags, fd, off0);
 
 	error = 0;
 out:
@@ -2115,7 +1831,7 @@ int arch_map_vdso(struct process_vm *vm)
 	vrflags |= VR_PROT_READ | VR_PROT_EXEC;
 	vrflags |= VRFLAG_PROT_TO_MAXPROT(vrflags);
 	error = add_process_memory_range(vm, (intptr_t)s, (intptr_t)e,
-			NOPHYS, vrflags, NULL, 0, PAGE_SHIFT, NULL, &range);
+			NOPHYS, vrflags, NULL, 0, PAGE_SHIFT, &range);
 	if (error) {
 		ekprintf("ERROR: adding memory range for vdso. %d\n", error);
 		goto out;
@@ -2147,8 +1863,7 @@ int arch_map_vdso(struct process_vm *vm)
 		vrflags |= VR_PROT_READ;
 		vrflags |= VRFLAG_PROT_TO_MAXPROT(vrflags);
 		error = add_process_memory_range(vm, (intptr_t)s, (intptr_t)e,
-				NOPHYS, vrflags, NULL, 0,
-				PAGE_SHIFT, NULL, &range);
+				NOPHYS, vrflags, NULL, 0, PAGE_SHIFT, &range);
 		if (error) {
 			ekprintf("ERROR: adding memory range for vvar. %d\n", error);
 			goto out;
@@ -2290,7 +2005,7 @@ int do_process_vm_read_writev(int pid,
 	}
 
 	/* Check if parameters are okay */
-	ihk_mc_spinlock_lock_noirq(&lthread->vm->memory_range_lock);
+	ihk_rwspinlock_read_lock_noirq(&lthread->vm->memory_range_lock);
 
 	range = lookup_process_memory_range(lthread->vm, 
 			(uintptr_t)local_iov, 
@@ -2312,7 +2027,7 @@ int do_process_vm_read_writev(int pid,
 
 	ret = 0;
 arg_out:
-	ihk_mc_spinlock_unlock_noirq(&lthread->vm->memory_range_lock);
+	ihk_rwspinlock_read_unlock_noirq(&lthread->vm->memory_range_lock);
 
 	if (ret != 0) {
 		goto out;
@@ -2381,7 +2096,7 @@ arg_out:
 		if (pli != li) {
 			struct vm_range *range;
 
-			ihk_mc_spinlock_lock_noirq(&lthread->vm->memory_range_lock);
+			ihk_rwspinlock_read_lock_noirq(&lthread->vm->memory_range_lock);
 
 			/* Is base valid? */
 			range = lookup_process_memory_range(lthread->vm,
@@ -2411,7 +2126,7 @@ arg_out:
 
 			ret = 0;
 pli_out:
-			ihk_mc_spinlock_unlock_noirq(&lthread->vm->memory_range_lock);
+			ihk_rwspinlock_read_unlock_noirq(&lthread->vm->memory_range_lock);
 
 			if (ret != 0) {
 				goto out;
@@ -2424,7 +2139,7 @@ pli_out:
 		if (pri != ri) {
 			struct vm_range *range;
 
-			ihk_mc_spinlock_lock_noirq(&rvm->memory_range_lock);
+			ihk_rwspinlock_read_lock_noirq(&rvm->memory_range_lock);
 
 			/* Is base valid? */
 			range = lookup_process_memory_range(rvm,
@@ -2454,7 +2169,7 @@ pli_out:
 
 			ret = 0;
 pri_out:
-			ihk_mc_spinlock_unlock_noirq(&rvm->memory_range_lock);
+			ihk_rwspinlock_read_unlock_noirq(&rvm->memory_range_lock);
 
 			if (ret != 0) {
 				goto out;

cmake/modules/AutoconfHelper.cmake

@@ -0,0 +1,383 @@
+# Helper functions for translating autoconf projects. Several functions
+# are lifted from the Mono sources
+
+include (CheckCSourceCompiles)
+include (CheckIncludeFile)
+include (TestBigEndian)
+include (CheckFunctionExists)
+include (CheckTypeSize)
+include (CheckCSourceRuns)
+
+
+# Function to get the version information from the configure.ac file in the
+# current directory. Its argument is the name of the library as passed to
+# AC_INIT. It will set the variables ${LIBNAME}_VERSION and ${LIBNAME}_SOVERSION
+function (ac_get_version libname)
+  string(TOUPPER "${libname}" libname_upper)
+
+  # Read the relevant content from configure.ac
+  file (STRINGS configure.ac tmp_configure_ac
+    REGEX "${libname_upper}_[_A-Z]+=[ \\t]*[0-9]+")
+
+  # Product version
+  string (REGEX REPLACE ".+MAJOR[_A-Z]+=([0-9]+).+MINOR[_A-Z]+=([0-9]+).+MICRO[_A-Z]+=([0-9]+).*"
+    "\\1.\\2.\\3" ${libname_upper}_VERSION "${tmp_configure_ac}")
+
+  # Library version for libtool
+  string (REGEX REPLACE ".+CURRENT=([0-9]+).+REVISION=([0-9]+).+AGE=([0-9]+).*"
+    "\\1.\\2.\\3" ${libname_upper}_SOVERSION "${tmp_configure_ac}")
+
+  # Checks if the string needs to be displayed
+  set (${libname_upper}_DISPLAYSTR_AUX
+    "Found ${libname} version ${${libname_upper}_VERSION}, soversion ${${libname_upper}_SOVERSION} from configure.ac"
+  )
+  if ((NOT ${libname_upper}_DISPLAYSTR) OR (NOT ${libname_upper}_DISPLAYSTR STREQUAL ${libname_upper}_DISPLAYSTR_AUX))
+    set (${libname_upper}_DISPLAYSTR ${${libname_upper}_DISPLAYSTR_AUX}
+      CACHE INTERNAL "Version string from ${libname}" FORCE)
+    message (STATUS ${${libname_upper}_DISPLAYSTR})
+  endif ()
+
+  # Export the result to the caller
+  set(${libname_upper}_VERSION "${${libname_upper}_VERSION}" PARENT_SCOPE)
+  set(${libname_upper}_SOVERSION "${${libname_upper}_SOVERSION}" PARENT_SCOPE)
+endfunction()
+
+
+# Also from mono's source code
+# Implementation of AC_CHECK_HEADERS
+# In addition, it also records the list of variables in the variable
+# 'autoheader_vars', and for each variable, a documentation string in the
+# variable ${var}_doc
+function(ac_check_headers)
+  foreach (header ${ARGV})
+	string(TOUPPER ${header} header_var)
+	string(REPLACE "." "_" header_var ${header_var})
+	string(REPLACE "/" "_" header_var ${header_var})
+	set(header_var "HAVE_${header_var}")
+	check_include_file (${header} ${header_var})
+	set("${header_var}_doc" "Define to 1 if you have the <${header}> header file." PARENT_SCOPE)
+	if (${header_var})
+	  set("${header_var}_defined" "1" PARENT_SCOPE)
+	endif()
+	set("${header_var}_val" "1" PARENT_SCOPE)
+	set (autoheader_vars ${autoheader_vars} ${header_var})
+  endforeach()
+  set (autoheader_vars ${autoheader_vars} PARENT_SCOPE)
+endfunction()
+
+# Function taken from mono's source code
+function (ac_check_funcs)
+  foreach (func ${ARGV})
+    string(TOUPPER ${func} var)
+    set(var "HAVE_${var}")
+    set(${var})
+    check_function_exists (${func} ${var})
+    set("${var}_doc" "Define to 1 if you have the '${func}' function." PARENT_SCOPE)
+    if (${var})
+      set("${var}_defined" "1" PARENT_SCOPE)
+      set(${var} yes PARENT_SCOPE)
+    endif()
+    set("${var}_val" "1" PARENT_SCOPE)
+    set (autoheader_vars ${autoheader_vars} ${var})
+  endforeach()
+  set (autoheader_vars ${autoheader_vars} PARENT_SCOPE)
+endfunction()
+
+
+# Specifically, this macro checks for stdlib.h', stdarg.h',
+# string.h', and float.h'; if the system has those, it probably
+# has the rest of the ANSI C header files.  This macro also checks
+# whether string.h' declares memchr' (and thus presumably the
+# other mem' functions), whether stdlib.h' declare free' (and
+# thus presumably malloc' and other related functions), and whether
+# the ctype.h' macros work on characters with the high bit set, as
+# ANSI C requires.
+function (ac_header_stdc)
+  if (STDC_HEADERS)
+    return()
+  endif()
+  message(STATUS "Looking for ANSI-C headers")
+  set(code "
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+#include <float.h>
+
+int main(int argc, char **argv)
+{
+  void *ptr;
+  free((void*)1);
+  ptr = memchr((void*)1, 0, 0);
+
+  return (int)ptr;
+}
+")
+  # FIXME Check the ctype.h high bit
+  CHECK_C_SOURCE_COMPILES("${code}" STDC_HEADERS)
+  if (STDC_HEADERS)
+    set(STDC_HEADERS 1 PARENT_SCOPE)
+	message(STATUS "Looking for ANSI-C headers - found")
+  else()
+    message(STATUS "Looking for ANSI-C headers - not found")
+  endif()
+endfunction()
+
+
+# Also from the mono sources, kind of implements AC_SYS_LARGEFILE
+function (ac_sys_largefile)
+  CHECK_C_SOURCE_RUNS("
+#include <sys/types.h>
+#define BIG_OFF_T (((off_t)1<<62)-1+((off_t)1<<62))
+int main (int argc, char **argv) {
+    int big_off_t=((BIG_OFF_T%2147483629==721) &&
+                   (BIG_OFF_T%2147483647==1));
+    return big_off ? 0 : 1;
+}
+" HAVE_LARGE_FILE_SUPPORT)
+
+# Check if it makes sense to define _LARGE_FILES or _FILE_OFFSET_BITS
+  if (HAVE_LARGE_FILE_SUPPORT)
+    return()
+  endif()
+
+  set (_LARGE_FILE_EXTRA_SRC "
+#include <sys/types.h>
+int main (int argc, char **argv) {
+  return sizeof(off_t) == 8 ? 0 : 1;
+}
+")
+  CHECK_C_SOURCE_RUNS ("#define _LARGE_FILES\n${_LARGE_FILE_EXTRA_SRC}"
+    HAVE_USEFUL_D_LARGE_FILES)
+  if (NOT HAVE_USEFUL_D_LARGE_FILES)
+    if (NOT DEFINED HAVE_USEFUL_D_FILE_OFFSET_BITS)
+      set (SHOW_LARGE_FILE_WARNING TRUE)
+    endif ()
+    CHECK_C_SOURCE_RUNS ("#define _FILE_OFFSET_BITS 64\n${_LARGE_FILE_EXTRA_SRC}"
+      HAVE_USEFUL_D_FILE_OFFSET_BITS)
+	  if (HAVE_USEFUL_D_FILE_OFFSET_BITS)
+      set (_FILE_OFFSET_BITS 64 PARENT_SCOPE)
+	  elseif (SHOW_LARGE_FILE_WARNING)
+	    message (WARNING "No 64 bit file support through off_t available.")
+	  endif ()
+  else ()
+    set (_LARGE_FILES 1 PARENT_SCOPE)
+  endif ()
+endfunction ()
+
+
+# Quick way to set some basic variables
+# FIXME add support for variable number of arguments: only package and version are mandatory
+# arguments are package version bug_report tarname url
+function (ac_init)
+    set(package ${ARGV0})
+    set(version ${ARGV1})
+    set(bug_report ${ARGV2})
+    set(tarname ${ARGV3})
+    set(url ${ARGV4})
+    set(PACKAGE_NAME "\"${package}\"" PARENT_SCOPE)
+    set(PACKAGE_VERSION "\"${version}\"" PARENT_SCOPE)
+    set(VERSION "\"${version}\"" PARENT_SCOPE)
+    if(version)
+        set(PACKAGE_STRING "\"${package} ${version}\"" PARENT_SCOPE)
+    else()
+        set(PACKAGE_STRING "\"${package}\"" PARENT_SCOPE)
+    endif()
+
+    set(PACKAGE_BUGREPORT "\"${bug_report}\"" PARENT_SCOPE)
+
+    if(NOT tarname)
+        string(REGEX REPLACE "[^a-zA-Z0-9_]" "-" tarname "${package}")
+    endif()
+    set(PACKAGE_TARNAME "\"${tarname}\"" PARENT_SCOPE)
+
+    set(PACKAGE_URL "\"${url}\"" PARENT_SCOPE)
+endfunction()
+
+
+# Checks for the const keyword, defining "HAS_CONST_SUPPORT"
+# If it does not have support, defines "const" to 0 in the parent scope
+function (ac_c_const)
+  CHECK_C_SOURCE_COMPILES(
+    "int main(int argc, char **argv){const int r = 0;return r;}"
+    HAS_CONST_SUPPORT)
+  if (NOT HAS_CONST_SUPPORT)
+    set(const 0 PARENT_SCOPE)
+  endif()
+endfunction()
+
+
+# Inline keyword support. Defines "inline" in the parent scope to the
+# compiler internal keyword for inline in C
+# TODO write a better test!
+function (ac_c_inline)
+  if (MSVC)
+    set (inline __inline)
+  elseif(CMAKE_COMPILER_IS_GNUC)
+    set (inline __inline__)
+  endif()
+  set(inline "${inline}" PARENT_SCOPE)
+endfunction()
+
+
+# Test if you can safely include both <sys/time.h> and <time.h>
+function (ac_header_time)
+  CHECK_C_SOURCE_COMPILES(
+    "#include <sys/time.h>\n#include <time.h>\nint main(int argc, char **argv) { return 0; }"
+    TIME_WITH_SYS_TIME)
+  set(TIME_WITH_SYS_TIME ${TIME_WITH_SYS_TIME} PARENT_SCOPE)
+endfunction()
+
+
+# Native cpu byte order: 1 if big-endian (Motorola) or 0 if little-endian
+# (Intel), setting "WORDS_BIGENDIAN" to 1 if big endian
+function (ac_c_bigendian)
+  TEST_BIG_ENDIAN(HOST_BIGENDIAN)
+  if (HOST_BIGENDIAN)
+    set(WORDS_BIGENDIAN 1 PARENT_SCOPE)
+  endif()
+endfunction()
+
+
+# Check for off_t, setting "off_t" in the parent scope
+function(ac_type_off_t)
+  CHECK_TYPE_SIZE("off_t" SIZEOF_OFF_T)
+  if (NOT SIZEOF_OFF_T)
+    set(off_t "long int")
+  endif()
+  set(off_t ${off_t} PARENT_SCOPE)
+endfunction()
+
+
+# Check for size_t, setting "size_t" in the parent scope
+function(ac_type_size_t)
+  CHECK_TYPE_SIZE("size_t" SIZEOF_SIZE_T)
+  if (NOT SIZEOF_SIZE_T)
+    set(size_t "unsigned int")
+  endif()
+  set(size_t ${size_t} PARENT_SCOPE)
+endfunction()
+
+
+# Define "TM_IN_SYS_TIME" to 1 if <sys/time.h> declares "struct tm"
+function(ac_struct_tm)
+  CHECK_C_SOURCE_COMPILES(
+    "#include <sys/time.h>\nint main(int argc, char **argv) { struct tm x; return 0; }"
+    TM_IN_SYS_TIME
+  )
+  if (TM_IN_SYS_TIME)
+    set (TM_IN_SYS_TIME 1 PARENT_SCOPE)
+  endif()
+endfunction()
+
+
+# Obtain size of an 'type' and define as SIZEOF_TYPE
+function (ac_check_sizeof typename)
+  string(TOUPPER "SIZEOF_${typename}" varname)
+	string(REPLACE " " "_" varname "${varname}")
+  string(REPLACE "*" "p" varname "${varname}")
+  CHECK_TYPE_SIZE("${typename}" ${varname} BUILTIN_TYPES_ONLY)
+  if(NOT ${varname})
+    set(${varname} 0 PARENT_SCOPE)
+  endif()
+endfunction()
+
+
+# Check if the type exists, defines HAVE_<type>
+function (ac_check_type typename)
+  string(TOUPPER "${typename}" varname)
+	string(REPLACE " " "_" varname "${varname}")
+  string(REPLACE "*" "p" varname "${varname}")
+  CHECK_TYPE_SIZE("${typename}" ${varname})
+  if (NOT "${varname}" STREQUAL "")
+    set("HAVE_${varname}" 1 PARENT_SCOPE)
+    set("${varname}" "${typename}" PARENT_SCOPE)
+  else()
+    set("${varname}" "unknown" PARENT_SCOPE)
+  endif()
+endfunction()
+
+
+# Verifies if each type on the list exists, using the given prelude
+function (ac_check_types type_list prelude)
+  foreach(typename ${type_list})
+    string(TOUPPER "HAVE_${typename}" varname)
+    string(REPLACE " " "_" varname "${varname}")
+    string(REPLACE "*" "p" varname "${varname}")
+    CHECK_C_SOURCE_COMPILES("${prelude}\n ${typename} foo;" ${varname})
+  endforeach()
+endfunction()
+
+function(ac_path_prog variable prog_to_check_for value_if_not_found env_var)
+    find_program(${variable} NAMES ${prog_to_check_for} PATHS ENV ${env_var} NO_DEFAULT_PATH)
+    if(NOT ${variable})
+        message(STATUS "Looking for ${prog_to_check_for} - not found")
+        set(${variable} ${value_if_not_fount} PARENT_SCOPE)
+    else()
+        message(STATUS "Looking for ${prog_to_check_for} - ${variable}")
+        set(${variable} ${${variable}} PARENT_SCOPE)
+    endif()
+endfunction()
+
+# check if function func exists in library lib
+function(ac_check_lib lib func)
+    string(TOUPPER "HAVE_${func}" varname)
+    set(CMAKE_REQUIRED_LIBRARIES ${lib})
+    check_function_exists(${func} ${varname})
+    set(CMAKE_REQUIRED_LIBRARIES)
+endfunction()
+
+# check if source compiles without linking
+function(ac_try_compile SOURCE VAR)
+    set(CMAKE_TMP_DIR ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp)
+    if(NOT DEFINED "${VAR}")
+        file(WRITE
+            "${CMAKE_TMP_DIR}/src.c"
+            "${SOURCE}\n"
+        )
+
+        if(NOT CMAKE_REQUIRED_QUIET)
+            message(STATUS "Performing Test ${VAR}")
+        endif()
+        # Set up CMakeLists.txt for static library:
+        file(WRITE
+            ${CMAKE_TMP_DIR}/CMakeLists.txt
+            "add_library(compile STATIC src.c)"
+        )
+
+        # Configure:
+        execute_process(
+            COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" .
+            WORKING_DIRECTORY ${CMAKE_TMP_DIR}
+        )
+
+        # Build:
+        execute_process(
+            COMMAND ${CMAKE_COMMAND} --build ${CMAKE_TMP_DIR}
+            RESULT_VARIABLE RESVAR
+            OUTPUT_VARIABLE OUTPUT
+            ERROR_VARIABLE OUTPUT
+        )
+
+        # Set up result:
+        if(RESVAR EQUAL 0)
+            set(${VAR} 1 CACHE INTERNAL "Test ${VAR}")
+            if(NOT CMAKE_REQUIRED_QUIET)
+                message(STATUS "Performing Test ${VAR} - Success")
+            endif()
+
+            file(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeOutput.log
+                "Performing C SOURCE FILE Test ${VAR} succeded with the following output:\n"
+                "${OUTPUT}\n"
+                "Source file was:\n${SOURCE}\n")
+        else()
+            if(NOT CMAKE_REQUIRED_QUIET)
+                message(STATUS "Performing Test ${VAR} - Failed")
+            endif()
+            set(${VAR} "" CACHE INTERNAL "Test ${VAR}")
+            file(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log
+            "Performing C SOURCE FILE Test ${VAR} failed with the following output:\n"
+            "${OUTPUT}\n"
+            "Source file was:\n${SOURCE}\n")
+        endif()
+    endif()
+endfunction()

cmake/modules/FindLibElf.cmake

@@ -0,0 +1,64 @@
+# - Try to find libelf
+# Once done this will define
+#
+#  LIBELF_FOUND - system has libelf
+#  LIBELF_INCLUDE_DIRS - the libelf include directory
+#  LIBELF_LIBRARIES - Link these to use libelf
+#  LIBELF_DEFINITIONS - Compiler switches required for using libelf
+#
+# This module reads hints about search locations from variables:
+#
+#  LIBELF_ROOT - Preferred installation prefix
+#
+#  Copyright (c) 2008 Bernhard Walle <bernhard.walle@gmx.de>
+#
+#  Redistribution and use is allowed according to the terms of the New
+#  BSD license.
+#  For details see the accompanying COPYING-CMAKE-SCRIPTS file.
+#
+
+
+if (LIBELF_LIBRARIES AND LIBELF_INCLUDE_DIRS)
+  set (LibElf_FIND_QUIETLY TRUE)
+endif (LIBELF_LIBRARIES AND LIBELF_INCLUDE_DIRS)
+
+find_path (LIBELF_INCLUDE_DIRS
+  NAMES
+    libelf/libelf.h libelf.h
+  HINTS
+    ${LIBELF_ROOT}
+  PATH_SUFFIXES
+    include
+	libelf/include
+)
+
+find_library (LIBELF_LIBRARIES
+  NAMES
+    elf libelf
+  HINTS
+    ${LIBELF_ROOT}
+  PATH_SUFFIXES
+    lib
+	libelf/lib
+)
+
+include (FindPackageHandleStandardArgs)
+
+
+# handle the QUIETLY and REQUIRED arguments and set LIBELF_FOUND to TRUE if all listed variables are TRUE
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(LibElf DEFAULT_MSG
+  LIBELF_LIBRARIES
+  LIBELF_INCLUDE_DIRS)
+
+set(CMAKE_REQUIRED_LIBRARIES elf)
+include(CheckCXXSourceCompiles)
+check_cxx_source_compiles("#include <libelf.h>
+int main() {
+  Elf *e = (Elf*)0;
+  size_t sz;
+  elf_getshdrstrndx(e, &sz);
+  return 0;
+}" ELF_GETSHDRSTRNDX)
+unset(CMAKE_REQUIRED_LIBRARIES)
+
+mark_as_advanced(LIBELF_INCLUDE_DIRS LIBELF_LIBRARIES ELF_GETSHDRSTRNDX)

docs/Makefile

@@ -0,0 +1,29 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+po:
+	sphinx-build -b gettext . _build/gettext
+	/opt/local/Library/Frameworks/Python.framework/Versions/3.8/bin/sphinx-intl update -p _build/gettext -l ja
+
+ja:
+	sphinx-build -b html -D language=ja . _build/html/ja
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	echo ".. figure:: docs/mckernel-logo.png" > ../README.rst; echo >> ../README.rst
+	cat summary.rst >> ../README.rst; echo >> ../README.rst
+	cat doc.rst >> ../README.rst
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/NEWS.rst

@@ -0,0 +1,421 @@
+=============================================
+Version 1.7.0rc4 (Apr 15, 2020)
+=============================================
+
+----------------------
+McKernel major updates
+----------------------
+1. arm64: Contiguous PTE support
+2. arm64: Scalable Vector Extension (SVE) support
+3. arm64: PMU overflow interrupt support
+4. xpmem: Support large page attachment
+5. arm64 port: Direct access to Mckernel memory from Linux
+6. arm64 port: utility thread offloading, which spawns thread onto Linux CPU
+7. eclair: support for live debug
+8. Crash utility extension
+9. Replace mcoverlayfs with a soft userspace overlay
+10. Build system is switched to cmake
+11. Core dump includes thread information
+
+------------------------
+McKernel major bug fixes
+------------------------
+#. shmobj: Fix rusage counting for large page
+#. mcctrl control: task start_time changed to u64 nsec
+#. mcctrl: add handling for one more level of page tables
+#. Add kernel argument to turn on/off time sharing
+#. flatten_string / process env: realign env and clear trailing bits
+#. madvise: Add MADV_HUGEPAGE support
+#. mcctrl: remove in-kernel calls to syscalls
+#. arch_cpu_read_write_register: error return fix.
+#. set_cputime(): interrupt enable/disable fix.
+#. set_mempolicy(): Add mode check.
+#. mbind(): Fix memory_range_lock deadlock.
+#. ihk_ikc_recv: Record channel to packet for release
+#. Add set_cputime() kernel to kernel case and mode enum.
+#. execve: Call preempt_enable() before error-exit
+#. memory/x86_64: fix linux safe_kernel_map
+#. do_kill(): fix pids table when nr of threads is larger than num_processors
+#. shmget: Use transparent huge pages when page size isn't specified
+#. prctl: Add support for PR_SET_THP_DISABLE and PR_GET_THP_DISABLE
+#. monitor_init: fix undetected hang on highest numbered core
+#. init_process_stack: change premapped stack size based on arch
+#. x86 syscalls: add a bunch of XXat() delegated syscalls
+#. do_pageout: fix direct kernel-user access
+#. stack: add hwcap auxval
+#. perf counters: add arch-specific perf counters
+#. Added check of nohost to terminate_host().
+#. kmalloc: Fix address order in free list
+#. sysfs: use nr_cpu_ids for cpumasks (fixes libnuma parsing error on ARM)
+#. monitor_init: Use ihk_mc_cpu_info()
+#. Fix ThunderX2 write-combined PTE flag insanity
+#. ARM: eliminate zero page mapping (i.e, init_low_area())
+#. eliminate futex_cmpxchg_enabled check (not used and dereffed a NULL pointer)
+#. page_table: Fix return value of lookup_pte when ptl4 is blank
+#. sysfs: add missing symlinks for cpu/node
+#. Make Linux handler run when mmap to procfs.
+#. Separate mmap area from program loading (relocation) area
+#. move rusage into kernel ELF image (avoid dynamic alloc before NUMA init)
+#. arm: turn off cpu on panic
+#. page fault handler: protect thread accesses
+#. Register PPD and release_handler at the same time.
+#. fix to missing exclusive processing between terminate() and finalize_process().
+#. perfctr_stop: add flags to no 'disable_intens'
+#. fileobj, shmobj: free pages in object destructor (as opposed to page_unmap())
+#. clear_range_l1, clear_range_middle: Fix handling contiguous PTE
+#. do_mmap: don't pre-populate the whole file when asked for smaller segment
+#. invalidate_one_page: Support shmobj and contiguous PTE
+#. ubsan: fix undefined shifts
+#. x86: disable zero mapping and add a boot pt for ap trampoline
+#. rusage: Don't count PF_PATCH change
+#. Fixed time processing.
+#. copy_user_pte: vmap area not owned by McKernel
+#. gencore: Zero-clear ELF header and memory range table
+#. rpm: ignore CMakeCache.txt in dist and relax BuildRequires on cross build
+#. gencore: Allocate ELF header to heap instead of stack
+#. nanosleep: add cpu_pause() in spinwait loop
+#. init_process: add missing initializations to proc struct
+#. rus_vm_fault: always use a packet on the stack
+#. process stack: use PAGE_SIZE in aux vector
+#. copy_user_pte: base memobj copy on range & VR_PRIVATE
+#. arm64: ptrace: Fix overwriting 1st argument with return value
+#. page fault: use cow for private device mappings
+#. reproductible builds: remove most install paths in c code
+#. page fault: clear writable bit for non-dirtying access to shared ranges
+#. mcreboot/mcstop+release: support for regular user execution
+#. irqbalance_mck: replace extra service with service drop-in
+#. do_mmap: give addr argument a chance even if not MAP_FIXED
+#. x86: fix xchg() and cmpxchg() macros
+#. IHK: support for using Linux work IRQ as IKC interrupt (optional)
+#. MCS: fix ARM64 issue by using smp_XXX() functions (i.e., barrier()s)
+#. procfs: add number of threads to stat and status
+#. memory_range_lock: Fix deadlock in procfs/sysfs handler
+#. flush instruction cache at context switch time if necessary
+#. arm64: Fix PMU related functions
+#. page_fault_process_memory_range: Disable COW for VM region with zeroobj
+#. extend_process_region: Fall back to demand paging when not contiguous
+#. munmap: fix deadlock with remote pagefault on vm range lock
+#. procfs: if memory_range_lock fails, process later
+#. migrate-cpu: Prevent migration target from calling schedule() twice
+#. sched_request_migrate(): fix race condition between migration req and IRQs
+#. get_one_cpu_topology: Renumber core_id (physical core id)
+#. bb7e140 procfs cpuinfo: use sequence number as processor
+#. set_host_vma(): do NOT read protect Linux VMA
+
+===========================================
+Version 1.6.0 (Nov 11, 2018)
+===========================================
+
+-----------------------------------------------
+McKernel major updates
+-----------------------------------------------
+#. McKernel and Linux share one unified kernel virtual address space.
+   That is, McKernel sections resides in Linux sections spared for
+   modules.  In this way, Linux can access the McKernel kernel memory area.
+#. hugetlbfs support
+#. IHK is now included as a git submodule
+#. Debug messages are turned on/off in per souce file basis at run-time.
+#. It's prohibited for McKernel to access physical memory ranges which Linux didn't give to McKernel.
+#. UTI (capability to spawn a thread on Linux CPU) improvement:
+
+   * System calls issued from the thread are hooked by modifying binary in memory.
+
+---------------------------
+McKernel major bug fixes
+---------------------------
+#<digits> below denotes the redmine issue number (https://postpeta.pccluster.org/redmine/).
+
+1. #926: shmget: Hide object with IPC_RMID from shmget
+2. #1028: init_process: Inherit parent cpu_set
+3. #995: Fix shebang recorded in argv[0]
+4. #1024: Fix VMAP virtual address leak
+5. #1109: init_process_stack: Support "ulimit -s unlimited"
+6. x86 mem init: do not map identity mapping
+7. mcexec_wait_syscall: requeue potential request on interrupted wait
+8. mcctrl_ikc_send_wait: fix interrupt with do_frees == NULL
+9. pager_req_read: handle short read
+10. kprintf: only call eventfd() if it is safe to interrupt
+11. process_procfs_request: Add Pid to /proc/<PID>/status
+12. terminate: fix oversubscribe hang when waiting for other threads on same CPU to die
+13. mcexec: Do not close fd returned to mckernel side
+14. #976: execve: Clear sigaltstack and fp_regs
+15. #1002: perf_event: Specify counter by bit_mask on start/stop
+16. #1027: schedule: Don't reschedule immediately when wake up on migrate
+17. #mcctrl: lookup unexported symbols at runtime
+18. __sched_wakeup_thread: Notify interrupt_exit() of re-schedule
+19. futex_wait_queue_me: Spin-sleep when timeout and idle_halt is specified
+20. #1167: ihk_os_getperfevent,setperfevent: Timeout IKC sent by mcctrl
+21. devobj: fix object size (POSTK_DEBUG_TEMP_FIX_36)
+22. mcctrl: remove rus page cache
+23. #1021: procfs: Support multiple reads of e.g. ``/proc/*/maps``
+24. #1006: wait: Delay wake-up parent within switch context
+25. #1164: mem: Check if phys-mem is within the range of McKernel memory
+26. #1039: page_fault_process_memory_range: Remove ihk_mc_map_virtual for CoW of device map
+27. partitioned execution: pass process rank to LWK
+28. process/vm: implement access_ok()
+29. spinlock: rewrite spinlock to use Linux ticket head/tail format
+30. #986: Fix deadlock involving mmap_sem and memory_range_lock
+31. Prevent one CPU from getting chosen by concurrent forks
+32. #1009: check_signal: system call restart is done only once
+33. #1176: syscall: the signal received during system call processing is not processed.
+34. #1036 syscall_time: Handle by McKernel
+35. #1165 do_syscall: Delegate system calls to the mcexec with the same pid
+36. #1194 execve: Fix calling ptrace_report_signal after preemption is disabled
+37. #1005 coredump: Exclude special areas
+38. #1018 procfs: Fix pread/pwrite to procfs fail when specified size is bigger than 4MB
+39. #1180 sched_setaffinity: Check migration after decrementing in_interrupt
+40. #771, #1179, #1143 ptrace supports threads
+41. #1189 procfs/do_fork: wait until procfs entries are registered
+42. #1114 procfs: add '/proc/pid/stat' to mckernel side and fix its comm
+43. #1116 mcctrl procfs: check entry was returned before using it
+44. #1167 ihk_os_getperfevent,setperfevent: Return -ETIME when IKC timeouts
+45. mcexec/execve: fix shebangs handling
+46. procfs: handle 'comm' on mckernel side
+47. ihk_os_setperfevent: Return number of registered events
+48. mcexec: fix terminating zero after readlink()
+
+===========================================
+Version 1.5.1 (July 9, 2018)
+===========================================
+
+-----------------------------------------------
+McKernel major updates
+-----------------------------------------------
+
+Watchdog timer to detect hang of McKernel
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+mcexec prints out the following line to its stderr when a hang of McKernel is detected.
+
+::
+   
+       mcexec detected hang of McKernel
+
+The watchdog timer is enabled by passing -i <timeout_in_sec> option to mcreboot.sh. <timeout_in_sec> specifies the interval of checking if McKernel is alive.
+
+For example, specify ``-i 600`` to detect the hang with 10 minutes interval:
+
+::
+
+   mcreboot.sh -i 600
+
+The detailed step of the hang detection is as follows.
+   #. mcexec acquires eventfd for notification from IHK and perform epoll() on it.
+   #. A daemon called ihkmond monitors the state of McKernel periodically with the interval specified by the -i option. It judges that McKernel is hanging and notifies mcexec by the eventfd if its state hasn't changed since the last check.
+
+---------------------------
+McKernel major bug fixes
+---------------------------
+1.  #1146: pager_req_map(): do not take mmap_sem if not needed
+2.  #1135: prepare_process_ranges_args_envs(): fix saving cmdline
+3.  #1144: fileobj/devobj: record path name
+4.  #1145: fileobj: use MCS locks for per-file page hash
+5.  #1076: mcctrl: refactor prepare_image into new generic ikc send&wait
+6.  #1072: execve: fix execve with oversubscribing
+7.  #1132: execve: use thread variable instead of cpu_local_var(current)
+8.  #1117: mprotect: do not set page table writable for cow pages
+9.  #1143: syscall wait4: add _WALL (POSTK_DEBUG_ARCH_DEP_44)
+10. #1064: rusage: Fix initialization of rusage->num_processors
+11. #1133: pager_req_unmap: Put per-process data at exit
+12. #731: do_fork: Propagate error code returned by mcexec
+13. #1149: execve: Reinitialize vm_regions's map area on execve
+14. #1065: procfs: Show file names in /proc/<PID>/maps
+15. #1112: mremap: Fix type of size arguments (from ssize_t to size_t)
+16. #1121: sched_getaffinity: Check arguments in the same order as in Linux
+17. #1137: mmap, mremap: Check arguments in the same order as in Linux
+18. #1122: fix return value of sched_getaffinity
+19. #732: fix: /proc/<PID>/maps outputs a unnecessary NULL character
+
+===================================
+Version 1.5.0 (Apr 5, 2018)
+===================================
+
+--------------------------------------
+McKernel major updates
+--------------------------------------
+1. Aid for Linux version migration: Detect /proc, /sys format change
+   between two kernel verions
+2. Swap out
+   * Only swap-out anonymous pages for now
+3. Improve support of /proc/maps
+4. mcstat: Linux tool to show resource usage
+
+---------------------------
+McKernel major bug fixes
+---------------------------
+#. #727: execve: Fix memory leak when receiving SIGKILL
+#. #829: perf_event_open: Support PERF_TYPE_HARDWARE and PERF_TYPE_HW_CACHE
+#. #906: mcexec: Check return code of fork()
+#. #1038: mcexec: Timeout when incorrect value is given to -n option
+#. #943 #945 #946 #960 #961: mcexec: Support strace
+#. #1029: struct thread is not released with stress-test involving signal and futex
+#. #863 #870 : Respond immediately to terminating signal when offloading system call
+#. #1119: translate_rva_to_rpa(): use 2MB blocks in 1GB pages on x86
+#. #898: Shutdown OS only after no in-flight IKC exist
+#. #882: release_handler: Destroy objects as the process which opened it
+#. #882: mcexec: Make child process exit if the parent is killed during fork()
+#. #925: XPMEM: Don't destroy per-process object of the parent
+#. #885: ptrace: Support the case where a process attaches its child
+#. #1031: sigaction: Support SA_RESETHAND
+#. #923: rus_vm_fault: Return error when a thread not performing system call offloading causes remote page fault
+#. #1032 #1033 #1034: getrusage: Fix ru_maxrss, RUSAGE_CHILDREN, ru_stime related bugs
+#. #1120: getrusage: Fix deadlock on thread->times_update
+#. #1123: Fix deadlock related to wait_queue_head_list_node
+#. #1124: Fix deadlock of calling terminate() from terminate()
+#. #1125: Fix deadlock related to thread status
+
+   * Related functions are: hold_thread(), do_kill() and terminate()
+
+#. #1126: uti: Fix uti thread on the McKernel side blocks others in do_syscall()
+#. #1066: procfs: Show Linux /proc/self/cgroup
+#. #1127: prepare_process_ranges_args_envs(): fix generating saved_cmdline to avoid PF in strlen()
+#. #1128: ihk_mc_map/unmap_virtual(): do proper TLB invalidation
+#. #1043: terminate(): fix update_lock and threads_lock order to avoid deadlock
+#. #1129: mcreboot.sh: Save ``/proc/irq/*/smp_affinity`` to ``/tmp/mcreboot``
+#. #1130: mcexec: drop READ_IMPLIES_EXEC from personality
+
+--------------------
+McKernel workarounds
+--------------------
+#. Forbid CPU oversubscription
+
+   * It can be turned on by mcreboot.sh -O option
+
+
+===================================
+Version 1.4.0 (Oct 30, 2017)
+===================================
+
+-----------------------------------------------------------
+Abstracted event type support in perf_event_open()
+-----------------------------------------------------------
+
+PERF_TYPE_HARDWARE and PERF_TYPE_CACHE types are supported.
+
+----------------------------------
+Direct user-space access
+----------------------------------
+Code lines using direct user-space access (e.g. passing user-space
+pointer to memcpy()) becomes more portable across processor
+architectures. The modification follows the following rules.
+
+1. Move the code section as it is to the architecture dependent
+   directory if it is a part of the critical-path.
+2. Otherwise, rewrite the code section by using the portable methods.
+   The methods include copy_from_user(), copy_to_user(),
+   pte_get_phys() and phys_to_virt().
+
+--------------------------------
+MPI and OpenMP micro-bench tests
+--------------------------------
+The performance figures of MPI and OpenMP primitives are compared with
+those of Linux by using Intel MPI Benchmarks and EPCC OpenMP Micro
+Benchmark.
+
+
+===================================
+Version 1.3.0 (Sep 30, 2017)
+===================================
+
+--------------------
+Kernel dump
+--------------------
+#. A dump level of "only kernel memory" is added.
+
+The following two levels are available now:
+
++--+-----------------------+
+| 0|Dump all               |
++--+-----------------------+
+|24|Dump only kernel memory|
++--+-----------------------+
+
+The dump level can be set by -d option in ihkosctl or the argument
+for ihk_os_makedumpfile(), as shown in the following examples:
+
+::
+
+   Command:		ihkosctl 0 dump -d 24
+   Function call:	ihk_os_makedumpfile(0, NULL, 24, 0);
+
+#. Dump file is created when Linux panics.
+
+The dump level can be set by dump_level kernel argument, as shown in the
+following example:
+
+::
+
+   ihkosctl 0 kargs "hidos dump_level=24"
+
+The IHK dump function is registered to panic_notifier_list when creating /dev/mcdX and called when Linux panics.
+
+-----------------------------
+Quick Process Launch
+-----------------------------
+
+MPI process launch time and some of the initialization time can be
+reduced in application consisting of multiple MPI programs which are
+launched in turn in the job script.
+
+The following two steps should be performed to use this feature:
+#. Replace mpiexec with ql_mpiexec_start and add some lines for ql_mpiexec_finalize in the job script
+#. Modify the app so that it can repeat calculations and wait for the instructions from ql_mpiexec_{start,finalize} at the end of the loop
+
+The first step is explained using an example. Assume the original job script looks like this:
+
+.. code-block:: none
+   
+   /* Execute ensamble simulation and then data assimilation, and repeat this ten times */
+   for i in {1..10}; do
+   
+      /* Each ensamble simulation execution uses 100 nodes, launch ten of them in parallel */
+      for j in {1..10}; do
+         mpiexec -n 100 -machinefile ./list1_$j p1.out a1 & pids[$i]=$!;
+      done
+
+      /* Wait until the ten ensamble simulation programs finish */
+      for j in {1..10}; do wait ${pids[$j]}; done
+      
+      /* Launch one data assimilation program using 1000 nodes */
+      mpiexec -n 1000 -machinefile ./list2 p2.out a2
+   done
+   
+The job script should be modified like this:
+
+.. code-block:: none
+
+   for i in {1..10}; do
+      for j in {1..10}; do
+         /*  Replace mpiexec with ql_mpiexec_start */
+         ql_mpiexec_start -n 100 -machinefile ./list1_$j p1.out a1 & pids[$j]=$!;
+      done
+      
+      for j in {1..10}; do wait ${pids[$j]}; done
+      
+      ql_mpiexec_start -n 1000 -machinefile ./list2 p2.out a2
+   done
+   
+   /* p1.out and p2.out don't exit but are waiting for the next calculation. So tell them to exit */
+   for j in {1..10}; do
+      ql_mpiexec_finalize -machinefile ./list1_$i p1.out a1;
+   done
+   
+   ql_mpiexec_finalize -machinefile ./list2 p2.out a2;
+
+The second step is explained using a pseudo-code.
+
+.. code-block:: none
+
+   MPI_Init();
+   Prepare data exchange with preceding / following MPI programs
+   loop:
+   foreach Fortran module
+      Initialize data using command-line argments, parameter files, environment variables
+      Input data from preceding MPI programs / Read snap-shot
+      Perform main calculation
+      Output data to following MPI programs / Write snap-shot
+      /* ql_client() waits for command of ql_mpiexec_{start,finish} */
+      if (ql_client() == QL_CONTINUE) { goto loop; }
+      MPI_Finalize();
+
+qlmpilib.h should be included in the code and libql{mpi,fort}.so should be linked to the executable file.

docs/archtecture.rst

@@ -0,0 +1,65 @@
+Architectural Overview
+======================
+
+At the heart of the stack is a low-level software infrastructure called
+Interface for Heterogeneous Kernels (IHK). IHK is a general framework
+that provides capabilities for partitioning resources in a many-core
+environment (e.g.,CPU cores and physical memory) and it enables
+management of lightweight kernels. IHK can allocate and release host
+resources dynamically and no reboot of the host machine is required when
+altering configuration. IHK also provides a low-level inter-kernel
+messaging infrastructure, called the Inter-Kernel Communication (IKC)
+layer. An architectural overview of the main system components is shown
+below.
+
+.. figure:: mckernel.png
+   :alt: arch
+
+McKernel is a lightweight kernel written from scratch. It is designed
+for HPC and is booted from IHK. McKernel retains a binary compatible ABI
+with Linux, however, it implements only a small set of performance
+sensitive system calls and the rest are offloaded to Linux.
+Specifically, McKernel has its own memory management, it supports
+processes and multi-threading with a simple round-robin cooperative
+(tick-less) scheduler, and it implements signaling. It also allows
+inter-process memory mappings and it provides interfaces to hardware
+performance counters.
+
+Functionality
+-------------
+
+An overview of some of the principal functionalities of the IHK/McKernel
+stack is provided below.
+
+System Call Offloading
+~~~~~~~~~~~~~~~~~~~~~~
+
+System call forwarding in McKernel is implemented as follows. When an
+offloaded system call occurs, McKernel marshals the system call number
+along with its arguments and sends a message to Linux via a dedicated
+IKC channel. The corresponding proxy process running on Linux is by
+default waiting for system call requests through an ioctl() call into
+IHK’s system call delegator kernel module. The delegator kernel module’s
+IKC interrupt handler wakes up the proxy process, which returns to
+userspace and simply invokes the requested system call. Once it obtains
+the return value, it instructs the delegator module to send the result
+back to McKernel, which subsequently passes the value to user-space.
+
+Unified Address Space
+~~~~~~~~~~~~~~~~~~~~~
+
+The unified address space model in IHK/McKernel ensures that offloaded
+system calls can seamlessly resolve arguments even in case of pointers.
+This mechanism is depicted below and is implemented as follows.
+
+.. figure:: unified_address_space_en.png
+   :alt: unified_ap
+
+First, the proxy process is compiled as a position independent binary,
+which enables us to map the code and data segments specific to the proxy
+process to an address range which is explicitly excluded from McKernel’s
+user space. The grey box on the right side of the figure demonstrates
+the excluded region. Second, the entire valid virtual address range of
+McKernel’s application user-space is covered by a special mapping in the
+proxy process for which we use a pseudo file mapping in Linux. This
+mapping is indicated by the blue box on the left side of the figure.

docs/background.rst

@@ -0,0 +1,47 @@
+Background and Motivation
+=========================
+
+With the growing complexity of high-end supercomputers, the current
+system software stack faces significant challenges as we move forward to
+exascale and beyond. The necessity to deal with extreme degree of
+parallelism, heterogeneous architectures, multiple levels of memory
+hierarchy, power constraints, etc., advocates operating systems that can
+rapidly adapt to new hardware requirements, and that can support novel
+programming paradigms and runtime systems. On the other hand, a new
+class of more dynamic and complex applications are also on the horizon,
+with an increasing demand for application constructs such as in-situ
+analysis, workflows, elaborate monitoring and performance tools. This
+complexity relies not only on the rich features of POSIX, but also on
+the Linux APIs (such as the */proc*, */sys* filesystems, etc.) in
+particular.
+
+Two Traditional HPC OS Approaches
+---------------------------------
+
+Traditionally, light-weight operating systems specialized for HPC
+followed two approaches to tackle scalable execution of large-scale
+applications. In the full weight kernel (FWK) approach, a full Linux
+environment is taken as the basis, and features that inhibit attaining
+HPC scalability are removed, i.e., making it light-weight. The pure
+light-weight kernel (LWK) approach, on the other hand, starts from
+scratch and effort is undertaken to add sufficient functionality so that
+it provides a familiar API, typically something close to that of a
+general purpose OS, while at the same time it retains the desired
+scalability and reliability attributes. Neither of these approaches
+yields a fully Linux compatible environment.
+
+The Multi-kernel Approach
+-------------------------
+
+A hybrid approach recognized recently by the system software community
+is to run Linux simultaneously with a lightweight kernel on compute
+nodes and multiple research projects are now pursuing this direction.
+The basic idea is that simulations run on an HPC tailored lightweight
+kernel, ensuring the necessary isolation for noiseless execution of
+parallel applications, but Linux is leveraged so that the full POSIX API
+is supported. Additionally, the small code base of the LWK can also
+facilitate rapid prototyping for new, exotic hardware features.
+Nevertheless, the questions of how to share node resources between the
+two types of kernels, where do device drivers execute, how exactly do
+the two kernels interact with each other and to what extent are they
+integrated, remain subjects of ongoing debate.

docs/boot_run_shutdown.rst

@@ -0,0 +1,60 @@
+Boot McKernel
+----------------
+
+A boot script called ``mcreboot.sh`` is provided under ``sbin`` in the install
+folder. To boot on logical CPU 1 with 512MB of memory, use the following
+invocation:
+
+::
+
+   export TOP=${HOME}/ihk+mckernel/
+   cd ${TOP}
+   sudo ./sbin/mcreboot.sh -c 1 -m 512m
+
+You should see something similar like this if you display the McKernel’s
+kernel message log:
+
+.. code-block:: none
+
+   ./sbin/ihkosctl 0 kmsg
+
+   IHK/McKernel started.
+   [ -1]: no_execute_available: 1
+   [ -1]: map_fixed: phys: 0xfee00000 => 0xffff860000009000 (1 pages)
+   [ -1]: setup_x86 done.
+   [ -1]: ns_per_tsc: 385
+   [ -1]: KCommand Line: hidos    dump_level=24
+   [ -1]: Physical memory: 0x1ad3000 - 0x21000000, 525520896 bytes, 128301 pages available @ NUMA: 0
+   [ -1]: NUMA: 0, Linux NUMA: 0, type: 1, available bytes: 525520896, pages: 128301
+   [ -1]: NUMA 0 distances: 0 (10),
+   [ -1]: map_fixed: phys: 0x28000 => 0xffff86000000a000 (2 pages)
+   [ -1]: Trampoline area: 0x28000
+   [ -1]: map_fixed: phys: 0x0 => 0xffff86000000c000 (1 pages)
+   [ -1]: # of cpus : 1
+   [ -1]: locals = ffff880001af6000
+   [  0]: BSP: 0 (HW ID: 1 @ NUMA 0)
+   [  0]: BSP: booted 0 AP CPUs
+   [  0]: Master channel init acked.
+   [  0]: vdso is enabled
+   IHK/McKernel booted.
+
+Run a simple program on McKernel
+-----------------------------------
+
+The mcexec command line tool (which is also the Linux proxy process) can
+be used for executing applications on McKernel:
+
+::
+
+   ./bin/mcexec hostname
+   centos-vm
+
+Shutdown McKernel
+--------------------
+
+Finally, to shutdown McKernel and release CPU/memory resources back to
+Linux use the following command:
+
+::
+
+   sudo ./sbin/mcstop+release.sh

docs/conf.py

@@ -0,0 +1,173 @@
+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file does only contain a selection of the most common options. For a
+# full list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = u'IHK/McKernel'
+copyright = u'2020, Masamichi Takagi, Balazs Gerofi, Yutaka Ishikawa'
+author = u'Masamichi Takagi, Balazs Gerofi, Yutaka Ishikawa'
+
+# The short X.Y version
+version = u''
+# The full version, including alpha/beta/rc tags
+release = u''
+
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'recommonmark',
+]
+numfig = True
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+source_suffix = ['.rst', '.md']
+
+# The master toctree document.
+master_doc = 'index'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = [u'_build', 'Thumbs.db', '.DS_Store']
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = None
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'sphinx_rtd_theme'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself.  Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'IHKMcKerneldoc'
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'IHKMcKernel.tex', u'IHK/McKernel Documentation',
+     u'Masamichi Takagi, Balazs Gerofi, Yutaka Ishikawa', 'manual'),
+]
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'ihkmckernel', u'IHK/McKernel Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'IHKMcKernel', u'IHK/McKernel Documentation',
+     author, 'IHKMcKernel', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+
+# -- Options for Epub output -------------------------------------------------
+
+# Bibliographic Dublin Core info.
+epub_title = project
+
+# The unique identifier of the text. This can be a ISBN number
+# or the project homepage.
+#
+# epub_identifier = ''
+
+# A unique identification for the text.
+#
+# epub_uid = ''
+
+# A list of files that should not be packed into the epub file.
+epub_exclude_files = ['search.html']

docs/contact.rst

@@ -0,0 +1,11 @@
+Contact
+=======
+
+Please give your feedback to us via one of the following mailing lists.
+Subscription via
+`www.pccluster.org <http://www.pccluster.org/mailman/listinfo/mckernel-users>`__
+is needed.
+
+-  English: mckernel-users@pccluster.org
+-  Japanese: mckernel-users-jp@pccluster.org
+

docs/cover.rst

@@ -0,0 +1,5 @@
+.. include:: logo.rst
+
+.. include:: summary.rst
+
+See `Quick Guide -- Installation <quick.html#installation>`__ for jump start.

docs/doc.rst

@@ -0,0 +1,5 @@
+Documentation
+=============
+
+Documentation is available
+`here <https://ihkmckernel.readthedocs.io>`__.

docs/ihk_developers.rst

@@ -0,0 +1,18 @@
+.. sectnum::
+   :suffix: .
+   :depth: 3
+
+External Specs
+==============
+
+Overview
+--------
+
+Function Specs
+--------------
+
+Command / Daemon Specs
+----------------------
+
+Booting LWK
+===========

docs/index.rst

@@ -0,0 +1,58 @@
+.. IHK/McKernel documentation master file, created by
+   sphinx-quickstart on Mon Jul 27 15:57:24 2020.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+===================
+IHK/McKernel
+===================
+
+.. include:: cover.rst
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Quick Guide
+
+   quick
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Users' Guide
+
+   users
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Operators' Guide
+
+   operators
+
+.. toctree::
+   :maxdepth: 2
+   :caption: IHK Developers' Guide
+
+   ihk_developers
+
+.. toctree::
+   :maxdepth: 2
+   :caption: McKernel Developers' Guide
+
+   mckernel_developers
+
+.. toctree::
+   :maxdepth: 2
+   :caption: IHK Specifications
+
+   spec/ihk
+
+.. toctree::
+   :maxdepth: 2
+   :caption: McKernel Specifications
+
+   spec/mckernel
+
+.. toctree::
+   :maxdepth: 2
+   :caption: What's New
+
+   NEWS

docs/install.rst

@@ -0,0 +1,202 @@
+.. highlight:: bash
+
+Installation
+============
+
+The following OS distributions and platforms are recommended:
+
+* OS distribution
+
+  * CentOS 7.3 or later
+  * RHEL 7.3 or later
+
+* Platform
+
+  * Intel Xeon
+  * Intel Xeon Phi
+  * Fujitsu A64FX
+
+Prepare files for building McKernel
+-----------------------------------
+
+Grant read permission to the System.map file of your kernel version on the build machine:
+
+::
+
+   sudo chmod a+r /boot/System.map-`uname -r`
+
+Install the following packages to the build machine:
+
+::
+
+   cmake kernel-devel binutils-devel systemd-devel numactl-devel gcc make nasm git libdwarf-devel
+
+When having access to repositories
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+On RHEL 8, enable the CodeReady Linux Builder (CLB) repository:
+
+::
+
+   sudo subscription-manager repos --enable codeready-builder-for-rhel-8-$(/bin/arch)-rpms
+
+On CentOS 8, enable the PowerTools repository:
+
+::
+
+   sudo dnf config-manager --set-enabled PowerTools
+
+Install with yum:
+
+::
+
+   sudo yum install cmake kernel-devel binutils-devel systemd-devel numactl-devel gcc make nasm git libdwarf-devel
+
+When not having access to repositories
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Ask the system administrator to install them. Note that ``libdwarf-devel`` is in the CodeReady Linux Builder repository on RHEL 8 or in the PowerTools repository on CentOS 8.
+
+Clone, compile, install
+--------------------------
+
+Clone the source code:
+
+::
+
+   mkdir -p ~/src/ihk+mckernel/
+   cd ~/src/ihk+mckernel/
+   git clone --recursive -b development https://github.com/RIKEN-SysSoft/mckernel.git
+
+(Optional) Checkout to the specific branch or version:
+
+::
+
+   cd mckernel
+   git checkout <pathspec>
+   git submodule update
+
+Foe example, if you want to try the development branch, use
+“development” as the pathspec. If you want to try the prerelease version
+1.7.0-0.2, use “1.7.0-0.2”.
+
+Move to build directory:
+
+::
+
+   mkdir -p ~/src/ihk+mckernel/build && cd ~/src/ihk+mckernel/build
+
+Run cmake:
+
+When not cross-compiling:
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+   cmake -DCMAKE_INSTALL_PREFIX=${HOME}/ihk+mckernel ../mckernel
+
+When cross-compiling:
+~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+   cmake -DCMAKE_INSTALL_PREFIX=${HOME}/ihk+mckernel \
+     -DUNAME_R=<target_uname_r> \
+     -DKERNEL_DIR=<kernnel_dir> \
+     -DBUILD_TARGET=smp-arm64 \
+     -DCMAKE_TOOLCHAIN_FILE=../mckernel/cmake/cross-aarch64.cmake \
+     ../mckernel
+
+Install with cmake
+~~~~~~~~~~~~~~~~~~~~~~
+
+Install with make:
+
+::
+
+   make -j install
+
+The kernel modules and McKernel kernel image should be installed
+under the **ihk+mckernel** folder in your home directory.
+
+Install with rpm
+~~~~~~~~~~~~~~~~~~~~
+
+Create the tarball and the spec file:
+
+::
+
+   make dist
+   cp mckernel-<version>.tar.gz <rpmbuild>/SOURCES
+
+Create the rpm package:
+
+When not cross-compiling:
+"""""""""""""""""""""""""
+
+::
+
+   rpmbuild -ba scripts/mckernel.spec
+
+When cross-compiling:
+"""""""""""""""""""""
+
+::
+
+   rpmbuild -ba scripts/mckernel.spec --target <target_uname_m> -D 'kernel_version <target_uname_r>' -D 'kernel_dir <kernel_source>'
+
+Install the rpm package:
+
+::
+
+   sudo rpm -ivh <rpmbuild>/RPMS/<arch>/mckernel-<version>-<release>_<linux_kernel_ver>_<dist>.<arch>.rpm
+
+The kernel modules and McKernel kernel image are installed under the
+standard system directories.
+
+Prepare files and change settings for installing McKernel
+---------------------------------------------------------
+
+Disable SELinux of the compute nodes:
+
+::
+
+   sudo vim /etc/selinux/config
+
+Change the file to SELINUX=disabled. And then reboot the compute nodes:
+
+::
+
+   sudo reboot
+
+Install the following packages to the compute nodes:
+
+::
+
+   systemd-libs numactl-libs libdwarf
+
+When having access to repositories
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+On RHEL 8, enable the CodeReady Linux Builder (CLB) repository:
+
+::
+
+   sudo subscription-manager repos --enable codeready-builder-for-rhel-8-$(/bin/arch)-rpms
+
+On CentOS 8, enable the PowerTools repository:
+
+::
+
+   sudo dnf config-manager --set-enabled PowerTools
+
+Install with yum:
+
+::
+
+   sudo yum install systemd-libs numactl-libs libdwarf
+
+When not having access to repositories
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Ask the system administrator to install them. Note that ``libdwarf`` is in the CodeReady Linux Builder repository on RHEL 8 or in the PowerTools repository on CentOS 8.

docs/license.rst

@@ -0,0 +1,5 @@
+License
+=======
+
+McKernel is GPL licensed, as found in the LICENSE file.
+

docs/locales/ja/LC_MESSAGES/README_.po

@@ -0,0 +1,469 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) 2020, Masamichi Takagi, Balazs Gerofi, Yutaka Ishikawa
+# This file is distributed under the same license as the IHK/McKernel
+# package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2020.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: IHK/McKernel \n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2020-08-04 16:40+0900\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=utf-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: Babel 2.7.0\n"
+
+#: ../../README_.rst:2
+msgid "|McKernel Logo|"
+msgstr ""
+
+#: ../../README_.rst:4
+msgid ""
+"IHK/McKernel is a light-weight multi-kernel operating system designed for"
+" high-end supercomputing. It runs Linux and McKernel, a light-weight "
+"kernel (LWK), side-by-side inside compute nodes and aims at the "
+"following:"
+msgstr ""
+
+#: ../../README_.rst:9
+msgid ""
+"Provide scalable and consistent execution of large-scale parallel "
+"scientific applications, but at the same time maintain the ability to "
+"rapidly adapt to new hardware features and emerging programming models"
+msgstr ""
+
+#: ../../README_.rst:13
+msgid ""
+"Provide efficient memory and device management so that resource "
+"contention and data movement are minimized at the system level"
+msgstr ""
+
+#: ../../README_.rst:15
+msgid ""
+"Eliminate OS noise by isolating OS services in Linux and provide jitter "
+"free execution on the LWK"
+msgstr ""
+
+#: ../../README_.rst:17
+msgid ""
+"Support the full POSIX/Linux APIs by selectively offloading (slow-path) "
+"system calls to Linux"
+msgstr ""
+
+#: ../../README_.rst:21
+msgid "Documentation"
+msgstr ""
+
+#: ../../README_.rst:23
+msgid "Documentation is available `here <https://ihkmckernel.readthedocs.io>`__."
+msgstr ""
+
+#: ../../README_.rst:27
+msgid "Contents"
+msgstr ""
+
+#: ../../README_.rst:29
+msgid "`Background <#background-and-motivation>`__"
+msgstr ""
+
+#: ../../README_.rst:30
+msgid "`Architectural Overview <#architectural-overview>`__"
+msgstr ""
+
+#: ../../README_.rst:31
+msgid "`Installation <#installation>`__"
+msgstr ""
+
+#: ../../README_.rst:32
+msgid "`The Team <#the-team>`__"
+msgstr ""
+
+#: ../../README_.rst:35
+msgid "Background and Motivation"
+msgstr ""
+
+#: ../../README_.rst:37
+msgid ""
+"With the growing complexity of high-end supercomputers, the current "
+"system software stack faces significant challenges as we move forward to "
+"exascale and beyond. The necessity to deal with extreme degree of "
+"parallelism, heterogeneous architectures, multiple levels of memory "
+"hierarchy, power constraints, etc., advocates operating systems that can "
+"rapidly adapt to new hardware requirements, and that can support novel "
+"programming paradigms and runtime systems. On the other hand, a new class"
+" of more dynamic and complex applications are also on the horizon, with "
+"an increasing demand for application constructs such as in-situ analysis,"
+" workflows, elaborate monitoring and performance tools. This complexity "
+"relies not only on the rich features of POSIX, but also on the Linux APIs"
+" (such as the */proc*, */sys* filesystems, etc.) in particular."
+msgstr ""
+
+#: ../../README_.rst:52
+msgid "Two Traditional HPC OS Approaches"
+msgstr ""
+
+#: ../../README_.rst:54
+msgid ""
+"Traditionally, light-weight operating systems specialized for HPC "
+"followed two approaches to tackle scalable execution of large-scale "
+"applications. In the full weight kernel (FWK) approach, a full Linux "
+"environment is taken as the basis, and features that inhibit attaining "
+"HPC scalability are removed, i.e., making it light-weight. The pure "
+"light-weight kernel (LWK) approach, on the other hand, starts from "
+"scratch and effort is undertaken to add sufficient functionality so that "
+"it provides a familiar API, typically something close to that of a "
+"general purpose OS, while at the same time it retains the desired "
+"scalability and reliability attributes. Neither of these approaches "
+"yields a fully Linux compatible environment."
+msgstr ""
+
+#: ../../README_.rst:67
+msgid "The Multi-kernel Approach"
+msgstr ""
+
+#: ../../README_.rst:69
+msgid ""
+"A hybrid approach recognized recently by the system software community is"
+" to run Linux simultaneously with a lightweight kernel on compute nodes "
+"and multiple research projects are now pursuing this direction. The basic"
+" idea is that simulations run on an HPC tailored lightweight kernel, "
+"ensuring the necessary isolation for noiseless execution of parallel "
+"applications, but Linux is leveraged so that the full POSIX API is "
+"supported. Additionally, the small code base of the LWK can also "
+"facilitate rapid prototyping for new, exotic hardware features. "
+"Nevertheless, the questions of how to share node resources between the "
+"two types of kernels, where do device drivers execute, how exactly do the"
+" two kernels interact with each other and to what extent are they "
+"integrated, remain subjects of ongoing debate."
+msgstr ""
+
+#: ../../README_.rst:83
+msgid "Architectural Overview"
+msgstr ""
+
+#: ../../README_.rst:85
+msgid ""
+"At the heart of the stack is a low-level software infrastructure called "
+"Interface for Heterogeneous Kernels (IHK). IHK is a general framework "
+"that provides capabilities for partitioning resources in a many-core "
+"environment (e.g.,CPU cores and physical memory) and it enables "
+"management of lightweight kernels. IHK can allocate and release host "
+"resources dynamically and no reboot of the host machine is required when "
+"altering configuration. IHK also provides a low-level inter-kernel "
+"messaging infrastructure, called the Inter-Kernel Communication (IKC) "
+"layer. An architectural overview of the main system components is shown "
+"below."
+msgstr ""
+
+#: ../../README_.rst:99
+msgid "arch"
+msgstr ""
+
+#: ../../README_.rst:101
+msgid ""
+"McKernel is a lightweight kernel written from scratch. It is designed for"
+" HPC and is booted from IHK. McKernel retains a binary compatible ABI "
+"with Linux, however, it implements only a small set of performance "
+"sensitive system calls and the rest are offloaded to Linux. Specifically,"
+" McKernel has its own memory management, it supports processes and multi-"
+"threading with a simple round-robin cooperative (tick-less) scheduler, "
+"and it implements signaling. It also allows inter-process memory mappings"
+" and it provides interfaces to hardware performance counters."
+msgstr ""
+
+#: ../../README_.rst:112
+msgid "Functionality"
+msgstr ""
+
+#: ../../README_.rst:114
+msgid ""
+"An overview of some of the principal functionalities of the IHK/McKernel "
+"stack is provided below."
+msgstr ""
+
+#: ../../README_.rst:118
+msgid "System Call Offloading"
+msgstr ""
+
+#: ../../README_.rst:120
+msgid ""
+"System call forwarding in McKernel is implemented as follows. When an "
+"offloaded system call occurs, McKernel marshals the system call number "
+"along with its arguments and sends a message to Linux via a dedicated IKC"
+" channel. The corresponding proxy process running on Linux is by default "
+"waiting for system call requests through an ioctl() call into IHK’s "
+"system call delegator kernel module. The delegator kernel module’s IKC "
+"interrupt handler wakes up the proxy process, which returns to userspace "
+"and simply invokes the requested system call. Once it obtains the return "
+"value, it instructs the delegator module to send the result back to "
+"McKernel, which subsequently passes the value to user-space."
+msgstr ""
+
+#: ../../README_.rst:132
+msgid "Unified Address Space"
+msgstr ""
+
+#: ../../README_.rst:134
+msgid ""
+"The unified address space model in IHK/McKernel ensures that offloaded "
+"system calls can seamlessly resolve arguments even in case of pointers. "
+"This mechanism is depicted below and is implemented as follows."
+msgstr ""
+
+#: ../../README_.rst:141
+msgid "unified_ap"
+msgstr ""
+
+#: ../../README_.rst:143
+msgid ""
+"First, the proxy process is compiled as a position independent binary, "
+"which enables us to map the code and data segments specific to the proxy "
+"process to an address range which is explicitly excluded from McKernel’s "
+"user space. The grey box on the right side of the figure demonstrates the"
+" excluded region. Second, the entire valid virtual address range of "
+"McKernel’s application user-space is covered by a special mapping in the "
+"proxy process for which we use a pseudo file mapping in Linux. This "
+"mapping is indicated by the blue box on the left side of the figure."
+msgstr ""
+
+#: ../../README_.rst:153
+msgid "Installation"
+msgstr ""
+
+#: ../../README_.rst:155
+msgid ""
+"For a smooth experience, we recommend the following combination of OS "
+"distributions and platforms:"
+msgstr ""
+
+#: ../../README_.rst:158
+msgid "CentOS 7.3+ running on Intel Xeon, Xeon Phi, Fujitsu A64FX"
+msgstr ""
+
+#: ../../README_.rst:161
+msgid "1. Change SELinux settings"
+msgstr ""
+
+#: ../../README_.rst:163
+msgid "Log in as the root and disable SELinux:"
+msgstr ""
+
+#: ../../README_.rst:169
+msgid "Change the file to SELINUX=disabled"
+msgstr ""
+
+#: ../../README_.rst:172
+msgid "2. Reboot the host machine"
+msgstr ""
+
+#: ../../README_.rst:179
+msgid "3. Prepare packages, kernel symbol table file"
+msgstr ""
+
+#: ../../README_.rst:181
+msgid "You will need the following packages installed:"
+msgstr ""
+
+#: ../../README_.rst:187
+msgid ""
+"Note that to install libdwarf-devel to RHEL-8.2, you need to enable the "
+"CodeReady Linux Builder (CLB) repository and the EPEL repository with the"
+" following commands:"
+msgstr ""
+
+#: ../../README_.rst:195
+msgid "Grant read permission to the System.map file of your kernel version:"
+msgstr ""
+
+#: ../../README_.rst:202
+msgid "4. Obtain sources and compile the kernel"
+msgstr ""
+
+#: ../../README_.rst:204
+msgid "Clone the source code:"
+msgstr ""
+
+#: ../../README_.rst:212
+msgid "(Optional) Checkout to the specific branch or version:"
+msgstr ""
+
+#: ../../README_.rst:220
+msgid ""
+"Foe example, if you want to try the development branch, use “development”"
+" as the pathspec. If you want to try the prerelease version 1.7.0-0.2, "
+"use “1.7.0-0.2”."
+msgstr ""
+
+#: ../../README_.rst:225
+msgid "4.1 Install with cmake"
+msgstr ""
+
+#: ../../README_.rst:227
+msgid "Configure and compile:"
+msgstr ""
+
+#: ../../README_.rst:235
+msgid ""
+"The IHK kernel modules and McKernel kernel image should be installed "
+"under the **ihk+mckernel** folder in your home directory."
+msgstr ""
+
+#: ../../README_.rst:239
+msgid "4.2 Install with rpm"
+msgstr ""
+
+#: ../../README_.rst:241
+msgid "Build rpm:"
+msgstr ""
+
+#: ../../README_.rst:252
+msgid ""
+"The IHK kernel modules and McKernel kernel image are installed under the "
+"system directory."
+msgstr ""
+
+#: ../../README_.rst:256
+msgid "5. Boot McKernel"
+msgstr ""
+
+#: ../../README_.rst:258
+msgid ""
+"A boot script called mcreboot.sh is provided under sbin in the install "
+"folder. To boot on logical CPU 1 with 512MB of memory, use the following "
+"invocation:"
+msgstr ""
+
+#: ../../README_.rst:268
+msgid ""
+"You should see something similar like this if you display the McKernel’s "
+"kernel message log:"
+msgstr ""
+
+#: ../../README_.rst:296
+msgid "6. Run a simple program on McKernel"
+msgstr ""
+
+#: ../../README_.rst:298
+msgid ""
+"The mcexec command line tool (which is also the Linux proxy process) can "
+"be used for executing applications on McKernel:"
+msgstr ""
+
+#: ../../README_.rst:307
+msgid "7. Shutdown McKernel"
+msgstr ""
+
+#: ../../README_.rst:309
+msgid ""
+"Finally, to shutdown McKernel and release CPU/memory resources back to "
+"Linux use the following command:"
+msgstr ""
+
+#: ../../README_.rst:317
+msgid "8. Advanced: Enable Utility Thread offloading Interface (UTI)"
+msgstr ""
+
+#: ../../README_.rst:319
+msgid ""
+"UTI enables a runtime such as MPI runtime to spawn utility threads such "
+"as MPI asynchronous progress threads to Linux cores."
+msgstr ""
+
+#: ../../README_.rst:323
+msgid "8.1 Install capstone"
+msgstr ""
+
+#: ../../README_.rst:325
+msgid "Install EPEL capstone-devel:"
+msgstr ""
+
+#: ../../README_.rst:333
+msgid "8.2 Install syscall_intercept"
+msgstr ""
+
+#: ../../README_.rst:342
+msgid "8.3 Install UTI for McKernel"
+msgstr ""
+
+#: ../../README_.rst:344
+msgid "Install:"
+msgstr ""
+
+#: ../../README_.rst:354
+msgid "8.4 Install McKernel"
+msgstr ""
+
+#: ../../README_.rst:361
+msgid "8.5 Run executable"
+msgstr ""
+
+#: ../../README_.rst:368
+msgid "8.6 Install UTI for Linux for performance comparison"
+msgstr ""
+
+#: ../../README_.rst:370
+msgid "Install by make:"
+msgstr ""
+
+#: ../../README_.rst:379
+msgid "Install by rpm:"
+msgstr ""
+
+#: ../../README_.rst:391
+msgid "The Team"
+msgstr ""
+
+#: ../../README_.rst:393
+msgid ""
+"The McKernel project was started at The University of Tokyo and currently"
+" it is mainly developed at RIKEN. Some of our collaborators include:"
+msgstr ""
+
+#: ../../README_.rst:397
+msgid "Hitachi"
+msgstr ""
+
+#: ../../README_.rst:398
+msgid "Fujitsu"
+msgstr ""
+
+#: ../../README_.rst:399
+msgid "CEA (France)"
+msgstr ""
+
+#: ../../README_.rst:400
+msgid "NEC"
+msgstr ""
+
+#: ../../README_.rst:403
+msgid "License"
+msgstr ""
+
+#: ../../README_.rst:405
+msgid "McKernel is GPL licensed, as found in the LICENSE file."
+msgstr ""
+
+#: ../../README_.rst:408
+msgid "Contact"
+msgstr ""
+
+#: ../../README_.rst:410
+msgid ""
+"Please give your feedback to us via one of the following mailing lists. "
+"Subscription via `www.pccluster.org "
+"<http://www.pccluster.org/mailman/listinfo/mckernel-users>`__ is needed."
+msgstr ""
+
+#: ../../README_.rst:415
+msgid "English: mckernel-users@pccluster.org"
+msgstr ""
+
+#: ../../README_.rst:416
+msgid "Japanese: mckernel-users-jp@pccluster.org"
+msgstr ""
+

docs/locales/ja/LC_MESSAGES/archtecture.po

@@ -0,0 +1,101 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) 2020, Masamichi Takagi, Balazs Gerofi, Yutaka Ishikawa
+# This file is distributed under the same license as the IHK/McKernel
+# package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2020.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: IHK/McKernel \n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2020-08-04 16:40+0900\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=utf-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: Babel 2.7.0\n"
+
+#: ../../archtecture.rst:2
+msgid "Architectural Overview"
+msgstr ""
+
+#: ../../archtecture.rst:4
+msgid ""
+"At the heart of the stack is a low-level software infrastructure called "
+"Interface for Heterogeneous Kernels (IHK). IHK is a general framework "
+"that provides capabilities for partitioning resources in a many-core "
+"environment (e.g.,CPU cores and physical memory) and it enables "
+"management of lightweight kernels. IHK can allocate and release host "
+"resources dynamically and no reboot of the host machine is required when "
+"altering configuration. IHK also provides a low-level inter-kernel "
+"messaging infrastructure, called the Inter-Kernel Communication (IKC) "
+"layer. An architectural overview of the main system components is shown "
+"below."
+msgstr ""
+
+#: ../../archtecture.rst:18
+msgid ""
+"McKernel is a lightweight kernel written from scratch. It is designed for"
+" HPC and is booted from IHK. McKernel retains a binary compatible ABI "
+"with Linux, however, it implements only a small set of performance "
+"sensitive system calls and the rest are offloaded to Linux. Specifically,"
+" McKernel has its own memory management, it supports processes and multi-"
+"threading with a simple round-robin cooperative (tick-less) scheduler, "
+"and it implements signaling. It also allows inter-process memory mappings"
+" and it provides interfaces to hardware performance counters."
+msgstr ""
+
+#: ../../archtecture.rst:29
+msgid "Functionality"
+msgstr ""
+
+#: ../../archtecture.rst:31
+msgid ""
+"An overview of some of the principal functionalities of the IHK/McKernel "
+"stack is provided below."
+msgstr ""
+
+#: ../../archtecture.rst:35
+msgid "System Call Offloading"
+msgstr ""
+
+#: ../../archtecture.rst:37
+msgid ""
+"System call forwarding in McKernel is implemented as follows. When an "
+"offloaded system call occurs, McKernel marshals the system call number "
+"along with its arguments and sends a message to Linux via a dedicated IKC"
+" channel. The corresponding proxy process running on Linux is by default "
+"waiting for system call requests through an ioctl() call into IHK’s "
+"system call delegator kernel module. The delegator kernel module’s IKC "
+"interrupt handler wakes up the proxy process, which returns to userspace "
+"and simply invokes the requested system call. Once it obtains the return "
+"value, it instructs the delegator module to send the result back to "
+"McKernel, which subsequently passes the value to user-space."
+msgstr ""
+
+#: ../../archtecture.rst:49
+msgid "Unified Address Space"
+msgstr ""
+
+#: ../../archtecture.rst:51
+msgid ""
+"The unified address space model in IHK/McKernel ensures that offloaded "
+"system calls can seamlessly resolve arguments even in case of pointers. "
+"This mechanism is depicted below and is implemented as follows."
+msgstr ""
+
+#: ../../archtecture.rst:58
+msgid ""
+"First, the proxy process is compiled as a position independent binary, "
+"which enables us to map the code and data segments specific to the proxy "
+"process to an address range which is explicitly excluded from McKernel’s "
+"user space. The grey box on the right side of the figure demonstrates the"
+" excluded region. Second, the entire valid virtual address range of "
+"McKernel’s application user-space is covered by a special mapping in the "
+"proxy process for which we use a pseudo file mapping in Linux. This "
+"mapping is indicated by the blue box on the left side of the figure."
+msgstr ""
+

docs/locales/ja/LC_MESSAGES/background.po

@@ -0,0 +1,79 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) 2020, Masamichi Takagi, Balazs Gerofi, Yutaka Ishikawa
+# This file is distributed under the same license as the IHK/McKernel
+# package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2020.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: IHK/McKernel \n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2020-08-04 16:40+0900\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=utf-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: Babel 2.7.0\n"
+
+#: ../../background.rst:2
+msgid "Background and Motivation"
+msgstr ""
+
+#: ../../background.rst:4
+msgid ""
+"With the growing complexity of high-end supercomputers, the current "
+"system software stack faces significant challenges as we move forward to "
+"exascale and beyond. The necessity to deal with extreme degree of "
+"parallelism, heterogeneous architectures, multiple levels of memory "
+"hierarchy, power constraints, etc., advocates operating systems that can "
+"rapidly adapt to new hardware requirements, and that can support novel "
+"programming paradigms and runtime systems. On the other hand, a new class"
+" of more dynamic and complex applications are also on the horizon, with "
+"an increasing demand for application constructs such as in-situ analysis,"
+" workflows, elaborate monitoring and performance tools. This complexity "
+"relies not only on the rich features of POSIX, but also on the Linux APIs"
+" (such as the */proc*, */sys* filesystems, etc.) in particular."
+msgstr ""
+
+#: ../../background.rst:19
+msgid "Two Traditional HPC OS Approaches"
+msgstr ""
+
+#: ../../background.rst:21
+msgid ""
+"Traditionally, light-weight operating systems specialized for HPC "
+"followed two approaches to tackle scalable execution of large-scale "
+"applications. In the full weight kernel (FWK) approach, a full Linux "
+"environment is taken as the basis, and features that inhibit attaining "
+"HPC scalability are removed, i.e., making it light-weight. The pure "
+"light-weight kernel (LWK) approach, on the other hand, starts from "
+"scratch and effort is undertaken to add sufficient functionality so that "
+"it provides a familiar API, typically something close to that of a "
+"general purpose OS, while at the same time it retains the desired "
+"scalability and reliability attributes. Neither of these approaches "
+"yields a fully Linux compatible environment."
+msgstr ""
+
+#: ../../background.rst:34
+msgid "The Multi-kernel Approach"
+msgstr ""
+
+#: ../../background.rst:36
+msgid ""
+"A hybrid approach recognized recently by the system software community is"
+" to run Linux simultaneously with a lightweight kernel on compute nodes "
+"and multiple research projects are now pursuing this direction. The basic"
+" idea is that simulations run on an HPC tailored lightweight kernel, "
+"ensuring the necessary isolation for noiseless execution of parallel "
+"applications, but Linux is leveraged so that the full POSIX API is "
+"supported. Additionally, the small code base of the LWK can also "
+"facilitate rapid prototyping for new, exotic hardware features. "
+"Nevertheless, the questions of how to share node resources between the "
+"two types of kernels, where do device drivers execute, how exactly do the"
+" two kernels interact with each other and to what extent are they "
+"integrated, remain subjects of ongoing debate."
+msgstr ""
+

docs/locales/ja/LC_MESSAGES/boot_run_shutdown.po

@@ -0,0 +1,57 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) 2020, Masamichi Takagi, Balazs Gerofi, Yutaka Ishikawa
+# This file is distributed under the same license as the IHK/McKernel
+# package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2020.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: IHK/McKernel \n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2020-08-04 16:40+0900\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=utf-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: Babel 2.7.0\n"
+
+#: ../../boot_run_shutdown.rst:2
+msgid "Boot McKernel"
+msgstr ""
+
+#: ../../boot_run_shutdown.rst:4
+msgid ""
+"A boot script called ``mcreboot.sh`` is provided under ``sbin`` in the "
+"install folder. To boot on logical CPU 1 with 512MB of memory, use the "
+"following invocation:"
+msgstr ""
+
+#: ../../boot_run_shutdown.rst:14
+msgid ""
+"You should see something similar like this if you display the McKernel’s "
+"kernel message log:"
+msgstr ""
+
+#: ../../boot_run_shutdown.rst:42
+msgid "Run a simple program on McKernel"
+msgstr ""
+
+#: ../../boot_run_shutdown.rst:44
+msgid ""
+"The mcexec command line tool (which is also the Linux proxy process) can "
+"be used for executing applications on McKernel:"
+msgstr ""
+
+#: ../../boot_run_shutdown.rst:53
+msgid "Shutdown McKernel"
+msgstr ""
+
+#: ../../boot_run_shutdown.rst:55
+msgid ""
+"Finally, to shutdown McKernel and release CPU/memory resources back to "
+"Linux use the following command:"
+msgstr ""
+

docs/locales/ja/LC_MESSAGES/contact.po

@@ -0,0 +1,39 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) 2020, Masamichi Takagi, Balazs Gerofi, Yutaka Ishikawa
+# This file is distributed under the same license as the IHK/McKernel
+# package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2020.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: IHK/McKernel \n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2020-08-04 16:40+0900\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=utf-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: Babel 2.7.0\n"
+
+#: ../../contact.rst:2
+msgid "Contact"
+msgstr ""
+
+#: ../../contact.rst:4
+msgid ""
+"Please give your feedback to us via one of the following mailing lists. "
+"Subscription via `www.pccluster.org "
+"<http://www.pccluster.org/mailman/listinfo/mckernel-users>`__ is needed."
+msgstr ""
+
+#: ../../contact.rst:9
+msgid "English: mckernel-users@pccluster.org"
+msgstr ""
+
+#: ../../contact.rst:10
+msgid "Japanese: mckernel-users-jp@pccluster.org"
+msgstr ""
+

docs/locales/ja/LC_MESSAGES/cover.po

@@ -0,0 +1,59 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) 2020, Masamichi Takagi, Balazs Gerofi, Yutaka Ishikawa
+# This file is distributed under the same license as the IHK/McKernel
+# package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2020.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: IHK/McKernel \n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2020-08-04 16:40+0900\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=utf-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: Babel 2.7.0\n"
+
+#: ../../summary.rst:1
+msgid ""
+"IHK/McKernel is a light-weight multi-kernel operating system designed for"
+" high-end supercomputing. It runs Linux and McKernel, a light-weight "
+"kernel (LWK), side-by-side inside compute nodes and aims at the "
+"following:"
+msgstr ""
+
+#: ../../summary.rst:6
+msgid ""
+"Provide scalable and consistent execution of large-scale parallel "
+"scientific applications, but at the same time maintain the ability to "
+"rapidly adapt to new hardware features and emerging programming models"
+msgstr ""
+
+#: ../../summary.rst:10
+msgid ""
+"Provide efficient memory and device management so that resource "
+"contention and data movement are minimized at the system level"
+msgstr ""
+
+#: ../../summary.rst:12
+msgid ""
+"Eliminate OS noise by isolating OS services in Linux and provide jitter "
+"free execution on the LWK"
+msgstr ""
+
+#: ../../summary.rst:14
+msgid ""
+"Support the full POSIX/Linux APIs by selectively offloading (slow-path) "
+"system calls to Linux"
+msgstr ""
+
+#: ../../cover.rst:5
+msgid ""
+"See `Quick Guide -- Installation <quick.html#installation>`__ for jump "
+"start."
+msgstr ""
+

docs/locales/ja/LC_MESSAGES/doc.po

@@ -0,0 +1,28 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) 2020, Masamichi Takagi, Balazs Gerofi, Yutaka Ishikawa
+# This file is distributed under the same license as the IHK/McKernel
+# package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2020.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: IHK/McKernel \n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2020-08-04 16:40+0900\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=utf-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: Babel 2.7.0\n"
+
+#: ../../doc.rst:2
+msgid "Documentation"
+msgstr ""
+
+#: ../../doc.rst:4
+msgid "Documentation is available `here <https://ihkmckernel.readthedocs.io>`__."
+msgstr ""
+

docs/locales/ja/LC_MESSAGES/ihk_developers.po

@@ -0,0 +1,40 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) 2020, Masamichi Takagi, Balazs Gerofi, Yutaka Ishikawa
+# This file is distributed under the same license as the IHK/McKernel
+# package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2020.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: IHK/McKernel \n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2020-08-04 16:40+0900\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=utf-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: Babel 2.7.0\n"
+
+#: ../../ihk_developers.rst:6
+msgid "External Specs"
+msgstr ""
+
+#: ../../ihk_developers.rst:9
+msgid "Overview"
+msgstr ""
+
+#: ../../ihk_developers.rst:12
+msgid "Function Specs"
+msgstr ""
+
+#: ../../ihk_developers.rst:15
+msgid "Command / Daemon Specs"
+msgstr ""
+
+#: ../../ihk_developers.rst:18
+msgid "Booting LWK"
+msgstr ""
+

docs/locales/ja/LC_MESSAGES/index.po

@@ -0,0 +1,95 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) 2020, Masamichi Takagi, Balazs Gerofi, Yutaka Ishikawa
+# This file is distributed under the same license as the IHK/McKernel
+# package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2020.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: IHK/McKernel \n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2020-08-07 10:00+0900\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=utf-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: Babel 2.7.0\n"
+
+#: ../../index.rst:12
+msgid "Quick Guide"
+msgstr "クイックガイド"
+
+#: ../../index.rst:18
+msgid "Users' Guide"
+msgstr "ユーザガイド"
+
+#: ../../index.rst:24
+msgid "Operators' Guide"
+msgstr "運用ガイド"
+
+#: ../../index.rst:30
+msgid "IHK Developers' Guide"
+msgstr ""
+
+#: ../../index.rst:36
+msgid "McKernel Developers' Guide"
+msgstr ""
+
+#: ../../index.rst:42
+msgid "IHK Specifications"
+msgstr ""
+
+#: ../../index.rst:48
+msgid "McKernel Specifications"
+msgstr ""
+
+#: ../../index.rst:54
+msgid "What's New"
+msgstr ""
+
+#: ../../index.rst:8
+msgid "IHK/McKernel"
+msgstr ""
+
+#: ../../summary.rst:1
+msgid ""
+"IHK/McKernel is a light-weight multi-kernel operating system designed for"
+" high-end supercomputing. It runs Linux and McKernel, a light-weight "
+"kernel (LWK), side-by-side inside compute nodes and aims at the "
+"following:"
+msgstr ""
+
+#: ../../summary.rst:6
+msgid ""
+"Provide scalable and consistent execution of large-scale parallel "
+"scientific applications, but at the same time maintain the ability to "
+"rapidly adapt to new hardware features and emerging programming models"
+msgstr ""
+
+#: ../../summary.rst:10
+msgid ""
+"Provide efficient memory and device management so that resource "
+"contention and data movement are minimized at the system level"
+msgstr ""
+
+#: ../../summary.rst:12
+msgid ""
+"Eliminate OS noise by isolating OS services in Linux and provide jitter "
+"free execution on the LWK"
+msgstr ""
+
+#: ../../summary.rst:14
+msgid ""
+"Support the full POSIX/Linux APIs by selectively offloading (slow-path) "
+"system calls to Linux"
+msgstr ""
+
+#: ../../cover.rst:5
+msgid ""
+"See `Quick Guide -- Installation <quick.html#installation>`__ for jump "
+"start."
+msgstr ""
+

docs/locales/ja/LC_MESSAGES/install.po

@@ -0,0 +1,191 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) 2020, Masamichi Takagi, Balazs Gerofi, Yutaka Ishikawa
+# This file is distributed under the same license as the IHK/McKernel
+# package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2020.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: IHK/McKernel \n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2020-08-04 16:40+0900\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=utf-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: Babel 2.7.0\n"
+
+#: ../../install.rst:4
+msgid "Installation"
+msgstr "インストール"
+
+#: ../../install.rst:6
+msgid "The following OS distributions and platforms are recommended:"
+msgstr "推奨OSディストリビューションとプロセッサは以下の通り。"
+
+#: ../../install.rst:8
+msgid "OS distribution"
+msgstr ""
+
+#: ../../install.rst:10
+msgid "CentOS 7.3 or later"
+msgstr ""
+
+#: ../../install.rst:11
+msgid "RHEL 7.3 or later"
+msgstr ""
+
+#: ../../install.rst:13
+msgid "Platform"
+msgstr ""
+
+#: ../../install.rst:15
+msgid "Intel Xeon"
+msgstr ""
+
+#: ../../install.rst:16
+msgid "Intel Xeon Phi"
+msgstr ""
+
+#: ../../install.rst:17
+msgid "Fujitsu A64FX"
+msgstr ""
+
+#: ../../install.rst:20
+msgid "Prepare files for building McKernel"
+msgstr ""
+
+#: ../../install.rst:22
+msgid ""
+"Grant read permission to the System.map file of your kernel version on "
+"the build machine:"
+msgstr ""
+
+#: ../../install.rst:28
+msgid "Install the following packages to the build machine:"
+msgstr ""
+
+#: ../../install.rst:35 ../../install.rst:179
+msgid "When having access to repositories"
+msgstr ""
+
+#: ../../install.rst:37 ../../install.rst:181
+msgid "On RHEL 8, enable the CodeReady Linux Builder (CLB) repository:"
+msgstr ""
+
+#: ../../install.rst:43 ../../install.rst:187
+msgid "On CentOS 8, enable the PowerTools repository:"
+msgstr ""
+
+#: ../../install.rst:49 ../../install.rst:193
+msgid "Install with yum:"
+msgstr ""
+
+#: ../../install.rst:56 ../../install.rst:200
+msgid "When not having access to repositories"
+msgstr ""
+
+#: ../../install.rst:58
+msgid ""
+"Ask the system administrator to install them. Note that ``libdwarf-"
+"devel`` is in the CodeReady Linux Builder repository on RHEL 8 or in the "
+"PowerTools repository on CentOS 8."
+msgstr ""
+
+#: ../../install.rst:61
+msgid "Clone, compile, install"
+msgstr ""
+
+#: ../../install.rst:63
+msgid "Clone the source code:"
+msgstr ""
+
+#: ../../install.rst:71
+msgid "(Optional) Checkout to the specific branch or version:"
+msgstr ""
+
+#: ../../install.rst:79
+msgid ""
+"Foe example, if you want to try the development branch, use “development”"
+" as the pathspec. If you want to try the prerelease version 1.7.0-0.2, "
+"use “1.7.0-0.2”."
+msgstr ""
+
+#: ../../install.rst:83
+msgid "Move to build directory:"
+msgstr ""
+
+#: ../../install.rst:89
+msgid "Run cmake:"
+msgstr ""
+
+#: ../../install.rst:92 ../../install.rst:135
+msgid "When not cross-compiling:"
+msgstr ""
+
+#: ../../install.rst:99 ../../install.rst:142
+msgid "When cross-compiling:"
+msgstr ""
+
+#: ../../install.rst:111
+msgid "Install with cmake"
+msgstr ""
+
+#: ../../install.rst:113
+msgid "Install with make:"
+msgstr ""
+
+#: ../../install.rst:119
+msgid ""
+"The kernel modules and McKernel kernel image should be installed under "
+"the **ihk+mckernel** folder in your home directory."
+msgstr ""
+
+#: ../../install.rst:123
+msgid "Install with rpm"
+msgstr ""
+
+#: ../../install.rst:125
+msgid "Create the tarball and the spec file:"
+msgstr ""
+
+#: ../../install.rst:132
+msgid "Create the rpm package:"
+msgstr ""
+
+#: ../../install.rst:148
+msgid "Install the rpm package:"
+msgstr ""
+
+#: ../../install.rst:154
+msgid ""
+"The kernel modules and McKernel kernel image are installed under the "
+"standard system directories."
+msgstr ""
+
+#: ../../install.rst:158
+msgid "Prepare files and change settings for installing McKernel"
+msgstr ""
+
+#: ../../install.rst:160
+msgid "Disable SELinux of the compute nodes:"
+msgstr ""
+
+#: ../../install.rst:166
+msgid "Change the file to SELINUX=disabled. And then reboot the compute nodes:"
+msgstr ""
+
+#: ../../install.rst:172
+msgid "Install the following packages to the compute nodes:"
+msgstr ""
+
+#: ../../install.rst:202
+msgid ""
+"Ask the system administrator to install them. Note that ``libdwarf`` is "
+"in the CodeReady Linux Builder repository on RHEL 8 or in the PowerTools "
+"repository on CentOS 8."
+msgstr ""
+

docs/locales/ja/LC_MESSAGES/license.po

@@ -0,0 +1,28 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) 2020, Masamichi Takagi, Balazs Gerofi, Yutaka Ishikawa
+# This file is distributed under the same license as the IHK/McKernel
+# package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2020.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: IHK/McKernel \n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2020-08-04 16:40+0900\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=utf-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: Babel 2.7.0\n"
+
+#: ../../license.rst:2
+msgid "License"
+msgstr ""
+
+#: ../../license.rst:4
+msgid "McKernel is GPL licensed, as found in the LICENSE file."
+msgstr ""
+

docs/locales/ja/LC_MESSAGES/logo.po

@@ -0,0 +1,20 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) 2020, Masamichi Takagi, Balazs Gerofi, Yutaka Ishikawa
+# This file is distributed under the same license as the IHK/McKernel
+# package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2020.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: IHK/McKernel \n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2020-08-04 16:40+0900\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=utf-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: Babel 2.7.0\n"
+

docs/locales/ja/LC_MESSAGES/mckernel_developers.po

@@ -0,0 +1,28 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) 2020, Masamichi Takagi, Balazs Gerofi, Yutaka Ishikawa
+# This file is distributed under the same license as the IHK/McKernel
+# package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2020.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: IHK/McKernel \n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2020-08-04 16:40+0900\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=utf-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: Babel 2.7.0\n"
+
+#: ../../mckernel_developers.rst:6
+msgid "Interfaces"
+msgstr ""
+
+#: ../../mckernel_developers.rst:9
+msgid "Interface details"
+msgstr ""
+

docs/locales/ja/LC_MESSAGES/operators.po

@@ -0,0 +1,886 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) 2020, Masamichi Takagi, Balazs Gerofi, Yutaka Ishikawa
+# This file is distributed under the same license as the IHK/McKernel
+# package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2020.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: IHK/McKernel \n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2020-08-07 10:26+0900\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=utf-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: Babel 2.7.0\n"
+
+#: ../../operators.rst:5
+#, fuzzy
+msgid "This document will explain how to operate system with McKernel."
+msgstr "McKernelを用いたシステムを運用するシステム管理者を対象として、運用手順を説明する。"
+
+#: ../../operators.rst:8
+msgid "Installation"
+msgstr "インストール"
+
+#: ../../operators.rst:10
+msgid "See `Quick Guide -- Installation <quick.html#installation>`__."
+msgstr "`クイックガイド ―  インストール <quick.html#installation>`__ に記載する。"
+
+#: ../../uti.rst:2
+msgid "Advanced: Enable Utility Thread offloading Interface (UTI)"
+msgstr ""
+
+#: ../../uti.rst:4
+msgid ""
+"UTI enables a runtime such as MPI runtime to spawn utility threads such "
+"as MPI asynchronous progress threads to Linux cores."
+msgstr ""
+
+#: ../../uti.rst:8
+msgid "Install capstone"
+msgstr ""
+
+#: ../../uti.rst:11 ../../uti.rst:22
+msgid "When compute nodes don't have access to repositories"
+msgstr ""
+
+#: ../../uti.rst:13
+msgid "Install EPEL capstone-devel:"
+msgstr ""
+
+#: ../../uti.rst:24
+msgid ""
+"Ask the system administrator to install ``capstone-devel``. Note that it "
+"is in the EPEL repository."
+msgstr ""
+
+#: ../../uti.rst:28
+msgid "Install syscall_intercept"
+msgstr ""
+
+#: ../../uti.rst:37
+msgid "Install UTI for McKernel"
+msgstr ""
+
+#: ../../uti.rst:39
+msgid "Install:"
+msgstr ""
+
+#: ../../uti.rst:49
+msgid "Install McKernel"
+msgstr ""
+
+#: ../../uti.rst:51
+msgid "Add ``-DENABLE_UTI=ON`` option to ``cmake``:"
+msgstr ""
+
+#: ../../uti.rst:58
+msgid "Run programs"
+msgstr ""
+
+#: ../../uti.rst:60
+msgid "Add ``--enable-uti`` option to ``mcexec``:"
+msgstr ""
+
+#: ../../uti.rst:67
+msgid "Install UTI for Linux"
+msgstr ""
+
+#: ../../uti.rst:69
+msgid ""
+"You should skip this step if it's already installed as with, for example,"
+" Fujitsu Technical Computing Suite."
+msgstr ""
+
+#: ../../uti.rst:72
+msgid "Install by make"
+msgstr ""
+
+#: ../../uti.rst:82
+msgid "Install by rpm"
+msgstr ""
+
+#: ../../operators.rst:15
+msgid "Boot and Shut-down"
+msgstr "起動停止"
+
+#: ../../operators.rst:18
+msgid "Related files"
+msgstr "関連ファイル"
+
+#: ../../operators.rst:20
+msgid ""
+"In the followings, the install directory of IHK/McKernel is shown as "
+"``<install>`` . The kernel modules and their locations are as follows."
+msgstr "以降、IHK/McKernelのインストールディレクトリを<install>とする。SMPプロセッサ向け、x86_64アーキ向けの関連ファイルの場所は以下の通り。"
+
+#: ../../operators.rst:24
+msgid "<install>/kmod/ihk.ko"
+msgstr ""
+
+#: ../../operators.rst:24
+msgid "IHK-master core"
+msgstr ""
+
+#: ../../operators.rst:26
+msgid "|ihk-smp|"
+msgstr ""
+
+#: ../../operators.rst:26
+msgid "IHK-master driver"
+msgstr ""
+
+#: ../../operators.rst:29
+msgid "|mcctrl|"
+msgstr ""
+
+#: ../../operators.rst:29
+msgid "Delegator module"
+msgstr ""
+
+#: ../../operators.rst:32
+msgid "|mckernel.img|"
+msgstr ""
+
+#: ../../operators.rst:32
+msgid "Kernel Image"
+msgstr "カーネルイメージ"
+
+#: ../../operators.rst:40
+msgid "The commands and daemons for operation and their locations are as follows."
+msgstr "運用向けコマンド・デーモンのファイルの場所は以下の通り。"
+
+#: ../../operators.rst:43
+msgid "|mcreboot|"
+msgstr ""
+
+#: ../../operators.rst:43
+msgid "Boot script"
+msgstr "起動スクリプト"
+
+#: ../../operators.rst:46
+msgid "|mcstop|"
+msgstr ""
+
+#: ../../operators.rst:46
+msgid "Shutdown script"
+msgstr "シャットダウンスクリプト"
+
+#: ../../operators.rst:49
+msgid "<install>/bin/mcexec"
+msgstr ""
+
+#: ../../operators.rst:49
+msgid "Process invocation command"
+msgstr "プロセス起動コマンド"
+
+#: ../../operators.rst:51
+msgid "<install>/bin/eclair"
+msgstr ""
+
+#: ../../operators.rst:51
+msgid "Kernel dump analysis tool"
+msgstr "カーネルダンプ解析ツール"
+
+#: ../../operators.rst:53
+msgid "|vmcore2mckdump|"
+msgstr ""
+
+#: ../../operators.rst:53
+msgid "Kernel dump format conversion tool"
+msgstr "カーネルダンプ形式変換ツール"
+
+#: ../../operators.rst:60
+msgid "以下、関連コマンドおよび関連関数のインターフェイスを説明する。"
+msgstr ""
+
+#: ../../operators.rst:63
+msgid "インターフェイス"
+msgstr ""
+
+#: ../../operators.rst:66
+msgid "カーネル引数"
+msgstr ""
+
+#: ../../operators.rst:68
+msgid "McKernelのカーネル引数を表 :numref:`tab-kargs` に示す。"
+msgstr ""
+
+#: ../../operators.rst:72
+msgid "McKernelのカーネル引数"
+msgstr ""
+
+#: ../../operators.rst:75
+msgid "hidos"
+msgstr ""
+
+#: ../../operators.rst:75
+msgid "IKCを有効にする。"
+msgstr ""
+
+#: ../../operators.rst:77
+msgid "|dlv|"
+msgstr ""
+
+#: ../../operators.rst
+msgid "Linuxのpanicハンドラ経由でダンプを行った場合の、ダ"
+msgstr ""
+
+#: ../../operators.rst
+msgid "ンプ対象とするメモリ領域の種類を<level>に設定する。"
+msgstr ""
+
+#: ../../operators.rst
+msgid "設定可能な値は以下の通り。"
+msgstr ""
+
+#: ../../operators.rst:83 ../../operators.rst:164 ../../operators.rst:196
+#: ../../operators.rst:234
+msgid "0"
+msgstr ""
+
+#: ../../operators.rst:84 ../../operators.rst:165
+msgid "IHKがMcKernelに割り当てたメモリ領域を出力する"
+msgstr ""
+
+#: ../../operators.rst:85 ../../operators.rst:166
+msgid "24"
+msgstr ""
+
+#: ../../operators.rst:86 ../../operators.rst:167
+msgid "カーネルが使用しているメモリ領域を出力する"
+msgstr ""
+
+#: ../../operators.rst:88 ../../operators.rst:169
+msgid "指定がなかった場合は24が用いられる。"
+msgstr ""
+
+#: ../../operators.rst:90
+msgid "|allow|"
+msgstr ""
+
+#: ../../operators.rst
+msgid "McKernelに割り当てられたCPU数より大きい数のスレッド"
+msgstr ""
+
+#: ../../operators.rst
+msgid "またはプロセスの生成を許可する。この引数が指定され"
+msgstr ""
+
+#: ../../operators.rst
+msgid "ない場合に、CPU数より大きい数のスレッドまたはプロセ"
+msgstr ""
+
+#: ../../operators.rst
+msgid "スをclone(), fork(), vfork()などで生成しようとする"
+msgstr ""
+
+#: ../../operators.rst
+msgid "と、当該システムコールがEINVALエラーを返す。"
+msgstr ""
+
+#: ../../operators.rst:102
+msgid "ブートスクリプト"
+msgstr "ブートスクリプト"
+
+#: ../../operators.rst:105 ../../operators.rst:207
+msgid "書式"
+msgstr ""
+
+#: ../../operators.rst:113 ../../operators.rst:217
+msgid "オプション"
+msgstr ""
+
+#: ../../operators.rst:116
+msgid "|opt-c|"
+msgstr ""
+
+#: ../../operators.rst
+msgid "McKernelに割り当てるCPUのリストを指定する。フォー"
+msgstr ""
+
+#: ../../operators.rst
+msgid "マットは以下の通り。"
+msgstr ""
+
+#: ../../operators.rst
+msgid "<CPU logical id>,<CPU logical id>...または"
+msgstr ""
+
+#: ../../operators.rst
+msgid "<CPU logical id>-<CPU logical id>,<CPU logical id>"
+msgstr ""
+
+#: ../../operators.rst
+msgid "-<CPU logical id>...または両者の混合。"
+msgstr ""
+
+#: ../../operators.rst:122
+msgid "|opt-r|"
+msgstr ""
+
+#: ../../operators.rst
+msgid "McKernelのCPUがIKCメッセージを送るLinux"
+msgstr ""
+
+#: ../../operators.rst
+msgid "CPUを指定する。フォーマットは以下の通り。"
+msgstr ""
+
+#: ../../operators.rst
+msgid "<CPU list>:<CPU id>+<CPU list>:<CPU id>..."
+msgstr ""
+
+#: ../../operators.rst
+msgid "<CPU list>のフォーマットは-cオプションにおけるもの"
+msgstr ""
+
+#: ../../operators.rst
+msgid "と同じである。"
+msgstr ""
+
+#: ../../operators.rst
+msgid "各<CPU list>:<CPU id>は<CPU list>で示されるMcKernel"
+msgstr ""
+
+#: ../../operators.rst
+msgid "のCPUが<CPU logical id>で示されるLinuxのCPUにIKC"
+msgstr ""
+
+#: ../../operators.rst
+msgid "メッセージを送信することを意味する。"
+msgstr ""
+
+#: ../../operators.rst:131
+msgid "|opt-m|"
+msgstr ""
+
+#: ../../operators.rst
+msgid "McKernelに割り当てるメモリ領域を指定する。フォーマッ"
+msgstr ""
+
+#: ../../operators.rst
+msgid "トは以下の通り。"
+msgstr ""
+
+#: ../../operators.rst
+msgid "<size>@<NUMA-id>, <size>@<NUMA-id>..."
+msgstr ""
+
+#: ../../operators.rst:135
+msgid "|opt-f|"
+msgstr ""
+
+#: ../../operators.rst
+msgid "ihkmondが使用するsyslogプロトコルのfacilityを指定す"
+msgstr ""
+
+#: ../../operators.rst
+msgid "る。デフォルトはLOG_LOCAL6。"
+msgstr ""
+
+#: ../../operators.rst:138
+msgid "|opt-o|"
+msgstr ""
+
+#: ../../operators.rst
+msgid "IHKのデバイスファイル(/dev/mcd*, /dev/mcos*)のオー"
+msgstr ""
+
+#: ../../operators.rst
+msgid "ナーとグループの値を<user>[:<group>]の形式で指定す"
+msgstr ""
+
+#: ../../operators.rst
+msgid "る。デフォルトはmcreboot.shを実行したユーザ。"
+msgstr ""
+
+#: ../../operators.rst:142
+msgid "|opt-i|"
+msgstr ""
+
+#: ../../operators.rst
+msgid "ihkmondがハングアップ検知のためにOS状態を確認する時"
+msgstr ""
+
+#: ../../operators.rst
+msgid "間間隔を秒単位で指定する。-1が指定された場合はハン"
+msgstr ""
+
+#: ../../operators.rst
+msgid "グアップ検知を行わない。指定がない場合はハングアッ"
+msgstr ""
+
+#: ../../operators.rst
+msgid "プ検知を行わない。"
+msgstr ""
+
+#: ../../operators.rst:147
+msgid "|opt-k|"
+msgstr ""
+
+#: ../../operators.rst
+msgid "カーネルメッセージの/dev/logへのリダイレクト有無を"
+msgstr ""
+
+#: ../../operators.rst
+msgid "指定する。0が指定された場合はリダイレクトを行わず、"
+msgstr ""
+
+#: ../../operators.rst
+msgid "0以外が指定された場合はリダイレクトを行う。指定がな"
+msgstr ""
+
+#: ../../operators.rst
+msgid "い場合はリダイレクトを行わない。"
+msgstr ""
+
+#: ../../operators.rst:152
+msgid "-q <irq>"
+msgstr ""
+
+#: ../../operators.rst
+msgid "IHKが使用するIRQ番号を指定する。指定がない場合は"
+msgstr ""
+
+#: ../../operators.rst
+msgid "64-255の範囲で空いているものを使用する。"
+msgstr ""
+
+#: ../../operators.rst:155
+msgid "-t"
+msgstr ""
+
+#: ../../operators.rst
+msgid "（x86_64アーキテクチャのみ）Turbo"
+msgstr ""
+
+#: ../../operators.rst
+msgid "Boostをオンにする。デフォルトはオフ。"
+msgstr ""
+
+#: ../../operators.rst:158
+msgid "-d <level>"
+msgstr ""
+
+#: ../../operators.rst:171
+msgid "-O"
+msgstr ""
+
+#: ../../operators.rst
+msgid "またはプロセスの生成を許可する。指定がない場合は許可"
+msgstr ""
+
+#: ../../operators.rst
+msgid "しない。すなわち、CPU数より大きい数のスレッドまたは"
+msgstr ""
+
+#: ../../operators.rst
+msgid "プロセスを生成しようとするとエラーとなる。"
+msgstr ""
+
+#: ../../operators.rst:187 ../../operators.rst:224
+msgid "説明"
+msgstr ""
+
+#: ../../operators.rst:189
+msgid ""
+"McKernel関連カーネルモジュールをinsmodし、<cpulist>で指定されたCPUと<memlist>で指定されたメモリ領域からなるパーティションを作成し、IKC"
+" mapを<ikcmap>に設定し、前記パーティションにMcKernelをブートする。"
+msgstr ""
+
+#: ../../operators.rst:193 ../../operators.rst:231
+msgid "戻り値"
+msgstr ""
+
+#: ../../operators.rst:196 ../../operators.rst:234
+msgid "正常終了"
+msgstr ""
+
+#: ../../operators.rst:198 ../../operators.rst:236
+msgid "0以外"
+msgstr ""
+
+#: ../../operators.rst:198 ../../operators.rst:236
+msgid "エラー"
+msgstr ""
+
+#: ../../operators.rst:202
+msgid "シャットダウンスクリプト"
+msgstr "シャットダウンスクリプト"
+
+#: ../../operators.rst:219
+msgid "なし"
+msgstr ""
+
+#: ../../operators.rst:226
+msgid "McKernelをシャットダウンし、McKernel用パーティションを削除し、関連カーネルモジュールをrmmodする。"
+msgstr ""
+
+#: ../../operators.rst:240
+msgid "プロセス起動コマンド"
+msgstr ""
+
+#: ../../operators.rst:245
+msgid "ダンプ解析コマンド"
+msgstr ""
+
+#: ../../operators.rst:250
+msgid "ダンプ形式変換コマンド"
+msgstr ""
+
+#: ../../operators.rst:255
+msgid "ブート手順"
+msgstr ""
+
+#: ../../operators.rst:257
+msgid "mcreboot.shを用いてブート手順を説明する。"
+msgstr ""
+
+#: ../../operators.rst:259 ../../operators.rst:906
+msgid "スクリプトは以下の通り。"
+msgstr ""
+
+#: ../../operators.rst:852 ../../operators.rst:1050
+msgid "手順は以下の通り。"
+msgstr ""
+
+#: ../../operators.rst:854
+msgid "ihkmondを起動する。ihkmondは任意のタイミングで起動してよい。これは、ihkmondはOSインスタンスの作成を検知して動作を開始するためである。（83行目）"
+msgstr ""
+
+#: ../../operators.rst:856
+msgid "Linuxのカーネルバージョンが、mcoverlayfsが動作するものであるかを確認する。（200–216行目）"
+msgstr ""
+
+#: ../../operators.rst:858
+msgid "irqbalanceを停止する。（251–257行目）"
+msgstr ""
+
+#: ../../operators.rst:860
+msgid ""
+"/proc/irq/[n]/affinityの設定を保存した上でMcKernel "
+"CPUを担当から外す。担当CPUが無くなる場合は、全てのLinux CPUを指定する。（269–303行目）"
+msgstr ""
+
+#: ../../operators.rst:864
+msgid "ihk.koをinsmodする。（307行目）"
+msgstr ""
+
+#: ../../operators.rst:866
+msgid "Linuxによるメモリフラグメンテーションを緩和するために以下を実施する。（313–320行目）"
+msgstr ""
+
+#: ../../operators.rst:868
+msgid "アクティブでないプロセスを積極的にスワップアウトするように設定する"
+msgstr ""
+
+#: ../../operators.rst:870
+msgid "クリーンなページキャッシュを無効化し、またdentriesやinodeのslabオブジェクトのうち可能なものを破棄する"
+msgstr ""
+
+#: ../../operators.rst:872
+msgid "連続する空き領域を結合してより大きな空き領域にまとめる"
+msgstr ""
+
+#: ../../operators.rst:874
+msgid ""
+"ihk-smp-x86.koをinsmodする。（340行目）ihk-smp-x86.koは関数をihk.koに登録する。このため、ihk-"
+"smp-x86.koはihk.koをinsmodした後にinsmodする必要がある。"
+msgstr ""
+
+#: ../../operators.rst:876
+msgid "メモリを予約する。（370行目）"
+msgstr ""
+
+#: ../../operators.rst:878
+msgid "CPUを予約する。（374行目）"
+msgstr ""
+
+#: ../../operators.rst:880
+msgid "McKernelのカーネルモジュールmcctrl.koをinsmodする。（382行目）mcctrl.koはMcKernelブート時に呼ばれる関数をihk.koに登録する。このため、mcctrl.koのinsmodはihk.koのinsmodの後に、またブートの前に行う必要がある。"
+msgstr ""
+
+#: ../../operators.rst:882
+msgid "OSインスタンスを作成する。（406行目）"
+msgstr ""
+
+#: ../../operators.rst:884
+msgid "OSインスタンスにCPUを割り当てる。（412行目）"
+msgstr ""
+
+#: ../../operators.rst:886
+msgid "McKernel CPUのIKCメッセージ送信先のLinux CPUを設定する。（419行目）"
+msgstr ""
+
+#: ../../operators.rst:888
+msgid "OSインスタンスにメモリを割り当てる。（426行目）"
+msgstr ""
+
+#: ../../operators.rst:890
+msgid "カーネルイメージをロードする。（432行目）"
+msgstr ""
+
+#: ../../operators.rst:892
+msgid "カーネル引数をカーネルに渡す。（438行目）"
+msgstr ""
+
+#: ../../operators.rst:894
+msgid "カーネルをブートする。（444行目）"
+msgstr ""
+
+#: ../../operators.rst:896
+msgid ""
+"/proc, "
+"/sysファイルの準備をする。また、その中でmcoverlayfs.koをinsmodする。mcoverlayfs.koは他モジュールとの依存関係を持たない。（454行目から567行目）なお、関数インターフェイスでの対応関数はihk_os_create_pseudofs()である。"
+msgstr ""
+
+#: ../../operators.rst:898
+msgid "irqbalanceを、Linux CPUのみを対象とする設定で開始する。（569–587行目）"
+msgstr ""
+
+#: ../../operators.rst:902
+msgid "シャットダウン手順"
+msgstr ""
+
+#: ../../operators.rst:904
+msgid "mcstop+release.shを用いてシャットダウン手順を説明する。"
+msgstr ""
+
+#: ../../operators.rst:1052
+msgid "ブート時にLinux CPUのみを対象とする設定で開始されたirqbalanceを停止する。（24–33行目）"
+msgstr ""
+
+#: ../../operators.rst:1055
+msgid "全てのOSインスタンスを破壊する。OSインスタンスに割り当てられていた資源はIHKがLWKのために予約した状態に移行する。（35–50行目）"
+msgstr ""
+
+#: ../../operators.rst:1057
+msgid "IHKがLWKのために予約していた資源を開放する。（52–77行目）"
+msgstr ""
+
+#: ../../operators.rst:1059
+msgid "mcctrl.koをrmmodする。（81行目）"
+msgstr ""
+
+#: ../../operators.rst:1061
+msgid ""
+"/proc, "
+"/sysファイルの準備をする。また、その中でmcoverlayfs.koをrmmodする。（87–100行目）なお、関数インターフェイスでの対応関数はihk_os_destroy_pseudofs()である。"
+msgstr ""
+
+#: ../../operators.rst:1063
+msgid "ihk-smp-x86.koをrmmodする。（104行目）"
+msgstr ""
+
+#: ../../operators.rst:1065
+msgid "ihk.koをrmmodする。（112行目）"
+msgstr ""
+
+#: ../../operators.rst:1067
+msgid "ihkmondを停止する。（121行目）"
+msgstr ""
+
+#: ../../operators.rst:1069
+msgid "/proc/irq/[n]/affinityの設定をブート時に保存しておいたものに戻し、ブート前の設定でirqbalanceを開始する。（124–135行目）"
+msgstr ""
+
+#: ../../operators.rst:1071
+msgid "Linuxカーネルのスワップアウト積極度の設定をデフォルトの値に戻す。（138行目）"
+msgstr ""
+
+#~ msgid "The following OS distributions and platforms are recommended:"
+#~ msgstr ""
+
+#~ msgid "OS distribution"
+#~ msgstr ""
+
+#~ msgid "CentOS 7.3 or later"
+#~ msgstr ""
+
+#~ msgid "RHEL 7.3 or later"
+#~ msgstr ""
+
+#~ msgid "Platform"
+#~ msgstr ""
+
+#~ msgid "Intel Xeon"
+#~ msgstr ""
+
+#~ msgid "Intel Xeon Phi"
+#~ msgstr ""
+
+#~ msgid "Fujitsu A64FX"
+#~ msgstr ""
+
+#~ msgid "Prepare files for building McKernel"
+#~ msgstr ""
+
+#~ msgid ""
+#~ "Grant read permission to the System.map"
+#~ " file of your kernel version on "
+#~ "the build machine:"
+#~ msgstr ""
+
+#~ msgid "Install the following packages to the build machine:"
+#~ msgstr ""
+
+#~ msgid "When having access to repositories"
+#~ msgstr ""
+
+#~ msgid "On RHEL 8, enable the CodeReady Linux Builder (CLB) repository:"
+#~ msgstr ""
+
+#~ msgid "On CentOS 8, enable the PowerTools repository:"
+#~ msgstr ""
+
+#~ msgid "Install with yum:"
+#~ msgstr ""
+
+#~ msgid "When not having access to repositories"
+#~ msgstr ""
+
+#~ msgid ""
+#~ "Ask the system administrator to install"
+#~ " them. Note that ``libdwarf-devel`` "
+#~ "is in the CodeReady Linux Builder "
+#~ "repository on RHEL 8 or in the "
+#~ "PowerTools repository on CentOS 8."
+#~ msgstr ""
+
+#~ msgid "Clone, compile, install"
+#~ msgstr ""
+
+#~ msgid "Clone the source code:"
+#~ msgstr ""
+
+#~ msgid "(Optional) Checkout to the specific branch or version:"
+#~ msgstr ""
+
+#~ msgid ""
+#~ "Foe example, if you want to try"
+#~ " the development branch, use “development”"
+#~ " as the pathspec. If you want "
+#~ "to try the prerelease version 1.7.0-0.2,"
+#~ " use “1.7.0-0.2”."
+#~ msgstr ""
+
+#~ msgid "Move to build directory:"
+#~ msgstr ""
+
+#~ msgid "Run cmake:"
+#~ msgstr ""
+
+#~ msgid "When not cross-compiling:"
+#~ msgstr ""
+
+#~ msgid "When cross-compiling:"
+#~ msgstr ""
+
+#~ msgid "Install with cmake"
+#~ msgstr ""
+
+#~ msgid "Install with make:"
+#~ msgstr ""
+
+#~ msgid ""
+#~ "The kernel modules and McKernel kernel"
+#~ " image should be installed under the"
+#~ " **ihk+mckernel** folder in your home "
+#~ "directory."
+#~ msgstr ""
+
+#~ msgid "Install with rpm"
+#~ msgstr ""
+
+#~ msgid "Create the tarball and the spec file:"
+#~ msgstr ""
+
+#~ msgid "Create the rpm package:"
+#~ msgstr ""
+
+#~ msgid "Install the rpm package:"
+#~ msgstr ""
+
+#~ msgid ""
+#~ "The kernel modules and McKernel kernel"
+#~ " image are installed under the "
+#~ "standard system directories."
+#~ msgstr ""
+
+#~ msgid "Prepare files and change settings for installing McKernel"
+#~ msgstr ""
+
+#~ msgid "Disable SELinux of the compute nodes:"
+#~ msgstr ""
+
+#~ msgid "Change the file to SELINUX=disabled. And then reboot the compute nodes:"
+#~ msgstr ""
+
+#~ msgid "Install the following packages to the compute nodes:"
+#~ msgstr ""
+
+#~ msgid ""
+#~ "Ask the system administrator to install"
+#~ " them. Note that ``libdwarf`` is in"
+#~ " the CodeReady Linux Builder repository "
+#~ "on RHEL 8 or in the PowerTools "
+#~ "repository on CentOS 8."
+#~ msgstr ""
+
+#~ msgid "起動停止"
+#~ msgstr "起動停止"
+
+#~ msgid "関連ファイル"
+#~ msgstr "関連ファイル"
+
+#~ msgid "McKernelを用いたシステムを運用するシステム管理者を対象として、運用手順を説明する。"
+#~ msgstr ""
+
+#~ msgid ""
+#~ "SMPプロセッサ向け、x86_64アーキ向けの関連ファイルの場所は以下の通り。 "
+#~ "なお、IHK/McKernelのインストールディレクトリを<install>とする。"
+#~ msgstr "運用向けコマンド・デーモンのファイルの場所は以下の通り。 なお、IHK/McKernelのインストールディレクトリを<install>とする。"
+
+#~ msgid ""
+#~ "SMPプロセッサ向け、x86_64アーキ向けの関連ファイルの場所は以下の通り。 "
+#~ "なお、IHK/McKernelのインストールディレクトリを<install>とする。 The related "
+#~ "files and their locations are as "
+#~ "follows. Denote by ``<install>`` the "
+#~ "install directory of IHK/McKernel."
+#~ msgstr ""
+
+#~ msgid ""
+#~ "The related files and their locations"
+#~ " are as follows. Denote by "
+#~ "``<install>`` the install directory of "
+#~ "IHK/McKernel."
+#~ msgstr ""
+#~ "SMPプロセッサ向け、x86_64アーキ向けの関連ファイルの場所は以下の通り。 "
+#~ "なお、IHK/McKernelのインストールディレクトリを<install>とする。"
+
+#~ msgid "運用向けコマンド・デーモンのファイルの場所は以下の通り。 なお、IHK/McKernelのインストールディレクトリを<install>とする。"
+#~ msgstr "運用向けコマンド・デーモンのファイルの場所は以下の通り。 なお、IHK/McKernelのインストールディレクトリを<install>とする。"
+
+#~ msgid "運用向けコマンド・デーモンのファイルの場所は以下の通り。"
+#~ msgstr ""
+#~ "The commands and daemons for operation"
+#~ " and their locations are as follows."
+
+#~ msgid "カーネルイメージ"
+#~ msgstr ""
+
+#~ msgid "ダンプ解析ツール"
+#~ msgstr ""
+
+#~ msgid "ダンプ形式変換ツール"
+#~ msgstr ""
+
+#~ msgid "tool"
+#~ msgstr "ツール"
+

docs/locales/ja/LC_MESSAGES/ops.po

@@ -0,0 +1,821 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) 2020, Masamichi Takagi, Balazs Gerofi, Yutaka Ishikawa
+# This file is distributed under the same license as the IHK/McKernel
+# package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2020.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: IHK/McKernel \n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2020-08-06 10:10+0900\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=utf-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: Babel 2.7.0\n"
+
+#: ../../ops.rst:5
+msgid "McKernelを用いたシステムを運用するシステム管理者を対象として、運用手順を説明する。"
+msgstr ""
+
+#: ../../ops.rst:8
+msgid "Installation"
+msgstr "インストール"
+
+#: ../../ops.rst:10
+msgid "See `Quick Guide -- Installation <quick.html#installation>`__."
+msgstr ""
+
+#: ../../uti.rst:2
+msgid "Advanced: Enable Utility Thread offloading Interface (UTI)"
+msgstr ""
+
+#: ../../uti.rst:4
+msgid ""
+"UTI enables a runtime such as MPI runtime to spawn utility threads such "
+"as MPI asynchronous progress threads to Linux cores."
+msgstr ""
+
+#: ../../uti.rst:8
+msgid "Install capstone"
+msgstr ""
+
+#: ../../uti.rst:11 ../../uti.rst:22
+msgid "When compute nodes don't have access to repositories"
+msgstr ""
+
+#: ../../uti.rst:13
+msgid "Install EPEL capstone-devel:"
+msgstr ""
+
+#: ../../uti.rst:24
+msgid ""
+"Ask the system administrator to install ``capstone-devel``. Note that it "
+"is in the EPEL repository."
+msgstr ""
+
+#: ../../uti.rst:28
+msgid "Install syscall_intercept"
+msgstr ""
+
+#: ../../uti.rst:37
+msgid "Install UTI for McKernel"
+msgstr ""
+
+#: ../../uti.rst:39
+msgid "Install:"
+msgstr ""
+
+#: ../../uti.rst:49
+msgid "Install McKernel"
+msgstr ""
+
+#: ../../uti.rst:51
+msgid "Add ``-DENABLE_UTI=ON`` option to ``cmake``:"
+msgstr ""
+
+#: ../../uti.rst:58
+msgid "Run programs"
+msgstr ""
+
+#: ../../uti.rst:60
+msgid "Add ``--enable-uti`` option to ``mcexec``:"
+msgstr ""
+
+#: ../../uti.rst:67
+msgid "Install UTI for Linux"
+msgstr ""
+
+#: ../../uti.rst:69
+msgid ""
+"You should skip this step if it's already installed as with, for example,"
+" Fujitsu Technical Computing Suite."
+msgstr ""
+
+#: ../../uti.rst:72
+msgid "Install by make"
+msgstr ""
+
+#: ../../uti.rst:82
+msgid "Install by rpm"
+msgstr ""
+
+#: ../../ops.rst:15
+msgid "起動停止"
+msgstr ""
+
+#: ../../ops.rst:18
+msgid "関連ファイル"
+msgstr ""
+
+#: ../../ops.rst:20
+msgid ""
+"SMPプロセッサ向け、x86_64アーキ向けの関連ファイルの場所は以下の通り。 "
+"なお、IHK/McKernelのインストールディレクトリを<install>とする。"
+msgstr ""
+
+#: ../../ops.rst:24
+msgid "<install>/kmod/ihk.ko"
+msgstr ""
+
+#: ../../ops.rst:24
+msgid "IHK-master core"
+msgstr ""
+
+#: ../../ops.rst:26
+msgid "|ihk-smp|"
+msgstr ""
+
+#: ../../ops.rst:26
+msgid "IHK-master driver"
+msgstr ""
+
+#: ../../ops.rst:29
+msgid "|mcctrl|"
+msgstr ""
+
+#: ../../ops.rst:29
+msgid "Delegator module"
+msgstr ""
+
+#: ../../ops.rst:32
+msgid "|mckernel.img|"
+msgstr ""
+
+#: ../../ops.rst:32
+msgid "カーネルイメージ"
+msgstr ""
+
+#: ../../ops.rst:40
+msgid "運用向けコマンド・デーモンのファイルの場所は以下の通り。 なお、IHK/McKernelのインストールディレクトリを<install>とする。"
+msgstr ""
+
+#: ../../ops.rst:44
+msgid "|mcreboot|"
+msgstr ""
+
+#: ../../ops.rst:44 ../../ops.rst:104
+msgid "ブートスクリプト"
+msgstr ""
+
+#: ../../ops.rst:47
+msgid "|mcstop|"
+msgstr ""
+
+#: ../../ops.rst:47 ../../ops.rst:204
+msgid "シャットダウンスクリプト"
+msgstr ""
+
+#: ../../ops.rst:50
+msgid "<install>/bin/mcexec"
+msgstr ""
+
+#: ../../ops.rst:50 ../../ops.rst:242
+msgid "プロセス起動コマンド"
+msgstr ""
+
+#: ../../ops.rst:52
+msgid "<install>/bin/eclair"
+msgstr ""
+
+#: ../../ops.rst:52
+msgid "ダンプ解析ツール"
+msgstr ""
+
+#: ../../ops.rst:54
+msgid "|vmcore2mckdump|"
+msgstr ""
+
+#: ../../ops.rst:54
+msgid "ダンプ形式変換ツール"
+msgstr ""
+
+#: ../../ops.rst:62
+msgid "以下、関連コマンドおよび関連関数のインターフェイスを説明する。"
+msgstr ""
+
+#: ../../ops.rst:65
+msgid "インターフェイス"
+msgstr ""
+
+#: ../../ops.rst:68
+msgid "カーネル引数"
+msgstr ""
+
+#: ../../ops.rst:70
+msgid "McKernelのカーネル引数を表 :numref:`tab-kargs` に示す。"
+msgstr ""
+
+#: ../../ops.rst:74
+msgid "McKernelのカーネル引数"
+msgstr ""
+
+#: ../../ops.rst:77
+msgid "hidos"
+msgstr ""
+
+#: ../../ops.rst:77
+msgid "IKCを有効にする。"
+msgstr ""
+
+#: ../../ops.rst:79
+msgid "|dlv|"
+msgstr ""
+
+#: ../../ops.rst
+msgid "Linuxのpanicハンドラ経由でダンプを行った場合の、ダ"
+msgstr ""
+
+#: ../../ops.rst
+msgid "ンプ対象とするメモリ領域の種類を<level>に設定する。"
+msgstr ""
+
+#: ../../ops.rst
+msgid "設定可能な値は以下の通り。"
+msgstr ""
+
+#: ../../ops.rst:85 ../../ops.rst:166 ../../ops.rst:198 ../../ops.rst:236
+msgid "0"
+msgstr ""
+
+#: ../../ops.rst:86 ../../ops.rst:167
+msgid "IHKがMcKernelに割り当てたメモリ領域を出力する"
+msgstr ""
+
+#: ../../ops.rst:87 ../../ops.rst:168
+msgid "24"
+msgstr ""
+
+#: ../../ops.rst:88 ../../ops.rst:169
+msgid "カーネルが使用しているメモリ領域を出力する"
+msgstr ""
+
+#: ../../ops.rst:90 ../../ops.rst:171
+msgid "指定がなかった場合は24が用いられる。"
+msgstr ""
+
+#: ../../ops.rst:92
+msgid "|allow|"
+msgstr ""
+
+#: ../../ops.rst
+msgid "McKernelに割り当てられたCPU数より大きい数のスレッド"
+msgstr ""
+
+#: ../../ops.rst
+msgid "またはプロセスの生成を許可する。この引数が指定され"
+msgstr ""
+
+#: ../../ops.rst
+msgid "ない場合に、CPU数より大きい数のスレッドまたはプロセ"
+msgstr ""
+
+#: ../../ops.rst
+msgid "スをclone(), fork(), vfork()などで生成しようとする"
+msgstr ""
+
+#: ../../ops.rst
+msgid "と、当該システムコールがEINVALエラーを返す。"
+msgstr ""
+
+#: ../../ops.rst:107 ../../ops.rst:209
+msgid "書式"
+msgstr ""
+
+#: ../../ops.rst:115 ../../ops.rst:219
+msgid "オプション"
+msgstr ""
+
+#: ../../ops.rst:118
+msgid "|opt-c|"
+msgstr ""
+
+#: ../../ops.rst
+msgid "McKernelに割り当てるCPUのリストを指定する。フォー"
+msgstr ""
+
+#: ../../ops.rst
+msgid "マットは以下の通り。"
+msgstr ""
+
+#: ../../ops.rst
+msgid "<CPU logical id>,<CPU logical id>...または"
+msgstr ""
+
+#: ../../ops.rst
+msgid "<CPU logical id>-<CPU logical id>,<CPU logical id>"
+msgstr ""
+
+#: ../../ops.rst
+msgid "-<CPU logical id>...または両者の混合。"
+msgstr ""
+
+#: ../../ops.rst:124
+msgid "|opt-r|"
+msgstr ""
+
+#: ../../ops.rst
+msgid "McKernelのCPUがIKCメッセージを送るLinux"
+msgstr ""
+
+#: ../../ops.rst
+msgid "CPUを指定する。フォーマットは以下の通り。"
+msgstr ""
+
+#: ../../ops.rst
+msgid "<CPU list>:<CPU id>+<CPU list>:<CPU id>..."
+msgstr ""
+
+#: ../../ops.rst
+msgid "<CPU list>のフォーマットは-cオプションにおけるもの"
+msgstr ""
+
+#: ../../ops.rst
+msgid "と同じである。"
+msgstr ""
+
+#: ../../ops.rst
+msgid "各<CPU list>:<CPU id>は<CPU list>で示されるMcKernel"
+msgstr ""
+
+#: ../../ops.rst
+msgid "のCPUが<CPU logical id>で示されるLinuxのCPUにIKC"
+msgstr ""
+
+#: ../../ops.rst
+msgid "メッセージを送信することを意味する。"
+msgstr ""
+
+#: ../../ops.rst:133
+msgid "|opt-m|"
+msgstr ""
+
+#: ../../ops.rst
+msgid "McKernelに割り当てるメモリ領域を指定する。フォーマッ"
+msgstr ""
+
+#: ../../ops.rst
+msgid "トは以下の通り。"
+msgstr ""
+
+#: ../../ops.rst
+msgid "<size>@<NUMA-id>, <size>@<NUMA-id>..."
+msgstr ""
+
+#: ../../ops.rst:137
+msgid "|opt-f|"
+msgstr ""
+
+#: ../../ops.rst
+msgid "ihkmondが使用するsyslogプロトコルのfacilityを指定す"
+msgstr ""
+
+#: ../../ops.rst
+msgid "る。デフォルトはLOG_LOCAL6。"
+msgstr ""
+
+#: ../../ops.rst:140
+msgid "|opt-o|"
+msgstr ""
+
+#: ../../ops.rst
+msgid "IHKのデバイスファイル(/dev/mcd*, /dev/mcos*)のオー"
+msgstr ""
+
+#: ../../ops.rst
+msgid "ナーとグループの値を<user>[:<group>]の形式で指定す"
+msgstr ""
+
+#: ../../ops.rst
+msgid "る。デフォルトはmcreboot.shを実行したユーザ。"
+msgstr ""
+
+#: ../../ops.rst:144
+msgid "|opt-i|"
+msgstr ""
+
+#: ../../ops.rst
+msgid "ihkmondがハングアップ検知のためにOS状態を確認する時"
+msgstr ""
+
+#: ../../ops.rst
+msgid "間間隔を秒単位で指定する。-1が指定された場合はハン"
+msgstr ""
+
+#: ../../ops.rst
+msgid "グアップ検知を行わない。指定がない場合はハングアッ"
+msgstr ""
+
+#: ../../ops.rst
+msgid "プ検知を行わない。"
+msgstr ""
+
+#: ../../ops.rst:149
+msgid "|opt-k|"
+msgstr ""
+
+#: ../../ops.rst
+msgid "カーネルメッセージの/dev/logへのリダイレクト有無を"
+msgstr ""
+
+#: ../../ops.rst
+msgid "指定する。0が指定された場合はリダイレクトを行わず、"
+msgstr ""
+
+#: ../../ops.rst
+msgid "0以外が指定された場合はリダイレクトを行う。指定がな"
+msgstr ""
+
+#: ../../ops.rst
+msgid "い場合はリダイレクトを行わない。"
+msgstr ""
+
+#: ../../ops.rst:154
+msgid "-q <irq>"
+msgstr ""
+
+#: ../../ops.rst
+msgid "IHKが使用するIRQ番号を指定する。指定がない場合は"
+msgstr ""
+
+#: ../../ops.rst
+msgid "64-255の範囲で空いているものを使用する。"
+msgstr ""
+
+#: ../../ops.rst:157
+msgid "-t"
+msgstr ""
+
+#: ../../ops.rst
+msgid "（x86_64アーキテクチャのみ）Turbo"
+msgstr ""
+
+#: ../../ops.rst
+msgid "Boostをオンにする。デフォルトはオフ。"
+msgstr ""
+
+#: ../../ops.rst:160
+msgid "-d <level>"
+msgstr ""
+
+#: ../../ops.rst:173
+msgid "-O"
+msgstr ""
+
+#: ../../ops.rst
+msgid "またはプロセスの生成を許可する。指定がない場合は許可"
+msgstr ""
+
+#: ../../ops.rst
+msgid "しない。すなわち、CPU数より大きい数のスレッドまたは"
+msgstr ""
+
+#: ../../ops.rst
+msgid "プロセスを生成しようとするとエラーとなる。"
+msgstr ""
+
+#: ../../ops.rst:189 ../../ops.rst:226
+msgid "説明"
+msgstr ""
+
+#: ../../ops.rst:191
+msgid ""
+"McKernel関連カーネルモジュールをinsmodし、<cpulist>で指定されたCPUと<memlist>で指定されたメモリ領域からなるパーティションを作成し、IKC"
+" mapを<ikcmap>に設定し、前記パーティションにMcKernelをブートする。"
+msgstr ""
+
+#: ../../ops.rst:195 ../../ops.rst:233
+msgid "戻り値"
+msgstr ""
+
+#: ../../ops.rst:198 ../../ops.rst:236
+msgid "正常終了"
+msgstr ""
+
+#: ../../ops.rst:200 ../../ops.rst:238
+msgid "0以外"
+msgstr ""
+
+#: ../../ops.rst:200 ../../ops.rst:238
+msgid "エラー"
+msgstr ""
+
+#: ../../ops.rst:221
+msgid "なし"
+msgstr ""
+
+#: ../../ops.rst:228
+msgid "McKernelをシャットダウンし、McKernel用パーティションを削除し、関連カーネルモジュールをrmmodする。"
+msgstr ""
+
+#: ../../ops.rst:247
+msgid "ダンプ解析コマンド"
+msgstr ""
+
+#: ../../ops.rst:252
+msgid "ダンプ形式変換コマンド"
+msgstr ""
+
+#: ../../ops.rst:257
+msgid "ブート手順"
+msgstr ""
+
+#: ../../ops.rst:259
+msgid "mcreboot.shを用いてブート手順を説明する。"
+msgstr ""
+
+#: ../../ops.rst:261 ../../ops.rst:908
+msgid "スクリプトは以下の通り。"
+msgstr ""
+
+#: ../../ops.rst:854 ../../ops.rst:1052
+msgid "手順は以下の通り。"
+msgstr ""
+
+#: ../../ops.rst:856
+msgid "ihkmondを起動する。ihkmondは任意のタイミングで起動してよい。これは、ihkmondはOSインスタンスの作成を検知して動作を開始するためである。（83行目）"
+msgstr ""
+
+#: ../../ops.rst:858
+msgid "Linuxのカーネルバージョンが、mcoverlayfsが動作するものであるかを確認する。（200–216行目）"
+msgstr ""
+
+#: ../../ops.rst:860
+msgid "irqbalanceを停止する。（251–257行目）"
+msgstr ""
+
+#: ../../ops.rst:862
+msgid ""
+"/proc/irq/[n]/affinityの設定を保存した上でMcKernel "
+"CPUを担当から外す。担当CPUが無くなる場合は、全てのLinux CPUを指定する。（269–303行目）"
+msgstr ""
+
+#: ../../ops.rst:866
+msgid "ihk.koをinsmodする。（307行目）"
+msgstr ""
+
+#: ../../ops.rst:868
+msgid "Linuxによるメモリフラグメンテーションを緩和するために以下を実施する。（313–320行目）"
+msgstr ""
+
+#: ../../ops.rst:870
+msgid "アクティブでないプロセスを積極的にスワップアウトするように設定する"
+msgstr ""
+
+#: ../../ops.rst:872
+msgid "クリーンなページキャッシュを無効化し、またdentriesやinodeのslabオブジェクトのうち可能なものを破棄する"
+msgstr ""
+
+#: ../../ops.rst:874
+msgid "連続する空き領域を結合してより大きな空き領域にまとめる"
+msgstr ""
+
+#: ../../ops.rst:876
+msgid ""
+"ihk-smp-x86.koをinsmodする。（340行目）ihk-smp-x86.koは関数をihk.koに登録する。このため、ihk-"
+"smp-x86.koはihk.koをinsmodした後にinsmodする必要がある。"
+msgstr ""
+
+#: ../../ops.rst:878
+msgid "メモリを予約する。（370行目）"
+msgstr ""
+
+#: ../../ops.rst:880
+msgid "CPUを予約する。（374行目）"
+msgstr ""
+
+#: ../../ops.rst:882
+msgid "McKernelのカーネルモジュールmcctrl.koをinsmodする。（382行目）mcctrl.koはMcKernelブート時に呼ばれる関数をihk.koに登録する。このため、mcctrl.koのinsmodはihk.koのinsmodの後に、またブートの前に行う必要がある。"
+msgstr ""
+
+#: ../../ops.rst:884
+msgid "OSインスタンスを作成する。（406行目）"
+msgstr ""
+
+#: ../../ops.rst:886
+msgid "OSインスタンスにCPUを割り当てる。（412行目）"
+msgstr ""
+
+#: ../../ops.rst:888
+msgid "McKernel CPUのIKCメッセージ送信先のLinux CPUを設定する。（419行目）"
+msgstr ""
+
+#: ../../ops.rst:890
+msgid "OSインスタンスにメモリを割り当てる。（426行目）"
+msgstr ""
+
+#: ../../ops.rst:892
+msgid "カーネルイメージをロードする。（432行目）"
+msgstr ""
+
+#: ../../ops.rst:894
+msgid "カーネル引数をカーネルに渡す。（438行目）"
+msgstr ""
+
+#: ../../ops.rst:896
+msgid "カーネルをブートする。（444行目）"
+msgstr ""
+
+#: ../../ops.rst:898
+msgid ""
+"/proc, "
+"/sysファイルの準備をする。また、その中でmcoverlayfs.koをinsmodする。mcoverlayfs.koは他モジュールとの依存関係を持たない。（454行目から567行目）なお、関数インターフェイスでの対応関数はihk_os_create_pseudofs()である。"
+msgstr ""
+
+#: ../../ops.rst:900
+msgid "irqbalanceを、Linux CPUのみを対象とする設定で開始する。（569–587行目）"
+msgstr ""
+
+#: ../../ops.rst:904
+msgid "シャットダウン手順"
+msgstr ""
+
+#: ../../ops.rst:906
+msgid "mcstop+release.shを用いてシャットダウン手順を説明する。"
+msgstr ""
+
+#: ../../ops.rst:1054
+msgid "ブート時にLinux CPUのみを対象とする設定で開始されたirqbalanceを停止する。（24–33行目）"
+msgstr ""
+
+#: ../../ops.rst:1057
+msgid "全てのOSインスタンスを破壊する。OSインスタンスに割り当てられていた資源はIHKがLWKのために予約した状態に移行する。（35–50行目）"
+msgstr ""
+
+#: ../../ops.rst:1059
+msgid "IHKがLWKのために予約していた資源を開放する。（52–77行目）"
+msgstr ""
+
+#: ../../ops.rst:1061
+msgid "mcctrl.koをrmmodする。（81行目）"
+msgstr ""
+
+#: ../../ops.rst:1063
+msgid ""
+"/proc, "
+"/sysファイルの準備をする。また、その中でmcoverlayfs.koをrmmodする。（87–100行目）なお、関数インターフェイスでの対応関数はihk_os_destroy_pseudofs()である。"
+msgstr ""
+
+#: ../../ops.rst:1065
+msgid "ihk-smp-x86.koをrmmodする。（104行目）"
+msgstr ""
+
+#: ../../ops.rst:1067
+msgid "ihk.koをrmmodする。（112行目）"
+msgstr ""
+
+#: ../../ops.rst:1069
+msgid "ihkmondを停止する。（121行目）"
+msgstr ""
+
+#: ../../ops.rst:1071
+msgid "/proc/irq/[n]/affinityの設定をブート時に保存しておいたものに戻し、ブート前の設定でirqbalanceを開始する。（124–135行目）"
+msgstr ""
+
+#: ../../ops.rst:1073
+msgid "Linuxカーネルのスワップアウト積極度の設定をデフォルトの値に戻す。（138行目）"
+msgstr ""
+
+#~ msgid "The following OS distributions and platforms are recommended:"
+#~ msgstr "推奨OSディストリビューションとプロセッサは以下の通り。"
+
+#~ msgid "OS distribution"
+#~ msgstr ""
+
+#~ msgid "CentOS 7.3 or later"
+#~ msgstr ""
+
+#~ msgid "RHEL 7.3 or later"
+#~ msgstr ""
+
+#~ msgid "Platform"
+#~ msgstr ""
+
+#~ msgid "Intel Xeon"
+#~ msgstr ""
+
+#~ msgid "Intel Xeon Phi"
+#~ msgstr ""
+
+#~ msgid "Fujitsu A64FX"
+#~ msgstr ""
+
+#~ msgid "Prepare files for building McKernel"
+#~ msgstr ""
+
+#~ msgid ""
+#~ "Grant read permission to the System.map"
+#~ " file of your kernel version on "
+#~ "the build machine:"
+#~ msgstr ""
+
+#~ msgid "Install the following packages to the build machine:"
+#~ msgstr ""
+
+#~ msgid "When having access to repositories"
+#~ msgstr ""
+
+#~ msgid "On RHEL 8, enable the CodeReady Linux Builder (CLB) repository:"
+#~ msgstr ""
+
+#~ msgid "On CentOS 8, enable the PowerTools repository:"
+#~ msgstr ""
+
+#~ msgid "Install with yum:"
+#~ msgstr ""
+
+#~ msgid "When not having access to repositories"
+#~ msgstr ""
+
+#~ msgid ""
+#~ "Ask the system administrator to install"
+#~ " them. Note that ``libdwarf-devel`` "
+#~ "is in the CodeReady Linux Builder "
+#~ "repository on RHEL 8 or in the "
+#~ "PowerTools repository on CentOS 8."
+#~ msgstr ""
+
+#~ msgid "Clone, compile, install"
+#~ msgstr ""
+
+#~ msgid "Clone the source code:"
+#~ msgstr ""
+
+#~ msgid "(Optional) Checkout to the specific branch or version:"
+#~ msgstr ""
+
+#~ msgid ""
+#~ "Foe example, if you want to try"
+#~ " the development branch, use “development”"
+#~ " as the pathspec. If you want "
+#~ "to try the prerelease version 1.7.0-0.2,"
+#~ " use “1.7.0-0.2”."
+#~ msgstr ""
+
+#~ msgid "Move to build directory:"
+#~ msgstr ""
+
+#~ msgid "Run cmake:"
+#~ msgstr ""
+
+#~ msgid "When not cross-compiling:"
+#~ msgstr ""
+
+#~ msgid "When cross-compiling:"
+#~ msgstr ""
+
+#~ msgid "Install with cmake"
+#~ msgstr ""
+
+#~ msgid "Install with make:"
+#~ msgstr ""
+
+#~ msgid ""
+#~ "The kernel modules and McKernel kernel"
+#~ " image should be installed under the"
+#~ " **ihk+mckernel** folder in your home "
+#~ "directory."
+#~ msgstr ""
+
+#~ msgid "Install with rpm"
+#~ msgstr ""
+
+#~ msgid "Create the tarball and the spec file:"
+#~ msgstr ""
+
+#~ msgid "Create the rpm package:"
+#~ msgstr ""
+
+#~ msgid "Install the rpm package:"
+#~ msgstr ""
+
+#~ msgid ""
+#~ "The kernel modules and McKernel kernel"
+#~ " image are installed under the "
+#~ "standard system directories."
+#~ msgstr ""
+
+#~ msgid "Prepare files and change settings for installing McKernel"
+#~ msgstr ""
+
+#~ msgid "Disable SELinux of the compute nodes:"
+#~ msgstr ""
+
+#~ msgid "Change the file to SELINUX=disabled. And then reboot the compute nodes:"
+#~ msgstr ""
+
+#~ msgid "Install the following packages to the compute nodes:"
+#~ msgstr ""
+
+#~ msgid ""
+#~ "Ask the system administrator to install"
+#~ " them. Note that ``libdwarf`` is in"
+#~ " the CodeReady Linux Builder repository "
+#~ "on RHEL 8 or in the PowerTools "
+#~ "repository on CentOS 8."
+#~ msgstr ""
+

docs/locales/ja/LC_MESSAGES/quick.po

@@ -0,0 +1,458 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) 2020, Masamichi Takagi, Balazs Gerofi, Yutaka Ishikawa
+# This file is distributed under the same license as the IHK/McKernel
+# package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2020.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: IHK/McKernel \n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2020-08-04 16:40+0900\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=utf-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: Babel 2.7.0\n"
+
+#: ../../quick.rst:6
+msgid "Introduction"
+msgstr ""
+
+#: ../../summary.rst:1
+msgid ""
+"IHK/McKernel is a light-weight multi-kernel operating system designed for"
+" high-end supercomputing. It runs Linux and McKernel, a light-weight "
+"kernel (LWK), side-by-side inside compute nodes and aims at the "
+"following:"
+msgstr ""
+
+#: ../../summary.rst:6
+msgid ""
+"Provide scalable and consistent execution of large-scale parallel "
+"scientific applications, but at the same time maintain the ability to "
+"rapidly adapt to new hardware features and emerging programming models"
+msgstr ""
+
+#: ../../summary.rst:10
+msgid ""
+"Provide efficient memory and device management so that resource "
+"contention and data movement are minimized at the system level"
+msgstr ""
+
+#: ../../summary.rst:12
+msgid ""
+"Eliminate OS noise by isolating OS services in Linux and provide jitter "
+"free execution on the LWK"
+msgstr ""
+
+#: ../../summary.rst:14
+msgid ""
+"Support the full POSIX/Linux APIs by selectively offloading (slow-path) "
+"system calls to Linux"
+msgstr ""
+
+#: ../../background.rst:2
+msgid "Background and Motivation"
+msgstr ""
+
+#: ../../background.rst:4
+msgid ""
+"With the growing complexity of high-end supercomputers, the current "
+"system software stack faces significant challenges as we move forward to "
+"exascale and beyond. The necessity to deal with extreme degree of "
+"parallelism, heterogeneous architectures, multiple levels of memory "
+"hierarchy, power constraints, etc., advocates operating systems that can "
+"rapidly adapt to new hardware requirements, and that can support novel "
+"programming paradigms and runtime systems. On the other hand, a new class"
+" of more dynamic and complex applications are also on the horizon, with "
+"an increasing demand for application constructs such as in-situ analysis,"
+" workflows, elaborate monitoring and performance tools. This complexity "
+"relies not only on the rich features of POSIX, but also on the Linux APIs"
+" (such as the */proc*, */sys* filesystems, etc.) in particular."
+msgstr ""
+
+#: ../../background.rst:19
+msgid "Two Traditional HPC OS Approaches"
+msgstr ""
+
+#: ../../background.rst:21
+msgid ""
+"Traditionally, light-weight operating systems specialized for HPC "
+"followed two approaches to tackle scalable execution of large-scale "
+"applications. In the full weight kernel (FWK) approach, a full Linux "
+"environment is taken as the basis, and features that inhibit attaining "
+"HPC scalability are removed, i.e., making it light-weight. The pure "
+"light-weight kernel (LWK) approach, on the other hand, starts from "
+"scratch and effort is undertaken to add sufficient functionality so that "
+"it provides a familiar API, typically something close to that of a "
+"general purpose OS, while at the same time it retains the desired "
+"scalability and reliability attributes. Neither of these approaches "
+"yields a fully Linux compatible environment."
+msgstr ""
+
+#: ../../background.rst:34
+msgid "The Multi-kernel Approach"
+msgstr ""
+
+#: ../../background.rst:36
+msgid ""
+"A hybrid approach recognized recently by the system software community is"
+" to run Linux simultaneously with a lightweight kernel on compute nodes "
+"and multiple research projects are now pursuing this direction. The basic"
+" idea is that simulations run on an HPC tailored lightweight kernel, "
+"ensuring the necessary isolation for noiseless execution of parallel "
+"applications, but Linux is leveraged so that the full POSIX API is "
+"supported. Additionally, the small code base of the LWK can also "
+"facilitate rapid prototyping for new, exotic hardware features. "
+"Nevertheless, the questions of how to share node resources between the "
+"two types of kernels, where do device drivers execute, how exactly do the"
+" two kernels interact with each other and to what extent are they "
+"integrated, remain subjects of ongoing debate."
+msgstr ""
+
+#: ../../archtecture.rst:2
+msgid "Architectural Overview"
+msgstr ""
+
+#: ../../archtecture.rst:4
+msgid ""
+"At the heart of the stack is a low-level software infrastructure called "
+"Interface for Heterogeneous Kernels (IHK). IHK is a general framework "
+"that provides capabilities for partitioning resources in a many-core "
+"environment (e.g.,CPU cores and physical memory) and it enables "
+"management of lightweight kernels. IHK can allocate and release host "
+"resources dynamically and no reboot of the host machine is required when "
+"altering configuration. IHK also provides a low-level inter-kernel "
+"messaging infrastructure, called the Inter-Kernel Communication (IKC) "
+"layer. An architectural overview of the main system components is shown "
+"below."
+msgstr ""
+
+#: ../../archtecture.rst:18
+msgid ""
+"McKernel is a lightweight kernel written from scratch. It is designed for"
+" HPC and is booted from IHK. McKernel retains a binary compatible ABI "
+"with Linux, however, it implements only a small set of performance "
+"sensitive system calls and the rest are offloaded to Linux. Specifically,"
+" McKernel has its own memory management, it supports processes and multi-"
+"threading with a simple round-robin cooperative (tick-less) scheduler, "
+"and it implements signaling. It also allows inter-process memory mappings"
+" and it provides interfaces to hardware performance counters."
+msgstr ""
+
+#: ../../archtecture.rst:29
+msgid "Functionality"
+msgstr ""
+
+#: ../../archtecture.rst:31
+msgid ""
+"An overview of some of the principal functionalities of the IHK/McKernel "
+"stack is provided below."
+msgstr ""
+
+#: ../../archtecture.rst:35
+msgid "System Call Offloading"
+msgstr ""
+
+#: ../../archtecture.rst:37
+msgid ""
+"System call forwarding in McKernel is implemented as follows. When an "
+"offloaded system call occurs, McKernel marshals the system call number "
+"along with its arguments and sends a message to Linux via a dedicated IKC"
+" channel. The corresponding proxy process running on Linux is by default "
+"waiting for system call requests through an ioctl() call into IHK’s "
+"system call delegator kernel module. The delegator kernel module’s IKC "
+"interrupt handler wakes up the proxy process, which returns to userspace "
+"and simply invokes the requested system call. Once it obtains the return "
+"value, it instructs the delegator module to send the result back to "
+"McKernel, which subsequently passes the value to user-space."
+msgstr ""
+
+#: ../../archtecture.rst:49
+msgid "Unified Address Space"
+msgstr ""
+
+#: ../../archtecture.rst:51
+msgid ""
+"The unified address space model in IHK/McKernel ensures that offloaded "
+"system calls can seamlessly resolve arguments even in case of pointers. "
+"This mechanism is depicted below and is implemented as follows."
+msgstr ""
+
+#: ../../archtecture.rst:58
+msgid ""
+"First, the proxy process is compiled as a position independent binary, "
+"which enables us to map the code and data segments specific to the proxy "
+"process to an address range which is explicitly excluded from McKernel’s "
+"user space. The grey box on the right side of the figure demonstrates the"
+" excluded region. Second, the entire valid virtual address range of "
+"McKernel’s application user-space is covered by a special mapping in the "
+"proxy process for which we use a pseudo file mapping in Linux. This "
+"mapping is indicated by the blue box on the left side of the figure."
+msgstr ""
+
+#: ../../install.rst:4
+msgid "Installation"
+msgstr "インストール"
+
+#: ../../install.rst:6
+msgid "The following OS distributions and platforms are recommended:"
+msgstr "推奨OSディストリビューションとプロセッサは以下の通り。"
+
+#: ../../install.rst:8
+msgid "OS distribution"
+msgstr ""
+
+#: ../../install.rst:10
+msgid "CentOS 7.3 or later"
+msgstr ""
+
+#: ../../install.rst:11
+msgid "RHEL 7.3 or later"
+msgstr ""
+
+#: ../../install.rst:13
+msgid "Platform"
+msgstr ""
+
+#: ../../install.rst:15
+msgid "Intel Xeon"
+msgstr ""
+
+#: ../../install.rst:16
+msgid "Intel Xeon Phi"
+msgstr ""
+
+#: ../../install.rst:17
+msgid "Fujitsu A64FX"
+msgstr ""
+
+#: ../../install.rst:20
+msgid "Prepare files for building McKernel"
+msgstr ""
+
+#: ../../install.rst:22
+msgid ""
+"Grant read permission to the System.map file of your kernel version on "
+"the build machine:"
+msgstr ""
+
+#: ../../install.rst:28
+msgid "Install the following packages to the build machine:"
+msgstr ""
+
+#: ../../install.rst:35 ../../install.rst:179
+msgid "When having access to repositories"
+msgstr ""
+
+#: ../../install.rst:37 ../../install.rst:181
+msgid "On RHEL 8, enable the CodeReady Linux Builder (CLB) repository:"
+msgstr ""
+
+#: ../../install.rst:43 ../../install.rst:187
+msgid "On CentOS 8, enable the PowerTools repository:"
+msgstr ""
+
+#: ../../install.rst:49 ../../install.rst:193
+msgid "Install with yum:"
+msgstr ""
+
+#: ../../install.rst:56 ../../install.rst:200
+msgid "When not having access to repositories"
+msgstr ""
+
+#: ../../install.rst:58
+msgid ""
+"Ask the system administrator to install them. Note that ``libdwarf-"
+"devel`` is in the CodeReady Linux Builder repository on RHEL 8 or in the "
+"PowerTools repository on CentOS 8."
+msgstr ""
+
+#: ../../install.rst:61
+msgid "Clone, compile, install"
+msgstr ""
+
+#: ../../install.rst:63
+msgid "Clone the source code:"
+msgstr ""
+
+#: ../../install.rst:71
+msgid "(Optional) Checkout to the specific branch or version:"
+msgstr ""
+
+#: ../../install.rst:79
+msgid ""
+"Foe example, if you want to try the development branch, use “development”"
+" as the pathspec. If you want to try the prerelease version 1.7.0-0.2, "
+"use “1.7.0-0.2”."
+msgstr ""
+
+#: ../../install.rst:83
+msgid "Move to build directory:"
+msgstr ""
+
+#: ../../install.rst:89
+msgid "Run cmake:"
+msgstr ""
+
+#: ../../install.rst:92 ../../install.rst:135
+msgid "When not cross-compiling:"
+msgstr ""
+
+#: ../../install.rst:99 ../../install.rst:142
+msgid "When cross-compiling:"
+msgstr ""
+
+#: ../../install.rst:111
+msgid "Install with cmake"
+msgstr ""
+
+#: ../../install.rst:113
+msgid "Install with make:"
+msgstr ""
+
+#: ../../install.rst:119
+msgid ""
+"The kernel modules and McKernel kernel image should be installed under "
+"the **ihk+mckernel** folder in your home directory."
+msgstr ""
+
+#: ../../install.rst:123
+msgid "Install with rpm"
+msgstr ""
+
+#: ../../install.rst:125
+msgid "Create the tarball and the spec file:"
+msgstr ""
+
+#: ../../install.rst:132
+msgid "Create the rpm package:"
+msgstr ""
+
+#: ../../install.rst:148
+msgid "Install the rpm package:"
+msgstr ""
+
+#: ../../install.rst:154
+msgid ""
+"The kernel modules and McKernel kernel image are installed under the "
+"standard system directories."
+msgstr ""
+
+#: ../../install.rst:158
+msgid "Prepare files and change settings for installing McKernel"
+msgstr ""
+
+#: ../../install.rst:160
+msgid "Disable SELinux of the compute nodes:"
+msgstr ""
+
+#: ../../install.rst:166
+msgid "Change the file to SELINUX=disabled. And then reboot the compute nodes:"
+msgstr ""
+
+#: ../../install.rst:172
+msgid "Install the following packages to the compute nodes:"
+msgstr ""
+
+#: ../../install.rst:202
+msgid ""
+"Ask the system administrator to install them. Note that ``libdwarf`` is "
+"in the CodeReady Linux Builder repository on RHEL 8 or in the PowerTools "
+"repository on CentOS 8."
+msgstr ""
+
+#: ../../boot_run_shutdown.rst:2
+msgid "Boot McKernel"
+msgstr ""
+
+#: ../../boot_run_shutdown.rst:4
+msgid ""
+"A boot script called ``mcreboot.sh`` is provided under ``sbin`` in the "
+"install folder. To boot on logical CPU 1 with 512MB of memory, use the "
+"following invocation:"
+msgstr ""
+
+#: ../../boot_run_shutdown.rst:14
+msgid ""
+"You should see something similar like this if you display the McKernel’s "
+"kernel message log:"
+msgstr ""
+
+#: ../../boot_run_shutdown.rst:42
+msgid "Run a simple program on McKernel"
+msgstr ""
+
+#: ../../boot_run_shutdown.rst:44
+msgid ""
+"The mcexec command line tool (which is also the Linux proxy process) can "
+"be used for executing applications on McKernel:"
+msgstr ""
+
+#: ../../boot_run_shutdown.rst:53
+msgid "Shutdown McKernel"
+msgstr ""
+
+#: ../../boot_run_shutdown.rst:55
+msgid ""
+"Finally, to shutdown McKernel and release CPU/memory resources back to "
+"Linux use the following command:"
+msgstr ""
+
+#: ../../team.rst:2
+msgid "The Team"
+msgstr ""
+
+#: ../../team.rst:4
+msgid ""
+"The McKernel project was started at The University of Tokyo and currently"
+" it is mainly developed at RIKEN. Some of our collaborators include:"
+msgstr ""
+
+#: ../../team.rst:8
+msgid "Hitachi"
+msgstr ""
+
+#: ../../team.rst:9
+msgid "Fujitsu"
+msgstr ""
+
+#: ../../team.rst:10
+msgid "CEA (France)"
+msgstr ""
+
+#: ../../team.rst:11
+msgid "NEC"
+msgstr ""
+
+#: ../../license.rst:2
+msgid "License"
+msgstr ""
+
+#: ../../license.rst:4
+msgid "McKernel is GPL licensed, as found in the LICENSE file."
+msgstr ""
+
+#: ../../contact.rst:2
+msgid "Contact"
+msgstr ""
+
+#: ../../contact.rst:4
+msgid ""
+"Please give your feedback to us via one of the following mailing lists. "
+"Subscription via `www.pccluster.org "
+"<http://www.pccluster.org/mailman/listinfo/mckernel-users>`__ is needed."
+msgstr ""
+
+#: ../../contact.rst:9
+msgid "English: mckernel-users@pccluster.org"
+msgstr ""
+
+#: ../../contact.rst:10
+msgid "Japanese: mckernel-users-jp@pccluster.org"
+msgstr ""
+

docs/locales/ja/LC_MESSAGES/spec-ihk.po

@@ -0,0 +1,28 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) 2020, Masamichi Takagi, Balazs Gerofi, Yutaka Ishikawa
+# This file is distributed under the same license as the IHK/McKernel
+# package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2020.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: IHK/McKernel \n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2020-08-04 16:40+0900\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=utf-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: Babel 2.7.0\n"
+
+#: ../../spec-ihk.md:1
+msgid "hi"
+msgstr ""
+
+#: ../../spec-ihk.md:3
+msgid ":download:IHK Spec <ihk.pdf>"
+msgstr ""
+

docs/locales/ja/LC_MESSAGES/spec-mckernel.po

@@ -0,0 +1,24 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) 2020, Masamichi Takagi, Balazs Gerofi, Yutaka Ishikawa
+# This file is distributed under the same license as the IHK/McKernel
+# package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2020.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: IHK/McKernel \n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2020-08-04 16:40+0900\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=utf-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: Babel 2.7.0\n"
+
+#: ../../spec-mckernel.md:1
+msgid "Hello"
+msgstr ""
+

docs/locales/ja/LC_MESSAGES/spec.po

@@ -0,0 +1,48 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) 2020, Masamichi Takagi, Balazs Gerofi, Yutaka Ishikawa
+# This file is distributed under the same license as the IHK/McKernel
+# package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2020.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: IHK/McKernel \n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2020-08-07 10:00+0900\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=utf-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: Babel 2.7.0\n"
+
+#: ../../spec/ihk.rst:6
+msgid "External Specs"
+msgstr ""
+
+#: ../../spec/ihk.rst:9
+msgid "Overview"
+msgstr ""
+
+#: ../../spec/ihk.rst:12
+msgid "Function Specs"
+msgstr ""
+
+#: ../../spec/ihk.rst:15
+msgid "Command / Daemon Specs"
+msgstr ""
+
+#: ../../spec/ihk.rst:18
+msgid "Booting LWK"
+msgstr ""
+
+#: ../../spec/mckernel.rst:6
+msgid "Interfaces"
+msgstr ""
+
+#: ../../spec/mckernel.rst:9
+msgid "Interface details"
+msgstr ""
+

docs/locales/ja/LC_MESSAGES/summary.po

@@ -0,0 +1,53 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) 2020, Masamichi Takagi, Balazs Gerofi, Yutaka Ishikawa
+# This file is distributed under the same license as the IHK/McKernel
+# package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2020.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: IHK/McKernel \n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2020-08-04 16:40+0900\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=utf-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: Babel 2.7.0\n"
+
+#: ../../summary.rst:1
+msgid ""
+"IHK/McKernel is a light-weight multi-kernel operating system designed for"
+" high-end supercomputing. It runs Linux and McKernel, a light-weight "
+"kernel (LWK), side-by-side inside compute nodes and aims at the "
+"following:"
+msgstr ""
+
+#: ../../summary.rst:6
+msgid ""
+"Provide scalable and consistent execution of large-scale parallel "
+"scientific applications, but at the same time maintain the ability to "
+"rapidly adapt to new hardware features and emerging programming models"
+msgstr ""
+
+#: ../../summary.rst:10
+msgid ""
+"Provide efficient memory and device management so that resource "
+"contention and data movement are minimized at the system level"
+msgstr ""
+
+#: ../../summary.rst:12
+msgid ""
+"Eliminate OS noise by isolating OS services in Linux and provide jitter "
+"free execution on the LWK"
+msgstr ""
+
+#: ../../summary.rst:14
+msgid ""
+"Support the full POSIX/Linux APIs by selectively offloading (slow-path) "
+"system calls to Linux"
+msgstr ""
+

docs/locales/ja/LC_MESSAGES/team.po

@@ -0,0 +1,46 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) 2020, Masamichi Takagi, Balazs Gerofi, Yutaka Ishikawa
+# This file is distributed under the same license as the IHK/McKernel
+# package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2020.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: IHK/McKernel \n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2020-08-04 16:40+0900\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=utf-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: Babel 2.7.0\n"
+
+#: ../../team.rst:2
+msgid "The Team"
+msgstr ""
+
+#: ../../team.rst:4
+msgid ""
+"The McKernel project was started at The University of Tokyo and currently"
+" it is mainly developed at RIKEN. Some of our collaborators include:"
+msgstr ""
+
+#: ../../team.rst:8
+msgid "Hitachi"
+msgstr ""
+
+#: ../../team.rst:9
+msgid "Fujitsu"
+msgstr ""
+
+#: ../../team.rst:10
+msgid "CEA (France)"
+msgstr ""
+
+#: ../../team.rst:11
+msgid "NEC"
+msgstr ""
+

docs/locales/ja/LC_MESSAGES/users.po

@@ -0,0 +1,454 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) 2020, Masamichi Takagi, Balazs Gerofi, Yutaka Ishikawa
+# This file is distributed under the same license as the IHK/McKernel
+# package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2020.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: IHK/McKernel \n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2020-08-07 10:00+0900\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=utf-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: Babel 2.7.0\n"
+
+#: ../../users.rst:6
+msgid "Architectural Overview"
+msgstr ""
+
+#: ../../users.rst:8
+msgid ""
+"See `Quick Guide -- Architectural Overview <quick.html#architectural-"
+"overview>`__."
+msgstr ""
+
+#: ../../users.rst:11
+msgid "Running Programs"
+msgstr ""
+
+#: ../../users.rst:13
+msgid ""
+"You need to check if your application and pre-/post-processing programs "
+"are suited to run with McKernel. Follow the guide below to choose to run "
+"the whole on McKernel, or run the whole on Linux, or run pre-/post-"
+"processing on Linux and the application on McKernel:"
+msgstr ""
+
+#: ../../users.rst:16
+msgid "Application"
+msgstr ""
+
+#: ../../users.rst:18
+msgid ""
+"Run the whole on Linux if it issues system calls frequently and becoming "
+"the bottleneck with McKernel. For example, it's better to run on Linux "
+"those programs performing many file I/O operations."
+msgstr ""
+
+#: ../../users.rst:19
+msgid "Otherwise, run it on McKernel."
+msgstr ""
+
+#: ../../users.rst:21
+msgid "Pre-/Post-processing"
+msgstr ""
+
+#: ../../users.rst:23
+msgid ""
+"Run it on McKernel if it consumes a large amount of memory or the "
+"execution time isn't prolonged prohivitively with McKernel. The reason "
+"for the first condition is that the resource could be limited for Linux "
+"CPUs in the nodes for McKernel."
+msgstr ""
+
+#: ../../users.rst:24
+msgid "Otherwise, run it on Linux."
+msgstr ""
+
+#: ../../users.rst:28
+msgid "Modify job script"
+msgstr ""
+
+#: ../../users.rst:30
+msgid ""
+"When using job submission system, you need to modify the job scripts so "
+"that the job script itself is going to run on Linux. For example, with "
+"Fujitsu Technical Computing Suite (TCS), you need to specify "
+"``jobenv=linux`` by inserting the following line into the job script:"
+msgstr ""
+
+#: ../../users.rst:38
+msgid "Insert ``mcexec`` into the command line"
+msgstr ""
+
+#: ../../users.rst:40
+msgid ""
+"You need to insert ``mcexec`` into the command lines invoking the "
+"programs that you chose to run on McKernel:"
+msgstr ""
+
+#: ../../users.rst:43
+msgid "Non-MPI programs"
+msgstr ""
+
+#: ../../users.rst:45
+msgid "Insert ``mcexec`` before an executable:"
+msgstr ""
+
+#: ../../users.rst:52
+msgid "MPI programs"
+msgstr ""
+
+#: ../../users.rst:54
+msgid ""
+"Insert ``mcexec -n <processes-per-node>`` **after mpirun** and before an "
+"executable:"
+msgstr ""
+
+#: ../../users.rst:61
+msgid ""
+"``<processes-per-node>`` is the number of the processes per node and "
+"calculated by (number of MPI processes) / (number of nodes)."
+msgstr ""
+
+#: ../../users.rst:64
+msgid ""
+"For example, ``<processes-per-node>`` equals to 4 (=32/8) when specifying"
+" the number of processes and nodes as follows with Fujitsu Technical "
+"Computing Suite."
+msgstr ""
+
+#: ../../users.rst:74
+msgid "Limitations"
+msgstr ""
+
+#: ../../users.rst:76
+msgid ""
+"Pseudo devices such as /dev/mem and /dev/zero are not mmap()ed correctly "
+"even if the mmap() returns a success. An access of their mapping receives"
+" the SIGSEGV signal."
+msgstr ""
+
+#: ../../users.rst:80
+msgid ""
+"clone() supports only the following flags. All the other flags cause "
+"clone() to return error or are simply ignored."
+msgstr ""
+
+#: ../../users.rst:83
+msgid "CLONE_CHILD_CLEARTID"
+msgstr ""
+
+#: ../../users.rst:84
+msgid "CLONE_CHILD_SETTID"
+msgstr ""
+
+#: ../../users.rst:85
+msgid "CLONE_PARENT_SETTID"
+msgstr ""
+
+#: ../../users.rst:86
+msgid "CLONE_SETTLS"
+msgstr ""
+
+#: ../../users.rst:87
+msgid "CLONE_SIGHAND"
+msgstr ""
+
+#: ../../users.rst:88
+msgid "CLONE_VM"
+msgstr ""
+
+#: ../../users.rst:90
+msgid "PAPI has the following restriction."
+msgstr ""
+
+#: ../../users.rst:92
+msgid ""
+"Number of counters a user can use at the same time is up to the number of"
+" the physical counters in the processor."
+msgstr ""
+
+#: ../../users.rst:95
+msgid "msync writes back only the modified pages mapped by the calling process."
+msgstr ""
+
+#: ../../users.rst:98
+msgid "The following syscalls always return the ENOSYS error."
+msgstr ""
+
+#: ../../users.rst:100
+msgid "migrate_pages()"
+msgstr ""
+
+#: ../../users.rst:101
+msgid "move_pages()"
+msgstr ""
+
+#: ../../users.rst:102
+msgid "set_robust_list()"
+msgstr ""
+
+#: ../../users.rst:104
+msgid "The following syscalls always return the EOPNOTSUPP error."
+msgstr ""
+
+#: ../../users.rst:106
+msgid "arch_prctl(ARCH_SET_GS)"
+msgstr ""
+
+#: ../../users.rst:107
+msgid "signalfd()"
+msgstr ""
+
+#: ../../users.rst:109
+msgid "signalfd4() returns a fd, but signal is not notified through the fd."
+msgstr ""
+
+#: ../../users.rst:111
+msgid "set_rlimit sets the limit values but they are not enforced."
+msgstr ""
+
+#: ../../users.rst:113
+msgid "Address randomization is not supported."
+msgstr ""
+
+#: ../../users.rst:115
+msgid ""
+"brk() extends the heap more than requestd when -h (–extend-heap-by=) "
+"option of mcexec is used with the value larger than 4 KiB. "
+"syscall_pwrite02 of LTP would fail for this reason. This is because the "
+"test expects that the end of the heap is set to the same address as the "
+"argument of sbrk() and expects a segmentation violation occurs when it "
+"tries to access the memory area right next to the boundary. However, the "
+"optimization sets the end to a value larger than the requested. "
+"Therefore, the expected segmentation violation doesn’t occur."
+msgstr ""
+
+#: ../../users.rst:125
+msgid ""
+"setpriority()/getpriority() won’t work. They might set/get the priority "
+"of a random mcexec thread. This is because there’s no fixed "
+"correspondence between a McKernel thread which issues the system call and"
+" a mcexec thread which handles the offload request."
+msgstr ""
+
+#: ../../users.rst:130
+msgid ""
+"mbind() can set the policy but it is not used when allocating physical "
+"pages."
+msgstr ""
+
+#: ../../users.rst:133
+msgid ""
+"MPOL_F_RELATIVE_NODES and MPOL_INTERLEAVE flags for "
+"set_mempolicy()/mbind() are not supported."
+msgstr ""
+
+#: ../../users.rst:136
+msgid ""
+"The MPOL_BIND policy for set_mempolicy()/mbind() works as the same as the"
+" MPOL_PREFERRED policy. That is, the physical page allocator doesn’t give"
+" up the allocation when the specified nodes are running out of pages but "
+"continues to search pages in the other nodes."
+msgstr ""
+
+#: ../../users.rst:141
+msgid ""
+"Kernel dump on Linux panic requires Linux kernel CentOS-7.4 and later. In"
+" addition, crash_kexec_post_notifiers kernel argument must be given to "
+"Linux kernel."
+msgstr ""
+
+#: ../../users.rst:145
+msgid ""
+"setfsuid()/setfsgid() cannot change the id of the calling thread. "
+"Instead, it changes that of the mcexec worker thread which takes the "
+"system-call offload request."
+msgstr ""
+
+#: ../../users.rst:149
+msgid ""
+"mmap (hugeTLBfs): The physical pages corresponding to a map are released "
+"when no McKernel process exist. The next map gets fresh physical pages."
+msgstr ""
+
+#: ../../users.rst:153
+msgid "Sticky bit on executable file has no effect."
+msgstr ""
+
+#: ../../users.rst:155
+msgid ""
+"Linux (RHEL-7 for x86_64) could hang when offlining CPUs in the process "
+"of booting McKernel due to the Linux bug, found in Linux-3.10 and fixed "
+"in the later version. One way to circumvent this is to always assign the "
+"same CPU set to McKernel."
+msgstr ""
+
+#: ../../users.rst:160
+msgid "madvise:"
+msgstr ""
+
+#: ../../users.rst:162
+msgid "MADV_HWPOISON and MADV_SOFT_OFFLINE always returns -EPERM."
+msgstr ""
+
+#: ../../users.rst:163
+msgid "MADV_MERGEABLE and MADV_UNMERGEABLE always returns -EINVAL."
+msgstr ""
+
+#: ../../users.rst:164
+msgid ""
+"MADV_HUGEPAGE and MADV_NOHUGEPAGE on file map returns -EINVAL except on "
+"RHEL-8 for aarch64."
+msgstr ""
+
+#: ../../users.rst:167
+msgid ""
+"brk() and mmap() doesn’t report out-of-memory through its return value. "
+"Instead, page-fault reports the error."
+msgstr ""
+
+#: ../../users.rst:170
+msgid ""
+"Anonymous mmap pre-maps requested number of pages when contiguous pages "
+"are available. Demand paging is used when not available."
+msgstr ""
+
+#: ../../users.rst:173
+msgid ""
+"Mixing page sizes in anonymous shared mapping is not allowed. mmap "
+"creates vm_range with one page size. And munmap or mremap that needs the "
+"reduced page size changes the sizes of all the pages of the vm_range."
+msgstr ""
+
+#: ../../users.rst:178
+msgid ""
+"ihk_os_getperfevent() could time-out when invoked from Fujitsu TCS (job-"
+"scheduler)."
+msgstr ""
+
+#: ../../users.rst:181
+msgid ""
+"The behaviors of madvise and mbind are changed to do nothing and report "
+"success as a workaround for Fugaku."
+msgstr ""
+
+#: ../../users.rst:184
+msgid ""
+"mmap() allows unlimited overcommit. Note that it corresponds to setting "
+"sysctl ``vm.overcommit_memory`` to 1."
+msgstr ""
+
+#~ msgid ""
+#~ "At the heart of the stack is "
+#~ "a low-level software infrastructure "
+#~ "called Interface for Heterogeneous Kernels "
+#~ "(IHK). IHK is a general framework "
+#~ "that provides capabilities for partitioning"
+#~ " resources in a many-core environment"
+#~ " (e.g.,CPU cores and physical memory) "
+#~ "and it enables management of lightweight"
+#~ " kernels. IHK can allocate and "
+#~ "release host resources dynamically and "
+#~ "no reboot of the host machine is"
+#~ " required when altering configuration. IHK"
+#~ " also provides a low-level inter-"
+#~ "kernel messaging infrastructure, called the"
+#~ " Inter-Kernel Communication (IKC) layer."
+#~ " An architectural overview of the "
+#~ "main system components is shown below."
+#~ msgstr ""
+
+#~ msgid ""
+#~ "McKernel is a lightweight kernel written"
+#~ " from scratch. It is designed for "
+#~ "HPC and is booted from IHK. "
+#~ "McKernel retains a binary compatible ABI"
+#~ " with Linux, however, it implements "
+#~ "only a small set of performance "
+#~ "sensitive system calls and the rest "
+#~ "are offloaded to Linux. Specifically, "
+#~ "McKernel has its own memory management,"
+#~ " it supports processes and multi-"
+#~ "threading with a simple round-robin "
+#~ "cooperative (tick-less) scheduler, and "
+#~ "it implements signaling. It also allows"
+#~ " inter-process memory mappings and it"
+#~ " provides interfaces to hardware "
+#~ "performance counters."
+#~ msgstr ""
+
+#~ msgid "Functionality"
+#~ msgstr ""
+
+#~ msgid ""
+#~ "An overview of some of the "
+#~ "principal functionalities of the IHK/McKernel"
+#~ " stack is provided below."
+#~ msgstr ""
+
+#~ msgid "System Call Offloading"
+#~ msgstr ""
+
+#~ msgid ""
+#~ "System call forwarding in McKernel is"
+#~ " implemented as follows. When an "
+#~ "offloaded system call occurs, McKernel "
+#~ "marshals the system call number along"
+#~ " with its arguments and sends a "
+#~ "message to Linux via a dedicated "
+#~ "IKC channel. The corresponding proxy "
+#~ "process running on Linux is by "
+#~ "default waiting for system call requests"
+#~ " through an ioctl() call into IHK’s"
+#~ " system call delegator kernel module. "
+#~ "The delegator kernel module’s IKC "
+#~ "interrupt handler wakes up the proxy "
+#~ "process, which returns to userspace and"
+#~ " simply invokes the requested system "
+#~ "call. Once it obtains the return "
+#~ "value, it instructs the delegator module"
+#~ " to send the result back to "
+#~ "McKernel, which subsequently passes the "
+#~ "value to user-space."
+#~ msgstr ""
+
+#~ msgid "Unified Address Space"
+#~ msgstr ""
+
+#~ msgid ""
+#~ "The unified address space model in "
+#~ "IHK/McKernel ensures that offloaded system "
+#~ "calls can seamlessly resolve arguments "
+#~ "even in case of pointers. This "
+#~ "mechanism is depicted below and is "
+#~ "implemented as follows."
+#~ msgstr ""
+
+#~ msgid ""
+#~ "First, the proxy process is compiled "
+#~ "as a position independent binary, which"
+#~ " enables us to map the code and"
+#~ " data segments specific to the proxy"
+#~ " process to an address range which"
+#~ " is explicitly excluded from McKernel’s "
+#~ "user space. The grey box on the"
+#~ " right side of the figure "
+#~ "demonstrates the excluded region. Second, "
+#~ "the entire valid virtual address range"
+#~ " of McKernel’s application user-space "
+#~ "is covered by a special mapping in"
+#~ " the proxy process for which we "
+#~ "use a pseudo file mapping in "
+#~ "Linux. This mapping is indicated by "
+#~ "the blue box on the left side "
+#~ "of the figure."
+#~ msgstr ""
+

docs/locales/ja/LC_MESSAGES/uti.po

@@ -0,0 +1,94 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) 2020, Masamichi Takagi, Balazs Gerofi, Yutaka Ishikawa
+# This file is distributed under the same license as the IHK/McKernel
+# package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2020.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: IHK/McKernel \n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2020-08-04 16:40+0900\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=utf-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: Babel 2.7.0\n"
+
+#: ../../uti.rst:2
+msgid "Advanced: Enable Utility Thread offloading Interface (UTI)"
+msgstr ""
+
+#: ../../uti.rst:4
+msgid ""
+"UTI enables a runtime such as MPI runtime to spawn utility threads such "
+"as MPI asynchronous progress threads to Linux cores."
+msgstr ""
+
+#: ../../uti.rst:8
+msgid "Install capstone"
+msgstr ""
+
+#: ../../uti.rst:11 ../../uti.rst:22
+msgid "When compute nodes don't have access to repositories"
+msgstr ""
+
+#: ../../uti.rst:13
+msgid "Install EPEL capstone-devel:"
+msgstr ""
+
+#: ../../uti.rst:24
+msgid ""
+"Ask the system administrator to install ``capstone-devel``. Note that it "
+"is in the EPEL repository."
+msgstr ""
+
+#: ../../uti.rst:28
+msgid "Install syscall_intercept"
+msgstr ""
+
+#: ../../uti.rst:37
+msgid "Install UTI for McKernel"
+msgstr ""
+
+#: ../../uti.rst:39
+msgid "Install:"
+msgstr ""
+
+#: ../../uti.rst:49
+msgid "Install McKernel"
+msgstr ""
+
+#: ../../uti.rst:51
+msgid "Add ``-DENABLE_UTI=ON`` option to ``cmake``:"
+msgstr ""
+
+#: ../../uti.rst:58
+msgid "Run programs"
+msgstr ""
+
+#: ../../uti.rst:60
+msgid "Add ``--enable-uti`` option to ``mcexec``:"
+msgstr ""
+
+#: ../../uti.rst:67
+msgid "Install UTI for Linux"
+msgstr ""
+
+#: ../../uti.rst:69
+msgid ""
+"You should skip this step if it's already installed as with, for example,"
+" Fujitsu Technical Computing Suite."
+msgstr ""
+
+#: ../../uti.rst:72
+msgid "Install by make"
+msgstr ""
+
+#: ../../uti.rst:82
+msgid "Install by rpm"
+msgstr ""
+

docs/logo.rst

@@ -0,0 +1 @@
+.. figure:: mckernel-logo.png

docs/mckernel_developers.rst

@@ -0,0 +1,9 @@
+.. sectnum::
+   :suffix: .
+   :depth: 3
+
+Interfaces
+==========
+
+Interface details
+=================

docs/quick.rst

@@ -0,0 +1,22 @@
+.. sectnum::
+   :suffix: .
+   :depth: 3
+
+Introduction
+============
+
+.. include:: summary.rst
+
+.. include:: background.rst
+
+.. include:: archtecture.rst
+
+.. include:: install.rst
+
+.. include:: boot_run_shutdown.rst
+
+.. include:: team.rst
+
+.. include:: license.rst
+
+.. include:: contact.rst

docs/requirements.txt

@@ -0,0 +1,30 @@
+alabaster==0.7.12
+attrs==19.3.0
+Babel==2.7.0
+certifi==2019.11.28
+chardet==3.0.4
+commonmark==0.9.1
+docutils==0.15
+idna==2.8
+imagesize==1.1.0
+Jinja2==2.10.3
+MarkupSafe==1.1.1
+mistune==0.8.4
+packaging==19.2
+Pygments==2.5.2
+pyparsing==2.4.5
+pytz==2019.3
+recommonmark==0.6.0
+requests==2.22.0
+roman==3.2
+six==1.13.0
+snowballstemmer==2.0.0
+Sphinx==2.2.2
+sphinx-rtd-theme==0.5.0
+sphinxcontrib-applehelp==1.0.1
+sphinxcontrib-devhelp==1.0.1
+sphinxcontrib-htmlhelp==1.0.2
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.2
+sphinxcontrib-serializinghtml==1.1.3
+urllib3==1.25.7

docs/spec-ihk.md

@@ -0,0 +1,3 @@
+## hi
+
+:download:`IHK Spec <ihk.pdf>`

docs/spec-mckernel.md

@@ -0,0 +1 @@
+## Hello

docs/spec/ihk.rst

@@ -0,0 +1,18 @@
+.. sectnum::
+   :suffix: .
+   :depth: 3
+
+External Specs
+==============
+
+Overview
+--------
+
+Function Specs
+--------------
+
+Command / Daemon Specs
+----------------------
+
+Booting LWK
+===========

docs/spec/mckernel.rst

@@ -0,0 +1,9 @@
+.. sectnum::
+   :suffix: .
+   :depth: 3
+
+Interfaces
+==========
+
+Interface details
+=================

docs/summary.rst

@@ -0,0 +1,15 @@
+IHK/McKernel is a light-weight multi-kernel operating system designed
+for high-end supercomputing. It runs Linux and McKernel, a light-weight
+kernel (LWK), side-by-side inside compute nodes and aims at the
+following:
+
+-  Provide scalable and consistent execution of large-scale parallel
+   scientific applications, but at the same time maintain the ability to
+   rapidly adapt to new hardware features and emerging programming
+   models
+-  Provide efficient memory and device management so that resource
+   contention and data movement are minimized at the system level
+-  Eliminate OS noise by isolating OS services in Linux and provide
+   jitter free execution on the LWK
+-  Support the full POSIX/Linux APIs by selectively offloading
+   (slow-path) system calls to Linux

docs/team.rst

@@ -0,0 +1,12 @@
+The Team
+========
+
+The McKernel project was started at The University of Tokyo and
+currently it is mainly developed at RIKEN. Some of our collaborators
+include:
+
+-  Hitachi
+-  Fujitsu
+-  CEA (France)
+-  NEC
+

docs/users.rst

@@ -0,0 +1,185 @@
+.. sectnum::
+   :suffix: .
+   :depth: 3
+
+Architectural Overview
+======================
+
+See `Quick Guide -- Architectural Overview <quick.html#architectural-overview>`__.
+
+Running Programs
+================
+
+You need to check if your application and pre-/post-processing programs are suited to run with McKernel.
+Follow the guide below to choose to run the whole on McKernel, or run the whole on Linux, or run pre-/post-processing on Linux and the application on McKernel:
+
+* Application
+
+  - Run the whole on Linux if it issues system calls frequently and becoming the bottleneck with McKernel, e.g., those performing many file I/O operations.
+  - Otherwise, run it on McKernel.
+
+* Pre-/Post-processing
+
+  - Run it on McKernel if it consumes a large amount of memory or the execution time isn't prolonged prohivitively with McKernel. The reason for the first condition is that the resource could be limited for Linux CPUs in the nodes for McKernel.
+  - Otherwise, run it on Linux.
+
+
+Modify job script
+-----------------
+
+When using job submission system, you need to modify the job scripts so that the job script itself is going to run on Linux.
+For example, with Fujitsu Technical Computing Suite (TCS), you need to specify ``jobenv=mck1`` by inserting the following line into the job script:
+
+.. code-block:: none
+
+   #PJM -L jobenv=mck1
+
+Insert ``mcexec`` into the command line
+---------------------------------------
+
+You need to insert ``mcexec`` into the command lines invoking the programs that you chose to run on McKernel:
+
+Non-MPI programs
+~~~~~~~~~~~~~~~~
+
+Insert ``mcexec`` before an executable:
+
+::
+
+   mcexec ./a.out
+
+MPI programs
+~~~~~~~~~~~~
+
+Insert ``mcexec -n <processes-per-node>`` **after mpirun** and before an
+executable:
+
+::
+
+   mpirun -n <number-of-MPI-processes> mcexec -n <processes-per-node> ./a.out
+
+``<processes-per-node>`` is the number of the processes per node and
+calculated by (number of MPI processes) / (number of nodes).
+
+For example, ``<processes-per-node>`` equals to 4 (=32/8) when
+specifying the number of processes and nodes as follows with
+Fujitsu Technical Computing Suite.
+
+.. code-block:: none
+
+   #PJM --mpi "proc=32"
+   #PJM -L "node=8"
+
+Limitations
+===========
+
+1.  Pseudo devices such as /dev/mem and /dev/zero are not mmap()ed
+    correctly even if the mmap() returns a success. An access of their
+    mapping receives the SIGSEGV signal.
+
+2.  clone() supports only the following flags. All the other flags cause
+    clone() to return error or are simply ignored.
+
+    -  CLONE_CHILD_CLEARTID
+    -  CLONE_CHILD_SETTID
+    -  CLONE_PARENT_SETTID
+    -  CLONE_SETTLS
+    -  CLONE_SIGHAND
+    -  CLONE_VM
+
+3.  PAPI has the following restriction.
+
+    -  Number of counters a user can use at the same time is up to the
+       number of the physical counters in the processor.
+
+4.  msync writes back only the modified pages mapped by the calling
+    process.
+
+5.  The following syscalls always return the ENOSYS error.
+
+    -  migrate_pages()
+    -  move_pages()
+    -  set_robust_list()
+
+6.  The following syscalls always return the EOPNOTSUPP error.
+
+    -  arch_prctl(ARCH_SET_GS)
+    -  signalfd()
+
+7.  signalfd4() returns a fd, but signal is not notified through the fd.
+
+8.  set_rlimit sets the limit values but they are not enforced.
+
+9.  Address randomization is not supported.
+
+10. brk() extends the heap more than requestd when -h (–extend-heap-by=)
+    option of mcexec is used with the value larger than 4 KiB.
+    syscall_pwrite02 of LTP would fail for this reason. This is because
+    the test expects that the end of the heap is set to the same address
+    as the argument of sbrk() and expects a segmentation violation
+    occurs when it tries to access the memory area right next to the
+    boundary. However, the optimization sets the end to a value larger
+    than the requested. Therefore, the expected segmentation violation
+    doesn’t occur.
+
+11. setpriority()/getpriority() won’t work. They might set/get the
+    priority of a random mcexec thread. This is because there’s no fixed
+    correspondence between a McKernel thread which issues the system
+    call and a mcexec thread which handles the offload request.
+
+12. mbind() can set the policy but it is not used when allocating
+    physical pages.
+
+13. MPOL_F_RELATIVE_NODES and MPOL_INTERLEAVE flags for
+    set_mempolicy()/mbind() are not supported.
+
+14. The MPOL_BIND policy for set_mempolicy()/mbind() works as the same
+    as the MPOL_PREFERRED policy. That is, the physical page allocator
+    doesn’t give up the allocation when the specified nodes are running
+    out of pages but continues to search pages in the other nodes.
+
+15. Kernel dump on Linux panic requires Linux kernel CentOS-7.4 and
+    later. In addition, crash_kexec_post_notifiers kernel argument must
+    be given to Linux kernel.
+
+16. setfsuid()/setfsgid() cannot change the id of the calling thread.
+    Instead, it changes that of the mcexec worker thread which takes the
+    system-call offload request.
+
+17. mmap (hugeTLBfs): The physical pages corresponding to a map are
+    released when no McKernel process exist. The next map gets fresh
+    physical pages.
+
+18. Sticky bit on executable file has no effect.
+
+19. Linux (RHEL-7 for x86_64) could hang when offlining CPUs in the
+    process of booting McKernel due to the Linux bug, found in
+    Linux-3.10 and fixed in the later version. One way to circumvent
+    this is to always assign the same CPU set to McKernel.
+
+20. madvise:
+
+    -  MADV_HWPOISON and MADV_SOFT_OFFLINE always returns -EPERM.
+    -  MADV_MERGEABLE and MADV_UNMERGEABLE always returns -EINVAL.
+    -  MADV_HUGEPAGE and MADV_NOHUGEPAGE on file map returns -EINVAL
+       except on RHEL-8 for aarch64.
+
+21. brk() and mmap() doesn’t report out-of-memory through its return
+    value. Instead, page-fault reports the error.
+
+22. Anonymous mmap pre-maps requested number of pages when contiguous
+    pages are available. Demand paging is used when not available.
+
+23. Mixing page sizes in anonymous shared mapping is not allowed. mmap
+    creates vm_range with one page size. And munmap or mremap that needs
+    the reduced page size changes the sizes of all the pages of the
+    vm_range.
+
+24. ihk_os_getperfevent() could time-out when invoked from Fujitsu TCS
+    (job-scheduler).
+
+25. The behaviors of madvise and mbind are changed to do nothing and
+    report success as a workaround for Fugaku.
+
+26. mmap() allows unlimited overcommit. Note that it corresponds to
+    setting sysctl ``vm.overcommit_memory`` to 1.

docs/uti.rst

@@ -0,0 +1,91 @@
+Advanced: Enable Utility Thread offloading Interface (UTI)
+-------------------------------------------------------------
+
+UTI enables a runtime such as MPI runtime to spawn utility threads such
+as MPI asynchronous progress threads to Linux cores.
+
+Install capstone
+~~~~~~~~~~~~~~~~~~~~
+
+When compute nodes don't have access to repositories
+""""""""""""""""""""""""""""""""""""""""""""""""""""
+
+Install EPEL capstone-devel:
+
+::
+
+   sudo yum install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
+   sudo yum install capstone-devel
+
+
+When compute nodes don't have access to repositories
+""""""""""""""""""""""""""""""""""""""""""""""""""""
+
+Ask the system administrator to install ``capstone-devel``. Note that it is in the EPEL repository.
+
+
+Install syscall_intercept
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+   git clone https://github.com/RIKEN-SysSoft/syscall_intercept.git
+   mkdir build && cd build
+   cmake <syscall_intercept>/arch/aarch64 -DCMAKE_INSTALL_PREFIX=<syscall-intercept-install> -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=gcc -DTREAT_WARNINGS_AS_ERRORS=OFF
+
+Install UTI for McKernel
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Install:
+
+.. code-block:: none
+
+   git clone https://github.com/RIKEN-SysSoft/uti.git
+   mkdir build && cd build
+   ../uti/configure --prefix=<mckernel-install> --with-rm=mckernel
+   make && make install
+
+Install McKernel
+~~~~~~~~~~~~~~~~~~~~
+
+Add ``-DENABLE_UTI=ON`` option to ``cmake``:
+
+::
+
+   CMAKE_PREFIX_PATH=<syscall-intercept-install> cmake -DCMAKE_INSTALL_PREFIX=${HOME}/ihk+mckernel -DENABLE_UTI=ON $HOME/src/ihk+mckernel/mckernel
+
+Run programs
+~~~~~~~~~~~~~~~~
+
+Add ``--enable-uti`` option to ``mcexec``:
+
+::
+
+   mcexec --enable-uti <command>
+
+Install UTI for Linux
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You should skip this step if it's already installed as with, for example, Fujitsu Technical Computing Suite.
+
+Install by make
+"""""""""""""""
+
+.. code-block:: none
+
+   git clone https://github.com/RIKEN-SysSoft/uti.git
+   mkdir build && cd build
+   ../uti/configure --prefix=<uti-install> --with-rm=linux
+   make && make install
+
+Install by rpm
+""""""""""""""
+
+.. code-block:: none
+
+   git clone https://github.com/RIKEN-SysSoft/uti.git
+   mkdir build && cd build
+   ../uti/configure --prefix=<uti-install> --with-rm=linux
+   rm -f ~/rpmbuild/SOURCES/<version>.tar.gz
+   rpmbuild -ba ./scripts/uti.spec
+   rpm -Uvh uti-<version>-<release>-<arch>.rpm

executer/include/uprotocol.h

@@ -91,8 +91,9 @@ struct program_image_section {
 
 struct get_cpu_set_arg {
 	int nr_processes;
+	char *req_cpu_list;   // Requested by user-space
+	int req_cpu_list_len; // Lenght of request string
 	int *process_rank;
-	pid_t ppid;
 	void *cpu_set;
 	size_t cpu_set_size;	// Size in bytes
 	int *target_core;

executer/kernel/mcctrl/control.c

@@ -587,14 +587,13 @@ extern int mckernel_cpu_2_linux_cpu(struct mcctrl_usrdata *udp, int cpu_id);
 static long mcexec_get_cpuset(ihk_os_t os, unsigned long arg)
 {
 	struct mcctrl_usrdata *udp = ihk_host_os_get_usrdata(os);
-	struct mcctrl_part_exec *pe = NULL, *pe_itr;
+	struct mcctrl_part_exec *pe;
 	struct get_cpu_set_arg req;
 	struct mcctrl_cpu_topology *cpu_top, *cpu_top_i;
 	struct cache_topology *cache_top;
 	int cpu, cpus_assigned, cpus_to_assign, cpu_prev;
 	int ret = 0;
 	int mcexec_linux_numa;
-	int pe_list_len = 0;
 	cpumask_t *mcexec_cpu_set = NULL;
 	cpumask_t *cpus_used = NULL;
 	cpumask_t *cpus_to_use = NULL;
@@ -615,54 +614,96 @@ static long mcexec_get_cpuset(ihk_os_t os, unsigned long arg)
 	}
 
 	if (copy_from_user(&req, (void *)arg, sizeof(req))) {
-		pr_err("%s: error copying user request\n", __func__);
+		printk("%s: error copying user request\n", __FUNCTION__);
 		ret = -EINVAL;
 		goto put_out;
 	}
 
-	mutex_lock(&udp->part_exec_lock);
-	/* Find part_exec having same node_proxy */
-	list_for_each_entry_reverse(pe_itr, &udp->part_exec_list, chain) {
-		pe_list_len++;
-		if (pe_itr->node_proxy_pid == req.ppid) {
-			pe = pe_itr;
-			break;
-		}
-	}
+	/* User requested CPU mask? */
+	if (req.req_cpu_list && req.req_cpu_list_len) {
+		char *cpu_list = NULL;
 
-	if (!pe) {
-		/* First process to enter CPU partitioning */
-		pr_debug("%s: pe_list_len:%d\n", __func__, pe_list_len);
-		if (pe_list_len >= PE_LIST_MAXLEN) {
-			/* delete head entry of pe_list */
-			pe_itr = list_first_entry(&udp->part_exec_list,
-					struct mcctrl_part_exec, chain);
-			list_del(&pe_itr->chain);
-			kfree(pe_itr);
-		}
-
-		pe = kzalloc(sizeof(struct mcctrl_part_exec), GFP_KERNEL);
-		if (!pe) {
-			mutex_unlock(&udp->part_exec_lock);
+		cpu_list = kmalloc(req.req_cpu_list_len, GFP_KERNEL);
+		if (!cpu_list) {
+			printk("%s: error: allocating CPU list\n", __FUNCTION__);
 			ret = -ENOMEM;
 			goto put_out;
 		}
-		/* Init part_exec */
-		mutex_init(&pe->lock);
-		INIT_LIST_HEAD(&pe->pli_list);
-		pe->nr_processes = req.nr_processes;
-		pe->nr_processes_left = req.nr_processes;
-		pe->nr_processes_joined = 0;
-		pe->node_proxy_pid = req.ppid;
 
-		list_add_tail(&pe->chain, &udp->part_exec_list);
-		dprintk("%s: nr_processes: %d (partitioned exec starts)\n",
-				__func__, pe->nr_processes);
+		if (copy_from_user(cpu_list,
+					req.req_cpu_list, req.req_cpu_list_len)) {
+			printk("%s: error copying CPU list request\n", __FUNCTION__);
+			kfree(cpu_list);
+			ret = -EINVAL;
+			goto put_out;
+		}
+
+		cpus_used = kmalloc(sizeof(cpumask_t), GFP_KERNEL);
+		cpus_to_use = kmalloc(sizeof(cpumask_t), GFP_KERNEL);
+		if (!cpus_to_use || !cpus_used) {
+			printk("%s: error: allocating CPU mask\n", __FUNCTION__);
+			ret = -ENOMEM;
+			kfree(cpu_list);
+			goto put_out;
+		}
+		memset(cpus_used, 0, sizeof(cpumask_t));
+		memset(cpus_to_use, 0, sizeof(cpumask_t));
+
+		/* Parse CPU list */
+		if (cpulist_parse(cpu_list, cpus_to_use) < 0) {
+			printk("%s: invalid CPUs requested: %s\n",
+				__FUNCTION__, cpu_list);
+			ret = -EINVAL;
+			kfree(cpu_list);
+			goto put_out;
+		}
+
+		memcpy(cpus_used, cpus_to_use, sizeof(cpumask_t));
+
+		/* Copy mask to user-space */
+		if (copy_to_user(req.cpu_set, cpus_used,
+					(req.cpu_set_size < sizeof(cpumask_t) ?
+					 req.cpu_set_size : sizeof(cpumask_t)))) {
+			printk("%s: error copying mask to user\n", __FUNCTION__);
+			ret = -EINVAL;
+			kfree(cpu_list);
+			goto put_out;
+		}
+
+		/* Copy IKC target core */
+		cpu = cpumask_next(-1, cpus_used);
+		if (copy_to_user(req.target_core, &cpu, sizeof(cpu))) {
+			printk("%s: error copying target core to user\n",
+					__FUNCTION__);
+			ret = -EINVAL;
+			kfree(cpu_list);
+			goto put_out;
+		}
+
+		/* Save in per-process structure */
+		memcpy(&ppd->cpu_set, cpus_used, sizeof(cpumask_t));
+		ppd->ikc_target_cpu = cpu;
+		printk("%s: %s -> target McKernel CPU: %d\n",
+			__func__, cpu_list, cpu);
+
+		ret = 0;
+		kfree(cpu_list);
+		goto put_out;
 	}
-	mutex_unlock(&udp->part_exec_lock);
+
+	pe = &udp->part_exec;
 
 	mutex_lock(&pe->lock);
 
+	/* First process to enter CPU partitioning */
+	if (pe->nr_processes == -1) {
+		pe->nr_processes = req.nr_processes;
+		pe->nr_processes_left = req.nr_processes;
+		dprintk("%s: nr_processes: %d (partitioned exec starts)\n",
+				__FUNCTION__,
+				pe->nr_processes);
+	}
+
 	if (pe->nr_processes != req.nr_processes) {
 		printk("%s: error: requested number of processes"
 				" doesn't match current partitioned execution\n",
@@ -671,15 +712,7 @@ static long mcexec_get_cpuset(ihk_os_t os, unsigned long arg)
 		goto put_and_unlock_out;
 	}
 
-	if (pe->nr_processes_joined >= pe->nr_processes) {
-		printk("%s: too many processes have joined to the group of %d\n",
-				__func__, req.ppid);
-		ret = -EINVAL;
-		goto put_and_unlock_out;
-	}
-
 	--pe->nr_processes_left;
-	++pe->nr_processes_joined;
 	dprintk("%s: nr_processes: %d, nr_processes_left: %d\n",
 			__FUNCTION__,
 			pe->nr_processes,
@@ -765,6 +798,8 @@ static long mcexec_get_cpuset(ihk_os_t os, unsigned long arg)
 				wake_up_interruptible(&pli_next->pli_wq);
 			}
 
+			/* Reset process counter to start state */
+			pe->nr_processes = -1;
 			ret = -ETIMEDOUT;
 			goto put_and_unlock_out;
 		}
@@ -1012,8 +1047,16 @@ next_cpu:
 	/* Commit used cores to OS structure */
 	memcpy(&pe->cpus_used, cpus_used, sizeof(*cpus_used));
 
-	/* If not last process, wake up next process in list */
-	if (pe->nr_processes_left != 0) {
+	/* Reset if last process */
+	if (pe->nr_processes_left == 0) {
+		dprintk("%s: nr_processes: %d (partitioned exec ends)\n",
+				__FUNCTION__,
+				pe->nr_processes);
+		pe->nr_processes = -1;
+		memset(&pe->cpus_used, 0, sizeof(pe->cpus_used));
+	}
+	/* Otherwise wake up next process in list */
+	else {
 		++pe->process_rank;
 		pli_next = list_first_entry(&pe->pli_list,
 			struct process_list_item, list);
@@ -1237,7 +1280,7 @@ int mcexec_syscall(struct mcctrl_usrdata *ud, struct ikc_scd_packet *packet)
 	ppd = mcctrl_get_per_proc_data(ud, pid);
 
 	if (unlikely(!ppd)) {
-		kprintf("%s: ERROR: no per-process structure for PID %d, "
+		dprintk("%s: ERROR: no per-process structure for PID %d, "
 				"syscall nr: %lu\n",
 				__FUNCTION__, pid, packet->req.number);
 
@@ -1452,7 +1495,7 @@ retry_alloc:
 			__FUNCTION__, task_pid_vnr(current), packet->ref);
 
 	mb();
-	if (!packet->req.valid) {
+	if (!smp_load_acquire(&packet->req.valid)) {
 		printk("%s: ERROR: stray wakeup pid: %d, tid: %d: SC %lu\n",
 				__FUNCTION__,
 				task_tgid_vnr(current),
@@ -1462,7 +1505,7 @@ retry_alloc:
 		goto retry;
 	}
 
-	packet->req.valid = 0; /* ack */
+	smp_store_release(&packet->req.valid,  0); /* ack */
 	dprintk("%s: system call: %d, args[0]: %lu, args[1]: %lu, args[2]: %lu, "
 			"args[3]: %lu, args[4]: %lu, args[5]: %lu\n",
 			__FUNCTION__,
@@ -3399,14 +3442,6 @@ long __mcctrl_control(ihk_os_t os, unsigned int req, unsigned long arg,
 	return -EINVAL;
 }
 
-/* Per-CPU register manipulation functions */
-struct mcctrl_os_cpu_response {
-	int done;
-	unsigned long val;
-	int err;
-	wait_queue_head_t wq;
-};
-
 int mcctrl_get_request_os_cpu(ihk_os_t os, int *ret_cpu)
 {
 	struct mcctrl_usrdata *usrdata;
@@ -3470,31 +3505,14 @@ out_put_ppd:
 	return ret;
 }
 
-void mcctrl_os_read_write_cpu_response(ihk_os_t os,
-		struct ikc_scd_packet *pisp)
-{
-	struct mcctrl_os_cpu_response *resp;
-
-	/* XXX: What if caller thread is unblocked by a signal
-	 * before this message arrives? */
-	resp = pisp->resp;
-	if (!resp) {
-		return;
-	}
-
-	resp->val = pisp->desc.val;
-	resp->done = 1;
-	resp->err = pisp->err;
-	wake_up_interruptible(&resp->wq);
-}
-
 int __mcctrl_os_read_write_cpu_register(ihk_os_t os, int cpu,
 		struct ihk_os_cpu_register *desc,
 		enum mcctrl_os_cpu_operation op)
 {
 	struct mcctrl_usrdata *udp = ihk_host_os_get_usrdata(os);
 	struct ikc_scd_packet isp;
-	struct mcctrl_os_cpu_response resp;
+	struct ihk_os_cpu_register *ldesc = NULL;
+	int do_free = 0;
 	int ret = -EINVAL;
 
 	if (!udp) {
@@ -3511,50 +3529,43 @@ int __mcctrl_os_read_write_cpu_register(ihk_os_t os, int cpu,
 
 	}
 
+	/* Keep a dynamic structure around that can
+	 * survive an early return due to a signal */
+	ldesc = kmalloc(sizeof(*ldesc), GFP_KERNEL);
+	if (!ldesc) {
+		printk("%s: ERROR: allocating cpu register desc\n", __FUNCTION__);
+		return -ENOMEM;
+	}
+	*ldesc = *desc;
+
 	memset(&isp, '\0', sizeof(struct ikc_scd_packet));
 	isp.msg = SCD_MSG_CPU_RW_REG;
 	isp.op = op;
-	isp.desc = *desc;
-	isp.resp = &resp;
+	isp.pdesc = virt_to_phys(ldesc);
 
-	resp.done = 0;
-	resp.err = 0;
-	init_waitqueue_head(&resp.wq);
-
-	mb();
-	ret = mcctrl_ikc_send(os, cpu, &isp);
-	if (ret < 0) {
+	ret = mcctrl_ikc_send_wait(os, cpu, &isp, 0, NULL, &do_free, 1, ldesc);
+	if (ret != 0) {
 		printk("%s: ERROR sending IKC msg: %d\n", __FUNCTION__, ret);
 		goto out;
 	}
 
-	/* Wait for response */
-	ret = wait_event_interruptible(resp.wq, resp.done);
-	if (ret < 0) {
-		printk("%s: ERROR after wait: %d\n", __FUNCTION__, ret);
-		goto out;
-	}
-
-	ret = resp.err;
-	if (ret != 0) {
-		printk("%s: ERROR receive: %d\n", __FUNCTION__, resp.err);
-		goto out;
-	}
-
 	/* Update if read */
-	if (ret == 0 && op == MCCTRL_OS_CPU_READ_REGISTER) {
-		desc->val = resp.val;
+	if (op == MCCTRL_OS_CPU_READ_REGISTER) {
+		desc->val = ldesc->val;
 	}
 
 	/* Notify caller (for future async implementation) */
 	atomic_set(&desc->sync, 1);
 
-	dprintk("%s: MCCTRL_OS_CPU_%s_REGISTER: reg: 0x%lx, val: 0x%lx\n",
+	dprintk("%s: MCCTRL_OS_CPU_%s_REGISTER: CPU: %d, addr_ext: 0x%lx, val: 0x%lx\n",
 		__FUNCTION__,
-		(op == MCCTRL_OS_CPU_READ_REGISTER ? "READ" : "WRITE"),
-		desc->addr, desc->val);
+		(op == MCCTRL_OS_CPU_READ_REGISTER ? "READ" : "WRITE"), cpu,
+		desc->addr_ext, desc->val);
 
 out:
+	if (do_free) {
+		kfree(ldesc);
+	}
 	return ret;
 }
 

executer/kernel/mcctrl/ikc.c

@@ -209,6 +209,7 @@ static int syscall_packet_handler(struct ihk_ikc_channel_desc *c,
 	case SCD_MSG_SEND_SIGNAL_ACK:
 	case SCD_MSG_PROCFS_ANSWER:
 	case SCD_MSG_REMOTE_PAGE_FAULT_ANSWER:
+	case SCD_MSG_CPU_RW_REG_RESP:
 		mcctrl_wakeup_cb(__os, pisp);
 		break;
 
@@ -239,10 +240,6 @@ static int syscall_packet_handler(struct ihk_ikc_channel_desc *c,
 		get_vdso_info(__os, pisp->arg);
 		break;
 
-	case SCD_MSG_CPU_RW_REG_RESP:
-		mcctrl_os_read_write_cpu_response(__os, pisp);
-		break;
-
 	case SCD_MSG_EVENTFD:
 		dkprintf("%s: SCD_MSG_EVENTFD,pisp->eventfd_type=%d\n", __FUNCTION__, pisp->eventfd_type);
 		mcctrl_eventfd(__os, pisp);
@@ -465,7 +462,7 @@ int prepare_ikc_channels(ihk_os_t os)
 	int i;
 	int ret = 0;
 
-	usrdata = kzalloc(sizeof(struct mcctrl_usrdata), GFP_KERNEL);
+	usrdata = kzalloc(sizeof(struct mcctrl_usrdata), GFP_ATOMIC);
 	if (!usrdata) {
 		printk("%s: error: allocating mcctrl_usrdata\n", __FUNCTION__);
 		ret = -ENOMEM;
@@ -491,7 +488,7 @@ int prepare_ikc_channels(ihk_os_t os)
 	usrdata->num_channels = usrdata->cpu_info->n_cpus;
 	usrdata->channels = kzalloc(sizeof(struct mcctrl_channel) *
 			usrdata->num_channels,
-			GFP_KERNEL);
+			GFP_ATOMIC);
 
 	if (!usrdata->channels) {
 		printk("Error: cannot allocate channels.\n");
@@ -500,7 +497,7 @@ int prepare_ikc_channels(ihk_os_t os)
 	}
 
 	usrdata->ikc2linux = kzalloc(sizeof(struct ihk_ikc_channel_desc *) *
-			nr_cpu_ids, GFP_KERNEL);
+			nr_cpu_ids, GFP_ATOMIC);
 
 	if (!usrdata->ikc2linux) {
 		printk("Error: cannot allocate ikc2linux channels.\n");
@@ -516,7 +513,6 @@ int prepare_ikc_channels(ihk_os_t os)
 
 	init_waitqueue_head(&usrdata->wq_procfs);
 	mutex_init(&usrdata->reserve_lock);
-	mutex_init(&usrdata->part_exec_lock);
 
 	for (i = 0; i < MCCTRL_PER_PROC_DATA_HASH_SIZE; ++i) {
 		INIT_LIST_HEAD(&usrdata->per_proc_data_hash[i]);
@@ -525,8 +521,10 @@ int prepare_ikc_channels(ihk_os_t os)
 
 	INIT_LIST_HEAD(&usrdata->cpu_topology_list);
 	INIT_LIST_HEAD(&usrdata->node_topology_list);
-	INIT_LIST_HEAD(&usrdata->part_exec_list);
 
+	mutex_init(&usrdata->part_exec.lock);
+	INIT_LIST_HEAD(&usrdata->part_exec.pli_list);
+	usrdata->part_exec.nr_processes = -1;
 	INIT_LIST_HEAD(&usrdata->wakeup_descs_list);
 	spin_lock_init(&usrdata->wakeup_descs_lock);
 
@@ -582,18 +580,6 @@ void destroy_ikc_channels(ihk_os_t os)
 
 	kfree(usrdata->channels);
 	kfree(usrdata->ikc2linux);
-
-	mutex_lock(&usrdata->part_exec_lock);
-	while (!list_empty(&usrdata->part_exec_list)) {
-		struct mcctrl_part_exec *pe;
-
-		pe = list_first_entry(&usrdata->part_exec_list,
-				struct mcctrl_part_exec, chain);
-		list_del(&pe->chain);
-		kfree(pe);
-	}
-	mutex_unlock(&usrdata->part_exec_lock);
-
 	kfree(usrdata);
 }
 

executer/kernel/mcctrl/mcctrl.h

@@ -154,7 +154,7 @@ struct ikc_scd_packet {
 
 		/* SCD_MSG_CPU_RW_REG */
 		struct {
-			struct ihk_os_cpu_register desc;
+			unsigned long pdesc; /* Physical addr of the descriptor */
 			enum mcctrl_os_cpu_operation op;
 			void *resp;
 		};
@@ -298,7 +298,6 @@ struct mcctrl_cpu_topology {
 	//struct mcctrl_usrdata *udp;
 	struct ihk_cpu_topology *saved;
 	int mckernel_cpu_id;
-	int mckernel_core_id;
 	cpumask_t core_siblings;
 	cpumask_t thread_siblings;
 
@@ -325,20 +324,13 @@ struct process_list_item {
 	wait_queue_head_t pli_wq;
 };
 
-#define PE_LIST_MAXLEN 5
-
 struct mcctrl_part_exec {
 	struct mutex lock;	
 	int nr_processes;
-	/* number of processes to let in / out the synchronization point */
 	int nr_processes_left;
-	/* number of processes which have joined the partition */
-	int nr_processes_joined;
 	int process_rank;
-	pid_t node_proxy_pid;
 	cpumask_t cpus_used;
 	struct list_head pli_list;
-	struct list_head chain;
 };
 
 #define CPU_LONGS (((NR_CPUS) + (BITS_PER_LONG) - 1) / (BITS_PER_LONG))
@@ -361,7 +353,6 @@ struct mcctrl_usrdata {
 	int	job_pos;
 	int	mcctrl_dma_abort;
 	struct mutex reserve_lock;
-	struct mutex part_exec_lock;
 	unsigned long	last_thread_exec;
 	wait_queue_head_t wq_procfs;
 	struct list_head per_proc_data_hash[MCCTRL_PER_PROC_DATA_HASH_SIZE];
@@ -377,7 +368,7 @@ struct mcctrl_usrdata {
 	nodemask_t numa_online;
 	struct list_head cpu_topology_list;
 	struct list_head node_topology_list;
-	struct list_head part_exec_list;
+	struct mcctrl_part_exec part_exec;
 	int perf_event_num;
 };
 

executer/kernel/mcctrl/procfs.c

@@ -1130,7 +1130,6 @@ static const struct procfs_entry base_entry_stuff[] = {
 //	PROC_REG("cpuinfo",    S_IRUGO, NULL),
 #endif /* POSTK_DEBUG_ARCH_DEP_42 */
 //	PROC_REG("meminfo",    S_IRUGO, NULL),
-	PROC_REG("meminfo",    S_IRUGO, &mckernel_buff_io),
 //	PROC_REG("pagetypeinfo",S_IRUGO, NULL),
 //	PROC_REG("softirq",    S_IRUGO, NULL),
 	PROC_REG("stat",       0444, &mckernel_buff_io),

executer/kernel/mcctrl/syscall.c

@@ -228,18 +228,19 @@ static int __notify_syscall_requester(ihk_os_t os, struct ikc_scd_packet *packet
 	c = (usrdata->channels + packet->ref)->c;
 
 	/* If spinning, no need for IKC message */
-	if (__sync_bool_compare_and_swap(&res->req_thread_status,
+	if (cmpxchg(&res->req_thread_status,
 				IHK_SCD_REQ_THREAD_SPINNING,
-				IHK_SCD_REQ_THREAD_TO_BE_WOKEN)) {
+				IHK_SCD_REQ_THREAD_TO_BE_WOKEN) ==
+			IHK_SCD_REQ_THREAD_SPINNING) {
 		dprintk("%s: no need to send IKC message for PID %d\n",
-			   __FUNCTION__, packet->pid);
+				__FUNCTION__, packet->pid);
 		return ret;
 	}
 
 	/* Wait until the status goes back to IHK_SCD_REQ_THREAD_SPINNING or
 	   IHK_SCD_REQ_THREAD_DESCHEDULED because two wake-up attempts are competing.
 	   Note that mcexec_terminate_thread() and returning EINTR would compete. */
-	if (res->req_thread_status == IHK_SCD_REQ_THREAD_TO_BE_WOKEN) {
+	if (smp_load_acquire(&res->req_thread_status) == IHK_SCD_REQ_THREAD_TO_BE_WOKEN) {
 		printk("%s: INFO: someone else is waking up the McKernel thread, "
 				"pid: %d, req status: %lu, syscall nr: %lu\n",
 				__FUNCTION__, packet->pid,
@@ -247,9 +248,10 @@ static int __notify_syscall_requester(ihk_os_t os, struct ikc_scd_packet *packet
 	}
 
 	/* The thread is not spinning any more, make sure it's descheduled */
-	if (!__sync_bool_compare_and_swap(&res->req_thread_status,
+	if (cmpxchg(&res->req_thread_status,
 				IHK_SCD_REQ_THREAD_DESCHEDULED,
-				IHK_SCD_REQ_THREAD_TO_BE_WOKEN)) {
+				IHK_SCD_REQ_THREAD_TO_BE_WOKEN) !=
+			IHK_SCD_REQ_THREAD_DESCHEDULED) {
 		printk("%s: WARNING: inconsistent requester status, "
 				"pid: %d, req status: %lu, syscall nr: %lu\n",
 				__FUNCTION__, packet->pid,
@@ -537,7 +539,11 @@ retry_alloc:
 #define	USE_VM_INSERT_PFN	1
 
 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
+#if defined(RHEL_RELEASE_CODE) && RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(8, 2)
+static vm_fault_t rus_vm_fault(struct vm_fault *vmf)
+#else
 static int rus_vm_fault(struct vm_fault *vmf)
+#endif
 {
 	struct vm_area_struct *vma = vmf->vma;
 #else
@@ -970,6 +976,30 @@ static int pager_req_create(ihk_os_t os, int fd, uintptr_t result_pa)
 		goto out;
 	}
 
+	/* Shared memory hack */
+	{
+		char *pathbuf, *fullpath;
+		pathbuf = kmalloc(PATH_MAX, GFP_ATOMIC);
+		if (pathbuf) {
+			fullpath = d_path(&file->f_path, pathbuf, PATH_MAX);
+			if (!IS_ERR(fullpath)) {
+				if (!strncmp("/tmp/ompi.", fullpath, 10) ||
+						!strncmp("/dev/shm/", fullpath, 9) ||
+						(!strncmp("/var/opt/FJSVtcs/ple/daemonif/",
+							fullpath, 30) && !strstr(fullpath, "dstore_sm.lock"))) {
+					printk("%s: treating %s as a device file..\n",
+						__func__, fullpath);
+					kfree(pathbuf);
+
+					error = -ESRCH;
+					goto out;
+				}
+
+				kfree(pathbuf);
+			}
+		}
+	}
+
 	inode = file->f_path.dentry->d_inode;
 	if (!inode) {
 		error = -EBADF;
@@ -1029,7 +1059,7 @@ static int pager_req_create(ihk_os_t os, int fd, uintptr_t result_pa)
 			pager = newpager;
 			newpager = NULL;
 
-			/* Intel MPI library and shared memory "prefetch" */
+			/* Shared libraries prefetch */
 			{
 				char *pathbuf, *fullpath;
 
@@ -1037,15 +1067,7 @@ static int pager_req_create(ihk_os_t os, int fd, uintptr_t result_pa)
 				if (pathbuf) {
 					fullpath = d_path(&file->f_path, pathbuf, PATH_MAX);
 					if (!IS_ERR(fullpath)) {
-						if (!strncmp("/dev/shm/Intel_MPI", fullpath, 18)) {
-							mf_flags = (MF_PREMAP | MF_ZEROFILL);
-							dprintk("%s: filename: %s, premap & zerofill\n",
-									__FUNCTION__, fullpath);
-						}
-						else if (strstr(fullpath, "libmpi") ||
-								strstr(fullpath, "libiomp") ||
-								strstr(fullpath, "libpthread") ||
-								strstr(fullpath, "libc.so")) {
+						if (strstr(fullpath, ".so")) {
 							mf_flags = MF_PREFETCH;
 							dprintk("%s: filename: %s, prefetch\n",
 									__FUNCTION__, fullpath);
@@ -1417,6 +1439,26 @@ static int pager_req_map(ihk_os_t os, int fd, size_t len, off_t off,
 #define	ANY_WHERE 0
 	if (prot_and_flags & MAP_LOCKED) prot_and_flags |= MAP_POPULATE;
 
+	/* Shared memory hack */
+	{
+		char *pathbuf, *fullpath;
+		pathbuf = kmalloc(PATH_MAX, GFP_ATOMIC);
+		if (pathbuf) {
+			fullpath = d_path(&file->f_path, pathbuf, PATH_MAX);
+			if (!IS_ERR(fullpath)) {
+				if (!strncmp("/tmp/ompi.", fullpath, 10) ||
+						!strncmp("/dev/shm/", fullpath, 9) ||
+						!strncmp("/var/opt/FJSVtcs/ple/daemonif/",
+							fullpath, 30)) {
+					dprintk("%s: pre-populating %s..\n",
+						__func__, fullpath);
+					prot_and_flags |= MAP_POPULATE;
+				}
+				kfree(pathbuf);
+			}
+		}
+	}
+
 #if LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0)
 	down_write(&current->mm->mmap_sem);
 
@@ -1430,7 +1472,12 @@ static int pager_req_map(ihk_os_t os, int fd, size_t len, off_t off,
 #endif
 
 	if (IS_ERR_VALUE(va)) {
-		printk("pager_req_map(%p,%d,%lx,%lx,%lx):do_mmap_pgoff failed. %d\n", os, fd, len, off, result_rpa, (int)va);
+		if ((int)va != -ENOTSUPP) {
+			pr_err("%s(%p,%d,%lx,%lx,%lx): "
+			       "do_mmap_pgoff failed. %d\n",
+			       __func__, os, fd, len, off, result_rpa,
+			       (int)va);
+		}
 		error = va;
 		goto out;
 	}
@@ -1579,10 +1626,27 @@ retry:
 #else
 		fault = handle_mm_fault(current->mm, vma, va, flags);
 #endif
+#ifdef SC_DEBUG
 		if (fault != 0) {
-			printk("%s: error: faulting %lx at off: %lu\n", 
-					__FUNCTION__, va, off);
+			char *pathbuf = NULL;
+			char *fullpath;
+
+			if (vma->vm_file) {
+				pathbuf = kmalloc(PATH_MAX, GFP_ATOMIC);
+				if (pathbuf) {
+					fullpath = d_path(&vma->vm_file->f_path,
+							pathbuf, PATH_MAX);
+					if (!IS_ERR(fullpath)) {
+						printk("%s: WARNING: couldn't fault 0x%lx"
+								" at off: %lu in %s\n",
+								__FUNCTION__, va, off, fullpath);
+					}
+
+					kfree(pathbuf);
+				}
+			}
 		}
+#endif
 
 		page_fault_attempted = 1;
 		goto retry;

executer/kernel/mcctrl/sysfs_files.c

@@ -12,7 +12,6 @@
  */
 
 #include <linux/kernel.h>
-#include <linux/hashtable.h>
 #include <linux/slab.h>
 #include <linux/uaccess.h>
 #include <linux/version.h>
@@ -24,14 +23,6 @@
 #define wprintk(...) do { if (1) printk(KERN_WARNING __VA_ARGS__); } while (0)
 #define eprintk(...) do { if (1) printk(KERN_ERR __VA_ARGS__); } while (0)
 
-struct physical_core_id {
-	int linux_core_id;
-	int mckernel_core_id;
-	struct hlist_node next;
-};
-
-DEFINE_HASHTABLE(physical_core_id_map, 10);
-
 static ssize_t
 show_int(struct sysfsm_ops *ops, void *instance, void *buf, size_t size)
 {
@@ -197,9 +188,6 @@ static void free_cpu_topology(struct mcctrl_usrdata *udp)
 void free_topology_info(ihk_os_t os)
 {
 	struct mcctrl_usrdata *udp = ihk_host_os_get_usrdata(os);
-	int bkt;
-	struct hlist_node *tmp;
-	struct physical_core_id *cur;
 
 	if (!udp) {
 		pr_warn("%s: warning: mcctrl_usrdata not found\n", __func__);
@@ -209,11 +197,6 @@ void free_topology_info(ihk_os_t os)
 	free_node_topology(udp);
 	free_cpu_topology(udp);
 
-	hash_for_each_safe(physical_core_id_map, bkt, tmp, cur, next) {
-		hash_del(&cur->next);
-		kfree(cur);
-	}
-
 	return;
 } /* free_topology_info() */
 
@@ -365,11 +348,6 @@ static struct mcctrl_cpu_topology *get_one_cpu_topology(struct mcctrl_usrdata *u
 	struct mcctrl_cpu_topology *topology = NULL;
 	struct cache_topology *cache;
 	struct ihk_cache_topology *saved_cache;
-	int linux_core_id;
-	int mckernel_core_id;
-	struct physical_core_id *entry;
-	struct physical_core_id *cur;
-	static int nr_mckernel_core;
 
 	dprintk("get_one_cpu_topology(%p,%d)\n", udp, index);
 	topology = kmalloc(sizeof(*topology), GFP_KERNEL);
@@ -413,22 +391,6 @@ static struct mcctrl_cpu_topology *get_one_cpu_topology(struct mcctrl_usrdata *u
 		goto out;
 	}
 
-	linux_core_id = topology->saved->core_id;
-	mckernel_core_id = -1;
-	hash_for_each_possible(physical_core_id_map, cur, next, linux_core_id) {
-		mckernel_core_id = cur->mckernel_core_id;
-		break;
-	}
-	if (mckernel_core_id < 0) {
-		mckernel_core_id = nr_mckernel_core++;
-		entry = kmalloc(sizeof(struct physical_core_id), GFP_KERNEL);
-		entry->linux_core_id = linux_core_id;
-		entry->mckernel_core_id = mckernel_core_id;
-		hash_add(physical_core_id_map,
-			 &entry->next, entry->linux_core_id);
-	}
-	topology->mckernel_core_id = mckernel_core_id;
-
 	list_for_each_entry(saved_cache,
 			&topology->saved->cache_topology_list, chain) {
 		cache = get_cache_topology(udp, topology, saved_cache);
@@ -550,7 +512,7 @@ static void setup_cpu_sysfs_files(struct mcctrl_usrdata *udp,
 			"%s/cpu%d/topology/physical_package_id",
 			prefix, cpu_number);
 	sysfsm_createf(udp->os, SYSFS_SNOOPING_OPS_d32,
-			&cpu->mckernel_core_id, 0444,
+			&cpu->saved->core_id, 0444,
 			"%s/cpu%d/topology/core_id",
 			prefix, cpu_number);
 

executer/user/CMakeLists.txt

@@ -8,7 +8,7 @@ include_directories(
 	"${PROJECT_BINARY_DIR}/ihk/linux/include"
 )
 
-
+add_subdirectory(lib)
 add_library(libmcexec STATIC arch/${ARCH}/archdep.S)
 SET_TARGET_PROPERTIES(libmcexec PROPERTIES OUTPUT_NAME mcexec)
 set_property(TARGET libmcexec PROPERTY POSITION_INDEPENDENT_CODE ON)
@@ -20,6 +20,18 @@ target_include_directories(mcexec PUBLIC "${KERNEL_DIR}")
 set_property(TARGET mcexec PROPERTY POSITION_INDEPENDENT_CODE ON)
 set_property(TARGET mcexec PROPERTY LINK_FLAGS "-fPIE -pie")
 
+add_executable(mcinspect mcinspect.c)
+if (NOT LIBDWARF)
+	target_include_directories(mcinspect PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/lib/")
+	target_include_directories(mcinspect PRIVATE
+		"${CMAKE_CURRENT_SOURCE_DIR}/lib/libdwarf/libdwarf/libdwarf/")
+	target_link_libraries(mcinspect dwarf z elf)
+else()
+	target_include_directories(mcinspect PRIVATE ${DWARF_H})
+	target_link_libraries(mcinspect ${LIBDWARF})
+endif()
+target_link_libraries(mcinspect ${LIBBFD})
+
 add_executable(eclair eclair.c arch/${ARCH}/arch-eclair.c)
 target_link_libraries(eclair ${LIBBFD})
 
@@ -74,10 +86,13 @@ add_library(ldump2mcdump SHARED ldump2mcdump.c)
 configure_file(vmcore2mckdump.in vmcore2mckdump @ONLY)
 
 configure_file(mcexec.1in mcexec.1 @ONLY)
+configure_file(mcps.in mcps @ONLY)
 
-install(TARGETS mcexec eclair
+install(TARGETS mcexec eclair mcinspect
 	DESTINATION "${CMAKE_INSTALL_BINDIR}")
-install(PROGRAMS "${CMAKE_CURRENT_BINARY_DIR}/vmcore2mckdump"
+install(PROGRAMS
+	"${CMAKE_CURRENT_BINARY_DIR}/vmcore2mckdump"
+	"${CMAKE_CURRENT_BINARY_DIR}/mcps"
 	DESTINATION "${CMAKE_INSTALL_BINDIR}")
 install(TARGETS sched_yield ldump2mcdump
 	DESTINATION "${CMAKE_INSTALL_LIBDIR}")

executer/user/lib/CMakeLists.txt

@@ -0,0 +1,3 @@
+if (NOT LIBDWARF)
+	add_subdirectory(libdwarf)
+endif()

executer/user/lib/libdwarf/CMakeLists.txt

@@ -0,0 +1,144 @@
+if (NOT LIBDWARF)
+	# view folders on supported IDEs
+	set_property(GLOBAL PROPERTY USE_FOLDERS ON)
+
+	# used when finding libelf
+	set_property(GLOBAL PROPERTY FIND_LIBRARY_USE_LIB64_PATHS TRUE)
+
+	find_package(LibElf REQUIRED)
+	list(APPEND CMAKE_REQUIRED_INCLUDES ${LIBELF_INCLUDE_DIRS})
+	include(AutoconfHelper)
+
+	ac_init()
+	ac_c_bigendian()
+	ac_check_headers(sys/types.h sys/stat.h stdlib.h string.h memory.h strings.h inttypes.h stdint.h unistd.h)
+	ac_check_headers(alloca.h elf.h elfaccess.h libelf.h libelf/libelf.h  sys/types.h sys/elf_386.h sys/elf_amd64.h sys/elf_sparc.h sys/ia64/elf.h)
+
+	#  The default libdwarf is the one with struct Elf
+	message(STATUS "Assuming struct Elf for the default libdwarf.h")
+
+	# Find out where the elf header is.
+	if(HAVE_ELF_H)
+		set(HAVE_LOCATION_OF_LIBELFHEADER "<elf.h>")
+	elseif(HAVE_LIBELF_H)
+		set(HAVE_LOCATION_OF_LIBELFHEADER "<libelf.h>")
+	elseif(HAVE_LIBELF_LIBELF_H)
+		set(HAVE_LOCATION_OF_LIBELFHEADER "<libelf/libelf.h>")
+	endif()
+
+	ac_check_lib(${LIBELF_LIBRARIES} elf64_getehdr)
+	ac_check_lib(${LIBELF_LIBRARIES} elf64_getshdr)
+
+	ac_try_compile("
+	int main()
+	{
+		__uint32_t p;
+		p = 3;
+		return 0;
+	}"
+	HAVE___UINT32_T)
+
+	ac_try_compile("
+	int main()
+	{
+		__uint64_t p;
+		p = 3;
+		return 0;
+	}"
+	HAVE___UINT64_T)
+
+	ac_try_compile("
+	#include <sys/types.h>
+	int main()
+	{
+		__uint32_t p;
+		p = 3;
+		return 0;
+	}"
+	HAVE___UINT32_T_IN_SYS_TYPES_H)
+
+	ac_try_compile("
+	#include <sys/types.h>
+	int main()
+	{
+		__uint64_t p;
+		p = 3;
+		return 0;
+	}"
+	HAVE___UINT64_T_IN_SYS_TYPES_H)
+
+	check_c_source_runs("
+	static unsigned foo( unsigned x, __attribute__ ((unused)) int y)
+	{
+		unsigned x2 = x + 1;
+		return x2;
+	}
+
+	int main(void) {
+		unsigned y = 0;
+		y = foo(12,y);
+		return 0;
+	}"
+	HAVE_UNUSED_ATTRIBUTE)
+
+	message(STATUS "Checking compiler supports __attribute__ unused... ${HAVE_UNUSED_ATTRIBUTE}")
+
+	ac_try_compile("
+	#include <zlib.h>
+	int main()
+	{
+		Bytef dest[100];
+		uLongf destlen = 100;
+		Bytef *src = 0;
+		uLong srclen = 3;
+		int res = uncompress(dest,&destlen,src,srclen);
+		if (res == Z_OK) {
+			 /* ALL IS WELL */
+		}
+		return 0;
+	}"
+	HAVE_ZLIB)
+	message(STATUS "Checking zlib.h usability... ${HAVE_ZLIB}")
+	set(dwfzlib $<$<BOOL:${HAVE_ZIB}>:"z")
+
+	configure_file(libdwarf/libdwarf/libdwarf.h.in libdwarf.h COPYONLY)
+	configure_file(libdwarf/libdwarf/config.h.in.cmake config.h)
+	set(DWARF_CONFIGURATION_FILES ${CMAKE_CURRENT_BINARY_DIR}/config.h)
+
+	set(DWARF_SOURCES dwarf_abbrev.c dwarf_alloc.c dwarf_arange.c dwarf_die_deliv.c dwarf_dnames.c dwarf_dsc.c dwarf_elf_access.c dwarf_error.c
+	dwarf_form.c dwarf_frame.c dwarf_frame2.c dwarf_funcs.c dwarf_gdbindex.c dwarf_global.c dwarf_groups.c dwarf_harmless.c dwarf_init_finish.c dwarf_leb.c
+	dwarf_line.c dwarf_loc.c dwarf_macro.c dwarf_macro5.c dwarf_original_elf_init.c dwarf_pubtypes.c dwarf_query.c dwarf_ranges.c dwarf_string.c dwarf_tied.c
+	dwarf_str_offsets.c
+	dwarf_tsearchhash.c dwarf_types.c dwarf_util.c dwarf_vars.c dwarf_weaks.c dwarf_xu_index.c dwarf_print_lines.c malloc_check.c pro_alloc.c pro_arange.c
+	pro_die.c pro_encode_nm.c pro_error.c pro_expr.c pro_finish.c pro_forms.c pro_funcs.c pro_frame.c pro_init.c pro_line.c pro_reloc.c pro_reloc_stream.c
+	pro_reloc_symbolic.c pro_pubnames.c pro_section.c pro_types.c pro_vars.c pro_macinfo.c pro_weaks.c)
+
+	set(DWARF_HEADERS dwarf.h dwarf_abbrev.h dwarf_alloc.h dwarf_arange.h dwarf_base_types.h dwarf_die_deliv.h dwarf_dnames.h dwarf_dsc.h
+	dwarf_elf_access.h dwarf_error.h dwarf_frame.h dwarf_funcs.h dwarf_gdbindex.h dwarf_global.h dwarf_harmless.h dwarf_incl.h dwarf_line.h dwarf_loc.h
+	dwarf_macro.h dwarf_macro5.h dwarf_opaque.h dwarf_reloc_arm.h dwarf_reloc_mips.h dwarf_reloc_ppc.h dwarf_reloc_ppc64.h dwarf_reloc_x86_64.h dwarf_tsearch.h
+	dwarf_str_offsets.h
+	dwarf_types.h dwarf_util.h dwarf_vars.h dwarf_weaks.h dwarf_xu_index.h dwgetopt.h libdwarfdefs.h malloc_check.h pro_alloc.h pro_arange.h pro_die.h
+	pro_encode_nm.h pro_error.h pro_expr.h pro_frame.h pro_incl.h pro_line.h pro_macinfo.h pro_opaque.h pro_reloc.h pro_reloc_stream.h pro_reloc_symbolic.h
+	pro_section.h pro_types.h pro_util.h)
+
+	SET(__SRCS "")
+		FOREACH(f ${DWARF_SOURCES})
+			LIST(APPEND __SRCS "libdwarf/libdwarf/${f}")
+		ENDFOREACH(f)
+	SET(DWARF_SOURCES ${__SRCS})
+
+	set(GENNAMES_SOURCES
+		libdwarf/libdwarf/gennames.c
+		libdwarf/libdwarf/dwgetopt.c
+		libdwarf/libdwarf/dwarf.h)
+	add_executable(gennames ${GENNAMES_SOURCES})
+	set(GENNAMES_OUTPUT dwarf_names.c dwarf_names.h dwarf_names_enum.h dwarf_names_new.h)
+
+	add_custom_command(OUTPUT ${GENNAMES_OUTPUT}
+		COMMAND gennames -s -i ${CMAKE_CURRENT_SOURCE_DIR}/libdwarf/libdwarf/ -o .
+		DEPENDS gennames libdwarf/libdwarf/libdwarf.h.in)
+
+	add_library(dwarf STATIC ${DWARF_SOURCES} ${GENNAMES_OUTPUT} ${DWARF_CONFIGURATION_FILES})
+	target_include_directories(dwarf PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/libdwarf/libdwarf/")
+	target_include_directories(dwarf BEFORE PRIVATE "${CMAKE_CURRENT_BINARY_DIR}")
+endif()

executer/user/libsched_yield.c

@@ -1,6 +1,7 @@
 #define _GNU_SOURCE
 #include <dlfcn.h>
 #include <sys/time.h>
+#include <sys/syscall.h>
 #include <sched.h>
 #include <unistd.h>
 #include <stdlib.h>
@@ -27,3 +28,110 @@ int sched_yield(void)
 
 	return 0;
 }
+
+#undef pthread_create
+
+typedef int (*__pthread_create_fn)(pthread_t *thread,
+		const pthread_attr_t *attr,
+		void *(*start_routine) (void *),
+		void *arg);
+
+static __pthread_create_fn orig_pthread_create = 0;
+
+
+int pthread_create(pthread_t *thread,
+		const pthread_attr_t *attr,
+		void *(*start_routine) (void *),
+		void *arg)
+{
+	if (!orig_pthread_create) {
+		orig_pthread_create =
+			(__pthread_create_fn)dlsym(RTLD_NEXT, "pthread_create");
+	}
+
+	/* CLONE_VM and newsp == parent_tidptr impiles pthread start routine addr */
+	syscall(__NR_clone, CLONE_VM, start_routine, start_routine, 0, 0, 0);
+
+	return orig_pthread_create(thread, attr, start_routine, arg);
+}
+
+#if 0
+#define PROCMAPS_MAX_LEN	131072
+
+char *addr_to_lib(void *addr, unsigned long *offset_in_lib)
+{
+	char maps_path[PATH_MAX];
+	char buf[PROCMAPS_MAX_LEN];
+	int fd;
+	void *start, *end;
+	char perms[4];
+	unsigned long offset;
+	unsigned long dev[2];
+	int inode;
+	char path[PATH_MAX];
+	char *line;
+
+	sprintf(maps_path,"/proc/self/maps");
+	fd = open(maps_path, O_RDONLY);
+	if (fd < 0) {
+		fprintf(stderr,"error: cannot open the memory maps, %s\n",
+				strerror(errno));
+		return NULL;
+	}
+
+	memset(buf, 0, PROCMAPS_MAX_LEN);
+	read(fd, buf, PROCMAPS_MAX_LEN);
+	line = strtok(buf, "\n");
+	while (line) {
+		memset(path, 0, sizeof(path));
+		sscanf(line, "%012lx-%012lx %4s %lx %lx:%lx %d\t\t\t%[^\n]",
+			&start, &end, perms, &offset, &dev[0], &dev[1], &inode, path);
+
+		if (start <= addr && end > addr) {
+			close(fd);
+			if (offset_in_lib)
+				*offset_in_lib = (unsigned long)(addr - start);
+			return strlen(path) > 0 ? strdup(path) : NULL;
+		}
+
+		line = strtok(NULL, "\n");
+	}
+
+	close(fd);
+	return NULL;
+}
+
+int pthread_create(pthread_t *thread,
+		const pthread_attr_t *attr,
+		void *(*start_routine) (void *),
+		void *arg)
+{
+	char *lib = NULL;
+	int util_thread = 1;
+	unsigned long offset;
+
+	if (!orig_pthread_create) {
+		orig_pthread_create =
+			(__pthread_create_fn)dlsym(RTLD_NEXT, "pthread_create");
+	}
+
+	lib = addr_to_lib(start_routine, &offset);
+	if (lib)
+		printf("%s: 0x%lx is in %s @ 0x%lx\n",
+				__func__, start_routine, lib, offset);
+
+	if (lib && (strstr(lib, "iomp") || strstr(lib, "psm"))) {
+		util_thread = 0;
+	}
+
+	if (util_thread) {
+		/* McKernel util_indicate_clone() */
+		syscall(731);
+	}
+
+	if (lib)
+		free(lib);
+
+	return orig_pthread_create(thread, attr, start_routine, arg);
+}
+#endif

executer/user/mcexec.c

@@ -1322,7 +1322,8 @@ void init_sigaction(void)
 
 	master_tid = gettid();
 	for (i = 1; i <= 64; i++) {
-		if (i != SIGKILL && i != SIGSTOP && i != SIGCHLD) {
+		if (i != SIGKILL && i != SIGSTOP && i != SIGCHLD &&
+				i != SIGTSTP && i != SIGTTIN && i != SIGTTOU) {
 			struct sigaction act;
 
 			sigaction(i, NULL, &act);
@@ -2211,6 +2212,64 @@ int main(int argc, char **argv)
 
 	pthread_spin_init(&overlay_fd_lock, 0);
 
+	/* XXX: Fugaku: Fujitsu process placement fix */
+	if (getenv("FLIB_AFFINITY_ON_PROCESS")) {
+		char *cpu_s;
+		int flib_size;
+		char *flib_aff_orig, *flib_aff;
+		int cpu, off = 0;
+
+		flib_aff_orig = strdup(getenv("FLIB_AFFINITY_ON_PROCESS"));
+		if (!flib_aff_orig) {
+			fprintf(stderr, "error: dupping FLIB_AFFINITY_ON_PROCESS\n");
+			exit(EXIT_FAILURE);
+		}
+
+		flib_size = strlen(flib_aff_orig) * 2;
+
+		flib_aff = malloc(flib_size);
+		if (!flib_aff) {
+			fprintf(stderr, "error: allocating memory for "
+					"FLIB_AFFINITY_ON_PROCESS\n");
+			exit(EXIT_FAILURE);
+		}
+		memset(flib_aff, 0, flib_size);
+
+		cpu_s = strtok(flib_aff_orig, ",");
+		while (cpu_s) {
+			int ret;
+
+			/* "Shift" left by 12 CPUs */
+			cpu = atoi(cpu_s) - 12;
+
+			/* Prepend "," */
+			if (off > 0) {
+				ret = snprintf(flib_aff + off, flib_size - off, "%s", ",");
+				if (ret < 0) {
+					fprintf(stderr, "error: constructing "
+							"FLIB_AFFINITY_ON_PROCESS\n");
+					exit(EXIT_FAILURE);
+				}
+
+				off += ret;
+			}
+
+			ret = snprintf(flib_aff + off, flib_size - off, "%d", cpu);
+			if (ret < 0) {
+				fprintf(stderr, "error: constructing "
+						"FLIB_AFFINITY_ON_PROCESS\n");
+				exit(EXIT_FAILURE);
+			}
+
+			off += ret;
+			cpu_s = strtok(NULL, ",");
+		}
+
+		__dprintf("FLIB_AFFINITY_ON_PROCESS: %s -> %s\n",
+				getenv("FLIB_AFFINITY_ON_PROCESS"), flib_aff);
+		setenv("FLIB_AFFINITY_ON_PROCESS", flib_aff, 1);
+	}
+
 	ld_preload_init();
 
 #ifdef ADD_ENVS_OPTION
@@ -2385,6 +2444,13 @@ int main(int argc, char **argv)
 		}
 	}
 
+	/* Fugaku: use FLIB_NUM_PROCESS_ON_NODE if -n is not specified */
+	if (getenv("FLIB_NUM_PROCESS_ON_NODE") && nr_processes == 0) {
+		nr_processes = atoi(getenv("FLIB_NUM_PROCESS_ON_NODE"));
+		__dprintf("%s: using FLIB_NUM_PROCESS_ON_NODE: %d\n",
+			__func__, nr_processes);
+	}
+
 	if (nr_processes > ncpu) {
 		fprintf(stderr, "error: nr_processes can't exceed nr. of CPUs\n");
 		return EINVAL;
@@ -2483,10 +2549,11 @@ int main(int argc, char **argv)
 
 		CPU_ZERO(&mcexec_cpu_set);
 
+		cpu_set_arg.req_cpu_list = NULL;
+		cpu_set_arg.req_cpu_list_len = 0;
 		cpu_set_arg.cpu_set = (void *)&desc->cpu_set;
 		cpu_set_arg.cpu_set_size = sizeof(desc->cpu_set);
 		cpu_set_arg.nr_processes = nr_processes;
-		cpu_set_arg.ppid = getppid();
 		cpu_set_arg.target_core = &target_core;
 		cpu_set_arg.process_rank = &process_rank;
 		cpu_set_arg.mcexec_linux_numa = &mcexec_linux_numa;
@@ -2494,6 +2561,16 @@ int main(int argc, char **argv)
 		cpu_set_arg.mcexec_cpu_set_size = sizeof(mcexec_cpu_set);
 		cpu_set_arg.ikc_mapped = &ikc_mapped;
 
+		/* Fugaku specific: Fujitsu CPU binding */
+		if (getenv("FLIB_AFFINITY_ON_PROCESS")) {
+			cpu_set_arg.req_cpu_list =
+				getenv("FLIB_AFFINITY_ON_PROCESS");
+			cpu_set_arg.req_cpu_list_len =
+				strlen(cpu_set_arg.req_cpu_list) + 1;
+			__dprintf("%s: requesting CPUs: %s\n",
+				__func__, cpu_set_arg.req_cpu_list);
+		}
+
 		if (ioctl(fd, MCEXEC_UP_GET_CPUSET, (void *)&cpu_set_arg) != 0) {
 			perror("getting CPU set for partitioned execution");
 			close(fd);
@@ -2503,6 +2580,13 @@ int main(int argc, char **argv)
 		desc->cpu = target_core;
 		desc->process_rank = process_rank;
 
+		/* Fugaku specific: Fujitsu node-local rank */
+		if (getenv("FLIB_RANK_ON_NODE")) {
+			desc->process_rank = atoi(getenv("FLIB_RANK_ON_NODE"));
+			__dprintf("%s: rank: %d, target CPU: %d\n",
+				__func__, desc->process_rank, desc->cpu);
+		}
+
 		/* Bind to CPU cores where the LWK process' IKC target maps to */
 		if (ikc_mapped && !no_bind_ikc_map) {
 			/* This call may not succeed, but that is fine */

executer/user/mcps.in

@@ -0,0 +1,10 @@
+#!/bin/bash
+# IHK/McKernel mcps script.
+# author: Balazs Gerofi <bgerofi@riken.jp>
+#      Copyright (C) 2019 RIKEN
+#
+prefix="@prefix@"
+BINDIR="${prefix}/bin"
+KERNDIR="@MCKERNELDIR@"
+
+${BINDIR}/mcinspect --kernel=${KERNDIR}/mckernel.img --ps

kernel/devobj.c

@@ -170,7 +170,7 @@ static void devobj_free(struct memobj *memobj)
 
 	error = syscall_generic_forwarding(__NR_mmap, &ctx);
 	if (error) {
-		kprintf("%s(%p %lx): release failed. %d\n",
+		dkprintf("%s(%p %lx): release failed. %d\n",
 			__func__, obj, handle, error);
 		/* through */
 	}

kernel/fileobj.c

@@ -398,8 +398,9 @@ static void fileobj_free(struct memobj *memobj)
 		}
 	}
 
-	/* Pre-mapped? */
-	if (to_memobj(obj)->flags & MF_PREMAP) {
+	/* Pre-mapped zerofilled? */
+	if (to_memobj(obj)->flags & MF_PREMAP &&
+			to_memobj(obj)->flags & MF_ZEROFILL) {
 		int i;
 
 		for (i = 0; i < to_memobj(obj)->nr_pages; ++i) {
@@ -440,7 +441,7 @@ static void fileobj_free(struct memobj *memobj)
 
 	error = syscall_generic_forwarding(__NR_mmap, &ctx);
 	if (error) {
-		kprintf("%s(%p %lx): free failed. %d\n", __func__,
+		dkprintf("%s(%p %lx): free failed. %d\n", __func__,
 			obj, obj->handle, error);
 		/* through */
 	}
@@ -473,20 +474,24 @@ static void fileobj_do_pageio(void *args0)
 	ssize_t ss;
 	struct mcs_lock_node mcs_node;
 	int hash = (off >> PAGE_SHIFT) & FILEOBJ_PAGE_HASH_MASK;	
+	int attempts = 0;
 
-	mcs_lock_lock_noirq(&obj->page_hash_locks[hash],
-			&mcs_node);
+	mcs_lock_lock(&obj->page_hash_locks[hash], &mcs_node);
 	page = __fileobj_page_hash_lookup(obj, hash, off);
 	if (!page) {
 		goto out;
 	}
 
 	while (page->mode == PM_PAGEIO) {
-		mcs_lock_unlock_noirq(&obj->page_hash_locks[hash],
-				&mcs_node);
+		mcs_lock_unlock(&obj->page_hash_locks[hash], &mcs_node);
+		++attempts;
+		if (attempts > 49) {
+			dkprintf("%s: %s:%lu PM_PAGEIO loop %d -> schedule()\n",
+				__func__, to_memobj(obj)->path, off, attempts);
+			schedule();
+		}
 		cpu_pause();
-		mcs_lock_lock_noirq(&obj->page_hash_locks[hash],
-				&mcs_node);
+		mcs_lock_lock(&obj->page_hash_locks[hash], &mcs_node);
 	}
 
 	if (page->mode == PM_WILL_PAGEIO) {
@@ -499,8 +504,7 @@ static void fileobj_do_pageio(void *args0)
 		}
 		else {
 			page->mode = PM_PAGEIO;
-			mcs_lock_unlock_noirq(&obj->page_hash_locks[hash],
-					&mcs_node);
+			mcs_lock_unlock(&obj->page_hash_locks[hash], &mcs_node);
 
 			ihk_mc_syscall_arg0(&ctx) = PAGER_REQ_READ;
 			ihk_mc_syscall_arg1(&ctx) = obj->handle;
@@ -512,8 +516,7 @@ static void fileobj_do_pageio(void *args0)
 					__FUNCTION__, obj->handle);
 			ss = syscall_generic_forwarding(__NR_mmap, &ctx);
 
-			mcs_lock_lock_noirq(&obj->page_hash_locks[hash],
-					&mcs_node);
+			mcs_lock_lock(&obj->page_hash_locks[hash], &mcs_node);
 			if (page->mode != PM_PAGEIO) {
 				kprintf("fileobj_do_pageio(%p,%lx,%lx):"
 						"invalid mode %x\n",
@@ -539,8 +542,7 @@ static void fileobj_do_pageio(void *args0)
 		page->mode = PM_DONE_PAGEIO;
 	}
 out:
-	mcs_lock_unlock_noirq(&obj->page_hash_locks[hash],
-			&mcs_node);
+	mcs_lock_unlock(&obj->page_hash_locks[hash], &mcs_node);
 	memobj_unref(&obj->memobj);		/* got fileobj_get_page() */
 	kfree(args0);
 	dkprintf("fileobj_do_pageio(%p,%lx,%lx):\n", obj, off, pgsize);
@@ -570,7 +572,8 @@ static int fileobj_get_page(struct memobj *memobj, off_t off,
 	profile_event_add(PROFILE_page_fault_file, PAGE_SIZE);
 #endif // PROFILE_ENABLE
 
-	if (memobj->flags & MF_PREMAP) {
+	if (memobj->flags & MF_PREMAP &&
+			memobj->flags & MF_ZEROFILL) {
 		int page_ind = off >> PAGE_SHIFT;
 
 		if (!memobj->pages[page_ind]) {
@@ -587,8 +590,7 @@ static int fileobj_get_page(struct memobj *memobj, off_t off,
 
 			/* Update the array but see if someone did it already and use
 			 * that if so */
-			if (!__sync_bool_compare_and_swap(&memobj->pages[page_ind],
-						NULL, virt)) {
+			if (cmpxchg(&memobj->pages[page_ind], NULL, virt) != NULL) {
 				ihk_mc_free_pages_user(virt, 1);
 			}
 			else {
@@ -609,8 +611,7 @@ static int fileobj_get_page(struct memobj *memobj, off_t off,
 		goto out_nolock;
 	}
 
-	mcs_lock_lock_noirq(&obj->page_hash_locks[hash],
-			&mcs_node);
+	mcs_lock_lock(&obj->page_hash_locks[hash], &mcs_node);
 	page = __fileobj_page_hash_lookup(obj, hash, off);
 	if (!page || (page->mode == PM_WILL_PAGEIO)
 			|| (page->mode == PM_PAGEIO)) {
@@ -688,8 +689,7 @@ static int fileobj_get_page(struct memobj *memobj, off_t off,
 	*physp = page_to_phys(page);
 	virt = NULL;
 out:
-	mcs_lock_unlock_noirq(&obj->page_hash_locks[hash],
-			&mcs_node);
+	mcs_lock_unlock(&obj->page_hash_locks[hash], &mcs_node);
 out_nolock:
 	if (virt) {
 		ihk_mc_free_pages_user(virt, npages);
@@ -767,8 +767,7 @@ static int fileobj_lookup_page(struct memobj *memobj, off_t off,
 		return -ENOMEM;
 	}
 
-	mcs_lock_lock_noirq(&obj->page_hash_locks[hash],
-			&mcs_node);
+	mcs_lock_lock(&obj->page_hash_locks[hash], &mcs_node);
 
 	page = __fileobj_page_hash_lookup(obj, hash, off);
 	if (!page) {
@@ -779,8 +778,7 @@ static int fileobj_lookup_page(struct memobj *memobj, off_t off,
 	error = 0;
 
 out:
-	mcs_lock_unlock_noirq(&obj->page_hash_locks[hash],
-			&mcs_node);
+	mcs_lock_unlock(&obj->page_hash_locks[hash], &mcs_node);
 
 	dkprintf("fileobj_lookup_page(%p,%lx,%x,%p): %d \n",
 			obj, off, p2align, physp, error);

kernel/futex.c

@@ -311,6 +311,7 @@ static int futex_wake(uint32_t *uaddr, int fshared, int nr_wake, uint32_t bitset
 	struct plist_head *head;
 	union futex_key key = FUTEX_KEY_INIT;
 	int ret;
+	unsigned long irqstate;
 
 	if (!bitset)
 		return -EINVAL;
@@ -320,7 +321,7 @@ static int futex_wake(uint32_t *uaddr, int fshared, int nr_wake, uint32_t bitset
 		goto out;
 
 	hb = hash_futex(&key);
-	ihk_mc_spinlock_lock_noirq(&hb->lock);
+	irqstate = ihk_mc_spinlock_lock(&hb->lock);
 	head = &hb->chain;
 
 	plist_for_each_entry_safe(this, next, head, list) {
@@ -337,7 +338,7 @@ static int futex_wake(uint32_t *uaddr, int fshared, int nr_wake, uint32_t bitset
 		}
 	}
 
-	ihk_mc_spinlock_unlock_noirq(&hb->lock);
+	ihk_mc_spinlock_unlock(&hb->lock, irqstate);
 	put_futex_key(fshared, &key);
 out:
 	return ret;

kernel/host.c

@@ -142,7 +142,7 @@ int prepare_process_ranges_args_envs(struct thread *thread,
 		if (add_process_memory_range(vm, s, e, NOPHYS, flags, NULL, 0,
 					pn->sections[i].len > LARGE_PAGE_SIZE ?
 					LARGE_PAGE_SHIFT : PAGE_SHIFT,
-					NULL, &range) != 0) {
+					&range) != 0) {
 			kprintf("ERROR: adding memory range for ELF section %i\n", i);
 			goto err;
 		}
@@ -284,7 +284,7 @@ int prepare_process_ranges_args_envs(struct thread *thread,
 	dkprintf("%s: args_envs: %d pages\n",
 			__func__, argenv_page_count);
 	if(add_process_memory_range(vm, addr, e, args_envs_p,
-				flags, NULL, 0, PAGE_SHIFT, NULL, NULL) != 0){
+				flags, NULL, 0, PAGE_SHIFT, NULL) != 0){
 		ihk_mc_free_pages_user(args_envs, argenv_page_count);
 		kprintf("ERROR: adding memory range for args/envs\n");
 		goto err;
@@ -500,6 +500,7 @@ static int process_msg_prepare_process(unsigned long rphys)
 		ihk_mc_unmap_memory(NULL, phys, sz);
 		return -ENOMEM;
 	}
+	sprintf(thread->pthread_routine, "%s", "[main]");
 	proc = thread->proc;
 	vm = thread->vm;
 
@@ -619,6 +620,7 @@ static int syscall_packet_handler(struct ihk_ikc_channel_desc *c,
 {
 	struct ikc_scd_packet *packet = __packet;
 	struct ikc_scd_packet pckt;
+	struct ihk_os_cpu_register *cpu_desc;
 	struct ihk_ikc_channel_desc *resp_channel = cpu_local_var(ikc2linux);
 	int rc;
 	struct thread *thread;
@@ -689,11 +691,11 @@ static int syscall_packet_handler(struct ihk_ikc_channel_desc *c,
 			ret = -EINVAL;
 			break;
 		}
-		thread_unlock(thread);
 
 		dkprintf("%s: SCD_MSG_WAKE_UP_SYSCALL_THREAD: waking up tid %d\n",
 			__FUNCTION__, packet->ttid);
 		waitq_wakeup(&thread->scd_wq);
+		thread_unlock(thread);
 		ret = 0;
 		break;
 
@@ -717,9 +719,11 @@ static int syscall_packet_handler(struct ihk_ikc_channel_desc *c,
 		dkprintf("remote page fault,pid=%d,va=%lx,reason=%x\n",
 			 thread->proc->pid, packet->fault_address,
 			 packet->fault_reason|PF_POPULATE);
+		preempt_disable();
 		pckt.err = page_fault_process_vm(thread->vm,
 					(void *)packet->fault_address,
 					packet->fault_reason|PF_POPULATE);
+		preempt_enable();
 
 #ifdef PROFILE_ENABLE
 		if (thread->profile) {
@@ -839,12 +843,17 @@ out_remote_pf:
 		break;
 
 	case SCD_MSG_CPU_RW_REG:
+		pp = ihk_mc_map_memory(NULL, packet->pdesc,
+				sizeof(struct ihk_os_cpu_register));
+		cpu_desc = (struct ihk_os_cpu_register *)ihk_mc_map_virtual(
+				pp, 1, PTATTR_WRITABLE | PTATTR_ACTIVE);
 
 		pckt.msg = SCD_MSG_CPU_RW_REG_RESP;
-		memcpy(&pckt.desc, &packet->desc,
-				sizeof(struct ihk_os_cpu_register));
-		pckt.resp = packet->resp;
-		pckt.err = arch_cpu_read_write_register(&pckt.desc, packet->op);
+		pckt.reply = packet->reply;
+		pckt.err = arch_cpu_read_write_register(cpu_desc, packet->op);
+
+		ihk_mc_unmap_virtual(cpu_desc, 1);
+		ihk_mc_unmap_memory(NULL, pp, sizeof(struct ihk_os_cpu_register));
 
 		ihk_ikc_send(resp_channel, &pckt, 0);
 		break;
@@ -895,7 +904,7 @@ void init_host_ikc2linux(int linux_cpu)
 		param.port = 503;
 		param.intr_cpu = linux_cpu;
 		param.pkt_size = sizeof(struct ikc_scd_packet);
-		param.queue_size = 2 * num_processors * sizeof(struct ikc_scd_packet);
+		param.queue_size = 4 * num_processors * sizeof(struct ikc_scd_packet);
 		if (param.queue_size < PAGE_SIZE * 4) {
 			param.queue_size = PAGE_SIZE * 4;
 		}

kernel/hugefileobj.c

@@ -139,7 +139,6 @@ static void hugefileobj_free(struct memobj *memobj)
 struct memobj_ops hugefileobj_ops = {
 	.free = hugefileobj_free,
 	.get_page = hugefileobj_get_page,
-
 };
 
 void hugefileobj_cleanup(void)

kernel/include/cls.h

@@ -100,6 +100,7 @@ struct cpu_local_var {
 	int in_interrupt;
 	int no_preempt;
 	int timer_enabled;
+	unsigned long nr_ctx_switches;
 	int kmalloc_initialized;
 	struct ihk_os_cpu_monitor *monitor;
 	struct rusage_percpu *rusage;

kernel/include/mman.h

@@ -72,4 +72,11 @@
 #define	MS_INVALIDATE	0x02
 #define	MS_SYNC		0x04
 
+/* include/uapi/linux/mman.h */
+#define OVERCOMMIT_GUESS		0
+#define OVERCOMMIT_ALWAYS		1
+#define OVERCOMMIT_NEVER		2
+
+extern int sysctl_overcommit_memory;
+
 #endif /* HEADER_MMAN_H */

kernel/include/process.h

@@ -55,7 +55,6 @@
 #define	VR_MEMTYPE_MASK    0x0f000000
 #define VR_PAGEOUT	   0x10000000
 #define VR_DONTDUMP	   0x20000000
-#define VR_XPMEM	   0x40000000
 #define VR_WIPEONFORK	   0x80000000
 
 #define	PROT_TO_VR_FLAG(prot)	(((unsigned long)(prot) << 16) & VR_PROT_MASK)
@@ -506,10 +505,12 @@ struct process {
 			// PS_STOPPED -----+
 			// (PS_TRACED)
 
-	/* Store exit_status for a group of threads when stopped by SIGSTOP.
-	   exit_status can't be used because values of exit_status of threads
-	   might divert while the threads are exiting by group_exit(). */
-	int group_exit_status;
+	/* Store exit_status for a group of threads when stopped by SIGSTOP.  */
+	/* exit_status can't be used because values of exit_status of threads */
+	/* might divert while the threads are exiting by group_exit().	      */
+	/* The upper 4 bytes of group_exit_status is the confirmation flag of */
+	/* exit status. The lower 4 bytes is the exit status.		      */
+	unsigned long group_exit_status;
 
 	/* Manage ptraced processes in the separate list to make it easy to
 	   restore the orginal parent child relationship when 
@@ -609,6 +610,7 @@ struct thread {
 	// thread info
 	int cpu_id;
 	int tid;
+	char pthread_routine[PATH_MAX + 64];
 	int status;	// PS_RUNNING -> PS_EXITED (-> ZOMBIE / ptrace)
 			// |       ^       ^
 			// |       |       |
@@ -718,6 +720,7 @@ struct thread {
 	/* Syscall offload wait queue head */
 	struct waitq scd_wq;
 
+	unsigned long clone_pthread_start_routine;
 	int uti_state;
 	int mod_clone;
 	struct uti_attr *mod_clone_arg;
@@ -749,7 +752,7 @@ struct process_vm {
 	void *vvar_addr;
  	
 	ihk_spinlock_t page_table_lock;
-	ihk_spinlock_t memory_range_lock;
+	ihk_rwspinlock_t memory_range_lock;
     // to protect the followings:
     // 1. addition of process "memory range" (extend_process_region, add_process_memory_range)
     // 2. addition of process page table (allocate_pages, update_process_page_table)
@@ -805,7 +808,7 @@ int add_process_memory_range(struct process_vm *vm,
 		unsigned long start, unsigned long end,
 		unsigned long phys, unsigned long flag,
 		struct memobj *memobj, off_t offset,
-		int pgshift, void *private_data, struct vm_range **rp);
+		int pgshift, struct vm_range **rp);
 int remove_process_memory_range(struct process_vm *vm, unsigned long start,
 		unsigned long end, int *ro_freedp);
 int split_process_memory_range(struct process_vm *vm,

kernel/include/profile.h

@@ -5,10 +5,9 @@
 #define PROFILE_ENABLE
 
 #ifdef PROFILE_ENABLE
-#define PROFILE_SYSCALL_MAX                          300
+#define PROFILE_SYSCALL_MAX                          2000
 #define PROFILE_OFFLOAD_MAX   (PROFILE_SYSCALL_MAX << 1)
 #define PROFILE_EVENT_MIN            PROFILE_OFFLOAD_MAX
-#define __NR_profile                                 701
 
 #define PROF_JOB                       0x40000000
 #define PROF_PROC                      0x80000000
@@ -47,6 +46,8 @@ enum profile_event_type {
 	PROFILE_EVENT_MAX	/* Should be the last event type */
 };
 
+#define __NR_profile                   PROFILE_EVENT_MAX
+
 struct thread;
 struct process;
 

kernel/include/rusage_private.h

@@ -11,6 +11,7 @@
 #include <ihk/ihk_monitor.h>
 #include <ihk/debug.h>
 #include <memory.h>
+#include <mman.h>
 
 #ifdef ENABLE_RUSAGE
 
@@ -27,30 +28,17 @@ static inline int rusage_pgsize_to_pgtype(size_t pgsize)
 	case 12:
 		ret = IHK_OS_PGSIZE_4KB;
 		break;
-	case 16:
-		ret = IHK_OS_PGSIZE_64KB;
-		break;
 	case 21:
 		ret = IHK_OS_PGSIZE_2MB;
 		break;
-	case 25:
-		ret = IHK_OS_PGSIZE_32MB;
-		break;
 	case 30:
 		ret = IHK_OS_PGSIZE_1GB;
 		break;
-	case 34:
-		ret = IHK_OS_PGSIZE_16GB;
-		break;
-	case 29:
-		ret = IHK_OS_PGSIZE_512MB;
-		break;
-	case 42:
-		ret = IHK_OS_PGSIZE_4TB;
-		break;
 	default:
+#if 0 /* 64KB page goes here when using mckernel_rusage-compatible ihk_os_rusage */
 		kprintf("%s: Error: Unknown pgsize=%ld\n",
 			__func__, pgsize);
+#endif
 		break;
 	}
 
@@ -324,6 +312,9 @@ rusage_check_overmap(size_t len, int pgshift)
 {
 	int npages = 0, remain_pages = 0;
 
+	if (sysctl_overcommit_memory == OVERCOMMIT_ALWAYS)
+		return 0;
+
 	npages = (len + (1UL << pgshift) - 1) >> pgshift;
 	remain_pages = (rusage.total_memory - rusage.total_memory_usage)
 			>> pgshift;

kernel/include/syscall.h

@@ -287,7 +287,7 @@ struct ikc_scd_packet {
 
 		/* SCD_MSG_CPU_RW_REG */
 		struct {
-			struct ihk_os_cpu_register desc;
+			unsigned long pdesc; /* Physical addr of the descriptor */
 			enum mcctrl_os_cpu_operation op;
 			void *resp;
 		};
@@ -508,7 +508,7 @@ enum set_cputime_mode {
 void set_cputime(enum set_cputime_mode mode);
 int do_munmap(void *addr, size_t len, int holding_memory_range_lock);
 intptr_t do_mmap(uintptr_t addr0, size_t len0, int prot, int flags, int fd,
-		off_t off0, const int vrf0, void *private_data);
+		off_t off0);
 void clear_host_pte(uintptr_t addr, size_t len, int holding_memory_range_lock);
 typedef int32_t key_t;
 int do_shmget(key_t key, size_t size, int shmflg);
@@ -651,4 +651,6 @@ extern int (*linux_clock_gettime)(clockid_t clk_id, struct timespec *tp);
 #define COREDUMP_TO_BE_WOKEN      2
 
 extern void terminate_host(int pid, struct thread *thread);
+struct sig_pending *getsigpending(struct thread *thread, int delflag);
+int interrupt_from_user(void *regs0);
 #endif