release: 1.7.8: fix ihklib/ihk_reserve_cpu when using krm

Change-Id: I57235d51f51ae7327cb08a9e3ae56be995157100

.gitignore

@@ -1,3 +1,4 @@
+*~
 *.o
 *.elf
 *.bin
@@ -8,9 +9,36 @@
 Module.symvers
 *.order
 .tmp_versions
-
-elfboot/elfboot
-elfboot/elfboot_test
-linux/executer/mcexec
-linux/mod_test*
-linux/target
+old_timestamp
+CMakeFiles
+CMakeCache.txt
+Makefile
+!test/*/*/Makefile
+!test/signalonfork+wait/Makefile
+!test/perf_overflow/Makefile
+!test/*/*/*.cmd
+Kbuild
+cmake_install.cmake
+config.h
+mcstop+release.sh
+mcreboot.sh
+mcreboot.1
+mcoverlay-destroy.sh
+mcoverlay-create.sh
+kernel/mckernel.img
+kernel/include/swapfmt.h
+executer/user/vmcore2mckdump
+executer/user/ql_talker
+executer/user/mcexec.1
+executer/user/mcexec
+executer/user/libsched_yield.so.1.0.0
+executer/user/libsched_yield.so
+executer/user/libmcexec.a
+executer/user/libldump2mcdump.so
+executer/user/eclair
+tools/mcstat/mcstat
+/_CPack_Packages
+/CPackSourceConfig.cmake
+CPackConfig.cmake
+/build
+mckernel-*.tar.gz

.gitmodules

@@ -0,0 +1,12 @@
+[submodule "ihk"]
+	path = ihk
+	url = https://github.com/RIKEN-SysSoft/ihk.git
+[submodule "executer/user/lib/libdwarf/libdwarf"]
+	path = executer/user/lib/libdwarf/libdwarf
+	url = https://github.com/bgerofi/libdwarf.git
+[submodule "executer/user/lib/syscall_intercept"]
+	path = executer/user/lib/syscall_intercept
+	url = https://github.com/RIKEN-SysSoft/syscall_intercept.git
+[submodule "executer/user/lib/uti"]
+	path = executer/user/lib/uti
+	url = https://github.com/RIKEN-SysSoft/uti.git

CMakeLists.txt

@@ -0,0 +1,352 @@
+cmake_minimum_required(VERSION 3.11)
+
+if (NOT CMAKE_BUILD_TYPE)
+	set (CMAKE_BUILD_TYPE "Debug" CACHE STRING "Build type: Debug Release..." FORCE)
+endif (NOT CMAKE_BUILD_TYPE)
+
+enable_language(C ASM)
+
+project(mckernel C ASM)
+set(MCKERNEL_VERSION "1.7.8")
+
+# See "Fedora Packaging Guidelines -- Versioning"
+set(MCKERNEL_RELEASE "")
+
+set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake/modules)
+# for rpmbuild
+if(DEFINED SYSCONF_INSTALL_DIR)
+	set(CMAKE_INSTALL_SYSCONFDIR "${SYSCONF_INSTALL_DIR}")
+endif()
+
+if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+	set(BUILD_TARGET "smp-x86" CACHE STRING "Build target: smp-x86 | smp-arm64")
+elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64")
+	set(BUILD_TARGET "smp-arm64" CACHE STRING "Build target: smp-x86 | smp-arm64")
+endif()
+
+if (BUILD_TARGET STREQUAL "smp-x86")
+	set(ARCH "x86_64")
+elseif (BUILD_TARGET STREQUAL "smp-arm64")
+	set(ARCH "arm64")
+endif()
+
+include(GNUInstallDirs)
+include(CMakeParseArguments)
+include(Kbuild)
+include(CheckCCompilerFlag)
+include(AutoconfHelper)
+
+CHECK_C_COMPILER_FLAG(-Wno-implicit-fallthrough IMPLICIT_FALLTHROUGH)
+if(IMPLICIT_FALLTHROUGH)
+	set(EXTRA_WARNINGS "-Wno-implicit-fallthrough")
+endif(IMPLICIT_FALLTHROUGH)
+
+CHECK_C_COMPILER_FLAG(-Wno-stringop-truncation STRINGOP_TRUNCATION)
+if(STRINGOP_TRUNCATION)
+	list(APPEND EXTRA_WARNINGS "-Wno-stringop-truncation")
+endif(STRINGOP_TRUNCATION)
+
+# build options
+set(CFLAGS_WARNING "-Wall" "-Wextra" "-Wno-unused-parameter" "-Wno-sign-compare" "-Wno-unused-function" ${EXTRA_WARNINGS} CACHE STRING "Warning flags")
+add_compile_options(${CFLAGS_WARNING})
+
+option(ENABLE_WERROR "Enable -Werror" OFF)
+if (ENABLE_WERROR)
+	add_compile_options("-Werror")
+endif(ENABLE_WERROR)
+
+execute_process(COMMAND bash -c "ls -ld /proc/tofu/ 2>/dev/null | wc -l"
+  OUTPUT_VARIABLE PROC_TOFU OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+if(PROC_TOFU STREQUAL "1")
+	option(ENABLE_TOFU "Built-in tofu driver support" ON)
+else()
+	option(ENABLE_TOFU "Built-in tofu driver support" OFF)
+endif()
+
+if(ENABLE_TOFU)
+	add_definitions(-DENABLE_TOFU)
+	set(KBUILD_C_FLAGS "${KBUILD_C_FLAGS} -DENABLE_TOFU")
+endif()
+
+# when compiling on a compute-node
+execute_process(COMMAND bash -c "grep $(hostname) /etc/opt/FJSVfefs/config/fefs_node1.csv 2>/dev/null | cut -d, -f2 | grep -o CN"
+  OUTPUT_VARIABLE FUGAKU_NODE_TYPE OUTPUT_STRIP_TRAILING_WHITESPACE)
+if(FUGAKU_NODE_TYPE STREQUAL "CN")
+	option(ENABLE_FUGAKU_HACKS "Fugaku hacks" ON)
+else()
+	option(ENABLE_FUGAKU_HACKS "Fugaku hacks" OFF)
+endif()
+
+if(ENABLE_FUGAKU_HACKS)
+	add_definitions(-DENABLE_FUGAKU_HACKS)
+	set(KBUILD_C_FLAGS "${KBUILD_C_FLAGS} -DENABLE_FUGAKU_HACKS")
+endif()
+
+# Fujitsu MPI tries to xpmem-attach segment with size of range size + 1?
+set(FJMPI_VERSION_COMMAND "a=\$(which mpifcc); b=\${a%/*/*}; c=\${b##*/}; d=\${c#*-}; echo \$d")
+execute_process(COMMAND bash -c "${FJMPI_VERSION_COMMAND}"
+  OUTPUT_VARIABLE FJMPI_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE)
+message("FJMPI_VERSION: ${FJMPI_VERSION}")
+
+if(NOT "${FJMPI_VERSION}" STREQUAL "")
+	option(ENABLE_FJMPI_WORKAROUND "Fujitsu MPI workaround" ON)
+else()
+	option(ENABLE_FJMPI_WORKAROUND "Fujitsu MPI workaround" OFF)
+endif()
+
+if(ENABLE_FJMPI_WORKAROUND)
+	add_definitions(-DENABLE_FJMPI_WORKAROUND)
+	set(KBUILD_C_FLAGS "${KBUILD_C_FLAGS} -DENABLE_FJMPI_WORKAROUND")
+endif()
+
+# krm that mandates reserved memory amount >= available at boot time?
+execute_process(COMMAND bash -c "rpm -qi FJSVpxkrm-plugin-mckernel | awk '$1 == \"Version\" && $2 == \":\" { print $3 }'"
+  OUTPUT_VARIABLE KRM_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE)
+message("KRM_VERSION: ${KRM_VERSION}")
+
+if(NOT "${KRM_VERSION}" STREQUAL "" AND "${KRM_VERSION}" VERSION_LESS_EQUAL 4.0.1)
+	option(ENABLE_KRM_WORKAROUND "krm workaround" ON)
+else()
+	option(ENABLE_KRM_WORKAROUND "krm workaround" OFF)
+endif()
+
+if(ENABLE_KRM_WORKAROUND)
+	add_definitions(-DENABLE_KRM_WORKAROUND)
+	set(KBUILD_C_FLAGS "${KBUILD_C_FLAGS} -DENABLE_KRM_WORKAROUND")
+endif()
+
+# SIGSTOP instead of SIGSEGV, additional IHK Linux kmsg
+option(ENABLE_FUGAKU_DEBUG "Fugaku debug instrumentation" OFF)
+
+if(ENABLE_FUGAKU_DEBUG)
+	add_definitions(-DENABLE_FUGAKU_DEBUG)
+	set(KBUILD_C_FLAGS "${KBUILD_C_FLAGS} -DENABLE_FUGAKU_DEBUG")
+endif()
+
+# redirect kernel messages to Linux's /dev/kmsg
+option(ENABLE_KMSG_REDIRECT "Redirect kernel message to Linux's /dev/kmsg" OFF)
+
+if(ENABLE_KMSG_REDIRECT)
+	add_definitions(-DENABLE_KMSG_REDIRECT)
+	set(KBUILD_C_FLAGS "${KBUILD_C_FLAGS} -DENABLE_KMSG_REDIRECT")
+endif()
+
+option(PROFILE_ENABLE "System call profile" ON)
+if(PROFILE_ENABLE)
+	add_definitions(-DPROFILE_ENABLE)
+	set(KBUILD_C_FLAGS "${KBUILD_C_FLAGS} -DPROFILE_ENABLE")
+endif()
+
+option(ENABLE_LINUX_WORK_IRQ_FOR_IKC "Use Linux work IRQ for IKC IPI" ON)
+if (ENABLE_LINUX_WORK_IRQ_FOR_IKC)
+	set(KBUILD_C_FLAGS "${KBUILD_C_FLAGS} -DIHK_IKC_USE_LINUX_WORK_IRQ")
+	add_definitions(-DIHK_IKC_USE_LINUX_WORK_IRQ)
+endif()
+
+if (BUILD_TARGET STREQUAL "smp-arm64")
+	set(POSTK_DEBUG_DEFINES "-DPOSTK_DEBUG_ARCH_DEP_100 -DPOSTK_DEBUG_ARCH_DEP_42 -DPOSTK_DEBUG_TEMP_FIX_29 -DPOSTK_DEBUG_TEMP_FIX_49")
+	add_definitions("${POSTK_DEBUG_DEFINES}")
+	set(KBUILD_C_FLAGS "${KBUILD_C_FLAGS} ${POSTK_DEBUG_DEFINES}")
+
+	execute_process(COMMAND awk -F= "$1 == \"CONFIG_ARM64_64K_PAGES\" { print $2; exit; }" "${KERNEL_DIR}/.config"
+			OUTPUT_VARIABLE CONFIG_ARM64_64K_PAGES OUTPUT_STRIP_TRAILING_WHITESPACE)
+	execute_process(COMMAND awk -F= "$1 == \"CONFIG_ARM64_VA_BITS\" { print $2; exit; }" "${KERNEL_DIR}/.config"
+			OUTPUT_VARIABLE CONFIG_ARM64_VA_BITS OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+	message("Host kernel CONFIG_ARM64_64K_PAGES=${CONFIG_ARM64_64K_PAGES}")
+	message("Host kernel CONFIG_ARM64_VA_BITS=${CONFIG_ARM64_VA_BITS}")
+
+	if(CONFIG_ARM64_64K_PAGES STREQUAL "y")
+		if(CONFIG_ARM64_VA_BITS STREQUAL 42)
+			add_definitions(-DCONFIG_ARM64_PGTABLE_LEVELS=2 -DCONFIG_ARM64_VA_BITS=42 -DCONFIG_ARM64_64K_PAGES)
+			set(LINKER_SCRIPT "smp-arm64_type3.lds")
+		elseif(CONFIG_ARM64_VA_BITS STREQUAL 48)
+			add_definitions(-DCONFIG_ARM64_PGTABLE_LEVELS=3 -DCONFIG_ARM64_VA_BITS=48 -DCONFIG_ARM64_64K_PAGES)
+			set(LINKER_SCRIPT "smp-arm64_type4.lds")
+		endif()
+	else(CONFIG_ARM64_64K_PAGES STREQUAL "y")
+		if(CONFIG_ARM64_VA_BITS STREQUAL 39)
+			add_definitions(-DCONFIG_ARM64_PGTABLE_LEVELS=3 -DCONFIG_ARM64_VA_BITS=39)
+			set(LINKER_SCRIPT "smp-arm64_type1.lds")
+		elseif(CONFIG_ARM64_VA_BITS STREQUAL 48)
+			add_definitions(-DCONFIG_ARM64_PGTABLE_LEVELS=4 -DCONFIG_ARM64_VA_BITS=48)
+			set(LINKER_SCRIPT "smp-arm64_type2.lds")
+		endif()
+	endif(CONFIG_ARM64_64K_PAGES STREQUAL "y")
+endif()
+set_property(CACHE BUILD_TARGET PROPERTY STRINGS smp-x86 smp-arm64)
+
+# define MAP_KERNEL_START
+
+set(tmpdir ${CMAKE_CURRENT_BINARY_DIR}/tmp.resolve_MODULES_END)
+file(REMOVE_RECURSE ${tmpdir})
+file(MAKE_DIRECTORY ${tmpdir})
+file(WRITE ${tmpdir}/driver.c "#include <linux/module.h>\n")
+file(APPEND ${tmpdir}/driver.c "unsigned long MAP_KERNEL_START = MODULES_END - (1UL << 23);\n")
+file(WRITE ${tmpdir}/Makefile "obj-m := driver.o\n")
+file(APPEND ${tmpdir}/Makefile "all:\n")
+file(APPEND ${tmpdir}/Makefile "\tmake ${KBUILD_MAKE_FLAGS_STR} -C ${KERNEL_DIR} M=${tmpdir} modules\n")
+
+execute_process(COMMAND make -C ${tmpdir})
+execute_process(COMMAND bash -c "offset=`readelf -S ${tmpdir}/driver.ko | grep .data | sed 's/.* //g'`; echo $((0x$offset))"
+  OUTPUT_VARIABLE MAP_KERNEL_START_OFFSET OUTPUT_STRIP_TRAILING_WHITESPACE)
+execute_process(COMMAND bash -c "dd if=${tmpdir}/driver.ko bs=1 skip=${MAP_KERNEL_START_OFFSET} count=8 2>/dev/null | od -tx8 -Ax | head -1 | sed 's|.* |0x|g'"
+  OUTPUT_VARIABLE MAP_KERNEL_START OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+
+set(ENABLE_MEMDUMP ON)
+option(ENABLE_PERF "Enable perf support" ON)
+option(ENABLE_RUSAGE "Enable rusage support" ON)
+option(ENABLE_QLMPI "Enable qlmpi programs" OFF)
+option(ENABLE_UTI "Enable uti support" OFF)
+option(ENABLE_UBSAN "Enable undefined behaviour sanitizer on mckernel size" OFF)
+option(ENABLE_PER_CPU_ALLOC_CACHE "Enable per-CPU allocator cache (ThunderX2 workaround)" OFF)
+
+find_package(PkgConfig REQUIRED)
+set(PKG_CONFIG_USE_CMAKE_PREFIX_PATH ON)
+
+find_library(LIBRT rt)
+if (NOT LIBRT)
+	message(FATAL_ERROR "error: couldn't find librt")
+endif()
+find_library(LIBNUMA numa)
+if (NOT LIBNUMA)
+	message(FATAL_ERROR "error: couldn't find libnuma")
+endif()
+find_library(LIBBFD bfd)
+if (NOT LIBBFD)
+	message(FATAL_ERROR "error: couldn't find libbfd")
+endif()
+find_library(LIBIBERTY iberty)
+if (NOT LIBIBERTY)
+	message(FATAL_ERROR "error: couldn't find libiberty")
+endif()
+
+find_library(LIBDWARF dwarf)
+
+if (NOT LIBDWARF)
+	if (CMAKE_CROSSCOMPILING)
+		message(FATAL_ERROR "Could not find libdwarf.so, install libdwarf-devel to ${CMAKE_FIND_ROOT_PATH}")
+	endif()
+	message("WARNING: libdwarf will be compiled locally")
+	enable_language(CXX)
+else()
+	# Note that libdwarf-devel provides /usr/include/libdwarf/dwarf.h
+	# but elfutils-devel provides /usr/include/dwarf.h
+	# while mcinspect.c performs "#include <dwarf.h>"
+	find_path(DWARF_H dwarf.h PATH_SUFFIXES libdwarf)
+endif()
+
+if (ENABLE_QLMPI)
+	find_package(MPI REQUIRED)
+endif()
+
+string(REGEX REPLACE "^([0-9]+)\\.([0-9]+)\\.([0-9]+)(-([0-9]+)(.*))?" "\\1;\\2;\\3;\\5;\\6" LINUX_VERSION ${UNAME_R})
+list(GET LINUX_VERSION 0 LINUX_VERSION_MAJOR)
+list(GET LINUX_VERSION 1 LINUX_VERSION_MINOR)
+list(GET LINUX_VERSION 2 LINUX_VERSION_PATCH)
+list(GET LINUX_VERSION 3 LINUX_VERSION_RELEASE)
+math(EXPR LINUX_VERSION_CODE "${LINUX_VERSION_MAJOR} * 65536 + ${LINUX_VERSION_MINOR} * 256 + ${LINUX_VERSION_PATCH}")
+
+# compat with various install paths
+set(BINDIR ${CMAKE_INSTALL_FULL_BINDIR})
+set(SBINDIR ${CMAKE_INSTALL_FULL_SBINDIR})
+set(LIBDIR ${CMAKE_INSTALL_FULL_LIBDIR})
+set(ETCDIR ${CMAKE_INSTALL_PREFIX}/etc)
+set(ROOTFSDIR "/rootfs")
+if (CMAKE_INSTALL_PREFIX STREQUAL "/usr")
+	set(KMODDIR "/lib/modules/${UNAME_R}/extra/mckernel")
+	set(MCKERNELDIR "${CMAKE_INSTALL_FULL_DATADIR}/mckernel/${BUILD_TARGET}")
+else()
+	set(KMODDIR "${CMAKE_INSTALL_PREFIX}/kmod")
+	set(MCKERNELDIR "${CMAKE_INSTALL_PREFIX}/${BUILD_TARGET}/kernel")
+endif()
+set(prefix ${CMAKE_INSTALL_PREFIX})
+
+# set rpath for everyone
+set(CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_FULL_LIBDIR})
+
+# ihk: ultimately should support extrnal build, but add as subproject for now
+if (EXISTS ${PROJECT_SOURCE_DIR}/ihk/CMakeLists.txt)
+	set(IHK_SOURCE_DIR "ihk" CACHE STRINGS "path to ihk source directory from mckernel sources")
+elseif (EXISTS ${PROJECT_SOURCE_DIR}/../ihk/CMakeLists.txt)
+	set(IHK_SOURCE_DIR "../ihk" CACHE STRINGS "path to ihk source directory from mckernel sources")
+else()
+	set(IHK_SOURCE_DIR "ihk" CACHE STRINGS "path to ihk source directory from mckernel sources")
+endif()
+if (EXISTS ${PROJECT_SOURCE_DIR}/${IHK_SOURCE_DIR}/CMakeLists.txt)
+	set(IHK_FULL_SOURCE_DIR ${PROJECT_SOURCE_DIR}/${IHK_SOURCE_DIR})
+elseif (EXISTS /${IHK_SOURCE_DIR}/CMakeLists.txt)
+	set(IHK_FULL_SOURCE_DIR /${IHK_SOURCE_DIR})
+else()
+	message(FATAL_ERROR "Could not find ihk dir, or it does not contain CMakeLists.txt, either clone ihk or run git submodule update --init")
+endif()
+
+add_subdirectory(${IHK_SOURCE_DIR} ihk)
+
+configure_file(config.h.in config.h)
+
+# actual build section - just subdirs
+add_subdirectory(executer/kernel/mcctrl)
+add_subdirectory(executer/user)
+add_subdirectory(kernel)
+add_subdirectory(tools/mcstat)
+add_subdirectory(tools/crash)
+
+configure_file(scripts/mcreboot-smp.sh.in mcreboot.sh @ONLY)
+configure_file(scripts/mcstop+release-smp.sh.in mcstop+release.sh @ONLY)
+configure_file(scripts/mcreboot.1in mcreboot.1 @ONLY)
+configure_file(scripts/eclair-dump-backtrace.exp.in eclair-dump-backtrace.exp @ONLY)
+install(PROGRAMS
+		"${CMAKE_CURRENT_BINARY_DIR}/mcreboot.sh"
+		"${CMAKE_CURRENT_BINARY_DIR}/mcstop+release.sh"
+	DESTINATION "${CMAKE_INSTALL_SBINDIR}")
+install(PROGRAMS
+		"${CMAKE_CURRENT_BINARY_DIR}/eclair-dump-backtrace.exp"
+	DESTINATION "${CMAKE_INSTALL_BINDIR}")
+install(FILES "scripts/irqbalance_mck.in"
+	DESTINATION "${CMAKE_INSTALL_SYSCONFDIR}")
+install(FILES "${CMAKE_CURRENT_BINARY_DIR}/mcreboot.1"
+	DESTINATION "${CMAKE_INSTALL_MANDIR}/man1")
+
+
+configure_file(scripts/mckernel.spec.in scripts/mckernel.spec @ONLY)
+set(CPACK_SOURCE_PACKAGE_FILE_NAME "${CMAKE_PROJECT_NAME}-${MCKERNEL_VERSION}")
+set(CPACK_SOURCE_IGNORE_FILES "/.git/;/build;/CMakeCache.txt$;/CMakeFiles$;/Makefile$")
+set(CPACK_SOURCE_INSTALLED_DIRECTORIES "${CMAKE_SOURCE_DIR};/;${IHK_FULL_SOURCE_DIR};/ihk;${CMAKE_BINARY_DIR}/scripts;/scripts")
+set(CPACK_SOURCE_GENERATOR "TGZ")
+include(CPack)
+add_custom_target(dist COMMAND ${CMAKE_MAKE_PROGRAM} package_source)
+
+
+# config report
+message("-------------------------------")
+message("Option summary")
+message("-------------------------------")
+message("Build type: ${CMAKE_BUILD_TYPE}")
+message("Build target: ${BUILD_TARGET}")
+message("IHK_SOURCE_DIR: ${IHK_SOURCE_DIR} (relative to mckernel source tree)")
+message("UNAME_R: ${UNAME_R}")
+message("KERNEL_DIR: ${KERNEL_DIR}")
+message("SYSTEM_MAP: ${SYSTEM_MAP}")
+message("VMLINUX: ${VMLINUX}")
+message("KBUILD_C_FLAGS: ${KBUILD_C_FLAGS}")
+message("MAP_KERNEL_START: ${MAP_KERNEL_START}")
+message("ENABLE_MEMDUMP: ${ENABLE_MEMDUMP}")
+message("ENABLE_PERF: ${ENABLE_PERF}")
+message("ENABLE_TOFU: ${ENABLE_TOFU}")
+message("ENABLE_FUGAKU_HACKS: ${ENABLE_FUGAKU_HACKS}")
+message("ENABLE_FUGAKU_DEBUG: ${ENABLE_FUGAKU_DEBUG}")
+message("ENABLE_KRM_WORKAROUND: ${ENABLE_KRM_WORKAROUND}")
+message("ENABLE_FJMPI_WORKAROUND: ${ENABLE_FJMPI_WORKAROUND}")
+message("PROFILE_ENABLE: ${PROFILE_ENABLE}")
+message("ENABLE_RUSAGE: ${ENABLE_RUSAGE}")
+message("ENABLE_QLMPI: ${ENABLE_QLMPI}")
+message("ENABLE_UTI: ${ENABLE_UTI}")
+message("ENABLE_WERROR: ${ENABLE_WERROR}")
+message("ENABLE_UBSAN: ${ENABLE_UBSAN}")
+message("ENABLE_LINUX_WORK_IRQ_FOR_IKC: ${ENABLE_LINUX_WORK_IRQ_FOR_IKC}")
+message("ENABLE_PER_CPU_ALLOC_CACHE: ${ENABLE_PER_CPU_ALLOC_CACHE}")
+message("ENABLE_KMSG_REDIRECT: ${ENABLE_KMSG_REDIRECT}")
+message("-------------------------------")

KNOWN_BUGS.md

@@ -0,0 +1,70 @@
+Linux crash when offlining CPU (el7, hardware-specific)
+=========================================================
+
+On some hardware with el7 kernel, linux can crash due to a bug in the
+irq handling when offlining CPUs (reserve cpu part of mcreboot)
+
+Example stack trace:
+```
+[ 4147.052753] BUG: unable to handle kernel NULL pointer dereference at 0000000000000040
+[ 4147.060677] IP: [<ffffffff8102ce26>] check_irq_vectors_for_cpu_disable+0x86/0x1c0
+[ 4147.068226] PGD 1057e44067 PUD 105f1e7067 PMD 0
+[ 4147.072935] Oops: 0000 [#1] SMP
+[ 4147.076230] Modules linked in: mcctrl(OE) ihk_smp_x86_64(OE) ihk(OE) xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core
+[ 4147.148619]  dm_mirror dm_region_hash dm_log dm_mod sb_edac edac_core intel_powerclamp coretemp ext4 mbcache jbd2 intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul ipmi_ssif glue_helper ablk_helper joydev iTCO_wdt iTCO_vendor_support cryptd ipmi_si ipmi_devintf ipmi_msghandler pcspkr wmi mei_me mei lpc_ich i2c_i801 sg ioatdma shpchp nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c mlx4_en sd_mod crc_t10dif crct10dif_generic mgag200 drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm isci igb drm mlx4_core libsas ahci libahci scsi_transport_sas libata crct10dif_pclmul ptp crct10dif_common pps_core crc32c_intel dca i2c_algo_bit i2c_core devlink [last unloaded: ihk]
+[ 4147.215370] CPU: 6 PID: 38 Comm: migration/6 Tainted: G           OE  ------------ T 3.10.0-693.2.2.el7.x86_64 #1
+[ 4147.225672] Hardware name: SGI.COM C1104G-RP5/X9DRG-HF, BIOS 3.0  10/25/2013
+[ 4147.232747] task: ffff880174689fa0 ti: ffff8801746ac000 task.ti: ffff8801746ac000
+[ 4147.240278] RIP: 0010:[<ffffffff8102ce26>]  [<ffffffff8102ce26>] check_irq_vectors_for_cpu_disable+0x86/0x1c0
+[ 4147.250275] RSP: 0018:ffff8801746afd30  EFLAGS: 00010046
+[ 4147.255608] RAX: 0000000000000000 RBX: 000000000000004e RCX: 0000000000000000
+[ 4147.262770] RDX: 0000000000000020 RSI: 000000000000005f RDI: 0000000000000023
+[ 4147.269936] RBP: ffff8801746afd58 R08: 0000000000000001 R09: ffff88017f800490
+[ 4147.277103] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000006
+[ 4147.284269] R13: 0000000000000000 R14: ffff88085ca82500 R15: 000000000000005f
+[ 4147.291429] FS:  0000000000000000(0000) GS:ffff88085fb80000(0000) knlGS:0000000000000000
+[ 4147.299556] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
+[ 4147.305326] CR2: 0000000000000040 CR3: 0000001059704000 CR4: 00000000001407e0
+[ 4147.312490] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
+[ 4147.319659] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
+[ 4147.326827] Stack:
+[ 4147.328857]  ffff8808f43078c8 ffff8808f4307850 0000000000000286 ffff8808f4307701
+[ 4147.336384]  0000000000000000 ffff8801746afd70 ffffffff81052a82 0000000200000000
+[ 4147.343915]  ffff8801746afd88 ffffffff81693ca3 0000000000000003 ffff8801746afdc0
+[ 4147.351447] Call Trace:
+[ 4147.353921]  [<ffffffff81052a82>] native_cpu_disable+0x12/0x40
+[ 4147.359795]  [<ffffffff81693ca3>] take_cpu_down+0x13/0x40
+[ 4147.365236]  [<ffffffff81116899>] multi_cpu_stop+0xd9/0x100
+[ 4147.370850]  [<ffffffff811167c0>] ? cpu_stop_should_run+0x50/0x50
+[ 4147.376983]  [<ffffffff81116ab7>] cpu_stopper_thread+0x97/0x150
+[ 4147.382942]  [<ffffffff816a8fad>] ? __schedule+0x39d/0x8b0
+[ 4147.388461]  [<ffffffff810b909f>] smpboot_thread_fn+0x12f/0x180
+[ 4147.394406]  [<ffffffff810b8f70>] ? lg_double_unlock+0x40/0x40
+[ 4147.400276]  [<ffffffff810b098f>] kthread+0xcf/0xe0
+[ 4147.405182]  [<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
+[ 4147.411319]  [<ffffffff816b4f58>] ret_from_fork+0x58/0x90
+[ 4147.418893]  [<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
+[ 4147.426524] Code: 81 fb 00 01 00 00 0f 84 8a 00 00 00 89 d8 65 44 8b 3c 85 20 c6 00 00 45 85 ff 78 e1 44 89 ff e8 91 31 10 00 48 63 15 7e 10 af 00 <48> 8b 70 40 48 c7 c7 80 71 cf 81 49 89 c6 48 83 c2 3f 48 c1 fa
+[ 4147.450352] RIP  [<ffffffff8102ce26>] check_irq_vectors_for_cpu_disable+0x86/0x1c0
+[ 4147.460135]  RSP <ffff8801746afd30>
+[ 4147.465154] CR2: 0000000000000040
+```
+
+This bug has been fixed upstream, but redhat will not backport the fixes.
+You can work around the problem with a kpatch by backporting the three
+following commits:
+
+x86: irq: Get correct available vectors for cpu disable
+https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=ac2a55395eddccd6e3e39532df9869d61e97b2ee
+
+x86/irq: Check for valid irq descriptor in check_irq_vectors_for_cpu_disable()
+https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=d97eb8966c91f2c9d05f0a22eb89ed5b76d966d1
+
+x86/irq: Use proper locking in check_irq_vectors_for_cpu_disable()
+https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=cbb24dc761d95fe39a7a122bb1b298e9604cae15
+
+
+Alternatively, since it is related to the irq configuration, it might
+be possible to mitigate the issue by setting the irq affinities early
+on and making sure none of the cpus that will be offlined have any irq
+configured.

LICENSE

@@ -0,0 +1,339 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+                            NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.

Makefile.in

@@ -1,76 +0,0 @@
-TARGET = @TARGET@
-SBINDIR = @SBINDIR@
-INCDIR =  @INCDIR@
-ETCDIR = @ETCDIR@
-MANDIR = @MANDIR@
-
-all: executer-mcctrl executer-mcoverlayfs executer-user mckernel
-
-executer-mcctrl:
-	+@(cd executer/kernel/mcctrl; $(MAKE) modules)
-
-executer-mcoverlayfs:
-	+@(cd executer/kernel/mcoverlayfs; $(MAKE) modules)
-
-executer-user:
-	+@(cd executer/user; $(MAKE))
-
-mckernel:
-	+@case "$(TARGET)" in \
-	    attached-mic | builtin-x86 | builtin-mic | smp-x86 | smp-arm64) \
-		(cd kernel; $(MAKE)) \
-		;; \
-	    *) \
-		echo "unknown target $(TARGET)" >&2 \
-		exit 1 \
-		;; \
-	esac
-
-install:
-	@(cd executer/kernel/mcctrl; $(MAKE) install)
-	@(cd executer/kernel/mcoverlayfs; $(MAKE) install)
-	@(cd executer/user; $(MAKE) install)
-	@case "$(TARGET)" in \
-	    attached-mic | builtin-x86 | builtin-mic | smp-x86 | smp-arm64) \
-		(cd kernel; $(MAKE) install) \
-		;; \
-	    *) \
-		echo "unknown target $(TARGET)" >&2 \
-		exit 1 \
-		;; \
-	esac
-	@case "$(TARGET)" in \
-		smp-x86 | smp-arm64) \
-		mkdir -p -m 755 $(SBINDIR); \
-		install -m 755 arch/x86_64/tools/mcreboot-smp-x86.sh $(SBINDIR)/mcreboot.sh; \
-		install -m 755 arch/x86_64/tools/mcstop+release-smp-x86.sh $(SBINDIR)/mcstop+release.sh; \
-		install -m 755 arch/x86_64/tools/mcoverlay-destroy-smp-x86.sh $(SBINDIR)/mcoverlay-destroy.sh; \
-		install -m 755 arch/x86_64/tools/mcoverlay-create-smp-x86.sh $(SBINDIR)/mcoverlay-create.sh; \
-		install -m 755 arch/x86_64/tools/eclair-dump-backtrace.exp $(SBINDIR)/eclair-dump-backtrace.exp;\
-		mkdir -p -m 755 $(ETCDIR); \
-		install -m 644 arch/x86_64/tools/irqbalance_mck.service $(ETCDIR)/irqbalance_mck.service; \
-		install -m 644 arch/x86_64/tools/irqbalance_mck.in $(ETCDIR)/irqbalance_mck.in; \
-		mkdir -p -m 755 $(INCDIR); \
-		install -m 644 kernel/include/swapfmt.h $(INCDIR); \
-		mkdir -p -m 755 $(MANDIR)/man1; \
-		install -m 644 arch/x86_64/tools/mcreboot.1 $(MANDIR)/man1/mcreboot.1; \
-		;; \
-	    *) \
-		echo "unknown target $(TARGET)" >&2 \
-		exit 1 \
-		;; \
-	esac
-
-clean:
-	@(cd executer/kernel/mcctrl; $(MAKE) clean)
-	@(cd executer/kernel/mcoverlayfs; $(MAKE) clean)
-	@(cd executer/user; $(MAKE) clean)
-	@case "$(TARGET)" in \
-	    attached-mic | builtin-x86 | builtin-mic | smp-x86 | smp-arm64) \
-		(cd kernel; $(MAKE) clean) \
-		;; \
-	    *) \
-		echo "unknown target $(TARGET)" >&2 \
-		exit 1 \
-		;; \
-	esac

README.rst

@@ -0,0 +1,23 @@
+.. figure:: docs/mckernel-logo.png
+
+IHK/McKernel is a light-weight multi-kernel operating system designed
+for high-end supercomputing. It runs Linux and McKernel, a light-weight
+kernel (LWK), side-by-side inside compute nodes and aims at the
+following:
+
+-  Provide scalable and consistent execution of large-scale parallel
+   scientific applications, but at the same time maintain the ability to
+   rapidly adapt to new hardware features and emerging programming
+   models
+-  Provide efficient memory and device management so that resource
+   contention and data movement are minimized at the system level
+-  Eliminate OS noise by isolating OS services in Linux and provide
+   jitter free execution on the LWK
+-  Support the full POSIX/Linux APIs by selectively offloading
+   (slow-path) system calls to Linux
+
+Documentation
+=============
+
+Documentation is available
+`here <https://ihkmckernel.readthedocs.io>`__.

arch/arm64/kernel/Makefile.arch.in

@@ -1,4 +1,4 @@
-# Makefile.arch COPYRIGHT FUJITSU LIMITED 2015-2017
+# Makefile.arch.in COPYRIGHT FUJITSU LIMITED 2015-2018
 VDSO_SRCDIR = $(SRC)/../arch/$(IHKARCH)/kernel/vdso
 VDSO_BUILDDIR = @abs_builddir@/vdso
 VDSO_SO_O = $(O)/vdso.so.o
@@ -6,23 +6,22 @@ VDSO_SO_O = $(O)/vdso.so.o
 IHK_OBJS += assert.o cache.o cpu.o cputable.o context.o entry.o entry-fpsimd.o
 IHK_OBJS += fault.o head.o hyp-stub.o local.o perfctr.o perfctr_armv8pmu.o proc.o proc-macros.o
 IHK_OBJS += psci.o smp.o trampoline.o traps.o fpsimd.o
-IHK_OBJS += debug-monitors.o hw_breakpoint.o ptrace.o
+IHK_OBJS += debug-monitors.o hw_breakpoint.o ptrace.o timer.o
 IHK_OBJS += $(notdir $(VDSO_SO_O)) memory.o syscall.o vdso.o
 
 IHK_OBJS += irq-gic-v2.o irq-gic-v3.o
 IHK_OBJS += memcpy.o memset.o
 IHK_OBJS += cpufeature.o
 
-# POSTK_DEBUG_ARCH_DEP_18 coredump arch separation.
-# IHK_OBJS added coredump.o
+IHK_OBJS += imp-sysreg.o
 IHK_OBJS += coredump.o
 
 $(VDSO_SO_O): $(VDSO_BUILDDIR)/vdso.so
 
 $(VDSO_BUILDDIR)/vdso.so: FORCE
 	$(call echo_cmd,BUILD VDSO,$(TARGET))
-	@mkdir -p $(O)/vdso
-	@TARGETDIR="$(TARGETDIR)" $(submake) -C $(VDSO_BUILDDIR) $(SUBOPTS) prepare
-	@TARGETDIR="$(TARGETDIR)" $(submake) -C $(VDSO_BUILDDIR) $(SUBOPTS)
+	mkdir -p $(O)/vdso
+	TARGETDIR="$(TARGETDIR)" $(submake) -C $(VDSO_BUILDDIR) $(SUBOPTS) prepare
+	TARGETDIR="$(TARGETDIR)" $(submake) -C $(VDSO_BUILDDIR) $(SUBOPTS)
 
 FORCE:

arch/arm64/kernel/assert.c

@@ -1,4 +1,4 @@
-/* assert.c COPYRIGHT FUJITSU LIMITED 2015-2017 */
+/* assert.c COPYRIGHT FUJITSU LIMITED 2015-2019 */
 
 #include <process.h>
 #include <list.h>
@@ -24,6 +24,7 @@ STATIC_ASSERT(offsetof(struct pt_regs, sp) == S_SP);
 STATIC_ASSERT(offsetof(struct pt_regs, pc) == S_PC);
 STATIC_ASSERT(offsetof(struct pt_regs, pstate) == S_PSTATE);
 STATIC_ASSERT(offsetof(struct pt_regs, orig_x0) == S_ORIG_X0);
+STATIC_ASSERT(offsetof(struct pt_regs, orig_pc) == S_ORIG_PC);
 STATIC_ASSERT(offsetof(struct pt_regs, syscallno) == S_SYSCALLNO);
 STATIC_ASSERT(sizeof(struct pt_regs) == S_FRAME_SIZE);
 
@@ -50,3 +51,6 @@ STATIC_ASSERT(sizeof(struct sigcontext) - offsetof(struct sigcontext, __reserved
 		ALIGN_UP(sizeof(struct _aarch64_ctx), 16) > sizeof(struct extra_context));
 STATIC_ASSERT(SVE_PT_FPSIMD_OFFSET == sizeof(struct user_sve_header));
 STATIC_ASSERT(SVE_PT_SVE_OFFSET == sizeof(struct user_sve_header));
+
+/* assert for struct arm64_cpu_local_thread member offset define */
+STATIC_ASSERT(offsetof(struct arm64_cpu_local_thread, panic_regs) == 168);

arch/arm64/kernel/coredump.c

@@ -1,10 +1,15 @@
-/* coredump.c COPYRIGHT FUJITSU LIMITED 2015-2016 */
-#ifdef POSTK_DEBUG_ARCH_DEP_18 /* coredump arch separation. */
+/* coredump.c COPYRIGHT FUJITSU LIMITED 2015-2019 */
 #include <process.h>
 #include <elfcore.h>
 #include <string.h>
+#include <ptrace.h>
+#include <cls.h>
+#include <hwcap.h>
 
-void arch_fill_prstatus(struct elf_prstatus64 *prstatus, struct thread *thread, void *regs0)
+#define	align32(x) ((((x) + 3) / 4) * 4)
+
+void arch_fill_prstatus(struct elf_prstatus64 *prstatus,
+	struct thread *thread, void *regs0, int sig)
 {
 	struct pt_regs *regs = regs0;
 	struct elf_prstatus64 tmp_prstatus;
@@ -15,8 +20,6 @@ void arch_fill_prstatus(struct elf_prstatus64 *prstatus, struct thread *thread,
 	short int pr_cursig;
 	a8_uint64_t pr_sigpend;
 	a8_uint64_t pr_sighold;
-	pid_t pr_pid;
-	pid_t pr_ppid;
 	pid_t pr_pgrp;
 	pid_t pr_sid;
 	struct prstatus64_timeval pr_utime;
@@ -24,12 +27,66 @@ void arch_fill_prstatus(struct elf_prstatus64 *prstatus, struct thread *thread,
 	struct prstatus64_timeval pr_cutime;
 	struct prstatus64_timeval pr_cstime;
  */
+
 	/* copy x0-30, sp, pc, pstate */
 	memcpy(&tmp_prstatus.pr_reg, &regs->user_regs, sizeof(tmp_prstatus.pr_reg));
 	tmp_prstatus.pr_fpvalid = 0;	/* We assume no fp */
 
 	/* copy unaligned prstatus addr */
 	memcpy(prstatus, &tmp_prstatus, sizeof(*prstatus));
+
+	prstatus->pr_pid = thread->tid;
+	if (thread->proc->parent) {
+		prstatus->pr_ppid = thread->proc->parent->pid;
+	}
+
+	prstatus->pr_info.si_signo = sig;
+	prstatus->pr_cursig = sig;
 }
 
-#endif /* POSTK_DEBUG_ARCH_DEP_18 */
+int arch_get_thread_core_info_size(void)
+{
+	const struct user_regset_view *view = current_user_regset_view();
+	const struct user_regset *regset = find_regset(view, NT_ARM_SVE);
+
+	if (unlikely(!(elf_hwcap & HWCAP_SVE))) {
+		return 0;
+	}
+	return sizeof(struct note) + align32(sizeof("LINUX"))
+		+ regset_size(cpu_local_var(current), regset);
+}
+
+void arch_fill_thread_core_info(struct note *head,
+				struct thread *thread, void *regs)
+{
+	const struct user_regset_view *view = current_user_regset_view();
+	const struct user_regset *regset = find_regset(view, NT_ARM_SVE);
+
+	if (unlikely(!(elf_hwcap & HWCAP_SVE))) {
+		return;
+	}
+
+	/* pre saved registers */
+	save_fp_regs(thread);
+
+	if (regset->core_note_type && regset->get &&
+	    (!regset->active || regset->active(thread, regset))) {
+		int ret;
+		size_t size = regset_size(thread, regset);
+		void *namep;
+		void *descp;
+
+		namep = (void *) (head + 1);
+		descp = namep + align32(sizeof("LINUX"));
+
+		ret = regset->get(thread, regset, 0, size, descp, NULL);
+		if (ret) {
+			return;
+		}
+
+		head->namesz = sizeof("LINUX");
+		head->descsz = size;
+		head->type = NT_ARM_SVE;
+		memcpy(namep, "LINUX", sizeof("LINUX"));
+	}
+}

arch/arm64/kernel/cpufeature.c

@@ -1,4 +1,4 @@
-/* cpufeature.c COPYRIGHT FUJITSU LIMITED 2017 */
+/* cpufeature.c COPYRIGHT FUJITSU LIMITED 2017-2018 */
 
 #include <cpufeature.h>
 #include <ihk/debug.h>
@@ -10,9 +10,7 @@
 #include <ptrace.h>
 #include <hwcap.h>
 
-#ifdef POSTK_DEBUG_ARCH_DEP_65
 unsigned long elf_hwcap;
-#endif /* POSTK_DEBUG_ARCH_DEP_65 */
 
 /* @ref.impl arch/arm64/kernel/cpufeature.c */
 #define __ARM64_FTR_BITS(SIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \
@@ -54,6 +52,19 @@ static const struct arm64_ftr_bits ftr_id_aa64isar0[] = {
 	ARM64_FTR_END,
 };
 
+/* @ref.impl linux4.16.0 arch/arm64/kernel/cpufeature.c */
+static const struct arm64_ftr_bits ftr_id_aa64isar1[] = {
+	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE,
+		       ID_AA64ISAR1_LRCPC_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE,
+		       ID_AA64ISAR1_FCMA_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE,
+		       ID_AA64ISAR1_JSCVT_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE,
+		       ID_AA64ISAR1_DPB_SHIFT, 4, 0),
+	ARM64_FTR_END,
+};
+
 /* @ref.impl arch/arm64/kernel/cpufeature.c */
 static const struct arm64_ftr_bits ftr_id_aa64pfr0[] = {
 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_SVE_SHIFT, 4, 0),
@@ -304,7 +315,7 @@ static const struct __ftr_reg_entry {
 
 	/* Op1 = 0, CRn = 0, CRm = 6 */
 	ARM64_FTR_REG(SYS_ID_AA64ISAR0_EL1, ftr_id_aa64isar0),
-	ARM64_FTR_REG(SYS_ID_AA64ISAR1_EL1, ftr_raz),
+	ARM64_FTR_REG(SYS_ID_AA64ISAR1_EL1, ftr_id_aa64isar1),
 
 	/* Op1 = 0, CRn = 0, CRm = 7 */
 	ARM64_FTR_REG(SYS_ID_AA64MMFR0_EL1, ftr_id_aa64mmfr0),
@@ -959,7 +970,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
 #ifdef CONFIG_ARM64_SVE
 	HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_SVE_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SVE),
 #endif
-	{},
+	{ 0 },
 };
 
 /* @ref.impl arch/arm64/kernel/cpufeature.c */
@@ -997,9 +1008,7 @@ void setup_cpu_features(void)
 	setup_elf_hwcaps(arm64_elf_hwcaps);
 }
 
-#ifdef POSTK_DEBUG_ARCH_DEP_65
 unsigned long arch_get_hwcap(void)
 {
 	return elf_hwcap;
 }
-#endif /* POSTK_DEBUG_ARCH_DEP_65 */

arch/arm64/kernel/cputable.c

@@ -10,5 +10,5 @@ struct cpu_info cpu_table[] = {
 		.cpu_name	= "AArch64 Processor",
 		.cpu_setup	= __cpu_setup,
 	},
-	{ /* Empty */ },
+	{ 0 },
 };

arch/arm64/kernel/debug-monitors.c

@@ -2,7 +2,6 @@
 #include <cputype.h>
 #include <irqflags.h>
 #include <ihk/context.h>
-#include <ihk/debug.h>
 #include <signal.h>
 #include <errno.h>
 #include <debug-monitors.h>

arch/arm64/kernel/entry.S

@@ -1,10 +1,11 @@
-/* entry.S COPYRIGHT FUJITSU LIMITED 2015-2017 */
+/* entry.S COPYRIGHT FUJITSU LIMITED 2015-2018 */
 
 #include <linkage.h>
 #include <assembler.h>
 #include <asm-offsets.h>
 #include <esr.h>
 #include <thread_info.h>
+#include <asm-syscall.h>
 
 /*
  * Bad Abort numbers
@@ -77,11 +78,12 @@
 
 	.macro	kernel_exit, el, need_enable_step = 0
 	.if 	\el == 0
+	bl	check_sig_pending
+	bl	check_need_resched		// or reschedule is needed.
 	mov	x0, #0
 	mov	x1, sp
 	mov	x2, #0
 	bl	check_signal			// check whether the signal is delivered
-	bl	check_need_resched		// or reschedule is needed.
 	mov	x0, #0
 	mov	x1, sp
 	mov	x2, #0
@@ -367,7 +369,12 @@ el0_sync:
 	b	el0_inv
 el0_svc:
 	uxtw	scno, w8			// syscall number in w8
-	stp	x0, scno, [sp, #S_ORIG_X0]	// save the original x0 and syscall number
+	cmp	scno, #__NR_rt_sigreturn
+	b.eq	1f
+	str	x0, [sp, #S_ORIG_X0]		// save the original x0
+	ldr	x16, [sp, #S_PC]
+	str	x16, [sp, #S_ORIG_PC]		// save the original pc
+1:	str	scno, [sp, #S_SYSCALLNO]	// save syscall number
 	enable_nmi
 	enable_dbg_and_irq x0
 	adrp	x16, __arm64_syscall_handler
@@ -550,9 +557,7 @@ ENTRY(ret_from_fork)
 	blr	x19
 1:	get_thread_info tsk
 	bl	release_runq_lock
+	bl	utilthr_migrate
 	b	ret_to_user
 ENDPROC(ret_from_fork)
 
-/* TODO: skeleton for rusage */
-ENTRY(__freeze)
-ENDPROC(__freeze)

arch/arm64/kernel/fault.c

@@ -1,4 +1,4 @@
-/* fault.c COPYRIGHT FUJITSU LIMITED 2015-2017 */
+/* fault.c COPYRIGHT FUJITSU LIMITED 2015-2018 */
 
 #include <ihk/context.h>
 #include <ihk/debug.h>
@@ -13,7 +13,6 @@
 unsigned long __page_fault_handler_address;
 extern int interrupt_from_user(void *);
 
-void set_signal(int sig, void *regs, struct siginfo *info);
 static void do_bad_area(unsigned long addr, unsigned int esr, struct pt_regs *regs);
 static int do_page_fault(unsigned long addr, unsigned int esr, struct pt_regs *regs);
 static int do_translation_fault(unsigned long addr, unsigned int esr, struct pt_regs *regs);
@@ -105,12 +104,13 @@ void do_mem_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs)
 {
 	const struct fault_info *inf = fault_info + (esr & 63);
 	struct siginfo info;
+	const int from_user = interrupt_from_user(regs);
 
 	/* set_cputime called in inf->fn() */
 	if (!inf->fn(addr, esr, regs))
 		return;
 
-	set_cputime(interrupt_from_user(regs)? 1: 2);
+	set_cputime(from_user ? CPUTIME_MODE_U2K : CPUTIME_MODE_K2K_IN);
 	kprintf("Unhandled fault: %s (0x%08x) at 0x%016lx\n", inf->name, esr, addr);
 	info.si_signo = inf->sig;
 	info.si_errno = 0;
@@ -118,7 +118,7 @@ void do_mem_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs)
 	info._sifields._sigfault.si_addr  = (void*)addr;
 
 	arm64_notify_die("", regs, &info, esr);
-	set_cputime(0);
+	set_cputime(from_user ? CPUTIME_MODE_K2U : CPUTIME_MODE_K2K_OUT);
 }
 
 /*
@@ -127,21 +127,24 @@ void do_mem_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs)
 void do_sp_pc_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs)
 {
 	struct siginfo info;
+	const int from_user = interrupt_from_user(regs);
 
-	set_cputime(interrupt_from_user(regs)? 1: 2);
+	set_cputime(from_user ? CPUTIME_MODE_U2K : CPUTIME_MODE_K2K_IN);
 
 	info.si_signo = SIGBUS;
 	info.si_errno = 0;
 	info.si_code  = BUS_ADRALN;
 	info._sifields._sigfault.si_addr  = (void*)addr;
 	arm64_notify_die("", regs, &info, esr);
-	set_cputime(0);
+	set_cputime(from_user ? CPUTIME_MODE_K2U : CPUTIME_MODE_K2K_OUT);
 }
 
 static void do_bad_area(unsigned long addr, unsigned int esr, struct pt_regs *regs)
 {
 	struct siginfo info;
-	set_cputime(interrupt_from_user(regs) ? 1: 2);
+	const int from_user = interrupt_from_user(regs);
+
+	set_cputime(from_user ? CPUTIME_MODE_U2K : CPUTIME_MODE_K2K_IN);
 	/*
 	 * If we are in kernel mode at this point, we have no context to
 	 * handle this fault with.
@@ -163,7 +166,7 @@ static void do_bad_area(unsigned long addr, unsigned int esr, struct pt_regs *re
 			 (addr < PAGE_SIZE) ? "NULL pointer dereference" : "paging request", addr);
 		panic("OOps.");
 	}
-	set_cputime(0);
+	set_cputime(from_user ? CPUTIME_MODE_K2U : CPUTIME_MODE_K2K_OUT);
 }
 
 static int is_el0_instruction_abort(unsigned int esr)
@@ -192,6 +195,7 @@ static int do_page_fault(unsigned long addr, unsigned int esr,
 		}
 	}
 
+	/* set_cputime() call in page_fault_handler() */
 	page_fault_handler = (void *)__page_fault_handler_address;
 	(*page_fault_handler)((void *)addr, reason, regs);
 
@@ -219,7 +223,12 @@ static int do_translation_fault(unsigned long addr,
 					  unsigned int esr,
 					  struct pt_regs *regs)
 {
+#ifdef ENABLE_TOFU
+	// XXX: Handle kernel space page faults for Tofu driver
+	//if (addr < USER_END)
+#else
 	if (addr < USER_END)
+#endif
 		return do_page_fault(addr, esr, regs);
 
 	do_bad_area(addr, esr, regs);
@@ -252,10 +261,10 @@ int do_debug_exception(unsigned long addr, unsigned int esr, struct pt_regs *reg
 {
 	const struct fault_info *inf = debug_fault_info + DBG_ESR_EVT(esr);
 	struct siginfo info;
-	int from_user = interrupt_from_user(regs);
+	const int from_user = interrupt_from_user(regs);
 	int ret = -1;
 
-	set_cputime(from_user ? 1: 2);
+	set_cputime(from_user ? CPUTIME_MODE_U2K : CPUTIME_MODE_K2K_IN);
 
 	if (!inf->fn(addr, esr, regs)) {
 		ret = 1;
@@ -274,7 +283,7 @@ int do_debug_exception(unsigned long addr, unsigned int esr, struct pt_regs *reg
 
 	ret = 0;
 out:
-	set_cputime(0);
+	set_cputime(from_user ? CPUTIME_MODE_K2U : CPUTIME_MODE_K2K_OUT);
 	return ret;
 }
 
@@ -283,7 +292,9 @@ out:
  */
 static int do_bad(unsigned long addr, unsigned int esr, struct pt_regs *regs)
 {
-	set_cputime(interrupt_from_user(regs) ? 1: 2);
-	set_cputime(0);
+	const int from_user = interrupt_from_user(regs);
+
+	set_cputime(from_user ? CPUTIME_MODE_U2K : CPUTIME_MODE_K2K_IN);
+	set_cputime(from_user ? CPUTIME_MODE_K2U : CPUTIME_MODE_K2K_OUT);
 	return 1;
 }

arch/arm64/kernel/fpsimd.c

@@ -1,4 +1,4 @@
-/* fpsimd.c COPYRIGHT FUJITSU LIMITED 2016-2017 */
+/* fpsimd.c COPYRIGHT FUJITSU LIMITED 2016-2019 */
 #include <thread_info.h>
 #include <fpsimd.h>
 #include <cpuinfo.h>
@@ -9,27 +9,100 @@
 #include <prctl.h>
 #include <cpufeature.h>
 #include <kmalloc.h>
+#include <ihk/debug.h>
+#include <process.h>
+#include <bitmap.h>
 
 //#define DEBUG_PRINT_FPSIMD
 
 #ifdef DEBUG_PRINT_FPSIMD
-#define dkprintf kprintf
-#define ekprintf kprintf
-#else
-#define dkprintf(...) do { if (0) kprintf(__VA_ARGS__); } while (0)
-#define ekprintf kprintf
+#undef DDEBUG_DEFAULT
+#define DDEBUG_DEFAULT DDEBUG_PRINT
 #endif
 
-#define BUG_ON(condition) do { if (condition) { kprintf("PANIC: %s: %s(line:%d)\n",\
-				__FILE__, __FUNCTION__, __LINE__); panic(""); } } while(0)
-
 #ifdef CONFIG_ARM64_SVE
 
+/* Set of available vector lengths, as vq_to_bit(vq): */
+static DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);
+
 /* Maximum supported vector length across all CPUs (initially poisoned) */
 int sve_max_vl = -1;
+
 /* Default VL for tasks that don't set it explicitly: */
 int sve_default_vl = -1;
 
+/*
+ * Helpers to translate bit indices in sve_vq_map to VQ values (and
+ * vice versa).  This allows find_next_bit() to be used to find the
+ * _maximum_ VQ not exceeding a certain value.
+ */
+
+static unsigned int vq_to_bit(unsigned int vq)
+{
+	return SVE_VQ_MAX - vq;
+}
+
+static unsigned int bit_to_vq(unsigned int bit)
+{
+	if (bit >= SVE_VQ_MAX) {
+		bit = SVE_VQ_MAX - 1;
+	}
+	return SVE_VQ_MAX - bit;
+}
+
+/*
+ * All vector length selection from userspace comes through here.
+ * We're on a slow path, so some sanity-checks are included.
+ * If things go wrong there's a bug somewhere, but try to fall back to a
+ * safe choice.
+ */
+static unsigned int find_supported_vector_length(unsigned int vl)
+{
+	int bit;
+	int max_vl = sve_max_vl;
+
+	if (!sve_vl_valid(vl)) {
+		vl = SVE_VL_MIN;
+	}
+
+	if (!sve_vl_valid(max_vl)) {
+		max_vl = SVE_VL_MIN;
+	}
+
+	if (vl > max_vl) {
+		vl = max_vl;
+	}
+
+	bit = find_next_bit(sve_vq_map, SVE_VQ_MAX,
+			    vq_to_bit(sve_vq_from_vl(vl)));
+	return sve_vl_from_vq(bit_to_vq(bit));
+}
+
+static void sve_probe_vqs(DECLARE_BITMAP(map, SVE_VQ_MAX))
+{
+	unsigned int vq, vl;
+	unsigned long zcr;
+
+	bitmap_zero(map, SVE_VQ_MAX);
+
+	zcr = ZCR_EL1_LEN_MASK;
+	zcr = read_sysreg_s(SYS_ZCR_EL1) & ~zcr;
+
+	for (vq = SVE_VQ_MAX; vq >= SVE_VQ_MIN; --vq) {
+		/* self-syncing */
+		write_sysreg_s(zcr | (vq - 1), SYS_ZCR_EL1);
+		vl = sve_get_vl();
+		/* skip intervening lengths */
+		vq = sve_vq_from_vl(vl);
+		set_bit(vq_to_bit(vq), map);
+	}
+}
+
+void sve_init_vq_map(void)
+{
+	sve_probe_vqs(sve_vq_map);
+}
+
 size_t sve_state_size(struct thread const *thread)
 {
 	unsigned int vl = thread->ctx.thread->sve_vl;
@@ -46,17 +119,19 @@ void sve_free(struct thread *thread)
 	}
 }
 
-void sve_alloc(struct thread *thread)
+int sve_alloc(struct thread *thread)
 {
 	if (thread->ctx.thread->sve_state) {
-		return;
+		return 0;
 	}
 
 	thread->ctx.thread->sve_state =
 		kmalloc(sve_state_size(thread), IHK_MC_AP_NOWAIT);
-	BUG_ON(!thread->ctx.thread->sve_state);
-
+	if (thread->ctx.thread->sve_state == NULL) {
+		return -ENOMEM;
+	}
 	memset(thread->ctx.thread->sve_state, 0, sve_state_size(thread));
+	return 0;
 }
 
 static int get_nr_threads(struct process *proc)
@@ -73,28 +148,13 @@ static int get_nr_threads(struct process *proc)
 	return nr_threads;
 }
 
-extern void save_fp_regs(struct thread *thread);
-extern void clear_fp_regs(struct thread *thread);
-extern void restore_fp_regs(struct thread *thread);
 /* @ref.impl arch/arm64/kernel/fpsimd.c::sve_set_vector_length */
 int sve_set_vector_length(struct thread *thread,
 			unsigned long vl, unsigned long flags)
 {
 	struct thread_info *ti = thread->ctx.thread;
 
-	BUG_ON(thread == cpu_local_var(current) && cpu_local_var(no_preempt) == 0);
-
-	/*
-	 * To avoid accidents, forbid setting for individual threads of a
-	 * multithreaded process.  User code that knows what it's doing can
-	 * pass PR_SVE_SET_VL_THREAD to override this restriction:
-	 */
-	if (!(flags & PR_SVE_SET_VL_THREAD) && get_nr_threads(thread->proc) != 1) {
-		return -EINVAL;
-	}
-	flags &= ~(unsigned long)PR_SVE_SET_VL_THREAD;
-
-	if (flags & ~(unsigned long)(PR_SVE_SET_VL_INHERIT |
+	if (flags & ~(unsigned long)(PR_SVE_VL_INHERIT |
 				     PR_SVE_SET_VL_ONEXEC)) {
 		return -EINVAL;
 	}
@@ -103,13 +163,19 @@ int sve_set_vector_length(struct thread *thread,
 		return -EINVAL;
 	}
 
-	if (vl > sve_max_vl) {
-		BUG_ON(!sve_vl_valid(sve_max_vl));
-		vl = sve_max_vl;
+	/*
+	 * Clamp to the maximum vector length that VL-agnostic SVE code can
+	 * work with.  A flag may be assigned in the future to allow setting
+	 * of larger vector lengths without confusing older software.
+	 */
+	if (vl > SVE_VL_ARCH_MAX) {
+		vl = SVE_VL_ARCH_MAX;
 	}
 
-	if (flags & (PR_SVE_SET_VL_ONEXEC |
-		     PR_SVE_SET_VL_INHERIT)) {
+	vl = find_supported_vector_length(vl);
+
+	if (flags & (PR_SVE_VL_INHERIT |
+		     PR_SVE_SET_VL_ONEXEC)) {
 		ti->sve_vl_onexec = vl;
 	} else {
 		/* Reset VL to system default on next exec: */
@@ -121,39 +187,42 @@ int sve_set_vector_length(struct thread *thread,
 		goto out;
 	}
 
-	if (vl != ti->sve_vl) {
-		if ((elf_hwcap & HWCAP_SVE)) {
-			fp_regs_struct fp_regs;
-			memset(&fp_regs, 0, sizeof(fp_regs));
+	if (vl == ti->sve_vl) {
+		goto out;
+	}
 
-			/* for self at prctl syscall */
-			if (thread == cpu_local_var(current)) {
-				save_fp_regs(thread);
-				clear_fp_regs(thread);
-				thread_sve_to_fpsimd(thread, &fp_regs);
-				sve_free(thread);
+	if ((elf_hwcap & HWCAP_SVE)) {
+		fp_regs_struct fp_regs;
 
-				ti->sve_vl = vl;
+		memset(&fp_regs, 0, sizeof(fp_regs));
 
-				sve_alloc(thread);
-				thread_fpsimd_to_sve(thread, &fp_regs);
-				restore_fp_regs(thread);
-			/* for target thread at ptrace */
-			} else {
-				thread_sve_to_fpsimd(thread, &fp_regs);
-				sve_free(thread);
+		/* for self at prctl syscall */
+		if (thread == cpu_local_var(current)) {
+			save_fp_regs(thread);
+			clear_fp_regs();
+			thread_sve_to_fpsimd(thread, &fp_regs);
+			sve_free(thread);
 
-				ti->sve_vl = vl;
+			ti->sve_vl = vl;
 
-				sve_alloc(thread);
-				thread_fpsimd_to_sve(thread, &fp_regs);
-			}
+			sve_alloc(thread);
+			thread_fpsimd_to_sve(thread, &fp_regs);
+			restore_fp_regs(thread);
+		/* for target thread at ptrace */
+		} else {
+			thread_sve_to_fpsimd(thread, &fp_regs);
+			sve_free(thread);
+
+			ti->sve_vl = vl;
+
+			sve_alloc(thread);
+			thread_fpsimd_to_sve(thread, &fp_regs);
 		}
 	}
 	ti->sve_vl = vl;
 
 out:
-	ti->sve_flags = flags & PR_SVE_SET_VL_INHERIT;
+	ti->sve_flags = flags & PR_SVE_VL_INHERIT;
 
 	return 0;
 }
@@ -163,44 +232,53 @@ out:
  * Encode the current vector length and flags for return.
  * This is only required for prctl(): ptrace has separate fields
  */
-static int sve_prctl_status(const struct thread_info *ti)
+static int sve_prctl_status(unsigned long flags)
 {
-	int ret = ti->sve_vl;
+	int ret;
+	struct thread_info *ti = cpu_local_var(current)->ctx.thread;
 
-	ret |= ti->sve_flags << 16;
+	if (flags & PR_SVE_SET_VL_ONEXEC) {
+		ret = ti->sve_vl_onexec;
+	}
+	else {
+		ret = ti->sve_vl;
+	}
 
+	if (ti->sve_flags & PR_SVE_VL_INHERIT) {
+		ret |= PR_SVE_VL_INHERIT;
+	}
 	return ret;
 }
 
 /* @ref.impl arch/arm64/kernel/fpsimd.c::sve_set_task_vl */
-int sve_set_thread_vl(struct thread *thread, const unsigned long vector_length,
-			const unsigned long flags)
+int sve_set_thread_vl(unsigned long arg)
 {
+	unsigned long vl, flags;
 	int ret;
 
-	if (!(elf_hwcap & HWCAP_SVE)) {
+	vl = arg & PR_SVE_VL_LEN_MASK;
+	flags = arg & ~vl;
+
+	/* Instead of system_supports_sve() */
+	if (unlikely(!(elf_hwcap & HWCAP_SVE))) {
 		return -EINVAL;
 	}
 
-	BUG_ON(thread != cpu_local_var(current));
-
-	preempt_disable();
-	ret = sve_set_vector_length(thread, vector_length, flags);
-	preempt_enable();
-
+	ret = sve_set_vector_length(cpu_local_var(current), vl, flags);
 	if (ret) {
 		return ret;
 	}
-	return sve_prctl_status(thread->ctx.thread);
+	return sve_prctl_status(flags);
 }
 
 /* @ref.impl arch/arm64/kernel/fpsimd.c::sve_get_ti_vl */
-int sve_get_thread_vl(const struct thread *thread)
+int sve_get_thread_vl(void)
 {
-	if (!(elf_hwcap & HWCAP_SVE)) {
+	/* Instead of system_supports_sve() */
+	if (unlikely(!(elf_hwcap & HWCAP_SVE))) {
 		return -EINVAL;
 	}
-	return sve_prctl_status(thread->ctx.thread);
+	return sve_prctl_status(0);
 }
 
 void do_sve_acc(unsigned int esr, struct pt_regs *regs)
@@ -210,25 +288,48 @@ void do_sve_acc(unsigned int esr, struct pt_regs *regs)
 	panic("");
 }
 
-void init_sve_vl(void)
+void sve_setup(void)
 {
 	extern unsigned long ihk_param_default_vl;
 	uint64_t zcr;
 
+	/* Instead of system_supports_sve() */
 	if (unlikely(!(elf_hwcap & HWCAP_SVE))) {
 		return;
 	}
 
-	zcr = read_system_reg(SYS_ZCR_EL1);
-	BUG_ON(((zcr & ZCR_EL1_LEN_MASK) + 1) * 16 > sve_max_vl);
+	/* init sve_vq_map bitmap */
+	sve_init_vq_map();
+
+	/*
+	 * The SVE architecture mandates support for 128-bit vectors,
+	 * so sve_vq_map must have at least SVE_VQ_MIN set.
+	 * If something went wrong, at least try to patch it up:
+	 */
+	if (!test_bit(vq_to_bit(SVE_VQ_MIN), sve_vq_map)) {
+		set_bit(vq_to_bit(SVE_VQ_MIN), sve_vq_map);
+	}
+
+	zcr = read_system_reg(SYS_ZCR_EL1);
+	sve_max_vl = sve_vl_from_vq((zcr & ZCR_EL1_LEN_MASK) + 1);
+
+	/*
+	 * Sanity-check that the max VL we determined through CPU features
+	 * corresponds properly to sve_vq_map.  If not, do our best:
+	 */
+	if (sve_max_vl != find_supported_vector_length(sve_max_vl)) {
+		sve_max_vl = find_supported_vector_length(sve_max_vl);
+	}
 
-	sve_max_vl = ((zcr & ZCR_EL1_LEN_MASK) + 1) * 16;
 	sve_default_vl = ihk_param_default_vl;
 
-	if (sve_default_vl == 0) {
-		kprintf("SVE: Getting default VL = 0 from HOST-Linux.\n");
-		sve_default_vl = sve_max_vl > 64 ? 64 : sve_max_vl;
-		kprintf("SVE: Using default vl(%d byte).\n", sve_default_vl);
+	if (ihk_param_default_vl !=
+		find_supported_vector_length(ihk_param_default_vl)) {
+		kprintf("SVE: Getting unsupported default VL = %d "
+			"from HOST-Linux.\n", sve_default_vl);
+		sve_default_vl = find_supported_vector_length(64);
+		kprintf("SVE: Using default vl(%d byte).\n",
+			sve_default_vl);
 	}
 
 	kprintf("SVE: maximum available vector length %u bytes per vector\n",
@@ -239,7 +340,7 @@ void init_sve_vl(void)
 
 #else /* CONFIG_ARM64_SVE */
 
-void init_sve_vl(void)
+void sve_setup(void)
 {
 	/* nothing to do. */
 }

arch/arm64/kernel/gencore.c

@@ -1,472 +0,0 @@
-/* gencore.c COPYRIGHT FUJITSU LIMITED 2015-2016 */
-#ifndef POSTK_DEBUG_ARCH_DEP_18 /* coredump arch separation. */
-#include <ihk/debug.h>
-#include <kmalloc.h>
-#include <cls.h>
-#include <list.h>
-#include <process.h>
-#include <string.h>
-#include <elfcore.h>
-
-#define	align32(x) ((((x) + 3) / 4) * 4)
-#define	alignpage(x) ((((x) + (PAGE_SIZE) - 1) / (PAGE_SIZE)) * (PAGE_SIZE))
-
-//#define DEBUG_PRINT_GENCORE
-
-#ifdef DEBUG_PRINT_GENCORE
-#define	dkprintf(...)	kprintf(__VA_ARGS__)
-#define	ekprintf(...)	kprintf(__VA_ARGS__)
-#else
-#define dkprintf(...)	do { if (0) kprintf(__VA_ARGS__); } while (0)
-#define	ekprintf(...)	kprintf(__VA_ARGS__)
-#endif
-
-/*
- * Generate a core file image, which consists of many chunks.
- * Returns an allocated table, an etnry of which is a pair of the address 
- * of a chunk and its length.
- */
-
-/**
- * \brief Fill the elf header.
- *
- * \param eh An Elf64_Ehdr structure.
- * \param segs Number of segments of the core file.
- */
-
-void fill_elf_header(Elf64_Ehdr *eh, int segs)
-{
-	eh->e_ident[EI_MAG0] = 0x7f;
-	eh->e_ident[EI_MAG1] = 'E';
-	eh->e_ident[EI_MAG2] = 'L';
-	eh->e_ident[EI_MAG3] = 'F';
-	eh->e_ident[EI_CLASS] = ELFCLASS64;
-	eh->e_ident[EI_DATA] = ELFDATA2LSB;
-	eh->e_ident[EI_VERSION] = El_VERSION;
-	eh->e_ident[EI_OSABI] = ELFOSABI_NONE;
-	eh->e_ident[EI_ABIVERSION] = El_ABIVERSION_NONE;
-
-	eh->e_type = ET_CORE;
-#ifdef CONFIG_MIC
-	eh->e_machine = EM_K10M;
-#else
-	eh->e_machine = EM_X86_64;
-#endif
-	eh->e_version = EV_CURRENT;
-	eh->e_entry = 0;	/* Do we really need this? */
-	eh->e_phoff = 64;	/* fixed */
-	eh->e_shoff = 0;	/* no section header */
-	eh->e_flags = 0;
-	eh->e_ehsize = 64;	/* fixed */
-	eh->e_phentsize = 56;	/* fixed */
-	eh->e_phnum = segs;
-	eh->e_shentsize = 0;
-	eh->e_shnum = 0;
-	eh->e_shstrndx = 0;
-}
-
-/**
- * \brief Return the size of the prstatus entry of the NOTE segment.
- *
- */
-
-int get_prstatus_size(void)
-{
-	return sizeof(struct note) + align32(sizeof("CORE")) 
-		+ align32(sizeof(struct elf_prstatus64));
-}
-
-/**
- * \brief Fill a prstatus structure.
- *
- * \param head A pointer to a note structure.
- * \param thread A pointer to the current thread structure.
- * \param regs0 A pointer to a x86_regs structure.
- */
-
-void fill_prstatus(struct note *head, struct thread *thread, void *regs0)
-{
-/* TODO(pka_idle) */
-} 
-
-/**
- * \brief Return the size of the prpsinfo entry of the NOTE segment.
- *
- */
-
-int get_prpsinfo_size(void)
-{
-	return sizeof(struct note) + align32(sizeof("CORE")) 
-		+ align32(sizeof(struct elf_prpsinfo64));
-}
-
-/**
- * \brief Fill a prpsinfo structure.
- *
- * \param head A pointer to a note structure.
- * \param thread A pointer to the current thread structure.
- * \param regs A pointer to a x86_regs structure.
- */
-
-void fill_prpsinfo(struct note *head, struct thread *thread, void *regs)
-{
-	void *name;
-	struct elf_prpsinfo64 *prpsinfo;
-
-	head->namesz = sizeof("CORE");
-	head->descsz = sizeof(struct elf_prpsinfo64);
-	head->type = NT_PRPSINFO;
-	name =  (void *) (head + 1);
-	memcpy(name, "CORE", sizeof("CORE"));
-	prpsinfo = (struct elf_prpsinfo64 *)(name + align32(sizeof("CORE")));
-
-	prpsinfo->pr_state = thread->status;
-	prpsinfo->pr_pid = thread->proc->pid;
-
-/*
-  We leave most of the fields unfilled.
-
-	char pr_sname;
-	char pr_zomb;
-	char pr_nice;
-	a8_uint64_t pr_flag;
-	unsigned int pr_uid;
-	unsigned int pr_gid;
-	int pr_ppid, pr_pgrp, pr_sid;
-	char pr_fname[16];
-	char pr_psargs[ELF_PRARGSZ];
-*/
-} 
-
-/**
- * \brief Return the size of the AUXV entry of the NOTE segment.
- *
- */
-
-int get_auxv_size(void)
-{
-	return sizeof(struct note) + align32(sizeof("CORE")) 
-		+ sizeof(unsigned long) * AUXV_LEN;
-}
-
-/**
- * \brief Fill an AUXV structure.
- *
- * \param head A pointer to a note structure.
- * \param thread A pointer to the current thread structure.
- * \param regs A pointer to a x86_regs structure.
- */
-
-void fill_auxv(struct note *head, struct thread *thread, void *regs)
-{
-	void *name;
-	void *auxv;
-
-	head->namesz = sizeof("CORE");
-	head->descsz = sizeof(unsigned long) * AUXV_LEN;
-	head->type = NT_AUXV;
-	name =  (void *) (head + 1);
-	memcpy(name, "CORE", sizeof("CORE"));
-	auxv = name + align32(sizeof("CORE"));
-	memcpy(auxv, thread->proc->saved_auxv, sizeof(unsigned long) * AUXV_LEN);
-} 
-
-/**
- * \brief Return the size of the whole NOTE segment.
- *
- */
-
-int get_note_size(void)
-{
-	return get_prstatus_size() + get_prpsinfo_size()
-		+ get_auxv_size();
-}
-
-/**
- * \brief Fill the NOTE segment.
- *
- * \param head A pointer to a note structure.
- * \param thread A pointer to the current thread structure.
- * \param regs A pointer to a x86_regs structure.
- */
-
-void fill_note(void *note, struct thread *thread, void *regs)
-{
-	fill_prstatus(note, thread, regs);
-	note += get_prstatus_size();
-	fill_prpsinfo(note, thread, regs);
-	note += get_prpsinfo_size();
-	fill_auxv(note, thread, regs);
-}
-
-/**
- * \brief Generate an image of the core file.
- *
- * \param thread A pointer to the current thread structure.
- * \param regs A pointer to a x86_regs structure.
- * \param coretable(out) An array of core chunks.
- * \param chunks(out) Number of the entires of coretable.
- *
- * A core chunk is represented by a pair of a physical 
- * address of memory region and its size. If there are
- * no corresponding physical address for a VM area 
- * (an unallocated demand-paging page, e.g.), the address
- * should be zero.
- */
-
-int gencore(struct thread *thread, void *regs, 
-	    struct coretable **coretable, int *chunks)
-{
-	struct coretable *ct = NULL;
-	Elf64_Ehdr eh;
-	Elf64_Phdr *ph = NULL;
-	void *note = NULL;
-	struct vm_range *range, *next;
-	struct process_vm *vm = thread->vm;
-	int segs = 1;	/* the first one is for NOTE */
-	int notesize, phsize, alignednotesize;
-	unsigned int offset = 0;
-	int i;
-
-	*chunks = 3; /* Elf header , header table and NOTE segment */
-
-	if (vm == NULL) {
-		dkprintf("no vm found.\n");
-		return -1;
-	}
-
-	next = lookup_process_memory_range(vm, 0, -1);
-	while ((range = next)) {
-		next = next_process_memory_range(vm, range);
-
-		dkprintf("start:%lx end:%lx flag:%lx objoff:%lx\n", 
-			 range->start, range->end, range->flag, range->objoff);
-		/* We omit reserved areas because they are only for
-		   mckernel's internal use. */		   
-		if (range->flag & VR_RESERVED)
-			continue;
-		/* We need a chunk for each page for a demand paging area.
-		   This can be optimized for spacial complexity but we would
-		   lose simplicity instead. */
-		if (range->flag & VR_DEMAND_PAGING) {
-			unsigned long p, phys;
-			int prevzero = 0;
-			for (p = range->start; p < range->end; p += PAGE_SIZE) {
-				if (ihk_mc_pt_virt_to_phys(thread->vm->address_space->page_table, 
-							    (void *)p, &phys) != 0) {
-					prevzero = 1;
-				} else {
-					if (prevzero == 1)
-						(*chunks)++;
-					(*chunks)++;
-					prevzero = 0;
-				}
-			}
-			if (prevzero == 1)
-				(*chunks)++;
-		} else {
-			(*chunks)++;
-		}
-		segs++;
-	}
-	dkprintf("we have %d segs and %d chunks.\n\n", segs, *chunks);
-
-	{
-		struct vm_regions region = thread->vm->region;
-
-		dkprintf("text:  %lx-%lx\n", region.text_start, region.text_end);
-		dkprintf("data:  %lx-%lx\n", region.data_start, region.data_end);
-		dkprintf("brk:   %lx-%lx\n", region.brk_start, region.brk_end);
-		dkprintf("map:   %lx-%lx\n", region.map_start, region.map_end);
-		dkprintf("stack: %lx-%lx\n", region.stack_start, region.stack_end);
-		dkprintf("user:  %lx-%lx\n\n", region.user_start, region.user_end);
-	}
-
-	dkprintf("now generate a core file image\n");
-
-	offset += sizeof(eh);
-	fill_elf_header(&eh, segs);
-
-	/* program header table */
-	phsize = sizeof(Elf64_Phdr) * segs;
-	ph = kmalloc(phsize, IHK_MC_AP_NOWAIT);
-	if (ph == NULL) {
-		dkprintf("could not alloc a program header table.\n");
-		goto fail;
-	}
-	memset(ph, 0, phsize);
-
-	offset += phsize;
-
-	/* NOTE segment
-	 * To align the next segment page-sized, we prepare a padded
-	 * region for our NOTE segment.
-	 */
-	notesize = get_note_size();
-	alignednotesize = alignpage(notesize + offset) - offset;
-	note = kmalloc(alignednotesize, IHK_MC_AP_NOWAIT);
-	if (note == NULL) {
-		dkprintf("could not alloc NOTE for core.\n");
-		goto fail;
-	}
-	memset(note, 0, alignednotesize);
-	fill_note(note, thread, regs);
-
-	/* prgram header for NOTE segment is exceptional */
-	ph[0].p_type = PT_NOTE;
-	ph[0].p_flags = 0;
-	ph[0].p_offset = offset;
-	ph[0].p_vaddr = 0;
-	ph[0].p_paddr = 0;
-	ph[0].p_filesz = notesize;
-	ph[0].p_memsz = notesize;
-	ph[0].p_align = 0;
-
-	offset += alignednotesize;
-
-	/* program header for each memory chunk */
-	i = 1;
-	next = lookup_process_memory_range(vm, 0, -1);
-	while ((range = next)) {
-		next = next_process_memory_range(vm, range);
-
-		unsigned long flag = range->flag;
-		unsigned long size = range->end - range->start;
-
-		if (range->flag & VR_RESERVED)
-			continue;
-
-		ph[i].p_type = PT_LOAD;
-		ph[i].p_flags = ((flag & VR_PROT_READ) ? PF_R : 0)
-			| ((flag & VR_PROT_WRITE) ? PF_W : 0)
-			| ((flag & VR_PROT_EXEC) ? PF_X : 0);
-		ph[i].p_offset = offset;
-		ph[i].p_vaddr = range->start;
-		ph[i].p_paddr = 0;
-		ph[i].p_filesz = size;
-		ph[i].p_memsz = size;
-		ph[i].p_align = PAGE_SIZE;
-		i++;
-		offset += size;
-	}
-
-	/* coretable to send to host */
-	ct = kmalloc(sizeof(struct coretable) * (*chunks), IHK_MC_AP_NOWAIT);
-	if (!ct) {
-		dkprintf("could not alloc a coretable.\n");
-		goto fail;
-	}
-
-	ct[0].addr = virt_to_phys(&eh);	/* ELF header */
-	ct[0].len = 64; 
-	dkprintf("coretable[0]: %lx@%lx(%lx)\n", ct[0].len, ct[0].addr, &eh);
-
-	ct[1].addr = virt_to_phys(ph);	/* program header table */
-	ct[1].len = phsize;
-	dkprintf("coretable[1]: %lx@%lx(%lx)\n", ct[1].len, ct[1].addr, ph);
-
-	ct[2].addr = virt_to_phys(note);	/* NOTE segment */
-	ct[2].len = alignednotesize;
-	dkprintf("coretable[2]: %lx@%lx(%lx)\n", ct[2].len, ct[2].addr, note);
-
-	i = 3;	/* memory segments */
-	next = lookup_process_memory_range(vm, 0, -1);
-	while ((range = next)) {
-		next = next_process_memory_range(vm, range);
-
-		unsigned long phys;
-
-		if (range->flag & VR_RESERVED)
-			continue;
-		if (range->flag & VR_DEMAND_PAGING) {
-			/* Just an ad hoc kluge. */
-			unsigned long p, start, phys;
-			int prevzero = 0;
-			unsigned long size = 0;
-
-			for (start = p = range->start; 
-			     p < range->end; p += PAGE_SIZE) {
-				if (ihk_mc_pt_virt_to_phys(thread->vm->address_space->page_table, 
-							    (void *)p, &phys) != 0) {
-					if (prevzero == 0) {
-						/* We begin a new chunk */
-						size = PAGE_SIZE;
-						start = p;
-					} else {
-						/* We extend the previous chunk */
-						size += PAGE_SIZE;
-					}
-					prevzero = 1;
-				} else {
-					if (prevzero == 1) {
-						/* Flush out an empty chunk */
-						ct[i].addr = 0;
-						ct[i].len = size;
-						dkprintf("coretable[%d]: %lx@%lx(%lx)\n", i, 
-							 ct[i].len, ct[i].addr, start);
-						i++;
-
-					}
-					ct[i].addr = phys;
-					ct[i].len = PAGE_SIZE;
-					dkprintf("coretable[%d]: %lx@%lx(%lx)\n", i, 
-						 ct[i].len, ct[i].addr, p);
-					i++;
-					prevzero = 0;
-				}
-			}
-			if (prevzero == 1) {
-				/* An empty chunk */
-				ct[i].addr = 0;
-				ct[i].len = size;
-				dkprintf("coretable[%d]: %lx@%lx(%lx)\n", i, 
-					 ct[i].len, ct[i].addr, start);
-				i++;
-			}		
-		} else {
-			if ((thread->vm->region.user_start <= range->start) &&
-			    (range->end <= thread->vm->region.user_end)) {
-				if (ihk_mc_pt_virt_to_phys(thread->vm->address_space->page_table, 
-							   (void *)range->start, &phys) != 0) {
-					dkprintf("could not convert user virtual address %lx"
-						 "to physical address", range->start);
-					goto fail;
-				}
-			} else {
-				phys = virt_to_phys((void *)range->start);
-			}
-			ct[i].addr = phys;
-			ct[i].len = range->end - range->start;
-			dkprintf("coretable[%d]: %lx@%lx(%lx)\n", i, 
-				 ct[i].len, ct[i].addr, range->start);
-			i++;
-		}
-	}
-	*coretable = ct;
-
-	return 0;
-
-	fail:
-	if (ct)
-		kfree(ct);
-	if (ph)
-		kfree(ph);
-	if (note)
-		kfree(note);
-	return -1;
-}
-
-/**
- * \brief Free all the allocated spaces for an image of the core file.
- *
- * \param coretable An array of core chunks.
- */
-
-void freecore(struct coretable **coretable)
-{
-	struct coretable *ct = *coretable;
-	kfree(phys_to_virt(ct[2].addr));	/* NOTE segment */
-	kfree(phys_to_virt(ct[1].addr));	/* ph */
-	kfree(*coretable);
-}
-#endif /* !POSTK_DEBUG_ARCH_DEP_18 */

arch/arm64/kernel/head.S

@@ -1,4 +1,4 @@
-/* head.S COPYRIGHT FUJITSU LIMITED 2015-2017 */
+/* head.S COPYRIGHT FUJITSU LIMITED 2015-2018 */
 
 #include <linkage.h>
 #include <ptrace.h>
@@ -10,9 +10,7 @@
 #include <smp.h>
 #include <arm-gic-v3.h>
 
-#define KERNEL_RAM_VADDR	MAP_KERNEL_START
-#define EARLY_ALLOC_VADDR	MAP_EARLY_ALLOC
-#define BOOT_PARAM_VADDR	MAP_BOOT_PARAM
+/* KERNEL_RAM_VADDR is defined by cmake */
 
 //#ifndef CONFIG_SMP
 //# define PTE_FLAGS	PTE_TYPE_PAGE | PTE_AF
@@ -50,16 +48,6 @@
 	add	\ttb0, \ttb0, \virt_to_phys
 	.endm
 
-#ifdef CONFIG_ARM64_64K_PAGES
-# define BLOCK_SHIFT	PAGE_SHIFT
-# define BLOCK_SIZE		PAGE_SIZE
-# define TABLE_SHIFT	PMD_SHIFT
-#else
-# define BLOCK_SHIFT	SECTION_SHIFT
-# define BLOCK_SIZE		SECTION_SIZE
-# define TABLE_SHIFT	PUD_SHIFT
-#endif
-
 #define KERNEL_START	KERNEL_RAM_VADDR
 #define KERNEL_END	_end
 
@@ -87,6 +75,7 @@
 #define TRAMPOLINE_DATA_CPU_MAP_SIZE_SIZE      0x08
 #define TRAMPOLINE_DATA_CPU_MAP_SIZE           (NR_CPUS * 8)
 #define TRAMPOLINE_DATA_DATA_RDISTS_PA_SIZE    (NR_CPUS * 8)
+#define TRAMPOLINE_DATA_RETENTION_STATE_FLAG_PA_SIZE 0x08
 #define TRAMPOLINE_DATA_NR_PMU_AFFI_SIZE       0x04
 #define TRAMPOLINE_DATA_PMU_AFF_SIZE           (CONFIG_SMP_MAX_CORES * 4)
 
@@ -105,9 +94,9 @@
 	.globl	ihk_param_gic_percpu_offset, ihk_param_gic_version
 	.globl	ihk_param_lpj, ihk_param_hz, ihk_param_psci_method
 	.globl	ihk_param_cpu_logical_map, ihk_param_gic_rdist_base_pa
-	.globl	ihk_param_pmu_irq_affiniry, ihk_param_nr_pmu_irq_affiniry
+	.globl	ihk_param_pmu_irq_affi, ihk_param_nr_pmu_irq_affi
 	.globl	ihk_param_use_virt_timer, ihk_param_evtstrm_timer_rate
-	.globl	ihk_param_default_vl
+	.globl	ihk_param_retention_state_flag_pa, ihk_param_default_vl
 ihk_param_head:
 ihk_param_param_addr:
 	.quad	0
@@ -145,9 +134,11 @@ ihk_param_cpu_logical_map:
 	.skip	NR_CPUS * 8	/* array of the MPIDR and the core number */
 ihk_param_gic_rdist_base_pa:
 	.skip	NR_CPUS * 8	/* per-cpu re-distributer PA */
-ihk_param_pmu_irq_affiniry:
+ihk_param_retention_state_flag_pa:
+	.quad	0
+ihk_param_pmu_irq_affi:
 	.skip	CONFIG_SMP_MAX_CORES * 4	/* array of the pmu affinity list */
-ihk_param_nr_pmu_irq_affiniry:
+ihk_param_nr_pmu_irq_affi:
 	.word	0		/* number of pmu affinity list elements. */
 
 /* @ref.impl arch/arm64/include/asm/kvm_arm.h */
@@ -265,13 +256,17 @@ ENTRY(arch_start)
 	mov x16, #NR_CPUS /* calc next data */
 	lsl x16, x16, 3
 	add x0, x0, x16
-	/* nr_pmu_irq_affiniry */
+	/* retention_state_flag_pa */
+	ldr	x16, [x0], #TRAMPOLINE_DATA_RETENTION_STATE_FLAG_PA_SIZE
+	adr	x15, ihk_param_retention_state_flag_pa
+	str	x16, [x15]
+	/* nr_pmu_irq_affi */
 	ldr	w16, [x0], #TRAMPOLINE_DATA_NR_PMU_AFFI_SIZE
-	adr	x15, ihk_param_nr_pmu_irq_affiniry
+	adr	x15, ihk_param_nr_pmu_irq_affi
 	str	w16, [x15]
-	/* pmu_irq_affiniry */
+	/* pmu_irq_affi */
 	mov x18, x0
-	adr	x15, ihk_param_pmu_irq_affiniry
+	adr	x15, ihk_param_pmu_irq_affi
 	b	2f
 1:	ldr	w17, [x18], #4
 	str	w17, [x15], #4
@@ -410,14 +405,17 @@ __create_page_tables:
 	 * Map the early_alloc_pages area, kernel_img next block
 	 */
 	ldr	x3, =KERNEL_END
-	add	x3, x3, x28				// __pa(KERNEL_END)
+	add	x3, x3, x28			// __pa(KERNEL_END)
 	add	x3, x3, #BLOCK_SIZE
-	sub x3, x3, #1
-	bic x3, x3, #(BLOCK_SIZE - 1)		// start PA calc.
-	ldr	x5, =EARLY_ALLOC_VADDR			// get start VA
-	mov x6, #1
-	lsl x6, x6, #(PAGE_SHIFT + MAP_EARLY_ALLOC_SHIFT)
+	sub	x3, x3, #1
+	bic	x3, x3, #(BLOCK_SIZE - 1)	// start PA calc.
+	ldr	x5, =KERNEL_END			// get start VA
+	add     x5, x5, #BLOCK_SIZE
+	sub	x5, x5, #1
+	bic	x5, x5, #(BLOCK_SIZE - 1)	// start VA calc.
+	mov	x6, #MAP_EARLY_ALLOC_SIZE
 	add	x6, x5, x6			// end VA calc
+	mov	x23, x6				// save end VA
 	sub	x6, x6, #1			// inclusive range
 	create_block_map x0, x7, x3, x5, x6
 
@@ -425,11 +423,13 @@ __create_page_tables:
 	 * Map the boot_param area
 	 */
 	adr	x3, ihk_param_param_addr
-	ldr	x3, [x3]				// get boot_param PA
-	ldr	x5, =BOOT_PARAM_VADDR			// get boot_param VA
-	mov x6, #1
-	lsl x6, x6, #MAP_BOOT_PARAM_SHIFT
-	add	x6, x5, x6			// end VA calc
+	ldr	x3, [x3]			// get boot_param PA
+	mov	x5, x23				// get start VA
+	add     x5, x5, #BLOCK_SIZE
+	sub	x5, x5, #1
+	bic	x5, x5, #(BLOCK_SIZE - 1)	// start VA calc
+	mov	x6, #MAP_BOOT_PARAM_SIZE
+	add	x6, x5, x6			// end VA calc.
 	sub	x6, x6, #1			// inclusive range
 	create_block_map x0, x7, x3, x5, x6
 

arch/arm64/kernel/hw_breakpoint.c

@@ -7,6 +7,7 @@
 #include <hw_breakpoint.h>
 #include <arch-memory.h>
 #include <signal.h>
+#include <process.h>
 
 /* @ref.impl arch/arm64/kernel/hw_breakpoint.c::core_num_[brps|wrps] */
 /* Number of BRP/WRP registers on this CPU. */

arch/arm64/kernel/imp-sysreg.c

@@ -0,0 +1,131 @@
+/* imp-sysreg.c COPYRIGHT FUJITSU LIMITED 2018 */
+#include <sysreg.h>
+
+/* hpc */
+ACCESS_REG_FUNC(fj_tag_address_ctrl_el1, IMP_FJ_TAG_ADDRESS_CTRL_EL1);
+ACCESS_REG_FUNC(pf_ctrl_el1, IMP_PF_CTRL_EL1);
+ACCESS_REG_FUNC(pf_stream_detect_ctrl_el0, IMP_PF_STREAM_DETECT_CTRL_EL0);
+ACCESS_REG_FUNC(pf_injection_ctrl0_el0, IMP_PF_INJECTION_CTRL0_EL0);
+ACCESS_REG_FUNC(pf_injection_ctrl1_el0, IMP_PF_INJECTION_CTRL1_EL0);
+ACCESS_REG_FUNC(pf_injection_ctrl2_el0, IMP_PF_INJECTION_CTRL2_EL0);
+ACCESS_REG_FUNC(pf_injection_ctrl3_el0, IMP_PF_INJECTION_CTRL3_EL0);
+ACCESS_REG_FUNC(pf_injection_ctrl4_el0, IMP_PF_INJECTION_CTRL4_EL0);
+ACCESS_REG_FUNC(pf_injection_ctrl5_el0, IMP_PF_INJECTION_CTRL5_EL0);
+ACCESS_REG_FUNC(pf_injection_ctrl6_el0, IMP_PF_INJECTION_CTRL6_EL0);
+ACCESS_REG_FUNC(pf_injection_ctrl7_el0, IMP_PF_INJECTION_CTRL7_EL0);
+ACCESS_REG_FUNC(pf_injection_distance0_el0, IMP_PF_INJECTION_DISTANCE0_EL0);
+ACCESS_REG_FUNC(pf_injection_distance1_el0, IMP_PF_INJECTION_DISTANCE1_EL0);
+ACCESS_REG_FUNC(pf_injection_distance2_el0, IMP_PF_INJECTION_DISTANCE2_EL0);
+ACCESS_REG_FUNC(pf_injection_distance3_el0, IMP_PF_INJECTION_DISTANCE3_EL0);
+ACCESS_REG_FUNC(pf_injection_distance4_el0, IMP_PF_INJECTION_DISTANCE4_EL0);
+ACCESS_REG_FUNC(pf_injection_distance5_el0, IMP_PF_INJECTION_DISTANCE5_EL0);
+ACCESS_REG_FUNC(pf_injection_distance6_el0, IMP_PF_INJECTION_DISTANCE6_EL0);
+ACCESS_REG_FUNC(pf_injection_distance7_el0, IMP_PF_INJECTION_DISTANCE7_EL0);
+
+static void hpc_prefetch_regs_init(void)
+{
+	uint64_t reg = 0;
+
+	/* PF_CTRL_EL1 */
+	reg = IMP_PF_CTRL_EL1_EL1AE_ENABLE | IMP_PF_CTRL_EL1_EL0AE_ENABLE;
+	xos_access_pf_ctrl_el1(WRITE_ACCESS, &reg);
+
+	/* PF_STREAM_DETECT_CTRL */
+	reg = 0;
+	xos_access_pf_stream_detect_ctrl_el0(WRITE_ACCESS, &reg);
+
+	/* PF_INJECTION_CTRL */
+	reg = 0;
+	xos_access_pf_injection_ctrl0_el0(WRITE_ACCESS, &reg);
+	xos_access_pf_injection_ctrl1_el0(WRITE_ACCESS, &reg);
+	xos_access_pf_injection_ctrl2_el0(WRITE_ACCESS, &reg);
+	xos_access_pf_injection_ctrl3_el0(WRITE_ACCESS, &reg);
+	xos_access_pf_injection_ctrl4_el0(WRITE_ACCESS, &reg);
+	xos_access_pf_injection_ctrl5_el0(WRITE_ACCESS, &reg);
+	xos_access_pf_injection_ctrl6_el0(WRITE_ACCESS, &reg);
+	xos_access_pf_injection_ctrl7_el0(WRITE_ACCESS, &reg);
+
+	/* PF_INJECTION_DISTANCE */
+	reg = 0;
+	xos_access_pf_injection_distance0_el0(WRITE_ACCESS, &reg);
+	xos_access_pf_injection_distance1_el0(WRITE_ACCESS, &reg);
+	xos_access_pf_injection_distance2_el0(WRITE_ACCESS, &reg);
+	xos_access_pf_injection_distance3_el0(WRITE_ACCESS, &reg);
+	xos_access_pf_injection_distance4_el0(WRITE_ACCESS, &reg);
+	xos_access_pf_injection_distance5_el0(WRITE_ACCESS, &reg);
+	xos_access_pf_injection_distance6_el0(WRITE_ACCESS, &reg);
+	xos_access_pf_injection_distance7_el0(WRITE_ACCESS, &reg);
+}
+
+static void hpc_tag_address_regs_init(void)
+{
+	uint64_t reg = IMP_FJ_TAG_ADDRESS_CTRL_EL1_TBO0_MASK |
+		IMP_FJ_TAG_ADDRESS_CTRL_EL1_SEC0_MASK |
+		IMP_FJ_TAG_ADDRESS_CTRL_EL1_PFE0_MASK;
+
+	/* FJ_TAG_ADDRESS_CTRL */
+	xos_access_fj_tag_address_ctrl_el1(WRITE_ACCESS, &reg);
+}
+
+void hpc_registers_init(void)
+{
+	hpc_prefetch_regs_init();
+	hpc_tag_address_regs_init();
+}
+
+/* vhbm */
+ACCESS_REG_FUNC(barrier_ctrl_el1, IMP_BARRIER_CTRL_EL1);
+ACCESS_REG_FUNC(barrier_bst_bit_el1, IMP_BARRIER_BST_BIT_EL1);
+ACCESS_REG_FUNC(barrier_init_sync_bb0_el1, IMP_BARRIER_INIT_SYNC_BB0_EL1);
+ACCESS_REG_FUNC(barrier_init_sync_bb1_el1, IMP_BARRIER_INIT_SYNC_BB1_EL1);
+ACCESS_REG_FUNC(barrier_init_sync_bb2_el1, IMP_BARRIER_INIT_SYNC_BB2_EL1);
+ACCESS_REG_FUNC(barrier_init_sync_bb3_el1, IMP_BARRIER_INIT_SYNC_BB3_EL1);
+ACCESS_REG_FUNC(barrier_init_sync_bb4_el1, IMP_BARRIER_INIT_SYNC_BB4_EL1);
+ACCESS_REG_FUNC(barrier_init_sync_bb5_el1, IMP_BARRIER_INIT_SYNC_BB5_EL1);
+ACCESS_REG_FUNC(barrier_assign_sync_w0_el1, IMP_BARRIER_ASSIGN_SYNC_W0_EL1);
+ACCESS_REG_FUNC(barrier_assign_sync_w1_el1, IMP_BARRIER_ASSIGN_SYNC_W1_EL1);
+ACCESS_REG_FUNC(barrier_assign_sync_w2_el1, IMP_BARRIER_ASSIGN_SYNC_W2_EL1);
+ACCESS_REG_FUNC(barrier_assign_sync_w3_el1, IMP_BARRIER_ASSIGN_SYNC_W3_EL1);
+
+void vhbm_barrier_registers_init(void)
+{
+	uint64_t reg = 0;
+
+	reg = IMP_BARRIER_CTRL_EL1_EL1AE_ENABLE |
+		IMP_BARRIER_CTRL_EL1_EL0AE_ENABLE;
+	xos_access_barrier_ctrl_el1(WRITE_ACCESS, &reg);
+
+	reg = 0;
+
+	xos_access_barrier_init_sync_bb0_el1(WRITE_ACCESS, &reg);
+	xos_access_barrier_init_sync_bb1_el1(WRITE_ACCESS, &reg);
+	xos_access_barrier_init_sync_bb2_el1(WRITE_ACCESS, &reg);
+	xos_access_barrier_init_sync_bb3_el1(WRITE_ACCESS, &reg);
+	xos_access_barrier_init_sync_bb4_el1(WRITE_ACCESS, &reg);
+	xos_access_barrier_init_sync_bb5_el1(WRITE_ACCESS, &reg);
+	xos_access_barrier_assign_sync_w0_el1(WRITE_ACCESS, &reg);
+	xos_access_barrier_assign_sync_w1_el1(WRITE_ACCESS, &reg);
+	xos_access_barrier_assign_sync_w2_el1(WRITE_ACCESS, &reg);
+	xos_access_barrier_assign_sync_w3_el1(WRITE_ACCESS, &reg);
+}
+
+/* sccr */
+ACCESS_REG_FUNC(sccr_ctrl_el1, IMP_SCCR_CTRL_EL1);
+ACCESS_REG_FUNC(sccr_assign_el1, IMP_SCCR_ASSIGN_EL1);
+ACCESS_REG_FUNC(sccr_set0_l2_el1, IMP_SCCR_SET0_L2_EL1);
+ACCESS_REG_FUNC(sccr_l1_el0, IMP_SCCR_L1_EL0);
+
+void scdrv_registers_init(void)
+{
+	uint64_t reg = 0;
+
+	reg = IMP_SCCR_CTRL_EL1_EL1AE_MASK;
+	xos_access_sccr_ctrl_el1(WRITE_ACCESS, &reg);
+
+	reg = 0;
+	xos_access_sccr_assign_el1(WRITE_ACCESS, &reg);
+	xos_access_sccr_l1_el0(WRITE_ACCESS, &reg);
+
+	reg = (14UL << IMP_SCCR_SET0_L2_EL1_L2_SEC0_SHIFT);
+	xos_access_sccr_set0_l2_el1(WRITE_ACCESS, &reg);
+}

arch/arm64/kernel/include/arch-futex.h

@@ -1,4 +1,4 @@
-/* arch-futex.h COPYRIGHT FUJITSU LIMITED 2015 */
+/* arch-futex.h COPYRIGHT FUJITSU LIMITED 2015-2018 */
 #ifndef __HEADER_ARM64_COMMON_ARCH_FUTEX_H
 #define __HEADER_ARM64_COMMON_ARCH_FUTEX_H
 
@@ -7,7 +7,8 @@
  * @ref.impl 
  * 	linux-linaro/arch/arm64/include/asm/futex.h:__futex_atomic_op
  */
-#define __futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg)		\
+#define ___futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg)		\
+do {								\
 	asm volatile(							\
 "1:	ldxr	%w1, %2\n"						\
 	insn "\n"							\
@@ -26,18 +27,36 @@
 "	.popsection\n"							\
 	: "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp)	\
 	: "r" (oparg), "Ir" (-EFAULT)					\
-	: "memory")
+	: "memory");						\
+} while (0);
+
+#ifndef IHK_OS_MANYCORE
+#include <linux/uaccess.h>
+
+#define __futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg)		\
+do {								\
+	uaccess_enable();					\
+	___futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg)		\
+	uaccess_disable();					\
+} while (0);
+
+#else
+#define __futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg)		\
+	___futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg)		\
+
+#endif
 
 /*
  * @ref.impl 
  * 	linux-linaro/arch/arm64/include/asm/futex.h:futex_atomic_op_inuser
  */
-static inline int futex_atomic_op_inuser(int encoded_op, int __user *uaddr)
+static inline int futex_atomic_op_inuser(int encoded_op,
+					 int __user *uaddr)
 {
 	int op = (encoded_op >> 28) & 7;
 	int cmp = (encoded_op >> 24) & 15;
-	int oparg = (encoded_op << 8) >> 20;
-	int cmparg = (encoded_op << 20) >> 20;
+	int oparg = (encoded_op & 0x00fff000) >> 12;
+	int cmparg = encoded_op & 0xfff;
 	int oldval = 0, ret, tmp;
 
 	if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28))
@@ -134,12 +153,4 @@ futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval)
 	return ret;
 }
 
-static inline int get_futex_value_locked(uint32_t *dest, uint32_t *from)
-{
-
-	*dest = *(volatile uint32_t *)from;
-
-	return 0;
-}
-
 #endif /* !__HEADER_ARM64_COMMON_ARCH_FUTEX_H */

arch/arm64/kernel/include/arch-lock.h

@@ -1,4 +1,4 @@
-/* arch-lock.h COPYRIGHT FUJITSU LIMITED 2015-2017 */
+/* arch-lock.h COPYRIGHT FUJITSU LIMITED 2015-2018 */
 #ifndef __HEADER_ARM64_COMMON_ARCH_LOCK_H
 #define __HEADER_ARM64_COMMON_ARCH_LOCK_H
 
@@ -6,6 +6,12 @@
 
 #include <ihk/cpu.h>
 #include <ihk/atomic.h>
+#include "affinity.h"
+#include <lwk/compiler.h>
+#include "config.h"
+#ifdef ENABLE_FUGAKU_HACKS
+#include <ihk/debug.h>
+#endif
 
 //#define DEBUG_SPINLOCK
 //#define DEBUG_MCS_RWLOCK
@@ -19,14 +25,18 @@ int __kprintf(const char *format, ...);
 
 /* @ref.impl arch/arm64/include/asm/spinlock_types.h::arch_spinlock_t */
 typedef struct {
-//#ifdef __AARCH64EB__
-//	uint16_t next;
-//	uint16_t owner;
-//#else /* __AARCH64EB__ */
+#ifdef __AARCH64EB__
+	uint16_t next;
+	uint16_t owner;
+#else /* __AARCH64EB__ */
 	uint16_t owner;
 	uint16_t next;
-//#endif /* __AARCH64EB__ */
-} ihk_spinlock_t;
+#endif /* __AARCH64EB__ */
+} __attribute__((aligned(4))) ihk_spinlock_t;
+
+#ifdef ENABLE_FUGAKU_HACKS
+extern ihk_spinlock_t *get_this_cpu_runq_lock(void);
+#endif
 
 extern void preempt_enable(void);
 extern void preempt_disable(void);
@@ -34,14 +44,112 @@ extern void preempt_disable(void);
 /* @ref.impl arch/arm64/include/asm/spinlock_types.h::__ARCH_SPIN_LOCK_UNLOCKED */
 #define SPIN_LOCK_UNLOCKED	{ 0, 0 }
 
+/* @ref.impl arch/arm64/include/asm/barrier.h::__nops */
+#define __nops(n)	".rept	" #n "\nnop\n.endr\n"
+
+/* @ref.impl ./arch/arm64/include/asm/lse.h::ARM64_LSE_ATOMIC_INSN */
+/* else defined(CONFIG_AS_LSE) && defined(CONFIG_ARM64_LSE_ATOMICS) */
+#define ARM64_LSE_ATOMIC_INSN(llsc, lse)	llsc
+
 /* initialized spinlock struct */
 static void ihk_mc_spinlock_init(ihk_spinlock_t *lock)
 {
 	*lock = (ihk_spinlock_t)SPIN_LOCK_UNLOCKED;
 }
 
-/* @ref.impl arch/arm64/include/asm/spinlock.h::arch_spin_lock */
-/* spinlock lock */
+#ifdef DEBUG_SPINLOCK
+#define ihk_mc_spinlock_trylock_noirq(l) { \
+	int rc; \
+	__kprintf("[%d] call ihk_mc_spinlock_trylock_noirq %p %s:%d\n", \
+		  ihk_mc_get_processor_id(), (l), __FILE__, __LINE__); \
+	rc = __ihk_mc_spinlock_trylock_noirq(l); \
+	__kprintf("[%d] ret ihk_mc_spinlock_trylock_noirq\n", \
+		  ihk_mc_get_processor_id()); \
+	rc; \
+}
+#else
+#define ihk_mc_spinlock_trylock_noirq __ihk_mc_spinlock_trylock_noirq
+#endif
+
+/* @ref.impl arch/arm64/include/asm/spinlock.h::arch_spin_trylock */
+/* spinlock trylock */
+static int __ihk_mc_spinlock_trylock_noirq(ihk_spinlock_t *lock)
+{
+	unsigned int tmp;
+	ihk_spinlock_t lockval;
+	int success;
+
+	preempt_disable();
+
+	asm volatile(ARM64_LSE_ATOMIC_INSN(
+	/* LL/SC */
+	"	prfm	pstl1strm, %2\n"
+	"1:	ldaxr	%w0, %2\n"
+	"	eor	%w1, %w0, %w0, ror #16\n"
+	"	cbnz	%w1, 2f\n"
+	"	add	%w0, %w0, %3\n"
+	"	stxr	%w1, %w0, %2\n"
+	"	cbnz	%w1, 1b\n"
+	"2:",
+	/* LSE atomics */
+	"	ldr	%w0, %2\n"
+	"	eor	%w1, %w0, %w0, ror #16\n"
+	"	cbnz	%w1, 1f\n"
+	"	add	%w1, %w0, %3\n"
+	"	casa	%w0, %w1, %2\n"
+	"	sub	%w1, %w1, %3\n"
+	"	eor	%w1, %w1, %w0\n"
+	"1:")
+	: "=&r" (lockval), "=&r" (tmp), "+Q" (*lock)
+	: "I" (1 << TICKET_SHIFT)
+	: "memory");
+
+	success = !tmp;
+
+#ifdef ENABLE_FUGAKU_HACKS
+#if 0
+	if (success) {
+		if (get_this_cpu_runq_lock() == lock &&
+				!cpu_interrupt_disabled()) {
+			kprintf("%s: WARNING: runq lock held without IRQs disabled?\n", __func__); \
+		}
+	}
+#endif
+#endif
+
+	if (!success) {
+		preempt_enable();
+	}
+	return success;
+}
+
+#ifdef DEBUG_SPINLOCK
+#define ihk_mc_spinlock_trylock(l, result) ({ \
+	unsigned long rc; \
+	__kprintf("[%d] call ihk_mc_spinlock_trylock %p %s:%d\n", \
+		  ihk_mc_get_processor_id(), (l), __FILE__, __LINE__); \
+	rc = __ihk_mc_spinlock_trylock(l, result); \
+	__kprintf("[%d] ret ihk_mc_spinlock_trylock\n", \
+		  ihk_mc_get_processor_id()); \
+	rc; \
+})
+#else
+#define ihk_mc_spinlock_trylock __ihk_mc_spinlock_trylock
+#endif
+
+/* spinlock trylock & interrupt disable & PSTATE.DAIF save */
+static unsigned long __ihk_mc_spinlock_trylock(ihk_spinlock_t *lock,
+					       int *result)
+{
+	unsigned long flags;
+
+	flags = cpu_disable_interrupt_save();
+
+	*result = __ihk_mc_spinlock_trylock_noirq(lock);
+
+	return flags;
+}
+
 #ifdef DEBUG_SPINLOCK
 #define ihk_mc_spinlock_lock_noirq(l) { \
 __kprintf("[%d] call ihk_mc_spinlock_lock_noirq %p %s:%d\n", ihk_mc_get_processor_id(), (l), __FILE__, __LINE__); \
@@ -52,6 +160,8 @@ __kprintf("[%d] ret ihk_mc_spinlock_lock_noirq\n", ihk_mc_get_processor_id()); \
 #define ihk_mc_spinlock_lock_noirq __ihk_mc_spinlock_lock_noirq
 #endif
 
+/* @ref.impl arch/arm64/include/asm/spinlock.h::arch_spin_lock */
+/* spinlock lock */
 static void __ihk_mc_spinlock_lock_noirq(ihk_spinlock_t *lock)
 {
 	unsigned int tmp;
@@ -61,11 +171,19 @@ static void __ihk_mc_spinlock_lock_noirq(ihk_spinlock_t *lock)
 
 	asm volatile(
 	/* Atomically increment the next ticket. */
+	ARM64_LSE_ATOMIC_INSN(
+	/* LL/SC */
 "	prfm	pstl1strm, %3\n"
 "1:	ldaxr	%w0, %3\n"
 "	add	%w1, %w0, %w5\n"
 "	stxr	%w2, %w1, %3\n"
-"	cbnz	%w2, 1b\n"
+"	cbnz	%w2, 1b\n",
+	/* LSE atomics */
+"	mov	%w2, %w5\n"
+"	ldadda	%w2, %w0, %3\n"
+	__nops(3)
+	)
+
 	/* Did we get the lock? */
 "	eor	%w1, %w0, %w0, ror #16\n"
 "	cbz	%w1, 3f\n"
@@ -83,9 +201,16 @@ static void __ihk_mc_spinlock_lock_noirq(ihk_spinlock_t *lock)
 	: "=&r" (lockval), "=&r" (newval), "=&r" (tmp), "+Q" (*lock)
 	: "Q" (lock->owner), "I" (1 << TICKET_SHIFT)
 	: "memory");
+#ifdef ENABLE_FUGAKU_HACKS
+#if 0
+	if (get_this_cpu_runq_lock() == lock &&
+			!cpu_interrupt_disabled()) {
+		kprintf("%s: WARNING: runq lock held without IRQs disabled?\n", __func__); \
+	}
+#endif
+#endif
 }
 
-/* spinlock lock & interrupt disable & PSTATE.DAIF save */
 #ifdef DEBUG_SPINLOCK
 #define ihk_mc_spinlock_lock(l) ({ unsigned long rc;\
 __kprintf("[%d] call ihk_mc_spinlock_lock %p %s:%d\n", ihk_mc_get_processor_id(), (l), __FILE__, __LINE__); \
@@ -95,6 +220,8 @@ __kprintf("[%d] ret ihk_mc_spinlock_lock\n", ihk_mc_get_processor_id()); rc;\
 #else
 #define ihk_mc_spinlock_lock __ihk_mc_spinlock_lock
 #endif
+
+/* spinlock lock & interrupt disable & PSTATE.DAIF save */
 static unsigned long __ihk_mc_spinlock_lock(ihk_spinlock_t *lock)
 {
 	unsigned long flags;
@@ -106,8 +233,6 @@ static unsigned long __ihk_mc_spinlock_lock(ihk_spinlock_t *lock)
 	return flags;
 }
 
-/* @ref.impl arch/arm64/include/asm/spinlock.h::arch_spin_unlock */
-/* spinlock unlock */
 #ifdef DEBUG_SPINLOCK
 #define ihk_mc_spinlock_unlock_noirq(l) { \
 __kprintf("[%d] call ihk_mc_spinlock_unlock_noirq %p %s:%d\n", ihk_mc_get_processor_id(), (l), __FILE__, __LINE__); \
@@ -117,12 +242,24 @@ __kprintf("[%d] ret ihk_mc_spinlock_unlock_noirq\n", ihk_mc_get_processor_id());
 #else
 #define ihk_mc_spinlock_unlock_noirq __ihk_mc_spinlock_unlock_noirq
 #endif
+
+/* @ref.impl arch/arm64/include/asm/spinlock.h::arch_spin_unlock */
+/* spinlock unlock */
 static void __ihk_mc_spinlock_unlock_noirq(ihk_spinlock_t *lock)
 {
-	asm volatile(
-"	stlrh	%w1, %0\n"
-	: "=Q" (lock->owner)
-	: "r" (lock->owner + 1)
+	unsigned long tmp;
+
+	asm volatile(ARM64_LSE_ATOMIC_INSN(
+	/* LL/SC */
+	"	ldrh	%w1, %0\n"
+	"	add	%w1, %w1, #1\n"
+	"	stlrh	%w1, %0",
+	/* LSE atomics */
+	"	mov	%w1, #1\n"
+	"	staddlh	%w1, %0\n"
+	__nops(1))
+	: "=Q" (lock->owner), "=&r" (tmp)
+	:
 	: "memory");
 
 	preempt_enable();
@@ -145,84 +282,6 @@ static void __ihk_mc_spinlock_unlock(ihk_spinlock_t *lock, unsigned long flags)
 	cpu_restore_interrupt(flags);
 }
 
-/* An implementation of the Mellor-Crummey Scott (MCS) lock */
-typedef struct mcs_lock_node {
-	unsigned long locked;
-	struct mcs_lock_node *next;
-	unsigned long irqsave;
-} __attribute__((aligned(64))) mcs_lock_node_t;
-
-static void mcs_lock_init(struct mcs_lock_node *node)
-{
-	node->locked = 0;
-	node->next = NULL;
-}
-
-static void __mcs_lock_lock(struct mcs_lock_node *lock,
-		struct mcs_lock_node *node)
-{
-	struct mcs_lock_node *pred;
-
-	node->next = NULL;
-	node->locked = 0;
-	pred = xchg8(&(lock->next), node);
-
-	if (pred) {
-		node->locked = 1;
-		pred->next = node;
-		while (node->locked != 0) {
-			cpu_pause();
-		}
-	}
-}
-
-static void __mcs_lock_unlock(struct mcs_lock_node *lock,
-		struct mcs_lock_node *node)
-{
-	if (node->next == NULL) {
-		struct mcs_lock_node *old = atomic_cmpxchg8(&(lock->next), node, 0);
-
-		if (old == node) {
-			return;
-		}
-
-		while (node->next == NULL) {
-			cpu_pause();
-		}
-	}
-
-	node->next->locked = 0;
-}
-
-static void mcs_lock_lock_noirq(struct mcs_lock_node *lock,
-		struct mcs_lock_node *node)
-{
-	preempt_disable();
-	__mcs_lock_lock(lock, node);
-}
-
-static void mcs_lock_unlock_noirq(struct mcs_lock_node *lock,
-		struct mcs_lock_node *node)
-{
-	__mcs_lock_unlock(lock, node);
-	preempt_enable();
-}
-
-static void mcs_lock_lock(struct mcs_lock_node *lock,
-		struct mcs_lock_node *node)
-{
-	node->irqsave = cpu_disable_interrupt_save();
-	mcs_lock_lock_noirq(lock, node);
-}
-
-static void mcs_lock_unlock(struct mcs_lock_node *lock,
-		struct mcs_lock_node *node)
-{
-	mcs_lock_unlock_noirq(lock, node);
-	cpu_restore_interrupt(node->irqsave);
-}
-
-
 #define SPINLOCK_IN_MCS_RWLOCK
 
 // reader/writer lock
@@ -238,14 +297,22 @@ typedef struct mcs_rwlock_node {
 	char dmy1;		// unused
 	char dmy2;		// unused
 	struct mcs_rwlock_node *next;
-} __attribute__((aligned(64))) mcs_rwlock_node_t;
+#ifndef ENABLE_UBSAN
+} __aligned(64) mcs_rwlock_node_t;
+#else
+} mcs_rwlock_node_t;
+#endif
 
 typedef struct mcs_rwlock_node_irqsave {
 #ifndef SPINLOCK_IN_MCS_RWLOCK
 	struct mcs_rwlock_node node;
 #endif
 	unsigned long irqsave;
-} __attribute__((aligned(64))) mcs_rwlock_node_irqsave_t;
+#ifndef ENABLE_UBSAN
+} __aligned(64) mcs_rwlock_node_irqsave_t;
+#else
+} mcs_rwlock_node_irqsave_t;
+#endif
 
 typedef struct mcs_rwlock_lock {
 #ifdef SPINLOCK_IN_MCS_RWLOCK
@@ -254,7 +321,11 @@ typedef struct mcs_rwlock_lock {
 	struct mcs_rwlock_node reader;		/* common reader lock */
 	struct mcs_rwlock_node *node;		/* base */
 #endif
-} __attribute__((aligned(64))) mcs_rwlock_lock_t;
+#ifndef ENABLE_UBSAN
+} __aligned(64) mcs_rwlock_lock_t;
+#else
+} mcs_rwlock_lock_t;
+#endif
 
 static void
 mcs_rwlock_init(struct mcs_rwlock_lock *lock)
@@ -602,4 +673,115 @@ __mcs_rwlock_reader_unlock(struct mcs_rwlock_lock *lock, struct mcs_rwlock_node_
 #endif
 }
 
+#if defined(CONFIG_HAS_NMI)
+#include <arm-gic-v3.h>
+static inline int irqflags_can_interrupt(unsigned long flags)
+{
+	return (flags == ICC_PMR_EL1_UNMASKED);
+}
+#else /* CONFIG_HAS_NMI */
+static inline int irqflags_can_interrupt(unsigned long flags)
+{
+	return !(flags & 0x2);
+}
+#endif /* CONFIG_HAS_NMI */
+
+struct ihk_rwlock {
+	unsigned int lock;
+};
+
+static inline void ihk_mc_rwlock_init(struct ihk_rwlock *rw)
+{
+	rw->lock = 0;
+}
+
+static inline void ihk_mc_read_lock(struct ihk_rwlock *rw)
+{
+	unsigned int tmp, tmp2;
+
+	asm volatile(
+	"       sevl\n"
+	"1:     wfe\n"
+	"2:     ldaxr   %w0, %2\n"
+	"       add     %w0, %w0, #1\n"
+	"       tbnz    %w0, #31, 1b\n"
+	"       stxr    %w1, %w0, %2\n"
+	"       cbnz    %w1, 2b\n"
+	: "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock)
+	:
+	: "cc", "memory");
+}
+
+static inline int ihk_mc_read_trylock(struct ihk_rwlock *rw)
+{
+	unsigned int tmp, tmp2 = 1;
+
+	asm volatile(
+	"       ldaxr   %w0, %2\n"
+	"       add     %w0, %w0, #1\n"
+	"       tbnz    %w0, #31, 1f\n"
+	"       stxr    %w1, %w0, %2\n"
+	"1:\n"
+	: "=&r" (tmp), "+r" (tmp2), "+Q" (rw->lock)
+	:
+	: "cc", "memory");
+
+	return !tmp2;
+}
+
+static inline void ihk_mc_read_unlock(struct ihk_rwlock *rw)
+{
+	unsigned int tmp, tmp2;
+
+	asm volatile(
+	"1:     ldxr    %w0, %2\n"
+	"       sub     %w0, %w0, #1\n"
+	"       stlxr   %w1, %w0, %2\n"
+	"       cbnz    %w1, 1b\n"
+	: "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock)
+	:
+	: "cc", "memory");
+}
+
+static inline void ihk_mc_write_lock(struct ihk_rwlock *rw)
+{
+	unsigned int tmp;
+
+	asm volatile(
+	"       sevl\n"
+	"1:     wfe\n"
+	"2:     ldaxr   %w0, %1\n"
+	"       cbnz    %w0, 1b\n"
+	"       stxr    %w0, %w2, %1\n"
+	"       cbnz    %w0, 2b\n"
+	: "=&r" (tmp), "+Q" (rw->lock)
+	: "r" (0x80000000)
+	: "cc", "memory");
+}
+
+static inline int ihk_mc_write_trylock(struct ihk_rwlock *rw)
+{
+	unsigned int tmp;
+
+	asm volatile(
+	"       ldaxr   %w0, %1\n"
+	"       cbnz    %w0, 1f\n"
+	"       stxr    %w0, %w2, %1\n"
+	"1:\n"
+	: "=&r" (tmp), "+Q" (rw->lock)
+	: "r" (0x80000000)
+	: "cc", "memory");
+
+	return !tmp;
+}
+
+static inline void ihk_mc_write_unlock(struct ihk_rwlock *rw)
+{
+	asm volatile(
+	"       stlr    %w1, %0\n"
+	: "=Q" (rw->lock) : "r" (0) : "memory");
+}
+
+#define ihk_mc_read_can_lock(rw) ((rw)->lock < 0x80000000)
+#define ihk_mc_write_can_lock(rw) ((rw)->lock == 0)
 #endif /* !__HEADER_ARM64_COMMON_ARCH_LOCK_H */

arch/arm64/kernel/include/arch-memory.h

@@ -1,96 +1,114 @@
-/* arch-memory.h COPYRIGHT FUJITSU LIMITED 2015-2017 */
+/* arch-memory.h COPYRIGHT FUJITSU LIMITED 2015-2018 */
 #ifndef __HEADER_ARM64_COMMON_ARCH_MEMORY_H
 #define __HEADER_ARM64_COMMON_ARCH_MEMORY_H
 
 #include <const.h>
+#include <errno.h>
+
+#ifndef __ASSEMBLY__
+#include <list.h>
+#include <page.h>
+void panic(const char *);
+#endif  /*__ASSEMBLY__*/
 
 #define _SZ4KB  (1UL<<12)
 #define _SZ16KB (1UL<<14)
 #define _SZ64KB (1UL<<16)
 
 #ifdef CONFIG_ARM64_64K_PAGES
-# define GRANULE_SIZE _SZ64KB
+# define GRANULE_SIZE	_SZ64KB
+# define BLOCK_SHIFT	PAGE_SHIFT
+# define BLOCK_SIZE	PAGE_SIZE
+# define TABLE_SHIFT	PMD_SHIFT
 #else
-# define GRANULE_SIZE _SZ4KB
+# define GRANULE_SIZE	_SZ4KB
+# define BLOCK_SHIFT	SECTION_SHIFT
+# define BLOCK_SIZE	SECTION_SIZE
+# define TABLE_SHIFT	PUD_SHIFT
 #endif
+
 #define VA_BITS		CONFIG_ARM64_VA_BITS
 
 /*
  * Address define
  */
-#define MAP_KERNEL_SHIFT	21
-#define MAP_KERNEL_SIZE		(UL(1) << MAP_KERNEL_SHIFT)
-#define MAP_EARLY_ALLOC_SHIFT	9
+/* early alloc area address */
+/* START:_end, SIZE:512 pages */
+#define MAP_EARLY_ALLOC_SHIFT	5
 #define MAP_EARLY_ALLOC_SIZE	(UL(1) << (PAGE_SHIFT + MAP_EARLY_ALLOC_SHIFT))
+
+#ifndef __ASSEMBLY__
+# define ALIGN_UP(x, align)     ALIGN_DOWN((x) + (align) - 1, align)
+# define ALIGN_DOWN(x, align)   ((x) & ~((align) - 1))
+extern char _end[];
+# define MAP_EARLY_ALLOC	(ALIGN_UP((unsigned long)_end, BLOCK_SIZE))
+# define MAP_EARLY_ALLOC_END	(MAP_EARLY_ALLOC + MAP_EARLY_ALLOC_SIZE)
+#endif /* !__ASSEMBLY__ */
+
+/* bootparam area address */
+/* START:early alloc area end, SIZE:2MiB */
 #define MAP_BOOT_PARAM_SHIFT	21
 #define MAP_BOOT_PARAM_SIZE	(UL(1) << MAP_BOOT_PARAM_SHIFT)
 
-#if (VA_BITS == 39 && GRANULE_SIZE == _SZ4KB)
+#ifndef __ASSEMBLY__
+# define MAP_BOOT_PARAM		(ALIGN_UP(MAP_EARLY_ALLOC_END, BLOCK_SIZE))
+# define MAP_BOOT_PARAM_END	(MAP_BOOT_PARAM + MAP_BOOT_PARAM_SIZE)
+#endif /* !__ASSEMBLY__ */
+
+/*
+ * MAP_KERNEL_START is HOST MODULES_END - 8MiB.
+ * It's defined by cmake.
+ */
+#if (VA_BITS == 39 && GRANULE_SIZE == _SZ4KB) /* ARM64_MEMORY_LAYOUT=1 */
 #
+# define LD_TASK_UNMAPPED_BASE	UL(0x0000000400000000)
 # define TASK_UNMAPPED_BASE	UL(0x0000000800000000)
 # define USER_END		UL(0x0000002000000000)
 # define MAP_VMAP_START		UL(0xffffffbdc0000000)
 # define MAP_VMAP_SIZE		UL(0x0000000100000000)
 # define MAP_FIXED_START	UL(0xffffffbffbdfd000)
 # define MAP_ST_START		UL(0xffffffc000000000)
-# define MAP_KERNEL_START	UL(0xffffffffff800000)                   // 0xffff_ffff_ff80_0000
-# define MAP_ST_SIZE		(MAP_KERNEL_START - MAP_ST_START)        // 0x0000_003f_ff80_0000
-# define MAP_EARLY_ALLOC	(MAP_KERNEL_START + MAP_KERNEL_SIZE)     // 0xffff_ffff_ffa0_0000
-# define MAP_EARLY_ALLOC_END	(MAP_EARLY_ALLOC + MAP_EARLY_ALLOC_SIZE)
-# define MAP_BOOT_PARAM		(MAP_EARLY_ALLOC_END)                    // 0xffff_ffff_ffc0_0000
-# define MAP_BOOT_PARAM_END	(MAP_BOOT_PARAM + MAP_BOOT_PARAM_SIZE)   // 0xffff_ffff_ffe0_0000
 #
-#elif (VA_BITS == 42 && GRANULE_SIZE == _SZ64KB)
+#elif (VA_BITS == 42 && GRANULE_SIZE == _SZ64KB) /* ARM64_MEMORY_LAYOUT=3 */
 #
+# define LD_TASK_UNMAPPED_BASE	UL(0x0000002000000000)
 # define TASK_UNMAPPED_BASE	UL(0x0000004000000000)
 # define USER_END		UL(0x0000010000000000)
 # define MAP_VMAP_START		UL(0xfffffdfee0000000)
 # define MAP_VMAP_SIZE		UL(0x0000000100000000)
 # define MAP_FIXED_START	UL(0xfffffdfffbdd0000)
 # define MAP_ST_START		UL(0xfffffe0000000000)
-# define MAP_KERNEL_START	UL(0xffffffffe0000000)                   // 0xffff_ffff_e000_0000
-# define MAP_ST_SIZE		(MAP_KERNEL_START - MAP_ST_START)        // 0x0000_01ff_e000_0000
-# define MAP_EARLY_ALLOC	(MAP_KERNEL_START + MAP_KERNEL_SIZE)     // 0xffff_ffff_e020_0000
-# define MAP_EARLY_ALLOC_END	(MAP_EARLY_ALLOC + MAP_EARLY_ALLOC_SIZE)
-# define MAP_BOOT_PARAM		(MAP_EARLY_ALLOC_END)                    // 0xffff_ffff_e220_0000
-# define MAP_BOOT_PARAM_END	(MAP_BOOT_PARAM + MAP_BOOT_PARAM_SIZE)   // 0xffff_ffff_e240_0000
 #
-#elif (VA_BITS == 48 && GRANULE_SIZE == _SZ4KB)
+#elif (VA_BITS == 48 && GRANULE_SIZE == _SZ4KB) /* ARM64_MEMORY_LAYOUT=2 */
 #
+# define LD_TASK_UNMAPPED_BASE	UL(0x0000080000000000)
 # define TASK_UNMAPPED_BASE	UL(0x0000100000000000)
 # define USER_END		UL(0x0000400000000000)
 # define MAP_VMAP_START		UL(0xffff7bffc0000000)
 # define MAP_VMAP_SIZE		UL(0x0000000100000000)
 # define MAP_FIXED_START	UL(0xffff7ffffbdfd000)
 # define MAP_ST_START		UL(0xffff800000000000)
-# define MAP_KERNEL_START	UL(0xffffffffff800000)                   // 0xffff_ffff_ff80_0000
-# define MAP_ST_SIZE		(MAP_KERNEL_START - MAP_ST_START)        // 0x0000_7fff_ff80_0000
-# define MAP_EARLY_ALLOC	(MAP_KERNEL_START + MAP_KERNEL_SIZE)     // 0xffff_ffff_ffa0_0000
-# define MAP_EARLY_ALLOC_END	(MAP_EARLY_ALLOC + MAP_EARLY_ALLOC_SIZE)
-# define MAP_BOOT_PARAM		(MAP_EARLY_ALLOC_END)                    // 0xffff_ffff_ffc0_0000
-# define MAP_BOOT_PARAM_END	(MAP_BOOT_PARAM + MAP_BOOT_PARAM_SIZE)   // 0xffff_ffff_ffe0_0000
 #
+#elif (VA_BITS == 48 && GRANULE_SIZE == _SZ64KB) /* ARM64_MEMORY_LAYOUT=4 */
 #
-#elif (VA_BITS == 48 && GRANULE_SIZE == _SZ64KB)
-#
+# define LD_TASK_UNMAPPED_BASE	UL(0x0000080000000000)
 # define TASK_UNMAPPED_BASE	UL(0x0000100000000000)
 # define USER_END		UL(0x0000400000000000)
+#ifdef ENABLE_TOFU
+# define MAP_VMAP_START		UL(0xffff7bdfffff0000)
+#else
 # define MAP_VMAP_START		UL(0xffff780000000000)
+#endif
 # define MAP_VMAP_SIZE		UL(0x0000000100000000)
 # define MAP_FIXED_START	UL(0xffff7ffffbdd0000)
 # define MAP_ST_START		UL(0xffff800000000000)
-# define MAP_KERNEL_START	UL(0xffffffffe0000000)                   // 0xffff_ffff_e000_0000
-# define MAP_ST_SIZE		(MAP_KERNEL_START - MAP_ST_START)        // 0x0000_7fff_e000_0000
-# define MAP_EARLY_ALLOC	(MAP_KERNEL_START + MAP_KERNEL_SIZE)     // 0xffff_ffff_e020_0000
-# define MAP_EARLY_ALLOC_END	(MAP_EARLY_ALLOC + MAP_EARLY_ALLOC_SIZE)
-# define MAP_BOOT_PARAM		(MAP_EARLY_ALLOC_END)                    // 0xffff_ffff_e220_0000
-# define MAP_BOOT_PARAM_END	(MAP_BOOT_PARAM + MAP_BOOT_PARAM_SIZE)   // 0xffff_ffff_e240_0000
 #
 #else
 # error address space is not defined.
 #endif
 
-#define STACK_TOP(region)  ((region)->user_end)
+#define MAP_ST_SIZE		(MAP_KERNEL_START - MAP_ST_START)
+#define STACK_TOP(region)	((region)->user_end)
 
 /*
  * pagetable define
@@ -104,7 +122,10 @@
 # define PTL3_INDEX_MASK PTL4_INDEX_MASK
 # define PTL2_INDEX_MASK PTL3_INDEX_MASK
 # define PTL1_INDEX_MASK PTL2_INDEX_MASK
-# define FIRST_LEVEL_BLOCK_SUPPORT  1
+# define __PTL4_CONT_SHIFT (__PTL4_SHIFT + 0)
+# define __PTL3_CONT_SHIFT (__PTL3_SHIFT + 4)
+# define __PTL2_CONT_SHIFT (__PTL2_SHIFT + 4)
+# define __PTL1_CONT_SHIFT (__PTL1_SHIFT + 4)
 #elif GRANULE_SIZE == _SZ16KB
 # define __PTL4_SHIFT  47
 # define __PTL3_SHIFT  36
@@ -114,29 +135,53 @@
 # define PTL3_INDEX_MASK ((UL(1) << 11) - 1)
 # define PTL2_INDEX_MASK PTL3_INDEX_MASK
 # define PTL1_INDEX_MASK PTL2_INDEX_MASK
-# define FIRST_LEVEL_BLOCK_SUPPORT  0
+# define __PTL4_CONT_SHIFT (__PTL4_SHIFT + 0)
+# define __PTL3_CONT_SHIFT (__PTL3_SHIFT + 0)
+# define __PTL2_CONT_SHIFT (__PTL2_SHIFT + 5)
+# define __PTL1_CONT_SHIFT (__PTL1_SHIFT + 7)
 #elif GRANULE_SIZE == _SZ64KB
-# define __PTL4_SHIFT  0
+# define __PTL4_SHIFT  55
 # define __PTL3_SHIFT  42
 # define __PTL2_SHIFT  29
 # define __PTL1_SHIFT  16
 # define PTL4_INDEX_MASK 0
 # define PTL3_INDEX_MASK ((UL(1) << 6) - 1)
+# define PTL3_INDEX_MASK_LINUX ((UL(1) << 10) - 1)
 # define PTL2_INDEX_MASK ((UL(1) << 13) - 1)
 # define PTL1_INDEX_MASK PTL2_INDEX_MASK
-# define FIRST_LEVEL_BLOCK_SUPPORT  0
+# define __PTL4_CONT_SHIFT (__PTL4_SHIFT + 0)
+# define __PTL3_CONT_SHIFT (__PTL3_SHIFT + 0)
+# define __PTL2_CONT_SHIFT (__PTL2_SHIFT + 5)
+# define __PTL1_CONT_SHIFT (__PTL1_SHIFT + 5)
 #else
 # error granule size error.
 #endif
 
+#ifndef __ASSEMBLY__
+extern int first_level_block_support;
+#endif /* __ASSEMBLY__ */
+
 # define __PTL4_SIZE  (UL(1) << __PTL4_SHIFT)
 # define __PTL3_SIZE  (UL(1) << __PTL3_SHIFT)
 # define __PTL2_SIZE  (UL(1) << __PTL2_SHIFT)
 # define __PTL1_SIZE  (UL(1) << __PTL1_SHIFT)
-# define __PTL4_MASK  (~__PTL4_SIZE - 1)
-# define __PTL3_MASK  (~__PTL3_SIZE - 1)
-# define __PTL2_MASK  (~__PTL2_SIZE - 1)
-# define __PTL1_MASK  (~__PTL1_SIZE - 1)
+# define __PTL4_MASK  (~(__PTL4_SIZE - 1))
+# define __PTL3_MASK  (~(__PTL3_SIZE - 1))
+# define __PTL2_MASK  (~(__PTL2_SIZE - 1))
+# define __PTL1_MASK  (~(__PTL1_SIZE - 1))
+
+# define __PTL4_CONT_SIZE  (UL(1) << __PTL4_CONT_SHIFT)
+# define __PTL3_CONT_SIZE  (UL(1) << __PTL3_CONT_SHIFT)
+# define __PTL2_CONT_SIZE  (UL(1) << __PTL2_CONT_SHIFT)
+# define __PTL1_CONT_SIZE  (UL(1) << __PTL1_CONT_SHIFT)
+# define __PTL4_CONT_MASK  (~(__PTL4_CONT_SIZE - 1))
+# define __PTL3_CONT_MASK  (~(__PTL3_CONT_SIZE - 1))
+# define __PTL2_CONT_MASK  (~(__PTL2_CONT_SIZE - 1))
+# define __PTL1_CONT_MASK  (~(__PTL1_CONT_SIZE - 1))
+# define __PTL4_CONT_COUNT  (UL(1) << (__PTL4_CONT_SHIFT - __PTL4_SHIFT))
+# define __PTL3_CONT_COUNT  (UL(1) << (__PTL3_CONT_SHIFT - __PTL3_SHIFT))
+# define __PTL2_CONT_COUNT  (UL(1) << (__PTL2_CONT_SHIFT - __PTL2_SHIFT))
+# define __PTL1_CONT_COUNT  (UL(1) << (__PTL1_CONT_SHIFT - __PTL1_SHIFT))
 
 /* calculate entries */
 #if (CONFIG_ARM64_PGTABLE_LEVELS > 3) && (VA_BITS > __PTL4_SHIFT)
@@ -183,6 +228,22 @@ static const unsigned int  PTL4_ENTRIES = __PTL4_ENTRIES;
 static const unsigned int  PTL3_ENTRIES = __PTL3_ENTRIES;
 static const unsigned int  PTL2_ENTRIES = __PTL2_ENTRIES;
 static const unsigned int  PTL1_ENTRIES = __PTL1_ENTRIES;
+static const unsigned int  PTL4_CONT_SHIFT = __PTL4_CONT_SHIFT;
+static const unsigned int  PTL3_CONT_SHIFT = __PTL3_CONT_SHIFT;
+static const unsigned int  PTL2_CONT_SHIFT = __PTL2_CONT_SHIFT;
+static const unsigned int  PTL1_CONT_SHIFT = __PTL1_CONT_SHIFT;
+static const unsigned long PTL4_CONT_SIZE  = __PTL4_CONT_SIZE;
+static const unsigned long PTL3_CONT_SIZE  = __PTL3_CONT_SIZE;
+static const unsigned long PTL2_CONT_SIZE  = __PTL2_CONT_SIZE;
+static const unsigned long PTL1_CONT_SIZE  = __PTL1_CONT_SIZE;
+static const unsigned long PTL4_CONT_MASK  = __PTL4_CONT_MASK;
+static const unsigned long PTL3_CONT_MASK  = __PTL3_CONT_MASK;
+static const unsigned long PTL2_CONT_MASK  = __PTL2_CONT_MASK;
+static const unsigned long PTL1_CONT_MASK  = __PTL1_CONT_MASK;
+static const unsigned int  PTL4_CONT_COUNT = __PTL4_CONT_COUNT;
+static const unsigned int  PTL3_CONT_COUNT = __PTL3_CONT_COUNT;
+static const unsigned int  PTL2_CONT_COUNT = __PTL2_CONT_COUNT;
+static const unsigned int  PTL1_CONT_COUNT = __PTL1_CONT_COUNT;
 #else
 # define PTL4_SHIFT   __PTL4_SHIFT
 # define PTL3_SHIFT   __PTL3_SHIFT
@@ -200,8 +261,26 @@ static const unsigned int  PTL1_ENTRIES = __PTL1_ENTRIES;
 # define PTL3_ENTRIES __PTL3_ENTRIES
 # define PTL2_ENTRIES __PTL2_ENTRIES
 # define PTL1_ENTRIES __PTL1_ENTRIES
+# define PTL4_CONT_SHIFT __PTL4_CONT_SHIFT
+# define PTL3_CONT_SHIFT __PTL3_CONT_SHIFT
+# define PTL2_CONT_SHIFT __PTL2_CONT_SHIFT
+# define PTL1_CONT_SHIFT __PTL1_CONT_SHIFT
+# define PTL4_CONT_SIZE __PTL4_CONT_SIZE
+# define PTL3_CONT_SIZE __PTL3_CONT_SIZE
+# define PTL2_CONT_SIZE __PTL2_CONT_SIZE
+# define PTL1_CONT_SIZE __PTL1_CONT_SIZE
+# define PTL4_CONT_MASK __PTL4_CONT_MASK
+# define PTL3_CONT_MASK __PTL3_CONT_MASK
+# define PTL2_CONT_MASK __PTL2_CONT_MASK
+# define PTL1_CONT_MASK __PTL1_CONT_MASK
+# define PTL4_CONT_COUNT __PTL4_CONT_COUNT
+# define PTL3_CONT_COUNT __PTL3_CONT_COUNT
+# define PTL2_CONT_COUNT __PTL2_CONT_COUNT
+# define PTL1_CONT_COUNT __PTL1_CONT_COUNT
 #endif/*__ASSEMBLY__*/
 
+#define __page_size(pgshift)        (UL(1) << (pgshift))
+#define __page_mask(pgsize)         (~((pgsize) - 1))
 #define __page_offset(addr, size)   ((unsigned long)(addr) & ((size) - 1))
 #define __page_align(addr, size)    ((unsigned long)(addr) & ~((size) - 1))
 #define __page_align_up(addr, size) __page_align((unsigned long)(addr) + (size) - 1, size)
@@ -210,8 +289,8 @@ static const unsigned int  PTL1_ENTRIES = __PTL1_ENTRIES;
  * nornal page
  */
 #define PAGE_SHIFT          __PTL1_SHIFT
-#define PAGE_SIZE           (UL(1) << __PTL1_SHIFT)
-#define PAGE_MASK           (~(PTL1_SIZE - 1))
+#define PAGE_SIZE           __page_size(PAGE_SHIFT)
+#define PAGE_MASK           __page_mask(PAGE_SIZE)
 #define PAGE_P2ALIGN        0
 #define page_offset(addr)   __page_offset(addr, PAGE_SIZE)
 #define page_align(addr)    __page_align(addr, PAGE_SIZE)
@@ -221,8 +300,8 @@ static const unsigned int  PTL1_ENTRIES = __PTL1_ENTRIES;
  * large page
  */
 #define LARGE_PAGE_SHIFT          __PTL2_SHIFT
-#define LARGE_PAGE_SIZE           (UL(1) << __PTL2_SHIFT)
-#define LARGE_PAGE_MASK           (~(PTL2_SIZE - 1))
+#define LARGE_PAGE_SIZE           __page_size(LARGE_PAGE_SHIFT)
+#define LARGE_PAGE_MASK           __page_mask(LARGE_PAGE_SIZE)
 #define LARGE_PAGE_P2ALIGN        (LARGE_PAGE_SHIFT - PAGE_SHIFT)
 #define large_page_offset(addr)   __page_offset(addr, LARGE_PAGE_SIZE)
 #define large_page_align(addr)    __page_align(addr, LARGE_PAGE_SIZE)
@@ -263,6 +342,18 @@ static const unsigned int  PTL1_ENTRIES = __PTL1_ENTRIES;
 
 #define PTE_FILEOFF		PTE_SPECIAL
 
+#ifdef CONFIG_ARM64_64K_PAGES
+# define USER_STACK_PREPAGE_SIZE	PAGE_SIZE
+# define USER_STACK_PAGE_MASK		PAGE_MASK
+# define USER_STACK_PAGE_P2ALIGN	PAGE_P2ALIGN
+# define USER_STACK_PAGE_SHIFT		PAGE_SHIFT
+#else
+# define USER_STACK_PREPAGE_SIZE	LARGE_PAGE_SIZE
+# define USER_STACK_PAGE_MASK		LARGE_PAGE_MASK
+# define USER_STACK_PAGE_P2ALIGN	LARGE_PAGE_P2ALIGN
+# define USER_STACK_PAGE_SHIFT		LARGE_PAGE_SHIFT
+#endif
+
 #define PT_ENTRIES		(PAGE_SIZE >> 3)
 
 #ifndef __ASSEMBLY__
@@ -312,6 +403,8 @@ enum ihk_mc_pt_attribute {
 	PTATTR_FOR_USER   = UL(1) << (PHYS_MASK_SHIFT - 1),
 	/* WriteCombine */
 	PTATTR_WRITE_COMBINED = PTE_ATTRINDX(2),
+	/* converted flag */
+	ARCH_PTATTR_FLIPPED = PTE_PROT_NONE,
 };
 extern enum ihk_mc_pt_attribute attr_mask;
 
@@ -323,18 +416,23 @@ static inline int pfn_is_write_combined(uintptr_t pfn)
 //共通部と意味がするビット定義
 #define attr_flip_bits (PTATTR_WRITABLE | PTATTR_LARGEPAGE)
 
+static inline int pgsize_to_tbllv(size_t pgsize);
 static inline int pte_is_type_page(const pte_t *ptep, size_t pgsize)
 {
 	int ret = 0; //default D_TABLE
-	if ((PTL4_SIZE == pgsize && CONFIG_ARM64_PGTABLE_LEVELS > 3) ||
-	    (PTL3_SIZE == pgsize && CONFIG_ARM64_PGTABLE_LEVELS > 2) ||
-	    (PTL2_SIZE == pgsize)) {
+	int level = pgsize_to_tbllv(pgsize);
+
+	switch (level) {
+	case 4:
+	case 3:
+	case 2:
 		// check D_BLOCK
 		ret = ((*ptep & PMD_TYPE_MASK) == PMD_TYPE_SECT);
-	}
-	else if (PTL1_SIZE == pgsize) {
+		break;
+	case 1:
 		// check D_PAGE
 		ret = ((*ptep & PTE_TYPE_MASK) == PTE_TYPE_PAGE);
+		break;
 	}
 	return ret;
 }
@@ -413,21 +511,18 @@ static inline enum ihk_mc_pt_attribute pte_get_attr(pte_t *ptep, size_t pgsize)
 
 static inline void pte_make_null(pte_t *ptep, size_t pgsize)
 {
-	if ((PTL4_SIZE == pgsize && CONFIG_ARM64_PGTABLE_LEVELS > 3) ||
-	    (PTL3_SIZE == pgsize && CONFIG_ARM64_PGTABLE_LEVELS > 2) ||
-	    (PTL2_SIZE == pgsize) || 
-	    (PTL1_SIZE == pgsize)) {
-		*ptep = PTE_NULL;
-	}
+	*ptep = PTE_NULL;
 }
 
 static inline void pte_make_fileoff(off_t off,
 		enum ihk_mc_pt_attribute ptattr, size_t pgsize, pte_t *ptep)
 {
-	if ((PTL4_SIZE == pgsize && CONFIG_ARM64_PGTABLE_LEVELS > 3) ||
-	    (PTL3_SIZE == pgsize && CONFIG_ARM64_PGTABLE_LEVELS > 2) ||
-	    (PTL2_SIZE == pgsize) ||
-	    (PTL1_SIZE == pgsize)) {
+	if (((PTL4_SIZE == pgsize || PTL4_CONT_SIZE == pgsize)
+				&& CONFIG_ARM64_PGTABLE_LEVELS > 3) ||
+	    ((PTL3_SIZE == pgsize || PTL3_CONT_SIZE == pgsize)
+				&& CONFIG_ARM64_PGTABLE_LEVELS > 2) ||
+	     (PTL2_SIZE == pgsize || PTL2_CONT_SIZE == pgsize) ||
+	     (PTL1_SIZE == pgsize || PTL1_CONT_SIZE == pgsize)) {
 		*ptep = PTE_FILEOFF | off | PTE_TYPE_PAGE;
 	}
 }
@@ -457,7 +552,276 @@ static inline void pte_set_dirty(pte_t *ptep, size_t pgsize)
 	}
 }
 
-struct page_table;
+static inline int pte_is_contiguous(const pte_t *ptep)
+{
+	return !!(*ptep & PTE_CONT);
+}
+
+static inline int pgsize_is_contiguous(size_t pgsize)
+{
+	int ret = 0;
+
+	if ((pgsize == PTL4_CONT_SIZE && CONFIG_ARM64_PGTABLE_LEVELS > 3) ||
+	    (pgsize == PTL3_CONT_SIZE && CONFIG_ARM64_PGTABLE_LEVELS > 2) ||
+	    (pgsize == PTL2_CONT_SIZE) ||
+	    (pgsize == PTL1_CONT_SIZE)) {
+		ret = 1;
+	}
+	return ret;
+}
+
+static inline int pgsize_to_tbllv(size_t pgsize)
+{
+	int level = -EINVAL;
+
+	if ((pgsize == PTL4_CONT_SIZE || pgsize == PTL4_SIZE)
+	    && (CONFIG_ARM64_PGTABLE_LEVELS > 3)) {
+		level = 4;
+	} else if ((pgsize == PTL3_CONT_SIZE || pgsize == PTL3_SIZE)
+		   && (CONFIG_ARM64_PGTABLE_LEVELS > 2)) {
+		level = 3;
+	} else if (pgsize == PTL2_CONT_SIZE || pgsize == PTL2_SIZE) {
+		level = 2;
+	} else if (pgsize == PTL1_CONT_SIZE || pgsize == PTL1_SIZE) {
+		level = 1;
+	}
+	return level;
+}
+
+static inline int pgsize_to_pgshift(size_t pgsize)
+{
+	/* We need to use if instead of switch because
+	 * sometimes PTLX_CONT_SIZE == PTLX_SIZE
+	 */
+	if (pgsize == PTL4_CONT_SIZE) {
+		if (CONFIG_ARM64_PGTABLE_LEVELS > 3) {
+			return PTL4_CONT_SHIFT;
+		}
+	} else if (pgsize == PTL4_SIZE) {
+		if (CONFIG_ARM64_PGTABLE_LEVELS > 3) {
+			return PTL4_SHIFT;
+		}
+	} else if (pgsize == PTL3_CONT_SIZE) {
+		if (CONFIG_ARM64_PGTABLE_LEVELS > 2) {
+			return PTL3_CONT_SHIFT;
+		}
+	} else if (pgsize == PTL3_SIZE) {
+		if (CONFIG_ARM64_PGTABLE_LEVELS > 2) {
+			return PTL3_SHIFT;
+		}
+	} else if (pgsize == PTL2_CONT_SIZE) {
+		return PTL2_CONT_SHIFT;
+	} else if (pgsize == PTL2_SIZE) {
+		return PTL2_SHIFT;
+	} else if (pgsize == PTL1_CONT_SIZE) {
+		return PTL1_CONT_SHIFT;
+	} else if (pgsize == PTL1_SIZE) {
+		return PTL1_SHIFT;
+	}
+
+	return -EINVAL;
+}
+
+static inline size_t tbllv_to_pgsize(int level)
+{
+	size_t pgsize = 0;
+
+	switch (level) {
+	case 4:
+		if (CONFIG_ARM64_PGTABLE_LEVELS > 3) {
+			pgsize = PTL4_SIZE;
+		} else {
+			panic("page table level 4 is invalid.");
+		}
+		break;
+	case 3:
+		if (CONFIG_ARM64_PGTABLE_LEVELS > 2) {
+			pgsize = PTL3_SIZE;
+		} else {
+			panic("page table level 3 is invalid.");
+		}
+		break;
+	case 2:
+		pgsize = PTL2_SIZE;
+		break;
+	case 1:
+		pgsize = PTL1_SIZE;
+		break;
+	default:
+		panic("page table level is invalid.");
+	}
+	return pgsize;
+}
+
+static inline size_t tbllv_to_contpgsize(int level)
+{
+	size_t pgsize = 0;
+
+	switch (level) {
+	case 4:
+		if (CONFIG_ARM64_PGTABLE_LEVELS > 3) {
+			pgsize = PTL4_CONT_SIZE;
+		} else {
+			panic("page table level 4 is invalid.");
+		}
+		break;
+	case 3:
+		if (CONFIG_ARM64_PGTABLE_LEVELS > 2) {
+			pgsize = PTL3_CONT_SIZE;
+		} else {
+			panic("page table level 3 is invalid.");
+		}
+		break;
+	case 2:
+		pgsize = PTL2_CONT_SIZE;
+		break;
+	case 1:
+		pgsize = PTL1_CONT_SIZE;
+		break;
+	default:
+		panic("page table level is invalid.");
+	}
+	return pgsize;
+}
+
+static inline int tbllv_to_contpgshift(int level)
+{
+	int ret = 0;
+
+	switch (level) {
+	case 4:
+		if (CONFIG_ARM64_PGTABLE_LEVELS > 3) {
+			ret = PTL4_CONT_SHIFT;
+		} else {
+			panic("page table level 4 is invalid.");
+		}
+
+		break;
+	case 3:
+		if (CONFIG_ARM64_PGTABLE_LEVELS > 2) {
+			ret = PTL3_CONT_SHIFT;
+		} else {
+			panic("page table level 3 is invalid.");
+		}
+
+		break;
+	case 2:
+		ret = PTL2_CONT_SHIFT;
+		break;
+	case 1:
+		ret = PTL1_CONT_SHIFT;
+		break;
+	default:
+		panic("page table level is invalid.");
+	}
+	return ret;
+}
+
+static inline pte_t *get_contiguous_head(pte_t *__ptep, size_t __pgsize)
+{
+	unsigned long align;
+	int shift = 0;
+
+	switch (pgsize_to_tbllv(__pgsize)) {
+	case 4:
+		if (CONFIG_ARM64_PGTABLE_LEVELS > 3) {
+			shift = PTL4_CONT_SHIFT - PTL4_SHIFT;
+		} else {
+			panic("page table level 4 is invalid.");
+		}
+		break;
+	case 3:
+		if (CONFIG_ARM64_PGTABLE_LEVELS > 2) {
+			shift = PTL3_CONT_SHIFT - PTL3_SHIFT;
+		} else {
+			panic("page table level 3 is invalid.");
+		}
+		break;
+	case 2:
+		shift = PTL2_CONT_SHIFT - PTL2_SHIFT;
+		break;
+	case 1:
+		shift = PTL1_CONT_SHIFT - PTL1_SHIFT;
+		break;
+	default:
+		panic("page table level is invalid.");
+	}
+	align = sizeof(*__ptep) << shift;
+	return  (pte_t *)__page_align(__ptep, align);
+}
+
+static inline pte_t *get_contiguous_tail(pte_t *__ptep, size_t __pgsize)
+{
+	unsigned long align;
+	int shift = 0;
+
+	switch (pgsize_to_tbllv(__pgsize)) {
+	case 4:
+		if (CONFIG_ARM64_PGTABLE_LEVELS > 3) {
+			shift = PTL4_CONT_SHIFT - PTL4_SHIFT;
+		} else {
+			panic("page table level 4 is invalid.");
+		}
+		break;
+	case 3:
+		if (CONFIG_ARM64_PGTABLE_LEVELS > 2) {
+			shift = PTL3_CONT_SHIFT - PTL3_SHIFT;
+		} else {
+			panic("page table level 3 is invalid.");
+		}
+		break;
+	case 2:
+		shift = PTL2_CONT_SHIFT - PTL2_SHIFT;
+		break;
+	case 1:
+		shift = PTL1_CONT_SHIFT - PTL1_SHIFT;
+		break;
+	default:
+		panic("page table level is invalid.");
+	}
+	align = sizeof(*__ptep) << shift;
+	return  (pte_t *)__page_align_up(__ptep + 1, align) - 1;
+}
+
+int split_contiguous_pages(pte_t *ptep, size_t pgsize,
+		uint32_t memobj_flags);
+
+static inline int page_is_contiguous_head(pte_t *ptep, size_t pgsize)
+{
+	pte_t *ptr = get_contiguous_head(ptep, pgsize);
+
+	return (ptr == ptep);
+}
+
+static inline int page_is_contiguous_tail(pte_t *ptep, size_t pgsize)
+{
+	pte_t *ptr = get_contiguous_tail(ptep, pgsize);
+
+	return (ptr == ptep);
+}
+
+/* Return true if PTE doesn't belong to a contiguous PTE group or PTE
+ * is the head of a contiguous PTE group
+ */
+static inline int pte_is_head(pte_t *ptep, pte_t *old, size_t cont_size)
+{
+	if (!pte_is_contiguous(old))
+		return 1;
+	return page_is_contiguous_head(ptep, cont_size);
+}
+
+typedef pte_t translation_table_t;
+struct page_table {
+	translation_table_t* tt;
+	translation_table_t* tt_pa;
+	int asid;
+};
+
+void arch_adjust_allocate_page_size(struct page_table *pt,
+				    uintptr_t fault_addr,
+				    pte_t *ptep,
+				    void **pgaddrp,
+				    size_t *pgsizep);
 void set_pte(pte_t *ppte, unsigned long phys, enum ihk_mc_pt_attribute attr);
 pte_t *get_pte(struct page_table *pt, void *virt, enum ihk_mc_pt_attribute attr);
 
@@ -472,7 +836,6 @@ void *map_fixed_area(unsigned long phys, unsigned long size, int uncachable);
 void set_address_space_id(struct page_table *pt, int asid);
 int get_address_space_id(const struct page_table *pt);
 
-typedef pte_t translation_table_t;
 void set_translation_table(struct page_table *pt, translation_table_t* tt);
 translation_table_t* get_translation_table(const struct page_table *pt);
 translation_table_t* get_translation_table_as_paddr(const struct page_table *pt);

arch/arm64/kernel/include/arch-perfctr.h

@@ -1,9 +1,16 @@
-/* arch-perfctr.h COPYRIGHT FUJITSU LIMITED 2016-2017 */
+/* arch-perfctr.h COPYRIGHT FUJITSU LIMITED 2016-2018 */
 #ifndef __ARCH_PERFCTR_H__
 #define __ARCH_PERFCTR_H__
 
 #include <ihk/types.h>
 #include <ihk/cpu.h>
+#include <bitops.h>
+
+struct per_cpu_arm_pmu {
+	int num_events;
+#define ARMV8_PMUV3_MAX_COMMON_EVENTS 0x40
+	DECLARE_BITMAP(pmceid_bitmap, ARMV8_PMUV3_MAX_COMMON_EVENTS);
+};
 
 /* @ref.impl arch/arm64/include/asm/pmu.h */
 struct arm_pmu {
@@ -13,15 +20,22 @@ struct arm_pmu {
 	void (*reset)(void*);
 	int (*enable_pmu)(void);
 	void (*disable_pmu)(void);
-	int (*enable_counter)(int);
-	int (*disable_counter)(int);
-	int (*enable_intens)(int);
-	int (*disable_intens)(int);
+	int (*enable_counter)(unsigned long counter_mask);
+	int (*disable_counter)(unsigned long counter_mask);
+	int (*enable_intens)(unsigned long counter_mask);
+	int (*disable_intens)(unsigned long counter_mask);
 	int (*set_event_filter)(unsigned long*, int);
 	void (*write_evtype)(int, uint32_t);
-	int (*get_event_idx)(int, unsigned long);
+	int (*get_event_idx)(int num_events, unsigned long used_mask,
+			     unsigned long config);
 	int (*map_event)(uint32_t, uint64_t);
-	int num_events;
+	int (*map_hw_event)(uint64_t config);
+	int (*map_cache_event)(uint64_t config);
+	int (*map_raw_event)(uint64_t config);
+	void (*enable_user_access_pmu_regs)(void);
+	void (*disable_user_access_pmu_regs)(void);
+	int (*counter_mask_valid)(unsigned long counter_mask);
+	struct per_cpu_arm_pmu *per_cpu;
 };
 
 static inline const struct arm_pmu* get_cpu_pmu(void)
@@ -29,44 +43,21 @@ static inline const struct arm_pmu* get_cpu_pmu(void)
 	extern struct arm_pmu cpu_pmu;
 	return &cpu_pmu;
 }
+
+static inline const struct per_cpu_arm_pmu *get_per_cpu_pmu(void)
+{
+	const struct arm_pmu *cpu_pmu = get_cpu_pmu();
+
+	return &cpu_pmu->per_cpu[ihk_mc_get_processor_id()];
+}
+
 int arm64_init_perfctr(void);
+void arm64_init_per_cpu_perfctr(void);
 int arm64_enable_pmu(void);
 void arm64_disable_pmu(void);
 int armv8pmu_init(struct arm_pmu* cpu_pmu);
-
-/* TODO[PMU]: 共通部に定義があっても良い。今後の動向を見てここの定義を削除する */
-/*
- * Generalized hardware cache events:
- *
- *       { L1-D, L1-I, LLC, ITLB, DTLB, BPU, NODE } x
- *       { read, write, prefetch } x
- *       { accesses, misses }
- */
-enum perf_hw_cache_id {
-	PERF_COUNT_HW_CACHE_L1D			= 0,
-	PERF_COUNT_HW_CACHE_L1I			= 1,
-	PERF_COUNT_HW_CACHE_LL			= 2,
-	PERF_COUNT_HW_CACHE_DTLB		= 3,
-	PERF_COUNT_HW_CACHE_ITLB		= 4,
-	PERF_COUNT_HW_CACHE_BPU			= 5,
-	PERF_COUNT_HW_CACHE_NODE		= 6,
-
-	PERF_COUNT_HW_CACHE_MAX,		/* non-ABI */
-};
-
-enum perf_hw_cache_op_id {
-	PERF_COUNT_HW_CACHE_OP_READ		= 0,
-	PERF_COUNT_HW_CACHE_OP_WRITE		= 1,
-	PERF_COUNT_HW_CACHE_OP_PREFETCH		= 2,
-
-	PERF_COUNT_HW_CACHE_OP_MAX,		/* non-ABI */
-};
-
-enum perf_hw_cache_op_result_id {
-	PERF_COUNT_HW_CACHE_RESULT_ACCESS	= 0,
-	PERF_COUNT_HW_CACHE_RESULT_MISS		= 1,
-
-	PERF_COUNT_HW_CACHE_RESULT_MAX,		/* non-ABI */
-};
+void armv8pmu_per_cpu_init(struct per_cpu_arm_pmu *per_cpu);
+void arm64_enable_user_access_pmu_regs(void);
+void arm64_disable_user_access_pmu_regs(void);
 
 #endif

arch/arm64/kernel/include/arch-string.h

@@ -10,4 +10,13 @@ extern void *__inline_memcpy(void *to, const void *from, size_t t);
 
 extern void *__inline_memset(void *s, unsigned long c, size_t count);
 
+#define ARCH_MEMCLEAR
+
+extern void __memclear(void *addr, unsigned long len, void *tmp);
+inline static void memclear(void *addr, unsigned long len)
+{
+	uint64_t q0q1[4];
+	__memclear(addr, len, (void *)&q0q1);
+}
+
 #endif	/* __HEADER_ARM64_COMMON_ARCH_TIMER_H */

arch/arm64/kernel/include/arch-timer.h

@@ -1,7 +1,9 @@
-/* arch-timer.h COPYRIGHT FUJITSU LIMITED 2016 */
+/* arch-timer.h COPYRIGHT FUJITSU LIMITED 2016-2018 */
 #ifndef __HEADER_ARM64_COMMON_ARCH_TIMER_H
 #define __HEADER_ARM64_COMMON_ARCH_TIMER_H
 
+#include <ihk/cpu.h>
+
 /* @ref.impl include/clocksource/arm_arch_timer.h */
 #define ARCH_TIMER_USR_PCT_ACCESS_EN	(1 << 0)	/* physical counter */
 #define ARCH_TIMER_USR_VCT_ACCESS_EN	(1 << 1)	/* virtual counter */
@@ -11,4 +13,19 @@
 #define ARCH_TIMER_USR_VT_ACCESS_EN	(1 << 8)	/* virtual timer registers */
 #define ARCH_TIMER_USR_PT_ACCESS_EN	(1 << 9)	/* physical timer registers */
 
+/* @ref.impl linux4.10.16 */
+/* include/clocksource/arm_arch_timer.h */
+#define ARCH_TIMER_CTRL_ENABLE		(1 << 0)
+#define ARCH_TIMER_CTRL_IT_MASK		(1 << 1)
+#define ARCH_TIMER_CTRL_IT_STAT		(1 << 2)
+
+enum arch_timer_reg {
+	ARCH_TIMER_REG_CTRL,
+	ARCH_TIMER_REG_TVAL,
+};
+
+extern int get_timer_intrid(void);
+extern void arch_timer_init(void);
+extern struct ihk_mc_interrupt_handler *get_timer_handler(void);
+
 #endif	/* __HEADER_ARM64_COMMON_ARCH_TIMER_H */

arch/arm64/kernel/include/arch/cpu.h

@@ -1,4 +1,4 @@
-/* cpu.h COPYRIGHT FUJITSU LIMITED 2016-2017 */
+/* cpu.h COPYRIGHT FUJITSU LIMITED 2016-2018 */
 #ifndef __HEADER_ARM64_ARCH_CPU_H
 #define __HEADER_ARM64_ARCH_CPU_H
 
@@ -12,6 +12,8 @@
 #define dmb(opt)	asm volatile("dmb " #opt : : : "memory")
 #define dsb(opt)	asm volatile("dsb " #opt : : : "memory")
 
+#include <registers.h>
+
 #define mb()		dsb(sy)
 #define rmb()		dsb(ld)
 #define wmb()		dsb(st)
@@ -69,12 +71,10 @@ do {									\
 #define smp_mb__before_atomic()	smp_mb()
 #define smp_mb__after_atomic()	smp_mb()
 
-/* @ref.impl linux-linaro/arch/arm64/include/asm/arch_timer.h::arch_counter_get_cntvct */
 #define read_tsc()						\
 ({								\
 	unsigned long cval;					\
-	isb();							\
-	asm volatile("mrs %0, cntvct_el0" : "=r" (cval));	\
+	cval = rdtsc();						\
 	cval;							\
 })
 
@@ -102,4 +102,6 @@ static inline void cpu_disable_nmi(void)
 
 #endif	/* __ASSEMBLY__ */
 
+void arch_flush_icache_all(void);
+
 #endif /* !__HEADER_ARM64_ARCH_CPU_H */

arch/arm64/kernel/include/arch/mman.h

@@ -21,12 +21,11 @@
 
 /* Bits [26:31] are reserved, see mman-common.h for MAP_HUGETLB usage */
 #define MAP_HUGE_SHIFT  26
-#if FIRST_LEVEL_BLOCK_SUPPORT
-# define MAP_HUGE_FIRST_BLOCK (__PTL3_SHIFT << MAP_HUGE_SHIFT)
-#else
-# define MAP_HUGE_FIRST_BLOCK -1 /* not supported */
-#endif
-#define MAP_HUGE_SECOND_BLOCK (__PTL2_SHIFT << MAP_HUGE_SHIFT)
+#define MAP_HUGE_FIRST_BLOCK       (__PTL3_SHIFT << MAP_HUGE_SHIFT)
+#define MAP_HUGE_FIRST_CONT_BLOCK  (__PTL3_CONT_SHIFT << MAP_HUGE_SHIFT)
+#define MAP_HUGE_SECOND_BLOCK      (__PTL2_SHIFT << MAP_HUGE_SHIFT)
+#define MAP_HUGE_SECOND_CONT_BLOCK (__PTL2_CONT_SHIFT << MAP_HUGE_SHIFT)
+#define MAP_HUGE_THIRD_CONT_BLOCK  (__PTL1_CONT_SHIFT << MAP_HUGE_SHIFT)
 
 /*
  * for mlockall()

arch/arm64/kernel/include/arch/rusage.h

@@ -1,60 +0,0 @@
-#ifndef ARCH_RUSAGE_H_INCLUDED
-#define ARCH_RUSAGE_H_INCLUDED
-
-#define DEBUG_RUSAGE
-
-#define IHK_OS_PGSIZE_4KB 0
-#define IHK_OS_PGSIZE_2MB 1
-#define IHK_OS_PGSIZE_1GB 2
-
-extern struct ihk_os_monitor *monitor;
-
-extern int sprintf(char * buf, const char *fmt, ...);
-
-#define DEBUG_ARCH_RUSAGE
-#ifdef DEBUG_ARCH_RUSAGE
-#define	dprintf(...)											\
-	do {														\
-		char msg[1024];											\
-		sprintf(msg, __VA_ARGS__);								\
-		kprintf("%s,%s", __FUNCTION__, msg);					\
-	} while (0);
-#define	eprintf(...)											\
-	do {														\
-		char msg[1024];											\
-		sprintf(msg, __VA_ARGS__);								\
-		kprintf("%s,%s", __FUNCTION__, msg);					\
-	} while (0);
-#else
-#define dprintf(...) do {  } while (0)
-#define	eprintf(...)											\
-	do {														\
-		char msg[1024];											\
-		sprintf(msg, __VA_ARGS__);								\
-		kprintf("%s,%s", __FUNCTION__, msg);					\
-	} while (0);
-#endif
-
-static inline int rusage_pgsize_to_pgtype(size_t pgsize)
-{
-	int ret = IHK_OS_PGSIZE_4KB;
-#if 0 /* postk-TODO */
-	switch (pgsize) {
-	case PTL1_SIZE:
-		ret = IHK_OS_PGSIZE_4KB;
-		break;
-	case PTL2_SIZE:
-		ret = IHK_OS_PGSIZE_2MB;
-		break;
-	case PTL3_SIZE:
-		ret = IHK_OS_PGSIZE_1GB;
-		break;
-	default:
-		eprintf("unknown pgsize=%ld\n", pgsize);
-		break;
-	}
-#endif
-	return ret;
-}
-
-#endif /* !defined(ARCH_RUSAGE_H_INCLUDED) */

arch/arm64/kernel/include/arch/shm.h

@@ -6,12 +6,11 @@
 
 /* shmflg */
 #define SHM_HUGE_SHIFT  26
-#if FIRST_LEVEL_BLOCK_SUPPORT
-# define SHM_HUGE_FIRST_BLOCK (__PTL3_SHIFT << SHM_HUGE_SHIFT)
-#else
-# define SHM_HUGE_FIRST_BLOCK -1  /* not supported */
-#endif
-#define SHM_HUGE_SECOND_BLOCK (__PTL2_SHIFT << SHM_HUGE_SHIFT)
+#define SHM_HUGE_FIRST_BLOCK       (__PTL3_SHIFT << SHM_HUGE_SHIFT)
+#define SHM_HUGE_FIRST_CONT_BLOCK  (__PTL3_CONT_SHIFT << SHM_HUGE_SHIFT)
+#define SHM_HUGE_SECOND_BLOCK      (__PTL2_SHIFT << SHM_HUGE_SHIFT)
+#define SHM_HUGE_SECOND_CONT_BLOCK (__PTL2_CONT_SHIFT << SHM_HUGE_SHIFT)
+#define SHM_HUGE_THIRD_CONT_BLOCK  (__PTL1_CONT_SHIFT << SHM_HUGE_SHIFT)
 
 struct ipc_perm {
 	key_t		key;

arch/arm64/kernel/include/arch_rusage.h

@@ -1,34 +0,0 @@
-#ifndef ARCH_RUSAGE_H_INCLUDED
-#define ARCH_RUSAGE_H_INCLUDED
-
-#include <arch-memory.h>
-
-//#define DEBUG_RUSAGE
-
-extern struct rusage_global *rusage;
-
-#define IHK_OS_PGSIZE_4KB  0
-#define IHK_OS_PGSIZE_16KB 1
-#define IHK_OS_PGSIZE_64KB 2
-
-static inline int rusage_pgsize_to_pgtype(size_t pgsize)
-{
-	int ret = IHK_OS_PGSIZE_4KB;
-	switch (pgsize) {
-	case __PTL1_SIZE:
-		ret = IHK_OS_PGSIZE_4KB;
-		break;
-	case __PTL2_SIZE:
-		ret = IHK_OS_PGSIZE_16KB;
-		break;
-	case __PTL3_SIZE:
-		ret = IHK_OS_PGSIZE_64KB;
-		break;
-	default:
-		kprintf("%s: Error: Unknown pgsize=%ld\n", __FUNCTION__, pgsize);
-		break;
-	}
-	return ret;
-}
-
-#endif /* !defined(ARCH_RUSAGE_H_INCLUDED) */

arch/arm64/kernel/include/arm-gic-v2.h

@@ -60,9 +60,9 @@
 
 #ifdef CONFIG_HAS_NMI
 #define GICD_INT_NMI_PRI		0x40
-#define GICD_INT_DEF_PRI		0xc0
+#define GICD_INT_DEF_PRI		0xc0U
 #else
-#define GICD_INT_DEF_PRI		0xa0
+#define GICD_INT_DEF_PRI		0xa0U
 #endif
 #define GICD_INT_DEF_PRI_X4		((GICD_INT_DEF_PRI << 24) |\
 					(GICD_INT_DEF_PRI << 16) |\

arch/arm64/kernel/include/arm-gic-v3.h

@@ -19,6 +19,7 @@
 #ifndef __LINUX_IRQCHIP_ARM_GIC_V3_H
 #define __LINUX_IRQCHIP_ARM_GIC_V3_H
 
+#include <stringify.h>
 /* @ref.impl include/linux/irqchip/arm-gic-v3.h */
 #include <sysreg.h>
 
@@ -381,11 +382,4 @@
 #define ICH_AP1R2_EL2			__AP1Rx_EL2(2)
 #define ICH_AP1R3_EL2			__AP1Rx_EL2(3)
 
-
-/**
- * @ref.impl host-kernel/include/linux/stringify.h
- */
-#define __stringify_1(x...)	#x
-#define __stringify(x...)	__stringify_1(x)
-
 #endif /* __LINUX_IRQCHIP_ARM_GIC_V3_H */

arch/arm64/kernel/include/asm-offsets.h

@@ -15,8 +15,9 @@
 #define S_PC		0x100	/* offsetof(struct pt_regs, pc) */
 #define S_PSTATE	0x108	/* offsetof(struct pt_regs, pstate) */
 #define S_ORIG_X0	0x110	/* offsetof(struct pt_regs, orig_x0) */
-#define S_SYSCALLNO	0x118	/* offsetof(struct pt_regs, syscallno) */
-#define S_FRAME_SIZE	0x120	/* sizeof(struct pt_regs) */
+#define S_ORIG_PC	0x118	/* offsetof(struct pt_regs, orig_pc) */
+#define S_SYSCALLNO	0x120	/* offsetof(struct pt_regs, syscallno) */
+#define S_FRAME_SIZE	0x130	/* sizeof(struct pt_regs) must be 16 byte align */
 
 #define CPU_INFO_SETUP	0x10	/* offsetof(struct cpu_info, cpu_setup) */
 #define CPU_INFO_SZ	0x18	/* sizeof(struct cpu_info) */

arch/arm64/kernel/include/asm-syscall.h

@@ -0,0 +1,20 @@
+/* asm-syscall.h COPYRIGHT FUJITSU LIMITED 2018 */
+#ifndef __HEADER_ARM64_ASM_SYSCALL_H
+#define __HEADER_ARM64_ASM_SYSCALL_H
+
+#ifdef __ASSEMBLY__
+
+#define DECLARATOR(number, name)	.equ __NR_##name, number
+#define SYSCALL_HANDLED(number, name)	DECLARATOR(number, name)
+#define SYSCALL_DELEGATED(number, name)	DECLARATOR(number, name)
+
+#include <config.h>
+#include <syscall_list.h>
+
+#undef DECLARATOR
+#undef SYSCALL_HANDLED
+#undef SYSCALL_DELEGATED
+
+#endif /* __ASSEMBLY__ */
+
+#endif	/* !__HEADER_ARM64_ASM_SYSCALL_H */

arch/arm64/kernel/include/cpufeature.h

@@ -67,21 +67,12 @@ struct arm64_cpu_capabilities {
 	int def_scope;/* default scope */
 	int (*matches)(const struct arm64_cpu_capabilities *caps, int scope);
 	int (*enable)(void *);/* Called on all active CPUs */
-	union {
-		struct {/* To be used for erratum handling only */
-			uint32_t midr_model;
-			uint32_t midr_range_min, midr_range_max;
-		};
-
-		struct {/* Feature register checking */
-			uint32_t sys_reg;
-			uint8_t field_pos;
-			uint8_t min_field_value;
-			uint8_t hwcap_type;
-			int sign;
-			unsigned long hwcap;
-		};
-	};
+	uint32_t sys_reg;
+	uint8_t field_pos;
+	uint8_t min_field_value;
+	uint8_t hwcap_type;
+	int sign;
+	unsigned long hwcap;
 };
 
 /* @ref.impl include/linux/bitops.h */

arch/arm64/kernel/include/cputype.h

@@ -25,17 +25,78 @@
 #define MIDR_PARTNUM(midr)	\
 	(((midr) & MIDR_PARTNUM_MASK) >> MIDR_PARTNUM_SHIFT)
 
+#define MIDR_ARCHITECTURE_SHIFT	16
+#define MIDR_ARCHITECTURE_MASK	(0xf << MIDR_ARCHITECTURE_SHIFT)
+#define MIDR_ARCHITECTURE(midr)	\
+	(((midr) & MIDR_ARCHITECTURE_MASK) >> MIDR_ARCHITECTURE_SHIFT)
+
 #define MIDR_VARIANT_SHIFT	20
 #define MIDR_VARIANT_MASK	(0xf << MIDR_VARIANT_SHIFT)
 #define MIDR_VARIANT(midr)	\
 	(((midr) & MIDR_VARIANT_MASK) >> MIDR_VARIANT_SHIFT)
 
 #define MIDR_IMPLEMENTOR_SHIFT	24
-#define MIDR_IMPLEMENTOR_MASK	(0xff << MIDR_IMPLEMENTOR_SHIFT)
+#define MIDR_IMPLEMENTOR_MASK	(0xffU << MIDR_IMPLEMENTOR_SHIFT)
 #define MIDR_IMPLEMENTOR(midr)	\
 	(((midr) & MIDR_IMPLEMENTOR_MASK) >> MIDR_IMPLEMENTOR_SHIFT)
 
-#define ARM_CPU_IMP_CAVIUM	0x43
+#define MIDR_CPU_MODEL(imp, partnum) \
+	(((imp)			<< MIDR_IMPLEMENTOR_SHIFT) | \
+	(0xf			<< MIDR_ARCHITECTURE_SHIFT) | \
+	((partnum)		<< MIDR_PARTNUM_SHIFT))
+
+#define MIDR_CPU_VAR_REV(var, rev) \
+	(((var)	<< MIDR_VARIANT_SHIFT) | (rev))
+
+#define MIDR_CPU_MODEL_MASK (MIDR_IMPLEMENTOR_MASK | MIDR_PARTNUM_MASK | \
+			     MIDR_ARCHITECTURE_MASK)
+
+#define MIDR_IS_CPU_MODEL_RANGE(midr, model, rv_min, rv_max)		\
+({									\
+	u32 _model = (midr) & MIDR_CPU_MODEL_MASK;			\
+	u32 rv = (midr) & (MIDR_REVISION_MASK | MIDR_VARIANT_MASK);	\
+									\
+	_model == (model) && rv >= (rv_min) && rv <= (rv_max);		\
+})
+
+#define ARM_CPU_IMP_ARM			0x41
+#define ARM_CPU_IMP_APM			0x50
+#define ARM_CPU_IMP_CAVIUM		0x43
+#define ARM_CPU_IMP_BRCM		0x42
+#define ARM_CPU_IMP_QCOM		0x51
+
+#define ARM_CPU_PART_AEM_V8		0xD0F
+#define ARM_CPU_PART_FOUNDATION		0xD00
+#define ARM_CPU_PART_CORTEX_A57		0xD07
+#define ARM_CPU_PART_CORTEX_A72		0xD08
+#define ARM_CPU_PART_CORTEX_A53		0xD03
+#define ARM_CPU_PART_CORTEX_A73		0xD09
+#define ARM_CPU_PART_CORTEX_A75		0xD0A
+
+#define APM_CPU_PART_POTENZA		0x000
+
+#define CAVIUM_CPU_PART_THUNDERX	0x0A1
+#define CAVIUM_CPU_PART_THUNDERX_81XX	0x0A2
+#define CAVIUM_CPU_PART_THUNDERX_83XX	0x0A3
+#define CAVIUM_CPU_PART_THUNDERX2	0x0AF
+
+#define BRCM_CPU_PART_VULCAN		0x516
+
+#define QCOM_CPU_PART_FALKOR_V1		0x800
+#define QCOM_CPU_PART_FALKOR		0xC00
+
+#define MIDR_CORTEX_A53 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53)
+#define MIDR_CORTEX_A57 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57)
+#define MIDR_CORTEX_A72 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A72)
+#define MIDR_CORTEX_A73 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A73)
+#define MIDR_CORTEX_A75 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A75)
+#define MIDR_THUNDERX	MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX)
+#define MIDR_THUNDERX_81XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX)
+#define MIDR_THUNDERX_83XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_83XX)
+#define MIDR_CAVIUM_THUNDERX2 MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX2)
+#define MIDR_BRCM_VULCAN MIDR_CPU_MODEL(ARM_CPU_IMP_BRCM, BRCM_CPU_PART_VULCAN)
+#define MIDR_QCOM_FALKOR_V1 MIDR_CPU_MODEL(ARM_CPU_IMP_QCOM, QCOM_CPU_PART_FALKOR_V1)
+#define MIDR_QCOM_FALKOR MIDR_CPU_MODEL(ARM_CPU_IMP_QCOM, QCOM_CPU_PART_FALKOR)
 
 #ifndef __ASSEMBLY__
 

arch/arm64/kernel/include/elfcore.h

@@ -1,92 +0,0 @@
-/* elfcore.h COPYRIGHT FUJITSU LIMITED 2015 */
-#ifndef POSTK_DEBUG_ARCH_DEP_18 /* coredump arch separation. */
-#ifndef __HEADER_ARM64_COMMON_ELFCORE_H
-#define __HEADER_ARM64_COMMON_ELFCORE_H
-
-typedef	uint16_t	Elf64_Half;
-typedef	uint32_t	Elf64_Word;
-typedef	uint64_t	Elf64_Xword;
-typedef	uint64_t	Elf64_Addr;
-typedef	uint64_t	Elf64_Off;
-
-#define	EI_NIDENT	16
-
-typedef struct {
-	unsigned char e_ident[EI_NIDENT];
-	Elf64_Half e_type;
-	Elf64_Half e_machine;
-	Elf64_Word e_version;
-	Elf64_Addr e_entry;
-	Elf64_Off e_phoff;
-	Elf64_Off e_shoff;
-	Elf64_Word e_flags;
-	Elf64_Half e_ehsize;
-	Elf64_Half e_phentsize;
-	Elf64_Half e_phnum;
-	Elf64_Half e_shentsize;
-	Elf64_Half e_shnum;
-	Elf64_Half e_shstrndx;
-} Elf64_Ehdr;
-
-#define	EI_MAG0		0
-#define	EI_MAG1		1
-#define EI_MAG2		2
-#define	EI_MAG3		3
-#define	EI_CLASS	4
-#define EI_DATA		5
-#define	EI_VERSION	6
-#define	EI_OSABI	7
-#define	EI_ABIVERSION	8
-#define EI_PAD		9
-
-
-#define	ELFMAG0		0x7f
-#define	ELFMAG1		'E'
-#define	ELFMAG2		'L'
-#define	ELFMAG3		'F'
-
-#define	ELFCLASS64	2	/* 64-bit object */
-#define	ELFDATA2LSB	1	/* LSB */
-#define	El_VERSION	1	/* defined to be the same as EV CURRENT */
-#define	ELFOSABI_NONE	0	/* unspecied */
-#define	El_ABIVERSION_NONE	0	/* unspecied */
-#define	ET_CORE		4	/* Core file */
-#define	EM_X86_64	62	/* AMD x86-64 architecture */
-#define	EM_K10M		181	/* Intel K10M */
-#define	EV_CURRENT	1	/* Current version */
-
-typedef struct {
-	Elf64_Word p_type;
-	Elf64_Word p_flags;
-	Elf64_Off p_offset;
-	Elf64_Addr p_vaddr;
-	Elf64_Addr p_paddr;
-	Elf64_Xword p_filesz;
-	Elf64_Xword p_memsz;
-	Elf64_Xword p_align;
-} Elf64_Phdr;
-
-#define	PT_LOAD		1
-#define	PT_NOTE		4
-
-#define	PF_X		1	/* executable bit */
-#define	PF_W		2	/* writable bit */
-#define	PF_R		4	/* readable bit */
-
-struct note {
-	Elf64_Word namesz;
-	Elf64_Word descsz;
-	Elf64_Word type;
-	/* name char[namesz] and desc[descsz] */
-};
-
-#define	NT_PRSTATUS	1
-#define	NT_PRFRPREG	2
-#define	NT_PRPSINFO	3
-#define	NT_AUXV		6
-#define	NT_X86_STATE	0x202
-
-#include "elfcoregpl.h"
-
-#endif /* !__HEADER_ARM64_COMMON_ELFCORE_H */
-#endif /* !POSTK_DEBUG_ARCH_DEP_18 */

arch/arm64/kernel/include/elfcoregpl.h

@@ -1,98 +0,0 @@
-/* elfcoregpl.h COPYRIGHT FUJITSU LIMITED 2015 */
-#ifndef POSTK_DEBUG_ARCH_DEP_18 /* coredump arch separation. */
-#ifndef __HEADER_ARM64_COMMON_ELFCOREGPL_H
-#define __HEADER_ARM64_COMMON_ELFCOREGPL_H
-
-#define	pid_t		int
-
-/* From /usr/include/linux/elfcore.h of Linux */
-
-#define	ELF_PRARGSZ	(80)
-
-/* From /usr/include/linux/elfcore.h fro Linux */
-
-struct elf_siginfo
-{
-	int si_signo;
-	int si_code;
-	int si_errno;
-};
-
-/* From bfd/hosts/x86-64linux.h of gdb. */
-
-typedef uint64_t __attribute__ ((__aligned__ (8))) a8_uint64_t;
-typedef	a8_uint64_t elf_greg64_t;
-
-struct user_regs64_struct
-{
-	a8_uint64_t r15;
-	a8_uint64_t r14;
-	a8_uint64_t r13;
-	a8_uint64_t r12;
-	a8_uint64_t rbp;
-	a8_uint64_t rbx;
-	a8_uint64_t r11;
-	a8_uint64_t r10;
-	a8_uint64_t r9;
-	a8_uint64_t r8;
-	a8_uint64_t rax;
-	a8_uint64_t rcx;
-	a8_uint64_t rdx;
-	a8_uint64_t rsi;
-	a8_uint64_t rdi;
-	a8_uint64_t orig_rax;
-	a8_uint64_t rip;
-	a8_uint64_t cs;
-	a8_uint64_t eflags;
-	a8_uint64_t rsp;
-	a8_uint64_t ss;
-	a8_uint64_t fs_base;
-	a8_uint64_t gs_base;
-	a8_uint64_t ds;
-	a8_uint64_t es;
-	a8_uint64_t fs;
-	a8_uint64_t gs;
-};
-
-#define ELF_NGREG64 (sizeof (struct user_regs64_struct) / sizeof(elf_greg64_t))
-
-typedef elf_greg64_t elf_gregset64_t[ELF_NGREG64];
-
-struct prstatus64_timeval
-{
-	a8_uint64_t tv_sec;
-	a8_uint64_t tv_usec;
-};
-struct elf_prstatus64
-{
-	struct elf_siginfo pr_info;
-	short int pr_cursig;
-	a8_uint64_t pr_sigpend;
-	a8_uint64_t pr_sighold;
-	pid_t pr_pid;
-	pid_t pr_ppid;
-	pid_t pr_pgrp;
-	pid_t pr_sid;
-	struct prstatus64_timeval pr_utime;
-	struct prstatus64_timeval pr_stime;
-	struct prstatus64_timeval pr_cutime;
-	struct prstatus64_timeval pr_cstime;
-	elf_gregset64_t pr_reg;
-	int pr_fpvalid;
-};
-struct elf_prpsinfo64
-{
-	char pr_state;
-	char pr_sname;
-	char pr_zomb;
-	char pr_nice;
-	a8_uint64_t pr_flag;
-	unsigned int pr_uid;
-	unsigned int pr_gid;
-	int pr_pid, pr_ppid, pr_pgrp, pr_sid;
-	char pr_fname[16];
-	char pr_psargs[ELF_PRARGSZ];
-};
-
-#endif /* !__HEADER_ARM64_COMMON_ELFCOREGPL_H */
-#endif /* !POSTK_DEBUG_ARCH_DEP_18 */

arch/arm64/kernel/include/fpsimd.h

@@ -1,4 +1,4 @@
-/* fpsimd.h COPYRIGHT FUJITSU LIMITED 2016-2017 */
+/* fpsimd.h COPYRIGHT FUJITSU LIMITED 2016-2019 */
 #ifndef __HEADER_ARM64_COMMON_FPSIMD_H
 #define __HEADER_ARM64_COMMON_FPSIMD_H
 
@@ -42,16 +42,19 @@ extern void thread_sve_to_fpsimd(struct thread *thread, fp_regs_struct *fp_regs)
 
 extern size_t sve_state_size(struct thread const *thread);
 extern void sve_free(struct thread *thread);
-extern void sve_alloc(struct thread *thread);
+extern int sve_alloc(struct thread *thread);
 extern void sve_save_state(void *state, unsigned int *pfpsr);
 extern void sve_load_state(void const *state, unsigned int const *pfpsr, unsigned long vq_minus_1);
 extern unsigned int sve_get_vl(void);
-extern int sve_set_thread_vl(struct thread *thread, const unsigned long vector_length, const unsigned long flags);
-extern int sve_get_thread_vl(const struct thread *thread);
+extern int sve_set_thread_vl(unsigned long arg);
+extern int sve_get_thread_vl(void);
 extern int sve_set_vector_length(struct thread *thread, unsigned long vl, unsigned long flags);
 
-#define SVE_SET_VL(thread, vector_length, flags)	sve_set_thread_vl(thread, vector_length, flags)
-#define SVE_GET_VL(thread)				sve_get_thread_vl(thread)
+#define SVE_SET_VL(arg)	sve_set_thread_vl(arg)
+#define SVE_GET_VL()	sve_get_thread_vl()
+
+/* Maximum VL that SVE VL-agnostic software can transparently support */
+#define SVE_VL_ARCH_MAX 0x100
 
 #else /* CONFIG_ARM64_SVE */
 
@@ -80,12 +83,12 @@ static int sve_set_vector_length(struct thread *thread, unsigned long vl, unsign
 }
 
 /* for prctl syscall */
-#define SVE_SET_VL(a,b,c)	(-EINVAL)
-#define SVE_GET_VL(a)		(-EINVAL)
+#define SVE_SET_VL(a)	(-EINVAL)
+#define SVE_GET_VL()	(-EINVAL)
 
 #endif /* CONFIG_ARM64_SVE */
 
-extern void init_sve_vl(void);
+extern void sve_setup(void);
 extern void fpsimd_save_state(struct fpsimd_state *state);
 extern void fpsimd_load_state(struct fpsimd_state *state);
 extern void thread_fpsimd_save(struct thread *thread);

arch/arm64/kernel/include/hwcap.h

@@ -1,5 +1,4 @@
 /* hwcap.h COPYRIGHT FUJITSU LIMITED 2017 */
-#ifdef POSTK_DEBUG_ARCH_DEP_65
 #ifndef _UAPI__ASM_HWCAP_H
 #define _UAPI__ASM_HWCAP_H
 
@@ -25,4 +24,3 @@ unsigned long arch_get_hwcap(void);
 extern unsigned long elf_hwcap;
 
 #endif /* _UAPI__ASM_HWCAP_H */
-#endif /* POSTK_DEBUG_ARCH_DEP_65 */

arch/arm64/kernel/include/ihk/atomic.h

@@ -124,7 +124,7 @@ static inline long ihk_atomic64_read(const ihk_atomic64_t *v)
 	return *(volatile long *)&(v)->counter64;
 }
 
-static inline void ihk_atomic64_set(ihk_atomic64_t *v, int i)
+static inline void ihk_atomic64_set(ihk_atomic64_t *v, long i)
 {
 	v->counter64 = i;
 }
@@ -147,6 +147,8 @@ static inline void ihk_atomic64_add(long i, ihk_atomic64_t *v)
 /* @ref.impl arch/arm64/include/asm/atomic.h::atomic64_inc */
 #define ihk_atomic64_inc(v)			ihk_atomic64_add(1LL, (v))
 
+#define ihk_atomic64_cmpxchg(p, o, n) cmpxchg(&((p)->counter64), o, n)
+
 /***********************************************************************
  * others
  */

arch/arm64/kernel/include/ihk/context.h

@@ -1,4 +1,4 @@
-/* context.h COPYRIGHT FUJITSU LIMITED 2015-2017 */
+/* context.h COPYRIGHT FUJITSU LIMITED 2015-2018 */
 #ifndef __HEADER_ARM64_IHK_CONTEXT_H
 #define __HEADER_ARM64_IHK_CONTEXT_H
 
@@ -27,7 +27,9 @@ struct pt_regs {
 		};
 	};
 	unsigned long orig_x0;
+	unsigned long orig_pc;
 	unsigned long syscallno;
+	unsigned long __padding;
 };
 
 typedef struct pt_regs ihk_mc_user_context_t;
@@ -65,17 +67,17 @@ static inline void pt_regs_write_reg(struct pt_regs *regs, int r,
 }
 
 /* temp */
-#define ihk_mc_syscall_arg0(uc) (uc)->regs[0]
-#define ihk_mc_syscall_arg1(uc) (uc)->regs[1]
-#define ihk_mc_syscall_arg2(uc) (uc)->regs[2]
-#define ihk_mc_syscall_arg3(uc) (uc)->regs[3]
-#define ihk_mc_syscall_arg4(uc) (uc)->regs[4]
-#define ihk_mc_syscall_arg5(uc) (uc)->regs[5]
+#define ihk_mc_syscall_arg0(uc)		((uc)->regs[0])
+#define ihk_mc_syscall_arg1(uc)		((uc)->regs[1])
+#define ihk_mc_syscall_arg2(uc)		((uc)->regs[2])
+#define ihk_mc_syscall_arg3(uc)		((uc)->regs[3])
+#define ihk_mc_syscall_arg4(uc)		((uc)->regs[4])
+#define ihk_mc_syscall_arg5(uc)		((uc)->regs[5])
 
-#define ihk_mc_syscall_ret(uc)  (uc)->regs[0]
-#define ihk_mc_syscall_number(uc)  (uc)->regs[8]
+#define ihk_mc_syscall_ret(uc)		((uc)->regs[0])
+#define ihk_mc_syscall_number(uc)	((uc)->regs[8])
 
-#define ihk_mc_syscall_pc(uc)   (uc)->pc
-#define ihk_mc_syscall_sp(uc)   (uc)->sp
+#define ihk_mc_syscall_pc(uc)		((uc)->pc)
+#define ihk_mc_syscall_sp(uc)		((uc)->sp)
 
 #endif /* !__HEADER_ARM64_IHK_CONTEXT_H */

arch/arm64/kernel/include/ihk/types.h

@@ -20,13 +20,11 @@ typedef uint64_t           size_t;
 typedef int64_t            ssize_t;
 typedef int64_t            off_t;
 
-#ifdef POSTK_DEBUG_ARCH_DEP_18 /* coredump arch separation. */
 typedef int32_t            key_t;
 typedef uint32_t           uid_t;
 typedef uint32_t           gid_t;
 typedef int64_t            time_t;
 typedef int32_t            pid_t;
-#endif /* POSTK_DEBUG_ARCH_DEP_18 */
 
 #endif /* __ASSEMBLY__ */
 

arch/arm64/kernel/include/imp-sysreg.h

@@ -0,0 +1,103 @@
+/* imp-sysreg.h COPYRIGHT FUJITSU LIMITED 2016-2018 */
+#ifndef __ASM_IMP_SYSREG_H
+#define __ASM_IMP_SYSREG_H
+
+#ifndef __ASSEMBLY__
+
+/* register sys_reg list */
+#define IMP_FJ_TAG_ADDRESS_CTRL_EL1		sys_reg(3, 0, 11, 2, 0)
+#define IMP_SCCR_CTRL_EL1			sys_reg(3, 0, 11, 8, 0)
+#define IMP_SCCR_ASSIGN_EL1			sys_reg(3, 0, 11, 8, 1)
+#define IMP_SCCR_SET0_L2_EL1			sys_reg(3, 0, 15, 8, 2)
+#define IMP_SCCR_SET1_L2_EL1			sys_reg(3, 0, 15, 8, 3)
+#define IMP_SCCR_L1_EL0				sys_reg(3, 3, 11, 8, 2)
+#define IMP_PF_CTRL_EL1				sys_reg(3, 0, 11, 4, 0)
+#define IMP_PF_STREAM_DETECT_CTRL_EL0		sys_reg(3, 3, 11, 4, 0)
+#define IMP_PF_INJECTION_CTRL0_EL0		sys_reg(3, 3, 11, 6, 0)
+#define IMP_PF_INJECTION_CTRL1_EL0		sys_reg(3, 3, 11, 6, 1)
+#define IMP_PF_INJECTION_CTRL2_EL0		sys_reg(3, 3, 11, 6, 2)
+#define IMP_PF_INJECTION_CTRL3_EL0		sys_reg(3, 3, 11, 6, 3)
+#define IMP_PF_INJECTION_CTRL4_EL0		sys_reg(3, 3, 11, 6, 4)
+#define IMP_PF_INJECTION_CTRL5_EL0		sys_reg(3, 3, 11, 6, 5)
+#define IMP_PF_INJECTION_CTRL6_EL0		sys_reg(3, 3, 11, 6, 6)
+#define IMP_PF_INJECTION_CTRL7_EL0		sys_reg(3, 3, 11, 6, 7)
+#define IMP_PF_INJECTION_DISTANCE0_EL0		sys_reg(3, 3, 11, 7, 0)
+#define IMP_PF_INJECTION_DISTANCE1_EL0		sys_reg(3, 3, 11, 7, 1)
+#define IMP_PF_INJECTION_DISTANCE2_EL0		sys_reg(3, 3, 11, 7, 2)
+#define IMP_PF_INJECTION_DISTANCE3_EL0		sys_reg(3, 3, 11, 7, 3)
+#define IMP_PF_INJECTION_DISTANCE4_EL0		sys_reg(3, 3, 11, 7, 4)
+#define IMP_PF_INJECTION_DISTANCE5_EL0		sys_reg(3, 3, 11, 7, 5)
+#define IMP_PF_INJECTION_DISTANCE6_EL0		sys_reg(3, 3, 11, 7, 6)
+#define IMP_PF_INJECTION_DISTANCE7_EL0		sys_reg(3, 3, 11, 7, 7)
+#define IMP_PF_PMUSERENR_EL0			sys_reg(3, 3, 9, 14, 0)
+#define IMP_BARRIER_CTRL_EL1			sys_reg(3, 0, 11, 12, 0)
+#define IMP_BARRIER_BST_BIT_EL1			sys_reg(3, 0, 11, 12, 4)
+#define IMP_BARRIER_INIT_SYNC_BB0_EL1		sys_reg(3, 0, 15, 13, 0)
+#define IMP_BARRIER_INIT_SYNC_BB1_EL1		sys_reg(3, 0, 15, 13, 1)
+#define IMP_BARRIER_INIT_SYNC_BB2_EL1		sys_reg(3, 0, 15, 13, 2)
+#define IMP_BARRIER_INIT_SYNC_BB3_EL1		sys_reg(3, 0, 15, 13, 3)
+#define IMP_BARRIER_INIT_SYNC_BB4_EL1		sys_reg(3, 0, 15, 13, 4)
+#define IMP_BARRIER_INIT_SYNC_BB5_EL1		sys_reg(3, 0, 15, 13, 5)
+#define IMP_BARRIER_ASSIGN_SYNC_W0_EL1		sys_reg(3, 0, 15, 15, 0)
+#define IMP_BARRIER_ASSIGN_SYNC_W1_EL1		sys_reg(3, 0, 15, 15, 1)
+#define IMP_BARRIER_ASSIGN_SYNC_W2_EL1		sys_reg(3, 0, 15, 15, 2)
+#define IMP_BARRIER_ASSIGN_SYNC_W3_EL1		sys_reg(3, 0, 15, 15, 3)
+#define IMP_SOC_STANDBY_CTRL_EL1		sys_reg(3, 0, 11, 0, 0)
+#define IMP_FJ_CORE_UARCH_CTRL_EL2		sys_reg(3, 4, 11, 0, 4)
+#define IMP_FJ_CORE_UARCH_RESTRECTION_EL1	sys_reg(3, 0, 11, 0, 5)
+
+/* macros */
+#define PWR_REG_MASK(reg, feild)  (((UL(1) << ((reg##_##feild##_MSB) - (reg##_##feild##_LSB) + 1)) - 1) << (reg##_##feild##_LSB))
+
+/* IMP_FJ_TAG_ADDRESS_CTRL_EL1 */
+#define IMP_FJ_TAG_ADDRESS_CTRL_EL1_TBO0_SHIFT	(0)
+#define IMP_FJ_TAG_ADDRESS_CTRL_EL1_SEC0_SHIFT	(8)
+#define IMP_FJ_TAG_ADDRESS_CTRL_EL1_PFE0_SHIFT	(9)
+#define IMP_FJ_TAG_ADDRESS_CTRL_EL1_TBO0_MASK	(1UL << IMP_FJ_TAG_ADDRESS_CTRL_EL1_TBO0_SHIFT)
+#define IMP_FJ_TAG_ADDRESS_CTRL_EL1_SEC0_MASK	(1UL << IMP_FJ_TAG_ADDRESS_CTRL_EL1_SEC0_SHIFT)
+#define IMP_FJ_TAG_ADDRESS_CTRL_EL1_PFE0_MASK	(1UL << IMP_FJ_TAG_ADDRESS_CTRL_EL1_PFE0_SHIFT)
+
+/* IMP_SCCR_CTRL_EL1 */
+#define IMP_SCCR_CTRL_EL1_EL1AE_SHIFT		(63)
+#define IMP_SCCR_CTRL_EL1_EL1AE_MASK		(1UL << IMP_SCCR_CTRL_EL1_EL1AE_SHIFT)
+
+/* IMP_SCCR_SET0_L2_EL1 */
+#define IMP_SCCR_SET0_L2_EL1_L2_SEC0_SHIFT	(0)
+
+/* IMP_PF_CTRL_EL1 */
+#define IMP_PF_CTRL_EL1_EL1AE_ENABLE		(1UL << 63)
+#define IMP_PF_CTRL_EL1_EL0AE_ENABLE		(1UL << 62)
+
+/* IMP_BARRIER_CTRL_EL1 */
+#define IMP_BARRIER_CTRL_EL1_EL1AE_ENABLE	(1UL << 63)
+#define IMP_BARRIER_CTRL_EL1_EL0AE_ENABLE	(1UL << 62)
+
+/* IMP_SOC_STANDBY_CTRL_EL1 */
+#define IMP_SOC_STANDBY_CTRL_EL1_ECO_MODE_MSB		2
+#define IMP_SOC_STANDBY_CTRL_EL1_ECO_MODE_LSB		2
+#define IMP_SOC_STANDBY_CTRL_EL1_MODE_CHANGE_MSB	1
+#define IMP_SOC_STANDBY_CTRL_EL1_MODE_CHANGE_LSB	1
+#define IMP_SOC_STANDBY_CTRL_EL1_RETENTION_MSB		0
+#define IMP_SOC_STANDBY_CTRL_EL1_RETENTION_LSB		0
+#define IMP_SOC_STANDBY_CTRL_EL1_ECO_MODE		PWR_REG_MASK(IMP_SOC_STANDBY_CTRL_EL1, ECO_MODE)
+#define IMP_SOC_STANDBY_CTRL_EL1_MODE_CHANGE		PWR_REG_MASK(IMP_SOC_STANDBY_CTRL_EL1, MODE_CHANGE)
+#define IMP_SOC_STANDBY_CTRL_EL1_RETENTION		PWR_REG_MASK(IMP_SOC_STANDBY_CTRL_EL1, RETENTION)
+
+/* IMP_FJ_CORE_UARCH_RESTRECTION_EL1 */
+#define IMP_FJ_CORE_UARCH_RESTRECTION_EL1_FL_RESTRICT_TRANS_MSB	33
+#define IMP_FJ_CORE_UARCH_RESTRECTION_EL1_FL_RESTRICT_TRANS_LSB	33
+#define IMP_FJ_CORE_UARCH_RESTRECTION_EL1_ISSUE_RESTRICTION_MSB	9
+#define IMP_FJ_CORE_UARCH_RESTRECTION_EL1_ISSUE_RESTRICTION_LSB	8
+#define IMP_FJ_CORE_UARCH_RESTRECTION_EL1_EX_RESTRICTION_MSB	0
+#define IMP_FJ_CORE_UARCH_RESTRECTION_EL1_EX_RESTRICTION_LSB	0
+#define IMP_FJ_CORE_UARCH_RESTRECTION_EL1_FL_RESTRICT_TRANS	PWR_REG_MASK(IMP_FJ_CORE_UARCH_RESTRECTION_EL1, FL_RESTRICT_TRANS)
+#define IMP_FJ_CORE_UARCH_RESTRECTION_EL1_ISSUE_RESTRICTION	PWR_REG_MASK(IMP_FJ_CORE_UARCH_RESTRECTION_EL1, ISSUE_RESTRICTION)
+#define IMP_FJ_CORE_UARCH_RESTRECTION_EL1_EX_RESTRICTION	PWR_REG_MASK(IMP_FJ_CORE_UARCH_RESTRECTION_EL1, EX_RESTRICTION)
+
+void scdrv_registers_init(void);
+void hpc_registers_init(void);
+void vhbm_barrier_registers_init(void);
+
+#endif /* __ASSEMBLY__ */
+
+#endif	/* __ASM_IMP_SYSREG_H */

arch/arm64/kernel/include/io.h

@@ -80,6 +80,10 @@ static inline uint64_t __raw_readq(const volatile void *addr)
 	return val;
 }
 
+/* IO barriers */
+#define __iormb()		rmb()
+#define __iowmb()		wmb()
+
 /*
  * Relaxed I/O memory access primitives. These follow the Device memory
  * ordering rules but do not guarantee any ordering relative to Normal memory
@@ -95,5 +99,20 @@ static inline uint64_t __raw_readq(const volatile void *addr)
 #define writel_relaxed(v,c)	((void)__raw_writel((uint32_t)(v),(c)))
 #define writeq_relaxed(v,c)	((void)__raw_writeq((uint64_t)(v),(c)))
 
+/*
+ * I/O memory access primitives. Reads are ordered relative to any
+ * following Normal memory access. Writes are ordered relative to any prior
+ * Normal memory access.
+ */
+#define readb(c)		({ uint8_t  __v = readb_relaxed(c); __iormb(); __v; })
+#define readw(c)		({ uint16_t __v = readw_relaxed(c); __iormb(); __v; })
+#define readl(c)		({ uint32_t __v = readl_relaxed(c); __iormb(); __v; })
+#define readq(c)		({ uint64_t __v = readq_relaxed(c); __iormb(); __v; })
+
+#define writeb(v,c)		({ __iowmb(); writeb_relaxed((v),(c)); })
+#define writew(v,c)		({ __iowmb(); writew_relaxed((v),(c)); })
+#define writel(v,c)		({ __iowmb(); writel_relaxed((v),(c)); })
+#define writeq(v,c)		({ __iowmb(); writeq_relaxed((v),(c)); })
+
 #endif /* __KERNEL__ */
 #endif /* __ASM_IO_H */

arch/arm64/kernel/include/irq.h

@@ -1,4 +1,4 @@
-/* irq.h COPYRIGHT FUJITSU LIMITED 2015-2017 */
+/* irq.h COPYRIGHT FUJITSU LIMITED 2015-2019 */
 
 #ifndef __HEADER_ARM64_IRQ_H
 #define __HEADER_ARM64_IRQ_H
@@ -14,48 +14,24 @@
 #define INTRID_QUERY_FREE_MEM	2
 #define INTRID_CPU_STOP		3
 #define INTRID_TLB_FLUSH	4
-#define INTRID_STACK_TRACE	6
-#define INTRID_MEMDUMP		7
+#define INTRID_STACK_TRACE	5
+#define INTRID_MULTI_INTR	6
+#define INTRID_MULTI_NMI	7
+#define LOCAL_SMP_FUNC_CALL_VECTOR   1 /* same as IKC */
 
 /* use PPI interrupt number */
+#define INTRID_PERF_OVF		23
 #define INTRID_HYP_PHYS_TIMER	26 /* cnthp */
 #define INTRID_VIRT_TIMER	27 /* cntv */
 #define INTRID_HYP_VIRT_TIMER	28 /* cnthv */
 #define INTRID_PHYS_TIMER	30 /* cntp */
 
-/* timer intrid getter */
-static int get_virt_timer_intrid(void)
-{
-#ifdef CONFIG_ARM64_VHE
-	unsigned long mmfr = read_cpuid(ID_AA64MMFR1_EL1);
-
-	if ((mmfr >> ID_AA64MMFR1_VHE_SHIFT) & 1UL) {
-		return INTRID_HYP_VIRT_TIMER;
-	}
-#endif /* CONFIG_ARM64_VHE */
-	return INTRID_VIRT_TIMER;
-}
-
-static int get_phys_timer_intrid(void)
-{
-#ifdef CONFIG_ARM64_VHE
-	unsigned long mmfr = read_cpuid(ID_AA64MMFR1_EL1);
-
-	if ((mmfr >> ID_AA64MMFR1_VHE_SHIFT) & 1UL) {
-		return INTRID_HYP_PHYS_TIMER;
-	}
-#endif /* CONFIG_ARM64_VHE */
-	return INTRID_PHYS_TIMER;
-}
-
-/* use timer checker */
-extern unsigned long is_use_virt_timer(void);
-
 /* Functions for GICv2 */
 extern void gic_dist_init_gicv2(unsigned long dist_base_pa, unsigned long size);
 extern void gic_cpu_init_gicv2(unsigned long cpu_base_pa, unsigned long size);
 extern void gic_enable_gicv2(void);
 extern void arm64_issue_ipi_gicv2(unsigned int cpuid, unsigned int vector);
+extern void arm64_issue_host_ipi_gicv2(uint32_t cpuid, uint32_t vector);
 extern void handle_interrupt_gicv2(struct pt_regs *regs);
 
 /* Functions for GICv3 */
@@ -63,6 +39,7 @@ extern void gic_dist_init_gicv3(unsigned long dist_base_pa, unsigned long size);
 extern void gic_cpu_init_gicv3(unsigned long cpu_base_pa, unsigned long size);
 extern void gic_enable_gicv3(void);
 extern void arm64_issue_ipi_gicv3(unsigned int cpuid, unsigned int vector);
+extern void arm64_issue_host_ipi_gicv3(uint32_t cpuid, uint32_t vector);
 extern void handle_interrupt_gicv3(struct pt_regs *regs);
 
 void handle_IPI(unsigned int vector, struct pt_regs *regs);

arch/arm64/kernel/include/pgtable-hwdef.h

@@ -72,6 +72,7 @@
 #define PMD_SECT_S		(UL(3) << 8)
 #define PMD_SECT_AF		(UL(1) << 10)
 #define PMD_SECT_NG		(UL(1) << 11)
+#define PMD_SECT_CONT		(UL(1) << 52)
 #define PMD_SECT_PXN		(UL(1) << 53)
 #define PMD_SECT_UXN		(UL(1) << 54)
 
@@ -93,6 +94,7 @@
 #define PTE_SHARED		(UL(3) << 8)	/* SH[1:0], inner shareable */
 #define PTE_AF			(UL(1) << 10)	/* Access Flag */
 #define PTE_NG			(UL(1) << 11)	/* nG */
+#define PTE_CONT		(UL(1) << 52)	/* Contiguous range */
 #define PTE_PXN			(UL(1) << 53)	/* Privileged XN */
 #define PTE_UXN			(UL(1) << 54)	/* User XN */
 /* Software defined PTE bits definition.*/

arch/arm64/kernel/include/prctl.h

@@ -1,17 +1,17 @@
-/* prctl.h COPYRIGHT FUJITSU LIMITED 2017 */
+/* prctl.h COPYRIGHT FUJITSU LIMITED 2017-2019 */
 #ifndef __HEADER_ARM64_COMMON_PRCTL_H
 #define __HEADER_ARM64_COMMON_PRCTL_H
 
-/* arm64 Scalable Vector Extension controls */
-#define PR_SVE_SET_VL		48			/* set task vector length */
-#define PR_SVE_SET_VL_THREAD	(1 << 1)		/* set just this thread */
-#define PR_SVE_SET_VL_INHERIT	(1 << 2)		/* inherit across exec */
-#define PR_SVE_SET_VL_ONEXEC	(1 << 3)		/* defer effect until exec */
+#define PR_SET_THP_DISABLE 41
+#define PR_GET_THP_DISABLE 42
 
-#define PR_SVE_GET_VL		49			/* get task vector length */
-/* Decode helpers for the return value from PR_SVE_GET_VL: */
-#define PR_SVE_GET_VL_LEN(ret)	((ret) & 0x3fff)	/* vector length */
-#define PR_SVE_GET_VL_INHERIT	(PR_SVE_SET_VL_INHERIT << 16)
-/* For conveinence, PR_SVE_SET_VL returns the result in the same encoding */
+/* arm64 Scalable Vector Extension controls */
+/* Flag values must be kept in sync with ptrace NT_ARM_SVE interface */
+#define PR_SVE_SET_VL		50		/* set task vector length */
+# define PR_SVE_SET_VL_ONEXEC	(1 << 18)	/* defer effect until exec */
+#define PR_SVE_GET_VL		51		/* get task vector length */
+/* Bits common to PR_SVE_SET_VL and PR_SVE_GET_VL */
+# define PR_SVE_VL_LEN_MASK	0xffff
+# define PR_SVE_VL_INHERIT	(1 << 17)	/* inherit across exec */
 
 #endif /* !__HEADER_ARM64_COMMON_PRCTL_H */

arch/arm64/kernel/include/ptrace.h

@@ -1,4 +1,4 @@
-/* ptrace.h COPYRIGHT FUJITSU LIMITED 2015-2017 */
+/* ptrace.h COPYRIGHT FUJITSU LIMITED 2015-2019 */
 #ifndef __HEADER_ARM64_COMMON_PTRACE_H
 #define __HEADER_ARM64_COMMON_PTRACE_H
 
@@ -46,6 +46,7 @@
 
 #ifndef __ASSEMBLY__
 
+#include <lwk/compiler.h>
 #include <ihk/types.h>
 
 struct user_hwdebug_state {
@@ -78,6 +79,70 @@ struct user_sve_header {
 	uint16_t __reserved;
 };
 
+enum aarch64_regset {
+	REGSET_GPR,
+	REGSET_FPR,
+	REGSET_TLS,
+	REGSET_HW_BREAK,
+	REGSET_HW_WATCH,
+	REGSET_SYSTEM_CALL,
+#ifdef CONFIG_ARM64_SVE
+	REGSET_SVE,
+#endif /* CONFIG_ARM64_SVE */
+};
+
+struct thread;
+struct user_regset;
+
+typedef int user_regset_active_fn(struct thread *target,
+				  const struct user_regset *regset);
+
+typedef long user_regset_get_fn(struct thread *target,
+				const struct user_regset *regset,
+				unsigned int pos, unsigned int count,
+				void *kbuf, void __user *ubuf);
+
+typedef long user_regset_set_fn(struct thread *target,
+				const struct user_regset *regset,
+				unsigned int pos, unsigned int count,
+				const void *kbuf, const void __user *ubuf);
+
+typedef int user_regset_writeback_fn(struct thread *target,
+				     const struct user_regset *regset,
+				     int immediate);
+
+typedef unsigned int user_regset_get_size_fn(struct thread *target,
+					     const struct user_regset *regset);
+
+struct user_regset {
+	user_regset_get_fn		*get;
+	user_regset_set_fn		*set;
+	user_regset_active_fn		*active;
+	user_regset_writeback_fn	*writeback;
+	user_regset_get_size_fn		*get_size;
+	unsigned int			n;
+	unsigned int			size;
+	unsigned int			align;
+	unsigned int			bias;
+	unsigned int			core_note_type;
+};
+
+struct user_regset_view {
+	const char *name;
+	const struct user_regset *regsets;
+	unsigned int n;
+	uint32_t e_flags;
+	uint16_t e_machine;
+	uint8_t ei_osabi;
+};
+
+extern const struct user_regset_view *current_user_regset_view(void);
+extern const struct user_regset *find_regset(
+		const struct user_regset_view *view,
+		unsigned int type);
+extern unsigned int regset_size(struct thread *target,
+		const struct user_regset *regset);
+
 /* Definitions for user_sve_header.flags: */
 #define SVE_PT_REGS_MASK	(1 << 0)
 
@@ -85,7 +150,7 @@ struct user_sve_header {
 #define SVE_PT_REGS_SVE		SVE_PT_REGS_MASK
 
 #define SVE_PT_VL_THREAD	PR_SVE_SET_VL_THREAD
-#define SVE_PT_VL_INHERIT	PR_SVE_SET_VL_INHERIT
+#define SVE_PT_VL_INHERIT	PR_SVE_VL_INHERIT
 #define SVE_PT_VL_ONEXEC	PR_SVE_SET_VL_ONEXEC
 
 /*
@@ -99,7 +164,9 @@ struct user_sve_header {
  */
 
 /* Offset from the start of struct user_sve_header to the register data */
-#define SVE_PT_REGS_OFFSET	((sizeof(struct sve_context) + 15) / 16 * 16)
+#define SVE_PT_REGS_OFFSET					\
+	((sizeof(struct sve_context) + (SVE_VQ_BYTES - 1))	\
+		/ SVE_VQ_BYTES * SVE_VQ_BYTES)
 
 /*
  * The register data content and layout depends on the value of the
@@ -174,8 +241,10 @@ struct user_sve_header {
 #define SVE_PT_SVE_FFR_OFFSET(vq) \
 	__SVE_SIG_TO_PT(SVE_SIG_FFR_OFFSET(vq))
 
-#define SVE_PT_SVE_FPSR_OFFSET(vq) \
-	((SVE_PT_SVE_FFR_OFFSET(vq) + SVE_PT_SVE_FFR_SIZE(vq) + 15) / 16 * 16)
+#define SVE_PT_SVE_FPSR_OFFSET(vq)				\
+	((SVE_PT_SVE_FFR_OFFSET(vq) + SVE_PT_SVE_FFR_SIZE(vq) + \
+			(SVE_VQ_BYTES - 1))			\
+		/ SVE_VQ_BYTES * SVE_VQ_BYTES)
 #define SVE_PT_SVE_FPCR_OFFSET(vq) \
 	(SVE_PT_SVE_FPSR_OFFSET(vq) + SVE_PT_SVE_FPSR_SIZE)
 
@@ -184,9 +253,10 @@ struct user_sve_header {
  * 128-bit boundary.
  */
 
-#define SVE_PT_SVE_SIZE(vq, flags)				\
-	((SVE_PT_SVE_FPCR_OFFSET(vq) + SVE_PT_SVE_FPCR_SIZE -	\
-		SVE_PT_SVE_OFFSET + 15) / 16 * 16)
+#define SVE_PT_SVE_SIZE(vq, flags)					\
+	((SVE_PT_SVE_FPCR_OFFSET(vq) + SVE_PT_SVE_FPCR_SIZE		\
+			- SVE_PT_SVE_OFFSET + (SVE_VQ_BYTES - 1))	\
+		/ SVE_VQ_BYTES * SVE_VQ_BYTES)
 
 #define SVE_PT_SIZE(vq, flags)						\
 	(((flags) & SVE_PT_REGS_MASK) == SVE_PT_REGS_SVE ?		\

arch/arm64/kernel/include/registers.h

@@ -1,9 +1,10 @@
-/* registers.h COPYRIGHT FUJITSU LIMITED 2015-2016 */
+/* registers.h COPYRIGHT FUJITSU LIMITED 2015-2018 */
 #ifndef __HEADER_ARM64_COMMON_REGISTERS_H
 #define __HEADER_ARM64_COMMON_REGISTERS_H
 
 #include <types.h>
 #include <arch/cpu.h>
+#include <sysreg.h>
 
 #define RFLAGS_CF      (1 << 0)
 #define RFLAGS_PF      (1 << 2)
@@ -76,15 +77,12 @@ static unsigned long rdmsr(unsigned int index)
 	return 0;
 }
 
-/* @ref.impl linux-linaro/arch/arm64/include/asm/arch_timer.h::arch_counter_get_cntvct */
-static unsigned long rdtsc(void)
+/* @ref.impl linux4.10.16 */
+/* arch/arm64/include/asm/arch_timer.h:arch_counter_get_cntvct() */
+static inline unsigned long rdtsc(void)
 {
-	unsigned long cval;
-
 	isb();
-	asm volatile("mrs %0, cntvct_el0" : "=r" (cval));
-
-	return cval;
+	return read_sysreg(cntvct_el0);
 }
 
 static void set_perfctl(int counter, int event, int mask)

arch/arm64/kernel/include/rlimit.h

@@ -85,7 +85,11 @@ enum __rlimit_resource
   __RLIMIT_RTPRIO = 14,
 #define RLIMIT_RTPRIO __RLIMIT_RTPRIO
 
-  __RLIMIT_NLIMITS = 15,
+  /* timeout for RT tasks in us */
+  __RLIMIT_RTTIME = 15,
+#define RLIMIT_RTTIME __RLIMIT_RTTIME
+
+  __RLIMIT_NLIMITS = 16,
   __RLIM_NLIMITS = __RLIMIT_NLIMITS
 #define RLIMIT_NLIMITS __RLIMIT_NLIMITS
 #define RLIM_NLIMITS __RLIM_NLIMITS

arch/arm64/kernel/include/signal.h

@@ -1,4 +1,4 @@
-/* signal.h COPYRIGHT FUJITSU LIMITED 2015-2017 */
+/* signal.h COPYRIGHT FUJITSU LIMITED 2015-2019 */
 #ifndef __HEADER_ARM64_COMMON_SIGNAL_H
 #define __HEADER_ARM64_COMMON_SIGNAL_H
 
@@ -9,6 +9,11 @@
 #define _NSIG_BPW	64
 #define _NSIG_WORDS (_NSIG / _NSIG_BPW)
 
+static inline int valid_signal(unsigned long sig)
+{
+	return sig <= _NSIG ? 1 : 0;
+}
+
 typedef unsigned long int __sigset_t;
 #define __sigmask(sig)  (((__sigset_t) 1) << ((sig) - 1))
 
@@ -293,6 +298,7 @@ struct extra_context {
 	struct _aarch64_ctx head;
 	void *data;	/* 16-byte aligned pointer to the extra space */
 	uint32_t size;	/* size in bytes of the extra space */
+	uint32_t __reserved[3];
 };
 
 #define SVE_MAGIC	0x53564501
@@ -313,19 +319,25 @@ struct sve_context {
  * The SVE architecture leaves space for future expansion of the
  * vector length beyond its initial architectural limit of 2048 bits
  * (16 quadwords).
+ *
+ * See linux/Documentation/arm64/sve.txt for a description of the VL/VQ
+ * terminology.
  */
-#define SVE_VQ_MIN	1
-#define SVE_VQ_MAX	0x200
+#define SVE_VQ_BYTES	16	/* number of bytes per quadword */
 
-#define SVE_VL_MIN	(SVE_VQ_MIN * 0x10)
-#define SVE_VL_MAX	(SVE_VQ_MAX * 0x10)
+#define SVE_VQ_MIN	1
+#define SVE_VQ_MAX	512
+
+#define SVE_VL_MIN	(SVE_VQ_MIN * SVE_VQ_BYTES)
+#define SVE_VL_MAX	(SVE_VQ_MAX * SVE_VQ_BYTES)
 
 #define SVE_NUM_ZREGS	32
 #define SVE_NUM_PREGS	16
 
 #define sve_vl_valid(vl) \
-	((vl) % 0x10 == 0 && (vl) >= SVE_VL_MIN && (vl) <= SVE_VL_MAX)
-#define sve_vq_from_vl(vl)	((vl) / 0x10)
+	((vl) % SVE_VQ_BYTES == 0 && (vl) >= SVE_VL_MIN && (vl) <= SVE_VL_MAX)
+#define sve_vq_from_vl(vl)	((vl) / SVE_VQ_BYTES)
+#define sve_vl_from_vq(vq)	((vq) * SVE_VQ_BYTES)
 
 /*
  * The total size of meaningful data in the SVE context in bytes,
@@ -360,11 +372,13 @@ struct sve_context {
  * Additional data might be appended in the future.
  */
 
-#define SVE_SIG_ZREG_SIZE(vq)	((uint32_t)(vq) * 16)
-#define SVE_SIG_PREG_SIZE(vq)	((uint32_t)(vq) * 2)
+#define SVE_SIG_ZREG_SIZE(vq)	((uint32_t)(vq) * SVE_VQ_BYTES)
+#define SVE_SIG_PREG_SIZE(vq)	((uint32_t)(vq) * (SVE_VQ_BYTES / 8))
 #define SVE_SIG_FFR_SIZE(vq)	SVE_SIG_PREG_SIZE(vq)
 
-#define SVE_SIG_REGS_OFFSET	((sizeof(struct sve_context) + 15) / 16 * 16)
+#define SVE_SIG_REGS_OFFSET					\
+	((sizeof(struct sve_context) + (SVE_VQ_BYTES - 1))	\
+		/ SVE_VQ_BYTES * SVE_VQ_BYTES)
 
 #define SVE_SIG_ZREGS_OFFSET	SVE_SIG_REGS_OFFSET
 #define SVE_SIG_ZREG_OFFSET(vq, n) \
@@ -402,8 +416,6 @@ struct ucontext {
 };
 
 void arm64_notify_die(const char *str, struct pt_regs *regs, struct siginfo *info, int err);
-void set_signal(int sig, void *regs, struct siginfo *info);
-void check_signal(unsigned long rc, void *regs, int num);
 void check_signal_irq_disabled(unsigned long rc, void *regs, int num);
 
 #endif /* __HEADER_ARM64_COMMON_SIGNAL_H */

arch/arm64/kernel/include/syscall_list.h

@@ -1,17 +1,14 @@
-/* syscall_list.h COPYRIGHT FUJITSU LIMITED 2015-2017 */
+/* syscall_list.h COPYRIGHT FUJITSU LIMITED 2015-2018 */
 
 SYSCALL_DELEGATED(4, io_getevents)
 SYSCALL_DELEGATED(17, getcwd)
-SYSCALL_DELEGATED(22, epoll_pwait)
+SYSCALL_HANDLED(22, epoll_pwait)
 SYSCALL_DELEGATED(25, fcntl)
 SYSCALL_HANDLED(29, ioctl)
+SYSCALL_DELEGATED(35, unlinkat)
 SYSCALL_DELEGATED(43, statfs)
 SYSCALL_DELEGATED(44, fstatfs)
-#ifdef POSTK_DEBUG_ARCH_DEP_62 /* Absorb the difference between open and openat args. */
 SYSCALL_HANDLED(56, openat)
-#else /* POSTK_DEBUG_ARCH_DEP_62 */
-SYSCALL_DELEGATED(56, openat)
-#endif /* POSTK_DEBUG_ARCH_DEP_62 */
 SYSCALL_HANDLED(57, close)
 SYSCALL_DELEGATED(61, getdents64)
 SYSCALL_DELEGATED(62, lseek)
@@ -20,8 +17,8 @@ SYSCALL_DELEGATED(64, write)
 SYSCALL_DELEGATED(66, writev)
 SYSCALL_DELEGATED(67, pread64)
 SYSCALL_DELEGATED(68, pwrite64)
-SYSCALL_DELEGATED(72, pselect6)
-SYSCALL_DELEGATED(73, ppoll)
+SYSCALL_HANDLED(72, pselect6)
+SYSCALL_HANDLED(73, ppoll)
 SYSCALL_HANDLED(74, signalfd4)
 SYSCALL_DELEGATED(78, readlinkat)
 SYSCALL_DELEGATED(80, fstat)
@@ -86,6 +83,7 @@ SYSCALL_HANDLED(175, geteuid)
 SYSCALL_HANDLED(176, getgid)
 SYSCALL_HANDLED(177, getegid)
 SYSCALL_HANDLED(178, gettid)
+SYSCALL_HANDLED(179, sysinfo)
 SYSCALL_DELEGATED(188, msgrcv)
 SYSCALL_DELEGATED(189, msgsnd)
 SYSCALL_DELEGATED(192, semtimedop)
@@ -114,21 +112,24 @@ SYSCALL_HANDLED(236, get_mempolicy)
 SYSCALL_HANDLED(237, set_mempolicy)
 SYSCALL_HANDLED(238, migrate_pages)
 SYSCALL_HANDLED(239, move_pages)
+#ifdef ENABLE_PERF
 SYSCALL_HANDLED(241, perf_event_open)
+#else // PERF_ENABLE
+SYSCALL_DELEGATED(241, perf_event_open)
+#endif // PERF_ENABLE
 SYSCALL_HANDLED(260, wait4)
+SYSCALL_HANDLED(261, prlimit64)
 SYSCALL_HANDLED(270, process_vm_readv)
 SYSCALL_HANDLED(271, process_vm_writev)
-SYSCALL_HANDLED(601, pmc_init)
-SYSCALL_HANDLED(602, pmc_start)
-SYSCALL_HANDLED(603, pmc_stop)
-SYSCALL_HANDLED(604, pmc_reset)
+SYSCALL_HANDLED(281, execveat)
 SYSCALL_HANDLED(700, get_cpu_id)
 #ifdef PROFILE_ENABLE
-SYSCALL_HANDLED(__NR_profile, profile)
+SYSCALL_HANDLED(PROFILE_EVENT_MAX, profile)
 #endif // PROFILE_ENABLE
 SYSCALL_HANDLED(730, util_migrate_inter_kernel)
 SYSCALL_HANDLED(731, util_indicate_clone)
 SYSCALL_HANDLED(732, get_system)
+SYSCALL_HANDLED(733, util_register_desc)
 
 /* McKernel Specific */
 SYSCALL_HANDLED(801, swapout)
@@ -138,11 +139,14 @@ SYSCALL_HANDLED(804, resume_threads)
 SYSCALL_HANDLED(811, linux_spawn)
 
 SYSCALL_DELEGATED(1024, open)
-SYSCALL_DELEGATED(1026, unlink)
 SYSCALL_DELEGATED(1035, readlink)
 SYSCALL_HANDLED(1045, signalfd)
 SYSCALL_DELEGATED(1049, stat)
 SYSCALL_DELEGATED(1060, getpgrp)
-SYSCALL_DELEGATED(1062, time)
-SYSCALL_HANDLED(1071, vfork)
-SYSCALL_DELEGATED(1079, fork)
+SYSCALL_HANDLED(1062, time)
+SYSCALL_DELEGATED(1069, epoll_wait)
+
+/* Do not edit the lines including this comment and
+ * EOF just after it because those are used as a
+ * robust marker for the autotest patch.
+ */

arch/arm64/kernel/include/sysreg.h

@@ -1,4 +1,4 @@
-/* sysreg.h COPYRIGHT FUJITSU LIMITED 2016-2017 */
+/* sysreg.h COPYRIGHT FUJITSU LIMITED 2016-2018 */
 /*
  * Macros for accessing system registers with older binutils.
  *
@@ -23,6 +23,7 @@
 
 #include <types.h>
 #include <stringify.h>
+#include <ihk/types.h>
 
 /*
  * ARMv8 ARM reserves the following encoding for system registers:
@@ -56,12 +57,6 @@
 #define sys_reg_CRm(id)	(((id) >> CRm_shift) & CRm_mask)
 #define sys_reg_Op2(id)	(((id) >> Op2_shift) & Op2_mask)
 
-#ifdef __ASSEMBLY__
-#define __emit_inst(x).inst (x)
-#else
-#define __emit_inst(x)".inst " __stringify((x)) "\n\t"
-#endif
-
 #define SYS_MIDR_EL1			sys_reg(3, 0, 0, 0, 0)
 #define SYS_MPIDR_EL1			sys_reg(3, 0, 0, 0, 5)
 #define SYS_REVIDR_EL1			sys_reg(3, 0, 0, 0, 6)
@@ -143,6 +138,12 @@
 #define ID_AA64ISAR0_SHA1_SHIFT		8
 #define ID_AA64ISAR0_AES_SHIFT		4
 
+/* id_aa64isar1 */
+#define ID_AA64ISAR1_LRCPC_SHIFT	20
+#define ID_AA64ISAR1_FCMA_SHIFT		16
+#define ID_AA64ISAR1_JSCVT_SHIFT	12
+#define ID_AA64ISAR1_DPB_SHIFT		0
+
 /* id_aa64pfr0 */
 #define ID_AA64PFR0_SVE_SHIFT		32
 #define ID_AA64PFR0_GIC_SHIFT		24
@@ -178,6 +179,14 @@
 #define ID_AA64MMFR0_TGRAN64_SUPPORTED	0x0
 #define ID_AA64MMFR0_TGRAN16_NI		0x0
 #define ID_AA64MMFR0_TGRAN16_SUPPORTED	0x1
+#define ID_AA64MMFR0_PARANGE_48		0x5
+#define ID_AA64MMFR0_PARANGE_52		0x6
+
+#ifdef CONFIG_ARM64_PA_BITS_52
+#define ID_AA64MMFR0_PARANGE_MAX	ID_AA64MMFR0_PARANGE_52
+#else
+#define ID_AA64MMFR0_PARANGE_MAX	ID_AA64MMFR0_PARANGE_48
+#endif
 
 /* id_aa64mmfr1 */
 #define ID_AA64MMFR1_PAN_SHIFT		20
@@ -264,15 +273,46 @@
 /* Safe value for MPIDR_EL1: Bit31:RES1, Bit30:U:0, Bit24:MT:0 */
 #define SYS_MPIDR_SAFE_VAL		(1UL << 31)
 
-#ifdef __ASSEMBLY__
+/* SYS_MIDR_EL1 */
+//mask
+#define SYS_MIDR_EL1_IMPLEMENTER_MASK	(0xFFUL)
+#define SYS_MIDR_EL1_PPNUM_MASK		(0xFFFUL)
+//shift
+#define SYS_MIDR_EL1_IMPLEMENTER_SHIFT	(24)
+#define SYS_MIDR_EL1_PPNUM_SHIFT	(0x4)
+//val
+#define SYS_MIDR_EL1_IMPLEMENTER_FJ	(0x46)
+#define SYS_MIDR_EL1_PPNUM_TCHIP	(0x1)
 
+#define READ_ACCESS	(0)
+#define WRITE_ACCESS	(1)
+#define ACCESS_REG_FUNC(name, reg) \
+	static void xos_access_##name(uint8_t flag, uint64_t *reg_value) \
+	{ \
+		if (flag == READ_ACCESS) { \
+			__asm__ __volatile__("mrs_s %0," __stringify(reg) "\n\t" \
+				:"=&r"(*reg_value)::); \
+		} \
+		else if (flag == WRITE_ACCESS) { \
+			__asm__ __volatile__("msr_s" __stringify(reg) ", %0\n\t" \
+				::"r"(*reg_value):); \
+		} else { \
+			; \
+		} \
+	}
+
+#define XOS_FALSE	(0)
+#define XOS_TRUE	(1)
+
+#ifdef __ASSEMBLY__
+#define __emit_inst(x).inst (x)
 	.irp	num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30
 	.equ	.L__reg_num_x\num, \num
 	.endr
 	.equ	.L__reg_num_xzr, 31
 
 	.macro	mrs_s, rt, sreg
-	 __emit_inst(0xd5200000|(\sreg)|(.L__reg_num_\rt))
+	__emit_inst(0xd5200000|(\sreg)|(.L__reg_num_\rt))
 	.endm
 
 	.macro	msr_s, sreg, rt
@@ -280,7 +320,7 @@
 	.endm
 
 #else
-
+#define __emit_inst(x)".inst " __stringify((x)) "\n\t"
 asm(
 "	.irp	num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30\n"
 "	.equ	.L__reg_num_x\\num, \\num\n"
@@ -296,6 +336,28 @@ asm(
 "	.endm\n"
 );
 
+ACCESS_REG_FUNC(midr_el1, SYS_MIDR_EL1);
+static int xos_is_tchip(void)
+{
+	uint64_t reg = 0;
+	int ret = 0, impl = 0, part = 0;
+
+	xos_access_midr_el1(READ_ACCESS, &reg);
+
+	impl = (reg >> SYS_MIDR_EL1_IMPLEMENTER_SHIFT) &
+		SYS_MIDR_EL1_IMPLEMENTER_MASK;
+	part = (reg >> SYS_MIDR_EL1_PPNUM_SHIFT) & SYS_MIDR_EL1_PPNUM_MASK;
+
+	if ((impl == SYS_MIDR_EL1_IMPLEMENTER_FJ) &&
+	    (part == SYS_MIDR_EL1_PPNUM_TCHIP)) {
+		ret = XOS_TRUE;
+	}
+	else {
+		ret = XOS_FALSE;
+	}
+
+	return ret;
+}
 #endif
 
 /*
@@ -336,4 +398,6 @@ asm(
 /* @ref.impl arch/arm64/include/asm/kvm_arm.h */
 #define CPTR_EL2_TZ		(1 << 8)
 
+#include "imp-sysreg.h"
+
 #endif	/* __ASM_SYSREG_H */

arch/arm64/kernel/include/thread_info.h

@@ -1,15 +1,22 @@
-/* thread_info.h COPYRIGHT FUJITSU LIMITED 2015-2017 */
+/* thread_info.h COPYRIGHT FUJITSU LIMITED 2015-2019 */
 #ifndef __HEADER_ARM64_COMMON_THREAD_INFO_H
 #define __HEADER_ARM64_COMMON_THREAD_INFO_H
 
-#define KERNEL_STACK_SIZE	32768	/* 8 page */
+#define MIN_KERNEL_STACK_SHIFT	18
+
+#include <arch-memory.h>
+
+#if (MIN_KERNEL_STACK_SHIFT < PAGE_SHIFT)
+#define KERNEL_STACK_SHIFT	PAGE_SHIFT
+#else
+#define KERNEL_STACK_SHIFT	MIN_KERNEL_STACK_SHIFT
+#endif
+
+#define KERNEL_STACK_SIZE	(UL(1) << KERNEL_STACK_SHIFT)
 #define THREAD_START_SP		KERNEL_STACK_SIZE - 16
 
 #ifndef __ASSEMBLY__
 
-#define ALIGN_UP(x, align)     ALIGN_DOWN((x) + (align) - 1, align)
-#define ALIGN_DOWN(x, align)   ((x) & ~((align) - 1))
-
 #include <process.h>
 #include <prctl.h>
 
@@ -39,9 +46,9 @@ struct thread_info {
 	int			cpu;		/* cpu */
 	struct cpu_context	cpu_context;	/* kernel_context */
 	void			*sve_state;	/* SVE registers, if any */
-	uint16_t		sve_vl;		/* SVE vector length */
-	uint16_t		sve_vl_onexec;	/* SVE vl after next exec */
-	uint16_t		sve_flags;	/* SVE related flags */
+	unsigned int		sve_vl;		/* SVE vector length */
+	unsigned int		sve_vl_onexec;	/* SVE vl after next exec */
+	unsigned long		sve_flags;	/* SVE related flags */
 	unsigned long		fault_address;	/* fault info */
 	unsigned long		fault_code;	/* ESR_EL1 value */
 };
@@ -49,12 +56,12 @@ struct thread_info {
 /* Flags for sve_flags (intentionally defined to match the prctl flags) */
 
 /* Inherit sve_vl and sve_flags across execve(): */
-#define THREAD_VL_INHERIT	PR_SVE_SET_VL_INHERIT
+#define THREAD_VL_INHERIT	PR_SVE_VL_INHERIT
 
 struct arm64_cpu_local_thread {
 	struct thread_info thread_info;
-	unsigned long paniced;		/* 136 */
-	uint64_t panic_regs[34];	/* 144 */
+	unsigned long paniced;
+	uint64_t panic_regs[34];
 };
 
 union arm64_cpu_local_variables {

arch/arm64/kernel/include/traps.h

@@ -4,6 +4,7 @@
 #define __ASM_TRAP_H
 
 #include <types.h>
+#include <arch-lock.h>
 
 struct pt_regs;
 

arch/arm64/kernel/include/virt.h

@@ -1,8 +1,22 @@
-/* virt.h COPYRIGHT FUJITSU LIMITED 2015 */
+/* virt.h COPYRIGHT FUJITSU LIMITED 2015-2017 */
 #ifndef __HEADER_ARM64_COMMON_VIRT_H
 #define __HEADER_ARM64_COMMON_VIRT_H
 
+/* @ref.impl linux-v4.15-rc3 arch/arm64/include/asm/virt.h */
 #define BOOT_CPU_MODE_EL1	(0xe11)
 #define BOOT_CPU_MODE_EL2	(0xe12)
 
+#ifndef __ASSEMBLY__
+
+#include <sysreg.h>
+#include <ptrace.h>
+
+/* @ref.impl linux-v4.15-rc3 arch/arm64/include/asm/virt.h */
+static inline int is_kernel_in_hyp_mode(void)
+{
+	return read_sysreg(CurrentEL) == CurrentEL_EL2;
+}
+
+#endif /* !__ASSEMBLY__ */
+
 #endif /* !__HEADER_ARM64_COMMON_VIRT_H */

arch/arm64/kernel/irq-gic-v2.c

@@ -1,21 +1,21 @@
-/* irq-gic-v2.c COPYRIGHT FUJITSU LIMITED 2015-2016 */
-
+/* irq-gic-v2.c COPYRIGHT FUJITSU LIMITED 2015-2018 */
 #include <ihk/cpu.h>
 #include <irq.h>
 #include <arm-gic-v2.h>
 #include <io.h>
 #include <arch/cpu.h>
 #include <memory.h>
+#include <affinity.h>
 #include <syscall.h>
+#include <ihk/debug.h>
+#include <arch-timer.h>
+#include <cls.h>
 
 // #define DEBUG_GICV2
 
 #ifdef DEBUG_GICV2
-#define	dkprintf(...)	kprintf(__VA_ARGS__)
-#define	ekprintf(...)	kprintf(__VA_ARGS__)
-#else
-#define dkprintf(...)
-#define	ekprintf(...)	kprintf(__VA_ARGS__)
+#undef DDEBUG_DEFAULT
+#define DDEBUG_DEFAULT DDEBUG_PRINT
 #endif
 
 void *dist_base;
@@ -31,10 +31,9 @@ void *cpu_base;
  * function, it is not necessary to perform the disable/enable
  * interrupts in this function as gic_raise_softirq() .
  */
-static void arm64_raise_sgi_gicv2(unsigned int cpuid, unsigned int vector)
+static void __arm64_raise_sgi_gicv2(unsigned int hw_cpuid, unsigned int vector)
 {
 	/* Build interrupt destination of the target cpu */
-	unsigned int hw_cpuid = ihk_mc_get_cpu_info()->hw_ids[cpuid];
 	uint8_t cpu_target_list = gic_hwid_to_affinity(hw_cpuid);
 
 	/*
@@ -50,21 +49,32 @@ static void arm64_raise_sgi_gicv2(unsigned int cpuid, unsigned int vector)
 	);
 }
 
+static void arm64_raise_sgi_gicv2(uint32_t cpuid, uint32_t vector)
+{
+	/* Build interrupt destination of the target CPU */
+	uint32_t hw_cpuid = ihk_mc_get_cpu_info()->hw_ids[cpuid];
+
+	__arm64_raise_sgi_gicv2(hw_cpuid, vector);
+}
+
+static void arm64_raise_sgi_to_host_gicv2(uint32_t cpuid, uint32_t vector)
+{
+	/* Build interrupt destination of the target Linux/host CPU */
+	uint32_t hw_cpuid = ihk_mc_get_apicid(cpuid);
+
+	__arm64_raise_sgi_gicv2(hw_cpuid, vector);
+}
+
+
 /**
  * arm64_raise_spi_gicv2
  * @ref.impl nothing.
  */
-extern unsigned int ihk_ikc_irq_apicid;
 static void arm64_raise_spi_gicv2(unsigned int cpuid, unsigned int vector)
 {
 	uint64_t spi_reg_offset;
 	uint32_t spi_set_pending_bitpos;
 
-	if (cpuid != ihk_ikc_irq_apicid) {
-		ekprintf("SPI(irq#%d) cannot send other than the host.\n", vector);
-		return;
-	}
-
 	/**
 	 * calculates register offset and bit position corresponding to the numbers.
 	 *
@@ -83,6 +93,11 @@ static void arm64_raise_spi_gicv2(unsigned int cpuid, unsigned int vector)
 	);
 }
 
+void arm64_issue_host_ipi_gicv2(uint32_t cpuid, uint32_t vector)
+{
+	arm64_raise_sgi_to_host_gicv2(cpuid, vector);
+}
+
 /**
  * arm64_issue_ipi_gicv2
  * @param cpuid : hardware cpu id
@@ -111,8 +126,9 @@ extern int interrupt_from_user(void *);
 void handle_interrupt_gicv2(struct pt_regs *regs)
 {
 	unsigned int irqstat, irqnr;
+	const int from_user = interrupt_from_user(regs);
 
-	set_cputime(interrupt_from_user(regs)? 1: 2);
+	set_cputime(from_user ? CPUTIME_MODE_U2K : CPUTIME_MODE_K2K_IN);
 	do {
 		// get GICC_IAR.InterruptID
 		irqstat = readl_relaxed(cpu_base + GIC_CPU_INTACK);
@@ -132,7 +148,13 @@ void handle_interrupt_gicv2(struct pt_regs *regs)
 		 */
 		break;
 	} while (1);
-	set_cputime(0);
+	set_cputime(from_user ? CPUTIME_MODE_K2U : CPUTIME_MODE_K2K_OUT);
+
+	/* for migration by IPI */
+	if (get_this_cpu_local_var()->flags & CPU_FLAG_NEED_MIGRATE) {
+		schedule();
+		check_signal(0, regs, 0);
+	}
 }
 
 void gic_dist_init_gicv2(unsigned long dist_base_pa, unsigned long size)
@@ -149,10 +171,6 @@ void gic_enable_gicv2(void)
 {
 	unsigned int enable_ppi_sgi = 0;
 
-	if (is_use_virt_timer()) {
-		enable_ppi_sgi |= GICD_ENABLE << get_virt_timer_intrid();
-	} else {
-		enable_ppi_sgi |= GICD_ENABLE << get_phys_timer_intrid();
-	}
+	enable_ppi_sgi |= GICD_ENABLE << get_timer_intrid();
 	writel_relaxed(enable_ppi_sgi, dist_base + GIC_DIST_ENABLE_SET);
 }

arch/arm64/kernel/irq-gic-v3.c

@@ -1,5 +1,4 @@
-/* irq-gic-v3.c COPYRIGHT FUJITSU LIMITED 2015-2017 */
-
+/* irq-gic-v3.c COPYRIGHT FUJITSU LIMITED 2015-2018 */
 #include <irq.h>
 #include <arm-gic-v2.h>
 #include <arm-gic-v3.h>
@@ -7,17 +6,20 @@
 #include <cputype.h>
 #include <process.h>
 #include <syscall.h>
+#include <ihk/debug.h>
+#ifdef ENABLE_FUGAKU_HACKS
+#include <ihk/monitor.h>
+#endif
+#include <arch-timer.h>
+#include <cls.h>
 
 //#define DEBUG_GICV3
 
 #define USE_CAVIUM_THUNDER_X
 
 #ifdef DEBUG_GICV3
-#define	dkprintf(...)	kprintf(__VA_ARGS__)
-#define	ekprintf(...)	kprintf(__VA_ARGS__)
-#else
-#define dkprintf(...)
-#define	ekprintf(...)	kprintf(__VA_ARGS__)
+#undef DDEBUG_DEFAULT
+#define DDEBUG_DEFAULT DDEBUG_PRINT
 #endif
 
 #ifdef USE_CAVIUM_THUNDER_X
@@ -196,15 +198,12 @@ static inline void gic_write_bpr1(uint32_t val)
 }
 #endif
 
-static void arm64_raise_sgi_gicv3(uint32_t cpuid, uint32_t vector)
+static void __arm64_raise_sgi_gicv3(uint32_t hw_cpuid, uint32_t vector)
 {
 	uint64_t mpidr, cluster_id;
 	uint16_t tlist;
 	uint64_t val;
 
-	/* Build interrupt destination of the target cpu */
-	uint32_t hw_cpuid = ihk_mc_get_cpu_info()->hw_ids[cpuid];
-
 	/*
 	 * Ensure that stores to Normal memory are visible to the
 	 * other CPUs before issuing the IPI.
@@ -240,6 +239,22 @@ static void arm64_raise_sgi_gicv3(uint32_t cpuid, uint32_t vector)
 	}
 }
 
+static void arm64_raise_sgi_gicv3(uint32_t cpuid, uint32_t vector)
+{
+	/* Build interrupt destination of the target CPU */
+	uint32_t hw_cpuid = ihk_mc_get_cpu_info()->hw_ids[cpuid];
+
+	__arm64_raise_sgi_gicv3(hw_cpuid, vector);
+}
+
+static void arm64_raise_sgi_to_host_gicv3(uint32_t cpuid, uint32_t vector)
+{
+	/* Build interrupt destination of the target Linux/host CPU */
+	uint32_t hw_cpuid = ihk_mc_get_apicid(cpuid);
+
+	__arm64_raise_sgi_gicv3(hw_cpuid, vector);
+}
+
 static void arm64_raise_spi_gicv3(uint32_t cpuid, uint32_t vector)
 {
 	uint64_t spi_reg_offset;
@@ -266,8 +281,14 @@ static void arm64_raise_spi_gicv3(uint32_t cpuid, uint32_t vector)
 static void arm64_raise_lpi_gicv3(uint32_t cpuid, uint32_t vector)
 {
 	// @todo.impl
+	ekprintf("%s called.\n", __func__);
 }
  
+void arm64_issue_host_ipi_gicv3(uint32_t cpuid, uint32_t vector)
+{
+	arm64_raise_sgi_to_host_gicv3(cpuid, vector);
+}
+
 void arm64_issue_ipi_gicv3(uint32_t cpuid, uint32_t vector)
 {
 	dkprintf("Send irq#%d to cpuid=%d\n", vector, cpuid);
@@ -283,7 +304,7 @@ void arm64_issue_ipi_gicv3(uint32_t cpuid, uint32_t vector)
 		// send LPI (allow only to host)
 		arm64_raise_lpi_gicv3(cpuid, vector);
 	} else {
-		ekprintf("#%d is bad irq number.", vector);
+		ekprintf("#%d is bad irq number.\n", vector);
 	}
 }
 
@@ -291,22 +312,97 @@ extern int interrupt_from_user(void *);
 void handle_interrupt_gicv3(struct pt_regs *regs)
 {
 	uint64_t irqnr;
+	const int from_user = interrupt_from_user(regs);
+	struct cpu_local_var *v = get_this_cpu_local_var();
+	//unsigned long irqflags;
+	int do_check = 0;
+#ifdef ENABLE_FUGAKU_HACKS
+	struct ihk_os_cpu_monitor *monitor = cpu_local_var(monitor);
 
+	++v->in_interrupt;
+#endif
 	irqnr = gic_read_iar();
 	cpu_enable_nmi();
-	set_cputime(interrupt_from_user(regs)? 1: 2);
+	set_cputime(from_user ? CPUTIME_MODE_U2K : CPUTIME_MODE_K2K_IN);
 	while (irqnr != ICC_IAR1_EL1_SPURIOUS) {
 		if ((irqnr < 1020) || (irqnr >= 8192)) {
 			gic_write_eoir(irqnr);
+#ifndef ENABLE_FUGAKU_HACKS
 			handle_IPI(irqnr, regs);
+#else
+			/* Once paniced, only allow CPU stop and NMI IRQs */
+			if (monitor->status != IHK_OS_MONITOR_PANIC ||
+					irqnr == INTRID_CPU_STOP ||
+					irqnr == INTRID_MULTI_NMI) {
+				handle_IPI(irqnr, regs);
+			}
+#endif
 		}
 		irqnr = gic_read_iar();
 	}
-	set_cputime(0);
+	set_cputime(from_user ? CPUTIME_MODE_K2U : CPUTIME_MODE_K2K_OUT);
+
+	//irqflags = ihk_mc_spinlock_lock(&v->runq_lock);
+	/* For migration by IPI or by timesharing */
+	if (v->flags & CPU_FLAG_NEED_RESCHED) {
+		if (v->flags & CPU_FLAG_NEED_MIGRATE && !from_user) {
+			// Don't migrate on K2K schedule
+		} else {
+			v->flags &= ~CPU_FLAG_NEED_RESCHED;
+			do_check = 1;
+		}
+	}
+	//ihk_mc_spinlock_unlock(&v->runq_lock, irqflags);
+
+#ifndef ENABLE_FUGAKU_HACKS
+	if (do_check) {
+#else
+	--v->in_interrupt;
+	if (monitor->status != IHK_OS_MONITOR_PANIC && do_check) {
+#endif
+		check_signal(0, regs, 0);
+		schedule();
+	}
+}
+
+static uint64_t gic_mpidr_to_affinity(unsigned long mpidr)
+{
+	uint64_t aff;
+
+	aff = ((uint64_t)MPIDR_AFFINITY_LEVEL(mpidr, 3) << 32 |
+			 MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
+			 MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8  |
+			 MPIDR_AFFINITY_LEVEL(mpidr, 0));
+	return aff;
+}
+
+static void init_spi_routing(uint32_t irq, uint32_t linux_cpu)
+{
+	uint64_t spi_route_reg_val, spi_route_reg_offset;
+
+	if (irq < 32 || 1020 <= irq) {
+		ekprintf("%s: irq is not spi number. (irq=%d)\n",
+			 __func__, irq);
+		return;
+	}
+
+	/* write to GICD_IROUTER */
+	spi_route_reg_offset = irq * 8;
+	spi_route_reg_val = gic_mpidr_to_affinity(cpu_logical_map(linux_cpu));
+
+	writeq_relaxed(spi_route_reg_val,
+		       (void *)(dist_base + GICD_IROUTER +
+				spi_route_reg_offset));
 }
 
 void gic_dist_init_gicv3(unsigned long dist_base_pa, unsigned long size)
 {
+#ifndef IHK_IKC_USE_LINUX_WORK_IRQ
+	extern int spi_table[];
+	extern int nr_spi_table;
+	int i;
+#endif // !IHK_IKC_USE_LINUX_WORK_IRQ
+
 	dist_base = map_fixed_area(dist_base_pa, size, 1 /*non chachable*/);
 
 #ifdef USE_CAVIUM_THUNDER_X
@@ -315,6 +411,16 @@ void gic_dist_init_gicv3(unsigned long dist_base_pa, unsigned long size)
 		is_cavium_thunderx = 1;
 	}
 #endif
+
+#ifndef IHK_IKC_USE_LINUX_WORK_IRQ
+	/* initialize spi routing */
+	for (i = 0; i < nr_spi_table; i++) {
+		if (spi_table[i] == -1) {
+			continue;
+		}
+		init_spi_routing(spi_table[i], i);
+	}
+#endif // !IHK_IKC_USE_LINUX_WORK_IRQ
 }
 
 void gic_cpu_init_gicv3(unsigned long cpu_base_pa, unsigned long size)
@@ -351,11 +457,23 @@ void gic_enable_gicv3(void)
 	void *rd_sgi_base = rbase + 0x10000 /* SZ_64K */;
 	int i;
 	unsigned int enable_ppi_sgi = GICD_INT_EN_SET_SGI;
+	extern int ihk_param_nr_pmu_irq_affi;
+	extern int ihk_param_pmu_irq_affi[CONFIG_SMP_MAX_CORES];
 
-	if (is_use_virt_timer()) {
-		enable_ppi_sgi |= GICD_ENABLE << get_virt_timer_intrid();
-	} else {
-		enable_ppi_sgi |= GICD_ENABLE << get_phys_timer_intrid();
+	enable_ppi_sgi |= GICD_ENABLE << get_timer_intrid();
+
+	if (0 < ihk_param_nr_pmu_irq_affi) {
+		for (i = 0; i < ihk_param_nr_pmu_irq_affi; i++) {
+			if ((0 <= ihk_param_pmu_irq_affi[i]) &&
+			    (ihk_param_pmu_irq_affi[i] <
+			     sizeof(enable_ppi_sgi) * BITS_PER_BYTE)) {
+				enable_ppi_sgi |= GICD_ENABLE <<
+					ihk_param_pmu_irq_affi[i];
+			}
+		}
+	}
+	else {
+		enable_ppi_sgi |= GICD_ENABLE << INTRID_PERF_OVF;
 	}
 
 	/*
@@ -368,9 +486,10 @@ void gic_enable_gicv3(void)
 	/*
 	 * Set priority on PPI and SGI interrupts
 	 */
-	for (i = 0; i < 32; i += 4)
+	for (i = 0; i < 32; i += 4) {
 		writel_relaxed(GICD_INT_DEF_PRI_X4,
-					rd_sgi_base + GIC_DIST_PRI + i * 4 / 4);
+			       rd_sgi_base + GIC_DIST_PRI + i);
+	}
 
 	/* sync wait */
 	gic_do_wait_for_rwp(rbase);
@@ -406,9 +525,12 @@ void gic_enable_gicv3(void)
 	gic_write_bpr1(0);
 
 	/* Set specific IPI to NMI */
-	writeb_relaxed(GICD_INT_NMI_PRI, rd_sgi_base + GIC_DIST_PRI + INTRID_CPU_STOP);
-	writeb_relaxed(GICD_INT_NMI_PRI, rd_sgi_base + GIC_DIST_PRI + INTRID_MEMDUMP);
-	writeb_relaxed(GICD_INT_NMI_PRI, rd_sgi_base + GIC_DIST_PRI + INTRID_STACK_TRACE);
+	writeb_relaxed(GICD_INT_NMI_PRI,
+		       rd_sgi_base + GIC_DIST_PRI + INTRID_CPU_STOP);
+	writeb_relaxed(GICD_INT_NMI_PRI,
+		       rd_sgi_base + GIC_DIST_PRI + INTRID_MULTI_NMI);
+	writeb_relaxed(GICD_INT_NMI_PRI,
+		       rd_sgi_base + GIC_DIST_PRI + INTRID_STACK_TRACE);
 
 	/* sync wait */
 	gic_do_wait_for_rwp(rbase);

arch/arm64/kernel/local.c

@@ -1,4 +1,4 @@
-/* local.c COPYRIGHT FUJITSU LIMITED 2015-2016 */
+/* local.c COPYRIGHT FUJITSU LIMITED 2015-2018 */
 #include <cpulocal.h>
 #include <ihk/atomic.h>
 #include <ihk/mm.h>
@@ -7,24 +7,31 @@
 #include <registers.h>
 #include <string.h>
 
-#define LOCALS_SPAN (8 * PAGE_SIZE)
-
 /* BSP initialized stack area */
 union arm64_cpu_local_variables init_thread_info __attribute__((aligned(KERNEL_STACK_SIZE)));
 
 /* BSP/AP idle stack pointer head */
 static union arm64_cpu_local_variables *locals;
-size_t arm64_cpu_local_variables_span = LOCALS_SPAN;	/* for debugger */
+size_t arm64_cpu_local_variables_span = KERNEL_STACK_SIZE; /* for debugger */
 
 /* allocate & initialize BSP/AP idle stack */
 void init_processors_local(int max_id)
 {
 	int i = 0;
-	const int sz = (max_id + 1) * KERNEL_STACK_SIZE;
 	union arm64_cpu_local_variables *tmp;
+	const int npages = ((max_id + 1) *
+			    (ALIGN_UP(KERNEL_STACK_SIZE, PAGE_SIZE) >>
+			     PAGE_SHIFT));
+
+	if (npages < 1) {
+		panic("idle kernel stack allocation failed.");
+	}
 
 	/* allocate one more for alignment */
-	locals = ihk_mc_alloc_pages(((sz + PAGE_SIZE - 1) / PAGE_SIZE), IHK_MC_AP_CRITICAL);
+	locals = ihk_mc_alloc_pages(npages, IHK_MC_AP_CRITICAL);
+	if (locals == NULL) {
+		panic("idle kernel stack allocation failed.");
+	}
 	locals = (union arm64_cpu_local_variables *)ALIGN_UP((unsigned long)locals, KERNEL_STACK_SIZE);
 
 	/* clear struct process, struct process_vm, struct thread_info area */

arch/arm64/kernel/memset.S

@@ -218,3 +218,41 @@ ENTRY(__inline_memset)
 	ret
 ENDPIPROC(__inline_memset)
 ENDPROC(____inline_memset)
+
+
+/*
+ * Non-temporal vector memory clear
+ *
+ * Parameters:
+ *     x0 - buf (assumed to be aligned to page size)
+ *     x1 - n (assumed to be at least page size)
+ */
+ENTRY(__memclear)
+       stp     q0, q1, [x2] /* Preserve two 128 bit vector regs */
+       eor v0.16B, v0.16B, v0.16B
+       eor v1.16B, v1.16B, v1.16B
+1:
+       stnp q0, q1, [x0, #32 * 0]
+       stnp q0, q1, [x0, #32 * 1]
+       stnp q0, q1, [x0, #32 * 2]
+       stnp q0, q1, [x0, #32 * 3]
+       stnp q0, q1, [x0, #32 * 4]
+       stnp q0, q1, [x0, #32 * 5]
+       stnp q0, q1, [x0, #32 * 6]
+       stnp q0, q1, [x0, #32 * 7]
+       stnp q0, q1, [x0, #32 * 8]
+       stnp q0, q1, [x0, #32 * 9]
+       stnp q0, q1, [x0, #32 * 10]
+       stnp q0, q1, [x0, #32 * 11]
+       stnp q0, q1, [x0, #32 * 12]
+       stnp q0, q1, [x0, #32 * 13]
+       stnp q0, q1, [x0, #32 * 14]
+       stnp q0, q1, [x0, #32 * 15]
+       add x0, x0, #512
+       subs x1, x1, #512
+       cmp x1, #0
+       b.ne    1b
+
+       ldp     q0, q1, [x2] /* Restore vector regs */
+       ret
+ENDPROC(__memclear)

arch/arm64/kernel/mikc.c

@@ -19,7 +19,7 @@ int ihk_mc_ikc_init_first_local(struct ihk_ikc_channel_desc *channel,
 
 	memset(channel, 0, sizeof(struct ihk_ikc_channel_desc));
 
-	mikc_queue_pages = ((2 * num_processors * MASTER_IKCQ_PKTSIZE)
+	mikc_queue_pages = ((8 * num_processors * MASTER_IKCQ_PKTSIZE)
 			+ (PAGE_SIZE - 1)) / PAGE_SIZE;
 
 	/* Place both sides in this side */

arch/arm64/kernel/perfctr.c

@@ -1,4 +1,4 @@
-/* perfctr.c COPYRIGHT FUJITSU LIMITED 2015-2017 */
+/* perfctr.c COPYRIGHT FUJITSU LIMITED 2015-2018 */
 #include <arch-perfctr.h>
 #include <ihk/perfctr.h>
 #include <mc_perf_event.h>
@@ -6,32 +6,61 @@
 #include <ihk/debug.h>
 #include <registers.h>
 #include <string.h>
+#include <ihk/mm.h>
+#include <irq.h>
+#include <process.h>
 
 /*
  * @ref.impl arch/arm64/kernel/perf_event.c
  * Set at runtime when we know what CPU type we are.
  */
 struct arm_pmu cpu_pmu;
-extern int ihk_param_pmu_irq_affiniry[CONFIG_SMP_MAX_CORES];
-extern int ihk_param_nr_pmu_irq_affiniry;
-
+extern int ihk_param_pmu_irq_affi[CONFIG_SMP_MAX_CORES];
+extern int ihk_param_nr_pmu_irq_affi;
 
 int arm64_init_perfctr(void)
 {
 	int ret;
 	int i;
+	int pages;
+	const struct ihk_mc_cpu_info *cpu_info;
 
 	memset(&cpu_pmu, 0, sizeof(cpu_pmu));
 	ret = armv8pmu_init(&cpu_pmu);
-	if (!ret) {
+	if (ret) {
 		return ret;
 	}
-	for (i = 0; i < ihk_param_nr_pmu_irq_affiniry; i++) {
-		ret = ihk_mc_register_interrupt_handler(ihk_param_pmu_irq_affiniry[i], cpu_pmu.handler);
+
+	cpu_info = ihk_mc_get_cpu_info();
+	pages = (sizeof(struct per_cpu_arm_pmu) * cpu_info->ncpus +
+		 PAGE_SIZE - 1) >> PAGE_SHIFT;
+	cpu_pmu.per_cpu = ihk_mc_alloc_pages(pages, IHK_MC_AP_NOWAIT);
+	if (cpu_pmu.per_cpu == NULL) {
+		return -ENOMEM;
+	}
+	memset(cpu_pmu.per_cpu, 0, pages * PAGE_SIZE);
+
+	if (0 < ihk_param_nr_pmu_irq_affi) {
+		for (i = 0; i < ihk_param_nr_pmu_irq_affi; i++) {
+			ret = ihk_mc_register_interrupt_handler(ihk_param_pmu_irq_affi[i],
+								cpu_pmu.handler);
+			if (ret) {
+				break;
+			}
+		}
+	}
+	else {
+		ret = ihk_mc_register_interrupt_handler(INTRID_PERF_OVF,
+							cpu_pmu.handler);
 	}
 	return ret;
 }
 
+void arm64_init_per_cpu_perfctr(void)
+{
+	armv8pmu_per_cpu_init(&cpu_pmu.per_cpu[ihk_mc_get_processor_id()]);
+}
+
 int arm64_enable_pmu(void)
 {
 	int ret;
@@ -47,34 +76,36 @@ void arm64_disable_pmu(void)
 	cpu_pmu.disable_pmu();
 }
 
-extern unsigned int *arm64_march_perfmap;
+void arm64_enable_user_access_pmu_regs(void)
+{
+	cpu_pmu.enable_user_access_pmu_regs();
+}
+
+void arm64_disable_user_access_pmu_regs(void)
+{
+	cpu_pmu.disable_user_access_pmu_regs();
+}
 
 static int __ihk_mc_perfctr_init(int counter, uint32_t type, uint64_t config, int mode)
 {
-	int ret;
+	int ret = -1;
 	unsigned long config_base = 0;
-	int mapping;
 
-	mapping = cpu_pmu.map_event(type, config);
-	if (mapping < 0) {
-		return mapping;
-	}
-
-	ret = cpu_pmu.disable_counter(counter);
-	if (!ret) {
+	ret = cpu_pmu.disable_counter(1UL << counter);
+	if (ret < 0) {
 		return ret;
 	}
 
-	ret = cpu_pmu.enable_intens(counter);
-	if (!ret) {
+	ret = cpu_pmu.enable_intens(1UL << counter);
+	if (ret < 0) {
 		return ret;
 	}
 
 	ret = cpu_pmu.set_event_filter(&config_base, mode);
-	if (!ret) {
+	if (ret) {
 		return ret;
 	}
-	config_base |= (unsigned long)mapping;
+	config_base |= config;
 	cpu_pmu.write_evtype(counter, config_base);
 	return ret;
 }
@@ -86,41 +117,24 @@ int ihk_mc_perfctr_init_raw(int counter, uint64_t config, int mode)
 	return ret;
 }
 
-int ihk_mc_perfctr_init(int counter, uint64_t config, int mode)
+int ihk_mc_perfctr_start(unsigned long counter_mask)
 {
-	int ret;
-	ret = __ihk_mc_perfctr_init(counter, PERF_TYPE_RAW, config, mode);
-	return ret;
+	return cpu_pmu.enable_counter(counter_mask);
 }
 
-int ihk_mc_perfctr_start(int counter)
+int ihk_mc_perfctr_stop(unsigned long counter_mask, int flags)
 {
-	int ret;
-	ret = cpu_pmu.enable_counter(counter);
-	return ret;
-}
-
-int ihk_mc_perfctr_stop(int counter)
-{
-	cpu_pmu.disable_counter(counter);
-
-	// ihk_mc_perfctr_startが呼ばれるときには、
-	// init系関数が呼ばれるのでdisableにする。
-	cpu_pmu.disable_intens(counter);
-	return 0;
+	return cpu_pmu.disable_counter(counter_mask);
 }
 
 int ihk_mc_perfctr_reset(int counter)
 {
-	// TODO[PMU]: ihk_mc_perfctr_setと同様にサンプリングレートの共通部実装の扱いを見てから本実装。
 	cpu_pmu.write_counter(counter, 0);
 	return 0;
 }
 
-//int ihk_mc_perfctr_set(int counter, unsigned long val)
-int ihk_mc_perfctr_set(int counter, long val) /* 0416_patchtemp */
+int ihk_mc_perfctr_set(int counter, long val)
 {
-	// TODO[PMU]: 共通部でサンプリングレートの計算をして、設定するカウンタ値をvalに渡してくるようになると想定。サンプリングレートの扱いを見てから本実装。
 	uint32_t v = val;
 	cpu_pmu.write_counter(counter, v);
 	return 0;
@@ -133,6 +147,15 @@ int ihk_mc_perfctr_read_mask(unsigned long counter_mask, unsigned long *value)
 	return 0;
 }
 
+int ihk_mc_perfctr_alloc(struct thread *thread, struct mc_perf_event *event)
+{
+	const int counters = ihk_mc_perf_get_num_counters();
+
+	return cpu_pmu.get_event_idx(counters,
+				     thread->pmc_alloc_map,
+				     event->hw_config);
+}
+
 unsigned long ihk_mc_perfctr_read(int counter)
 {
 	unsigned long count;
@@ -140,17 +163,135 @@ unsigned long ihk_mc_perfctr_read(int counter)
 	return count;
 }
 
-//int ihk_mc_perfctr_alloc_counter(unsigned long pmc_status)
-int ihk_mc_perfctr_alloc_counter(unsigned int *type, unsigned long *config, unsigned long pmc_status) /* 0416_patchtemp */
+unsigned long ihk_mc_perfctr_value(int counter, unsigned long correction)
+{
+	unsigned long count = ihk_mc_perfctr_read(counter) + correction;
+
+	count &= ((1UL << 32) - 1);
+	return count;
+}
+
+int ihk_mc_perfctr_alloc_counter(unsigned int *type, unsigned long *config,
+				 unsigned long pmc_status)
 {
 	int ret;
-	ret = cpu_pmu.get_event_idx(cpu_pmu.num_events, pmc_status);
+
+	if (*type == PERF_TYPE_HARDWARE) {
+		switch (*config) {
+		case PERF_COUNT_HW_INSTRUCTIONS:
+			ret = cpu_pmu.map_event(*type, *config);
+			if (ret < 0) {
+				return -1;
+			}
+			*type = PERF_TYPE_RAW;
+			break;
+		default:
+			// Unexpected config
+			return -1;
+		}
+	}
+	else if (*type != PERF_TYPE_RAW) {
+		return -1;
+	}
+	ret = cpu_pmu.get_event_idx(get_per_cpu_pmu()->num_events, pmc_status,
+				    *config);
         return ret;
 }
 
-/* 0416_patchtemp */
-/* ihk_mc_perfctr_fixed_init() stub added. */
-int ihk_mc_perfctr_fixed_init(int counter, int mode)
+int ihk_mc_perf_counter_mask_check(unsigned long counter_mask)
 {
-	return -1;
+	return cpu_pmu.counter_mask_valid(counter_mask);
+}
+
+int ihk_mc_perf_get_num_counters(void)
+{
+	const struct per_cpu_arm_pmu *per_cpu_arm_pmu = get_per_cpu_pmu();
+
+	return per_cpu_arm_pmu->num_events;
+}
+
+int ihk_mc_perfctr_set_extra(struct mc_perf_event *event)
+{
+	/* Nothing to do. */
+	return 0;
+}
+
+static inline uint64_t arm_pmu_event_max_period(struct mc_perf_event *event)
+{
+	return 0xFFFFFFFF;
+}
+
+int hw_perf_event_init(struct mc_perf_event *event)
+{
+	struct hw_perf_event *hwc = &event->hw;
+
+	if (!is_sampling_event(event)) {
+		hwc->sample_period  = arm_pmu_event_max_period(event) >> 1;
+		hwc->last_period    = hwc->sample_period;
+		ihk_atomic64_set(&hwc->period_left, hwc->sample_period);
+	}
+	return 0;
+}
+
+int ihk_mc_event_set_period(struct mc_perf_event *event)
+{
+	struct hw_perf_event *hwc = &event->hw;
+	int64_t left = ihk_atomic64_read(&hwc->period_left);
+	int64_t period = hwc->sample_period;
+	uint64_t max_period;
+	int ret = 0;
+
+	max_period = arm_pmu_event_max_period(event);
+	if (unlikely(left <= -period)) {
+		left = period;
+		ihk_atomic64_set(&hwc->period_left, left);
+		hwc->last_period = period;
+		ret = 1;
+	}
+
+	if (unlikely(left <= 0)) {
+		left += period;
+		ihk_atomic64_set(&hwc->period_left, left);
+		hwc->last_period = period;
+		ret = 1;
+	}
+
+	/*
+	 * Limit the maximum period to prevent the counter value
+	 * from overtaking the one we are about to program. In
+	 * effect we are reducing max_period to account for
+	 * interrupt latency (and we are being very conservative).
+	 */
+	if (left > (max_period >> 1))
+		left = (max_period >> 1);
+
+	ihk_atomic64_set(&hwc->prev_count, (uint64_t)-left);
+
+	cpu_pmu.write_counter(event->counter_id,
+			      (uint64_t)(-left) & max_period);
+
+	return ret;
+}
+
+uint64_t ihk_mc_event_update(struct mc_perf_event *event)
+{
+	struct hw_perf_event *hwc = &event->hw;
+	int64_t delta;
+	uint64_t prev_raw_count, new_raw_count;
+	uint64_t max_period = arm_pmu_event_max_period(event);
+
+again:
+	prev_raw_count = ihk_atomic64_read(&hwc->prev_count);
+	new_raw_count = cpu_pmu.read_counter(event->counter_id);
+
+	if (ihk_atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
+			     new_raw_count) != prev_raw_count)
+		goto again;
+
+	delta = (new_raw_count - prev_raw_count) & max_period;
+
+	ihk_atomic64_add(delta, &event->count);
+	ihk_atomic64_add(-delta, &hwc->period_left);
+
+	return new_raw_count;
 }

arch/arm64/kernel/proc.S

@@ -30,7 +30,7 @@
  */
 #if defined(CONFIG_HAS_NMI)
 #include <arm-gic-v3.h>
-ENTRY(cpu_do_idle)
+ENTRY(__cpu_do_idle)
 	mrs	x0, daif			// save I bit
 	msr	daifset, #2			// set I bit
 	mrs_s	x1, ICC_PMR_EL1			// save PMR
@@ -41,13 +41,13 @@ ENTRY(cpu_do_idle)
 	msr_s	ICC_PMR_EL1, x1			// restore PMR
 	msr	daif, x0			// restore I bit
 	ret
-ENDPROC(cpu_do_idle)
+ENDPROC(__cpu_do_idle)
 #else /* defined(CONFIG_HAS_NMI) */
-ENTRY(cpu_do_idle)
+ENTRY(__cpu_do_idle)
 	dsb	sy				// WFI may enter a low-power mode
 	wfi
 	ret
-ENDPROC(cpu_do_idle)
+ENDPROC(__cpu_do_idle)
 #endif /* defined(CONFIG_HAS_NMI) */
 
 /*

arch/arm64/kernel/psci.c

@@ -1,4 +1,4 @@
-/* psci.c COPYRIGHT FUJITSU LIMITED 2015-2016 */
+/* psci.c COPYRIGHT FUJITSU LIMITED 2015-2018 */
 /* @ref.impl arch/arm64/kernel/psci.c */
 /*
  * This program is free software; you can redistribute it and/or modify
@@ -18,18 +18,15 @@
 #include <psci.h>
 #include <errno.h>
 #include <ihk/types.h>
-#include <ihk/debug.h>
 #include <compiler.h>
 #include <lwk/compiler.h>
+#include <ihk/debug.h>
 
 //#define DEBUG_PRINT_PSCI
 
 #ifdef DEBUG_PRINT_PSCI
-#define dkprintf(...) kprintf(__VA_ARGS__)
-#define ekprintf(...) kprintf(__VA_ARGS__)
-#else
-#define dkprintf(...) do { if (0) kprintf(__VA_ARGS__); } while (0)
-#define ekprintf(...) kprintf(__VA_ARGS__)
+#undef DDEBUG_DEFAULT
+#define DDEBUG_DEFAULT DDEBUG_PRINT
 #endif
 
 #define PSCI_POWER_STATE_TYPE_POWER_DOWN	1

arch/arm64/kernel/ptrace.c

@@ -1,4 +1,4 @@
-/* ptrace.c COPYRIGHT FUJITSU LIMITED 2016-2017 */
+/* ptrace.c COPYRIGHT FUJITSU LIMITED 2016-2019 */
 #include <errno.h>
 #include <debug-monitors.h>
 #include <hw_breakpoint.h>
@@ -11,57 +11,21 @@
 #include <hwcap.h>
 #include <string.h>
 #include <thread_info.h>
+#include <ptrace.h>
+#include <ihk/debug.h>
 
 //#define DEBUG_PRINT_SC
 
 #ifdef DEBUG_PRINT_SC
-#define dkprintf kprintf
-#define ekprintf(...) do { if (0) kprintf(__VA_ARGS__); } while (0)
-#else
-#define dkprintf(...) do { if (0) kprintf(__VA_ARGS__); } while (0)
-#define ekprintf(...) do { if (0) kprintf(__VA_ARGS__); } while (0)
+#undef DDEBUG_DEFAULT
+#define DDEBUG_DEFAULT DDEBUG_PRINT
 #endif
 
 #define NOT_IMPLEMENTED()  do { kprintf("%s is not implemented\n", __func__); while(1);} while(0)
 
-#define BUG_ON(condition) do { if (condition) { kprintf("PANIC: %s: %s(line:%d)\n",\
-				__FILE__, __FUNCTION__, __LINE__); panic(""); } } while(0)
-
 extern void save_debugreg(unsigned long *debugreg);
-extern unsigned long do_kill(struct thread *thread, int pid, int tid, int sig, struct siginfo *info, int ptracecont);
 extern int interrupt_from_user(void *);
 
-enum aarch64_regset {
-	REGSET_GPR,
-	REGSET_FPR,
-	REGSET_TLS,
-	REGSET_HW_BREAK,
-	REGSET_HW_WATCH,
-	REGSET_SYSTEM_CALL,
-#ifdef CONFIG_ARM64_SVE
-	REGSET_SVE,
-#endif /* CONFIG_ARM64_SVE */
-};
-
-struct user_regset;
-typedef long user_regset_get_fn(struct thread *target,
-				const struct user_regset *regset,
-				unsigned int pos, unsigned int count,
-				void *kbuf, void __user *ubuf);
-
-typedef long user_regset_set_fn(struct thread *target,
-				const struct user_regset *regset,
-				unsigned int pos, unsigned int count,
-				const void *kbuf, const void __user *ubuf);
-
-struct user_regset {
-	user_regset_get_fn *get;
-	user_regset_set_fn *set;
-	unsigned int n;
-	unsigned int size;
-	unsigned int core_note_type;
-};
-
 long ptrace_read_user(struct thread *thread, long addr, unsigned long *value)
 {
 	return -EIO;
@@ -279,6 +243,17 @@ static inline long copy_regset_from_user(struct thread *target,
 	return regset->set(target, regset, offset, size, NULL, data);
 }
 
+unsigned int regset_size(struct thread *target,
+			 const struct user_regset *regset)
+{
+	if (!regset->get_size) {
+		return regset->n * regset->size;
+	}
+	else {
+		return regset->get_size(target, regset);
+	}
+}
+
 /*
  * Bits which are always architecturally RES0 per ARM DDI 0487A.h
  * Userspace cannot use these until they have an architectural meaning.
@@ -630,6 +605,48 @@ out:
 
 #ifdef CONFIG_ARM64_SVE
 
+static void sve_init_header_from_thread(struct user_sve_header *header,
+					struct thread *target)
+{
+	unsigned int vq;
+
+	memset(header, 0, sizeof(*header));
+
+	/* McKernel processes always enable SVE. */
+	header->flags = SVE_PT_REGS_SVE;
+
+	if (target->ctx.thread->sve_flags & SVE_PT_VL_INHERIT) {
+		header->flags |= SVE_PT_VL_INHERIT;
+	}
+
+	header->vl = target->ctx.thread->sve_vl;
+	vq = sve_vq_from_vl(header->vl);
+
+	header->max_vl = sve_max_vl;
+	header->size = SVE_PT_SIZE(vq, header->flags);
+	header->max_size = SVE_PT_SIZE(sve_vq_from_vl(header->max_vl),
+				       SVE_PT_REGS_SVE);
+}
+
+static unsigned int sve_size_from_header(struct user_sve_header const *header)
+{
+	return ALIGN(header->size, SVE_VQ_BYTES);
+}
+
+static unsigned int sve_get_size(struct thread *target,
+				 const struct user_regset *regset)
+{
+	struct user_sve_header header;
+
+	/* Instead of system_supports_sve() */
+	if (unlikely(!(elf_hwcap & HWCAP_SVE))) {
+		return 0;
+	}
+
+	sve_init_header_from_thread(&header, target);
+	return sve_size_from_header(&header);
+}
+
 /* read NT_ARM_SVE */
 static long sve_get(struct thread *target,
 		    const struct user_regset *regset,
@@ -652,23 +669,9 @@ static long sve_get(struct thread *target,
 	}
 
 	/* Header */
-	memset(&header, 0, sizeof(header));
-
-	header.vl = target->ctx.thread->sve_vl;
-
-	BUG_ON(!sve_vl_valid(header.vl));
+	sve_init_header_from_thread(&header, target);
 	vq = sve_vq_from_vl(header.vl);
 
-	BUG_ON(!sve_vl_valid(sve_max_vl));
-	header.max_vl = sve_max_vl;
-
-	/* McKernel processes always enable SVE. */
-	header.flags = SVE_PT_REGS_SVE;
-
-	header.size = SVE_PT_SIZE(vq, header.flags);
-	header.max_size = SVE_PT_SIZE(sve_vq_from_vl(header.max_vl),
-				      SVE_PT_REGS_SVE);
-
 	ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, &header,
 				  0, sizeof(header));
 	if (ret) {
@@ -682,11 +685,9 @@ static long sve_get(struct thread *target,
 	 */
 
 	/* Otherwise: full SVE case */
+
 	start = SVE_PT_SVE_OFFSET;
 	end = SVE_PT_SVE_FFR_OFFSET(vq) + SVE_PT_SVE_FFR_SIZE(vq);
-
-	BUG_ON(end < start);
-	BUG_ON(end - start > sve_state_size(target));
 	ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
 				  target->ctx.thread->sve_state,
 				  start, end);
@@ -696,24 +697,18 @@ static long sve_get(struct thread *target,
 
 	start = end;
 	end = SVE_PT_SVE_FPSR_OFFSET(vq);
-
-	BUG_ON(end < start);
 	ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
 				       start, end);
 	if (ret) {
 		goto out;
 	}
 
+	/*
+	 * Copy fpsr, and fpcr which must follow contiguously in
+	 * struct fpsimd_state:
+	 */
 	start = end;
 	end = SVE_PT_SVE_FPCR_OFFSET(vq) + SVE_PT_SVE_FPCR_SIZE;
-
-	BUG_ON((char *)(&target->fp_regs->fpcr + 1) <
-	       (char *)&target->fp_regs->fpsr);
-	BUG_ON(end < start);
-	BUG_ON((char *)(&target->fp_regs->fpcr + 1) -
-	       (char *)&target->fp_regs->fpsr !=
-	        end - start);
-
 	ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
 				  &target->fp_regs->fpsr,
 				  start, end);
@@ -722,9 +717,7 @@ static long sve_get(struct thread *target,
 	}
 
 	start = end;
-	end = (SVE_PT_SIZE(SVE_VQ_MAX, SVE_PT_REGS_SVE) + 15) / 16 * 16;
-
-	BUG_ON(end < start);
+	end = sve_size_from_header(&header);
 	ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
 				       start, end);
 out:
@@ -768,13 +761,12 @@ static long sve_set(struct thread *target,
 	 * sve_set_vector_length(), which will also validate them for us:
 	 */
 	ret = sve_set_vector_length(target, header.vl,
-				    header.flags & ~SVE_PT_REGS_MASK);
+		((unsigned long)header.flags & ~SVE_PT_REGS_MASK) << 16);
 	if (ret) {
 		goto out;
 	}
 
 	/* Actual VL set may be less than the user asked for: */
-	BUG_ON(!sve_vl_valid(target->ctx.thread->sve_vl));
 	vq = sve_vq_from_vl(target->ctx.thread->sve_vl);
 
 	/* Registers: FPSIMD-only case */
@@ -785,11 +777,19 @@ static long sve_set(struct thread *target,
 	}
 
 	/* Otherwise: full SVE case */
+
+	/*
+	 * If setting a different VL from the requested VL and there is
+	 * register data, the data layout will be wrong: don't even
+	 * try to set the registers in this case.
+	 */
+	if (count && vq != sve_vq_from_vl(header.vl)) {
+		ret = -EIO;
+		goto out;
+	}
+
 	start = SVE_PT_SVE_OFFSET;
 	end = SVE_PT_SVE_FFR_OFFSET(vq) + SVE_PT_SVE_FFR_SIZE(vq);
-
-	BUG_ON(end < start);
-	BUG_ON(end - start > sve_state_size(target));
 	ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
 				 target->ctx.thread->sve_state,
 				 start, end);
@@ -799,27 +799,21 @@ static long sve_set(struct thread *target,
 
 	start = end;
 	end = SVE_PT_SVE_FPSR_OFFSET(vq);
-
-	BUG_ON(end < start);
 	ret = user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf,
 					start, end);
 	if (ret) {
 		goto out;
 	}
 
+	/*
+	 * Copy fpsr, and fpcr which must follow contiguously in
+	 * struct fpsimd_state:
+	 */
 	start = end;
 	end = SVE_PT_SVE_FPCR_OFFSET(vq) + SVE_PT_SVE_FPCR_SIZE;
-
-	BUG_ON((char *)(&target->fp_regs->fpcr + 1) <
-		(char *)&target->fp_regs->fpsr);
-	BUG_ON(end < start);
-	BUG_ON((char *)(&target->fp_regs->fpcr + 1) -
-		(char *)&target->fp_regs->fpsr !=
-		 end - start);
-
-	user_regset_copyin(&pos, &count, &kbuf, &ubuf,
-			   &target->fp_regs->fpsr,
-			   start, end);
+	ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
+				 &target->fp_regs->fpsr,
+				 start, end);
 out:
 	return ret;
 }
@@ -831,8 +825,9 @@ static const struct user_regset aarch64_regsets[] = {
 		.core_note_type = NT_PRSTATUS,
 		.n = sizeof(struct user_pt_regs) / sizeof(uint64_t),
 		.size = sizeof(uint64_t),
+		.align = sizeof(uint64_t),
 		.get = gpr_get,
-		.set = gpr_set
+		.set = gpr_set,
 	},
 	[REGSET_FPR] = {
 		.core_note_type = NT_PRFPREG,
@@ -842,56 +837,75 @@ static const struct user_regset aarch64_regsets[] = {
 		 * fpcr are 32-bits wide.
 		 */
 		.size = sizeof(uint32_t),
+		.align = sizeof(uint32_t),
 		.get = fpr_get,
-		.set = fpr_set
+		.set = fpr_set,
 	},
 	[REGSET_TLS] = {
 		.core_note_type = NT_ARM_TLS,
 		.n = 1,
 		.size = sizeof(void *),
+		.align = sizeof(void *),
 		.get = tls_get,
-		.set = tls_set
+		.set = tls_set,
 	},
 	[REGSET_HW_BREAK] = {
 		.core_note_type = NT_ARM_HW_BREAK,
 		.n = sizeof(struct user_hwdebug_state) / sizeof(uint32_t),
 		.size = sizeof(uint32_t),
+		.align = sizeof(uint32_t),
 		.get = hw_break_get,
-		.set = hw_break_set
+		.set = hw_break_set,
 	},
 	[REGSET_HW_WATCH] = {
 		.core_note_type = NT_ARM_HW_WATCH,
 		.n = sizeof(struct user_hwdebug_state) / sizeof(uint32_t),
 		.size = sizeof(uint32_t),
+		.align = sizeof(uint32_t),
 		.get = hw_break_get,
-		.set = hw_break_set
+		.set = hw_break_set,
 	},
 	[REGSET_SYSTEM_CALL] = {
 		.core_note_type = NT_ARM_SYSTEM_CALL,
 		.n = 1,
 		.size = sizeof(int),
+		.align = sizeof(int),
 		.get = system_call_get,
-		.set = system_call_set
+		.set = system_call_set,
 	},
 #ifdef CONFIG_ARM64_SVE
 	[REGSET_SVE] = { /* Scalable Vector Extension */
 		.core_note_type = NT_ARM_SVE,
-		.n = (SVE_PT_SIZE(SVE_VQ_MAX, SVE_PT_REGS_SVE) + 15) / 16,
-		.size = 16,
+		.n = (SVE_PT_SIZE(SVE_VQ_MAX, SVE_PT_REGS_SVE) +
+			(SVE_VQ_BYTES - 1)) / SVE_VQ_BYTES,
+		.size = SVE_VQ_BYTES,
+		.align = SVE_VQ_BYTES,
 		.get = sve_get,
-		.set = sve_set
+		.set = sve_set,
+		.get_size = sve_get_size,
 	},
 #endif /* CONFIG_ARM64_SVE */
 };
 
-static const struct user_regset *
-find_regset(const struct user_regset *regset, unsigned int type, int n)
+static const struct user_regset_view user_aarch64_view = {
+	.name = "aarch64", .e_machine = EM_AARCH64,
+	.regsets = aarch64_regsets,
+	.n = sizeof(aarch64_regsets) / sizeof(aarch64_regsets[0])
+};
+
+const struct user_regset_view *current_user_regset_view(void)
+{
+	return &user_aarch64_view;
+}
+
+const struct user_regset *find_regset(const struct user_regset_view *view,
+				      unsigned int type)
 {
 	int i = 0;
 
-	for (i = 0; i < n; i++) {
-		if (regset[i].core_note_type == type) {
-			return &regset[i];
+	for (i = 0; i < view->n; i++) {
+		if (view->regsets[i].core_note_type == type) {
+			return &view->regsets[i];
 		}
 	}
 	return NULL;
@@ -900,8 +914,8 @@ find_regset(const struct user_regset *regset, unsigned int type, int n)
 static long ptrace_regset(struct thread *thread, int req, long type, struct iovec *iov)
 {
 	long rc = -EINVAL;
-	const struct user_regset *regset = find_regset(aarch64_regsets, type,
-					sizeof(aarch64_regsets) / sizeof(aarch64_regsets[0]));
+	const struct user_regset *regset =
+		find_regset(&user_aarch64_view, type);
 
 	if (!regset) {
 		kprintf("%s: not supported type 0x%x\n", __FUNCTION__, type);
@@ -950,30 +964,43 @@ void ptrace_report_signal(struct thread *thread, int sig)
 	/* save thread_info, if called by ptrace_report_exec() */
 	if (sig == ((SIGTRAP | (PTRACE_EVENT_EXEC << 8)))) {
 		memcpy(&tinfo, thread->ctx.thread, sizeof(struct thread_info));
+		thread->uctx->user_regs.regs[0] = 0;
 	}
 
 	mcs_rwlock_writer_lock(&proc->update_lock, &lock);
-	if(!(proc->ptrace & PT_TRACED)){
+	if (!(thread->ptrace & PT_TRACED)) {
 		mcs_rwlock_writer_unlock(&proc->update_lock, &lock);
 		return;
 	}
-	thread->exit_status = sig;
+
 	/* Transition thread state */
-#ifdef POSTK_DEBUG_TEMP_FIX_41 /* early to wait4() wakeup for ptrace, fix. */
-	proc->status = PS_DELAY_TRACED;
-#else /* POSTK_DEBUG_TEMP_FIX_41 */
-	proc->status = PS_TRACED;
-#endif /* POSTK_DEBUG_TEMP_FIX_41 */
+	thread->exit_status = sig;
 	thread->status = PS_TRACED;
-	proc->ptrace &= ~PT_TRACE_SYSCALL;
-	if (sig == SIGSTOP || sig == SIGTSTP ||
-			sig == SIGTTIN || sig == SIGTTOU) {
-		proc->signal_flags |= SIGNAL_STOP_STOPPED;
-	} else {
-		proc->signal_flags &= ~SIGNAL_STOP_STOPPED;
+	thread->ptrace &= ~PT_TRACE_SYSCALL;
+	if (sig == ((SIGTRAP | (PTRACE_EVENT_EXEC << 8))) &&
+				thread->ptrace & PTRACE_O_TRACEEXEC) {
+		/* PTRACE_O_TRACEEXEC: since Linux 3.0, the former
+		 * thread ID can be retrieved with PTRACE_GETEVENTMSG.
+		 * Report no change. */
+		thread->ptrace_eventmsg = thread->tid;
 	}
-	parent_pid = proc->parent->pid;
 	save_debugreg(thread->ptrace_debugreg);
+	if (sig == SIGSTOP || sig == SIGTSTP ||
+	    sig == SIGTTIN || sig == SIGTTOU) {
+		thread->signal_flags |= SIGNAL_STOP_STOPPED;
+	}
+	else {
+		thread->signal_flags &= ~SIGNAL_STOP_STOPPED;
+	}
+
+	if (thread == proc->main_thread) {
+		proc->status = PS_DELAY_TRACED;
+		parent_pid = proc->parent->pid;
+	}
+	else {
+		parent_pid = thread->report_proc->pid;
+		waitq_wakeup(&thread->report_proc->waitpid_q);
+	}
 	mcs_rwlock_writer_unlock(&proc->update_lock, &lock);
 
 	memset(&info, '\0', sizeof info);
@@ -982,10 +1009,6 @@ void ptrace_report_signal(struct thread *thread, int sig)
 	info._sifields._sigchld.si_pid = thread->tid;
 	info._sifields._sigchld.si_status = thread->exit_status;
 	do_kill(cpu_local_var(current), parent_pid, -1, SIGCHLD, &info, 0);
-#ifndef POSTK_DEBUG_TEMP_FIX_41 /* early to wait4() wakeup for ptrace, fix. */
-	/* Wake parent (if sleeping in wait4()) */
-	waitq_wakeup(&proc->parent->waitpid_q);
-#endif /* !POSTK_DEBUG_TEMP_FIX_41 */
 
 	dkprintf("ptrace_report_signal,sleeping\n");
 	/* Sleep */
@@ -996,6 +1019,7 @@ void ptrace_report_signal(struct thread *thread, int sig)
 	if (sig == ((SIGTRAP | (PTRACE_EVENT_EXEC << 8)))) {
 		memcpy(thread->ctx.thread, &tinfo, sizeof(struct thread_info));
 	}
+	arch_flush_icache_all();
 }
 
 long

arch/arm64/kernel/timer.c

@@ -0,0 +1,203 @@
+/* timer.c COPYRIGHT FUJITSU LIMITED 2018 */
+#include <ihk/types.h>
+#include <ihk/cpu.h>
+#include <ihk/lock.h>
+#include <sysreg.h>
+#include <kmalloc.h>
+#include <cls.h>
+#include <cputype.h>
+#include <irq.h>
+#include <arch-timer.h>
+#include <ihk/debug.h>
+
+//#define DEBUG_PRINT_TIMER
+
+#ifdef DEBUG_PRINT_TIMER
+#undef DDEBUG_DEFAULT
+#define DDEBUG_DEFAULT DDEBUG_PRINT
+#endif
+
+static unsigned int per_cpu_timer_val[NR_CPUS] = { 0 };
+static int timer_intrid = INTRID_VIRT_TIMER;
+
+static void arch_timer_virt_reg_write(enum arch_timer_reg reg, uint32_t val);
+static void (*arch_timer_reg_write)(enum arch_timer_reg, uint32_t) =
+	arch_timer_virt_reg_write;
+
+static uint32_t arch_timer_virt_reg_read(enum arch_timer_reg reg);
+static uint32_t (*arch_timer_reg_read)(enum arch_timer_reg) =
+	arch_timer_virt_reg_read;
+
+static void arch_timer_phys_reg_write(enum arch_timer_reg reg, uint32_t val)
+{
+	switch (reg) {
+	case ARCH_TIMER_REG_CTRL:
+		write_sysreg(val, cntp_ctl_el0);
+		break;
+	case ARCH_TIMER_REG_TVAL:
+		write_sysreg(val, cntp_tval_el0);
+		break;
+	}
+	isb();
+}
+
+static void arch_timer_virt_reg_write(enum arch_timer_reg reg, uint32_t val)
+{
+	switch (reg) {
+	case ARCH_TIMER_REG_CTRL:
+		write_sysreg(val, cntv_ctl_el0);
+		break;
+	case ARCH_TIMER_REG_TVAL:
+		write_sysreg(val, cntv_tval_el0);
+		break;
+	}
+	isb();
+}
+
+static uint32_t arch_timer_phys_reg_read(enum arch_timer_reg reg)
+{
+	uint32_t val = 0;
+
+	switch (reg) {
+	case ARCH_TIMER_REG_CTRL:
+		val = read_sysreg(cntp_ctl_el0);
+		break;
+	case ARCH_TIMER_REG_TVAL:
+		val = read_sysreg(cntp_tval_el0);
+		break;
+	}
+	return val;
+}
+
+static uint32_t arch_timer_virt_reg_read(enum arch_timer_reg reg)
+{
+	uint32_t val = 0;
+
+	switch (reg) {
+	case ARCH_TIMER_REG_CTRL:
+		val = read_sysreg(cntv_ctl_el0);
+		break;
+	case ARCH_TIMER_REG_TVAL:
+		val = read_sysreg(cntv_tval_el0);
+		break;
+	}
+	return val;
+}
+
+static void timer_handler(void *priv)
+{
+	unsigned long ctrl;
+	const int cpu = ihk_mc_get_processor_id();
+
+	dkprintf("CPU%d: catch %s timer\n", cpu,
+		 ((timer_intrid == INTRID_PHYS_TIMER) ||
+		  (timer_intrid == INTRID_HYP_PHYS_TIMER)) ?
+		 "physical" : "virtual");
+
+	ctrl = arch_timer_reg_read(ARCH_TIMER_REG_CTRL);
+	if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
+		const unsigned int clocks = per_cpu_timer_val[cpu];
+		struct cpu_local_var *v = get_this_cpu_local_var();
+		unsigned long irqstate;
+
+		/* set resched flag */
+		irqstate = ihk_mc_spinlock_lock(&v->runq_lock);
+		v->flags |= CPU_FLAG_NEED_RESCHED;
+		ihk_mc_spinlock_unlock(&v->runq_lock, irqstate);
+
+		/* gen control register value */
+		ctrl &= ~ARCH_TIMER_CTRL_IT_STAT;
+
+		/* set timer re-enable for periodic */
+		arch_timer_reg_write(ARCH_TIMER_REG_TVAL, clocks);
+		arch_timer_reg_write(ARCH_TIMER_REG_CTRL, ctrl);
+
+		do_backlog();
+	}
+}
+
+static unsigned long is_use_virt_timer(void)
+{
+	extern unsigned long ihk_param_use_virt_timer;
+
+	switch (ihk_param_use_virt_timer) {
+	case 0: /* physical */
+	case 1: /* virtual */
+		break;
+	default: /* invalid */
+		panic("PANIC: is_use_virt_timer(): timer select neither phys-timer nor virt-timer.\n");
+		break;
+	}
+	return ihk_param_use_virt_timer;
+}
+
+static struct ihk_mc_interrupt_handler timer_interrupt_handler = {
+	.func = timer_handler,
+	.priv = NULL,
+};
+
+/* other source use functions */
+
+struct ihk_mc_interrupt_handler *get_timer_handler(void)
+{
+	return &timer_interrupt_handler;
+}
+
+void
+lapic_timer_enable(unsigned int clocks)
+{
+	unsigned long ctrl = 0;
+
+	/* gen control register value */
+	ctrl = arch_timer_reg_read(ARCH_TIMER_REG_CTRL);
+	ctrl |= ARCH_TIMER_CTRL_ENABLE;
+	ctrl &= ~(ARCH_TIMER_CTRL_IT_MASK | ARCH_TIMER_CTRL_IT_STAT);
+	arch_timer_reg_write(ARCH_TIMER_REG_TVAL, clocks);
+	arch_timer_reg_write(ARCH_TIMER_REG_CTRL, ctrl);
+
+	per_cpu_timer_val[ihk_mc_get_processor_id()] = clocks;
+}
+
+void
+lapic_timer_disable()
+{
+	unsigned long ctrl = 0;
+
+	ctrl = arch_timer_reg_read(ARCH_TIMER_REG_CTRL);
+	ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
+	arch_timer_reg_write(ARCH_TIMER_REG_CTRL, ctrl);
+
+	per_cpu_timer_val[ihk_mc_get_processor_id()] = 0;
+}
+
+int get_timer_intrid(void)
+{
+	return timer_intrid;
+}
+
+void arch_timer_init(void)
+{
+	const unsigned long is_virt = is_use_virt_timer();
+#ifdef CONFIG_ARM64_VHE
+	const unsigned long mmfr = read_cpuid(ID_AA64MMFR1_EL1);
+#endif /* CONFIG_ARM64_VHE */
+
+	if (is_virt) {
+		timer_intrid = INTRID_VIRT_TIMER;
+		arch_timer_reg_write = arch_timer_virt_reg_write;
+		arch_timer_reg_read = arch_timer_virt_reg_read;
+	} else {
+		timer_intrid = INTRID_PHYS_TIMER;
+		arch_timer_reg_write = arch_timer_phys_reg_write;
+		arch_timer_reg_read = arch_timer_phys_reg_read;
+	}
+#ifdef CONFIG_ARM64_VHE
+	if ((mmfr >> ID_AA64MMFR1_VHE_SHIFT) & 1UL) {
+		if (is_virt) {
+			timer_intrid = INTRID_HYP_VIRT_TIMER;
+		} else {
+			timer_intrid = INTRID_HYP_PHYS_TIMER;
+		}
+	}
+#endif /* CONFIG_ARM64_VHE */
+}

arch/arm64/kernel/traps.c

@@ -1,4 +1,4 @@
-/* traps.c COPYRIGHT FUJITSU LIMITED 2015-2017 */
+/* traps.c COPYRIGHT FUJITSU LIMITED 2015-2018 */
 #include <ihk/context.h>
 #include <ihk/debug.h>
 #include <traps.h>
@@ -29,12 +29,14 @@ void arm64_notify_die(const char *str, struct pt_regs *regs, struct siginfo *inf
  */
 void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs)
 {
+	const int from_user = interrupt_from_user(regs);
+
 	// /* TODO: implement lazy context saving/restoring */
-	set_cputime(1);
+	set_cputime(from_user ? CPUTIME_MODE_U2K : CPUTIME_MODE_K2K_IN);
 	// WARN_ON(1);
 	kprintf("WARNING: CPU: %d PID: %d Trapped FP/ASIMD access.\n",
 		ihk_mc_get_processor_id(), cpu_local_var(current)->proc->pid);
-	set_cputime(0);
+	set_cputime(from_user ? CPUTIME_MODE_K2U : CPUTIME_MODE_K2K_OUT);
 }
 
 /*
@@ -51,7 +53,9 @@ void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)
 {
 	siginfo_t info;
 	unsigned int si_code = 0;
-	set_cputime(1);
+	const int from_user = interrupt_from_user(regs);
+
+	set_cputime(from_user ? CPUTIME_MODE_U2K : CPUTIME_MODE_K2K_IN);
 
 	if (esr & FPEXC_IOF)
 		si_code = FPE_FLTINV;
@@ -70,7 +74,7 @@ void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)
 	info._sifields._sigfault.si_addr  = (void*)regs->pc;
 
 	set_signal(SIGFPE, regs, &info);
-	set_cputime(0);
+	set_cputime(from_user ? CPUTIME_MODE_K2U : CPUTIME_MODE_K2K_OUT);
 }
 
 /* @ref.impl arch/arm64/kernel/traps.c */
@@ -133,8 +137,9 @@ exit:
 void do_undefinstr(struct pt_regs *regs)
 {
 	siginfo_t info;
+	const int from_user = interrupt_from_user(regs);
 
-	set_cputime(interrupt_from_user(regs)? 1: 2);
+	set_cputime(from_user ? CPUTIME_MODE_U2K : CPUTIME_MODE_K2K_IN);
 
 	if (call_undef_hook(regs) == 0) {
 		goto out;
@@ -147,7 +152,7 @@ void do_undefinstr(struct pt_regs *regs)
 
 	arm64_notify_die("Oops - undefined instruction", regs, &info, 0);
 out:
-	set_cputime(0);
+	set_cputime(from_user ? CPUTIME_MODE_K2U : CPUTIME_MODE_K2K_OUT);
 }
 
 /*
@@ -157,7 +162,9 @@ out:
 void bad_mode(struct pt_regs *regs, int reason, unsigned int esr)
 {
 	siginfo_t info;
-	set_cputime(interrupt_from_user(regs)? 1: 2);
+	const int from_user = interrupt_from_user(regs);
+
+	set_cputime(from_user ? CPUTIME_MODE_U2K : CPUTIME_MODE_K2K_IN);
 	kprintf("entering bad_mode !! (regs:0x%p, reason:%d, esr:0x%x)\n", regs, reason, esr);
 
 	kprintf("esr Analyse:\n");
@@ -167,11 +174,16 @@ void bad_mode(struct pt_regs *regs, int reason, unsigned int esr)
 
 	arch_show_interrupt_context(regs);
 
+#ifdef ENABLE_TOFU
+	info.si_signo = SIGSTOP;
+	info.si_errno = 0;
+#else
 	info.si_signo = SIGILL;
 	info.si_errno = 0;
 	info.si_code  = ILL_ILLOPC;
+#endif
 	info._sifields._sigfault.si_addr  = (void*)regs->pc;
 
 	arm64_notify_die("Oops - bad mode", regs, &info, 0);
-	set_cputime(0);
+	set_cputime(from_user ? CPUTIME_MODE_K2U : CPUTIME_MODE_K2K_OUT);
 }

arch/arm64/kernel/vdso.c

@@ -1,4 +1,4 @@
-/* vdso.c COPYRIGHT FUJITSU LIMITED 2016 */
+/* vdso.c COPYRIGHT FUJITSU LIMITED 2016-2018 */
 /* @ref.impl arch/arm64/kernel/vdso.c */
 
 #include <arch-memory.h>
@@ -11,21 +11,17 @@
 #include <process.h>
 #include <string.h>
 #include <syscall.h>
-#include <ihk/debug.h>
 #include <ikc/queue.h>
 #include <vdso.h>
+#include <ihk/debug.h>
 
 //#define DEBUG_PRINT_VDSO
 
 #ifdef DEBUG_PRINT_VDSO
-#define	dkprintf(...)	kprintf(__VA_ARGS__)
-#define	ekprintf(...)	kprintf(__VA_ARGS__)
-#else
-#define dkprintf(...)	do { if (0) kprintf(__VA_ARGS__); } while (0)
-#define	ekprintf(...)	kprintf(__VA_ARGS__)
+#undef DDEBUG_DEFAULT
+#define DDEBUG_DEFAULT DDEBUG_PRINT
 #endif
 
-#ifdef POSTK_DEBUG_ARCH_DEP_52
 #define VDSO_MAXPAGES 1
 struct vdso {
 	long busy;
@@ -36,7 +32,6 @@ struct vdso {
 	long lbase;
 	long offset_sigtramp;
 };
-#endif /*POSTK_DEBUG_ARCH_DEP_52*/
 
 extern char vdso_start, vdso_end;
 static struct vdso vdso;
@@ -90,26 +85,9 @@ int arch_setup_vdso(void)
 		kprintf("Enable Host mapping vDSO.\n");
 		return 0;
 	}
-	kprintf("Enable McK mapping vDSO.\n");
 
-	if (memcmp(&vdso_start, "\177ELF", 4)) {
-		panic("vDSO is not a valid ELF object!\n");
-	}
-
-	vdso.vdso_npages = (&vdso_end - &vdso_start) >> PAGE_SHIFT;
-	dkprintf("vdso: %ld pages (%ld code @ %p, %ld data @ %p)\n",
-		vdso.vdso_npages + 1, vdso.vdso_npages, &vdso_start, 1L, &tod_data);
-	if (vdso.vdso_npages != 1) {
-		panic("vDSO is not a valid number of pages!\n");
-	}
-
-	vdso.vvar_phys = virt_to_phys((void *)&tod_data);
-	vdso.vdso_physlist[0] = virt_to_phys((void *)&vdso_start);
-
-	vdso.lbase = VDSO_LBASE;
-	vdso.offset_sigtramp = vdso_offset_sigtramp;
-
-	return 0;
+	panic("Only support host mapping vDSO");
+	return -1;
 }
 
 static int get_free_area(struct process_vm *vm, size_t len, intptr_t hint,
@@ -158,6 +136,7 @@ int arch_map_vdso(struct process_vm *vm)
 	unsigned long start, end;
 	unsigned long flag;
 	int ret;
+	struct vm_range *range;
 
 	vdso_text_len = vdso.vdso_npages << PAGE_SHIFT;
 	/* Be sure to map the data page */
@@ -176,7 +155,7 @@ int arch_map_vdso(struct process_vm *vm)
 	flag = VR_REMOTE | VR_PROT_READ;
 	flag |= VRFLAG_PROT_TO_MAXPROT(flag);
 	ret = add_process_memory_range(vm, start, end, vdso.vvar_phys, flag,
-				       NULL, 0, PAGE_SHIFT, NULL);
+				       NULL, 0, PAGE_SHIFT, NULL, &range);
 	if (ret != 0){
 		dkprintf("ERROR: adding memory range for tod_data\n");
 		goto exit;
@@ -188,7 +167,7 @@ int arch_map_vdso(struct process_vm *vm)
 	flag = VR_REMOTE | VR_PROT_READ | VR_PROT_EXEC;
 	flag |= VRFLAG_PROT_TO_MAXPROT(flag);
 	ret = add_process_memory_range(vm, start, end, vdso.vdso_physlist[0], flag,
-				       NULL, 0, PAGE_SHIFT, NULL);
+				       NULL, 0, PAGE_SHIFT, NULL, &range);
 	if (ret != 0) {
 		dkprintf("ERROR: adding memory range for vdso_text\n");
 

arch/arm64/kernel/vdso.so.S

@@ -1,33 +0,0 @@
-/* vdso.so.S COPYRIGHT FUJITSU LIMITED 2016 */
-/* @ref.impl arch/arm64/kernel/vdso/vdso.S */
-/*
- * Copyright (C) 2012 ARM Limited
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * Author: Will Deacon <will.deacon@arm.com>
- */
-
-#include <arch-memory.h>
-#include <vdso-so-path.h>
-
-	.section ".vdso.txet", "aw"
-
-	.globl vdso_start, vdso_end
-	.balign PAGE_SIZE
-vdso_start:
-	.incbin VDSO_SO_PATH
-	.balign PAGE_SIZE
-vdso_end:
-
-	.previous

arch/arm64/kernel/vdso/Makefile.in

@@ -1,131 +0,0 @@
-# Makefile.in COPYRIGHT FUJITSU LIMITED 2016
-# @ref.impl arch/arm64/kernel/vdso/Makefile
-# Building a vDSO image for AArch64.
-
-HOST_DIR=@KDIR@
-HOST_CONFIG=$(HOST_DIR)/.config
-HOST_KERNEL_CONFIG_ARM64_4K_PAGES=$(shell grep -E "^CONFIG_ARM64_4K_PAGES=y" $(HOST_CONFIG) | sed 's|CONFIG_ARM64_4K_PAGES=||g')
-HOST_KERNEL_CONFIG_ARM64_16K_PAGES=$(shell grep -E "^CONFIG_ARM64_16K_PAGES=y" $(HOST_CONFIG) | sed 's|CONFIG_ARM64_16K_PAGES=||g')
-HOST_KERNEL_CONFIG_ARM64_64K_PAGES=$(shell grep -E "^CONFIG_ARM64_64K_PAGES=y" $(HOST_CONFIG) | sed 's|CONFIG_ARM64_64K_PAGES=||g')
-
-VDSOSRC = @abs_srcdir@
-VDSOBUILD = @abs_builddir@
-INCDIR = $(VDSOSRC)/../include
-ECHO_SUFFIX = [VDSO]
-
-VDSOOBJS := gettimeofday.o
-DESTOBJS = $(addprefix $(VDSOBUILD)/, $(VDSOOBJS))
-
-VDSOASMOBJS := note.o sigreturn.o
-DESTASMOBJS = $(addprefix $(VDSOBUILD)/, $(VDSOASMOBJS))
-
-$(if $(VDSOSRC),,$(error IHK output directory is not specified))
-$(if $(TARGET),,$(error Target is not specified))
-
-#CFLAGS := -nostdinc -mlittle-endian -Wall -mabi=lp64 -Wa,-gdwarf-2
-CFLAGS := -nostdinc -mlittle-endian -Wall -Wa,-gdwarf-2
-CFLAGS += -D__KERNEL__ -I$(SRC)/include
-CFLAGS += -I$(SRC)/../lib/include -I$(INCDIR) -I$(IHKBASE)/smp/arm64/include
-CFLAGS += $(foreach i, $(shell seq 1 100), $(addprefix -DPOSTK_DEBUG_ARCH_DEP_, $(i)))
-CFLAGS += $(foreach i, $(shell seq 1 100), $(addprefix -DPOSTK_DEBUG_TEMP_FIX_, $(i)))
-
-LDFLAGS := -nostdinc -mlittle-endian -Wall -Wundef -Wstrict-prototypes
-LDFLAGS += -Wno-trigraphs -fno-strict-aliasing -fno-common
-LDFLAGS += -Werror-implicit-function-declaration -Wno-format-security
-#LDFLAGS += -std=gnu89 -mgeneral-regs-only -mabi=lp64 -O2
-LDFLAGS += -std=gnu89 -mgeneral-regs-only -O2
-LDFLAGS += -Wframe-larger-than=2048 -fno-stack-protector
-LDFLAGS += -fno-delete-null-pointer-checks -Wno-unused-but-set-variable
-LDFLAGS += -fno-omit-frame-pointer -fno-optimize-sibling-calls
-LDFLAGS += -fno-var-tracking-assignments -g -Wdeclaration-after-statement
-LDFLAGS += -Wno-pointer-sign -fno-strict-overflow -fconserve-stack
-LDFLAGS += -Werror=implicit-int -Werror=strict-prototypes -Werror=date-time
-LDFLAGS += -shared -fno-common -fno-builtin -nostdlib
-LDFLAGS += -Wl,-soname=linux-vdso.so.1 -Wl,--hash-style=sysv -Wl,-n -Wl,-T
-LDFLAGS += --param=allow-store-data-races=0 -DCC_HAVE_ASM_GOTO
-LDFLAGS += -D"KBUILD_STR(s)=\#s" -D"KBUILD_BASENAME=KBUILD_STR(vdso.so)"
-LDFLAGS += -D"KBUILD_MODNAME=KBUILD_STR(vdso.so)" -D__KERNEL__
-
-DEPSRCS = $(wildcard $(VDSOSRC)/*.c $(VDSOSRC)/*.S)
-
-CFLAGS_lds := -E -P -C -U$(ARCH)
-CFLAGS_lds += -nostdinc
-CFLAGS_lds += -mlittle-endian
-CFLAGS_lds += -D__KERNEL__
-CFLAGS_lds += -D__ASSEMBLY__
-CFLAGS_lds += -DLINKER_SCRIPT
-CFLAGS_lds += -DVDSO_LBASE=0
-ifeq ($(HOST_KERNEL_CONFIG_ARM64_4K_PAGES), y)
-CFLAGS_lds += -DPAGE_SIZE=0x1000
-endif
-ifeq ($(HOST_KERNEL_CONFIG_ARM64_16K_PAGES), y)
-CFLAGS_lds += -DPAGE_SIZE=0x4000
-endif
-ifeq ($(HOST_KERNEL_CONFIG_ARM64_64K_PAGES), y)
-CFLAGS_lds += -DPAGE_SIZE=0x10000
-endif
-
-#load mckernel config (append CPPFLAGS)
-include @abs_top_builddir@/../ihk/cokernel/$(TARGETDIR)/Makefile.predefines
-
-default: all
-
-.PHONY: all clean depend prepare
-
-all: depend $(VDSOBUILD)/vdso.so $(VDSOBUILD)/../include/vdso-offsets.h $(VDSOBUILD)/../include/vdso-so-path.h
-
-# Strip rule for the .so file
-$(VDSOBUILD)/vdso.so: OBJCOPYFLAGS := -S
-$(VDSOBUILD)/vdso.so: $(VDSOBUILD)/vdso.so.dbg
-	$(objcopy_cmd)
-
-# Generate VDSO offsets using helper script
-$(VDSOBUILD)/../include/vdso-offsets.h: $(VDSOBUILD)/vdso.so.dbg
-	$(call echo_cmd,VDSOSYM,$<)
-	@mkdir -p $(VDSOBUILD)/../include
-	@nm $< | sh $(VDSOSRC)/gen_vdso_offsets.sh | LC_ALL=C sort > $@
-
-$(VDSOBUILD)/../include/vdso-so-path.h:
-	@echo "#define VDSO_SO_PATH \"@abs_builddir@/vdso.so\"" > $@
-
-# Link rule for the .so file, .lds has to be first
-$(VDSOBUILD)/vdso.so.dbg: $(VDSOBUILD)/vdso.lds $(DESTOBJS) $(DESTASMOBJS)
-	$(ld_cmd)
-
-$(VDSOBUILD)/vdso.lds: $(VDSOSRC)/vdso.lds.S
-	$(lds_cmd)
-
-clean:
-	$(rm_cmd) $(DESTOBJS) $(DESTASMOBJS) $(VDSOBUILD)/Makefile.dep $(VDSOBUILD)/vdso.* -r $(VDSOBUILD)/../include
-
-depend: $(VDSOBUILD)/Makefile.dep
-
-$(VDSOBUILD)/Makefile.dep:
-	$(call dep_cmd,$(DEPSRCS))
-
-prepare:
-	@$(RM) $(VDSOBUILD)/Makefile.dep
-
--include $(VDSOBUILD)/Makefile.dep
-
-# Actual build commands
-ifeq ($(V),1)
-echo_cmd =
-submake = make
-else
-echo_cmd = @echo ' ($(TARGET))' $1 $(ECHO_SUFFIX) $2;
-submake = make --no-print-directory
-endif
-
-cc_cmd = $(call echo_cmd,CC,$<)$(CC) $(CFLAGS) -c -o $@
-ld_cmd = $(call echo_cmd,LD,$@)$(CC) $(LDFLAGS) $^ -o $@
-dep_cmd = $(call echo_cmd,DEPEND,)$(CC) $(CFLAGS) -MM $1 > $@
-rm_cmd = $(call echo_cmd,CLEAN,)$(RM)
-objcopy_cmd = $(call echo_cmd,OBJCOPY,$<)$(OBJCOPY) $(OBJCOPYFLAGS) $< $@
-lds_cmd = $(call echo_cmd,LDS,$<)$(CC) $(CFLAGS_lds) -c -o $@ $<
-
-$(DESTOBJS):
-	$(cc_cmd) $(addprefix $(VDSOSRC)/, $(notdir $(@:.o=.c)))
-$(DESTASMOBJS):
-	$(cc_cmd) $(addprefix $(VDSOSRC)/, $(notdir $(@:.o=.S))) -D__ASSEMBLY__
-

arch/arm64/kernel/vdso/gen_vdso_offsets.sh

@@ -1,17 +0,0 @@
-#!/bin/sh
-
-# gen_vdso_offsets.sh COPYRIGHT FUJITSU LIMITED 2016
-# @ref.impl arch/arm64/kernel/vdso/gen_vdso_offsets.sh
-#
-# Match symbols in the DSO that look like VDSO_*; produce a header file
-# of constant offsets into the shared object.
-#
-# Doing this inside the Makefile will break the $(filter-out) function,
-# causing Kbuild to rebuild the vdso-offsets header file every time.
-#
-# Author: Will Deacon <will.deacon@arm.com
-#
-
-LC_ALL=C
-sed -n -e 's/^00*/0/' -e \
-'s/^\([0-9a-fA-F]*\) . VDSO_\([a-zA-Z0-9_]*\)$/\#define vdso_offset_\2\t0x\1/p'

arch/arm64/kernel/vdso/gettimeofday.c

@@ -1,205 +0,0 @@
-/* gettimeofday.c COPYRIGHT FUJITSU LIMITED 2016 */
-
-#include <time.h>
-#include <syscall.h>
-#include <registers.h>
-#include <ihk/atomic.h>
-
-extern int __kernel_gettimeofday(struct timeval *tv, void *tz);
-
-static inline void cpu_pause_for_vsyscall(void)
-{
-	asm volatile ("yield" ::: "memory");
-	return;
-}
-
-static inline void calculate_time_from_tsc(struct timespec *ts,
-					   struct tod_data_s *tod_data)
-{
-	long ver;
-	unsigned long current_tsc;
-	__time_t sec_delta;
-	long ns_delta;
-
-	for (;;) {
-		while ((ver = ihk_atomic64_read(&tod_data->version)) & 1) {
-			/* settimeofday() is in progress */
-			cpu_pause_for_vsyscall();
-		}
-		rmb();
-		*ts = tod_data->origin;
-		rmb();
-		if (ver == ihk_atomic64_read(&tod_data->version)) {
-			break;
-		}
-
-		/* settimeofday() has intervened */
-		cpu_pause_for_vsyscall();
-	}
-
-	current_tsc = rdtsc();
-	sec_delta = current_tsc / tod_data->clocks_per_sec;
-	ns_delta = NS_PER_SEC * (current_tsc % tod_data->clocks_per_sec)
-		/ tod_data->clocks_per_sec;
-	/* calc. of ns_delta overflows if clocks_per_sec exceeds 18.44 GHz */
-
-	ts->tv_sec += sec_delta;
-	ts->tv_nsec += ns_delta;
-	if (ts->tv_nsec >= NS_PER_SEC) {
-		ts->tv_nsec -= NS_PER_SEC;
-		++ts->tv_sec;
-	}
-
-	return;
-}
-
-static inline struct tod_data_s *get_tod_data_addr(void)
-{
-	unsigned long addr;
-
-	asm volatile("adr	%0, _tod_data\n"
-		     : "=r" (addr)
-		     :
-		     : "memory");
-
-	return (struct tod_data_s *)addr;
-}
-
-int __kernel_gettimeofday(struct timeval *tv, void *tz)
-{
-	long ret;
-	struct tod_data_s *tod_data;
-	struct timespec ats;
-
-	if(!tv && !tz) {
-		/* nothing to do */
-		return 0;
-	}
-
-	tod_data = get_tod_data_addr();
-
-	/* DO it locally if supported */
-	if (!tz && tod_data->do_local) {
-		calculate_time_from_tsc(&ats, tod_data);
-
-		tv->tv_sec = ats.tv_sec;
-		tv->tv_usec = ats.tv_nsec / 1000;
-
-		return 0;
-	}
-
-	/* Otherwize syscall */
-	asm volatile("mov	w8, %w1\n"
-		     "mov	x0, %2\n"
-		     "mov	x1, %3\n"
-		     "svc	#0\n"
-		     "mov	%0, x0\n"
-		     : "=r" (ret)
-		     : "r" (__NR_gettimeofday), "r"(tv), "r"(tz)
-		     : "memory");
-
-	if (ret) {
-		*(int *)0 = 0;	/* i.e. raise(SIGSEGV) */
-	}
-	return (int)ret;
-}
-
-
-/*
- * The IDs of the various system clocks (for POSIX.1b interval timers):
- * @ref.impl include/uapi/linux/time.h
- */
-// #define CLOCK_REALTIME		0
-// #define CLOCK_MONOTONIC		1
-// #define CLOCK_PROCESS_CPUTIME_ID	2
-// #define CLOCK_THREAD_CPUTIME_ID	3
-#define CLOCK_MONOTONIC_RAW		4
-#define CLOCK_REALTIME_COARSE		5
-#define CLOCK_MONOTONIC_COARSE		6
-#define CLOCK_BOOTTIME			7
-#define CLOCK_REALTIME_ALARM		8
-#define CLOCK_BOOTTIME_ALARM		9
-#define CLOCK_SGI_CYCLE			10	/* Hardware specific */
-#define CLOCK_TAI			11
-
-#define HIGH_RES_NSEC		1 /* nsec. */
-#define CLOCK_REALTIME_RES	HIGH_RES_NSEC
-
-#define CLOCK_COARSE_RES	((NS_PER_SEC+CONFIG_HZ/2)/CONFIG_HZ) /* 10,000,000 nsec*/
-
-typedef int	clockid_t;
-
-int __kernel_clock_gettime(clockid_t clk_id, struct timespec *tp)
-{
-	long ret;
-	struct tod_data_s *tod_data;
-	struct timespec ats;
-
-	if (!tp) {
-		/* nothing to do */
-		return 0;
-	}
-
-	tod_data = get_tod_data_addr();
-
-	/* DO it locally if supported */
-	if (tod_data->do_local && clk_id == CLOCK_REALTIME) {
-		calculate_time_from_tsc(&ats, tod_data);
-
-		tp->tv_sec = ats.tv_sec;
-		tp->tv_nsec = ats.tv_nsec;
-
-		return 0;
-	}
-
-	/* Otherwize syscall */
-	asm volatile("mov	w8, %w1\n"
-		     "mov	x0, %2\n"
-		     "mov	x1, %3\n"
-		     "svc	#0\n"
-		     "mov	%0, x0\n"
-		     : "=r" (ret)
-		     : "r" (__NR_clock_gettime), "r"(clk_id), "r"(tp)
-		     : "memory");
-
-	return (int)ret;
-}
-
-int __kernel_clock_getres(clockid_t clk_id, struct timespec *res)
-{
-	long ret;
-
-	if (!res) {
-		/* nothing to do */
-		return 0;
-	}
-
-	switch (clk_id) {
-		case CLOCK_REALTIME:
-		case CLOCK_MONOTONIC:
-			res->tv_sec = 0;
-			res->tv_nsec = CLOCK_REALTIME_RES;
-			return 0;
-			break;
-		case CLOCK_REALTIME_COARSE:
-		case CLOCK_MONOTONIC_COARSE:
-			res->tv_sec = 0;
-			res->tv_nsec = CLOCK_COARSE_RES;
-			return 0;
-			break;
-		default:
-			break;
-	}
-
-	/* Otherwise syscall */
-	asm volatile("mov	w8, %w1\n"
-		     "mov	x0, %2\n"
-		     "mov	x1, %3\n"
-		     "svc	#0\n"
-		     "mov	%0, x0\n"
-		     : "=r" (ret)
-		     : "r" (__NR_clock_getres), "r"(clk_id), "r"(res)
-		     : "memory");
-
-	return (int)ret;
-}

arch/arm64/kernel/vdso/note.S

@@ -1,28 +0,0 @@
-/* note.S COPYRIGHT FUJITSU LIMITED 2016 */
-/* @ref.impl arch/arm64/kernel/vdso/note.S */
-/*
- * Copyright (C) 2012 ARM Limited
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * Author: Will Deacon <will.deacon@arm.com>
- *
- * This supplies .note.* sections to go into the PT_NOTE inside the vDSO text.
- * Here we can supply some information useful to userland.
- */
-
-#include <elfnote.h>
-
-ELFNOTE_START(McKernel, 0, "a")
-	.long 0x10000	/* MCKERNEL_VERSION_CODE */
-ELFNOTE_END

arch/arm64/kernel/vdso/sigreturn.S

@@ -1,39 +0,0 @@
-/* sigreturn.S COPYRIGHT FUJITSU LIMITED 2016 */
-/* @ref.impl arch/arm64/kernel/vdso/sigreturn.S */
-/*
- * Sigreturn trampoline for returning from a signal when the SA_RESTORER
- * flag is not set.
- *
- * Copyright (C) 2012 ARM Limited
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * Author: Will Deacon <will.deacon@arm.com>
- */
-
-#include <linkage.h>
-#include "syscall.h"
-
-	.text
-
-	nop
-ENTRY(__kernel_rt_sigreturn)
-	.cfi_startproc
-	.cfi_signal_frame
-	.cfi_def_cfa	x29, 0
-	.cfi_offset	x29, 0 * 8
-	.cfi_offset	x30, 1 * 8
-	mov	x8, #__NR_rt_sigreturn
-	svc	#0
-	.cfi_endproc
-ENDPROC(__kernel_rt_sigreturn)

arch/arm64/kernel/vdso/syscall.h

@@ -1,15 +0,0 @@
-/* syscall.h COPYRIGHT FUJITSU LIMITED 2016 */
-#ifndef __HEADER_ARM64_VDSO_SYSCALL_H
-#define __HEADER_ARM64_VDSO_SYSCALL_H
-
-#define DECLARATOR(number,name)		.equ __NR_##name, number
-#define SYSCALL_HANDLED(number,name)	DECLARATOR(number,name)
-#define SYSCALL_DELEGATED(number,name)	DECLARATOR(number,name)
-
-#include <syscall_list.h>
-
-#undef DECLARATOR
-#undef SYSCALL_HANDLED
-#undef SYSCALL_DELEGATED
-
-#endif	/* !__HEADER_ARM64_VDSO_SYSCALL_H */

arch/arm64/kernel/vdso/vdso.lds.S

@@ -1,96 +0,0 @@
-/* vdso.lds.S COPYRIGHT FUJITSU LIMITED 2016 */
-/* @ref.impl arch/arm64/kernel/vdso/vdso.lds.S */
-/*
- * GNU linker script for the VDSO library.
- *
- * Copyright (C) 2012 ARM Limited
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * Author: Will Deacon <will.deacon@arm.com>
- * Heavily based on the vDSO linker scripts for other archs.
- */
-
-OUTPUT_FORMAT("elf64-littleaarch64", "elf64-bigaarch64", "elf64-littleaarch64")
-OUTPUT_ARCH(aarch64)
-
-SECTIONS
-{
-	PROVIDE(_tod_data = . - PAGE_SIZE);
-	. = VDSO_LBASE + SIZEOF_HEADERS;
-
-	.hash		: { *(.hash) }			:text
-	.gnu.hash	: { *(.gnu.hash) }
-	.dynsym		: { *(.dynsym) }
-	.dynstr		: { *(.dynstr) }
-	.gnu.version	: { *(.gnu.version) }
-	.gnu.version_d	: { *(.gnu.version_d) }
-	.gnu.version_r	: { *(.gnu.version_r) }
-
-	.note		: { *(.note.*) }		:text	:note
-
-	. = ALIGN(16);
-
-	.text		: { *(.text*) }			:text	=0xd503201f
-	PROVIDE (__etext = .);
-	PROVIDE (_etext = .);
-	PROVIDE (etext = .);
-
-	.eh_frame_hdr	: { *(.eh_frame_hdr) }		:text	:eh_frame_hdr
-	.eh_frame	: { KEEP (*(.eh_frame)) }	:text
-
-	.dynamic	: { *(.dynamic) }		:text	:dynamic
-
-	.rodata		: { *(.rodata*) }		:text
-
-	_end = .;
-	PROVIDE(end = .);
-
-	/DISCARD/	: {
-		*(.note.GNU-stack)
-		*(.data .data.* .gnu.linkonce.d.* .sdata*)
-		*(.bss .sbss .dynbss .dynsbss)
-	}
-}
-
-/*
- * We must supply the ELF program headers explicitly to get just one
- * PT_LOAD segment, and set the flags explicitly to make segments read-only.
- */
-PHDRS
-{
-	text		PT_LOAD		FLAGS(5) FILEHDR PHDRS; /* PF_R|PF_X */
-	dynamic		PT_DYNAMIC	FLAGS(4);		/* PF_R */
-	note		PT_NOTE		FLAGS(4);		/* PF_R */
-	eh_frame_hdr	PT_GNU_EH_FRAME;
-}
-
-/*
- * This controls what symbols we export from the DSO.
- */
-VERSION
-{
-	LINUX_2.6.39 {
-	global:
-		__kernel_rt_sigreturn;
-		__kernel_gettimeofday;
-		__kernel_clock_gettime;
-		__kernel_clock_getres;
-	local: *;
-	};
-}
-
-/*
- * Make the sigreturn code visible to the kernel.
- */
-VDSO_sigtramp	= __kernel_rt_sigreturn;

arch/x86_64/elfboot/raw.lds

@@ -9,29 +9,29 @@ PHDRS
 SECTIONS
 {
 	. = SIZEOF_HEADERS;
-    . = ALIGN(4096);
+	. = ALIGN(4096);
 	.text :	{
-	      *(.text)
+		*(.text)
 	} :text
 	.data : {
-	      *(.data)
-	      *(.data.*)
+		*(.data)
+		*(.data.*)
 	} :data
 	.rodata : {
-	      *(.rodata .rodata.*)
+		*(.rodata .rodata.*)
 	} :data
 
 	. = ALIGN(8);
 	.bss : {
-	     _bss_start = .;
-	     *(.bss .bss.*)
-	     _bss_end = .;
-         . = ALIGN(4096);
-         _stack_end = .;
-    } :data
+		_bss_start = .;
+		*(.bss .bss.*)
+		_bss_end = .;
+		. = ALIGN(4096);
+		_stack_end = .;
+	} :data
 
 	/DISCARD/ : {
-	     *(.eh_frame)
-         *(.note.gnu.build-id)
+		*(.eh_frame)
+		*(.note.gnu.build-id)
 	}
-}
+}

arch/x86_64/kboot/main.c

@@ -18,7 +18,7 @@ extern char data_start[], data_end[];
 #define LARGE_PAGE_MASK  (~((unsigned long)LARGE_PAGE_SIZE - 1))
 
 #define MAP_ST_START       0xffff800000000000UL
-#define MAP_KERNEL_START   0xffffffff80000000UL
+/* MAP_KERNEL_START is defined by cmake */
 
 #define PTL4_SHIFT         39
 #define PTL3_SHIFT         30

arch/x86_64/kernel/Makefile.arch.in

@@ -1,2 +1,2 @@
 IHK_OBJS += cpu.o interrupt.o memory.o trampoline.o local.o context.o
-IHK_OBJS += perfctr.o syscall.o vsyscall.o
+IHK_OBJS += perfctr.o syscall.o vsyscall.o coredump.o

arch/x86_64/kernel/coredump.c

@@ -1,8 +1,9 @@
-#ifdef POSTK_DEBUG_ARCH_DEP_18 /* coredump arch separation. */
+/* coredump.c COPYRIGHT FUJITSU LIMITED 2018-2019 */
 #include <process.h>
 #include <elfcore.h>
 
-void arch_fill_prstatus(struct elf_prstatus64 *prstatus, struct thread *thread, void *regs0)
+void arch_fill_prstatus(struct elf_prstatus64 *prstatus,
+	struct thread *thread, void *regs0, int sig)
 {
 	struct x86_user_context *uctx = regs0;
 	struct x86_basic_regs *regs = &uctx->gpr;
@@ -18,8 +19,6 @@ void arch_fill_prstatus(struct elf_prstatus64 *prstatus, struct thread *thread,
 	short int pr_cursig;
 	a8_uint64_t pr_sigpend;
 	a8_uint64_t pr_sighold;
-	pid_t pr_pid;
-	pid_t pr_ppid;
 	pid_t pr_pgrp;
 	pid_t pr_sid;
 	struct prstatus64_timeval pr_utime;
@@ -28,6 +27,14 @@ void arch_fill_prstatus(struct elf_prstatus64 *prstatus, struct thread *thread,
 	struct prstatus64_timeval pr_cstime;
  */
 
+	prstatus->pr_pid = thread->tid;
+	if (thread->proc->parent) {
+		prstatus->pr_ppid = thread->proc->parent->pid;
+	}
+
+	prstatus->pr_info.si_signo = sig;
+	prstatus->pr_cursig = sig;
+
 	prstatus->pr_reg[0] = _r15;
 	prstatus->pr_reg[1] = _r14;
 	prstatus->pr_reg[2] = _r13;
@@ -56,4 +63,12 @@ void arch_fill_prstatus(struct elf_prstatus64 *prstatus, struct thread *thread,
 	prstatus->pr_fpvalid = 0;	/* We assume no fp */
 }
 
-#endif /* POSTK_DEBUG_ARCH_DEP_18 */
+void arch_fill_thread_core_info(struct note *head,
+				struct thread *thread, void *regs)
+{
+}
+
+int arch_get_thread_core_info_size(void)
+{
+	return 0;
+}

arch/x86_64/kernel/cpu.c

@@ -1,3 +1,4 @@
+/* cpu.c COPYRIGHT FUJITSU LIMITED 2018-2019 */
 /**
  * \file cpu.c
  *  License details are found in the file LICENSE.
@@ -15,7 +16,6 @@
  */
 
 #include <ihk/cpu.h>
-#include <ihk/debug.h>
 #include <ihk/mm.h>
 #include <types.h>
 #include <errno.h>
@@ -31,6 +31,7 @@
 #include <prctl.h>
 #include <page.h>
 #include <kmalloc.h>
+#include <ihk/debug.h>
 
 #define LAPIC_ID            0x020
 #define LAPIC_TIMER         0x320
@@ -43,11 +44,9 @@
 #define LAPIC_ICR0          0x300
 #define LAPIC_ICR2          0x310
 #define LAPIC_ESR           0x280
-#ifdef POSTK_DEBUG_ARCH_DEP_75 /* x86 depend hide */
 #define LOCAL_TIMER_VECTOR  0xef
 #define LOCAL_PERF_VECTOR   0xf0
 #define LOCAL_SMP_FUNC_CALL_VECTOR   0xf1
-#endif /* POSTK_DEBUG_ARCH_DEP_75 */
 
 #define APIC_INT_LEVELTRIG      0x08000
 #define APIC_INT_ASSERT         0x04000
@@ -69,11 +68,8 @@
 //#define DEBUG_PRINT_CPU
 
 #ifdef DEBUG_PRINT_CPU
-#define dkprintf kprintf
-#define ekprintf kprintf
-#else
-#define dkprintf(...) do { if (0) kprintf(__VA_ARGS__); } while (0)
-#define ekprintf kprintf
+#undef DDEBUG_DEFAULT
+#define DDEBUG_DEFAULT DDEBUG_PRINT
 #endif
 
 static void *lapic_vp;
@@ -84,7 +80,11 @@ static void (*lapic_icr_write)(unsigned int h, unsigned int l);
 static void (*lapic_wait_icr_idle)(void);
 void (*x86_issue_ipi)(unsigned int apicid, unsigned int low);
 int running_on_kvm(void);
-static void smp_func_call_handler(void);
+void smp_func_call_handler(void);
+int ihk_mc_get_smp_handler_irq(void)
+{
+	return LOCAL_SMP_FUNC_CALL_VECTOR;
+}
 
 void init_processors_local(int max_id);
 void assign_processor_id(void);
@@ -93,9 +93,10 @@ void x86_set_warm_reset(unsigned long ip, char *first_page_va);
 void x86_init_perfctr(void);
 int gettime_local_support = 0;
 
-extern int ihk_mc_pt_print_pte(struct page_table *pt, void *virt);
 extern int kprintf(const char *format, ...);
 extern int interrupt_from_user(void *);
+extern void perf_start(struct mc_perf_event *event);
+extern void perf_reset(struct mc_perf_event *event);
 
 static struct idt_entry{
 	uint32_t desc[4];
@@ -148,7 +149,7 @@ void reload_idt(void)
 }
 
 static struct list_head handlers[256 - 32];
-extern char nmi[];
+extern char nmi_handler[];
 extern char page_fault[], general_protection_exception[];
 extern char debug_exception[], int3_exception[];
 
@@ -175,7 +176,7 @@ static void init_idt(void)
 		set_idt_entry(i, generic_common_handlers[i]);
 	}
 
-	set_idt_entry(2, (uintptr_t)nmi);
+	set_idt_entry(2, (uintptr_t)nmi_handler);
 	set_idt_entry(13, (unsigned long)general_protection_exception);
 	set_idt_entry(14, (unsigned long)page_fault);
 
@@ -824,11 +825,14 @@ void call_ap_func(void (*next_func)(void))
 	next_func();
 }
 
+struct page_table *get_init_page_table(void);
 void setup_x86_ap(void (*next_func)(void))
 {
 	unsigned long rsp;
 	cpu_disable_interrupt();
 
+	ihk_mc_load_page_table(get_init_page_table());
+
 	assign_processor_id();
 
 	init_smp_processor();
@@ -847,8 +851,6 @@ void setup_x86_ap(void (*next_func)(void))
 }
 
 void arch_show_interrupt_context(const void *reg);
-void set_signal(int sig, void *regs, struct siginfo *info);
-void check_signal(unsigned long, void *, int);
 extern void tlb_flush_handler(int vector);
 
 void __show_stack(uintptr_t *sp) {
@@ -870,15 +872,70 @@ void show_context_stack(uintptr_t *rbp) {
 	return;
 }
 
+#ifdef ENABLE_FUGAKU_HACKS
+void __show_context_stack(struct thread *thread,
+        unsigned long pc, uintptr_t sp, int kprintf_locked)
+{
+    uintptr_t stack_top;
+    unsigned long irqflags = 0;
+
+    stack_top = ALIGN_UP(sp, (uintptr_t)KERNEL_STACK_SIZE);
+
+    if (!kprintf_locked)
+        irqflags = kprintf_lock();
+
+    __kprintf("TID: %d, call stack (most recent first):\n",
+        thread->tid);
+    __kprintf("PC: %016lx, SP: %016lx\n", pc, sp);
+    for (;;) {
+        extern char _head[], _end[];
+        uintptr_t *fp, *lr;
+        fp = (uintptr_t *)sp;
+        lr = (uintptr_t *)(sp + 8);
+
+        if ((*fp <= sp)) {
+            break;
+        }
+
+        if ((*fp > stack_top)) {
+            break;
+        }
+
+        if ((*lr < (unsigned long)_head) ||
+            (*lr > (unsigned long)_end)) {
+            break;
+        }
+
+        __kprintf("PC: %016lx, SP: %016lx, FP: %016lx\n", *lr - 4, sp, *fp);
+        sp = *fp;
+    }
+
+    if (!kprintf_locked)
+        kprintf_unlock(irqflags);
+}
+#endif
+
+void interrupt_exit(struct x86_user_context *regs)
+{
+	if (interrupt_from_user(regs)) {
+		cpu_enable_interrupt();
+		check_sig_pending();
+		check_need_resched();
+		check_signal(0, regs, -1);
+	}
+}
+
 void handle_interrupt(int vector, struct x86_user_context *regs)
 {
 	struct ihk_mc_interrupt_handler *h;
 	struct cpu_local_var *v = get_this_cpu_local_var();
+	int from_user = interrupt_from_user(regs);
 
 	lapic_ack();
 	++v->in_interrupt;
 
-	set_cputime(interrupt_from_user(regs)? 1: 2);
+	set_cputime(from_user ?
+		CPUTIME_MODE_U2K : CPUTIME_MODE_K2K_IN);
 
 	dkprintf("CPU[%d] got interrupt, vector: %d, RIP: 0x%lX\n", 
 	         ihk_mc_get_processor_id(), vector, regs->gpr.rip);
@@ -940,6 +997,8 @@ void handle_interrupt(int vector, struct x86_user_context *regs)
 		v->flags |= CPU_FLAG_NEED_RESCHED;
 		ihk_mc_spinlock_unlock(&v->runq_lock, irqstate);
 		dkprintf("timer[%lu]: CPU_FLAG_NEED_RESCHED \n", rdtsc());
+
+		do_backlog();
 	}
 	else if (vector == LOCAL_PERF_VECTOR) {
 		struct siginfo info;
@@ -992,19 +1051,26 @@ void handle_interrupt(int vector, struct x86_user_context *regs)
 		}
 	}
 
-	if(interrupt_from_user(regs)){
-		cpu_enable_interrupt();
-		check_signal(0, regs, 0);
-		check_need_resched();
-	}
-	set_cputime(0);
+	interrupt_exit(regs);
+	set_cputime(from_user ?
+		CPUTIME_MODE_K2U : CPUTIME_MODE_K2K_OUT);
 
 	--v->in_interrupt;
+
+	/* for migration by IPI */
+	if (v->flags & CPU_FLAG_NEED_MIGRATE) {
+		// Don't migrate on K2K schedule
+		if (from_user) {
+			schedule();
+			check_signal(0, regs, 0);
+		}
+	}
 }
 
 void gpe_handler(struct x86_user_context *regs)
 {
-	set_cputime(interrupt_from_user(regs)? 1: 2);
+	set_cputime(interrupt_from_user(regs) ?
+		CPUTIME_MODE_U2K : CPUTIME_MODE_K2K_IN);
 	kprintf("General protection fault (err: %lx, %lx:%lx)\n",
 	        regs->gpr.error, regs->gpr.cs, regs->gpr.rip);
 	arch_show_interrupt_context(regs);
@@ -1012,13 +1078,10 @@ void gpe_handler(struct x86_user_context *regs)
 		panic("gpe_handler");
 	}
 	set_signal(SIGSEGV, regs, NULL);
-	if(interrupt_from_user(regs)){
-		cpu_enable_interrupt();
-		check_signal(0, regs, 0);
-		check_need_resched();
-	}
-	set_cputime(0);
-	// panic("GPF");
+	interrupt_exit(regs);
+	set_cputime(interrupt_from_user(regs) ?
+		CPUTIME_MODE_K2U : CPUTIME_MODE_K2K_OUT);
+	panic("GPF");
 }
 
 void debug_handler(struct x86_user_context *regs)
@@ -1027,7 +1090,8 @@ void debug_handler(struct x86_user_context *regs)
 	int si_code = 0;
 	struct siginfo info;
 
-	set_cputime(interrupt_from_user(regs)? 1: 2);
+	set_cputime(interrupt_from_user(regs) ?
+		CPUTIME_MODE_U2K : CPUTIME_MODE_K2K_IN);
 #ifdef DEBUG_PRINT_CPU
 	kprintf("debug exception (err: %lx, %lx:%lx)\n",
 	        regs->gpr.error, regs->gpr.cs, regs->gpr.rip);
@@ -1045,19 +1109,17 @@ void debug_handler(struct x86_user_context *regs)
 	memset(&info, '\0', sizeof info);
 	info.si_code = si_code;
 	set_signal(SIGTRAP, regs, &info);
-	if(interrupt_from_user(regs)){
-		cpu_enable_interrupt();
-		check_signal(0, regs, 0);
-		check_need_resched();
-	}
-	set_cputime(0);
+	interrupt_exit(regs);
+	set_cputime(interrupt_from_user(regs) ?
+		CPUTIME_MODE_K2U : CPUTIME_MODE_K2K_OUT);
 }
 
 void int3_handler(struct x86_user_context *regs)
 {
 	struct siginfo info;
 
-	set_cputime(interrupt_from_user(regs)? 1: 2);
+	set_cputime(interrupt_from_user(regs) ?
+		CPUTIME_MODE_U2K : CPUTIME_MODE_K2K_IN);
 #ifdef DEBUG_PRINT_CPU
 	kprintf("int3 exception (err: %lx, %lx:%lx)\n",
 	        regs->gpr.error, regs->gpr.cs, regs->gpr.rip);
@@ -1067,64 +1129,9 @@ void int3_handler(struct x86_user_context *regs)
 	memset(&info, '\0', sizeof info);
 	info.si_code = TRAP_BRKPT;
 	set_signal(SIGTRAP, regs, &info);
-	if(interrupt_from_user(regs)){
-		cpu_enable_interrupt();
-		check_signal(0, regs, 0);
-		check_need_resched();
-	}
-	set_cputime(0);
-}
-
-void
-unhandled_page_fault(struct thread *thread, void *fault_addr, void *regs)
-{
-	const uintptr_t address = (uintptr_t)fault_addr;
-	struct process_vm *vm = thread->vm;
-	struct vm_range *range;
-	unsigned long irqflags;
-	unsigned long error = ((struct x86_user_context *)regs)->gpr.error;
-
-	irqflags = kprintf_lock();
-	__kprintf("Page fault for 0x%lx\n", address);
-	__kprintf("%s for %s access in %s mode (reserved bit %s set), "
-			"it %s an instruction fetch\n",
-			(error & PF_PROT ? "protection fault" : "no page found"),
-			(error & PF_WRITE ? "write" : "read"),
-			(error & PF_USER ? "user" : "kernel"),
-			(error & PF_RSVD ? "was" : "wasn't"),
-			(error & PF_INSTR ? "was" : "wasn't"));
-
-	range = lookup_process_memory_range(vm, address, address+1);
-	if (range) {
-		__kprintf("address is in range, flag: 0x%lx\n",
-				range->flag);
-		ihk_mc_pt_print_pte(vm->address_space->page_table, (void*)address);
-	} else {
-		__kprintf("address is out of range! \n");
-	}
-
-	kprintf_unlock(irqflags);
-
-	/* TODO */
-	ihk_mc_debug_show_interrupt_context(regs);
-
-	if (!(error & PF_USER)) {
-		panic("panic: kernel mode PF");
-	}
-
-	//dkprintf("now dump a core file\n");
-	//coredump(proc, regs);
-
-#ifdef DEBUG_PRINT_MEM
-	{
-		uint64_t *sp = (void *)REGS_GET_STACK_POINTER(regs);
-
-		kprintf("*rsp:%lx,*rsp+8:%lx,*rsp+16:%lx,*rsp+24:%lx,\n",
-				sp[0], sp[1], sp[2], sp[3]);
-	}
-#endif
-
-	return;
+	interrupt_exit(regs);
+	set_cputime(interrupt_from_user(regs) ?
+		CPUTIME_MODE_K2U : CPUTIME_MODE_K2K_OUT);
 }
 
 static void outb(uint8_t v, uint16_t port)
@@ -1178,6 +1185,17 @@ void cpu_halt(void)
 	asm volatile("hlt");
 }
 
+#ifdef ENABLE_FUGAKU_HACKS
+/*@
+  @ assigns \nothing;
+  @ ensures \interrupt_disabled == 0;
+  @*/
+void cpu_halt_panic(void)
+{
+    cpu_halt();
+}
+#endif
+
 /*@
   @ assigns \nothing;
   @ ensures \interrupt_disabled == 0;
@@ -1246,6 +1264,24 @@ unsigned long cpu_disable_interrupt_save(void)
 	return flags;
 }
 
+unsigned long cpu_enable_interrupt_save(void)
+{
+	unsigned long flags;
+
+	asm volatile("pushf; pop %0; sti" : "=r"(flags) : : "memory", "cc");
+
+	return flags;
+}
+
+int cpu_interrupt_disabled(void)
+{
+	unsigned long flags;
+
+	asm volatile("pushf; pop %0" : "=r"(flags) : : "memory", "cc");
+
+	return !(flags & 0x200);
+}
+
 /*@
   @ behavior valid_vector:
   @   assumes 32 <= vector <= 255;
@@ -1289,7 +1325,7 @@ void ihk_mc_set_page_fault_handler(void (*h)(void *, uint64_t, void *))
 }
 
 extern char trampoline_code_data[], trampoline_code_data_end[];
-struct page_table *get_init_page_table(void);
+struct page_table *get_boot_page_table(void);
 unsigned long get_transit_page_table(void);
 
 /* reusable, but not reentrant */
@@ -1313,9 +1349,10 @@ void ihk_mc_boot_cpu(int cpuid, unsigned long pc)
 	memcpy(p, trampoline_code_data, 
 	       trampoline_code_data_end - trampoline_code_data);
 
-	p[1] = (unsigned long)virt_to_phys(get_init_page_table());
+	p[1] = (unsigned long)virt_to_phys(get_boot_page_table());
 	p[2] = (unsigned long)setup_x86_ap;
 	p[3] = pc;
+	p[4] = (unsigned long)get_x86_cpu_local_kstack(cpuid);
 	p[6] = (unsigned long)get_transit_page_table();
 	if (!p[6]) {
 		p[6] = p[1];
@@ -1433,13 +1470,11 @@ long ihk_mc_show_cpuinfo(char *buf, size_t buf_size, unsigned long read_off, int
 }
 #endif /* POSTK_DEBUG_ARCH_DEP_42 */
 
-#ifdef POSTK_DEBUG_ARCH_DEP_23 /* add arch dep. clone_thread() function */
 void arch_clone_thread(struct thread *othread, unsigned long pc,
 			unsigned long sp, struct thread *nthread)
 {
 	return;
 }
-#endif /* POSTK_DEBUG_ARCH_DEP_23 */
 
 void ihk_mc_print_user_context(ihk_mc_user_context_t *uctx)
 {
@@ -1543,7 +1578,8 @@ void arch_print_pre_interrupt_stack(const struct x86_basic_regs *regs) {
 	__print_stack(rbp, regs->rip);
 }
 
-void arch_print_stack() {
+void arch_print_stack(void)
+{
 	struct stack *rbp;
 
 	__kprintf("Approximative stack trace:\n");
@@ -1553,6 +1589,16 @@ void arch_print_stack() {
 	__print_stack(rbp, 0);
 }
 
+#ifdef ENABLE_FUGAKU_HACKS
+unsigned long arch_get_instruction_address(const void *reg)
+{
+	const struct x86_user_context *uctx = reg;
+	const struct x86_basic_regs *regs = &uctx->gpr;
+
+	return regs->rip;
+}
+#endif
+
 /*@
   @ requires \valid(reg);
   @ assigns \nothing;
@@ -1582,6 +1628,8 @@ void arch_show_interrupt_context(const void *reg)
 	__kprintf("%16lx %16lx %16lx %16lx\n",
 	        regs->cs, regs->ss, regs->rflags, regs->error);
 
+kprintf_unlock(irqflags);
+return;
 	arch_show_extended_context();
 
 	arch_print_pre_interrupt_stack(regs);
@@ -1589,6 +1637,13 @@ void arch_show_interrupt_context(const void *reg)
 	kprintf_unlock(irqflags);
 }
 
+void arch_cpu_stop(void)
+{
+	while (1) {
+		cpu_halt();
+	}
+}
+
 /*@
   @ behavior fs_base:
   @   assumes type == IHK_ASR_X86_FS;
@@ -1632,24 +1687,31 @@ int ihk_mc_arch_get_special_register(enum ihk_asr_type type,
 	}
 }
 
+int ihk_mc_get_interrupt_id(int cpu)
+{
+	return get_x86_cpu_local_variable(cpu)->apic_id;
+}
+
 /*@
-  @ requires \valid_apicid(cpu);	// valid APIC ID or not
+  @ requires \valid_cpuid(cpu);     // valid CPU logical ID
   @ ensures \result == 0
   @*/
 int ihk_mc_interrupt_cpu(int cpu, int vector)
 {
+	if (cpu < 0 || cpu >= num_processors) {
+		kprintf("%s: invalid CPU id: %d\n", __func__, cpu);
+		return -1;
+	}
 	dkprintf("[%d] ihk_mc_interrupt_cpu: %d\n", ihk_mc_get_processor_id(), cpu);
 
-	x86_issue_ipi(cpu, vector);
+	x86_issue_ipi(get_x86_cpu_local_variable(cpu)->apic_id, vector);
 	return 0;
 }
 
-#ifdef POSTK_DEBUG_ARCH_DEP_22
-extern void perf_start(struct mc_perf_event *event);
-extern void perf_reset(struct mc_perf_event *event);
 struct thread *arch_switch_context(struct thread *prev, struct thread *next)
 {
 	struct thread *last;
+	struct mcs_rwlock_node_irqsave lock;
 
 	dkprintf("[%d] schedule: tlsblock_base: 0x%lX\n",
 	         ihk_mc_get_processor_id(), next->tlsblock_base);
@@ -1657,6 +1719,7 @@ struct thread *arch_switch_context(struct thread *prev, struct thread *next)
 	/* Set up new TLS.. */
 	ihk_mc_init_user_tlsbase(next->uctx, next->tlsblock_base);
 
+#ifdef ENABLE_PERF
 	/* Performance monitoring inherit */
 	if(next->proc->monitoring_event) {
 		if(next->proc->perf_status == PP_RESET)
@@ -1666,9 +1729,10 @@ struct thread *arch_switch_context(struct thread *prev, struct thread *next)
 			perf_start(next->proc->monitoring_event);
 		}
 	}
+#endif
 
 #ifdef PROFILE_ENABLE
-	if (prev->profile && prev->profile_start_ts != 0) {
+	if (prev && prev->profile && prev->profile_start_ts != 0) {
 		prev->profile_elapsed_ts +=
 			(rdtsc() - prev->profile_start_ts);
 		prev->profile_start_ts = 0;
@@ -1680,6 +1744,28 @@ struct thread *arch_switch_context(struct thread *prev, struct thread *next)
 #endif
 
 	if (prev) {
+		mcs_rwlock_writer_lock(&prev->proc->update_lock, &lock);
+		if (prev->proc->status & (PS_DELAY_STOPPED | PS_DELAY_TRACED)) {
+			switch (prev->proc->status) {
+			case PS_DELAY_STOPPED:
+				prev->proc->status = PS_STOPPED;
+				break;
+			case PS_DELAY_TRACED:
+				prev->proc->status = PS_TRACED;
+				break;
+			default:
+				break;
+			}
+			mcs_rwlock_writer_unlock(&prev->proc->update_lock,
+						&lock);
+
+			/* Wake up the parent who tried wait4 and sleeping */
+			waitq_wakeup(&prev->proc->parent->waitpid_q);
+		} else {
+			mcs_rwlock_writer_unlock(&prev->proc->update_lock,
+						&lock);
+		}
+
 		last = ihk_mc_switch_context(&prev->ctx, &next->ctx, prev);
 	}
 	else {
@@ -1687,7 +1773,6 @@ struct thread *arch_switch_context(struct thread *prev, struct thread *next)
 	}
 	return last;
 }
-#endif
 
 /*@
   @ requires \valid(thread);
@@ -1720,7 +1805,7 @@ check_and_allocate_fp_regs(struct thread *thread)
 
 		if (!thread->fp_regs) {
 			kprintf("error: allocating fp_regs pages\n");
-			result = 1;
+			result = -ENOMEM;
 			goto out;
 		}
 
@@ -1733,12 +1818,14 @@ out:
 /*@
   @ requires \valid(thread);
   @*/
-void
+int
 save_fp_regs(struct thread *thread)
 {
-	if (check_and_allocate_fp_regs(thread) != 0) {
-		// alloc error
-		return;
+	int ret = 0;
+
+	ret = check_and_allocate_fp_regs(thread);
+	if (ret) {
+		goto out;
 	}
 
 	if (xsave_available) {
@@ -1753,23 +1840,25 @@ save_fp_regs(struct thread *thread)
 
 		dkprintf("fp_regs for TID %d saved\n", thread->tid);
 	}
+out:
+	return ret;
 }
 
-void copy_fp_regs(struct thread *from, struct thread *to)
+int copy_fp_regs(struct thread *from, struct thread *to)
 {
-	if ((from->fp_regs != NULL) && (check_and_allocate_fp_regs(to) == 0)) {
-		memcpy(to->fp_regs, from->fp_regs, sizeof(fp_regs_struct));
+	int ret = 0;
+
+	if (from->fp_regs != NULL) {
+		ret = check_and_allocate_fp_regs(to);
+		if (!ret) {
+			memcpy(to->fp_regs,
+				from->fp_regs,
+				sizeof(fp_regs_struct));
+		}
 	}
+	return ret;
 }
 
-#ifdef POSTK_DEBUG_TEMP_FIX_19
-void
-clear_fp_regs(struct thread *thread)
-{
-	return;
-}
-#endif /* POSTK_DEBUG_TEMP_FIX_19 */
-
 /*@
   @ requires \valid(thread);
   @ assigns thread->fp_regs;
@@ -1777,8 +1866,11 @@ clear_fp_regs(struct thread *thread)
 void
 restore_fp_regs(struct thread *thread)
 {
-	if (!thread->fp_regs)
+	if (!thread->fp_regs) {
+		// only clear fpregs.
+		clear_fp_regs();
 		return;
+	}
 
 	if (xsave_available) {
 		unsigned int low, high;
@@ -1797,6 +1889,13 @@ restore_fp_regs(struct thread *thread)
 	//release_fp_regs(thread);
 }
 
+void clear_fp_regs(void)
+{
+	struct cpu_local_var *v = get_this_cpu_local_var();
+
+	restore_fp_regs(&v->idle);
+}
+
 ihk_mc_user_context_t *lookup_user_context(struct thread *thread)
 {
 	ihk_mc_user_context_t *uctx = thread->uctx;
@@ -1830,6 +1929,10 @@ ihk_mc_init_user_tlsbase(ihk_mc_user_context_t *ctx,
 	do_arch_prctl(ARCH_SET_FS, tls_base_addr);
 }
 
+void arch_flush_icache_all(void)
+{
+	return;
+}
 
 /*@
   @ assigns \nothing;
@@ -1983,6 +2086,92 @@ mod_nmi_ctx(void *nmi_ctx, void (*func)())
 	l[i++] = 0x28;				// KERNEL DS
 }
 
+void arch_save_panic_regs(void *irq_regs)
+{
+	struct thread *current = cpu_local_var(current);
+	struct x86_user_context *regs =
+		(struct x86_user_context *)irq_regs;
+	struct x86_cpu_local_variables *x86v =
+		get_x86_cpu_local_variable(ihk_mc_get_processor_id());
+	struct segment_regs {
+		uint32_t rflags;
+		uint32_t cs;
+		uint32_t ss;
+		uint32_t ds;
+		uint32_t es;
+		uint32_t fs;
+		uint32_t gs;
+	} *sregs;
+
+	/* Kernel space? */
+	if (regs->gpr.rip > USER_END) {
+		x86v->panic_regs[0] = regs->gpr.rax;
+		x86v->panic_regs[1] = regs->gpr.rbx;
+		x86v->panic_regs[2] = regs->gpr.rcx;
+		x86v->panic_regs[3] = regs->gpr.rdx;
+		x86v->panic_regs[4] = regs->gpr.rsi;
+		x86v->panic_regs[5] = regs->gpr.rdi;
+		x86v->panic_regs[6] = regs->gpr.rbp;
+		x86v->panic_regs[7] = regs->gpr.rsp;
+		x86v->panic_regs[8] = regs->gpr.r8;
+		x86v->panic_regs[9] = regs->gpr.r9;
+		x86v->panic_regs[10] = regs->gpr.r10;
+		x86v->panic_regs[11] = regs->gpr.r11;
+		x86v->panic_regs[12] = regs->gpr.r12;
+		x86v->panic_regs[13] = regs->gpr.r13;
+		x86v->panic_regs[14] = regs->gpr.r14;
+		x86v->panic_regs[15] = regs->gpr.r15;
+		x86v->panic_regs[16] = regs->gpr.rip;
+		sregs = (struct segment_regs *)&x86v->panic_regs[17];
+		sregs->rflags = regs->gpr.rflags;
+		sregs->cs = regs->gpr.cs;
+		sregs->ss = regs->gpr.ss;
+		sregs->ds = regs->sr.ds;
+		sregs->es = regs->sr.es;
+		sregs->fs = regs->sr.fs;
+		sregs->gs = regs->sr.gs;
+	}
+	/* User-space, show kernel context */
+	else {
+		kprintf("%s: in user-space: %p\n", __func__, regs->gpr.rip);
+		x86v->panic_regs[0] = 0;
+		x86v->panic_regs[1] = current->ctx.rbx;
+		x86v->panic_regs[2] = 0;
+		x86v->panic_regs[3] = 0;
+		x86v->panic_regs[4] = current->ctx.rsi;
+		x86v->panic_regs[5] = current->ctx.rdi;
+		x86v->panic_regs[6] = current->ctx.rbp;
+		x86v->panic_regs[7] = current->ctx.rsp;
+		x86v->panic_regs[8] = 0;
+		x86v->panic_regs[9] = 0;
+		x86v->panic_regs[10] = 0;
+		x86v->panic_regs[11] = 0;
+		x86v->panic_regs[12] = regs->gpr.r12;
+		x86v->panic_regs[13] = regs->gpr.r13;
+		x86v->panic_regs[14] = regs->gpr.r14;
+		x86v->panic_regs[15] = regs->gpr.r15;
+		x86v->panic_regs[16] = (unsigned long)enter_user_mode;
+		sregs = (struct segment_regs *)&x86v->panic_regs[17];
+		sregs->rflags = regs->gpr.rflags;
+		sregs->cs = regs->gpr.cs;
+		sregs->ss = regs->gpr.ss;
+		sregs->ds = regs->sr.ds;
+		sregs->es = regs->sr.es;
+		sregs->fs = regs->sr.fs;
+		sregs->gs = regs->sr.gs;
+	}
+
+	x86v->paniced = 1;
+}
+
+void arch_clear_panic(void)
+{
+	struct x86_cpu_local_variables *x86v =
+		get_x86_cpu_local_variable(ihk_mc_get_processor_id());
+
+	x86v->paniced = 0;
+}
+
 int arch_cpu_read_write_register(
 		struct ihk_os_cpu_register *desc,
 		enum mcctrl_os_cpu_operation op)
@@ -2000,144 +2189,48 @@ int arch_cpu_read_write_register(
 	return 0;
 }
 
-/*
- * Generic remote CPU function invocation facility.
- */
-static void smp_func_call_handler(void)
+extern int nmi_mode;
+extern long freeze_thaw(void *nmi_ctx);
+
+void multi_nm_interrupt_handler(void *irq_regs)
 {
-	int irq_flags;
-	struct smp_func_call_request *req;
-	int reqs_left;
+	dkprintf("%s: ...\n", __func__);
+	switch (nmi_mode) {
+	case 1:
+	case 2:
+		/* mode == 1 or 2, for FREEZER NMI */
+		dkprintf("%s: freeze mode NMI catch. (nmi_mode=%d)\n",
+			 __func__, nmi_mode);
+		freeze_thaw(NULL);
+		break;
 
-reiterate:
-	req = NULL;
-	reqs_left = 0;
+	case 0:
+		/* mode == 0, for MEMDUMP NMI */
+		arch_save_panic_regs(irq_regs);
+		ihk_mc_query_mem_areas();
+		/* memdump-nmi is halted McKernel, break is unnecessary. */
+		/* fall through */
+	case 3:
+		/* mode == 3, for SHUTDOWN-WAIT NMI */
+		kprintf("%s: STOP\n", __func__);
+		while (nmi_mode != 4)
+			cpu_halt();
+		break;
 
-	irq_flags = ihk_mc_spinlock_lock(
-			&cpu_local_var(smp_func_req_lock));
-
-	/* Take requests one-by-one */
-	if (!list_empty(&cpu_local_var(smp_func_req_list))) {
-		req = list_first_entry(&cpu_local_var(smp_func_req_list),
-			struct smp_func_call_request, list);
-		list_del(&req->list);
-
-		reqs_left = !list_empty(&cpu_local_var(smp_func_req_list));
-	}
-
-	ihk_mc_spinlock_unlock(&cpu_local_var(smp_func_req_lock),
-			irq_flags);
-
-	if (req) {
-		req->ret = req->sfcd->func(req->cpu_index,
-				req->sfcd->nr_cpus, req->sfcd->arg);
-		ihk_atomic_dec(&req->sfcd->cpus_left);
-	}
-
-	if (reqs_left)
-		goto reiterate;
-}
-
-int smp_call_func(cpu_set_t *__cpu_set, smp_func_t __func, void *__arg)
-{
-	int cpu, nr_cpus = 0;
-	int cpu_index = 0;
-	int this_cpu_index = 0;
-	struct smp_func_call_data sfcd;
-	struct smp_func_call_request *reqs;
-	int ret = 0;
-	int call_on_this_cpu = 0;
-	cpu_set_t cpu_set;
-
-	/* Sanity checks */
-	if (!__cpu_set || !__func) {
-		return -EINVAL;
-	}
-
-	/* Make sure it won't change in between */
-	cpu_set = *__cpu_set;
-
-	for_each_set_bit(cpu, (unsigned long *)&cpu_set,
-			sizeof(cpu_set) * BITS_PER_BYTE) {
-
-		if (cpu == ihk_mc_get_processor_id()) {
-			call_on_this_cpu = 1;
+	case 4:
+		/* mode == 4, continue NMI */
+		arch_clear_panic();
+		if (!ihk_mc_get_processor_id()) {
+			ihk_mc_clear_dump_page_completion();
 		}
-		++nr_cpus;
+		kprintf("%s: RESUME, nmi_mode: %d\n", __func__, nmi_mode);
+		break;
+
+	default:
+		ekprintf("%s: Unknown nmi-mode(%d) detected.\n",
+			 __func__, nmi_mode);
+		break;
 	}
-
-	if (!nr_cpus) {
-		return -EINVAL;
-	}
-
-	reqs = kmalloc(sizeof(*reqs) * nr_cpus, IHK_MC_AP_NOWAIT);
-	if (!reqs) {
-		ret = -ENOMEM;
-		goto free_out;
-	}
-
-	sfcd.nr_cpus = nr_cpus;
-	sfcd.func = __func;
-	sfcd.arg = __arg;
-	ihk_atomic_set(&sfcd.cpus_left,
-			call_on_this_cpu ? nr_cpus - 1 : nr_cpus);
-
-	/* Add requests and send IPIs */
-	cpu_index = 0;
-	for_each_set_bit(cpu, (unsigned long *)&cpu_set,
-			sizeof(cpu_set) * BITS_PER_BYTE) {
-		unsigned long irq_flags;
-
-		reqs[cpu_index].cpu_index = cpu_index;
-		reqs[cpu_index].ret = 0;
-
-		if (cpu == ihk_mc_get_processor_id()) {
-			this_cpu_index = cpu_index;
-			++cpu_index;
-			continue;
-		}
-
-		reqs[cpu_index].sfcd = &sfcd;
-
-		irq_flags =
-			ihk_mc_spinlock_lock(&get_cpu_local_var(cpu)->smp_func_req_lock);
-		list_add_tail(&reqs[cpu_index].list,
-				&get_cpu_local_var(cpu)->smp_func_req_list);
-		ihk_mc_spinlock_unlock(&get_cpu_local_var(cpu)->smp_func_req_lock,
-			irq_flags);
-
-		ihk_mc_interrupt_cpu(
-				get_x86_cpu_local_variable(cpu)->apic_id,
-				LOCAL_SMP_FUNC_CALL_VECTOR);
-
-		++cpu_index;
-	}
-
-	/* Is this CPU involved? */
-	if (call_on_this_cpu) {
-		reqs[this_cpu_index].ret =
-			__func(this_cpu_index, nr_cpus, __arg);
-	}
-
-	/* Wait for the rest of the CPUs */
-	while (ihk_atomic_read(&sfcd.cpus_left) > 0) {
-		cpu_pause();
-	}
-
-	/* Check return values, if error, report the first non-zero */
-	for (cpu_index = 0; cpu_index < nr_cpus; ++cpu_index) {
-		if (reqs[cpu_index].ret != 0) {
-			ret = reqs[cpu_index].ret;
-			goto free_out;
-		}
-	}
-
-	ret = 0;
-
-free_out:
-	kfree(reqs);
-
-	return ret;
 }
 
 /*** end of file ***/

arch/x86_64/kernel/include/arch-futex.h

@@ -64,13 +64,13 @@ static inline int futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval,
 	return oldval;
 }
 
-#ifdef POSTK_DEBUG_ARCH_DEP_8 /* arch depend hide */
-static inline int futex_atomic_op_inuser(int encoded_op, int __user *uaddr)
+static inline int futex_atomic_op_inuser(int encoded_op,
+					 int __user *uaddr)
 {
 	int op = (encoded_op >> 28) & 7;
 	int cmp = (encoded_op >> 24) & 15;
-	int oparg = (encoded_op << 8) >> 20;
-	int cmparg = (encoded_op << 20) >> 20;
+	int oparg = (encoded_op & 0x00fff000) >> 12;
+	int cmparg = encoded_op & 0xfff;
 	int oldval = 0, ret, tem;
 
 	if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28))
@@ -128,14 +128,5 @@ static inline int futex_atomic_op_inuser(int encoded_op, int __user *uaddr)
 	}
 	return ret;
 }
-#endif	/* !POSTK_DEBUG_ARCH_DEP_8 */
-
-static inline int get_futex_value_locked(uint32_t *dest, uint32_t *from)
-{
-
-	*dest = *(volatile uint32_t *)from;
-
-	return 0;
-}
 
 #endif

arch/x86_64/kernel/include/arch-lock.h

@@ -6,6 +6,8 @@
 
 #include <ihk/cpu.h>
 #include <ihk/atomic.h>
+#include <lwk/compiler.h>
+#include "config.h"
 
 //#define DEBUG_SPINLOCK
 //#define DEBUG_MCS_RWLOCK
@@ -14,18 +16,84 @@
 int __kprintf(const char *format, ...);
 #endif
 
-typedef int ihk_spinlock_t;
+typedef unsigned short __ticket_t;
+typedef unsigned int __ticketpair_t;
+
+typedef struct ihk_spinlock {
+	union {
+		__ticketpair_t head_tail;
+		struct __raw_tickets {
+			__ticket_t head, tail;
+		} tickets;
+	};
+} ihk_spinlock_t;
 
 extern void preempt_enable(void);
 extern void preempt_disable(void);
 
 #define IHK_STATIC_SPINLOCK_FUNCS
 
-static void ihk_mc_spinlock_init(ihk_spinlock_t *lock)
+static inline void ihk_mc_spinlock_init(ihk_spinlock_t *lock)
 {
-	*lock = 0;
+	lock->head_tail = 0;
+}
+#define SPIN_LOCK_UNLOCKED { .head_tail = 0 }
+
+
+#ifdef DEBUG_SPINLOCK
+#define ihk_mc_spinlock_trylock_noirq(l) { int rc;						\
+__kprintf("[%d] call ihk_mc_spinlock_trylock_noirq %p %s:%d\n", ihk_mc_get_processor_id(), (l), __FILE__, __LINE__); \
+rc = __ihk_mc_spinlock_trylock_noirq(l); \
+ __kprintf("[%d] ret ihk_mc_spinlock_trylock_noirq\n", ihk_mc_get_processor_id()); rc; \
+}
+#else
+#define ihk_mc_spinlock_trylock_noirq __ihk_mc_spinlock_trylock_noirq
+#endif
+
+static inline int __ihk_mc_spinlock_trylock_noirq(ihk_spinlock_t *lock)
+{
+	ihk_spinlock_t cur = { .head_tail = lock->head_tail };
+	ihk_spinlock_t next = { .tickets = {
+		.head = cur.tickets.head,
+		.tail = cur.tickets.tail + 2
+	} };
+	int success;
+
+	if (cur.tickets.head != cur.tickets.tail) {
+		return 0;
+	}
+
+	preempt_disable();
+
+	/* Use the same increment amount as other functions! */
+	success = __sync_bool_compare_and_swap((__ticketpair_t*)lock, cur.head_tail, next.head_tail);
+
+	if (!success) {
+		preempt_enable();
+	}
+	return success;
+}
+
+#ifdef DEBUG_SPINLOCK
+#define ihk_mc_spinlock_trylock(l, result) ({ unsigned long rc;		\
+__kprintf("[%d] call ihk_mc_spinlock_trylock %p %s:%d\n", ihk_mc_get_processor_id(), (l), __FILE__, __LINE__); \
+ rc = __ihk_mc_spinlock_trylock(l, result);									\
+__kprintf("[%d] ret ihk_mc_spinlock_trylock\n", ihk_mc_get_processor_id()); rc;\
+})
+#else
+#define ihk_mc_spinlock_trylock __ihk_mc_spinlock_trylock
+#endif
+static inline unsigned long __ihk_mc_spinlock_trylock(ihk_spinlock_t *lock,
+						      int *result)
+{
+	unsigned long flags;
+	
+	flags = cpu_disable_interrupt_save();
+
+	*result = __ihk_mc_spinlock_trylock_noirq(lock);
+
+	return flags;
 }
-#define SPIN_LOCK_UNLOCKED 0
 
 #ifdef DEBUG_SPINLOCK
 #define ihk_mc_spinlock_lock_noirq(l) { \
@@ -37,42 +105,26 @@ __kprintf("[%d] ret ihk_mc_spinlock_lock_noirq\n", ihk_mc_get_processor_id()); \
 #define ihk_mc_spinlock_lock_noirq __ihk_mc_spinlock_lock_noirq
 #endif
 
-static void __ihk_mc_spinlock_lock_noirq(ihk_spinlock_t *lock)
+static inline void __ihk_mc_spinlock_lock_noirq(ihk_spinlock_t *lock)
 {
-	int inc = 0x00010000;
-	int tmp;
-
-#if 0
-	asm volatile("lock ; xaddl %0, %1\n"
-	             "movzwl %w0, %2\n\t"
-	             "shrl $16, %0\n\t"
-	             "1:\t"
-	             "cmpl %0, %2\n\t"
-	             "je 2f\n\t"
-	             "rep ; nop\n\t"
-	             "movzwl %1, %2\n\t"
-	             "jmp 1b\n"
-	             "2:"
-	             : "+Q" (inc), "+m" (*lock), "=r" (tmp) : : "memory", "cc");
-#endif
+	register struct __raw_tickets inc = { .tail = 0x0002 };
 
 	preempt_disable();
 
-	asm volatile("lock; xaddl %0, %1\n"
-			"movzwl %w0, %2\n\t"
-			"shrl $16, %0\n\t"
-			"1:\t"
-			"cmpl %0, %2\n\t"
-			"je 2f\n\t"
-			"rep ; nop\n\t"
-			"movzwl %1, %2\n\t"
-			/* don't need lfence here, because loads are in-order */
-			"jmp 1b\n"
-			"2:"
-			: "+r" (inc), "+m" (*lock), "=&r" (tmp)
-			:
-			: "memory", "cc");
+	asm volatile ("lock xaddl %0, %1\n"
+			: "+r" (inc), "+m" (*(lock)) : : "memory", "cc");
 
+	if (inc.head == inc.tail)
+		goto out;
+
+	for (;;) {
+		if (*((volatile __ticket_t *)&lock->tickets.head) == inc.tail)
+			goto out;
+		cpu_pause();
+	}
+
+out:
+	barrier();	/* make sure nothing creeps before the lock is taken */
 }
 
 #ifdef DEBUG_SPINLOCK
@@ -84,7 +136,7 @@ __kprintf("[%d] ret ihk_mc_spinlock_lock\n", ihk_mc_get_processor_id()); rc;\
 #else
 #define ihk_mc_spinlock_lock __ihk_mc_spinlock_lock
 #endif
-static unsigned long __ihk_mc_spinlock_lock(ihk_spinlock_t *lock)
+static inline unsigned long __ihk_mc_spinlock_lock(ihk_spinlock_t *lock)
 {
 	unsigned long flags;
 	
@@ -104,10 +156,13 @@ __kprintf("[%d] ret ihk_mc_spinlock_unlock_noirq\n", ihk_mc_get_processor_id());
 #else
 #define ihk_mc_spinlock_unlock_noirq __ihk_mc_spinlock_unlock_noirq
 #endif
-static void __ihk_mc_spinlock_unlock_noirq(ihk_spinlock_t *lock)
+static inline void __ihk_mc_spinlock_unlock_noirq(ihk_spinlock_t *lock)
 {
-	asm volatile ("lock incw %0" : "+m"(*lock) : : "memory", "cc");
-	
+	__ticket_t inc = 0x0002;
+
+	asm volatile ("lock addw %1, %0\n"
+			: "+m" (lock->tickets.head) : "ri" (inc) : "memory", "cc");
+
 	preempt_enable();
 }
 
@@ -120,94 +175,14 @@ __kprintf("[%d] ret ihk_mc_spinlock_unlock\n", ihk_mc_get_processor_id()); \
 #else
 #define ihk_mc_spinlock_unlock __ihk_mc_spinlock_unlock
 #endif
-static void __ihk_mc_spinlock_unlock(ihk_spinlock_t *lock, unsigned long flags)
+static inline void __ihk_mc_spinlock_unlock(ihk_spinlock_t *lock,
+					    unsigned long flags)
 {
 	__ihk_mc_spinlock_unlock_noirq(lock);
 
 	cpu_restore_interrupt(flags);
 }
 
-/* An implementation of the Mellor-Crummey Scott (MCS) lock */
-typedef struct mcs_lock_node {
-	unsigned long locked;
-	struct mcs_lock_node *next;
-	unsigned long irqsave;
-} __attribute__((aligned(64))) mcs_lock_node_t;
-
-static void mcs_lock_init(struct mcs_lock_node *node)
-{
-	node->locked = 0;
-	node->next = NULL;
-}
-
-static void __mcs_lock_lock(struct mcs_lock_node *lock,
-		struct mcs_lock_node *node)
-{
-	struct mcs_lock_node *pred;
-
-	node->next = NULL;
-	node->locked = 0;
-	pred = (struct mcs_lock_node *)xchg8((unsigned long *)&lock->next,
-			(unsigned long)node);
-
-	if (pred) {
-		node->locked = 1;
-		pred->next = node;
-		while (node->locked != 0) {
-			cpu_pause();
-		}
-	}
-}
-
-static void __mcs_lock_unlock(struct mcs_lock_node *lock,
-		struct mcs_lock_node *node)
-{
-	if (node->next == NULL) {
-		struct mcs_lock_node *old = (struct mcs_lock_node *)
-			atomic_cmpxchg8((unsigned long *)&lock->next,
-					(unsigned long)node, (unsigned long)0);
-
-		if (old == node) {
-			return;
-		}
-
-		while (node->next == NULL) {
-			cpu_pause();
-		}
-	}
-
-	node->next->locked = 0;
-}
-
-static void mcs_lock_lock_noirq(struct mcs_lock_node *lock,
-		struct mcs_lock_node *node)
-{
-	preempt_disable();
-	__mcs_lock_lock(lock, node);
-}
-
-static void mcs_lock_unlock_noirq(struct mcs_lock_node *lock,
-		struct mcs_lock_node *node)
-{
-	__mcs_lock_unlock(lock, node);
-	preempt_enable();
-}
-
-static void mcs_lock_lock(struct mcs_lock_node *lock,
-		struct mcs_lock_node *node)
-{
-	node->irqsave = cpu_disable_interrupt_save();
-	mcs_lock_lock_noirq(lock, node);
-}
-
-static void mcs_lock_unlock(struct mcs_lock_node *lock,
-		struct mcs_lock_node *node)
-{
-	mcs_lock_unlock_noirq(lock, node);
-	cpu_restore_interrupt(node->irqsave);
-}
-
-
 #define SPINLOCK_IN_MCS_RWLOCK
 
 // reader/writer lock
@@ -223,14 +198,22 @@ typedef struct mcs_rwlock_node {
 	char dmy1;		// unused
 	char dmy2;		// unused
 	struct mcs_rwlock_node *next;
-} __attribute__((aligned(64))) mcs_rwlock_node_t;
+#ifndef ENABLE_UBSAN
+} __aligned(64) mcs_rwlock_node_t;
+#else
+} mcs_rwlock_node_t;
+#endif
 
 typedef struct mcs_rwlock_node_irqsave {
 #ifndef SPINLOCK_IN_MCS_RWLOCK
 	struct mcs_rwlock_node node;
 #endif
 	unsigned long irqsave;
-} __attribute__((aligned(64))) mcs_rwlock_node_irqsave_t;
+#ifndef ENABLE_UBSAN
+} __aligned(64) mcs_rwlock_node_irqsave_t;
+#else
+} mcs_rwlock_node_irqsave_t;
+#endif
 
 typedef struct mcs_rwlock_lock {
 #ifdef SPINLOCK_IN_MCS_RWLOCK
@@ -239,9 +222,13 @@ typedef struct mcs_rwlock_lock {
 	struct mcs_rwlock_node reader;		/* common reader lock */
 	struct mcs_rwlock_node *node;		/* base */
 #endif
-} __attribute__((aligned(64))) mcs_rwlock_lock_t;
+#ifndef ENABLE_UBSAN
+} __aligned(64) mcs_rwlock_lock_t;
+#else
+} mcs_rwlock_lock_t;
+#endif
 
-static void
+static inline void
 mcs_rwlock_init(struct mcs_rwlock_lock *lock)
 {
 #ifdef SPINLOCK_IN_MCS_RWLOCK
@@ -262,7 +249,7 @@ __kprintf("[%d] ret mcs_rwlock_writer_lock_noirq\n", ihk_mc_get_processor_id());
 #else
 #define mcs_rwlock_writer_lock_noirq __mcs_rwlock_writer_lock_noirq
 #endif
-static void
+static inline void
 __mcs_rwlock_writer_lock_noirq(struct mcs_rwlock_lock *lock, struct mcs_rwlock_node *node)
 {
 #ifdef SPINLOCK_IN_MCS_RWLOCK
@@ -289,7 +276,7 @@ __mcs_rwlock_writer_lock_noirq(struct mcs_rwlock_lock *lock, struct mcs_rwlock_n
 }
 
 #ifndef SPINLOCK_IN_MCS_RWLOCK
-static void
+static inline void
 mcs_rwlock_unlock_readers(struct mcs_rwlock_lock *lock)
 {
 	struct mcs_rwlock_node *p;
@@ -356,7 +343,7 @@ __kprintf("[%d] ret mcs_rwlock_writer_unlock_noirq\n", ihk_mc_get_processor_id()
 #else
 #define mcs_rwlock_writer_unlock_noirq __mcs_rwlock_writer_unlock_noirq
 #endif
-static void
+static inline void
 __mcs_rwlock_writer_unlock_noirq(struct mcs_rwlock_lock *lock, struct mcs_rwlock_node *node)
 {
 #ifdef SPINLOCK_IN_MCS_RWLOCK
@@ -416,7 +403,7 @@ atomic_inc_ifnot0(ihk_atomic_t *v)
 	return old;
 }
 
-static void
+static inline void
 __mcs_rwlock_reader_lock_noirq(struct mcs_rwlock_lock *lock, struct mcs_rwlock_node *node)
 {
 #ifdef SPINLOCK_IN_MCS_RWLOCK
@@ -482,7 +469,7 @@ __kprintf("[%d] ret mcs_rwlock_reader_unlock_noirq\n", ihk_mc_get_processor_id()
 #else
 #define mcs_rwlock_reader_unlock_noirq __mcs_rwlock_reader_unlock_noirq
 #endif
-static void
+static inline void
 __mcs_rwlock_reader_unlock_noirq(struct mcs_rwlock_lock *lock, struct mcs_rwlock_node *node)
 {
 #ifdef SPINLOCK_IN_MCS_RWLOCK
@@ -529,7 +516,7 @@ __kprintf("[%d] ret mcs_rwlock_writer_lock\n", ihk_mc_get_processor_id()); \
 #else
 #define mcs_rwlock_writer_lock __mcs_rwlock_writer_lock
 #endif
-static void
+static inline void
 __mcs_rwlock_writer_lock(struct mcs_rwlock_lock *lock, struct mcs_rwlock_node_irqsave *node)
 {
 #ifdef SPINLOCK_IN_MCS_RWLOCK
@@ -549,7 +536,7 @@ __kprintf("[%d] ret mcs_rwlock_writer_unlock\n", ihk_mc_get_processor_id()); \
 #else
 #define mcs_rwlock_writer_unlock __mcs_rwlock_writer_unlock
 #endif
-static void
+static inline void
 __mcs_rwlock_writer_unlock(struct mcs_rwlock_lock *lock, struct mcs_rwlock_node_irqsave *node)
 {
 #ifdef SPINLOCK_IN_MCS_RWLOCK
@@ -569,7 +556,7 @@ __kprintf("[%d] ret mcs_rwlock_reader_lock\n", ihk_mc_get_processor_id()); \
 #else
 #define mcs_rwlock_reader_lock __mcs_rwlock_reader_lock
 #endif
-static void
+static inline void
 __mcs_rwlock_reader_lock(struct mcs_rwlock_lock *lock, struct mcs_rwlock_node_irqsave *node)
 {
 #ifdef SPINLOCK_IN_MCS_RWLOCK
@@ -589,7 +576,7 @@ __kprintf("[%d] ret mcs_rwlock_reader_unlock\n", ihk_mc_get_processor_id()); \
 #else
 #define mcs_rwlock_reader_unlock __mcs_rwlock_reader_unlock
 #endif
-static void
+static inline void
 __mcs_rwlock_reader_unlock(struct mcs_rwlock_lock *lock, struct mcs_rwlock_node_irqsave *node)
 {
 #ifdef SPINLOCK_IN_MCS_RWLOCK
@@ -600,4 +587,95 @@ __mcs_rwlock_reader_unlock(struct mcs_rwlock_lock *lock, struct mcs_rwlock_node_
 #endif
 }
 
+static inline int irqflags_can_interrupt(unsigned long flags)
+{
+	return !!(flags & 0x200);
+}
+
+struct ihk_rwlock {
+	union {
+		long lock;
+		struct {
+			unsigned int read;
+			int write;
+		};
+	} lock;
+};
+
+static inline void ihk_mc_rwlock_init(struct ihk_rwlock *rw)
+{
+	rw->lock.read = 0;
+	rw->lock.write = 1;
+}
+
+static inline void ihk_mc_read_lock(struct ihk_rwlock *rw)
+{
+	asm volatile("1:\t"
+		     "lock; decq %0\n\t"
+		     "jns 3f\n\t"
+		     "lock incq %0\n\t"
+		     "2:\t"
+		     "pause\n\t"
+		     "cmpq $0x1, %0\n\t"
+		     "jns 1b\n\t"
+		     "jmp 2b\n\t"
+		     "3:"
+		     : "+m" (rw->lock.lock) : : "memory");
+}
+
+static inline void ihk_mc_write_lock(struct ihk_rwlock *rw)
+{
+	asm volatile("1:\t"
+		     "lock; decl %0\n\t"
+		     "je 3f\n\t"
+		     "lock; incl %0\n\t"
+		     "2:\t"
+		     "pause\n\t"
+		     "cmpl $0x1,%0\n\t"
+		     "je 1b\n\t"
+		     "jmp 2b\n\t"
+		     "3:"
+		     : "+m" (rw->lock.write) : "i" (((1L) << 32)) : "memory");
+}
+
+static inline int ihk_mc_read_trylock(struct ihk_rwlock *rw)
+{
+	ihk_atomic64_t *count = (ihk_atomic64_t *)rw;
+
+	if (ihk_atomic64_sub_return(1, count) >= 0)
+		return 1;
+	ihk_atomic64_inc(count);
+	return 0;
+}
+
+static inline int ihk_mc_write_trylock(struct ihk_rwlock *rw)
+{
+	ihk_atomic_t *count = (ihk_atomic_t *)&rw->lock.write;
+
+	if (ihk_atomic_dec_and_test(count))
+		return 1;
+	ihk_atomic_inc(count);
+	return 0;
+}
+
+static inline void ihk_mc_read_unlock(struct ihk_rwlock *rw)
+{
+	asm volatile("lock; incq %0" : "+m" (rw->lock.lock) : : "memory");
+}
+
+static inline void ihk_mc_write_unlock(struct ihk_rwlock *rw)
+{
+	asm volatile("lock; incl %0"
+		     : "+m" (rw->lock.write) : "i" (((1L) << 32)) : "memory");
+}
+
+static inline int ihk_mc_write_can_lock(struct ihk_rwlock *rw)
+{
+	return rw->lock.write == 1;
+}
+
+static inline int ihk_mc_read_can_lock(struct ihk_rwlock *rw)
+{
+	return rw->lock.lock > 0;
+}
 #endif

arch/x86_64/kernel/include/arch-memory.h

@@ -1,3 +1,4 @@
+/* arch-memory.h COPYRIGHT FUJITSU LIMITED 2018 */
 /**
  * \file arch-memomry.h
  *  License details are found in the file LICENSE.
@@ -16,7 +17,9 @@
 #define __HEADER_X86_COMMON_ARCH_MEMORY_H
 
 #include <ihk/types.h>
+#include <errno.h>
 
+struct memobj;
 #define KERNEL_CS_ENTRY    4
 #define KERNEL_DS_ENTRY    5
 #define USER_CS_ENTRY      6
@@ -40,18 +43,37 @@
 #define LARGE_PAGE_MASK    (~((unsigned long)LARGE_PAGE_SIZE - 1))
 #define LARGE_PAGE_P2ALIGN (LARGE_PAGE_SHIFT - PAGE_SHIFT)
 
-#define USER_END           0x0000800000000000UL
-#define TASK_UNMAPPED_BASE 0x00002AAAAAA00000UL
+#define USER_END              0x0000800000000000UL
+#define LD_TASK_UNMAPPED_BASE 0x0000155555500000UL
+#define TASK_UNMAPPED_BASE    0x00002AAAAAA00000UL
+
+/*
+ * Canonical negative addresses (i.e., the smallest kernel virtual address)
+ * on x86 64 bit mode (in its most restricted 48 bit format) starts from
+ * 0xffff800000000000, but Linux starts mapping physical memory at 0xffff880000000000.
+ * The 0x80000000000 long gap (8TBs, i.e., 16 PGD level entries in the page tables)
+ * is used for Xen hyervisor (see arch/x86/include/asm/page.h) and that is
+ * what we utilize for McKernel.
+ * This gives us the benefit of being able to use Linux kernel virtual
+ * addresses identically as in Linux.
+ *
+ * NOTE: update these also in eclair.c when modified!
+ */
 #define MAP_ST_START       0xffff800000000000UL
-#define MAP_VMAP_START     0xfffff00000000000UL
-#define MAP_FIXED_START    0xffffffff70000000UL
-#define MAP_KERNEL_START   0xffffffff80000000UL
+#define MAP_VMAP_START     0xffff850000000000UL
+#define MAP_FIXED_START    0xffff860000000000UL
+#define LINUX_PAGE_OFFSET  0xffff880000000000UL
+/*
+ * MAP_KERNEL_START is 8MB below MODULES_END in Linux.
+ * Placing the LWK image in the virtual address space at the end of
+ * the Linux modules section enables us to map the LWK TEXT in Linux
+ * as well, so that Linux can also call into LWK text.
+ * It's defined by cmake.
+ */
 #define STACK_TOP(region)  ((region)->user_end)
 
 #define MAP_VMAP_SIZE      0x0000000100000000UL
 
-#define KERNEL_PHYS_OFFSET MAP_ST_START
-
 #define PTL4_SHIFT         39
 #define PTL4_SIZE          (1UL << PTL4_SHIFT)
 #define PTL3_SHIFT         30
@@ -142,6 +164,10 @@ typedef unsigned long pte_t;
 #define PM_PRESENT          PM_STATUS(4LL)
 #define PM_SWAP             PM_STATUS(2LL)
 
+#define USER_STACK_PREPAGE_SIZE	LARGE_PAGE_SIZE
+#define USER_STACK_PAGE_MASK	LARGE_PAGE_MASK
+#define USER_STACK_PAGE_P2ALIGN	LARGE_PAGE_P2ALIGN
+#define USER_STACK_PAGE_SHIFT	LARGE_PAGE_SHIFT
 
 /* For easy conversion, it is better to be the same as architecture's ones */
 enum ihk_mc_pt_attribute {
@@ -159,12 +185,10 @@ enum ihk_mc_pt_attribute {
 
 enum ihk_mc_pt_attribute attr_mask;
 
-#ifdef POSTK_DEBUG_ARCH_DEP_12
 static inline int pfn_is_write_combined(uintptr_t pfn)
 {
 	return ((pfn & PFL1_PWT) && !(pfn & PFL1_PCD));
 }
-#endif /* #ifdef POSTK_DEBUG_ARCH_DEP_12 */
 
 static inline int pte_is_null(pte_t *ptep)
 {
@@ -314,7 +338,102 @@ static inline void pte_set_dirty(pte_t *ptep, size_t pgsize)
 	return;
 }
 
+static inline int pte_is_contiguous(pte_t *ptep)
+{
+	return 0;
+}
+
+static inline int pgsize_is_contiguous(size_t pgsize)
+{
+	return 0;
+}
+
+static inline int pgsize_to_tbllv(size_t pgsize)
+{
+	switch (pgsize) {
+	case PTL1_SIZE:	return 1;
+	case PTL2_SIZE:	return 2;
+	case PTL3_SIZE:	return 3;
+	case PTL4_SIZE:	return 4;
+	default:
+#if 0	/* XXX: workaround. cannot use panic() here */
+		panic("pgsize_to_tbllv");
+#else
+		return 0;
+#endif
+	}
+	return 0;
+}
+
+static inline int pgsize_to_pgshift(size_t pgsize)
+{
+	switch (pgsize) {
+	case PTL1_SIZE:	return PTL1_SHIFT;
+	case PTL2_SIZE:	return PTL2_SHIFT;
+	case PTL3_SIZE:	return PTL3_SHIFT;
+	case PTL4_SIZE:	return PTL4_SHIFT;
+	default: return -EINVAL;
+	}
+}
+
+static inline size_t tbllv_to_pgsize(int level)
+{
+	switch (level) {
+	case 1: return PTL1_SIZE;
+	case 2: return PTL2_SIZE;
+	case 3: return PTL3_SIZE;
+	case 4: return PTL4_SIZE;
+	default:
+#if 0	/* XXX: workaround. cannot use panic() here */
+		panic("tbllv_to_pgsize");
+#else
+		return 0;
+#endif
+	}
+	return 0;
+}
+
+static inline size_t tbllv_to_contpgsize(int level)
+{
+	return 0;
+}
+
+static inline int tbllv_to_contpgshift(int level)
+{
+	return 0;
+}
+
+static inline pte_t *get_contiguous_head(pte_t *__ptep, size_t __pgsize)
+{
+	return __ptep;
+}
+
+static inline pte_t *get_contiguous_tail(pte_t *__ptep, size_t __pgsize)
+{
+	return __ptep;
+}
+
+int split_contiguous_pages(pte_t *ptep, size_t pgsize,
+		uint32_t memobj_flags);
+
+static inline int page_is_contiguous_head(pte_t *ptep, size_t pgsize)
+{
+	return 0;
+}
+
+static inline int page_is_contiguous_tail(pte_t *ptep, size_t pgsize)
+{
+	return 0;
+}
+
 struct page_table;
+static inline void arch_adjust_allocate_page_size(struct page_table *pt,
+						  uintptr_t fault_addr,
+						  pte_t *ptep,
+						  void **pgaddrp,
+						  size_t *pgsizep)
+{
+}
 void set_pte(pte_t *ppte, unsigned long phys, enum ihk_mc_pt_attribute attr);
 pte_t *get_pte(struct page_table *pt, void *virt, enum ihk_mc_pt_attribute attr);
 
@@ -331,4 +450,12 @@ extern unsigned long ap_trampoline;
 
 /* Local is cachable */
 #define IHK_IKC_QUEUE_PT_ATTR (PTATTR_NO_EXECUTE | PTATTR_WRITABLE)
+
+#ifdef ENABLE_FUGAKU_HACKS
+#ifndef __ASSEMBLY__
+# define ALIGN_UP(x, align)     ALIGN_DOWN((x) + (align) - 1, align)
+# define ALIGN_DOWN(x, align)   ((x) & ~((align) - 1))
+#endif /* !__ASSEMBLY__ */
+#endif
+
 #endif

arch/x86_64/kernel/include/arch/cpu.h

@@ -13,19 +13,17 @@
 #ifndef ARCH_CPU_H
 #define ARCH_CPU_H
 
+#define mb()    asm volatile("mfence":::"memory")
+#define rmb()   asm volatile("lfence":::"memory")
+#define wmb()   asm volatile("sfence" ::: "memory")
+
+#define smp_mb()    mb()
+#define smp_rmb()   rmb()
+#define smp_wmb()	barrier()
+
 #define arch_barrier()	asm volatile("" : : : "memory")
 
-static inline void rmb(void)
-{
-	arch_barrier();
-}
-
-static inline void wmb(void)
-{
-	arch_barrier();
-}
-
-static unsigned long read_tsc(void)
+static inline unsigned long read_tsc(void)
 {
 	unsigned int low, high;
 
@@ -34,4 +32,21 @@ static unsigned long read_tsc(void)
 	return (low | ((unsigned long)high << 32));
 }
 
+#define smp_load_acquire(p)						\
+({									\
+	typeof(*p) ___p1 = ACCESS_ONCE(*p);				\
+	compiletime_assert_atomic_type(*p);				\
+	barrier();							\
+	___p1;								\
+})
+
+#define smp_store_release(p, v)			\
+({							\
+	compiletime_assert_atomic_type(*p);	\
+	barrier();							\
+	WRITE_ONCE(*p, v);					\
+})
+
+void arch_flush_icache_all(void);
+
 #endif /* ARCH_CPU_H */

arch/x86_64/kernel/include/arch_rusage.h

@@ -1,32 +0,0 @@
-#ifndef ARCH_RUSAGE_H_INCLUDED
-#define ARCH_RUSAGE_H_INCLUDED
-
-#define DEBUG_RUSAGE
-
-#define IHK_OS_PGSIZE_4KB 0
-#define IHK_OS_PGSIZE_2MB 1
-#define IHK_OS_PGSIZE_1GB 2
-
-extern struct rusage_global *rusage;
-
-static inline int rusage_pgsize_to_pgtype(size_t pgsize)
-{
-	int ret = IHK_OS_PGSIZE_4KB;
-	switch (pgsize) {
-	case PTL1_SIZE:
-		ret = IHK_OS_PGSIZE_4KB;
-		break;
-	case PTL2_SIZE:
-		ret = IHK_OS_PGSIZE_2MB;
-		break;
-	case PTL3_SIZE:
-		ret = IHK_OS_PGSIZE_1GB;
-		break;
-	default:
-		kprintf("%s: Error: Unknown pgsize=%ld\n", __FUNCTION__, pgsize);
-		break;
-	}
-	return ret;
-}
-
-#endif /* !defined(ARCH_RUSAGE_H_INCLUDED) */

arch/x86_64/kernel/include/cpulocal.h

@@ -50,7 +50,12 @@ struct x86_cpu_local_variables {
 
 struct x86_cpu_local_variables *get_x86_cpu_local_variable(int id);
 struct x86_cpu_local_variables *get_x86_this_cpu_local(void);
+void *get_x86_cpu_local_kstack(int id);
 void *get_x86_this_cpu_kstack(void);
 
+#ifdef ENABLE_FUGAKU_HACKS
+#define LOCALS_SPAN (4 * PAGE_SIZE)
+#define KERNEL_STACK_SIZE LOCALS_SPAN
+#endif
 
 #endif

arch/x86_64/kernel/include/elf.h

@@ -1,4 +1,4 @@
-#ifdef POSTK_DEBUG_ARCH_DEP_18 /* coredump arch separation. */
+/* elf.h COPYRIGHT FUJITSU LIMITED 2018 */
 #ifndef __HEADER_X86_COMMON_ELF_H
 #define __HEADER_X86_COMMON_ELF_H
 
@@ -56,4 +56,3 @@ struct user_regs64_struct
 typedef elf_greg64_t elf_gregset64_t[ELF_NGREG64];
 
 #endif /* __HEADER_S64FX_COMMON_ELF_H */
-#endif /* !POSTK_DEBUG_ARCH_DEP_18 */

arch/x86_64/kernel/include/elfcore.h

@@ -1,94 +0,0 @@
-#ifndef POSTK_DEBUG_ARCH_DEP_18 /* coredump arch separation. */
-/*
- * Structures and definitions for ELF core file.
- * Extracted from 
- * System V Application Binary Interface - DRAFT - 10 June 2013,
- * http://www.sco.com/developers/gabi/latest/contents.html
- */
-
-typedef	uint16_t	Elf64_Half;
-typedef	uint32_t	Elf64_Word;
-typedef	uint64_t	Elf64_Xword;
-typedef	uint64_t	Elf64_Addr;
-typedef	uint64_t	Elf64_Off;
-
-#define	EI_NIDENT	16
-
-typedef struct {
-	unsigned char e_ident[EI_NIDENT];
-	Elf64_Half e_type;
-	Elf64_Half e_machine;
-	Elf64_Word e_version;
-	Elf64_Addr e_entry;
-	Elf64_Off e_phoff;
-	Elf64_Off e_shoff;
-	Elf64_Word e_flags;
-	Elf64_Half e_ehsize;
-	Elf64_Half e_phentsize;
-	Elf64_Half e_phnum;
-	Elf64_Half e_shentsize;
-	Elf64_Half e_shnum;
-	Elf64_Half e_shstrndx;
-} Elf64_Ehdr;
-
-#define	EI_MAG0		0
-#define	EI_MAG1		1
-#define EI_MAG2		2
-#define	EI_MAG3		3
-#define	EI_CLASS	4
-#define EI_DATA		5
-#define	EI_VERSION	6
-#define	EI_OSABI	7
-#define	EI_ABIVERSION	8
-#define EI_PAD		9
-
-
-#define	ELFMAG0		0x7f
-#define	ELFMAG1		'E'
-#define	ELFMAG2		'L'
-#define	ELFMAG3		'F'
-
-#define	ELFCLASS64	2	/* 64-bit object */
-#define	ELFDATA2LSB	1	/* LSB */
-#define	El_VERSION	1	/* defined to be the same as EV CURRENT */
-#define	ELFOSABI_NONE	0	/* unspecied */
-#define	El_ABIVERSION_NONE	0	/* unspecied */
-#define	ET_CORE		4	/* Core file */
-#define	EM_X86_64	62	/* AMD x86-64 architecture */
-#define	EM_K10M		181	/* Intel K10M */
-#define	EV_CURRENT	1	/* Current version */
-
-typedef struct {
-	Elf64_Word p_type;
-	Elf64_Word p_flags;
-	Elf64_Off p_offset;
-	Elf64_Addr p_vaddr;
-	Elf64_Addr p_paddr;
-	Elf64_Xword p_filesz;
-	Elf64_Xword p_memsz;
-	Elf64_Xword p_align;
-} Elf64_Phdr;
-
-#define	PT_LOAD		1
-#define	PT_NOTE		4
-
-#define	PF_X		1	/* executable bit */
-#define	PF_W		2	/* writable bit */
-#define	PF_R		4	/* readable bit */
-
-struct note {
-	Elf64_Word namesz;
-	Elf64_Word descsz;
-	Elf64_Word type;
-	/* name char[namesz] and desc[descsz] */
-};
-
-#define	NT_PRSTATUS	1
-#define	NT_PRFRPREG	2
-#define	NT_PRPSINFO	3
-#define	NT_AUXV		6
-#define	NT_X86_STATE	0x202
-
-#include "elfcoregpl.h"
-
-#endif /* !POSTK_DEBUG_ARCH_DEP_18 */

arch/x86_64/kernel/include/elfcoregpl.h

@@ -1,96 +0,0 @@
-#ifndef POSTK_DEBUG_ARCH_DEP_18 /* coredump arch separation. */
-/* 
- * Structures and defines from GPLed file.
- */
-
-#define	pid_t		int
-
-/* From /usr/include/linux/elfcore.h of Linux */
-
-#define	ELF_PRARGSZ	(80)
-
-/* From /usr/include/linux/elfcore.h fro Linux */
-
-struct elf_siginfo
-{
-	int si_signo;
-	int si_code;
-	int si_errno;
-};
-
-/* From bfd/hosts/x86-64linux.h of gdb. */
-
-typedef uint64_t __attribute__ ((__aligned__ (8))) a8_uint64_t;
-typedef	a8_uint64_t elf_greg64_t;
-
-struct user_regs64_struct
-{
-	a8_uint64_t r15;
-	a8_uint64_t r14;
-	a8_uint64_t r13;
-	a8_uint64_t r12;
-	a8_uint64_t rbp;
-	a8_uint64_t rbx;
-	a8_uint64_t r11;
-	a8_uint64_t r10;
-	a8_uint64_t r9;
-	a8_uint64_t r8;
-	a8_uint64_t rax;
-	a8_uint64_t rcx;
-	a8_uint64_t rdx;
-	a8_uint64_t rsi;
-	a8_uint64_t rdi;
-	a8_uint64_t orig_rax;
-	a8_uint64_t rip;
-	a8_uint64_t cs;
-	a8_uint64_t eflags;
-	a8_uint64_t rsp;
-	a8_uint64_t ss;
-	a8_uint64_t fs_base;
-	a8_uint64_t gs_base;
-	a8_uint64_t ds;
-	a8_uint64_t es;
-	a8_uint64_t fs;
-	a8_uint64_t gs;
-};
-
-#define ELF_NGREG64 (sizeof (struct user_regs64_struct) / sizeof(elf_greg64_t))
-
-typedef elf_greg64_t elf_gregset64_t[ELF_NGREG64];
-
-struct prstatus64_timeval
-{
-	a8_uint64_t tv_sec;
-	a8_uint64_t tv_usec;
-};
-struct elf_prstatus64
-{
-	struct elf_siginfo pr_info;
-	short int pr_cursig;
-	a8_uint64_t pr_sigpend;
-	a8_uint64_t pr_sighold;
-	pid_t pr_pid;
-	pid_t pr_ppid;
-	pid_t pr_pgrp;
-	pid_t pr_sid;
-	struct prstatus64_timeval pr_utime;
-	struct prstatus64_timeval pr_stime;
-	struct prstatus64_timeval pr_cutime;
-	struct prstatus64_timeval pr_cstime;
-	elf_gregset64_t pr_reg;
-	int pr_fpvalid;
-};
-struct elf_prpsinfo64
-{
-	char pr_state;
-	char pr_sname;
-	char pr_zomb;
-	char pr_nice;
-	a8_uint64_t pr_flag;
-	unsigned int pr_uid;
-	unsigned int pr_gid;
-	int pr_pid, pr_ppid, pr_pgrp, pr_sid;
-	char pr_fname[16];
-	char pr_psargs[ELF_PRARGSZ];
-};
-#endif /* !POSTK_DEBUG_ARCH_DEP_18 */

arch/x86_64/kernel/include/hwcap.h

@@ -1,5 +1,4 @@
-/* hwcap.h COPYRIGHT FUJITSU LIMITED 2017 */
-#ifdef POSTK_DEBUG_ARCH_DEP_65
+/* hwcap.h COPYRIGHT FUJITSU LIMITED 2017-2018 */
 #ifndef _UAPI__ASM_HWCAP_H
 #define _UAPI__ASM_HWCAP_H
 
@@ -9,4 +8,3 @@ static unsigned long arch_get_hwcap(void)
 }
 
 #endif /* _UAPI__ASM_HWCAP_H */
-#endif /* POSTK_DEBUG_ARCH_DEP_65 */

arch/x86_64/kernel/include/ihk/atomic.h

@@ -13,6 +13,8 @@
 #ifndef HEADER_X86_COMMON_IHK_ATOMIC_H
 #define HEADER_X86_COMMON_IHK_ATOMIC_H
  
+#include <lwk/compiler.h>
+
 /***********************************************************************
  * ihk_atomic_t
  */
@@ -114,7 +116,7 @@ static inline long ihk_atomic64_read(const ihk_atomic64_t *v)
 	return *(volatile long *)&(v)->counter64;
 }
 
-static inline void ihk_atomic64_set(ihk_atomic64_t *v, int i)
+static inline void ihk_atomic64_set(ihk_atomic64_t *v, long i)
 {
 	v->counter64 = i;
 }
@@ -124,6 +126,22 @@ static inline void ihk_atomic64_inc(ihk_atomic64_t *v)
 	asm volatile ("lock incq %0" : "+m"(v->counter64));
 }
 
+static inline long ihk_atomic64_add_return(long i, ihk_atomic64_t *v)
+{
+	long __i;
+
+	__i = i;
+	asm volatile("lock xaddq %0, %1"
+		     : "+r" (i), "+m" (v->counter64)
+		     : : "memory");
+	return i + __i;
+}
+
+static inline long ihk_atomic64_sub_return(long i, ihk_atomic64_t *v)
+{
+	return ihk_atomic64_add_return(-i, v);
+}
+
 /***********************************************************************
  * others
  */
@@ -156,43 +174,55 @@ static inline unsigned long xchg8(unsigned long *ptr, unsigned long x)
 	return __x;
 }
 
-#define __xchg(x, ptr, size)						\
-({									\
-	__typeof(*(ptr)) __x = (x);					\
-	switch (size) {							\
-	case 1:								\
-		asm volatile("xchgb %b0,%1"				\
-			     : "=q" (__x)				\
-			     : "m" (*__xg(ptr)), "0" (__x)		\
-			     : "memory");				\
-		break;							\
-	case 2:								\
-		asm volatile("xchgw %w0,%1"				\
-			     : "=r" (__x)				\
-			     : "m" (*__xg(ptr)), "0" (__x)		\
-			     : "memory");				\
-		break;							\
-	case 4:								\
-		asm volatile("xchgl %k0,%1"				\
-			     : "=r" (__x)				\
-			     : "m" (*__xg(ptr)), "0" (__x)		\
-			     : "memory");				\
-		break;							\
-	case 8:								\
-		asm volatile("xchgq %0,%1"				\
-			     : "=r" (__x)				\
-			     : "m" (*__xg(ptr)), "0" (__x)		\
-			     : "memory");				\
-		break;							\
-	default:							\
-		panic("xchg for wrong size");					\
-	}								\
-	__x;								\
-})
+#define __X86_CASE_B	1
+#define __X86_CASE_W	2
+#define __X86_CASE_L	4
+#define __X86_CASE_Q	8
 
+extern void __xchg_wrong_size(void)
+	__compiletime_error("Bad argument size for xchg");
 
-#define xchg(ptr, v)							\
-	__xchg((v), (ptr), sizeof(*ptr))
+/*
+ * An exchange-type operation, which takes a value and a pointer, and
+ * returns the old value.
+ */
+#define __xchg_op(ptr, arg, op, lock)					\
+	({								\
+		__typeof__(*(ptr)) __ret = (arg);			\
+		switch (sizeof(*(ptr))) {				\
+		case __X86_CASE_B:					\
+			asm volatile (lock #op "b %b0, %1\n"		\
+				      : "+q" (__ret), "+m" (*(ptr))	\
+				      : : "memory", "cc");		\
+			break;						\
+		case __X86_CASE_W:					\
+			asm volatile (lock #op "w %w0, %1\n"		\
+				      : "+r" (__ret), "+m" (*(ptr))	\
+				      : : "memory", "cc");		\
+			break;						\
+		case __X86_CASE_L:					\
+			asm volatile (lock #op "l %0, %1\n"		\
+				      : "+r" (__ret), "+m" (*(ptr))	\
+				      : : "memory", "cc");		\
+			break;						\
+		case __X86_CASE_Q:					\
+			asm volatile (lock #op "q %q0, %1\n"		\
+				      : "+r" (__ret), "+m" (*(ptr))	\
+				      : : "memory", "cc");		\
+			break;						\
+		default:						\
+			__xchg_wrong_size();			\
+		}							\
+		__ret;							\
+	})
+
+/*
+ * Note: no "lock" prefix even on SMP: xchg always implies lock anyway.
+ * Since this is generally used to protect other memory information, we
+ * use "asm volatile" and "memory" clobbers to prevent gcc from moving
+ * information around.
+ */
+#define xchg(ptr, v)	__xchg_op((ptr), (v), xchg, "")
 
 static inline unsigned long atomic_cmpxchg8(unsigned long *addr,
 		unsigned long oldval,
@@ -241,4 +271,66 @@ static inline unsigned long ihk_atomic_add_long_return(long i, long *v) {
         return i + __i;
 }
 
+extern void __cmpxchg_wrong_size(void)
+	__compiletime_error("Bad argument size for cmpxchg");
+
+/*
+ * Atomic compare and exchange.  Compare OLD with MEM, if identical,
+ * store NEW in MEM.  Return the initial value in MEM.  Success is
+ * indicated by comparing RETURN with OLD.
+ */
+#define __raw_cmpxchg(ptr, old, new, size, lock)		\
+({									\
+	__typeof__(*(ptr)) __ret;					\
+	__typeof__(*(ptr)) __old = (old);				\
+	__typeof__(*(ptr)) __new = (new);				\
+	switch (size) {							\
+	case __X86_CASE_B:						\
+	{								\
+		volatile uint8_t *__ptr = (volatile uint8_t *)(ptr);\
+		asm volatile(lock "cmpxchgb %2,%1"			\
+			     : "=a" (__ret), "+m" (*__ptr)		\
+			     : "q" (__new), "0" (__old)			\
+			     : "memory");				\
+		break;							\
+	}								\
+	case __X86_CASE_W:						\
+	{								\
+		volatile uint16_t *__ptr = (volatile uint16_t *)(ptr);\
+		asm volatile(lock "cmpxchgw %2,%1"			\
+			     : "=a" (__ret), "+m" (*__ptr)		\
+			     : "r" (__new), "0" (__old)			\
+			     : "memory");				\
+		break;							\
+	}								\
+	case __X86_CASE_L:						\
+	{								\
+		volatile uint32_t *__ptr = (volatile uint32_t *)(ptr);\
+		asm volatile(lock "cmpxchgl %2,%1"			\
+			     : "=a" (__ret), "+m" (*__ptr)		\
+			     : "r" (__new), "0" (__old)			\
+			     : "memory");				\
+		break;							\
+	}								\
+	case __X86_CASE_Q:						\
+	{								\
+		volatile uint64_t *__ptr = (volatile uint64_t *)(ptr);\
+		asm volatile(lock "cmpxchgq %2,%1"			\
+			     : "=a" (__ret), "+m" (*__ptr)		\
+			     : "r" (__new), "0" (__old)			\
+			     : "memory");				\
+		break;							\
+	}								\
+	default:							\
+		__cmpxchg_wrong_size();		\
+	}								\
+	__ret;								\
+})
+
+#define __cmpxchg(ptr, old, new, size)					\
+	__raw_cmpxchg((ptr), (old), (new), (size), "lock; ")
+
+#define cmpxchg(ptr, old, new)						\
+	__cmpxchg(ptr, old, new, sizeof(*(ptr)))
+
 #endif

arch/x86_64/kernel/include/ihk/types.h

@@ -1,3 +1,4 @@
+/* types.h COPYRIGHT FUJITSU LIMITED 2018 */
 /**
  * \file types.h
  *  Licence details are found in the file LICENSE.
@@ -29,13 +30,11 @@ typedef uint64_t           size_t;
 typedef int64_t            ssize_t;
 typedef int64_t            off_t;
 
-#ifdef POSTK_DEBUG_ARCH_DEP_18 /* coredump arch separation. */
 typedef int32_t            key_t;
 typedef uint32_t           uid_t;
 typedef uint32_t           gid_t;
 typedef int64_t            time_t;
 typedef int32_t            pid_t;
-#endif /* POSTK_DEBUG_ARCH_DEP_18 */
 
 #define NULL ((void *)0)
 

arch/x86_64/kernel/include/prctl.h

@@ -9,6 +9,9 @@
 #ifndef __ARCH_PRCTL_H
 #define __ARCH_PRCTL_H
 
+#define PR_SET_THP_DISABLE 41
+#define PR_GET_THP_DISABLE 42
+
 #define ARCH_SET_GS 0x1001
 #define ARCH_SET_FS 0x1002
 #define ARCH_GET_FS 0x1003