The build is successful, testing is required

Change-Id: I503233b393e5bfec003d407512d1028de3a60946

.gitignore

@@ -1,3 +1,4 @@
+*~
 *.o
 *.elf
 *.bin
@@ -8,9 +9,36 @@
 Module.symvers
 *.order
 .tmp_versions
-
-elfboot/elfboot
-elfboot/elfboot_test
-linux/executer/mcexec
-linux/mod_test*
-linux/target
+old_timestamp
+CMakeFiles
+CMakeCache.txt
+Makefile
+!test/*/*/Makefile
+!test/signalonfork+wait/Makefile
+!test/perf_overflow/Makefile
+!test/*/*/*.cmd
+Kbuild
+cmake_install.cmake
+config.h
+mcstop+release.sh
+mcreboot.sh
+mcreboot.1
+mcoverlay-destroy.sh
+mcoverlay-create.sh
+kernel/mckernel.img
+kernel/include/swapfmt.h
+executer/user/vmcore2mckdump
+executer/user/ql_talker
+executer/user/mcexec.1
+executer/user/mcexec
+executer/user/libsched_yield.so.1.0.0
+executer/user/libsched_yield.so
+executer/user/libmcexec.a
+executer/user/libldump2mcdump.so
+executer/user/eclair
+tools/mcstat/mcstat
+/_CPack_Packages
+/CPackSourceConfig.cmake
+CPackConfig.cmake
+/build
+mckernel-*.tar.gz

.gitmodules

@@ -0,0 +1,12 @@
+[submodule "ihk"]
+	path = ihk
+	url = https://github.com/ihkmckernel/ihk.git
+[submodule "executer/user/lib/libdwarf/libdwarf"]
+	path = executer/user/lib/libdwarf/libdwarf
+	url = https://github.com/bgerofi/libdwarf.git
+[submodule "executer/user/lib/syscall_intercept"]
+	path = executer/user/lib/syscall_intercept
+	url = https://github.com/ihkmckernel/syscall_intercept.git
+[submodule "executer/user/lib/uti"]
+	path = executer/user/lib/uti
+	url = https://github.com/ihkmckernel/uti.git

CMakeLists.txt

@@ -0,0 +1,361 @@
+cmake_minimum_required(VERSION 3.11)
+
+if (NOT CMAKE_BUILD_TYPE)
+	set (CMAKE_BUILD_TYPE "Debug" CACHE STRING "Build type: Debug Release..." FORCE)
+endif (NOT CMAKE_BUILD_TYPE)
+
+enable_language(C ASM)
+
+project(mckernel C ASM)
+set(MCKERNEL_VERSION "1.8.0")
+
+# See "Fedora Packaging Guidelines -- Versioning"
+set(MCKERNEL_RELEASE "")
+
+set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake/modules)
+# for rpmbuild
+if(DEFINED SYSCONF_INSTALL_DIR)
+	set(CMAKE_INSTALL_SYSCONFDIR "${SYSCONF_INSTALL_DIR}")
+endif()
+
+if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+	set(BUILD_TARGET "smp-x86" CACHE STRING "Build target: smp-x86 | smp-arm64")
+elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64")
+	set(BUILD_TARGET "smp-arm64" CACHE STRING "Build target: smp-x86 | smp-arm64")
+endif()
+
+if (BUILD_TARGET STREQUAL "smp-x86")
+	set(ARCH "x86_64")
+elseif (BUILD_TARGET STREQUAL "smp-arm64")
+	set(ARCH "arm64")
+endif()
+
+include(GNUInstallDirs)
+include(CMakeParseArguments)
+include(Kbuild)
+include(CheckCCompilerFlag)
+include(AutoconfHelper)
+
+CHECK_C_COMPILER_FLAG(-Wno-implicit-fallthrough IMPLICIT_FALLTHROUGH)
+if(IMPLICIT_FALLTHROUGH)
+	set(EXTRA_WARNINGS "-Wno-implicit-fallthrough")
+endif(IMPLICIT_FALLTHROUGH)
+
+CHECK_C_COMPILER_FLAG(-Wno-stringop-truncation STRINGOP_TRUNCATION)
+if(STRINGOP_TRUNCATION)
+	list(APPEND EXTRA_WARNINGS "-Wno-stringop-truncation")
+endif(STRINGOP_TRUNCATION)
+
+# build options
+set(CFLAGS_WARNING "-Wall" "-Wextra" "-Wno-unused-parameter" "-Wno-sign-compare" "-Wno-unused-function" ${EXTRA_WARNINGS} CACHE STRING "Warning flags")
+add_compile_options(${CFLAGS_WARNING})
+
+option(ENABLE_WERROR "Enable -Werror" OFF)
+if (ENABLE_WERROR)
+	add_compile_options("-Werror")
+endif(ENABLE_WERROR)
+
+execute_process(COMMAND bash -c "ls -ld /proc/tofu/ 2>/dev/null | wc -l"
+  OUTPUT_VARIABLE PROC_TOFU OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+if(PROC_TOFU STREQUAL "1")
+	option(ENABLE_TOFU "Built-in tofu driver support" ON)
+else()
+	option(ENABLE_TOFU "Built-in tofu driver support" OFF)
+endif()
+
+if(ENABLE_TOFU)
+	add_definitions(-DENABLE_TOFU)
+	set(KBUILD_C_FLAGS "${KBUILD_C_FLAGS} -DENABLE_TOFU")
+endif()
+
+# when compiling on a compute-node
+execute_process(COMMAND bash -c "grep $(hostname) /etc/opt/FJSVfefs/config/fefs_node1.csv 2>/dev/null | cut -d, -f2 | grep -o CN"
+  OUTPUT_VARIABLE FUGAKU_NODE_TYPE OUTPUT_STRIP_TRAILING_WHITESPACE)
+if(FUGAKU_NODE_TYPE STREQUAL "CN")
+	option(ENABLE_FUGAKU_HACKS "Fugaku hacks" ON)
+else()
+	option(ENABLE_FUGAKU_HACKS "Fugaku hacks" OFF)
+endif()
+
+if(ENABLE_FUGAKU_HACKS)
+	add_definitions(-DENABLE_FUGAKU_HACKS)
+	set(KBUILD_C_FLAGS "${KBUILD_C_FLAGS} -DENABLE_FUGAKU_HACKS")
+endif()
+
+# Fujitsu MPI tries to xpmem-attach segment with size of range size + 1?
+#set(FJMPI_VERSION_COMMAND "a=\$(which mpifcc); b=\${a%/*/*}; c=\${b##*/}; d=\${c#*-}; echo \$d")
+set(FJMPI_VERSION_COMMAND "a=\$(which mpifort); b=\${a%/*/*}; c=\${b##*/}; d=\${c#*-}; echo \$d")
+execute_process(COMMAND bash -c "${FJMPI_VERSION_COMMAND}"
+  OUTPUT_VARIABLE FJMPI_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE)
+message("FJMPI_VERSION: ${FJMPI_VERSION}")
+
+if(NOT "${FJMPI_VERSION}" STREQUAL "")
+	option(ENABLE_FJMPI_WORKAROUND "Fujitsu MPI workaround" ON)
+else()
+	option(ENABLE_FJMPI_WORKAROUND "Fujitsu MPI workaround" OFF)
+endif()
+
+if(ENABLE_FJMPI_WORKAROUND)
+	add_definitions(-DENABLE_FJMPI_WORKAROUND)
+	set(KBUILD_C_FLAGS "${KBUILD_C_FLAGS} -DENABLE_FJMPI_WORKAROUND")
+endif()
+
+# krm that mandates reserved memory amount >= available at boot time?
+execute_process(COMMAND bash -c "rpm -qi FJSVpxkrm-plugin-mckernel | awk '$1 == \"Version\" && $2 == \":\" { print $3 }'"
+  OUTPUT_VARIABLE KRM_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE)
+message("KRM_VERSION: ${KRM_VERSION}")
+
+execute_process(COMMAND bash -c "rpm -qi FJSVpxkrm-plugin-mckernel | awk '$1 == \"Release\" && $2 == \":\" { print $3 }'"
+  OUTPUT_VARIABLE KRM_RELEASE OUTPUT_STRIP_TRAILING_WHITESPACE)
+message("KRM_RELEASE: ${KRM_RELEASE}")
+
+if("${KRM_VERSION}" STREQUAL "")
+	option(ENABLE_KRM_WORKAROUND "krm workaround" OFF)
+elseif("${KRM_VERSION}" VERSION_GREATER_EQUAL 4.0.2 OR
+    ("${KRM_VERSION}" VERSION_EQUAL 4.0.1 AND "${KRM_RELEASE}" VERSION_GREATER_EQUAL 25.13.1.0))
+	option(ENABLE_KRM_WORKAROUND "krm workaround" OFF)
+else()
+	option(ENABLE_KRM_WORKAROUND "krm workaround" ON)
+endif()
+
+if(ENABLE_KRM_WORKAROUND)
+	add_definitions(-DENABLE_KRM_WORKAROUND)
+	set(KBUILD_C_FLAGS "${KBUILD_C_FLAGS} -DENABLE_KRM_WORKAROUND")
+endif()
+
+# SIGSTOP instead of SIGSEGV, additional IHK Linux kmsg
+option(ENABLE_FUGAKU_DEBUG "Fugaku debug instrumentation" OFF)
+
+if(ENABLE_FUGAKU_DEBUG)
+	add_definitions(-DENABLE_FUGAKU_DEBUG)
+	set(KBUILD_C_FLAGS "${KBUILD_C_FLAGS} -DENABLE_FUGAKU_DEBUG")
+endif()
+
+# redirect kernel messages to Linux's /dev/kmsg
+option(ENABLE_KMSG_REDIRECT "Redirect kernel message to Linux's /dev/kmsg" OFF)
+
+if(ENABLE_KMSG_REDIRECT)
+	add_definitions(-DENABLE_KMSG_REDIRECT)
+	set(KBUILD_C_FLAGS "${KBUILD_C_FLAGS} -DENABLE_KMSG_REDIRECT")
+endif()
+
+option(PROFILE_ENABLE "System call profile" ON)
+if(PROFILE_ENABLE)
+	add_definitions(-DPROFILE_ENABLE)
+	set(KBUILD_C_FLAGS "${KBUILD_C_FLAGS} -DPROFILE_ENABLE")
+endif()
+
+option(ENABLE_LINUX_WORK_IRQ_FOR_IKC "Use Linux work IRQ for IKC IPI" ON)
+if (ENABLE_LINUX_WORK_IRQ_FOR_IKC)
+	set(KBUILD_C_FLAGS "${KBUILD_C_FLAGS} -DIHK_IKC_USE_LINUX_WORK_IRQ")
+	add_definitions(-DIHK_IKC_USE_LINUX_WORK_IRQ)
+endif()
+
+if (BUILD_TARGET STREQUAL "smp-arm64")
+	set(POSTK_DEBUG_DEFINES "-DPOSTK_DEBUG_ARCH_DEP_100 -DPOSTK_DEBUG_ARCH_DEP_42 -DPOSTK_DEBUG_TEMP_FIX_29 -DPOSTK_DEBUG_TEMP_FIX_49")
+	add_definitions("${POSTK_DEBUG_DEFINES}")
+	set(KBUILD_C_FLAGS "${KBUILD_C_FLAGS} ${POSTK_DEBUG_DEFINES}")
+
+	execute_process(COMMAND awk -F= "$1 == \"CONFIG_ARM64_64K_PAGES\" { print $2; exit; }" "${KERNEL_DIR}/.config"
+			OUTPUT_VARIABLE CONFIG_ARM64_64K_PAGES OUTPUT_STRIP_TRAILING_WHITESPACE)
+	execute_process(COMMAND awk -F= "$1 == \"CONFIG_ARM64_VA_BITS\" { print $2; exit; }" "${KERNEL_DIR}/.config"
+			OUTPUT_VARIABLE CONFIG_ARM64_VA_BITS OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+	message("Host kernel CONFIG_ARM64_64K_PAGES=${CONFIG_ARM64_64K_PAGES}")
+	message("Host kernel CONFIG_ARM64_VA_BITS=${CONFIG_ARM64_VA_BITS}")
+
+	if(CONFIG_ARM64_64K_PAGES STREQUAL "y")
+		if(CONFIG_ARM64_VA_BITS STREQUAL 42)
+			add_definitions(-DCONFIG_ARM64_PGTABLE_LEVELS=2 -DCONFIG_ARM64_VA_BITS=42 -DCONFIG_ARM64_64K_PAGES)
+			set(LINKER_SCRIPT "smp-arm64_type3.lds")
+		elseif(CONFIG_ARM64_VA_BITS STREQUAL 48)
+			add_definitions(-DCONFIG_ARM64_PGTABLE_LEVELS=3 -DCONFIG_ARM64_VA_BITS=48 -DCONFIG_ARM64_64K_PAGES)
+			set(LINKER_SCRIPT "smp-arm64_type4.lds")
+		endif()
+	else(CONFIG_ARM64_64K_PAGES STREQUAL "y")
+		if(CONFIG_ARM64_VA_BITS STREQUAL 39)
+			add_definitions(-DCONFIG_ARM64_PGTABLE_LEVELS=3 -DCONFIG_ARM64_VA_BITS=39)
+			set(LINKER_SCRIPT "smp-arm64_type1.lds")
+		elseif(CONFIG_ARM64_VA_BITS STREQUAL 48)
+			add_definitions(-DCONFIG_ARM64_PGTABLE_LEVELS=4 -DCONFIG_ARM64_VA_BITS=48)
+			set(LINKER_SCRIPT "smp-arm64_type2.lds")
+		endif()
+	endif(CONFIG_ARM64_64K_PAGES STREQUAL "y")
+endif()
+set_property(CACHE BUILD_TARGET PROPERTY STRINGS smp-x86 smp-arm64)
+
+# define MAP_KERNEL_START
+
+set(tmpdir ${CMAKE_CURRENT_BINARY_DIR}/tmp.resolve_MODULES_END)
+file(REMOVE_RECURSE ${tmpdir})
+file(MAKE_DIRECTORY ${tmpdir})
+file(WRITE ${tmpdir}/driver.c "#include <linux/module.h>\n")
+file(APPEND ${tmpdir}/driver.c "unsigned long MAP_KERNEL_START = MODULES_END - (1UL << 23);\n")
+file(APPEND ${tmpdir}/driver.c "MODULE_LICENSE(\"GPL\");\n")
+file(WRITE ${tmpdir}/Makefile "obj-m := driver.o\n")
+file(APPEND ${tmpdir}/Makefile "all:\n")
+file(APPEND ${tmpdir}/Makefile "\tmake ${KBUILD_MAKE_FLAGS_STR} -C ${KERNEL_DIR} M=${tmpdir} modules\n")
+
+execute_process(COMMAND make -C ${tmpdir})
+execute_process(COMMAND bash -c "offset=`readelf -S ${tmpdir}/driver.ko | grep .data | sed 's/.* //g'`; echo $((0x$offset))"
+  OUTPUT_VARIABLE MAP_KERNEL_START_OFFSET OUTPUT_STRIP_TRAILING_WHITESPACE)
+execute_process(COMMAND bash -c "dd if=${tmpdir}/driver.ko bs=1 skip=${MAP_KERNEL_START_OFFSET} count=8 2>/dev/null | od -tx8 -Ax | head -1 | sed 's|.* |0x|g'"
+  OUTPUT_VARIABLE MAP_KERNEL_START OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+
+set(ENABLE_MEMDUMP ON)
+option(ENABLE_PERF "Enable perf support" ON)
+option(ENABLE_RUSAGE "Enable rusage support" ON)
+option(ENABLE_QLMPI "Enable qlmpi programs" OFF)
+option(ENABLE_UTI "Enable uti support" OFF)
+option(ENABLE_UBSAN "Enable undefined behaviour sanitizer on mckernel size" OFF)
+option(ENABLE_PER_CPU_ALLOC_CACHE "Enable per-CPU allocator cache (ThunderX2 workaround)" OFF)
+
+find_package(PkgConfig REQUIRED)
+set(PKG_CONFIG_USE_CMAKE_PREFIX_PATH ON)
+
+find_library(LIBRT rt)
+if (NOT LIBRT)
+	message(FATAL_ERROR "error: couldn't find librt")
+endif()
+find_library(LIBNUMA numa)
+if (NOT LIBNUMA)
+	message(FATAL_ERROR "error: couldn't find libnuma")
+endif()
+find_library(LIBBFD bfd)
+if (NOT LIBBFD)
+	message(FATAL_ERROR "error: couldn't find libbfd")
+endif()
+find_library(LIBIBERTY iberty)
+if (NOT LIBIBERTY)
+	message(FATAL_ERROR "error: couldn't find libiberty")
+endif()
+
+find_library(LIBDWARF dwarf)
+
+if (NOT LIBDWARF)
+	if (CMAKE_CROSSCOMPILING)
+		message(FATAL_ERROR "Could not find libdwarf.so, install libdwarf-devel to ${CMAKE_FIND_ROOT_PATH}")
+	endif()
+	message("WARNING: libdwarf will be compiled locally")
+	enable_language(CXX)
+else()
+	# Note that libdwarf-devel provides /usr/include/libdwarf/dwarf.h
+	# but elfutils-devel provides /usr/include/dwarf.h
+	# while mcinspect.c performs "#include <dwarf.h>"
+	find_path(DWARF_H dwarf.h PATH_SUFFIXES libdwarf)
+endif()
+
+if (ENABLE_QLMPI)
+	find_package(MPI REQUIRED)
+endif()
+
+string(REGEX REPLACE "^([0-9]+)\\.([0-9]+)\\.([0-9]+)(-([0-9]+)(.*))?" "\\1;\\2;\\3;\\5;\\6" LINUX_VERSION ${UNAME_R})
+list(GET LINUX_VERSION 0 LINUX_VERSION_MAJOR)
+list(GET LINUX_VERSION 1 LINUX_VERSION_MINOR)
+list(GET LINUX_VERSION 2 LINUX_VERSION_PATCH)
+list(GET LINUX_VERSION 3 LINUX_VERSION_RELEASE)
+math(EXPR LINUX_VERSION_CODE "${LINUX_VERSION_MAJOR} * 65536 + ${LINUX_VERSION_MINOR} * 256 + ${LINUX_VERSION_PATCH}")
+
+# compat with various install paths
+set(BINDIR ${CMAKE_INSTALL_FULL_BINDIR})
+set(SBINDIR ${CMAKE_INSTALL_FULL_SBINDIR})
+set(LIBDIR ${CMAKE_INSTALL_FULL_LIBDIR})
+set(ETCDIR ${CMAKE_INSTALL_PREFIX}/etc)
+set(ROOTFSDIR "/rootfs")
+if (CMAKE_INSTALL_PREFIX STREQUAL "/usr")
+	set(KMODDIR "/lib/modules/${UNAME_R}/extra/mckernel")
+	set(MCKERNELDIR "${CMAKE_INSTALL_FULL_DATADIR}/mckernel/${BUILD_TARGET}")
+else()
+	set(KMODDIR "${CMAKE_INSTALL_PREFIX}/kmod")
+	set(MCKERNELDIR "${CMAKE_INSTALL_PREFIX}/${BUILD_TARGET}/kernel")
+endif()
+set(prefix ${CMAKE_INSTALL_PREFIX})
+
+# set rpath for everyone
+set(CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_FULL_LIBDIR})
+
+# ihk: ultimately should support extrnal build, but add as subproject for now
+if (EXISTS ${PROJECT_SOURCE_DIR}/ihk/CMakeLists.txt)
+	set(IHK_SOURCE_DIR "ihk" CACHE STRING "path to ihk source directory from mckernel sources")
+elseif (EXISTS ${PROJECT_SOURCE_DIR}/../ihk/CMakeLists.txt)
+	set(IHK_SOURCE_DIR "../ihk" CACHE STRING "path to ihk source directory from mckernel sources")
+else()
+	set(IHK_SOURCE_DIR "ihk" CACHE STRING "path to ihk source directory from mckernel sources")
+endif()
+if (EXISTS ${PROJECT_SOURCE_DIR}/${IHK_SOURCE_DIR}/CMakeLists.txt)
+	set(IHK_FULL_SOURCE_DIR ${PROJECT_SOURCE_DIR}/${IHK_SOURCE_DIR})
+elseif (EXISTS /${IHK_SOURCE_DIR}/CMakeLists.txt)
+	set(IHK_FULL_SOURCE_DIR /${IHK_SOURCE_DIR})
+else()
+	message(FATAL_ERROR "Could not find ihk dir, or it does not contain CMakeLists.txt, either clone ihk or run git submodule update --init")
+endif()
+
+add_subdirectory(${IHK_SOURCE_DIR} ihk)
+
+configure_file(config.h.in config.h)
+
+# actual build section - just subdirs
+add_subdirectory(executer/kernel/mcctrl)
+add_subdirectory(executer/user)
+add_subdirectory(kernel)
+add_subdirectory(tools/mcstat)
+add_subdirectory(tools/crash)
+
+configure_file(scripts/mcreboot-smp.sh.in mcreboot.sh @ONLY)
+configure_file(scripts/mcstop+release-smp.sh.in mcstop+release.sh @ONLY)
+configure_file(scripts/mcreboot.1in mcreboot.1 @ONLY)
+configure_file(scripts/eclair-dump-backtrace.exp.in eclair-dump-backtrace.exp @ONLY)
+install(PROGRAMS
+		"${CMAKE_CURRENT_BINARY_DIR}/mcreboot.sh"
+		"${CMAKE_CURRENT_BINARY_DIR}/mcstop+release.sh"
+	DESTINATION "${CMAKE_INSTALL_SBINDIR}")
+install(PROGRAMS
+		"${CMAKE_CURRENT_BINARY_DIR}/eclair-dump-backtrace.exp"
+	DESTINATION "${CMAKE_INSTALL_BINDIR}")
+install(FILES "scripts/irqbalance_mck.in"
+	DESTINATION "${CMAKE_INSTALL_SYSCONFDIR}")
+install(FILES "${CMAKE_CURRENT_BINARY_DIR}/mcreboot.1"
+	DESTINATION "${CMAKE_INSTALL_MANDIR}/man1")
+
+
+configure_file(scripts/mckernel.spec.in scripts/mckernel.spec @ONLY)
+set(CPACK_SOURCE_PACKAGE_FILE_NAME "${CMAKE_PROJECT_NAME}-${MCKERNEL_VERSION}")
+set(CPACK_SOURCE_IGNORE_FILES "/.git/;/build;/CMakeCache.txt$;/CMakeFiles$;/Makefile$")
+set(CPACK_SOURCE_INSTALLED_DIRECTORIES "${CMAKE_SOURCE_DIR};/;${IHK_FULL_SOURCE_DIR};/ihk;${CMAKE_BINARY_DIR}/scripts;/scripts")
+set(CPACK_SOURCE_GENERATOR "TGZ")
+include(CPack)
+add_custom_target(dist COMMAND ${CMAKE_MAKE_PROGRAM} package_source)
+
+
+# config report
+message("-------------------------------")
+message("Option summary")
+message("-------------------------------")
+message("Build type: ${CMAKE_BUILD_TYPE}")
+message("Build target: ${BUILD_TARGET}")
+message("IHK_SOURCE_DIR: ${IHK_SOURCE_DIR} (relative to mckernel source tree)")
+message("UNAME_R: ${UNAME_R}")
+message("KERNEL_DIR: ${KERNEL_DIR}")
+message("SYSTEM_MAP: ${SYSTEM_MAP}")
+message("VMLINUX: ${VMLINUX}")
+message("KBUILD_C_FLAGS: ${KBUILD_C_FLAGS}")
+message("MAP_KERNEL_START: ${MAP_KERNEL_START}")
+message("ENABLE_MEMDUMP: ${ENABLE_MEMDUMP}")
+message("ENABLE_PERF: ${ENABLE_PERF}")
+message("ENABLE_TOFU: ${ENABLE_TOFU}")
+message("ENABLE_FUGAKU_HACKS: ${ENABLE_FUGAKU_HACKS}")
+message("ENABLE_FUGAKU_DEBUG: ${ENABLE_FUGAKU_DEBUG}")
+message("ENABLE_KRM_WORKAROUND: ${ENABLE_KRM_WORKAROUND}")
+message("ENABLE_FJMPI_WORKAROUND: ${ENABLE_FJMPI_WORKAROUND}")
+message("PROFILE_ENABLE: ${PROFILE_ENABLE}")
+message("ENABLE_RUSAGE: ${ENABLE_RUSAGE}")
+message("ENABLE_QLMPI: ${ENABLE_QLMPI}")
+message("ENABLE_UTI: ${ENABLE_UTI}")
+message("ENABLE_WERROR: ${ENABLE_WERROR}")
+message("ENABLE_UBSAN: ${ENABLE_UBSAN}")
+message("ENABLE_LINUX_WORK_IRQ_FOR_IKC: ${ENABLE_LINUX_WORK_IRQ_FOR_IKC}")
+message("ENABLE_PER_CPU_ALLOC_CACHE: ${ENABLE_PER_CPU_ALLOC_CACHE}")
+message("ENABLE_KMSG_REDIRECT: ${ENABLE_KMSG_REDIRECT}")
+message("-------------------------------")

KNOWN_BUGS.md

@@ -0,0 +1,70 @@
+Linux crash when offlining CPU (el7, hardware-specific)
+=========================================================
+
+On some hardware with el7 kernel, linux can crash due to a bug in the
+irq handling when offlining CPUs (reserve cpu part of mcreboot)
+
+Example stack trace:
+```
+[ 4147.052753] BUG: unable to handle kernel NULL pointer dereference at 0000000000000040
+[ 4147.060677] IP: [<ffffffff8102ce26>] check_irq_vectors_for_cpu_disable+0x86/0x1c0
+[ 4147.068226] PGD 1057e44067 PUD 105f1e7067 PMD 0
+[ 4147.072935] Oops: 0000 [#1] SMP
+[ 4147.076230] Modules linked in: mcctrl(OE) ihk_smp_x86_64(OE) ihk(OE) xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core
+[ 4147.148619]  dm_mirror dm_region_hash dm_log dm_mod sb_edac edac_core intel_powerclamp coretemp ext4 mbcache jbd2 intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul ipmi_ssif glue_helper ablk_helper joydev iTCO_wdt iTCO_vendor_support cryptd ipmi_si ipmi_devintf ipmi_msghandler pcspkr wmi mei_me mei lpc_ich i2c_i801 sg ioatdma shpchp nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c mlx4_en sd_mod crc_t10dif crct10dif_generic mgag200 drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm isci igb drm mlx4_core libsas ahci libahci scsi_transport_sas libata crct10dif_pclmul ptp crct10dif_common pps_core crc32c_intel dca i2c_algo_bit i2c_core devlink [last unloaded: ihk]
+[ 4147.215370] CPU: 6 PID: 38 Comm: migration/6 Tainted: G           OE  ------------ T 3.10.0-693.2.2.el7.x86_64 #1
+[ 4147.225672] Hardware name: SGI.COM C1104G-RP5/X9DRG-HF, BIOS 3.0  10/25/2013
+[ 4147.232747] task: ffff880174689fa0 ti: ffff8801746ac000 task.ti: ffff8801746ac000
+[ 4147.240278] RIP: 0010:[<ffffffff8102ce26>]  [<ffffffff8102ce26>] check_irq_vectors_for_cpu_disable+0x86/0x1c0
+[ 4147.250275] RSP: 0018:ffff8801746afd30  EFLAGS: 00010046
+[ 4147.255608] RAX: 0000000000000000 RBX: 000000000000004e RCX: 0000000000000000
+[ 4147.262770] RDX: 0000000000000020 RSI: 000000000000005f RDI: 0000000000000023
+[ 4147.269936] RBP: ffff8801746afd58 R08: 0000000000000001 R09: ffff88017f800490
+[ 4147.277103] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000006
+[ 4147.284269] R13: 0000000000000000 R14: ffff88085ca82500 R15: 000000000000005f
+[ 4147.291429] FS:  0000000000000000(0000) GS:ffff88085fb80000(0000) knlGS:0000000000000000
+[ 4147.299556] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
+[ 4147.305326] CR2: 0000000000000040 CR3: 0000001059704000 CR4: 00000000001407e0
+[ 4147.312490] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
+[ 4147.319659] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
+[ 4147.326827] Stack:
+[ 4147.328857]  ffff8808f43078c8 ffff8808f4307850 0000000000000286 ffff8808f4307701
+[ 4147.336384]  0000000000000000 ffff8801746afd70 ffffffff81052a82 0000000200000000
+[ 4147.343915]  ffff8801746afd88 ffffffff81693ca3 0000000000000003 ffff8801746afdc0
+[ 4147.351447] Call Trace:
+[ 4147.353921]  [<ffffffff81052a82>] native_cpu_disable+0x12/0x40
+[ 4147.359795]  [<ffffffff81693ca3>] take_cpu_down+0x13/0x40
+[ 4147.365236]  [<ffffffff81116899>] multi_cpu_stop+0xd9/0x100
+[ 4147.370850]  [<ffffffff811167c0>] ? cpu_stop_should_run+0x50/0x50
+[ 4147.376983]  [<ffffffff81116ab7>] cpu_stopper_thread+0x97/0x150
+[ 4147.382942]  [<ffffffff816a8fad>] ? __schedule+0x39d/0x8b0
+[ 4147.388461]  [<ffffffff810b909f>] smpboot_thread_fn+0x12f/0x180
+[ 4147.394406]  [<ffffffff810b8f70>] ? lg_double_unlock+0x40/0x40
+[ 4147.400276]  [<ffffffff810b098f>] kthread+0xcf/0xe0
+[ 4147.405182]  [<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
+[ 4147.411319]  [<ffffffff816b4f58>] ret_from_fork+0x58/0x90
+[ 4147.418893]  [<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
+[ 4147.426524] Code: 81 fb 00 01 00 00 0f 84 8a 00 00 00 89 d8 65 44 8b 3c 85 20 c6 00 00 45 85 ff 78 e1 44 89 ff e8 91 31 10 00 48 63 15 7e 10 af 00 <48> 8b 70 40 48 c7 c7 80 71 cf 81 49 89 c6 48 83 c2 3f 48 c1 fa
+[ 4147.450352] RIP  [<ffffffff8102ce26>] check_irq_vectors_for_cpu_disable+0x86/0x1c0
+[ 4147.460135]  RSP <ffff8801746afd30>
+[ 4147.465154] CR2: 0000000000000040
+```
+
+This bug has been fixed upstream, but redhat will not backport the fixes.
+You can work around the problem with a kpatch by backporting the three
+following commits:
+
+x86: irq: Get correct available vectors for cpu disable
+https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=ac2a55395eddccd6e3e39532df9869d61e97b2ee
+
+x86/irq: Check for valid irq descriptor in check_irq_vectors_for_cpu_disable()
+https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=d97eb8966c91f2c9d05f0a22eb89ed5b76d966d1
+
+x86/irq: Use proper locking in check_irq_vectors_for_cpu_disable()
+https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=cbb24dc761d95fe39a7a122bb1b298e9604cae15
+
+
+Alternatively, since it is related to the irq configuration, it might
+be possible to mitigate the issue by setting the irq affinities early
+on and making sure none of the cpus that will be offlined have any irq
+configured.

LICENSE

@@ -0,0 +1,339 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+                            NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.

Makefile.in

@@ -1,68 +0,0 @@
-TARGET = @TARGET@
-SBINDIR = @SBINDIR@
-MANDIR = @MANDIR@
-
-all::
-	@(cd executer/kernel; make modules)
-	@(cd executer/user; make)
-	@case "$(TARGET)" in \
-	    attached-mic | builtin-x86 | builtin-mic | smp-x86) \
-		(cd kernel; make) \
-		;; \
-	    *) \
-		echo "unknown target $(TARGET)" >&2 \
-		exit 1 \
-		;; \
-	esac
-
-install::
-	@(cd executer/kernel; make install)
-	@(cd executer/user; make install)
-	@case "$(TARGET)" in \
-	    attached-mic | builtin-x86 | builtin-mic | smp-x86) \
-		(cd kernel; make install) \
-		;; \
-	    *) \
-		echo "unknown target $(TARGET)" >&2 \
-		exit 1 \
-		;; \
-	esac
-	@case "$(TARGET)" in \
-	    attached-mic) \
-		mkdir -p -m 755 $(SBINDIR); \
-		install -m 755 arch/x86/tools/mcreboot-attached-mic.sh $(SBINDIR)/mcreboot; \
-		install -m 755 arch/x86/tools/mcshutdown-attached-mic.sh $(SBINDIR)/mcshutdown; \
-		mkdir -p -m 755 $(MANDIR)/man1; \
-		install -m 644 arch/x86/tools/mcreboot.1 $(MANDIR)/man1/mcreboot.1; \
-		;; \
-	    builtin-x86) \
-		mkdir -p -m 755 $(SBINDIR); \
-		install -m 755 arch/x86/tools/mcreboot-builtin-x86.sh $(SBINDIR)/mcreboot; \
-		install -m 755 arch/x86/tools/mcshutdown-builtin-x86.sh $(SBINDIR)/mcshutdown; \
-		mkdir -p -m 755 $(MANDIR)/man1; \
-		install -m 644 arch/x86/tools/mcreboot.1 $(MANDIR)/man1/mcreboot.1; \
-		;; \
-	    smp-x86) \
-		mkdir -p -m 755 $(SBINDIR); \
-		install -m 755 arch/x86/tools/mcreboot-smp-x86.sh $(SBINDIR)/mcreboot; \
-		mkdir -p -m 755 $(MANDIR)/man1; \
-		install -m 644 arch/x86/tools/mcreboot.1 $(MANDIR)/man1/mcreboot.1; \
-		;; \
-	    *) \
-		echo "unknown target $(TARGET)" >&2 \
-		exit 1 \
-		;; \
-	esac
-
-clean::
-	@(cd executer/kernel; make clean)
-	@(cd executer/user; make clean)
-	@case "$(TARGET)" in \
-	    attached-mic | builtin-x86 | builtin-mic | smp-x86) \
-		(cd kernel; make clean) \
-		;; \
-	    *) \
-		echo "unknown target $(TARGET)" >&2 \
-		exit 1 \
-		;; \
-	esac

README.rst

@@ -0,0 +1,23 @@
+.. figure:: docs/mckernel-logo.png
+
+IHK/McKernel is a light-weight multi-kernel operating system designed
+for high-end supercomputing. It runs Linux and McKernel, a light-weight
+kernel (LWK), side-by-side inside compute nodes and aims at the
+following:
+
+-  Provide scalable and consistent execution of large-scale parallel
+   scientific applications, but at the same time maintain the ability to
+   rapidly adapt to new hardware features and emerging programming
+   models
+-  Provide efficient memory and device management so that resource
+   contention and data movement are minimized at the system level
+-  Eliminate OS noise by isolating OS services in Linux and provide
+   jitter free execution on the LWK
+-  Support the full POSIX/Linux APIs by selectively offloading
+   (slow-path) system calls to Linux
+
+Documentation
+=============
+
+Documentation is available
+`here <https://ihkmckernel.readthedocs.io>`__.

arch/arm64/kernel/Makefile.arch.in

@@ -0,0 +1,27 @@
+# Makefile.arch.in COPYRIGHT FUJITSU LIMITED 2015-2018
+VDSO_SRCDIR = $(SRC)/../arch/$(IHKARCH)/kernel/vdso
+VDSO_BUILDDIR = @abs_builddir@/vdso
+VDSO_SO_O = $(O)/vdso.so.o
+
+IHK_OBJS += assert.o cache.o cpu.o cputable.o context.o entry.o entry-fpsimd.o
+IHK_OBJS += fault.o head.o hyp-stub.o local.o perfctr.o perfctr_armv8pmu.o proc.o proc-macros.o
+IHK_OBJS += psci.o smp.o trampoline.o traps.o fpsimd.o
+IHK_OBJS += debug-monitors.o hw_breakpoint.o ptrace.o timer.o
+IHK_OBJS += $(notdir $(VDSO_SO_O)) memory.o syscall.o vdso.o
+
+IHK_OBJS += irq-gic-v2.o irq-gic-v3.o
+IHK_OBJS += memcpy.o memset.o
+IHK_OBJS += cpufeature.o
+
+IHK_OBJS += imp-sysreg.o
+IHK_OBJS += coredump.o
+
+$(VDSO_SO_O): $(VDSO_BUILDDIR)/vdso.so
+
+$(VDSO_BUILDDIR)/vdso.so: FORCE
+	$(call echo_cmd,BUILD VDSO,$(TARGET))
+	mkdir -p $(O)/vdso
+	TARGETDIR="$(TARGETDIR)" $(submake) -C $(VDSO_BUILDDIR) $(SUBOPTS) prepare
+	TARGETDIR="$(TARGETDIR)" $(submake) -C $(VDSO_BUILDDIR) $(SUBOPTS)
+
+FORCE:

arch/arm64/kernel/assert.c

@@ -0,0 +1,56 @@
+/* assert.c COPYRIGHT FUJITSU LIMITED 2015-2019 */
+
+#include <process.h>
+#include <list.h>
+#include <ihk/debug.h>
+#include <ihk/context.h>
+#include <asm-offsets.h>
+#include <cputable.h>
+#include <thread_info.h>
+#include <smp.h>
+#include <ptrace.h>
+
+/* assert for struct pt_regs member offset & size define */
+STATIC_ASSERT(offsetof(struct pt_regs, regs[0]) == S_X0);
+STATIC_ASSERT(offsetof(struct pt_regs, regs[1]) == S_X1);
+STATIC_ASSERT(offsetof(struct pt_regs, regs[2]) == S_X2);
+STATIC_ASSERT(offsetof(struct pt_regs, regs[3]) == S_X3);
+STATIC_ASSERT(offsetof(struct pt_regs, regs[4]) == S_X4);
+STATIC_ASSERT(offsetof(struct pt_regs, regs[5]) == S_X5);
+STATIC_ASSERT(offsetof(struct pt_regs, regs[6]) == S_X6);
+STATIC_ASSERT(offsetof(struct pt_regs, regs[7]) == S_X7);
+STATIC_ASSERT(offsetof(struct pt_regs, regs[30]) == S_LR);
+STATIC_ASSERT(offsetof(struct pt_regs, sp) == S_SP);
+STATIC_ASSERT(offsetof(struct pt_regs, pc) == S_PC);
+STATIC_ASSERT(offsetof(struct pt_regs, pstate) == S_PSTATE);
+STATIC_ASSERT(offsetof(struct pt_regs, orig_x0) == S_ORIG_X0);
+STATIC_ASSERT(offsetof(struct pt_regs, orig_pc) == S_ORIG_PC);
+STATIC_ASSERT(offsetof(struct pt_regs, syscallno) == S_SYSCALLNO);
+STATIC_ASSERT(sizeof(struct pt_regs) == S_FRAME_SIZE);
+
+/* assert for struct cpu_info member offset & size define */
+STATIC_ASSERT(offsetof(struct cpu_info, cpu_setup) == CPU_INFO_SETUP);
+STATIC_ASSERT(sizeof(struct cpu_info) == CPU_INFO_SZ);
+
+/* assert for struct thread_info member offset define */
+STATIC_ASSERT(offsetof(struct thread_info, flags) == TI_FLAGS);
+STATIC_ASSERT(offsetof(struct thread_info, cpu_context) == TI_CPU_CONTEXT);
+
+/* assert for arch depend kernel stack size and common kernel stack pages */
+STATIC_ASSERT((KERNEL_STACK_SIZE * 2) < (KERNEL_STACK_NR_PAGES * PAGE_SIZE));
+
+/* assert for struct secondary_data member offset define */
+STATIC_ASSERT(offsetof(struct secondary_data, stack) == SECONDARY_DATA_STACK);
+STATIC_ASSERT(offsetof(struct secondary_data, next_pc) == SECONDARY_DATA_NEXT_PC);
+STATIC_ASSERT(offsetof(struct secondary_data, arg) == SECONDARY_DATA_ARG);
+
+/* assert for sve defines */
+/* @ref.impl arch/arm64/kernel/signal.c::BUILD_BUG_ON in the init_user_layout */
+STATIC_ASSERT(sizeof(struct sigcontext) - offsetof(struct sigcontext, __reserved) > ALIGN_UP(sizeof(struct _aarch64_ctx), 16));
+STATIC_ASSERT(sizeof(struct sigcontext) - offsetof(struct sigcontext, __reserved) -
+		ALIGN_UP(sizeof(struct _aarch64_ctx), 16) > sizeof(struct extra_context));
+STATIC_ASSERT(SVE_PT_FPSIMD_OFFSET == sizeof(struct user_sve_header));
+STATIC_ASSERT(SVE_PT_SVE_OFFSET == sizeof(struct user_sve_header));
+
+/* assert for struct arm64_cpu_local_thread member offset define */
+STATIC_ASSERT(offsetof(struct arm64_cpu_local_thread, panic_regs) == 168);

arch/arm64/kernel/cache.S

@@ -0,0 +1,39 @@
+/* cache.S COPYRIGHT FUJITSU LIMITED 2015 */
+
+#include <linkage.h>
+#include "proc-macros.S"
+
+/*
+ *	__inval_cache_range(start, end)
+ *	- start   - start address of region
+ *	- end     - end address of region
+ */
+ENTRY(__inval_cache_range)
+	/* FALLTHROUGH */
+
+/*
+ *	__dma_inv_range(start, end)
+ *	- start   - virtual start address of region
+ *	- end     - virtual end address of region
+ */
+__dma_inv_range:
+	dcache_line_size x2, x3
+	sub	x3, x2, #1
+	tst	x1, x3				// end cache line aligned?
+	bic	x1, x1, x3
+	b.eq	1f
+	dc	civac, x1			// clean & invalidate D / U line
+1:	tst	x0, x3				// start cache line aligned?
+	bic	x0, x0, x3
+	b.eq	2f
+	dc	civac, x0			// clean & invalidate D / U line
+	b	3f
+2:	dc	ivac, x0			// invalidate D / U line
+3:	add	x0, x0, x2
+	cmp	x0, x1
+	b.lo	2b
+	dsb	sy
+	ret
+ENDPROC(__inval_cache_range)
+ENDPROC(__dma_inv_range)
+

arch/arm64/kernel/context.c

@@ -0,0 +1,191 @@
+/* context.c COPYRIGHT FUJITSU LIMITED 2015-2017 */
+#include <ihk/context.h>
+#include <ihk/debug.h>
+#include <thread_info.h>
+#include <cputype.h>
+#include <mmu_context.h>
+#include <arch-memory.h>
+#include <irqflags.h>
+#include <lwk/compiler.h>
+#include <bitops.h>
+
+/* @ref.impl arch/arm64/include/asm/mmu_context.h::MAX_ASID_BITS */
+#define MAX_ASID_BITS		16
+#define ASID_FIRST_VERSION	(1 << MAX_ASID_BITS)
+#define ASID_MASK		((1 << MAX_ASID_BITS) - 1)
+#define VERSION_MASK		(0xFFFF << MAX_ASID_BITS)
+
+/* @ref.impl arch/arm64/mm/context.c::asid_bits */
+#define asid_bits(reg) \
+	(((read_cpuid(ID_AA64MMFR0_EL1) & 0xf0) >> 2) + 8)
+
+#define MAX_CTX_NR		(1UL << MAX_ASID_BITS)
+DECLARE_BITMAP(mmu_context_bmap, MAX_CTX_NR) = { 1 };	/* context number 0 reserved. */
+
+/* cpu_asid lock */
+static ihk_spinlock_t cpu_asid_lock = SPIN_LOCK_UNLOCKED;
+
+/* last allocation ASID, initialized by 0x0001_0000 */
+static unsigned int cpu_last_asid = ASID_FIRST_VERSION;
+
+/* @ref.impl arch/arm64/mm/context.c::set_mm_context */
+/* set asid for kernel_context_t.context */
+static void set_mm_context(struct page_table *pgtbl, unsigned int asid)
+{
+	unsigned int context = get_address_space_id(pgtbl);
+	if (likely((context ^ cpu_last_asid) >> MAX_ASID_BITS)) {
+		set_address_space_id(pgtbl, asid);
+	}
+}
+
+/* @ref.impl arch/arm64/mm/context.c::__new_context */
+/* ASID allocation for new process function */
+static inline void __new_context(struct page_table *pgtbl)
+{
+	unsigned int asid;
+	unsigned int bits = asid_bits();
+	unsigned long flags;
+	unsigned int context = get_address_space_id(pgtbl);
+	unsigned long index = 0;
+
+	flags = ihk_mc_spinlock_lock(&cpu_asid_lock);
+
+	/* already assigned context number? */
+	if (!unlikely((context ^ cpu_last_asid) >> MAX_ASID_BITS)) {
+		/* true, unnecessary assigned context number */
+		ihk_mc_spinlock_unlock(&cpu_asid_lock, flags);
+		return;
+	}
+
+	/* false, necessary assigned context number */
+	/* search from the previous assigned number */
+	index = (cpu_last_asid & ASID_MASK) + 1;
+	asid = find_next_zero_bit(mmu_context_bmap, MAX_CTX_NR, index);
+
+	/* upper limit exceeded */
+	if (asid >= (1 << bits)) {
+		/* re assigned context number, search from 1 */
+		asid = find_next_zero_bit(mmu_context_bmap, index, 1);
+
+		/* upper previous assigned number, goto panic */
+		if (unlikely(asid >= index)) {
+			ihk_mc_spinlock_unlock(&cpu_asid_lock, flags);
+			panic("__new_context(): PANIC: Context Number Depletion.\n");
+		}
+	}
+
+	/* set assigned context number bitmap */
+	mmu_context_bmap[asid >> 6] |= (1UL << (asid & 63));
+
+	/* set previous assigned context number */
+	cpu_last_asid = asid | (cpu_last_asid & VERSION_MASK);
+
+	set_mm_context(pgtbl, cpu_last_asid);
+	ihk_mc_spinlock_unlock(&cpu_asid_lock, flags);
+}
+
+void free_mmu_context(struct page_table *pgtbl)
+{
+	unsigned int context = get_address_space_id(pgtbl);
+	unsigned int nr = context & ASID_MASK;
+	unsigned long flags = ihk_mc_spinlock_lock(&cpu_asid_lock);
+
+	/* clear used context number bitmap */
+	mmu_context_bmap[nr >> 6] &= ~(1UL << (nr & 63));
+	ihk_mc_spinlock_unlock(&cpu_asid_lock, flags);
+}
+
+/* set ttbr0 assembler code extern */
+/* in arch/arm64/kernel/proc.S */
+extern void *cpu_do_switch_mm(translation_table_t* tt_pa, unsigned int asid);
+
+/* @ref.impl arch/arm64/include/asm/mmu_context.h::switch_new_context */
+/* ASID allocation for new process */
+static inline void switch_new_context(struct page_table *pgtbl)
+{
+	unsigned long flags;
+	translation_table_t* tt_pa;
+	unsigned int context;
+
+	/* ASID allocation */
+	__new_context(pgtbl);
+	context = get_address_space_id(pgtbl);
+
+	/* disable interrupt save */
+	flags = cpu_disable_interrupt_save();
+
+	tt_pa = get_translation_table_as_paddr(pgtbl);
+	cpu_do_switch_mm(tt_pa, context & ASID_MASK);
+
+	/* interrupt restore */
+	cpu_restore_interrupt(flags);
+}
+
+/* @ref.impl arch/arm64/include/asm/mmu_context.h::check_and_switch_context */
+/* ASID allocation */
+void switch_mm(struct page_table *pgtbl)
+{
+	unsigned int context = get_address_space_id(pgtbl);
+
+	/* During switch_mm, you want to disable the TTBR */
+	cpu_set_reserved_ttbr0();
+
+	/* check new process or existing process */
+	if (!((context ^ cpu_last_asid) >> MAX_ASID_BITS)) {
+		translation_table_t* tt_pa;
+
+		/* for existing process */
+		tt_pa = get_translation_table_as_paddr(pgtbl);
+		cpu_do_switch_mm(tt_pa, context & ASID_MASK);
+
+/* TODO: tif_switch_mm / after context switch */
+//	} else if (irqs_disabled()) {
+//		/*
+//		 * Defer the new ASID allocation until after the context
+//		 * switch critical region since __new_context() cannot be
+//		 * called with interrupts disabled.
+//		 */
+//		set_ti_thread_flag(task_thread_info(tsk), TIF_SWITCH_MM);
+	} else {
+		/* for new process */
+		/* ASID allocation & set ttbr0 */
+		switch_new_context(pgtbl);
+	}
+}
+
+/* context switch assembler code extern */
+/* in arch/arm64/kernel/entry.S */
+extern void *cpu_switch_to(struct thread_info *prev, struct thread_info *next, void *prev_proc);
+
+/* context switch C function */
+/* TODO: fpreg etc.. save & restore */
+static inline void *switch_to(struct thread_info *prev,
+		       struct thread_info *next,
+		       void *prev_proc)
+{
+	void *last = NULL;
+
+	next->cpu = ihk_mc_get_processor_id();
+	last = cpu_switch_to(prev, next, prev_proc);
+
+	return last;
+}
+
+/* common unit I/F, for context switch */
+void *ihk_mc_switch_context(ihk_mc_kernel_context_t *old_ctx,
+			    ihk_mc_kernel_context_t *new_ctx,
+			    void *prev)
+{
+	struct thread_info *prev_ti = NULL;
+	struct thread_info *next_ti = NULL;
+
+	/* get next thread_info addr */
+	next_ti = new_ctx->thread;
+	if (likely(old_ctx)) {
+		/* get prev thread_info addr */
+		prev_ti = old_ctx->thread;
+	}
+
+	/* switch next thread_info & process */
+	return switch_to(prev_ti, next_ti, prev);
+}

arch/arm64/kernel/copy_template.S

@@ -0,0 +1,194 @@
+/* copy_template.S COPYRIGHT FUJITSU LIMITED 2017 */
+/*
+ * Copyright (C) 2013 ARM Ltd.
+ * Copyright (C) 2013 Linaro.
+ *
+ * This code is based on glibc cortex strings work originally authored by Linaro
+ * and re-licensed under GPLv2 for the Linux kernel. The original code can
+ * be found @
+ *
+ * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/
+ * files/head:/src/aarch64/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+/*
+ * Copy a buffer from src to dest (alignment handled by the hardware)
+ *
+ * Parameters:
+ *	x0 - dest
+ *	x1 - src
+ *	x2 - n
+ * Returns:
+ *	x0 - dest
+ */
+dstin	.req	x0
+src	.req	x1
+count	.req	x2
+tmp1	.req	x3
+tmp1w	.req	w3
+tmp2	.req	x4
+tmp2w	.req	w4
+dst	.req	x6
+
+A_l	.req	x7
+A_h	.req	x8
+B_l	.req	x9
+B_h	.req	x10
+C_l	.req	x11
+C_h	.req	x12
+D_l	.req	x13
+D_h	.req	x14
+
+	mov	dst, dstin
+	cmp	count, #16
+	/*When memory length is less than 16, the accessed are not aligned.*/
+	b.lo	.Ltiny15
+
+	neg	tmp2, src
+	ands	tmp2, tmp2, #15/* Bytes to reach alignment. */
+	b.eq	.LSrcAligned
+	sub	count, count, tmp2
+	/*
+	* Copy the leading memory data from src to dst in an increasing
+	* address order.By this way,the risk of overwritting the source
+	* memory data is eliminated when the distance between src and
+	* dst is less than 16. The memory accesses here are alignment.
+	*/
+	tbz	tmp2, #0, 1f
+	ldrb1	tmp1w, src, #1
+	strb1	tmp1w, dst, #1
+1:
+	tbz	tmp2, #1, 2f
+	ldrh1	tmp1w, src, #2
+	strh1	tmp1w, dst, #2
+2:
+	tbz	tmp2, #2, 3f
+	ldr1	tmp1w, src, #4
+	str1	tmp1w, dst, #4
+3:
+	tbz	tmp2, #3, .LSrcAligned
+	ldr1	tmp1, src, #8
+	str1	tmp1, dst, #8
+
+.LSrcAligned:
+	cmp	count, #64
+	b.ge	.Lcpy_over64
+	/*
+	* Deal with small copies quickly by dropping straight into the
+	* exit block.
+	*/
+.Ltail63:
+	/*
+	* Copy up to 48 bytes of data. At this point we only need the
+	* bottom 6 bits of count to be accurate.
+	*/
+	ands	tmp1, count, #0x30
+	b.eq	.Ltiny15
+	cmp	tmp1w, #0x20
+	b.eq	1f
+	b.lt	2f
+	ldp1	A_l, A_h, src, #16
+	stp1	A_l, A_h, dst, #16
+1:
+	ldp1	A_l, A_h, src, #16
+	stp1	A_l, A_h, dst, #16
+2:
+	ldp1	A_l, A_h, src, #16
+	stp1	A_l, A_h, dst, #16
+.Ltiny15:
+	/*
+	* Prefer to break one ldp/stp into several load/store to access
+	* memory in an increasing address order,rather than to load/store 16
+	* bytes from (src-16) to (dst-16) and to backward the src to aligned
+	* address,which way is used in original cortex memcpy. If keeping
+	* the original memcpy process here, memmove need to satisfy the
+	* precondition that src address is at least 16 bytes bigger than dst
+	* address,otherwise some source data will be overwritten when memove
+	* call memcpy directly. To make memmove simpler and decouple the
+	* memcpy's dependency on memmove, withdrew the original process.
+	*/
+	tbz	count, #3, 1f
+	ldr1	tmp1, src, #8
+	str1	tmp1, dst, #8
+1:
+	tbz	count, #2, 2f
+	ldr1	tmp1w, src, #4
+	str1	tmp1w, dst, #4
+2:
+	tbz	count, #1, 3f
+	ldrh1	tmp1w, src, #2
+	strh1	tmp1w, dst, #2
+3:
+	tbz	count, #0, .Lexitfunc
+	ldrb1	tmp1w, src, #1
+	strb1	tmp1w, dst, #1
+
+	b	.Lexitfunc
+
+.Lcpy_over64:
+	subs	count, count, #128
+	b.ge	.Lcpy_body_large
+	/*
+	* Less than 128 bytes to copy, so handle 64 here and then jump
+	* to the tail.
+	*/
+	ldp1	A_l, A_h, src, #16
+	stp1	A_l, A_h, dst, #16
+	ldp1	B_l, B_h, src, #16
+	ldp1	C_l, C_h, src, #16
+	stp1	B_l, B_h, dst, #16
+	stp1	C_l, C_h, dst, #16
+	ldp1	D_l, D_h, src, #16
+	stp1	D_l, D_h, dst, #16
+
+	tst	count, #0x3f
+	b.ne	.Ltail63
+	b	.Lexitfunc
+
+	/*
+	* Critical loop.  Start at a new cache line boundary.  Assuming
+	* 64 bytes per line this ensures the entire loop is in one line.
+	*/
+	.p2align	L1_CACHE_SHIFT
+.Lcpy_body_large:
+	/* pre-get 64 bytes data. */
+	ldp1	A_l, A_h, src, #16
+	ldp1	B_l, B_h, src, #16
+	ldp1	C_l, C_h, src, #16
+	ldp1	D_l, D_h, src, #16
+1:
+	/*
+	* interlace the load of next 64 bytes data block with store of the last
+	* loaded 64 bytes data.
+	*/
+	stp1	A_l, A_h, dst, #16
+	ldp1	A_l, A_h, src, #16
+	stp1	B_l, B_h, dst, #16
+	ldp1	B_l, B_h, src, #16
+	stp1	C_l, C_h, dst, #16
+	ldp1	C_l, C_h, src, #16
+	stp1	D_l, D_h, dst, #16
+	ldp1	D_l, D_h, src, #16
+	subs	count, count, #64
+	b.ge	1b
+	stp1	A_l, A_h, dst, #16
+	stp1	B_l, B_h, dst, #16
+	stp1	C_l, C_h, dst, #16
+	stp1	D_l, D_h, dst, #16
+
+	tst	count, #0x3f
+	b.ne	.Ltail63
+.Lexitfunc:

arch/arm64/kernel/coredump.c

@@ -0,0 +1,92 @@
+/* coredump.c COPYRIGHT FUJITSU LIMITED 2015-2019 */
+#include <process.h>
+#include <elfcore.h>
+#include <string.h>
+#include <ptrace.h>
+#include <cls.h>
+#include <hwcap.h>
+
+#define	align32(x) ((((x) + 3) / 4) * 4)
+
+void arch_fill_prstatus(struct elf_prstatus64 *prstatus,
+	struct thread *thread, void *regs0, int sig)
+{
+	struct pt_regs *regs = regs0;
+	struct elf_prstatus64 tmp_prstatus;
+/*
+  We ignore following entries for now.
+
+	struct elf_siginfo pr_info;
+	short int pr_cursig;
+	a8_uint64_t pr_sigpend;
+	a8_uint64_t pr_sighold;
+	pid_t pr_pgrp;
+	pid_t pr_sid;
+	struct prstatus64_timeval pr_utime;
+	struct prstatus64_timeval pr_stime;
+	struct prstatus64_timeval pr_cutime;
+	struct prstatus64_timeval pr_cstime;
+ */
+
+	/* copy x0-30, sp, pc, pstate */
+	memcpy(&tmp_prstatus.pr_reg, &regs->user_regs, sizeof(tmp_prstatus.pr_reg));
+	tmp_prstatus.pr_fpvalid = 0;	/* We assume no fp */
+
+	/* copy unaligned prstatus addr */
+	memcpy(prstatus, &tmp_prstatus, sizeof(*prstatus));
+
+	prstatus->pr_pid = thread->tid;
+	if (thread->proc->parent) {
+		prstatus->pr_ppid = thread->proc->parent->pid;
+	}
+
+	prstatus->pr_info.si_signo = sig;
+	prstatus->pr_cursig = sig;
+}
+
+int arch_get_thread_core_info_size(void)
+{
+	const struct user_regset_view *view = current_user_regset_view();
+	const struct user_regset *regset = find_regset(view, NT_ARM_SVE);
+
+	if (unlikely(!(elf_hwcap & HWCAP_SVE))) {
+		return 0;
+	}
+	return sizeof(struct note) + align32(sizeof("LINUX"))
+		+ regset_size(cpu_local_var(current), regset);
+}
+
+void arch_fill_thread_core_info(struct note *head,
+				struct thread *thread, void *regs)
+{
+	const struct user_regset_view *view = current_user_regset_view();
+	const struct user_regset *regset = find_regset(view, NT_ARM_SVE);
+
+	if (unlikely(!(elf_hwcap & HWCAP_SVE))) {
+		return;
+	}
+
+	/* pre saved registers */
+	save_fp_regs(thread);
+
+	if (regset->core_note_type && regset->get &&
+	    (!regset->active || regset->active(thread, regset))) {
+		int ret;
+		size_t size = regset_size(thread, regset);
+		void *namep;
+		void *descp;
+
+		namep = (void *) (head + 1);
+		descp = namep + align32(sizeof("LINUX"));
+
+		ret = regset->get(thread, regset, 0, size, descp, NULL);
+		if (ret) {
+			return;
+		}
+
+		head->namesz = sizeof("LINUX");
+		head->descsz = size;
+		head->type = NT_ARM_SVE;
+		memcpy(namep, "LINUX", sizeof("LINUX"));
+	}
+}

arch/arm64/kernel/cputable.c

@@ -0,0 +1,14 @@
+/* cputable.c COPYRIGHT FUJITSU LIMITED 2015 */
+
+#include <cputable.h>
+
+extern unsigned long __cpu_setup(void);
+struct cpu_info cpu_table[] = {
+	{
+		.cpu_id_val	= 0x000f0000,
+		.cpu_id_mask	= 0x000f0000,
+		.cpu_name	= "AArch64 Processor",
+		.cpu_setup	= __cpu_setup,
+	},
+	{ 0 },
+};

arch/arm64/kernel/debug-monitors.c

@@ -0,0 +1,109 @@
+/* debug-monitors.c COPYRIGHT FUJITSU LIMITED 2016-2017 */
+#include <cputype.h>
+#include <irqflags.h>
+#include <ihk/context.h>
+#include <signal.h>
+#include <errno.h>
+#include <debug-monitors.h>
+#include <cls.h>
+#include <thread_info.h>
+
+/* @ref.impl arch/arm64/kernel/debug-monitors.c::debug_monitors_arch */
+/* Determine debug architecture. */
+unsigned char debug_monitors_arch(void)
+{
+	return read_cpuid(ID_AA64DFR0_EL1) & 0xf;
+}
+
+/* @ref.impl arch/arm64/kernel/debug-monitors.c::mdscr_write */
+void mdscr_write(unsigned int mdscr)
+{
+	unsigned long flags = local_dbg_save();
+	asm volatile("msr mdscr_el1, %0" :: "r" (mdscr));
+	local_dbg_restore(flags);
+}
+
+/* @ref.impl arch/arm64/kernel/debug-monitors.c::mdscr_read */
+unsigned int mdscr_read(void)
+{
+	unsigned int mdscr;
+	asm volatile("mrs %0, mdscr_el1" : "=r" (mdscr));
+	return mdscr;
+}
+
+/* @ref.impl arch/arm64/kernel/debug-monitors.c::clear_os_lock */
+static void clear_os_lock(void)
+{
+	asm volatile("msr oslar_el1, %0" : : "r" (0));
+}
+
+/* @ref.impl arch/arm64/kernel/debug-monitors.c::debug_monitors_init */
+void debug_monitors_init(void)
+{
+	clear_os_lock();
+}
+
+/* @ref.impl arch/arm64/kernel/debug-monitors.c::set_regs_spsr_ss */
+void set_regs_spsr_ss(struct pt_regs *regs)
+{
+	unsigned long spsr;
+
+	spsr = regs->pstate;
+	spsr &= ~DBG_SPSR_SS;
+	spsr |= DBG_SPSR_SS;
+	regs->pstate = spsr;
+}
+
+/* @ref.impl arch/arm64/kernel/debug-monitors.c::set_regs_spsr_ss */
+void clear_regs_spsr_ss(struct pt_regs *regs)
+{
+	unsigned long spsr;
+
+	spsr = regs->pstate;
+	spsr &= ~DBG_SPSR_SS;
+	regs->pstate = spsr;
+}
+
+extern int interrupt_from_user(void *);
+extern void clear_single_step(struct thread *thread);
+
+/* @ref.impl arch/arm64/kernel/debug-monitors.c::single_step_handler */
+int single_step_handler(unsigned long addr, unsigned int esr, struct pt_regs *regs)
+{
+	siginfo_t info;
+	int ret = -EFAULT;
+
+	if (interrupt_from_user(regs)) {
+		info.si_signo = SIGTRAP;
+		info.si_errno = 0;
+		info.si_code  = TRAP_HWBKPT;
+		info._sifields._sigfault.si_addr = (void *)regs->pc;
+		set_signal(SIGTRAP, regs, &info);
+		clear_single_step(cpu_local_var(current));
+
+		ret = 0;
+	} else {
+		kprintf("Unexpected kernel single-step exception at EL1\n");
+	}
+	return ret;
+}
+
+/* @ref.impl arch/arm64/kernel/debug-monitors.c::brk_handler */
+int brk_handler(unsigned long addr, unsigned int esr, struct pt_regs *regs)
+{
+	siginfo_t info;
+	int ret = -EFAULT;
+
+	if (interrupt_from_user(regs)) {
+		info.si_signo = SIGTRAP;
+		info.si_errno = 0;
+		info.si_code  = TRAP_BRKPT;
+		info._sifields._sigfault.si_addr = (void *)regs->pc;
+		set_signal(SIGTRAP, regs, &info);
+
+		ret = 0;
+	} else {
+		kprintf("Unexpected kernel BRK exception at EL1\n");
+	}
+	return ret;
+}

arch/arm64/kernel/entry-fpsimd.S

@@ -0,0 +1,126 @@
+/* entry-fpsimd.S COPYRIGHT FUJITSU LIMITED 2015-2017 */
+
+#include <linkage.h>
+#include <assembler.h>
+#include <fpsimdmacros.h>
+
+/*
+ * @ref.impl linux-linaro/arch/arm64/include/asm/fpsimdmacros.h
+ */
+/*
+ * FP/SIMD state saving and restoring macros
+ *
+ * Copyright (C) 2012 ARM Ltd.
+ * Author: Catalin Marinas <catalin.marinas@arm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+.macro fpsimd_save state, tmpnr
+	stp	q0, q1, [\state, #16 * 0]
+	stp	q2, q3, [\state, #16 * 2]
+	stp	q4, q5, [\state, #16 * 4]
+	stp	q6, q7, [\state, #16 * 6]
+	stp	q8, q9, [\state, #16 * 8]
+	stp	q10, q11, [\state, #16 * 10]
+	stp	q12, q13, [\state, #16 * 12]
+	stp	q14, q15, [\state, #16 * 14]
+	stp	q16, q17, [\state, #16 * 16]
+	stp	q18, q19, [\state, #16 * 18]
+	stp	q20, q21, [\state, #16 * 20]
+	stp	q22, q23, [\state, #16 * 22]
+	stp	q24, q25, [\state, #16 * 24]
+	stp	q26, q27, [\state, #16 * 26]
+	stp	q28, q29, [\state, #16 * 28]
+	stp	q30, q31, [\state, #16 * 30]!
+	mrs	x\tmpnr, fpsr
+	str	w\tmpnr, [\state, #16 * 2]
+	mrs	x\tmpnr, fpcr
+	str	w\tmpnr, [\state, #16 * 2 + 4]
+.endm
+
+.macro fpsimd_restore_fpcr state, tmp
+	/*
+	 * Writes to fpcr may be self-synchronising, so avoid restoring
+	 * the register if it hasn't changed.
+	 */
+	mrs	\tmp, fpcr
+	cmp	\tmp, \state
+	b.eq	9999f
+	msr	fpcr, \state
+9999:
+.endm
+
+/* Clobbers \state */
+.macro fpsimd_restore state, tmpnr
+	ldp	q0, q1, [\state, #16 * 0]
+	ldp	q2, q3, [\state, #16 * 2]
+	ldp	q4, q5, [\state, #16 * 4]
+	ldp	q6, q7, [\state, #16 * 6]
+	ldp	q8, q9, [\state, #16 * 8]
+	ldp	q10, q11, [\state, #16 * 10]
+	ldp	q12, q13, [\state, #16 * 12]
+	ldp	q14, q15, [\state, #16 * 14]
+	ldp	q16, q17, [\state, #16 * 16]
+	ldp	q18, q19, [\state, #16 * 18]
+	ldp	q20, q21, [\state, #16 * 20]
+	ldp	q22, q23, [\state, #16 * 22]
+	ldp	q24, q25, [\state, #16 * 24]
+	ldp	q26, q27, [\state, #16 * 26]
+	ldp	q28, q29, [\state, #16 * 28]
+	ldp	q30, q31, [\state, #16 * 30]!
+	ldr	w\tmpnr, [\state, #16 * 2]
+	msr	fpsr, x\tmpnr
+	ldr	w\tmpnr, [\state, #16 * 2 + 4]
+	fpsimd_restore_fpcr x\tmpnr, \state
+.endm
+
+/*
+ * @ref.impl linux-linaro/arch/arm64/kernel/entry-fpsimd.S
+ */
+/*
+ * Save the FP registers.
+ *
+ * x0 - pointer to struct fpsimd_state
+ */
+ENTRY(fpsimd_save_state)
+	fpsimd_save x0, 8
+	ret
+ENDPROC(fpsimd_save_state)
+
+/*
+ * Load the FP registers.
+ *
+ * x0 - pointer to struct fpsimd_state
+ */
+ENTRY(fpsimd_load_state)
+	fpsimd_restore x0, 8
+	ret
+ENDPROC(fpsimd_load_state)
+
+#ifdef CONFIG_ARM64_SVE
+ENTRY(sve_save_state)
+	sve_save 0, x1, 2
+	ret
+ENDPROC(sve_save_state)
+
+ENTRY(sve_load_state)
+	sve_load 0, x1, x2, 3
+	ret
+ENDPROC(sve_load_state)
+
+ENTRY(sve_get_vl)
+	_zrdvl	0, 1
+	ret
+ENDPROC(sve_get_vl)
+#endif /* CONFIG_ARM64_SVE */

arch/arm64/kernel/entry.S

@@ -0,0 +1,563 @@
+/* entry.S COPYRIGHT FUJITSU LIMITED 2015-2018 */
+
+#include <linkage.h>
+#include <assembler.h>
+#include <asm-offsets.h>
+#include <esr.h>
+#include <thread_info.h>
+#include <asm-syscall.h>
+
+/*
+ * Bad Abort numbers
+ *-----------------
+ */
+#define BAD_SYNC	0
+#define BAD_IRQ		1
+#define BAD_FIQ		2
+#define BAD_ERROR	3
+
+	.macro	kernel_entry, el, regsize = 64
+	sub	sp, sp, #S_FRAME_SIZE
+	.if	\regsize == 32
+	mov	w0, w0				// zero upper 32 bits of x0
+	.endif
+	stp	x0, x1, [sp, #16 * 0]
+	stp	x2, x3, [sp, #16 * 1]
+	stp	x4, x5, [sp, #16 * 2]
+	stp	x6, x7, [sp, #16 * 3]
+	stp	x8, x9, [sp, #16 * 4]
+	stp	x10, x11, [sp, #16 * 5]
+	stp	x12, x13, [sp, #16 * 6]
+	stp	x14, x15, [sp, #16 * 7]
+	stp	x16, x17, [sp, #16 * 8]
+	stp	x18, x19, [sp, #16 * 9]
+	stp	x20, x21, [sp, #16 * 10]
+	stp	x22, x23, [sp, #16 * 11]
+	stp	x24, x25, [sp, #16 * 12]
+	stp	x26, x27, [sp, #16 * 13]
+	stp	x28, x29, [sp, #16 * 14]
+
+	.if	\el == 0
+	mrs	x21, sp_el0
+	get_thread_info tsk			// Ensure MDSCR_EL1.SS is clear,
+	ldr	x19, [tsk, #TI_FLAGS]		// since we can unmask debug
+	disable_step_tsk x19, x20		// exceptions when scheduling.
+	.else
+	add	x21, sp, #S_FRAME_SIZE
+	.endif
+	mrs	x22, elr_el1
+	mrs	x23, spsr_el1
+#if defined(CONFIG_HAS_NMI)
+	mrs_s	x20, ICC_PMR_EL1		// Get PMR
+	and	x20, x20, #ICC_PMR_EL1_G_BIT	// Extract mask bit
+	lsl	x20, x20, #PSR_G_PMR_G_SHIFT	// Shift to a PSTATE RES0 bit
+	eor	x20, x20, #PSR_G_BIT		// Invert bit
+	orr	x23, x20, x23		 	// Store PMR within PSTATE
+	mov	x20, #ICC_PMR_EL1_MASKED
+	msr_s	ICC_PMR_EL1, x20		// Mask normal interrupts at PMR
+#endif /* defined(CONFIG_HAS_NMI) */
+	stp	lr, x21, [sp, #S_LR]
+	stp	x22, x23, [sp, #S_PC]
+
+	/*
+	 * Set syscallno to -1 by default (overridden later if real syscall).
+	 */
+	.if	\el == 0
+	mvn	x21, xzr
+	str	x21, [sp, #S_SYSCALLNO]
+	.endif
+
+	/*
+	 * Registers that may be useful after this macro is invoked:
+	 *
+	 * x21 - aborted SP
+	 * x22 - aborted PC
+	 * x23 - aborted PSTATE
+	*/
+	.endm
+
+	.macro	kernel_exit, el, need_enable_step = 0
+	.if 	\el == 0
+	bl	check_sig_pending
+	bl	check_need_resched		// or reschedule is needed.
+	mov	x0, #0
+	mov	x1, sp
+	mov	x2, #0
+	bl	check_signal			// check whether the signal is delivered
+	mov	x0, #0
+	mov	x1, sp
+	mov	x2, #0
+	bl	check_signal_irq_disabled	// check whether the signal is delivered(for kernel_exit)
+	.endif
+	disable_irq x1				// disable interrupts
+	.if	\need_enable_step == 1
+	ldr	x1, [tsk, #TI_FLAGS]
+	enable_step_tsk x1, x2
+	.endif
+	disable_nmi
+	ldp	x21, x22, [sp, #S_PC]		// load ELR, SPSR
+	.if	\el == 0
+//	ct_user_enter				// McKernel, disable (debugcode?)
+	ldr	x23, [sp, #S_SP]		// load return stack pointer
+	msr	sp_el0, x23
+	.endif
+#if defined(CONFIG_HAS_NMI)
+	and	x20, x22, #PSR_G_BIT		// Get stolen PSTATE bit
+	and	x22, x22, #~PSR_G_BIT		// Clear stolen bit
+	lsr	x20, x20, #PSR_G_PMR_G_SHIFT	// Shift back to PMR mask
+	eor	x20, x20, #ICC_PMR_EL1_UNMASKED	// x20 gets 0xf0 or 0xb0
+	msr_s	ICC_PMR_EL1, x20		// Write to PMR
+#endif /* defined(CONFIG_HAS_NMI) */
+	msr	elr_el1, x21			// set up the return data
+	msr	spsr_el1, x22
+	ldp	x0, x1, [sp, #16 * 0]
+	ldp	x2, x3, [sp, #16 * 1]
+	ldp	x4, x5, [sp, #16 * 2]
+	ldp	x6, x7, [sp, #16 * 3]
+	ldp	x8, x9, [sp, #16 * 4]
+	ldp	x10, x11, [sp, #16 * 5]
+	ldp	x12, x13, [sp, #16 * 6]
+	ldp	x14, x15, [sp, #16 * 7]
+	ldp	x16, x17, [sp, #16 * 8]
+	ldp	x18, x19, [sp, #16 * 9]
+	ldp	x20, x21, [sp, #16 * 10]
+	ldp	x22, x23, [sp, #16 * 11]
+	ldp	x24, x25, [sp, #16 * 12]
+	ldp	x26, x27, [sp, #16 * 13]
+	ldp	x28, x29, [sp, #16 * 14]
+	ldr	lr, [sp, #S_LR]
+	add	sp, sp, #S_FRAME_SIZE		// restore sp
+	eret					// return to kernel
+	.endm
+
+	.macro	get_thread_info, rd
+	mov	\rd, sp
+	and	\rd, \rd, #~(KERNEL_STACK_SIZE - 1)	// top of stack
+	.endm
+
+/*
+ * These are the registers used in the syscall handler, and allow us to
+ * have in theory up to 7 arguments to a function - x0 to x6.
+ *
+ * x7 is reserved for the system call number in 32-bit mode.
+ */
+sc_nr	.req	x25		// number of system calls
+scno	.req	x26		// syscall number
+stbl	.req	x27		// syscall table pointer
+tsk	.req	x28		// current thread_info
+
+/*
+ * Interrupt handling.
+ */
+	.macro	irq_handler
+	adrp	x1, handle_arch_irq
+	ldr	x1, [x1, #:lo12:handle_arch_irq]
+	mov	x0, sp
+	blr	x1
+	.endm
+
+	.text
+
+/*
+ * Exception vectors.
+ */
+
+	.align	11
+ENTRY(vectors)
+	ventry	el1_sync_invalid		// Synchronous EL1t
+	ventry	el1_irq_invalid			// IRQ EL1t
+	ventry	el1_fiq_invalid			// FIQ EL1t
+	ventry	el1_error_invalid		// Error EL1t
+
+	ventry	el1_sync			// Synchronous EL1h
+	ventry	el1_irq				// IRQ EL1h
+	ventry	el1_fiq_invalid			// FIQ EL1h
+	ventry	el1_error_invalid		// Error EL1h
+
+	ventry	el0_sync			// Synchronous 64-bit EL0
+	ventry	el0_irq				// IRQ 64-bit EL0
+	ventry	el0_fiq_invalid			// FIQ 64-bit EL0
+	ventry	el0_error_invalid		// Error 64-bit EL0
+
+	ventry	el0_sync_invalid		// Synchronous 32-bit EL0
+	ventry	el0_irq_invalid			// IRQ 32-bit EL0
+	ventry	el0_fiq_invalid			// FIQ 32-bit EL0
+	ventry	el0_error_invalid		// Error 32-bit EL0
+END(vectors)
+
+/*
+ * Invalid mode handlers
+ */
+	.macro	inv_entry, el, reason, regsize = 64
+	kernel_entry el, \regsize
+	mov	x0, sp
+	mov	x1, #\reason
+	mrs	x2, esr_el1
+	enable_nmi
+	.if	\el == 0
+	bl	bad_mode
+	b	ret_to_user
+	.else
+	b	bad_mode
+	.endif
+	.endm
+
+el0_sync_invalid:
+	inv_entry 0, BAD_SYNC
+ENDPROC(el0_sync_invalid)
+
+el0_irq_invalid:
+	inv_entry 0, BAD_IRQ
+ENDPROC(el0_irq_invalid)
+
+el0_fiq_invalid:
+	inv_entry 0, BAD_FIQ
+ENDPROC(el0_fiq_invalid)
+
+el0_error_invalid:
+	inv_entry 0, BAD_ERROR
+ENDPROC(el0_error_invalid)
+
+el1_sync_invalid:
+	inv_entry 1, BAD_SYNC
+ENDPROC(el1_sync_invalid)
+
+el1_irq_invalid:
+	inv_entry 1, BAD_IRQ
+ENDPROC(el1_irq_invalid)
+
+el1_fiq_invalid:
+	inv_entry 1, BAD_FIQ
+ENDPROC(el1_fiq_invalid)
+
+el1_error_invalid:
+	inv_entry 1, BAD_ERROR
+ENDPROC(el1_error_invalid)
+
+/*
+ * EL1 mode handlers.
+ */
+	.align	6
+el1_sync:
+	kernel_entry 1
+	mrs	x1, esr_el1			// read the syndrome register
+	lsr	x24, x1, #ESR_ELx_EC_SHIFT	// exception class
+	cmp	x24, #ESR_ELx_EC_DABT_CUR	// data abort in EL1
+	b.eq	el1_da
+//	cmp	x24, #ESR_ELx_EC_IABT_CUR	// instruction abort in EL1
+//	b.eq	el1_ia
+	cmp	x24, #ESR_ELx_EC_SYS64		// configurable trap
+	b.eq	el1_undef
+	cmp	x24, #ESR_ELx_EC_SP_ALIGN	// stack alignment exception
+	b.eq	el1_sp_pc
+	cmp	x24, #ESR_ELx_EC_PC_ALIGN	// pc alignment exception
+	b.eq	el1_sp_pc
+	cmp	x24, #ESR_ELx_EC_UNKNOWN	// unknown exception in EL1
+	b.eq	el1_undef
+	// cmp	x24, #ESR_ELx_EC_BREAKPT_CUR	// debug exception in EL1
+	// b.ge	el1_dbg
+	b	el1_inv
+el1_ia:
+	/*
+	 * Fall through to the Data abort case
+	 */
+el1_da:
+	/*
+	 * Data abort handling
+	 */
+	mrs	x0, far_el1
+	enable_nmi
+	enable_dbg
+#if defined(CONFIG_HAS_NMI)
+# define PSR_INTR_SHIFT PSR_G_SHIFT  // PSR_G_BIT
+#else /* defined(CONFIG_HAS_NMI) */
+# define PSR_INTR_SHIFT 7            // PSR_I_BIT
+#endif /* defined(CONFIG_HAS_NMI) */
+	// re-enable interrupts if they were enabled in the aborted context
+	tbnz	x23, #PSR_INTR_SHIFT, 1f
+	enable_irq	x2
+1:
+	mov	x2, sp				// struct pt_regs
+	bl	do_mem_abort
+
+	// disable interrupts before pulling preserved data off the stack
+	kernel_exit 1
+
+el1_sp_pc:
+	/*
+	 * Stack or PC alignment exception handling
+	 */
+	mrs	x0, far_el1
+	enable_nmi
+	enable_dbg
+	mov	x2, sp
+	b	do_sp_pc_abort
+el1_undef:
+	/*
+	 * Undefined instruction
+	 */
+	enable_nmi
+	enable_dbg
+	mov	x0, sp
+	b	do_undefinstr
+// el1_dbg:
+// 	/*
+// 	 * Debug exception handling
+// 	 */
+// 	cmp	x24, #ESR_ELx_EC_BRK64		// if BRK64
+// 	cinc	x24, x24, eq			// set bit '0'
+// 	tbz	x24, #0, el1_inv		// EL1 only
+// 	mrs	x0, far_el1
+// 	mov	x2, sp				// struct pt_regs
+// 	bl	do_debug_exception
+// 	kernel_exit 1
+el1_inv:
+	// TODO: add support for undefined instructions in kernel mode
+	mov	x0, sp
+	mov	x1, #BAD_SYNC
+	mrs	x2, esr_el1
+	enable_nmi
+	enable_dbg
+	b	bad_mode
+ENDPROC(el1_sync)
+
+/*
+ * EL1 mode handlers.
+ */
+	.align	6
+el1_irq:
+	kernel_entry 1
+	enable_dbg
+
+	irq_handler
+
+	kernel_exit 1
+ENDPROC(el1_irq)
+
+/*
+ * EL0 mode handlers.
+ */
+	.align	6
+el0_sync:
+	kernel_entry 0
+	mrs	x25, esr_el1			// read the syndrome register
+	lsr	x24, x25, #ESR_ELx_EC_SHIFT	// exception class
+	cmp	x24, #ESR_ELx_EC_SVC64		// SVC in 64-bit state
+	b.eq	el0_svc
+	cmp	x24, #ESR_ELx_EC_DABT_LOW	// data abort in EL0
+	b.eq	el0_da
+	cmp	x24, #ESR_ELx_EC_IABT_LOW	// instruction abort in EL0
+	b.eq	el0_ia
+	cmp	x24, #ESR_ELx_EC_FP_ASIMD	// FP/ASIMD access
+	b.eq	el0_fpsimd_acc
+#ifdef CONFIG_ARM64_SVE
+	cmp	x24, #ESR_ELx_EC_SVE		// SVE access
+	b.eq	el0_sve_acc
+#endif
+	cmp	x24, #ESR_ELx_EC_FP_EXC64	// FP/ASIMD exception
+	b.eq	el0_fpsimd_exc
+	cmp	x24, #ESR_ELx_EC_SYS64		// configurable trap
+	b.eq	el0_undef
+	cmp	x24, #ESR_ELx_EC_SP_ALIGN	// stack alignment exception
+	b.eq	el0_sp_pc
+	cmp	x24, #ESR_ELx_EC_PC_ALIGN	// pc alignment exception
+	b.eq	el0_sp_pc
+	cmp	x24, #ESR_ELx_EC_UNKNOWN	// unknown exception in EL0
+	b.eq	el0_undef
+	cmp	x24, #ESR_ELx_EC_BREAKPT_LOW	// debug exception in EL0
+	b.ge	el0_dbg
+	b	el0_inv
+el0_svc:
+	uxtw	scno, w8			// syscall number in w8
+	cmp	scno, #__NR_rt_sigreturn
+	b.eq	1f
+	str	x0, [sp, #S_ORIG_X0]		// save the original x0
+	ldr	x16, [sp, #S_PC]
+	str	x16, [sp, #S_ORIG_PC]		// save the original pc
+1:	str	scno, [sp, #S_SYSCALLNO]	// save syscall number
+	enable_nmi
+	enable_dbg_and_irq x0
+	adrp	x16, __arm64_syscall_handler
+	ldr	x16, [x16, #:lo12:__arm64_syscall_handler]
+	mov	x0, scno
+	mov	x1, sp
+	blr	x16				// __arm64_syscall_handler(int, syscall_num, ihk_mc_user_context_t *uctx);
+	/* Signal check has been completed at the stage of came back. */
+	b	ret_fast_syscall
+el0_da:
+	/*
+	 * Data abort handling
+	 */
+	mrs	x26, far_el1
+	// enable interrupts before calling the main handler
+	enable_nmi
+	enable_dbg_and_irq x0
+	// ct_user_exit
+	bic	x0, x26, #(0xff << 56)
+	mov	x1, x25
+	mov	x2, sp
+	bl	do_mem_abort
+	b	ret_to_user
+el0_ia:
+	/*
+	 * Instruction abort handling
+	 */
+	mrs	x26, far_el1
+	// enable interrupts before calling the main handler
+	enable_nmi
+	enable_dbg_and_irq x0
+	// ct_user_exit
+	mov	x0, x26
+	mov	x1, x25
+	mov	x2, sp
+	bl	do_mem_abort
+	b	ret_to_user
+el0_fpsimd_acc:
+	/*
+	 * Floating Point or Advanced SIMD access
+	 */
+	enable_nmi
+	enable_dbg
+	// ct_user_exit
+	mov	x0, x25
+	mov	x1, sp
+	bl	do_fpsimd_acc
+	b	ret_to_user
+#ifdef CONFIG_ARM64_SVE
+	/*
+	 * Scalable Vector Extension access
+	 */
+el0_sve_acc:
+	enable_nmi
+	enable_dbg
+	// ct_user_exit
+	mov	x0, x25
+	mov	x1, sp
+	bl	do_sve_acc
+	b	ret_to_user
+#endif
+el0_fpsimd_exc:
+	/*
+	 * Floating Point, Advanced SIMD or SVE exception
+	 */
+	enable_nmi
+	enable_dbg
+	// ct_user_exit
+	mov	x0, x25
+	mov	x1, sp
+	bl	do_fpsimd_exc
+	b	ret_to_user
+el0_sp_pc:
+	/*
+	 * Stack or PC alignment exception handling
+	 */
+	mrs	x26, far_el1
+	// enable interrupts before calling the main handler
+	enable_nmi
+	enable_dbg_and_irq x0
+	mov	x0, x26
+	mov	x1, x25
+	mov	x2, sp
+	bl	do_sp_pc_abort
+	b	ret_to_user
+el0_undef:
+	/*
+	 * Undefined instruction
+	 */
+	// enable interrupts before calling the main handler
+	enable_nmi
+	enable_dbg_and_irq x0
+	// ct_user_exit
+	mov	x0, sp
+	bl	do_undefinstr
+	b	ret_to_user
+el0_dbg:
+ 	/*
+ 	 * Debug exception handling
+ 	 */
+ 	tbnz	x24, #0, el0_inv		// EL0 only
+ 	mrs	x0, far_el1
+ 	mov	x1, x25
+ 	mov	x2, sp
+	enable_nmi
+ 	bl	do_debug_exception
+ 	enable_dbg
+ 	// ct_user_exit
+ 	b	ret_to_user
+el0_inv:
+	enable_dbg
+	mov	x0, sp
+	mov	x1, #BAD_SYNC
+	mrs	x2, esr_el1
+	enable_nmi
+	bl	bad_mode
+	b	ret_to_user
+ENDPROC(el0_sync)
+	.align	6
+el0_irq:
+	kernel_entry 0
+	enable_dbg
+	irq_handler
+	b	ret_to_user
+ENDPROC(el0_irq)
+
+/*
+ * Register switch for AArch64. The callee-saved registers need to be saved
+ * and restored. On entry:
+ *   x0 = previous task_struct (must be preserved across the switch)
+ *   x1 = next task_struct
+ * Previous and next are guaranteed not to be the same.
+ *
+ */
+ENTRY(cpu_switch_to)
+	cmp	x0, xzr				// for idle process branch(skip save)
+	b.eq	1f
+	add	x8, x0, #TI_CPU_CONTEXT
+	mov	x9, sp
+	stp	x19, x20, [x8], #16		// store callee-saved registers
+	stp	x21, x22, [x8], #16
+	stp	x23, x24, [x8], #16
+	stp	x25, x26, [x8], #16
+	stp	x27, x28, [x8], #16
+	stp	x29, x9, [x8], #16
+	str	lr, [x8]
+1:	add	x8, x1, #TI_CPU_CONTEXT
+	ldp	x19, x20, [x8], #16		// restore callee-saved registers
+	ldp	x21, x22, [x8], #16
+	ldp	x23, x24, [x8], #16
+	ldp	x25, x26, [x8], #16
+	ldp	x27, x28, [x8], #16
+	ldp	x29, x9, [x8], #16
+	ldr	lr, [x8]
+	mov	sp, x9
+	mov	x0, x2				// return void *prev
+	ret
+ENDPROC(cpu_switch_to)
+
+
+ret_fast_syscall:
+	kernel_exit 0, 1
+ENDPROC(ret_fast_syscall)
+
+/*
+ * "slow" syscall return path.
+ */
+ret_to_user:
+no_work_pending:
+	kernel_exit 0, 1
+ENDPROC(ret_to_user)
+
+/*
+ * This is how we return from a fork.
+ */
+ENTRY(ret_from_fork)
+//	bl	schedule_tail
+	cbz	x19, 1f				// not a kernel thread
+	mov	x0, x20
+	blr	x19
+1:	get_thread_info tsk
+	bl	release_runq_lock
+	bl	utilthr_migrate
+	b	ret_to_user
+ENDPROC(ret_from_fork)
+

arch/arm64/kernel/fault.c

@@ -0,0 +1,300 @@
+/* fault.c COPYRIGHT FUJITSU LIMITED 2015-2018 */
+
+#include <ihk/context.h>
+#include <ihk/debug.h>
+#include <ptrace.h>
+#include <esr.h>
+#include <signal.h>
+#include <arch-memory.h>
+#include <thread_info.h>
+#include <syscall.h>
+#include <debug-monitors.h>
+
+unsigned long __page_fault_handler_address;
+extern int interrupt_from_user(void *);
+
+static void do_bad_area(unsigned long addr, unsigned int esr, struct pt_regs *regs);
+static int do_page_fault(unsigned long addr, unsigned int esr, struct pt_regs *regs);
+static int do_translation_fault(unsigned long addr, unsigned int esr, struct pt_regs *regs);
+static int do_bad(unsigned long addr, unsigned int esr, struct pt_regs *regs);
+static int do_alignment_fault(unsigned long addr, unsigned int esr, struct pt_regs *regs);
+
+static struct fault_info {
+	int	(*fn)(unsigned long addr, unsigned int esr, struct pt_regs *regs);
+	int	sig;
+	int	code;
+	const char *name;
+} fault_info[] = {
+	{ do_bad,		SIGBUS,  0,		"ttbr address size fault"	},
+	{ do_bad,		SIGBUS,  0,		"level 1 address size fault"	},
+	{ do_bad,		SIGBUS,  0,		"level 2 address size fault"	},
+	{ do_bad,		SIGBUS,  0,		"level 3 address size fault"	},
+	{ do_translation_fault,	SIGSEGV, SEGV_MAPERR,	"level 0 translation fault"	},
+	{ do_translation_fault,	SIGSEGV, SEGV_MAPERR,	"level 1 translation fault"	},
+	{ do_translation_fault,	SIGSEGV, SEGV_MAPERR,	"level 2 translation fault"	},
+	{ do_page_fault,	SIGSEGV, SEGV_MAPERR,	"level 3 translation fault"	},
+	{ do_bad,		SIGBUS,  0,		"unknown 8"			},
+	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 1 access flag fault"	},
+	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 2 access flag fault"	},
+	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 3 access flag fault"	},
+	{ do_bad,		SIGBUS,  0,		"unknown 12"			},
+	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 1 permission fault"	},
+	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 2 permission fault"	},
+	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 3 permission fault"	},
+	{ do_bad,		SIGBUS,  0,		"synchronous external abort"	},
+	{ do_bad,		SIGBUS,  0,		"unknown 17"			},
+	{ do_bad,		SIGBUS,  0,		"unknown 18"			},
+	{ do_bad,		SIGBUS,  0,		"unknown 19"			},
+	{ do_bad,		SIGBUS,  0,		"synchronous external abort (translation table walk)" },
+	{ do_bad,		SIGBUS,  0,		"synchronous external abort (translation table walk)" },
+	{ do_bad,		SIGBUS,  0,		"synchronous external abort (translation table walk)" },
+	{ do_bad,		SIGBUS,  0,		"synchronous external abort (translation table walk)" },
+	{ do_bad,		SIGBUS,  0,		"synchronous parity error"	},
+	{ do_bad,		SIGBUS,  0,		"unknown 25"			},
+	{ do_bad,		SIGBUS,  0,		"unknown 26"			},
+	{ do_bad,		SIGBUS,  0,		"unknown 27"			},
+	{ do_bad,		SIGBUS,  0,		"synchronous parity error (translation table walk)" },
+	{ do_bad,		SIGBUS,  0,		"synchronous parity error (translation table walk)" },
+	{ do_bad,		SIGBUS,  0,		"synchronous parity error (translation table walk)" },
+	{ do_bad,		SIGBUS,  0,		"synchronous parity error (translation table walk)" },
+	{ do_bad,		SIGBUS,  0,		"unknown 32"			},
+	{ do_alignment_fault,	SIGBUS,  BUS_ADRALN,	"alignment fault"		},
+	{ do_bad,		SIGBUS,  0,		"unknown 34"			},
+	{ do_bad,		SIGBUS,  0,		"unknown 35"			},
+	{ do_bad,		SIGBUS,  0,		"unknown 36"			},
+	{ do_bad,		SIGBUS,  0,		"unknown 37"			},
+	{ do_bad,		SIGBUS,  0,		"unknown 38"			},
+	{ do_bad,		SIGBUS,  0,		"unknown 39"			},
+	{ do_bad,		SIGBUS,  0,		"unknown 40"			},
+	{ do_bad,		SIGBUS,  0,		"unknown 41"			},
+	{ do_bad,		SIGBUS,  0,		"unknown 42"			},
+	{ do_bad,		SIGBUS,  0,		"unknown 43"			},
+	{ do_bad,		SIGBUS,  0,		"unknown 44"			},
+	{ do_bad,		SIGBUS,  0,		"unknown 45"			},
+	{ do_bad,		SIGBUS,  0,		"unknown 46"			},
+	{ do_bad,		SIGBUS,  0,		"unknown 47"			},
+	{ do_bad,		SIGBUS,  0,		"TLB conflict abort"		},
+	{ do_bad,		SIGBUS,  0,		"unknown 49"			},
+	{ do_bad,		SIGBUS,  0,		"unknown 50"			},
+	{ do_bad,		SIGBUS,  0,		"unknown 51"			},
+	{ do_bad,		SIGBUS,  0,		"implementation fault (lockdown abort)" },
+	{ do_bad,		SIGBUS,  0,		"implementation fault (unsupported exclusive)" },
+	{ do_bad,		SIGBUS,  0,		"unknown 54"			},
+	{ do_bad,		SIGBUS,  0,		"unknown 55"			},
+	{ do_bad,		SIGBUS,  0,		"unknown 56"			},
+	{ do_bad,		SIGBUS,  0,		"unknown 57"			},
+	{ do_bad,		SIGBUS,  0,		"unknown 58" 			},
+	{ do_bad,		SIGBUS,  0,		"unknown 59"			},
+	{ do_bad,		SIGBUS,  0,		"unknown 60"			},
+	{ do_bad,		SIGBUS,  0,		"section domain fault"		},
+	{ do_bad,		SIGBUS,  0,		"page domain fault"		},
+	{ do_bad,		SIGBUS,  0,		"unknown 63"			},
+};
+
+static const char *fault_name(unsigned int esr)
+{
+	const struct fault_info *inf = fault_info + (esr & 63);
+	return inf->name;
+}
+
+/*
+ * Dispatch a data abort to the relevant handler.
+ */
+void do_mem_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs)
+{
+	const struct fault_info *inf = fault_info + (esr & 63);
+	struct siginfo info;
+	const int from_user = interrupt_from_user(regs);
+
+	/* set_cputime called in inf->fn() */
+	if (!inf->fn(addr, esr, regs))
+		return;
+
+	set_cputime(from_user ? CPUTIME_MODE_U2K : CPUTIME_MODE_K2K_IN);
+	kprintf("Unhandled fault: %s (0x%08x) at 0x%016lx\n", inf->name, esr, addr);
+	info.si_signo = inf->sig;
+	info.si_errno = 0;
+	info.si_code  = inf->code;
+	info._sifields._sigfault.si_addr  = (void*)addr;
+
+	arm64_notify_die("", regs, &info, esr);
+	set_cputime(from_user ? CPUTIME_MODE_K2U : CPUTIME_MODE_K2K_OUT);
+}
+
+/*
+ * Handle stack alignment exceptions.
+ */
+void do_sp_pc_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs)
+{
+	struct siginfo info;
+	const int from_user = interrupt_from_user(regs);
+
+	set_cputime(from_user ? CPUTIME_MODE_U2K : CPUTIME_MODE_K2K_IN);
+
+	info.si_signo = SIGBUS;
+	info.si_errno = 0;
+	info.si_code  = BUS_ADRALN;
+	info._sifields._sigfault.si_addr  = (void*)addr;
+	arm64_notify_die("", regs, &info, esr);
+	set_cputime(from_user ? CPUTIME_MODE_K2U : CPUTIME_MODE_K2K_OUT);
+}
+
+static void do_bad_area(unsigned long addr, unsigned int esr, struct pt_regs *regs)
+{
+	struct siginfo info;
+	const int from_user = interrupt_from_user(regs);
+
+	set_cputime(from_user ? CPUTIME_MODE_U2K : CPUTIME_MODE_K2K_IN);
+	/*
+	 * If we are in kernel mode at this point, we have no context to
+	 * handle this fault with.
+	 */
+	if (interrupt_from_user(regs)) {
+		kprintf("unhandled %s (%d) at 0x%08lx, esr 0x%03x\n",
+			fault_name(esr), SIGSEGV, addr, esr);
+
+		current_thread_info()->fault_address = addr;
+		current_thread_info()->fault_code = esr;
+		info.si_signo = SIGSEGV;
+		info.si_errno = 0;
+		info.si_code = SEGV_MAPERR;
+		info._sifields._sigfault.si_addr = (void *)addr;
+		set_signal(SIGSEGV, regs, &info); 
+
+	} else {
+		kprintf("Unable to handle kernel %s at virtual address %08lx\n",
+			 (addr < PAGE_SIZE) ? "NULL pointer dereference" : "paging request", addr);
+		panic("OOps.");
+	}
+	set_cputime(from_user ? CPUTIME_MODE_K2U : CPUTIME_MODE_K2K_OUT);
+}
+
+static int is_el0_instruction_abort(unsigned int esr)
+{
+	return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_LOW;
+}
+
+static int do_page_fault(unsigned long addr, unsigned int esr,
+				   struct pt_regs *regs)
+{
+	void (*page_fault_handler)(void *, uint64_t, void *);
+	uint64_t reason = 0;
+	int esr_ec_dfsc = (esr & 63);
+
+	if (interrupt_from_user(regs)) {
+		reason |= PF_USER;
+	}
+
+	if (is_el0_instruction_abort(esr)) {
+		reason |= PF_INSTR;
+	} else if ((esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM)) {
+		reason |= PF_WRITE;
+		if (13 <= esr_ec_dfsc && esr_ec_dfsc <= 15 ) {
+			/* level [1-3] permission fault */
+			reason |= PF_PROT;
+		}
+	}
+
+	/* set_cputime() call in page_fault_handler() */
+	page_fault_handler = (void *)__page_fault_handler_address;
+	(*page_fault_handler)((void *)addr, reason, regs);
+
+	return 0;
+}
+
+/*
+ * First Level Translation Fault Handler
+ *
+ * We enter here because the first level page table doesn't contain a valid
+ * entry for the address.
+ *
+ * If the address is in kernel space (>= TASK_SIZE), then we are probably
+ * faulting in the vmalloc() area.
+ *
+ * If the init_task's first level page tables contains the relevant entry, we
+ * copy the it to this task.  If not, we send the process a signal, fixup the
+ * exception, or oops the kernel.
+ *
+ * NOTE! We MUST NOT take any locks for this case. We may be in an interrupt
+ * or a critical region, and should only copy the information from the master
+ * page table, nothing more.
+ */
+static int do_translation_fault(unsigned long addr,
+					  unsigned int esr,
+					  struct pt_regs *regs)
+{
+#ifdef ENABLE_TOFU
+	// XXX: Handle kernel space page faults for Tofu driver
+	//if (addr < USER_END)
+#else
+	if (addr < USER_END)
+#endif
+		return do_page_fault(addr, esr, regs);
+
+	do_bad_area(addr, esr, regs);
+	return 0;
+}
+
+static int do_alignment_fault(unsigned long addr, unsigned int esr,
+			      struct pt_regs *regs)
+{
+	do_bad_area(addr, esr, regs);
+	return 0;
+}
+
+extern int breakpoint_handler(unsigned long unused, unsigned int esr, struct pt_regs *regs);
+extern int single_step_handler(unsigned long addr, unsigned int esr, struct pt_regs *regs);
+extern int watchpoint_handler(unsigned long addr, unsigned int esr, struct pt_regs *regs);
+extern int brk_handler(unsigned long addr, unsigned int esr, struct pt_regs *regs);
+static struct fault_info debug_fault_info[] = {
+	{ breakpoint_handler,	SIGTRAP,	TRAP_HWBKPT,	"hw-breakpoint handler"	},
+	{ single_step_handler,	SIGTRAP,	TRAP_HWBKPT,	"single-step handler"	},
+	{ watchpoint_handler,	SIGTRAP,	TRAP_HWBKPT,	"hw-watchpoint handler"	},
+	{ do_bad,		SIGBUS,		0,		"unknown 3"		},
+	{ do_bad,		SIGTRAP,	TRAP_BRKPT,	"aarch32 BKPT"		},
+	{ do_bad,		SIGTRAP,	0,		"aarch32 vector catch"	},
+	{ brk_handler,		SIGTRAP,	TRAP_BRKPT,	"ptrace BRK handler"	},
+	{ do_bad,		SIGBUS,		0,		"unknown 7"		},
+};
+
+int do_debug_exception(unsigned long addr, unsigned int esr, struct pt_regs *regs)
+{
+	const struct fault_info *inf = debug_fault_info + DBG_ESR_EVT(esr);
+	struct siginfo info;
+	const int from_user = interrupt_from_user(regs);
+	int ret = -1;
+
+	set_cputime(from_user ? CPUTIME_MODE_U2K : CPUTIME_MODE_K2K_IN);
+
+	if (!inf->fn(addr, esr, regs)) {
+		ret = 1;
+		goto out;
+	}
+
+	kprintf("Unhandled debug exception: %s (0x%08x) at 0x%016lx\n",
+		inf->name, esr, addr);
+
+	info.si_signo = inf->sig;
+	info.si_errno = 0;
+	info.si_code  = inf->code;
+	info._sifields._sigfault.si_addr = (void *)addr;
+
+	arm64_notify_die("", regs, &info, 0);
+
+	ret = 0;
+out:
+	set_cputime(from_user ? CPUTIME_MODE_K2U : CPUTIME_MODE_K2K_OUT);
+	return ret;
+}
+
+/*
+ * This abort handler always returns "fault".
+ */
+static int do_bad(unsigned long addr, unsigned int esr, struct pt_regs *regs)
+{
+	const int from_user = interrupt_from_user(regs);
+
+	set_cputime(from_user ? CPUTIME_MODE_U2K : CPUTIME_MODE_K2K_IN);
+	set_cputime(from_user ? CPUTIME_MODE_K2U : CPUTIME_MODE_K2K_OUT);
+	return 1;
+}

arch/arm64/kernel/fpsimd.c

@@ -0,0 +1,426 @@
+/* fpsimd.c COPYRIGHT FUJITSU LIMITED 2016-2019 */
+#include <thread_info.h>
+#include <fpsimd.h>
+#include <cpuinfo.h>
+#include <lwk/compiler.h>
+#include <ikc/ihk.h>
+#include <hwcap.h>
+#include <cls.h>
+#include <prctl.h>
+#include <cpufeature.h>
+#include <kmalloc.h>
+#include <ihk/debug.h>
+#include <process.h>
+#include <bitmap.h>
+
+//#define DEBUG_PRINT_FPSIMD
+
+#ifdef DEBUG_PRINT_FPSIMD
+#undef DDEBUG_DEFAULT
+#define DDEBUG_DEFAULT DDEBUG_PRINT
+#endif
+
+#ifdef CONFIG_ARM64_SVE
+
+/* Set of available vector lengths, as vq_to_bit(vq): */
+static DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);
+
+/* Maximum supported vector length across all CPUs (initially poisoned) */
+int sve_max_vl = -1;
+
+/* Default VL for tasks that don't set it explicitly: */
+int sve_default_vl = -1;
+
+/*
+ * Helpers to translate bit indices in sve_vq_map to VQ values (and
+ * vice versa).  This allows find_next_bit() to be used to find the
+ * _maximum_ VQ not exceeding a certain value.
+ */
+
+static unsigned int vq_to_bit(unsigned int vq)
+{
+	return SVE_VQ_MAX - vq;
+}
+
+static unsigned int bit_to_vq(unsigned int bit)
+{
+	if (bit >= SVE_VQ_MAX) {
+		bit = SVE_VQ_MAX - 1;
+	}
+	return SVE_VQ_MAX - bit;
+}
+
+/*
+ * All vector length selection from userspace comes through here.
+ * We're on a slow path, so some sanity-checks are included.
+ * If things go wrong there's a bug somewhere, but try to fall back to a
+ * safe choice.
+ */
+static unsigned int find_supported_vector_length(unsigned int vl)
+{
+	int bit;
+	int max_vl = sve_max_vl;
+
+	if (!sve_vl_valid(vl)) {
+		vl = SVE_VL_MIN;
+	}
+
+	if (!sve_vl_valid(max_vl)) {
+		max_vl = SVE_VL_MIN;
+	}
+
+	if (vl > max_vl) {
+		vl = max_vl;
+	}
+
+	bit = find_next_bit(sve_vq_map, SVE_VQ_MAX,
+			    vq_to_bit(sve_vq_from_vl(vl)));
+	return sve_vl_from_vq(bit_to_vq(bit));
+}
+
+static void sve_probe_vqs(DECLARE_BITMAP(map, SVE_VQ_MAX))
+{
+	unsigned int vq, vl;
+	unsigned long zcr;
+
+	bitmap_zero(map, SVE_VQ_MAX);
+
+	zcr = ZCR_EL1_LEN_MASK;
+	zcr = read_sysreg_s(SYS_ZCR_EL1) & ~zcr;
+
+	for (vq = SVE_VQ_MAX; vq >= SVE_VQ_MIN; --vq) {
+		/* self-syncing */
+		write_sysreg_s(zcr | (vq - 1), SYS_ZCR_EL1);
+		vl = sve_get_vl();
+		/* skip intervening lengths */
+		vq = sve_vq_from_vl(vl);
+		set_bit(vq_to_bit(vq), map);
+	}
+}
+
+void sve_init_vq_map(void)
+{
+	sve_probe_vqs(sve_vq_map);
+}
+
+size_t sve_state_size(struct thread const *thread)
+{
+	unsigned int vl = thread->ctx.thread->sve_vl;
+
+	BUG_ON(!sve_vl_valid(vl));
+	return SVE_SIG_REGS_SIZE(sve_vq_from_vl(vl));
+}
+
+void sve_free(struct thread *thread)
+{
+	if (thread->ctx.thread->sve_state) {
+		kfree(thread->ctx.thread->sve_state);
+		thread->ctx.thread->sve_state = NULL;
+	}
+}
+
+int sve_alloc(struct thread *thread)
+{
+	if (thread->ctx.thread->sve_state) {
+		return 0;
+	}
+
+	thread->ctx.thread->sve_state =
+		kmalloc(sve_state_size(thread), IHK_MC_AP_NOWAIT);
+	if (thread->ctx.thread->sve_state == NULL) {
+		return -ENOMEM;
+	}
+	memset(thread->ctx.thread->sve_state, 0, sve_state_size(thread));
+	return 0;
+}
+
+static int get_nr_threads(struct process *proc)
+{
+	struct thread *child;
+	struct mcs_rwlock_node_irqsave lock;
+	int nr_threads = 0;
+
+	mcs_rwlock_reader_lock(&proc->threads_lock, &lock);
+	list_for_each_entry(child, &proc->threads_list, siblings_list){
+		nr_threads++;
+	}
+	mcs_rwlock_reader_unlock(&proc->threads_lock, &lock);
+	return nr_threads;
+}
+
+/* @ref.impl arch/arm64/kernel/fpsimd.c::sve_set_vector_length */
+int sve_set_vector_length(struct thread *thread,
+			unsigned long vl, unsigned long flags)
+{
+	struct thread_info *ti = thread->ctx.thread;
+
+	if (flags & ~(unsigned long)(PR_SVE_VL_INHERIT |
+				     PR_SVE_SET_VL_ONEXEC)) {
+		return -EINVAL;
+	}
+
+	if (!sve_vl_valid(vl)) {
+		return -EINVAL;
+	}
+
+	/*
+	 * Clamp to the maximum vector length that VL-agnostic SVE code can
+	 * work with.  A flag may be assigned in the future to allow setting
+	 * of larger vector lengths without confusing older software.
+	 */
+	if (vl > SVE_VL_ARCH_MAX) {
+		vl = SVE_VL_ARCH_MAX;
+	}
+
+	vl = find_supported_vector_length(vl);
+
+	if (flags & (PR_SVE_VL_INHERIT |
+		     PR_SVE_SET_VL_ONEXEC)) {
+		ti->sve_vl_onexec = vl;
+	} else {
+		/* Reset VL to system default on next exec: */
+		ti->sve_vl_onexec = 0;
+	}
+
+	/* Only actually set the VL if not deferred: */
+	if (flags & PR_SVE_SET_VL_ONEXEC) {
+		goto out;
+	}
+
+	if (vl == ti->sve_vl) {
+		goto out;
+	}
+
+	if ((elf_hwcap & HWCAP_SVE)) {
+		fp_regs_struct fp_regs;
+
+		memset(&fp_regs, 0, sizeof(fp_regs));
+
+		/* for self at prctl syscall */
+		if (thread == cpu_local_var(current)) {
+			save_fp_regs(thread);
+			clear_fp_regs();
+			thread_sve_to_fpsimd(thread, &fp_regs);
+			sve_free(thread);
+
+			ti->sve_vl = vl;
+
+			sve_alloc(thread);
+			thread_fpsimd_to_sve(thread, &fp_regs);
+			restore_fp_regs(thread);
+		/* for target thread at ptrace */
+		} else {
+			thread_sve_to_fpsimd(thread, &fp_regs);
+			sve_free(thread);
+
+			ti->sve_vl = vl;
+
+			sve_alloc(thread);
+			thread_fpsimd_to_sve(thread, &fp_regs);
+		}
+	}
+	ti->sve_vl = vl;
+
+out:
+	ti->sve_flags = flags & PR_SVE_VL_INHERIT;
+
+	return 0;
+}
+
+/* @ref.impl arch/arm64/kernel/fpsimd.c::sve_prctl_status */
+/*
+ * Encode the current vector length and flags for return.
+ * This is only required for prctl(): ptrace has separate fields
+ */
+static int sve_prctl_status(unsigned long flags)
+{
+	int ret;
+	struct thread_info *ti = cpu_local_var(current)->ctx.thread;
+
+	if (flags & PR_SVE_SET_VL_ONEXEC) {
+		ret = ti->sve_vl_onexec;
+	}
+	else {
+		ret = ti->sve_vl;
+	}
+
+	if (ti->sve_flags & PR_SVE_VL_INHERIT) {
+		ret |= PR_SVE_VL_INHERIT;
+	}
+	return ret;
+}
+
+/* @ref.impl arch/arm64/kernel/fpsimd.c::sve_set_task_vl */
+int sve_set_thread_vl(unsigned long arg)
+{
+	unsigned long vl, flags;
+	int ret;
+
+	vl = arg & PR_SVE_VL_LEN_MASK;
+	flags = arg & ~vl;
+
+	/* Instead of system_supports_sve() */
+	if (unlikely(!(elf_hwcap & HWCAP_SVE))) {
+		return -EINVAL;
+	}
+
+	ret = sve_set_vector_length(cpu_local_var(current), vl, flags);
+	if (ret) {
+		return ret;
+	}
+	return sve_prctl_status(flags);
+}
+
+/* @ref.impl arch/arm64/kernel/fpsimd.c::sve_get_ti_vl */
+int sve_get_thread_vl(void)
+{
+	/* Instead of system_supports_sve() */
+	if (unlikely(!(elf_hwcap & HWCAP_SVE))) {
+		return -EINVAL;
+	}
+	return sve_prctl_status(0);
+}
+
+void do_sve_acc(unsigned int esr, struct pt_regs *regs)
+{
+	kprintf("PANIC: CPU: %d PID: %d ESR: %x Trapped SVE access.\n",
+		ihk_mc_get_processor_id(), cpu_local_var(current)->proc->pid, esr);
+	panic("");
+}
+
+void sve_setup(void)
+{
+	extern unsigned long ihk_param_default_vl;
+	uint64_t zcr;
+
+	/* Instead of system_supports_sve() */
+	if (unlikely(!(elf_hwcap & HWCAP_SVE))) {
+		return;
+	}
+
+	/* init sve_vq_map bitmap */
+	sve_init_vq_map();
+
+	/*
+	 * The SVE architecture mandates support for 128-bit vectors,
+	 * so sve_vq_map must have at least SVE_VQ_MIN set.
+	 * If something went wrong, at least try to patch it up:
+	 */
+	if (!test_bit(vq_to_bit(SVE_VQ_MIN), sve_vq_map)) {
+		set_bit(vq_to_bit(SVE_VQ_MIN), sve_vq_map);
+	}
+
+	zcr = read_system_reg(SYS_ZCR_EL1);
+	sve_max_vl = sve_vl_from_vq((zcr & ZCR_EL1_LEN_MASK) + 1);
+
+	/*
+	 * Sanity-check that the max VL we determined through CPU features
+	 * corresponds properly to sve_vq_map.  If not, do our best:
+	 */
+	if (sve_max_vl != find_supported_vector_length(sve_max_vl)) {
+		sve_max_vl = find_supported_vector_length(sve_max_vl);
+	}
+
+	sve_default_vl = ihk_param_default_vl;
+
+	if (ihk_param_default_vl !=
+		find_supported_vector_length(ihk_param_default_vl)) {
+		kprintf("SVE: Getting unsupported default VL = %d "
+			"from HOST-Linux.\n", sve_default_vl);
+		sve_default_vl = find_supported_vector_length(64);
+		kprintf("SVE: Using default vl(%d byte).\n",
+			sve_default_vl);
+	}
+
+	kprintf("SVE: maximum available vector length %u bytes per vector\n",
+		sve_max_vl);
+	kprintf("SVE: default vector length %u bytes per vector\n",
+		sve_default_vl);
+}
+
+#else /* CONFIG_ARM64_SVE */
+
+void sve_setup(void)
+{
+	/* nothing to do. */
+}
+
+#endif /* CONFIG_ARM64_SVE */
+
+/* @ref.impl arch/arm64/kernel/fpsimd.c::__task_pffr */
+static void *__thread_pffr(struct thread *thread)
+{
+	unsigned int vl = thread->ctx.thread->sve_vl;
+
+	BUG_ON(!sve_vl_valid(vl));
+	return (char *)thread->ctx.thread->sve_state + 34 * vl;
+}
+
+/* There is a need to call from to check the HWCAP_FP and HWCAP_ASIMD state. */
+void thread_fpsimd_load(struct thread *thread)
+{
+	if (likely(elf_hwcap & HWCAP_SVE)) {
+		unsigned int vl = thread->ctx.thread->sve_vl;
+
+		BUG_ON(!sve_vl_valid(vl));
+		sve_load_state(__thread_pffr(thread), &thread->fp_regs->fpsr, sve_vq_from_vl(vl) - 1);
+		dkprintf("sve for TID %d restored\n", thread->tid);
+	} else {
+		// Load the current FPSIMD state to memory.
+		fpsimd_load_state(thread->fp_regs);
+		dkprintf("fp_regs for TID %d restored\n", thread->tid);
+	}
+}
+
+/* There is a need to call from to check the HWCAP_FP and HWCAP_ASIMD state. */
+void thread_fpsimd_save(struct thread *thread)
+{
+	if (likely(elf_hwcap & HWCAP_SVE)) {
+		sve_save_state(__thread_pffr(thread), &thread->fp_regs->fpsr);
+		dkprintf("sve for TID %d saved\n", thread->tid);
+	} else {
+		// Save the current FPSIMD state to memory.
+		fpsimd_save_state(thread->fp_regs);
+		dkprintf("fp_regs for TID %d saved\n", thread->tid);
+	}
+}
+
+/* @ref.impl arch/arm64/kernel/fpsimd.c::__task_fpsimd_to_sve */
+static void __thread_fpsimd_to_sve(struct thread *thread, fp_regs_struct *fp_regs, unsigned int vq)
+{
+	struct fpsimd_sve_state(vq) *sst = thread->ctx.thread->sve_state;
+	unsigned int i;
+
+	for (i = 0; i < 32; i++) {
+		sst->zregs[i][0] = fp_regs->vregs[i];
+	}
+}
+
+/* @ref.impl arch/arm64/kernel/fpsimd.c::task_fpsimd_to_sve */
+void thread_fpsimd_to_sve(struct thread *thread, fp_regs_struct *fp_regs)
+{
+	unsigned int vl = thread->ctx.thread->sve_vl;
+
+	BUG_ON(!sve_vl_valid(vl));
+	__thread_fpsimd_to_sve(thread, fp_regs, sve_vq_from_vl(vl));
+}
+
+/* @ref.impl arch/arm64/kernel/fpsimd.c::__task_sve_to_fpsimd */
+static void __thread_sve_to_fpsimd(struct thread *thread, fp_regs_struct *fp_regs, unsigned int vq)
+{
+	struct fpsimd_sve_state(vq) *sst = thread->ctx.thread->sve_state;
+	unsigned int i;
+
+	for (i = 0; i < 32; i++) {
+		fp_regs->vregs[i] = sst->zregs[i][0];
+	}
+}
+
+/* @ref.impl arch/arm64/kernel/fpsimd.c::task_sve_to_fpsimd */
+void thread_sve_to_fpsimd(struct thread *thread, fp_regs_struct *fp_regs)
+{
+	unsigned int vl = thread->ctx.thread->sve_vl;
+
+	BUG_ON(!sve_vl_valid(vl));
+	__thread_sve_to_fpsimd(thread, fp_regs, sve_vq_from_vl(vl));
+}

arch/arm64/kernel/head.S

@@ -0,0 +1,805 @@
+/* head.S COPYRIGHT FUJITSU LIMITED 2015-2018 */
+
+#include <linkage.h>
+#include <ptrace.h>
+#include <assembler.h>
+#include <asm-offsets.h>
+#include <virt.h>
+#include <cache.h>
+#include <arch-memory.h>
+#include <smp.h>
+#include <arm-gic-v3.h>
+
+/* KERNEL_RAM_VADDR is defined by cmake */
+
+//#ifndef CONFIG_SMP
+//# define PTE_FLAGS	PTE_TYPE_PAGE | PTE_AF
+//# define PMD_FLAGS	PMD_TYPE_SECT | PMD_SECT_AF
+//#else
+# define PTE_FLAGS	PTE_TYPE_PAGE | PTE_AF | PTE_SHARED
+# define PMD_FLAGS	PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S
+//#endif /*CONFIG_SMP*/
+
+#ifdef CONFIG_ARM64_64K_PAGES
+# define MM_MMUFLAGS	PTE_ATTRINDX(MT_NORMAL) | PTE_FLAGS
+#else
+# define MM_MMUFLAGS	PMD_ATTRINDX(MT_NORMAL) | PMD_FLAGS
+#endif
+
+	.macro	pgtbl_init_core, name, dir, tbl, ents, virt_to_phys
+	ldr	\tbl, =\name
+	ldr	\ents, =\dir
+	add	\tbl, \tbl, \virt_to_phys
+	str	\ents, [\tbl]
+	add	\tbl, \tbl, #8
+	add	\ents, \ents, \virt_to_phys
+	str	\ents, [\tbl]
+	.endm
+		
+	.macro	pgtbl_init, tbl, ents, virt_to_phys
+	pgtbl_init_core	swapper_page_table, swapper_pg_dir, \tbl, \ents, \virt_to_phys
+	pgtbl_init_core	idmap_page_table, idmap_pg_dir, \tbl, \ents, \virt_to_phys
+	.endm
+
+	.macro	pgtbl, ttb0, ttb1, virt_to_phys
+	ldr	\ttb1, =swapper_pg_dir
+	ldr	\ttb0, =idmap_pg_dir
+	add	\ttb1, \ttb1, \virt_to_phys
+	add	\ttb0, \ttb0, \virt_to_phys
+	.endm
+
+#define KERNEL_START	KERNEL_RAM_VADDR
+#define KERNEL_END	_end
+
+/* ihk param offset */
+#define TRAMPOLINE_DATA_RESERVED_SIZE          0x08
+#define TRAMPOLINE_DATA_PGTBL_SIZE             0x08
+#define TRAMPOLINE_DATA_LOAD_SIZE              0x08
+#define TRAMPOLINE_DATA_STACK_SIZE             0x08
+#define TRAMPOLINE_DATA_BOOT_PARAM_SIZE        0x08
+#define TRAMPOLINE_DATA_STARTUP_DATA_SIZE      0x08
+#define TRAMPOLINE_DATA_ST_PHYS_BASE_SIZE      0x08
+#define TRAMPOLINE_DATA_ST_PHYS_SIZE_SIZE      0x08
+#define TRAMPOLINE_DATA_GIC_DIST_PA_SIZE       0x08
+#define TRAMPOLINE_DATA_GIC_DIST_MAP_SIZE_SIZE 0x08
+#define TRAMPOLINE_DATA_GIC_CPU_PA_SIZE        0x08
+#define TRAMPOLINE_DATA_GIC_CPU_MAP_SIZE_SIZE  0x08
+#define TRAMPOLINE_DATA_GIC_PERCPU_OFF_SIZE    0x04
+#define TRAMPOLINE_DATA_GIC_VERSION_SIZE       0x04
+#define TRAMPOLINE_DATA_LPJ_SIZE               0x08
+#define TRAMPOLINE_DATA_HZ_SIZE                0x08
+#define TRAMPOLINE_DATA_PSCI_METHOD_SIZE       0x08
+#define TRAMPOLINE_DATA_USE_VIRT_TIMER_SIZE    0x08
+#define TRAMPOLINE_DATA_EVTSTRM_TIMER_RATE_SIZE	0x08
+#define TRAMPOLINE_DATA_DEFAULT_VL_SIZE        0x08
+#define TRAMPOLINE_DATA_CPU_MAP_SIZE_SIZE      0x08
+#define TRAMPOLINE_DATA_CPU_MAP_SIZE           (NR_CPUS * 8)
+#define TRAMPOLINE_DATA_DATA_RDISTS_PA_SIZE    (NR_CPUS * 8)
+#define TRAMPOLINE_DATA_RETENTION_STATE_FLAG_PA_SIZE 0x08
+#define TRAMPOLINE_DATA_NR_PMU_AFFI_SIZE       0x04
+#define TRAMPOLINE_DATA_PMU_AFF_SIZE           (CONFIG_SMP_MAX_CORES * 4)
+
+#define STARTUP_DATA_RESERVED		0x00
+#define STARTUP_DATA_BASE		0x08
+#define STARTUP_DATA_PGTBL		0x10
+#define STARTUP_DATA_STACK		0x18
+#define STARTUP_DATA_ARG2		0x20
+#define STARTUP_DATA_TRAMPILINE		0x28
+#define STARTUP_DATA_NEXT_PC		0x30
+
+/* ihk param save area */
+	.globl	ihk_param_head
+	.globl	ihk_param_gic_dist_base_pa, ihk_param_gic_cpu_base_pa
+	.globl	ihk_param_gic_dist_map_size, ihk_param_gic_cpu_map_size
+	.globl	ihk_param_gic_percpu_offset, ihk_param_gic_version
+	.globl	ihk_param_lpj, ihk_param_hz, ihk_param_psci_method
+	.globl	ihk_param_cpu_logical_map, ihk_param_gic_rdist_base_pa
+	.globl	ihk_param_pmu_irq_affi, ihk_param_nr_pmu_irq_affi
+	.globl	ihk_param_use_virt_timer, ihk_param_evtstrm_timer_rate
+	.globl	ihk_param_retention_state_flag_pa, ihk_param_default_vl
+ihk_param_head:
+ihk_param_param_addr:
+	.quad	0
+ihk_param_phys_addr:
+	.quad	0
+ihk_param_st_phys_base:
+	.quad	0
+ihk_param_st_phys_size:
+	.quad	0
+ihk_param_gic_dist_base_pa:
+	.quad	0
+ihk_param_gic_dist_map_size:
+	.quad	0
+ihk_param_gic_cpu_base_pa:
+	.quad	0
+ihk_param_gic_cpu_map_size:
+	.quad	0
+ihk_param_gic_percpu_offset:
+	.word	0
+ihk_param_gic_version:
+	.word	0
+ihk_param_lpj:
+	.quad	0		/* udelay loops value */
+ihk_param_hz:
+	.quad	0		/* host HZ value */
+ihk_param_psci_method:
+	.quad	0		/* hvc or smc ? */
+ihk_param_use_virt_timer:
+	.quad	0		/* virt timer or phys timer ? */
+ihk_param_evtstrm_timer_rate:
+	.quad	0		/* event stream timer rate */
+ihk_param_default_vl:
+	.quad	0		/* SVE default VL */
+ihk_param_cpu_logical_map:
+	.skip	NR_CPUS * 8	/* array of the MPIDR and the core number */
+ihk_param_gic_rdist_base_pa:
+	.skip	NR_CPUS * 8	/* per-cpu re-distributer PA */
+ihk_param_retention_state_flag_pa:
+	.quad	0
+ihk_param_pmu_irq_affi:
+	.skip	CONFIG_SMP_MAX_CORES * 4	/* array of the pmu affinity list */
+ihk_param_nr_pmu_irq_affi:
+	.word	0		/* number of pmu affinity list elements. */
+
+/* @ref.impl arch/arm64/include/asm/kvm_arm.h */
+#define HCR_E2H		(UL(1) << 34)
+#define HCR_RW_SHIFT	31
+#define HCR_RW		(UL(1) << HCR_RW_SHIFT)
+#define HCR_TGE		(UL(1) << 27)
+
+/*
+ * end early head section, begin head code that is also used for
+ * hotplug and needs to have the same protections as the text region
+ */
+	.section ".text","ax"
+
+ENTRY(arch_start)
+	/* store ihk param */
+	/* x4 = ihk_smp_trampoline_data PA */
+	add x0, x4, #TRAMPOLINE_DATA_RESERVED_SIZE
+	/* header_pgtbl */
+	add x0, x0, #TRAMPOLINE_DATA_PGTBL_SIZE
+	/* header_load */
+	add x0, x0, #TRAMPOLINE_DATA_LOAD_SIZE
+	/* stack_ptr */
+	add x0, x0, #TRAMPOLINE_DATA_STACK_SIZE
+	/* notify_address */
+	ldr	x16, [x0], #TRAMPOLINE_DATA_BOOT_PARAM_SIZE
+	adr	x15, ihk_param_param_addr
+	str	x16, [x15]
+	/* startup_data */
+	ldr	x16, [x0], #TRAMPOLINE_DATA_STARTUP_DATA_SIZE
+	ldr	x15, [x16, #STARTUP_DATA_ARG2]
+	adr	x17, ihk_param_phys_addr
+	str	x15, [x17]
+	/* st_phys_base */
+	ldr	x16, [x0], #TRAMPOLINE_DATA_ST_PHYS_BASE_SIZE
+	adr	x15, ihk_param_st_phys_base
+	str	x16, [x15]
+	/* st_phys_size */
+	ldr	x16, [x0], #TRAMPOLINE_DATA_ST_PHYS_SIZE_SIZE
+	adr	x15, ihk_param_st_phys_size
+	str	x16, [x15]
+	/* dist_base_pa */
+	ldr	x16, [x0], #TRAMPOLINE_DATA_GIC_DIST_PA_SIZE
+	adr	x15, ihk_param_gic_dist_base_pa
+	str	x16, [x15]
+	/* dist_map_size */
+	ldr	x16, [x0], #TRAMPOLINE_DATA_GIC_DIST_MAP_SIZE_SIZE
+	adr	x15, ihk_param_gic_dist_map_size
+	str	x16, [x15]
+	/* cpu_base_pa */
+	ldr	x16, [x0], #TRAMPOLINE_DATA_GIC_CPU_PA_SIZE
+	adr	x15, ihk_param_gic_cpu_base_pa
+	str	x16, [x15]
+	/* cpu_map_size */
+	ldr	x16, [x0], #TRAMPOLINE_DATA_GIC_CPU_MAP_SIZE_SIZE
+	adr	x15, ihk_param_gic_cpu_map_size
+	str	x16, [x15]
+	/* percpu_offset */
+	ldr	w16, [x0], #TRAMPOLINE_DATA_GIC_PERCPU_OFF_SIZE
+	adr	x15, ihk_param_gic_percpu_offset
+	str	w16, [x15]
+	/* gic_version */
+	ldr	w16, [x0], #TRAMPOLINE_DATA_GIC_VERSION_SIZE
+	adr	x15, ihk_param_gic_version
+	str	w16, [x15]
+	/* loops_per_jiffy */
+	ldr	x16, [x0], #TRAMPOLINE_DATA_LPJ_SIZE
+	adr	x15, ihk_param_lpj
+	str	x16, [x15]
+	/* hz */
+	ldr	x16, [x0], #TRAMPOLINE_DATA_HZ_SIZE
+	adr	x15, ihk_param_hz
+	str	x16, [x15]
+	/* psci_method */
+	ldr	x16, [x0], #TRAMPOLINE_DATA_PSCI_METHOD_SIZE
+	adr	x15, ihk_param_psci_method
+	str	x16, [x15]
+	/* use_virt_timer */
+	ldr	x16, [x0], #TRAMPOLINE_DATA_USE_VIRT_TIMER_SIZE
+	adr	x15, ihk_param_use_virt_timer
+	str	x16, [x15]
+	/* evtstrm_timer_rate */
+	ldr	x16, [x0], #TRAMPOLINE_DATA_EVTSTRM_TIMER_RATE_SIZE
+	adr	x15, ihk_param_evtstrm_timer_rate
+	str	x16, [x15]
+	/* SVE default VL */
+	ldr	x16, [x0], #TRAMPOLINE_DATA_DEFAULT_VL_SIZE
+	adr	x15, ihk_param_default_vl
+	str	x16, [x15]
+	/* cpu_logical_map_size */
+	ldr	x16, [x0], #TRAMPOLINE_DATA_CPU_MAP_SIZE_SIZE
+	mov	x1, x16
+	/* cpu_logical_map */
+	adr	x15, ihk_param_cpu_logical_map
+	mov	x18, x0
+1:	ldr	x17, [x18], #8
+	str	x17, [x15], #8
+	sub	x16, x16, #1
+	cmp	x16, #0
+	b.ne	1b
+	mov x16, #NR_CPUS /* calc next data */
+	lsl x16, x16, 3
+	add x0, x0, x16
+
+	/* reset cpu_logical_map_size */
+	mov	x16, x1
+	/* gic_rdist_base_pa */
+	adr	x15, ihk_param_gic_rdist_base_pa
+	mov	x18, x0
+1:	ldr	x17, [x18], #8
+	str	x17, [x15], #8
+	sub	x16, x16, #1
+	cmp	x16, #0
+	b.ne	1b
+	mov x16, #NR_CPUS /* calc next data */
+	lsl x16, x16, 3
+	add x0, x0, x16
+	/* retention_state_flag_pa */
+	ldr	x16, [x0], #TRAMPOLINE_DATA_RETENTION_STATE_FLAG_PA_SIZE
+	adr	x15, ihk_param_retention_state_flag_pa
+	str	x16, [x15]
+	/* nr_pmu_irq_affi */
+	ldr	w16, [x0], #TRAMPOLINE_DATA_NR_PMU_AFFI_SIZE
+	adr	x15, ihk_param_nr_pmu_irq_affi
+	str	w16, [x15]
+	/* pmu_irq_affi */
+	mov x18, x0
+	adr	x15, ihk_param_pmu_irq_affi
+	b	2f
+1:	ldr	w17, [x18], #4
+	str	w17, [x15], #4
+	sub	w16, w16, #1
+2:	cmp	w16, #0
+	b.ne	1b
+
+	mov x16, #CONFIG_SMP_MAX_CORES  /* calc next data */
+	lsl x16, x16, 2
+	add x0, x0, x16
+	/* */
+	bl	__calc_phys_offset	// x24=PHYS_OFFSET, x28=PHYS_OFFSET-KERNEL_START
+	bl	__create_page_tables	// x25=TTBR0, x26=TTBR1
+	b	secondary_entry_common
+ENDPROC(arch_start)
+
+ENTRY(arch_ap_start)
+	bl	__calc_phys_offset	// x24=PHYS_OFFSET, x28=PHYS_OFFSET-KERNEL_START
+	b	secondary_entry_common
+ENDPROC(arch_ap_start)
+
+/*
+ * Macro to create a table entry to the next page.
+ *
+ *	tbl:	page table address
+ *	virt:	virtual address
+ *	shift:	#imm page table shift
+ *	ptrs:	#imm pointers per table page
+ *
+ * Preserves:	virt
+ * Corrupts:	tmp1, tmp2
+ * Returns:	tbl -> next level table page address
+ */
+	.macro	create_table_entry, tbl, virt, shift, ptrs, tmp1, tmp2
+	lsr	\tmp1, \virt, #\shift
+	and	\tmp1, \tmp1, #\ptrs - 1	// table index
+	add	\tmp2, \tbl, #PAGE_SIZE
+	orr	\tmp2, \tmp2, #PMD_TYPE_TABLE	// address of next table and entry type
+	str	\tmp2, [\tbl, \tmp1, lsl #3]
+	add	\tbl, \tbl, #PAGE_SIZE		// next level table page
+	.endm
+
+/*
+ * Macro to populate the PGD (and possibily PUD) for the corresponding
+ * block entry in the next level (tbl) for the given virtual address.
+ *
+ * Preserves:	tbl, next, virt
+ * Corrupts:	tmp1, tmp2
+ */
+	.macro	create_pgd_entry, tbl, virt, tmp1, tmp2
+	create_table_entry \tbl, \virt, PGDIR_SHIFT, PTRS_PER_PGD, \tmp1, \tmp2
+#if SWAPPER_PGTABLE_LEVELS == 3
+	create_table_entry \tbl, \virt, TABLE_SHIFT, PTRS_PER_PTE, \tmp1, \tmp2
+#endif
+	.endm
+
+/*
+ * Macro to populate block entries in the page table for the start..end
+ * virtual range (inclusive).
+ *
+ * Preserves:	tbl, flags
+ * Corrupts:	phys, start, end, pstate
+ */
+	.macro	create_block_map, tbl, flags, phys, start, end
+	lsr	\phys, \phys, #BLOCK_SHIFT
+	lsr	\start, \start, #BLOCK_SHIFT
+	and	\start, \start, #PTRS_PER_PTE - 1	// table index
+	orr	\phys, \flags, \phys, lsl #BLOCK_SHIFT	// table entry
+	lsr	\end, \end, #BLOCK_SHIFT
+	and	\end, \end, #PTRS_PER_PTE - 1		// table end index
+9999:	str	\phys, [\tbl, \start, lsl #3]		// store the entry
+	add	\start, \start, #1			// next entry
+	add	\phys, \phys, #BLOCK_SIZE		// next block
+	cmp	\start, \end
+	b.ls	9999b
+	.endm
+
+/*
+ * Setup the initial page tables. We only setup the barest amount which is
+ * required to get the kernel running. The following sections are required:
+ *   - identity mapping to enable the MMU (low address, TTBR0)
+ *   - first few MB of the kernel linear mapping to jump to once the MMU has
+ *     been enabled, including the FDT blob (TTBR1)
+ *   - pgd entry for fixed mappings (TTBR1)
+ */
+__create_page_tables:
+	pgtbl_init x25, x26, x28
+	pgtbl	x25, x26, x28			// idmap_pg_dir and swapper_pg_dir addresses
+	mov	x27, lr
+
+	/*
+	 * Invalidate the idmap and swapper page tables to avoid potential
+	 * dirty cache lines being evicted.
+	 */
+	mov	x0, x25
+	add	x1, x26, #SWAPPER_DIR_SIZE
+	bl	__inval_cache_range
+
+	/*
+	 * Clear the idmap and swapper page tables.
+	 */
+	mov	x0, x25
+	add	x6, x26, #SWAPPER_DIR_SIZE
+1:	stp	xzr, xzr, [x0], #16
+	stp	xzr, xzr, [x0], #16
+	stp	xzr, xzr, [x0], #16
+	stp	xzr, xzr, [x0], #16
+	cmp	x0, x6
+	b.lo	1b
+
+	ldr	x7, =MM_MMUFLAGS
+
+	/*
+	 * Create the identity mapping.
+	 */
+	mov	x0, x25				// idmap_pg_dir
+	ldr	x3, =KERNEL_START
+	add	x3, x3, x28			// __pa(KERNEL_START)
+	create_pgd_entry x0, x3, x5, x6
+	ldr	x6, =KERNEL_END
+	mov	x5, x3				// __pa(KERNEL_START)
+	add	x6, x6, x28			// __pa(KERNEL_END)
+	create_block_map x0, x7, x3, x5, x6
+
+	/*
+	 * Map the kernel image (starting with PHYS_OFFSET).
+	 */
+	mov	x0, x26				// swapper_pg_dir
+	ldr	x5, =KERNEL_START
+	create_pgd_entry x0, x5, x3, x6
+	ldr	x6, =KERNEL_END
+	mov	x3, x24				// phys offset
+	create_block_map x0, x7, x3, x5, x6
+
+	/*
+	 * Map the early_alloc_pages area, kernel_img next block
+	 */
+	ldr	x3, =KERNEL_END
+	add	x3, x3, x28			// __pa(KERNEL_END)
+	add	x3, x3, #BLOCK_SIZE
+	sub	x3, x3, #1
+	bic	x3, x3, #(BLOCK_SIZE - 1)	// start PA calc.
+	ldr	x5, =KERNEL_END			// get start VA
+	add     x5, x5, #BLOCK_SIZE
+	sub	x5, x5, #1
+	bic	x5, x5, #(BLOCK_SIZE - 1)	// start VA calc.
+	mov	x6, #MAP_EARLY_ALLOC_SIZE
+	add	x6, x5, x6			// end VA calc
+	mov	x23, x6				// save end VA
+	sub	x6, x6, #1			// inclusive range
+	create_block_map x0, x7, x3, x5, x6
+
+	/*
+	 * Map the boot_param area
+	 */
+	adr	x3, ihk_param_param_addr
+	ldr	x3, [x3]			// get boot_param PA
+	mov	x5, x23				// get start VA
+	add     x5, x5, #BLOCK_SIZE
+	sub	x5, x5, #1
+	bic	x5, x5, #(BLOCK_SIZE - 1)	// start VA calc
+	mov	x6, #MAP_BOOT_PARAM_SIZE
+	add	x6, x5, x6			// end VA calc.
+	sub	x6, x6, #1			// inclusive range
+	create_block_map x0, x7, x3, x5, x6
+
+	/*
+	 * Map the FDT blob (maximum 2MB; must be within 512MB of
+	 * PHYS_OFFSET).
+	 */
+/* FDT disable for McKernel */
+//	mov	x3, x21				// FDT phys address
+//	and	x3, x3, #~((1 << 21) - 1)	// 2MB aligned
+//	mov	x6, #PAGE_OFFSET
+//	sub	x5, x3, x24			// subtract PHYS_OFFSET
+//	tst	x5, #~((1 << 29) - 1)		// within 512MB?
+//	csel	x21, xzr, x21, ne		// zero the FDT pointer
+//	b.ne	1f
+//	add	x5, x5, x6			// __va(FDT blob)
+//	add	x6, x5, #1 << 21		// 2MB for the FDT blob
+//	sub	x6, x6, #1			// inclusive range
+//	create_block_map x0, x7, x3, x5, x6
+1:
+	/*
+	 * Since the page tables have been populated with non-cacheable
+	 * accesses (MMU disabled), invalidate the idmap and swapper page
+	 * tables again to remove any speculatively loaded cache lines.
+	 */
+	mov	x0, x25
+	add	x1, x26, #SWAPPER_DIR_SIZE
+	bl	__inval_cache_range
+
+	mov	lr, x27
+	ret
+ENDPROC(__create_page_tables)
+	.ltorg
+
+/*
+ * If we're fortunate enough to boot at EL2, ensure that the world is
+ * sane before dropping to EL1.
+ *
+ * Returns either BOOT_CPU_MODE_EL1 or BOOT_CPU_MODE_EL2 in x20 if
+ * booted in EL1 or EL2 respectively.
+ */
+ENTRY(el2_setup)
+	mrs	x0, CurrentEL
+	cmp	x0, #CurrentEL_EL2
+	b.ne	1f
+	mrs	x0, sctlr_el2
+CPU_BE(	orr	x0, x0, #(1 << 25)	)	// Set the EE bit for EL2
+CPU_LE(	bic	x0, x0, #(1 << 25)	)	// Clear the EE bit for EL2
+	msr	sctlr_el2, x0
+	b	2f
+1:	mrs	x0, sctlr_el1
+CPU_BE(	orr	x0, x0, #(3 << 24)	)	// Set the EE and E0E bits for EL1
+CPU_LE(	bic	x0, x0, #(3 << 24)	)	// Clear the EE and E0E bits for EL1
+	msr	sctlr_el1, x0
+	mov	w20, #BOOT_CPU_MODE_EL1		// This cpu booted in EL1
+	isb
+	ret
+
+2:
+#ifdef CONFIG_ARM64_VHE
+	/*
+	 * Check for VHE being present. For the rest of the EL2 setup,
+	 * x2 being non-zero indicates that we do have VHE, and that the
+	 * kernel is intended to run at EL2.
+	 */
+	mrs	x2, id_aa64mmfr1_el1
+	ubfx	x2, x2, #8, #4
+#else /* CONFIG_ARM64_VHE */
+	mov	x2, xzr
+#endif /* CONFIG_ARM64_VHE */
+
+	/* Hyp configuration. */
+	mov	x0, #HCR_RW			// 64-bit EL1
+	cbz	x2, set_hcr
+	orr	x0, x0, #HCR_TGE		// Enable Host Extensions
+	orr	x0, x0, #HCR_E2H
+set_hcr:
+	msr	hcr_el2, x0
+	isb
+
+	/* Generic timers. */
+	mrs	x0, cnthctl_el2
+	orr	x0, x0, #3			// Enable EL1 physical timers
+	msr	cnthctl_el2, x0
+	msr	cntvoff_el2, xzr		// Clear virtual offset
+
+#ifdef CONFIG_ARM_GIC_V3
+	/* GICv3 system register access */
+	mrs	x0, id_aa64pfr0_el1
+	ubfx	x0, x0, #24, #4
+	cmp	x0, #1
+	b.ne	3f
+
+	mrs_s	x0, ICC_SRE_EL2
+	orr	x0, x0, #ICC_SRE_EL2_SRE	// Set ICC_SRE_EL2.SRE==1
+	orr	x0, x0, #ICC_SRE_EL2_ENABLE	// Set ICC_SRE_EL2.Enable==1
+	msr_s	ICC_SRE_EL2, x0
+	isb					// Make sure SRE is now set
+	msr_s	ICH_HCR_EL2, xzr		// Reset ICC_HCR_EL2 to defaults
+
+3:
+#endif
+
+	/* Populate ID registers. */
+	mrs	x0, midr_el1
+	mrs	x1, mpidr_el1
+	msr	vpidr_el2, x0
+	msr	vmpidr_el2, x1
+
+	/*
+	 * When VHE is not in use, early init of EL2 and EL1 needs to be
+	 * done here.
+	 * When VHE _is_ in use, EL1 will not be used in the host and
+	 * requires no configuration, and all non-hyp-specific EL2 setup
+	 * will be done via the _EL1 system register aliases in __cpu_setup.
+	 */
+	cbnz	x2, 1f
+
+	/* sctlr_el1 */
+	mov	x0, #0x0800			// Set/clear RES{1,0} bits
+CPU_BE(	movk	x0, #0x33d0, lsl #16	)	// Set EE and E0E on BE systems
+CPU_LE(	movk	x0, #0x30d0, lsl #16	)	// Clear EE and E0E on LE systems
+	msr	sctlr_el1, x0
+
+	/* Coprocessor traps. */
+	mov	x0, #0x33ff
+
+	/* SVE register access */
+	mrs	x1, id_aa64pfr0_el1
+	ubfx	x1, x1, #ID_AA64PFR0_SVE_SHIFT, #4
+	cbz	x1, 4f
+
+	bic	x0, x0, #CPTR_EL2_TZ		// Disable SVE traps to EL2
+	msr	cptr_el2, x0			// Disable copro. traps to EL2
+	isb
+ 
+	mov	x1, #ZCR_EL1_LEN_MASK		// SVE: Enable full vector
+	msr_s	SYS_ZCR_EL1, x1			// length for EL1.
+	b	1f
+
+4:	msr	cptr_el2, x0			// Disable copro. traps to EL2
+1:
+#ifdef CONFIG_COMPAT
+	msr	hstr_el2, xzr			// Disable CP15 traps to EL2
+#endif
+
+	/* Stage-2 translation */
+	msr	vttbr_el2, xzr
+
+	cbz	x2, install_el2_stub
+
+	mov	w20, #BOOT_CPU_MODE_EL2		// This CPU booted in EL2
+	isb
+	ret
+
+install_el2_stub:
+	/* Hypervisor stub */
+	adrp	x0, __hyp_stub_vectors
+	add	x0, x0, #:lo12:__hyp_stub_vectors
+	msr	vbar_el2, x0
+
+	/* spsr */
+	mov	x0, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\
+		      PSR_MODE_EL1h)
+	msr	spsr_el2, x0
+	msr	elr_el2, lr
+	mov	w20, #BOOT_CPU_MODE_EL2		// This CPU booted in EL2
+	eret
+ENDPROC(el2_setup)
+
+/*
+ * Sets the __boot_cpu_mode flag depending on the CPU boot mode passed
+ * in x20. See arch/arm64/include/asm/virt.h for more info.
+ */
+ENTRY(set_cpu_boot_mode_flag)
+	ldr	x1, =__boot_cpu_mode		// Compute __boot_cpu_mode
+	add	x1, x1, x28
+	cmp	w20, #BOOT_CPU_MODE_EL2
+	b.ne	1f
+	add	x1, x1, #4
+1:	str	w20, [x1]			// This CPU has booted in EL1
+	dmb	sy
+	dc	ivac, x1			// Invalidate potentially stale cache line
+	ret
+ENDPROC(set_cpu_boot_mode_flag)
+
+#if defined(CONFIG_HAS_NMI)
+/*
+ * void maybe_switch_to_sysreg_gic_cpuif(void)
+ *
+ * Enable interrupt controller system register access if this feature
+ * has been detected by the alternatives system.
+ *
+ * Before we jump into generic code we must enable interrupt controller system
+ * register access because this is required by the irqflags macros.  We must
+ * also mask interrupts at the PMR and unmask them within the PSR. That leaves
+ * us set up and ready for the kernel to make its first call to
+ * arch_local_irq_enable().
+ *
+ */
+ENTRY(maybe_switch_to_sysreg_gic_cpuif)
+	mrs_s	x0, ICC_SRE_EL1
+	orr	x0, x0, #1
+	msr_s	ICC_SRE_EL1, x0		// Set ICC_SRE_EL1.SRE==1
+	isb				// Make sure SRE is now set
+	mov	x0, ICC_PMR_EL1_MASKED
+	msr_s	ICC_PMR_EL1, x0		// Prepare for unmask of I bit
+	msr	daifclr, #2		// Clear the I bit
+	ret
+ENDPROC(maybe_switch_to_sysreg_gic_cpuif)
+#else
+ENTRY(maybe_switch_to_sysreg_gic_cpuif)
+	ret
+ENDPROC(maybe_switch_to_sysreg_gic_cpuif)
+#endif /* defined(CONFIG_HAS_NMI) */
+
+/*
+ * We need to find out the CPU boot mode long after boot, so we need to
+ * store it in a writable variable.
+ *
+ * This is not in .bss, because we set it sufficiently early that the boot-time
+ * zeroing of .bss would clobber it.
+ */
+	.pushsection	.data..cacheline_aligned
+ENTRY(__boot_cpu_mode)
+	.align	L1_CACHE_SHIFT
+	.long	BOOT_CPU_MODE_EL2
+	.long	0
+	.popsection
+
+ENTRY(secondary_entry_common)
+	bl	el2_setup		// Drop to EL1
+	bl	set_cpu_boot_mode_flag
+	b	secondary_startup
+ENDPROC(secondary_entry_common)
+
+ENTRY(secondary_startup)
+	/*
+	 * Common entry point for secondary CPUs.
+	 */
+	mrs	x22, midr_el1			// x22=cpuid
+	mov	x0, x22
+	bl	lookup_processor_type
+	mov	x23, x0				// x23=current cpu_table
+	cbz	x23, __error_p			// invalid processor (x23=0)?
+
+	pgtbl	x25, x26, x28			// x25=TTBR0, x26=TTBR1
+	ldr	x12, [x23, #CPU_INFO_SETUP]
+	add	x12, x12, x28			// __virt_to_phys
+	blr	x12				// initialise processor
+
+	ldr	x21, =secondary_data
+	ldr	x27, =__secondary_switched	// address to jump to after enabling the MMU
+	b	__enable_mmu
+ENDPROC(secondary_startup)
+
+ENTRY(__secondary_switched)
+	ldr	x0, [x21, #SECONDARY_DATA_STACK]	// get secondary_data.stack
+	mov	sp, x0
+	
+	/*
+	 * Conditionally switch to GIC PMR for interrupt masking (this
+	 * will be a nop if we are using normal interrupt masking)
+	 */
+	bl	maybe_switch_to_sysreg_gic_cpuif
+	mov	x29, #0
+
+	adr	x1, secondary_data
+	ldr	x0, [x1, #SECONDARY_DATA_ARG]		// get secondary_data.arg
+	ldr	x27, [x1, #SECONDARY_DATA_NEXT_PC]	// get secondary_data.next_pc
+	br	x27					// secondary_data.next_pc(secondary_data.arg);
+ENDPROC(__secondary_switched)
+
+/*
+ * Setup common bits before finally enabling the MMU. Essentially this is just
+ * loading the page table pointer and vector base registers.
+ *
+ * On entry to this code, x0 must contain the SCTLR_EL1 value for turning on
+ * the MMU.
+ */
+__enable_mmu:
+	ldr	x5, =vectors
+	msr	vbar_el1, x5
+	msr	ttbr0_el1, x25			// load TTBR0
+	msr	ttbr1_el1, x26			// load TTBR1
+	isb
+	b	__turn_mmu_on
+ENDPROC(__enable_mmu)
+
+/*
+ * Enable the MMU. This completely changes the structure of the visible memory
+ * space. You will not be able to trace execution through this.
+ *
+ *  x0  = system control register
+ *  x27 = *virtual* address to jump to upon completion
+ *
+ * other registers depend on the function called upon completion
+ *
+ * We align the entire function to the smallest power of two larger than it to
+ * ensure it fits within a single block map entry. Otherwise were PHYS_OFFSET
+ * close to the end of a 512MB or 1GB block we might require an additional
+ * table to map the entire function.
+ */
+	.align	4
+__turn_mmu_on:
+	msr	sctlr_el1, x0
+	isb
+	br	x27
+ENDPROC(__turn_mmu_on)
+
+/*
+ * Calculate the start of physical memory.
+ */
+__calc_phys_offset:
+	adr	x0, 1f
+	ldp	x1, x2, [x0]
+	sub	x28, x0, x1		// x28 = PHYS_OFFSET - KERNEL_START
+	add	x24, x2, x28		// x24 = PHYS_OFFSET
+	ret
+ENDPROC(__calc_phys_offset)
+
+	.align 3
+1:	.quad	.
+	.quad	KERNEL_START
+
+/*
+ * Exception handling. Something went wrong and we can't proceed. We ought to
+ * tell the user, but since we don't have any guarantee that we're even
+ * running on the right architecture, we do virtually nothing.
+ */
+__error_p:
+ENDPROC(__error_p)
+
+__error:
+1:	nop
+	b	1b
+ENDPROC(__error)
+
+/*
+ * This function gets the processor ID in w0 and searches the cpu_table[] for
+ * a match. It returns a pointer to the struct cpu_info it found. The
+ * cpu_table[] must end with an empty (all zeros) structure.
+ *
+ * This routine can be called via C code and it needs to work with the MMU
+ * both disabled and enabled (the offset is calculated automatically).
+ */
+ENTRY(lookup_processor_type)
+	adr	x1, __lookup_processor_type_data
+	ldp	x2, x3, [x1]
+	sub	x1, x1, x2			// get offset between VA and PA
+	add	x3, x3, x1			// convert VA to PA
+1:
+	ldp	w5, w6, [x3]			// load cpu_id_val and cpu_id_mask
+	cbz	w5, 2f				// end of list?
+	and	w6, w6, w0
+	cmp	w5, w6
+	b.eq	3f
+	add	x3, x3, #CPU_INFO_SZ
+	b	1b
+2:
+	mov	x3, #0				// unknown processor
+3:
+	mov	x0, x3
+	ret
+ENDPROC(lookup_processor_type)
+
+	.align	3
+	.type	__lookup_processor_type_data, %object
+__lookup_processor_type_data:
+	.quad	.
+	.quad	cpu_table
+	.size	__lookup_processor_type_data, . - __lookup_processor_type_data
+

arch/arm64/kernel/hw_breakpoint.c

@@ -0,0 +1,410 @@
+/* hw_breakpoint.c COPYRIGHT FUJITSU LIMITED 2016 */
+#include <ihk/debug.h>
+#include <cputype.h>
+#include <errno.h>
+#include <elfcore.h>
+#include <ptrace.h>
+#include <hw_breakpoint.h>
+#include <arch-memory.h>
+#include <signal.h>
+#include <process.h>
+
+/* @ref.impl arch/arm64/kernel/hw_breakpoint.c::core_num_[brps|wrps] */
+/* Number of BRP/WRP registers on this CPU. */
+int core_num_brps;
+int core_num_wrps;
+
+/* @ref.impl arch/arm64/kernel/hw_breakpoint.c::get_num_brps */
+/* Determine number of BRP registers available. */
+int get_num_brps(void)
+{
+	return ((read_cpuid(ID_AA64DFR0_EL1) >> 12) & 0xf) + 1;
+}
+
+/* @ref.impl arch/arm64/kernel/hw_breakpoint.c::get_num_wrps */
+/* Determine number of WRP registers available. */
+int get_num_wrps(void)
+{
+	return ((read_cpuid(ID_AA64DFR0_EL1) >> 20) & 0xf) + 1;
+}
+
+/* @ref.impl arch/arm64/kernel/hw_breakpoint.c::hw_breakpoint_slots */
+int hw_breakpoint_slots(int type)
+{
+	/*
+	 * We can be called early, so don't rely on
+	 * our static variables being initialised.
+	 */
+	switch (type) {
+	case TYPE_INST:
+		return get_num_brps();
+	case TYPE_DATA:
+		return get_num_wrps();
+	default:
+		kprintf("unknown slot type: %d\n", type);
+		return 0;
+	}
+}
+
+/* @ref.impl arch/arm64/kernel/hw_breakpoint.c::READ_WB_REG_CASE */
+#define READ_WB_REG_CASE(OFF, N, REG, VAL)	\
+	case (OFF + N):				\
+		AARCH64_DBG_READ(N, REG, VAL);	\
+		break
+
+/* @ref.impl arch/arm64/kernel/hw_breakpoint.c::READ_WB_REG_CASE */
+#define WRITE_WB_REG_CASE(OFF, N, REG, VAL)	\
+	case (OFF + N):				\
+		AARCH64_DBG_WRITE(N, REG, VAL);	\
+		break
+
+/* @ref.impl arch/arm64/kernel/hw_breakpoint.c::GEN_READ_WB_REG_CASES */
+#define GEN_READ_WB_REG_CASES(OFF, REG, VAL)	\
+	READ_WB_REG_CASE(OFF,  0, REG, VAL);	\
+	READ_WB_REG_CASE(OFF,  1, REG, VAL);	\
+	READ_WB_REG_CASE(OFF,  2, REG, VAL);	\
+	READ_WB_REG_CASE(OFF,  3, REG, VAL);	\
+	READ_WB_REG_CASE(OFF,  4, REG, VAL);	\
+	READ_WB_REG_CASE(OFF,  5, REG, VAL);	\
+	READ_WB_REG_CASE(OFF,  6, REG, VAL);	\
+	READ_WB_REG_CASE(OFF,  7, REG, VAL);	\
+	READ_WB_REG_CASE(OFF,  8, REG, VAL);	\
+	READ_WB_REG_CASE(OFF,  9, REG, VAL);	\
+	READ_WB_REG_CASE(OFF, 10, REG, VAL);	\
+	READ_WB_REG_CASE(OFF, 11, REG, VAL);	\
+	READ_WB_REG_CASE(OFF, 12, REG, VAL);	\
+	READ_WB_REG_CASE(OFF, 13, REG, VAL);	\
+	READ_WB_REG_CASE(OFF, 14, REG, VAL);	\
+	READ_WB_REG_CASE(OFF, 15, REG, VAL)
+
+/* @ref.impl arch/arm64/kernel/hw_breakpoint.c::GEN_WRITE_WB_REG_CASES */
+#define GEN_WRITE_WB_REG_CASES(OFF, REG, VAL)	\
+	WRITE_WB_REG_CASE(OFF,  0, REG, VAL);	\
+	WRITE_WB_REG_CASE(OFF,  1, REG, VAL);	\
+	WRITE_WB_REG_CASE(OFF,  2, REG, VAL);	\
+	WRITE_WB_REG_CASE(OFF,  3, REG, VAL);	\
+	WRITE_WB_REG_CASE(OFF,  4, REG, VAL);	\
+	WRITE_WB_REG_CASE(OFF,  5, REG, VAL);	\
+	WRITE_WB_REG_CASE(OFF,  6, REG, VAL);	\
+	WRITE_WB_REG_CASE(OFF,  7, REG, VAL);	\
+	WRITE_WB_REG_CASE(OFF,  8, REG, VAL);	\
+	WRITE_WB_REG_CASE(OFF,  9, REG, VAL);	\
+	WRITE_WB_REG_CASE(OFF, 10, REG, VAL);	\
+	WRITE_WB_REG_CASE(OFF, 11, REG, VAL);	\
+	WRITE_WB_REG_CASE(OFF, 12, REG, VAL);	\
+	WRITE_WB_REG_CASE(OFF, 13, REG, VAL);	\
+	WRITE_WB_REG_CASE(OFF, 14, REG, VAL);	\
+	WRITE_WB_REG_CASE(OFF, 15, REG, VAL)
+
+/* @ref.impl arch/arm64/kernel/hw_breakpoint.c::read_wb_reg */
+unsigned long read_wb_reg(int reg, int n)
+{
+	unsigned long val = 0;
+
+	switch (reg + n) {
+	GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val);
+	GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val);
+	GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_WVR, AARCH64_DBG_REG_NAME_WVR, val);
+	GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_WCR, AARCH64_DBG_REG_NAME_WCR, val);
+	default:
+		kprintf("attempt to read from unknown breakpoint register %d\n", n);
+	}
+
+	return val;
+}
+
+/* @ref.impl arch/arm64/kernel/hw_breakpoint.c::write_wb_reg */
+void write_wb_reg(int reg, int n, unsigned long val)
+{
+	switch (reg + n) {
+	GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val);
+	GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val);
+	GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_WVR, AARCH64_DBG_REG_NAME_WVR, val);
+	GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_WCR, AARCH64_DBG_REG_NAME_WCR, val);
+	default:
+		kprintf("attempt to write to unknown breakpoint register %d\n", n);
+	}
+	isb();
+}
+
+/* @ref.impl arch/arm64/kernel/hw_breakpoint.c::hw_breakpoint_reset */
+void hw_breakpoint_reset(void)
+{
+	int i = 0;
+
+	/* clear DBGBVR<n>_EL1 and DBGBCR<n>_EL1 (n=0-(core_num_brps-1)) */
+	for (i = 0; i < core_num_brps; i++) {
+		write_wb_reg(AARCH64_DBG_REG_BVR, i, 0UL);
+		write_wb_reg(AARCH64_DBG_REG_BCR, i, 0UL);
+	}
+
+	/* clear DBGWVR<n>_EL1 and DBGWCR<n>_EL1 (n=0-(core_num_wrps-1)) */
+	for (i = 0; i < core_num_wrps; i++) {
+		write_wb_reg(AARCH64_DBG_REG_WVR, i, 0UL);
+		write_wb_reg(AARCH64_DBG_REG_WCR, i, 0UL);
+	}
+}
+
+/* @ref.impl arch/arm64/kernel/hw_breakpoint.c::arch_hw_breakpoint_init */
+void arch_hw_breakpoint_init(void)
+{
+	struct user_hwdebug_state hws;
+	int max_hws_dbg_regs = sizeof(hws.dbg_regs) / sizeof(hws.dbg_regs[0]);
+
+	core_num_brps = get_num_brps();
+	core_num_wrps = get_num_wrps();
+
+	if (max_hws_dbg_regs < core_num_brps) {
+		kprintf("debugreg struct size is less than Determine number of BRP registers available.\n");
+		core_num_brps = max_hws_dbg_regs;
+	}
+
+	if (max_hws_dbg_regs < core_num_wrps) {
+		kprintf("debugreg struct size is less than Determine number of WRP registers available.\n");
+		core_num_wrps = max_hws_dbg_regs;
+	}
+	hw_breakpoint_reset();
+}
+
+struct arch_hw_breakpoint_ctrl {
+	unsigned int __reserved	: 19,
+	len			: 8,
+	type			: 2,
+	privilege		: 2,
+	enabled			: 1;
+};
+
+static inline unsigned int encode_ctrl_reg(struct arch_hw_breakpoint_ctrl ctrl)
+{
+	return (ctrl.len << 5) | (ctrl.type << 3) | (ctrl.privilege << 1) |
+		ctrl.enabled;
+}
+
+static inline void decode_ctrl_reg(unsigned int reg, struct arch_hw_breakpoint_ctrl *ctrl)
+{
+	ctrl->enabled	= reg & 0x1;
+	reg >>= 1;
+	ctrl->privilege	= reg & 0x3;
+	reg >>= 2;
+	ctrl->type	= reg & 0x3;
+	reg >>= 2;
+	ctrl->len	= reg & 0xff;
+}
+
+/* @ref.impl arch/arm64/kernel/hw_breakpoint.c::arch_bp_generic_fields */
+/*
+ * Extract generic type and length encodings from an arch_hw_breakpoint_ctrl.
+ * Hopefully this will disappear when ptrace can bypass the conversion
+ * to generic breakpoint descriptions.
+ */
+int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl,
+			   int *gen_len, int *gen_type)
+{
+	/* Type */
+	switch (ctrl.type) {
+	case ARM_BREAKPOINT_EXECUTE:
+		*gen_type = HW_BREAKPOINT_X;
+		break;
+	case ARM_BREAKPOINT_LOAD:
+		*gen_type = HW_BREAKPOINT_R;
+		break;
+	case ARM_BREAKPOINT_STORE:
+		*gen_type = HW_BREAKPOINT_W;
+		break;
+	case ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE:
+		*gen_type = HW_BREAKPOINT_RW;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* Len */
+	switch (ctrl.len) {
+	case ARM_BREAKPOINT_LEN_1:
+		*gen_len = HW_BREAKPOINT_LEN_1;
+		break;
+	case ARM_BREAKPOINT_LEN_2:
+		*gen_len = HW_BREAKPOINT_LEN_2;
+		break;
+	case ARM_BREAKPOINT_LEN_4:
+		*gen_len = HW_BREAKPOINT_LEN_4;
+		break;
+	case ARM_BREAKPOINT_LEN_8:
+		*gen_len = HW_BREAKPOINT_LEN_8;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/* @ref.impl arch/arm64/kernel/hw_breakpoint.c::arch_check_bp_in_kernelspace */
+/*
+ * Check whether bp virtual address is in kernel space.
+ */
+int arch_check_bp_in_kernelspace(unsigned long addr, unsigned int len)
+{
+	return (addr >= USER_END) && ((addr + len - 1) >= USER_END);
+}
+
+/* @ref.impl arch/arm64/kernel/hw_breakpoint.c::arch_validate_hwbkpt_settings */
+int arch_validate_hwbkpt_settings(long note_type, struct user_hwdebug_state *hws, size_t len)
+{
+	int i;
+	unsigned long alignment_mask;
+	size_t cpysize, cpynum;
+
+	switch(note_type) {
+	case NT_ARM_HW_BREAK: /* breakpoint */
+		alignment_mask = 0x3;
+		break;
+	case NT_ARM_HW_WATCH: /* watchpoint */
+		alignment_mask = 0x7;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	cpysize = len - offsetof(struct user_hwdebug_state, dbg_regs[0]);
+	cpynum = cpysize / sizeof(hws->dbg_regs[0]);
+
+	for (i = 0; i < cpynum; i++) {
+		unsigned long addr = hws->dbg_regs[i].addr;
+		unsigned int uctrl = hws->dbg_regs[i].ctrl;
+		struct arch_hw_breakpoint_ctrl ctrl;
+		int err, len, type;
+
+		/* empty dbg_regs check skip */
+		if (addr == 0 && uctrl == 0) {
+			continue;
+		}
+
+		/* check address alignment */
+		if (addr & alignment_mask) {
+			return -EINVAL;
+		}
+
+		/* decode control bit */
+		decode_ctrl_reg(uctrl, &ctrl);
+
+		/* disabled, continue */
+		if (!ctrl.enabled) {
+			continue;
+		}
+
+		err = arch_bp_generic_fields(ctrl, &len, &type);
+		if (err) {
+			return err;
+		}
+
+		/* type check */
+		switch (note_type) {
+		case NT_ARM_HW_BREAK: /* breakpoint */
+			if ((type & HW_BREAKPOINT_X) != type) {
+				return -EINVAL;
+			}
+			break;
+		case NT_ARM_HW_WATCH: /* watchpoint */
+			if ((type & HW_BREAKPOINT_RW) != type) {
+				return -EINVAL;
+			}
+			break;
+		default:
+			return -EINVAL;
+		}
+
+		/* privilege generate */
+		if (arch_check_bp_in_kernelspace(addr, len)) {
+			/* kernel space breakpoint unsupported. */
+			return -EINVAL;
+		} else {
+			ctrl.privilege = AARCH64_BREAKPOINT_EL0;
+		}
+
+		/* ctrl check OK. */
+		hws->dbg_regs[i].ctrl = encode_ctrl_reg(ctrl);
+	}
+	return 0;
+}
+
+/* @ref.impl arch/arm64/kernel/hw_breakpoint.c::breakpoint_handler */
+/*
+ * Debug exception handlers.
+ */
+int breakpoint_handler(unsigned long unused, unsigned int esr, struct pt_regs *regs)
+{
+	int i = 0;
+	unsigned long val;
+	unsigned int ctrl_reg;
+	struct arch_hw_breakpoint_ctrl ctrl;
+	siginfo_t info;
+
+	for (i = 0; i < core_num_brps; i++) {
+
+		/* Check if the breakpoint value matches. */
+		val = read_wb_reg(AARCH64_DBG_REG_BVR, i);
+		if (val != (regs->pc & ~0x3)) {
+			continue;
+		}
+
+		/* Possible match, check the byte address select to confirm. */
+		ctrl_reg = read_wb_reg(AARCH64_DBG_REG_BCR, i);
+		decode_ctrl_reg(ctrl_reg, &ctrl);
+		if (!((1 << (regs->pc & 0x3)) & ctrl.len)) {
+			continue;
+		}
+
+		/* send SIGTRAP */
+		info.si_signo = SIGTRAP;
+		info.si_errno = 0;
+		info.si_code  = TRAP_HWBKPT;
+		info._sifields._sigfault.si_addr = (void *)regs->pc;
+		set_signal(SIGTRAP, regs, &info);
+	}
+	return 0;
+}
+
+/* @ref.impl arch/arm64/kernel/hw_breakpoint.c::watchpoint_handler */
+int watchpoint_handler(unsigned long addr, unsigned int esr, struct pt_regs *regs)
+{
+	int i = 0;
+	int access;
+	unsigned long val;
+	unsigned int ctrl_reg;
+	struct arch_hw_breakpoint_ctrl ctrl;
+	siginfo_t info;
+
+	for (i = 0; i < core_num_wrps; i++) {
+		/* Check if the watchpoint value matches. */
+		val = read_wb_reg(AARCH64_DBG_REG_WVR, i);
+		if (val != (addr & ~0x7)) {
+			continue;
+		}
+
+		/* Possible match, check the byte address select to confirm. */
+		ctrl_reg = read_wb_reg(AARCH64_DBG_REG_WCR, i);
+		decode_ctrl_reg(ctrl_reg, &ctrl);
+		if (!((1 << (addr & 0x7)) & ctrl.len)) {
+			continue;
+		}
+
+		/*
+		 * Check that the access type matches.
+		 * 0 => load, otherwise => store
+		 */
+		access = (esr & AARCH64_ESR_ACCESS_MASK) ? ARM_BREAKPOINT_STORE :
+			 ARM_BREAKPOINT_LOAD;
+		if (!(access & ctrl.type)) {
+			continue;
+		}
+
+		/* send SIGTRAP */
+		info.si_signo = SIGTRAP;
+		info.si_errno = 0;
+		info.si_code  = TRAP_HWBKPT;
+		info._sifields._sigfault.si_addr = (void *)addr;
+		set_signal(SIGTRAP, regs, &info);
+	}
+	return 0;
+}

arch/arm64/kernel/hyp-stub.S

@@ -0,0 +1,58 @@
+/* hyp-stub.S COPYRIGHT FUJITSU LIMITED 2015 */
+#include <linkage.h>
+#include <assembler.h>
+
+	.text
+	.align 11
+
+ENTRY(__hyp_stub_vectors)
+	ventry	el2_sync_invalid		// Synchronous EL2t
+	ventry	el2_irq_invalid			// IRQ EL2t
+	ventry	el2_fiq_invalid			// FIQ EL2t
+	ventry	el2_error_invalid		// Error EL2t
+
+	ventry	el2_sync_invalid		// Synchronous EL2h
+	ventry	el2_irq_invalid			// IRQ EL2h
+	ventry	el2_fiq_invalid			// FIQ EL2h
+	ventry	el2_error_invalid		// Error EL2h
+
+	ventry	el1_sync			// Synchronous 64-bit EL1
+	ventry	el1_irq_invalid			// IRQ 64-bit EL1
+	ventry	el1_fiq_invalid			// FIQ 64-bit EL1
+	ventry	el1_error_invalid		// Error 64-bit EL1
+
+	ventry	el1_sync_invalid		// Synchronous 32-bit EL1
+	ventry	el1_irq_invalid			// IRQ 32-bit EL1
+	ventry	el1_fiq_invalid			// FIQ 32-bit EL1
+	ventry	el1_error_invalid		// Error 32-bit EL1
+ENDPROC(__hyp_stub_vectors)
+
+	.align 11
+
+el1_sync:
+	mrs	x1, esr_el2
+	lsr	x1, x1, #26
+	cmp	x1, #0x16
+	b.ne	2f				// Not an HVC trap
+	cbz	x0, 1f
+	msr	vbar_el2, x0			// Set vbar_el2
+	b	2f
+1:	mrs	x0, vbar_el2			// Return vbar_el2
+2:	eret
+ENDPROC(el1_sync)
+
+.macro invalid_vector	label
+\label:
+	b \label
+ENDPROC(\label)
+.endm
+
+	invalid_vector	el2_sync_invalid
+	invalid_vector	el2_irq_invalid
+	invalid_vector	el2_fiq_invalid
+	invalid_vector	el2_error_invalid
+	invalid_vector	el1_sync_invalid
+	invalid_vector	el1_irq_invalid
+	invalid_vector	el1_fiq_invalid
+	invalid_vector	el1_error_invalid
+

arch/arm64/kernel/imp-sysreg.c

@@ -0,0 +1,131 @@
+/* imp-sysreg.c COPYRIGHT FUJITSU LIMITED 2018 */
+#include <sysreg.h>
+
+/* hpc */
+ACCESS_REG_FUNC(fj_tag_address_ctrl_el1, IMP_FJ_TAG_ADDRESS_CTRL_EL1);
+ACCESS_REG_FUNC(pf_ctrl_el1, IMP_PF_CTRL_EL1);
+ACCESS_REG_FUNC(pf_stream_detect_ctrl_el0, IMP_PF_STREAM_DETECT_CTRL_EL0);
+ACCESS_REG_FUNC(pf_injection_ctrl0_el0, IMP_PF_INJECTION_CTRL0_EL0);
+ACCESS_REG_FUNC(pf_injection_ctrl1_el0, IMP_PF_INJECTION_CTRL1_EL0);
+ACCESS_REG_FUNC(pf_injection_ctrl2_el0, IMP_PF_INJECTION_CTRL2_EL0);
+ACCESS_REG_FUNC(pf_injection_ctrl3_el0, IMP_PF_INJECTION_CTRL3_EL0);
+ACCESS_REG_FUNC(pf_injection_ctrl4_el0, IMP_PF_INJECTION_CTRL4_EL0);
+ACCESS_REG_FUNC(pf_injection_ctrl5_el0, IMP_PF_INJECTION_CTRL5_EL0);
+ACCESS_REG_FUNC(pf_injection_ctrl6_el0, IMP_PF_INJECTION_CTRL6_EL0);
+ACCESS_REG_FUNC(pf_injection_ctrl7_el0, IMP_PF_INJECTION_CTRL7_EL0);
+ACCESS_REG_FUNC(pf_injection_distance0_el0, IMP_PF_INJECTION_DISTANCE0_EL0);
+ACCESS_REG_FUNC(pf_injection_distance1_el0, IMP_PF_INJECTION_DISTANCE1_EL0);
+ACCESS_REG_FUNC(pf_injection_distance2_el0, IMP_PF_INJECTION_DISTANCE2_EL0);
+ACCESS_REG_FUNC(pf_injection_distance3_el0, IMP_PF_INJECTION_DISTANCE3_EL0);
+ACCESS_REG_FUNC(pf_injection_distance4_el0, IMP_PF_INJECTION_DISTANCE4_EL0);
+ACCESS_REG_FUNC(pf_injection_distance5_el0, IMP_PF_INJECTION_DISTANCE5_EL0);
+ACCESS_REG_FUNC(pf_injection_distance6_el0, IMP_PF_INJECTION_DISTANCE6_EL0);
+ACCESS_REG_FUNC(pf_injection_distance7_el0, IMP_PF_INJECTION_DISTANCE7_EL0);
+
+static void hpc_prefetch_regs_init(void)
+{
+	uint64_t reg = 0;
+
+	/* PF_CTRL_EL1 */
+	reg = IMP_PF_CTRL_EL1_EL1AE_ENABLE | IMP_PF_CTRL_EL1_EL0AE_ENABLE;
+	xos_access_pf_ctrl_el1(WRITE_ACCESS, &reg);
+
+	/* PF_STREAM_DETECT_CTRL */
+	reg = 0;
+	xos_access_pf_stream_detect_ctrl_el0(WRITE_ACCESS, &reg);
+
+	/* PF_INJECTION_CTRL */
+	reg = 0;
+	xos_access_pf_injection_ctrl0_el0(WRITE_ACCESS, &reg);
+	xos_access_pf_injection_ctrl1_el0(WRITE_ACCESS, &reg);
+	xos_access_pf_injection_ctrl2_el0(WRITE_ACCESS, &reg);
+	xos_access_pf_injection_ctrl3_el0(WRITE_ACCESS, &reg);
+	xos_access_pf_injection_ctrl4_el0(WRITE_ACCESS, &reg);
+	xos_access_pf_injection_ctrl5_el0(WRITE_ACCESS, &reg);
+	xos_access_pf_injection_ctrl6_el0(WRITE_ACCESS, &reg);
+	xos_access_pf_injection_ctrl7_el0(WRITE_ACCESS, &reg);
+
+	/* PF_INJECTION_DISTANCE */
+	reg = 0;
+	xos_access_pf_injection_distance0_el0(WRITE_ACCESS, &reg);
+	xos_access_pf_injection_distance1_el0(WRITE_ACCESS, &reg);
+	xos_access_pf_injection_distance2_el0(WRITE_ACCESS, &reg);
+	xos_access_pf_injection_distance3_el0(WRITE_ACCESS, &reg);
+	xos_access_pf_injection_distance4_el0(WRITE_ACCESS, &reg);
+	xos_access_pf_injection_distance5_el0(WRITE_ACCESS, &reg);
+	xos_access_pf_injection_distance6_el0(WRITE_ACCESS, &reg);
+	xos_access_pf_injection_distance7_el0(WRITE_ACCESS, &reg);
+}
+
+static void hpc_tag_address_regs_init(void)
+{
+	uint64_t reg = IMP_FJ_TAG_ADDRESS_CTRL_EL1_TBO0_MASK |
+		IMP_FJ_TAG_ADDRESS_CTRL_EL1_SEC0_MASK |
+		IMP_FJ_TAG_ADDRESS_CTRL_EL1_PFE0_MASK;
+
+	/* FJ_TAG_ADDRESS_CTRL */
+	xos_access_fj_tag_address_ctrl_el1(WRITE_ACCESS, &reg);
+}
+
+void hpc_registers_init(void)
+{
+	hpc_prefetch_regs_init();
+	hpc_tag_address_regs_init();
+}
+
+/* vhbm */
+ACCESS_REG_FUNC(barrier_ctrl_el1, IMP_BARRIER_CTRL_EL1);
+ACCESS_REG_FUNC(barrier_bst_bit_el1, IMP_BARRIER_BST_BIT_EL1);
+ACCESS_REG_FUNC(barrier_init_sync_bb0_el1, IMP_BARRIER_INIT_SYNC_BB0_EL1);
+ACCESS_REG_FUNC(barrier_init_sync_bb1_el1, IMP_BARRIER_INIT_SYNC_BB1_EL1);
+ACCESS_REG_FUNC(barrier_init_sync_bb2_el1, IMP_BARRIER_INIT_SYNC_BB2_EL1);
+ACCESS_REG_FUNC(barrier_init_sync_bb3_el1, IMP_BARRIER_INIT_SYNC_BB3_EL1);
+ACCESS_REG_FUNC(barrier_init_sync_bb4_el1, IMP_BARRIER_INIT_SYNC_BB4_EL1);
+ACCESS_REG_FUNC(barrier_init_sync_bb5_el1, IMP_BARRIER_INIT_SYNC_BB5_EL1);
+ACCESS_REG_FUNC(barrier_assign_sync_w0_el1, IMP_BARRIER_ASSIGN_SYNC_W0_EL1);
+ACCESS_REG_FUNC(barrier_assign_sync_w1_el1, IMP_BARRIER_ASSIGN_SYNC_W1_EL1);
+ACCESS_REG_FUNC(barrier_assign_sync_w2_el1, IMP_BARRIER_ASSIGN_SYNC_W2_EL1);
+ACCESS_REG_FUNC(barrier_assign_sync_w3_el1, IMP_BARRIER_ASSIGN_SYNC_W3_EL1);
+
+void vhbm_barrier_registers_init(void)
+{
+	uint64_t reg = 0;
+
+	reg = IMP_BARRIER_CTRL_EL1_EL1AE_ENABLE |
+		IMP_BARRIER_CTRL_EL1_EL0AE_ENABLE;
+	xos_access_barrier_ctrl_el1(WRITE_ACCESS, &reg);
+
+	reg = 0;
+
+	xos_access_barrier_init_sync_bb0_el1(WRITE_ACCESS, &reg);
+	xos_access_barrier_init_sync_bb1_el1(WRITE_ACCESS, &reg);
+	xos_access_barrier_init_sync_bb2_el1(WRITE_ACCESS, &reg);
+	xos_access_barrier_init_sync_bb3_el1(WRITE_ACCESS, &reg);
+	xos_access_barrier_init_sync_bb4_el1(WRITE_ACCESS, &reg);
+	xos_access_barrier_init_sync_bb5_el1(WRITE_ACCESS, &reg);
+	xos_access_barrier_assign_sync_w0_el1(WRITE_ACCESS, &reg);
+	xos_access_barrier_assign_sync_w1_el1(WRITE_ACCESS, &reg);
+	xos_access_barrier_assign_sync_w2_el1(WRITE_ACCESS, &reg);
+	xos_access_barrier_assign_sync_w3_el1(WRITE_ACCESS, &reg);
+}
+
+/* sccr */
+ACCESS_REG_FUNC(sccr_ctrl_el1, IMP_SCCR_CTRL_EL1);
+ACCESS_REG_FUNC(sccr_assign_el1, IMP_SCCR_ASSIGN_EL1);
+ACCESS_REG_FUNC(sccr_set0_l2_el1, IMP_SCCR_SET0_L2_EL1);
+ACCESS_REG_FUNC(sccr_l1_el0, IMP_SCCR_L1_EL0);
+
+void scdrv_registers_init(void)
+{
+	uint64_t reg = 0;
+
+	reg = IMP_SCCR_CTRL_EL1_EL1AE_MASK;
+	xos_access_sccr_ctrl_el1(WRITE_ACCESS, &reg);
+
+	reg = 0;
+	xos_access_sccr_assign_el1(WRITE_ACCESS, &reg);
+	xos_access_sccr_l1_el0(WRITE_ACCESS, &reg);
+
+	reg = (14UL << IMP_SCCR_SET0_L2_EL1_L2_SEC0_SHIFT);
+	xos_access_sccr_set0_l2_el1(WRITE_ACCESS, &reg);
+}

arch/arm64/kernel/include/arch-bitops.h

@@ -0,0 +1,19 @@
+/* arch-bitops.h COPYRIGHT FUJITSU LIMITED 2015-2016 */
+#ifndef __HEADER_ARM64_COMMON_BITOPS_H
+#define __HEADER_ARM64_COMMON_BITOPS_H
+
+#ifndef INCLUDE_BITOPS_H
+# error only <bitops.h> can be included directly
+#endif
+
+#ifndef __ASSEMBLY__
+
+#include "bitops-fls.h"
+#include "bitops-__ffs.h"
+#include "bitops-ffz.h"
+#include "bitops-set_bit.h"
+#include "bitops-clear_bit.h"
+
+#endif /*__ASSEMBLY__*/
+#endif /* !__HEADER_ARM64_COMMON_BITOPS_H */
+

arch/arm64/kernel/include/arch-futex.h

@@ -0,0 +1,156 @@
+/* arch-futex.h COPYRIGHT FUJITSU LIMITED 2015-2018 */
+#ifndef __HEADER_ARM64_COMMON_ARCH_FUTEX_H
+#define __HEADER_ARM64_COMMON_ARCH_FUTEX_H
+
+
+/*
+ * @ref.impl 
+ * 	linux-linaro/arch/arm64/include/asm/futex.h:__futex_atomic_op
+ */
+#define ___futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg)		\
+do {								\
+	asm volatile(							\
+"1:	ldxr	%w1, %2\n"						\
+	insn "\n"							\
+"2:	stlxr	%w3, %w0, %2\n"						\
+"	cbnz	%w3, 1b\n"						\
+"	dmb	ish\n"							\
+"3:\n"									\
+"	.pushsection .fixup,\"ax\"\n"					\
+"	.align	2\n"							\
+"4:	mov	%w0, %w5\n"						\
+"	b	3b\n"							\
+"	.popsection\n"							\
+"	.pushsection __ex_table,\"a\"\n"				\
+"	.align	3\n"							\
+"	.quad	1b, 4b, 2b, 4b\n"					\
+"	.popsection\n"							\
+	: "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp)	\
+	: "r" (oparg), "Ir" (-EFAULT)					\
+	: "memory");						\
+} while (0);
+
+#ifndef IHK_OS_MANYCORE
+#include <linux/uaccess.h>
+
+#define __futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg)		\
+do {								\
+	uaccess_enable();					\
+	___futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg)		\
+	uaccess_disable();					\
+} while (0);
+
+#else
+#define __futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg)		\
+	___futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg)		\
+
+#endif
+
+/*
+ * @ref.impl 
+ * 	linux-linaro/arch/arm64/include/asm/futex.h:futex_atomic_op_inuser
+ */
+static inline int futex_atomic_op_inuser(int encoded_op,
+					 int __user *uaddr)
+{
+	int op = (encoded_op >> 28) & 7;
+	int cmp = (encoded_op >> 24) & 15;
+	int oparg = (encoded_op & 0x00fff000) >> 12;
+	int cmparg = encoded_op & 0xfff;
+	int oldval = 0, ret, tmp;
+
+	if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28))
+		oparg = 1 << oparg;
+
+#ifdef __UACCESS__
+	if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int)))
+		return -EFAULT;
+#endif
+
+	// pagefault_disable();	/* implies preempt_disable() */
+
+	switch (op) {
+	case FUTEX_OP_SET:
+		__futex_atomic_op("mov	%w0, %w4",
+				  ret, oldval, uaddr, tmp, oparg);
+		break;
+	case FUTEX_OP_ADD:
+		__futex_atomic_op("add	%w0, %w1, %w4",
+				  ret, oldval, uaddr, tmp, oparg);
+		break;
+	case FUTEX_OP_OR:
+		__futex_atomic_op("orr	%w0, %w1, %w4",
+				  ret, oldval, uaddr, tmp, oparg);
+		break;
+	case FUTEX_OP_ANDN:
+		__futex_atomic_op("and	%w0, %w1, %w4",
+				  ret, oldval, uaddr, tmp, ~oparg);
+		break;
+	case FUTEX_OP_XOR:
+		__futex_atomic_op("eor	%w0, %w1, %w4",
+				  ret, oldval, uaddr, tmp, oparg);
+		break;
+	default:
+		ret = -ENOSYS;
+	}
+
+	// pagefault_enable();	/* subsumes preempt_enable() */
+
+	if (!ret) {
+		switch (cmp) {
+		case FUTEX_OP_CMP_EQ: ret = (oldval == cmparg); break;
+		case FUTEX_OP_CMP_NE: ret = (oldval != cmparg); break;
+		case FUTEX_OP_CMP_LT: ret = (oldval < cmparg); break;
+		case FUTEX_OP_CMP_GE: ret = (oldval >= cmparg); break;
+		case FUTEX_OP_CMP_LE: ret = (oldval <= cmparg); break;
+		case FUTEX_OP_CMP_GT: ret = (oldval > cmparg); break;
+		default: ret = -ENOSYS;
+		}
+	}
+	return ret;
+}
+
+/*
+ * @ref.impl 
+ * 	linux-linaro/arch/arm64/include/asm/futex.h:futex_atomic_cmpxchg_inatomic
+ * 	mckernel/kernel/include/futex.h:futex_atomic_cmpxchg_inatomic (x86 depend)
+ */
+static inline int
+futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval)
+{
+	int ret = 0;
+	int val, tmp;
+
+	if(uaddr == NULL) {
+		return -EFAULT;
+	}
+#ifdef __UACCESS__
+	if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int))) {
+		return -EFAULT;
+	}
+#endif
+
+	asm volatile("// futex_atomic_cmpxchg_inatomic\n"
+"1:	ldxr	%w1, %2\n"
+"	sub	%w3, %w1, %w4\n"
+"	cbnz	%w3, 3f\n"
+"2:	stlxr	%w3, %w5, %2\n"
+"	cbnz	%w3, 1b\n"
+"	dmb	ish\n"
+"3:\n"
+"	.pushsection .fixup,\"ax\"\n"
+"4:	mov	%w0, %w6\n"
+"	b	3b\n"
+"	.popsection\n"
+"	.pushsection __ex_table,\"a\"\n"
+"	.align	3\n"
+"	.quad	1b, 4b, 2b, 4b\n"
+"	.popsection\n"
+	: "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp)
+	: "r" (oldval), "r" (newval), "Ir" (-EFAULT)
+	: "memory");
+
+	return ret;
+}
+
+#endif /* !__HEADER_ARM64_COMMON_ARCH_FUTEX_H */

arch/arm64/kernel/include/arch-lock.h

@@ -0,0 +1,787 @@
+/* arch-lock.h COPYRIGHT FUJITSU LIMITED 2015-2018 */
+#ifndef __HEADER_ARM64_COMMON_ARCH_LOCK_H
+#define __HEADER_ARM64_COMMON_ARCH_LOCK_H
+
+#define IHK_STATIC_SPINLOCK_FUNCS
+
+#include <ihk/cpu.h>
+#include <ihk/atomic.h>
+#include "affinity.h"
+#include <lwk/compiler.h>
+#include "config.h"
+#ifdef ENABLE_FUGAKU_HACKS
+#include <ihk/debug.h>
+#endif
+
+//#define DEBUG_SPINLOCK
+//#define DEBUG_MCS_RWLOCK
+
+#if defined(DEBUG_SPINLOCK) || defined(DEBUG_MCS_RWLOCK)
+int __kprintf(const char *format, ...);
+#endif
+
+/* @ref.impl arch/arm64/include/asm/spinlock_types.h::TICKET_SHIFT */
+#define TICKET_SHIFT	16
+
+/* @ref.impl arch/arm64/include/asm/spinlock_types.h::arch_spinlock_t */
+typedef struct {
+#ifdef __AARCH64EB__
+	uint16_t next;
+	uint16_t owner;
+#else /* __AARCH64EB__ */
+	uint16_t owner;
+	uint16_t next;
+#endif /* __AARCH64EB__ */
+} __attribute__((aligned(4))) ihk_spinlock_t;
+
+#ifdef ENABLE_FUGAKU_HACKS
+extern ihk_spinlock_t *get_this_cpu_runq_lock(void);
+#endif
+
+extern void preempt_enable(void);
+extern void preempt_disable(void);
+
+/* @ref.impl arch/arm64/include/asm/spinlock_types.h::__ARCH_SPIN_LOCK_UNLOCKED */
+#define SPIN_LOCK_UNLOCKED	{ 0, 0 }
+
+/* @ref.impl arch/arm64/include/asm/barrier.h::__nops */
+#define __nops(n)	".rept	" #n "\nnop\n.endr\n"
+
+/* @ref.impl ./arch/arm64/include/asm/lse.h::ARM64_LSE_ATOMIC_INSN */
+/* else defined(CONFIG_AS_LSE) && defined(CONFIG_ARM64_LSE_ATOMICS) */
+#define ARM64_LSE_ATOMIC_INSN(llsc, lse)	llsc
+
+/* initialized spinlock struct */
+static void ihk_mc_spinlock_init(ihk_spinlock_t *lock)
+{
+	*lock = (ihk_spinlock_t)SPIN_LOCK_UNLOCKED;
+}
+
+#ifdef DEBUG_SPINLOCK
+#define ihk_mc_spinlock_trylock_noirq(l) { \
+	int rc; \
+	__kprintf("[%d] call ihk_mc_spinlock_trylock_noirq %p %s:%d\n", \
+		  ihk_mc_get_processor_id(), (l), __FILE__, __LINE__); \
+	rc = __ihk_mc_spinlock_trylock_noirq(l); \
+	__kprintf("[%d] ret ihk_mc_spinlock_trylock_noirq\n", \
+		  ihk_mc_get_processor_id()); \
+	rc; \
+}
+#else
+#define ihk_mc_spinlock_trylock_noirq __ihk_mc_spinlock_trylock_noirq
+#endif
+
+/* @ref.impl arch/arm64/include/asm/spinlock.h::arch_spin_trylock */
+/* spinlock trylock */
+static int __ihk_mc_spinlock_trylock_noirq(ihk_spinlock_t *lock)
+{
+	unsigned int tmp;
+	ihk_spinlock_t lockval;
+	int success;
+
+	preempt_disable();
+
+	asm volatile(ARM64_LSE_ATOMIC_INSN(
+	/* LL/SC */
+	"	prfm	pstl1strm, %2\n"
+	"1:	ldaxr	%w0, %2\n"
+	"	eor	%w1, %w0, %w0, ror #16\n"
+	"	cbnz	%w1, 2f\n"
+	"	add	%w0, %w0, %3\n"
+	"	stxr	%w1, %w0, %2\n"
+	"	cbnz	%w1, 1b\n"
+	"2:",
+	/* LSE atomics */
+	"	ldr	%w0, %2\n"
+	"	eor	%w1, %w0, %w0, ror #16\n"
+	"	cbnz	%w1, 1f\n"
+	"	add	%w1, %w0, %3\n"
+	"	casa	%w0, %w1, %2\n"
+	"	sub	%w1, %w1, %3\n"
+	"	eor	%w1, %w1, %w0\n"
+	"1:")
+	: "=&r" (lockval), "=&r" (tmp), "+Q" (*lock)
+	: "I" (1 << TICKET_SHIFT)
+	: "memory");
+
+	success = !tmp;
+
+#ifdef ENABLE_FUGAKU_HACKS
+#if 0
+	if (success) {
+		if (get_this_cpu_runq_lock() == lock &&
+				!cpu_interrupt_disabled()) {
+			kprintf("%s: WARNING: runq lock held without IRQs disabled?\n", __func__); \
+		}
+	}
+#endif
+#endif
+
+	if (!success) {
+		preempt_enable();
+	}
+	return success;
+}
+
+#ifdef DEBUG_SPINLOCK
+#define ihk_mc_spinlock_trylock(l, result) ({ \
+	unsigned long rc; \
+	__kprintf("[%d] call ihk_mc_spinlock_trylock %p %s:%d\n", \
+		  ihk_mc_get_processor_id(), (l), __FILE__, __LINE__); \
+	rc = __ihk_mc_spinlock_trylock(l, result); \
+	__kprintf("[%d] ret ihk_mc_spinlock_trylock\n", \
+		  ihk_mc_get_processor_id()); \
+	rc; \
+})
+#else
+#define ihk_mc_spinlock_trylock __ihk_mc_spinlock_trylock
+#endif
+
+/* spinlock trylock & interrupt disable & PSTATE.DAIF save */
+static unsigned long __ihk_mc_spinlock_trylock(ihk_spinlock_t *lock,
+					       int *result)
+{
+	unsigned long flags;
+
+	flags = cpu_disable_interrupt_save();
+
+	*result = __ihk_mc_spinlock_trylock_noirq(lock);
+
+	return flags;
+}
+
+#ifdef DEBUG_SPINLOCK
+#define ihk_mc_spinlock_lock_noirq(l) { \
+__kprintf("[%d] call ihk_mc_spinlock_lock_noirq %p %s:%d\n", ihk_mc_get_processor_id(), (l), __FILE__, __LINE__); \
+__ihk_mc_spinlock_lock_noirq(l); \
+__kprintf("[%d] ret ihk_mc_spinlock_lock_noirq\n", ihk_mc_get_processor_id()); \
+}
+#else
+#define ihk_mc_spinlock_lock_noirq __ihk_mc_spinlock_lock_noirq
+#endif
+
+/* @ref.impl arch/arm64/include/asm/spinlock.h::arch_spin_lock */
+/* spinlock lock */
+static void __ihk_mc_spinlock_lock_noirq(ihk_spinlock_t *lock)
+{
+	unsigned int tmp;
+	ihk_spinlock_t lockval, newval;
+
+	preempt_disable();
+
+	asm volatile(
+	/* Atomically increment the next ticket. */
+	ARM64_LSE_ATOMIC_INSN(
+	/* LL/SC */
+"	prfm	pstl1strm, %3\n"
+"1:	ldaxr	%w0, %3\n"
+"	add	%w1, %w0, %w5\n"
+"	stxr	%w2, %w1, %3\n"
+"	cbnz	%w2, 1b\n",
+	/* LSE atomics */
+"	mov	%w2, %w5\n"
+"	ldadda	%w2, %w0, %3\n"
+	__nops(3)
+	)
+
+	/* Did we get the lock? */
+"	eor	%w1, %w0, %w0, ror #16\n"
+"	cbz	%w1, 3f\n"
+	/*
+	 * No: spin on the owner. Send a local event to avoid missing an
+	 * unlock before the exclusive load.
+	 */
+"	sevl\n"
+"2:	wfe\n"
+"	ldaxrh	%w2, %4\n"
+"	eor	%w1, %w2, %w0, lsr #16\n"
+"	cbnz	%w1, 2b\n"
+	/* We got the lock. Critical section starts here. */
+"3:"
+	: "=&r" (lockval), "=&r" (newval), "=&r" (tmp), "+Q" (*lock)
+	: "Q" (lock->owner), "I" (1 << TICKET_SHIFT)
+	: "memory");
+#ifdef ENABLE_FUGAKU_HACKS
+#if 0
+	if (get_this_cpu_runq_lock() == lock &&
+			!cpu_interrupt_disabled()) {
+		kprintf("%s: WARNING: runq lock held without IRQs disabled?\n", __func__); \
+	}
+#endif
+#endif
+}
+
+#ifdef DEBUG_SPINLOCK
+#define ihk_mc_spinlock_lock(l) ({ unsigned long rc;\
+__kprintf("[%d] call ihk_mc_spinlock_lock %p %s:%d\n", ihk_mc_get_processor_id(), (l), __FILE__, __LINE__); \
+rc = __ihk_mc_spinlock_lock(l);\
+__kprintf("[%d] ret ihk_mc_spinlock_lock\n", ihk_mc_get_processor_id()); rc;\
+})
+#else
+#define ihk_mc_spinlock_lock __ihk_mc_spinlock_lock
+#endif
+
+/* spinlock lock & interrupt disable & PSTATE.DAIF save */
+static unsigned long __ihk_mc_spinlock_lock(ihk_spinlock_t *lock)
+{
+	unsigned long flags;
+	
+	flags = cpu_disable_interrupt_save();
+
+	__ihk_mc_spinlock_lock_noirq(lock);
+
+	return flags;
+}
+
+#ifdef DEBUG_SPINLOCK
+#define ihk_mc_spinlock_unlock_noirq(l) { \
+__kprintf("[%d] call ihk_mc_spinlock_unlock_noirq %p %s:%d\n", ihk_mc_get_processor_id(), (l), __FILE__, __LINE__); \
+__ihk_mc_spinlock_unlock_noirq(l); \
+__kprintf("[%d] ret ihk_mc_spinlock_unlock_noirq\n", ihk_mc_get_processor_id()); \
+}
+#else
+#define ihk_mc_spinlock_unlock_noirq __ihk_mc_spinlock_unlock_noirq
+#endif
+
+/* @ref.impl arch/arm64/include/asm/spinlock.h::arch_spin_unlock */
+/* spinlock unlock */
+static void __ihk_mc_spinlock_unlock_noirq(ihk_spinlock_t *lock)
+{
+	unsigned long tmp;
+
+	asm volatile(ARM64_LSE_ATOMIC_INSN(
+	/* LL/SC */
+	"	ldrh	%w1, %0\n"
+	"	add	%w1, %w1, #1\n"
+	"	stlrh	%w1, %0",
+	/* LSE atomics */
+	"	mov	%w1, #1\n"
+	"	staddlh	%w1, %0\n"
+	__nops(1))
+	: "=Q" (lock->owner), "=&r" (tmp)
+	:
+	: "memory");
+
+	preempt_enable();
+}
+
+/* spinlock unlock & restore PSTATE.DAIF */
+#ifdef DEBUG_SPINLOCK
+#define ihk_mc_spinlock_unlock(l, f) { \
+__kprintf("[%d] call ihk_mc_spinlock_unlock %p %s:%d\n", ihk_mc_get_processor_id(), (l), __FILE__, __LINE__); \
+__ihk_mc_spinlock_unlock((l), (f)); \
+__kprintf("[%d] ret ihk_mc_spinlock_unlock\n", ihk_mc_get_processor_id()); \
+}
+#else
+#define ihk_mc_spinlock_unlock __ihk_mc_spinlock_unlock
+#endif
+static void __ihk_mc_spinlock_unlock(ihk_spinlock_t *lock, unsigned long flags)
+{
+	__ihk_mc_spinlock_unlock_noirq(lock);
+
+	cpu_restore_interrupt(flags);
+}
+
+#define SPINLOCK_IN_MCS_RWLOCK
+
+// reader/writer lock
+typedef struct mcs_rwlock_node {
+	ihk_atomic_t count;	// num of readers (use only common reader)
+	char type;		// lock type
+#define MCS_RWLOCK_TYPE_COMMON_READER 0
+#define MCS_RWLOCK_TYPE_READER 1
+#define MCS_RWLOCK_TYPE_WRITER 2
+	char locked;		// lock
+#define MCS_RWLOCK_LOCKED	1
+#define MCS_RWLOCK_UNLOCKED	0
+	char dmy1;		// unused
+	char dmy2;		// unused
+	struct mcs_rwlock_node *next;
+#ifndef ENABLE_UBSAN
+} __aligned(64) mcs_rwlock_node_t;
+#else
+} mcs_rwlock_node_t;
+#endif
+
+typedef struct mcs_rwlock_node_irqsave {
+#ifndef SPINLOCK_IN_MCS_RWLOCK
+	struct mcs_rwlock_node node;
+#endif
+	unsigned long irqsave;
+#ifndef ENABLE_UBSAN
+} __aligned(64) mcs_rwlock_node_irqsave_t;
+#else
+} mcs_rwlock_node_irqsave_t;
+#endif
+
+typedef struct mcs_rwlock_lock {
+#ifdef SPINLOCK_IN_MCS_RWLOCK
+	ihk_spinlock_t slock;
+#else
+	struct mcs_rwlock_node reader;		/* common reader lock */
+	struct mcs_rwlock_node *node;		/* base */
+#endif
+#ifndef ENABLE_UBSAN
+} __aligned(64) mcs_rwlock_lock_t;
+#else
+} mcs_rwlock_lock_t;
+#endif
+
+static void
+mcs_rwlock_init(struct mcs_rwlock_lock *lock)
+{
+#ifdef SPINLOCK_IN_MCS_RWLOCK
+	ihk_mc_spinlock_init(&lock->slock);
+#else
+	ihk_atomic_set(&lock->reader.count, 0);
+	lock->reader.type = MCS_RWLOCK_TYPE_COMMON_READER;
+	lock->node = NULL;
+#endif
+}
+
+#ifdef DEBUG_MCS_RWLOCK
+#define mcs_rwlock_writer_lock_noirq(l, n) { \
+__kprintf("[%d] call mcs_rwlock_writer_lock_noirq %p %s:%d\n", ihk_mc_get_processor_id(), (l), __FILE__, __LINE__); \
+__mcs_rwlock_writer_lock_noirq((l), (n)); \
+__kprintf("[%d] ret mcs_rwlock_writer_lock_noirq\n", ihk_mc_get_processor_id()); \
+}
+#else
+#define mcs_rwlock_writer_lock_noirq __mcs_rwlock_writer_lock_noirq
+#endif
+static void
+__mcs_rwlock_writer_lock_noirq(struct mcs_rwlock_lock *lock, struct mcs_rwlock_node *node)
+{
+#ifdef SPINLOCK_IN_MCS_RWLOCK
+	ihk_mc_spinlock_lock_noirq(&lock->slock);
+#else
+	struct mcs_rwlock_node *pred;
+
+	preempt_disable();
+
+	node->type = MCS_RWLOCK_TYPE_WRITER;
+	node->next = NULL;
+
+	pred = xchg8(&(lock->node), node);
+
+	if (pred) {
+		node->locked = MCS_RWLOCK_LOCKED;
+		pred->next = node;
+		while (node->locked != MCS_RWLOCK_UNLOCKED) {
+			cpu_pause();
+		}
+	}
+#endif
+}
+
+#ifndef SPINLOCK_IN_MCS_RWLOCK
+static void
+mcs_rwlock_unlock_readers(struct mcs_rwlock_lock *lock)
+{
+	struct mcs_rwlock_node *p;
+	struct mcs_rwlock_node *f = NULL;
+	struct mcs_rwlock_node *n;
+	int breakf = 0;
+
+	ihk_atomic_inc(&lock->reader.count); // protect to unlock reader
+	for(p = &lock->reader; p->next; p = n){
+		n = p->next;
+		if(p->next->type == MCS_RWLOCK_TYPE_READER){
+			p->next = n->next;
+			if(lock->node == n){
+				struct mcs_rwlock_node *old;
+
+				old = atomic_cmpxchg8(&(lock->node), n, p);
+
+				if(old != n){ // couldn't change
+					while (n->next == NULL) {
+						cpu_pause();
+					}
+					p->next = n->next;
+				}
+				else{
+					breakf = 1;
+				}
+			}
+			else if(p->next == NULL){
+				while (n->next == NULL) {
+					cpu_pause();
+				}
+				p->next = n->next;
+			}
+			if(f){
+				ihk_atomic_inc(&lock->reader.count);
+				n->locked = MCS_RWLOCK_UNLOCKED;
+			}
+			else
+				f = n;
+			n = p;
+			if(breakf)
+				break;
+		}
+		if(n->next == NULL && lock->node != n){
+			while (n->next == NULL && lock->node != n) {
+				cpu_pause();
+			}
+		}
+	}
+
+	f->locked = MCS_RWLOCK_UNLOCKED;
+}
+#endif
+
+#ifdef DEBUG_MCS_RWLOCK
+#define mcs_rwlock_writer_unlock_noirq(l, n) { \
+__kprintf("[%d] call mcs_rwlock_writer_unlock_noirq %p %s:%d\n", ihk_mc_get_processor_id(), (l), __FILE__, __LINE__); \
+__mcs_rwlock_writer_unlock_noirq((l), (n)); \
+__kprintf("[%d] ret mcs_rwlock_writer_unlock_noirq\n", ihk_mc_get_processor_id()); \
+}
+#else
+#define mcs_rwlock_writer_unlock_noirq __mcs_rwlock_writer_unlock_noirq
+#endif
+static void
+__mcs_rwlock_writer_unlock_noirq(struct mcs_rwlock_lock *lock, struct mcs_rwlock_node *node)
+{
+#ifdef SPINLOCK_IN_MCS_RWLOCK
+	ihk_mc_spinlock_unlock_noirq(&lock->slock);
+#else
+	if (node->next == NULL) {
+		struct mcs_rwlock_node *old = atomic_cmpxchg8(&(lock->node), node, 0);
+
+		if (old == node) {
+			goto out;
+		}
+
+		while (node->next == NULL) {
+			cpu_pause();
+		}
+	}
+
+	if(node->next->type == MCS_RWLOCK_TYPE_READER){
+		lock->reader.next = node->next;
+		mcs_rwlock_unlock_readers(lock);
+	}
+	else{
+		node->next->locked = MCS_RWLOCK_UNLOCKED;
+	}
+
+out:
+	preempt_enable();
+#endif
+}
+
+#ifdef DEBUG_MCS_RWLOCK
+#define mcs_rwlock_reader_lock_noirq(l, n) { \
+__kprintf("[%d] call mcs_rwlock_reader_lock_noirq %p %s:%d\n", ihk_mc_get_processor_id(), (l), __FILE__, __LINE__); \
+__mcs_rwlock_reader_lock_noirq((l), (n)); \
+__kprintf("[%d] ret mcs_rwlock_reader_lock_noirq\n", ihk_mc_get_processor_id()); \
+}
+#else
+#define mcs_rwlock_reader_lock_noirq __mcs_rwlock_reader_lock_noirq
+#endif
+
+static inline unsigned int
+atomic_inc_ifnot0(ihk_atomic_t *v)
+{
+	unsigned int *p = (unsigned int *)(&(v)->counter);
+	unsigned int old;
+	unsigned int new;
+	unsigned int val;
+
+	do{
+		if(!(old = *p))
+			break;
+		new = old + 1;
+		val = atomic_cmpxchg4(p, old, new);
+	}while(val != old);
+	return old;
+}
+
+static void
+__mcs_rwlock_reader_lock_noirq(struct mcs_rwlock_lock *lock, struct mcs_rwlock_node *node)
+{
+#ifdef SPINLOCK_IN_MCS_RWLOCK
+	ihk_mc_spinlock_lock_noirq(&lock->slock);
+#else
+	struct mcs_rwlock_node *pred;
+
+	preempt_disable();
+
+	node->type = MCS_RWLOCK_TYPE_READER;
+	node->next = NULL;
+	node->dmy1 = ihk_mc_get_processor_id();
+
+	pred = xchg8(&(lock->node), node);
+
+	if (pred) {
+		if(pred == &lock->reader){
+			if(atomic_inc_ifnot0(&pred->count)){
+				struct mcs_rwlock_node *old;
+
+				old = atomic_cmpxchg8(&(lock->node), node, pred);
+
+				if (old == node) {
+					goto out;
+				}
+
+				while (node->next == NULL) {
+					cpu_pause();
+				}
+
+				node->locked = MCS_RWLOCK_LOCKED;
+				lock->reader.next = node;
+				mcs_rwlock_unlock_readers(lock);
+				ihk_atomic_dec(&pred->count);
+				goto out;
+			}
+		}
+		node->locked = MCS_RWLOCK_LOCKED;
+		pred->next = node;
+		while (node->locked != MCS_RWLOCK_UNLOCKED) {
+			cpu_pause();
+		}
+	}
+	else {
+		lock->reader.next = node;
+		mcs_rwlock_unlock_readers(lock);
+	}
+out:
+	return;
+#endif
+}
+
+#ifdef DEBUG_MCS_RWLOCK
+#define mcs_rwlock_reader_unlock_noirq(l, n) { \
+__kprintf("[%d] call mcs_rwlock_reader_unlock_noirq %p %s:%d\n", ihk_mc_get_processor_id(), (l), __FILE__, __LINE__); \
+__mcs_rwlock_reader_unlock_noirq((l), (n)); \
+__kprintf("[%d] ret mcs_rwlock_reader_unlock_noirq\n", ihk_mc_get_processor_id()); \
+}
+#else
+#define mcs_rwlock_reader_unlock_noirq __mcs_rwlock_reader_unlock_noirq
+#endif
+static void
+__mcs_rwlock_reader_unlock_noirq(struct mcs_rwlock_lock *lock, struct mcs_rwlock_node *node)
+{
+#ifdef SPINLOCK_IN_MCS_RWLOCK
+	ihk_mc_spinlock_unlock_noirq(&lock->slock);
+#else
+	if(ihk_atomic_dec_return(&lock->reader.count))
+		goto out;
+
+	if (lock->reader.next == NULL) {
+		struct mcs_rwlock_node *old;
+
+		old = atomic_cmpxchg8(&(lock->node), &(lock->reader), 0);
+
+		if (old == &lock->reader) {
+			goto out;
+		}
+
+		while (lock->reader.next == NULL) {
+			cpu_pause();
+		}
+	}
+
+	if(lock->reader.next->type == MCS_RWLOCK_TYPE_READER){
+		mcs_rwlock_unlock_readers(lock);
+	}
+	else{
+		lock->reader.next->locked = MCS_RWLOCK_UNLOCKED;
+	}
+
+out:
+	preempt_enable();
+#endif
+}
+
+#ifdef DEBUG_MCS_RWLOCK
+#define mcs_rwlock_writer_lock(l, n) { \
+__kprintf("[%d] call mcs_rwlock_writer_lock %p %s:%d\n", ihk_mc_get_processor_id(), (l), __FILE__, __LINE__); \
+__mcs_rwlock_writer_lock((l), (n)); \
+__kprintf("[%d] ret mcs_rwlock_writer_lock\n", ihk_mc_get_processor_id()); \
+}
+#else
+#define mcs_rwlock_writer_lock __mcs_rwlock_writer_lock
+#endif
+static void
+__mcs_rwlock_writer_lock(struct mcs_rwlock_lock *lock, struct mcs_rwlock_node_irqsave *node)
+{
+#ifdef SPINLOCK_IN_MCS_RWLOCK
+	node->irqsave = ihk_mc_spinlock_lock(&lock->slock);
+#else
+	node->irqsave = cpu_disable_interrupt_save();
+	__mcs_rwlock_writer_lock_noirq(lock, &node->node);
+#endif
+}
+
+#ifdef DEBUG_MCS_RWLOCK
+#define mcs_rwlock_writer_unlock(l, n) { \
+__kprintf("[%d] call mcs_rwlock_writer_unlock %p %s:%d\n", ihk_mc_get_processor_id(), (l), __FILE__, __LINE__); \
+__mcs_rwlock_writer_unlock((l), (n)); \
+__kprintf("[%d] ret mcs_rwlock_writer_unlock\n", ihk_mc_get_processor_id()); \
+}
+#else
+#define mcs_rwlock_writer_unlock __mcs_rwlock_writer_unlock
+#endif
+static void
+__mcs_rwlock_writer_unlock(struct mcs_rwlock_lock *lock, struct mcs_rwlock_node_irqsave *node)
+{
+#ifdef SPINLOCK_IN_MCS_RWLOCK
+	ihk_mc_spinlock_unlock(&lock->slock, node->irqsave);
+#else
+	__mcs_rwlock_writer_unlock_noirq(lock, &node->node);
+	cpu_restore_interrupt(node->irqsave);
+#endif
+}
+
+#ifdef DEBUG_MCS_RWLOCK
+#define mcs_rwlock_reader_lock(l, n) { \
+__kprintf("[%d] call mcs_rwlock_reader_lock %p %s:%d\n", ihk_mc_get_processor_id(), (l), __FILE__, __LINE__); \
+__mcs_rwlock_reader_lock((l), (n)); \
+__kprintf("[%d] ret mcs_rwlock_reader_lock\n", ihk_mc_get_processor_id()); \
+}
+#else
+#define mcs_rwlock_reader_lock __mcs_rwlock_reader_lock
+#endif
+static void
+__mcs_rwlock_reader_lock(struct mcs_rwlock_lock *lock, struct mcs_rwlock_node_irqsave *node)
+{
+#ifdef SPINLOCK_IN_MCS_RWLOCK
+	node->irqsave = ihk_mc_spinlock_lock(&lock->slock);
+#else
+	node->irqsave = cpu_disable_interrupt_save();
+	__mcs_rwlock_reader_lock_noirq(lock, &node->node);
+#endif
+}
+
+#ifdef DEBUG_MCS_RWLOCK
+#define mcs_rwlock_reader_unlock(l, n) { \
+__kprintf("[%d] call mcs_rwlock_reader_unlock %p %s:%d\n", ihk_mc_get_processor_id(), (l), __FILE__, __LINE__); \
+__mcs_rwlock_reader_unlock((l), (n)); \
+__kprintf("[%d] ret mcs_rwlock_reader_unlock\n", ihk_mc_get_processor_id()); \
+}
+#else
+#define mcs_rwlock_reader_unlock __mcs_rwlock_reader_unlock
+#endif
+static void
+__mcs_rwlock_reader_unlock(struct mcs_rwlock_lock *lock, struct mcs_rwlock_node_irqsave *node)
+{
+#ifdef SPINLOCK_IN_MCS_RWLOCK
+	ihk_mc_spinlock_unlock(&lock->slock, node->irqsave);
+#else
+	__mcs_rwlock_reader_unlock_noirq(lock, &node->node);
+	cpu_restore_interrupt(node->irqsave);
+#endif
+}
+
+#if defined(CONFIG_HAS_NMI)
+#include <arm-gic-v3.h>
+static inline int irqflags_can_interrupt(unsigned long flags)
+{
+	return (flags == ICC_PMR_EL1_UNMASKED);
+}
+#else /* CONFIG_HAS_NMI */
+static inline int irqflags_can_interrupt(unsigned long flags)
+{
+	return !(flags & 0x2);
+}
+#endif /* CONFIG_HAS_NMI */
+
+struct ihk_rwlock {
+	unsigned int lock;
+};
+
+static inline void ihk_mc_rwlock_init(struct ihk_rwlock *rw)
+{
+	rw->lock = 0;
+}
+
+static inline void ihk_mc_read_lock(struct ihk_rwlock *rw)
+{
+	unsigned int tmp, tmp2;
+
+	asm volatile(
+	"       sevl\n"
+	"1:     wfe\n"
+	"2:     ldaxr   %w0, %2\n"
+	"       add     %w0, %w0, #1\n"
+	"       tbnz    %w0, #31, 1b\n"
+	"       stxr    %w1, %w0, %2\n"
+	"       cbnz    %w1, 2b\n"
+	: "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock)
+	:
+	: "cc", "memory");
+}
+
+static inline int ihk_mc_read_trylock(struct ihk_rwlock *rw)
+{
+	unsigned int tmp, tmp2 = 1;
+
+	asm volatile(
+	"       ldaxr   %w0, %2\n"
+	"       add     %w0, %w0, #1\n"
+	"       tbnz    %w0, #31, 1f\n"
+	"       stxr    %w1, %w0, %2\n"
+	"1:\n"
+	: "=&r" (tmp), "+r" (tmp2), "+Q" (rw->lock)
+	:
+	: "cc", "memory");
+
+	return !tmp2;
+}
+
+static inline void ihk_mc_read_unlock(struct ihk_rwlock *rw)
+{
+	unsigned int tmp, tmp2;
+
+	asm volatile(
+	"1:     ldxr    %w0, %2\n"
+	"       sub     %w0, %w0, #1\n"
+	"       stlxr   %w1, %w0, %2\n"
+	"       cbnz    %w1, 1b\n"
+	: "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock)
+	:
+	: "cc", "memory");
+}
+
+static inline void ihk_mc_write_lock(struct ihk_rwlock *rw)
+{
+	unsigned int tmp;
+
+	asm volatile(
+	"       sevl\n"
+	"1:     wfe\n"
+	"2:     ldaxr   %w0, %1\n"
+	"       cbnz    %w0, 1b\n"
+	"       stxr    %w0, %w2, %1\n"
+	"       cbnz    %w0, 2b\n"
+	: "=&r" (tmp), "+Q" (rw->lock)
+	: "r" (0x80000000)
+	: "cc", "memory");
+}
+
+static inline int ihk_mc_write_trylock(struct ihk_rwlock *rw)
+{
+	unsigned int tmp;
+
+	asm volatile(
+	"       ldaxr   %w0, %1\n"
+	"       cbnz    %w0, 1f\n"
+	"       stxr    %w0, %w2, %1\n"
+	"1:\n"
+	: "=&r" (tmp), "+Q" (rw->lock)
+	: "r" (0x80000000)
+	: "cc", "memory");
+
+	return !tmp;
+}
+
+static inline void ihk_mc_write_unlock(struct ihk_rwlock *rw)
+{
+	asm volatile(
+	"       stlr    %w1, %0\n"
+	: "=Q" (rw->lock) : "r" (0) : "memory");
+}
+
+#define ihk_mc_read_can_lock(rw) ((rw)->lock < 0x80000000)
+#define ihk_mc_write_can_lock(rw) ((rw)->lock == 0)
+#endif /* !__HEADER_ARM64_COMMON_ARCH_LOCK_H */

arch/arm64/kernel/include/arch-memory.h

@@ -0,0 +1,852 @@
+/* arch-memory.h COPYRIGHT FUJITSU LIMITED 2015-2018 */
+#ifndef __HEADER_ARM64_COMMON_ARCH_MEMORY_H
+#define __HEADER_ARM64_COMMON_ARCH_MEMORY_H
+
+#include <const.h>
+#include <errno.h>
+
+#ifndef __ASSEMBLY__
+#include <list.h>
+#include <page.h>
+void panic(const char *);
+#endif  /*__ASSEMBLY__*/
+
+#define _SZ4KB  (1UL<<12)
+#define _SZ16KB (1UL<<14)
+#define _SZ64KB (1UL<<16)
+
+#ifdef CONFIG_ARM64_64K_PAGES
+# define GRANULE_SIZE	_SZ64KB
+# define BLOCK_SHIFT	PAGE_SHIFT
+# define BLOCK_SIZE	PAGE_SIZE
+# define TABLE_SHIFT	PMD_SHIFT
+#else
+# define GRANULE_SIZE	_SZ4KB
+# define BLOCK_SHIFT	SECTION_SHIFT
+# define BLOCK_SIZE	SECTION_SIZE
+# define TABLE_SHIFT	PUD_SHIFT
+#endif
+
+#define VA_BITS		CONFIG_ARM64_VA_BITS
+
+/*
+ * Address define
+ */
+/* early alloc area address */
+/* START:_end, SIZE:512 pages */
+#define MAP_EARLY_ALLOC_SHIFT	5
+#define MAP_EARLY_ALLOC_SIZE	(UL(1) << (PAGE_SHIFT + MAP_EARLY_ALLOC_SHIFT))
+
+#ifndef __ASSEMBLY__
+# define ALIGN_UP(x, align)     ALIGN_DOWN((x) + (align) - 1, align)
+# define ALIGN_DOWN(x, align)   ((x) & ~((align) - 1))
+extern char _end[];
+# define MAP_EARLY_ALLOC	(ALIGN_UP((unsigned long)_end, BLOCK_SIZE))
+# define MAP_EARLY_ALLOC_END	(MAP_EARLY_ALLOC + MAP_EARLY_ALLOC_SIZE)
+#endif /* !__ASSEMBLY__ */
+
+/* bootparam area address */
+/* START:early alloc area end, SIZE:2MiB */
+#define MAP_BOOT_PARAM_SHIFT	21
+#define MAP_BOOT_PARAM_SIZE	(UL(1) << MAP_BOOT_PARAM_SHIFT)
+
+#ifndef __ASSEMBLY__
+# define MAP_BOOT_PARAM		(ALIGN_UP(MAP_EARLY_ALLOC_END, BLOCK_SIZE))
+# define MAP_BOOT_PARAM_END	(MAP_BOOT_PARAM + MAP_BOOT_PARAM_SIZE)
+#endif /* !__ASSEMBLY__ */
+
+/*
+ * MAP_KERNEL_START is HOST MODULES_END - 8MiB.
+ * It's defined by cmake.
+ */
+#if (VA_BITS == 39 && GRANULE_SIZE == _SZ4KB) /* ARM64_MEMORY_LAYOUT=1 */
+#
+# define LD_TASK_UNMAPPED_BASE	UL(0x0000000400000000)
+# define TASK_UNMAPPED_BASE	UL(0x0000000800000000)
+# define USER_END		UL(0x0000002000000000)
+# define MAP_VMAP_START		UL(0xffffffbdc0000000)
+# define MAP_VMAP_SIZE		UL(0x0000000100000000)
+# define MAP_FIXED_START	UL(0xffffffbffbdfd000)
+# define MAP_ST_START		UL(0xffffffc000000000)
+#
+#elif (VA_BITS == 42 && GRANULE_SIZE == _SZ64KB) /* ARM64_MEMORY_LAYOUT=3 */
+#
+# define LD_TASK_UNMAPPED_BASE	UL(0x0000002000000000)
+# define TASK_UNMAPPED_BASE	UL(0x0000004000000000)
+# define USER_END		UL(0x0000010000000000)
+# define MAP_VMAP_START		UL(0xfffffdfee0000000)
+# define MAP_VMAP_SIZE		UL(0x0000000100000000)
+# define MAP_FIXED_START	UL(0xfffffdfffbdd0000)
+# define MAP_ST_START		UL(0xfffffe0000000000)
+#
+#elif (VA_BITS == 48 && GRANULE_SIZE == _SZ4KB) /* ARM64_MEMORY_LAYOUT=2 */
+#
+# define LD_TASK_UNMAPPED_BASE	UL(0x0000080000000000)
+# define TASK_UNMAPPED_BASE	UL(0x0000100000000000)
+# define USER_END		UL(0x0000400000000000)
+# define MAP_VMAP_START		UL(0xffff7bffc0000000)
+# define MAP_VMAP_SIZE		UL(0x0000000100000000)
+# define MAP_FIXED_START	UL(0xffff7ffffbdfd000)
+# define MAP_ST_START		UL(0xffff800000000000)
+#
+#elif (VA_BITS == 48 && GRANULE_SIZE == _SZ64KB) /* ARM64_MEMORY_LAYOUT=4 */
+#
+# define LD_TASK_UNMAPPED_BASE	UL(0x0000080000000000)
+# define TASK_UNMAPPED_BASE	UL(0x0000100000000000)
+# define USER_END		UL(0x0000400000000000)
+#ifdef ENABLE_TOFU
+# define MAP_VMAP_START		UL(0xffff7bdfffff0000)
+#else
+# define MAP_VMAP_START		UL(0xffff780000000000)
+#endif
+# define MAP_VMAP_SIZE		UL(0x0000000100000000)
+# define MAP_FIXED_START	UL(0xffff7ffffbdd0000)
+# define MAP_ST_START		UL(0xffff800000000000)
+#
+#else
+# error address space is not defined.
+#endif
+
+#define MAP_ST_SIZE		(MAP_KERNEL_START - MAP_ST_START)
+#define STACK_TOP(region)	((region)->user_end)
+
+/*
+ * pagetable define
+ */
+#if GRANULE_SIZE == _SZ4KB
+# define __PTL4_SHIFT  39
+# define __PTL3_SHIFT  30
+# define __PTL2_SHIFT  21
+# define __PTL1_SHIFT  12
+# define PTL4_INDEX_MASK ((UL(1) << 9) - 1)
+# define PTL3_INDEX_MASK PTL4_INDEX_MASK
+# define PTL2_INDEX_MASK PTL3_INDEX_MASK
+# define PTL1_INDEX_MASK PTL2_INDEX_MASK
+# define __PTL4_CONT_SHIFT (__PTL4_SHIFT + 0)
+# define __PTL3_CONT_SHIFT (__PTL3_SHIFT + 4)
+# define __PTL2_CONT_SHIFT (__PTL2_SHIFT + 4)
+# define __PTL1_CONT_SHIFT (__PTL1_SHIFT + 4)
+#elif GRANULE_SIZE == _SZ16KB
+# define __PTL4_SHIFT  47
+# define __PTL3_SHIFT  36
+# define __PTL2_SHIFT  25
+# define __PTL1_SHIFT  14
+# define PTL4_INDEX_MASK ((UL(1) << 1) - 1)
+# define PTL3_INDEX_MASK ((UL(1) << 11) - 1)
+# define PTL2_INDEX_MASK PTL3_INDEX_MASK
+# define PTL1_INDEX_MASK PTL2_INDEX_MASK
+# define __PTL4_CONT_SHIFT (__PTL4_SHIFT + 0)
+# define __PTL3_CONT_SHIFT (__PTL3_SHIFT + 0)
+# define __PTL2_CONT_SHIFT (__PTL2_SHIFT + 5)
+# define __PTL1_CONT_SHIFT (__PTL1_SHIFT + 7)
+#elif GRANULE_SIZE == _SZ64KB
+# define __PTL4_SHIFT  55
+# define __PTL3_SHIFT  42
+# define __PTL2_SHIFT  29
+# define __PTL1_SHIFT  16
+# define PTL4_INDEX_MASK 0
+# define PTL3_INDEX_MASK ((UL(1) << 6) - 1)
+# define PTL3_INDEX_MASK_LINUX ((UL(1) << 10) - 1)
+# define PTL2_INDEX_MASK ((UL(1) << 13) - 1)
+# define PTL1_INDEX_MASK PTL2_INDEX_MASK
+# define __PTL4_CONT_SHIFT (__PTL4_SHIFT + 0)
+# define __PTL3_CONT_SHIFT (__PTL3_SHIFT + 0)
+# define __PTL2_CONT_SHIFT (__PTL2_SHIFT + 5)
+# define __PTL1_CONT_SHIFT (__PTL1_SHIFT + 5)
+#else
+# error granule size error.
+#endif
+
+#ifndef __ASSEMBLY__
+extern int first_level_block_support;
+#endif /* __ASSEMBLY__ */
+
+# define __PTL4_SIZE  (UL(1) << __PTL4_SHIFT)
+# define __PTL3_SIZE  (UL(1) << __PTL3_SHIFT)
+# define __PTL2_SIZE  (UL(1) << __PTL2_SHIFT)
+# define __PTL1_SIZE  (UL(1) << __PTL1_SHIFT)
+# define __PTL4_MASK  (~(__PTL4_SIZE - 1))
+# define __PTL3_MASK  (~(__PTL3_SIZE - 1))
+# define __PTL2_MASK  (~(__PTL2_SIZE - 1))
+# define __PTL1_MASK  (~(__PTL1_SIZE - 1))
+
+# define __PTL4_CONT_SIZE  (UL(1) << __PTL4_CONT_SHIFT)
+# define __PTL3_CONT_SIZE  (UL(1) << __PTL3_CONT_SHIFT)
+# define __PTL2_CONT_SIZE  (UL(1) << __PTL2_CONT_SHIFT)
+# define __PTL1_CONT_SIZE  (UL(1) << __PTL1_CONT_SHIFT)
+# define __PTL4_CONT_MASK  (~(__PTL4_CONT_SIZE - 1))
+# define __PTL3_CONT_MASK  (~(__PTL3_CONT_SIZE - 1))
+# define __PTL2_CONT_MASK  (~(__PTL2_CONT_SIZE - 1))
+# define __PTL1_CONT_MASK  (~(__PTL1_CONT_SIZE - 1))
+# define __PTL4_CONT_COUNT  (UL(1) << (__PTL4_CONT_SHIFT - __PTL4_SHIFT))
+# define __PTL3_CONT_COUNT  (UL(1) << (__PTL3_CONT_SHIFT - __PTL3_SHIFT))
+# define __PTL2_CONT_COUNT  (UL(1) << (__PTL2_CONT_SHIFT - __PTL2_SHIFT))
+# define __PTL1_CONT_COUNT  (UL(1) << (__PTL1_CONT_SHIFT - __PTL1_SHIFT))
+
+/* calculate entries */
+#if (CONFIG_ARM64_PGTABLE_LEVELS > 3) && (VA_BITS > __PTL4_SHIFT)
+# define __PTL4_ENTRIES  (UL(1) << (VA_BITS - __PTL4_SHIFT))
+# define __PTL3_ENTRIES  (UL(1) << (__PTL1_SHIFT - 3))
+# define __PTL2_ENTRIES  (UL(1) << (__PTL1_SHIFT - 3))
+# define __PTL1_ENTRIES  (UL(1) << (__PTL1_SHIFT - 3))
+#elif (CONFIG_ARM64_PGTABLE_LEVELS > 2) && (VA_BITS > __PTL3_SHIFT)
+# define __PTL4_ENTRIES  1
+# define __PTL3_ENTRIES  (UL(1) << (VA_BITS - __PTL3_SHIFT))
+# define __PTL2_ENTRIES  (UL(1) << (__PTL1_SHIFT - 3))
+# define __PTL1_ENTRIES  (UL(1) << (__PTL1_SHIFT - 3))
+#elif (CONFIG_ARM64_PGTABLE_LEVELS > 1) && (VA_BITS > __PTL2_SHIFT)
+# define __PTL4_ENTRIES  1
+# define __PTL3_ENTRIES  1
+# define __PTL2_ENTRIES  (UL(1) << (VA_BITS - __PTL2_SHIFT))
+# define __PTL1_ENTRIES  (UL(1) << (__PTL1_SHIFT - 3))
+#elif VA_BITS > __PTL1_SHIFT
+# define __PTL4_ENTRIES  1
+# define __PTL3_ENTRIES  1
+# define __PTL2_ENTRIES  1
+# define __PTL1_ENTRIES  (UL(1) << (VA_BITS - __PTL1_SHIFT))
+#else
+# define __PTL4_ENTRIES  1
+# define __PTL3_ENTRIES  1
+# define __PTL2_ENTRIES  1
+# define __PTL1_ENTRIES  1
+#endif
+
+#ifndef __ASSEMBLY__
+static const unsigned int  PTL4_SHIFT   = __PTL4_SHIFT;
+static const unsigned int  PTL3_SHIFT   = __PTL3_SHIFT;
+static const unsigned int  PTL2_SHIFT   = __PTL2_SHIFT;
+static const unsigned int  PTL1_SHIFT   = __PTL1_SHIFT;
+static const unsigned long PTL4_SIZE    = __PTL4_SIZE; 
+static const unsigned long PTL3_SIZE    = __PTL3_SIZE; 
+static const unsigned long PTL2_SIZE    = __PTL2_SIZE; 
+static const unsigned long PTL1_SIZE    = __PTL1_SIZE; 
+static const unsigned long PTL4_MASK    = __PTL4_MASK;
+static const unsigned long PTL3_MASK    = __PTL3_MASK;
+static const unsigned long PTL2_MASK    = __PTL2_MASK;
+static const unsigned long PTL1_MASK    = __PTL1_MASK;
+static const unsigned int  PTL4_ENTRIES = __PTL4_ENTRIES;
+static const unsigned int  PTL3_ENTRIES = __PTL3_ENTRIES;
+static const unsigned int  PTL2_ENTRIES = __PTL2_ENTRIES;
+static const unsigned int  PTL1_ENTRIES = __PTL1_ENTRIES;
+static const unsigned int  PTL4_CONT_SHIFT = __PTL4_CONT_SHIFT;
+static const unsigned int  PTL3_CONT_SHIFT = __PTL3_CONT_SHIFT;
+static const unsigned int  PTL2_CONT_SHIFT = __PTL2_CONT_SHIFT;
+static const unsigned int  PTL1_CONT_SHIFT = __PTL1_CONT_SHIFT;
+static const unsigned long PTL4_CONT_SIZE  = __PTL4_CONT_SIZE;
+static const unsigned long PTL3_CONT_SIZE  = __PTL3_CONT_SIZE;
+static const unsigned long PTL2_CONT_SIZE  = __PTL2_CONT_SIZE;
+static const unsigned long PTL1_CONT_SIZE  = __PTL1_CONT_SIZE;
+static const unsigned long PTL4_CONT_MASK  = __PTL4_CONT_MASK;
+static const unsigned long PTL3_CONT_MASK  = __PTL3_CONT_MASK;
+static const unsigned long PTL2_CONT_MASK  = __PTL2_CONT_MASK;
+static const unsigned long PTL1_CONT_MASK  = __PTL1_CONT_MASK;
+static const unsigned int  PTL4_CONT_COUNT = __PTL4_CONT_COUNT;
+static const unsigned int  PTL3_CONT_COUNT = __PTL3_CONT_COUNT;
+static const unsigned int  PTL2_CONT_COUNT = __PTL2_CONT_COUNT;
+static const unsigned int  PTL1_CONT_COUNT = __PTL1_CONT_COUNT;
+#else
+# define PTL4_SHIFT   __PTL4_SHIFT
+# define PTL3_SHIFT   __PTL3_SHIFT
+# define PTL2_SHIFT   __PTL2_SHIFT
+# define PTL1_SHIFT   __PTL1_SHIFT
+# define PTL4_SIZE    __PTL4_SIZE
+# define PTL3_SIZE    __PTL3_SIZE
+# define PTL2_SIZE    __PTL2_SIZE
+# define PTL1_SIZE    __PTL1_SIZE
+# define PTL4_MASK    __PTL4_MASK
+# define PTL3_MASK    __PTL3_MASK
+# define PTL2_MASK    __PTL2_MASK
+# define PTL1_MASK    __PTL1_MASK
+# define PTL4_ENTRIES __PTL4_ENTRIES
+# define PTL3_ENTRIES __PTL3_ENTRIES
+# define PTL2_ENTRIES __PTL2_ENTRIES
+# define PTL1_ENTRIES __PTL1_ENTRIES
+# define PTL4_CONT_SHIFT __PTL4_CONT_SHIFT
+# define PTL3_CONT_SHIFT __PTL3_CONT_SHIFT
+# define PTL2_CONT_SHIFT __PTL2_CONT_SHIFT
+# define PTL1_CONT_SHIFT __PTL1_CONT_SHIFT
+# define PTL4_CONT_SIZE __PTL4_CONT_SIZE
+# define PTL3_CONT_SIZE __PTL3_CONT_SIZE
+# define PTL2_CONT_SIZE __PTL2_CONT_SIZE
+# define PTL1_CONT_SIZE __PTL1_CONT_SIZE
+# define PTL4_CONT_MASK __PTL4_CONT_MASK
+# define PTL3_CONT_MASK __PTL3_CONT_MASK
+# define PTL2_CONT_MASK __PTL2_CONT_MASK
+# define PTL1_CONT_MASK __PTL1_CONT_MASK
+# define PTL4_CONT_COUNT __PTL4_CONT_COUNT
+# define PTL3_CONT_COUNT __PTL3_CONT_COUNT
+# define PTL2_CONT_COUNT __PTL2_CONT_COUNT
+# define PTL1_CONT_COUNT __PTL1_CONT_COUNT
+#endif/*__ASSEMBLY__*/
+
+#define __page_size(pgshift)        (UL(1) << (pgshift))
+#define __page_mask(pgsize)         (~((pgsize) - 1))
+#define __page_offset(addr, size)   ((unsigned long)(addr) & ((size) - 1))
+#define __page_align(addr, size)    ((unsigned long)(addr) & ~((size) - 1))
+#define __page_align_up(addr, size) __page_align((unsigned long)(addr) + (size) - 1, size)
+
+/*
+ * nornal page
+ */
+#define PAGE_SHIFT          __PTL1_SHIFT
+#define PAGE_SIZE           __page_size(PAGE_SHIFT)
+#define PAGE_MASK           __page_mask(PAGE_SIZE)
+#define PAGE_P2ALIGN        0
+#define page_offset(addr)   __page_offset(addr, PAGE_SIZE)
+#define page_align(addr)    __page_align(addr, PAGE_SIZE)
+#define page_align_up(addr) __page_align_up(addr, PAGE_SIZE)
+
+/*
+ * large page
+ */
+#define LARGE_PAGE_SHIFT          __PTL2_SHIFT
+#define LARGE_PAGE_SIZE           __page_size(LARGE_PAGE_SHIFT)
+#define LARGE_PAGE_MASK           __page_mask(LARGE_PAGE_SIZE)
+#define LARGE_PAGE_P2ALIGN        (LARGE_PAGE_SHIFT - PAGE_SHIFT)
+#define large_page_offset(addr)   __page_offset(addr, LARGE_PAGE_SIZE)
+#define large_page_align(addr)    __page_align(addr, LARGE_PAGE_SIZE)
+#define large_page_align_up(addr) __page_align_up(addr, LARGE_PAGE_SIZE)
+
+/*
+ *
+ */
+#define TTBR_ASID_SHIFT  48
+#define TTBR_ASID_MASK   (0xFFFFUL << TTBR_ASID_SHIFT)
+#define TTBR_BADDR_MASK  (~TTBR_ASID_MASK)
+
+#include "pgtable-hwdef.h"
+
+#define KERNEL_PHYS_OFFSET
+
+#define PT_PHYSMASK PHYS_MASK
+/* We allow user programs to access all the memory (D_Block, D_Page) */
+#define PFL_KERN_BLK_ATTR		PROT_SECT_NORMAL_EXEC
+#define PFL_KERN_PAGE_ATTR		PAGE_KERNEL_EXEC
+/* for the page table entry that points another page table (D_Table) */
+#define PFL_PDIR_TBL_ATTR		PMD_TYPE_TABLE
+
+#ifdef CONFIG_ARM64_64K_PAGES
+# define SWAPPER_PGTABLE_LEVELS	(CONFIG_ARM64_PGTABLE_LEVELS)
+#else
+# define SWAPPER_PGTABLE_LEVELS	(CONFIG_ARM64_PGTABLE_LEVELS - 1)
+#endif
+#define SWAPPER_DIR_SIZE	(SWAPPER_PGTABLE_LEVELS * PAGE_SIZE)
+#define IDMAP_DIR_SIZE		(3 * PAGE_SIZE)
+
+/* [Page level Write Throgh] ページキャッシュ方式  0:ライトバック 1:ライトスルー */
+#define PFL1_PWT		0 //< DEBUG_ARCH_DEP, devobj.cの直接参照を関数化 (is_pte_pwd)
+/* [Page level Cache Disable] ページキャッシュ 0:有効 1:無効 */
+#define PFL1_PCD		0 //< DEBUG_ARCH_DEP, devobj.cの直接参照を関数化 (is_pte_pcd)
+
+#define	PTE_NULL		(0)
+
+#define PTE_FILEOFF		PTE_SPECIAL
+
+#ifdef CONFIG_ARM64_64K_PAGES
+# define USER_STACK_PREPAGE_SIZE	PAGE_SIZE
+# define USER_STACK_PAGE_MASK		PAGE_MASK
+# define USER_STACK_PAGE_P2ALIGN	PAGE_P2ALIGN
+# define USER_STACK_PAGE_SHIFT		PAGE_SHIFT
+#else
+# define USER_STACK_PREPAGE_SIZE	LARGE_PAGE_SIZE
+# define USER_STACK_PAGE_MASK		LARGE_PAGE_MASK
+# define USER_STACK_PAGE_P2ALIGN	LARGE_PAGE_P2ALIGN
+# define USER_STACK_PAGE_SHIFT		LARGE_PAGE_SHIFT
+#endif
+
+#define PT_ENTRIES		(PAGE_SIZE >> 3)
+
+#ifndef __ASSEMBLY__
+
+#include <ihk/types.h>
+
+typedef unsigned long pte_t;
+
+/*
+ * pagemap kernel ABI bits
+ */
+#define PM_ENTRY_BYTES      sizeof(uint64_t)
+#define PM_STATUS_BITS      3
+#define PM_STATUS_OFFSET    (64 - PM_STATUS_BITS)
+#define PM_STATUS_MASK      (((1LL << PM_STATUS_BITS) - 1) << PM_STATUS_OFFSET)
+#define PM_STATUS(nr)       (((nr) << PM_STATUS_OFFSET) & PM_STATUS_MASK)
+#define PM_PSHIFT_BITS      6
+#define PM_PSHIFT_OFFSET    (PM_STATUS_OFFSET - PM_PSHIFT_BITS)
+#define PM_PSHIFT_MASK      (((1LL << PM_PSHIFT_BITS) - 1) << PM_PSHIFT_OFFSET)
+#define PM_PSHIFT(x)        (((uint64_t) (x) << PM_PSHIFT_OFFSET) & PM_PSHIFT_MASK)
+#define PM_PFRAME_MASK      ((1LL << PM_PSHIFT_OFFSET) - 1)
+#define PM_PFRAME(x)        ((x) & PM_PFRAME_MASK)
+
+#define PM_PRESENT          PM_STATUS(4LL)
+#define PM_SWAP             PM_STATUS(2LL)
+
+
+/* For easy conversion, it is better to be the same as architecture's ones */
+enum ihk_mc_pt_attribute {
+	/* ページが物理メモリにロードされているか */
+	PTATTR_ACTIVE     = PTE_VALID,
+	/* Read/Writeフラグ */
+	PTATTR_WRITABLE   = PTE_RDONLY,	//共通定義と意味が反転するので注意
+	/* ユーザ/特権フラグ */
+	PTATTR_USER       = PTE_USER | PTE_NG,
+	/* ページの変更を示す */
+	PTATTR_DIRTY      = PTE_DIRTY,
+	/* ラージページを示す */
+	PTATTR_LARGEPAGE  = PMD_TABLE_BIT, //共通定義と意味が反転するので注意
+	/* remap_file_page フラグ */
+	PTATTR_FILEOFF    = PTE_FILEOFF,
+	/* 実行不可フラグ */
+	PTATTR_NO_EXECUTE = PTE_UXN,
+	/* キャッシュ無し */
+	PTATTR_UNCACHABLE = PTE_ATTRINDX(1),
+	/* ユーザ空間向けを示す */
+	PTATTR_FOR_USER   = UL(1) << (PHYS_MASK_SHIFT - 1),
+	/* WriteCombine */
+	PTATTR_WRITE_COMBINED = PTE_ATTRINDX(2),
+	/* converted flag */
+	ARCH_PTATTR_FLIPPED = PTE_PROT_NONE,
+};
+extern enum ihk_mc_pt_attribute attr_mask;
+
+static inline int pfn_is_write_combined(uintptr_t pfn)
+{
+	return ((pfn & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL_NC));
+}
+
+//共通部と意味がするビット定義
+#define attr_flip_bits (PTATTR_WRITABLE | PTATTR_LARGEPAGE)
+
+static inline int pgsize_to_tbllv(size_t pgsize);
+static inline int pte_is_type_page(const pte_t *ptep, size_t pgsize)
+{
+	int ret = 0; //default D_TABLE
+	int level = pgsize_to_tbllv(pgsize);
+
+	switch (level) {
+	case 4:
+	case 3:
+	case 2:
+		// check D_BLOCK
+		ret = ((*ptep & PMD_TYPE_MASK) == PMD_TYPE_SECT);
+		break;
+	case 1:
+		// check D_PAGE
+		ret = ((*ptep & PTE_TYPE_MASK) == PTE_TYPE_PAGE);
+		break;
+	}
+	return ret;
+}
+
+static inline int pte_is_null(pte_t *ptep)
+{
+	return (*ptep == PTE_NULL);
+}
+
+static inline int pte_is_present(pte_t *ptep)
+{
+	return !!(*ptep & PMD_SECT_VALID);
+}
+
+static inline int pte_is_writable(pte_t *ptep)
+{
+	extern int kprintf(const char *format, ...);
+	kprintf("ERROR: %s is not implemented. \n", __func__);
+	return 0;
+}
+
+static inline int pte_is_dirty(pte_t *ptep, size_t pgsize)
+{
+	int ret = 0;
+	int do_check = pte_is_type_page(ptep, pgsize);
+	if (do_check) {
+		ret = !!(*ptep & PTE_DIRTY);
+	}	
+	return ret;
+}
+
+static inline int pte_is_fileoff(pte_t *ptep, size_t pgsize)
+{
+	int ret = 0;
+	int do_check = pte_is_type_page(ptep, pgsize);
+	if (do_check) {
+		ret = !!(*ptep & PTE_FILEOFF);
+	}
+
+	return ret;
+}
+
+static inline void pte_update_phys(pte_t *ptep, unsigned long phys)
+{
+	*ptep = (*ptep & ~PT_PHYSMASK) | (phys & PT_PHYSMASK);
+}
+
+static inline uintptr_t pte_get_phys(pte_t *ptep)
+{
+	return (uintptr_t)(*ptep & PT_PHYSMASK);
+}
+
+static inline off_t pte_get_off(pte_t *ptep, size_t pgsize)
+{
+	return (off_t)(*ptep & PHYS_MASK);
+}
+
+static inline enum ihk_mc_pt_attribute pte_get_attr(pte_t *ptep, size_t pgsize)
+{
+	enum ihk_mc_pt_attribute attr;
+
+	attr = *ptep & attr_mask;
+	attr ^= attr_flip_bits;
+	if ((*ptep & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_DEVICE_nGnRE)) {
+		attr |= PTATTR_UNCACHABLE;
+	} else if ((*ptep & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL_NC)) {
+		attr |= PTATTR_WRITE_COMBINED;
+	}
+	if (((pgsize == PTL2_SIZE) || (pgsize == PTL3_SIZE))
+	    && ((*ptep & PMD_TYPE_MASK) == PMD_TYPE_SECT)) {
+		attr |= PTATTR_LARGEPAGE;
+	}
+
+	return attr;
+}
+
+static inline void pte_make_null(pte_t *ptep, size_t pgsize)
+{
+	*ptep = PTE_NULL;
+}
+
+static inline void pte_make_fileoff(off_t off,
+		enum ihk_mc_pt_attribute ptattr, size_t pgsize, pte_t *ptep)
+{
+	if (((PTL4_SIZE == pgsize || PTL4_CONT_SIZE == pgsize)
+				&& CONFIG_ARM64_PGTABLE_LEVELS > 3) ||
+	    ((PTL3_SIZE == pgsize || PTL3_CONT_SIZE == pgsize)
+				&& CONFIG_ARM64_PGTABLE_LEVELS > 2) ||
+	     (PTL2_SIZE == pgsize || PTL2_CONT_SIZE == pgsize) ||
+	     (PTL1_SIZE == pgsize || PTL1_CONT_SIZE == pgsize)) {
+		*ptep = PTE_FILEOFF | off | PTE_TYPE_PAGE;
+	}
+}
+
+#if 0	/* XXX: workaround. cannot use panic() here */
+static inline void pte_xchg(pte_t *ptep, pte_t *valp)
+{
+	*valp = xchg(ptep, *valp);
+}
+#else
+#define	pte_xchg(p,vp)	do { *(vp) = xchg((p), *(vp)); } while (0)
+#endif
+
+static inline void pte_clear_dirty(pte_t *ptep, size_t pgsize)
+{
+	int do_clear = pte_is_type_page(ptep, pgsize);
+	if (do_clear) {
+		*ptep = *ptep & ~PTE_DIRTY;
+	}
+}
+
+static inline void pte_set_dirty(pte_t *ptep, size_t pgsize)
+{
+	int do_set = pte_is_type_page(ptep, pgsize);
+	if (do_set) {
+		*ptep |= PTE_DIRTY;
+	}
+}
+
+static inline int pte_is_contiguous(const pte_t *ptep)
+{
+	return !!(*ptep & PTE_CONT);
+}
+
+static inline int pgsize_is_contiguous(size_t pgsize)
+{
+	int ret = 0;
+
+	if ((pgsize == PTL4_CONT_SIZE && CONFIG_ARM64_PGTABLE_LEVELS > 3) ||
+	    (pgsize == PTL3_CONT_SIZE && CONFIG_ARM64_PGTABLE_LEVELS > 2) ||
+	    (pgsize == PTL2_CONT_SIZE) ||
+	    (pgsize == PTL1_CONT_SIZE)) {
+		ret = 1;
+	}
+	return ret;
+}
+
+static inline int pgsize_to_tbllv(size_t pgsize)
+{
+	int level = -EINVAL;
+
+	if ((pgsize == PTL4_CONT_SIZE || pgsize == PTL4_SIZE)
+	    && (CONFIG_ARM64_PGTABLE_LEVELS > 3)) {
+		level = 4;
+	} else if ((pgsize == PTL3_CONT_SIZE || pgsize == PTL3_SIZE)
+		   && (CONFIG_ARM64_PGTABLE_LEVELS > 2)) {
+		level = 3;
+	} else if (pgsize == PTL2_CONT_SIZE || pgsize == PTL2_SIZE) {
+		level = 2;
+	} else if (pgsize == PTL1_CONT_SIZE || pgsize == PTL1_SIZE) {
+		level = 1;
+	}
+	return level;
+}
+
+static inline int pgsize_to_pgshift(size_t pgsize)
+{
+	/* We need to use if instead of switch because
+	 * sometimes PTLX_CONT_SIZE == PTLX_SIZE
+	 */
+	if (pgsize == PTL4_CONT_SIZE) {
+		if (CONFIG_ARM64_PGTABLE_LEVELS > 3) {
+			return PTL4_CONT_SHIFT;
+		}
+	} else if (pgsize == PTL4_SIZE) {
+		if (CONFIG_ARM64_PGTABLE_LEVELS > 3) {
+			return PTL4_SHIFT;
+		}
+	} else if (pgsize == PTL3_CONT_SIZE) {
+		if (CONFIG_ARM64_PGTABLE_LEVELS > 2) {
+			return PTL3_CONT_SHIFT;
+		}
+	} else if (pgsize == PTL3_SIZE) {
+		if (CONFIG_ARM64_PGTABLE_LEVELS > 2) {
+			return PTL3_SHIFT;
+		}
+	} else if (pgsize == PTL2_CONT_SIZE) {
+		return PTL2_CONT_SHIFT;
+	} else if (pgsize == PTL2_SIZE) {
+		return PTL2_SHIFT;
+	} else if (pgsize == PTL1_CONT_SIZE) {
+		return PTL1_CONT_SHIFT;
+	} else if (pgsize == PTL1_SIZE) {
+		return PTL1_SHIFT;
+	}
+
+	return -EINVAL;
+}
+
+static inline size_t tbllv_to_pgsize(int level)
+{
+	size_t pgsize = 0;
+
+	switch (level) {
+	case 4:
+		if (CONFIG_ARM64_PGTABLE_LEVELS > 3) {
+			pgsize = PTL4_SIZE;
+		} else {
+			panic("page table level 4 is invalid.");
+		}
+		break;
+	case 3:
+		if (CONFIG_ARM64_PGTABLE_LEVELS > 2) {
+			pgsize = PTL3_SIZE;
+		} else {
+			panic("page table level 3 is invalid.");
+		}
+		break;
+	case 2:
+		pgsize = PTL2_SIZE;
+		break;
+	case 1:
+		pgsize = PTL1_SIZE;
+		break;
+	default:
+		panic("page table level is invalid.");
+	}
+	return pgsize;
+}
+
+static inline size_t tbllv_to_contpgsize(int level)
+{
+	size_t pgsize = 0;
+
+	switch (level) {
+	case 4:
+		if (CONFIG_ARM64_PGTABLE_LEVELS > 3) {
+			pgsize = PTL4_CONT_SIZE;
+		} else {
+			panic("page table level 4 is invalid.");
+		}
+		break;
+	case 3:
+		if (CONFIG_ARM64_PGTABLE_LEVELS > 2) {
+			pgsize = PTL3_CONT_SIZE;
+		} else {
+			panic("page table level 3 is invalid.");
+		}
+		break;
+	case 2:
+		pgsize = PTL2_CONT_SIZE;
+		break;
+	case 1:
+		pgsize = PTL1_CONT_SIZE;
+		break;
+	default:
+		panic("page table level is invalid.");
+	}
+	return pgsize;
+}
+
+static inline int tbllv_to_contpgshift(int level)
+{
+	int ret = 0;
+
+	switch (level) {
+	case 4:
+		if (CONFIG_ARM64_PGTABLE_LEVELS > 3) {
+			ret = PTL4_CONT_SHIFT;
+		} else {
+			panic("page table level 4 is invalid.");
+		}
+
+		break;
+	case 3:
+		if (CONFIG_ARM64_PGTABLE_LEVELS > 2) {
+			ret = PTL3_CONT_SHIFT;
+		} else {
+			panic("page table level 3 is invalid.");
+		}
+
+		break;
+	case 2:
+		ret = PTL2_CONT_SHIFT;
+		break;
+	case 1:
+		ret = PTL1_CONT_SHIFT;
+		break;
+	default:
+		panic("page table level is invalid.");
+	}
+	return ret;
+}
+
+static inline pte_t *get_contiguous_head(pte_t *__ptep, size_t __pgsize)
+{
+	unsigned long align;
+	int shift = 0;
+
+	switch (pgsize_to_tbllv(__pgsize)) {
+	case 4:
+		if (CONFIG_ARM64_PGTABLE_LEVELS > 3) {
+			shift = PTL4_CONT_SHIFT - PTL4_SHIFT;
+		} else {
+			panic("page table level 4 is invalid.");
+		}
+		break;
+	case 3:
+		if (CONFIG_ARM64_PGTABLE_LEVELS > 2) {
+			shift = PTL3_CONT_SHIFT - PTL3_SHIFT;
+		} else {
+			panic("page table level 3 is invalid.");
+		}
+		break;
+	case 2:
+		shift = PTL2_CONT_SHIFT - PTL2_SHIFT;
+		break;
+	case 1:
+		shift = PTL1_CONT_SHIFT - PTL1_SHIFT;
+		break;
+	default:
+		panic("page table level is invalid.");
+	}
+	align = sizeof(*__ptep) << shift;
+	return  (pte_t *)__page_align(__ptep, align);
+}
+
+static inline pte_t *get_contiguous_tail(pte_t *__ptep, size_t __pgsize)
+{
+	unsigned long align;
+	int shift = 0;
+
+	switch (pgsize_to_tbllv(__pgsize)) {
+	case 4:
+		if (CONFIG_ARM64_PGTABLE_LEVELS > 3) {
+			shift = PTL4_CONT_SHIFT - PTL4_SHIFT;
+		} else {
+			panic("page table level 4 is invalid.");
+		}
+		break;
+	case 3:
+		if (CONFIG_ARM64_PGTABLE_LEVELS > 2) {
+			shift = PTL3_CONT_SHIFT - PTL3_SHIFT;
+		} else {
+			panic("page table level 3 is invalid.");
+		}
+		break;
+	case 2:
+		shift = PTL2_CONT_SHIFT - PTL2_SHIFT;
+		break;
+	case 1:
+		shift = PTL1_CONT_SHIFT - PTL1_SHIFT;
+		break;
+	default:
+		panic("page table level is invalid.");
+	}
+	align = sizeof(*__ptep) << shift;
+	return  (pte_t *)__page_align_up(__ptep + 1, align) - 1;
+}
+
+int split_contiguous_pages(pte_t *ptep, size_t pgsize,
+		uint32_t memobj_flags);
+
+static inline int page_is_contiguous_head(pte_t *ptep, size_t pgsize)
+{
+	pte_t *ptr = get_contiguous_head(ptep, pgsize);
+
+	return (ptr == ptep);
+}
+
+static inline int page_is_contiguous_tail(pte_t *ptep, size_t pgsize)
+{
+	pte_t *ptr = get_contiguous_tail(ptep, pgsize);
+
+	return (ptr == ptep);
+}
+
+/* Return true if PTE doesn't belong to a contiguous PTE group or PTE
+ * is the head of a contiguous PTE group
+ */
+static inline int pte_is_head(pte_t *ptep, pte_t *old, size_t cont_size)
+{
+	if (!pte_is_contiguous(old))
+		return 1;
+	return page_is_contiguous_head(ptep, cont_size);
+}
+
+typedef pte_t translation_table_t;
+struct page_table {
+	translation_table_t* tt;
+	translation_table_t* tt_pa;
+	int asid;
+};
+
+void arch_adjust_allocate_page_size(struct page_table *pt,
+				    uintptr_t fault_addr,
+				    pte_t *ptep,
+				    void **pgaddrp,
+				    size_t *pgsizep);
+void set_pte(pte_t *ppte, unsigned long phys, enum ihk_mc_pt_attribute attr);
+pte_t *get_pte(struct page_table *pt, void *virt, enum ihk_mc_pt_attribute attr);
+
+struct page_table *get_init_page_table(void);
+void *early_alloc_pages(int nr_pages);
+void *get_last_early_heap(void);
+void flush_tlb(void);
+void flush_tlb_single(unsigned long addr);
+
+void *map_fixed_area(unsigned long phys, unsigned long size, int uncachable);
+
+void set_address_space_id(struct page_table *pt, int asid);
+int get_address_space_id(const struct page_table *pt);
+
+void set_translation_table(struct page_table *pt, translation_table_t* tt);
+translation_table_t* get_translation_table(const struct page_table *pt);
+translation_table_t* get_translation_table_as_paddr(const struct page_table *pt);
+
+extern unsigned long ap_trampoline;
+//#define AP_TRAMPOLINE       0x10000
+#define AP_TRAMPOLINE_SIZE  0x2000
+
+/* Local is cachable */
+#define IHK_IKC_QUEUE_PT_ATTR (PTATTR_NO_EXECUTE | PTATTR_WRITABLE)
+
+#endif /* !__ASSEMBLY__ */
+
+#endif /* !__HEADER_ARM64_COMMON_ARCH_MEMORY_H */

arch/arm64/kernel/include/arch-perfctr.h

@@ -0,0 +1,63 @@
+/* arch-perfctr.h COPYRIGHT FUJITSU LIMITED 2016-2018 */
+#ifndef __ARCH_PERFCTR_H__
+#define __ARCH_PERFCTR_H__
+
+#include <ihk/types.h>
+#include <ihk/cpu.h>
+#include <bitops.h>
+
+struct per_cpu_arm_pmu {
+	int num_events;
+#define ARMV8_PMUV3_MAX_COMMON_EVENTS 0x40
+	DECLARE_BITMAP(pmceid_bitmap, ARMV8_PMUV3_MAX_COMMON_EVENTS);
+};
+
+/* @ref.impl arch/arm64/include/asm/pmu.h */
+struct arm_pmu {
+	struct ihk_mc_interrupt_handler* handler;
+	uint32_t (*read_counter)(int);
+	void (*write_counter)(int, uint32_t);
+	void (*reset)(void*);
+	int (*enable_pmu)(void);
+	void (*disable_pmu)(void);
+	int (*enable_counter)(unsigned long counter_mask);
+	int (*disable_counter)(unsigned long counter_mask);
+	int (*enable_intens)(unsigned long counter_mask);
+	int (*disable_intens)(unsigned long counter_mask);
+	int (*set_event_filter)(unsigned long*, int);
+	void (*write_evtype)(int, uint32_t);
+	int (*get_event_idx)(int num_events, unsigned long used_mask,
+			     unsigned long config);
+	int (*map_event)(uint32_t, uint64_t);
+	int (*map_hw_event)(uint64_t config);
+	int (*map_cache_event)(uint64_t config);
+	int (*map_raw_event)(uint64_t config);
+	void (*enable_user_access_pmu_regs)(void);
+	void (*disable_user_access_pmu_regs)(void);
+	int (*counter_mask_valid)(unsigned long counter_mask);
+	struct per_cpu_arm_pmu *per_cpu;
+};
+
+static inline const struct arm_pmu* get_cpu_pmu(void)
+{
+	extern struct arm_pmu cpu_pmu;
+	return &cpu_pmu;
+}
+
+static inline const struct per_cpu_arm_pmu *get_per_cpu_pmu(void)
+{
+	const struct arm_pmu *cpu_pmu = get_cpu_pmu();
+
+	return &cpu_pmu->per_cpu[ihk_mc_get_processor_id()];
+}
+
+int arm64_init_perfctr(void);
+void arm64_init_per_cpu_perfctr(void);
+int arm64_enable_pmu(void);
+void arm64_disable_pmu(void);
+int armv8pmu_init(struct arm_pmu* cpu_pmu);
+void armv8pmu_per_cpu_init(struct per_cpu_arm_pmu *per_cpu);
+void arm64_enable_user_access_pmu_regs(void);
+void arm64_disable_user_access_pmu_regs(void);
+
+#endif

arch/arm64/kernel/include/arch-string.h

@@ -0,0 +1,22 @@
+/* arch-string.h COPYRIGHT FUJITSU LIMITED 2016-2017 */
+#ifndef __HEADER_ARM64_COMMON_ARCH_STRING_H
+#define __HEADER_ARM64_COMMON_ARCH_STRING_H
+
+#define ARCH_FAST_MEMCPY
+
+extern void *__inline_memcpy(void *to, const void *from, size_t t);
+
+#define ARCH_FAST_MEMSET
+
+extern void *__inline_memset(void *s, unsigned long c, size_t count);
+
+#define ARCH_MEMCLEAR
+
+extern void __memclear(void *addr, unsigned long len, void *tmp);
+inline static void memclear(void *addr, unsigned long len)
+{
+	uint64_t q0q1[4];
+	__memclear(addr, len, (void *)&q0q1);
+}
+
+#endif	/* __HEADER_ARM64_COMMON_ARCH_TIMER_H */

arch/arm64/kernel/include/arch-timer.h

@@ -0,0 +1,31 @@
+/* arch-timer.h COPYRIGHT FUJITSU LIMITED 2016-2018 */
+#ifndef __HEADER_ARM64_COMMON_ARCH_TIMER_H
+#define __HEADER_ARM64_COMMON_ARCH_TIMER_H
+
+#include <ihk/cpu.h>
+
+/* @ref.impl include/clocksource/arm_arch_timer.h */
+#define ARCH_TIMER_USR_PCT_ACCESS_EN	(1 << 0)	/* physical counter */
+#define ARCH_TIMER_USR_VCT_ACCESS_EN	(1 << 1)	/* virtual counter */
+#define ARCH_TIMER_VIRT_EVT_EN		(1 << 2)
+#define ARCH_TIMER_EVT_TRIGGER_SHIFT	(4)
+#define ARCH_TIMER_EVT_TRIGGER_MASK	(0xF << ARCH_TIMER_EVT_TRIGGER_SHIFT)
+#define ARCH_TIMER_USR_VT_ACCESS_EN	(1 << 8)	/* virtual timer registers */
+#define ARCH_TIMER_USR_PT_ACCESS_EN	(1 << 9)	/* physical timer registers */
+
+/* @ref.impl linux4.10.16 */
+/* include/clocksource/arm_arch_timer.h */
+#define ARCH_TIMER_CTRL_ENABLE		(1 << 0)
+#define ARCH_TIMER_CTRL_IT_MASK		(1 << 1)
+#define ARCH_TIMER_CTRL_IT_STAT		(1 << 2)
+
+enum arch_timer_reg {
+	ARCH_TIMER_REG_CTRL,
+	ARCH_TIMER_REG_TVAL,
+};
+
+extern int get_timer_intrid(void);
+extern void arch_timer_init(void);
+extern struct ihk_mc_interrupt_handler *get_timer_handler(void);
+
+#endif	/* __HEADER_ARM64_COMMON_ARCH_TIMER_H */

arch/arm64/kernel/include/arch/auxvec.h

@@ -0,0 +1,7 @@
+/* auxvec.h COPYRIGHT FUJITSU LIMITED 2016 */
+#ifndef __HEADER_ARM64_ARCH_AUXVEC_H
+#define __HEADER_ARM64_ARCH_AUXVEC_H
+
+#define AT_SYSINFO_EHDR 33
+
+#endif /* __HEADER_ARM64_ARCH_AUXVEC_H */

arch/arm64/kernel/include/arch/cpu.h

@@ -0,0 +1,107 @@
+/* cpu.h COPYRIGHT FUJITSU LIMITED 2016-2018 */
+#ifndef __HEADER_ARM64_ARCH_CPU_H
+#define __HEADER_ARM64_ARCH_CPU_H
+
+#ifndef __ASSEMBLY__
+
+#define sev()		asm volatile("sev" : : : "memory")
+#define wfe()		asm volatile("wfe" : : : "memory")
+#define wfi()		asm volatile("wfi" : : : "memory")
+
+#define isb()		asm volatile("isb" : : : "memory")
+#define dmb(opt)	asm volatile("dmb " #opt : : : "memory")
+#define dsb(opt)	asm volatile("dsb " #opt : : : "memory")
+
+#include <registers.h>
+
+#define mb()		dsb(sy)
+#define rmb()		dsb(ld)
+#define wmb()		dsb(st)
+
+#define dma_rmb()	dmb(oshld)
+#define dma_wmb()	dmb(oshst)
+
+//#ifndef CONFIG_SMP
+//#else
+#define smp_mb()	dmb(ish)
+#define smp_rmb()	dmb(ishld)
+#define smp_wmb()	dmb(ishst)
+
+#define arch_barrier()	smp_mb()
+
+#define smp_store_release(p, v)						\
+do {									\
+	compiletime_assert_atomic_type(*p);				\
+	switch (sizeof(*p)) {						\
+	case 4:								\
+		asm volatile ("stlr %w1, %0"				\
+				: "=Q" (*p) : "r" (v) : "memory");	\
+		break;							\
+	case 8:								\
+		asm volatile ("stlr %1, %0"				\
+				: "=Q" (*p) : "r" (v) : "memory");	\
+		break;							\
+	}								\
+} while (0)
+
+#define smp_load_acquire(p)						\
+({									\
+	typeof(*p) ___p1;						\
+	compiletime_assert_atomic_type(*p);				\
+	switch (sizeof(*p)) {						\
+	case 4:								\
+		asm volatile ("ldar %w0, %1"				\
+			: "=r" (___p1) : "Q" (*p) : "memory");		\
+		break;							\
+	case 8:								\
+		asm volatile ("ldar %0, %1"				\
+			: "=r" (___p1) : "Q" (*p) : "memory");		\
+		break;							\
+	}								\
+	___p1;								\
+})
+//#endif /*CONFIG_SMP*/
+
+#define read_barrier_depends()		do { } while(0)
+#define smp_read_barrier_depends()	do { } while(0)
+
+#define set_mb(var, value)	do { var = value; smp_mb(); } while (0)
+#define nop()		asm volatile("nop");
+
+#define smp_mb__before_atomic()	smp_mb()
+#define smp_mb__after_atomic()	smp_mb()
+
+#define read_tsc()						\
+({								\
+	unsigned long cval;					\
+	cval = rdtsc();						\
+	cval;							\
+})
+
+void init_tod_data(void);
+
+#if defined(CONFIG_HAS_NMI)
+static inline void cpu_enable_nmi(void)
+{
+	asm volatile("msr    daifclr, #2": : : "memory");
+}
+
+static inline void cpu_disable_nmi(void)
+{
+	asm volatile("msr    daifset, #2": : : "memory");
+}
+#else/*defined(CONFIG_HAS_NMI)*/
+static inline void cpu_enable_nmi(void)
+{
+}
+
+static inline void cpu_disable_nmi(void)
+{
+}
+#endif/*defined(CONFIG_HAS_NMI)*/
+
+#endif	/* __ASSEMBLY__ */
+
+void arch_flush_icache_all(void);
+
+#endif /* !__HEADER_ARM64_ARCH_CPU_H */

arch/arm64/kernel/include/arch/mm.h

@@ -0,0 +1,17 @@
+/* mm.h COPYRIGHT FUJITSU LIMITED 2016 */
+#ifndef __HEADER_ARM64_ARCH_MM_H
+#define __HEADER_ARM64_ARCH_MM_H
+
+struct process_vm;
+
+static inline void
+flush_nfo_tlb()
+{
+}
+
+static inline void
+flush_nfo_tlb_mm(struct process_vm *vm)
+{
+}
+
+#endif /* __HEADER_ARM64_ARCH_MM_H */

arch/arm64/kernel/include/arch/mman.h

@@ -0,0 +1,36 @@
+/* mman.h COPYRIGHT FUJITSU LIMITED 2015-2016 */
+/* @ref.impl linux-linaro/include/uapi/asm-generic/mman.h */
+
+#ifndef __HEADER_ARM64_ARCH_MMAN_H
+#define __HEADER_ARM64_ARCH_MMAN_H
+
+#include <arch-memory.h>
+
+/*
+ * mapping flags
+ */
+#define MAP_GROWSDOWN	0x0100		/* stack-like segment */
+#define MAP_DENYWRITE	0x0800		/* ETXTBSY */
+#define MAP_EXECUTABLE	0x1000		/* mark it as an executable */
+#define MAP_LOCKED	0x2000		/* pages are locked */
+#define MAP_NORESERVE	0x4000		/* don't check for reservations */
+#define MAP_POPULATE	0x8000		/* populate (prefault) pagetables */
+#define MAP_NONBLOCK	0x10000		/* do not block on IO */
+#define MAP_STACK	0x20000		/* give out an address that is best suited for process/thread stacks */
+#define MAP_HUGETLB	0x40000		/* create a huge page mapping */
+
+/* Bits [26:31] are reserved, see mman-common.h for MAP_HUGETLB usage */
+#define MAP_HUGE_SHIFT  26
+#define MAP_HUGE_FIRST_BLOCK       (__PTL3_SHIFT << MAP_HUGE_SHIFT)
+#define MAP_HUGE_FIRST_CONT_BLOCK  (__PTL3_CONT_SHIFT << MAP_HUGE_SHIFT)
+#define MAP_HUGE_SECOND_BLOCK      (__PTL2_SHIFT << MAP_HUGE_SHIFT)
+#define MAP_HUGE_SECOND_CONT_BLOCK (__PTL2_CONT_SHIFT << MAP_HUGE_SHIFT)
+#define MAP_HUGE_THIRD_CONT_BLOCK  (__PTL1_CONT_SHIFT << MAP_HUGE_SHIFT)
+
+/*
+ * for mlockall()
+ */
+#define MCL_CURRENT	1		/* lock all current mappings */
+#define MCL_FUTURE	2		/* lock all future mappings */
+
+#endif /* __HEADER_ARM64_ARCH_MMAN_H */

arch/arm64/kernel/include/arch/shm.h

@@ -0,0 +1,40 @@
+/* shm.h COPYRIGHT FUJITSU LIMITED 2015-2016 */
+#ifndef __HEADER_ARM64_ARCH_SHM_H
+#define __HEADER_ARM64_ARCH_SHM_H
+
+#include <arch-memory.h>
+
+/* shmflg */
+#define SHM_HUGE_SHIFT  26
+#define SHM_HUGE_FIRST_BLOCK       (__PTL3_SHIFT << SHM_HUGE_SHIFT)
+#define SHM_HUGE_FIRST_CONT_BLOCK  (__PTL3_CONT_SHIFT << SHM_HUGE_SHIFT)
+#define SHM_HUGE_SECOND_BLOCK      (__PTL2_SHIFT << SHM_HUGE_SHIFT)
+#define SHM_HUGE_SECOND_CONT_BLOCK (__PTL2_CONT_SHIFT << SHM_HUGE_SHIFT)
+#define SHM_HUGE_THIRD_CONT_BLOCK  (__PTL1_CONT_SHIFT << SHM_HUGE_SHIFT)
+
+struct ipc_perm {
+	key_t		key;
+	uid_t		uid;
+	gid_t		gid;
+	uid_t		cuid;
+	gid_t		cgid;
+	uint16_t	mode;
+	uint8_t		padding[2];
+	uint16_t	seq;
+	uint8_t		padding2[22];
+};
+
+struct shmid_ds {
+	struct ipc_perm	shm_perm;
+	size_t		shm_segsz;
+	time_t		shm_atime;
+	time_t		shm_dtime;
+	time_t		shm_ctime;
+	pid_t		shm_cpid;
+	pid_t		shm_lpid;
+	uint64_t	shm_nattch;
+	uint8_t		padding[12];
+	int		init_pgshift;
+};
+
+#endif /* __HEADER_ARM64_ARCH_SHM_H */

arch/arm64/kernel/include/arm-gic-v2.h

@@ -0,0 +1,106 @@
+/* arm-gic-v2.h COPYRIGHT FUJITSU LIMITED 2015-2017 */
+/*
+ *  include/linux/irqchip/arm-gic.h
+ *
+ *  Copyright (C) 2002 ARM Limited, All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#ifndef __LINUX_IRQCHIP_ARM_GIC_H
+#define __LINUX_IRQCHIP_ARM_GIC_H
+
+/* check config */
+#if defined(CONFIG_HAS_NMI) && !defined(CONFIG_ARM_GIC_V3)
+# error GICv2 is not support NMI
+#endif
+
+/* @ref.impl include/linux/irqchip/arm-gic.h */
+
+#define GIC_CPU_CTRL			0x00
+#define GIC_CPU_PRIMASK			0x04
+#define GIC_CPU_BINPOINT		0x08
+#define GIC_CPU_INTACK			0x0c
+#define GIC_CPU_EOI			0x10
+#define GIC_CPU_RUNNINGPRI		0x14
+#define GIC_CPU_HIGHPRI			0x18
+#define GIC_CPU_ALIAS_BINPOINT		0x1c
+#define GIC_CPU_ACTIVEPRIO		0xd0
+#define GIC_CPU_IDENT			0xfc
+
+#define GICC_ENABLE			0x1
+#define GICC_INT_PRI_THRESHOLD		0xf0
+#define GICC_IAR_INT_ID_MASK		0x3ff
+#define GICC_INT_SPURIOUS		1023
+#define GICC_DIS_BYPASS_MASK		0x1e0
+
+#define GIC_DIST_CTRL			0x000
+#define GIC_DIST_CTR			0x004
+#define GIC_DIST_IGROUP			0x080
+#define GIC_DIST_ENABLE_SET		0x100
+#define GIC_DIST_ENABLE_CLEAR		0x180
+#define GIC_DIST_PENDING_SET		0x200
+#define GIC_DIST_PENDING_CLEAR		0x280
+#define GIC_DIST_ACTIVE_SET		0x300
+#define GIC_DIST_ACTIVE_CLEAR		0x380
+#define GIC_DIST_PRI			0x400
+#define GIC_DIST_TARGET			0x800
+#define GIC_DIST_CONFIG			0xc00
+#define GIC_DIST_SOFTINT		0xf00
+#define GIC_DIST_SGI_PENDING_CLEAR	0xf10
+#define GIC_DIST_SGI_PENDING_SET	0xf20
+
+#define GICD_ENABLE			0x1
+#define GICD_DISABLE			0x0
+#define GICD_INT_ACTLOW_LVLTRIG		0x0
+#define GICD_INT_EN_CLR_X32		0xffffffff
+#define GICD_INT_EN_SET_SGI		0x0000ffff
+#define GICD_INT_EN_CLR_PPI		0xffff0000
+
+#ifdef CONFIG_HAS_NMI
+#define GICD_INT_NMI_PRI		0x40
+#define GICD_INT_DEF_PRI		0xc0U
+#else
+#define GICD_INT_DEF_PRI		0xa0U
+#endif
+#define GICD_INT_DEF_PRI_X4		((GICD_INT_DEF_PRI << 24) |\
+					(GICD_INT_DEF_PRI << 16) |\
+					(GICD_INT_DEF_PRI << 8) |\
+					GICD_INT_DEF_PRI)
+
+#define GICH_HCR			0x0
+#define GICH_VTR			0x4
+#define GICH_VMCR			0x8
+#define GICH_MISR			0x10
+#define GICH_EISR0 			0x20
+#define GICH_EISR1 			0x24
+#define GICH_ELRSR0 			0x30
+#define GICH_ELRSR1 			0x34
+#define GICH_APR			0xf0
+#define GICH_LR0			0x100
+
+#define GICH_HCR_EN			(1 << 0)
+#define GICH_HCR_UIE			(1 << 1)
+
+#define GICH_LR_VIRTUALID		(0x3ff << 0)
+#define GICH_LR_PHYSID_CPUID_SHIFT	(10)
+#define GICH_LR_PHYSID_CPUID		(7 << GICH_LR_PHYSID_CPUID_SHIFT)
+#define GICH_LR_STATE			(3 << 28)
+#define GICH_LR_PENDING_BIT		(1 << 28)
+#define GICH_LR_ACTIVE_BIT		(1 << 29)
+#define GICH_LR_EOI			(1 << 19)
+
+#define GICH_VMCR_CTRL_SHIFT		0
+#define GICH_VMCR_CTRL_MASK		(0x21f << GICH_VMCR_CTRL_SHIFT)
+#define GICH_VMCR_PRIMASK_SHIFT		27
+#define GICH_VMCR_PRIMASK_MASK		(0x1f << GICH_VMCR_PRIMASK_SHIFT)
+#define GICH_VMCR_BINPOINT_SHIFT	21
+#define GICH_VMCR_BINPOINT_MASK		(0x7 << GICH_VMCR_BINPOINT_SHIFT)
+#define GICH_VMCR_ALIAS_BINPOINT_SHIFT	18
+#define GICH_VMCR_ALIAS_BINPOINT_MASK	(0x7 << GICH_VMCR_ALIAS_BINPOINT_SHIFT)
+
+#define GICH_MISR_EOI			(1 << 0)
+#define GICH_MISR_U			(1 << 1)
+
+#endif /* __LINUX_IRQCHIP_ARM_GIC_H */

arch/arm64/kernel/include/arm-gic-v3.h

@@ -0,0 +1,385 @@
+/* arm-gic-v3.h COPYRIGHT FUJITSU LIMITED 2015-2017 */
+/*
+ * Copyright (C) 2013, 2014 ARM Limited, All Rights Reserved.
+ * Author: Marc Zyngier <marc.zyngier@arm.com>
+ *
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef __LINUX_IRQCHIP_ARM_GIC_V3_H
+#define __LINUX_IRQCHIP_ARM_GIC_V3_H
+
+#include <stringify.h>
+/* @ref.impl include/linux/irqchip/arm-gic-v3.h */
+#include <sysreg.h>
+
+/*
+ * Distributor registers. We assume we're running non-secure, with ARE
+ * being set. Secure-only and non-ARE registers are not described.
+ */
+#define GICD_CTLR			0x0000
+#define GICD_TYPER			0x0004
+#define GICD_IIDR			0x0008
+#define GICD_STATUSR			0x0010
+#define GICD_SETSPI_NSR			0x0040
+#define GICD_CLRSPI_NSR			0x0048
+#define GICD_SETSPI_SR			0x0050
+#define GICD_CLRSPI_SR			0x0058
+#define GICD_SEIR			0x0068
+#define GICD_IGROUPR			0x0080
+#define GICD_ISENABLER			0x0100
+#define GICD_ICENABLER			0x0180
+#define GICD_ISPENDR			0x0200
+#define GICD_ICPENDR			0x0280
+#define GICD_ISACTIVER			0x0300
+#define GICD_ICACTIVER			0x0380
+#define GICD_IPRIORITYR			0x0400
+#define GICD_ICFGR			0x0C00
+#define GICD_IGRPMODR			0x0D00
+#define GICD_NSACR			0x0E00
+#define GICD_IROUTER			0x6000
+#define GICD_IDREGS			0xFFD0
+#define GICD_PIDR2			0xFFE8
+
+/*
+ * Those registers are actually from GICv2, but the spec demands that they
+ * are implemented as RES0 if ARE is 1 (which we do in KVM's emulated GICv3).
+ */
+#define GICD_ITARGETSR			0x0800
+#define GICD_SGIR			0x0F00
+#define GICD_CPENDSGIR			0x0F10
+#define GICD_SPENDSGIR			0x0F20
+
+#define GICD_CTLR_RWP			(1U << 31)
+#define GICD_CTLR_DS			(1U << 6)
+#define GICD_CTLR_ARE_NS		(1U << 4)
+#define GICD_CTLR_ENABLE_G1A		(1U << 1)
+#define GICD_CTLR_ENABLE_G1		(1U << 0)
+
+/*
+ * In systems with a single security state (what we emulate in KVM)
+ * the meaning of the interrupt group enable bits is slightly different
+ */
+#define GICD_CTLR_ENABLE_SS_G1		(1U << 1)
+#define GICD_CTLR_ENABLE_SS_G0		(1U << 0)
+
+#define GICD_TYPER_LPIS			(1U << 17)
+#define GICD_TYPER_MBIS			(1U << 16)
+
+#define GICD_TYPER_ID_BITS(typer)	((((typer) >> 19) & 0x1f) + 1)
+#define GICD_TYPER_IRQS(typer)		((((typer) & 0x1f) + 1) * 32)
+#define GICD_TYPER_LPIS			(1U << 17)
+
+#define GICD_IROUTER_SPI_MODE_ONE	(0U << 31)
+#define GICD_IROUTER_SPI_MODE_ANY	(1U << 31)
+
+#define GIC_PIDR2_ARCH_MASK		0xf0
+#define GIC_PIDR2_ARCH_GICv3		0x30
+#define GIC_PIDR2_ARCH_GICv4		0x40
+
+#define GIC_V3_DIST_SIZE		0x10000
+
+/*
+ * Re-Distributor registers, offsets from RD_base
+ */
+#define GICR_CTLR			GICD_CTLR
+#define GICR_IIDR			0x0004
+#define GICR_TYPER			0x0008
+#define GICR_STATUSR			GICD_STATUSR
+#define GICR_WAKER			0x0014
+#define GICR_SETLPIR			0x0040
+#define GICR_CLRLPIR			0x0048
+#define GICR_SEIR			GICD_SEIR
+#define GICR_PROPBASER			0x0070
+#define GICR_PENDBASER			0x0078
+#define GICR_INVLPIR			0x00A0
+#define GICR_INVALLR			0x00B0
+#define GICR_SYNCR			0x00C0
+#define GICR_MOVLPIR			0x0100
+#define GICR_MOVALLR			0x0110
+#define GICR_IDREGS			GICD_IDREGS
+#define GICR_PIDR2			GICD_PIDR2
+
+#define GICR_CTLR_ENABLE_LPIS		(1UL << 0)
+
+#define GICR_TYPER_CPU_NUMBER(r)	(((r) >> 8) & 0xffff)
+
+#define GICR_WAKER_ProcessorSleep	(1U << 1)
+#define GICR_WAKER_ChildrenAsleep	(1U << 2)
+
+#define GICR_PROPBASER_NonShareable	(0U << 10)
+#define GICR_PROPBASER_InnerShareable	(1U << 10)
+#define GICR_PROPBASER_OuterShareable	(2U << 10)
+#define GICR_PROPBASER_SHAREABILITY_MASK (3UL << 10)
+#define GICR_PROPBASER_nCnB		(0U << 7)
+#define GICR_PROPBASER_nC		(1U << 7)
+#define GICR_PROPBASER_RaWt		(2U << 7)
+#define GICR_PROPBASER_RaWb		(3U << 7)
+#define GICR_PROPBASER_WaWt		(4U << 7)
+#define GICR_PROPBASER_WaWb		(5U << 7)
+#define GICR_PROPBASER_RaWaWt		(6U << 7)
+#define GICR_PROPBASER_RaWaWb		(7U << 7)
+#define GICR_PROPBASER_CACHEABILITY_MASK (7U << 7)
+#define GICR_PROPBASER_IDBITS_MASK	(0x1f)
+
+#define GICR_PENDBASER_NonShareable	(0U << 10)
+#define GICR_PENDBASER_InnerShareable	(1U << 10)
+#define GICR_PENDBASER_OuterShareable	(2U << 10)
+#define GICR_PENDBASER_SHAREABILITY_MASK (3UL << 10)
+#define GICR_PENDBASER_nCnB		(0U << 7)
+#define GICR_PENDBASER_nC		(1U << 7)
+#define GICR_PENDBASER_RaWt		(2U << 7)
+#define GICR_PENDBASER_RaWb		(3U << 7)
+#define GICR_PENDBASER_WaWt		(4U << 7)
+#define GICR_PENDBASER_WaWb		(5U << 7)
+#define GICR_PENDBASER_RaWaWt		(6U << 7)
+#define GICR_PENDBASER_RaWaWb		(7U << 7)
+#define GICR_PENDBASER_CACHEABILITY_MASK (7U << 7)
+
+/*
+ * Re-Distributor registers, offsets from SGI_base
+ */
+#define GICR_IGROUPR0			GICD_IGROUPR
+#define GICR_ISENABLER0			GICD_ISENABLER
+#define GICR_ICENABLER0			GICD_ICENABLER
+#define GICR_ISPENDR0			GICD_ISPENDR
+#define GICR_ICPENDR0			GICD_ICPENDR
+#define GICR_ISACTIVER0			GICD_ISACTIVER
+#define GICR_ICACTIVER0			GICD_ICACTIVER
+#define GICR_IPRIORITYR0		GICD_IPRIORITYR
+#define GICR_ICFGR0			GICD_ICFGR
+#define GICR_IGRPMODR0			GICD_IGRPMODR
+#define GICR_NSACR			GICD_NSACR
+
+#define GICR_TYPER_PLPIS		(1U << 0)
+#define GICR_TYPER_VLPIS		(1U << 1)
+#define GICR_TYPER_LAST			(1U << 4)
+
+#define GIC_V3_REDIST_SIZE		0x20000
+
+#define LPI_PROP_GROUP1			(1 << 1)
+#define LPI_PROP_ENABLED		(1 << 0)
+
+/*
+ * ITS registers, offsets from ITS_base
+ */
+#define GITS_CTLR			0x0000
+#define GITS_IIDR			0x0004
+#define GITS_TYPER			0x0008
+#define GITS_CBASER			0x0080
+#define GITS_CWRITER			0x0088
+#define GITS_CREADR			0x0090
+#define GITS_BASER			0x0100
+#define GITS_PIDR2			GICR_PIDR2
+
+#define GITS_TRANSLATER			0x10040
+
+#define GITS_CTLR_ENABLE		(1U << 0)
+#define GITS_CTLR_QUIESCENT		(1U << 31)
+
+#define GITS_TYPER_DEVBITS_SHIFT	13
+#define GITS_TYPER_DEVBITS(r)		((((r) >> GITS_TYPER_DEVBITS_SHIFT) & 0x1f) + 1)
+#define GITS_TYPER_PTA			(1UL << 19)
+
+#define GITS_CBASER_VALID		(1UL << 63)
+#define GITS_CBASER_nCnB		(0UL << 59)
+#define GITS_CBASER_nC			(1UL << 59)
+#define GITS_CBASER_RaWt		(2UL << 59)
+#define GITS_CBASER_RaWb		(3UL << 59)
+#define GITS_CBASER_WaWt		(4UL << 59)
+#define GITS_CBASER_WaWb		(5UL << 59)
+#define GITS_CBASER_RaWaWt		(6UL << 59)
+#define GITS_CBASER_RaWaWb		(7UL << 59)
+#define GITS_CBASER_CACHEABILITY_MASK	(7UL << 59)
+#define GITS_CBASER_NonShareable	(0UL << 10)
+#define GITS_CBASER_InnerShareable	(1UL << 10)
+#define GITS_CBASER_OuterShareable	(2UL << 10)
+#define GITS_CBASER_SHAREABILITY_MASK	(3UL << 10)
+
+#define GITS_BASER_NR_REGS		8
+
+#define GITS_BASER_VALID		(1UL << 63)
+#define GITS_BASER_nCnB			(0UL << 59)
+#define GITS_BASER_nC			(1UL << 59)
+#define GITS_BASER_RaWt			(2UL << 59)
+#define GITS_BASER_RaWb			(3UL << 59)
+#define GITS_BASER_WaWt			(4UL << 59)
+#define GITS_BASER_WaWb			(5UL << 59)
+#define GITS_BASER_RaWaWt		(6UL << 59)
+#define GITS_BASER_RaWaWb		(7UL << 59)
+#define GITS_BASER_CACHEABILITY_MASK	(7UL << 59)
+#define GITS_BASER_TYPE_SHIFT		(56)
+#define GITS_BASER_TYPE(r)		(((r) >> GITS_BASER_TYPE_SHIFT) & 7)
+#define GITS_BASER_ENTRY_SIZE_SHIFT	(48)
+#define GITS_BASER_ENTRY_SIZE(r)	((((r) >> GITS_BASER_ENTRY_SIZE_SHIFT) & 0xff) + 1)
+#define GITS_BASER_NonShareable		(0UL << 10)
+#define GITS_BASER_InnerShareable	(1UL << 10)
+#define GITS_BASER_OuterShareable	(2UL << 10)
+#define GITS_BASER_SHAREABILITY_SHIFT	(10)
+#define GITS_BASER_SHAREABILITY_MASK	(3UL << GITS_BASER_SHAREABILITY_SHIFT)
+#define GITS_BASER_PAGE_SIZE_SHIFT	(8)
+#define GITS_BASER_PAGE_SIZE_4K		(0UL << GITS_BASER_PAGE_SIZE_SHIFT)
+#define GITS_BASER_PAGE_SIZE_16K	(1UL << GITS_BASER_PAGE_SIZE_SHIFT)
+#define GITS_BASER_PAGE_SIZE_64K	(2UL << GITS_BASER_PAGE_SIZE_SHIFT)
+#define GITS_BASER_PAGE_SIZE_MASK	(3UL << GITS_BASER_PAGE_SIZE_SHIFT)
+#define GITS_BASER_PAGES_MAX		256
+
+#define GITS_BASER_TYPE_NONE		0
+#define GITS_BASER_TYPE_DEVICE		1
+#define GITS_BASER_TYPE_VCPU		2
+#define GITS_BASER_TYPE_CPU		3
+#define GITS_BASER_TYPE_COLLECTION	4
+#define GITS_BASER_TYPE_RESERVED5	5
+#define GITS_BASER_TYPE_RESERVED6	6
+#define GITS_BASER_TYPE_RESERVED7	7
+
+/*
+ * ITS commands
+ */
+#define GITS_CMD_MAPD			0x08
+#define GITS_CMD_MAPC			0x09
+#define GITS_CMD_MAPVI			0x0a
+#define GITS_CMD_MOVI			0x01
+#define GITS_CMD_DISCARD		0x0f
+#define GITS_CMD_INV			0x0c
+#define GITS_CMD_MOVALL			0x0e
+#define GITS_CMD_INVALL			0x0d
+#define GITS_CMD_INT			0x03
+#define GITS_CMD_CLEAR			0x04
+#define GITS_CMD_SYNC			0x05
+
+/*
+ * CPU interface registers
+ */
+#define ICC_CTLR_EL1_EOImode_drop_dir	(0U << 1)
+#define ICC_CTLR_EL1_EOImode_drop	(1U << 1)
+#define ICC_SRE_EL1_SRE			(1U << 0)
+
+/*
+ * Hypervisor interface registers (SRE only)
+ */
+#define ICH_LR_VIRTUAL_ID_MASK		((1UL << 32) - 1)
+
+#define ICH_LR_EOI			(1UL << 41)
+#define ICH_LR_GROUP			(1UL << 60)
+#define ICH_LR_STATE			(3UL << 62)
+#define ICH_LR_PENDING_BIT		(1UL << 62)
+#define ICH_LR_ACTIVE_BIT		(1UL << 63)
+
+#define ICH_MISR_EOI			(1 << 0)
+#define ICH_MISR_U			(1 << 1)
+
+#define ICH_HCR_EN			(1 << 0)
+#define ICH_HCR_UIE			(1 << 1)
+
+#define ICH_VMCR_CTLR_SHIFT		0
+#define ICH_VMCR_CTLR_MASK		(0x21f << ICH_VMCR_CTLR_SHIFT)
+#define ICH_VMCR_BPR1_SHIFT		18
+#define ICH_VMCR_BPR1_MASK		(7 << ICH_VMCR_BPR1_SHIFT)
+#define ICH_VMCR_BPR0_SHIFT		21
+#define ICH_VMCR_BPR0_MASK		(7 << ICH_VMCR_BPR0_SHIFT)
+#define ICH_VMCR_PMR_SHIFT		24
+#define ICH_VMCR_PMR_MASK		(0xffUL << ICH_VMCR_PMR_SHIFT)
+
+#define ICC_EOIR1_EL1			sys_reg(3, 0, 12, 12, 1)
+#define ICC_IAR1_EL1			sys_reg(3, 0, 12, 12, 0)
+#define ICC_SGI1R_EL1			sys_reg(3, 0, 12, 11, 5)
+#define ICC_PMR_EL1			sys_reg(3, 0, 4, 6, 0)
+#define ICC_CTLR_EL1			sys_reg(3, 0, 12, 12, 4)
+#define ICC_SRE_EL1			sys_reg(3, 0, 12, 12, 5)
+#define ICC_GRPEN1_EL1			sys_reg(3, 0, 12, 12, 7)
+#define ICC_BPR1_EL1			sys_reg(3, 0, 12, 12, 3)
+
+#define ICC_IAR1_EL1_SPURIOUS		0x3ff
+
+#define ICC_SRE_EL2			sys_reg(3, 4, 12, 9, 5)
+
+#define ICC_SRE_EL2_SRE			(1 << 0)
+#define ICC_SRE_EL2_ENABLE		(1 << 3)
+
+#define ICC_SGI1R_TARGET_LIST_SHIFT	0
+#define ICC_SGI1R_TARGET_LIST_MASK	(0xffff << ICC_SGI1R_TARGET_LIST_SHIFT)
+#define ICC_SGI1R_AFFINITY_1_SHIFT	16
+#define ICC_SGI1R_AFFINITY_1_MASK	(0xff << ICC_SGI1R_AFFINITY_1_SHIFT)
+#define ICC_SGI1R_SGI_ID_SHIFT		24
+#define ICC_SGI1R_SGI_ID_MASK		(0xff << ICC_SGI1R_SGI_ID_SHIFT)
+#define ICC_SGI1R_AFFINITY_2_SHIFT	32
+#define ICC_SGI1R_AFFINITY_2_MASK	(0xffULL << ICC_SGI1R_AFFINITY_1_SHIFT)
+#define ICC_SGI1R_IRQ_ROUTING_MODE_BIT	40
+#define ICC_SGI1R_AFFINITY_3_SHIFT	48
+#define ICC_SGI1R_AFFINITY_3_MASK	(0xffULL << ICC_SGI1R_AFFINITY_1_SHIFT)
+
+#ifdef CONFIG_HAS_NMI
+/* PMR values used to mask/unmask interrupts */
+#define ICC_PMR_EL1_G_SHIFT		6
+#define ICC_PMR_EL1_G_BIT		(1 << ICC_PMR_EL1_G_SHIFT)
+#define ICC_PMR_EL1_UNMASKED		0xf0
+#define ICC_PMR_EL1_MASKED		(ICC_PMR_EL1_UNMASKED ^ ICC_PMR_EL1_G_BIT)
+
+/* 
+ * This is the GIC interrupt mask bit. It is not actually part of the
+ * PSR and so does not appear in the user API, we are simply using some
+ * reserved bits in the PSR to store some state from the interrupt
+ * controller. The context save/restore functions will extract the
+ * ICC_PMR_EL1_G_BIT and save it as the PSR_G_BIT.
+ */
+#define PSR_G_BIT			0x00400000
+#define PSR_G_SHIFT			22
+#define PSR_G_PMR_G_SHIFT		(PSR_G_SHIFT - ICC_PMR_EL1_G_SHIFT)
+#define PSR_I_PMR_G_SHIFT		(7 - ICC_PMR_EL1_G_SHIFT)
+#endif /* CONFIG_HAS_NMI */
+
+/*
+ * System register definitions
+ */
+#define ICH_VSEIR_EL2			sys_reg(3, 4, 12, 9, 4)
+#define ICH_HCR_EL2			sys_reg(3, 4, 12, 11, 0)
+#define ICH_VTR_EL2			sys_reg(3, 4, 12, 11, 1)
+#define ICH_MISR_EL2			sys_reg(3, 4, 12, 11, 2)
+#define ICH_EISR_EL2			sys_reg(3, 4, 12, 11, 3)
+#define ICH_ELSR_EL2			sys_reg(3, 4, 12, 11, 5)
+#define ICH_VMCR_EL2			sys_reg(3, 4, 12, 11, 7)
+
+#define __LR0_EL2(x)			sys_reg(3, 4, 12, 12, x)
+#define __LR8_EL2(x)			sys_reg(3, 4, 12, 13, x)
+
+#define ICH_LR0_EL2			__LR0_EL2(0)
+#define ICH_LR1_EL2			__LR0_EL2(1)
+#define ICH_LR2_EL2			__LR0_EL2(2)
+#define ICH_LR3_EL2			__LR0_EL2(3)
+#define ICH_LR4_EL2			__LR0_EL2(4)
+#define ICH_LR5_EL2			__LR0_EL2(5)
+#define ICH_LR6_EL2			__LR0_EL2(6)
+#define ICH_LR7_EL2			__LR0_EL2(7)
+#define ICH_LR8_EL2			__LR8_EL2(0)
+#define ICH_LR9_EL2			__LR8_EL2(1)
+#define ICH_LR10_EL2			__LR8_EL2(2)
+#define ICH_LR11_EL2			__LR8_EL2(3)
+#define ICH_LR12_EL2			__LR8_EL2(4)
+#define ICH_LR13_EL2			__LR8_EL2(5)
+#define ICH_LR14_EL2			__LR8_EL2(6)
+#define ICH_LR15_EL2			__LR8_EL2(7)
+
+#define __AP0Rx_EL2(x)			sys_reg(3, 4, 12, 8, x)
+#define ICH_AP0R0_EL2			__AP0Rx_EL2(0)
+#define ICH_AP0R1_EL2			__AP0Rx_EL2(1)
+#define ICH_AP0R2_EL2			__AP0Rx_EL2(2)
+#define ICH_AP0R3_EL2			__AP0Rx_EL2(3)
+
+#define __AP1Rx_EL2(x)			sys_reg(3, 4, 12, 9, x)
+#define ICH_AP1R0_EL2			__AP1Rx_EL2(0)
+#define ICH_AP1R1_EL2			__AP1Rx_EL2(1)
+#define ICH_AP1R2_EL2			__AP1Rx_EL2(2)
+#define ICH_AP1R3_EL2			__AP1Rx_EL2(3)
+
+#endif /* __LINUX_IRQCHIP_ARM_GIC_V3_H */

arch/arm64/kernel/include/asm-offsets.h

@@ -0,0 +1,28 @@
+/* asm-offsets.h COPYRIGHT FUJITSU LIMITED 2015-2016 */
+#ifndef __HEADER_ARM64_COMMON_ASM_OFFSETS_H
+#define __HEADER_ARM64_COMMON_ASM_OFFSETS_H
+
+#define S_X0		0x00	/* offsetof(struct pt_regs, regs[0]) */
+#define S_X1		0x08	/* offsetof(struct pt_regs, regs[1]) */
+#define S_X2		0x10	/* offsetof(struct pt_regs, regs[2]) */
+#define S_X3		0x18	/* offsetof(struct pt_regs, regs[3]) */
+#define S_X4		0x20	/* offsetof(struct pt_regs, regs[4]) */
+#define S_X5		0x28	/* offsetof(struct pt_regs, regs[5]) */
+#define S_X6		0x30	/* offsetof(struct pt_regs, regs[6]) */
+#define S_X7		0x38	/* offsetof(struct pt_regs, regs[7]) */
+#define S_LR		0xf0	/* offsetof(struct pt_regs, regs[30]) */
+#define S_SP		0xf8	/* offsetof(struct pt_regs, sp) */
+#define S_PC		0x100	/* offsetof(struct pt_regs, pc) */
+#define S_PSTATE	0x108	/* offsetof(struct pt_regs, pstate) */
+#define S_ORIG_X0	0x110	/* offsetof(struct pt_regs, orig_x0) */
+#define S_ORIG_PC	0x118	/* offsetof(struct pt_regs, orig_pc) */
+#define S_SYSCALLNO	0x120	/* offsetof(struct pt_regs, syscallno) */
+#define S_FRAME_SIZE	0x130	/* sizeof(struct pt_regs) must be 16 byte align */
+
+#define CPU_INFO_SETUP	0x10	/* offsetof(struct cpu_info, cpu_setup) */
+#define CPU_INFO_SZ	0x18	/* sizeof(struct cpu_info) */
+
+#define TI_FLAGS	0x00	/* offsetof(struct thread_info, flags) */
+#define TI_CPU_CONTEXT	0x10	/* offsetof(struct thread_info, cpu_context) */
+
+#endif /* !__HEADER_ARM64_COMMON_ASM_OFFSETS_H */

arch/arm64/kernel/include/asm-syscall.h

@@ -0,0 +1,20 @@
+/* asm-syscall.h COPYRIGHT FUJITSU LIMITED 2018 */
+#ifndef __HEADER_ARM64_ASM_SYSCALL_H
+#define __HEADER_ARM64_ASM_SYSCALL_H
+
+#ifdef __ASSEMBLY__
+
+#define DECLARATOR(number, name)	.equ __NR_##name, number
+#define SYSCALL_HANDLED(number, name)	DECLARATOR(number, name)
+#define SYSCALL_DELEGATED(number, name)	DECLARATOR(number, name)
+
+#include <config.h>
+#include <syscall_list.h>
+
+#undef DECLARATOR
+#undef SYSCALL_HANDLED
+#undef SYSCALL_DELEGATED
+
+#endif /* __ASSEMBLY__ */
+
+#endif	/* !__HEADER_ARM64_ASM_SYSCALL_H */

arch/arm64/kernel/include/assembler.h

@@ -0,0 +1,149 @@
+/* assembler.h COPYRIGHT FUJITSU LIMITED 2015-2017 */
+#ifndef __HEADER_ARM64_COMMON_ASSEMBLER_H
+#define __HEADER_ARM64_COMMON_ASSEMBLER_H
+
+#include <thread_info.h>
+
+#if defined(CONFIG_HAS_NMI)
+#include <arm-gic-v3.h>
+#else /* defined(CONFIG_HAS_NMI) */
+#include <sysreg.h>
+#endif /* defined(CONFIG_HAS_NMI) */
+
+#if defined(CONFIG_HAS_NMI)
+/*
+ * Enable and disable pseudo NMI.
+ */
+	.macro	disable_nmi
+	msr	daifset, #2
+	.endm
+
+	.macro	enable_nmi
+	msr	daifclr, #2
+	.endm
+
+/*
+ * Enable and disable interrupts.
+ */
+	.macro	disable_irq, tmp
+	mov	\tmp, #ICC_PMR_EL1_MASKED
+	msr_s	ICC_PMR_EL1, \tmp
+	.endm
+
+	.macro	enable_irq, tmp
+	mov	\tmp, #ICC_PMR_EL1_UNMASKED
+	msr_s	ICC_PMR_EL1, \tmp
+	.endm
+
+#else /* defined(CONFIG_HAS_NMI) */
+/*
+ * Enable and disable pseudo NMI.
+ */
+	.macro	disable_nmi
+	.endm
+
+	.macro	enable_nmi
+	.endm
+
+/*
+ * Enable and disable interrupts.
+ */
+	.macro	disable_irq, tmp
+	msr	daifset, #2
+	.endm
+
+	.macro	enable_irq, tmp
+	msr	daifclr, #2
+	.endm
+#endif /* defined(CONFIG_HAS_NMI) */
+
+/*
+ * Enable and disable debug exceptions.
+ */
+	.macro	disable_dbg
+	msr	daifset, #8
+	.endm
+
+	.macro	enable_dbg
+	msr	daifclr, #8
+	.endm
+
+	.macro	disable_step_tsk, flgs, tmp
+	tbz	\flgs, #TIF_SINGLESTEP, 9990f
+	mrs	\tmp, mdscr_el1
+	bic	\tmp, \tmp, #1
+	msr	mdscr_el1, \tmp
+	isb	// Synchronise with enable_dbg
+9990:
+	.endm
+
+	.macro	enable_step_tsk, flgs, tmp
+	tbz	\flgs, #TIF_SINGLESTEP, 9990f
+	disable_dbg
+	mrs	\tmp, mdscr_el1
+	orr	\tmp, \tmp, #1
+	msr	mdscr_el1, \tmp
+	b	9991f
+9990:
+	mrs	\tmp, mdscr_el1
+	bic	\tmp, \tmp, #1
+	msr	mdscr_el1, \tmp
+	isb	// Synchronise with enable_dbg
+9991:
+	.endm
+
+/*
+ * Enable both debug exceptions and interrupts. This is likely to be
+ * faster than two daifclr operations, since writes to this register
+ * are self-synchronising.
+ */
+#if defined(CONFIG_HAS_NMI)
+ 	.macro  enable_dbg_and_irq, tmp
+ 	enable_dbg
+ 	enable_irq \tmp
+	.endm
+#else /* defined(CONFIG_HAS_NMI) */
+	.macro	enable_dbg_and_irq, tmp
+	msr	daifclr, #(8 | 2)
+	.endm
+#endif /* defined(CONFIG_HAS_NMI) */
+
+/*
+ * Register aliases.
+ */
+lr	.req	x30		// link register
+
+/*
+ * Vector entry
+ */
+	 .macro ventry  label
+	.align 7
+	b	\label
+	.endm
+
+/*
+ * Select code when configured for BE.
+ */
+//#ifdef CONFIG_CPU_BIG_ENDIAN
+//#define CPU_BE(code...) code
+//#else
+#define CPU_BE(code...)
+//#endif
+
+/*
+ * Select code when configured for LE.
+ */
+//#ifdef CONFIG_CPU_BIG_ENDIAN
+//#define CPU_LE(code...)
+//#else
+#define CPU_LE(code...) code
+//#endif
+
+#define ENDPIPROC(x)                    \
+	.globl  __pi_##x;               \
+	.type   __pi_##x, %function;    \
+	.set    __pi_##x, x;            \
+	.size   __pi_##x, . - x;        \
+	ENDPROC(x)
+
+#endif /* !__HEADER_ARM64_COMMON_ASSEMBLER_H */

arch/arm64/kernel/include/cache.h

@@ -0,0 +1,7 @@
+/* cache.h COPYRIGHT FUJITSU LIMITED 2015 */
+#ifndef __HEADER_ARM64_COMMON_CACHE_H
+#define __HEADER_ARM64_COMMON_CACHE_H
+
+#define L1_CACHE_SHIFT	6
+
+#endif /* !__HEADER_ARM64_COMMON_CACHE_H */

arch/arm64/kernel/include/cas.h

@@ -0,0 +1,32 @@
+/* cas.h COPYRIGHT FUJITSU LIMITED 2015-2016 */
+#ifndef __HEADER_ARM64_COMMON_CAS_H
+#define __HEADER_ARM64_COMMON_CAS_H
+
+#include <arch/cpu.h>
+
+/* @ref.impl arch/arm64/include/asm/cmpxchg.h::__cmpxchg (size == 8 case) */
+/* 8 byte compare and swap, return 0:fail, 1:success */
+static inline int
+compare_and_swap(void *addr, unsigned long olddata, unsigned long newdata)
+{
+	unsigned long oldval = 0, res = 0;
+
+	smp_mb();
+	do {
+		asm volatile("// __cmpxchg8\n"
+		"	ldxr	%1, %2\n"
+		"	mov	%w0, #0\n"
+		"	cmp	%1, %3\n"
+		"	b.ne	1f\n"
+		"	stxr	%w0, %4, %2\n"
+		"1:\n"
+			: "=&r" (res), "=&r" (oldval), "+Q" (*(unsigned long *)addr)
+			: "Ir" (olddata), "r" (newdata)
+			: "cc");
+	} while (res);
+	smp_mb();
+
+	return (oldval == olddata);
+}
+
+#endif /* !__HEADER_ARM64_COMMON_CAS_H */

arch/arm64/kernel/include/compiler.h

@@ -0,0 +1,32 @@
+/* compiler.h COPYRIGHT FUJITSU LIMITED 2015-2016 */
+#ifndef __ASM_COMPILER_H
+#define __ASM_COMPILER_H
+
+/* @ref.impl arch/arm64/include/asm/compiler.h::__asmeq(x,y) */
+/*
+ * This is used to ensure the compiler did actually allocate the register we
+ * asked it for some inline assembly sequences.  Apparently we can't trust the
+ * compiler from one version to another so a bit of paranoia won't hurt.  This
+ * string is meant to be concatenated with the inline asm string and will
+ * cause compilation to stop on mismatch.  (for details, see gcc PR 15089)
+ */
+#define __asmeq(x, y)  ".ifnc " x "," y " ; .err ; .endif\n\t"
+
+/* @ref.impl include/linux/compiler.h::__section(S) */
+/* Simple shorthand for a section definition */
+# define __section(S)	__attribute__ ((__section__(#S)))
+
+/* @ref.impl include/linux/compiler.h::__aligned(x) */
+/*
+ * From the GCC manual:
+ *
+ * Many functions have no effects except the return value and their
+ * return value depends only on the parameters and/or global
+ * variables.  Such a function can be subject to common subexpression
+ * elimination and loop optimization just as an arithmetic operator
+ * would be.
+ * [...]
+ */
+#define __aligned(x)	__attribute__((aligned(x)))
+
+#endif	/* __ASM_COMPILER_H */

arch/arm64/kernel/include/const.h

@@ -0,0 +1,23 @@
+/* const.h COPYRIGHT FUJITSU LIMITED 2015 */
+#ifndef __HEADER_ARM64_COMMON_CONST_H
+#define __HEADER_ARM64_COMMON_CONST_H
+
+#ifndef __ASSEMBLY__
+#define __AC(X,Y)	(X##Y)
+#define _AC(X,Y)	__AC(X,Y)
+#define _AT(T,X)	((T)(X))
+#else /* !__ASSEMBLY__ */
+#define _AC(X,Y)	X
+#define _AT(T,X)	X
+#endif /* !__ASSEMBLY__ */
+
+#define _BITUL(x)	(_AC(1,UL) << (x))
+#define _BITULL(x)	(_AC(1,ULL) << (x))
+
+/*
+ * Allow for constants defined here to be used from assembly code
+ * by prepending the UL suffix only with actual C code compilation.
+ */
+#define UL(x) _AC(x, UL)
+
+#endif /* !__HEADER_ARM64_COMMON_CONST_H */

arch/arm64/kernel/include/context.h

@@ -0,0 +1,8 @@
+/* context.h COPYRIGHT FUJITSU LIMITED 2015 */
+#ifndef __HEADER_ARM64_COMMON_CONTEXT_H
+#define __HEADER_ARM64_COMMON_CONTEXT_H
+
+void switch_mm(struct page_table *pgtbl);
+void free_mmu_context(struct page_table *pgtbl);
+
+#endif /*__HEADER_ARM64_COMMON_CONTEXT_H*/

arch/arm64/kernel/include/cpufeature.h

@@ -0,0 +1,182 @@
+/* cpufeature.h COPYRIGHT FUJITSU LIMITED 2017 */
+
+#ifndef __ASM_CPUFEATURE_H
+#define __ASM_CPUFEATURE_H
+
+#include <types.h>
+#include <cpuinfo.h>
+#include <sysreg.h>
+
+/* @ref.impl arch/arm64/include/asm/cpufeature.h */
+/* CPU feature register tracking */
+enum ftr_type {
+	FTR_EXACT,	/* Use a predefined safe value */
+	FTR_LOWER_SAFE,	/* Smaller value is safe */
+	FTR_HIGHER_SAFE,/* Bigger value is safe */
+};
+
+#define FTR_STRICT	(1)	/* SANITY check strict matching required */
+#define FTR_NONSTRICT	(0)	/* SANITY check ignored */
+
+#define FTR_SIGNED	(1)	/* Value should be treated as signed */
+#define FTR_UNSIGNED	(0)	/* Value should be treated as unsigned */
+
+#define FTR_VISIBLE	(1)	/* Feature visible to the user space */
+#define FTR_HIDDEN	(0)	/* Feature is hidden from the user */
+
+/* @ref.impl arch/arm64/include/asm/cpufeature.h */
+struct arm64_ftr_bits {
+	int		sign;	/* Value is signed ? */
+	int		visible;
+	int		strict;	/* CPU Sanity check: strict matching required ? */
+	enum ftr_type	type;
+	uint8_t		shift;
+	uint8_t		width;
+	int64_t		safe_val; /* safe value for FTR_EXACT features */
+};
+
+/* @ref.impl arch/arm64/include/asm/cpufeature.h */
+/*
+ * @arm64_ftr_reg - Feature register
+ * @strict_mask		Bits which should match across all CPUs for sanity.
+ * @sys_val		Safe value across the CPUs (system view)
+ */
+struct arm64_ftr_reg {
+	const char			*name;
+	uint64_t			strict_mask;
+	uint64_t			user_mask;
+	uint64_t			sys_val;
+	uint64_t			user_val;
+	const struct arm64_ftr_bits	*ftr_bits;
+};
+
+/* @ref.impl arch/arm64/include/asm/cpufeature.h */
+extern struct arm64_ftr_reg arm64_ftr_reg_ctrel0;
+
+/* @ref.impl arch/arm64/include/asm/cpufeature.h */
+/* scope of capability check */
+enum {
+	SCOPE_SYSTEM,
+	SCOPE_LOCAL_CPU,
+};
+
+/* @ref.impl arch/arm64/include/asm/cpufeature.h */
+struct arm64_cpu_capabilities {
+	const char *desc;
+	uint16_t capability;
+	int def_scope;/* default scope */
+	int (*matches)(const struct arm64_cpu_capabilities *caps, int scope);
+	int (*enable)(void *);/* Called on all active CPUs */
+	uint32_t sys_reg;
+	uint8_t field_pos;
+	uint8_t min_field_value;
+	uint8_t hwcap_type;
+	int sign;
+	unsigned long hwcap;
+};
+
+/* @ref.impl include/linux/bitops.h */
+/*
+ * Create a contiguous bitmask starting at bit position @l and ending at
+ * position @h. For example
+ * GENMASK_ULL(39, 21) gives us the 64bit vector 0x000000ffffe00000.
+ */
+#define GENMASK(h, l) \
+	(((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h))))
+
+/* @ref.impl arch/arm64/include/asm/cpufeature.h */
+static inline uint64_t arm64_ftr_mask(const struct arm64_ftr_bits *ftrp)
+{
+	return (uint64_t)GENMASK(ftrp->shift + ftrp->width - 1, ftrp->shift);
+}
+
+/* @ref.impl arch/arm64/include/asm/cpufeature.h */
+static inline int
+cpuid_feature_extract_signed_field_width(uint64_t features, int field, int width)
+{
+	return (int64_t)(features << (64 - width - field)) >> (64 - width);
+}
+
+/* @ref.impl arch/arm64/include/asm/cpufeature.h */
+static inline int
+cpuid_feature_extract_signed_field(uint64_t features, int field)
+{
+	return cpuid_feature_extract_signed_field_width(features, field, 4);
+}
+
+/* @ref.impl arch/arm64/include/asm/cpufeature.h */
+static inline unsigned int
+cpuid_feature_extract_unsigned_field_width(uint64_t features, int field, int width)
+{
+	return (uint64_t)(features << (64 - width - field)) >> (64 - width);
+}
+
+/* @ref.impl arch/arm64/include/asm/cpufeature.h */
+static inline unsigned int
+cpuid_feature_extract_unsigned_field(uint64_t features, int field)
+{
+	return cpuid_feature_extract_unsigned_field_width(features, field, 4);
+}
+
+/* @ref.impl arch/arm64/include/asm/cpufeature.h */
+static inline uint64_t arm64_ftr_reg_user_value(const struct arm64_ftr_reg *reg)
+{
+	return (reg->user_val | (reg->sys_val & reg->user_mask));
+}
+
+/* @ref.impl arch/arm64/include/asm/cpufeature.h */
+static inline int
+cpuid_feature_extract_field_width(uint64_t features, int field, int width, int sign)
+{
+	return (sign) ?
+		cpuid_feature_extract_signed_field_width(features, field, width) :
+		cpuid_feature_extract_unsigned_field_width(features, field, width);
+}
+
+/* @ref.impl arch/arm64/include/asm/cpufeature.h */
+static inline int
+cpuid_feature_extract_field(uint64_t features, int field, int sign)
+{
+	return cpuid_feature_extract_field_width(features, field, 4, sign);
+}
+
+/* @ref.impl arch/arm64/include/asm/cpufeature.h */
+static inline int64_t arm64_ftr_value(const struct arm64_ftr_bits *ftrp, uint64_t val)
+{
+	return (int64_t)cpuid_feature_extract_field_width(val, ftrp->shift, ftrp->width, ftrp->sign);
+}
+
+/* @ref.impl arch/arm64/include/asm/cpufeature.h */
+static inline int id_aa64pfr0_32bit_el0(uint64_t pfr0)
+{
+	uint32_t val = cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_EL0_SHIFT);
+
+	return val == ID_AA64PFR0_EL0_32BIT_64BIT;
+}
+
+/* @ref.impl arch/arm64/include/asm/cpufeature.h */
+static inline int id_aa64pfr0_sve(uint64_t pfr0)
+{
+	uint32_t val = cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_SVE_SHIFT);
+
+	return val > 0;
+}
+
+void setup_cpu_features(void);
+void update_cpu_features(int cpu,
+			 struct cpuinfo_arm64 *info,
+			 struct cpuinfo_arm64 *boot);
+uint64_t read_system_reg(uint32_t id);
+void init_cpu_features(struct cpuinfo_arm64 *info);
+int enable_mrs_emulation(void);
+
+/* @ref.impl arch/arm64/include/asm/hwcap.h */
+enum {
+	CAP_HWCAP = 1,
+#ifdef CONFIG_COMPAT
+	CAP_COMPAT_HWCAP,
+	CAP_COMPAT_HWCAP2,
+#endif
+};
+
+#endif /* __ASM_CPUFEATURE_H */

arch/arm64/kernel/include/cpuinfo.h

@@ -0,0 +1,34 @@
+/* cpuinfo.h COPYRIGHT FUJITSU LIMITED 2016-2017 */
+#ifndef __HEADER_ARM64_COMMON_CPUINFO_H
+#define __HEADER_ARM64_COMMON_CPUINFO_H
+
+#include <types.h>
+
+/* @ref.impl arch/arm64/include/cpu.h */
+/*
+ * Records attributes of an individual CPU.
+ */
+struct cpuinfo_arm64 {
+	uint32_t reg_midr;
+	unsigned int hwid;	/* McKernel Original. */
+
+	uint32_t reg_ctr;
+	uint32_t reg_cntfrq;
+	uint32_t reg_dczid;
+	uint32_t reg_revidr;
+
+	uint64_t reg_id_aa64dfr0;
+	uint64_t reg_id_aa64dfr1;
+	uint64_t reg_id_aa64isar0;
+	uint64_t reg_id_aa64isar1;
+	uint64_t reg_id_aa64mmfr0;
+	uint64_t reg_id_aa64mmfr1;
+	uint64_t reg_id_aa64mmfr2;
+	uint64_t reg_id_aa64pfr0;
+	uint64_t reg_id_aa64pfr1;
+	uint64_t reg_id_aa64zfr0;
+
+	uint64_t reg_zcr;
+};
+
+#endif /* !__HEADER_ARM64_COMMON_CPUINFO_H */

arch/arm64/kernel/include/cpulocal.h

@@ -0,0 +1,13 @@
+/* cpulocal.h COPYRIGHT FUJITSU LIMITED 2015 */
+#ifndef __HEADER_ARM64_COMMON_CPULOCAL_H
+#define __HEADER_ARM64_COMMON_CPULOCAL_H
+
+#include <types.h>
+#include <registers.h>
+#include <thread_info.h>
+
+union arm64_cpu_local_variables *get_arm64_cpu_local_variable(int id);
+union arm64_cpu_local_variables *get_arm64_this_cpu_local(void);
+void *get_arm64_this_cpu_kstack(void);
+
+#endif /* !__HEADER_ARM64_COMMON_CPULOCAL_H */

arch/arm64/kernel/include/cputable.h

@@ -0,0 +1,12 @@
+/* cputable.h COPYRIGHT FUJITSU LIMITED 2015 */
+#ifndef __HEADER_ARM64_COMMON_CPUTABLE_H
+#define __HEADER_ARM64_COMMON_CPUTABLE_H
+
+struct cpu_info {
+	unsigned int	cpu_id_val;
+	unsigned int	cpu_id_mask;
+	const char	*cpu_name;
+	unsigned long	(*cpu_setup)(void);
+};
+
+#endif /* !__HEADER_ARM64_COMMON_CPUTABLE_H */

arch/arm64/kernel/include/cputype.h

@@ -0,0 +1,110 @@
+/* cputype.h COPYRIGHT FUJITSU LIMITED 2015-2017 */
+/* @ref.impl arch/arm64/include/asm/cputype.h */
+#ifndef __HEADER_ARM64_COMMON_CPUTYPE_H
+#define __HEADER_ARM64_COMMON_CPUTYPE_H
+
+#include <sysreg.h>
+
+#define MPIDR_LEVEL_BITS_SHIFT	3
+#define MPIDR_LEVEL_BITS	(1 << MPIDR_LEVEL_BITS_SHIFT)
+#define MPIDR_LEVEL_MASK	((1 << MPIDR_LEVEL_BITS) - 1)
+
+#define MPIDR_LEVEL_SHIFT(level) \
+	(((1 << level) >> 1) << MPIDR_LEVEL_BITS_SHIFT)
+
+#define MPIDR_AFFINITY_LEVEL(mpidr, level) \
+	((mpidr >> MPIDR_LEVEL_SHIFT(level)) & MPIDR_LEVEL_MASK)
+
+#define read_cpuid(reg) read_sysreg_s(SYS_ ## reg)
+
+#define MIDR_REVISION_MASK	0xf
+#define MIDR_REVISION(midr)	((midr) & MIDR_REVISION_MASK)
+
+#define MIDR_PARTNUM_SHIFT	4
+#define MIDR_PARTNUM_MASK	(0xfff << MIDR_PARTNUM_SHIFT)
+#define MIDR_PARTNUM(midr)	\
+	(((midr) & MIDR_PARTNUM_MASK) >> MIDR_PARTNUM_SHIFT)
+
+#define MIDR_ARCHITECTURE_SHIFT	16
+#define MIDR_ARCHITECTURE_MASK	(0xf << MIDR_ARCHITECTURE_SHIFT)
+#define MIDR_ARCHITECTURE(midr)	\
+	(((midr) & MIDR_ARCHITECTURE_MASK) >> MIDR_ARCHITECTURE_SHIFT)
+
+#define MIDR_VARIANT_SHIFT	20
+#define MIDR_VARIANT_MASK	(0xf << MIDR_VARIANT_SHIFT)
+#define MIDR_VARIANT(midr)	\
+	(((midr) & MIDR_VARIANT_MASK) >> MIDR_VARIANT_SHIFT)
+
+#define MIDR_IMPLEMENTOR_SHIFT	24
+#define MIDR_IMPLEMENTOR_MASK	(0xffU << MIDR_IMPLEMENTOR_SHIFT)
+#define MIDR_IMPLEMENTOR(midr)	\
+	(((midr) & MIDR_IMPLEMENTOR_MASK) >> MIDR_IMPLEMENTOR_SHIFT)
+
+#define MIDR_CPU_MODEL(imp, partnum) \
+	(((imp)			<< MIDR_IMPLEMENTOR_SHIFT) | \
+	(0xf			<< MIDR_ARCHITECTURE_SHIFT) | \
+	((partnum)		<< MIDR_PARTNUM_SHIFT))
+
+#define MIDR_CPU_VAR_REV(var, rev) \
+	(((var)	<< MIDR_VARIANT_SHIFT) | (rev))
+
+#define MIDR_CPU_MODEL_MASK (MIDR_IMPLEMENTOR_MASK | MIDR_PARTNUM_MASK | \
+			     MIDR_ARCHITECTURE_MASK)
+
+#define MIDR_IS_CPU_MODEL_RANGE(midr, model, rv_min, rv_max)		\
+({									\
+	u32 _model = (midr) & MIDR_CPU_MODEL_MASK;			\
+	u32 rv = (midr) & (MIDR_REVISION_MASK | MIDR_VARIANT_MASK);	\
+									\
+	_model == (model) && rv >= (rv_min) && rv <= (rv_max);		\
+})
+
+#define ARM_CPU_IMP_ARM			0x41
+#define ARM_CPU_IMP_APM			0x50
+#define ARM_CPU_IMP_CAVIUM		0x43
+#define ARM_CPU_IMP_BRCM		0x42
+#define ARM_CPU_IMP_QCOM		0x51
+
+#define ARM_CPU_PART_AEM_V8		0xD0F
+#define ARM_CPU_PART_FOUNDATION		0xD00
+#define ARM_CPU_PART_CORTEX_A57		0xD07
+#define ARM_CPU_PART_CORTEX_A72		0xD08
+#define ARM_CPU_PART_CORTEX_A53		0xD03
+#define ARM_CPU_PART_CORTEX_A73		0xD09
+#define ARM_CPU_PART_CORTEX_A75		0xD0A
+
+#define APM_CPU_PART_POTENZA		0x000
+
+#define CAVIUM_CPU_PART_THUNDERX	0x0A1
+#define CAVIUM_CPU_PART_THUNDERX_81XX	0x0A2
+#define CAVIUM_CPU_PART_THUNDERX_83XX	0x0A3
+#define CAVIUM_CPU_PART_THUNDERX2	0x0AF
+
+#define BRCM_CPU_PART_VULCAN		0x516
+
+#define QCOM_CPU_PART_FALKOR_V1		0x800
+#define QCOM_CPU_PART_FALKOR		0xC00
+
+#define MIDR_CORTEX_A53 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53)
+#define MIDR_CORTEX_A57 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57)
+#define MIDR_CORTEX_A72 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A72)
+#define MIDR_CORTEX_A73 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A73)
+#define MIDR_CORTEX_A75 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A75)
+#define MIDR_THUNDERX	MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX)
+#define MIDR_THUNDERX_81XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX)
+#define MIDR_THUNDERX_83XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_83XX)
+#define MIDR_CAVIUM_THUNDERX2 MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX2)
+#define MIDR_BRCM_VULCAN MIDR_CPU_MODEL(ARM_CPU_IMP_BRCM, BRCM_CPU_PART_VULCAN)
+#define MIDR_QCOM_FALKOR_V1 MIDR_CPU_MODEL(ARM_CPU_IMP_QCOM, QCOM_CPU_PART_FALKOR_V1)
+#define MIDR_QCOM_FALKOR MIDR_CPU_MODEL(ARM_CPU_IMP_QCOM, QCOM_CPU_PART_FALKOR)
+
+#ifndef __ASSEMBLY__
+
+static unsigned int read_cpuid_id(void)
+{
+	return read_cpuid(MIDR_EL1);
+}
+
+#endif /* !__ASSEMBLY__ */
+
+#endif /* !__HEADER_ARM64_COMMON_CPUTYPE_H */

arch/arm64/kernel/include/debug-monitors.h

@@ -0,0 +1,35 @@
+/* debug-monitors.h COPYRIGHT FUJITSU LIMITED 2016-2017 */
+#ifndef __HEADER_ARM64_COMMON_DEBUG_MONITORS_H
+#define __HEADER_ARM64_COMMON_DEBUG_MONITORS_H
+
+/* Low-level stepping controls. */
+#define DBG_MDSCR_SS		(1 << 0)
+#define DBG_SPSR_SS		(1 << 21)
+
+/* MDSCR_EL1 enabling bits */
+#define DBG_MDSCR_KDE		(1 << 13)
+#define DBG_MDSCR_MDE		(1 << 15)
+#define DBG_MDSCR_MASK		~(DBG_MDSCR_KDE | DBG_MDSCR_MDE)
+
+#define DBG_ESR_EVT(x)		(((x) >> 27) & 0x7)
+
+/* AArch64 */
+#define DBG_ESR_EVT_HWBP	0x0
+#define DBG_ESR_EVT_HWSS	0x1
+#define DBG_ESR_EVT_HWWP	0x2
+#define DBG_ESR_EVT_BRK		0x6
+
+#ifndef __ASSEMBLY__
+
+unsigned char debug_monitors_arch(void);
+void mdscr_write(unsigned int mdscr);
+unsigned int mdscr_read(void);
+void debug_monitors_init(void);
+
+struct pt_regs;
+void set_regs_spsr_ss(struct pt_regs *regs);
+void clear_regs_spsr_ss(struct pt_regs *regs);
+
+#endif /* !__ASSEMBLY__ */
+
+#endif /* !__HEADER_ARM64_COMMON_DEBUG_MONITORS_H */

arch/arm64/kernel/include/elf.h

@@ -0,0 +1,28 @@
+/* elf.h COPYRIGHT FUJITSU LIMITED 2015-2017 */
+#ifndef __HEADER_ARM64_COMMON_ELF_H
+#define __HEADER_ARM64_COMMON_ELF_H
+
+#include <ihk/context.h>
+
+/* ELF target machines defined */
+#define EM_AARCH64	183
+
+/* ELF header defined */
+#define ELF_CLASS	ELFCLASS64
+#define ELF_DATA	ELFDATA2LSB
+#define ELF_OSABI	ELFOSABI_NONE
+#define ELF_ABIVERSION	El_ABIVERSION_NONE
+#define ELF_ARCH	EM_AARCH64
+
+#define ELF_NGREG64 (sizeof (struct user_pt_regs) / sizeof(elf_greg64_t))
+
+/* PTRACE_GETREGSET and PTRACE_SETREGSET requests. */
+#define NT_ARM_TLS		0x401	/* ARM TLS register */
+#define NT_ARM_HW_BREAK		0x402	/* ARM hardware breakpoint registers */
+#define NT_ARM_HW_WATCH		0x403	/* ARM hardware watchpoint registers */
+#define NT_ARM_SYSTEM_CALL	0x404	/* ARM system call number */
+#define NT_ARM_SVE		0x405	/* ARM Scalable Vector Extension registers */
+
+typedef elf_greg64_t elf_gregset64_t[ELF_NGREG64];
+
+#endif /* __HEADER_ARM64_COMMON_ELF_H */

arch/arm64/kernel/include/elfnote.h

@@ -0,0 +1,60 @@
+/* elfnote.h COPYRIGHT FUJITSU LIMITED 2016 */
+/* @ref.impl include/linux/elfnote.h */
+/*
+ * Helper macros to generate ELF Note structures, which are put into a
+ * PT_NOTE segment of the final vmlinux image.  These are useful for
+ * including name-value pairs of metadata into the kernel binary (or
+ * modules?) for use by external programs.
+ *
+ * Each note has three parts: a name, a type and a desc.  The name is
+ * intended to distinguish the note's originator, so it would be a
+ * company, project, subsystem, etc; it must be in a suitable form for
+ * use in a section name.  The type is an integer which is used to tag
+ * the data, and is considered to be within the "name" namespace (so
+ * "FooCo"'s type 42 is distinct from "BarProj"'s type 42).  The
+ * "desc" field is the actual data.  There are no constraints on the
+ * desc field's contents, though typically they're fairly small.
+ *
+ * All notes from a given NAME are put into a section named
+ * .note.NAME.  When the kernel image is finally linked, all the notes
+ * are packed into a single .notes section, which is mapped into the
+ * PT_NOTE segment.  Because notes for a given name are grouped into
+ * the same section, they'll all be adjacent the output file.
+ *
+ * This file defines macros for both C and assembler use.  Their
+ * syntax is slightly different, but they're semantically similar.
+ *
+ * See the ELF specification for more detail about ELF notes.
+ */
+#ifndef __HEADER_ARM64_COMMON_ELFNOTE_H
+#define __HEADER_ARM64_COMMON_ELFNOTE_H
+
+#ifdef __ASSEMBLER__
+
+/*
+ * Generate a structure with the same shape as Elf{32,64}_Nhdr (which
+ * turn out to be the same size and shape), followed by the name and
+ * desc data with appropriate padding.  The 'desctype' argument is the
+ * assembler pseudo op defining the type of the data e.g. .asciz while
+ * 'descdata' is the data itself e.g.  "hello, world".
+ *
+ * e.g. ELFNOTE(XYZCo, 42, .asciz, "forty-two")
+ *      ELFNOTE(XYZCo, 12, .long, 0xdeadbeef)
+ */
+#define ELFNOTE_START(name, type, flags)        \
+.pushsection .note.name, flags,@note    ;       \
+  .balign 4                             ;       \
+  .long 2f - 1f         /* namesz */    ;       \
+  .long 4484f - 3f      /* descsz */    ;       \
+  .long type                            ;       \
+1:.asciz #name                          ;       \
+2:.balign 4                             ;       \
+3:
+
+#define ELFNOTE_END                             \
+4484:.balign 4                          ;       \
+.popsection                             ;
+
+#endif	/* __ASSEMBLER__ */
+
+#endif	/* !__HEADER_ARM64_COMMON_ELFNOTE_H */

arch/arm64/kernel/include/errno.h

@@ -1,40 +1,8 @@
-#ifndef _ASM_GENERIC_ERRNO_BASE_H
-#define _ASM_GENERIC_ERRNO_BASE_H
+/* errno.h COPYRIGHT FUJITSU LIMITED 2016 */
+#ifndef __HEADER_ARM64_COMMON_ERRNO_H
+#define __HEADER_ARM64_COMMON_ERRNO_H
 
-#define	EPERM		 1	/* Operation not permitted */
-#define	ENOENT		 2	/* No such file or directory */
-#define	ESRCH		 3	/* No such process */
-#define	EINTR		 4	/* Interrupted system call */
-#define	EIO		 5	/* I/O error */
-#define	ENXIO		 6	/* No such device or address */
-#define	E2BIG		 7	/* Argument list too long */
-#define	ENOEXEC		 8	/* Exec format error */
-#define	EBADF		 9	/* Bad file number */
-#define	ECHILD		10	/* No child processes */
-#define	EAGAIN		11	/* Try again */
-#define	ENOMEM		12	/* Out of memory */
-#define	EACCES		13	/* Permission denied */
-#define	EFAULT		14	/* Bad address */
-#define	ENOTBLK		15	/* Block device required */
-#define	EBUSY		16	/* Device or resource busy */
-#define	EEXIST		17	/* File exists */
-#define	EXDEV		18	/* Cross-device link */
-#define	ENODEV		19	/* No such device */
-#define	ENOTDIR		20	/* Not a directory */
-#define	EISDIR		21	/* Is a directory */
-#define	EINVAL		22	/* Invalid argument */
-#define	ENFILE		23	/* File table overflow */
-#define	EMFILE		24	/* Too many open files */
-#define	ENOTTY		25	/* Not a typewriter */
-#define	ETXTBSY		26	/* Text file busy */
-#define	EFBIG		27	/* File too large */
-#define	ENOSPC		28	/* No space left on device */
-#define	ESPIPE		29	/* Illegal seek */
-#define	EROFS		30	/* Read-only file system */
-#define	EMLINK		31	/* Too many links */
-#define	EPIPE		32	/* Broken pipe */
-#define	EDOM		33	/* Math argument out of domain of func */
-#define	ERANGE		34	/* Math result not representable */
+#include <generic-errno.h>
 
 #define	EDEADLK		35	/* Resource deadlock would occur */
 #define	ENAMETOOLONG	36	/* File name too long */
@@ -141,29 +109,4 @@
 
 #define ERFKILL		132	/* Operation not possible due to RF-kill */
 
-
-#ifdef __KERNEL__
-
-/* Should never be seen by user programs */
-#define ERESTARTSYS	512
-#define ERESTARTNOINTR	513
-#define ERESTARTNOHAND	514	/* restart if no handler.. */
-#define ENOIOCTLCMD	515	/* No ioctl command */
-#define ERESTART_RESTARTBLOCK 516 /* restart by calling sys_restart_syscall */
-
-/* Defined for the NFSv3 protocol */
-#define EBADHANDLE	521	/* Illegal NFS file handle */
-#define ENOTSYNC	522	/* Update synchronization mismatch */
-#define EBADCOOKIE	523	/* Cookie is stale */
-#define ENOTSUPP	524	/* Operation is not supported */
-#define ETOOSMALL	525	/* Buffer or request is too small */
-#define ESERVERFAULT	526	/* An untranslatable error occurred */
-#define EBADTYPE	527	/* Type not supported by server */
-#define EJUKEBOX	528	/* Request initiated, but will not complete before timeout */
-#define EIOCBQUEUED	529	/* iocb queued, will get completion event */
-#define EIOCBRETRY	530	/* iocb queued, will trigger a retry */
-
-#endif
-
-
 #endif

arch/arm64/kernel/include/esr.h

@@ -0,0 +1,180 @@
+/* esr.h COPYRIGHT FUJITSU LIMITED 2015-2017 */
+/*
+ * Copyright (C) 2013 - ARM Ltd
+ * Author: Marc Zyngier <marc.zyngier@arm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __ASM_ESR_H
+#define __ASM_ESR_H
+
+#include <const.h>
+
+#define ESR_ELx_EC_UNKNOWN	(0x00)
+#define ESR_ELx_EC_WFx		(0x01)
+/* Unallocated EC: 0x02 */
+#define ESR_ELx_EC_CP15_32	(0x03)
+#define ESR_ELx_EC_CP15_64	(0x04)
+#define ESR_ELx_EC_CP14_MR	(0x05)
+#define ESR_ELx_EC_CP14_LS	(0x06)
+#define ESR_ELx_EC_FP_ASIMD	(0x07)
+#define ESR_ELx_EC_CP10_ID	(0x08)
+/* Unallocated EC: 0x09 - 0x0B */
+#define ESR_ELx_EC_CP14_64	(0x0C)
+/* Unallocated EC: 0x0d */
+#define ESR_ELx_EC_ILL		(0x0E)
+/* Unallocated EC: 0x0F - 0x10 */
+#define ESR_ELx_EC_SVC32	(0x11)
+#define ESR_ELx_EC_HVC32	(0x12)
+#define ESR_ELx_EC_SMC32	(0x13)
+/* Unallocated EC: 0x14 */
+#define ESR_ELx_EC_SVC64	(0x15)
+#define ESR_ELx_EC_HVC64	(0x16)
+#define ESR_ELx_EC_SMC64	(0x17)
+#define ESR_ELx_EC_SYS64	(0x18)
+#define ESR_ELx_EC_SVE		(0x19)
+/* Unallocated EC: 0x1A - 0x1E */
+#define ESR_ELx_EC_IMP_DEF	(0x1f)
+#define ESR_ELx_EC_IABT_LOW	(0x20)
+#define ESR_ELx_EC_IABT_CUR	(0x21)
+#define ESR_ELx_EC_PC_ALIGN	(0x22)
+/* Unallocated EC: 0x23 */
+#define ESR_ELx_EC_DABT_LOW	(0x24)
+#define ESR_ELx_EC_DABT_CUR	(0x25)
+#define ESR_ELx_EC_SP_ALIGN	(0x26)
+/* Unallocated EC: 0x27 */
+#define ESR_ELx_EC_FP_EXC32	(0x28)
+/* Unallocated EC: 0x29 - 0x2B */
+#define ESR_ELx_EC_FP_EXC64	(0x2C)
+/* Unallocated EC: 0x2D - 0x2E */
+#define ESR_ELx_EC_SERROR	(0x2F)
+#define ESR_ELx_EC_BREAKPT_LOW	(0x30)
+#define ESR_ELx_EC_BREAKPT_CUR	(0x31)
+#define ESR_ELx_EC_SOFTSTP_LOW	(0x32)
+#define ESR_ELx_EC_SOFTSTP_CUR	(0x33)
+#define ESR_ELx_EC_WATCHPT_LOW	(0x34)
+#define ESR_ELx_EC_WATCHPT_CUR	(0x35)
+/* Unallocated EC: 0x36 - 0x37 */
+#define ESR_ELx_EC_BKPT32	(0x38)
+/* Unallocated EC: 0x39 */
+#define ESR_ELx_EC_VECTOR32	(0x3A)
+/* Unallocted EC: 0x3B */
+#define ESR_ELx_EC_BRK64	(0x3C)
+/* Unallocated EC: 0x3D - 0x3F */
+#define ESR_ELx_EC_MAX		(0x3F)
+
+#define ESR_ELx_EC_SHIFT	(26)
+#define ESR_ELx_EC_MASK		(UL(0x3F) << ESR_ELx_EC_SHIFT)
+#define ESR_ELx_EC(esr)		(((esr) & ESR_ELx_EC_MASK) >> ESR_ELx_EC_SHIFT)
+
+#define ESR_ELx_IL		(UL(1) << 25)
+#define ESR_ELx_ISS_MASK	(ESR_ELx_IL - 1)
+
+/* ISS field definitions shared by different classes */
+#define ESR_ELx_WNR		(UL(1) << 6)
+
+/* Shared ISS field definitions for Data/Instruction aborts */
+#define ESR_ELx_EA		(UL(1) << 9)
+#define ESR_ELx_S1PTW		(UL(1) << 7)
+
+/* Shared ISS fault status code(IFSC/DFSC) for Data/Instruction aborts */
+#define ESR_ELx_FSC		(0x3F)
+#define ESR_ELx_FSC_TYPE	(0x3C)
+#define ESR_ELx_FSC_EXTABT	(0x10)
+#define ESR_ELx_FSC_ACCESS	(0x08)
+#define ESR_ELx_FSC_FAULT	(0x04)
+#define ESR_ELx_FSC_PERM	(0x0C)
+
+/* ISS field definitions for Data Aborts */
+#define ESR_ELx_ISV		(UL(1) << 24)
+#define ESR_ELx_SAS_SHIFT	(22)
+#define ESR_ELx_SAS		(UL(3) << ESR_ELx_SAS_SHIFT)
+#define ESR_ELx_SSE		(UL(1) << 21)
+#define ESR_ELx_SRT_SHIFT	(16)
+#define ESR_ELx_SRT_MASK	(UL(0x1F) << ESR_ELx_SRT_SHIFT)
+#define ESR_ELx_SF 		(UL(1) << 15)
+#define ESR_ELx_AR 		(UL(1) << 14)
+#define ESR_ELx_CM 		(UL(1) << 8)
+
+/* ISS field definitions for exceptions taken in to Hyp */
+#define ESR_ELx_CV		(UL(1) << 24)
+#define ESR_ELx_COND_SHIFT	(20)
+#define ESR_ELx_COND_MASK	(UL(0xF) << ESR_ELx_COND_SHIFT)
+#define ESR_ELx_WFx_ISS_WFE	(UL(1) << 0)
+#define ESR_ELx_xVC_IMM_MASK	((1UL << 16) - 1)
+
+/* ESR value templates for specific events */
+
+/* BRK instruction trap from AArch64 state */
+#define ESR_ELx_VAL_BRK64(imm)					\
+	((ESR_ELx_EC_BRK64 << ESR_ELx_EC_SHIFT) | ESR_ELx_IL |	\
+	 ((imm) & 0xffff))
+
+/* ISS field definitions for System instruction traps */
+#define ESR_ELx_SYS64_ISS_RES0_SHIFT	22
+#define ESR_ELx_SYS64_ISS_RES0_MASK	(UL(0x7) << ESR_ELx_SYS64_ISS_RES0_SHIFT)
+#define ESR_ELx_SYS64_ISS_DIR_MASK	0x1
+#define ESR_ELx_SYS64_ISS_DIR_READ	0x1
+#define ESR_ELx_SYS64_ISS_DIR_WRITE	0x0
+
+#define ESR_ELx_SYS64_ISS_RT_SHIFT	5
+#define ESR_ELx_SYS64_ISS_RT_MASK	(UL(0x1f) << ESR_ELx_SYS64_ISS_RT_SHIFT)
+#define ESR_ELx_SYS64_ISS_CRM_SHIFT	1
+#define ESR_ELx_SYS64_ISS_CRM_MASK	(UL(0xf) << ESR_ELx_SYS64_ISS_CRM_SHIFT)
+#define ESR_ELx_SYS64_ISS_CRN_SHIFT	10
+#define ESR_ELx_SYS64_ISS_CRN_MASK	(UL(0xf) << ESR_ELx_SYS64_ISS_CRN_SHIFT)
+#define ESR_ELx_SYS64_ISS_OP1_SHIFT	14
+#define ESR_ELx_SYS64_ISS_OP1_MASK	(UL(0x7) << ESR_ELx_SYS64_ISS_OP1_SHIFT)
+#define ESR_ELx_SYS64_ISS_OP2_SHIFT	17
+#define ESR_ELx_SYS64_ISS_OP2_MASK	(UL(0x7) << ESR_ELx_SYS64_ISS_OP2_SHIFT)
+#define ESR_ELx_SYS64_ISS_OP0_SHIFT	20
+#define ESR_ELx_SYS64_ISS_OP0_MASK	(UL(0x3) << ESR_ELx_SYS64_ISS_OP0_SHIFT)
+#define ESR_ELx_SYS64_ISS_SYS_MASK	(ESR_ELx_SYS64_ISS_OP0_MASK | \
+					 ESR_ELx_SYS64_ISS_OP1_MASK | \
+					 ESR_ELx_SYS64_ISS_OP2_MASK | \
+					 ESR_ELx_SYS64_ISS_CRN_MASK | \
+					 ESR_ELx_SYS64_ISS_CRM_MASK)
+#define ESR_ELx_SYS64_ISS_SYS_VAL(op0, op1, op2, crn, crm) \
+					(((op0) << ESR_ELx_SYS64_ISS_OP0_SHIFT) | \
+					 ((op1) << ESR_ELx_SYS64_ISS_OP1_SHIFT) | \
+					 ((op2) << ESR_ELx_SYS64_ISS_OP2_SHIFT) | \
+					 ((crn) << ESR_ELx_SYS64_ISS_CRN_SHIFT) | \
+					 ((crm) << ESR_ELx_SYS64_ISS_CRM_SHIFT))
+
+#define ESR_ELx_SYS64_ISS_SYS_OP_MASK	(ESR_ELx_SYS64_ISS_SYS_MASK | \
+					 ESR_ELx_SYS64_ISS_DIR_MASK)
+/*
+ * User space cache operations have the following sysreg encoding
+ * in System instructions.
+ * op0=1, op1=3, op2=1, crn=7, crm={ 5, 10, 11, 14 }, WRITE (L=0)
+ */
+#define ESR_ELx_SYS64_ISS_CRM_DC_CIVAC	14
+#define ESR_ELx_SYS64_ISS_CRM_DC_CVAU	11
+#define ESR_ELx_SYS64_ISS_CRM_DC_CVAC	10
+#define ESR_ELx_SYS64_ISS_CRM_IC_IVAU	5
+
+#define ESR_ELx_SYS64_ISS_EL0_CACHE_OP_MASK	(ESR_ELx_SYS64_ISS_OP0_MASK | \
+						 ESR_ELx_SYS64_ISS_OP1_MASK | \
+						 ESR_ELx_SYS64_ISS_OP2_MASK | \
+						 ESR_ELx_SYS64_ISS_CRN_MASK | \
+						 ESR_ELx_SYS64_ISS_DIR_MASK)
+#define ESR_ELx_SYS64_ISS_EL0_CACHE_OP_VAL \
+				(ESR_ELx_SYS64_ISS_SYS_VAL(1, 3, 1, 7, 0) | \
+				 ESR_ELx_SYS64_ISS_DIR_WRITE)
+
+#define ESR_ELx_SYS64_ISS_SYS_CTR	ESR_ELx_SYS64_ISS_SYS_VAL(3, 3, 1, 0, 0)
+#define ESR_ELx_SYS64_ISS_SYS_CTR_READ	(ESR_ELx_SYS64_ISS_SYS_CTR | \
+					 ESR_ELx_SYS64_ISS_DIR_READ)
+
+#endif /* __ASM_ESR_H */

arch/arm64/kernel/include/fpsimd.h

@@ -0,0 +1,102 @@
+/* fpsimd.h COPYRIGHT FUJITSU LIMITED 2016-2019 */
+#ifndef __HEADER_ARM64_COMMON_FPSIMD_H
+#define __HEADER_ARM64_COMMON_FPSIMD_H
+
+#include <ptrace.h>
+
+#ifndef __ASSEMBLY__
+
+/*
+ * FP/SIMD storage area has:
+ *  - FPSR and FPCR
+ *  - 32 128-bit data registers
+ *
+ * Note that user_fpsimd forms a prefix of this structure, which is
+ * relied upon in the ptrace FP/SIMD accessors.
+ */
+/* @ref.impl arch/arm64/include/asm/fpsimd.h::struct fpsimd_state */
+struct fpsimd_state {
+	union {
+		struct user_fpsimd_state user_fpsimd;
+		struct {
+			__uint128_t vregs[32];
+			unsigned int fpsr;
+			unsigned int fpcr;
+			/*
+			 * For ptrace compatibility, pad to next 128-bit
+			 * boundary here if extending this struct.
+			 */
+		};
+	};
+	/* the id of the last cpu to have restored this state */
+	unsigned int cpu;
+};
+
+/* need for struct process */
+typedef struct fpsimd_state fp_regs_struct;
+
+extern void thread_fpsimd_to_sve(struct thread *thread, fp_regs_struct *fp_regs);
+extern void thread_sve_to_fpsimd(struct thread *thread, fp_regs_struct *fp_regs);
+
+#ifdef CONFIG_ARM64_SVE
+
+extern size_t sve_state_size(struct thread const *thread);
+extern void sve_free(struct thread *thread);
+extern int sve_alloc(struct thread *thread);
+extern void sve_save_state(void *state, unsigned int *pfpsr);
+extern void sve_load_state(void const *state, unsigned int const *pfpsr, unsigned long vq_minus_1);
+extern unsigned int sve_get_vl(void);
+extern int sve_set_thread_vl(unsigned long arg);
+extern int sve_get_thread_vl(void);
+extern int sve_set_vector_length(struct thread *thread, unsigned long vl, unsigned long flags);
+
+#define SVE_SET_VL(arg)	sve_set_thread_vl(arg)
+#define SVE_GET_VL()	sve_get_thread_vl()
+
+/* Maximum VL that SVE VL-agnostic software can transparently support */
+#define SVE_VL_ARCH_MAX 0x100
+
+#else /* CONFIG_ARM64_SVE */
+
+#include <ihk/debug.h>
+#include <errno.h>
+
+static void sve_save_state(void *state, unsigned int *pfpsr)
+{
+	panic("PANIC:sve_save_state() was called CONFIG_ARM64_SVE off.\n");
+}
+
+static void sve_load_state(void const *state, unsigned int const *pfpsr, unsigned long vq_minus_1)
+{
+	panic("PANIC:sve_load_state() was called CONFIG_ARM64_SVE off.\n");
+}
+
+static unsigned int sve_get_vl(void)
+{
+	panic("PANIC:sve_get_vl() was called CONFIG_ARM64_SVE off.\n");
+	return (unsigned int)-1;
+}
+
+static int sve_set_vector_length(struct thread *thread, unsigned long vl, unsigned long flags)
+{
+	return -EINVAL;
+}
+
+/* for prctl syscall */
+#define SVE_SET_VL(a)	(-EINVAL)
+#define SVE_GET_VL()	(-EINVAL)
+
+#endif /* CONFIG_ARM64_SVE */
+
+extern void sve_setup(void);
+extern void fpsimd_save_state(struct fpsimd_state *state);
+extern void fpsimd_load_state(struct fpsimd_state *state);
+extern void thread_fpsimd_save(struct thread *thread);
+extern void thread_fpsimd_load(struct thread *thread);
+
+extern int sve_max_vl;
+extern int sve_default_vl;
+
+#endif /* !__ASSEMBLY__ */
+
+#endif /* !__HEADER_ARM64_COMMON_FPSIMD_H */

arch/arm64/kernel/include/fpsimdmacros.h

@@ -0,0 +1,151 @@
+/* fpsimdmacros.h COPYRIGHT FUJITSU LIMITED 2016-2017 */
+
+.macro _check_reg nr
+	.if (\nr) < 0 || (\nr) > 31
+		.error "Bad register number \nr."
+	.endif
+.endm
+
+.macro _check_zreg znr
+	.if (\znr) < 0 || (\znr) > 31
+		.error "Bad Scalable Vector Extension vector register number \znr."
+	.endif
+.endm
+
+.macro _check_preg pnr
+	.if (\pnr) < 0 || (\pnr) > 15
+		.error "Bad Scalable Vector Extension predicate register number \pnr."
+	.endif
+.endm
+
+.macro _check_num n, min, max
+	.if (\n) < (\min) || (\n) > (\max)
+		.error "Number \n out of range [\min,\max]"
+	.endif
+.endm
+
+.macro _zstrv znt, nspb, ioff=0
+	_check_zreg \znt
+	_check_reg \nspb
+	_check_num (\ioff), -0x100, 0xff
+	.inst	0xe5804000			\
+		| (\znt)			\
+		| ((\nspb) << 5)		\
+		| (((\ioff) & 7) << 10)		\
+		| (((\ioff) & 0x1f8) << 13)
+.endm
+
+.macro _zldrv znt, nspb, ioff=0
+	_check_zreg \znt
+	_check_reg \nspb
+	_check_num (\ioff), -0x100, 0xff
+	.inst	0x85804000			\
+		| (\znt)			\
+		| ((\nspb) << 5)		\
+		| (((\ioff) & 7) << 10)		\
+		| (((\ioff) & 0x1f8) << 13)
+.endm
+
+.macro _zstrp pnt, nspb, ioff=0
+	_check_preg \pnt
+	_check_reg \nspb
+	_check_num (\ioff), -0x100, 0xff
+	.inst	0xe5800000			\
+		| (\pnt)			\
+		| ((\nspb) << 5)		\
+		| (((\ioff) & 7) << 10)		\
+		| (((\ioff) & 0x1f8) << 13)
+.endm
+
+.macro _zldrp pnt, nspb, ioff=0
+	_check_preg \pnt
+	_check_reg \nspb
+	_check_num (\ioff), -0x100, 0xff
+	.inst	0x85800000			\
+		| (\pnt)			\
+		| ((\nspb) << 5)		\
+		| (((\ioff) & 7) << 10)		\
+		| (((\ioff) & 0x1f8) << 13)
+.endm
+
+.macro _zrdvl nspd, is1
+	_check_reg \nspd
+	_check_num (\is1), -0x20, 0x1f
+	.inst	0x04bf5000			\
+		| (\nspd)			\
+		| (((\is1) & 0x3f) << 5)
+.endm
+
+.macro _zrdffr pnd
+	_check_preg \pnd
+	.inst	0x2519f000			\
+		| (\pnd)
+.endm
+
+.macro _zwrffr pnd
+	_check_preg \pnd
+	.inst	0x25289000			\
+		| ((\pnd) << 5)
+.endm
+
+.macro for from, to, insn
+	.if (\from) >= (\to)
+		\insn	(\from)
+		.exitm
+	.endif
+
+	for \from, ((\from) + (\to)) / 2, \insn
+	for ((\from) + (\to)) / 2 + 1, \to, \insn
+.endm
+
+.macro sve_save nb, xpfpsr, ntmp
+	.macro savez n
+		_zstrv	\n, \nb, (\n) - 34
+	.endm
+
+	.macro savep n
+		_zstrp	\n, \nb, (\n) - 16
+	.endm
+
+	for	0, 31, savez
+	for	0, 15, savep
+	_zrdffr	0
+	_zstrp	0, \nb
+	_zldrp	0, \nb, -16
+
+	mrs	x\ntmp, fpsr
+	str	w\ntmp, [\xpfpsr]
+	mrs	x\ntmp, fpcr
+	str	w\ntmp, [\xpfpsr, #4]
+
+	.purgem savez
+	.purgem savep
+.endm
+
+.macro sve_load nb, xpfpsr, xvqminus1 ntmp
+	mrs_s	x\ntmp, SYS_ZCR_EL1
+	bic	x\ntmp, x\ntmp, ZCR_EL1_LEN_MASK
+	orr	x\ntmp, x\ntmp, \xvqminus1
+	msr_s	SYS_ZCR_EL1, x\ntmp // self-synchronising
+
+	.macro loadz n	
+		_zldrv	\n, \nb, (\n) - 34
+	.endm
+
+	.macro loadp n
+		_zldrp	\n, \nb, (\n) - 16
+	.endm
+
+	for	0, 31, loadz
+	_zldrp	0, \nb
+	_zwrffr	0
+	for	0, 15, loadp
+
+	ldr	w\ntmp, [\xpfpsr]
+	msr	fpsr, x\ntmp
+	ldr	w\ntmp, [\xpfpsr, #4]
+	msr	fpcr, x\ntmp
+
+	.purgem loadz
+	.purgem loadp
+.endm

arch/arm64/kernel/include/hw_breakpoint.h

@@ -0,0 +1,92 @@
+/* hw_breakpoint.h COPYRIGHT FUJITSU LIMITED 2016 */
+#ifndef __HEADER_ARM64_COMMON_HW_BREAKPOINT_H
+#define __HEADER_ARM64_COMMON_HW_BREAKPOINT_H
+
+#include <ihk/types.h>
+
+int hw_breakpoint_slots(int type);
+unsigned long read_wb_reg(int reg, int n);
+void write_wb_reg(int reg, int n, unsigned long val);
+void hw_breakpoint_reset(void);
+void arch_hw_breakpoint_init(void);
+
+struct user_hwdebug_state;
+int arch_validate_hwbkpt_settings(long note_type, struct user_hwdebug_state *hws, size_t len);
+
+extern int core_num_brps;
+extern int core_num_wrps;
+
+/* @ref.impl include/uapi/linux/hw_breakpoint.h::HW_BREAKPOINT_LEN_n, HW_BREAKPOINT_xxx, bp_type_idx */
+enum {
+	HW_BREAKPOINT_LEN_1 = 1,
+	HW_BREAKPOINT_LEN_2 = 2,
+	HW_BREAKPOINT_LEN_4 = 4,
+	HW_BREAKPOINT_LEN_8 = 8,
+};
+
+enum {
+	HW_BREAKPOINT_EMPTY	= 0,
+	HW_BREAKPOINT_R		= 1,
+	HW_BREAKPOINT_W		= 2,
+	HW_BREAKPOINT_RW	= HW_BREAKPOINT_R | HW_BREAKPOINT_W,
+	HW_BREAKPOINT_X		= 4,
+	HW_BREAKPOINT_INVALID	= HW_BREAKPOINT_RW | HW_BREAKPOINT_X,
+};
+
+enum bp_type_idx {
+	TYPE_INST	= 0,
+	TYPE_DATA	= 1,
+	TYPE_MAX
+};
+
+/* Breakpoint */
+#define ARM_BREAKPOINT_EXECUTE	0
+
+/* Watchpoints */
+#define ARM_BREAKPOINT_LOAD	1
+#define ARM_BREAKPOINT_STORE	2
+#define AARCH64_ESR_ACCESS_MASK	(1 << 6)
+
+/* Privilege Levels */
+#define AARCH64_BREAKPOINT_EL1	1
+#define AARCH64_BREAKPOINT_EL0	2
+
+/* Lengths */
+#define ARM_BREAKPOINT_LEN_1	0x1
+#define ARM_BREAKPOINT_LEN_2	0x3
+#define ARM_BREAKPOINT_LEN_4	0xf
+#define ARM_BREAKPOINT_LEN_8	0xff
+
+/* @ref.impl arch/arm64/include/asm/hw_breakpoint.h::ARM_MAX_[BRP|WRP] */
+/*
+ * Limits.
+ * Changing these will require modifications to the register accessors.
+ */
+#define ARM_MAX_BRP		16
+#define ARM_MAX_WRP		16
+
+/* @ref.impl arch/arm64/include/asm/hw_breakpoint.h::AARCH64_DBG_REG_xxx */
+/* Virtual debug register bases. */
+#define AARCH64_DBG_REG_BVR	0
+#define AARCH64_DBG_REG_BCR	(AARCH64_DBG_REG_BVR + ARM_MAX_BRP)
+#define AARCH64_DBG_REG_WVR	(AARCH64_DBG_REG_BCR + ARM_MAX_BRP)
+#define AARCH64_DBG_REG_WCR	(AARCH64_DBG_REG_WVR + ARM_MAX_WRP)
+
+/* @ref.impl arch/arm64/include/asm/hw_breakpoint.h::AARCH64_DBG_REG_NAME_xxx */
+/* Debug register names. */
+#define AARCH64_DBG_REG_NAME_BVR	"bvr"
+#define AARCH64_DBG_REG_NAME_BCR	"bcr"
+#define AARCH64_DBG_REG_NAME_WVR	"wvr"
+#define AARCH64_DBG_REG_NAME_WCR	"wcr"
+
+/* @ref.impl arch/arm64/include/asm/hw_breakpoint.h::AARCH64_DBG_[READ|WRITE] */
+/* Accessor macros for the debug registers. */
+#define AARCH64_DBG_READ(N, REG, VAL) do {\
+	asm volatile("mrs %0, dbg" REG #N "_el1" : "=r" (VAL));\
+} while (0)
+
+#define AARCH64_DBG_WRITE(N, REG, VAL) do {\
+	asm volatile("msr dbg" REG #N "_el1, %0" :: "r" (VAL));\
+} while (0)
+
+#endif /* !__HEADER_ARM64_COMMON_HW_BREAKPOINT_H */

arch/arm64/kernel/include/hwcap.h

@@ -0,0 +1,26 @@
+/* hwcap.h COPYRIGHT FUJITSU LIMITED 2017 */
+#ifndef _UAPI__ASM_HWCAP_H
+#define _UAPI__ASM_HWCAP_H
+
+/*
+ * HWCAP flags - for elf_hwcap (in kernel) and AT_HWCAP
+ */
+#define HWCAP_FP		(1 << 0)
+#define HWCAP_ASIMD		(1 << 1)
+#define HWCAP_EVTSTRM		(1 << 2)
+#define HWCAP_AES		(1 << 3)
+#define HWCAP_PMULL		(1 << 4)
+#define HWCAP_SHA1		(1 << 5)
+#define HWCAP_SHA2		(1 << 6)
+#define HWCAP_CRC32		(1 << 7)
+#define HWCAP_ATOMICS		(1 << 8)
+#define HWCAP_FPHP		(1 << 9)
+#define HWCAP_ASIMDHP		(1 << 10)
+#define HWCAP_CPUID		(1 << 11)
+#define HWCAP_ASIMDRDM		(1 << 12)
+#define HWCAP_SVE		(1 << 13)
+
+unsigned long arch_get_hwcap(void);
+extern unsigned long elf_hwcap;
+
+#endif /* _UAPI__ASM_HWCAP_H */

arch/arm64/kernel/include/ihk/atomic.h

@@ -0,0 +1,365 @@
+/* atomic.h COPYRIGHT FUJITSU LIMITED 2015-2016 */
+#ifndef __HEADER_ARM64_IHK_ATOMIC_H
+#define __HEADER_ARM64_IHK_ATOMIC_H
+
+#include <arch/cpu.h>
+
+/***********************************************************************
+ * ihk_atomic_t
+ */
+
+typedef struct {
+	int counter;
+} ihk_atomic_t;
+
+#define IHK_ATOMIC_INIT(i)	{ (i) }
+
+static inline int ihk_atomic_read(const ihk_atomic_t *v)
+{
+	return (*(volatile int *)&(v)->counter);
+}
+
+static inline void ihk_atomic_set(ihk_atomic_t *v, int i)
+{
+	v->counter = i;
+}
+
+/* @ref.impl arch/arm64/include/asm/atomic.h::atomic_add (atomic_##op) */
+static inline void ihk_atomic_add(int i, ihk_atomic_t *v)
+{
+	unsigned long tmp;
+	int result;
+
+	asm volatile("// atomic_add\n"
+"1:	ldxr	%w0, %2\n"
+"	add	%w0, %w0, %w3\n"
+"	stxr	%w1, %w0, %2\n"
+"	cbnz	%w1, 1b"
+	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
+	: "Ir" (i));
+}
+
+/* @ref.impl arch/arm64/include/asm/atomic.h::atomic_sub (atomic_##op) */
+static inline void ihk_atomic_sub(int i, ihk_atomic_t *v)
+{
+	unsigned long tmp;
+	int result;
+
+	asm volatile("// atomic_sub\n"
+"1:	ldxr	%w0, %2\n"
+"	sub	%w0, %w0, %w3\n"
+"	stxr	%w1, %w0, %2\n"
+"	cbnz	%w1, 1b"
+	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
+	: "Ir" (i));
+}
+
+/* @ref.impl arch/arm64/include/asm/atomic.h::atomic_inc */
+#define ihk_atomic_inc(v)		ihk_atomic_add(1, v)
+
+/* @ref.impl arch/arm64/include/asm/atomic.h::atomic_dec */
+#define ihk_atomic_dec(v)		ihk_atomic_sub(1, v)
+
+/* @ref.impl arch/arm64/include/asm/atomic.h::atomic_add_return (atomic_##op##_return) */
+static inline int ihk_atomic_add_return(int i, ihk_atomic_t *v)
+{
+	unsigned long tmp;
+	int result;
+
+	asm volatile("// atomic_add_return\n"
+"1:	ldxr	%w0, %2\n"
+"	add	%w0, %w0, %w3\n"
+"	stlxr	%w1, %w0, %2\n"
+"	cbnz	%w1, 1b"
+	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
+	: "Ir" (i)
+	: "memory");
+
+	smp_mb();
+	return result;
+}
+
+/* @ref.impl arch/arm64/include/asm/atomic.h::atomic_sub_return (atomic_##op##_return) */
+static inline int ihk_atomic_sub_return(int i, ihk_atomic_t *v)
+{
+	unsigned long tmp;
+	int result;
+
+	asm volatile("// atomic_sub_return\n"
+"1:	ldxr	%w0, %2\n"
+"	sub	%w0, %w0, %w3\n"
+"	stlxr	%w1, %w0, %2\n"
+"	cbnz	%w1, 1b"
+	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
+	: "Ir" (i)
+	: "memory");
+
+	smp_mb();
+	return result;
+}
+
+/* @ref.impl arch/arm64/include/asm/atomic.h::atomic_inc_and_test */
+#define ihk_atomic_inc_and_test(v)	(ihk_atomic_add_return(1, v) == 0)
+
+/* @ref.impl arch/arm64/include/asm/atomic.h::atomic_dec_and_test */
+#define ihk_atomic_dec_and_test(v)	(ihk_atomic_sub_return(1, v) == 0)
+
+/* @ref.impl arch/arm64/include/asm/atomic.h::atomic_inc_return */
+#define ihk_atomic_inc_return(v)  	(ihk_atomic_add_return(1, v))
+
+/* @ref.impl arch/arm64/include/asm/atomic.h::atomic_dec_return */
+#define ihk_atomic_dec_return(v)  	(ihk_atomic_sub_return(1, v))
+
+/***********************************************************************
+ * ihk_atomic64_t
+ */
+typedef struct {
+	long counter64;
+} ihk_atomic64_t;
+
+#define IHK_ATOMIC64_INIT(i)	{ .counter64 = (i) }
+
+static inline long ihk_atomic64_read(const ihk_atomic64_t *v)
+{
+	return *(volatile long *)&(v)->counter64;
+}
+
+static inline void ihk_atomic64_set(ihk_atomic64_t *v, long i)
+{
+	v->counter64 = i;
+}
+
+/* @ref.impl arch/arm64/include/asm/atomic.h::atomic64_add (atomic64_##op) */
+static inline void ihk_atomic64_add(long i, ihk_atomic64_t *v)
+{
+	long result;
+	unsigned long tmp;
+
+	asm volatile("// atomic64_add\n"
+"1:	ldxr	%0, %2\n"
+"	add	%0, %0, %3\n"
+"	stxr	%w1, %0, %2\n"
+"	cbnz	%w1, 1b"
+	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter64)
+	: "Ir" (i));
+}
+
+/* @ref.impl arch/arm64/include/asm/atomic.h::atomic64_inc */
+#define ihk_atomic64_inc(v)			ihk_atomic64_add(1LL, (v))
+
+#define ihk_atomic64_cmpxchg(p, o, n) cmpxchg(&((p)->counter64), o, n)
+
+/***********************************************************************
+ * others
+ */
+/* @ref.impl arch/arm64/include/asm/cmpxchg.h::__xchg */
+static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size)
+{
+	unsigned long ret = 0, tmp;
+
+	switch (size) {
+	case 1:
+		asm volatile("//	__xchg1\n"
+		"1:	ldxrb	%w0, %2\n"
+		"	stlxrb	%w1, %w3, %2\n"
+		"	cbnz	%w1, 1b\n"
+			: "=&r" (ret), "=&r" (tmp), "+Q" (*(unsigned char *)ptr)
+			: "r" (x)
+			: "memory");
+		break;
+	case 2:
+		asm volatile("//	__xchg2\n"
+		"1:	ldxrh	%w0, %2\n"
+		"	stlxrh	%w1, %w3, %2\n"
+		"	cbnz	%w1, 1b\n"
+			: "=&r" (ret), "=&r" (tmp), "+Q" (*(unsigned short *)ptr)
+			: "r" (x)
+			: "memory");
+		break;
+	case 4:
+		asm volatile("//	__xchg4\n"
+		"1:	ldxr	%w0, %2\n"
+		"	stlxr	%w1, %w3, %2\n"
+		"	cbnz	%w1, 1b\n"
+			: "=&r" (ret), "=&r" (tmp), "+Q" (*(unsigned int *)ptr)
+			: "r" (x)
+			: "memory");
+		break;
+	case 8:
+		asm volatile("//	__xchg8\n"
+		"1:	ldxr	%0, %2\n"
+		"	stlxr	%w1, %3, %2\n"
+		"	cbnz	%w1, 1b\n"
+			: "=&r" (ret), "=&r" (tmp), "+Q" (*(unsigned long *)ptr)
+			: "r" (x)
+			: "memory");
+		break;
+/*
+	default:
+		BUILD_BUG();
+*/
+	}
+
+	smp_mb();
+	return ret;
+}
+
+/* @ref.impl arch/arm64/include/asm/cmpxchg.h::xchg */
+#define xchg(ptr,x) \
+({ \
+	__typeof__(*(ptr)) __ret; \
+	__ret = (__typeof__(*(ptr))) \
+		__xchg((unsigned long)(x), (ptr), sizeof(*(ptr))); \
+	__ret; \
+})
+
+#define xchg4(ptr, x) xchg(ptr,x)
+#define xchg8(ptr, x) xchg(ptr,x)
+
+/* @ref.impl arch/arm64/include/asm/cmpxchg.h::__cmpxchg */
+static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
+				unsigned long new, int size)
+{
+	unsigned long oldval = 0, res;
+
+	switch (size) {
+	case 1:
+		do {
+			asm volatile("// __cmpxchg1\n"
+			"	ldxrb	%w1, %2\n"
+			"	mov	%w0, #0\n"
+			"	cmp	%w1, %w3\n"
+			"	b.ne	1f\n"
+			"	stxrb	%w0, %w4, %2\n"
+			"1:\n"
+				: "=&r" (res), "=&r" (oldval), "+Q" (*(unsigned char *)ptr)
+				: "Ir" (old), "r" (new)                                                               				: "cc");
+		} while (res);
+		break;
+
+	case 2:
+		do {
+			asm volatile("// __cmpxchg2\n"
+			"	ldxrh	%w1, %2\n"
+			"	mov	%w0, #0\n"
+			"	cmp	%w1, %w3\n"
+			"	b.ne	1f\n"
+			"	stxrh	%w0, %w4, %2\n"
+			"1:\n"
+				: "=&r" (res), "=&r" (oldval), "+Q" (*(unsigned short *)ptr)
+				: "Ir" (old), "r" (new)
+				: "cc");
+		} while (res);
+		break;
+
+	case 4:
+		do {
+			asm volatile("// __cmpxchg4\n"
+			"	ldxr	%w1, %2\n"
+			"	mov	%w0, #0\n"
+			"	cmp	%w1, %w3\n"
+			"	b.ne	1f\n"
+			"	stxr	%w0, %w4, %2\n"
+			"1:\n"
+				: "=&r" (res), "=&r" (oldval), "+Q" (*(unsigned int *)ptr)
+				: "Ir" (old), "r" (new)
+				: "cc");
+		} while (res);
+		break;
+
+	case 8:
+		do {
+			asm volatile("// __cmpxchg8\n"
+			"	ldxr	%1, %2\n"
+			"	mov	%w0, #0\n"
+			"	cmp	%1, %3\n"
+			"	b.ne	1f\n"
+			"	stxr	%w0, %4, %2\n"
+			"1:\n"
+				: "=&r" (res), "=&r" (oldval), "+Q" (*(unsigned long *)ptr)
+				: "Ir" (old), "r" (new)
+				: "cc");
+		} while (res);
+		break;
+/*
+	default:
+		BUILD_BUG();
+*/
+	}
+
+	return oldval;
+}
+
+/* @ref.impl arch/arm64/include/asm/cmpxchg.h::__cmpxchg_mb */
+static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
+					 unsigned long new, int size)
+{
+	unsigned long ret;
+
+	smp_mb();
+	ret = __cmpxchg(ptr, old, new, size);
+	smp_mb();
+
+	return ret;
+}
+
+/* @ref.impl arch/arm64/include/asm/cmpxchg.h::cmpxchg */
+#define cmpxchg(ptr, o, n) \
+({ \
+	__typeof__(*(ptr)) __ret; \
+	__ret = (__typeof__(*(ptr))) \
+		__cmpxchg_mb((ptr), (unsigned long)(o), (unsigned long)(n), \
+			sizeof(*(ptr))); \
+	__ret; \
+})
+
+#define atomic_cmpxchg4(ptr, o, n)	cmpxchg(ptr,o,n)
+#define atomic_cmpxchg8(ptr, o, n)	cmpxchg(ptr,o,n)
+
+static inline void ihk_atomic_add_long(long i, long *v)
+{
+	long result;
+	unsigned long tmp;
+
+	asm volatile("// atomic64_add\n"
+"1:	ldxr	%0, %2\n"
+"	add	%0, %0, %3\n"
+"	stxr	%w1, %0, %2\n"
+"	cbnz	%w1, 1b"
+	: "=&r" (result), "=&r" (tmp), "+Q" (*v)
+	: "Ir" (i));
+}
+
+static inline void ihk_atomic_add_ulong(long i, unsigned long *v)
+{
+	long result;
+	unsigned long tmp;
+
+	asm volatile("// atomic64_add\n"
+"1:	ldxr	%0, %2\n"
+"	add	%0, %0, %3\n"
+"	stxr	%w1, %0, %2\n"
+"	cbnz	%w1, 1b"
+	: "=&r" (result), "=&r" (tmp), "+Q" (*v)
+	: "Ir" (i));
+}
+
+static inline unsigned long ihk_atomic_add_long_return(long i, long *v)
+{
+	unsigned long result;
+	unsigned long tmp;
+
+	asm volatile("// atomic64_add_return\n"
+"1:	ldxr	%0, %2\n"
+"	add	%0, %0, %3\n"
+"	stlxr	%w1, %0, %2\n"
+"	cbnz	%w1, 1b"
+	: "=&r" (result), "=&r" (tmp), "+Q" (*v)
+	: "Ir" (i)
+	: "memory");
+
+	smp_mb();
+	return result;
+}
+
+#endif /* !__HEADER_ARM64_COMMON_IHK_ATOMIC_H */

arch/arm64/kernel/include/ihk/context.h

@@ -0,0 +1,83 @@
+/* context.h COPYRIGHT FUJITSU LIMITED 2015-2018 */
+#ifndef __HEADER_ARM64_IHK_CONTEXT_H
+#define __HEADER_ARM64_IHK_CONTEXT_H
+
+#include <registers.h>
+
+struct thread_info;
+typedef struct {
+	struct thread_info *thread;
+} ihk_mc_kernel_context_t;
+
+struct user_pt_regs {
+	unsigned long regs[31];
+	unsigned long sp;
+	unsigned long pc;
+	unsigned long pstate;
+};
+
+struct pt_regs {
+	union {
+		struct user_pt_regs user_regs;
+		struct {
+			unsigned long regs[31];
+			unsigned long sp;
+			unsigned long pc;
+			unsigned long pstate;
+		};
+	};
+	unsigned long orig_x0;
+	unsigned long orig_pc;
+	unsigned long syscallno;
+	unsigned long __padding;
+};
+
+typedef struct pt_regs ihk_mc_user_context_t;
+
+/* @ref.impl arch/arm64/include/asm/ptrace.h */
+#define GET_IP(regs)		((unsigned long)(regs)->pc)
+#define SET_IP(regs, value)	((regs)->pc = ((uint64_t) (value)))
+
+/* @ref.impl arch/arm64/include/asm/ptrace.h */
+/* AArch32 CPSR bits */
+#define COMPAT_PSR_MODE_MASK	0x0000001f
+
+/* @ref.impl include/asm-generic/ptrace.h */
+static inline unsigned long instruction_pointer(struct pt_regs *regs)
+{
+	return GET_IP(regs);
+}
+/* @ref.impl include/asm-generic/ptrace.h */
+static inline void instruction_pointer_set(struct pt_regs *regs,
+					   unsigned long val)
+{
+	SET_IP(regs, val);
+}
+
+/* @ref.impl arch/arm64/include/asm/ptrace.h */
+/*
+ * Write a register given an architectural register index r.
+ * This handles the common case where 31 means XZR, not SP.
+ */
+static inline void pt_regs_write_reg(struct pt_regs *regs, int r,
+				     unsigned long val)
+{
+	if (r != 31)
+		regs->regs[r] = val;
+}
+
+/* temp */
+#define ihk_mc_syscall_arg0(uc)		((uc)->regs[0])
+#define ihk_mc_syscall_arg1(uc)		((uc)->regs[1])
+#define ihk_mc_syscall_arg2(uc)		((uc)->regs[2])
+#define ihk_mc_syscall_arg3(uc)		((uc)->regs[3])
+#define ihk_mc_syscall_arg4(uc)		((uc)->regs[4])
+#define ihk_mc_syscall_arg5(uc)		((uc)->regs[5])
+
+#define ihk_mc_syscall_ret(uc)		((uc)->regs[0])
+#define ihk_mc_syscall_number(uc)	((uc)->regs[8])
+
+#define ihk_mc_syscall_pc(uc)		((uc)->pc)
+#define ihk_mc_syscall_sp(uc)		((uc)->sp)
+
+#endif /* !__HEADER_ARM64_IHK_CONTEXT_H */

arch/arm64/kernel/include/ihk/ikc.h

@@ -0,0 +1,14 @@
+/* ikc.h COPYRIGHT FUJITSU LIMITED 2015 */
+#ifndef __HEADER_ARM64_IHK_IKC_H
+#define __HEADER_ARM64_IHK_IKC_H
+
+#include <ikc/ihk.h>
+
+#define IKC_PORT_IKC2MCKERNEL 501
+#define IKC_PORT_IKC2LINUX    503
+
+/* manycore side */
+int ihk_mc_ikc_init_first(struct ihk_ikc_channel_desc *,
+                          ihk_ikc_ph_t handler);
+
+#endif /* !__HEADER_ARM64_IHK_IKC_H */

arch/arm64/kernel/include/ihk/types.h

@@ -0,0 +1,33 @@
+/* types.h COPYRIGHT FUJITSU LIMITED 2015-2017 */
+#ifndef __HEADER_ARM64_IHK_TYPES_H
+#define __HEADER_ARM64_IHK_TYPES_H
+
+#ifndef __ASSEMBLY__
+
+typedef unsigned char      uint8_t;
+typedef unsigned short     uint16_t;
+typedef unsigned int       uint32_t;
+typedef unsigned long long uint64_t;
+typedef signed char        int8_t;
+typedef signed short       int16_t;
+typedef signed int         int32_t;
+typedef signed long long   int64_t;
+
+typedef int64_t            ptrdiff_t;
+typedef int64_t            intptr_t;
+typedef uint64_t           uintptr_t;
+typedef uint64_t           size_t;
+typedef int64_t            ssize_t;
+typedef int64_t            off_t;
+
+typedef int32_t            key_t;
+typedef uint32_t           uid_t;
+typedef uint32_t           gid_t;
+typedef int64_t            time_t;
+typedef int32_t            pid_t;
+
+#endif /* __ASSEMBLY__ */
+
+#define NULL ((void *)0)
+
+#endif /* !__HEADER_ARM64_IHK_TYPES_H */

arch/arm64/kernel/include/imp-sysreg.h

@@ -0,0 +1,103 @@
+/* imp-sysreg.h COPYRIGHT FUJITSU LIMITED 2016-2018 */
+#ifndef __ASM_IMP_SYSREG_H
+#define __ASM_IMP_SYSREG_H
+
+#ifndef __ASSEMBLY__
+
+/* register sys_reg list */
+#define IMP_FJ_TAG_ADDRESS_CTRL_EL1		sys_reg(3, 0, 11, 2, 0)
+#define IMP_SCCR_CTRL_EL1			sys_reg(3, 0, 11, 8, 0)
+#define IMP_SCCR_ASSIGN_EL1			sys_reg(3, 0, 11, 8, 1)
+#define IMP_SCCR_SET0_L2_EL1			sys_reg(3, 0, 15, 8, 2)
+#define IMP_SCCR_SET1_L2_EL1			sys_reg(3, 0, 15, 8, 3)
+#define IMP_SCCR_L1_EL0				sys_reg(3, 3, 11, 8, 2)
+#define IMP_PF_CTRL_EL1				sys_reg(3, 0, 11, 4, 0)
+#define IMP_PF_STREAM_DETECT_CTRL_EL0		sys_reg(3, 3, 11, 4, 0)
+#define IMP_PF_INJECTION_CTRL0_EL0		sys_reg(3, 3, 11, 6, 0)
+#define IMP_PF_INJECTION_CTRL1_EL0		sys_reg(3, 3, 11, 6, 1)
+#define IMP_PF_INJECTION_CTRL2_EL0		sys_reg(3, 3, 11, 6, 2)
+#define IMP_PF_INJECTION_CTRL3_EL0		sys_reg(3, 3, 11, 6, 3)
+#define IMP_PF_INJECTION_CTRL4_EL0		sys_reg(3, 3, 11, 6, 4)
+#define IMP_PF_INJECTION_CTRL5_EL0		sys_reg(3, 3, 11, 6, 5)
+#define IMP_PF_INJECTION_CTRL6_EL0		sys_reg(3, 3, 11, 6, 6)
+#define IMP_PF_INJECTION_CTRL7_EL0		sys_reg(3, 3, 11, 6, 7)
+#define IMP_PF_INJECTION_DISTANCE0_EL0		sys_reg(3, 3, 11, 7, 0)
+#define IMP_PF_INJECTION_DISTANCE1_EL0		sys_reg(3, 3, 11, 7, 1)
+#define IMP_PF_INJECTION_DISTANCE2_EL0		sys_reg(3, 3, 11, 7, 2)
+#define IMP_PF_INJECTION_DISTANCE3_EL0		sys_reg(3, 3, 11, 7, 3)
+#define IMP_PF_INJECTION_DISTANCE4_EL0		sys_reg(3, 3, 11, 7, 4)
+#define IMP_PF_INJECTION_DISTANCE5_EL0		sys_reg(3, 3, 11, 7, 5)
+#define IMP_PF_INJECTION_DISTANCE6_EL0		sys_reg(3, 3, 11, 7, 6)
+#define IMP_PF_INJECTION_DISTANCE7_EL0		sys_reg(3, 3, 11, 7, 7)
+#define IMP_PF_PMUSERENR_EL0			sys_reg(3, 3, 9, 14, 0)
+#define IMP_BARRIER_CTRL_EL1			sys_reg(3, 0, 11, 12, 0)
+#define IMP_BARRIER_BST_BIT_EL1			sys_reg(3, 0, 11, 12, 4)
+#define IMP_BARRIER_INIT_SYNC_BB0_EL1		sys_reg(3, 0, 15, 13, 0)
+#define IMP_BARRIER_INIT_SYNC_BB1_EL1		sys_reg(3, 0, 15, 13, 1)
+#define IMP_BARRIER_INIT_SYNC_BB2_EL1		sys_reg(3, 0, 15, 13, 2)
+#define IMP_BARRIER_INIT_SYNC_BB3_EL1		sys_reg(3, 0, 15, 13, 3)
+#define IMP_BARRIER_INIT_SYNC_BB4_EL1		sys_reg(3, 0, 15, 13, 4)
+#define IMP_BARRIER_INIT_SYNC_BB5_EL1		sys_reg(3, 0, 15, 13, 5)
+#define IMP_BARRIER_ASSIGN_SYNC_W0_EL1		sys_reg(3, 0, 15, 15, 0)
+#define IMP_BARRIER_ASSIGN_SYNC_W1_EL1		sys_reg(3, 0, 15, 15, 1)
+#define IMP_BARRIER_ASSIGN_SYNC_W2_EL1		sys_reg(3, 0, 15, 15, 2)
+#define IMP_BARRIER_ASSIGN_SYNC_W3_EL1		sys_reg(3, 0, 15, 15, 3)
+#define IMP_SOC_STANDBY_CTRL_EL1		sys_reg(3, 0, 11, 0, 0)
+#define IMP_FJ_CORE_UARCH_CTRL_EL2		sys_reg(3, 4, 11, 0, 4)
+#define IMP_FJ_CORE_UARCH_RESTRECTION_EL1	sys_reg(3, 0, 11, 0, 5)
+
+/* macros */
+#define PWR_REG_MASK(reg, feild)  (((UL(1) << ((reg##_##feild##_MSB) - (reg##_##feild##_LSB) + 1)) - 1) << (reg##_##feild##_LSB))
+
+/* IMP_FJ_TAG_ADDRESS_CTRL_EL1 */
+#define IMP_FJ_TAG_ADDRESS_CTRL_EL1_TBO0_SHIFT	(0)
+#define IMP_FJ_TAG_ADDRESS_CTRL_EL1_SEC0_SHIFT	(8)
+#define IMP_FJ_TAG_ADDRESS_CTRL_EL1_PFE0_SHIFT	(9)
+#define IMP_FJ_TAG_ADDRESS_CTRL_EL1_TBO0_MASK	(1UL << IMP_FJ_TAG_ADDRESS_CTRL_EL1_TBO0_SHIFT)
+#define IMP_FJ_TAG_ADDRESS_CTRL_EL1_SEC0_MASK	(1UL << IMP_FJ_TAG_ADDRESS_CTRL_EL1_SEC0_SHIFT)
+#define IMP_FJ_TAG_ADDRESS_CTRL_EL1_PFE0_MASK	(1UL << IMP_FJ_TAG_ADDRESS_CTRL_EL1_PFE0_SHIFT)
+
+/* IMP_SCCR_CTRL_EL1 */
+#define IMP_SCCR_CTRL_EL1_EL1AE_SHIFT		(63)
+#define IMP_SCCR_CTRL_EL1_EL1AE_MASK		(1UL << IMP_SCCR_CTRL_EL1_EL1AE_SHIFT)
+
+/* IMP_SCCR_SET0_L2_EL1 */
+#define IMP_SCCR_SET0_L2_EL1_L2_SEC0_SHIFT	(0)
+
+/* IMP_PF_CTRL_EL1 */
+#define IMP_PF_CTRL_EL1_EL1AE_ENABLE		(1UL << 63)
+#define IMP_PF_CTRL_EL1_EL0AE_ENABLE		(1UL << 62)
+
+/* IMP_BARRIER_CTRL_EL1 */
+#define IMP_BARRIER_CTRL_EL1_EL1AE_ENABLE	(1UL << 63)
+#define IMP_BARRIER_CTRL_EL1_EL0AE_ENABLE	(1UL << 62)
+
+/* IMP_SOC_STANDBY_CTRL_EL1 */
+#define IMP_SOC_STANDBY_CTRL_EL1_ECO_MODE_MSB		2
+#define IMP_SOC_STANDBY_CTRL_EL1_ECO_MODE_LSB		2
+#define IMP_SOC_STANDBY_CTRL_EL1_MODE_CHANGE_MSB	1
+#define IMP_SOC_STANDBY_CTRL_EL1_MODE_CHANGE_LSB	1
+#define IMP_SOC_STANDBY_CTRL_EL1_RETENTION_MSB		0
+#define IMP_SOC_STANDBY_CTRL_EL1_RETENTION_LSB		0
+#define IMP_SOC_STANDBY_CTRL_EL1_ECO_MODE		PWR_REG_MASK(IMP_SOC_STANDBY_CTRL_EL1, ECO_MODE)
+#define IMP_SOC_STANDBY_CTRL_EL1_MODE_CHANGE		PWR_REG_MASK(IMP_SOC_STANDBY_CTRL_EL1, MODE_CHANGE)
+#define IMP_SOC_STANDBY_CTRL_EL1_RETENTION		PWR_REG_MASK(IMP_SOC_STANDBY_CTRL_EL1, RETENTION)
+
+/* IMP_FJ_CORE_UARCH_RESTRECTION_EL1 */
+#define IMP_FJ_CORE_UARCH_RESTRECTION_EL1_FL_RESTRICT_TRANS_MSB	33
+#define IMP_FJ_CORE_UARCH_RESTRECTION_EL1_FL_RESTRICT_TRANS_LSB	33
+#define IMP_FJ_CORE_UARCH_RESTRECTION_EL1_ISSUE_RESTRICTION_MSB	9
+#define IMP_FJ_CORE_UARCH_RESTRECTION_EL1_ISSUE_RESTRICTION_LSB	8
+#define IMP_FJ_CORE_UARCH_RESTRECTION_EL1_EX_RESTRICTION_MSB	0
+#define IMP_FJ_CORE_UARCH_RESTRECTION_EL1_EX_RESTRICTION_LSB	0
+#define IMP_FJ_CORE_UARCH_RESTRECTION_EL1_FL_RESTRICT_TRANS	PWR_REG_MASK(IMP_FJ_CORE_UARCH_RESTRECTION_EL1, FL_RESTRICT_TRANS)
+#define IMP_FJ_CORE_UARCH_RESTRECTION_EL1_ISSUE_RESTRICTION	PWR_REG_MASK(IMP_FJ_CORE_UARCH_RESTRECTION_EL1, ISSUE_RESTRICTION)
+#define IMP_FJ_CORE_UARCH_RESTRECTION_EL1_EX_RESTRICTION	PWR_REG_MASK(IMP_FJ_CORE_UARCH_RESTRECTION_EL1, EX_RESTRICTION)
+
+void scdrv_registers_init(void);
+void hpc_registers_init(void);
+void vhbm_barrier_registers_init(void);
+
+#endif /* __ASSEMBLY__ */
+
+#endif	/* __ASM_IMP_SYSREG_H */

arch/arm64/kernel/include/io.h

@@ -0,0 +1,118 @@
+/* io.h COPYRIGHT FUJITSU LIMITED 2015 */
+/*
+ * Based on arch/arm/include/asm/io.h
+ *
+ * Copyright (C) 1996-2000 Russell King
+ * Copyright (C) 2012 ARM Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef __ASM_IO_H
+#define __ASM_IO_H
+
+#include <ihk/types.h>
+
+#ifdef __KERNEL__
+
+/*
+ * Generic IO read/write.  These perform native-endian accesses.
+ */
+static inline void __raw_writeb(uint8_t val, volatile void *addr)
+{
+	asm volatile("strb %w0, [%1]" : : "r" (val), "r" (addr));
+}
+
+static inline void __raw_writew(uint16_t val, volatile void *addr)
+{
+	asm volatile("strh %w0, [%1]" : : "r" (val), "r" (addr));
+}
+
+static inline void __raw_writel(uint32_t val, volatile void *addr)
+{
+	asm volatile("str %w0, [%1]" : : "r" (val), "r" (addr));
+}
+
+static inline void __raw_writeq(uint64_t val, volatile void *addr)
+{
+	asm volatile("str %0, [%1]" : : "r" (val), "r" (addr));
+}
+
+static inline uint8_t __raw_readb(const volatile void *addr)
+{
+	uint8_t val;
+	asm volatile("ldarb %w0, [%1]"
+		     : "=r" (val) : "r" (addr));
+	return val;
+}
+
+static inline uint16_t __raw_readw(const volatile void *addr)
+{
+	uint16_t val;
+
+	asm volatile("ldarh %w0, [%1]"
+		     : "=r" (val) : "r" (addr));
+	return val;
+}
+
+static inline uint32_t __raw_readl(const volatile void *addr)
+{
+	uint32_t val;
+	asm volatile("ldar %w0, [%1]"
+		     : "=r" (val) : "r" (addr));
+	return val;
+}
+
+static inline uint64_t __raw_readq(const volatile void *addr)
+{
+	uint64_t val;
+	asm volatile("ldar %0, [%1]"
+		     : "=r" (val) : "r" (addr));
+	return val;
+}
+
+/* IO barriers */
+#define __iormb()		rmb()
+#define __iowmb()		wmb()
+
+/*
+ * Relaxed I/O memory access primitives. These follow the Device memory
+ * ordering rules but do not guarantee any ordering relative to Normal memory
+ * accesses.
+ */
+#define readb_relaxed(c)	({ uint8_t  __v = (uint8_t)__raw_readb(c); __v; })
+#define readw_relaxed(c)	({ uint16_t __v = (uint16_t)__raw_readw(c); __v; })
+#define readl_relaxed(c)	({ uint32_t __v = (uint32_t)__raw_readl(c); __v; })
+#define readq_relaxed(c)	({ uint64_t __v = (uint64_t)__raw_readq(c); __v; })
+
+#define writeb_relaxed(v,c)	((void)__raw_writeb((uint8_t)(v),(c)))
+#define writew_relaxed(v,c)	((void)__raw_writew((uint16_t)(v),(c)))
+#define writel_relaxed(v,c)	((void)__raw_writel((uint32_t)(v),(c)))
+#define writeq_relaxed(v,c)	((void)__raw_writeq((uint64_t)(v),(c)))
+
+/*
+ * I/O memory access primitives. Reads are ordered relative to any
+ * following Normal memory access. Writes are ordered relative to any prior
+ * Normal memory access.
+ */
+#define readb(c)		({ uint8_t  __v = readb_relaxed(c); __iormb(); __v; })
+#define readw(c)		({ uint16_t __v = readw_relaxed(c); __iormb(); __v; })
+#define readl(c)		({ uint32_t __v = readl_relaxed(c); __iormb(); __v; })
+#define readq(c)		({ uint64_t __v = readq_relaxed(c); __iormb(); __v; })
+
+#define writeb(v,c)		({ __iowmb(); writeb_relaxed((v),(c)); })
+#define writew(v,c)		({ __iowmb(); writew_relaxed((v),(c)); })
+#define writel(v,c)		({ __iowmb(); writel_relaxed((v),(c)); })
+#define writeq(v,c)		({ __iowmb(); writeq_relaxed((v),(c)); })
+
+#endif /* __KERNEL__ */
+#endif /* __ASM_IO_H */

arch/arm64/kernel/include/irq.h

@@ -0,0 +1,47 @@
+/* irq.h COPYRIGHT FUJITSU LIMITED 2015-2019 */
+
+#ifndef __HEADER_ARM64_IRQ_H
+#define __HEADER_ARM64_IRQ_H
+
+#include <ihk/debug.h>
+#include <ihk/context.h>
+#include <sysreg.h>
+#include <cputype.h>
+
+/* use SGI interrupt number */
+#define INTRID_CPU_NOTIFY	0
+#define INTRID_IKC		1
+#define INTRID_QUERY_FREE_MEM	2
+#define INTRID_CPU_STOP		3
+#define INTRID_TLB_FLUSH	4
+#define INTRID_STACK_TRACE	5
+#define INTRID_MULTI_INTR	6
+#define INTRID_MULTI_NMI	7
+#define LOCAL_SMP_FUNC_CALL_VECTOR   1 /* same as IKC */
+
+/* use PPI interrupt number */
+#define INTRID_PERF_OVF		23
+#define INTRID_HYP_PHYS_TIMER	26 /* cnthp */
+#define INTRID_VIRT_TIMER	27 /* cntv */
+#define INTRID_HYP_VIRT_TIMER	28 /* cnthv */
+#define INTRID_PHYS_TIMER	30 /* cntp */
+
+/* Functions for GICv2 */
+extern void gic_dist_init_gicv2(unsigned long dist_base_pa, unsigned long size);
+extern void gic_cpu_init_gicv2(unsigned long cpu_base_pa, unsigned long size);
+extern void gic_enable_gicv2(void);
+extern void arm64_issue_ipi_gicv2(unsigned int cpuid, unsigned int vector);
+extern void arm64_issue_host_ipi_gicv2(uint32_t cpuid, uint32_t vector);
+extern void handle_interrupt_gicv2(struct pt_regs *regs);
+
+/* Functions for GICv3 */
+extern void gic_dist_init_gicv3(unsigned long dist_base_pa, unsigned long size);
+extern void gic_cpu_init_gicv3(unsigned long cpu_base_pa, unsigned long size);
+extern void gic_enable_gicv3(void);
+extern void arm64_issue_ipi_gicv3(unsigned int cpuid, unsigned int vector);
+extern void arm64_issue_host_ipi_gicv3(uint32_t cpuid, uint32_t vector);
+extern void handle_interrupt_gicv3(struct pt_regs *regs);
+
+void handle_IPI(unsigned int vector, struct pt_regs *regs);
+
+#endif /* __HEADER_ARM64_IRQ_H */

arch/arm64/kernel/include/irqflags.h

@@ -0,0 +1,31 @@
+/* irqflags.h COPYRIGHT FUJITSU LIMITED 2015-2017 */
+#ifndef __HEADER_ARM64_COMMON_IRQFLAGS_H
+#define __HEADER_ARM64_COMMON_IRQFLAGS_H
+
+#include <ptrace.h>
+
+/*
+ * save and restore debug state
+ */
+static inline unsigned long local_dbg_save(void)
+{
+	unsigned long flags;
+	asm volatile(
+		"mrs    %0, daif		// local_dbg_save\n"
+		"msr    daifset, #8"
+		: "=r" (flags)
+		:
+		: "memory");
+	return flags;
+}
+
+static inline void local_dbg_restore(unsigned long flags)
+{
+	asm volatile(
+		"msr    daif, %0		// local_dbg_restore"
+		:
+		: "r" (flags)
+		: "memory");
+}
+
+#endif /* !__HEADER_ARM64_COMMON_IRQFLAGS_H */

arch/arm64/kernel/include/linkage.h

@@ -0,0 +1,25 @@
+/* linkage.h COPYRIGHT FUJITSU LIMITED 2015-2016 */
+#ifndef __HEADER_ARM64_COMMON_LINKAGE_H
+#define __HEADER_ARM64_COMMON_LINKAGE_H
+
+#include <arch-memory.h>
+#include <compiler.h>
+
+#define ASM_NL		;
+
+#define __ALIGN		.align 4
+#define __ALIGN_STR	".align 4"
+
+#define ENTRY(name)		\
+	.globl name ASM_NL	\
+	__ALIGN ASM_NL		\
+	name:
+
+#define END(name)		\
+	.size name, .-name
+
+#define ENDPROC(name)			\
+	.type name, @function ASM_NL	\
+	END(name)
+
+#endif /* !__HEADER_ARM64_COMMON_LINKAGE_H */

arch/arm64/kernel/include/mmu_context.h

@@ -0,0 +1,22 @@
+/* mmu_context.h COPYRIGHT FUJITSU LIMITED 2015 */
+#ifndef __HEADER_ARM64_COMMON_MMU_CONTEXT_H
+#define __HEADER_ARM64_COMMON_MMU_CONTEXT_H
+
+#include <pgtable.h>
+#include <memory.h>
+
+/*
+ * Set TTBR0 to empty_zero_page. No translations will be possible via TTBR0.
+ */
+static inline void cpu_set_reserved_ttbr0(void)
+{
+	unsigned long ttbr = virt_to_phys(empty_zero_page);
+
+	asm(
+	"	msr	ttbr0_el1, %0			// set TTBR0\n"
+	"	isb"
+	:
+	: "r" (ttbr));
+}
+
+#endif /* !__HEADER_ARM64_COMMON_MMU_CONTEXT_H */

arch/arm64/kernel/include/pgtable-hwdef.h

@@ -0,0 +1,198 @@
+/* pgtable-hwdef.h COPYRIGHT FUJITSU LIMITED 2015 */
+#ifndef __HEADER_ARM64_COMMON_PGTABLE_HWDEF_H
+#define __HEADER_ARM64_COMMON_PGTABLE_HWDEF_H
+
+#ifndef __HEADER_ARM64_COMMON_ARCH_MEMORY_H
+# error arch-memory.h
+#endif
+
+#define PTRS_PER_PTE		(1 << (PAGE_SHIFT - 3))
+
+/*
+ * PMD_SHIFT determines the size a level 2 page table entry can map.
+ */
+#if CONFIG_ARM64_PGTABLE_LEVELS > 2
+# define PMD_SHIFT		((PAGE_SHIFT - 3) * 2 + 3)
+# define PMD_SIZE		(1UL << PMD_SHIFT)
+# define PMD_MASK		(~(PMD_SIZE-1))
+# define PTRS_PER_PMD		PTRS_PER_PTE
+#endif
+
+ /*
+  * PUD_SHIFT determines the size a level 1 page table entry can map.
+  */
+#if CONFIG_ARM64_PGTABLE_LEVELS > 3
+# define PUD_SHIFT		((PAGE_SHIFT - 3) * 3 + 3)
+# define PUD_SIZE		(1UL << PUD_SHIFT)
+# define PUD_MASK		(~(PUD_SIZE-1))
+# define PTRS_PER_PUD		PTRS_PER_PTE
+#endif
+ 
+/*
+ * PGDIR_SHIFT determines the size a top-level page table entry can map
+ * (depending on the configuration, this level can be 0, 1 or 2).
+ */
+#define PGDIR_SHIFT		((PAGE_SHIFT - 3) * CONFIG_ARM64_PGTABLE_LEVELS + 3)
+#define PGDIR_SIZE		(_AC(1, UL) << PGDIR_SHIFT)
+#define PGDIR_MASK		(~(PGDIR_SIZE-1))
+#define PTRS_PER_PGD		(1 << (VA_BITS - PGDIR_SHIFT))
+
+/*
+ * Section address mask and size definitions.
+ */
+#define SECTION_SHIFT		PMD_SHIFT
+#define SECTION_SIZE		(UL(1) << SECTION_SHIFT)
+#define SECTION_MASK		(~(SECTION_SIZE-1))
+
+/*
+ * Level 2 descriptor (PMD).
+ */
+#define PMD_TYPE_MASK		(UL(3) << 0)
+#define PMD_TYPE_FAULT		(UL(0) << 0)
+#define PMD_TYPE_TABLE		(UL(3) << 0)
+#define PMD_TYPE_SECT		(UL(1) << 0)
+#define PMD_TABLE_BIT		(UL(1) << 1)
+
+/*
+ * Table (D_Block)
+ */
+#define PMD_TBL_PXNT		(UL(1) << 59)
+#define PMD_TBL_UXNT		(UL(1) << 60)
+#define PMD_TBL_APT_USER	(UL(1) << 61) /* 0:Access at EL0 permitted, 1:Access at EL0 not permitted */
+#define PMD_TBL_APT_RDONLY	(UL(2) << 61) /* 0:read write(EL0-3) 0:read only(EL0-3) */
+#define PMD_TBL_NST		(UL(1) << 63) /* 0:secure, 1:non-secure */
+
+/*
+ * Section (D_Page)
+ */
+#define PMD_SECT_VALID		(UL(1) << 0)
+#define PMD_SECT_PROT_NONE	(UL(1) << 58)
+#define PMD_SECT_USER		(UL(1) << 6)		/* AP[1] */
+#define PMD_SECT_RDONLY		(UL(1) << 7)		/* AP[2] */
+#define PMD_SECT_S		(UL(3) << 8)
+#define PMD_SECT_AF		(UL(1) << 10)
+#define PMD_SECT_NG		(UL(1) << 11)
+#define PMD_SECT_CONT		(UL(1) << 52)
+#define PMD_SECT_PXN		(UL(1) << 53)
+#define PMD_SECT_UXN		(UL(1) << 54)
+
+/*
+ * AttrIndx[2:0] encoding (mapping attributes defined in the MAIR* registers).
+ */
+#define PMD_ATTRINDX(t)		(UL(t) << 2)
+#define PMD_ATTRINDX_MASK	(UL(7) << 2)
+
+/*
+ * Level 3 descriptor (PTE).
+ */
+#define PTE_TYPE_MASK		(UL(3) << 0)
+#define PTE_TYPE_FAULT		(UL(0) << 0)
+#define PTE_TYPE_PAGE		(UL(3) << 0)
+#define PTE_TABLE_BIT		(UL(1) << 1)
+#define PTE_USER		(UL(1) << 6)	/* AP[1] */
+#define PTE_RDONLY		(UL(1) << 7)	/* AP[2] */
+#define PTE_SHARED		(UL(3) << 8)	/* SH[1:0], inner shareable */
+#define PTE_AF			(UL(1) << 10)	/* Access Flag */
+#define PTE_NG			(UL(1) << 11)	/* nG */
+#define PTE_CONT		(UL(1) << 52)	/* Contiguous range */
+#define PTE_PXN			(UL(1) << 53)	/* Privileged XN */
+#define PTE_UXN			(UL(1) << 54)	/* User XN */
+/* Software defined PTE bits definition.*/
+#define PTE_VALID		(UL(1) << 0)
+#define PTE_FILE		(UL(1) << 2)	/* only when !pte_present() */
+#define PTE_DIRTY		(UL(1) << 55)
+#define PTE_SPECIAL		(UL(1) << 56)
+#define PTE_WRITE		(UL(1) << 57)
+#define PTE_PROT_NONE		(UL(1) << 58) /* only when !PTE_VALID */
+
+/*
+ * AttrIndx[2:0] encoding (mapping attributes defined in the MAIR* registers).
+ */
+#define PTE_ATTRINDX(t)		(UL(t) << 2)
+#define PTE_ATTRINDX_MASK	(UL(7) << 2)
+
+/*
+ * Highest possible physical address supported.
+ */
+#define PHYS_MASK_SHIFT		(48)
+#define PHYS_MASK		(((UL(1) << PHYS_MASK_SHIFT) - 1) & PAGE_MASK)
+
+/*
+ * TCR flags.
+ */
+#define TCR_TxSZ(x)		(((UL(64) - (x)) << 16) | ((UL(64) - (x)) << 0))
+#define TCR_IRGN_NC		((UL(0) << 8) | (UL(0) << 24))
+#define TCR_IRGN_WBWA		((UL(1) << 8) | (UL(1) << 24))
+#define TCR_IRGN_WT		((UL(2) << 8) | (UL(2) << 24))
+#define TCR_IRGN_WBnWA		((UL(3) << 8) | (UL(3) << 24))
+#define TCR_IRGN_MASK		((UL(3) << 8) | (UL(3) << 24))
+#define TCR_ORGN_NC		((UL(0) << 10) | (UL(0) << 26))
+#define TCR_ORGN_WBWA		((UL(1) << 10) | (UL(1) << 26))
+#define TCR_ORGN_WT		((UL(2) << 10) | (UL(2) << 26))
+#define TCR_ORGN_WBnWA		((UL(3) << 10) | (UL(3) << 26))
+#define TCR_ORGN_MASK		((UL(3) << 10) | (UL(3) << 26))
+#define TCR_SHARED		((UL(3) << 12) | (UL(3) << 28))
+#define TCR_TG0_4K		(UL(0) << 14)
+#define TCR_TG0_64K		(UL(1) << 14)
+#define TCR_TG0_16K		(UL(2) << 14)
+#define TCR_TG1_16K		(UL(1) << 30)
+#define TCR_TG1_4K		(UL(2) << 30)
+#define TCR_TG1_64K		(UL(3) << 30)
+#define TCR_ASID16		(UL(1) << 36)
+#define TCR_TBI0		(UL(1) << 37)
+
+/*
+ * Memory types available.
+ */
+#define MT_DEVICE_nGnRnE	0
+#define MT_DEVICE_nGnRE		1
+#define MT_DEVICE_GRE		2
+#define MT_NORMAL_NC		3
+#define MT_NORMAL		4
+
+/*
+ * page table entry attribute set.
+ */
+#define PROT_DEFAULT		(PTE_TYPE_PAGE | PTE_AF | PTE_SHARED)
+#define PROT_SECT_DEFAULT	(PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S)
+
+#define PROT_DEVICE_nGnRE	(PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_ATTRINDX(MT_DEVICE_nGnRE))
+#define PROT_NORMAL_NC		(PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_ATTRINDX(MT_NORMAL_NC))
+#define PROT_NORMAL		(PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_ATTRINDX(MT_NORMAL))
+
+#define PROT_SECT_DEVICE_nGnRE	(PROT_SECT_DEFAULT | PMD_SECT_PXN | PMD_SECT_UXN | PMD_ATTRINDX(MT_DEVICE_nGnRE))
+#define PROT_SECT_NORMAL	(PROT_SECT_DEFAULT | PMD_SECT_PXN | PMD_SECT_UXN | PMD_ATTRINDX(MT_NORMAL))
+#define PROT_SECT_NORMAL_EXEC	(PROT_SECT_DEFAULT | PMD_SECT_UXN | PMD_ATTRINDX(MT_NORMAL))
+
+#define _PAGE_DEFAULT		(PROT_DEFAULT | PTE_ATTRINDX(MT_NORMAL))
+
+#define PAGE_KERNEL		(_PAGE_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE)
+#define PAGE_KERNEL_EXEC	(_PAGE_DEFAULT | PTE_UXN | PTE_DIRTY | PTE_WRITE)
+
+#define PAGE_NONE		(((_PAGE_DEFAULT) & ~PTE_TYPE_MASK) | PTE_PROT_NONE | PTE_PXN | PTE_UXN)
+#define PAGE_SHARED		(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN | PTE_WRITE)
+#define PAGE_SHARED_EXEC	(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_WRITE)
+#define PAGE_COPY		(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN)
+#define PAGE_COPY_EXEC		(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN)
+#define PAGE_READONLY		(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN)
+#define PAGE_READONLY_EXEC	(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN)
+
+#define __P000  PAGE_NONE
+#define __P001  PAGE_READONLY
+#define __P010  PAGE_COPY
+#define __P011  PAGE_COPY
+#define __P100  PAGE_READONLY_EXEC
+#define __P101  PAGE_READONLY_EXEC
+#define __P110  PAGE_COPY_EXEC
+#define __P111  PAGE_COPY_EXEC
+
+#define __S000  PAGE_NONE
+#define __S001  PAGE_READONLY
+#define __S010  PAGE_SHARED
+#define __S011  PAGE_SHARED
+#define __S100  PAGE_READONLY_EXEC
+#define __S101  PAGE_READONLY_EXEC
+#define __S110  PAGE_SHARED_EXEC
+#define __S111  PAGE_SHARED_EXEC
+
+#endif /* !__HEADER_ARM64_COMMON_PGTABLE_HWDEF_H */

arch/arm64/kernel/include/pgtable.h

@@ -0,0 +1,7 @@
+/* pgtable.h COPYRIGHT FUJITSU LIMITED 2015 */
+#ifndef __HEADER_ARM64_COMMON_PGTABLE_H
+#define __HEADER_ARM64_COMMON_PGTABLE_H
+
+extern char empty_zero_page[];
+
+#endif /* !__HEADER_ARM64_COMMON_PGTABLE_H */

arch/arm64/kernel/include/prctl.h

@@ -0,0 +1,17 @@
+/* prctl.h COPYRIGHT FUJITSU LIMITED 2017-2019 */
+#ifndef __HEADER_ARM64_COMMON_PRCTL_H
+#define __HEADER_ARM64_COMMON_PRCTL_H
+
+#define PR_SET_THP_DISABLE 41
+#define PR_GET_THP_DISABLE 42
+
+/* arm64 Scalable Vector Extension controls */
+/* Flag values must be kept in sync with ptrace NT_ARM_SVE interface */
+#define PR_SVE_SET_VL		50		/* set task vector length */
+# define PR_SVE_SET_VL_ONEXEC	(1 << 18)	/* defer effect until exec */
+#define PR_SVE_GET_VL		51		/* get task vector length */
+/* Bits common to PR_SVE_SET_VL and PR_SVE_GET_VL */
+# define PR_SVE_VL_LEN_MASK	0xffff
+# define PR_SVE_VL_INHERIT	(1 << 17)	/* inherit across exec */
+
+#endif /* !__HEADER_ARM64_COMMON_PRCTL_H */

arch/arm64/kernel/include/psci.h

@@ -0,0 +1,68 @@
+/* psci.h COPYRIGHT FUJITSU LIMITED 2015-2016 */
+/* @ref.impl include/uapi/linux/psci.h */
+/*
+ * ARM Power State and Coordination Interface (PSCI) header
+ *
+ * This header holds common PSCI defines and macros shared
+ * by: ARM kernel, ARM64 kernel, KVM ARM/ARM64 and user space.
+ *
+ * Copyright (C) 2014 Linaro Ltd.
+ * Author: Anup Patel <anup.patel@linaro.org>
+ */
+
+#ifndef __HEADER_ARM64_PSCI_H
+#define __HEADER_ARM64_PSCI_H
+
+/*
+ * PSCI v0.1 interface
+ *
+ * The PSCI v0.1 function numbers are implementation defined.
+ *
+ * Only PSCI return values such as: SUCCESS, NOT_SUPPORTED,
+ * INVALID_PARAMS, and DENIED defined below are applicable
+ * to PSCI v0.1.
+ */
+
+/* PSCI v0.2 interface */
+#define PSCI_0_2_FN_BASE			0x84000000
+#define PSCI_0_2_FN(n)				(PSCI_0_2_FN_BASE + (n))
+#define PSCI_0_2_64BIT				0x40000000
+#define PSCI_0_2_FN64_BASE			(PSCI_0_2_FN_BASE + PSCI_0_2_64BIT)
+#define PSCI_0_2_FN64(n)			(PSCI_0_2_FN64_BASE + (n))
+
+#define PSCI_0_2_FN_PSCI_VERSION		PSCI_0_2_FN(0)
+#define PSCI_0_2_FN_CPU_OFF			PSCI_0_2_FN(2)
+#define PSCI_0_2_FN64_CPU_ON			PSCI_0_2_FN64(3)
+#define PSCI_0_2_FN64_AFFINITY_INFO		PSCI_0_2_FN64(4)
+
+/* PSCI v0.2 power state encoding for CPU_SUSPEND function */
+#define PSCI_0_2_POWER_STATE_ID_MASK		0xffff
+#define PSCI_0_2_POWER_STATE_ID_SHIFT		0
+#define PSCI_0_2_POWER_STATE_TYPE_SHIFT		16
+#define PSCI_0_2_POWER_STATE_TYPE_MASK		\
+				(0x1 << PSCI_0_2_POWER_STATE_TYPE_SHIFT)
+#define PSCI_0_2_POWER_STATE_AFFL_SHIFT		24
+#define PSCI_0_2_POWER_STATE_AFFL_MASK		\
+				(0x3 << PSCI_0_2_POWER_STATE_AFFL_SHIFT)
+
+/* PSCI version decoding (independent of PSCI version) */
+#define PSCI_VERSION_MAJOR_SHIFT		16
+#define PSCI_VERSION_MINOR_MASK			\
+		((1U << PSCI_VERSION_MAJOR_SHIFT) - 1)
+#define PSCI_VERSION_MAJOR_MASK			~PSCI_VERSION_MINOR_MASK
+#define PSCI_VERSION_MAJOR(ver)			\
+		(((ver) & PSCI_VERSION_MAJOR_MASK) >> PSCI_VERSION_MAJOR_SHIFT)
+#define PSCI_VERSION_MINOR(ver)			\
+		((ver) & PSCI_VERSION_MINOR_MASK)
+
+/* PSCI return values (inclusive of all PSCI versions) */
+#define PSCI_RET_SUCCESS			0
+#define PSCI_RET_NOT_SUPPORTED			-1
+#define PSCI_RET_INVALID_PARAMS			-2
+#define PSCI_RET_DENIED				-3
+
+int psci_init(void);
+int psci_cpu_off(void);
+int cpu_psci_cpu_boot(unsigned int cpu, unsigned long pc);
+
+#endif /* __HEADER_ARM64_PSCI_H */

arch/arm64/kernel/include/ptrace.h

@@ -0,0 +1,268 @@
+/* ptrace.h COPYRIGHT FUJITSU LIMITED 2015-2019 */
+#ifndef __HEADER_ARM64_COMMON_PTRACE_H
+#define __HEADER_ARM64_COMMON_PTRACE_H
+
+/*
+ * PSR bits
+ */
+#define PSR_MODE_EL0t   0x00000000
+#define PSR_MODE_EL1t   0x00000004
+#define PSR_MODE_EL1h   0x00000005
+#define PSR_MODE_EL2t   0x00000008
+#define PSR_MODE_EL2h   0x00000009
+#define PSR_MODE_EL3t   0x0000000c
+#define PSR_MODE_EL3h   0x0000000d
+#define PSR_MODE_MASK   0x0000000f
+
+/* AArch32 CPSR bits */
+#define PSR_MODE32_BIT	0x00000010
+
+/* AArch64 SPSR bits */
+#define PSR_F_BIT       0x00000040
+#define PSR_I_BIT       0x00000080
+#define PSR_A_BIT       0x00000100
+#define PSR_D_BIT       0x00000200
+#define PSR_Q_BIT       0x08000000
+#define PSR_V_BIT       0x10000000
+#define PSR_C_BIT       0x20000000
+#define PSR_Z_BIT       0x40000000
+#define PSR_N_BIT       0x80000000
+
+/*
+ * Groups of PSR bits
+ */
+#define PSR_f           0xff000000      /* Flags                */
+#define PSR_s           0x00ff0000      /* Status               */
+#define PSR_x           0x0000ff00      /* Extension            */
+#define PSR_c           0x000000ff      /* Control              */
+
+/* Current Exception Level values, as contained in CurrentEL */
+#define CurrentEL_EL1	(1 << 2)
+#define CurrentEL_EL2	(2 << 2)
+
+/* thread->ptrace_debugreg lower-area and higher-area */
+#define HWS_BREAK	0
+#define HWS_WATCH	1
+
+#ifndef __ASSEMBLY__
+
+#include <lwk/compiler.h>
+#include <ihk/types.h>
+
+struct user_hwdebug_state {
+	uint32_t dbg_info;
+	uint32_t pad;
+	struct {
+		uint64_t addr;
+		uint32_t ctrl;
+		uint32_t pad;
+	} dbg_regs[16];
+};
+
+struct user_fpsimd_state {
+	__uint128_t	vregs[32];
+	uint32_t	fpsr;
+	uint32_t	fpcr;
+	uint32_t	__reserved[2];
+};
+
+extern unsigned int ptrace_hbp_get_resource_info(unsigned int note_type);
+
+/* SVE/FP/SIMD state (NT_ARM_SVE) */
+
+struct user_sve_header {
+	uint32_t size;		/* total meaningful regset content in bytes */
+	uint32_t max_size;	/* maxmium possible size for this thread */
+	uint16_t vl;		/* current vector length */
+	uint16_t max_vl;	/* maximum possible vector length */
+	uint16_t flags;
+	uint16_t __reserved;
+};
+
+enum aarch64_regset {
+	REGSET_GPR,
+	REGSET_FPR,
+	REGSET_TLS,
+	REGSET_HW_BREAK,
+	REGSET_HW_WATCH,
+	REGSET_SYSTEM_CALL,
+#ifdef CONFIG_ARM64_SVE
+	REGSET_SVE,
+#endif /* CONFIG_ARM64_SVE */
+};
+
+struct thread;
+struct user_regset;
+
+typedef int user_regset_active_fn(struct thread *target,
+				  const struct user_regset *regset);
+
+typedef long user_regset_get_fn(struct thread *target,
+				const struct user_regset *regset,
+				unsigned int pos, unsigned int count,
+				void *kbuf, void __user *ubuf);
+
+typedef long user_regset_set_fn(struct thread *target,
+				const struct user_regset *regset,
+				unsigned int pos, unsigned int count,
+				const void *kbuf, const void __user *ubuf);
+
+typedef int user_regset_writeback_fn(struct thread *target,
+				     const struct user_regset *regset,
+				     int immediate);
+
+typedef unsigned int user_regset_get_size_fn(struct thread *target,
+					     const struct user_regset *regset);
+
+struct user_regset {
+	user_regset_get_fn		*get;
+	user_regset_set_fn		*set;
+	user_regset_active_fn		*active;
+	user_regset_writeback_fn	*writeback;
+	user_regset_get_size_fn		*get_size;
+	unsigned int			n;
+	unsigned int			size;
+	unsigned int			align;
+	unsigned int			bias;
+	unsigned int			core_note_type;
+};
+
+struct user_regset_view {
+	const char *name;
+	const struct user_regset *regsets;
+	unsigned int n;
+	uint32_t e_flags;
+	uint16_t e_machine;
+	uint8_t ei_osabi;
+};
+
+extern const struct user_regset_view *current_user_regset_view(void);
+extern const struct user_regset *find_regset(
+		const struct user_regset_view *view,
+		unsigned int type);
+extern unsigned int regset_size(struct thread *target,
+		const struct user_regset *regset);
+
+/* Definitions for user_sve_header.flags: */
+#define SVE_PT_REGS_MASK	(1 << 0)
+
+#define SVE_PT_REGS_FPSIMD	0
+#define SVE_PT_REGS_SVE		SVE_PT_REGS_MASK
+
+#define SVE_PT_VL_THREAD	PR_SVE_SET_VL_THREAD
+#define SVE_PT_VL_INHERIT	PR_SVE_VL_INHERIT
+#define SVE_PT_VL_ONEXEC	PR_SVE_SET_VL_ONEXEC
+
+/*
+ * The remainder of the SVE state follows struct user_sve_header.  The
+ * total size of the SVE state (including header) depends on the
+ * metadata in the header:  SVE_PT_SIZE(vq, flags) gives the total size
+ * of the state in bytes, including the header.
+ *
+ * Refer to <asm/sigcontext.h> for details of how to pass the correct
+ * "vq" argument to these macros.
+ */
+
+/* Offset from the start of struct user_sve_header to the register data */
+#define SVE_PT_REGS_OFFSET					\
+	((sizeof(struct sve_context) + (SVE_VQ_BYTES - 1))	\
+		/ SVE_VQ_BYTES * SVE_VQ_BYTES)
+
+/*
+ * The register data content and layout depends on the value of the
+ * flags field.
+ */
+
+/*
+ * (flags & SVE_PT_REGS_MASK) == SVE_PT_REGS_FPSIMD case:
+ *
+ * The payload starts at offset SVE_PT_FPSIMD_OFFSET, and is of type
+ * struct user_fpsimd_state.  Additional data might be appended in the
+ * future: use SVE_PT_FPSIMD_SIZE(vq, flags) to compute the total size.
+ * SVE_PT_FPSIMD_SIZE(vq, flags) will never be less than
+ * sizeof(struct user_fpsimd_state).
+ */
+
+#define SVE_PT_FPSIMD_OFFSET		SVE_PT_REGS_OFFSET
+
+#define SVE_PT_FPSIMD_SIZE(vq, flags)	(sizeof(struct user_fpsimd_state))
+
+/*
+ * (flags & SVE_PT_REGS_MASK) == SVE_PT_REGS_SVE case:
+ *
+ * The payload starts at offset SVE_PT_SVE_OFFSET, and is of size
+ * SVE_PT_SVE_SIZE(vq, flags).
+ *
+ * Additional macros describe the contents and layout of the payload.
+ * For each, SVE_PT_SVE_x_OFFSET(args) is the start offset relative to
+ * the start of struct user_sve_header, and SVE_PT_SVE_x_SIZE(args) is
+ * the size in bytes:
+ *
+ *	x	type				description
+ *	-	----				-----------
+ *	ZREGS		\
+ *	ZREG		|
+ *	PREGS		| refer to <asm/sigcontext.h>
+ *	PREG		|
+ *	FFR		/
+ *
+ *	FPSR	uint32_t			FPSR
+ *	FPCR	uint32_t			FPCR
+ *
+ * Additional data might be appended in the future.
+ */
+
+#define SVE_PT_SVE_ZREG_SIZE(vq)	SVE_SIG_ZREG_SIZE(vq)
+#define SVE_PT_SVE_PREG_SIZE(vq)	SVE_SIG_PREG_SIZE(vq)
+#define SVE_PT_SVE_FFR_SIZE(vq)		SVE_SIG_FFR_SIZE(vq)
+#define SVE_PT_SVE_FPSR_SIZE		sizeof(uint32_t)
+#define SVE_PT_SVE_FPCR_SIZE		sizeof(uint32_t)
+
+#define __SVE_SIG_TO_PT(offset) \
+	((offset) - SVE_SIG_REGS_OFFSET + SVE_PT_REGS_OFFSET)
+
+#define SVE_PT_SVE_OFFSET		SVE_PT_REGS_OFFSET
+
+#define SVE_PT_SVE_ZREGS_OFFSET \
+	__SVE_SIG_TO_PT(SVE_SIG_ZREGS_OFFSET)
+#define SVE_PT_SVE_ZREG_OFFSET(vq, n) \
+	__SVE_SIG_TO_PT(SVE_SIG_ZREG_OFFSET(vq, n))
+#define SVE_PT_SVE_ZREGS_SIZE(vq) \
+	(SVE_PT_SVE_ZREG_OFFSET(vq, SVE_NUM_ZREGS) - SVE_PT_SVE_ZREGS_OFFSET)
+
+#define SVE_PT_SVE_PREGS_OFFSET(vq) \
+	__SVE_SIG_TO_PT(SVE_SIG_PREGS_OFFSET(vq))
+#define SVE_PT_SVE_PREG_OFFSET(vq, n) \
+	__SVE_SIG_TO_PT(SVE_SIG_PREG_OFFSET(vq, n))
+#define SVE_PT_SVE_PREGS_SIZE(vq) \
+	(SVE_PT_SVE_PREG_OFFSET(vq, SVE_NUM_PREGS) - \
+		SVE_PT_SVE_PREGS_OFFSET(vq))
+
+#define SVE_PT_SVE_FFR_OFFSET(vq) \
+	__SVE_SIG_TO_PT(SVE_SIG_FFR_OFFSET(vq))
+
+#define SVE_PT_SVE_FPSR_OFFSET(vq)				\
+	((SVE_PT_SVE_FFR_OFFSET(vq) + SVE_PT_SVE_FFR_SIZE(vq) + \
+			(SVE_VQ_BYTES - 1))			\
+		/ SVE_VQ_BYTES * SVE_VQ_BYTES)
+#define SVE_PT_SVE_FPCR_OFFSET(vq) \
+	(SVE_PT_SVE_FPSR_OFFSET(vq) + SVE_PT_SVE_FPSR_SIZE)
+
+/*
+ * Any future extension appended after FPCR must be aligned to the next
+ * 128-bit boundary.
+ */
+
+#define SVE_PT_SVE_SIZE(vq, flags)					\
+	((SVE_PT_SVE_FPCR_OFFSET(vq) + SVE_PT_SVE_FPCR_SIZE		\
+			- SVE_PT_SVE_OFFSET + (SVE_VQ_BYTES - 1))	\
+		/ SVE_VQ_BYTES * SVE_VQ_BYTES)
+
+#define SVE_PT_SIZE(vq, flags)						\
+	(((flags) & SVE_PT_REGS_MASK) == SVE_PT_REGS_SVE ?		\
+		  SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, flags)	\
+		: SVE_PT_FPSIMD_OFFSET + SVE_PT_FPSIMD_SIZE(vq, flags))
+
+#endif /* !__ASSEMBLY__ */
+
+#endif /* !__HEADER_ARM64_COMMON_PTRACE_H */

arch/arm64/kernel/include/registers.h

@@ -0,0 +1,127 @@
+/* registers.h COPYRIGHT FUJITSU LIMITED 2015-2018 */
+#ifndef __HEADER_ARM64_COMMON_REGISTERS_H
+#define __HEADER_ARM64_COMMON_REGISTERS_H
+
+#include <types.h>
+#include <arch/cpu.h>
+#include <sysreg.h>
+
+#define RFLAGS_CF      (1 << 0)
+#define RFLAGS_PF      (1 << 2)
+#define RFLAGS_AF      (1 << 4)
+#define RFLAGS_ZF      (1 << 6)
+#define RFLAGS_SF      (1 << 7)
+#define RFLAGS_TF      (1 << 8)
+#define RFLAGS_IF      (1 << 9)
+#define RFLAGS_DF      (1 << 10)
+#define RFLAGS_OF      (1 << 11)
+#define RFLAGS_IOPL    (3 << 12)
+#define RFLAGS_NT      (1 << 14)
+#define RFLAGS_RF      (1 << 16)
+#define RFLAGS_VM      (1 << 17)
+#define RFLAGS_AC      (1 << 18)
+#define RFLAGS_VIF     (1 << 19)
+#define RFLAGS_VIP     (1 << 20)
+#define RFLAGS_ID      (1 << 21)
+
+#define DB6_B0         (1 << 0)
+#define DB6_B1         (1 << 1)
+#define DB6_B2         (1 << 2)
+#define DB6_B3         (1 << 3)
+#define DB6_BD         (1 << 13)
+#define DB6_BS         (1 << 14)
+#define DB6_BT         (1 << 15)
+
+#define MSR_EFER       0xc0000080
+#define MSR_STAR       0xc0000081
+#define MSR_LSTAR      0xc0000082
+#define MSR_FMASK      0xc0000084
+#define MSR_FS_BASE    0xc0000100
+#define MSR_GS_BASE    0xc0000101
+
+#define MSR_IA32_APIC_BASE 0x000000001b
+#define MSR_PLATFORM_INFO  0x000000ce
+#define MSR_IA32_PERF_CTL  0x00000199
+#define MSR_IA32_MISC_ENABLE		0x000001a0
+#define MSR_IA32_ENERGY_PERF_BIAS	0x000001b0
+#define MSR_NHM_TURBO_RATIO_LIMIT	0x000001ad
+#define MSR_IA32_CR_PAT			0x00000277
+
+
+#define CVAL(event, mask) \
+        ((((event) & 0xf00) << 24) | ((mask) << 8) | ((event) & 0xff))
+#define CVAL2(event, mask, inv, count)    \
+        ((((event) & 0xf00) << 24) | ((mask) << 8) | ((event) & 0xff) | \
+         ((inv & 1) << 23) | ((count & 0xff) << 24))
+
+/* AMD */
+#define MSR_PERF_CTL_0 0xc0010000
+#define MSR_PERF_CTR_0 0xc0010004
+
+static unsigned long xgetbv(unsigned int index)
+{
+	return 0;
+}
+
+static void xsetbv(unsigned int index, unsigned long val)
+{
+}
+
+static unsigned long rdpmc(unsigned int counter)
+{
+	return 0;
+}
+
+static unsigned long rdmsr(unsigned int index)
+{
+	return 0;
+}
+
+/* @ref.impl linux4.10.16 */
+/* arch/arm64/include/asm/arch_timer.h:arch_counter_get_cntvct() */
+static inline unsigned long rdtsc(void)
+{
+	isb();
+	return read_sysreg(cntvct_el0);
+}
+
+static void set_perfctl(int counter, int event, int mask)
+{
+}
+
+static void start_perfctr(int counter)
+{
+}
+static void stop_perfctr(int counter)
+{
+}
+
+static void clear_perfctl(int counter)
+{
+}
+
+static void set_perfctr(int counter, unsigned long value)
+{
+}
+
+static unsigned long read_perfctr(int counter)
+{
+	return 0;
+}
+
+#define ihk_mc_mb()	do {} while(0);
+
+#define	REGS_GET_STACK_POINTER(regs)	(((struct pt_regs *)regs)->sp)
+
+enum arm64_pf_error_code {
+	PF_PROT		=	1 << 0,
+	PF_WRITE	=	1 << 1,
+	PF_USER		=	1 << 2,
+	PF_RSVD		=	1 << 3,
+	PF_INSTR	=	1 << 4,
+
+	PF_PATCH	=	1 << 29,
+	PF_POPULATE	=	1 << 30,
+};
+
+#endif /* !__HEADER_ARM64_COMMON_REGISTERS_H */

arch/arm64/kernel/include/rlimit.h

@@ -0,0 +1,100 @@
+/* rlimit.h COPYRIGHT FUJITSU LIMITED 2016 */
+/**
+ * \file rlimit.h
+ *  License details are found in the file LICENSE.
+ * \brief
+ *  Kinds of resource limit
+ * \author Taku Shimosawa  <shimosawa@is.s.u-tokyo.ac.jp> \par
+ * Copyright (C) 2011 - 2012  Taku Shimosawa
+ */
+/*
+ * HISTORY
+ */
+
+#ifndef __HEADER_ARM64_COMMON_RLIMIT_H
+#define __HEADER_ARM64_COMMON_RLIMIT_H
+
+/* Kinds of resource limit.  */
+enum __rlimit_resource
+{
+  /* Per-process CPU limit, in seconds.  */
+  RLIMIT_CPU = 0,
+#define RLIMIT_CPU RLIMIT_CPU
+
+  /* Largest file that can be created, in bytes.  */
+  RLIMIT_FSIZE = 1,
+#define	RLIMIT_FSIZE RLIMIT_FSIZE
+
+  /* Maximum size of data segment, in bytes.  */
+  RLIMIT_DATA = 2,
+#define	RLIMIT_DATA RLIMIT_DATA
+
+  /* Maximum size of stack segment, in bytes.  */
+  RLIMIT_STACK = 3,
+#define	RLIMIT_STACK RLIMIT_STACK
+
+  /* Largest core file that can be created, in bytes.  */
+  RLIMIT_CORE = 4,
+#define	RLIMIT_CORE RLIMIT_CORE
+
+  /* Largest resident set size, in bytes.
+     This affects swapping; processes that are exceeding their
+     resident set size will be more likely to have physical memory
+     taken from them.  */
+  __RLIMIT_RSS = 5,
+#define	RLIMIT_RSS __RLIMIT_RSS
+
+  /* Number of open files.  */
+  RLIMIT_NOFILE = 7,
+  __RLIMIT_OFILE = RLIMIT_NOFILE, /* BSD name for same.  */
+#define RLIMIT_NOFILE RLIMIT_NOFILE
+#define RLIMIT_OFILE __RLIMIT_OFILE
+
+  /* Address space limit.  */
+  RLIMIT_AS = 9,
+#define RLIMIT_AS RLIMIT_AS
+
+  /* Number of processes.  */
+  __RLIMIT_NPROC = 6,
+#define RLIMIT_NPROC __RLIMIT_NPROC
+
+  /* Locked-in-memory address space.  */
+  __RLIMIT_MEMLOCK = 8,
+#define RLIMIT_MEMLOCK __RLIMIT_MEMLOCK
+
+  /* Maximum number of file locks.  */
+  __RLIMIT_LOCKS = 10,
+#define RLIMIT_LOCKS __RLIMIT_LOCKS
+
+  /* Maximum number of pending signals.  */
+  __RLIMIT_SIGPENDING = 11,
+#define RLIMIT_SIGPENDING __RLIMIT_SIGPENDING
+
+  /* Maximum bytes in POSIX message queues.  */
+  __RLIMIT_MSGQUEUE = 12,
+#define RLIMIT_MSGQUEUE __RLIMIT_MSGQUEUE
+
+  /* Maximum nice priority allowed to raise to.
+     Nice levels 19 .. -20 correspond to 0 .. 39
+     values of this resource limit.  */
+  __RLIMIT_NICE = 13,
+#define RLIMIT_NICE __RLIMIT_NICE
+
+  /* Maximum realtime priority allowed for non-priviledged
+     processes.  */
+  __RLIMIT_RTPRIO = 14,
+#define RLIMIT_RTPRIO __RLIMIT_RTPRIO
+
+  /* timeout for RT tasks in us */
+  __RLIMIT_RTTIME = 15,
+#define RLIMIT_RTTIME __RLIMIT_RTTIME
+
+  __RLIMIT_NLIMITS = 16,
+  __RLIM_NLIMITS = __RLIMIT_NLIMITS
+#define RLIMIT_NLIMITS __RLIMIT_NLIMITS
+#define RLIM_NLIMITS __RLIM_NLIMITS
+};
+
+#include <generic-rlimit.h>
+
+#endif

arch/arm64/kernel/include/signal.h

@@ -0,0 +1,421 @@
+/* signal.h COPYRIGHT FUJITSU LIMITED 2015-2019 */
+#ifndef __HEADER_ARM64_COMMON_SIGNAL_H
+#define __HEADER_ARM64_COMMON_SIGNAL_H
+
+#include <fpsimd.h>
+#include <ihk/types.h>
+
+#define _NSIG		64
+#define _NSIG_BPW	64
+#define _NSIG_WORDS (_NSIG / _NSIG_BPW)
+
+static inline int valid_signal(unsigned long sig)
+{
+	return sig <= _NSIG ? 1 : 0;
+}
+
+typedef unsigned long int __sigset_t;
+#define __sigmask(sig)  (((__sigset_t) 1) << ((sig) - 1))
+
+typedef struct {
+	__sigset_t __val[_NSIG_WORDS];
+} sigset_t;
+
+#define SIG_BLOCK 0
+#define SIG_UNBLOCK 1
+#define SIG_SETMASK 2
+
+struct sigaction {
+	void (*sa_handler)(int);
+	unsigned long sa_flags;
+	void (*sa_restorer)(int);
+	sigset_t sa_mask;
+};
+
+typedef void __sig_fn_t(int);
+typedef __sig_fn_t *__sig_handler_t;
+#define SIG_DFL (__sig_handler_t)0
+#define SIG_IGN (__sig_handler_t)1
+#define SIG_ERR (__sig_handler_t)-1
+
+#define SA_NOCLDSTOP    0x00000001U
+#define SA_NOCLDWAIT    0x00000002U
+#define SA_NODEFER      0x40000000U
+#define SA_ONSTACK      0x08000000U
+#define SA_RESETHAND    0x80000000U
+#define SA_RESTART      0x10000000U
+#define SA_SIGINFO      0x00000004U
+
+/* Required for AArch32 compatibility. */
+#define SA_RESTORER	0x04000000U
+
+struct k_sigaction {
+        struct sigaction sa;
+};
+
+typedef struct sigaltstack {
+	void	*ss_sp;
+	int	ss_flags;
+	size_t	ss_size;
+} stack_t;
+
+#define MINSIGSTKSZ 5120
+#define SS_ONSTACK 1
+#define SS_DISABLE 2
+
+typedef union sigval {
+	int sival_int;
+	void *sival_ptr;
+} sigval_t;
+
+#define __SI_MAX_SIZE 128
+#define __SI_PAD_SIZE ((__SI_MAX_SIZE / sizeof (int)) - 4)
+
+typedef struct siginfo {
+	int si_signo;		/* Signal number.  */
+	int si_errno;		/* If non-zero, an errno value associated with
+				   this signal, as defined in <errno.h>.  */
+	int si_code;		/* Signal code.  */
+#define SI_USER         0  /* sent by kill, sigsend, raise */
+#define SI_KERNEL       0x80 /* sent by the kernel from somewhere */
+#define SI_QUEUE        -1 /* sent by sigqueue */
+#define SI_TIMER __SI_CODE(__SI_TIMER,-2) /* sent by timer expiration */
+#define SI_MESGQ __SI_CODE(__SI_MESGQ,-3) /* sent by real time mesq state change
+ */
+#define SI_ASYNCIO      -4 /* sent by AIO completion */
+#define SI_SIGIO        -5 /* sent by queued SIGIO */
+#define SI_TKILL        -6 /* sent by tkill system call */
+#define SI_DETHREAD     -7 /* sent by execve() killing subsidiary threads */
+
+#define ILL_ILLOPC      1  /* illegal opcode */
+#define ILL_ILLOPN      2  /* illegal operand */
+#define ILL_ILLADR      3  /* illegal addressing mode */
+#define ILL_ILLTRP      4  /* illegal trap */
+#define ILL_PRVOPC      5  /* privileged opcode */
+#define ILL_PRVREG      6  /* privileged register */
+#define ILL_COPROC      7  /* coprocessor error */
+#define ILL_BADSTK      8  /* internal stack error */
+
+#define FPE_INTDIV      1  /* integer divide by zero */
+#define FPE_INTOVF      2  /* integer overflow */
+#define FPE_FLTDIV      3  /* floating point divide by zero */
+#define FPE_FLTOVF      4  /* floating point overflow */
+#define FPE_FLTUND      5  /* floating point underflow */
+#define FPE_FLTRES      6  /* floating point inexact result */
+#define FPE_FLTINV      7  /* floating point invalid operation */
+#define FPE_FLTSUB      8  /* subscript out of range */
+
+#define SEGV_MAPERR     1  /* address not mapped to object */
+#define SEGV_ACCERR     2  /* invalid permissions for mapped object */
+
+#define BUS_ADRALN      1  /* invalid address alignment */
+#define BUS_ADRERR      2  /* non-existant physical address */
+#define BUS_OBJERR      3  /* object specific hardware error */
+/* hardware memory error consumed on a machine check: action required */
+#define BUS_MCEERR_AR   4
+/* hardware memory error detected in process but not consumed: action optional*/
+#define BUS_MCEERR_AO   5
+
+#define TRAP_BRKPT      1  /* process breakpoint */
+#define TRAP_TRACE      2  /* process trace trap */
+#define TRAP_BRANCH     3  /* process taken branch trap */
+#define TRAP_HWBKPT     4  /* hardware breakpoint/watchpoint */
+
+#define CLD_EXITED      1   /* child has exited */
+#define CLD_KILLED      2   /* child was killed */
+#define CLD_DUMPED      3   /* child terminated abnormally */
+#define CLD_TRAPPED     4   /* traced child has trapped */
+#define CLD_STOPPED     5   /* child has stopped */
+#define CLD_CONTINUED   6   /* stopped child has continued */
+
+#define POLL_IN         1   /* data input available */
+#define POLL_OUT        2   /* output buffers available */
+#define POLL_MSG        3   /* input message available */
+#define POLL_ERR        4   /* i/o error */
+#define POLL_PRI        5   /* high priority input available */
+#define POLL_HUP        6   /* device disconnected */
+
+#define SIGEV_SIGNAL    0   /* notify via signal */
+#define SIGEV_NONE      1   /* other notification: meaningless */
+#define SIGEV_THREAD    2   /* deliver via thread creation */
+#define SIGEV_THREAD_ID 4   /* deliver to thread */
+
+	union {
+		int _pad[__SI_PAD_SIZE];
+
+		/* kill().  */
+		struct {
+			int si_pid;/* Sending process ID.  */
+			int si_uid;/* Real user ID of sending process.  */
+		} _kill;
+
+		/* POSIX.1b timers.  */
+		struct {
+			int si_tid;         /* Timer ID.  */
+			int si_overrun;     /* Overrun count.  */
+			sigval_t si_sigval; /* Signal value.  */
+		} _timer;
+
+		/* POSIX.1b signals.  */
+		struct {
+			int si_pid;     /* Sending process ID.  */
+			int si_uid;     /* Real user ID of sending process.  */
+			sigval_t si_sigval; /* Signal value.  */
+		} _rt;
+
+		/* SIGCHLD.  */
+		struct {
+			int si_pid;     /* Which child.  */
+			int si_uid;     /* Real user ID of sending process.  */
+			int si_status;      /* Exit value or signal.  */
+			long si_utime;
+			long si_stime;
+		} _sigchld;
+
+		/* SIGILL, SIGFPE, SIGSEGV, SIGBUS.  */
+		struct {
+			void *si_addr;      /* Faulting insn/memory ref.  */
+		} _sigfault;
+
+		/* SIGPOLL.  */
+		struct {
+			long int si_band;   /* Band event for SIGPOLL.  */
+			int si_fd;
+		} _sigpoll;
+	} _sifields;
+} siginfo_t;
+
+struct signalfd_siginfo {
+	unsigned int ssi_signo;
+	int ssi_errno;
+	int ssi_code;
+	unsigned int ssi_pid;
+	unsigned int ssi_uid;
+	int ssi_fd;
+	unsigned int ssi_tid;
+	unsigned int ssi_band;
+	unsigned int ssi_overrun;
+	unsigned int ssi_trapno;
+	int ssi_status;
+	int ssi_int;
+	unsigned long ssi_ptr;
+	unsigned long ssi_utime;
+	unsigned long ssi_stime;
+	unsigned long ssi_addr;
+	unsigned short ssi_addr_lsb;
+
+	char __pad[46];
+};
+
+
+#define SIGHUP           1
+#define SIGINT           2
+#define SIGQUIT          3
+#define SIGILL           4
+#define SIGTRAP          5
+#define SIGABRT          6
+#define SIGIOT           6
+#define SIGBUS           7
+#define SIGFPE           8
+#define SIGKILL          9
+#define SIGUSR1         10
+#define SIGSEGV         11
+#define SIGUSR2         12
+#define SIGPIPE         13
+#define SIGALRM         14
+#define SIGTERM         15
+#define SIGSTKFLT       16
+#define SIGCHLD         17
+#define SIGCONT         18
+#define SIGSTOP         19
+#define SIGTSTP         20
+#define SIGTTIN         21
+#define SIGTTOU         22
+#define SIGURG          23
+#define SIGXCPU         24
+#define SIGXFSZ         25
+#define SIGVTALRM       26
+#define SIGPROF         27
+#define SIGWINCH        28
+#define SIGIO           29
+#define SIGPOLL         SIGIO
+#define SIGPWR          30
+#define SIGSYS          31
+#define SIGUNUSED       31
+#define SIGRTMIN        32
+
+#ifndef SIGRTMAX
+#define SIGRTMAX	_NSIG
+#endif
+
+#define PTRACE_EVENT_EXEC 4
+
+/*
+ * @ref.impl linux-linaro/arch/arm64/include/uapi/asm/sigcontext.h
+ */
+struct sigcontext {
+	unsigned long fault_address;
+	/* AArch64 registers */
+	unsigned long regs[31];
+	unsigned long sp;
+	unsigned long pc;
+	unsigned long pstate;
+	/* 4K reserved for FP/SIMD state and future expansion */
+	unsigned char __reserved[4096] /*__attribute__((__aligned__(16)))*/;
+};
+
+/*
+ * Header to be used at the beginning of structures extending the user
+ * context. Such structures must be placed after the rt_sigframe on the stack
+ * and be 16-byte aligned. The last structure must be a dummy one with the
+ * magic and size set to 0.
+ */
+struct _aarch64_ctx {
+	unsigned int magic;
+	unsigned int size;
+};
+
+#define FPSIMD_MAGIC	0x46508001
+
+struct fpsimd_context {
+	struct _aarch64_ctx head;
+	unsigned int fpsr;
+	unsigned int fpcr;
+	__uint128_t vregs[32];
+};
+
+/* ESR_EL1 context */
+#define ESR_MAGIC	0x45535201
+
+struct esr_context {
+	struct _aarch64_ctx head;
+	unsigned long esr;
+};
+
+#define EXTRA_MAGIC	0x45585401
+
+struct extra_context {
+	struct _aarch64_ctx head;
+	void *data;	/* 16-byte aligned pointer to the extra space */
+	uint32_t size;	/* size in bytes of the extra space */
+	uint32_t __reserved[3];
+};
+
+#define SVE_MAGIC	0x53564501
+
+#define fpsimd_sve_state(vq) {			\
+	__uint128_t zregs[32][vq];		\
+	uint16_t pregs[16][vq];			\
+	uint16_t ffr[vq];			\
+}
+
+struct sve_context {
+	struct _aarch64_ctx head;
+	uint16_t vl;
+	uint16_t __reserved[3];
+};
+
+/*
+ * The SVE architecture leaves space for future expansion of the
+ * vector length beyond its initial architectural limit of 2048 bits
+ * (16 quadwords).
+ *
+ * See linux/Documentation/arm64/sve.txt for a description of the VL/VQ
+ * terminology.
+ */
+#define SVE_VQ_BYTES	16	/* number of bytes per quadword */
+
+#define SVE_VQ_MIN	1
+#define SVE_VQ_MAX	512
+
+#define SVE_VL_MIN	(SVE_VQ_MIN * SVE_VQ_BYTES)
+#define SVE_VL_MAX	(SVE_VQ_MAX * SVE_VQ_BYTES)
+
+#define SVE_NUM_ZREGS	32
+#define SVE_NUM_PREGS	16
+
+#define sve_vl_valid(vl) \
+	((vl) % SVE_VQ_BYTES == 0 && (vl) >= SVE_VL_MIN && (vl) <= SVE_VL_MAX)
+#define sve_vq_from_vl(vl)	((vl) / SVE_VQ_BYTES)
+#define sve_vl_from_vq(vq)	((vq) * SVE_VQ_BYTES)
+
+/*
+ * The total size of meaningful data in the SVE context in bytes,
+ * including the header, is given by SVE_SIG_CONTEXT_SIZE(vq).
+ *
+ * Note: for all these macros, the "vq" argument denotes the SVE
+ * vector length in quadwords (i.e., units of 128 bits).
+ *
+ * The correct way to obtain vq is to use sve_vq_from_vl(vl).  The
+ * result is valid if and only if sve_vl_valid(vl) is true.  This is
+ * guaranteed for a struct sve_context written by the kernel.
+ *
+ *
+ * Additional macros describe the contents and layout of the payload.
+ * For each, SVE_SIG_x_OFFSET(args) is the start offset relative to
+ * the start of struct sve_context, and SVE_SIG_x_SIZE(args) is the
+ * size in bytes:
+ *
+ *
+ *	x	type				description
+ *	-	----				-----------
+ *	REGS					the entire SVE context
+ *
+ *	ZREGS	__uint128_t[SVE_NUM_ZREGS][vq]	all Z-registers
+ *	ZREG	__uint128_t[vq]			individual Z-register Zn
+ *
+ *	PREGS	uint16_t[SVE_NUM_PREGS][vq]	all P-registers
+ *	PREG	uint16_t[vq]			individual P-register Pn
+ *
+ *	FFR	uint16_t[vq]			first-fault status register
+ *
+ * Additional data might be appended in the future.
+ */
+
+#define SVE_SIG_ZREG_SIZE(vq)	((uint32_t)(vq) * SVE_VQ_BYTES)
+#define SVE_SIG_PREG_SIZE(vq)	((uint32_t)(vq) * (SVE_VQ_BYTES / 8))
+#define SVE_SIG_FFR_SIZE(vq)	SVE_SIG_PREG_SIZE(vq)
+
+#define SVE_SIG_REGS_OFFSET					\
+	((sizeof(struct sve_context) + (SVE_VQ_BYTES - 1))	\
+		/ SVE_VQ_BYTES * SVE_VQ_BYTES)
+
+#define SVE_SIG_ZREGS_OFFSET	SVE_SIG_REGS_OFFSET
+#define SVE_SIG_ZREG_OFFSET(vq, n) \
+	(SVE_SIG_ZREGS_OFFSET + SVE_SIG_ZREG_SIZE(vq) * (n))
+#define SVE_SIG_ZREGS_SIZE(vq) \
+	(SVE_SIG_ZREG_OFFSET(vq, SVE_NUM_ZREGS) - SVE_SIG_ZREGS_OFFSET)
+
+#define SVE_SIG_PREGS_OFFSET(vq) \
+	(SVE_SIG_ZREGS_OFFSET + SVE_SIG_ZREGS_SIZE(vq))
+#define SVE_SIG_PREG_OFFSET(vq, n) \
+	(SVE_SIG_PREGS_OFFSET(vq) + SVE_SIG_PREG_SIZE(vq) * (n))
+#define SVE_SIG_PREGS_SIZE(vq) \
+	(SVE_SIG_PREG_OFFSET(vq, SVE_NUM_PREGS) - SVE_SIG_PREGS_OFFSET(vq))
+
+#define SVE_SIG_FFR_OFFSET(vq) \
+	(SVE_SIG_PREGS_OFFSET(vq) + SVE_SIG_PREGS_SIZE(vq))
+
+#define SVE_SIG_REGS_SIZE(vq) \
+	(SVE_SIG_FFR_OFFSET(vq) + SVE_SIG_FFR_SIZE(vq) - SVE_SIG_REGS_OFFSET)
+
+#define SVE_SIG_CONTEXT_SIZE(vq) (SVE_SIG_REGS_OFFSET + SVE_SIG_REGS_SIZE(vq))
+
+/*
+ * @ref.impl linux-linaro/arch/arm64/include/asm/ucontext.h
+ */
+struct ucontext {
+	unsigned long uc_flags;
+	struct ucontext *uc_link;
+	stack_t uc_stack;
+	sigset_t uc_sigmask;
+	/* glibc uses a 1024-bit sigset_t */
+	unsigned char __unused[1024 / 8 - sizeof(sigset_t)];
+	/* last for future expansion */
+	struct sigcontext uc_mcontext;
+};
+
+void arm64_notify_die(const char *str, struct pt_regs *regs, struct siginfo *info, int err);
+void check_signal_irq_disabled(unsigned long rc, void *regs, int num);
+
+#endif /* __HEADER_ARM64_COMMON_SIGNAL_H */

arch/arm64/kernel/include/smp.h

@@ -0,0 +1,23 @@
+/* smp.h COPYRIGHT FUJITSU LIMITED 2015 */
+#ifndef __HEADER_ARM64_COMMON_SMP_H
+#define __HEADER_ARM64_COMMON_SMP_H
+
+#ifndef __ASSEMBLY__
+/*
+ * Initial data for bringing up a secondary CPU.
+ */
+struct secondary_data {
+	void *stack;
+	unsigned long next_pc;
+	unsigned long arg;
+};
+extern struct secondary_data secondary_data;
+
+#endif /* __ASSEMBLY__ */
+
+/* struct secondary_data offsets */
+#define SECONDARY_DATA_STACK	0x00
+#define SECONDARY_DATA_NEXT_PC	0x08
+#define SECONDARY_DATA_ARG	0x10
+
+#endif /* !__HEADER_ARM64_COMMON_SMP_H */

arch/arm64/kernel/include/stringify.h

@@ -0,0 +1,17 @@
+/* stringify.h COPYRIGHT FUJITSU LIMITED 2017 */
+
+/**
+ * @ref.impl host-kernel/include/linux/stringify.h
+ */
+#ifndef __LINUX_STRINGIFY_H
+#define __LINUX_STRINGIFY_H
+
+/* Indirect stringification.  Doing two levels allows the parameter to be a
+ * macro itself.  For example, compile with -DFOO=bar, __stringify(FOO)
+ * converts to "bar".
+ */
+
+#define __stringify_1(x...)#x
+#define __stringify(x...)__stringify_1(x)
+
+#endif/* !__LINUX_STRINGIFY_H */

arch/arm64/kernel/include/syscall_list.h

@@ -0,0 +1,152 @@
+/* syscall_list.h COPYRIGHT FUJITSU LIMITED 2015-2018 */
+
+SYSCALL_DELEGATED(4, io_getevents)
+SYSCALL_DELEGATED(17, getcwd)
+SYSCALL_HANDLED(22, epoll_pwait)
+SYSCALL_DELEGATED(25, fcntl)
+SYSCALL_HANDLED(29, ioctl)
+SYSCALL_DELEGATED(35, unlinkat)
+SYSCALL_DELEGATED(43, statfs)
+SYSCALL_DELEGATED(44, fstatfs)
+SYSCALL_HANDLED(56, openat)
+SYSCALL_HANDLED(57, close)
+SYSCALL_DELEGATED(61, getdents64)
+SYSCALL_DELEGATED(62, lseek)
+SYSCALL_HANDLED(63, read)
+SYSCALL_DELEGATED(64, write)
+SYSCALL_DELEGATED(66, writev)
+SYSCALL_DELEGATED(67, pread64)
+SYSCALL_DELEGATED(68, pwrite64)
+SYSCALL_HANDLED(72, pselect6)
+SYSCALL_HANDLED(73, ppoll)
+SYSCALL_HANDLED(74, signalfd4)
+SYSCALL_DELEGATED(78, readlinkat)
+SYSCALL_DELEGATED(80, fstat)
+SYSCALL_HANDLED(93, exit)
+SYSCALL_HANDLED(94, exit_group)
+SYSCALL_HANDLED(95, waitid)
+SYSCALL_HANDLED(96, set_tid_address)
+SYSCALL_HANDLED(98, futex)
+SYSCALL_HANDLED(99, set_robust_list)
+SYSCALL_HANDLED(101, nanosleep)
+SYSCALL_HANDLED(102, getitimer)
+SYSCALL_HANDLED(103, setitimer)
+SYSCALL_HANDLED(113, clock_gettime)
+SYSCALL_DELEGATED(114, clock_getres)
+SYSCALL_DELEGATED(115, clock_nanosleep)
+SYSCALL_HANDLED(117, ptrace)
+SYSCALL_HANDLED(118, sched_setparam)
+SYSCALL_HANDLED(119, sched_setscheduler)
+SYSCALL_HANDLED(120, sched_getscheduler)
+SYSCALL_HANDLED(121, sched_getparam)
+SYSCALL_HANDLED(122, sched_setaffinity)
+SYSCALL_HANDLED(123, sched_getaffinity)
+SYSCALL_HANDLED(124, sched_yield)
+SYSCALL_HANDLED(125, sched_get_priority_max)
+SYSCALL_HANDLED(126, sched_get_priority_min)
+SYSCALL_HANDLED(127, sched_rr_get_interval)
+SYSCALL_HANDLED(129, kill)
+SYSCALL_HANDLED(130, tkill)
+SYSCALL_HANDLED(131, tgkill)
+SYSCALL_HANDLED(132, sigaltstack)
+SYSCALL_HANDLED(133, rt_sigsuspend)
+SYSCALL_HANDLED(134, rt_sigaction)
+SYSCALL_HANDLED(135, rt_sigprocmask)
+SYSCALL_HANDLED(136, rt_sigpending)
+SYSCALL_HANDLED(137, rt_sigtimedwait)
+SYSCALL_HANDLED(138, rt_sigqueueinfo)
+SYSCALL_HANDLED(139, rt_sigreturn)
+SYSCALL_HANDLED(143, setregid)
+SYSCALL_HANDLED(144, setgid)
+SYSCALL_HANDLED(145, setreuid)
+SYSCALL_HANDLED(146, setuid)
+SYSCALL_HANDLED(147, setresuid)
+SYSCALL_HANDLED(148, getresuid)
+SYSCALL_HANDLED(149, setresgid)
+SYSCALL_HANDLED(150, getresgid)
+SYSCALL_HANDLED(151, setfsuid)
+SYSCALL_HANDLED(152, setfsgid)
+SYSCALL_HANDLED(153, times)
+SYSCALL_HANDLED(154, setpgid)
+SYSCALL_DELEGATED(160, uname)
+SYSCALL_HANDLED(163, getrlimit)
+SYSCALL_HANDLED(164, setrlimit)
+SYSCALL_HANDLED(165, getrusage)
+SYSCALL_HANDLED(167, prctl)
+SYSCALL_HANDLED(168, getcpu)
+SYSCALL_HANDLED(169, gettimeofday)
+SYSCALL_HANDLED(170, settimeofday)
+SYSCALL_HANDLED(172, getpid)
+SYSCALL_HANDLED(173, getppid)
+SYSCALL_HANDLED(174, getuid)
+SYSCALL_HANDLED(175, geteuid)
+SYSCALL_HANDLED(176, getgid)
+SYSCALL_HANDLED(177, getegid)
+SYSCALL_HANDLED(178, gettid)
+SYSCALL_HANDLED(179, sysinfo)
+SYSCALL_DELEGATED(188, msgrcv)
+SYSCALL_DELEGATED(189, msgsnd)
+SYSCALL_DELEGATED(192, semtimedop)
+SYSCALL_DELEGATED(193, semop)
+SYSCALL_HANDLED(194, shmget)
+SYSCALL_HANDLED(195, shmctl)
+SYSCALL_HANDLED(196, shmat)
+SYSCALL_HANDLED(197, shmdt)
+SYSCALL_HANDLED(214, brk)
+SYSCALL_HANDLED(215, munmap)
+SYSCALL_HANDLED(216, mremap)
+SYSCALL_HANDLED(220, clone)
+SYSCALL_HANDLED(221, execve)
+SYSCALL_HANDLED(222, mmap)
+SYSCALL_HANDLED(226, mprotect)
+SYSCALL_HANDLED(227, msync)
+SYSCALL_HANDLED(228, mlock)
+SYSCALL_HANDLED(229, munlock)
+SYSCALL_HANDLED(230, mlockall)
+SYSCALL_HANDLED(231, munlockall)
+SYSCALL_HANDLED(232, mincore)
+SYSCALL_HANDLED(233, madvise)
+SYSCALL_HANDLED(234, remap_file_pages)
+SYSCALL_HANDLED(235, mbind)
+SYSCALL_HANDLED(236, get_mempolicy)
+SYSCALL_HANDLED(237, set_mempolicy)
+SYSCALL_HANDLED(238, migrate_pages)
+SYSCALL_HANDLED(239, move_pages)
+#ifdef ENABLE_PERF
+SYSCALL_HANDLED(241, perf_event_open)
+#else // PERF_ENABLE
+SYSCALL_DELEGATED(241, perf_event_open)
+#endif // PERF_ENABLE
+SYSCALL_HANDLED(260, wait4)
+SYSCALL_HANDLED(261, prlimit64)
+SYSCALL_HANDLED(270, process_vm_readv)
+SYSCALL_HANDLED(271, process_vm_writev)
+SYSCALL_HANDLED(281, execveat)
+SYSCALL_HANDLED(700, get_cpu_id)
+#ifdef PROFILE_ENABLE
+SYSCALL_HANDLED(PROFILE_EVENT_MAX, profile)
+#endif // PROFILE_ENABLE
+SYSCALL_HANDLED(730, util_migrate_inter_kernel)
+SYSCALL_HANDLED(731, util_indicate_clone)
+SYSCALL_HANDLED(732, get_system)
+SYSCALL_HANDLED(733, util_register_desc)
+
+/* McKernel Specific */
+SYSCALL_HANDLED(801, swapout)
+SYSCALL_HANDLED(802, linux_mlock)
+SYSCALL_HANDLED(803, suspend_threads)
+SYSCALL_HANDLED(804, resume_threads)
+SYSCALL_HANDLED(811, linux_spawn)
+
+SYSCALL_DELEGATED(1024, open)
+SYSCALL_DELEGATED(1035, readlink)
+SYSCALL_HANDLED(1045, signalfd)
+SYSCALL_DELEGATED(1049, stat)
+SYSCALL_DELEGATED(1060, getpgrp)
+SYSCALL_HANDLED(1062, time)
+SYSCALL_DELEGATED(1069, epoll_wait)
+
+/* Do not edit the lines including this comment and
+ * EOF just after it because those are used as a
+ * robust marker for the autotest patch.
+ */

arch/arm64/kernel/include/sysreg.h

@@ -0,0 +1,403 @@
+/* sysreg.h COPYRIGHT FUJITSU LIMITED 2016-2018 */
+/*
+ * Macros for accessing system registers with older binutils.
+ *
+ * Copyright (C) 2014 ARM Ltd.
+ * Author: Catalin Marinas <catalin.marinas@arm.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __ASM_SYSREG_H
+#define __ASM_SYSREG_H
+
+#include <types.h>
+#include <stringify.h>
+#include <ihk/types.h>
+
+/*
+ * ARMv8 ARM reserves the following encoding for system registers:
+ * (Ref: ARMv8 ARM, Section: "System instruction class encoding overview",
+ *  C5.2, version:ARM DDI 0487A.f)
+ *	[20-19] : Op0
+ *	[18-16] : Op1
+ *	[15-12] : CRn
+ *	[11-8]  : CRm
+ *	[7-5]   : Op2
+ */
+#define Op0_shift	19
+#define Op0_mask	0x3
+#define Op1_shift	16
+#define Op1_mask	0x7
+#define CRn_shift	12
+#define CRn_mask	0xf
+#define CRm_shift	8
+#define CRm_mask	0xf
+#define Op2_shift	5
+#define Op2_mask	0x7
+
+#define sys_reg(op0, op1, crn, crm, op2) \
+	(((op0) << Op0_shift) | ((op1) << Op1_shift) | \
+	 ((crn) << CRn_shift) | ((crm) << CRm_shift) | \
+	 ((op2) << Op2_shift))
+
+#define sys_reg_Op0(id)	(((id) >> Op0_shift) & Op0_mask)
+#define sys_reg_Op1(id)	(((id) >> Op1_shift) & Op1_mask)
+#define sys_reg_CRn(id)	(((id) >> CRn_shift) & CRn_mask)
+#define sys_reg_CRm(id)	(((id) >> CRm_shift) & CRm_mask)
+#define sys_reg_Op2(id)	(((id) >> Op2_shift) & Op2_mask)
+
+#define SYS_MIDR_EL1			sys_reg(3, 0, 0, 0, 0)
+#define SYS_MPIDR_EL1			sys_reg(3, 0, 0, 0, 5)
+#define SYS_REVIDR_EL1			sys_reg(3, 0, 0, 0, 6)
+
+#define SYS_ID_PFR0_EL1			sys_reg(3, 0, 0, 1, 0)
+#define SYS_ID_PFR1_EL1			sys_reg(3, 0, 0, 1, 1)
+#define SYS_ID_DFR0_EL1			sys_reg(3, 0, 0, 1, 2)
+#define SYS_ID_MMFR0_EL1		sys_reg(3, 0, 0, 1, 4)
+#define SYS_ID_MMFR1_EL1		sys_reg(3, 0, 0, 1, 5)
+#define SYS_ID_MMFR2_EL1		sys_reg(3, 0, 0, 1, 6)
+#define SYS_ID_MMFR3_EL1		sys_reg(3, 0, 0, 1, 7)
+
+#define SYS_ID_ISAR0_EL1		sys_reg(3, 0, 0, 2, 0)
+#define SYS_ID_ISAR1_EL1		sys_reg(3, 0, 0, 2, 1)
+#define SYS_ID_ISAR2_EL1		sys_reg(3, 0, 0, 2, 2)
+#define SYS_ID_ISAR3_EL1		sys_reg(3, 0, 0, 2, 3)
+#define SYS_ID_ISAR4_EL1		sys_reg(3, 0, 0, 2, 4)
+#define SYS_ID_ISAR5_EL1		sys_reg(3, 0, 0, 2, 5)
+#define SYS_ID_MMFR4_EL1		sys_reg(3, 0, 0, 2, 6)
+
+#define SYS_MVFR0_EL1			sys_reg(3, 0, 0, 3, 0)
+#define SYS_MVFR1_EL1			sys_reg(3, 0, 0, 3, 1)
+#define SYS_MVFR2_EL1			sys_reg(3, 0, 0, 3, 2)
+
+#define SYS_ID_AA64PFR0_EL1		sys_reg(3, 0, 0, 4, 0)
+#define SYS_ID_AA64PFR1_EL1		sys_reg(3, 0, 0, 4, 1)
+#define SYS_ID_AA64ZFR0_EL1		sys_reg(3, 0, 0, 4, 4)
+
+#define SYS_ID_AA64DFR0_EL1		sys_reg(3, 0, 0, 5, 0)
+#define SYS_ID_AA64DFR1_EL1		sys_reg(3, 0, 0, 5, 1)
+
+#define SYS_ID_AA64ISAR0_EL1		sys_reg(3, 0, 0, 6, 0)
+#define SYS_ID_AA64ISAR1_EL1		sys_reg(3, 0, 0, 6, 1)
+
+#define SYS_ID_AA64MMFR0_EL1		sys_reg(3, 0, 0, 7, 0)
+#define SYS_ID_AA64MMFR1_EL1		sys_reg(3, 0, 0, 7, 1)
+#define SYS_ID_AA64MMFR2_EL1		sys_reg(3, 0, 0, 7, 2)
+
+#define SYS_ZCR_EL1			sys_reg(3, 0, 1, 2, 0)
+#define SYS_ZCR_EL2			sys_reg(3, 4, 1, 2, 0)
+
+#define SYS_CNTFRQ_EL0			sys_reg(3, 3, 14, 0, 0)
+#define SYS_CTR_EL0			sys_reg(3, 3, 0, 0, 1)
+#define SYS_DCZID_EL0			sys_reg(3, 3, 0, 0, 7)
+
+#define REG_PSTATE_PAN_IMM		sys_reg(0, 0, 4, 0, 4)
+#define REG_PSTATE_UAO_IMM		sys_reg(0, 0, 4, 0, 3)
+
+/*
+#define SET_PSTATE_PAN(x) __inst_arm(0xd5000000 | REG_PSTATE_PAN_IMM | \
+				     (!!x)<<8 | 0x1f)
+#define SET_PSTATE_UAO(x) __inst_arm(0xd5000000 | REG_PSTATE_UAO_IMM | \
+				     (!!x)<<8 | 0x1f)
+*/
+
+/* Common SCTLR_ELx flags. */
+#define SCTLR_ELx_EE    (1 << 25)
+#define SCTLR_ELx_I	(1 << 12)
+#define SCTLR_ELx_SA	(1 << 3)
+#define SCTLR_ELx_C	(1 << 2)
+#define SCTLR_ELx_A	(1 << 1)
+#define SCTLR_ELx_M	1
+
+#define SCTLR_ELx_FLAGS	(SCTLR_ELx_M | SCTLR_ELx_A | SCTLR_ELx_C | \
+			 SCTLR_ELx_SA | SCTLR_ELx_I)
+
+/* SCTLR_EL1 specific flags. */
+#define SCTLR_EL1_UCI		(1 << 26)
+#define SCTLR_EL1_SPAN		(1 << 23)
+#define SCTLR_EL1_UCT		(1 << 15)
+#define SCTLR_EL1_SED		(1 << 8)
+#define SCTLR_EL1_CP15BEN	(1 << 5)
+
+/* id_aa64isar0 */
+#define ID_AA64ISAR0_RDM_SHIFT		28
+#define ID_AA64ISAR0_ATOMICS_SHIFT	20
+#define ID_AA64ISAR0_CRC32_SHIFT	16
+#define ID_AA64ISAR0_SHA2_SHIFT		12
+#define ID_AA64ISAR0_SHA1_SHIFT		8
+#define ID_AA64ISAR0_AES_SHIFT		4
+
+/* id_aa64isar1 */
+#define ID_AA64ISAR1_LRCPC_SHIFT	20
+#define ID_AA64ISAR1_FCMA_SHIFT		16
+#define ID_AA64ISAR1_JSCVT_SHIFT	12
+#define ID_AA64ISAR1_DPB_SHIFT		0
+
+/* id_aa64pfr0 */
+#define ID_AA64PFR0_SVE_SHIFT		32
+#define ID_AA64PFR0_GIC_SHIFT		24
+#define ID_AA64PFR0_ASIMD_SHIFT		20
+#define ID_AA64PFR0_FP_SHIFT		16
+#define ID_AA64PFR0_EL3_SHIFT		12
+#define ID_AA64PFR0_EL2_SHIFT		8
+#define ID_AA64PFR0_EL1_SHIFT		4
+#define ID_AA64PFR0_EL0_SHIFT		0
+
+#define ID_AA64PFR0_SVE			0x1
+#define ID_AA64PFR0_FP_NI		0xf
+#define ID_AA64PFR0_FP_SUPPORTED	0x0
+#define ID_AA64PFR0_ASIMD_NI		0xf
+#define ID_AA64PFR0_ASIMD_SUPPORTED	0x0
+#define ID_AA64PFR0_EL1_64BIT_ONLY	0x1
+#define ID_AA64PFR0_EL0_64BIT_ONLY	0x1
+#define ID_AA64PFR0_EL0_32BIT_64BIT	0x2
+
+/* id_aa64mmfr0 */
+#define ID_AA64MMFR0_TGRAN4_SHIFT	28
+#define ID_AA64MMFR0_TGRAN64_SHIFT	24
+#define ID_AA64MMFR0_TGRAN16_SHIFT	20
+#define ID_AA64MMFR0_BIGENDEL0_SHIFT	16
+#define ID_AA64MMFR0_SNSMEM_SHIFT	12
+#define ID_AA64MMFR0_BIGENDEL_SHIFT	8
+#define ID_AA64MMFR0_ASID_SHIFT		4
+#define ID_AA64MMFR0_PARANGE_SHIFT	0
+
+#define ID_AA64MMFR0_TGRAN4_NI		0xf
+#define ID_AA64MMFR0_TGRAN4_SUPPORTED	0x0
+#define ID_AA64MMFR0_TGRAN64_NI		0xf
+#define ID_AA64MMFR0_TGRAN64_SUPPORTED	0x0
+#define ID_AA64MMFR0_TGRAN16_NI		0x0
+#define ID_AA64MMFR0_TGRAN16_SUPPORTED	0x1
+#define ID_AA64MMFR0_PARANGE_48		0x5
+#define ID_AA64MMFR0_PARANGE_52		0x6
+
+#ifdef CONFIG_ARM64_PA_BITS_52
+#define ID_AA64MMFR0_PARANGE_MAX	ID_AA64MMFR0_PARANGE_52
+#else
+#define ID_AA64MMFR0_PARANGE_MAX	ID_AA64MMFR0_PARANGE_48
+#endif
+
+/* id_aa64mmfr1 */
+#define ID_AA64MMFR1_PAN_SHIFT		20
+#define ID_AA64MMFR1_LOR_SHIFT		16
+#define ID_AA64MMFR1_HPD_SHIFT		12
+#define ID_AA64MMFR1_VHE_SHIFT		8
+#define ID_AA64MMFR1_VMIDBITS_SHIFT	4
+#define ID_AA64MMFR1_HADBS_SHIFT	0
+
+#define ID_AA64MMFR1_VMIDBITS_8		0
+#define ID_AA64MMFR1_VMIDBITS_16	2
+
+/* id_aa64mmfr2 */
+#define ID_AA64MMFR2_LVA_SHIFT		16
+#define ID_AA64MMFR2_IESB_SHIFT		12
+#define ID_AA64MMFR2_LSM_SHIFT		8
+#define ID_AA64MMFR2_UAO_SHIFT		4
+#define ID_AA64MMFR2_CNP_SHIFT		0
+
+/* id_aa64dfr0 */
+#define ID_AA64DFR0_PMSVER_SHIFT	32
+#define ID_AA64DFR0_CTX_CMPS_SHIFT	28
+#define ID_AA64DFR0_WRPS_SHIFT		20
+#define ID_AA64DFR0_BRPS_SHIFT		12
+#define ID_AA64DFR0_PMUVER_SHIFT	8
+#define ID_AA64DFR0_TRACEVER_SHIFT	4
+#define ID_AA64DFR0_DEBUGVER_SHIFT	0
+
+#define ID_ISAR5_RDM_SHIFT		24
+#define ID_ISAR5_CRC32_SHIFT		16
+#define ID_ISAR5_SHA2_SHIFT		12
+#define ID_ISAR5_SHA1_SHIFT		8
+#define ID_ISAR5_AES_SHIFT		4
+#define ID_ISAR5_SEVL_SHIFT		0
+
+#define MVFR0_FPROUND_SHIFT		28
+#define MVFR0_FPSHVEC_SHIFT		24
+#define MVFR0_FPSQRT_SHIFT		20
+#define MVFR0_FPDIVIDE_SHIFT		16
+#define MVFR0_FPTRAP_SHIFT		12
+#define MVFR0_FPDP_SHIFT		8
+#define MVFR0_FPSP_SHIFT		4
+#define MVFR0_SIMD_SHIFT		0
+
+#define MVFR1_SIMDFMAC_SHIFT		28
+#define MVFR1_FPHP_SHIFT		24
+#define MVFR1_SIMDHP_SHIFT		20
+#define MVFR1_SIMDSP_SHIFT		16
+#define MVFR1_SIMDINT_SHIFT		12
+#define MVFR1_SIMDLS_SHIFT		8
+#define MVFR1_FPDNAN_SHIFT		4
+#define MVFR1_FPFTZ_SHIFT		0
+
+#define ID_AA64MMFR0_TGRAN4_SHIFT	28
+#define ID_AA64MMFR0_TGRAN64_SHIFT	24
+#define ID_AA64MMFR0_TGRAN16_SHIFT	20
+
+#define ID_AA64MMFR0_TGRAN4_NI		0xf
+#define ID_AA64MMFR0_TGRAN4_SUPPORTED	0x0
+#define ID_AA64MMFR0_TGRAN64_NI		0xf
+#define ID_AA64MMFR0_TGRAN64_SUPPORTED	0x0
+#define ID_AA64MMFR0_TGRAN16_NI		0x0
+#define ID_AA64MMFR0_TGRAN16_SUPPORTED	0x1
+
+#if defined(CONFIG_ARM64_4K_PAGES)
+#define ID_AA64MMFR0_TGRAN_SHIFT	ID_AA64MMFR0_TGRAN4_SHIFT
+#define ID_AA64MMFR0_TGRAN_SUPPORTED	ID_AA64MMFR0_TGRAN4_SUPPORTED
+#elif defined(CONFIG_ARM64_16K_PAGES)
+#define ID_AA64MMFR0_TGRAN_SHIFT	ID_AA64MMFR0_TGRAN16_SHIFT
+#define ID_AA64MMFR0_TGRAN_SUPPORTED	ID_AA64MMFR0_TGRAN16_SUPPORTED
+#elif defined(CONFIG_ARM64_64K_PAGES)
+#define ID_AA64MMFR0_TGRAN_SHIFT	ID_AA64MMFR0_TGRAN64_SHIFT
+#define ID_AA64MMFR0_TGRAN_SUPPORTED	ID_AA64MMFR0_TGRAN64_SUPPORTED
+#endif
+
+#define ZCR_EL1_LEN_SHIFT	0
+#define ZCR_EL1_LEN_SIZE	9
+#define ZCR_EL1_LEN_MASK	0x1ff
+
+#define CPACR_EL1_ZEN_EL1EN	(1 << 16)
+#define CPACR_EL1_ZEN_EL0EN	(1 << 17)
+#define CPACR_EL1_ZEN		(CPACR_EL1_ZEN_EL1EN | CPACR_EL1_ZEN_EL0EN)
+
+/* Safe value for MPIDR_EL1: Bit31:RES1, Bit30:U:0, Bit24:MT:0 */
+#define SYS_MPIDR_SAFE_VAL		(1UL << 31)
+
+/* SYS_MIDR_EL1 */
+//mask
+#define SYS_MIDR_EL1_IMPLEMENTER_MASK	(0xFFUL)
+#define SYS_MIDR_EL1_PPNUM_MASK		(0xFFFUL)
+//shift
+#define SYS_MIDR_EL1_IMPLEMENTER_SHIFT	(24)
+#define SYS_MIDR_EL1_PPNUM_SHIFT	(0x4)
+//val
+#define SYS_MIDR_EL1_IMPLEMENTER_FJ	(0x46)
+#define SYS_MIDR_EL1_PPNUM_TCHIP	(0x1)
+
+#define READ_ACCESS	(0)
+#define WRITE_ACCESS	(1)
+#define ACCESS_REG_FUNC(name, reg) \
+	static void xos_access_##name(uint8_t flag, uint64_t *reg_value) \
+	{ \
+		if (flag == READ_ACCESS) { \
+			__asm__ __volatile__("mrs_s %0," __stringify(reg) "\n\t" \
+				:"=&r"(*reg_value)::); \
+		} \
+		else if (flag == WRITE_ACCESS) { \
+			__asm__ __volatile__("msr_s" __stringify(reg) ", %0\n\t" \
+				::"r"(*reg_value):); \
+		} else { \
+			; \
+		} \
+	}
+
+#define XOS_FALSE	(0)
+#define XOS_TRUE	(1)
+
+#ifdef __ASSEMBLY__
+#define __emit_inst(x).inst (x)
+	.irp	num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30
+	.equ	.L__reg_num_x\num, \num
+	.endr
+	.equ	.L__reg_num_xzr, 31
+
+	.macro	mrs_s, rt, sreg
+	__emit_inst(0xd5200000|(\sreg)|(.L__reg_num_\rt))
+	.endm
+
+	.macro	msr_s, sreg, rt
+	__emit_inst(0xd5000000|(\sreg)|(.L__reg_num_\rt))
+	.endm
+
+#else
+#define __emit_inst(x)".inst " __stringify((x)) "\n\t"
+asm(
+"	.irp	num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30\n"
+"	.equ	.L__reg_num_x\\num, \\num\n"
+"	.endr\n"
+"	.equ	.L__reg_num_xzr, 31\n"
+"\n"
+"	.macro	mrs_s, rt, sreg\n"
+	__emit_inst(0xd5200000|(\\sreg)|(.L__reg_num_\\rt))
+"	.endm\n"
+"\n"
+"	.macro	msr_s, sreg, rt\n"
+	__emit_inst(0xd5000000|(\\sreg)|(.L__reg_num_\\rt))
+"	.endm\n"
+);
+
+ACCESS_REG_FUNC(midr_el1, SYS_MIDR_EL1);
+static int xos_is_tchip(void)
+{
+	uint64_t reg = 0;
+	int ret = 0, impl = 0, part = 0;
+
+	xos_access_midr_el1(READ_ACCESS, &reg);
+
+	impl = (reg >> SYS_MIDR_EL1_IMPLEMENTER_SHIFT) &
+		SYS_MIDR_EL1_IMPLEMENTER_MASK;
+	part = (reg >> SYS_MIDR_EL1_PPNUM_SHIFT) & SYS_MIDR_EL1_PPNUM_MASK;
+
+	if ((impl == SYS_MIDR_EL1_IMPLEMENTER_FJ) &&
+	    (part == SYS_MIDR_EL1_PPNUM_TCHIP)) {
+		ret = XOS_TRUE;
+	}
+	else {
+		ret = XOS_FALSE;
+	}
+
+	return ret;
+}
+#endif
+
+/*
+ * Unlike read_cpuid, calls to read_sysreg are never expected to be
+ * optimized away or replaced with synthetic values.
+ */
+#define read_sysreg(r) ({						\
+			uint64_t __val;					\
+			asm volatile("mrs %0, " __stringify(r) : "=r" (__val));	\
+			__val;						\
+		})
+
+/*
+ * The "Z" constraint normally means a zero immediate, but when combined with
+ * the "%x0" template means XZR.
+ */
+#define write_sysreg(v, r) do {				\
+	uint64_t __val = (uint64_t)v;			\
+	asm volatile("msr " __stringify(r) ", %x0"	\
+		     : : "rZ" (__val));			\
+} while (0)
+
+/*
+ * For registers without architectural names, or simply unsupported by
+ * GAS.
+ */
+#define read_sysreg_s(r) ({						\
+			uint64_t __val;					\
+			asm volatile("mrs_s %0, " __stringify(r) : "=r" (__val)); \
+			__val;						\
+		})
+
+#define write_sysreg_s(v, r) do {					\
+	uint64_t __val = (uint64_t)v;					\
+	asm volatile("msr_s " __stringify(r) ", %x0" : : "rZ" (__val));	\
+} while (0)
+
+/* @ref.impl arch/arm64/include/asm/kvm_arm.h */
+#define CPTR_EL2_TZ		(1 << 8)
+
+#include "imp-sysreg.h"
+
+#endif	/* __ASM_SYSREG_H */

arch/arm64/kernel/include/thread_info.h

@@ -0,0 +1,107 @@
+/* thread_info.h COPYRIGHT FUJITSU LIMITED 2015-2019 */
+#ifndef __HEADER_ARM64_COMMON_THREAD_INFO_H
+#define __HEADER_ARM64_COMMON_THREAD_INFO_H
+
+#define MIN_KERNEL_STACK_SHIFT	18
+
+#include <arch-memory.h>
+
+#if (MIN_KERNEL_STACK_SHIFT < PAGE_SHIFT)
+#define KERNEL_STACK_SHIFT	PAGE_SHIFT
+#else
+#define KERNEL_STACK_SHIFT	MIN_KERNEL_STACK_SHIFT
+#endif
+
+#define KERNEL_STACK_SIZE	(UL(1) << KERNEL_STACK_SHIFT)
+#define THREAD_START_SP		KERNEL_STACK_SIZE - 16
+
+#ifndef __ASSEMBLY__
+
+#include <process.h>
+#include <prctl.h>
+
+struct cpu_context {
+	unsigned long x19;
+	unsigned long x20;
+	unsigned long x21;
+	unsigned long x22;
+	unsigned long x23;
+	unsigned long x24;
+	unsigned long x25;
+	unsigned long x26;
+	unsigned long x27;
+	unsigned long x28;
+	unsigned long fp;
+	unsigned long sp;
+	unsigned long pc;
+};
+
+struct thread_info {
+	unsigned long		flags;		/* low level flags */
+//	mm_segment_t		addr_limit;	/* address limit */
+//	struct task_struct	*task;		/* main task structure */
+//	struct exec_domain	*exec_domain;	/* execution domain */
+//	struct restart_block	restart_block;
+//	int			preempt_count;	/* 0 => preemptable, <0 => bug */
+	int			cpu;		/* cpu */
+	struct cpu_context	cpu_context;	/* kernel_context */
+	void			*sve_state;	/* SVE registers, if any */
+	unsigned int		sve_vl;		/* SVE vector length */
+	unsigned int		sve_vl_onexec;	/* SVE vl after next exec */
+	unsigned long		sve_flags;	/* SVE related flags */
+	unsigned long		fault_address;	/* fault info */
+	unsigned long		fault_code;	/* ESR_EL1 value */
+};
+
+/* Flags for sve_flags (intentionally defined to match the prctl flags) */
+
+/* Inherit sve_vl and sve_flags across execve(): */
+#define THREAD_VL_INHERIT	PR_SVE_VL_INHERIT
+
+struct arm64_cpu_local_thread {
+	struct thread_info thread_info;
+	unsigned long paniced;
+	uint64_t panic_regs[34];
+};
+
+union arm64_cpu_local_variables {
+	struct arm64_cpu_local_thread arm64_cpu_local_thread;
+	unsigned long stack[KERNEL_STACK_SIZE / sizeof(unsigned long)];
+};
+extern union arm64_cpu_local_variables init_thread_info;
+
+/*
+ * how to get the current stack pointer from C
+ */
+register unsigned long current_stack_pointer asm ("sp");
+
+/*
+ * how to get the thread information struct from C
+ */
+static inline struct thread_info *current_thread_info(void)
+{
+	unsigned long ti = 0;
+
+	ti = ALIGN_DOWN(current_stack_pointer, KERNEL_STACK_SIZE);
+
+	return (struct thread_info *)ti;
+}
+
+/*
+ * how to get the pt_regs struct from C
+ */
+static inline struct pt_regs *current_pt_regs(void)
+{
+	unsigned long regs = 0;
+
+	regs = ALIGN_DOWN(current_stack_pointer, KERNEL_STACK_SIZE);
+	regs += THREAD_START_SP - sizeof(struct pt_regs);
+
+	return (struct pt_regs *)regs;
+}
+
+#endif /* !__ASSEMBLY__ */
+
+#define TIF_SINGLESTEP		21
+
+#endif /* !__HEADER_ARM64_COMMON_THREAD_INFO_H */

arch/arm64/kernel/include/traps.h

@@ -0,0 +1,28 @@
+/* traps.h COPYRIGHT FUJITSU LIMITED 2017 */
+
+#ifndef __ASM_TRAP_H
+#define __ASM_TRAP_H
+
+#include <types.h>
+#include <arch-lock.h>
+
+struct pt_regs;
+
+/* @ref.impl arch/arm64/include/asm/traps.h */
+struct undef_hook {
+	struct list_head node;
+	uint32_t instr_mask;
+	uint32_t instr_val;
+	uint64_t pstate_mask;
+	uint64_t pstate_val;
+	int (*fn)(struct pt_regs *regs, uint32_t instr);
+};
+
+/* @ref.impl arch/arm64/include/asm/traps.h */
+void register_undef_hook(struct undef_hook *hook);
+
+/* @ref.impl arch/arm64/include/asm/traps.h */
+void unregister_undef_hook(struct undef_hook *hook);
+
+#endif /* __ASM_TRAP_H */
+

arch/arm64/kernel/include/vdso.h

@@ -0,0 +1,30 @@
+/* vdso.h COPYRIGHT FUJITSU LIMITED 2016 */
+#ifndef __HEADER_ARM64_COMMON_VDSO_H
+#define __HEADER_ARM64_COMMON_VDSO_H
+
+#ifdef __KERNEL__
+
+/* @ref.impl arch/arm64/include/asm/vsdo.h::VDSO_LBASE */
+/*
+ * Default link address for the vDSO.
+ * Since we randomise the VDSO mapping, there's little point in trying
+ * to prelink this.
+ */
+#define VDSO_LBASE	0x0
+
+#ifndef __ASSEMBLY__
+
+#include <vdso-offsets.h>
+
+/* @ref.impl arch/arm64/include/asm/vsdo.h::VDSO_SYMBOL */
+#define VDSO_SYMBOL(base, name)		vdso_symbol_##name((unsigned long)(base))
+void* vdso_symbol_sigtramp(unsigned long base);
+
+int add_vdso_pages(struct thread *thread);
+
+#endif /* !__ASSEMBLY__ */
+
+#endif /* __KERNEL__ */
+
+#endif /* !__HEADER_ARM64_COMMON_VDSO_H */
+

arch/arm64/kernel/include/virt.h

@@ -0,0 +1,22 @@
+/* virt.h COPYRIGHT FUJITSU LIMITED 2015-2017 */
+#ifndef __HEADER_ARM64_COMMON_VIRT_H
+#define __HEADER_ARM64_COMMON_VIRT_H
+
+/* @ref.impl linux-v4.15-rc3 arch/arm64/include/asm/virt.h */
+#define BOOT_CPU_MODE_EL1	(0xe11)
+#define BOOT_CPU_MODE_EL2	(0xe12)
+
+#ifndef __ASSEMBLY__
+
+#include <sysreg.h>
+#include <ptrace.h>
+
+/* @ref.impl linux-v4.15-rc3 arch/arm64/include/asm/virt.h */
+static inline int is_kernel_in_hyp_mode(void)
+{
+	return read_sysreg(CurrentEL) == CurrentEL_EL2;
+}
+
+#endif /* !__ASSEMBLY__ */
+
+#endif /* !__HEADER_ARM64_COMMON_VIRT_H */

arch/arm64/kernel/irq-gic-v2.c

@@ -0,0 +1,176 @@
+/* irq-gic-v2.c COPYRIGHT FUJITSU LIMITED 2015-2018 */
+#include <ihk/cpu.h>
+#include <irq.h>
+#include <arm-gic-v2.h>
+#include <io.h>
+#include <arch/cpu.h>
+#include <memory.h>
+#include <affinity.h>
+#include <syscall.h>
+#include <ihk/debug.h>
+#include <arch-timer.h>
+#include <cls.h>
+
+// #define DEBUG_GICV2
+
+#ifdef DEBUG_GICV2
+#undef DDEBUG_DEFAULT
+#define DDEBUG_DEFAULT DDEBUG_PRINT
+#endif
+
+void *dist_base;
+void *cpu_base;
+
+#define gic_hwid_to_affinity(hw_cpuid)	(1UL << hw_cpuid)
+
+/**
+ * arm64_raise_sgi_gicv2
+ * @ref.impl drivers/irqchip/irq-gic.c:gic_raise_softirq
+ *
+ * @note Because it performs interrupt control at a higher 
+ * function, it is not necessary to perform the disable/enable
+ * interrupts in this function as gic_raise_softirq() .
+ */
+static void __arm64_raise_sgi_gicv2(unsigned int hw_cpuid, unsigned int vector)
+{
+	/* Build interrupt destination of the target cpu */
+	uint8_t cpu_target_list = gic_hwid_to_affinity(hw_cpuid);
+
+	/*
+	 * Ensure that stores to Normal memory are visible to the
+	 * other CPUs before they observe us issuing the IPI.
+	 */
+	dmb(ishst);
+
+	/* write to GICD_SGIR */
+	writel_relaxed(
+		cpu_target_list << 16 | vector, 
+		(void *)(dist_base + GIC_DIST_SOFTINT)
+	);
+}
+
+static void arm64_raise_sgi_gicv2(uint32_t cpuid, uint32_t vector)
+{
+	/* Build interrupt destination of the target CPU */
+	uint32_t hw_cpuid = ihk_mc_get_cpu_info()->hw_ids[cpuid];
+
+	__arm64_raise_sgi_gicv2(hw_cpuid, vector);
+}
+
+static void arm64_raise_sgi_to_host_gicv2(uint32_t cpuid, uint32_t vector)
+{
+	/* Build interrupt destination of the target Linux/host CPU */
+	uint32_t hw_cpuid = ihk_mc_get_apicid(cpuid);
+
+	__arm64_raise_sgi_gicv2(hw_cpuid, vector);
+}
+
+
+/**
+ * arm64_raise_spi_gicv2
+ * @ref.impl nothing.
+ */
+static void arm64_raise_spi_gicv2(unsigned int cpuid, unsigned int vector)
+{
+	uint64_t spi_reg_offset;
+	uint32_t spi_set_pending_bitpos;
+
+	/**
+	 * calculates register offset and bit position corresponding to the numbers.
+	 *
+	 * For interrupt vector m, 
+	 * - the corresponding GICD_ISPENDR number, n, is given by n = m / 32 
+	 * - the offset of the required GICD_ISPENDR is (0x200 + (4*n)) 
+	 * - the bit number of the required Set-pending bit in this register is m % 32.
+	 */
+	spi_reg_offset = vector / 32 * 4;
+	spi_set_pending_bitpos = vector % 32;
+
+	/* write to GICD_ISPENDR */
+	writel_relaxed(
+		1 << spi_set_pending_bitpos, 
+		(void *)(dist_base + GIC_DIST_PENDING_SET + spi_reg_offset)
+	);
+}
+
+void arm64_issue_host_ipi_gicv2(uint32_t cpuid, uint32_t vector)
+{
+	arm64_raise_sgi_to_host_gicv2(cpuid, vector);
+}
+
+/**
+ * arm64_issue_ipi_gicv2
+ * @param cpuid : hardware cpu id
+ * @param vector : interrupt vector number
+ */
+void arm64_issue_ipi_gicv2(unsigned int cpuid, unsigned int vector)
+{
+	dkprintf("Send irq#%d to cpuid=%d\n", vector, cpuid);
+
+	if(vector < 16){
+		// send SGI
+		arm64_raise_sgi_gicv2(cpuid, vector);
+	} else if (32 <= vector && vector < 1020) {
+		// send SPI (allow only to host)
+		arm64_raise_spi_gicv2(cpuid, vector);
+	} else {
+		ekprintf("#%d is bad irq number.", vector);
+	}
+}
+
+/**
+ * handle_interrupt_gicv2
+ * @ref.impl drivers/irqchip/irq-gic.c:gic_handle_irq
+ */
+extern int interrupt_from_user(void *);
+void handle_interrupt_gicv2(struct pt_regs *regs)
+{
+	unsigned int irqstat, irqnr;
+	const int from_user = interrupt_from_user(regs);
+
+	set_cputime(from_user ? CPUTIME_MODE_U2K : CPUTIME_MODE_K2K_IN);
+	do {
+		// get GICC_IAR.InterruptID
+		irqstat = readl_relaxed(cpu_base + GIC_CPU_INTACK);
+		irqnr = irqstat & GICC_IAR_INT_ID_MASK;
+
+		if (irqnr < 32) {
+			writel_relaxed(irqstat, cpu_base + GIC_CPU_EOI);
+			handle_IPI(irqnr, regs);
+			continue;
+		} else if (irqnr != 1023) {
+			panic("PANIC: handle_interrupt_gicv2(): catch invalid interrupt.");
+		}
+
+		/*
+		 * If another interrupt is not pending, GICC_IAR.InterruptID
+		 * returns 1023 (see GICv2 spec. Chap. 4.4.4) .
+		 */
+		break;
+	} while (1);
+	set_cputime(from_user ? CPUTIME_MODE_K2U : CPUTIME_MODE_K2K_OUT);
+
+	/* for migration by IPI */
+	if (get_this_cpu_local_var()->flags & CPU_FLAG_NEED_MIGRATE) {
+		schedule();
+		check_signal(0, regs, 0);
+	}
+}
+
+void gic_dist_init_gicv2(unsigned long dist_base_pa, unsigned long size)
+{
+	dist_base = map_fixed_area(dist_base_pa, size, 1 /*non chachable*/);
+}
+
+void gic_cpu_init_gicv2(unsigned long cpu_base_pa, unsigned long size)
+{
+	cpu_base = map_fixed_area(cpu_base_pa, size, 1 /*non chachable*/);
+}
+
+void gic_enable_gicv2(void)
+{
+	unsigned int enable_ppi_sgi = 0;
+
+	enable_ppi_sgi |= GICD_ENABLE << get_timer_intrid();
+	writel_relaxed(enable_ppi_sgi, dist_base + GIC_DIST_ENABLE_SET);
+}

arch/arm64/kernel/irq-gic-v3.c

@@ -0,0 +1,538 @@
+/* irq-gic-v3.c COPYRIGHT FUJITSU LIMITED 2015-2018 */
+#include <irq.h>
+#include <arm-gic-v2.h>
+#include <arm-gic-v3.h>
+#include <io.h>
+#include <cputype.h>
+#include <process.h>
+#include <syscall.h>
+#include <ihk/debug.h>
+#ifdef ENABLE_FUGAKU_HACKS
+#include <ihk/monitor.h>
+#endif
+#include <arch-timer.h>
+#include <cls.h>
+
+//#define DEBUG_GICV3
+
+#define USE_CAVIUM_THUNDER_X
+
+#ifdef DEBUG_GICV3
+#undef DDEBUG_DEFAULT
+#define DDEBUG_DEFAULT DDEBUG_PRINT
+#endif
+
+#ifdef USE_CAVIUM_THUNDER_X
+static char is_cavium_thunderx = 0;
+#endif
+
+void *dist_base;
+void *rdist_base[NR_CPUS];
+
+extern uint64_t ihk_param_cpu_logical_map;
+static uint64_t *__cpu_logical_map = &ihk_param_cpu_logical_map;
+
+extern uint64_t ihk_param_gic_rdist_base_pa[NR_CPUS];
+
+#define cpu_logical_map(cpu)    __cpu_logical_map[cpu]
+
+/* Our default, arbitrary priority value. Linux only uses one anyway. */
+#define DEFAULT_PMR_VALUE	0xf0
+
+/**
+ * Low level accessors
+ * @ref.impl host-kernel/drivers/irqchip/irq-gic-v3.c
+ */
+static uint64_t gic_read_iar_common(void)
+{
+	uint64_t irqstat;
+
+#ifdef CONFIG_HAS_NMI
+	uint64_t daif;
+	uint64_t pmr;
+	uint64_t default_pmr_value = DEFAULT_PMR_VALUE;
+
+	/*
+	 * The PMR may be configured to mask interrupts when this code is
+	 * called, thus in order to acknowledge interrupts we must set the
+	 * PMR to its default value before reading from the IAR.
+	 *
+	 * To do this without taking an interrupt we also ensure the I bit
+	 * is set whilst we are interfering with the value of the PMR.
+	 */
+	asm volatile(
+		"mrs  %1, daif\n\t"				/* save I bit  */
+		"msr  daifset, #2\n\t"				/* set I bit   */
+		"mrs_s  %2, " __stringify(ICC_PMR_EL1) "\n\t"	/* save PMR    */
+		"msr_s  " __stringify(ICC_PMR_EL1) ",%3\n\t"	/* set PMR     */
+		"mrs_s  %0, " __stringify(ICC_IAR1_EL1) "\n\t"	/* ack int   */
+		"msr_s  " __stringify(ICC_PMR_EL1) ",%2\n\t"	/* restore PMR */
+		"isb\n\t"
+		"msr  daif, %1"					/* restore I   */
+		: "=r" (irqstat), "=&r" (daif), "=&r" (pmr)
+		: "r" (default_pmr_value));
+#else /* CONFIG_HAS_NMI */
+	asm volatile("mrs_s %0, " __stringify(ICC_IAR1_EL1) : "=r" (irqstat));
+#endif /* CONFIG_HAS_NMI */
+
+	return irqstat;
+}
+
+#ifdef USE_CAVIUM_THUNDER_X
+/* Cavium ThunderX erratum 23154 */
+static uint64_t gic_read_iar_cavium_thunderx(void)
+{
+	uint64_t irqstat;
+
+#ifdef CONFIG_HAS_NMI
+	uint64_t daif;
+	uint64_t pmr;
+	uint64_t default_pmr_value = DEFAULT_PMR_VALUE;
+
+	/*
+	 * The PMR may be configured to mask interrupts when this code is
+	 * called, thus in order to acknowledge interrupts we must set the
+	 * PMR to its default value before reading from the IAR.
+	 *
+	 * To do this without taking an interrupt we also ensure the I bit
+	 * is set whilst we are interfering with the value of the PMR.
+	 */
+	asm volatile(
+		"mrs  %1, daif\n\t"				/* save I bit  */
+		"msr  daifset, #2\n\t"				/* set I bit   */
+		"mrs_s  %2, " __stringify(ICC_PMR_EL1) "\n\t"	/* save PMR    */
+		"msr_s  " __stringify(ICC_PMR_EL1) ",%3\n\t"	/* set PMR     */
+		"nop;nop;nop;nop\n\t"
+		"nop;nop;nop;nop\n\t"
+		"mrs_s  %0, " __stringify(ICC_IAR1_EL1) "\n\t"	/* ack int   */
+		"nop;nop;nop;nop\n\t"
+		"msr_s  " __stringify(ICC_PMR_EL1) ",%2\n\t"	/* restore PMR */
+		"isb\n\t"
+		"msr  daif, %1"					/* restore I   */
+		: "=r" (irqstat), "=&r" (daif), "=&r" (pmr)
+		: "r" (default_pmr_value));
+#else /* CONFIG_HAS_NMI */
+	asm volatile("nop;nop;nop;nop;");
+	asm volatile("nop;nop;nop;nop;");
+	asm volatile("mrs_s %0, " __stringify(ICC_IAR1_EL1) : "=r" (irqstat));
+	asm volatile("nop;nop;nop;nop;");
+#endif /* CONFIG_HAS_NMI */
+	mb();
+
+	return irqstat;
+}
+#endif
+
+static uint64_t gic_read_iar(void)
+{
+#ifdef USE_CAVIUM_THUNDER_X
+	if (is_cavium_thunderx)
+		return gic_read_iar_cavium_thunderx();
+	else
+#endif
+		return gic_read_iar_common();
+
+}
+
+static void gic_write_pmr(uint64_t val)
+{
+	asm volatile("msr_s " __stringify(ICC_PMR_EL1) ", %0" : : "r" (val));
+}
+
+static void gic_write_ctlr(uint64_t val)
+{
+	asm volatile("msr_s " __stringify(ICC_CTLR_EL1) ", %0" : : "r" (val));
+	isb();
+}
+
+static void gic_write_grpen1(uint64_t val)
+{
+	asm volatile("msr_s " __stringify(ICC_GRPEN1_EL1) ", %0" : : "r" (val));
+	isb();
+}
+
+static inline void gic_write_eoir(uint64_t irq)
+{
+	asm volatile("msr_s " __stringify(ICC_EOIR1_EL1) ", %0" : : "r" (irq));
+	isb();
+}
+
+static void gic_write_sgi1r(uint64_t val)
+{
+	asm volatile("msr_s " __stringify(ICC_SGI1R_EL1) ", %0" : : "r" (val));
+}
+
+static inline uint32_t gic_read_sre(void)
+{
+	uint64_t val;
+
+	asm volatile("mrs_s %0, " __stringify(ICC_SRE_EL1) : "=r" (val));
+	return val;
+}
+
+static inline void gic_write_sre(uint32_t val)
+{
+	asm volatile("msr_s " __stringify(ICC_SRE_EL1) ", %0" : : "r" ((uint64_t)val));
+	isb();
+}
+
+static uint32_t gic_enable_sre(void)
+{
+	uint32_t val;
+
+	val = gic_read_sre();
+	if (val & ICC_SRE_EL1_SRE)
+		return 1; /*ok*/
+
+	val |= ICC_SRE_EL1_SRE;
+	gic_write_sre(val);
+	val = gic_read_sre();
+
+	return !!(val & ICC_SRE_EL1_SRE);
+}
+
+#ifdef CONFIG_HAS_NMI
+static inline void gic_write_bpr1(uint32_t val)
+{
+	asm volatile("msr_s " __stringify(ICC_BPR1_EL1) ", %0" : : "r" (val));
+}
+#endif
+
+static void __arm64_raise_sgi_gicv3(uint32_t hw_cpuid, uint32_t vector)
+{
+	uint64_t mpidr, cluster_id;
+	uint16_t tlist;
+	uint64_t val;
+
+	/*
+	 * Ensure that stores to Normal memory are visible to the
+	 * other CPUs before issuing the IPI.
+	 */
+	smp_wmb();
+
+	mpidr = cpu_logical_map(hw_cpuid);
+	if((mpidr & 0xffUL) < 16) {
+		cluster_id = cpu_logical_map(hw_cpuid) & ~0xffUL;
+		tlist = (uint16_t)(1 << (mpidr & 0xf));
+	
+#define MPIDR_TO_SGI_AFFINITY(cluster_id, level) \
+	(MPIDR_AFFINITY_LEVEL(cluster_id, level) \
+		<< ICC_SGI1R_AFFINITY_## level ##_SHIFT)
+
+		val = (MPIDR_TO_SGI_AFFINITY(cluster_id, 3)	|
+		       MPIDR_TO_SGI_AFFINITY(cluster_id, 2)	|
+		       vector << ICC_SGI1R_SGI_ID_SHIFT		|
+		       MPIDR_TO_SGI_AFFINITY(cluster_id, 1)	|
+		       tlist << ICC_SGI1R_TARGET_LIST_SHIFT);
+	
+		dkprintf("CPU%d: ICC_SGI1R_EL1 %llx\n", ihk_mc_get_processor_id(), val);
+		gic_write_sgi1r(val);
+
+		/* Force the above writes to ICC_SGI1R_EL1 to be executed */
+		isb();
+	}  else {
+		/*
+		 * If we ever get a cluster of more than 16 CPUs, just
+		 * scream and skip that CPU.
+		 */
+		ekprintf("GICv3 can't send SGI for TargetList=%d\n", (mpidr & 0xffUL));
+	}
+}
+
+static void arm64_raise_sgi_gicv3(uint32_t cpuid, uint32_t vector)
+{
+	/* Build interrupt destination of the target CPU */
+	uint32_t hw_cpuid = ihk_mc_get_cpu_info()->hw_ids[cpuid];
+
+	__arm64_raise_sgi_gicv3(hw_cpuid, vector);
+}
+
+static void arm64_raise_sgi_to_host_gicv3(uint32_t cpuid, uint32_t vector)
+{
+	/* Build interrupt destination of the target Linux/host CPU */
+	uint32_t hw_cpuid = ihk_mc_get_apicid(cpuid);
+
+	__arm64_raise_sgi_gicv3(hw_cpuid, vector);
+}
+
+static void arm64_raise_spi_gicv3(uint32_t cpuid, uint32_t vector)
+{
+	uint64_t spi_reg_offset;
+	uint32_t spi_set_pending_bitpos;
+
+	/**
+	 * calculates register offset and bit position corresponding to the numbers.
+	 *
+	 * For interrupt vector m, 
+	 * - the corresponding GICD_ISPENDR number, n, is given by n = m / 32 
+	 * - the offset of the required GICD_ISPENDR is (0x200 + (4*n)) 
+	 * - the bit number of the required Set-pending bit in this register is m % 32.
+	 */
+	spi_reg_offset = vector / 32 * 4;
+	spi_set_pending_bitpos = vector % 32;
+
+	/* write to GICD_ISPENDR */
+	writel_relaxed(
+		1 << spi_set_pending_bitpos, 
+		(void *)(dist_base + GICD_ISPENDR + spi_reg_offset)
+	);
+}
+
+static void arm64_raise_lpi_gicv3(uint32_t cpuid, uint32_t vector)
+{
+	// @todo.impl
+	ekprintf("%s called.\n", __func__);
+}
+ 
+void arm64_issue_host_ipi_gicv3(uint32_t cpuid, uint32_t vector)
+{
+	arm64_raise_sgi_to_host_gicv3(cpuid, vector);
+}
+
+void arm64_issue_ipi_gicv3(uint32_t cpuid, uint32_t vector)
+{
+	dkprintf("Send irq#%d to cpuid=%d\n", vector, cpuid);
+
+	barrier();
+	if(vector < 16){
+		// send SGI
+		arm64_raise_sgi_gicv3(cpuid, vector);
+	} else if (32 <= vector && vector < 1020) {
+		// send SPI (allow only to host)
+		arm64_raise_spi_gicv3(cpuid, vector);
+	} else if (8192 <= vector) {
+		// send LPI (allow only to host)
+		arm64_raise_lpi_gicv3(cpuid, vector);
+	} else {
+		ekprintf("#%d is bad irq number.\n", vector);
+	}
+}
+
+extern int interrupt_from_user(void *);
+void handle_interrupt_gicv3(struct pt_regs *regs)
+{
+	uint64_t irqnr;
+	const int from_user = interrupt_from_user(regs);
+	struct cpu_local_var *v = get_this_cpu_local_var();
+	//unsigned long irqflags;
+	int do_check = 0;
+#ifdef ENABLE_FUGAKU_HACKS
+	struct ihk_os_cpu_monitor *monitor = cpu_local_var(monitor);
+
+	++v->in_interrupt;
+#endif
+	irqnr = gic_read_iar();
+	cpu_enable_nmi();
+	set_cputime(from_user ? CPUTIME_MODE_U2K : CPUTIME_MODE_K2K_IN);
+	while (irqnr != ICC_IAR1_EL1_SPURIOUS) {
+		if ((irqnr < 1020) || (irqnr >= 8192)) {
+			gic_write_eoir(irqnr);
+#ifndef ENABLE_FUGAKU_HACKS
+			handle_IPI(irqnr, regs);
+#else
+			/* Once paniced, only allow CPU stop and NMI IRQs */
+			if (monitor->status != IHK_OS_MONITOR_PANIC ||
+					irqnr == INTRID_CPU_STOP ||
+					irqnr == INTRID_MULTI_NMI) {
+				handle_IPI(irqnr, regs);
+			}
+#endif
+		}
+		irqnr = gic_read_iar();
+	}
+	set_cputime(from_user ? CPUTIME_MODE_K2U : CPUTIME_MODE_K2K_OUT);
+
+	//irqflags = ihk_mc_spinlock_lock(&v->runq_lock);
+	/* For migration by IPI or by timesharing */
+	if (v->flags & CPU_FLAG_NEED_RESCHED) {
+		if (v->flags & CPU_FLAG_NEED_MIGRATE && !from_user) {
+			// Don't migrate on K2K schedule
+		} else {
+			v->flags &= ~CPU_FLAG_NEED_RESCHED;
+			do_check = 1;
+		}
+	}
+	//ihk_mc_spinlock_unlock(&v->runq_lock, irqflags);
+
+#ifndef ENABLE_FUGAKU_HACKS
+	if (do_check) {
+#else
+	--v->in_interrupt;
+	if (monitor->status != IHK_OS_MONITOR_PANIC && do_check) {
+#endif
+		check_signal(0, regs, 0);
+		schedule();
+	}
+}
+
+static uint64_t gic_mpidr_to_affinity(unsigned long mpidr)
+{
+	uint64_t aff;
+
+	aff = ((uint64_t)MPIDR_AFFINITY_LEVEL(mpidr, 3) << 32 |
+			 MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
+			 MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8  |
+			 MPIDR_AFFINITY_LEVEL(mpidr, 0));
+	return aff;
+}
+
+static void init_spi_routing(uint32_t irq, uint32_t linux_cpu)
+{
+	uint64_t spi_route_reg_val, spi_route_reg_offset;
+
+	if (irq < 32 || 1020 <= irq) {
+		ekprintf("%s: irq is not spi number. (irq=%d)\n",
+			 __func__, irq);
+		return;
+	}
+
+	/* write to GICD_IROUTER */
+	spi_route_reg_offset = irq * 8;
+	spi_route_reg_val = gic_mpidr_to_affinity(cpu_logical_map(linux_cpu));
+
+	writeq_relaxed(spi_route_reg_val,
+		       (void *)(dist_base + GICD_IROUTER +
+				spi_route_reg_offset));
+}
+
+void gic_dist_init_gicv3(unsigned long dist_base_pa, unsigned long size)
+{
+#ifndef IHK_IKC_USE_LINUX_WORK_IRQ
+	extern int spi_table[];
+	extern int nr_spi_table;
+	int i;
+#endif // !IHK_IKC_USE_LINUX_WORK_IRQ
+
+	dist_base = map_fixed_area(dist_base_pa, size, 1 /*non chachable*/);
+
+#ifdef USE_CAVIUM_THUNDER_X
+	/* Cavium ThunderX erratum 23154 */
+	if (MIDR_IMPLEMENTOR(read_cpuid_id()) == ARM_CPU_IMP_CAVIUM) {
+		is_cavium_thunderx = 1;
+	}
+#endif
+
+#ifndef IHK_IKC_USE_LINUX_WORK_IRQ
+	/* initialize spi routing */
+	for (i = 0; i < nr_spi_table; i++) {
+		if (spi_table[i] == -1) {
+			continue;
+		}
+		init_spi_routing(spi_table[i], i);
+	}
+#endif // !IHK_IKC_USE_LINUX_WORK_IRQ
+}
+
+void gic_cpu_init_gicv3(unsigned long cpu_base_pa, unsigned long size)
+{
+	int32_t cpuid, hw_cpuid;
+	struct ihk_mc_cpu_info *cpu_info = ihk_mc_get_cpu_info();
+
+	for(cpuid = 0; cpuid < cpu_info->ncpus; cpuid++) {
+		hw_cpuid = cpu_info->hw_ids[cpuid];
+		if(ihk_param_gic_rdist_base_pa[hw_cpuid] != 0) {
+			rdist_base[hw_cpuid] = 
+				map_fixed_area(ihk_param_gic_rdist_base_pa[hw_cpuid], size, 1 /*non chachable*/);
+		}
+	}
+}
+
+static void gic_do_wait_for_rwp(void *base)
+{
+	uint32_t count = 1000000;	/* 1s! */
+
+	while (readl_relaxed(base + GICD_CTLR) & GICD_CTLR_RWP) {
+		count--;
+		if (!count) {
+			ekprintf("RWP timeout, gone fishing\n");
+			return;
+		}
+		cpu_pause();
+	};
+}
+
+void gic_enable_gicv3(void)
+{
+	void *rbase = rdist_base[ihk_mc_get_hardware_processor_id()];
+	void *rd_sgi_base = rbase + 0x10000 /* SZ_64K */;
+	int i;
+	unsigned int enable_ppi_sgi = GICD_INT_EN_SET_SGI;
+	extern int ihk_param_nr_pmu_irq_affi;
+	extern int ihk_param_pmu_irq_affi[CONFIG_SMP_MAX_CORES];
+
+	enable_ppi_sgi |= GICD_ENABLE << get_timer_intrid();
+
+	if (0 < ihk_param_nr_pmu_irq_affi) {
+		for (i = 0; i < ihk_param_nr_pmu_irq_affi; i++) {
+			if ((0 <= ihk_param_pmu_irq_affi[i]) &&
+			    (ihk_param_pmu_irq_affi[i] <
+			     sizeof(enable_ppi_sgi) * BITS_PER_BYTE)) {
+				enable_ppi_sgi |= GICD_ENABLE <<
+					ihk_param_pmu_irq_affi[i];
+			}
+		}
+	}
+	else {
+		enable_ppi_sgi |= GICD_ENABLE << INTRID_PERF_OVF;
+	}
+
+	/*
+	 * Deal with the banked PPI and SGI interrupts - disable all
+	 * PPI interrupts, ensure all SGI interrupts are enabled.
+	 */
+	writel_relaxed(~enable_ppi_sgi, rd_sgi_base + GIC_DIST_ENABLE_CLEAR);
+	writel_relaxed(enable_ppi_sgi, rd_sgi_base + GIC_DIST_ENABLE_SET);
+
+	/*
+	 * Set priority on PPI and SGI interrupts
+	 */
+	for (i = 0; i < 32; i += 4) {
+		writel_relaxed(GICD_INT_DEF_PRI_X4,
+			       rd_sgi_base + GIC_DIST_PRI + i);
+	}
+
+	/* sync wait */
+	gic_do_wait_for_rwp(rbase);
+
+	/*
+	 * Need to check that the SRE bit has actually been set. If
+	 * not, it means that SRE is disabled at EL2. We're going to
+	 * die painfully, and there is nothing we can do about it.
+	 *
+	 * Kindly inform the luser.
+	 */
+	if (!gic_enable_sre())
+		panic("GIC: unable to set SRE (disabled at EL2), panic ahead\n");
+
+#ifndef CONFIG_HAS_NMI
+	/* Set priority mask register */
+	gic_write_pmr(DEFAULT_PMR_VALUE);
+#endif
+	
+	/* EOI deactivates interrupt too (mode 0) */
+	gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop_dir);
+
+	/* ... and let's hit the road... */
+	gic_write_grpen1(1);
+
+#ifdef CONFIG_HAS_NMI
+	/*
+	 * Some firmwares hand over to the kernel with the BPR changed from
+	 * its reset value (and with a value large enough to prevent
+	 * any pre-emptive interrupts from working at all). Writing a zero
+	 * to BPR restores is reset value.
+	 */
+	gic_write_bpr1(0);
+
+	/* Set specific IPI to NMI */
+	writeb_relaxed(GICD_INT_NMI_PRI,
+		       rd_sgi_base + GIC_DIST_PRI + INTRID_CPU_STOP);
+	writeb_relaxed(GICD_INT_NMI_PRI,
+		       rd_sgi_base + GIC_DIST_PRI + INTRID_MULTI_NMI);
+	writeb_relaxed(GICD_INT_NMI_PRI,
+		       rd_sgi_base + GIC_DIST_PRI + INTRID_STACK_TRACE);
+
+	/* sync wait */
+	gic_do_wait_for_rwp(rbase);
+#endif /* CONFIG_HAS_NMI */
+}

arch/arm64/kernel/local.c

@@ -0,0 +1,95 @@
+/* local.c COPYRIGHT FUJITSU LIMITED 2015-2018 */
+#include <cpulocal.h>
+#include <ihk/atomic.h>
+#include <ihk/mm.h>
+#include <ihk/cpu.h>
+#include <ihk/debug.h>
+#include <registers.h>
+#include <string.h>
+
+/* BSP initialized stack area */
+union arm64_cpu_local_variables init_thread_info __attribute__((aligned(KERNEL_STACK_SIZE)));
+
+/* BSP/AP idle stack pointer head */
+static union arm64_cpu_local_variables *locals;
+size_t arm64_cpu_local_variables_span = KERNEL_STACK_SIZE; /* for debugger */
+
+/* allocate & initialize BSP/AP idle stack */
+void init_processors_local(int max_id)
+{
+	int i = 0;
+	union arm64_cpu_local_variables *tmp;
+	const int npages = ((max_id + 1) *
+			    (ALIGN_UP(KERNEL_STACK_SIZE, PAGE_SIZE) >>
+			     PAGE_SHIFT));
+
+	if (npages < 1) {
+		panic("idle kernel stack allocation failed.");
+	}
+
+	/* allocate one more for alignment */
+	locals = ihk_mc_alloc_pages(npages, IHK_MC_AP_CRITICAL);
+	if (locals == NULL) {
+		panic("idle kernel stack allocation failed.");
+	}
+	locals = (union arm64_cpu_local_variables *)ALIGN_UP((unsigned long)locals, KERNEL_STACK_SIZE);
+
+	/* clear struct process, struct process_vm, struct thread_info area */
+	for (i = 0, tmp = locals; i < max_id; i++, tmp++) {
+		memset(tmp, 0, sizeof(struct thread_info));
+	}
+	kprintf("locals = %p\n", locals);
+}
+
+/* get id (logical processor id) local variable address */
+union arm64_cpu_local_variables *get_arm64_cpu_local_variable(int id)
+{
+	return locals + id;
+}
+
+/* get id (logical processor id) kernel stack address */
+static void *get_arm64_cpu_local_kstack(int id)
+{
+	return (char *)get_arm64_cpu_local_variable(id) + THREAD_START_SP;
+}
+
+/* get current cpu local variable address */
+union arm64_cpu_local_variables *get_arm64_this_cpu_local(void)
+{
+	int id = ihk_mc_get_processor_id();
+	return get_arm64_cpu_local_variable(id);
+}
+
+/* get current kernel stack address */
+void *get_arm64_this_cpu_kstack(void)
+{
+	int id = ihk_mc_get_processor_id();
+	return get_arm64_cpu_local_kstack(id);
+}
+
+/* assign logical processor id for current_thread_info.cpu */
+/* logical processor id BSP:0, AP0:1, AP1:2, ... APn:n-1 */
+static ihk_atomic_t last_processor_id = IHK_ATOMIC_INIT(-1);
+void assign_processor_id(void)
+{
+	int id;
+	union arm64_cpu_local_variables *v;
+
+	id = ihk_atomic_inc_return(&last_processor_id);
+
+	v = get_arm64_cpu_local_variable(id);
+	v->arm64_cpu_local_thread.thread_info.cpu = id;
+}
+
+/** IHK **/
+/* get current logical processor id */
+int ihk_mc_get_processor_id(void)
+{
+	return current_thread_info()->cpu;
+}
+
+/* get current physical processor id (not equal AFFINITY !!) */
+int ihk_mc_get_hardware_processor_id(void)
+{
+	return ihk_mc_get_cpu_info()->hw_ids[ihk_mc_get_processor_id()];
+}

arch/arm64/kernel/memcpy.S

@@ -0,0 +1,78 @@
+/* memcpy.S COPYRIGHT FUJITSU LIMITED 2017 */
+/*
+ * Copyright (C) 2013 ARM Ltd.
+ * Copyright (C) 2013 Linaro.
+ *
+ * This code is based on glibc cortex strings work originally authored by Linaro
+ * and re-licensed under GPLv2 for the Linux kernel. The original code can
+ * be found @
+ *
+ * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/
+ * files/head:/src/aarch64/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linkage.h>
+#include <assembler.h>
+#include <cache.h>
+
+/*
+ * Copy a buffer from src to dest (alignment handled by the hardware)
+ *
+ * Parameters:
+ *	x0 - dest
+ *	x1 - src
+ *	x2 - n
+ * Returns:
+ *	x0 - dest
+ */
+	.macro ldrb1 ptr, regB, val
+	ldrb  \ptr, [\regB], \val
+	.endm
+
+	.macro strb1 ptr, regB, val
+	strb \ptr, [\regB], \val
+	.endm
+
+	.macro ldrh1 ptr, regB, val
+	ldrh  \ptr, [\regB], \val
+	.endm
+
+	.macro strh1 ptr, regB, val
+	strh \ptr, [\regB], \val
+	.endm
+
+	.macro ldr1 ptr, regB, val
+	ldr \ptr, [\regB], \val
+	.endm
+
+	.macro str1 ptr, regB, val
+	str \ptr, [\regB], \val
+	.endm
+
+	.macro ldp1 ptr, regB, regC, val
+	ldp \ptr, \regB, [\regC], \val
+	.endm
+
+	.macro stp1 ptr, regB, regC, val
+	stp \ptr, \regB, [\regC], \val
+	.endm
+
+	.weak memcpy
+ENTRY(____inline_memcpy)
+ENTRY(__inline_memcpy)
+#include "copy_template.S"
+	ret
+ENDPIPROC(__inline_memcpy)
+ENDPROC(____inline_memcpy)

arch/arm64/kernel/memset.S

@@ -0,0 +1,258 @@
+/* memset.S COPYRIGHT FUJITSU LIMITED 2017 */
+/*
+ * Copyright (C) 2013 ARM Ltd.
+ * Copyright (C) 2013 Linaro.
+ *
+ * This code is based on glibc cortex strings work originally authored by Linaro
+ * and re-licensed under GPLv2 for the Linux kernel. The original code can
+ * be found @
+ *
+ * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/
+ * files/head:/src/aarch64/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linkage.h>
+#include <assembler.h>
+#include <cache.h>
+
+/*
+ * Fill in the buffer with character c (alignment handled by the hardware)
+ *
+ * Parameters:
+ *	x0 - buf
+ *	x1 - c
+ *	x2 - n
+ * Returns:
+ *	x0 - buf
+ */
+
+dstin		.req	x0
+val		.req	w1
+count		.req	x2
+tmp1		.req	x3
+tmp1w		.req	w3
+tmp2		.req	x4
+tmp2w		.req	w4
+zva_len_x	.req	x5
+zva_len		.req	w5
+zva_bits_x	.req	x6
+
+A_l		.req	x7
+A_lw		.req	w7
+dst		.req	x8
+tmp3w		.req	w9
+tmp3		.req	x9
+
+	.weak memset
+ENTRY(____inline_memset)
+ENTRY(__inline_memset)
+	mov	dst, dstin	/* Preserve return value.  */
+	and	A_lw, val, #255
+	orr	A_lw, A_lw, A_lw, lsl #8
+	orr	A_lw, A_lw, A_lw, lsl #16
+	orr	A_l, A_l, A_l, lsl #32
+
+	cmp	count, #15
+	b.hi	.Lover16_proc
+	/*All store maybe are non-aligned..*/
+	tbz	count, #3, 1f
+	str	A_l, [dst], #8
+1:
+	tbz	count, #2, 2f
+	str	A_lw, [dst], #4
+2:
+	tbz	count, #1, 3f
+	strh	A_lw, [dst], #2
+3:
+	tbz	count, #0, 4f
+	strb	A_lw, [dst]
+4:
+	ret
+
+.Lover16_proc:
+	/*Whether  the start address is aligned with 16.*/
+	neg	tmp2, dst
+	ands	tmp2, tmp2, #15
+	b.eq	.Laligned
+/*
+* The count is not less than 16, we can use stp to store the start 16 bytes,
+* then adjust the dst aligned with 16.This process will make the current
+* memory address at alignment boundary.
+*/
+	stp	A_l, A_l, [dst] /*non-aligned store..*/
+	/*make the dst aligned..*/
+	sub	count, count, tmp2
+	add	dst, dst, tmp2
+
+.Laligned:
+	cbz	A_l, .Lzero_mem
+
+.Ltail_maybe_long:
+	cmp	count, #64
+	b.ge	.Lnot_short
+.Ltail63:
+	ands	tmp1, count, #0x30
+	b.eq	3f
+	cmp	tmp1w, #0x20
+	b.eq	1f
+	b.lt	2f
+	stp	A_l, A_l, [dst], #16
+1:
+	stp	A_l, A_l, [dst], #16
+2:
+	stp	A_l, A_l, [dst], #16
+/*
+* The last store length is less than 16,use stp to write last 16 bytes.
+* It will lead some bytes written twice and the access is non-aligned.
+*/
+3:
+	ands	count, count, #15
+	cbz	count, 4f
+	add	dst, dst, count
+	stp	A_l, A_l, [dst, #-16]	/* Repeat some/all of last store. */
+4:
+	ret
+
+	/*
+	* Critical loop. Start at a new cache line boundary. Assuming
+	* 64 bytes per line, this ensures the entire loop is in one line.
+	*/
+	.p2align	L1_CACHE_SHIFT
+.Lnot_short:
+	sub	dst, dst, #16/* Pre-bias.  */
+	sub	count, count, #64
+1:
+	stp	A_l, A_l, [dst, #16]
+	stp	A_l, A_l, [dst, #32]
+	stp	A_l, A_l, [dst, #48]
+	stp	A_l, A_l, [dst, #64]!
+	subs	count, count, #64
+	b.ge	1b
+	tst	count, #0x3f
+	add	dst, dst, #16
+	b.ne	.Ltail63
+.Lexitfunc:
+	ret
+
+	/*
+	* For zeroing memory, check to see if we can use the ZVA feature to
+	* zero entire 'cache' lines.
+	*/
+.Lzero_mem:
+	cmp	count, #63
+	b.le	.Ltail63
+	/*
+	* For zeroing small amounts of memory, it's not worth setting up
+	* the line-clear code.
+	*/
+	cmp	count, #128
+	b.lt	.Lnot_short /*count is at least  128 bytes*/
+
+	mrs	tmp1, dczid_el0
+	tbnz	tmp1, #4, .Lnot_short
+	mov	tmp3w, #4
+	and	zva_len, tmp1w, #15	/* Safety: other bits reserved.  */
+	lsl	zva_len, tmp3w, zva_len
+
+	ands	tmp3w, zva_len, #63
+	/*
+	* ensure the zva_len is not less than 64.
+	* It is not meaningful to use ZVA if the block size is less than 64.
+	*/
+	b.ne	.Lnot_short
+.Lzero_by_line:
+	/*
+	* Compute how far we need to go to become suitably aligned. We're
+	* already at quad-word alignment.
+	*/
+	cmp	count, zva_len_x
+	b.lt	.Lnot_short		/* Not enough to reach alignment.  */
+	sub	zva_bits_x, zva_len_x, #1
+	neg	tmp2, dst
+	ands	tmp2, tmp2, zva_bits_x
+	b.eq	2f			/* Already aligned.  */
+	/* Not aligned, check that there's enough to copy after alignment.*/
+	sub	tmp1, count, tmp2
+	/*
+	* grantee the remain length to be ZVA is bigger than 64,
+	* avoid to make the 2f's process over mem range.*/
+	cmp	tmp1, #64
+	ccmp	tmp1, zva_len_x, #8, ge	/* NZCV=0b1000 */
+	b.lt	.Lnot_short
+	/*
+	* We know that there's at least 64 bytes to zero and that it's safe
+	* to overrun by 64 bytes.
+	*/
+	mov	count, tmp1
+1:
+	stp	A_l, A_l, [dst]
+	stp	A_l, A_l, [dst, #16]
+	stp	A_l, A_l, [dst, #32]
+	subs	tmp2, tmp2, #64
+	stp	A_l, A_l, [dst, #48]
+	add	dst, dst, #64
+	b.ge	1b
+	/* We've overrun a bit, so adjust dst downwards.*/
+	add	dst, dst, tmp2
+2:
+	sub	count, count, zva_len_x
+3:
+	dc	zva, dst
+	add	dst, dst, zva_len_x
+	subs	count, count, zva_len_x
+	b.ge	3b
+	ands	count, count, zva_bits_x
+	b.ne	.Ltail_maybe_long
+	ret
+ENDPIPROC(__inline_memset)
+ENDPROC(____inline_memset)
+
+
+/*
+ * Non-temporal vector memory clear
+ *
+ * Parameters:
+ *     x0 - buf (assumed to be aligned to page size)
+ *     x1 - n (assumed to be at least page size)
+ */
+ENTRY(__memclear)
+       stp     q0, q1, [x2] /* Preserve two 128 bit vector regs */
+       eor v0.16B, v0.16B, v0.16B
+       eor v1.16B, v1.16B, v1.16B
+1:
+       stnp q0, q1, [x0, #32 * 0]
+       stnp q0, q1, [x0, #32 * 1]
+       stnp q0, q1, [x0, #32 * 2]
+       stnp q0, q1, [x0, #32 * 3]
+       stnp q0, q1, [x0, #32 * 4]
+       stnp q0, q1, [x0, #32 * 5]
+       stnp q0, q1, [x0, #32 * 6]
+       stnp q0, q1, [x0, #32 * 7]
+       stnp q0, q1, [x0, #32 * 8]
+       stnp q0, q1, [x0, #32 * 9]
+       stnp q0, q1, [x0, #32 * 10]
+       stnp q0, q1, [x0, #32 * 11]
+       stnp q0, q1, [x0, #32 * 12]
+       stnp q0, q1, [x0, #32 * 13]
+       stnp q0, q1, [x0, #32 * 14]
+       stnp q0, q1, [x0, #32 * 15]
+       add x0, x0, #512
+       subs x1, x1, #512
+       cmp x1, #0
+       b.ne    1b
+
+       ldp     q0, q1, [x2] /* Restore vector regs */
+       ret
+ENDPROC(__memclear)

arch/arm64/kernel/mikc.c

@@ -0,0 +1,44 @@
+/* mikc.c COPYRIGHT FUJITSU LIMITED 2015-2016 */
+#include <ihk/ikc.h>
+#include <ihk/lock.h>
+#include <ikc/msg.h>
+#include <memory.h>
+#include <string.h>
+
+extern int num_processors;
+extern void arch_set_mikc_queue(void *r, void *w);
+ihk_ikc_ph_t arch_master_channel_packet_handler;
+
+int ihk_mc_ikc_init_first_local(struct ihk_ikc_channel_desc *channel,
+                                ihk_ikc_ph_t packet_handler)
+{
+	struct ihk_ikc_queue_head *rq, *wq;
+	size_t mikc_queue_pages;
+
+	ihk_ikc_system_init(NULL);
+
+	memset(channel, 0, sizeof(struct ihk_ikc_channel_desc));
+
+	mikc_queue_pages = ((8 * num_processors * MASTER_IKCQ_PKTSIZE)
+			+ (PAGE_SIZE - 1)) / PAGE_SIZE;
+
+	/* Place both sides in this side */
+	rq = ihk_mc_alloc_pages(mikc_queue_pages, IHK_MC_AP_CRITICAL);
+	wq = ihk_mc_alloc_pages(mikc_queue_pages, IHK_MC_AP_CRITICAL);
+
+	ihk_ikc_init_queue(rq, 0, 0,
+			mikc_queue_pages * PAGE_SIZE, MASTER_IKCQ_PKTSIZE);
+	ihk_ikc_init_queue(wq, 0, 0,
+			mikc_queue_pages * PAGE_SIZE, MASTER_IKCQ_PKTSIZE);
+
+	arch_master_channel_packet_handler = packet_handler;
+
+	ihk_ikc_init_desc(channel, IKC_OS_HOST, 0, rq, wq,
+	                  ihk_ikc_master_channel_packet_handler, channel);
+	ihk_ikc_enable_channel(channel);
+
+	/* Set boot parameter */
+	arch_set_mikc_queue(rq, wq);
+
+	return 0;
+}

arch/arm64/kernel/perfctr.c

@@ -0,0 +1,297 @@
+/* perfctr.c COPYRIGHT FUJITSU LIMITED 2015-2018 */
+#include <arch-perfctr.h>
+#include <ihk/perfctr.h>
+#include <mc_perf_event.h>
+#include <errno.h>
+#include <ihk/debug.h>
+#include <registers.h>
+#include <string.h>
+#include <ihk/mm.h>
+#include <irq.h>
+#include <process.h>
+
+/*
+ * @ref.impl arch/arm64/kernel/perf_event.c
+ * Set at runtime when we know what CPU type we are.
+ */
+struct arm_pmu cpu_pmu;
+extern int ihk_param_pmu_irq_affi[CONFIG_SMP_MAX_CORES];
+extern int ihk_param_nr_pmu_irq_affi;
+
+int arm64_init_perfctr(void)
+{
+	int ret;
+	int i;
+	int pages;
+	const struct ihk_mc_cpu_info *cpu_info;
+
+	memset(&cpu_pmu, 0, sizeof(cpu_pmu));
+	ret = armv8pmu_init(&cpu_pmu);
+	if (ret) {
+		return ret;
+	}
+
+	cpu_info = ihk_mc_get_cpu_info();
+	pages = (sizeof(struct per_cpu_arm_pmu) * cpu_info->ncpus +
+		 PAGE_SIZE - 1) >> PAGE_SHIFT;
+	cpu_pmu.per_cpu = ihk_mc_alloc_pages(pages, IHK_MC_AP_NOWAIT);
+	if (cpu_pmu.per_cpu == NULL) {
+		return -ENOMEM;
+	}
+	memset(cpu_pmu.per_cpu, 0, pages * PAGE_SIZE);
+
+	if (0 < ihk_param_nr_pmu_irq_affi) {
+		for (i = 0; i < ihk_param_nr_pmu_irq_affi; i++) {
+			ret = ihk_mc_register_interrupt_handler(ihk_param_pmu_irq_affi[i],
+								cpu_pmu.handler);
+			if (ret) {
+				break;
+			}
+		}
+	}
+	else {
+		ret = ihk_mc_register_interrupt_handler(INTRID_PERF_OVF,
+							cpu_pmu.handler);
+	}
+	return ret;
+}
+
+void arm64_init_per_cpu_perfctr(void)
+{
+	armv8pmu_per_cpu_init(&cpu_pmu.per_cpu[ihk_mc_get_processor_id()]);
+}
+
+int arm64_enable_pmu(void)
+{
+	int ret;
+	if (cpu_pmu.reset) {
+		cpu_pmu.reset(&cpu_pmu);
+	}
+	ret = cpu_pmu.enable_pmu();
+	return ret;
+}
+
+void arm64_disable_pmu(void)
+{
+	cpu_pmu.disable_pmu();
+}
+
+void arm64_enable_user_access_pmu_regs(void)
+{
+	cpu_pmu.enable_user_access_pmu_regs();
+}
+
+void arm64_disable_user_access_pmu_regs(void)
+{
+	cpu_pmu.disable_user_access_pmu_regs();
+}
+
+static int __ihk_mc_perfctr_init(int counter, uint32_t type, uint64_t config, int mode)
+{
+	int ret = -1;
+	unsigned long config_base = 0;
+
+	ret = cpu_pmu.disable_counter(1UL << counter);
+	if (ret < 0) {
+		return ret;
+	}
+
+	ret = cpu_pmu.enable_intens(1UL << counter);
+	if (ret < 0) {
+		return ret;
+	}
+
+	ret = cpu_pmu.set_event_filter(&config_base, mode);
+	if (ret) {
+		return ret;
+	}
+	config_base |= config;
+	cpu_pmu.write_evtype(counter, config_base);
+	return ret;
+}
+
+int ihk_mc_perfctr_init_raw(int counter, uint64_t config, int mode)
+{
+	int ret;
+	ret = __ihk_mc_perfctr_init(counter, PERF_TYPE_RAW, config, mode);
+	return ret;
+}
+
+int ihk_mc_perfctr_start(unsigned long counter_mask)
+{
+	return cpu_pmu.enable_counter(counter_mask);
+}
+
+int ihk_mc_perfctr_stop(unsigned long counter_mask, int flags)
+{
+	return cpu_pmu.disable_counter(counter_mask);
+}
+
+int ihk_mc_perfctr_reset(int counter)
+{
+	cpu_pmu.write_counter(counter, 0);
+	return 0;
+}
+
+int ihk_mc_perfctr_set(int counter, long val)
+{
+	uint32_t v = val;
+	cpu_pmu.write_counter(counter, v);
+	return 0;
+}
+
+int ihk_mc_perfctr_read_mask(unsigned long counter_mask, unsigned long *value)
+{
+	/* this function not used yet. */
+	panic("not implemented.");
+	return 0;
+}
+
+int ihk_mc_perfctr_alloc(struct thread *thread, struct mc_perf_event *event)
+{
+	const int counters = ihk_mc_perf_get_num_counters();
+
+	return cpu_pmu.get_event_idx(counters,
+				     thread->pmc_alloc_map,
+				     event->hw_config);
+}
+
+unsigned long ihk_mc_perfctr_read(int counter)
+{
+	unsigned long count;
+	count = cpu_pmu.read_counter(counter);
+	return count;
+}
+
+unsigned long ihk_mc_perfctr_value(int counter, unsigned long correction)
+{
+	unsigned long count = ihk_mc_perfctr_read(counter) + correction;
+
+	count &= ((1UL << 32) - 1);
+	return count;
+}
+
+int ihk_mc_perfctr_alloc_counter(unsigned int *type, unsigned long *config,
+				 unsigned long pmc_status)
+{
+	int ret;
+
+	if (*type == PERF_TYPE_HARDWARE) {
+		switch (*config) {
+		case PERF_COUNT_HW_INSTRUCTIONS:
+			ret = cpu_pmu.map_event(*type, *config);
+			if (ret < 0) {
+				return -1;
+			}
+			*type = PERF_TYPE_RAW;
+			break;
+		default:
+			// Unexpected config
+			return -1;
+		}
+	}
+	else if (*type != PERF_TYPE_RAW) {
+		return -1;
+	}
+	ret = cpu_pmu.get_event_idx(get_per_cpu_pmu()->num_events, pmc_status,
+				    *config);
+        return ret;
+}
+
+int ihk_mc_perf_counter_mask_check(unsigned long counter_mask)
+{
+	return cpu_pmu.counter_mask_valid(counter_mask);
+}
+
+int ihk_mc_perf_get_num_counters(void)
+{
+	const struct per_cpu_arm_pmu *per_cpu_arm_pmu = get_per_cpu_pmu();
+
+	return per_cpu_arm_pmu->num_events;
+}
+
+int ihk_mc_perfctr_set_extra(struct mc_perf_event *event)
+{
+	/* Nothing to do. */
+	return 0;
+}
+
+static inline uint64_t arm_pmu_event_max_period(struct mc_perf_event *event)
+{
+	return 0xFFFFFFFF;
+}
+
+int hw_perf_event_init(struct mc_perf_event *event)
+{
+	struct hw_perf_event *hwc = &event->hw;
+
+	if (!is_sampling_event(event)) {
+		hwc->sample_period  = arm_pmu_event_max_period(event) >> 1;
+		hwc->last_period    = hwc->sample_period;
+		ihk_atomic64_set(&hwc->period_left, hwc->sample_period);
+	}
+	return 0;
+}
+
+int ihk_mc_event_set_period(struct mc_perf_event *event)
+{
+	struct hw_perf_event *hwc = &event->hw;
+	int64_t left = ihk_atomic64_read(&hwc->period_left);
+	int64_t period = hwc->sample_period;
+	uint64_t max_period;
+	int ret = 0;
+
+	max_period = arm_pmu_event_max_period(event);
+	if (unlikely(left <= -period)) {
+		left = period;
+		ihk_atomic64_set(&hwc->period_left, left);
+		hwc->last_period = period;
+		ret = 1;
+	}
+
+	if (unlikely(left <= 0)) {
+		left += period;
+		ihk_atomic64_set(&hwc->period_left, left);
+		hwc->last_period = period;
+		ret = 1;
+	}
+
+	/*
+	 * Limit the maximum period to prevent the counter value
+	 * from overtaking the one we are about to program. In
+	 * effect we are reducing max_period to account for
+	 * interrupt latency (and we are being very conservative).
+	 */
+	if (left > (max_period >> 1))
+		left = (max_period >> 1);
+
+	ihk_atomic64_set(&hwc->prev_count, (uint64_t)-left);
+
+	cpu_pmu.write_counter(event->counter_id,
+			      (uint64_t)(-left) & max_period);
+
+	return ret;
+}
+
+uint64_t ihk_mc_event_update(struct mc_perf_event *event)
+{
+	struct hw_perf_event *hwc = &event->hw;
+	int64_t delta;
+	uint64_t prev_raw_count, new_raw_count;
+	uint64_t max_period = arm_pmu_event_max_period(event);
+
+again:
+	prev_raw_count = ihk_atomic64_read(&hwc->prev_count);
+	new_raw_count = cpu_pmu.read_counter(event->counter_id);
+
+	if (ihk_atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
+			     new_raw_count) != prev_raw_count)
+		goto again;
+
+	delta = (new_raw_count - prev_raw_count) & max_period;
+
+	ihk_atomic64_add(delta, &event->count);
+	ihk_atomic64_add(-delta, &hwc->period_left);
+
+	return new_raw_count;
+}

arch/arm64/kernel/perfctr_armv8pmu.c

@@ -0,0 +1,910 @@
+/* perfctr_armv8pmu.c COPYRIGHT FUJITSU LIMITED 2016-2018 */
+#include <arch-perfctr.h>
+#include <mc_perf_event.h>
+#include <ihk/perfctr.h>
+#include <errno.h>
+#include <ihk/debug.h>
+#include <sysreg.h>
+#include <virt.h>
+#include <bitops.h>
+#include <string.h>
+#include <signal.h>
+#include <cls.h>
+#include <process.h>
+
+#define BIT(nr) (1UL << (nr))
+
+//#define DEBUG_PRINT_PMU
+#ifdef DEBUG_PRINT_PMU
+#undef DDEBUG_DEFAULT
+#define DDEBUG_DEFAULT DDEBUG_PRINT
+#endif
+
+/*
+ * read pmevcntr<n>_el0 functions
+ */
+#define read_pmevcntrN_el0(N) \
+static uint32_t read_pmevcntr##N##_el0(void) \
+{ \
+	return read_sysreg(pmevcntr##N##_el0); \
+}
+
+read_pmevcntrN_el0(0)
+read_pmevcntrN_el0(1)
+read_pmevcntrN_el0(2)
+read_pmevcntrN_el0(3)
+read_pmevcntrN_el0(4)
+read_pmevcntrN_el0(5)
+read_pmevcntrN_el0(6)
+read_pmevcntrN_el0(7)
+read_pmevcntrN_el0(8)
+read_pmevcntrN_el0(9)
+read_pmevcntrN_el0(10)
+read_pmevcntrN_el0(11)
+read_pmevcntrN_el0(12)
+read_pmevcntrN_el0(13)
+read_pmevcntrN_el0(14)
+read_pmevcntrN_el0(15)
+read_pmevcntrN_el0(16)
+read_pmevcntrN_el0(17)
+read_pmevcntrN_el0(18)
+read_pmevcntrN_el0(19)
+read_pmevcntrN_el0(20)
+read_pmevcntrN_el0(21)
+read_pmevcntrN_el0(22)
+read_pmevcntrN_el0(23)
+read_pmevcntrN_el0(24)
+read_pmevcntrN_el0(25)
+read_pmevcntrN_el0(26)
+read_pmevcntrN_el0(27)
+read_pmevcntrN_el0(28)
+read_pmevcntrN_el0(29)
+read_pmevcntrN_el0(30)
+
+static uint32_t (* const read_pmevcntr_el0[])(void) = {
+	read_pmevcntr0_el0, read_pmevcntr1_el0, read_pmevcntr2_el0,
+	read_pmevcntr3_el0, read_pmevcntr4_el0, read_pmevcntr5_el0,
+	read_pmevcntr6_el0, read_pmevcntr7_el0, read_pmevcntr8_el0,
+	read_pmevcntr9_el0, read_pmevcntr10_el0, read_pmevcntr11_el0,
+	read_pmevcntr12_el0, read_pmevcntr13_el0, read_pmevcntr14_el0,
+	read_pmevcntr15_el0, read_pmevcntr16_el0, read_pmevcntr17_el0,
+	read_pmevcntr18_el0, read_pmevcntr19_el0, read_pmevcntr20_el0,
+	read_pmevcntr21_el0, read_pmevcntr22_el0, read_pmevcntr23_el0,
+	read_pmevcntr24_el0, read_pmevcntr25_el0, read_pmevcntr26_el0,
+	read_pmevcntr27_el0, read_pmevcntr28_el0, read_pmevcntr29_el0,
+	read_pmevcntr30_el0,
+};
+
+
+/*
+ * write pmevcntr<n>_el0 functions
+ */
+#define write_pmevcntrN_el0(N) \
+static void write_pmevcntr##N##_el0(uint32_t v) \
+{ \
+	write_sysreg(v, pmevcntr##N##_el0); \
+}
+
+write_pmevcntrN_el0(0)
+write_pmevcntrN_el0(1)
+write_pmevcntrN_el0(2)
+write_pmevcntrN_el0(3)
+write_pmevcntrN_el0(4)
+write_pmevcntrN_el0(5)
+write_pmevcntrN_el0(6)
+write_pmevcntrN_el0(7)
+write_pmevcntrN_el0(8)
+write_pmevcntrN_el0(9)
+write_pmevcntrN_el0(10)
+write_pmevcntrN_el0(11)
+write_pmevcntrN_el0(12)
+write_pmevcntrN_el0(13)
+write_pmevcntrN_el0(14)
+write_pmevcntrN_el0(15)
+write_pmevcntrN_el0(16)
+write_pmevcntrN_el0(17)
+write_pmevcntrN_el0(18)
+write_pmevcntrN_el0(19)
+write_pmevcntrN_el0(20)
+write_pmevcntrN_el0(21)
+write_pmevcntrN_el0(22)
+write_pmevcntrN_el0(23)
+write_pmevcntrN_el0(24)
+write_pmevcntrN_el0(25)
+write_pmevcntrN_el0(26)
+write_pmevcntrN_el0(27)
+write_pmevcntrN_el0(28)
+write_pmevcntrN_el0(29)
+write_pmevcntrN_el0(30)
+
+static void (* const write_pmevcntr_el0[])(uint32_t) = {
+	write_pmevcntr0_el0, write_pmevcntr1_el0, write_pmevcntr2_el0,
+	write_pmevcntr3_el0, write_pmevcntr4_el0, write_pmevcntr5_el0,
+	write_pmevcntr6_el0, write_pmevcntr7_el0, write_pmevcntr8_el0,
+	write_pmevcntr9_el0, write_pmevcntr10_el0, write_pmevcntr11_el0,
+	write_pmevcntr12_el0, write_pmevcntr13_el0, write_pmevcntr14_el0,
+	write_pmevcntr15_el0, write_pmevcntr16_el0, write_pmevcntr17_el0,
+	write_pmevcntr18_el0, write_pmevcntr19_el0, write_pmevcntr20_el0,
+	write_pmevcntr21_el0, write_pmevcntr22_el0, write_pmevcntr23_el0,
+	write_pmevcntr24_el0, write_pmevcntr25_el0, write_pmevcntr26_el0,
+	write_pmevcntr27_el0, write_pmevcntr28_el0, write_pmevcntr29_el0,
+	write_pmevcntr30_el0,
+};
+
+/*
+ * write pmevtyper<n>_el0 functions
+ */
+#define write_pmevtyperN_el0(N) \
+static void write_pmevtyper##N##_el0(uint32_t v) \
+{ \
+	write_sysreg(v, pmevtyper##N##_el0); \
+}
+
+write_pmevtyperN_el0(0)
+write_pmevtyperN_el0(1)
+write_pmevtyperN_el0(2)
+write_pmevtyperN_el0(3)
+write_pmevtyperN_el0(4)
+write_pmevtyperN_el0(5)
+write_pmevtyperN_el0(6)
+write_pmevtyperN_el0(7)
+write_pmevtyperN_el0(8)
+write_pmevtyperN_el0(9)
+write_pmevtyperN_el0(10)
+write_pmevtyperN_el0(11)
+write_pmevtyperN_el0(12)
+write_pmevtyperN_el0(13)
+write_pmevtyperN_el0(14)
+write_pmevtyperN_el0(15)
+write_pmevtyperN_el0(16)
+write_pmevtyperN_el0(17)
+write_pmevtyperN_el0(18)
+write_pmevtyperN_el0(19)
+write_pmevtyperN_el0(20)
+write_pmevtyperN_el0(21)
+write_pmevtyperN_el0(22)
+write_pmevtyperN_el0(23)
+write_pmevtyperN_el0(24)
+write_pmevtyperN_el0(25)
+write_pmevtyperN_el0(26)
+write_pmevtyperN_el0(27)
+write_pmevtyperN_el0(28)
+write_pmevtyperN_el0(29)
+write_pmevtyperN_el0(30)
+
+static void (* const write_pmevtyper_el0[])(uint32_t) = {
+	write_pmevtyper0_el0, write_pmevtyper1_el0, write_pmevtyper2_el0,
+	write_pmevtyper3_el0, write_pmevtyper4_el0, write_pmevtyper5_el0,
+	write_pmevtyper6_el0, write_pmevtyper7_el0, write_pmevtyper8_el0,
+	write_pmevtyper9_el0, write_pmevtyper10_el0, write_pmevtyper11_el0,
+	write_pmevtyper12_el0, write_pmevtyper13_el0, write_pmevtyper14_el0,
+	write_pmevtyper15_el0, write_pmevtyper16_el0, write_pmevtyper17_el0,
+	write_pmevtyper18_el0, write_pmevtyper19_el0, write_pmevtyper20_el0,
+	write_pmevtyper21_el0, write_pmevtyper22_el0, write_pmevtyper23_el0,
+	write_pmevtyper24_el0, write_pmevtyper25_el0, write_pmevtyper26_el0,
+	write_pmevtyper27_el0, write_pmevtyper28_el0, write_pmevtyper29_el0,
+	write_pmevtyper30_el0,
+};
+
+#define	ARMV8_IDX_CYCLE_COUNTER	31
+#define	ARMV8_IDX_COUNTER0	0
+
+/*
+ * @ref.impl linux-v4.15-rc3 arch/arm64/include/asm/perf_event.h
+ * Per-CPU PMCR: config reg
+ */
+#define ARMV8_PMU_PMCR_E	(1 << 0) /* Enable all counters */
+#define ARMV8_PMU_PMCR_P	(1 << 1) /* Reset all counters */
+#define ARMV8_PMU_PMCR_C	(1 << 2) /* Cycle counter reset */
+#define ARMV8_PMU_PMCR_D	(1 << 3) /* CCNT counts every 64th cpu cycle */
+#define ARMV8_PMU_PMCR_X	(1 << 4) /* Export to ETM */
+#define ARMV8_PMU_PMCR_DP	(1 << 5) /* Disable CCNT if non-invasive debug*/
+#define ARMV8_PMU_PMCR_LC	(1 << 6) /* Overflow on 64 bit cycle counter */
+#define	ARMV8_PMU_PMCR_N_SHIFT	11	 /* Number of counters supported */
+#define	ARMV8_PMU_PMCR_N_MASK	0x1f
+#define	ARMV8_PMU_PMCR_MASK	0x7f	 /* Mask for writable bits */
+
+/*
+ * @ref.impl linux-v4.15-rc3 arch/arm64/include/asm/perf_event.h
+ * PMOVSR: counters overflow flag status reg
+ */
+#define	ARMV8_PMU_OVSR_MASK		0xffffffff	/* Mask for writable bits */
+#define	ARMV8_PMU_OVERFLOWED_MASK	ARMV8_PMU_OVSR_MASK
+
+/*
+ * @ref.impl linux-v4.15-rc3 arch/arm64/include/asm/perf_event.h
+ * PMXEVTYPER: Event selection reg
+ */
+#define	ARMV8_PMU_EVTYPE_MASK	0xc800ffff	/* Mask for writable bits */
+#define	ARMV8_PMU_EVTYPE_EVENT	0xffff		/* Mask for EVENT bits */
+
+/*
+ * @ref.impl linux-v4.15-rc3 arch/arm64/include/asm/perf_event.h
+ * Event filters for PMUv3
+ */
+#define	ARMV8_PMU_EXCLUDE_EL1	(1 << 31)
+#define	ARMV8_PMU_EXCLUDE_EL0	(1 << 30)
+#define	ARMV8_PMU_INCLUDE_EL2	(1 << 27)
+
+/*
+ * @ref.impl linux-v4.15-rc3 arch/arm64/include/asm/perf_event.h
+ * PMUSERENR: user enable reg
+ */
+#define ARMV8_PMU_USERENR_MASK	0xf		/* Mask for writable bits */
+#define ARMV8_PMU_USERENR_EN	(1 << 0) /* PMU regs can be accessed at EL0 */
+#define ARMV8_PMU_USERENR_SW	(1 << 1) /* PMSWINC can be written at EL0 */
+#define ARMV8_PMU_USERENR_CR	(1 << 2) /* Cycle counter can be read at EL0 */
+#define ARMV8_PMU_USERENR_ER	(1 << 3) /* Event counter can be read at EL0 */
+
+/*
+ * @ref.impl linux-v4.15-rc3 arch/arm64/include/asm/perf_event.h
+ * PMUv3 event types: required events
+ */
+#define ARMV8_PMUV3_PERFCTR_SW_INCR				0x00
+#define ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL			0x03
+#define ARMV8_PMUV3_PERFCTR_L1D_CACHE				0x04
+#define ARMV8_PMUV3_PERFCTR_BR_MIS_PRED				0x10
+#define ARMV8_PMUV3_PERFCTR_CPU_CYCLES				0x11
+#define ARMV8_PMUV3_PERFCTR_BR_PRED				0x12
+
+/*
+ * @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c
+ * ARMv8 PMUv3 Performance Events handling code.
+ * Common event types (some are defined in asm/perf_event.h).
+ */
+
+/* At least one of the following is required. */
+#define ARMV8_PMUV3_PERFCTR_INST_RETIRED			0x08
+#define ARMV8_PMUV3_PERFCTR_INST_SPEC				0x1B
+
+/* Common architectural events. */
+#define ARMV8_PMUV3_PERFCTR_LD_RETIRED				0x06
+#define ARMV8_PMUV3_PERFCTR_ST_RETIRED				0x07
+#define ARMV8_PMUV3_PERFCTR_EXC_TAKEN				0x09
+#define ARMV8_PMUV3_PERFCTR_EXC_RETURN				0x0A
+#define ARMV8_PMUV3_PERFCTR_CID_WRITE_RETIRED			0x0B
+#define ARMV8_PMUV3_PERFCTR_PC_WRITE_RETIRED			0x0C
+#define ARMV8_PMUV3_PERFCTR_BR_IMMED_RETIRED			0x0D
+#define ARMV8_PMUV3_PERFCTR_BR_RETURN_RETIRED			0x0E
+#define ARMV8_PMUV3_PERFCTR_UNALIGNED_LDST_RETIRED		0x0F
+#define ARMV8_PMUV3_PERFCTR_TTBR_WRITE_RETIRED			0x1C
+#define ARMV8_PMUV3_PERFCTR_CHAIN				0x1E
+#define ARMV8_PMUV3_PERFCTR_BR_RETIRED				0x21
+
+/* Common microarchitectural events. */
+#define ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL			0x01
+#define ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL			0x02
+#define ARMV8_PMUV3_PERFCTR_L1D_TLB_REFILL			0x05
+#define ARMV8_PMUV3_PERFCTR_MEM_ACCESS				0x13
+#define ARMV8_PMUV3_PERFCTR_L1I_CACHE				0x14
+#define ARMV8_PMUV3_PERFCTR_L1D_CACHE_WB			0x15
+#define ARMV8_PMUV3_PERFCTR_L2D_CACHE				0x16
+#define ARMV8_PMUV3_PERFCTR_L2D_CACHE_REFILL			0x17
+#define ARMV8_PMUV3_PERFCTR_L2D_CACHE_WB			0x18
+#define ARMV8_PMUV3_PERFCTR_BUS_ACCESS				0x19
+#define ARMV8_PMUV3_PERFCTR_MEMORY_ERROR			0x1A
+#define ARMV8_PMUV3_PERFCTR_BUS_CYCLES				0x1D
+#define ARMV8_PMUV3_PERFCTR_L1D_CACHE_ALLOCATE			0x1F
+#define ARMV8_PMUV3_PERFCTR_L2D_CACHE_ALLOCATE			0x20
+#define ARMV8_PMUV3_PERFCTR_BR_MIS_PRED_RETIRED			0x22
+#define ARMV8_PMUV3_PERFCTR_STALL_FRONTEND			0x23
+#define ARMV8_PMUV3_PERFCTR_STALL_BACKEND			0x24
+#define ARMV8_PMUV3_PERFCTR_L1D_TLB				0x25
+#define ARMV8_PMUV3_PERFCTR_L1I_TLB				0x26
+#define ARMV8_PMUV3_PERFCTR_L2I_CACHE				0x27
+#define ARMV8_PMUV3_PERFCTR_L2I_CACHE_REFILL			0x28
+#define ARMV8_PMUV3_PERFCTR_L3D_CACHE_ALLOCATE			0x29
+#define ARMV8_PMUV3_PERFCTR_L3D_CACHE_REFILL			0x2A
+#define ARMV8_PMUV3_PERFCTR_L3D_CACHE				0x2B
+#define ARMV8_PMUV3_PERFCTR_L3D_CACHE_WB			0x2C
+#define ARMV8_PMUV3_PERFCTR_L2D_TLB_REFILL			0x2D
+#define ARMV8_PMUV3_PERFCTR_L2I_TLB_REFILL			0x2E
+#define ARMV8_PMUV3_PERFCTR_L2D_TLB				0x2F
+#define ARMV8_PMUV3_PERFCTR_L2I_TLB				0x30
+
+/* @ref.impl linux-v4.15-rc3 include/linux/perf/arm_pmu.h */
+#define HW_OP_UNSUPPORTED		0xFFFF
+#define C(_x)				PERF_COUNT_HW_CACHE_##_x
+#define CACHE_OP_UNSUPPORTED		0xFFFF
+
+#define PERF_MAP_ALL_UNSUPPORTED					\
+	[0 ... PERF_COUNT_HW_MAX - 1] = HW_OP_UNSUPPORTED
+
+#define PERF_CACHE_MAP_ALL_UNSUPPORTED					\
+[0 ... C(MAX) - 1] = {							\
+	[0 ... C(OP_MAX) - 1] = {					\
+		[0 ... C(RESULT_MAX) - 1] = CACHE_OP_UNSUPPORTED,	\
+	},								\
+}
+
+/* PMUv3 HW events mapping. */
+
+/* disable -Woverride-init for the following initializations */
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Woverride-init"
+
+/*
+ * @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c
+ * ARMv8 Architectural defined events, not all of these may
+ * be supported on any given implementation. Undefined events will
+ * be disabled at run-time.
+ */
+static const unsigned armv8_pmuv3_perf_map[PERF_COUNT_HW_MAX] = {
+	PERF_MAP_ALL_UNSUPPORTED,
+	[PERF_COUNT_HW_CPU_CYCLES]		= ARMV8_PMUV3_PERFCTR_CPU_CYCLES,
+	[PERF_COUNT_HW_INSTRUCTIONS]		= ARMV8_PMUV3_PERFCTR_INST_RETIRED,
+	[PERF_COUNT_HW_CACHE_REFERENCES]	= ARMV8_PMUV3_PERFCTR_L1D_CACHE,
+	[PERF_COUNT_HW_CACHE_MISSES]		= ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL,
+	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS]	= ARMV8_PMUV3_PERFCTR_PC_WRITE_RETIRED,
+	[PERF_COUNT_HW_BRANCH_MISSES]		= ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
+	[PERF_COUNT_HW_BUS_CYCLES]		= ARMV8_PMUV3_PERFCTR_BUS_CYCLES,
+	[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND]	= ARMV8_PMUV3_PERFCTR_STALL_FRONTEND,
+	[PERF_COUNT_HW_STALLED_CYCLES_BACKEND]	= ARMV8_PMUV3_PERFCTR_STALL_BACKEND,
+};
+
+/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
+static const unsigned armv8_pmuv3_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
+						[PERF_COUNT_HW_CACHE_OP_MAX]
+						[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
+	PERF_CACHE_MAP_ALL_UNSUPPORTED,
+
+	[C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV8_PMUV3_PERFCTR_L1D_CACHE,
+	[C(L1D)][C(OP_READ)][C(RESULT_MISS)]	= ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL,
+	[C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV8_PMUV3_PERFCTR_L1D_CACHE,
+	[C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL,
+
+	[C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV8_PMUV3_PERFCTR_L1I_CACHE,
+	[C(L1I)][C(OP_READ)][C(RESULT_MISS)]	= ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL,
+
+	[C(DTLB)][C(OP_READ)][C(RESULT_MISS)]	= ARMV8_PMUV3_PERFCTR_L1D_TLB_REFILL,
+	[C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV8_PMUV3_PERFCTR_L1D_TLB,
+
+	[C(ITLB)][C(OP_READ)][C(RESULT_MISS)]	= ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL,
+	[C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV8_PMUV3_PERFCTR_L1I_TLB,
+
+	[C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV8_PMUV3_PERFCTR_BR_PRED,
+	[C(BPU)][C(OP_READ)][C(RESULT_MISS)]	= ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
+	[C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV8_PMUV3_PERFCTR_BR_PRED,
+	[C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
+};
+
+/* restore warnings */
+#pragma GCC diagnostic pop
+
+/* @ref.impl linux-v4.15-rc3 drivers/perf/arm_pmu.c */
+static int
+armpmu_map_cache_event(const unsigned (*cache_map)
+				      [PERF_COUNT_HW_CACHE_MAX]
+				      [PERF_COUNT_HW_CACHE_OP_MAX]
+				      [PERF_COUNT_HW_CACHE_RESULT_MAX],
+		       uint64_t config)
+{
+	unsigned int cache_type, cache_op, cache_result, ret;
+
+	cache_type = (config >>  0) & 0xff;
+	if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
+		return -EINVAL;
+
+	cache_op = (config >>  8) & 0xff;
+	if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
+		return -EINVAL;
+
+	cache_result = (config >> 16) & 0xff;
+	if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
+		return -EINVAL;
+
+	if (!cache_map)
+		return -ENOENT;
+
+	ret = (int)(*cache_map)[cache_type][cache_op][cache_result];
+
+	if (ret == CACHE_OP_UNSUPPORTED)
+		return -ENOENT;
+
+	return ret;
+}
+
+/* @ref.impl linux-v4.15-rc3 drivers/perf/arm_pmu.c */
+static int
+armpmu_map_hw_event(const unsigned int (*event_map)[PERF_COUNT_HW_MAX],
+		    uint64_t config)
+{
+	int mapping;
+
+	if (config >= PERF_COUNT_HW_MAX)
+		return -EINVAL;
+
+	if (!event_map)
+		return -ENOENT;
+
+	mapping = (*event_map)[config];
+	return mapping == HW_OP_UNSUPPORTED ? -ENOENT : mapping;
+}
+
+/* @ref.impl linux-v4.15-rc3 drivers/perf/arm_pmu.c */
+static int
+armpmu_map_raw_event(uint32_t raw_event_mask, uint64_t config)
+{
+	return (int)(config & raw_event_mask);
+}
+
+/* @ref.impl linux-v4.15-rc3 drivers/perf/arm_pmu.c */
+static int
+armpmu_map_event(uint32_t type, uint64_t config,
+		 const unsigned int (*event_map)[PERF_COUNT_HW_MAX],
+		 const unsigned int (*cache_map)
+				[PERF_COUNT_HW_CACHE_MAX]
+				[PERF_COUNT_HW_CACHE_OP_MAX]
+				[PERF_COUNT_HW_CACHE_RESULT_MAX],
+		 uint32_t raw_event_mask)
+{
+	switch (type) {
+	case PERF_TYPE_HARDWARE:
+		return armpmu_map_hw_event(event_map, config);
+	case PERF_TYPE_HW_CACHE:
+		return armpmu_map_cache_event(cache_map, config);
+	case PERF_TYPE_RAW:
+		return armpmu_map_raw_event(raw_event_mask, config);
+	}
+	return -ENOENT;
+}
+
+static inline int armv8pmu_counter_mask_valid(unsigned long counter_mask)
+{
+	int num;
+	unsigned long event;
+	unsigned long cycle;
+	unsigned long invalid_mask;
+
+	num = get_per_cpu_pmu()->num_events;
+	num--; /* Sub the CPU cycles counter */
+	event = ((1UL << num) - 1) << ARMV8_IDX_COUNTER0;
+	cycle = 1UL << ARMV8_IDX_CYCLE_COUNTER;
+	invalid_mask = ~(event | cycle);
+
+	return !(counter_mask & invalid_mask);
+}
+
+static inline int armv8pmu_counter_valid(int idx)
+{
+	return armv8pmu_counter_mask_valid(1UL << idx);
+}
+
+/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
+static inline uint32_t armv8pmu_getreset_flags(void)
+{
+	uint32_t value;
+
+	/* Read */
+	value = read_sysreg(pmovsclr_el0);
+
+	/* Write to clear flags */
+	value &= ARMV8_PMU_OVSR_MASK;
+	write_sysreg(value, pmovsclr_el0);
+
+	return value;
+}
+
+/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
+static inline int armv8pmu_has_overflowed(uint32_t pmovsr)
+{
+	return pmovsr & ARMV8_PMU_OVERFLOWED_MASK;
+}
+
+static inline int armv8pmu_counter_has_overflowed(uint32_t pmnc, int idx)
+{
+	return pmnc & BIT(idx);
+}
+
+/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
+static int __armv8_pmuv3_map_event(uint32_t type, uint64_t config,
+				   const unsigned int (*extra_event_map)
+						  [PERF_COUNT_HW_MAX],
+				   const unsigned int (*extra_cache_map)
+						  [PERF_COUNT_HW_CACHE_MAX]
+						  [PERF_COUNT_HW_CACHE_OP_MAX]
+						  [PERF_COUNT_HW_CACHE_RESULT_MAX])
+{
+	int hw_event_id;
+
+	hw_event_id = armpmu_map_event(type, config, &armv8_pmuv3_perf_map,
+				       &armv8_pmuv3_perf_cache_map,
+				       ARMV8_PMU_EVTYPE_EVENT);
+
+	/* Onl expose micro/arch events supported by this PMU */
+	if ((hw_event_id > 0) && (hw_event_id < ARMV8_PMUV3_MAX_COMMON_EVENTS)
+	    && test_bit(hw_event_id, get_per_cpu_pmu()->pmceid_bitmap)) {
+		return hw_event_id;
+	}
+
+	return armpmu_map_event(type, config, extra_event_map, extra_cache_map,
+				ARMV8_PMU_EVTYPE_EVENT);
+}
+
+/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
+static int armv8_pmuv3_map_event(uint32_t type, uint64_t config)
+{
+	return __armv8_pmuv3_map_event(type, config, NULL, NULL);
+}
+
+
+static int armv8_pmuv3_map_hw_event(uint64_t config)
+{
+	return __armv8_pmuv3_map_event(PERF_TYPE_HARDWARE, config, NULL, NULL);
+}
+
+
+static int armv8_pmuv3_map_cache_event(uint64_t config)
+{
+	return __armv8_pmuv3_map_event(PERF_TYPE_HW_CACHE, config, NULL, NULL);
+}
+
+static int armv8_pmuv3_map_raw_event(uint64_t config)
+{
+	return __armv8_pmuv3_map_event(PERF_TYPE_RAW, config, NULL, NULL);
+}
+
+/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
+static inline uint32_t armv8pmu_pmcr_read(void)
+{
+	return read_sysreg(pmcr_el0);
+}
+
+/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
+static inline void armv8pmu_pmcr_write(uint32_t val)
+{
+	val &= ARMV8_PMU_PMCR_MASK;
+	isb();
+	write_sysreg(val, pmcr_el0);
+}
+
+/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
+static inline uint32_t armv8pmu_read_counter(int idx)
+{
+	uint32_t value = 0;
+
+	if (!armv8pmu_counter_valid(idx)) {
+		ekprintf("%s: The count_register#%d is not implemented.\n",
+			__func__, idx);
+	}
+	else if (idx == ARMV8_IDX_CYCLE_COUNTER) {
+		value = read_sysreg(pmccntr_el0);
+	}
+	else {
+		value = read_pmevcntr_el0[idx]();
+	}
+
+	return value;
+}
+
+/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
+static inline void armv8pmu_write_counter(int idx, uint32_t value)
+{
+	if (!armv8pmu_counter_valid(idx)) {
+		ekprintf("%s: The count_register#%d is not implemented.\n",
+			__func__, idx);
+	}
+	else if (idx == ARMV8_IDX_CYCLE_COUNTER) {
+		/*
+		 * Set the upper 32bits as this is a 64bit counter but we only
+		 * count using the lower 32bits and we want an interrupt when
+		 * it overflows.
+		 */
+		uint64_t value64 = (int32_t)value;
+
+		write_sysreg(value64, pmccntr_el0);
+	}
+	else {
+		write_pmevcntr_el0[idx](value);
+	}
+}
+
+/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
+static inline int armv8pmu_enable_intens(unsigned long counter_mask)
+{
+	if (!armv8pmu_counter_mask_valid(counter_mask)) {
+		ekprintf("%s: invalid counter mask(%#lx)\n",
+			__func__, counter_mask);
+		return -EINVAL;
+	}
+
+	write_sysreg(counter_mask, pmintenset_el1);
+	return 0;
+}
+
+/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
+static inline int armv8pmu_disable_intens(unsigned long counter_mask)
+{
+	if (!armv8pmu_counter_mask_valid(counter_mask)) {
+		ekprintf("%s: invalid counter mask(%#lx)\n",
+			__func__, counter_mask);
+		return -EINVAL;
+	}
+	write_sysreg(counter_mask, pmintenclr_el1);
+	isb();
+	/* Clear the overflow flag in case an interrupt is pending. */
+	write_sysreg(counter_mask, pmovsclr_el0);
+	isb();
+	return 0;
+}
+
+/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
+static int armv8pmu_set_event_filter(unsigned long *config_base, int mode)
+{
+	/* exclude_idle is unused mode, unsupported */
+//	if (attr->exclude_idle)
+//		return -EPERM;
+
+	/*
+	 * If we're running in hyp mode, then we *are* the hypervisor.
+	 * Therefore we ignore exclude_hv in this configuration, since
+	 * there's no hypervisor to sample anyway. This is consistent
+	 * with other architectures (x86 and Power).
+	 */
+	if (is_kernel_in_hyp_mode()) {
+		if (mode & PERFCTR_KERNEL_MODE)
+			*config_base |= ARMV8_PMU_INCLUDE_EL2;
+	} else {
+		if (!(mode & PERFCTR_KERNEL_MODE))
+			*config_base |= ARMV8_PMU_EXCLUDE_EL1;
+		/* exclude_hv is unused mode, unsupported */
+//		if (!attr->exclude_hv)
+//			config_base |= ARMV8_PMU_INCLUDE_EL2;
+	}
+	if (!(mode & PERFCTR_USER_MODE))
+		*config_base |= ARMV8_PMU_EXCLUDE_EL0;
+
+	return 0;
+}
+
+/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
+static inline void armv8pmu_write_evtype(int idx, uint32_t val)
+{
+	if (!armv8pmu_counter_valid(idx)) {
+		ekprintf("%s: The count_register#%d is not implemented.\n",
+			 __func__, idx);
+		return;
+	} else if (idx != ARMV8_IDX_CYCLE_COUNTER) {
+		write_pmevtyper_el0[idx](val);
+	}
+}
+
+/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
+static inline int armv8pmu_enable_counter(unsigned long counter_mask)
+{
+	if (!armv8pmu_counter_mask_valid(counter_mask)) {
+		ekprintf("%s: invalid counter mask 0x%lx.\n",
+			 __func__, counter_mask);
+		return -EINVAL;
+	}
+	write_sysreg(counter_mask, pmcntenset_el0);
+	return 0;
+}
+
+/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
+static inline int armv8pmu_disable_counter(unsigned long counter_mask)
+{
+	if (!armv8pmu_counter_mask_valid(counter_mask)) {
+		ekprintf("%s: invalid counter mask 0x%lx.\n",
+			 __func__, counter_mask);
+		return -EINVAL;
+	}
+	write_sysreg(counter_mask, pmcntenclr_el0);
+	return 0;
+}
+
+/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
+static ihk_spinlock_t pmu_lock = SPIN_LOCK_UNLOCKED;
+static int armv8pmu_start(void)
+{
+	unsigned long flags;
+
+	flags = ihk_mc_spinlock_lock(&pmu_lock);
+	/* Enable all counters */
+	armv8pmu_pmcr_write(armv8pmu_pmcr_read() | ARMV8_PMU_PMCR_E);
+	ihk_mc_spinlock_unlock(&pmu_lock, flags);
+
+	return 0;
+}
+
+/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
+static void armv8pmu_stop(void)
+{
+	unsigned long flags;
+
+	flags = ihk_mc_spinlock_lock(&pmu_lock);
+	/* Disable all counters */
+	armv8pmu_pmcr_write(armv8pmu_pmcr_read() & ~ARMV8_PMU_PMCR_E);
+	ihk_mc_spinlock_unlock(&pmu_lock, flags);
+}
+
+/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
+static void armv8pmu_reset(void *info)
+{
+	struct arm_pmu *cpu_pmu = (struct arm_pmu *)info;
+	uint32_t nb_cnt =
+		cpu_pmu->per_cpu[ihk_mc_get_processor_id()].num_events;
+	nb_cnt--; /* Sub the CPU cycles counter */
+	unsigned long event = ((1UL << nb_cnt) - 1) << ARMV8_IDX_COUNTER0;
+	unsigned long cycle = 1UL << ARMV8_IDX_CYCLE_COUNTER;
+	unsigned long valid_mask = event | cycle;
+
+	/* The counter and interrupt enable registers are unknown at reset. */
+	armv8pmu_disable_counter(valid_mask);
+	armv8pmu_disable_intens(valid_mask);
+
+	/*
+	 * Initialize & Reset PMNC. Request overflow interrupt for
+	 * 64 bit cycle counter but cheat in armv8pmu_write_counter().
+	 */
+	armv8pmu_pmcr_write(ARMV8_PMU_PMCR_P | ARMV8_PMU_PMCR_C |
+			    ARMV8_PMU_PMCR_LC);
+}
+
+/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
+static int armv8pmu_get_event_idx(int num_events, unsigned long used_mask,
+				  unsigned long config)
+{
+	int idx, end;
+	unsigned long evtype = config & ARMV8_PMU_EVTYPE_EVENT;
+
+	/* Always prefer to place a cycle counter into the cycle counter. */
+	if (evtype == ARMV8_PMUV3_PERFCTR_CPU_CYCLES) {
+		if (!(used_mask & (1UL << ARMV8_IDX_CYCLE_COUNTER)))
+			return ARMV8_IDX_CYCLE_COUNTER;
+	}
+
+	/*
+	 * Otherwise use events counters
+	 */
+	end = ARMV8_IDX_COUNTER0 + num_events;
+	end--; /* Sub the CPU cycles counter */
+	for (idx = ARMV8_IDX_COUNTER0; idx < end; ++idx) {
+		if (!(used_mask & (1UL << idx)))
+			return idx;
+	}
+
+	/* The counters are all in use. */
+	return -EAGAIN;
+}
+
+/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c:__armv8pmu_probe_pmu() */
+/* Extract get num_events processing. */
+static uint32_t armv8pmu_read_num_pmnc_events(void)
+{
+	uint32_t num_events = 0;
+
+	/* Read the nb of CNTx counters supported from PMNC */
+	num_events = (armv8pmu_pmcr_read() >> ARMV8_PMU_PMCR_N_SHIFT)
+		& ARMV8_PMU_PMCR_N_MASK;
+
+	/* Add the CPU cycles counter */
+	num_events += 1;
+
+	return num_events;
+}
+
+static void armv8pmu_handle_irq(void *priv)
+{
+	uint32_t pmovsr;
+	struct thread *thread = cpu_local_var(current);
+	struct process *proc = thread->proc;
+	const struct per_cpu_arm_pmu *cpu_pmu = get_per_cpu_pmu();
+	int idx;
+
+	/*
+	 * Get and reset the IRQ flags
+	 */
+	pmovsr = armv8pmu_getreset_flags();
+
+	/*
+	 * Did an overflow occur?
+	 */
+	if (!armv8pmu_has_overflowed(pmovsr))
+		return;
+
+	if (!proc->monitoring_event) {
+		return;
+	}
+	/*
+	 * Handle the counter(s) overflow(s)
+	 */
+	for (idx = 0; idx < cpu_pmu->num_events; idx++) {
+		struct mc_perf_event *event = NULL;
+		struct mc_perf_event *sub;
+
+		if (!armv8pmu_counter_has_overflowed(pmovsr, idx)) {
+			continue;
+		}
+
+		if (proc->monitoring_event->counter_id == idx) {
+			event = proc->monitoring_event;
+		} else {
+			list_for_each_entry(sub,
+					&proc->monitoring_event->sibling_list,
+					group_entry) {
+				if (sub->counter_id == idx) {
+					event = sub;
+					break;
+				}
+			}
+		}
+
+		if (!event) {
+			continue;
+		}
+		ihk_mc_event_update(event);
+		ihk_mc_event_set_period(event);
+	}
+	return;
+}
+
+static void armv8pmu_enable_user_access_pmu_regs(void)
+{
+	uint32_t value = 0;
+
+	value = read_sysreg(pmuserenr_el0);
+	write_sysreg(value | (ARMV8_PMU_USERENR_ER | ARMV8_PMU_USERENR_CR),
+		     pmuserenr_el0);
+}
+
+static void armv8pmu_disable_user_access_pmu_regs(void)
+{
+	uint32_t value = 0;
+
+	value = read_sysreg(pmuserenr_el0);
+	write_sysreg(value & ~(ARMV8_PMU_USERENR_ER | ARMV8_PMU_USERENR_CR),
+		     pmuserenr_el0);
+}
+
+static void armv8pmu_create_pmceid_bitmap(unsigned long *bitmap, uint32_t nbits)
+{
+	uint32_t pmceid[2];
+
+	memset(bitmap, 0, BITS_TO_LONGS(nbits) * sizeof(unsigned long));
+
+	pmceid[0] = read_sysreg(pmceid0_el0);
+	bitmap[0] = (unsigned long)pmceid[0];
+
+	pmceid[1] = read_sysreg(pmceid1_el0);
+	bitmap[0] |= (unsigned long)pmceid[1] << 32;
+}
+
+static struct ihk_mc_interrupt_handler armv8pmu_handler = {
+	.func = armv8pmu_handle_irq,
+	.priv = NULL,
+};
+
+int armv8pmu_init(struct arm_pmu* cpu_pmu)
+{
+	cpu_pmu->read_counter = armv8pmu_read_counter;
+	cpu_pmu->write_counter = armv8pmu_write_counter;
+	cpu_pmu->reset = armv8pmu_reset;
+	cpu_pmu->enable_pmu = armv8pmu_start;
+	cpu_pmu->disable_pmu = armv8pmu_stop;
+	cpu_pmu->enable_counter = armv8pmu_enable_counter;
+	cpu_pmu->disable_counter = armv8pmu_disable_counter;
+	cpu_pmu->enable_intens = armv8pmu_enable_intens;
+	cpu_pmu->disable_intens = armv8pmu_disable_intens;
+	cpu_pmu->set_event_filter = armv8pmu_set_event_filter;
+	cpu_pmu->write_evtype = armv8pmu_write_evtype;
+	cpu_pmu->get_event_idx = armv8pmu_get_event_idx;
+	cpu_pmu->map_event = armv8_pmuv3_map_event;
+	cpu_pmu->map_hw_event = armv8_pmuv3_map_hw_event;
+	cpu_pmu->map_cache_event = armv8_pmuv3_map_cache_event;
+	cpu_pmu->map_raw_event = armv8_pmuv3_map_raw_event;
+	cpu_pmu->enable_user_access_pmu_regs =
+		armv8pmu_enable_user_access_pmu_regs;
+	cpu_pmu->disable_user_access_pmu_regs =
+		armv8pmu_disable_user_access_pmu_regs;
+	cpu_pmu->handler = &armv8pmu_handler;
+	cpu_pmu->counter_mask_valid = &armv8pmu_counter_mask_valid;
+	return 0;
+}
+
+void armv8pmu_per_cpu_init(struct per_cpu_arm_pmu *per_cpu)
+{
+	per_cpu->num_events = armv8pmu_read_num_pmnc_events();
+	armv8pmu_create_pmceid_bitmap(per_cpu->pmceid_bitmap,
+					ARMV8_PMUV3_MAX_COMMON_EVENTS);
+}

arch/arm64/kernel/postk_print_sysreg.c

@@ -0,0 +1,311 @@
+/* postk_print_sysreg.c COPYRIGHT FUJITSU LIMITED 2016 */
+/*
+ * usage:
+ *    	(gdb) call/x postk_debug_sysreg_ttbr1_el1()
+ *      $1 = 0x4e64f000
+ */
+#define postk_debug_sysreg(sysreg)  __postk_debug_sysreg(sysreg, sysreg)
+
+#define __postk_debug_sysreg(fname, regname)		\
+	unsigned long postk_debug_sysreg_ ## fname (void)	\
+	{							\
+		unsigned long sysreg;				\
+		asm volatile(					\
+			"mrs    %0, " # regname "\n"		\
+			: "=r" (sysreg)				\
+			:					\
+			: "memory");				\
+		return sysreg;					\
+	}
+
+/*
+ * ARMR Architecture Reference Manual ARMv8, for ARMv8-A architecture profile Errata markup Beta
+ *  - Table J-5 Alphabetical index of AArch64 Registers
+ */
+postk_debug_sysreg(actlr_el1)
+postk_debug_sysreg(actlr_el2)
+postk_debug_sysreg(actlr_el3)
+postk_debug_sysreg(afsr0_el1)
+postk_debug_sysreg(afsr0_el2)
+postk_debug_sysreg(afsr0_el3)
+postk_debug_sysreg(afsr1_el1)
+postk_debug_sysreg(afsr1_el2)
+postk_debug_sysreg(afsr1_el3)
+postk_debug_sysreg(aidr_el1)
+postk_debug_sysreg(amair_el1)
+postk_debug_sysreg(amair_el2)
+postk_debug_sysreg(amair_el3)
+/*postk_debug_sysreg(at s12e0r)*/
+/*postk_debug_sysreg(at s12e0w)*/
+/*postk_debug_sysreg(at s12e1r)*/
+/*postk_debug_sysreg(at s12e1w)*/
+/*postk_debug_sysreg(at s1e0r)*/
+/*postk_debug_sysreg(at s1e0w)*/
+/*postk_debug_sysreg(at s1e1r)*/
+/*postk_debug_sysreg(at s1e1w)*/
+/*postk_debug_sysreg(at s1e2r)*/
+/*postk_debug_sysreg(at s1e2w)*/
+/*postk_debug_sysreg(at s1e3r)*/
+/*postk_debug_sysreg(at s1e3w)*/
+postk_debug_sysreg(ccsidr_el1)
+postk_debug_sysreg(clidr_el1)
+postk_debug_sysreg(cntfrq_el0)
+postk_debug_sysreg(cnthctl_el2)
+postk_debug_sysreg(cnthp_ctl_el2)
+postk_debug_sysreg(cnthp_cval_el2)
+postk_debug_sysreg(cnthp_tval_el2)
+postk_debug_sysreg(cntkctl_el1)
+postk_debug_sysreg(cntp_ctl_el0)
+postk_debug_sysreg(cntp_cval_el0)
+postk_debug_sysreg(cntp_tval_el0)
+postk_debug_sysreg(cntpct_el0)
+postk_debug_sysreg(cntps_ctl_el1)
+postk_debug_sysreg(cntps_cval_el1)
+postk_debug_sysreg(cntps_tval_el1)
+postk_debug_sysreg(cntv_ctl_el0)
+postk_debug_sysreg(cntv_cval_el0)
+postk_debug_sysreg(cntv_tval_el0)
+postk_debug_sysreg(cntvct_el0)
+postk_debug_sysreg(cntvoff_el2)
+postk_debug_sysreg(contextidr_el1)
+postk_debug_sysreg(cpacr_el1)
+postk_debug_sysreg(cptr_el2)
+postk_debug_sysreg(cptr_el3)
+postk_debug_sysreg(csselr_el1)
+postk_debug_sysreg(ctr_el0)
+postk_debug_sysreg(currentel)
+postk_debug_sysreg(dacr32_el2)
+postk_debug_sysreg(daif)
+postk_debug_sysreg(dbgauthstatus_el1)
+/*postk_debug_sysreg(dbgbcr<n>_el1)*/
+/*postk_debug_sysreg(dbgbvr<n>_el1)*/
+postk_debug_sysreg(dbgclaimclr_el1)
+postk_debug_sysreg(dbgclaimset_el1)
+postk_debug_sysreg(dbgdtr_el0)
+postk_debug_sysreg(dbgdtrrx_el0)
+postk_debug_sysreg(dbgdtrtx_el0)
+postk_debug_sysreg(dbgprcr_el1)
+postk_debug_sysreg(dbgvcr32_el2)
+/*postk_debug_sysreg(dbgwcr<n>_el1)*/
+/*postk_debug_sysreg(dbgwvr<n>_el1)*/
+/*postk_debug_sysreg(dc cisw)*/
+/*postk_debug_sysreg(dc civac)*/
+/*postk_debug_sysreg(dc csw)*/
+/*postk_debug_sysreg(dc cvac)*/
+/*postk_debug_sysreg(dc cvau)*/
+/*postk_debug_sysreg(dc isw)*/
+/*postk_debug_sysreg(dc ivac)*/
+/*postk_debug_sysreg(dc zva)*/
+postk_debug_sysreg(dczid_el0)
+postk_debug_sysreg(dlr_el0)
+postk_debug_sysreg(dspsr_el0)
+postk_debug_sysreg(elr_el1)
+postk_debug_sysreg(elr_el2)
+postk_debug_sysreg(elr_el3)
+postk_debug_sysreg(esr_el1)
+postk_debug_sysreg(esr_el2)
+postk_debug_sysreg(esr_el3)
+postk_debug_sysreg(far_el1)
+postk_debug_sysreg(far_el2)
+postk_debug_sysreg(far_el3)
+postk_debug_sysreg(fpcr)
+postk_debug_sysreg(fpexc32_el2)
+postk_debug_sysreg(fpsr)
+postk_debug_sysreg(hacr_el2)
+postk_debug_sysreg(hcr_el2)
+postk_debug_sysreg(hpfar_el2)
+postk_debug_sysreg(hstr_el2)
+/*postk_debug_sysreg(ic iallu)*/
+/*postk_debug_sysreg(ic ialluis)*/
+/*postk_debug_sysreg(ic ivau)*/
+/*postk_debug_sysreg(icc_ap0r0_el1)*/
+/*postk_debug_sysreg(icc_ap0r1_el1)*/
+/*postk_debug_sysreg(icc_ap0r2_el1)*/
+/*postk_debug_sysreg(icc_ap0r3_el1)*/
+/*postk_debug_sysreg(icc_ap1r0_el1)*/
+/*postk_debug_sysreg(icc_ap1r1_el1)*/
+/*postk_debug_sysreg(icc_ap1r2_el1)*/
+/*postk_debug_sysreg(icc_ap1r3_el1)*/
+/*postk_debug_sysreg(icc_asgi1r_el1)*/
+/*postk_debug_sysreg(icc_bpr0_el1)*/
+/*postk_debug_sysreg(icc_bpr1_el1)*/
+/*postk_debug_sysreg(icc_ctlr_el1)*/
+/*postk_debug_sysreg(icc_ctlr_el3)*/
+/*postk_debug_sysreg(icc_dir_el1)*/
+/*postk_debug_sysreg(icc_eoir0_el1)*/
+/*postk_debug_sysreg(icc_eoir1_el1)*/
+/*postk_debug_sysreg(icc_hppir0_el1)*/
+/*postk_debug_sysreg(icc_hppir1_el1)*/
+/*postk_debug_sysreg(icc_iar0_el1)*/
+/*postk_debug_sysreg(icc_iar1_el1)*/
+/*postk_debug_sysreg(icc_igrpen0_el1)*/
+/*postk_debug_sysreg(icc_igrpen1_el1)*/
+/*postk_debug_sysreg(icc_igrpen1_el3)*/
+/*postk_debug_sysreg(icc_pmr_el1)*/
+/*postk_debug_sysreg(icc_rpr_el1)*/
+/*postk_debug_sysreg(icc_seien_el1)*/
+/*postk_debug_sysreg(icc_sgi0r_el1)*/
+/*postk_debug_sysreg(icc_sgi1r_el1)*/
+/*postk_debug_sysreg(icc_sre_el1)*/
+/*postk_debug_sysreg(icc_sre_el2)*/
+/*postk_debug_sysreg(icc_sre_el3)*/
+/*postk_debug_sysreg(ich_ap0r0_el2)*/
+/*postk_debug_sysreg(ich_ap0r1_el2)*/
+/*postk_debug_sysreg(ich_ap0r2_el2)*/
+/*postk_debug_sysreg(ich_ap0r3_el2)*/
+/*postk_debug_sysreg(ich_ap1r0_el2)*/
+/*postk_debug_sysreg(ich_ap1r1_el2)*/
+/*postk_debug_sysreg(ich_ap1r2_el2)*/
+/*postk_debug_sysreg(ich_ap1r3_el2)*/
+/*postk_debug_sysreg(ich_eisr_el2)*/
+/*postk_debug_sysreg(ich_elsr_el2)*/
+/*postk_debug_sysreg(ich_hcr_el2)*/
+/*postk_debug_sysreg(ich_lr<n>_el2)*/
+/*postk_debug_sysreg(ich_misr_el2)*/
+/*postk_debug_sysreg(ich_vmcr_el2)*/
+/*postk_debug_sysreg(ich_vseir_el2)*/
+/*postk_debug_sysreg(ich_vtr_el2)*/
+postk_debug_sysreg(id_aa64afr0_el1)
+postk_debug_sysreg(id_aa64afr1_el1)
+postk_debug_sysreg(id_aa64dfr0_el1)
+postk_debug_sysreg(id_aa64dfr1_el1)
+postk_debug_sysreg(id_aa64isar0_el1)
+postk_debug_sysreg(id_aa64isar1_el1)
+postk_debug_sysreg(id_aa64mmfr0_el1)
+postk_debug_sysreg(id_aa64mmfr1_el1)
+postk_debug_sysreg(id_aa64pfr0_el1)
+postk_debug_sysreg(id_aa64pfr1_el1)
+postk_debug_sysreg(id_afr0_el1)
+postk_debug_sysreg(id_dfr0_el1)
+postk_debug_sysreg(id_isar0_el1)
+postk_debug_sysreg(id_isar1_el1)
+postk_debug_sysreg(id_isar2_el1)
+postk_debug_sysreg(id_isar3_el1)
+postk_debug_sysreg(id_isar4_el1)
+postk_debug_sysreg(id_isar5_el1)
+postk_debug_sysreg(id_mmfr0_el1)
+postk_debug_sysreg(id_mmfr1_el1)
+postk_debug_sysreg(id_mmfr2_el1)
+postk_debug_sysreg(id_mmfr3_el1)
+postk_debug_sysreg(id_pfr0_el1)
+postk_debug_sysreg(id_pfr1_el1)
+postk_debug_sysreg(ifsr32_el2)
+postk_debug_sysreg(isr_el1)
+postk_debug_sysreg(mair_el1)
+postk_debug_sysreg(mair_el2)
+postk_debug_sysreg(mair_el3)
+postk_debug_sysreg(mdccint_el1)
+postk_debug_sysreg(mdccsr_el0)
+postk_debug_sysreg(mdcr_el2)
+postk_debug_sysreg(mdcr_el3)
+postk_debug_sysreg(mdrar_el1)
+postk_debug_sysreg(mdscr_el1)
+postk_debug_sysreg(midr_el1)
+postk_debug_sysreg(mpidr_el1)
+postk_debug_sysreg(mvfr0_el1)
+postk_debug_sysreg(mvfr1_el1)
+postk_debug_sysreg(mvfr2_el1)
+postk_debug_sysreg(nzcv)
+postk_debug_sysreg(osdlr_el1)
+postk_debug_sysreg(osdtrrx_el1)
+postk_debug_sysreg(osdtrtx_el1)
+postk_debug_sysreg(oseccr_el1)
+postk_debug_sysreg(oslar_el1)
+postk_debug_sysreg(oslsr_el1)
+postk_debug_sysreg(par_el1)
+postk_debug_sysreg(pmccfiltr_el0)
+postk_debug_sysreg(pmccntr_el0)
+postk_debug_sysreg(pmceid0_el0)
+postk_debug_sysreg(pmceid1_el0)
+postk_debug_sysreg(pmcntenclr_el0)
+postk_debug_sysreg(pmcntenset_el0)
+postk_debug_sysreg(pmcr_el0)
+/*postk_debug_sysreg(pmevcntr<n>_el0)*/
+/*postk_debug_sysreg(pmevtyper<n>_el0)*/
+postk_debug_sysreg(pmintenclr_el1)
+postk_debug_sysreg(pmintenset_el1)
+postk_debug_sysreg(pmovsclr_el0)
+postk_debug_sysreg(pmovsset_el0)
+postk_debug_sysreg(pmselr_el0)
+postk_debug_sysreg(pmswinc_el0)
+postk_debug_sysreg(pmuserenr_el0)
+postk_debug_sysreg(pmxevcntr_el0)
+postk_debug_sysreg(pmxevtyper_el0)
+postk_debug_sysreg(revidr_el1)
+postk_debug_sysreg(rmr_el1)
+postk_debug_sysreg(rmr_el2)
+postk_debug_sysreg(rmr_el3)
+postk_debug_sysreg(rvbar_el1)
+postk_debug_sysreg(rvbar_el2)
+postk_debug_sysreg(rvbar_el3)
+/*postk_debug_sysreg(s3_<op1>_<cn>_<cm>_<op2>)*/
+postk_debug_sysreg(scr_el3)
+postk_debug_sysreg(sctlr_el1)
+postk_debug_sysreg(sctlr_el2)
+postk_debug_sysreg(sctlr_el3)
+postk_debug_sysreg(sder32_el3)
+postk_debug_sysreg(sp_el0)
+postk_debug_sysreg(sp_el1)
+postk_debug_sysreg(sp_el2)
+/*postk_debug_sysreg(sp_el3)*/
+postk_debug_sysreg(spsel)
+postk_debug_sysreg(spsr_abt)
+postk_debug_sysreg(spsr_el1)
+postk_debug_sysreg(spsr_el2)
+postk_debug_sysreg(spsr_el3)
+postk_debug_sysreg(spsr_fiq)
+postk_debug_sysreg(spsr_irq)
+postk_debug_sysreg(spsr_und)
+postk_debug_sysreg(tcr_el1)
+postk_debug_sysreg(tcr_el2)
+postk_debug_sysreg(tcr_el3)
+postk_debug_sysreg(teecr32_el1)
+postk_debug_sysreg(teehbr32_el1)
+/*postk_debug_sysreg(tlbi alle1)*/
+/*postk_debug_sysreg(tlbi alle1is)*/
+/*postk_debug_sysreg(tlbi alle2)*/
+/*postk_debug_sysreg(tlbi alle2is)*/
+/*postk_debug_sysreg(tlbi alle3)*/
+/*postk_debug_sysreg(tlbi alle3is)*/
+/*postk_debug_sysreg(tlbi aside1)*/
+/*postk_debug_sysreg(tlbi aside1is)*/
+/*postk_debug_sysreg(tlbi ipas2e1)*/
+/*postk_debug_sysreg(tlbi ipas2e1is)*/
+/*postk_debug_sysreg(tlbi ipas2le1)*/
+/*postk_debug_sysreg(tlbi ipas2le1is)*/
+/*postk_debug_sysreg(tlbi vaae1)*/
+/*postk_debug_sysreg(tlbi vaae1is)*/
+/*postk_debug_sysreg(tlbi vaale1)*/
+/*postk_debug_sysreg(tlbi vaale1is)*/
+/*postk_debug_sysreg(tlbi vae1)*/
+/*postk_debug_sysreg(tlbi vae1is)*/
+/*postk_debug_sysreg(tlbi vae2)*/
+/*postk_debug_sysreg(tlbi vae2is)*/
+/*postk_debug_sysreg(tlbi vae3)*/
+/*postk_debug_sysreg(tlbi vae3is)*/
+/*postk_debug_sysreg(tlbi vale1)*/
+/*postk_debug_sysreg(tlbi vale1is)*/
+/*postk_debug_sysreg(tlbi vale2)*/
+/*postk_debug_sysreg(tlbi vale2is)*/
+/*postk_debug_sysreg(tlbi vale3)*/
+/*postk_debug_sysreg(tlbi vale3is)*/
+/*postk_debug_sysreg(tlbi vmalle1)*/
+/*postk_debug_sysreg(tlbi vmalle1is)*/
+/*postk_debug_sysreg(tlbi vmalls12e1)*/
+/*postk_debug_sysreg(tlbi vmalls12e1is)*/
+postk_debug_sysreg(tpidr_el0)
+postk_debug_sysreg(tpidr_el1)
+postk_debug_sysreg(tpidr_el2)
+postk_debug_sysreg(tpidr_el3)
+postk_debug_sysreg(tpidrro_el0)
+postk_debug_sysreg(ttbr0_el1)
+postk_debug_sysreg(ttbr0_el2)
+postk_debug_sysreg(ttbr0_el3)
+postk_debug_sysreg(ttbr1_el1)
+postk_debug_sysreg(vbar_el1)
+postk_debug_sysreg(vbar_el2)
+postk_debug_sysreg(vbar_el3)
+postk_debug_sysreg(vmpidr_el2)
+postk_debug_sysreg(vpidr_el2)
+postk_debug_sysreg(vtcr_el2)
+postk_debug_sysreg(vttbr_el2)

arch/arm64/kernel/proc-macros.S

@@ -0,0 +1,13 @@
+/* proc-macros.S COPYRIGHT FUJITSU LIMITED 2015 */
+
+#include <arch-memory.h>
+
+/*
+ * dcache_line_size - get the minimum D-cache line size from the CTR register.
+ */
+	.macro	dcache_line_size, reg, tmp
+	mrs	\tmp, ctr_el0			// read CTR
+	ubfm	\tmp, \tmp, #16, #19		// cache line size encoding
+	mov	\reg, #4			// bytes per word
+	lsl	\reg, \reg, \tmp		// actual cache line size
+	.endm

arch/arm64/kernel/proc.S

@@ -0,0 +1,148 @@
+/* proc.S COPYRIGHT FUJITSU LIMITED 2015-2017 */
+
+#include <linkage.h>
+#include <arch-memory.h>
+#include <sysreg.h>
+#include <assembler.h>
+#include "proc-macros.S"
+
+#ifdef CONFIG_ARM64_64K_PAGES
+# define TCR_TG_FLAGS	TCR_TG0_64K | TCR_TG1_64K
+#else
+# define TCR_TG_FLAGS	TCR_TG0_4K | TCR_TG1_4K
+#endif
+
+//#ifdef CONFIG_SMP
+#define TCR_SMP_FLAGS	TCR_SHARED
+//#else
+//#define TCR_SMP_FLAGS	0
+//#endif
+
+/* PTWs cacheable, inner/outer WBWA */
+#define TCR_CACHE_FLAGS	TCR_IRGN_WBWA | TCR_ORGN_WBWA
+
+#define MAIR(attr, mt)	((attr) << ((mt) * 8))
+
+/*
+ *	cpu_do_idle()
+ *
+ *	Idle the processor (wait for interrupt).
+ */
+#if defined(CONFIG_HAS_NMI)
+#include <arm-gic-v3.h>
+ENTRY(__cpu_do_idle)
+	mrs	x0, daif			// save I bit
+	msr	daifset, #2			// set I bit
+	mrs_s	x1, ICC_PMR_EL1			// save PMR
+	mov	x2, #ICC_PMR_EL1_UNMASKED
+	msr_s	ICC_PMR_EL1, x2			// unmask at PMR
+	dsb	sy				// WFI may enter a low-power mode
+	wfi
+	msr_s	ICC_PMR_EL1, x1			// restore PMR
+	msr	daif, x0			// restore I bit
+	ret
+ENDPROC(__cpu_do_idle)
+#else /* defined(CONFIG_HAS_NMI) */
+ENTRY(__cpu_do_idle)
+	dsb	sy				// WFI may enter a low-power mode
+	wfi
+	ret
+ENDPROC(__cpu_do_idle)
+#endif /* defined(CONFIG_HAS_NMI) */
+
+/*
+ *	cpu_do_switch_mm(pgd_phys, tsk)
+ *
+ *	Set the translation table base pointer to be pgd_phys.
+ *
+ *	- pgd_phys - physical address of new TTB
+ */
+ENTRY(cpu_do_switch_mm)
+	//mmid	w1, x1				// get mm->context.id
+	bfi	x0, x1, #48, #16		// set the ASID
+	msr	ttbr0_el1, x0			// set TTBR0
+	isb
+	ret
+ENDPROC(cpu_do_switch_mm)
+
+	.section ".text.init", #alloc, #execinstr
+
+/*
+ *	__cpu_setup
+ *
+ *	Initialise the processor for turning the MMU on.  Return in x0 the
+ *	value of the SCTLR_EL1 register.
+ */
+ENTRY(__cpu_setup)
+	tlbi	vmalle1				// Invalidate local TLB
+	dsb	nsh
+
+	mov	x0, #3 << 20
+
+	/* SVE */
+	mrs	x5, id_aa64pfr0_el1
+	ubfx	x5, x5, #ID_AA64PFR0_SVE_SHIFT, #4
+	cbz	x5, 1f
+
+	orr	x0, x0, #CPACR_EL1_ZEN		// SVE: trap disabled EL1 and EL0
+1:	msr	cpacr_el1, x0			// Enable FP/ASIMD
+
+	mov	x0, #1 << 12			// Reset mdscr_el1 and disable
+	msr	mdscr_el1, x0			// access to the DCC from EL0
+	isb					// Unmask debug exceptions now,
+	enable_dbg				// since this is per-cpu
+
+	/*
+	 * Memory region attributes for LPAE:
+	 *
+	 *   n = AttrIndx[2:0]
+	 *			n	MAIR
+	 *   DEVICE_nGnRnE	000	00000000
+	 *   DEVICE_nGnRE	001	00000100
+	 *   DEVICE_GRE		010	00001100
+	 *   NORMAL_NC		011	01000100
+	 *   NORMAL		100	11111111
+	 */
+	ldr	x5, =MAIR(0x00, MT_DEVICE_nGnRnE) | \
+		     MAIR(0x04, MT_DEVICE_nGnRE) | \
+		     MAIR(0x0c, MT_DEVICE_GRE) | \
+		     MAIR(0x44, MT_NORMAL_NC) | \
+		     MAIR(0xff, MT_NORMAL)
+	msr	mair_el1, x5
+	/*
+	 * Prepare SCTLR
+	 */
+	adr	x5, crval
+	ldp	w5, w6, [x5]
+	mrs	x0, sctlr_el1
+	bic	x0, x0, x5			// clear bits
+	orr	x0, x0, x6			// set bits
+	/*
+	 * Set/prepare TCR and TTBR. We use 512GB (39-bit) address range for
+	 * both user and kernel.
+	 */
+	ldr	x10, =TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \
+			TCR_TG_FLAGS | TCR_ASID16 | TCR_TBI0
+	/*
+	 * Read the PARange bits from ID_AA64MMFR0_EL1 and set the IPS bits in
+	 * TCR_EL1.
+	 */
+	mrs	x9, ID_AA64MMFR0_EL1
+	bfi	x10, x9, #32, #3
+	msr	tcr_el1, x10
+	ret					// return to head.S
+ENDPROC(__cpu_setup)
+
+	/*
+	 *                 n n            T
+	 *       U E      WT T UD     US IHBS
+	 *       CE0      XWHW CZ     ME TEEA S
+	 * .... .IEE .... NEAI TE.I ..AD DEN0 ACAM
+	 * 0011 0... 1101 ..0. ..0. 10.. .... .... < hardware reserved
+	 * .... .1.. .... 01.1 11.1 ..01 0001 1101 < software settings
+	 */
+	.type	crval, #object
+crval:
+	.word	0x000802e2			// clear
+	.word	0x0405d11d			// set
+

arch/arm64/kernel/psci.c

@@ -0,0 +1,152 @@
+/* psci.c COPYRIGHT FUJITSU LIMITED 2015-2018 */
+/* @ref.impl arch/arm64/kernel/psci.c */
+/*
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * Copyright (C) 2013 ARM Limited
+ *
+ * Author: Will Deacon <will.deacon@arm.com>
+ */
+
+#include <psci.h>
+#include <errno.h>
+#include <ihk/types.h>
+#include <compiler.h>
+#include <lwk/compiler.h>
+#include <ihk/debug.h>
+
+//#define DEBUG_PRINT_PSCI
+
+#ifdef DEBUG_PRINT_PSCI
+#undef DDEBUG_DEFAULT
+#define DDEBUG_DEFAULT DDEBUG_PRINT
+#endif
+
+#define PSCI_POWER_STATE_TYPE_POWER_DOWN	1
+
+extern uint64_t ihk_param_cpu_logical_map;
+static uint64_t *__cpu_logical_map = &ihk_param_cpu_logical_map;
+#define cpu_logical_map(cpu)    __cpu_logical_map[cpu]
+
+struct psci_power_state {
+	uint16_t id;
+	uint8_t type;
+	uint8_t affinity_level;
+};
+
+static int psci_to_linux_errno(int errno)
+{
+	switch (errno) {
+	case PSCI_RET_SUCCESS:
+		return 0;
+	case PSCI_RET_NOT_SUPPORTED:
+		return -EOPNOTSUPP;
+	case PSCI_RET_INVALID_PARAMS:
+		return -EINVAL;
+	case PSCI_RET_DENIED:
+		return -EPERM;
+	};
+
+	return -EINVAL;
+}
+
+static uint32_t psci_power_state_pack(struct psci_power_state state)
+{
+	return ((state.id << PSCI_0_2_POWER_STATE_ID_SHIFT)
+			& PSCI_0_2_POWER_STATE_ID_MASK) |
+		((state.type << PSCI_0_2_POWER_STATE_TYPE_SHIFT)
+		 & PSCI_0_2_POWER_STATE_TYPE_MASK) |
+		((state.affinity_level << PSCI_0_2_POWER_STATE_AFFL_SHIFT)
+		 & PSCI_0_2_POWER_STATE_AFFL_MASK);
+}
+
+static noinline int __invoke_psci_fn_hvc(uint64_t function_id, uint64_t arg0, uint64_t arg1,
+					 uint64_t arg2)
+{
+	asm volatile(
+			__asmeq("%0", "x0")
+			__asmeq("%1", "x1")
+			__asmeq("%2", "x2")
+			__asmeq("%3", "x3")
+			"hvc	#0\n"
+		: "+r" (function_id)
+		: "r" (arg0), "r" (arg1), "r" (arg2));
+
+	return function_id;
+}
+
+static noinline int __invoke_psci_fn_smc(uint64_t function_id, uint64_t arg0, uint64_t arg1,
+					 uint64_t arg2)
+{
+	asm volatile(
+			__asmeq("%0", "x0")
+			__asmeq("%1", "x1")
+			__asmeq("%2", "x2")
+			__asmeq("%3", "x3")
+			"smc	#0\n"
+		: "+r" (function_id)
+		: "r" (arg0), "r" (arg1), "r" (arg2));
+
+	return function_id;
+}
+
+
+static int (*invoke_psci_fn)(uint64_t, uint64_t, uint64_t, uint64_t) = NULL;
+
+#define PSCI_METHOD_INVALID	-1
+#define PSCI_METHOD_HVC		0
+#define PSCI_METHOD_SMC		1
+int psci_init(void)
+{
+	extern unsigned long ihk_param_psci_method;
+	int ret = 0;
+
+	if (ihk_param_psci_method == PSCI_METHOD_SMC) {
+		dkprintf("psci_init(): set invoke_psci_fn = __invoke_psci_fn_smc\n");
+		invoke_psci_fn = __invoke_psci_fn_smc;
+	} else if (ihk_param_psci_method == PSCI_METHOD_HVC) {
+		dkprintf("psci_init(): set invoke_psci_fn = __invoke_psci_fn_hvc\n");
+		invoke_psci_fn = __invoke_psci_fn_hvc;
+	} else {
+		ekprintf("psci_init(): ihk_param_psci_method invalid value. %ld\n", ihk_param_psci_method);
+		ret = -1;
+	}
+	return ret;
+}
+
+int psci_cpu_off(void)
+{
+	int err;
+	uint32_t fn, power_state;
+
+	struct psci_power_state state = {
+		.type = PSCI_POWER_STATE_TYPE_POWER_DOWN,
+	};
+
+	fn = PSCI_0_2_FN_CPU_OFF;
+	power_state = psci_power_state_pack(state);
+	err = invoke_psci_fn(fn, power_state, 0, 0);
+	return psci_to_linux_errno(err);
+}
+
+static int psci_cpu_on(unsigned long cpuid, unsigned long entry_point)
+{
+	int err;
+	uint32_t fn;
+
+	fn = PSCI_0_2_FN64_CPU_ON;
+	err = invoke_psci_fn(fn, cpuid, entry_point, 0);
+	return psci_to_linux_errno(err);
+}
+
+int cpu_psci_cpu_boot(unsigned int cpu, unsigned long pc)
+{
+	return psci_cpu_on(cpu_logical_map(cpu), pc);
+}