prerelease: 0.93: investigate smp_ihk_os_panic_notifier

Change-Id: I997b41f80038603261de2e8232b6b8ca200cd8cd

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.6)
+cmake_minimum_required(VERSION 3.11)
 
 if (NOT CMAKE_BUILD_TYPE)
 	set (CMAKE_BUILD_TYPE "Debug" CACHE STRING "Build type: Debug Release..." FORCE)
@@ -7,10 +7,10 @@ endif (NOT CMAKE_BUILD_TYPE)
 enable_language(C ASM)
 
 project(mckernel C ASM)
-set(MCKERNEL_VERSION "1.7.0")
+set(MCKERNEL_VERSION "1.7.1")
 
-# See "Fedora Packaging Guidlines -- Versioning"
-set(MCKERNEL_RELEASE "0.95")
+# See "Fedora Packaging Guidelines -- Versioning"
+set(MCKERNEL_RELEASE "0.93")
 
 set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake/modules)
 # for rpmbuild
@@ -50,6 +50,64 @@ if (ENABLE_WERROR)
 	add_compile_options("-Werror")
 endif(ENABLE_WERROR)
 
+execute_process(COMMAND bash -c "ls -ld /proc/tofu/ 2>/dev/null | wc -l"
+  OUTPUT_VARIABLE PROC_TOFU OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+if(PROC_TOFU STREQUAL "1")
+	option(ENABLE_TOFU "Built-in tofu driver support" ON)
+else()
+	option(ENABLE_TOFU "Built-in tofu driver support" OFF)
+endif()
+
+if(ENABLE_TOFU)
+	add_definitions(-DENABLE_TOFU)
+	set(KBUILD_C_FLAGS "${KBUILD_C_FLAGS} -DENABLE_TOFU")
+endif()
+
+# when compiling on a compute-node
+execute_process(COMMAND bash -c "grep $(hostname) /etc/opt/FJSVfefs/config/fefs_node1.csv 2>/dev/null | cut -d, -f2 | grep -o CN"
+  OUTPUT_VARIABLE FUGAKU_NODE_TYPE OUTPUT_STRIP_TRAILING_WHITESPACE)
+if(FUGAKU_NODE_TYPE STREQUAL "CN")
+	option(ENABLE_FUGAKU_HACKS "Fugaku hacks" ON)
+else()
+	option(ENABLE_FUGAKU_HACKS "Fugaku hacks" OFF)
+endif()
+
+if(ENABLE_FUGAKU_HACKS)
+	add_definitions(-DENABLE_FUGAKU_HACKS)
+	set(KBUILD_C_FLAGS "${KBUILD_C_FLAGS} -DENABLE_FUGAKU_HACKS")
+endif()
+
+# krm that mandates reserved memory amount >= available at boot time?
+execute_process(COMMAND bash -c "rpm -qi FJSVpxkrm-plugin-mckernel | awk '$1 == \"Version\" && $2 == \":\" { print $3 }'"
+  OUTPUT_VARIABLE KRM_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE)
+message("KRM_VERSION: ${KRM_VERSION}")
+
+if(NOT "${KRM_VERSION}" STREQUAL "" AND "${KRM_VERSION}" VERSION_LESS_EQUAL 4.0.1)
+	option(ENABLE_KRM_WORKAROUND "krm workaround" ON)
+else()
+	option(ENABLE_KRM_WORKAROUND "krm workaround" OFF)
+endif()
+
+if(ENABLE_KRM_WORKAROUND)
+	add_definitions(-DENABLE_KRM_WORKAROUND)
+	set(KBUILD_C_FLAGS "${KBUILD_C_FLAGS} -DENABLE_KRM_WORKAROUND")
+endif()
+
+# SIGSTOP instead of SIGSEGV, additional IHK Linux kmsg
+option(ENABLE_FUGAKU_DEBUG "Fugaku debug instrumentation" OFF)
+
+if(ENABLE_FUGAKU_DEBUG)
+	add_definitions(-DENABLE_FUGAKU_DEBUG)
+	set(KBUILD_C_FLAGS "${KBUILD_C_FLAGS} -DENABLE_FUGAKU_DEBUG")
+endif()
+
+option(PROFILE_ENABLE "System call profile" ON)
+if(PROFILE_ENABLE)
+	add_definitions(-DPROFILE_ENABLE)
+	set(KBUILD_C_FLAGS "${KBUILD_C_FLAGS} -DPROFILE_ENABLE")
+endif()
+
 option(ENABLE_LINUX_WORK_IRQ_FOR_IKC "Use Linux work IRQ for IKC IPI" ON)
 if (ENABLE_LINUX_WORK_IRQ_FOR_IKC)
 	set(KBUILD_C_FLAGS "${KBUILD_C_FLAGS} -DIHK_IKC_USE_LINUX_WORK_IRQ")
@@ -252,6 +310,11 @@ message("KBUILD_C_FLAGS: ${KBUILD_C_FLAGS}")
 message("MAP_KERNEL_START: ${MAP_KERNEL_START}")
 message("ENABLE_MEMDUMP: ${ENABLE_MEMDUMP}")
 message("ENABLE_PERF: ${ENABLE_PERF}")
+message("ENABLE_TOFU: ${ENABLE_TOFU}")
+message("ENABLE_FUGAKU_HACKS: ${ENABLE_FUGAKU_HACKS}")
+message("ENABLE_FUGAKU_DEBUG: ${ENABLE_FUGAKU_DEBUG}")
+message("ENABLE_KRM_WORKAROUND: ${ENABLE_KRM_WORKAROUND}")
+message("PROFILE_ENABLE: ${PROFILE_ENABLE}")
 message("ENABLE_RUSAGE: ${ENABLE_RUSAGE}")
 message("ENABLE_QLMPI: ${ENABLE_QLMPI}")
 message("ENABLE_UTI: ${ENABLE_UTI}")

arch/arm64/kernel/cpu.c

@@ -143,6 +143,11 @@ void arch_save_panic_regs(void *irq_regs)
 
 	clv = get_arm64_this_cpu_local();
 
+	/* If kernel mode PF occurred, unroll the causing call stack */
+	if (cpu_local_var(kernel_mode_pf_regs)) {
+		regs = cpu_local_var(kernel_mode_pf_regs);
+	}
+
 	/* For user-space, use saved kernel context */
 	if (regs->pc < USER_END) {
 		memset(clv->arm64_cpu_local_thread.panic_regs,
@@ -725,6 +730,49 @@ static void show_context_stack(struct pt_regs *regs)
 	}
 }
 
+#ifdef ENABLE_FUGAKU_HACKS
+void __show_context_stack(struct thread *thread,
+        unsigned long pc, uintptr_t sp, int kprintf_locked)
+{
+    uintptr_t stack_top;
+    unsigned long irqflags = 0;
+
+    stack_top = ALIGN_UP(sp, (uintptr_t)KERNEL_STACK_SIZE);
+
+    if (!kprintf_locked)
+        irqflags = kprintf_lock();
+
+    __kprintf("TID: %d, call stack (most recent first):\n",
+        thread->tid);
+    __kprintf("PC: %016lx, SP: %016lx\n", pc, sp);
+    for (;;) {
+        extern char _head[], _end[];
+        uintptr_t *fp, *lr;
+        fp = (uintptr_t *)sp;
+        lr = (uintptr_t *)(sp + 8);
+
+        if ((*fp <= sp)) {
+            break;
+        }
+
+        if ((*fp > stack_top)) {
+            break;
+        }
+
+        if ((*lr < (unsigned long)_head) ||
+            (*lr > (unsigned long)_end)) {
+            break;
+        }
+
+        __kprintf("PC: %016lx, SP: %016lx, FP: %016lx\n", *lr - 4, sp, *fp);
+        sp = *fp;
+    }
+
+    if (!kprintf_locked)
+        kprintf_unlock(irqflags);
+}
+#endif
+
 void handle_IPI(unsigned int vector, struct pt_regs *regs)
 {
 	struct ihk_mc_interrupt_handler *h;
@@ -786,6 +834,19 @@ void cpu_safe_halt(void)
 	cpu_enable_interrupt();
 }
 
+#ifdef ENABLE_FUGAKU_HACKS
+/*@
+  @ assigns \nothing;
+  @ ensures \interrupt_disabled == 0;
+  @*/
+void cpu_halt_panic(void)
+{
+	extern void __cpu_do_idle(void);
+	cpu_enable_interrupt();
+	__cpu_do_idle();
+}
+#endif
+
 #if defined(CONFIG_HAS_NMI)
 #include <arm-gic-v3.h>
 
@@ -851,6 +912,21 @@ unsigned long cpu_enable_interrupt_save(void)
 	return flags;
 }
 
+#ifdef ENABLE_FUGAKU_HACKS
+int cpu_interrupt_disabled(void)
+{
+	unsigned long flags;
+	unsigned long masked = ICC_PMR_EL1_MASKED;
+
+	asm volatile(
+		"mrs_s  %0, " __stringify(ICC_PMR_EL1)
+		: "=&r" (flags)
+		:
+		: "memory");
+	return (flags == masked);
+}
+#endif
+
 #else /* defined(CONFIG_HAS_NMI) */
 
 /* @ref.impl arch/arm64/include/asm/irqflags.h::arch_local_irq_enable */
@@ -1372,6 +1448,14 @@ void arch_print_stack(void)
 {
 }
 
+#ifdef ENABLE_FUGAKU_HACKS
+unsigned long arch_get_instruction_address(const void *reg)
+{
+	const struct pt_regs *regs = (struct pt_regs *)reg;
+	return regs->pc;
+}
+#endif
+
 void arch_show_interrupt_context(const void *reg)
 {
 	const struct pt_regs *regs = (struct pt_regs *)reg;

arch/arm64/kernel/fault.c

@@ -223,7 +223,12 @@ static int do_translation_fault(unsigned long addr,
 					  unsigned int esr,
 					  struct pt_regs *regs)
 {
+#ifdef ENABLE_TOFU
+	// XXX: Handle kernel space page faults for Tofu driver
+	//if (addr < USER_END)
+#else
 	if (addr < USER_END)
+#endif
 		return do_page_fault(addr, esr, regs);
 
 	do_bad_area(addr, esr, regs);

arch/arm64/kernel/include/arch-lock.h

@@ -9,6 +9,9 @@
 #include "affinity.h"
 #include <lwk/compiler.h>
 #include "config.h"
+#ifdef ENABLE_FUGAKU_HACKS
+#include <ihk/debug.h>
+#endif
 
 //#define DEBUG_SPINLOCK
 //#define DEBUG_MCS_RWLOCK
@@ -31,6 +34,10 @@ typedef struct {
 #endif /* __AARCH64EB__ */
 } __attribute__((aligned(4))) ihk_spinlock_t;
 
+#ifdef ENABLE_FUGAKU_HACKS
+extern ihk_spinlock_t *get_this_cpu_runq_lock(void);
+#endif
+
 extern void preempt_enable(void);
 extern void preempt_disable(void);
 
@@ -98,6 +105,18 @@ static int __ihk_mc_spinlock_trylock_noirq(ihk_spinlock_t *lock)
 	: "memory");
 
 	success = !tmp;
+
+#ifdef ENABLE_FUGAKU_HACKS
+#if 0
+	if (success) {
+		if (get_this_cpu_runq_lock() == lock &&
+				!cpu_interrupt_disabled()) {
+			kprintf("%s: WARNING: runq lock held without IRQs disabled?\n", __func__); \
+		}
+	}
+#endif
+#endif
+
 	if (!success) {
 		preempt_enable();
 	}
@@ -182,6 +201,14 @@ static void __ihk_mc_spinlock_lock_noirq(ihk_spinlock_t *lock)
 	: "=&r" (lockval), "=&r" (newval), "=&r" (tmp), "+Q" (*lock)
 	: "Q" (lock->owner), "I" (1 << TICKET_SHIFT)
 	: "memory");
+#ifdef ENABLE_FUGAKU_HACKS
+#if 0
+	if (get_this_cpu_runq_lock() == lock &&
+			!cpu_interrupt_disabled()) {
+		kprintf("%s: WARNING: runq lock held without IRQs disabled?\n", __func__); \
+	}
+#endif
+#endif
 }
 
 #ifdef DEBUG_SPINLOCK

arch/arm64/kernel/include/arch-memory.h

@@ -94,7 +94,11 @@ extern char _end[];
 # define LD_TASK_UNMAPPED_BASE	UL(0x0000080000000000)
 # define TASK_UNMAPPED_BASE	UL(0x0000100000000000)
 # define USER_END		UL(0x0000400000000000)
+#ifdef ENABLE_TOFU
+# define MAP_VMAP_START		UL(0xffff7bdfffff0000)
+#else
 # define MAP_VMAP_START		UL(0xffff780000000000)
+#endif
 # define MAP_VMAP_SIZE		UL(0x0000000100000000)
 # define MAP_FIXED_START	UL(0xffff7ffffbdd0000)
 # define MAP_ST_START		UL(0xffff800000000000)
@@ -142,6 +146,7 @@ extern char _end[];
 # define __PTL1_SHIFT  16
 # define PTL4_INDEX_MASK 0
 # define PTL3_INDEX_MASK ((UL(1) << 6) - 1)
+# define PTL3_INDEX_MASK_LINUX ((UL(1) << 10) - 1)
 # define PTL2_INDEX_MASK ((UL(1) << 13) - 1)
 # define PTL1_INDEX_MASK PTL2_INDEX_MASK
 # define __PTL4_CONT_SHIFT (__PTL4_SHIFT + 0)
@@ -829,7 +834,13 @@ static inline int pte_is_head(pte_t *ptep, pte_t *old, size_t cont_size)
 	return page_is_contiguous_head(ptep, cont_size);
 }
 
-struct page_table;
+typedef pte_t translation_table_t;
+struct page_table {
+	translation_table_t* tt;
+	translation_table_t* tt_pa;
+	int asid;
+};
+
 void arch_adjust_allocate_page_size(struct page_table *pt,
 				    uintptr_t fault_addr,
 				    pte_t *ptep,
@@ -849,7 +860,6 @@ void *map_fixed_area(unsigned long phys, unsigned long size, int uncachable);
 void set_address_space_id(struct page_table *pt, int asid);
 int get_address_space_id(const struct page_table *pt);
 
-typedef pte_t translation_table_t;
 void set_translation_table(struct page_table *pt, translation_table_t* tt);
 translation_table_t* get_translation_table(const struct page_table *pt);
 translation_table_t* get_translation_table_as_paddr(const struct page_table *pt);

arch/arm64/kernel/include/arch-string.h

@@ -10,4 +10,13 @@ extern void *__inline_memcpy(void *to, const void *from, size_t t);
 
 extern void *__inline_memset(void *s, unsigned long c, size_t count);
 
+#define ARCH_MEMCLEAR
+
+extern void __memclear(void *addr, unsigned long len, void *tmp);
+inline static void memclear(void *addr, unsigned long len)
+{
+	uint64_t q0q1[4];
+	__memclear(addr, len, (void *)&q0q1);
+}
+
 #endif	/* __HEADER_ARM64_COMMON_ARCH_TIMER_H */

arch/arm64/kernel/include/io.h

@@ -80,6 +80,10 @@ static inline uint64_t __raw_readq(const volatile void *addr)
 	return val;
 }
 
+/* IO barriers */
+#define __iormb()		rmb()
+#define __iowmb()		wmb()
+
 /*
  * Relaxed I/O memory access primitives. These follow the Device memory
  * ordering rules but do not guarantee any ordering relative to Normal memory
@@ -95,5 +99,20 @@ static inline uint64_t __raw_readq(const volatile void *addr)
 #define writel_relaxed(v,c)	((void)__raw_writel((uint32_t)(v),(c)))
 #define writeq_relaxed(v,c)	((void)__raw_writeq((uint64_t)(v),(c)))
 
+/*
+ * I/O memory access primitives. Reads are ordered relative to any
+ * following Normal memory access. Writes are ordered relative to any prior
+ * Normal memory access.
+ */
+#define readb(c)		({ uint8_t  __v = readb_relaxed(c); __iormb(); __v; })
+#define readw(c)		({ uint16_t __v = readw_relaxed(c); __iormb(); __v; })
+#define readl(c)		({ uint32_t __v = readl_relaxed(c); __iormb(); __v; })
+#define readq(c)		({ uint64_t __v = readq_relaxed(c); __iormb(); __v; })
+
+#define writeb(v,c)		({ __iowmb(); writeb_relaxed((v),(c)); })
+#define writew(v,c)		({ __iowmb(); writew_relaxed((v),(c)); })
+#define writel(v,c)		({ __iowmb(); writel_relaxed((v),(c)); })
+#define writeq(v,c)		({ __iowmb(); writeq_relaxed((v),(c)); })
+
 #endif /* __KERNEL__ */
 #endif /* __ASM_IO_H */

arch/arm64/kernel/include/syscall_list.h

@@ -124,7 +124,7 @@ SYSCALL_HANDLED(271, process_vm_writev)
 SYSCALL_HANDLED(281, execveat)
 SYSCALL_HANDLED(700, get_cpu_id)
 #ifdef PROFILE_ENABLE
-SYSCALL_HANDLED(__NR_profile, profile)
+SYSCALL_HANDLED(PROFILE_EVENT_MAX, profile)
 #endif // PROFILE_ENABLE
 SYSCALL_HANDLED(730, util_migrate_inter_kernel)
 SYSCALL_HANDLED(731, util_indicate_clone)

arch/arm64/kernel/include/thread_info.h

@@ -2,7 +2,7 @@
 #ifndef __HEADER_ARM64_COMMON_THREAD_INFO_H
 #define __HEADER_ARM64_COMMON_THREAD_INFO_H
 
-#define MIN_KERNEL_STACK_SHIFT	15
+#define MIN_KERNEL_STACK_SHIFT	18
 
 #include <arch-memory.h>
 

arch/arm64/kernel/irq-gic-v3.c

@@ -7,6 +7,9 @@
 #include <process.h>
 #include <syscall.h>
 #include <ihk/debug.h>
+#ifdef ENABLE_FUGAKU_HACKS
+#include <ihk/monitor.h>
+#endif
 #include <arch-timer.h>
 #include <cls.h>
 
@@ -313,14 +316,27 @@ void handle_interrupt_gicv3(struct pt_regs *regs)
 	struct cpu_local_var *v = get_this_cpu_local_var();
 	//unsigned long irqflags;
 	int do_check = 0;
+#ifdef ENABLE_FUGAKU_HACKS
+	struct ihk_os_cpu_monitor *monitor = cpu_local_var(monitor);
 
+	++v->in_interrupt;
+#endif
 	irqnr = gic_read_iar();
 	cpu_enable_nmi();
 	set_cputime(from_user ? CPUTIME_MODE_U2K : CPUTIME_MODE_K2K_IN);
 	while (irqnr != ICC_IAR1_EL1_SPURIOUS) {
 		if ((irqnr < 1020) || (irqnr >= 8192)) {
 			gic_write_eoir(irqnr);
+#ifndef ENABLE_FUGAKU_HACKS
 			handle_IPI(irqnr, regs);
+#else
+			/* Once paniced, only allow CPU stop and NMI IRQs */
+			if (monitor->status != IHK_OS_MONITOR_PANIC ||
+					irqnr == INTRID_CPU_STOP ||
+					irqnr == INTRID_MULTI_NMI) {
+				handle_IPI(irqnr, regs);
+			}
+#endif
 		}
 		irqnr = gic_read_iar();
 	}
@@ -335,7 +351,12 @@ void handle_interrupt_gicv3(struct pt_regs *regs)
 	}
 	//ihk_mc_spinlock_unlock(&v->runq_lock, irqflags);
 
+#ifndef ENABLE_FUGAKU_HACKS
 	if (do_check) {
+#else
+	--v->in_interrupt;
+	if (monitor->status != IHK_OS_MONITOR_PANIC && do_check) {
+#endif
 		check_signal(0, regs, 0);
 		schedule();
 	}

arch/arm64/kernel/memory.c

@@ -150,12 +150,6 @@ void flush_tlb_single(unsigned long addr)
 	arch_flush_tlb_single(asid, addr);
 }
 
-struct page_table {
-	translation_table_t* tt;
-	translation_table_t* tt_pa;
-	int asid;
-};
-
 extern struct page_table swapper_page_table;
 static struct page_table *init_pt = &swapper_page_table;
 static ihk_spinlock_t init_pt_lock;
@@ -223,6 +217,13 @@ static inline int ptl4_index(unsigned long addr)
 	int idx = (addr >> PTL4_SHIFT) & PTL4_INDEX_MASK;
 	return idx;
 }
+#ifdef ENABLE_TOFU
+static inline int ptl3_index_linux(unsigned long addr)
+{
+	int idx = (addr >> PTL3_SHIFT) & PTL3_INDEX_MASK_LINUX;
+	return idx;
+}
+#endif
 static inline int ptl3_index(unsigned long addr)
 {
 	int idx = (addr >> PTL3_SHIFT) & PTL3_INDEX_MASK;
@@ -281,6 +282,40 @@ static inline pte_t* ptl4_offset(const translation_table_t* ptl4, unsigned long
 	}
 	return ptep;
 }
+
+#ifdef ENABLE_TOFU
+static inline pte_t* ptl3_offset_linux(const pte_t* l4p, unsigned long addr)
+{
+	pte_t* ptep = NULL;
+	pte_t pte = 0;
+	unsigned long phys = 0;
+	translation_table_t* ptl3 = NULL;
+	int idx = 0;
+
+	switch (CONFIG_ARM64_PGTABLE_LEVELS)
+	{
+	case 4:
+		pte = ptl4_val(l4p);
+		phys = pte & PT_PHYSMASK;
+		ptl3 = phys_to_virt(phys);
+		idx = ptl3_index_linux(addr);
+		ptep = (pte_t*)ptl3 + idx;
+		break;
+	case 3:
+		ptl3 = (translation_table_t*)l4p;
+		idx = ptl3_index_linux(addr);
+		ptep = (pte_t*)ptl3 + idx;
+		break;
+	case 2:
+	case 1:
+		/* PTL3が無いときにはエントリではなくページテーブルのアドレスを引渡していく。*/
+		ptep = (pte_t*)l4p;
+		break;
+	}
+	return ptep;
+}
+#endif
+
 static inline pte_t* ptl3_offset(const pte_t* l4p, unsigned long addr)
 {
 	pte_t* ptep = NULL;
@@ -959,7 +994,14 @@ static void init_normal_area(struct page_table *pt)
 	int i;
 	
 	tt = get_translation_table(pt);
-	
+
+#ifdef ENABLE_TOFU
+	setup(tt,
+			arm64_st_phys_base,
+			arm64_st_phys_base  + (1UL << 40));
+	return;
+#endif
+
 	for (i = 0; i < ihk_mc_get_nr_memory_chunks(); i++) {
 		unsigned long map_start, map_end;
 		int numa_id;
@@ -1287,6 +1329,58 @@ out:
 	return ret;
 }
 
+#ifdef ENABLE_TOFU
+int ihk_mc_linux_pt_virt_to_phys_size(struct page_table *pt,
+                           const void *virt,
+						   unsigned long *phys,
+						   unsigned long *size)
+{
+	unsigned long v = (unsigned long)virt;
+	pte_t* ptep;
+	translation_table_t* tt;
+
+	unsigned long paddr;
+	unsigned long lsize;
+
+	tt = get_translation_table(pt);
+
+	ptep = ptl4_offset(tt, v);
+	if (!ptl4_present(ptep)) {
+		return -EFAULT;
+	}
+
+	ptep = ptl3_offset_linux(ptep, v);
+	if (!ptl3_present(ptep)) {
+		return -EFAULT;
+	}
+	if (ptl3_type_block(ptep)) {
+		paddr = ptl3_phys(ptep);
+		lsize = PTL3_SIZE;
+		goto out;
+	}
+
+	ptep = ptl2_offset(ptep, v);
+	if (!ptl2_present(ptep)) {
+		return -EFAULT;
+	}
+	if (ptl2_type_block(ptep)) {
+		paddr = ptl2_phys(ptep);
+		lsize = PTL2_SIZE;
+		goto out;
+	}
+
+	ptep = ptl1_offset(ptep, v);
+	if (!ptl1_present(ptep)) {
+		return -EFAULT;
+	}
+	paddr = ptl1_phys(ptep);
+	lsize = PTL1_SIZE;
+out:
+	*phys = paddr | (v & (lsize - 1));
+	if(size) *size = lsize;
+	return 0;
+}
+#endif
 
 int ihk_mc_pt_virt_to_phys_size(struct page_table *pt,
                            const void *virt,
@@ -1348,7 +1442,6 @@ int ihk_mc_pt_virt_to_phys(struct page_table *pt,
 	return ihk_mc_pt_virt_to_phys_size(pt, virt, phys, NULL);
 }
 
-
 int ihk_mc_pt_print_pte(struct page_table *pt, void *virt)
 {
 	const unsigned long v = (unsigned long)virt;
@@ -1360,6 +1453,15 @@ int ihk_mc_pt_print_pte(struct page_table *pt, void *virt)
 	}
 	tt = get_translation_table(pt);
 
+	__kprintf("%s: 0x%lx, CONFIG_ARM64_PGTABLE_LEVELS: %d, ptl4_index: %ld, ptl3_index: %ld, ptl2_index: %ld, ptl1_index: %ld\n", 
+		__func__,
+		v,
+		CONFIG_ARM64_PGTABLE_LEVELS,
+		ptl4_index(v),
+		ptl3_index(v),
+		ptl2_index(v),
+		ptl1_index(v));
+
 	ptep = ptl4_offset(tt, v);
 	__kprintf("l4 table: 0x%lX l4idx: %d\n", virt_to_phys(tt), ptl4_index(v));
 	if (!(ptl4_present(ptep))) {
@@ -2147,6 +2249,198 @@ static void unmap_free_stat(struct page *page, unsigned long phys,
 	}
 }
 
+/*
+ * Kernel space page table clearing functions.
+ */
+struct clear_kernel_range_args {
+	int free_physical;
+};
+
+static int clear_kernel_range_middle(void *args0, pte_t *ptep, uint64_t base,
+			      uint64_t start, uint64_t end, int level);
+
+static int clear_kernel_range_l1(void *args0, pte_t *ptep, uint64_t base,
+		uint64_t start, uint64_t end)
+{
+	const struct table {
+		unsigned long pgsize;
+		unsigned long cont_pgsize;
+	} tbl = {
+		.pgsize = PTL1_SIZE,
+		.cont_pgsize = PTL1_CONT_SIZE
+	};
+
+	struct clear_kernel_range_args *args = args0;
+	uint64_t phys = 0;
+	pte_t old;
+	size_t clear_size;
+
+	if (ptl1_null(ptep)) {
+		return -ENOENT;
+	}
+
+	old = xchg(ptep, PTE_NULL);
+	if (!pte_is_present(&old))
+		return 0;
+
+	arch_flush_tlb_single(0, base);
+	clear_size = pte_is_contiguous(&old) ?
+		tbl.cont_pgsize : tbl.pgsize;
+
+	dkprintf("%s: 0x%lx:%lu unmapped\n",
+		__func__, base, clear_size);
+
+	if (args->free_physical) {
+		phys = ptl1_phys(&old);
+		ihk_mc_free_pages(phys_to_virt(phys), clear_size >> PAGE_SHIFT);
+	}
+
+	return 0;
+}
+
+static int clear_kernel_range_l2(void *args0, pte_t *ptep, uint64_t base,
+		uint64_t start, uint64_t end)
+{
+	return clear_kernel_range_middle(args0, ptep, base, start, end, 2);
+}
+
+static int clear_kernel_range_l3(void *args0, pte_t *ptep, uint64_t base,
+		uint64_t start, uint64_t end)
+{
+	return clear_kernel_range_middle(args0, ptep, base, start, end, 3);
+}
+
+static int clear_kernel_range_l4(void *args0, pte_t *ptep, uint64_t base,
+		uint64_t start, uint64_t end)
+{
+	return clear_kernel_range_middle(args0, ptep, base, start, end, 4);
+}
+
+static int clear_kernel_range_middle(void *args0, pte_t *ptep, uint64_t base,
+			      uint64_t start, uint64_t end, int level)
+{
+	const struct table {
+		walk_pte_t* walk;
+		walk_pte_fn_t* callback;
+		unsigned long pgsize;
+		unsigned long cont_pgsize;
+	} table[] = {
+		{walk_pte_l1, clear_kernel_range_l1, PTL2_SIZE, PTL2_CONT_SIZE}, /*PTL2*/
+		{walk_pte_l2, clear_kernel_range_l2, PTL3_SIZE, PTL3_CONT_SIZE}, /*PTL3*/
+		{walk_pte_l3, clear_kernel_range_l3, PTL4_SIZE, PTL4_CONT_SIZE}, /*PTL4*/
+	};
+	const struct table tbl = table[level-2];
+
+	struct clear_kernel_range_args *args = args0;
+	uint64_t phys = 0;
+	translation_table_t *tt;
+	int error;
+	pte_t old;
+	size_t clear_size;
+
+	if (ptl_null(ptep, level)) {
+		return -ENOENT;
+	}
+
+	dkprintf("%s(level: %d): 0x%lx in 0x%lx-0x%lx\n",
+			__func__, level, base, start, end);
+
+	if (ptl_type_page(ptep, level)
+			&& ((base < start) || (end < (base + tbl.pgsize)))) {
+		error = -EINVAL;
+		ekprintf("clear_range_middle(%p,%p,%lx,%lx,%lx,%d):"
+			 "split page. %d\n",
+			 args0, ptep, base, start, end, level, error);
+		return error;
+	}
+
+	if (ptl_type_page(ptep, level)) {
+		old = xchg(ptep, PTE_NULL);
+
+		if (!ptl_present(&old, level)) {
+			return 0;
+		}
+
+		arch_flush_tlb_single(0, base);
+
+		clear_size = pte_is_contiguous(&old) ?
+			tbl.cont_pgsize : tbl.pgsize;
+
+		dkprintf("%s(level: %d): 0x%lx:%lu unmapped\n",
+				__func__, level, base, clear_size);
+
+		if (args->free_physical) {
+			phys = ptl_phys(&old, level);
+			ihk_mc_free_pages(phys_to_virt(phys), clear_size >> PAGE_SHIFT);
+		}
+
+		return 0;
+	}
+
+	tt = (translation_table_t*)phys_to_virt(ptl_phys(ptep, level));
+	error = tbl.walk(tt, base, start, end, tbl.callback, args0);
+	if (error && (error != -ENOENT)) {
+		return error;
+	}
+
+	if (args->free_physical) {
+		if ((start <= base) && ((base + tbl.pgsize) <= end)) {
+			ptl_clear(ptep, level);
+			arch_flush_tlb_single(0, base);
+			ihk_mc_free_pages(tt, 1);
+		}
+	}
+
+	return 0;
+}
+
+static int clear_kernel_range(uintptr_t start, uintptr_t end, int free_physical)
+{
+	const struct table {
+		walk_pte_t* walk;
+		walk_pte_fn_t* callback;
+	} tables[] = {
+		{walk_pte_l2, clear_kernel_range_l2}, /*second*/
+		{walk_pte_l3, clear_kernel_range_l3}, /*first*/
+		{walk_pte_l4, clear_kernel_range_l4}, /*zero*/
+	};
+	const struct table initial_lookup = tables[CONFIG_ARM64_PGTABLE_LEVELS - 2];
+
+	int error;
+	struct clear_kernel_range_args args;
+	translation_table_t* tt;
+	unsigned long irqflags;
+
+	dkprintf("%s: start: 0x%lx, end: 0x%lx, free phys: %d\n",
+		 __func__, start, end, free_physical);
+
+	if (start <= USER_END)
+		return -EINVAL;
+
+	args.free_physical = free_physical;
+
+	irqflags = ihk_mc_spinlock_lock(&init_pt_lock);
+	tt = get_translation_table(get_init_page_table());
+	error = initial_lookup.walk(tt, 0,
+			(start & ~(0xffff000000000000)),
+			(end & ~(0xffff000000000000)),
+			initial_lookup.callback, &args);
+	dkprintf("%s: start: 0x%lx, end: 0x%lx, free phys: %d, ret: %d\n",
+		 __func__, start, end, free_physical, error);
+
+	ihk_mc_spinlock_unlock(&init_pt_lock, irqflags);
+	return error;
+}
+
+int ihk_mc_clear_kernel_range(void *start, void *end)
+{
+#define	KEEP_PHYSICAL	0
+	return clear_kernel_range((uintptr_t)start, (uintptr_t)end, KEEP_PHYSICAL);
+}
+
+/*
+ * User space page table clearing functions.
+ */
 struct clear_range_args {
 	int free_physical;
 	struct memobj *memobj;
@@ -2344,6 +2638,14 @@ static int clear_range(struct page_table *pt, struct process_vm *vm,
 	if (memobj && ((memobj->flags & MF_PREMAP))) {
 		args.free_physical = 0;
 	}
+
+	if (vm->proc->straight_va &&
+			(void *)start == vm->proc->straight_va &&
+			(void *)end == (vm->proc->straight_va +
+				vm->proc->straight_len)) {
+		args.free_physical = 0;
+	}
+
 	args.memobj = memobj;
 	args.vm = vm;
 

arch/arm64/kernel/memset.S

@@ -218,3 +218,41 @@ ENTRY(__inline_memset)
 	ret
 ENDPIPROC(__inline_memset)
 ENDPROC(____inline_memset)
+
+
+/*
+ * Non-temporal vector memory clear
+ *
+ * Parameters:
+ *     x0 - buf (assumed to be aligned to page size)
+ *     x1 - n (assumed to be at least page size)
+ */
+ENTRY(__memclear)
+       stp     q0, q1, [x2] /* Preserve two 128 bit vector regs */
+       eor v0.16B, v0.16B, v0.16B
+       eor v1.16B, v1.16B, v1.16B
+1:
+       stnp q0, q1, [x0, #32 * 0]
+       stnp q0, q1, [x0, #32 * 1]
+       stnp q0, q1, [x0, #32 * 2]
+       stnp q0, q1, [x0, #32 * 3]
+       stnp q0, q1, [x0, #32 * 4]
+       stnp q0, q1, [x0, #32 * 5]
+       stnp q0, q1, [x0, #32 * 6]
+       stnp q0, q1, [x0, #32 * 7]
+       stnp q0, q1, [x0, #32 * 8]
+       stnp q0, q1, [x0, #32 * 9]
+       stnp q0, q1, [x0, #32 * 10]
+       stnp q0, q1, [x0, #32 * 11]
+       stnp q0, q1, [x0, #32 * 12]
+       stnp q0, q1, [x0, #32 * 13]
+       stnp q0, q1, [x0, #32 * 14]
+       stnp q0, q1, [x0, #32 * 15]
+       add x0, x0, #512
+       subs x1, x1, #512
+       cmp x1, #0
+       b.ne    1b
+
+       ldp     q0, q1, [x2] /* Restore vector regs */
+       ret
+ENDPROC(__memclear)

arch/arm64/kernel/syscall.c

@@ -1071,6 +1071,9 @@ static int setup_rt_frame(int usig, unsigned long rc, int to_restart,
 
 	if (k->sa.sa_flags & SA_RESTORER){
 		regs->regs[30] = (unsigned long)k->sa.sa_restorer;
+#ifdef ENABLE_FUGAKU_HACKS
+		kprintf("%s: SA_RESTORER: 0x%lx\n", __func__, regs->regs[30]);
+#endif
 	} else {
 		regs->regs[30] = (unsigned long)VDSO_SYMBOL(thread->vm->vdso_addr, sigtramp);
 	}
@@ -1723,9 +1726,18 @@ SYSCALL_DECLARE(mmap)
 
 	/* check arguments */
 	pgsize = PAGE_SIZE;
+#ifndef ENABLE_FUGAKU_HACKS
 	if (flags & MAP_HUGETLB) {
 		int hugeshift = flags & (0x3F << MAP_HUGE_SHIFT);
 
+		/* OpenMPI expects -EINVAL when trying to map
+		 * /dev/shm/ file with MAP_SHARED | MAP_HUGETLB
+		 */
+		if (!(flags & MAP_ANONYMOUS)) {
+			error = -EINVAL;
+			goto out;
+		}
+
 		if (hugeshift == 0) {
 			/* default hugepage size */
 			flags |= ihk_mc_get_linux_default_huge_page_shift() <<
@@ -1755,6 +1767,11 @@ SYSCALL_DECLARE(mmap)
 			goto out;
 		}
 	}
+#else
+	if (flags & MAP_HUGETLB) {
+		flags &= ~(MAP_HUGETLB);
+	}
+#endif
 
 #define	VALID_DUMMY_ADDR	((region->user_start + PTL3_SIZE - 1) & ~(PTL3_SIZE - 1))
 	addr = (flags & MAP_FIXED)? addr0: VALID_DUMMY_ADDR;

arch/arm64/kernel/traps.c

@@ -174,9 +174,14 @@ void bad_mode(struct pt_regs *regs, int reason, unsigned int esr)
 
 	arch_show_interrupt_context(regs);
 
+#ifdef ENABLE_TOFU
+	info.si_signo = SIGSTOP;
+	info.si_errno = 0;
+#else
 	info.si_signo = SIGILL;
 	info.si_errno = 0;
 	info.si_code  = ILL_ILLOPC;
+#endif
 	info._sifields._sigfault.si_addr  = (void*)regs->pc;
 
 	arm64_notify_die("Oops - bad mode", regs, &info, 0);

arch/x86_64/kernel/cpu.c

@@ -868,6 +868,49 @@ void show_context_stack(uintptr_t *rbp) {
 	return;
 }
 
+#ifdef ENABLE_FUGAKU_HACKS
+void __show_context_stack(struct thread *thread,
+        unsigned long pc, uintptr_t sp, int kprintf_locked)
+{
+    uintptr_t stack_top;
+    unsigned long irqflags = 0;
+
+    stack_top = ALIGN_UP(sp, (uintptr_t)KERNEL_STACK_SIZE);
+
+    if (!kprintf_locked)
+        irqflags = kprintf_lock();
+
+    __kprintf("TID: %d, call stack (most recent first):\n",
+        thread->tid);
+    __kprintf("PC: %016lx, SP: %016lx\n", pc, sp);
+    for (;;) {
+        extern char _head[], _end[];
+        uintptr_t *fp, *lr;
+        fp = (uintptr_t *)sp;
+        lr = (uintptr_t *)(sp + 8);
+
+        if ((*fp <= sp)) {
+            break;
+        }
+
+        if ((*fp > stack_top)) {
+            break;
+        }
+
+        if ((*lr < (unsigned long)_head) ||
+            (*lr > (unsigned long)_end)) {
+            break;
+        }
+
+        __kprintf("PC: %016lx, SP: %016lx, FP: %016lx\n", *lr - 4, sp, *fp);
+        sp = *fp;
+    }
+
+    if (!kprintf_locked)
+        kprintf_unlock(irqflags);
+}
+#endif
+
 void interrupt_exit(struct x86_user_context *regs)
 {
 	if (interrupt_from_user(regs)) {
@@ -1137,6 +1180,17 @@ void cpu_halt(void)
 	asm volatile("hlt");
 }
 
+#ifdef ENABLE_FUGAKU_HACKS
+/*@
+  @ assigns \nothing;
+  @ ensures \interrupt_disabled == 0;
+  @*/
+void cpu_halt_panic(void)
+{
+    cpu_halt();
+}
+#endif
+
 /*@
   @ assigns \nothing;
   @ ensures \interrupt_disabled == 0;
@@ -1521,6 +1575,16 @@ void arch_print_stack(void)
 	__print_stack(rbp, 0);
 }
 
+#ifdef ENABLE_FUGAKU_HACKS
+unsigned long arch_get_instruction_address(const void *reg)
+{
+	const struct x86_user_context *uctx = reg;
+	const struct x86_basic_regs *regs = &uctx->gpr;
+
+	return regs->rip;
+}
+#endif
+
 /*@
   @ requires \valid(reg);
   @ assigns \nothing;

arch/x86_64/kernel/include/arch-memory.h

@@ -451,4 +451,12 @@ extern unsigned long ap_trampoline;
 
 /* Local is cachable */
 #define IHK_IKC_QUEUE_PT_ATTR (PTATTR_NO_EXECUTE | PTATTR_WRITABLE)
+
+#ifdef ENABLE_FUGAKU_HACKS
+#ifndef __ASSEMBLY__
+# define ALIGN_UP(x, align)     ALIGN_DOWN((x) + (align) - 1, align)
+# define ALIGN_DOWN(x, align)   ((x) & ~((align) - 1))
+#endif /* !__ASSEMBLY__ */
+#endif
+
 #endif

arch/x86_64/kernel/include/cpulocal.h

@@ -53,5 +53,9 @@ struct x86_cpu_local_variables *get_x86_this_cpu_local(void);
 void *get_x86_cpu_local_kstack(int id);
 void *get_x86_this_cpu_kstack(void);
 
+#ifdef ENABLE_FUGAKU_HACKS
+#define LOCALS_SPAN (4 * PAGE_SIZE)
+#define KERNEL_STACK_SIZE LOCALS_SPAN
+#endif
 
 #endif

arch/x86_64/kernel/include/syscall_list.h

@@ -168,7 +168,7 @@ SYSCALL_HANDLED(311, process_vm_writev)
 SYSCALL_HANDLED(322, execveat)
 SYSCALL_HANDLED(700, get_cpu_id)
 #ifdef PROFILE_ENABLE
-SYSCALL_HANDLED(__NR_profile, profile)
+SYSCALL_HANDLED(PROFILE_EVENT_MAX, profile)
 #endif // PROFILE_ENABLE
 SYSCALL_HANDLED(730, util_migrate_inter_kernel)
 SYSCALL_HANDLED(731, util_indicate_clone)

arch/x86_64/kernel/local.c

@@ -21,7 +21,9 @@
 #include <registers.h>
 #include <string.h>
 
+#ifndef ENABLE_FUGAKU_HACKS
 #define LOCALS_SPAN (4 * PAGE_SIZE)
+#endif
 
 struct x86_cpu_local_variables *locals;
 size_t x86_cpu_local_variables_span = LOCALS_SPAN;	/* for debugger */

arch/x86_64/kernel/memory.c

@@ -1651,6 +1651,14 @@ static int clear_range(struct page_table *pt, struct process_vm *vm,
 	if (memobj && ((memobj->flags & MF_PREMAP))) {
 		args.free_physical = 0;
 	}
+
+	if (vm->proc->straight_va &&
+			(void *)start == vm->proc->straight_va &&
+			(void *)end == (vm->proc->straight_va +
+				vm->proc->straight_len)) {
+		args.free_physical = 0;
+	}
+
 	args.memobj = memobj;
 	args.vm = vm;
 

arch/x86_64/kernel/syscall.c

@@ -1430,6 +1430,14 @@ SYSCALL_DECLARE(mmap)
 	/* check arguments */
 	pgsize = PAGE_SIZE;
 	if (flags & MAP_HUGETLB) {
+		/* OpenMPI expects -EINVAL when trying to map
+		 * /dev/shm/ file with MAP_SHARED | MAP_HUGETLB
+		 */
+		if (!(flags & MAP_ANONYMOUS)) {
+			error = -EINVAL;
+			goto out;
+		}
+
 		switch (flags & (0x3F << MAP_HUGE_SHIFT)) {
 		case 0:
 			/* default hugepage size */

cmake/modules/Kbuild.cmake

@@ -30,6 +30,9 @@ endif ()
 
 if (NOT "${LINUX_ARCH}" STREQUAL "${CMAKE_HOST_SYSTEM_PROCESSOR}")
 	string(REGEX REPLACE "ld$" "" CROSS_COMPILE "${CMAKE_LINKER}")
+	if (CMAKE_CROSSCOMPILING)
+	   list(APPEND KBUILD_MAKE_FLAGS "QEMU_LD_PREFIX=${CMAKE_FIND_ROOT_PATH}")
+	endif()
 	list(APPEND KBUILD_MAKE_FLAGS "ARCH=${ARCH}")
 	list(APPEND KBUILD_MAKE_FLAGS "CROSS_COMPILE=${CROSS_COMPILE}")
 endif()

docs/NEWS.rst

@@ -1,11 +1,12 @@
 =============================================
-Version 1.7.0-0.93 (Aug 1, 2020)
+Version 1.7.0 (Nov 25, 2020)
 =============================================
 
 ----------------------
 IHK major updates
 ----------------------
 #. ihklib: add ihk_create_os_str
+#. ihklib: ihk_reserve_mem: add capped best effort to balanced
 
 ------------------------
 IHK major bug fixes

docs/install.rst

@@ -129,11 +129,29 @@ Create the tarball and the spec file:
    make dist
    cp mckernel-<version>.tar.gz <rpmbuild>/SOURCES
 
+(optional) Edit the following line in ``scripts/mckernel.spec`` to change
+cmake options. For example:
+
+::
+
+   %cmake -DCMAKE_BUILD_TYPE=Release \
+	-DUNAME_R=%{kernel_version} \
+	-DKERNEL_DIR=%{kernel_dir} \
+	%{?cmake_libdir:-DCMAKE_INSTALL_LIBDIR=%{cmake_libdir}} \
+	%{?build_target:-DBUILD_TARGET=%{build_target}} \
+	%{?toolchain_file:-DCMAKE_TOOLCHAIN_FILE=%{toolchain_file}} \
+	-DENABLE_TOFU=ON -DENABLE_FUGAKU_HACKS=ON \
+	-DENABLE_KRM_WORKAROUND=OFF -DWITH_KRM=ON \
+	-DENABLE_FUGAKU_DEBUG=OFF \
+	.
+
 Create the rpm package:
 
 When not cross-compiling:
 """""""""""""""""""""""""
 
+Then build the rpm:
+
 ::
 
    rpmbuild -ba scripts/mckernel.spec

docs/spec/ihk.rst

@@ -2,17 +2,6 @@
    :suffix: .
    :depth: 3
 
-External Specs
+Specifications
 ==============
-
-Overview
---------
-
-Function Specs
---------------
-
-Command / Daemon Specs
-----------------------
-
-Booting LWK
-===========
+The specifications pdf is :download:`here <ihk.pdf>`

docs/spec/mckernel.rst

@@ -2,8 +2,6 @@
    :suffix: .
    :depth: 3
 
-Interfaces
-==========
-
-Interface details
-=================
+Specifications
+==============
+The specifications pdf is :download:`here <mckernel.pdf>`

docs/users.rst

@@ -34,6 +34,19 @@ For example, with Fujitsu Technical Computing Suite (TCS), you need to specify `
 
    #PJM -L jobenv=mck1
 
+(Optional, Fujitsu TCS only) Specify boot parameters
+----------------------------------------------------
+
+You can specify the boot parameters by defining environmental variables and pass them to Fujitsu TCS.
+The parameters include the resource reservation settings, resource reservation amount, kernel arguments and routing of message channels between McKernel CPUs and Linux CPUs.
+See `IHK Specifications - ihk_create_os_str() <spec/ihk.html>`__ for the parameter names and allowed values.
+The example of setting the memory amount is shown below.
+
+.. code-block:: none
+
+   export IHK_MEM="7G@4,7G@5,7G@6,7G@7"
+   pjsub -X run.sh
+
 Insert ``mcexec`` into the command line
 ---------------------------------------
 
@@ -183,3 +196,9 @@ Limitations
 
 26. mmap() allows unlimited overcommit. Note that it corresponds to
     setting sysctl ``vm.overcommit_memory`` to 1.
+
+27. mlockall() is not supported and returns -EPERM.
+
+28. munlockall() is not supported and returns zero.
+
+29. scheduling behavior is not Linux compatible. For example, sometimes one of the two processes on the same CPU continues to run after yielding.

docs/uti.rst

@@ -4,24 +4,41 @@ Advanced: Enable Utility Thread offloading Interface (UTI)
 UTI enables a runtime such as MPI runtime to spawn utility threads such
 as MPI asynchronous progress threads to Linux cores.
 
-Install capstone
-~~~~~~~~~~~~~~~~~~~~
+Install ``capstone`` and ``capstone-devel``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-When compute nodes don't have access to repositories
-""""""""""""""""""""""""""""""""""""""""""""""""""""
+When compute nodes don't have access to EPEL repository
+"""""""""""""""""""""""""""""""""""""""""""""""""""""""
 
-Install EPEL capstone-devel:
+Install EPEL ``capstone`` and ``capstone-devel``:
 
 ::
 
    sudo yum install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
-   sudo yum install capstone-devel
+   sudo yum install capstone capstone-devel
 
 
-When compute nodes don't have access to repositories
-""""""""""""""""""""""""""""""""""""""""""""""""""""
+When compute nodes don't have access to EPEL repository
+"""""""""""""""""""""""""""""""""""""""""""""""""""""""
 
-Ask the system administrator to install ``capstone-devel``. Note that it is in the EPEL repository.
+A. Ask the system administrator to install ``capstone`` and ``capstone-devel``. Note that it is in the EPEL repository.
+
+B. Download the rpm with the machine in which you are the administrator:
+
+::
+
+   sudo yum install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
+   sudo yum install yum-utils
+   yumdownloader capstone capstone-devel
+
+and then install it to your home directory of the login node:
+
+::
+
+   cd $HOME/$(uname -p)
+   rpm2cpio capstone-4.0.1-9.el8.aarch64.rpm | cpio -idv
+   rpm2cpio capstone-devel-4.0.1-9.el8.aarch64.rpm | cpio -idv
+   sed -i 's#/usr/#'"$HOME"'/'"$(uname -p)"'/usr/#' $HOME/$(uname -p)/usr/lib64/pkgconfig/capstone.pc
 
 
 Install syscall_intercept
@@ -31,7 +48,24 @@ Install syscall_intercept
 
    git clone https://github.com/RIKEN-SysSoft/syscall_intercept.git
    mkdir build && cd build
-   cmake <syscall_intercept>/arch/aarch64 -DCMAKE_INSTALL_PREFIX=<syscall-intercept-install> -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=gcc -DTREAT_WARNINGS_AS_ERRORS=OFF
+
+When ``capstone`` and ``capstone-devel`` are installed into the system directory:
+
+::
+
+   cmake ../syscall_intercept/arch/aarch64 -DCMAKE_INSTALL_PREFIX=${HOME}/$(uname -p)/usr -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=gcc -DTREAT_WARNINGS_AS_ERRORS=OFF
+
+When ``capstone`` and ``capstone-devel`` are installed into your home directory:
+
+::
+
+   CMAKE_PREFIX_PATH=${HOME}/$(uname -p)/usr cmake ../syscall_intercept/arch/aarch64 -DCMAKE_INSTALL_PREFIX=${HOME}/$(uname -p)/usr -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=gcc -DTREAT_WARNINGS_AS_ERRORS=OFF
+
+Install:
+
+::
+
+   make && make install && make test
 
 Install UTI for McKernel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -48,16 +82,17 @@ Install:
 Install McKernel
 ~~~~~~~~~~~~~~~~~~~~
 
-Add ``-DENABLE_UTI=ON`` option to ``cmake``:
+``cmake`` with the additional options:
 
 ::
 
-   CMAKE_PREFIX_PATH=<syscall-intercept-install> cmake -DCMAKE_INSTALL_PREFIX=${HOME}/ihk+mckernel -DENABLE_UTI=ON $HOME/src/ihk+mckernel/mckernel
+   CMAKE_PREFIX_PATH=${HOME}/$(uname -p)/usr cmake -DCMAKE_INSTALL_PREFIX=${HOME}/ihk+mckernel -DENABLE_UTI=ON $HOME/src/ihk+mckernel/mckernel
+   make -j install
 
 Run programs
 ~~~~~~~~~~~~~~~~
 
-Add ``--enable-uti`` option to ``mcexec``:
+``mcexec`` with ``--enable-uti`` option:
 
 ::
 

executer/include/defs.h

@@ -4671,7 +4671,7 @@ void cmd_ipcs(void);         /* ipcs.c */
 /*
  *  main.c
  */
-void main_loop(void);
+//void main_loop(void);
 void exec_command(void);
 struct command_table_entry *get_command_table_entry(char *);
 void program_usage(int);

executer/include/uprotocol.h

@@ -94,6 +94,7 @@ struct get_cpu_set_arg {
 	char *req_cpu_list;   // Requested by user-space
 	int req_cpu_list_len; // Lenght of request string
 	int *process_rank;
+	pid_t ppid;
 	void *cpu_set;
 	size_t cpu_set_size;	// Size in bytes
 	int *target_core;
@@ -112,6 +113,18 @@ typedef unsigned long __cpu_set_unit;
 #define MPOL_NO_BSS               0x04
 #define MPOL_SHM_PREMAP           0x08
 
+/* should be the same as process.h */
+#define PLD_PROCESS_NUMA_MASK_BITS 256
+
+enum {
+	PLD_MPOL_DEFAULT,
+	PLD_MPOL_PREFERRED,
+	PLD_MPOL_BIND,
+	PLD_MPOL_INTERLEAVE,
+	PLD_MPOL_LOCAL,
+	PLD_MPOL_MAX,	/* always last member of enum */
+};
+
 #define PLD_MAGIC 0xcafecafe44332211UL
 
 struct program_load_desc {
@@ -146,9 +159,18 @@ struct program_load_desc {
 	unsigned long heap_extension;
 	long stack_premap;
 	unsigned long mpol_bind_mask;
+	int mpol_mode;
+	unsigned long mpol_nodemask[PLD_PROCESS_NUMA_MASK_BITS /
+				    (sizeof(unsigned long) * 8)];
+
 	int thp_disable;
 	int uti_thread_rank; /* N-th clone() spawns a thread on Linux CPU */
 	int uti_use_last_cpu; /* Work-around not to share CPU with OpenMP thread */
+	int straight_map;
+	size_t straight_map_threshold;
+#ifdef ENABLE_TOFU
+	int enable_tofu;
+#endif
 	int nr_processes;
 	int process_rank;
 	__cpu_set_unit cpu_set[PLD_CPU_SET_SIZE];
@@ -195,6 +217,9 @@ struct syscall_response {
 	unsigned long req_thread_status;
 	long ret;
 	unsigned long fault_address;
+#ifdef ENABLE_TOFU
+	void *pde_data;
+#endif
 };
 
 struct syscall_ret_desc {

executer/kernel/mcctrl/arch/arm64/archdeps.c

@@ -2,11 +2,16 @@
 #include <linux/version.h>
 #include <linux/mm_types.h>
 #include <linux/kallsyms.h>
+#include <linux/delay.h>
 #if KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE
 #include <linux/sched/task_stack.h>
 #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) */
 #include <linux/ptrace.h>
 #include <linux/uaccess.h>
+#include <linux/mmu_notifier.h>
+#include <linux/kref.h>
+#include <linux/file.h>
+#include <linux/proc_fs.h>
 #include <asm/vdso.h>
 #include "config.h"
 #include "../../mcctrl.h"
@@ -27,6 +32,39 @@ void *vdso_end;
 static struct vm_special_mapping (*vdso_spec)[2];
 #endif
 
+#ifdef ENABLE_TOFU
+/* Tofu CQ and barrier gate release functions */
+struct file_operations *mcctrl_tof_utofu_procfs_ops_cq;
+int (*mcctrl_tof_utofu_release_cq)(struct inode *inode,
+		struct file *filp);
+struct file_operations *mcctrl_tof_utofu_procfs_ops_bch;
+int (*mcctrl_tof_utofu_release_bch)(struct inode *inode,
+		struct file *filp);
+int (*mcctrl_tof_core_cq_cacheflush)(int tni, int cqid);
+int (*mcctrl_tof_core_disable_bch)(int tni, int bgid);
+int (*mcctrl_tof_core_unset_bg)(int tni, int bgid);
+typedef void (*tof_core_signal_handler)(int, int, uint64_t, uint64_t);
+void (*mcctrl_tof_core_register_signal_bg)(int tni, int bgid,
+			tof_core_signal_handler handler);
+struct tof_utofu_bg;
+struct tof_utofu_bg *mcctrl_tof_utofu_bg;
+
+
+/* Tofu MMU notifier */
+struct mmu_notifier_ops *mcctrl_tof_utofu_mn_ops;
+struct mmu_notifier_ops __mcctrl_tof_utofu_mn_ops;
+static void (*mcctrl_tof_utofu_mn_invalidate_range_end)(
+	struct mmu_notifier *mn,
+	struct mm_struct *mm,
+	unsigned long start,
+	unsigned long end);
+void __mcctrl_tof_utofu_mn_invalidate_range_end(
+	struct mmu_notifier *mn,
+	struct mm_struct *mm,
+	unsigned long start,
+	unsigned long end);
+#endif
+
 int arch_symbols_init(void)
 {
 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 0, 0)
@@ -43,6 +81,71 @@ int arch_symbols_init(void)
 		return -EFAULT;
 #endif
 
+#ifdef ENABLE_TOFU
+	mcctrl_tof_utofu_procfs_ops_cq =
+		(void *)kallsyms_lookup_name("tof_utofu_procfs_ops_cq");
+	if (WARN_ON(!mcctrl_tof_utofu_procfs_ops_cq))
+		return -EFAULT;
+
+	mcctrl_tof_utofu_procfs_ops_bch =
+		(void *)kallsyms_lookup_name("tof_utofu_procfs_ops_bch");
+	if (WARN_ON(!mcctrl_tof_utofu_procfs_ops_bch))
+		return -EFAULT;
+
+	mcctrl_tof_utofu_release_cq =
+		(void *)kallsyms_lookup_name("tof_utofu_release_cq");
+	if (WARN_ON(!mcctrl_tof_utofu_release_cq))
+		return -EFAULT;
+
+	mcctrl_tof_utofu_release_bch =
+		(void *)kallsyms_lookup_name("tof_utofu_release_bch");
+	if (WARN_ON(!mcctrl_tof_utofu_release_bch))
+		return -EFAULT;
+
+	mcctrl_tof_core_cq_cacheflush =
+		(void *)kallsyms_lookup_name("tof_core_cq_cacheflush");
+	if (WARN_ON(!mcctrl_tof_core_cq_cacheflush))
+		return -EFAULT;
+
+	mcctrl_tof_core_disable_bch =
+		(void *)kallsyms_lookup_name("tof_core_disable_bch");
+	if (WARN_ON(!mcctrl_tof_core_disable_bch))
+		return -EFAULT;
+
+	mcctrl_tof_core_unset_bg =
+		(void *)kallsyms_lookup_name("tof_core_unset_bg");
+	if (WARN_ON(!mcctrl_tof_core_unset_bg))
+		return -EFAULT;
+
+	mcctrl_tof_core_register_signal_bg =
+		(void *)kallsyms_lookup_name("tof_core_register_signal_bg");
+	if (WARN_ON(!mcctrl_tof_core_register_signal_bg))
+		return -EFAULT;
+
+	mcctrl_tof_utofu_bg =
+		(void *)kallsyms_lookup_name("tof_utofu_bg");
+	if (WARN_ON(!mcctrl_tof_utofu_bg))
+		return -EFAULT;
+
+	mcctrl_tof_utofu_mn_ops =
+		(void *)kallsyms_lookup_name("tof_utofu_mn_ops");
+	if (WARN_ON(!mcctrl_tof_utofu_mn_ops))
+		return -EFAULT;
+	/*
+	 * Copy original content and update redirected function,
+	 * CQ will be pointed to this structure after init ioctl()
+	 */
+	memcpy(&__mcctrl_tof_utofu_mn_ops, mcctrl_tof_utofu_mn_ops,
+			sizeof(*mcctrl_tof_utofu_mn_ops));
+	__mcctrl_tof_utofu_mn_ops.invalidate_range =
+		__mcctrl_tof_utofu_mn_invalidate_range_end;
+
+	mcctrl_tof_utofu_mn_invalidate_range_end =
+		(void *)kallsyms_lookup_name("tof_utofu_mn_invalidate_range_end");
+	if (WARN_ON(!mcctrl_tof_utofu_mn_invalidate_range_end))
+		return -EFAULT;
+#endif
+
 	return 0;
 }
 
@@ -331,6 +434,15 @@ int translate_rva_to_rpa(ihk_os_t os, unsigned long rpt, unsigned long rva,
 
 	// page table to translation_table.
 	phys = ihk_device_map_memory(ihk_os_to_dev(os), rpt, PAGE_SIZE);
+#ifdef ENABLE_FUGAKU_HACKS
+	if (!phys) {
+		pr_err("%s(): ERROR: VA: 0x%lx, rpt is NULL for PID %d\n",
+			__func__, rva, task_tgid_vnr(current));
+		error = -EFAULT;
+		goto out;
+	}
+#endif
+
 	tbl = ihk_device_map_virtual(ihk_os_to_dev(os), phys, PAGE_SIZE, NULL, 0);
 	rpa = (unsigned long)tbl->tt_pa;
 
@@ -417,3 +529,488 @@ long arch_switch_ctx(struct uti_switch_ctx_desc *desc)
 out:
 	return rc;
 }
+
+
+#ifdef ENABLE_TOFU
+/*
+ * Tofu CQ and BCH release handlers
+ */
+int __mcctrl_tof_utofu_release_cq(struct inode *inode, struct file *filp);
+int __mcctrl_tof_utofu_release_bch(struct inode *inode, struct file *filp);
+
+void mcctrl_tofu_hijack_release_handlers(void)
+{
+	mcctrl_tof_utofu_procfs_ops_cq->release =
+		__mcctrl_tof_utofu_release_cq;
+	mcctrl_tof_utofu_procfs_ops_bch->release =
+		__mcctrl_tof_utofu_release_bch;
+	wmb();
+}
+
+void mcctrl_tofu_restore_release_handlers(void)
+{
+	mcctrl_tof_utofu_procfs_ops_cq->release =
+		mcctrl_tof_utofu_release_cq;
+	mcctrl_tof_utofu_procfs_ops_bch->release =
+		mcctrl_tof_utofu_release_bch;
+	wmb();
+}
+
+/*
+ * Tofu cleanup functions
+ */
+#include <tofu/tof_uapi.h>
+#include <tofu/tof_icc.h>
+#include <tofu/tofu_generated-tof_core_cq.h>
+#include <tofu/tofu_generated-tof_utofu_device.h>
+#include <tofu/tofu_generated-tof_utofu_cq.h>
+#include <tofu/tofu_generated-tof_utofu_mbpt.h>
+#include <tofu/tofu_generated-tof_utofu_bg.h>
+
+#define TOF_UTOFU_VERSION TOF_UAPI_VERSION
+#define TOF_UTOFU_NUM_STAG_NTYPES 3
+#define TOF_UTOFU_NUM_STAG_BITS(size) ((size) + 13)
+#define TOF_UTOFU_NUM_STAG(size) ((uint64_t)1 << TOF_UTOFU_NUM_STAG_BITS(size))
+#define TOF_UTOFU_STAG_TRANS_BITS 3
+#define TOF_UTOFU_STAG_TRANS_SIZE ((uint64_t)1 << TOF_UTOFU_STAG_TRANS_BITS)
+#define TOF_UTOFU_STAG_TRANS_TABLE_LEN(size) (TOF_UTOFU_NUM_STAG(size) * TOF_UTOFU_STAG_TRANS_SIZE)
+#define TOF_UTOFU_STEERING_TABLE_LEN(size) (TOF_UTOFU_NUM_STAG(size) * TOF_ICC_STEERING_SIZE)
+#define TOF_UTOFU_MB_TABLE_LEN(size) (TOF_UTOFU_NUM_STAG(size) * TOF_ICC_MB_SIZE)
+#define TOF_UTOFU_STAG_MEM_LEN(size) (TOF_UTOFU_STEERING_TABLE_LEN(size) * 4)
+#define TOF_UTOFU_SPECIAL_STAG 4096
+
+#define TOF_UTOFU_ICC_COMMON_REGISTER (tof_icc_reg_pa + 0x0B000000)
+#define TOF_UTOFU_REG_START tof_icc_reg_pa
+#define TOF_UTOFU_REG_END (TOF_UTOFU_ICC_COMMON_REGISTER + 0x000FFFFF)
+
+#define TOF_UTOFU_SET_SUBNET_TNI	0	/* This number is kernel TNIs number in setting subnet */
+#define TOF_UTOFU_KCQ			11
+#define TOF_UTOFU_LINKDOWN_PORT_MASK	0x000003FF
+
+#define TOF_UTOFU_ALLOC_STAG_LPG	0x2
+#define TOF_UTOFU_BLANK_MBVA (-1)
+
+#define TOF_UTOFU_MRU_EMPTY (-1)
+
+struct tof_utofu_trans_list {
+	int16_t prev;
+	int16_t next;
+	uint8_t pgszbits;
+	struct tof_utofu_mbpt *mbpt;
+};
+
+
+/*
+ * Bit 30 marks a kref as McKernel internal.
+ * This can be used to distinguish krefs from Linux and
+ * it also ensures that a non deallocated kref will not
+ * crash the Linux allocator.
+ */
+#define MCKERNEL_KREF_MARK	(1U << 30)
+static inline unsigned int mcctrl_kref_is_mckernel(const struct kref *kref)
+{
+	return (refcount_read(&kref->refcount) & (MCKERNEL_KREF_MARK));
+}
+
+/**
+ * kref_put - decrement refcount for object.
+ * @kref: object.
+ * @release: pointer to the function that will clean up the object when the
+ *	     last reference to the object is released.
+ *	     This pointer is required, and it is not acceptable to pass kfree
+ *	     in as this function.  If the caller does pass kfree to this
+ *	     function, you will be publicly mocked mercilessly by the kref
+ *	     maintainer, and anyone else who happens to notice it.  You have
+ *	     been warned.
+ *
+ * Decrement the refcount, and if 0, call release().
+ * Return 1 if the object was removed, otherwise return 0.  Beware, if this
+ * function returns 0, you still can not count on the kref from remaining in
+ * memory.  Only use the return value if you want to see if the kref is now
+ * gone, not present.
+ */
+static inline int mcctrl_kref_put(struct kref *kref, void (*release)(struct kref *kref))
+{
+	if (atomic_dec_return(&kref->refcount.refs) == MCKERNEL_KREF_MARK) {
+		release(kref);
+		return 1;
+	}
+	return 0;
+}
+
+static int tof_utofu_cq_cacheflush(struct tof_utofu_cq *ucq){
+	return mcctrl_tof_core_cq_cacheflush(ucq->tni, ucq->cqid);
+}
+
+static void tof_utofu_trans_mru_delete(struct tof_utofu_cq *ucq, int stag){
+	struct tof_utofu_trans_list *mru = ucq->trans.mru;
+	int prev = mru[stag].prev;
+	int next = mru[stag].next;
+	if(prev == TOF_UTOFU_MRU_EMPTY || next == TOF_UTOFU_MRU_EMPTY){ /* already deleted */
+		return;
+	}
+	if(prev == stag){  /* a single entry */
+		ucq->trans.mruhead = TOF_UTOFU_MRU_EMPTY;
+	}else{
+		if(ucq->trans.mruhead == stag){
+			ucq->trans.mruhead = next;
+		}
+		mru[prev].next = next;
+		mru[next].prev = prev;
+	}
+	mru[stag].prev = TOF_UTOFU_MRU_EMPTY;
+	mru[stag].next = TOF_UTOFU_MRU_EMPTY;
+}
+
+static void tof_utofu_trans_disable(struct tof_utofu_cq *ucq, int stag){
+	struct tof_trans_table *table = ucq->trans.table;
+	atomic64_set((atomic64_t *)&table[stag], 0);
+	tof_utofu_trans_mru_delete(ucq, stag);
+}
+
+/* McKernel scatterlist is simply a contiguous buffer. */
+struct scatterlist {
+	void *pages;
+	unsigned int	offset;
+	unsigned int	length;
+	unsigned long	dma_address;
+	unsigned int	dma_length;
+};
+
+static uintptr_t tof_utofu_disable_mbpt(struct tof_utofu_mbpt *mbpt, int idx){
+	int i0, i1;
+	struct tof_icc_mbpt_entry *ent;
+	uintptr_t ipa;
+	i0 = idx / (PAGE_SIZE / TOF_ICC_MBPT_SIZE);
+	i1 = idx - i0 * (PAGE_SIZE / TOF_ICC_MBPT_SIZE);
+	//ent = sg_virt(&mbpt->sg[i0]);
+	ent = mbpt->sg->pages + (i0 * PAGE_SIZE);
+	if(!ent[i1].enable){
+		return 0;
+	}
+	ent[i1].enable = 0;
+	ipa = (uint64_t)ent[i1].ipa << 12;
+	ent[i1].ipa = 0;
+	return ipa;
+}
+
+static void tof_utofu_free_mbpt(struct tof_utofu_cq *ucq, struct tof_utofu_mbpt *mbpt){
+	int i;
+
+	for(i = 0; i < mbpt->nsgents * PAGE_SIZE / sizeof(struct tof_icc_mbpt_entry); i++){
+		uintptr_t iova;
+		iova = tof_utofu_disable_mbpt(mbpt, i);
+#if 0
+		/*
+		 * NOTE: Not performed for McKernel managed stags.
+		 */
+		if(iova){
+			tof_smmu_release_ipa_cq(ucq->tni, ucq->cqid, iova, mbpt->pgsz);
+		}
+#endif
+	}
+
+#if 0
+	/*
+	 * NOTE: Everyhing below has been allocated in McKernel, do nothing here!!
+	 * This leaks memory in McKernel, but it doesn't crash Linux.
+	 * Memory will be released once McKernel is unbooted.
+	 */
+	tof_smmu_iova_unmap_sg(ucq->tni, ucq->cqid, mbpt->sg, mbpt->nsgents);
+
+	for(i = 0; i < mbpt->nsgents; i++){
+		tof_util_free_pages((unsigned long)sg_virt(&mbpt->sg[i]), 0);
+	}
+	tof_util_free(mbpt->sg);
+	tof_util_free(mbpt);
+#endif
+}
+
+static void tof_utofu_mbpt_release(struct kref *kref)
+{
+	struct tof_utofu_mbpt *mbpt = container_of(kref, struct tof_utofu_mbpt, kref);
+	//atomic64_inc((atomic64_t *)&kref_free_count);
+	tof_utofu_free_mbpt(mbpt->ucq, mbpt);
+}
+
+static int tof_utofu_free_stag(struct tof_utofu_cq *ucq, int stag){
+	if(stag < 0 || stag >= TOF_UTOFU_NUM_STAG(ucq->num_stag) ||
+	   ucq->steering == NULL){
+		return -EINVAL;
+	}
+	if(!(ucq->steering[stag].enable)){
+		return -ENOENT;
+	}
+	if (!mcctrl_kref_is_mckernel(&ucq->trans.mru[stag].mbpt->kref)) {
+		printk("%s: stag: %d is not an McKernel kref\n", __func__, stag);
+		return -EINVAL;
+	}
+	ucq->steering[stag].enable = 0;
+	ucq->mb[stag].enable = 0;
+	tof_utofu_trans_disable(ucq, stag);
+	dma_wmb();
+	tof_utofu_cq_cacheflush(ucq);
+	mcctrl_kref_put(&ucq->trans.mru[stag].mbpt->kref, tof_utofu_mbpt_release);
+	ucq->trans.mru[stag].mbpt = NULL;
+	dprintk("%s: TNI: %d, CQ: %d: stag %d deallocated\n",
+			__func__, ucq->tni, ucq->cqid, stag);
+	return 0;
+}
+
+void mcctrl_mckernel_tof_utofu_release_cq(void *pde_data)
+{
+	struct tof_utofu_cq *ucq;
+	struct tof_utofu_device *dev;
+	unsigned long irqflags;
+	int stag;
+
+	dev = (struct tof_utofu_device *)pde_data;
+	ucq = container_of(dev, struct tof_utofu_cq, common);
+
+	if (!ucq->common.enabled) {
+		return;
+	}
+
+	dprintk("%s: UCQ (PDE: 0x%lx) TNI %d CQ %d\n",
+		__func__, (unsigned long)pde_data, ucq->tni, ucq->cqid);
+
+	/*
+	 * Only release stags here, actual cleanup is still performed
+	 * in the Tofu driver
+	 */
+	for (stag = 0; stag < TOF_UTOFU_NUM_STAG(ucq->num_stag); stag++) {
+		spin_lock_irqsave(&ucq->trans.mru_lock, irqflags);
+		tof_utofu_free_stag(ucq, stag);
+		spin_unlock_irqrestore(&ucq->trans.mru_lock, irqflags);
+	}
+}
+
+static inline void tof_core_unregister_signal_bg(int tni, int bgid)
+{
+	return mcctrl_tof_core_register_signal_bg(tni, bgid, NULL);
+}
+
+static struct tof_utofu_bg *tof_utofu_bg_get(int tni, int bgid){
+	if((unsigned int)tni >= TOF_ICC_NTNIS ||
+	   (unsigned int)bgid >= TOF_ICC_NBGS){
+		return NULL;
+	}
+	//return &tof_utofu_bg[tni][bgid];
+
+	// Convert [][] notion into pointer aritmethic
+	return mcctrl_tof_utofu_bg + (tni * TOF_ICC_NBGS) + bgid;
+}
+
+static int __tof_utofu_unset_bg(struct tof_utofu_bg *ubg){
+	if(ubg->common.enabled){
+		mcctrl_tof_core_unset_bg(ubg->tni, ubg->bgid);
+		ubg->common.enabled = false;
+		tof_core_unregister_signal_bg(ubg->tni, ubg->bgid);
+	}
+	return 0;
+}
+
+static int mcctrl_tof_utofu_disable_bch(struct tof_utofu_bg *ubg){
+	int ret;
+	int tni, bgid;
+
+	if(!ubg->bch.enabled){
+		return -EPERM;
+	}
+
+	ret = mcctrl_tof_core_disable_bch(ubg->tni, ubg->bgid);
+	if(ret < 0){
+		return ret;
+	}
+
+	for(tni = 0; tni < TOF_ICC_NTNIS; tni++){
+		uint64_t mask = ubg->bch.bgmask[tni];
+		for(bgid = 0; bgid < TOF_ICC_NBGS; bgid++){
+			if((mask >> bgid) & 1){
+				ret = __tof_utofu_unset_bg(tof_utofu_bg_get(tni, bgid));
+				if(ret < 0){
+					/* OK? */
+					//BUG();
+					return ret;
+				}
+			}
+		}
+	}
+
+	/* Not performed in McKernel handler */
+	//tof_smmu_release_ipa_bg(ubg->tni, ubg->bgid, ubg->bch.iova, TOF_ICC_BCH_DMA_ALIGN);
+	//put_page(ubg->bch.page);
+	ubg->bch.enabled = false;
+	smp_mb();
+	dprintk("%s: tni=%d bgid=%d\n", __func__, ubg->tni, ubg->bgid);
+	return 0;
+}
+
+void mcctrl_mckernel_tof_utofu_release_bch(void *pde_data)
+{
+	struct tof_utofu_bg *ubg;
+	struct tof_utofu_device *dev = (struct tof_utofu_device *)pde_data;
+
+	ubg = container_of(dev, struct tof_utofu_bg, common);
+	//tof_log_if("tni=%d bgid=%d\n", ubg->tni, ubg->bgid);
+	dprintk("%s: tni=%d bgid=%d\n", __func__, ubg->tni, ubg->bgid);
+	mcctrl_tof_utofu_disable_bch(ubg);
+}
+
+void mcctrl_tofu_cleanup_file(struct mcctrl_file_to_pidfd *f2pfd)
+{
+	/* Figure out whether CQ or BCH */
+	if (strstr(f2pfd->tofu_dev_path, "cq")) {
+		dprintk("%s: PID: %d, fd: %d (%s) -> release CQ\n",
+			__func__, f2pfd->pid, f2pfd->fd, f2pfd->tofu_dev_path);
+		mcctrl_mckernel_tof_utofu_release_cq(f2pfd->pde_data);
+	}
+
+	else if (strstr(f2pfd->tofu_dev_path, "bch")) {
+		dprintk("%s: PID: %d, fd: %d (%s) -> release BCH\n",
+			__func__, f2pfd->pid, f2pfd->fd, f2pfd->tofu_dev_path);
+		mcctrl_mckernel_tof_utofu_release_bch(f2pfd->pde_data);
+	}
+}
+
+
+int __mcctrl_tof_utofu_release_handler(struct inode *inode, struct file *filp,
+		int (*__release_func)(struct inode *inode, struct file *filp))
+{
+	struct mcctrl_usrdata *usrdata;
+	struct mcctrl_file_to_pidfd *f2pfd;
+	struct mcctrl_per_proc_data *ppd;
+	struct ikc_scd_packet isp;
+	int ret;
+
+	dprintk("%s: current PID: %d, comm: %s \n",
+			__func__, task_tgid_vnr(current), current->comm);
+
+	f2pfd = mcctrl_file_to_pidfd_hash_lookup(filp, current->group_leader);
+	if (!f2pfd) {
+		goto out;
+	}
+
+	dprintk("%s: current PID: %d, PID: %d, fd: %d ...\n",
+			__func__, task_tgid_vnr(current), f2pfd->pid, f2pfd->fd);
+	usrdata = ihk_host_os_get_usrdata(f2pfd->os);
+
+	/* Look up per-process structure */
+	ppd = mcctrl_get_per_proc_data(usrdata, f2pfd->pid);
+	if (!ppd) {
+		pr_err("%s: PID: %d, fd: %d no PPD\n",
+				__func__, f2pfd->pid, f2pfd->fd);
+		goto out;
+	}
+
+	dprintk("%s: PID: %d, fd: %d PPD OK\n",
+			__func__, f2pfd->pid, f2pfd->fd);
+
+	/*
+	 * We are in release() due to the process being killed,
+	 * or because the application didn't close the file properly.
+	 * Ask McKernel to clean up this fd.
+	 */
+	isp.msg = SCD_MSG_CLEANUP_FD;
+	isp.pid = f2pfd->pid;
+	isp.arg = f2pfd->fd;
+
+	ret = mcctrl_ikc_send_wait(f2pfd->os, ppd->ikc_target_cpu,
+			&isp, -20, NULL, NULL, 0);
+	if (ret != 0) {
+		pr_err("%s: WARNING: IKC req for PID: %d, fd: %d failed\n",
+				__func__, f2pfd->pid, f2pfd->fd);
+	}
+
+	/* Disable any remaining STAGs/BCH in mcctrl anyway */
+	mcctrl_tofu_cleanup_file(f2pfd);
+
+	mcctrl_file_to_pidfd_hash_remove(filp, f2pfd->os,
+			current->group_leader, f2pfd->fd);
+
+	mcctrl_put_per_proc_data(ppd);
+
+out:
+	dprintk("%s: current PID: %d, comm: %s -> calling release\n",
+			__func__, task_tgid_vnr(current), current->comm);
+	return __release_func(inode, filp);
+}
+
+int __mcctrl_tof_utofu_release_cq(struct inode *inode, struct file *filp)
+{
+	return __mcctrl_tof_utofu_release_handler(inode, filp,
+			mcctrl_tof_utofu_release_cq);
+}
+
+int __mcctrl_tof_utofu_release_bch(struct inode *inode, struct file *filp)
+{
+	return __mcctrl_tof_utofu_release_handler(inode, filp,
+			mcctrl_tof_utofu_release_bch);
+}
+
+/*
+ * Tofu MMU notifier functions
+ */
+void __mcctrl_tof_utofu_mn_invalidate_range_end(
+	struct mmu_notifier *mn,
+	struct mm_struct *mm,
+	unsigned long start,
+	unsigned long end)
+{
+	char tmpname[TASK_COMM_LEN];
+
+	/* Not an offloaded syscall? */
+	if (current->mm != mm) {
+		goto out_call_real;
+	}
+
+	/* Not mcexec? Just in case.. */
+	get_task_comm(tmpname, current);
+	if (strncmp(tmpname, "mcexec", TASK_COMM_LEN)) {
+		goto out_call_real;
+	}
+
+	/* This is only called for Tofu enabled mcexec processes */
+	dprintk("%s: skipping tof_utofu_mn_invalidate_range_end() "
+			"for mcexec PID %d\n",
+			__func__, task_tgid_vnr(current));
+	return;
+
+out_call_real:
+	return mcctrl_tof_utofu_mn_invalidate_range_end(mn, mm, start, end);
+}
+
+int __mcctrl_tof_utofu_ioctl_init_cq(struct tof_utofu_device *dev,
+		unsigned long arg) {
+	struct tof_utofu_cq *ucq;
+
+	ucq = container_of(dev, struct tof_utofu_cq, common);
+	if (!ucq->common.enabled) {
+		return -EINVAL;
+	}
+
+	dprintk("%s: Tofu TNI %d CQ %d (PDE: 0x%lx) MMU notifier to be hijacked\n",
+		__func__, ucq->tni, ucq->cqid, (unsigned long)dev);
+	/* Override the MMU notifier */
+	ucq->mn.ops = &__mcctrl_tof_utofu_mn_ops;
+
+	return 0;
+}
+
+long __mcctrl_tof_utofu_unlocked_ioctl_cq(void *pde_data, unsigned int cmd,
+					unsigned long arg) {
+	struct tof_utofu_device *dev = (struct tof_utofu_device *)pde_data;
+	int ret;
+
+	switch (cmd) {
+		/* We only care about init, where we hijack the MMU notifier */
+		case TOF_IOCTL_INIT_CQ:
+			ret = __mcctrl_tof_utofu_ioctl_init_cq(dev, arg);
+			break;
+		default:
+			ret = 0;
+	}
+
+	return ret;
+}
+#endif

executer/kernel/mcctrl/arch/arm64/include/tofu/generate_headers.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+SCRIPT="`readlink -f ${BASH_SOURCE[0]:-}`"
+SCRIPT_DIR=$(dirname ${SCRIPT})
+CURRENT_DIR=`pwd`
+
+cd ${SCRIPT_DIR}
+
+DWARF_TOOL=${SCRIPT_DIR}/../../../../../../../tools/dwarf-extract-struct/dwarf-extract-struct
+if [ ! -x ${DWARF_TOOL} ]; then
+	echo "error: couldn't find DWARF extractor executable (${DWARF_TOOL}), have you compiled it?"
+	cd -
+	exit 1
+fi
+
+echo "Looking for Tofu driver debug symbols..."
+if [ "`find /lib/modules/ -name "tof_module.tar.gz" | xargs -r ls -t | head -n 1 | wc -l`" == "0" ]; then
+	echo "error: couldn't find Tofu modules with debug symbols"
+	cd -
+	exit 1
+fi
+
+MODULE_TAR_GZ=`find /lib/modules/ -name "tof_module.tar.gz" | xargs ls -t | head -n 1`
+echo "Using Tofu driver debug symbols: ${MODULE_TAR_GZ}"
+
+KMODULE=tof_utofu.ko
+if ! tar zxvf ${MODULE_TAR_GZ} ${KMODULE} 2>&1 > /dev/null; then
+	echo "error: uncompressing kernel module with debug symbols"
+	cd -
+	exit 1
+fi
+
+${DWARF_TOOL} ${KMODULE} tof_utofu_device enabled subnet gpid > tofu_generated-tof_utofu_device.h
+${DWARF_TOOL} ${KMODULE} tof_utofu_cq common tni cqid mn trans steering mb num_stag | sed "s/struct FILL_IN_MANUALLY trans;/#include \"tof_utofu_cq_trans.h\"/g" > tofu_generated-tof_utofu_cq.h
+${DWARF_TOOL} ${KMODULE} tof_utofu_mbpt ucq iova sg nsgents mbptstart pgsz kref > tofu_generated-tof_utofu_mbpt.h
+${DWARF_TOOL} ${KMODULE} tof_utofu_bg common tni bgid bch | sed "s/struct FILL_IN_MANUALLY bch;/#include \"tof_utofu_bg_bch.h\"/g" > tofu_generated-tof_utofu_bg.h
+rm ${KMODULE}
+
+
+#cat tofu_generated*.h
+cd - > /dev/null

executer/kernel/mcctrl/arch/arm64/include/tofu/tof_icc.h

@@ -0,0 +1,831 @@
+#ifndef _TOF_ICC_H_
+#define _TOF_ICC_H_
+
+#include <linux/types.h>
+#ifdef __KERNEL__
+#include <linux/bitops.h>
+#else
+#include <stdint.h>
+typedef uint64_t phys_addr_t;
+#endif
+
+/* constants related to the Tofu Interconnect D */
+
+#define TOF_ICC_NTNIS 6
+#define TOF_ICC_NCQS 12
+#define TOF_ICC_NBGS 48
+#define TOF_ICC_NBCHS 16
+#define TOF_ICC_NPORTS 10
+#define TOF_ICC_NVMSIDS 16
+
+#define TOF_ICC_RH_LEN 8
+#define TOF_ICC_ECRC_LEN 4
+#define TOF_ICC_FRAME_ALIGN 32
+#define TOF_ICC_TLP_LEN(len) (((len) + 1) * TOF_ICC_FRAME_ALIGN)
+#define TOF_ICC_TLP_PAYLOAD_MAX (TOF_ICC_TLP_LEN(61) - TOF_ICC_ECRC_LEN)
+#define TOF_ICC_FRAME_LEN(len) (TOF_ICC_RH_LEN + TOF_ICC_TLP_LEN(len))
+#define TOF_ICC_FRAME_LEN_MIN TOF_ICC_FRAME_LEN(2)
+#define TOF_ICC_FRAME_LEN_MAX TOF_ICC_FRAME_LEN(61)
+#define TOF_ICC_FRAME_BUF_SIZE_BITS 11
+#define TOF_ICC_FRAME_BUF_SIZE (1 << TOF_ICC_FRAME_BUF_SIZE_BITS)
+#define TOF_ICC_FRAME_BUF_ALIGN_BITS 8
+#define TOF_ICC_FRAME_BUF_ALIGN (1 << TOF_ICC_FRAME_BUF_ALIGN_BITS)
+#define TOF_ICC_PB_SIZE_BITS 11
+#define TOF_ICC_PB_SIZE (1 << TOF_ICC_PB_SIZE_BITS)
+#define TOF_ICC_PB_ALIGN_BITS 11
+#define TOF_ICC_PB_ALIGN (1 << TOF_ICC_PB_ALIGN_BITS)
+
+#define TOF_ICC_ST_ALIGN_BITS 8
+#define TOF_ICC_ST_ALIGN (1 << TOF_ICC_ST_ALIGN_BITS)
+
+#define TOF_ICC_MBT_ALIGN_BITS 8
+#define TOF_ICC_MBT_ALIGN (1 << TOF_ICC_MBT_ALIGN_BITS)
+
+#define TOF_ICC_MBPT_ALIGN_BITS 8
+#define TOF_ICC_MBPT_ALIGN (1 << TOF_ICC_MBPT_ALIGN_BITS)
+
+#define TOF_ICC_BG_BSEQ_SIZE_BITS 24
+#define TOF_ICC_BG_BSEQ_SIZE (1 << TOF_ICC_BG_BSEQ_SIZE_BITS)
+
+#define TOF_ICC_BCH_DMA_ALIGN_BITS 8
+#define TOF_ICC_BCH_DMA_ALIGN (1 << TOF_ICC_BCH_DMA_ALIGN_BITS)
+
+/* this is a CPU-specific constant, but referred in the ICC spec. */
+#define TOF_ICC_CACHE_LINE_SIZE_BITS 8
+#define TOF_ICC_CACHE_LINE_SIZE (1 << TOF_ICC_CACHE_LINE_SIZE_BITS)
+
+#define TOF_ICC_TOQ_DESC_SIZE_BITS 5
+#define TOF_ICC_TOQ_DESC_SIZE (1 << TOF_ICC_TOQ_DESC_SIZE_BITS)
+#define TOF_ICC_TCQ_DESC_SIZE_BITS 3
+#define TOF_ICC_TCQ_DESC_SIZE (1 << TOF_ICC_TCQ_DESC_SIZE_BITS)
+#define TOF_ICC_TCQ_NLINE_BITS (TOF_ICC_CACHE_LINE_SIZE_BITS - TOF_ICC_TCQ_DESC_SIZE_BITS)
+#define TOF_ICC_MRQ_DESC_SIZE_BITS 5
+#define TOF_ICC_MRQ_DESC_SIZE (1 << TOF_ICC_MRQ_DESC_SIZE_BITS)
+#define TOF_ICC_PBQ_DESC_SIZE_BITS 3
+#define TOF_ICC_PBQ_DESC_SIZE (1 << TOF_ICC_PBQ_DESC_SIZE_BITS)
+#define TOF_ICC_PRQ_DESC_SIZE_BITS 3
+#define TOF_ICC_PRQ_DESC_SIZE (1 << TOF_ICC_PRQ_DESC_SIZE_BITS)
+#define TOF_ICC_PRQ_NLINE_BITS (TOF_ICC_CACHE_LINE_SIZE_BITS - TOF_ICC_PBQ_DESC_SIZE_BITS)
+
+#define TOF_ICC_TOQ_SIZE_NTYPES 6
+#define TOF_ICC_TOQ_SIZE_BITS(size) ((size) * 2 + 11)
+#define TOF_ICC_TOQ_SIZE(size) (1 << TOF_ICC_TOQ_SIZE_BITS(size))
+#define TOF_ICC_TOQ_LEN(size) (TOF_ICC_TOQ_SIZE(size) * TOF_ICC_TOQ_DESC_SIZE)
+#define TOF_ICC_TCQ_LEN(size) (TOF_ICC_TOQ_SIZE(size) * TOF_ICC_TCQ_DESC_SIZE)
+
+#define TOF_ICC_MRQ_SIZE_NTYPES 6
+#define TOF_ICC_MRQ_SIZE_BITS(size) ((size) * 2 + 11)
+#define TOF_ICC_MRQ_SIZE(size) (1 << TOF_ICC_MRQ_SIZE_BITS(size))
+#define TOF_ICC_MRQ_LEN(size) (TOF_ICC_MRQ_SIZE(size) * TOF_ICC_MRQ_DESC_SIZE)
+
+#define TOF_ICC_PBQ_SIZE_NTYPES 6
+#define TOF_ICC_PBQ_SIZE_BITS(size) ((size) * 2 + 11)
+#define TOF_ICC_PBQ_SIZE(size) (1 << TOF_ICC_PBQ_SIZE_BITS(size))
+#define TOF_ICC_PBQ_LEN(size) (TOF_ICC_PBQ_SIZE(size) * TOF_ICC_PBQ_DESC_SIZE)
+
+#define TOF_ICC_PRQ_SIZE_NTYPES 6
+#define TOF_ICC_PRQ_SIZE_BITS(size) ((size) * 2 + 11)
+#define TOF_ICC_PRQ_SIZE(size) (1 << TOF_ICC_PRQ_SIZE_BITS(size))
+#define TOF_ICC_PRQ_LEN(size) (TOF_ICC_PRQ_SIZE(size) * TOF_ICC_PRQ_DESC_SIZE)
+
+#define TOF_ICC_STEERING_TABLE_ALIGN_BITS 8
+#define TOF_ICC_STEERING_TABLE_ALIGN (1 << TOF_ICC_STEERING_TABLE_ALIGN_BITS)
+#define TOF_ICC_STEERING_SIZE_BITS 4
+#define TOF_ICC_STEERING_SIZE (1 << TOF_ICC_STEERING_SIZE_BITS)
+
+#define TOF_ICC_MB_TABLE_ALIGN_BITS 8
+#define TOF_ICC_MB_TABLE_ALIGN (1 << TOF_ICC_MB_TABLE_ALIGN_BITS)
+#define TOF_ICC_MB_SIZE_BITS 4
+#define TOF_ICC_MB_SIZE (1 << TOF_ICC_MB_SIZE_BITS)
+#define TOF_ICC_MB_PS_ENCODE(bits) ((bits) % 9 == 3 ? (bits) / 9 - 1 : (bits) / 13 + 3)
+
+#define TOF_ICC_MBPT_ALIGN_BITS 8
+#define TOF_ICC_MBPT_ALIGN (1 << TOF_ICC_MBPT_ALIGN_BITS)
+#define TOF_ICC_MBPT_SIZE_BITS 3
+#define TOF_ICC_MBPT_SIZE (1 << TOF_ICC_MBPT_SIZE_BITS)
+
+#define TOF_ICC_X_BITS 5
+#define TOF_ICC_Y_BITS 5
+#define TOF_ICC_Z_BITS 5
+#define TOF_ICC_A_BITS 1
+#define TOF_ICC_B_BITS 2
+#define TOF_ICC_C_BITS 1
+#define TOF_ICC_MAX_X_SIZE (1 << TOF_ICC_X_BITS)
+#define TOF_ICC_MAX_Y_SIZE (1 << TOF_ICC_Y_BITS)
+#define TOF_ICC_MAX_Z_SIZE (1 << TOF_ICC_Z_BITS)
+#define TOF_ICC_A_SIZE 2
+#define TOF_ICC_B_SIZE 3
+#define TOF_ICC_C_SIZE 2
+#define TOF_ICC_X_MASK ((1 << TOF_ICC_X_BITS) - 1)
+#define TOF_ICC_Y_MASK ((1 << TOF_ICC_Y_BITS) - 1)
+#define TOF_ICC_Z_MASK ((1 << TOF_ICC_Z_BITS) - 1)
+#define TOF_ICC_A_MASK ((1 << TOF_ICC_A_BITS) - 1)
+#define TOF_ICC_B_MASK ((1 << TOF_ICC_B_BITS) - 1)
+#define TOF_ICC_C_MASK ((1 << TOF_ICC_C_BITS) - 1)
+#define TOF_ICC_ABC_SIZE (TOF_ICC_A_SIZE * TOF_ICC_B_SIZE * TOF_ICC_C_SIZE)
+
+static inline int tof_icc_get_framelen(int len){
+	len = TOF_ICC_RH_LEN + round_up(len + TOF_ICC_ECRC_LEN, TOF_ICC_FRAME_ALIGN);
+	if(len < TOF_ICC_FRAME_LEN_MIN){
+		len = TOF_ICC_FRAME_LEN_MIN;
+	}
+	return len;
+}
+
+/** Descriptors **/
+/** commands and rcodes **/
+enum {
+	TOF_ICC_TOQ_NOP,
+	TOF_ICC_TOQ_PUT,
+	TOF_ICC_TOQ_WRITE_PIGGYBACK_BUFFER,
+	TOF_ICC_TOQ_PUT_PIGGYBACK,
+	TOF_ICC_TOQ_GET,
+	TOF_ICC_TOQ_GETL,
+	TOF_ICC_TOQ_ATOMIC_READ_MODIFY_WRITE = 0xe,
+	TOF_ICC_TOQ_TRANSMIT_RAW_PACKET1 = 0x10,
+	TOF_ICC_TOQ_TRANSMIT_RAW_PACKET2,
+	TOF_ICC_TOQ_TRANSMIT_SYSTEM_PACKET1,
+	TOF_ICC_TOQ_TRANSMIT_SYSTEM_PACKET2,
+
+	TOF_ICC_TOQ_NCOMMANDS,
+};
+
+enum {
+	TOF_ICC_MRQ_ATOMIC_READ_MODIFY_WRITE_HALFWAY_NOTICE = 0x1,
+	TOF_ICC_MRQ_ATOMIC_READ_MODIFY_WRITE_NOTICE,
+	TOF_ICC_MRQ_ATOMIC_READ_MODIFY_WRITE_REMOTE_ERROR,
+	TOF_ICC_MRQ_PUT_HALFWAY_NOTICE,
+	TOF_ICC_MRQ_PUT_LAST_HALFWAY_NOTICE,
+	TOF_ICC_MRQ_GET_HALFWAY_NOTICE,
+	TOF_ICC_MRQ_GET_LAST_HALFWAY_NOTICE,
+	TOF_ICC_MRQ_PUT_NOTICE,
+	TOF_ICC_MRQ_PUT_LAST_NOTICE,
+	TOF_ICC_MRQ_GET_NOTICE,
+	TOF_ICC_MRQ_GET_LAST_NOTICE,
+	TOF_ICC_MRQ_PUT_REMOTE_ERROR,
+	TOF_ICC_MRQ_PUT_LAST_REMOTE_ERROR,
+	TOF_ICC_MRQ_GET_REMOTE_ERROR,
+	TOF_ICC_MRQ_GET_LAST_REMOTE_ERROR,
+
+	TOF_ICC_MRQ_NCOMMANDS,
+};
+
+enum {
+	TOF_ICC_PRQ_UNKNOWN_TLP,
+	TOF_ICC_PRQ_SYSTEM_TLP,
+	TOF_ICC_PRQ_ADDRESS_RANGE_EXCEPTION = 0x6,
+	TOF_ICC_PRQ_CQ_EXCEPTION = 0x8,
+	TOF_ICC_PRQ_ILLEGAL_TLP_FLAGS,
+	TOF_ICC_PRQ_ILLEGAL_TLP_LENGTH,
+	TOF_ICC_PRQ_CQ_ERROR = 0xc,
+};
+
+/** structures **/
+struct tof_icc_steering_entry {
+	uint64_t res1:6;
+	uint64_t readonly:1;
+	uint64_t enable:1;
+	uint64_t mbva:32;
+	uint64_t res2:8;
+	uint64_t mbid:16;
+	uint64_t length;  /* for optimization */
+};
+
+struct tof_icc_mb_entry {
+	uint64_t ps:3;
+	uint64_t res1:4;
+	uint64_t enable:1;
+	uint64_t ipa:32;
+	uint64_t res2:24;
+	uint64_t npage;  /* for optimization */
+};
+
+struct tof_icc_mbpt_entry {
+	uint64_t res1:7;
+	uint64_t enable:1;
+	uint64_t res2:4;
+	uint64_t ipa:28;
+	uint64_t res3:24;
+};
+
+struct tof_icc_cq_stag_offset {
+	uint64_t offset:40;
+	uint64_t stag:18;
+	uint64_t cqid:6;
+};
+
+struct tof_icc_toq_common_header1 {
+	uint8_t interrupt:1;
+	uint8_t res1:4;
+	uint8_t source_type:2;
+	uint8_t flip:1;
+	uint8_t command;
+	union {
+		uint8_t mtu;
+		struct {
+			uint8_t res:4;
+			uint8_t op:4;
+		} armw;
+	} mtuop;
+	uint8_t sps:4;
+	uint8_t pa:1;
+	uint8_t pb:2;
+	uint8_t pc:1;
+	uint8_t rx;
+	uint8_t ry;
+	uint8_t rz;
+	uint8_t ra:1;
+	uint8_t rb:2;
+	uint8_t rc:1;
+	uint8_t res3:1;
+	uint8_t ri:3;
+};
+
+struct tof_icc_toq_common_header2 {
+	uint8_t gap;
+	uint8_t s:1;
+	uint8_t r:1;
+	uint8_t q:1;
+	uint8_t p:1;
+	uint8_t res1:1;
+	uint8_t j:1;
+	uint8_t res2:2;
+	uint16_t edata;
+	union{
+		struct {
+			uint32_t length:24;
+			uint32_t res:8;
+		} normal;
+		struct {
+			uint32_t length:6;
+			uint32_t res:26;
+		} piggyback;
+	} len;
+};
+
+struct tof_icc_toq_descriptor {
+	struct tof_icc_toq_common_header1 head1;
+	uint64_t res[3];
+};
+
+struct tof_icc_toq_nop {
+	struct tof_icc_toq_common_header1 head1;
+	uint64_t res[3];
+};
+
+struct tof_icc_toq_put {
+	struct tof_icc_toq_common_header1 head1;
+	struct tof_icc_toq_common_header2 head2;
+	struct tof_icc_cq_stag_offset remote;
+	struct tof_icc_cq_stag_offset local;
+};
+
+struct tof_icc_toq_write_piggyback_buffer {
+	struct tof_icc_toq_common_header1 head1;
+	uint64_t data[3];
+};
+
+struct tof_icc_toq_put_piggyback {
+	struct tof_icc_toq_common_header1 head1;
+	struct tof_icc_toq_common_header2 head2;
+	struct tof_icc_cq_stag_offset remote;
+	uint64_t data;
+};
+
+struct tof_icc_toq_get {
+	struct tof_icc_toq_common_header1 head1;
+	struct tof_icc_toq_common_header2 head2;
+	struct tof_icc_cq_stag_offset remote;
+	struct tof_icc_cq_stag_offset local;
+};
+
+struct tof_icc_toq_atomic_read_modify_write {
+	struct tof_icc_toq_common_header1 head1;
+	struct tof_icc_toq_common_header2 head2;
+	struct tof_icc_cq_stag_offset remote;
+	uint64_t data;
+};
+
+struct tof_icc_toq_transmit_raw_packet1 {
+	struct tof_icc_toq_common_header1 head1;
+	uint8_t gap;
+	uint8_t res4[3];
+	uint32_t length:12;
+	uint32_t res5:20;
+	uint64_t res6;
+	uint64_t pa:48;  /* for optimization */
+	uint64_t res7:16;
+};
+
+struct tof_icc_toq_transmit_raw_packet2 {
+	uint8_t interrupt:1;
+	uint8_t res1:4;
+	uint8_t source_type:2;
+	uint8_t flip:1;
+	uint8_t command;
+	uint8_t res2:7;
+	uint8_t e:1;
+	uint8_t res3[4];
+	uint8_t port:5;
+	uint8_t res4:1;
+	uint8_t vc:2;
+	uint8_t gap;
+	uint8_t res5[3];
+	uint32_t length:12;
+	uint32_t res6:20;
+	uint64_t res7;
+	uint64_t pa:48;  /* for optimization */
+	uint64_t res8:16;
+};
+
+struct tof_icc_toq_transmit_system_packet {
+	struct tof_icc_toq_common_header1 head1;  /* rx, ry, rz should be rdx, rdy, rdz */
+	uint8_t gap;
+	uint8_t res4[3];
+	uint32_t length:12;
+	uint32_t res5:20;
+	uint64_t res6;
+	uint64_t pa:48;  /* for optimization */
+	uint64_t res7:16;
+};
+
+struct tof_icc_tcq_descriptor {
+	uint8_t res1:5;
+	uint8_t counter_unmatch:1;
+	uint8_t res2:1;
+	uint8_t flip:1;
+	uint8_t rcode;
+	uint8_t res3[2];
+	union{
+		struct {
+			uint32_t length:24;
+			uint32_t res:8;
+		} normal;
+		struct {
+			uint32_t length:6;
+			uint32_t res:26;
+		} piggyback;
+	} len;
+};
+
+struct tof_icc_mrq_common_header1 {
+	uint8_t res1:7;
+	uint8_t flip:1;
+	uint8_t id;
+	uint8_t rcode;
+	uint8_t res2:4;
+	uint8_t pa:1;
+	uint8_t pb:2;
+	uint8_t pc:1;
+	uint8_t x;
+	uint8_t y;
+	uint8_t z;
+	uint8_t a:1;
+	uint8_t b:2;
+	uint8_t c:1;
+	uint8_t res3:1;
+	uint8_t i:3;
+};
+
+struct tof_icc_mrq_common_header2 {
+	uint8_t res1;
+	uint8_t res2:4;
+	uint8_t initial:1;
+	uint8_t res3:3;
+	uint16_t edata;
+	union {
+		struct {
+			uint32_t length:11;
+			uint32_t res:21;
+		} normal;
+		struct {
+			uint32_t op:4;
+			uint32_t res:28;
+		} armw;
+	} lenop;
+};
+
+struct tof_icc_mrq_atomic_read_modify_write_halfway_notice {
+	struct tof_icc_mrq_common_header1 head1;
+	struct tof_icc_mrq_common_header2 head2;
+	struct tof_icc_cq_stag_offset local;
+	struct tof_icc_cq_stag_offset remote;
+};
+
+struct tof_icc_mrq_descriptor {
+	struct tof_icc_mrq_common_header1 head1;
+	struct tof_icc_mrq_common_header2 head2;
+	struct tof_icc_cq_stag_offset cso1;
+	struct tof_icc_cq_stag_offset cso2;
+};
+
+struct tof_icc_pbq_descriptor {
+	uint64_t res1:7;
+	uint64_t f:1;
+	uint64_t res2:3;
+	uint64_t pa:29;
+	uint64_t res3:24;
+};
+
+struct tof_icc_prq_descriptor {
+	uint64_t rcode:7;
+	uint64_t f:1;
+	uint64_t res1:3;
+	uint64_t pa:29;
+	uint64_t res2:8;
+	uint64_t w:1;
+	uint64_t res3:5;
+	uint64_t l:1;
+	uint64_t e:1;
+	uint64_t res4:8;
+};
+
+
+/** Registers **/
+/* useful packed structures */
+struct tof_icc_reg_subnet {
+	uint64_t lz:6;
+	uint64_t sz:6;
+	uint64_t nz:6;
+	uint64_t ly:6;
+	uint64_t sy:6;
+	uint64_t ny:6;
+	uint64_t lx:6;
+	uint64_t sx:6;
+	uint64_t nx:6;
+	uint64_t res:10;
+};
+
+struct tof_icc_reg_bg_address {
+	uint32_t bgid:6;
+	uint32_t tni:3;
+	uint32_t c:1;
+	uint32_t b:2;
+	uint32_t a:1;
+	uint32_t z:5;
+	uint32_t y:5;
+	uint32_t x:5;
+	uint32_t pc:1;
+	uint32_t pb:2;
+	uint32_t pa:1;
+};
+
+/* relative offset of interrupt controller registers */
+#define TOF_ICC_IRQREG_IRR 0x0
+#define TOF_ICC_IRQREG_IMR 0x8
+#define TOF_ICC_IRQREG_IRC 0x10
+#define TOF_ICC_IRQREG_IMC 0x18
+#define TOF_ICC_IRQREG_ICL 0x20
+
+/* TOFU REGISTERS */
+#define  tof_icc_reg_pa 0x40000000
+
+/* CQ */
+#define TOF_ICC_REG_CQ_PA(tni, cqid) (tof_icc_reg_pa + 0 + (tni) * 0x1000000 + (cqid) * 0x10000)
+#define TOF_ICC_REG_CQ_TOQ_DIRECT_DESCRIPTOR 0x0
+#define TOF_ICC_REG_CQ_TOQ_FETCH_START 0x40
+#define TOF_ICC_REG_CQ_MRQ_FULL_POINTER 0x48
+#define TOF_ICC_REG_CQ_TOQ_PIGGYBACK_BUFFER0 0x50
+#define TOF_ICC_REG_CQ_TOQ_PIGGYBACK_BUFFER1 0x58
+#define TOF_ICC_REG_CQ_TOQ_PIGGYBACK_BUFFER2 0x60
+#define TOF_ICC_REG_CQ_TCQ_NUM_NOTICE 0x68
+#define TOF_ICC_REG_CQ_MRQ_NUM_NOTICE 0x70
+#define TOF_ICC_REG_CQ_TX_PAYLOAD_BYTE 0x78
+#define TOF_ICC_REG_CQ_RX_PAYLOAD_BYTE 0x80
+#define TOF_ICC_REG_CQ_DUMP_START 0x0
+#define TOF_ICC_REG_CQ_DUMP_END 0x88
+
+/* BCH */
+#define TOF_ICC_REG_BCH_PA(tni, bgid) (tof_icc_reg_pa + 0x0000e00000 + (tni) * 0x1000000 + (bgid) * 0x10000)
+#define TOF_ICC_REG_BCH_IDATA 0x800
+#define TOF_ICC_REG_BCH_READY 0x840
+#define TOF_ICC_REG_BCH_READY_STATE BIT(63)
+#define TOF_ICC_REG_BCH_IGNORED_SIGNAL_COUNT 0x848
+#define TOF_ICC_REG_BCH_DUMP_START 0x800
+#define TOF_ICC_REG_BCH_DUMP_END 0x850
+
+/* CQS */
+#define TOF_ICC_REG_CQS_PA(tni, cqid) (tof_icc_reg_pa + 0x0000400000 + (tni) * 0x1000000 + (cqid) * 0x10000)
+#define TOF_ICC_REG_CQS_STATUS 0x0
+#define TOF_ICC_REG_CQS_STATUS_DESCRIPTOR_PROCESS_STOP BIT(63)
+#define TOF_ICC_REG_CQS_STATUS_DESCRIPTOR_FETCH_STOP BIT(62)
+#define TOF_ICC_REG_CQS_STATUS_BLANK_ENTRY_FLIP_BIT BIT(61)
+#define TOF_ICC_REG_CQS_STATUS_CACHE_FLUSH_BUSY BIT(60)
+#define TOF_ICC_REG_CQS_STATUS_CQ_ENABLE BIT(59)
+#define TOF_ICC_REG_CQS_STATUS_SESSION_DEAD BIT(58)
+#define TOF_ICC_REG_CQS_STATUS_SESSION_OFFSET_OVERFLOW BIT(57)
+#define TOF_ICC_REG_CQS_STATUS_SESSION_OFFSET GENMASK(56, 32)
+#define TOF_ICC_REG_CQS_STATUS_NEXT_DESCRIPTOR_OFFSET GENMASK(29, 5)
+#define TOF_ICC_REG_CQS_ENABLE 0x8
+#define TOF_ICC_REG_CQS_CACHE_FLUSH 0x10
+#define TOF_ICC_REG_CQS_FETCH_STOP 0x18
+#define TOF_ICC_REG_CQS_MODE 0x20
+#define TOF_ICC_REG_CQS_MODE_SYSTEM BIT(63)
+#define TOF_ICC_REG_CQS_MODE_TRP2_ENABLE BIT(62)
+#define TOF_ICC_REG_CQS_MODE_TRP1_ENABLE BIT(61)
+#define TOF_ICC_REG_CQS_MODE_SESSION BIT(60)
+#define TOF_ICC_REG_CQS_MODE_SUBNET_NX GENMASK(53, 48)
+#define TOF_ICC_REG_CQS_MODE_SUBNET_SX GENMASK(47, 42)
+#define TOF_ICC_REG_CQS_MODE_SUBNET_LX GENMASK(41, 36)
+#define TOF_ICC_REG_CQS_MODE_SUBNET_NY GENMASK(35, 30)
+#define TOF_ICC_REG_CQS_MODE_SUBNET_SY GENMASK(29, 24)
+#define TOF_ICC_REG_CQS_MODE_SUBNET_LY GENMASK(23, 18)
+#define TOF_ICC_REG_CQS_MODE_SUBNET_NZ GENMASK(17, 12)
+#define TOF_ICC_REG_CQS_MODE_SUBNET_SZ GENMASK(11, 6)
+#define TOF_ICC_REG_CQS_MODE_SUBNET_LZ GENMASK(5, 0)
+#define TOF_ICC_REG_CQS_GPID 0x28
+#define TOF_ICC_REG_CQS_TOQ_IPA 0x30
+#define TOF_ICC_REG_CQS_TOQ_SIZE 0x38
+#define TOF_ICC_REG_CQS_TCQ_IPA 0x40
+#define TOF_ICC_REG_CQS_TCQ_IPA_CACHE_INJECTION BIT(63)
+#define TOF_ICC_REG_CQS_MRQ_IPA 0x48
+#define TOF_ICC_REG_CQS_MRQ_IPA_CACHE_INJECTION BIT(63)
+#define TOF_ICC_REG_CQS_MRQ_SIZE 0x50
+#define TOF_ICC_REG_CQS_MRQ_MASK 0x58
+#define TOF_ICC_REG_CQS_TCQ_DESCRIPTOR_COALESCING_TIMER 0x60
+#define TOF_ICC_REG_CQS_MRQ_DESCRIPTOR_COALESCING_TIMER 0x68
+#define TOF_ICC_REG_CQS_MRQ_INTERRUPT_COALESCING_TIMER 0x70
+#define TOF_ICC_REG_CQS_MRQ_INTERRUPT_COALESCING_COUNT 0x78
+#define TOF_ICC_REG_CQS_TOQ_DIRECT_SOURCE_COUNT 0x80
+#define TOF_ICC_REG_CQS_TOQ_DIRECT_DESCRIPTOR_COUNT 0x88
+#define TOF_ICC_REG_CQS_MEMORY_BLOCK_TABLE_ENABLE 0x90
+#define TOF_ICC_REG_CQS_MEMORY_BLOCK_TABLE_IPA 0x98
+#define TOF_ICC_REG_CQS_MEMORY_BLOCK_TABLE_SIZE 0xa0
+#define TOF_ICC_REG_CQS_STEERING_TABLE_ENABLE 0xa8
+#define TOF_ICC_REG_CQS_STEERING_TABLE_IPA 0xb0
+#define TOF_ICC_REG_CQS_STEERING_TABLE_SIZE 0xb8
+#define TOF_ICC_REG_CQS_MRQ_INTERRUPT_MASK 0xc0
+#define TOF_ICC_REG_CQS_IRR 0xc8
+#define TOF_ICC_REG_CQS_IMR 0xd0
+#define TOF_ICC_REG_CQS_IRC 0xd8
+#define TOF_ICC_REG_CQS_IMC 0xe0
+#define TOF_ICC_REG_CQS_ICL 0xe8
+#define TOF_ICC_REG_CQS_DUMP_START 0x0
+#define TOF_ICC_REG_CQS_DUMP_END 0xf0
+
+/* BGS */
+#define TOF_ICC_REG_BGS_PA(tni, bgid) (tof_icc_reg_pa + 0x0000800000 + (tni) * 0x1000000 + (bgid) * 0x10000)
+#define TOF_ICC_REG_BGS_ENABLE 0x0
+#define TOF_ICC_REG_BGS_IRR 0x8
+#define TOF_ICC_REG_BGS_IMR 0x10
+#define TOF_ICC_REG_BGS_IRC 0x18
+#define TOF_ICC_REG_BGS_IMC 0x20
+#define TOF_ICC_REG_BGS_ICL 0x28
+#define TOF_ICC_REG_BGS_STATE 0x30
+#define TOF_ICC_REG_BGS_STATE_ENABLE BIT(0)
+#define TOF_ICC_REG_BGS_EXCEPTION_INFO_GPID_UNMATCH 0x38
+#define TOF_ICC_REG_BGS_EXCEPTION_INFO_GPID_UNMATCH_BG_ADDRESS GENMASK(27, 0)
+#define TOF_ICC_REG_BGS_EXCEPTION_INFO_ADDRESS_UNMATCH 0x40
+#define TOF_ICC_REG_BGS_EXCEPTION_INFO_ADDRESS_UNMATCH_BG_ADDRESS GENMASK(27, 0)
+#define TOF_ICC_REG_BGS_SIGNAL_A 0x48
+#define TOF_ICC_REG_BGS_SIGNAL_A_SIG_RECV BIT(63)
+#define TOF_ICC_REG_BGS_SIGNAL_A_TLP_RECV BIT(62)
+#define TOF_ICC_REG_BGS_SIGNAL_A_SIG_SEND BIT(61)
+#define TOF_ICC_REG_BGS_SIGNAL_A_OP_TYPE GENMASK(3, 0)
+#define TOF_ICC_REG_BGS_SIGNAL_B 0x50
+#define TOF_ICC_REG_BGS_SIGNAL_B_SIG_RECV BIT(63)
+#define TOF_ICC_REG_BGS_SIGNAL_B_TLP_RECV BIT(62)
+#define TOF_ICC_REG_BGS_SIGNAL_B_SIG_SEND BIT(61)
+#define TOF_ICC_REG_BGS_SIGNAL_B_OP_TYPE GENMASK(3, 0)
+#define TOF_ICC_REG_BGS_SIGNAL_MASK 0x58
+#define TOF_ICC_REG_BGS_SIGNAL_MASK_SIG_RECV BIT(63)
+#define TOF_ICC_REG_BGS_SIGNAL_MASK_TLP_RECV BIT(62)
+#define TOF_ICC_REG_BGS_SIGNAL_MASK_SIG_SEND BIT(61)
+#define TOF_ICC_REG_BGS_SIGNAL_MASK_TLP_SEND BIT(60)
+#define TOF_ICC_REG_BGS_LOCAL_LINK 0x60
+#define TOF_ICC_REG_BGS_LOCAL_LINK_BGID_RECV GENMASK(37, 32)
+#define TOF_ICC_REG_BGS_LOCAL_LINK_BGID_SEND GENMASK(5, 0)
+#define TOF_ICC_REG_BGS_REMOTE_LINK 0x68
+#define TOF_ICC_REG_BGS_REMOTE_LINK_BG_ADDRESS_RECV GENMASK(59, 32)
+#define TOF_ICC_REG_BGS_REMOTE_LINK_BG_ADDRESS_SEND GENMASK(31, 0)
+#define TOF_ICC_REG_BGS_SUBNET_SIZE 0x70
+#define TOF_ICC_REG_BGS_GPID_BSEQ 0x78
+#define TOF_ICC_REG_BGS_DATA_A0 0x108
+#define TOF_ICC_REG_BGS_DATA_AE 0x178
+#define TOF_ICC_REG_BGS_DATA_B0 0x188
+#define TOF_ICC_REG_BGS_DATA_BE 0x1f8
+#define TOF_ICC_REG_BGS_BCH_MASK 0x800
+#define TOF_ICC_REG_BGS_BCH_MASK_MASK BIT(63)
+#define TOF_ICC_REG_BGS_BCH_MASK_STATUS 0x808
+#define TOF_ICC_REG_BGS_BCH_MASK_STATUS_RUN BIT(63)
+#define TOF_ICC_REG_BGS_BCH_NOTICE_IPA 0x810
+#define TOF_ICC_REG_BGS_DUMP_START 0x0
+#define TOF_ICC_REG_BGS_DUMP_END 0x818
+
+/* TNI */
+#define TOF_ICC_REG_TNI_PA(tni) (tof_icc_reg_pa + 0x0000c00000 + (tni) * 0x1000000)
+#define TOF_ICC_REG_TNI_IRR 0x8
+#define TOF_ICC_REG_TNI_IMR 0x10
+#define TOF_ICC_REG_TNI_IRC 0x18
+#define TOF_ICC_REG_TNI_IMC 0x20
+#define TOF_ICC_REG_TNI_ICL 0x28
+#define TOF_ICC_REG_TNI_STATE 0x30
+#define TOF_ICC_REG_TNI_STATE_MASK GENMASK(1, 0)
+#define TOF_ICC_REG_TNI_STATE_DISABLE 0
+#define TOF_ICC_REG_TNI_STATE_NORMAL 2
+#define TOF_ICC_REG_TNI_STATE_ERROR 3
+#define TOF_ICC_REG_TNI_ENABLE 0x38
+#define TOF_ICC_REG_TNI_CQ_PRESENT 0x40
+#define TOF_ICC_REG_TNI_EXCEPTION_INFO_INACTIVE_BG 0x48
+#define TOF_ICC_REG_TNI_EXCEPTION_INFO_INACTIVE_BG_DEST_BG GENMASK(37, 32)
+#define TOF_ICC_REG_TNI_EXCEPTION_INFO_INACTIVE_BG_SOURCE_BG_ADDRESS GENMASK(27, 0)
+#define TOF_ICC_REG_TNI_PRQ_FULL_POINTER 0x100
+#define TOF_ICC_REG_TNI_PBQ_PA 0x108
+#define TOF_ICC_REG_TNI_PBQ_SIZE 0x110
+#define TOF_ICC_REG_TNI_PRQ_PA 0x118
+#define TOF_ICC_REG_TNI_PRQ_PA_CACHE_INJECTION BIT(63)
+#define TOF_ICC_REG_TNI_PRQ_SIZE 0x120
+#define TOF_ICC_REG_TNI_PRQ_MASK 0x128
+#define TOF_ICC_REG_TNI_PRQ_ENTRY_COALESCING_TIMER 0x130
+#define TOF_ICC_REG_TNI_PRQ_INTERRUPT_COALESCING_TIMER 0x138
+#define TOF_ICC_REG_TNI_PRQ_INTERRUPT_COALESCING_COUNT 0x140
+#define TOF_ICC_REG_TNI_SEND_COUNT 0x148
+#define TOF_ICC_REG_TNI_NO_SEND_COUNT 0x150
+#define TOF_ICC_REG_TNI_BLOCK_SEND_COUNT 0x158
+#define TOF_ICC_REG_TNI_RECEIVE_COUNT 0x160
+#define TOF_ICC_REG_TNI_NO_RECEIVE_COUNT 0x168
+#define TOF_ICC_REG_TNI_NUM_SEND_TLP 0x170
+#define TOF_ICC_REG_TNI_BYTE_SEND_TLP 0x178
+#define TOF_ICC_REG_TNI_NUM_SEND_SYSTEM_TLP 0x180
+#define TOF_ICC_REG_TNI_NUM_RECEIVE_TLP 0x188
+#define TOF_ICC_REG_TNI_BYTE_RECEIVE_TLP 0x190
+#define TOF_ICC_REG_TNI_NUM_RECEIVE_NULLIFIED_TLP 0x198
+#define TOF_ICC_REG_TNI_RX_NUM_UNKNOWN_TLP 0x1a0
+#define TOF_ICC_REG_TNI_RX_NUM_SYSTEM_TLP 0x1a8
+#define TOF_ICC_REG_TNI_RX_NUM_EXCEPTION_TLP 0x1b0
+#define TOF_ICC_REG_TNI_RX_NUM_DISCARD_UNKNOWN_TLP 0x1b8
+#define TOF_ICC_REG_TNI_RX_NUM_DISCARD_SYSTEM_TLP 0x1c0
+#define TOF_ICC_REG_TNI_RX_NUM_DISCARD_EXCEPTION_TLP 0x1c8
+#define TOF_ICC_REG_TNI_DUMP_START 0x8
+#define TOF_ICC_REG_TNI_DUMP_END 0x1d0
+
+/* Port */
+#define TOF_ICC_REG_PORT_PA(port) (tof_icc_reg_pa + 0x0006000000 + (port) * 0x1000)
+#define TOF_ICC_REG_PORT_TX_VC0_ZERO_CREDIT_COUNT 0x0
+#define TOF_ICC_REG_PORT_TX_VC1_ZERO_CREDIT_COUNT 0x8
+#define TOF_ICC_REG_PORT_TX_VC2_ZERO_CREDIT_COUNT 0x10
+#define TOF_ICC_REG_PORT_TX_VC3_ZERO_CREDIT_COUNT 0x18
+#define TOF_ICC_REG_PORT_FREE_RUN_COUNT 0x80
+#define TOF_ICC_REG_PORT_NUM_SEND_DLLP 0xc0
+#define TOF_ICC_REG_PORT_NUM_SEND_TLP 0xc8
+#define TOF_ICC_REG_PORT_BYTE_SEND_TLP 0xd0
+#define TOF_ICC_REG_PORT_NUM_SEND_SYSTEM_TLP 0xd8
+#define TOF_ICC_REG_PORT_NUM_SEND_NULLIFIED_TLP 0xe0
+#define TOF_ICC_REG_PORT_NUM_TX_DISCARD_SYSTEM_TLP 0xe8
+#define TOF_ICC_REG_PORT_NUM_TX_DISCARD_NORMAL_TLP 0xf0
+#define TOF_ICC_REG_PORT_NUM_TX_FILTERED_NORMAL_TLP 0xf8
+#define TOF_ICC_REG_PORT_NUM_VIRTUAL_CUT_THROUGH_TLP 0x100
+#define TOF_ICC_REG_PORT_NUM_GENERATE_NULLIFIED_TLP 0x108
+#define TOF_ICC_REG_PORT_NUM_RECEIVE_DLLP 0x110
+#define TOF_ICC_REG_PORT_NUM_RECEIVE_TLP 0x118
+#define TOF_ICC_REG_PORT_BYTE_RECEIVE_TLP 0x120
+#define TOF_ICC_REG_PORT_NUM_RECEIVE_SYSTEM_TLP 0x128
+#define TOF_ICC_REG_PORT_NUM_RECEIVE_NULLIFIED_TLP 0x130
+#define TOF_ICC_REG_PORT_NUM_RX_DISCARD_SYSTEM_TLP 0x138
+#define TOF_ICC_REG_PORT_NUM_RX_DISCARD_NORMAL_TLP 0x140
+#define TOF_ICC_REG_PORT_NUM_RX_FILTERED_NORMAL_TLP 0x158
+#define TOF_ICC_REG_PORT_NUM_RX_DISCARD_NULLIFIED_TLP 0x160
+#define TOF_ICC_REG_PORT_FRAME_LCRC_ERROR_COUNT 0x170
+#define TOF_ICC_REG_PORT_TX_RETRY_BUFFER_CE_COUNT 0x180
+#define TOF_ICC_REG_PORT_RX_VC_BUFFER_CE_COUNT 0x188
+#define TOF_ICC_REG_PORT_XB_CE_COUNT 0x190
+#define TOF_ICC_REG_PORT_ACK_NACK_TIME_OUT_COUNT 0x198
+#define TOF_ICC_REG_PORT_SLICE0_FCS_ERROR_COUNT 0x1a0
+#define TOF_ICC_REG_PORT_SLICE1_FCS_ERROR_COUNT 0x1a8
+#define TOF_ICC_REG_PORT_DUMP_START 0x0
+#define TOF_ICC_REG_PORT_DUMP_END 0x1b0
+
+/* XB */
+#define TOF_ICC_REG_XB_PA (tof_icc_reg_pa + 0x000600f000)
+#define TOF_ICC_REG_XB_STQ_ENABLE 0x0
+#define TOF_ICC_REG_XB_STQ_UPDATE_INTERVAL 0x8
+#define TOF_ICC_REG_XB_STQ_PA 0x10
+#define TOF_ICC_REG_XB_STQ_SIZE 0x18
+#define TOF_ICC_REG_XB_STQ_NEXT_OFFSET 0x20
+#define TOF_ICC_REG_XB_DUMP_START 0x0
+#define TOF_ICC_REG_XB_DUMP_END 0x28
+
+#define TOF_ICC_XB_TC_DATA_CYCLE_COUNT(tni) ((tni) * 0x10 + 0x0)
+#define TOF_ICC_XB_TC_WAIT_CYCLE_COUNT(tni) ((tni) * 0x10 + 0x8)
+#define TOF_ICC_XB_TD_DATA_CYCLE_COUNT(tnr) ((tnr) * 0x10 + 0x60)
+#define TOF_ICC_XB_TD_WAIT_CYCLE_COUNT(tnr) ((tnr) * 0x10 + 0x68)
+
+/* Tofu */
+#define TOF_ICC_REG_TOFU_PA (tof_icc_reg_pa + 0x0007000000)
+#define TOF_ICC_REG_TOFU_NODE_ADDRESS 0x0
+#define TOF_ICC_REG_TOFU_NODE_ADDRESS_X GENMASK(22, 18)
+#define TOF_ICC_REG_TOFU_NODE_ADDRESS_Y GENMASK(17, 13)
+#define TOF_ICC_REG_TOFU_NODE_ADDRESS_Z GENMASK(12, 8)
+#define TOF_ICC_REG_TOFU_NODE_ADDRESS_A BIT(7)
+#define TOF_ICC_REG_TOFU_NODE_ADDRESS_B GENMASK(6, 5)
+#define TOF_ICC_REG_TOFU_NODE_ADDRESS_C BIT(4)
+#define TOF_ICC_REG_TOFU_PORT_SETTING 0x8
+#define TOF_ICC_REG_TOFU_TD_TLP_FILTER(tnr) ((tnr) * 0x10 + 0x10)
+#define TOF_ICC_REG_TOFU_TD_SETTINGS(tnr) ((tnr) * 0x10 + 0x18)
+#define TOF_ICC_REG_TOFU_TNR_MSI_BASE 0xc0
+#define TOF_ICC_REG_TOFU_TNR_IRR 0xc8
+#define TOF_ICC_REG_TOFU_TNR_IMR 0xd0
+#define TOF_ICC_REG_TOFU_TNR_IRC 0xd8
+#define TOF_ICC_REG_TOFU_TNR_IMC 0xe0
+#define TOF_ICC_REG_TOFU_TNR_ICL 0xe8
+#define TOF_ICC_REG_TOFU_TNI_VMS(tni, vmsid) ((tni) * 0x100 + (vmsid) * 0x8 + 0x100)
+#define TOF_ICC_REG_TOFU_TNI_VMS_CQ00(tni) ((tni) * 0x100 + 0x180)
+#define TOF_ICC_REG_TOFU_TNI_VMS_BG00(tni) ((tni) * 0x100 + 0x1a0)
+#define TOF_ICC_REG_TOFU_TNI_VMS_BG16(tni) ((tni) * 0x100 + 0x1a8)
+#define TOF_ICC_REG_TOFU_TNI_VMS_BG32(tni) ((tni) * 0x100 + 0x1b0)
+#define TOF_ICC_REG_TOFU_TNI_MSI_BASE(tni) ((tni) * 0x100 + 0x1c0)
+#define TOF_ICC_REG_TOFU_DUMP_START 0x0
+#define TOF_ICC_REG_TOFU_DUMP_END 0x6c8
+
+/** Interrupts **/
+#define TOF_ICC_IRQ_CQS_TOQ_READ_EXCEPTION BIT(0)
+#define TOF_ICC_IRQ_CQS_TOQ_DIRECT_DESCRIPTOR_EXCEPTION BIT(1)
+#define TOF_ICC_IRQ_CQS_TOQ_MARKED_UE BIT(2)
+#define TOF_ICC_IRQ_CQS_TCQ_WRITE_EXCEPTION BIT(3)
+#define TOF_ICC_IRQ_CQS_TOQ_SOURCE_TYPE_EXCEPTION BIT(4)
+#define TOF_ICC_IRQ_CQS_TCQ_WRITE_ACKNOWLEDGE BIT(5)
+#define TOF_ICC_IRQ_CQS_MRQ_WRITE_ACKNOWLEDGE BIT(7)
+#define TOF_ICC_IRQ_CQS_MRQ_WRITE_EXCEPTION BIT(8)
+#define TOF_ICC_IRQ_CQS_MRQ_OVERFLOW BIT(9)
+#define TOF_ICC_IRQ_CQS_STEERING_READ_EXCEPTION BIT(36)
+#define TOF_ICC_IRQ_CQS_MB_READ_EXCEPTION BIT(38)
+#define TOF_ICC_IRQ_CQS_PAYLOAD_READ_EXCEPTION BIT(39)
+#define TOF_ICC_IRQ_CQS_PAYLOAD_WRITE_EXCEPTION BIT(40)
+/* Just for convinience of irr value, no exists CQS CACHEFLUSH_TIMEOUT interrupt  */
+#define TOF_ICC_DUMMY_IRQ_CQS_CACHEFLUSH_TIMEOUT BIT(63)
+
+#define TOF_ICC_IRQ_BGS_NODE_ADDRESS_UNMATCH BIT(0)
+#define TOF_ICC_IRQ_BGS_BG_RECV_ADDRESS_EXCEPTION BIT(1)
+#define TOF_ICC_IRQ_BGS_BG_SEND_ADDRESS_EXCEPTION BIT(2)
+#define TOF_ICC_IRQ_BGS_GPID_UNMATCH BIT(3)
+#define TOF_ICC_IRQ_BGS_BSEQ_UNMATCH BIT(4)
+#define TOF_ICC_IRQ_BGS_SIGNAL_STATE_ERROR BIT(5)
+#define TOF_ICC_IRQ_BGS_SYNCHRONIZATION_ACKNOWLEDGE BIT(24)
+#define TOF_ICC_IRQ_BGS_ERROR_SYNCHRONIZATION_ACKNOWLEDGE BIT(25)
+#define TOF_ICC_IRQ_BGS_DMA_COMPLETION_EXCEPTION BIT(26)
+
+#define TOF_ICC_IRQ_TNI_PBQ_READ_EXCEPTION BIT(0)
+#define TOF_ICC_IRQ_TNI_PBQ_MARKED_UE BIT(1)
+#define TOF_ICC_IRQ_TNI_PBQ_UNDERFLOW BIT(2)
+#define TOF_ICC_IRQ_TNI_PRQ_PACKET_DISCARD BIT(3)
+#define TOF_ICC_IRQ_TNI_PRQ_WRITE_ACKNOWLEDGE BIT(4)
+#define TOF_ICC_IRQ_TNI_PRQ_WRITE_EXCEPTION BIT(5)
+#define TOF_ICC_IRQ_TNI_PRQ_OVERFLOW BIT(6)
+#define TOF_ICC_IRQ_TNI_INACTIVE_BG BIT(16)
+#define TOF_ICC_IRQ_TNI_STAGE2_TRANSLATION_FAULT BIT(32)
+
+#define TOF_ICC_IRQ_TNR_TNR0_RX_FILTER_OUT BIT(0)
+#define TOF_ICC_IRQ_TNR_TNR0_TX_FILTER_OUT BIT(1)
+#define TOF_ICC_IRQ_TNR_TNR0_PORT_ERROR BIT(2)
+#define TOF_ICC_IRQ_TNR_TNR0_DATELINE_ERROR BIT(3)
+#define TOF_ICC_IRQ_TNR_TNR0_ROUTING_ERROR BIT(4)
+#define TOF_ICC_IRQ_TNR_TNR1_RX_FILTER_OUT BIT(6)
+#define TOF_ICC_IRQ_TNR_TNR1_TX_FILTER_OUT BIT(7)
+#define TOF_ICC_IRQ_TNR_TNR1_PORT_ERROR BIT(8)
+#define TOF_ICC_IRQ_TNR_TNR1_DATELINE_ERROR BIT(9)
+#define TOF_ICC_IRQ_TNR_TNR1_ROUTING_ERROR BIT(10)
+#define TOF_ICC_IRQ_TNR_TNR2_RX_FILTER_OUT BIT(12)
+#define TOF_ICC_IRQ_TNR_TNR2_TX_FILTER_OUT BIT(13)
+#define TOF_ICC_IRQ_TNR_TNR2_PORT_ERROR BIT(14)
+#define TOF_ICC_IRQ_TNR_TNR2_DATELINE_ERROR BIT(15)
+#define TOF_ICC_IRQ_TNR_TNR2_ROUTING_ERROR BIT(16)
+#define TOF_ICC_IRQ_TNR_TNR3_RX_FILTER_OUT BIT(18)
+#define TOF_ICC_IRQ_TNR_TNR3_TX_FILTER_OUT BIT(19)
+#define TOF_ICC_IRQ_TNR_TNR3_PORT_ERROR BIT(20)
+#define TOF_ICC_IRQ_TNR_TNR3_DATELINE_ERROR BIT(21)
+#define TOF_ICC_IRQ_TNR_TNR3_ROUTING_ERROR BIT(22)
+#define TOF_ICC_IRQ_TNR_TNR4_RX_FILTER_OUT BIT(24)
+#define TOF_ICC_IRQ_TNR_TNR4_TX_FILTER_OUT BIT(25)
+#define TOF_ICC_IRQ_TNR_TNR4_PORT_ERROR BIT(26)
+#define TOF_ICC_IRQ_TNR_TNR4_DATELINE_ERROR BIT(27)
+#define TOF_ICC_IRQ_TNR_TNR4_ROUTING_ERROR BIT(28)
+#define TOF_ICC_IRQ_TNR_TNR5_RX_FILTER_OUT BIT(30)
+#define TOF_ICC_IRQ_TNR_TNR5_TX_FILTER_OUT BIT(31)
+#define TOF_ICC_IRQ_TNR_TNR5_PORT_ERROR BIT(32)
+#define TOF_ICC_IRQ_TNR_TNR5_DATELINE_ERROR BIT(33)
+#define TOF_ICC_IRQ_TNR_TNR5_ROUTING_ERROR BIT(34)
+#define TOF_ICC_IRQ_TNR_TNR6_RX_FILTER_OUT BIT(36)
+#define TOF_ICC_IRQ_TNR_TNR6_TX_FILTER_OUT BIT(37)
+#define TOF_ICC_IRQ_TNR_TNR6_PORT_ERROR BIT(38)
+#define TOF_ICC_IRQ_TNR_TNR6_DATELINE_ERROR BIT(39)
+#define TOF_ICC_IRQ_TNR_TNR6_ROUTING_ERROR BIT(40)
+#define TOF_ICC_IRQ_TNR_TNR7_RX_FILTER_OUT BIT(42)
+#define TOF_ICC_IRQ_TNR_TNR7_TX_FILTER_OUT BIT(43)
+#define TOF_ICC_IRQ_TNR_TNR7_PORT_ERROR BIT(44)
+#define TOF_ICC_IRQ_TNR_TNR7_DATELINE_ERROR BIT(45)
+#define TOF_ICC_IRQ_TNR_TNR7_ROUTING_ERROR BIT(46)
+#define TOF_ICC_IRQ_TNR_TNR8_RX_FILTER_OUT BIT(48)
+#define TOF_ICC_IRQ_TNR_TNR8_TX_FILTER_OUT BIT(49)
+#define TOF_ICC_IRQ_TNR_TNR8_PORT_ERROR BIT(50)
+#define TOF_ICC_IRQ_TNR_TNR8_DATELINE_ERROR BIT(51)
+#define TOF_ICC_IRQ_TNR_TNR8_ROUTING_ERROR BIT(52)
+#define TOF_ICC_IRQ_TNR_TNR9_RX_FILTER_OUT BIT(54)
+#define TOF_ICC_IRQ_TNR_TNR9_TX_FILTER_OUT BIT(55)
+#define TOF_ICC_IRQ_TNR_TNR9_PORT_ERROR BIT(56)
+#define TOF_ICC_IRQ_TNR_TNR9_DATELINE_ERROR BIT(57)
+#define TOF_ICC_IRQ_TNR_TNR9_ROUTING_ERROR BIT(58)
+
+#endif
+
+/* vim: set noet ts=8 sw=8 sts=0 tw=0 : */
+

executer/kernel/mcctrl/arch/arm64/include/tofu/tof_uapi.h

@@ -0,0 +1,345 @@
+#ifndef _TOF_UAPI_H_
+#define _TOF_UAPI_H_
+
+#ifdef __KERNEL__
+#include <linux/types.h>
+#else
+#include <stdint.h>
+#include <unistd.h>
+#include <sys/ioctl.h>
+#include <sys/types.h>
+#endif
+
+enum tof_sig_errno_cq {
+	TOF_TOQ_DIRECT_DESCRIPTOR_EXCEPTION,
+	TOF_TOQ_SOURCE_TYPE_EXCEPTION,
+	TOF_MRQ_OVERFLOW,
+	TOF_CQS_CACHEFLUSH_TIMEOUT,
+};
+
+enum tof_sig_errno_bg {
+	TOF_NODE_ADDRESS_UNMATCH,
+	TOF_BSEQ_UNMATCH,
+	TOF_SIGNAL_STATE_ERROR,
+	TOF_ERROR_SYNCHRONIZATION_ACKNOWLEDGE,
+};
+
+#define TOF_UAPI_VERSION 0x2a00
+
+struct tof_init_cq {
+	uint16_t version;
+	uint8_t session_mode;
+	uint8_t toq_size;
+	uint8_t mrq_size;
+	uint8_t num_stag;
+	uint8_t tcq_cinj;
+	uint8_t mrq_cinj;
+	void *toq_mem;
+	void *tcq_mem;
+	void *mrq_mem;
+};
+
+struct tof_alloc_stag {
+	uint32_t flags;
+	int stag;
+	uint64_t offset;
+	void *va;
+	uint64_t len;
+};
+
+struct tof_free_stags {
+	uint16_t num;
+	int *stags;
+};
+
+struct tof_addr {
+	uint8_t pa;
+	uint8_t pb;
+	uint8_t pc;
+	uint8_t x;
+	uint8_t y;
+	uint8_t z;
+	uint8_t a;
+	uint8_t b;
+	uint8_t c;
+};
+
+struct tof_set_bg {
+	int tni;
+	int gate;
+	int source_lgate;
+	struct tof_addr source_raddr;
+	int source_rtni;
+	int source_rgate;
+	int dest_lgate;
+	struct tof_addr dest_raddr;
+	int dest_rtni;
+	int dest_rgate;
+};
+
+struct tof_enable_bch {
+	void *addr;
+	int bseq;
+	int num;
+	struct tof_set_bg *bgs;
+};
+
+struct tof_set_subnet {
+	int res0;
+	int res1;
+	uint8_t nx;
+	uint8_t sx;
+	uint8_t lx;
+	uint8_t ny;
+	uint8_t sy;
+	uint8_t ly;
+	uint8_t nz;
+	uint8_t sz;
+	uint8_t lz;
+};
+
+struct tof_reg_user {
+	uid_t uid;
+	uint32_t gpid;
+	struct tof_set_subnet subnet;
+	uint64_t *cqmask;
+	uint64_t *bgmask;
+};
+
+struct tof_notify_linkdown {
+	int num;
+	struct {
+		uint8_t x;
+		uint8_t y;
+		uint8_t z;
+		uint8_t a;
+		uint8_t b;
+		uint8_t c;
+		uint16_t ports;
+	} *items;
+};
+
+struct tof_get_port_stat {
+	int port_no;
+	uint64_t mask;
+	uint64_t pa[31];
+};
+
+struct tof_get_cq_stat {
+	int tni;
+	int cqid;
+	uint64_t txbyte;
+	uint64_t rxbyte;
+};
+
+struct tof_load_register {
+	uint64_t pa;
+	uint64_t len;
+	void *buf;
+};
+
+struct tof_load_resource {
+	uint64_t rsc_id;
+	uint64_t offset;
+	uint64_t len;
+	void *buf;
+};
+
+union tof_trans_table_bitfield {
+	struct {
+		uint64_t start:36;
+		uint64_t len:27;
+		uint64_t ps_code:1;
+	} bits;
+	uint64_t atomic;
+};
+
+struct tof_trans_table {
+	union tof_trans_table_bitfield steering;
+	union tof_trans_table_bitfield mbpt;
+};
+
+void tof_utofu_set_linkdown_callback(void (*callback)(int, const void *));
+void tof_utofu_unset_linkdown_callback(void);
+
+#define TOF_MMAP_CQ_REGISTER 0
+#ifdef __KERNEL__
+#define TOF_MMAP_CQ_TRANSTABLE (PAGE_SIZE)
+#else
+#define TOF_MMAP_CQ_TRANSTABLE (sysconf(_SC_PAGESIZE))
+#endif
+#define TOF_MMAP_BCH_REGISTER 0
+#define TOF_MMAP_XB_STQ 0
+
+#define TOF_ST_RDWR 0x0
+#define TOF_ST_RDONLY 0x1
+#define TOF_ST_LPG 0x2
+
+#define TOF_STAG_TRANS_PS_CODE_64KB 0
+#define TOF_STAG_TRANS_PS_CODE_2MB  1
+
+#define TOF_IOC_MAGIC 'd'
+#define TOF_IOCTL_INIT_CQ _IOWR(TOF_IOC_MAGIC, 0, long)
+#define TOF_IOCTL_ALLOC_STAG _IOWR(TOF_IOC_MAGIC, 1, long)
+#define TOF_IOCTL_FREE_STAGS _IOWR(TOF_IOC_MAGIC, 2, long)
+#define TOF_IOCTL_ENABLE_BCH _IOWR(TOF_IOC_MAGIC, 3, long)
+#define TOF_IOCTL_DISABLE_BCH _IOWR(TOF_IOC_MAGIC, 4, long)
+#define TOF_IOCTL_SET_RT_SIGNAL _IOWR(TOF_IOC_MAGIC, 5, long)
+#define TOF_IOCTL_SET_SUBNET _IOWR(TOF_IOC_MAGIC, 6, long)
+#define TOF_IOCTL_REG_USER _IOWR(TOF_IOC_MAGIC, 7, long)
+#define TOF_IOCTL_NOTIFY_LINKDOWN _IOWR(TOF_IOC_MAGIC, 8, long)
+#define TOF_IOCTL_GET_PORT_STAT _IOWR(TOF_IOC_MAGIC, 9, long)
+#define TOF_IOCTL_GET_CQ_STAT _IOWR(TOF_IOC_MAGIC, 10, long)
+#define TOF_IOCTL_LOAD_REGISTER _IOWR(TOF_IOC_MAGIC, 11, long)
+#define TOF_IOCTL_LOAD_RESOURCE _IOWR(TOF_IOC_MAGIC, 12, long)
+
+#ifndef __KERNEL__
+#define TOF_INIT_CQ TOF_IOCTL_INIT_CQ
+#define TOF_ALLOC_STAG TOF_IOCTL_ALLOC_STAG
+#define TOF_FREE_STAGS TOF_IOCTL_FREE_STAGS
+#define TOF_ENABLE_BCH TOF_IOCTL_ENABLE_BCH
+#define TOF_DISABLE_BCH TOF_IOCTL_DISABLE_BCH
+#define TOF_SET_RT_SIGNAL TOF_IOCTL_SET_RT_SIGNAL
+#define TOF_SET_SUBNET TOF_IOCTL_SET_SUBNET
+#define TOF_REG_USER TOF_IOCTL_REG_USER
+#define TOF_NOTIFY_LINKDOWN TOF_IOCTL_NOTIFY_LINKDOWN
+#define TOF_GET_PORT_STAT TOF_IOCTL_GET_PORT_STAT
+#define TOF_GET_CQ_STAT TOF_IOCTL_GET_CQ_STAT
+#define TOF_LOAD_REGISTER TOF_IOCTL_LOAD_REGISTER
+#define TOF_LOAD_RESOURCE TOF_IOCTL_LOAD_RESOURCE
+#endif
+
+enum {
+	/* TOQ (0 - 71) */
+	TOF_RSC_TNI0_TOQ0 = 0, TOF_RSC_TNI0_TOQ1, TOF_RSC_TNI0_TOQ2, TOF_RSC_TNI0_TOQ3,
+	TOF_RSC_TNI0_TOQ4, TOF_RSC_TNI0_TOQ5, TOF_RSC_TNI0_TOQ6, TOF_RSC_TNI0_TOQ7,
+	TOF_RSC_TNI0_TOQ8, TOF_RSC_TNI0_TOQ9, TOF_RSC_TNI0_TOQ10, TOF_RSC_TNI0_TOQ11,
+	TOF_RSC_TNI1_TOQ0, TOF_RSC_TNI1_TOQ1, TOF_RSC_TNI1_TOQ2, TOF_RSC_TNI1_TOQ3,
+	TOF_RSC_TNI1_TOQ4, TOF_RSC_TNI1_TOQ5, TOF_RSC_TNI1_TOQ6, TOF_RSC_TNI1_TOQ7,
+	TOF_RSC_TNI1_TOQ8, TOF_RSC_TNI1_TOQ9, TOF_RSC_TNI1_TOQ10, TOF_RSC_TNI1_TOQ11,
+	TOF_RSC_TNI2_TOQ0, TOF_RSC_TNI2_TOQ1, TOF_RSC_TNI2_TOQ2, TOF_RSC_TNI2_TOQ3,
+	TOF_RSC_TNI2_TOQ4, TOF_RSC_TNI2_TOQ5, TOF_RSC_TNI2_TOQ6, TOF_RSC_TNI2_TOQ7,
+	TOF_RSC_TNI2_TOQ8, TOF_RSC_TNI2_TOQ9, TOF_RSC_TNI2_TOQ10, TOF_RSC_TNI2_TOQ11,
+	TOF_RSC_TNI3_TOQ0, TOF_RSC_TNI3_TOQ1, TOF_RSC_TNI3_TOQ2, TOF_RSC_TNI3_TOQ3,
+	TOF_RSC_TNI3_TOQ4, TOF_RSC_TNI3_TOQ5, TOF_RSC_TNI3_TOQ6, TOF_RSC_TNI3_TOQ7,
+	TOF_RSC_TNI3_TOQ8, TOF_RSC_TNI3_TOQ9, TOF_RSC_TNI3_TOQ10, TOF_RSC_TNI3_TOQ11,
+	TOF_RSC_TNI4_TOQ0, TOF_RSC_TNI4_TOQ1, TOF_RSC_TNI4_TOQ2, TOF_RSC_TNI4_TOQ3,
+	TOF_RSC_TNI4_TOQ4, TOF_RSC_TNI4_TOQ5, TOF_RSC_TNI4_TOQ6, TOF_RSC_TNI4_TOQ7,
+	TOF_RSC_TNI4_TOQ8, TOF_RSC_TNI4_TOQ9, TOF_RSC_TNI4_TOQ10, TOF_RSC_TNI4_TOQ11,
+	TOF_RSC_TNI5_TOQ0, TOF_RSC_TNI5_TOQ1, TOF_RSC_TNI5_TOQ2, TOF_RSC_TNI5_TOQ3,
+	TOF_RSC_TNI5_TOQ4, TOF_RSC_TNI5_TOQ5, TOF_RSC_TNI5_TOQ6, TOF_RSC_TNI5_TOQ7,
+	TOF_RSC_TNI5_TOQ8, TOF_RSC_TNI5_TOQ9, TOF_RSC_TNI5_TOQ10, TOF_RSC_TNI5_TOQ11,
+
+	/* TOQ (72 - 143) */
+	TOF_RSC_TNI0_TCQ0, TOF_RSC_TNI0_TCQ1, TOF_RSC_TNI0_TCQ2, TOF_RSC_TNI0_TCQ3,
+	TOF_RSC_TNI0_TCQ4, TOF_RSC_TNI0_TCQ5, TOF_RSC_TNI0_TCQ6, TOF_RSC_TNI0_TCQ7,
+	TOF_RSC_TNI0_TCQ8, TOF_RSC_TNI0_TCQ9, TOF_RSC_TNI0_TCQ10, TOF_RSC_TNI0_TCQ11,
+	TOF_RSC_TNI1_TCQ0, TOF_RSC_TNI1_TCQ1, TOF_RSC_TNI1_TCQ2, TOF_RSC_TNI1_TCQ3,
+	TOF_RSC_TNI1_TCQ4, TOF_RSC_TNI1_TCQ5, TOF_RSC_TNI1_TCQ6, TOF_RSC_TNI1_TCQ7,
+	TOF_RSC_TNI1_TCQ8, TOF_RSC_TNI1_TCQ9, TOF_RSC_TNI1_TCQ10, TOF_RSC_TNI1_TCQ11,
+	TOF_RSC_TNI2_TCQ0, TOF_RSC_TNI2_TCQ1, TOF_RSC_TNI2_TCQ2, TOF_RSC_TNI2_TCQ3,
+	TOF_RSC_TNI2_TCQ4, TOF_RSC_TNI2_TCQ5, TOF_RSC_TNI2_TCQ6, TOF_RSC_TNI2_TCQ7,
+	TOF_RSC_TNI2_TCQ8, TOF_RSC_TNI2_TCQ9, TOF_RSC_TNI2_TCQ10, TOF_RSC_TNI2_TCQ11,
+	TOF_RSC_TNI3_TCQ0, TOF_RSC_TNI3_TCQ1, TOF_RSC_TNI3_TCQ2, TOF_RSC_TNI3_TCQ3,
+	TOF_RSC_TNI3_TCQ4, TOF_RSC_TNI3_TCQ5, TOF_RSC_TNI3_TCQ6, TOF_RSC_TNI3_TCQ7,
+	TOF_RSC_TNI3_TCQ8, TOF_RSC_TNI3_TCQ9, TOF_RSC_TNI3_TCQ10, TOF_RSC_TNI3_TCQ11,
+	TOF_RSC_TNI4_TCQ0, TOF_RSC_TNI4_TCQ1, TOF_RSC_TNI4_TCQ2, TOF_RSC_TNI4_TCQ3,
+	TOF_RSC_TNI4_TCQ4, TOF_RSC_TNI4_TCQ5, TOF_RSC_TNI4_TCQ6, TOF_RSC_TNI4_TCQ7,
+	TOF_RSC_TNI4_TCQ8, TOF_RSC_TNI4_TCQ9, TOF_RSC_TNI4_TCQ10, TOF_RSC_TNI4_TCQ11,
+	TOF_RSC_TNI5_TCQ0, TOF_RSC_TNI5_TCQ1, TOF_RSC_TNI5_TCQ2, TOF_RSC_TNI5_TCQ3,
+	TOF_RSC_TNI5_TCQ4, TOF_RSC_TNI5_TCQ5, TOF_RSC_TNI5_TCQ6, TOF_RSC_TNI5_TCQ7,
+	TOF_RSC_TNI5_TCQ8, TOF_RSC_TNI5_TCQ9, TOF_RSC_TNI5_TCQ10, TOF_RSC_TNI5_TCQ11,
+
+	/* MRQ (144 - 215) */
+	TOF_RSC_TNI0_MRQ0, TOF_RSC_TNI0_MRQ1, TOF_RSC_TNI0_MRQ2, TOF_RSC_TNI0_MRQ3,
+	TOF_RSC_TNI0_MRQ4, TOF_RSC_TNI0_MRQ5, TOF_RSC_TNI0_MRQ6, TOF_RSC_TNI0_MRQ7,
+	TOF_RSC_TNI0_MRQ8, TOF_RSC_TNI0_MRQ9, TOF_RSC_TNI0_MRQ10, TOF_RSC_TNI0_MRQ11,
+	TOF_RSC_TNI1_MRQ0, TOF_RSC_TNI1_MRQ1, TOF_RSC_TNI1_MRQ2, TOF_RSC_TNI1_MRQ3,
+	TOF_RSC_TNI1_MRQ4, TOF_RSC_TNI1_MRQ5, TOF_RSC_TNI1_MRQ6, TOF_RSC_TNI1_MRQ7,
+	TOF_RSC_TNI1_MRQ8, TOF_RSC_TNI1_MRQ9, TOF_RSC_TNI1_MRQ10, TOF_RSC_TNI1_MRQ11,
+	TOF_RSC_TNI2_MRQ0, TOF_RSC_TNI2_MRQ1, TOF_RSC_TNI2_MRQ2, TOF_RSC_TNI2_MRQ3,
+	TOF_RSC_TNI2_MRQ4, TOF_RSC_TNI2_MRQ5, TOF_RSC_TNI2_MRQ6, TOF_RSC_TNI2_MRQ7,
+	TOF_RSC_TNI2_MRQ8, TOF_RSC_TNI2_MRQ9, TOF_RSC_TNI2_MRQ10, TOF_RSC_TNI2_MRQ11,
+	TOF_RSC_TNI3_MRQ0, TOF_RSC_TNI3_MRQ1, TOF_RSC_TNI3_MRQ2, TOF_RSC_TNI3_MRQ3,
+	TOF_RSC_TNI3_MRQ4, TOF_RSC_TNI3_MRQ5, TOF_RSC_TNI3_MRQ6, TOF_RSC_TNI3_MRQ7,
+	TOF_RSC_TNI3_MRQ8, TOF_RSC_TNI3_MRQ9, TOF_RSC_TNI3_MRQ10, TOF_RSC_TNI3_MRQ11,
+	TOF_RSC_TNI4_MRQ0, TOF_RSC_TNI4_MRQ1, TOF_RSC_TNI4_MRQ2, TOF_RSC_TNI4_MRQ3,
+	TOF_RSC_TNI4_MRQ4, TOF_RSC_TNI4_MRQ5, TOF_RSC_TNI4_MRQ6, TOF_RSC_TNI4_MRQ7,
+	TOF_RSC_TNI4_MRQ8, TOF_RSC_TNI4_MRQ9, TOF_RSC_TNI4_MRQ10, TOF_RSC_TNI4_MRQ11,
+	TOF_RSC_TNI5_MRQ0, TOF_RSC_TNI5_MRQ1, TOF_RSC_TNI5_MRQ2, TOF_RSC_TNI5_MRQ3,
+	TOF_RSC_TNI5_MRQ4, TOF_RSC_TNI5_MRQ5, TOF_RSC_TNI5_MRQ6, TOF_RSC_TNI5_MRQ7,
+	TOF_RSC_TNI5_MRQ8, TOF_RSC_TNI5_MRQ9, TOF_RSC_TNI5_MRQ10, TOF_RSC_TNI5_MRQ11,
+
+	/* PBQ (216 - 221) */
+	TOF_RSC_TNI0_PBQ, TOF_RSC_TNI1_PBQ, TOF_RSC_TNI2_PBQ, TOF_RSC_TNI3_PBQ,
+	TOF_RSC_TNI4_PBQ, TOF_RSC_TNI5_PBQ,
+
+	/* PRQ (222 - 227) */
+	TOF_RSC_TNI0_PRQ, TOF_RSC_TNI1_PRQ, TOF_RSC_TNI2_PRQ, TOF_RSC_TNI3_PRQ,
+	TOF_RSC_TNI4_PRQ, TOF_RSC_TNI5_PRQ,
+
+	/* STEERINGTABLE (228 - 299) */
+	TOF_RSC_TNI0_STEERINGTABLE0, TOF_RSC_TNI0_STEERINGTABLE1, TOF_RSC_TNI0_STEERINGTABLE2,
+	TOF_RSC_TNI0_STEERINGTABLE3, TOF_RSC_TNI0_STEERINGTABLE4, TOF_RSC_TNI0_STEERINGTABLE5,
+	TOF_RSC_TNI0_STEERINGTABLE6, TOF_RSC_TNI0_STEERINGTABLE7, TOF_RSC_TNI0_STEERINGTABLE8,
+	TOF_RSC_TNI0_STEERINGTABLE9, TOF_RSC_TNI0_STEERINGTABLE10, TOF_RSC_TNI0_STEERINGTABLE11,
+	TOF_RSC_TNI1_STEERINGTABLE0, TOF_RSC_TNI1_STEERINGTABLE1, TOF_RSC_TNI1_STEERINGTABLE2,
+	TOF_RSC_TNI1_STEERINGTABLE3, TOF_RSC_TNI1_STEERINGTABLE4, TOF_RSC_TNI1_STEERINGTABLE5,
+	TOF_RSC_TNI1_STEERINGTABLE6, TOF_RSC_TNI1_STEERINGTABLE7, TOF_RSC_TNI1_STEERINGTABLE8,
+	TOF_RSC_TNI1_STEERINGTABLE9, TOF_RSC_TNI1_STEERINGTABLE10, TOF_RSC_TNI1_STEERINGTABLE11,
+	TOF_RSC_TNI2_STEERINGTABLE0, TOF_RSC_TNI2_STEERINGTABLE1, TOF_RSC_TNI2_STEERINGTABLE2,
+	TOF_RSC_TNI2_STEERINGTABLE3, TOF_RSC_TNI2_STEERINGTABLE4, TOF_RSC_TNI2_STEERINGTABLE5,
+	TOF_RSC_TNI2_STEERINGTABLE6, TOF_RSC_TNI2_STEERINGTABLE7, TOF_RSC_TNI2_STEERINGTABLE8,
+	TOF_RSC_TNI2_STEERINGTABLE9, TOF_RSC_TNI2_STEERINGTABLE10, TOF_RSC_TNI2_STEERINGTABLE11,
+	TOF_RSC_TNI3_STEERINGTABLE0, TOF_RSC_TNI3_STEERINGTABLE1, TOF_RSC_TNI3_STEERINGTABLE2,
+	TOF_RSC_TNI3_STEERINGTABLE3, TOF_RSC_TNI3_STEERINGTABLE4, TOF_RSC_TNI3_STEERINGTABLE5,
+	TOF_RSC_TNI3_STEERINGTABLE6, TOF_RSC_TNI3_STEERINGTABLE7, TOF_RSC_TNI3_STEERINGTABLE8,
+        TOF_RSC_TNI3_STEERINGTABLE9, TOF_RSC_TNI3_STEERINGTABLE10, TOF_RSC_TNI3_STEERINGTABLE11,
+        TOF_RSC_TNI4_STEERINGTABLE0, TOF_RSC_TNI4_STEERINGTABLE1, TOF_RSC_TNI4_STEERINGTABLE2,
+        TOF_RSC_TNI4_STEERINGTABLE3, TOF_RSC_TNI4_STEERINGTABLE4, TOF_RSC_TNI4_STEERINGTABLE5,
+        TOF_RSC_TNI4_STEERINGTABLE6, TOF_RSC_TNI4_STEERINGTABLE7, TOF_RSC_TNI4_STEERINGTABLE8,
+        TOF_RSC_TNI4_STEERINGTABLE9, TOF_RSC_TNI4_STEERINGTABLE10, TOF_RSC_TNI4_STEERINGTABLE11,
+        TOF_RSC_TNI5_STEERINGTABLE3, TOF_RSC_TNI5_STEERINGTABLE4, TOF_RSC_TNI5_STEERINGTABLE5,
+        TOF_RSC_TNI5_STEERINGTABLE6, TOF_RSC_TNI5_STEERINGTABLE7, TOF_RSC_TNI5_STEERINGTABLE8,
+        TOF_RSC_TNI5_STEERINGTABLE9, TOF_RSC_TNI5_STEERINGTABLE10, TOF_RSC_TNI5_STEERINGTABLE11,
+
+        /* MBTABLE (300 - 371) */
+        TOF_RSC_TNI0_MBTABLE0, TOF_RSC_TNI0_MBTABLE1, TOF_RSC_TNI0_MBTABLE2,
+        TOF_RSC_TNI0_MBTABLE3, TOF_RSC_TNI0_MBTABLE4, TOF_RSC_TNI0_MBTABLE5,
+        TOF_RSC_TNI0_MBTABLE6, TOF_RSC_TNI0_MBTABLE7, TOF_RSC_TNI0_MBTABLE8,
+        TOF_RSC_TNI0_MBTABLE9, TOF_RSC_TNI0_MBTABLE10, TOF_RSC_TNI0_MBTABLE11,
+        TOF_RSC_TNI1_MBTABLE0, TOF_RSC_TNI1_MBTABLE1, TOF_RSC_TNI1_MBTABLE2,
+        TOF_RSC_TNI1_MBTABLE3, TOF_RSC_TNI1_MBTABLE4, TOF_RSC_TNI1_MBTABLE5,
+        TOF_RSC_TNI1_MBTABLE6, TOF_RSC_TNI1_MBTABLE7, TOF_RSC_TNI1_MBTABLE8,
+        TOF_RSC_TNI1_MBTABLE9, TOF_RSC_TNI1_MBTABLE10, TOF_RSC_TNI1_MBTABLE11,
+        TOF_RSC_TNI2_MBTABLE0, TOF_RSC_TNI2_MBTABLE1, TOF_RSC_TNI2_MBTABLE2,
+        TOF_RSC_TNI2_MBTABLE3, TOF_RSC_TNI2_MBTABLE4, TOF_RSC_TNI2_MBTABLE5,
+        TOF_RSC_TNI2_MBTABLE6, TOF_RSC_TNI2_MBTABLE7, TOF_RSC_TNI2_MBTABLE8,
+        TOF_RSC_TNI2_MBTABLE9, TOF_RSC_TNI2_MBTABLE10, TOF_RSC_TNI2_MBTABLE11,
+        TOF_RSC_TNI3_MBTABLE0, TOF_RSC_TNI3_MBTABLE1, TOF_RSC_TNI3_MBTABLE2,
+        TOF_RSC_TNI3_MBTABLE3, TOF_RSC_TNI3_MBTABLE4, TOF_RSC_TNI3_MBTABLE5,
+        TOF_RSC_TNI3_MBTABLE6, TOF_RSC_TNI3_MBTABLE7, TOF_RSC_TNI3_MBTABLE8,
+        TOF_RSC_TNI3_MBTABLE9, TOF_RSC_TNI3_MBTABLE10, TOF_RSC_TNI3_MBTABLE11,
+        TOF_RSC_TNI4_MBTABLE0, TOF_RSC_TNI4_MBTABLE1, TOF_RSC_TNI4_MBTABLE2,
+        TOF_RSC_TNI4_MBTABLE3, TOF_RSC_TNI4_MBTABLE4, TOF_RSC_TNI4_MBTABLE5,
+        TOF_RSC_TNI4_MBTABLE6, TOF_RSC_TNI4_MBTABLE7, TOF_RSC_TNI4_MBTABLE8,
+        TOF_RSC_TNI4_MBTABLE9, TOF_RSC_TNI4_MBTABLE10, TOF_RSC_TNI4_MBTABLE11,
+        TOF_RSC_TNI5_MBTABLE0, TOF_RSC_TNI5_MBTABLE1, TOF_RSC_TNI5_MBTABLE2,
+        TOF_RSC_TNI5_MBTABLE3, TOF_RSC_TNI5_MBTABLE4, TOF_RSC_TNI5_MBTABLE5,
+        TOF_RSC_TNI5_MBTABLE6, TOF_RSC_TNI5_MBTABLE7, TOF_RSC_TNI5_MBTABLE8,
+        TOF_RSC_TNI5_MBTABLE9, TOF_RSC_TNI5_MBTABLE10, TOF_RSC_TNI5_MBTABLE11,
+
+        TOF_RSC_NUM     /* 372 */
+};
+#define TOF_RSC_TOQ(TNI, CQID)  (TOF_RSC_TNI0_TOQ0 + (TNI * 12) + CQID)
+#define TOF_RSC_TCQ(TNI, CQID)  (TOF_RSC_TNI0_TCQ0 + (TNI * 12) + CQID)
+#define TOF_RSC_MRQ(TNI, CQID)  (TOF_RSC_TNI0_MRQ0 + (TNI * 12) + CQID)
+#define TOF_RSC_PBQ(TNI)        (TOF_RSC_TNI0_PBQ + TNI)
+#define TOF_RSC_PRQ(TNI)        (TOF_RSC_TNI0_PRQ + TNI)
+#define TOF_RSC_STT(TNI, CQID)  (TOF_RSC_TNI0_STEERINGTABLE0 + (TNI * 12) + CQID)
+#define TOF_RSC_MBT(TNI, CQID)  (TOF_RSC_TNI0_MBTABLE0 + (TNI * 12) + CQID)
+
+#endif
+
+/* vim: set noet ts=8 sw=8 sts=0 tw=0 : */
+

executer/kernel/mcctrl/arch/arm64/include/tofu/tof_utofu_bg_bch.h

@@ -0,0 +1,6 @@
+	struct {
+		bool enabled;
+		uint64_t bgmask[TOF_ICC_NTNIS];
+		uintptr_t iova;
+		void *kaddr;
+	} bch;

executer/kernel/mcctrl/arch/arm64/include/tofu/tof_utofu_cq_trans.h

@@ -0,0 +1,6 @@
+			struct {
+				struct tof_utofu_trans_list *mru;
+				struct tof_trans_table *table;
+				int mruhead;
+				ihk_spinlock_t mru_lock;
+			} trans;

executer/kernel/mcctrl/arch/arm64/include/tofu/tofu_generated-tof_utofu_bg.h

@@ -0,0 +1,21 @@
+struct tof_utofu_bg {
+	union {
+		char whole_struct[160];
+		struct {
+			char padding0[0];
+			struct tof_utofu_device common;
+		};
+		struct {
+			char padding1[80];
+			uint8_t tni;
+		};
+		struct {
+			char padding2[81];
+			uint8_t bgid;
+		};
+		struct {
+			char padding3[88];
+			#include "tof_utofu_bg_bch.h"
+		};
+	};
+};

executer/kernel/mcctrl/arch/arm64/include/tofu/tofu_generated-tof_utofu_cq.h

@@ -0,0 +1,37 @@
+struct tof_utofu_cq {
+	union {
+		char whole_struct[384];
+		struct {
+			char padding0[0];
+			struct tof_utofu_device common;
+		};
+		struct {
+			char padding1[80];
+			uint8_t tni;
+		};
+		struct {
+			char padding2[81];
+			uint8_t cqid;
+		};
+		struct {
+			char padding3[104];
+			#include "tof_utofu_cq_trans.h"
+		};
+		struct {
+			char padding4[128];
+			struct tof_icc_steering_entry *steering;
+		};
+		struct {
+			char padding5[136];
+			struct tof_icc_mb_entry *mb;
+		};
+		struct {
+			char padding6[186];
+			uint8_t num_stag;
+		};
+		struct {
+			char padding7[336];
+			struct mmu_notifier mn;
+		};
+	};
+};

executer/kernel/mcctrl/arch/arm64/include/tofu/tofu_generated-tof_utofu_device.h

@@ -0,0 +1,17 @@
+struct tof_utofu_device {
+	union {
+		char whole_struct[80];
+		struct {
+			char padding0[0];
+			bool enabled;
+		};
+		struct {
+			char padding1[12];
+			uint32_t gpid;
+		};
+		struct {
+			char padding2[24];
+			uint64_t subnet;
+		};
+	};
+};

executer/kernel/mcctrl/arch/arm64/include/tofu/tofu_generated-tof_utofu_mbpt.h

@@ -0,0 +1,33 @@
+struct tof_utofu_mbpt {
+	union {
+		char whole_struct[56];
+		struct {
+			char padding0[0];
+			struct kref kref;
+		};
+		struct {
+			char padding1[8];
+			struct tof_utofu_cq *ucq;
+		};
+		struct {
+			char padding2[16];
+			uintptr_t iova;
+		};
+		struct {
+			char padding3[24];
+			struct scatterlist *sg;
+		};
+		struct {
+			char padding4[32];
+			size_t nsgents;
+		};
+		struct {
+			char padding5[40];
+			uintptr_t mbptstart;
+		};
+		struct {
+			char padding6[48];
+			size_t pgsz;
+		};
+	};
+};

executer/kernel/mcctrl/control.c

@@ -36,6 +36,7 @@
 #include <linux/semaphore.h>
 #include <linux/interrupt.h>
 #include <linux/cpumask.h>
+#include <linux/delay.h>
 #include <asm/uaccess.h>
 #include <asm/delay.h>
 #include <asm/io.h>
@@ -229,6 +230,9 @@ static long mcexec_prepare_image(ihk_os_t os,
 
 	dprintk("%s: pid %d, rpgtable: 0x%lx added\n",
 		__FUNCTION__, ppd->pid, ppd->rpgtable);
+#ifdef ENABLE_TOFU
+	ppd->enable_tofu = pdesc->enable_tofu;
+#endif
 
 	ret = 0;
 
@@ -266,18 +270,24 @@ int mcexec_transfer_image(ihk_os_t os, struct remote_transfer *__user upt)
 		return -EFAULT;
 	}
 
+#ifdef CONFIG_MIC
 	if (pt.size > PAGE_SIZE) {
 		printk("mcexec_transfer_image(): ERROR: size exceeds PAGE_SIZE\n");
 		return -EFAULT;
 	}
-	
+
 	phys = ihk_device_map_memory(ihk_os_to_dev(os), pt.rphys, PAGE_SIZE);
-#ifdef CONFIG_MIC
 	rpm = ioremap_wc(phys, PAGE_SIZE);
 #else
-	rpm = ihk_device_map_virtual(ihk_os_to_dev(os), phys, PAGE_SIZE, NULL, 0);
+	phys = ihk_device_map_memory(ihk_os_to_dev(os), pt.rphys, pt.size);
+	rpm = ihk_device_map_virtual(ihk_os_to_dev(os), phys, pt.size, NULL, 0);
 #endif
-	
+
+	if (!rpm) {
+		pr_err("%s(): error: invalid remote address\n", __func__);
+		return -EFAULT;
+	}
+
 	if (pt.direction == MCEXEC_UP_TRANSFER_TO_REMOTE) {
 		if (copy_from_user(rpm, pt.userp, pt.size)) {
 			ret = -EFAULT;
@@ -295,10 +305,11 @@ int mcexec_transfer_image(ihk_os_t os, struct remote_transfer *__user upt)
 
 #ifdef CONFIG_MIC
 	iounmap(rpm);
+	ihk_device_unmap_memory(ihk_os_to_dev(os), phys, PAGE_SIZE);
 #else
-	ihk_device_unmap_virtual(ihk_os_to_dev(os), rpm, PAGE_SIZE);
+	ihk_device_unmap_virtual(ihk_os_to_dev(os), rpm, pt.size);
+	ihk_device_unmap_memory(ihk_os_to_dev(os), phys, pt.size);
 #endif
-	ihk_device_unmap_memory(ihk_os_to_dev(os), phys, PAGE_SIZE);	
 
 	return ret;
 
@@ -378,6 +389,7 @@ static void release_handler(ihk_os_t os, void *param)
 	int os_ind = ihk_host_os_get_index(os);
 	unsigned long flags;
 	struct host_thread *thread;
+	int ret;
 
 	/* Finalize FS switch for uti threads */ 
 	write_lock_irqsave(&host_thread_lock, flags);
@@ -399,7 +411,13 @@ static void release_handler(ihk_os_t os, void *param)
 
 	dprintk("%s: SCD_MSG_CLEANUP_PROCESS, info: %p, cpu: %d\n",
 			__FUNCTION__, info, info->cpu);
-	mcctrl_ikc_send(os, info->cpu, &isp);
+	ret = mcctrl_ikc_send_wait(os, info->cpu,
+			&isp, -20, NULL, NULL, 0);
+	if (ret != 0) {
+		printk("%s: WARNING: failed to send IKC msg: %d\n",
+				__func__, ret);
+	}
+
 	if (os_ind >= 0) {
 		delete_pid_entry(os_ind, info->pid);
 	}
@@ -587,13 +605,14 @@ extern int mckernel_cpu_2_linux_cpu(struct mcctrl_usrdata *udp, int cpu_id);
 static long mcexec_get_cpuset(ihk_os_t os, unsigned long arg)
 {
 	struct mcctrl_usrdata *udp = ihk_host_os_get_usrdata(os);
-	struct mcctrl_part_exec *pe;
+	struct mcctrl_part_exec *pe = NULL, *pe_itr;
 	struct get_cpu_set_arg req;
 	struct mcctrl_cpu_topology *cpu_top, *cpu_top_i;
 	struct cache_topology *cache_top;
 	int cpu, cpus_assigned, cpus_to_assign, cpu_prev;
 	int ret = 0;
 	int mcexec_linux_numa;
+	int pe_list_len = 0;
 	cpumask_t *mcexec_cpu_set = NULL;
 	cpumask_t *cpus_used = NULL;
 	cpumask_t *cpus_to_use = NULL;
@@ -614,7 +633,7 @@ static long mcexec_get_cpuset(ihk_os_t os, unsigned long arg)
 	}
 
 	if (copy_from_user(&req, (void *)arg, sizeof(req))) {
-		printk("%s: error copying user request\n", __FUNCTION__);
+		pr_err("%s: error copying user request\n", __func__);
 		ret = -EINVAL;
 		goto put_out;
 	}
@@ -691,18 +710,48 @@ static long mcexec_get_cpuset(ihk_os_t os, unsigned long arg)
 		goto put_out;
 	}
 
-	pe = &udp->part_exec;
+	mutex_lock(&udp->part_exec_lock);
+	/* Find part_exec having same node_proxy */
+	list_for_each_entry_reverse(pe_itr, &udp->part_exec_list, chain) {
+		pe_list_len++;
+		if (pe_itr->node_proxy_pid == req.ppid) {
+			pe = pe_itr;
+			break;
+		}
+	}
 
-	mutex_lock(&pe->lock);
+	if (!pe) {
+		/* First process to enter CPU partitioning */
+		pr_debug("%s: pe_list_len:%d\n", __func__, pe_list_len);
+		if (pe_list_len >= PE_LIST_MAXLEN) {
+			/* delete head entry of pe_list */
+			pe_itr = list_first_entry(&udp->part_exec_list,
+					struct mcctrl_part_exec, chain);
+			list_del(&pe_itr->chain);
+			kfree(pe_itr);
+		}
 
-	/* First process to enter CPU partitioning */
-	if (pe->nr_processes == -1) {
+		pe = kzalloc(sizeof(struct mcctrl_part_exec), GFP_KERNEL);
+		if (!pe) {
+			mutex_unlock(&udp->part_exec_lock);
+			ret = -ENOMEM;
+			goto put_out;
+		}
+		/* Init part_exec */
+		mutex_init(&pe->lock);
+		INIT_LIST_HEAD(&pe->pli_list);
 		pe->nr_processes = req.nr_processes;
 		pe->nr_processes_left = req.nr_processes;
+		pe->nr_processes_joined = 0;
+		pe->node_proxy_pid = req.ppid;
+
+		list_add_tail(&pe->chain, &udp->part_exec_list);
 		dprintk("%s: nr_processes: %d (partitioned exec starts)\n",
-				__FUNCTION__,
-				pe->nr_processes);
+				__func__, pe->nr_processes);
 	}
+	mutex_unlock(&udp->part_exec_lock);
+
+	mutex_lock(&pe->lock);
 
 	if (pe->nr_processes != req.nr_processes) {
 		printk("%s: error: requested number of processes"
@@ -712,7 +761,15 @@ static long mcexec_get_cpuset(ihk_os_t os, unsigned long arg)
 		goto put_and_unlock_out;
 	}
 
+	if (pe->nr_processes_joined >= pe->nr_processes) {
+		printk("%s: too many processes have joined to the group of %d\n",
+				__func__, req.ppid);
+		ret = -EINVAL;
+		goto put_and_unlock_out;
+	}
+
 	--pe->nr_processes_left;
+	++pe->nr_processes_joined;
 	dprintk("%s: nr_processes: %d, nr_processes_left: %d\n",
 			__FUNCTION__,
 			pe->nr_processes,
@@ -798,8 +855,6 @@ static long mcexec_get_cpuset(ihk_os_t os, unsigned long arg)
 				wake_up_interruptible(&pli_next->pli_wq);
 			}
 
-			/* Reset process counter to start state */
-			pe->nr_processes = -1;
 			ret = -ETIMEDOUT;
 			goto put_and_unlock_out;
 		}
@@ -1047,16 +1102,8 @@ next_cpu:
 	/* Commit used cores to OS structure */
 	memcpy(&pe->cpus_used, cpus_used, sizeof(*cpus_used));
 
-	/* Reset if last process */
-	if (pe->nr_processes_left == 0) {
-		dprintk("%s: nr_processes: %d (partitioned exec ends)\n",
-				__FUNCTION__,
-				pe->nr_processes);
-		pe->nr_processes = -1;
-		memset(&pe->cpus_used, 0, sizeof(pe->cpus_used));
-	}
-	/* Otherwise wake up next process in list */
-	else {
+	/* If not last process, wake up next process in list */
+	if (pe->nr_processes_left != 0) {
 		++pe->process_rank;
 		pli_next = list_first_entry(&pe->pli_list,
 			struct process_list_item, list);
@@ -1221,7 +1268,7 @@ void mcctrl_put_per_proc_data(struct mcctrl_per_proc_data *ppd)
 			   process is gone and the application should be terminated. */
 			packet = (struct ikc_scd_packet *)ptd->data;
 			dprintk("%s: calling __return_syscall (hash),target pid=%d,tid=%d\n", __FUNCTION__, ppd->pid, packet->req.rtid);
-			__return_syscall(ppd->ud->os, packet, -ERESTARTSYS,
+			__return_syscall(ppd->ud->os, ppd, packet, -ERESTARTSYS,
 					 packet->req.rtid);
 			ihk_ikc_release_packet((struct ihk_ikc_free_packet *)packet);
 
@@ -1245,7 +1292,7 @@ void mcctrl_put_per_proc_data(struct mcctrl_per_proc_data *ppd)
 
 		/* We use ERESTARTSYS to tell the LWK that the proxy
 		 * process is gone and the application should be terminated */
-		__return_syscall(ppd->ud->os, packet, -ERESTARTSYS,
+		__return_syscall(ppd->ud->os, ppd, packet, -ERESTARTSYS,
 				packet->req.rtid);
 		ihk_ikc_release_packet((struct ihk_ikc_free_packet *)packet);
 	}
@@ -1286,7 +1333,7 @@ int mcexec_syscall(struct mcctrl_usrdata *ud, struct ikc_scd_packet *packet)
 
 		/* We use ERESTARTSYS to tell the LWK that the proxy
 		 * process is gone and the application should be terminated */
-		__return_syscall(ud->os, packet, -ERESTARTSYS,
+		__return_syscall(ud->os, NULL, packet, -ERESTARTSYS,
 				packet->req.rtid);
 		ihk_ikc_release_packet((struct ihk_ikc_free_packet *)packet);
 
@@ -1729,7 +1776,7 @@ long mcexec_ret_syscall(ihk_os_t os, struct syscall_ret_desc *__user arg)
 		ihk_device_unmap_memory(ihk_os_to_dev(os), phys, ret.size);
 	} 
 
-	__return_syscall(os, packet, ret.ret, task_pid_vnr(current));
+	__return_syscall(os, ppd, packet, ret.ret, task_pid_vnr(current));
 
 	error = 0;
 out:
@@ -2172,7 +2219,13 @@ static DECLARE_WAIT_QUEUE_HEAD(perfctrlq);
 
 long mcctrl_perf_num(ihk_os_t os, unsigned long arg)
 {
-	struct mcctrl_usrdata *usrdata = ihk_host_os_get_usrdata(os);
+	struct mcctrl_usrdata *usrdata;
+
+	if (!os || ihk_host_validate_os(os)) {
+		return -EINVAL;
+	}
+
+	usrdata = ihk_host_os_get_usrdata(os);
 
 	if (!usrdata) {
 		pr_err("%s: error: mcctrl_usrdata not found\n", __func__);
@@ -2197,22 +2250,34 @@ struct mcctrl_perf_ctrl_desc {
  */
 long mcctrl_perf_set(ihk_os_t os, struct ihk_perf_event_attr *__user arg)
 {
-	struct mcctrl_usrdata *usrdata = ihk_host_os_get_usrdata(os);
+	struct mcctrl_usrdata *usrdata = NULL;
 	struct ikc_scd_packet isp;
 	struct perf_ctrl_desc *perf_desc;
 	struct ihk_perf_event_attr attr;
-	struct ihk_cpu_info *info = ihk_os_get_cpu_info(os);
+	struct ihk_cpu_info *info = NULL;
 	int ret = 0;
 	int i = 0, j = 0;
 	int need_free;
 	int num_registered = 0;
 	int err = 0;
 
+	if (!os || ihk_host_validate_os(os)) {
+		return -EINVAL;
+	}
+
+	usrdata = ihk_host_os_get_usrdata(os);
+
 	if (!usrdata) {
 		pr_err("%s: error: mcctrl_usrdata not found\n", __func__);
 		return -EINVAL;
 	}
 
+	info = ihk_os_get_cpu_info(os);
+	if (!info) {
+		pr_err("%s: error: cannot get cpu info\n", __func__);
+		return -EINVAL;
+	}
+
 	for (i = 0; i < usrdata->perf_event_num; i++) {
 		ret = copy_from_user(&attr, &arg[i],
 				     sizeof(struct ihk_perf_event_attr));
@@ -2272,20 +2337,30 @@ long mcctrl_perf_set(ihk_os_t os, struct ihk_perf_event_attr *__user arg)
 
 long mcctrl_perf_get(ihk_os_t os, unsigned long *__user arg)
 {
-	struct mcctrl_usrdata *usrdata = ihk_host_os_get_usrdata(os);
+	struct mcctrl_usrdata *usrdata = NULL;
 	struct ikc_scd_packet isp;
 	struct perf_ctrl_desc *perf_desc;
-	struct ihk_cpu_info *info = ihk_os_get_cpu_info(os);
+	struct ihk_cpu_info *info = NULL;
 	unsigned long value_sum = 0;
 	int ret = 0;
 	int i = 0, j = 0;
 	int need_free;
 
+	if (!os || ihk_host_validate_os(os)) {
+		return -EINVAL;
+	}
+
+	usrdata = ihk_host_os_get_usrdata(os);
 	if (!usrdata) {
 		pr_err("%s: error: mcctrl_usrdata not found\n", __func__);
 		return -EINVAL;
 	}
 
+	info = ihk_os_get_cpu_info(os);
+	if (!info || info->n_cpus < 1) {
+		return -EINVAL;
+	}
+
 	for (i = 0; i < usrdata->perf_event_num; i++) {
 		perf_desc = kmalloc(sizeof(struct mcctrl_perf_ctrl_desc),
 				    GFP_KERNEL);
@@ -2333,15 +2408,20 @@ long mcctrl_perf_get(ihk_os_t os, unsigned long *__user arg)
 
 long mcctrl_perf_enable(ihk_os_t os)
 {
-	struct mcctrl_usrdata *usrdata = ihk_host_os_get_usrdata(os);
+	struct mcctrl_usrdata *usrdata = NULL;
 	struct ikc_scd_packet isp;
 	struct perf_ctrl_desc *perf_desc;
-	struct ihk_cpu_info *info = ihk_os_get_cpu_info(os);
+	struct ihk_cpu_info *info = NULL;
 	unsigned long cntr_mask = 0;
 	int ret = 0;
 	int i = 0, j = 0;
 	int need_free;
 
+	if (!os || ihk_host_validate_os(os)) {
+		return -EINVAL;
+	}
+
+	usrdata = ihk_host_os_get_usrdata(os);
 	if (!usrdata) {
 		pr_err("%s: error: mcctrl_usrdata not found\n", __func__);
 		return -EINVAL;
@@ -2364,6 +2444,11 @@ long mcctrl_perf_enable(ihk_os_t os)
 	isp.msg = SCD_MSG_PERF_CTRL;
 	isp.arg = virt_to_phys(perf_desc);
 
+	info = ihk_os_get_cpu_info(os);
+	if (!info || info->n_cpus < 1) {
+		kfree(perf_desc);
+		return -EINVAL;
+	}
 	for (j = 0; j < info->n_cpus; j++) {
 		ret = mcctrl_ikc_send_wait(os, j, &isp, 0,
 					   wakeup_desc_of_perf_desc(perf_desc),
@@ -2391,15 +2476,20 @@ long mcctrl_perf_enable(ihk_os_t os)
 
 long mcctrl_perf_disable(ihk_os_t os)
 {
-	struct mcctrl_usrdata *usrdata = ihk_host_os_get_usrdata(os);
+	struct mcctrl_usrdata *usrdata = NULL;
 	struct ikc_scd_packet isp;
 	struct perf_ctrl_desc *perf_desc;
-	struct ihk_cpu_info *info = ihk_os_get_cpu_info(os);
+	struct ihk_cpu_info *info = NULL;
 	unsigned long cntr_mask = 0;
 	int ret = 0;
 	int i = 0, j = 0;
 	int need_free;
 
+	if (!os || ihk_host_validate_os(os)) {
+		return -EINVAL;
+	}
+
+	usrdata = ihk_host_os_get_usrdata(os);
 	if (!usrdata) {
 		pr_err("%s: error: mcctrl_usrdata not found\n", __func__);
 		return -EINVAL;
@@ -2422,6 +2512,11 @@ long mcctrl_perf_disable(ihk_os_t os)
 	isp.msg = SCD_MSG_PERF_CTRL;
 	isp.arg = virt_to_phys(perf_desc);
 
+	info = ihk_os_get_cpu_info(os);
+	if (!info || info->n_cpus < 1) {
+		kfree(perf_desc);
+		return -EINVAL;
+	}
 	for (j = 0; j < info->n_cpus; j++) {
 		ret = mcctrl_ikc_send_wait(os, j, &isp, 0,
 				wakeup_desc_of_perf_desc(perf_desc),
@@ -2463,6 +2558,10 @@ long mcctrl_getrusage(ihk_os_t ihk_os, struct mcctrl_ioctl_getrusage_desc *__use
 	unsigned long ut;
 	unsigned long st;
 
+	if (!ihk_os || ihk_host_validate_os(ihk_os)) {
+		return -EINVAL;
+	}
+
 	ret = copy_from_user(&desc, _desc, sizeof(struct mcctrl_ioctl_getrusage_desc));
 	if (ret != 0) {
 		printk("%s: copy_from_user failed\n", __FUNCTION__);
@@ -2705,7 +2804,7 @@ static long mcexec_terminate_thread_unsafe(ihk_os_t os, int pid, int tid, long c
 				__FUNCTION__, tid);
 		goto no_ptd;
 	}
-	__return_syscall(usrdata->os, packet, code, tid);
+	__return_syscall(usrdata->os, ppd, packet, code, tid);
 	ihk_ikc_release_packet((struct ihk_ikc_free_packet *)packet);
 
 	/* Drop reference for this function */
@@ -3451,7 +3550,7 @@ int mcctrl_get_request_os_cpu(ihk_os_t os, int *ret_cpu)
 	struct ihk_ikc_channel_desc *ch;
 	int ret = 0;
 
-	if (!os) {
+	if (!os || ihk_host_validate_os(os) || !ret_cpu) {
 		return -EINVAL;
 	}
 
@@ -3493,7 +3592,11 @@ int mcctrl_get_request_os_cpu(ihk_os_t os, int *ret_cpu)
 	*ret_cpu = ch->send.queue->read_cpu;
 	ret = 0;
 
+#ifndef ENABLE_FUGAKU_HACKS
 	pr_info("%s: OS: %lx, CPU: %d\n",
+#else
+	dprintk("%s: OS: %lx, CPU: %d\n",
+#endif
 		__func__, (unsigned long)os, *ret_cpu);
 
 out_put_ppd:
@@ -3543,7 +3646,8 @@ int __mcctrl_os_read_write_cpu_register(ihk_os_t os, int cpu,
 	isp.op = op;
 	isp.pdesc = virt_to_phys(ldesc);
 
-	ret = mcctrl_ikc_send_wait(os, cpu, &isp, 0, NULL, &do_free, 1, ldesc);
+	/* 1 sec timeout for the case where McKernel can't respond */
+	ret = mcctrl_ikc_send_wait(os, cpu, &isp, 1000, NULL, &do_free, 1, ldesc);
 	if (ret != 0) {
 		printk("%s: ERROR sending IKC msg: %d\n", __FUNCTION__, ret);
 		goto out;
@@ -3557,7 +3661,11 @@ int __mcctrl_os_read_write_cpu_register(ihk_os_t os, int cpu,
 	/* Notify caller (for future async implementation) */
 	atomic_set(&desc->sync, 1);
 
+#ifndef ENABLE_FUGAKU_HACKS
 	dprintk("%s: MCCTRL_OS_CPU_%s_REGISTER: CPU: %d, addr_ext: 0x%lx, val: 0x%lx\n",
+#else
+	printk("%s: MCCTRL_OS_CPU_%s_REGISTER: CPU: %d, addr_ext: 0x%lx, val: 0x%lx\n",
+#endif
 		__FUNCTION__,
 		(op == MCCTRL_OS_CPU_READ_REGISTER ? "READ" : "WRITE"), cpu,
 		desc->addr_ext, desc->val);

executer/kernel/mcctrl/driver.c

@@ -50,6 +50,9 @@ extern void procfs_exit(int);
 extern void uti_attr_finalize(void);
 extern void binfmt_mcexec_init(void);
 extern void binfmt_mcexec_exit(void);
+#ifdef ENABLE_TOFU
+extern void mcctrl_file_to_pidfd_hash_init(void);
+#endif
 
 extern int mcctrl_os_read_cpu_register(ihk_os_t os, int cpu,
 		struct ihk_os_cpu_register *desc);
@@ -57,6 +60,11 @@ extern int mcctrl_os_write_cpu_register(ihk_os_t os, int cpu,
 		struct ihk_os_cpu_register *desc);
 extern int mcctrl_get_request_os_cpu(ihk_os_t os, int *cpu);
 
+#ifdef ENABLE_TOFU
+extern void mcctrl_tofu_hijack_release_handlers(void);
+extern void mcctrl_tofu_restore_release_handlers(void);
+#endif
+
 static long mcctrl_ioctl(ihk_os_t os, unsigned int request, void *priv,
                          unsigned long arg, struct file *file)
 {
@@ -227,7 +235,6 @@ void (*mcctrl_zap_page_range)(struct vm_area_struct *vma,
 
 struct inode_operations *mcctrl_hugetlbfs_inode_operations;
 
-
 static int symbols_init(void)
 {
 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4,17,0)
@@ -319,10 +326,17 @@ static int __init mcctrl_init(void)
 	}
 
 	binfmt_mcexec_init();
+#ifdef ENABLE_TOFU
+	mcctrl_file_to_pidfd_hash_init();
+#endif
 
 	if ((ret = symbols_init()))
 		goto error;
 
+#ifdef ENABLE_TOFU
+	mcctrl_tofu_hijack_release_handlers();
+#endif
+
 	if ((ret = ihk_host_register_os_notifier(&mcctrl_os_notifier)) != 0) {
 		printk("mcctrl: error: registering OS notifier\n");
 		goto error;
@@ -345,6 +359,9 @@ static void __exit mcctrl_exit(void)
 
 	binfmt_mcexec_exit();
 	uti_attr_finalize();
+#ifdef ENABLE_TOFU
+	mcctrl_tofu_restore_release_handlers();
+#endif
 
 	printk("mcctrl: unregistered.\n");
 }

executer/kernel/mcctrl/ikc.c

@@ -142,13 +142,35 @@ int mcctrl_ikc_send_wait(ihk_os_t os, int cpu, struct ikc_scd_packet *pisp,
 	ret = mcctrl_ikc_send(os, cpu, pisp);
 	if (ret < 0) {
 		pr_warn("%s: mcctrl_ikc_send failed: %d\n", __func__, ret);
-		kfree(desc);
+		if (alloc_desc)
+			kfree(desc);
 		return ret;
 	}
 
 	if (timeout) {
-		ret = wait_event_interruptible_timeout(desc->wq,
-			desc->status, timeout);
+		/*
+		 * Negative timeout indicates busy waiting, which can be used
+		 * in situations where wait_event_interruptible_XXX() would
+		 * fail, e.g., in a signal handler, at the time the process
+		 * is being killed, etc.
+		 */
+		if (timeout < 0) {
+			unsigned long timeout_jiffies =
+				jiffies + msecs_to_jiffies(timeout * -1);
+			ret = -ETIME;
+
+			while (time_before(jiffies, timeout_jiffies)) {
+				schedule();
+				if (READ_ONCE(desc->status)) {
+					ret = 0;
+					break;
+				}
+			}
+		}
+		else {
+			ret = wait_event_interruptible_timeout(desc->wq,
+					desc->status, msecs_to_jiffies(timeout));
+		}
 	} else {
 		ret = wait_event_interruptible(desc->wq, desc->status);
 	}
@@ -210,6 +232,8 @@ static int syscall_packet_handler(struct ihk_ikc_channel_desc *c,
 	case SCD_MSG_PROCFS_ANSWER:
 	case SCD_MSG_REMOTE_PAGE_FAULT_ANSWER:
 	case SCD_MSG_CPU_RW_REG_RESP:
+	case SCD_MSG_CLEANUP_PROCESS_RESP:
+	case SCD_MSG_CLEANUP_FD_RESP:
 		mcctrl_wakeup_cb(__os, pisp);
 		break;
 
@@ -280,7 +304,11 @@ int mcctrl_ikc_send(ihk_os_t os, int cpu, struct ikc_scd_packet *pisp)
 {
 	struct mcctrl_usrdata *usrdata;
 
-	if (!os || cpu < 0) {
+	if (!os || ihk_host_validate_os(os) || !pisp) {
+		return -EINVAL;
+	}
+
+	if (cpu < 0) {
 		return -EINVAL;
 	}
 
@@ -508,11 +536,9 @@ int prepare_ikc_channels(ihk_os_t os)
 	usrdata->os = os;
 	ihk_host_os_set_usrdata(os, usrdata);
 
-	ihk_ikc_listen_port(os, &lp_ikc2linux);
-	ihk_ikc_listen_port(os, &lp_ikc2mckernel);
-
 	init_waitqueue_head(&usrdata->wq_procfs);
 	mutex_init(&usrdata->reserve_lock);
+	mutex_init(&usrdata->part_exec_lock);
 
 	for (i = 0; i < MCCTRL_PER_PROC_DATA_HASH_SIZE; ++i) {
 		INIT_LIST_HEAD(&usrdata->per_proc_data_hash[i]);
@@ -521,13 +547,21 @@ int prepare_ikc_channels(ihk_os_t os)
 
 	INIT_LIST_HEAD(&usrdata->cpu_topology_list);
 	INIT_LIST_HEAD(&usrdata->node_topology_list);
+	INIT_LIST_HEAD(&usrdata->part_exec_list);
 
-	mutex_init(&usrdata->part_exec.lock);
-	INIT_LIST_HEAD(&usrdata->part_exec.pli_list);
-	usrdata->part_exec.nr_processes = -1;
 	INIT_LIST_HEAD(&usrdata->wakeup_descs_list);
 	spin_lock_init(&usrdata->wakeup_descs_lock);
 
+	/* ihk_ikc_listen_port should be performed after
+	 * usrdata->cpu_topology_list is initialized because the
+	 * function enables syscall_packet_handler which accesses
+	 * the list (the call path is sysfsm_packet_handler -->
+	 * sysfsm_work_main --> sysfsm_setup --> setup_sysfs_files
+	 * --> setup_cpus_sysfs_files).
+	 */
+	ihk_ikc_listen_port(os, &lp_ikc2linux);
+	ihk_ikc_listen_port(os, &lp_ikc2mckernel);
+
 	return 0;
 
 error:
@@ -580,6 +614,18 @@ void destroy_ikc_channels(ihk_os_t os)
 
 	kfree(usrdata->channels);
 	kfree(usrdata->ikc2linux);
+
+	mutex_lock(&usrdata->part_exec_lock);
+	while (!list_empty(&usrdata->part_exec_list)) {
+		struct mcctrl_part_exec *pe;
+
+		pe = list_first_entry(&usrdata->part_exec_list,
+				struct mcctrl_part_exec, chain);
+		list_del(&pe->chain);
+		kfree(pe);
+	}
+	mutex_unlock(&usrdata->part_exec_lock);
+
 	kfree(usrdata);
 }
 

executer/kernel/mcctrl/mcctrl.h

@@ -58,7 +58,8 @@
 #define SCD_MSG_SEND_SIGNAL             0x7
 #define SCD_MSG_SEND_SIGNAL_ACK         0x8
 #define SCD_MSG_CLEANUP_PROCESS         0x9
-#define SCD_MSG_GET_VDSO_INFO           0xa
+#define SCD_MSG_CLEANUP_PROCESS_RESP    0xa
+#define SCD_MSG_GET_VDSO_INFO           0xb
 
 //#define SCD_MSG_GET_CPU_MAPPING         0xc
 //#define SCD_MSG_REPLY_GET_CPU_MAPPING   0xd
@@ -104,6 +105,8 @@
 
 #define SCD_MSG_CPU_RW_REG              0x52
 #define SCD_MSG_CPU_RW_REG_RESP         0x53
+#define SCD_MSG_CLEANUP_FD              0x54
+#define SCD_MSG_CLEANUP_FD_RESP         0x55
 
 #define SCD_MSG_FUTEX_WAKE              0x60
 
@@ -260,6 +263,7 @@ struct mcctrl_per_proc_data {
 
 	struct list_head devobj_pager_list;
 	struct semaphore devobj_pager_lock;
+	int enable_tofu;
 };
 
 struct sysfsm_req {
@@ -324,13 +328,20 @@ struct process_list_item {
 	wait_queue_head_t pli_wq;
 };
 
+#define PE_LIST_MAXLEN 5
+
 struct mcctrl_part_exec {
 	struct mutex lock;	
 	int nr_processes;
+	/* number of processes to let in / out the synchronization point */
 	int nr_processes_left;
+	/* number of processes which have joined the partition */
+	int nr_processes_joined;
 	int process_rank;
+	pid_t node_proxy_pid;
 	cpumask_t cpus_used;
 	struct list_head pli_list;
+	struct list_head chain;
 };
 
 #define CPU_LONGS (((NR_CPUS) + (BITS_PER_LONG) - 1) / (BITS_PER_LONG))
@@ -353,6 +364,7 @@ struct mcctrl_usrdata {
 	int	job_pos;
 	int	mcctrl_dma_abort;
 	struct mutex reserve_lock;
+	struct mutex part_exec_lock;
 	unsigned long	last_thread_exec;
 	wait_queue_head_t wq_procfs;
 	struct list_head per_proc_data_hash[MCCTRL_PER_PROC_DATA_HASH_SIZE];
@@ -368,7 +380,7 @@ struct mcctrl_usrdata {
 	nodemask_t numa_online;
 	struct list_head cpu_topology_list;
 	struct list_head node_topology_list;
-	struct mcctrl_part_exec part_exec;
+	struct list_head part_exec_list;
 	int perf_event_num;
 };
 
@@ -453,7 +465,8 @@ struct mcctrl_per_thread_data *mcctrl_get_per_thread_data(struct mcctrl_per_proc
 							  struct task_struct *task);
 int mcctrl_clear_pte_range(uintptr_t start, uintptr_t len);
 
-void __return_syscall(ihk_os_t os, struct ikc_scd_packet *packet, 
+void __return_syscall(ihk_os_t os, struct mcctrl_per_proc_data *ppd,
+		struct ikc_scd_packet *packet,
 		long ret, int stid);
 int clear_pte_range(uintptr_t start, uintptr_t len);
 
@@ -548,4 +561,34 @@ struct uti_futex_resp {
 	int done;
 	wait_queue_head_t wq;
 };
+
+#ifdef ENABLE_TOFU
+/*
+ * Hash table to keep track of files and related processes
+ * and file descriptors.
+ * NOTE: Used for Tofu driver release handlers.
+ */
+#define MCCTRL_FILE_2_PIDFD_HASH_SHIFT 4
+#define MCCTRL_FILE_2_PIDFD_HASH_SIZE (1 << MCCTRL_FILE_2_PIDFD_HASH_SHIFT)
+#define MCCTRL_FILE_2_PIDFD_HASH_MASK (MCCTRL_FILE_2_PIDFD_HASH_SIZE - 1)
+
+struct mcctrl_file_to_pidfd {
+	struct file *filp;
+	ihk_os_t os;
+	struct task_struct *group_leader;
+	int pid;
+	int fd;
+	struct list_head hash;
+	char tofu_dev_path[128];
+	void *pde_data;
+};
+
+int mcctrl_file_to_pidfd_hash_insert(struct file *filp,
+	ihk_os_t os, int pid, struct task_struct *group_leader, int fd,
+	char *path, void *pde_data);
+struct mcctrl_file_to_pidfd *mcctrl_file_to_pidfd_hash_lookup(
+	struct file *filp, struct task_struct *group_leader);
+int mcctrl_file_to_pidfd_hash_remove(struct file *filp,
+	ihk_os_t os, struct task_struct *group_leader, int fd);
+#endif
 #endif

executer/kernel/mcctrl/procfs.c

@@ -126,7 +126,7 @@ find_procfs_entry(struct procfs_list_entry *parent, const char *name)
 static void
 delete_procfs_entries(struct procfs_list_entry *top)
 {
-	struct procfs_list_entry *e;
+	struct procfs_list_entry *e = NULL;
 	struct procfs_list_entry *n;
 
 	list_del(&top->list);
@@ -136,8 +136,10 @@ delete_procfs_entries(struct procfs_list_entry *top)
 	}
 
 #if LINUX_VERSION_CODE < KERNEL_VERSION(3,10,0)
-	e->entry->read_proc = NULL;
-	e->entry->data = NULL;
+	if (e) {
+		e->entry->read_proc = NULL;
+		e->entry->data = NULL;
+	}
 #endif
 	remove_proc_entry(top->name, top->parent? top->parent->entry: NULL);
 	if(top->data)

executer/kernel/mcctrl/syscall.c

@@ -45,6 +45,9 @@
 #include <linux/mount.h>
 #include <linux/kdev_t.h>
 #include <linux/hugetlb.h>
+#include <linux/proc_fs.h>
+#include <linux/rbtree.h>
+#include <linux/llist.h>
 #include <asm/uaccess.h>
 #include <asm/delay.h>
 #include <asm/io.h>
@@ -52,6 +55,7 @@
 #include "mcctrl.h"
 #include <linux/version.h>
 #include <archdeps.h>
+#include <asm/pgtable.h>
 
 #define ALIGN_WAIT_BUF(z)   (((z + 63) >> 6) << 6)
 
@@ -655,6 +659,9 @@ static int rus_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 		goto put_and_out;
 	}
 
+	// Force regular page size
+	pgsize = PAGE_SIZE;
+
 	rva = (unsigned long)addr & ~(pgsize - 1);
 	rpa = rpa & ~(pgsize - 1);
 
@@ -666,7 +673,8 @@ static int rus_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 
 		/* LWK may hold large page based mappings that align rva outside
 		 * Linux' VMA, make sure we don't try to map to those pages */
-		if (rva + (pix * PAGE_SIZE) < vma->vm_start) {
+		if (rva + (pix * PAGE_SIZE) < vma->vm_start ||
+			rva + (pix * PAGE_SIZE) > vma->vm_end) {
 			continue;
 		}
 
@@ -677,21 +685,27 @@ static int rus_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 			if (error) {
 				pr_err("%s: error inserting mapping for 0x%#lx "
 					"(req: TID: %d, syscall: %lu) error: %d,"
-					" vm_start: 0x%lx, vm_end: 0x%lx\n",
+					" vm_start: 0x%lx, vm_end: 0x%lx, pgsize: %lu, ind: %lu\n",
 					__func__,
 					(unsigned long)addr, packet.fault_tid,
 					rsysnum, error,
-					vma->vm_start, vma->vm_end);
+					vma->vm_start, vma->vm_end, pgsize, pix);
 			}
 		}
-		else
+		else {
 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 18, 0)
 			error = vmf_insert_pfn(vma, rva+(pix*PAGE_SIZE),
 					       pfn+pix);
+			if (error == VM_FAULT_NOPAGE) {
+				dprintk("%s: vmf_insert_pfn returned %d\n",
+					__func__, error);
+				error = 0;
+			}
 #else
 			error = vm_insert_pfn(vma, rva+(pix*PAGE_SIZE),
 					      pfn+pix);
 #endif
+		}
 		if (error) {
 			pr_err("%s: vm_insert_pfn returned %d\n",
 			       __func__, error);
@@ -1835,20 +1849,165 @@ static long pager_call(ihk_os_t os, struct syscall_request *req)
 	return ret;
 }
 
-void __return_syscall(ihk_os_t os, struct ikc_scd_packet *packet,
+#ifdef ENABLE_TOFU
+struct list_head mcctrl_file_to_pidfd_hash[MCCTRL_FILE_2_PIDFD_HASH_SIZE];
+spinlock_t mcctrl_file_to_pidfd_hash_lock;
+
+void mcctrl_file_to_pidfd_hash_init(void)
+{
+	int hash;
+	spin_lock_init(&mcctrl_file_to_pidfd_hash_lock);
+
+	for (hash = 0; hash < MCCTRL_FILE_2_PIDFD_HASH_SIZE; ++hash) {
+		INIT_LIST_HEAD(&mcctrl_file_to_pidfd_hash[hash]);
+	}
+}
+
+int mcctrl_file_to_pidfd_hash_insert(struct file *filp,
+	ihk_os_t os, int pid, struct task_struct *group_leader, int fd,
+	char *path, void *pde_data)
+{
+	unsigned long irqflags;
+	struct mcctrl_file_to_pidfd *file2pidfd_iter;
+	struct mcctrl_file_to_pidfd *file2pidfd;
+	int hash = (int)((unsigned long)filp &
+			(unsigned long)MCCTRL_FILE_2_PIDFD_HASH_MASK);
+	int ret = 0;
+
+	file2pidfd = kmalloc(sizeof(*file2pidfd), GFP_ATOMIC);
+	if (!file2pidfd)
+		return -ENOMEM;
+
+	file2pidfd->filp = filp;
+	file2pidfd->os = os;
+	file2pidfd->pid = pid;
+	file2pidfd->group_leader = group_leader;
+	file2pidfd->fd = fd;
+	/* Only copy the name under /proc/tofu/dev/ */
+	strncpy(file2pidfd->tofu_dev_path, path + 15, 128);
+	file2pidfd->pde_data = pde_data;
+
+	spin_lock_irqsave(&mcctrl_file_to_pidfd_hash_lock, irqflags);
+	list_for_each_entry(file2pidfd_iter,
+			&mcctrl_file_to_pidfd_hash[hash], hash) {
+		if (file2pidfd_iter->filp == filp) {
+			printk("%s: WARNING: filp: %p, pid: %d, fd: %d exists\n",
+					__func__, filp, pid, fd);
+			ret = -EBUSY;
+			goto free_out;
+		}
+	}
+
+	list_add_tail(&file2pidfd->hash,
+			&mcctrl_file_to_pidfd_hash[hash]);
+	dprintk("%s: filp: %p, pid: %d, fd: %d added\n",
+			__func__, filp, pid, fd);
+
+	spin_unlock_irqrestore(&mcctrl_file_to_pidfd_hash_lock, irqflags);
+	return ret;
+
+free_out:
+	kfree(file2pidfd);
+	spin_unlock_irqrestore(&mcctrl_file_to_pidfd_hash_lock, irqflags);
+	return ret;
+}
+
+/*
+ * XXX: lookup relies on group_leader to identify the process
+ * because PIDs might be different across name spaces (e.g.,
+ * when using Docker)
+ */
+struct mcctrl_file_to_pidfd *mcctrl_file_to_pidfd_hash_lookup(
+	struct file *filp, struct task_struct *group_leader)
+{
+	unsigned long irqflags;
+	struct mcctrl_file_to_pidfd *file2pidfd_iter;
+	struct mcctrl_file_to_pidfd *file2pidfd = NULL;
+	int hash = (int)((unsigned long)filp &
+			(unsigned long)MCCTRL_FILE_2_PIDFD_HASH_MASK);
+
+	spin_lock_irqsave(&mcctrl_file_to_pidfd_hash_lock, irqflags);
+	list_for_each_entry(file2pidfd_iter,
+			&mcctrl_file_to_pidfd_hash[hash], hash) {
+		if (file2pidfd_iter->filp == filp && 
+				file2pidfd_iter->group_leader == group_leader) {
+			file2pidfd = file2pidfd_iter;
+			dprintk("%s: filp: %p, pid: %d, fd: %d found\n",
+					__func__, filp, file2pidfd->pid, file2pidfd->fd);
+			break;
+		}
+	}
+
+	spin_unlock_irqrestore(&mcctrl_file_to_pidfd_hash_lock, irqflags);
+
+	return file2pidfd;
+}
+
+int mcctrl_file_to_pidfd_hash_remove(struct file *filp,
+	ihk_os_t os, struct task_struct *group_leader, int fd)
+{
+	unsigned long irqflags;
+	struct mcctrl_file_to_pidfd *file2pidfd_iter;
+	int hash = (int)((unsigned long)filp &
+			(unsigned long)MCCTRL_FILE_2_PIDFD_HASH_MASK);
+	int ret = 0;
+
+	spin_lock_irqsave(&mcctrl_file_to_pidfd_hash_lock, irqflags);
+	list_for_each_entry(file2pidfd_iter,
+			&mcctrl_file_to_pidfd_hash[hash], hash) {
+		if (file2pidfd_iter->filp != filp)
+			continue;
+
+		if (file2pidfd_iter->os != os)
+			continue;
+
+		if (file2pidfd_iter->group_leader != group_leader)
+			continue;
+
+		if (file2pidfd_iter->fd != fd)
+			continue;
+
+		list_del(&file2pidfd_iter->hash);
+		dprintk("%s: filp: %p, pid: %d, fd: %d removed\n",
+				__func__, filp, file2pidfd_iter->pid, fd);
+		kfree(file2pidfd_iter);
+		goto unlock_out;
+	}
+
+	dprintk("%s: filp: %p, pid: %d, fd: %d couldn't be found\n",
+			__func__, filp, pid, fd);
+	ret = -ENOENT;
+
+unlock_out:
+	spin_unlock_irqrestore(&mcctrl_file_to_pidfd_hash_lock, irqflags);
+	return ret;
+}
+#endif
+
+void __return_syscall(ihk_os_t os, struct mcctrl_per_proc_data *ppd,
+		struct ikc_scd_packet *packet,
 		long ret, int stid)
 {
 	unsigned long phys;
 	struct syscall_response *res;
 
+	if (!os || ihk_host_validate_os(os) || !packet) {
+		return;
+	}
+
 	phys = ihk_device_map_memory(ihk_os_to_dev(os),
 			packet->resp_pa, sizeof(*res));
+	if (!phys) {
+		return;
+	}
+
 	res = ihk_device_map_virtual(ihk_os_to_dev(os),
 			phys, sizeof(*res), NULL, 0);
 
 	if (!res) {
 		printk("%s: ERROR: invalid response structure address\n",
 			__FUNCTION__);
+		ihk_device_unmap_memory(ihk_os_to_dev(os), phys, sizeof(*res));
 		return;
 	}
 
@@ -1856,6 +2015,109 @@ void __return_syscall(ihk_os_t os, struct ikc_scd_packet *packet,
 	res->ret = ret;
 	res->stid = stid;
 
+#ifdef ENABLE_TOFU
+	/* Tofu enabled process? */
+	if (ppd && ppd->enable_tofu) {
+		char *pathbuf, *fullpath;
+
+		/* Record PDE_DATA after open() calls for Tofu driver */
+		if (packet->req.number == __NR_openat && ret > 1) {
+			struct fd f;
+			int fd;
+
+			fd = ret;
+			f = fdget(fd);
+
+			if (!f.file) {
+				goto out_notify;
+			}
+
+			pathbuf = (char *)__get_free_page(GFP_ATOMIC);
+			if (!pathbuf) {
+				goto out_fdput_open;
+			}
+
+			fullpath = d_path(&f.file->f_path, pathbuf, PAGE_SIZE);
+			if (IS_ERR(fullpath)) {
+				goto out_free_open;
+			}
+
+			if (!strncmp("/proc/tofu/dev/", fullpath, 15)) {
+				res->pde_data = PDE_DATA(file_inode(f.file));
+				dprintk("%s: fd: %d, path: %s, PDE_DATA: 0x%lx\n",
+						__func__,
+						fd,
+						fullpath,
+						(unsigned long)res->pde_data);
+				dprintk("%s: pgd_index: %ld, pmd_index: %ld, pte_index: %ld\n",
+						__func__,
+						pgd_index((unsigned long)res->pde_data),
+						pmd_index((unsigned long)res->pde_data),
+						pte_index((unsigned long)res->pde_data));
+				dprintk("MAX_USER_VA_BITS: %d, PGDIR_SHIFT: %d\n",
+						MAX_USER_VA_BITS, PGDIR_SHIFT);
+				mcctrl_file_to_pidfd_hash_insert(f.file, os,
+						task_tgid_vnr(current),
+						current->group_leader, fd,
+						fullpath, res->pde_data);
+			}
+
+out_free_open:
+			free_page((unsigned long)pathbuf);
+out_fdput_open:
+			fdput(f);
+		}
+
+		/* Ioctl on Tofu CQ? */
+		else if (packet->req.number == __NR_ioctl &&
+				packet->req.args[0] > 0 && ret == 0) {
+			struct fd f;
+			int fd;
+			int tni, cq;
+			long __ret;
+
+			fd = packet->req.args[0];
+			f = fdget(fd);
+
+			if (!f.file) {
+				goto out_notify;
+			}
+
+			pathbuf = (char *)__get_free_page(GFP_ATOMIC);
+			if (!pathbuf) {
+				goto out_fdput_ioctl;
+			}
+
+			fullpath = d_path(&f.file->f_path, pathbuf, PAGE_SIZE);
+			if (IS_ERR(fullpath)) {
+				goto out_free_ioctl;
+			}
+
+			/* Looking for /proc/tofu/dev/tniXcqY pattern */
+			__ret = sscanf(fullpath, "/proc/tofu/dev/tni%dcq%d", &tni, &cq);
+			if (__ret == 2) {
+				extern long __mcctrl_tof_utofu_unlocked_ioctl_cq(void *pde_data,
+						unsigned int cmd, unsigned long arg);
+
+				dprintk("%s: ioctl(): fd: %d, path: %s\n",
+						__func__,
+						fd,
+						fullpath);
+
+				__ret = __mcctrl_tof_utofu_unlocked_ioctl_cq(
+						PDE_DATA(file_inode(f.file)),
+						packet->req.args[1], packet->req.args[2]);
+			}
+
+out_free_ioctl:
+			free_page((unsigned long)pathbuf);
+out_fdput_ioctl:
+			fdput(f);
+		}
+	}
+
+out_notify:
+#endif
 	if (__notify_syscall_requester(os, packet, res) < 0) {
 		printk("%s: WARNING: failed to notify PID %d\n",
 			__FUNCTION__, packet->pid);
@@ -2158,11 +2420,98 @@ int __do_in_kernel_irq_syscall(ihk_os_t os, struct ikc_scd_packet *packet)
 	if (ret == -ENOSYS)
 		return -ENOSYS;
 
-	__return_syscall(os, packet, ret, 0);
+	__return_syscall(os, NULL, packet, ret, 0);
 
 	return 0;
 }
 
+/*
+ * Memory clearing helpers.
+ */
+struct node_distance;
+
+#define IHK_RBTREE_ALLOCATOR
+
+#ifdef IHK_RBTREE_ALLOCATOR
+struct free_chunk {
+	unsigned long addr, size;
+	struct rb_node node;
+	struct llist_node list;
+};
+#endif
+
+typedef struct mcs_lock_node {
+#ifndef SPIN_LOCK_IN_MCS
+	unsigned long locked;
+	struct mcs_lock_node *next;
+#endif
+	unsigned long irqsave;
+#ifdef SPIN_LOCK_IN_MCS
+	ihk_spinlock_t spinlock;
+#endif
+#ifndef ENABLE_UBSAN
+} __aligned(64) mcs_lock_node_t;
+#else
+} mcs_lock_node_t;
+#endif
+
+struct ihk_mc_numa_node {
+	int id;
+	int linux_numa_id;
+	int type;
+	struct list_head allocators;
+	struct node_distance *nodes_by_distance;
+#ifdef IHK_RBTREE_ALLOCATOR
+	atomic_t zeroing_workers;
+	atomic_t nr_to_zero_pages;
+	struct llist_head zeroed_list;
+	struct llist_head to_zero_list;
+	struct rb_root free_chunks;
+	mcs_lock_node_t lock;
+
+	unsigned long nr_pages;
+	/*
+	 * nr_free_pages: all freed pages, zeroed if zero_at_free
+	 */
+	unsigned long nr_free_pages;
+	unsigned long min_addr;
+	unsigned long max_addr;
+#endif
+};
+
+void mcctrl_zero_mckernel_pages(unsigned long arg)
+{
+	struct llist_node *llnode;
+	struct ihk_mc_numa_node *node =
+		(struct ihk_mc_numa_node *)arg;
+
+	/* Iterate free chunks */
+	while ((llnode = llist_del_first(&node->to_zero_list))) {
+		unsigned long addr;
+		unsigned long size;
+		struct free_chunk *chunk =
+			container_of(llnode, struct free_chunk, list);
+
+		addr = chunk->addr;
+		size = chunk->size;
+
+		memset(phys_to_virt(addr) + sizeof(*chunk), 0,
+				chunk->size - sizeof(*chunk));
+		llist_add(&chunk->list, &node->zeroed_list);
+
+		dprintk("%s: zeroed %lu pages @ McKernel NUMA %d (chunk: 0x%lx:%lu)\n",
+				__func__,
+				size >> PAGE_SHIFT,
+				node->id,
+				addr, size);
+		barrier();
+		atomic_sub((int)(size >> PAGE_SHIFT), &node->nr_to_zero_pages);
+	}
+
+	atomic_dec(&node->zeroing_workers);
+}
+
+
 int __do_in_kernel_syscall(ihk_os_t os, struct ikc_scd_packet *packet)
 {
 	struct syscall_request *sc = &packet->req;
@@ -2171,6 +2520,28 @@ int __do_in_kernel_syscall(ihk_os_t os, struct ikc_scd_packet *packet)
 
 	dprintk("%s: system call: %lx\n", __FUNCTION__, sc->args[0]);
 	switch (sc->number) {
+#ifdef ENABLE_TOFU
+	case __NR_close: {
+		struct fd f;
+		int fd;
+
+		fd = (int)sc->args[0];
+		if (fd > 2) {
+			f = fdget(fd);
+
+			if (f.file) {
+				mcctrl_file_to_pidfd_hash_remove(f.file, os,
+						current->group_leader, fd);
+				fdput(f);
+			}
+		}
+
+		error = -ENOSYS;
+		goto out;
+
+		break;
+	}
+#endif
 	case __NR_mmap:
 		ret = pager_call(os, sc);
 		break;
@@ -2183,6 +2554,14 @@ int __do_in_kernel_syscall(ihk_os_t os, struct ikc_scd_packet *packet)
 		ret = remap_user_space(sc->args[0], sc->args[1], sc->args[2]);
 		break;
 
+	case __NR_move_pages:
+		/*
+		 * move pages is used for zeroing McKernel side memory,
+		 * this call is NOT offloaded by applications.
+		 */
+		mcctrl_zero_mckernel_pages(sc->args[0]);
+		goto out_no_syscall_return;
+
 	case __NR_exit_group: {
 	
 		/* Make sure the user space handler will be called as well */
@@ -2266,7 +2645,9 @@ sched_setparam_out:
 		break;
 	}
 
-	__return_syscall(os, packet, ret, 0);
+	__return_syscall(os, NULL, packet, ret, 0);
+
+out_no_syscall_return:
 	ihk_ikc_release_packet((struct ihk_ikc_free_packet *)packet);
 
 	error = 0;

executer/user/eclair.c

@@ -1179,7 +1179,7 @@ static int start_gdb(void) {
 
 		sprintf(buf, "target remote :%d", ntohs(sin.sin_port));
 		execlp("gdb", "eclair", "-q", "-ex", "set prompt (eclair) ",
-				"-ex", buf, opt.kernel_path, NULL);
+				"-ex", buf, opt.kernel_path, "-ex", "set pagination off", NULL);
 		perror("execlp");
 		return 3;
 	}

executer/user/mcexec.c

@@ -68,13 +68,13 @@
 #include <sys/user.h>
 #endif /* !__aarch64__ */
 #include <sys/prctl.h>
+#include "../../config.h"
 #include "../include/uprotocol.h"
 #include <ihk/ihk_host_user.h>
 #include "../include/uti.h"
 #include <getopt.h>
 #include "archdep.h"
 #include "arch_args.h"
-#include "../../config.h"
 #include <numa.h>
 #include <numaif.h>
 #include <spawn.h>
@@ -84,7 +84,11 @@
 #include "../include/pmi.h"
 #include "../include/qlmpi.h"
 #include <sys/xattr.h>
+#include "../include/defs.h"
 #include "../../lib/include/list.h"
+#include "../../lib/include/bitops-set_bit.h"
+#include "../../lib/include/bitops-clear_bit.h"
+#include "../../lib/include/bitops-test_bit.h"
 
 //#define DEBUG
 #define ADD_ENVS_OPTION
@@ -187,6 +191,8 @@ static int mpol_no_stack = 0;
 static int mpol_no_bss = 0;
 static int mpol_shm_premap = 0;
 static int no_bind_ikc_map = 0;
+static int straight_map = 0;
+static unsigned long straight_map_threshold = (1024*1024);
 static unsigned long mpol_threshold = 0;
 static unsigned long heap_extension = -1;
 static int profile = 0;
@@ -198,6 +204,9 @@ static char *mpol_bind_nodes = NULL;
 static int uti_thread_rank = 0;
 static int uti_use_last_cpu = 0;
 static int enable_uti = 0;
+#ifdef ENABLE_TOFU
+static int enable_tofu = 0;
+#endif
 
 /* Partitioned execution (e.g., for MPI) */
 static int nr_processes = 0;
@@ -1053,6 +1062,64 @@ static inline cpu_set_t *numa_node_set(int n)
 	return (cpu_set_t *)(numa_nodes + n * cpu_set_size);
 }
 
+static inline void _numa_local(__cpu_set_unit *localset,
+			       unsigned long *nodemask, int nonlocal)
+{
+	int i;
+
+	memset(nodemask, 0, PLD_PROCESS_NUMA_MASK_BITS / 8);
+
+	for (i = 0; i < nnodes; i++) {
+		cpu_set_t *nodeset = numa_node_set(i);
+		int j;
+
+		if (nonlocal) {
+			set_bit(i, nodemask);
+		}
+
+		for (j = 0; j < ncpu; j++) {
+			if (test_bit(j, localset)) {
+				__dprintf("%d belongs to local set\n", j);
+			}
+
+			if (CPU_ISSET_S(j, cpu_set_size, nodeset)) {
+				__dprintf("%d belongs to node %d\n", j, i);
+			}
+
+			if (test_bit(j, localset) &&
+			    CPU_ISSET_S(j, cpu_set_size, nodeset)) {
+				if (nonlocal) {
+					clear_bit(i, nodemask);
+				} else {
+					set_bit(i, nodemask);
+				}
+			}
+		}
+	}
+}
+
+static inline void numa_local(__cpu_set_unit *localset, unsigned long *nodemask)
+{
+	_numa_local(localset, nodemask, 0);
+}
+
+static inline void numa_nonlocal(__cpu_set_unit *localset,
+				 unsigned long *nodemask)
+{
+	_numa_local(localset, nodemask, 1);
+}
+
+static inline void numa_all(unsigned long *nodemask)
+{
+	int i;
+
+	memset(nodemask, 0, PLD_PROCESS_NUMA_MASK_BITS / 8);
+
+	for (i = 0; i < nnodes; i++) {
+		set_bit(i, nodemask);
+	}
+}
+
 pid_t master_tid;
 
 pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
@@ -1674,6 +1741,18 @@ static struct option mcexec_options[] = {
 		.flag =		NULL,
 		.val =		'M',
 	},
+	{
+		.name =		"enable-straight-map",
+		.has_arg =	no_argument,
+		.flag =		&straight_map,
+		.val =		1,
+	},
+	{
+		.name =		"straight-map-threshold",
+		.has_arg =	required_argument,
+		.flag =		NULL,
+		.val =		'S',
+	},
 	{
 		.name =		"disable-sched-yield",
 		.has_arg =	no_argument,
@@ -1710,6 +1789,14 @@ static struct option mcexec_options[] = {
 		.flag =		&enable_uti,
 		.val =		1,
 	},
+#ifdef ENABLE_TOFU
+	{
+		.name =		"enable-tofu",
+		.has_arg =	no_argument,
+		.flag =		&enable_tofu,
+		.val =		1,
+	},
+#endif
 	{
 		.name =		"debug-mcexec",
 		.has_arg =	no_argument,
@@ -1870,14 +1957,14 @@ opendev()
 			fprintf(stderr, "%s: warning: LD_PRELOAD line is too long\n", __FUNCTION__); \
 			return; \
 		} \
-		strncat(envbuf, elembuf, remainder); \
+		strncat(envbuf, elembuf, remainder - 1); \
 		remainder = PATH_MAX - (strlen(envbuf) + 1); \
 		nelem++; \
 	} while (0)
 
 static ssize_t find_libdir(char *libdir, size_t len)
 {
-	FILE *filep;
+	FILE *filep = NULL;
 	ssize_t rc;
 	size_t linelen = 0;
 	char *line = NULL;
@@ -1933,7 +2020,9 @@ static ssize_t find_libdir(char *libdir, size_t len)
 	}
 
 out:
-	pclose(filep);
+	if (filep) {
+		pclose(filep);
+	}
 	free(line);
 	return rc;
 }
@@ -2095,10 +2184,10 @@ int main(int argc, char **argv)
 
 	/* Parse options ("+" denotes stop at the first non-option) */
 #ifdef ADD_ENVS_OPTION
-	while ((opt = getopt_long(argc, argv, "+c:n:t:M:h:e:s:m:u:",
+	while ((opt = getopt_long(argc, argv, "+c:n:t:M:h:e:s:m:u:S:",
 				  mcexec_options, NULL)) != -1) {
 #else /* ADD_ENVS_OPTION */
-	while ((opt = getopt_long(argc, argv, "+c:n:t:M:h:s:m:u:",
+	while ((opt = getopt_long(argc, argv, "+c:n:t:M:h:s:m:u:S:",
 				  mcexec_options, NULL)) != -1) {
 #endif /* ADD_ENVS_OPTION */
 		switch (opt) {
@@ -2140,6 +2229,10 @@ int main(int argc, char **argv)
 				heap_extension = atobytes(optarg);
 				break;
 
+			case 'S':
+				straight_map_threshold = atobytes(optarg);
+				break;
+
 #ifdef ADD_ENVS_OPTION
 			case 'e':
 				add_env_list(&extra_env, optarg);
@@ -2554,6 +2647,7 @@ int main(int argc, char **argv)
 		cpu_set_arg.cpu_set = (void *)&desc->cpu_set;
 		cpu_set_arg.cpu_set_size = sizeof(desc->cpu_set);
 		cpu_set_arg.nr_processes = nr_processes;
+		cpu_set_arg.ppid = getppid();
 		cpu_set_arg.target_core = &target_core;
 		cpu_set_arg.process_rank = &process_rank;
 		cpu_set_arg.mcexec_linux_numa = &mcexec_linux_numa;
@@ -2659,6 +2753,7 @@ int main(int argc, char **argv)
 	desc->heap_extension = heap_extension;
 
 	desc->mpol_bind_mask = 0;
+	desc->mpol_mode = PLD_MPOL_MAX; /* not specified */
 	if (mpol_bind_nodes) {
 		struct bitmask *bind_mask;
 		bind_mask = numa_parse_nodestring_all(mpol_bind_nodes);
@@ -2672,11 +2767,65 @@ int main(int argc, char **argv)
 			}
 		}
 	}
+	/* Fujitsu TCS specific: mempolicy */
+	else if (getenv("OMPI_MCA_plm_ple_memory_allocation_policy")) {
+		char *mpol =
+			getenv("OMPI_MCA_plm_ple_memory_allocation_policy");
+
+		__dprintf("OMPI_MCA_plm_ple_memory_allocation_policy: %s\n",
+			  mpol);
+
+		if (!strncmp(mpol, "localalloc", 10)) {
+			/* MPOL_DEFAULT has the same effect as MPOL_LOCAL */
+			desc->mpol_mode = MPOL_DEFAULT;
+		}
+		else if (!strncmp(mpol, "interleave_local", 16)) {
+			desc->mpol_mode = MPOL_INTERLEAVE;
+			numa_local(desc->cpu_set, desc->mpol_nodemask);
+		}
+		else if (!strncmp(mpol, "interleave_nonlocal", 19)) {
+			desc->mpol_mode = MPOL_INTERLEAVE;
+			numa_nonlocal(desc->cpu_set, desc->mpol_nodemask);
+		}
+		else if (!strncmp(mpol, "interleave_all", 14)) {
+			desc->mpol_mode = MPOL_INTERLEAVE;
+			numa_all(desc->mpol_nodemask);
+		}
+		else if (!strncmp(mpol, "bind_local", 10)) {
+			desc->mpol_mode = MPOL_BIND;
+			numa_local(desc->cpu_set, desc->mpol_nodemask);
+		}
+		else if (!strncmp(mpol, "bind_nonlocal", 13)) {
+			desc->mpol_mode = MPOL_BIND;
+			numa_nonlocal(desc->cpu_set, desc->mpol_nodemask);
+		}
+		else if (!strncmp(mpol, "bind_all", 8)) {
+			desc->mpol_mode = MPOL_BIND;
+			numa_all(desc->mpol_nodemask);
+		}
+		else if (!strncmp(mpol, "prefer_local", 12)) {
+			desc->mpol_mode = MPOL_PREFERRED;
+			numa_local(desc->cpu_set, desc->mpol_nodemask);
+		}
+		else if (!strncmp(mpol, "prefer_nonlocal", 15)) {
+			desc->mpol_mode = MPOL_PREFERRED;
+			numa_nonlocal(desc->cpu_set, desc->mpol_nodemask);
+		}
+
+		__dprintf("mpol_mode: %d, mpol_nodemask: %ld\n",
+			  desc->mpol_mode, desc->mpol_nodemask[0]);
+	}
 
 	desc->uti_thread_rank = uti_thread_rank;
 	desc->uti_use_last_cpu = uti_use_last_cpu;
 	desc->thp_disable = get_thp_disable();
 
+	desc->straight_map = straight_map;
+	desc->straight_map_threshold = straight_map_threshold;
+#ifdef ENABLE_TOFU
+	desc->enable_tofu = enable_tofu;
+#endif
+
 	/* user_start and user_end are set by this call */
 	if (ioctl(fd, MCEXEC_UP_PREPARE_IMAGE, (unsigned long)desc) != 0) {
 		perror("prepare");
@@ -3974,6 +4123,7 @@ int main_loop(struct thread_data_s *my_thread)
 #endif
 
 		case __NR_gettid:{
+			int rc = 0;
 			/*
 			 * Number of TIDs and the remote physical address where TIDs are
 			 * expected are passed in arg 4 and 5, respectively.
@@ -3985,6 +4135,7 @@ int main_loop(struct thread_data_s *my_thread)
 				int *tids = malloc(sizeof(int) * w.sr.args[4]);
 				if (!tids) {
 					fprintf(stderr, "__NR_gettid(): error allocating TIDs\n");
+					rc = -ENOMEM;
 					goto gettid_out;
 				}
 
@@ -4005,13 +4156,14 @@ int main_loop(struct thread_data_s *my_thread)
 				trans.direction = MCEXEC_UP_TRANSFER_TO_REMOTE;
 
 				if (ioctl(fd, MCEXEC_UP_TRANSFER, &trans) != 0) {
+					rc = -EFAULT;
 					fprintf(stderr, "__NR_gettid(): error transfering TIDs\n");
 				}
 
 				free(tids);
 			}
 gettid_out:
-			do_syscall_return(fd, cpu, 0, 0, 0, 0, 0);
+			do_syscall_return(fd, cpu, rc, 0, 0, 0, 0);
 			break;
 		}
 

kernel/CMakeLists.txt

@@ -51,6 +51,12 @@ set(MCKERNEL_SRCS
 	${IHK_FULL_SOURCE_DIR}/cokernel/smp/${ARCH}/setup.c
 )
 
+if (ENABLE_TOFU)
+	list(APPEND MCKERNEL_SRCS
+		tofu/tof_utofu_main.c
+	)
+endif()
+
 if (ENABLE_UBSAN)
 	add_compile_options(-fsanitize=undefined)
 	list(APPEND MCKERNEL_SRCS ubsan.c)

kernel/cls.c

@@ -58,16 +58,43 @@ struct cpu_local_var *get_cpu_local_var(int id)
 	return clv + id;
 }
 
+#ifdef ENABLE_FUGAKU_HACKS
+void __show_context_stack(struct thread *thread,
+        unsigned long pc, uintptr_t sp, int kprintf_locked);
+#endif
 void preempt_enable(void)
 {
+#ifndef ENABLE_FUGAKU_HACKS
 	if (cpu_local_var_initialized)
 		--cpu_local_var(no_preempt);
+#else
+	if (cpu_local_var_initialized) {
+		--cpu_local_var(no_preempt);
+
+		if (cpu_local_var(no_preempt) < 0) {
+			//cpu_disable_interrupt();
+
+	__kprintf("%s: %d\n", __func__, cpu_local_var(no_preempt));
+    __kprintf("TID: %d, call stack from builtin frame (most recent first):\n",
+        cpu_local_var(current)->tid);
+	__show_context_stack(cpu_local_var(current), (uintptr_t)&preempt_enable,
+			(unsigned long)__builtin_frame_address(0), 1);
+
+			//arch_cpu_stop();
+			//cpu_halt();
+#ifdef ENABLE_FUGAKU_HACKS
+		panic("panic: negative preemption??");
+#endif
+		}
+	}
+#endif
 }
 
 void preempt_disable(void)
 {
-	if (cpu_local_var_initialized)
+	if (cpu_local_var_initialized) {
 		++cpu_local_var(no_preempt);
+	}
 }
 
 int add_backlog(int (*func)(void *arg), void *arg)
@@ -120,3 +147,10 @@ void do_backlog(void)
 		}
 	}
 }
+
+#ifdef ENABLE_FUGAKU_HACKS
+ihk_spinlock_t *get_this_cpu_runq_lock(void)
+{
+	return &get_this_cpu_local_var()->runq_lock;
+}
+#endif

kernel/host.c

@@ -542,10 +542,42 @@ static int process_msg_prepare_process(unsigned long rphys)
 		}
 		vm->numa_mem_policy = MPOL_BIND;
 	}
+	else if (pn->mpol_mode != MPOL_MAX) {
+		int bit;
+
+		vm->numa_mem_policy = pn->mpol_mode;
+
+		memset(&vm->numa_mask, 0, sizeof(vm->numa_mask));
+
+		for_each_set_bit(bit, pn->mpol_nodemask,
+				PLD_PROCESS_NUMA_MASK_BITS) {
+			if (bit >= ihk_mc_get_nr_numa_nodes()) {
+				kprintf("%s: error: NUMA id %d is larger than mask size!\n",
+						__func__, bit);
+				return -EINVAL;
+			}
+			set_bit(bit, &vm->numa_mask[0]);
+		}
+
+		dkprintf("%s: numa_mem_policy: %d, numa_mask: %ld\n",
+			 __func__, vm->numa_mem_policy, vm->numa_mask[0]);
+	}
 
 	proc->uti_thread_rank = pn->uti_thread_rank;
 	proc->uti_use_last_cpu = pn->uti_use_last_cpu;
 
+	proc->straight_map = pn->straight_map;
+	proc->straight_map_threshold = pn->straight_map_threshold;
+
+#ifdef ENABLE_TOFU
+	proc->enable_tofu = pn->enable_tofu;
+	if (proc->enable_tofu) {
+		extern void tof_utofu_finalize(void);
+
+		tof_utofu_finalize();
+	}
+#endif
+
 #ifdef PROFILE_ENABLE
 	proc->profile = pn->profile;
 	thread->profile = pn->profile;
@@ -756,7 +788,11 @@ out_remote_pf:
 		syscall_channel_send(resp_channel, &pckt);
 
 		rc = do_kill(NULL, info.pid, info.tid, info.sig, &info.info, 0);
+#ifndef ENABLE_FUGAKU_HACKS
 		dkprintf("SCD_MSG_SEND_SIGNAL: do_kill(pid=%d, tid=%d, sig=%d)=%d\n", info.pid, info.tid, info.sig, rc);
+#else
+		kprintf("SCD_MSG_SEND_SIGNAL: do_kill(pid=%d, tid=%d, sig=%d)=%d\n", info.pid, info.tid, info.sig, rc);
+#endif
 		ret = 0;
 		break;
 
@@ -766,12 +802,36 @@ out_remote_pf:
 		ret = 0;
 		break;
 
-	case SCD_MSG_CLEANUP_PROCESS:
+	case SCD_MSG_CLEANUP_PROCESS: {
+		extern int process_cleanup_before_terminate(int pid);
 		dkprintf("SCD_MSG_CLEANUP_PROCESS pid=%d, thread=0x%llx\n",
 				packet->pid, packet->arg);
+
+		pckt.msg = SCD_MSG_CLEANUP_PROCESS_RESP;
+		pckt.err = process_cleanup_before_terminate(packet->pid);
+		pckt.ref = packet->ref;
+		pckt.arg = packet->arg;
+		pckt.reply = packet->reply;
+		syscall_channel_send(resp_channel, &pckt);
 		terminate_host(packet->pid, (struct thread *)packet->arg);
 		ret = 0;
 		break;
+	}
+
+	case SCD_MSG_CLEANUP_FD: {
+		extern int process_cleanup_fd(int pid, int fd);
+		pckt.msg = SCD_MSG_CLEANUP_FD_RESP;
+		pckt.err = process_cleanup_fd(packet->pid, packet->arg);
+		dkprintf("SCD_MSG_CLEANUP_FD pid=%d, fd=%d -> err: %d\n",
+				packet->pid, packet->arg, pckt.err);
+
+		pckt.ref = packet->ref;
+		pckt.arg = packet->arg;
+		pckt.reply = packet->reply;
+		syscall_channel_send(resp_channel, &pckt);
+		ret = 0;
+		break;
+	}
 
 	case SCD_MSG_DEBUG_LOG:
 		dkprintf("SCD_MSG_DEBUG_LOG code=%lx\n", packet->arg);

kernel/hugefileobj.c

@@ -85,7 +85,11 @@ static int hugefileobj_get_page(struct memobj *memobj, off_t off,
 		}
 
 		memset(obj->pages[pgind], 0, obj->pgsize);
+#ifndef ENABLE_FUGAKU_HACKS
 		dkprintf("%s: obj: 0x%lx, allocated page for off: %lu"
+#else
+		kprintf("%s: obj: 0x%lx, allocated page for off: %lu"
+#endif
 				" (ind: %d), page size: %lu\n",
 				__func__, obj, off, pgind, obj->pgsize);
 	}
@@ -274,13 +278,51 @@ int hugefileobj_create(struct memobj *memobj, size_t len, off_t off,
 
 		obj->nr_pages = nr_pages;
 		obj->pages = pages;
+#ifndef ENABLE_FUGAKU_HACKS
 		dkprintf("%s: obj: 0x%lx, VA: 0x%lx, page array allocated"
+#else
+		kprintf("%s: obj: 0x%lx, VA: 0x%lx, page array allocated"
+#endif
 				" for %d pages, pagesize: %lu\n",
 				__func__,
 				obj,
 				virt_addr,
 				nr_pages,
 				obj->pgsize);
+
+#ifdef ENABLE_FUGAKU_HACKS
+		if (!hugetlbfs_on_demand) {
+			int pgind;
+			int npages;
+
+#ifndef ENABLE_FUGAKU_HACKS
+			for (pgind = 0; pgind < obj->nr_pages; ++pgind) {
+#else
+			/* Map in only the last 8 pages */
+			for (pgind = ((obj->nr_pages > 8) ? (obj->nr_pages - 8) : 0);
+					pgind < obj->nr_pages; ++pgind) {
+#endif
+				if (obj->pages[pgind]) {
+					continue;
+				}
+
+				npages = obj->pgsize >> PAGE_SHIFT;
+				obj->pages[pgind] = ihk_mc_alloc_aligned_pages_user(npages,
+						obj->pgshift - PTL1_SHIFT,
+						IHK_MC_AP_NOWAIT | IHK_MC_AP_USER, 0);
+				if (!obj->pages[pgind]) {
+					kprintf("%s: error: could not allocate page for off: %lu"
+							", page size: %lu\n", __func__, off, obj->pgsize);
+					continue;
+				}
+
+				memset(obj->pages[pgind], 0, obj->pgsize);
+				dkprintf("%s: obj: 0x%lx, pre-allocated page for off: %lu"
+						" (ind: %d), page size: %lu\n",
+						__func__, obj, off, pgind, obj->pgsize);
+			}
+		}
+#endif
 	}
 
 	obj->memobj.size = len;

kernel/include/cls.h

@@ -20,10 +20,17 @@
  * CPU Local Storage (cls)
  */
 
+struct kmalloc_cache_header {
+	struct kmalloc_cache_header *next;
+};
+
 struct kmalloc_header {
 	unsigned int front_magic;
 	int cpu_id;
-	struct list_head list;
+	union {
+		struct list_head list;
+		struct kmalloc_cache_header *cache;
+	};
 	int size; /* The size of this chunk without the header */
 	unsigned int end_magic;
 	/* 32 bytes */
@@ -79,6 +86,7 @@ struct cpu_local_var {
 	ihk_spinlock_t runq_lock;
 	unsigned long runq_irqstate;
 	struct thread *current;
+	void *kernel_mode_pf_regs;
 	int prevpid;
 	struct list_head runq;
 	size_t runq_len;
@@ -98,6 +106,7 @@ struct cpu_local_var {
 	ihk_spinlock_t migq_lock;
 	struct list_head migq;
 	int in_interrupt;
+	int in_page_fault;
 	int no_preempt;
 	int timer_enabled;
 	unsigned long nr_ctx_switches;

kernel/include/kmalloc.h

@@ -36,4 +36,98 @@ int memcheckall(void);
 int freecheck(int runcount);
 void kmalloc_consolidate_free_list(void);
 
+#ifndef unlikely
+#define unlikely(x) __builtin_expect(!!(x), 0)
+#endif
+
+/*
+ * Generic lockless kmalloc cache.
+ */
+static inline void kmalloc_cache_free(void *elem)
+{
+	struct kmalloc_cache_header *current = NULL;
+	struct kmalloc_cache_header *new =
+		(struct kmalloc_cache_header *)elem;
+	struct kmalloc_header *header;
+	register struct kmalloc_cache_header *cache;
+
+	if (unlikely(!elem))
+		return;
+
+	/* Get cache pointer from kmalloc header */
+	header = (struct kmalloc_header *)((void *)elem -
+				sizeof(struct kmalloc_header));
+	if (unlikely(!header->cache)) {
+		kprintf("%s: WARNING: no cache for 0x%lx\n",
+				__func__, elem);
+		return;
+	}
+
+	cache = header->cache;
+
+retry:
+	current = cache->next;
+	new->next = current;
+
+	if (!__sync_bool_compare_and_swap(&cache->next, current, new)) {
+		goto retry;
+	}
+}
+
+static inline void kmalloc_cache_prealloc(struct kmalloc_cache_header *cache,
+		size_t size, int nr_elem)
+{
+	struct kmalloc_cache_header *elem;
+	int i;
+
+	if (unlikely(cache->next))
+		return;
+
+	for (i = 0; i < nr_elem; ++i) {
+		struct kmalloc_header *header;
+
+		elem = (struct kmalloc_cache_header *)
+			kmalloc(size, IHK_MC_AP_NOWAIT);
+
+		if (!elem) {
+			kprintf("%s: ERROR: allocating cache element\n", __func__);
+			continue;
+		}
+
+		/* Store cache pointer in kmalloc_header */
+		header = (struct kmalloc_header *)((void *)elem -
+				sizeof(struct kmalloc_header));
+		header->cache = cache;
+
+		kmalloc_cache_free(elem);
+	}
+}
+
+static inline void *kmalloc_cache_alloc(struct kmalloc_cache_header *cache,
+		size_t size)
+{
+	register struct kmalloc_cache_header *first, *next;
+
+retry:
+	next = NULL;
+	first = cache->next;
+
+	if (first) {
+		next = first->next;
+
+		if (!__sync_bool_compare_and_swap(&cache->next,
+					first, next)) {
+			goto retry;
+		}
+	}
+	else {
+		kprintf("%s: calling pre-alloc for 0x%lx...\n", __func__, cache);
+
+		kmalloc_cache_prealloc(cache, size, 384);
+		goto retry;
+	}
+
+	return (void *)first;
+}
+
 #endif

kernel/include/kref.h

@@ -0,0 +1,84 @@
+/*
+ * kref.h - library routines for handling generic reference counted objects
+ * (based on Linux implementation)
+ *
+ * This file is released under the GPLv2.
+ *
+ */
+
+#ifndef _KREF_H_
+#define _KREF_H_
+
+#include <ihk/atomic.h>
+#include <ihk/lock.h>
+
+/*
+ * Bit 30 marks a kref as McKernel internal.
+ * This can be used to distinguish krefs from Linux and
+ * it also ensures that a non deallocated kref will not
+ * crash the Linux allocator.
+ */
+#define MCKERNEL_KREF_MARK	(1U << 30)
+
+struct kref {
+	ihk_atomic_t		refcount;
+};
+
+#define KREF_INIT(n)	{ .refcount = IHK_ATOMIC_INIT(MCKERNEL_KREF_MARK + n), }
+
+/**
+ * kref_init - initialize object.
+ * @kref: object in question.
+ */
+static inline void kref_init(struct kref *kref)
+{
+	ihk_atomic_set(&kref->refcount, MCKERNEL_KREF_MARK + 1);
+}
+
+static inline unsigned int kref_read(const struct kref *kref)
+{
+	return (ihk_atomic_read(&kref->refcount) & ~(MCKERNEL_KREF_MARK));
+}
+
+static inline unsigned int kref_is_mckernel(const struct kref *kref)
+{
+	return (ihk_atomic_read(&kref->refcount) & (MCKERNEL_KREF_MARK));
+}
+
+/**
+ * kref_get - increment refcount for object.
+ * @kref: object.
+ */
+static inline void kref_get(struct kref *kref)
+{
+	ihk_atomic_inc(&kref->refcount);
+}
+
+/**
+ * kref_put - decrement refcount for object.
+ * @kref: object.
+ * @release: pointer to the function that will clean up the object when the
+ *	     last reference to the object is released.
+ *	     This pointer is required, and it is not acceptable to pass kfree
+ *	     in as this function.  If the caller does pass kfree to this
+ *	     function, you will be publicly mocked mercilessly by the kref
+ *	     maintainer, and anyone else who happens to notice it.  You have
+ *	     been warned.
+ *
+ * Decrement the refcount, and if 0, call release().
+ * Return 1 if the object was removed, otherwise return 0.  Beware, if this
+ * function returns 0, you still can not count on the kref from remaining in
+ * memory.  Only use the return value if you want to see if the kref is now
+ * gone, not present.
+ */
+static inline int kref_put(struct kref *kref, void (*release)(struct kref *kref))
+{
+	//if (ihk_atomic_dec_and_test(&kref->refcount)) {
+	if (ihk_atomic_sub_return(1, &kref->refcount) == MCKERNEL_KREF_MARK) {
+		release(kref);
+		return 1;
+	}
+	return 0;
+}
+
+#endif /* _KREF_H_ */

kernel/include/mman.h

@@ -79,4 +79,14 @@
 
 extern int sysctl_overcommit_memory;
 
+/*
+ * This looks more complex than it should be. But we need to
+ * get the type for the ~ right in round_down (it needs to be
+ * as wide as the result!), and we want to evaluate the macro
+ * arguments just once each.
+ */
+#define __round_mask(x, y) ((__typeof__(x))((y)-1))
+#define round_up(x, y) ((((x)-1) | __round_mask(x, y))+1)
+#define round_down(x, y) ((x) & ~__round_mask(x, y))
+
 #endif /* HEADER_MMAN_H */

kernel/include/page.h

@@ -69,4 +69,7 @@ static inline int page_is_multi_mapped(struct page *page)
 
 /* Should we take page faults on ANONYMOUS mappings? */
 extern int anon_on_demand;
+#ifdef ENABLE_FUGAKU_HACKS
+extern int hugetlbfs_on_demand;
+#endif
 #endif

kernel/include/process.h

@@ -390,10 +390,14 @@ struct vm_range {
 	struct rb_node vm_rb_node;
 	unsigned long start, end;
 	unsigned long flag;
+	unsigned long straight_start;
 	struct memobj *memobj;
 	off_t objoff;
 	int pgshift;	/* page size. 0 means THP */
 	int padding;
+#ifdef ENABLE_TOFU
+	struct list_head tofu_stag_list;
+#endif
 	void *private_data;
 };
 
@@ -558,11 +562,19 @@ struct process {
 	size_t mpol_threshold;
 	unsigned long heap_extension;
 	unsigned long mpol_bind_mask;
+	int mpol_mode;
+
 	int uti_thread_rank; /* Spawn on Linux CPU when clone_count reaches this */
 	int uti_use_last_cpu; /* Work-around not to share CPU with OpenMP thread */
 	int clone_count;
 	int thp_disable;
 
+	int straight_map;
+#ifdef ENABLE_TOFU
+	int enable_tofu;
+#endif
+	size_t straight_map_threshold;
+
 	// perf_event
 	int perf_status;
 #define PP_NONE 0
@@ -578,8 +590,18 @@ struct process {
 #endif // PROFILE_ENABLE
 	int nr_processes; /* For partitioned execution */
 	int process_rank; /* Rank in partition */
+
+	void *straight_va;
+	size_t straight_len;
+	unsigned long straight_pa;
+
 	int coredump_barrier_count, coredump_barrier_count2;
 	mcs_rwlock_lock_t coredump_lock; // lock for coredump
+#ifdef ENABLE_TOFU
+#define MAX_FD_PDE 1024
+	void *fd_pde_data[MAX_FD_PDE];
+	char *fd_path[MAX_FD_PDE];
+#endif
 };
 
 /*
@@ -737,9 +759,17 @@ struct thread {
 	void *coredump_regs;
 	struct waitq coredump_wq;
 	int coredump_status;
+
+#ifdef ENABLE_TOFU
+	/* Path of file being opened */
+	char *fd_path_in_open;
+#endif
 };
 
 #define VM_RANGE_CACHE_SIZE	4
+#ifdef ENABLE_TOFU
+#define TOFU_STAG_HASH_SIZE 4
+#endif
 
 struct process_vm {
 	struct address_space *address_space;
@@ -772,6 +802,12 @@ struct process_vm {
 	struct vm_range *range_cache[VM_RANGE_CACHE_SIZE];
 	int range_cache_ind;
 	struct swapinfo *swapinfo;
+
+#ifdef ENABLE_TOFU
+	/* Tofu STAG hash */
+	ihk_spinlock_t tofu_stag_lock;
+	struct list_head tofu_stag_hash[TOFU_STAG_HASH_SIZE];
+#endif
 };
 
 static inline int has_cap_ipc_lock(struct thread *th)

kernel/include/profile.h

@@ -1,9 +1,6 @@
 #ifndef __PROCESS_PROFILE_H_
 #define __PROCESS_PROFILE_H_
 
-/* Uncomment this to enable profiling */
-#define PROFILE_ENABLE
-
 #ifdef PROFILE_ENABLE
 #define PROFILE_SYSCALL_MAX                          2000
 #define PROFILE_OFFLOAD_MAX   (PROFILE_SYSCALL_MAX << 1)
@@ -40,14 +37,25 @@ enum profile_event_type {
 	PROFILE_remote_page_fault,
 	PROFILE_mpol_alloc_missed,
 	PROFILE_mmap_anon_contig_phys,
+	PROFILE_mmap_anon_straight,
+	PROFILE_mmap_anon_not_straight,
 	PROFILE_mmap_anon_no_contig_phys,
 	PROFILE_mmap_regular_file,
 	PROFILE_mmap_device_file,
+	PROFILE_tofu_stag_alloc,
+	PROFILE_tofu_stag_alloc_new_steering,
+	PROFILE_tofu_stag_alloc_new_steering_alloc_mbpt,
+	PROFILE_tofu_stag_alloc_new_steering_update_mbpt,
+	PROFILE_tofu_stag_free_stags,
+	PROFILE_tofu_stag_free_stag,
+	PROFILE_tofu_stag_free_stag_pre,
+	PROFILE_tofu_stag_free_stag_cqflush,
+	PROFILE_tofu_stag_free_stag_dealloc,
+	PROFILE_tofu_stag_free_stag_dealloc_free_pages,
 	PROFILE_EVENT_MAX	/* Should be the last event type */
 };
 
-#define __NR_profile                   PROFILE_EVENT_MAX
-
+#ifdef __KERNEL__
 struct thread;
 struct process;
 
@@ -61,6 +69,77 @@ int profile_accumulate_and_print_job_events(struct process *proc);
 int profile_alloc_events(struct thread *thread);
 void profile_dealloc_thread_events(struct thread *thread);
 void profile_dealloc_proc_events(struct process *proc);
+
+#else // User space interface
+#include <unistd.h>
+#include <sys/syscall.h>
+
+#define __NR_profile                   PROFILE_EVENT_MAX
+
+/* Per-thread */
+static inline void mckernel_profile_thread_on(void)
+{
+	syscall(__NR_profile, PROF_ON);
+}
+
+static inline void mckernel_profile_thread_off(void)
+{
+	syscall(__NR_profile, PROF_OFF);
+}
+
+static inline void mckernel_profile_thread_print(void)
+{
+	syscall(__NR_profile, PROF_PRINT);
+}
+
+static inline void mckernel_profile_thread_print_off(void)
+{
+	syscall(__NR_profile, PROF_OFF | PROF_PRINT);
+}
+
+/* Per-process */
+static inline void mckernel_profile_process_on(void)
+{
+	syscall(__NR_profile, PROF_PROC | PROF_ON);
+}
+
+static inline void mckernel_profile_process_off(void)
+{
+	syscall(__NR_profile, PROF_PROC | PROF_OFF);
+}
+
+static inline void mckernel_profile_process_print(void)
+{
+	syscall(__NR_profile, PROF_PROC | PROF_PRINT);
+}
+
+static inline void mckernel_profile_process_print_off(void)
+{
+	syscall(__NR_profile, PROF_PROC | PROF_OFF | PROF_PRINT);
+}
+
+/* Per-job */
+static inline void mckernel_profile_job_on(void)
+{
+	syscall(__NR_profile, PROF_JOB | PROF_ON);
+}
+
+static inline void mckernel_profile_job_off(void)
+{
+	syscall(__NR_profile, PROF_JOB | PROF_OFF);
+}
+
+static inline void mckernel_profile_job_print(void)
+{
+	syscall(__NR_profile, PROF_JOB | PROF_PRINT);
+}
+
+static inline void mckernel_profile_job_print_off(void)
+{
+	syscall(__NR_profile, PROF_JOB | PROF_OFF | PROF_PRINT);
+}
+
+#endif // __KERNEL__
 #endif // PROFILE_ENABLE
 
 

kernel/include/rbtree.h

@@ -108,4 +108,6 @@ static inline void rb_link_node(struct rb_node * node, struct rb_node * parent,
 			typeof(*pos), field); 1; }); \
 	     pos = n)
 
+struct rb_node *rb_preorder_dfs_search(const struct rb_root *root,
+	bool (*__cond)(struct rb_node *, void *arg), void *__cond_arg);
 #endif	/* _LINUX_RBTREE_H */

kernel/include/syscall.h

@@ -20,6 +20,7 @@
 #include <ihk/ikc.h>
 #include <rlimit.h>
 #include <time.h>
+#include <profile.h>
 
 #define NUM_SYSCALLS 255
 
@@ -39,7 +40,8 @@
 #define SCD_MSG_SEND_SIGNAL             0x7
 #define SCD_MSG_SEND_SIGNAL_ACK         0x8
 #define SCD_MSG_CLEANUP_PROCESS         0x9
-#define SCD_MSG_GET_VDSO_INFO           0xa
+#define SCD_MSG_CLEANUP_PROCESS_RESP    0xa
+#define SCD_MSG_GET_VDSO_INFO           0xb
 
 #define SCD_MSG_GET_CPU_MAPPING         0xc
 #define SCD_MSG_REPLY_GET_CPU_MAPPING   0xd
@@ -84,6 +86,8 @@
 
 #define SCD_MSG_CPU_RW_REG              0x52
 #define SCD_MSG_CPU_RW_REG_RESP         0x53
+#define SCD_MSG_CLEANUP_FD              0x54
+#define SCD_MSG_CLEANUP_FD_RESP         0x55
 
 #define SCD_MSG_FUTEX_WAKE              0x60
 
@@ -180,6 +184,18 @@ typedef unsigned long __cpu_set_unit;
 #define MPOL_NO_BSS               0x04
 #define MPOL_SHM_PREMAP           0x08
 
+/* should be the same as process.h */
+#define PLD_PROCESS_NUMA_MASK_BITS 256
+
+enum {
+	PLD_MPOL_DEFAULT,
+	PLD_MPOL_PREFERRED,
+	PLD_MPOL_BIND,
+	PLD_MPOL_INTERLEAVE,
+	PLD_MPOL_LOCAL,
+	PLD_MPOL_MAX,	/* always last member of enum */
+};
+
 #define PLD_MAGIC 0xcafecafe44332211UL
 
 struct program_load_desc {
@@ -214,9 +230,18 @@ struct program_load_desc {
 	unsigned long heap_extension;
 	long stack_premap;
 	unsigned long mpol_bind_mask;
+	int mpol_mode;
+	unsigned long mpol_nodemask[PLD_PROCESS_NUMA_MASK_BITS /
+				    (sizeof(unsigned long) * 8)];
+
 	int thp_disable;
 	int uti_thread_rank; /* N-th clone() spawns a thread on Linux CPU */
 	int uti_use_last_cpu; /* Work-around not to share CPU with OpenMP thread */
+	int straight_map;
+	size_t straight_map_threshold;
+#ifdef ENABLE_TOFU
+	int enable_tofu;
+#endif
 	int nr_processes;
 	int process_rank;
 	__cpu_set_unit cpu_set[PLD_CPU_SET_SIZE];
@@ -327,6 +352,7 @@ struct syscall_response {
 	unsigned long req_thread_status;
 	long ret;
 	unsigned long fault_address;
+	void *pde_data;
 };
 
 struct syscall_post {

kernel/include/tofu/generate_headers.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+SCRIPT="`readlink -f ${BASH_SOURCE[0]:-}`"
+SCRIPT_DIR=$(dirname ${SCRIPT})
+CURRENT_DIR=`pwd`
+
+cd ${SCRIPT_DIR}
+
+DWARF_TOOL=${SCRIPT_DIR}/../../../tools/dwarf-extract-struct/dwarf-extract-struct
+if [ ! -x ${DWARF_TOOL} ]; then
+	echo "error: couldn't find DWARF extractor executable (${DWARF_TOOL}), have you compiled it?"
+	cd -
+	exit 1
+fi
+
+echo "Looking for Tofu driver debug symbols..."
+if [ "`find /lib/modules/ -name "tof_module.tar.gz" | xargs -r ls -t | head -n 1 | wc -l`" == "0" ]; then
+	echo "error: couldn't find Tofu modules with debug symbols"
+	cd -
+	exit 1
+fi
+
+MODULE_TAR_GZ=`find /lib/modules/ -name "tof_module.tar.gz" | xargs ls -t | head -n 1`
+echo "Using Tofu driver debug symbols: ${MODULE_TAR_GZ}"
+
+KMODULE=tof_utofu.ko 
+if ! tar zxvf ${MODULE_TAR_GZ} ${KMODULE} 2>&1 > /dev/null; then
+	echo "error: uncompressing kernel module with debug symbols"
+	cd -
+	exit 1
+fi
+
+${DWARF_TOOL} ${KMODULE} tof_utofu_device enabled subnet gpid > tofu_generated-tof_utofu_device.h
+${DWARF_TOOL} ${KMODULE} tof_utofu_cq common tni cqid trans steering mb num_stag | sed "s/struct FILL_IN_MANUALLY trans;/#include \"tof_utofu_cq_trans.h\"/g" > tofu_generated-tof_utofu_cq.h
+${DWARF_TOOL} ${KMODULE} tof_utofu_mbpt ucq iova sg nsgents mbptstart pgsz kref > tofu_generated-tof_utofu_mbpt.h
+${DWARF_TOOL} ${KMODULE} tof_utofu_bg common tni bgid bch | sed "s/struct FILL_IN_MANUALLY bch;/#include \"tof_utofu_bg_bch.h\"/g" > tofu_generated-tof_utofu_bg.h
+rm ${KMODULE}
+
+KMODULE=tof_core.ko 
+if ! tar zxvf ${MODULE_TAR_GZ} ${KMODULE} 2>&1 > /dev/null; then
+	echo "error: uncompressing kernel module with debug symbols"
+	cd -
+	exit 1
+fi
+
+${DWARF_TOOL} ${KMODULE} tof_core_cq reg | sed "s/struct FILL_IN_MANUALLY reg;/#include \"tof_core_cq_reg.h\"/g" > tofu_generated-tof_core_cq.h 
+${DWARF_TOOL} ${KMODULE} tof_core_bg lock reg irq subnet gpid sighandler | sed "s/struct FILL_IN_MANUALLY reg;/#include \"tof_core_bg_reg.h\"/g" > tofu_generated-tof_core_bg.h 
+rm ${KMODULE}
+
+#cat tofu_generated*.h
+cd - > /dev/null

kernel/include/tofu/tof_core_bg_reg.h

@@ -0,0 +1,4 @@
+	struct {
+		void *bgs;
+		void *bch;
+	} reg;

kernel/include/tofu/tof_core_cq_reg.h

@@ -0,0 +1,4 @@
+	struct {
+		void *cq;
+		void *cqs;
+	} reg;

kernel/include/tofu/tof_icc.h

@@ -0,0 +1,836 @@
+#ifndef _TOF_ICC_H_
+#define _TOF_ICC_H_
+
+#include <types.h>
+#include <bitops.h>
+typedef uint64_t phys_addr_t;
+
+/* @ref.impl include/linux/bitops.h */
+/*
+ * Create a contiguous bitmask starting at bit position @l and ending at
+ * position @h. For example
+ * GENMASK_ULL(39, 21) gives us the 64bit vector 0x000000ffffe00000.
+ */
+#define GENMASK(h, l) \
+	(((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h))))
+
+/* constants related to the Tofu Interconnect D */
+
+#define TOF_ICC_NTNIS 6
+#define TOF_ICC_NCQS 12
+#define TOF_ICC_NBGS 48
+#define TOF_ICC_NBCHS 16
+#define TOF_ICC_NPORTS 10
+#define TOF_ICC_NVMSIDS 16
+
+#define TOF_ICC_RH_LEN 8
+#define TOF_ICC_ECRC_LEN 4
+#define TOF_ICC_FRAME_ALIGN 32
+#define TOF_ICC_TLP_LEN(len) (((len) + 1) * TOF_ICC_FRAME_ALIGN)
+#define TOF_ICC_TLP_PAYLOAD_MAX (TOF_ICC_TLP_LEN(61) - TOF_ICC_ECRC_LEN)
+#define TOF_ICC_FRAME_LEN(len) (TOF_ICC_RH_LEN + TOF_ICC_TLP_LEN(len))
+#define TOF_ICC_FRAME_LEN_MIN TOF_ICC_FRAME_LEN(2)
+#define TOF_ICC_FRAME_LEN_MAX TOF_ICC_FRAME_LEN(61)
+#define TOF_ICC_FRAME_BUF_SIZE_BITS 11
+#define TOF_ICC_FRAME_BUF_SIZE (1 << TOF_ICC_FRAME_BUF_SIZE_BITS)
+#define TOF_ICC_FRAME_BUF_ALIGN_BITS 8
+#define TOF_ICC_FRAME_BUF_ALIGN (1 << TOF_ICC_FRAME_BUF_ALIGN_BITS)
+#define TOF_ICC_PB_SIZE_BITS 11
+#define TOF_ICC_PB_SIZE (1 << TOF_ICC_PB_SIZE_BITS)
+#define TOF_ICC_PB_ALIGN_BITS 11
+#define TOF_ICC_PB_ALIGN (1 << TOF_ICC_PB_ALIGN_BITS)
+
+#define TOF_ICC_ST_ALIGN_BITS 8
+#define TOF_ICC_ST_ALIGN (1 << TOF_ICC_ST_ALIGN_BITS)
+
+#define TOF_ICC_MBT_ALIGN_BITS 8
+#define TOF_ICC_MBT_ALIGN (1 << TOF_ICC_MBT_ALIGN_BITS)
+
+#define TOF_ICC_MBPT_ALIGN_BITS 8
+#define TOF_ICC_MBPT_ALIGN (1 << TOF_ICC_MBPT_ALIGN_BITS)
+
+#define TOF_ICC_BG_BSEQ_SIZE_BITS 24
+#define TOF_ICC_BG_BSEQ_SIZE (1 << TOF_ICC_BG_BSEQ_SIZE_BITS)
+
+#define TOF_ICC_BCH_DMA_ALIGN_BITS 8
+#define TOF_ICC_BCH_DMA_ALIGN (1 << TOF_ICC_BCH_DMA_ALIGN_BITS)
+
+/* this is a CPU-specific constant, but referred in the ICC spec. */
+#define TOF_ICC_CACHE_LINE_SIZE_BITS 8
+#define TOF_ICC_CACHE_LINE_SIZE (1 << TOF_ICC_CACHE_LINE_SIZE_BITS)
+
+#define TOF_ICC_TOQ_DESC_SIZE_BITS 5
+#define TOF_ICC_TOQ_DESC_SIZE (1 << TOF_ICC_TOQ_DESC_SIZE_BITS)
+#define TOF_ICC_TCQ_DESC_SIZE_BITS 3
+#define TOF_ICC_TCQ_DESC_SIZE (1 << TOF_ICC_TCQ_DESC_SIZE_BITS)
+#define TOF_ICC_TCQ_NLINE_BITS (TOF_ICC_CACHE_LINE_SIZE_BITS - TOF_ICC_TCQ_DESC_SIZE_BITS)
+#define TOF_ICC_MRQ_DESC_SIZE_BITS 5
+#define TOF_ICC_MRQ_DESC_SIZE (1 << TOF_ICC_MRQ_DESC_SIZE_BITS)
+#define TOF_ICC_PBQ_DESC_SIZE_BITS 3
+#define TOF_ICC_PBQ_DESC_SIZE (1 << TOF_ICC_PBQ_DESC_SIZE_BITS)
+#define TOF_ICC_PRQ_DESC_SIZE_BITS 3
+#define TOF_ICC_PRQ_DESC_SIZE (1 << TOF_ICC_PRQ_DESC_SIZE_BITS)
+#define TOF_ICC_PRQ_NLINE_BITS (TOF_ICC_CACHE_LINE_SIZE_BITS - TOF_ICC_PBQ_DESC_SIZE_BITS)
+
+#define TOF_ICC_TOQ_SIZE_NTYPES 6
+#define TOF_ICC_TOQ_SIZE_BITS(size) ((size) * 2 + 11)
+#define TOF_ICC_TOQ_SIZE(size) (1 << TOF_ICC_TOQ_SIZE_BITS(size))
+#define TOF_ICC_TOQ_LEN(size) (TOF_ICC_TOQ_SIZE(size) * TOF_ICC_TOQ_DESC_SIZE)
+#define TOF_ICC_TCQ_LEN(size) (TOF_ICC_TOQ_SIZE(size) * TOF_ICC_TCQ_DESC_SIZE)
+
+#define TOF_ICC_MRQ_SIZE_NTYPES 6
+#define TOF_ICC_MRQ_SIZE_BITS(size) ((size) * 2 + 11)
+#define TOF_ICC_MRQ_SIZE(size) (1 << TOF_ICC_MRQ_SIZE_BITS(size))
+#define TOF_ICC_MRQ_LEN(size) (TOF_ICC_MRQ_SIZE(size) * TOF_ICC_MRQ_DESC_SIZE)
+
+#define TOF_ICC_PBQ_SIZE_NTYPES 6
+#define TOF_ICC_PBQ_SIZE_BITS(size) ((size) * 2 + 11)
+#define TOF_ICC_PBQ_SIZE(size) (1 << TOF_ICC_PBQ_SIZE_BITS(size))
+#define TOF_ICC_PBQ_LEN(size) (TOF_ICC_PBQ_SIZE(size) * TOF_ICC_PBQ_DESC_SIZE)
+
+#define TOF_ICC_PRQ_SIZE_NTYPES 6
+#define TOF_ICC_PRQ_SIZE_BITS(size) ((size) * 2 + 11)
+#define TOF_ICC_PRQ_SIZE(size) (1 << TOF_ICC_PRQ_SIZE_BITS(size))
+#define TOF_ICC_PRQ_LEN(size) (TOF_ICC_PRQ_SIZE(size) * TOF_ICC_PRQ_DESC_SIZE)
+
+#define TOF_ICC_STEERING_TABLE_ALIGN_BITS 8
+#define TOF_ICC_STEERING_TABLE_ALIGN (1 << TOF_ICC_STEERING_TABLE_ALIGN_BITS)
+#define TOF_ICC_STEERING_SIZE_BITS 4
+#define TOF_ICC_STEERING_SIZE (1 << TOF_ICC_STEERING_SIZE_BITS)
+
+#define TOF_ICC_MB_TABLE_ALIGN_BITS 8
+#define TOF_ICC_MB_TABLE_ALIGN (1 << TOF_ICC_MB_TABLE_ALIGN_BITS)
+#define TOF_ICC_MB_SIZE_BITS 4
+#define TOF_ICC_MB_SIZE (1 << TOF_ICC_MB_SIZE_BITS)
+#define TOF_ICC_MB_PS_ENCODE(bits) ((bits) % 9 == 3 ? (bits) / 9 - 1 : (bits) / 13 + 3)
+
+#define TOF_ICC_MBPT_ALIGN_BITS 8
+#define TOF_ICC_MBPT_ALIGN (1 << TOF_ICC_MBPT_ALIGN_BITS)
+#define TOF_ICC_MBPT_SIZE_BITS 3
+#define TOF_ICC_MBPT_SIZE (1 << TOF_ICC_MBPT_SIZE_BITS)
+
+#define TOF_ICC_X_BITS 5
+#define TOF_ICC_Y_BITS 5
+#define TOF_ICC_Z_BITS 5
+#define TOF_ICC_A_BITS 1
+#define TOF_ICC_B_BITS 2
+#define TOF_ICC_C_BITS 1
+#define TOF_ICC_MAX_X_SIZE (1 << TOF_ICC_X_BITS)
+#define TOF_ICC_MAX_Y_SIZE (1 << TOF_ICC_Y_BITS)
+#define TOF_ICC_MAX_Z_SIZE (1 << TOF_ICC_Z_BITS)
+#define TOF_ICC_A_SIZE 2
+#define TOF_ICC_B_SIZE 3
+#define TOF_ICC_C_SIZE 2
+#define TOF_ICC_X_MASK ((1 << TOF_ICC_X_BITS) - 1)
+#define TOF_ICC_Y_MASK ((1 << TOF_ICC_Y_BITS) - 1)
+#define TOF_ICC_Z_MASK ((1 << TOF_ICC_Z_BITS) - 1)
+#define TOF_ICC_A_MASK ((1 << TOF_ICC_A_BITS) - 1)
+#define TOF_ICC_B_MASK ((1 << TOF_ICC_B_BITS) - 1)
+#define TOF_ICC_C_MASK ((1 << TOF_ICC_C_BITS) - 1)
+#define TOF_ICC_ABC_SIZE (TOF_ICC_A_SIZE * TOF_ICC_B_SIZE * TOF_ICC_C_SIZE)
+
+static inline int tof_icc_get_framelen(int len){
+	len = TOF_ICC_RH_LEN + round_up(len + TOF_ICC_ECRC_LEN, TOF_ICC_FRAME_ALIGN);
+	if(len < TOF_ICC_FRAME_LEN_MIN){
+		len = TOF_ICC_FRAME_LEN_MIN;
+	}
+	return len;
+}
+
+/** Descriptors **/
+/** commands and rcodes **/
+enum {
+	TOF_ICC_TOQ_NOP,
+	TOF_ICC_TOQ_PUT,
+	TOF_ICC_TOQ_WRITE_PIGGYBACK_BUFFER,
+	TOF_ICC_TOQ_PUT_PIGGYBACK,
+	TOF_ICC_TOQ_GET,
+	TOF_ICC_TOQ_GETL,
+	TOF_ICC_TOQ_ATOMIC_READ_MODIFY_WRITE = 0xe,
+	TOF_ICC_TOQ_TRANSMIT_RAW_PACKET1 = 0x10,
+	TOF_ICC_TOQ_TRANSMIT_RAW_PACKET2,
+	TOF_ICC_TOQ_TRANSMIT_SYSTEM_PACKET1,
+	TOF_ICC_TOQ_TRANSMIT_SYSTEM_PACKET2,
+
+	TOF_ICC_TOQ_NCOMMANDS,
+};
+
+enum {
+	TOF_ICC_MRQ_ATOMIC_READ_MODIFY_WRITE_HALFWAY_NOTICE = 0x1,
+	TOF_ICC_MRQ_ATOMIC_READ_MODIFY_WRITE_NOTICE,
+	TOF_ICC_MRQ_ATOMIC_READ_MODIFY_WRITE_REMOTE_ERROR,
+	TOF_ICC_MRQ_PUT_HALFWAY_NOTICE,
+	TOF_ICC_MRQ_PUT_LAST_HALFWAY_NOTICE,
+	TOF_ICC_MRQ_GET_HALFWAY_NOTICE,
+	TOF_ICC_MRQ_GET_LAST_HALFWAY_NOTICE,
+	TOF_ICC_MRQ_PUT_NOTICE,
+	TOF_ICC_MRQ_PUT_LAST_NOTICE,
+	TOF_ICC_MRQ_GET_NOTICE,
+	TOF_ICC_MRQ_GET_LAST_NOTICE,
+	TOF_ICC_MRQ_PUT_REMOTE_ERROR,
+	TOF_ICC_MRQ_PUT_LAST_REMOTE_ERROR,
+	TOF_ICC_MRQ_GET_REMOTE_ERROR,
+	TOF_ICC_MRQ_GET_LAST_REMOTE_ERROR,
+
+	TOF_ICC_MRQ_NCOMMANDS,
+};
+
+enum {
+	TOF_ICC_PRQ_UNKNOWN_TLP,
+	TOF_ICC_PRQ_SYSTEM_TLP,
+	TOF_ICC_PRQ_ADDRESS_RANGE_EXCEPTION = 0x6,
+	TOF_ICC_PRQ_CQ_EXCEPTION = 0x8,
+	TOF_ICC_PRQ_ILLEGAL_TLP_FLAGS,
+	TOF_ICC_PRQ_ILLEGAL_TLP_LENGTH,
+	TOF_ICC_PRQ_CQ_ERROR = 0xc,
+};
+
+/** structures **/
+struct tof_icc_steering_entry {
+	uint64_t res1:6;
+	uint64_t readonly:1;
+	uint64_t enable:1;
+	uint64_t mbva:32;
+	uint64_t res2:8;
+	uint64_t mbid:16;
+	uint64_t length;  /* for optimization */
+};
+
+struct tof_icc_mb_entry {
+	uint64_t ps:3;
+	uint64_t res1:4;
+	uint64_t enable:1;
+	uint64_t ipa:32;
+	uint64_t res2:24;
+	uint64_t npage;  /* for optimization */
+};
+
+struct tof_icc_mbpt_entry {
+	uint64_t res1:7;
+	uint64_t enable:1;
+	uint64_t res2:4;
+	uint64_t ipa:28;
+	uint64_t res3:24;
+};
+
+struct tof_icc_cq_stag_offset {
+	uint64_t offset:40;
+	uint64_t stag:18;
+	uint64_t cqid:6;
+};
+
+struct tof_icc_toq_common_header1 {
+	uint8_t interrupt:1;
+	uint8_t res1:4;
+	uint8_t source_type:2;
+	uint8_t flip:1;
+	uint8_t command;
+	union {
+		uint8_t mtu;
+		struct {
+			uint8_t res:4;
+			uint8_t op:4;
+		} armw;
+	} mtuop;
+	uint8_t sps:4;
+	uint8_t pa:1;
+	uint8_t pb:2;
+	uint8_t pc:1;
+	uint8_t rx;
+	uint8_t ry;
+	uint8_t rz;
+	uint8_t ra:1;
+	uint8_t rb:2;
+	uint8_t rc:1;
+	uint8_t res3:1;
+	uint8_t ri:3;
+};
+
+struct tof_icc_toq_common_header2 {
+	uint8_t gap;
+	uint8_t s:1;
+	uint8_t r:1;
+	uint8_t q:1;
+	uint8_t p:1;
+	uint8_t res1:1;
+	uint8_t j:1;
+	uint8_t res2:2;
+	uint16_t edata;
+	union{
+		struct {
+			uint32_t length:24;
+			uint32_t res:8;
+		} normal;
+		struct {
+			uint32_t length:6;
+			uint32_t res:26;
+		} piggyback;
+	} len;
+};
+
+struct tof_icc_toq_descriptor {
+	struct tof_icc_toq_common_header1 head1;
+	uint64_t res[3];
+};
+
+struct tof_icc_toq_nop {
+	struct tof_icc_toq_common_header1 head1;
+	uint64_t res[3];
+};
+
+struct tof_icc_toq_put {
+	struct tof_icc_toq_common_header1 head1;
+	struct tof_icc_toq_common_header2 head2;
+	struct tof_icc_cq_stag_offset remote;
+	struct tof_icc_cq_stag_offset local;
+};
+
+struct tof_icc_toq_write_piggyback_buffer {
+	struct tof_icc_toq_common_header1 head1;
+	uint64_t data[3];
+};
+
+struct tof_icc_toq_put_piggyback {
+	struct tof_icc_toq_common_header1 head1;
+	struct tof_icc_toq_common_header2 head2;
+	struct tof_icc_cq_stag_offset remote;
+	uint64_t data;
+};
+
+struct tof_icc_toq_get {
+	struct tof_icc_toq_common_header1 head1;
+	struct tof_icc_toq_common_header2 head2;
+	struct tof_icc_cq_stag_offset remote;
+	struct tof_icc_cq_stag_offset local;
+};
+
+struct tof_icc_toq_atomic_read_modify_write {
+	struct tof_icc_toq_common_header1 head1;
+	struct tof_icc_toq_common_header2 head2;
+	struct tof_icc_cq_stag_offset remote;
+	uint64_t data;
+};
+
+struct tof_icc_toq_transmit_raw_packet1 {
+	struct tof_icc_toq_common_header1 head1;
+	uint8_t gap;
+	uint8_t res4[3];
+	uint32_t length:12;
+	uint32_t res5:20;
+	uint64_t res6;
+	uint64_t pa:48;  /* for optimization */
+	uint64_t res7:16;
+};
+
+struct tof_icc_toq_transmit_raw_packet2 {
+	uint8_t interrupt:1;
+	uint8_t res1:4;
+	uint8_t source_type:2;
+	uint8_t flip:1;
+	uint8_t command;
+	uint8_t res2:7;
+	uint8_t e:1;
+	uint8_t res3[4];
+	uint8_t port:5;
+	uint8_t res4:1;
+	uint8_t vc:2;
+	uint8_t gap;
+	uint8_t res5[3];
+	uint32_t length:12;
+	uint32_t res6:20;
+	uint64_t res7;
+	uint64_t pa:48;  /* for optimization */
+	uint64_t res8:16;
+};
+
+struct tof_icc_toq_transmit_system_packet {
+	struct tof_icc_toq_common_header1 head1;  /* rx, ry, rz should be rdx, rdy, rdz */
+	uint8_t gap;
+	uint8_t res4[3];
+	uint32_t length:12;
+	uint32_t res5:20;
+	uint64_t res6;
+	uint64_t pa:48;  /* for optimization */
+	uint64_t res7:16;
+};
+
+struct tof_icc_tcq_descriptor {
+	uint8_t res1:5;
+	uint8_t counter_unmatch:1;
+	uint8_t res2:1;
+	uint8_t flip:1;
+	uint8_t rcode;
+	uint8_t res3[2];
+	union{
+		struct {
+			uint32_t length:24;
+			uint32_t res:8;
+		} normal;
+		struct {
+			uint32_t length:6;
+			uint32_t res:26;
+		} piggyback;
+	} len;
+};
+
+struct tof_icc_mrq_common_header1 {
+	uint8_t res1:7;
+	uint8_t flip:1;
+	uint8_t id;
+	uint8_t rcode;
+	uint8_t res2:4;
+	uint8_t pa:1;
+	uint8_t pb:2;
+	uint8_t pc:1;
+	uint8_t x;
+	uint8_t y;
+	uint8_t z;
+	uint8_t a:1;
+	uint8_t b:2;
+	uint8_t c:1;
+	uint8_t res3:1;
+	uint8_t i:3;
+};
+
+struct tof_icc_mrq_common_header2 {
+	uint8_t res1;
+	uint8_t res2:4;
+	uint8_t initial:1;
+	uint8_t res3:3;
+	uint16_t edata;
+	union {
+		struct {
+			uint32_t length:11;
+			uint32_t res:21;
+		} normal;
+		struct {
+			uint32_t op:4;
+			uint32_t res:28;
+		} armw;
+	} lenop;
+};
+
+struct tof_icc_mrq_atomic_read_modify_write_halfway_notice {
+	struct tof_icc_mrq_common_header1 head1;
+	struct tof_icc_mrq_common_header2 head2;
+	struct tof_icc_cq_stag_offset local;
+	struct tof_icc_cq_stag_offset remote;
+};
+
+struct tof_icc_mrq_descriptor {
+	struct tof_icc_mrq_common_header1 head1;
+	struct tof_icc_mrq_common_header2 head2;
+	struct tof_icc_cq_stag_offset cso1;
+	struct tof_icc_cq_stag_offset cso2;
+};
+
+struct tof_icc_pbq_descriptor {
+	uint64_t res1:7;
+	uint64_t f:1;
+	uint64_t res2:3;
+	uint64_t pa:29;
+	uint64_t res3:24;
+};
+
+struct tof_icc_prq_descriptor {
+	uint64_t rcode:7;
+	uint64_t f:1;
+	uint64_t res1:3;
+	uint64_t pa:29;
+	uint64_t res2:8;
+	uint64_t w:1;
+	uint64_t res3:5;
+	uint64_t l:1;
+	uint64_t e:1;
+	uint64_t res4:8;
+};
+
+
+/** Registers **/
+/* useful packed structures */
+struct tof_icc_reg_subnet {
+	uint64_t lz:6;
+	uint64_t sz:6;
+	uint64_t nz:6;
+	uint64_t ly:6;
+	uint64_t sy:6;
+	uint64_t ny:6;
+	uint64_t lx:6;
+	uint64_t sx:6;
+	uint64_t nx:6;
+	uint64_t res:10;
+};
+
+struct tof_icc_reg_bg_address {
+	uint32_t bgid:6;
+	uint32_t tni:3;
+	uint32_t c:1;
+	uint32_t b:2;
+	uint32_t a:1;
+	uint32_t z:5;
+	uint32_t y:5;
+	uint32_t x:5;
+	uint32_t pc:1;
+	uint32_t pb:2;
+	uint32_t pa:1;
+};
+
+/* relative offset of interrupt controller registers */
+#define TOF_ICC_IRQREG_IRR 0x0
+#define TOF_ICC_IRQREG_IMR 0x8
+#define TOF_ICC_IRQREG_IRC 0x10
+#define TOF_ICC_IRQREG_IMC 0x18
+#define TOF_ICC_IRQREG_ICL 0x20
+
+/* TOFU REGISTERS */
+#define  tof_icc_reg_pa 0x40000000
+
+/* CQ */
+#define TOF_ICC_REG_CQ_PA(tni, cqid) (tof_icc_reg_pa + 0 + (tni) * 0x1000000 + (cqid) * 0x10000)
+#define TOF_ICC_REG_CQ_TOQ_DIRECT_DESCRIPTOR 0x0
+#define TOF_ICC_REG_CQ_TOQ_FETCH_START 0x40
+#define TOF_ICC_REG_CQ_MRQ_FULL_POINTER 0x48
+#define TOF_ICC_REG_CQ_TOQ_PIGGYBACK_BUFFER0 0x50
+#define TOF_ICC_REG_CQ_TOQ_PIGGYBACK_BUFFER1 0x58
+#define TOF_ICC_REG_CQ_TOQ_PIGGYBACK_BUFFER2 0x60
+#define TOF_ICC_REG_CQ_TCQ_NUM_NOTICE 0x68
+#define TOF_ICC_REG_CQ_MRQ_NUM_NOTICE 0x70
+#define TOF_ICC_REG_CQ_TX_PAYLOAD_BYTE 0x78
+#define TOF_ICC_REG_CQ_RX_PAYLOAD_BYTE 0x80
+#define TOF_ICC_REG_CQ_DUMP_START 0x0
+#define TOF_ICC_REG_CQ_DUMP_END 0x88
+
+/* BCH */
+#define TOF_ICC_REG_BCH_PA(tni, bgid) (tof_icc_reg_pa + 0x0000e00000 + (tni) * 0x1000000 + (bgid) * 0x10000)
+#define TOF_ICC_REG_BCH_IDATA 0x800
+#define TOF_ICC_REG_BCH_READY 0x840
+#define TOF_ICC_REG_BCH_READY_STATE BIT(63)
+#define TOF_ICC_REG_BCH_IGNORED_SIGNAL_COUNT 0x848
+#define TOF_ICC_REG_BCH_DUMP_START 0x800
+#define TOF_ICC_REG_BCH_DUMP_END 0x850
+
+/* CQS */
+#define TOF_ICC_REG_CQS_PA(tni, cqid) (tof_icc_reg_pa + 0x0000400000 + (tni) * 0x1000000 + (cqid) * 0x10000)
+#define TOF_ICC_REG_CQS_STATUS 0x0
+#define TOF_ICC_REG_CQS_STATUS_DESCRIPTOR_PROCESS_STOP BIT(63)
+#define TOF_ICC_REG_CQS_STATUS_DESCRIPTOR_FETCH_STOP BIT(62)
+#define TOF_ICC_REG_CQS_STATUS_BLANK_ENTRY_FLIP_BIT BIT(61)
+#define TOF_ICC_REG_CQS_STATUS_CACHE_FLUSH_BUSY BIT(60)
+#define TOF_ICC_REG_CQS_STATUS_CQ_ENABLE BIT(59)
+#define TOF_ICC_REG_CQS_STATUS_SESSION_DEAD BIT(58)
+#define TOF_ICC_REG_CQS_STATUS_SESSION_OFFSET_OVERFLOW BIT(57)
+#define TOF_ICC_REG_CQS_STATUS_SESSION_OFFSET GENMASK(56, 32)
+#define TOF_ICC_REG_CQS_STATUS_NEXT_DESCRIPTOR_OFFSET GENMASK(29, 5)
+#define TOF_ICC_REG_CQS_ENABLE 0x8
+#define TOF_ICC_REG_CQS_CACHE_FLUSH 0x10
+#define TOF_ICC_REG_CQS_FETCH_STOP 0x18
+#define TOF_ICC_REG_CQS_MODE 0x20
+#define TOF_ICC_REG_CQS_MODE_SYSTEM BIT(63)
+#define TOF_ICC_REG_CQS_MODE_TRP2_ENABLE BIT(62)
+#define TOF_ICC_REG_CQS_MODE_TRP1_ENABLE BIT(61)
+#define TOF_ICC_REG_CQS_MODE_SESSION BIT(60)
+#define TOF_ICC_REG_CQS_MODE_SUBNET_NX GENMASK(53, 48)
+#define TOF_ICC_REG_CQS_MODE_SUBNET_SX GENMASK(47, 42)
+#define TOF_ICC_REG_CQS_MODE_SUBNET_LX GENMASK(41, 36)
+#define TOF_ICC_REG_CQS_MODE_SUBNET_NY GENMASK(35, 30)
+#define TOF_ICC_REG_CQS_MODE_SUBNET_SY GENMASK(29, 24)
+#define TOF_ICC_REG_CQS_MODE_SUBNET_LY GENMASK(23, 18)
+#define TOF_ICC_REG_CQS_MODE_SUBNET_NZ GENMASK(17, 12)
+#define TOF_ICC_REG_CQS_MODE_SUBNET_SZ GENMASK(11, 6)
+#define TOF_ICC_REG_CQS_MODE_SUBNET_LZ GENMASK(5, 0)
+#define TOF_ICC_REG_CQS_GPID 0x28
+#define TOF_ICC_REG_CQS_TOQ_IPA 0x30
+#define TOF_ICC_REG_CQS_TOQ_SIZE 0x38
+#define TOF_ICC_REG_CQS_TCQ_IPA 0x40
+#define TOF_ICC_REG_CQS_TCQ_IPA_CACHE_INJECTION BIT(63)
+#define TOF_ICC_REG_CQS_MRQ_IPA 0x48
+#define TOF_ICC_REG_CQS_MRQ_IPA_CACHE_INJECTION BIT(63)
+#define TOF_ICC_REG_CQS_MRQ_SIZE 0x50
+#define TOF_ICC_REG_CQS_MRQ_MASK 0x58
+#define TOF_ICC_REG_CQS_TCQ_DESCRIPTOR_COALESCING_TIMER 0x60
+#define TOF_ICC_REG_CQS_MRQ_DESCRIPTOR_COALESCING_TIMER 0x68
+#define TOF_ICC_REG_CQS_MRQ_INTERRUPT_COALESCING_TIMER 0x70
+#define TOF_ICC_REG_CQS_MRQ_INTERRUPT_COALESCING_COUNT 0x78
+#define TOF_ICC_REG_CQS_TOQ_DIRECT_SOURCE_COUNT 0x80
+#define TOF_ICC_REG_CQS_TOQ_DIRECT_DESCRIPTOR_COUNT 0x88
+#define TOF_ICC_REG_CQS_MEMORY_BLOCK_TABLE_ENABLE 0x90
+#define TOF_ICC_REG_CQS_MEMORY_BLOCK_TABLE_IPA 0x98
+#define TOF_ICC_REG_CQS_MEMORY_BLOCK_TABLE_SIZE 0xa0
+#define TOF_ICC_REG_CQS_STEERING_TABLE_ENABLE 0xa8
+#define TOF_ICC_REG_CQS_STEERING_TABLE_IPA 0xb0
+#define TOF_ICC_REG_CQS_STEERING_TABLE_SIZE 0xb8
+#define TOF_ICC_REG_CQS_MRQ_INTERRUPT_MASK 0xc0
+#define TOF_ICC_REG_CQS_IRR 0xc8
+#define TOF_ICC_REG_CQS_IMR 0xd0
+#define TOF_ICC_REG_CQS_IRC 0xd8
+#define TOF_ICC_REG_CQS_IMC 0xe0
+#define TOF_ICC_REG_CQS_ICL 0xe8
+#define TOF_ICC_REG_CQS_DUMP_START 0x0
+#define TOF_ICC_REG_CQS_DUMP_END 0xf0
+
+/* BGS */
+#define TOF_ICC_REG_BGS_PA(tni, bgid) (tof_icc_reg_pa + 0x0000800000 + (tni) * 0x1000000 + (bgid) * 0x10000)
+#define TOF_ICC_REG_BGS_ENABLE 0x0
+#define TOF_ICC_REG_BGS_IRR 0x8
+#define TOF_ICC_REG_BGS_IMR 0x10
+#define TOF_ICC_REG_BGS_IRC 0x18
+#define TOF_ICC_REG_BGS_IMC 0x20
+#define TOF_ICC_REG_BGS_ICL 0x28
+#define TOF_ICC_REG_BGS_STATE 0x30
+#define TOF_ICC_REG_BGS_STATE_ENABLE BIT(0)
+#define TOF_ICC_REG_BGS_EXCEPTION_INFO_GPID_UNMATCH 0x38
+#define TOF_ICC_REG_BGS_EXCEPTION_INFO_GPID_UNMATCH_BG_ADDRESS GENMASK(27, 0)
+#define TOF_ICC_REG_BGS_EXCEPTION_INFO_ADDRESS_UNMATCH 0x40
+#define TOF_ICC_REG_BGS_EXCEPTION_INFO_ADDRESS_UNMATCH_BG_ADDRESS GENMASK(27, 0)
+#define TOF_ICC_REG_BGS_SIGNAL_A 0x48
+#define TOF_ICC_REG_BGS_SIGNAL_A_SIG_RECV BIT(63)
+#define TOF_ICC_REG_BGS_SIGNAL_A_TLP_RECV BIT(62)
+#define TOF_ICC_REG_BGS_SIGNAL_A_SIG_SEND BIT(61)
+#define TOF_ICC_REG_BGS_SIGNAL_A_OP_TYPE GENMASK(3, 0)
+#define TOF_ICC_REG_BGS_SIGNAL_B 0x50
+#define TOF_ICC_REG_BGS_SIGNAL_B_SIG_RECV BIT(63)
+#define TOF_ICC_REG_BGS_SIGNAL_B_TLP_RECV BIT(62)
+#define TOF_ICC_REG_BGS_SIGNAL_B_SIG_SEND BIT(61)
+#define TOF_ICC_REG_BGS_SIGNAL_B_OP_TYPE GENMASK(3, 0)
+#define TOF_ICC_REG_BGS_SIGNAL_MASK 0x58
+#define TOF_ICC_REG_BGS_SIGNAL_MASK_SIG_RECV BIT(63)
+#define TOF_ICC_REG_BGS_SIGNAL_MASK_TLP_RECV BIT(62)
+#define TOF_ICC_REG_BGS_SIGNAL_MASK_SIG_SEND BIT(61)
+#define TOF_ICC_REG_BGS_SIGNAL_MASK_TLP_SEND BIT(60)
+#define TOF_ICC_REG_BGS_LOCAL_LINK 0x60
+#define TOF_ICC_REG_BGS_LOCAL_LINK_BGID_RECV GENMASK(37, 32)
+#define TOF_ICC_REG_BGS_LOCAL_LINK_BGID_SEND GENMASK(5, 0)
+#define TOF_ICC_REG_BGS_REMOTE_LINK 0x68
+#define TOF_ICC_REG_BGS_REMOTE_LINK_BG_ADDRESS_RECV GENMASK(59, 32)
+#define TOF_ICC_REG_BGS_REMOTE_LINK_BG_ADDRESS_SEND GENMASK(31, 0)
+#define TOF_ICC_REG_BGS_SUBNET_SIZE 0x70
+#define TOF_ICC_REG_BGS_GPID_BSEQ 0x78
+#define TOF_ICC_REG_BGS_DATA_A0 0x108
+#define TOF_ICC_REG_BGS_DATA_AE 0x178
+#define TOF_ICC_REG_BGS_DATA_B0 0x188
+#define TOF_ICC_REG_BGS_DATA_BE 0x1f8
+#define TOF_ICC_REG_BGS_BCH_MASK 0x800
+#define TOF_ICC_REG_BGS_BCH_MASK_MASK BIT(63)
+#define TOF_ICC_REG_BGS_BCH_MASK_STATUS 0x808
+#define TOF_ICC_REG_BGS_BCH_MASK_STATUS_RUN BIT(63)
+#define TOF_ICC_REG_BGS_BCH_NOTICE_IPA 0x810
+#define TOF_ICC_REG_BGS_DUMP_START 0x0
+#define TOF_ICC_REG_BGS_DUMP_END 0x818
+
+/* TNI */
+#define TOF_ICC_REG_TNI_PA(tni) (tof_icc_reg_pa + 0x0000c00000 + (tni) * 0x1000000)
+#define TOF_ICC_REG_TNI_IRR 0x8
+#define TOF_ICC_REG_TNI_IMR 0x10
+#define TOF_ICC_REG_TNI_IRC 0x18
+#define TOF_ICC_REG_TNI_IMC 0x20
+#define TOF_ICC_REG_TNI_ICL 0x28
+#define TOF_ICC_REG_TNI_STATE 0x30
+#define TOF_ICC_REG_TNI_STATE_MASK GENMASK(1, 0)
+#define TOF_ICC_REG_TNI_STATE_DISABLE 0
+#define TOF_ICC_REG_TNI_STATE_NORMAL 2
+#define TOF_ICC_REG_TNI_STATE_ERROR 3
+#define TOF_ICC_REG_TNI_ENABLE 0x38
+#define TOF_ICC_REG_TNI_CQ_PRESENT 0x40
+#define TOF_ICC_REG_TNI_EXCEPTION_INFO_INACTIVE_BG 0x48
+#define TOF_ICC_REG_TNI_EXCEPTION_INFO_INACTIVE_BG_DEST_BG GENMASK(37, 32)
+#define TOF_ICC_REG_TNI_EXCEPTION_INFO_INACTIVE_BG_SOURCE_BG_ADDRESS GENMASK(27, 0)
+#define TOF_ICC_REG_TNI_PRQ_FULL_POINTER 0x100
+#define TOF_ICC_REG_TNI_PBQ_PA 0x108
+#define TOF_ICC_REG_TNI_PBQ_SIZE 0x110
+#define TOF_ICC_REG_TNI_PRQ_PA 0x118
+#define TOF_ICC_REG_TNI_PRQ_PA_CACHE_INJECTION BIT(63)
+#define TOF_ICC_REG_TNI_PRQ_SIZE 0x120
+#define TOF_ICC_REG_TNI_PRQ_MASK 0x128
+#define TOF_ICC_REG_TNI_PRQ_ENTRY_COALESCING_TIMER 0x130
+#define TOF_ICC_REG_TNI_PRQ_INTERRUPT_COALESCING_TIMER 0x138
+#define TOF_ICC_REG_TNI_PRQ_INTERRUPT_COALESCING_COUNT 0x140
+#define TOF_ICC_REG_TNI_SEND_COUNT 0x148
+#define TOF_ICC_REG_TNI_NO_SEND_COUNT 0x150
+#define TOF_ICC_REG_TNI_BLOCK_SEND_COUNT 0x158
+#define TOF_ICC_REG_TNI_RECEIVE_COUNT 0x160
+#define TOF_ICC_REG_TNI_NO_RECEIVE_COUNT 0x168
+#define TOF_ICC_REG_TNI_NUM_SEND_TLP 0x170
+#define TOF_ICC_REG_TNI_BYTE_SEND_TLP 0x178
+#define TOF_ICC_REG_TNI_NUM_SEND_SYSTEM_TLP 0x180
+#define TOF_ICC_REG_TNI_NUM_RECEIVE_TLP 0x188
+#define TOF_ICC_REG_TNI_BYTE_RECEIVE_TLP 0x190
+#define TOF_ICC_REG_TNI_NUM_RECEIVE_NULLIFIED_TLP 0x198
+#define TOF_ICC_REG_TNI_RX_NUM_UNKNOWN_TLP 0x1a0
+#define TOF_ICC_REG_TNI_RX_NUM_SYSTEM_TLP 0x1a8
+#define TOF_ICC_REG_TNI_RX_NUM_EXCEPTION_TLP 0x1b0
+#define TOF_ICC_REG_TNI_RX_NUM_DISCARD_UNKNOWN_TLP 0x1b8
+#define TOF_ICC_REG_TNI_RX_NUM_DISCARD_SYSTEM_TLP 0x1c0
+#define TOF_ICC_REG_TNI_RX_NUM_DISCARD_EXCEPTION_TLP 0x1c8
+#define TOF_ICC_REG_TNI_DUMP_START 0x8
+#define TOF_ICC_REG_TNI_DUMP_END 0x1d0
+
+/* Port */
+#define TOF_ICC_REG_PORT_PA(port) (tof_icc_reg_pa + 0x0006000000 + (port) * 0x1000)
+#define TOF_ICC_REG_PORT_TX_VC0_ZERO_CREDIT_COUNT 0x0
+#define TOF_ICC_REG_PORT_TX_VC1_ZERO_CREDIT_COUNT 0x8
+#define TOF_ICC_REG_PORT_TX_VC2_ZERO_CREDIT_COUNT 0x10
+#define TOF_ICC_REG_PORT_TX_VC3_ZERO_CREDIT_COUNT 0x18
+#define TOF_ICC_REG_PORT_FREE_RUN_COUNT 0x80
+#define TOF_ICC_REG_PORT_NUM_SEND_DLLP 0xc0
+#define TOF_ICC_REG_PORT_NUM_SEND_TLP 0xc8
+#define TOF_ICC_REG_PORT_BYTE_SEND_TLP 0xd0
+#define TOF_ICC_REG_PORT_NUM_SEND_SYSTEM_TLP 0xd8
+#define TOF_ICC_REG_PORT_NUM_SEND_NULLIFIED_TLP 0xe0
+#define TOF_ICC_REG_PORT_NUM_TX_DISCARD_SYSTEM_TLP 0xe8
+#define TOF_ICC_REG_PORT_NUM_TX_DISCARD_NORMAL_TLP 0xf0
+#define TOF_ICC_REG_PORT_NUM_TX_FILTERED_NORMAL_TLP 0xf8
+#define TOF_ICC_REG_PORT_NUM_VIRTUAL_CUT_THROUGH_TLP 0x100
+#define TOF_ICC_REG_PORT_NUM_GENERATE_NULLIFIED_TLP 0x108
+#define TOF_ICC_REG_PORT_NUM_RECEIVE_DLLP 0x110
+#define TOF_ICC_REG_PORT_NUM_RECEIVE_TLP 0x118
+#define TOF_ICC_REG_PORT_BYTE_RECEIVE_TLP 0x120
+#define TOF_ICC_REG_PORT_NUM_RECEIVE_SYSTEM_TLP 0x128
+#define TOF_ICC_REG_PORT_NUM_RECEIVE_NULLIFIED_TLP 0x130
+#define TOF_ICC_REG_PORT_NUM_RX_DISCARD_SYSTEM_TLP 0x138
+#define TOF_ICC_REG_PORT_NUM_RX_DISCARD_NORMAL_TLP 0x140
+#define TOF_ICC_REG_PORT_NUM_RX_FILTERED_NORMAL_TLP 0x158
+#define TOF_ICC_REG_PORT_NUM_RX_DISCARD_NULLIFIED_TLP 0x160
+#define TOF_ICC_REG_PORT_FRAME_LCRC_ERROR_COUNT 0x170
+#define TOF_ICC_REG_PORT_TX_RETRY_BUFFER_CE_COUNT 0x180
+#define TOF_ICC_REG_PORT_RX_VC_BUFFER_CE_COUNT 0x188
+#define TOF_ICC_REG_PORT_XB_CE_COUNT 0x190
+#define TOF_ICC_REG_PORT_ACK_NACK_TIME_OUT_COUNT 0x198
+#define TOF_ICC_REG_PORT_SLICE0_FCS_ERROR_COUNT 0x1a0
+#define TOF_ICC_REG_PORT_SLICE1_FCS_ERROR_COUNT 0x1a8
+#define TOF_ICC_REG_PORT_DUMP_START 0x0
+#define TOF_ICC_REG_PORT_DUMP_END 0x1b0
+
+/* XB */
+#define TOF_ICC_REG_XB_PA (tof_icc_reg_pa + 0x000600f000)
+#define TOF_ICC_REG_XB_STQ_ENABLE 0x0
+#define TOF_ICC_REG_XB_STQ_UPDATE_INTERVAL 0x8
+#define TOF_ICC_REG_XB_STQ_PA 0x10
+#define TOF_ICC_REG_XB_STQ_SIZE 0x18
+#define TOF_ICC_REG_XB_STQ_NEXT_OFFSET 0x20
+#define TOF_ICC_REG_XB_DUMP_START 0x0
+#define TOF_ICC_REG_XB_DUMP_END 0x28
+
+#define TOF_ICC_XB_TC_DATA_CYCLE_COUNT(tni) ((tni) * 0x10 + 0x0)
+#define TOF_ICC_XB_TC_WAIT_CYCLE_COUNT(tni) ((tni) * 0x10 + 0x8)
+#define TOF_ICC_XB_TD_DATA_CYCLE_COUNT(tnr) ((tnr) * 0x10 + 0x60)
+#define TOF_ICC_XB_TD_WAIT_CYCLE_COUNT(tnr) ((tnr) * 0x10 + 0x68)
+
+/* Tofu */
+#define TOF_ICC_REG_TOFU_PA (tof_icc_reg_pa + 0x0007000000)
+#define TOF_ICC_REG_TOFU_NODE_ADDRESS 0x0
+#define TOF_ICC_REG_TOFU_NODE_ADDRESS_X GENMASK(22, 18)
+#define TOF_ICC_REG_TOFU_NODE_ADDRESS_Y GENMASK(17, 13)
+#define TOF_ICC_REG_TOFU_NODE_ADDRESS_Z GENMASK(12, 8)
+#define TOF_ICC_REG_TOFU_NODE_ADDRESS_A BIT(7)
+#define TOF_ICC_REG_TOFU_NODE_ADDRESS_B GENMASK(6, 5)
+#define TOF_ICC_REG_TOFU_NODE_ADDRESS_C BIT(4)
+#define TOF_ICC_REG_TOFU_PORT_SETTING 0x8
+#define TOF_ICC_REG_TOFU_TD_TLP_FILTER(tnr) ((tnr) * 0x10 + 0x10)
+#define TOF_ICC_REG_TOFU_TD_SETTINGS(tnr) ((tnr) * 0x10 + 0x18)
+#define TOF_ICC_REG_TOFU_TNR_MSI_BASE 0xc0
+#define TOF_ICC_REG_TOFU_TNR_IRR 0xc8
+#define TOF_ICC_REG_TOFU_TNR_IMR 0xd0
+#define TOF_ICC_REG_TOFU_TNR_IRC 0xd8
+#define TOF_ICC_REG_TOFU_TNR_IMC 0xe0
+#define TOF_ICC_REG_TOFU_TNR_ICL 0xe8
+#define TOF_ICC_REG_TOFU_TNI_VMS(tni, vmsid) ((tni) * 0x100 + (vmsid) * 0x8 + 0x100)
+#define TOF_ICC_REG_TOFU_TNI_VMS_CQ00(tni) ((tni) * 0x100 + 0x180)
+#define TOF_ICC_REG_TOFU_TNI_VMS_BG00(tni) ((tni) * 0x100 + 0x1a0)
+#define TOF_ICC_REG_TOFU_TNI_VMS_BG16(tni) ((tni) * 0x100 + 0x1a8)
+#define TOF_ICC_REG_TOFU_TNI_VMS_BG32(tni) ((tni) * 0x100 + 0x1b0)
+#define TOF_ICC_REG_TOFU_TNI_MSI_BASE(tni) ((tni) * 0x100 + 0x1c0)
+#define TOF_ICC_REG_TOFU_DUMP_START 0x0
+#define TOF_ICC_REG_TOFU_DUMP_END 0x6c8
+
+/** Interrupts **/
+#define TOF_ICC_IRQ_CQS_TOQ_READ_EXCEPTION BIT(0)
+#define TOF_ICC_IRQ_CQS_TOQ_DIRECT_DESCRIPTOR_EXCEPTION BIT(1)
+#define TOF_ICC_IRQ_CQS_TOQ_MARKED_UE BIT(2)
+#define TOF_ICC_IRQ_CQS_TCQ_WRITE_EXCEPTION BIT(3)
+#define TOF_ICC_IRQ_CQS_TOQ_SOURCE_TYPE_EXCEPTION BIT(4)
+#define TOF_ICC_IRQ_CQS_TCQ_WRITE_ACKNOWLEDGE BIT(5)
+#define TOF_ICC_IRQ_CQS_MRQ_WRITE_ACKNOWLEDGE BIT(7)
+#define TOF_ICC_IRQ_CQS_MRQ_WRITE_EXCEPTION BIT(8)
+#define TOF_ICC_IRQ_CQS_MRQ_OVERFLOW BIT(9)
+#define TOF_ICC_IRQ_CQS_STEERING_READ_EXCEPTION BIT(36)
+#define TOF_ICC_IRQ_CQS_MB_READ_EXCEPTION BIT(38)
+#define TOF_ICC_IRQ_CQS_PAYLOAD_READ_EXCEPTION BIT(39)
+#define TOF_ICC_IRQ_CQS_PAYLOAD_WRITE_EXCEPTION BIT(40)
+/* Just for convinience of irr value, no exists CQS CACHEFLUSH_TIMEOUT interrupt  */
+#define TOF_ICC_DUMMY_IRQ_CQS_CACHEFLUSH_TIMEOUT BIT(63)
+
+#define TOF_ICC_IRQ_BGS_NODE_ADDRESS_UNMATCH BIT(0)
+#define TOF_ICC_IRQ_BGS_BG_RECV_ADDRESS_EXCEPTION BIT(1)
+#define TOF_ICC_IRQ_BGS_BG_SEND_ADDRESS_EXCEPTION BIT(2)
+#define TOF_ICC_IRQ_BGS_GPID_UNMATCH BIT(3)
+#define TOF_ICC_IRQ_BGS_BSEQ_UNMATCH BIT(4)
+#define TOF_ICC_IRQ_BGS_SIGNAL_STATE_ERROR BIT(5)
+#define TOF_ICC_IRQ_BGS_SYNCHRONIZATION_ACKNOWLEDGE BIT(24)
+#define TOF_ICC_IRQ_BGS_ERROR_SYNCHRONIZATION_ACKNOWLEDGE BIT(25)
+#define TOF_ICC_IRQ_BGS_DMA_COMPLETION_EXCEPTION BIT(26)
+
+#define TOF_ICC_IRQ_TNI_PBQ_READ_EXCEPTION BIT(0)
+#define TOF_ICC_IRQ_TNI_PBQ_MARKED_UE BIT(1)
+#define TOF_ICC_IRQ_TNI_PBQ_UNDERFLOW BIT(2)
+#define TOF_ICC_IRQ_TNI_PRQ_PACKET_DISCARD BIT(3)
+#define TOF_ICC_IRQ_TNI_PRQ_WRITE_ACKNOWLEDGE BIT(4)
+#define TOF_ICC_IRQ_TNI_PRQ_WRITE_EXCEPTION BIT(5)
+#define TOF_ICC_IRQ_TNI_PRQ_OVERFLOW BIT(6)
+#define TOF_ICC_IRQ_TNI_INACTIVE_BG BIT(16)
+#define TOF_ICC_IRQ_TNI_STAGE2_TRANSLATION_FAULT BIT(32)
+
+#define TOF_ICC_IRQ_TNR_TNR0_RX_FILTER_OUT BIT(0)
+#define TOF_ICC_IRQ_TNR_TNR0_TX_FILTER_OUT BIT(1)
+#define TOF_ICC_IRQ_TNR_TNR0_PORT_ERROR BIT(2)
+#define TOF_ICC_IRQ_TNR_TNR0_DATELINE_ERROR BIT(3)
+#define TOF_ICC_IRQ_TNR_TNR0_ROUTING_ERROR BIT(4)
+#define TOF_ICC_IRQ_TNR_TNR1_RX_FILTER_OUT BIT(6)
+#define TOF_ICC_IRQ_TNR_TNR1_TX_FILTER_OUT BIT(7)
+#define TOF_ICC_IRQ_TNR_TNR1_PORT_ERROR BIT(8)
+#define TOF_ICC_IRQ_TNR_TNR1_DATELINE_ERROR BIT(9)
+#define TOF_ICC_IRQ_TNR_TNR1_ROUTING_ERROR BIT(10)
+#define TOF_ICC_IRQ_TNR_TNR2_RX_FILTER_OUT BIT(12)
+#define TOF_ICC_IRQ_TNR_TNR2_TX_FILTER_OUT BIT(13)
+#define TOF_ICC_IRQ_TNR_TNR2_PORT_ERROR BIT(14)
+#define TOF_ICC_IRQ_TNR_TNR2_DATELINE_ERROR BIT(15)
+#define TOF_ICC_IRQ_TNR_TNR2_ROUTING_ERROR BIT(16)
+#define TOF_ICC_IRQ_TNR_TNR3_RX_FILTER_OUT BIT(18)
+#define TOF_ICC_IRQ_TNR_TNR3_TX_FILTER_OUT BIT(19)
+#define TOF_ICC_IRQ_TNR_TNR3_PORT_ERROR BIT(20)
+#define TOF_ICC_IRQ_TNR_TNR3_DATELINE_ERROR BIT(21)
+#define TOF_ICC_IRQ_TNR_TNR3_ROUTING_ERROR BIT(22)
+#define TOF_ICC_IRQ_TNR_TNR4_RX_FILTER_OUT BIT(24)
+#define TOF_ICC_IRQ_TNR_TNR4_TX_FILTER_OUT BIT(25)
+#define TOF_ICC_IRQ_TNR_TNR4_PORT_ERROR BIT(26)
+#define TOF_ICC_IRQ_TNR_TNR4_DATELINE_ERROR BIT(27)
+#define TOF_ICC_IRQ_TNR_TNR4_ROUTING_ERROR BIT(28)
+#define TOF_ICC_IRQ_TNR_TNR5_RX_FILTER_OUT BIT(30)
+#define TOF_ICC_IRQ_TNR_TNR5_TX_FILTER_OUT BIT(31)
+#define TOF_ICC_IRQ_TNR_TNR5_PORT_ERROR BIT(32)
+#define TOF_ICC_IRQ_TNR_TNR5_DATELINE_ERROR BIT(33)
+#define TOF_ICC_IRQ_TNR_TNR5_ROUTING_ERROR BIT(34)
+#define TOF_ICC_IRQ_TNR_TNR6_RX_FILTER_OUT BIT(36)
+#define TOF_ICC_IRQ_TNR_TNR6_TX_FILTER_OUT BIT(37)
+#define TOF_ICC_IRQ_TNR_TNR6_PORT_ERROR BIT(38)
+#define TOF_ICC_IRQ_TNR_TNR6_DATELINE_ERROR BIT(39)
+#define TOF_ICC_IRQ_TNR_TNR6_ROUTING_ERROR BIT(40)
+#define TOF_ICC_IRQ_TNR_TNR7_RX_FILTER_OUT BIT(42)
+#define TOF_ICC_IRQ_TNR_TNR7_TX_FILTER_OUT BIT(43)
+#define TOF_ICC_IRQ_TNR_TNR7_PORT_ERROR BIT(44)
+#define TOF_ICC_IRQ_TNR_TNR7_DATELINE_ERROR BIT(45)
+#define TOF_ICC_IRQ_TNR_TNR7_ROUTING_ERROR BIT(46)
+#define TOF_ICC_IRQ_TNR_TNR8_RX_FILTER_OUT BIT(48)
+#define TOF_ICC_IRQ_TNR_TNR8_TX_FILTER_OUT BIT(49)
+#define TOF_ICC_IRQ_TNR_TNR8_PORT_ERROR BIT(50)
+#define TOF_ICC_IRQ_TNR_TNR8_DATELINE_ERROR BIT(51)
+#define TOF_ICC_IRQ_TNR_TNR8_ROUTING_ERROR BIT(52)
+#define TOF_ICC_IRQ_TNR_TNR9_RX_FILTER_OUT BIT(54)
+#define TOF_ICC_IRQ_TNR_TNR9_TX_FILTER_OUT BIT(55)
+#define TOF_ICC_IRQ_TNR_TNR9_PORT_ERROR BIT(56)
+#define TOF_ICC_IRQ_TNR_TNR9_DATELINE_ERROR BIT(57)
+#define TOF_ICC_IRQ_TNR_TNR9_ROUTING_ERROR BIT(58)
+
+#endif
+
+/* vim: set noet ts=8 sw=8 sts=0 tw=0 : */
+

kernel/include/tofu/tof_uapi.h

@@ -0,0 +1,319 @@
+#ifndef _TOF_UAPI_H_
+#define _TOF_UAPI_H_
+
+#include <ihk/types.h>
+#include <arch-memory.h>
+
+enum tof_sig_errno_cq {
+	TOF_TOQ_DIRECT_DESCRIPTOR_EXCEPTION,
+	TOF_TOQ_SOURCE_TYPE_EXCEPTION,
+	TOF_MRQ_OVERFLOW,
+	TOF_CQS_CACHEFLUSH_TIMEOUT,
+};
+
+enum tof_sig_errno_bg {
+	TOF_NODE_ADDRESS_UNMATCH,
+	TOF_BSEQ_UNMATCH,
+	TOF_SIGNAL_STATE_ERROR,
+	TOF_ERROR_SYNCHRONIZATION_ACKNOWLEDGE,
+};
+
+#define TOF_UAPI_VERSION 0x2a00
+
+struct tof_init_cq {
+	uint16_t version;
+	uint8_t session_mode;
+	uint8_t toq_size;
+	uint8_t mrq_size;
+	uint8_t num_stag;
+	uint8_t tcq_cinj;
+	uint8_t mrq_cinj;
+	void *toq_mem;
+	void *tcq_mem;
+	void *mrq_mem;
+};
+
+struct tof_alloc_stag {
+	uint32_t flags;
+	int stag;
+	uint64_t offset;
+	void *va;
+	uint64_t len;
+};
+
+struct tof_free_stags {
+	uint16_t num;
+	int *stags;
+};
+
+struct tof_addr {
+	uint8_t pa;
+	uint8_t pb;
+	uint8_t pc;
+	uint8_t x;
+	uint8_t y;
+	uint8_t z;
+	uint8_t a;
+	uint8_t b;
+	uint8_t c;
+};
+
+struct tof_set_bg {
+	int tni;
+	int gate;
+	int source_lgate;
+	struct tof_addr source_raddr;
+	int source_rtni;
+	int source_rgate;
+	int dest_lgate;
+	struct tof_addr dest_raddr;
+	int dest_rtni;
+	int dest_rgate;
+};
+
+struct tof_enable_bch {
+	void *addr;
+	int bseq;
+	int num;
+	struct tof_set_bg *bgs;
+};
+
+struct tof_set_subnet {
+	int res0;
+	int res1;
+	uint8_t nx;
+	uint8_t sx;
+	uint8_t lx;
+	uint8_t ny;
+	uint8_t sy;
+	uint8_t ly;
+	uint8_t nz;
+	uint8_t sz;
+	uint8_t lz;
+};
+
+struct tof_reg_user {
+	uid_t uid;
+	uint32_t gpid;
+	struct tof_set_subnet subnet;
+	uint64_t *cqmask;
+	uint64_t *bgmask;
+};
+
+struct tof_notify_linkdown {
+	int num;
+	struct {
+		uint8_t x;
+		uint8_t y;
+		uint8_t z;
+		uint8_t a;
+		uint8_t b;
+		uint8_t c;
+		uint16_t ports;
+	} *items;
+};
+
+struct tof_get_port_stat {
+	int port_no;
+	uint64_t mask;
+	uint64_t pa[31];
+};
+
+struct tof_get_cq_stat {
+	int tni;
+	int cqid;
+	uint64_t txbyte;
+	uint64_t rxbyte;
+};
+
+struct tof_load_register {
+	uint64_t pa;
+	uint64_t len;
+	void *buf;
+};
+
+struct tof_load_resource {
+	uint64_t rsc_id;
+	uint64_t offset;
+	uint64_t len;
+	void *buf;
+};
+
+union tof_trans_table_bitfield {
+	struct {
+		uint64_t start:36;
+		uint64_t len:27;
+		uint64_t ps_code:1;
+	} bits;
+	uint64_t atomic;
+};
+
+struct tof_trans_table {
+	union tof_trans_table_bitfield steering;
+	union tof_trans_table_bitfield mbpt;
+};
+
+void tof_utofu_set_linkdown_callback(void (*callback)(int, const void *));
+void tof_utofu_unset_linkdown_callback(void);
+
+#define TOF_MMAP_CQ_REGISTER 0
+#define TOF_MMAP_CQ_TRANSTABLE (PAGE_SIZE)
+#define TOF_MMAP_BCH_REGISTER 0
+#define TOF_MMAP_XB_STQ 0
+
+#define TOF_ST_RDWR 0x0
+#define TOF_ST_RDONLY 0x1
+#define TOF_ST_LPG 0x2
+
+#define TOF_STAG_TRANS_PS_CODE_64KB 0
+#define TOF_STAG_TRANS_PS_CODE_2MB  1
+
+#define TOF_IOC_MAGIC 'd'
+#define TOF_IOCTL_INIT_CQ _IOWR(TOF_IOC_MAGIC, 0, long)
+#define TOF_IOCTL_ALLOC_STAG _IOWR(TOF_IOC_MAGIC, 1, long)
+#define TOF_IOCTL_FREE_STAGS _IOWR(TOF_IOC_MAGIC, 2, long)
+#define TOF_IOCTL_ENABLE_BCH _IOWR(TOF_IOC_MAGIC, 3, long)
+#define TOF_IOCTL_DISABLE_BCH _IOWR(TOF_IOC_MAGIC, 4, long)
+#define TOF_IOCTL_SET_RT_SIGNAL _IOWR(TOF_IOC_MAGIC, 5, long)
+#define TOF_IOCTL_SET_SUBNET _IOWR(TOF_IOC_MAGIC, 6, long)
+#define TOF_IOCTL_REG_USER _IOWR(TOF_IOC_MAGIC, 7, long)
+#define TOF_IOCTL_NOTIFY_LINKDOWN _IOWR(TOF_IOC_MAGIC, 8, long)
+#define TOF_IOCTL_GET_PORT_STAT _IOWR(TOF_IOC_MAGIC, 9, long)
+#define TOF_IOCTL_GET_CQ_STAT _IOWR(TOF_IOC_MAGIC, 10, long)
+#define TOF_IOCTL_LOAD_REGISTER _IOWR(TOF_IOC_MAGIC, 11, long)
+#define TOF_IOCTL_LOAD_RESOURCE _IOWR(TOF_IOC_MAGIC, 12, long)
+
+enum {
+	/* TOQ (0 - 71) */
+	TOF_RSC_TNI0_TOQ0 = 0, TOF_RSC_TNI0_TOQ1, TOF_RSC_TNI0_TOQ2, TOF_RSC_TNI0_TOQ3,
+	TOF_RSC_TNI0_TOQ4, TOF_RSC_TNI0_TOQ5, TOF_RSC_TNI0_TOQ6, TOF_RSC_TNI0_TOQ7,
+	TOF_RSC_TNI0_TOQ8, TOF_RSC_TNI0_TOQ9, TOF_RSC_TNI0_TOQ10, TOF_RSC_TNI0_TOQ11,
+	TOF_RSC_TNI1_TOQ0, TOF_RSC_TNI1_TOQ1, TOF_RSC_TNI1_TOQ2, TOF_RSC_TNI1_TOQ3,
+	TOF_RSC_TNI1_TOQ4, TOF_RSC_TNI1_TOQ5, TOF_RSC_TNI1_TOQ6, TOF_RSC_TNI1_TOQ7,
+	TOF_RSC_TNI1_TOQ8, TOF_RSC_TNI1_TOQ9, TOF_RSC_TNI1_TOQ10, TOF_RSC_TNI1_TOQ11,
+	TOF_RSC_TNI2_TOQ0, TOF_RSC_TNI2_TOQ1, TOF_RSC_TNI2_TOQ2, TOF_RSC_TNI2_TOQ3,
+	TOF_RSC_TNI2_TOQ4, TOF_RSC_TNI2_TOQ5, TOF_RSC_TNI2_TOQ6, TOF_RSC_TNI2_TOQ7,
+	TOF_RSC_TNI2_TOQ8, TOF_RSC_TNI2_TOQ9, TOF_RSC_TNI2_TOQ10, TOF_RSC_TNI2_TOQ11,
+	TOF_RSC_TNI3_TOQ0, TOF_RSC_TNI3_TOQ1, TOF_RSC_TNI3_TOQ2, TOF_RSC_TNI3_TOQ3,
+	TOF_RSC_TNI3_TOQ4, TOF_RSC_TNI3_TOQ5, TOF_RSC_TNI3_TOQ6, TOF_RSC_TNI3_TOQ7,
+	TOF_RSC_TNI3_TOQ8, TOF_RSC_TNI3_TOQ9, TOF_RSC_TNI3_TOQ10, TOF_RSC_TNI3_TOQ11,
+	TOF_RSC_TNI4_TOQ0, TOF_RSC_TNI4_TOQ1, TOF_RSC_TNI4_TOQ2, TOF_RSC_TNI4_TOQ3,
+	TOF_RSC_TNI4_TOQ4, TOF_RSC_TNI4_TOQ5, TOF_RSC_TNI4_TOQ6, TOF_RSC_TNI4_TOQ7,
+	TOF_RSC_TNI4_TOQ8, TOF_RSC_TNI4_TOQ9, TOF_RSC_TNI4_TOQ10, TOF_RSC_TNI4_TOQ11,
+	TOF_RSC_TNI5_TOQ0, TOF_RSC_TNI5_TOQ1, TOF_RSC_TNI5_TOQ2, TOF_RSC_TNI5_TOQ3,
+	TOF_RSC_TNI5_TOQ4, TOF_RSC_TNI5_TOQ5, TOF_RSC_TNI5_TOQ6, TOF_RSC_TNI5_TOQ7,
+	TOF_RSC_TNI5_TOQ8, TOF_RSC_TNI5_TOQ9, TOF_RSC_TNI5_TOQ10, TOF_RSC_TNI5_TOQ11,
+
+	/* TOQ (72 - 143) */
+	TOF_RSC_TNI0_TCQ0, TOF_RSC_TNI0_TCQ1, TOF_RSC_TNI0_TCQ2, TOF_RSC_TNI0_TCQ3,
+	TOF_RSC_TNI0_TCQ4, TOF_RSC_TNI0_TCQ5, TOF_RSC_TNI0_TCQ6, TOF_RSC_TNI0_TCQ7,
+	TOF_RSC_TNI0_TCQ8, TOF_RSC_TNI0_TCQ9, TOF_RSC_TNI0_TCQ10, TOF_RSC_TNI0_TCQ11,
+	TOF_RSC_TNI1_TCQ0, TOF_RSC_TNI1_TCQ1, TOF_RSC_TNI1_TCQ2, TOF_RSC_TNI1_TCQ3,
+	TOF_RSC_TNI1_TCQ4, TOF_RSC_TNI1_TCQ5, TOF_RSC_TNI1_TCQ6, TOF_RSC_TNI1_TCQ7,
+	TOF_RSC_TNI1_TCQ8, TOF_RSC_TNI1_TCQ9, TOF_RSC_TNI1_TCQ10, TOF_RSC_TNI1_TCQ11,
+	TOF_RSC_TNI2_TCQ0, TOF_RSC_TNI2_TCQ1, TOF_RSC_TNI2_TCQ2, TOF_RSC_TNI2_TCQ3,
+	TOF_RSC_TNI2_TCQ4, TOF_RSC_TNI2_TCQ5, TOF_RSC_TNI2_TCQ6, TOF_RSC_TNI2_TCQ7,
+	TOF_RSC_TNI2_TCQ8, TOF_RSC_TNI2_TCQ9, TOF_RSC_TNI2_TCQ10, TOF_RSC_TNI2_TCQ11,
+	TOF_RSC_TNI3_TCQ0, TOF_RSC_TNI3_TCQ1, TOF_RSC_TNI3_TCQ2, TOF_RSC_TNI3_TCQ3,
+	TOF_RSC_TNI3_TCQ4, TOF_RSC_TNI3_TCQ5, TOF_RSC_TNI3_TCQ6, TOF_RSC_TNI3_TCQ7,
+	TOF_RSC_TNI3_TCQ8, TOF_RSC_TNI3_TCQ9, TOF_RSC_TNI3_TCQ10, TOF_RSC_TNI3_TCQ11,
+	TOF_RSC_TNI4_TCQ0, TOF_RSC_TNI4_TCQ1, TOF_RSC_TNI4_TCQ2, TOF_RSC_TNI4_TCQ3,
+	TOF_RSC_TNI4_TCQ4, TOF_RSC_TNI4_TCQ5, TOF_RSC_TNI4_TCQ6, TOF_RSC_TNI4_TCQ7,
+	TOF_RSC_TNI4_TCQ8, TOF_RSC_TNI4_TCQ9, TOF_RSC_TNI4_TCQ10, TOF_RSC_TNI4_TCQ11,
+	TOF_RSC_TNI5_TCQ0, TOF_RSC_TNI5_TCQ1, TOF_RSC_TNI5_TCQ2, TOF_RSC_TNI5_TCQ3,
+	TOF_RSC_TNI5_TCQ4, TOF_RSC_TNI5_TCQ5, TOF_RSC_TNI5_TCQ6, TOF_RSC_TNI5_TCQ7,
+	TOF_RSC_TNI5_TCQ8, TOF_RSC_TNI5_TCQ9, TOF_RSC_TNI5_TCQ10, TOF_RSC_TNI5_TCQ11,
+
+	/* MRQ (144 - 215) */
+	TOF_RSC_TNI0_MRQ0, TOF_RSC_TNI0_MRQ1, TOF_RSC_TNI0_MRQ2, TOF_RSC_TNI0_MRQ3,
+	TOF_RSC_TNI0_MRQ4, TOF_RSC_TNI0_MRQ5, TOF_RSC_TNI0_MRQ6, TOF_RSC_TNI0_MRQ7,
+	TOF_RSC_TNI0_MRQ8, TOF_RSC_TNI0_MRQ9, TOF_RSC_TNI0_MRQ10, TOF_RSC_TNI0_MRQ11,
+	TOF_RSC_TNI1_MRQ0, TOF_RSC_TNI1_MRQ1, TOF_RSC_TNI1_MRQ2, TOF_RSC_TNI1_MRQ3,
+	TOF_RSC_TNI1_MRQ4, TOF_RSC_TNI1_MRQ5, TOF_RSC_TNI1_MRQ6, TOF_RSC_TNI1_MRQ7,
+	TOF_RSC_TNI1_MRQ8, TOF_RSC_TNI1_MRQ9, TOF_RSC_TNI1_MRQ10, TOF_RSC_TNI1_MRQ11,
+	TOF_RSC_TNI2_MRQ0, TOF_RSC_TNI2_MRQ1, TOF_RSC_TNI2_MRQ2, TOF_RSC_TNI2_MRQ3,
+	TOF_RSC_TNI2_MRQ4, TOF_RSC_TNI2_MRQ5, TOF_RSC_TNI2_MRQ6, TOF_RSC_TNI2_MRQ7,
+	TOF_RSC_TNI2_MRQ8, TOF_RSC_TNI2_MRQ9, TOF_RSC_TNI2_MRQ10, TOF_RSC_TNI2_MRQ11,
+	TOF_RSC_TNI3_MRQ0, TOF_RSC_TNI3_MRQ1, TOF_RSC_TNI3_MRQ2, TOF_RSC_TNI3_MRQ3,
+	TOF_RSC_TNI3_MRQ4, TOF_RSC_TNI3_MRQ5, TOF_RSC_TNI3_MRQ6, TOF_RSC_TNI3_MRQ7,
+	TOF_RSC_TNI3_MRQ8, TOF_RSC_TNI3_MRQ9, TOF_RSC_TNI3_MRQ10, TOF_RSC_TNI3_MRQ11,
+	TOF_RSC_TNI4_MRQ0, TOF_RSC_TNI4_MRQ1, TOF_RSC_TNI4_MRQ2, TOF_RSC_TNI4_MRQ3,
+	TOF_RSC_TNI4_MRQ4, TOF_RSC_TNI4_MRQ5, TOF_RSC_TNI4_MRQ6, TOF_RSC_TNI4_MRQ7,
+	TOF_RSC_TNI4_MRQ8, TOF_RSC_TNI4_MRQ9, TOF_RSC_TNI4_MRQ10, TOF_RSC_TNI4_MRQ11,
+	TOF_RSC_TNI5_MRQ0, TOF_RSC_TNI5_MRQ1, TOF_RSC_TNI5_MRQ2, TOF_RSC_TNI5_MRQ3,
+	TOF_RSC_TNI5_MRQ4, TOF_RSC_TNI5_MRQ5, TOF_RSC_TNI5_MRQ6, TOF_RSC_TNI5_MRQ7,
+	TOF_RSC_TNI5_MRQ8, TOF_RSC_TNI5_MRQ9, TOF_RSC_TNI5_MRQ10, TOF_RSC_TNI5_MRQ11,
+
+	/* PBQ (216 - 221) */
+	TOF_RSC_TNI0_PBQ, TOF_RSC_TNI1_PBQ, TOF_RSC_TNI2_PBQ, TOF_RSC_TNI3_PBQ,
+	TOF_RSC_TNI4_PBQ, TOF_RSC_TNI5_PBQ,
+
+	/* PRQ (222 - 227) */
+	TOF_RSC_TNI0_PRQ, TOF_RSC_TNI1_PRQ, TOF_RSC_TNI2_PRQ, TOF_RSC_TNI3_PRQ,
+	TOF_RSC_TNI4_PRQ, TOF_RSC_TNI5_PRQ,
+
+	/* STEERINGTABLE (228 - 299) */
+	TOF_RSC_TNI0_STEERINGTABLE0, TOF_RSC_TNI0_STEERINGTABLE1, TOF_RSC_TNI0_STEERINGTABLE2,
+	TOF_RSC_TNI0_STEERINGTABLE3, TOF_RSC_TNI0_STEERINGTABLE4, TOF_RSC_TNI0_STEERINGTABLE5,
+	TOF_RSC_TNI0_STEERINGTABLE6, TOF_RSC_TNI0_STEERINGTABLE7, TOF_RSC_TNI0_STEERINGTABLE8,
+	TOF_RSC_TNI0_STEERINGTABLE9, TOF_RSC_TNI0_STEERINGTABLE10, TOF_RSC_TNI0_STEERINGTABLE11,
+	TOF_RSC_TNI1_STEERINGTABLE0, TOF_RSC_TNI1_STEERINGTABLE1, TOF_RSC_TNI1_STEERINGTABLE2,
+	TOF_RSC_TNI1_STEERINGTABLE3, TOF_RSC_TNI1_STEERINGTABLE4, TOF_RSC_TNI1_STEERINGTABLE5,
+	TOF_RSC_TNI1_STEERINGTABLE6, TOF_RSC_TNI1_STEERINGTABLE7, TOF_RSC_TNI1_STEERINGTABLE8,
+	TOF_RSC_TNI1_STEERINGTABLE9, TOF_RSC_TNI1_STEERINGTABLE10, TOF_RSC_TNI1_STEERINGTABLE11,
+	TOF_RSC_TNI2_STEERINGTABLE0, TOF_RSC_TNI2_STEERINGTABLE1, TOF_RSC_TNI2_STEERINGTABLE2,
+	TOF_RSC_TNI2_STEERINGTABLE3, TOF_RSC_TNI2_STEERINGTABLE4, TOF_RSC_TNI2_STEERINGTABLE5,
+	TOF_RSC_TNI2_STEERINGTABLE6, TOF_RSC_TNI2_STEERINGTABLE7, TOF_RSC_TNI2_STEERINGTABLE8,
+	TOF_RSC_TNI2_STEERINGTABLE9, TOF_RSC_TNI2_STEERINGTABLE10, TOF_RSC_TNI2_STEERINGTABLE11,
+	TOF_RSC_TNI3_STEERINGTABLE0, TOF_RSC_TNI3_STEERINGTABLE1, TOF_RSC_TNI3_STEERINGTABLE2,
+	TOF_RSC_TNI3_STEERINGTABLE3, TOF_RSC_TNI3_STEERINGTABLE4, TOF_RSC_TNI3_STEERINGTABLE5,
+	TOF_RSC_TNI3_STEERINGTABLE6, TOF_RSC_TNI3_STEERINGTABLE7, TOF_RSC_TNI3_STEERINGTABLE8,
+        TOF_RSC_TNI3_STEERINGTABLE9, TOF_RSC_TNI3_STEERINGTABLE10, TOF_RSC_TNI3_STEERINGTABLE11,
+        TOF_RSC_TNI4_STEERINGTABLE0, TOF_RSC_TNI4_STEERINGTABLE1, TOF_RSC_TNI4_STEERINGTABLE2,
+        TOF_RSC_TNI4_STEERINGTABLE3, TOF_RSC_TNI4_STEERINGTABLE4, TOF_RSC_TNI4_STEERINGTABLE5,
+        TOF_RSC_TNI4_STEERINGTABLE6, TOF_RSC_TNI4_STEERINGTABLE7, TOF_RSC_TNI4_STEERINGTABLE8,
+        TOF_RSC_TNI4_STEERINGTABLE9, TOF_RSC_TNI4_STEERINGTABLE10, TOF_RSC_TNI4_STEERINGTABLE11,
+        TOF_RSC_TNI5_STEERINGTABLE3, TOF_RSC_TNI5_STEERINGTABLE4, TOF_RSC_TNI5_STEERINGTABLE5,
+        TOF_RSC_TNI5_STEERINGTABLE6, TOF_RSC_TNI5_STEERINGTABLE7, TOF_RSC_TNI5_STEERINGTABLE8,
+        TOF_RSC_TNI5_STEERINGTABLE9, TOF_RSC_TNI5_STEERINGTABLE10, TOF_RSC_TNI5_STEERINGTABLE11,
+
+        /* MBTABLE (300 - 371) */
+        TOF_RSC_TNI0_MBTABLE0, TOF_RSC_TNI0_MBTABLE1, TOF_RSC_TNI0_MBTABLE2,
+        TOF_RSC_TNI0_MBTABLE3, TOF_RSC_TNI0_MBTABLE4, TOF_RSC_TNI0_MBTABLE5,
+        TOF_RSC_TNI0_MBTABLE6, TOF_RSC_TNI0_MBTABLE7, TOF_RSC_TNI0_MBTABLE8,
+        TOF_RSC_TNI0_MBTABLE9, TOF_RSC_TNI0_MBTABLE10, TOF_RSC_TNI0_MBTABLE11,
+        TOF_RSC_TNI1_MBTABLE0, TOF_RSC_TNI1_MBTABLE1, TOF_RSC_TNI1_MBTABLE2,
+        TOF_RSC_TNI1_MBTABLE3, TOF_RSC_TNI1_MBTABLE4, TOF_RSC_TNI1_MBTABLE5,
+        TOF_RSC_TNI1_MBTABLE6, TOF_RSC_TNI1_MBTABLE7, TOF_RSC_TNI1_MBTABLE8,
+        TOF_RSC_TNI1_MBTABLE9, TOF_RSC_TNI1_MBTABLE10, TOF_RSC_TNI1_MBTABLE11,
+        TOF_RSC_TNI2_MBTABLE0, TOF_RSC_TNI2_MBTABLE1, TOF_RSC_TNI2_MBTABLE2,
+        TOF_RSC_TNI2_MBTABLE3, TOF_RSC_TNI2_MBTABLE4, TOF_RSC_TNI2_MBTABLE5,
+        TOF_RSC_TNI2_MBTABLE6, TOF_RSC_TNI2_MBTABLE7, TOF_RSC_TNI2_MBTABLE8,
+        TOF_RSC_TNI2_MBTABLE9, TOF_RSC_TNI2_MBTABLE10, TOF_RSC_TNI2_MBTABLE11,
+        TOF_RSC_TNI3_MBTABLE0, TOF_RSC_TNI3_MBTABLE1, TOF_RSC_TNI3_MBTABLE2,
+        TOF_RSC_TNI3_MBTABLE3, TOF_RSC_TNI3_MBTABLE4, TOF_RSC_TNI3_MBTABLE5,
+        TOF_RSC_TNI3_MBTABLE6, TOF_RSC_TNI3_MBTABLE7, TOF_RSC_TNI3_MBTABLE8,
+        TOF_RSC_TNI3_MBTABLE9, TOF_RSC_TNI3_MBTABLE10, TOF_RSC_TNI3_MBTABLE11,
+        TOF_RSC_TNI4_MBTABLE0, TOF_RSC_TNI4_MBTABLE1, TOF_RSC_TNI4_MBTABLE2,
+        TOF_RSC_TNI4_MBTABLE3, TOF_RSC_TNI4_MBTABLE4, TOF_RSC_TNI4_MBTABLE5,
+        TOF_RSC_TNI4_MBTABLE6, TOF_RSC_TNI4_MBTABLE7, TOF_RSC_TNI4_MBTABLE8,
+        TOF_RSC_TNI4_MBTABLE9, TOF_RSC_TNI4_MBTABLE10, TOF_RSC_TNI4_MBTABLE11,
+        TOF_RSC_TNI5_MBTABLE0, TOF_RSC_TNI5_MBTABLE1, TOF_RSC_TNI5_MBTABLE2,
+        TOF_RSC_TNI5_MBTABLE3, TOF_RSC_TNI5_MBTABLE4, TOF_RSC_TNI5_MBTABLE5,
+        TOF_RSC_TNI5_MBTABLE6, TOF_RSC_TNI5_MBTABLE7, TOF_RSC_TNI5_MBTABLE8,
+        TOF_RSC_TNI5_MBTABLE9, TOF_RSC_TNI5_MBTABLE10, TOF_RSC_TNI5_MBTABLE11,
+
+        TOF_RSC_NUM     /* 372 */
+};
+#define TOF_RSC_TOQ(TNI, CQID)  (TOF_RSC_TNI0_TOQ0 + (TNI * 12) + CQID)
+#define TOF_RSC_TCQ(TNI, CQID)  (TOF_RSC_TNI0_TCQ0 + (TNI * 12) + CQID)
+#define TOF_RSC_MRQ(TNI, CQID)  (TOF_RSC_TNI0_MRQ0 + (TNI * 12) + CQID)
+#define TOF_RSC_PBQ(TNI)        (TOF_RSC_TNI0_PBQ + TNI)
+#define TOF_RSC_PRQ(TNI)        (TOF_RSC_TNI0_PRQ + TNI)
+#define TOF_RSC_STT(TNI, CQID)  (TOF_RSC_TNI0_STEERINGTABLE0 + (TNI * 12) + CQID)
+#define TOF_RSC_MBT(TNI, CQID)  (TOF_RSC_TNI0_MBTABLE0 + (TNI * 12) + CQID)
+
+#endif
+
+/* vim: set noet ts=8 sw=8 sts=0 tw=0 : */
+

kernel/include/tofu/tof_utofu_bg_bch.h

@@ -0,0 +1,6 @@
+	struct {
+		bool enabled;
+		uint64_t bgmask[TOF_ICC_NTNIS];
+		uintptr_t iova;
+		void *kaddr;
+	} bch;

kernel/include/tofu/tof_utofu_cq_trans.h

@@ -0,0 +1,6 @@
+			struct {
+				struct tof_utofu_trans_list *mru;
+				struct tof_trans_table *table;
+				int mruhead;
+				ihk_spinlock_t mru_lock;
+			} trans;

kernel/include/tofu/tofu_generated-tof_core_bg.h

@@ -0,0 +1,29 @@
+struct tof_core_bg {
+	union {
+		char whole_struct[120];
+		struct {
+			char padding0[0];
+			spinlock_t lock;
+		};
+		struct {
+			char padding1[8];
+			#include "tof_core_bg_reg.h"
+		};
+		struct {
+			char padding2[24];
+			struct tof_core_irq irq;
+		};
+		struct {
+			char padding3[88];
+			tof_core_signal_handler sighandler;
+		};
+		struct {
+			char padding4[104];
+			uint64_t subnet;
+		};
+		struct {
+			char padding5[112];
+			uint32_t gpid;
+		};
+	};
+};

kernel/include/tofu/tofu_generated-tof_core_cq.h

@@ -0,0 +1,9 @@
+struct tof_core_cq {
+	union {
+		char whole_struct[264];
+		struct {
+			char padding0[56];
+			#include "tof_core_cq_reg.h"
+		};
+	};
+};

kernel/include/tofu/tofu_generated-tof_utofu_bg.h

@@ -0,0 +1,21 @@
+struct tof_utofu_bg {
+	union {
+		char whole_struct[160];
+		struct {
+			char padding0[0];
+			struct tof_utofu_device common;
+		};
+		struct {
+			char padding1[80];
+			uint8_t tni;
+		};
+		struct {
+			char padding2[81];
+			uint8_t bgid;
+		};
+		struct {
+			char padding3[88];
+			#include "tof_utofu_bg_bch.h"
+		};
+	};
+};

kernel/include/tofu/tofu_generated-tof_utofu_cq.h

@@ -0,0 +1,33 @@
+struct tof_utofu_cq {
+	union {
+		char whole_struct[384];
+		struct {
+			char padding0[0];
+			struct tof_utofu_device common;
+		};
+		struct {
+			char padding1[80];
+			uint8_t tni;
+		};
+		struct {
+			char padding2[81];
+			uint8_t cqid;
+		};
+		struct {
+			char padding3[104];
+			#include "tof_utofu_cq_trans.h"
+		};
+		struct {
+			char padding4[128];
+			struct tof_icc_steering_entry *steering;
+		};
+		struct {
+			char padding5[136];
+			struct tof_icc_mb_entry *mb;
+		};
+		struct {
+			char padding6[186];
+			uint8_t num_stag;
+		};
+	};
+};

kernel/include/tofu/tofu_generated-tof_utofu_device.h

@@ -0,0 +1,17 @@
+struct tof_utofu_device {
+	union {
+		char whole_struct[80];
+		struct {
+			char padding0[0];
+			bool enabled;
+		};
+		struct {
+			char padding1[12];
+			uint32_t gpid;
+		};
+		struct {
+			char padding2[24];
+			uint64_t subnet;
+		};
+	};
+};

kernel/include/tofu/tofu_generated-tof_utofu_mbpt.h

@@ -0,0 +1,33 @@
+struct tof_utofu_mbpt {
+	union {
+		char whole_struct[56];
+		struct {
+			char padding0[0];
+			struct kref kref;
+		};
+		struct {
+			char padding1[8];
+			struct tof_utofu_cq *ucq;
+		};
+		struct {
+			char padding2[16];
+			uintptr_t iova;
+		};
+		struct {
+			char padding3[24];
+			struct scatterlist *sg;
+		};
+		struct {
+			char padding4[32];
+			size_t nsgents;
+		};
+		struct {
+			char padding5[40];
+			uintptr_t mbptstart;
+		};
+		struct {
+			char padding6[48];
+			size_t pgsz;
+		};
+	};
+};

kernel/include/tofu/tofu_stag_range.h

@@ -0,0 +1,15 @@
+#ifndef __TOFU_STAG_RANGE_HEADER__
+#define __TOFU_STAG_RANGE_HEADER__
+
+struct tof_utofu_cq;
+
+struct tofu_stag_range {
+	uintptr_t start, end;
+	int stag;
+	struct tof_utofu_cq *ucq;
+	struct list_head list; // per-vm_range list
+	struct list_head hash; // per-process stag hash
+};
+
+#endif // __TOFU_STAG_RANGE_HEADER__
+

kernel/init.c

@@ -345,6 +345,9 @@ static void populate_sysfs(void)
 
 int host_ikc_inited = 0;
 extern int num_processors;
+#ifdef ENABLE_TOFU
+extern void tof_utofu_init_globals(void);
+#endif
 
 static void post_init(void)
 {
@@ -370,6 +373,9 @@ static void post_init(void)
 
 	sysfs_init();
 	populate_sysfs();
+#ifdef ENABLE_TOFU
+	tof_utofu_init_globals();
+#endif
 }
 #ifdef DCFA_RUN
 extern void user_main();

kernel/mem.c

@@ -44,6 +44,7 @@
 #include <limits.h>
 #include <sysfs.h>
 #include <ihk/debug.h>
+#include <llist.h>
 #include <bootparam.h>
 
 //#define DEBUG_PRINT_MEM
@@ -62,6 +63,9 @@ extern int interrupt_from_user(void *);
 struct tlb_flush_entry tlb_flush_vector[IHK_TLB_FLUSH_IRQ_VECTOR_SIZE];
 
 int anon_on_demand = 0;
+#ifdef ENABLE_FUGAKU_HACKS
+int hugetlbfs_on_demand;
+#endif
 int sysctl_overcommit_memory = OVERCOMMIT_ALWAYS;
 
 static struct ihk_mc_pa_ops *pa_ops;
@@ -741,6 +745,19 @@ distance_based:
 						memory_nodes[node].nodes_by_distance[i].id);
 				break;
 			}
+			else {
+			if (i == 0)
+#ifndef ENABLE_FUGAKU_HACKS
+				kprintf("%s: distance: CPU @ node %d failed to allocate "
+#else
+				dkprintf("%s: distance: CPU @ node %d failed to allocate "
+#endif
+						"%d pages from node %d\n",
+						__FUNCTION__,
+						ihk_mc_get_numa_id(),
+						npages,
+						memory_nodes[node].nodes_by_distance[i].id);
+			}
 		}
 
 		if (pa) break;
@@ -792,6 +809,27 @@ order_based:
 	return NULL;
 }
 
+/*
+ * Get NUMA node structure offsetted by index in the order of distance
+ */
+struct ihk_mc_numa_node *ihk_mc_get_numa_node_by_distance(int i)
+{
+	int numa_id;
+
+	if (!cpu_local_var_initialized)
+		return NULL;
+
+	if (i < 0 || i > ihk_mc_get_nr_numa_nodes()) {
+		return NULL;
+	}
+
+	numa_id = ihk_mc_get_numa_id();
+	if (!memory_nodes[numa_id].nodes_by_distance)
+		return NULL;
+
+	return &memory_nodes[memory_nodes[numa_id].nodes_by_distance[i].id];
+}
+
 static void __mckernel_free_pages_in_allocator(void *va, int npages,
                                                int is_user)
 {
@@ -920,6 +958,9 @@ static void query_free_mem_interrupt_handler(void *priv)
 	}
 
 	kprintf("McKernel free pages in total: %d\n", pages);
+#ifdef ENABLE_FUGAKU_HACKS
+	panic("PANIC");
+#endif
 
 	if (find_command_line("memdebug")) {
 		extern void kmalloc_memcheck(void);
@@ -1255,6 +1296,9 @@ void tlb_flush_handler(int vector)
 	}
 #endif // PROFILE_ENABLE
 }
+#ifdef ENABLE_FUGAKU_HACKS
+extern unsigned long arch_get_instruction_address(const void *reg);
+#endif
 
 static void unhandled_page_fault(struct thread *thread, void *fault_addr,
 				 uint64_t reason, void *regs)
@@ -1286,13 +1330,36 @@ static void unhandled_page_fault(struct thread *thread, void *fault_addr,
 		__kprintf("address is out of range!\n");
 	}
 
+#ifdef ENABLE_FUGAKU_HACKS
+	{
+		unsigned long pc = arch_get_instruction_address(regs);
+		range = lookup_process_memory_range(vm, pc, pc + 1);
+		if (range) {
+			__kprintf("PC: 0x%lx (%lx in %s)\n",
+					pc,
+					(range->memobj && range->memobj->flags & MF_REG_FILE) ?
+					pc - range->start + range->objoff :
+					pc - range->start,
+					(range->memobj && range->memobj->path) ?
+						range->memobj->path : "(unknown)");
+		}
+	}
+#endif
+
 	kprintf_unlock(irqflags);
 
 	/* TODO */
 	ihk_mc_debug_show_interrupt_context(regs);
 
 	if (!(reason & PF_USER)) {
+		cpu_local_var(kernel_mode_pf_regs) = regs;
+#ifndef ENABLE_FUGAKU_HACKS
 		panic("panic: kernel mode PF");
+#else
+		kprintf("panic: kernel mode PF");
+		for (;;) cpu_pause();
+		//panic("panic: kernel mode PF");
+#endif
 	}
 
 	//dkprintf("now dump a core file\n");
@@ -1312,6 +1379,9 @@ static void unhandled_page_fault(struct thread *thread, void *fault_addr,
 static void page_fault_handler(void *fault_addr, uint64_t reason, void *regs)
 {
 	struct thread *thread = cpu_local_var(current);
+#ifdef ENABLE_TOFU
+	unsigned long addr = (unsigned long)fault_addr;
+#endif
 	int error;
 #ifdef PROFILE_ENABLE
 	uint64_t t_s = 0;
@@ -1325,9 +1395,64 @@ static void page_fault_handler(void *fault_addr, uint64_t reason, void *regs)
 			__FUNCTION__, fault_addr, reason, regs);
 
 	preempt_disable();
+	++cpu_local_var(in_page_fault);
+	if (cpu_local_var(in_page_fault) > 1) {
+		kprintf("%s: PF in PF??\n", __func__);
+		cpu_disable_interrupt();
+		if (!(reason & PF_USER)) {
+			cpu_local_var(kernel_mode_pf_regs) = regs;
+			panic("panic: kernel mode PF in PF");
+		}
+		while (1) {
+			panic("PANIC");
+		}
+	}
 
 	cpu_enable_interrupt();
 
+#ifdef ENABLE_TOFU
+	if (!(reason & PF_USER) &&
+			(addr > 0xffff000000000000 &&
+			 addr < 0xffff800000000000)) {
+		int error;
+		int ihk_mc_linux_pt_virt_to_phys_size(struct page_table *pt,
+				const void *virt,
+				unsigned long *phys,
+				unsigned long *size);
+
+		unsigned long phys, size;
+		enum ihk_mc_pt_attribute attr = PTATTR_WRITABLE | PTATTR_ACTIVE;
+
+		if (ihk_mc_linux_pt_virt_to_phys_size(ihk_mc_get_linux_kernel_pgt(),
+					fault_addr, &phys, &size) < 0) {
+			kprintf("%s: failed to resolve 0x%lx from Linux PT..\n",
+				__func__, addr);
+			goto out_linux;	
+		}
+
+retry_linux:
+		if ((error = ihk_mc_pt_set_page(NULL, fault_addr, phys, attr)) < 0) {
+			if (error == -EBUSY) {
+				kprintf("%s: WARNING: updating 0x%lx -> 0x%lx"
+						" to reflect Linux kernel mapping..\n",
+						__func__, addr, phys);
+				ihk_mc_clear_kernel_range(fault_addr, fault_addr + PAGE_SIZE);
+				goto retry_linux;
+			}
+			else {
+				kprintf("%s: failed to set up 0x%lx -> 0x%lx Linux kernel mapping..\n",
+						__func__, addr, phys);
+				goto out_linux;
+			}
+		}
+
+		dkprintf("%s: Linux kernel mapping 0x%lx -> 0x%lx set\n",
+				__func__, addr, phys);
+		goto out_ok;
+	}
+out_linux:
+#endif
+
 	if ((uintptr_t)fault_addr < PAGE_SIZE || !thread) {
 		error = -EINVAL;
 	} else {
@@ -1348,7 +1473,15 @@ static void page_fault_handler(void *fault_addr, uint64_t reason, void *regs)
 			__func__, thread ? thread->tid : -1, fault_addr,
 			reason, error);
 		unhandled_page_fault(thread, fault_addr, reason, regs);
+		--cpu_local_var(in_page_fault);
 		preempt_enable();
+
+#ifdef ENABLE_FUGAKU_DEBUG
+		kprintf("%s: sending SIGSTOP to TID: %d\n", __func__, thread->tid);
+		do_kill(thread, thread->proc->pid, thread->tid, SIGSTOP, NULL, 0);
+		goto out;
+#endif
+
 		memset(&info, '\0', sizeof info);
 		if (error == -ERANGE) {
 			info.si_signo = SIGBUS;
@@ -1373,7 +1506,11 @@ static void page_fault_handler(void *fault_addr, uint64_t reason, void *regs)
 		goto out;
 	}
 
+#ifdef ENABLE_TOFU
+out_ok:
+#endif
 	error = 0;
+	--cpu_local_var(in_page_fault);
 	preempt_enable();
 out:
 	dkprintf("%s: addr: %p, reason: %lx, regs: %p -> error: %d\n",
@@ -1465,7 +1602,11 @@ static void numa_init(void)
 		INIT_LIST_HEAD(&memory_nodes[i].allocators);
 		memory_nodes[i].nodes_by_distance = 0;
 #ifdef IHK_RBTREE_ALLOCATOR
+		ihk_atomic_set(&memory_nodes[i].zeroing_workers, 0);
+		ihk_atomic_set(&memory_nodes[i].nr_to_zero_pages, 0);
 		memory_nodes[i].free_chunks.rb_node = 0;
+		init_llist_head(&memory_nodes[i].zeroed_list);
+		init_llist_head(&memory_nodes[i].to_zero_list);
 		mcs_lock_init(&memory_nodes[i].lock);
 		memory_nodes[i].min_addr = 0xFFFFFFFFFFFFFFFF;
 		memory_nodes[i].max_addr = 0;
@@ -1956,6 +2097,13 @@ void mem_init(void)
 		anon_on_demand = 1;
 	}
 	
+#ifdef ENABLE_FUGAKU_HACKS
+	if (find_command_line("hugetlbfs_on_demand")) {
+		kprintf("Demand paging on hugetlbfs mappings enabled.\n");
+		hugetlbfs_on_demand = 1;
+	}
+#endif
+
 	/* Init distance vectors */
 	numa_distances_init();
 }
@@ -2712,3 +2860,25 @@ int ihk_mc_get_mem_user_page(void *arg0, page_table_t pt, pte_t *ptep, void *pga
 
 	return 0;
 }
+
+pte_t *ihk_mc_pt_lookup_fault_pte(struct process_vm *vm, void *virt,
+		int pgshift, void **basep, size_t *sizep, int *p2alignp)
+{
+	int faulted = 0;
+	pte_t *ptep;
+
+retry:
+	ptep = ihk_mc_pt_lookup_pte(vm->address_space->page_table,
+			virt, pgshift, basep, sizep, p2alignp);
+	if (!faulted && (!ptep || !pte_is_present(ptep))) {
+		page_fault_process_vm(vm, virt, PF_POPULATE | PF_USER);
+		faulted = 1;
+		goto retry;
+	}
+
+	if (faulted && ptep && pte_is_present(ptep)) {
+		kprintf("%s: successfully faulted 0x%lx\n", __FUNCTION__, virt);
+	}
+
+	return ptep;
+}

kernel/process.c

@@ -36,6 +36,9 @@
 #include <rusage_private.h>
 #include <ihk/monitor.h>
 #include <ihk/debug.h>
+#ifdef ENABLE_TOFU
+#include <tofu/tofu_stag_range.h>
+#endif
 
 //#define DEBUG_PRINT_PROCESS
 
@@ -269,6 +272,12 @@ init_process_vm(struct process *owner, struct address_space *asp, struct process
 	}
 	vm->range_cache_ind = 0;
 
+#ifdef ENABLE_TOFU
+	ihk_mc_spinlock_init(&vm->tofu_stag_lock);
+	for (i = 0; i < TOFU_STAG_HASH_SIZE; ++i) {
+		INIT_LIST_HEAD(&vm->tofu_stag_hash[i]);
+	}
+#endif
 	return 0;
 }
 
@@ -824,6 +833,8 @@ static int copy_user_ranges(struct process_vm *vm, struct process_vm *orgvm)
 		range->objoff = src_range->objoff;
 		range->pgshift = src_range->pgshift;
 		range->private_data = src_range->private_data;
+		range->straight_start = src_range->straight_start;
+
 		if (range->memobj) {
 			memobj_ref(range->memobj);
 		}
@@ -945,11 +956,31 @@ int split_process_memory_range(struct process_vm *vm, struct vm_range *range,
 	}
 
 	newrange->start = addr;
+	newrange->straight_start = 0;
+	if (range->straight_start) {
+		newrange->straight_start =
+			range->straight_start + (addr - range->start);
+	}
 	newrange->end = range->end;
 	newrange->flag = range->flag;
 	newrange->pgshift = range->pgshift;
 	newrange->private_data = range->private_data;
 
+#ifdef ENABLE_TOFU
+	INIT_LIST_HEAD(&newrange->tofu_stag_list);
+	{
+		extern int tofu_stag_split_vm_range_on_addr(struct process_vm *vm,
+				struct vm_range *range_low, struct vm_range *range_high,
+				uintptr_t addr);
+
+		int moved =
+			tofu_stag_split_vm_range_on_addr(vm, range, newrange, addr);
+		if (moved > 0) {
+			kprintf("%s: moved %d stag ranges\n", __func__, moved);
+		}
+	}
+#endif
+
 	if (range->memobj) {
 		memobj_ref(range->memobj);
 		newrange->memobj = range->memobj;
@@ -1018,6 +1049,28 @@ int join_process_memory_range(struct process_vm *vm,
 		if (vm->range_cache[i] == merging)
 			vm->range_cache[i] = surviving;
 	}
+
+#ifdef ENABLE_TOFU
+	/* Move Tofu stag range entries */
+	if (vm->proc->enable_tofu) {
+		struct tofu_stag_range *tsr, *next;
+
+		ihk_mc_spinlock_lock_noirq(&vm->tofu_stag_lock);
+		list_for_each_entry_safe(tsr, next,
+				&merging->tofu_stag_list, list) {
+
+			list_del(&tsr->list);
+			list_add_tail(&tsr->list, &surviving->tofu_stag_list);
+			dkprintf("%s: stag: %d @ %p:%lu moved in VM range merge\n",
+					__func__,
+					tsr->stag,
+					tsr->start,
+					(unsigned long)(tsr->end - tsr->start));
+		}
+		ihk_mc_spinlock_unlock_noirq(&vm->tofu_stag_lock);
+	}
+#endif
+
 	kfree(merging);
 
 	error = 0;
@@ -1045,6 +1098,11 @@ static int free_process_memory_range(struct process_vm *vm,
 
 	start = range->start;
 	end = range->end;
+
+	/* No regular page table manipulation for straight mappings */
+	if (range->straight_start || ((void *)start == vm->proc->straight_va))
+		goto straight_out;
+
 	if (!(range->flag & (VR_REMOTE | VR_IO_NOCACHE | VR_RESERVED))) {
 		neighbor = previous_process_memory_range(vm, range);
 		pgsize = -1;
@@ -1126,11 +1184,57 @@ static int free_process_memory_range(struct process_vm *vm,
 		memobj_unref(range->memobj);
 	}
 
+straight_out:
+
+#ifdef ENABLE_TOFU
+	if (vm->proc->enable_tofu) {
+		int entries;
+		extern int tofu_stag_range_remove_overlapping(struct process_vm *vm,
+				struct vm_range *range);
+
+		entries = tofu_stag_range_remove_overlapping(vm, range);
+		if (entries > 0) {
+			dkprintf("%s: removed %d Tofu stag entries for range 0x%lx:%lu\n",
+				__func__,
+				entries,
+				range->start,
+				range->end - range->start);
+		}
+	}
+#endif
+
 	rb_erase(&range->vm_rb_node, &vm->vm_range_tree);
 	for (i = 0; i < VM_RANGE_CACHE_SIZE; ++i) {
 		if (vm->range_cache[i] == range)
 			vm->range_cache[i] = NULL;
 	}
+
+	/* For straight ranges just free physical memory */
+	if (range->straight_start) {
+		ihk_mc_free_pages(phys_to_virt(vm->proc->straight_pa +
+					(range->straight_start - (unsigned long)vm->proc->straight_va)),
+				(range->end - range->start) >> PAGE_SHIFT);
+
+		dkprintf("%s: straight range 0x%lx @ straight 0x%lx (phys: 0x%lx)"
+				" physical memory freed\n",
+				__FUNCTION__, range->start, range->straight_start,
+				vm->proc->straight_pa +
+				(range->straight_start - (unsigned long)vm->proc->straight_va));
+	}
+	/* For the main straight mapping, free page tables */
+	else if (range->start == (unsigned long)vm->proc->straight_va &&
+			range->end == ((unsigned long)vm->proc->straight_va +
+				vm->proc->straight_len)) {
+		ihk_mc_spinlock_lock_noirq(&vm->page_table_lock);
+		error = ihk_mc_pt_clear_range(vm->address_space->page_table, vm,
+				(void *)start, (void *)end);
+		ihk_mc_spinlock_unlock_noirq(&vm->page_table_lock);
+
+		dkprintf("%s: main straight mapping 0x%lx unmapped\n",
+				__FUNCTION__, vm->proc->straight_va);
+		vm->proc->straight_len = 0;
+	}
+
 	kfree(range);
 
 	dkprintf("free_process_memory_range(%p,%lx-%lx): 0\n",
@@ -1148,6 +1252,50 @@ int remove_process_memory_range(struct process_vm *vm,
 	dkprintf("remove_process_memory_range(%p,%lx,%lx)\n",
 			vm, start, end);
 
+	/*
+	 * Convert to real virtual address for straight ranges,
+	 * but not for the main straight mapping
+	 */
+	if (vm->proc->straight_va &&
+			start >= (unsigned long)vm->proc->straight_va &&
+			end <= ((unsigned long)vm->proc->straight_va +
+				vm->proc->straight_len) &&
+			!(start == (unsigned long)vm->proc->straight_va &&
+				end == ((unsigned long)vm->proc->straight_va +
+					vm->proc->straight_len))) {
+		struct vm_range *range_iter;
+		struct vm_range *range = NULL;
+		unsigned long len = end - start;
+
+		range_iter = lookup_process_memory_range(vm, 0, -1);
+
+		while (range_iter) {
+			if (range_iter->straight_start &&
+					start >= range_iter->straight_start &&
+					start < (range_iter->straight_start +
+						(range_iter->end - range_iter->start))) {
+				range = range_iter;
+				break;
+			}
+
+			range_iter = next_process_memory_range(vm, range_iter);
+		}
+
+		if (!range) {
+			kprintf("%s: WARNING: no straight mapping range found for 0x%lx\n",
+					__FUNCTION__, start);
+			return 0;
+		}
+
+		dkprintf("%s: straight range converted from 0x%lx:%lu -> 0x%lx:%lu\n",
+				__FUNCTION__,
+				start, len,
+				range->start + (start - range->straight_start), len);
+
+		start = range->start + (start - range->straight_start);
+		end = start + len;
+	}
+
 	next = lookup_process_memory_range(vm, start, end);
 	while ((range = next) && range->start < end) {
 		next = next_process_memory_range(vm, range);
@@ -1345,6 +1493,10 @@ int add_process_memory_range(struct process_vm *vm,
 	range->objoff = offset;
 	range->pgshift = pgshift;
 	range->private_data = NULL;
+	range->straight_start = 0;
+#ifdef ENABLE_TOFU
+	INIT_LIST_HEAD(&range->tofu_stag_list);
+#endif
 
 	rc = 0;
 	if (phys == NOPHYS) {
@@ -1385,17 +1537,14 @@ int add_process_memory_range(struct process_vm *vm,
 	/* Clear content! */
 	if (phys != NOPHYS && !(flag & (VR_REMOTE | VR_DEMAND_PAGING))
 			&& ((flag & VR_PROT_MASK) != VR_PROT_NONE)) {
-#if 1
-		memset((void *)phys_to_virt(phys), 0, end - start);
+
+		if (!zero_at_free) {
+#ifdef ARCH_MEMCLEAR
+			memclear((void *)phys_to_virt(phys), end - start);
 #else
-		if (end - start < (1024*1024)) {
-			memset((void*)phys_to_virt(phys), 0, end - start);
-		}
-		else {
-			memset_smp(&cpu_local_var(current)->cpu_set,
-					(void *)phys_to_virt(phys), 0, end - start);
-		}
+			memset((void *)phys_to_virt(phys), 0, end - start);
 #endif
+		}
 	}
 
 	/* Return range object if requested */
@@ -1935,6 +2084,14 @@ static int page_fault_process_memory_range(struct process_vm *vm, struct vm_rang
 	int private_range, patching_to_rdonly;
 	int devfile_or_hugetlbfs_or_premap, regfile_or_shm;
 
+	if (cpu_local_var(current)->profile) {
+		dkprintf("%s: 0x%lx @ %s\n",
+				__func__, fault_addr,
+				range->memobj && range->memobj->path ?
+				range->memobj->path :
+				range->private_data ? "XPMEM" : "<unknown>");
+	}
+
 	dkprintf("page_fault_process_memory_range(%p,%lx-%lx %lx,%lx,%lx)\n", vm, range->start, range->end, range->flag, fault_addr, reason);
 	ihk_mc_spinlock_lock_noirq(&vm->page_table_lock);
 	/*****/
@@ -2433,6 +2590,14 @@ int init_process_stack(struct thread *thread, struct program_load_desc *pn,
 			__FUNCTION__, size, minsz,
 			ap_flag ? "(IHK_MC_AP_USER)" : "");
 
+#ifdef ENABLE_FUGAKU_HACKS
+	/*
+	 * XXX: Fugaku: Fujitsu's runtime remaps the stack
+	 * using hugetlbfs so don't bother allocating too much here..
+	 */
+	minsz = 8*1024*1024;
+#endif
+
 	stack = ihk_mc_alloc_aligned_pages_user(minsz >> PAGE_SHIFT,
 						USER_STACK_PAGE_P2ALIGN,
 						IHK_MC_AP_NOWAIT | ap_flag,
@@ -2774,6 +2939,11 @@ release_process(struct process *proc)
 	/* no process left */
 	mcs_rwlock_reader_lock(&rset->pid1->children_lock, &lock);
 	if (list_empty(&rset->pid1->children_list)) {
+#ifdef ENABLE_TOFU
+		extern void tof_utofu_finalize(void);
+
+		tof_utofu_finalize();
+#endif
 		hugefileobj_cleanup();
 	}
 	mcs_rwlock_reader_unlock(&rset->pid1->children_lock, &lock);
@@ -3122,6 +3292,7 @@ static void idle(void)
 		    v->status == CPU_STATUS_RESERVED) {
 			/* No work to do? Consolidate the kmalloc free list */
 			kmalloc_consolidate_free_list();
+			ihk_numa_zero_free_pages(ihk_mc_get_numa_node_by_distance(0));
 			monitor->status = IHK_OS_MONITOR_IDLE;
 			cpu_local_var(current)->status = PS_INTERRUPTIBLE;
 			cpu_safe_halt();
@@ -3477,6 +3648,7 @@ void spin_sleep_or_schedule(void)
 			break;
 		}
 
+		ihk_numa_zero_free_pages(ihk_mc_get_numa_node_by_distance(0));
 		cpu_pause();
 	}
 

kernel/profile.c

@@ -63,10 +63,21 @@ char *profile_event_names[] =
 	"remote_page_fault",
 	"mpol_alloc_missed",
 	"mmap_anon_contig_phys",
+	"|-------mmap_straight",
+	"|---mmap_not_straight",
 	"mmap_anon_no_contig_phys",
 	"mmap_regular_file",
 	"mmap_device_file",
-	""
+	"tofu_stag_alloc ",
+	"|--new_steering ",
+	"   |-alloc_mbpt ",
+	"   |-update_mbpt",
+	"tofu_stag_free_stags",
+	"tofu_stag_free_stag",
+	"   |--------pre",
+	"   |----cqflush",
+	"   |----dealloc",
+	"      |---free_pages",
 };
 
 mcs_lock_node_t job_profile_lock = { 0 };
@@ -471,7 +482,7 @@ int do_profile(int flag)
 		if (flag & PROF_ON) {
 			if (!thread->profile) {
 				thread->profile = 1;
-				thread->profile_start_ts = 0;
+				thread->profile_start_ts = now_ts;
 			}
 		}
 		else if (flag & PROF_OFF) {

kernel/rbtree.c

@@ -432,6 +432,43 @@ struct rb_node *rb_first(const struct rb_root *root)
 }
 EXPORT_SYMBOL(rb_first);
 
+/*
+ * Pre-order depth first search.
+ * Return a node where __cond is true.
+ */
+static struct rb_node *__rb_preorder_dfs(struct rb_node *n,
+	bool (*__cond)(struct rb_node *, void *arg), void *__cond_arg)
+{
+	struct rb_node *left_res = NULL;
+
+	if (__cond(n, __cond_arg))
+		return n;
+
+	if (n->rb_left) {
+		left_res = __rb_preorder_dfs(n->rb_left, __cond, __cond_arg);
+		if (left_res) {
+			return left_res;
+		}
+	}
+	if (n->rb_right)
+		return __rb_preorder_dfs(n->rb_right, __cond, __cond_arg);
+
+	return NULL;
+}
+
+struct rb_node *rb_preorder_dfs_search(const struct rb_root *root,
+	bool (*__cond)(struct rb_node *, void *arg), void *__cond_arg)
+{
+	struct rb_node *n;
+
+	n = root->rb_node;
+	if (!n)
+		return NULL;
+
+	return __rb_preorder_dfs(n, __cond, __cond_arg);
+}
+
+
 struct rb_node *rb_first_safe(const struct rb_root *root)
 {
 	struct rb_node	*n;

kernel/syscall.c

@@ -204,6 +204,14 @@ long do_syscall(struct syscall_request *req, int cpu)
 		++thread->in_syscall_offload;
 	}
 
+#ifdef ENABLE_FUGAKU_HACKS
+#if 0
+	if (req->number == __NR_write && req->args[0] == 1) {
+		return req->args[2];
+	}
+#endif
+#endif
+
 	/* The current thread is the requester */
 	req->rtid = cpu_local_var(current)->tid;
 
@@ -220,6 +228,9 @@ long do_syscall(struct syscall_request *req, int cpu)
 		req->ttid = 0;
 	}
 	res.req_thread_status = IHK_SCD_REQ_THREAD_SPINNING;
+#ifdef ENABLE_TOFU
+	res.pde_data = NULL;
+#endif
 	send_syscall(req, cpu, &res);
 
 	if (req->rtid == -1) {
@@ -266,6 +277,7 @@ long do_syscall(struct syscall_request *req, int cpu)
 			cpu_restore_interrupt(runq_irqstate);
 
 			if (!do_schedule) {
+				ihk_numa_zero_free_pages(ihk_mc_get_numa_node_by_distance(0));
 				continue;
 			}
 
@@ -379,6 +391,35 @@ long do_syscall(struct syscall_request *req, int cpu)
 
 	rc = res.ret;
 
+#ifdef ENABLE_TOFU
+	if ((req->number == __NR_ioctl && rc == 0) ||
+			(req->number == __NR_openat && rc > 0)) {
+		int fd = req->number == __NR_ioctl ? req->args[0] : rc;
+		char *path = req->number == __NR_ioctl ?
+			thread->proc->fd_path[fd] : thread->fd_path_in_open;
+
+		if (cpu_local_var(current)->proc->enable_tofu &&
+				res.pde_data &&
+				fd < MAX_FD_PDE &&
+				!thread->proc->fd_pde_data[fd] &&
+				!strncmp(path, "/proc/tofu/dev/", 15)) {
+			unsigned long irqstate;
+
+			irqstate = ihk_mc_spinlock_lock(&thread->proc->mckfd_lock);
+			thread->proc->fd_pde_data[fd] = res.pde_data;
+			ihk_mc_spinlock_unlock(&thread->proc->mckfd_lock, irqstate);
+
+			dkprintf("%s: PID: %d, ioctl fd: %d, filename: "
+					"%s, pde_data: 0x%lx\n",
+					__FUNCTION__,
+					thread->proc->pid,
+					fd,
+					path,
+					res.pde_data);
+		}
+	}
+#endif
+
 	if(req->number != __NR_exit_group){
 		--thread->in_syscall_offload;
 	}
@@ -1264,6 +1305,30 @@ void terminate(int rc, int sig)
 	mcs_rwlock_writer_unlock(&proc->threads_lock, &lock);
 	mcs_rwlock_writer_unlock_noirq(&proc->update_lock, &updatelock);
 
+#ifdef ENABLE_TOFU
+	/* Tofu: cleanup, must be done before mcexec is gone */
+	if (proc->enable_tofu) {
+		int fd;
+
+		for (fd = 0; fd < MAX_FD_PDE; ++fd) {
+			/* Tofu? */
+			if (proc->enable_tofu && proc->fd_pde_data[fd]) {
+				extern void tof_utofu_release_fd(struct process *proc, int fd);
+
+				dkprintf("%s: -> tof_utofu_release_fd() @ fd: %d (%s)\n",
+						__func__, fd, proc->fd_path[fd]);
+				tof_utofu_release_fd(proc, fd);
+				proc->fd_pde_data[fd] = NULL;
+			}
+
+			if (proc->fd_path[fd]) {
+				kfree(proc->fd_path[fd]);
+				proc->fd_path[fd] = NULL;
+			}
+		}
+	}
+#endif
+
 	terminate_mcexec(rc, sig);
 
 	mcs_rwlock_writer_lock(&proc->threads_lock, &lock);
@@ -1325,6 +1390,15 @@ void terminate(int rc, int sig)
 	mcs_rwlock_writer_unlock(&proc->threads_lock, &lock);
 
 	vm = proc->vm;
+
+#ifdef ENABLE_TOFU
+	if (proc->enable_tofu) {
+		extern void tof_utofu_finalize();
+
+		tof_utofu_finalize();
+	}
+#endif
+
 	free_all_process_memory_range(vm);
 
 	if (proc->saved_cmdline) {
@@ -1418,7 +1492,6 @@ void terminate(int rc, int sig)
 #endif
 
 	// clean up memory
-
 	finalize_process(proc);
 
 	preempt_disable();
@@ -1433,6 +1506,71 @@ void terminate(int rc, int sig)
 	panic("panic");
 }
 
+int __process_cleanup_fd(struct process *proc, int fd)
+{
+#ifdef ENABLE_TOFU
+	/* Tofu? */
+	if (proc->enable_tofu) {
+		extern void tof_utofu_release_fd(struct process *proc, int fd);
+
+		dkprintf("%s: -> tof_utofu_release_fd() @ fd: %d (%s)\n",
+				__func__, fd, proc->fd_path[fd]);
+		tof_utofu_release_fd(proc, fd);
+		proc->fd_pde_data[fd] = NULL;
+
+		if (proc->fd_path[fd]) {
+			kfree(proc->fd_path[fd]);
+			proc->fd_path[fd] = NULL;
+		}
+	}
+#endif
+	return 0;
+}
+
+int process_cleanup_fd(int pid, int fd)
+{
+	struct process *proc;
+	struct mcs_rwlock_node_irqsave lock;
+
+	proc = find_process(pid, &lock);
+	if (!proc) {
+		/* This is normal behavior */
+		dkprintf("%s: PID %d couldn't be found\n", __func__, pid);
+		return 0;
+	}
+
+	__process_cleanup_fd(proc, fd);
+
+	process_unlock(proc, &lock);
+	return 0;
+}
+
+int process_cleanup_before_terminate(int pid)
+{
+	struct process *proc;
+	struct mcs_rwlock_node_irqsave lock;
+#ifdef ENABLE_TOFU
+	int fd;
+#endif
+
+	proc = find_process(pid, &lock);
+	if (!proc) {
+		/* This is normal behavior */
+		return 0;
+	}
+
+#ifdef ENABLE_TOFU
+	/* Clean up PDE file descriptors */
+	for (fd = 2; fd < MAX_FD_PDE; ++fd) {
+		__process_cleanup_fd(proc, fd);
+	}
+#endif
+
+	process_unlock(proc, &lock);
+	return 0;
+}
+
+
 void
 terminate_host(int pid, struct thread *thread)
 {
@@ -1581,10 +1719,20 @@ int do_munmap(void *addr, size_t len, int holding_memory_range_lock)
 {
 	int error;
 	int ro_freed;
+	struct thread *thread = cpu_local_var(current);
 
 	begin_free_pages_pending();
 	error = remove_process_memory_range(cpu_local_var(current)->vm,
 			(intptr_t)addr, (intptr_t)addr+len, &ro_freed);
+
+	/* No host involvement for straight mapping ranges */
+	if (thread->proc->straight_va &&
+			addr >= thread->proc->straight_va &&
+			(addr + len) <=
+			(thread->proc->straight_va + thread->proc->straight_len)) {
+		goto out;
+	}
+
 	if (error || !ro_freed) {
 		clear_host_pte((uintptr_t)addr, len, holding_memory_range_lock);
 	}
@@ -1595,6 +1743,8 @@ int do_munmap(void *addr, size_t len, int holding_memory_range_lock)
 			/* through */
 		}
 	}
+
+out:
 	finish_free_pages_pending();
 
 	dkprintf("%s: 0x%lx:%lu, error: %ld\n",
@@ -1666,6 +1816,7 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot,
 	void *p = NULL;
 	int vrflags;
 	uintptr_t phys;
+	intptr_t straight_phys;
 	struct memobj *memobj = NULL;
 	int maxprot;
 	int denied;
@@ -1707,6 +1858,212 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot,
 
 	flush_nfo_tlb();
 
+	/* Initialize straight large memory mapping */
+	if (proc->straight_map && !proc->straight_va) {
+		unsigned long straight_pa_start = 0xFFFFFFFFFFFFFFFF;
+		unsigned long straight_pa_end = 0;
+		int i;
+		int p2align = PAGE_P2ALIGN;
+		size_t psize = PAGE_SIZE;
+		unsigned long vrflags;
+		enum ihk_mc_pt_attribute ptattr;
+		struct vm_range *range;
+
+		vrflags = PROT_TO_VR_FLAG(PROT_READ | PROT_WRITE);
+		vrflags |= VRFLAG_PROT_TO_MAXPROT(vrflags);
+		vrflags |= VR_DEMAND_PAGING;
+
+		for (i = 0; i < ihk_mc_get_nr_memory_chunks(); ++i) {
+			unsigned long start, end;
+
+			ihk_mc_get_memory_chunk(i, &start, &end, NULL);
+
+			if (straight_pa_start > start) {
+				straight_pa_start = start;
+			}
+
+			if (straight_pa_end < end) {
+				straight_pa_end = end;
+			}
+		}
+
+		kprintf("%s: straight_pa_start: 0x%lx, straight_pa_end: 0x%lx\n",
+				__FUNCTION__, straight_pa_start, straight_pa_end);
+
+		error = arch_get_smaller_page_size(NULL,
+				straight_pa_end - straight_pa_start,
+				&psize, &p2align);
+
+		if (error) {
+			kprintf("%s: arch_get_smaller_page_size failed: %d\n",
+					__FUNCTION__, error);
+			goto straight_out;
+		}
+		//psize = PTL2_SIZE;
+		//p2align = PTL2_SHIFT - PTL1_SHIFT;
+
+		// Force 512G page
+		//psize = (1UL << 39);
+		//p2align = 39 - PAGE_SHIFT;
+
+		// Force 512MB page
+		psize = (1UL << 29);
+		p2align = 29 - PAGE_SHIFT;
+
+		kprintf("%s: using page shift: %d, psize: %lu\n",
+				__FUNCTION__, p2align + PAGE_SHIFT, psize);
+
+		straight_pa_start &= ~(psize - 1);
+		straight_pa_end = (straight_pa_end + psize - 1) & ~(psize - 1);
+
+		kprintf("%s: aligned straight_pa_start: 0x%lx, straight_pa_end: 0x%lx\n",
+				__FUNCTION__, straight_pa_start, straight_pa_end);
+
+		proc->straight_len = straight_pa_end - straight_pa_start;
+		error = search_free_space(proc->straight_len,
+				PAGE_SHIFT + p2align, (uintptr_t *)&proc->straight_va);
+
+		if (error) {
+			kprintf("%s: search_free_space() failed: %d\n",
+					__FUNCTION__, error);
+			proc->straight_va = 0;
+			goto straight_out;
+		}
+
+		dkprintf("%s: straight_va: 0x%lx to be used\n",
+				__FUNCTION__, proc->straight_va);
+
+		if (add_process_memory_range(proc->vm, (unsigned long)proc->straight_va,
+					(unsigned long)proc->straight_va + proc->straight_len,
+					NOPHYS, vrflags, NULL, 0,
+					PAGE_SHIFT + p2align, &range) != 0) {
+			kprintf("%s: error: adding straight memory range \n",
+					__FUNCTION__);
+			proc->straight_va = 0;
+			goto straight_out;
+		}
+
+		kprintf("%s: straight_va: 0x%lx, range->pgshift: %d, range OK\n",
+				__FUNCTION__, proc->straight_va, range->pgshift);
+
+		ptattr = arch_vrflag_to_ptattr(range->flag, PF_POPULATE, NULL);
+
+#ifdef ENABLE_FUGAKU_HACKS
+		if (1) { // Un-safe mapping of covering physical range
+#endif
+		error = ihk_mc_pt_set_range(proc->vm->address_space->page_table,
+				proc->vm,
+				(void *)range->start,
+				(void *)range->end,
+				straight_pa_start, ptattr,
+				range->pgshift,
+				range, 0);
+
+		if (error) {
+			kprintf("%s: ihk_mc_pt_set_range() failed: %d\n",
+					__FUNCTION__, error);
+			proc->straight_va = 0;
+			goto straight_out;
+		}
+		//ihk_mc_pt_print_pte(proc->vm->address_space->page_table, range->start);
+
+		region->map_end = (unsigned long)proc->straight_va + proc->straight_len;
+		proc->straight_pa = straight_pa_start;
+		kprintf("%s: straight mapping: 0x%lx:%lu @ 0x%lx, "
+				"psize: %lu, straight_map_threshold: %lu\n",
+				__FUNCTION__,
+				proc->straight_va,
+				proc->straight_len,
+				proc->straight_pa,
+				psize,
+				proc->straight_map_threshold);
+
+#ifdef ENABLE_FUGAKU_HACKS
+		}
+		else { // Safe mapping of only LWK memory ranges
+			size_t max_pgsize = 0;
+			size_t min_pgsize = 0xFFFFFFFFFFFFFFFF;
+
+			/*
+			 * Iterate LWK phsyical memory chunks and map them to their
+			 * corresponding offset in the straight range using the largest
+			 * suitable pages.
+			 */
+			for (i = 0; i < ihk_mc_get_nr_memory_chunks(); ++i) {
+				unsigned long start, end, pa;
+				void *va, *va_end;
+				size_t pgsize;
+				int pg2align;
+
+				ihk_mc_get_memory_chunk(i, &start, &end, NULL);
+				va = proc->straight_va + (start - straight_pa_start);
+				va_end = va + (end - start);
+				pa = start;
+
+				while (va < va_end) {
+					pgsize = (va_end - va) + 1;
+retry:
+					error = arch_get_smaller_page_size(NULL, pgsize,
+							&pgsize, &pg2align);
+					if (error) {
+						ekprintf("%s: arch_get_smaller_page_size() failed"
+								" during straight mapping: %d\n",
+								__func__, error);
+						proc->straight_va = 0;
+						goto straight_out;
+					}
+
+					/* Are virtual or physical not page aligned for this size? */
+					if (((unsigned long)va & (pgsize - 1)) ||
+							(pa & (pgsize - 1))) {
+						goto retry;
+					}
+
+					error = ihk_mc_pt_set_range(
+							proc->vm->address_space->page_table,
+							proc->vm,
+							va,
+							va + pgsize,
+							pa,
+							ptattr,
+							pg2align + PAGE_SHIFT,
+							range,
+							0);
+
+					if (error) {
+						kprintf("%s: ihk_mc_pt_set_range() failed"
+								" during straight mapping: %d\n",
+								__func__, error);
+						proc->straight_va = 0;
+						goto straight_out;
+					}
+
+					if (pgsize > max_pgsize)
+						max_pgsize = pgsize;
+
+					if (pgsize < min_pgsize)
+						min_pgsize = pgsize;
+
+					va += pgsize;
+					pa += pgsize;
+				}
+			}
+
+			region->map_end = (unsigned long)proc->straight_va +
+				proc->straight_len;
+			proc->straight_pa = straight_pa_start;
+			kprintf("%s: straight mapping: 0x%lx:%lu @ "
+					"min_pgsize: %lu, max_pgsize: %lu\n",
+					__FUNCTION__,
+					proc->straight_va,
+					proc->straight_len,
+					min_pgsize,
+					max_pgsize);
+		}
+#endif
+	}
+straight_out:
+
 	if (flags & MAP_HUGETLB) {
 		pgshift = (flags >> MAP_HUGE_SHIFT) & 0x3F;
 		if (!pgshift) {
@@ -1734,6 +2091,15 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot,
 
 	ihk_rwspinlock_write_lock_noirq(&thread->vm->memory_range_lock);
 
+	if ((flags & MAP_FIXED) && proc->straight_va &&
+			((void *)addr >= proc->straight_va) &&
+			((void *)addr + len) <= (proc->straight_va + proc->straight_len)) {
+		kprintf("%s: can't map MAP_FIXED into straight mapping\n",
+				__FUNCTION__);
+		error = -EINVAL;
+		goto out;
+	}
+
 	if (flags & MAP_FIXED) {
 		/* clear specified address range */
 		error = do_munmap((void *)addr, len, 1/* holding memory_range_lock */);
@@ -1766,6 +2132,10 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot,
 	}
 
 	if (flags & (MAP_POPULATE | MAP_LOCKED)) {
+		dkprintf("%s: 0x%lx:%lu %s%s|\n",
+			__func__, addr, len,
+				flags & MAP_POPULATE ? "|MAP_POPULATE" : "",
+				flags & MAP_LOCKED ? "|MAP_LOCKED" : "");
 		populated_mapping = 1;
 	}
 
@@ -1790,6 +2160,7 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot,
 	}
 
 	phys = 0;
+	straight_phys = 0;
 	off = 0;
 	maxprot = PROT_READ | PROT_WRITE | PROT_EXEC;
 	if (!(flags & MAP_ANONYMOUS)) {
@@ -1977,6 +2348,31 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot,
 	}
 	vrflags |= VRFLAG_PROT_TO_MAXPROT(PROT_TO_VR_FLAG(maxprot));
 
+	/*
+	 * Large anonymous non-fix allocations are in straight mapping,
+	 * pretend demand paging to avoid filling in PTEs
+	 */
+	if ((flags & MAP_ANONYMOUS) && proc->straight_map &&
+			!(flags & MAP_FIXED) && phys) {
+		if (len >= proc->straight_map_threshold) {
+			dkprintf("%s: range 0x%lx:%lu will be straight, addding VR_DEMAND\n",
+					__FUNCTION__, addr, len);
+			vrflags |= VR_DEMAND_PAGING;
+			straight_phys = phys;
+			phys = 0;
+#ifdef PROFILE_ENABLE
+			profile_event_add(PROFILE_mmap_anon_straight, len);
+#endif // PROFILE_ENABLE
+		}
+		else {
+#ifdef PROFILE_ENABLE
+			if (cpu_local_var(current)->profile)
+				kprintf("%s: contiguous but not straight? len: %lu\n", __func__, len);
+			profile_event_add(PROFILE_mmap_anon_not_straight, len);
+#endif // PROFILE_ENABLE
+		}
+	}
+
 	error = add_process_memory_range(thread->vm, addr, addr+len, phys,
 			vrflags, memobj, off, pgshift, &range);
 	if (error) {
@@ -1987,6 +2383,26 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot,
 		goto out;
 	}
 
+	/* Update straight mapping start address */
+	if (straight_phys) {
+		extern int zero_at_free;
+		range->straight_start =
+			(unsigned long)proc->straight_va +
+			(straight_phys - proc->straight_pa);
+#ifndef ENABLE_FUGAKU_HACKS
+		dkprintf("%s: range 0x%lx:%lu is straight starting at 0x%lx\n",
+			 __FUNCTION__, addr, len, range->straight_start);
+#else
+		dkprintf("%s: range 0x%lx:%lu is straight starting at 0x%lx"
+				" (phys: 0x%lx)\n",
+				__FUNCTION__, addr, len, range->straight_start,
+				straight_phys);
+#endif
+		if (!zero_at_free) {
+			memset((void *)phys_to_virt(straight_phys), 0, len);
+		}
+	}
+
 	/* Determine pre-populated size */
 	populate_len = memobj ? min(len, memobj->size) : len;
 
@@ -2041,12 +2457,13 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot,
 	ro_vma_mapped = 0;
 
 out:
-	if (ro_vma_mapped) {
+	if (ro_vma_mapped && !range->straight_start) {
 		(void)set_host_vma(addr, len, PROT_READ | PROT_WRITE | PROT_EXEC, 1/* holding memory_range_lock */);
 	}
 	ihk_rwspinlock_write_unlock_noirq(&thread->vm->memory_range_lock);
 
-	if (!error && populated_mapping && !((vrflags & VR_PROT_MASK) == VR_PROT_NONE)) {
+	if (!error && populated_mapping &&
+			!((vrflags & VR_PROT_MASK) == VR_PROT_NONE) && !range->straight_start) {
 		error = populate_process_memory(thread->vm,
 				(void *)addr, populate_len);
 
@@ -2080,13 +2497,24 @@ out:
 	if (memobj) {
 		memobj_unref(memobj);
 	}
+
+#ifndef ENABLE_FUGAKU_HACKS
 	dkprintf("%s: 0x%lx:%8lu, (req: 0x%lx:%lu), prot: %x, flags: %x, "
+#else
+	if (cpu_local_var(current)->profile) {
+		kprintf("%s: 0x%lx:%8lu, (req: 0x%lx:%lu), prot: %x, flags: %x, "
+#endif
 			"fd: %d, off: %lu, error: %ld, addr: 0x%lx\n",
 			__FUNCTION__,
 			addr, len, addr0, len0, prot, flags,
 			fd, off0, error, addr);
+#ifdef ENABLE_FUGAKU_HACKS
+	}
+#endif
 
-	return (!error)? addr: error;
+	return !error ?
+		(range->straight_start ? range->straight_start : addr) :
+		error;
 }
 
 SYSCALL_DECLARE(munmap)
@@ -2119,6 +2547,11 @@ SYSCALL_DECLARE(munmap)
 out:
 	dkprintf("[%d]sys_munmap(%lx,%lx): %d\n",
 			ihk_mc_get_processor_id(), addr, len0, error);
+#ifdef ENABLE_FUGAKU_HACKS
+	if (error) {
+		kprintf("%s: error: %d\n", __func__, error);
+	}
+#endif
 	return error;
 }
 
@@ -2166,6 +2599,16 @@ SYSCALL_DECLARE(mprotect)
 		return 0;
 	}
 
+	if (thread->proc->straight_va &&
+			((void *)start >= thread->proc->straight_va) &&
+			(void *)end <= (thread->proc->straight_va +
+				thread->proc->straight_len)) {
+		kprintf("%s: ignored for straight mapping 0x%lx\n",
+				__FUNCTION__, start);
+		error = 0;
+		goto out_straight;
+	}
+
 	flush_nfo_tlb();
 
 	ihk_rwspinlock_write_lock_noirq(&thread->vm->memory_range_lock);
@@ -2259,6 +2702,8 @@ out:
 		}
 	}
 	ihk_rwspinlock_write_unlock_noirq(&thread->vm->memory_range_lock);
+
+out_straight:
 	dkprintf("[%d]sys_mprotect(%lx,%lx,%x): %d\n",
 			ihk_mc_get_processor_id(), start, len0, prot, error);
 	return error;
@@ -3615,6 +4060,23 @@ SYSCALL_DECLARE(ioctl)
 			break;
 	ihk_mc_spinlock_unlock(&proc->mckfd_lock, irqstate);
 
+#ifdef ENABLE_TOFU
+	/* Tofu? */
+	if (proc->enable_tofu &&
+			fd < MAX_FD_PDE && thread->proc->fd_pde_data[fd]) {
+		extern long tof_utofu_unlocked_ioctl(int fd,
+				unsigned int cmd, unsigned long arg);
+
+		rc = tof_utofu_unlocked_ioctl(fd,
+			ihk_mc_syscall_arg1(ctx),
+			ihk_mc_syscall_arg2(ctx));
+
+		/* Do we need to offload? */
+		if (rc != -ENOTSUPP)
+			return rc;
+	}
+#endif
+
 	if(fdp && fdp->ioctl_cb){
 		//kprintf("ioctl: found system fd %d\n", fd);
 		rc = fdp->ioctl_cb(fdp, ctx);
@@ -3622,6 +4084,7 @@ SYSCALL_DECLARE(ioctl)
 	else{
 		rc = syscall_generic_forwarding(__NR_ioctl, ctx);
 	}
+
 	return rc;
 }
 
@@ -3649,6 +4112,10 @@ SYSCALL_DECLARE(open)
 		goto out;
 	}
 
+#ifdef ENABLE_TOFU
+	cpu_local_var(current)->fd_path_in_open = pathname;
+#endif
+
 	dkprintf("open(): pathname=%s\n", pathname);
 	if (!strncmp(pathname, XPMEM_DEV_PATH, len)) {
 		rc = xpmem_open(pathname, flags, ctx);
@@ -3656,8 +4123,21 @@ SYSCALL_DECLARE(open)
 		rc = syscall_generic_forwarding(__NR_open, ctx);
 	}
 
+#ifdef ENABLE_TOFU
+	cpu_local_var(current)->fd_path_in_open = NULL;
+#endif
+
  out:
+#ifdef ENABLE_TOFU
+	if (rc > 0 && rc < MAX_FD_PDE) {
+		cpu_local_var(current)->proc->fd_path[rc] = pathname;
+	}
+	else {
+		kfree(pathname);
+	}
+#else
 	kfree(pathname);
+#endif
 	return rc;
 }
 
@@ -3685,6 +4165,10 @@ SYSCALL_DECLARE(openat)
 		goto out;
 	}
 
+#ifdef ENABLE_TOFU
+	cpu_local_var(current)->fd_path_in_open = pathname;
+#endif
+
 	dkprintf("openat(): pathname=%s\n", pathname);
 	if (!strncmp(pathname, XPMEM_DEV_PATH, len)) {
 		rc = xpmem_openat(pathname, flags, ctx);
@@ -3692,8 +4176,21 @@ SYSCALL_DECLARE(openat)
 		rc = syscall_generic_forwarding(__NR_openat, ctx);
 	}
 
+#ifdef ENABLE_TOFU
+	cpu_local_var(current)->fd_path_in_open = NULL;
+#endif
+
 out:
+#ifdef ENABLE_TOFU
+	if (rc > 0 && rc < MAX_FD_PDE) {
+		cpu_local_var(current)->proc->fd_path[rc] = pathname;
+	}
+	else {
+		kfree(pathname);
+	}
+#else
 	kfree(pathname);
+#endif
 	return rc;
 }
 
@@ -3741,6 +4238,29 @@ SYSCALL_DECLARE(close)
 	long irqstate;
 
 	irqstate = ihk_mc_spinlock_lock(&proc->mckfd_lock);
+
+#ifdef ENABLE_TOFU
+	/* Clear path and PDE data */
+	if (thread->proc->enable_tofu &&
+			fd >= 0 && fd < MAX_FD_PDE) {
+		/* Tofu? */
+		if (thread->proc->fd_pde_data[fd]) {
+			extern void tof_utofu_release_fd(struct process *proc, int fd);
+
+			dkprintf("%s: -> tof_utofu_release_fd() @ fd: %d (%s)\n",
+					__func__, fd, thread->proc->fd_path[fd]);
+			tof_utofu_release_fd(thread->proc, fd);
+			thread->proc->fd_pde_data[fd] = NULL;
+		}
+
+		if (thread->proc->fd_path[fd]) {
+			dkprintf("%s: %d -> %s\n", __func__, fd, thread->proc->fd_path[fd]);
+			kfree(thread->proc->fd_path[fd]);
+			thread->proc->fd_path[fd] = NULL;
+		}
+	}
+#endif
+
 	for(fdp = proc->mckfd, fdq = NULL; fdp; fdq = fdp, fdp = fdp->next)
 		if(fdp->fd == fd)
 			break;
@@ -8729,6 +9249,15 @@ SYSCALL_DECLARE(mremap)
 
 	dkprintf("sys_mremap(%#lx,%#lx,%#lx,%#x,%#lx)\n",
 			oldaddr, oldsize0, newsize0, flags, newaddr);
+
+	if (vm->proc->straight_va &&
+			(void *)oldaddr >= vm->proc->straight_va &&
+			(void *)oldaddr < vm->proc->straight_va + vm->proc->straight_len) {
+		kprintf("%s: reject for straight range 0x%lx\n",
+				__FUNCTION__, oldaddr);
+		return -EINVAL;
+	}
+
 	ihk_rwspinlock_write_lock_noirq(&vm->memory_range_lock);
 
 	/* check arguments */
@@ -9132,6 +9661,11 @@ SYSCALL_DECLARE(mbind)
 		__FUNCTION__,
 		addr, len, mode, nodemask, flags);
 
+	/* No bind support for straight mapped processes */
+	if (cpu_local_var(current)->proc->straight_va) {
+		return 0;
+	}
+
 	/* Validate arguments */
 	if (addr & ~PAGE_MASK) {
 		return -EINVAL;
@@ -10473,6 +11007,7 @@ long syscall(int num, ihk_mc_user_context_t *ctx)
 		 */
 		if (num < PROFILE_SYSCALL_MAX) {
 			profile_event_add(num, (ts - thread->profile_start_ts));
+			thread->profile_start_ts = rdtsc();
 		}
 		else {
 			if (num != __NR_profile) {

kernel/timer.c

@@ -102,6 +102,7 @@ uint64_t schedule_timeout(uint64_t timeout)
 
 		/* Spin wait */
 		while ((rdtsc() - t_s) < LOOP_TIMEOUT) {
+			ihk_numa_zero_free_pages(ihk_mc_get_numa_node_by_distance(0));
 			cpu_pause();
 		}
 

kernel/xpmem.c

@@ -1919,21 +1919,35 @@ static int xpmem_remap_pte(
 		goto out;
 	}
 
-	seg_pte = ihk_mc_pt_lookup_pte(seg_tg->vm->address_space->page_table, 
-		(void *)seg_vaddr, seg_vmr->pgshift, &seg_pgaddr, &seg_pgsize, 
-		&seg_p2align);
-	if (!seg_pte) {
-		ret = -EFAULT;
-		ekprintf("%s: ERROR: ihk_mc_pt_lookup_pte() failed\n", 
-			__FUNCTION__);
-		goto out;
+	if (seg_tg->vm->proc->straight_va &&
+	    seg_vaddr >= (unsigned long)seg_tg->vm->proc->straight_va &&
+	    seg_vaddr < ((unsigned long)seg_tg->vm->proc->straight_va +
+				seg_tg->vm->proc->straight_len)) {
+		seg_phys = (((unsigned long)seg_vaddr & PAGE_MASK) -
+				(unsigned long)seg_tg->vm->proc->straight_va) +
+			seg_tg->vm->proc->straight_pa;
+		dkprintf("%s: 0x%lx in PID %d is straight -> phys: 0x%lx\n",
+				__func__, (unsigned long)seg_vaddr & PAGE_MASK,
+				seg_tg->tgid, seg_phys);
 	}
-	XPMEM_DEBUG("seg_pte=0x%016lx, seg_pgaddr=0x%p, seg_pgsize=%lu, " 
-		"seg_p2align=%d", 
-		*seg_pte, seg_pgaddr, seg_pgsize, seg_p2align);
+	else {
 
-	seg_phys = pte_get_phys(seg_pte);
-	XPMEM_DEBUG("seg_phys=0x%lx", seg_phys);
+		seg_pte = ihk_mc_pt_lookup_pte(seg_tg->vm->address_space->page_table, 
+				(void *)seg_vaddr, seg_vmr->pgshift, &seg_pgaddr, &seg_pgsize, 
+				&seg_p2align);
+		if (!seg_pte) {
+			ret = -EFAULT;
+			ekprintf("%s: ERROR: ihk_mc_pt_lookup_pte() failed\n", 
+					__FUNCTION__);
+			goto out;
+		}
+		XPMEM_DEBUG("seg_pte=0x%016lx, seg_pgaddr=0x%p, seg_pgsize=%lu, " 
+				"seg_p2align=%d", 
+				*seg_pte, seg_pgaddr, seg_pgsize, seg_p2align);
+
+		seg_phys = pte_get_phys(seg_pte);
+		XPMEM_DEBUG("seg_phys=0x%lx", seg_phys);
+	}
 
 	att_pte = ihk_mc_pt_lookup_pte(vm->address_space->page_table, 
 		(void *)vaddr, vmr->pgshift, &att_pgaddr, &att_pgsize, 

lib/abort.c

@@ -22,12 +22,18 @@ void panic(const char *msg)
 
 	arch_print_stack();
 
+#ifndef ENABLE_FUGAKU_HACKS
 	/* do not assume anything after this is executed */
 	arch_cpu_stop();
 
 	while (1) {
 		cpu_halt();
 	}
+#else
+	while (1) {
+		cpu_halt_panic();
+	}
+#endif
 }
 
 extern void arch_show_interrupt_context(const void*);

lib/include/bitops-test_bit.h

@@ -0,0 +1,11 @@
+#ifndef INCLUDE_BITOPS_TEST_BIT_H
+#define INCLUDE_BITOPS_TEST_BIT_H
+
+static inline int test_bit(int nr, const void *addr)
+{
+	const uint32_t *p = (const uint32_t *)addr;
+
+	return ((1UL << (nr & 31)) & (p[nr >> 5])) != 0;
+}
+
+#endif

lib/include/bitops.h

@@ -27,11 +27,7 @@ unsigned long find_first_bit(const unsigned long *addr,
 unsigned long find_first_zero_bit(const unsigned long *addr, 
 				  unsigned long size);
 
-static inline int test_bit(int nr, const void *addr)
-{
-	const uint32_t *p = (const uint32_t *)addr;
-	return ((1UL << (nr & 31)) & (p[nr >> 5])) != 0;
-}
+#include <bitops-test_bit.h>
 
 extern unsigned int __sw_hweight32(unsigned int w);
 extern unsigned int __sw_hweight16(unsigned int w);

lib/include/ihk/cpu.h

@@ -23,7 +23,13 @@ extern int num_processors;
 
 void cpu_enable_interrupt(void);
 void cpu_disable_interrupt(void);
+#ifdef ENABLE_FUGAKU_HACKS
+int cpu_interrupt_disabled(void);
+#endif
 void cpu_halt(void);
+#ifdef ENABLE_FUGAKU_HACKS
+void cpu_halt_panic(void);
+#endif
 void cpu_safe_halt(void);
 void cpu_restore_interrupt(unsigned long);
 void cpu_pause(void);

lib/include/ihk/lock.h

@@ -292,5 +292,38 @@ void ihk_mc_spinlock_lock(ihk_spinlock_t *, unsigned long *);
 void ihk_mc_spinlock_unlock(ihk_spinlock_t *, unsigned long *);
 #endif
 
+/*
+ * Linux queued_spin_lock compatible spin_lock, without the queue.
+ */
+#define _Q_LOCKED_OFFSET    0
+#define _Q_LOCKED_VAL       (1U << _Q_LOCKED_OFFSET)
+
+#define linux_spin_lock(lock)                    \
+	do {                                         \
+		while (!__sync_bool_compare_and_swap(    \
+					(unsigned int *)lock, 0,     \
+					_Q_LOCKED_VAL)) {            \
+			cpu_pause();                         \
+		}                                        \
+	} while (0)
+
+#define linux_spin_unlock(lock)                               \
+	do {                                                      \
+		smp_store_release(lock, 0);                           \
+	} while (0)
+
+#define linux_spin_lock_irqsave(lock, flags)     \
+	do {                                         \
+		flags = cpu_disable_interrupt_save();    \
+		linux_spin_lock(lock);                   \
+	} while (0)
+
+#define linux_spin_unlock_irqrestore(lock, flags) \
+	do {                                          \
+		linux_spin_unlock(lock);                  \
+		cpu_restore_interrupt(flags);             \
+	} while (0)
+
+
 #endif
 

lib/include/ihk/mm.h

@@ -20,6 +20,7 @@
 #include <ihk/lock.h>
 #include <ihk/atomic.h>
 #include <arch/mm.h>
+#include <ihk/debug.h>
 
 struct memobj;
 struct process_vm;
@@ -55,6 +56,8 @@ typedef unsigned long ihk_mc_ap_flag;
 
 #define IHK_MC_AP_BANDWIDTH       0x010000
 #define IHK_MC_AP_LATENCY         0x020000
+/* Only allocate from the closest NUMA node */
+#define IHK_MC_AP_NUMA_STRICT     0x040000
 
 #define IHK_MC_PG_KERNEL       0
 #define IHK_MC_PG_USER         1
@@ -170,6 +173,7 @@ int ihk_mc_pt_change_page(page_table_t pt, void *virt,
                           enum ihk_mc_pt_attribute);
 int ihk_mc_pt_clear_page(page_table_t pt, void *virt);
 int ihk_mc_pt_clear_large_page(page_table_t pt, void *virt);
+int ihk_mc_clear_kernel_range(void *start, void *end);
 int ihk_mc_pt_clear_range(page_table_t pt, struct process_vm *vm, 
 		void *start, void *end);
 int ihk_mc_pt_free_range(page_table_t pt, struct process_vm *vm, 
@@ -178,6 +182,8 @@ int ihk_mc_pt_change_attr_range(page_table_t pt, void *start, void *end,
 		enum ihk_mc_pt_attribute clrattr,
 		enum ihk_mc_pt_attribute setattr);
 pte_t *ihk_mc_pt_lookup_pte(page_table_t pt, void *virt, int pgshift, void **pgbasep, size_t *pgsizep, int *p2alignp);
+pte_t *ihk_mc_pt_lookup_fault_pte(struct process_vm *vm, void *virt,
+		int pgshift, void **basep, size_t *sizep, int *p2alignp);
 int ihk_mc_pt_set_range(page_table_t pt, struct process_vm *vm, void *start, 
 		void *end, uintptr_t phys, enum ihk_mc_pt_attribute attr,
 		int pgshift, struct vm_range *range, int overwrite);
@@ -208,6 +214,10 @@ int ihk_mc_pt_virt_to_phys(struct page_table *pt,
 uint64_t ihk_mc_pt_virt_to_pagemap(struct page_table *pt, unsigned long virt);
 
 int ihk_mc_get_nr_numa_nodes(void);
+struct ihk_mc_numa_node *ihk_mc_get_numa_node_by_distance(int i);
+void ihk_numa_zero_free_pages(struct ihk_mc_numa_node *__node);
+extern int zero_at_free;
+
 struct smp_coreset;
 int ihk_mc_get_numa_node(int id, int *linux_numa_id, int *type);
 int ihk_mc_get_numa_distance(int i, int j);
@@ -217,6 +227,11 @@ int ihk_mc_get_memory_chunk(int id,
 	unsigned long *start,
 	unsigned long *end,
 	int *numa_id);
+#ifdef ENABLE_TOFU
+int ihk_mc_get_memory_chunk_dma_addr(int id,
+		int tni, int cqid,
+		uintptr_t *dma_addr);
+#endif
 
 void remote_flush_tlb_cpumask(struct process_vm *vm, 
 		unsigned long addr, int cpu_id);
@@ -257,4 +272,92 @@ void ihk_mc_query_mem_free_page(void *dump_page_set);
 int ihk_mc_chk_page_address(pte_t mem_addr);
 int ihk_mc_get_mem_user_page(void *arg0, page_table_t pt, pte_t *ptep, void *pgaddr, int pgshift);
 
+#ifndef unlikely
+#define unlikely(x) __builtin_expect(!!(x), 0)
+#endif
+
+extern int zero_at_free;
+
+/*
+ * Generic lockless page cache.
+ * TODO: Store nr of pages in header and double-check at alloc time..
+ */
+struct ihk_mc_page_cache_header;
+
+struct ihk_mc_page_cache_header {
+	struct ihk_mc_page_cache_header *next;
+};
+
+
+static inline void ihk_mc_page_cache_free(
+	struct ihk_mc_page_cache_header *cache, void *page)
+{
+	struct ihk_mc_page_cache_header *current = NULL;
+	struct ihk_mc_page_cache_header *new =
+		(struct ihk_mc_page_cache_header *)page;
+
+	if (unlikely(!page))
+		return;
+
+retry:
+	current = cache->next;
+	new->next = current;
+
+	if (!__sync_bool_compare_and_swap(&cache->next, current, new)) {
+		goto retry;
+	}
+}
+
+static inline void ihk_mc_page_cache_prealloc(
+	struct ihk_mc_page_cache_header *cache,
+	int nr_pages,
+	int nr_elem)
+{
+	int i;
+
+	if (unlikely(cache->next))
+		return;
+
+	for (i = 0; i < nr_elem; ++i) {
+		void *pages;
+
+		pages = ihk_mc_alloc_pages(nr_pages, IHK_MC_AP_NOWAIT);
+
+		if (!pages) {
+			kprintf("%s: ERROR: allocating pages..\n", __func__);
+			continue;
+		}
+
+		ihk_mc_page_cache_free(cache, pages);
+	}
+}
+
+static inline void *ihk_mc_page_cache_alloc(
+	struct ihk_mc_page_cache_header *cache,
+	int nr_pages)
+{
+	register struct ihk_mc_page_cache_header *first, *next;
+
+retry:
+	next = NULL;
+	first = cache->next;
+
+	if (first) {
+		next = first->next;
+
+		if (!__sync_bool_compare_and_swap(&cache->next,
+					first, next)) {
+			goto retry;
+		}
+	}
+	else {
+		kprintf("%s: calling pre-alloc for 0x%lx...\n", __func__, cache);
+
+		ihk_mc_page_cache_prealloc(cache, nr_pages, 256);
+		goto retry;
+	}
+
+	return (void *)first;
+}
+
 #endif

lib/include/ihk/page_alloc.h

@@ -17,6 +17,7 @@
 #define __HEADER_GENERIC_IHK_PAGE_ALLOC
 
 #include <list.h>
+#include <llist.h>
 #include <rbtree.h>
 
 /* XXX: Physical memory management shouldn't be part of IHK */
@@ -31,9 +32,11 @@ struct node_distance {
 struct free_chunk {
 	unsigned long addr, size;
 	struct rb_node node;
+	struct llist_node list;
 };
 #endif
 
+
 struct ihk_mc_numa_node {
 	int id;
 	int linux_numa_id;
@@ -41,10 +44,17 @@ struct ihk_mc_numa_node {
 	struct list_head allocators;
 	struct node_distance *nodes_by_distance;
 #ifdef IHK_RBTREE_ALLOCATOR
+	ihk_atomic_t zeroing_workers;
+	ihk_atomic_t nr_to_zero_pages;
+	struct llist_head zeroed_list;
+	struct llist_head to_zero_list;
 	struct rb_root free_chunks;
 	mcs_lock_node_t lock;
 
 	unsigned long nr_pages;
+	/*
+	 * nr_free_pages: all freed pages, zeroed if zero_at_free
+	 */
 	unsigned long nr_free_pages;
 	unsigned long min_addr;
 	unsigned long max_addr;

lib/include/limits.h

@@ -13,12 +13,35 @@
 #ifndef __HEADER_LIMITS
 #define __HEADER_LIMITS
 
-#define INT_MAX 0x7fffffff
-#define INT_MIN -0x80000000
-#define UINT_MAX 0xffffffff
-#define LONG_MAX 0x7fffffffffffffffL
-#define LONG_MIN -0x8000000000000000L
-#define ULONG_MAX 0xffffffffffffffffL
+#define USHRT_MAX	((uint16_t)(~0U))
+#define SHRT_MAX	((int16_t)(USHRT_MAX>>1))
+#define SHRT_MIN	((int16_t)(-SHRT_MAX - 1))
+#define INT_MAX		((int)(~0U>>1))
+#define INT_MIN		(-INT_MAX - 1)
+#define UINT_MAX	(~0U)
+#define LONG_MAX	((long)(~0UL>>1))
+#define LONG_MIN	(-LONG_MAX - 1)
+#define ULONG_MAX	(~0UL)
+#define LLONG_MAX	((long long)(~0ULL>>1))
+#define LLONG_MIN	(-LLONG_MAX - 1)
+#define ULLONG_MAX	(~0ULL)
+#define SIZE_MAX	(~(size_t)0)
+typedef uint64_t phys_addr_t;
+#define PHYS_ADDR_MAX	(~(phys_addr_t)0)
+
+#define U8_MAX		((uint8_t)~0U)
+#define S8_MAX		((int8_t)(U8_MAX>>1))
+#define S8_MIN		((int8_t)(-S8_MAX - 1))
+#define U16_MAX		((uint16_t)~0U)
+#define S16_MAX		((int16_t)(U16_MAX>>1))
+#define S16_MIN		((int16_t)(-S16_MAX - 1))
+#define U32_MAX		((uint32_t)~0U)
+#define S32_MAX		((int32_t)(U32_MAX>>1))
+#define S32_MIN		((int32_t)(-S32_MAX - 1))
+#define U64_MAX		((uint64_t)~0ULL)
+#define S64_MAX		((int64_t)(U64_MAX>>1))
+#define S64_MIN		((int64_t)(-S64_MAX - 1))
+
 #define IOV_MAX 1024
 
 #ifndef PATH_MAX

lib/include/mc_perf_event.h

@@ -45,7 +45,7 @@ struct perf_event_attr;
          ((nr)   << _IOC_NRSHIFT) | \
          ((size) << _IOC_SIZESHIFT))
 
-#ifndef __KERNEL__
+#ifndef _IOC_TYPECHECK
 #define _IOC_TYPECHECK(t) (sizeof(t))
 #endif
 

lib/page_alloc.c

@@ -319,6 +319,9 @@ kprintf("\nzeroing done\n");
 
 #ifdef IHK_RBTREE_ALLOCATOR
 
+int zero_at_free = 1;
+int deferred_zero_at_free = 1;
+
 /*
  * Simple red-black tree based physical memory management routines.
  *
@@ -329,6 +332,23 @@ kprintf("\nzeroing done\n");
  * of their corresponding memory (i.e., they are on the free memory chunk itself).
  */
 
+#ifdef ENABLE_FUGAKU_HACKS
+size_t __count_free_bytes(struct rb_root *root)
+{
+	struct free_chunk *chunk;
+	struct rb_node *node;
+	size_t size = 0;
+
+	for (node = rb_first(root); node; node = rb_next(node)) {
+		chunk = container_of(node, struct free_chunk, node);
+
+		size += chunk->size;
+	}
+
+	return size;
+}
+#endif
+
 /*
  * Free pages.
  * NOTE: locking must be managed by the caller.
@@ -356,6 +376,7 @@ static int __page_alloc_rbtree_free_range(struct rb_root *root,
 		/* Is ichunk contigous from the left? */
 		if (ichunk->addr + ichunk->size == addr) {
 			struct rb_node *right;
+
 			/* Extend it to the right */
 			ichunk->size += size;
 			dkprintf("%s: chunk extended to right: 0x%lx:%lu\n",
@@ -370,6 +391,10 @@ static int __page_alloc_rbtree_free_range(struct rb_root *root,
 				if (ichunk->addr + ichunk->size == right_chunk->addr) {
 					ichunk->size += right_chunk->size;
 					rb_erase(right, root);
+
+					/* Clear old structure */
+					memset(right_chunk, 0, sizeof(*right_chunk));
+
 					dkprintf("%s: chunk merged to right: 0x%lx:%lu\n",
 							__FUNCTION__, ichunk->addr, ichunk->size);
 				}
@@ -381,6 +406,7 @@ static int __page_alloc_rbtree_free_range(struct rb_root *root,
 		/* Is ichunk contigous from the right? */
 		if (addr + size == ichunk->addr) {
 			struct rb_node *left;
+
 			/* Extend it to the left */
 			ichunk->addr -= size;
 			ichunk->size += size;
@@ -397,6 +423,10 @@ static int __page_alloc_rbtree_free_range(struct rb_root *root,
 					ichunk->addr -= left_chunk->size;
 					ichunk->size += left_chunk->size;
 					rb_erase(left, root);
+
+					/* Clear old structure */
+					memset(left_chunk, 0, sizeof(*left_chunk));
+
 					dkprintf("%s: chunk merged to left: 0x%lx:%lu\n",
 							__FUNCTION__, ichunk->addr, ichunk->size);
 				}
@@ -406,6 +436,10 @@ static int __page_alloc_rbtree_free_range(struct rb_root *root,
 			new_chunk = (struct free_chunk *)phys_to_virt(ichunk->addr);
 			*new_chunk = *ichunk;
 			rb_replace_node(&ichunk->node, &new_chunk->node, root);
+
+			/* Clear old structure */
+			memset(ichunk, 0, sizeof(*ichunk));
+
 			dkprintf("%s: chunk moved to front: 0x%lx:%lu\n",
 					__FUNCTION__, new_chunk->addr, new_chunk->size);
 
@@ -496,6 +530,30 @@ static int __page_alloc_rbtree_mark_range_allocated(struct rb_root *root,
  *
  * NOTE: locking must be managed by the caller.
  */
+struct chunk_fits_arg {
+	unsigned long size;
+	unsigned long align_size;
+	unsigned long align_mask;
+};
+
+bool chunk_fits(struct rb_node *node, void *arg)
+{
+	struct free_chunk *chunk;
+	unsigned long aligned_addr = 0;
+	struct chunk_fits_arg *cfa = (struct chunk_fits_arg *)arg;
+
+	chunk = container_of(node, struct free_chunk, node);
+	aligned_addr = (chunk->addr + (cfa->align_size - 1)) & cfa->align_mask;
+
+	/* Is this a suitable chunk? */
+	if ((aligned_addr + cfa->size) <= (chunk->addr + chunk->size)) {
+		return true;
+	}
+
+	return false;
+}
+
+
 static unsigned long __page_alloc_rbtree_alloc_pages(struct rb_root *root,
 		int npages, int p2align)
 {
@@ -506,6 +564,19 @@ static unsigned long __page_alloc_rbtree_alloc_pages(struct rb_root *root,
 	unsigned long align_mask = ~(align_size - 1);
 	unsigned long aligned_addr = 0;
 
+#if 0
+	struct chunk_fits_arg cfa = {
+		.size = size,
+		.align_size = align_size,
+		.align_mask = align_mask
+	};
+
+	/* Find first maching chunk */
+	node = rb_preorder_dfs_search(root, chunk_fits, &cfa);
+
+	chunk = container_of(node, struct free_chunk, node);
+	aligned_addr = (chunk->addr + (align_size - 1)) & align_mask;
+#else
 	for (node = rb_first(root); node; node = rb_next(node)) {
 		chunk = container_of(node, struct free_chunk, node);
 		aligned_addr = (chunk->addr + (align_size - 1)) & align_mask;
@@ -520,6 +591,7 @@ static unsigned long __page_alloc_rbtree_alloc_pages(struct rb_root *root,
 	if (!node) {
 		return 0;
 	}
+#endif
 
 	dkprintf("%s: allocating: 0x%lx:%lu\n",
 			__FUNCTION__, aligned_addr, size);
@@ -530,6 +602,11 @@ static unsigned long __page_alloc_rbtree_alloc_pages(struct rb_root *root,
 		return 0;
 	}
 
+	if (zero_at_free) {
+		memset(phys_to_virt(aligned_addr),
+				0, sizeof(struct free_chunk));
+	}
+
 	return aligned_addr;
 }
 
@@ -576,6 +653,17 @@ static unsigned long __page_alloc_rbtree_reserve_pages(struct rb_root *root,
 	return aligned_addr;
 }
 
+static struct free_chunk *__page_alloc_rbtree_get_root_chunk(
+	struct rb_root *root)
+{
+	struct rb_node *node = root->rb_node;
+	if (!node) {
+		return NULL;
+	}
+
+	rb_erase(node, root);
+	return container_of(node, struct free_chunk, node);
+}
 
 /*
  * External routines.
@@ -583,9 +671,14 @@ static unsigned long __page_alloc_rbtree_reserve_pages(struct rb_root *root,
 int ihk_numa_add_free_pages(struct ihk_mc_numa_node *node,
 		unsigned long addr, unsigned long size)
 {
+	if (zero_at_free) {
+		/* Zero chunk */
+		memset(phys_to_virt(addr), 0, size);
+	}
+
 	if (__page_alloc_rbtree_free_range(&node->free_chunks, addr, size)) {
 		kprintf("%s: ERROR: adding 0x%lx:%lu\n",
-			__FUNCTION__, addr, size);
+				__FUNCTION__, addr, size);
 		return EINVAL;
 	}
 
@@ -602,6 +695,99 @@ int ihk_numa_add_free_pages(struct ihk_mc_numa_node *node,
 	return 0;
 }
 
+#define IHK_NUMA_ALL_PAGES	(0)
+
+int __ihk_numa_zero_free_pages(struct ihk_mc_numa_node *__node, int nr_pages)
+{
+	int i, max_i;
+	int nr_zeroed_pages = 0;
+
+	if (!zero_at_free)
+		return 0;
+
+	/* If explicitly specified, zero only in __node */
+	max_i = __node ? 1 : ihk_mc_get_nr_numa_nodes();
+
+	/* Look at NUMA nodes in the order of distance */
+	for (i = 0; i < max_i; ++i) {
+		struct ihk_mc_numa_node *node;
+		struct llist_node *llnode;
+
+		/* Unless explicitly specified.. */
+		node = __node ? __node : ihk_mc_get_numa_node_by_distance(i);
+		if (!node) {
+			break;
+		}
+
+		/*
+		 * If number of pages specified, look for a big enough chunk
+		 */
+		if (nr_pages) {
+			struct llist_head tmp;
+
+			init_llist_head(&tmp);
+
+			/* Look for a suitable chunk */
+			while ((llnode = llist_del_first(&node->to_zero_list))) {
+				unsigned long addr;
+				unsigned long size;
+				struct free_chunk *chunk =
+					container_of(llnode, struct free_chunk, list);
+
+				addr = chunk->addr;
+				size = chunk->size;
+
+				if (size < (nr_pages << PAGE_SHIFT)) {
+					llist_add(llnode, &tmp);
+					continue;
+				}
+
+				memset(phys_to_virt(addr) + sizeof(*chunk), 0,
+						size - sizeof(*chunk));
+				llist_add(&chunk->list, &node->zeroed_list);
+				barrier();
+				ihk_atomic_sub((int)(size >> PAGE_SHIFT),
+						&node->nr_to_zero_pages);
+				nr_zeroed_pages += (chunk->size >> PAGE_SHIFT);
+				kprintf("%s: zeroed chunk 0x%lx:%lu in allocate path\n",
+						__func__, addr, size);
+				break;
+			}
+
+			/* Add back the ones that didn't match */
+			while ((llnode = llist_del_first(&tmp))) {
+				llist_add(llnode, &node->to_zero_list);
+			}
+		}
+		/* Otherwise iterate all to_zero chunks */
+		else {
+			while ((llnode = llist_del_first(&node->to_zero_list))) {
+				unsigned long addr;
+				unsigned long size;
+				struct free_chunk *chunk =
+					container_of(llnode, struct free_chunk, list);
+
+				addr = chunk->addr;
+				size = chunk->size;
+
+				memset(phys_to_virt(addr) + sizeof(*chunk), 0,
+						size - sizeof(*chunk));
+				llist_add(&chunk->list, &node->zeroed_list);
+				barrier();
+				ihk_atomic_sub((int)(size >> PAGE_SHIFT),
+						&node->nr_to_zero_pages);
+				nr_zeroed_pages += (chunk->size >> PAGE_SHIFT);
+			}
+		}
+	}
+
+	return nr_zeroed_pages;
+}
+
+void ihk_numa_zero_free_pages(struct ihk_mc_numa_node *__node)
+{
+	__ihk_numa_zero_free_pages(__node, IHK_NUMA_ALL_PAGES);
+}
 
 unsigned long ihk_numa_alloc_pages(struct ihk_mc_numa_node *node,
 	int npages, int p2align)
@@ -628,7 +814,44 @@ unsigned long ihk_numa_alloc_pages(struct ihk_mc_numa_node *node,
 #endif
 
 	mcs_lock_lock(&node->lock, &mcs_node);
+retry:
+	if (zero_at_free) {
+		struct llist_node *llnode;
 
+		/*
+		 * Process zeroed chunks that are not
+		 * on the free tree yet.
+		 */
+		while ((llnode = llist_del_first(&node->zeroed_list))) {
+			unsigned long addr;
+			unsigned long size;
+			struct free_chunk *chunk =
+				container_of(llnode, struct free_chunk, list);
+
+			addr = chunk->addr;
+			size = chunk->size;
+
+			if (__page_alloc_rbtree_free_range(&node->free_chunks,
+						addr, size)) {
+				kprintf("%s: ERROR: freeing zeroed chunk 0x%lx:%lu\n",
+						__FUNCTION__, addr, npages << PAGE_SHIFT);
+			}
+			else {
+				node->nr_free_pages += (size >> PAGE_SHIFT);
+				dkprintf("%s: freed zeroed chunk 0x%lx:%lu\n",
+						__FUNCTION__, addr, size);
+			}
+		}
+
+		/* Not enough? Check if we can zero pages now */
+		if (node->nr_free_pages < npages) {
+			if (__ihk_numa_zero_free_pages(node, npages) >= npages) {
+				goto retry;
+			}
+		}
+	}
+
+	/* Not enough pages? Give up.. */
 	if (node->nr_free_pages < npages) {
 		goto unlock_out;
 	}
@@ -639,6 +862,16 @@ unsigned long ihk_numa_alloc_pages(struct ihk_mc_numa_node *node,
 	/* Does not necessarily succeed due to alignment */
 	if (addr) {
 		node->nr_free_pages -= npages;
+#if 0
+		{
+			size_t free_bytes = __count_free_bytes(&node->free_chunks);
+			if (free_bytes != node->nr_free_pages * PAGE_SIZE) {
+				kprintf("%s: inconsistent free count? node: %lu vs. cnt: %lu\n",
+						__func__, node->nr_free_pages * PAGE_SIZE, free_bytes);
+				panic("");
+			}
+		}
+#endif
 		dkprintf("%s: allocated pages 0x%lx:%lu\n",
 				__FUNCTION__, addr, npages << PAGE_SHIFT);
 	}
@@ -653,6 +886,7 @@ void ihk_numa_free_pages(struct ihk_mc_numa_node *node,
 	unsigned long addr, int npages)
 {
 	mcs_lock_node_t mcs_node;
+	int defer_zero_at_free = deferred_zero_at_free;
 
 #ifdef ENABLE_PER_CPU_ALLOC_CACHE
 	/* CPU local cache */
@@ -684,18 +918,105 @@ void ihk_numa_free_pages(struct ihk_mc_numa_node *node,
 		return;
 	}
 
-	mcs_lock_lock(&node->lock, &mcs_node);
-	if (__page_alloc_rbtree_free_range(&node->free_chunks, addr,
-				npages << PAGE_SHIFT)) {
-		kprintf("%s: ERROR: freeing 0x%lx:%lu\n",
-				__FUNCTION__, addr, npages << PAGE_SHIFT);
+#if 0
+	/* Do not defer zeroing when the number of free pages is low */
+	if (zero_at_free && defer_zero_at_free) {
+		mcs_lock_lock(&node->lock, &mcs_node);
+		if (node->nr_free_pages < (node->nr_pages * 3 / 100))
+			defer_zero_at_free = 0;
+		mcs_lock_unlock(&node->lock, &mcs_node);
 	}
+#endif
+
+
+	/* Zero chunk right here if needed */
+	if (zero_at_free && !defer_zero_at_free) {
+		memset(phys_to_virt(addr), 0, npages << PAGE_SHIFT);
+	}
+
+	/*
+	 * If we don't zero at free() or we zeroed the chunk
+	 * already, simply add it to the free tree.
+	 */
+	if (!zero_at_free ||
+			(zero_at_free && !defer_zero_at_free)) {
+		mcs_lock_lock(&node->lock, &mcs_node);
+
+		if (__page_alloc_rbtree_free_range(&node->free_chunks, addr,
+					npages << PAGE_SHIFT)) {
+			kprintf("%s: ERROR: freeing 0x%lx:%lu\n",
+					__FUNCTION__, addr, npages << PAGE_SHIFT);
+		}
+		else {
+			node->nr_free_pages += npages;
+#if 0
+			{
+				size_t free_bytes = __count_free_bytes(&node->free_chunks);
+				if (free_bytes != node->nr_free_pages * PAGE_SIZE) {
+					kprintf("%s: inconsistent free count? node: %lu vs. cnt: %lu\n",
+							__func__, node->nr_free_pages * PAGE_SIZE, free_bytes);
+					panic("");
+				}
+			}
+#endif
+			dkprintf("%s: freed%s chunk 0x%lx:%lu\n",
+					__FUNCTION__,
+					zero_at_free ? " and zeroed" : "",
+					addr, npages << PAGE_SHIFT);
+		}
+		mcs_lock_unlock(&node->lock, &mcs_node);
+	}
+	/*
+	 * Deferred zeroing.
+	 * Put the chunk to the to_zero list.
+	 */
 	else {
-		node->nr_free_pages += npages;
-		dkprintf("%s: freed pages 0x%lx:%lu\n",
-				__FUNCTION__, addr, npages << PAGE_SHIFT);
+		struct free_chunk *chunk =
+			(struct free_chunk *)phys_to_virt(addr);
+		chunk->addr = addr;
+		chunk->size = npages << PAGE_SHIFT;
+		ihk_atomic_add(npages, &node->nr_to_zero_pages);
+		barrier();
+		llist_add(&chunk->list, &node->to_zero_list);
+
+		/* Ask Linux to clear memory */
+		if (cpu_local_var_initialized &&
+				cpu_local_var(current) &&
+				cpu_local_var(current) != &cpu_local_var(idle) &&
+				!cpu_local_var(current)->proc->nohost) {
+			struct ihk_ikc_channel_desc *syscall_channel =
+				cpu_local_var(ikc2linux);
+			struct ikc_scd_packet packet IHK_DMA_ALIGN;
+
+			if (ihk_atomic_read(&node->zeroing_workers) > 0) {
+				dkprintf("%s: skipping Linux zero request..\n", __func__);
+				return;
+			}
+
+			ihk_atomic_inc(&node->zeroing_workers);
+
+			memset(&packet, 0, sizeof(packet));
+			packet.req.number = __NR_move_pages;
+			packet.req.args[0] = (unsigned long)node;
+
+			barrier();
+			smp_store_release(&packet.req.valid, 1);
+			packet.msg = SCD_MSG_SYSCALL_ONESIDE;
+			packet.ref = ihk_mc_get_processor_id();
+			packet.pid = cpu_local_var(current)->proc->pid;
+			packet.resp_pa = 0;
+
+			if (ihk_ikc_send(syscall_channel, &packet, 0) < 0) {
+				kprintf("%s: WARNING: failed to send memory clear"
+						" send IKC req..\n", __func__);
+			}
+			else {
+				dkprintf("%s: clear mem req for NUMA %d sent in req"
+						" for addr: 0x%lx\n",
+						__func__, node->id, addr);
+			}
+		}
 	}
-	mcs_lock_unlock(&node->lock, &mcs_node);
 }
 
 #endif // IHK_RBTREE_ALLOCATOR

scripts/mckernel.spec.in

@@ -25,9 +25,17 @@ Source0: mckernel-%{version}.tar.gz
 
 Requires: systemd-libs numactl-libs libdwarf
 
-# kernel_module_package macro does not handle cross build...
+# don't use kernel_module_package so that one rpm including .ko and binaries are created
 %if "%{?_host_cpu}" == "x86_64" && "%{?_target_cpu}" == "aarch64"
 %define cross_compile 1
+%else
+BuildRequires: systemd-devel numactl-devel binutils-devel kernel-devel libdwarf-devel
+# Friendly reminder of the fact that kernel-rpm-macros is no longer included in kernel-devel
+%if 0%{?rhel} >= 8
+BuildRequires: redhat-rpm-config kernel-rpm-macros elfutils-libelf-devel
+%endif
+%endif
+
 %if 0%{?rhel} >= 8
 Requires: kernel >= %{krequires}
 %else
@@ -35,17 +43,6 @@ Requires: kernel = %{krequires}
 %endif
 Requires(post):   /usr/sbin/depmod
 Requires(postun): /usr/sbin/depmod
-%else
-BuildRequires: systemd-devel numactl-devel binutils-devel kernel-devel libdwarf-devel
-# Friendly reminder of the fact that kernel-rpm-macros is no longer included in kernel-devel
-%if 0%{?rhel} >= 8
-BuildRequires: redhat-rpm-config kernel-rpm-macros elfutils-libelf-devel kmod
-%endif
-%if %{defined kernel_module_package_buildreqs}
-BuildRequires: %kernel_module_package_buildreqs
-%kernel_module_package %{?kmod_flavors}
-%endif
-%endif
 
 %description
 Interface for Heterogeneous Kernels and McKernel.
@@ -78,6 +75,9 @@ This package contains headers and libraries required for build apps using IHK/Mc
 	%{?cmake_libdir:-DCMAKE_INSTALL_LIBDIR=%{cmake_libdir}} \
 	%{?build_target:-DBUILD_TARGET=%{build_target}} \
 	%{?toolchain_file:-DCMAKE_TOOLCHAIN_FILE=%{toolchain_file}} \
+	-DENABLE_TOFU=ON -DENABLE_FUGAKU_HACKS=ON \
+	-DENABLE_KRM_WORKAROUND=OFF -DWITH_KRM=ON \
+	-DENABLE_FUGAKU_DEBUG=OFF \
 	.
 %make_build
 
@@ -113,7 +113,6 @@ This package contains headers and libraries required for build apps using IHK/Mc
 %{_mandir}/man1/ihkosctl.1.gz
 %{_mandir}/man1/mcexec.1.gz
 
-%if 0%{?cross_compile}
 /lib/modules/%{kernel_version}/extra/mckernel/ihk.ko
 /lib/modules/%{kernel_version}/extra/mckernel/mcctrl.ko
 %ifarch x86_64
@@ -122,7 +121,6 @@ This package contains headers and libraries required for build apps using IHK/Mc
 %ifarch aarch64
 /lib/modules/%{kernel_version}/extra/mckernel/ihk-smp-arm64.ko
 %endif
-%endif
 
 %files	devel
 %{_includedir}/ihklib.h
@@ -135,8 +133,7 @@ This package contains headers and libraries required for build apps using IHK/Mc
 %{_includedir}/ihk/ihk_host_driver.h
 /lib/modules/%{kernel_version}/extra/mckernel/ihk/linux/core/Module.symvers
 
-%if 0%{?cross_compile}
-# scripts from /usr/lib/rpm/redhat/kmodtool (kernel_module_package) as well
+# taken from /usr/lib/rpm/redhat/kmodtool (kernel_module_package)
 %post
 if [ -e "/boot/System.map-%{kernel_version}" ]; then
     /usr/sbin/depmod -aeF "/boot/System.map-%{kernel_version}" "%{kernel_version}" > /dev/null || :
@@ -162,7 +159,6 @@ if [ -x "/sbin/weak-modules" ]; then
     printf '%s\n' "${modules[@]}" \
     | /sbin/weak-modules --remove-modules
 fi
-%endif
 
 %changelog
 * Tue Feb 12 2019 Dominique Martinet <dominique.martinet@cea.fr> - 1.6.0-0

scripts/mcreboot-lwk.sh.in

@@ -0,0 +1,99 @@
+#!/bin/bash
+# IHK/McKernel user priviledge reboot script.
+# author: Balazs Gerofi <bgerofi@riken.jp>
+#      Copyright (C) 2019 RIKEN
+#
+prefix="@prefix@"
+BINDIR="${prefix}/bin"
+SBINDIR="${prefix}/sbin"
+KERNDIR="@MCKERNELDIR@"
+
+mem=""
+cpus=""
+ikc_map=""
+
+while getopts c:m:r: OPT
+do
+	case ${OPT} in
+	c) cpus=${OPTARG}
+		;;
+	m) mem=${OPTARG}
+		;;
+	r) ikc_map=${OPTARG}
+		;;
+	\?) exit 1
+		;;
+	esac
+done
+
+if [ "${ikc_map}" == "" ]; then
+	# Query IKC map
+	if ! ${SBINDIR}/ihkosctl 0 get ikc_map > /dev/null; then
+		echo "error: querying IKC map" >&2
+		exit 1
+	fi
+
+	ikc_map=`${SBINDIR}/ihkosctl 0 get ikc_map`
+fi
+
+
+# Shutdown OS
+if ! ${SBINDIR}/ihkosctl 0 shutdown; then
+	echo "error: shuting down OS" >&2
+	exit 1
+fi
+
+sleep 2
+
+# Query IHK-SMP resources and reassign
+if ! ${SBINDIR}/ihkconfig 0 query cpu > /dev/null; then
+	echo "error: querying cpus" >&2
+	exit 1
+fi
+
+cpus=`${SBINDIR}/ihkconfig 0 query cpu`
+if [ "${cpus}" == "" ]; then
+	echo "error: querying CPUs" >&2
+	exit 1
+fi
+
+# Assign CPUs
+if ! ${SBINDIR}/ihkosctl 0 assign cpu ${cpus}; then
+	echo "error: assign CPUs" >&2
+	exit 1
+fi
+
+# Assign memory
+for i in `seq 0 15`; do
+	if ! ${SBINDIR}/ihkosctl 0 assign mem all@${i}; then
+		echo "error: assign memory" >&2
+		exit 1
+	fi
+done
+
+if [ "${ikc_map}" != "" ]; then
+	# Set IKC map
+	if ! ${SBINDIR}/ihkosctl 0 set ikc_map ${ikc_map}; then
+		echo "error: setting IKC map" >&2
+		exit 1
+	fi
+fi
+
+# Load kernel image
+if ! ${SBINDIR}/ihkosctl 0 load ${KERNDIR}/mckernel.img; then
+	echo "error: loading kernel image: ${KERNDIR}/mckernel.img" >&2
+	exit 1
+fi
+
+# Set kernel arguments
+if ! ${SBINDIR}/ihkosctl 0 kargs "hidos dump_level=24"; then
+	echo "error: setting kernel arguments" >&2
+	exit 1
+fi
+
+# Boot OS instance
+if ! ${SBINDIR}/ihkosctl 0 boot; then
+	echo "error: booting" >&2
+	exit 1
+fi
+