uti: futex call function in mcctrl

Previously, futex code of McKerenl was called by mccontrol,
but there ware some problems with this method.
(Mainly, location of McKernel image on memory)

Call futex code in mcctrl instead of the one in McKernel image,
giving the following benefits:
1. Not relying on shared kernel virtual address space with Linux any more
2. The cpu id store / retrieve is not needed and resulting in the code

Change-Id: Ic40929b64a655b270c435859fa287fedb713ee5c
refe: #1428

arch/arm64/kernel/cpu.c

@@ -1524,6 +1524,11 @@ int ihk_mc_arch_get_special_register(enum ihk_asr_type type,
 	return -1;
 }
 
+int ihk_mc_get_interrupt_id(int cpu)
+{
+	return cpu;
+}
+
 /*@
   @ requires \valid_cpuid(cpu);     // valid CPU logical ID
   @ ensures \result == 0

arch/arm64/kernel/include/arch-futex.h

@@ -7,7 +7,8 @@
  * @ref.impl 
  * 	linux-linaro/arch/arm64/include/asm/futex.h:__futex_atomic_op
  */
-#define __futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg)		\
+#define ___futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg)		\
+do {								\
 	asm volatile(							\
 "1:	ldxr	%w1, %2\n"						\
 	insn "\n"							\
@@ -26,7 +27,24 @@
 "	.popsection\n"							\
 	: "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp)	\
 	: "r" (oparg), "Ir" (-EFAULT)					\
-	: "memory")
+	: "memory");						\
+} while (0);
+
+#ifndef IHK_OS_MANYCORE
+#include <linux/uaccess.h>
+
+#define __futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg)		\
+do {								\
+	uaccess_enable();					\
+	___futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg)		\
+	uaccess_disable();					\
+} while (0);
+
+#else
+#define __futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg)		\
+	___futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg)		\
+
+#endif
 
 /*
  * @ref.impl 
@@ -135,12 +153,4 @@ futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval)
 	return ret;
 }
 
-static inline int get_futex_value_locked(uint32_t *dest, uint32_t *from)
-{
-
-	*dest = *(volatile uint32_t *)from;
-
-	return 0;
-}
-
 #endif /* !__HEADER_ARM64_COMMON_ARCH_FUTEX_H */

arch/x86_64/kernel/cpu.c

@@ -1678,6 +1678,11 @@ int ihk_mc_arch_get_special_register(enum ihk_asr_type type,
 	}
 }
 
+int ihk_mc_get_interrupt_id(int cpu)
+{
+	return get_x86_cpu_local_variable(cpu)->apic_id;
+}
+
 /*@
   @ requires \valid_cpuid(cpu);     // valid CPU logical ID
   @ ensures \result == 0

arch/x86_64/kernel/include/arch-futex.h

@@ -129,12 +129,4 @@ static inline int futex_atomic_op_inuser(int encoded_op,
 	return ret;
 }
 
-static inline int get_futex_value_locked(uint32_t *dest, uint32_t *from)
-{
-
-	*dest = *(volatile uint32_t *)from;
-
-	return 0;
-}
-
 #endif

executer/include/uti.h

@@ -5,7 +5,7 @@ struct syscall_struct {
 	int number;
 	unsigned long args[6];
 	unsigned long ret;
-	unsigned long uti_clv; /* copy of a clv in McKernel */
+	unsigned long uti_info; /* reference to data in McKernel */
 };
 
 #define UTI_SZ_SYSCALL_STACK 16
@@ -17,7 +17,7 @@ struct uti_desc {
 	int mck_tid; /* TODO: Move this out for multiple migrated-to-Linux threads */
 	unsigned long key; /* struct task_struct* of mcexec thread, used to search struct host_thread */
 	int pid, tid; /* Used as the id of tracee when issuing MCEXEC_UP_TERMINATE_THREAD */
-	unsigned long uti_clv; /* copy of McKernel clv */
+	unsigned long uti_info; /* reference to data in McKernel */
 
 	int fd; /* /dev/mcosX */
 	struct syscall_struct syscall_stack[UTI_SZ_SYSCALL_STACK]; /* stack of system call arguments and return values */
@@ -26,6 +26,36 @@ struct uti_desc {
 	int start_syscall_intercept; /* Used to sync between mcexec.c and syscall_intercept.c */
 };
 
+/* Reference to McKernel variables accessed by mcctrl */
+struct uti_info {
+	/* clv info */
+	unsigned long thread_va;
+	void *uti_futex_resp;
+	void *ikc2linux;
+	unsigned long uti_futex_resp_pa;
+	unsigned long ikc2linux_pa;
+
+	/* thread info */
+	int tid;
+	int cpu;
+	void *status;
+	void *spin_sleep_lock;
+	void *spin_sleep;
+	void *vm;
+	void *futex_q;
+	unsigned long status_pa;
+	unsigned long spin_sleep_lock_pa;
+	unsigned long spin_sleep_pa;
+	unsigned long vm_pa;
+	unsigned long futex_q_pa;
+
+	/* global info */
+	int mc_idle_halt;
+	void *futex_queue;
+	void *os;              // set by mcctrl
+	unsigned long futex_queue_pa;
+
+};
 
 #endif
 

executer/kernel/mcctrl/CMakeLists.txt

@@ -16,13 +16,15 @@ kmod(mcctrl
 		-I${IHK_FULL_SOURCE_DIR}/include/arch/${ARCH}
 		-I${PROJECT_SOURCE_DIR}/executer/include
 		-I${CMAKE_CURRENT_SOURCE_DIR}/arch/${ARCH}/include
+		-I${CMAKE_CURRENT_SOURCE_DIR}/include
 		-I${PROJECT_BINARY_DIR}
 		-I${PROJECT_SOURCE_DIR}/kernel/include
+		-I${PROJECT_SOURCE_DIR}/arch/${ARCH}/kernel/include
 		-DMCEXEC_PATH=\\"${MCEXEC_PATH}\\"
 		${ARCH_C_FLAGS}
 	SOURCES
 		driver.c control.c ikc.c syscall.c procfs.c binfmt_mcexec.c
-		sysfs.c sysfs_files.c arch/${ARCH}/archdeps.c
+		sysfs.c sysfs_files.c mc_plist.c futex.c arch/${ARCH}/archdeps.c arch/${ARCH}/cpu.c
 	EXTRA_SYMBOLS
 		${PROJECT_BINARY_DIR}/ihk/linux/core/Module.symvers
 	DEPENDS

executer/kernel/mcctrl/arch/arm64/cpu.c

@@ -0,0 +1,96 @@
+/* cpu.c COPYRIGHT FUJITSU LIMITED 2015-2019 */
+
+#include <cpu.h>
+
+/* we not have "pause" instruction, instead "yield" instruction */
+void cpu_pause(void)
+{
+	asm volatile("yield" ::: "memory");
+}
+
+#if defined(CONFIG_HAS_NMI)
+#include <arm-gic-v3.h>
+
+/* restore interrupt (ICC_PMR_EL1 <= flags) */
+void cpu_restore_interrupt(unsigned long flags)
+{
+	asm volatile(
+		"msr_s  " __stringify(ICC_PMR_EL1) ",%0"
+		:
+		: "r" (flags)
+		: "memory");
+}
+
+/* save ICC_PMR_EL1 & disable interrupt (ICC_PMR_EL1 <= ICC_PMR_EL1_MASKED) */
+unsigned long cpu_disable_interrupt_save(void)
+{
+	unsigned long flags;
+	unsigned long masked = ICC_PMR_EL1_MASKED;
+
+	asm volatile(
+		"mrs_s  %0, " __stringify(ICC_PMR_EL1) "\n"
+		"msr_s  " __stringify(ICC_PMR_EL1) ",%1"
+		: "=&r" (flags)
+		: "r" (masked)
+		: "memory");
+	return flags;
+}
+
+/* save ICC_PMR_EL1 & enable interrupt (ICC_PMR_EL1 <= ICC_PMR_EL1_UNMASKED) */
+unsigned long cpu_enable_interrupt_save(void)
+{
+	unsigned long flags;
+	unsigned long masked = ICC_PMR_EL1_UNMASKED;
+
+	asm volatile(
+		"mrs_s  %0, " __stringify(ICC_PMR_EL1) "\n"
+		"msr_s  " __stringify(ICC_PMR_EL1) ",%1"
+		: "=&r" (flags)
+		: "r" (masked)
+		: "memory");
+	return flags;
+}
+
+#else /* defined(CONFIG_HAS_NMI) */
+
+/* @ref.impl arch/arm64/include/asm/spinlock.h::arch_local_irq_restore */
+/* restore interrupt (PSTATE.DAIF = flags restore) */
+void cpu_restore_interrupt(unsigned long flags)
+{
+	asm volatile(
+		"msr    daif, %0	// arch_local_irq_restore"
+		:
+		: "r" (flags)
+		: "memory");
+}
+
+/* @ref.impl arch/arm64/include/asm/irqflags.h::arch_local_irq_save */
+/* save PSTATE.DAIF & disable interrupt (PSTATE.DAIF I bit set) */
+unsigned long cpu_disable_interrupt_save(void)
+{
+	unsigned long flags;
+
+	asm volatile(
+		"mrs    %0, daif	// arch_local_irq_save\n"
+		"msr    daifset, #2"
+		: "=r" (flags)
+		:
+		: "memory");
+	return flags;
+}
+
+/* save PSTATE.DAIF & enable interrupt (PSTATE.DAIF I bit set) */
+unsigned long cpu_enable_interrupt_save(void)
+{
+	unsigned long flags;
+
+	asm volatile(
+		"mrs    %0, daif	// arch_local_irq_save\n"
+		"msr    daifclr, #2"
+		: "=r" (flags)
+		:
+		: "memory");
+	return flags;
+}
+#endif /* defined(CONFIG_HAS_NMI) */
+

executer/kernel/mcctrl/arch/arm64/include/arch-lock.h

@@ -0,0 +1,142 @@
+/* This is copy of the necessary part from McKernel, for uti-futex */
+
+/* arch-lock.h COPYRIGHT FUJITSU LIMITED 2015-2018 */
+#ifndef __HEADER_ARM64_COMMON_ARCH_LOCK_H
+#define __HEADER_ARM64_COMMON_ARCH_LOCK_H
+
+#include <linux/preempt.h>
+#include <cpu.h>
+
+#define ihk_mc_spinlock_lock __ihk_mc_spinlock_lock
+#define ihk_mc_spinlock_unlock __ihk_mc_spinlock_unlock
+
+#define ihk_mc_spinlock_lock_noirq __ihk_mc_spinlock_lock_noirq
+#define ihk_mc_spinlock_unlock_noirq __ihk_mc_spinlock_unlock_noirq
+
+/* @ref.impl arch/arm64/include/asm/spinlock_types.h::TICKET_SHIFT */
+#define TICKET_SHIFT	16
+
+/* @ref.impl ./arch/arm64/include/asm/lse.h::ARM64_LSE_ATOMIC_INSN */
+/* else defined(CONFIG_AS_LSE) && defined(CONFIG_ARM64_LSE_ATOMICS) */
+#define _ARM64_LSE_ATOMIC_INSN(llsc, lse)    llsc
+
+/* @ref.impl arch/arm64/include/asm/spinlock_types.h::arch_spinlock_t */
+typedef struct {
+#ifdef __AARCH64EB__
+	uint16_t next;
+	uint16_t owner;
+#else /* __AARCH64EB__ */
+	uint16_t owner;
+	uint16_t next;
+#endif /* __AARCH64EB__ */
+} __attribute__((aligned(4))) _ihk_spinlock_t;
+
+/* @ref.impl arch/arm64/include/asm/spinlock.h::arch_spin_lock */
+/* spinlock lock */
+static inline void
+__ihk_mc_spinlock_lock_noirq(_ihk_spinlock_t *lock)
+{
+	unsigned int tmp;
+	_ihk_spinlock_t lockval, newval;
+
+	preempt_disable();
+
+	asm volatile(
+	/* Atomically increment the next ticket. */
+	_ARM64_LSE_ATOMIC_INSN(
+	/* LL/SC */
+"	prfm	pstl1strm, %3\n"
+"1:	ldaxr	%w0, %3\n"
+"	add	%w1, %w0, %w5\n"
+"	stxr	%w2, %w1, %3\n"
+"	cbnz	%w2, 1b\n",
+	/* LSE atomics */
+"	mov	%w2, %w5\n"
+"	ldadda	%w2, %w0, %3\n"
+	__nops(3)
+	)
+
+	/* Did we get the lock? */
+"	eor	%w1, %w0, %w0, ror #16\n"
+"	cbz	%w1, 3f\n"
+	/*
+	 * No: spin on the owner. Send a local event to avoid missing an
+	 * unlock before the exclusive load.
+	 */
+"	sevl\n"
+"2:	wfe\n"
+"	ldaxrh	%w2, %4\n"
+"	eor	%w1, %w2, %w0, lsr #16\n"
+"	cbnz	%w1, 2b\n"
+	/* We got the lock. Critical section starts here. */
+"3:"
+	: "=&r" (lockval), "=&r" (newval), "=&r" (tmp), "+Q" (*lock)
+	: "Q" (lock->owner), "I" (1 << TICKET_SHIFT)
+	: "memory");
+}
+
+/* spinlock lock & interrupt disable & PSTATE.DAIF save */
+static inline unsigned long
+__ihk_mc_spinlock_lock(_ihk_spinlock_t *lock)
+{
+	unsigned long flags;
+
+	flags = cpu_disable_interrupt_save();
+
+	__ihk_mc_spinlock_lock_noirq(lock);
+
+	return flags;
+}
+
+/* @ref.impl arch/arm64/include/asm/spinlock.h::arch_spin_unlock */
+/* spinlock unlock */
+static inline void
+__ihk_mc_spinlock_unlock_noirq(_ihk_spinlock_t *lock)
+{
+	unsigned long tmp;
+
+	asm volatile(_ARM64_LSE_ATOMIC_INSN(
+	/* LL/SC */
+	"	ldrh	%w1, %0\n"
+	"	add	%w1, %w1, #1\n"
+	"	stlrh	%w1, %0",
+	/* LSE atomics */
+	"	mov	%w1, #1\n"
+	"	staddlh	%w1, %0\n"
+	__nops(1))
+	: "=Q" (lock->owner), "=&r" (tmp)
+	:
+	: "memory");
+
+	preempt_enable();
+}
+
+static inline void
+__ihk_mc_spinlock_unlock(_ihk_spinlock_t *lock, unsigned long flags)
+{
+	__ihk_mc_spinlock_unlock_noirq(lock);
+
+	cpu_restore_interrupt(flags);
+}
+
+typedef struct mcs_rwlock_lock {
+	_ihk_spinlock_t slock;
+#ifndef ENABLE_UBSAN
+} __aligned(64) mcs_rwlock_lock_t;
+#else
+} mcs_rwlock_lock_t;
+#endif
+
+static inline void
+mcs_rwlock_writer_lock_noirq(struct mcs_rwlock_lock *lock)
+{
+	ihk_mc_spinlock_lock_noirq(&lock->slock);
+}
+
+static inline void
+mcs_rwlock_writer_unlock_noirq(struct mcs_rwlock_lock *lock)
+{
+	ihk_mc_spinlock_unlock_noirq(&lock->slock);
+}
+
+#endif /* !__HEADER_ARM64_COMMON_ARCH_LOCK_H */

executer/kernel/mcctrl/arch/arm64/include/archdeps.h

@@ -38,4 +38,26 @@ static const unsigned long arch_rus_vm_flags = VM_RESERVED | VM_MIXEDMAP | VM_EX
 #else
 static const unsigned long arch_rus_vm_flags = VM_DONTDUMP | VM_MIXEDMAP | VM_EXEC;
 #endif
+
+#define _xchg(ptr, x) \
+({ \
+	__typeof__(*(ptr)) __ret; \
+	__ret = (__typeof__(*(ptr))) \
+		__xchg((unsigned long)(x), (ptr), sizeof(*(ptr))); \
+	__ret; \
+})
+
+#define xchg4(ptr, x) _xchg(ptr, x)
+#define xchg8(ptr, x) _xchg(ptr, x)
+
+enum arm64_pf_error_code {
+	PF_PROT		=	1 << 0,
+	PF_WRITE	=	1 << 1,
+	PF_USER		=	1 << 2,
+	PF_RSVD		=	1 << 3,
+	PF_INSTR	=	1 << 4,
+
+	PF_PATCH	=	1 << 29,
+	PF_POPULATE	=	1 << 30,
+};
 #endif /* __HEADER_MCCTRL_ARM64_ARCHDEPS_H */

executer/kernel/mcctrl/arch/x86_64/cpu.c

@@ -0,0 +1,51 @@
+/* This is copy of the necessary part from McKernel, for uti-futex */
+
+#include <cpu.h>
+
+/*@
+  @ assigns \nothing;
+  @ behavior to_enabled:
+  @   assumes flags & RFLAGS_IF;
+  @   ensures \interrupt_disabled == 0;
+  @ behavior to_disabled:
+  @   assumes !(flags & RFLAGS_IF);
+  @   ensures \interrupt_disabled > 0;
+  @*/
+void cpu_restore_interrupt(unsigned long flags)
+{
+	asm volatile("push %0; popf" : : "g"(flags) : "memory", "cc");
+}
+
+void cpu_pause(void)
+{
+	asm volatile("pause" ::: "memory");
+}
+
+/*@
+  @ assigns \nothing;
+  @ ensures \interrupt_disabled > 0;
+  @ behavior from_enabled:
+  @   assumes \interrupt_disabled == 0;
+  @   ensures \result & RFLAGS_IF;
+  @ behavior from_disabled:
+  @   assumes \interrupt_disabled > 0;
+  @   ensures !(\result & RFLAGS_IF);
+  @*/
+unsigned long cpu_disable_interrupt_save(void)
+{
+	unsigned long flags;
+
+	asm volatile("pushf; pop %0; cli" : "=r"(flags) : : "memory", "cc");
+
+	return flags;
+}
+
+unsigned long cpu_enable_interrupt_save(void)
+{
+	unsigned long flags;
+
+	asm volatile("pushf; pop %0; sti" : "=r"(flags) : : "memory", "cc");
+
+	return flags;
+}
+

executer/kernel/mcctrl/arch/x86_64/include/arch-lock.h

@@ -0,0 +1,106 @@
+/* This is copy of the necessary part from McKernel, for uti-futex */
+
+#ifndef __HEADER_X86_COMMON_ARCH_LOCK
+#define __HEADER_X86_COMMON_ARCH_LOCK
+
+#include <linux/preempt.h>
+#include <cpu.h>
+
+#define ihk_mc_spinlock_lock __ihk_mc_spinlock_lock
+#define ihk_mc_spinlock_unlock __ihk_mc_spinlock_unlock
+
+#define ihk_mc_spinlock_lock_noirq __ihk_mc_spinlock_lock_noirq
+#define ihk_mc_spinlock_unlock_noirq __ihk_mc_spinlock_unlock_noirq
+
+typedef unsigned short __ticket_t;
+typedef unsigned int __ticketpair_t;
+
+/* arch/x86/include/asm/spinlock_types.h defines struct __raw_tickets */
+typedef struct ihk_spinlock {
+	union {
+		__ticketpair_t head_tail;
+		struct ihk__raw_tickets {
+			__ticket_t head, tail;
+		} tickets;
+	};
+} _ihk_spinlock_t;
+
+static inline void ihk_mc_spinlock_init(_ihk_spinlock_t *lock)
+{
+	lock->head_tail = 0;
+}
+
+static inline void __ihk_mc_spinlock_lock_noirq(_ihk_spinlock_t *lock)
+{
+	register struct ihk__raw_tickets inc = { .tail = 0x0002 };
+
+	preempt_disable();
+
+	asm volatile ("lock xaddl %0, %1\n"
+			: "+r" (inc), "+m" (*(lock)) : : "memory", "cc");
+
+	if (inc.head == inc.tail)
+		goto out;
+
+	for (;;) {
+		if (*((volatile __ticket_t *)&lock->tickets.head) == inc.tail)
+			goto out;
+		cpu_pause();
+	}
+
+out:
+	barrier();  /* make sure nothing creeps before the lock is taken */
+}
+
+static inline void __ihk_mc_spinlock_unlock_noirq(_ihk_spinlock_t *lock)
+{
+	__ticket_t inc = 0x0002;
+
+	asm volatile ("lock addw %1, %0\n"
+			: "+m" (lock->tickets.head)
+			: "ri" (inc) : "memory", "cc");
+
+	preempt_enable();
+}
+
+static inline unsigned long __ihk_mc_spinlock_lock(_ihk_spinlock_t *lock)
+{
+	unsigned long flags;
+
+	flags = cpu_disable_interrupt_save();
+
+	__ihk_mc_spinlock_lock_noirq(lock);
+
+	return flags;
+}
+
+static inline void __ihk_mc_spinlock_unlock(_ihk_spinlock_t *lock,
+		unsigned long flags)
+{
+	__ihk_mc_spinlock_unlock_noirq(lock);
+
+	cpu_restore_interrupt(flags);
+}
+
+typedef struct mcs_rwlock_lock {
+	_ihk_spinlock_t slock;
+
+#ifndef ENABLE_UBSAN
+} __aligned(64) mcs_rwlock_lock_t;
+#else
+} mcs_rwlock_lock_t;
+#endif
+
+static inline void
+mcs_rwlock_writer_lock_noirq(struct mcs_rwlock_lock *lock)
+{
+	ihk_mc_spinlock_lock_noirq(&lock->slock);
+}
+
+static inline void
+mcs_rwlock_writer_unlock_noirq(struct mcs_rwlock_lock *lock)
+{
+	ihk_mc_spinlock_unlock_noirq(&lock->slock);
+}
+
+#endif

executer/kernel/mcctrl/arch/x86_64/include/archdeps.h

@@ -23,4 +23,26 @@ static const unsigned long arch_rus_vm_flags = VM_RESERVED | VM_MIXEDMAP;
 #else
 static const unsigned long arch_rus_vm_flags = VM_DONTDUMP | VM_MIXEDMAP;
 #endif
+
+#define xchg4(ptr, x)						\
+({									\
+	int __x = (x);					\
+	asm volatile("xchgl %k0,%1"				\
+			 : "=r" (__x)				\
+			 : "m" (*ptr), "0" (__x)		\
+			 : "memory");				\
+	__x;								\
+})
+
+enum x86_pf_error_code {
+	PF_PROT     =       1 << 0,
+	PF_WRITE    =       1 << 1,
+	PF_USER     =       1 << 2,
+	PF_RSVD     =       1 << 3,
+	PF_INSTR    =       1 << 4,
+
+	PF_PATCH    =       1 << 29,
+	PF_POPULATE =       1 << 30,
+};
+
 #endif /* __HEADER_MCCTRL_X86_64_ARCHDEPS_H */

executer/kernel/mcctrl/control.c

@@ -50,6 +50,8 @@
 #include <uapi/linux/sched/types.h>
 #endif
 #include <archdeps.h>
+#include <uti.h>
+#include <futex.h>
 
 //#define DEBUG
 
@@ -2891,57 +2893,28 @@ static long mcexec_release_user_space(struct release_user_space_desc *__user arg
 #endif
 }
 
- static long (*mckernel_do_futex)(int n, unsigned long arg0, unsigned long arg1,
-			  unsigned long arg2, unsigned long arg3,
-			  unsigned long arg4, unsigned long arg5,
-				   unsigned long _uti_clv,
-				   void *uti_futex_resp,
-				   void *_linux_wait_event,
-							  void *_linux_printk,
-							  void *_linux_clock_gettime);
+/* Convert phys_addr to virt_addr on Linux */
+static void
+uti_info_p2v(struct uti_info *info)
+{
+	info->uti_futex_resp =
+		(void *)phys_to_virt(info->uti_futex_resp_pa);
+	info->ikc2linux =
+		(void *)phys_to_virt(info->ikc2linux_pa);
 
- long uti_wait_event(void *_resp, unsigned long nsec_timeout) {
-	 struct uti_futex_resp *resp = _resp;
-	 if (nsec_timeout) {
-		 return wait_event_interruptible_timeout(resp->wq, resp->done, nsecs_to_jiffies(nsec_timeout));
-	 } else {
-		 return wait_event_interruptible(resp->wq, resp->done);
-	 }
- }
+	info->status =
+		(void *)phys_to_virt(info->status_pa);
+	info->spin_sleep_lock =
+		(void *)phys_to_virt(info->spin_sleep_lock_pa);
+	info->spin_sleep =
+		(void *)phys_to_virt(info->spin_sleep_pa);
+	info->vm =
+		(void *)phys_to_virt(info->vm_pa);
+	info->futex_q =
+		(void *)phys_to_virt(info->futex_q_pa);
 
- int uti_printk(const char *fmt, ...) {
-	 int sum = 0, nwritten;
-	 va_list args;
-	 va_start(args, fmt);
-	 nwritten = vprintk(fmt, args);
-	 sum += nwritten;
-	 va_end(args);
-	 return sum;
- }
-
-int uti_clock_gettime(clockid_t clk_id, struct timespec *tp) {
-	int ret = 0;
-	struct timespec64 ts64;
-	dprintk("%s: clk_id=%x,REALTIME=%x,MONOTONIC=%x\n", __FUNCTION__, clk_id, CLOCK_REALTIME, CLOCK_MONOTONIC);
-	switch(clk_id) {
-	case CLOCK_REALTIME:
-		getnstimeofday64(&ts64);
-		tp->tv_sec = ts64.tv_sec;
-		tp->tv_nsec = ts64.tv_nsec;
-		dprintk("%s: CLOCK_REALTIME,%ld.%09ld\n", __FUNCTION__, tp->tv_sec, tp->tv_nsec);
-		break;
-	case CLOCK_MONOTONIC: {
-		/* Do not use getrawmonotonic() because it returns different value than clock_gettime() */
-		ktime_get_ts64(&ts64);
-		tp->tv_sec = ts64.tv_sec;
-		tp->tv_nsec = ts64.tv_nsec;
-		dprintk("%s: CLOCK_MONOTONIC,%ld.%09ld\n", __FUNCTION__, tp->tv_sec, tp->tv_nsec);
-		break; }
-	default:
-		ret = -EINVAL;
-		break;
-	}
-	return ret;
+	info->futex_queue =
+		(void *)phys_to_virt(info->futex_queue_pa);
 }
 
 long mcexec_syscall_thread(ihk_os_t os, unsigned long arg, struct file *file)
@@ -2950,36 +2923,38 @@ long mcexec_syscall_thread(ihk_os_t os, unsigned long arg, struct file *file)
 		int number;
 		unsigned long args[6];
 		unsigned long ret;
-		unsigned long uti_clv; /* copy of a clv in McKernel */
+		unsigned long uti_info; /* reference to data in McKernel */
 	};
 	struct syscall_struct param;
 	struct syscall_struct __user *uparam =
 	                              (struct syscall_struct __user *)arg;
 	long rc;
 
-
 	if (copy_from_user(&param, uparam, sizeof param)) {
 		return -EFAULT;
 	}
 
+
 	if (param.number == __NR_futex) {
 		struct uti_futex_resp resp = {
 			.done = 0
 		};
+		struct uti_info *_uti_info = NULL;
+
 		init_waitqueue_head(&resp.wq);
+		_uti_info = (struct uti_info *)param.uti_info;
 
- 		if (!mckernel_do_futex) {
-			if (ihk_os_get_special_address(os, IHK_SPADDR_MCKERNEL_DO_FUTEX,
-										   (unsigned long *)&mckernel_do_futex,
-										   NULL)) {
-				kprintf("%s: ihk_os_get_special_address failed\n", __FUNCTION__);
-				return -EINVAL;
-			}
-			dprintk("%s: mckernel_do_futex=%p\n", __FUNCTION__, mckernel_do_futex);
-		}
+		/* Convert phys_addr to virt_addr on Linux */
+		uti_info_p2v(_uti_info);
+
+		_uti_info->os = (void *)os;
+		
+		rc = do_futex(param.number, param.args[0],
+				param.args[1], param.args[2],
+				param.args[3], param.args[4], param.args[5],
+				(struct uti_info *)param.uti_info,
+				(void *)&resp);
 
-		rc = (*mckernel_do_futex)(param.number, param.args[0], param.args[1], param.args[2],
-							  param.args[3], param.args[4], param.args[5], param.uti_clv, (void *)&resp, (void *)uti_wait_event, (void *)uti_printk, (void *)uti_clock_gettime);
 		param.ret = rc;
 	} else {
 		struct mcctrl_usrdata *usrdata = ihk_host_os_get_usrdata(os);

executer/kernel/mcctrl/include/cpu.h

@@ -0,0 +1,10 @@
+/* This is copy of the necessary part from McKernel, for uti-futex */
+#ifndef MC_CPU_H
+#define MC_CPU_H
+
+void cpu_restore_interrupt(unsigned long flags);
+void cpu_pause(void);
+unsigned long cpu_disable_interrupt_save(void);
+unsigned long cpu_enable_interrupt_save(void);
+
+#endif

executer/kernel/mcctrl/include/futex.h

@@ -0,0 +1,169 @@
+/* This is copy of the necessary part from McKernel, for uti-futex */
+
+#ifndef _FUTEX_H
+#define _FUTEX_H
+
+#include <mc_plist.h>
+#include <arch-lock.h>
+#include <linux/uaccess.h>
+
+/** \name Futex Commands
+ * @{
+ */
+#define FUTEX_WAIT		0
+#define FUTEX_WAKE		1
+#define FUTEX_FD		2
+#define FUTEX_REQUEUE		3
+#define FUTEX_CMP_REQUEUE	4
+#define FUTEX_WAKE_OP		5
+#define FUTEX_LOCK_PI		6
+#define FUTEX_UNLOCK_PI		7
+#define FUTEX_TRYLOCK_PI	8
+#define FUTEX_WAIT_BITSET	9
+#define FUTEX_WAKE_BITSET	10
+#define FUTEX_WAIT_REQUEUE_PI	11
+#define FUTEX_CMP_REQUEUE_PI	12
+// @}
+
+#define FUTEX_PRIVATE_FLAG	128
+#define FUTEX_CLOCK_REALTIME	256
+#define FUTEX_CMD_MASK		~(FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME)
+
+#define FUTEX_WAIT_PRIVATE	(FUTEX_WAIT | FUTEX_PRIVATE_FLAG)
+#define FUTEX_WAKE_PRIVATE	(FUTEX_WAKE | FUTEX_PRIVATE_FLAG)
+#define FUTEX_REQUEUE_PRIVATE	(FUTEX_REQUEUE | FUTEX_PRIVATE_FLAG)
+#define FUTEX_CMP_REQUEUE_PRIVATE (FUTEX_CMP_REQUEUE | FUTEX_PRIVATE_FLAG)
+#define FUTEX_WAKE_OP_PRIVATE	(FUTEX_WAKE_OP | FUTEX_PRIVATE_FLAG)
+#define FUTEX_LOCK_PI_PRIVATE	(FUTEX_LOCK_PI | FUTEX_PRIVATE_FLAG)
+#define FUTEX_UNLOCK_PI_PRIVATE	(FUTEX_UNLOCK_PI | FUTEX_PRIVATE_FLAG)
+#define FUTEX_TRYLOCK_PI_PRIVATE (FUTEX_TRYLOCK_PI | FUTEX_PRIVATE_FLAG)
+#define FUTEX_WAIT_BITSET_PRIVATE	(FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG)
+#define FUTEX_WAKE_BITSET_PRIVATE	(FUTEX_WAKE_BITSET | FUTEX_PRIVATE_FLAG)
+#define FUTEX_WAIT_REQUEUE_PI_PRIVATE	(FUTEX_WAIT_REQUEUE_PI | \
+					 FUTEX_PRIVATE_FLAG)
+#define FUTEX_CMP_REQUEUE_PI_PRIVATE	(FUTEX_CMP_REQUEUE_PI | \
+					 FUTEX_PRIVATE_FLAG)
+
+
+/** \name Futex Operations, used for FUTEX_WAKE_OP
+ * @{
+ */
+#define FUTEX_OP_SET		0	/* *(int *)UADDR2 = OPARG; */
+#define FUTEX_OP_ADD		1	/* *(int *)UADDR2 += OPARG; */
+#define FUTEX_OP_OR		2	/* *(int *)UADDR2 |= OPARG; */
+#define FUTEX_OP_ANDN		3	/* *(int *)UADDR2 &= ~OPARG; */
+#define FUTEX_OP_XOR		4	/* *(int *)UADDR2 ^= OPARG; */
+
+#define FUTEX_OP_OPARG_SHIFT	8U	/* Use (1 << OPARG) instead of OPARG. */
+
+#define FUTEX_OP_CMP_EQ		0	/* if (oldval == CMPARG) wake */
+#define FUTEX_OP_CMP_NE		1	/* if (oldval != CMPARG) wake */
+#define FUTEX_OP_CMP_LT		2	/* if (oldval < CMPARG) wake */
+#define FUTEX_OP_CMP_LE		3	/* if (oldval <= CMPARG) wake */
+#define FUTEX_OP_CMP_GT		4	/* if (oldval > CMPARG) wake */
+#define FUTEX_OP_CMP_GE		5	/* if (oldval >= CMPARG) wake */
+// @}
+
+#define FUT_OFF_INODE    1 /* We set bit 0 if key has a reference on inode */
+#define FUT_OFF_MMSHARED 2 /* We set bit 1 if key has a reference on mm */
+
+#define FUTEX_HASHBITS		8	/* 256 entries in each futex hash tbl */
+
+#define PS_RUNNING           0x1
+#define PS_INTERRUPTIBLE     0x2
+#define PS_UNINTERRUPTIBLE   0x4
+#define PS_ZOMBIE            0x8
+#define PS_EXITED            0x10
+#define PS_STOPPED           0x20
+
+static inline int get_futex_value_locked(uint32_t *dest, uint32_t *from)
+{
+	int ret;
+
+	pagefault_disable();
+	ret = __get_user(*dest, from);
+	pagefault_enable();
+
+	return ret ? -EFAULT : 0;
+}
+
+union futex_key {
+	struct {
+		unsigned long pgoff;
+		void *phys;
+		int offset;
+	} shared;
+	struct {
+		unsigned long address;
+		void *mm;    // Acctually, process_vm
+		int offset;
+	} private;
+	struct {
+		unsigned long word;
+		void *ptr;
+		int offset;
+	} both;
+};
+
+#define FUTEX_KEY_INIT ((union futex_key) { .both = { .ptr = NULL } })
+
+#define FUTEX_BITSET_MATCH_ANY	0xffffffff
+
+/**
+ * struct futex_q - The hashed futex queue entry, one per waiting task
+ * @task:		the task waiting on the futex
+ * @lock_ptr:		the hash bucket lock
+ * @key:		the key the futex is hashed on
+ * @requeue_pi_key:	the requeue_pi target futex key
+ * @bitset:		bitset for the optional bitmasked wakeup
+ *
+ * We use this hashed waitqueue, instead of a normal wait_queue_t, so
+ * we can wake only the relevant ones (hashed queues may be shared).
+ *
+ * A futex_q has a woken state, just like tasks have TASK_RUNNING.
+ * It is considered woken when plist_node_empty(&q->list) || q->lock_ptr == 0.
+ * The order of wakup is always to make the first condition true, then
+ * the second.
+ *
+ * PI futexes are typically woken before they are removed from the hash list via
+ * the rt_mutex code. See unqueue_me_pi().
+ */
+struct futex_q {
+	struct mc_plist_node list;
+
+	void *task;    // Actually, struct thread
+	_ihk_spinlock_t *lock_ptr;
+	union futex_key key;
+	union futex_key *requeue_pi_key;
+	uint32_t bitset;
+
+	/* Used to wake-up a thread running on a Linux CPU */
+	void *uti_futex_resp;
+
+	/* Used to wake-up a thread running on a McKernel from Linux */
+	void *th_spin_sleep;
+	void *th_status;
+	void *th_spin_sleep_lock;
+	void *proc_status;
+	void *proc_update_lock;
+	void *runq_lock;
+	void *clv_flags;
+	int intr_id;
+	int intr_vector;
+
+	unsigned long th_spin_sleep_pa;
+	unsigned long th_status_pa;
+	unsigned long th_spin_sleep_lock_pa;
+	unsigned long proc_status_pa;
+	unsigned long proc_update_lock_pa;
+	unsigned long runq_lock_pa;
+	unsigned long clv_flags_pa;
+};
+
+long do_futex(int n, unsigned long arg0, unsigned long arg1,
+		unsigned long arg2, unsigned long arg3,
+		unsigned long arg4, unsigned long arg5,
+		struct uti_info *uti_info,
+		void *uti_futex_resp);
+
+#endif

executer/kernel/mcctrl/include/mc_plist.h

@@ -0,0 +1,277 @@
+/* This is copy of the necessary part from McKernel, for uti-futex */
+
+/*
+ * Descending-priority-sorted double-linked list
+ *
+ * (C) 2002-2003 Intel Corp
+ * Inaky Perez-Gonzalez <inaky.perez-gonzalez@intel.com>.
+ *
+ * 2001-2005 (c) MontaVista Software, Inc.
+ * Daniel Walker <dwalker@mvista.com>
+ *
+ * (C) 2005 Thomas Gleixner <tglx@linutronix.de>
+ *
+ * Simplifications of the original code by
+ * Oleg Nesterov <oleg@tv-sign.ru>
+ *
+ * Licensed under the FSF's GNU Public License v2 or later.
+ *
+ * Based on simple lists (include/linux/list.h).
+ *
+ * This is a priority-sorted list of nodes; each node has a
+ * priority from INT_MIN (highest) to INT_MAX (lowest).
+ *
+ * Addition is O(K), removal is O(1), change of priority of a node is
+ * O(K) and K is the number of RT priority levels used in the system.
+ * (1 <= K <= 99)
+ *
+ * This list is really a list of lists:
+ *
+ *  - The tier 1 list is the prio_list, different priority nodes.
+ *
+ *  - The tier 2 list is the node_list, serialized nodes.
+ *
+ * Simple ASCII art explanation:
+ *
+ * |HEAD          |
+ * |              |
+ * |prio_list.prev|<------------------------------------|
+ * |prio_list.next|<->|pl|<->|pl|<--------------->|pl|<-|
+ * |10            |   |10|   |21|   |21|   |21|   |40|   (prio)
+ * |              |   |  |   |  |   |  |   |  |   |  |
+ * |              |   |  |   |  |   |  |   |  |   |  |
+ * |node_list.next|<->|nl|<->|nl|<->|nl|<->|nl|<->|nl|<-|
+ * |node_list.prev|<------------------------------------|
+ *
+ * The nodes on the prio_list list are sorted by priority to simplify
+ * the insertion of new nodes. There are no nodes with duplicate
+ * priorites on the list.
+ *
+ * The nodes on the node_list are ordered by priority and can contain
+ * entries which have the same priority. Those entries are ordered
+ * FIFO
+ *
+ * Addition means: look for the prio_list node in the prio_list
+ * for the priority of the node and insert it before the node_list
+ * entry of the next prio_list node. If it is the first node of
+ * that priority, add it to the prio_list in the right position and
+ * insert it into the serialized node_list list
+ *
+ * Removal means remove it from the node_list and remove it from
+ * the prio_list if the node_list list_head is non empty. In case
+ * of removal from the prio_list it must be checked whether other
+ * entries of the same priority are on the list or not. If there
+ * is another entry of the same priority then this entry has to
+ * replace the removed entry on the prio_list. If the entry which
+ * is removed is the only entry of this priority then a simple
+ * remove from both list is sufficient.
+ *
+ * INT_MIN is the highest priority, 0 is the medium highest, INT_MAX
+ * is lowest priority.
+ *
+ * No locking is done, up to the caller.
+ *
+ */
+#ifndef _MC_PLIST_H_
+#define _MC_PLIST_H_
+
+#include <arch-lock.h>
+
+struct mc_plist_head {
+	struct list_head prio_list;
+	struct list_head node_list;
+#ifdef CONFIG_DEBUG_PI_LIST
+	raw_spinlock_t *rawlock;
+	spinlock_t *spinlock;
+#endif
+};
+
+struct mc_plist_node {
+	int			prio;
+	struct mc_plist_head	plist;
+};
+
+#ifdef CONFIG_DEBUG_PI_LIST
+# define PLIST_HEAD_LOCK_INIT(_lock)		.spinlock = _lock
+# define PLIST_HEAD_LOCK_INIT_RAW(_lock)	.rawlock = _lock
+#else
+# define PLIST_HEAD_LOCK_INIT(_lock)
+# define PLIST_HEAD_LOCK_INIT_RAW(_lock)
+#endif
+
+#define _MCK_PLIST_HEAD_INIT(head)				\
+	.prio_list = LIST_HEAD_INIT((head).prio_list),	\
+	.node_list = LIST_HEAD_INIT((head).node_list)
+
+/**
+ * PLIST_HEAD_INIT - static struct plist_head initializer
+ * @head:	struct plist_head variable name
+ * @_lock:	lock to initialize for this list
+ */
+#define MCK_PLIST_HEAD_INIT(head, _lock)			\
+{							\
+	_MCK_PLIST_HEAD_INIT(head),				\
+	MCK_PLIST_HEAD_LOCK_INIT(&(_lock))			\
+}
+
+/**
+ * PLIST_HEAD_INIT_RAW - static struct plist_head initializer
+ * @head:	struct plist_head variable name
+ * @_lock:	lock to initialize for this list
+ */
+#define MCK_PLIST_HEAD_INIT_RAW(head, _lock)		\
+{							\
+	_MCK_PLIST_HEAD_INIT(head),				\
+	MCK_PLIST_HEAD_LOCK_INIT_RAW(&(_lock))		\
+}
+
+/**
+ * PLIST_NODE_INIT - static struct plist_node initializer
+ * @node:	struct plist_node variable name
+ * @__prio:	initial node priority
+ */
+#define MCK_PLIST_NODE_INIT(node, __prio)			\
+{							\
+	.prio  = (__prio),				\
+	.plist = { _MCK_PLIST_HEAD_INIT((node).plist) },	\
+}
+
+/**
+ * plist_head_init - dynamic struct plist_head initializer
+ * @head:	&struct plist_head pointer
+ * @lock:	spinlock protecting the list (debugging)
+ */
+static inline void
+mc_plist_head_init(struct mc_plist_head *head, _ihk_spinlock_t *lock)
+{
+	INIT_LIST_HEAD(&head->prio_list);
+	INIT_LIST_HEAD(&head->node_list);
+#ifdef CONFIG_DEBUG_PI_LIST
+	head->spinlock = lock;
+	head->rawlock = NULL;
+#endif
+}
+
+/**
+ * plist_head_init_raw - dynamic struct plist_head initializer
+ * @head:	&struct plist_head pointer
+ * @lock:	raw_spinlock protecting the list (debugging)
+ */
+static inline void
+mc_plist_head_init_raw(struct mc_plist_head *head, _ihk_spinlock_t *lock)
+{
+	INIT_LIST_HEAD(&head->prio_list);
+	INIT_LIST_HEAD(&head->node_list);
+#ifdef CONFIG_DEBUG_PI_LIST
+	head->rawlock = lock;
+	head->spinlock = NULL;
+#endif
+}
+
+/**
+ * plist_node_init - Dynamic struct plist_node initializer
+ * @node:	&struct plist_node pointer
+ * @prio:	initial node priority
+ */
+static inline void mc_plist_node_init(struct mc_plist_node *node, int prio)
+{
+	node->prio = prio;
+	mc_plist_head_init(&node->plist, NULL);
+}
+
+extern void mc_plist_add(struct mc_plist_node *node,
+		struct mc_plist_head *head);
+extern void mc_plist_del(struct mc_plist_node *node,
+		struct mc_plist_head *head);
+
+/**
+ * plist_for_each - iterate over the plist
+ * @pos:	the type * to use as a loop counter
+ * @head:	the head for your list
+ */
+#define mc_plist_for_each(pos, head)	\
+	 list_for_each_entry(pos, &(head)->node_list, plist.node_list)
+
+/**
+ * plist_for_each_safe - iterate safely over a plist of given type
+ * @pos:	the type * to use as a loop counter
+ * @n:	another type * to use as temporary storage
+ * @head:	the head for your list
+ *
+ * Iterate over a plist of given type, safe against removal of list entry.
+ */
+#define mc_plist_for_each_safe(pos, n, head)	\
+	 list_for_each_entry_safe(pos, n, &(head)->node_list, plist.node_list)
+
+/**
+ * plist_for_each_entry	- iterate over list of given type
+ * @pos:	the type * to use as a loop counter
+ * @head:	the head for your list
+ * @mem:	the name of the list_struct within the struct
+ */
+#define mc_plist_for_each_entry(pos, head, mem)	\
+	 list_for_each_entry(pos, &(head)->node_list, mem.plist.node_list)
+
+/**
+ * plist_for_each_entry_safe - iterate safely over list of given type
+ * @pos:	the type * to use as a loop counter
+ * @n:		another type * to use as temporary storage
+ * @head:	the head for your list
+ * @m:		the name of the list_struct within the struct
+ *
+ * Iterate over list of given type, safe against removal of list entry.
+ */
+#define mc_plist_for_each_entry_safe(pos, n, head, m)	\
+	list_for_each_entry_safe(pos, n, &(head)->node_list, m.plist.node_list)
+
+/**
+ * plist_head_empty - return !0 if a plist_head is empty
+ * @head:	&struct plist_head pointer
+ */
+static inline int mc_plist_head_empty(const struct mc_plist_head *head)
+{
+	return list_empty(&head->node_list);
+}
+
+/**
+ * plist_node_empty - return !0 if plist_node is not on a list
+ * @node:	&struct plist_node pointer
+ */
+static inline int mc_plist_node_empty(const struct mc_plist_node *node)
+{
+	return mc_plist_head_empty(&node->plist);
+}
+
+/* All functions below assume the plist_head is not empty. */
+
+/**
+ * plist_first_entry - get the struct for the first entry
+ * @head:	the &struct plist_head pointer
+ * @type:	the type of the struct this is embedded in
+ * @member:	the name of the list_struct within the struct
+ */
+#ifdef CONFIG_DEBUG_PI_LIST
+# define mc_plist_first_entry(head, type, member)	\
+({ \
+	WARN_ON(mc_plist_head_empty(head)); \
+	container_of(mc_plist_first(head), type, member); \
+})
+#else
+# define mc_plist_first_entry(head, type, member)	\
+	container_of(mc_plist_first(head), type, member)
+#endif
+
+/**
+ * plist_first - return the first node (and thus, highest priority)
+ * @head:	the &struct plist_head pointer
+ *
+ * Assumes the plist is _not_ empty.
+ */
+static inline struct mc_plist_node *mc_plist_first(
+		const struct mc_plist_head *head)
+{
+	return list_entry(head->node_list.next,
+			  struct mc_plist_node, plist.node_list);
+}
+
+#endif

executer/kernel/mcctrl/mc_plist.c

@@ -0,0 +1,100 @@
+/* This is copy of the necessary part from McKernel, for uti-futex */
+
+#include <mc_plist.h>
+#include <arch-lock.h>
+
+#ifdef CONFIG_DEBUG_PI_LIST
+
+static void mc_plist_check_prev_next(struct list_head *t, struct list_head *p,
+				  struct list_head *n)
+{
+	WARN(n->prev != p || p->next != n,
+			"top: %p, n: %p, p: %p\n"
+			"prev: %p, n: %p, p: %p\n"
+			"next: %p, n: %p, p: %p\n",
+			 t, t->next, t->prev,
+			p, p->next, p->prev,
+			n, n->next, n->prev);
+}
+
+static void mc_plist_check_list(struct list_head *top)
+{
+	struct list_head *prev = top, *next = top->next;
+
+	mc_plist_check_prev_next(top, prev, next);
+	while (next != top) {
+		prev = next;
+		next = prev->next;
+		mc_plist_check_prev_next(top, prev, next);
+	}
+}
+
+static void mc_plist_check_head(struct mc_plist_head *head)
+{
+	WARN_ON(!head->rawlock && !head->spinlock);
+	if (head->rawlock)
+		WARN_ON_SMP(!raw_spin_is_locked(head->rawlock));
+	if (head->spinlock)
+		WARN_ON_SMP(!spin_is_locked(head->spinlock));
+	mc_plist_check_list(&head->prio_list);
+	mc_plist_check_list(&head->node_list);
+}
+
+#else
+# define mc_plist_check_head(h)	do { } while (0)
+#endif
+
+/**
+ * plist_add - add @node to @head
+ *
+ * @node:	&struct plist_node pointer
+ * @head:	&struct plist_head pointer
+ */
+void mc_plist_add(struct mc_plist_node *node, struct mc_plist_head *head)
+{
+	struct mc_plist_node *iter;
+
+	mc_plist_check_head(head);
+#if 0
+	WARN_ON(!plist_node_empty(node));
+#endif
+
+	list_for_each_entry(iter, &head->prio_list, plist.prio_list) {
+		if (node->prio < iter->prio)
+			goto lt_prio;
+		else if (node->prio == iter->prio) {
+			iter = list_entry(iter->plist.prio_list.next,
+					struct mc_plist_node, plist.prio_list);
+			goto eq_prio;
+		}
+	}
+
+lt_prio:
+	list_add_tail(&node->plist.prio_list, &iter->plist.prio_list);
+eq_prio:
+	list_add_tail(&node->plist.node_list, &iter->plist.node_list);
+
+	mc_plist_check_head(head);
+}
+
+/**
+ * plist_del - Remove a @node from plist.
+ *
+ * @node:	&struct plist_node pointer - entry to be removed
+ * @head:	&struct plist_head pointer - list head
+ */
+void mc_plist_del(struct mc_plist_node *node, struct mc_plist_head *head)
+{
+	mc_plist_check_head(head);
+
+	if (!list_empty(&node->plist.prio_list)) {
+		struct mc_plist_node *next = mc_plist_first(&node->plist);
+
+		list_move_tail(&next->plist.prio_list, &node->plist.prio_list);
+		list_del_init(&node->plist.prio_list);
+	}
+
+	list_del_init(&node->plist.node_list);
+
+	mc_plist_check_head(head);
+}

executer/user/mcexec.c

@@ -2962,7 +2962,9 @@ static void kill_thread(unsigned long tid, int sig,
 	}
 }
 
-static long util_thread(struct thread_data_s *my_thread, unsigned long rp_rctx, int remote_tid, unsigned long pattr, unsigned long uti_clv, unsigned long _uti_desc)
+static long util_thread(struct thread_data_s *my_thread,
+		unsigned long rp_rctx, int remote_tid, unsigned long pattr,
+		unsigned long uti_info, unsigned long _uti_desc)
 {
 	struct uti_get_ctx_desc get_ctx_desc;
 	struct uti_switch_ctx_desc switch_ctx_desc;
@@ -3014,7 +3016,7 @@ static long util_thread(struct thread_data_s *my_thread, unsigned long rp_rctx,
 	uti_desc->key = get_ctx_desc.key;
 	uti_desc->pid = getpid();
 	uti_desc->tid = gettid();
-	uti_desc->uti_clv = uti_clv;
+	uti_desc->uti_info = uti_info;
 	
 	/* Initialize list of syscall arguments for syscall_intercept */
 	if (sizeof(struct syscall_struct) * 11 > page_size) {
@@ -4823,7 +4825,8 @@ return_execve2:
 		case __NR_sched_setaffinity:
 			if (w.sr.args[0] == 0) {
 				ret = util_thread(my_thread, w.sr.args[1], w.sr.rtid,
-				                  w.sr.args[2], w.sr.args[3], w.sr.args[4]);
+						w.sr.args[2], w.sr.args[3],
+						w.sr.args[4]);
 			}
 			else {
 				__eprintf("__NR_sched_setaffinity: invalid argument (%lx)\n", w.sr.args[0]);

executer/user/syscall_intercept.c

@@ -5,6 +5,7 @@
 #include <syscall.h>
 #include <sys/time.h>
 #include <sys/resource.h>
+#include <sys/types.h> /* for pid_t in uprotocol.h */
 #include "../include/uprotocol.h"
 #include "../include/uti.h"
 #include "./archdep_uti.h"
@@ -76,7 +77,7 @@ hook(long syscall_number,
 		uti_desc.syscall_stack[stack_top].args[3] = arg3;
 		uti_desc.syscall_stack[stack_top].args[4] = arg4;
 		uti_desc.syscall_stack[stack_top].args[5] = arg5;
-		uti_desc.syscall_stack[stack_top].uti_clv = uti_desc.uti_clv;
+		uti_desc.syscall_stack[stack_top].uti_info = uti_desc.uti_info;
 		uti_desc.syscall_stack[stack_top].ret = -EINVAL;
 
 		ret = uti_syscall3(__NR_ioctl, uti_desc.fd,

kernel/futex.c

@@ -62,7 +62,7 @@
 
 #include <process.h>
 #include <futex.h>
-#include <jhash.h>
+#include <mc_jhash.h>
 #include <ihk/lock.h>
 #include <ihk/atomic.h>
 #include <list.h>
@@ -72,39 +72,24 @@
 #include <timer.h>
 #include <ihk/debug.h>
 #include <syscall.h>
-
-//#define DEBUG_PRINT_FUTEX
-
-#ifdef DEBUG_PRINT_FUTEX
-#undef DDEBUG_DEFAULT
-#define DDEBUG_DEFAULT DDEBUG_PRINT
-#define uti_dkprintf(...) do { ((clv_override && linux_printk) ? (*linux_printk) : kprintf)(__VA_ARGS__); } while (0)
-#else
-#define uti_dkprintf(...) do { } while (0)
-#endif
-#define uti_kprintf(...) do { ((clv_override && linux_printk) ? (*linux_printk) : kprintf)(__VA_ARGS__); } while (0)
+#include <kmalloc.h>
 
 
 unsigned long ihk_mc_get_ns_per_tsc(void);
 
-/*
- * Hash buckets are shared by all the futex_keys that hash to the same
- * location.  Each key may have multiple futex_q structures, one for each task
- * waiting on a futex.
- */
-struct futex_hash_bucket {
-	ihk_spinlock_t lock;
-	struct plist_head chain;
-};
+struct futex_hash_bucket *futex_queues;
 
-static struct futex_hash_bucket futex_queues[1<<FUTEX_HASHBITS];
+struct futex_hash_bucket *get_futex_queues(void)
+{
+	return futex_queues;
+}
 
 /*
  * We hash on the keys returned from get_futex_key (see below).
  */
 static struct futex_hash_bucket *hash_futex(union futex_key *key)
 {
-	uint32_t hash = jhash2((uint32_t*)&key->both.word,
+	uint32_t hash = mc_jhash2((uint32_t *)&key->both.word,
 			  (sizeof(key->both.word)+sizeof(key->both.ptr))/4,
 			  key->both.offset);
 	return &futex_queues[hash & ((1 << FUTEX_HASHBITS)-1)];
@@ -157,11 +142,11 @@ static void drop_futex_key_refs(union futex_key *key)
  * lock_page() might sleep, the caller should not hold a spinlock.
  */
 static int
-get_futex_key(uint32_t *uaddr, int fshared, union futex_key *key, struct cpu_local_var *clv_override)
+get_futex_key(uint32_t *uaddr, int fshared, union futex_key *key)
 {
 	unsigned long address = (unsigned long)uaddr;
 	unsigned long phys;
-	struct thread *thread = cpu_local_var_with_override(current, clv_override);
+	struct thread *thread = cpu_local_var(current);
 	struct process_vm *mm = thread->vm;
 
 	/*
@@ -228,7 +213,7 @@ static int cmpxchg_futex_value_locked(uint32_t __user *uaddr, uint32_t uval, uin
  * The hash bucket lock must be held when this is called.
  * Afterwards, the futex_q must not be accessed.
  */
-static void wake_futex(struct futex_q *q, struct cpu_local_var *clv_override)
+static void wake_futex(struct futex_q *q)
 {
 	struct thread *p = q->task;
 
@@ -253,26 +238,22 @@ static void wake_futex(struct futex_q *q, struct cpu_local_var *clv_override)
 
 	if (q->uti_futex_resp) { 
 		int rc;
-		uti_dkprintf("wake_futex(): waking up migrated-to-Linux thread (tid %d),uti_futex_resp=%p\n", p->tid, q->uti_futex_resp);
-		/* TODO: Add the case when a Linux thread waking up another Linux thread */
-		if (clv_override) {
-			uti_dkprintf("%s: ERROR: A Linux thread is waking up migrated-to-Linux thread\n", __FUNCTION__);
-		}
-		if (p->spin_sleep == 0) {
-			uti_dkprintf("%s: INFO: woken up by someone else\n", __FUNCTION__);
-		}
+		dkprintf("%s: waking up migrated-to-Linux thread (tid %d),uti_futex_resp=%p\n",
+				__func__, p->tid, q->uti_futex_resp);
 
 		struct ikc_scd_packet pckt;
-		struct ihk_ikc_channel_desc *resp_channel = cpu_local_var_with_override(ikc2linux, clv_override);
+		struct ihk_ikc_channel_desc *resp_channel = cpu_local_var(ikc2linux);
 		pckt.msg = SCD_MSG_FUTEX_WAKE;
 		pckt.futex.resp = q->uti_futex_resp;
 		pckt.futex.spin_sleep = &p->spin_sleep;
 		rc = ihk_ikc_send(resp_channel, &pckt, 0);
 		if (rc) {
-			uti_dkprintf("%s: ERROR: ihk_ikc_send returned %d, resp_channel=%p\n", __FUNCTION__, rc, resp_channel);
+			dkprintf("%s: ERROR: ihk_ikc_send returned %d, resp_channel=%p\n",
+					__func__, rc, resp_channel);
 		}
 	} else {
-		uti_dkprintf("wake_futex(): waking up McKernel thread (tid %d)\n", p->tid);
+		dkprintf("%s: waking up McKernel thread (tid %d)\n",
+				__func__, p->tid);
 		sched_wakeup_thread(p, PS_NORMAL);
 	}
 }
@@ -304,7 +285,8 @@ double_unlock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2)
 /*
  * Wake up waiters matching bitset queued on this futex (uaddr).
  */
-static int futex_wake(uint32_t *uaddr, int fshared, int nr_wake, uint32_t bitset, struct cpu_local_var *clv_override)
+static int futex_wake(uint32_t *uaddr, int fshared, int nr_wake,
+		uint32_t bitset)
 {
 	struct futex_hash_bucket *hb;
 	struct futex_q *this, *next;
@@ -316,7 +298,7 @@ static int futex_wake(uint32_t *uaddr, int fshared, int nr_wake, uint32_t bitset
 	if (!bitset)
 		return -EINVAL;
 
-	ret = get_futex_key(uaddr, fshared, &key, clv_override);
+	ret = get_futex_key(uaddr, fshared, &key);
 	if ((ret != 0))
 		goto out;
 
@@ -332,7 +314,7 @@ static int futex_wake(uint32_t *uaddr, int fshared, int nr_wake, uint32_t bitset
 			if (!(this->bitset & bitset))
 				continue;
 
-			wake_futex(this, clv_override);
+			wake_futex(this);
 			if (++ret >= nr_wake)
 				break;
 		}
@@ -350,8 +332,7 @@ out:
  */
 static int
 futex_wake_op(uint32_t *uaddr1, int fshared, uint32_t *uaddr2,
-			  int nr_wake, int nr_wake2, int op,
-			  struct cpu_local_var *clv_override)
+			  int nr_wake, int nr_wake2, int op)
 {
 	union futex_key key1 = FUTEX_KEY_INIT, key2 = FUTEX_KEY_INIT;
 	struct futex_hash_bucket *hb1, *hb2;
@@ -360,10 +341,10 @@ futex_wake_op(uint32_t *uaddr1, int fshared, uint32_t *uaddr2,
 	int ret, op_ret;
 
 retry:
-	ret = get_futex_key(uaddr1, fshared, &key1, clv_override);
+	ret = get_futex_key(uaddr1, fshared, &key1);
 	if ((ret != 0))
 		goto out;
-	ret = get_futex_key(uaddr2, fshared, &key2, clv_override);
+	ret = get_futex_key(uaddr2, fshared, &key2);
 	if ((ret != 0))
 		goto out_put_key1;
 
@@ -397,7 +378,7 @@ retry_private:
 
 	plist_for_each_entry_safe(this, next, head, list) {
 		if (match_futex (&this->key, &key1)) {
-			wake_futex(this, clv_override);
+			wake_futex(this);
 			if (++ret >= nr_wake)
 				break;
 		}
@@ -409,7 +390,7 @@ retry_private:
 		op_ret = 0;
 		plist_for_each_entry_safe(this, next, head, list) {
 			if (match_futex (&this->key, &key2)) {
-				wake_futex(this, clv_override);
+				wake_futex(this);
 				if (++op_ret >= nr_wake2)
 					break;
 			}
@@ -471,8 +452,8 @@ void requeue_futex(struct futex_q *q, struct futex_hash_bucket *hb1,
  *  <0 - on error
  */
 static int futex_requeue(uint32_t *uaddr1, int fshared, uint32_t *uaddr2,
-			 int nr_wake, int nr_requeue, uint32_t *cmpval,
-						 int requeue_pi, struct cpu_local_var *clv_override)
+		int nr_wake, int nr_requeue, uint32_t *cmpval,
+		int requeue_pi)
 {
 	union futex_key key1 = FUTEX_KEY_INIT, key2 = FUTEX_KEY_INIT;
 	int drop_count = 0, task_count = 0, ret;
@@ -480,10 +461,10 @@ static int futex_requeue(uint32_t *uaddr1, int fshared, uint32_t *uaddr2,
 	struct plist_head *head1;
 	struct futex_q *this, *next;
 
-	ret = get_futex_key(uaddr1, fshared, &key1, clv_override);
+	ret = get_futex_key(uaddr1, fshared, &key1);
 	if ((ret != 0))
 		goto out;
-	ret = get_futex_key(uaddr2, fshared, &key2, clv_override);
+	ret = get_futex_key(uaddr2, fshared, &key2);
 	if ((ret != 0))
 		goto out_put_key1;
 
@@ -518,7 +499,7 @@ static int futex_requeue(uint32_t *uaddr1, int fshared, uint32_t *uaddr2,
 		 */
 		/* RIKEN: no requeue_pi at this moment */
 		if (++task_count <= nr_wake) {
-			wake_futex(this, clv_override);
+			wake_futex(this);
 			continue;
 		}
 
@@ -577,9 +558,12 @@ queue_unlock(struct futex_q *q, struct futex_hash_bucket *hb)
  * state is implicit in the state of woken task (see futex_wait_requeue_pi() for
  * an example).
  */
-static inline void queue_me(struct futex_q *q, struct futex_hash_bucket *hb, struct cpu_local_var *clv_override)
+static inline void queue_me(struct futex_q *q, struct futex_hash_bucket *hb)
 {
 	int prio;
+	struct thread *thread = cpu_local_var(current);
+	ihk_spinlock_t *_runq_lock = &cpu_local_var(runq_lock);
+	unsigned int *_flags = &cpu_local_var(flags);
 
 	/*
 	 * The priority used to register this element is
@@ -598,7 +582,19 @@ static inline void queue_me(struct futex_q *q, struct futex_hash_bucket *hb, str
 	q->list.plist.spinlock = &hb->lock;
 #endif
 	plist_add(&q->list, &hb->chain);
-	q->task = cpu_local_var_with_override(current, clv_override);
+
+	/* Store information about wait thread for uti-futex*/
+	q->task = thread;
+	q->th_spin_sleep_pa = virt_to_phys((void *)&thread->spin_sleep);
+	q->th_status_pa = virt_to_phys((void *)&thread->status);
+	q->th_spin_sleep_lock_pa = virt_to_phys((void *)&thread->spin_sleep_lock);
+	q->proc_status_pa = virt_to_phys((void *)&thread->proc->status);
+	q->proc_update_lock_pa = virt_to_phys((void *)&thread->proc->update_lock);
+	q->runq_lock_pa = virt_to_phys((void *)_runq_lock);
+	q->clv_flags_pa = virt_to_phys((void *)_flags);
+	q->intr_id = ihk_mc_get_interrupt_id(thread->cpu_id);
+	q->intr_vector = ihk_mc_get_vector(IHK_GV_IKC);
+
 	ihk_mc_spinlock_unlock_noirq(&hb->lock);
 }
 
@@ -661,12 +657,12 @@ retry:
 /* RIKEN: this function has been rewritten so that it returns the remaining
  * time in case we are waken.
  */
-static int64_t futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q *q,
-				   uint64_t timeout, struct cpu_local_var *clv_override)
+static int64_t futex_wait_queue_me(struct futex_hash_bucket *hb,
+		struct futex_q *q, uint64_t timeout)
 {
 	int64_t time_remain = 0;
 	unsigned long irqstate;
-	struct thread *thread = cpu_local_var_with_override(current, clv_override);
+	struct thread *thread = cpu_local_var(current);
 	/*
 	 * The task state is guaranteed to be set before another task can
 	 * wake it. 
@@ -685,25 +681,9 @@ static int64_t futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q
 		ihk_mc_spinlock_unlock(&thread->spin_sleep_lock, irqstate);
 	}
 
-	queue_me(q, hb, clv_override);
+	queue_me(q, hb);
 	
 	if (!plist_node_empty(&q->list)) {
-		if (clv_override) {
-			uti_dkprintf("%s: tid: %d is trying to sleep\n", __FUNCTION__, thread->tid);
-			/* Note that the unit of timeout is nsec */
-			time_remain = (*linux_wait_event)(q->uti_futex_resp, timeout);
-			
-			/* Note that time_remain == 0 indicates contidion evaluated to false after the timeout elapsed */
-			if (time_remain < 0) {
-				if (time_remain == -ERESTARTSYS) { /* Interrupted by signal */
-					uti_dkprintf("%s: DEBUG: wait_event returned -ERESTARTSYS\n", __FUNCTION__);
-				} else {
-					uti_kprintf("%s: ERROR: wait_event returned %d\n", __FUNCTION__, time_remain);
-				}
-			}
-			uti_dkprintf("%s: tid: %d woken up\n", __FUNCTION__, thread->tid);
-		} else {
-		
 		if (timeout) {
 			dkprintf("futex_wait_queue_me(): tid: %d schedule_timeout()\n", thread->tid);
 			time_remain = schedule_timeout(timeout);
@@ -714,7 +694,6 @@ static int64_t futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q
 			time_remain = 0;
 		}
 		dkprintf("futex_wait_queue_me(): tid: %d woken up\n", thread->tid);
-		}
 	}
 	
 	/* This does not need to be serialized */
@@ -742,8 +721,7 @@ static int64_t futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q
  * <1 - -EFAULT or -EWOULDBLOCK (uaddr does not contain val) and hb is unlcoked
  */
 static int futex_wait_setup(uint32_t __user *uaddr, uint32_t val, int fshared,
-			    struct futex_q *q, struct futex_hash_bucket **hb,
-			    struct cpu_local_var *clv_override)
+		struct futex_q *q, struct futex_hash_bucket **hb)
 {
 	uint32_t uval;
 	int ret;
@@ -766,7 +744,7 @@ static int futex_wait_setup(uint32_t __user *uaddr, uint32_t val, int fshared,
 	 * rare, but normal.
 	 */
 	q->key = FUTEX_KEY_INIT;
-	ret = get_futex_key(uaddr, fshared, &q->key, clv_override);
+	ret = get_futex_key(uaddr, fshared, &q->key);
 	if (ret != 0)
 		return ret;
 
@@ -790,8 +768,7 @@ static int futex_wait_setup(uint32_t __user *uaddr, uint32_t val, int fshared,
 }
 
 static int futex_wait(uint32_t __user *uaddr, int fshared,
-		      uint32_t val, uint64_t timeout, uint32_t bitset, int clockrt,
-		      struct cpu_local_var *clv_override)
+		uint32_t val, uint64_t timeout, uint32_t bitset, int clockrt)
 {
 	struct futex_hash_bucket *hb;
 	int64_t time_remain;
@@ -802,57 +779,55 @@ static int futex_wait(uint32_t __user *uaddr, int fshared,
 	if (!bitset)
 		return -EINVAL;
 
-	if (!clv_override) {
-		q = &lq;
-	}
-	else {
-		q = &cpu_local_var_with_override(current,
-						 clv_override)->futex_q;
-	}
+	q = &lq;
 
 #ifdef PROFILE_ENABLE
-	if (cpu_local_var_with_override(current, clv_override)->profile &&
-		cpu_local_var_with_override(current, clv_override)->profile_start_ts) {
-		cpu_local_var_with_override(current, clv_override)->profile_elapsed_ts +=
-			(rdtsc() - cpu_local_var_with_override(current, clv_override)->profile_start_ts);
-		cpu_local_var_with_override(current, clv_override)->profile_start_ts = 0;
+	if (cpu_local_var(current)->profile &&
+		cpu_local_var(current)->profile_start_ts) {
+		cpu_local_var(current)->profile_elapsed_ts +=
+			(rdtsc() - cpu_local_var(current)->profile_start_ts);
+		cpu_local_var(current)->profile_start_ts = 0;
 	}
 #endif
 
 	q->bitset = bitset;
 	q->requeue_pi_key = NULL;
-	q->uti_futex_resp = cpu_local_var_with_override(uti_futex_resp,
-							clv_override);
+	q->uti_futex_resp = cpu_local_var(uti_futex_resp);
 
 retry:
 	/* Prepare to wait on uaddr. */
-	ret = futex_wait_setup(uaddr, val, fshared, q, &hb, clv_override);
+	ret = futex_wait_setup(uaddr, val, fshared, q, &hb);
 	if (ret) {
-		uti_dkprintf("%s: tid=%d futex_wait_setup returns zero, no need to sleep\n", __FUNCTION__, cpu_local_var_with_override(current, clv_override)->tid);
+		dkprintf("%s: tid=%d futex_wait_setup returns zero, no need to sleep\n",
+			__func__, cpu_local_var(current)->tid);
 		goto out;
 	}
 
 	/* queue_me and wait for wakeup, timeout, or a signal. */
-	time_remain = futex_wait_queue_me(hb, q, timeout, clv_override);
+	time_remain = futex_wait_queue_me(hb, q, timeout);
 
 	/* If we were woken (and unqueued), we succeeded, whatever. */
 	ret = 0;
 	if (!unqueue_me(q)) {
-		uti_dkprintf("%s: tid=%d unqueued\n", __FUNCTION__, cpu_local_var_with_override(current, clv_override)->tid);
+		dkprintf("%s: tid=%d unqueued\n",
+				__func__, cpu_local_var(current)->tid);
 		goto out_put_key;
 	}
 	ret = -ETIMEDOUT;
 
 	/* RIKEN: timer expired case (indicated by !time_remain) */
 	if (timeout && !time_remain) {
-		uti_dkprintf("%s: tid=%d timer expired\n", __FUNCTION__, cpu_local_var_with_override(current, clv_override)->tid);
+		dkprintf("%s: tid=%d timer expired\n",
+				__func__, cpu_local_var(current)->tid);
 		goto out_put_key;
 	}
 
 	/* RIKEN: futex_wait_queue_me() returns -ERESTARTSYS when waiting on Linux CPU and woken up by signal */
-	if (hassigpending(cpu_local_var_with_override(current, clv_override)) || time_remain == -ERESTARTSYS) {
+	if (hassigpending(cpu_local_var(current)) ||
+			time_remain == -ERESTARTSYS) {
 		ret = -EINTR;
-		uti_dkprintf("%s: tid=%d woken up by signal\n", __FUNCTION__, cpu_local_var_with_override(current, clv_override)->tid);
+		dkprintf("%s: tid=%d woken up by signal\n",
+				__func__, cpu_local_var(current)->tid);
 		goto out_put_key;
 	}
 
@@ -864,21 +839,22 @@ out_put_key:
 	put_futex_key(fshared, &q->key);
 out:
 #ifdef PROFILE_ENABLE
-	if (cpu_local_var_with_override(current, clv_override)->profile) {
-		cpu_local_var_with_override(current, clv_override)->profile_start_ts = rdtsc();
+	if (cpu_local_var(current)->profile) {
+		cpu_local_var(current)->profile_start_ts = rdtsc();
 	}
 #endif
 	return ret;
 }
 
 int futex(uint32_t *uaddr, int op, uint32_t val, uint64_t timeout,
-	  uint32_t *uaddr2, uint32_t val2, uint32_t val3, int fshared,
-	  struct cpu_local_var *clv_override)
+		uint32_t *uaddr2, uint32_t val2, uint32_t val3, int fshared)
 {
 	int clockrt, ret = -ENOSYS;
 	int cmd = op & FUTEX_CMD_MASK;
 
-	uti_dkprintf("%s: uaddr=%p, op=%x, val=%x, timeout=%ld, uaddr2=%p, val2=%x, val3=%x, fshared=%d, clv=%p\n", __FUNCTION__, uaddr, op, val, timeout, uaddr2, val2, val3, fshared, clv_override);
+	dkprintf("%s: uaddr=%p, op=%x, val=%x, timeout=%ld, uaddr2=%p, val2=%x, val3=%x, fshared=%d\n",
+			__func__, uaddr, op, val, timeout, uaddr2,
+			val2, val3, fshared);
 
 	clockrt = op & FUTEX_CLOCK_REALTIME;
 	if (clockrt && cmd != FUTEX_WAIT_BITSET && cmd != FUTEX_WAIT_REQUEUE_PI)
@@ -888,21 +864,23 @@ int futex(uint32_t *uaddr, int op, uint32_t val, uint64_t timeout,
 	case FUTEX_WAIT:
 		val3 = FUTEX_BITSET_MATCH_ANY;
 	case FUTEX_WAIT_BITSET:
-		ret = futex_wait(uaddr, fshared, val, timeout, val3, clockrt, clv_override);
+		ret = futex_wait(uaddr, fshared, val, timeout, val3, clockrt);
 		break;
 	case FUTEX_WAKE:
 		val3 = FUTEX_BITSET_MATCH_ANY;
 	case FUTEX_WAKE_BITSET:
-		ret = futex_wake(uaddr, fshared, val, val3, clv_override);
+		ret = futex_wake(uaddr, fshared, val, val3);
 		break;
 	case FUTEX_REQUEUE:
-		ret = futex_requeue(uaddr, fshared, uaddr2, val, val2, NULL, 0, clv_override);
+		ret = futex_requeue(uaddr, fshared, uaddr2,
+				val, val2, NULL, 0);
 		break;
 	case FUTEX_CMP_REQUEUE:
-		ret = futex_requeue(uaddr, fshared, uaddr2, val, val2, &val3, 0, clv_override);
+		ret = futex_requeue(uaddr, fshared, uaddr2,
+				val, val2, &val3, 0);
 		break;
 	case FUTEX_WAKE_OP:
-		ret = futex_wake_op(uaddr, fshared, uaddr2, val, val2, val3, clv_override);
+		ret = futex_wake_op(uaddr, fshared, uaddr2, val, val2, val3);
 		break;
 	/* RIKEN: these calls are not supported for now.	
 	case FUTEX_LOCK_PI:
@@ -942,7 +920,9 @@ int futex_init(void)
 {
 	int i;
 
-	for (i = 0; i < ARRAY_SIZE(futex_queues); i++) {
+	futex_queues = kmalloc(sizeof(struct futex_hash_bucket) *
+			(1 << FUTEX_HASHBITS), IHK_MC_AP_NOWAIT);
+	for (i = 0; i < (1 << FUTEX_HASHBITS); i++) {
 		plist_head_init(&futex_queues[i].chain, &futex_queues[i].lock);
 		ihk_mc_spinlock_init(&futex_queues[i].lock);
 	}

kernel/include/futex.h

@@ -128,6 +128,26 @@
 
 struct process_vm;
 
+static inline int get_futex_value_locked(uint32_t *dest, uint32_t *from)
+{
+
+	*dest = *(volatile uint32_t *)from;
+
+	return 0;
+}
+
+/*
+ * Hash buckets are shared by all the futex_keys that hash to the same
+ * location.  Each key may have multiple futex_q structures, one for each task
+ * waiting on a futex.
+ */
+struct futex_hash_bucket {
+	ihk_spinlock_t lock;
+	struct plist_head chain;
+};
+
+struct futex_hash_bucket *get_futex_queues(void);
+
 union futex_key {
 	struct {
 		unsigned long pgoff;
@@ -161,8 +181,7 @@ futex(
 	uint32_t __user *		uaddr2,
 	uint32_t				val2,
 	uint32_t				val3,
-	int                     fshared,
-	struct cpu_local_var *clv_override
+	int                     fshared
 );
 
 
@@ -196,6 +215,25 @@ struct futex_q {
 
 	/* Used to wake-up a thread running on a Linux CPU */
 	void *uti_futex_resp;
+
+	/* Used to wake-up a thread running on a McKernel from Linux */
+	void *th_spin_sleep;
+	void *th_status;
+	void *th_spin_sleep_lock;
+	void *proc_status;
+	void *proc_update_lock;
+	void *runq_lock;
+	void *clv_flags;
+	int intr_id;
+	int intr_vector;
+
+	unsigned long th_spin_sleep_pa;
+	unsigned long th_status_pa;
+	unsigned long th_spin_sleep_lock_pa;
+	unsigned long proc_status_pa;
+	unsigned long proc_update_lock_pa;
+	unsigned long runq_lock_pa;
+	unsigned long clv_flags_pa;
 };
 
 #endif

kernel/include/jhash.h

@@ -1,158 +0,0 @@
-#ifndef _LINUX_JHASH_H
-#define _LINUX_JHASH_H
-/**
- * \file futex.c
- * Licence details are found in the file LICENSE.
- *  
- * \brief
- * Adaptation to McKernel
- *
- * \author Balazs Gerofi  <bgerofi@riken.jp> \par
- * Copyright (C) 2012  RIKEN AICS
- *
- *
- * HISTORY:
- */
-
-/* 
- * jhash.h: Jenkins hash support.
- *
- * Copyright (C) 1996 Bob Jenkins (bob_jenkins@burtleburtle.net)
- *
- * http://burtleburtle.net/bob/hash/
- *
- * These are the credits from Bob's sources:
- *
- * lookup2.c, by Bob Jenkins, December 1996, Public Domain.
- * hash(), hash2(), hash3, and mix() are externally useful functions.
- * Routines to test the hash are included if SELF_TEST is defined.
- * You can use this free for any purpose.  It has no warranty.
- *
- * Copyright (C) 2003 David S. Miller (davem@redhat.com)
- *
- * I've modified Bob's hash to be useful in the Linux kernel, and
- * any bugs present are surely my fault.  -DaveM
- * 
- */
-
-/* NOTE: Arguments are modified. */
-#define __jhash_mix(a, b, c) \
-{ \
-  a -= b; a -= c; a ^= (c>>13); \
-  b -= c; b -= a; b ^= (a<<8); \
-  c -= a; c -= b; c ^= (b>>13); \
-  a -= b; a -= c; a ^= (c>>12);  \
-  b -= c; b -= a; b ^= (a<<16); \
-  c -= a; c -= b; c ^= (b>>5); \
-  a -= b; a -= c; a ^= (c>>3);  \
-  b -= c; b -= a; b ^= (a<<10); \
-  c -= a; c -= b; c ^= (b>>15); \
-}
-
-/* The golden ration: an arbitrary value */
-#define JHASH_GOLDEN_RATIO	0x9e3779b9
-
-/* The most generic version, hashes an arbitrary sequence
- * of bytes.  No alignment or length assumptions are made about
- * the input key.
- */
-static inline uint32_t jhash(const void *key, uint32_t length, uint32_t initval)
-{
-	uint32_t a, b, c, len;
-	const uint8_t *k = key;
-
-	len = length;
-	a = b = JHASH_GOLDEN_RATIO;
-	c = initval;
-
-	while (len >= 12) {
-		a += (k[0] +((uint32_t)k[1]<<8) +((uint32_t)k[2]<<16) +((uint32_t)k[3]<<24));
-		b += (k[4] +((uint32_t)k[5]<<8) +((uint32_t)k[6]<<16) +((uint32_t)k[7]<<24));
-		c += (k[8] +((uint32_t)k[9]<<8) +((uint32_t)k[10]<<16)+((uint32_t)k[11]<<24));
-
-		__jhash_mix(a,b,c);
-
-		k += 12;
-		len -= 12;
-	}
-
-	c += length;
-	switch (len) {
-	case 11: c += ((uint32_t)k[10]<<24);
-	case 10: c += ((uint32_t)k[9]<<16);
-	case 9 : c += ((uint32_t)k[8]<<8);
-	case 8 : b += ((uint32_t)k[7]<<24);
-	case 7 : b += ((uint32_t)k[6]<<16);
-	case 6 : b += ((uint32_t)k[5]<<8);
-	case 5 : b += k[4];
-	case 4 : a += ((uint32_t)k[3]<<24);
-	case 3 : a += ((uint32_t)k[2]<<16);
-	case 2 : a += ((uint32_t)k[1]<<8);
-	case 1 : a += k[0];
-	};
-
-	__jhash_mix(a,b,c);
-
-	return c;
-}
-
-/* A special optimized version that handles 1 or more of uint32_ts.
- * The length parameter here is the number of uint32_ts in the key.
- */
-static inline uint32_t jhash2(const uint32_t *k, uint32_t length, uint32_t initval)
-{
-	uint32_t a, b, c, len;
-
-	a = b = JHASH_GOLDEN_RATIO;
-	c = initval;
-	len = length;
-
-	while (len >= 3) {
-		a += k[0];
-		b += k[1];
-		c += k[2];
-		__jhash_mix(a, b, c);
-		k += 3; len -= 3;
-	}
-
-	c += length * 4;
-
-	switch (len) {
-	case 2 : b += k[1];
-	case 1 : a += k[0];
-	};
-
-	__jhash_mix(a,b,c);
-
-	return c;
-}
-
-
-/* A special ultra-optimized versions that knows they are hashing exactly
- * 3, 2 or 1 word(s).
- *
- * NOTE: In partilar the "c += length; __jhash_mix(a,b,c);" normally
- *       done at the end is not done here.
- */
-static inline uint32_t jhash_3words(uint32_t a, uint32_t b, uint32_t c, uint32_t initval)
-{
-	a += JHASH_GOLDEN_RATIO;
-	b += JHASH_GOLDEN_RATIO;
-	c += initval;
-
-	__jhash_mix(a, b, c);
-
-	return c;
-}
-
-static inline uint32_t jhash_2words(uint32_t a, uint32_t b, uint32_t initval)
-{
-	return jhash_3words(a, b, 0, initval);
-}
-
-static inline uint32_t jhash_1word(uint32_t a, uint32_t initval)
-{
-	return jhash_3words(a, 0, 0, initval);
-}
-
-#endif /* _LINUX_JHASH_H */

kernel/include/mc_jhash.h

@@ -0,0 +1,88 @@
+#ifndef _MC_JHASH_H
+#define _MC_JHASH_H
+/**
+ * \file mc_jhash.h
+ * Licence details are found in the file LICENSE.
+ *
+ * \brief
+ * Adaptation to McKernel
+ *
+ * \author Balazs Gerofi  <bgerofi@riken.jp> \par
+ * Copyright (C) 2012  RIKEN AICS
+ *
+ *
+ * HISTORY:
+ */
+
+/*
+ * jhash.h: Jenkins hash support.
+ *
+ * Copyright (C) 1996 Bob Jenkins (bob_jenkins@burtleburtle.net)
+ *
+ * http://burtleburtle.net/bob/hash/
+ *
+ * These are the credits from Bob's sources:
+ *
+ * lookup2.c, by Bob Jenkins, December 1996, Public Domain.
+ * hash(), hash2(), hash3, and mix() are externally useful functions.
+ * Routines to test the hash are included if SELF_TEST is defined.
+ * You can use this free for any purpose.  It has no warranty.
+ *
+ * Copyright (C) 2003 David S. Miller (davem@redhat.com)
+ *
+ * I've modified Bob's hash to be useful in the Linux kernel, and
+ * any bugs present are surely my fault.  -DaveM
+ *
+ */
+
+/* NOTE: Arguments are modified. */
+#define __mc_jhash_mix(a, b, c) \
+{ \
+  a -= b; a -= c; a ^= (c>>13); \
+  b -= c; b -= a; b ^= (a<<8); \
+  c -= a; c -= b; c ^= (b>>13); \
+  a -= b; a -= c; a ^= (c>>12);  \
+  b -= c; b -= a; b ^= (a<<16); \
+  c -= a; c -= b; c ^= (b>>5); \
+  a -= b; a -= c; a ^= (c>>3);  \
+  b -= c; b -= a; b ^= (a<<10); \
+  c -= a; c -= b; c ^= (b>>15); \
+}
+
+/* The golden ration: an arbitrary value */
+#define JHASH_GOLDEN_RATIO	0x9e3779b9
+
+/* A special optimized version that handles 1 or more of uint32_ts.
+ * The length parameter here is the number of uint32_ts in the key.
+ */
+static inline uint32_t mc_jhash2(const uint32_t *k, uint32_t length, uint32_t initval)
+{
+	uint32_t a, b, c, len;
+
+	a = b = JHASH_GOLDEN_RATIO;
+	c = initval;
+	len = length;
+
+	while (len >= 3) {
+		a += k[0];
+		b += k[1];
+		c += k[2];
+		__mc_jhash_mix(a, b, c);
+		k += 3; len -= 3;
+	}
+
+	c += length * 4;
+
+	switch (len) {
+	case 2:
+		b += k[1];
+	case 1:
+		a += k[0];
+	};
+
+	__mc_jhash_mix(a, b, c);
+
+	return c;
+}
+
+#endif /* _MC_JHASH_H */

kernel/init.c

@@ -252,7 +252,6 @@ static void nmi_init()
 
 static void uti_init()
 {
-	ihk_set_mckernel_do_futex((unsigned long)do_futex);
 }
 
 static void rest_init(void)

kernel/syscall.c

@@ -74,13 +74,6 @@
 #define DDEBUG_DEFAULT DDEBUG_PRINT
 #endif
 
-//#define DEBUG_UTI
-#ifdef DEBUG_UTI
-#define uti_dkprintf(...) do { ((uti_clv && linux_printk) ? (*linux_printk) : kprintf)(__VA_ARGS__); } while (0)
-#else
-#define uti_dkprintf(...) do { } while (0)
-#endif
-
 //static ihk_atomic_t pid_cnt = IHK_ATOMIC_INIT(1024);
 
 /* generate system call handler's prototypes */
@@ -6907,7 +6900,7 @@ long do_futex(int n, unsigned long arg0, unsigned long arg1,
 	}
 	op = (op & FUTEX_CMD_MASK);
 	
-	uti_dkprintf("futex op=[%x, %s],uaddr=%lx, val=%x, utime=%lx, uaddr2=%lx, val3=%x, []=%x, shared: %d\n", 
+	dkprintf("futex op=[%x, %s],uaddr=%lx, val=%x, utime=%lx, uaddr2=%lx, val3=%x, []=%x, shared: %d\n",
 			flags,
 			(op == FUTEX_WAIT) ? "FUTEX_WAIT" :
 			(op == FUTEX_WAIT_BITSET) ? "FUTEX_WAIT_BITSET" :
@@ -6919,7 +6912,8 @@ long do_futex(int n, unsigned long arg0, unsigned long arg1,
 			(unsigned long)uaddr, val, utime, uaddr2, val3, *uaddr, fshared);
 
 	if ((op == FUTEX_WAIT || op == FUTEX_WAIT_BITSET) && utime) {
-		uti_dkprintf("%s: utime=%ld.%09ld\n", __FUNCTION__, utime->tv_sec, utime->tv_nsec);
+		dkprintf("%s: utime=%ld.%09ld\n",
+				__func__, utime->tv_sec, utime->tv_nsec);
 	}
 	if (utime && (op == FUTEX_WAIT_BITSET || op == FUTEX_WAIT)) {
 		unsigned long nsec_timeout;
@@ -6977,7 +6971,8 @@ long do_futex(int n, unsigned long arg0, unsigned long arg1,
 				if (ret) {
 					return ret;
 				}
-				uti_dkprintf("%s: ats=%ld.%09ld\n", __FUNCTION__, ats.tv_sec, ats.tv_nsec);
+				dkprintf("%s: ats=%ld.%09ld\n",
+						__func__, ats.tv_sec, ats.tv_nsec);
 				/* Use nsec for UTI case */
 				timeout = (utime->tv_sec * NS_PER_SEC + utime->tv_nsec) -
 					(ats.tv_sec * NS_PER_SEC + ats.tv_nsec);
@@ -6993,9 +6988,9 @@ long do_futex(int n, unsigned long arg0, unsigned long arg1,
 	if (op == FUTEX_CMP_REQUEUE || op == FUTEX_WAKE_OP)
 		val2 = (uint32_t) (unsigned long) arg3;
 
-	ret = futex(uaddr, op, val, timeout, uaddr2, val2, val3, fshared, uti_clv);
+	ret = futex(uaddr, op, val, timeout, uaddr2, val2, val3, fshared);
 
-	uti_dkprintf("futex op=[%x, %s],uaddr=%lx, val=%x, utime=%lx, uaddr2=%lx, val3=%x, []=%x, shared: %d, ret: %d\n", 
+	dkprintf("futex op=[%x, %s],uaddr=%lx, val=%x, utime=%lx, uaddr2=%lx, val3=%x, []=%x, shared: %d, ret: %d\n",
 			op,
 			(op == FUTEX_WAIT) ? "FUTEX_WAIT" :
 			(op == FUTEX_WAIT_BITSET) ? "FUTEX_WAIT_BITSET" :
@@ -7043,7 +7038,7 @@ do_exit(int code)
 		setint_user((int*)thread->clear_child_tid, 0);
 		barrier();
 		futex((uint32_t *)thread->clear_child_tid,
-		      FUTEX_WAKE, 1, 0, NULL, 0, 0, 1, NULL);
+		      FUTEX_WAKE, 1, 0, NULL, 0, 0, 1);
 		thread->clear_child_tid = NULL;
 	}
 
@@ -10596,7 +10591,7 @@ int util_thread(struct uti_attr *arg)
 {
 	struct uti_ctx *rctx = NULL;
 	unsigned long rp_rctx;
-	struct cpu_local_var *uti_clv = NULL;
+	struct uti_info *uti_info = NULL;
 	struct syscall_request request IHK_DMA_ALIGN;
 	long rc;
 	struct thread *thread = cpu_local_var(current);
@@ -10615,13 +10610,29 @@ int util_thread(struct uti_attr *arg)
 	rp_rctx = virt_to_phys((void *)rctx);
 	save_uctx((void *)rctx->ctx, NULL);
 
-	/* Create a copy of clv and replace clv with it when the Linux thread calls in a McKernel function */
-	uti_clv = kmalloc(sizeof(struct cpu_local_var), IHK_MC_AP_NOWAIT);
-	if (!uti_clv) {
+	/* Create a information for Linux thread */
+	uti_info = kmalloc(sizeof(struct uti_info), IHK_MC_AP_NOWAIT);
+	if (!uti_info) {
 		rc = -ENOMEM;
 		goto out;
 	}
-	memcpy(uti_clv, get_this_cpu_local_var(), sizeof(struct cpu_local_var));
+	/* clv info */
+	uti_info->thread_va = (unsigned long)cpu_local_var(current);
+	uti_info->uti_futex_resp_pa = virt_to_phys((void *)cpu_local_var(uti_futex_resp));
+	uti_info->ikc2linux_pa = virt_to_phys((void *)cpu_local_var(ikc2linux));
+
+	/* thread info */
+	uti_info->tid = thread->tid;
+	uti_info->cpu = ihk_mc_get_processor_id();
+	uti_info->status_pa = virt_to_phys((void *)&thread->status);
+	uti_info->spin_sleep_lock_pa = virt_to_phys((void *)&thread->spin_sleep_lock);
+	uti_info->spin_sleep_pa = virt_to_phys((void *)&thread->spin_sleep);
+	uti_info->vm_pa = virt_to_phys((void *)thread->vm);
+	uti_info->futex_q_pa = virt_to_phys((void *)&thread->futex_q);
+
+	/* global info */
+	uti_info->mc_idle_halt = idle_halt;
+	uti_info->futex_queue_pa = virt_to_phys((void *)get_futex_queues());
 
 	request.number = __NR_sched_setaffinity;
 	request.args[0] = 0;
@@ -10632,7 +10643,7 @@ int util_thread(struct uti_attr *arg)
 		kattr.parent_cpuid = thread->parent_cpuid;
 		request.args[2] = virt_to_phys(&kattr);
 	}
-	request.args[3] = (unsigned long)uti_clv;
+	request.args[3] = (unsigned long)uti_info;
 	request.args[4] = uti_desc;
 	thread->uti_state = UTI_STATE_RUNNING_IN_LINUX;
 	rc = do_syscall(&request, ihk_mc_get_processor_id());
@@ -10649,8 +10660,8 @@ int util_thread(struct uti_attr *arg)
 	kfree(rctx);
 	rctx = NULL;
 
-	kfree(uti_clv);
-	uti_clv = NULL;
+	kfree(uti_info);
+	uti_info = NULL;
 
 	if (rc >= 0) {
 		if (rc & 0x100000000) { /* exit_group */
@@ -10673,7 +10684,7 @@ int util_thread(struct uti_attr *arg)
 
  out:
 	kfree(rctx);
-	kfree(uti_clv);
+	kfree(uti_info);
 
 	return rc;
 }

lib/include/ihk/cpu.h

@@ -109,6 +109,7 @@ static inline int is_sampling_event(struct mc_perf_event *event)
 void *ihk_mc_switch_context(ihk_mc_kernel_context_t *old_ctx,
                            ihk_mc_kernel_context_t *new_ctx,
 			   void *prev);
+int ihk_mc_get_interrupt_id(int cpu);
 int ihk_mc_interrupt_cpu(int cpu, int vector);
 
 void ihk_mc_init_user_process(ihk_mc_kernel_context_t *ctx,

lib/include/ihk/mm.h

@@ -247,7 +247,6 @@ int ihk_set_monitor(unsigned long addr, unsigned long size);
 int ihk_set_rusage(unsigned long addr, unsigned long size);
 int ihk_set_multi_intr_mode_addr(unsigned long addr);
 int ihk_set_nmi_mode_addr(unsigned long addr);
-int ihk_set_mckernel_do_futex(unsigned long addr);
 
 extern void (*__tlb_flush_handler)(int vector);
 

test/issues/1428/C1428.sh

@@ -0,0 +1,87 @@
+#/bin/sh
+
+USELTP=1
+USEOSTEST=0
+
+. ../../common.sh
+
+issue="1428"
+tid=01
+
+arch="`uname -p`"
+if [ "${arch}" == "x86_64" ]; then
+	UTI_TEST_DIR="../../uti"
+elif [ "${arch}" == "aarch64" ]; then
+	UTI_TEST_DIR="../../uti/arm64"
+else
+	echo "Error: ${arch} is unexpected arch"
+	exit 1
+fi
+
+# make uti test
+pushd ${UTI_TEST_DIR}
+make
+popd
+
+for tno in `seq 12 20`
+do
+	tname=`printf "C${issue}T%02d" ${tid}`
+	echo "*** ${tname} start *******************************"
+	sudo ${MCEXEC} --enable-uti ${UTI_TEST_DIR}/CT${tno} 2>&1 | tee ./${tname}.txt
+	rc=$?
+	ngs=`grep "NG" ./${tname}.txt | wc -l`
+
+	if [ ${ngs} -eq 0 ]; then
+		echo "*** ${tname} PASSED ******************************"
+	else
+		echo "*** ${tname} FAILED ******************************"
+	fi
+	let tid++
+	echo ""
+done
+
+echo "*** Stop mckernel to exec CT31-34 on Linux"
+mcstop
+for tno in `seq 31 34`
+do
+	sudo ${UTI_TEST_DIR}/CT${tno} -l &> ./lnx_CT${tno}.txt
+done
+echo "*** Boot mckernel"
+mcreboot
+echo ""
+
+for tno in `seq 31 34`
+do
+	tname=`printf "C${issue}T%02d" ${tid}`
+	echo "*** ${tname} start *******************************"
+	sudo ${MCEXEC} --enable-uti ${UTI_TEST_DIR}/CT${tno} 2>&1 | tee ./${tname}.txt
+	rc=$?
+	ngs=`grep "NG" ./${tname}.txt | wc -l`
+	echo "** Result on Linux **"
+	grep "waiter" ./lnx_CT${tno}.txt
+
+	if [ ${ngs} -eq 0 ]; then
+		echo "*** ${tname} PASSED ******************************"
+	else
+		echo "*** ${tname} FAILED ******************************"
+	fi
+	let tid++
+	echo ""
+done
+
+for tp in futex_wait01 futex_wait02 futex_wait03 futex_wait04 futex_wait_bitset01 futex_wait_bitset02 futex_wake01 futex_wake02 futex_wake03
+do
+	tname=`printf "C${issue}T%02d" ${tid}`
+	echo "*** ${tname} start *******************************"
+	sudo $MCEXEC $LTPBIN/$tp 2>&1 | tee $tp.txt
+	ok=`grep PASS $tp.txt | wc -l`
+	ng=`grep FAIL $tp.txt | wc -l`
+	if [ $ng = 0 ]; then
+		echo "*** ${tname} PASSED ($ok)"
+	else
+		echo "*** ${tname} FAILED (ok=$ok ng=$ng)"
+	fi
+	let tid++
+	echo ""
+done
+

test/issues/1428/Makefile

@@ -0,0 +1,11 @@
+CFLAGS=-g
+LDFLAGS=
+
+TARGET=
+
+all: $(TARGET)
+
+test: all
+	./C1428.sh
+clean:
+	rm -f $(TARGET) *.o *.txt

test/issues/1428/README

@@ -0,0 +1,32 @@
+【Issue#1428 動作確認】
+□ テスト内容
+1. McKernelのuti用テストプログラムのうち、futex機能を用いるテストを実行し、
+  utiスレッドを用いたfutex機能が正常に動作することを確認
+  実行するのは、test/utiの中のCT12～20、および、CT31～34
+  テスト内容の詳細は、test/uti/README を参照
+
+2. 以下のLTPを用いて既存のfutex機能に影響が無いことを確認
+  - futex_wait01
+  - futex_wait02
+  - futex_wait03
+  - futex_wait04
+  - futex_wake_bitset01
+  - futex_wake_bitset02
+  - futex_wake01
+  - futex_wake02
+  - futex_wake03
+
+□ 実行手順
+test/uti/Makefile または test/uti/arm64/Makefile 中の
+UTI_DIR の内容を環境に合わせて変更する。
+
+$ make test
+
+McKernelのインストール先や、OSTEST, LTPの配置場所は、
+$HOME/.mck_test_config を参照している
+.mck_test_config は、McKernelをビルドした際に生成されるmck_test_config.sample ファイルを
+$HOMEにコピーし、適宜編集する
+
+□ 実行結果
+x86_64_result.log aarch64_result.log 参照。
+すべての項目をPASSしていることを確認。

test/issues/1428/aarch64_result.log

@@ -0,0 +1,248 @@
+./C1428.sh
+mcstop+release.sh ... done
+mcreboot.sh -c 37-43,49-55 -m 2G@2,2G@3 -r 37-43:36+49-55:48 -O ... done
+~/src/mckernel/test/uti/arm64 ~/src/mckernel/test/issues/1428
+~/src/mckernel/test/issues/1428
+*** C1428T01 start *******************************
+CT12001 futex START
+CT12002 pthread_create OK
+CT12100 running on Linux CPU OK
+CT12003 FUTEX_WAKE OK
+CT12101 FUTEX_WAIT OK
+CT12004 pthread_join OK
+CT12005 END
+*** C1428T01 PASSED ******************************
+
+*** C1428T02 start *******************************
+CT13001 futex START
+CT13002 pthread_create OK
+CT13100 running on Linux CPU OK
+CT13101 FUTEX_WAKE OK
+CT13003 FUTEX_WAIT OK
+CT13004 pthread_join OK
+CT13005 END
+*** C1428T02 PASSED ******************************
+
+*** C1428T03 start *******************************
+CT14001 futex START
+CT14002 util_indicate_clone OK
+CT14003 pthread_create OK
+CT14004 lock first OK
+CT14100 running on Linux OK
+CT14101 lock second OK
+CT14005 pthread_join OK
+CT14006 END
+nsec=94214570, nspw=9.421457
+*** C1428T03 PASSED ******************************
+
+*** C1428T04 start *******************************
+CT15001 futex START
+CT15002 util_indicate_clone OK
+CT15003 pthread_create OK
+CT15100 running on Linux OK
+CT15101 lock first OK
+CT15004 lock second OK
+CT15005 pthread_join OK
+CT15006 END
+nsec=94214620, nspw=9.421462
+*** C1428T04 PASSED ******************************
+
+*** C1428T05 start *******************************
+CT16001 futex START
+CT16002 util_indicate_clone OK
+CT16003 pthread_create OK
+CT16101 running on Linux OK
+CT16102 return from pthread_cond_wait() OK
+CT16004 pthread_join OK
+CT16005 END
+*** C1428T05 PASSED ******************************
+
+*** C1428T06 start *******************************
+CT17001 futex START
+CT17002 util_indicate_clone OK
+CT17003 pthread_create OK
+CT17004 lock on 0x4200a8 OK
+CT17100 running on Linux OK
+CT17005 wake on 0x4200e0 OK
+CT17006 pthread_join OK
+CT17007 END
+*** C1428T06 PASSED ******************************
+
+*** C1428T07 start *******************************
+CT18001 futex START
+CT18002 pthread_create OK
+CT18101 running on Linux CPU OK
+start=1613528413.931088714
+op=109
+end=1613528414.759485821
+CT18102 FUTEX_WAIT OK
+CT18103 timeout OK
+CT18003 FUTEX_WAKE missing the waiter OK
+CT18004 pthread_join OK
+CT18005 END
+*** C1428T07 PASSED ******************************
+
+*** C1428T08 start *******************************
+CT19001 futex START
+CT19002 pthread_create OK
+CT19100 running on Linux CPU OK
+start=7347844.370216897
+op=9
+end=7347845.190062937
+CT19101 FUTEX_WAIT OK
+CT19102 timeout OK
+CT19003 FUTEX_WAKE missing the waiter OK
+CT19004 pthread_join OK
+CT19005 END
+*** C1428T08 PASSED ******************************
+
+*** C1428T09 start *******************************
+CT20001 futex START
+CT20002 pthread_create OK
+CT20100 running on Linux CPU OK
+start=1613528425.067456921
+end=1613528425.879490654
+CT20101 FUTEX_WAIT OK
+CT20102 timeout OK
+CT20003 FUTEX_WAKE missing the waiter OK
+CT20004 pthread_join OK
+CT20005 END
+*** C1428T09 PASSED ******************************
+
+*** Stop mckernel to exec CT31-34 on Linux
+mcstop+release.sh ... done
+*** Boot mckernel
+mcreboot.sh -c 37-43,49-55 -m 2G@2,2G@3 -r 37-43:36+49-55:48 -O ... done
+
+*** C1428T10 start *******************************
+[INFO] nloop=1000,blocktime=10000000
+[INFO] Master thread (tid: 54422) is running on 00,00
+nsec=94970130, nspw=9.497013
+[ OK ] Master thread is running on McKernel
+[ OK ] util_indicate_clone
+[INFO] Utility thread (tid: 54440) is running on 29,29
+[ OK ] Utility thread is running on Linux
+[INFO] waker: 9756747550 nsec, waiter: 9762950100 nsec, (waiter - waker) / nloop: 6202 nsec
+** Result on Linux **
+[INFO] waker: 9984565840 nsec, waiter: 9988427640 nsec, (waiter - waker) / nloop: 3861 nsec
+*** C1428T10 PASSED ******************************
+
+*** C1428T11 start *******************************
+[INFO] nloop=1000,blocktime=10000000
+[INFO] Master thread (tid: 54449) is running on 00,00
+nsec=95183170, nspw=9.518317
+[ OK ] Master thread is running on McKernel
+[ OK ] util_indicate_clone
+[INFO] Utility thread (tid: 54467) is running on 27,27
+[ OK ] Utility thread is running on Linux
+[INFO] waker: 9724199060 nsec, waiter: 9730702360 nsec, (waiter - waker) / nloop: 6503 nsec
+** Result on Linux **
+[INFO] waker: 9987888970 nsec, waiter: 9991459180 nsec, (waiter - waker) / nloop: 3570 nsec
+*** C1428T11 PASSED ******************************
+
+*** C1428T12 start *******************************
+[INFO] nloop=1000,blocktime=10000000
+[INFO] Master thread (tid: 54476) is running on 00,00
+nsec=96968310, nspw=9.696831
+[ OK ] Master thread is running on McKernel
+[ OK ] util_indicate_clone
+[INFO] Utility thread (tid: 54494) is running on 27,27
+[ OK ] Utility thread is running on Linux
+[INFO] waiter: 9747346620 nsec, waker: 9736919490 nsec, (waiter - waker) / nloop: 10427 nsec
+** Result on Linux **
+[INFO] waiter: 9922548360 nsec, waker: 9918225010 nsec, (waiter - waker) / nloop: 4323 nsec
+*** C1428T12 PASSED ******************************
+
+*** C1428T13 start *******************************
+[INFO] nloop=1000,blocktime=10000000
+[INFO] Master thread (tid: 54503) is running on 01,01
+nsec=94160460, nspw=9.416046
+[ OK ] Master thread is running on McKernel
+[ OK ] util_indicate_clone
+[INFO] Utility thread (tid: 54521) is running on 01,01
+[ OK ] Utility thread is running on Linux
+[INFO] waiter: 10112660440 nsec, waker: 10105975190 nsec, (waiter - waker) / nloop: 6685 nsec
+** Result on Linux **
+[INFO] waiter: 10082423010 nsec, waker: 10078381240 nsec, (waiter - waker) / nloop: 4041 nsec
+*** C1428T13 PASSED ******************************
+
+*** C1428T14 start *******************************
+futex_wait01    1  TPASS  :  futex_wait(): errno=ETIMEDOUT(110): Connection timed out
+futex_wait01    2  TPASS  :  futex_wait(): errno=EAGAIN/EWOULDBLOCK(11): Resource temporarily unavailable
+futex_wait01    3  TPASS  :  futex_wait(): errno=ETIMEDOUT(110): Connection timed out
+futex_wait01    4  TPASS  :  futex_wait(): errno=EAGAIN/EWOULDBLOCK(11): Resource temporarily unavailable
+*** C1428T14 PASSED (4)
+
+*** C1428T15 start *******************************
+futex_wait02    1  TPASS  :  futex_wait() woken up
+*** C1428T15 PASSED (1)
+
+*** C1428T16 start *******************************
+futex_wait03    1  TPASS  :  futex_wait() woken up
+*** C1428T16 PASSED (1)
+
+*** C1428T17 start *******************************
+futex_wait04    1  TPASS  :  futex_wait() returned -1: errno=EAGAIN/EWOULDBLOCK(11): Resource temporarily unavailable
+*** C1428T17 PASSED (1)
+
+*** C1428T18 start *******************************
+tst_test.c:1096: INFO: Timeout per run is 0h 05m 00s
+futex_wait_bitset.h:17: INFO: testing futex_wait_bitset() timeout with CLOCK_MONOTONIC
+futex_wait_bitset.h:59: PASS: futex_wait_bitset() waited 102024us, expected 100010us
+
+Summary:
+passed   1
+failed   0
+skipped  0
+warnings 0
+*** C1428T18 PASSED (1)
+
+*** C1428T19 start *******************************
+tst_test.c:1096: INFO: Timeout per run is 0h 05m 00s
+futex_wait_bitset.h:17: INFO: testing futex_wait_bitset() timeout with CLOCK_REALTIME
+futex_wait_bitset.h:59: PASS: futex_wait_bitset() waited 101995us, expected 100010us
+
+Summary:
+passed   1
+failed   0
+skipped  0
+warnings 0
+*** C1428T19 PASSED (1)
+
+*** C1428T20 start *******************************
+futex_wake01    1  TPASS  :  futex_wake() returned 0
+futex_wake01    2  TPASS  :  futex_wake() returned 0
+futex_wake01    3  TPASS  :  futex_wake() returned 0
+futex_wake01    4  TPASS  :  futex_wake() returned 0
+futex_wake01    5  TPASS  :  futex_wake() returned 0
+futex_wake01    6  TPASS  :  futex_wake() returned 0
+*** C1428T20 PASSED (6)
+
+*** C1428T21 start *******************************
+futex_wake02    1  TPASS  :  futex_wake() woken up 1 threads
+futex_wake02    2  TPASS  :  futex_wake() woken up 2 threads
+futex_wake02    3  TPASS  :  futex_wake() woken up 3 threads
+futex_wake02    4  TPASS  :  futex_wake() woken up 4 threads
+futex_wake02    5  TPASS  :  futex_wake() woken up 5 threads
+futex_wake02    6  TPASS  :  futex_wake() woken up 6 threads
+futex_wake02    7  TPASS  :  futex_wake() woken up 7 threads
+futex_wake02    8  TPASS  :  futex_wake() woken up 8 threads
+futex_wake02    9  TPASS  :  futex_wake() woken up 9 threads
+futex_wake02   10  TPASS  :  futex_wake() woken up 10 threads
+futex_wake02   11  TPASS  :  futex_wake() woken up 0 threads
+futex_wake02    0  TINFO  :  Child process returned TPASS
+*** C1428T21 PASSED (12)
+
+*** C1428T22 start *******************************
+futex_wake03    1  TPASS  :  futex_wake() woken up 1 childs
+futex_wake03    2  TPASS  :  futex_wake() woken up 2 childs
+futex_wake03    3  TPASS  :  futex_wake() woken up 3 childs
+futex_wake03    4  TPASS  :  futex_wake() woken up 4 childs
+futex_wake03    5  TPASS  :  futex_wake() woken up 5 childs
+futex_wake03    6  TPASS  :  futex_wake() woken up 6 childs
+futex_wake03    7  TPASS  :  futex_wake() woken up 7 childs
+futex_wake03    8  TPASS  :  futex_wake() woken up 8 childs
+futex_wake03    9  TPASS  :  futex_wake() woken up 9 childs
+futex_wake03   10  TPASS  :  futex_wake() woken up 10 childs
+futex_wake03   11  TPASS  :  futex_wake() woken up 0 children
+*** C1428T22 PASSED (11)

test/issues/1428/x86_64_result.log

@@ -0,0 +1,131 @@
+[m-takagi@wallaby14 1428]$ make test
+./C1428.sh
+mcstop+release.sh ... done
+mcreboot.sh -k 0 -f LOG_LOCAL6  -c 1-7,9-15,17-23,25-31 -m 10G@0,10G@1 -r 1-7:0+9-15:8+17-23:16+25-31:24 -O ... done
+~/project/os/mckernel/test/uti ~/project/os/mckernel/test/issues/1428
+make[1]: Entering directory `/home/m-takagi/project/os/mckernel/test/uti'
+dd bs=4096 count=1000 if=/dev/zero of=./file
+1000+0 records in
+1000+0 records out
+4096000 bytes (4.1 MB) copied, 0.0398667 s, 103 MB/s
+make[1]: Leaving directory `/home/m-takagi/project/os/mckernel/test/uti'
+~/project/os/mckernel/test/issues/1428
+*** Stop mckernel to exec CT31-34 on Linux
+mcstop+release.sh ... done
+*** Boot mckernel
+mcreboot.sh -k 0 -f LOG_LOCAL6  -c 1-7,9-15,17-23,25-31 -m 10G@0,10G@1 -r 1-7:0+9-15:8+17-23:16+25-31:24 -O ... done
+
+*** C1428T01 start *******************************
+[INFO] nloop=1000,blocktime=10000000
+[INFO] Master thread (tid: 1518) is running on 00,00
+[ OK ] Master thread is running on McKernel
+[ OK ] util_indicate_clone
+[INFO] Utility thread (tid: 1551) is running on 00,00
+[ OK ] Utility thread is running on Linux
+[INFO] waker: 26037705112 cycles, waiter: 26042430924 cycles, (waiter - waker) / nloop: 4725 cycles
+** Result on Linux **
+[INFO] waker: 19797701232 cycles, waiter: 19799301694 cycles, (waiter - waker) / nloop: 1600 cycles
+*** C1428T01 PASSED ******************************
+
+*** C1428T02 start *******************************
+[INFO] nloop=1000,blocktime=10000000
+[INFO] Master thread (tid: 1568) is running on 00,00
+[ OK ] Master thread is running on McKernel
+[ OK ] util_indicate_clone
+[INFO] Utility thread (tid: 1600) is running on 00,00
+[ OK ] Utility thread is running on Linux
+[INFO] waker: 26064839352 cycles, waiter: 26070575240 cycles, (waiter - waker) / nloop: 5735 cycles
+** Result on Linux **
+[INFO] waker: 24762320086 cycles, waiter: 24764268665 cycles, (waiter - waker) / nloop: 1948 cycles
+*** C1428T02 PASSED ******************************
+
+*** C1428T03 start *******************************
+[INFO] nloop=1000,blocktime=10000000
+[INFO] Master thread (tid: 1609) is running on 00,00
+[ OK ] Master thread is running on McKernel
+[ OK ] util_indicate_clone
+[INFO] Utility thread (tid: 1641) is running on 00,00
+[ OK ] Utility thread is running on Linux
+[INFO] waiter: 26042752992 cycles, waker: 26037367808 cycles, (waiter - waker) / nloop: 5385 cycles
+** Result on Linux **
+[INFO] waiter: 25124067612 cycles, waker: 25122513727 cycles, (waiter - waker) / nloop: 1553 cycles
+*** C1428T03 PASSED ******************************
+
+*** C1428T04 start *******************************
+[INFO] nloop=1000,blocktime=10000000
+[INFO] Master thread (tid: 1651) is running on 01,01
+[ OK ] Master thread is running on McKernel
+[ OK ] util_indicate_clone
+[INFO] Utility thread (tid: 1684) is running on 00,00
+[ OK ] Utility thread is running on Linux
+[INFO] waiter: 26004096360 cycles, waker: 25998796808 cycles, (waiter - waker) / nloop: 5299 cycles
+** Result on Linux **
+[INFO] waiter: 26289569877 cycles, waker: 26287829592 cycles, (waiter - waker) / nloop: 1740 cycles
+*** C1428T04 PASSED ******************************
+
+*** C1428T05 start *******************************
+futex_wait01    1  TPASS  :  futex_wait(): errno=ETIMEDOUT(110): Connection timed out
+futex_wait01    2  TPASS  :  futex_wait(): errno=EAGAIN/EWOULDBLOCK(11): Resource temporarily unavailable
+futex_wait01    3  TPASS  :  futex_wait(): errno=ETIMEDOUT(110): Connection timed out
+futex_wait01    4  TPASS  :  futex_wait(): errno=EAGAIN/EWOULDBLOCK(11): Resource temporarily unavailable
+*** C1428T05 PASSED (4)
+
+*** C1428T06 start *******************************
+futex_wait02    1  TPASS  :  futex_wait() woken up
+*** C1428T06 PASSED (1)
+
+*** C1428T07 start *******************************
+futex_wait03    1  TPASS  :  futex_wait() woken up
+*** C1428T07 PASSED (1)
+
+*** C1428T08 start *******************************
+futex_wait04    1  TPASS  :  futex_wait() returned -1: errno=EAGAIN/EWOULDBLOCK(11): Resource temporarily unavailable
+*** C1428T08 PASSED (1)
+
+*** C1428T09 start *******************************
+futex_wait_bitset01    0  TINFO  :  testing futex_wait_bitset() timeout with CLOCK_MONOTONIC
+futex_wait_bitset01    1  TPASS  :  futex_wait_bitset() waited 146706us, expected 100010us
+*** C1428T09 PASSED (1)
+
+*** C1428T10 start *******************************
+futex_wait_bitset02    0  TINFO  :  testing futex_wait_bitset() timeout with CLOCK_REALTIME
+futex_wait_bitset02    1  TPASS  :  futex_wait_bitset() waited 146709us, expected 100010us
+*** C1428T10 PASSED (1)
+
+*** C1428T11 start *******************************
+futex_wake01    1  TPASS  :  futex_wake() returned 0
+futex_wake01    2  TPASS  :  futex_wake() returned 0
+futex_wake01    3  TPASS  :  futex_wake() returned 0
+futex_wake01    4  TPASS  :  futex_wake() returned 0
+futex_wake01    5  TPASS  :  futex_wake() returned 0
+futex_wake01    6  TPASS  :  futex_wake() returned 0
+*** C1428T11 PASSED (6)
+
+*** C1428T12 start *******************************
+futex_wake02    1  TPASS  :  futex_wake() woken up 1 threads
+futex_wake02    2  TPASS  :  futex_wake() woken up 2 threads
+futex_wake02    3  TPASS  :  futex_wake() woken up 3 threads
+futex_wake02    4  TPASS  :  futex_wake() woken up 4 threads
+futex_wake02    5  TPASS  :  futex_wake() woken up 5 threads
+futex_wake02    6  TPASS  :  futex_wake() woken up 6 threads
+futex_wake02    7  TPASS  :  futex_wake() woken up 7 threads
+futex_wake02    8  TPASS  :  futex_wake() woken up 8 threads
+futex_wake02    9  TPASS  :  futex_wake() woken up 9 threads
+futex_wake02   10  TPASS  :  futex_wake() woken up 10 threads
+futex_wake02   11  TPASS  :  futex_wake() woken up 0 threads
+futex_wake02    0  TINFO  :  Child process returned TPASS
+*** C1428T12 PASSED (12)
+
+*** C1428T13 start *******************************
+futex_wake03    1  TPASS  :  futex_wake() woken up 1 childs
+futex_wake03    2  TPASS  :  futex_wake() woken up 2 childs
+futex_wake03    3  TPASS  :  futex_wake() woken up 3 childs
+futex_wake03    4  TPASS  :  futex_wake() woken up 4 childs
+futex_wake03    5  TPASS  :  futex_wake() woken up 5 childs
+futex_wake03    6  TPASS  :  futex_wake() woken up 6 childs
+futex_wake03    7  TPASS  :  futex_wake() woken up 7 childs
+futex_wake03    8  TPASS  :  futex_wake() woken up 8 childs
+futex_wake03    9  TPASS  :  futex_wake() woken up 9 childs
+futex_wake03   10  TPASS  :  futex_wake() woken up 10 childs
+futex_wake03   11  TPASS  :  futex_wake() woken up 0 children
+*** C1428T13 PASSED (11)

test/uti/CT09.c

@@ -83,26 +83,6 @@ double nspw; /* nsec per work */
 
 #define N_INIT 10000000
 
-void fwq_init(unsigned long *mem) {
-	struct timespec start, end;
-	unsigned long nsec;
-	int i;
-	clock_gettime(CLOCK_THREAD_CPUTIME_ID, &start);
-	BULK_FSW(N_INIT, mem);
-	clock_gettime(CLOCK_THREAD_CPUTIME_ID, &end);
-	nsec = (TS2NS(end.tv_sec, end.tv_nsec) - TS2NS(start.tv_sec, start.tv_nsec));
-	nspw = nsec / (double)N_INIT;
-	printf("nsec=%ld, nspw=%f\n", nsec, nspw);
-}
-
-void fwq(long delay_nsec, unsigned long* mem) {
-	if (delay_nsec < 0) {
-		printf("%s: delay_nsec<0\n", __FUNCTION__);
-	}
-	//printf("delay_nsec=%ld,count=%f\n", delay_nsec, delay_nsec / nspw);
-	BULK_FSW(delay_nsec / nspw, mem);
-}
-
 void mydelay(long delay_nsec, long *mem) {
 	struct timespec start, end;
 	clock_gettime(CLOCK_THREAD_CPUTIME_ID, &start);
@@ -154,11 +134,11 @@ void *progress_fn(void *_arg) {
 			break;
 		}
 
-		fwq(POLL_DELAY, &arg->mem);
+		fwq(POLL_DELAY);
 		
 		/* Event found */
 		if (nevents > 0) {
-			fwq(COMPL_DELAY, &arg->mem); /* Simulate MPI protocol response */
+			fwq(COMPL_DELAY); /* Simulate MPI protocol response */
 			nevents = 0;
 		}
 
@@ -190,7 +170,7 @@ int main(int argc, char **argv) {
 		fprintf(stdout, "CT09002 main running on McKernel INFO\n");
 	}
 
-	fwq_init(&mem);
+	fwq_init();
 	pthread_mutex_init(&ep_lock, NULL);
 
 	thr_args.bar_count = 0;
@@ -243,16 +223,16 @@ int main(int argc, char **argv) {
 
 			/* Acquire endpoint and send request-to-send packet */
 			pthread_mutex_lock(&ep_lock);
-			fwq(RTS_DELAY, &mem); 
+			fwq(RTS_DELAY);
 			pthread_mutex_unlock(&ep_lock);
 
 			/* Start calculation */
 
 			/* Generate event on behaf of responder */
-			fwq(NIC_DELAY, &mem); 
+			fwq(NIC_DELAY);
 			nevents++;
 
-			fwq(CALC_DELAY - NIC_DELAY, &mem); /* Overlap remainder */
+			fwq(CALC_DELAY - NIC_DELAY); /* Overlap remainder */
 
 			/* Wait until async thread consumes the event */
 			while (nevents > 0) {
@@ -260,7 +240,7 @@ int main(int argc, char **argv) {
 			}
 		} else {
 			/* No overlap case */
-			fwq(RTS_DELAY + CALC_DELAY + POLL_DELAY + COMPL_DELAY, &mem);
+			fwq(RTS_DELAY + CALC_DELAY + POLL_DELAY + COMPL_DELAY);
 		}
 	}
 	clock_gettime(CLOCK_THREAD_CPUTIME_ID, &end);

test/uti/CT14.c

@@ -34,23 +34,6 @@ double nspw; /* nsec per work */
 
 #define N_INIT 10000000
 
-void fwq_init(unsigned long *mem) {
-	struct timespec start, end;
-	unsigned long nsec;
-	int i;
-	clock_gettime(CLOCK_THREAD_CPUTIME_ID, &start);
-	BULK_FSW(N_INIT, mem);
-	clock_gettime(CLOCK_THREAD_CPUTIME_ID, &end);
-	nsec = (TS2NS(end.tv_sec, end.tv_nsec) - TS2NS(start.tv_sec, start.tv_nsec));
-	nspw = nsec / (double)N_INIT;
-	printf("nsec=%ld, nspw=%f\n", nsec, nspw);
-}
-
-void fwq(unsigned long delay_nsec, unsigned long* mem) {
-	//printf("delay_nsec=%ld,count=%f\n", delay_nsec, delay_nsec / nspw);
-	BULK_FSW(delay_nsec / nspw, mem);
-}
-
 void *
 util_thread(void *arg)
 {
@@ -65,7 +48,7 @@ util_thread(void *arg)
 	}
 	errno = 0;
 
-	fwq(500 * 1000 * 1000UL, &mem); /* Sending debug messages through serial takes 0.05 sec */
+	fwq(500 * 1000 * 1000UL); /* Sending debug messages through serial takes 0.05 sec */
 
 	pthread_mutex_lock(&mutex);
 	if (owned) {
@@ -84,7 +67,7 @@ int main(int argc, char **argv) {
 	unsigned long mem;
 
 	pthread_mutex_init(&mutex, NULL);
-	fwq_init(&mem);
+	fwq_init();
 
 	fprintf(stderr, "CT14001 futex START\n");
 
@@ -110,7 +93,7 @@ int main(int argc, char **argv) {
 		fprintf(stderr, "CT14004 lock first NG\n");
 	}
 	owned = 1;
-	fwq(2000 * 1000 * 1000UL, &mem); /* Need 2 sec to make child sleep */
+	fwq(2000 * 1000 * 1000UL); /* Need 2 sec to make child sleep */
 	pthread_mutex_unlock(&mutex);
 
 	pthread_join(thr, NULL);

test/uti/CT15.c

@@ -34,23 +34,6 @@ double nspw; /* nsec per work */
 
 #define N_INIT 10000000
 
-void fwq_init(unsigned long *mem) {
-	struct timespec start, end;
-	unsigned long nsec;
-	int i;
-	clock_gettime(CLOCK_THREAD_CPUTIME_ID, &start);
-	BULK_FSW(N_INIT, mem);
-	clock_gettime(CLOCK_THREAD_CPUTIME_ID, &end);
-	nsec = (TS2NS(end.tv_sec, end.tv_nsec) - TS2NS(start.tv_sec, start.tv_nsec));
-	nspw = nsec / (double)N_INIT;
-	printf("nsec=%ld, nspw=%f\n", nsec, nspw);
-}
-
-void fwq(unsigned long delay_nsec, unsigned long* mem) {
-	//printf("delay_nsec=%ld,count=%f\n", delay_nsec, delay_nsec / nspw);
-	BULK_FSW(delay_nsec / nspw, mem);
-}
-
 void *
 util_thread(void *arg)
 {
@@ -72,7 +55,7 @@ util_thread(void *arg)
 		fprintf(stderr, "CT14101 lock first NG\n");
 	}
 	owned = 1;
-	fwq(2000 * 1000 * 1000UL, &mem); /* Need 2 sec to make parent sleep */
+	fwq(2000 * 1000 * 1000UL); /* Need 2 sec to make parent sleep */
 	pthread_mutex_unlock(&mutex);
 
 	return NULL;
@@ -83,7 +66,7 @@ int main(int argc, char **argv) {
 	unsigned long mem;
 
 	pthread_mutex_init(&mutex, NULL);
-	fwq_init(&mem);
+	fwq_init();
 
 	fprintf(stderr, "CT14001 futex START\n");
 
@@ -102,7 +85,7 @@ int main(int argc, char **argv) {
 	}
 	fprintf(stderr, "CT14003 pthread_create OK\n");
 
-	fwq(500 * 1000 * 1000UL, &mem); /* Sending debug messages through serial takes 0.05 sec */
+	fwq(500 * 1000 * 1000UL); /* Sending debug messages through serial takes 0.05 sec */
 
 	pthread_mutex_lock(&mutex);
 	if (owned) {

test/uti/CT21.c

@@ -74,23 +74,6 @@ double nspw; /* nsec per work */
 
 #define N_INIT 10000000
 
-void fwq_init(unsigned long *mem) {
-	struct timespec start, end;
-	unsigned long nsec;
-	int i;
-	clock_gettime(CLOCK_THREAD_CPUTIME_ID, &start);
-	BULK_FSW(N_INIT, mem);
-	clock_gettime(CLOCK_THREAD_CPUTIME_ID, &end);
-	nsec = (TS2NS(end.tv_sec, end.tv_nsec) - TS2NS(start.tv_sec, start.tv_nsec));
-	nspw = nsec / (double)N_INIT;
-	printf("nsec=%ld, nspw=%f\n", nsec, nspw);
-}
-
-void fwq(unsigned long delay_nsec, unsigned long* mem) {
-	//printf("delay_nsec=%ld,count=%f\n", delay_nsec, delay_nsec / nspw);
-	BULK_FSW(delay_nsec / nspw, mem);
-}
-
 void mydelay(long delay_nsec, long *mem) {
 	struct timespec start, end;
 	clock_gettime(CLOCK_THREAD_CPUTIME_ID, &start);
@@ -136,7 +119,7 @@ void *progress_fn(void *_arg) {
 		}
 
 		pthread_mutex_unlock(&ep_lock);
-		fwq(random() % 1000000000, &mem); /* 0 - 1 sec */
+		fwq(random() % 1000000000); /* 0 - 1 sec */
 		pthread_mutex_lock(&ep_lock);
 	}
 	return NULL;
@@ -156,7 +139,7 @@ int main(int argc, char **argv) {
 		fprintf(stdout, "CT09002 main running on McKernel INFO\n");
 	}
 
-	fwq_init(&mem);
+	fwq_init();
 	pthread_mutex_init(&ep_lock, NULL);
 
 	for(i = 0; i < NTHR; i++) {
@@ -189,7 +172,7 @@ int main(int argc, char **argv) {
 	for (i = 0; i < 10; i++) {
 		pthread_mutex_lock(&ep_lock);
 		nevents++;
-		fwq(random() % 1000000000, &mem); /* 0 - 1 sec */
+		fwq(random() % 1000000000); /* 0 - 1 sec */
 		pthread_mutex_unlock(&ep_lock);
 		while (nevents > 0) {
 			FIXED_SIZE_WORK(&mem);

test/uti/CT22.c

@@ -74,23 +74,6 @@ double nspw; /* nsec per work */
 
 #define N_INIT 10000000
 
-void fwq_init(unsigned long *mem) {
-	struct timespec start, end;
-	unsigned long nsec;
-	int i;
-	clock_gettime(CLOCK_THREAD_CPUTIME_ID, &start);
-	BULK_FSW(N_INIT, mem);
-	clock_gettime(CLOCK_THREAD_CPUTIME_ID, &end);
-	nsec = (TS2NS(end.tv_sec, end.tv_nsec) - TS2NS(start.tv_sec, start.tv_nsec));
-	nspw = nsec / (double)N_INIT;
-	printf("nsec=%ld, nspw=%f\n", nsec, nspw);
-}
-
-void fwq(unsigned long delay_nsec, unsigned long* mem) {
-	//printf("delay_nsec=%ld,count=%f\n", delay_nsec, delay_nsec / nspw);
-	BULK_FSW(delay_nsec / nspw, mem);
-}
-
 void mydelay(long delay_nsec, long *mem) {
 	struct timespec start, end;
 	clock_gettime(CLOCK_THREAD_CPUTIME_ID, &start);
@@ -126,7 +109,7 @@ void *progress_fn(void *_arg) {
 	for (i = 0; i < 100; i++) {
 		pthread_mutex_lock(&ep_lock);
 		nevents++;
-		fwq(random() % 100000000, &mem); /* 0 - 0.1 sec */
+		fwq(random() % 100000000); /* 0 - 0.1 sec */
 		pthread_mutex_unlock(&ep_lock);
 		while (nevents > 0) {
 			FIXED_SIZE_WORK(&mem);
@@ -150,7 +133,7 @@ int main(int argc, char **argv) {
 		fprintf(stdout, "CT09002 main running on McKernel INFO\n");
 	}
 
-	fwq_init(&mem);
+	fwq_init();
 	pthread_mutex_init(&ep_lock, NULL);
 
 	for(i = 0; i < NTHR; i++) {
@@ -193,7 +176,7 @@ int main(int argc, char **argv) {
 		}
 
 		pthread_mutex_unlock(&ep_lock);
-		fwq(random() % 100000000, &mem); /* 0 - 0.1 sec */
+		fwq(random() % 100000000); /* 0 - 0.1 sec */
 		pthread_mutex_lock(&ep_lock);
 	}
 	pthread_mutex_unlock(&ep_lock);

test/uti/CT23.c

@@ -75,23 +75,6 @@ double nspw; /* nsec per work */
 
 #define N_INIT 10000000
 
-void fwq_init(unsigned long *mem) {
-	struct timespec start, end;
-	unsigned long nsec;
-	int i;
-	clock_gettime(CLOCK_THREAD_CPUTIME_ID, &start);
-	BULK_FSW(N_INIT, mem);
-	clock_gettime(CLOCK_THREAD_CPUTIME_ID, &end);
-	nsec = (TS2NS(end.tv_sec, end.tv_nsec) - TS2NS(start.tv_sec, start.tv_nsec));
-	nspw = nsec / (double)N_INIT;
-	printf("nsec=%ld, nspw=%f\n", nsec, nspw);
-}
-
-void fwq(unsigned long delay_nsec, unsigned long* mem) {
-	//printf("delay_nsec=%ld,count=%f\n", delay_nsec, delay_nsec / nspw);
-	BULK_FSW(delay_nsec / nspw, mem);
-}
-
 void mydelay(long delay_nsec, long *mem) {
 	struct timespec start, end;
 	clock_gettime(CLOCK_THREAD_CPUTIME_ID, &start);
@@ -135,7 +118,7 @@ void *progress_fn(void *_arg) {
 		}
 		nevents = 0;
 		pthread_mutex_unlock(&ep_lock);
-		fwq(random() % 100000000, &mem); /* 0 - 0.1 sec */
+		fwq(random() % 100000000); /* 0 - 0.1 sec */
 		pthread_mutex_lock(&ep_lock);
 	}
 	pthread_mutex_unlock(&ep_lock);
@@ -156,7 +139,7 @@ int main(int argc, char **argv) {
 		fprintf(stdout, "CT09002 main running on McKernel INFO\n");
 	}
 
-	fwq_init(&mem);
+	fwq_init();
 	pthread_cond_init(&ep_cond, NULL);
 	pthread_mutex_init(&ep_lock, NULL);
 
@@ -188,7 +171,7 @@ int main(int argc, char **argv) {
 	fprintf(stdout, "CT09004 pthread_create OK\n");
 	clock_gettime(CLOCK_THREAD_CPUTIME_ID, &start);
 	for (i = 0; i < 100; i++) {
-		fwq(random() % 100000000, &mem); /* 0 - 0.1 sec */
+		fwq(random() % 100000000); /* 0 - 0.1 sec */
 		pthread_mutex_lock(&ep_lock);
 		nevents++;
 		pthread_cond_signal(&ep_cond);

test/uti/CT24.c

@@ -75,23 +75,6 @@ double nspw; /* nsec per work */
 
 #define N_INIT 10000000
 
-void fwq_init(unsigned long *mem) {
-	struct timespec start, end;
-	unsigned long nsec;
-	int i;
-	clock_gettime(CLOCK_THREAD_CPUTIME_ID, &start);
-	BULK_FSW(N_INIT, mem);
-	clock_gettime(CLOCK_THREAD_CPUTIME_ID, &end);
-	nsec = (TS2NS(end.tv_sec, end.tv_nsec) - TS2NS(start.tv_sec, start.tv_nsec));
-	nspw = nsec / (double)N_INIT;
-	printf("nsec=%ld, nspw=%f\n", nsec, nspw);
-}
-
-void fwq(unsigned long delay_nsec, unsigned long* mem) {
-	//printf("delay_nsec=%ld,count=%f\n", delay_nsec, delay_nsec / nspw);
-	BULK_FSW(delay_nsec / nspw, mem);
-}
-
 void mydelay(long delay_nsec, long *mem) {
 	struct timespec start, end;
 	clock_gettime(CLOCK_THREAD_CPUTIME_ID, &start);
@@ -125,7 +108,7 @@ void *progress_fn(void *_arg) {
 	pthread_mutex_unlock(&arg->bar_lock);
 
 	for (i = 0; i < 100; i++) {
-		fwq(random() % 100000000, &mem); /* 0 - 0.1 sec */
+		fwq(random() % 100000000); /* 0 - 0.1 sec */
 		pthread_mutex_lock(&ep_lock);
 		nevents++;
 		pthread_cond_signal(&ep_cond);
@@ -152,7 +135,7 @@ int main(int argc, char **argv) {
 		fprintf(stdout, "CT09002 main running on McKernel INFO\n");
 	}
 
-	fwq_init(&mem);
+	fwq_init();
 	pthread_mutex_init(&ep_lock, NULL);
 
 	for(i = 0; i < NTHR; i++) {
@@ -193,7 +176,7 @@ int main(int argc, char **argv) {
 		}
 		nevents = 0;
 		pthread_mutex_unlock(&ep_lock);
-		fwq(random() % 100000000, &mem); /* 0 - 0.1 sec */
+		fwq(random() % 100000000); /* 0 - 0.1 sec */
 		pthread_mutex_lock(&ep_lock);
 	}
 	pthread_mutex_unlock(&ep_lock);

test/uti/CT27.c

@@ -102,25 +102,6 @@ double nspw; /* nsec per work */
 
 #define N_INIT 10000000
 
-void fwq_init() {
-	struct timespec start, end;
-	unsigned long nsec;
-	int i;
-	clock_gettime(CLOCK_THREAD_CPUTIME_ID, &start);
-	bulk_fsw(N_INIT);
-	clock_gettime(CLOCK_THREAD_CPUTIME_ID, &end);
-	nsec = (TS2NS(end.tv_sec, end.tv_nsec) - TS2NS(start.tv_sec, start.tv_nsec));
-	nspw = nsec / (double)N_INIT;
-	printf("nsec=%ld, nspw=%f\n", nsec, nspw);
-}
-
-void fwq(long delay_nsec) {
-	if (delay_nsec < 0) {
-		printf("%s: delay_nsec<0\n", __FUNCTION__);
-	}
-	//printf("delay_nsec=%ld,count=%f\n", delay_nsec, delay_nsec / nspw);
-	bulk_fsw(delay_nsec / nspw);
-}
 int progress_responder(struct thr_arg *thr_arg) {
 	int ret = 0;
 	int j;

test/uti/CT28.c

@@ -152,25 +152,6 @@ static int print_cpu_last_executed_on() {
     goto fn_exit;
 }
 
-void fwq_init() {
-	struct timespec start, end;
-	unsigned long nsec;
-	int i;
-	clock_gettime(CLOCK_THREAD_CPUTIME_ID, &start);
-	bulk_fsw(N_INIT);
-	clock_gettime(CLOCK_THREAD_CPUTIME_ID, &end);
-	nsec = (TS2NS(end.tv_sec, end.tv_nsec) - TS2NS(start.tv_sec, start.tv_nsec));
-	nspw = nsec / (double)N_INIT;
-	printf("nsec=%ld, nspw=%f\n", nsec, nspw);
-}
-
-void fwq(long delay_nsec) {
-	if (delay_nsec < 0) {
-		printf("%s: delay_nsec<0\n", __FUNCTION__);
-	}
-	bulk_fsw(delay_nsec / nspw);
-}
-
 void init_bar(struct thr_arg* thr_arg) {
 	pthread_mutex_lock(&thr_arg->bar_lock);
 	thr_arg->bar_count= 0;

test/uti/CT29.c

@@ -44,26 +44,6 @@ static inline void bulk_fsw(unsigned long n) {
 
 #define N_INIT 1000000
 
-void fwq_init() {
-	struct timespec start, end;
-	unsigned long nsec;
-	int i;
-	clock_gettime(CLOCK_THREAD_CPUTIME_ID, &start);
-	bulk_fsw(N_INIT);
-	clock_gettime(CLOCK_THREAD_CPUTIME_ID, &end);
-	nsec = (TS2NS(end.tv_sec, end.tv_nsec) - TS2NS(start.tv_sec, start.tv_nsec));
-	nspw = nsec / (double)N_INIT;
-	printf("nsec=%ld, nspw=%f\n", nsec, nspw);
-}
-
-void fwq(long delay_nsec) {
-	if (delay_nsec < 0) {
-		printf("%s: delay_nsec<0\n", __FUNCTION__);
-	}
-	//printf("delay_nsec=%ld,count=%f\n", delay_nsec, delay_nsec / nspw);
-	bulk_fsw(delay_nsec / nspw);
-}
-
 void *util_thread(void *arg) {
 	int rc;
 

test/uti/CT30.c

@@ -53,23 +53,6 @@ double nspw; /* nsec per work */
 
 #define N_INIT 10000000
 
-void fwq_init(unsigned long *mem) {
-	struct timespec start, end;
-	unsigned long nsec;
-	int i;
-	clock_gettime(CLOCK_THREAD_CPUTIME_ID, &start);
-	BULK_FSW(N_INIT, mem);
-	clock_gettime(CLOCK_THREAD_CPUTIME_ID, &end);
-	nsec = (TS2NS(end.tv_sec, end.tv_nsec) - TS2NS(start.tv_sec, start.tv_nsec));
-	nspw = nsec / (double)N_INIT;
-	printf("[INFO] nsec=%ld, nspw=%f\n", nsec, nspw);
-}
-
-void fwq(unsigned long delay_nsec, unsigned long* mem) {
-	//printf("delay_nsec=%ld,count=%f\n", delay_nsec, delay_nsec / nspw);
-	BULK_FSW(delay_nsec / nspw, mem);
-}
-
 void fwq_omp(unsigned long delay_nsec, unsigned long* mem) {
 #pragma omp parallel
 	{
@@ -110,7 +93,7 @@ void *util_fn(void *_arg) {
 
 		if (nevents > 0) {
 			nevents--;
-			fwq(random() % 100000000, &mem); /* 0 - 0.1 sec */
+			fwq(random() % 100000000); /* 0 - 0.1 sec */
 		}
 		pthread_mutex_unlock(&ep_lock);
 	}
@@ -127,7 +110,7 @@ int main(int argc, char **argv) {
 	ret = syscall(732);
 	OKNGNOJUMP(ret != -1, "Master is running on McKernel\n");
 
-	fwq_init(&mem);
+	fwq_init();
 	pthread_mutex_init(&ep_lock, NULL);
 
 	pthread_barrier_init(&bar, NULL, NTHR + 1);

test/uti/CT31.c

@@ -25,17 +25,20 @@ pthread_t thr;
 long t_cond_wait, t_fwq;
 long nloop;
 long blocktime = 10L * 1000 * 1000;
+int linux_run;
 
 void *util_fn(void *arg)
 {
 	int i;
 	int ret;
-    long start, end;
+	long start, end;
 
 	print_cpu_last_executed_on("Utility thread");
 
-	ret = syscall(732);
-	OKNGNOJUMP(ret == -1, "Utility thread is running on Linux\n");
+	if (!linux_run) {
+		ret = syscall(732);
+		OKNGNOJUMP(ret == -1, "Utility thread is running on Linux\n");
+	}
 
 	pthread_barrier_wait(&bar);
 	for (i = 0; i < nloop; i++) {
@@ -50,7 +53,6 @@ void *util_fn(void *arg)
 		flag = 1;
 		pthread_cond_signal(&cond);
 		pthread_mutex_unlock(&mutex);
-		
 	}
 
  fn_fail:
@@ -66,21 +68,24 @@ int main(int argc, char **argv)
 {
 	int i;
 	int ret;
-    long start, end;
+	long start, end;
 	cpu_set_t cpuset;
 	pthread_attr_t attr;
 	pthread_barrierattr_t bar_attr;
 	struct sched_param param = { .sched_priority = 99 };
 	int opt;
 
-	while ((opt = getopt_long(argc, argv, "+b:", options, NULL)) != -1) {
+	while ((opt = getopt_long(argc, argv, "+b:l", options, NULL)) != -1) {
 		switch (opt) {
-			case 'b':
-				blocktime = atoi(optarg);
-				break;
-			default: /* '?' */
-				printf("unknown option %c\n", optopt);
-				exit(1);
+		case 'b':
+			blocktime = atoi(optarg);
+			break;
+		case 'l':
+			linux_run = 1;
+			break;
+		default: /* '?' */
+			printf("unknown option %c\n", optopt);
+			exit(1);
 		}
 	}
 	nloop = (10 * 1000000000UL) / blocktime;
@@ -103,11 +108,13 @@ int main(int argc, char **argv)
 	pthread_barrierattr_init(&bar_attr);
 	pthread_barrier_init(&bar, &bar_attr, 2);
 
-	ret = syscall(732);
-	OKNGNOJUMP(ret != -1, "Master thread is running on McKernel\n");
+	if (!linux_run) {
+		ret = syscall(732);
+		OKNGNOJUMP(ret != -1, "Master thread is running on McKernel\n");
 
-	ret = syscall(731, 1, NULL);
-	OKNGNOJUMP(ret != -1, "util_indicate_clone\n");
+		ret = syscall(731, 1, NULL);
+		OKNGNOJUMP(ret != -1, "util_indicate_clone\n");
+	}
 
 	if ((ret = pthread_attr_init(&attr))) {
  		printf("%s: Error: pthread_attr_init failed (%d)\n", __FUNCTION__, ret);
@@ -128,11 +135,13 @@ int main(int argc, char **argv)
 	}
 
 	if ((ret = sched_setscheduler(0, SCHED_FIFO, &param))) {
-		fprintf(stderr, "Error: sched_setscheduler failed (%d)\n", ret);
-		goto fn_fail;
+		fprintf(stderr, "Warning: sched_setscheduler: %s\n",
+			strerror(errno));
 	}
 
-	syscall(701, 1 | 2);
+	if (!linux_run) {
+		syscall(701, 1 | 2);
+	}
 	pthread_barrier_wait(&bar);
 	for (i = 0; i < nloop; i++) {
 		start = rdtsc_light();
@@ -147,7 +156,9 @@ int main(int argc, char **argv)
 		end = rdtsc_light();
 		t_cond_wait += end - start;
 	}
-	syscall(701, 4 | 8);
+	if (!linux_run) {
+		syscall(701, 4 | 8);
+	}
 
 	pthread_join(thr, NULL);
 	printf("[INFO] waker: %ld cycles, waiter: %ld cycles, (waiter - waker) / nloop: %ld cycles\n", t_fwq, t_cond_wait, (t_cond_wait - t_fwq) / nloop);

test/uti/CT31.sh

@@ -1,102 +0,0 @@
-#!/usr/bin/bash
-
-bn=`basename $0`
-fn=`echo $bn | sed 's/.sh//'`
-
-nloop=800
-stop=0
-reboot=0
-go=0
-mck=0
-NNODES=1
-NPROC=$((1 * NNODES))
-LASTNODE=8200
-use_hfi=0
-
-while getopts srgmh:N:P:L: OPT
-do
-        case ${OPT} in
-	    s) stop=1
-		;;
-            r) reboot=1
-                ;;
-            g) go=1
-                ;;
-            m) mck=1
-                ;;
-	    h) use_hfi=1
-		;;
-	    N) NNODES=$OPTARG
-		;;
-	    P) NPROC=$OPTARG
-		;;
-	    L) LASTNODE=$OPTARG
-		;;
-            *) echo "invalid option -${OPT}" >&2
-                exit 1
-        esac
-done
-
-MYHOME=/work/gg10/e29005
-ABS_SRCDIR=${MYHOME}/project/os/mckernel/test/uti
-MCK=${MYHOME}/project/os/install
-
-NODES=`echo $(seq -s ",c" $(($LASTNODE + 1 - $NNODES)) $LASTNODE) | sed 's/^/c/'`
-PPN=$((NPROC / NNODES))
-echo NPROC=$NPROC NNODES=$NNODES PPN=$PPN NODES=$NODES
-
-if [ ${mck} -eq 1 ]; then
-    MCEXEC="${MCK}/bin/mcexec"
-    mcexecopt="--enable-uti"
-    if [ ${use_hfi} -eq 1 ]; then
-	mcexecopt="--enable-hfi1 $mcexecopt"
-    fi
-else
-    MCEXEC=
-    mcexecopt=
-fi
-
-if [ ${stop} -eq 1 ]; then
-    PDSH_SSH_ARGS_APPEND="-tt -q" pdsh -t 2 -w $NODES \
-	sudo mount /work
-
-    if [ ${mck} -eq 1 ]; then
-	PDSH_SSH_ARGS_APPEND="-tt -q" pdsh -t 2 -w $NODES \
-	    /sbin/pidof mcexec \| xargs -r kill -9
-	PDSH_SSH_ARGS_APPEND="-tt -q" pdsh -t 2 -w $NODES \
-	    sudo ${MCK}/sbin/mcstop+release.sh
-    else
-	:
-    fi
-fi
-
-if [ ${reboot} -eq 1 ]; then
-    PDSH_SSH_ARGS_APPEND="-tt -q" pdsh -t 2 -w $NODES \
-    sudo mount /work
-
-    if [ ${mck} -eq 1 ]; then
-	PDSH_SSH_ARGS_APPEND="-tt -q" pdsh -t 2 -w $NODES \
-	    sudo ${MCK}/sbin/mcreboot.sh -c 2-17,70-85,138-153,206-221,20-35,88-103,156-171,224-239,36-51,104-119,172-187,240-255,52-67,120-135,188-203,256-271 -r 2-5,70-73,138-141,206-209:0+6-9,74-77,142-145,210-213:1+10-13,78-81,146-149,214-217:68+14-17,82-85,150-153,218-221:69+20-23,88-91,156-159,224-227:136+24-27,92-95,160-163,228-231:137+28-31,96-99,164-167,232-235:204+32-35,100-103,168-171,236-239:205+36-39,104-107,172-175,240-243:18+40-43,108-111,176-179,244-247:19+44-47,112-115,180-183,248-251:86+48-51,116-119,184-187,252-255:87+52-55,120-123,188-191,256-259:154+56-59,124-127,192-195,260-263:155+60-63,128-131,196-199,264-267:222+64-67,132-135,200-203,268-271:223 -m 32G@0,12G@1
-    else
-	:
-    fi
-fi
-
-if [ ${go} -eq 1 ]; then
-    cd $ABS_SRCDIR
-    make $fn
-
-    PDSH_SSH_ARGS_APPEND="-tt -q" pdsh -t 2 -w $NODES \
-	ulimit -u 16384; 
-    PDSH_SSH_ARGS_APPEND="-tt -q" pdsh -t 2 -w $NODES \
-	ulimit -s unlimited
-
-    for((count=0;count<nloop;count++)); do
-	sudo $MCEXEC $mcexecopt ./$fn
-	echo =====
-	echo $count
-	echo =====
-    done
-
-fi
-

test/uti/CT32.c

@@ -4,60 +4,57 @@
 #include <string.h>
 #include <pthread.h>
 #include <errno.h>
-#include <sys/mman.h>
 #include <unistd.h>
 #include <sys/syscall.h>
-#include <linux/futex.h>
+#include <sys/mman.h>
+#include <signal.h>
 #include <getopt.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <fcntl.h>
-#include <stdint.h>
-#include <uti.h>
 #include "util.h"
 
 #define WAITER_CPU 0
 #define WAKER_CPU 1
 
-int sem;
+pthread_mutex_t mutex;
+pthread_cond_t cond;
 pthread_barrier_t bar;
 int flag;
 pthread_t thr;
-long t_futex_wait, t_fwq;
+long t_cond_wait, t_fwq;
 long nloop;
 long blocktime = 10L * 1000 * 1000;
+int linux_run;
 
 void *util_fn(void *arg)
 {
 	int i;
 	int ret;
-    long start, end;
-	int testid = 32101;
+	long start, end;
 
 	print_cpu_last_executed_on("Utility thread");
 
-	ret = syscall(732);
-	OKNGNOJUMP(ret == -1, "Utility thread is running on Linux\n");
+	if (!linux_run) {
+		ret = syscall(732);
+		OKNGNOJUMP(ret == -1, "Utility thread is running on Linux\n");
+	}
 
 	pthread_barrier_wait(&bar);
-
 	for (i = 0; i < nloop; i++) {
 		start = rdtsc_light();
 		
-		fwq(blocktime);
+		pthread_mutex_lock(&mutex); /* no futex */
+		while(!flag) {
+			pthread_cond_wait(&cond, &mutex); /* 1st futex */
+		}
+		flag = 0;
+		pthread_mutex_unlock(&mutex); /* 2nd futex */
 
 		end = rdtsc_light();
-		t_fwq += end - start;
-
-		if ((ret = syscall(__NR_futex, &sem, FUTEX_WAKE, 1, NULL, NULL, 0)) == -1) {
-			printf("Error: futex wake: %s\n", strerror(errno));
-		}
-
-		//pthread_barrier_wait(&bar);
-
+		t_cond_wait += end - start;
 	}
 
-	ret = 0;
  fn_fail:
 	return NULL;
 }
@@ -71,21 +68,24 @@ int main(int argc, char **argv)
 {
 	int i;
 	int ret;
-    long start, end;
+	long start, end;
 	cpu_set_t cpuset;
 	pthread_attr_t attr;
 	pthread_barrierattr_t bar_attr;
 	struct sched_param param = { .sched_priority = 99 };
 	int opt;
 
-	while ((opt = getopt_long(argc, argv, "+b:", options, NULL)) != -1) {
+	while ((opt = getopt_long(argc, argv, "+b:l", options, NULL)) != -1) {
 		switch (opt) {
-			case 'b':
-				blocktime = atoi(optarg);
-				break;
-			default: /* '?' */
-				printf("unknown option %c\n", optopt);
-				exit(1);
+		case 'b':
+			blocktime = atoi(optarg);
+			break;
+		case 'l':
+			linux_run = 1;
+			break;
+		default: /* '?' */
+			printf("unknown option %c\n", optopt);
+			exit(1);
 		}
 	}
 	nloop = (10 * 1000000000UL) / blocktime;
@@ -102,88 +102,66 @@ int main(int argc, char **argv)
 
 	fwq_init();
 
+	pthread_mutex_init(&mutex, NULL);
+	pthread_cond_init(&cond, NULL);
+
 	pthread_barrierattr_init(&bar_attr);
 	pthread_barrier_init(&bar, &bar_attr, 2);
 
+	if (!linux_run) {
+		ret = syscall(732);
+		OKNGNOJUMP(ret != -1, "Master thread is running on McKernel\n");
+
+		ret = syscall(731, 1, NULL);
+		OKNGNOJUMP(ret != -1, "util_indicate_clone\n");
+	}
+
 	if ((ret = pthread_attr_init(&attr))) {
- 		printf("Error: pthread_attr_init: %s\n", strerror(errno));
+ 		printf("%s: Error: pthread_attr_init failed (%d)\n", __FUNCTION__, ret);
 		goto fn_fail;
 	}
 
-#if 0
-	uti_attr_t uti_attr;
-	ret = uti_attr_init(&uti_attr);
-	if (ret) {
-		printf("%s: Error: uti_attr_init failed (%d)\n", __FUNCTION__, ret);
-		exit(1);
-	}
-
-	/* Give a hint that it's beneficial to prioritize it in scheduling. */
-	ret = UTI_ATTR_HIGH_PRIORITY(&uti_attr);
-	if (ret) {
-		printf("%s: Error: UTI_ATTR_HIGH_PRIORITY failed (%d)\n", __FUNCTION__, ret);
-		exit(1);
-	}
-	
-	if ((ret = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED))) {
-		printf("%s: Error: pthread_attr_setdetachstate failed (%d)\n", __FUNCTION__, ret);
-		exit(1);
-	}
-	
-	if ((ret = uti_pthread_create(&thr, &attr, progress_function, NULL, &uti_attr))) {
-		printf("%s: Error: uti_pthread_create: %s\n", __FUNCTION__, strerror(errno));
-		exit(1);
-	}
-	
-	if ((ret = uti_attr_destroy(&uti_attr))) {
-		printf("%s: Error: uti_attr_destroy failed (%d)\n", __FUNCTION__, ret);
-		exit(1);
-	}
-#else
  	CPU_ZERO(&cpuset);
 	CPU_SET(WAKER_CPU, &cpuset);
 
 	if ((ret = pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset))) {
- 		printf("Error: pthread_attr_setaffinity_np: %s\n", strerror(errno));
+ 		printf("%s: Error: pthread_attr_setaffinity_np failed (%d)\n", __FUNCTION__, ret);
 		goto fn_fail;
 	}
 
-	ret = syscall(732);
-	OKNGNOJUMP(ret != -1, "Master thread is running on McKernel\n");
-
-	ret = syscall(731, 1, NULL);
-	OKNGNOJUMP(ret != -1, "util_indicate_clone\n");
-
 	if ((ret = pthread_create(&thr, &attr, util_fn, NULL))) {
-		printf("Error: pthread_create: %s\n", strerror(errno));
+		fprintf(stderr, "Error: pthread_create failed (%d)\n", ret);
 		goto fn_fail;
 	}
 
-#endif
-
 	if ((ret = sched_setscheduler(0, SCHED_FIFO, &param))) {
-		printf("Error: sched_setscheduler: %s\n", strerror(errno));
-		ret = -errno;
-		goto fn_fail;
+		fprintf(stderr, "Warning: sched_setscheduler: %s\n",
+			strerror(errno));
 	}
 
-	syscall(701, 1 | 2);
+	if (!linux_run) {
+		syscall(701, 1 | 2);
+	}
 	pthread_barrier_wait(&bar);
-	start = rdtsc_light();
 	for (i = 0; i < nloop; i++) {
+		start = rdtsc_light();
 
-		if ((ret = syscall(__NR_futex, &sem, FUTEX_WAIT, 0, NULL, NULL, 0))) {
-			printf("Error: futex wait failed (%s)\n", strerror(errno));
-		}
+		fwq(blocktime);
 
-		//pthread_barrier_wait(&bar); /* 2nd futex */
+		end = rdtsc_light();
+		t_fwq += end - start;
+
+		pthread_mutex_lock(&mutex);
+		flag = 1;
+		pthread_cond_signal(&cond);
+		pthread_mutex_unlock(&mutex);
+	}
+	if (!linux_run) {
+		syscall(701, 4 | 8);
 	}
-	end = rdtsc_light();
-	t_futex_wait += end - start;
-	syscall(701, 4 | 8);
 
 	pthread_join(thr, NULL);
-	printf("[INFO] waiter: %ld cycles, waker: %ld cycles, (waiter - waker) / nloop: %ld cycles\n", t_fwq, t_futex_wait, (t_futex_wait - t_fwq) / nloop);
+	printf("[INFO] waker: %ld cycles, waiter: %ld cycles, (waiter - waker) / nloop: %ld cycles\n", t_fwq, t_cond_wait, (t_cond_wait - t_fwq) / nloop);
 
 	ret = 0;
  fn_fail:

test/uti/CT32.sh

@@ -1,104 +0,0 @@
-#!/usr/bin/bash
-
-bn=`basename $0`
-fn=`echo $bn | sed 's/.sh//'`
-
-stop=0
-reboot=0
-go=0
-mck=0
-disable_uti=1
-NNODES=1
-NPROC=$((1 * NNODES))
-LASTNODE=8200
-use_hfi=0
-
-while getopts srgmh:N:P:L:d: OPT
-do
-        case ${OPT} in
-	    s) stop=1
-		;;
-            r) reboot=1
-                ;;
-            g) go=1
-                ;;
-            m) mck=1
-                ;;
-	    h) use_hfi=1
-		;;
-            d) disable_uti=$OPTARG
-                ;;
-	    N) NNODES=$OPTARG
-		;;
-	    P) NPROC=$OPTARG
-		;;
-	    L) LASTNODE=$OPTARG
-		;;
-            *) echo "invalid option -${OPT}" >&2
-                exit 1
-        esac
-done
-
-MYHOME=/work/gg10/e29005
-ABS_SRCDIR=${MYHOME}/project/os/mckernel/test/uti
-MCK=${MYHOME}/project/os/install
-
-NODES=`echo $(seq -s ",c" $(($LASTNODE + 1 - $NNODES)) $LASTNODE) | sed 's/^/c/'`
-PPN=$((NPROC / NNODES))
-echo NPROC=$NPROC NNODES=$NNODES PPN=$PPN NODES=$NODES
-
-if [ $disable_uti -eq 1 ]; then
-    export DISABLE_UTI=1
-else
-    unset DISABLE_UTI
-fi
-
-if [ ${mck} -eq 1 ]; then
-    MCEXEC="${MCK}/bin/mcexec"
-    mcexecopt="--enable-uti"
-    if [ ${use_hfi} -eq 1 ]; then
-	mcexecopt="--enable-hfi1 $mcexecopt"
-    fi
-else
-    MCEXEC=
-    mcexecopt=
-fi
-
-if [ ${stop} -eq 1 ]; then
-    PDSH_SSH_ARGS_APPEND="-tt -q" pdsh -t 2 -w $NODES \
-	sudo mount /work
-
-    if [ ${mck} -eq 1 ]; then
-	PDSH_SSH_ARGS_APPEND="-tt -q" pdsh -t 2 -w $NODES \
-	    /sbin/pidof mcexec \| xargs -r kill -9
-	PDSH_SSH_ARGS_APPEND="-tt -q" pdsh -t 2 -w $NODES \
-	    sudo ${MCK}/sbin/mcstop+release.sh
-    else
-	:
-    fi
-fi
-
-if [ ${reboot} -eq 1 ]; then
-    PDSH_SSH_ARGS_APPEND="-tt -q" pdsh -t 2 -w $NODES \
-    sudo mount /work
-
-    if [ ${mck} -eq 1 ]; then
-	PDSH_SSH_ARGS_APPEND="-tt -q" pdsh -t 2 -w $NODES \
-	    sudo ${MCK}/sbin/mcreboot.sh -c 2-17,70-85,138-153,206-221,20-35,88-103,156-171,224-239,36-51,104-119,172-187,240-255,52-67,120-135,188-203,256-271 -r 2-5,70-73,138-141,206-209:0+6-9,74-77,142-145,210-213:1+10-13,78-81,146-149,214-217:68+14-17,82-85,150-153,218-221:69+20-23,88-91,156-159,224-227:136+24-27,92-95,160-163,228-231:137+28-31,96-99,164-167,232-235:204+32-35,100-103,168-171,236-239:205+36-39,104-107,172-175,240-243:18+40-43,108-111,176-179,244-247:19+44-47,112-115,180-183,248-251:86+48-51,116-119,184-187,252-255:87+52-55,120-123,188-191,256-259:154+56-59,124-127,192-195,260-263:155+60-63,128-131,196-199,264-267:222+64-67,132-135,200-203,268-271:223 -m 32G@0,12G@1
-    else
-	:
-    fi
-fi
-
-if [ ${go} -eq 1 ]; then
-    cd $ABS_SRCDIR
-    make $fn
-
-    PDSH_SSH_ARGS_APPEND="-tt -q" pdsh -t 2 -w $NODES \
-	ulimit -u 16384; 
-    PDSH_SSH_ARGS_APPEND="-tt -q" pdsh -t 2 -w $NODES \
-	ulimit -s unlimited
-
-    sudo $MCEXEC $mcexecopt ./$fn
-fi
-

test/uti/CT33.c

@@ -22,54 +22,46 @@ int sem;
 pthread_barrier_t bar;
 int flag;
 pthread_t thr;
-long t_fwq, t_futex_wake, t_futex_wait;
-long t_fwq2;
+long t_futex_wait, t_fwq;
 long nloop;
-long blocktime = 10 * 1000 * 1000L;
+long blocktime = 10L * 1000 * 1000;
+int linux_run;
 
 void *util_fn(void *arg)
 {
 	int i;
 	int ret;
-    long start, end;
-    long start2, end2;
+	long start, end;
+	int testid = 32101;
 
 	print_cpu_last_executed_on("Utility thread");
 
-	ret = syscall(732);
-	OKNGNOJUMP(ret == -1, "Utility thread is running on Linux\n");
-
-	/* Measure fwq time */
-	start = rdtsc_light();
-	for (i = 0; i < nloop; i++) {
-		fwq(blocktime);
+	if (!linux_run) {
+		ret = syscall(732);
+		OKNGNOJUMP(ret == -1, "Utility thread is running on Linux\n");
 	}
-	end = rdtsc_light();
-	t_fwq2 += end - start;
 
-	/* Measure fwq + futex time */
-	syscall(701, 1 | 2 | 0x80000000);
 	pthread_barrier_wait(&bar);
-	start = rdtsc_light();
+
 	for (i = 0; i < nloop; i++) {
-		start2 = rdtsc_light();
+		start = rdtsc_light();
 
 		fwq(blocktime);
 
-		end2 = rdtsc_light();
-		t_fwq += end2 - start2;
+		end = rdtsc_light();
+		t_fwq += end - start;
 
-		if ((ret = syscall(__NR_futex, &sem, FUTEX_WAKE, 1, NULL, NULL, 0)) != 1) {
-			printf("Error: futex wake failed (%d,%s)\n", ret, strerror(errno));
+		sem = i + 1;
+		if ((ret = syscall(__NR_futex, &sem, FUTEX_WAKE, 1,
+				   NULL, NULL, 0)) != 1) {
+			printf("Error: futex wake: %d,%d\n", ret, errno);
 		}
 
 		//pthread_barrier_wait(&bar);
+
 	}
-	end = rdtsc_light();
-	t_futex_wake += end - start;
-
-	syscall(701, 4 | 8 | 0x80000000);
 
+	ret = 0;
  fn_fail:
 	return NULL;
 }
@@ -81,26 +73,29 @@ static struct option options[] = {
 
 int main(int argc, char **argv)
 {
-	int i, j;
+	int i;
 	int ret;
-	long start, end;
+    long start, end;
 	cpu_set_t cpuset;
 	pthread_attr_t attr;
 	pthread_barrierattr_t bar_attr;
 	struct sched_param param = { .sched_priority = 99 };
 	int opt;
 
-	while ((opt = getopt_long(argc, argv, "+b:", options, NULL)) != -1) {
+	while ((opt = getopt_long(argc, argv, "+b:l", options, NULL)) != -1) {
 		switch (opt) {
-			case 'b':
-				blocktime = atoi(optarg);
-				break;
-			default: /* '?' */
-				printf("unknown option %c\n", optopt);
-				exit(1);
+		case 'b':
+			blocktime = atoi(optarg);
+			break;
+		case 'l':
+			linux_run = 1;
+			break;
+		default: /* '?' */
+			printf("unknown option %c\n", optopt);
+			exit(1);
 		}
 	}
-	nloop = 10 * 1000000000UL / blocktime;
+	nloop = (10 * 1000000000UL) / blocktime;
 	printf("[INFO] nloop=%ld,blocktime=%ld\n", nloop, blocktime);
 
 	
@@ -117,14 +112,8 @@ int main(int argc, char **argv)
 	pthread_barrierattr_init(&bar_attr);
 	pthread_barrier_init(&bar, &bar_attr, 2);
 
-	ret = syscall(732);
-	OKNGNOJUMP(ret != -1, "Master thread is running on McKernel\n");
-
-	ret = syscall(731, 1, NULL);
-	OKNGNOJUMP(ret != -1, "util_indicate_clone\n");
-
 	if ((ret = pthread_attr_init(&attr))) {
- 		printf("Error: pthread_attr_init failed: %s\n", strerror(errno));
+ 		printf("Error: pthread_attr_init: %s\n", strerror(errno));
 		goto fn_fail;
 	}
 
@@ -136,32 +125,45 @@ int main(int argc, char **argv)
 		goto fn_fail;
 	}
 
+	if (!linux_run) {
+		ret = syscall(732);
+		OKNGNOJUMP(ret != -1, "Master thread is running on McKernel\n");
+
+		ret = syscall(731, 1, NULL);
+		OKNGNOJUMP(ret != -1, "util_indicate_clone\n");
+	}
+
 	if ((ret = pthread_create(&thr, &attr, util_fn, NULL))) {
 		printf("Error: pthread_create: %s\n", strerror(errno));
 		goto fn_fail;
 	}
 
 	if ((ret = sched_setscheduler(0, SCHED_FIFO, &param))) {
-		printf("Error: sched_setscheduler: %s\n", strerror(errno));
-		goto fn_fail;
+		printf("Warning: sched_setscheduler: %s\n", strerror(errno));
 	}
 
+	if (!linux_run) {
+		syscall(701, 1 | 2);
+	}
 	pthread_barrier_wait(&bar);
 	start = rdtsc_light();
 	for (i = 0; i < nloop; i++) {
-		
-		if ((ret = syscall(__NR_futex, &sem, FUTEX_WAIT, 0, NULL, NULL, 0))) {
-			printf("Error: futex wait: %s\n", strerror(errno));
+		if ((ret = syscall(__NR_futex, &sem, FUTEX_WAIT, i, NULL, NULL, 0))) {
+			printf("Error: futex wait failed (%s)\n", strerror(errno));
 		}
 
-		//pthread_barrier_wait(&bar);
+		//pthread_barrier_wait(&bar); /* 2nd futex */
 	}
 	end = rdtsc_light();
 	t_futex_wait += end - start;
+	if (!linux_run) {
+		syscall(701, 4 | 8);
+	}
 
 	pthread_join(thr, NULL);
-	printf("[INFO] compute: %ld, wake: %ld, wait: %ld, wake - compute: %ld, wait - compute: %ld (cycles)\n", t_fwq, t_futex_wake, t_futex_wait, (t_futex_wake - t_fwq) / nloop, (t_futex_wait - t_fwq) / nloop);
+	printf("[INFO] waiter: %ld cycles, waker: %ld cycles, (waiter - waker) / nloop: %ld cycles\n", t_futex_wait, t_fwq, (t_futex_wait - t_fwq) / nloop);
 
+	ret = 0;
  fn_fail:
 	return ret;
 }

test/uti/CT33.sh

@@ -1,93 +0,0 @@
-#!/usr/bin/bash
-
-bn=`basename $0`
-fn=`echo $bn | sed 's/.sh//'`
-
-stop=0
-reboot=0
-go=0
-mck=0
-NNODES=1
-NPROC=$((1 * NNODES))
-LASTNODE=8200
-use_hfi=0
-
-while getopts srgmh:N:P:L: OPT
-do
-        case ${OPT} in
-	    s) stop=1
-		;;
-            r) reboot=1
-                ;;
-            g) go=1
-                ;;
-            m) mck=1
-                ;;
-	    h) use_hfi=1
-		;;
-	    N) NNODES=$OPTARG
-		;;
-	    P) NPROC=$OPTARG
-		;;
-	    L) LASTNODE=$OPTARG
-		;;
-            *) echo "invalid option -${OPT}" >&2
-                exit 1
-        esac
-done
-
-MYHOME=/work/gg10/e29005
-ABS_SRCDIR=${MYHOME}/project/os/mckernel/test/uti
-MCK=${MYHOME}/project/os/install
-
-nodes=`echo $(seq -s ",c" $(($LASTNODE + 1 - $NNODES)) $LASTNODE) | sed 's/^/c/'`
-PPN=$((NPROC / NNODES))
-echo NPROC=$NPROC NNODES=$NNODES PPN=$PPN nodes=$nodes
-
-if [ "`cat /etc/mtab | while read line; do cut -d" " -f 2; done | grep /work`" == "" ]; then
-    PDSH_SSH_ARGS_APPEND="-tt -q" pdsh -t 2 -w $nodes sudo mount /work
-fi
-
-if [ ${mck} -eq 1 ]; then
-    MCEXEC="${MCK}/bin/mcexec"
-    mcexecopt="--enable-uti"
-    if [ ${use_hfi} -eq 1 ]; then
-	mcexecopt="--enable-hfi1 $mcexecopt"
-    fi
-else
-    MCEXEC=
-    mcexecopt=
-fi
-
-if [ ${stop} -eq 1 ]; then
-    if [ ${mck} -eq 1 ]; then
-	PDSH_SSH_ARGS_APPEND="-tt -q" pdsh -t 2 -w $nodes \
-	    /sbin/pidof mcexec \| xargs -r kill -9
-	PDSH_SSH_ARGS_APPEND="-tt -q" pdsh -t 2 -w $nodes \
-	    sudo ${MCK}/sbin/mcstop+release.sh
-    else
-	:
-    fi
-fi
-
-if [ ${reboot} -eq 1 ]; then
-    if [ ${mck} -eq 1 ]; then
-	PDSH_SSH_ARGS_APPEND="-tt -q" pdsh -t 2 -w $nodes \
-	    sudo ${MCK}/sbin/mcreboot.sh -c 2-17,70-85,138-153,206-221,20-35,88-103,156-171,224-239,36-51,104-119,172-187,240-255,52-67,120-135,188-203,256-271 -r 2-5,70-73,138-141,206-209:0+6-9,74-77,142-145,210-213:1+10-13,78-81,146-149,214-217:68+14-17,82-85,150-153,218-221:69+20-23,88-91,156-159,224-227:136+24-27,92-95,160-163,228-231:137+28-31,96-99,164-167,232-235:204+32-35,100-103,168-171,236-239:205+36-39,104-107,172-175,240-243:18+40-43,108-111,176-179,244-247:19+44-47,112-115,180-183,248-251:86+48-51,116-119,184-187,252-255:87+52-55,120-123,188-191,256-259:154+56-59,124-127,192-195,260-263:155+60-63,128-131,196-199,264-267:222+64-67,132-135,200-203,268-271:223 -m 32G@0,12G@1
-    else
-	:
-    fi
-fi
-
-if [ ${go} -eq 1 ]; then
-    cd $ABS_SRCDIR
-    make $fn
-
-    PDSH_SSH_ARGS_APPEND="-tt -q" pdsh -t 2 -w $nodes \
-	ulimit -u 16384; 
-    PDSH_SSH_ARGS_APPEND="-tt -q" pdsh -t 2 -w $nodes \
-	ulimit -s unlimited
-
-    sudo $MCEXEC $mcexecopt ./$fn
-fi
-

test/uti/CT34.c

@@ -1,62 +1,168 @@
 #define _GNU_SOURCE
 #include <stdlib.h>
 #include <stdio.h>
+#include <string.h>
 #include <pthread.h>
 #include <errno.h>
-#include <unistd.h>
-#include <string.h>
-#include <stdint.h>
-#include <sys/syscall.h>
 #include <sys/mman.h>
-#include <sched.h>
+#include <unistd.h>
+#include <sys/syscall.h>
+#include <linux/futex.h>
+#include <getopt.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdint.h>
 #include "util.h"
 
+#define WAITER_CPU 0
+#define WAKER_CPU 1
+
+int sem;
+pthread_barrier_t bar;
+int flag;
+pthread_t thr;
+long t_futex_wait, t_fwq;
+long nloop;
+long blocktime = 10L * 1000 * 1000;
+int linux_run;
+
 void *util_fn(void *arg)
 {
+	int i;
 	int ret;
-	ret = syscall(732);
-	OKNGNOJUMP(ret == -1, "Utility thread is running on Linux\n");
+	long start, end;
+	int testid = 32101;
+
+	print_cpu_last_executed_on("Utility thread");
+
+	if (!linux_run) {
+		ret = syscall(732);
+		OKNGNOJUMP(ret == -1, "Utility thread is running on Linux\n");
+	}
+
+	pthread_barrier_wait(&bar);
+	start = rdtsc_light();
+	for (i = 0; i < nloop; i++) {
+		
+		if ((ret = syscall(__NR_futex, &sem, FUTEX_WAIT, i, NULL, NULL, 0))) {
+			printf("Error: futex wait failed (%s)\n", strerror(errno));
+		}
+
+		//pthread_barrier_wait(&bar); /* 2nd futex */
+	}
+	end = rdtsc_light();
+	t_futex_wait += end - start;
+
+	ret = 0;
  fn_fail:
 	return NULL;
 }
 
-int my_thread_create()
+static struct option options[] = {
+	/* end */
+	{ NULL, 0, NULL, 0, }
+};
+
+int main(int argc, char **argv)
 {
-	pthread_t thr;
-	int ret = 0;
+	int i;
+	int ret;
+	long start, end;
+	cpu_set_t cpuset;
+	pthread_attr_t attr;
+	pthread_barrierattr_t bar_attr;
+	struct sched_param param = { .sched_priority = 99 };
+	int opt;
 
-	ret = syscall(731, 1, NULL);
-	OKNGNOJUMP(ret == 0, "util_indicate_clone,ret=%d,errno=%d\n", ret, errno);
+	while ((opt = getopt_long(argc, argv, "+b:l", options, NULL)) != -1) {
+		switch (opt) {
+		case 'b':
+			blocktime = atoi(optarg);
+			break;
+		case 'l':
+			linux_run = 1;
+			break;
+		default: /* '?' */
+			printf("unknown option %c\n", optopt);
+			exit(1);
+		}
+	}
+	nloop = (10 * 1000000000UL) / blocktime;
+	printf("[INFO] nloop=%ld,blocktime=%ld\n", nloop, blocktime);
 
-	if ((ret = pthread_create(&thr, NULL, util_fn, NULL))) {
+	
+ 	CPU_ZERO(&cpuset);
+	CPU_SET(WAKER_CPU, &cpuset);
+	if ((ret = sched_setaffinity(0, sizeof(cpu_set_t), &cpuset))) {
+ 		printf("Error: sched_setaffinity: %s\n", strerror(errno));
+		goto fn_fail;
+	}
+	print_cpu_last_executed_on("Master thread");
+
+	fwq_init();
+
+	pthread_barrierattr_init(&bar_attr);
+	pthread_barrier_init(&bar, &bar_attr, 2);
+
+	if ((ret = pthread_attr_init(&attr))) {
+ 		printf("Error: pthread_attr_init: %s\n", strerror(errno));
+		goto fn_fail;
+	}
+
+ 	CPU_ZERO(&cpuset);
+	CPU_SET(WAITER_CPU, &cpuset);
+
+	if ((ret = pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset))) {
+ 		printf("Error: pthread_attr_setaffinity_np: %s\n", strerror(errno));
+		goto fn_fail;
+	}
+
+	if (!linux_run) {
+		ret = syscall(732);
+		OKNGNOJUMP(ret != -1, "Master thread is running on McKernel\n");
+
+		ret = syscall(731, 1, NULL);
+		OKNGNOJUMP(ret != -1, "util_indicate_clone\n");
+	}
+
+	if ((ret = pthread_create(&thr, &attr, util_fn, NULL))) {
 		printf("Error: pthread_create: %s\n", strerror(errno));
+		goto fn_fail;
 	}
 
-	if ((ret = pthread_join(thr, NULL))) {
-		printf("Error: pthread_join: %s\n", strerror(errno));
+	if ((ret = sched_setscheduler(0, SCHED_FIFO, &param))) {
+		printf("Warning: sched_setscheduler: %s\n", strerror(errno));
 	}
 
- fn_exit:
-	return ret;
+	if (!linux_run) {
+		syscall(701, 1 | 2);
+	}
+	pthread_barrier_wait(&bar);
+	for (i = 0; i < nloop; i++) {
+		start = rdtsc_light();
 
+		fwq(blocktime);
+
+		end = rdtsc_light();
+		t_fwq += end - start;
+
+		sem = i + 1;
+		if ((ret = syscall(__NR_futex, &sem, FUTEX_WAKE, 1,
+				   NULL, NULL, 0)) != 1) {
+			printf("Error: futex wake: %d, %d\n", ret, errno);
+		}
+
+		//pthread_barrier_wait(&bar);
+	}
+	if (!linux_run) {
+		syscall(701, 4 | 8);
+	}
+
+	pthread_join(thr, NULL);
+	printf("[INFO] waiter: %ld cycles, waker: %ld cycles, (waiter - waker) / nloop: %ld cycles\n", t_futex_wait, t_fwq, (t_futex_wait - t_fwq) / nloop);
+
+	ret = 0;
  fn_fail:
-	ret = -1;
-	goto fn_exit;
-}
-
-int
-main(int argc, char **argv)
-{
-	int ret = 0;
-
-	if ((ret = my_thread_create())) {
-		printf("Error: my_thread_create,ret=%d\n", ret);
-	}
-
- fn_exit:
 	return ret;
-
- fn_fail:
-	ret = -1;
-	goto fn_exit;
 }

test/uti/CT35.c

@@ -0,0 +1,62 @@
+#define _GNU_SOURCE
+#include <stdlib.h>
+#include <stdio.h>
+#include <pthread.h>
+#include <errno.h>
+#include <unistd.h>
+#include <string.h>
+#include <stdint.h>
+#include <sys/syscall.h>
+#include <sys/mman.h>
+#include <sched.h>
+#include "util.h"
+
+void *util_fn(void *arg)
+{
+	int ret;
+	ret = syscall(732);
+	OKNGNOJUMP(ret == -1, "Utility thread is running on Linux\n");
+ fn_fail:
+	return NULL;
+}
+
+int my_thread_create()
+{
+	pthread_t thr;
+	int ret = 0;
+
+	ret = syscall(731, 1, NULL);
+	OKNGNOJUMP(ret == 0, "util_indicate_clone,ret=%d,errno=%d\n", ret, errno);
+
+	if ((ret = pthread_create(&thr, NULL, util_fn, NULL))) {
+		printf("Error: pthread_create: %s\n", strerror(errno));
+	}
+	
+	if ((ret = pthread_join(thr, NULL))) {
+		printf("Error: pthread_join: %s\n", strerror(errno));
+	}
+
+ fn_exit:
+	return ret;
+
+ fn_fail:
+	ret = -1;
+	goto fn_exit;
+}
+
+int
+main(int argc, char **argv)
+{
+	int ret = 0;
+
+	if ((ret = my_thread_create())) {
+		printf("Error: my_thread_create,ret=%d\n", ret);
+	}
+
+ fn_exit:
+	return ret;
+
+ fn_fail:
+	ret = -1;
+	goto fn_exit;
+}

test/uti/Makefile

@@ -1,13 +1,14 @@
 .SUFFIXES:	# Disable implicit rules
 
-SYSCALLL_INTERCEPT_DIR=$(HOME)/usr
-UTI_DIR=$(HOME)/project/uti/install
+SYSCALLL_INTERCEPT_DIR=$(MCK_DIR)
+include $(HOME)/.mck_test_config.mk
 
 CC = gcc
 
 CPPFLAGS = -I$(UTI_DIR)/include
 CCFLAGS = -g -O0
 LDFLAGS = -L$(UTI_DIR)/lib -Wl,-rpath,$(UTI_DIR)/lib -luti -lpthread -lrt
+LDFLAGS2 = -lpthread -lrt
 
 SRCS = $(shell ls CT*.c)
 EXES = $(SRCS:.c=)
@@ -27,9 +28,21 @@ file::
 CT30.o:: CT30.c
 	icc $(CCFLAGS) -qopenmp $(CPPFLAGS) -c $<
 
-CT30: CT30.o
+CT30: CT30.o util.o
 	icc -o $@ $^ $(LDFLAGS) -qopenmp
 
+CT31: CT31.o util.o
+	$(CC) -o $@ $^ $(LDFLAGS2) $(CPPFLAGS)
+
+CT32: CT32.o util.o
+	$(CC) -o $@ $^ $(LDFLAGS2) $(CPPFLAGS)
+
+CT33: CT33.o util.o
+	$(CC) -o $@ $^ $(LDFLAGS2) $(CPPFLAGS)
+
+CT34: CT34.o util.o
+	$(CC) -o $@ $^ $(LDFLAGS2) $(CPPFLAGS)
+
 %.o:: %.c
 	$(CC) $(CCFLAGS) $(CPPFLAGS) -c $<
 

test/uti/README

@@ -338,14 +338,46 @@ CT29 no reverse offload
 CT30 CT21にopenmpスレッドを追加したテスト
 
 CT31 pthread_cond_waitオーバーヘッド測定
-* waiterとwakerのCPUは、それぞれ、WAITER_CPU、WAKER_CPUで設定
+* Linuxがwaker、McKernelがwaiter
 
-CT32 futex waitオーバーヘッド測定
-* waiterとwakerのCPUは、それぞれ、WAITER_CPU、WAKER_CPUで設定
+CT32 pthread_cond_waitオーバーヘッド測定
+* Linuxがwaiter、McKernelがwaker
 
-CT33 futex wakeオーバーヘッド測定
-* waiterとwakerのCPUは、それぞれ、WAITER_CPU、WAKER_CPUで設定
+CT33 Main threadでfutex_wait, UTI threadでfutex_wakeをした場合のオーバーヘッド測定
+* Linuxがwaker、McKernelがwaiter
 
-CT34 繰り返しpthread_create
+CT34 UTI threadでfutex_wait, Main threadでfutex_wakeをした場合のオーバーヘッド測定
+* Linuxがwaiter、McKernelがwaker
 
-CT35 LD_PRELOADでsyscall_interceptを用いたsoをつけた場合のテスト
+CT35 繰り返しpthread_create
+
+CT36 LD_PRELOADでsyscall_interceptを用いたsoをつけた場合のテスト
+
+
+==========
+How to run
+==========
+
+Prepare $HOME/.mck_test_config. Example:
+
+# Config file for McKernel tests
+MCK_DIR=/home/m-takagi/project/os/install
+BIN=/home/m-takagi/project/os/install/bin
+SBIN=/home/m-takagi/project/os/install/sbin
+: ${OSTEST:=/home/m-takagi/project/src/ostest}
+: ${LTP:=/home/m-takagi/project/src/ltp/install}
+BOOTPARAM="-k 0 -f LOG_LOCAL6  -c 1-7,9-15,17-23,25-31 -m 10G@0,10G@1 -r 1-7:0+9-15:8+17-23:16+25-31:24 -O"
+: ${MCKERNEL_VERSION:=1.5.0}
+
+Prepare $HOME/.mck_test_config.mk. Example:
+
+# Config file for McKernel tests
+BIN ?= /home/m-takagi/project/os/install/bin
+SBIN ?= /home/m-takagi/project/os/install/sbin
+OSTEST ?=
+LTP ?=
+BOOTPARAM ?= -c 1-7,9-15,17-23,25-31 -m 10G@0,10G@1 -r 1-7:0+9-15:8+17-23:16+25-31:24
+MCK_DIR ?= /home/m-takagi/project/os/install
+ARCH ?= x86_64
+TARGET ?= smp-x86
+UTI_DIR ?= /home/m-takagi/project/uti/install_mckernel

test/uti/arm64/CT31.c

@@ -0,0 +1,171 @@
+#define _GNU_SOURCE
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <pthread.h>
+#include <errno.h>
+#include <unistd.h>
+#include <sys/syscall.h>
+#include <sys/mman.h>
+#include <signal.h>
+#include <getopt.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include "util.h"
+
+#define WAITER_CPU 0
+#define WAKER_CPU 1
+
+pthread_mutex_t mutex;
+pthread_cond_t cond;
+pthread_barrier_t bar;
+int flag;
+pthread_t thr;
+long t_cond_wait, t_fwq;
+long nloop;
+long blocktime = 10L * 1000 * 1000;
+int linux_run;
+
+void *util_fn(void *arg)
+{
+	int i;
+	int ret;
+	long start, end;
+	unsigned long mem;
+
+	print_cpu_last_executed_on("Utility thread");
+
+	if (!linux_run) {
+		ret = syscall(732);
+		OKNGNOJUMP(ret == -1, "Utility thread is running on Linux\n");
+	}
+
+	pthread_barrier_wait(&bar);
+	for (i = 0; i < nloop; i++) {
+		start = rdtsc_light();
+
+		fwq(blocktime, &mem);
+
+		end = rdtsc_light();
+		t_fwq += end - start;
+
+		pthread_mutex_lock(&mutex);
+		flag = 1;
+		pthread_cond_signal(&cond);
+		pthread_mutex_unlock(&mutex);
+	}
+
+ fn_fail:
+	return NULL;
+}
+
+static struct option options[] = {
+	/* end */
+	{ NULL, 0, NULL, 0, }
+};
+
+int main(int argc, char **argv)
+{
+	int i;
+	int ret;
+	long start, end;
+	cpu_set_t cpuset;
+	pthread_attr_t attr;
+	pthread_barrierattr_t bar_attr;
+	struct sched_param param = { .sched_priority = 99 };
+	int opt;
+	unsigned long mem;
+
+	while ((opt = getopt_long(argc, argv, "+b:l", options, NULL)) != -1) {
+		switch (opt) {
+		case 'b':
+			blocktime = atoi(optarg);
+			break;
+		case 'l':
+			linux_run = 1;
+			break;
+		default: /* '?' */
+			printf("unknown option %c\n", optopt);
+			exit(1);
+		}
+	}
+	nloop = (10 * 1000000000UL) / blocktime;
+	printf("[INFO] nloop=%ld,blocktime=%ld\n", nloop, blocktime);
+
+	
+ 	CPU_ZERO(&cpuset);
+	CPU_SET(WAITER_CPU, &cpuset);
+	if ((ret = sched_setaffinity(0, sizeof(cpu_set_t), &cpuset))) {
+ 		printf("Error: sched_setaffinity: %s\n", strerror(errno));
+		goto fn_fail;
+	}
+	print_cpu_last_executed_on("Master thread");
+
+	fwq_init(&mem);
+
+	pthread_mutex_init(&mutex, NULL);
+	pthread_cond_init(&cond, NULL);
+
+	pthread_barrierattr_init(&bar_attr);
+	pthread_barrier_init(&bar, &bar_attr, 2);
+
+	if (!linux_run) {
+		ret = syscall(732);
+		OKNGNOJUMP(ret != -1, "Master thread is running on McKernel\n");
+
+		ret = syscall(731, 1, NULL);
+		OKNGNOJUMP(ret != -1, "util_indicate_clone\n");
+	}
+
+	if ((ret = pthread_attr_init(&attr))) {
+ 		printf("%s: Error: pthread_attr_init failed (%d)\n", __FUNCTION__, ret);
+		goto fn_fail;
+	}
+
+ 	CPU_ZERO(&cpuset);
+	CPU_SET(WAKER_CPU, &cpuset);
+
+	if ((ret = pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset))) {
+ 		printf("%s: Error: pthread_attr_setaffinity_np failed (%d)\n", __FUNCTION__, ret);
+		goto fn_fail;
+	}
+
+	if ((ret = pthread_create(&thr, &attr, util_fn, NULL))) {
+		fprintf(stderr, "Error: pthread_create failed (%d)\n", ret);
+		goto fn_fail;
+	}
+
+	if ((ret = sched_setscheduler(0, SCHED_FIFO, &param))) {
+		fprintf(stderr, "Warning: sched_setscheduler: %s\n",
+			strerror(errno));
+	}
+
+	if (!linux_run) {
+		syscall(701, 1 | 2);
+	}
+	pthread_barrier_wait(&bar);
+	for (i = 0; i < nloop; i++) {
+		start = rdtsc_light();
+		
+		pthread_mutex_lock(&mutex); /* no futex */
+		while(!flag) {
+			pthread_cond_wait(&cond, &mutex); /* 1st futex */
+		}
+		flag = 0;
+		pthread_mutex_unlock(&mutex); /* 2nd futex */
+
+		end = rdtsc_light();
+		t_cond_wait += end - start;
+	}
+	if (!linux_run) {
+		syscall(701, 4 | 8);
+	}
+
+	pthread_join(thr, NULL);
+	printf("[INFO] waker: %ld nsec, waiter: %ld nsec, (waiter - waker) / nloop: %ld nsec\n", t_fwq * 10, t_cond_wait * 10, (t_cond_wait - t_fwq) * 10 / nloop);
+
+	ret = 0;
+ fn_fail:
+	return ret;
+}

test/uti/arm64/CT32.c

@@ -0,0 +1,171 @@
+#define _GNU_SOURCE
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <pthread.h>
+#include <errno.h>
+#include <unistd.h>
+#include <sys/syscall.h>
+#include <sys/mman.h>
+#include <signal.h>
+#include <getopt.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include "util.h"
+
+#define WAITER_CPU 0
+#define WAKER_CPU 1
+
+pthread_mutex_t mutex;
+pthread_cond_t cond;
+pthread_barrier_t bar;
+int flag;
+pthread_t thr;
+long t_cond_wait, t_fwq;
+long nloop;
+long blocktime = 10L * 1000 * 1000;
+int linux_run;
+
+void *util_fn(void *arg)
+{
+	int i;
+	int ret;
+	long start, end;
+	unsigned long mem;
+
+	print_cpu_last_executed_on("Utility thread");
+
+	if (!linux_run) {
+		ret = syscall(732);
+		OKNGNOJUMP(ret == -1, "Utility thread is running on Linux\n");
+	}
+
+	pthread_barrier_wait(&bar);
+	for (i = 0; i < nloop; i++) {
+		start = rdtsc_light();
+		
+		pthread_mutex_lock(&mutex); /* no futex */
+		while(!flag) {
+			pthread_cond_wait(&cond, &mutex); /* 1st futex */
+		}
+		flag = 0;
+		pthread_mutex_unlock(&mutex); /* 2nd futex */
+
+		end = rdtsc_light();
+		t_cond_wait += end - start;
+	}
+
+ fn_fail:
+	return NULL;
+}
+
+static struct option options[] = {
+	/* end */
+	{ NULL, 0, NULL, 0, }
+};
+
+int main(int argc, char **argv)
+{
+	int i;
+	int ret;
+	long start, end;
+	cpu_set_t cpuset;
+	pthread_attr_t attr;
+	pthread_barrierattr_t bar_attr;
+	struct sched_param param = { .sched_priority = 99 };
+	int opt;
+	unsigned long mem;
+
+	while ((opt = getopt_long(argc, argv, "+b:l", options, NULL)) != -1) {
+		switch (opt) {
+		case 'b':
+			blocktime = atoi(optarg);
+			break;
+		case 'l':
+			linux_run = 1;
+			break;
+		default: /* '?' */
+			printf("unknown option %c\n", optopt);
+			exit(1);
+		}
+	}
+	nloop = (10 * 1000000000UL) / blocktime;
+	printf("[INFO] nloop=%ld,blocktime=%ld\n", nloop, blocktime);
+
+	
+ 	CPU_ZERO(&cpuset);
+	CPU_SET(WAITER_CPU, &cpuset);
+	if ((ret = sched_setaffinity(0, sizeof(cpu_set_t), &cpuset))) {
+ 		printf("Error: sched_setaffinity: %s\n", strerror(errno));
+		goto fn_fail;
+	}
+	print_cpu_last_executed_on("Master thread");
+
+	fwq_init(&mem);
+
+	pthread_mutex_init(&mutex, NULL);
+	pthread_cond_init(&cond, NULL);
+
+	pthread_barrierattr_init(&bar_attr);
+	pthread_barrier_init(&bar, &bar_attr, 2);
+
+	if (!linux_run) {
+		ret = syscall(732);
+		OKNGNOJUMP(ret != -1, "Master thread is running on McKernel\n");
+
+		ret = syscall(731, 1, NULL);
+		OKNGNOJUMP(ret != -1, "util_indicate_clone\n");
+	}
+
+	if ((ret = pthread_attr_init(&attr))) {
+ 		printf("%s: Error: pthread_attr_init failed (%d)\n", __FUNCTION__, ret);
+		goto fn_fail;
+	}
+
+ 	CPU_ZERO(&cpuset);
+	CPU_SET(WAKER_CPU, &cpuset);
+
+	if ((ret = pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset))) {
+ 		printf("%s: Error: pthread_attr_setaffinity_np failed (%d)\n", __FUNCTION__, ret);
+		goto fn_fail;
+	}
+
+	if ((ret = pthread_create(&thr, &attr, util_fn, NULL))) {
+		fprintf(stderr, "Error: pthread_create failed (%d)\n", ret);
+		goto fn_fail;
+	}
+
+	if ((ret = sched_setscheduler(0, SCHED_FIFO, &param))) {
+		fprintf(stderr, "Warning: sched_setscheduler: %s\n",
+			strerror(errno));
+	}
+
+	if (!linux_run) {
+		syscall(701, 1 | 2);
+	}
+	pthread_barrier_wait(&bar);
+	for (i = 0; i < nloop; i++) {
+		start = rdtsc_light();
+
+		fwq(blocktime, &mem);
+
+		end = rdtsc_light();
+		t_fwq += end - start;
+
+		pthread_mutex_lock(&mutex);
+		flag = 1;
+		pthread_cond_signal(&cond);
+		pthread_mutex_unlock(&mutex);
+	}
+	if (!linux_run) {
+		syscall(701, 4 | 8);
+	}
+
+	pthread_join(thr, NULL);
+	printf("[INFO] waker: %ld nsec, waiter: %ld nsec, (waiter - waker) / nloop: %ld nsec\n", t_fwq * 10, t_cond_wait * 10, (t_cond_wait - t_fwq) * 10 / nloop);
+
+	ret = 0;
+ fn_fail:
+	return ret;
+}

test/uti/arm64/CT33.c

@@ -0,0 +1,171 @@
+#define _GNU_SOURCE
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <pthread.h>
+#include <errno.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#include <sys/syscall.h>
+#include <linux/futex.h>
+#include <getopt.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdint.h>
+#include "util.h"
+
+#define WAITER_CPU 0
+#define WAKER_CPU 1
+
+int sem;
+pthread_barrier_t bar;
+int flag;
+pthread_t thr;
+long t_futex_wait, t_fwq;
+long nloop;
+long blocktime = 10L * 1000 * 1000;
+int linux_run;
+
+void *util_fn(void *arg)
+{
+	int i;
+	int ret;
+	long start, end;
+	int testid = 32101;
+	unsigned long mem;
+
+	print_cpu_last_executed_on("Utility thread");
+
+	if (!linux_run) {
+		ret = syscall(732);
+		OKNGNOJUMP(ret == -1, "Utility thread is running on Linux\n");
+	}
+
+	pthread_barrier_wait(&bar);
+
+	for (i = 0; i < nloop; i++) {
+		start = rdtsc_light();
+
+		fwq(blocktime, &mem);
+
+		end = rdtsc_light();
+		t_fwq += end - start;
+
+		sem = i + 1;
+		if ((ret = syscall(__NR_futex, &sem, FUTEX_WAKE, 1,
+				   NULL, NULL, 0)) != 1) {
+			printf("Error: futex wake: %d,%d\n", ret, errno);
+		}
+
+		//pthread_barrier_wait(&bar);
+
+	}
+
+	ret = 0;
+ fn_fail:
+	return NULL;
+}
+
+static struct option options[] = {
+	/* end */
+	{ NULL, 0, NULL, 0, }
+};
+
+int main(int argc, char **argv)
+{
+	int i;
+	int ret;
+    long start, end;
+	cpu_set_t cpuset;
+	pthread_attr_t attr;
+	pthread_barrierattr_t bar_attr;
+	struct sched_param param = { .sched_priority = 99 };
+	int opt;
+	unsigned long mem;
+
+	while ((opt = getopt_long(argc, argv, "+b:l", options, NULL)) != -1) {
+		switch (opt) {
+		case 'b':
+			blocktime = atoi(optarg);
+			break;
+		case 'l':
+			linux_run = 1;
+			break;
+		default: /* '?' */
+			printf("unknown option %c\n", optopt);
+			exit(1);
+		}
+	}
+	nloop = (10 * 1000000000UL) / blocktime;
+	printf("[INFO] nloop=%ld,blocktime=%ld\n", nloop, blocktime);
+
+	
+ 	CPU_ZERO(&cpuset);
+	CPU_SET(WAITER_CPU, &cpuset);
+	if ((ret = sched_setaffinity(0, sizeof(cpu_set_t), &cpuset))) {
+ 		printf("Error: sched_setaffinity: %s\n", strerror(errno));
+		goto fn_fail;
+	}
+	print_cpu_last_executed_on("Master thread");
+
+	fwq_init(&mem);
+
+	pthread_barrierattr_init(&bar_attr);
+	pthread_barrier_init(&bar, &bar_attr, 2);
+
+	if ((ret = pthread_attr_init(&attr))) {
+ 		printf("Error: pthread_attr_init: %s\n", strerror(errno));
+		goto fn_fail;
+	}
+
+ 	CPU_ZERO(&cpuset);
+	CPU_SET(WAKER_CPU, &cpuset);
+
+	if ((ret = pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset))) {
+ 		printf("Error: pthread_attr_setaffinity_np: %s\n", strerror(errno));
+		goto fn_fail;
+	}
+
+	if (!linux_run) {
+		ret = syscall(732);
+		OKNGNOJUMP(ret != -1, "Master thread is running on McKernel\n");
+
+		ret = syscall(731, 1, NULL);
+		OKNGNOJUMP(ret != -1, "util_indicate_clone\n");
+	}
+
+	if ((ret = pthread_create(&thr, &attr, util_fn, NULL))) {
+		printf("Error: pthread_create: %s\n", strerror(errno));
+		goto fn_fail;
+	}
+
+	if ((ret = sched_setscheduler(0, SCHED_FIFO, &param))) {
+		printf("Warning: sched_setscheduler: %s\n", strerror(errno));
+	}
+
+	if (!linux_run) {
+		syscall(701, 1 | 2);
+	}
+	pthread_barrier_wait(&bar);
+	start = rdtsc_light();
+	for (i = 0; i < nloop; i++) {
+		if ((ret = syscall(__NR_futex, &sem, FUTEX_WAIT, i, NULL, NULL, 0))) {
+			printf("Error: futex wait failed (%s)\n", strerror(errno));
+		}
+
+		//pthread_barrier_wait(&bar); /* 2nd futex */
+	}
+	end = rdtsc_light();
+	t_futex_wait += end - start;
+	if (!linux_run) {
+		syscall(701, 4 | 8);
+	}
+
+	pthread_join(thr, NULL);
+	printf("[INFO] waiter: %ld nsec, waker: %ld nsec, (waiter - waker) / nloop: %ld nsec\n", t_futex_wait * 10, t_fwq * 10, (t_futex_wait - t_fwq) * 10 / nloop);
+
+	ret = 0;
+ fn_fail:
+	return ret;
+}

test/uti/arm64/CT34.c

@@ -0,0 +1,170 @@
+#define _GNU_SOURCE
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <pthread.h>
+#include <errno.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#include <sys/syscall.h>
+#include <linux/futex.h>
+#include <getopt.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdint.h>
+#include "util.h"
+
+#define WAITER_CPU 0
+#define WAKER_CPU 1
+
+int sem;
+pthread_barrier_t bar;
+int flag;
+pthread_t thr;
+long t_futex_wait, t_fwq;
+long nloop;
+long blocktime = 10L * 1000 * 1000;
+int linux_run;
+
+void *util_fn(void *arg)
+{
+	int i;
+	int ret;
+	long start, end;
+	int testid = 32101;
+	unsigned long mem;
+
+	print_cpu_last_executed_on("Utility thread");
+
+	if (!linux_run) {
+		ret = syscall(732);
+		OKNGNOJUMP(ret == -1, "Utility thread is running on Linux\n");
+	}
+
+	pthread_barrier_wait(&bar);
+	start = rdtsc_light();
+	for (i = 0; i < nloop; i++) {
+		
+		if ((ret = syscall(__NR_futex, &sem, FUTEX_WAIT, i, NULL, NULL, 0))) {
+			printf("Error: futex wait failed (%s)\n", strerror(errno));
+		}
+
+		//pthread_barrier_wait(&bar); /* 2nd futex */
+	}
+	end = rdtsc_light();
+	t_futex_wait += end - start;
+
+	ret = 0;
+ fn_fail:
+	return NULL;
+}
+
+static struct option options[] = {
+	/* end */
+	{ NULL, 0, NULL, 0, }
+};
+
+int main(int argc, char **argv)
+{
+	int i;
+	int ret;
+	long start, end;
+	cpu_set_t cpuset;
+	pthread_attr_t attr;
+	pthread_barrierattr_t bar_attr;
+	struct sched_param param = { .sched_priority = 99 };
+	int opt;
+	unsigned long mem;
+
+	while ((opt = getopt_long(argc, argv, "+b:l", options, NULL)) != -1) {
+		switch (opt) {
+		case 'b':
+			blocktime = atoi(optarg);
+			break;
+		case 'l':
+			linux_run = 1;
+			break;
+		default: /* '?' */
+			printf("unknown option %c\n", optopt);
+			exit(1);
+		}
+	}
+	nloop = (10 * 1000000000UL) / blocktime;
+	printf("[INFO] nloop=%ld,blocktime=%ld\n", nloop, blocktime);
+
+	
+ 	CPU_ZERO(&cpuset);
+	CPU_SET(WAKER_CPU, &cpuset);
+	if ((ret = sched_setaffinity(0, sizeof(cpu_set_t), &cpuset))) {
+ 		printf("Error: sched_setaffinity: %s\n", strerror(errno));
+		goto fn_fail;
+	}
+	print_cpu_last_executed_on("Master thread");
+
+	fwq_init(&mem);
+
+	pthread_barrierattr_init(&bar_attr);
+	pthread_barrier_init(&bar, &bar_attr, 2);
+
+	if ((ret = pthread_attr_init(&attr))) {
+ 		printf("Error: pthread_attr_init: %s\n", strerror(errno));
+		goto fn_fail;
+	}
+
+ 	CPU_ZERO(&cpuset);
+	CPU_SET(WAITER_CPU, &cpuset);
+
+	if ((ret = pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset))) {
+ 		printf("Error: pthread_attr_setaffinity_np: %s\n", strerror(errno));
+		goto fn_fail;
+	}
+
+	if (!linux_run) {
+		ret = syscall(732);
+		OKNGNOJUMP(ret != -1, "Master thread is running on McKernel\n");
+
+		ret = syscall(731, 1, NULL);
+		OKNGNOJUMP(ret != -1, "util_indicate_clone\n");
+	}
+
+	if ((ret = pthread_create(&thr, &attr, util_fn, NULL))) {
+		printf("Error: pthread_create: %s\n", strerror(errno));
+		goto fn_fail;
+	}
+
+	if ((ret = sched_setscheduler(0, SCHED_FIFO, &param))) {
+		printf("Warning: sched_setscheduler: %s\n", strerror(errno));
+	}
+
+	if (!linux_run) {
+		syscall(701, 1 | 2);
+	}
+	pthread_barrier_wait(&bar);
+	for (i = 0; i < nloop; i++) {
+		start = rdtsc_light();
+
+		fwq(blocktime, &mem);
+
+		end = rdtsc_light();
+		t_fwq += end - start;
+
+		sem = i + 1;
+		if ((ret = syscall(__NR_futex, &sem, FUTEX_WAKE, 1,
+				   NULL, NULL, 0)) != 1) {
+			printf("Error: futex wake: %d, %d\n", ret, errno);
+		}
+
+		//pthread_barrier_wait(&bar);
+	}
+	if (!linux_run) {
+		syscall(701, 4 | 8);
+	}
+
+	pthread_join(thr, NULL);
+	printf("[INFO] waiter: %ld nsec, waker: %ld nsec, (waiter - waker) / nloop: %ld nsec\n", t_futex_wait * 10, t_fwq * 10, (t_futex_wait - t_fwq) * 10 / nloop);
+
+	ret = 0;
+ fn_fail:
+	return ret;
+}

test/uti/arm64/Makefile

@@ -1,6 +1,9 @@
 # Makefile COPYRIGHT FUJITSU LIMITED 2019
 CC = gcc
-LDFLAGS = -Wall -lpthread
+LDFLAGS =  -Wall -lpthread
+
+CCFLAGS = -g -O0
+CPPFLAGS =
 
 SRCS = $(shell ls CT*.c)
 TARGET = $(SRCS:.c=)
@@ -10,5 +13,23 @@ all: $(TARGET)
 test: all
 	./run.sh
 
+%.o:: %.c
+	$(CC) $(CCFLAGS) $(CPPFLAGS) -c $<
+
+util.o:: util.c
+	$(CC) $(CCFLAGS) $(CPPFLAGS) -c $<
+
+CT31: CT31.o util.o
+	$(CC) -o $@ $^ $(LDFLAGS) $(CPPFLAGS)
+
+CT32: CT32.o util.o
+	$(CC) -o $@ $^ $(LDFLAGS) $(CPPFLAGS)
+
+CT33: CT33.o util.o
+	$(CC) -o $@ $^ $(LDFLAGS) $(CPPFLAGS)
+
+CT34: CT34.o util.o
+	$(CC) -o $@ $^ $(LDFLAGS) $(CPPFLAGS)
+
 clean:
-	rm -f $(TARGET)
+	rm -f $(TARGET) *.o

test/uti/arm64/util.c

@@ -0,0 +1,122 @@
+#define _GNU_SOURCE		 /* See feature_test_macros(7) */
+#include <unistd.h>
+#include <sys/syscall.h>   /* For SYS_xxx definitions */
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <sched.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <time.h>
+#include "util.h"
+
+#define TS2NS(sec, nsec) \
+	((unsigned long)(sec) * 1000000000ULL + \
+	 (unsigned long)(nsec))
+
+#define N_INIT 10000000
+
+static inline void FIXED_SIZE_WORK(unsigned long *ptr)
+{
+	asm volatile("mov %x0, x20\n"
+			 "add x20, x20, #1\n"
+			 "mov x20, %x0\n"
+			 : "+rm" (*ptr)
+			 :
+			 : "x20", "cc", "memory");
+}
+
+static inline void BULK_FSW(unsigned long n,
+				unsigned long *ptr)
+{
+	int j;
+
+	for (j = 0; j < (n); j++) {
+		FIXED_SIZE_WORK(ptr);
+	}
+}
+
+double nspw; /* nsec per work */
+unsigned long nsec;
+
+void fwq_init(unsigned long *mem)
+{
+	struct timespec start, end;
+	unsigned long nsec;
+
+	clock_gettime(CLOCK_THREAD_CPUTIME_ID, &start);
+	BULK_FSW(N_INIT, mem);
+	clock_gettime(CLOCK_THREAD_CPUTIME_ID, &end);
+	nsec = (TS2NS(end.tv_sec, end.tv_nsec) -
+		TS2NS(start.tv_sec, start.tv_nsec));
+	nspw = nsec / (double)N_INIT;
+	printf("nsec=%ld, nspw=%f\n", nsec, nspw);
+}
+
+void fwq(long delay_nsec, unsigned long *mem)
+{
+	BULK_FSW(delay_nsec / nspw, mem);
+}
+
+int print_cpu_last_executed_on(const char *name) {
+	char fn[256];
+	char* result;
+	pid_t tid = syscall(SYS_gettid);
+	int fd;
+	int offset;
+	int mpi_errno = 0;
+
+	sprintf(fn, "/proc/%d/task/%d/stat", getpid(), (int)tid);
+	//printf("fn=%s\n", fn);
+	fd = open(fn, O_RDONLY);
+	if(fd == -1) {
+		printf("open() failed\n");
+		goto fn_fail;
+	}
+
+	result = malloc(65536);
+	if(result == NULL) {
+		printf("malloc() failed");
+		goto fn_fail;
+	}
+
+	int amount = 0;
+	offset = 0;
+	while(1) {
+		amount = read(fd, result + offset, 65536);
+		//		printf("amount=%d\n", amount);
+		if(amount == -1) {
+			printf("read() failed");
+			goto fn_fail;
+		}
+		if(amount == 0) {
+			goto eof;
+		}
+		offset += amount;
+	}
+ eof:;
+	//printf("result:%s\n", result);
+
+	char* next_delim = result;
+	char* field;
+	int i;
+	for(i = 0; i < 39; i++) {
+		field = strsep(&next_delim, " ");
+	}
+
+	int cpu = sched_getcpu();
+	if(cpu == -1) {
+		printf("getcpu() failed\n");
+		goto fn_fail;
+	}
+
+	printf("[INFO] %s (tid: %d) is running on %02d,%02d\n", name, tid, atoi(field), cpu);
+ fn_exit:
+	free(result);
+	return mpi_errno;
+ fn_fail:
+	mpi_errno = -1;
+	goto fn_exit;
+}
+

test/uti/arm64/util.h

@@ -0,0 +1,70 @@
+#ifndef __UTIL_H_INCLUDED__
+#define __UTIL_H_INCLUDED__
+
+#include <stdint.h>
+
+#define isb()       asm volatile("isb" : : : "memory")
+
+#define DEBUG
+
+#ifdef DEBUG
+#define dprintf(...) do {			 \
+	char msg[1024];			 \
+	sprintf(msg, __VA_ARGS__);		 \
+	fprintf(stderr, "%s,%s", __func__, msg); \
+} while (0)
+#else
+#define dprintf(...) do {  } while (0)
+#endif
+
+#define eprintf(...) do {			 \
+	char msg[1024];			 \
+	sprintf(msg, __VA_ARGS__);		 \
+	fprintf(stderr, "%s,%s", __func__, msg); \
+} while (0)
+
+#define CHKANDJUMP(cond, err, ...) do { \
+	if (cond) {			\
+		eprintf(__VA_ARGS__);   \
+		ret = err;		\
+		goto fn_fail;		\
+	}				\
+} while (0)
+
+#define _OKNG(verb, jump, cond, fmt, args...) do {	\
+	if (cond) {					\
+		if (verb)				\
+			printf("[ OK ] " fmt, ##args);	\
+	} else {					\
+		printf("[ NG ] " fmt, ##args);		\
+		if (jump)				\
+			goto fn_fail;			\
+	}						\
+} while (0)
+
+#define OKNG(args...) _OKNG(1, 1, ##args)
+#define NG(args...) _OKNG(0, 1, ##args)
+#define OKNGNOJUMP(args...) _OKNG(1, 0, ##args)
+
+#define DIFFNSEC(end, start) ((end.tv_sec - start.tv_sec) * 1000000000UL + (end.tv_nsec - start.tv_nsec))
+#define TIMER_KIND CLOCK_MONOTONIC_RAW /* CLOCK_THREAD_CPUTIME_ID */
+
+static inline uint64_t rdtsc_light(void )
+{
+	unsigned long cval;
+
+	isb();
+	asm volatile("mrs %0, cntvct_el0" : "=r" (cval));
+
+	return cval;
+}
+
+extern double nspw; /* nsec per work */
+extern unsigned long nsec;
+
+void fwq_init(unsigned long *mem);
+void fwq(long delay_nsec, unsigned long *mem);
+int print_cpu_last_executed_on(const char *name);
+
+#endif
+