mcklogd: fixed termination method of mcklogd

Conflicts:
	configure
	kernel/process.c

Makefile.in

@@ -49,6 +49,7 @@ install::
 		mkdir -p -m 755 $(SBINDIR); \
 		install -m 755 arch/x86/tools/mcreboot-smp-x86.sh $(SBINDIR)/mcreboot.sh; \
 		install -m 755 arch/x86/tools/mcstop+release-smp-x86.sh $(SBINDIR)/mcstop+release.sh; \
+		install -m 755 arch/x86/tools/eclair-dump-backtrace.exp $(SBINDIR)/eclair-dump-backtrace.exp;\
 		mkdir -p -m 755 $(ETCDIR); \
 		install -m 644 arch/x86/tools/irqbalance_mck.service $(ETCDIR)/irqbalance_mck.service; \
 		install -m 644 arch/x86/tools/irqbalance_mck.in $(ETCDIR)/irqbalance_mck.in; \

arch/x86/kernel/cpu.c

@@ -148,7 +148,7 @@ extern char page_fault[], general_protection_exception[];
 extern char debug_exception[], int3_exception[];
 
 uint64_t boot_pat_state = 0;
-int no_turbo = 0; /* May be updated by early parsing of kargs */
+int no_turbo = 1; /* May be updated by early parsing of kargs */
 
 extern int num_processors; /* kernel/ap.c */
 struct pvclock_vsyscall_time_info *pvti = NULL;
@@ -844,6 +844,25 @@ void set_signal(int sig, void *regs, struct siginfo *info);
 void check_signal(unsigned long, void *, int);
 extern void tlb_flush_handler(int vector);
 
+void __show_stack(uintptr_t *sp) {
+	while (((uintptr_t)sp >= 0xffff800000000000)
+			&& ((uintptr_t)sp <  0xffffffff80000000)) {
+		uintptr_t fp;
+		uintptr_t ip;
+
+		fp = sp[0];
+		ip = sp[1];
+		kprintf("IP: %016lx, SP: %016lx, FP: %016lx\n", ip, (uintptr_t)sp, fp);
+		sp = (void *)fp;
+	}
+	return;
+}
+
+void show_context_stack(uintptr_t *rbp) {
+	__show_stack(rbp);
+	return;
+}
+
 void handle_interrupt(int vector, struct x86_user_context *regs)
 {
 	struct ihk_mc_interrupt_handler *h;
@@ -952,6 +971,9 @@ void handle_interrupt(int vector, struct x86_user_context *regs)
 
 			tlb_flush_handler(vector);
 	} 
+	else if (vector == 133) {
+		show_context_stack((uintptr_t *)regs->gpr.rbp);
+	}
 	else {
 		list_for_each_entry(h, &handlers[vector - 32], list) {
 			if (h->func) {
@@ -1079,6 +1101,10 @@ unhandled_page_fault(struct thread *thread, void *fault_addr, void *regs)
 
 	kprintf_unlock(irqflags);
 
+	if (!(error & PF_USER)) {
+		panic("panic: kernel mode PF");
+	}
+
 	/* TODO */
 	ihk_mc_debug_show_interrupt_context(regs);
 
@@ -1787,4 +1813,43 @@ int running_on_kvm(void) {
 	return 0;
 }
 
+void
+mod_nmi_ctx(void *nmi_ctx, void (*func)())
+{
+	unsigned long *l = nmi_ctx;
+	int i;
+	unsigned long flags;
+
+//struct x86_cpu_local_variables *v;
+
+//if(!ihk_mc_get_processor_id()) {
+//v = get_x86_this_cpu_local();
+//}
+	asm volatile("pushf; pop %0" : "=r"(flags) : : "memory", "cc");
+	for (i = 0; i < 22; i++)
+		l[i] = l[i + 5];
+	l[i++] = (unsigned long)func;		// return address
+	l[i++] = 0x20;				// KERNEL CS
+	l[i++] = flags & ~RFLAGS_IF;		// rflags (disable interrupt)
+	l[i++] = (unsigned long)(l + 27);	// ols rsp
+	l[i++] = 0x28;				// KERNEL DS
+}
+
+int arch_cpu_read_write_register(
+		struct mcctrl_os_cpu_register *desc,
+		enum mcctrl_os_cpu_operation op)
+{
+	if (op == MCCTRL_OS_CPU_READ_REGISTER) {
+		desc->val = rdmsr(desc->addr);
+	}
+	else if (op == MCCTRL_OS_CPU_WRITE_REGISTER) {
+		wrmsr(desc->addr, desc->val);
+	}
+	else {
+		return -1;
+	}
+
+	return 0;
+}
+
 /*** end of file ***/

arch/x86/kernel/include/arch-lock.h

@@ -131,6 +131,7 @@ static void __ihk_mc_spinlock_unlock(ihk_spinlock_t *lock, unsigned long flags)
 typedef struct mcs_lock_node {
 	unsigned long locked;
 	struct mcs_lock_node *next;
+	unsigned long irqsave;
 } __attribute__((aligned(64))) mcs_lock_node_t;
 
 static void mcs_lock_init(struct mcs_lock_node *node)
@@ -139,7 +140,7 @@ static void mcs_lock_init(struct mcs_lock_node *node)
 	node->next = NULL;
 }
 
-static void mcs_lock_lock(struct mcs_lock_node *lock,
+static void __mcs_lock_lock(struct mcs_lock_node *lock,
 		struct mcs_lock_node *node)
 {
 	struct mcs_lock_node *pred;
@@ -158,7 +159,7 @@ static void mcs_lock_lock(struct mcs_lock_node *lock,
 	}
 }
 
-static void mcs_lock_unlock(struct mcs_lock_node *lock,
+static void __mcs_lock_unlock(struct mcs_lock_node *lock,
 		struct mcs_lock_node *node)
 {
 	if (node->next == NULL) {
@@ -178,6 +179,35 @@ static void mcs_lock_unlock(struct mcs_lock_node *lock,
 	node->next->locked = 0;
 }
 
+static void mcs_lock_lock_noirq(struct mcs_lock_node *lock,
+		struct mcs_lock_node *node)
+{
+	preempt_disable();
+	__mcs_lock_lock(lock, node);
+}
+
+static void mcs_lock_unlock_noirq(struct mcs_lock_node *lock,
+		struct mcs_lock_node *node)
+{
+	__mcs_lock_unlock(lock, node);
+	preempt_enable();
+}
+
+static void mcs_lock_lock(struct mcs_lock_node *lock,
+		struct mcs_lock_node *node)
+{
+	node->irqsave = cpu_disable_interrupt_save();
+	mcs_lock_lock_noirq(lock, node);
+}
+
+static void mcs_lock_unlock(struct mcs_lock_node *lock,
+		struct mcs_lock_node *node)
+{
+	mcs_lock_unlock_noirq(lock, node);
+	cpu_restore_interrupt(node->irqsave);
+}
+
+
 // reader/writer lock
 typedef struct mcs_rwlock_node {
 	ihk_atomic_t count;	// num of readers (use only common reader)

arch/x86/kernel/include/ihk/atomic.h

@@ -215,4 +215,25 @@ static inline unsigned long atomic_cmpxchg4(unsigned int *addr,
 	return oldval;
 }
 
+static inline void ihk_atomic_add_long(long i, long *v) {
+	asm volatile("lock addq %1,%0"
+					: "+m" (*v)
+					: "ir" (i));
+}
+static inline void ihk_atomic_add_ulong(long i, unsigned long *v) {
+	asm volatile("lock addq %1,%0"
+					: "+m" (*v)
+					: "ir" (i));
+}
+
+static inline unsigned long ihk_atomic_add_long_return(long i, long *v) {
+        long __i;
+
+        __i = i;
+        asm volatile("lock xaddq %0, %1"
+                     : "+r" (i), "+m" (*v)
+                     : : "memory");
+        return i + __i;
+}
+
 #endif

arch/x86/kernel/include/syscall_list.h

@@ -22,7 +22,7 @@
 
 SYSCALL_HANDLED(0, read)
 SYSCALL_DELEGATED(1, write)
-SYSCALL_DELEGATED(2, open)
+SYSCALL_HANDLED(2, open)
 SYSCALL_HANDLED(3, close)
 SYSCALL_DELEGATED(4, stat)
 SYSCALL_DELEGATED(5, fstat)
@@ -150,5 +150,8 @@ SYSCALL_HANDLED(602, pmc_start)
 SYSCALL_HANDLED(603, pmc_stop)
 SYSCALL_HANDLED(604, pmc_reset)
 SYSCALL_HANDLED(700, get_cpu_id)
+#ifdef TRACK_SYSCALLS
+SYSCALL_HANDLED(__NR_track_syscalls, track_syscalls)
+#endif // TRACK_SYSCALLS
 
 /**** End of File ****/

arch/x86/kernel/interrupt.S

@@ -130,11 +130,40 @@ general_protection_exception:
 		addq	$8, %rsp
 		iretq
 
+.global __freeze
+__freeze:
+		PUSH_ALL_REGS
+		callq	freeze
+		POP_ALL_REGS
+		iretq
+
 .globl nmi
 nmi:
 #define PANICED 232
 #define PANIC_REGS 240
 		movq	%rax,%gs:PANIC_REGS+0x00
+		movq	%rsp,%gs:PANIC_REGS+0x08
+
+		movl	nmi_mode(%rip),%eax
+		cmp	$1,%rax
+		je	1f
+		cmp	$2,%rax
+		jne	3f
+1:
+		cld
+		movq	%gs:PANIC_REGS+0x00,%rax
+		PUSH_ALL_REGS
+		subq	$40, %rsp
+		movq	%rsp,%gs:PANIC_REGS+0x10
+		movq	%rsp, %rdi
+		call	freeze_thaw
+		cmpq	$0, %rax
+		jnz	2f
+		addq	$40, %rsp
+2:
+		POP_ALL_REGS
+		iretq
+3:
 		movq	%rbx,%gs:PANIC_REGS+0x08
 		movq	%rcx,%gs:PANIC_REGS+0x10
 		movq	%rdx,%gs:PANIC_REGS+0x18

arch/x86/kernel/memory.c

@@ -45,7 +45,11 @@ void *early_alloc_pages(int nr_pages)
 		last_page = phys_to_virt(virt_to_phys(last_page));
 	} else if (last_page == (void *)-1) {
 		panic("Early allocator is already finalized. Do not use it.\n");
-	}
+	} else {
+        if(virt_to_phys(last_page) >= bootstrap_mem_end) {
+            panic("Early allocator: Out of memory\n");
+        }
+    }
 	p = last_page;
 	last_page += (nr_pages * PAGE_SIZE);
 
@@ -179,7 +183,7 @@ static void init_normal_area(struct page_table *pt)
 	}
 }
 
-static struct page_table *__alloc_new_pt(enum ihk_mc_ap_flag ap_flag)
+static struct page_table *__alloc_new_pt(ihk_mc_ap_flag ap_flag)
 {
 	struct page_table *newpt = ihk_mc_alloc_pages(1, ap_flag);
 
@@ -278,7 +282,7 @@ void set_pte(pte_t *ppte, unsigned long phys, enum ihk_mc_pt_attribute attr)
  *             and returns a pointer to the PTE corresponding to the
  *             virtual address.
  */
-pte_t *get_pte(struct page_table *pt, void *virt, enum ihk_mc_pt_attribute attr, enum ihk_mc_ap_flag ap_flag)
+pte_t *get_pte(struct page_table *pt, void *virt, enum ihk_mc_pt_attribute attr, ihk_mc_ap_flag ap_flag)
 {
 	int l4idx, l3idx, l2idx, l1idx;
 	unsigned long v = (unsigned long)virt;
@@ -339,7 +343,7 @@ static int __set_pt_page(struct page_table *pt, void *virt, unsigned long phys,
 	int l4idx, l3idx, l2idx, l1idx;
 	unsigned long v = (unsigned long)virt;
 	struct page_table *newpt;
-	enum ihk_mc_ap_flag ap_flag;
+	ihk_mc_ap_flag ap_flag;
 	int in_kernel =
 		(((unsigned long long)virt) >= 0xffff000000000000ULL);
 	unsigned long init_pt_lock_flags;
@@ -558,28 +562,34 @@ int ihk_mc_pt_print_pte(struct page_table *pt, void *virt)
 
 	GET_VIRT_INDICES(v, l4idx, l3idx, l2idx, l1idx);
 
+	__kprintf("l4 table: 0x%lX l4idx: %d \n", virt_to_phys(pt), l4idx);
 	if (!(pt->entry[l4idx] & PFL4_PRESENT)) {
 		__kprintf("0x%lX l4idx not present! \n", (unsigned long)virt);
-		__kprintf("l4 entry: 0x%lX\n", pt->entry[l4idx]);
 		return -EFAULT;
 	}
+	__kprintf("l4 entry: 0x%lX\n", pt->entry[l4idx]);
 	pt = phys_to_virt(pt->entry[l4idx] & PAGE_MASK);
 
 	__kprintf("l3 table: 0x%lX l3idx: %d \n", virt_to_phys(pt), l3idx);
 	if (!(pt->entry[l3idx] & PFL3_PRESENT)) {
 		__kprintf("0x%lX l3idx not present! \n", (unsigned long)virt);
-		__kprintf("l3 entry: 0x%lX\n", pt->entry[l3idx]);
 		return -EFAULT;
 	}
+	__kprintf("l3 entry: 0x%lX\n", pt->entry[l3idx]);
+	if ((pt->entry[l3idx] & PFL3_SIZE)) {
+		__kprintf("l3 entry is 1G page\n");
+		return 0;
+	}
 	pt = phys_to_virt(pt->entry[l3idx] & PAGE_MASK);
 	
 	__kprintf("l2 table: 0x%lX l2idx: %d \n", virt_to_phys(pt), l2idx);
 	if (!(pt->entry[l2idx] & PFL2_PRESENT)) {
 		__kprintf("0x%lX l2idx not present! \n", (unsigned long)virt);
-		__kprintf("l2 entry: 0x%lX\n", pt->entry[l2idx]);
 		return -EFAULT;
 	}
+	__kprintf("l2 entry: 0x%lX\n", pt->entry[l2idx]);
 	if ((pt->entry[l2idx] & PFL2_SIZE)) {
+		__kprintf("l2 entry is 2M page\n");
 		return 0;
 	}
 	pt = phys_to_virt(pt->entry[l2idx] & PAGE_MASK);
@@ -658,7 +668,7 @@ int ihk_mc_pt_prepare_map(page_table_t p, void *virt, unsigned long size,
 	return ret;
 }
 
-struct page_table *ihk_mc_pt_create(enum ihk_mc_ap_flag ap_flag)
+struct page_table *ihk_mc_pt_create(ihk_mc_ap_flag ap_flag)
 {
 	struct page_table *pt = ihk_mc_alloc_pages(1, ap_flag);
 
@@ -1091,7 +1101,8 @@ static int clear_range_l1(void *args0, pte_t *ptep, uint64_t base,
 		page = phys_to_page(phys);
 	}
 
-	if (page && page_is_in_memobj(page) && (old & PFL1_DIRTY)) {
+	if (page && page_is_in_memobj(page) && (old & PFL1_DIRTY) && (args->memobj) &&
+			!(args->memobj->flags & MF_ZEROFILL)) {
 		memobj_flush_page(args->memobj, phys, PTL1_SIZE);
 	}
 
@@ -1265,6 +1276,9 @@ static int clear_range(struct page_table *pt, struct process_vm *vm,
 	}
 
 	args.free_physical = free_physical;
+	if (memobj && (memobj->flags & MF_DEV_FILE)) {
+		args.free_physical = 0;
+	}
 	args.memobj = memobj;
 	args.vm = vm;
 
@@ -1773,9 +1787,19 @@ int ihk_mc_pt_set_pte(page_table_t pt, pte_t *ptep, size_t pgsize,
 		*ptep = phys | attr_to_l1attr(attr);
 	}
 	else if (pgsize == PTL2_SIZE) {
+		if (phys & (PTL2_SIZE - 1)) {
+			kprintf("%s: error: phys needs to be PTL2_SIZE aligned\n", __FUNCTION__);
+			error = -1;
+			goto out;
+		}
 		*ptep = phys | attr_to_l2attr(attr | PTATTR_LARGEPAGE);
 	}
 	else if ((pgsize == PTL3_SIZE) && (use_1gb_page)) {
+		if (phys & (PTL3_SIZE - 1)) {
+			kprintf("%s: error: phys needs to be PTL3_SIZE aligned\n", __FUNCTION__);
+			error = -1;
+			goto out;
+		}
 		*ptep = phys | attr_to_l3attr(attr | PTATTR_LARGEPAGE);
 	}
 	else {
@@ -2213,30 +2237,28 @@ int strcpy_from_user(char *dst, const char *src)
 	return err;
 }
 
-long getlong_user(const long *p)
+long getlong_user(long *dest, const long *p)
 {
 	int error;
-	long l;
 
-	error = copy_from_user(&l, p, sizeof(l));
+	error = copy_from_user(dest, p, sizeof(long));
 	if (error) {
 		return error;
 	}
 
-	return l;
+	return 0;
 }
 
-int getint_user(const int *p)
+int getint_user(int *dest, const int *p)
 {
 	int error;
-	int i;
 
-	error = copy_from_user(&i, p, sizeof(i));
+	error = copy_from_user(dest, p, sizeof(int));
 	if (error) {
 		return error;
 	}
 
-	return i;
+	return 0;
 }
 
 int read_process_vm(struct process_vm *vm, void *kdst, const void *usrc, size_t siz)
@@ -2367,8 +2389,18 @@ int write_process_vm(struct process_vm *vm, void *udst, const void *ksrc, size_t
 			return error;
 		}
 
-		va = phys_to_virt(pa);
-		memcpy(va, from, cpsize);
+		if (pa < ihk_mc_get_memory_address(IHK_MC_GMA_MAP_START, 0) ||
+			pa >= ihk_mc_get_memory_address(IHK_MC_GMA_MAP_END, 0)) {
+			dkprintf("%s: pa is outside of LWK memory, from: %p,"
+				"pa: %p, cpsize: %d\n", __FUNCTION__, from, pa, cpsize);
+			va = ihk_mc_map_virtual(pa, 1, PTATTR_ACTIVE);
+			memcpy(va, from, cpsize);
+			ihk_mc_unmap_virtual(va, 1, 1);
+		}
+		else {
+			va = phys_to_virt(pa);
+			memcpy(va, from, cpsize);
+		}
 
 		from += cpsize;
 		to += cpsize;
@@ -2392,7 +2424,7 @@ int patch_process_vm(struct process_vm *vm, void *udst, const void *ksrc, size_t
 	unsigned long pa;
 	void *va;
 
-	kprintf("patch_process_vm(%p,%p,%p,%lx)\n", vm, udst, ksrc, siz);
+	dkprintf("patch_process_vm(%p,%p,%p,%lx)\n", vm, udst, ksrc, siz);
 	if ((ustart < vm->region.user_start)
 			|| (vm->region.user_end <= ustart)
 			|| ((vm->region.user_end - ustart) < siz)) {
@@ -2442,6 +2474,6 @@ int patch_process_vm(struct process_vm *vm, void *udst, const void *ksrc, size_t
 		remain -= cpsize;
 	}
 
-	kprintf("patch_process_vm(%p,%p,%p,%lx):%d\n", vm, udst, ksrc, siz, 0);
+	dkprintf("patch_process_vm(%p,%p,%p,%lx):%d\n", vm, udst, ksrc, siz, 0);
 	return 0;
 } /* patch_process_vm() */

arch/x86/kernel/mikc.c

@@ -30,7 +30,7 @@ int ihk_mc_ikc_init_first_local(struct ihk_ikc_channel_desc *channel,
 
 	memset(channel, 0, sizeof(struct ihk_ikc_channel_desc));
 
-	mikc_queue_pages = ((num_processors * MASTER_IKCQ_PKTSIZE)
+	mikc_queue_pages = ((2 * num_processors * MASTER_IKCQ_PKTSIZE)
 			+ (PAGE_SIZE - 1)) / PAGE_SIZE;
 
 	/* Place both sides in this side */

arch/x86/kernel/syscall.c

@@ -70,71 +70,37 @@ static struct vdso vdso;
 static size_t container_size = 0;
 static ptrdiff_t vdso_offset;
 
-/*
-See dkprintf("BSP HW ID = %d, ", bsp_hw_id); (in ./mcos/kernel/ap.c)
+extern int num_processors;
 
-Core with BSP HW ID 224 is 1st logical core of last physical core.
-It                      boots first and is given SW-ID of 0
+int obtain_clone_cpuid(cpu_set_t *cpu_set) {
+	int min_queue_len = -1;
+	int cpu, min_cpu = -1;
 
-Core with BSP HW ID 0 is 1st logical core of 1st physical core. 
-It                      boots next and is given  SW-ID of 1.
-Core with BSP HW ID 1   boots next and is given  SW-ID of 2.
-Core with BSP HW ID 2   boots next and is given  SW-ID of 3.
-Core with BSP HW ID 3   boots next and is given  SW-ID of 4.
-...
-Core with BSP HW ID 220 is 1st logical core of 56-th physical core.
-It                      boots next and is given  SW-ID of 221.
-Core with BSP HW ID 221 boots next and is given  SW-ID of 222.
-Core with BSP HW ID 222 boots next and is given  SW-ID of 223.
-Core with BSP HW ID 223 boots next and is given  SW-ID of 224.
+	/* Find the first allowed core with the shortest run queue */
+	for (cpu = 0; cpu < num_processors; ++cpu) {
+		struct cpu_local_var *v;
+		unsigned long irqstate;
 
-Core with BSP HW ID 225 is 2nd logical core of last physical core.
-It                      boots next and is given  SW-ID of 225.
-Core with BSP HW ID 226 boots next and is given  SW-ID of 226.
-Core with BSP HW ID 227 boots next and is given  SW-ID of 227.
-*/
-ihk_spinlock_t cpuid_head_lock = 0;
-static int cpuid_head = 0;
+		if (!CPU_ISSET(cpu, cpu_set)) continue;
 
-/* archtecture-depended syscall handlers */
-int obtain_clone_cpuid() {
-    /* see above on BSP HW ID */
-	struct ihk_mc_cpu_info *cpu_info = ihk_mc_get_cpu_info();
-    int cpuid, nretry = 0;
-    ihk_mc_spinlock_lock_noirq(&cpuid_head_lock);
-	
-	/* Always start from 0 to fill in LWK cores linearily */
-	cpuid_head = 0;
- retry:
-    /* Try to obtain next physical core */
-    cpuid = cpuid_head;
+		v = get_cpu_local_var(cpu);
+		irqstate = ihk_mc_spinlock_lock(&v->runq_lock);
+		if (min_queue_len == -1 || v->runq_len < min_queue_len) {
+			min_queue_len = v->runq_len;
+			min_cpu = cpu;
+		}
+		ihk_mc_spinlock_unlock(&v->runq_lock, irqstate);
 
-    /* A hyper-threading core on the same physical core as
-       the parent process might be chosen. Use sched_setaffinity
-       if you want to skip that kind of busy physical core for
-       performance reason. */
-    cpuid_head += 1;
-    if(cpuid_head >= cpu_info->ncpus) {
-        cpuid_head = 0;
-    }
+		if (min_queue_len == 0)
+			break;
+	}
 
-    /* A hyper-threading core whose parent physical core has a
-       process on one of its hyper-threading core might
-       be chosen. Use sched_setaffinity if you want to skip that
-       kind of busy physical core for performance reason. */
-    if(get_cpu_local_var(cpuid)->status != CPU_STATUS_IDLE) {
-        nretry++;
-        if(nretry >= cpu_info->ncpus) {
-            cpuid = -1;
-            ihk_mc_spinlock_unlock_noirq(&cpuid_head_lock);
-            goto out;
-        }
-        goto retry; 
-    }
-	get_cpu_local_var(cpuid)->status = CPU_STATUS_RESERVED;
-    ihk_mc_spinlock_unlock_noirq(&cpuid_head_lock);
- out:
-    return cpuid;
+	if (min_cpu != -1) {
+		if (get_cpu_local_var(min_cpu)->status != CPU_STATUS_RESERVED)
+			get_cpu_local_var(min_cpu)->status = CPU_STATUS_RESERVED;
+	}
+
+    return min_cpu;
 }
 
 int
@@ -544,14 +510,14 @@ void ptrace_report_signal(struct thread *thread, int sig)
 	int parent_pid;
 	struct siginfo info;
 
-	dkprintf("ptrace_report_signal,pid=%d\n", thread->proc->pid);
+	dkprintf("ptrace_report_signal, tid=%d, pid=%d\n", thread->tid, thread->proc->pid);
 
 	mcs_rwlock_writer_lock(&proc->update_lock, &lock);	
 	if(!(proc->ptrace & PT_TRACED)){
 		mcs_rwlock_writer_unlock(&proc->update_lock, &lock);
 		return;
 	}
-	proc->exit_status = sig;
+	thread->exit_status = sig;
 	/* Transition thread state */
 	proc->status = PS_TRACED;
 	thread->status = PS_TRACED;
@@ -569,8 +535,8 @@ void ptrace_report_signal(struct thread *thread, int sig)
 	memset(&info, '\0', sizeof info);
 	info.si_signo = SIGCHLD;
 	info.si_code = CLD_TRAPPED;
-	info._sifields._sigchld.si_pid = thread->proc->pid;
-	info._sifields._sigchld.si_status = thread->proc->exit_status;
+	info._sifields._sigchld.si_pid = thread->tid;
+	info._sifields._sigchld.si_status = thread->exit_status;
 	do_kill(cpu_local_var(current), parent_pid, -1, SIGCHLD, &info, 0);
 	/* Wake parent (if sleeping in wait4()) */
 	waitq_wakeup(&proc->parent->waitpid_q);
@@ -695,10 +661,10 @@ do_signal(unsigned long rc, void *regs0, struct thread *thread, struct sig_pendi
 	int	orgsig;
 	int	ptraceflag = 0;
 	struct mcs_rwlock_node_irqsave lock;
-	unsigned long irqstate;
+	struct mcs_rwlock_node_irqsave mcs_rw_node;
 
 	for(w = pending->sigmask.__val[0], sig = 0; w; sig++, w >>= 1);
-	dkprintf("do_signal,pid=%d,sig=%d\n", proc->pid, sig);
+	dkprintf("do_signal(): tid=%d, pid=%d, sig=%d\n", thread->tid, proc->pid, sig);
 	orgsig = sig;
 
 	if((proc->ptrace & PT_TRACED) &&
@@ -718,12 +684,12 @@ do_signal(unsigned long rc, void *regs0, struct thread *thread, struct sig_pendi
 		rc = regs->gpr.rax;
 	}
 
-	irqstate = ihk_mc_spinlock_lock(&thread->sigcommon->lock);
+	mcs_rwlock_writer_lock(&thread->sigcommon->lock, &mcs_rw_node);
 	k = thread->sigcommon->action + sig - 1;
 
 	if(k->sa.sa_handler == SIG_IGN){
 		kfree(pending);
-		ihk_mc_spinlock_unlock(&thread->sigcommon->lock, irqstate);
+		mcs_rwlock_writer_unlock(&thread->sigcommon->lock, &mcs_rw_node);
 		return;
 	}
 	else if(k->sa.sa_handler){
@@ -808,7 +774,7 @@ do_signal(unsigned long rc, void *regs0, struct thread *thread, struct sig_pendi
 
 		if(copy_to_user(sigsp, &ksigsp, sizeof ksigsp)){
 			kfree(pending);
-			ihk_mc_spinlock_unlock(&thread->sigcommon->lock, irqstate);
+			mcs_rwlock_writer_unlock(&thread->sigcommon->lock, &mcs_rw_node);
 			kprintf("do_signal,write_process_vm failed\n");
 			terminate(0, sig);
 			return;
@@ -827,7 +793,7 @@ do_signal(unsigned long rc, void *regs0, struct thread *thread, struct sig_pendi
 		if(!(k->sa.sa_flags & SA_NODEFER))
 			thread->sigmask.__val[0] |= pending->sigmask.__val[0];
 		kfree(pending);
-		ihk_mc_spinlock_unlock(&thread->sigcommon->lock, irqstate);
+		mcs_rwlock_writer_unlock(&thread->sigcommon->lock, &mcs_rw_node);
 		if(regs->gpr.rflags & RFLAGS_TF){
 			struct siginfo info;
 
@@ -853,7 +819,7 @@ do_signal(unsigned long rc, void *regs0, struct thread *thread, struct sig_pendi
 		}
 		else
 			kfree(pending);
-		ihk_mc_spinlock_unlock(&thread->sigcommon->lock, irqstate);
+		mcs_rwlock_writer_unlock(&thread->sigcommon->lock, &mcs_rw_node);
 		switch (sig) {
 		case SIGSTOP:
 		case SIGTSTP:
@@ -885,7 +851,8 @@ do_signal(unsigned long rc, void *regs0, struct thread *thread, struct sig_pendi
 				/* Wake up the parent who tried wait4 and sleeping */
 				waitq_wakeup(&proc->parent->waitpid_q);
 
-				dkprintf("do_signal,SIGSTOP,sleeping\n");
+				dkprintf("do_signal(): pid: %d, tid: %d SIGSTOP, sleeping\n", 
+					proc->pid, thread->tid);
 				/* Sleep */
 				schedule();
 				dkprintf("SIGSTOP(): woken up\n");
@@ -899,7 +866,7 @@ do_signal(unsigned long rc, void *regs0, struct thread *thread, struct sig_pendi
 
 			/* Update thread state in fork tree */
 			mcs_rwlock_writer_lock(&proc->update_lock, &lock);	
-			proc->exit_status = SIGTRAP;
+			thread->exit_status = SIGTRAP;
 			proc->status = PS_TRACED;
 			thread->status = PS_TRACED;
 			mcs_rwlock_writer_unlock(&proc->update_lock, &lock);	
@@ -953,11 +920,11 @@ do_signal(unsigned long rc, void *regs0, struct thread *thread, struct sig_pendi
 static struct sig_pending *
 getsigpending(struct thread *thread, int delflag){
 	struct list_head *head;
-	ihk_spinlock_t *lock;
+	mcs_rwlock_lock_t *lock;
+	struct mcs_rwlock_node_irqsave mcs_rw_node;
 	struct sig_pending *next;
 	struct sig_pending *pending;
 	__sigset_t w;
-	int	irqstate;
 	__sigset_t x;
 	int sig;
 	struct k_sigaction *k;
@@ -966,8 +933,12 @@ getsigpending(struct thread *thread, int delflag){
 
 	lock = &thread->sigcommon->lock;
 	head = &thread->sigcommon->sigpending;
-	for(;;){
-		irqstate = ihk_mc_spinlock_lock(lock);
+	for(;;) {
+		if (delflag)
+			mcs_rwlock_writer_lock(lock, &mcs_rw_node);
+		else 
+			mcs_rwlock_reader_lock(lock, &mcs_rw_node);
+
 		list_for_each_entry_safe(pending, next, head, list){
 			for(x = pending->sigmask.__val[0], sig = 0; x; sig++, x >>= 1);
 			k = thread->sigcommon->action + sig - 1;
@@ -976,17 +947,26 @@ getsigpending(struct thread *thread, int delflag){
 			   (k->sa.sa_handler != (void *)1 &&
 			    k->sa.sa_handler != NULL)){
 				if(!(pending->sigmask.__val[0] & w)){
-					if(delflag)
+					if(delflag) 
 						list_del(&pending->list);
-					ihk_mc_spinlock_unlock(lock, irqstate);
+
+					if (delflag)
+						mcs_rwlock_writer_unlock(lock, &mcs_rw_node);
+					else 
+						mcs_rwlock_reader_unlock(lock, &mcs_rw_node);
 					return pending;
 				}
 			}
 		}
-		ihk_mc_spinlock_unlock(lock, irqstate);
+
+		if (delflag)
+			mcs_rwlock_writer_unlock(lock, &mcs_rw_node);
+		else 
+			mcs_rwlock_reader_unlock(lock, &mcs_rw_node);
 
 		if(lock == &thread->sigpendinglock)
 			return NULL;
+
 		lock = &thread->sigpendinglock;
 		head = &thread->sigpending;
 	}
@@ -1034,22 +1014,25 @@ check_signal(unsigned long rc, void *regs0, int num)
 			}
 		}
 		ihk_mc_spinlock_unlock(&(cpu_local_var(runq_lock)), irqstate);
-		return;
+		goto out;
 	}
 
 	if(regs != NULL && !interrupt_from_user(regs)) {
-		return;
+		goto out;
 	}
 
 	for(;;){
 		pending = getsigpending(thread, 1);
 		if(!pending) {
 			dkprintf("check_signal,queue is empty\n");
-			return;
+			goto out;
 		}
 
 		do_signal(rc, regs, thread, pending, num);
 	}
+
+out:
+	return;
 }
 
 unsigned long
@@ -1063,7 +1046,8 @@ do_kill(struct thread *thread, int pid, int tid, int sig, siginfo_t *info,
 	struct thread *tthread = NULL;
 	int i;
 	__sigset_t mask;
-	ihk_spinlock_t *savelock = NULL;
+	mcs_rwlock_lock_t *savelock = NULL;
+	struct mcs_rwlock_node mcs_rw_node;
 	struct list_head *head = NULL;
 	int rc;
 	unsigned long irqstate = 0;
@@ -1195,7 +1179,8 @@ done:
 			if(pid != -1 && tthread->proc->pid != pid){
 				continue;
 			}
-			if(tthread->tid == tid){
+			if (tthread->tid == tid &&
+			    tthread->status != PS_EXITED) {
 				found = 1;
 				break;
 			}
@@ -1247,7 +1232,7 @@ done:
 
 	doint = 0;
 
-	ihk_mc_spinlock_lock_noirq(savelock);
+	mcs_rwlock_writer_lock_noirq(savelock, &mcs_rw_node);
 
 	/* Put signal event even when handler is SIG_IGN or SIG_DFL
 	   because target ptraced thread must call ptrace_report_signal 
@@ -1286,7 +1271,7 @@ done:
 			}
 		}
 	}
-	ihk_mc_spinlock_unlock_noirq(savelock);
+	mcs_rwlock_writer_unlock_noirq(savelock, &mcs_rw_node);
 	cpu_restore_interrupt(irqstate);
 
 	if (doint && !(mask & tthread->sigmask.__val[0])) {
@@ -1313,6 +1298,9 @@ done:
 				sched_wakeup_thread(tthread, PS_STOPPED);
 				tthread->proc->status = PS_RUNNING;
 			}
+			else {
+				sched_wakeup_thread(tthread, PS_INTERRUPTIBLE);
+			}
 		}
 	}
 	release_thread(tthread);
@@ -1757,7 +1745,8 @@ int arch_map_vdso(struct process_vm *vm)
 	vrflags = VR_REMOTE;
 	vrflags |= VR_PROT_READ | VR_PROT_EXEC;
 	vrflags |= VRFLAG_PROT_TO_MAXPROT(vrflags);
-	error = add_process_memory_range(vm, (intptr_t)s, (intptr_t)e, NOPHYS, vrflags, NULL, 0, PAGE_SHIFT);
+	error = add_process_memory_range(vm, (intptr_t)s, (intptr_t)e,
+			NOPHYS, vrflags, NULL, 0, PAGE_SHIFT, NULL);
 	if (error) {
 		ekprintf("ERROR: adding memory range for vdso. %d\n", error);
 		goto out;
@@ -1788,7 +1777,8 @@ int arch_map_vdso(struct process_vm *vm)
 		vrflags = VR_REMOTE;
 		vrflags |= VR_PROT_READ;
 		vrflags |= VRFLAG_PROT_TO_MAXPROT(vrflags);
-		error = add_process_memory_range(vm, (intptr_t)s, (intptr_t)e, NOPHYS, vrflags, NULL, 0, PAGE_SHIFT);
+		error = add_process_memory_range(vm, (intptr_t)s, (intptr_t)e,
+				NOPHYS, vrflags, NULL, 0, PAGE_SHIFT, NULL);
 		if (error) {
 			ekprintf("ERROR: adding memory range for vvar. %d\n", error);
 			goto out;

arch/x86/kernel/vsyscall.c

@@ -17,6 +17,7 @@
  * make sure that these are position-independent codes.
  */
 
+#include <cls.h>
 #include <syscall.h>
 #include <ihk/atomic.h>
 #include <arch/cpu.h>

arch/x86/tools/eclair-dump-backtrace.exp.in

@@ -0,0 +1,67 @@
+#!/usr/bin/expect
+
+set INST_DIR "@prefix@"
+
+spawn $INST_DIR/bin/eclair -d /tmp/mckernel.dump -k $INST_DIR/smp-x86/kernel/mckernel.img -i
+
+set state "init"
+set thread_id 0
+
+expect {
+	"in ?? ()" {
+		switch -- $state {
+			"thread_chosen" {
+				set state "thread_skip"
+			}
+			"thread_bt" {
+				set state "thread_skip"
+			}
+		}
+
+		exp_continue
+	}
+	"(eclair) " {
+		switch -- $state {
+			"init" {
+				set state "threads_list"
+				send "info threads\r"
+			}
+			"threads_list" {
+				incr thread_id
+				set state "thread_chosen"
+				send "thread $thread_id\r"
+			}
+			"thread_skip" {
+				incr thread_id
+				set state "thread_chosen"
+				send "thread $thread_id\r"
+			}
+			"thread_chosen" {
+				set state "thread_bt"
+				send "bt\r"
+			}
+		}
+
+		exp_continue
+	}
+	"Type <return> to continue, or q <return> to quit" { 
+		switch -- $state {
+			"threads_list" {
+				send "\r"
+			}
+			"thread_bt" {
+				send "\r"
+			}
+			"thread_skip" {
+				send "q\r"
+			}
+		}
+		exp_continue 
+	}
+	" not known." { 
+		expect "(eclair) " { send "quit\r" }
+		expect "Quit anyway? (y or n) " { send "y\r" }
+		exit 0
+	}
+}
+

arch/x86/tools/irqbalance_mck.service.in

@@ -3,7 +3,7 @@ Description=irqbalance daemon
 After=syslog.target
 
 [Service]
-EnvironmentFile=@ETCDIR@/irqbalance_mck
+EnvironmentFile=/tmp/irqbalance_mck
 ExecStart=/usr/sbin/irqbalance --foreground $IRQBALANCE_ARGS
 
 [Install]

arch/x86/tools/mcreboot-smp-x86.sh.in

@@ -39,7 +39,9 @@ else
 	irqbalance_used="no"
 fi
 
-while getopts :i:k:c:m:o:f: OPT
+turbo=""
+
+while getopts :ti:k:c:m:o:f: OPT
 do
 	case ${OPT} in
 	f)	facility=${OPTARG}
@@ -76,6 +78,8 @@ do
 		;;
 	m) mem=${OPTARG}
 		;;
+	t) turbo="turbo"
+		;;
 	*)  echo "invalid option -${OPT}" >&2
 		exit 1
 	esac
@@ -100,7 +104,7 @@ error_exit() {
 		;&
 	mcoverlayfs_loaded)
 		if [ "$enable_mcoverlay" == "yes" ]; then
-			rmmod mcoverlay
+			rmmod mcoverlay 2>/dev/null
 		fi
 		;&
 	linux_proc_bind_mounted)
@@ -130,15 +134,7 @@ error_exit() {
 		fi
 		;&
 	mcctrl_loaded)
-		rmmod mcctrl || echo "warning: failed to remove mcctrl" >&2
-		;&
-	mem_reserved)
-		mem=`${SBINDIR}/ihkconfig 0 query mem`
-		if [ "${mem}" != "" ]; then
-			if ! ${SBINDIR}/ihkconfig 0 release mem $mem > /dev/null; then
-				echo "warning: failed to release memory" >&2
-			fi
-		fi
+		rmmod mcctrl 2>/dev/null || echo "warning: failed to remove mcctrl" >&2
 		;&
 	cpus_reserved)
 		cpus=`${SBINDIR}/ihkconfig 0 query cpu`
@@ -148,11 +144,19 @@ error_exit() {
 			fi
 		fi
 		;&
+	mem_reserved)
+		mem=`${SBINDIR}/ihkconfig 0 query mem`
+		if [ "${mem}" != "" ]; then
+			if ! ${SBINDIR}/ihkconfig 0 release mem $mem > /dev/null; then
+				echo "warning: failed to release memory" >&2
+			fi
+		fi
+		;&
 	ihk_smp_loaded)
-		rmmod ihk_smp_x86 || echo "warning: failed to remove ihk_smp_x86" >&2
+		rmmod ihk_smp_x86 2>/dev/null || echo "warning: failed to remove ihk_smp_x86" >&2
 		;&
 	ihk_loaded)
-		rmmod ihk || echo "warning: failed to remove ihk" >&2
+		rmmod ihk 2>/dev/null || echo "warning: failed to remove ihk" >&2
 		;&
 	irqbalance_stopped)
 		if [ "`systemctl status irqbalance_mck.service 2> /dev/null |grep -E 'Active: active'`" != "" ]; then
@@ -170,6 +174,11 @@ error_exit() {
 			fi
 		fi
 		;&
+	aslr_disabled)
+		if [ -f /tmp/mckernel_randomize_va_space ]; then
+			cat /tmp/mckernel_randomize_va_space > /proc/sys/kernel/randomize_va_space
+		fi
+		;&
 	initial)
 		# Nothing more to revert
 		;;
@@ -221,17 +230,23 @@ if [ "$cpus" == "" ]; then
 	fi
 fi
 
+# Disable address space layout randomization
+if [ -f /proc/sys/kernel/randomize_va_space ] && [ "`cat /proc/sys/kernel/randomize_va_space`" != "0" ]; then
+	cat /proc/sys/kernel/randomize_va_space > /tmp/mckernel_randomize_va_space
+	echo "0" > /proc/sys/kernel/randomize_va_space
+fi
+
 # Remove mcoverlay if loaded
 if [ "$enable_mcoverlay" == "yes" ]; then
-	if [ "`lsmod | grep mcoverlay`" != "" ]; then
+	if grep mcoverlay /proc/modules &>/dev/null; then
 		if [ "`cat /proc/mounts | grep /tmp/mcos/mcos0_sys`" != "" ]; then umount -l /tmp/mcos/mcos0_sys; fi
 		if [ "`cat /proc/mounts | grep /tmp/mcos/mcos0_proc`" != "" ]; then umount -l /tmp/mcos/mcos0_proc; fi
 		if [ "`cat /proc/mounts | grep /tmp/mcos/linux_proc`" != "" ]; then umount -l /tmp/mcos/linux_proc; fi
 		if [ "`cat /proc/mounts | grep /tmp/mcos`" != "" ]; then umount -l /tmp/mcos; fi
 		if [ -e /tmp/mcos ]; then rm -rf /tmp/mcos; fi
-		if ! rmmod mcoverlay; then
+		if ! rmmod mcoverlay 2>/dev/null; then
 			echo "error: removing mcoverlay" >&2
-			error_exit "initial"
+			error_exit "aslr_disabled"
 		fi
 	fi
 fi
@@ -241,36 +256,31 @@ if [ "${irqbalance_used}" == "yes" ]; then
     systemctl stop irqbalance_mck.service 2>/dev/null
     if ! systemctl stop irqbalance.service 2>/dev/null ; then
 		echo "error: stopping irqbalance" >&2
-		error_exit "initial"
+		error_exit "aslr_disabled"
 	fi;
 fi
 
-# Start mcklogd. Note that McKernel blocks when kmsg buffer is full
-# with '-k 1' until mcklogd unblocks it so starting mcklogd must preceed
-# booting McKernel
-if [ ${LOGMODE} -ne 0 ]; then
-	# Stop mcklogd which has survived McKernel shutdown because
-	# mcstop+release.sh is not used
-	pkill mcklogd
-	SBINDIR=${SBINDIR} ${SBINDIR}/mcklogd -i ${INTERVAL} -f ${facility}
-fi
-
 # Load IHK if not loaded
-if [ "`lsmod | grep ihk`" == "" ]; then
-	if ! insmod ${KMODDIR}/ihk.ko; then
+if ! grep -E 'ihk\s' /proc/modules &>/dev/null; then
+	if ! insmod ${KMODDIR}/ihk.ko 2>/dev/null; then
 		echo "error: loading ihk" >&2
 		error_exit "irqbalance_stopped"
 	fi
 fi
 
+# Increase swappiness so that we have better chance to allocate memory for IHK
+echo 100 > /proc/sys/vm/swappiness
+
 # Drop Linux caches to free memory
 sync && echo 3 > /proc/sys/vm/drop_caches
 
 # Merge free memory areas into large, physically contigous ones
 echo 1 > /proc/sys/vm/compact_memory 2>/dev/null
 
+sync
+
 # Load IHK-SMP if not loaded and reserve CPUs and memory
-if [ "`lsmod | grep ihk_smp_x86`" == "" ]; then
+if ! grep ihk_smp_x86 /proc/modules &>/dev/null; then
 	ihk_irq=""
 	for i in `seq 64 255`; do
 		if [ ! -d /proc/irq/$i ] && [ "`cat /proc/interrupts | grep ":" | awk '{print $1}' | grep -o '[0-9]*' | grep -e '^$i$'`" == "" ]; then
@@ -282,25 +292,38 @@ if [ "`lsmod | grep ihk_smp_x86`" == "" ]; then
 		echo "error: no IRQ available" >&2
 		error_exit "ihk_loaded"
 	fi
-	if ! insmod ${KMODDIR}/ihk-smp-x86.ko ihk_start_irq=$ihk_irq ihk_ikc_irq_core=$ihk_ikc_irq_core; then
+	if ! insmod ${KMODDIR}/ihk-smp-x86.ko ihk_start_irq=$ihk_irq ihk_ikc_irq_core=$ihk_ikc_irq_core 2>/dev/null; then
 		echo "error: loading ihk-smp-x86" >&2
 		error_exit "ihk_loaded"
 	fi
-	if ! ${SBINDIR}/ihkconfig 0 reserve cpu ${cpus}; then
-		echo "error: reserving CPUs" >&2;
-		error_exit "ihk_smp_loaded"
+
+	# Free MCDRAM (special case for OFP SNC-4 mode)
+	if [ "`hostname | grep "c[0-9][0-9][0-9][0-9].ofp"`" != "" ] && [ "`cat /sys/devices/system/node/online`" == "0-7" ]; then
+		for i in 4 5 6 7; do
+			find /sys/devices/system/node/node$i/memory*/ -name "online" | while read f; do
+				echo 0 > $f 2>&1 > /dev/null;
+			done
+			find /sys/devices/system/node/node$i/memory*/ -name "online" | while read f; do
+				echo 1 > $f 2>&1 > /dev/null;
+			done
+		done
 	fi
+
 	if ! ${SBINDIR}/ihkconfig 0 reserve mem ${mem}; then
 		echo "error: reserving memory" >&2
-		error_exit "cpus_reserved"
+		error_exit "ihk_smp_loaded"
+	fi
+	if ! ${SBINDIR}/ihkconfig 0 reserve cpu ${cpus}; then
+		echo "error: reserving CPUs" >&2;
+		error_exit "mem_reserved"
 	fi
 fi
 
 # Load mcctrl if not loaded
-if [ "`lsmod | grep mcctrl`" == "" ]; then
-	if ! insmod ${KMODDIR}/mcctrl.ko; then
+if ! grep mcctrl /proc/modules &>/dev/null; then
+	if ! insmod ${KMODDIR}/mcctrl.ko 2>/dev/null; then
 		echo "error: inserting mcctrl.ko" >&2
-		error_exit "mem_reserved"
+		error_exit "cpus_reserved"
 	fi
 fi
 
@@ -340,7 +363,7 @@ if ! ${SBINDIR}/ihkosctl 0 load ${KERNDIR}/mckernel.img; then
 fi
 
 # Set kernel arguments
-if ! ${SBINDIR}/ihkosctl 0 kargs "hidos ksyslogd=${LOGMODE}"; then
+if ! ${SBINDIR}/ihkosctl 0 kargs "hidos ksyslogd=${LOGMODE} $turbo"; then
 	echo "error: setting kernel arguments" >&2
 	error_exit "os_created"
 fi
@@ -358,27 +381,37 @@ fi
 
 # Overlay /proc, /sys with McKernel specific contents
 if [ "$enable_mcoverlay" == "yes" ]; then
-	if [ ! -e /tmp/mcos ]; then mkdir -p /tmp/mcos; fi
+	if [ ! -e /tmp/mcos ]; then 
+		mkdir -p /tmp/mcos; 
+	fi
 	if ! mount -t tmpfs tmpfs /tmp/mcos; then
 		echo "error: mount /tmp/mcos" >&2
 		error_exit "tmp_mcos_created"
 	fi
-	if [ ! -e /tmp/mcos/linux_proc ]; then mkdir -p /tmp/mcos/linux_proc; fi
+	if [ ! -e /tmp/mcos/linux_proc ]; then 
+		mkdir -p /tmp/mcos/linux_proc; 
+	fi
 	if ! mount --bind /proc /tmp/mcos/linux_proc; then
 		echo "error: mount /tmp/mcos/linux_proc" >&2
 		error_exit "tmp_mcos_mounted"
 	fi
-	if ! insmod ${KMODDIR}/mcoverlay.ko; then
+	if ! insmod ${KMODDIR}/mcoverlay.ko 2>/dev/null; then
 		echo "error: inserting mcoverlay.ko" >&2
 		error_exit "linux_proc_bind_mounted"
 	fi
 	while [ ! -e /proc/mcos0 ]
 	do
-		sleep 1
+		sleep 0.1
 	done
-	if [ ! -e /tmp/mcos/mcos0_proc ]; then mkdir -p /tmp/mcos/mcos0_proc; fi
-	if [ ! -e /tmp/mcos/mcos0_proc_upper ]; then mkdir -p /tmp/mcos/mcos0_proc_upper; fi
-	if [ ! -e /tmp/mcos/mcos0_proc_work ]; then mkdir -p /tmp/mcos/mcos0_proc_work; fi
+	if [ ! -e /tmp/mcos/mcos0_proc ]; then 
+		mkdir -p /tmp/mcos/mcos0_proc; 
+	fi
+	if [ ! -e /tmp/mcos/mcos0_proc_upper ]; then 
+		mkdir -p /tmp/mcos/mcos0_proc_upper; 
+	fi
+	if [ ! -e /tmp/mcos/mcos0_proc_work ]; then 
+		mkdir -p /tmp/mcos/mcos0_proc_work; 
+	fi
 	if ! mount -t mcoverlay mcoverlay -o lowerdir=/proc/mcos0:/proc,upperdir=/tmp/mcos/mcos0_proc_upper,workdir=/tmp/mcos/mcos0_proc_work,nocopyupw,nofscheck /tmp/mcos/mcos0_proc; then
 		echo "error: mounting /tmp/mcos/mcos0_proc" >&2
 		error_exit "mcoverlayfs_loaded"
@@ -390,9 +423,15 @@ if [ "$enable_mcoverlay" == "yes" ]; then
 	do
 		sleep 0.1
 	done
-	if [ ! -e /tmp/mcos/mcos0_sys ]; then mkdir -p /tmp/mcos/mcos0_sys; fi
-	if [ ! -e /tmp/mcos/mcos0_sys_upper ]; then mkdir -p /tmp/mcos/mcos0_sys_upper; fi
-	if [ ! -e /tmp/mcos/mcos0_sys_work ]; then mkdir -p /tmp/mcos/mcos0_sys_work; fi
+	if [ ! -e /tmp/mcos/mcos0_sys ]; then 
+		mkdir -p /tmp/mcos/mcos0_sys; 
+	fi
+	if [ ! -e /tmp/mcos/mcos0_sys_upper ]; then 
+		mkdir -p /tmp/mcos/mcos0_sys_upper; 
+	fi
+	if [ ! -e /tmp/mcos/mcos0_sys_work ]; then 
+		mkdir -p /tmp/mcos/mcos0_sys_work; 
+	fi
 	if ! mount -t mcoverlay mcoverlay -o lowerdir=/sys/devices/virtual/mcos/mcos0/sys:/sys,upperdir=/tmp/mcos/mcos0_sys_upper,workdir=/tmp/mcos/mcos0_sys_work,nocopyupw,nofscheck /tmp/mcos/mcos0_sys; then
 		echo "error: mount /tmp/mcos/mcos0_sys" >&2
 		error_exit "mcos_proc_mounted"
@@ -400,6 +439,8 @@ if [ "$enable_mcoverlay" == "yes" ]; then
 	# TODO: How de we revert this in case of failure??
 	mount --make-rprivate /sys
 
+	touch /tmp/mcos/mcos0_proc/mckernel
+
 	rm -rf /tmp/mcos/mcos0_sys/setup_complete
 
 	# Hide NUMA related files which are outside the LWK partition
@@ -456,8 +497,9 @@ if [ "${irqbalance_used}" == "yes" ]; then
 
     banirq=`cat /proc/interrupts| perl -e 'while(<>) { if(/^\s*(\d+).*IHK\-SMP\s*$/) {print $1;}}'`
 
-    sed "s/%mask%/$smp_affinity_mask/g" $ETCDIR/irqbalance_mck.in | sed "s/%banirq%/$banirq/g" > $ETCDIR/irqbalance_mck
-    if ! systemctl link $ETCDIR/irqbalance_mck.service >/dev/null 2>/dev/null; then
+    sed "s/%mask%/$smp_affinity_mask/g" $ETCDIR/irqbalance_mck.in | sed "s/%banirq%/$banirq/g" > /tmp/irqbalance_mck
+	systemctl disable irqbalance_mck.service >/dev/null 2>/dev/null
+	if ! systemctl link $ETCDIR/irqbalance_mck.service >/dev/null 2>/dev/null; then
 		echo "error: linking irqbalance_mck" >&2
 		error_exit "mcos_sys_mounted"
 	fi
@@ -469,3 +511,13 @@ if [ "${irqbalance_used}" == "yes" ]; then
 #    echo cpus=$cpus mask=$smp_affinity_mask banirq=$banirq
 fi
 
+# Start mcklogd. Note that McKernel blocks when kmsg buffer is full
+# with '-k 1' until mcklogd unblocks it so starting mcklogd must preceed
+# booting McKernel
+if [ ${LOGMODE} -ne 0 ]; then
+	# Stop mcklogd which has survived McKernel shutdown because
+	# mcstop+release.sh is not used
+	pkill mcklogd
+	SBINDIR=${SBINDIR} ${SBINDIR}/mcklogd -i ${INTERVAL} -f ${facility}
+fi
+

arch/x86/tools/mcstop+release-smp-x86.sh.in

@@ -18,7 +18,13 @@ mem=""
 cpus=""
 
 # No SMP module? Exit.
-if [ "`lsmod | grep ihk_smp_x86`" == "" ]; then exit 0; fi
+if ! grep ihk_smp_x86 /proc/modules &>/dev/null; then exit 0; fi
+
+# Stop mcklogd
+while pgrep "mcklogd" > /dev/null 2>&1;
+do
+	pkill -9 mcklogd
+done
 
 # Destroy all LWK instances
 if ls /dev/mcos* 1>/dev/null 2>&1; then
@@ -59,44 +65,41 @@ if [ "${mem}" != "" ]; then
 fi
 
 # Remove delegator if loaded
-if [ "`lsmod | grep mcctrl`" != "" ]; then
-	if ! rmmod mcctrl; then
+if grep mcctrl /proc/modules &>/dev/null; then
+	if ! rmmod mcctrl 2>/dev/null; then
 		echo "error: removing mcctrl" >&2
 		exit 1
 	fi
 fi
 
 # Remove mcoverlay if loaded
-if [ "`lsmod | grep mcoverlay`" != "" ]; then
+if grep mcoverlay /proc/modules &>/dev/null; then
 	if [ "`cat /proc/mounts | grep /tmp/mcos/mcos0_sys`" != "" ]; then umount -l /tmp/mcos/mcos0_sys; fi
 	if [ "`cat /proc/mounts | grep /tmp/mcos/mcos0_proc`" != "" ]; then umount -l /tmp/mcos/mcos0_proc; fi
 	if [ "`cat /proc/mounts | grep /tmp/mcos/linux_proc`" != "" ]; then umount -l /tmp/mcos/linux_proc; fi
 	if [ "`cat /proc/mounts | grep /tmp/mcos`" != "" ]; then umount -l /tmp/mcos; fi
 	if [ -e /tmp/mcos ]; then rm -rf /tmp/mcos; fi
-	if ! rmmod mcoverlay; then
+	if ! rmmod mcoverlay 2>/dev/null; then
 		echo "warning: failed to remove mcoverlay" >&2
 	fi
 fi
 
 # Remove SMP module
-if [ "`lsmod | grep ihk_smp_x86`" != "" ]; then
-	if ! rmmod ihk_smp_x86; then
+if grep ihk_smp_x86 /proc/modules &>/dev/null; then
+	if ! rmmod ihk_smp_x86 2>/dev/null; then
 		echo "error: removing ihk_smp_x86" >&2
 		exit 1
 	fi
 fi
 
 # Remove core module
-if [ "`lsmod | grep -E 'ihk\s' | awk '{print $1}'`" != "" ]; then
-	if ! rmmod ihk; then
+if grep -E 'ihk\s' /proc/modules &>/dev/null; then
+	if ! rmmod ihk 2>/dev/null; then
 		echo "error: removing ihk" >&2
 		exit 1
 	fi
 fi
 
-# Stop mcklogd
-pkill mcklogd
-
 # Start irqbalance with the original settings
 if [ "`systemctl status irqbalance_mck.service 2> /dev/null |grep -E 'Active: active'`" != "" ]; then
 	if ! systemctl stop irqbalance_mck.service 2>/dev/null; then
@@ -113,3 +116,10 @@ if [ "`systemctl status irqbalance_mck.service 2> /dev/null |grep -E 'Active: ac
 	fi
 fi
 
+# Re-enable ASLR
+if [ -f /tmp/mckernel_randomize_va_space ]; then
+	cat /tmp/mckernel_randomize_va_space > /proc/sys/kernel/randomize_va_space
+fi
+
+# Set back default swappiness
+echo 60 > /proc/sys/vm/swappiness

configure.ac

@@ -17,6 +17,13 @@ DCFA_RELEASE_DATE=DCFA_RELEASE_DATE_m4
 
 AC_PREFIX_DEFAULT([/opt/ppos])
 
+AC_CHECK_HEADER([numa.h],[numa_header_found=yes])
+AS_IF([test "x$numa_header_found" != "xyes"],
+	[AC_MSG_ERROR([Unable to find numa.h header file, missing numactl-devel?])])
+AC_CHECK_LIB([numa],[numa_run_on_node],[numa_lib_found=yes])
+AS_IF([test "x$numa_lib_found" != "xyes"],
+	[AC_MSG_ERROR([Unable to find NUMA library, missing numactl-devel?])])
+
 AC_ARG_WITH([kernelsrc],
   AC_HELP_STRING(
     [--with-kernelsrc=path],[Path to 'kernel src', default is /lib/modules/uname_r/build]),
@@ -48,6 +55,17 @@ AC_ARG_ENABLE([mcoverlayfs],
 	      [ENABLE_MCOVERLAYFS=$enableval],
 	      [ENABLE_MCOVERLAYFS=yes])
 
+AC_ARG_WITH([uname_r],
+  AC_HELP_STRING(
+    [--with-uname_r=uname_r],[Value of '`uname -r`' on the target platform, default is local value]),
+  [WITH_UNAME_R=$withval],[WITH_UNAME_R=yes])
+
+case "X$WITH_UNAME_R" in
+    Xyes | Xno | X)
+	WITH_UNAME_R='`uname -r`'
+	;;
+esac
+
 case "X$WITH_KERNELSRC" in
     Xyes | Xno | X)
 	WITH_KERNELSRC='/lib/modules/`uname -r`/build'
@@ -70,6 +88,7 @@ case $WITH_TARGET in
 	ARCH=`uname -m`
 	AC_PROG_CC
 	XCC=$CC
+	CFLAGS="$CFLAGS -ffreestanding -fno-tree-loop-distribute-patterns"
 	;;
     builtin-mic)
 	ARCH=k1om
@@ -162,6 +181,7 @@ case $WITH_TARGET in
 esac
 
 KDIR="$WITH_KERNELSRC"
+UNAME_R="$WITH_UNAME_R"
 TARGET="$WITH_TARGET"
 
 MCCTRL_LINUX_SYMTAB=""
@@ -279,6 +299,7 @@ AC_SUBST(CC)
 AC_SUBST(XCC)
 AC_SUBST(ARCH)
 AC_SUBST(KDIR)
+AC_SUBST(UNAME_R)
 AC_SUBST(TARGET)
 AC_SUBST(BINDIR)
 AC_SUBST(SBINDIR)
@@ -286,6 +307,7 @@ AC_SUBST(ETCDIR)
 AC_SUBST(KMODDIR)
 AC_SUBST(KERNDIR)
 AC_SUBST(MANDIR)
+AC_SUBST(CFLAGS)
 AC_SUBST(ENABLE_MCOVERLAYFS)
 
 AC_SUBST(IHK_VERSION)
@@ -313,6 +335,7 @@ AC_CONFIG_FILES([
 	arch/x86/tools/mcreboot-builtin-x86.sh
 	arch/x86/tools/mcreboot-smp-x86.sh
 	arch/x86/tools/mcstop+release-smp-x86.sh
+	arch/x86/tools/eclair-dump-backtrace.exp
 	arch/x86/tools/mcshutdown-builtin-x86.sh
 	arch/x86/tools/mcreboot.1:arch/x86/tools/mcreboot.1in
 	arch/x86/tools/irqbalance_mck.service

executer/include/uprotocol.h

@@ -42,6 +42,7 @@
 #define MCEXEC_UP_GET_CRED	 0x30a0290a
 #define MCEXEC_UP_GET_CREDV	 0x30a0290b
 #define MCEXEC_UP_GET_NODES  0x30a0290c
+#define MCEXEC_UP_GET_CPUSET  0x30a0290d
 
 #define MCEXEC_UP_PREPARE_DMA    0x30a02910
 #define MCEXEC_UP_FREE_DMA       0x30a02911
@@ -79,6 +80,18 @@ struct program_image_section {
 #define SHELL_PATH_MAX_LEN	1024
 #define MCK_RLIM_MAX	20
 
+struct get_cpu_set_arg {
+	int nr_processes;
+	void *cpu_set;
+	size_t cpu_set_size;	// Size in bytes
+	int *target_core;
+	int *mcexec_linux_numa; // NUMA domain to bind mcexec to
+};
+
+#define PLD_CPU_SET_MAX_CPUS 1024
+typedef unsigned long __cpu_set_unit;
+#define PLD_CPU_SET_SIZE (PLD_CPU_SET_MAX_CPUS / (8 * sizeof(__cpu_set_unit)))
+
 struct program_load_desc {
 	int num_sections;
 	int status;
@@ -108,6 +121,7 @@ struct program_load_desc {
 	struct rlimit rlimit[MCK_RLIM_MAX];
 	unsigned long interp_align;
 	char shell_path[SHELL_PATH_MAX_LEN];
+	__cpu_set_unit cpu_set[PLD_CPU_SET_SIZE];
 	struct program_image_section sections[0];
 };
 
@@ -206,4 +220,34 @@ struct sys_unshare_desc {
 	unsigned long unshare_flags;
 };
 
+enum perf_ctrl_type {
+	PERF_CTRL_SET,
+	PERF_CTRL_GET,
+	PERF_CTRL_ENABLE,
+	PERF_CTRL_DISABLE,
+};
+
+struct perf_ctrl_desc {
+	enum perf_ctrl_type ctrl_type;
+	int status;
+	union {
+		/* for SET, GET */
+		struct {
+			unsigned int target_cntr;
+			unsigned long config;
+			unsigned long read_value;
+			unsigned disabled        :1,
+			         pinned          :1,
+			         exclude_user    :1,
+			         exclude_kernel  :1,
+			         exclude_hv      :1,
+			         exclude_idle    :1;
+		};
+
+		/* for START, STOP*/
+		struct {
+			unsigned long target_cntr_mask;
+		};
+	};
+};
 #endif

executer/kernel/mcctrl/arch/x86_64/archdeps.c

@@ -64,6 +64,8 @@ reserve_user_space(struct mcctrl_usrdata *usrdata, unsigned long *startp, unsign
 	unsigned long start = 0L;
 	unsigned long end;
 
+	mutex_lock(&usrdata->reserve_lock);
+
 #define	DESIRED_USER_END	0x800000000000
 #define	GAP_FOR_MCEXEC		0x008000000000UL
 	end = DESIRED_USER_END;
@@ -81,6 +83,8 @@ reserve_user_space(struct mcctrl_usrdata *usrdata, unsigned long *startp, unsign
 	up_write(&current->mm->mmap_sem);
 #endif
 
+	mutex_unlock(&usrdata->reserve_lock);
+
 	if (IS_ERR_VALUE(start)) {
 		return start;
 	}

executer/kernel/mcctrl/driver.c

@@ -27,6 +27,7 @@
 #include <linux/slab.h>
 #include <linux/device.h>
 #include "mcctrl.h"
+#include <ihk/ihk_host_user.h>
 
 #define OS_MAX_MINOR 64
 
@@ -61,6 +62,7 @@ static struct ihk_os_user_call_handler mcctrl_uchs[] = {
 	{ .request = MCEXEC_UP_SEND_SIGNAL, .func = mcctrl_ioctl },
 	{ .request = MCEXEC_UP_GET_CPU, .func = mcctrl_ioctl },
 	{ .request = MCEXEC_UP_GET_NODES, .func = mcctrl_ioctl },
+	{ .request = MCEXEC_UP_GET_CPUSET, .func = mcctrl_ioctl },
 	{ .request = MCEXEC_UP_STRNCPY_FROM_USER, .func = mcctrl_ioctl },
 	{ .request = MCEXEC_UP_NEW_PROCESS, .func = mcctrl_ioctl },
 	{ .request = MCEXEC_UP_PREPARE_DMA, .func = mcctrl_ioctl },
@@ -73,6 +75,12 @@ static struct ihk_os_user_call_handler mcctrl_uchs[] = {
 	{ .request = MCEXEC_UP_SYS_UMOUNT, .func = mcctrl_ioctl },
 	{ .request = MCEXEC_UP_SYS_UNSHARE, .func = mcctrl_ioctl },
 	{ .request = MCEXEC_UP_DEBUG_LOG, .func = mcctrl_ioctl },
+	{ .request = IHK_OS_AUX_PERF_NUM, .func = mcctrl_ioctl },
+	{ .request = IHK_OS_AUX_PERF_SET, .func = mcctrl_ioctl },
+	{ .request = IHK_OS_AUX_PERF_GET, .func = mcctrl_ioctl },
+	{ .request = IHK_OS_AUX_PERF_ENABLE, .func = mcctrl_ioctl },
+	{ .request = IHK_OS_AUX_PERF_DISABLE, .func = mcctrl_ioctl },
+	{ .request = IHK_OS_AUX_PERF_DESTROY, .func = mcctrl_ioctl },
 };
 
 static struct ihk_os_user_call mcctrl_uc_proto = {

executer/kernel/mcctrl/ikc.c

@@ -53,6 +53,9 @@ void mcexec_prepare_ack(ihk_os_t os, unsigned long arg, int err);
 static void mcctrl_ikc_init(ihk_os_t os, int cpu, unsigned long rphys, struct ihk_ikc_channel_desc *c);
 int mcexec_syscall(struct mcctrl_usrdata *ud, struct ikc_scd_packet *packet);
 void sig_done(unsigned long arg, int err);
+void mcctrl_perf_ack(ihk_os_t os, struct ikc_scd_packet *packet);
+void mcctrl_os_read_write_cpu_response(ihk_os_t os,
+		struct ikc_scd_packet *pisp);
 
 /* XXX: this runs in atomic context! */
 static int syscall_packet_handler(struct ihk_ikc_channel_desc *c,
@@ -80,7 +83,7 @@ static int syscall_packet_handler(struct ihk_ikc_channel_desc *c,
 		break;
 
 	case SCD_MSG_PROCFS_ANSWER:
-		procfs_answer(pisp->arg, pisp->err);
+		procfs_answer(usrdata, pisp->pid);
 		break;
 
 	case SCD_MSG_SEND_SIGNAL:
@@ -109,6 +112,14 @@ static int syscall_packet_handler(struct ihk_ikc_channel_desc *c,
 		get_vdso_info(__os, pisp->arg);
 		break;
 
+	case SCD_MSG_PERF_ACK:
+		mcctrl_perf_ack(__os, pisp);
+		break;
+
+	case SCD_MSG_CPU_RW_REG_RESP:
+		mcctrl_os_read_write_cpu_response(__os, pisp);
+		break;
+
 	default:
 		printk(KERN_ERR "mcctrl:syscall_packet_handler:"
 				"unknown message (%d.%d.%d.%d.%d.%#lx)\n",
@@ -240,7 +251,7 @@ static struct ihk_ikc_listen_param listen_param = {
 	.port = 501,
 	.handler = connect_handler,
 	.pkt_size = sizeof(struct ikc_scd_packet),
-	.queue_size = PAGE_SIZE,
+	.queue_size = PAGE_SIZE * 4,
 	.magic = 0x1129,
 };
 
@@ -248,7 +259,7 @@ static struct ihk_ikc_listen_param listen_param2 = {
 	.port = 502,
 	.handler = connect_handler2,
 	.pkt_size = sizeof(struct ikc_scd_packet),
-	.queue_size = PAGE_SIZE,
+	.queue_size = PAGE_SIZE * 4,
 	.magic = 0x1329,
 };
 
@@ -283,12 +294,13 @@ int prepare_ikc_channels(ihk_os_t os)
 	}
 
 	usrdata->os = os;
-	init_waitqueue_head(&usrdata->wq_prepare);
 	ihk_host_os_set_usrdata(os, usrdata);
 	memcpy(&usrdata->listen_param, &listen_param, sizeof listen_param);
 	ihk_ikc_listen_port(os, &usrdata->listen_param);
 	memcpy(&usrdata->listen_param2, &listen_param2, sizeof listen_param2);
 	ihk_ikc_listen_port(os, &usrdata->listen_param2);
+	init_waitqueue_head(&usrdata->wq_procfs);
+	mutex_init(&usrdata->reserve_lock);
 
 	for (i = 0; i < MCCTRL_PER_PROC_DATA_HASH_SIZE; ++i) {
 		INIT_LIST_HEAD(&usrdata->per_proc_data_hash[i]);
@@ -298,6 +310,9 @@ int prepare_ikc_channels(ihk_os_t os)
 	INIT_LIST_HEAD(&usrdata->cpu_topology_list);
 	INIT_LIST_HEAD(&usrdata->node_topology_list);
 
+	mutex_init(&usrdata->part_exec.lock);
+	usrdata->part_exec.nr_processes = -1;
+
 	return 0;
 }
 
@@ -323,7 +338,6 @@ void destroy_ikc_channels(ihk_os_t os)
 //			ihk_ikc_disconnect(usrdata->channels[i].c);
 			ihk_ikc_free_channel(usrdata->channels[i].c);
 			__destroy_ikc_channel(os, usrdata->channels + i);
-			printk("Channel #%d freed.\n", i);
 		}
 	}
 

executer/kernel/mcctrl/mcctrl.h

@@ -44,6 +44,7 @@
 #include <linux/rwlock.h>
 #include <linux/threads.h>
 #include "sysfs.h"
+#include "mcctrl_public.h"
 
 #define SCD_MSG_PREPARE_PROCESS         0x1
 #define SCD_MSG_PREPARE_PROCESS_ACKED   0x2
@@ -92,6 +93,12 @@
 #define SCD_MSG_PROCFS_TID_CREATE	0x44
 #define SCD_MSG_PROCFS_TID_DELETE	0x45
 
+#define SCD_MSG_PERF_CTRL               0x50
+#define SCD_MSG_PERF_ACK                0x51
+
+#define SCD_MSG_CPU_RW_REG              0x52
+#define SCD_MSG_CPU_RW_REG_RESP         0x53
+
 #define DMA_PIN_SHIFT                   21
 
 #define DO_USER_MODE
@@ -103,6 +110,12 @@ struct coretable {
 	unsigned long addr;
 };
 
+enum mcctrl_os_cpu_operation {
+	MCCTRL_OS_CPU_READ_REGISTER,
+	MCCTRL_OS_CPU_WRITE_REGISTER,
+	MCCTRL_OS_CPU_MAX_OP
+};
+
 struct ikc_scd_packet {
 	int msg;
 	int err;
@@ -128,6 +141,13 @@ struct ikc_scd_packet {
 		struct {
 			int ttid;
 		};
+
+		/* SCD_MSG_CPU_RW_REG */
+		struct {
+			struct mcctrl_os_cpu_register desc;
+			enum mcctrl_os_cpu_operation op;
+			void *resp;
+		};
 	};
 	char padding[12];
 };
@@ -187,17 +207,24 @@ struct mcctrl_per_thread_data {
 #define MCCTRL_PER_THREAD_DATA_HASH_MASK (MCCTRL_PER_THREAD_DATA_HASH_SIZE - 1) 
 
 struct mcctrl_per_proc_data {
+	struct mcctrl_usrdata *ud;
 	struct list_head hash;
 	int pid;
 	unsigned long rpgtable;	/* per process, not per OS */
 
-	struct list_head wq_list;
-	struct list_head wq_req_list;
-	struct list_head wq_list_exact;
+	struct list_head wq_list; /* All these requests come from mcexec */
+	struct list_head wq_req_list; /* These requests come from IKC IRQ handler (can be processed by any threads) */
+	struct list_head wq_list_exact;  /* These requests come from IKC IRQ handler targeting a particular thread */
+
 	ihk_spinlock_t wq_list_lock;
+	wait_queue_head_t wq_prepare;
+	wait_queue_head_t wq_procfs;
 
 	struct list_head per_thread_data_hash[MCCTRL_PER_THREAD_DATA_HASH_SIZE];
 	rwlock_t per_thread_data_hash_lock[MCCTRL_PER_THREAD_DATA_HASH_SIZE];
+	cpumask_t cpu_set;
+	int ikc_target_cpu;
+	atomic_t refcount;
 };
 
 struct sysfsm_req {
@@ -254,6 +281,13 @@ struct node_topology {
 	struct list_head chain;
 };
 
+struct mcctrl_part_exec {
+	struct mutex lock;	
+	int nr_processes;
+	int nr_processes_left;
+	cpumask_t cpus_used;
+};
+
 #define CPU_LONGS (((NR_CPUS) + (BITS_PER_LONG) - 1) / (BITS_PER_LONG))
 
 #define MCCTRL_PER_PROC_DATA_HASH_SHIFT 7
@@ -270,9 +304,9 @@ struct mcctrl_usrdata {
 	int	base_cpu;
 	int	job_pos;
 	int	mcctrl_dma_abort;
+	struct mutex reserve_lock;
 	unsigned long	last_thread_exec;
-	wait_queue_head_t	wq_prepare;
-	
+	wait_queue_head_t wq_procfs;
 	struct list_head per_proc_data_hash[MCCTRL_PER_PROC_DATA_HASH_SIZE];
 	rwlock_t per_proc_data_hash_lock[MCCTRL_PER_PROC_DATA_HASH_SIZE];
 
@@ -284,6 +318,8 @@ struct mcctrl_usrdata {
 	nodemask_t numa_online;
 	struct list_head cpu_topology_list;
 	struct list_head node_topology_list;
+	struct mcctrl_part_exec part_exec;
+	int perf_event_num;
 };
 
 struct mcctrl_signal {
@@ -305,8 +341,9 @@ int __do_in_kernel_syscall(ihk_os_t os, struct ikc_scd_packet *packet);
 int mcctrl_add_per_proc_data(struct mcctrl_usrdata *ud, int pid, 
 	struct mcctrl_per_proc_data *ppd);
 int mcctrl_delete_per_proc_data(struct mcctrl_usrdata *ud, int pid);
-inline struct mcctrl_per_proc_data *mcctrl_get_per_proc_data(
+struct mcctrl_per_proc_data *mcctrl_get_per_proc_data(
 		struct mcctrl_usrdata *ud, int pid);
+void mcctrl_put_per_proc_data(struct mcctrl_per_proc_data *ppd);
 
 int mcctrl_add_per_thread_data(struct mcctrl_per_proc_data* ppd,
 	struct task_struct *task, void *data);
@@ -338,7 +375,7 @@ struct procfs_file {
 	char fname[PROCFS_NAME_MAX];	/* procfs filename (request) */
 };
 
-void procfs_answer(unsigned int arg, int err);
+void procfs_answer(struct mcctrl_usrdata *ud, int pid);
 int procfsm_packet_handler(void *os, int msg, int pid, unsigned long arg);
 void add_tid_entry(int osnum, int pid, int tid);
 void add_pid_entry(int osnum, int pid);
@@ -386,4 +423,14 @@ struct get_cpu_mapping_req {
 	wait_queue_head_t wq;
 };
 
+struct ihk_perf_event_attr{
+	unsigned long config; 
+	unsigned disabled:1;
+	unsigned pinned:1;
+	unsigned exclude_user:1;
+	unsigned exclude_kernel:1;
+	unsigned exclude_hv:1;
+	unsigned exclude_idle:1;
+};
+
 #endif

executer/kernel/mcctrl/mcctrl_public.h

@@ -0,0 +1,20 @@
+#ifndef __MCCTRL_PUBLIC_H
+#define __MCCTRL_PUBLIC_H
+
+#include <ihk/ihk_host_user.h>
+#include <ikc/queue.h>
+
+struct mcctrl_os_cpu_register {
+	unsigned long addr;
+	unsigned long val;
+	unsigned long addr_ext;
+};
+
+int mcctrl_os_read_cpu_register(ihk_os_t os, int cpu,
+		struct mcctrl_os_cpu_register *desc);
+int mcctrl_os_write_cpu_register(ihk_os_t os, int cpu,
+		struct mcctrl_os_cpu_register *desc);
+int mcctrl_get_request_os_cpu(ihk_os_t *os, int *cpu);
+
+
+#endif // __MCCTRL_PUBLIC_H

executer/kernel/mcctrl/procfs.c

@@ -59,7 +59,6 @@ static const struct procfs_entry base_entry_stuff[];
 static const struct file_operations mckernel_forward_ro;
 static const struct file_operations mckernel_forward;
 
-static DECLARE_WAIT_QUEUE_HEAD(procfsq);
 static ssize_t mckernel_procfs_read(struct file *file, char __user *buf, 
 		size_t nbytes, loff_t *ppos);
 
@@ -106,14 +105,28 @@ getpath(struct procfs_list_entry *e, char *buf, int bufsize)
 /**
  * \brief Process SCD_MSG_PROCFS_ANSWER message.
  *
- * \param arg sent argument
- * \param err error info (redundant)
+ * \param ud mcctrl_usrdata pointer
+ * \param pid PID of the requesting process
  */
-void
-procfs_answer(unsigned int arg, int err)
+void procfs_answer(struct mcctrl_usrdata *ud, int pid)
 {
-	dprintk("procfs: received SCD_MSG_PROCFS_ANSWER message(err = %d).\n", err);
-	wake_up_interruptible(&procfsq);
+	struct mcctrl_per_proc_data *ppd = NULL;
+
+	if (pid > 0) {
+		ppd = mcctrl_get_per_proc_data(ud, pid);
+
+		if (unlikely(!ppd)) {
+			kprintf("%s: ERROR: no per-process structure for PID %d\n",
+					__FUNCTION__, pid);
+			return;
+		}
+	}
+
+	wake_up_all(pid > 0 ? &ppd->wq_procfs : &ud->wq_procfs);
+
+	if (pid > 0) {
+		mcctrl_put_per_proc_data(ppd);
+	}
 }
 
 static struct procfs_list_entry *
@@ -248,9 +261,11 @@ get_pid_cred(int pid)
 {
 	struct task_struct *task = NULL;
 
-	if(pid > 0){
+	if (pid > 0) {
+		rcu_read_lock();
 		task = pid_task(find_vpid(pid), PIDTYPE_PID);
-		if(task){
+		rcu_read_unlock();
+		if (task) {
 			return __task_cred(task);
 		}
 	}
@@ -493,36 +508,84 @@ procfs_exit(int osnum)
  * This function conforms to the 2) way of fs/proc/generic.c
  * from linux-2.6.39.4.
  */
-static ssize_t
-mckernel_procfs_read(struct file *file, char __user *buf, size_t nbytes,
-	       loff_t *ppos)
+static ssize_t __mckernel_procfs_read_write(
+		struct file *file,
+		char __user *buf, size_t nbytes,
+		loff_t *ppos, int read_write)
 {
 	struct inode * inode = file->f_inode;
 	char *kern_buffer = NULL;
 	int order = 0;
 	volatile struct procfs_read *r = NULL;
 	struct ikc_scd_packet isp;
-	int ret;
+	int ret, osnum, pid, retw;
 	unsigned long pbuf;
 	unsigned long count = nbytes;
 #if LINUX_VERSION_CODE < KERNEL_VERSION(3,10,0)
 	struct proc_dir_entry *dp = PDE(inode);
 	struct procfs_list_entry *e = dp->data;
-#else	
+#else
 	struct procfs_list_entry *e = PDE_DATA(inode);
-#endif	
+#endif
 	loff_t offset = *ppos;
 	char pathbuf[PROCFS_NAME_MAX];
-	char *path;
+	char *path, *p;
+	ihk_os_t os = NULL;
+	struct mcctrl_usrdata *udp = NULL;
+	struct mcctrl_per_proc_data *ppd = NULL;
 
-	path = getpath(e, pathbuf, 256);
-	dprintk("mckernel_procfs_read: invoked for %s, offset: %lu, count: %d\n", 
-			path, offset, count); 
-	
 	if (count <= 0 || offset < 0) {
 		return 0;
 	}
-	
+
+	path = getpath(e, pathbuf, PROCFS_NAME_MAX);
+	dprintk("%s: invoked for %s, offset: %lu, count: %lu\n",
+			__FUNCTION__, path,
+			(unsigned long)offset, count);
+
+	/* Verify OS number */
+	ret = sscanf(path, "mcos%d/", &osnum);
+	if (ret != 1) {
+		printk("%s: error: couldn't determine OS number\n", __FUNCTION__);
+		return -EINVAL;
+	}
+
+	if (osnum != e->osnum) {
+		printk("%s: error: OS numbers don't match\n", __FUNCTION__);
+		return -EINVAL;
+	}
+
+	/* Is this request for a specific process? */
+	p = strchr(path, '/') + 1;
+	ret = sscanf(p, "%d/", &pid);
+	if (ret != 1) {
+		pid = -1;
+	}
+
+	os = osnum_to_os(osnum);
+	if (!os) {
+		printk("%s: error: no IHK OS data found for OS %d\n",
+				__FUNCTION__, osnum);
+		return -EINVAL;
+	}
+
+	udp = ihk_host_os_get_usrdata(os);
+	if (!udp) {
+		printk("%s: error: no MCCTRL data found for OS %d\n",
+				__FUNCTION__, osnum);
+		return -EINVAL;
+	}
+
+	if (pid > 0) {
+		ppd = mcctrl_get_per_proc_data(udp, pid);
+
+		if (unlikely(!ppd)) {
+			printk("%s: error: no per-process structure for PID %d",
+					__FUNCTION__, pid);
+			return -EINVAL;
+		}
+	}
+
 	while ((1 << order) < count) ++order;
 	if (order > 12) {
 		order -= 12;
@@ -534,10 +597,11 @@ mckernel_procfs_read(struct file *file, char __user *buf, size_t nbytes,
 	/* NOTE: we need physically contigous memory to pass through IKC */
 	kern_buffer = (char *)__get_free_pages(GFP_KERNEL, order);
 	if (!kern_buffer) {
-		printk("mckernel_procfs_read(): ERROR: allocating kernel buffer\n");
-		return -ENOMEM;
+		printk("%s: ERROR: allocating kernel buffer\n", __FUNCTION__);
+		ret = -ENOMEM;
+		goto out;
 	}
-	
+
 	pbuf = virt_to_phys(kern_buffer);
 
 	r = kmalloc(sizeof(struct procfs_read), GFP_KERNEL);
@@ -551,152 +615,96 @@ mckernel_procfs_read(struct file *file, char __user *buf, size_t nbytes,
 	r->status = 0;
 	r->offset = offset;
 	r->count = count;
-	r->readwrite = 0;
+	r->readwrite = read_write;
 	strncpy((char *)r->fname, path, PROCFS_NAME_MAX);
 	isp.msg = SCD_MSG_PROCFS_REQUEST;
 	isp.ref = 0;
 	isp.arg = virt_to_phys(r);
-	
-	ret = mcctrl_ikc_send(osnum_to_os(e->osnum), 0, &isp);
-	
+	isp.pid = pid;
+
+	ret = mcctrl_ikc_send(osnum_to_os(e->osnum),
+			(pid > 0) ? ppd->ikc_target_cpu : 0, &isp);
+
 	if (ret < 0) {
 		goto out; /* error */
 	}
-	
+
 	/* Wait for a reply. */
 	ret = -EIO; /* default exit code */
-	dprintk("now wait for a relpy\n");
-	
-	/* Wait for the status field of the procfs_read structure set ready. */
-	if (wait_event_interruptible_timeout(procfsq, r->status != 0, HZ) == 0) {
-		kprintf("ERROR: mckernel_procfs_read: timeout (1 sec).\n");
+	dprintk("%s: waiting for reply\n", __FUNCTION__);
+
+retry_wait:
+	/* Wait for the status field of the procfs_read structure,
+	 * wait on per-process or OS specific data depending on
+	 * who the request is for.
+	 */
+	if (pid > 0) {
+		retw = wait_event_interruptible_timeout(ppd->wq_procfs,
+				r->status != 0, HZ);
+	}
+	else {
+		retw = wait_event_interruptible_timeout(udp->wq_procfs,
+				r->status != 0, HZ);
+	}
+
+	/* Timeout? */
+	if (retw == 0 && r->status == 0) {
+		printk("%s: error: timeout (1 sec)\n", __FUNCTION__);
 		goto out;
 	}
-	
-	/* Wake up and check the result. */
-	dprintk("mckernel_procfs_read: woke up. ret: %d, eof: %d\n", r->ret, r->eof);
-	
-	if (r->ret > 0) {
-		if (copy_to_user(buf, kern_buffer, r->ret)) {
-			kprintf("ERROR: mckernel_procfs_read: copy_to_user failed.\n");
-			ret = -EFAULT;
-			goto out;
-		}
+	/* Interrupted? */
+	else if (retw == -ERESTARTSYS) {
+		ret = -ERESTART;
+		goto out;
+	}
+	/* Were we woken up by a reply to another procfs request? */
+	else if (r->status == 0) {
+		/* TODO: r->status is not set atomically, we could be woken
+		 * up with status == 0 and it could change to 1 while in this
+		 * code, we could potentially miss the wake_up()... 
+		 */
+		printk("%s: stale wake-up, retrying\n", __FUNCTION__);
+		goto retry_wait;
+	}
 
+	/* Wake up and check the result. */
+	dprintk("%s: woke up. ret: %d, eof: %d\n",
+			__FUNCTION__, r->ret, r->eof);
+
+	if (r->ret > 0) {
+		if (read_write == 0) {
+			if (copy_to_user(buf, kern_buffer, r->ret)) {
+				printk("%s: ERROR: copy_to_user failed.\n", __FUNCTION__);
+				ret = -EFAULT;
+				goto out;
+			}
+		}
 		*ppos += r->ret;
 	}
 	ret = r->ret;
 
 out:
-	if(kern_buffer)
+	if (ppd)
+		mcctrl_put_per_proc_data(ppd);
+	if (kern_buffer)
 		free_pages((uintptr_t)kern_buffer, order);
-	if(r)
+	if (r)
 		kfree((void *)r);
-	
+
 	return ret;
 }
 
-static ssize_t
-mckernel_procfs_write(struct file *file, const char __user *buf, size_t nbytes,
-	       loff_t *ppos)
+static ssize_t mckernel_procfs_read(struct file *file,
+		char __user *buf, size_t nbytes, loff_t *ppos)
 {
-	struct inode * inode = file->f_inode;
-	char *kern_buffer = NULL;
-	int order = 0;
-	volatile struct procfs_read *r = NULL;
-	struct ikc_scd_packet isp;
-	int ret;
-	unsigned long pbuf;
-	unsigned long count = nbytes;
-#if LINUX_VERSION_CODE < KERNEL_VERSION(3,10,0)
-	struct proc_dir_entry *dp = PDE(inode);
-	struct procfs_list_entry *e = dp->data;
-#else	
-	struct procfs_list_entry *e = PDE_DATA(inode);
-#endif	
-	loff_t offset = *ppos;
-	char pathbuf[PROCFS_NAME_MAX];
-	char *path;
+	return __mckernel_procfs_read_write(file, buf, nbytes, ppos, 0);
+}
 
-	path = getpath(e, pathbuf, 256);
-	dprintk("mckernel_procfs_read: invoked for %s, offset: %lu, count: %d\n", 
-			path, offset, count); 
-	
-	if (count <= 0 || offset < 0) {
-		return 0;
-	}
-	
-	while ((1 << order) < count) ++order;
-	if (order > 12) {
-		order -= 12;
-	}
-	else {
-		order = 1;
-	}
-
-	/* NOTE: we need physically contigous memory to pass through IKC */
-	kern_buffer = (char *)__get_free_pages(GFP_KERNEL, order);
-	if (!kern_buffer) {
-		printk("mckernel_procfs_read(): ERROR: allocating kernel buffer\n");
-		return -ENOMEM;
-	}
-	if (copy_from_user(kern_buffer, buf, nbytes)) {
-		ret = -EFAULT;
-		goto out;
-	}
-	
-	pbuf = virt_to_phys(kern_buffer);
-
-	r = kmalloc(sizeof(struct procfs_read), GFP_KERNEL);
-	if (r == NULL) {
-		ret = -ENOMEM;
-		goto out;
-	}
-	dprintk("offset: %lx, count: %d, cpu: %d\n", offset, count, e->cpu);
-
-	r->pbuf = pbuf;
-	r->eof = 0;
-	r->ret = -EIO; /* default */
-	r->status = 0;
-	r->offset = offset;
-	r->count = count;
-	r->readwrite = 1;
-	strncpy((char *)r->fname, path, PROCFS_NAME_MAX);
-	isp.msg = SCD_MSG_PROCFS_REQUEST;
-	isp.ref = 0;
-	isp.arg = virt_to_phys(r);
-	
-	ret = mcctrl_ikc_send(osnum_to_os(e->osnum), 0, &isp);
-	
-	if (ret < 0) {
-		goto out; /* error */
-	}
-	
-	/* Wait for a reply. */
-	ret = -EIO; /* default exit code */
-	dprintk("now wait for a relpy\n");
-	
-	/* Wait for the status field of the procfs_read structure set ready. */
-	if (wait_event_interruptible_timeout(procfsq, r->status != 0, HZ) == 0) {
-		kprintf("ERROR: mckernel_procfs_read: timeout (1 sec).\n");
-		goto out;
-	}
-	
-	/* Wake up and check the result. */
-	dprintk("mckernel_procfs_read: woke up. ret: %d, eof: %d\n", r->ret, r->eof);
-	
-	if (r->ret > 0) {
-		*ppos += r->ret;
-	}
-	ret = r->ret;
-
-out:
-	if(kern_buffer)
-		free_pages((uintptr_t)kern_buffer, order);
-	if(r)
-		kfree((void *)r);
-	
-	return ret;
+static ssize_t mckernel_procfs_write(struct file *file,
+		const char __user *buf, size_t nbytes, loff_t *ppos)
+{
+	return __mckernel_procfs_read_write(file,
+			(char __user *)buf, nbytes, ppos, 1);
 }
 
 static loff_t

executer/kernel/mcctrl/syscall.c

@@ -306,7 +306,7 @@ static int remote_page_fault(struct mcctrl_usrdata *usrdata, void *fault_addr, u
 		error = -ENOENT;
 		printk("%s: no packet registered for TID %d\n",
 				__FUNCTION__, task_pid_vnr(current));
-		goto out_no_unmap;
+		goto out_put_ppd;
 	}
 
 	req = &packet->req;
@@ -326,6 +326,9 @@ retry_alloc:
 
 	/* Prepare per-thread wait queue head */
 	wqhln->task = current;
+	/* Save the TID explicitly, because mcexec_syscall(), where the request
+	 * will be matched, is in IRQ context and can't call task_pid_vnr() */
+	wqhln->rtid = task_pid_vnr(current);
 	wqhln->req = 0;
 	init_waitqueue_head(&wqhln->wq_syscall);
 
@@ -434,9 +437,11 @@ out:
 	ihk_device_unmap_virtual(ihk_os_to_dev(usrdata->os), resp, sizeof(*resp));
 	ihk_device_unmap_memory(ihk_os_to_dev(usrdata->os), phys, sizeof(*resp));
 
-out_no_unmap:
+out_put_ppd:
 	dprintk("%s: tid: %d, fault_addr: %lu, reason: %lu, error: %d\n",
 		__FUNCTION__, task_pid_vnr(current), fault_addr, reason, error);
+
+	mcctrl_put_per_proc_data(ppd);
 	return error;
 }
 
@@ -574,6 +579,7 @@ static int rus_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 	size_t			pix;
 #endif
 	struct mcctrl_per_proc_data *ppd;
+	int ret = 0;
 
 	dprintk("mcctrl:page fault:flags %#x pgoff %#lx va %p page %p\n",
 			vmf->flags, vmf->pgoff, vmf->virtual_address, vmf->page);
@@ -584,7 +590,6 @@ static int rus_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 		ppd = mcctrl_get_per_proc_data(usrdata, vma->vm_mm->owner->pid);
 	}
 
-
 	if (!ppd) {
 		kprintf("%s: ERROR: no per-process structure for PID %d??\n", 
 				__FUNCTION__, task_tgid_vnr(current));
@@ -618,7 +623,8 @@ static int rus_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 	if (error) {
 		printk("mcctrl:page fault error:flags %#x pgoff %#lx va %p page %p\n",
 				vmf->flags, vmf->pgoff, vmf->virtual_address, vmf->page);
-		return VM_FAULT_SIGBUS;
+		ret = VM_FAULT_SIGBUS;
+		goto put_and_out;
 	}
 
 	rva = (unsigned long)vmf->virtual_address & ~(pgsize - 1);
@@ -655,10 +661,15 @@ static int rus_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 	if (error) {
 		printk("mcctrl:page fault:remap error:flags %#x pgoff %#lx va %p page %p\n",
 				vmf->flags, vmf->pgoff, vmf->virtual_address, vmf->page);
-		return VM_FAULT_SIGBUS;
+		ret = VM_FAULT_SIGBUS;
+		goto put_and_out;
 	}
 
-	return VM_FAULT_NOPAGE;
+	ret = VM_FAULT_NOPAGE;
+
+put_and_out:
+	mcctrl_put_per_proc_data(ppd);
+	return ret;
 }
 
 static struct vm_operations_struct rus_vmops = {
@@ -746,6 +757,20 @@ static struct list_head pager_list = LIST_HEAD_INIT(pager_list);
 struct pager_create_result {
 	uintptr_t	handle;
 	int		maxprot;
+	uint32_t flags;
+	size_t size;
+};
+
+enum {
+	/* for memobj.flags */
+	MF_HAS_PAGER	= 0x0001,
+	MF_SHMDT_OK	= 0x0002,
+	MF_IS_REMOVABLE	= 0x0004,
+	MF_PREFETCH = 0x0008,
+	MF_ZEROFILL = 0x0010,
+	MF_REG_FILE = 0x1000,
+	MF_DEV_FILE = 0x2000,
+	MF_END
 };
 
 static int pager_req_create(ihk_os_t os, int fd, uintptr_t result_pa)
@@ -760,6 +785,7 @@ static int pager_req_create(ihk_os_t os, int fd, uintptr_t result_pa)
 	struct pager *newpager = NULL;
 	uintptr_t phys;
 	struct kstat st;
+	int mf_flags = 0;
 
 	dprintk("pager_req_create(%d,%lx)\n", fd, (long)result_pa);
 
@@ -827,6 +853,32 @@ static int pager_req_create(ihk_os_t os, int fd, uintptr_t result_pa)
 			list_add(&newpager->list, &pager_list);
 			pager = newpager;
 			newpager = NULL;
+
+			/* Intel MPI library and shared memory "prefetch" */
+			{
+				char *pathbuf, *fullpath;
+
+				pathbuf = kmalloc(PATH_MAX, GFP_TEMPORARY);
+				if (pathbuf) {
+					fullpath = d_path(&file->f_path, pathbuf, PATH_MAX);
+					if (!IS_ERR(fullpath)) {
+						if (!strncmp("/dev/shm/Intel_MPI", fullpath, 18)) {
+							//mf_flags = (MF_PREFETCH | MF_ZEROFILL);
+							mf_flags = (MF_ZEROFILL);
+							dprintk("%s: filename: %s, zerofill\n",
+									__FUNCTION__, fullpath);
+						}
+						else if (strstr(fullpath, "libmpi") != NULL) {
+							mf_flags = MF_PREFETCH;
+							dprintk("%s: filename: %s, prefetch\n",
+									__FUNCTION__, fullpath);
+						}
+					}
+
+					kfree(pathbuf);
+				}
+			}
+
 			break;
 		}
 
@@ -856,6 +908,8 @@ found:
 	resp = ihk_device_map_virtual(dev, phys, sizeof(*resp), NULL, 0);
 	resp->handle = (uintptr_t)pager;
 	resp->maxprot = maxprot;
+	resp->flags = mf_flags;
+	resp->size = st.size;
 	ihk_device_unmap_virtual(dev, resp, sizeof(*resp));
 	ihk_device_unmap_memory(dev, phys, sizeof(*resp));
 
@@ -1582,6 +1636,7 @@ int __do_in_kernel_syscall(ihk_os_t os, struct ikc_scd_packet *packet)
 
 			dprintk("%s: pid: %d, rpgtable: 0x%lx updated\n",
 				__FUNCTION__, ppd->pid, ppd->rpgtable);
+			mcctrl_put_per_proc_data(ppd);
 		}
 
 		ret = clear_pte_range(sc->args[0], sc->args[1]);

executer/kernel/mcctrl/sysfs_files.c

@@ -197,19 +197,19 @@ void free_topology_info(ihk_os_t os)
 /*
  * CPU and NUMA node mapping conversion functions.
  */
-static int mckernel_cpu_2_linux_cpu(struct mcctrl_usrdata *udp, int cpu_id)
+int mckernel_cpu_2_linux_cpu(struct mcctrl_usrdata *udp, int cpu_id)
 {
 	return (cpu_id < udp->cpu_info->n_cpus) ?
 		udp->cpu_info->mapping[cpu_id] : -1;
 }
 
-static int mckernel_cpu_2_hw_id(struct mcctrl_usrdata *udp, int cpu_id)
+int mckernel_cpu_2_hw_id(struct mcctrl_usrdata *udp, int cpu_id)
 {
 	return (cpu_id < udp->cpu_info->n_cpus) ?
 		udp->cpu_info->hw_ids[cpu_id] : -1;
 }
 
-static int linux_cpu_2_mckernel_cpu(struct mcctrl_usrdata *udp, int cpu_id)
+int linux_cpu_2_mckernel_cpu(struct mcctrl_usrdata *udp, int cpu_id)
 {
 	int i;
 
@@ -222,7 +222,7 @@ static int linux_cpu_2_mckernel_cpu(struct mcctrl_usrdata *udp, int cpu_id)
 }
 
 #if 0
-static int hw_id_2_mckernel_cpu(struct mcctrl_usrdata *udp, int hw_id)
+int hw_id_2_mckernel_cpu(struct mcctrl_usrdata *udp, int hw_id)
 {
 	int i;
 
@@ -235,7 +235,7 @@ static int hw_id_2_mckernel_cpu(struct mcctrl_usrdata *udp, int hw_id)
 	return -1;
 }
 
-static int hw_id_2_linux_cpu(struct mcctrl_usrdata *udp, int hw_id)
+int hw_id_2_linux_cpu(struct mcctrl_usrdata *udp, int hw_id)
 {
 	int i;
 
@@ -248,7 +248,7 @@ static int hw_id_2_linux_cpu(struct mcctrl_usrdata *udp, int hw_id)
 	return -1;
 }
 
-static int linux_cpu_2_hw_id(struct mcctrl_usrdata *udp, int cpu)
+int linux_cpu_2_hw_id(struct mcctrl_usrdata *udp, int cpu)
 {
 	int mckernel_cpu = linux_cpu_2_mckernel_cpu(udp, cpu);
 
@@ -257,13 +257,13 @@ static int linux_cpu_2_hw_id(struct mcctrl_usrdata *udp, int cpu)
 }
 #endif
 
-static int mckernel_numa_2_linux_numa(struct mcctrl_usrdata *udp, int numa_id)
+int mckernel_numa_2_linux_numa(struct mcctrl_usrdata *udp, int numa_id)
 {
 	return (numa_id < udp->mem_info->n_numa_nodes) ?
 		udp->mem_info->numa_mapping[numa_id] : -1;
 }
 
-static int linux_numa_2_mckernel_numa(struct mcctrl_usrdata *udp, int numa_id)
+int linux_numa_2_mckernel_numa(struct mcctrl_usrdata *udp, int numa_id)
 {
 	int i;
 

executer/kernel/mcoverlayfs/Makefile.in

@@ -1,6 +1,6 @@
 ENABLE_MCOVERLAYFS=@ENABLE_MCOVERLAYFS@
+RELEASE=@UNAME_R@
 
-RELEASE=$(shell uname -r)
 MAJOR=$(shell echo ${RELEASE} | sed -e 's/^\([0-9]*\).*/\1/')
 MINOR=$(shell echo ${RELEASE} | sed -e 's/^[0-9]*.\([0-9]*\).*/\1/')
 PATCH=$(shell echo ${RELEASE} | sed -e 's/^[0-9]*.[0-9]*.\([0-9]*\).*/\1/')
@@ -9,6 +9,7 @@ RHEL_RELEASE_TMP=$(shell echo ${RELEASE} | sed -e 's/^[0-9]*.[0-9]*.[0-9]*-\([0-
 RHEL_RELEASE=$(shell if [ "${RELEASE}" == "${RHEL_RELEASE_TMP}" ]; then echo ""; else echo ${RHEL_RELEASE_TMP}; fi)
 BUILD_MODULE_TMP=$(shell if [ "${RHEL_RELEASE}" == "" ]; then echo "org"; else echo "rhel"; fi)
 BUILD_MODULE=none
+#$(info "LINUX_VERSION_CODE: ${LINUX_VERSION_CODE}, RHEL_RELEASE: ${RHEL_RELEASE}")
 ifeq ($(ENABLE_MCOVERLAYFS),yes)
 ifeq ($(BUILD_MODULE_TMP),org)
 ifeq ($(BUILD_MODULE),none)
@@ -20,7 +21,7 @@ endif
 endif
 ifeq ($(BUILD_MODULE_TMP),rhel)
 ifeq ($(BUILD_MODULE),none)
-BUILD_MODULE=$(shell if [ ${LINUX_VERSION_CODE} -eq 199168 -a ${RHEL_RELEASE} -eq 327 ]; then echo "linux-3.10.0-327.36.1.el7"; else echo "none"; fi)
+BUILD_MODULE=$(shell if [ ${LINUX_VERSION_CODE} -eq 199168 -a ${RHEL_RELEASE} -ge 327 -a ${RHEL_RELEASE} -le 514 ]; then echo "linux-3.10.0-327.36.1.el7"; else echo "none"; fi)
 endif
 endif
 endif

executer/kernel/mcoverlayfs/linux-4.6.7/inode.c

@@ -420,8 +420,8 @@ struct inode *ovl_d_select_inode(struct dentry *dentry, unsigned file_flags)
 				dentry, dentry->d_inode->i_ino);
 			OVL_DEBUG("sysfs: realpath.dentry=%pd4, i_ino=%lu\n", 
 				realpath.dentry, realpath.dentry->d_inode->i_ino);
-			if (!dentry->d_inode->i_private) {
-				dentry->d_inode->i_private = dentry->d_fsdata;
+			if (!ovl_find_d_fsdata(dentry)) {
+				ovl_add_d_fsdata(dentry);
 				dentry->d_fsdata = realpath.dentry->d_fsdata;
 			}
 		}

executer/kernel/mcoverlayfs/linux-4.6.7/overlayfs.h

@@ -43,6 +43,12 @@ enum ovl_opt_bit {
 #define OVL_OPT_NOCOPYUPW(opt)	((opt) & __OVL_OPT_NOCOPYUPW)
 #define OVL_OPT_NOFSCHECK(opt)	((opt) & __OVL_OPT_NOFSCHECK)
 
+struct ovl_d_fsdata {
+	struct list_head list;
+	struct dentry *d;
+	struct ovl_entry *oe;
+};
+
 static inline int ovl_do_rmdir(struct inode *dir, struct dentry *dentry)
 {
 	int err = vfs_rmdir(dir, dentry);
@@ -149,6 +155,8 @@ static inline int ovl_do_whiteout(struct inode *dir, struct dentry *dentry)
 
 unsigned ovl_get_config_opt(struct dentry *dentry);
 void ovl_reset_ovl_entry(struct ovl_entry **oe, struct dentry *dentry);
+struct ovl_entry *ovl_find_d_fsdata(struct dentry *dentry);
+int ovl_add_d_fsdata(struct dentry *dentry);
 enum ovl_path_type ovl_path_type(struct dentry *dentry);
 u64 ovl_dentry_version_get(struct dentry *dentry);
 void ovl_dentry_version_inc(struct dentry *dentry);

executer/kernel/mcoverlayfs/linux-4.6.7/super.c

@@ -45,6 +45,7 @@ struct ovl_fs {
 	long lower_namelen;
 	/* pathnames of lower and upper dirs, for show_options */
 	struct ovl_config config;
+	struct list_head d_fsdata_list;
 };
 
 struct ovl_dir_cache;
@@ -76,15 +77,76 @@ unsigned ovl_get_config_opt(struct dentry *dentry)
 void ovl_reset_ovl_entry(struct ovl_entry **oe, struct dentry *dentry)
 {
 	unsigned opt = ovl_get_config_opt(dentry);
+	struct ovl_entry *d_fsdata;
 
 	if (OVL_OPT_NOFSCHECK(opt)) {
-		if (dentry->d_inode && dentry->d_inode->i_private && 
-			!S_ISDIR(dentry->d_inode->i_mode)) {
-			*oe = dentry->d_inode->i_private;
+		if (dentry->d_inode && S_ISDIR(dentry->d_inode->i_mode)) {
+			return;
+		}
+
+		d_fsdata = ovl_find_d_fsdata(dentry);
+		if (d_fsdata) {
+			OVL_DEBUG("reset: dentry=%pd4, 0x%p, oe=0x%p\n", 
+				dentry, dentry, d_fsdata);
+			*oe = d_fsdata;
 		}
 	}
 }
 
+struct ovl_entry *ovl_find_d_fsdata(struct dentry *dentry)
+{
+	struct ovl_fs *ofs = dentry->d_sb->s_fs_info;
+	struct ovl_d_fsdata *d_fsdata;
+
+	list_for_each_entry(d_fsdata, &ofs->d_fsdata_list, list) {
+		if (dentry == d_fsdata->d) {
+			OVL_DEBUG("exist: dentry=%pd4, 0x%p, oe=0x%p\n", 
+				d_fsdata->d, d_fsdata->d, d_fsdata->oe);
+			return d_fsdata->oe;
+		}
+	}
+
+	return NULL;
+}
+
+int ovl_add_d_fsdata(struct dentry *dentry)
+{
+	struct ovl_fs *ofs = dentry->d_sb->s_fs_info;
+	struct ovl_d_fsdata *d_fsdata;
+
+	d_fsdata = kzalloc(sizeof(struct ovl_d_fsdata), GFP_KERNEL);
+	if (!d_fsdata) {
+		return -1;
+	}
+
+	d_fsdata->d = dentry;
+	d_fsdata->oe = dentry->d_fsdata;
+
+	list_add(&d_fsdata->list, &ofs->d_fsdata_list);
+
+	OVL_DEBUG("add: dentry=%pd4, 0x%p, oe=0x%p\n", 
+		d_fsdata->d, d_fsdata->d, d_fsdata->oe);
+
+	return 0;
+}
+
+static int ovl_clear_d_fsdata(struct ovl_fs *ofs)
+{
+	struct ovl_d_fsdata *d_fsdata;
+	struct ovl_d_fsdata *d_fsdata_next;
+
+	list_for_each_entry_safe(d_fsdata, d_fsdata_next, &ofs->d_fsdata_list, 
+		list) {
+		OVL_DEBUG("delete: dentry=%pd4, 0x%p\n", 
+			d_fsdata->d, d_fsdata->d);
+		list_del(&d_fsdata->list);
+
+		kfree(d_fsdata);
+	}
+
+	return 0;
+}
+
 static struct dentry *__ovl_dentry_lower(struct ovl_entry *oe)
 {
 	return oe->numlower ? oe->lowerstack[0].dentry : NULL;
@@ -658,6 +720,8 @@ static void ovl_put_super(struct super_block *sb)
 	struct ovl_fs *ufs = sb->s_fs_info;
 	unsigned i;
 
+	ovl_clear_d_fsdata(ufs);
+
 	dput(ufs->workdir);
 	mntput(ufs->upper_mnt);
 	for (i = 0; i < ufs->numlower; i++)
@@ -1049,6 +1113,7 @@ static int ovl_fill_super(struct super_block *sb, void *data, int silent)
 	if (!ufs)
 		goto out;
 
+	INIT_LIST_HEAD(&ufs->d_fsdata_list);
 	err = ovl_parse_opt((char *) data, &ufs->config);
 	if (err)
 		goto out_free_config;

executer/user/Makefile.in

@@ -6,14 +6,15 @@ VPATH=@abs_srcdir@
 TARGET=mcexec
 @uncomment_if_ENABLE_MEMDUMP@TARGET+=eclair
 LIBS=@LIBS@
+IHKDIR ?= $(VPATH)/../../../ihk/linux/include/
 
 all: $(TARGET)
 
 mcexec: mcexec.c
-	$(CC) -I${KDIR} $(CFLAGS) $(EXTRA_CFLAGS) -fPIE -pie -lrt -pthread -o $@ $^ $(EXTRA_OBJS)
+	$(CC) -I${KDIR} $(CFLAGS) $(EXTRA_CFLAGS) -fPIE -pie -lrt -lnuma -pthread -o $@ $^ $(EXTRA_OBJS)
 
 eclair: eclair.c
-	$(CC) $(CFLAGS) -o $@ $^ $(LIBS)
+	$(CC) $(CFLAGS) -I${IHKDIR} -o $@ $^ $(LIBS)
 
 clean:
 	$(RM) $(TARGET) *.o

executer/user/eclair.c

@@ -16,20 +16,8 @@
 #include <unistd.h>
 #include <sys/socket.h>
 #include <arpa/inet.h>
-
-/* From ihk/linux/include/ihk/ihk_host_user.h */
-#define PHYS_CHUNKS_DESC_SIZE 8192
-
-struct dump_mem_chunk {
-	unsigned long addr;
-	unsigned long size;
-};
-
-typedef struct dump_mem_chunks_s {
-	int nr_chunks;
-	struct dump_mem_chunk chunks[];
-} dump_mem_chunks_t;
-/* ---------- */
+#include <sys/ioctl.h>
+#include <ihk/ihk_host_user.h>
 
 #define CPU_TID_BASE 1000000
 
@@ -39,6 +27,10 @@ struct options {
 	char *kernel_path;
 	char *dump_path;
 	char *log_path;
+	int interactive;
+	int os_id;
+	int mcos_fd;
+	int print_idle;
 }; /* struct options */
 
 struct thread_info {
@@ -56,7 +48,7 @@ struct thread_info {
 	int tid;
 	int cpu;
 	int lcpu;
-	int padding;
+	int idle;
 	uintptr_t process;
 	uintptr_t clv;
 	uintptr_t x86_clv;
@@ -150,7 +142,21 @@ static int read_mem(uintptr_t va, void *buf, size_t size) {
 		}
 		return 1;
 	}
-	error = read_physmem(pa, buf, size);
+
+	if (opt.interactive) {
+		dumpargs_t args;
+
+		args.cmd = DUMP_READ;
+		args.start = pa;
+		args.size = size;
+		args.buf = buf;
+
+		error = ioctl(opt.mcos_fd, IHK_OS_DUMP, &args);
+	}
+	else {
+		error = read_physmem(pa, buf, size);
+	}
+
 	if (error) {
 		perror("read_mem:read_physmem");
 		return 1;
@@ -256,6 +262,7 @@ static int setup_threads(void) {
 		perror("num_processors");
 		return 1;
 	}
+	printf("%s: num_processors: %d\n", __FUNCTION__, num_processors);
 
 	error = read_symbol_64("locals", &locals);
 	if (error) {
@@ -278,64 +285,6 @@ static int setup_threads(void) {
 	ihk_mc_switch_context = lookup_symbol("ihk_mc_switch_context");
 	if (0) printf("ihk_mc_switch_context: %lx\n", ihk_mc_switch_context);
 
-	/* Set up idle threads first */
-	for (cpu = 0; cpu < num_processors; ++cpu) {
-		uintptr_t v;
-		uintptr_t thread;
-		uintptr_t proc;
-		int pid;
-		int tid;
-		struct thread_info *ti;
-		int status;
-
-		v = clv + (cpu * K(CPU_LOCAL_VAR_SIZE));
-
-		ti = malloc(sizeof(*ti));
-		if (!ti) {
-			perror("malloc");
-			return 1;
-		}
-
-		thread = v+K(IDLE_THREAD_OFFSET);
-
-		error = read_64(thread+K(PROC_OFFSET), &proc);
-		if (error) {
-			perror("proc");
-			return 1;
-		}
-
-		error = read_32(thread+K(STATUS_OFFSET), &status);
-		if (error) {
-			perror("status");
-			return 1;
-		}
-
-		error = read_32(proc+K(PID_OFFSET), &pid);
-		if (error) {
-			perror("pid");
-			return 1;
-		}
-
-		error = read_32(thread+K(TID_OFFSET), &tid);
-		if (error) {
-			perror("tid");
-			return 1;
-		}
-
-		ti->next = NULL;
-		ti->status = status;
-		ti->pid = pid;
-		ti->tid = tid;
-		ti->cpu = cpu;
-		ti->lcpu = cpu;
-		ti->process = thread;
-		ti->clv = v;
-		ti->x86_clv = locals + locals_span*cpu;
-
-		*titailp = ti;
-		titailp = &ti->next;
-	}
-
 	for (cpu = 0; cpu < num_processors; ++cpu) {
 		uintptr_t v;
 		uintptr_t head;
@@ -400,15 +349,19 @@ static int setup_threads(void) {
 			ti->status = status;
 			ti->pid = pid;
 			ti->tid = tid;
-			ti->cpu = (thread == current)? cpu: -1;
+			ti->cpu = (thread == current) ? cpu : -1;
 			ti->lcpu = cpu;
 			ti->process = thread;
+			ti->idle = 0;
 			ti->clv = v;
 			ti->x86_clv = locals + locals_span*cpu;
 
 			*titailp = ti;
 			titailp = &ti->next;
 
+			if (!curr_thread)
+				curr_thread = ti;
+
 			error = read_64(entry, &entry);
 			if (error) {
 				perror("process2");
@@ -417,8 +370,78 @@ static int setup_threads(void) {
 		}
 	}
 
+	/* Set up idle threads */
+	if (opt.print_idle) {
+		for (cpu = 0; cpu < num_processors; ++cpu) {
+			uintptr_t v;
+			uintptr_t thread;
+			uintptr_t proc;
+			int pid;
+			int tid;
+			struct thread_info *ti;
+			int status;
+
+			v = clv + (cpu * K(CPU_LOCAL_VAR_SIZE));
+
+			error = read_64(v+K(CURRENT_OFFSET), &current);
+			if (error) {
+				perror("current");
+				return 1;
+			}
+
+			ti = malloc(sizeof(*ti));
+			if (!ti) {
+				perror("malloc");
+				return 1;
+			}
+
+			thread = v+K(IDLE_THREAD_OFFSET);
+
+			error = read_64(thread+K(PROC_OFFSET), &proc);
+			if (error) {
+				perror("proc");
+				return 1;
+			}
+
+			error = read_32(thread+K(STATUS_OFFSET), &status);
+			if (error) {
+				perror("status");
+				return 1;
+			}
+
+			error = read_32(proc+K(PID_OFFSET), &pid);
+			if (error) {
+				perror("pid");
+				return 1;
+			}
+
+			error = read_32(thread+K(TID_OFFSET), &tid);
+			if (error) {
+				perror("tid");
+				return 1;
+			}
+
+			ti->next = NULL;
+			ti->status = status;
+			ti->pid = 1;
+			ti->tid = 2000000000 + tid;
+			ti->cpu = (thread == current) ? cpu : -1;
+			ti->lcpu = cpu;
+			ti->process = thread;
+			ti->idle = 1;
+			ti->clv = v;
+			ti->x86_clv = locals + locals_span*cpu;
+
+			*titailp = ti;
+			titailp = &ti->next;
+
+			if (!curr_thread)
+				curr_thread = ti;
+		}
+	}
+
 	if (!tihead) {
-		printf("thread not found. cpu mode forcibly\n");
+		printf("No threads found, forcing CPU mode.\n");
 		opt.cpu = 1;
 	}
 
@@ -459,6 +482,7 @@ static int setup_threads(void) {
 			ti->tid = CPU_TID_BASE + cpu;
 			ti->cpu = cpu;
 			ti->process = current;
+			ti->idle = 1;
 			ti->clv = v;
 			ti->x86_clv = locals + locals_span*cpu;
 
@@ -471,7 +495,9 @@ static int setup_threads(void) {
 		printf("thread not found\n");
 		return 1;
 	}
-	curr_thread = tihead;
+
+	if (!curr_thread)
+		curr_thread = tihead;
 
 	return 0;
 } /* setup_threads() */
@@ -713,18 +739,21 @@ static void command(char *cmd, char *res) {
 					break;
 				}
 
+				//if (regs[17] > MAP_KERNEL) {}
 				pu8 = (void *)&regs;
 				for (i = 0; i < sizeof(regs)-4; ++i) {
 					rbp += sprintf(rbp, "%02x", pu8[i]);
 				}
 			}
 		}
+		/*
 		else if (!strcmp(p, "mffffffff80018a82,1")) {
 			rbp += sprintf(rbp, "b8");
 		}
 		else if (!strcmp(p, "mffffffff80018a82,9")) {
 			rbp += sprintf(rbp, "b8f2ffffff41564155");
 		}
+		*/
 		else if (!strncmp(p, "m", 1)) {
 			int n;
 			uintptr_t start;
@@ -820,33 +849,35 @@ static void command(char *cmd, char *res) {
 				break;
 			}
 			q = buf;
+			q += sprintf(q, "PID %d, ", ti->pid);
 			if (ti->status & PS_RUNNING) {
-				q += sprintf(q, "running on cpu%d", ti->cpu);
+				q += sprintf(q, "%srunning on cpu %d",
+					ti->idle ? "idle " : "", ti->lcpu);
 			}
 			else if (ti->status & (PS_INTERRUPTIBLE | PS_UNINTERRUPTIBLE)) {
-				q += sprintf(q, "waiting on cpu%d", ti->lcpu);
+				q += sprintf(q, "%swaiting on cpu %d",
+					ti->idle ? "idle " : "", ti->lcpu);
 			}
 			else if (ti->status & PS_STOPPED) {
-				q += sprintf(q, "stopped on cpu%d", ti->lcpu);
+				q += sprintf(q, "%sstopped on cpu %d",
+					ti->idle ? "idle " : "", ti->lcpu);
 			}
 			else if (ti->status & PS_TRACED) {
-				q += sprintf(q, "traced on cpu%d", ti->lcpu);
+				q += sprintf(q, "%straced on cpu %d",
+					ti->idle ? "idle " : "", ti->lcpu);
 			}
 			else if (ti->status == CS_IDLE) {
-				q += sprintf(q, "cpu%d idle", ti->cpu);
+				q += sprintf(q, "cpu %d idle", ti->cpu);
 			}
 			else if (ti->status == CS_RUNNING) {
-				q += sprintf(q, "cpu%d running", ti->cpu);
+				q += sprintf(q, "cpu %d running", ti->cpu);
 			}
 			else if (ti->status == CS_RESERVED) {
-				q += sprintf(q, "cpu%d reserved", ti->cpu);
+				q += sprintf(q, "cpu %d reserved", ti->cpu);
 			}
 			else {
 				q += sprintf(q, "status=%#x", ti->status);
 			}
-			if (ti->tid != ti->pid) {
-				q += sprintf(q, ",pid=%d", ti->pid);
-			}
 			rbp += print_hex(rbp, buf);
 		}
 	} while (0);
@@ -859,11 +890,12 @@ static void options(int argc, char *argv[]) {
 	memset(&opt, 0, sizeof(opt));
 	opt.kernel_path = "./mckernel.img";
 	opt.dump_path = "./mcdump";
+	opt.mcos_fd = -1;
 
 	for (;;) {
 		int c;
 
-		c = getopt(argc, argv, "cd:hk:");
+		c = getopt(argc, argv, "ilcd:hk:o:");
 		if (c < 0) {
 			break;
 		}
@@ -881,12 +913,32 @@ static void options(int argc, char *argv[]) {
 		case 'd':
 			opt.dump_path = optarg;
 			break;
+		case 'i':
+			opt.interactive = 1;
+			break;
+		case 'o':
+			opt.os_id = atoi(optarg);
+			break;
+		case 'l':
+			opt.print_idle = 1;
+			break;
 		}
 	}
 	if (optind < argc) {
 		opt.help = 1;
 	}
 
+	if (opt.interactive) {
+		char fn[128];
+		sprintf(fn, "/dev/mcos%d", opt.os_id);
+
+		opt.mcos_fd = open(fn, O_RDONLY);
+		if (opt.mcos_fd < 0) {
+			perror("open");
+			exit(1);
+		}
+	}
+
 	return;
 } /* options() */
 
@@ -969,7 +1021,7 @@ int main(int argc, char *argv[]) {
 	uint8_t sum;
 	uint8_t check;
 	static char lbuf[1024];
-	static char rbuf[1024];
+	static char rbuf[8192];
 	static char cbuf[3];
 	char *lbp;
 	char *p;

executer/user/mcexec.c

@@ -66,6 +66,8 @@
 #include "../include/uprotocol.h"
 #include <getopt.h>
 #include "../config.h"
+#include <numa.h>
+#include <numaif.h>
 
 //#define DEBUG
 
@@ -112,9 +114,9 @@ char **__glob_argv = 0;
 #define ENABLE_MCOVERLAYFS 1
 #endif // LINUX_VERSION_CODE == 4.6
 #else
-#if RHEL_RELEASE_CODE == RHEL_RELEASE_VERSION(7,2)
+#if RHEL_RELEASE_CODE <= RHEL_RELEASE_VERSION(7,3) 
 #define ENABLE_MCOVERLAYFS 1
-#endif // RHEL_RELEASE_CODE == 7.2
+#endif // RHEL_RELEASE_CODE <= 7.3
 #endif // RHEL_RELEASE_CODE
 #endif // ENABLE_MCOVERLAYFS
 
@@ -153,6 +155,10 @@ static const char rlimit_stack_envname[] = "MCKERNEL_RLIMIT_STACK";
 static int ischild;
 static int enable_vdso = 1;
 
+/* Partitioned execution (e.g., for MPI) */
+static int nr_processes = 0;
+static int nr_threads = -1;
+
 struct fork_sync {
 	pid_t pid;
 	int status;
@@ -502,7 +508,7 @@ retry:
 
 	/* Check whether the resolved path is a symlink */
 	if (lstat(path, &sb) == -1) {
-		fprintf(stderr, "lookup_exec_path(): error stat\n");
+		__dprintf(stderr, "lookup_exec_path(): error stat\n");
 		return errno;
 	}
 
@@ -1102,7 +1108,7 @@ static int reduce_stack(struct rlimit *orig_rlim, char *argv[])
 
 void print_usage(char **argv)
 {
-	fprintf(stderr, "Usage: %s [-c target_core] [<mcos-id>] (program) [args...]\n", argv[0]);
+	fprintf(stderr, "Usage: %s [-c target_core] [-n nr_partitions] [<mcos-id>] (program) [args...]\n", argv[0]);
 }
 
 void init_sigaction(void)
@@ -1329,12 +1335,20 @@ int main(int argc, char **argv)
 	}
            
 	/* Parse options ("+" denotes stop at the first non-option) */
-	while ((opt = getopt_long(argc, argv, "+c:", mcexec_options, NULL)) != -1) {
+	while ((opt = getopt_long(argc, argv, "+c:n:t:", mcexec_options, NULL)) != -1) {
 		switch (opt) {
 			case 'c':
 				target_core = atoi(optarg);
 				break;
-			
+
+			case 'n':
+				nr_processes = atoi(optarg);
+				break;
+
+			case 't':
+				nr_threads = atoi(optarg);
+				break;
+
 			case 0:	/* long opt */
 				break;
 
@@ -1550,7 +1564,16 @@ int main(int argc, char **argv)
 		return 1;
 	}
 
-	n_threads = ncpu + 1;
+	if (nr_threads > 0) {
+		n_threads = nr_threads;
+	}
+	else if (getenv("OMP_NUM_THREADS")) {
+		/* Leave some headroom for helper threads.. */
+		n_threads = atoi(getenv("OMP_NUM_THREADS")) + 4;
+	}
+	else {
+		n_threads = ncpu;
+	}
 
 	/* 
 	 * XXX: keep thread_data ncpu sized despite that there are only
@@ -1561,6 +1584,10 @@ int main(int argc, char **argv)
 	 * TODO: implement dynaic thread pool resizing.
 	 */
 	thread_data = (struct thread_data_s *)malloc(sizeof(struct thread_data_s) * (ncpu + 1));
+	if (!thread_data) {
+		fprintf(stderr, "error: allocating thread pool data\n");
+		return 1;
+	}
 	memset(thread_data, '\0', sizeof(struct thread_data_s) * (ncpu + 1));
 
 #if 0	
@@ -1595,6 +1622,53 @@ int main(int argc, char **argv)
 		exit(1);
 	}
 
+	/* Partitioned execution, obtain CPU set */
+	if (nr_processes > 0) {
+		struct get_cpu_set_arg cpu_set_arg;
+		int mcexec_linux_numa = 0;
+
+		cpu_set_arg.cpu_set = (void *)&desc->cpu_set;
+		cpu_set_arg.cpu_set_size = sizeof(desc->cpu_set);
+		cpu_set_arg.nr_processes = nr_processes;
+		cpu_set_arg.target_core = &target_core;
+		cpu_set_arg.mcexec_linux_numa = &mcexec_linux_numa;
+
+		if (ioctl(fd, MCEXEC_UP_GET_CPUSET, (void *)&cpu_set_arg) != 0) {
+			perror("getting CPU set for partitioned execution");
+			close(fd);
+			return 1;
+		}
+
+		desc->cpu = target_core;
+
+		/* This call may not succeed, but that is fine */
+		if (numa_run_on_node(mcexec_linux_numa) < 0) {
+			__dprint("%s: WARNING: couldn't bind to NUMA %d\n",
+				__FUNCTION__, mcexec_linux_numa);
+		}
+#ifdef DEBUG
+		else {
+			cpu_set_t cpuset;
+			char affinity[BUFSIZ];
+
+			CPU_ZERO(&cpuset);
+			if ((sched_getaffinity(0, sizeof(cpu_set_t), &cpuset)) != 0) {
+				perror("Error sched_getaffinity");
+				exit(1);
+			}
+
+			affinity[0] = '\0';
+			for (i = 0; i < 512; i++) {
+				if (CPU_ISSET(i, &cpuset) == 1) {
+					sprintf(affinity, "%s %d", affinity, i);
+				}
+			}
+			__dprint("%s: PID: %d affinity: %s\n",
+					__FUNCTION__, getpid(), affinity);
+		}
+#endif
+	}
+
 	if (ioctl(fd, MCEXEC_UP_PREPARE_IMAGE, (unsigned long)desc) != 0) {
 		perror("prepare");
 		close(fd);
@@ -1703,8 +1777,8 @@ do_generic_syscall(
 
 	/* Overlayfs /sys/X directory lseek() problem work around */
 	if (w->sr.number == __NR_lseek && ret == -EINVAL) {
-		char proc_path[512];
-		char path[512];
+		char proc_path[PATH_MAX];
+		char path[PATH_MAX];
 		struct stat sb;
 
 		sprintf(proc_path, "/proc/self/fd/%d", (int)w->sr.args[0]);
@@ -1713,6 +1787,7 @@ do_generic_syscall(
 		if (readlink(proc_path, path, sizeof(path)) < 0) {
 			fprintf(stderr, "%s: error: readlink() failed for %s\n",
 				__FUNCTION__, proc_path);
+			perror(": ");
 			goto out;
 		}
 
@@ -1906,9 +1981,18 @@ int close_cloexec_fds(int mcos_fd)
 	return 0;
 }
 
+void chgdevpath(char *in, char *buf)
+{
+	if(!strcmp(in, "/dev/xpmem")){
+		sprintf(in, "/dev/null");
+	}
+}
+
 char *
 chgpath(char *in, char *buf)
 {
+	chgdevpath(in, buf);
+
 #ifdef ENABLE_MCOVERLAYFS
 	return in;
 #endif // ENABLE_MCOVERLAYFS
@@ -2552,6 +2636,24 @@ return_execve2:
 			do_syscall_return(fd, cpu, ret, 0, 0, 0, 0);
 			break;
 
+		case __NR_stat:
+			ret = do_strncpy_from_user(fd, pathbuf, (void *)w.sr.args[0], PATH_MAX);
+			if (ret >= PATH_MAX) {
+				ret = -ENAMETOOLONG;
+			}
+			if (ret < 0) {
+				do_syscall_return(fd, cpu, ret, 0, 0, 0, 0);
+				break;
+			}
+
+			fn = chgpath(pathbuf, tmpbuf);
+
+			ret = stat(fn, (struct stat *)w.sr.args[1]);
+			__dprintf("stat: path=%s, ret=%ld\n", fn, ret);
+			SET_ERR(ret);
+			do_syscall_return(fd, cpu, ret, 0, 0, 0, 0);
+			break;
+
 		default:
 			ret = do_generic_syscall(&w);
 			do_syscall_return(fd, cpu, ret, 0, 0, 0, 0);

kernel/Makefile.build.in

@@ -1,17 +1,22 @@
+ENABLE_RUSAGE=@ENABLE_RUSAGE@
 VPATH=@abs_srcdir@
 SRC=$(VPATH)
 IHKDIR=$(IHKBASE)/$(TARGETDIR)
 OBJS = init.o mem.o debug.o mikc.o listeners.o ap.o syscall.o cls.o host.o
 OBJS += process.o copy.o waitq.o futex.o timer.o plist.o fileobj.o shmobj.o
-OBJS += zeroobj.o procfs.o devobj.o sysfs.o
+OBJS += zeroobj.o procfs.o devobj.o sysfs.o xpmem.o rusage.o freeze.o
 DEPSRCS=$(wildcard $(SRC)/*.c)
+DOPT=
+ifeq ($(ENABLE_RUSAGE),yes)
+DOPT=-DENABLE_RUSAGE
+endif
 
-CFLAGS += -I$(SRC)/include -D__KERNEL__ -g
+CFLAGS += -I$(SRC)/include -D__KERNEL__ $(DOPT) -g -fno-omit-frame-pointer -fno-inline -fno-inline-small-functions
 LDFLAGS += -e arch_start
 IHKOBJ = ihk/ihk.o
 
 include $(SRC)/config/config.$(TARGET)
-include $(IHKBASE)/Makefile.common
+include @abs_builddir@/../../ihk/cokernel/Makefile.common
 
 # CFLAGS += -I$(SRC)/../arch/$(IHKARCH)/kernel/include -I$(SRC)/../lib/include
 

kernel/Makefile.in

@@ -9,7 +9,7 @@ V ?= $(VERBOSE)
 KERNEL = kernel.img
 KERNELS = $(addsuffix /$(KERNEL),$(addprefix $(O)/,$(BUILD_TARGET)))
 
-SUBCMD_OPTS = V='$(V)'
+SUBCMD_OPTS = V='$(V)' BUILD_IHK_COKERNEL=@abs_builddir@/../../ihk/cokernel
 
 $(if $(O),,$(error Specify the compilation target directory))
 #$(if $(shell ls $(IHKBASE)/Makefile),,\

kernel/ap.c

@@ -25,10 +25,27 @@
 #include <init.h>
 #include <march.h>
 #include <cls.h>
+#include <time.h>
+#include <syscall.h>
+#include <rusage.h>
+
+//#define DEBUG_PRINT_AP
+
+#ifdef DEBUG_PRINT_AP
+#define dkprintf(...) kprintf(__VA_ARGS__)
+#define ekprintf(...) kprintf(__VA_ARGS__)
+#else
+#define dkprintf(...) do { if (0) kprintf(__VA_ARGS__); } while (0)
+#define ekprintf(...) kprintf(__VA_ARGS__)
+#endif
 
 int num_processors = 1;
 static volatile int ap_stop = 1;
 
+mcs_lock_node_t ap_syscall_semaphore;
+
+extern struct ihk_os_monitor *monitor;
+
 static void ap_wait(void)
 {
 	init_tick();
@@ -43,7 +60,11 @@ static void ap_wait(void)
 	arch_start_pvclock();
 
 	if (find_command_line("hidos")) {
+		mcs_lock_node_t mcs_node;
+
+		mcs_lock_lock_noirq(&ap_syscall_semaphore, &mcs_node);
 		init_host_syscall_channel();
+		mcs_lock_unlock_noirq(&ap_syscall_semaphore, &mcs_node);
 	}
 	
 	pc_ap_init();
@@ -57,6 +78,7 @@ static void ap_wait(void)
 void ap_start(void)
 {
 	init_tick();
+	mcs_lock_init(&ap_syscall_semaphore);
 	ap_stop = 0;
 	sync_tick();
 }
@@ -93,13 +115,17 @@ void ap_init(void)
 		if (cpu_info->hw_ids[i] == bsp_hw_id) {
 			continue;
 		}
-		kprintf("AP Booting: %d (HW ID: %d @ NUMA %d)\n", i,
+		dkprintf("AP Booting: %d (HW ID: %d @ NUMA %d)\n", i,
 			cpu_info->hw_ids[i], cpu_info->nodes[i]);
 		ihk_mc_boot_cpu(cpu_info->hw_ids[i], (unsigned long)ap_wait);
 
 		num_processors++;
 	}
-	kprintf("AP Booting: Done\n");
+	kprintf("BSP: booted %d AP CPUs\n", cpu_info->ncpus - 1);
+#ifdef ENABLE_RUSAGE
+    rusage_num_threads = 0;
+    rusage_max_num_threads = 0;
+#endif
 }
 
 #include <sysfs.h>

kernel/cls.c

@@ -23,17 +23,23 @@
 extern int num_processors;
 
 struct cpu_local_var *clv;
-static int cpu_local_var_initialized = 0;
+int cpu_local_var_initialized = 0;
+extern struct ihk_os_monitor *monitor;
 
 void cpu_local_var_init(void)
 {
 	int z;
+	int i;
 
 	z = sizeof(struct cpu_local_var) * num_processors;
 	z = (z + PAGE_SIZE - 1) >> PAGE_SHIFT;
 
 	clv = ihk_mc_alloc_pages(z, IHK_MC_AP_CRITICAL);
 	memset(clv, 0, z * PAGE_SIZE);
+
+	for(i = 0; i < num_processors; i++)
+		clv[i].monitor = monitor + i;
+
 	cpu_local_var_initialized = 1;
 }
 

kernel/devobj.c

@@ -126,7 +126,8 @@ int devobj_create(int fd, size_t len, off_t off, struct memobj **objp, int *maxp
 		__FUNCTION__, fd, len, off, result.handle, result.maxprot);
 
 	obj->memobj.ops = &devobj_ops;
-	obj->memobj.flags = MF_HAS_PAGER;
+	obj->memobj.flags = MF_HAS_PAGER | MF_DEV_FILE;
+	obj->memobj.size = len;
 	obj->handle = result.handle;
 	obj->ref = 1;
 	obj->pfn_pgoff = off / PAGE_SIZE;
@@ -180,19 +181,21 @@ static void devobj_release(struct memobj *memobj)
 	memobj_unlock(&obj->memobj);
 
 	if (free_obj) {
-		int error;
-		ihk_mc_user_context_t ctx;
+		if (!(free_obj->memobj.flags & MF_HOST_RELEASED)) {
+			int error;
+			ihk_mc_user_context_t ctx;
 
-		ihk_mc_syscall_arg0(&ctx) = PAGER_REQ_UNMAP;
-		ihk_mc_syscall_arg1(&ctx) = handle;
-		ihk_mc_syscall_arg2(&ctx) = 1;
+			ihk_mc_syscall_arg0(&ctx) = PAGER_REQ_UNMAP;
+			ihk_mc_syscall_arg1(&ctx) = handle;
+			ihk_mc_syscall_arg2(&ctx) = 1;
 
-		error = syscall_generic_forwarding(__NR_mmap, &ctx);
-		if (error) {
-			kprintf("devobj_release(%p %lx):"
-					"release failed. %d\n",
-					free_obj, handle, error);
-			/* through */
+			error = syscall_generic_forwarding(__NR_mmap, &ctx);
+			if (error) {
+				kprintf("devobj_release(%p %lx):"
+						"release failed. %d\n",
+						free_obj, handle, error);
+				/* through */
+			}
 		}
 
 		if (obj->pfn_table) {

kernel/fileobj.c

@@ -29,22 +29,27 @@
 #define	dkprintf(...)	do { if (0) kprintf(__VA_ARGS__); } while (0)
 #define	ekprintf(...)	kprintf(__VA_ARGS__)
 
-static ihk_spinlock_t fileobj_list_lock = SPIN_LOCK_UNLOCKED;
+mcs_rwlock_lock_t fileobj_list_lock =
+	{{{0}, MCS_RWLOCK_TYPE_COMMON_READER, 0, 0, 0, NULL}, NULL};
 static LIST_HEAD(fileobj_list);
 
+#define FILEOBJ_PAGE_HASH_SHIFT 9
+#define FILEOBJ_PAGE_HASH_SIZE (1 << FILEOBJ_PAGE_HASH_SHIFT)
+#define FILEOBJ_PAGE_HASH_MASK (FILEOBJ_PAGE_HASH_SIZE - 1)
+
 struct fileobj {
-	struct memobj		memobj;		/* must be first */
-	long			sref;
-	long			cref;
-	uintptr_t		handle;
-	struct list_head	page_list;
-	struct list_head	list;
+	struct memobj memobj;		/* must be first */
+	long sref;
+	long cref;
+	uintptr_t handle;
+	struct list_head list;
+	struct list_head page_hash[FILEOBJ_PAGE_HASH_SIZE];
+	mcs_rwlock_lock_t page_hash_locks[FILEOBJ_PAGE_HASH_SIZE];
 };
 
 static memobj_release_func_t fileobj_release;
 static memobj_ref_func_t fileobj_ref;
 static memobj_get_page_func_t fileobj_get_page;
-static memobj_copy_page_func_t fileobj_copy_page;
 static memobj_flush_page_func_t fileobj_flush_page;
 static memobj_invalidate_page_func_t fileobj_invalidate_page;
 static memobj_lookup_page_func_t fileobj_lookup_page;
@@ -53,7 +58,7 @@ static struct memobj_ops fileobj_ops = {
 	.release =	&fileobj_release,
 	.ref =		&fileobj_ref,
 	.get_page =	&fileobj_get_page,
-	.copy_page =	&fileobj_copy_page,
+	.copy_page =	NULL,
 	.flush_page =	&fileobj_flush_page,
 	.invalidate_page =	&fileobj_invalidate_page,
 	.lookup_page =	&fileobj_lookup_page,
@@ -72,28 +77,36 @@ static struct memobj *to_memobj(struct fileobj *fileobj)
 /***********************************************************************
  * page_list
  */
-static void page_list_init(struct fileobj *obj)
+static void fileobj_page_hash_init(struct fileobj *obj)
 {
-	INIT_LIST_HEAD(&obj->page_list);
+	int i;
+	for (i = 0; i < FILEOBJ_PAGE_HASH_SIZE; ++i) {
+		mcs_rwlock_init(&obj->page_hash_locks[i]);
+		INIT_LIST_HEAD(&obj->page_hash[i]);
+	}
 	return;
 }
 
-static void page_list_insert(struct fileobj *obj, struct page *page)
+/* NOTE: caller must hold page_hash_locks[hash] */
+static void __fileobj_page_hash_insert(struct fileobj *obj,
+		struct page *page, int hash)
 {
-	list_add(&page->list, &obj->page_list);
-	return;
+	list_add(&page->list, &obj->page_hash[hash]);
 }
 
-static void page_list_remove(struct fileobj *obj, struct page *page)
+/* NOTE: caller must hold page_hash_locks[hash] */
+static void __fileobj_page_hash_remove(struct page *page)
 {
 	list_del(&page->list);
 }
 
-static struct page *page_list_lookup(struct fileobj *obj, off_t off)
+/* NOTE: caller must hold page_hash_locks[hash] */
+static struct page *__fileobj_page_hash_lookup(struct fileobj *obj,
+		int hash, off_t off)
 {
 	struct page *page;
 
-	list_for_each_entry(page, &obj->page_list, list) {
+	list_for_each_entry(page, &obj->page_hash[hash], list) {
 		if ((page->mode != PM_WILL_PAGEIO)
 				&& (page->mode != PM_PAGEIO)
 				&& (page->mode != PM_DONE_PAGEIO)
@@ -104,6 +117,7 @@ static struct page *page_list_lookup(struct fileobj *obj, off_t off)
 					obj, off, page->mode);
 			panic("page_list_lookup:invalid obj page");
 		}
+
 		if (page->offset == off) {
 			goto out;
 		}
@@ -114,13 +128,22 @@ out:
 	return page;
 }
 
-static struct page *page_list_first(struct fileobj *obj)
+static struct page *fileobj_page_hash_first(struct fileobj *obj)
 {
-	if (list_empty(&obj->page_list)) {
-		return NULL;
+	int i;
+
+	for (i = 0; i < FILEOBJ_PAGE_HASH_SIZE; ++i) {
+		if (!list_empty(&obj->page_hash[i])) {
+			break;
+		}
 	}
 
-	return list_first_entry(&obj->page_list, struct page, list);
+	if (i != FILEOBJ_PAGE_HASH_SIZE) {
+		return list_first_entry(&obj->page_hash[i], struct page, list);
+	}
+	else {
+		return NULL;
+	}
 }
 
 /***********************************************************************
@@ -163,10 +186,11 @@ static struct fileobj *obj_list_lookup(uintptr_t handle)
 int fileobj_create(int fd, struct memobj **objp, int *maxprotp)
 {
 	ihk_mc_user_context_t ctx;
-	struct pager_create_result result;	// XXX: assumes contiguous physical
+	struct pager_create_result result __attribute__((aligned(64)));	
 	int error;
 	struct fileobj *newobj  = NULL;
 	struct fileobj *obj;
+	struct mcs_rwlock_node node;
 
 	dkprintf("fileobj_create(%d)\n", fd);
 	newobj = kmalloc(sizeof(*newobj), IHK_MC_AP_NOWAIT);
@@ -179,6 +203,7 @@ int fileobj_create(int fd, struct memobj **objp, int *maxprotp)
 	ihk_mc_syscall_arg0(&ctx) = PAGER_REQ_CREATE;
 	ihk_mc_syscall_arg1(&ctx) = fd;
 	ihk_mc_syscall_arg2(&ctx) = virt_to_phys(&result);
+	memset(&result, 0, sizeof(result));
 
 	error = syscall_generic_forwarding(__NR_mmap, &ctx);
 	if (error) {
@@ -188,27 +213,43 @@ int fileobj_create(int fd, struct memobj **objp, int *maxprotp)
 
 	memset(newobj, 0, sizeof(*newobj));
 	newobj->memobj.ops = &fileobj_ops;
-	newobj->memobj.flags = MF_HAS_PAGER;
+	newobj->memobj.flags = MF_HAS_PAGER | MF_REG_FILE;
 	newobj->handle = result.handle;
 	newobj->sref = 1;
 	newobj->cref = 1;
-	page_list_init(newobj);
+	fileobj_page_hash_init(newobj);
 	ihk_mc_spinlock_init(&newobj->memobj.lock);
 
-	ihk_mc_spinlock_lock_noirq(&fileobj_list_lock);
+	mcs_rwlock_writer_lock_noirq(&fileobj_list_lock, &node);
 	obj = obj_list_lookup(result.handle);
 	if (!obj) {
 		obj_list_insert(newobj);
 		obj = newobj;
+		to_memobj(obj)->size = result.size;
+		to_memobj(obj)->flags |= result.flags;
+		to_memobj(obj)->status = MEMOBJ_READY;
+		if (to_memobj(obj)->flags & MF_PREFETCH) {
+			to_memobj(obj)->status = MEMOBJ_TO_BE_PREFETCHED;
+		}
 		newobj = NULL;
+		dkprintf("%s: new obj 0x%lx cref: %d, %s\n",
+			__FUNCTION__,
+			obj,
+			obj->cref,
+			to_memobj(obj)->flags & MF_ZEROFILL ? "zerofill" : "");
 	}
 	else {
 		++obj->sref;
 		++obj->cref;
 		memobj_unlock(&obj->memobj);	/* locked by obj_list_lookup() */
+		dkprintf("%s: existing obj 0x%lx cref: %d, %s\n",
+			__FUNCTION__,
+			obj,
+			obj->cref,
+			to_memobj(obj)->flags & MF_ZEROFILL ? "zerofill" : "");
 	}
 
-	ihk_mc_spinlock_unlock_noirq(&fileobj_list_lock);
+	mcs_rwlock_writer_unlock_noirq(&fileobj_list_lock, &node);
 
 	error = 0;
 	*objp = to_memobj(obj);
@@ -239,6 +280,7 @@ static void fileobj_release(struct memobj *memobj)
 	long free_sref = 0;
 	uintptr_t free_handle;
 	struct fileobj *free_obj = NULL;
+	struct mcs_rwlock_node node;
 
 	dkprintf("fileobj_release(%p %lx)\n", obj, obj->handle);
 
@@ -252,19 +294,28 @@ static void fileobj_release(struct memobj *memobj)
 	obj->sref -= free_sref;
 	free_handle = obj->handle;
 	memobj_unlock(&obj->memobj);
+	if (obj->memobj.flags & MF_HOST_RELEASED) {
+		free_sref = 0; // don't call syscall_generic_forwarding
+	}
 
 	if (free_obj) {
-		ihk_mc_spinlock_lock_noirq(&fileobj_list_lock);
+		dkprintf("%s: release obj 0x%lx cref: %d, free_obj: 0x%lx, %s\n",
+				__FUNCTION__,
+				obj,
+				obj->cref,
+				free_obj,
+				to_memobj(obj)->flags & MF_ZEROFILL ? "zerofill" : "");
+		mcs_rwlock_writer_lock_noirq(&fileobj_list_lock, &node);
 		/* zap page_list */
 		for (;;) {
 			struct page *page;
 			void *page_va;
 
-			page = page_list_first(obj);
+			page = fileobj_page_hash_first(obj);
 			if (!page) {
 				break;
 			}
-			page_list_remove(obj, page);
+			__fileobj_page_hash_remove(page);
 			page_va = phys_to_virt(page_to_phys(page));
 
 			if (ihk_atomic_read(&page->count) != 1) {
@@ -295,7 +346,7 @@ static void fileobj_release(struct memobj *memobj)
 #endif
 		}
 		obj_list_remove(free_obj);
-		ihk_mc_spinlock_unlock_noirq(&fileobj_list_lock);
+		mcs_rwlock_writer_unlock_noirq(&fileobj_list_lock, &node);
 		kfree(free_obj);
 	}
 
@@ -341,83 +392,101 @@ static void fileobj_do_pageio(void *args0)
 	struct page *page;
 	ihk_mc_user_context_t ctx;
 	ssize_t ss;
+	struct mcs_rwlock_node mcs_node;
+	int hash = (off >> PAGE_SHIFT) & FILEOBJ_PAGE_HASH_MASK;	
 
-	memobj_lock(&obj->memobj);
-	page = page_list_lookup(obj, off);
+	mcs_rwlock_writer_lock_noirq(&obj->page_hash_locks[hash],
+			&mcs_node);
+	page = __fileobj_page_hash_lookup(obj, hash, off);
 	if (!page) {
 		goto out;
 	}
 
 	while (page->mode == PM_PAGEIO) {
-		memobj_unlock(&obj->memobj);
+		mcs_rwlock_writer_unlock_noirq(&obj->page_hash_locks[hash],
+				&mcs_node);
 		cpu_pause();
-		memobj_lock(&obj->memobj);
+		mcs_rwlock_writer_lock_noirq(&obj->page_hash_locks[hash],
+				&mcs_node);
 	}
 
 	if (page->mode == PM_WILL_PAGEIO) {
-		page->mode = PM_PAGEIO;
-		memobj_unlock(&obj->memobj);
-
-		ihk_mc_syscall_arg0(&ctx) = PAGER_REQ_READ;
-		ihk_mc_syscall_arg1(&ctx) = obj->handle;
-		ihk_mc_syscall_arg2(&ctx) = off;
-		ihk_mc_syscall_arg3(&ctx) = pgsize;
-		ihk_mc_syscall_arg4(&ctx) = page_to_phys(page);
-
-		ss = syscall_generic_forwarding(__NR_mmap, &ctx);
-
-		memobj_lock(&obj->memobj);
-		if (page->mode != PM_PAGEIO) {
-			kprintf("fileobj_do_pageio(%p,%lx,%lx):"
-					"invalid mode %x\n",
-					obj, off, pgsize, page->mode);
-			panic("fileobj_do_pageio:invalid page mode");
+		if (to_memobj(obj)->flags & MF_ZEROFILL) {
+			void *virt = phys_to_virt(page_to_phys(page));
+			memset(virt, 0, PAGE_SIZE);
 		}
+		else {
+			page->mode = PM_PAGEIO;
+			mcs_rwlock_writer_unlock_noirq(&obj->page_hash_locks[hash],
+					&mcs_node);
 
-		if (ss == 0) {
-			dkprintf("fileobj_do_pageio(%p,%lx,%lx):EOF? %ld\n",
-					obj, off, pgsize, ss);
-			page->mode = PM_PAGEIO_EOF;
-			goto out;
-		}
-		else if (ss != pgsize) {
-			kprintf("fileobj_do_pageio(%p,%lx,%lx):"
-					"read failed. %ld\n",
-					obj, off, pgsize, ss);
-			page->mode = PM_PAGEIO_ERROR;
-			goto out;
+			ihk_mc_syscall_arg0(&ctx) = PAGER_REQ_READ;
+			ihk_mc_syscall_arg1(&ctx) = obj->handle;
+			ihk_mc_syscall_arg2(&ctx) = off;
+			ihk_mc_syscall_arg3(&ctx) = pgsize;
+			ihk_mc_syscall_arg4(&ctx) = page_to_phys(page);
+
+			dkprintf("%s: __NR_mmap for handle 0x%lx\n",
+					__FUNCTION__, obj->handle);
+			ss = syscall_generic_forwarding(__NR_mmap, &ctx);
+
+			mcs_rwlock_writer_lock_noirq(&obj->page_hash_locks[hash],
+					&mcs_node);
+			if (page->mode != PM_PAGEIO) {
+				kprintf("fileobj_do_pageio(%p,%lx,%lx):"
+						"invalid mode %x\n",
+						obj, off, pgsize, page->mode);
+				panic("fileobj_do_pageio:invalid page mode");
+			}
+
+			if (ss == 0) {
+				dkprintf("fileobj_do_pageio(%p,%lx,%lx):EOF? %ld\n",
+						obj, off, pgsize, ss);
+				page->mode = PM_PAGEIO_EOF;
+				goto out;
+			}
+			else if (ss != pgsize) {
+				kprintf("fileobj_do_pageio(%p,%lx,%lx):"
+						"read failed. %ld\n",
+						obj, off, pgsize, ss);
+				page->mode = PM_PAGEIO_ERROR;
+				goto out;
+			}
 		}
 
 		page->mode = PM_DONE_PAGEIO;
 	}
 out:
-	memobj_unlock(&obj->memobj);
+	mcs_rwlock_writer_unlock_noirq(&obj->page_hash_locks[hash],
+			&mcs_node);
 	fileobj_release(&obj->memobj);		/* got fileobj_get_page() */
 	kfree(args0);
 	dkprintf("fileobj_do_pageio(%p,%lx,%lx):\n", obj, off, pgsize);
 	return;
 }
 
-static int fileobj_get_page(struct memobj *memobj, off_t off, int p2align, uintptr_t *physp, unsigned long *pflag)
+static int fileobj_get_page(struct memobj *memobj, off_t off,
+		int p2align, uintptr_t *physp, unsigned long *pflag)
 {
 	struct thread *proc = cpu_local_var(current);
 	struct fileobj *obj = to_fileobj(memobj);
-	int error;
+	int error = -1;
 	void *virt = NULL;
 	int npages;
 	uintptr_t phys = -1;
 	struct page *page;
 	struct pageio_args *args = NULL;
+	struct mcs_rwlock_node mcs_node;
+	int hash = (off >> PAGE_SHIFT) & FILEOBJ_PAGE_HASH_MASK;	
 
 	dkprintf("fileobj_get_page(%p,%lx,%x,%p)\n", obj, off, p2align, physp);
-
-	memobj_lock(&obj->memobj);
 	if (p2align != PAGE_P2ALIGN) {
-		error = -ENOMEM;
-		goto out;
+		return -ENOMEM;
 	}
 
-	page = page_list_lookup(obj, off);
+	mcs_rwlock_writer_lock_noirq(&obj->page_hash_locks[hash],
+			&mcs_node);
+	page = __fileobj_page_hash_lookup(obj, hash, off);
 	if (!page || (page->mode == PM_WILL_PAGEIO)
 			|| (page->mode == PM_PAGEIO)) {
 		args = kmalloc(sizeof(*args), IHK_MC_AP_NOWAIT);
@@ -431,7 +500,10 @@ static int fileobj_get_page(struct memobj *memobj, off_t off, int p2align, uintp
 
 		if (!page) {
 			npages = 1 << p2align;
-			virt = ihk_mc_alloc_pages(npages, IHK_MC_AP_NOWAIT);
+
+			virt = ihk_mc_alloc_pages(npages, IHK_MC_AP_NOWAIT |
+					(to_memobj(obj)->flags & MF_ZEROFILL) ? IHK_MC_AP_USER : 0);
+
 			if (!virt) {
 				error = -ENOMEM;
 				kprintf("fileobj_get_page(%p,%lx,%x,%p):"
@@ -445,13 +517,15 @@ static int fileobj_get_page(struct memobj *memobj, off_t off, int p2align, uintp
 			if (page->mode != PM_NONE) {
 				panic("fileobj_get_page:invalid new page");
 			}
-			page->mode = PM_WILL_PAGEIO;
 			page->offset = off;
 			ihk_atomic_set(&page->count, 1);
-			page_list_insert(obj, page);
+			__fileobj_page_hash_insert(obj, page, hash);
+			page->mode = PM_WILL_PAGEIO;
 		}
 
+		memobj_lock(&obj->memobj);
 		++obj->cref;	/* for fileobj_do_pageio() */
+		memobj_unlock(&obj->memobj);
 
 		args->fileobj = obj;
 		args->objoff = off;
@@ -483,7 +557,8 @@ static int fileobj_get_page(struct memobj *memobj, off_t off, int p2align, uintp
 	*physp = page_to_phys(page);
 	virt = NULL;
 out:
-	memobj_unlock(&obj->memobj);
+	mcs_rwlock_writer_unlock_noirq(&obj->page_hash_locks[hash],
+			&mcs_node);
 	if (virt) {
 		ihk_mc_free_pages(virt, npages);
 	}
@@ -495,78 +570,6 @@ out:
 	return error;
 }
 
-static uintptr_t fileobj_copy_page(
-		struct memobj *memobj, uintptr_t orgpa, int p2align)
-{
-	struct page *orgpage = phys_to_page(orgpa);
-	size_t pgsize = PAGE_SIZE << p2align;
-	int npages = 1 << p2align;
-	void *newkva = NULL;
-	uintptr_t newpa = -1;
-	void *orgkva;
-	int count;
-
-	dkprintf("fileobj_copy_page(%p,%lx,%d)\n", memobj, orgpa, p2align);
-	if (p2align != PAGE_P2ALIGN) {
-		panic("p2align");
-	}
-
-	memobj_lock(memobj);
-	for (;;) {
-		if (!orgpage || orgpage->mode != PM_MAPPED) {
-			kprintf("fileobj_copy_page(%p,%lx,%d):"
-					"invalid cow page. %x\n",
-					memobj, orgpa, p2align, orgpage ? orgpage->mode : 0);
-			panic("fileobj_copy_page:invalid cow page");
-		}
-		count = ihk_atomic_read(&orgpage->count);
-		if (count == 2) {	// XXX: private only
-			list_del(&orgpage->list);
-			ihk_atomic_dec(&orgpage->count);
-			orgpage->mode = PM_NONE;
-			newpa = orgpa;
-			break;
-		}
-		if (count <= 0) {
-			kprintf("fileobj_copy_page(%p,%lx,%d):"
-					"orgpage count corrupted. %x\n",
-					memobj, orgpa, p2align, count);
-			panic("fileobj_copy_page:orgpage count corrupted");
-		}
-		if (newkva) {
-			orgkva = phys_to_virt(orgpa);
-			memcpy(newkva, orgkva, pgsize);
-			ihk_atomic_dec(&orgpage->count);
-			newpa = virt_to_phys(newkva);
-			if (phys_to_page(newpa)) {
-				page_map(phys_to_page(newpa));
-			}
-			newkva = NULL;	/* avoid ihk_mc_free_pages() */
-			break;
-		}
-
-		memobj_unlock(memobj);
-		newkva = ihk_mc_alloc_aligned_pages(npages, p2align,
-				IHK_MC_AP_NOWAIT);
-		if (!newkva) {
-			kprintf("fileobj_copy_page(%p,%lx,%d):"
-					"alloc page failed\n",
-					memobj, orgpa, p2align);
-			goto out;
-		}
-		memobj_lock(memobj);
-	}
-	memobj_unlock(memobj);
-
-out:
-	if (newkva) {
-		ihk_mc_free_pages(newkva, npages);
-	}
-	dkprintf("fileobj_copy_page(%p,%lx,%d): %lx\n",
-			memobj, orgpa, p2align, newpa);
-	return newpa;
-}
-
 static int fileobj_flush_page(struct memobj *memobj, uintptr_t phys,
 		size_t pgsize)
 {
@@ -575,6 +578,10 @@ static int fileobj_flush_page(struct memobj *memobj, uintptr_t phys,
 	ihk_mc_user_context_t ctx;
 	ssize_t ss;
 
+	if (to_memobj(obj)->flags & MF_ZEROFILL) {
+		return 0;
+	}
+
 	page = phys_to_page(phys);
 	if (!page) {
 		kprintf("%s: warning: tried to flush non-existing page for phys addr: 0x%lx\n", 
@@ -603,63 +610,48 @@ static int fileobj_flush_page(struct memobj *memobj, uintptr_t phys,
 static int fileobj_invalidate_page(struct memobj *memobj, uintptr_t phys,
 		size_t pgsize)
 {
-	struct fileobj *obj = to_fileobj(memobj);
-	int error;
-	struct page *page;
-
 	dkprintf("fileobj_invalidate_page(%p,%#lx,%#lx)\n",
 			memobj, phys, pgsize);
 
-	if (!(page = phys_to_page(phys))
-			|| !(page = page_list_lookup(obj, page->offset))) {
-		error = 0;
-		goto out;
-	}
-
-	if (ihk_atomic_read(&page->count) == 1) {
-		if (page_unmap(page)) {
-			ihk_mc_free_pages(phys_to_virt(phys),
-					pgsize/PAGE_SIZE);
-		}
-	}
-
-	error = 0;
-out:
-	dkprintf("fileobj_invalidate_page(%p,%#lx,%#lx):%d\n",
-			memobj, phys, pgsize, error);
-	return error;
+	/* TODO: keep track of reverse mappings so that invalidation
+	 * can be performed */
+	kprintf("%s: WARNING: file mapping invalidation not supported\n",
+		__FUNCTION__);
+	return 0;
 }
 
-static int fileobj_lookup_page(struct memobj *memobj, off_t off, int p2align, uintptr_t *physp, unsigned long *pflag)
+static int fileobj_lookup_page(struct memobj *memobj, off_t off,
+		int p2align, uintptr_t *physp, unsigned long *pflag)
 {
 	struct fileobj *obj = to_fileobj(memobj);
-	int error;
-	uintptr_t phys = -1;
+	int error = -1;
 	struct page *page;
+	struct mcs_rwlock_node mcs_node;
+	int hash = (off >> PAGE_SHIFT) & FILEOBJ_PAGE_HASH_MASK;
 
 	dkprintf("fileobj_lookup_page(%p,%lx,%x,%p)\n", obj, off, p2align, physp);
 
-	memobj_lock(&obj->memobj);
 	if (p2align != PAGE_P2ALIGN) {
-		error = -ENOMEM;
-		goto out;
+		return -ENOMEM;
 	}
 
-	page = page_list_lookup(obj, off);
+	mcs_rwlock_reader_lock_noirq(&obj->page_hash_locks[hash],
+			&mcs_node);
+
+	page = __fileobj_page_hash_lookup(obj, hash, off);
 	if (!page) {
-		error = -ENOENT;
-		dkprintf("fileobj_lookup_page(%p,%lx,%x,%p): page not found. %d\n", obj, off, p2align, physp, error);
 		goto out;
 	}
-	phys = page_to_phys(page);
 
+	*physp = page_to_phys(page);
 	error = 0;
-	if (physp) {
-		*physp = phys;
-	}
+
 out:
-	memobj_unlock(&obj->memobj);
-	dkprintf("fileobj_lookup_page(%p,%lx,%x,%p): %d %lx\n",
-			obj, off, p2align, physp, error, phys);
+	mcs_rwlock_reader_unlock_noirq(&obj->page_hash_locks[hash],
+			&mcs_node);
+
+	dkprintf("fileobj_lookup_page(%p,%lx,%x,%p): %d \n",
+			obj, off, p2align, physp, error);
 	return error;
 }
+

kernel/freeze.c

@@ -0,0 +1,54 @@
+#include <kmsg.h>
+#include <string.h>
+#include <ihk/cpu.h>
+#include <ihk/debug.h>
+#include <cls.h>
+
+extern int nmi_mode;
+extern void mod_nmi_ctx(void *, void(*)());
+extern void lapic_ack();
+extern void __freeze();
+
+void
+freeze()
+{
+	struct ihk_os_monitor *monitor = cpu_local_var(monitor);
+
+	monitor->status_bak = monitor->status;
+	monitor->status = IHK_OS_MONITOR_KERNEL_FROZEN;
+	while (monitor->status == IHK_OS_MONITOR_KERNEL_FROZEN)
+		cpu_halt();
+	monitor->status = monitor->status_bak;
+}
+
+long
+freeze_thaw(void *nmi_ctx)
+{
+	struct ihk_os_monitor *monitor = cpu_local_var(monitor);
+
+	if (nmi_mode == 1) {
+		if (monitor->status != IHK_OS_MONITOR_KERNEL_FROZEN) {
+#if 1
+			mod_nmi_ctx(nmi_ctx, __freeze);
+			return 1;
+#else
+			unsigned long flags;
+
+			flags = cpu_disable_interrupt_save();
+			monitor->status_bak = monitor->status;
+			monitor->status = IHK_OS_MONITOR_KERNEL_FROZEN;
+			lapic_ack();
+			while (monitor->status == IHK_OS_MONITOR_KERNEL_FROZEN)
+				cpu_halt();
+			monitor->status = monitor->status_bak;
+			cpu_restore_interrupt(flags);
+#endif
+		}
+	}
+	else if(nmi_mode == 2) {
+		if (monitor->status == IHK_OS_MONITOR_KERNEL_FROZEN) {
+			monitor->status = IHK_OS_MONITOR_KERNEL_THAW;
+		}
+	}
+	return 0;
+}

kernel/futex.c

@@ -248,9 +248,13 @@ static int cmpxchg_futex_value_locked(uint32_t __user *uaddr, uint32_t uval, uin
 
 static int get_futex_value_locked(uint32_t *dest, uint32_t *from)
 {
-	/* RIKEN: futexes are always on not swappable pages */
-	*dest = getint_user((int *)from);
-
+	/*
+	 * Officially we should call:
+	 * return getint_user((int *)dest, (int *)from);
+	 *
+	 * but McKernel on x86 can just access user-space.
+	 */
+	*dest = *(volatile uint32_t *)from;
 	return 0;
 }
 
@@ -670,25 +674,32 @@ static uint64_t futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q
 				uint64_t timeout)
 {
 	uint64_t time_remain = 0;
+	unsigned long irqstate;
+	struct thread *thread = cpu_local_var(current);
 	/*
 	 * The task state is guaranteed to be set before another task can
-	 * wake it. set_current_state() is implemented using set_mb() and
-	 * queue_me() calls spin_unlock() upon completion, both serializing
-	 * access to the hash list and forcing another memory barrier.
+	 * wake it. 
+	 * queue_me() calls spin_unlock() upon completion, serializing
+	 * access to the hash list and forcing a memory barrier.
 	 */
 	xchg4(&(cpu_local_var(current)->status), PS_INTERRUPTIBLE);
+
+	/* Indicate spin sleep */
+	irqstate = ihk_mc_spinlock_lock(&thread->spin_sleep_lock);
+	thread->spin_sleep = 1;
+	ihk_mc_spinlock_unlock(&thread->spin_sleep_lock, irqstate);
+
 	queue_me(q, hb);
 	
 	if (!plist_node_empty(&q->list)) {
 		
-		/* RIKEN: use mcos timers */
 		if (timeout) {
 			dkprintf("futex_wait_queue_me(): tid: %d schedule_timeout()\n", cpu_local_var(current)->tid);
 			time_remain = schedule_timeout(timeout);
 		}
 		else {
 			dkprintf("futex_wait_queue_me(): tid: %d schedule()\n", cpu_local_var(current)->tid);
-			schedule();
+			spin_sleep_or_schedule();
 			time_remain = 0;
 		}
 		
@@ -697,6 +708,7 @@ static uint64_t futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q
 	
 	/* This does not need to be serialized */
 	cpu_local_var(current)->status = PS_RUNNING;
+	thread->spin_sleep = 0;
 	
 	return time_remain;
 }
@@ -743,14 +755,17 @@ static int futex_wait_setup(uint32_t __user *uaddr, uint32_t val, int fshared,
 	 */
 	q->key = FUTEX_KEY_INIT;
 	ret = get_futex_key(uaddr, fshared, &q->key);
-	if ((ret != 0))
+	if (ret != 0)
 		return ret;
 
 	*hb = queue_lock(q);
 
 	ret = get_futex_value_locked(&uval, uaddr);
-
-	/* RIKEN: get_futex_value_locked() always returns 0 on mckernel */
+	if (ret) {
+		queue_unlock(q, *hb);
+		put_futex_key(fshared, &q->key);
+		return ret;
+	}
 
 	if (uval != val) {
 		queue_unlock(q, *hb);
@@ -776,8 +791,6 @@ static int futex_wait(uint32_t __user *uaddr, int fshared,
 	q.bitset = bitset;
 	q.requeue_pi_key = NULL;
 
-	/* RIKEN: futex_wait_queue_me() calls schedule_timeout() if timer is set */
-
 retry:
 	/* Prepare to wait on uaddr. */
 	ret = futex_wait_setup(uaddr, val, fshared, &q, &hb);

kernel/host.c

@@ -23,14 +23,15 @@
 #include <ihk/debug.h>
 #include <ihk/ikc.h>
 #include <ikc/master.h>
-#include <syscall.h>
 #include <cls.h>
+#include <syscall.h>
 #include <process.h>
 #include <page.h>
 #include <mman.h>
 #include <init.h>
 #include <kmalloc.h>
 #include <sysfs.h>
+#include <ihk/perfctr.h>
 
 //#define DEBUG_PRINT_HOST
 
@@ -125,7 +126,7 @@ int prepare_process_ranges_args_envs(struct thread *thread,
 		
 		up = virt_to_phys(up_v);
 		if (add_process_memory_range(vm, s, e, up, flags, NULL, 0,
-					PAGE_SHIFT) != 0) {
+					PAGE_SHIFT, NULL) != 0) {
 			ihk_mc_free_pages(up_v, range_npages);
 			kprintf("ERROR: adding memory range for ELF section %i\n", i);
 			goto err;
@@ -213,7 +214,7 @@ int prepare_process_ranges_args_envs(struct thread *thread,
 	args_envs_p = virt_to_phys(args_envs);
 
 	if(add_process_memory_range(vm, addr, e, args_envs_p,
-				flags, NULL, 0, PAGE_SHIFT) != 0){
+				flags, NULL, 0, PAGE_SHIFT, NULL) != 0){
 		ihk_mc_free_pages(args_envs, ARGENV_PAGE_COUNT);
 		kprintf("ERROR: adding memory range for args/envs\n");
 		goto err;
@@ -393,7 +394,9 @@ static int process_msg_prepare_process(unsigned long rphys)
 	memcpy_long(pn, p, sizeof(struct program_load_desc) 
 	            + sizeof(struct program_image_section) * n);
 
-	if((thread = create_thread(p->entry)) == NULL){
+	if ((thread = create_thread(p->entry,
+					(unsigned long *)&p->cpu_set,
+					sizeof(p->cpu_set))) == NULL) {
 		kfree(pn);
 		ihk_mc_unmap_virtual(p, npages, 1);
 		ihk_mc_unmap_memory(NULL, phys, sz);
@@ -432,9 +435,6 @@ static int process_msg_prepare_process(unsigned long rphys)
 	vm->region.map_end = vm->region.map_start;
 	memcpy(proc->rlimit, pn->rlimit, sizeof(struct rlimit) * MCK_RLIM_MAX);
 
-	/* TODO: Clear it at the proper timing */
-	cpu_local_var(scp).post_idx = 0;
-
 	if (prepare_process_ranges_args_envs(thread, pn, p, attr, 
 				NULL, 0, NULL, 0) != 0) {
 		kprintf("error: preparing process ranges, args, envs, stack\n");
@@ -459,70 +459,6 @@ err:
 	return -ENOMEM;
 }
 
-static void process_msg_init(struct ikc_scd_init_param *pcp, struct syscall_params *lparam)
-{
-	lparam->response_va = ihk_mc_alloc_pages(RESPONSE_PAGE_COUNT, 0);
-	lparam->response_pa = virt_to_phys(lparam->response_va);
-
-	pcp->request_page = 0;
-	pcp->doorbell_page = 0;
-	pcp->response_page = lparam->response_pa;
-}
-
-static void process_msg_init_acked(struct ihk_ikc_channel_desc *c, unsigned long pphys)
-{
-	struct ikc_scd_init_param *param = phys_to_virt(pphys);
-	struct syscall_params *lparam;
-	enum ihk_mc_pt_attribute attr;
-
-	attr = PTATTR_NO_EXECUTE | PTATTR_WRITABLE | PTATTR_FOR_USER;
-
-	lparam = &cpu_local_var(scp);
-	if(cpu_local_var(syscall_channel2) == c)
-		lparam = &cpu_local_var(scp2);
-	lparam->request_rpa = param->request_page;
-	lparam->request_pa = ihk_mc_map_memory(NULL, param->request_page,
-	                                       REQUEST_PAGE_COUNT * PAGE_SIZE);
-	if((lparam->request_va = ihk_mc_map_virtual(lparam->request_pa,
-	                                        REQUEST_PAGE_COUNT,
-	                                        attr)) == NULL){
-		// TODO: 
-		panic("ENOMEM");
-	}
-
-	lparam->doorbell_rpa = param->doorbell_page;
-	lparam->doorbell_pa = ihk_mc_map_memory(NULL, param->doorbell_page,
-	                                        DOORBELL_PAGE_COUNT * 
-	                                        PAGE_SIZE);
-	if((lparam->doorbell_va = ihk_mc_map_virtual(lparam->doorbell_pa,
-	                                         DOORBELL_PAGE_COUNT,
-	                                         attr)) == NULL){
-		// TODO: 
-		panic("ENOMEM");
-	}
-
-	lparam->post_rpa = param->post_page;
-	lparam->post_pa = ihk_mc_map_memory(NULL, param->post_page,
-	                                    PAGE_SIZE);
-	if((lparam->post_va = ihk_mc_map_virtual(lparam->post_pa, 1,
-	                                     attr)) == NULL){
-		// TODO: 
-		panic("ENOMEM");
-	}
-
-	lparam->post_fin = 1;
-
-	dkprintf("Syscall parameters: (%d)\n", ihk_mc_get_processor_id());
-	dkprintf(" Response: %lx, %p\n",
-	        lparam->response_pa, lparam->response_va);
-	dkprintf(" Request : %lx, %lx, %p\n",
-	        lparam->request_pa, lparam->request_rpa, lparam->request_va);
-	dkprintf(" Doorbell: %lx, %lx, %p\n",
-	        lparam->doorbell_pa, lparam->doorbell_rpa, lparam->doorbell_va);
-	dkprintf(" Post: %lx, %lx, %p\n",
-	        lparam->post_pa, lparam->post_rpa, lparam->post_va);
-}
-
 static void syscall_channel_send(struct ihk_ikc_channel_desc *c,
                                  struct ikc_scd_packet *packet)
 {
@@ -530,7 +466,7 @@ static void syscall_channel_send(struct ihk_ikc_channel_desc *c,
 }
 
 extern unsigned long do_kill(struct thread *, int, int, int, struct siginfo *, int ptracecont);
-extern void process_procfs_request(unsigned long rarg);
+extern void process_procfs_request(struct ikc_scd_packet *rpacket);
 extern void terminate_host(int pid);
 extern void debug_log(long);
 
@@ -553,11 +489,12 @@ static int syscall_packet_handler(struct ihk_ikc_channel_desc *c,
 	unsigned long pp;
 	int cpuid;
 	int ret = 0;
+	struct perf_ctrl_desc *pcd;
+	unsigned int mode = 0;
 
 	switch (packet->msg) {
 	case SCD_MSG_INIT_CHANNEL_ACKED:
 		dkprintf("SCD_MSG_INIT_CHANNEL_ACKED\n");
-		process_msg_init_acked(c, packet->arg);
 		ret = 0;
 		break;
 
@@ -579,14 +516,16 @@ static int syscall_packet_handler(struct ihk_ikc_channel_desc *c,
 		break;
 
 	case SCD_MSG_SCHEDULE_PROCESS:
-		cpuid = obtain_clone_cpuid();
-		if(cpuid == -1){
+		thread = (struct thread *)packet->arg;
+
+		cpuid = obtain_clone_cpuid(&thread->cpu_set);
+		if (cpuid == -1) {
 			kprintf("No CPU available\n");
 			ret = -1;
 			break;
 		}
+
 		dkprintf("SCD_MSG_SCHEDULE_PROCESS: %lx\n", packet->arg);
-		thread = (struct thread *)packet->arg;
 		proc = thread->proc;
 		thread->tid = proc->pid;
 		proc->status = PS_RUNNING;
@@ -594,8 +533,7 @@ static int syscall_packet_handler(struct ihk_ikc_channel_desc *c,
 		chain_thread(thread);
 		chain_process(proc);
 		runq_add_thread(thread, cpuid);
-					  
-		//cpu_local_var(next) = (struct thread *)packet->arg;
+
 		ret = 0;
 		break;
 
@@ -637,7 +575,7 @@ static int syscall_packet_handler(struct ihk_ikc_channel_desc *c,
 		break;
 
 	case SCD_MSG_PROCFS_REQUEST:
-		process_procfs_request(packet->arg);
+		process_procfs_request(packet);
 		ret = 0;
 		break;
 
@@ -662,6 +600,61 @@ static int syscall_packet_handler(struct ihk_ikc_channel_desc *c,
 		ret = 0;
 		break;
 
+	case SCD_MSG_PERF_CTRL:
+		pp = ihk_mc_map_memory(NULL, packet->arg, sizeof(struct perf_ctrl_desc));
+		pcd = (struct perf_ctrl_desc *)ihk_mc_map_virtual(pp, 1, PTATTR_WRITABLE | PTATTR_ACTIVE);
+
+		switch (pcd->ctrl_type) {
+		case PERF_CTRL_SET:
+			if (!pcd->exclude_kernel) {
+				mode |= PERFCTR_KERNEL_MODE;
+			}
+			if (!pcd->exclude_user) {
+				mode |= PERFCTR_USER_MODE;
+			}
+			ihk_mc_perfctr_init_raw(pcd->target_cntr, pcd->config, mode);
+			ihk_mc_perfctr_stop(1 << pcd->target_cntr);
+			ihk_mc_perfctr_reset(pcd->target_cntr);
+			break;
+
+		case PERF_CTRL_ENABLE:
+			ihk_mc_perfctr_start(pcd->target_cntr_mask);
+			break;
+			
+		case PERF_CTRL_DISABLE:
+			ihk_mc_perfctr_stop(pcd->target_cntr_mask);
+			break;
+
+		case PERF_CTRL_GET:
+			pcd->read_value = ihk_mc_perfctr_read(pcd->target_cntr);
+			break;
+			
+		default:
+			kprintf("%s: SCD_MSG_PERF_CTRL unexpected ctrl_type\n", __FUNCTION__);
+		}
+
+		ihk_mc_unmap_virtual(pcd, 1, 0);
+		ihk_mc_unmap_memory(NULL, pp, sizeof(struct perf_ctrl_desc));
+
+		pckt.msg = SCD_MSG_PERF_ACK;
+		pckt.err = 0;
+		pckt.arg = packet->arg;
+		ihk_ikc_send(c, &pckt, 0);
+
+		ret = 0;
+		break;
+
+	case SCD_MSG_CPU_RW_REG:
+
+		pckt.msg = SCD_MSG_CPU_RW_REG_RESP;
+		memcpy(&pckt.desc, &packet->desc,
+				sizeof(struct mcctrl_os_cpu_register));
+		pckt.resp = packet->resp;
+		pckt.err = arch_cpu_read_write_register(&pckt.desc, pckt.op);
+
+		ihk_ikc_send(c, &pckt, 0);
+		break;
+
 	default:
 		kprintf("syscall_pakcet_handler:unknown message "
 				"(%d.%d.%d.%d.%d.%#lx)\n",
@@ -683,7 +676,7 @@ void init_host_syscall_channel(void)
 
 	param.port = 501;
 	param.pkt_size = sizeof(struct ikc_scd_packet);
-	param.queue_size = PAGE_SIZE;
+	param.queue_size = PAGE_SIZE * 4;
 	param.magic = 0x1129;
 	param.handler = syscall_packet_handler;
 
@@ -696,7 +689,6 @@ void init_host_syscall_channel(void)
 
 	get_this_cpu_local_var()->syscall_channel = param.channel;
 
-	process_msg_init(&cpu_local_var(iip), &cpu_local_var(scp));
 	pckt.msg = SCD_MSG_INIT_CHANNEL;
 	pckt.ref = ihk_mc_get_processor_id();
 	pckt.arg = virt_to_phys(&cpu_local_var(iip));
@@ -710,7 +702,7 @@ void init_host_syscall_channel2(void)
 
 	param.port = 502;
 	param.pkt_size = sizeof(struct ikc_scd_packet);
-	param.queue_size = PAGE_SIZE;
+	param.queue_size = PAGE_SIZE * 4;
 	param.magic = 0x1329;
 	param.handler = syscall_packet_handler;
 
@@ -723,7 +715,6 @@ void init_host_syscall_channel2(void)
 
 	get_this_cpu_local_var()->syscall_channel2 = param.channel;
 
-	process_msg_init(&cpu_local_var(iip2), &cpu_local_var(scp2));
 	pckt.msg = SCD_MSG_INIT_CHANNEL;
 	pckt.ref = ihk_mc_get_processor_id();
 	pckt.arg = virt_to_phys(&cpu_local_var(iip2));

kernel/include/cls.h

@@ -28,6 +28,26 @@ struct kmalloc_header {
 	/* 32 bytes */
 };
 
+#define IHK_OS_MONITOR_NOT_BOOT 0
+#define IHK_OS_MONITOR_IDLE 1
+#define IHK_OS_MONITOR_USER 2
+#define IHK_OS_MONITOR_KERNEL 3
+#define IHK_OS_MONITOR_KERNEL_HEAVY 4
+#define IHK_OS_MONITOR_KERNEL_OFFLOAD 5
+#define IHK_OS_MONITOR_KERNEL_FREEZING 8
+#define IHK_OS_MONITOR_KERNEL_FROZEN 9
+#define IHK_OS_MONITOR_KERNEL_THAW 10
+#define IHK_OS_MONITOR_PANIC 99
+
+struct ihk_os_monitor {
+	int status;
+	int status_bak;
+	unsigned long counter;
+	unsigned long ocounter;
+	unsigned long user_tsc;
+	unsigned long system_tsc;
+};
+
 #include <ihk/lock.h>
 #define CPU_STATUS_DISABLE	(0)
 #define CPU_STATUS_IDLE		(1)
@@ -56,11 +76,9 @@ struct cpu_local_var {
 	size_t runq_len;
 
 	struct ihk_ikc_channel_desc *syscall_channel;
-	struct syscall_params scp;
 	struct ikc_scd_init_param iip;
 
 	struct ihk_ikc_channel_desc *syscall_channel2;
-	struct syscall_params scp2;
 	struct ikc_scd_init_param iip2;
 	struct resource_set *resource_set;
 	
@@ -77,6 +95,7 @@ struct cpu_local_var {
 	int no_preempt;
 	int timer_enabled;
 	int kmalloc_initialized;
+	struct ihk_os_monitor *monitor;
 } __attribute__((aligned(64)));
 
 

kernel/include/init.h

@@ -16,7 +16,7 @@
 extern void arch_init(void);
 extern void kmsg_init(int);
 extern void mem_init(void);
-extern void ikc_master_init(void);
+extern void ihk_ikc_master_init(void);
 extern void ap_init(void);
 extern void arch_ready(void);
 extern void mc_ikc_test_init(void);
@@ -29,7 +29,11 @@ extern void init_host_syscall_channel2(void);
 extern void sched_init(void);
 extern void pc_ap_init(void);
 extern void cpu_sysfs_setup(void);
+extern void rusage_sysfs_setup(void);
+extern void status_sysfs_setup(void);
 
 extern char *find_command_line(char *name);
 
+extern int num_processors;
+
 #endif

kernel/include/kmalloc.h

@@ -28,9 +28,9 @@ r;\
 })
 #define kfree(ptr) _kfree(ptr, __FILE__, __LINE__)
 #define memcheck(ptr, msg) _memcheck(ptr, msg, __FILE__, __LINE__, 0)
-void *_kmalloc(int size, enum ihk_mc_ap_flag flag, char *file, int line);
+void *_kmalloc(int size, ihk_mc_ap_flag flag, char *file, int line);
 void _kfree(void *ptr, char *file, int line);
-void *__kmalloc(int size, enum ihk_mc_ap_flag flag);
+void *__kmalloc(int size, ihk_mc_ap_flag flag);
 void __kfree(void *ptr);
 
 int _memcheck(void *ptr, char *msg, char *file, int line, int free);

kernel/include/memobj.h

@@ -32,13 +32,23 @@ enum {
 	MF_HAS_PAGER	= 0x0001,
 	MF_SHMDT_OK	= 0x0002,
 	MF_IS_REMOVABLE	= 0x0004,
+	MF_PREFETCH = 0x0008,
+	MF_ZEROFILL = 0x0010,
+	MF_REG_FILE = 0x1000,
+	MF_DEV_FILE = 0x2000,
+	MF_HOST_RELEASED = 0x80000000,
+	MF_END
 };
 
+#define MEMOBJ_READY              0
+#define MEMOBJ_TO_BE_PREFETCHED   1
+
 struct memobj {
-	struct memobj_ops *	ops;
-	uint32_t		flags;
-	int8_t			padding[4];
-	ihk_spinlock_t		lock;
+	struct memobj_ops *ops;
+	uint32_t flags;
+	uint32_t status;
+	size_t size;
+	ihk_spinlock_t lock;
 };
 
 typedef void memobj_release_func_t(struct memobj *obj);

kernel/include/pager.h

@@ -30,7 +30,8 @@ enum pager_op {
 struct pager_create_result {
 	uintptr_t	handle;
 	int		maxprot;
-	int8_t		padding[4];
+	uint32_t flags;
+	size_t size;
 };
 
 /*

kernel/include/process.h

@@ -166,7 +166,7 @@
 
 #define NOPHYS ((uintptr_t)-1)
 
-#define PROCESS_NUMA_MASK_BITS 64
+#define PROCESS_NUMA_MASK_BITS 256
 
 /*
  * Both the MPOL_* mempolicy mode and the MPOL_F_* optional mode flags are
@@ -242,6 +242,28 @@ struct process_vm;
 struct vm_regions;
 struct vm_range;
 
+//#define TRACK_SYSCALLS
+
+#ifdef TRACK_SYSCALLS
+#define TRACK_SYSCALLS_MAX               300
+#define __NR_track_syscalls              701
+
+#define TRACK_SYSCALLS_CLEAR             0x01
+#define TRACK_SYSCALLS_ON                0x02
+#define TRACK_SYSCALLS_OFF               0x04
+#define TRACK_SYSCALLS_PRINT             0x08
+#define TRACK_SYSCALLS_PRINT_PROC        0x10
+
+void track_syscalls_print_thread_stats(struct thread *thread);
+void track_syscalls_print_proc_stats(struct process *proc);
+void track_syscalls_accumulate_counters(struct thread *thread,
+		struct process *proc);
+void track_syscalls_alloc_counters(struct thread *thread);
+void track_syscalls_dealloc_thread_counters(struct thread *thread);
+void track_syscalls_dealloc_proc_counters(struct process *proc);
+#endif // TRACK_SYSCALLS
+
+
 #define HASH_SIZE	73
 
 struct resource_set {
@@ -367,6 +389,7 @@ struct vm_range {
 	off_t objoff;
 	int pgshift;	/* page size. 0 means THP */
 	int padding;
+	void *private_data;
 };
 
 struct vm_range_numa_policy {
@@ -405,7 +428,7 @@ struct mckfd {
 #define SFD_NONBLOCK 04000
 
 struct sig_common {
-	ihk_spinlock_t	lock;
+	mcs_rwlock_lock_t lock;
 	ihk_atomic_t use;
 	struct k_sigaction action[_NSIG];
 	struct list_head sigpending;
@@ -466,7 +489,7 @@ struct process {
 			// V       +----   |
 			// PS_STOPPED -----+
 			// (PS_TRACED)
-	int exit_status;
+	int exit_status; // only for zombie
 
 	/* Store exit_status for a group of threads when stopped by SIGSTOP.
 	   exit_status can't be used because values of exit_status of threads
@@ -537,6 +560,13 @@ struct process {
 #define PP_COUNT 2
 #define PP_STOP 3
 	struct mc_perf_event *monitoring_event;
+#ifdef TRACK_SYSCALLS
+	mcs_lock_node_t st_lock;
+	uint64_t *syscall_times;
+	uint32_t *syscall_cnts;
+	uint64_t *offload_times;
+	uint32_t *offload_cnts;
+#endif // TRACK_SYSCALLS
 };
 
 void hold_thread(struct thread *ftn);
@@ -578,6 +608,7 @@ struct thread {
 			// PS_TRACED
 			// PS_INTERRPUTIBLE
 			// PS_UNINTERRUPTIBLE
+	int exit_status;
 
 	// process vm
 	struct process_vm *vm;
@@ -608,12 +639,20 @@ struct thread {
 	fp_regs_struct *fp_regs;
 	int in_syscall_offload;
 
+#ifdef TRACK_SYSCALLS
+	int track_syscalls;
+	uint64_t *syscall_times;
+	uint32_t *syscall_cnts;
+	uint64_t *offload_times;
+	uint32_t *offload_cnts;
+#endif // TRACK_SYSCALLS
+
 	// signal
 	struct sig_common *sigcommon;
 	sigset_t sigmask;
 	stack_t sigstack;
 	struct list_head sigpending;
-	ihk_spinlock_t	sigpendinglock;
+	mcs_rwlock_lock_t sigpendinglock;
 	volatile int sigevent;
 
 	// gpio
@@ -626,9 +665,14 @@ struct thread {
 	struct sig_pending *ptrace_sendsig;
 
 	// cpu time
+	/*
 	struct timespec stime;
 	struct timespec utime;
 	struct timespec btime;
+	*/
+	unsigned long system_tsc;
+	unsigned long user_tsc;
+	unsigned long base_tsc;
 	int times_update;
 	int in_kernel;
 
@@ -689,7 +733,8 @@ static inline int has_cap_sys_admin(struct thread *th)
 
 void hold_address_space(struct address_space *);
 void release_address_space(struct address_space *);
-struct thread *create_thread(unsigned long user_pc);
+struct thread *create_thread(unsigned long user_pc,
+		unsigned long *__cpu_set, size_t cpu_set_size);
 struct thread *clone_thread(struct thread *org, unsigned long pc,
                               unsigned long sp, int clone_flags);
 void destroy_thread(struct thread *thread);
@@ -704,9 +749,10 @@ void free_process_memory_ranges(struct process_vm *vm);
 int populate_process_memory(struct process_vm *vm, void *start, size_t len);
 
 int add_process_memory_range(struct process_vm *vm,
-                             unsigned long start, unsigned long end,
-                             unsigned long phys, unsigned long flag,
-			     struct memobj *memobj, off_t objoff, int pgshift);
+		unsigned long start, unsigned long end,
+		unsigned long phys, unsigned long flag,
+		struct memobj *memobj, off_t offset,
+		int pgshift, struct vm_range **rp);
 int remove_process_memory_range(struct process_vm *vm, unsigned long start,
 		unsigned long end, int *ro_freedp);
 int split_process_memory_range(struct process_vm *vm,
@@ -746,9 +792,11 @@ extern enum ihk_mc_pt_attribute arch_vrflag_to_ptattr(unsigned long flag, uint64
 enum ihk_mc_pt_attribute common_vrflag_to_ptattr(unsigned long flag, uint64_t fault, pte_t *ptep);
 
 void schedule(void);
+void spin_sleep_or_schedule(void);
 void runq_add_thread(struct thread *thread, int cpu_id);
 void runq_del_thread(struct thread *thread, int cpu_id);
 int sched_wakeup_thread(struct thread *thread, int valid_states);
+int sched_wakeup_thread_locked(struct thread *thread, int valid_states);
 
 void sched_request_migrate(int cpu_id, struct thread *thread);
 void check_need_resched(void);

kernel/include/rusage.h

@@ -0,0 +1,79 @@
+#ifndef __RUSAGE_H
+#define __RUSAGE_H
+
+#define RUSAGE_DEFAULT_SIZE 10
+
+enum RUSAGE_MEMBER {
+    RUSAGE_RSS,
+    RUSAGE_CACHE,
+    RUSAGE_RSS_HUGE,
+    RUSAGE_MAPPED_FILE,
+    RUSAGE_MAX_USAGE,
+    RUSAGE_KMEM_USAGE,
+    RUSAGE_KMAX_USAGE,
+    RUSAGE_NUM_NUMA_NODES,
+    RUSAGE_NUMA_STAT,
+    RUSAGE_HUGETLB ,
+    RUSAGE_HUGETLB_MAX ,
+    RUSAGE_STAT_SYSTEM ,
+    RUSAGE_STAT_USER ,
+    RUSAGE_USAGE ,
+    RUSAGE_USAGE_PER_CPU ,
+    RUSAGE_NUM_THREADS ,
+    RUSAGE_MAX_NUM_THREADS
+};
+
+struct  r_data{
+    unsigned long pid;
+    unsigned long rss;
+    unsigned long cache;
+    unsigned long rss_huge;
+    unsigned long mapped_file;
+    unsigned long max_usage;
+    unsigned long kmem_usage;
+    unsigned long kmax_usage;
+    unsigned long hugetlb;
+    unsigned long hugetlb_max;
+    unsigned long stat_system;
+    unsigned long stat_user;
+    unsigned long usage;
+    struct r_data *next;
+} ;
+typedef struct r_data rusage_data;
+
+rusage_data *rdata[RUSAGE_DEFAULT_SIZE];
+unsigned long rusage_max_num_threads;
+unsigned long rusage_num_threads;
+
+enum ihk_os_status {
+    IHK_STATUS_INACTIVE,
+    IHK_STATUS_BOOTING,
+    IHK_STATUS_RUNNING,
+    IHK_STATUS_SHUTDOWN,
+    IHK_STATUS_PANIC,
+    IHK_STATUS_HUNGUP,
+    IHK_STATUS_FREEZING,
+    IHK_STATUS_FROZEN,
+};
+
+enum ihk_os_status os_status;
+unsigned long sys_delegate_count;
+enum sys_delegate_state_enum {
+	ENTER_KERNEL,
+	EXIT_KERNEL,
+};
+enum sys_delegate_state_enum sys_delegate_state;
+
+unsigned long rusage_rss[sizeof(cpu_set_t)/8];
+unsigned long rusage_rss_max;
+long rusage_rss_current;
+unsigned long rusage_kmem_usage;
+unsigned long rusage_kmem_max_usage;
+unsigned long rusage_hugetlb_usage;
+unsigned long rusage_hugetlb_max_usage;
+unsigned long rusage_numa_stat[1024];
+unsigned long rusage_max_memory;
+
+#define RUSAGE_MEM_LIMIT 2000000
+
+#endif

kernel/include/syscall.h

@@ -73,6 +73,13 @@
 /* #define SCD_MSG_SYSFS_RESP_CLEANUP   0x43 */
 #define SCD_MSG_PROCFS_TID_CREATE	0x44
 #define SCD_MSG_PROCFS_TID_DELETE	0x45
+#define SCD_MSG_EVENT_SIGNAL		0x46
+
+#define SCD_MSG_PERF_CTRL               0x50
+#define SCD_MSG_PERF_ACK                0x51
+
+#define SCD_MSG_CPU_RW_REG              0x52
+#define SCD_MSG_CPU_RW_REG_RESP         0x53
 
 /* Cloning flags.  */
 # define CSIGNAL       0x000000ff /* Signal mask to be sent at exit.  */
@@ -149,6 +156,10 @@ struct program_image_section {
 #define MCK_RLIMIT_SIGPENDING	14
 #define MCK_RLIMIT_STACK	15
 
+#define PLD_CPU_SET_MAX_CPUS 1024
+typedef unsigned long __cpu_set_unit;
+#define PLD_CPU_SET_SIZE (PLD_CPU_SET_MAX_CPUS / (8 * sizeof(__cpu_set_unit)))
+
 struct program_load_desc {
 	int num_sections;
 	int status;
@@ -178,6 +189,7 @@ struct program_load_desc {
 	struct rlimit rlimit[MCK_RLIM_MAX];
 	unsigned long interp_align;
 	char shell_path[SHELL_PATH_MAX_LEN];
+	__cpu_set_unit cpu_set[PLD_CPU_SET_SIZE];
 	struct program_image_section sections[0];
 };
 
@@ -201,6 +213,18 @@ struct syscall_request {
 	unsigned long args[6];
 };
 
+struct mcctrl_os_cpu_register {
+	unsigned long addr;
+	unsigned long val;
+	unsigned long addr_ext;
+};
+
+enum mcctrl_os_cpu_operation {
+	MCCTRL_OS_CPU_READ_REGISTER,
+	MCCTRL_OS_CPU_WRITE_REGISTER,
+	MCCTRL_OS_CPU_MAX_OP
+};
+
 struct ikc_scd_packet {
 	int msg;
 	int err;
@@ -226,6 +250,13 @@ struct ikc_scd_packet {
 		struct {
 			int ttid;
 		};
+
+		/* SCD_MSG_CPU_RW_REG */
+		struct {
+			struct mcctrl_os_cpu_register desc;
+			enum mcctrl_os_cpu_operation op;
+			void *resp;
+		};
 	};
 	char padding[12];
 };
@@ -250,22 +281,6 @@ struct syscall_post {
 	unsigned long v[8];
 };
 
-struct syscall_params {
-	unsigned long request_rpa, request_pa;
-	struct syscall_request *request_va;
-	unsigned long response_pa;
-	struct syscall_response *response_va;
-
-	unsigned long doorbell_rpa, doorbell_pa;
-	unsigned long *doorbell_va;
-
-	unsigned int post_idx;
-	unsigned long post_rpa, post_pa;
-	struct syscall_post *post_va;
-	unsigned long post_fin;
-	struct syscall_post post_buf IHK_DMA_ALIGN;
-};
-
 #define SYSCALL_DECLARE(name) long sys_##name(int n, ihk_mc_user_context_t *ctx)
 #define SYSCALL_HEADER struct syscall_request request IHK_DMA_ALIGN; \
 	request.number = n
@@ -385,8 +400,32 @@ struct tod_data_s {
 };
 extern struct tod_data_s tod_data;	/* residing in arch-dependent file */
 
+static inline void tsc_to_ts(unsigned long tsc, struct timespec *ts)
+{
+	time_t sec_delta;
+	long ns_delta;
+
+	sec_delta = tsc / tod_data.clocks_per_sec;
+	ns_delta = NS_PER_SEC * (tsc % tod_data.clocks_per_sec)
+	           / tod_data.clocks_per_sec;
+	/* calc. of ns_delta overflows if clocks_per_sec exceeds 18.44 GHz */
+
+	ts->tv_sec = sec_delta;
+	ts->tv_nsec = ns_delta;
+	if (ts->tv_nsec >= NS_PER_SEC) {
+		ts->tv_nsec -= NS_PER_SEC;
+		++ts->tv_sec;
+	}
+}
+
+static inline unsigned long timespec_to_jiffy(const struct timespec *ats)
+{
+	return ats->tv_sec * 100 + ats->tv_nsec / 10000000;
+}
+
 void reset_cputime();
 void set_cputime(int mode);
+int do_munmap(void *addr, size_t len);
 intptr_t do_mmap(intptr_t addr0, size_t len0, int prot, int flags, int fd,
 		off_t off0);
 void clear_host_pte(uintptr_t addr, size_t len);
@@ -395,6 +434,8 @@ int do_shmget(key_t key, size_t size, int shmflg);
 struct process_vm;
 int arch_map_vdso(struct process_vm *vm);	/* arch dependent */
 int arch_setup_vdso(void);
+int arch_cpu_read_write_register(struct mcctrl_os_cpu_register *desc,
+		enum mcctrl_os_cpu_operation op);
 
 #define VDSO_MAXPAGES 2
 struct vdso {
@@ -431,4 +472,34 @@ struct get_cpu_mapping_req {
 #endif
 };
 
+enum perf_ctrl_type {
+	PERF_CTRL_SET,
+	PERF_CTRL_GET,
+	PERF_CTRL_ENABLE,
+	PERF_CTRL_DISABLE,
+};
+
+struct perf_ctrl_desc {
+	enum perf_ctrl_type ctrl_type;
+	int status;
+	union {
+		/* for SET, GET */
+		struct {
+			unsigned int target_cntr;
+			unsigned long config;
+			unsigned long read_value;
+			unsigned disabled        :1,
+			         pinned          :1,
+			         exclude_user    :1,
+			         exclude_kernel  :1,
+			         exclude_hv      :1,
+			         exclude_idle    :1;
+		};
+
+		/* for START, STOP*/
+		struct {
+			unsigned long target_cntr_mask;
+		};
+	};
+};
 #endif

kernel/include/waitq.h

@@ -27,6 +27,8 @@ typedef int (*waitq_func_t)(struct waitq_entry *wait, unsigned mode,
 
 int default_wake_function(struct waitq_entry *wait, unsigned mode, int flags,
 			              void *key);
+int locked_wake_function(struct waitq_entry *wait, unsigned mode, int flags,
+			              void *key);
 
 typedef struct waitq {
 	ihk_spinlock_t lock;
@@ -57,6 +59,13 @@ typedef struct waitq_entry {
 	    .link  = { &(name).link, &(name).link }			\
 	}
 
+#define DECLARE_WAITQ_ENTRY_LOCKED(name, tsk)           \
+	waitq_entry_t name = {								\
+	    .private  = tsk,								\
+		.func = locked_wake_function,					\
+	    .link  = { &(name).link, &(name).link }			\
+	}
+
 extern void waitq_init(waitq_t *waitq);
 extern void waitq_init_entry(waitq_entry_t *entry, struct thread *proc);
 extern int waitq_active(waitq_t *waitq);

kernel/include/xpmem.h

@@ -0,0 +1,26 @@
+/**
+ * \file xpmem.h
+ *  License details are found in the file LICENSE.
+ * \brief
+ *  Structures and functions of xpmem
+ */
+/*
+ * HISTORY
+ */
+
+#ifndef _XPMEM_H
+#define _XPMEM_H
+
+#include <process.h>
+#include <ihk/context.h>
+
+#define XPMEM_DEV_PATH  "/dev/xpmem"
+
+extern int xpmem_open(ihk_mc_user_context_t *ctx);
+extern int xpmem_remove_process_memory_range(struct process_vm *vm, 
+	struct vm_range *vmr);
+extern int xpmem_fault_process_memory_range(struct process_vm *vm, 
+	struct vm_range *vmr, unsigned long vaddr, uint64_t reason);
+
+#endif /* _XPMEM_H */
+

kernel/include/xpmem_private.h

@@ -0,0 +1,490 @@
+/**
+ * \file xpmem_private.h
+ *  License details are found in the file LICENSE.
+ * \brief
+ *  Private Cross Partition Memory (XPMEM) structures and macros.
+ */
+/*
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * Copyright (c) 2004-2007 Silicon Graphics, Inc.  All Rights Reserved.
+ * Copyright 2009, 2010, 2014 Cray Inc. All Rights Reserved
+ * Copyright (c) 2014-2016 Los Alamos National Security, LCC. All rights
+ *                         reserved.
+ */
+/*
+ * HISTORY
+ */
+
+#ifndef _XPMEM_PRIVATE_H
+#define _XPMEM_PRIVATE_H
+
+#include <mc_xpmem.h>
+#include <xpmem.h>
+
+#define XPMEM_CURRENT_VERSION           0x00026003
+
+//#define DEBUG_PRINT_XPMEM
+
+#ifdef DEBUG_PRINT_XPMEM
+#define dkprintf(...) kprintf(__VA_ARGS__)
+#define ekprintf(...) kprintf(__VA_ARGS__)
+#define XPMEM_DEBUG(format, a...) kprintf("[%d] %s: "format"\n", cpu_local_var(current)->proc->rgid, __func__, ##a)
+#else
+#define dkprintf(...) do { if (0) kprintf(__VA_ARGS__); } while (0)
+#define ekprintf(...) kprintf(__VA_ARGS__)
+#define XPMEM_DEBUG(format, a...) do { if (0) kprintf("\n"); } while (0)
+#endif
+
+//#define USE_DBUG_ON
+
+#ifdef USE_DBUG_ON
+#define DBUG_ON(condition) do { if (condition) kprintf("[%d] BUG: func=%s\n", cpu_local_var(current)->proc->rgid, __func__); } while (0)
+#else
+#define DBUG_ON(condition)
+#endif
+
+#define offset_in_page(p)	((unsigned long)(p) & ~PAGE_MASK)
+
+#define min(x, y)	({                                              \
+	__typeof__(x) _min1 = (x);                                      \
+	__typeof__(y) _min2 = (y);                                      \
+	(void) (&_min1 == &_min2);                                      \
+	_min1 < _min2 ? _min1 : _min2;})
+
+#define max(x, y)	({                                              \
+	__typeof__(x) _max1 = (x);                                      \
+	__typeof__(y) _max2 = (y);                                      \
+	(void) (&_max1 == &_max2);                                      \
+	_max1 > _max2 ? _max1 : _max2;})
+
+#define MAX_ERRNO	4095
+
+#define IS_ERR_VALUE(x)	((x) >= (unsigned long)-MAX_ERRNO)
+
+static inline void * ERR_PTR(long error)
+{
+	return (void *)error;
+}
+
+static inline long PTR_ERR(const void *ptr)
+{
+	return (long)ptr;
+}
+
+static inline long IS_ERR(const void *ptr)
+{
+	return IS_ERR_VALUE((unsigned long)ptr);
+}
+
+static inline long IS_ERR_OR_NULL(const void *ptr)
+{
+	return !ptr || IS_ERR_VALUE((unsigned long)ptr);
+}
+
+/*
+ * Both the xpmem_segid_t and xpmem_apid_t are of type __s64 and designed
+ * to be opaque to the user. Both consist of the same underlying fields.
+ *
+ * The 'uniq' field is designed to give each segid or apid a unique value.
+ * Each type is only unique with respect to itself.
+ *
+ * An ID is never less than or equal to zero.
+ */
+struct xpmem_id {
+	pid_t tgid;		/* thread group that owns ID */
+	unsigned int uniq;	/* this value makes the ID unique */
+};
+
+typedef union {
+	struct xpmem_id xpmem_id;
+	xpmem_segid_t segid;
+	xpmem_apid_t apid;
+} xpmem_id_t;
+
+/* Shift INT_MAX by one so we can tell when we overflow. */
+#define XPMEM_MAX_UNIQ_ID	(INT_MAX >> 1)
+
+static inline pid_t xpmem_segid_to_tgid(xpmem_segid_t segid)
+{
+	DBUG_ON(segid <= 0);
+	return ((xpmem_id_t *)&segid)->xpmem_id.tgid;
+}
+
+static inline pid_t xpmem_apid_to_tgid(xpmem_apid_t apid)
+{
+	DBUG_ON(apid <= 0);
+	return ((xpmem_id_t *)&apid)->xpmem_id.tgid;
+}
+
+/*
+ * Hash Tables
+ *
+ * XPMEM utilizes hash tables to enable faster lookups of list entries.
+ * These hash tables are implemented as arrays. A simple modulus of the hash
+ * key yields the appropriate array index. A hash table's array element (i.e.,
+ * hash table bucket) consists of a hash list and the lock that protects it.
+ *
+ * XPMEM has the following two hash tables:
+ *
+ * table                bucket                                  key
+ * part->tg_hashtable   list of struct xpmem_thread_group       tgid
+ * tg->ap_hashtable     list of struct xpmem_access_permit      apid.uniq
+ */
+struct xpmem_hashlist {
+        mcs_rwlock_lock_t lock;	/* lock for hash list */
+        struct list_head list;	/* hash list */
+};
+
+#define XPMEM_TG_HASHTABLE_SIZE 8
+#define XPMEM_AP_HASHTABLE_SIZE 8
+
+static inline int xpmem_tg_hashtable_index(pid_t tgid)
+{
+	int index;
+
+	index = (unsigned int)tgid % XPMEM_TG_HASHTABLE_SIZE;
+
+	XPMEM_DEBUG("return: tgid=%lu, index=%d", tgid, index);
+
+	return index;
+}
+
+static inline int xpmem_ap_hashtable_index(xpmem_apid_t apid)
+{
+	int index;
+
+        DBUG_ON(apid <= 0);
+
+	index = ((xpmem_id_t *)&apid)->xpmem_id.uniq % XPMEM_AP_HASHTABLE_SIZE;
+
+	XPMEM_DEBUG("return: apid=0x%lx, index=%d", apid, index);
+
+	return index;
+}
+
+/*
+ * general internal driver structures
+ */
+struct xpmem_thread_group {
+	ihk_spinlock_t lock;	/* tg lock */
+	pid_t tgid;		/* tg's tgid */
+	uid_t uid;		/* tg's uid */
+	gid_t gid;		/* tg's gid */
+	volatile int flags;	/* tg attributes and state */
+	ihk_atomic_t uniq_segid;	/* segid uniq */
+	ihk_atomic_t uniq_apid;	/* apid uniq */
+	mcs_rwlock_lock_t seg_list_lock;	/* tg's list of segs lock */
+	struct list_head seg_list;	/* tg's list of segs */
+	ihk_atomic_t refcnt;	/* references to tg */
+	ihk_atomic_t n_pinned;	/* #of pages pinned by this tg */
+	struct list_head tg_hashlist;	/* tg hash list */
+	struct thread *group_leader;	/* thread group leader */
+	struct process_vm *vm;		/* tg's process_vm */
+	struct xpmem_hashlist ap_hashtable[];	/* locks + ap hash lists */
+};
+
+struct xpmem_segment {
+	ihk_spinlock_t lock;	/* seg lock */
+	xpmem_segid_t segid;	/* unique segid */
+	unsigned long vaddr;	/* starting address */
+	size_t size;		/* size of seg */
+	int permit_type;	/* permission scheme */
+	void *permit_value;	/* permission data */
+	volatile int flags;	/* seg attributes and state */
+	ihk_atomic_t refcnt;	/* references to seg */
+	struct xpmem_thread_group *tg;	/* creator tg */
+	struct list_head ap_list;	/* local access permits of seg */
+	struct list_head seg_list;	/* tg's list of segs */
+};
+
+struct xpmem_access_permit {
+	ihk_spinlock_t lock;	/* access permit lock */
+	xpmem_apid_t apid;	/* unique apid */
+	int mode;		/* read/write mode */
+	volatile int flags;	/* access permit attributes and state */
+	ihk_atomic_t refcnt;	/* references to access permit */
+	struct xpmem_segment *seg;	/* seg permitted to be accessed */
+	struct xpmem_thread_group *tg;	/* access permit's tg */
+	struct list_head att_list;	/* atts of this access permit's seg */
+	struct list_head ap_list;	/* access permits linked to seg */
+	struct list_head ap_hashlist;	/* access permit hash list */
+};
+
+struct xpmem_attachment {
+	mcs_rwlock_lock_t at_lock;	/* att lock */
+	unsigned long vaddr;	/* starting address of seg attached */
+	unsigned long at_vaddr;	/* address where seg is attached */
+	size_t at_size;		/* size of seg attachment */
+	struct vm_range *at_vmr;	/* vm_range where seg is attachment */
+	volatile int flags;	/* att attributes and state */
+	ihk_atomic_t refcnt;	/* references to att */
+	struct xpmem_access_permit *ap;	/* associated access permit */
+	struct list_head att_list;	/* atts linked to access permit */
+	struct process_vm *vm;	/* process_vm attached to */
+};
+
+struct xpmem_partition {
+	ihk_atomic_t n_opened;	/* # of /dev/xpmem opened */
+	struct xpmem_hashlist tg_hashtable[];	/* locks + tg hash lists */
+};
+
+#define XPMEM_FLAG_DESTROYING		0x00040 /* being destroyed */
+#define XPMEM_FLAG_DESTROYED		0x00080 /* 'being destroyed' finished */
+
+#define XPMEM_FLAG_VALIDPTEs		0x00200 /* valid PTEs exist */
+
+struct xpmem_perm {
+	uid_t uid;
+	gid_t gid;
+	unsigned long mode;
+};
+
+#define XPMEM_PERM_IRUSR 00400
+#define XPMEM_PERM_IWUSR 00200
+
+extern struct xpmem_partition *xpmem_my_part;
+
+static int xpmem_ioctl(struct mckfd *mckfd, ihk_mc_user_context_t *ctx);
+static int xpmem_close(struct mckfd *mckfd, ihk_mc_user_context_t *ctx);
+
+static int xpmem_init(void);
+static void xpmem_exit(void);
+static int __xpmem_open(void);
+static void xpmem_destroy_tg(struct xpmem_thread_group *);
+
+static int xpmem_make(unsigned long, size_t, int, void *, xpmem_segid_t *);
+static xpmem_segid_t xpmem_make_segid(struct xpmem_thread_group *);
+
+static int xpmem_remove(xpmem_segid_t);
+static void xpmem_remove_seg(struct xpmem_thread_group *,
+        struct xpmem_segment *);
+static void xpmem_remove_segs_of_tg(struct xpmem_thread_group *seg_tg);
+
+static int xpmem_get(xpmem_segid_t, int, int, void *, xpmem_apid_t *);
+static int xpmem_check_permit_mode(int, struct xpmem_segment *);
+static int xpmem_perms(struct xpmem_perm *, short);
+static xpmem_apid_t xpmem_make_apid(struct xpmem_thread_group *);
+
+static int xpmem_release(xpmem_apid_t);
+static void xpmem_release_ap(struct xpmem_thread_group *,
+	struct xpmem_access_permit *);
+static void xpmem_release_aps_of_tg(struct xpmem_thread_group *ap_tg);
+
+static int xpmem_attach(struct mckfd *, xpmem_apid_t, off_t, size_t, 
+	unsigned long, int, int, unsigned long *);
+
+static int xpmem_detach(unsigned long);
+static int xpmem_vm_munmap(struct process_vm *vm, void *addr, size_t len);
+static int xpmem_remove_process_range(struct process_vm *vm, 
+	unsigned long start, unsigned long end, int *ro_freedp);
+static int xpmem_free_process_memory_range(struct process_vm *vm,
+	struct vm_range *range);
+static void xpmem_detach_att(struct xpmem_access_permit *, 
+	struct xpmem_attachment *);
+static void xpmem_clear_PTEs(struct xpmem_segment *);
+static void xpmem_clear_PTEs_range(struct xpmem_segment *, unsigned long,
+	unsigned long);
+static void xpmem_clear_PTEs_of_ap(struct xpmem_access_permit *, unsigned long, 
+	unsigned long);
+static void xpmem_clear_PTEs_of_att(struct xpmem_attachment *, unsigned long, 
+	unsigned long);
+
+static int xpmem_remap_pte(struct process_vm *, struct vm_range *,
+	unsigned long, uint64_t, struct xpmem_segment *, unsigned long);
+
+static int xpmem_ensure_valid_page(struct xpmem_segment *, unsigned long);
+static pte_t * xpmem_vaddr_to_pte(struct process_vm *, unsigned long, 
+	size_t *pgsize);
+static int xpmem_pin_page(struct xpmem_thread_group *, struct thread *,
+	struct process_vm *, unsigned long);
+static void xpmem_unpin_pages(struct xpmem_segment *, struct process_vm *, 
+	unsigned long, size_t);
+
+static struct xpmem_thread_group * __xpmem_tg_ref_by_tgid_nolock_internal(
+	pid_t, int, int);
+
+static inline struct xpmem_thread_group *__xpmem_tg_ref_by_tgid(
+	pid_t tgid,
+	int return_destroying)
+{
+	struct xpmem_thread_group *tg;
+	int index;
+	struct mcs_rwlock_node_irqsave lock;
+
+	XPMEM_DEBUG("call: tgid=%d, return_destroying=%d", 
+		tgid, return_destroying);
+
+	index = xpmem_tg_hashtable_index(tgid);
+	mcs_rwlock_reader_lock(&xpmem_my_part->tg_hashtable[index].lock, &lock);
+	tg = __xpmem_tg_ref_by_tgid_nolock_internal(tgid, index, 
+		return_destroying);
+        mcs_rwlock_reader_unlock(&xpmem_my_part->tg_hashtable[index].lock, 
+		&lock);
+
+	XPMEM_DEBUG("return: tg=0x%p", tg);
+
+        return tg;
+}
+
+static inline struct xpmem_thread_group *__xpmem_tg_ref_by_tgid_nolock(
+	pid_t tgid,
+	int return_destroying)
+{
+	struct xpmem_thread_group *tg;
+
+	XPMEM_DEBUG("call: tgid=%d, return_destroying=%d", 
+		tgid, return_destroying);
+
+        tg = __xpmem_tg_ref_by_tgid_nolock_internal(tgid, 
+		xpmem_tg_hashtable_index(tgid), return_destroying);
+
+	XPMEM_DEBUG("return: tg=0x%p", tg);
+
+        return tg;
+}
+
+#define xpmem_tg_ref_by_tgid(t)             __xpmem_tg_ref_by_tgid(t, 0)
+#define xpmem_tg_ref_by_tgid_all(t)         __xpmem_tg_ref_by_tgid(t, 1)
+#define xpmem_tg_ref_by_tgid_nolock(t)      __xpmem_tg_ref_by_tgid_nolock(t, 0)
+#define xpmem_tg_ref_by_tgid_all_nolock(t)  __xpmem_tg_ref_by_tgid_nolock(t, 1)
+
+static struct xpmem_thread_group * xpmem_tg_ref_by_segid(xpmem_segid_t);
+static struct xpmem_thread_group * xpmem_tg_ref_by_apid(xpmem_apid_t);
+static void xpmem_tg_deref(struct xpmem_thread_group *);
+static struct xpmem_segment *xpmem_seg_ref_by_segid(struct xpmem_thread_group *,
+	xpmem_segid_t);
+static void xpmem_seg_deref(struct xpmem_segment *);
+static struct xpmem_access_permit * xpmem_ap_ref_by_apid(
+	struct xpmem_thread_group *, xpmem_apid_t);
+static void xpmem_ap_deref(struct xpmem_access_permit *);
+static void xpmem_att_deref(struct xpmem_attachment *);
+static int xpmem_validate_access(struct xpmem_access_permit *, off_t, size_t,
+	int, unsigned long *);
+
+/*
+ * Inlines that mark an internal driver structure as being destroyable or not.
+ * The idea is to set the refcnt to 1 at structure creation time and then
+ * drop that reference at the time the structure is to be destroyed.
+ */
+static inline void xpmem_tg_not_destroyable(
+	struct xpmem_thread_group *tg)
+{
+	ihk_atomic_set(&tg->refcnt, 1);
+
+	XPMEM_DEBUG("return: tg->refcnt=%d", tg->refcnt);
+}
+
+static inline void xpmem_tg_destroyable(
+	struct xpmem_thread_group *tg)
+{
+	XPMEM_DEBUG("call: ");
+
+	xpmem_tg_deref(tg);
+
+	XPMEM_DEBUG("return: ");
+}
+
+static inline void xpmem_seg_not_destroyable(
+	struct xpmem_segment *seg)
+{
+	ihk_atomic_set(&seg->refcnt, 1);
+
+	XPMEM_DEBUG("return: seg->refcnt=%d", seg->refcnt);
+}
+
+static inline void xpmem_seg_destroyable(
+	struct xpmem_segment *seg)
+{
+	XPMEM_DEBUG("call: ");
+
+	xpmem_seg_deref(seg);
+
+	XPMEM_DEBUG("return: ");
+}
+
+static inline void xpmem_ap_not_destroyable(
+	struct xpmem_access_permit *ap)
+{
+	ihk_atomic_set(&ap->refcnt, 1);
+
+	XPMEM_DEBUG("return: ap->refcnt=%d", ap->refcnt);
+}
+
+static inline void xpmem_ap_destroyable(
+	struct xpmem_access_permit *ap)
+{
+	XPMEM_DEBUG("call: ");
+
+	xpmem_ap_deref(ap);
+
+	XPMEM_DEBUG("return: ");
+}
+
+static inline void xpmem_att_not_destroyable(
+	struct xpmem_attachment *att)
+{
+	ihk_atomic_set(&att->refcnt, 1);
+
+	XPMEM_DEBUG("return: att->refcnt=%d", att->refcnt);
+}
+
+static inline void xpmem_att_destroyable(
+	struct xpmem_attachment *att)
+{
+	XPMEM_DEBUG("call: ");
+
+	xpmem_att_deref(att);
+
+	XPMEM_DEBUG("return: ");
+}
+
+/*
+ * Inlines that increment the refcnt for the specified structure.
+ */
+static inline void xpmem_tg_ref(
+	struct xpmem_thread_group *tg)
+{
+	DBUG_ON(ihk_atomic_read(&tg->refcnt) <= 0);
+	ihk_atomic_inc(&tg->refcnt);
+
+	XPMEM_DEBUG("return: tg->refcnt=%d", tg->refcnt);
+}
+
+static inline void xpmem_seg_ref(
+	struct xpmem_segment *seg)
+{
+	DBUG_ON(ihk_atomic_read(&seg->refcnt) <= 0);
+	ihk_atomic_inc(&seg->refcnt);
+
+	XPMEM_DEBUG("return: seg->refcnt=%d", seg->refcnt);
+}
+
+static inline void xpmem_ap_ref(
+	struct xpmem_access_permit *ap)
+{
+	DBUG_ON(ihk_atomic_read(&ap->refcnt) <= 0);
+	ihk_atomic_inc(&ap->refcnt);
+
+	XPMEM_DEBUG("return: ap->refcnt=%d", ap->refcnt);
+}
+
+static inline void xpmem_att_ref(
+	struct xpmem_attachment *att)
+{
+	DBUG_ON(ihk_atomic_read(&att->refcnt) <= 0);
+	ihk_atomic_inc(&att->refcnt);
+
+	XPMEM_DEBUG("return: att->refcnt=%d", att->refcnt);
+}
+
+static inline int xpmem_is_private_data(
+	struct vm_range *vmr)
+{
+        return (vmr->private_data != NULL);
+}
+
+#endif /* _XPMEM_PRIVATE_H */
+

kernel/init.c

@@ -31,6 +31,7 @@
 #include <cls.h>
 #include <syscall.h>
 #include <sysfs.h>
+#include <rusage.h>
 
 //#define IOCTL_FUNC_EXTENSION
 #ifdef IOCTL_FUNC_EXTENSION
@@ -108,11 +109,11 @@ static void dma_test(void)
 }
 #endif
 
-extern char *ihk_mc_get_kernel_args(void);
+extern char *ihk_get_kargs(void);
 
 char *find_command_line(char *name)
 {
-	char *cmdline = ihk_mc_get_kernel_args();
+	char *cmdline = ihk_get_kargs();
 
 	if (!cmdline) {
 		return NULL;
@@ -122,7 +123,7 @@ char *find_command_line(char *name)
 
 static void parse_kargs(void)
 {
-	kprintf("KCommand Line: %s\n", ihk_mc_get_kernel_args());
+	kprintf("KCommand Line: %s\n", ihk_get_kargs());
 
 	if (1) {
 		char *key = "osnum=";
@@ -239,6 +240,31 @@ static void time_init(void)
 	return;
 }
 
+struct ihk_os_monitor *monitor;
+
+static void monitor_init()
+{
+	int z;
+	unsigned long phys;
+
+	z = sizeof(struct ihk_os_monitor) * num_processors;
+	z = (z + PAGE_SIZE -1) >> PAGE_SHIFT;
+	monitor = ihk_mc_alloc_pages(z, IHK_MC_AP_CRITICAL);
+	memset(monitor, 0, z * PAGE_SIZE);
+	phys = virt_to_phys(monitor);
+	ihk_set_monitor(phys, sizeof(struct ihk_os_monitor) * num_processors);
+}
+
+int nmi_mode;
+
+static void nmi_init()
+{
+	unsigned long phys;
+
+	phys = virt_to_phys(&nmi_mode);
+	ihk_set_nmi_mode_addr(phys);
+}
+
 static void rest_init(void)
 {
 	handler_init();
@@ -250,11 +276,13 @@ static void rest_init(void)
 	//pc_test();
 
 	ap_init();
+	monitor_init();
 	cpu_local_var_init();
+	nmi_init();
 	time_init();
 	kmalloc_init();
 
-	ikc_master_init();
+	ihk_ikc_master_init();
 
 	proc_init();
 
@@ -320,6 +348,10 @@ static void setup_remote_snooping_samples(void)
 static void populate_sysfs(void)
 {
 	cpu_sysfs_setup();
+#ifdef ENABLE_RUSAGE
+	rusage_sysfs_setup();
+	status_sysfs_setup();
+#endif
 	//setup_remote_snooping_samples();
 } /* populate_sysfs() */
 
@@ -336,11 +368,8 @@ static void post_init(void)
 	}
 
 	if (find_command_line("hidos")) {
-		extern ihk_spinlock_t syscall_lock;
-
 		init_host_syscall_channel();
 		init_host_syscall_channel2();
-		ihk_mc_spinlock_init(&syscall_lock);
 	}
 
 	arch_setup_vdso();
@@ -364,6 +393,21 @@ int main(void)
 	char *ptr;
 	int mode = 0;
 
+#ifdef ENABLE_RUSAGE
+	int i;
+	os_status = IHK_STATUS_INACTIVE;
+	rusage_hugetlb_usage = 0;
+	rusage_hugetlb_max_usage = 0;
+	for (i = 0; i < sizeof(cpu_set_t)/8; i++) {
+		rusage_rss[i] = 0;
+	}
+	for (i = 0; i < 1024; i++) {
+		rusage_numa_stat[i] = 0;
+	}
+	rusage_rss_current = 0;
+	rusage_rss_max = 0;
+#endif
+
 	ptr = find_command_line("ksyslogd=");
 	if (ptr) {
 	    mode = ptr[9] - 0x30;
@@ -372,7 +416,10 @@ int main(void)
 	kmsg_init(mode);
 
 	kputs("IHK/McKernel started.\n");
-
+#ifdef ENABLE_RUSAGE
+	os_status = IHK_STATUS_BOOTING;
+#endif
+	ihk_set_kmsg(virt_to_phys(&kmsg_buf), IHK_KMSG_SIZE);
 	arch_init();
 
 	/*
@@ -394,6 +441,9 @@ int main(void)
 	futex_init();
 
 	kputs("IHK/McKernel booted.\n");
+#ifdef ENABLE_RUSAGE
+	os_status = IHK_STATUS_RUNNING;
+#endif
 
 #ifdef DCFA_KMOD
 	mc_cmd_client_init();

kernel/mem.c

@@ -37,6 +37,7 @@
 #include <cpulocal.h>
 #include <init.h>
 #include <cas.h>
+#include <rusage.h>
 
 //#define DEBUG_PRINT_MEM
 
@@ -49,7 +50,7 @@
 #endif
 
 static unsigned long pa_start, pa_end;
-static struct ihk_mc_numa_node *memory_nodes = NULL;
+static struct ihk_mc_numa_node memory_nodes[512];
 
 extern void unhandled_page_fault(struct thread *, void *, void *);
 extern int interrupt_from_user(void *);
@@ -65,12 +66,12 @@ extern void early_alloc_invalidate(void);
 
 static char *memdebug = NULL;
 
-static void *___kmalloc(int size, enum ihk_mc_ap_flag flag);
+static void *___kmalloc(int size, ihk_mc_ap_flag flag);
 static void ___kfree(void *ptr);
 
-static void *___ihk_mc_alloc_aligned_pages(int npages,
-		int p2align, enum ihk_mc_ap_flag flag);
-static void *___ihk_mc_alloc_pages(int npages, enum ihk_mc_ap_flag flag);
+static void *___ihk_mc_alloc_aligned_pages_node(int npages,
+		int p2align, ihk_mc_ap_flag flag, int node);
+static void *___ihk_mc_alloc_pages(int npages, ihk_mc_ap_flag flag);
 static void ___ihk_mc_free_pages(void *p, int npages);
 
 /*
@@ -151,14 +152,15 @@ struct pagealloc_track_entry *__pagealloc_track_find_entry(
 }
 
 /* Top level routines called from macros */
-void *_ihk_mc_alloc_aligned_pages(int npages, int p2align,
-	enum ihk_mc_ap_flag flag, char *file, int line)
+void *_ihk_mc_alloc_aligned_pages_node(int npages, int p2align,
+	ihk_mc_ap_flag flag, int node, char *file, int line)
 {
 	unsigned long irqflags;
 	struct pagealloc_track_entry *entry;
 	struct pagealloc_track_addr_entry *addr_entry;
 	int hash, addr_hash;
-	void *r = ___ihk_mc_alloc_aligned_pages(npages, p2align, flag);
+	void *r = ___ihk_mc_alloc_aligned_pages_node(npages,
+					p2align, flag, node);
 
 	if (!memdebug || !pagealloc_track_initialized)
 		return r;
@@ -230,12 +232,6 @@ out:
 	return r;
 }
 
-void *_ihk_mc_alloc_pages(int npages, enum ihk_mc_ap_flag flag,
-		char *file, int line)
-{
-	return _ihk_mc_alloc_aligned_pages(npages, PAGE_P2ALIGN, flag, file, line);
-}
-
 void _ihk_mc_free_pages(void *ptr, int npages, char *file, int line)
 {
 	unsigned long irqflags;
@@ -449,18 +445,18 @@ void pagealloc_memcheck(void)
 
 
 /* Actual allocation routines */
-static void *___ihk_mc_alloc_aligned_pages(int npages, int p2align,
-	enum ihk_mc_ap_flag flag)
+static void *___ihk_mc_alloc_aligned_pages_node(int npages, int p2align,
+	ihk_mc_ap_flag flag, int node)
 {
 	if (pa_ops)
-		return pa_ops->alloc_page(npages, p2align, flag);
+		return pa_ops->alloc_page(npages, p2align, flag, node);
 	else
 		return early_alloc_pages(npages);
 }
 
-static void *___ihk_mc_alloc_pages(int npages, enum ihk_mc_ap_flag flag)
+static void *___ihk_mc_alloc_pages(int npages, ihk_mc_ap_flag flag)
 {
-	return ___ihk_mc_alloc_aligned_pages(npages, PAGE_P2ALIGN, flag);
+	return ___ihk_mc_alloc_aligned_pages_node(npages, PAGE_P2ALIGN, flag, -1);
 }
 
 static void ___ihk_mc_free_pages(void *p, int npages)
@@ -494,21 +490,128 @@ static void reserve_pages(struct ihk_page_allocator_desc *pa_allocator,
 	ihk_pagealloc_reserve(pa_allocator, start, end);
 }
 
-static void *allocate_aligned_pages(int npages, int p2align, 
-		enum ihk_mc_ap_flag flag)
+extern int cpu_local_var_initialized;
+static void *mckernel_allocate_aligned_pages_node(int npages, int p2align,
+		ihk_mc_ap_flag flag, int pref_node)
 {
-	unsigned long pa;
-	int i;
+	unsigned long pa = 0;
+	int i, node;
+	struct ihk_page_allocator_desc *pa_allocator;
+
+	/* Not yet initialized or idle process */
+	if (!cpu_local_var_initialized ||
+			!cpu_local_var(current) ||
+			!cpu_local_var(current)->vm)
+		goto distance_based;
+
+	/* User requested policy? */
+	if (!(flag & IHK_MC_AP_USER)) {
+		goto distance_based;
+	}
+
+	node = ihk_mc_get_numa_id();
+	if (!memory_nodes[node].nodes_by_distance)
+		goto order_based;
+
+	switch (cpu_local_var(current)->vm->numa_mem_policy) {
+		case MPOL_BIND:
+		case MPOL_PREFERRED:
+
+			/* Look at nodes in the order of distance but consider
+			 * only the ones requested in user policy */
+			for (i = 0; i < ihk_mc_get_nr_numa_nodes(); ++i) {
+
+				/* Not part of user requested policy? */
+				if (!test_bit(memory_nodes[node].nodes_by_distance[i].id,
+						cpu_local_var(current)->proc->vm->numa_mask)) {
+					continue;
+				}
+
+				list_for_each_entry(pa_allocator,
+						&memory_nodes[memory_nodes[node].
+						nodes_by_distance[i].id].allocators, list) {
+					pa = ihk_pagealloc_alloc(pa_allocator, npages, p2align);
+
+					if (pa) {
+						dkprintf("%s: policy: CPU @ node %d allocated "
+								"%d pages from node %d\n",
+								__FUNCTION__,
+								ihk_mc_get_numa_id(),
+								npages, node);
+#ifdef ENABLE_RUSAGE
+						rusage_numa_stat[ihk_mc_get_numa_id()] += npages * PAGE_SIZE;
+#endif
+						break;
+					}
+				}
+
+				if (pa) break;
+			}
+			break;
+
+		case MPOL_INTERLEAVE:
+			/* TODO: */
+			break;
+
+		default:
+			break;
+	}
+
+	if (pa) {
+		return phys_to_virt(pa);
+	}
+	else {
+		dkprintf("%s: couldn't fulfill user policy for %d pages\n",
+			__FUNCTION__, npages);
+	}
+
+distance_based:
+	node = ihk_mc_get_numa_id();
+
+	/* Look at nodes in the order of distance */
+	if (!memory_nodes[node].nodes_by_distance)
+		goto order_based;
 
-	/* TODO: match NUMA id and distance matrix with allocating core */
 	for (i = 0; i < ihk_mc_get_nr_numa_nodes(); ++i) {
-		struct ihk_page_allocator_desc *pa_allocator;
 
 		list_for_each_entry(pa_allocator,
-				&memory_nodes[(ihk_mc_get_numa_id() + i) %
-				ihk_mc_get_nr_numa_nodes()].allocators, list) {
+				&memory_nodes[memory_nodes[node].
+				nodes_by_distance[i].id].allocators, list) {
 			pa = ihk_pagealloc_alloc(pa_allocator, npages, p2align);
 
+			if (pa) {
+				dkprintf("%s: distance: CPU @ node %d allocated "
+						"%d pages from node %d\n",
+						__FUNCTION__,
+						ihk_mc_get_numa_id(),
+						npages,
+						memory_nodes[node].nodes_by_distance[i].id);
+#ifdef ENABLE_RUSAGE
+				rusage_numa_stat[ihk_mc_get_numa_id()] += npages * PAGE_SIZE;
+#endif
+				break;
+			}
+		}
+
+		if (pa) break;
+	}
+
+	if (pa)
+		return phys_to_virt(pa);
+
+order_based:
+	node = ihk_mc_get_numa_id();
+
+	/* Fall back to regular order */
+	for (i = 0; i < ihk_mc_get_nr_numa_nodes(); ++i) {
+
+		list_for_each_entry(pa_allocator,
+				&memory_nodes[(node + i) %
+				ihk_mc_get_nr_numa_nodes()].allocators, list) {
+			pa = ihk_pagealloc_alloc(pa_allocator, npages, p2align);
+#ifdef ENABLE_RUSAGE
+			rusage_numa_stat[ihk_mc_get_numa_id()] += npages * PAGE_SIZE;
+#endif
 			if (pa) break;
 		}
 
@@ -524,12 +627,7 @@ static void *allocate_aligned_pages(int npages, int p2align,
 	return NULL;
 }
 
-static void *allocate_pages(int npages, enum ihk_mc_ap_flag flag)
-{
-	return allocate_aligned_pages(npages, PAGE_P2ALIGN, flag);
-}
-
-static void __free_pages_in_allocator(void *va, int npages)
+static void __mckernel_free_pages_in_allocator(void *va, int npages)
 {
 	int i;
 	unsigned long pa_start = virt_to_phys(va);
@@ -545,6 +643,9 @@ static void __free_pages_in_allocator(void *va, int npages)
 			if (pa_start >= pa_allocator->start &&
 					pa_end <= pa_allocator->end) {
 				ihk_pagealloc_free(pa_allocator, pa_start, npages);
+#ifdef ENABLE_RUSAGE
+				rusage_numa_stat[i] -= npages * PAGE_SIZE;
+#endif
 				return;
 			}
 		}
@@ -552,7 +653,7 @@ static void __free_pages_in_allocator(void *va, int npages)
 }
 
 
-static void free_pages(void *va, int npages)
+static void mckernel_free_pages(void *va, int npages)
 {
 	struct list_head *pendings = &cpu_local_var(pending_free_pages);
 	struct page *page;
@@ -560,7 +661,8 @@ static void free_pages(void *va, int npages)
 	page = phys_to_page(virt_to_phys(va));
 	if (page) {
 		if (page->mode != PM_NONE) {
-			panic("free_pages:not PM_NONE");
+			kprintf("%s: WARNING: page phys 0x%lx is not PM_NONE",
+					__FUNCTION__, page->phys);
 		}
 		if (pendings->next != NULL) {
 			page->mode = PM_PENDING_FREE;
@@ -570,7 +672,7 @@ static void free_pages(void *va, int npages)
 		}
 	}
 
-	__free_pages_in_allocator(va, npages);
+	__mckernel_free_pages_in_allocator(va, npages);
 }
 
 void begin_free_pages_pending(void) {
@@ -599,7 +701,7 @@ void finish_free_pages_pending(void)
 		}
 		page->mode = PM_NONE;
 		list_del(&page->list);
-		__free_pages_in_allocator(phys_to_virt(page_to_phys(page)), 
+		__mckernel_free_pages_in_allocator(phys_to_virt(page_to_phys(page)),
 				page->offset);
 	}
 
@@ -608,8 +710,8 @@ void finish_free_pages_pending(void)
 }
 
 static struct ihk_mc_pa_ops allocator = {
-	.alloc_page = allocate_aligned_pages,
-	.free_page = free_pages,
+	.alloc_page = mckernel_allocate_aligned_pages_node,
+	.free_page = mckernel_free_pages,
 };
 
 void sbox_write(int offset, unsigned int value);
@@ -754,6 +856,8 @@ void remote_flush_tlb_cpumask(struct process_vm *vm,
 			flush_tlb();
 		}
 
+		/* Flush on this core */
+		flush_tlb_single(addr & PAGE_MASK);
 		/* Wait for all cores */
 		while (ihk_atomic_read(&flush_entry->pending) != 0) {
 			cpu_pause();
@@ -804,8 +908,8 @@ static void page_fault_handler(void *fault_addr, uint64_t reason, void *regs)
 	int error;
 
 	set_cputime(interrupt_from_user(regs)? 1: 2);
-	dkprintf("[%d]page_fault_handler(%p,%lx,%p)\n",
-			ihk_mc_get_processor_id(), fault_addr, reason, regs);
+	dkprintf("%s: addr: %p, reason: %lx, regs: %p\n",
+			__FUNCTION__, fault_addr, reason, regs);
 
 	preempt_disable();
 
@@ -860,21 +964,22 @@ static void page_fault_handler(void *fault_addr, uint64_t reason, void *regs)
 	error = 0;
 	preempt_enable();
 out:
-	dkprintf("[%d]page_fault_handler(%p,%lx,%p): (%d)\n",
-			ihk_mc_get_processor_id(), fault_addr, reason,
-			regs, error);
+	dkprintf("%s: addr: %p, reason: %lx, regs: %p -> error: %d\n",
+			__FUNCTION__, fault_addr, reason, regs, error);
 	check_need_resched();
 	set_cputime(0);
 	return;
 }
 
 static struct ihk_page_allocator_desc *page_allocator_init(uint64_t start, 
-		uint64_t end, int initial)
+		uint64_t end)
 {
 	struct ihk_page_allocator_desc *pa_allocator;
 	unsigned long page_map_pa, pages;
 	void *page_map;
 	unsigned int i;
+	extern char _end[];
+	unsigned long phys_end = virt_to_phys(_end);
 
 	start &= PAGE_MASK;
 	pa_start = (start + PAGE_SIZE - 1) & PAGE_MASK;
@@ -887,7 +992,12 @@ static struct ihk_page_allocator_desc *page_allocator_init(uint64_t start,
 	*/
 	page_map_pa = 0x100000;
 #else
-	page_map_pa = initial ? virt_to_phys(get_last_early_heap()) : pa_start;
+	if (pa_start <= phys_end && phys_end <= pa_end) {
+		page_map_pa = virt_to_phys(get_last_early_heap());
+	}
+	else {
+		page_map_pa = pa_start;
+	}
 #endif
 
 	page_map = phys_to_virt(page_map_pa);
@@ -918,18 +1028,21 @@ static struct ihk_page_allocator_desc *page_allocator_init(uint64_t start,
 static void numa_init(void)
 {
 	int i, j;
-	memory_nodes = early_alloc_pages((sizeof(*memory_nodes) * 
-				ihk_mc_get_nr_numa_nodes() + PAGE_SIZE - 1) 
-			>> PAGE_SHIFT);
 
 	for (i = 0; i < ihk_mc_get_nr_numa_nodes(); ++i) {
 		int linux_numa_id, type;
 
-		ihk_mc_get_numa_node(i, &linux_numa_id, &type);
+		if (ihk_mc_get_numa_node(i, &linux_numa_id, &type) != 0) {
+			kprintf("%s: error: obtaining NUMA info for node %d\n",
+					__FUNCTION__, i);
+			panic("");
+		}
+
 		memory_nodes[i].id = i;
 		memory_nodes[i].linux_numa_id = linux_numa_id;
 		memory_nodes[i].type = type;
 		INIT_LIST_HEAD(&memory_nodes[i].allocators);
+		memory_nodes[i].nodes_by_distance = 0;
 
 		kprintf("NUMA: %d, Linux NUMA: %d, type: %d\n",
 			i, linux_numa_id, type);
@@ -942,7 +1055,7 @@ static void numa_init(void)
 
 		ihk_mc_get_memory_chunk(j, &start, &end, &numa_id);
 
-		allocator = page_allocator_init(start, end, (j == 0));
+		allocator = page_allocator_init(start, end);
 		list_add_tail(&allocator->list, &memory_nodes[numa_id].allocators);
 
 		kprintf("Physical memory: 0x%lx - 0x%lx, %lu bytes, %d pages available @ NUMA: %d\n",
@@ -950,6 +1063,75 @@ static void numa_init(void)
 				ihk_pagealloc_count(allocator) * PAGE_SIZE,
 				ihk_pagealloc_count(allocator),
 				numa_id);
+#ifdef ENABLE_RUSAGE
+				rusage_max_memory = ihk_pagealloc_count(allocator) * PAGE_SIZE;
+#endif
+	}
+}
+
+static void numa_distances_init()
+{
+	int i, j, swapped;
+
+	for (i = 0; i < ihk_mc_get_nr_numa_nodes(); ++i) {
+		/* TODO: allocate on target node */
+		memory_nodes[i].nodes_by_distance =
+			ihk_mc_alloc_pages((sizeof(struct node_distance) *
+						ihk_mc_get_nr_numa_nodes() + PAGE_SIZE - 1)
+					>> PAGE_SHIFT, IHK_MC_AP_NOWAIT);
+
+		if (!memory_nodes[i].nodes_by_distance) {
+			kprintf("%s: error: allocating nodes_by_distance\n",
+				__FUNCTION__);
+			continue;
+		}
+
+		for (j = 0; j < ihk_mc_get_nr_numa_nodes(); ++j) {
+			memory_nodes[i].nodes_by_distance[j].id = j;
+			memory_nodes[i].nodes_by_distance[j].distance =
+				ihk_mc_get_numa_distance(i, j);
+		}
+
+		/* Sort by distance and node ID */
+		swapped = 1;
+		while (swapped) {
+			swapped = 0;
+			for (j = 1; j < ihk_mc_get_nr_numa_nodes(); ++j) {
+				if ((memory_nodes[i].nodes_by_distance[j - 1].distance >
+							memory_nodes[i].nodes_by_distance[j].distance) ||
+						((memory_nodes[i].nodes_by_distance[j - 1].distance ==
+						  memory_nodes[i].nodes_by_distance[j].distance) &&
+						 (memory_nodes[i].nodes_by_distance[j - 1].id >
+						  memory_nodes[i].nodes_by_distance[j].id))) {
+					memory_nodes[i].nodes_by_distance[j - 1].id ^=
+						memory_nodes[i].nodes_by_distance[j].id;
+					memory_nodes[i].nodes_by_distance[j].id ^=
+						memory_nodes[i].nodes_by_distance[j - 1].id;
+					memory_nodes[i].nodes_by_distance[j - 1].id ^=
+						memory_nodes[i].nodes_by_distance[j].id;
+
+					memory_nodes[i].nodes_by_distance[j - 1].distance ^=
+						memory_nodes[i].nodes_by_distance[j].distance;
+					memory_nodes[i].nodes_by_distance[j].distance ^=
+						memory_nodes[i].nodes_by_distance[j - 1].distance;
+					memory_nodes[i].nodes_by_distance[j - 1].distance ^=
+						memory_nodes[i].nodes_by_distance[j].distance;
+					swapped = 1;
+				}
+			}
+		}
+		{
+			char buf[1024];
+			char *pbuf = buf;
+
+			pbuf += sprintf(pbuf, "NUMA %d distances: ", i);
+			for (j = 0; j < ihk_mc_get_nr_numa_nodes(); ++j) {
+				pbuf += sprintf(pbuf, "%d (%d), ",
+						memory_nodes[i].nodes_by_distance[j].id,
+						memory_nodes[i].nodes_by_distance[j].distance);
+			}
+			kprintf("%s\n", buf);
+		}
 	}
 }
 
@@ -1234,6 +1416,9 @@ void mem_init(void)
 		kprintf("Demand paging on ANONYMOUS mappings enabled.\n");
 		anon_on_demand = 1;
 	}
+	
+	/* Init distance vectors */
+	numa_distances_init();
 }
 
 #define KMALLOC_TRACK_HASH_SHIFT	(8)
@@ -1323,7 +1508,7 @@ struct kmalloc_track_entry *__kmalloc_track_find_entry(
 }
 
 /* Top level routines called from macro */
-void *_kmalloc(int size, enum ihk_mc_ap_flag flag, char *file, int line)
+void *_kmalloc(int size, ihk_mc_ap_flag flag, char *file, int line)
 {
 	unsigned long irqflags;
 	struct kmalloc_track_entry *entry;
@@ -1513,7 +1698,7 @@ void kmalloc_memcheck(void)
 }
 
 /* Redirection routines registered in alloc structure */
-void *__kmalloc(int size, enum ihk_mc_ap_flag flag)
+void *__kmalloc(int size, ihk_mc_ap_flag flag)
 {
 	return kmalloc(size, flag);
 }
@@ -1611,7 +1796,7 @@ void kmalloc_consolidate_free_list(void)
 #define KMALLOC_MIN_MASK    (KMALLOC_MIN_SIZE - 1)
 
 /* Actual low-level allocation routines */
-static void *___kmalloc(int size, enum ihk_mc_ap_flag flag)
+static void *___kmalloc(int size, ihk_mc_ap_flag flag)
 {
 	struct kmalloc_header *chunk_iter;
 	struct kmalloc_header *chunk = NULL;

kernel/mikc.c

@@ -21,7 +21,7 @@ static struct ihk_ikc_channel_desc *mchannel;
 static int arch_master_channel_packet_handler(struct ihk_ikc_channel_desc *,
                                          void *__packet, void *arg);
 
-void ikc_master_init(void)
+void ihk_ikc_master_init(void)
 {
 	mchannel = kmalloc(sizeof(struct ihk_ikc_channel_desc) +
 	                   sizeof(struct ihk_ikc_master_packet),

kernel/process.c

@@ -31,6 +31,8 @@
 #include <auxvec.h>
 #include <timer.h>
 #include <mman.h>
+#include <xpmem.h>
+#include <rusage.h>
 
 //#define DEBUG_PRINT_PROCESS
 
@@ -65,6 +67,7 @@ extern void procfs_create_thread(struct thread *);
 extern void procfs_delete_thread(struct thread *);
 extern void perf_start(struct mc_perf_event *event);
 extern void perf_reset(struct mc_perf_event *event);
+extern void event_signal();
 
 struct list_head resource_set_list;
 mcs_rwlock_lock_t    resource_set_lock;
@@ -74,7 +77,6 @@ init_process(struct process *proc, struct process *parent)
 {
 	/* These will be filled out when changing status */
 	proc->pid = -1;
-	proc->exit_status = -1;
 	proc->status = PS_RUNNING;
 
 	if(parent){
@@ -102,6 +104,13 @@ init_process(struct process *proc, struct process *parent)
 	waitq_init(&proc->waitpid_q);
 	ihk_atomic_set(&proc->refcount, 2);
 	proc->monitoring_event = NULL;
+#ifdef TRACK_SYSCALLS
+	mcs_lock_init(&proc->st_lock);
+	proc->syscall_times = NULL;
+	proc->syscall_cnts = NULL;
+	proc->offload_times = NULL;
+	proc->offload_cnts = NULL;
+#endif
 }
 
 void
@@ -234,13 +243,15 @@ init_process_vm(struct process *owner, struct address_space *asp, struct process
 	return 0;
 }
 
-struct thread *
-create_thread(unsigned long user_pc)
+struct thread *create_thread(unsigned long user_pc,
+		unsigned long *__cpu_set, size_t cpu_set_size)
 {
 	struct thread *thread;
 	struct process *proc;
 	struct process_vm *vm = NULL;
 	struct address_space *asp = NULL;
+	int cpu;
+	int cpu_set_empty = 1;
 
 	thread = ihk_mc_alloc_pages(KERNEL_STACK_NR_PAGES, IHK_MC_AP_NOWAIT);
 	if (!thread)
@@ -256,7 +267,22 @@ create_thread(unsigned long user_pc)
 	memset(vm, 0, sizeof(struct process_vm));
 	init_process(proc, cpu_local_var(resource_set)->pid1);
 
-	if (1) {
+	/* Use requested CPU cores */
+	for_each_set_bit(cpu, __cpu_set, cpu_set_size * BITS_PER_BYTE) {
+		if (cpu >= num_processors) {
+			kprintf("%s: invalid CPU requested in initial cpu_set\n",
+				__FUNCTION__);
+			goto err;
+		}
+
+		dkprintf("%s: pid: %d, CPU: %d\n",
+			__FUNCTION__, proc->pid, cpu); 
+		CPU_SET(cpu, &thread->cpu_set);
+		cpu_set_empty = 0;
+	}
+
+	/* Default allows all cores */
+	if (cpu_set_empty) {
 		struct ihk_mc_cpu_info *infop;
 		int i;
 
@@ -278,10 +304,10 @@ create_thread(unsigned long user_pc)
 	dkprintf("fork(): sigshared\n");
 
 	ihk_atomic_set(&thread->sigcommon->use, 1);
-	ihk_mc_spinlock_init(&thread->sigcommon->lock);
+	mcs_rwlock_init(&thread->sigcommon->lock);
 	INIT_LIST_HEAD(&thread->sigcommon->sigpending);
 
-	ihk_mc_spinlock_init(&thread->sigpendinglock);
+	mcs_rwlock_init(&thread->sigpendinglock);
 	INIT_LIST_HEAD(&thread->sigpending);
 
 	thread->sigstack.ss_sp = NULL;
@@ -298,13 +324,32 @@ create_thread(unsigned long user_pc)
 	if(init_process_vm(proc, asp, vm) != 0){
 		goto err;
 	}
+	thread->exit_status = -1;
 
 	cpu_set(ihk_mc_get_processor_id(), &thread->vm->address_space->cpu_set,
 			&thread->vm->address_space->cpu_set_lock);
 
 	ihk_mc_spinlock_init(&thread->spin_sleep_lock);
 	thread->spin_sleep = 0;
-
+#ifdef ENABLE_RUSAGE
+	{
+		int processor_id;
+		unsigned long curr;
+		processor_id = ihk_mc_get_processor_id();
+		rusage_rss[processor_id] += KERNEL_STACK_NR_PAGES * PAGE_SIZE;
+		curr = ihk_atomic_add_long_return ( KERNEL_STACK_NR_PAGES * PAGE_SIZE, &rusage_rss_current);
+		if (rusage_rss_max < curr) {
+			atomic_cmpxchg8(&rusage_rss_max, rusage_rss_max, curr);
+		}
+		if (rusage_max_memory - curr < RUSAGE_MEM_LIMIT) {
+			event_signal();
+		}
+		ihk_atomic_add_ulong ( 1, &rusage_num_threads);
+		if (rusage_max_num_threads < rusage_num_threads) {
+			atomic_cmpxchg8(&rusage_max_num_threads, rusage_max_num_threads, rusage_num_threads);
+		}
+	}
+#endif
 	return thread;
 
 err:
@@ -441,17 +486,43 @@ clone_thread(struct thread *org, unsigned long pc, unsigned long sp,
 		memcpy(thread->sigcommon->action, org->sigcommon->action,
 		       sizeof(struct k_sigaction) * _NSIG);
 		ihk_atomic_set(&thread->sigcommon->use, 1);
-		ihk_mc_spinlock_init(&thread->sigcommon->lock);
+		mcs_rwlock_init(&thread->sigcommon->lock);
 		INIT_LIST_HEAD(&thread->sigcommon->sigpending);
 		// TODO: copy signalfd
 	}
-	ihk_mc_spinlock_init(&thread->sigpendinglock);
+	mcs_rwlock_init(&thread->sigpendinglock);
 	INIT_LIST_HEAD(&thread->sigpending);
 	thread->sigmask = org->sigmask;
 
 	ihk_mc_spinlock_init(&thread->spin_sleep_lock);
 	thread->spin_sleep = 0;
 
+#ifdef ENABLE_RUSAGE
+	{
+		int processor_id;
+		long curr;
+		processor_id = ihk_mc_get_processor_id();
+		rusage_rss[processor_id] += KERNEL_STACK_NR_PAGES * PAGE_SIZE;
+		curr = ihk_atomic_add_long_return (KERNEL_STACK_NR_PAGES * PAGE_SIZE, &rusage_rss_current);
+		if (rusage_rss_max < curr) {
+			atomic_cmpxchg8(&rusage_rss_max, rusage_rss_max, curr);
+		}
+		if (rusage_max_memory - curr < RUSAGE_MEM_LIMIT) {
+			event_signal();
+		}
+
+		ihk_atomic_add_ulong ( 1, &rusage_num_threads);
+
+		if (rusage_max_num_threads < rusage_num_threads) {
+			atomic_cmpxchg8(&rusage_max_num_threads, rusage_max_num_threads, rusage_num_threads);
+		}
+	}
+#endif
+
+#ifdef TRACK_SYSCALLS
+	thread->track_syscalls = org->track_syscalls;
+#endif
+
 	return thread;
 
 err_free_proc:
@@ -610,6 +681,7 @@ static int copy_user_ranges(struct process_vm *vm, struct process_vm *orgvm)
 		range->memobj = src_range->memobj;
 		range->objoff = src_range->objoff;
 		range->pgshift = src_range->pgshift;
+		range->private_data = src_range->private_data;
 		if (range->memobj) {
 			memobj_ref(range->memobj);
 		}
@@ -707,6 +779,7 @@ int split_process_memory_range(struct process_vm *vm, struct vm_range *range,
 	newrange->end = range->end;
 	newrange->flag = range->flag;
 	newrange->pgshift = range->pgshift;
+	newrange->private_data = range->private_data;
 
 	if (range->memobj) {
 		memobj_ref(range->memobj);
@@ -926,6 +999,10 @@ int remove_process_memory_range(struct process_vm *vm,
 			ro_freed = 1;
 		}
 
+		if (freerange->private_data) {
+			xpmem_remove_process_memory_range(vm, freerange);
+		}
+
 		error = free_process_memory_range(vm, freerange);
 		if (error) {
 			ekprintf("remove_process_memory_range(%p,%lx,%lx):"
@@ -1001,21 +1078,18 @@ enum ihk_mc_pt_attribute common_vrflag_to_ptattr(unsigned long flag, uint64_t fa
 }
 
 int add_process_memory_range(struct process_vm *vm,
-                             unsigned long start, unsigned long end,
-                             unsigned long phys, unsigned long flag,
-			     struct memobj *memobj, off_t offset,
-			     int pgshift)
+		unsigned long start, unsigned long end,
+		unsigned long phys, unsigned long flag,
+		struct memobj *memobj, off_t offset,
+		int pgshift, struct vm_range **rp)
 {
 	struct vm_range *range;
 	int rc;
-#if 0
-	extern void __host_update_process_range(struct thread *process,
-						struct vm_range *range);
-#endif
 
 	if ((start < vm->region.user_start)
 			|| (vm->region.user_end < end)) {
-		kprintf("range(%#lx - %#lx) is not in user avail(%#lx - %#lx)\n",
+		kprintf("%s: error: range %lx - %lx is not in user available area\n",
+				__FUNCTION__,
 				start, end, vm->region.user_start,
 				vm->region.user_end);
 		return -EINVAL;
@@ -1023,9 +1097,10 @@ int add_process_memory_range(struct process_vm *vm,
 
 	range = kmalloc(sizeof(struct vm_range), IHK_MC_AP_NOWAIT);
 	if (!range) {
-		kprintf("ERROR: allocating pages for range\n");
+		kprintf("%s: ERROR: allocating pages for range\n", __FUNCTION__);
 		return -ENOMEM;
 	}
+
 	INIT_LIST_HEAD(&range->list);
 	range->start = start;
 	range->end = end;
@@ -1033,49 +1108,36 @@ int add_process_memory_range(struct process_vm *vm,
 	range->memobj = memobj;
 	range->objoff = offset;
 	range->pgshift = pgshift;
+	range->private_data = NULL;
 
-    if(range->flag & VR_DEMAND_PAGING) {
-	dkprintf("range: 0x%lX - 0x%lX => physicall memory area is allocated on demand (%ld) [%lx]\n",
-	        range->start, range->end, range->end - range->start,
-		range->flag);
-    } else {
-		dkprintf("range: 0x%lX - 0x%lX (%ld) [%lx]\n",
-				range->start, range->end, range->end - range->start,
-				range->flag);
-    }
-
-    rc = 0;
-    if (0) {
-	    /* dummy */
-    }
-    else if (phys == NOPHYS) {
-	    /* nothing to map */
-    }
-    else if (flag & VR_REMOTE) {
+	rc = 0;
+	if (phys == NOPHYS) {
+		/* Nothing to map */
+	}
+	else if (flag & VR_REMOTE) {
 		rc = update_process_page_table(vm, range, phys, IHK_PTA_REMOTE);
-	} else if (flag & VR_IO_NOCACHE) {
+	}
+	else if (flag & VR_IO_NOCACHE) {
 		rc = update_process_page_table(vm, range, phys, PTATTR_UNCACHABLE);
-	} else if(flag & VR_DEMAND_PAGING){
-	  //demand paging no need to update process table now
-	  dkprintf("demand paging do not update process page table\n");
-      rc = 0;
-	} else if ((range->flag & VR_PROT_MASK) == VR_PROT_NONE) {
+	}
+	else if (flag & VR_DEMAND_PAGING) {
+		dkprintf("%s: range: 0x%lx - 0x%lx is demand paging\n",
+				__FUNCTION__, range->start, range->end);
 		rc = 0;
-	} else {
+	}
+	else if ((range->flag & VR_PROT_MASK) == VR_PROT_NONE) {
+		rc = 0;
+	}
+	else {
 		rc = update_process_page_table(vm, range, phys, 0);
 	}
-	if(rc != 0){
-		kprintf("ERROR: preparing page tables\n");
+
+	if (rc != 0) {
+		kprintf("%s: ERROR: preparing page tables\n", __FUNCTION__);
 		kfree(range);
 		return rc;
 	}
 
-#if 0 // disable __host_update_process_range() in add_process_memory_range(), because it has no effect on the actual mapping on the MICs side.
-	if (!(flag & VR_REMOTE)) {
-		__host_update_process_range(process, range);
-	}
-#endif
-
 	insert_vm_range_list(vm, range);
 
 	/* Clear content! */
@@ -1084,6 +1146,11 @@ int add_process_memory_range(struct process_vm *vm,
 		memset((void*)phys_to_virt(phys), 0, end - start);
 	}
 
+	/* Return range object if requested */
+	if (rp) {
+		*rp = range;
+	}
+
 	return 0;
 }
 
@@ -1367,6 +1434,11 @@ static int sync_one_page(void *arg0, page_table_t pt, pte_t *ptep,
 	flush_tlb_single((uintptr_t)pgaddr);	/* XXX: TLB flush */
 
 	phys = pte_get_phys(ptep);
+	if (args->memobj->flags & MF_ZEROFILL) {
+		error = 0;
+		goto out;
+	}
+
 	error = memobj_flush_page(args->memobj, phys, pgsize);
 	if (error) {
 		ekprintf("sync_one_page(%p,%p,%p %#lx,%p,%d):"
@@ -1394,11 +1466,19 @@ int sync_process_memory_range(struct process_vm *vm, struct vm_range *range,
 	args.memobj = range->memobj;
 
 	ihk_mc_spinlock_lock_noirq(&vm->page_table_lock);
-	memobj_lock(range->memobj);
+
+	if (!(range->memobj->flags & MF_ZEROFILL)) {
+		memobj_lock(range->memobj);
+	}
+
 	error = visit_pte_range(vm->address_space->page_table, (void *)start,
-	                        (void *)end, range->pgshift, VPTEF_SKIP_NULL,
-				&sync_one_page, &args);
-	memobj_unlock(range->memobj);
+			(void *)end, range->pgshift, VPTEF_SKIP_NULL,
+			&sync_one_page, &args);
+
+	if (!(range->memobj->flags & MF_ZEROFILL)) {
+		memobj_unlock(range->memobj);
+	}
+
 	ihk_mc_spinlock_unlock_noirq(&vm->page_table_lock);
 	if (error) {
 		ekprintf("sync_process_memory_range(%p,%p,%#lx,%#lx):"
@@ -1690,10 +1770,9 @@ static int do_page_fault_process_vm(struct process_vm *vm, void *fault_addr0, ui
 	range = lookup_process_memory_range(vm, fault_addr, fault_addr+1);
 	if (range == NULL) {
 		error = -EFAULT;
-		dkprintf("[%d]do_page_fault_process_vm(%p,%lx,%lx):"
-				"out of range. %d\n",
-				ihk_mc_get_processor_id(), vm,
-				fault_addr0, reason, error);
+		dkprintf("do_page_fault_process_vm(): vm: %p, addr: %p, reason: %lx):"
+				"out of range: %d\n",
+				vm, fault_addr0, reason, error);
 		goto out;
 	}
 
@@ -1723,10 +1802,18 @@ static int do_page_fault_process_vm(struct process_vm *vm, void *fault_addr0, ui
 			kprintf("if (((range->flag & VR_PROT_MASK) == VR_PROT_NONE))\n");
 		if (((reason & PF_WRITE) && !(reason & PF_PATCH)))
 			kprintf("if (((reason & PF_WRITE) && !(reason & PF_PATCH)))\n");
-		if (!(range->flag & VR_PROT_WRITE))
+		if (!(range->flag & VR_PROT_WRITE)) {
 			kprintf("if (!(range->flag & VR_PROT_WRITE))\n");
-		if ((reason & PF_INSTR) && !(range->flag & VR_PROT_EXEC))
+			//kprintf("setting VR_PROT_WRITE\n");
+			//range->flag |= VR_PROT_WRITE;
+			//goto cont;
+		}
+		if ((reason & PF_INSTR) && !(range->flag & VR_PROT_EXEC)) {
 			kprintf("if ((reason & PF_INSTR) && !(range->flag & VR_PROT_EXEC))\n");
+			//kprintf("setting VR_PROT_EXEC\n");
+			//range->flag |= VR_PROT_EXEC;
+			//goto cont;
+		}
 		goto out;
 	}
 
@@ -1757,7 +1844,12 @@ static int do_page_fault_process_vm(struct process_vm *vm, void *fault_addr0, ui
 		}
 	}
 
-	error = page_fault_process_memory_range(vm, range, fault_addr, reason);
+	if (!range->private_data) {
+		error = page_fault_process_memory_range(vm, range, fault_addr, reason);
+	}
+	else {
+		error = xpmem_fault_process_memory_range(vm, range, fault_addr, reason);
+	}
 	if (error == -ERESTART) {
 		goto out;
 	}
@@ -1815,6 +1907,7 @@ int init_process_stack(struct thread *thread, struct program_load_desc *pn,
 	unsigned long minsz;
 	unsigned long at_rand;
 	struct process *proc = thread->proc;
+	unsigned long __flag;
 
 	/* create stack range */
 	end = STACK_TOP(&thread->vm->region);
@@ -1833,12 +1926,15 @@ int init_process_stack(struct thread *thread, struct program_load_desc *pn,
 	vrflag |= VR_MAXPROT_READ | VR_MAXPROT_WRITE | VR_MAXPROT_EXEC;
 #define	NOPHYS	((uintptr_t)-1)
 	if ((rc = add_process_memory_range(thread->vm, start, end, NOPHYS,
-					vrflag, NULL, 0, PAGE_SHIFT)) != 0) {
+					vrflag, NULL, 0, PAGE_SHIFT, NULL)) != 0) {
 		return rc;
 	}
 
+	__flag = (size >= 16777216) ? IHK_MC_AP_USER : 0;
 	/* map physical pages for initial stack frame */
-	stack = ihk_mc_alloc_pages(minsz >> PAGE_SHIFT, IHK_MC_AP_NOWAIT);
+	stack = ihk_mc_alloc_pages(minsz >> PAGE_SHIFT,
+			IHK_MC_AP_NOWAIT | __flag);
+
 	if (!stack) {
 		return -ENOMEM;
 	}
@@ -1911,6 +2007,24 @@ int init_process_stack(struct thread *thread, struct program_load_desc *pn,
 	                           end + sizeof(unsigned long) * s_ind);
 	thread->vm->region.stack_end = end;
 	thread->vm->region.stack_start = start;
+
+#ifdef ENABLE_RUSAGE
+{
+	int processor_id;
+	long curr;
+
+	processor_id = ihk_mc_get_processor_id();
+	rusage_rss[processor_id] += (minsz >> PAGE_SHIFT) * PAGE_SIZE;
+	curr = ihk_atomic_add_long_return ((minsz >> PAGE_SHIFT) * PAGE_SIZE, &rusage_rss_current);
+	if (rusage_rss_max < curr) {
+		atomic_cmpxchg8(&rusage_rss_max, rusage_rss_max, curr);
+	}
+	if (rusage_max_memory - curr < RUSAGE_MEM_LIMIT) {
+		event_signal();
+	}
+}
+#endif
+
 	return 0;
 }
 
@@ -1952,7 +2066,7 @@ unsigned long extend_process_region(struct process_vm *vm,
 			}
 			if((rc = add_process_memory_range(vm, old_aligned_end,
                                         aligned_end, virt_to_phys(p), flag,
-					LARGE_PAGE_SHIFT)) != 0){
+					LARGE_PAGE_SHIFT, NULL)) != 0){
 				ihk_mc_free_pages(p, (aligned_end - old_aligned_end) >> PAGE_SHIFT);
 				return end;
 			}
@@ -1982,7 +2096,7 @@ unsigned long extend_process_region(struct process_vm *vm,
 
 		if((rc = add_process_memory_range(vm, aligned_end,
                                aligned_new_end, virt_to_phys((void *)p_aligned),
-                               flag, LARGE_PAGE_SHIFT)) != 0){
+                               flag, LARGE_PAGE_SHIFT, NULL)) != 0){
 			ihk_mc_free_pages(p, (aligned_new_end - aligned_end + LARGE_PAGE_SIZE) >> PAGE_SHIFT);
 			return end;
 		}
@@ -2001,19 +2115,34 @@ unsigned long extend_process_region(struct process_vm *vm,
 	  p=0;
 	}else{
 
-	p = ihk_mc_alloc_pages((aligned_new_end - aligned_end) >> PAGE_SHIFT, IHK_MC_AP_NOWAIT);
+	p = ihk_mc_alloc_pages((aligned_new_end - aligned_end) >> PAGE_SHIFT,
+		IHK_MC_AP_NOWAIT | IHK_MC_AP_USER);
 
 	if (!p) {
 		return end;
 	}
     }
-	if((rc = add_process_memory_range(vm, aligned_end, aligned_new_end,
-                                      (p==0?0:virt_to_phys(p)), flag, NULL, 0,
-				      PAGE_SHIFT)) != 0){
+	if ((rc = add_process_memory_range(vm, aligned_end, aligned_new_end,
+					(p == 0 ? 0 : virt_to_phys(p)), flag, NULL, 0,
+					PAGE_SHIFT, NULL)) != 0) {
 		ihk_mc_free_pages(p, (aligned_new_end - aligned_end) >> PAGE_SHIFT);
 		return end;
 	}
-
+#ifdef ENABLE_RUSAGE
+{
+	int processor_id;
+	long curr;
+	processor_id = ihk_mc_get_processor_id();
+	rusage_rss[processor_id] += ((aligned_new_end - aligned_end) >> PAGE_SHIFT) * PAGE_SIZE;
+	curr = ihk_atomic_add_long_return (((aligned_new_end - aligned_end) >> PAGE_SHIFT) * PAGE_SIZE, &rusage_rss_current);
+	if (rusage_rss_max < curr) {
+		atomic_cmpxchg8(&rusage_rss_max, rusage_rss_max, curr);
+	}
+	if (rusage_max_memory - curr < RUSAGE_MEM_LIMIT) {
+		event_signal();
+	}
+}
+#endif
 	return address;
 }
 
@@ -2124,6 +2253,10 @@ release_process(struct process *proc)
 	}
 
 	if (proc->tids) kfree(proc->tids);
+#ifdef TRACK_SYSCALLS
+	track_syscalls_print_proc_stats(proc);
+	track_syscalls_dealloc_proc_counters(proc);
+#endif // TRACK_SYSCALLS
 	kfree(proc);
 }
 
@@ -2141,6 +2274,9 @@ free_all_process_memory_range(struct process_vm *vm)
 
 	ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
 	list_for_each_entry_safe(range, next, &vm->vm_range_list, list) {
+		if (range->memobj) {
+			range->memobj->flags |= MF_HOST_RELEASED;
+		}
 		error = free_process_memory_range(vm, range);
 		if (error) {
 			ekprintf("free_process_memory(%p):"
@@ -2161,6 +2297,19 @@ release_process_vm(struct process_vm *vm)
 		return;
 	}
 
+	{
+		long irqstate;
+		struct mckfd *fdp;
+
+		irqstate = ihk_mc_spinlock_lock(&proc->mckfd_lock);
+		for (fdp = proc->mckfd; fdp; fdp = fdp->next) {
+			if (fdp->close_cb) {
+				fdp->close_cb(fdp, NULL);
+			}
+		}
+		ihk_mc_spinlock_unlock(&proc->mckfd_lock, irqstate);
+	}
+
 	if(vm->free_cb)
 		vm->free_cb(vm, vm->opt);
 
@@ -2184,9 +2333,10 @@ int populate_process_memory(struct process_vm *vm, void *start, size_t len)
 	for (addr = (uintptr_t)start; addr < end; addr += PAGE_SIZE) {
 		error = page_fault_process_vm(vm, (void *)addr, reason);
 		if (error) {
-			ekprintf("populate_process_range:page_fault_process_vm"
-					"(%p,%lx,%lx) failed %d\n",
-					vm, addr, reason, error);
+			ekprintf("%s: WARNING: page_fault_process_vm(): vm: %p, "
+					"addr: %lx, reason: %lx, off: %lu, len: %lu returns %d\n",
+					__FUNCTION__, vm, addr, reason,
+					((void *)addr - start), len, error);
 			goto out;
 		}
 	}
@@ -2287,6 +2437,16 @@ void destroy_thread(struct thread *thread)
 
 	release_sigcommon(thread->sigcommon);
 
+#ifdef ENABLE_RUSAGE
+{
+	int processor_id;
+	processor_id = ihk_mc_get_processor_id();
+	rusage_rss[processor_id] -= KERNEL_STACK_NR_PAGES * PAGE_SIZE;
+	ihk_atomic_add_long_return(KERNEL_STACK_NR_PAGES * PAGE_SIZE * (-1) , &rusage_rss_current);
+	ihk_atomic_add_ulong ( -1, &rusage_num_threads);
+}
+#endif
+
 	ihk_mc_free_pages(thread, KERNEL_STACK_NR_PAGES);
 }
 
@@ -2294,18 +2454,26 @@ void release_thread(struct thread *thread)
 {
 	struct process_vm *vm;
 	struct mcs_rwlock_node lock;
+	struct timespec ats;
 
 	if (!ihk_atomic_dec_and_test(&thread->refcount)) {
 		return;
 	}
 
 	mcs_rwlock_writer_lock_noirq(&thread->proc->update_lock, &lock);
-	ts_add(&thread->proc->stime, &thread->stime);
-	ts_add(&thread->proc->utime, &thread->utime);
+	tsc_to_ts(thread->system_tsc, &ats);
+	ts_add(&thread->proc->stime, &ats);
+	tsc_to_ts(thread->user_tsc, &ats);
+	ts_add(&thread->proc->utime, &ats);
 	mcs_rwlock_writer_unlock_noirq(&thread->proc->update_lock, &lock);
 
 	vm = thread->vm;
 
+#ifdef TRACK_SYSCALLS
+	track_syscalls_accumulate_counters(thread, thread->proc);
+	//track_syscalls_print_thread_stats(thread);
+	track_syscalls_dealloc_thread_counters(thread);
+#endif // TRACK_SYSCALLS
 	procfs_delete_thread(thread);
 	destroy_thread(thread);
 
@@ -2314,7 +2482,7 @@ void release_thread(struct thread *thread)
 
 void cpu_set(int cpu, cpu_set_t *cpu_set, ihk_spinlock_t *lock)
 {
-	unsigned int flags;
+	unsigned long flags;
 	flags = ihk_mc_spinlock_lock(lock);
 	CPU_SET(cpu, cpu_set);
 	ihk_mc_spinlock_unlock(lock, flags);
@@ -2322,7 +2490,7 @@ void cpu_set(int cpu, cpu_set_t *cpu_set, ihk_spinlock_t *lock)
 
 void cpu_clear(int cpu, cpu_set_t *cpu_set, ihk_spinlock_t *lock)
 {
-	unsigned int flags;
+	unsigned long flags;
 	flags = ihk_mc_spinlock_lock(lock);
 	CPU_CLR(cpu, cpu_set);
 	ihk_mc_spinlock_unlock(lock, flags);
@@ -2331,7 +2499,7 @@ void cpu_clear(int cpu, cpu_set_t *cpu_set, ihk_spinlock_t *lock)
 void cpu_clear_and_set(int c_cpu, int s_cpu,
 	cpu_set_t *cpu_set, ihk_spinlock_t *lock)
 {
-	unsigned int flags;
+	unsigned long flags;
 	flags = ihk_mc_spinlock_lock(lock);
 	CPU_CLR(c_cpu, cpu_set);
 	CPU_SET(s_cpu, cpu_set);
@@ -2344,6 +2512,7 @@ static void do_migrate(void);
 static void idle(void)
 {
 	struct cpu_local_var *v = get_this_cpu_local_var();
+	struct ihk_os_monitor *monitor = v->monitor;
 
 	/* Release runq_lock before starting the idle loop.
 	 * See comments at release_runq_lock().
@@ -2356,7 +2525,9 @@ static void idle(void)
 	cpu_enable_interrupt();
 
 	while (1) {
+		cpu_local_var(current)->status = PS_STOPPED;
 		schedule();
+		cpu_local_var(current)->status = PS_RUNNING;
 		cpu_disable_interrupt();
 
 		/* See if we need to migrate a process somewhere */
@@ -2402,7 +2573,12 @@ static void idle(void)
 		    v->status == CPU_STATUS_RESERVED) {
 			/* No work to do? Consolidate the kmalloc free list */
 			kmalloc_consolidate_free_list();
+			monitor->status = IHK_OS_MONITOR_IDLE;
+			cpu_local_var(current)->status = PS_INTERRUPTIBLE;
 			cpu_safe_halt();
+			monitor->status = IHK_OS_MONITOR_KERNEL;
+			monitor->counter++;
+			cpu_local_var(current)->status = PS_RUNNING;
 		}
 		else {
 			cpu_enable_interrupt();
@@ -2599,13 +2775,13 @@ static void do_migrate(void)
 		                  &req->thread->vm->address_space->cpu_set,
 		                  &req->thread->vm->address_space->cpu_set_lock);
 
-		dkprintf("do_migrate(): migrated TID %d from CPU %d to CPU %d\n",
-			req->thread->tid, old_cpu_id, cpu_id);
+		dkprintf("%s: migrated TID %d from CPU %d to CPU %d\n",
+			__FUNCTION__, req->thread->tid, old_cpu_id, cpu_id);
 		
 		v->flags |= CPU_FLAG_NEED_RESCHED;
-		ihk_mc_interrupt_cpu(get_x86_cpu_local_variable(cpu_id)->apic_id, 0xd1);
+		waitq_wakeup(&req->wq);
 		double_rq_unlock(cur_v, v, irqstate);
-
+		continue;
 ack:
 		waitq_wakeup(&req->wq);
 	}
@@ -2632,6 +2808,65 @@ set_timer()
 	}
 }
 
+/*
+ * NOTE: it is assumed that a wait-queue (or futex queue) is
+ * set before calling this function.
+ * NOTE: one must set thread->spin_sleep to 1 before evaluating
+ * the wait condition to avoid lost wake-ups.
+ */
+void spin_sleep_or_schedule(void)
+{
+	struct thread *thread = cpu_local_var(current);
+	struct cpu_local_var *v;
+	int do_schedule = 0;
+	int woken = 0;
+	long irqstate;
+
+	/* Try to spin sleep */
+	irqstate = ihk_mc_spinlock_lock(&thread->spin_sleep_lock);
+	if (thread->spin_sleep == 0) {
+		dkprintf("%s: caught a lost wake-up!\n", __FUNCTION__);
+	}
+	ihk_mc_spinlock_unlock(&thread->spin_sleep_lock, irqstate);
+
+	for (;;) {
+		/* Check if we need to reschedule */
+		irqstate =
+			ihk_mc_spinlock_lock(&(get_this_cpu_local_var()->runq_lock));
+		v = get_this_cpu_local_var();
+
+		if (v->flags & CPU_FLAG_NEED_RESCHED || v->runq_len > 1) {
+			do_schedule = 1;
+		}
+
+		ihk_mc_spinlock_unlock(&v->runq_lock, irqstate);
+
+		/* Check if we were woken up */
+		irqstate = ihk_mc_spinlock_lock(&thread->spin_sleep_lock);
+		if (thread->spin_sleep == 0) {
+			woken = 1;
+		}
+
+		/* Indicate that we are not spinning any more */
+		if (do_schedule) {
+			thread->spin_sleep = 0;
+		}
+		ihk_mc_spinlock_unlock(&thread->spin_sleep_lock, irqstate);
+
+		if (woken) {
+			return;
+		}
+
+		if (do_schedule) {
+			break;
+		}
+
+		cpu_pause();
+	}
+
+	schedule();
+}
+
 void schedule(void)
 {
 	struct cpu_local_var *v;
@@ -2717,7 +2952,9 @@ redo:
 			restore_fp_regs(next);
 		}
 
-		ihk_mc_load_page_table(next->vm->address_space->page_table);
+		if (prev && prev->vm->address_space->page_table !=
+				next->vm->address_space->page_table)
+			ihk_mc_load_page_table(next->vm->address_space->page_table);
 
 		dkprintf("[%d] schedule: tlsblock_base: 0x%lX\n",
 		         ihk_mc_get_processor_id(), next->tlsblock_base);
@@ -2792,39 +3029,38 @@ void check_need_resched(void)
 	}
 }
 
-int
-sched_wakeup_thread(struct thread *thread, int valid_states)
+int __sched_wakeup_thread(struct thread *thread,
+		int valid_states, int runq_locked)
 {
 	int status;
-	int spin_slept = 0;
 	unsigned long irqstate;
 	struct cpu_local_var *v = get_cpu_local_var(thread->cpu_id);
 	struct process *proc = thread->proc;
 	struct mcs_rwlock_node updatelock;
 
-	dkprintf("sched_wakeup_process,proc->pid=%d,valid_states=%08x,proc->status=%08x,proc->cpu_id=%d,my cpu_id=%d\n",
-			 proc->pid, valid_states, thread->status, thread->cpu_id, ihk_mc_get_processor_id());
+	dkprintf("%s: proc->pid=%d, valid_states=%08x, "
+			"proc->status=%08x, proc->cpu_id=%d,my cpu_id=%d\n",
+			__FUNCTION__,
+			proc->pid, valid_states, thread->status,
+			thread->cpu_id, ihk_mc_get_processor_id());
 
 	irqstate = ihk_mc_spinlock_lock(&(thread->spin_sleep_lock));
-	if (thread->spin_sleep > 0) {
-		dkprintf("sched_wakeup_process() spin wakeup: cpu_id: %d\n",
-				 thread->cpu_id);
+	if (thread->spin_sleep == 1) {
+		dkprintf("%s: spin wakeup: cpu_id: %d\n",
+				__FUNCTION__, thread->cpu_id);
 
-		spin_slept = 1;
 		status = 0;
 	}
-	--thread->spin_sleep;
+	thread->spin_sleep = 0;
 	ihk_mc_spinlock_unlock(&(thread->spin_sleep_lock), irqstate);
 
-	if (spin_slept) {
-		return status;
+	if (!runq_locked) {
+		irqstate = ihk_mc_spinlock_lock(&(v->runq_lock));
 	}
 
-	irqstate = ihk_mc_spinlock_lock(&(v->runq_lock));
-
 	if (thread->status & valid_states) {
 		mcs_rwlock_writer_lock_noirq(&proc->update_lock, &updatelock);
-		if(proc->status != PS_EXITED)
+		if (proc->status != PS_EXITED)
 			proc->status = PS_RUNNING;
 		mcs_rwlock_writer_unlock_noirq(&proc->update_lock, &updatelock);
 		xchg4((int *)(&thread->status), PS_RUNNING);
@@ -2834,18 +3070,32 @@ sched_wakeup_thread(struct thread *thread, int valid_states)
 		status = -EINVAL;
 	}
 
-	ihk_mc_spinlock_unlock(&(v->runq_lock), irqstate);
+	if (!runq_locked) {
+		ihk_mc_spinlock_unlock(&(v->runq_lock), irqstate);
+	}
 
 	if (!status && (thread->cpu_id != ihk_mc_get_processor_id())) {
-		dkprintf("sched_wakeup_process,issuing IPI,thread->cpu_id=%d\n",
-				 thread->cpu_id);
-		ihk_mc_interrupt_cpu(get_x86_cpu_local_variable(thread->cpu_id)->apic_id,
-		                     0xd1);
+		dkprintf("%s: issuing IPI, thread->cpu_id=%d\n",
+				__FUNCTION__, thread->cpu_id);
+		ihk_mc_interrupt_cpu(
+				get_x86_cpu_local_variable(thread->cpu_id)->apic_id,
+				0xd1);
 	}
 
 	return status;
 }
 
+int sched_wakeup_thread_locked(struct thread *thread, int valid_states)
+{
+	return __sched_wakeup_thread(thread, valid_states, 1);
+}
+
+int sched_wakeup_thread(struct thread *thread, int valid_states)
+{
+	return __sched_wakeup_thread(thread, valid_states, 0);
+}
+
+
 /*
  * 1. Add current process to waitq
  * 2. Queue migration request into the target CPU's queue
@@ -2869,7 +3119,7 @@ void sched_request_migrate(int cpu_id, struct thread *thread)
 	struct cpu_local_var *v = get_cpu_local_var(cpu_id);
 	struct migrate_request req = { .thread = thread };
 	unsigned long irqstate;
-	DECLARE_WAITQ_ENTRY(entry, cpu_local_var(current));
+	DECLARE_WAITQ_ENTRY_LOCKED(entry, cpu_local_var(current));
 
 	waitq_init(&req.wq);
 	waitq_prepare_to_wait(&req.wq, &entry, PS_UNINTERRUPTIBLE);
@@ -2886,6 +3136,8 @@ void sched_request_migrate(int cpu_id, struct thread *thread)
 	if (cpu_id != ihk_mc_get_processor_id())
 		ihk_mc_interrupt_cpu(/* Kick scheduler */
 				get_x86_cpu_local_variable(cpu_id)->apic_id, 0xd1);
+	dkprintf("%s: tid: %d -> cpu: %d\n",
+			__FUNCTION__, thread->tid, cpu_id);
 
 	schedule();
 	waitq_finish_wait(&req.wq, &entry);
@@ -2949,6 +3201,7 @@ find_thread(int pid, int tid, struct mcs_rwlock_node_irqsave *lock)
 	if(tid <= 0)
 		return NULL;
 	mcs_rwlock_reader_lock(&thash->lock[hash], lock);
+retry:
 	list_for_each_entry(thread, &thash->list[hash], hash_list){
 		if(thread->tid == tid){
 			if(pid <= 0)
@@ -2957,6 +3210,13 @@ find_thread(int pid, int tid, struct mcs_rwlock_node_irqsave *lock)
 				return thread;
 		}
 	}
+	/* If no thread with pid == tid was found, then we may be looking for a
+	 * specific thread (not the main thread of the process), try to find it
+	 * based on tid only */
+	if (pid > 0 && pid == tid) {
+		pid = 0;
+		goto retry;
+	}
 	mcs_rwlock_reader_unlock(&thash->lock[hash], lock);
 	return NULL;
 }

kernel/procfs.c

@@ -17,13 +17,14 @@
 #include <ihk/debug.h>
 #include <ihk/ikc.h>
 #include <ikc/master.h>
-#include <syscall.h>
 #include <cls.h>
+#include <syscall.h>
 #include <kmalloc.h>
 #include <process.h>
 #include <page.h>
 #include <mman.h>
 #include <bitmap.h>
+#include <init.h>
 
 //#define DEBUG_PRINT_PROCFS
 
@@ -75,11 +76,11 @@ procfs_delete_thread(struct thread *thread)
  *
  * \param rarg returned argument
  */
-void
-process_procfs_request(unsigned long rarg)
+void process_procfs_request(struct ikc_scd_packet *rpacket)
 {
+	unsigned long rarg = rpacket->arg;
 	unsigned long parg, pbuf;
-        struct thread *thread = NULL;
+	struct thread *thread = NULL;
 	struct process *proc = NULL;
 	struct process_vm *vm = NULL;
 	struct procfs_read *r;
@@ -160,7 +161,7 @@ process_procfs_request(unsigned long rarg)
 	 */
 	ret = sscanf(p, "%d/", &pid);
 	if (ret == 1) {
-		struct mcs_rwlock_node tlock;
+		struct mcs_rwlock_node_irqsave tlock;
 		int tids;
 		struct thread *thread1 = NULL;
 
@@ -177,7 +178,7 @@ process_procfs_request(unsigned long rarg)
 		else
 			tid = pid;
 
-		mcs_rwlock_reader_lock_noirq(&proc->threads_lock, &tlock);
+		mcs_rwlock_reader_lock(&proc->threads_lock, &tlock);
 		list_for_each_entry(thread, &proc->threads_list, siblings_list){
 			if(thread->tid == tid)
 				break;
@@ -187,15 +188,15 @@ process_procfs_request(unsigned long rarg)
 		if(thread == NULL){
 			kprintf("process_procfs_request: no such tid %d-%d\n", pid, tid);
 			if(tids){
+				mcs_rwlock_reader_unlock(&proc->threads_lock, &tlock);
 				process_unlock(proc, &lock);
-				mcs_rwlock_reader_unlock_noirq(&proc->threads_lock, &tlock);
 				goto end;
 			}
 			thread = thread1;
 		}
 		if(thread)
 			hold_thread(thread);
-		mcs_rwlock_reader_unlock_noirq(&proc->threads_lock, &tlock);
+		mcs_rwlock_reader_unlock(&proc->threads_lock, &tlock);
 		hold_process(proc);
 		vm = proc->vm;
 		if(vm)
@@ -632,6 +633,7 @@ dataunavail:
 
 	packet.msg = SCD_MSG_PROCFS_ANSWER;
 	packet.arg = rarg;
+	packet.pid = rpacket->pid;
 
 	ret = ihk_ikc_send(syscall_channel, &packet, 0);
 	if (ret < 0) {

kernel/rusage.c

@@ -0,0 +1,389 @@
+/**
+ * \file rusage.c
+ */
+#include <types.h>
+#include <kmsg.h>
+#include <ihk/cpu.h>
+#include <ihk/mm.h>
+#include <ihk/debug.h>
+#include <process.h>
+#include <init.h>
+#include <march.h>
+#include <cls.h>
+#include <time.h>
+#include <syscall.h>
+#include <string.h>
+#include <rusage.h>
+
+//#define DEBUG_PRINT_AP
+
+#ifdef DEBUG_PRINT_AP
+#define dkprintf(...) kprintf(__VA_ARGS__)
+#define ekprintf(...) kprintf(__VA_ARGS__)
+#else
+#define dkprintf(...) do { if (0) kprintf(__VA_ARGS__); } while (0)
+#define ekprintf(...) kprintf(__VA_ARGS__)
+#endif
+
+extern int num_processors ;
+static volatile int ap_stop = 1;
+
+mcs_lock_node_t ap_syscall_semaphore;
+
+extern struct ihk_os_monitor *monitor;
+
+
+#ifdef ENABLE_RUSAGE
+/* count total rss */
+unsigned long count_rss () {
+	int i;
+	unsigned long val = 0;
+	for(i = 0; i < sizeof(cpu_set_t)/8; i++){
+		val += rusage_rss[i];
+	}
+	return val;
+}
+
+/* count total cache */
+unsigned long count_cache () {
+   return 0;
+}
+
+/* count total rss_huge */
+unsigned long count_rss_huge () {
+	return 0;
+}
+
+/* count total mapped_file */
+unsigned long count_mapped_file () {
+	return 0;
+}
+
+/* count total max_usage */
+unsigned long count_max_usage() {
+	return rusage_rss_max;
+}
+
+/* count total kmem_usage */
+unsigned long count_kmem_usage() {
+	return 0;
+}
+
+/* count total kmax_usage */
+unsigned long count_kmem_max_usage() {
+	return 0;
+}
+
+#endif 
+
+#include <sysfs.h>
+#include <vsprintf.h>
+
+#ifdef ENABLE_RUSAGE
+char* strcat_rusage(char *s1, char *s2) {
+	int i;
+	int j;
+	for (i = 0; s1[i] != '\0'; i++); //skip chars.
+	for (j = 0; s2[j] != '\0'; j++) {
+		s1[i+j] = s2[j];
+	}
+	s1[i+j] = '\0';
+	return s1;	
+}
+
+static ssize_t
+show_rusage_memory_data(struct sysfs_ops *ops, void *instance, void *buf, size_t size)
+{
+	return snprintf(buf, size, "rss %lu\ncache %lu\nrss_huge %lu\nmapped_file %lu\n",
+			count_rss(),
+			count_cache(),
+			count_rss_huge(),
+			count_mapped_file()
+	);
+}
+
+static ssize_t 
+show_rusage_memory_max_usage_data(struct sysfs_ops *ops, void *instance, void *buf, size_t size)
+{
+	return snprintf(buf,size,"%lu\n",count_max_usage());
+}
+
+static ssize_t 
+show_rusage_memory_kmem_usage_data(struct sysfs_ops *ops, void *instance, void *buf, size_t size)
+{
+	return snprintf(buf,size,"%lu\n",count_kmem_usage());
+}
+
+static ssize_t 
+show_rusage_memory_kmem_max_usage_data(struct sysfs_ops *ops, void *instance, void *buf, size_t size)
+{
+	return snprintf(buf,size,"%lu\n",count_kmem_max_usage());
+}
+
+static ssize_t 
+show_rusage_num_numa_nodes_data(struct sysfs_ops *ops, void *instance, void *buf, size_t size)
+{
+	return snprintf(buf,size,"%d\n",ihk_mc_get_nr_numa_nodes());
+}
+
+static ssize_t 
+show_rusage_memory_numa_stat_data(struct sysfs_ops *ops, void *instance, void *buf, size_t size)
+{
+	int i;
+	int num_numa;
+	char tmp_buf1[1024];
+	char tmp_buf2[1024];
+	unsigned long total = 0;
+	memset(tmp_buf1, 0, 1024);
+	num_numa = ihk_mc_get_nr_numa_nodes();
+
+	for (i = 0; i < num_numa; i++) {
+		total += rusage_numa_stat[i];
+	}
+	sprintf(tmp_buf1, "total=%lu ", total);
+
+	for (i = 0; i < num_numa; i++) {
+		sprintf(tmp_buf2, "N%d=%lu ", i, rusage_numa_stat[i]);
+		strcat_rusage(tmp_buf1, tmp_buf2);
+		memset(tmp_buf2, 0, 1024);
+	}
+	return snprintf(buf, size, "%s\n", tmp_buf1);
+}
+
+static ssize_t 
+show_rusage_hugetlb_usage_data(struct sysfs_ops *ops, void *instance, void *buf, size_t size)
+{
+	return snprintf(buf, size, "%lu\n", rusage_hugetlb_usage);
+}
+
+static ssize_t 
+show_rusage_hugetlb_max_usage_data(struct sysfs_ops *ops, void *instance, void *buf, size_t size)
+{
+	return snprintf(buf, size, "%lu\n", rusage_hugetlb_max_usage);
+}
+static ssize_t 
+show_rusage_cpuacct_stat_data(struct sysfs_ops *ops, void *instance, void *buf, size_t size)
+{
+	struct timespec uts;
+	struct timespec sts;
+	int i;
+	int r = 0;
+
+	uts.tv_sec = 0;
+	uts.tv_nsec = 0;
+	sts.tv_sec = 0;
+	sts.tv_nsec = 0;
+	if (monitor)
+		for (i = 0; i < num_processors; i++) {
+			struct timespec ats;
+
+			tsc_to_ts(monitor[i].user_tsc, &ats);
+			ts_add(&uts, &ats);
+			tsc_to_ts(monitor[i].system_tsc, &ats);
+			ts_add(&sts, &ats);
+		}
+	r = snprintf(buf, size, "user %lu\n", timespec_to_jiffy(&uts));
+	r += snprintf(strchr(buf, '\0'), size - r, "system %lu\n",
+				  timespec_to_jiffy(&sts));
+	return r;
+}
+static ssize_t 
+show_rusage_cpuacct_usage_data(struct sysfs_ops *ops, void *instance, void *buf, size_t size)
+{
+	struct timespec uts;
+	int i;
+	int r = 0;
+
+	uts.tv_sec = 0;
+	uts.tv_nsec = 0;
+	if (monitor)
+		for (i = 0; i < num_processors; i++) {
+			struct timespec ats;
+
+			tsc_to_ts(monitor[i].user_tsc + monitor[i].system_tsc,
+					  &ats);
+			ts_add(&uts, &ats);
+		}
+	if (uts.tv_sec)
+		r = snprintf(buf, size, "%lu%09lu\n", uts.tv_sec, uts.tv_nsec);
+	else
+		r = snprintf(buf, size, "%lu\n", uts.tv_nsec);
+	return r;
+}
+
+
+static ssize_t 
+show_rusage_cpuacct_usage_percpu_data(struct sysfs_ops *ops, void *instance, void *buf, size_t size)
+{
+	struct timespec uts;
+	int i;
+	int r = 0;
+
+	((char *)buf)[0] = '\0';
+	for (i = 0; i < num_processors; i++) {
+		if (monitor) {
+			tsc_to_ts(monitor[i].user_tsc + monitor[i].system_tsc,
+					  &uts);
+		}
+		else {
+			uts.tv_sec = 0;
+			uts.tv_nsec = 0;
+		}
+		if (uts.tv_sec)
+			r += snprintf(strchr(buf, '\0'), size - r,
+						  "%lu%09lu ", uts.tv_sec, uts.tv_nsec);
+		else
+			r += snprintf(strchr(buf, '\0'), size - r,
+						  "%lu ", uts.tv_nsec);
+	}
+	((char *)buf)[r - 1] = '\n';
+	return r;
+}
+
+/* callback funciton of rusage(threads) sysfs */
+static ssize_t
+show_rusage_num_threads_data(struct sysfs_ops *ops, void *instance, void *buf, size_t size)
+{
+	return snprintf(buf, size, "%lu\n", rusage_num_threads);
+}
+
+/* callback funciton of rusage(max threads) sysfs */
+static ssize_t
+show_rusage_max_num_threads_data(struct sysfs_ops *ops, void *instance, void *buf, size_t size)
+{
+	return snprintf(buf, size, "%lu\n", rusage_max_num_threads);
+}
+
+/* definition of sysfs ops */
+struct sysfs_ops show_rusage_memory = {
+	.show = &show_rusage_memory_data,
+};
+struct sysfs_ops show_rusage_memory_max_usage = {
+	.show = &show_rusage_memory_max_usage_data,
+};
+struct sysfs_ops show_rusage_memory_kmem_usage = {
+	.show = &show_rusage_memory_kmem_usage_data,
+};
+struct sysfs_ops show_rusage_memory_kmem_max_usage = {
+	.show = &show_rusage_memory_kmem_max_usage_data,
+};
+struct sysfs_ops show_rusage_num_numa_nodes = {
+	.show = &show_rusage_num_numa_nodes_data,
+};
+struct sysfs_ops show_rusage_memory_numa_stat = {
+	.show = &show_rusage_memory_numa_stat_data,
+};
+struct sysfs_ops show_rusage_hugetlb_usage = {
+	.show = &show_rusage_hugetlb_usage_data,
+};
+struct sysfs_ops show_rusage_hugetlb_max_usage = {
+	.show = &show_rusage_hugetlb_max_usage_data,
+};
+struct sysfs_ops show_rusage_cpuacct_stat = {
+	.show = &show_rusage_cpuacct_stat_data,
+};
+struct sysfs_ops show_rusage_cpuacct_usage = {
+	.show = &show_rusage_cpuacct_usage_data,
+};
+struct sysfs_ops show_rusage_num_threads = {
+	.show = &show_rusage_num_threads_data,
+};
+struct sysfs_ops show_rusage_cpuacct_usage_percpu = {
+	.show = &show_rusage_cpuacct_usage_percpu_data,
+};
+struct sysfs_ops show_rusage_max_num_threads = {
+	.show = &show_rusage_max_num_threads_data,
+};
+
+/* create sysfs files for rusage. */
+void rusage_sysfs_setup(void) {
+	int error;
+	error = sysfs_createf(&show_rusage_memory, &rdata, 0444,
+		"/sys/fs/cgroup/memory/memory.stat");
+	if (error) {
+		panic("rusage_sysfs_setup:sysfs_createf() failed\n");
+	}
+	error = sysfs_createf(&show_rusage_memory_max_usage, &rdata, 0444,
+		"/sys/fs/cgroup/memory/memory.max_usage_in_bytes");
+	if (error) {
+		panic("rusage_sysfs_setup:sysfs_createf() failed\n");
+	}
+	error = sysfs_createf(&show_rusage_memory_kmem_usage, &rdata, 0444,
+		"/sys/fs/cgroup/memory/memory.kmem.usage_in_bytes");
+	if (error) {
+		panic("rusage_sysfs_setup:sysfs_createf() failed\n");
+	}
+	error = sysfs_createf(&show_rusage_memory_kmem_max_usage, &rdata, 0444,
+		"/sys/fs/cgroup/memory/memory.kmem.max_usage_in_bytes");
+	if (error) {
+		panic("rusage_sysfs_setup:sysfs_createf() failed\n");
+	}
+	error = sysfs_createf(&show_rusage_num_numa_nodes, &rdata, 0444,
+		"/sys/fs/cgroup/cpu/num_numa_nodes.txt");
+	if (error) {
+		panic("rusage_sysfs_setup:sysfs_createf() failed\n");
+	}
+	error = sysfs_createf(&show_rusage_memory_numa_stat, &rdata, 0444,
+		"/sys/fs/cgroup/memory/memory.numa_stat");
+	if (error) {
+		panic("rusage_sysfs_setup:sysfs_createf() failed\n");
+	}
+	error = sysfs_createf(&show_rusage_hugetlb_usage, &rdata, 0444,
+		"/sys/fs/cgroup/hugetlb/hugetlb.1GB.usage_in_bytes");
+	if (error) {
+		panic("rusage_sysfs_setup:sysfs_createf() failed\n");
+	}
+	error = sysfs_createf(&show_rusage_hugetlb_max_usage, &rdata, 0444,
+		"/sys/fs/cgroup/hugetlb/hugetlb.1GB.max_usage_in_bytes");
+	if (error) {
+		panic("rusage_sysfs_setup:sysfs_createf() failed\n");
+	}
+	error = sysfs_createf(&show_rusage_cpuacct_stat, &rdata, 0444,
+		"/sys/fs/cgroup/cpuacct/cpuacct.stat");
+	if (error) {
+		panic("rusage_sysfs_setup:sysfs_createf() failed\n");
+	}
+	error = sysfs_createf(&show_rusage_cpuacct_usage, &rdata, 0444,
+		"/sys/fs/cgroup/cpuacct/cpuacct.usage");
+	if (error) {
+		panic("rusage_sysfs_setup:sysfs_createf() failed\n");
+	}
+	error = sysfs_createf(&show_rusage_cpuacct_usage_percpu, &rdata, 0444,
+		"/sys/fs/cgroup/cpuacct/cpuacct.usage_percpu");
+	if (error) {
+		panic("rusage_sysfs_setup:sysfs_createf() failed\n");
+	}
+	error = sysfs_createf(&show_rusage_num_threads, &rdata, 0444,
+		"/sys/fs/cgroup/num_threads");
+	if (error) {
+		panic("rusage_sysfs_setup:sysfs_createf() failed\n");
+	}
+	error = sysfs_createf(&show_rusage_max_num_threads, &rdata, 0444,
+		"/sys/fs/cgroup/max_num_threads");
+	if (error) {
+		panic("rusage_sysfs_setup:sysfs_createf() failed\n");
+	}
+}
+
+/* callback funciton of os_status sysfs */
+static ssize_t
+show_ihk_status_data(struct sysfs_ops *ops, void *instance, void *buf, size_t size)
+{
+	return snprintf(buf, size, "%d\n", os_status);
+}
+
+struct sysfs_ops show_ihk_status = {
+	.show = &show_ihk_status_data,
+};
+
+/* create sysfs files for monitoring status.*/
+void status_sysfs_setup(void) {
+	int error;
+	error = sysfs_createf(&show_ihk_status, &rdata, 0444,
+		"/sys/fs/cgroup/mck_status");
+	if (error) {
+		panic("status_sysfs_setup:sysfs_createf() failed\n");
+	}
+}
+#endif
+

kernel/shmobj.c

@@ -179,6 +179,7 @@ int shmobj_create(struct shmid_ds *ds, struct memobj **objp)
 
 	memset(obj, 0, sizeof(*obj));
 	obj->memobj.ops = &shmobj_ops;
+	obj->memobj.size = ds->shm_segsz;
 	obj->ds = *ds;
 	obj->ds.shm_perm.seq = the_seq++;
 	obj->ds.shm_nattch = 1;

kernel/timer.c

@@ -54,136 +54,75 @@ void init_timers(void)
 }
 
 uint64_t schedule_timeout(uint64_t timeout)
-{	
-	struct waitq_entry my_wait;
-	struct timer my_timer;
+{
 	struct thread *thread = cpu_local_var(current);
-	int irqstate;
-	int spin_sleep;
-
-	irqstate = ihk_mc_spinlock_lock(&thread->spin_sleep_lock);
-	dkprintf("schedule_timeout() spin sleep timeout: %lu\n", timeout);
-	spin_sleep = ++thread->spin_sleep;
-	ihk_mc_spinlock_unlock(&thread->spin_sleep_lock, irqstate);
+	long irqstate;
 
 	/* Spin sleep.. */
 	for (;;) {
+		int need_schedule;
+		struct cpu_local_var *v = get_this_cpu_local_var();
 		uint64_t t_s = rdtsc();
 		uint64_t t_e;
-		int spin_over = 0;
-		
+
 		irqstate = ihk_mc_spinlock_lock(&thread->spin_sleep_lock);
-		
+
 		/* Woken up by someone? */
-		if (thread->spin_sleep < 1) {
+		if (thread->spin_sleep == 0) {
 			t_e = rdtsc();
 
-			spin_over = 1;
 			if ((t_e - t_s) < timeout) {
 				timeout -= (t_e - t_s);
 			}
 			else {
 				timeout = 1;
 			}
+
+			ihk_mc_spinlock_unlock(&thread->spin_sleep_lock, irqstate);
+			break;
 		}
-		
+
 		ihk_mc_spinlock_unlock(&thread->spin_sleep_lock, irqstate);
 
-		if (!spin_over) {
-			t_s = rdtsc();
-			int need_schedule;
-			struct cpu_local_var *v = get_this_cpu_local_var();
-			int irqstate = ihk_mc_spinlock_lock(&(v->runq_lock));
-			need_schedule = v->runq_len > 1 ? 1 : 0;
+		/* Give a chance to another thread (if any) in case the core is
+		 * oversubscribed, but make sure we will be re-scheduled */
+		irqstate = ihk_mc_spinlock_lock(&(v->runq_lock));
+		need_schedule = v->runq_len > 1 ? 1 : 0;
+
+		if (need_schedule) {
+			xchg4(&(cpu_local_var(current)->status), PS_RUNNING);
 			ihk_mc_spinlock_unlock(&(v->runq_lock), irqstate);
+			schedule();
 
-			/* Give a chance to another thread (if any) in case the core is
-			 * oversubscribed, but make sure we will be re-scheduled */
-			if (need_schedule) {
-				xchg4(&(cpu_local_var(current)->status), PS_RUNNING);
-				schedule();
-				xchg4(&(cpu_local_var(current)->status), 
-						PS_INTERRUPTIBLE);
-			}
-			else {
-				/* Spin wait */
-				while ((rdtsc() - t_s) < LOOP_TIMEOUT) {
-					cpu_pause();
-				}
-
-				if (timeout < LOOP_TIMEOUT) {
-					timeout = 0;
-					spin_over = 1;
-				}
-				else {
-					timeout -= LOOP_TIMEOUT;
-				}
-			}
+			/* Recheck if woken */
+			continue;
+		}
+		else {
+			ihk_mc_spinlock_unlock(&(v->runq_lock), irqstate);
 		}
-		
-		if (spin_over) {
-			dkprintf("schedule_timeout() spin woken up, timeout: %lu\n", 
-					timeout);
-			
-			/* Give a chance to another thread (if any) in case we timed out, 
-			 * but make sure we will be re-scheduled */
-			if (timeout == 0) {
-				int need_schedule;
-				struct cpu_local_var *v = get_this_cpu_local_var();
 
-				int irqstate = 
-					ihk_mc_spinlock_lock(&(v->runq_lock));
-				need_schedule = v->runq_len > 1 ? 1 : 0;
-				ihk_mc_spinlock_unlock(&(v->runq_lock), irqstate);
+		/* Spin wait */
+		while ((rdtsc() - t_s) < LOOP_TIMEOUT) {
+			cpu_pause();
+		}
 
-				if (need_schedule) {
-					xchg4(&(cpu_local_var(current)->status), PS_RUNNING);
-					schedule();
-					xchg4(&(cpu_local_var(current)->status), 
-							PS_INTERRUPTIBLE);
-				}
-			}
-			
+		/* Time out? */
+		if (timeout < LOOP_TIMEOUT) {
+			timeout = 0;
+
+			/* We are not sleeping any more */
 			irqstate = ihk_mc_spinlock_lock(&thread->spin_sleep_lock);
-			if (spin_sleep == thread->spin_sleep) {
-				--thread->spin_sleep;
-			}
+			thread->spin_sleep = 0;
 			ihk_mc_spinlock_unlock(&thread->spin_sleep_lock, irqstate);
 
-			return timeout;
+			break;
+		}
+		else {
+			timeout -= LOOP_TIMEOUT;
 		}
 	}
 
-	/* Init waitq and wait entry for this timer */
-	my_timer.timeout = (timeout < LOOP_TIMEOUT) ? LOOP_TIMEOUT : timeout;
-	my_timer.thread = cpu_local_var(current);
-	waitq_init(&my_timer.processes);
-	waitq_init_entry(&my_wait, cpu_local_var(current));
-
-	/* Add ourself to the timer queue */
-	ihk_mc_spinlock_lock_noirq(&timers_lock);
-	list_add_tail(&my_timer.list, &timers);
-
-	dkprintf("schedule_timeout() sleep timeout: %lu\n", my_timer.timeout);
-
-	/* Add ourself to the waitqueue and sleep */ 
-	waitq_prepare_to_wait(&my_timer.processes, &my_wait, PS_INTERRUPTIBLE);
-	ihk_mc_spinlock_unlock_noirq(&timers_lock);
-	schedule();
-	waitq_finish_wait(&my_timer.processes, &my_wait);
-
-	ihk_mc_spinlock_lock_noirq(&timers_lock);
-	
-	/* Waken up by someone else then timeout? */
-	if (my_timer.timeout) {
-		list_del(&my_timer.list);
-	}
-	ihk_mc_spinlock_unlock_noirq(&timers_lock);
-
-	dkprintf("schedule_timeout() woken up, timeout: %lu\n", 
-			my_timer.timeout);
-
-	return my_timer.timeout;
+	return timeout;
 }
 
 

kernel/waitq.c

@@ -22,6 +22,13 @@ default_wake_function(waitq_entry_t *entry, unsigned mode,
 	return sched_wakeup_thread(entry->private, PS_NORMAL);
 }
 
+int
+locked_wake_function(waitq_entry_t *entry, unsigned mode,
+					  int flags, void *key)
+{
+	return sched_wakeup_thread_locked(entry->private, PS_NORMAL);
+}
+
 void
 waitq_init(waitq_t *waitq)
 {

kernel/zeroobj.c

@@ -102,6 +102,7 @@ static int alloc_zeroobj(void)
 
 	memset(obj, 0, sizeof(*obj));
 	obj->memobj.ops = &zeroobj_ops;
+	obj->memobj.size = 0;
 	page_list_init(obj);
 	ihk_mc_spinlock_init(&obj->memobj.lock);
 

lib/abort.c

@@ -1,8 +1,17 @@
 #include <ihk/debug.h>
 #include <ihk/cpu.h>
+#include <cls.h>
+#include <ihk/rusage.h>
+
+extern struct cpu_local_var *clv;
 
 void panic(const char *msg)
 {
+	if (clv) {
+		struct ihk_os_monitor *monitor = cpu_local_var(monitor);
+
+		monitor->status = IHK_OS_MONITOR_PANIC;
+	}
 	cpu_disable_interrupt();
 
 	kprintf(msg);

lib/include/ihk/cpu.h

@@ -49,6 +49,7 @@ struct ihk_mc_cpu_info {
 	int ncpus;
 	int *hw_ids;
 	int *nodes;
+	int *linux_cpu_ids;
 };
 
 struct ihk_mc_cpu_info *ihk_mc_get_cpu_info(void);
@@ -56,6 +57,9 @@ void ihk_mc_boot_cpu(int cpuid, unsigned long pc);
 int ihk_mc_get_processor_id(void);
 int ihk_mc_get_hardware_processor_id(void);
 int ihk_mc_get_numa_id(void);
+int ihk_mc_get_nr_cores();
+int ihk_mc_get_core(int id, unsigned long *linux_core_id, unsigned long *apic_id,
+                    int *numa_id);
 
 void ihk_mc_delay_us(int us);
 void ihk_mc_set_syscall_handler(long (*handler)(int, ihk_mc_user_context_t *));

lib/include/ihk/mm.h

@@ -34,18 +34,25 @@ enum ihk_mc_gma_type {
 	IHK_MC_RESERVED_AREA_END,
 };
 
+extern unsigned long bootstrap_mem_end;
+
 enum ihk_mc_ma_type {
 	IHK_MC_MA_AVAILABLE,
 	IHK_MC_MA_RESERVED,
 	IHK_MC_MA_SPECIAL,
 };
 
-enum ihk_mc_ap_flag {
-	IHK_MC_AP_FLAG,
-	IHK_MC_AP_CRITICAL, /* panic on no memory space */
-	IHK_MC_AP_NOWAIT,   /* error return on no memory space */
-	IHK_MC_AP_WAIT      /* wait on no memory space */
-};
+typedef unsigned long ihk_mc_ap_flag;
+/* Panic on no memory space */
+#define IHK_MC_AP_CRITICAL        0x000001
+/* Error return on no memory space */
+#define IHK_MC_AP_NOWAIT          0x000002
+/* Wait on no memory space */
+#define IHK_MC_AP_WAIT            0x000004
+#define IHK_MC_AP_USER            0x001000
+
+#define IHK_MC_AP_BANDWIDTH       0x010000
+#define IHK_MC_AP_LATENCY         0x020000
 
 enum ihk_mc_pt_prepare_flag {
 	IHK_MC_PT_FIRST_LEVEL,
@@ -79,10 +86,10 @@ void ihk_mc_reserve_arch_pages(struct ihk_page_allocator_desc *pa_allocator,
 			unsigned long, unsigned long, int));
 
 struct ihk_mc_pa_ops {
-	void *(*alloc_page)(int, int, enum ihk_mc_ap_flag);
+	void *(*alloc_page)(int, int, ihk_mc_ap_flag, int node);
 	void (*free_page)(void *, int);
 
-	void *(*alloc)(int, enum ihk_mc_ap_flag);
+	void *(*alloc)(int, ihk_mc_ap_flag);
 	void (*free)(void *);
 };
 
@@ -103,17 +110,20 @@ void ihk_mc_map_micpa(unsigned long host_pa, unsigned long* mic_pa);
 int ihk_mc_free_micpa(unsigned long mic_pa);
 void ihk_mc_clean_micpa(void);
 
-void *_ihk_mc_alloc_aligned_pages(int npages, int p2align,
-	enum ihk_mc_ap_flag flag, char *file, int line);
-#define ihk_mc_alloc_aligned_pages(npages, p2align, flag) ({\
-void *r = _ihk_mc_alloc_aligned_pages(npages, p2align, flag, __FILE__, __LINE__);\
+void *_ihk_mc_alloc_aligned_pages_node(int npages, int p2align,
+	ihk_mc_ap_flag flag, int node, char *file, int line);
+#define ihk_mc_alloc_aligned_pages_node(npages, p2align, flag, node) ({\
+void *r = _ihk_mc_alloc_aligned_pages_node(npages, p2align, flag, node, __FILE__, __LINE__);\
+r;\
+})
+
+#define ihk_mc_alloc_aligned_pages(npages, p2align, flag) ({\
+void *r = _ihk_mc_alloc_aligned_pages_node(npages, p2align, flag, -1, __FILE__, __LINE__);\
 r;\
 })
 
-void *_ihk_mc_alloc_pages(int npages, enum ihk_mc_ap_flag flag,
-		char *file, int line);
 #define ihk_mc_alloc_pages(npages, flag) ({\
-void *r = _ihk_mc_alloc_pages(npages, flag, __FILE__, __LINE__);\
+void *r = _ihk_mc_alloc_aligned_pages_node(npages, PAGE_P2ALIGN, flag, -1, __FILE__, __LINE__);\
 r;\
 })
 
@@ -160,7 +170,7 @@ int visit_pte_range(page_table_t pt, void *start, void *end, int pgshift,
 int move_pte_range(page_table_t pt, struct process_vm *vm, 
 		void *src, void *dest, size_t size);
 
-struct page_table *ihk_mc_pt_create(enum ihk_mc_ap_flag ap_flag);
+struct page_table *ihk_mc_pt_create(ihk_mc_ap_flag ap_flag);
 /* XXX: proper use of struct page_table and page_table_t is unknown */
 void ihk_mc_pt_destroy(struct page_table *pt);
 void ihk_mc_load_page_table(struct page_table *pt);
@@ -185,6 +195,12 @@ int ihk_mc_get_memory_chunk(int id,
 void remote_flush_tlb_cpumask(struct process_vm *vm, 
 		unsigned long addr, int cpu_id);
 
+int ihk_set_kmsg(unsigned long addr, unsigned long size);
+char *ihk_get_kargs();
+
+int ihk_set_monitor(unsigned long addr, unsigned long size);
+int ihk_set_nmi_mode_addr(unsigned long addr);
+
 extern void (*__tlb_flush_handler)(int vector);
 
 struct tlb_flush_entry {

lib/include/ihk/page_alloc.h

@@ -17,11 +17,17 @@
 #include <list.h>
 
 /* XXX: Physical memory management shouldn't be part of IHK */
+struct node_distance {
+	int id;
+	int distance;
+};
+
 struct ihk_mc_numa_node {
 	int id;
 	int linux_numa_id;
 	int type;
 	struct list_head allocators;
+	struct node_distance *nodes_by_distance;
 };
 
 struct ihk_page_allocator_desc {
@@ -30,7 +36,7 @@ struct ihk_page_allocator_desc {
 	unsigned int count;
 	unsigned int flag;
 	unsigned int shift;
-	ihk_spinlock_t lock;
+	mcs_lock_node_t lock;
 	struct list_head list;
 	
 	unsigned long map[0];

lib/include/ihk/rusage.h

@@ -0,0 +1,74 @@
+#ifndef __RUSAGE_H
+#define __RUSAGE_H
+
+#define RUSAGE_DEFAULT_SIZE 10
+
+enum RUSAGE_MEMBER {
+    RUSAGE_RSS,
+    RUSAGE_CACHE,
+    RUSAGE_RSS_HUGE,
+    RUSAGE_MAPPED_FILE,
+    RUSAGE_MAX_USAGE,
+    RUSAGE_KMEM_USAGE,
+    RUSAGE_KMAX_USAGE,
+    RUSAGE_NUM_NUMA_NODES,
+    RUSAGE_NUMA_STAT,
+    RUSAGE_HUGETLB ,
+    RUSAGE_HUGETLB_MAX ,
+    RUSAGE_STAT_SYSTEM ,
+    RUSAGE_STAT_USER ,
+    RUSAGE_USAGE ,
+    RUSAGE_USAGE_PER_CPU ,
+    RUSAGE_NUM_THREADS ,
+    RUSAGE_MAX_NUM_THREADS
+};
+
+struct  r_data{
+    unsigned long pid;
+    unsigned long rss;
+    unsigned long cache;
+    unsigned long rss_huge;
+    unsigned long mapped_file;
+    unsigned long max_usage;
+    unsigned long kmem_usage;
+    unsigned long kmax_usage;
+    unsigned long hugetlb;
+    unsigned long hugetlb_max;
+    unsigned long stat_system;
+    unsigned long stat_user;
+    unsigned long usage;
+    struct r_data *next;
+} ;
+typedef struct r_data rusage_data;
+
+rusage_data *rdata[RUSAGE_DEFAULT_SIZE];
+unsigned long rusage_max_num_threads;
+unsigned long rusage_num_threads;
+
+enum ihk_os_status {
+    IHK_STATUS_INACTIVE,
+    IHK_STATUS_BOOTING,
+    IHK_STATUS_RUNNING,
+    IHK_STATUS_SHUTDOWN,
+    IHK_STATUS_PANIC,
+    IHK_STATUS_HUNGUP,
+    IHK_STATUS_FREEZING,
+    IHK_STATUS_FROZEN,
+};
+
+enum ihk_os_status os_status;
+unsigned long sys_delegate_count;
+enum sys_delegate_state_enum {
+	ENTER_KERNEL,
+	EXIT_KERNEL,
+};
+enum sys_delegate_state_enum sys_delegate_state;
+
+unsigned long rusage_kmem_usage;
+unsigned long rusage_kmem_max_usage;
+unsigned long rusage_hugetlb_usage;
+unsigned long rusage_hugetlb_max_usage;
+unsigned long rusage_usage_per_cpu[sizeof(cpu_set_t)/8];
+unsigned long rusage_numa_stat[1024];
+
+#endif

lib/include/mc_xpmem.h

@@ -0,0 +1,153 @@
+/**
+ * \file mc_xpmem.h
+ *  License details are found in the file LICENSE.
+ * \brief
+ *  Cross Partition Memory (XPMEM) structures and macros.
+ */
+/*
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * Copyright (c) 2004-2007 Silicon Graphics, Inc.  All Rights Reserved.
+ */
+/*
+ * HISTORY
+ */
+
+#ifndef _MC_XPMEM_H
+#define _MC_XPMEM_H
+
+#ifndef __KERNEL__
+#include <sys/types.h>
+#endif
+
+/*
+ * _IOC definitions for McKernel
+ */
+#define _IOC_NRBITS	8
+#define _IOC_TYPEBITS	8
+
+#define _IOC_SIZEBITS	14
+
+#define _IOC_DIRBITS	2
+
+#define _IOC_NRSHIFT	0
+#define _IOC_TYPESHIFT	(_IOC_NRSHIFT+_IOC_NRBITS)
+#define _IOC_SIZESHIFT	(_IOC_TYPESHIFT+_IOC_TYPEBITS)
+#define _IOC_DIRSHIFT	(_IOC_SIZESHIFT+_IOC_SIZEBITS)
+
+#define _IOC_NONE	0U
+
+#define _IOC(dir,type,nr,size) \
+	(((dir)  << _IOC_DIRSHIFT) | \
+	 ((type) << _IOC_TYPESHIFT) | \
+	 ((nr)   << _IOC_NRSHIFT) | \
+	 ((size) << _IOC_SIZESHIFT))
+
+#define _IO(type,nr) _IOC(_IOC_NONE,(type),(nr),0)
+
+/*
+ * basic argument type definitions for McKernel
+ */
+typedef uint64_t u64;
+typedef uint64_t __u64;
+typedef int64_t __s64;
+
+/*
+ * basic argument type definitions
+ */
+typedef __s64 xpmem_segid_t;	/* segid returned from xpmem_make() */
+typedef __s64 xpmem_apid_t;	/* apid returned from xpmem_get() */
+
+struct xpmem_addr {
+	xpmem_apid_t apid;	/* apid that represents memory */
+	off_t offset;		/* offset into apid's memory */
+};
+
+#define XPMEM_MAXADDR_SIZE	(size_t)(-1L)
+
+/*
+ * path to XPMEM device
+ */
+#define XPMEM_DEV_PATH  "/dev/xpmem"
+
+/*
+ * The following are the possible XPMEM related errors.
+ */
+#define XPMEM_ERRNO_NOPROC	2004	/* unknown thread due to fork() */
+
+/*
+ * flags for segment permissions
+ */
+#define XPMEM_RDONLY	0x1
+#define XPMEM_RDWR	0x2
+
+/*
+ * Valid permit_type values for xpmem_make().
+ */
+#define XPMEM_PERMIT_MODE	0x1
+
+/*
+ * ioctl() commands used to interface to the kernel module.
+ */
+#define XPMEM_IOC_MAGIC		'x'
+#define XPMEM_CMD_VERSION	_IO(XPMEM_IOC_MAGIC, 0)
+#define XPMEM_CMD_MAKE		_IO(XPMEM_IOC_MAGIC, 1)
+#define XPMEM_CMD_REMOVE	_IO(XPMEM_IOC_MAGIC, 2)
+#define XPMEM_CMD_GET		_IO(XPMEM_IOC_MAGIC, 3)
+#define XPMEM_CMD_RELEASE	_IO(XPMEM_IOC_MAGIC, 4)
+#define XPMEM_CMD_ATTACH	_IO(XPMEM_IOC_MAGIC, 5)
+#define XPMEM_CMD_DETACH	_IO(XPMEM_IOC_MAGIC, 6)
+
+/*
+ * Structures used with the preceding ioctl() commands to pass data.
+ */
+struct xpmem_cmd_make {
+	__u64 vaddr;
+	size_t size;
+	int permit_type;
+	__u64 permit_value;
+	xpmem_segid_t segid;	/* returned on success */
+};
+
+struct xpmem_cmd_remove {
+	xpmem_segid_t segid;
+};
+
+struct xpmem_cmd_get {
+	xpmem_segid_t segid;
+	int flags;
+	int permit_type;
+	__u64 permit_value;
+	xpmem_apid_t apid;	/* returned on success */
+};
+
+struct xpmem_cmd_release {
+	xpmem_apid_t apid;
+};
+
+struct xpmem_cmd_attach {
+	xpmem_apid_t apid;
+	off_t offset;
+	size_t size;
+	__u64 vaddr;
+	int fd;
+	int flags;
+};
+
+struct xpmem_cmd_detach {
+	__u64 vaddr;
+};
+
+#ifndef __KERNEL__
+extern int xpmem_version(void);
+extern xpmem_segid_t xpmem_make(void *, size_t, int, void *);
+extern int xpmem_remove(xpmem_segid_t);
+extern xpmem_apid_t xpmem_get(xpmem_segid_t, int, int, void *);
+extern int xpmem_release(xpmem_apid_t);
+extern void *xpmem_attach(struct xpmem_addr, size_t, void *);
+extern int xpmem_detach(void *);
+#endif
+
+#endif /* _MC_XPMEM_H */

lib/include/memory.h

@@ -35,8 +35,8 @@ void *phys_to_virt(unsigned long p);
 int copy_from_user(void *dst, const void *src, size_t siz);
 int strlen_user(const char *s);
 int strcpy_from_user(char *dst, const char *src);
-long getlong_user(const long *p);
-int getint_user(const int *p);
+long getlong_user(long *dest, const long *p);
+int getint_user(int *dest, const int *p);
 int read_process_vm(struct process_vm *vm, void *kdst, const void *usrc, size_t siz);
 int copy_to_user(void *dst, const void *src, size_t siz);
 int setlong_user(long *dst, long data);

lib/page_alloc.c

@@ -19,7 +19,6 @@
 #include <memory.h>
 #include <bitops.h>
 
-void *allocate_pages(int npages, enum ihk_mc_ap_flag flag);
 void free_pages(void *, int npages);
 
 #define MAP_INDEX(n)    ((n) >> 6)
@@ -73,7 +72,7 @@ void *__ihk_pagealloc_init(unsigned long start, unsigned long size,
 	//kprintf("page allocator @ %lx - %lx (%d)\n", start, start + size,
 	//        page_shift);
 
-	ihk_mc_spinlock_init(&desc->lock);
+	mcs_lock_init(&desc->lock);
 
 	/* Reserve align padding area */
 	for (i = mapsize; i < mapaligned * 8; i++) {
@@ -99,12 +98,12 @@ void ihk_pagealloc_destroy(void *__desc)
 static unsigned long __ihk_pagealloc_large(struct ihk_page_allocator_desc *desc,
                                            int npages, int p2align)
 {
-	unsigned long flags;
 	unsigned int i, j, mi;
 	int nblocks;
 	int nfrags;
 	unsigned long mask;
-	int mialign;
+	unsigned long align_mask = ((PAGE_SIZE << p2align) - 1);
+	mcs_lock_node_t node;
 
 	nblocks = (npages / 64);
 	mask = -1;
@@ -113,14 +112,13 @@ static unsigned long __ihk_pagealloc_large(struct ihk_page_allocator_desc *desc,
 		++nblocks;
 		mask = (1UL << nfrags) - 1;
 	}
-	mialign = (p2align <= 6)? 1: (1 << (p2align - 6));
 
-	flags = ihk_mc_spinlock_lock(&desc->lock);
+	mcs_lock_lock(&desc->lock, &node);
 	for (i = 0, mi = desc->last; i < desc->count; i++, mi++) {
 		if (mi >= desc->count) {
 			mi = 0;
 		}
-		if ((mi + nblocks >= desc->count) || (mi % mialign)) {
+		if ((mi + nblocks >= desc->count) || (ADDRESS(desc, mi, 0) & align_mask)) {
 			continue;
 		}
 		for (j = mi; j < mi + nblocks - 1; j++) {
@@ -133,11 +131,11 @@ static unsigned long __ihk_pagealloc_large(struct ihk_page_allocator_desc *desc,
 				desc->map[j] = (unsigned long)-1;
 			}
 			desc->map[j] |= mask;
-			ihk_mc_spinlock_unlock(&desc->lock, flags);
+			mcs_lock_unlock(&desc->lock, &node);
 			return ADDRESS(desc, mi, 0);
 		}
 	}
-	ihk_mc_spinlock_unlock(&desc->lock, flags);
+	mcs_lock_unlock(&desc->lock, &node);
 
 	return 0;
 }
@@ -147,8 +145,9 @@ unsigned long ihk_pagealloc_alloc(void *__desc, int npages, int p2align)
 	struct ihk_page_allocator_desc *desc = __desc;
 	unsigned int i, mi;
 	int j;
-	unsigned long v, mask, flags;
+	unsigned long v, mask;
 	int jalign;
+	mcs_lock_node_t node;
 
 	if ((npages >= 32) || (p2align >= 5)) {
 		return __ihk_pagealloc_large(desc, npages, p2align);
@@ -157,7 +156,7 @@ unsigned long ihk_pagealloc_alloc(void *__desc, int npages, int p2align)
 	mask = (1UL << npages) - 1;
 	jalign = (p2align <= 0)? 1: (1 << p2align);
 
-	flags = ihk_mc_spinlock_lock(&desc->lock);
+	mcs_lock_lock(&desc->lock, &node);
 	for (i = 0, mi = desc->last; i < desc->count; i++, mi++) {
 		if (mi >= desc->count) {
 			mi = 0;
@@ -174,12 +173,12 @@ unsigned long ihk_pagealloc_alloc(void *__desc, int npages, int p2align)
 			if (!(v & (mask << j))) { /* free */
 				desc->map[mi] |= (mask << j);
 
-				ihk_mc_spinlock_unlock(&desc->lock, flags);
+				mcs_lock_unlock(&desc->lock, &node);
 				return ADDRESS(desc, mi, j);
 			}
 		}
 	}
-	ihk_mc_spinlock_unlock(&desc->lock, flags);
+	mcs_lock_unlock(&desc->lock, &node);
 
 	/* We use null pointer for failure */
 	return 0;
@@ -189,7 +188,7 @@ void ihk_pagealloc_reserve(void *__desc, unsigned long start, unsigned long end)
 {
 	int i, n;
 	struct ihk_page_allocator_desc *desc = __desc;
-	unsigned long flags;
+	mcs_lock_node_t node;
 
 	n = (end + (1 << desc->shift) - 1 - desc->start) >> desc->shift;
 	i = ((start - desc->start) >> desc->shift);
@@ -197,7 +196,7 @@ void ihk_pagealloc_reserve(void *__desc, unsigned long start, unsigned long end)
 		return;
 	}
 
-	flags = ihk_mc_spinlock_lock(&desc->lock);
+	mcs_lock_lock(&desc->lock, &node);
 	for (; i < n; i++) {
 		if (!(i & 63) && i + 63 < n) {
 			desc->map[MAP_INDEX(i)] = (unsigned long)-1L;
@@ -206,7 +205,7 @@ void ihk_pagealloc_reserve(void *__desc, unsigned long start, unsigned long end)
 			desc->map[MAP_INDEX(i)] |= (1UL << MAP_BIT(i));
 		}
 	}
-	ihk_mc_spinlock_unlock(&desc->lock, flags);
+	mcs_lock_unlock(&desc->lock, &node);
 }
 
 void ihk_pagealloc_free(void *__desc, unsigned long address, int npages)
@@ -214,24 +213,24 @@ void ihk_pagealloc_free(void *__desc, unsigned long address, int npages)
 	struct ihk_page_allocator_desc *desc = __desc;
 	int i;
 	unsigned mi;
-	unsigned long flags;
+	mcs_lock_node_t node;
 
 	/* XXX: Parameter check */
-	flags = ihk_mc_spinlock_lock(&desc->lock);
+	mcs_lock_lock(&desc->lock, &node);
 	mi = (address - desc->start) >> desc->shift;
 	for (i = 0; i < npages; i++, mi++) {
 		desc->map[MAP_INDEX(mi)] &= ~(1UL << MAP_BIT(mi));
 	}
-	ihk_mc_spinlock_unlock(&desc->lock, flags);
+	mcs_lock_unlock(&desc->lock, &node);
 }
 
 unsigned long ihk_pagealloc_count(void *__desc)
 {
 	struct ihk_page_allocator_desc *desc = __desc;
 	unsigned long i, j, n = 0;
-	unsigned long flags;
+	mcs_lock_node_t node;
 
-	flags = ihk_mc_spinlock_lock(&desc->lock);
+	mcs_lock_lock(&desc->lock, &node);
 	/* XXX: Very silly counting */
 	for (i = 0; i < desc->count; i++) {
 		for (j = 0; j < 64; j++) {
@@ -240,7 +239,7 @@ unsigned long ihk_pagealloc_count(void *__desc)
 			}
 		}
 	}
-	ihk_mc_spinlock_unlock(&desc->lock, flags);
+	mcs_lock_unlock(&desc->lock, &node);
 	
 	return n;
 }
@@ -250,10 +249,11 @@ int ihk_pagealloc_query_free(void *__desc)
 	struct ihk_page_allocator_desc *desc = __desc;
 	unsigned int mi;
 	int j;
-	unsigned long v, flags;
+	unsigned long v;
 	int npages = 0;
+	mcs_lock_node_t node;
 
-	flags = ihk_mc_spinlock_lock(&desc->lock);
+	mcs_lock_lock(&desc->lock, &node);
 	for (mi = 0; mi < desc->count; mi++) {
 		
 		v = desc->map[mi];
@@ -266,7 +266,7 @@ int ihk_pagealloc_query_free(void *__desc)
 			}
 		}
 	}
-	ihk_mc_spinlock_unlock(&desc->lock, flags);
+	mcs_lock_unlock(&desc->lock, &node);
 
 	return npages;
 }
@@ -276,11 +276,12 @@ void __ihk_pagealloc_zero_free_pages(void *__desc)
 	struct ihk_page_allocator_desc *desc = __desc;
 	unsigned int mi;
 	int j;
-	unsigned long v, flags;
+	unsigned long v;
+	mcs_lock_node_t node;
 
 kprintf("zeroing free memory... ");
 
-	flags = ihk_mc_spinlock_lock(&desc->lock);
+	mcs_lock_lock(&desc->lock, &node);
 	for (mi = 0; mi < desc->count; mi++) {
 		
 		v = desc->map[mi];
@@ -294,7 +295,7 @@ kprintf("zeroing free memory... ");
 			}
 		}
 	}
-	ihk_mc_spinlock_unlock(&desc->lock, flags);
+	mcs_lock_unlock(&desc->lock, &node);
 
 kprintf("\nzeroing done\n");
 }

lib/string.c

@@ -275,13 +275,21 @@ int flatten_strings_from_user(int nr_strings, char *first, char **strings, char
 	long *_flat;
 	char *p;
 	long r;
-	int n;
+	int n, ret;
 
 	/* How many strings do we have? */
 	if (nr_strings == -1) {
-		for (nr_strings = 0; (r = getlong_user((void *)(strings + nr_strings))) > 0; ++nr_strings);
-		if(r < 0)
-			return r;
+		nr_strings = 0;
+		for (;;) {
+			ret = getlong_user(&r, (void *)(strings + nr_strings));
+			if (ret < 0)
+				return ret;
+
+			if (r == 0)
+				break;
+
+			++nr_strings;
+		}
 	}
 
 	/* Count full length */
@@ -295,13 +303,19 @@ int flatten_strings_from_user(int nr_strings, char *first, char **strings, char
 	}
 
 	for (string_i = 0; string_i < nr_strings; ++string_i) {
-		char *userp = (char *)getlong_user((void *)(strings + string_i));
-		int len = strlen_user(userp);
+		char *userp;
+		int len;
+
+		ret = getlong_user((long *)&userp, (void *)(strings + string_i));
+		if (ret < 0)
+			return ret;
+
+		len = strlen_user(userp);
 
 		if(len < 0)
 			return len;
 		// Pointer + actual value
-		full_len += sizeof(char *) + len + 1; 
+		full_len += sizeof(char *) + len + 1;
 	}
 
 	full_len = (full_len + sizeof(long) - 1) & ~(sizeof(long) - 1);
@@ -326,8 +340,13 @@ int flatten_strings_from_user(int nr_strings, char *first, char **strings, char
 	}
 
 	for (string_i = 0; string_i < nr_strings; ++string_i) {
-		char *userp = (char *)getlong_user((void *)(strings + string_i));
+		char *userp;
 		_flat[n++] = p - (char *)_flat;
+
+		ret = getlong_user((long *)&userp, (void *)(strings + string_i));
+		if (ret < 0)
+			return ret;
+
 		strcpy_from_user(p, userp);
 		p = strchr(p, '\0') + 1;
 	}