Make clone system-call create a new thread on the next available physical core

and skip the physical core with a system process so that
programs using Intel OpenMP bind threads to cores in that manner.

arch/x86/kernel/syscall.c

@@ -3,6 +3,7 @@
  */
 
 #include <ihk/cpu.h>
+#include <ihk/debug.h>
 #include <cls.h>
 #include <syscall.h>
 
@@ -14,4 +15,55 @@
 #define dkprintf(...)
 #endif
 
+/*
+See dkprintf("BSP HW ID = %d, ", bsp_hw_id); (in ./mcos/kernel/ap.c)
+
+Core with BSP HW ID 224 is 1st logical core of last physical core.
+It                      boots first and is given SW-ID of 0
+
+Core with BSP HW ID 0 is 1st logical core of 1st physical core. 
+It                      boots next and is given  SW-ID of 1.
+Core with BSP HW ID 1   boots next and is given  SW-ID of 2.
+Core with BSP HW ID 2   boots next and is given  SW-ID of 3.
+Core with BSP HW ID 3   boots next and is given  SW-ID of 4.
+...
+Core with BSP HW ID 220 is 1st logical core of 56-th physical core.
+It                      boots next and is given  SW-ID of 221.
+Core with BSP HW ID 221 boots next and is given  SW-ID of 222.
+Core with BSP HW ID 222 boots next and is given  SW-ID of 223.
+Core with BSP HW ID 223 boots next and is given  SW-ID of 224.
+
+Core with BSP HW ID 225 is 2nd logical core of last physical core.
+It                      boots next and is given  SW-ID of 225.
+Core with BSP HW ID 226 boots next and is given  SW-ID of 226.
+Core with BSP HW ID 227 boots next and is given  SW-ID of 227.
+*/
+static ihk_spinlock_t cpuid_head_lock = 0;
+static int cpuid_head = 1;
+
 /* archtecture-depended syscall handlers */
+int obtain_clone_cpuid() {
+    /* see above on BSP HW ID */
+	struct ihk_mc_cpu_info *cpu_info = ihk_mc_get_cpu_info();
+    int cpuid, nretry = 0;
+    ihk_mc_spinlock_lock_noirq(&cpuid_head_lock);
+ retry:
+    /* Try to obtain next physical core */
+    cpuid = cpuid_head;
+    cpuid_head += 4;
+    if(cpuid_head >= cpu_info->ncpus) {
+        cpuid_head = ((cpuid_head % 4) + 1) % 4;
+    }
+    /* Don't use a physical core with a system process (e.g. MPI)
+       because using it degrades performance */
+    if((cpu_info->ncpus - 3 <= cpuid && cpuid <= cpu_info->ncpus - 1) ||
+       get_cpu_local_var(cpuid)->status != CPU_STATUS_IDLE) {
+        nretry++;
+        if(nretry >= cpu_info->ncpus) {
+            panic("there is no cpu with empty runq\n");
+        }
+        goto retry; 
+    }
+    ihk_mc_spinlock_unlock_noirq(&cpuid_head_lock);
+    return cpuid;
+}

kernel/include/syscall.h

@@ -185,5 +185,6 @@ struct syscall_params {
 #define SYSCALL_FOOTER return do_syscall(&request, ctx)
 
 extern int do_syscall(struct syscall_request *req, ihk_mc_user_context_t *ctx);
+extern int obtain_clone_cpuid();
 
 #endif

kernel/syscall.c

@@ -487,27 +487,16 @@ SYSCALL_DECLARE(arch_prctl)
 
 SYSCALL_DECLARE(clone)
 {
-	int i;
-	int cpuid = -1;
+	int cpuid;
 	int clone_flags = ihk_mc_syscall_arg0(ctx);
-	struct ihk_mc_cpu_info *cpu_info = ihk_mc_get_cpu_info();
 	struct process *new;
 	
 	dkprintf("[%d] clone(): stack_pointr: 0x%lX\n",
 	         ihk_mc_get_processor_id(), 
 			 (unsigned long)ihk_mc_syscall_arg1(ctx));
 
-	//ihk_mc_spinlock_lock_noirq(&cpu_status_lock);
-	for (i = 0; i < cpu_info->ncpus; i++) {
-		if (get_cpu_local_var(i)->status == CPU_STATUS_IDLE) {
-			cpuid = i;
-			break;
-		}
-	}
+    cpuid = obtain_clone_cpuid();
 
-	if (cpuid < 0) 
-		return -EAGAIN;
-	
 	new = clone_process(cpu_local_var(current), ihk_mc_syscall_pc(ctx),
 	                    ihk_mc_syscall_arg1(ctx));
 	
@@ -546,7 +535,7 @@ SYSCALL_DECLARE(clone)
 
 	ihk_mc_syscall_ret(new->uctx) = 0;
 	
-	dkprintf("clone: kicking scheduler!\n");
+	dkprintf("clone: kicking scheduler!,cpuid=%d\n", cpuid);
 	runq_add_proc(new, cpuid);
 
 	return new->pid;