do_syscall(): allow schedule for another thread (Intel MPI+OpenMP issue)

kernel/process.c

@@ -2503,7 +2503,6 @@ static void do_migrate(void)
 		cur_v->runq_len -= 1;
 		old_cpu_id = req->thread->cpu_id;
 		req->thread->cpu_id = cpu_id;
-		settid(req->thread, 2, cpu_id, old_cpu_id);
 		list_add_tail(&req->thread->sched_list, &v->runq);
 		v->runq_len += 1;
 		
@@ -2518,6 +2517,7 @@ static void do_migrate(void)
 		v->flags |= CPU_FLAG_NEED_RESCHED;
 		ihk_mc_interrupt_cpu(get_x86_cpu_local_variable(cpu_id)->apic_id, 0xd1);
 		double_rq_unlock(cur_v, v, irqstate);
+		settid(req->thread, 2, cpu_id, old_cpu_id);
 
 ack:
 		waitq_wakeup(&req->wq);

kernel/syscall.c

@@ -231,16 +231,43 @@ long do_syscall(struct syscall_request *req, int cpu, int pid)
 
 	send_syscall(req, cpu, pid);
 
-	dkprintf("SC(%d)[%3d] waiting for host.. \n", 
+	dkprintf("%s: syscall num: %d waiting for Linux.. \n",
-	        ihk_mc_get_processor_id(),
+		__FUNCTION__, req->number);
-	        req->number);
 	
 #define	STATUS_IN_PROGRESS	0
 #define	STATUS_COMPLETED	1
 #define	STATUS_PAGE_FAULT	3
 	while (res->status != STATUS_COMPLETED) {
 		while (res->status == STATUS_IN_PROGRESS) {
+			struct cpu_local_var *v;
+			int call_schedule = 0;
+			long runq_irqstate;
+
 			cpu_pause();
+
+			/* XXX: Intel MPI + Intel OpenMP situation:
+			 * While the MPI helper thread waits in a poll() call the OpenMP master
+			 * thread is iterating through the CPU cores using setaffinity().
+			 * Unless we give a chance to it on this core the two threads seem to
+			 * hang in deadlock. If the new thread would make a system call on this
+			 * core we would be in trouble. For now, allow it, but in the future
+			 * we should have syscall channels for each thread instead of per core,
+			 * or we should multiplex syscall threads in mcexec */
+			runq_irqstate =
+				ihk_mc_spinlock_lock(&(get_this_cpu_local_var()->runq_lock));
+			v = get_this_cpu_local_var();
+
+			if (v->flags & CPU_FLAG_NEED_RESCHED) {
+				call_schedule = 1;
+				--thread->in_syscall_offload;
+			}
+
+			ihk_mc_spinlock_unlock(&v->runq_lock, runq_irqstate);
+
+			if (call_schedule) {
+				schedule();
+				++thread->in_syscall_offload;
+			}
 		}
 	
 		if (res->status == STATUS_PAGE_FAULT) {
@@ -260,9 +287,8 @@ long do_syscall(struct syscall_request *req, int cpu, int pid)
 		}
 	}
 
-	dkprintf("SC(%d)[%3d] got host reply: %d \n", 
+	dkprintf("%s: syscall num: %d got host reply: %d \n",
-	        ihk_mc_get_processor_id(),
+		__FUNCTION__, req->number, res->ret);
-	        req->number, res->ret);
 
 	rc = res->ret;
 	if(islock){
@@ -1068,6 +1094,11 @@ do_mmap(const intptr_t addr0, const size_t len0, const int prot,
 		populated_mapping = 1;
 	}
 
+	/* XXX: Intel MPI 128MB mapping.. */
+	if (len == 134217728) {
+		populated_mapping = 0;
+	}
+
 	if (!(prot & PROT_WRITE)) {
 		error = set_host_vma(addr, len, PROT_READ);
 		if (error) {