process: transfer TIDs in bulk and reuse them locally

executer/user/mcexec.c

@@ -1944,6 +1944,39 @@ int main_loop(int fd, int cpu, pthread_mutex_t *lock)
 				thread_data[oldcpuid].remote_tid = wtid;
 			}
 
+			/*
+			 * Number of TIDs and the remote physical address where TIDs are
+			 * expected are passed in arg 4 and 5, respectively.
+			 */
+			if (w.sr.args[4] > 0) {
+				struct remote_transfer trans;
+				int i = 0;
+				int *tids = malloc(sizeof(int) * w.sr.args[4]);
+				if (!tids) {
+					fprintf(stderr, "__NR_gettid(): error allocating TIDs\n");
+					goto gettid_out;
+				}
+
+				for (i = 0; i < ncpu && i < w.sr.args[4]; ++i) {
+					tids[i] = thread_data[i].tid;
+				}
+
+				for (; i < ncpu; ++i) {
+					tids[i] = 0;
+				}
+
+				trans.userp = (void*)tids;
+				trans.rphys = w.sr.args[5];
+				trans.size = sizeof(int) * w.sr.args[4];
+				trans.direction = MCEXEC_UP_TRANSFER_TO_REMOTE;
+
+				if (ioctl(fd, MCEXEC_UP_TRANSFER, &trans) != 0) {
+					fprintf(stderr, "__NR_gettid(): error transfering TIDs\n");
+				}
+
+				free(tids);
+			}
+gettid_out:
 			do_syscall_return(fd, cpu, thread_data[newcpuid].remote_tid, 0, 0, 0, 0);
 			break;
 		}

kernel/host.c

@@ -521,8 +521,6 @@ static void syscall_channel_send(struct ihk_ikc_channel_desc *c,
 }
 
 extern unsigned long do_kill(struct thread *, int, int, int, struct siginfo *, int ptracecont);
-extern void settid(struct thread *proc, int mode, int newcpuid, int oldcpuid);
-
 extern void process_procfs_request(unsigned long rarg);
 extern int memcheckall();
 extern int freecheck(int runcount);
@@ -612,7 +610,7 @@ static int syscall_packet_handler(struct ihk_ikc_channel_desc *c,
 		thread = (struct thread *)packet->arg;
 		proc = thread->proc;
 
-		settid(thread, 0, cpuid, -1);
+		settid(thread, 0, cpuid, -1, 0, NULL);
 		proc->status = PS_RUNNING;
 		thread->status = PS_RUNNING;
 		chain_thread(thread);

kernel/include/process.h

@@ -348,6 +348,11 @@ struct sig_pending {
 
 typedef void pgio_func_t(void *arg);
 
+struct mcexec_tid {
+	int tid;
+	struct thread *thread;
+};
+
 /* Represents a node in the process fork tree, it may exist even after the 
  * corresponding process exited due to references from the parent and/or 
  * children and is used for implementing wait/waitpid without having a 
@@ -362,6 +367,9 @@ struct process {
 	// threads and children
 	struct list_head threads_list;
 	mcs_rwlock_lock_t threads_lock; // lock for threads_list
+	/* TID set of proxy process */
+	struct mcexec_tid *tids;
+	int nr_tids;
 
 	/* The ptracing process behave as the parent of the ptraced process
 	   after using PTRACE_ATTACH except getppid. So we save it here. */
@@ -678,5 +686,7 @@ void chain_thread(struct thread *);
 void proc_init();
 void set_timer();
 struct sig_pending *hassigpending(struct thread *thread);
+void settid(struct thread *thread, int mode, int newcpuid, int oldcpuid,
+	int nr_tids, int *tids);
 
 #endif

kernel/process.c

@@ -53,7 +53,6 @@ static int copy_user_ranges(struct process_vm *vm, struct process_vm *orgvm);
 extern void release_fp_regs(struct thread *proc);
 extern void save_fp_regs(struct thread *proc);
 extern void restore_fp_regs(struct thread *proc);
-void settid(struct thread *proc, int mode, int newcpuid, int oldcpuid);
 extern void __runq_add_proc(struct thread *proc, int cpu_id);
 extern void terminate_host(int pid);
 extern void lapic_timer_enable(unsigned int clocks);
@@ -2062,6 +2061,7 @@ release_process(struct process *proc)
 		mcs_rwlock_writer_unlock(&parent->children_lock, &lock);
 	}
 
+	if (proc->tids) kfree(proc->tids);
 	kfree(proc);
 }
 
@@ -2167,6 +2167,23 @@ release_sigcommon(struct sig_common *sigcommon)
 	kfree(sigcommon);
 }
 
+/*
+ * Release the TID from the process' TID set corresponding to this thread.
+ * NOTE: threads_lock must be held.
+ */
+void __release_tid(struct process *proc, struct thread *thread) {
+	int i;
+
+	for (i = 0; i < proc->nr_tids; ++i) {
+		if (proc->tids[i].thread != thread) continue;
+
+		proc->tids[i].thread = NULL;
+		dkprintf("%s: tid %d has been released by %p\n",
+			__FUNCTION__, thread->tid, thread);
+		break;
+	}
+}
+
 void destroy_thread(struct thread *thread)
 {
 	struct sig_pending *pending;
@@ -2183,6 +2200,7 @@ void destroy_thread(struct thread *thread)
 
 	mcs_rwlock_writer_lock(&proc->threads_lock, &lock);
 	list_del(&thread->siblings_list);
+	__release_tid(proc, thread);
 	mcs_rwlock_writer_unlock(&proc->threads_lock, &lock);
 
 	cpu_clear(thread->cpu_id, &thread->vm->address_space->cpu_set,
@@ -2522,7 +2540,7 @@ static void do_migrate(void)
 		v->flags |= CPU_FLAG_NEED_RESCHED;
 		ihk_mc_interrupt_cpu(get_x86_cpu_local_variable(cpu_id)->apic_id, 0xd1);
 		double_rq_unlock(cur_v, v, irqstate);
-		settid(req->thread, 2, cpu_id, old_cpu_id);
+		//settid(req->thread, 2, cpu_id, old_cpu_id, 0, NULL);
 
 ack:
 		waitq_wakeup(&req->wq);

kernel/syscall.c

@@ -1478,8 +1478,8 @@ SYSCALL_DECLARE(getppid)
 	return thread->proc->ppid_parent->pid;
 }
 
-void
+void settid(struct thread *thread, int mode, int newcpuid, int oldcpuid,
-settid(struct thread *thread, int mode, int newcpuid, int oldcpuid)
+	int nr_tids, int *tids)
 {
 	struct syscall_request request IHK_DMA_ALIGN;
 	unsigned long rc;
@@ -1489,6 +1489,12 @@ settid(struct thread *thread, int mode, int newcpuid, int oldcpuid)
 	request.args[1] = thread->proc->pid;
 	request.args[2] = newcpuid;
 	request.args[3] = oldcpuid;
+	/*
+	 * If nr_tids is non-zero, tids should point to an array of ints
+	 * where the thread ids of the mcexec process are expected.
+	 */
+	request.args[4] = nr_tids;
+	request.args[5] = virt_to_phys(tids);
 	rc = do_syscall(&request, ihk_mc_get_processor_id(), thread->proc->pid);
 	if (mode != 2) {
 		thread->tid = rc;
@@ -1893,7 +1899,55 @@ unsigned long do_fork(int clone_flags, unsigned long newsp,
 	        &new->vm->address_space->cpu_set_lock);
 
 	if (clone_flags & CLONE_VM) {
-		settid(new, 1, cpuid, -1);
+		int *tids = NULL;
+		int i;
+		struct mcs_rwlock_node_irqsave lock;
+
+		mcs_rwlock_writer_lock(&newproc->threads_lock, &lock);
+		/* Obtain mcexec TIDs if not known yet */
+		if (!newproc->nr_tids) {
+			tids = kmalloc(sizeof(int) * num_processors, IHK_MC_AP_NOWAIT);
+			if (!tids) {
+				mcs_rwlock_writer_unlock(&newproc->threads_lock, &lock);
+				release_cpuid(cpuid);
+				return -ENOMEM;
+			}
+
+			newproc->tids = kmalloc(sizeof(struct mcexec_tid) * num_processors, IHK_MC_AP_NOWAIT);
+			if (!newproc->tids) {
+				mcs_rwlock_writer_unlock(&newproc->threads_lock, &lock);
+				kfree(tids);
+				release_cpuid(cpuid);
+				return -ENOMEM;
+			}
+
+			settid(new, 1, cpuid, -1, num_processors, tids);
+
+			for (i = 0; (i < num_processors) && tids[i]; ++i) {
+				dkprintf("%s: tid[%d]: %d\n", __FUNCTION__, i, tids[i]);
+				newproc->tids[i].tid = tids[i];
+				newproc->tids[i].thread = NULL;
+				++newproc->nr_tids;
+			}
+		}
+
+		/* Find an unused TID */
+		for (i = 0; i < newproc->nr_tids; ++i) {
+			if (!newproc->tids[i].thread) {
+				newproc->tids[i].thread = new;
+				new->tid = newproc->tids[i].tid;
+				dkprintf("%s: tid %d assigned to %p\n", __FUNCTION__, new->tid, new);
+				break;
+			}
+		}
+
+		/* TODO: spawn more mcexec threads */
+		if (!new->tid) {
+			kprintf("%s: no more TIDs available\n");
+			panic("");
+		}
+
+		mcs_rwlock_writer_unlock(&newproc->threads_lock, &lock);
 	}
 	/* fork() a new process on the host */
 	else {
@@ -1913,7 +1967,7 @@ unsigned long do_fork(int clone_flags, unsigned long newsp,
 		}
 
 		/* In a single threaded process TID equals to PID */
-		settid(new, 0, cpuid, -1);
+		settid(new, 0, cpuid, -1, 0, NULL);
 		new->vm->address_space->pids[0] = new->proc->pid;
 
 		dkprintf("fork(): new pid: %d\n", new->proc->pid);