From 04d4145b3e27bd59e65678f533c61000ff91b1ab Mon Sep 17 00:00:00 2001 From: Masamichi Takagi Date: Mon, 3 Sep 2018 18:02:35 +0900 Subject: [PATCH] uti: Replace dead uti thread with new mcexec thread in proc->tids Change-Id: Ic6e906dd1bfac1b07f1317732cbe0a5191831cd8 --- arch/arm64/kernel/syscall.c | 2 +- arch/x86_64/kernel/syscall.c | 2 +- executer/kernel/mcctrl/control.c | 5 +++-- executer/user/mcexec.c | 35 ++++++++++++++++++++++++-------- kernel/include/process.h | 8 +++++++- kernel/process.c | 21 ++++++++++++++++++- kernel/syscall.c | 11 ++++++++-- 7 files changed, 68 insertions(+), 16 deletions(-) diff --git a/arch/arm64/kernel/syscall.c b/arch/arm64/kernel/syscall.c index a8e4b9fb..d2a309c0 100644 --- a/arch/arm64/kernel/syscall.c +++ b/arch/arm64/kernel/syscall.c @@ -1575,7 +1575,7 @@ done: return 0; } - if (tthread->thread_offloaded) { + if (tthread->uti_state == UTI_STATE_RUNNING_IN_LINUX) { interrupt_syscall(tthread, sig); release_thread(tthread); return 0; diff --git a/arch/x86_64/kernel/syscall.c b/arch/x86_64/kernel/syscall.c index c2ca64d1..a7fd9e82 100644 --- a/arch/x86_64/kernel/syscall.c +++ b/arch/x86_64/kernel/syscall.c @@ -1370,7 +1370,7 @@ done: } /* Forward signal to Linux by interrupt_syscall mechanism */ - if (tthread->thread_offloaded) { + if (tthread->uti_state == UTI_STATE_RUNNING_IN_LINUX) { if (!tthread->proc->nohost) { interrupt_syscall(tthread, sig); } diff --git a/executer/kernel/mcctrl/control.c b/executer/kernel/mcctrl/control.c index 5f085bac..4515939b 100644 --- a/executer/kernel/mcctrl/control.c +++ b/executer/kernel/mcctrl/control.c @@ -2391,7 +2391,7 @@ long mcexec_util_thread1(ihk_os_t os, unsigned long arg, struct file *file) { void **__user uparam = (void ** __user)arg; - void *param[6]; + void *param[7]; unsigned long p_rctx; unsigned long phys; void *__user u_rctx; @@ -2401,7 +2401,7 @@ mcexec_util_thread1(ihk_os_t os, unsigned long arg, struct file *file) unsigned long free_size; unsigned long icurrent = (unsigned long)current; - if(copy_from_user(param, uparam, sizeof(void *) * 6)) { + if(copy_from_user(param, uparam, sizeof(void *) * 7)) { return -EFAULT; } p_rctx = (unsigned long)param[0]; @@ -2422,6 +2422,7 @@ mcexec_util_thread1(ihk_os_t os, unsigned long arg, struct file *file) ((unsigned long *)rctx)[0] = free_address; ((unsigned long *)rctx)[1] = free_size; + ((unsigned long *)rctx)[2] = (unsigned long)param[6]; #ifdef CONFIG_MIC iounmap(rctx); diff --git a/executer/user/mcexec.c b/executer/user/mcexec.c index 3979e357..ad1fb93a 100644 --- a/executer/user/mcexec.c +++ b/executer/user/mcexec.c @@ -1040,6 +1040,7 @@ pid_t master_tid; pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; pthread_barrier_t init_ready; +pthread_barrier_t uti_init_ready; pthread_attr_t watchdog_thread_attr; pthread_t watchdog_thread; @@ -1390,8 +1391,7 @@ void init_sigaction(void) static int max_cpuid; -static int -create_worker_thread(pthread_barrier_t *init_ready) +static int create_worker_thread(struct thread_data_s **tp_out, pthread_barrier_t *init_ready) { struct thread_data_s *tp; @@ -1409,6 +1409,10 @@ create_worker_thread(pthread_barrier_t *init_ready) tp->next = thread_data; thread_data = tp; + if (tp_out) { + *tp_out = tp; + } + return pthread_create(&tp->thread_id, NULL, &main_loop_thread_func, tp); } @@ -1422,7 +1426,7 @@ int init_worker_threads(int fd) max_cpuid = 0; for (i = 0; i <= n_threads; ++i) { - int ret = create_worker_thread(&init_ready); + int ret = create_worker_thread(NULL, &init_ready); if (ret) { printf("ERROR: creating syscall threads (%d), check ulimit?\n", ret); @@ -2882,10 +2886,18 @@ static long util_thread(struct thread_data_s *my_thread, unsigned long uctx_pa, int i; void *lctx; void *rctx; - void *param[6]; + void *param[7]; int rc = 0; void *uti_wp = (void*)-1; + pthread_barrier_init(&uti_init_ready, NULL, 2); + if ((rc = create_worker_thread(&tp, &uti_init_ready))) { + printf("%s: Error: create_worker_thread failed (%d)\n", __FUNCTION__, rc); + goto out; + } + pthread_barrier_wait(&uti_init_ready); + __dprintf("%s: worker tid: %d\n", __FUNCTION__, tp->tid); + uti_desc = (struct uti_desc *)_uti_desc; if (!uti_desc) { fprintf(stderr, "%s: ERROR: uti_desc isn't set. Use mcexec.sh instead of mcexec\n", __FUNCTION__); @@ -2916,8 +2928,8 @@ static long util_thread(struct thread_data_s *my_thread, unsigned long uctx_pa, rctx = (char *)lctx + PAGE_SIZE; #endif /* POSTK_DEBUG_ARCH_DEP_35 */ - param[0] = (void *)uctx_pa; - param[1] = rctx; + param[0] = (void *)uctx_pa; /* Source address, kernel space */ + param[1] = rctx; /* Destination address, user space */ param[2] = lctx; param[4] = uti_desc->wp; #ifdef POSTK_DEBUG_ARCH_DEP_35 @@ -2925,12 +2937,19 @@ static long util_thread(struct thread_data_s *my_thread, unsigned long uctx_pa, #else /* POSTK_DEBUG_ARCH_DEP_35 */ param[5] = (void *)(PAGE_SIZE * 3); #endif /* POSTK_DEBUG_ARCH_DEP_35 */ + param[6] = (void *)tp->tid; + printf("%s: param[6]=%ld,tid=%ld\n", __FUNCTION__, (long)param[6], (long)((void *)tp->tid)); + + __dprintf("%s: before MCEXEC_UP_UTIL_THREAD1\n", __FUNCTION__); + + /* 1. Copy context from kernel space to user space + 2. Write uti_wp addr, its size for McKernel to uctx_pa + Note that this overwrites context */ if ((rc = ioctl(fd, MCEXEC_UP_UTIL_THREAD1, param)) == -1) { fprintf(stderr, "util_thread1: %d errno=%d\n", rc, errno); rc = -errno; goto out; } - create_worker_thread(NULL); /* Record the info of the thread migrating to Linux */ uti_desc->wp = uti_wp; @@ -3372,7 +3391,7 @@ int main_loop(struct thread_data_s *my_thread) tids[i++] = tp->tid; } - for (; i < ncpu; ++i) { + for (; i < w.sr.args[4]; ++i) { tids[i] = 0; } diff --git a/kernel/include/process.h b/kernel/include/process.h index 5d8abb0a..c714f718 100644 --- a/kernel/include/process.h +++ b/kernel/include/process.h @@ -242,6 +242,11 @@ enum mpol_rebind_step { #define SPAWN_TO_REMOTE 1 #define SPAWNING_TO_REMOTE 1001 +#define UTI_STATE_DEAD 0 +#define UTI_STATE_PROLOGUE 1 +#define UTI_STATE_RUNNING_IN_LINUX 2 +#define UTI_STATE_EPILOGUE 3 + #include #include @@ -690,10 +695,11 @@ struct thread { /* Syscall offload wait queue head */ struct waitq scd_wq; - int thread_offloaded; + int uti_state; int mod_clone; struct uti_attr *mod_clone_arg; int parent_cpuid; + int uti_refill_tid; // for performance counter unsigned long pmc_alloc_map; diff --git a/kernel/process.c b/kernel/process.c index 7e7ba63e..0ca55c0c 100644 --- a/kernel/process.c +++ b/kernel/process.c @@ -2623,6 +2623,21 @@ void __release_tid(struct process *proc, struct thread *thread) { } } +/* Replace tid specified by thread with tid specified by new_tid */ +void __find_and_replace_tid(struct process *proc, struct thread *thread, int new_tid) { + int i; + + for (i = 0; i < proc->nr_tids; ++i) { + if (proc->tids[i].thread != thread) continue; + + proc->tids[i].thread = NULL; + proc->tids[i].tid = new_tid; + kprintf("%s: tid %d (thread %p) has been relaced with tid %d\n", + __FUNCTION__, thread->tid, thread, new_tid); + break; + } +} + void destroy_thread(struct thread *thread) { struct sig_pending *pending; @@ -2646,7 +2661,11 @@ void destroy_thread(struct thread *thread) mcs_rwlock_writer_lock(&proc->threads_lock, &lock); list_del(&thread->siblings_list); - __release_tid(proc, thread); + if (thread->uti_state == UTI_STATE_EPILOGUE) { + __find_and_replace_tid(proc, thread, thread->uti_refill_tid); + } else { + __release_tid(proc, thread); + } mcs_rwlock_writer_unlock(&proc->threads_lock, &lock); mcs_rwlock_writer_unlock(&proc->update_lock, &updatelock); diff --git a/kernel/syscall.c b/kernel/syscall.c index 7aa557c2..72e7fcc5 100644 --- a/kernel/syscall.c +++ b/kernel/syscall.c @@ -9127,6 +9127,8 @@ int util_thread(struct uti_attr *arg) struct uti_attr attr; } kattr; + thread->uti_state = UTI_STATE_PROLOGUE; + context = (volatile unsigned long *)ihk_mc_alloc_pages(1, IHK_MC_AP_NOWAIT); if (!context) { @@ -9154,15 +9156,20 @@ int util_thread(struct uti_attr *arg) } request.args[3] = (unsigned long)uti_clv; request.args[4] = uti_desc; - thread->thread_offloaded = 1; + thread->uti_state = UTI_STATE_RUNNING_IN_LINUX; rc = do_syscall(&request, ihk_mc_get_processor_id(), 0); dkprintf("%s: returned from do_syscall,tid=%d,rc=%lx\n", __FUNCTION__, thread->tid, rc); + thread->uti_state = UTI_STATE_EPILOGUE; + util_show_syscall_profile(); - thread->thread_offloaded = 0; + /* These are written by mcexec_util_thread1() */ free_address = context[0]; free_size = context[1]; + thread->uti_refill_tid = context[2]; + dkprintf("%s: mcexec worker tid=%d\n", __FUNCTION__, context[2]); + ihk_mc_free_pages((void *)context, 1); kfree(uti_clv);