diff --git a/executer/kernel/mcctrl/control.c b/executer/kernel/mcctrl/control.c index 19ae61bc..6b2f603c 100644 --- a/executer/kernel/mcctrl/control.c +++ b/executer/kernel/mcctrl/control.c @@ -1301,6 +1301,7 @@ void mcctrl_put_per_proc_data(struct mcctrl_per_proc_data *ppd) ihk_ikc_spinlock_unlock(&ppd->wq_list_lock, flags); pager_remove_process(ppd); + futex_remove_process(ppd); kfree(ppd); } @@ -1901,6 +1902,7 @@ int mcexec_create_per_process_data(ihk_os_t os, spin_lock_init(&ppd->wq_list_lock); memset(&ppd->cpu_set, 0, sizeof(cpumask_t)); ppd->ikc_target_cpu = 0; + ppd->rva_to_rpa_cache = RB_ROOT; /* Final ref will be dropped in release_handler() through * mcexec_destroy_per_process_data() */ atomic_set(&ppd->refcount, 1); @@ -3014,6 +3016,8 @@ void mcctrl_futex_wake(struct ikc_scd_packet *pisp) } resp->done = 1; + dprintk("%s: cpu: %d\n", __func__, ihk_ikc_get_processor_id()); + wake_up_interruptible(&resp->wq); } diff --git a/executer/kernel/mcctrl/futex.c b/executer/kernel/mcctrl/futex.c index 2a986a17..060f90bb 100644 --- a/executer/kernel/mcctrl/futex.c +++ b/executer/kernel/mcctrl/futex.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -213,6 +214,76 @@ retry_alloc: return error; } +struct rva_to_rpa_cache_node { + struct rb_node node; + unsigned long rva; + unsigned long rpa; +}; + +void futex_remove_process(struct mcctrl_per_proc_data *ppd) +{ + struct rb_node *node; + + while ((node = rb_first(&ppd->rva_to_rpa_cache))) { + struct rva_to_rpa_cache_node *cache_node; + + cache_node = container_of(node, struct rva_to_rpa_cache_node, + node); + rb_erase(node, &ppd->rva_to_rpa_cache); + kfree(cache_node); + } +} + +struct rva_to_rpa_cache_node *rva_to_rpa_cache_search(struct rb_root *root, + unsigned long rva) +{ + struct rb_node **iter = &root->rb_node, *parent = NULL; + + while (*iter) { + struct rva_to_rpa_cache_node *inode = + container_of(*iter, struct rva_to_rpa_cache_node, node); + + parent = *iter; + + if (rva == inode->rva) { + return inode; + } + + if (rva < inode->rva) + iter = &((*iter)->rb_left); + else + iter = &((*iter)->rb_right); + } + + return NULL; +} + +int rva_to_rpa_cache_insert(struct rb_root *root, + struct rva_to_rpa_cache_node *cache_node) +{ + struct rb_node **iter = &root->rb_node, *parent = NULL; + + while (*iter) { + struct rva_to_rpa_cache_node *inode = + container_of(*iter, struct rva_to_rpa_cache_node, node); + + parent = *iter; + + if (cache_node->rva == inode->rva) + return -EINVAL; + + if (cache_node->rva < inode->rva) + iter = &((*iter)->rb_left); + else + iter = &((*iter)->rb_right); + } + + rb_link_node(&cache_node->node, parent, iter); + rb_insert_color(&cache_node->node, root); + + return 0; +} + /** * get_futex_key() - Get parameters which are the keys for a futex * @uaddr: virtual address of the futex @@ -238,6 +309,7 @@ get_futex_key(uint32_t *uaddr, int fshared, union futex_key *key, struct mcctrl_usrdata *usrdata; struct mcctrl_per_proc_data *ppd; int ret = 0, error = 0; + struct rva_to_rpa_cache_node *cache_node; /* * The futex address must be "naturally" aligned. @@ -281,6 +353,17 @@ get_futex_key(uint32_t *uaddr, int fshared, union futex_key *key, goto out; } + /* cache because translate_rva_to_rpa calls smp_ihk_arch_dcache_flush + * via ihk_device_unmap_virtual + */ + cache_node = rva_to_rpa_cache_search(&ppd->rva_to_rpa_cache, + (unsigned long)uaddr); + if (cache_node) { + phys = cache_node->rpa; + dprintk("%s: cache hit, rva: %lx, rpa: %lx\n", + __func__, (unsigned long)uaddr, phys); + goto found; + } retry_v2p: error = translate_rva_to_rpa((ihk_os_t)uti_info->os, ppd->rpgtable, (unsigned long)uaddr, &phys, &pgsize); @@ -299,6 +382,23 @@ retry_v2p: goto retry_v2p; } + cache_node = kmalloc(sizeof(struct rva_to_rpa_cache_node), GFP_KERNEL); + if (!cache_node) { + ret = -ENOMEM; + goto put_out; + } + cache_node->rva = (unsigned long)uaddr; + cache_node->rpa = phys; + dprintk("%s: cache insert, rva: %lx, rpa: %lx\n", + __func__, (unsigned long)uaddr, phys); + ret = rva_to_rpa_cache_insert(&ppd->rva_to_rpa_cache, cache_node); + if (ret) { + pr_err("%s: error: cache entry found, rva: %lx, rpa: %lx\n", + __func__, (unsigned long)uaddr, phys); + goto put_out; + } + + found: key->shared.phys = (void *)phys; key->shared.pgoff = 0; @@ -444,8 +544,7 @@ static int uti_sched_wakeup_thread(struct futex_q *q, int valid_states, irqstate = ihk_mc_spinlock_lock( (_ihk_spinlock_t *)q->th_spin_sleep_lock); if (*(int *)q->th_spin_sleep == 1) { - dprintk("%s: spin wakeup: cpu_id: %d\n", - __func__, uti_info->cpu); + dprintk("%s: spin wakeup: cpu_id: %d\n", __func__, uti_info->cpu); status = 0; } *(int *)q->th_spin_sleep = 0; @@ -477,11 +576,11 @@ static int uti_sched_wakeup_thread(struct futex_q *q, int valid_states, ihk_mc_spinlock_unlock((_ihk_spinlock_t *)q->runq_lock, irqstate); if (!status) { - dprintk("%s: issuing IPI, thread->cpu_id=%d\n", - __func__, uti_info->cpu); + dprintk("%s: issuing IPI, thread->cpu_id=%d, intr_id: %d\n", + __func__, uti_info->cpu, q->intr_id); - ihk_os_issue_interrupt(uti_info->os, - q->intr_id, q->intr_vector); + ihk_os_issue_interrupt(uti_info->os, q->intr_id, + q->intr_vector); } return status; @@ -582,6 +681,7 @@ static int64_t futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q { int64_t time_remain = 0; unsigned long irqstate; + /* * The task state is guaranteed to be set before another task can * wake it. @@ -606,8 +706,8 @@ static int64_t futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q queue_me(q, hb, uti_info); if (!mc_plist_node_empty(&q->list)) { - dprintk("%s: tid: %d is trying to sleep\n", __func__, - uti_info->tid); + dprintk("%s: tid: %d is trying to sleep, cpu: %d\n", + __func__, uti_info->tid, ihk_ikc_get_processor_id()); /* Note that the unit of timeout is nsec */ time_remain = uti_wait_event(q->uti_futex_resp, timeout); @@ -619,7 +719,8 @@ static int64_t futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q pr_err("%s: ERROR: wait_event returned %lld\n", __func__, time_remain); } } - dprintk("%s: tid: %d woken up\n", __func__, uti_info->tid); + dprintk("%s: tid: %d woken up, cpu: %d\n", + __func__, uti_info->tid, ihk_ikc_get_processor_id()); } /* This does not need to be serialized */ @@ -1052,7 +1153,7 @@ long do_futex(int n, unsigned long arg0, unsigned long arg1, } op = (op & FUTEX_CMD_MASK); - dprintk("futex op=[%x, %s],uaddr=%lx, val=%x, utime=%p, uaddr2=%p, val3=%x, []=%x, shared: %d\n", + dprintk("futex op=[%x, %s],uaddr=%lx, val=%x, utime=%p, uaddr2=%p, val3=%x, shared: %d\n", flags, (op == FUTEX_WAIT) ? "FUTEX_WAIT" : (op == FUTEX_WAIT_BITSET) ? "FUTEX_WAIT_BITSET" : @@ -1061,7 +1162,7 @@ long do_futex(int n, unsigned long arg0, unsigned long arg1, (op == FUTEX_WAKE_BITSET) ? "FUTEX_WAKE_BITSET" : (op == FUTEX_CMP_REQUEUE) ? "FUTEX_CMP_REQUEUE" : (op == FUTEX_REQUEUE) ? "FUTEX_REQUEUE (NOT IMPL!)" : "unknown", - (unsigned long)uaddr, val, utime, uaddr2, val3, *uaddr, fshared); + (unsigned long)uaddr, val, utime, uaddr2, val3, fshared); if (utime && (op == FUTEX_WAIT_BITSET || op == FUTEX_WAIT)) { if (copy_from_user(&ts, utime, sizeof(ts)) != 0) { @@ -1100,7 +1201,7 @@ long do_futex(int n, unsigned long arg0, unsigned long arg1, ret = futex(uaddr, op, val, timeout, uaddr2, val2, val3, fshared, uti_info); - dprintk("futex op=[%x, %s],uaddr=%lx, val=%x, utime=%p, uaddr2=%p, val3=%x, []=%x, shared: %d, ret: %d\n", + dprintk("futex op=[%x, %s],uaddr=%lx, val=%x, utime=%p, uaddr2=%p, val3=%x, shared: %d, ret: %d\n", op, (op == FUTEX_WAIT) ? "FUTEX_WAIT" : (op == FUTEX_WAIT_BITSET) ? "FUTEX_WAIT_BITSET" : @@ -1109,7 +1210,7 @@ long do_futex(int n, unsigned long arg0, unsigned long arg1, (op == FUTEX_WAKE_BITSET) ? "FUTEX_WAKE_BITSET" : (op == FUTEX_CMP_REQUEUE) ? "FUTEX_CMP_REQUEUE" : (op == FUTEX_REQUEUE) ? "FUTEX_REQUEUE (NOT IMPL!)" : "unknown", - (unsigned long)uaddr, val, utime, uaddr2, val3, *uaddr, fshared, ret); + (unsigned long)uaddr, val, utime, uaddr2, val3, fshared, ret); return ret; } diff --git a/executer/kernel/mcctrl/include/futex.h b/executer/kernel/mcctrl/include/futex.h index 11824747..3ab83081 100644 --- a/executer/kernel/mcctrl/include/futex.h +++ b/executer/kernel/mcctrl/include/futex.h @@ -166,4 +166,6 @@ long do_futex(int n, unsigned long arg0, unsigned long arg1, struct uti_info *uti_info, void *uti_futex_resp); +void futex_remove_process(struct mcctrl_per_proc_data *ppd); + #endif diff --git a/executer/kernel/mcctrl/mcctrl.h b/executer/kernel/mcctrl/mcctrl.h index f0f320c8..eefca7ff 100644 --- a/executer/kernel/mcctrl/mcctrl.h +++ b/executer/kernel/mcctrl/mcctrl.h @@ -264,6 +264,8 @@ struct mcctrl_per_proc_data { struct list_head devobj_pager_list; struct semaphore devobj_pager_lock; int enable_tofu; + + struct rb_root rva_to_rpa_cache; }; struct sysfsm_req { diff --git a/kernel/futex.c b/kernel/futex.c index abef2d23..7b5a91f9 100644 --- a/kernel/futex.c +++ b/kernel/futex.c @@ -239,7 +239,7 @@ static void wake_futex(struct futex_q *q) if (q->uti_futex_resp) { int rc; dkprintf("%s: waking up migrated-to-Linux thread (tid %d),uti_futex_resp=%p\n", - __func__, p->tid, q->uti_futex_resp); + __func__, p->tid, q->uti_futex_resp); struct ikc_scd_packet pckt; struct ihk_ikc_channel_desc *resp_channel = cpu_local_var(ikc2linux); diff --git a/kernel/process.c b/kernel/process.c index 2ef74a0d..fbde6c71 100644 --- a/kernel/process.c +++ b/kernel/process.c @@ -3649,6 +3649,8 @@ void spin_sleep_or_schedule(void) } if (woken) { + dkprintf("%s: woken while spinning, cpu: %d, do_schedule: %d\n", + __func__, ihk_ikc_get_processor_id(), do_schedule); if (do_schedule) { irqstate = ihk_mc_spinlock_lock(&v->runq_lock); v->flags |= CPU_FLAG_NEED_RESCHED; @@ -3667,6 +3669,8 @@ void spin_sleep_or_schedule(void) out_schedule: schedule(); + dkprintf("%s: woken while sleeping, cpu: %d\n", + __func__, ihk_ikc_get_processor_id()); } void schedule(void) @@ -3680,7 +3684,7 @@ void schedule(void) if (cpu_local_var(no_preempt)) { kprintf("%s: WARNING can't schedule() while no preemption, cnt: %d\n", - __FUNCTION__, cpu_local_var(no_preempt)); + __func__, cpu_local_var(no_preempt)); irqstate = cpu_disable_interrupt_save(); ihk_mc_spinlock_lock_noirq( diff --git a/test/issues/1428/C1428.sh b/test/issues/1428/C1428.sh index f2df5f64..8ee08dfb 100755 --- a/test/issues/1428/C1428.sh +++ b/test/issues/1428/C1428.sh @@ -2,6 +2,7 @@ USELTP=1 USEOSTEST=0 +MCREBOOT=0 . ../../common.sh @@ -23,6 +24,7 @@ pushd ${UTI_TEST_DIR} make popd +mcreboot for tno in `seq 12 20` do tname=`printf "C${issue}T%02d" ${tid}` @@ -46,6 +48,7 @@ for tno in `seq 31 34` do sudo ${UTI_TEST_DIR}/CT${tno} -l &> ./lnx_CT${tno}.txt done + echo "*** Boot mckernel" mcreboot echo "" @@ -84,4 +87,3 @@ do let tid++ echo "" done - diff --git a/test/uti/arm64/CT13.c b/test/uti/arm64/CT13.c index 8e12c0f5..982401d7 100644 --- a/test/uti/arm64/CT13.c +++ b/test/uti/arm64/CT13.c @@ -40,8 +40,8 @@ void *util_thread(void *arg) FUTEX_WAKE, 1, NULL, NULL, 0); if (rc != 1) { fprintf(stderr, - "CT13101 FUTEX_WAKE NG (%d,%s)\n", - rc, strerror(errno)); + "CT13101 FUTEX_WAKE NG (%d,%d)\n", + rc, errno); } else { fprintf(stderr, "CT13101 FUTEX_WAKE OK\n"); diff --git a/test/uti/arm64/util.c b/test/uti/arm64/util.c index 59e0b9ff..c5bf3fa7 100644 --- a/test/uti/arm64/util.c +++ b/test/uti/arm64/util.c @@ -20,11 +20,11 @@ static inline void FIXED_SIZE_WORK(unsigned long *ptr) { asm volatile("mov %x0, x20\n" - "add x20, x20, #1\n" - "mov x20, %x0\n" - : "+rm" (*ptr) - : - : "x20", "cc", "memory"); + "add x20, x20, #1\n" + "mov x20, %x0\n" + : "+rm" (*ptr) + : + : "x20", "cc", "memory"); } static inline void BULK_FSW(unsigned long n,