uti: futex: cache remote va to remote pa result

Change-Id: Idbbb3f2981b76a0235615fceaa6281d2c7134ca2

executer/kernel/mcctrl/control.c

@@ -1301,6 +1301,7 @@ void mcctrl_put_per_proc_data(struct mcctrl_per_proc_data *ppd)
 	ihk_ikc_spinlock_unlock(&ppd->wq_list_lock, flags);
 
 	pager_remove_process(ppd);
+	futex_remove_process(ppd);
 	kfree(ppd);
 }
 
@@ -1901,6 +1902,7 @@ int mcexec_create_per_process_data(ihk_os_t os,
 	spin_lock_init(&ppd->wq_list_lock);
 	memset(&ppd->cpu_set, 0, sizeof(cpumask_t));
 	ppd->ikc_target_cpu = 0;
+	ppd->rva_to_rpa_cache = RB_ROOT;
 	/* Final ref will be dropped in release_handler() through
 	 * mcexec_destroy_per_process_data() */
 	atomic_set(&ppd->refcount, 1);
@@ -3014,6 +3016,8 @@ void mcctrl_futex_wake(struct ikc_scd_packet *pisp)
 	}
 
 	resp->done = 1;
+	dprintk("%s: cpu: %d\n", __func__, ihk_ikc_get_processor_id());
+
 	wake_up_interruptible(&resp->wq);
 }
 

executer/kernel/mcctrl/futex.c

@@ -10,6 +10,7 @@
 #include <linux/semaphore.h>
 #include <linux/interrupt.h>
 #include <linux/cpumask.h>
+#include <linux/rbtree.h>
 #include <asm/uaccess.h>
 #include <asm/delay.h>
 #include <asm/io.h>
@@ -213,6 +214,76 @@ retry_alloc:
 	return error;
 }
 
+struct rva_to_rpa_cache_node {
+	struct rb_node node;
+	unsigned long rva;
+	unsigned long rpa;
+};
+
+void futex_remove_process(struct mcctrl_per_proc_data *ppd)
+{
+	struct rb_node *node;
+
+	while ((node = rb_first(&ppd->rva_to_rpa_cache))) {
+		struct rva_to_rpa_cache_node *cache_node;
+
+		cache_node = container_of(node, struct rva_to_rpa_cache_node,
+					  node);
+		rb_erase(node, &ppd->rva_to_rpa_cache);
+		kfree(cache_node);
+	}
+}
+
+struct rva_to_rpa_cache_node *rva_to_rpa_cache_search(struct rb_root *root,
+						      unsigned long rva)
+{
+	struct rb_node **iter = &root->rb_node, *parent = NULL;
+
+	while (*iter) {
+		struct rva_to_rpa_cache_node *inode =
+			container_of(*iter, struct rva_to_rpa_cache_node, node);
+
+		parent = *iter;
+
+		if (rva == inode->rva) {
+			return inode;
+		}
+
+		if (rva < inode->rva)
+			iter = &((*iter)->rb_left);
+		else
+			iter = &((*iter)->rb_right);
+	}
+
+	return NULL;
+}
+
+int rva_to_rpa_cache_insert(struct rb_root *root,
+			    struct rva_to_rpa_cache_node *cache_node)
+{
+	struct rb_node **iter = &root->rb_node, *parent = NULL;
+
+	while (*iter) {
+		struct rva_to_rpa_cache_node *inode =
+			container_of(*iter, struct rva_to_rpa_cache_node, node);
+
+		parent = *iter;
+
+		if (cache_node->rva == inode->rva)
+			return -EINVAL;
+
+		if (cache_node->rva < inode->rva)
+			iter = &((*iter)->rb_left);
+		else
+			iter = &((*iter)->rb_right);
+	}
+
+	rb_link_node(&cache_node->node, parent, iter);
+	rb_insert_color(&cache_node->node, root);
+
+	return 0;
+}
+
 /**
  * get_futex_key() - Get parameters which are the keys for a futex
  * @uaddr:	virtual address of the futex
@@ -238,6 +309,7 @@ get_futex_key(uint32_t *uaddr, int fshared, union futex_key *key,
 	struct mcctrl_usrdata *usrdata;
 	struct mcctrl_per_proc_data *ppd;
 	int ret = 0, error = 0;
+	struct rva_to_rpa_cache_node *cache_node;
 
 	/*
 	 * The futex address must be "naturally" aligned.
@@ -281,6 +353,17 @@ get_futex_key(uint32_t *uaddr, int fshared, union futex_key *key,
 		goto out;
 	}
 
+	/* cache because translate_rva_to_rpa calls smp_ihk_arch_dcache_flush
+	 * via ihk_device_unmap_virtual
+	 */
+	cache_node = rva_to_rpa_cache_search(&ppd->rva_to_rpa_cache,
+					     (unsigned long)uaddr);
+	if (cache_node) {
+		phys = cache_node->rpa;
+		dprintk("%s: cache hit, rva: %lx, rpa: %lx\n",
+			__func__, (unsigned long)uaddr, phys);
+		goto found;
+	}
 retry_v2p:
 	error = translate_rva_to_rpa((ihk_os_t)uti_info->os, ppd->rpgtable,
 			(unsigned long)uaddr, &phys, &pgsize);
@@ -299,6 +382,23 @@ retry_v2p:
 		goto retry_v2p;
 	}
 
+	cache_node = kmalloc(sizeof(struct rva_to_rpa_cache_node), GFP_KERNEL);
+	if (!cache_node) {
+		ret = -ENOMEM;
+		goto put_out;
+	}
+	cache_node->rva = (unsigned long)uaddr;
+	cache_node->rpa = phys;
+	dprintk("%s: cache insert, rva: %lx, rpa: %lx\n",
+		__func__, (unsigned long)uaddr, phys);
+	ret = rva_to_rpa_cache_insert(&ppd->rva_to_rpa_cache, cache_node);
+	if (ret) {
+		pr_err("%s: error: cache entry found, rva: %lx, rpa: %lx\n",
+		       __func__, (unsigned long)uaddr, phys);
+		goto put_out;
+	}
+
+ found:
 	key->shared.phys = (void *)phys;
 	key->shared.pgoff = 0;
 
@@ -444,8 +544,7 @@ static int uti_sched_wakeup_thread(struct futex_q *q, int valid_states,
 	irqstate = ihk_mc_spinlock_lock(
 			(_ihk_spinlock_t *)q->th_spin_sleep_lock);
 	if (*(int *)q->th_spin_sleep == 1) {
-		dprintk("%s: spin wakeup: cpu_id: %d\n",
+		dprintk("%s: spin wakeup: cpu_id: %d\n", __func__, uti_info->cpu);
-				__func__, uti_info->cpu);
 		status = 0;
 	}
 	*(int *)q->th_spin_sleep = 0;
@@ -477,11 +576,11 @@ static int uti_sched_wakeup_thread(struct futex_q *q, int valid_states,
 	ihk_mc_spinlock_unlock((_ihk_spinlock_t *)q->runq_lock, irqstate);
 
 	if (!status) {
-		dprintk("%s: issuing IPI, thread->cpu_id=%d\n",
+		dprintk("%s: issuing IPI, thread->cpu_id=%d, intr_id: %d\n",
-				__func__, uti_info->cpu);
+			__func__, uti_info->cpu, q->intr_id);
 
-		ihk_os_issue_interrupt(uti_info->os,
+		ihk_os_issue_interrupt(uti_info->os, q->intr_id,
-				q->intr_id, q->intr_vector);
+				       q->intr_vector);
 	}
 
 	return status;
@@ -582,6 +681,7 @@ static int64_t futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q
 {
 	int64_t time_remain = 0;
 	unsigned long irqstate;
+
 	/*
 	 * The task state is guaranteed to be set before another task can
 	 * wake it.
@@ -606,8 +706,8 @@ static int64_t futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q
 	queue_me(q, hb, uti_info);
 
 	if (!mc_plist_node_empty(&q->list)) {
-		dprintk("%s: tid: %d is trying to sleep\n", __func__,
+		dprintk("%s: tid: %d is trying to sleep, cpu: %d\n",
-				uti_info->tid);
+			__func__, uti_info->tid, ihk_ikc_get_processor_id());
 		/* Note that the unit of timeout is nsec */
 		time_remain = uti_wait_event(q->uti_futex_resp, timeout);
 
@@ -619,7 +719,8 @@ static int64_t futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q
 				pr_err("%s: ERROR: wait_event returned %lld\n", __func__, time_remain);
 			}
 		}
-		dprintk("%s: tid: %d woken up\n", __func__, uti_info->tid);
+		dprintk("%s: tid: %d woken up, cpu: %d\n",
+			__func__, uti_info->tid, ihk_ikc_get_processor_id());
 	}
 
 	/* This does not need to be serialized */
@@ -1052,7 +1153,7 @@ long do_futex(int n, unsigned long arg0, unsigned long arg1,
 	}
 	op = (op & FUTEX_CMD_MASK);
 
-	dprintk("futex op=[%x, %s],uaddr=%lx, val=%x, utime=%p, uaddr2=%p, val3=%x, []=%x, shared: %d\n",
+	dprintk("futex op=[%x, %s],uaddr=%lx, val=%x, utime=%p, uaddr2=%p, val3=%x, shared: %d\n",
 			flags,
 			(op == FUTEX_WAIT) ? "FUTEX_WAIT" :
 			(op == FUTEX_WAIT_BITSET) ? "FUTEX_WAIT_BITSET" :
@@ -1061,7 +1162,7 @@ long do_futex(int n, unsigned long arg0, unsigned long arg1,
 			(op == FUTEX_WAKE_BITSET) ? "FUTEX_WAKE_BITSET" :
 			(op == FUTEX_CMP_REQUEUE) ? "FUTEX_CMP_REQUEUE" :
 			(op == FUTEX_REQUEUE) ? "FUTEX_REQUEUE (NOT IMPL!)" : "unknown",
-			(unsigned long)uaddr, val, utime, uaddr2, val3, *uaddr, fshared);
+			(unsigned long)uaddr, val, utime, uaddr2, val3, fshared);
 
 	if (utime && (op == FUTEX_WAIT_BITSET || op == FUTEX_WAIT)) {
 		if (copy_from_user(&ts, utime, sizeof(ts)) != 0) {
@@ -1100,7 +1201,7 @@ long do_futex(int n, unsigned long arg0, unsigned long arg1,
 	ret = futex(uaddr, op, val, timeout, uaddr2,
 			val2, val3, fshared, uti_info);
 
-	dprintk("futex op=[%x, %s],uaddr=%lx, val=%x, utime=%p, uaddr2=%p, val3=%x, []=%x, shared: %d, ret: %d\n",
+	dprintk("futex op=[%x, %s],uaddr=%lx, val=%x, utime=%p, uaddr2=%p, val3=%x, shared: %d, ret: %d\n",
 			op,
 			(op == FUTEX_WAIT) ? "FUTEX_WAIT" :
 			(op == FUTEX_WAIT_BITSET) ? "FUTEX_WAIT_BITSET" :
@@ -1109,7 +1210,7 @@ long do_futex(int n, unsigned long arg0, unsigned long arg1,
 			(op == FUTEX_WAKE_BITSET) ? "FUTEX_WAKE_BITSET" :
 			(op == FUTEX_CMP_REQUEUE) ? "FUTEX_CMP_REQUEUE" :
 			(op == FUTEX_REQUEUE) ? "FUTEX_REQUEUE (NOT IMPL!)" : "unknown",
-			(unsigned long)uaddr, val, utime, uaddr2, val3, *uaddr, fshared, ret);
+			(unsigned long)uaddr, val, utime, uaddr2, val3, fshared, ret);
 
 	return ret;
 }

executer/kernel/mcctrl/include/futex.h

@@ -166,4 +166,6 @@ long do_futex(int n, unsigned long arg0, unsigned long arg1,
 		struct uti_info *uti_info,
 		void *uti_futex_resp);
 
+void futex_remove_process(struct mcctrl_per_proc_data *ppd);
+
 #endif

executer/kernel/mcctrl/mcctrl.h

@@ -264,6 +264,8 @@ struct mcctrl_per_proc_data {
 	struct list_head devobj_pager_list;
 	struct semaphore devobj_pager_lock;
 	int enable_tofu;
+
+	struct rb_root rva_to_rpa_cache;
 };
 
 struct sysfsm_req {

kernel/futex.c

@@ -239,7 +239,7 @@ static void wake_futex(struct futex_q *q)
 	if (q->uti_futex_resp) { 
 		int rc;
 		dkprintf("%s: waking up migrated-to-Linux thread (tid %d),uti_futex_resp=%p\n",
-				__func__, p->tid, q->uti_futex_resp);
+			__func__, p->tid, q->uti_futex_resp);
 
 		struct ikc_scd_packet pckt;
 		struct ihk_ikc_channel_desc *resp_channel = cpu_local_var(ikc2linux);

kernel/process.c

@@ -3649,6 +3649,8 @@ void spin_sleep_or_schedule(void)
 		}
 
 		if (woken) {
+			dkprintf("%s: woken while spinning, cpu: %d, do_schedule: %d\n",
+				 __func__, ihk_ikc_get_processor_id(), do_schedule);
 			if (do_schedule) {
 				irqstate = ihk_mc_spinlock_lock(&v->runq_lock);
 				v->flags |= CPU_FLAG_NEED_RESCHED;
@@ -3667,6 +3669,8 @@ void spin_sleep_or_schedule(void)
 
 out_schedule:
 	schedule();
+	dkprintf("%s: woken while sleeping, cpu: %d\n",
+		 __func__, ihk_ikc_get_processor_id());
 }
 
 void schedule(void)
@@ -3680,7 +3684,7 @@ void schedule(void)
 
 	if (cpu_local_var(no_preempt)) {
 		kprintf("%s: WARNING can't schedule() while no preemption, cnt: %d\n",
-			__FUNCTION__, cpu_local_var(no_preempt));
+			__func__, cpu_local_var(no_preempt));
 
 		irqstate = cpu_disable_interrupt_save();
 		ihk_mc_spinlock_lock_noirq(

test/issues/1428/C1428.sh

@@ -2,6 +2,7 @@
 
 USELTP=1
 USEOSTEST=0
+MCREBOOT=0
 
 . ../../common.sh
 
@@ -23,6 +24,7 @@ pushd ${UTI_TEST_DIR}
 make
 popd
 
+mcreboot
 for tno in `seq 12 20`
 do
 	tname=`printf "C${issue}T%02d" ${tid}`
@@ -46,6 +48,7 @@ for tno in `seq 31 34`
 do
 	sudo ${UTI_TEST_DIR}/CT${tno} -l &> ./lnx_CT${tno}.txt
 done
+
 echo "*** Boot mckernel"
 mcreboot
 echo ""
@@ -84,4 +87,3 @@ do
 	let tid++
 	echo ""
 done
-

test/uti/arm64/CT13.c

@@ -40,8 +40,8 @@ void *util_thread(void *arg)
 			FUTEX_WAKE, 1, NULL, NULL, 0);
 	if (rc != 1) {
 		fprintf(stderr,
-			"CT13101 FUTEX_WAKE NG (%d,%s)\n",
+			"CT13101 FUTEX_WAKE NG (%d,%d)\n",
-			rc, strerror(errno));
+			rc, errno);
 	}
 	else {
 		fprintf(stderr, "CT13101 FUTEX_WAKE OK\n");

test/uti/arm64/util.c

@@ -20,11 +20,11 @@
 static inline void FIXED_SIZE_WORK(unsigned long *ptr)
 {
 	asm volatile("mov %x0, x20\n"
-			 "add x20, x20, #1\n"
+		     "add x20, x20, #1\n"
-			 "mov x20, %x0\n"
+		     "mov x20, %x0\n"
-			 : "+rm" (*ptr)
+		     : "+rm" (*ptr)
-			 :
+		     :
-			 : "x20", "cc", "memory");
+		     : "x20", "cc", "memory");
 }
 
 static inline void BULK_FSW(unsigned long n,