uti: futex call function in mcctrl
Previously, futex code of McKerenl was called by mccontrol, but there ware some problems with this method. (Mainly, location of McKernel image on memory) Call futex code in mcctrl instead of the one in McKernel image, giving the following benefits: 1. Not relying on shared kernel virtual address space with Linux any more 2. The cpu id store / retrieve is not needed and resulting in the code Change-Id: Ic40929b64a655b270c435859fa287fedb713ee5c refe: #1428
This commit is contained in:
Ken Sato
committed by
Masamichi Takagi
Masamichi Takagi
parent
35296c8210
commit
a9973e913d
202
kernel/futex.c
202
kernel/futex.c
@ -62,7 +62,7 @@
|
||||
|
||||
#include <process.h>
|
||||
#include <futex.h>
|
||||
#include <jhash.h>
|
||||
#include <mc_jhash.h>
|
||||
#include <ihk/lock.h>
|
||||
#include <ihk/atomic.h>
|
||||
#include <list.h>
|
||||
@ -72,39 +72,24 @@
|
||||
#include <timer.h>
|
||||
#include <ihk/debug.h>
|
||||
#include <syscall.h>
|
||||
|
||||
//#define DEBUG_PRINT_FUTEX
|
||||
|
||||
#ifdef DEBUG_PRINT_FUTEX
|
||||
#undef DDEBUG_DEFAULT
|
||||
#define DDEBUG_DEFAULT DDEBUG_PRINT
|
||||
#define uti_dkprintf(...) do { ((clv_override && linux_printk) ? (*linux_printk) : kprintf)(__VA_ARGS__); } while (0)
|
||||
#else
|
||||
#define uti_dkprintf(...) do { } while (0)
|
||||
#endif
|
||||
#define uti_kprintf(...) do { ((clv_override && linux_printk) ? (*linux_printk) : kprintf)(__VA_ARGS__); } while (0)
|
||||
#include <kmalloc.h>
|
||||
|
||||
|
||||
unsigned long ihk_mc_get_ns_per_tsc(void);
|
||||
|
||||
/*
|
||||
* Hash buckets are shared by all the futex_keys that hash to the same
|
||||
* location. Each key may have multiple futex_q structures, one for each task
|
||||
* waiting on a futex.
|
||||
*/
|
||||
struct futex_hash_bucket {
|
||||
ihk_spinlock_t lock;
|
||||
struct plist_head chain;
|
||||
};
|
||||
struct futex_hash_bucket *futex_queues;
|
||||
|
||||
static struct futex_hash_bucket futex_queues[1<<FUTEX_HASHBITS];
|
||||
struct futex_hash_bucket *get_futex_queues(void)
|
||||
{
|
||||
return futex_queues;
|
||||
}
|
||||
|
||||
/*
|
||||
* We hash on the keys returned from get_futex_key (see below).
|
||||
*/
|
||||
static struct futex_hash_bucket *hash_futex(union futex_key *key)
|
||||
{
|
||||
uint32_t hash = jhash2((uint32_t*)&key->both.word,
|
||||
uint32_t hash = mc_jhash2((uint32_t *)&key->both.word,
|
||||
(sizeof(key->both.word)+sizeof(key->both.ptr))/4,
|
||||
key->both.offset);
|
||||
return &futex_queues[hash & ((1 << FUTEX_HASHBITS)-1)];
|
||||
@ -157,11 +142,11 @@ static void drop_futex_key_refs(union futex_key *key)
|
||||
* lock_page() might sleep, the caller should not hold a spinlock.
|
||||
*/
|
||||
static int
|
||||
get_futex_key(uint32_t *uaddr, int fshared, union futex_key *key, struct cpu_local_var *clv_override)
|
||||
get_futex_key(uint32_t *uaddr, int fshared, union futex_key *key)
|
||||
{
|
||||
unsigned long address = (unsigned long)uaddr;
|
||||
unsigned long phys;
|
||||
struct thread *thread = cpu_local_var_with_override(current, clv_override);
|
||||
struct thread *thread = cpu_local_var(current);
|
||||
struct process_vm *mm = thread->vm;
|
||||
|
||||
/*
|
||||
@ -228,7 +213,7 @@ static int cmpxchg_futex_value_locked(uint32_t __user *uaddr, uint32_t uval, uin
|
||||
* The hash bucket lock must be held when this is called.
|
||||
* Afterwards, the futex_q must not be accessed.
|
||||
*/
|
||||
static void wake_futex(struct futex_q *q, struct cpu_local_var *clv_override)
|
||||
static void wake_futex(struct futex_q *q)
|
||||
{
|
||||
struct thread *p = q->task;
|
||||
|
||||
@ -253,26 +238,22 @@ static void wake_futex(struct futex_q *q, struct cpu_local_var *clv_override)
|
||||
|
||||
if (q->uti_futex_resp) {
|
||||
int rc;
|
||||
uti_dkprintf("wake_futex(): waking up migrated-to-Linux thread (tid %d),uti_futex_resp=%p\n", p->tid, q->uti_futex_resp);
|
||||
/* TODO: Add the case when a Linux thread waking up another Linux thread */
|
||||
if (clv_override) {
|
||||
uti_dkprintf("%s: ERROR: A Linux thread is waking up migrated-to-Linux thread\n", __FUNCTION__);
|
||||
}
|
||||
if (p->spin_sleep == 0) {
|
||||
uti_dkprintf("%s: INFO: woken up by someone else\n", __FUNCTION__);
|
||||
}
|
||||
dkprintf("%s: waking up migrated-to-Linux thread (tid %d),uti_futex_resp=%p\n",
|
||||
__func__, p->tid, q->uti_futex_resp);
|
||||
|
||||
struct ikc_scd_packet pckt;
|
||||
struct ihk_ikc_channel_desc *resp_channel = cpu_local_var_with_override(ikc2linux, clv_override);
|
||||
struct ihk_ikc_channel_desc *resp_channel = cpu_local_var(ikc2linux);
|
||||
pckt.msg = SCD_MSG_FUTEX_WAKE;
|
||||
pckt.futex.resp = q->uti_futex_resp;
|
||||
pckt.futex.spin_sleep = &p->spin_sleep;
|
||||
rc = ihk_ikc_send(resp_channel, &pckt, 0);
|
||||
if (rc) {
|
||||
uti_dkprintf("%s: ERROR: ihk_ikc_send returned %d, resp_channel=%p\n", __FUNCTION__, rc, resp_channel);
|
||||
dkprintf("%s: ERROR: ihk_ikc_send returned %d, resp_channel=%p\n",
|
||||
__func__, rc, resp_channel);
|
||||
}
|
||||
} else {
|
||||
uti_dkprintf("wake_futex(): waking up McKernel thread (tid %d)\n", p->tid);
|
||||
dkprintf("%s: waking up McKernel thread (tid %d)\n",
|
||||
__func__, p->tid);
|
||||
sched_wakeup_thread(p, PS_NORMAL);
|
||||
}
|
||||
}
|
||||
@ -304,7 +285,8 @@ double_unlock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2)
|
||||
/*
|
||||
* Wake up waiters matching bitset queued on this futex (uaddr).
|
||||
*/
|
||||
static int futex_wake(uint32_t *uaddr, int fshared, int nr_wake, uint32_t bitset, struct cpu_local_var *clv_override)
|
||||
static int futex_wake(uint32_t *uaddr, int fshared, int nr_wake,
|
||||
uint32_t bitset)
|
||||
{
|
||||
struct futex_hash_bucket *hb;
|
||||
struct futex_q *this, *next;
|
||||
@ -316,7 +298,7 @@ static int futex_wake(uint32_t *uaddr, int fshared, int nr_wake, uint32_t bitset
|
||||
if (!bitset)
|
||||
return -EINVAL;
|
||||
|
||||
ret = get_futex_key(uaddr, fshared, &key, clv_override);
|
||||
ret = get_futex_key(uaddr, fshared, &key);
|
||||
if ((ret != 0))
|
||||
goto out;
|
||||
|
||||
@ -332,7 +314,7 @@ static int futex_wake(uint32_t *uaddr, int fshared, int nr_wake, uint32_t bitset
|
||||
if (!(this->bitset & bitset))
|
||||
continue;
|
||||
|
||||
wake_futex(this, clv_override);
|
||||
wake_futex(this);
|
||||
if (++ret >= nr_wake)
|
||||
break;
|
||||
}
|
||||
@ -350,8 +332,7 @@ out:
|
||||
*/
|
||||
static int
|
||||
futex_wake_op(uint32_t *uaddr1, int fshared, uint32_t *uaddr2,
|
||||
int nr_wake, int nr_wake2, int op,
|
||||
struct cpu_local_var *clv_override)
|
||||
int nr_wake, int nr_wake2, int op)
|
||||
{
|
||||
union futex_key key1 = FUTEX_KEY_INIT, key2 = FUTEX_KEY_INIT;
|
||||
struct futex_hash_bucket *hb1, *hb2;
|
||||
@ -360,10 +341,10 @@ futex_wake_op(uint32_t *uaddr1, int fshared, uint32_t *uaddr2,
|
||||
int ret, op_ret;
|
||||
|
||||
retry:
|
||||
ret = get_futex_key(uaddr1, fshared, &key1, clv_override);
|
||||
ret = get_futex_key(uaddr1, fshared, &key1);
|
||||
if ((ret != 0))
|
||||
goto out;
|
||||
ret = get_futex_key(uaddr2, fshared, &key2, clv_override);
|
||||
ret = get_futex_key(uaddr2, fshared, &key2);
|
||||
if ((ret != 0))
|
||||
goto out_put_key1;
|
||||
|
||||
@ -397,7 +378,7 @@ retry_private:
|
||||
|
||||
plist_for_each_entry_safe(this, next, head, list) {
|
||||
if (match_futex (&this->key, &key1)) {
|
||||
wake_futex(this, clv_override);
|
||||
wake_futex(this);
|
||||
if (++ret >= nr_wake)
|
||||
break;
|
||||
}
|
||||
@ -409,7 +390,7 @@ retry_private:
|
||||
op_ret = 0;
|
||||
plist_for_each_entry_safe(this, next, head, list) {
|
||||
if (match_futex (&this->key, &key2)) {
|
||||
wake_futex(this, clv_override);
|
||||
wake_futex(this);
|
||||
if (++op_ret >= nr_wake2)
|
||||
break;
|
||||
}
|
||||
@ -471,8 +452,8 @@ void requeue_futex(struct futex_q *q, struct futex_hash_bucket *hb1,
|
||||
* <0 - on error
|
||||
*/
|
||||
static int futex_requeue(uint32_t *uaddr1, int fshared, uint32_t *uaddr2,
|
||||
int nr_wake, int nr_requeue, uint32_t *cmpval,
|
||||
int requeue_pi, struct cpu_local_var *clv_override)
|
||||
int nr_wake, int nr_requeue, uint32_t *cmpval,
|
||||
int requeue_pi)
|
||||
{
|
||||
union futex_key key1 = FUTEX_KEY_INIT, key2 = FUTEX_KEY_INIT;
|
||||
int drop_count = 0, task_count = 0, ret;
|
||||
@ -480,10 +461,10 @@ static int futex_requeue(uint32_t *uaddr1, int fshared, uint32_t *uaddr2,
|
||||
struct plist_head *head1;
|
||||
struct futex_q *this, *next;
|
||||
|
||||
ret = get_futex_key(uaddr1, fshared, &key1, clv_override);
|
||||
ret = get_futex_key(uaddr1, fshared, &key1);
|
||||
if ((ret != 0))
|
||||
goto out;
|
||||
ret = get_futex_key(uaddr2, fshared, &key2, clv_override);
|
||||
ret = get_futex_key(uaddr2, fshared, &key2);
|
||||
if ((ret != 0))
|
||||
goto out_put_key1;
|
||||
|
||||
@ -518,7 +499,7 @@ static int futex_requeue(uint32_t *uaddr1, int fshared, uint32_t *uaddr2,
|
||||
*/
|
||||
/* RIKEN: no requeue_pi at this moment */
|
||||
if (++task_count <= nr_wake) {
|
||||
wake_futex(this, clv_override);
|
||||
wake_futex(this);
|
||||
continue;
|
||||
}
|
||||
|
||||
@ -577,9 +558,12 @@ queue_unlock(struct futex_q *q, struct futex_hash_bucket *hb)
|
||||
* state is implicit in the state of woken task (see futex_wait_requeue_pi() for
|
||||
* an example).
|
||||
*/
|
||||
static inline void queue_me(struct futex_q *q, struct futex_hash_bucket *hb, struct cpu_local_var *clv_override)
|
||||
static inline void queue_me(struct futex_q *q, struct futex_hash_bucket *hb)
|
||||
{
|
||||
int prio;
|
||||
struct thread *thread = cpu_local_var(current);
|
||||
ihk_spinlock_t *_runq_lock = &cpu_local_var(runq_lock);
|
||||
unsigned int *_flags = &cpu_local_var(flags);
|
||||
|
||||
/*
|
||||
* The priority used to register this element is
|
||||
@ -598,7 +582,19 @@ static inline void queue_me(struct futex_q *q, struct futex_hash_bucket *hb, str
|
||||
q->list.plist.spinlock = &hb->lock;
|
||||
#endif
|
||||
plist_add(&q->list, &hb->chain);
|
||||
q->task = cpu_local_var_with_override(current, clv_override);
|
||||
|
||||
/* Store information about wait thread for uti-futex*/
|
||||
q->task = thread;
|
||||
q->th_spin_sleep_pa = virt_to_phys((void *)&thread->spin_sleep);
|
||||
q->th_status_pa = virt_to_phys((void *)&thread->status);
|
||||
q->th_spin_sleep_lock_pa = virt_to_phys((void *)&thread->spin_sleep_lock);
|
||||
q->proc_status_pa = virt_to_phys((void *)&thread->proc->status);
|
||||
q->proc_update_lock_pa = virt_to_phys((void *)&thread->proc->update_lock);
|
||||
q->runq_lock_pa = virt_to_phys((void *)_runq_lock);
|
||||
q->clv_flags_pa = virt_to_phys((void *)_flags);
|
||||
q->intr_id = ihk_mc_get_interrupt_id(thread->cpu_id);
|
||||
q->intr_vector = ihk_mc_get_vector(IHK_GV_IKC);
|
||||
|
||||
ihk_mc_spinlock_unlock_noirq(&hb->lock);
|
||||
}
|
||||
|
||||
@ -661,12 +657,12 @@ retry:
|
||||
/* RIKEN: this function has been rewritten so that it returns the remaining
|
||||
* time in case we are waken.
|
||||
*/
|
||||
static int64_t futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q *q,
|
||||
uint64_t timeout, struct cpu_local_var *clv_override)
|
||||
static int64_t futex_wait_queue_me(struct futex_hash_bucket *hb,
|
||||
struct futex_q *q, uint64_t timeout)
|
||||
{
|
||||
int64_t time_remain = 0;
|
||||
unsigned long irqstate;
|
||||
struct thread *thread = cpu_local_var_with_override(current, clv_override);
|
||||
struct thread *thread = cpu_local_var(current);
|
||||
/*
|
||||
* The task state is guaranteed to be set before another task can
|
||||
* wake it.
|
||||
@ -685,25 +681,9 @@ static int64_t futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q
|
||||
ihk_mc_spinlock_unlock(&thread->spin_sleep_lock, irqstate);
|
||||
}
|
||||
|
||||
queue_me(q, hb, clv_override);
|
||||
queue_me(q, hb);
|
||||
|
||||
if (!plist_node_empty(&q->list)) {
|
||||
if (clv_override) {
|
||||
uti_dkprintf("%s: tid: %d is trying to sleep\n", __FUNCTION__, thread->tid);
|
||||
/* Note that the unit of timeout is nsec */
|
||||
time_remain = (*linux_wait_event)(q->uti_futex_resp, timeout);
|
||||
|
||||
/* Note that time_remain == 0 indicates contidion evaluated to false after the timeout elapsed */
|
||||
if (time_remain < 0) {
|
||||
if (time_remain == -ERESTARTSYS) { /* Interrupted by signal */
|
||||
uti_dkprintf("%s: DEBUG: wait_event returned -ERESTARTSYS\n", __FUNCTION__);
|
||||
} else {
|
||||
uti_kprintf("%s: ERROR: wait_event returned %d\n", __FUNCTION__, time_remain);
|
||||
}
|
||||
}
|
||||
uti_dkprintf("%s: tid: %d woken up\n", __FUNCTION__, thread->tid);
|
||||
} else {
|
||||
|
||||
if (timeout) {
|
||||
dkprintf("futex_wait_queue_me(): tid: %d schedule_timeout()\n", thread->tid);
|
||||
time_remain = schedule_timeout(timeout);
|
||||
@ -714,7 +694,6 @@ static int64_t futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q
|
||||
time_remain = 0;
|
||||
}
|
||||
dkprintf("futex_wait_queue_me(): tid: %d woken up\n", thread->tid);
|
||||
}
|
||||
}
|
||||
|
||||
/* This does not need to be serialized */
|
||||
@ -742,8 +721,7 @@ static int64_t futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q
|
||||
* <1 - -EFAULT or -EWOULDBLOCK (uaddr does not contain val) and hb is unlcoked
|
||||
*/
|
||||
static int futex_wait_setup(uint32_t __user *uaddr, uint32_t val, int fshared,
|
||||
struct futex_q *q, struct futex_hash_bucket **hb,
|
||||
struct cpu_local_var *clv_override)
|
||||
struct futex_q *q, struct futex_hash_bucket **hb)
|
||||
{
|
||||
uint32_t uval;
|
||||
int ret;
|
||||
@ -766,7 +744,7 @@ static int futex_wait_setup(uint32_t __user *uaddr, uint32_t val, int fshared,
|
||||
* rare, but normal.
|
||||
*/
|
||||
q->key = FUTEX_KEY_INIT;
|
||||
ret = get_futex_key(uaddr, fshared, &q->key, clv_override);
|
||||
ret = get_futex_key(uaddr, fshared, &q->key);
|
||||
if (ret != 0)
|
||||
return ret;
|
||||
|
||||
@ -790,8 +768,7 @@ static int futex_wait_setup(uint32_t __user *uaddr, uint32_t val, int fshared,
|
||||
}
|
||||
|
||||
static int futex_wait(uint32_t __user *uaddr, int fshared,
|
||||
uint32_t val, uint64_t timeout, uint32_t bitset, int clockrt,
|
||||
struct cpu_local_var *clv_override)
|
||||
uint32_t val, uint64_t timeout, uint32_t bitset, int clockrt)
|
||||
{
|
||||
struct futex_hash_bucket *hb;
|
||||
int64_t time_remain;
|
||||
@ -802,57 +779,55 @@ static int futex_wait(uint32_t __user *uaddr, int fshared,
|
||||
if (!bitset)
|
||||
return -EINVAL;
|
||||
|
||||
if (!clv_override) {
|
||||
q = &lq;
|
||||
}
|
||||
else {
|
||||
q = &cpu_local_var_with_override(current,
|
||||
clv_override)->futex_q;
|
||||
}
|
||||
q = &lq;
|
||||
|
||||
#ifdef PROFILE_ENABLE
|
||||
if (cpu_local_var_with_override(current, clv_override)->profile &&
|
||||
cpu_local_var_with_override(current, clv_override)->profile_start_ts) {
|
||||
cpu_local_var_with_override(current, clv_override)->profile_elapsed_ts +=
|
||||
(rdtsc() - cpu_local_var_with_override(current, clv_override)->profile_start_ts);
|
||||
cpu_local_var_with_override(current, clv_override)->profile_start_ts = 0;
|
||||
if (cpu_local_var(current)->profile &&
|
||||
cpu_local_var(current)->profile_start_ts) {
|
||||
cpu_local_var(current)->profile_elapsed_ts +=
|
||||
(rdtsc() - cpu_local_var(current)->profile_start_ts);
|
||||
cpu_local_var(current)->profile_start_ts = 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
q->bitset = bitset;
|
||||
q->requeue_pi_key = NULL;
|
||||
q->uti_futex_resp = cpu_local_var_with_override(uti_futex_resp,
|
||||
clv_override);
|
||||
q->uti_futex_resp = cpu_local_var(uti_futex_resp);
|
||||
|
||||
retry:
|
||||
/* Prepare to wait on uaddr. */
|
||||
ret = futex_wait_setup(uaddr, val, fshared, q, &hb, clv_override);
|
||||
ret = futex_wait_setup(uaddr, val, fshared, q, &hb);
|
||||
if (ret) {
|
||||
uti_dkprintf("%s: tid=%d futex_wait_setup returns zero, no need to sleep\n", __FUNCTION__, cpu_local_var_with_override(current, clv_override)->tid);
|
||||
dkprintf("%s: tid=%d futex_wait_setup returns zero, no need to sleep\n",
|
||||
__func__, cpu_local_var(current)->tid);
|
||||
goto out;
|
||||
}
|
||||
|
||||
/* queue_me and wait for wakeup, timeout, or a signal. */
|
||||
time_remain = futex_wait_queue_me(hb, q, timeout, clv_override);
|
||||
time_remain = futex_wait_queue_me(hb, q, timeout);
|
||||
|
||||
/* If we were woken (and unqueued), we succeeded, whatever. */
|
||||
ret = 0;
|
||||
if (!unqueue_me(q)) {
|
||||
uti_dkprintf("%s: tid=%d unqueued\n", __FUNCTION__, cpu_local_var_with_override(current, clv_override)->tid);
|
||||
dkprintf("%s: tid=%d unqueued\n",
|
||||
__func__, cpu_local_var(current)->tid);
|
||||
goto out_put_key;
|
||||
}
|
||||
ret = -ETIMEDOUT;
|
||||
|
||||
/* RIKEN: timer expired case (indicated by !time_remain) */
|
||||
if (timeout && !time_remain) {
|
||||
uti_dkprintf("%s: tid=%d timer expired\n", __FUNCTION__, cpu_local_var_with_override(current, clv_override)->tid);
|
||||
dkprintf("%s: tid=%d timer expired\n",
|
||||
__func__, cpu_local_var(current)->tid);
|
||||
goto out_put_key;
|
||||
}
|
||||
|
||||
/* RIKEN: futex_wait_queue_me() returns -ERESTARTSYS when waiting on Linux CPU and woken up by signal */
|
||||
if (hassigpending(cpu_local_var_with_override(current, clv_override)) || time_remain == -ERESTARTSYS) {
|
||||
if (hassigpending(cpu_local_var(current)) ||
|
||||
time_remain == -ERESTARTSYS) {
|
||||
ret = -EINTR;
|
||||
uti_dkprintf("%s: tid=%d woken up by signal\n", __FUNCTION__, cpu_local_var_with_override(current, clv_override)->tid);
|
||||
dkprintf("%s: tid=%d woken up by signal\n",
|
||||
__func__, cpu_local_var(current)->tid);
|
||||
goto out_put_key;
|
||||
}
|
||||
|
||||
@ -864,21 +839,22 @@ out_put_key:
|
||||
put_futex_key(fshared, &q->key);
|
||||
out:
|
||||
#ifdef PROFILE_ENABLE
|
||||
if (cpu_local_var_with_override(current, clv_override)->profile) {
|
||||
cpu_local_var_with_override(current, clv_override)->profile_start_ts = rdtsc();
|
||||
if (cpu_local_var(current)->profile) {
|
||||
cpu_local_var(current)->profile_start_ts = rdtsc();
|
||||
}
|
||||
#endif
|
||||
return ret;
|
||||
}
|
||||
|
||||
int futex(uint32_t *uaddr, int op, uint32_t val, uint64_t timeout,
|
||||
uint32_t *uaddr2, uint32_t val2, uint32_t val3, int fshared,
|
||||
struct cpu_local_var *clv_override)
|
||||
uint32_t *uaddr2, uint32_t val2, uint32_t val3, int fshared)
|
||||
{
|
||||
int clockrt, ret = -ENOSYS;
|
||||
int cmd = op & FUTEX_CMD_MASK;
|
||||
|
||||
uti_dkprintf("%s: uaddr=%p, op=%x, val=%x, timeout=%ld, uaddr2=%p, val2=%x, val3=%x, fshared=%d, clv=%p\n", __FUNCTION__, uaddr, op, val, timeout, uaddr2, val2, val3, fshared, clv_override);
|
||||
dkprintf("%s: uaddr=%p, op=%x, val=%x, timeout=%ld, uaddr2=%p, val2=%x, val3=%x, fshared=%d\n",
|
||||
__func__, uaddr, op, val, timeout, uaddr2,
|
||||
val2, val3, fshared);
|
||||
|
||||
clockrt = op & FUTEX_CLOCK_REALTIME;
|
||||
if (clockrt && cmd != FUTEX_WAIT_BITSET && cmd != FUTEX_WAIT_REQUEUE_PI)
|
||||
@ -888,21 +864,23 @@ int futex(uint32_t *uaddr, int op, uint32_t val, uint64_t timeout,
|
||||
case FUTEX_WAIT:
|
||||
val3 = FUTEX_BITSET_MATCH_ANY;
|
||||
case FUTEX_WAIT_BITSET:
|
||||
ret = futex_wait(uaddr, fshared, val, timeout, val3, clockrt, clv_override);
|
||||
ret = futex_wait(uaddr, fshared, val, timeout, val3, clockrt);
|
||||
break;
|
||||
case FUTEX_WAKE:
|
||||
val3 = FUTEX_BITSET_MATCH_ANY;
|
||||
case FUTEX_WAKE_BITSET:
|
||||
ret = futex_wake(uaddr, fshared, val, val3, clv_override);
|
||||
ret = futex_wake(uaddr, fshared, val, val3);
|
||||
break;
|
||||
case FUTEX_REQUEUE:
|
||||
ret = futex_requeue(uaddr, fshared, uaddr2, val, val2, NULL, 0, clv_override);
|
||||
ret = futex_requeue(uaddr, fshared, uaddr2,
|
||||
val, val2, NULL, 0);
|
||||
break;
|
||||
case FUTEX_CMP_REQUEUE:
|
||||
ret = futex_requeue(uaddr, fshared, uaddr2, val, val2, &val3, 0, clv_override);
|
||||
ret = futex_requeue(uaddr, fshared, uaddr2,
|
||||
val, val2, &val3, 0);
|
||||
break;
|
||||
case FUTEX_WAKE_OP:
|
||||
ret = futex_wake_op(uaddr, fshared, uaddr2, val, val2, val3, clv_override);
|
||||
ret = futex_wake_op(uaddr, fshared, uaddr2, val, val2, val3);
|
||||
break;
|
||||
/* RIKEN: these calls are not supported for now.
|
||||
case FUTEX_LOCK_PI:
|
||||
@ -942,7 +920,9 @@ int futex_init(void)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = 0; i < ARRAY_SIZE(futex_queues); i++) {
|
||||
futex_queues = kmalloc(sizeof(struct futex_hash_bucket) *
|
||||
(1 << FUTEX_HASHBITS), IHK_MC_AP_NOWAIT);
|
||||
for (i = 0; i < (1 << FUTEX_HASHBITS); i++) {
|
||||
plist_head_init(&futex_queues[i].chain, &futex_queues[i].lock);
|
||||
ihk_mc_spinlock_init(&futex_queues[i].lock);
|
||||
}
|
||||
|
||||
@ -128,6 +128,26 @@
|
||||
|
||||
struct process_vm;
|
||||
|
||||
static inline int get_futex_value_locked(uint32_t *dest, uint32_t *from)
|
||||
{
|
||||
|
||||
*dest = *(volatile uint32_t *)from;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Hash buckets are shared by all the futex_keys that hash to the same
|
||||
* location. Each key may have multiple futex_q structures, one for each task
|
||||
* waiting on a futex.
|
||||
*/
|
||||
struct futex_hash_bucket {
|
||||
ihk_spinlock_t lock;
|
||||
struct plist_head chain;
|
||||
};
|
||||
|
||||
struct futex_hash_bucket *get_futex_queues(void);
|
||||
|
||||
union futex_key {
|
||||
struct {
|
||||
unsigned long pgoff;
|
||||
@ -161,8 +181,7 @@ futex(
|
||||
uint32_t __user * uaddr2,
|
||||
uint32_t val2,
|
||||
uint32_t val3,
|
||||
int fshared,
|
||||
struct cpu_local_var *clv_override
|
||||
int fshared
|
||||
);
|
||||
|
||||
|
||||
@ -196,6 +215,25 @@ struct futex_q {
|
||||
|
||||
/* Used to wake-up a thread running on a Linux CPU */
|
||||
void *uti_futex_resp;
|
||||
|
||||
/* Used to wake-up a thread running on a McKernel from Linux */
|
||||
void *th_spin_sleep;
|
||||
void *th_status;
|
||||
void *th_spin_sleep_lock;
|
||||
void *proc_status;
|
||||
void *proc_update_lock;
|
||||
void *runq_lock;
|
||||
void *clv_flags;
|
||||
int intr_id;
|
||||
int intr_vector;
|
||||
|
||||
unsigned long th_spin_sleep_pa;
|
||||
unsigned long th_status_pa;
|
||||
unsigned long th_spin_sleep_lock_pa;
|
||||
unsigned long proc_status_pa;
|
||||
unsigned long proc_update_lock_pa;
|
||||
unsigned long runq_lock_pa;
|
||||
unsigned long clv_flags_pa;
|
||||
};
|
||||
|
||||
#endif
|
||||
|
||||
@ -1,158 +0,0 @@
|
||||
#ifndef _LINUX_JHASH_H
|
||||
#define _LINUX_JHASH_H
|
||||
/**
|
||||
* \file futex.c
|
||||
* Licence details are found in the file LICENSE.
|
||||
*
|
||||
* \brief
|
||||
* Adaptation to McKernel
|
||||
*
|
||||
* \author Balazs Gerofi <bgerofi@riken.jp> \par
|
||||
* Copyright (C) 2012 RIKEN AICS
|
||||
*
|
||||
*
|
||||
* HISTORY:
|
||||
*/
|
||||
|
||||
/*
|
||||
* jhash.h: Jenkins hash support.
|
||||
*
|
||||
* Copyright (C) 1996 Bob Jenkins (bob_jenkins@burtleburtle.net)
|
||||
*
|
||||
* http://burtleburtle.net/bob/hash/
|
||||
*
|
||||
* These are the credits from Bob's sources:
|
||||
*
|
||||
* lookup2.c, by Bob Jenkins, December 1996, Public Domain.
|
||||
* hash(), hash2(), hash3, and mix() are externally useful functions.
|
||||
* Routines to test the hash are included if SELF_TEST is defined.
|
||||
* You can use this free for any purpose. It has no warranty.
|
||||
*
|
||||
* Copyright (C) 2003 David S. Miller (davem@redhat.com)
|
||||
*
|
||||
* I've modified Bob's hash to be useful in the Linux kernel, and
|
||||
* any bugs present are surely my fault. -DaveM
|
||||
*
|
||||
*/
|
||||
|
||||
/* NOTE: Arguments are modified. */
|
||||
#define __jhash_mix(a, b, c) \
|
||||
{ \
|
||||
a -= b; a -= c; a ^= (c>>13); \
|
||||
b -= c; b -= a; b ^= (a<<8); \
|
||||
c -= a; c -= b; c ^= (b>>13); \
|
||||
a -= b; a -= c; a ^= (c>>12); \
|
||||
b -= c; b -= a; b ^= (a<<16); \
|
||||
c -= a; c -= b; c ^= (b>>5); \
|
||||
a -= b; a -= c; a ^= (c>>3); \
|
||||
b -= c; b -= a; b ^= (a<<10); \
|
||||
c -= a; c -= b; c ^= (b>>15); \
|
||||
}
|
||||
|
||||
/* The golden ration: an arbitrary value */
|
||||
#define JHASH_GOLDEN_RATIO 0x9e3779b9
|
||||
|
||||
/* The most generic version, hashes an arbitrary sequence
|
||||
* of bytes. No alignment or length assumptions are made about
|
||||
* the input key.
|
||||
*/
|
||||
static inline uint32_t jhash(const void *key, uint32_t length, uint32_t initval)
|
||||
{
|
||||
uint32_t a, b, c, len;
|
||||
const uint8_t *k = key;
|
||||
|
||||
len = length;
|
||||
a = b = JHASH_GOLDEN_RATIO;
|
||||
c = initval;
|
||||
|
||||
while (len >= 12) {
|
||||
a += (k[0] +((uint32_t)k[1]<<8) +((uint32_t)k[2]<<16) +((uint32_t)k[3]<<24));
|
||||
b += (k[4] +((uint32_t)k[5]<<8) +((uint32_t)k[6]<<16) +((uint32_t)k[7]<<24));
|
||||
c += (k[8] +((uint32_t)k[9]<<8) +((uint32_t)k[10]<<16)+((uint32_t)k[11]<<24));
|
||||
|
||||
__jhash_mix(a,b,c);
|
||||
|
||||
k += 12;
|
||||
len -= 12;
|
||||
}
|
||||
|
||||
c += length;
|
||||
switch (len) {
|
||||
case 11: c += ((uint32_t)k[10]<<24);
|
||||
case 10: c += ((uint32_t)k[9]<<16);
|
||||
case 9 : c += ((uint32_t)k[8]<<8);
|
||||
case 8 : b += ((uint32_t)k[7]<<24);
|
||||
case 7 : b += ((uint32_t)k[6]<<16);
|
||||
case 6 : b += ((uint32_t)k[5]<<8);
|
||||
case 5 : b += k[4];
|
||||
case 4 : a += ((uint32_t)k[3]<<24);
|
||||
case 3 : a += ((uint32_t)k[2]<<16);
|
||||
case 2 : a += ((uint32_t)k[1]<<8);
|
||||
case 1 : a += k[0];
|
||||
};
|
||||
|
||||
__jhash_mix(a,b,c);
|
||||
|
||||
return c;
|
||||
}
|
||||
|
||||
/* A special optimized version that handles 1 or more of uint32_ts.
|
||||
* The length parameter here is the number of uint32_ts in the key.
|
||||
*/
|
||||
static inline uint32_t jhash2(const uint32_t *k, uint32_t length, uint32_t initval)
|
||||
{
|
||||
uint32_t a, b, c, len;
|
||||
|
||||
a = b = JHASH_GOLDEN_RATIO;
|
||||
c = initval;
|
||||
len = length;
|
||||
|
||||
while (len >= 3) {
|
||||
a += k[0];
|
||||
b += k[1];
|
||||
c += k[2];
|
||||
__jhash_mix(a, b, c);
|
||||
k += 3; len -= 3;
|
||||
}
|
||||
|
||||
c += length * 4;
|
||||
|
||||
switch (len) {
|
||||
case 2 : b += k[1];
|
||||
case 1 : a += k[0];
|
||||
};
|
||||
|
||||
__jhash_mix(a,b,c);
|
||||
|
||||
return c;
|
||||
}
|
||||
|
||||
|
||||
/* A special ultra-optimized versions that knows they are hashing exactly
|
||||
* 3, 2 or 1 word(s).
|
||||
*
|
||||
* NOTE: In partilar the "c += length; __jhash_mix(a,b,c);" normally
|
||||
* done at the end is not done here.
|
||||
*/
|
||||
static inline uint32_t jhash_3words(uint32_t a, uint32_t b, uint32_t c, uint32_t initval)
|
||||
{
|
||||
a += JHASH_GOLDEN_RATIO;
|
||||
b += JHASH_GOLDEN_RATIO;
|
||||
c += initval;
|
||||
|
||||
__jhash_mix(a, b, c);
|
||||
|
||||
return c;
|
||||
}
|
||||
|
||||
static inline uint32_t jhash_2words(uint32_t a, uint32_t b, uint32_t initval)
|
||||
{
|
||||
return jhash_3words(a, b, 0, initval);
|
||||
}
|
||||
|
||||
static inline uint32_t jhash_1word(uint32_t a, uint32_t initval)
|
||||
{
|
||||
return jhash_3words(a, 0, 0, initval);
|
||||
}
|
||||
|
||||
#endif /* _LINUX_JHASH_H */
|
||||
88
kernel/include/mc_jhash.h
Normal file
88
kernel/include/mc_jhash.h
Normal file
@ -0,0 +1,88 @@
|
||||
#ifndef _MC_JHASH_H
|
||||
#define _MC_JHASH_H
|
||||
/**
|
||||
* \file mc_jhash.h
|
||||
* Licence details are found in the file LICENSE.
|
||||
*
|
||||
* \brief
|
||||
* Adaptation to McKernel
|
||||
*
|
||||
* \author Balazs Gerofi <bgerofi@riken.jp> \par
|
||||
* Copyright (C) 2012 RIKEN AICS
|
||||
*
|
||||
*
|
||||
* HISTORY:
|
||||
*/
|
||||
|
||||
/*
|
||||
* jhash.h: Jenkins hash support.
|
||||
*
|
||||
* Copyright (C) 1996 Bob Jenkins (bob_jenkins@burtleburtle.net)
|
||||
*
|
||||
* http://burtleburtle.net/bob/hash/
|
||||
*
|
||||
* These are the credits from Bob's sources:
|
||||
*
|
||||
* lookup2.c, by Bob Jenkins, December 1996, Public Domain.
|
||||
* hash(), hash2(), hash3, and mix() are externally useful functions.
|
||||
* Routines to test the hash are included if SELF_TEST is defined.
|
||||
* You can use this free for any purpose. It has no warranty.
|
||||
*
|
||||
* Copyright (C) 2003 David S. Miller (davem@redhat.com)
|
||||
*
|
||||
* I've modified Bob's hash to be useful in the Linux kernel, and
|
||||
* any bugs present are surely my fault. -DaveM
|
||||
*
|
||||
*/
|
||||
|
||||
/* NOTE: Arguments are modified. */
|
||||
#define __mc_jhash_mix(a, b, c) \
|
||||
{ \
|
||||
a -= b; a -= c; a ^= (c>>13); \
|
||||
b -= c; b -= a; b ^= (a<<8); \
|
||||
c -= a; c -= b; c ^= (b>>13); \
|
||||
a -= b; a -= c; a ^= (c>>12); \
|
||||
b -= c; b -= a; b ^= (a<<16); \
|
||||
c -= a; c -= b; c ^= (b>>5); \
|
||||
a -= b; a -= c; a ^= (c>>3); \
|
||||
b -= c; b -= a; b ^= (a<<10); \
|
||||
c -= a; c -= b; c ^= (b>>15); \
|
||||
}
|
||||
|
||||
/* The golden ration: an arbitrary value */
|
||||
#define JHASH_GOLDEN_RATIO 0x9e3779b9
|
||||
|
||||
/* A special optimized version that handles 1 or more of uint32_ts.
|
||||
* The length parameter here is the number of uint32_ts in the key.
|
||||
*/
|
||||
static inline uint32_t mc_jhash2(const uint32_t *k, uint32_t length, uint32_t initval)
|
||||
{
|
||||
uint32_t a, b, c, len;
|
||||
|
||||
a = b = JHASH_GOLDEN_RATIO;
|
||||
c = initval;
|
||||
len = length;
|
||||
|
||||
while (len >= 3) {
|
||||
a += k[0];
|
||||
b += k[1];
|
||||
c += k[2];
|
||||
__mc_jhash_mix(a, b, c);
|
||||
k += 3; len -= 3;
|
||||
}
|
||||
|
||||
c += length * 4;
|
||||
|
||||
switch (len) {
|
||||
case 2:
|
||||
b += k[1];
|
||||
case 1:
|
||||
a += k[0];
|
||||
};
|
||||
|
||||
__mc_jhash_mix(a, b, c);
|
||||
|
||||
return c;
|
||||
}
|
||||
|
||||
#endif /* _MC_JHASH_H */
|
||||
@ -252,7 +252,6 @@ static void nmi_init()
|
||||
|
||||
static void uti_init()
|
||||
{
|
||||
ihk_set_mckernel_do_futex((unsigned long)do_futex);
|
||||
}
|
||||
|
||||
static void rest_init(void)
|
||||
|
||||
@ -74,13 +74,6 @@
|
||||
#define DDEBUG_DEFAULT DDEBUG_PRINT
|
||||
#endif
|
||||
|
||||
//#define DEBUG_UTI
|
||||
#ifdef DEBUG_UTI
|
||||
#define uti_dkprintf(...) do { ((uti_clv && linux_printk) ? (*linux_printk) : kprintf)(__VA_ARGS__); } while (0)
|
||||
#else
|
||||
#define uti_dkprintf(...) do { } while (0)
|
||||
#endif
|
||||
|
||||
//static ihk_atomic_t pid_cnt = IHK_ATOMIC_INIT(1024);
|
||||
|
||||
/* generate system call handler's prototypes */
|
||||
@ -6907,7 +6900,7 @@ long do_futex(int n, unsigned long arg0, unsigned long arg1,
|
||||
}
|
||||
op = (op & FUTEX_CMD_MASK);
|
||||
|
||||
uti_dkprintf("futex op=[%x, %s],uaddr=%lx, val=%x, utime=%lx, uaddr2=%lx, val3=%x, []=%x, shared: %d\n",
|
||||
dkprintf("futex op=[%x, %s],uaddr=%lx, val=%x, utime=%lx, uaddr2=%lx, val3=%x, []=%x, shared: %d\n",
|
||||
flags,
|
||||
(op == FUTEX_WAIT) ? "FUTEX_WAIT" :
|
||||
(op == FUTEX_WAIT_BITSET) ? "FUTEX_WAIT_BITSET" :
|
||||
@ -6919,7 +6912,8 @@ long do_futex(int n, unsigned long arg0, unsigned long arg1,
|
||||
(unsigned long)uaddr, val, utime, uaddr2, val3, *uaddr, fshared);
|
||||
|
||||
if ((op == FUTEX_WAIT || op == FUTEX_WAIT_BITSET) && utime) {
|
||||
uti_dkprintf("%s: utime=%ld.%09ld\n", __FUNCTION__, utime->tv_sec, utime->tv_nsec);
|
||||
dkprintf("%s: utime=%ld.%09ld\n",
|
||||
__func__, utime->tv_sec, utime->tv_nsec);
|
||||
}
|
||||
if (utime && (op == FUTEX_WAIT_BITSET || op == FUTEX_WAIT)) {
|
||||
unsigned long nsec_timeout;
|
||||
@ -6977,7 +6971,8 @@ long do_futex(int n, unsigned long arg0, unsigned long arg1,
|
||||
if (ret) {
|
||||
return ret;
|
||||
}
|
||||
uti_dkprintf("%s: ats=%ld.%09ld\n", __FUNCTION__, ats.tv_sec, ats.tv_nsec);
|
||||
dkprintf("%s: ats=%ld.%09ld\n",
|
||||
__func__, ats.tv_sec, ats.tv_nsec);
|
||||
/* Use nsec for UTI case */
|
||||
timeout = (utime->tv_sec * NS_PER_SEC + utime->tv_nsec) -
|
||||
(ats.tv_sec * NS_PER_SEC + ats.tv_nsec);
|
||||
@ -6993,9 +6988,9 @@ long do_futex(int n, unsigned long arg0, unsigned long arg1,
|
||||
if (op == FUTEX_CMP_REQUEUE || op == FUTEX_WAKE_OP)
|
||||
val2 = (uint32_t) (unsigned long) arg3;
|
||||
|
||||
ret = futex(uaddr, op, val, timeout, uaddr2, val2, val3, fshared, uti_clv);
|
||||
ret = futex(uaddr, op, val, timeout, uaddr2, val2, val3, fshared);
|
||||
|
||||
uti_dkprintf("futex op=[%x, %s],uaddr=%lx, val=%x, utime=%lx, uaddr2=%lx, val3=%x, []=%x, shared: %d, ret: %d\n",
|
||||
dkprintf("futex op=[%x, %s],uaddr=%lx, val=%x, utime=%lx, uaddr2=%lx, val3=%x, []=%x, shared: %d, ret: %d\n",
|
||||
op,
|
||||
(op == FUTEX_WAIT) ? "FUTEX_WAIT" :
|
||||
(op == FUTEX_WAIT_BITSET) ? "FUTEX_WAIT_BITSET" :
|
||||
@ -7043,7 +7038,7 @@ do_exit(int code)
|
||||
setint_user((int*)thread->clear_child_tid, 0);
|
||||
barrier();
|
||||
futex((uint32_t *)thread->clear_child_tid,
|
||||
FUTEX_WAKE, 1, 0, NULL, 0, 0, 1, NULL);
|
||||
FUTEX_WAKE, 1, 0, NULL, 0, 0, 1);
|
||||
thread->clear_child_tid = NULL;
|
||||
}
|
||||
|
||||
@ -10596,7 +10591,7 @@ int util_thread(struct uti_attr *arg)
|
||||
{
|
||||
struct uti_ctx *rctx = NULL;
|
||||
unsigned long rp_rctx;
|
||||
struct cpu_local_var *uti_clv = NULL;
|
||||
struct uti_info *uti_info = NULL;
|
||||
struct syscall_request request IHK_DMA_ALIGN;
|
||||
long rc;
|
||||
struct thread *thread = cpu_local_var(current);
|
||||
@ -10615,13 +10610,29 @@ int util_thread(struct uti_attr *arg)
|
||||
rp_rctx = virt_to_phys((void *)rctx);
|
||||
save_uctx((void *)rctx->ctx, NULL);
|
||||
|
||||
/* Create a copy of clv and replace clv with it when the Linux thread calls in a McKernel function */
|
||||
uti_clv = kmalloc(sizeof(struct cpu_local_var), IHK_MC_AP_NOWAIT);
|
||||
if (!uti_clv) {
|
||||
/* Create a information for Linux thread */
|
||||
uti_info = kmalloc(sizeof(struct uti_info), IHK_MC_AP_NOWAIT);
|
||||
if (!uti_info) {
|
||||
rc = -ENOMEM;
|
||||
goto out;
|
||||
}
|
||||
memcpy(uti_clv, get_this_cpu_local_var(), sizeof(struct cpu_local_var));
|
||||
/* clv info */
|
||||
uti_info->thread_va = (unsigned long)cpu_local_var(current);
|
||||
uti_info->uti_futex_resp_pa = virt_to_phys((void *)cpu_local_var(uti_futex_resp));
|
||||
uti_info->ikc2linux_pa = virt_to_phys((void *)cpu_local_var(ikc2linux));
|
||||
|
||||
/* thread info */
|
||||
uti_info->tid = thread->tid;
|
||||
uti_info->cpu = ihk_mc_get_processor_id();
|
||||
uti_info->status_pa = virt_to_phys((void *)&thread->status);
|
||||
uti_info->spin_sleep_lock_pa = virt_to_phys((void *)&thread->spin_sleep_lock);
|
||||
uti_info->spin_sleep_pa = virt_to_phys((void *)&thread->spin_sleep);
|
||||
uti_info->vm_pa = virt_to_phys((void *)thread->vm);
|
||||
uti_info->futex_q_pa = virt_to_phys((void *)&thread->futex_q);
|
||||
|
||||
/* global info */
|
||||
uti_info->mc_idle_halt = idle_halt;
|
||||
uti_info->futex_queue_pa = virt_to_phys((void *)get_futex_queues());
|
||||
|
||||
request.number = __NR_sched_setaffinity;
|
||||
request.args[0] = 0;
|
||||
@ -10632,7 +10643,7 @@ int util_thread(struct uti_attr *arg)
|
||||
kattr.parent_cpuid = thread->parent_cpuid;
|
||||
request.args[2] = virt_to_phys(&kattr);
|
||||
}
|
||||
request.args[3] = (unsigned long)uti_clv;
|
||||
request.args[3] = (unsigned long)uti_info;
|
||||
request.args[4] = uti_desc;
|
||||
thread->uti_state = UTI_STATE_RUNNING_IN_LINUX;
|
||||
rc = do_syscall(&request, ihk_mc_get_processor_id());
|
||||
@ -10649,8 +10660,8 @@ int util_thread(struct uti_attr *arg)
|
||||
kfree(rctx);
|
||||
rctx = NULL;
|
||||
|
||||
kfree(uti_clv);
|
||||
uti_clv = NULL;
|
||||
kfree(uti_info);
|
||||
uti_info = NULL;
|
||||
|
||||
if (rc >= 0) {
|
||||
if (rc & 0x100000000) { /* exit_group */
|
||||
@ -10673,7 +10684,7 @@ int util_thread(struct uti_attr *arg)
|
||||
|
||||
out:
|
||||
kfree(rctx);
|
||||
kfree(uti_clv);
|
||||
kfree(uti_info);
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user