gh0s7/mckernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reference counting per-thread data

Browse Source 
It is accompanied by the following fixes:
(1) Fix put ppd locations in mcexec_wait_syscall()
(2) Move put ptd to end of mcexec_terminate_thread_unsafe() and mcexec_ret_syscall()
(3) Add debug messages for ptd add/get/put
(4) Fix ptd-add/get/put matching in mcexec_wait_syscall()
    * Skip put when woken-up from wait_event_interruptible() by signal

Change-Id: Ib9be3f5e62a7a370197cb36c9fa7c4d79f44c314
This commit is contained in:
Masamichi Takagi
2018-09-03 19:36:28 +09:00
parent a121ffc785
commit b8bacdd2de
3 changed files with 269 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

178
executer/kernel/mcctrl/control.c
View File

@ -58,6 +58,13 @@
#define dprintk(...)
#endif
//#define DEBUG_PTD
#ifdef DEBUG_PTD
#define pr_ptd(msg, tid, ptd) do { printk("%s: " msg ",tid=%d,refc=%d\n", __FUNCTION__, tid, atomic_read(&ptd->refcount)); } while(0)
#else
#define pr_ptd(msg, tid, ptd) do { } while(0)
#endif
//extern struct mcctrl_channel *channels;
int mcctrl_ikc_set_recv_cpu(ihk_os_t os, int cpu);
int syscall_backward(struct mcctrl_usrdata *, int, unsigned long, unsigned long,
@ -1077,6 +1084,8 @@ void mcctrl_put_per_proc_data(struct mcctrl_per_proc_data *ppd)
struct wait_queue_head_list_node *wqhln;
struct wait_queue_head_list_node *wqhln_next;
struct ikc_scd_packet *packet;
struct mcctrl_per_thread_data *ptd;
struct mcctrl_per_thread_data *next;
if (!ppd)
return;
@ -1095,25 +1104,32 @@ void mcctrl_put_per_proc_data(struct mcctrl_per_proc_data *ppd)
write_unlock_irqrestore(&ppd->ud->per_proc_data_hash_lock[hash], flags);
dprintk("%s: deallocating PPD for pid %d\n", __FUNCTION__, ppd->pid);
for (i = 0; i < MCCTRL_PER_THREAD_DATA_HASH_SIZE; i++) {
struct mcctrl_per_thread_data *ptd;
struct mcctrl_per_thread_data *next;
for (i = 0; i < MCCTRL_PER_THREAD_DATA_HASH_SIZE; i++) {
write_lock_irqsave(&ppd->per_thread_data_hash_lock[i], flags);
list_for_each_entry_safe(ptd, next,
ppd->per_thread_data_hash + i, hash) {
packet = ptd->data;
list_del(&ptd->hash);
kfree(ptd);
/* We use ERESTARTSYS to tell the LWK that the proxy
* process is gone and the application should be terminated */
process is gone and the application should be terminated. */
packet = (struct ikc_scd_packet *)ptd->data;
dprintk("%s: calling __return_syscall (hash),target pid=%d,tid=%d\n", __FUNCTION__, ppd->pid, packet->req.rtid);
__return_syscall(ppd->ud->os, packet, -ERESTARTSYS,
packet->req.rtid);
packet->req.rtid);
ihk_ikc_release_packet(
(struct ihk_ikc_free_packet *)packet,
(ppd->ud->ikc2linux[smp_processor_id()] ?
ppd->ud->ikc2linux[smp_processor_id()] :
ppd->ud->ikc2linux[0]));
/* Note that uti ptd needs another put by mcexec_terminate_thread()
(see mcexec_syscall_wait()).
TODO: Detect tracer has died before calling mcexec_terminate_thread() and put uti ptd */
if (atomic_read(&ptd->refcount) != 1) {
printk("%s: WARNING: ptd->refcount != 1 but %d\n", __FUNCTION__, atomic_read(&ptd->refcount));
}
mcctrl_put_per_thread_data_unsafe(ptd);
pr_ptd("put", ptd->tid, ptd);
}
write_unlock_irqrestore(&ppd->per_thread_data_hash_lock[i], flags);
}
@ -1283,6 +1299,7 @@ int mcexec_wait_syscall(ihk_os_t os, struct syscall_wait_desc *__user req)
int ret = 0;
unsigned long irqflags;
struct mcctrl_per_proc_data *ppd;
struct mcctrl_per_thread_data *ptd = NULL;
/* Get a reference to per-process structure */
ppd = mcctrl_get_per_proc_data(usrdata, task_tgid_vnr(current));
@ -1293,12 +1310,13 @@ int mcexec_wait_syscall(ihk_os_t os, struct syscall_wait_desc *__user req)
return -EINVAL;
}
packet = (struct ikc_scd_packet *)mcctrl_get_per_thread_data(ppd, current);
if (packet) {
ptd = mcctrl_get_per_thread_data(ppd, current);
if (ptd) {
printk("%s: ERROR: packet %p is already registered for thread %d\n",
__FUNCTION__, packet, task_pid_vnr(current));
__FUNCTION__, ptd->data, task_pid_vnr(current));
mcctrl_put_per_thread_data(ptd);
ret = -EBUSY;
goto put_ppd_out;
goto no_ptd;
}
retry:
@ -1364,7 +1382,7 @@ retry_alloc:
}
wqhln = NULL;
ret = -EINTR;
goto put_ppd_out;
goto no_ptd;
}
packet = wqhln->packet;
@ -1400,27 +1418,43 @@ retry_alloc:
packet->req.args[4],
packet->req.args[5]);
if (mcctrl_add_per_thread_data(ppd, current, packet) < 0) {
kprintf("%s: error adding per-thread data\n", __FUNCTION__);
ret = -EINVAL;;
goto put_ppd_out;
/* Create ptd */
if ((ret = mcctrl_add_per_thread_data(ppd, packet))) {
kprintf("%s: error adding per-thread data (%d)\n", __FUNCTION__, ret);
ret = -EINVAL;
goto no_ptd;
}
/* Get a reference valid until offload is done */
ptd = mcctrl_get_per_thread_data(ppd, current);
if (!ptd) {
kprintf("%s: ERROR: ptd not found\n", __FUNCTION__);
ret = -EINVAL;
goto no_ptd;
}
pr_ptd("get", task_pid_vnr(current), ptd);
if (packet->req.number == __NR_sched_setaffinity && packet->req.args[0] == 0) {
dprintk("%s: uti,packet=%p,tid=%d\n", __FUNCTION__, packet, task_pid_vnr(current));
/* Get a reference valid until thread-offload is done */
ptd = mcctrl_get_per_thread_data(ppd, current);
if (!ptd) {
kprintf("%s: ptd not found\n", __FUNCTION__);
ret = -EINVAL;
goto no_ptd;
}
pr_ptd("get", task_pid_vnr(current), ptd);
}
if (__do_in_kernel_syscall(os, packet)) {
if (copy_to_user(&req->sr, &packet->req,
sizeof(struct syscall_request))) {
if (mcctrl_delete_per_thread_data(ppd, current) < 0) {
kprintf("%s: error deleting per-thread data\n", __FUNCTION__);
}
ret = -EINVAL;;
ret = -EINVAL;
goto put_ppd_out;
}
if (copy_to_user(&req->cpu, &packet->ref, sizeof(req->cpu))) {
if (mcctrl_delete_per_thread_data(ppd, current) < 0) {
kprintf("%s: error deleting per-thread data\n", __FUNCTION__);
}
ret = -EINVAL;
goto put_ppd_out;
}
@ -1434,15 +1468,18 @@ retry_alloc:
usrdata->ikc2linux[smp_processor_id()] :
usrdata->ikc2linux[0]));
if (mcctrl_delete_per_thread_data(ppd, current) < 0) {
kprintf("%s: error deleting per-thread data\n", __FUNCTION__);
ret = -EINVAL;;
goto put_ppd_out;
}
/* Drop reference to zero and restart from add */
mcctrl_put_per_thread_data(ptd);
pr_ptd("put,in_kernel", task_pid_vnr(current), ptd);
mcctrl_put_per_thread_data(ptd);
pr_ptd("put,in_kernel", task_pid_vnr(current), ptd);
goto retry;
put_ppd_out:
mcctrl_put_per_thread_data(ptd);
pr_ptd("put,in_mcexec", task_pid_vnr(current), ptd);
no_ptd:
mcctrl_put_per_proc_data(ppd);
return ret;
}
@ -1537,6 +1574,7 @@ long mcexec_ret_syscall(ihk_os_t os, struct syscall_ret_desc *__user arg)
struct ikc_scd_packet *packet;
struct mcctrl_usrdata *usrdata = ihk_host_os_get_usrdata(os);
struct mcctrl_per_proc_data *ppd;
struct mcctrl_per_thread_data *ptd;
int error = 0;
if (copy_from_user(&ret, arg, sizeof(struct syscall_ret_desc))) {
@ -1551,16 +1589,22 @@ long mcexec_ret_syscall(ihk_os_t os, struct syscall_ret_desc *__user arg)
return -EINVAL;
}
packet = (struct ikc_scd_packet *)mcctrl_get_per_thread_data(ppd, current);
/* Get a reference for this function */
ptd = mcctrl_get_per_thread_data(ppd, current);
if (!ptd) {
printk("%s: ERROR: mcctrl_get_per_thread_data failed\n", __FUNCTION__);
error = -EINVAL;
goto no_ptd;
}
pr_ptd("get", task_pid_vnr(current), ptd);
packet = (struct ikc_scd_packet *)ptd->data;
if (!packet) {
kprintf("%s: ERROR: no packet registered for TID %d\n",
__FUNCTION__, task_pid_vnr(current));
error = -EINVAL;
goto out;
goto put_ppd_out;
}
mcctrl_delete_per_thread_data(ppd, current);
if (ret.size > 0) {
/* Host => Accel. Write is fast. */
unsigned long phys;
@ -1595,7 +1639,15 @@ out:
(usrdata->ikc2linux[smp_processor_id()] ?
usrdata->ikc2linux[smp_processor_id()] :
usrdata->ikc2linux[0]));
put_ppd_out:
/* Drop a reference for this function */
mcctrl_put_per_thread_data(ptd);
pr_ptd("put", task_pid_vnr(current), ptd);
/* Final drop of the reference for non-uti syscall offloading */
mcctrl_put_per_thread_data(ptd);
pr_ptd("put", task_pid_vnr(current), ptd);
no_ptd:
mcctrl_put_per_proc_data(ppd);
return error;
}
@ -2452,6 +2504,7 @@ mcexec_terminate_thread_unsafe(ihk_os_t os, int pid, int tid, long sig, struct t
struct ikc_scd_packet *packet;
struct mcctrl_usrdata *usrdata = ihk_host_os_get_usrdata(os);
struct mcctrl_per_proc_data *ppd;
struct mcctrl_per_thread_data *ptd;
dprintk("%s: target pid=%d,tid=%d,sig=%lx,task=%p\n", __FUNCTION__, pid, tid, sig, tsk);
@ -2461,30 +2514,48 @@ mcexec_terminate_thread_unsafe(ihk_os_t os, int pid, int tid, long sig, struct t
__FUNCTION__, pid);
goto no_ppd;
}
packet = (struct ikc_scd_packet *)mcctrl_get_per_thread_data(ppd, tsk);
ptd = mcctrl_get_per_thread_data(ppd, tsk);
if (!ptd) {
printk("%s: ERROR: mcctrl_get_per_thread_data failed\n", __FUNCTION__);
goto no_ptd;
}
if (ptd->tid != tid) {
printk("%s: ERROR: ptd->tid(%d) != tid(%d)\n", __FUNCTION__, ptd->tid, tid);
goto no_ptd;
}
pr_ptd("get", tid, ptd);
packet = (struct ikc_scd_packet *)ptd->data;
if (!packet) {
kprintf("%s: ERROR: no packet registered for TID %d\n",
__FUNCTION__, tid);
goto no_ptd;
}
if ((rc = mcctrl_delete_per_thread_data(ppd, tsk))) {
kprintf("%s: ERROR: mcctrl_delete_per_thread_data failed (%d)\n", __FUNCTION__, rc);
goto no_ptd;
}
__return_syscall(usrdata->os, packet, sig, tid);
ihk_ikc_release_packet((struct ihk_ikc_free_packet *)packet,
(usrdata->ikc2linux[smp_processor_id()] ?
usrdata->ikc2linux[smp_processor_id()] :
usrdata->ikc2linux[0]));
/* Drop reference for this function */
mcctrl_put_per_thread_data(ptd);
pr_ptd("put", tid, ptd);
/* Final drop of reference for uti ptd */
mcctrl_put_per_thread_data(ptd);
pr_ptd("put", tid, ptd);
if (atomic_read(&ptd->refcount) != 1) {
printk("%s: WARNING: ptd->refcount != 1 but %d\n", __FUNCTION__, atomic_read(&ptd->refcount));
}
mcctrl_put_per_thread_data(ptd);
pr_ptd("put", tid, ptd);
no_ptd:
/* Destroy per_proc_data with the following two puts. Note that
it survived the signal-kill of tracee thanks to the additional put
done in mcexec_util_thread2 */
mcctrl_put_per_proc_data(ppd);
mcctrl_put_per_proc_data(ppd);
/* This is the final drop of uti-ppd */
mcctrl_put_per_proc_data(ppd);
no_ppd:
return 0;
}
@ -3094,6 +3165,7 @@ int mcctrl_get_request_os_cpu(ihk_os_t os, int *ret_cpu)
{
struct mcctrl_usrdata *usrdata;
struct mcctrl_per_proc_data *ppd;
struct mcctrl_per_thread_data *ptd;
struct ikc_scd_packet *packet;
struct ihk_ikc_channel_desc *ch;
int ret = 0;
@ -3118,11 +3190,18 @@ int mcctrl_get_request_os_cpu(ihk_os_t os, int *ret_cpu)
}
/* Look up per-thread structure */
packet = (struct ikc_scd_packet *)mcctrl_get_per_thread_data(ppd, current);
if (!packet) {
ptd = mcctrl_get_per_thread_data(ppd, current);
if (!ptd) {
printk("%s: ERROR: mcctrl_get_per_thread_data failed\n", __FUNCTION__);
ret = -EINVAL;
goto no_ptd;
}
pr_ptd("get", task_pid_vnr(current), ptd);
packet = (struct ikc_scd_packet *)ptd->data;
if (!packet) {
printk("%s: ERROR: no packet registered for TID %d\n",
__FUNCTION__, task_pid_vnr(current));
ret = -EINVAL;
goto out_put_ppd;
}
@ -3135,6 +3214,9 @@ int mcctrl_get_request_os_cpu(ihk_os_t os, int *ret_cpu)
printk("%s: OS: %p, CPU: %d\n", __FUNCTION__, os, *ret_cpu);
out_put_ppd:
mcctrl_put_per_thread_data(ptd);
pr_ptd("put", task_pid_vnr(current), ptd);
no_ptd:
mcctrl_put_per_proc_data(ppd);
return ret;

34
executer/kernel/mcctrl/mcctrl.h
View File

@ -228,9 +228,12 @@ struct mcctrl_channel {
};
struct mcctrl_per_thread_data {
struct mcctrl_per_proc_data *ppd;
struct list_head hash;
struct task_struct *task;
void *data;
int tid; /* debug */
atomic_t refcount;
};
#define MCCTRL_PER_THREAD_DATA_HASH_SHIFT 8
@ -447,20 +450,18 @@ struct mcctrl_per_proc_data *mcctrl_get_per_proc_data(
struct mcctrl_usrdata *ud, int pid);
void mcctrl_put_per_proc_data(struct mcctrl_per_proc_data *ppd);
int mcctrl_add_per_thread_data(struct mcctrl_per_proc_data* ppd,
struct task_struct *task, void *data);
int mcctrl_delete_per_thread_data(struct mcctrl_per_proc_data* ppd,
struct task_struct *task);
int mcctrl_add_per_thread_data(struct mcctrl_per_proc_data *ppd, void *data);
void mcctrl_put_per_thread_data_unsafe(struct mcctrl_per_thread_data *ptd);
void mcctrl_put_per_thread_data(struct mcctrl_per_thread_data* ptd);
#ifdef POSTK_DEBUG_ARCH_DEP_56 /* Strange how to use inline declaration fix. */
static inline struct mcctrl_per_thread_data *mcctrl_get_per_thread_data(
struct mcctrl_per_proc_data *ppd, struct task_struct *task)
inline struct mcctrl_per_thread_data *mcctrl_get_per_thread_data(struct mcctrl_per_proc_data *ppd, struct task_struct *task)
{
struct mcctrl_per_thread_data *ptd_iter, *ptd = NULL;
int hash = (((uint64_t)task >> 4) & MCCTRL_PER_THREAD_DATA_HASH_MASK);
unsigned long flags;
/* Check if data for this thread exists and return it */
read_lock_irqsave(&ppd->per_thread_data_hash_lock[hash], flags);
/* Check if data for this thread exists */
write_lock_irqsave(&ppd->per_thread_data_hash_lock[hash], flags);
list_for_each_entry(ptd_iter, &ppd->per_thread_data_hash[hash], hash) {
if (ptd_iter->task == task) {
@ -469,12 +470,21 @@ static inline struct mcctrl_per_thread_data *mcctrl_get_per_thread_data(
}
}
read_unlock_irqrestore(&ppd->per_thread_data_hash_lock[hash], flags);
return ptd ? ptd->data : NULL;
if (ptd) {
if (atomic_read(&ptd->refcount) <= 0) {
printk("%s: ERROR: use-after-free detected (%d)", __FUNCTION__, atomic_read(&ptd->refcount));
ptd = NULL;
goto out;
}
atomic_inc(&ptd->refcount);
}
out:
write_unlock_irqrestore(&ppd->per_thread_data_hash_lock[hash], flags);
return ptd;
}
#else /* POSTK_DEBUG_ARCH_DEP_56 */
inline struct mcctrl_per_thread_data *mcctrl_get_per_thread_data(
struct mcctrl_per_proc_data *ppd, struct task_struct *task);
inline struct mcctrl_per_thread_data *mcctrl_get_per_thread_data(struct mcctrl_per_proc_data *ppd, struct task_struct *task);
#endif /* POSTK_DEBUG_ARCH_DEP_56 */
void __return_syscall(ihk_os_t os, struct ikc_scd_packet *packet,

169
executer/kernel/mcctrl/syscall.c
View File

@ -63,6 +63,13 @@
#define dprintk(...)
#endif
//#define DEBUG_PTD
#ifdef DEBUG_PTD
#define pr_ptd(msg, tid, ptd) do { printk("%s: " msg ",tid=%d,refc=%d\n", __FUNCTION__, tid, atomic_read(&ptd->refcount)); } while(0)
#else
#define pr_ptd(msg, tid, ptd) do { } while(0)
#endif
static long pager_call_irq(ihk_os_t os, struct syscall_request *req);
static long pager_call(ihk_os_t os, struct syscall_request *req);
@ -80,75 +87,90 @@ static void print_dma_lastreq(void)
}
#endif
int mcctrl_add_per_thread_data(struct mcctrl_per_proc_data* ppd,
struct task_struct *task, void *data)
void mcctrl_put_per_thread_data_unsafe(struct mcctrl_per_thread_data *ptd)
{
struct mcctrl_per_thread_data *ptd_iter, *ptd = NULL;
struct mcctrl_per_thread_data *ptd_alloc = NULL;
int hash = (((uint64_t)task >> 4) & MCCTRL_PER_THREAD_DATA_HASH_MASK);
int ret = 0;
unsigned long flags;
ptd_alloc = kmalloc(sizeof(*ptd), GFP_ATOMIC);
if (!ptd_alloc) {
kprintf("%s: error allocate per thread data\n", __FUNCTION__);
ret = -ENOMEM;
goto out_noalloc;
}
/* Check if data for this thread exists and add if not */
write_lock_irqsave(&ppd->per_thread_data_hash_lock[hash], flags);
list_for_each_entry(ptd_iter, &ppd->per_thread_data_hash[hash], hash) {
if (ptd_iter->task == task) {
ptd = ptd_iter;
break;
if (!atomic_dec_and_test(&ptd->refcount)) {
int ret = atomic_read(&ptd->refcount);
if (ret < 0) {
printk("%s: ERROR: invalid refcount=%d\n", __FUNCTION__, ret);
}
return;
}
if (unlikely(ptd)) {
ret = -EBUSY;
kfree(ptd_alloc);
goto out;
}
ptd = ptd_alloc;
ptd->task = task;
ptd->data = data;
list_add_tail(&ptd->hash, &ppd->per_thread_data_hash[hash]);
out:
write_unlock_irqrestore(&ppd->per_thread_data_hash_lock[hash], flags);
out_noalloc:
return ret;
list_del(&ptd->hash);
kfree(ptd);
}
int mcctrl_delete_per_thread_data(struct mcctrl_per_proc_data* ppd,
struct task_struct *task)
void mcctrl_put_per_thread_data(struct mcctrl_per_thread_data* _ptd)
{
struct mcctrl_per_proc_data *ppd = _ptd->ppd;
struct mcctrl_per_thread_data *ptd_iter, *ptd = NULL;
int hash = (((uint64_t)task >> 4) & MCCTRL_PER_THREAD_DATA_HASH_MASK);
int ret = 0;
int hash = (((uint64_t)_ptd->task >> 4) & MCCTRL_PER_THREAD_DATA_HASH_MASK);
unsigned long flags;
/* Check if data for this thread exists and delete it */
write_lock_irqsave(&ppd->per_thread_data_hash_lock[hash], flags);
list_for_each_entry(ptd_iter, &ppd->per_thread_data_hash[hash], hash) {
if (ptd_iter->task == task) {
if (ptd_iter->task == _ptd->task) {
ptd = ptd_iter;
break;
}
}
if (!ptd) {
ret = -EINVAL;
printk("%s: ERROR: ptd not found\n", __FUNCTION__);
goto out;
}
list_del(&ptd->hash);
kfree(ptd);
mcctrl_put_per_thread_data_unsafe(ptd);
out:
write_unlock_irqrestore(&ppd->per_thread_data_hash_lock[hash], flags);
}
int mcctrl_add_per_thread_data(struct mcctrl_per_proc_data *ppd, void *data)
{
struct mcctrl_per_thread_data *ptd_iter, *ptd = NULL;
struct mcctrl_per_thread_data *ptd_alloc = NULL;
int hash = (((uint64_t)current >> 4) & MCCTRL_PER_THREAD_DATA_HASH_MASK);
int ret = 0;
unsigned long flags;
ptd_alloc = kmalloc(sizeof(struct mcctrl_per_thread_data), GFP_ATOMIC);
if (!ptd_alloc) {
kprintf("%s: error allocate per thread data\n", __FUNCTION__);
ret = -ENOMEM;
goto out_noalloc;
}
memset(ptd_alloc, 0, sizeof(struct mcctrl_per_thread_data));
/* Check if data for this thread exists and add if not */
write_lock_irqsave(&ppd->per_thread_data_hash_lock[hash], flags);
list_for_each_entry(ptd_iter, &ppd->per_thread_data_hash[hash], hash) {
if (ptd_iter->task == current) {
ptd = ptd_iter;
break;
}
}
if (unlikely(ptd)) {
kprintf("%s: WARNING: ptd of tid: %d exists\n", __FUNCTION__, task_pid_vnr(current));
ret = -EBUSY;
kfree(ptd_alloc);
goto out;
}
ptd = ptd_alloc;
ptd->ppd = ppd;
ptd->task = current;
ptd->tid = task_pid_vnr(current);
ptd->data = data;
atomic_set(&ptd->refcount, 1);
list_add_tail(&ptd->hash, &ppd->per_thread_data_hash[hash]);
out:
write_unlock_irqrestore(&ppd->per_thread_data_hash_lock[hash], flags);
out_noalloc:
return ret;
}
@ -159,7 +181,7 @@ struct mcctrl_per_thread_data *mcctrl_get_per_thread_data(struct mcctrl_per_proc
int hash = (((uint64_t)task >> 4) & MCCTRL_PER_THREAD_DATA_HASH_MASK);
unsigned long flags;
/* Check if data for this thread exists and return it */
/* Check if data for this thread exists */
read_lock_irqsave(&ppd->per_thread_data_hash_lock[hash], flags);
list_for_each_entry(ptd_iter, &ppd->per_thread_data_hash[hash], hash) {
@ -169,8 +191,18 @@ struct mcctrl_per_thread_data *mcctrl_get_per_thread_data(struct mcctrl_per_proc
}
}
if (ptd) {
if (atomic_read(&ptd->refcount) <= 0) {
printk("%s: ERROR: use-after-free detected (%d)", __FUNCTION__, atomic_read(&ptd->refcount));
ptd = NULL;
goto out;
}
atomic_inc(&ptd->refcount);
}
out:
read_unlock_irqrestore(&ppd->per_thread_data_hash_lock[hash], flags);
return ptd ? ptd->data : NULL;
return ptd;
}
#endif /* !POSTK_DEBUG_ARCH_DEP_56 */
@ -299,6 +331,7 @@ long syscall_backward(struct mcctrl_usrdata *usrdata, int num,
struct wait_queue_head_list_node *wqhln;
unsigned long irqflags;
struct mcctrl_per_proc_data *ppd;
struct mcctrl_per_thread_data *ptd;
unsigned long phys;
struct syscall_request _request[2];
struct syscall_request *request;
@ -327,7 +360,14 @@ long syscall_backward(struct mcctrl_usrdata *usrdata, int num,
return -EINVAL;
}
packet = (struct ikc_scd_packet *)mcctrl_get_per_thread_data(ppd, current);
ptd = mcctrl_get_per_thread_data(ppd, current);
if (!ptd) {
printk("%s: ERROR: mcctrl_get_per_thread_data failed\n", __FUNCTION__);
syscall_ret = -ENOENT;
goto no_ptd;
}
pr_ptd("get", task_pid_vnr(current), ptd);
packet = (struct ikc_scd_packet *)ptd->data;
if (!packet) {
syscall_ret = -ENOENT;
printk("%s: no packet registered for TID %d\n",
@ -466,6 +506,9 @@ out:
ihk_device_unmap_memory(ihk_os_to_dev(usrdata->os), phys, sizeof(*resp));
out_put_ppd:
mcctrl_put_per_thread_data(ptd);
pr_ptd("put", task_pid_vnr(current), ptd);
no_ptd:
dprintk("%s: tid: %d, syscall: %d, syscall_ret: %lx\n",
__FUNCTION__, task_pid_vnr(current), num, syscall_ret);
@ -483,6 +526,7 @@ int remote_page_fault(struct mcctrl_usrdata *usrdata, void *fault_addr, uint64_t
struct wait_queue_head_list_node *wqhln;
unsigned long irqflags;
struct mcctrl_per_proc_data *ppd;
struct mcctrl_per_thread_data *ptd;
unsigned long phys;
int retry;
@ -498,11 +542,18 @@ int remote_page_fault(struct mcctrl_usrdata *usrdata, void *fault_addr, uint64_t
return -EINVAL;
}
packet = (struct ikc_scd_packet *)mcctrl_get_per_thread_data(ppd, current);
if (!packet) {
ptd = mcctrl_get_per_thread_data(ppd, current);
if (!ptd) {
printk("%s: ERROR: mcctrl_get_per_thread_data failed\n", __FUNCTION__);
error = -ENOENT;
goto no_ptd;
}
pr_ptd("get", task_pid_vnr(current), ptd);
packet = (struct ikc_scd_packet *)ptd->data;
if (!packet) {
printk("%s: no packet registered for TID %d\n",
__FUNCTION__, task_pid_vnr(current));
error = -ENOENT;
goto out_put_ppd;
}
@ -669,6 +720,9 @@ out:
ihk_device_unmap_memory(ihk_os_to_dev(usrdata->os), phys, sizeof(*resp));
out_put_ppd:
mcctrl_put_per_thread_data(ptd);
pr_ptd("put", task_pid_vnr(current), ptd);
no_ptd:
dprintk("%s: tid: %d, fault_addr: %p, reason: %lu, error: %d\n",
__FUNCTION__, task_pid_vnr(current), fault_addr, (unsigned long)reason, error);
@ -711,6 +765,7 @@ static int rus_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
size_t pix;
#endif
struct mcctrl_per_proc_data *ppd;
struct mcctrl_per_thread_data *ptd;
struct ikc_scd_packet *packet;
int ret = 0;
@ -740,7 +795,14 @@ static int rus_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
goto no_ppd;
}
packet = (struct ikc_scd_packet *)mcctrl_get_per_thread_data(ppd, current);
ptd = mcctrl_get_per_thread_data(ppd, current);
if (!ptd) {
printk("%s: ERROR: mcctrl_get_per_thread_data failed\n", __FUNCTION__);
ret = VM_FAULT_SIGBUS;
goto no_ptd;
}
pr_ptd("get", task_pid_vnr(current), ptd);
packet = (struct ikc_scd_packet *)ptd->data;
if (!packet) {
ret = VM_FAULT_SIGBUS;
printk("%s: no packet registered for TID %d\n",
@ -929,6 +991,9 @@ static int rus_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
ret = VM_FAULT_NOPAGE;
put_and_out:
mcctrl_put_per_thread_data(ptd);
pr_ptd("put", task_pid_vnr(current), ptd);
no_ptd:
mcctrl_put_per_proc_data(ppd);
no_ppd:
return ret;