System call delegation supports DMA alignment

kernel/include/syscall.h

@@ -1,6 +1,8 @@
 #ifndef __HEADER_SYSCALL_H
 #define __HEADER_SYSCALL_H
 
+#include <aal/memconst.h>
+
 #define NUM_SYSCALLS 255
 
 #define REQUEST_PAGE_COUNT              16
@@ -61,7 +63,7 @@ struct syscall_response {
 };
 
 struct syscall_post {
-	unsigned long v[4];
+	unsigned long v[8];
 };
 
 struct syscall_params {
@@ -77,7 +79,7 @@ struct syscall_params {
 	unsigned long post_rpa, post_pa;
 	struct syscall_post *post_va;
 	unsigned long post_fin;
-	struct syscall_post post_buf;
+	struct syscall_post post_buf AAL_DMA_ALIGN;
 };
 
 #endif

kernel/syscall.c

@@ -72,7 +72,7 @@ long sys_brk(int n, aal_mc_user_context_t *ctx)
 }
 
 #define SYSCALL_DECLARE(name) long sys_##name(int n, aal_mc_user_context_t *ctx)
-#define SYSCALL_HEADER struct syscall_request request; \
+#define SYSCALL_HEADER struct syscall_request request AAL_DMA_ALIGN; \
 	request.number = n
 #define SYSCALL_ARG_D(n)    request.args[n] = aal_mc_syscall_arg##n(ctx)
 #define SYSCALL_ARG_MO(n) \
@@ -257,7 +257,7 @@ SYSCALL_DECLARE(uname)
 
 long sys_getxid(int n, aal_mc_user_context_t *ctx)
 {
-	struct syscall_request request;
+	struct syscall_request request AAL_DMA_ALIGN;
 
 	request.number = n;
 
@@ -287,7 +287,7 @@ SYSCALL_DECLARE(clone)
 {
 	/* Clone a new thread */
 	struct process *new;
-	struct syscall_request request;
+	struct syscall_request request AAL_DMA_ALIGN;
 
 	new = clone_process(cpu_local_var(current), aal_mc_syscall_pc(ctx),
 	                    aal_mc_syscall_arg1(ctx));
@@ -332,8 +332,6 @@ static long (*syscall_table[])(int, aal_mc_user_context_t *) = {
 	[231] = sys_exit_group,
 };
 
-#define DEBUG_PRINT_SC
-
 long syscall(int num, aal_mc_user_context_t *ctx)
 {
 	long l;
@@ -345,9 +343,7 @@ long syscall(int num, aal_mc_user_context_t *ctx)
 	        aal_mc_get_processor_id(),
 	        num,
 	        aal_mc_syscall_arg0(ctx), aal_mc_syscall_arg1(ctx),
-	        aal_mc_syscall_arg2(ctx), aal_mc_syscall_arg3(ctx),
+	        aal_mc_syscall_pc(ctx), aal_mc_syscall_sp(ctx));
-	        aal_mc_syscall_arg4(ctx), aal_mc_syscall_pc(ctx),
-	        aal_mc_syscall_sp(ctx));
 #endif
 
 	if (syscall_table[num]) {

linux/mod_mcctrl/control.c

@@ -8,6 +8,12 @@
 #include <asm/msr.h>
 #include "mcctrl.h"
 
+#ifdef DEBUG
+#define dprintk printk
+#else
+#define dprintk(...)
+#endif
+
 static DECLARE_WAIT_QUEUE_HEAD(wq_prepare);
 extern struct mcctrl_channel *channels;
 int mcctrl_ikc_set_recv_cpu(int cpu);
@@ -145,6 +151,7 @@ int __do_in_kernel_syscall(aal_os_t os, struct mcctrl_channel *c,
 
 static int remaining_job, base_cpu, job_pos;
 extern int num_channels;
+extern int mcctrl_dma_abort;
 
 int mcexec_wait_syscall(aal_os_t os, struct syscall_wait_desc *__user req)
 {
@@ -163,6 +170,7 @@ int mcexec_wait_syscall(aal_os_t os, struct syscall_wait_desc *__user req)
 	c->req = 0;
 #else
 	while (1) {
+		c = channels + swd.cpu;
 		rdtscll(s);
 		if (!remaining_job) {
 			while (!(*c->param.doorbell_va)) {
@@ -193,9 +201,15 @@ int mcexec_wait_syscall(aal_os_t os, struct syscall_wait_desc *__user req)
 			} else {
 				c = channels + base_cpu + job_pos;
 			}
+			if (!c) {
+				continue;
+			}
 			if (c->param.request_va &&
 			    c->param.request_va->valid) {
 				c->param.request_va->valid = 0; /* ack */
+				dprintk("SC #%lx, %lx\n",
+				        c->param.request_va->number,
+				        c->param.request_va->args[0]);
 		if (__do_in_kernel_syscall(os, c, c->param.request_va)) {
 #endif
 			if (copy_to_user(&req->sr, c->param.request_va,
@@ -204,6 +218,9 @@ int mcexec_wait_syscall(aal_os_t os, struct syscall_wait_desc *__user req)
 			}
 #ifndef DO_USER_MODE
 			return 0;
+		}
+		if (mcctrl_dma_abort) {
+			return -2;
 		}
 			}
 		}
@@ -283,16 +300,33 @@ long mcexec_ret_syscall(aal_os_t os, struct syscall_ret_desc *__user arg)
 	mc->param.response_va->ret = ret.ret;
 
 	if (ret.size > 0) {
+		/* Host => Accel. Write is fast. */
+		unsigned long phys;
+		void *rpm;
+
+		phys = aal_device_map_memory(aal_os_to_dev(os), ret.dest,
+		                             ret.size);
+		rpm = ioremap_wc(phys, ret.size);
+		
+		memcpy(rpm, phys_to_virt(ret.src), ret.size);
+		mc->param.response_va->status = 1;
+
+		iounmap(rpm);
+		aal_device_unmap_memory(aal_os_to_dev(os), phys, ret.size);
+
+/*
 		memset(&request, 0, sizeof(request));
 		request.src_os = NULL;
 		request.src_phys = ret.src;
 		request.dest_os = os;
 		request.dest_phys = ret.dest;
 		request.size = ret.size;
-		request.notify = (void *)mc->param.response_pa;
+		request.notify_os = os;
+		request.notify = (void *)mc->param.response_rpa;
 		request.priv = (void *)1;
 		
 		aal_dma_request(channel, &request);
+*/
 	} else {
 		mc->param.response_va->status = 1;
 	}

linux/mod_mcctrl/driver.c

@@ -11,6 +11,7 @@
 
 extern long __mcctrl_control(aal_os_t, unsigned int, unsigned long);
 extern int prepare_ikc_channels(aal_os_t os);
+extern void destroy_ikc_channels(aal_os_t os);
 
 static long mcctrl_ioctl(aal_os_t os, unsigned int request, void *priv,
                          unsigned long arg)
@@ -47,13 +48,14 @@ static int __init mcctrl_init(void)
 		return -EINVAL;
 	}
 
-
 	return aal_os_register_user_call_handlers(os, &mcctrl_uc);
 }
 
 static void __exit mcctrl_exit(void)
 {
+	printk("mcctrl: unregistered.\n");
 	aal_os_unregister_user_call_handlers(os, &mcctrl_uc);
+	destroy_ikc_channels(os);
 }
 
 MODULE_LICENSE("GPL v2");

linux/mod_mcctrl/ikc.c

@@ -7,6 +7,8 @@
 #include <aal/ikc.h>
 #include <ikc/master.h>
 
+#define REQUEST_SHIFT    16
+
 int num_channels;
 
 struct mcctrl_channel *channels;
@@ -98,7 +100,9 @@ static void mcctrl_ikc_init(aal_os_t os, int cpu, unsigned long rphys)
 	                             sizeof(struct ikc_scd_init_param));
 	rpm = ioremap_wc(phys, sizeof(struct ikc_scd_init_param));
 
-	pmc->param.request_va = (void *)__get_free_pages(GFP_KERNEL, 4);
+	pmc->param.request_va =
+		(void *)__get_free_pages(GFP_KERNEL,
+		                         REQUEST_SHIFT - PAGE_SHIFT);
 	pmc->param.request_pa = virt_to_phys(pmc->param.request_va);
 	pmc->param.doorbell_va = mcctrl_doorbell_va;
 	pmc->param.doorbell_pa = mcctrl_doorbell_pa;
@@ -199,3 +203,33 @@ int prepare_ikc_channels(aal_os_t os)
 	aal_ikc_listen_port(os, &listen_param);
 	return 0;
 }
+
+void __destroy_ikc_channel(aal_os_t os, struct mcctrl_channel *pmc)
+{
+	free_pages((unsigned long)pmc->param.request_va,
+	           REQUEST_SHIFT - PAGE_SHIFT);
+	free_page((unsigned long)pmc->param.post_va);
+
+	iounmap(pmc->param.response_va);
+	aal_device_unmap_memory(aal_os_to_dev(os),
+	                        pmc->param.response_pa, PAGE_SIZE);
+	free_pages((unsigned long)pmc->dma_buf,
+	           DMA_PIN_SHIFT - PAGE_SHIFT);
+}
+
+void destroy_ikc_channels(aal_os_t os)
+{
+	int i;
+
+	for (i = 0; i < num_channels; i++) {
+		if (channels[i].c) {
+//			aal_ikc_disconnect(channels[i].c);
+			aal_ikc_free_channel(channels[i].c);
+			__destroy_ikc_channel(os, channels + i);
+			printk("Channel #%d freed.\n", i);
+		}
+	}
+	free_page((unsigned long)mcctrl_doorbell_va);
+
+	kfree(channels);
+}

linux/mod_mcctrl/mcctrl.h

@@ -14,7 +14,7 @@
 
 #define SCD_MSG_SYSCALL_ONESIDE         0x4
 
-#define DMA_PIN_SHIFT                   16
+#define DMA_PIN_SHIFT                   21
 
 struct ikc_scd_packet {
 	int msg;
@@ -35,7 +35,7 @@ struct ikc_scd_init_param {
 };
 
 struct syscall_post {
-	unsigned long v[4];
+	unsigned long v[8];
 };
 
 struct syscall_params {

linux/mod_mcctrl/syscall.c

@@ -9,11 +9,28 @@
 #include <asm/delay.h>
 #include "mcctrl.h"
 
+#define ALIGN_WAIT_BUF(z)   (((z + 63) >> 6) << 6)
+
+#define SC_DEBUG
+#ifdef SC_DEBUG
+static struct aal_dma_request last_request;
+
+static void print_dma_lastreq(void)
+{
+	printk("SRC OS : %p | %lx\nDESTOS : %p | %lx\n", last_request.src_os,
+	       last_request.src_phys, last_request.dest_os,
+	       last_request.dest_phys);
+	printk("SIZE   : %lx | NOTIFY : %p | PRIV : %p\n",
+	       last_request.size, last_request.notify, last_request.priv);
+}
+#endif
+
 static int do_async_copy(aal_os_t os, unsigned long dest, unsigned long src,
                          unsigned long size, unsigned int inbound)
 {
 	struct aal_dma_request request;
 	aal_dma_channel_t channel;
+	unsigned long asize = ALIGN_WAIT_BUF(size);
 
 	channel = aal_device_get_dma_channel(aal_os_to_dev(os), 0);
 	if (!channel) {
@@ -21,28 +38,48 @@ static int do_async_copy(aal_os_t os, unsigned long dest, unsigned long src,
 	}
 
 	memset(&request, 0, sizeof(request));
-	request.src_os = os;
+	request.src_os = inbound ? os : NULL;
 	request.src_phys = src;
-	request.dest_os = NULL;
+	request.dest_os = inbound ? NULL : os;
 	request.dest_phys = dest;
 	request.size = size;
-	request.notify = (void *)(inbound ? dest + size : src + size);
+	request.notify = (void *)(inbound ? dest + asize : src + asize);
 	request.priv = (void *)1;
 
+	*(unsigned long *)phys_to_virt((unsigned long)request.notify) = 0;
+#ifdef SC_DEBUG
+	last_request = request;
+#endif
+
 	aal_dma_request(channel, &request);
 
 	return 0;
 }
 
+int mcctrl_dma_abort;
+
 static void async_wait(unsigned char *p, int size)
 {
-	while (!p[size]) {
+	int asize = ALIGN_WAIT_BUF(size);
+	unsigned long long s, w;
+
+	rdtscll(s);
+	while (!p[asize]) {
 		mb();
 		cpu_relax();
+		rdtscll(w);
+		if (w > s + 1024UL * 1024 * 1024 * 10) {
+			printk("DMA Timed out : %p (%p + %d) => %d\n",
+			       p + asize, p, size, p[asize]);
+			print_dma_lastreq();
+			mcctrl_dma_abort = 1;
+			return;
+		}
 	}
 }
 static void clear_wait(unsigned char *p, int size)
 {
+	int asize = ALIGN_WAIT_BUF(size);
 	p[size] = 0;
 }
 
@@ -75,7 +112,6 @@ static unsigned long translate_remote_va(struct mcctrl_channel *c,
 
 	return -EFAULT;
 }
-
 unsigned long last_thread_exec = 0;
 
 extern struct mcctrl_channel *channels;