Almost implemented.

arch/x86/kernel/gencore.c

@@ -273,10 +273,10 @@ void fill_note(void *note, struct process *proc, void *regs)
 int gencore(struct process *proc, void *regs, 
 	    struct coretable **coretable, int *chunks)
 {
-	struct coretable *ct;
+	struct coretable *ct = NULL;
 	Elf64_Ehdr eh;
-	Elf64_Phdr *ph;
+	Elf64_Phdr *ph = NULL;
-	void *note;
+	void *note = NULL;
 	struct vm_range *range;
 	struct process_vm *vm = proc->vm;
 	int segs = 1;	/* the first one is for NOTE */

executer/kernel/driver.c

@@ -35,6 +35,9 @@ extern void destroy_ikc_channels(ihk_os_t os);
 #ifndef DO_USER_MODE
 extern void mcctrl_syscall_init(void);
 #endif
+extern void procfs_init(int);
+extern void procfs_exit(int);
+
 
 static long mcctrl_ioctl(ihk_os_t os, unsigned int request, void *priv,
                          unsigned long arg)
@@ -104,8 +107,10 @@ static int __init mcctrl_init(void)
 				destroy_ikc_channels(os[i]);
 				os[i] = NULL;
 			}
+			procfs_init(i);
 		}
 	}
+
 	return 0;
 }
 
@@ -118,6 +123,7 @@ static void __exit mcctrl_exit(void)
 		if(os[i]){
 			ihk_os_unregister_user_call_handlers(os[i], mcctrl_uc + i);
 			destroy_ikc_channels(os[i]);
+			procfs_exit(i);
 		}
 	}
 }

executer/kernel/ikc.c

@@ -88,7 +88,8 @@ int mcctrl_ikc_send(ihk_os_t os, int cpu, struct ikc_scd_packet *pisp)
 {
 	struct mcctrl_usrdata *usrdata = ihk_host_os_get_usrdata(os);
 
-	if (cpu < 0 || cpu >= usrdata->num_channels || !usrdata->channels[cpu].c) {
+	if (cpu < 0 || os == NULL || usrdata == NULL ||
+	    cpu >= usrdata->num_channels || !usrdata->channels[cpu].c) {
 		return -EINVAL;
 	}
 	return ihk_ikc_send(usrdata->channels[cpu].c, pisp, 0);

executer/kernel/mcctrl.h

@@ -166,6 +166,7 @@ struct procfs_read {
 	int count;		/* bytes to read (request) */
 	int eof;		/* if eof is detected, 1 otherwise 0. (answer)*/
 	int ret;		/* read bytes (answer) */
+	int newcpu;		/* migrated new cpu (answer) */
 	char fname[PROCFS_NAME_MAX];	/* procfs filename (request) */
 };
 

executer/kernel/procfs.c

@@ -0,0 +1,321 @@
+/**
+ * \file procfs.c
+ *  License details are found in the file LICENSE.
+ * \brief
+ *  mcctrl procfs
+ * \author Naoki Hamada <nao@axe.bz> \par
+ * 	Copyright (C) 2014  AXE, Inc.
+ */
+/*
+ * HISTORY:
+ */
+
+#include <linux/string.h>
+#include <linux/proc_fs.h>
+#include <linux/list.h>
+#include <linux/uaccess.h>
+#include <linux/fs.h>
+#include "mcctrl.h"
+
+#define PROCFS_DEBUG
+
+#ifdef PROCFS_DEBUG
+#define	dprintk(...)	printk(__VA_ARGS__)
+#else
+#define	dprintk(...)
+#endif
+
+static DECLARE_WAIT_QUEUE_HEAD(procfsq);
+static unsigned long procfsq_channel;
+
+int mckernel_procfs_read(char *buffer, char **start, off_t offset,
+			 int count, int *peof, void *dat);
+
+/* A private data for the procfs driver. */
+
+struct procfs_list_entry {
+	struct list_head list;
+	struct proc_dir_entry *entry;
+	struct proc_dir_entry *parent;
+	ihk_os_t os;
+	int osnum;
+	int pid;
+	int cpu;
+	char fname[PROCFS_NAME_MAX];
+};
+
+LIST_HEAD(procfs_file_list);
+static ihk_spinlock_t procfs_file_list_lock;
+
+/*
+ * char *p a name of the procfs file
+ * int mode if zero create a directory otherwise a file
+ *
+ * return value: NULL: Something wrong has occurred.
+ *               otherwise: address of the proc_dir_entry structure of the procfs file
+ *
+ * p should not be NULL nor terminated by "/".
+ *
+ * We create a procfs entry if there is not already one.
+ * This process is recursive to the root of the procfs tree.
+ */
+
+static struct proc_dir_entry *get_procfs_entry(char *p, int mode)
+{
+	char *r;
+	struct proc_dir_entry *ret = NULL, *parent = NULL;
+	struct procfs_list_entry *e;
+	char name[PROCFS_NAME_MAX];
+	unsigned long irqflags;
+
+	dprintk("get_procfs_entry: %s for mode %o\n", p, mode);
+	irqflags = ihk_ikc_spinlock_lock(&procfs_file_list_lock);
+	list_for_each_entry(e, &procfs_file_list, list) {
+		if (e == NULL) {
+			kprintf("ERROR: The procfs_file_list has a null entry.\n");
+			return NULL;
+		}
+		if (strncmp(e->fname, p, PROCFS_NAME_MAX) == 0) {
+			/* We found the entry */
+			ret = e->entry;
+		}
+	}
+	ihk_ikc_spinlock_unlock(&procfs_file_list_lock, irqflags);
+	if (ret != NULL) {
+		return ret;
+	}
+	r = strrchr(p, '/');
+	if (r != NULL) {
+		/* We have non-null parent dir. */
+		strncpy(name, p, r - p);
+		name[r - p] = '\0'; 
+		parent = get_procfs_entry(name, 0);
+		if (parent == NULL) {
+			/* We counld not get a parent procfs entry. Give up.*/
+			return NULL;
+		}
+	}
+	e = kmalloc(sizeof(struct procfs_list_entry), GFP_KERNEL);
+	if (e == NULL) {
+		kprintf("ERROR: not enough memory to create PROCFS entry.\n");
+		return NULL;
+	}
+	/* Fill the fname field of the entry */
+	strncpy(e->fname, p, PROCFS_NAME_MAX);
+
+	if (r != NULL) {
+		strncpy(name, r + 1, p + PROCFS_NAME_MAX - r - 1);
+	} else {
+		strncpy(name, p, PROCFS_NAME_MAX);
+	}
+	if (mode == 0) {
+		ret = proc_mkdir(name, parent);
+	} else {	
+		ret = create_proc_entry(name, mode, parent);
+	}
+	if (ret == NULL) {
+		kprintf("ERROR: cannot create a PROCFS entry for %s.\n", p);
+		kfree(e);
+		return NULL;
+	}
+	ret->data = e;
+	e->entry = ret;
+	e->parent = parent;
+
+	irqflags = ihk_ikc_spinlock_lock(&procfs_file_list_lock);
+	list_add(&(e->list), &procfs_file_list);
+	ihk_ikc_spinlock_unlock(&procfs_file_list_lock, irqflags);
+
+	dprintk("get_procfs_entry: %s done\n", p);
+	return ret;
+}
+
+void procfs_create(void *__os, int ref, int osnum, int pid, unsigned long arg)
+{
+	struct proc_dir_entry *entry;
+	struct procfs_list_entry *e;
+	ihk_device_t dev = ihk_os_to_dev(__os);
+	unsigned long parg;
+	struct procfs_file *f;
+	int mode;
+	char name[PROCFS_NAME_MAX];
+
+	dprintk("procfs_create: osnum: %d, cpu: %d, pid: %d\n", osnum, ref, pid);
+
+	parg = ihk_device_map_memory(dev, arg, sizeof(struct procfs_file));
+	f = ihk_device_map_virtual(dev, parg, sizeof(struct procfs_file), NULL, 0);
+
+	dprintk("name: %s mode: %o\n", f->fname, f->mode);
+
+	strncpy(name, f->fname, PROCFS_NAME_MAX);
+	mode = f->mode;
+	f->status = 1; /* Now the peer can free the data. */
+
+	ihk_device_unmap_virtual(dev, f, sizeof(struct procfs_file));
+	ihk_device_unmap_memory(dev, parg, sizeof(struct procfs_file));
+
+	if (name[PROCFS_NAME_MAX - 1] != '\0') {
+			printk("ERROR: procfs_creat: file name not properly terminated.\n");
+			goto quit;
+	}
+	entry = get_procfs_entry(f->fname, mode);
+	if (entry == NULL) {
+		printk("ERROR: could not create a procfs entry for %s.\n", f->fname);
+		goto quit;
+	}
+
+	e = entry->data;
+	e->osnum = osnum;
+	e->os = __os;
+	e->cpu = ref;
+	e->pid = pid;
+
+	entry->read_proc = mckernel_procfs_read;
+quit:
+	dprintk("procfs_create: done\n");
+}
+
+void procfs_delete(void *__os, unsigned long arg)
+{
+	ihk_device_t dev = ihk_os_to_dev(__os);
+	unsigned long parg;
+	struct procfs_file *f;
+	struct procfs_list_entry *e;
+	struct proc_dir_entry *parent = NULL;
+	char name[PROCFS_NAME_MAX];
+	char *r;
+	unsigned long irqflags;
+
+	dprintk("procfs_delete: \n");
+	parg = ihk_device_map_memory(dev, arg, sizeof(struct procfs_file));
+	f = ihk_device_map_virtual(dev, parg, sizeof(struct procfs_file), NULL, 0);
+	dprintk("fname: %s.\n", f->fname);
+	irqflags = ihk_ikc_spinlock_lock(&procfs_file_list_lock);
+	list_for_each_entry(e, &procfs_file_list, list) {
+		if (strncmp(e->fname, f->fname, PROCFS_NAME_MAX) == 0) {
+			dprintk("found and delete an entry in the list.\n");
+			list_del(&e->list);
+			e->entry->read_proc = NULL;
+			e->entry->data = NULL;
+			parent = e->parent;
+			kfree(e);
+			break;
+		}
+	}
+	ihk_ikc_spinlock_unlock(&procfs_file_list_lock, irqflags);
+	r = strrchr(f->fname, '/');
+	if (r == NULL) {
+		strncpy(name, f->fname, PROCFS_NAME_MAX);
+	} else {
+		strncpy(name, r + 1, PROCFS_NAME_MAX);
+	}
+	remove_proc_entry(name, parent);
+	f->status = 1; /* Now the peer can free the data. */
+	ihk_device_unmap_virtual(dev, f, sizeof(struct procfs_file));
+	ihk_device_unmap_memory(dev, parg, sizeof(struct procfs_file));
+	dprintk("procfs_delete: done\n");
+}
+
+void procfs_answer(unsigned int arg)
+{
+	dprintk("procfs: received SCD_MSG_PROCFS_ANSWER message.\n");
+	procfsq_channel = arg;
+	wake_up_interruptible(&procfsq);
+}
+
+/*
+ * callback funciton for McKernel procfs
+ *
+ * This function conforms to the 2) way of fs/proc/generic.c
+ * from linux-2.6.39.4.
+ */
+
+static void *channel;
+
+int mckernel_procfs_read(char *buffer, char **start, off_t offset,
+			 int count, int *peof, void *dat)
+{
+	struct procfs_list_entry *e = dat;
+	struct procfs_read *r;
+	struct ikc_scd_packet isp;
+	int ret;
+	unsigned long pbuf;
+
+	dprintk("mckernel_procfs_read: invoked for %s\n", e->fname); 
+	dprintk("offset: %lx, count: %d\n", offset, count);
+
+	if (count <= 0 || dat == NULL) {
+		return 0;
+	}
+
+	pbuf = virt_to_phys(buffer);
+	if (pbuf / PAGE_SIZE != (pbuf + count - 1) / PAGE_SIZE) {
+		/* Truncate the read count upto the nearest page boundary */
+		count = ((pbuf + count - 1) / PAGE_SIZE) * PAGE_SIZE - pbuf;
+	}
+	r = kmalloc(sizeof(struct procfs_read), GFP_KERNEL);
+	if (r == NULL) {
+		return -ENOMEM;
+	}
+retry:
+	r->pbuf = pbuf;
+	r->eof = 0;
+	r->ret = 0;
+	r->offset = offset;
+	r->count = count;
+	strncpy(r->fname, e->fname, PROCFS_NAME_MAX);
+	isp.msg = SCD_MSG_PROCFS_REQUEST;
+	isp.ref = e->cpu;
+	isp.arg = virt_to_phys(r);
+	ret = mcctrl_ikc_send(e->os, e->cpu, &isp);
+	if (ret < 0) {
+		return ret; /* error */
+	}
+	channel = NULL;
+	/* Wait for a reply. */
+	dprintk("now wait for a relpy\n");
+	wait_event_interruptible(procfsq, procfsq_channel == virt_to_phys(r));
+	/* Wake up and check the result. */
+	dprintk("mckernel_procfs_read: woke up. ret: %d, eof: %d\n", r->ret, r->eof);
+	if ((r->ret == 0) && (r->eof != 1)) {
+		/* A miss-hit caused by migration has occurred.
+		 * We simply retry the query with a new CPU.
+		 */
+		e->cpu = r->newcpu;
+		dprintk("retry\n");
+		goto retry;
+	}
+	if (r->eof == 1) {
+		*peof = 1;
+	}
+	/* We employ the 2) method for a proc read function.
+	 * Refer to fs/proc/generic.c of the Linux kernel 
+	 * for the details.
+	 */
+	*start = buffer;
+	ret = r->ret;
+	kfree(r);
+	
+	return ret;
+}
+
+void procfs_init(int i) {
+}
+
+void procfs_exit(int i) {
+	char buf[20];
+	int error;
+	mm_segment_t old_fs = get_fs();
+	struct kstat stat;
+
+	sprintf(buf, "/proc/mcos%d", i);
+
+	set_fs(KERNEL_DS);
+	error = vfs_stat (buf, &stat);
+	set_fs(old_fs);
+	if (error != 0) {
+		return;
+	}
+
+	remove_proc_entry(buf + 6, NULL);
+}

executer/kernel/syscall.c

@@ -1169,7 +1169,7 @@ static void clear_pte_range(uintptr_t start, uintptr_t len)
 static int writecore(ihk_os_t os, unsigned long rcoretable, int chunks) {
 	struct file *file;
 	struct coretable *coretable;
-	int ret, len, i, tablesize, size, error = 0;
+	int ret, i, tablesize, size, error = 0;
 	mm_segment_t oldfs = get_fs(); 
 	unsigned long phys, tablephys, rphys;
 	ihk_device_t dev = ihk_os_to_dev(os);

kernel/include/syscall.h

@@ -250,6 +250,7 @@ struct procfs_read {
 	int count;		/* bytes to read (request) */
 	int eof;		/* if eof is detected, 1 otherwise 0. (answer)*/
 	int ret;		/* read bytes (answer) */
+	int newcpu;		/* migrated new cpu (answer) */
 	char fname[PROCFS_NAME_MAX];	/* procfs filename (request) */
 };
 

kernel/init.c

@@ -35,7 +35,7 @@
 #include <ioctl.h>
 #endif
 
-//#define DEBUG_PRINT_INIT
+#define DEBUG_PRINT_INIT
 
 #ifdef DEBUG_PRINT_INIT
 #define dkprintf kprintf
@@ -141,7 +141,7 @@ void pc_init(void)
 			osnum += *p++ - '0';
 		}
 	}
-	kprintf("osnum: %d\n");
+	kprintf("osnum: %d\n", osnum);
 
 
 	if (!(p = find_command_line("perfctr"))) {

kernel/procfs.c

@@ -26,7 +26,7 @@
 
 #define DEBUG_PRINT_PROCFS
 
-#ifdef DEBUG_PRINT_SC
+#ifdef DEBUG_PRINT_PROCFS
 #define	dprintf(...) kprintf(__VA_ARGS__)
 #else
 #define dprintf(...)
@@ -50,13 +50,17 @@ void create_proc_procfs_files(int pid, int cpuid)
 {
 	char fname[PROCFS_NAME_MAX];
 
-	snprintf(fname, PROCFS_NAME_MAX, "mcos%d/%d/mem", osnum, pid);
+	dprintf("create procfs files:\n");
+
+	snprintf(fname, PROCFS_NAME_MAX, "mcos%d/%d/task/%d/mem", osnum, pid, pid);
 	create_proc_procfs_file(pid, fname, 0400, cpuid);
+
+	dprintf("create procfs files: done\n");
 }
 
 static void create_proc_procfs_file(int pid, char *fname, int mode, int cpuid)
 {
-	dprintf("create procfs file: %s, mode: %s, cpuid: %d\n", fname, mode, cpuid);
+	dprintf("create procfs file: %s, mode: %o, cpuid: %d\n", fname, mode, cpuid);
 	operate_proc_procfs_file(pid, fname, SCD_MSG_PROCFS_CREATE, mode, cpuid);
 }
 
@@ -64,14 +68,32 @@ void delete_proc_procfs_files(int pid)
 {
 	char fname[PROCFS_NAME_MAX];
 
-	snprintf(fname, PROCFS_NAME_MAX, "mcos%d/%d/mem", osnum, pid);
+	dprintf("delete procfs files\n");
+	snprintf(fname, PROCFS_NAME_MAX, "mcos%d/%d/task/%d/mem", osnum, pid, pid);
 	delete_proc_procfs_file(pid, fname);
+
+	snprintf(fname, PROCFS_NAME_MAX, "mcos%d/%d/task/%d", osnum, pid, pid);
+	delete_proc_procfs_file(pid, fname);
+
+	snprintf(fname, PROCFS_NAME_MAX, "mcos%d/%d/task", osnum, pid);
+	delete_proc_procfs_file(pid, fname);
+
+	snprintf(fname, PROCFS_NAME_MAX, "mcos%d/%d", osnum, pid);
+	delete_proc_procfs_file(pid, fname);
+
+	/* CAVEAT: deleting mcos%d level procfs directory should be located 
+	   in delete_mckernel_procfs_files().*/
+	snprintf(fname, PROCFS_NAME_MAX, "mcos%d", osnum);
+	delete_proc_procfs_file(pid, fname);
+
+	dprintf("delete procfs files: done\n");
 }
 
 static void delete_proc_procfs_file(int pid, char *fname)
 {
 	dprintf("delete procfs file: %s\n", fname);
 	operate_proc_procfs_file(pid, fname, SCD_MSG_PROCFS_DELETE, 0, 0);
+	dprintf("delete procfs file: %s done\n", fname);
 }
 
 static void operate_proc_procfs_file(int pid, char *fname, int msg, int mode, int cpuid)
@@ -131,14 +153,20 @@ void process_procfs_request(unsigned long rarg)
 	pbuf = ihk_mc_map_memory(NULL, r->pbuf, r->count);
 	buf = ihk_mc_map_virtual(pbuf, r->count, PTATTR_WRITABLE | PTATTR_ACTIVE);
 
-	dprintf("fname:%s, offset: %lx, count:%d.\n", r->fname, r->offset, r->count);
+	dprintf("fname: %s, offset: %lx, count:%d.\n", r->fname, r->offset, r->count);
 
-	/* mcos0/PID/taks/PID/mem */
+	/* mcos0/PID/taks/PID/mem
+	 *
+	 * The offset is treated as the beginning of the virtual address area
+	 * of the process. The count is the length of the area.
+	 */
 	ret = sscanf(r->fname, "mcos%d/%d/task/%d/mem", &rosnum, &pid, &tid);
 	if ((ret == 3) && (pid == tid) && (osnum == rosnum)) {
 		if (cpu_local_var(current)->pid != pid) {
-			/* When the target process is no more here.
+			/* The target process has gone by migration. */
-			   This kind of hit-misses are caused by migration */ 
+#ifdef FIXME
+			r->newcpu = ...
+#endif
 			ans = 0;
 		} else {
 			struct vm_range *range;
@@ -147,16 +175,13 @@ void process_procfs_request(unsigned long rarg)
 			list_for_each_entry(range, &vm->vm_range_list, list) {
 				dprintf("range: %lx - %lx\n", range->start, range->end);
 				if ((range->start <= r->offset) && 
-				    (r->offset <= range->end)) {
+				    (r->offset < range->end)) {
-					if (r->offset + r->count <= range->end) {
+					unsigned int len = r->count;
-						memcpy((void *) buf, (void *) range->start, r->count);
+					if (range->end < r->offset + r->count) {
-						ans = r->count;
+						len = range->end - r->offset;
-					} else {
-						unsigned int remain;
-						remain = range->end - r->offset;
-						memcpy((void *) buf, (void *)range->start, remain);
-						ans = remain;
 					}
+					memcpy((void *) buf, (void *)range->start, len);
+					ans = len;
 					break;
 				}
 			}