add shmobj for shared anonymous mappings

kernel/Makefile.build

@@ -1,6 +1,6 @@
 IHKDIR=$(IHKBASE)/$(TARGETDIR)
 OBJS = init.o mem.o debug.o mikc.o listeners.o ap.o syscall.o cls.o host.o
-OBJS += process.o copy.o waitq.o futex.o timer.o plist.o fileobj.o
+OBJS += process.o copy.o waitq.o futex.o timer.o plist.o fileobj.o shmobj.o
 DEPSRCS=$(wildcard $(SRC)/*.c)
 
 CFLAGS += -I$(SRC)/include -mcmodel=kernel -D__KERNEL__

kernel/include/memobj.h

@@ -18,6 +18,7 @@
 #include <ihk/lock.h>
 #include <errno.h>
 #include <list.h>
+#include <shm.h>
 
 enum {
 	/* for memobj.flags */
@@ -101,5 +102,6 @@ static inline int memobj_has_pager(struct memobj *obj)
 }
 
 int fileobj_create(int fd, struct memobj **objp, int *maxprotp);
+int shmobj_create(struct shmid_ds *ds, struct memobj **objp);
 
 #endif /* HEADER_MEMOBJ_H */

kernel/include/shm.h

@@ -0,0 +1,49 @@
+/**
+ * \file shm.h
+ *  License details are found in the file LICENSE.
+ * \brief
+ *  header file for System V shared memory
+ * \author Gou Nakamura  <go.nakamura.yw@hitachi-solutions.com>
+ */
+/*
+ * HISTORY:
+ */
+
+#ifndef HEADER_SHM_H
+#define HEADER_SHM_H
+
+/* begin types.h */
+typedef int32_t key_t;
+typedef uint32_t uid_t;
+typedef uint32_t gid_t;
+typedef int64_t time_t;
+typedef int32_t pid_t;
+/* end types.h */
+
+typedef uint64_t shmatt_t;
+
+struct ipc_perm {
+	key_t		key;
+	uid_t		uid;
+	gid_t		gid;
+	uid_t		cuid;
+	gid_t		cgid;
+	uint16_t	mode;
+	uint8_t		padding[2];
+	uint16_t	seq;
+	uint8_t		padding2[22];
+};
+
+struct shmid_ds {
+	struct ipc_perm	shm_perm;
+	size_t		shm_segsz;
+	time_t		shm_atime;
+	time_t		shm_dtime;
+	time_t		shm_ctime;
+	pid_t		shm_cpid;
+	pid_t		shm_lpid;
+	shmatt_t	shm_nattch;
+	uint8_t		padding[16];
+};
+
+#endif /* HEADER_SHM_H */

kernel/shmobj.c

@@ -0,0 +1,287 @@
+/**
+ * \file shmobj.c
+ *  License details are found in the file LICENSE.
+ * \brief
+ *  shared memory object
+ * \author Gou Nakamura  <go.nakamura.yw@hitachi-solutions.com>
+ */
+/*
+ * HISTORY:
+ */
+
+#include <ihk/atomic.h>
+#include <ihk/debug.h>
+#include <ihk/lock.h>
+#include <ihk/mm.h>
+#include <errno.h>
+#include <kmalloc.h>
+#include <list.h>
+#include <memobj.h>
+#include <memory.h>
+#include <page.h>
+#include <shm.h>
+#include <string.h>
+
+#define	dkprintf(...)	do { if (0) kprintf(__VA_ARGS__); } while (0)
+#define	ekprintf(...)	kprintf(__VA_ARGS__)
+#define	fkprintf(...)	kprintf(__VA_ARGS__)
+
+struct shmobj {
+	struct memobj		memobj;		/* must be first */
+	long			ref;
+	struct shmid_ds		ds;
+	struct list_head	page_list;
+};
+
+static memobj_release_func_t shmobj_release;
+static memobj_ref_func_t shmobj_ref;
+static memobj_get_page_func_t shmobj_get_page;
+
+static struct memobj_ops shmobj_ops = {
+	.release =	&shmobj_release,
+	.ref =		&shmobj_ref,
+	.get_page =	&shmobj_get_page,
+};
+
+static struct shmobj *to_shmobj(struct memobj *memobj)
+{
+	return (struct shmobj *)memobj;
+}
+
+static struct memobj *to_memobj(struct shmobj *shmobj)
+{
+	return &shmobj->memobj;
+}
+
+/***********************************************************************
+ * page_list
+ */
+static void page_list_init(struct shmobj *obj)
+{
+	INIT_LIST_HEAD(&obj->page_list);
+	return;
+}
+
+static void page_list_insert(struct shmobj *obj, struct page *page)
+{
+	list_add(&page->list, &obj->page_list);
+	return;
+}
+
+static void page_list_remove(struct shmobj *obj, struct page *page)
+{
+	list_del(&page->list);
+	return;
+}
+
+static struct page *page_list_lookup(struct shmobj *obj, off_t off)
+{
+	struct page *page;
+
+	list_for_each_entry(page, &obj->page_list, list) {
+		if (page->offset == off) {
+			goto out;
+		}
+	}
+	page = NULL;
+
+out:
+	return page;
+}
+
+static struct page *page_list_first(struct shmobj *obj)
+{
+	if (list_empty(&obj->page_list)) {
+		return NULL;
+	}
+
+	return list_first_entry(&obj->page_list, struct page, list);
+}
+
+int shmobj_create(struct shmid_ds *ds, struct memobj **objp)
+{
+	struct shmobj *obj = NULL;
+	int error;
+
+	dkprintf("shmobj_create(%p %#lx,%p)\n", ds, ds->shm_segsz, objp);
+	obj = kmalloc(sizeof(*obj), IHK_MC_AP_NOWAIT);
+	if (!obj) {
+		error = -ENOMEM;
+		ekprintf("shmobj_create(%p %#lx,%p):kmalloc failed. %d\n",
+				ds, ds->shm_segsz, objp, error);
+		goto out;
+	}
+
+	memset(obj, 0, sizeof(*obj));
+	obj->memobj.ops = &shmobj_ops;
+	obj->ref = 1;
+	obj->ds = *ds;
+	page_list_init(obj);
+	ihk_mc_spinlock_init(&obj->memobj.lock);
+
+	error = 0;
+	*objp = to_memobj(obj);
+	obj = NULL;
+
+out:
+	if (obj) {
+		kfree(obj);
+	}
+	dkprintf("shmobj_create(%p %#lx,%p):%d %p\n",
+			ds, ds->shm_segsz, objp, error, *objp);
+	return error;
+}
+
+static void shmobj_release(struct memobj *memobj)
+{
+	struct shmobj *obj = to_shmobj(memobj);
+	struct shmobj *freeobj = NULL;
+
+	dkprintf("shmobj_release(%p)\n", memobj);
+	memobj_lock(&obj->memobj);
+	--obj->ref;
+	if (obj->ref <= 0) {
+		if (obj->ref < 0) {
+			fkprintf("shmobj_release(%p):ref %ld\n",
+					memobj, obj->ref);
+			panic("shmobj_release:freeing free shmobj");
+		}
+		freeobj = obj;
+	}
+	memobj_unlock(&obj->memobj);
+
+	if (freeobj) {
+		/* zap page_list */
+		for (;;) {
+			struct page *page;
+			int count;
+
+			page = page_list_first(obj);
+			if (!page) {
+				break;
+			}
+			page_list_remove(obj, page);
+
+			dkprintf("shmobj_release(%p):"
+					"release page. %p %#lx %d %d",
+					memobj, page, page_to_phys(page),
+					page->mode, page->count);
+			count = ihk_atomic_sub_return(1, &page->count);
+			if (!((page->mode == PM_MAPPED) && (count == 0))) {
+				fkprintf("shmobj_release(%p): "
+						"page %p phys %#lx mode %#x"
+						" count %d off %#lx\n",
+						memobj, page,
+						page_to_phys(page),
+						page->mode, count,
+						page->offset);
+				panic("shmobj_release");
+			}
+
+			/* XXX:NYI: large pages */
+			page->mode = PM_NONE;
+			free_pages(phys_to_virt(page_to_phys(page)), 1);
+		}
+		dkprintf("shmobj_release(%p):free shmobj", memobj);
+		kfree(freeobj);
+	}
+	dkprintf("shmobj_release(%p):\n", memobj);
+	return;
+}
+
+static void shmobj_ref(struct memobj *memobj)
+{
+	struct shmobj *obj = to_shmobj(memobj);
+	long newref;
+
+	dkprintf("shmobj_ref(%p)\n", memobj);
+	memobj_lock(&obj->memobj);
+	newref = ++obj->ref;
+	memobj_unlock(&obj->memobj);
+	dkprintf("shmobj_ref(%p): newref %ld\n", memobj, newref);
+	return;
+}
+
+static int shmobj_get_page(struct memobj *memobj, off_t off, int p2align,
+		uintptr_t *physp)
+{
+	struct shmobj *obj = to_shmobj(memobj);
+	int error;
+	struct page *page;
+	int npages;
+	void *virt = NULL;
+	uintptr_t phys = -1;
+
+	dkprintf("shmobj_get_page(%p,%#lx,%d,%p)\n",
+			memobj, off, p2align, physp);
+	memobj_lock(&obj->memobj);
+	if (off & ~PAGE_MASK) {
+		error = -EINVAL;
+		ekprintf("shmobj_get_page(%p,%#lx,%d,%p):invalid argument. %d\n",
+				memobj, off, p2align, physp, error);
+		goto out;
+	}
+	if (p2align != PAGE_P2ALIGN) {		/* XXX:NYI:large pages */
+		error = -ENOMEM;
+		ekprintf("shmobj_get_page(%p,%#lx,%d,%p):large page. %d\n",
+				memobj, off, p2align, physp, error);
+		goto out;
+	}
+	if (obj->ds.shm_segsz <= off) {
+		error = -ERANGE;
+		ekprintf("shmobj_get_page(%p,%#lx,%d,%p):beyond the end. %d\n",
+				memobj, off, p2align, physp, error);
+		goto out;
+	}
+	if ((obj->ds.shm_segsz - off) < (PAGE_SIZE << p2align)) {
+		error = -ENOSPC;
+		ekprintf("shmobj_get_page(%p,%#lx,%d,%p):too large. %d\n",
+				memobj, off, p2align, physp, error);
+		goto out;
+	}
+
+	page = page_list_lookup(obj, off);
+	if (!page) {
+		npages = 1 << p2align;
+		virt = ihk_mc_alloc_pages(npages, IHK_MC_AP_NOWAIT);
+		if (!virt) {
+			error = -ENOMEM;
+			ekprintf("shmobj_get_page(%p,%#lx,%d,%p):"
+					"alloc failed. %d\n",
+					memobj, off, p2align, physp, error);
+			goto out;
+		}
+		phys = virt_to_phys(virt);
+		page = phys_to_page(phys);
+		if (page->mode != PM_NONE) {
+			fkprintf("shmobj_get_page(%p,%#lx,%d,%p):"
+					"page %p %#lx %d %d %#lx\n",
+					memobj, off, p2align, physp,
+					page, page_to_phys(page), page->mode,
+					page->count, page->offset);
+			panic("shmobj_get_page()");
+		}
+		memset(virt, 0, npages*PAGE_SIZE);
+		page->mode = PM_MAPPED;
+		page->offset = off;
+		ihk_atomic_set(&page->count, 1);
+		page_list_insert(obj, page);
+		virt = NULL;
+		dkprintf("shmobj_get_page(%p,%#lx,%d,%p):alloc page. %p %#lx\n",
+				memobj, off, p2align, physp, page, phys);
+	}
+
+	ihk_atomic_inc(&page->count);
+
+	error = 0;
+	*physp = page_to_phys(page);
+
+out:
+	memobj_unlock(&obj->memobj);
+	if (virt) {
+		ihk_mc_free_pages(virt, npages);
+	}
+	dkprintf("shmobj_get_page(%p,%#lx,%d,%p):%d\n",
+			memobj, off, p2align, physp, error);
+	return error;
+}

kernel/syscall.c

@@ -45,6 +45,7 @@
 #include <mman.h>
 #include <kmalloc.h>
 #include <memobj.h>
+#include <shm.h>
 
 /* Headers taken from kitten LWK */
 #include <lwk/stddef.h>
@@ -630,12 +631,13 @@ SYSCALL_DECLARE(mmap)
 	const int prot = ihk_mc_syscall_arg2(ctx);
 	const int flags = ihk_mc_syscall_arg3(ctx);
 	const int fd = ihk_mc_syscall_arg4(ctx);
-	const off_t off = ihk_mc_syscall_arg5(ctx);
+	const off_t off0 = ihk_mc_syscall_arg5(ctx);
 
 	struct process *proc = cpu_local_var(current);
 	struct vm_regions *region = &proc->vm->region;
 	intptr_t addr;
 	size_t len;
+	off_t off;
 	int error;
 	intptr_t npages;
 	int p2align;
@@ -646,10 +648,11 @@ SYSCALL_DECLARE(mmap)
 	int maxprot;
 	int denied;
 	int ro_vma_mapped = 0;
+	struct shmid_ds ads;
 
 	dkprintf("[%d]sys_mmap(%lx,%lx,%x,%x,%d,%lx)\n",
 			ihk_mc_get_processor_id(),
-			addr0, len0, prot, flags, fd, off);
+			addr0, len0, prot, flags, fd, off0);
 
 	/* check constants for flags */
 	if (1) {
@@ -681,9 +684,9 @@ SYSCALL_DECLARE(mmap)
 			|| ((region->user_end - len) < addr)
 			|| !(flags & (MAP_SHARED | MAP_PRIVATE))
 			|| ((flags & MAP_SHARED) && (flags & MAP_PRIVATE))
-			|| (off & (PAGE_SIZE - 1))) {
+			|| (off0 & (PAGE_SIZE - 1))) {
 		ekprintf("sys_mmap(%lx,%lx,%x,%x,%x,%lx):EINVAL\n",
-				addr0, len0, prot, flags, fd, off);
+				addr0, len0, prot, flags, fd, off0);
 		error = -EINVAL;
 		goto out2;
 	}
@@ -692,7 +695,7 @@ SYSCALL_DECLARE(mmap)
 	if ((flags & error_flags)
 			|| (flags & ~(supported_flags | ignored_flags))) {
 		ekprintf("sys_mmap(%lx,%lx,%x,%x,%x,%lx):unknown flags %x\n",
-				addr0, len0, prot, flags, fd, off,
+				addr0, len0, prot, flags, fd, off0,
 				(flags & ~(supported_flags | ignored_flags)));
 		error = -EINVAL;
 		goto out2;
@@ -754,8 +757,10 @@ SYSCALL_DECLARE(mmap)
 	}
 
 	phys = 0;
+	off = 0;
 	maxprot = PROT_READ | PROT_WRITE | PROT_EXEC;
 	if (!(flags & MAP_ANONYMOUS)) {
+		off = off0;
 		error = fileobj_create(fd, &memobj, &maxprot);
 		if (error) {
 			ekprintf("sys_mmap:fileobj_create failed. %d\n", error);
@@ -781,6 +786,15 @@ SYSCALL_DECLARE(mmap)
 		}
 		phys = virt_to_phys(p);
 	}
+	else if (flags & MAP_SHARED) {
+		memset(&ads, 0, sizeof(ads));
+		ads.shm_segsz = len;
+		error = shmobj_create(&ads, &memobj);
+		if (error) {
+			ekprintf("sys_mmap:shmobj_create failed. %d\n", error);
+			goto out;
+		}
+	}
 
 	if ((flags & MAP_PRIVATE) && (maxprot & PROT_READ)) {
 		maxprot |= PROT_WRITE;
@@ -844,7 +858,7 @@ out2:
 	}
 	dkprintf("[%d]sys_mmap(%lx,%lx,%x,%x,%d,%lx): %ld %lx\n",
 			ihk_mc_get_processor_id(),
-			addr0, len0, prot, flags, fd, off, error, addr);
+			addr0, len0, prot, flags, fd, off0, error, addr);
 	return (!error)? addr: error;
 }