remote_page_fault is handled by the offloaded thread.

Change-Id: I9cfad509260cceada74abdf39ca6a1822561e0d9
Refs: #1474

kernel/host.c

@@ -658,6 +658,58 @@ void send_procfs_answer(struct ikc_scd_packet *packet, int err)
 	syscall_channel_send(resp_channel, &pckt);
 }
 
+static void do_remote_page_fault(struct ikc_scd_packet *packet, int err)
+{
+	struct ikc_scd_packet pckt;
+	struct ihk_ikc_channel_desc *resp_channel = cpu_local_var(ikc2linux);
+	struct thread *thread = cpu_local_var(current);
+	unsigned long t_s = 0;
+
+	if (err) {
+		goto out_remote_pf;
+	}
+
+#ifdef PROFILE_ENABLE
+	/* We cannot use thread->profile_start_ts here because the
+	 * caller may be utilizing it already
+	 */
+
+	if (thread->profile) {
+		t_s = rdtsc();
+	}
+#endif // PROFILE_ENABLE
+
+	dkprintf("remote page fault,pid=%d,va=%lx,reason=%x\n",
+		 thread->proc->pid, packet->fault_address,
+		 packet->fault_reason|PF_POPULATE);
+	preempt_disable();
+	pckt.err = page_fault_process_vm(thread->vm,
+				(void *)packet->fault_address,
+				packet->fault_reason|PF_POPULATE);
+	preempt_enable();
+
+#ifdef PROFILE_ENABLE
+	if (thread->profile) {
+		profile_event_add(PROFILE_remote_page_fault,
+				(rdtsc() - t_s));
+	}
+#endif // PROFILE_ENABLE
+
+out_remote_pf:
+	pckt.err = err;
+	pckt.msg = SCD_MSG_REMOTE_PAGE_FAULT_ANSWER;
+	pckt.ref = packet->ref;
+	pckt.arg = packet->arg;
+	pckt.reply = packet->reply;
+	pckt.pid = packet->pid;
+	syscall_channel_send(resp_channel, &pckt);
+}
+
+static void remote_page_fault(void *arg)
+{
+	do_remote_page_fault(arg, 0);
+}
+
 static int syscall_packet_handler(struct ihk_ikc_channel_desc *c,
                                   void *__packet, void *ihk_os)
 {
@@ -680,7 +732,6 @@ static int syscall_packet_handler(struct ihk_ikc_channel_desc *c,
 	int ret = 0;
 	struct perf_ctrl_desc *pcd;
 	unsigned int mode = 0;
-	unsigned long t_s = 0;
 
 	switch (packet->msg) {
 	case SCD_MSG_INIT_CHANNEL_ACKED:
@@ -747,42 +798,22 @@ static int syscall_packet_handler(struct ihk_ikc_channel_desc *c,
 		if (!thread) {
 			kprintf("%s: WARNING: no thread for remote pf %d\n",
 				__func__, packet->fault_tid);
-			pckt.err = ret = -EINVAL;
+			do_remote_page_fault(packet, -EINVAL);
-			goto out_remote_pf;
+			break;
 		}
-#ifdef PROFILE_ENABLE
-		/* We cannot use thread->profile_start_ts here because the
-		 * caller may be utilizing it already */
 
-		if (thread->profile) {
+		if (thread == cpu_local_var(current)) {
-			t_s = rdtsc();
+			do_remote_page_fault(packet, 0);
 		}
-#endif // PROFILE_ENABLE
+		else {
-
+			thread->rpf_arg = kmalloc(sizeof(struct ikc_scd_packet),
-		dkprintf("remote page fault,pid=%d,va=%lx,reason=%x\n",
+						  IHK_MC_AP_NOWAIT);
-			 thread->proc->pid, packet->fault_address,
+			memcpy(thread->rpf_arg, packet,
-			 packet->fault_reason|PF_POPULATE);
+			       sizeof(struct ikc_scd_packet));
-		preempt_disable();
+			thread->rpf_backlog = remote_page_fault;
-		pckt.err = page_fault_process_vm(thread->vm,
+			sched_wakeup_thread(thread, PS_INTERRUPTIBLE);
-					(void *)packet->fault_address,
-					packet->fault_reason|PF_POPULATE);
-		preempt_enable();
-
-#ifdef PROFILE_ENABLE
-		if (thread->profile) {
-			profile_event_add(PROFILE_remote_page_fault,
-					(rdtsc() - t_s));
 		}
-#endif // PROFILE_ENABLE
 		thread_unlock(thread);
-
-out_remote_pf:
-		pckt.msg = SCD_MSG_REMOTE_PAGE_FAULT_ANSWER;
-		pckt.ref = packet->ref;
-		pckt.arg = packet->arg;
-		pckt.reply = packet->reply;
-		pckt.pid = packet->pid;
-		syscall_channel_send(resp_channel, &pckt);
 		break;
 
 	case SCD_MSG_SEND_SIGNAL:

kernel/include/process.h

@@ -762,6 +762,9 @@ struct thread {
 	struct waitq coredump_wq;
 	int coredump_status;
 
+	void (*rpf_backlog)(void *arg);
+	void *rpf_arg;
+
 #ifdef ENABLE_TOFU
 	/* Path of file being opened */
 	char *fd_path_in_open;

kernel/syscall.c

@@ -246,6 +246,16 @@ long do_syscall(struct syscall_request *req, int cpu)
 			unsigned long flags;
 			DECLARE_WAITQ_ENTRY(scd_wq_entry, cpu_local_var(current));
 
+			if (thread->rpf_backlog) {
+				void (*func)(void *) = thread->rpf_backlog;
+				void *arg = thread->rpf_arg;
+
+				thread->rpf_backlog = NULL;
+				thread->rpf_arg = NULL;
+				func(arg);
+				kfree(arg);
+			}
+
 			check_sig_pending();
 			cpu_pause();
 

test/issues/1474/C1474.sh

@@ -0,0 +1,21 @@
+ELTP=1
+USEOSTEST=0
+
+. ../../common.sh
+
+################################################################################
+uname -m
+dd if=/dev/zero of=testfile bs=4096 count=1
+$MCEXEC ./C1474T01
+rm -f outfile
+$MCEXEC ./C1474T02
+rm -f outfile
+$MCEXEC ./C1474T03
+rm -f outfile
+$MCEXEC --enable-uti ./C1474T04
+rm -f outfile
+$MCEXEC --enable-uti ./C1474T05
+rm -f outfile
+$MCEXEC --enable-uti ./C1474T06
+rm -f outfile
+rm -f testfile

test/issues/1474/C1474T01.c

@@ -0,0 +1,102 @@
+#define _GNU_SOURCE
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/mman.h>
+#include <sys/wait.h>
+#include <sched.h>
+#include <fcntl.h>
+#include <errno.h>
+
+int
+main(int argc, char **argv)
+{
+	pid_t pid1;
+	pid_t pid2;
+	cpu_set_t cpu_set0;
+	cpu_set_t cpu_set;
+	int st;
+	int st1;
+	int i;
+
+	fprintf(stderr, "*** C1474T01 START ***\n");
+	if (sched_getaffinity(getpid(), sizeof(cpu_set_t), &cpu_set0) == -1) {
+		perror("sched_getaffinity");
+		goto ng;
+	}
+
+	CPU_ZERO(&cpu_set);
+	for (i = 0; i < sizeof(cpu_set) * 8; i++) {
+		if (CPU_ISSET(i, &cpu_set0)) {
+			break;
+		}
+	}
+	for (i++; i < sizeof(cpu_set) * 8; i++) {
+		if (CPU_ISSET(i, &cpu_set0)) {
+			CPU_SET(i, &cpu_set);
+			break;
+		}
+	}
+
+	if ((pid1 = fork()) == 0) {
+		if (sched_setaffinity(getpid(), sizeof(cpu_set_t), &cpu_set)) {
+			perror("sched_setaffinity");
+			goto ng;
+		}
+		for (;;)
+			;
+	}
+	if ((pid2 = fork()) == 0) {
+		int f;
+		char *p;
+		int r;
+
+		if (sched_setaffinity(getpid(), sizeof(cpu_set_t), &cpu_set)) {
+			perror("sched_setaffinity");
+			exit(1);
+		}
+		if ((f = open("testfile", O_RDONLY)) == -1) {
+			perror("open(testfile)");
+			exit(1);
+		}
+		p = mmap(NULL, 4096, PROT_READ, MAP_SHARED, f, 0);
+		if (p == (void *)-1) {
+			perror("mmap");
+			exit(1);
+		}
+		close(f);
+		f = open("outfile", O_WRONLY|O_CREAT|O_TRUNC, 0600);
+		if (f == -1) {
+			perror("open(outfile)");
+			exit(1);
+		}
+		if ((r = write(f, p, 1024)) == -1) {
+			perror("write");
+			exit(1);
+		}
+		fprintf(stderr, "remote page fault OK\n");
+		close(f);
+		if (r != 1024) {
+			fprintf(stderr, "BAD out size: %d\n", r);
+			exit(1);
+		}
+		exit(0);
+	}
+	if (waitpid(pid2, &st, 0) == -1) {
+		perror("waitpid");
+		goto ng;
+	}
+	kill(pid1, SIGKILL);
+	if (waitpid(pid1, &st1, 0) == -1) {
+		perror("waitpid");
+		goto ng;
+	}
+	if (WEXITSTATUS(st) == 0) {
+		fprintf(stderr, "*** C1474T01 OK ***\n");
+		exit(0);
+	}
+ng:
+	fprintf(stderr, "*** C1474T01 NG ***\n");
+	exit(1);
+}

test/issues/1474/C1474T02.c

@@ -0,0 +1,108 @@
+#define _GNU_SOURCE
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/mman.h>
+#include <sys/wait.h>
+#include <sched.h>
+#include <fcntl.h>
+#include <errno.h>
+
+int
+main(int argc, char **argv)
+{
+	pid_t pid1;
+	pid_t pid2;
+	cpu_set_t cpu_set0;
+	cpu_set_t cpu_set;
+	int st;
+	int st1;
+	int i;
+	int pfd[2];
+
+	fprintf(stderr, "*** C1474T02 START ***\n");
+	pipe(pfd);
+	if (sched_getaffinity(getpid(), sizeof(cpu_set_t), &cpu_set0) == -1) {
+		perror("sched_getaffinity");
+		goto ng;
+	}
+
+	CPU_ZERO(&cpu_set);
+	for (i = 0; i < sizeof(cpu_set) * 8; i++) {
+		if (CPU_ISSET(i, &cpu_set0)) {
+			break;
+		}
+	}
+	for (i++; i < sizeof(cpu_set) * 8; i++) {
+		if (CPU_ISSET(i, &cpu_set0)) {
+			CPU_SET(i, &cpu_set);
+			break;
+		}
+	}
+
+	if ((pid1 = fork()) == 0) {
+		close(pfd[1]);
+		if (sched_setaffinity(getpid(), sizeof(cpu_set_t), &cpu_set)) {
+			perror("sched_setaffinity");
+			exit(1);
+		}
+		read(pfd[0], &i, 1);
+		exit(1);
+	}
+	if ((pid2 = fork()) == 0) {
+		int f;
+		char *p;
+		int r;
+
+		close(pfd[0]);
+		close(pfd[1]);
+		if (sched_setaffinity(getpid(), sizeof(cpu_set_t), &cpu_set)) {
+			perror("sched_setaffinity");
+			exit(1);
+		}
+		if ((f = open("testfile", O_RDONLY)) == -1) {
+			perror("open(testfile)");
+			exit(1);
+		}
+		p = mmap(NULL, 4096, PROT_READ, MAP_SHARED, f, 0);
+		if (p == (void *)-1) {
+			perror("mmap");
+			exit(1);
+		}
+		close(f);
+		f = open("outfile", O_WRONLY|O_CREAT|O_TRUNC, 0600);
+		if (f == -1) {
+			perror("open(outfile)");
+			exit(1);
+		}
+		if ((r = write(f, p, 1024)) == -1) {
+			perror("write");
+			exit(1);
+		}
+		fprintf(stderr, "remote page fault OK\n");
+		close(f);
+		if (r != 1024) {
+			fprintf(stderr, "BAD out size: %d\n", r);
+			exit(1);
+		}
+		exit(0);
+	}
+	close(pfd[0]);
+	if (waitpid(pid2, &st, 0) == -1) {
+		perror("waitpid");
+		goto ng;
+	}
+	write(pfd[1], &st, 1);
+	if (waitpid(pid1, &st1, 0) == -1) {
+		perror("waitpid");
+		goto ng;
+	}
+	if (WEXITSTATUS(st) == 0) {
+		fprintf(stderr, "*** C1474T02 OK ***\n");
+		exit(0);
+	}
+ng:
+	fprintf(stderr, "*** C1474T02 NG ***\n");
+	exit(1);
+}

test/issues/1474/C1474T03.c

@@ -0,0 +1,95 @@
+#define _GNU_SOURCE
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/mman.h>
+#include <sys/wait.h>
+#include <sched.h>
+#include <fcntl.h>
+#include <errno.h>
+
+int
+main(int argc, char **argv)
+{
+	pid_t pid2;
+	cpu_set_t cpu_set0;
+	cpu_set_t cpu_set;
+	int st;
+	int i;
+	int j = 0;
+
+	fprintf(stderr, "*** C1474T03 START ***\n");
+	if (sched_getaffinity(getpid(), sizeof(cpu_set_t), &cpu_set0) == -1) {
+		perror("sched_getaffinity");
+		goto ng;
+	}
+
+	CPU_ZERO(&cpu_set);
+	for (i = 0; i < sizeof(cpu_set) * 8; i++) {
+		if (CPU_ISSET(i, &cpu_set0)) {
+			j = i;
+			break;
+		}
+	}
+	for (i++; i < sizeof(cpu_set) * 8; i++) {
+		if (CPU_ISSET(i, &cpu_set0)) {
+			CPU_SET(i, &cpu_set);
+			break;
+		}
+	}
+	if (sched_setaffinity(getpid(), sizeof(cpu_set_t), &cpu_set)) {
+		perror("sched_setaffinity");
+		goto ng;
+	}
+	CPU_ZERO(&cpu_set);
+	CPU_SET(j, &cpu_set);
+
+	if ((pid2 = fork()) == 0) {
+		int f;
+		char *p;
+		int r;
+
+		if (sched_setaffinity(getpid(), sizeof(cpu_set_t), &cpu_set)) {
+			perror("sched_setaffinity");
+			exit(1);
+		}
+		if ((f = open("testfile", O_RDONLY)) == -1) {
+			perror("open(testfile)");
+			exit(1);
+		}
+		p = mmap(NULL, 4096, PROT_READ, MAP_SHARED, f, 0);
+		if (p == (void *)-1) {
+			perror("mmap");
+			exit(1);
+		}
+		close(f);
+		f = open("outfile", O_WRONLY|O_CREAT|O_TRUNC, 0600);
+		if (f == -1) {
+			perror("open(outfile)");
+			exit(1);
+		}
+		if ((r = write(f, p, 1024)) == -1) {
+			perror("write");
+			exit(1);
+		}
+		fprintf(stderr, "remote page fault OK\n");
+		close(f);
+		if (r != 1024) {
+			fprintf(stderr, "BAD out size: %d\n", r);
+			exit(1);
+		}
+		exit(0);
+	}
+	if (waitpid(pid2, &st, 0) == -1) {
+		perror("waitpid");
+		goto ng;
+	}
+	if (WEXITSTATUS(st) == 0) {
+		fprintf(stderr, "*** C1474T03 OK ***\n");
+		exit(0);
+	}
+ng:
+	fprintf(stderr, "*** C1474T03 NG ***\n");
+	exit(1);
+}

test/issues/1474/C1474T04.c

@@ -0,0 +1,151 @@
+#define _GNU_SOURCE
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <sys/syscall.h>
+#include <sys/types.h>
+#include <sys/mman.h>
+#include <sys/wait.h>
+#include <sched.h>
+#include <fcntl.h>
+#include <pthread.h>
+#include <errno.h>
+
+int ok;
+
+void *
+util_thread(void *arg)
+{
+	char *p = arg;
+	int f;
+	int r;
+
+	r = syscall(732);
+	if (r == -1) {
+		fprintf(stderr, "thread is running on Linux OK\n");
+	}
+	else {
+		fprintf(stderr, "thread is running on McKernel NG(%d)\n", r);
+		ok = -1;
+		return NULL;
+	}
+	while (ok == 0)
+		;
+
+	f = open("outfile", O_WRONLY|O_CREAT|O_TRUNC, 0600);
+	if (f == -1) {
+		perror("open(outfile)");
+		ok = -1;
+		return NULL;
+	}
+	if ((r = write(f, p, 1024)) == -1) {
+		perror("write");
+		ok = -1;
+		return NULL;
+	}
+	fprintf(stderr, "remote page fault OK\n");
+	close(f);
+	if (r != 1024) {
+		fprintf(stderr, "BAD out size: %d\n", r);
+		ok = -1;
+		return NULL;
+	}
+	ok = 1;
+	return NULL;
+}
+
+int o2;
+
+void *
+wait_thread(void *arg)
+{
+	while (o2 == 0)
+		;
+	return NULL;
+}
+
+
+int
+main(int argc, char **argv)
+{
+	int f;
+	int r;
+	char *p;
+	pthread_t thr;
+	pthread_t wthr;
+	cpu_set_t cpu_set0;
+	cpu_set_t cpu_set1;
+	cpu_set_t cpu_set;
+	int i;
+
+	fprintf(stderr, "*** C1474T04 START ***\n");
+	if ((f = open("testfile", O_RDONLY)) == -1) {
+		perror("open(testfile)");
+		goto ng;
+	}
+	p = mmap(NULL, 4096, PROT_READ, MAP_SHARED, f, 0);
+	if (p == (void *)-1) {
+		perror("mmap");
+		goto ng;
+	}
+	close(f);
+
+	if (sched_getaffinity(getpid(), sizeof(cpu_set_t), &cpu_set0) == -1) {
+		perror("sched_getaffinity");
+		goto ng;
+	}
+	CPU_ZERO(&cpu_set);
+	CPU_ZERO(&cpu_set1);
+	for (i = 0; i < sizeof(cpu_set) * 8; i++) {
+		if (CPU_ISSET(i, &cpu_set0)) {
+			CPU_SET(i, &cpu_set1);
+			break;
+		}
+	}
+	for (i++; i < sizeof(cpu_set) * 8; i++) {
+		if (CPU_ISSET(i, &cpu_set0)) {
+			CPU_SET(i, &cpu_set);
+			break;
+		}
+	}
+	if (sched_setaffinity(getpid(), sizeof(cpu_set_t), &cpu_set)) {
+		perror("sched_setaffinity");
+		goto ng;
+	}
+
+	r = pthread_create(&wthr, NULL, wait_thread, NULL);
+	if (r) {
+		fprintf(stderr, "pthread_create: %d\n", r);
+		goto ng;
+	}
+
+	r = syscall(731, 1, NULL);
+	if (r) {
+		fprintf(stderr, "util_indicate_clone r=%d, err=%d\n", r, errno);
+		goto ng;
+	}
+	r = pthread_create(&thr, NULL, util_thread, p);
+	if (r) {
+		fprintf(stderr, "pthread_create: %d\n", r);
+		goto ng;
+	}
+
+	if (sched_setaffinity(getpid(), sizeof(cpu_set_t), &cpu_set1)) {
+		perror("sched_setaffinity");
+		goto ng;
+	}
+
+	ok = 2;
+	while (ok == 2)
+		;
+	pthread_join(thr, NULL);
+	o2 = 1;
+	pthread_join(wthr, NULL);
+	if (ok == 1) {
+		fprintf(stderr, "*** C1474T04 OK ***\n");
+		exit(0);
+	}
+ng:
+	fprintf(stderr, "*** C1474T04 NG ***\n");
+	exit(1);
+}

test/issues/1474/C1474T05.c

@@ -0,0 +1,147 @@
+#define _GNU_SOURCE
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <sys/syscall.h>
+#include <sys/types.h>
+#include <sys/mman.h>
+#include <sys/wait.h>
+#include <sched.h>
+#include <fcntl.h>
+#include <pthread.h>
+#include <errno.h>
+
+int ok;
+
+void *
+util_thread(void *arg)
+{
+	char *p = arg;
+	int f;
+	int r;
+
+	r = syscall(732);
+	if (r == -1) {
+		fprintf(stderr, "thread is running on Linux OK\n");
+	}
+	else {
+		fprintf(stderr, "thread is running on McKernel NG(%d)\n", r);
+		ok = -1;
+		return NULL;
+	}
+	while (ok == 0)
+		;
+
+	f = open("outfile", O_WRONLY|O_CREAT|O_TRUNC, 0600);
+	if (f == -1) {
+		perror("open(outfile)");
+		ok = -1;
+		return NULL;
+	}
+	if ((r = write(f, p, 1024)) == -1) {
+		perror("write");
+		ok = -1;
+		return NULL;
+	}
+	fprintf(stderr, "remote page fault OK\n");
+	close(f);
+	if (r != 1024) {
+		fprintf(stderr, "BAD out size: %d\n", r);
+		ok = -1;
+		return NULL;
+	}
+	ok = 1;
+	return NULL;
+}
+
+void *
+wait_thread(void *arg)
+{
+	sleep(5);
+	return NULL;
+}
+
+
+int
+main(int argc, char **argv)
+{
+	int f;
+	int r;
+	char *p;
+	pthread_t thr;
+	pthread_t wthr;
+	cpu_set_t cpu_set0;
+	cpu_set_t cpu_set1;
+	cpu_set_t cpu_set;
+	int i;
+
+	fprintf(stderr, "*** C1474T05 START ***\n");
+	if ((f = open("testfile", O_RDONLY)) == -1) {
+		perror("open(testfile)");
+		goto ng;
+	}
+	p = mmap(NULL, 4096, PROT_READ, MAP_SHARED, f, 0);
+	if (p == (void *)-1) {
+		perror("mmap");
+		goto ng;
+	}
+	close(f);
+
+	if (sched_getaffinity(getpid(), sizeof(cpu_set_t), &cpu_set0) == -1) {
+		perror("sched_getaffinity");
+		goto ng;
+	}
+	CPU_ZERO(&cpu_set);
+	CPU_ZERO(&cpu_set1);
+	for (i = 0; i < sizeof(cpu_set) * 8; i++) {
+		if (CPU_ISSET(i, &cpu_set0)) {
+			CPU_SET(i, &cpu_set1);
+			break;
+		}
+	}
+	for (i++; i < sizeof(cpu_set) * 8; i++) {
+		if (CPU_ISSET(i, &cpu_set0)) {
+			CPU_SET(i, &cpu_set);
+			break;
+		}
+	}
+	if (sched_setaffinity(getpid(), sizeof(cpu_set_t), &cpu_set)) {
+		perror("sched_setaffinity");
+		goto ng;
+	}
+
+	r = pthread_create(&wthr, NULL, wait_thread, NULL);
+	if (r) {
+		fprintf(stderr, "pthread_create: %d\n", r);
+		goto ng;
+	}
+
+	r = syscall(731, 1, NULL);
+	if (r) {
+		fprintf(stderr, "util_indicate_clone r=%d, err=%d\n", r, errno);
+		goto ng;
+	}
+	r = pthread_create(&thr, NULL, util_thread, p);
+	if (r) {
+		fprintf(stderr, "pthread_create: %d\n", r);
+		goto ng;
+	}
+
+	if (sched_setaffinity(getpid(), sizeof(cpu_set_t), &cpu_set1)) {
+		perror("sched_setaffinity");
+		goto ng;
+	}
+
+	ok = 2;
+	while (ok == 2)
+		;
+	pthread_join(thr, NULL);
+	pthread_join(wthr, NULL);
+	if (ok == 1) {
+		fprintf(stderr, "*** C1474T05 OK ***\n");
+		exit(0);
+	}
+ng:
+	fprintf(stderr, "*** C1474T05 NG ***\n");
+	exit(1);
+}

test/issues/1474/C1474T06.c

@@ -0,0 +1,99 @@
+#define _GNU_SOURCE
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <sys/syscall.h>
+#include <sys/types.h>
+#include <sys/mman.h>
+#include <sys/wait.h>
+#include <sched.h>
+#include <fcntl.h>
+#include <pthread.h>
+#include <errno.h>
+
+int ok;
+
+void *
+util_thread(void *arg)
+{
+	char *p = arg;
+	int f;
+	int r;
+
+	r = syscall(732);
+	if (r == -1) {
+		fprintf(stderr, "thread is running on Linux OK\n");
+	}
+	else {
+		fprintf(stderr, "thread is running on McKernel NG(%d)\n", r);
+		ok = -1;
+		return NULL;
+	}
+	while (ok == 0)
+		;
+
+	f = open("outfile", O_WRONLY|O_CREAT|O_TRUNC, 0600);
+	if (f == -1) {
+		perror("open(outfile)");
+		ok = -1;
+		return NULL;
+	}
+	if ((r = write(f, p, 1024)) == -1) {
+		perror("write");
+		ok = -1;
+		return NULL;
+	}
+	fprintf(stderr, "remote page fault OK\n");
+	close(f);
+	if (r != 1024) {
+		fprintf(stderr, "BAD out size: %d\n", r);
+		ok = -1;
+		return NULL;
+	}
+	ok = 1;
+	return NULL;
+}
+
+int
+main(int argc, char **argv)
+{
+	int f;
+	int r;
+	char *p;
+	pthread_t thr;
+
+	fprintf(stderr, "*** C1474T06 START ***\n");
+	if ((f = open("testfile", O_RDONLY)) == -1) {
+		perror("open(testfile)");
+		goto ng;
+	}
+	p = mmap(NULL, 4096, PROT_READ, MAP_SHARED, f, 0);
+	if (p == (void *)-1) {
+		perror("mmap");
+		goto ng;
+	}
+	close(f);
+
+	r = syscall(731, 1, NULL);
+	if (r) {
+		fprintf(stderr, "util_indicate_clone r=%d, err=%d\n", r, errno);
+		goto ng;
+	}
+	r = pthread_create(&thr, NULL, util_thread, p);
+	if (r) {
+		fprintf(stderr, "pthread_create: %d\n", r);
+		goto ng;
+	}
+
+	ok = 2;
+	while (ok == 2)
+		;
+	pthread_join(thr, NULL);
+	if (ok == 1) {
+		fprintf(stderr, "*** C1474T06 OK ***\n");
+		exit(0);
+	}
+ng:
+	fprintf(stderr, "*** C1474T06 NG ***\n");
+	exit(1);
+}

test/issues/1474/C1474_arm64.txt

@@ -0,0 +1,40 @@
+Script started on 2021-03-01 06:19:04+00:00
+bash-4.4$ make test
+gcc -g -Wall -o C1474T01 C1474T01.c
+gcc -g -Wall -o C1474T02 C1474T02.c
+gcc -g -Wall -o C1474T03 C1474T03.c
+gcc -g -Wall -O0 -pthread -o C1474T04 C1474T04.c
+gcc -g -Wall -O0 -pthread -o C1474T05 C1474T05.c
+gcc -g -Wall -O0 -pthread -o C1474T06 C1474T06.c
+sh ./C1474.sh
+mcstop+release.sh ... done
+mcreboot.sh -c 2-7 -m 2G@0 -O ... done
+aarch64
+1+0 records in
+1+0 records out
+4096 bytes (4.1 kB, 4.0 KiB) copied, 0.0002746 s, 14.9 MB/s
+*** C1474T01 START ***
+remote page fault OK
+*** C1474T01 OK ***
+*** C1474T02 START ***
+remote page fault OK
+*** C1474T02 OK ***
+*** C1474T03 START ***
+remote page fault OK
+*** C1474T03 OK ***
+*** C1474T04 START ***
+thread is running on Linux OK
+remote page fault OK
+*** C1474T04 OK ***
+*** C1474T05 START ***
+thread is running on Linux OK
+remote page fault OK
+*** C1474T05 OK ***
+*** C1474T06 START ***
+thread is running on Linux OK
+remote page fault OK
+*** C1474T06 OK ***
+bash-4.4$ exit
+exit
+
+Script done on 2021-03-01 06:19:25+00:00

test/issues/1474/C1474_x86_64.txt

@@ -0,0 +1,40 @@
+Script started on Mon Mar  1 15:20:29 2021
+bash-4.2$ make test
+gcc -g -Wall -o C1474T01 C1474T01.c
+gcc -g -Wall -o C1474T02 C1474T02.c
+gcc -g -Wall -o C1474T03 C1474T03.c
+gcc -g -Wall -O0 -pthread -o C1474T04 C1474T04.c
+gcc -g -Wall -O0 -pthread -o C1474T05 C1474T05.c
+gcc -g -Wall -O0 -pthread -o C1474T06 C1474T06.c
+sh ./C1474.sh
+mcstop+release.sh ... done
+mcreboot.sh -O -c 1-7 -m 2G@0 ... done
+x86_64
+1+0 records in
+1+0 records out
+4096 bytes (4.1 kB) copied, 0.000548447 s, 7.5 MB/s
+*** C1474T01 START ***
+remote page fault OK
+*** C1474T01 OK ***
+*** C1474T02 START ***
+remote page fault OK
+*** C1474T02 OK ***
+*** C1474T03 START ***
+remote page fault OK
+*** C1474T03 OK ***
+*** C1474T04 START ***
+thread is running on Linux OK
+remote page fault OK
+*** C1474T04 OK ***
+*** C1474T05 START ***
+thread is running on Linux OK
+remote page fault OK
+*** C1474T05 OK ***
+*** C1474T06 START ***
+thread is running on Linux OK
+remote page fault OK
+*** C1474T06 OK ***
+bash-4.2$ exit
+exit
+
+Script done on Mon Mar  1 15:20:46 2021

test/issues/1474/Makefile

@@ -0,0 +1,28 @@
+CC = gcc
+TARGET = C1474T01 C1474T02 C1474T03 C1474T04 C1474T05 C1474T06
+
+all:: $(TARGET)
+
+C1474T01: C1474T01.c
+	$(CC) -g -Wall -o $@ $^
+
+C1474T02: C1474T02.c
+	$(CC) -g -Wall -o $@ $^
+
+C1474T03: C1474T03.c
+	$(CC) -g -Wall -o $@ $^
+
+C1474T04: C1474T04.c
+	$(CC) -g -Wall -O0 -pthread -o $@ $^
+
+C1474T05: C1474T05.c
+	$(CC) -g -Wall -O0 -pthread -o $@ $^
+
+C1474T06: C1474T06.c
+	$(CC) -g -Wall -O0 -pthread -o $@ $^
+
+test:: all
+	sh ./C1474.sh
+
+clean::
+	rm -f $(TARGET) *.o

test/issues/1474/README

@@ -0,0 +1,50 @@
+【Issue#1474 動作確認】
+□ テスト内容
+ファイルマップ領域に対するリモートページフォルト発生時にMakernelのプロセス
+状態に無関係に処理が成功することを確認する。
+
+C1474T01 システムコールオフロード中にリモートページフォルトが発生したスレッドと
+	 同じCPUに別スレッドが割り当てられていて、そのスレッドがユーザ空間で
+	 処理中の場合に正常にリモートページフォルトが処理されることを確認する。
+	 このとき、当該CPUのカレントスレッドは別スレッドになっている。
+
+C1474T02 システムコールオフロード中にリモートページフォルトが発生したスレッドと
+	 同じCPUに別スレッドが割り当てられていて、別スレッドがシステムコール
+	 オフロードを処理中の場合に正常にリモートページフォルトが処理される
+	 ことを確認する。
+	 このとき、当該CPUのカレントスレッドはIDLEになっている。
+	 Issue 指摘事項がこの状態に該当する。
+
+C1474T03 システムコールオフロード中にリモートページフォルトが発生したスレッドが
+	 割り当てられているCPUに他のスレッドが割り当てられていない場合に
+	 正常にリモートページフォルトが処理されることを確認する。
+	 このとき、当該CPUのカレントスレッドはリモートページフォルト発生スレッド
+	 になっている。
+
+C1474T04 UTIスレッドでリモートページフォルトが発生したとき、McKernelの
+	 同じCPUに別スレッドが割り当てられていて、別スレッドがユーザ空間で
+	 処理中の場合に正常にリモートページフォルトが処理されることを確認する。
+	 このとき、当該CPUのカレントスレッドは別スレッドになっている。
+
+C1474T05 UTIスレッドでリモートページフォルトが発生したとき、McKernelの
+	 同じCPUに別スレッドが割り当てられていて、別スレッドがシステムコール
+	 オフロード処理中の場合に正常にリモートページフォルトが処理される
+	 ことを確認する。
+	 このとき、当該CPUのカレントスレッドはIDLEになっている。
+
+C1474T05 UTIスレッドでリモートページフォルトが発生したとき、McKernelのCPUに
+	 他のスレッドが割り当てられていない場合に正常にリモートページフォルトが
+	 処理されることを確認する。
+	 このとき、当該CPUのカレントスレッドはUTIスレッドになっている。
+
+
+□ 実行手順
+$ make test
+
+McKernelのインストール先や LTP の配置場所は、$HOME/.mck_test_config を
+参照する。.mck_test_config は、McKernel をビルドした際に生成される
+mck_test_config.sample ファイルを $HOME にコピーし、適宜編集すること。
+
+□ 実行結果
+C1474_x86_64.txt(x86_64実行結果)、C1474_arm64.txt(arm64実行結果)参照。
+全ての項目が PASS していることを確認。