From 8daffa939e413623ef612134529e4e794fa71137 Mon Sep 17 00:00:00 2001
From: Ken Sato <ken.sato.ty@hitachi-solutions.com>
Date: Thu, 9 Feb 2017 17:18:26 +0900
Subject: [PATCH] IKC: distribute IKC-interrupt to Linux cpus.

---
 arch/x86/kernel/include/ihk/ikc.h |  3 ++
 arch/x86/kernel/mikc.c            |  2 +
 arch/x86/kernel/syscall.c         |  2 +-
 executer/kernel/mcctrl/ikc.c      | 86 +++++++++++++++++-------------
 kernel/ap.c                       | 10 +++-
 kernel/host.c                     | 88 +++++++++++++++++++------------
 kernel/include/cls.h              |  6 +--
 kernel/include/init.h             |  5 +-
 kernel/init.c                     |  5 +-
 kernel/procfs.c                   |  4 +-
 kernel/syscall.c                  |  9 +---
 kernel/sysfs.c                    | 18 +++----
 lib/include/ihk/cpu.h             |  2 +
 13 files changed, 138 insertions(+), 102 deletions(-)

diff --git a/arch/x86/kernel/include/ihk/ikc.h b/arch/x86/kernel/include/ihk/ikc.h
index 0132ca6d..f6504f8d 100644
--- a/arch/x86/kernel/include/ihk/ikc.h
+++ b/arch/x86/kernel/include/ihk/ikc.h
@@ -15,6 +15,9 @@
 
 #include <ikc/ihk.h>
 
+#define IKC_PORT_IKC2MCKERNEL 501
+#define IKC_PORT_IKC2LINUX    503
+
 /* manycore side */
 int ihk_mc_ikc_init_first(struct ihk_ikc_channel_desc *,
                           ihk_ikc_ph_t handler);
diff --git a/arch/x86/kernel/mikc.c b/arch/x86/kernel/mikc.c
index 79ced05a..ef57e891 100644
--- a/arch/x86/kernel/mikc.c
+++ b/arch/x86/kernel/mikc.c
@@ -47,6 +47,8 @@ int ihk_mc_ikc_init_first_local(struct ihk_ikc_channel_desc *channel,
 	ihk_ikc_init_desc(channel, IKC_OS_HOST, 0, rq, wq,
 	                  ihk_ikc_master_channel_packet_handler, channel);
 	ihk_ikc_enable_channel(channel);
+/* Comment: McKernel側のmaster_channel 割り込み処理channel_list に追加 */
+	ihk_ikc_add_intr_channel(NULL, channel, 0);
 
 	/* Set boot parameter */
 	arch_set_mikc_queue(rq, wq);
diff --git a/arch/x86/kernel/syscall.c b/arch/x86/kernel/syscall.c
index 7f2059c7..e4a42647 100644
--- a/arch/x86/kernel/syscall.c
+++ b/arch/x86/kernel/syscall.c
@@ -1547,7 +1547,7 @@ static int vdso_get_vdso_info(void)
 {
 	int error;
 	struct ikc_scd_packet packet;
-	struct ihk_ikc_channel_desc *ch = cpu_local_var(syscall_channel);
+	struct ihk_ikc_channel_desc *ch = cpu_local_var(ikc2linux);
 
 	dkprintf("vdso_get_vdso_info()\n");
 	memset(&vdso, '\0', sizeof vdso);
diff --git a/executer/kernel/mcctrl/ikc.c b/executer/kernel/mcctrl/ikc.c
index f245c1e2..953f72a5 100644
--- a/executer/kernel/mcctrl/ikc.c
+++ b/executer/kernel/mcctrl/ikc.c
@@ -138,6 +138,14 @@ static int syscall_packet_handler(struct ihk_ikc_channel_desc *c,
 	return 0;
 }
 
+/* Comment: 受信を想定しないchannel用のパケットハンドラ */
+static int dummy_packet_handler(struct ihk_ikc_channel_desc *c,
+                                  void *__packet, void *__os)
+{
+	ihk_ikc_release_packet((struct ihk_ikc_free_packet *)__packet, c);
+	return 0;
+}
+
 int mcctrl_ikc_send(ihk_os_t os, int cpu, struct ikc_scd_packet *pisp)
 {
 	struct mcctrl_usrdata *usrdata = ihk_host_os_get_usrdata(os);
@@ -207,57 +215,58 @@ static void mcctrl_ikc_init(ihk_os_t os, int cpu, unsigned long rphys, struct ih
 	ihk_ikc_send(pmc->c, &packet, 0);
 }
 
-static int connect_handler(struct ihk_ikc_channel_info *param)
+/* Comment: ikc2linuxのaccept後のchannelの処理 */
+static int connect_handler_ikc2linux(struct ihk_ikc_channel_info *param)
 {
 	struct ihk_ikc_channel_desc *c;
-	int cpu;
+	int linux_cpu;
 	ihk_os_t os = param->channel->remote_os;
-	struct mcctrl_usrdata   *usrdata = ihk_host_os_get_usrdata(os);
 
 	c = param->channel;
-	cpu = c->send.queue->read_cpu;
+	linux_cpu = c->recv.queue->read_cpu;
 
-	if (cpu < 0 || cpu >= usrdata->num_channels) {
-		kprintf("Invalid connect source processor: %d\n", cpu);
+	param->packet_handler = syscall_packet_handler;
+
+/* Comment: 指定されたCPUの割り込み処理channel_list へ追加 */
+	ihk_ikc_add_intr_channel(os, param->channel, linux_cpu);
+
+	return 0;
+}
+/* Comment: ikc2mckernelのaccept後のchannelの処理 */
+static int connect_handler_ikc2mckernel(struct ihk_ikc_channel_info *param)
+{
+	struct ihk_ikc_channel_desc *c;
+	int mck_cpu;
+	ihk_os_t os = param->channel->remote_os;
+	struct mcctrl_usrdata  *usrdata = ihk_host_os_get_usrdata(os);
+
+	c = param->channel;
+	mck_cpu = c->send.queue->read_cpu;
+
+	if (mck_cpu < 0 || mck_cpu >= usrdata->num_channels) {
+		kprintf("Invalid connect source processor: %d\n", mck_cpu);
 		return 1;
 	}
-	param->packet_handler = syscall_packet_handler;
+	param->packet_handler = dummy_packet_handler;
 	
-	usrdata->channels[cpu].c = c;
-	dkprintf("syscall: MC CPU %d connected. c=%p\n", cpu, c);
+/* Comment: MCK_CPU毎の送信channelとして管理 (従来通り) */
+	usrdata->channels[mck_cpu].c = c;
 
 	return 0;
 }
 
-static int connect_handler2(struct ihk_ikc_channel_info *param)
-{
-	struct ihk_ikc_channel_desc *c;
-	int cpu;
-	ihk_os_t os = param->channel->remote_os;
-	struct mcctrl_usrdata   *usrdata = ihk_host_os_get_usrdata(os);
-
-	c = param->channel;
-	cpu = usrdata->num_channels - 1;
-
-	param->packet_handler = syscall_packet_handler;
-	
-	usrdata->channels[cpu].c = c;
-	dkprintf("syscall: MC CPU %d connected. c=%p\n", cpu, c);
-
-	return 0;
-}
-
-static struct ihk_ikc_listen_param listen_param = {
-	.port = 501,
-	.handler = connect_handler,
+/* Comment: listen_paramの設定 */
+static struct ihk_ikc_listen_param lp_ikc2linux = {
+	.port = 503,
+	.handler = connect_handler_ikc2linux,
 	.pkt_size = sizeof(struct ikc_scd_packet),
 	.queue_size = PAGE_SIZE * 4,
 	.magic = 0x1129,
 };
 
-static struct ihk_ikc_listen_param listen_param2 = {
-	.port = 502,
-	.handler = connect_handler2,
+static struct ihk_ikc_listen_param lp_ikc2mckernel = {
+	.port = 501,
+	.handler = connect_handler_ikc2mckernel,
 	.pkt_size = sizeof(struct ikc_scd_packet),
 	.queue_size = PAGE_SIZE * 4,
 	.magic = 0x1329,
@@ -283,7 +292,8 @@ int prepare_ikc_channels(ihk_os_t os)
 		return -EINVAL;
 	}
 
-	usrdata->num_channels = usrdata->cpu_info->n_cpus + 1;
+/* Comment: syscall_channel2 廃止に伴い、num_channelsも減らす */
+	usrdata->num_channels = usrdata->cpu_info->n_cpus;
 	usrdata->channels = kzalloc(sizeof(struct mcctrl_channel) *
 			usrdata->num_channels,
 			GFP_KERNEL);
@@ -295,10 +305,10 @@ int prepare_ikc_channels(ihk_os_t os)
 
 	usrdata->os = os;
 	ihk_host_os_set_usrdata(os, usrdata);
-	memcpy(&usrdata->listen_param, &listen_param, sizeof listen_param);
-	ihk_ikc_listen_port(os, &usrdata->listen_param);
-	memcpy(&usrdata->listen_param2, &listen_param2, sizeof listen_param2);
-	ihk_ikc_listen_port(os, &usrdata->listen_param2);
+
+	ihk_ikc_listen_port(os, &lp_ikc2linux);
+	ihk_ikc_listen_port(os, &lp_ikc2mckernel);
+
 	init_waitqueue_head(&usrdata->wq_procfs);
 	mutex_init(&usrdata->reserve_lock);
 
diff --git a/kernel/ap.c b/kernel/ap.c
index 799a05f8..10ada143 100644
--- a/kernel/ap.c
+++ b/kernel/ap.c
@@ -48,6 +48,7 @@ extern struct ihk_os_monitor *monitor;
 
 static void ap_wait(void)
 {
+	struct ihk_mc_cpu_info *cpu_info = ihk_mc_get_cpu_info();
 	init_tick();
 	while (ap_stop) {
 		barrier();
@@ -63,7 +64,14 @@ static void ap_wait(void)
 		mcs_lock_node_t mcs_node;
 
 		mcs_lock_lock_noirq(&ap_syscall_semaphore, &mcs_node);
-		init_host_syscall_channel();
+/* Comment: 自CPUのikc2mckernel と ikc2linuxの準備 */
+		init_host_ikc2mckernel();
+		int num = cpu_info->ikc_cpus[ihk_mc_get_processor_id()];
+		if (num == 1 ) {
+			num = 0;
+		}
+		init_host_ikc2linux(num);
+		//init_host_ikc2linux(cpu_info->ikc_cpus[ihk_mc_get_processor_id()] - 1);
 		mcs_lock_unlock_noirq(&ap_syscall_semaphore, &mcs_node);
 	}
 	
diff --git a/kernel/host.c b/kernel/host.c
index 29a34135..e85c93b4 100644
--- a/kernel/host.c
+++ b/kernel/host.c
@@ -41,6 +41,11 @@
 #define dkprintf(...) do { if (0) kprintf(__VA_ARGS__); } while (0)
 #endif
 
+/* Comment: McKernel側でのikc2linux(送信channel)の管理
+   nr_cpu_ids が利用できない？
+   配置場所の再考が必要？*/
+static struct ihk_ikc_channel_desc *ikc2linuxs[512];
+
 void check_mapping_for_proc(struct thread *thread, unsigned long addr)
 {
 	unsigned long __phys;
@@ -475,6 +480,9 @@ static int syscall_packet_handler(struct ihk_ikc_channel_desc *c,
 {
 	struct ikc_scd_packet *packet = __packet;
 	struct ikc_scd_packet pckt;
+/* Comment: 受信したchannel に返事をする方式から、
+   自CPUのikc2linux に返事をするよう変更 */
+	struct ihk_ikc_channel_desc *resp_channel = cpu_local_var(ikc2linux);
 	int rc;
 	struct mcs_rwlock_node_irqsave lock;
 	struct thread *thread;
@@ -510,7 +518,7 @@ static int syscall_packet_handler(struct ihk_ikc_channel_desc *c,
 		}
 		pckt.ref = packet->ref;
 		pckt.arg = packet->arg;
-		syscall_channel_send(c, &pckt);
+		syscall_channel_send(resp_channel, &pckt);
 
 		ret = 0;
 		break;
@@ -567,7 +575,7 @@ static int syscall_packet_handler(struct ihk_ikc_channel_desc *c,
 		pckt.err = 0;
 		pckt.ref = packet->ref;
 		pckt.arg = packet->arg;
-		syscall_channel_send(c, &pckt);
+		syscall_channel_send(resp_channel, &pckt);
 
 		rc = do_kill(NULL, info.pid, info.tid, info.sig, &info.info, 0);
 		kprintf("SCD_MSG_SEND_SIGNAL: do_kill(pid=%d, tid=%d, sig=%d)=%d\n", info.pid, info.tid, info.sig, rc);
@@ -669,54 +677,64 @@ static int syscall_packet_handler(struct ihk_ikc_channel_desc *c,
 	return ret;
 }
 
-void init_host_syscall_channel(void)
+/* Comment: パケット受信を想定しないchannel用のハンドラ */
+static int dummy_packet_handler(struct ihk_ikc_channel_desc *c,
+                                  void *__packet, void *__os)
 {
-	struct ihk_ikc_connect_param param;
-	struct ikc_scd_packet pckt;
-
-	param.port = 501;
-	param.pkt_size = sizeof(struct ikc_scd_packet);
-	param.queue_size = PAGE_SIZE * 4;
-	param.magic = 0x1129;
-	param.handler = syscall_packet_handler;
-
-	dkprintf("(syscall) Trying to connect host ...");
-	while (ihk_ikc_connect(NULL, &param) != 0) {
-		dkprintf(".");
-		ihk_mc_delay_us(1000 * 1000);
-	}
-	dkprintf("connected.\n");
-
-	get_this_cpu_local_var()->syscall_channel = param.channel;
-
-	pckt.msg = SCD_MSG_INIT_CHANNEL;
-	pckt.ref = ihk_mc_get_processor_id();
-	pckt.arg = virt_to_phys(&cpu_local_var(iip));
-	syscall_channel_send(param.channel, &pckt);
+	struct ikc_scd_packet *packet = __packet;
+	ihk_ikc_release_packet((struct ihk_ikc_free_packet *)packet, c);
+	return 0;
 }
 
-void init_host_syscall_channel2(void)
+/* Comment: ikc2linux の接続と自CPUへの設定を行う */
+void init_host_ikc2linux(int linux_cpu)
 {
 	struct ihk_ikc_connect_param param;
-	struct ikc_scd_packet pckt;
+	struct ihk_ikc_channel_desc *c = ikc2linuxs[linux_cpu];
 
-	param.port = 502;
+	if (!c) {
+/* Comment: 対象Linux_cpu 宛のikc2linuxが存在しなければ、接続 */
+		param.port = 503;
+		param.intr_cpu = linux_cpu;
+		param.pkt_size = sizeof(struct ikc_scd_packet);
+		param.queue_size = PAGE_SIZE * 4;
+		param.magic = 0x1129;
+		param.handler = dummy_packet_handler;
+
+		dkprintf("(ikc2linux) Trying to connect host ...");
+		while (ihk_ikc_connect(NULL, &param) != 0) {
+			dkprintf(".");
+			ihk_mc_delay_us(1000 * 1000);
+		}
+		dkprintf("connected.\n");
+
+		ikc2linuxs[linux_cpu] = param.channel;
+		c = param.channel;
+	}
+
+	get_this_cpu_local_var()->ikc2linux = c;
+}
+
+/* Comment: ikc2mckernelの接続と自CPUへの設定を行う */
+void init_host_ikc2mckernel(void)
+{
+	struct ihk_ikc_connect_param param;
+
+	param.port = 501;
+	param.intr_cpu = -1;
 	param.pkt_size = sizeof(struct ikc_scd_packet);
 	param.queue_size = PAGE_SIZE * 4;
 	param.magic = 0x1329;
 	param.handler = syscall_packet_handler;
 
-	dkprintf("(syscall) Trying to connect host ...");
+	dkprintf("(ikc2mckernel) Trying to connect host ...");
 	while (ihk_ikc_connect(NULL, &param) != 0) {
 		dkprintf(".");
 		ihk_mc_delay_us(1000 * 1000);
 	}
 	dkprintf("connected.\n");
 
-	get_this_cpu_local_var()->syscall_channel2 = param.channel;
-
-	pckt.msg = SCD_MSG_INIT_CHANNEL;
-	pckt.ref = ihk_mc_get_processor_id();
-	pckt.arg = virt_to_phys(&cpu_local_var(iip2));
-	syscall_channel_send(param.channel, &pckt);
+/* Comment: 待ち受け処理channel_list に追加する */
+	ihk_ikc_add_intr_channel(NULL, param.channel, ihk_ikc_get_processor_id());
 }
+
diff --git a/kernel/include/cls.h b/kernel/include/cls.h
index d91cf4e6..9f364916 100644
--- a/kernel/include/cls.h
+++ b/kernel/include/cls.h
@@ -75,11 +75,9 @@ struct cpu_local_var {
 	struct list_head runq;
 	size_t runq_len;
 
-	struct ihk_ikc_channel_desc *syscall_channel;
-	struct ikc_scd_init_param iip;
+/* Comment: 送信用するchannelをsyscall_channel から ikc2linux へ変更 */
+	struct ihk_ikc_channel_desc *ikc2linux;
 
-	struct ihk_ikc_channel_desc *syscall_channel2;
-	struct ikc_scd_init_param iip2;
 	struct resource_set *resource_set;
 	
 	int status;
diff --git a/kernel/include/init.h b/kernel/include/init.h
index 60729375..57f31f53 100644
--- a/kernel/include/init.h
+++ b/kernel/include/init.h
@@ -24,8 +24,9 @@ extern void cpu_local_var_init(void);
 extern void kmalloc_init(void);
 extern void ap_start(void);
 extern void ihk_mc_dma_init(void);
-extern void init_host_syscall_channel(void);
-extern void init_host_syscall_channel2(void);
+extern void init_host_ikc2linux(int linux_cpu);
+extern void init_host_ikc2mckernel(void);
+//extern void set_ikc2linux_to_local(int linux_cpu);
 extern void sched_init(void);
 extern void pc_ap_init(void);
 extern void cpu_sysfs_setup(void);
diff --git a/kernel/init.c b/kernel/init.c
index 37428d13..e605da8b 100644
--- a/kernel/init.c
+++ b/kernel/init.c
@@ -360,6 +360,7 @@ extern int num_processors;
 
 static void post_init(void)
 {
+	struct ihk_mc_cpu_info *cpu_info = ihk_mc_get_cpu_info();
 	cpu_enable_interrupt();
 
 	while (!host_ikc_inited) {
@@ -368,8 +369,8 @@ static void post_init(void)
 	}
 
 	if (find_command_line("hidos")) {
-		init_host_syscall_channel();
-		init_host_syscall_channel2();
+		init_host_ikc2mckernel();
+		init_host_ikc2linux(cpu_info->ikc_cpus[ihk_mc_get_processor_id()]);
 	}
 
 	arch_setup_vdso();
diff --git a/kernel/procfs.c b/kernel/procfs.c
index 4d5d4c7d..7c5122c4 100644
--- a/kernel/procfs.c
+++ b/kernel/procfs.c
@@ -47,7 +47,7 @@ procfs_thread_ctl(struct thread *thread, int msg)
 	struct ihk_ikc_channel_desc *syscall_channel;
 	struct ikc_scd_packet packet;
 
-	syscall_channel = cpu_local_var(syscall_channel);
+	syscall_channel = cpu_local_var(ikc2linux);
 	memset(&packet, '\0', sizeof packet);
 	packet.arg = thread->tid;
 	packet.msg = msg;
@@ -96,7 +96,7 @@ void process_procfs_request(struct ikc_scd_packet *rpacket)
 
 	dprintf("process_procfs_request: invoked.\n");
 
-	syscall_channel = get_cpu_local_var(0)->syscall_channel;
+	syscall_channel = get_cpu_local_var(0)->ikc2linux;
 
 	dprintf("rarg: %x\n", rarg);
 	parg = ihk_mc_map_memory(NULL, rarg, sizeof(struct procfs_read));
diff --git a/kernel/syscall.c b/kernel/syscall.c
index 6b510db1..e7d6d3ef 100644
--- a/kernel/syscall.c
+++ b/kernel/syscall.c
@@ -337,18 +337,14 @@ SYSCALL_DECLARE(track_syscalls)
 static void send_syscall(struct syscall_request *req, int cpu, int pid, struct syscall_response *res)
 {
 	struct ikc_scd_packet packet IHK_DMA_ALIGN;
-	struct ihk_ikc_channel_desc *syscall_channel;
+	struct ihk_ikc_channel_desc *syscall_channel = get_cpu_local_var(cpu)->ikc2linux;
 	int ret;
 
 	if(req->number == __NR_exit_group ||
 	   req->number == __NR_kill){ // interrupt syscall
-		extern int num_processors;
 
-		syscall_channel = get_cpu_local_var(0)->syscall_channel2;
-		
 		/* XXX: is this really going to work if multiple processes 
 		 * exit/receive signals at the same time?? */
-		cpu = num_processors;
 		if (req->number == __NR_kill) {
 			req->rtid = -1;
 			pid = req->args[0];
@@ -356,9 +352,6 @@ static void send_syscall(struct syscall_request *req, int cpu, int pid, struct s
 		if (req->number == __NR_gettid)
 			pid = req->args[1];
 	}
-	else{
-		syscall_channel = get_cpu_local_var(cpu)->syscall_channel;
-	}
 
 	res->status = 0;
 	req->valid = 0;
diff --git a/kernel/sysfs.c b/kernel/sysfs.c
index dc97e58d..fc07cd32 100644
--- a/kernel/sysfs.c
+++ b/kernel/sysfs.c
@@ -113,7 +113,7 @@ sysfs_createf(struct sysfs_ops *ops, void *instance, int mode,
 	packet.msg = SCD_MSG_SYSFS_REQ_CREATE;
 	packet.sysfs_arg1 = virt_to_phys(param);
 
-	error = ihk_ikc_send(cpu_local_var(syscall_channel), &packet, 0);
+	error = ihk_ikc_send(cpu_local_var(ikc2linux), &packet, 0);
 	if (error) {
 		ekprintf("sysfs_createf:ihk_ikc_send failed. %d\n", error);
 		goto out;
@@ -183,7 +183,7 @@ sysfs_mkdirf(sysfs_handle_t *dirhp, const char *fmt, ...)
 	packet.msg = SCD_MSG_SYSFS_REQ_MKDIR;
 	packet.sysfs_arg1 = virt_to_phys(param);
 
-	error = ihk_ikc_send(cpu_local_var(syscall_channel), &packet, 0);
+	error = ihk_ikc_send(cpu_local_var(ikc2linux), &packet, 0);
 	if (error) {
 		ekprintf("sysfs_mkdirf:ihk_ikc_send failed. %d\n", error);
 		goto out;
@@ -257,7 +257,7 @@ sysfs_symlinkf(sysfs_handle_t targeth, const char *fmt, ...)
 	packet.msg = SCD_MSG_SYSFS_REQ_SYMLINK;
 	packet.sysfs_arg1 = virt_to_phys(param);
 
-	error = ihk_ikc_send(cpu_local_var(syscall_channel), &packet, 0);
+	error = ihk_ikc_send(cpu_local_var(ikc2linux), &packet, 0);
 	if (error) {
 		ekprintf("sysfs_symlinkf:ihk_ikc_send failed. %d\n", error);
 		goto out;
@@ -328,7 +328,7 @@ sysfs_lookupf(sysfs_handle_t *objhp, const char *fmt, ...)
 	packet.msg = SCD_MSG_SYSFS_REQ_LOOKUP;
 	packet.sysfs_arg1 = virt_to_phys(param);
 
-	error = ihk_ikc_send(cpu_local_var(syscall_channel), &packet, 0);
+	error = ihk_ikc_send(cpu_local_var(ikc2linux), &packet, 0);
 	if (error) {
 		ekprintf("sysfs_lookupf:ihk_ikc_send failed. %d\n", error);
 		goto out;
@@ -402,7 +402,7 @@ sysfs_unlinkf(int flags, const char *fmt, ...)
 	packet.msg = SCD_MSG_SYSFS_REQ_UNLINK;
 	packet.sysfs_arg1 = virt_to_phys(param);
 
-	error = ihk_ikc_send(cpu_local_var(syscall_channel), &packet, 0);
+	error = ihk_ikc_send(cpu_local_var(ikc2linux), &packet, 0);
 	if (error) {
 		ekprintf("sysfs_unlinkf:ihk_ikc_send failed. %d\n", error);
 		goto out;
@@ -462,7 +462,7 @@ sysfss_req_show(long nodeh, struct sysfs_ops *ops, void *instance)
 	packet.sysfs_arg1 = nodeh;
 	packet.sysfs_arg2 = ssize;
 
-	error = ihk_ikc_send(cpu_local_var(syscall_channel), &packet, 0);
+	error = ihk_ikc_send(cpu_local_var(ikc2linux), &packet, 0);
 	if (error) {
 		ekprintf("sysfss_req_show:ihk_ikc_send failed. %d\n", error);
 		/* through */
@@ -508,7 +508,7 @@ sysfss_req_store(long nodeh, struct sysfs_ops *ops, void *instance,
 	packet.sysfs_arg1 = nodeh;
 	packet.sysfs_arg2 = ssize;
 
-	error = ihk_ikc_send(cpu_local_var(syscall_channel), &packet, 0);
+	error = ihk_ikc_send(cpu_local_var(ikc2linux), &packet, 0);
 	if (error) {
 		ekprintf("sysfss_req_store:ihk_ikc_send failed. %d\n", error);
 		/* through */
@@ -539,7 +539,7 @@ sysfss_req_release(long nodeh, struct sysfs_ops *ops, void *instance)
 	packet.err = 0;
 	packet.sysfs_arg1 = nodeh;
 
-	error = ihk_ikc_send(cpu_local_var(syscall_channel), &packet, 0);
+	error = ihk_ikc_send(cpu_local_var(ikc2linux), &packet, 0);
 	if (error) {
 		ekprintf("sysfss_req_release:ihk_ikc_send failed. %d\n",
 				error);
@@ -623,7 +623,7 @@ sysfs_init(void)
 	packet.msg = SCD_MSG_SYSFS_REQ_SETUP;
 	packet.sysfs_arg1 = virt_to_phys(param);
 
-	error = ihk_ikc_send(cpu_local_var(syscall_channel), &packet, 0);
+	error = ihk_ikc_send(cpu_local_var(ikc2linux), &packet, 0);
 	if (error) {
 		ekprintf("sysfs_init:ihk_ikc_send failed. %d\n", error);
 		goto out;
diff --git a/lib/include/ihk/cpu.h b/lib/include/ihk/cpu.h
index 67ba79f8..71600773 100644
--- a/lib/include/ihk/cpu.h
+++ b/lib/include/ihk/cpu.h
@@ -50,6 +50,7 @@ struct ihk_mc_cpu_info {
 	int *hw_ids;
 	int *nodes;
 	int *linux_cpu_ids;
+	int *ikc_cpus;
 };
 
 struct ihk_mc_cpu_info *ihk_mc_get_cpu_info(void);
@@ -60,6 +61,7 @@ int ihk_mc_get_numa_id(void);
 int ihk_mc_get_nr_cores();
 int ihk_mc_get_core(int id, unsigned long *linux_core_id, unsigned long *apic_id,
                     int *numa_id);
+int ihk_mc_get_apicid(int linux_core_id);
 
 void ihk_mc_delay_us(int us);
 void ihk_mc_set_syscall_handler(long (*handler)(int, ihk_mc_user_context_t *));