SMP: generic function call facility for CPU sets

arch/x86/kernel/cpu.c

@@ -30,6 +30,7 @@
 #include <cls.h>
 #include <prctl.h>
 #include <page.h>
+#include <kmalloc.h>
 
 #define LAPIC_ID            0x020
 #define LAPIC_TIMER         0x320
@@ -78,6 +79,7 @@ static void (*lapic_icr_write)(unsigned int h, unsigned int l);
 static void (*lapic_wait_icr_idle)(void);
 void (*x86_issue_ipi)(unsigned int apicid, unsigned int low);
 int running_on_kvm(void);
+static void smp_func_call_handler(void);
 
 void init_processors_local(int max_id);
 void assign_processor_id(void);
@@ -969,6 +971,9 @@ void handle_interrupt(int vector, struct x86_user_context *regs)
 
 			tlb_flush_handler(vector);
 	} 
+	else if (vector == LOCAL_SMP_FUNC_CALL_VECTOR) {
+		smp_func_call_handler();
+	}
 	else if (vector == 133) {
 		show_context_stack((uintptr_t *)regs->gpr.rbp);
 	}
@@ -1844,4 +1849,144 @@ int arch_cpu_read_write_register(
 	return 0;
 }
 
+/*
+ * Generic remote CPU function invocation facility.
+ */
+static void smp_func_call_handler(void)
+{
+	int irq_flags;
+	struct smp_func_call_request *req;
+	int reqs_left;
+
+reiterate:
+	req = NULL;
+	reqs_left = 0;
+
+	irq_flags = ihk_mc_spinlock_lock(
+			&cpu_local_var(smp_func_req_lock));
+
+	/* Take requests one-by-one */
+	if (!list_empty(&cpu_local_var(smp_func_req_list))) {
+		req = list_first_entry(&cpu_local_var(smp_func_req_list),
+			struct smp_func_call_request, list);
+		list_del(&req->list);
+
+		reqs_left = !list_empty(&cpu_local_var(smp_func_req_list));
+	}
+
+	ihk_mc_spinlock_unlock(&cpu_local_var(smp_func_req_lock),
+			irq_flags);
+
+	if (req) {
+		req->ret = req->sfcd->func(req->cpu_index,
+				req->sfcd->nr_cpus, req->sfcd->arg);
+		ihk_atomic_dec(&req->sfcd->cpus_left);
+	}
+
+	if (reqs_left)
+		goto reiterate;
+}
+
+int smp_call_func(cpu_set_t *__cpu_set, smp_func_t __func, void *__arg)
+{
+	int cpu, nr_cpus = 0;
+	int cpu_index = 0;
+	int this_cpu_index = 0;
+	struct smp_func_call_data sfcd;
+	struct smp_func_call_request *reqs;
+	int ret = 0;
+	int call_on_this_cpu = 0;
+	cpu_set_t cpu_set;
+
+	/* Sanity checks */
+	if (!__cpu_set || !__func) {
+		return -EINVAL;
+	}
+
+	/* Make sure it won't change in between */
+	cpu_set = *__cpu_set;
+
+	for_each_set_bit(cpu, (unsigned long *)&cpu_set,
+			sizeof(cpu_set) * BITS_PER_BYTE) {
+
+		if (cpu == ihk_mc_get_processor_id()) {
+			call_on_this_cpu = 1;
+		}
+		++nr_cpus;
+	}
+
+	if (!nr_cpus) {
+		return -EINVAL;
+	}
+
+	reqs = kmalloc(sizeof(*reqs) * nr_cpus, IHK_MC_AP_NOWAIT);
+	if (!reqs) {
+		ret = -ENOMEM;
+		goto free_out;
+	}
+
+	sfcd.nr_cpus = nr_cpus;
+	sfcd.func = __func;
+	sfcd.arg = __arg;
+	ihk_atomic_set(&sfcd.cpus_left,
+			call_on_this_cpu ? nr_cpus - 1 : nr_cpus);
+
+	/* Add requests and send IPIs */
+	cpu_index = 0;
+	for_each_set_bit(cpu, (unsigned long *)&cpu_set,
+			sizeof(cpu_set) * BITS_PER_BYTE) {
+		unsigned long irq_flags;
+
+		reqs[cpu_index].cpu_index = cpu_index;
+		reqs[cpu_index].ret = 0;
+
+		if (cpu == ihk_mc_get_processor_id()) {
+			this_cpu_index = cpu_index;
+			++cpu_index;
+			continue;
+		}
+
+		reqs[cpu_index].sfcd = &sfcd;
+
+		irq_flags =
+			ihk_mc_spinlock_lock(&get_cpu_local_var(cpu)->smp_func_req_lock);
+		list_add_tail(&reqs[cpu_index].list,
+				&get_cpu_local_var(cpu)->smp_func_req_list);
+		ihk_mc_spinlock_unlock(&get_cpu_local_var(cpu)->smp_func_req_lock,
+			irq_flags);
+
+		ihk_mc_interrupt_cpu(
+				get_x86_cpu_local_variable(cpu)->apic_id,
+				LOCAL_SMP_FUNC_CALL_VECTOR);
+
+		++cpu_index;
+	}
+
+	/* Is this CPU involved? */
+	if (call_on_this_cpu) {
+		reqs[this_cpu_index].ret =
+			__func(this_cpu_index, nr_cpus, __arg);
+	}
+
+	/* Wait for the rest of the CPUs */
+	while (ihk_atomic_read(&sfcd.cpus_left) > 0) {
+		cpu_pause();
+	}
+
+	/* Check return values, if error, report the first non-zero */
+	for (cpu_index = 0; cpu_index < nr_cpus; ++cpu_index) {
+		if (reqs[cpu_index].ret != 0) {
+			ret = reqs[cpu_index].ret;
+			goto free_out;
+		}
+	}
+
+	ret = 0;
+
+free_out:
+	kfree(reqs);
+
+	return ret;
+}
+
 /*** end of file ***/

kernel/cls.c

@@ -37,8 +37,10 @@ void cpu_local_var_init(void)
 	clv = ihk_mc_alloc_pages(z, IHK_MC_AP_CRITICAL);
 	memset(clv, 0, z * PAGE_SIZE);
 
-	for(i = 0; i < num_processors; i++)
+	for (i = 0; i < num_processors; i++) {
 		clv[i].monitor = monitor + i;
+		INIT_LIST_HEAD(&clv[i].smp_func_req_list);
+	}
 
 	cpu_local_var_initialized = 1;
 }

kernel/include/cls.h

@@ -58,6 +58,26 @@ extern ihk_spinlock_t	cpu_status_lock;
 #define CPU_FLAG_NEED_RESCHED	0x1U
 #define CPU_FLAG_NEED_MIGRATE	0x2U
 
+typedef int (*smp_func_t)(int cpu_index, int nr_cpus, void *arg);
+int smp_call_func(cpu_set_t *__cpu_set, smp_func_t __func, void *__arg);
+
+struct smp_func_call_data {
+	/* XXX: Sync MCS lock to avoid contention on counter */
+	// mcs_lock_node_t lock;
+	int nr_cpus;
+	ihk_atomic_t cpus_left;
+
+	smp_func_t func;
+	void *arg;
+};
+
+struct smp_func_call_request {
+	struct smp_func_call_data *sfcd;
+	int cpu_index;
+	int ret;
+	struct list_head list;
+};
+
 struct cpu_local_var {
 	/* malloc */
 	struct list_head free_list;
@@ -93,6 +113,9 @@ struct cpu_local_var {
 	int timer_enabled;
 	int kmalloc_initialized;
 	struct ihk_os_monitor *monitor;
+
+	ihk_spinlock_t smp_func_req_lock;
+	struct list_head smp_func_req_list;
 } __attribute__((aligned(64)));
 
 

lib/include/ihk/cpu.h

@@ -111,6 +111,8 @@ enum ihk_asr_type {
 #define IHK_TLB_FLUSH_IRQ_VECTOR_SIZE		64
 #define IHK_TLB_FLUSH_IRQ_VECTOR_END		(IHK_TLB_FLUSH_IRQ_VECTOR_START + IHK_TLB_FLUSH_IRQ_VECTOR_SIZE)
 
+#define LOCAL_SMP_FUNC_CALL_VECTOR   0xf1
+
 int ihk_mc_arch_set_special_register(enum ihk_asr_type, unsigned long value);
 int ihk_mc_arch_get_special_register(enum ihk_asr_type, unsigned long *value);