diff --git a/arch/x86/kernel/cpu.c b/arch/x86/kernel/cpu.c index cf14bd91..8d7a02e6 100644 --- a/arch/x86/kernel/cpu.c +++ b/arch/x86/kernel/cpu.c @@ -30,6 +30,7 @@ #include #include #include +#include #define LAPIC_ID 0x020 #define LAPIC_TIMER 0x320 @@ -78,6 +79,7 @@ static void (*lapic_icr_write)(unsigned int h, unsigned int l); static void (*lapic_wait_icr_idle)(void); void (*x86_issue_ipi)(unsigned int apicid, unsigned int low); int running_on_kvm(void); +static void smp_func_call_handler(void); void init_processors_local(int max_id); void assign_processor_id(void); @@ -969,6 +971,9 @@ void handle_interrupt(int vector, struct x86_user_context *regs) tlb_flush_handler(vector); } + else if (vector == LOCAL_SMP_FUNC_CALL_VECTOR) { + smp_func_call_handler(); + } else if (vector == 133) { show_context_stack((uintptr_t *)regs->gpr.rbp); } @@ -1844,4 +1849,144 @@ int arch_cpu_read_write_register( return 0; } +/* + * Generic remote CPU function invocation facility. + */ +static void smp_func_call_handler(void) +{ + int irq_flags; + struct smp_func_call_request *req; + int reqs_left; + +reiterate: + req = NULL; + reqs_left = 0; + + irq_flags = ihk_mc_spinlock_lock( + &cpu_local_var(smp_func_req_lock)); + + /* Take requests one-by-one */ + if (!list_empty(&cpu_local_var(smp_func_req_list))) { + req = list_first_entry(&cpu_local_var(smp_func_req_list), + struct smp_func_call_request, list); + list_del(&req->list); + + reqs_left = !list_empty(&cpu_local_var(smp_func_req_list)); + } + + ihk_mc_spinlock_unlock(&cpu_local_var(smp_func_req_lock), + irq_flags); + + if (req) { + req->ret = req->sfcd->func(req->cpu_index, + req->sfcd->nr_cpus, req->sfcd->arg); + ihk_atomic_dec(&req->sfcd->cpus_left); + } + + if (reqs_left) + goto reiterate; +} + +int smp_call_func(cpu_set_t *__cpu_set, smp_func_t __func, void *__arg) +{ + int cpu, nr_cpus = 0; + int cpu_index = 0; + int this_cpu_index = 0; + struct smp_func_call_data sfcd; + struct smp_func_call_request *reqs; + int ret = 0; + int call_on_this_cpu = 0; + cpu_set_t cpu_set; + + /* Sanity checks */ + if (!__cpu_set || !__func) { + return -EINVAL; + } + + /* Make sure it won't change in between */ + cpu_set = *__cpu_set; + + for_each_set_bit(cpu, (unsigned long *)&cpu_set, + sizeof(cpu_set) * BITS_PER_BYTE) { + + if (cpu == ihk_mc_get_processor_id()) { + call_on_this_cpu = 1; + } + ++nr_cpus; + } + + if (!nr_cpus) { + return -EINVAL; + } + + reqs = kmalloc(sizeof(*reqs) * nr_cpus, IHK_MC_AP_NOWAIT); + if (!reqs) { + ret = -ENOMEM; + goto free_out; + } + + sfcd.nr_cpus = nr_cpus; + sfcd.func = __func; + sfcd.arg = __arg; + ihk_atomic_set(&sfcd.cpus_left, + call_on_this_cpu ? nr_cpus - 1 : nr_cpus); + + /* Add requests and send IPIs */ + cpu_index = 0; + for_each_set_bit(cpu, (unsigned long *)&cpu_set, + sizeof(cpu_set) * BITS_PER_BYTE) { + unsigned long irq_flags; + + reqs[cpu_index].cpu_index = cpu_index; + reqs[cpu_index].ret = 0; + + if (cpu == ihk_mc_get_processor_id()) { + this_cpu_index = cpu_index; + ++cpu_index; + continue; + } + + reqs[cpu_index].sfcd = &sfcd; + + irq_flags = + ihk_mc_spinlock_lock(&get_cpu_local_var(cpu)->smp_func_req_lock); + list_add_tail(&reqs[cpu_index].list, + &get_cpu_local_var(cpu)->smp_func_req_list); + ihk_mc_spinlock_unlock(&get_cpu_local_var(cpu)->smp_func_req_lock, + irq_flags); + + ihk_mc_interrupt_cpu( + get_x86_cpu_local_variable(cpu)->apic_id, + LOCAL_SMP_FUNC_CALL_VECTOR); + + ++cpu_index; + } + + /* Is this CPU involved? */ + if (call_on_this_cpu) { + reqs[this_cpu_index].ret = + __func(this_cpu_index, nr_cpus, __arg); + } + + /* Wait for the rest of the CPUs */ + while (ihk_atomic_read(&sfcd.cpus_left) > 0) { + cpu_pause(); + } + + /* Check return values, if error, report the first non-zero */ + for (cpu_index = 0; cpu_index < nr_cpus; ++cpu_index) { + if (reqs[cpu_index].ret != 0) { + ret = reqs[cpu_index].ret; + goto free_out; + } + } + + ret = 0; + +free_out: + kfree(reqs); + + return ret; +} + /*** end of file ***/ diff --git a/kernel/cls.c b/kernel/cls.c index e1696a69..a1088950 100644 --- a/kernel/cls.c +++ b/kernel/cls.c @@ -37,8 +37,10 @@ void cpu_local_var_init(void) clv = ihk_mc_alloc_pages(z, IHK_MC_AP_CRITICAL); memset(clv, 0, z * PAGE_SIZE); - for(i = 0; i < num_processors; i++) + for (i = 0; i < num_processors; i++) { clv[i].monitor = monitor + i; + INIT_LIST_HEAD(&clv[i].smp_func_req_list); + } cpu_local_var_initialized = 1; } diff --git a/kernel/include/cls.h b/kernel/include/cls.h index 85aa1906..f754c170 100644 --- a/kernel/include/cls.h +++ b/kernel/include/cls.h @@ -58,6 +58,26 @@ extern ihk_spinlock_t cpu_status_lock; #define CPU_FLAG_NEED_RESCHED 0x1U #define CPU_FLAG_NEED_MIGRATE 0x2U +typedef int (*smp_func_t)(int cpu_index, int nr_cpus, void *arg); +int smp_call_func(cpu_set_t *__cpu_set, smp_func_t __func, void *__arg); + +struct smp_func_call_data { + /* XXX: Sync MCS lock to avoid contention on counter */ + // mcs_lock_node_t lock; + int nr_cpus; + ihk_atomic_t cpus_left; + + smp_func_t func; + void *arg; +}; + +struct smp_func_call_request { + struct smp_func_call_data *sfcd; + int cpu_index; + int ret; + struct list_head list; +}; + struct cpu_local_var { /* malloc */ struct list_head free_list; @@ -93,6 +113,9 @@ struct cpu_local_var { int timer_enabled; int kmalloc_initialized; struct ihk_os_monitor *monitor; + + ihk_spinlock_t smp_func_req_lock; + struct list_head smp_func_req_list; } __attribute__((aligned(64))); diff --git a/lib/include/ihk/cpu.h b/lib/include/ihk/cpu.h index 45330769..75737e69 100644 --- a/lib/include/ihk/cpu.h +++ b/lib/include/ihk/cpu.h @@ -111,6 +111,8 @@ enum ihk_asr_type { #define IHK_TLB_FLUSH_IRQ_VECTOR_SIZE 64 #define IHK_TLB_FLUSH_IRQ_VECTOR_END (IHK_TLB_FLUSH_IRQ_VECTOR_START + IHK_TLB_FLUSH_IRQ_VECTOR_SIZE) +#define LOCAL_SMP_FUNC_CALL_VECTOR 0xf1 + int ihk_mc_arch_set_special_register(enum ihk_asr_type, unsigned long value); int ihk_mc_arch_get_special_register(enum ihk_asr_type, unsigned long *value);