From de042b2cb2e400b4eb17ae2a381f12d92c7e20c5 Mon Sep 17 00:00:00 2001 From: Balazs Gerofi Date: Tue, 19 Mar 2019 16:33:28 +0900 Subject: [PATCH] IPI: use logical CPU ids in ihk_mc_interrupt_cpu() Also make remote TLB invalidation arch independent, removes POSTK_DEBUG_ARCH_DEP_8. Change-Id: I2b0fbcfa2bfe5da07607863e3e772d8e892e8525 --- arch/arm64/kernel/cpu.c | 7 ++- arch/arm64/kernel/memory.c | 38 ------------- arch/x86_64/kernel/cpu.c | 12 ++-- arch/x86_64/kernel/memory.c | 103 ----------------------------------- arch/x86_64/kernel/syscall.c | 3 +- ihk | 2 +- kernel/mem.c | 8 --- kernel/process.c | 33 +++-------- kernel/syscall.c | 14 ++--- 9 files changed, 29 insertions(+), 191 deletions(-) diff --git a/arch/arm64/kernel/cpu.c b/arch/arm64/kernel/cpu.c index c9cbe9f5..2297bd9e 100644 --- a/arch/arm64/kernel/cpu.c +++ b/arch/arm64/kernel/cpu.c @@ -32,6 +32,7 @@ #include #include #include +#include //#define DEBUG_PRINT_CPU @@ -1346,11 +1347,15 @@ int ihk_mc_arch_get_special_register(enum ihk_asr_type type, } /*@ - @ requires \valid_apicid(cpu); // valid APIC ID or not + @ requires \valid_cpuid(cpu); // valid CPU logical ID @ ensures \result == 0 @*/ int ihk_mc_interrupt_cpu(int cpu, int vector) { + if (cpu < 0 || cpu >= num_processors) { + kprintf("%s: invalid CPU id: %d\n", __func__, cpu); + return -1; + } dkprintf("[%d] ihk_mc_interrupt_cpu: %d\n", ihk_mc_get_processor_id(), cpu); (*arm64_issue_ipi)(cpu, vector); return 0; diff --git a/arch/arm64/kernel/memory.c b/arch/arm64/kernel/memory.c index d441e924..739b03e2 100644 --- a/arch/arm64/kernel/memory.c +++ b/arch/arm64/kernel/memory.c @@ -3715,44 +3715,6 @@ translation_table_t* get_translation_table_as_paddr(const struct page_table *pt) return pt->tt_pa; } -#ifdef POSTK_DEBUG_ARCH_DEP_8 -void remote_flush_tlb_cpumask(struct process_vm *vm, - unsigned long addr, int cpu_id) -{ - unsigned long cpu; - cpu_set_t _cpu_set; - int flush_ind; - - if (addr) { - flush_ind = (addr >> PAGE_SHIFT) % IHK_TLB_FLUSH_IRQ_VECTOR_SIZE; - } - /* Zero address denotes full TLB flush */ - else { - /* Random.. */ - flush_ind = (rdtsc()) % IHK_TLB_FLUSH_IRQ_VECTOR_SIZE; - } - - /* Take a copy of the cpu set so that we don't hold the lock - * all the way while interrupting other cores */ - ihk_mc_spinlock_lock_noirq(&vm->address_space->cpu_set_lock); - memcpy(&_cpu_set, &vm->address_space->cpu_set, sizeof(cpu_set_t)); - ihk_mc_spinlock_unlock_noirq(&vm->address_space->cpu_set_lock); - - /* Loop through CPUs in this address space and interrupt them for - * TLB flush on the specified address */ - for_each_set_bit(cpu, (const unsigned long*)&_cpu_set.__bits, CPU_SETSIZE) { - if (ihk_mc_get_processor_id() == cpu) - continue; - - dkprintf("remote_flush_tlb_cpumask: flush_ind: %d, addr: 0x%lX, interrupting cpu: %d\n", - flush_ind, addr, cpu); - - ihk_mc_interrupt_cpu(cpu, - ihk_mc_get_vector(flush_ind + IHK_TLB_FLUSH_IRQ_VECTOR_START)); - } -} -#endif /* POSTK_DEBUG_ARCH_DEP_8 */ - void arch_adjust_allocate_page_size(struct page_table *pt, uintptr_t fault_addr, pte_t *ptep, diff --git a/arch/x86_64/kernel/cpu.c b/arch/x86_64/kernel/cpu.c index abe7f4c1..1dbb2bab 100644 --- a/arch/x86_64/kernel/cpu.c +++ b/arch/x86_64/kernel/cpu.c @@ -1601,14 +1601,18 @@ int ihk_mc_arch_get_special_register(enum ihk_asr_type type, } /*@ - @ requires \valid_apicid(cpu); // valid APIC ID or not + @ requires \valid_cpuid(cpu); // valid CPU logical ID @ ensures \result == 0 @*/ int ihk_mc_interrupt_cpu(int cpu, int vector) { + if (cpu < 0 || cpu >= num_processors) { + kprintf("%s: invalid CPU id: %d\n", __func__, cpu); + return -1; + } dkprintf("[%d] ihk_mc_interrupt_cpu: %d\n", ihk_mc_get_processor_id(), cpu); - x86_issue_ipi(cpu, vector); + x86_issue_ipi(get_x86_cpu_local_variable(cpu)->apic_id, vector); return 0; } @@ -2095,9 +2099,7 @@ int smp_call_func(cpu_set_t *__cpu_set, smp_func_t __func, void *__arg) ihk_mc_spinlock_unlock(&get_cpu_local_var(cpu)->smp_func_req_lock, irq_flags); - ihk_mc_interrupt_cpu( - get_x86_cpu_local_variable(cpu)->apic_id, - LOCAL_SMP_FUNC_CALL_VECTOR); + ihk_mc_interrupt_cpu(cpu, LOCAL_SMP_FUNC_CALL_VECTOR); ++cpu_index; } diff --git a/arch/x86_64/kernel/memory.c b/arch/x86_64/kernel/memory.c index 8f9d02ba..8d32e51f 100644 --- a/arch/x86_64/kernel/memory.c +++ b/arch/x86_64/kernel/memory.c @@ -1354,109 +1354,6 @@ struct clear_range_args { int max_nr_addr; }; -#ifdef POSTK_DEBUG_ARCH_DEP_8 -void remote_flush_tlb_cpumask(struct process_vm *vm, - unsigned long addr, int cpu_id) -{ - unsigned long __addr = addr; - return remote_flush_tlb_array_cpumask(vm, &__addr, 1, cpu_id); -} - -void remote_flush_tlb_array_cpumask(struct process_vm *vm, - unsigned long *addr, - int nr_addr, - int cpu_id) -{ - unsigned long cpu; - int flush_ind; - struct tlb_flush_entry *flush_entry; - cpu_set_t _cpu_set; - - if (addr[0]) { - flush_ind = (addr[0] >> PAGE_SHIFT) % IHK_TLB_FLUSH_IRQ_VECTOR_SIZE; - } - /* Zero address denotes full TLB flush */ - else { - /* Random.. */ - flush_ind = (rdtsc()) % IHK_TLB_FLUSH_IRQ_VECTOR_SIZE; - } - - flush_entry = &tlb_flush_vector[flush_ind]; - - /* Take a copy of the cpu set so that we don't hold the lock - * all the way while interrupting other cores */ - ihk_mc_spinlock_lock_noirq(&vm->address_space->cpu_set_lock); - memcpy(&_cpu_set, &vm->address_space->cpu_set, sizeof(cpu_set_t)); - ihk_mc_spinlock_unlock_noirq(&vm->address_space->cpu_set_lock); - - dkprintf("trying to aquire flush_entry->lock flush_ind: %d\n", flush_ind); - - ihk_mc_spinlock_lock_noirq(&flush_entry->lock); - - flush_entry->vm = vm; - flush_entry->addr = addr; - flush_entry->nr_addr = nr_addr; - ihk_atomic_set(&flush_entry->pending, 0); - - dkprintf("lock aquired, iterating cpu mask.. flush_ind: %d\n", flush_ind); - - /* Loop through CPUs in this address space and interrupt them for - * TLB flush on the specified address */ - for_each_set_bit(cpu, (const unsigned long*)&_cpu_set.__bits, CPU_SETSIZE) { - - if (ihk_mc_get_processor_id() == cpu) - continue; - - ihk_atomic_inc(&flush_entry->pending); - dkprintf("remote_flush_tlb_cpumask: flush_ind: %d, addr: 0x%lX, interrupting cpu: %d\n", - flush_ind, addr, cpu); - -#ifdef POSTK_DEBUG_ARCH_DEP_8 /* arch depend hide */ - /* TODO(pka_idke) Interim support */ - ihk_mc_interrupt_cpu(cpu, - ihk_mc_get_vector(flush_ind + IHK_TLB_FLUSH_IRQ_VECTOR_START)); -#else /* POSTK_DEBUG_ARCH_DEP_8 */ - ihk_mc_interrupt_cpu(get_x86_cpu_local_variable(cpu)->apic_id, - flush_ind + IHK_TLB_FLUSH_IRQ_VECTOR_START); -#endif /* POSTK_DEBUG_ARCH_DEP_8 */ - } - -#ifdef DEBUG_IC_TLB - { - unsigned long tsc; - tsc = rdtsc() + 12884901888; /* 1.2GHz =>10 sec */ -#endif - if (flush_entry->addr[0]) { - int i; - - for (i = 0; i < flush_entry->nr_addr; ++i) { - flush_tlb_single(flush_entry->addr[i] & PAGE_MASK); - } - } - /* Zero address denotes full TLB flush */ - else { - flush_tlb(); - } - - /* Wait for all cores */ - while (ihk_atomic_read(&flush_entry->pending) != 0) { - cpu_pause(); - -#ifdef DEBUG_IC_TLB - if (rdtsc() > tsc) { - kprintf("waited 10 secs for remote TLB!! -> panic_all()\n"); - panic_all_cores("waited 10 secs for remote TLB!!\n"); - } -#endif - } -#ifdef DEBUG_IC_TLB - } -#endif - - ihk_mc_spinlock_unlock_noirq(&flush_entry->lock); -} -#endif /* POSTK_DEBUG_ARCH_DEP_8 */ - static void remote_flush_tlb_add_addr(struct clear_range_args *args, unsigned long addr) { diff --git a/arch/x86_64/kernel/syscall.c b/arch/x86_64/kernel/syscall.c index bd5664f9..d01dcdc4 100644 --- a/arch/x86_64/kernel/syscall.c +++ b/arch/x86_64/kernel/syscall.c @@ -1570,7 +1570,8 @@ done: if (thread != tthread) { dkprintf("do_kill,ipi,pid=%d,cpu_id=%d\n", tproc->pid, tthread->cpu_id); - ihk_mc_interrupt_cpu(get_x86_cpu_local_variable(tthread->cpu_id)->apic_id, 0xd0); + ihk_mc_interrupt_cpu(tthread->cpu_id, + ihk_mc_get_vector(IHK_GV_IKC)); } if (status != PS_RUNNING) { diff --git a/ihk b/ihk index 0681d7a0..70adc3dc 160000 --- a/ihk +++ b/ihk @@ -1 +1 @@ -Subproject commit 0681d7a055ef42f840846ca1738849a9b1e35dfa +Subproject commit 70adc3dcfd02e878141634a2fe84e859216d431b diff --git a/kernel/mem.c b/kernel/mem.c index c154769a..50e24a21 100644 --- a/kernel/mem.c +++ b/kernel/mem.c @@ -996,7 +996,6 @@ void coredump(struct thread *thread, void *regs) freecore(&coretable); } -#ifndef POSTK_DEBUG_ARCH_DEP_8 void remote_flush_tlb_cpumask(struct process_vm *vm, unsigned long addr, int cpu_id) { @@ -1053,14 +1052,8 @@ void remote_flush_tlb_array_cpumask(struct process_vm *vm, dkprintf("remote_flush_tlb_cpumask: flush_ind: %d, addr: 0x%lX, interrupting cpu: %d\n", flush_ind, addr, cpu); -#ifdef POSTK_DEBUG_ARCH_DEP_8 /* arch depend hide */ - /* TODO(pka_idke) Interim support */ ihk_mc_interrupt_cpu(cpu, ihk_mc_get_vector(flush_ind + IHK_TLB_FLUSH_IRQ_VECTOR_START)); -#else /* POSTK_DEBUG_ARCH_DEP_8 */ - ihk_mc_interrupt_cpu(get_x86_cpu_local_variable(cpu)->apic_id, - flush_ind + IHK_TLB_FLUSH_IRQ_VECTOR_START); -#endif /* POSTK_DEBUG_ARCH_DEP_8 */ } #ifdef DEBUG_IC_TLB @@ -1097,7 +1090,6 @@ void remote_flush_tlb_array_cpumask(struct process_vm *vm, ihk_mc_spinlock_unlock_noirq(&flush_entry->lock); } -#endif /* POSTK_DEBUG_ARCH_DEP_8 */ void tlb_flush_handler(int vector) { diff --git a/kernel/process.c b/kernel/process.c index 816c0318..d1856ad0 100644 --- a/kernel/process.c +++ b/kernel/process.c @@ -3518,14 +3518,8 @@ int __sched_wakeup_thread(struct thread *thread, if (!status && (thread->cpu_id != ihk_mc_get_processor_id())) { dkprintf("%s: issuing IPI, thread->cpu_id=%d\n", __FUNCTION__, thread->cpu_id); -#ifdef POSTK_DEBUG_ARCH_DEP_8 /* arch depend hide */ ihk_mc_interrupt_cpu(thread->cpu_id, ihk_mc_get_vector(IHK_GV_IKC)); -#else /* POSTK_DEBUG_ARCH_DEP_8 */ - ihk_mc_interrupt_cpu( - get_x86_cpu_local_variable(thread->cpu_id)->apic_id, - 0xd1); -#endif /* POSTK_DEBUG_ARCH_DEP_8 */ } return status; @@ -3579,15 +3573,11 @@ void sched_request_migrate(int cpu_id, struct thread *thread) v->status = CPU_STATUS_RUNNING; ihk_mc_spinlock_unlock(&v->runq_lock, irqstate); -#ifdef POSTK_DEBUG_ARCH_DEP_8 /* arch depend hide */ - if (cpu_id != ihk_mc_get_processor_id()) - ihk_mc_interrupt_cpu(/* Kick scheduler */ - thread->cpu_id, ihk_mc_get_vector(IHK_GV_IKC)); -#else /* POSTK_DEBUG_ARCH_DEP_8 */ - if (cpu_id != ihk_mc_get_processor_id()) - ihk_mc_interrupt_cpu(/* Kick scheduler */ - get_x86_cpu_local_variable(cpu_id)->apic_id, 0xd1); -#endif /* POSTK_DEBUG_ARCH_DEP_8 */ + if (cpu_id != ihk_mc_get_processor_id()) { + /* Kick scheduler */ + ihk_mc_interrupt_cpu(thread->cpu_id, + ihk_mc_get_vector(IHK_GV_IKC)); + } dkprintf("%s: tid: %d -> cpu: %d\n", __FUNCTION__, thread->tid, cpu_id); @@ -3640,15 +3630,10 @@ void runq_add_thread(struct thread *thread, int cpu_id) #endif /* Kick scheduler */ -#ifdef POSTK_DEBUG_ARCH_DEP_8 /* arch depend hide */ - if (cpu_id != ihk_mc_get_processor_id()) - ihk_mc_interrupt_cpu( - thread->cpu_id, ihk_mc_get_vector(IHK_GV_IKC)); -#else /* POSTK_DEBUG_ARCH_DEP_8 */ - if (cpu_id != ihk_mc_get_processor_id()) - ihk_mc_interrupt_cpu( - get_x86_cpu_local_variable(cpu_id)->apic_id, 0xd1); -#endif /* POSTK_DEBUG_ARCH_DEP_8 */ + if (cpu_id != ihk_mc_get_processor_id()) { + ihk_mc_interrupt_cpu(thread->cpu_id, + ihk_mc_get_vector(IHK_GV_IKC)); + } } /* NOTE: shouldn't remove a running process! */ diff --git a/kernel/syscall.c b/kernel/syscall.c index 8de5039c..182dab66 100644 --- a/kernel/syscall.c +++ b/kernel/syscall.c @@ -5988,11 +5988,8 @@ SYSCALL_DECLARE(getrusage) child->status == PS_RUNNING && !child->in_kernel){ child->times_update = 0; -#ifdef POSTK_DEBUG_ARCH_DEP_8 /* arch depend hide */ - ihk_mc_interrupt_cpu(child->cpu_id, ihk_mc_get_vector(IHK_GV_IKC)); -#else /* POSTK_DEBUG_ARCH_DEP_8 */ - ihk_mc_interrupt_cpu(get_x86_cpu_local_variable(child->cpu_id)->apic_id, 0xd1); -#endif /* POSTK_DEBUG_ARCH_DEP_8 */ + ihk_mc_interrupt_cpu(child->cpu_id, + ihk_mc_get_vector(IHK_GV_IKC)); } else child->times_update = 1; @@ -7231,11 +7228,8 @@ SYSCALL_DECLARE(clock_gettime) child->status == PS_RUNNING && !child->in_kernel){ child->times_update = 0; -#ifdef POSTK_DEBUG_ARCH_DEP_8 /* arch depend hide */ - ihk_mc_interrupt_cpu(child->cpu_id, ihk_mc_get_vector(IHK_GV_IKC)); -#else /* POSTK_DEBUG_ARCH_DEP_8 */ - ihk_mc_interrupt_cpu(get_x86_cpu_local_variable(child->cpu_id)->apic_id, 0xd1); -#endif /* POSTK_DEBUG_ARCH_DEP_8 */ + ihk_mc_interrupt_cpu(child->cpu_id, + ihk_mc_get_vector(IHK_GV_IKC)); } } ats.tv_sec = proc->utime.tv_sec;