a9973e913d
Previously, futex code of McKerenl was called by mccontrol, but there ware some problems with this method. (Mainly, location of McKernel image on memory) Call futex code in mcctrl instead of the one in McKernel image, giving the following benefits: 1. Not relying on shared kernel virtual address space with Linux any more 2. The cpu id store / retrieve is not needed and resulting in the code Change-Id: Ic40929b64a655b270c435859fa287fedb713ee5c refe: #1428
1992 lines
50 KiB
C
1992 lines
50 KiB
C
/* cpu.c COPYRIGHT FUJITSU LIMITED 2015-2019 */
|
|
#include <ihk/cpu.h>
|
|
#include <ihk/mm.h>
|
|
#include <types.h>
|
|
#include <errno.h>
|
|
#include <list.h>
|
|
#include <memory.h>
|
|
#include <string.h>
|
|
#include <registers.h>
|
|
#include <cpulocal.h>
|
|
#include <signal.h>
|
|
#include <process.h>
|
|
#include <cls.h>
|
|
#include <thread_info.h>
|
|
#include <arch-memory.h>
|
|
#include <irq.h>
|
|
#include <lwk/compiler.h>
|
|
#include <ptrace.h>
|
|
#include <psci.h>
|
|
#include <smp.h>
|
|
#include <arch-timer.h>
|
|
#include <page.h>
|
|
#include <kmalloc.h>
|
|
#include <cpuinfo.h>
|
|
#include <cputype.h>
|
|
#include <hw_breakpoint.h>
|
|
#include <arch-perfctr.h>
|
|
#include <bitops-fls.h>
|
|
#include <debug-monitors.h>
|
|
#include <sysreg.h>
|
|
#include <cpufeature.h>
|
|
#include <ihk/debug.h>
|
|
#include <hwcap.h>
|
|
#include <virt.h>
|
|
#include <init.h>
|
|
#include <bootparam.h>
|
|
|
|
//#define DEBUG_PRINT_CPU
|
|
|
|
#include "postk_print_sysreg.c"
|
|
|
|
#ifdef DEBUG_PRINT_CPU
|
|
#undef DDEBUG_DEFAULT
|
|
#define DDEBUG_DEFAULT DDEBUG_PRINT
|
|
#endif
|
|
|
|
struct cpuinfo_arm64 cpuinfo_data[NR_CPUS]; /* index is logical cpuid */
|
|
|
|
static struct list_head handlers[1024];
|
|
static void cpu_init_interrupt_handler(void);
|
|
|
|
void init_processors_local(int max_id);
|
|
void assign_processor_id(void);
|
|
void arch_delay(int);
|
|
int gettime_local_support = 0;
|
|
|
|
extern int interrupt_from_user(void *);
|
|
|
|
extern unsigned long ihk_param_gic_dist_base_pa;
|
|
extern unsigned long ihk_param_gic_dist_map_size;
|
|
extern unsigned long ihk_param_gic_cpu_base_pa;
|
|
extern unsigned long ihk_param_gic_cpu_map_size;
|
|
extern unsigned int ihk_param_gic_version;
|
|
extern int snprintf(char * buf, size_t size, const char *fmt, ...);
|
|
|
|
/* Function pointers for GIC */
|
|
void (*gic_dist_init)(unsigned long dist_base_pa, unsigned long size);
|
|
void (*gic_cpu_init)(unsigned long cpu_base_pa, unsigned long size);
|
|
void (*gic_enable)(void);
|
|
void (*arm64_issue_ipi)(unsigned int cpid, unsigned int vector);
|
|
void (*arm64_issue_host_ipi)(unsigned int cpid, unsigned int vector);
|
|
void (*handle_arch_irq)(struct pt_regs *);
|
|
|
|
static void gic_init(void)
|
|
{
|
|
if(ihk_param_gic_version >= 3) {
|
|
/* Setup functions for GICv3 */
|
|
gic_dist_init = gic_dist_init_gicv3;
|
|
gic_cpu_init = gic_cpu_init_gicv3;
|
|
gic_enable = gic_enable_gicv3;
|
|
arm64_issue_ipi = arm64_issue_ipi_gicv3;
|
|
arm64_issue_host_ipi = arm64_issue_host_ipi_gicv3;
|
|
handle_arch_irq = handle_interrupt_gicv3;
|
|
kprintf("%: GICv3\n", __func__);
|
|
} else {
|
|
/* Setup functions for GICv2 */
|
|
gic_dist_init = gic_dist_init_gicv2;
|
|
gic_cpu_init = gic_cpu_init_gicv2;
|
|
gic_enable = gic_enable_gicv2;
|
|
arm64_issue_ipi = arm64_issue_ipi_gicv2;
|
|
arm64_issue_host_ipi = arm64_issue_host_ipi_gicv2;
|
|
handle_arch_irq = handle_interrupt_gicv2;
|
|
kprintf("%: GICv2\n", __func__);
|
|
}
|
|
|
|
gic_dist_init(ihk_param_gic_dist_base_pa, ihk_param_gic_dist_map_size);
|
|
gic_cpu_init(ihk_param_gic_cpu_base_pa, ihk_param_gic_cpu_map_size);
|
|
}
|
|
|
|
static void remote_tlb_flush_interrupt_handler(void *priv)
|
|
{
|
|
/*Interim support*/
|
|
flush_tlb();
|
|
}
|
|
|
|
static struct ihk_mc_interrupt_handler remote_tlb_flush_handler = {
|
|
.func = remote_tlb_flush_interrupt_handler,
|
|
.priv = NULL,
|
|
};
|
|
|
|
static void cpu_stop_interrupt_handler(void *priv)
|
|
{
|
|
kprintf("CPU%d: shutdown.\n", ihk_mc_get_processor_id());
|
|
psci_cpu_off();
|
|
}
|
|
|
|
static struct ihk_mc_interrupt_handler cpu_stop_handler = {
|
|
.func = cpu_stop_interrupt_handler,
|
|
.priv = NULL,
|
|
};
|
|
|
|
extern long freeze_thaw(void *nmi_ctx);
|
|
static void multi_interrupt_handler(void *priv)
|
|
{
|
|
switch (multi_intr_mode) {
|
|
case 1:
|
|
case 2: /* mode == 1or2, for FREEZER intr */
|
|
dkprintf("%s: freeze mode intr catch. (multi_intr_mode=%d)\n",
|
|
__func__, multi_intr_mode);
|
|
freeze_thaw(NULL);
|
|
break;
|
|
default:
|
|
ekprintf("%s: Unknown multi-intr-mode(%d) detected.\n",
|
|
__func__, multi_intr_mode);
|
|
break;
|
|
}
|
|
}
|
|
|
|
void arch_save_panic_regs(void *irq_regs)
|
|
{
|
|
struct pt_regs *regs = (struct pt_regs *)irq_regs;
|
|
union arm64_cpu_local_variables *clv;
|
|
|
|
clv = get_arm64_this_cpu_local();
|
|
|
|
/* If kernel mode PF occurred, unroll the causing call stack */
|
|
if (cpu_local_var(kernel_mode_pf_regs)) {
|
|
regs = cpu_local_var(kernel_mode_pf_regs);
|
|
}
|
|
|
|
/* For user-space, use saved kernel context */
|
|
if (regs->pc < USER_END) {
|
|
memset(clv->arm64_cpu_local_thread.panic_regs,
|
|
0, sizeof(clv->arm64_cpu_local_thread.panic_regs));
|
|
clv->arm64_cpu_local_thread.panic_regs[29] =
|
|
current_thread_info()->cpu_context.fp;
|
|
clv->arm64_cpu_local_thread.panic_regs[31] =
|
|
current_thread_info()->cpu_context.sp;
|
|
clv->arm64_cpu_local_thread.panic_regs[32] =
|
|
current_thread_info()->cpu_context.pc;
|
|
clv->arm64_cpu_local_thread.panic_regs[33] =
|
|
PSR_MODE_EL1h;
|
|
}
|
|
else {
|
|
memcpy(clv->arm64_cpu_local_thread.panic_regs,
|
|
regs->regs, sizeof(regs->regs));
|
|
clv->arm64_cpu_local_thread.panic_regs[31] = regs->sp;
|
|
clv->arm64_cpu_local_thread.panic_regs[32] = regs->pc;
|
|
clv->arm64_cpu_local_thread.panic_regs[33] =
|
|
regs->pstate;
|
|
|
|
}
|
|
|
|
clv->arm64_cpu_local_thread.paniced = 1;
|
|
}
|
|
|
|
void arch_clear_panic(void)
|
|
{
|
|
union arm64_cpu_local_variables *clv;
|
|
|
|
clv = get_arm64_this_cpu_local();
|
|
clv->arm64_cpu_local_thread.paniced = 0;
|
|
}
|
|
|
|
static struct ihk_mc_interrupt_handler multi_intr_handler = {
|
|
.func = multi_interrupt_handler,
|
|
.priv = NULL,
|
|
};
|
|
|
|
static void multi_nm_interrupt_handler(void *irq_regs)
|
|
{
|
|
extern int nmi_mode;
|
|
|
|
dkprintf("%s: ...\n", __func__);
|
|
switch (nmi_mode) {
|
|
case 0:
|
|
/* mode == 0, for MEMDUMP NMI */
|
|
arch_save_panic_regs(irq_regs);
|
|
ihk_mc_query_mem_areas();
|
|
/* memdump-nmi is halted McKernel, break is unnecessary. */
|
|
/* fall through */
|
|
case 3:
|
|
/* mode == 3, for SHUTDOWN-WAIT NMI */
|
|
kprintf("%s: STOP\n", __func__);
|
|
while (nmi_mode != 4)
|
|
cpu_halt();
|
|
break;
|
|
|
|
case 4:
|
|
/* mode == 4, continue NMI */
|
|
arch_clear_panic();
|
|
if (!ihk_mc_get_processor_id()) {
|
|
ihk_mc_clear_dump_page_completion();
|
|
}
|
|
kprintf("%s: RESUME, nmi_mode: %d\n", __func__, nmi_mode);
|
|
break;
|
|
|
|
default:
|
|
ekprintf("%s: Unknown nmi-mode(%d) detected.\n",
|
|
__func__, nmi_mode);
|
|
break;
|
|
}
|
|
}
|
|
|
|
static struct ihk_mc_interrupt_handler multi_nmi_handler = {
|
|
.func = multi_nm_interrupt_handler,
|
|
.priv = NULL,
|
|
};
|
|
|
|
static void init_smp_processor(void)
|
|
{
|
|
/* nothing */
|
|
}
|
|
|
|
/* @ref.impl arch/arm64/include/asm/cputype.h */
|
|
static inline uint32_t read_cpuid_cachetype(void)
|
|
{
|
|
return read_cpuid(CTR_EL0);
|
|
}
|
|
|
|
/* @ref.impl arch/arm64/include/asm/arch_timer.h */
|
|
static inline uint32_t arch_timer_get_cntfrq(void)
|
|
{
|
|
return read_sysreg(cntfrq_el0);
|
|
}
|
|
|
|
/* @ref.impl arch/arm64/kernel/cpuinfo.c::__cpuinfo_store_cpu */
|
|
static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info)
|
|
{
|
|
info->hwid = ihk_mc_get_hardware_processor_id(); /* McKernel Original. */
|
|
|
|
info->reg_cntfrq = arch_timer_get_cntfrq();
|
|
info->reg_ctr = read_cpuid_cachetype();
|
|
info->reg_dczid = read_cpuid(DCZID_EL0);
|
|
info->reg_midr = read_cpuid_id();
|
|
info->reg_revidr = read_cpuid(REVIDR_EL1);
|
|
|
|
info->reg_id_aa64dfr0 = read_cpuid(ID_AA64DFR0_EL1);
|
|
info->reg_id_aa64dfr1 = read_cpuid(ID_AA64DFR1_EL1);
|
|
info->reg_id_aa64isar0 = read_cpuid(ID_AA64ISAR0_EL1);
|
|
info->reg_id_aa64isar1 = read_cpuid(ID_AA64ISAR1_EL1);
|
|
info->reg_id_aa64mmfr0 = read_cpuid(ID_AA64MMFR0_EL1);
|
|
info->reg_id_aa64mmfr1 = read_cpuid(ID_AA64MMFR1_EL1);
|
|
info->reg_id_aa64mmfr2 = read_cpuid(ID_AA64MMFR2_EL1);
|
|
info->reg_id_aa64pfr0 = read_cpuid(ID_AA64PFR0_EL1);
|
|
info->reg_id_aa64pfr1 = read_cpuid(ID_AA64PFR1_EL1);
|
|
info->reg_id_aa64zfr0 = read_cpuid(ID_AA64ZFR0_EL1);
|
|
|
|
/* Update the 32bit ID registers only if AArch32 is implemented */
|
|
// if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) {
|
|
// panic("AArch32 is not supported.");
|
|
// }
|
|
|
|
if (id_aa64pfr0_sve(info->reg_id_aa64pfr0)) {
|
|
uint64_t zcr;
|
|
|
|
write_sysreg_s(ZCR_EL1_LEN_MASK, SYS_ZCR_EL1);
|
|
zcr = read_sysreg_s(SYS_ZCR_EL1);
|
|
zcr &= ~(uint64_t)ZCR_EL1_LEN_MASK;
|
|
zcr |= sve_get_vl() / 16 - 1;
|
|
|
|
info->reg_zcr = zcr;
|
|
}
|
|
}
|
|
|
|
/* @ref.impl arch/arm64/kernel/cpuinfo.c */
|
|
static void cpuinfo_store_boot_cpu(void)
|
|
{
|
|
struct cpuinfo_arm64 *info = &cpuinfo_data[0];
|
|
__cpuinfo_store_cpu(info);
|
|
init_cpu_features(info);
|
|
}
|
|
|
|
/* @ref.impl arch/arm64/kernel/cpuinfo.c */
|
|
static void cpuinfo_store_cpu(void)
|
|
{
|
|
int cpuid = ihk_mc_get_processor_id();
|
|
struct cpuinfo_arm64 *boot_cpu_data = &cpuinfo_data[0];
|
|
struct cpuinfo_arm64 *info = &cpuinfo_data[cpuid];
|
|
__cpuinfo_store_cpu(info);
|
|
update_cpu_features(cpuid, info, boot_cpu_data);
|
|
}
|
|
|
|
/* @ref.impl arch/arm64/kernel/setup.c::setup_processor */
|
|
static void setup_processor(void)
|
|
{
|
|
cpuinfo_store_boot_cpu();
|
|
enable_mrs_emulation();
|
|
}
|
|
|
|
static char *trampoline_va, *first_page_va;
|
|
|
|
static inline uint64_t pwr_arm64hpc_read_imp_fj_core_uarch_restrection_el1(void)
|
|
{
|
|
uint64_t reg;
|
|
|
|
asm volatile("mrs_s %0, " __stringify(IMP_FJ_CORE_UARCH_RESTRECTION_EL1)
|
|
: "=r" (reg) : : "memory");
|
|
return reg;
|
|
}
|
|
|
|
static ihk_spinlock_t imp_fj_core_uarch_restrection_el1_lock =
|
|
SPIN_LOCK_UNLOCKED;
|
|
static inline void pwr_arm64hpc_write_imp_fj_core_uarch_restrection_el1(uint64_t set_bit,
|
|
uint64_t clear_bit)
|
|
{
|
|
uint64_t reg;
|
|
unsigned long flags;
|
|
|
|
flags = ihk_mc_spinlock_lock(&imp_fj_core_uarch_restrection_el1_lock);
|
|
reg = pwr_arm64hpc_read_imp_fj_core_uarch_restrection_el1();
|
|
reg = (reg & ~clear_bit) | set_bit;
|
|
asm volatile("msr_s " __stringify(IMP_FJ_CORE_UARCH_RESTRECTION_EL1) ", %0"
|
|
: : "r" (reg) : "memory");
|
|
ihk_mc_spinlock_unlock(&imp_fj_core_uarch_restrection_el1_lock, flags);
|
|
}
|
|
|
|
static inline uint64_t pwr_arm64hpc_read_imp_soc_standby_ctrl_el1(void)
|
|
{
|
|
uint64_t reg;
|
|
|
|
asm volatile("mrs_s %0, " __stringify(IMP_SOC_STANDBY_CTRL_EL1)
|
|
: "=r" (reg) : : "memory");
|
|
return reg;
|
|
}
|
|
|
|
static ihk_spinlock_t imp_soc_standby_ctrl_el1_lock = SPIN_LOCK_UNLOCKED;
|
|
static inline void pwr_arm64hpc_write_imp_soc_standby_ctrl_el1(uint64_t set_bit,
|
|
uint64_t clear_bit)
|
|
{
|
|
unsigned long flags;
|
|
uint64_t reg;
|
|
|
|
flags = ihk_mc_spinlock_lock(&imp_soc_standby_ctrl_el1_lock);
|
|
reg = pwr_arm64hpc_read_imp_soc_standby_ctrl_el1();
|
|
reg = (reg & ~clear_bit) | set_bit;
|
|
asm volatile("msr_s " __stringify(IMP_SOC_STANDBY_CTRL_EL1) ", %0"
|
|
: : "r" (reg) : "memory");
|
|
ihk_mc_spinlock_unlock(&imp_soc_standby_ctrl_el1_lock, flags);
|
|
}
|
|
|
|
static unsigned long *retention_state_flag;
|
|
|
|
static inline int is_hpcpwr_available(void)
|
|
{
|
|
if (retention_state_flag)
|
|
return 1;
|
|
else
|
|
return 0;
|
|
}
|
|
|
|
static inline void pwr_arm64hpc_map_retention_state_flag(void)
|
|
{
|
|
extern unsigned long ihk_param_retention_state_flag_pa;
|
|
unsigned long size = BITS_TO_LONGS(NR_CPUS) * sizeof(unsigned long);
|
|
|
|
if (!ihk_param_retention_state_flag_pa) {
|
|
return;
|
|
}
|
|
retention_state_flag = map_fixed_area(ihk_param_retention_state_flag_pa,
|
|
size, 0);
|
|
}
|
|
|
|
static inline int pwr_arm64hpc_retention_state_get(uint64_t *val)
|
|
{
|
|
unsigned long linux_core_id;
|
|
int cpu = ihk_mc_get_processor_id();
|
|
int ret;
|
|
|
|
if (!is_hpcpwr_available()) {
|
|
*val = 0;
|
|
return 0;
|
|
}
|
|
|
|
ret = ihk_mc_get_core(cpu, &linux_core_id, NULL, NULL);
|
|
if (ret) {
|
|
return ret;
|
|
}
|
|
*val = test_bit(linux_core_id, retention_state_flag);
|
|
return ret;
|
|
}
|
|
|
|
static inline int pwr_arm64hpc_retention_state_set(uint64_t val)
|
|
{
|
|
unsigned long linux_core_id;
|
|
int cpu = ihk_mc_get_processor_id();
|
|
int ret;
|
|
|
|
if (!is_hpcpwr_available()) {
|
|
return 0;
|
|
}
|
|
|
|
ret = ihk_mc_get_core(cpu, &linux_core_id, NULL, NULL);
|
|
if (ret) {
|
|
return ret;
|
|
}
|
|
|
|
if (val) {
|
|
set_bit(cpu, retention_state_flag);
|
|
} else {
|
|
clear_bit(cpu, retention_state_flag);
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
static inline int pwr_arm64hpc_enable_retention_state(void)
|
|
{
|
|
if (!is_hpcpwr_available()) {
|
|
return 0;
|
|
}
|
|
|
|
pwr_arm64hpc_write_imp_soc_standby_ctrl_el1(IMP_SOC_STANDBY_CTRL_EL1_RETENTION,
|
|
0);
|
|
return 0;
|
|
}
|
|
|
|
static inline void init_power_management(void)
|
|
{
|
|
int state;
|
|
uint64_t imp_fj_clear_bit = 0;
|
|
uint64_t imp_soc_clear_bit = 0;
|
|
|
|
if (!is_hpcpwr_available()) {
|
|
return;
|
|
}
|
|
|
|
/* retention state */
|
|
state = pwr_arm64hpc_retention_state_set(0);
|
|
if (state) {
|
|
panic("error: initialize power management\n");
|
|
}
|
|
|
|
/* issue state */
|
|
imp_fj_clear_bit |= IMP_FJ_CORE_UARCH_RESTRECTION_EL1_ISSUE_RESTRICTION;
|
|
|
|
/* eco_state */
|
|
imp_fj_clear_bit |= IMP_FJ_CORE_UARCH_RESTRECTION_EL1_FL_RESTRICT_TRANS;
|
|
imp_soc_clear_bit |= IMP_SOC_STANDBY_CTRL_EL1_ECO_MODE;
|
|
|
|
/* ex_pipe_state */
|
|
imp_fj_clear_bit |= IMP_FJ_CORE_UARCH_RESTRECTION_EL1_EX_RESTRICTION;
|
|
|
|
/* write */
|
|
pwr_arm64hpc_write_imp_fj_core_uarch_restrection_el1(0,
|
|
imp_fj_clear_bit);
|
|
pwr_arm64hpc_write_imp_soc_standby_ctrl_el1(0, imp_soc_clear_bit);
|
|
}
|
|
|
|
/*@
|
|
@ assigns torampoline_va;
|
|
@ assigns first_page_va;
|
|
@*/
|
|
void ihk_mc_init_ap(void)
|
|
{
|
|
struct ihk_mc_cpu_info *cpu_info = ihk_mc_get_cpu_info();
|
|
|
|
trampoline_va = map_fixed_area(ap_trampoline, AP_TRAMPOLINE_SIZE, 0);
|
|
kprintf("Trampoline area: 0x%lx \n", ap_trampoline);
|
|
first_page_va = map_fixed_area(0, PAGE_SIZE, 0);
|
|
|
|
kprintf("# of cpus : %d\n", cpu_info->ncpus);
|
|
init_processors_local(cpu_info->ncpus);
|
|
|
|
/* Do initialization for THIS cpu (BSP) */
|
|
assign_processor_id();
|
|
|
|
ihk_mc_register_interrupt_handler(INTRID_CPU_STOP, &cpu_stop_handler);
|
|
ihk_mc_register_interrupt_handler(INTRID_MULTI_NMI, &multi_nmi_handler);
|
|
ihk_mc_register_interrupt_handler(INTRID_MULTI_INTR,
|
|
&multi_intr_handler);
|
|
ihk_mc_register_interrupt_handler(
|
|
ihk_mc_get_vector(IHK_TLB_FLUSH_IRQ_VECTOR_START),
|
|
&remote_tlb_flush_handler);
|
|
ihk_mc_register_interrupt_handler(get_timer_intrid(),
|
|
get_timer_handler());
|
|
|
|
init_smp_processor();
|
|
init_power_management();
|
|
}
|
|
|
|
extern void vdso_init(void);
|
|
long (*__arm64_syscall_handler)(int, ihk_mc_user_context_t *);
|
|
|
|
/* @ref.impl arch/arm64/include/asm/arch_timer.h::arch_timer_get_cntkctl */
|
|
static inline unsigned int arch_timer_get_cntkctl(void)
|
|
{
|
|
return read_sysreg(cntkctl_el1);
|
|
}
|
|
|
|
/* @ref.impl arch/arm64/include/asm/arch_timer.h::arch_timer_set_cntkctl */
|
|
static inline void arch_timer_set_cntkctl(unsigned int cntkctl)
|
|
{
|
|
write_sysreg(cntkctl, cntkctl_el1);
|
|
}
|
|
|
|
#ifdef CONFIG_ARM_ARCH_TIMER_EVTSTREAM
|
|
/* @ref.impl drivers/clocksource/arm_arch_timer.c::arch_timer_evtstrm_enable */
|
|
static void arch_timer_evtstrm_enable(int divider)
|
|
{
|
|
uint32_t cntkctl = arch_timer_get_cntkctl();
|
|
|
|
cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK;
|
|
/* Set the divider and enable virtual event stream */
|
|
cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT)
|
|
| ARCH_TIMER_VIRT_EVT_EN;
|
|
arch_timer_set_cntkctl(cntkctl);
|
|
}
|
|
|
|
/* @ref.impl include/clocksource/arm_arch_timer.h::ARCH_TIMER_EVT_STREAM_FREQ */
|
|
#define ARCH_TIMER_EVT_STREAM_FREQ 10000 /* 100us */
|
|
|
|
/* @ref.impl drivers/clocksource/arm_arch_timer.c::arch_timer_configure_evtstream */
|
|
static void arch_timer_configure_evtstream(void)
|
|
{
|
|
int evt_stream_div, pos;
|
|
extern unsigned long ihk_param_evtstrm_timer_rate;
|
|
|
|
/* Find the closest power of two to the divisor */
|
|
evt_stream_div = ihk_param_evtstrm_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ;
|
|
pos = fls(evt_stream_div);
|
|
if (pos > 1 && !(evt_stream_div & (1 << (pos - 2))))
|
|
pos--;
|
|
/* enable event stream */
|
|
arch_timer_evtstrm_enable(pos > 15 ? 15 : pos);
|
|
}
|
|
#else /* CONFIG_ARM_ARCH_TIMER_EVTSTREAM */
|
|
static inline void arch_timer_configure_evtstream(void) {}
|
|
#endif /* CONFIG_ARM_ARCH_TIMER_EVTSTREAM */
|
|
|
|
/* @ref.impl drivers/clocksource/arm_arch_timer.c::arch_counter_set_user_access */
|
|
static void arch_counter_set_user_access(void)
|
|
{
|
|
unsigned int cntkctl = arch_timer_get_cntkctl();
|
|
|
|
/* Disable user access to the timers and the physical counter */
|
|
/* Also disable virtual event stream */
|
|
|
|
cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN
|
|
| ARCH_TIMER_USR_VT_ACCESS_EN
|
|
| ARCH_TIMER_VIRT_EVT_EN
|
|
| ARCH_TIMER_USR_PCT_ACCESS_EN);
|
|
|
|
/* Enable user access to the virtual counter */
|
|
cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN;
|
|
arch_timer_set_cntkctl(cntkctl);
|
|
}
|
|
|
|
static void init_gettime_support(void)
|
|
{
|
|
arch_counter_set_user_access();
|
|
|
|
gettime_local_support = 1;
|
|
}
|
|
|
|
void init_cpu(void)
|
|
{
|
|
if(gic_enable)
|
|
gic_enable();
|
|
|
|
arm64_init_per_cpu_perfctr();
|
|
arm64_enable_pmu();
|
|
|
|
if (xos_is_tchip()) {
|
|
vhbm_barrier_registers_init();
|
|
scdrv_registers_init();
|
|
hpc_registers_init();
|
|
}
|
|
arm64_enable_user_access_pmu_regs();
|
|
}
|
|
|
|
#ifdef CONFIG_ARM64_VHE
|
|
/* @ref.impl arch/arm64/kernel/smp.c */
|
|
/* Whether the boot CPU is running in HYP mode or not */
|
|
static int boot_cpu_hyp_mode;
|
|
|
|
static inline void save_boot_cpu_run_el(void)
|
|
{
|
|
boot_cpu_hyp_mode = is_kernel_in_hyp_mode();
|
|
}
|
|
|
|
static inline int is_boot_cpu_in_hyp_mode(void)
|
|
{
|
|
return boot_cpu_hyp_mode;
|
|
}
|
|
|
|
/*
|
|
* Verify that a secondary CPU is running the kernel at the same
|
|
* EL as that of the boot CPU.
|
|
*/
|
|
static void verify_cpu_run_el(void)
|
|
{
|
|
int in_el2 = is_kernel_in_hyp_mode();
|
|
int boot_cpu_el2 = is_boot_cpu_in_hyp_mode();
|
|
|
|
if (in_el2 ^ boot_cpu_el2) {
|
|
kprintf("CPU%d: mismatched Exception Level(EL%d) with boot CPU(EL%d)\n",
|
|
ihk_mc_get_processor_id(),
|
|
in_el2 ? 2 : 1,
|
|
boot_cpu_el2 ? 2 : 1);
|
|
panic("verify_cpu_run_el(): PANIC: mismatched Exception Level.\n");
|
|
}
|
|
}
|
|
#else /* CONFIG_ARM64_VHE */
|
|
static inline void save_boot_cpu_run_el(void) {}
|
|
static inline void verify_cpu_run_el(void) {}
|
|
#endif /* CONFIG_ARM64_VHE */
|
|
|
|
void setup_arm64(void)
|
|
{
|
|
cpu_disable_interrupt();
|
|
|
|
cpu_init_interrupt_handler();
|
|
|
|
arm64_init_perfctr();
|
|
|
|
arch_timer_init();
|
|
|
|
gic_init();
|
|
|
|
pwr_arm64hpc_map_retention_state_flag();
|
|
|
|
init_cpu();
|
|
|
|
init_gettime_support();
|
|
|
|
setup_processor();
|
|
|
|
save_boot_cpu_run_el();
|
|
|
|
arch_hw_breakpoint_init();
|
|
|
|
debug_monitors_init();
|
|
|
|
arch_timer_configure_evtstream();
|
|
|
|
if (psci_init()) {
|
|
panic("setup_arm64(): PANIC: HOST-Linux does not have a psci -> method property.\n");
|
|
}
|
|
|
|
kprintf("setup_arm64 done.\n");
|
|
}
|
|
|
|
static volatile int cpu_boot_status;
|
|
|
|
void call_ap_func(void (*next_func)(void))
|
|
{
|
|
/* ap boot flag ON */
|
|
cpu_boot_status = 1;
|
|
cpu_enable_interrupt();
|
|
next_func();
|
|
}
|
|
|
|
void setup_arm64_ap(void (*next_func)(void))
|
|
{
|
|
/* set this core logical cpuid for struct thread_info */
|
|
assign_processor_id();
|
|
verify_cpu_run_el();
|
|
arch_counter_set_user_access();
|
|
cpuinfo_store_cpu();
|
|
hw_breakpoint_reset();
|
|
debug_monitors_init();
|
|
arch_timer_configure_evtstream();
|
|
init_cpu();
|
|
init_power_management();
|
|
call_ap_func(next_func);
|
|
|
|
/* BUG */
|
|
while(1);
|
|
}
|
|
|
|
void arch_show_interrupt_context(const void *reg);
|
|
extern void tlb_flush_handler(int vector);
|
|
|
|
static void show_context_stack(struct pt_regs *regs)
|
|
{
|
|
const int min_stack_frame_size = 0x10;
|
|
uintptr_t sp;
|
|
uintptr_t stack_top;
|
|
int max_loop;
|
|
int i;
|
|
|
|
if (interrupt_from_user(regs)) {
|
|
kprintf("It is a user stack region and it ends.\n");
|
|
return;
|
|
}
|
|
|
|
ihk_mc_debug_show_interrupt_context(regs);
|
|
|
|
sp = (uintptr_t)regs + sizeof(*regs);
|
|
stack_top = ALIGN_UP(sp, (uintptr_t)KERNEL_STACK_SIZE);
|
|
max_loop = (stack_top - sp) / min_stack_frame_size;
|
|
|
|
for (i = 0; i < max_loop; i++) {
|
|
extern char _head[], _end[];
|
|
uintptr_t *fp, *lr;
|
|
fp = (uintptr_t *)sp;
|
|
lr = (uintptr_t *)(sp + 8);
|
|
|
|
if ((*fp <= sp) || (*fp > stack_top)) {
|
|
break;
|
|
}
|
|
|
|
if ((*lr < (unsigned long)_head) ||
|
|
(*lr > (unsigned long)_end)) {
|
|
break;
|
|
}
|
|
|
|
kprintf("LR: %016lx, SP: %016lx, FP: %016lx\n", *lr, sp, *fp);
|
|
sp = *fp;
|
|
}
|
|
}
|
|
|
|
#ifdef ENABLE_FUGAKU_HACKS
|
|
void __show_context_stack(struct thread *thread,
|
|
unsigned long pc, uintptr_t sp, int kprintf_locked)
|
|
{
|
|
uintptr_t stack_top;
|
|
unsigned long irqflags = 0;
|
|
|
|
stack_top = ALIGN_UP(sp, (uintptr_t)KERNEL_STACK_SIZE);
|
|
|
|
if (!kprintf_locked)
|
|
irqflags = kprintf_lock();
|
|
|
|
__kprintf("TID: %d, call stack (most recent first):\n",
|
|
thread->tid);
|
|
__kprintf("PC: %016lx, SP: %016lx\n", pc, sp);
|
|
for (;;) {
|
|
extern char _head[], _end[];
|
|
uintptr_t *fp, *lr;
|
|
fp = (uintptr_t *)sp;
|
|
lr = (uintptr_t *)(sp + 8);
|
|
|
|
if ((*fp <= sp)) {
|
|
break;
|
|
}
|
|
|
|
if ((*fp > stack_top)) {
|
|
break;
|
|
}
|
|
|
|
if ((*lr < (unsigned long)_head) ||
|
|
(*lr > (unsigned long)_end)) {
|
|
break;
|
|
}
|
|
|
|
__kprintf("PC: %016lx, SP: %016lx, FP: %016lx\n", *lr - 4, sp, *fp);
|
|
sp = *fp;
|
|
}
|
|
|
|
if (!kprintf_locked)
|
|
kprintf_unlock(irqflags);
|
|
}
|
|
#endif
|
|
|
|
void handle_IPI(unsigned int vector, struct pt_regs *regs)
|
|
{
|
|
struct ihk_mc_interrupt_handler *h;
|
|
|
|
dkprintf("CPU[%d] got interrupt, vector: %d\n",
|
|
ihk_mc_get_processor_id(), vector);
|
|
|
|
if (vector > ((sizeof(handlers) / sizeof(handlers[0])) - 1)) {
|
|
panic("Maybe BUG.");
|
|
}
|
|
else if (vector == INTRID_STACK_TRACE) {
|
|
show_context_stack(regs);
|
|
}
|
|
else {
|
|
list_for_each_entry(h, &handlers[vector], list) {
|
|
if (h->func) {
|
|
h->func(h->priv == NULL ? regs : h->priv);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
static void __arm64_wakeup(int hw_cpuid, unsigned long entry)
|
|
{
|
|
if (cpu_psci_cpu_boot(hw_cpuid, entry)) {
|
|
panic("ap kickup cpu_psci_cpu_boot() failed.\n");
|
|
}
|
|
}
|
|
|
|
/** IHK Functions **/
|
|
|
|
/* send WFI(Wait For Interrupt) instruction */
|
|
static inline void cpu_do_idle(void)
|
|
{
|
|
extern void __cpu_do_idle(void);
|
|
uint64_t retention;
|
|
int state;
|
|
|
|
state = pwr_arm64hpc_retention_state_get(&retention);
|
|
if ((state == 0) && (retention != 0)) {
|
|
pwr_arm64hpc_enable_retention_state();
|
|
}
|
|
__cpu_do_idle();
|
|
}
|
|
|
|
/* halt by WFI(Wait For Interrupt) */
|
|
void cpu_halt(void)
|
|
{
|
|
cpu_do_idle();
|
|
}
|
|
|
|
/*@
|
|
@ assigns \nothing;
|
|
@ ensures \interrupt_disabled == 0;
|
|
@*/
|
|
void cpu_safe_halt(void)
|
|
{
|
|
cpu_do_idle();
|
|
cpu_enable_interrupt();
|
|
}
|
|
|
|
#ifdef ENABLE_FUGAKU_HACKS
|
|
/*@
|
|
@ assigns \nothing;
|
|
@ ensures \interrupt_disabled == 0;
|
|
@*/
|
|
void cpu_halt_panic(void)
|
|
{
|
|
extern void __cpu_do_idle(void);
|
|
cpu_enable_interrupt();
|
|
__cpu_do_idle();
|
|
}
|
|
#endif
|
|
|
|
#if defined(CONFIG_HAS_NMI)
|
|
#include <arm-gic-v3.h>
|
|
|
|
/* enable interrupt (ICC_PMR_EL1 <= ICC_PMR_EL1_UNMASKED) */
|
|
void cpu_enable_interrupt(void)
|
|
{
|
|
unsigned long unmasked = ICC_PMR_EL1_UNMASKED;
|
|
asm volatile(
|
|
"msr_s " __stringify(ICC_PMR_EL1) ",%0"
|
|
:
|
|
: "r" (unmasked)
|
|
: "memory");
|
|
}
|
|
|
|
/* disable interrupt (ICC_PMR_EL1 <= ICC_PMR_EL1_MASKED) */
|
|
void cpu_disable_interrupt(void)
|
|
{
|
|
unsigned long masked = ICC_PMR_EL1_MASKED;
|
|
asm volatile(
|
|
"msr_s " __stringify(ICC_PMR_EL1) ",%0"
|
|
:
|
|
: "r" (masked)
|
|
: "memory");
|
|
}
|
|
|
|
/* restore interrupt (ICC_PMR_EL1 <= flags) */
|
|
void cpu_restore_interrupt(unsigned long flags)
|
|
{
|
|
asm volatile(
|
|
"msr_s " __stringify(ICC_PMR_EL1) ",%0"
|
|
:
|
|
: "r" (flags)
|
|
: "memory");
|
|
}
|
|
|
|
/* save ICC_PMR_EL1 & disable interrupt (ICC_PMR_EL1 <= ICC_PMR_EL1_MASKED) */
|
|
unsigned long cpu_disable_interrupt_save(void)
|
|
{
|
|
unsigned long flags;
|
|
unsigned long masked = ICC_PMR_EL1_MASKED;
|
|
|
|
asm volatile(
|
|
"mrs_s %0, " __stringify(ICC_PMR_EL1) "\n"
|
|
"msr_s " __stringify(ICC_PMR_EL1) ",%1"
|
|
: "=&r" (flags)
|
|
: "r" (masked)
|
|
: "memory");
|
|
return flags;
|
|
}
|
|
|
|
/* save ICC_PMR_EL1 & enable interrupt (ICC_PMR_EL1 <= ICC_PMR_EL1_UNMASKED) */
|
|
unsigned long cpu_enable_interrupt_save(void)
|
|
{
|
|
unsigned long flags;
|
|
unsigned long masked = ICC_PMR_EL1_UNMASKED;
|
|
|
|
asm volatile(
|
|
"mrs_s %0, " __stringify(ICC_PMR_EL1) "\n"
|
|
"msr_s " __stringify(ICC_PMR_EL1) ",%1"
|
|
: "=&r" (flags)
|
|
: "r" (masked)
|
|
: "memory");
|
|
return flags;
|
|
}
|
|
|
|
#ifdef ENABLE_FUGAKU_HACKS
|
|
int cpu_interrupt_disabled(void)
|
|
{
|
|
unsigned long flags;
|
|
unsigned long masked = ICC_PMR_EL1_MASKED;
|
|
|
|
asm volatile(
|
|
"mrs_s %0, " __stringify(ICC_PMR_EL1)
|
|
: "=&r" (flags)
|
|
:
|
|
: "memory");
|
|
return (flags == masked);
|
|
}
|
|
#endif
|
|
|
|
#else /* defined(CONFIG_HAS_NMI) */
|
|
|
|
/* @ref.impl arch/arm64/include/asm/irqflags.h::arch_local_irq_enable */
|
|
/* enable interrupt (PSTATE.DAIF I bit clear) */
|
|
void cpu_enable_interrupt(void)
|
|
{
|
|
asm volatile(
|
|
"msr daifclr, #2 // arch_local_irq_enable"
|
|
:
|
|
:
|
|
: "memory");
|
|
}
|
|
|
|
/* @ref.impl arch/arm64/include/asm/irqflags.h::arch_local_irq_disable */
|
|
/* disable interrupt (PSTATE.DAIF I bit set) */
|
|
void cpu_disable_interrupt(void)
|
|
{
|
|
asm volatile(
|
|
"msr daifset, #2 // arch_local_irq_disable"
|
|
:
|
|
:
|
|
: "memory");
|
|
}
|
|
|
|
/* @ref.impl arch/arm64/include/asm/spinlock.h::arch_local_irq_restore */
|
|
/* restore interrupt (PSTATE.DAIF = flags restore) */
|
|
void cpu_restore_interrupt(unsigned long flags)
|
|
{
|
|
asm volatile(
|
|
"msr daif, %0 // arch_local_irq_restore"
|
|
:
|
|
: "r" (flags)
|
|
: "memory");
|
|
}
|
|
|
|
/* @ref.impl arch/arm64/include/asm/irqflags.h::arch_local_irq_save */
|
|
/* save PSTATE.DAIF & disable interrupt (PSTATE.DAIF I bit set) */
|
|
unsigned long cpu_disable_interrupt_save(void)
|
|
{
|
|
unsigned long flags;
|
|
asm volatile(
|
|
"mrs %0, daif // arch_local_irq_save\n"
|
|
"msr daifset, #2"
|
|
: "=r" (flags)
|
|
:
|
|
: "memory");
|
|
return flags;
|
|
}
|
|
|
|
/* save PSTATE.DAIF & enable interrupt (PSTATE.DAIF I bit set) */
|
|
unsigned long cpu_enable_interrupt_save(void)
|
|
{
|
|
unsigned long flags;
|
|
|
|
asm volatile(
|
|
"mrs %0, daif // arch_local_irq_save\n"
|
|
"msr daifclr, #2"
|
|
: "=r" (flags)
|
|
:
|
|
: "memory");
|
|
return flags;
|
|
}
|
|
#endif /* defined(CONFIG_HAS_NMI) */
|
|
|
|
/* we not have "pause" instruction, instead "yield" instruction */
|
|
void cpu_pause(void)
|
|
{
|
|
asm volatile("yield" ::: "memory");
|
|
}
|
|
|
|
static void cpu_init_interrupt_handler(void)
|
|
{
|
|
int i;
|
|
for (i = 0; i < (sizeof(handlers) / sizeof(handlers[0])); i++) {
|
|
INIT_LIST_HEAD(&handlers[i]);
|
|
}
|
|
}
|
|
|
|
/*@
|
|
@ behavior valid_vector:
|
|
@ assumes 0 <= vector <= 15;
|
|
@ requires \valid(h);
|
|
@ assigns handlers[vector-32];
|
|
@ ensures \result == 0;
|
|
@ behavior invalid_vector:
|
|
@ assumes (vector > 15);
|
|
@ assigns \nothing;
|
|
@ ensures \result == -EINVAL;
|
|
@*/
|
|
int ihk_mc_register_interrupt_handler(int vector,
|
|
struct ihk_mc_interrupt_handler *h)
|
|
{
|
|
if ((vector < 0) || (vector > ((sizeof(handlers) / sizeof(handlers[0])) - 1))) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
list_add_tail(&h->list, &handlers[vector]);
|
|
|
|
return 0;
|
|
}
|
|
int ihk_mc_unregister_interrupt_handler(int vector,
|
|
struct ihk_mc_interrupt_handler *h)
|
|
{
|
|
list_del(&h->list);
|
|
|
|
return 0;
|
|
}
|
|
|
|
extern unsigned long __page_fault_handler_address;
|
|
|
|
/*@
|
|
@ requires \valid(h);
|
|
@ assigns __page_fault_handler_address;
|
|
@ ensures __page_fault_handler_address == h;
|
|
@*/
|
|
void ihk_mc_set_page_fault_handler(void (*h)(void *, uint64_t, void *))
|
|
{
|
|
__page_fault_handler_address = (unsigned long)h;
|
|
}
|
|
|
|
extern char trampoline_code_data[], trampoline_code_data_end[];
|
|
unsigned long get_transit_page_table(void);
|
|
|
|
int get_virt_cpuid(int hw_cpuid)
|
|
{
|
|
int virt_cpuid = -1;
|
|
const struct ihk_mc_cpu_info *cpu_info;
|
|
int i;
|
|
|
|
cpu_info = ihk_mc_get_cpu_info();
|
|
for (i = 0; i < cpu_info->ncpus; i++) {
|
|
if (cpu_info->hw_ids[i] == hw_cpuid) {
|
|
virt_cpuid = i;
|
|
break;
|
|
}
|
|
}
|
|
return virt_cpuid;
|
|
}
|
|
|
|
/* reusable, but not reentrant */
|
|
/*@
|
|
@ requires \valid_apicid(cpuid); // valid APIC ID or not
|
|
@ requires \valid(pc);
|
|
@ requires \valid(trampoline_va);
|
|
@ requires \valid(trampoline_code_data
|
|
@ +(0..(trampoline_code_data_end - trampoline_code_data)));
|
|
@ requires \valid_physical(ap_trampoline); // valid physical address or not
|
|
@ assigns (char *)trampoline_va+(0..trampoline_code_data_end - trampoline_code_data);
|
|
@ assigns cpu_boot_status;
|
|
@ ensures cpu_boot_status != 0;
|
|
@*/
|
|
void ihk_mc_boot_cpu(int cpuid, unsigned long pc)
|
|
{
|
|
int virt_cpuid = get_virt_cpuid(cpuid);
|
|
extern void arch_ap_start();
|
|
extern int num_processors;
|
|
int ncpus;
|
|
|
|
/* virt cpuid check */
|
|
if (virt_cpuid == -1) {
|
|
panic("exchange failed, PHYSCPUID --> VIRTCPUID\n");
|
|
}
|
|
|
|
/* ap stack address set for secondary_data */
|
|
secondary_data.stack =
|
|
(void *)get_arm64_cpu_local_variable(virt_cpuid) + THREAD_START_SP - sizeof(ihk_mc_user_context_t);
|
|
|
|
/* next_pc address set for secondary_data (setup_arm64_ap) */
|
|
secondary_data.next_pc = (uint64_t)setup_arm64_ap;
|
|
|
|
/* next_pc argument set for secondary_data (ihk_mc_boot_cpu argument 2) */
|
|
secondary_data.arg = pc;
|
|
|
|
/* ap wait flag initialize */
|
|
cpu_boot_status = 0;
|
|
|
|
/* ap kickup */
|
|
__arm64_wakeup(cpuid, virt_to_phys(arch_ap_start));
|
|
|
|
/* wait for ap call call_ap_func() */
|
|
while (!cpu_boot_status) {
|
|
cpu_pause();
|
|
}
|
|
|
|
ncpus = ihk_mc_get_cpu_info()->ncpus;
|
|
if (ncpus - 1 <= num_processors) {
|
|
setup_cpu_features();
|
|
}
|
|
|
|
sve_setup();
|
|
}
|
|
|
|
/* for ihk_mc_init_context() */
|
|
extern void ret_from_fork(void);
|
|
|
|
/*@
|
|
@ requires \valid(new_ctx);
|
|
@ requires (stack_pointer == NULL) || \valid((unsigned long *)stack_pointer-1);
|
|
@ requires \valid(next_function);
|
|
@*/
|
|
/* initialize context */
|
|
/* stack_pointer == NULL is idle process context */
|
|
/* stack_pointer != NULL is user thread context */
|
|
void ihk_mc_init_context(ihk_mc_kernel_context_t *new_ctx,
|
|
void *stack_pointer, void (*next_function)(void))
|
|
{
|
|
unsigned long sp = 0;
|
|
ihk_mc_user_context_t *new_uctx = NULL;
|
|
|
|
if (unlikely(!stack_pointer)) {
|
|
/* for idle process */
|
|
/* get idle stack address */
|
|
sp = (unsigned long)get_arm64_this_cpu_kstack();
|
|
|
|
/* get thread_info address */
|
|
new_ctx->thread = (struct thread_info *)((unsigned long)ALIGN_DOWN(sp, KERNEL_STACK_SIZE));
|
|
|
|
/* set ret_from_fork address */
|
|
new_ctx->thread->cpu_context.pc = (unsigned long)ret_from_fork;
|
|
|
|
/* set idle address */
|
|
/* branch in ret_from_fork */
|
|
new_ctx->thread->cpu_context.x19 = (unsigned long)next_function;
|
|
|
|
sp -= 16;
|
|
new_ctx->thread->cpu_context.fp = sp;
|
|
|
|
/* set stack_pointer */
|
|
new_ctx->thread->cpu_context.sp = sp - sizeof(ihk_mc_user_context_t);
|
|
|
|
/* clear pt_regs area */
|
|
new_uctx = (ihk_mc_user_context_t *)new_ctx->thread->cpu_context.sp;
|
|
memset(new_uctx, 0, sizeof(ihk_mc_user_context_t));
|
|
|
|
/* set pt_regs->pstate */
|
|
new_uctx->pstate = PSR_MODE_EL1h;
|
|
} else {
|
|
/* for user thread, kernel stack */
|
|
/* save logical cpuid (for execve) */
|
|
const int lcpuid = ihk_mc_get_processor_id();
|
|
const unsigned long syscallno = current_pt_regs()->syscallno;
|
|
#ifdef CONFIG_ARM64_SVE
|
|
struct thread_info *ti = current_thread_info();
|
|
const unsigned int orig_sve_vl = ti->sve_vl;
|
|
const unsigned int orig_sve_vl_onexec = ti->sve_vl_onexec;
|
|
const unsigned long orig_sve_flags = ti->sve_flags;
|
|
#endif /* CONFIG_ARM64_SVE */
|
|
|
|
/* get kernel stack address */
|
|
sp = (unsigned long)stack_pointer;
|
|
|
|
/* get thread_info address */
|
|
new_ctx->thread = (struct thread_info *)((unsigned long)ALIGN_DOWN(sp, KERNEL_STACK_SIZE));
|
|
|
|
/* clear thread_info */
|
|
memset(new_ctx->thread, 0, sizeof(struct thread_info));
|
|
|
|
/* restore logical cpuid (for execve) */
|
|
new_ctx->thread->cpu = lcpuid;
|
|
|
|
/* set ret_from_fork address */
|
|
new_ctx->thread->cpu_context.pc = (unsigned long)ret_from_fork;
|
|
|
|
/* set stack_pointer */
|
|
new_ctx->thread->cpu_context.sp = sp;
|
|
/* use the 16 bytes padding in ihk_mc_init_user_process()
|
|
* as closing frame in the frame chain */
|
|
new_ctx->thread->cpu_context.fp = sp + sizeof(ihk_mc_user_context_t);
|
|
|
|
/* clear pt_regs area */
|
|
new_uctx = (ihk_mc_user_context_t *)new_ctx->thread->cpu_context.sp;
|
|
memset(new_uctx, 0, sizeof(ihk_mc_user_context_t));
|
|
|
|
/* initialize user context */
|
|
/* copy from current_pt_regs */
|
|
*new_uctx = *((ihk_mc_user_context_t *)current_pt_regs());
|
|
new_uctx->regs[0] = 0;
|
|
new_uctx->pc = (unsigned long)next_function;
|
|
new_uctx->pstate = (new_uctx->pstate & ~PSR_MODE_MASK) | PSR_MODE_EL0t;
|
|
|
|
#ifdef CONFIG_ARM64_SVE
|
|
/* SVE-VL inherit */
|
|
if (likely(elf_hwcap & HWCAP_SVE)) {
|
|
new_ctx->thread->sve_vl_onexec = orig_sve_vl_onexec;
|
|
new_ctx->thread->sve_flags = orig_sve_flags;
|
|
|
|
if (syscallno == __NR_execve) {
|
|
new_ctx->thread->sve_vl = orig_sve_vl_onexec ?
|
|
orig_sve_vl_onexec : sve_default_vl;
|
|
|
|
BUG_ON(!sve_vl_valid(new_ctx->thread->sve_vl));
|
|
|
|
if (!(new_ctx->thread->sve_flags & THREAD_VL_INHERIT)) {
|
|
new_ctx->thread->sve_vl_onexec = 0;
|
|
}
|
|
} else {
|
|
new_ctx->thread->sve_vl = orig_sve_vl ?
|
|
orig_sve_vl : sve_default_vl;
|
|
}
|
|
}
|
|
#endif /* CONFIG_ARM64_SVE */
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Release runq_lock before entering user space.
|
|
* This is needed because schedule() holds the runq lock throughout
|
|
* the context switch and when a new process is created it starts
|
|
* execution in enter_user_mode, which in turn calls this function.
|
|
*/
|
|
void release_runq_lock(void)
|
|
{
|
|
ihk_mc_spinlock_unlock(&(cpu_local_var(runq_lock)),
|
|
cpu_local_var(runq_irqstate));
|
|
}
|
|
|
|
/*@
|
|
@ requires \valid(ctx);
|
|
@ requires \valid(puctx);
|
|
@ requires \valid((ihk_mc_user_context_t *)stack_pointer-1);
|
|
@ requires \valid_user(new_pc); // valid user space address or not
|
|
@ requires \valid_user(user_sp-1);
|
|
@ assigns *((ihk_mc_user_context_t *)stack_pointer-1);
|
|
@ assigns ctx->rsp0;
|
|
@*/
|
|
void ihk_mc_init_user_process(ihk_mc_kernel_context_t *ctx,
|
|
ihk_mc_user_context_t **puctx,
|
|
void *stack_pointer, unsigned long new_pc,
|
|
unsigned long user_sp)
|
|
{
|
|
char *sp = NULL;
|
|
|
|
/* calc aligned kernel stack address */
|
|
/* higher 16 byte area is padding area */
|
|
sp = (char *)(ALIGN_DOWN((unsigned long)stack_pointer, KERNEL_STACK_SIZE) - 16);
|
|
|
|
/* get pt_regs address */
|
|
sp -= sizeof(ihk_mc_user_context_t);
|
|
|
|
/* puctx return value set */
|
|
*puctx = (ihk_mc_user_context_t *)sp;
|
|
|
|
/* initialize kernel context */
|
|
ihk_mc_init_context(ctx, sp, (void (*)(void))new_pc);
|
|
}
|
|
|
|
/*@
|
|
@ behavior rsp:
|
|
@ assumes reg == IHK_UCR_STACK_POINTER;
|
|
@ requires \valid(uctx);
|
|
@ assigns uctx->gpr.rsp;
|
|
@ ensures uctx->gpr.rsp == value;
|
|
@ behavior rip:
|
|
@ assumes reg == IHK_UCR_PROGRAM_COUNTER;
|
|
@ requires \valid(uctx);
|
|
@ assigns uctx->gpr.rip;
|
|
@ ensures uctx->gpr.rip == value;
|
|
@*/
|
|
void ihk_mc_modify_user_context(ihk_mc_user_context_t *uctx,
|
|
enum ihk_mc_user_context_regtype reg,
|
|
unsigned long value)
|
|
{
|
|
if (reg == IHK_UCR_STACK_POINTER) {
|
|
if (value & 15) {
|
|
panic("User Stack Pointer Unaligned !!\n");
|
|
}
|
|
uctx->sp = value;
|
|
} else if (reg == IHK_UCR_PROGRAM_COUNTER) {
|
|
uctx->pc = value;
|
|
}
|
|
}
|
|
|
|
/* @ref.impl arch/arm64/kernel/setup.c::hwcap_str */
|
|
static const char *const hwcap_str[] = {
|
|
"fp",
|
|
"asimd",
|
|
"evtstrm",
|
|
"aes",
|
|
"pmull",
|
|
"sha1",
|
|
"sha2",
|
|
"crc32",
|
|
"atomics",
|
|
"fphp",
|
|
"asimdhp",
|
|
"cpuid",
|
|
"asimdrdm",
|
|
"sve",
|
|
NULL
|
|
};
|
|
|
|
#define CPUINFO_LEN_PER_CORE 0x100
|
|
long ihk_mc_show_cpuinfo(char *buf, size_t buf_size, unsigned long read_off, int *eofp)
|
|
{
|
|
extern int num_processors;
|
|
int i = 0;
|
|
char *lbuf = NULL;
|
|
const size_t lbuf_size = CPUINFO_LEN_PER_CORE * num_processors;
|
|
size_t loff = 0;
|
|
long ret = 0;
|
|
|
|
/* eof flag initialization */
|
|
*eofp = 0;
|
|
|
|
/* offset is over lbuf_size, return */
|
|
if (read_off >= lbuf_size) {
|
|
*eofp = 1;
|
|
return 0;
|
|
}
|
|
|
|
/* local buffer allocate */
|
|
lbuf = kmalloc(lbuf_size, IHK_MC_AP_NOWAIT);
|
|
if (lbuf == NULL) {
|
|
ekprintf("%s: ERROR Local buffer allocation failed.\n");
|
|
ret = -ENOMEM;
|
|
*eofp = 1;
|
|
goto err;
|
|
}
|
|
memset(lbuf, '\0', lbuf_size);
|
|
|
|
/* cpuinfo strings generate and copy */
|
|
for (i = 0; i < num_processors; i++) {
|
|
const struct cpuinfo_arm64 *cpuinfo = &cpuinfo_data[i];
|
|
const unsigned int midr = cpuinfo->reg_midr;
|
|
int j = 0;
|
|
|
|
/* generate strings */
|
|
loff += scnprintf(lbuf + loff, lbuf_size - loff,
|
|
"processor\t: %d\n", i);
|
|
loff += scnprintf(lbuf + loff, lbuf_size - loff, "Features\t:");
|
|
|
|
for (j = 0; hwcap_str[j]; j++) {
|
|
if (elf_hwcap & (1 << j)) {
|
|
loff += scnprintf(lbuf + loff,
|
|
lbuf_size - loff,
|
|
" %s", hwcap_str[j]);
|
|
}
|
|
}
|
|
loff += scnprintf(lbuf + loff, lbuf_size - loff, "\n");
|
|
loff += scnprintf(lbuf + loff, lbuf_size - loff,
|
|
"CPU implementer\t: 0x%02x\n",
|
|
MIDR_IMPLEMENTOR(midr));
|
|
loff += scnprintf(lbuf + loff, lbuf_size - loff,
|
|
"CPU architecture: 8\n");
|
|
loff += scnprintf(lbuf + loff, lbuf_size - loff,
|
|
"CPU variant\t: 0x%x\n",
|
|
MIDR_VARIANT(midr));
|
|
loff += scnprintf(lbuf + loff, lbuf_size - loff,
|
|
"CPU part\t: 0x%03x\n",
|
|
MIDR_PARTNUM(midr));
|
|
loff += scnprintf(lbuf + loff, lbuf_size - loff,
|
|
"CPU revision\t: %d\n\n",
|
|
MIDR_REVISION(midr));
|
|
|
|
/* check buffer depletion */
|
|
if ((i < num_processors - 1) && ((lbuf_size - loff) == 1)) {
|
|
ekprintf("%s: ERROR Local buffer size shortage.\n", __FUNCTION__);
|
|
ret = -ENOMEM;
|
|
*eofp = 1;
|
|
goto err_free;
|
|
}
|
|
}
|
|
|
|
/* copy to host buffer */
|
|
memcpy(buf, lbuf + read_off, buf_size);
|
|
if (read_off + buf_size >= loff) {
|
|
*eofp = 1;
|
|
ret = loff - read_off;
|
|
} else {
|
|
ret = buf_size;
|
|
}
|
|
|
|
err_free:
|
|
kfree(lbuf);
|
|
err:
|
|
return ret;
|
|
}
|
|
|
|
static int check_and_allocate_fp_regs(struct thread *thread);
|
|
|
|
void arch_clone_thread(struct thread *othread, unsigned long pc,
|
|
unsigned long sp, struct thread *nthread)
|
|
{
|
|
unsigned long tls = 0;
|
|
|
|
/* get tpidr_el0 value, and set original-thread->tlsblock_base, new-thread->tlsblock_base */
|
|
asm("mrs %0, tpidr_el0" : "=r" (tls));
|
|
othread->tlsblock_base = nthread->tlsblock_base = tls;
|
|
|
|
/* if SVE enable, takeover lower 128 bit register */
|
|
if (likely(elf_hwcap & HWCAP_SVE)) {
|
|
fp_regs_struct fp_regs;
|
|
|
|
memset(&fp_regs, 0, sizeof(fp_regs_struct));
|
|
fpsimd_save_state(&fp_regs);
|
|
thread_fpsimd_to_sve(nthread, &fp_regs);
|
|
}
|
|
}
|
|
|
|
/*@
|
|
@ requires \valid(handler);
|
|
@ assigns __arm64_syscall_handler;
|
|
@ ensures __arm64_syscall_handler == handler;
|
|
@*/
|
|
void ihk_mc_set_syscall_handler(long (*handler)(int, ihk_mc_user_context_t *))
|
|
{
|
|
__arm64_syscall_handler = handler;
|
|
}
|
|
|
|
/*@
|
|
@ assigns \nothing;
|
|
@*/
|
|
void ihk_mc_delay_us(int us)
|
|
{
|
|
arch_delay(us);
|
|
}
|
|
|
|
void arch_print_stack(void)
|
|
{
|
|
}
|
|
|
|
#ifdef ENABLE_FUGAKU_HACKS
|
|
unsigned long arch_get_instruction_address(const void *reg)
|
|
{
|
|
const struct pt_regs *regs = (struct pt_regs *)reg;
|
|
return regs->pc;
|
|
}
|
|
#endif
|
|
|
|
void arch_show_interrupt_context(const void *reg)
|
|
{
|
|
const struct pt_regs *regs = (struct pt_regs *)reg;
|
|
kprintf("dump pt_regs:\n");
|
|
kprintf(" x0 : %016lx x1 : %016lx x2 : %016lx x3 : %016lx\n",
|
|
regs->regs[0], regs->regs[1], regs->regs[2], regs->regs[3]);
|
|
kprintf(" x4 : %016lx x5 : %016lx x6 : %016lx x7 : %016lx\n",
|
|
regs->regs[4], regs->regs[5], regs->regs[6], regs->regs[7]);
|
|
kprintf(" x8 : %016lx x9 : %016lx x10 : %016lx x11 : %016lx\n",
|
|
regs->regs[8], regs->regs[9], regs->regs[10], regs->regs[11]);
|
|
kprintf(" x12 : %016lx x13 : %016lx x14 : %016lx x15 : %016lx\n",
|
|
regs->regs[12], regs->regs[13], regs->regs[14], regs->regs[15]);
|
|
kprintf(" x16 : %016lx x17 : %016lx x18 : %016lx x19 : %016lx\n",
|
|
regs->regs[16], regs->regs[17], regs->regs[18], regs->regs[19]);
|
|
kprintf(" x20 : %016lx x21 : %016lx x22 : %016lx x23 : %016lx\n",
|
|
regs->regs[20], regs->regs[21], regs->regs[22], regs->regs[23]);
|
|
kprintf(" x24 : %016lx x25 : %016lx x26 : %016lx x27 : %016lx\n",
|
|
regs->regs[24], regs->regs[25], regs->regs[26], regs->regs[27]);
|
|
kprintf(" x28 : %016lx x29 : %016lx x30 : %016lx\n",
|
|
regs->regs[28], regs->regs[29], regs->regs[30]);
|
|
kprintf(" sp : %016lx\n", regs->sp);
|
|
kprintf(" pc : %016lx\n", regs->pc);
|
|
kprintf(" pstate : %016lx(N:%d Z:%d C:%d V:%d SS:%d IL:%d D:%d A:%d I:%d F:%d M[4]:%d M:%d)\n",
|
|
regs->pstate,
|
|
(regs->pstate >> 31 & 1), (regs->pstate >> 30 & 1), (regs->pstate >> 29 & 1),
|
|
(regs->pstate >> 28 & 1), (regs->pstate >> 21 & 1), (regs->pstate >> 20 & 1),
|
|
(regs->pstate >> 9 & 1), (regs->pstate >> 8 & 1), (regs->pstate >> 7 & 1),
|
|
(regs->pstate >> 6 & 1), (regs->pstate >> 4 & 1), (regs->pstate & 7));
|
|
kprintf(" orig_x0 : %016lx\n", regs->orig_x0);
|
|
kprintf(" syscallno : %016lx\n", regs->syscallno);
|
|
}
|
|
|
|
void arch_cpu_stop(void)
|
|
{
|
|
psci_cpu_off();
|
|
}
|
|
|
|
/*@
|
|
@ behavior fs_base:
|
|
@ assumes type == IHK_ASR_X86_FS;
|
|
@ ensures \result == 0;
|
|
@ behavior invaiid_type:
|
|
@ assumes type != IHK_ASR_X86_FS;
|
|
@ ensures \result == -EINVAL;
|
|
@*/
|
|
int ihk_mc_arch_set_special_register(enum ihk_asr_type type,
|
|
unsigned long value)
|
|
{
|
|
/* TODO(pka_idle) */
|
|
return -1;
|
|
}
|
|
|
|
/*@
|
|
@ behavior fs_base:
|
|
@ assumes type == IHK_ASR_X86_FS;
|
|
@ requires \valid(value);
|
|
@ ensures \result == 0;
|
|
@ behavior invalid_type:
|
|
@ assumes type != IHK_ASR_X86_FS;
|
|
@ ensures \result == -EINVAL;
|
|
@*/
|
|
int ihk_mc_arch_get_special_register(enum ihk_asr_type type,
|
|
unsigned long *value)
|
|
{
|
|
/* TODO(pka_idle) */
|
|
return -1;
|
|
}
|
|
|
|
int ihk_mc_get_interrupt_id(int cpu)
|
|
{
|
|
return cpu;
|
|
}
|
|
|
|
/*@
|
|
@ requires \valid_cpuid(cpu); // valid CPU logical ID
|
|
@ ensures \result == 0
|
|
@*/
|
|
int ihk_mc_interrupt_cpu(int cpu, int vector)
|
|
{
|
|
if (cpu < 0 || cpu >= num_processors) {
|
|
kprintf("%s: invalid CPU id: %d\n", __func__, cpu);
|
|
return -1;
|
|
}
|
|
dkprintf("[%d] ihk_mc_interrupt_cpu: %d\n", ihk_mc_get_processor_id(), cpu);
|
|
(*arm64_issue_ipi)(cpu, vector);
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* @ref.impl linux-linaro/arch/arm64/kernel/process.c::tls_thread_switch()
|
|
*/
|
|
static void tls_thread_switch(struct thread *prev, struct thread *next)
|
|
{
|
|
unsigned long tpidr, tpidrro;
|
|
|
|
asm("mrs %0, tpidr_el0" : "=r" (tpidr));
|
|
prev->tlsblock_base = tpidr;
|
|
|
|
tpidr = next->tlsblock_base;
|
|
tpidrro = 0;
|
|
|
|
asm(
|
|
" msr tpidr_el0, %0\n"
|
|
" msr tpidrro_el0, %1"
|
|
: : "r" (tpidr), "r" (tpidrro));
|
|
}
|
|
|
|
struct thread *arch_switch_context(struct thread *prev, struct thread *next)
|
|
{
|
|
// TODO[PMU]: 暫定的にここに関数宣言を置く。共通部のヘッダに書くのが作法だと思うが、今後の動向を様子見。
|
|
extern void perf_start(struct mc_perf_event *event);
|
|
extern void perf_reset(struct mc_perf_event *event);
|
|
struct thread *last;
|
|
struct mcs_rwlock_node_irqsave lock;
|
|
|
|
/* Set up new TLS.. */
|
|
dkprintf("[%d] arch_switch_context: tlsblock_base: 0x%lX\n",
|
|
ihk_mc_get_processor_id(), next->tlsblock_base);
|
|
|
|
#ifdef ENABLE_PERF
|
|
/* Performance monitoring inherit */
|
|
if(next->proc->monitoring_event) {
|
|
if(next->proc->perf_status == PP_RESET)
|
|
perf_reset(next->proc->monitoring_event);
|
|
if(next->proc->perf_status != PP_COUNT) {
|
|
perf_reset(next->proc->monitoring_event);
|
|
perf_start(next->proc->monitoring_event);
|
|
}
|
|
}
|
|
#endif /*ENABLE_PERF*/
|
|
|
|
#ifdef PROFILE_ENABLE
|
|
if (prev && prev->profile && prev->profile_start_ts != 0) {
|
|
prev->profile_elapsed_ts +=
|
|
(rdtsc() - prev->profile_start_ts);
|
|
prev->profile_start_ts = 0;
|
|
}
|
|
|
|
if (next->profile && next->profile_start_ts == 0) {
|
|
next->profile_start_ts = rdtsc();
|
|
}
|
|
#endif
|
|
|
|
if (likely(prev)) {
|
|
tls_thread_switch(prev, next);
|
|
|
|
mcs_rwlock_writer_lock(&prev->proc->update_lock, &lock);
|
|
if (prev->proc->status & (PS_DELAY_STOPPED | PS_DELAY_TRACED)) {
|
|
switch (prev->proc->status) {
|
|
case PS_DELAY_STOPPED:
|
|
prev->proc->status = PS_STOPPED;
|
|
break;
|
|
case PS_DELAY_TRACED:
|
|
prev->proc->status = PS_TRACED;
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
mcs_rwlock_writer_unlock(&prev->proc->update_lock, &lock);
|
|
|
|
/* Wake up the parent who tried wait4 and sleeping */
|
|
waitq_wakeup(&prev->proc->parent->waitpid_q);
|
|
} else {
|
|
mcs_rwlock_writer_unlock(&prev->proc->update_lock, &lock);
|
|
}
|
|
|
|
last = ihk_mc_switch_context(&prev->ctx, &next->ctx, prev);
|
|
}
|
|
else {
|
|
last = ihk_mc_switch_context(NULL, &next->ctx, prev);
|
|
}
|
|
|
|
return last;
|
|
}
|
|
|
|
/*@
|
|
@ requires \valid(thread);
|
|
@ ensures thread->fp_regs == NULL;
|
|
@*/
|
|
void
|
|
release_fp_regs(struct thread *thread)
|
|
{
|
|
if (!thread) {
|
|
return;
|
|
}
|
|
|
|
if (likely(elf_hwcap & (HWCAP_FP | HWCAP_ASIMD))) {
|
|
int pages;
|
|
|
|
if (thread->fp_regs) {
|
|
// calcurate number of pages for fp regs area
|
|
pages = (sizeof(fp_regs_struct) + PAGE_SIZE -1) >> PAGE_SHIFT;
|
|
ihk_mc_free_pages(thread->fp_regs, pages);
|
|
thread->fp_regs = NULL;
|
|
}
|
|
|
|
#ifdef CONFIG_ARM64_SVE
|
|
if (likely(elf_hwcap & HWCAP_SVE)) {
|
|
sve_free(thread);
|
|
}
|
|
#endif /* CONFIG_ARM64_SVE */
|
|
}
|
|
}
|
|
|
|
static int
|
|
check_and_allocate_fp_regs(struct thread *thread)
|
|
{
|
|
int result = 0;
|
|
int pages;
|
|
|
|
if (!thread->fp_regs) {
|
|
pages = (sizeof(fp_regs_struct) + PAGE_SIZE -1) >> PAGE_SHIFT;
|
|
thread->fp_regs = ihk_mc_alloc_pages(pages, IHK_MC_AP_NOWAIT);
|
|
|
|
if (!thread->fp_regs) {
|
|
kprintf("error: allocating fp_regs pages\n");
|
|
result = -ENOMEM;
|
|
goto out;
|
|
}
|
|
|
|
memset(thread->fp_regs, 0, sizeof(fp_regs_struct));
|
|
}
|
|
|
|
#ifdef CONFIG_ARM64_SVE
|
|
if (likely(elf_hwcap & HWCAP_SVE)) {
|
|
result = sve_alloc(thread);
|
|
}
|
|
#endif /* CONFIG_ARM64_SVE */
|
|
out:
|
|
if (result) {
|
|
release_fp_regs(thread);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
/*@
|
|
@ requires \valid(thread);
|
|
@*/
|
|
int
|
|
save_fp_regs(struct thread *thread)
|
|
{
|
|
int ret = 0;
|
|
if (thread == &cpu_local_var(idle)) {
|
|
goto out;
|
|
}
|
|
|
|
if (likely(elf_hwcap & (HWCAP_FP | HWCAP_ASIMD))) {
|
|
ret = check_and_allocate_fp_regs(thread);
|
|
if (ret) {
|
|
goto out;
|
|
}
|
|
thread_fpsimd_save(thread);
|
|
}
|
|
out:
|
|
return ret;
|
|
}
|
|
|
|
int copy_fp_regs(struct thread *from, struct thread *to)
|
|
{
|
|
int ret = 0;
|
|
|
|
if (from->fp_regs != NULL) {
|
|
ret = check_and_allocate_fp_regs(to);
|
|
if (!ret) {
|
|
memcpy(to->fp_regs,
|
|
from->fp_regs,
|
|
sizeof(fp_regs_struct));
|
|
}
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
void clear_fp_regs(void)
|
|
{
|
|
if (likely(elf_hwcap & (HWCAP_FP | HWCAP_ASIMD))) {
|
|
#ifdef CONFIG_ARM64_SVE
|
|
if (likely(elf_hwcap & HWCAP_SVE)) {
|
|
unsigned int fpscr[2] = { 0, 0 };
|
|
unsigned int vl = current_thread_info()->sve_vl;
|
|
struct fpsimd_sve_state(sve_vq_from_vl(sve_max_vl)) clear_sve;
|
|
|
|
if (vl == 0) {
|
|
vl = sve_default_vl;
|
|
}
|
|
memset(&clear_sve, 0, sizeof(clear_sve));
|
|
sve_load_state(clear_sve.ffr, fpscr, sve_vq_from_vl(vl) - 1);
|
|
} else {
|
|
fp_regs_struct clear_fp;
|
|
memset(&clear_fp, 0, sizeof(fp_regs_struct));
|
|
fpsimd_load_state(&clear_fp);
|
|
}
|
|
#else /* CONFIG_ARM64_SVE */
|
|
fp_regs_struct clear_fp;
|
|
memset(&clear_fp, 0, sizeof(fp_regs_struct));
|
|
fpsimd_load_state(&clear_fp);
|
|
#endif /* CONFIG_ARM64_SVE */
|
|
}
|
|
}
|
|
|
|
/*@
|
|
@ requires \valid(thread);
|
|
@ assigns thread->fp_regs;
|
|
@*/
|
|
void
|
|
restore_fp_regs(struct thread *thread)
|
|
{
|
|
if (likely(elf_hwcap & (HWCAP_FP | HWCAP_ASIMD))) {
|
|
if (!thread->fp_regs) {
|
|
// only clear fpregs.
|
|
clear_fp_regs();
|
|
return;
|
|
}
|
|
thread_fpsimd_load(thread);
|
|
}
|
|
}
|
|
|
|
void init_tick(void)
|
|
{
|
|
dkprintf("init_tick():\n");
|
|
return;
|
|
}
|
|
|
|
void init_delay(void)
|
|
{
|
|
dkprintf("init_delay():\n");
|
|
return;
|
|
}
|
|
|
|
void sync_tick(void)
|
|
{
|
|
dkprintf("sync_tick():\n");
|
|
return;
|
|
}
|
|
|
|
void arch_start_pvclock(void)
|
|
{
|
|
/* linux-linaro(aarch64)ではKVM向けpvclockの処理が未サポート */
|
|
dkprintf("arch_start_pvclock(): not supported\n");
|
|
return;
|
|
}
|
|
|
|
void
|
|
mod_nmi_ctx(void *nmi_ctx, void (*func)())
|
|
{
|
|
func();
|
|
}
|
|
|
|
extern void freeze(void);
|
|
void __freeze(void)
|
|
{
|
|
freeze();
|
|
}
|
|
|
|
|
|
#define SYSREG_READ_S(sys_reg) case (sys_reg): asm volatile("mrs_s %0, " __stringify(sys_reg) : "=r" (*val)); break
|
|
static inline int arch_cpu_mrs(uint32_t sys_reg, uint64_t *val)
|
|
{
|
|
int ret = 0;
|
|
|
|
switch (sys_reg) {
|
|
SYSREG_READ_S(IMP_FJ_TAG_ADDRESS_CTRL_EL1);
|
|
SYSREG_READ_S(IMP_SCCR_CTRL_EL1);
|
|
SYSREG_READ_S(IMP_SCCR_ASSIGN_EL1);
|
|
SYSREG_READ_S(IMP_SCCR_SET0_L2_EL1);
|
|
SYSREG_READ_S(IMP_SCCR_SET1_L2_EL1);
|
|
SYSREG_READ_S(IMP_SCCR_L1_EL0);
|
|
SYSREG_READ_S(IMP_PF_CTRL_EL1);
|
|
SYSREG_READ_S(IMP_PF_STREAM_DETECT_CTRL_EL0);
|
|
SYSREG_READ_S(IMP_PF_INJECTION_CTRL0_EL0);
|
|
SYSREG_READ_S(IMP_PF_INJECTION_CTRL1_EL0);
|
|
SYSREG_READ_S(IMP_PF_INJECTION_CTRL2_EL0);
|
|
SYSREG_READ_S(IMP_PF_INJECTION_CTRL3_EL0);
|
|
SYSREG_READ_S(IMP_PF_INJECTION_CTRL4_EL0);
|
|
SYSREG_READ_S(IMP_PF_INJECTION_CTRL5_EL0);
|
|
SYSREG_READ_S(IMP_PF_INJECTION_CTRL6_EL0);
|
|
SYSREG_READ_S(IMP_PF_INJECTION_CTRL7_EL0);
|
|
SYSREG_READ_S(IMP_PF_INJECTION_DISTANCE0_EL0);
|
|
SYSREG_READ_S(IMP_PF_INJECTION_DISTANCE1_EL0);
|
|
SYSREG_READ_S(IMP_PF_INJECTION_DISTANCE2_EL0);
|
|
SYSREG_READ_S(IMP_PF_INJECTION_DISTANCE3_EL0);
|
|
SYSREG_READ_S(IMP_PF_INJECTION_DISTANCE4_EL0);
|
|
SYSREG_READ_S(IMP_PF_INJECTION_DISTANCE5_EL0);
|
|
SYSREG_READ_S(IMP_PF_INJECTION_DISTANCE6_EL0);
|
|
SYSREG_READ_S(IMP_PF_INJECTION_DISTANCE7_EL0);
|
|
SYSREG_READ_S(IMP_PF_PMUSERENR_EL0);
|
|
SYSREG_READ_S(IMP_BARRIER_CTRL_EL1);
|
|
SYSREG_READ_S(IMP_BARRIER_BST_BIT_EL1);
|
|
SYSREG_READ_S(IMP_BARRIER_INIT_SYNC_BB0_EL1);
|
|
SYSREG_READ_S(IMP_BARRIER_INIT_SYNC_BB1_EL1);
|
|
SYSREG_READ_S(IMP_BARRIER_INIT_SYNC_BB2_EL1);
|
|
SYSREG_READ_S(IMP_BARRIER_INIT_SYNC_BB3_EL1);
|
|
SYSREG_READ_S(IMP_BARRIER_INIT_SYNC_BB4_EL1);
|
|
SYSREG_READ_S(IMP_BARRIER_INIT_SYNC_BB5_EL1);
|
|
SYSREG_READ_S(IMP_BARRIER_ASSIGN_SYNC_W0_EL1);
|
|
SYSREG_READ_S(IMP_BARRIER_ASSIGN_SYNC_W1_EL1);
|
|
SYSREG_READ_S(IMP_BARRIER_ASSIGN_SYNC_W2_EL1);
|
|
SYSREG_READ_S(IMP_BARRIER_ASSIGN_SYNC_W3_EL1);
|
|
SYSREG_READ_S(IMP_SOC_STANDBY_CTRL_EL1);
|
|
SYSREG_READ_S(IMP_FJ_CORE_UARCH_CTRL_EL2);
|
|
SYSREG_READ_S(IMP_FJ_CORE_UARCH_RESTRECTION_EL1);
|
|
/* fall through */
|
|
default:
|
|
return -EINVAL;
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
static inline int arch_cpu_read_register(struct ihk_os_cpu_register *desc)
|
|
{
|
|
int ret = 0;
|
|
uint64_t value;
|
|
|
|
if (desc->addr) {
|
|
panic("memory mapped register is not supported.\n");
|
|
} else if (desc->addr_ext) {
|
|
ret = arch_cpu_mrs(desc->addr_ext, &value);
|
|
if (ret == 0) {
|
|
desc->val = value;
|
|
}
|
|
} else {
|
|
ret = -EINVAL;
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
#define SYSREG_WRITE_S(sys_reg) case (sys_reg): asm volatile("msr_s " __stringify(sys_reg) ", %0" :: "r" (val)); break
|
|
static inline int arch_cpu_msr(uint32_t sys_reg, uint64_t val)
|
|
{
|
|
int ret = 0;
|
|
|
|
switch (sys_reg) {
|
|
SYSREG_WRITE_S(IMP_FJ_TAG_ADDRESS_CTRL_EL1);
|
|
SYSREG_WRITE_S(IMP_SCCR_CTRL_EL1);
|
|
SYSREG_WRITE_S(IMP_SCCR_ASSIGN_EL1);
|
|
SYSREG_WRITE_S(IMP_SCCR_SET0_L2_EL1);
|
|
SYSREG_WRITE_S(IMP_SCCR_SET1_L2_EL1);
|
|
SYSREG_WRITE_S(IMP_SCCR_L1_EL0);
|
|
SYSREG_WRITE_S(IMP_PF_CTRL_EL1);
|
|
SYSREG_WRITE_S(IMP_PF_STREAM_DETECT_CTRL_EL0);
|
|
SYSREG_WRITE_S(IMP_PF_INJECTION_CTRL0_EL0);
|
|
SYSREG_WRITE_S(IMP_PF_INJECTION_CTRL1_EL0);
|
|
SYSREG_WRITE_S(IMP_PF_INJECTION_CTRL2_EL0);
|
|
SYSREG_WRITE_S(IMP_PF_INJECTION_CTRL3_EL0);
|
|
SYSREG_WRITE_S(IMP_PF_INJECTION_CTRL4_EL0);
|
|
SYSREG_WRITE_S(IMP_PF_INJECTION_CTRL5_EL0);
|
|
SYSREG_WRITE_S(IMP_PF_INJECTION_CTRL6_EL0);
|
|
SYSREG_WRITE_S(IMP_PF_INJECTION_CTRL7_EL0);
|
|
SYSREG_WRITE_S(IMP_PF_INJECTION_DISTANCE0_EL0);
|
|
SYSREG_WRITE_S(IMP_PF_INJECTION_DISTANCE1_EL0);
|
|
SYSREG_WRITE_S(IMP_PF_INJECTION_DISTANCE2_EL0);
|
|
SYSREG_WRITE_S(IMP_PF_INJECTION_DISTANCE3_EL0);
|
|
SYSREG_WRITE_S(IMP_PF_INJECTION_DISTANCE4_EL0);
|
|
SYSREG_WRITE_S(IMP_PF_INJECTION_DISTANCE5_EL0);
|
|
SYSREG_WRITE_S(IMP_PF_INJECTION_DISTANCE6_EL0);
|
|
SYSREG_WRITE_S(IMP_PF_INJECTION_DISTANCE7_EL0);
|
|
SYSREG_WRITE_S(IMP_PF_PMUSERENR_EL0);
|
|
SYSREG_WRITE_S(IMP_BARRIER_CTRL_EL1);
|
|
SYSREG_WRITE_S(IMP_BARRIER_BST_BIT_EL1);
|
|
SYSREG_WRITE_S(IMP_BARRIER_INIT_SYNC_BB0_EL1);
|
|
SYSREG_WRITE_S(IMP_BARRIER_INIT_SYNC_BB1_EL1);
|
|
SYSREG_WRITE_S(IMP_BARRIER_INIT_SYNC_BB2_EL1);
|
|
SYSREG_WRITE_S(IMP_BARRIER_INIT_SYNC_BB3_EL1);
|
|
SYSREG_WRITE_S(IMP_BARRIER_INIT_SYNC_BB4_EL1);
|
|
SYSREG_WRITE_S(IMP_BARRIER_INIT_SYNC_BB5_EL1);
|
|
SYSREG_WRITE_S(IMP_BARRIER_ASSIGN_SYNC_W0_EL1);
|
|
SYSREG_WRITE_S(IMP_BARRIER_ASSIGN_SYNC_W1_EL1);
|
|
SYSREG_WRITE_S(IMP_BARRIER_ASSIGN_SYNC_W2_EL1);
|
|
SYSREG_WRITE_S(IMP_BARRIER_ASSIGN_SYNC_W3_EL1);
|
|
SYSREG_WRITE_S(IMP_FJ_CORE_UARCH_CTRL_EL2);
|
|
SYSREG_WRITE_S(IMP_FJ_CORE_UARCH_RESTRECTION_EL1);
|
|
/* fallthrough */
|
|
case IMP_SOC_STANDBY_CTRL_EL1:
|
|
asm volatile("msr_s " __stringify(IMP_SOC_STANDBY_CTRL_EL1) ", %0"
|
|
: : "r" (val) : "memory");
|
|
if (val & IMP_SOC_STANDBY_CTRL_EL1_MODE_CHANGE) {
|
|
wfe();
|
|
}
|
|
break;
|
|
default:
|
|
return -EINVAL;
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
static inline int arch_cpu_write_register(struct ihk_os_cpu_register *desc)
|
|
{
|
|
int ret = 0;
|
|
|
|
if (desc->addr) {
|
|
panic("memory mapped register is not supported.\n");
|
|
} else if (desc->addr_ext) {
|
|
ret = arch_cpu_msr(desc->addr_ext, desc->val);
|
|
} else {
|
|
ret = -EINVAL;
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
int arch_cpu_read_write_register(
|
|
struct ihk_os_cpu_register *desc,
|
|
enum mcctrl_os_cpu_operation op)
|
|
{
|
|
int ret;
|
|
if (op == MCCTRL_OS_CPU_READ_REGISTER) {
|
|
ret = arch_cpu_read_register(desc);
|
|
}
|
|
else if (op == MCCTRL_OS_CPU_WRITE_REGISTER) {
|
|
ret = arch_cpu_write_register(desc);
|
|
}
|
|
else {
|
|
ret = -1;
|
|
}
|
|
|
|
dkprintf("%s: MCCTRL_OS_CPU_%s_REGISTER: reg: 0x%lx, val: 0x%lx\n",
|
|
__FUNCTION__,
|
|
(op == MCCTRL_OS_CPU_READ_REGISTER ? "READ" : "WRITE"),
|
|
desc->addr, desc->val);
|
|
|
|
return ret;
|
|
}
|
|
|
|
void arch_flush_icache_all(void)
|
|
{
|
|
asm("ic ialluis");
|
|
dsb(ish);
|
|
}
|
|
|
|
int ihk_mc_get_smp_handler_irq(void)
|
|
{
|
|
return LOCAL_SMP_FUNC_CALL_VECTOR;
|
|
}
|
|
|
|
/*** end of file ***/
|