911 lines
27 KiB
C
911 lines
27 KiB
C
/* perfctr_armv8pmu.c COPYRIGHT FUJITSU LIMITED 2016-2018 */
|
|
#include <arch-perfctr.h>
|
|
#include <mc_perf_event.h>
|
|
#include <ihk/perfctr.h>
|
|
#include <errno.h>
|
|
#include <ihk/debug.h>
|
|
#include <sysreg.h>
|
|
#include <virt.h>
|
|
#include <bitops.h>
|
|
#include <string.h>
|
|
#include <signal.h>
|
|
#include <cls.h>
|
|
#include <process.h>
|
|
|
|
#define BIT(nr) (1UL << (nr))
|
|
|
|
//#define DEBUG_PRINT_PMU
|
|
#ifdef DEBUG_PRINT_PMU
|
|
#undef DDEBUG_DEFAULT
|
|
#define DDEBUG_DEFAULT DDEBUG_PRINT
|
|
#endif
|
|
|
|
/*
|
|
* read pmevcntr<n>_el0 functions
|
|
*/
|
|
#define read_pmevcntrN_el0(N) \
|
|
static uint32_t read_pmevcntr##N##_el0(void) \
|
|
{ \
|
|
return read_sysreg(pmevcntr##N##_el0); \
|
|
}
|
|
|
|
read_pmevcntrN_el0(0)
|
|
read_pmevcntrN_el0(1)
|
|
read_pmevcntrN_el0(2)
|
|
read_pmevcntrN_el0(3)
|
|
read_pmevcntrN_el0(4)
|
|
read_pmevcntrN_el0(5)
|
|
read_pmevcntrN_el0(6)
|
|
read_pmevcntrN_el0(7)
|
|
read_pmevcntrN_el0(8)
|
|
read_pmevcntrN_el0(9)
|
|
read_pmevcntrN_el0(10)
|
|
read_pmevcntrN_el0(11)
|
|
read_pmevcntrN_el0(12)
|
|
read_pmevcntrN_el0(13)
|
|
read_pmevcntrN_el0(14)
|
|
read_pmevcntrN_el0(15)
|
|
read_pmevcntrN_el0(16)
|
|
read_pmevcntrN_el0(17)
|
|
read_pmevcntrN_el0(18)
|
|
read_pmevcntrN_el0(19)
|
|
read_pmevcntrN_el0(20)
|
|
read_pmevcntrN_el0(21)
|
|
read_pmevcntrN_el0(22)
|
|
read_pmevcntrN_el0(23)
|
|
read_pmevcntrN_el0(24)
|
|
read_pmevcntrN_el0(25)
|
|
read_pmevcntrN_el0(26)
|
|
read_pmevcntrN_el0(27)
|
|
read_pmevcntrN_el0(28)
|
|
read_pmevcntrN_el0(29)
|
|
read_pmevcntrN_el0(30)
|
|
|
|
static uint32_t (* const read_pmevcntr_el0[])(void) = {
|
|
read_pmevcntr0_el0, read_pmevcntr1_el0, read_pmevcntr2_el0,
|
|
read_pmevcntr3_el0, read_pmevcntr4_el0, read_pmevcntr5_el0,
|
|
read_pmevcntr6_el0, read_pmevcntr7_el0, read_pmevcntr8_el0,
|
|
read_pmevcntr9_el0, read_pmevcntr10_el0, read_pmevcntr11_el0,
|
|
read_pmevcntr12_el0, read_pmevcntr13_el0, read_pmevcntr14_el0,
|
|
read_pmevcntr15_el0, read_pmevcntr16_el0, read_pmevcntr17_el0,
|
|
read_pmevcntr18_el0, read_pmevcntr19_el0, read_pmevcntr20_el0,
|
|
read_pmevcntr21_el0, read_pmevcntr22_el0, read_pmevcntr23_el0,
|
|
read_pmevcntr24_el0, read_pmevcntr25_el0, read_pmevcntr26_el0,
|
|
read_pmevcntr27_el0, read_pmevcntr28_el0, read_pmevcntr29_el0,
|
|
read_pmevcntr30_el0,
|
|
};
|
|
|
|
|
|
/*
|
|
* write pmevcntr<n>_el0 functions
|
|
*/
|
|
#define write_pmevcntrN_el0(N) \
|
|
static void write_pmevcntr##N##_el0(uint32_t v) \
|
|
{ \
|
|
write_sysreg(v, pmevcntr##N##_el0); \
|
|
}
|
|
|
|
write_pmevcntrN_el0(0)
|
|
write_pmevcntrN_el0(1)
|
|
write_pmevcntrN_el0(2)
|
|
write_pmevcntrN_el0(3)
|
|
write_pmevcntrN_el0(4)
|
|
write_pmevcntrN_el0(5)
|
|
write_pmevcntrN_el0(6)
|
|
write_pmevcntrN_el0(7)
|
|
write_pmevcntrN_el0(8)
|
|
write_pmevcntrN_el0(9)
|
|
write_pmevcntrN_el0(10)
|
|
write_pmevcntrN_el0(11)
|
|
write_pmevcntrN_el0(12)
|
|
write_pmevcntrN_el0(13)
|
|
write_pmevcntrN_el0(14)
|
|
write_pmevcntrN_el0(15)
|
|
write_pmevcntrN_el0(16)
|
|
write_pmevcntrN_el0(17)
|
|
write_pmevcntrN_el0(18)
|
|
write_pmevcntrN_el0(19)
|
|
write_pmevcntrN_el0(20)
|
|
write_pmevcntrN_el0(21)
|
|
write_pmevcntrN_el0(22)
|
|
write_pmevcntrN_el0(23)
|
|
write_pmevcntrN_el0(24)
|
|
write_pmevcntrN_el0(25)
|
|
write_pmevcntrN_el0(26)
|
|
write_pmevcntrN_el0(27)
|
|
write_pmevcntrN_el0(28)
|
|
write_pmevcntrN_el0(29)
|
|
write_pmevcntrN_el0(30)
|
|
|
|
static void (* const write_pmevcntr_el0[])(uint32_t) = {
|
|
write_pmevcntr0_el0, write_pmevcntr1_el0, write_pmevcntr2_el0,
|
|
write_pmevcntr3_el0, write_pmevcntr4_el0, write_pmevcntr5_el0,
|
|
write_pmevcntr6_el0, write_pmevcntr7_el0, write_pmevcntr8_el0,
|
|
write_pmevcntr9_el0, write_pmevcntr10_el0, write_pmevcntr11_el0,
|
|
write_pmevcntr12_el0, write_pmevcntr13_el0, write_pmevcntr14_el0,
|
|
write_pmevcntr15_el0, write_pmevcntr16_el0, write_pmevcntr17_el0,
|
|
write_pmevcntr18_el0, write_pmevcntr19_el0, write_pmevcntr20_el0,
|
|
write_pmevcntr21_el0, write_pmevcntr22_el0, write_pmevcntr23_el0,
|
|
write_pmevcntr24_el0, write_pmevcntr25_el0, write_pmevcntr26_el0,
|
|
write_pmevcntr27_el0, write_pmevcntr28_el0, write_pmevcntr29_el0,
|
|
write_pmevcntr30_el0,
|
|
};
|
|
|
|
/*
|
|
* write pmevtyper<n>_el0 functions
|
|
*/
|
|
#define write_pmevtyperN_el0(N) \
|
|
static void write_pmevtyper##N##_el0(uint32_t v) \
|
|
{ \
|
|
write_sysreg(v, pmevtyper##N##_el0); \
|
|
}
|
|
|
|
write_pmevtyperN_el0(0)
|
|
write_pmevtyperN_el0(1)
|
|
write_pmevtyperN_el0(2)
|
|
write_pmevtyperN_el0(3)
|
|
write_pmevtyperN_el0(4)
|
|
write_pmevtyperN_el0(5)
|
|
write_pmevtyperN_el0(6)
|
|
write_pmevtyperN_el0(7)
|
|
write_pmevtyperN_el0(8)
|
|
write_pmevtyperN_el0(9)
|
|
write_pmevtyperN_el0(10)
|
|
write_pmevtyperN_el0(11)
|
|
write_pmevtyperN_el0(12)
|
|
write_pmevtyperN_el0(13)
|
|
write_pmevtyperN_el0(14)
|
|
write_pmevtyperN_el0(15)
|
|
write_pmevtyperN_el0(16)
|
|
write_pmevtyperN_el0(17)
|
|
write_pmevtyperN_el0(18)
|
|
write_pmevtyperN_el0(19)
|
|
write_pmevtyperN_el0(20)
|
|
write_pmevtyperN_el0(21)
|
|
write_pmevtyperN_el0(22)
|
|
write_pmevtyperN_el0(23)
|
|
write_pmevtyperN_el0(24)
|
|
write_pmevtyperN_el0(25)
|
|
write_pmevtyperN_el0(26)
|
|
write_pmevtyperN_el0(27)
|
|
write_pmevtyperN_el0(28)
|
|
write_pmevtyperN_el0(29)
|
|
write_pmevtyperN_el0(30)
|
|
|
|
static void (* const write_pmevtyper_el0[])(uint32_t) = {
|
|
write_pmevtyper0_el0, write_pmevtyper1_el0, write_pmevtyper2_el0,
|
|
write_pmevtyper3_el0, write_pmevtyper4_el0, write_pmevtyper5_el0,
|
|
write_pmevtyper6_el0, write_pmevtyper7_el0, write_pmevtyper8_el0,
|
|
write_pmevtyper9_el0, write_pmevtyper10_el0, write_pmevtyper11_el0,
|
|
write_pmevtyper12_el0, write_pmevtyper13_el0, write_pmevtyper14_el0,
|
|
write_pmevtyper15_el0, write_pmevtyper16_el0, write_pmevtyper17_el0,
|
|
write_pmevtyper18_el0, write_pmevtyper19_el0, write_pmevtyper20_el0,
|
|
write_pmevtyper21_el0, write_pmevtyper22_el0, write_pmevtyper23_el0,
|
|
write_pmevtyper24_el0, write_pmevtyper25_el0, write_pmevtyper26_el0,
|
|
write_pmevtyper27_el0, write_pmevtyper28_el0, write_pmevtyper29_el0,
|
|
write_pmevtyper30_el0,
|
|
};
|
|
|
|
#define ARMV8_IDX_CYCLE_COUNTER 31
|
|
#define ARMV8_IDX_COUNTER0 0
|
|
|
|
/*
|
|
* @ref.impl linux-v4.15-rc3 arch/arm64/include/asm/perf_event.h
|
|
* Per-CPU PMCR: config reg
|
|
*/
|
|
#define ARMV8_PMU_PMCR_E (1 << 0) /* Enable all counters */
|
|
#define ARMV8_PMU_PMCR_P (1 << 1) /* Reset all counters */
|
|
#define ARMV8_PMU_PMCR_C (1 << 2) /* Cycle counter reset */
|
|
#define ARMV8_PMU_PMCR_D (1 << 3) /* CCNT counts every 64th cpu cycle */
|
|
#define ARMV8_PMU_PMCR_X (1 << 4) /* Export to ETM */
|
|
#define ARMV8_PMU_PMCR_DP (1 << 5) /* Disable CCNT if non-invasive debug*/
|
|
#define ARMV8_PMU_PMCR_LC (1 << 6) /* Overflow on 64 bit cycle counter */
|
|
#define ARMV8_PMU_PMCR_N_SHIFT 11 /* Number of counters supported */
|
|
#define ARMV8_PMU_PMCR_N_MASK 0x1f
|
|
#define ARMV8_PMU_PMCR_MASK 0x7f /* Mask for writable bits */
|
|
|
|
/*
|
|
* @ref.impl linux-v4.15-rc3 arch/arm64/include/asm/perf_event.h
|
|
* PMOVSR: counters overflow flag status reg
|
|
*/
|
|
#define ARMV8_PMU_OVSR_MASK 0xffffffff /* Mask for writable bits */
|
|
#define ARMV8_PMU_OVERFLOWED_MASK ARMV8_PMU_OVSR_MASK
|
|
|
|
/*
|
|
* @ref.impl linux-v4.15-rc3 arch/arm64/include/asm/perf_event.h
|
|
* PMXEVTYPER: Event selection reg
|
|
*/
|
|
#define ARMV8_PMU_EVTYPE_MASK 0xc800ffff /* Mask for writable bits */
|
|
#define ARMV8_PMU_EVTYPE_EVENT 0xffff /* Mask for EVENT bits */
|
|
|
|
/*
|
|
* @ref.impl linux-v4.15-rc3 arch/arm64/include/asm/perf_event.h
|
|
* Event filters for PMUv3
|
|
*/
|
|
#define ARMV8_PMU_EXCLUDE_EL1 (1 << 31)
|
|
#define ARMV8_PMU_EXCLUDE_EL0 (1 << 30)
|
|
#define ARMV8_PMU_INCLUDE_EL2 (1 << 27)
|
|
|
|
/*
|
|
* @ref.impl linux-v4.15-rc3 arch/arm64/include/asm/perf_event.h
|
|
* PMUSERENR: user enable reg
|
|
*/
|
|
#define ARMV8_PMU_USERENR_MASK 0xf /* Mask for writable bits */
|
|
#define ARMV8_PMU_USERENR_EN (1 << 0) /* PMU regs can be accessed at EL0 */
|
|
#define ARMV8_PMU_USERENR_SW (1 << 1) /* PMSWINC can be written at EL0 */
|
|
#define ARMV8_PMU_USERENR_CR (1 << 2) /* Cycle counter can be read at EL0 */
|
|
#define ARMV8_PMU_USERENR_ER (1 << 3) /* Event counter can be read at EL0 */
|
|
|
|
/*
|
|
* @ref.impl linux-v4.15-rc3 arch/arm64/include/asm/perf_event.h
|
|
* PMUv3 event types: required events
|
|
*/
|
|
#define ARMV8_PMUV3_PERFCTR_SW_INCR 0x00
|
|
#define ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL 0x03
|
|
#define ARMV8_PMUV3_PERFCTR_L1D_CACHE 0x04
|
|
#define ARMV8_PMUV3_PERFCTR_BR_MIS_PRED 0x10
|
|
#define ARMV8_PMUV3_PERFCTR_CPU_CYCLES 0x11
|
|
#define ARMV8_PMUV3_PERFCTR_BR_PRED 0x12
|
|
|
|
/*
|
|
* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c
|
|
* ARMv8 PMUv3 Performance Events handling code.
|
|
* Common event types (some are defined in asm/perf_event.h).
|
|
*/
|
|
|
|
/* At least one of the following is required. */
|
|
#define ARMV8_PMUV3_PERFCTR_INST_RETIRED 0x08
|
|
#define ARMV8_PMUV3_PERFCTR_INST_SPEC 0x1B
|
|
|
|
/* Common architectural events. */
|
|
#define ARMV8_PMUV3_PERFCTR_LD_RETIRED 0x06
|
|
#define ARMV8_PMUV3_PERFCTR_ST_RETIRED 0x07
|
|
#define ARMV8_PMUV3_PERFCTR_EXC_TAKEN 0x09
|
|
#define ARMV8_PMUV3_PERFCTR_EXC_RETURN 0x0A
|
|
#define ARMV8_PMUV3_PERFCTR_CID_WRITE_RETIRED 0x0B
|
|
#define ARMV8_PMUV3_PERFCTR_PC_WRITE_RETIRED 0x0C
|
|
#define ARMV8_PMUV3_PERFCTR_BR_IMMED_RETIRED 0x0D
|
|
#define ARMV8_PMUV3_PERFCTR_BR_RETURN_RETIRED 0x0E
|
|
#define ARMV8_PMUV3_PERFCTR_UNALIGNED_LDST_RETIRED 0x0F
|
|
#define ARMV8_PMUV3_PERFCTR_TTBR_WRITE_RETIRED 0x1C
|
|
#define ARMV8_PMUV3_PERFCTR_CHAIN 0x1E
|
|
#define ARMV8_PMUV3_PERFCTR_BR_RETIRED 0x21
|
|
|
|
/* Common microarchitectural events. */
|
|
#define ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL 0x01
|
|
#define ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL 0x02
|
|
#define ARMV8_PMUV3_PERFCTR_L1D_TLB_REFILL 0x05
|
|
#define ARMV8_PMUV3_PERFCTR_MEM_ACCESS 0x13
|
|
#define ARMV8_PMUV3_PERFCTR_L1I_CACHE 0x14
|
|
#define ARMV8_PMUV3_PERFCTR_L1D_CACHE_WB 0x15
|
|
#define ARMV8_PMUV3_PERFCTR_L2D_CACHE 0x16
|
|
#define ARMV8_PMUV3_PERFCTR_L2D_CACHE_REFILL 0x17
|
|
#define ARMV8_PMUV3_PERFCTR_L2D_CACHE_WB 0x18
|
|
#define ARMV8_PMUV3_PERFCTR_BUS_ACCESS 0x19
|
|
#define ARMV8_PMUV3_PERFCTR_MEMORY_ERROR 0x1A
|
|
#define ARMV8_PMUV3_PERFCTR_BUS_CYCLES 0x1D
|
|
#define ARMV8_PMUV3_PERFCTR_L1D_CACHE_ALLOCATE 0x1F
|
|
#define ARMV8_PMUV3_PERFCTR_L2D_CACHE_ALLOCATE 0x20
|
|
#define ARMV8_PMUV3_PERFCTR_BR_MIS_PRED_RETIRED 0x22
|
|
#define ARMV8_PMUV3_PERFCTR_STALL_FRONTEND 0x23
|
|
#define ARMV8_PMUV3_PERFCTR_STALL_BACKEND 0x24
|
|
#define ARMV8_PMUV3_PERFCTR_L1D_TLB 0x25
|
|
#define ARMV8_PMUV3_PERFCTR_L1I_TLB 0x26
|
|
#define ARMV8_PMUV3_PERFCTR_L2I_CACHE 0x27
|
|
#define ARMV8_PMUV3_PERFCTR_L2I_CACHE_REFILL 0x28
|
|
#define ARMV8_PMUV3_PERFCTR_L3D_CACHE_ALLOCATE 0x29
|
|
#define ARMV8_PMUV3_PERFCTR_L3D_CACHE_REFILL 0x2A
|
|
#define ARMV8_PMUV3_PERFCTR_L3D_CACHE 0x2B
|
|
#define ARMV8_PMUV3_PERFCTR_L3D_CACHE_WB 0x2C
|
|
#define ARMV8_PMUV3_PERFCTR_L2D_TLB_REFILL 0x2D
|
|
#define ARMV8_PMUV3_PERFCTR_L2I_TLB_REFILL 0x2E
|
|
#define ARMV8_PMUV3_PERFCTR_L2D_TLB 0x2F
|
|
#define ARMV8_PMUV3_PERFCTR_L2I_TLB 0x30
|
|
|
|
/* @ref.impl linux-v4.15-rc3 include/linux/perf/arm_pmu.h */
|
|
#define HW_OP_UNSUPPORTED 0xFFFF
|
|
#define C(_x) PERF_COUNT_HW_CACHE_##_x
|
|
#define CACHE_OP_UNSUPPORTED 0xFFFF
|
|
|
|
#define PERF_MAP_ALL_UNSUPPORTED \
|
|
[0 ... PERF_COUNT_HW_MAX - 1] = HW_OP_UNSUPPORTED
|
|
|
|
#define PERF_CACHE_MAP_ALL_UNSUPPORTED \
|
|
[0 ... C(MAX) - 1] = { \
|
|
[0 ... C(OP_MAX) - 1] = { \
|
|
[0 ... C(RESULT_MAX) - 1] = CACHE_OP_UNSUPPORTED, \
|
|
}, \
|
|
}
|
|
|
|
/* PMUv3 HW events mapping. */
|
|
|
|
/* disable -Woverride-init for the following initializations */
|
|
#pragma GCC diagnostic push
|
|
#pragma GCC diagnostic ignored "-Woverride-init"
|
|
|
|
/*
|
|
* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c
|
|
* ARMv8 Architectural defined events, not all of these may
|
|
* be supported on any given implementation. Undefined events will
|
|
* be disabled at run-time.
|
|
*/
|
|
static const unsigned armv8_pmuv3_perf_map[PERF_COUNT_HW_MAX] = {
|
|
PERF_MAP_ALL_UNSUPPORTED,
|
|
[PERF_COUNT_HW_CPU_CYCLES] = ARMV8_PMUV3_PERFCTR_CPU_CYCLES,
|
|
[PERF_COUNT_HW_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_INST_RETIRED,
|
|
[PERF_COUNT_HW_CACHE_REFERENCES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE,
|
|
[PERF_COUNT_HW_CACHE_MISSES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL,
|
|
[PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_PC_WRITE_RETIRED,
|
|
[PERF_COUNT_HW_BRANCH_MISSES] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
|
|
[PERF_COUNT_HW_BUS_CYCLES] = ARMV8_PMUV3_PERFCTR_BUS_CYCLES,
|
|
[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV8_PMUV3_PERFCTR_STALL_FRONTEND,
|
|
[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV8_PMUV3_PERFCTR_STALL_BACKEND,
|
|
};
|
|
|
|
/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
|
|
static const unsigned armv8_pmuv3_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
|
|
[PERF_COUNT_HW_CACHE_OP_MAX]
|
|
[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
|
|
PERF_CACHE_MAP_ALL_UNSUPPORTED,
|
|
|
|
[C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1D_CACHE,
|
|
[C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL,
|
|
[C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1D_CACHE,
|
|
[C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL,
|
|
|
|
[C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE,
|
|
[C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL,
|
|
|
|
[C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1D_TLB_REFILL,
|
|
[C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1D_TLB,
|
|
|
|
[C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL,
|
|
[C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1I_TLB,
|
|
|
|
[C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED,
|
|
[C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
|
|
[C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED,
|
|
[C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
|
|
};
|
|
|
|
/* restore warnings */
|
|
#pragma GCC diagnostic pop
|
|
|
|
/* @ref.impl linux-v4.15-rc3 drivers/perf/arm_pmu.c */
|
|
static int
|
|
armpmu_map_cache_event(const unsigned (*cache_map)
|
|
[PERF_COUNT_HW_CACHE_MAX]
|
|
[PERF_COUNT_HW_CACHE_OP_MAX]
|
|
[PERF_COUNT_HW_CACHE_RESULT_MAX],
|
|
uint64_t config)
|
|
{
|
|
unsigned int cache_type, cache_op, cache_result, ret;
|
|
|
|
cache_type = (config >> 0) & 0xff;
|
|
if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
|
|
return -EINVAL;
|
|
|
|
cache_op = (config >> 8) & 0xff;
|
|
if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
|
|
return -EINVAL;
|
|
|
|
cache_result = (config >> 16) & 0xff;
|
|
if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
|
|
return -EINVAL;
|
|
|
|
if (!cache_map)
|
|
return -ENOENT;
|
|
|
|
ret = (int)(*cache_map)[cache_type][cache_op][cache_result];
|
|
|
|
if (ret == CACHE_OP_UNSUPPORTED)
|
|
return -ENOENT;
|
|
|
|
return ret;
|
|
}
|
|
|
|
/* @ref.impl linux-v4.15-rc3 drivers/perf/arm_pmu.c */
|
|
static int
|
|
armpmu_map_hw_event(const unsigned int (*event_map)[PERF_COUNT_HW_MAX],
|
|
uint64_t config)
|
|
{
|
|
int mapping;
|
|
|
|
if (config >= PERF_COUNT_HW_MAX)
|
|
return -EINVAL;
|
|
|
|
if (!event_map)
|
|
return -ENOENT;
|
|
|
|
mapping = (*event_map)[config];
|
|
return mapping == HW_OP_UNSUPPORTED ? -ENOENT : mapping;
|
|
}
|
|
|
|
/* @ref.impl linux-v4.15-rc3 drivers/perf/arm_pmu.c */
|
|
static int
|
|
armpmu_map_raw_event(uint32_t raw_event_mask, uint64_t config)
|
|
{
|
|
return (int)(config & raw_event_mask);
|
|
}
|
|
|
|
/* @ref.impl linux-v4.15-rc3 drivers/perf/arm_pmu.c */
|
|
static int
|
|
armpmu_map_event(uint32_t type, uint64_t config,
|
|
const unsigned int (*event_map)[PERF_COUNT_HW_MAX],
|
|
const unsigned int (*cache_map)
|
|
[PERF_COUNT_HW_CACHE_MAX]
|
|
[PERF_COUNT_HW_CACHE_OP_MAX]
|
|
[PERF_COUNT_HW_CACHE_RESULT_MAX],
|
|
uint32_t raw_event_mask)
|
|
{
|
|
switch (type) {
|
|
case PERF_TYPE_HARDWARE:
|
|
return armpmu_map_hw_event(event_map, config);
|
|
case PERF_TYPE_HW_CACHE:
|
|
return armpmu_map_cache_event(cache_map, config);
|
|
case PERF_TYPE_RAW:
|
|
return armpmu_map_raw_event(raw_event_mask, config);
|
|
}
|
|
return -ENOENT;
|
|
}
|
|
|
|
static inline int armv8pmu_counter_mask_valid(unsigned long counter_mask)
|
|
{
|
|
int num;
|
|
unsigned long event;
|
|
unsigned long cycle;
|
|
unsigned long invalid_mask;
|
|
|
|
num = get_per_cpu_pmu()->num_events;
|
|
num--; /* Sub the CPU cycles counter */
|
|
event = ((1UL << num) - 1) << ARMV8_IDX_COUNTER0;
|
|
cycle = 1UL << ARMV8_IDX_CYCLE_COUNTER;
|
|
invalid_mask = ~(event | cycle);
|
|
|
|
return !(counter_mask & invalid_mask);
|
|
}
|
|
|
|
static inline int armv8pmu_counter_valid(int idx)
|
|
{
|
|
return armv8pmu_counter_mask_valid(1UL << idx);
|
|
}
|
|
|
|
/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
|
|
static inline uint32_t armv8pmu_getreset_flags(void)
|
|
{
|
|
uint32_t value;
|
|
|
|
/* Read */
|
|
value = read_sysreg(pmovsclr_el0);
|
|
|
|
/* Write to clear flags */
|
|
value &= ARMV8_PMU_OVSR_MASK;
|
|
write_sysreg(value, pmovsclr_el0);
|
|
|
|
return value;
|
|
}
|
|
|
|
/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
|
|
static inline int armv8pmu_has_overflowed(uint32_t pmovsr)
|
|
{
|
|
return pmovsr & ARMV8_PMU_OVERFLOWED_MASK;
|
|
}
|
|
|
|
static inline int armv8pmu_counter_has_overflowed(uint32_t pmnc, int idx)
|
|
{
|
|
return pmnc & BIT(idx);
|
|
}
|
|
|
|
/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
|
|
static int __armv8_pmuv3_map_event(uint32_t type, uint64_t config,
|
|
const unsigned int (*extra_event_map)
|
|
[PERF_COUNT_HW_MAX],
|
|
const unsigned int (*extra_cache_map)
|
|
[PERF_COUNT_HW_CACHE_MAX]
|
|
[PERF_COUNT_HW_CACHE_OP_MAX]
|
|
[PERF_COUNT_HW_CACHE_RESULT_MAX])
|
|
{
|
|
int hw_event_id;
|
|
|
|
hw_event_id = armpmu_map_event(type, config, &armv8_pmuv3_perf_map,
|
|
&armv8_pmuv3_perf_cache_map,
|
|
ARMV8_PMU_EVTYPE_EVENT);
|
|
|
|
/* Onl expose micro/arch events supported by this PMU */
|
|
if ((hw_event_id > 0) && (hw_event_id < ARMV8_PMUV3_MAX_COMMON_EVENTS)
|
|
&& test_bit(hw_event_id, get_per_cpu_pmu()->pmceid_bitmap)) {
|
|
return hw_event_id;
|
|
}
|
|
|
|
return armpmu_map_event(type, config, extra_event_map, extra_cache_map,
|
|
ARMV8_PMU_EVTYPE_EVENT);
|
|
}
|
|
|
|
/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
|
|
static int armv8_pmuv3_map_event(uint32_t type, uint64_t config)
|
|
{
|
|
return __armv8_pmuv3_map_event(type, config, NULL, NULL);
|
|
}
|
|
|
|
|
|
static int armv8_pmuv3_map_hw_event(uint64_t config)
|
|
{
|
|
return __armv8_pmuv3_map_event(PERF_TYPE_HARDWARE, config, NULL, NULL);
|
|
}
|
|
|
|
|
|
static int armv8_pmuv3_map_cache_event(uint64_t config)
|
|
{
|
|
return __armv8_pmuv3_map_event(PERF_TYPE_HW_CACHE, config, NULL, NULL);
|
|
}
|
|
|
|
static int armv8_pmuv3_map_raw_event(uint64_t config)
|
|
{
|
|
return __armv8_pmuv3_map_event(PERF_TYPE_RAW, config, NULL, NULL);
|
|
}
|
|
|
|
/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
|
|
static inline uint32_t armv8pmu_pmcr_read(void)
|
|
{
|
|
return read_sysreg(pmcr_el0);
|
|
}
|
|
|
|
/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
|
|
static inline void armv8pmu_pmcr_write(uint32_t val)
|
|
{
|
|
val &= ARMV8_PMU_PMCR_MASK;
|
|
isb();
|
|
write_sysreg(val, pmcr_el0);
|
|
}
|
|
|
|
/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
|
|
static inline uint32_t armv8pmu_read_counter(int idx)
|
|
{
|
|
uint32_t value = 0;
|
|
|
|
if (!armv8pmu_counter_valid(idx)) {
|
|
ekprintf("%s: The count_register#%d is not implemented.\n",
|
|
__func__, idx);
|
|
}
|
|
else if (idx == ARMV8_IDX_CYCLE_COUNTER) {
|
|
value = read_sysreg(pmccntr_el0);
|
|
}
|
|
else {
|
|
value = read_pmevcntr_el0[idx]();
|
|
}
|
|
|
|
return value;
|
|
}
|
|
|
|
/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
|
|
static inline void armv8pmu_write_counter(int idx, uint32_t value)
|
|
{
|
|
if (!armv8pmu_counter_valid(idx)) {
|
|
ekprintf("%s: The count_register#%d is not implemented.\n",
|
|
__func__, idx);
|
|
}
|
|
else if (idx == ARMV8_IDX_CYCLE_COUNTER) {
|
|
/*
|
|
* Set the upper 32bits as this is a 64bit counter but we only
|
|
* count using the lower 32bits and we want an interrupt when
|
|
* it overflows.
|
|
*/
|
|
uint64_t value64 = 0xffffffff00000000ULL | value;
|
|
|
|
write_sysreg(value64, pmccntr_el0);
|
|
}
|
|
else {
|
|
write_pmevcntr_el0[idx](value);
|
|
}
|
|
}
|
|
|
|
/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
|
|
static inline int armv8pmu_enable_intens(unsigned long counter_mask)
|
|
{
|
|
if (!armv8pmu_counter_mask_valid(counter_mask)) {
|
|
ekprintf("%s: invalid counter mask(%#lx)\n",
|
|
__func__, counter_mask);
|
|
return -EINVAL;
|
|
}
|
|
|
|
write_sysreg(counter_mask, pmintenset_el1);
|
|
return 0;
|
|
}
|
|
|
|
/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
|
|
static inline int armv8pmu_disable_intens(unsigned long counter_mask)
|
|
{
|
|
if (!armv8pmu_counter_mask_valid(counter_mask)) {
|
|
ekprintf("%s: invalid counter mask(%#lx)\n",
|
|
__func__, counter_mask);
|
|
return -EINVAL;
|
|
}
|
|
write_sysreg(counter_mask, pmintenclr_el1);
|
|
isb();
|
|
/* Clear the overflow flag in case an interrupt is pending. */
|
|
write_sysreg(counter_mask, pmovsclr_el0);
|
|
isb();
|
|
return 0;
|
|
}
|
|
|
|
/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
|
|
static int armv8pmu_set_event_filter(unsigned long *config_base, int mode)
|
|
{
|
|
/* exclude_idle is unused mode, unsupported */
|
|
// if (attr->exclude_idle)
|
|
// return -EPERM;
|
|
|
|
/*
|
|
* If we're running in hyp mode, then we *are* the hypervisor.
|
|
* Therefore we ignore exclude_hv in this configuration, since
|
|
* there's no hypervisor to sample anyway. This is consistent
|
|
* with other architectures (x86 and Power).
|
|
*/
|
|
if (is_kernel_in_hyp_mode()) {
|
|
if (mode & PERFCTR_KERNEL_MODE)
|
|
*config_base |= ARMV8_PMU_INCLUDE_EL2;
|
|
} else {
|
|
if (!(mode & PERFCTR_KERNEL_MODE))
|
|
*config_base |= ARMV8_PMU_EXCLUDE_EL1;
|
|
/* exclude_hv is unused mode, unsupported */
|
|
// if (!attr->exclude_hv)
|
|
// config_base |= ARMV8_PMU_INCLUDE_EL2;
|
|
}
|
|
if (!(mode & PERFCTR_USER_MODE))
|
|
*config_base |= ARMV8_PMU_EXCLUDE_EL0;
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
|
|
static inline void armv8pmu_write_evtype(int idx, uint32_t val)
|
|
{
|
|
if (!armv8pmu_counter_valid(idx)) {
|
|
ekprintf("%s: The count_register#%d is not implemented.\n",
|
|
__func__, idx);
|
|
return;
|
|
} else if (idx != ARMV8_IDX_CYCLE_COUNTER) {
|
|
write_pmevtyper_el0[idx](val);
|
|
}
|
|
}
|
|
|
|
/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
|
|
static inline int armv8pmu_enable_counter(unsigned long counter_mask)
|
|
{
|
|
if (!armv8pmu_counter_mask_valid(counter_mask)) {
|
|
ekprintf("%s: invalid counter mask 0x%lx.\n",
|
|
__func__, counter_mask);
|
|
return -EINVAL;
|
|
}
|
|
write_sysreg(counter_mask, pmcntenset_el0);
|
|
return 0;
|
|
}
|
|
|
|
/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
|
|
static inline int armv8pmu_disable_counter(unsigned long counter_mask)
|
|
{
|
|
if (!armv8pmu_counter_mask_valid(counter_mask)) {
|
|
ekprintf("%s: invalid counter mask 0x%lx.\n",
|
|
__func__, counter_mask);
|
|
return -EINVAL;
|
|
}
|
|
write_sysreg(counter_mask, pmcntenclr_el0);
|
|
return 0;
|
|
}
|
|
|
|
/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
|
|
static ihk_spinlock_t pmu_lock = SPIN_LOCK_UNLOCKED;
|
|
static int armv8pmu_start(void)
|
|
{
|
|
unsigned long flags;
|
|
|
|
flags = ihk_mc_spinlock_lock(&pmu_lock);
|
|
/* Enable all counters */
|
|
armv8pmu_pmcr_write(armv8pmu_pmcr_read() | ARMV8_PMU_PMCR_E);
|
|
ihk_mc_spinlock_unlock(&pmu_lock, flags);
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
|
|
static void armv8pmu_stop(void)
|
|
{
|
|
unsigned long flags;
|
|
|
|
flags = ihk_mc_spinlock_lock(&pmu_lock);
|
|
/* Disable all counters */
|
|
armv8pmu_pmcr_write(armv8pmu_pmcr_read() & ~ARMV8_PMU_PMCR_E);
|
|
ihk_mc_spinlock_unlock(&pmu_lock, flags);
|
|
}
|
|
|
|
/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
|
|
static void armv8pmu_reset(void *info)
|
|
{
|
|
struct arm_pmu *cpu_pmu = (struct arm_pmu *)info;
|
|
uint32_t nb_cnt =
|
|
cpu_pmu->per_cpu[ihk_mc_get_processor_id()].num_events;
|
|
nb_cnt--; /* Sub the CPU cycles counter */
|
|
unsigned long event = ((1UL << nb_cnt) - 1) << ARMV8_IDX_COUNTER0;
|
|
unsigned long cycle = 1UL << ARMV8_IDX_CYCLE_COUNTER;
|
|
unsigned long valid_mask = event | cycle;
|
|
|
|
/* The counter and interrupt enable registers are unknown at reset. */
|
|
armv8pmu_disable_counter(valid_mask);
|
|
armv8pmu_disable_intens(valid_mask);
|
|
|
|
/*
|
|
* Initialize & Reset PMNC. Request overflow interrupt for
|
|
* 64 bit cycle counter but cheat in armv8pmu_write_counter().
|
|
*/
|
|
armv8pmu_pmcr_write(ARMV8_PMU_PMCR_P | ARMV8_PMU_PMCR_C |
|
|
ARMV8_PMU_PMCR_LC);
|
|
}
|
|
|
|
/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
|
|
static int armv8pmu_get_event_idx(int num_events, unsigned long used_mask,
|
|
unsigned long config)
|
|
{
|
|
int idx, end;
|
|
unsigned long evtype = config & ARMV8_PMU_EVTYPE_EVENT;
|
|
|
|
/* Always prefer to place a cycle counter into the cycle counter. */
|
|
if (evtype == ARMV8_PMUV3_PERFCTR_CPU_CYCLES) {
|
|
if (!(used_mask & (1UL << ARMV8_IDX_CYCLE_COUNTER)))
|
|
return ARMV8_IDX_CYCLE_COUNTER;
|
|
}
|
|
|
|
/*
|
|
* Otherwise use events counters
|
|
*/
|
|
end = ARMV8_IDX_COUNTER0 + num_events;
|
|
end--; /* Sub the CPU cycles counter */
|
|
for (idx = ARMV8_IDX_COUNTER0; idx < end; ++idx) {
|
|
if (!(used_mask & (1UL << idx)))
|
|
return idx;
|
|
}
|
|
|
|
/* The counters are all in use. */
|
|
return -EAGAIN;
|
|
}
|
|
|
|
/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c:__armv8pmu_probe_pmu() */
|
|
/* Extract get num_events processing. */
|
|
static uint32_t armv8pmu_read_num_pmnc_events(void)
|
|
{
|
|
uint32_t num_events = 0;
|
|
|
|
/* Read the nb of CNTx counters supported from PMNC */
|
|
num_events = (armv8pmu_pmcr_read() >> ARMV8_PMU_PMCR_N_SHIFT)
|
|
& ARMV8_PMU_PMCR_N_MASK;
|
|
|
|
/* Add the CPU cycles counter */
|
|
num_events += 1;
|
|
|
|
return num_events;
|
|
}
|
|
|
|
static void armv8pmu_handle_irq(void *priv)
|
|
{
|
|
uint32_t pmovsr;
|
|
struct thread *thread = cpu_local_var(current);
|
|
struct process *proc = thread->proc;
|
|
const struct per_cpu_arm_pmu *cpu_pmu = get_per_cpu_pmu();
|
|
int idx;
|
|
|
|
/*
|
|
* Get and reset the IRQ flags
|
|
*/
|
|
pmovsr = armv8pmu_getreset_flags();
|
|
|
|
/*
|
|
* Did an overflow occur?
|
|
*/
|
|
if (!armv8pmu_has_overflowed(pmovsr))
|
|
return;
|
|
|
|
if (!proc->monitoring_event) {
|
|
return;
|
|
}
|
|
/*
|
|
* Handle the counter(s) overflow(s)
|
|
*/
|
|
for (idx = 0; idx < cpu_pmu->num_events; idx++) {
|
|
struct mc_perf_event *event = NULL;
|
|
struct mc_perf_event *sub;
|
|
|
|
if (!armv8pmu_counter_has_overflowed(pmovsr, idx)) {
|
|
continue;
|
|
}
|
|
|
|
if (proc->monitoring_event->counter_id == idx) {
|
|
event = proc->monitoring_event;
|
|
} else {
|
|
list_for_each_entry(sub,
|
|
&proc->monitoring_event->sibling_list,
|
|
group_entry) {
|
|
if (sub->counter_id == idx) {
|
|
event = sub;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!event) {
|
|
continue;
|
|
}
|
|
ihk_mc_event_update(event);
|
|
ihk_mc_event_set_period(event);
|
|
}
|
|
return;
|
|
}
|
|
|
|
static void armv8pmu_enable_user_access_pmu_regs(void)
|
|
{
|
|
uint32_t value = 0;
|
|
|
|
value = read_sysreg(pmuserenr_el0);
|
|
write_sysreg(value | (ARMV8_PMU_USERENR_ER | ARMV8_PMU_USERENR_CR),
|
|
pmuserenr_el0);
|
|
}
|
|
|
|
static void armv8pmu_disable_user_access_pmu_regs(void)
|
|
{
|
|
uint32_t value = 0;
|
|
|
|
value = read_sysreg(pmuserenr_el0);
|
|
write_sysreg(value & ~(ARMV8_PMU_USERENR_ER | ARMV8_PMU_USERENR_CR),
|
|
pmuserenr_el0);
|
|
}
|
|
|
|
static void armv8pmu_create_pmceid_bitmap(unsigned long *bitmap, uint32_t nbits)
|
|
{
|
|
uint32_t pmceid[2];
|
|
|
|
memset(bitmap, 0, BITS_TO_LONGS(nbits) * sizeof(unsigned long));
|
|
|
|
pmceid[0] = read_sysreg(pmceid0_el0);
|
|
bitmap[0] = (unsigned long)pmceid[0];
|
|
|
|
pmceid[1] = read_sysreg(pmceid1_el0);
|
|
bitmap[0] |= (unsigned long)pmceid[1] << 32;
|
|
}
|
|
|
|
static struct ihk_mc_interrupt_handler armv8pmu_handler = {
|
|
.func = armv8pmu_handle_irq,
|
|
.priv = NULL,
|
|
};
|
|
|
|
int armv8pmu_init(struct arm_pmu* cpu_pmu)
|
|
{
|
|
cpu_pmu->read_counter = armv8pmu_read_counter;
|
|
cpu_pmu->write_counter = armv8pmu_write_counter;
|
|
cpu_pmu->reset = armv8pmu_reset;
|
|
cpu_pmu->enable_pmu = armv8pmu_start;
|
|
cpu_pmu->disable_pmu = armv8pmu_stop;
|
|
cpu_pmu->enable_counter = armv8pmu_enable_counter;
|
|
cpu_pmu->disable_counter = armv8pmu_disable_counter;
|
|
cpu_pmu->enable_intens = armv8pmu_enable_intens;
|
|
cpu_pmu->disable_intens = armv8pmu_disable_intens;
|
|
cpu_pmu->set_event_filter = armv8pmu_set_event_filter;
|
|
cpu_pmu->write_evtype = armv8pmu_write_evtype;
|
|
cpu_pmu->get_event_idx = armv8pmu_get_event_idx;
|
|
cpu_pmu->map_event = armv8_pmuv3_map_event;
|
|
cpu_pmu->map_hw_event = armv8_pmuv3_map_hw_event;
|
|
cpu_pmu->map_cache_event = armv8_pmuv3_map_cache_event;
|
|
cpu_pmu->map_raw_event = armv8_pmuv3_map_raw_event;
|
|
cpu_pmu->enable_user_access_pmu_regs =
|
|
armv8pmu_enable_user_access_pmu_regs;
|
|
cpu_pmu->disable_user_access_pmu_regs =
|
|
armv8pmu_disable_user_access_pmu_regs;
|
|
cpu_pmu->handler = &armv8pmu_handler;
|
|
cpu_pmu->counter_mask_valid = &armv8pmu_counter_mask_valid;
|
|
return 0;
|
|
}
|
|
|
|
void armv8pmu_per_cpu_init(struct per_cpu_arm_pmu *per_cpu)
|
|
{
|
|
per_cpu->num_events = armv8pmu_read_num_pmnc_events();
|
|
armv8pmu_create_pmceid_bitmap(per_cpu->pmceid_bitmap,
|
|
ARMV8_PMUV3_MAX_COMMON_EVENTS);
|
|
}
|