gh0s7/mckernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

pmu: Use bitmap instead of index to specify counters / events

Browse Source 
Let the software index (or number) same as the hardware index at the
same time.

Change-Id: I847180e94bf2c57644ae2f8f571cdb4a84eac991
This commit is contained in:
TOIDA,Suguru
2019-04-09 13:26:25 +09:00
parent b945367c90
commit b6285c9aa9
4 changed files with 72 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
arch/arm64/kernel/include/arch-perfctr.h
View File

@ -20,10 +20,10 @@ struct arm_pmu {
void (*reset)(void*); void (*reset)(void*);
int (*enable_pmu)(void); int (*enable_pmu)(void);
void (*disable_pmu)(void); void (*disable_pmu)(void);
int (*enable_counter)(int); int (*enable_counter)(unsigned long counter_mask);
int (*disable_counter)(int); int (*disable_counter)(unsigned long counter_mask);
int (*enable_intens)(int); int (*enable_intens)(unsigned long counter_mask);
int (*disable_intens)(int); int (*disable_intens)(unsigned long counter_mask);
int (*set_event_filter)(unsigned long*, int); int (*set_event_filter)(unsigned long*, int);
void (*write_evtype)(int, uint32_t); void (*write_evtype)(int, uint32_t);
int (*get_event_idx)(int num_events, unsigned long used_mask, int (*get_event_idx)(int num_events, unsigned long used_mask,
@ -34,6 +34,7 @@ struct arm_pmu {
int (*map_raw_event)(uint64_t config); int (*map_raw_event)(uint64_t config);
void (*enable_user_access_pmu_regs)(void); void (*enable_user_access_pmu_regs)(void);
void (*disable_user_access_pmu_regs)(void); void (*disable_user_access_pmu_regs)(void);
int (*counter_mask_valid)(unsigned long counter_mask);
struct per_cpu_arm_pmu *per_cpu; struct per_cpu_arm_pmu *per_cpu;
}; };

45
arch/arm64/kernel/perfctr.c
View File

@ -91,12 +91,12 @@ static int __ihk_mc_perfctr_init(int counter, uint32_t type, uint64_t config, in
int ret = -1; int ret = -1;
unsigned long config_base = 0; unsigned long config_base = 0;
ret = cpu_pmu.disable_counter(counter); ret = cpu_pmu.disable_counter(1UL << counter);
if (ret < 0) { if (ret < 0) {
return ret; return ret;
} }
ret = cpu_pmu.enable_intens(counter); ret = cpu_pmu.enable_intens(1UL << counter);
if (ret < 0) { if (ret < 0) {
return ret; return ret;
} }
@ -119,47 +119,12 @@ int ihk_mc_perfctr_init_raw(int counter, uint64_t config, int mode)
int ihk_mc_perfctr_start(unsigned long counter_mask) int ihk_mc_perfctr_start(unsigned long counter_mask)
{ {
int ret = 0, i; return cpu_pmu.enable_counter(counter_mask);
for (i = 0; i < sizeof(counter_mask) * BITS_PER_BYTE; i++) {
if (counter_mask & (1UL << i)) {
ret = cpu_pmu.enable_counter(i);
if (ret < 0) {
kprintf("%s: enable failed(idx=%d)\n",
__func__, i);
break;
}
}
}
return ret;
} }
int ihk_mc_perfctr_stop(unsigned long counter_mask, int flags) int ihk_mc_perfctr_stop(unsigned long counter_mask, int flags)
{ {
int i = 0; return cpu_pmu.disable_counter(counter_mask);
for (i = 0; i < sizeof(counter_mask) * BITS_PER_BYTE; i++) {
if (!(counter_mask & (1UL << i)))
continue;
int ret = 0;
ret = cpu_pmu.disable_counter(i);
if (ret < 0) {
continue;
}
if (flags & IHK_MC_PERFCTR_DISABLE_INTERRUPT) {
// when ihk_mc_perfctr_start is called,
// ihk_mc_perfctr_init is also called so disable
// interrupt
ret = cpu_pmu.disable_intens(i);
if (ret < 0) {
continue;
}
}
}
return 0;
} }
int ihk_mc_perfctr_reset(int counter) int ihk_mc_perfctr_reset(int counter)
@ -235,7 +200,7 @@ int ihk_mc_perfctr_alloc_counter(unsigned int *type, unsigned long *config,
int ihk_mc_perf_counter_mask_check(unsigned long counter_mask) int ihk_mc_perf_counter_mask_check(unsigned long counter_mask)
{ {
return 1; return cpu_pmu.counter_mask_valid(counter_mask);
} }
int ihk_mc_perf_get_num_counters(void) int ihk_mc_perf_get_num_counters(void)

161
arch/arm64/kernel/perfctr_armv8pmu.c
View File

@ -186,28 +186,8 @@ static void (* const write_pmevtyper_el0[])(uint32_t) = {
write_pmevtyper30_el0, write_pmevtyper30_el0,
}; };
/* #define ARMV8_IDX_CYCLE_COUNTER 31
* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c #define ARMV8_IDX_COUNTER0 0
* Perf Events' indices
*/
#define ARMV8_IDX_CYCLE_COUNTER 0
#define ARMV8_IDX_COUNTER0 1
#define ARMV8_IDX_COUNTER_LAST (ARMV8_IDX_CYCLE_COUNTER + get_per_cpu_pmu()->num_events - 1)
/* @ref.impl linux-v4.15-rc3 arch/arm64/include/asm/perf_event.h */
#define ARMV8_PMU_MAX_COUNTERS 32
#define ARMV8_PMU_COUNTER_MASK (ARMV8_PMU_MAX_COUNTERS - 1)
/*
* ARMv8 low level PMU access
*/
/*
* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c
* Perf Event to low level counters mapping
*/
#define ARMV8_IDX_TO_COUNTER(x) \
(((x) - ARMV8_IDX_COUNTER0) & ARMV8_PMU_COUNTER_MASK)
/* /*
* @ref.impl linux-v4.15-rc3 arch/arm64/include/asm/perf_event.h * @ref.impl linux-v4.15-rc3 arch/arm64/include/asm/perf_event.h
@ -469,11 +449,25 @@ armpmu_map_event(uint32_t type, uint64_t config,
return -ENOENT; return -ENOENT;
} }
/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */ static inline int armv8pmu_counter_mask_valid(unsigned long counter_mask)
{
int num;
unsigned long event;
unsigned long cycle;
unsigned long invalid_mask;
num = get_per_cpu_pmu()->num_events;
num--; /* Sub the CPU cycles counter */
event = ((1UL << num) - 1) << ARMV8_IDX_COUNTER0;
cycle = 1UL << ARMV8_IDX_CYCLE_COUNTER;
invalid_mask = ~(event | cycle);
return !(counter_mask & invalid_mask);
}
static inline int armv8pmu_counter_valid(int idx) static inline int armv8pmu_counter_valid(int idx)
{ {
return idx >= ARMV8_IDX_CYCLE_COUNTER && return armv8pmu_counter_mask_valid(1UL << idx);
idx <= ARMV8_IDX_COUNTER_LAST;
} }
/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */ /* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
@ -572,9 +566,7 @@ static inline uint32_t armv8pmu_read_counter(int idx)
value = read_sysreg(pmccntr_el0); value = read_sysreg(pmccntr_el0);
} }
else { else {
uint32_t counter = ARMV8_IDX_TO_COUNTER(idx); value = read_pmevcntr_el0[idx]();
value = read_pmevcntr_el0[counter]();
} }
return value; return value;
@ -598,40 +590,37 @@ static inline void armv8pmu_write_counter(int idx, uint32_t value)
write_sysreg(value64, pmccntr_el0); write_sysreg(value64, pmccntr_el0);
} }
else { else {
uint32_t counter = ARMV8_IDX_TO_COUNTER(idx); write_pmevcntr_el0[idx](value);
write_pmevcntr_el0[counter](value);
} }
} }
/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */ /* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
static inline int armv8pmu_enable_intens(int idx) static inline int armv8pmu_enable_intens(unsigned long counter_mask)
{ {
uint32_t counter; if (!armv8pmu_counter_mask_valid(counter_mask)) {
ekprintf("%s: invalid counter mask(%#lx)\n",
if (!armv8pmu_counter_valid(idx)) { __func__, counter_mask);
ekprintf("%s: The count_register#%d is not implemented.\n",
__func__, idx);
return -EINVAL; return -EINVAL;
} }
counter = ARMV8_IDX_TO_COUNTER(idx); write_sysreg(counter_mask, pmintenset_el1);
write_sysreg(BIT(counter), pmintenset_el1); return 0;
return idx;
} }
/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */ /* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
static inline int armv8pmu_disable_intens(int idx) static inline int armv8pmu_disable_intens(unsigned long counter_mask)
{ {
uint32_t counter = ARMV8_IDX_TO_COUNTER(idx); if (!armv8pmu_counter_mask_valid(counter_mask)) {
ekprintf("%s: invalid counter mask(%#lx)\n",
write_sysreg(BIT(counter), pmintenclr_el1); __func__, counter_mask);
return -EINVAL;
}
write_sysreg(counter_mask, pmintenclr_el1);
isb(); isb();
/* Clear the overflow flag in case an interrupt is pending. */ /* Clear the overflow flag in case an interrupt is pending. */
write_sysreg(BIT(counter), pmovsclr_el0); write_sysreg(counter_mask, pmovsclr_el0);
isb(); isb();
return 0;
return idx;
} }
/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */ /* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
@ -671,42 +660,32 @@ static inline void armv8pmu_write_evtype(int idx, uint32_t val)
__func__, idx); __func__, idx);
return; return;
} else if (idx != ARMV8_IDX_CYCLE_COUNTER) { } else if (idx != ARMV8_IDX_CYCLE_COUNTER) {
uint32_t counter = ARMV8_IDX_TO_COUNTER(idx); write_pmevtyper_el0[idx](val);
write_pmevtyper_el0[counter](val);
} }
} }
/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */ /* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
static inline int armv8pmu_enable_counter(int idx) static inline int armv8pmu_enable_counter(unsigned long counter_mask)
{ {
uint32_t counter; if (!armv8pmu_counter_mask_valid(counter_mask)) {
ekprintf("%s: invalid counter mask 0x%lx.\n",
if (!armv8pmu_counter_valid(idx)) { __func__, counter_mask);
ekprintf("%s: The count_register#%d is not implemented.\n",
__func__, idx);
return -EINVAL; return -EINVAL;
} }
write_sysreg(counter_mask, pmcntenset_el0);
counter = ARMV8_IDX_TO_COUNTER(idx); return 0;
write_sysreg(BIT(counter), pmcntenset_el0);
return idx;
} }
/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */ /* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
static inline int armv8pmu_disable_counter(int idx) static inline int armv8pmu_disable_counter(unsigned long counter_mask)
{ {
uint32_t counter; if (!armv8pmu_counter_mask_valid(counter_mask)) {
ekprintf("%s: invalid counter mask 0x%lx.\n",
if (!armv8pmu_counter_valid(idx)) { __func__, counter_mask);
ekprintf("%s: The count_register#%d is not implemented.\n",
__func__, idx);
return -EINVAL; return -EINVAL;
} }
write_sysreg(counter_mask, pmcntenclr_el0);
counter = ARMV8_IDX_TO_COUNTER(idx); return 0;
write_sysreg(BIT(counter), pmcntenclr_el0);
return idx;
} }
/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */ /* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
@ -734,41 +713,20 @@ static void armv8pmu_stop(void)
ihk_mc_spinlock_unlock(&pmu_lock, flags); ihk_mc_spinlock_unlock(&pmu_lock, flags);
} }
/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
static void armv8pmu_disable_event(int idx)
{
unsigned long flags;
/*
* Disable counter and interrupt
*/
flags = ihk_mc_spinlock_lock(&pmu_lock);
/*
* Disable counter
*/
armv8pmu_disable_counter(idx);
/*
* Disable interrupt for this counter
*/
armv8pmu_disable_intens(idx);
ihk_mc_spinlock_unlock(&pmu_lock, flags);
}
/* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */ /* @ref.impl linux-v4.15-rc3 arch/arm64/kernel/perf_event.c */
static void armv8pmu_reset(void *info) static void armv8pmu_reset(void *info)
{ {
struct arm_pmu *cpu_pmu = (struct arm_pmu *)info; struct arm_pmu *cpu_pmu = (struct arm_pmu *)info;
uint32_t idx, nb_cnt = uint32_t nb_cnt =
cpu_pmu->per_cpu[ihk_mc_get_processor_id()].num_events; cpu_pmu->per_cpu[ihk_mc_get_processor_id()].num_events;
nb_cnt--; /* Sub the CPU cycles counter */
unsigned long event = ((1UL << nb_cnt) - 1) << ARMV8_IDX_COUNTER0;
unsigned long cycle = 1UL << ARMV8_IDX_CYCLE_COUNTER;
unsigned long valid_mask = event | cycle;
/* The counter and interrupt enable registers are unknown at reset. */ /* The counter and interrupt enable registers are unknown at reset. */
for (idx = ARMV8_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) { armv8pmu_disable_counter(valid_mask);
armv8pmu_disable_counter(idx); armv8pmu_disable_intens(valid_mask);
armv8pmu_disable_intens(idx);
}
/* /*
* Initialize & Reset PMNC. Request overflow interrupt for * Initialize & Reset PMNC. Request overflow interrupt for
@ -782,7 +740,7 @@ static void armv8pmu_reset(void *info)
static int armv8pmu_get_event_idx(int num_events, unsigned long used_mask, static int armv8pmu_get_event_idx(int num_events, unsigned long used_mask,
unsigned long config) unsigned long config)
{ {
int idx; int idx, end;
unsigned long evtype = config & ARMV8_PMU_EVTYPE_EVENT; unsigned long evtype = config & ARMV8_PMU_EVTYPE_EVENT;
/* Always prefer to place a cycle counter into the cycle counter. */ /* Always prefer to place a cycle counter into the cycle counter. */
@ -794,7 +752,9 @@ static int armv8pmu_get_event_idx(int num_events, unsigned long used_mask,
/* /*
* Otherwise use events counters * Otherwise use events counters
*/ */
for (idx = ARMV8_IDX_COUNTER0; idx < num_events; ++idx) { end = ARMV8_IDX_COUNTER0 + num_events;
end--; /* Sub the CPU cycles counter */
for (idx = ARMV8_IDX_COUNTER0; idx < end; ++idx) {
if (!(used_mask & (1UL << idx))) if (!(used_mask & (1UL << idx)))
return idx; return idx;
} }
@ -922,6 +882,7 @@ int armv8pmu_init(struct arm_pmu* cpu_pmu)
cpu_pmu->disable_user_access_pmu_regs = cpu_pmu->disable_user_access_pmu_regs =
armv8pmu_disable_user_access_pmu_regs; armv8pmu_disable_user_access_pmu_regs;
cpu_pmu->handler = &armv8pmu_handler; cpu_pmu->handler = &armv8pmu_handler;
cpu_pmu->counter_mask_valid = &armv8pmu_counter_mask_valid;
return 0; return 0;
} }

2
executer/kernel/mcctrl/arch/arm64/include/archdeps.h
View File

@ -33,7 +33,7 @@ static inline bool pte_is_write_combined(pte_t pte)
} }
#endif /* POSTK_DEBUG_ARCH_DEP_12 */ #endif /* POSTK_DEBUG_ARCH_DEP_12 */
#define ARMV8_IDX_COUNTER0 1 #define ARMV8_IDX_COUNTER0 0
#define ARCH_PERF_COUNTER_START ARMV8_IDX_COUNTER0 #define ARCH_PERF_COUNTER_START ARMV8_IDX_COUNTER0
#if LINUX_VERSION_CODE < KERNEL_VERSION(3,7,0) #if LINUX_VERSION_CODE < KERNEL_VERSION(3,7,0)