diff --git a/arch/arm64/kernel/include/ihk/atomic.h b/arch/arm64/kernel/include/ihk/atomic.h index 02978a90..f894a3ec 100644 --- a/arch/arm64/kernel/include/ihk/atomic.h +++ b/arch/arm64/kernel/include/ihk/atomic.h @@ -147,6 +147,8 @@ static inline void ihk_atomic64_add(long i, ihk_atomic64_t *v) /* @ref.impl arch/arm64/include/asm/atomic.h::atomic64_inc */ #define ihk_atomic64_inc(v) ihk_atomic64_add(1LL, (v)) +#define ihk_atomic64_cmpxchg(p, o, n) cmpxchg(&((p)->counter64), o, n) + /*********************************************************************** * others */ diff --git a/arch/arm64/kernel/perfctr.c b/arch/arm64/kernel/perfctr.c index 2ffc4222..2e6f025c 100644 --- a/arch/arm64/kernel/perfctr.c +++ b/arch/arm64/kernel/perfctr.c @@ -272,3 +272,26 @@ int ihk_mc_event_set_period(struct mc_perf_event *event) return ret; } + +uint64_t ihk_mc_event_update(struct mc_perf_event *event) +{ + struct hw_perf_event *hwc = &event->hw; + int64_t delta; + uint64_t prev_raw_count, new_raw_count; + uint64_t max_period = arm_pmu_event_max_period(event); + +again: + prev_raw_count = ihk_atomic64_read(&hwc->prev_count); + new_raw_count = cpu_pmu.read_counter(event->counter_id); + + if (ihk_atomic64_cmpxchg(&hwc->prev_count, prev_raw_count, + new_raw_count) != prev_raw_count) + goto again; + + delta = (new_raw_count - prev_raw_count) & max_period; + + ihk_atomic64_add(delta, &event->count); + ihk_atomic64_add(-delta, &hwc->period_left); + + return new_raw_count; +} diff --git a/arch/arm64/kernel/perfctr_armv8pmu.c b/arch/arm64/kernel/perfctr_armv8pmu.c index a0973ca9..c31bd9f3 100644 --- a/arch/arm64/kernel/perfctr_armv8pmu.c +++ b/arch/arm64/kernel/perfctr_armv8pmu.c @@ -824,6 +824,7 @@ static void armv8pmu_handle_irq(void *priv) } if (event) { + ihk_mc_event_update(event); ihk_mc_event_set_period(event); } return; diff --git a/arch/x86_64/kernel/perfctr.c b/arch/x86_64/kernel/perfctr.c index 5d668cb0..581022c4 100644 --- a/arch/x86_64/kernel/perfctr.c +++ b/arch/x86_64/kernel/perfctr.c @@ -513,3 +513,8 @@ int ihk_mc_event_set_period(struct mc_perf_event *event) { return 0; } + +uint64_t ihk_mc_event_update(struct mc_perf_event *event) +{ + return 0; +} diff --git a/kernel/include/process.h b/kernel/include/process.h index e9c9f9e0..2adb111a 100644 --- a/kernel/include/process.h +++ b/kernel/include/process.h @@ -725,6 +725,7 @@ struct thread { struct futex_q futex_q; // for performance counter +#define PMC_ALLOC_MAP_BITS BITS_PER_LONG unsigned long pmc_alloc_map; unsigned long extra_reg_alloc_map; diff --git a/kernel/syscall.c b/kernel/syscall.c index d70f2e8e..9118c117 100644 --- a/kernel/syscall.c +++ b/kernel/syscall.c @@ -3914,7 +3914,6 @@ unsigned long perf_event_read_value(struct mc_perf_event *event) { unsigned long rtn_count = 0; unsigned long pmc_count = 0; - int counter_id = event->counter_id; struct thread *thread = cpu_local_var(current); unsigned long cur_user_tsc, cur_system_tsc; @@ -3954,8 +3953,7 @@ unsigned long perf_event_read_value(struct mc_perf_event *event) } } else { - pmc_count = ihk_mc_perfctr_read(counter_id) + - event->attr.sample_freq; + ihk_mc_event_update(event); } } @@ -4186,6 +4184,10 @@ perf_stop(struct mc_perf_event *event) struct mc_perf_event *leader = event->group_leader, *sub; struct thread *thread = cpu_local_var(current); + struct mc_perf_event *stop_event[PMC_ALLOC_MAP_BITS + 1]; + int stop_event_idx = 0; + + stop_event[0] = NULL; counter_id = leader->counter_id; if (ihk_mc_perf_counter_mask_check(1UL << counter_id) && leader->state == PERF_EVENT_STATE_ACTIVE) { @@ -4199,6 +4201,8 @@ perf_stop(struct mc_perf_event *event) } else { counter_mask |= 1UL << counter_id; + stop_event[stop_event_idx++] = leader; + stop_event[stop_event_idx] = NULL; } leader->state = PERF_EVENT_STATE_INACTIVE; @@ -4220,6 +4224,8 @@ perf_stop(struct mc_perf_event *event) } else { counter_mask |= 1UL << counter_id; + stop_event[stop_event_idx++] = sub; + stop_event[stop_event_idx] = NULL; } sub->state = PERF_EVENT_STATE_INACTIVE; } @@ -4227,6 +4233,10 @@ perf_stop(struct mc_perf_event *event) if (counter_mask) { ihk_mc_perfctr_stop(counter_mask, 0); + stop_event_idx = 0; + while (stop_event[stop_event_idx]) { + ihk_mc_event_update(stop_event[stop_event_idx++]); + } } cpu_local_var(current)->proc->monitoring_event = NULL; cpu_local_var(current)->proc->perf_status = PP_NONE; diff --git a/lib/include/ihk/cpu.h b/lib/include/ihk/cpu.h index 7a47505f..f572432e 100644 --- a/lib/include/ihk/cpu.h +++ b/lib/include/ihk/cpu.h @@ -92,6 +92,7 @@ unsigned long ihk_mc_raw_event_map(unsigned long raw_event); int ihk_mc_validate_event(unsigned long hw_config); int hw_perf_event_init(struct mc_perf_event *event); int ihk_mc_event_set_period(struct mc_perf_event *event); +uint64_t ihk_mc_event_update(struct mc_perf_event *event); static inline int is_sampling_event(struct mc_perf_event *event) {