1 // SPDX-License-Identifier: GPL-2.0-only
3 * ARMv8 PMUv3 Performance Events handling code.
5 * Copyright (C) 2012 ARM Limited
6 * Author: Will Deacon <will.deacon@arm.com>
8 * This code is based heavily on the ARMv7 perf event code.
11 #include <asm/irq_regs.h>
12 #include <asm/perf_event.h>
13 #include <asm/sysreg.h>
16 #include <linux/acpi.h>
17 #include <linux/clocksource.h>
18 #include <linux/kvm_host.h>
20 #include <linux/perf/arm_pmu.h>
21 #include <linux/platform_device.h>
22 #include <linux/smp.h>
24 /* ARMv8 Cortex-A53 specific event types. */
25 #define ARMV8_A53_PERFCTR_PREF_LINEFILL 0xC2
27 /* ARMv8 Cavium ThunderX specific event types. */
28 #define ARMV8_THUNDER_PERFCTR_L1D_CACHE_MISS_ST 0xE9
29 #define ARMV8_THUNDER_PERFCTR_L1D_CACHE_PREF_ACCESS 0xEA
30 #define ARMV8_THUNDER_PERFCTR_L1D_CACHE_PREF_MISS 0xEB
31 #define ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_ACCESS 0xEC
32 #define ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_MISS 0xED
35 * ARMv8 Architectural defined events, not all of these may
36 * be supported on any given implementation. Unsupported events will
37 * be disabled at run-time based on the PMCEID registers.
39 static const unsigned armv8_pmuv3_perf_map
[PERF_COUNT_HW_MAX
] = {
40 PERF_MAP_ALL_UNSUPPORTED
,
41 [PERF_COUNT_HW_CPU_CYCLES
] = ARMV8_PMUV3_PERFCTR_CPU_CYCLES
,
42 [PERF_COUNT_HW_INSTRUCTIONS
] = ARMV8_PMUV3_PERFCTR_INST_RETIRED
,
43 [PERF_COUNT_HW_CACHE_REFERENCES
] = ARMV8_PMUV3_PERFCTR_L1D_CACHE
,
44 [PERF_COUNT_HW_CACHE_MISSES
] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL
,
45 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS
] = ARMV8_PMUV3_PERFCTR_PC_WRITE_RETIRED
,
46 [PERF_COUNT_HW_BRANCH_MISSES
] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED
,
47 [PERF_COUNT_HW_BUS_CYCLES
] = ARMV8_PMUV3_PERFCTR_BUS_CYCLES
,
48 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND
] = ARMV8_PMUV3_PERFCTR_STALL_FRONTEND
,
49 [PERF_COUNT_HW_STALLED_CYCLES_BACKEND
] = ARMV8_PMUV3_PERFCTR_STALL_BACKEND
,
52 static const unsigned armv8_pmuv3_perf_cache_map
[PERF_COUNT_HW_CACHE_MAX
]
53 [PERF_COUNT_HW_CACHE_OP_MAX
]
54 [PERF_COUNT_HW_CACHE_RESULT_MAX
] = {
55 PERF_CACHE_MAP_ALL_UNSUPPORTED
,
57 [C(L1D
)][C(OP_READ
)][C(RESULT_ACCESS
)] = ARMV8_PMUV3_PERFCTR_L1D_CACHE
,
58 [C(L1D
)][C(OP_READ
)][C(RESULT_MISS
)] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL
,
60 [C(L1I
)][C(OP_READ
)][C(RESULT_ACCESS
)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE
,
61 [C(L1I
)][C(OP_READ
)][C(RESULT_MISS
)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL
,
63 [C(DTLB
)][C(OP_READ
)][C(RESULT_MISS
)] = ARMV8_PMUV3_PERFCTR_L1D_TLB_REFILL
,
64 [C(DTLB
)][C(OP_READ
)][C(RESULT_ACCESS
)] = ARMV8_PMUV3_PERFCTR_L1D_TLB
,
66 [C(ITLB
)][C(OP_READ
)][C(RESULT_MISS
)] = ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL
,
67 [C(ITLB
)][C(OP_READ
)][C(RESULT_ACCESS
)] = ARMV8_PMUV3_PERFCTR_L1I_TLB
,
69 [C(BPU
)][C(OP_READ
)][C(RESULT_ACCESS
)] = ARMV8_PMUV3_PERFCTR_BR_PRED
,
70 [C(BPU
)][C(OP_READ
)][C(RESULT_MISS
)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED
,
73 static const unsigned armv8_a53_perf_cache_map
[PERF_COUNT_HW_CACHE_MAX
]
74 [PERF_COUNT_HW_CACHE_OP_MAX
]
75 [PERF_COUNT_HW_CACHE_RESULT_MAX
] = {
76 PERF_CACHE_MAP_ALL_UNSUPPORTED
,
78 [C(L1D
)][C(OP_PREFETCH
)][C(RESULT_MISS
)] = ARMV8_A53_PERFCTR_PREF_LINEFILL
,
80 [C(NODE
)][C(OP_READ
)][C(RESULT_ACCESS
)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD
,
81 [C(NODE
)][C(OP_WRITE
)][C(RESULT_ACCESS
)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR
,
84 static const unsigned armv8_a57_perf_cache_map
[PERF_COUNT_HW_CACHE_MAX
]
85 [PERF_COUNT_HW_CACHE_OP_MAX
]
86 [PERF_COUNT_HW_CACHE_RESULT_MAX
] = {
87 PERF_CACHE_MAP_ALL_UNSUPPORTED
,
89 [C(L1D
)][C(OP_READ
)][C(RESULT_ACCESS
)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_RD
,
90 [C(L1D
)][C(OP_READ
)][C(RESULT_MISS
)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_RD
,
91 [C(L1D
)][C(OP_WRITE
)][C(RESULT_ACCESS
)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR
,
92 [C(L1D
)][C(OP_WRITE
)][C(RESULT_MISS
)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_WR
,
94 [C(DTLB
)][C(OP_READ
)][C(RESULT_MISS
)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD
,
95 [C(DTLB
)][C(OP_WRITE
)][C(RESULT_MISS
)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR
,
97 [C(NODE
)][C(OP_READ
)][C(RESULT_ACCESS
)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD
,
98 [C(NODE
)][C(OP_WRITE
)][C(RESULT_ACCESS
)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR
,
101 static const unsigned armv8_a73_perf_cache_map
[PERF_COUNT_HW_CACHE_MAX
]
102 [PERF_COUNT_HW_CACHE_OP_MAX
]
103 [PERF_COUNT_HW_CACHE_RESULT_MAX
] = {
104 PERF_CACHE_MAP_ALL_UNSUPPORTED
,
106 [C(L1D
)][C(OP_READ
)][C(RESULT_ACCESS
)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_RD
,
107 [C(L1D
)][C(OP_WRITE
)][C(RESULT_ACCESS
)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR
,
110 static const unsigned armv8_thunder_perf_cache_map
[PERF_COUNT_HW_CACHE_MAX
]
111 [PERF_COUNT_HW_CACHE_OP_MAX
]
112 [PERF_COUNT_HW_CACHE_RESULT_MAX
] = {
113 PERF_CACHE_MAP_ALL_UNSUPPORTED
,
115 [C(L1D
)][C(OP_READ
)][C(RESULT_ACCESS
)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_RD
,
116 [C(L1D
)][C(OP_READ
)][C(RESULT_MISS
)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_RD
,
117 [C(L1D
)][C(OP_WRITE
)][C(RESULT_ACCESS
)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR
,
118 [C(L1D
)][C(OP_WRITE
)][C(RESULT_MISS
)] = ARMV8_THUNDER_PERFCTR_L1D_CACHE_MISS_ST
,
119 [C(L1D
)][C(OP_PREFETCH
)][C(RESULT_ACCESS
)] = ARMV8_THUNDER_PERFCTR_L1D_CACHE_PREF_ACCESS
,
120 [C(L1D
)][C(OP_PREFETCH
)][C(RESULT_MISS
)] = ARMV8_THUNDER_PERFCTR_L1D_CACHE_PREF_MISS
,
122 [C(L1I
)][C(OP_PREFETCH
)][C(RESULT_ACCESS
)] = ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_ACCESS
,
123 [C(L1I
)][C(OP_PREFETCH
)][C(RESULT_MISS
)] = ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_MISS
,
125 [C(DTLB
)][C(OP_READ
)][C(RESULT_ACCESS
)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_RD
,
126 [C(DTLB
)][C(OP_READ
)][C(RESULT_MISS
)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD
,
127 [C(DTLB
)][C(OP_WRITE
)][C(RESULT_ACCESS
)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_WR
,
128 [C(DTLB
)][C(OP_WRITE
)][C(RESULT_MISS
)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR
,
131 static const unsigned armv8_vulcan_perf_cache_map
[PERF_COUNT_HW_CACHE_MAX
]
132 [PERF_COUNT_HW_CACHE_OP_MAX
]
133 [PERF_COUNT_HW_CACHE_RESULT_MAX
] = {
134 PERF_CACHE_MAP_ALL_UNSUPPORTED
,
136 [C(L1D
)][C(OP_READ
)][C(RESULT_ACCESS
)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_RD
,
137 [C(L1D
)][C(OP_READ
)][C(RESULT_MISS
)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_RD
,
138 [C(L1D
)][C(OP_WRITE
)][C(RESULT_ACCESS
)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR
,
139 [C(L1D
)][C(OP_WRITE
)][C(RESULT_MISS
)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_WR
,
141 [C(DTLB
)][C(OP_READ
)][C(RESULT_ACCESS
)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_RD
,
142 [C(DTLB
)][C(OP_WRITE
)][C(RESULT_ACCESS
)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_WR
,
143 [C(DTLB
)][C(OP_READ
)][C(RESULT_MISS
)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD
,
144 [C(DTLB
)][C(OP_WRITE
)][C(RESULT_MISS
)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR
,
146 [C(NODE
)][C(OP_READ
)][C(RESULT_ACCESS
)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD
,
147 [C(NODE
)][C(OP_WRITE
)][C(RESULT_ACCESS
)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR
,
151 armv8pmu_events_sysfs_show(struct device
*dev
,
152 struct device_attribute
*attr
, char *page
)
154 struct perf_pmu_events_attr
*pmu_attr
;
156 pmu_attr
= container_of(attr
, struct perf_pmu_events_attr
, attr
);
158 return sprintf(page
, "event=0x%03llx\n", pmu_attr
->id
);
161 #define ARMV8_EVENT_ATTR(name, config) \
162 (&((struct perf_pmu_events_attr) { \
163 .attr = __ATTR(name, 0444, armv8pmu_events_sysfs_show, NULL), \
167 static struct attribute
*armv8_pmuv3_event_attrs
[] = {
168 ARMV8_EVENT_ATTR(sw_incr
, ARMV8_PMUV3_PERFCTR_SW_INCR
),
169 ARMV8_EVENT_ATTR(l1i_cache_refill
, ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL
),
170 ARMV8_EVENT_ATTR(l1i_tlb_refill
, ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL
),
171 ARMV8_EVENT_ATTR(l1d_cache_refill
, ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL
),
172 ARMV8_EVENT_ATTR(l1d_cache
, ARMV8_PMUV3_PERFCTR_L1D_CACHE
),
173 ARMV8_EVENT_ATTR(l1d_tlb_refill
, ARMV8_PMUV3_PERFCTR_L1D_TLB_REFILL
),
174 ARMV8_EVENT_ATTR(ld_retired
, ARMV8_PMUV3_PERFCTR_LD_RETIRED
),
175 ARMV8_EVENT_ATTR(st_retired
, ARMV8_PMUV3_PERFCTR_ST_RETIRED
),
176 ARMV8_EVENT_ATTR(inst_retired
, ARMV8_PMUV3_PERFCTR_INST_RETIRED
),
177 ARMV8_EVENT_ATTR(exc_taken
, ARMV8_PMUV3_PERFCTR_EXC_TAKEN
),
178 ARMV8_EVENT_ATTR(exc_return
, ARMV8_PMUV3_PERFCTR_EXC_RETURN
),
179 ARMV8_EVENT_ATTR(cid_write_retired
, ARMV8_PMUV3_PERFCTR_CID_WRITE_RETIRED
),
180 ARMV8_EVENT_ATTR(pc_write_retired
, ARMV8_PMUV3_PERFCTR_PC_WRITE_RETIRED
),
181 ARMV8_EVENT_ATTR(br_immed_retired
, ARMV8_PMUV3_PERFCTR_BR_IMMED_RETIRED
),
182 ARMV8_EVENT_ATTR(br_return_retired
, ARMV8_PMUV3_PERFCTR_BR_RETURN_RETIRED
),
183 ARMV8_EVENT_ATTR(unaligned_ldst_retired
, ARMV8_PMUV3_PERFCTR_UNALIGNED_LDST_RETIRED
),
184 ARMV8_EVENT_ATTR(br_mis_pred
, ARMV8_PMUV3_PERFCTR_BR_MIS_PRED
),
185 ARMV8_EVENT_ATTR(cpu_cycles
, ARMV8_PMUV3_PERFCTR_CPU_CYCLES
),
186 ARMV8_EVENT_ATTR(br_pred
, ARMV8_PMUV3_PERFCTR_BR_PRED
),
187 ARMV8_EVENT_ATTR(mem_access
, ARMV8_PMUV3_PERFCTR_MEM_ACCESS
),
188 ARMV8_EVENT_ATTR(l1i_cache
, ARMV8_PMUV3_PERFCTR_L1I_CACHE
),
189 ARMV8_EVENT_ATTR(l1d_cache_wb
, ARMV8_PMUV3_PERFCTR_L1D_CACHE_WB
),
190 ARMV8_EVENT_ATTR(l2d_cache
, ARMV8_PMUV3_PERFCTR_L2D_CACHE
),
191 ARMV8_EVENT_ATTR(l2d_cache_refill
, ARMV8_PMUV3_PERFCTR_L2D_CACHE_REFILL
),
192 ARMV8_EVENT_ATTR(l2d_cache_wb
, ARMV8_PMUV3_PERFCTR_L2D_CACHE_WB
),
193 ARMV8_EVENT_ATTR(bus_access
, ARMV8_PMUV3_PERFCTR_BUS_ACCESS
),
194 ARMV8_EVENT_ATTR(memory_error
, ARMV8_PMUV3_PERFCTR_MEMORY_ERROR
),
195 ARMV8_EVENT_ATTR(inst_spec
, ARMV8_PMUV3_PERFCTR_INST_SPEC
),
196 ARMV8_EVENT_ATTR(ttbr_write_retired
, ARMV8_PMUV3_PERFCTR_TTBR_WRITE_RETIRED
),
197 ARMV8_EVENT_ATTR(bus_cycles
, ARMV8_PMUV3_PERFCTR_BUS_CYCLES
),
198 /* Don't expose the chain event in /sys, since it's useless in isolation */
199 ARMV8_EVENT_ATTR(l1d_cache_allocate
, ARMV8_PMUV3_PERFCTR_L1D_CACHE_ALLOCATE
),
200 ARMV8_EVENT_ATTR(l2d_cache_allocate
, ARMV8_PMUV3_PERFCTR_L2D_CACHE_ALLOCATE
),
201 ARMV8_EVENT_ATTR(br_retired
, ARMV8_PMUV3_PERFCTR_BR_RETIRED
),
202 ARMV8_EVENT_ATTR(br_mis_pred_retired
, ARMV8_PMUV3_PERFCTR_BR_MIS_PRED_RETIRED
),
203 ARMV8_EVENT_ATTR(stall_frontend
, ARMV8_PMUV3_PERFCTR_STALL_FRONTEND
),
204 ARMV8_EVENT_ATTR(stall_backend
, ARMV8_PMUV3_PERFCTR_STALL_BACKEND
),
205 ARMV8_EVENT_ATTR(l1d_tlb
, ARMV8_PMUV3_PERFCTR_L1D_TLB
),
206 ARMV8_EVENT_ATTR(l1i_tlb
, ARMV8_PMUV3_PERFCTR_L1I_TLB
),
207 ARMV8_EVENT_ATTR(l2i_cache
, ARMV8_PMUV3_PERFCTR_L2I_CACHE
),
208 ARMV8_EVENT_ATTR(l2i_cache_refill
, ARMV8_PMUV3_PERFCTR_L2I_CACHE_REFILL
),
209 ARMV8_EVENT_ATTR(l3d_cache_allocate
, ARMV8_PMUV3_PERFCTR_L3D_CACHE_ALLOCATE
),
210 ARMV8_EVENT_ATTR(l3d_cache_refill
, ARMV8_PMUV3_PERFCTR_L3D_CACHE_REFILL
),
211 ARMV8_EVENT_ATTR(l3d_cache
, ARMV8_PMUV3_PERFCTR_L3D_CACHE
),
212 ARMV8_EVENT_ATTR(l3d_cache_wb
, ARMV8_PMUV3_PERFCTR_L3D_CACHE_WB
),
213 ARMV8_EVENT_ATTR(l2d_tlb_refill
, ARMV8_PMUV3_PERFCTR_L2D_TLB_REFILL
),
214 ARMV8_EVENT_ATTR(l2i_tlb_refill
, ARMV8_PMUV3_PERFCTR_L2I_TLB_REFILL
),
215 ARMV8_EVENT_ATTR(l2d_tlb
, ARMV8_PMUV3_PERFCTR_L2D_TLB
),
216 ARMV8_EVENT_ATTR(l2i_tlb
, ARMV8_PMUV3_PERFCTR_L2I_TLB
),
217 ARMV8_EVENT_ATTR(remote_access
, ARMV8_PMUV3_PERFCTR_REMOTE_ACCESS
),
218 ARMV8_EVENT_ATTR(ll_cache
, ARMV8_PMUV3_PERFCTR_LL_CACHE
),
219 ARMV8_EVENT_ATTR(ll_cache_miss
, ARMV8_PMUV3_PERFCTR_LL_CACHE_MISS
),
220 ARMV8_EVENT_ATTR(dtlb_walk
, ARMV8_PMUV3_PERFCTR_DTLB_WALK
),
221 ARMV8_EVENT_ATTR(itlb_walk
, ARMV8_PMUV3_PERFCTR_ITLB_WALK
),
222 ARMV8_EVENT_ATTR(ll_cache_rd
, ARMV8_PMUV3_PERFCTR_LL_CACHE_RD
),
223 ARMV8_EVENT_ATTR(ll_cache_miss_rd
, ARMV8_PMUV3_PERFCTR_LL_CACHE_MISS_RD
),
224 ARMV8_EVENT_ATTR(remote_access_rd
, ARMV8_PMUV3_PERFCTR_REMOTE_ACCESS_RD
),
225 ARMV8_EVENT_ATTR(sample_pop
, ARMV8_SPE_PERFCTR_SAMPLE_POP
),
226 ARMV8_EVENT_ATTR(sample_feed
, ARMV8_SPE_PERFCTR_SAMPLE_FEED
),
227 ARMV8_EVENT_ATTR(sample_filtrate
, ARMV8_SPE_PERFCTR_SAMPLE_FILTRATE
),
228 ARMV8_EVENT_ATTR(sample_collision
, ARMV8_SPE_PERFCTR_SAMPLE_COLLISION
),
233 armv8pmu_event_attr_is_visible(struct kobject
*kobj
,
234 struct attribute
*attr
, int unused
)
236 struct device
*dev
= kobj_to_dev(kobj
);
237 struct pmu
*pmu
= dev_get_drvdata(dev
);
238 struct arm_pmu
*cpu_pmu
= container_of(pmu
, struct arm_pmu
, pmu
);
239 struct perf_pmu_events_attr
*pmu_attr
;
241 pmu_attr
= container_of(attr
, struct perf_pmu_events_attr
, attr
.attr
);
243 if (pmu_attr
->id
< ARMV8_PMUV3_MAX_COMMON_EVENTS
&&
244 test_bit(pmu_attr
->id
, cpu_pmu
->pmceid_bitmap
))
247 pmu_attr
->id
-= ARMV8_PMUV3_EXT_COMMON_EVENT_BASE
;
248 if (pmu_attr
->id
< ARMV8_PMUV3_MAX_COMMON_EVENTS
&&
249 test_bit(pmu_attr
->id
, cpu_pmu
->pmceid_ext_bitmap
))
255 static struct attribute_group armv8_pmuv3_events_attr_group
= {
257 .attrs
= armv8_pmuv3_event_attrs
,
258 .is_visible
= armv8pmu_event_attr_is_visible
,
261 PMU_FORMAT_ATTR(event
, "config:0-15");
262 PMU_FORMAT_ATTR(long, "config1:0");
264 static inline bool armv8pmu_event_is_64bit(struct perf_event
*event
)
266 return event
->attr
.config1
& 0x1;
269 static struct attribute
*armv8_pmuv3_format_attrs
[] = {
270 &format_attr_event
.attr
,
271 &format_attr_long
.attr
,
275 static struct attribute_group armv8_pmuv3_format_attr_group
= {
277 .attrs
= armv8_pmuv3_format_attrs
,
281 * Perf Events' indices
283 #define ARMV8_IDX_CYCLE_COUNTER 0
284 #define ARMV8_IDX_COUNTER0 1
285 #define ARMV8_IDX_COUNTER_LAST(cpu_pmu) \
286 (ARMV8_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1)
290 * We unconditionally enable ARMv8.5-PMU long event counter support
291 * (64-bit events) where supported. Indicate if this arm_pmu has long
292 * event counter support.
294 static bool armv8pmu_has_long_event(struct arm_pmu
*cpu_pmu
)
296 return (cpu_pmu
->pmuver
>= ID_AA64DFR0_PMUVER_8_5
);
300 * We must chain two programmable counters for 64 bit events,
301 * except when we have allocated the 64bit cycle counter (for CPU
302 * cycles event). This must be called only when the event has
303 * a counter allocated.
305 static inline bool armv8pmu_event_is_chained(struct perf_event
*event
)
307 int idx
= event
->hw
.idx
;
308 struct arm_pmu
*cpu_pmu
= to_arm_pmu(event
->pmu
);
310 return !WARN_ON(idx
< 0) &&
311 armv8pmu_event_is_64bit(event
) &&
312 !armv8pmu_has_long_event(cpu_pmu
) &&
313 (idx
!= ARMV8_IDX_CYCLE_COUNTER
);
317 * ARMv8 low level PMU access
321 * Perf Event to low level counters mapping
323 #define ARMV8_IDX_TO_COUNTER(x) \
324 (((x) - ARMV8_IDX_COUNTER0) & ARMV8_PMU_COUNTER_MASK)
326 static inline u32
armv8pmu_pmcr_read(void)
328 return read_sysreg(pmcr_el0
);
331 static inline void armv8pmu_pmcr_write(u32 val
)
333 val
&= ARMV8_PMU_PMCR_MASK
;
335 write_sysreg(val
, pmcr_el0
);
338 static inline int armv8pmu_has_overflowed(u32 pmovsr
)
340 return pmovsr
& ARMV8_PMU_OVERFLOWED_MASK
;
343 static inline int armv8pmu_counter_valid(struct arm_pmu
*cpu_pmu
, int idx
)
345 return idx
>= ARMV8_IDX_CYCLE_COUNTER
&&
346 idx
<= ARMV8_IDX_COUNTER_LAST(cpu_pmu
);
349 static inline int armv8pmu_counter_has_overflowed(u32 pmnc
, int idx
)
351 return pmnc
& BIT(ARMV8_IDX_TO_COUNTER(idx
));
354 static inline void armv8pmu_select_counter(int idx
)
356 u32 counter
= ARMV8_IDX_TO_COUNTER(idx
);
357 write_sysreg(counter
, pmselr_el0
);
361 static inline u64
armv8pmu_read_evcntr(int idx
)
363 armv8pmu_select_counter(idx
);
364 return read_sysreg(pmxevcntr_el0
);
367 static inline u64
armv8pmu_read_hw_counter(struct perf_event
*event
)
369 int idx
= event
->hw
.idx
;
372 val
= armv8pmu_read_evcntr(idx
);
373 if (armv8pmu_event_is_chained(event
))
374 val
= (val
<< 32) | armv8pmu_read_evcntr(idx
- 1);
379 * The cycle counter is always a 64-bit counter. When ARMV8_PMU_PMCR_LP
380 * is set the event counters also become 64-bit counters. Unless the
381 * user has requested a long counter (attr.config1) then we want to
382 * interrupt upon 32-bit overflow - we achieve this by applying a bias.
384 static bool armv8pmu_event_needs_bias(struct perf_event
*event
)
386 struct arm_pmu
*cpu_pmu
= to_arm_pmu(event
->pmu
);
387 struct hw_perf_event
*hwc
= &event
->hw
;
390 if (armv8pmu_event_is_64bit(event
))
393 if (armv8pmu_has_long_event(cpu_pmu
) ||
394 idx
== ARMV8_IDX_CYCLE_COUNTER
)
400 static u64
armv8pmu_bias_long_counter(struct perf_event
*event
, u64 value
)
402 if (armv8pmu_event_needs_bias(event
))
403 value
|= GENMASK(63, 32);
408 static u64
armv8pmu_unbias_long_counter(struct perf_event
*event
, u64 value
)
410 if (armv8pmu_event_needs_bias(event
))
411 value
&= ~GENMASK(63, 32);
416 static u64
armv8pmu_read_counter(struct perf_event
*event
)
418 struct arm_pmu
*cpu_pmu
= to_arm_pmu(event
->pmu
);
419 struct hw_perf_event
*hwc
= &event
->hw
;
423 if (!armv8pmu_counter_valid(cpu_pmu
, idx
))
424 pr_err("CPU%u reading wrong counter %d\n",
425 smp_processor_id(), idx
);
426 else if (idx
== ARMV8_IDX_CYCLE_COUNTER
)
427 value
= read_sysreg(pmccntr_el0
);
429 value
= armv8pmu_read_hw_counter(event
);
431 return armv8pmu_unbias_long_counter(event
, value
);
434 static inline void armv8pmu_write_evcntr(int idx
, u64 value
)
436 armv8pmu_select_counter(idx
);
437 write_sysreg(value
, pmxevcntr_el0
);
440 static inline void armv8pmu_write_hw_counter(struct perf_event
*event
,
443 int idx
= event
->hw
.idx
;
445 if (armv8pmu_event_is_chained(event
)) {
446 armv8pmu_write_evcntr(idx
, upper_32_bits(value
));
447 armv8pmu_write_evcntr(idx
- 1, lower_32_bits(value
));
449 armv8pmu_write_evcntr(idx
, value
);
453 static void armv8pmu_write_counter(struct perf_event
*event
, u64 value
)
455 struct arm_pmu
*cpu_pmu
= to_arm_pmu(event
->pmu
);
456 struct hw_perf_event
*hwc
= &event
->hw
;
459 value
= armv8pmu_bias_long_counter(event
, value
);
461 if (!armv8pmu_counter_valid(cpu_pmu
, idx
))
462 pr_err("CPU%u writing wrong counter %d\n",
463 smp_processor_id(), idx
);
464 else if (idx
== ARMV8_IDX_CYCLE_COUNTER
)
465 write_sysreg(value
, pmccntr_el0
);
467 armv8pmu_write_hw_counter(event
, value
);
470 static inline void armv8pmu_write_evtype(int idx
, u32 val
)
472 armv8pmu_select_counter(idx
);
473 val
&= ARMV8_PMU_EVTYPE_MASK
;
474 write_sysreg(val
, pmxevtyper_el0
);
477 static inline void armv8pmu_write_event_type(struct perf_event
*event
)
479 struct hw_perf_event
*hwc
= &event
->hw
;
483 * For chained events, the low counter is programmed to count
484 * the event of interest and the high counter is programmed
485 * with CHAIN event code with filters set to count at all ELs.
487 if (armv8pmu_event_is_chained(event
)) {
488 u32 chain_evt
= ARMV8_PMUV3_PERFCTR_CHAIN
|
489 ARMV8_PMU_INCLUDE_EL2
;
491 armv8pmu_write_evtype(idx
- 1, hwc
->config_base
);
492 armv8pmu_write_evtype(idx
, chain_evt
);
494 armv8pmu_write_evtype(idx
, hwc
->config_base
);
498 static u32
armv8pmu_event_cnten_mask(struct perf_event
*event
)
500 int counter
= ARMV8_IDX_TO_COUNTER(event
->hw
.idx
);
501 u32 mask
= BIT(counter
);
503 if (armv8pmu_event_is_chained(event
))
504 mask
|= BIT(counter
- 1);
508 static inline void armv8pmu_enable_counter(u32 mask
)
510 write_sysreg(mask
, pmcntenset_el0
);
513 static inline void armv8pmu_enable_event_counter(struct perf_event
*event
)
515 struct perf_event_attr
*attr
= &event
->attr
;
516 u32 mask
= armv8pmu_event_cnten_mask(event
);
518 kvm_set_pmu_events(mask
, attr
);
520 /* We rely on the hypervisor switch code to enable guest counters */
521 if (!kvm_pmu_counter_deferred(attr
))
522 armv8pmu_enable_counter(mask
);
525 static inline void armv8pmu_disable_counter(u32 mask
)
527 write_sysreg(mask
, pmcntenclr_el0
);
530 static inline void armv8pmu_disable_event_counter(struct perf_event
*event
)
532 struct perf_event_attr
*attr
= &event
->attr
;
533 u32 mask
= armv8pmu_event_cnten_mask(event
);
535 kvm_clr_pmu_events(mask
);
537 /* We rely on the hypervisor switch code to disable guest counters */
538 if (!kvm_pmu_counter_deferred(attr
))
539 armv8pmu_disable_counter(mask
);
542 static inline void armv8pmu_enable_intens(u32 mask
)
544 write_sysreg(mask
, pmintenset_el1
);
547 static inline void armv8pmu_enable_event_irq(struct perf_event
*event
)
549 u32 counter
= ARMV8_IDX_TO_COUNTER(event
->hw
.idx
);
550 armv8pmu_enable_intens(BIT(counter
));
553 static inline void armv8pmu_disable_intens(u32 mask
)
555 write_sysreg(mask
, pmintenclr_el1
);
557 /* Clear the overflow flag in case an interrupt is pending. */
558 write_sysreg(mask
, pmovsclr_el0
);
562 static inline void armv8pmu_disable_event_irq(struct perf_event
*event
)
564 u32 counter
= ARMV8_IDX_TO_COUNTER(event
->hw
.idx
);
565 armv8pmu_disable_intens(BIT(counter
));
568 static inline u32
armv8pmu_getreset_flags(void)
573 value
= read_sysreg(pmovsclr_el0
);
575 /* Write to clear flags */
576 value
&= ARMV8_PMU_OVSR_MASK
;
577 write_sysreg(value
, pmovsclr_el0
);
582 static void armv8pmu_enable_event(struct perf_event
*event
)
585 struct arm_pmu
*cpu_pmu
= to_arm_pmu(event
->pmu
);
586 struct pmu_hw_events
*events
= this_cpu_ptr(cpu_pmu
->hw_events
);
589 * Enable counter and interrupt, and set the counter to count
590 * the event that we're interested in.
592 raw_spin_lock_irqsave(&events
->pmu_lock
, flags
);
597 armv8pmu_disable_event_counter(event
);
600 * Set event (if destined for PMNx counters).
602 armv8pmu_write_event_type(event
);
605 * Enable interrupt for this counter
607 armv8pmu_enable_event_irq(event
);
612 armv8pmu_enable_event_counter(event
);
614 raw_spin_unlock_irqrestore(&events
->pmu_lock
, flags
);
617 static void armv8pmu_disable_event(struct perf_event
*event
)
620 struct arm_pmu
*cpu_pmu
= to_arm_pmu(event
->pmu
);
621 struct pmu_hw_events
*events
= this_cpu_ptr(cpu_pmu
->hw_events
);
624 * Disable counter and interrupt
626 raw_spin_lock_irqsave(&events
->pmu_lock
, flags
);
631 armv8pmu_disable_event_counter(event
);
634 * Disable interrupt for this counter
636 armv8pmu_disable_event_irq(event
);
638 raw_spin_unlock_irqrestore(&events
->pmu_lock
, flags
);
641 static void armv8pmu_start(struct arm_pmu
*cpu_pmu
)
644 struct pmu_hw_events
*events
= this_cpu_ptr(cpu_pmu
->hw_events
);
646 raw_spin_lock_irqsave(&events
->pmu_lock
, flags
);
647 /* Enable all counters */
648 armv8pmu_pmcr_write(armv8pmu_pmcr_read() | ARMV8_PMU_PMCR_E
);
649 raw_spin_unlock_irqrestore(&events
->pmu_lock
, flags
);
652 static void armv8pmu_stop(struct arm_pmu
*cpu_pmu
)
655 struct pmu_hw_events
*events
= this_cpu_ptr(cpu_pmu
->hw_events
);
657 raw_spin_lock_irqsave(&events
->pmu_lock
, flags
);
658 /* Disable all counters */
659 armv8pmu_pmcr_write(armv8pmu_pmcr_read() & ~ARMV8_PMU_PMCR_E
);
660 raw_spin_unlock_irqrestore(&events
->pmu_lock
, flags
);
663 static irqreturn_t
armv8pmu_handle_irq(struct arm_pmu
*cpu_pmu
)
666 struct perf_sample_data data
;
667 struct pmu_hw_events
*cpuc
= this_cpu_ptr(cpu_pmu
->hw_events
);
668 struct pt_regs
*regs
;
672 * Get and reset the IRQ flags
674 pmovsr
= armv8pmu_getreset_flags();
677 * Did an overflow occur?
679 if (!armv8pmu_has_overflowed(pmovsr
))
683 * Handle the counter(s) overflow(s)
685 regs
= get_irq_regs();
688 * Stop the PMU while processing the counter overflows
689 * to prevent skews in group events.
691 armv8pmu_stop(cpu_pmu
);
692 for (idx
= 0; idx
< cpu_pmu
->num_events
; ++idx
) {
693 struct perf_event
*event
= cpuc
->events
[idx
];
694 struct hw_perf_event
*hwc
;
696 /* Ignore if we don't have an event. */
701 * We have a single interrupt for all counters. Check that
702 * each counter has overflowed before we process it.
704 if (!armv8pmu_counter_has_overflowed(pmovsr
, idx
))
708 armpmu_event_update(event
);
709 perf_sample_data_init(&data
, 0, hwc
->last_period
);
710 if (!armpmu_event_set_period(event
))
713 if (perf_event_overflow(event
, &data
, regs
))
714 cpu_pmu
->disable(event
);
716 armv8pmu_start(cpu_pmu
);
719 * Handle the pending perf events.
721 * Note: this call *must* be run with interrupts disabled. For
722 * platforms that can have the PMU interrupts raised as an NMI, this
730 static int armv8pmu_get_single_idx(struct pmu_hw_events
*cpuc
,
731 struct arm_pmu
*cpu_pmu
)
735 for (idx
= ARMV8_IDX_COUNTER0
; idx
< cpu_pmu
->num_events
; idx
++) {
736 if (!test_and_set_bit(idx
, cpuc
->used_mask
))
742 static int armv8pmu_get_chain_idx(struct pmu_hw_events
*cpuc
,
743 struct arm_pmu
*cpu_pmu
)
748 * Chaining requires two consecutive event counters, where
749 * the lower idx must be even.
751 for (idx
= ARMV8_IDX_COUNTER0
+ 1; idx
< cpu_pmu
->num_events
; idx
+= 2) {
752 if (!test_and_set_bit(idx
, cpuc
->used_mask
)) {
753 /* Check if the preceding even counter is available */
754 if (!test_and_set_bit(idx
- 1, cpuc
->used_mask
))
756 /* Release the Odd counter */
757 clear_bit(idx
, cpuc
->used_mask
);
763 static int armv8pmu_get_event_idx(struct pmu_hw_events
*cpuc
,
764 struct perf_event
*event
)
766 struct arm_pmu
*cpu_pmu
= to_arm_pmu(event
->pmu
);
767 struct hw_perf_event
*hwc
= &event
->hw
;
768 unsigned long evtype
= hwc
->config_base
& ARMV8_PMU_EVTYPE_EVENT
;
770 /* Always prefer to place a cycle counter into the cycle counter. */
771 if (evtype
== ARMV8_PMUV3_PERFCTR_CPU_CYCLES
) {
772 if (!test_and_set_bit(ARMV8_IDX_CYCLE_COUNTER
, cpuc
->used_mask
))
773 return ARMV8_IDX_CYCLE_COUNTER
;
777 * Otherwise use events counters
779 if (armv8pmu_event_is_64bit(event
) &&
780 !armv8pmu_has_long_event(cpu_pmu
))
781 return armv8pmu_get_chain_idx(cpuc
, cpu_pmu
);
783 return armv8pmu_get_single_idx(cpuc
, cpu_pmu
);
786 static void armv8pmu_clear_event_idx(struct pmu_hw_events
*cpuc
,
787 struct perf_event
*event
)
789 int idx
= event
->hw
.idx
;
791 clear_bit(idx
, cpuc
->used_mask
);
792 if (armv8pmu_event_is_chained(event
))
793 clear_bit(idx
- 1, cpuc
->used_mask
);
797 * Add an event filter to a given event.
799 static int armv8pmu_set_event_filter(struct hw_perf_event
*event
,
800 struct perf_event_attr
*attr
)
802 unsigned long config_base
= 0;
804 if (attr
->exclude_idle
)
808 * If we're running in hyp mode, then we *are* the hypervisor.
809 * Therefore we ignore exclude_hv in this configuration, since
810 * there's no hypervisor to sample anyway. This is consistent
811 * with other architectures (x86 and Power).
813 if (is_kernel_in_hyp_mode()) {
814 if (!attr
->exclude_kernel
&& !attr
->exclude_host
)
815 config_base
|= ARMV8_PMU_INCLUDE_EL2
;
816 if (attr
->exclude_guest
)
817 config_base
|= ARMV8_PMU_EXCLUDE_EL1
;
818 if (attr
->exclude_host
)
819 config_base
|= ARMV8_PMU_EXCLUDE_EL0
;
821 if (!attr
->exclude_hv
&& !attr
->exclude_host
)
822 config_base
|= ARMV8_PMU_INCLUDE_EL2
;
826 * Filter out !VHE kernels and guest kernels
828 if (attr
->exclude_kernel
)
829 config_base
|= ARMV8_PMU_EXCLUDE_EL1
;
831 if (attr
->exclude_user
)
832 config_base
|= ARMV8_PMU_EXCLUDE_EL0
;
835 * Install the filter into config_base as this is used to
836 * construct the event type.
838 event
->config_base
= config_base
;
843 static int armv8pmu_filter_match(struct perf_event
*event
)
845 unsigned long evtype
= event
->hw
.config_base
& ARMV8_PMU_EVTYPE_EVENT
;
846 return evtype
!= ARMV8_PMUV3_PERFCTR_CHAIN
;
849 static void armv8pmu_reset(void *info
)
851 struct arm_pmu
*cpu_pmu
= (struct arm_pmu
*)info
;
854 /* The counter and interrupt enable registers are unknown at reset. */
855 armv8pmu_disable_counter(U32_MAX
);
856 armv8pmu_disable_intens(U32_MAX
);
858 /* Clear the counters we flip at guest entry/exit */
859 kvm_clr_pmu_events(U32_MAX
);
862 * Initialize & Reset PMNC. Request overflow interrupt for
863 * 64 bit cycle counter but cheat in armv8pmu_write_counter().
865 pmcr
= ARMV8_PMU_PMCR_P
| ARMV8_PMU_PMCR_C
| ARMV8_PMU_PMCR_LC
;
867 /* Enable long event counter support where available */
868 if (armv8pmu_has_long_event(cpu_pmu
))
869 pmcr
|= ARMV8_PMU_PMCR_LP
;
871 armv8pmu_pmcr_write(pmcr
);
874 static int __armv8_pmuv3_map_event(struct perf_event
*event
,
875 const unsigned (*extra_event_map
)
877 const unsigned (*extra_cache_map
)
878 [PERF_COUNT_HW_CACHE_MAX
]
879 [PERF_COUNT_HW_CACHE_OP_MAX
]
880 [PERF_COUNT_HW_CACHE_RESULT_MAX
])
883 struct arm_pmu
*armpmu
= to_arm_pmu(event
->pmu
);
885 hw_event_id
= armpmu_map_event(event
, &armv8_pmuv3_perf_map
,
886 &armv8_pmuv3_perf_cache_map
,
887 ARMV8_PMU_EVTYPE_EVENT
);
889 if (armv8pmu_event_is_64bit(event
))
890 event
->hw
.flags
|= ARMPMU_EVT_64BIT
;
892 /* Only expose micro/arch events supported by this PMU */
893 if ((hw_event_id
> 0) && (hw_event_id
< ARMV8_PMUV3_MAX_COMMON_EVENTS
)
894 && test_bit(hw_event_id
, armpmu
->pmceid_bitmap
)) {
898 return armpmu_map_event(event
, extra_event_map
, extra_cache_map
,
899 ARMV8_PMU_EVTYPE_EVENT
);
902 static int armv8_pmuv3_map_event(struct perf_event
*event
)
904 return __armv8_pmuv3_map_event(event
, NULL
, NULL
);
907 static int armv8_a53_map_event(struct perf_event
*event
)
909 return __armv8_pmuv3_map_event(event
, NULL
, &armv8_a53_perf_cache_map
);
912 static int armv8_a57_map_event(struct perf_event
*event
)
914 return __armv8_pmuv3_map_event(event
, NULL
, &armv8_a57_perf_cache_map
);
917 static int armv8_a73_map_event(struct perf_event
*event
)
919 return __armv8_pmuv3_map_event(event
, NULL
, &armv8_a73_perf_cache_map
);
922 static int armv8_thunder_map_event(struct perf_event
*event
)
924 return __armv8_pmuv3_map_event(event
, NULL
,
925 &armv8_thunder_perf_cache_map
);
928 static int armv8_vulcan_map_event(struct perf_event
*event
)
930 return __armv8_pmuv3_map_event(event
, NULL
,
931 &armv8_vulcan_perf_cache_map
);
934 struct armv8pmu_probe_info
{
939 static void __armv8pmu_probe_pmu(void *info
)
941 struct armv8pmu_probe_info
*probe
= info
;
942 struct arm_pmu
*cpu_pmu
= probe
->pmu
;
948 dfr0
= read_sysreg(id_aa64dfr0_el1
);
949 pmuver
= cpuid_feature_extract_unsigned_field(dfr0
,
950 ID_AA64DFR0_PMUVER_SHIFT
);
951 if (pmuver
== 0xf || pmuver
== 0)
954 cpu_pmu
->pmuver
= pmuver
;
955 probe
->present
= true;
957 /* Read the nb of CNTx counters supported from PMNC */
958 cpu_pmu
->num_events
= (armv8pmu_pmcr_read() >> ARMV8_PMU_PMCR_N_SHIFT
)
959 & ARMV8_PMU_PMCR_N_MASK
;
961 /* Add the CPU cycles counter */
962 cpu_pmu
->num_events
+= 1;
964 pmceid
[0] = pmceid_raw
[0] = read_sysreg(pmceid0_el0
);
965 pmceid
[1] = pmceid_raw
[1] = read_sysreg(pmceid1_el0
);
967 bitmap_from_arr32(cpu_pmu
->pmceid_bitmap
,
968 pmceid
, ARMV8_PMUV3_MAX_COMMON_EVENTS
);
970 pmceid
[0] = pmceid_raw
[0] >> 32;
971 pmceid
[1] = pmceid_raw
[1] >> 32;
973 bitmap_from_arr32(cpu_pmu
->pmceid_ext_bitmap
,
974 pmceid
, ARMV8_PMUV3_MAX_COMMON_EVENTS
);
977 static int armv8pmu_probe_pmu(struct arm_pmu
*cpu_pmu
)
979 struct armv8pmu_probe_info probe
= {
985 ret
= smp_call_function_any(&cpu_pmu
->supported_cpus
,
986 __armv8pmu_probe_pmu
,
991 return probe
.present
? 0 : -ENODEV
;
994 static int armv8_pmu_init(struct arm_pmu
*cpu_pmu
, char *name
,
995 int (*map_event
)(struct perf_event
*event
),
996 const struct attribute_group
*events
,
997 const struct attribute_group
*format
)
999 int ret
= armv8pmu_probe_pmu(cpu_pmu
);
1003 cpu_pmu
->handle_irq
= armv8pmu_handle_irq
;
1004 cpu_pmu
->enable
= armv8pmu_enable_event
;
1005 cpu_pmu
->disable
= armv8pmu_disable_event
;
1006 cpu_pmu
->read_counter
= armv8pmu_read_counter
;
1007 cpu_pmu
->write_counter
= armv8pmu_write_counter
;
1008 cpu_pmu
->get_event_idx
= armv8pmu_get_event_idx
;
1009 cpu_pmu
->clear_event_idx
= armv8pmu_clear_event_idx
;
1010 cpu_pmu
->start
= armv8pmu_start
;
1011 cpu_pmu
->stop
= armv8pmu_stop
;
1012 cpu_pmu
->reset
= armv8pmu_reset
;
1013 cpu_pmu
->set_event_filter
= armv8pmu_set_event_filter
;
1014 cpu_pmu
->filter_match
= armv8pmu_filter_match
;
1016 cpu_pmu
->name
= name
;
1017 cpu_pmu
->map_event
= map_event
;
1018 cpu_pmu
->attr_groups
[ARMPMU_ATTR_GROUP_EVENTS
] = events
?
1019 events
: &armv8_pmuv3_events_attr_group
;
1020 cpu_pmu
->attr_groups
[ARMPMU_ATTR_GROUP_FORMATS
] = format
?
1021 format
: &armv8_pmuv3_format_attr_group
;
1026 static int armv8_pmuv3_init(struct arm_pmu
*cpu_pmu
)
1028 return armv8_pmu_init(cpu_pmu
, "armv8_pmuv3",
1029 armv8_pmuv3_map_event
, NULL
, NULL
);
1032 static int armv8_a34_pmu_init(struct arm_pmu
*cpu_pmu
)
1034 return armv8_pmu_init(cpu_pmu
, "armv8_cortex_a34",
1035 armv8_pmuv3_map_event
, NULL
, NULL
);
1038 static int armv8_a35_pmu_init(struct arm_pmu
*cpu_pmu
)
1040 return armv8_pmu_init(cpu_pmu
, "armv8_cortex_a35",
1041 armv8_a53_map_event
, NULL
, NULL
);
1044 static int armv8_a53_pmu_init(struct arm_pmu
*cpu_pmu
)
1046 return armv8_pmu_init(cpu_pmu
, "armv8_cortex_a53",
1047 armv8_a53_map_event
, NULL
, NULL
);
1050 static int armv8_a55_pmu_init(struct arm_pmu
*cpu_pmu
)
1052 return armv8_pmu_init(cpu_pmu
, "armv8_cortex_a55",
1053 armv8_pmuv3_map_event
, NULL
, NULL
);
1056 static int armv8_a57_pmu_init(struct arm_pmu
*cpu_pmu
)
1058 return armv8_pmu_init(cpu_pmu
, "armv8_cortex_a57",
1059 armv8_a57_map_event
, NULL
, NULL
);
1062 static int armv8_a65_pmu_init(struct arm_pmu
*cpu_pmu
)
1064 return armv8_pmu_init(cpu_pmu
, "armv8_cortex_a65",
1065 armv8_pmuv3_map_event
, NULL
, NULL
);
1068 static int armv8_a72_pmu_init(struct arm_pmu
*cpu_pmu
)
1070 return armv8_pmu_init(cpu_pmu
, "armv8_cortex_a72",
1071 armv8_a57_map_event
, NULL
, NULL
);
1074 static int armv8_a73_pmu_init(struct arm_pmu
*cpu_pmu
)
1076 return armv8_pmu_init(cpu_pmu
, "armv8_cortex_a73",
1077 armv8_a73_map_event
, NULL
, NULL
);
1080 static int armv8_a75_pmu_init(struct arm_pmu
*cpu_pmu
)
1082 return armv8_pmu_init(cpu_pmu
, "armv8_cortex_a75",
1083 armv8_pmuv3_map_event
, NULL
, NULL
);
1086 static int armv8_a76_pmu_init(struct arm_pmu
*cpu_pmu
)
1088 return armv8_pmu_init(cpu_pmu
, "armv8_cortex_a76",
1089 armv8_pmuv3_map_event
, NULL
, NULL
);
1092 static int armv8_a77_pmu_init(struct arm_pmu
*cpu_pmu
)
1094 return armv8_pmu_init(cpu_pmu
, "armv8_cortex_a77",
1095 armv8_pmuv3_map_event
, NULL
, NULL
);
1098 static int armv8_e1_pmu_init(struct arm_pmu
*cpu_pmu
)
1100 return armv8_pmu_init(cpu_pmu
, "armv8_neoverse_e1",
1101 armv8_pmuv3_map_event
, NULL
, NULL
);
1104 static int armv8_n1_pmu_init(struct arm_pmu
*cpu_pmu
)
1106 return armv8_pmu_init(cpu_pmu
, "armv8_neoverse_n1",
1107 armv8_pmuv3_map_event
, NULL
, NULL
);
1110 static int armv8_thunder_pmu_init(struct arm_pmu
*cpu_pmu
)
1112 return armv8_pmu_init(cpu_pmu
, "armv8_cavium_thunder",
1113 armv8_thunder_map_event
, NULL
, NULL
);
1116 static int armv8_vulcan_pmu_init(struct arm_pmu
*cpu_pmu
)
1118 return armv8_pmu_init(cpu_pmu
, "armv8_brcm_vulcan",
1119 armv8_vulcan_map_event
, NULL
, NULL
);
1122 static const struct of_device_id armv8_pmu_of_device_ids
[] = {
1123 {.compatible
= "arm,armv8-pmuv3", .data
= armv8_pmuv3_init
},
1124 {.compatible
= "arm,cortex-a34-pmu", .data
= armv8_a34_pmu_init
},
1125 {.compatible
= "arm,cortex-a35-pmu", .data
= armv8_a35_pmu_init
},
1126 {.compatible
= "arm,cortex-a53-pmu", .data
= armv8_a53_pmu_init
},
1127 {.compatible
= "arm,cortex-a55-pmu", .data
= armv8_a55_pmu_init
},
1128 {.compatible
= "arm,cortex-a57-pmu", .data
= armv8_a57_pmu_init
},
1129 {.compatible
= "arm,cortex-a65-pmu", .data
= armv8_a65_pmu_init
},
1130 {.compatible
= "arm,cortex-a72-pmu", .data
= armv8_a72_pmu_init
},
1131 {.compatible
= "arm,cortex-a73-pmu", .data
= armv8_a73_pmu_init
},
1132 {.compatible
= "arm,cortex-a75-pmu", .data
= armv8_a75_pmu_init
},
1133 {.compatible
= "arm,cortex-a76-pmu", .data
= armv8_a76_pmu_init
},
1134 {.compatible
= "arm,cortex-a77-pmu", .data
= armv8_a77_pmu_init
},
1135 {.compatible
= "arm,neoverse-e1-pmu", .data
= armv8_e1_pmu_init
},
1136 {.compatible
= "arm,neoverse-n1-pmu", .data
= armv8_n1_pmu_init
},
1137 {.compatible
= "cavium,thunder-pmu", .data
= armv8_thunder_pmu_init
},
1138 {.compatible
= "brcm,vulcan-pmu", .data
= armv8_vulcan_pmu_init
},
1142 static int armv8_pmu_device_probe(struct platform_device
*pdev
)
1144 return arm_pmu_device_probe(pdev
, armv8_pmu_of_device_ids
, NULL
);
1147 static struct platform_driver armv8_pmu_driver
= {
1149 .name
= ARMV8_PMU_PDEV_NAME
,
1150 .of_match_table
= armv8_pmu_of_device_ids
,
1151 .suppress_bind_attrs
= true,
1153 .probe
= armv8_pmu_device_probe
,
1156 static int __init
armv8_pmu_driver_init(void)
1159 return platform_driver_register(&armv8_pmu_driver
);
1161 return arm_pmu_acpi_probe(armv8_pmuv3_init
);
1163 device_initcall(armv8_pmu_driver_init
)
1165 void arch_perf_update_userpage(struct perf_event
*event
,
1166 struct perf_event_mmap_page
*userpg
, u64 now
)
1172 * Internal timekeeping for enabled/running/stopped times
1173 * is always computed with the sched_clock.
1175 freq
= arch_timer_get_rate();
1176 userpg
->cap_user_time
= 1;
1178 clocks_calc_mult_shift(&userpg
->time_mult
, &shift
, freq
,
1181 * time_shift is not expected to be greater than 31 due to
1182 * the original published conversion algorithm shifting a
1183 * 32-bit value (now specifies a 64-bit value) - refer
1184 * perf_event_mmap_page documentation in perf_event.h.
1188 userpg
->time_mult
>>= 1;
1190 userpg
->time_shift
= (u16
)shift
;
1191 userpg
->time_offset
= -now
;