?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'v3.3.7' into 3.3/master
[zen-stable.git] / arch / arm / kernel / perf_event_v7.c
blob4d7095af2ab3a55075f080b3d4fee66b84d39f17
1 /*
2 * ARMv7 Cortex-A8 and Cortex-A9 Performance Events handling code.
3 *
4 * ARMv7 support: Jean Pihet <jpihet@mvista.com>
5 * 2010 (c) MontaVista Software, LLC.
6 *
7 * Copied from ARMv6 code, with the low level code inspired
8 * by the ARMv7 Oprofile code.
9 *
10 * Cortex-A8 has up to 4 configurable performance counters and
11 * a single cycle counter.
12 * Cortex-A9 has up to 31 configurable performance counters and
13 * a single cycle counter.
14 *
15 * All counters can be enabled/disabled and IRQ masked separately. The cycle
16 * counter and all 4 performance counters together can be reset separately.
17 */
18
19 #ifdef CONFIG_CPU_V7
20
21 static struct arm_pmu armv7pmu;
22
23 /*
24 * Common ARMv7 event types
25 *
26 * Note: An implementation may not be able to count all of these events
27 * but the encodings are considered to be `reserved' in the case that
28 * they are not available.
29 */
30 enum armv7_perf_types {
31 ARMV7_PERFCTR_PMNC_SW_INCR = 0x00,
32 ARMV7_PERFCTR_L1_ICACHE_REFILL = 0x01,
33 ARMV7_PERFCTR_ITLB_REFILL = 0x02,
34 ARMV7_PERFCTR_L1_DCACHE_REFILL = 0x03,
35 ARMV7_PERFCTR_L1_DCACHE_ACCESS = 0x04,
36 ARMV7_PERFCTR_DTLB_REFILL = 0x05,
37 ARMV7_PERFCTR_MEM_READ = 0x06,
38 ARMV7_PERFCTR_MEM_WRITE = 0x07,
39 ARMV7_PERFCTR_INSTR_EXECUTED = 0x08,
40 ARMV7_PERFCTR_EXC_TAKEN = 0x09,
41 ARMV7_PERFCTR_EXC_EXECUTED = 0x0A,
42 ARMV7_PERFCTR_CID_WRITE = 0x0B,
43
44 /*
45 * ARMV7_PERFCTR_PC_WRITE is equivalent to HW_BRANCH_INSTRUCTIONS.
46 * It counts:
47 * - all (taken) branch instructions,
48 * - instructions that explicitly write the PC,
49 * - exception generating instructions.
50 */
51 ARMV7_PERFCTR_PC_WRITE = 0x0C,
52 ARMV7_PERFCTR_PC_IMM_BRANCH = 0x0D,
53 ARMV7_PERFCTR_PC_PROC_RETURN = 0x0E,
54 ARMV7_PERFCTR_MEM_UNALIGNED_ACCESS = 0x0F,
55 ARMV7_PERFCTR_PC_BRANCH_MIS_PRED = 0x10,
56 ARMV7_PERFCTR_CLOCK_CYCLES = 0x11,
57 ARMV7_PERFCTR_PC_BRANCH_PRED = 0x12,
58
59 /* These events are defined by the PMUv2 supplement (ARM DDI 0457A). */
60 ARMV7_PERFCTR_MEM_ACCESS = 0x13,
61 ARMV7_PERFCTR_L1_ICACHE_ACCESS = 0x14,
62 ARMV7_PERFCTR_L1_DCACHE_WB = 0x15,
63 ARMV7_PERFCTR_L2_CACHE_ACCESS = 0x16,
64 ARMV7_PERFCTR_L2_CACHE_REFILL = 0x17,
65 ARMV7_PERFCTR_L2_CACHE_WB = 0x18,
66 ARMV7_PERFCTR_BUS_ACCESS = 0x19,
67 ARMV7_PERFCTR_MEM_ERROR = 0x1A,
68 ARMV7_PERFCTR_INSTR_SPEC = 0x1B,
69 ARMV7_PERFCTR_TTBR_WRITE = 0x1C,
70 ARMV7_PERFCTR_BUS_CYCLES = 0x1D,
71
72 ARMV7_PERFCTR_CPU_CYCLES = 0xFF
73 };
74
75 /* ARMv7 Cortex-A8 specific event types */
76 enum armv7_a8_perf_types {
77 ARMV7_A8_PERFCTR_L2_CACHE_ACCESS = 0x43,
78 ARMV7_A8_PERFCTR_L2_CACHE_REFILL = 0x44,
79 ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS = 0x50,
80 ARMV7_A8_PERFCTR_STALL_ISIDE = 0x56,
81 };
82
83 /* ARMv7 Cortex-A9 specific event types */
84 enum armv7_a9_perf_types {
85 ARMV7_A9_PERFCTR_INSTR_CORE_RENAME = 0x68,
86 ARMV7_A9_PERFCTR_STALL_ICACHE = 0x60,
87 ARMV7_A9_PERFCTR_STALL_DISPATCH = 0x66,
88 };
89
90 /* ARMv7 Cortex-A5 specific event types */
91 enum armv7_a5_perf_types {
92 ARMV7_A5_PERFCTR_PREFETCH_LINEFILL = 0xc2,
93 ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP = 0xc3,
94 };
95
96 /* ARMv7 Cortex-A15 specific event types */
97 enum armv7_a15_perf_types {
98 ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ = 0x40,
99 ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE = 0x41,
100 ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ = 0x42,
101 ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE = 0x43,
102
103 ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ = 0x4C,
104 ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE = 0x4D,
105
106 ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ = 0x50,
107 ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE = 0x51,
108 ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ = 0x52,
109 ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE = 0x53,
110
111 ARMV7_A15_PERFCTR_PC_WRITE_SPEC = 0x76,
112 };
113
114 /*
115 * Cortex-A8 HW events mapping
116 *
117 * The hardware events that we support. We do support cache operations but
118 * we have harvard caches and no way to combine instruction and data
119 * accesses/misses in hardware.
120 */
121 static const unsigned armv7_a8_perf_map[PERF_COUNT_HW_MAX] = {
122 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
123 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
124 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
125 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
126 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
127 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
128 [PERF_COUNT_HW_BUS_CYCLES] = HW_OP_UNSUPPORTED,
129 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A8_PERFCTR_STALL_ISIDE,
130 [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = HW_OP_UNSUPPORTED,
131 };
132
133 static const unsigned armv7_a8_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
134 [PERF_COUNT_HW_CACHE_OP_MAX]
135 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
136 [C(L1D)] = {
137 /*
138 * The performance counters don't differentiate between read
139 * and write accesses/misses so this isn't strictly correct,
140 * but it's the best we can do. Writes and reads get
141 * combined.
142 */
143 [C(OP_READ)] = {
144 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
145 [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
146 },
147 [C(OP_WRITE)] = {
148 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
149 [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
150 },
151 [C(OP_PREFETCH)] = {
152 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
153 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
154 },
155 },
156 [C(L1I)] = {
157 [C(OP_READ)] = {
158 [C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS,
159 [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
160 },
161 [C(OP_WRITE)] = {
162 [C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS,
163 [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
164 },
165 [C(OP_PREFETCH)] = {
166 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
167 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
168 },
169 },
170 [C(LL)] = {
171 [C(OP_READ)] = {
172 [C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,
173 [C(RESULT_MISS)] = ARMV7_A8_PERFCTR_L2_CACHE_REFILL,
174 },
175 [C(OP_WRITE)] = {
176 [C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,
177 [C(RESULT_MISS)] = ARMV7_A8_PERFCTR_L2_CACHE_REFILL,
178 },
179 [C(OP_PREFETCH)] = {
180 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
181 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
182 },
183 },
184 [C(DTLB)] = {
185 [C(OP_READ)] = {
186 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
187 [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
188 },
189 [C(OP_WRITE)] = {
190 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
191 [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
192 },
193 [C(OP_PREFETCH)] = {
194 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
195 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
196 },
197 },
198 [C(ITLB)] = {
199 [C(OP_READ)] = {
200 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
201 [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
202 },
203 [C(OP_WRITE)] = {
204 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
205 [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
206 },
207 [C(OP_PREFETCH)] = {
208 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
209 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
210 },
211 },
212 [C(BPU)] = {
213 [C(OP_READ)] = {
214 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
215 [C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
216 },
217 [C(OP_WRITE)] = {
218 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
219 [C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
220 },
221 [C(OP_PREFETCH)] = {
222 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
223 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
224 },
225 },
226 [C(NODE)] = {
227 [C(OP_READ)] = {
228 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
229 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
230 },
231 [C(OP_WRITE)] = {
232 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
233 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
234 },
235 [C(OP_PREFETCH)] = {
236 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
237 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
238 },
239 },
240 };
241
242 /*
243 * Cortex-A9 HW events mapping
244 */
245 static const unsigned armv7_a9_perf_map[PERF_COUNT_HW_MAX] = {
246 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
247 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_A9_PERFCTR_INSTR_CORE_RENAME,
248 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
249 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
250 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
251 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
252 [PERF_COUNT_HW_BUS_CYCLES] = HW_OP_UNSUPPORTED,
253 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A9_PERFCTR_STALL_ICACHE,
254 [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV7_A9_PERFCTR_STALL_DISPATCH,
255 };
256
257 static const unsigned armv7_a9_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
258 [PERF_COUNT_HW_CACHE_OP_MAX]
259 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
260 [C(L1D)] = {
261 /*
262 * The performance counters don't differentiate between read
263 * and write accesses/misses so this isn't strictly correct,
264 * but it's the best we can do. Writes and reads get
265 * combined.
266 */
267 [C(OP_READ)] = {
268 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
269 [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
270 },
271 [C(OP_WRITE)] = {
272 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
273 [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
274 },
275 [C(OP_PREFETCH)] = {
276 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
277 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
278 },
279 },
280 [C(L1I)] = {
281 [C(OP_READ)] = {
282 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
283 [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
284 },
285 [C(OP_WRITE)] = {
286 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
287 [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
288 },
289 [C(OP_PREFETCH)] = {
290 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
291 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
292 },
293 },
294 [C(LL)] = {
295 [C(OP_READ)] = {
296 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
297 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
298 },
299 [C(OP_WRITE)] = {
300 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
301 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
302 },
303 [C(OP_PREFETCH)] = {
304 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
305 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
306 },
307 },
308 [C(DTLB)] = {
309 [C(OP_READ)] = {
310 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
311 [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
312 },
313 [C(OP_WRITE)] = {
314 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
315 [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
316 },
317 [C(OP_PREFETCH)] = {
318 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
319 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
320 },
321 },
322 [C(ITLB)] = {
323 [C(OP_READ)] = {
324 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
325 [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
326 },
327 [C(OP_WRITE)] = {
328 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
329 [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
330 },
331 [C(OP_PREFETCH)] = {
332 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
333 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
334 },
335 },
336 [C(BPU)] = {
337 [C(OP_READ)] = {
338 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
339 [C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
340 },
341 [C(OP_WRITE)] = {
342 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
343 [C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
344 },
345 [C(OP_PREFETCH)] = {
346 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
347 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
348 },
349 },
350 [C(NODE)] = {
351 [C(OP_READ)] = {
352 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
353 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
354 },
355 [C(OP_WRITE)] = {
356 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
357 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
358 },
359 [C(OP_PREFETCH)] = {
360 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
361 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
362 },
363 },
364 };
365
366 /*
367 * Cortex-A5 HW events mapping
368 */
369 static const unsigned armv7_a5_perf_map[PERF_COUNT_HW_MAX] = {
370 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
371 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
372 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
373 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
374 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
375 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
376 [PERF_COUNT_HW_BUS_CYCLES] = HW_OP_UNSUPPORTED,
377 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = HW_OP_UNSUPPORTED,
378 [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = HW_OP_UNSUPPORTED,
379 };
380
381 static const unsigned armv7_a5_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
382 [PERF_COUNT_HW_CACHE_OP_MAX]
383 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
384 [C(L1D)] = {
385 [C(OP_READ)] = {
386 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
387 [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
388 },
389 [C(OP_WRITE)] = {
390 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
391 [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
392 },
393 [C(OP_PREFETCH)] = {
394 [C(RESULT_ACCESS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,
395 [C(RESULT_MISS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,
396 },
397 },
398 [C(L1I)] = {
399 [C(OP_READ)] = {
400 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
401 [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
402 },
403 [C(OP_WRITE)] = {
404 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
405 [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
406 },
407 /*
408 * The prefetch counters don't differentiate between the I
409 * side and the D side.
410 */
411 [C(OP_PREFETCH)] = {
412 [C(RESULT_ACCESS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,
413 [C(RESULT_MISS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,
414 },
415 },
416 [C(LL)] = {
417 [C(OP_READ)] = {
418 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
419 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
420 },
421 [C(OP_WRITE)] = {
422 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
423 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
424 },
425 [C(OP_PREFETCH)] = {
426 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
427 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
428 },
429 },
430 [C(DTLB)] = {
431 [C(OP_READ)] = {
432 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
433 [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
434 },
435 [C(OP_WRITE)] = {
436 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
437 [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
438 },
439 [C(OP_PREFETCH)] = {
440 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
441 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
442 },
443 },
444 [C(ITLB)] = {
445 [C(OP_READ)] = {
446 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
447 [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
448 },
449 [C(OP_WRITE)] = {
450 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
451 [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
452 },
453 [C(OP_PREFETCH)] = {
454 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
455 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
456 },
457 },
458 [C(BPU)] = {
459 [C(OP_READ)] = {
460 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
461 [C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
462 },
463 [C(OP_WRITE)] = {
464 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
465 [C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
466 },
467 [C(OP_PREFETCH)] = {
468 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
469 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
470 },
471 },
472 [C(NODE)] = {
473 [C(OP_READ)] = {
474 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
475 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
476 },
477 [C(OP_WRITE)] = {
478 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
479 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
480 },
481 [C(OP_PREFETCH)] = {
482 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
483 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
484 },
485 },
486 };
487
488 /*
489 * Cortex-A15 HW events mapping
490 */
491 static const unsigned armv7_a15_perf_map[PERF_COUNT_HW_MAX] = {
492 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
493 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
494 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
495 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
496 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_A15_PERFCTR_PC_WRITE_SPEC,
497 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
498 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_BUS_CYCLES,
499 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = HW_OP_UNSUPPORTED,
500 [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = HW_OP_UNSUPPORTED,
501 };
502
503 static const unsigned armv7_a15_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
504 [PERF_COUNT_HW_CACHE_OP_MAX]
505 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
506 [C(L1D)] = {
507 [C(OP_READ)] = {
508 [C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ,
509 [C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ,
510 },
511 [C(OP_WRITE)] = {
512 [C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE,
513 [C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE,
514 },
515 [C(OP_PREFETCH)] = {
516 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
517 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
518 },
519 },
520 [C(L1I)] = {
521 /*
522 * Not all performance counters differentiate between read
523 * and write accesses/misses so we're not always strictly
524 * correct, but it's the best we can do. Writes and reads get
525 * combined in these cases.
526 */
527 [C(OP_READ)] = {
528 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
529 [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
530 },
531 [C(OP_WRITE)] = {
532 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
533 [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
534 },
535 [C(OP_PREFETCH)] = {
536 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
537 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
538 },
539 },
540 [C(LL)] = {
541 [C(OP_READ)] = {
542 [C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ,
543 [C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ,
544 },
545 [C(OP_WRITE)] = {
546 [C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE,
547 [C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE,
548 },
549 [C(OP_PREFETCH)] = {
550 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
551 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
552 },
553 },
554 [C(DTLB)] = {
555 [C(OP_READ)] = {
556 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
557 [C(RESULT_MISS)] = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ,
558 },
559 [C(OP_WRITE)] = {
560 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
561 [C(RESULT_MISS)] = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE,
562 },
563 [C(OP_PREFETCH)] = {
564 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
565 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
566 },
567 },
568 [C(ITLB)] = {
569 [C(OP_READ)] = {
570 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
571 [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
572 },
573 [C(OP_WRITE)] = {
574 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
575 [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
576 },
577 [C(OP_PREFETCH)] = {
578 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
579 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
580 },
581 },
582 [C(BPU)] = {
583 [C(OP_READ)] = {
584 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
585 [C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
586 },
587 [C(OP_WRITE)] = {
588 [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
589 [C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
590 },
591 [C(OP_PREFETCH)] = {
592 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
593 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
594 },
595 },
596 [C(NODE)] = {
597 [C(OP_READ)] = {
598 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
599 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
600 },
601 [C(OP_WRITE)] = {
602 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
603 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
604 },
605 [C(OP_PREFETCH)] = {
606 [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
607 [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
608 },
609 },
610 };
611
612 /*
613 * Perf Events' indices
614 */
615 #define ARMV7_IDX_CYCLE_COUNTER 0
616 #define ARMV7_IDX_COUNTER0 1
617 #define ARMV7_IDX_COUNTER_LAST (ARMV7_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1)
618
619 #define ARMV7_MAX_COUNTERS 32
620 #define ARMV7_COUNTER_MASK (ARMV7_MAX_COUNTERS - 1)
621
622 /*
623 * ARMv7 low level PMNC access
624 */
625
626 /*
627 * Perf Event to low level counters mapping
628 */
629 #define ARMV7_IDX_TO_COUNTER(x) \
630 (((x) - ARMV7_IDX_COUNTER0) & ARMV7_COUNTER_MASK)
631
632 /*
633 * Per-CPU PMNC: config reg
634 */
635 #define ARMV7_PMNC_E (1 << 0) /* Enable all counters */
636 #define ARMV7_PMNC_P (1 << 1) /* Reset all counters */
637 #define ARMV7_PMNC_C (1 << 2) /* Cycle counter reset */
638 #define ARMV7_PMNC_D (1 << 3) /* CCNT counts every 64th cpu cycle */
639 #define ARMV7_PMNC_X (1 << 4) /* Export to ETM */
640 #define ARMV7_PMNC_DP (1 << 5) /* Disable CCNT if non-invasive debug*/
641 #define ARMV7_PMNC_N_SHIFT 11 /* Number of counters supported */
642 #define ARMV7_PMNC_N_MASK 0x1f
643 #define ARMV7_PMNC_MASK 0x3f /* Mask for writable bits */
644
645 /*
646 * FLAG: counters overflow flag status reg
647 */
648 #define ARMV7_FLAG_MASK 0xffffffff /* Mask for writable bits */
649 #define ARMV7_OVERFLOWED_MASK ARMV7_FLAG_MASK
650
651 /*
652 * PMXEVTYPER: Event selection reg
653 */
654 #define ARMV7_EVTYPE_MASK 0xc00000ff /* Mask for writable bits */
655 #define ARMV7_EVTYPE_EVENT 0xff /* Mask for EVENT bits */
656
657 /*
658 * Event filters for PMUv2
659 */
660 #define ARMV7_EXCLUDE_PL1 (1 << 31)
661 #define ARMV7_EXCLUDE_USER (1 << 30)
662 #define ARMV7_INCLUDE_HYP (1 << 27)
663
664 static inline u32 armv7_pmnc_read(void)
665 {
666 u32 val;
667 asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r"(val));
668 return val;
669 }
670
671 static inline void armv7_pmnc_write(u32 val)
672 {
673 val &= ARMV7_PMNC_MASK;
674 isb();
675 asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r"(val));
676 }
677
678 static inline int armv7_pmnc_has_overflowed(u32 pmnc)
679 {
680 return pmnc & ARMV7_OVERFLOWED_MASK;
681 }
682
683 static inline int armv7_pmnc_counter_valid(int idx)
684 {
685 return idx >= ARMV7_IDX_CYCLE_COUNTER && idx <= ARMV7_IDX_COUNTER_LAST;
686 }
687
688 static inline int armv7_pmnc_counter_has_overflowed(u32 pmnc, int idx)
689 {
690 int ret = 0;
691 u32 counter;
692
693 if (!armv7_pmnc_counter_valid(idx)) {
694 pr_err("CPU%u checking wrong counter %d overflow status\n",
695 smp_processor_id(), idx);
696 } else {
697 counter = ARMV7_IDX_TO_COUNTER(idx);
698 ret = pmnc & BIT(counter);
699 }
700
701 return ret;
702 }
703
704 static inline int armv7_pmnc_select_counter(int idx)
705 {
706 u32 counter;
707
708 if (!armv7_pmnc_counter_valid(idx)) {
709 pr_err("CPU%u selecting wrong PMNC counter %d\n",
710 smp_processor_id(), idx);
711 return -EINVAL;
712 }
713
714 counter = ARMV7_IDX_TO_COUNTER(idx);
715 asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (counter));
716 isb();
717
718 return idx;
719 }
720
721 static inline u32 armv7pmu_read_counter(int idx)
722 {
723 u32 value = 0;
724
725 if (!armv7_pmnc_counter_valid(idx))
726 pr_err("CPU%u reading wrong counter %d\n",
727 smp_processor_id(), idx);
728 else if (idx == ARMV7_IDX_CYCLE_COUNTER)
729 asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (value));
730 else if (armv7_pmnc_select_counter(idx) == idx)
731 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (value));
732
733 return value;
734 }
735
736 static inline void armv7pmu_write_counter(int idx, u32 value)
737 {
738 if (!armv7_pmnc_counter_valid(idx))
739 pr_err("CPU%u writing wrong counter %d\n",
740 smp_processor_id(), idx);
741 else if (idx == ARMV7_IDX_CYCLE_COUNTER)
742 asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (value));
743 else if (armv7_pmnc_select_counter(idx) == idx)
744 asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" (value));
745 }
746
747 static inline void armv7_pmnc_write_evtsel(int idx, u32 val)
748 {
749 if (armv7_pmnc_select_counter(idx) == idx) {
750 val &= ARMV7_EVTYPE_MASK;
751 asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val));
752 }
753 }
754
755 static inline int armv7_pmnc_enable_counter(int idx)
756 {
757 u32 counter;
758
759 if (!armv7_pmnc_counter_valid(idx)) {
760 pr_err("CPU%u enabling wrong PMNC counter %d\n",
761 smp_processor_id(), idx);
762 return -EINVAL;
763 }
764
765 counter = ARMV7_IDX_TO_COUNTER(idx);
766 asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (BIT(counter)));
767 return idx;
768 }
769
770 static inline int armv7_pmnc_disable_counter(int idx)
771 {
772 u32 counter;
773
774 if (!armv7_pmnc_counter_valid(idx)) {
775 pr_err("CPU%u disabling wrong PMNC counter %d\n",
776 smp_processor_id(), idx);
777 return -EINVAL;
778 }
779
780 counter = ARMV7_IDX_TO_COUNTER(idx);
781 asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (BIT(counter)));
782 return idx;
783 }
784
785 static inline int armv7_pmnc_enable_intens(int idx)
786 {
787 u32 counter;
788
789 if (!armv7_pmnc_counter_valid(idx)) {
790 pr_err("CPU%u enabling wrong PMNC counter IRQ enable %d\n",
791 smp_processor_id(), idx);
792 return -EINVAL;
793 }
794
795 counter = ARMV7_IDX_TO_COUNTER(idx);
796 asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (BIT(counter)));
797 return idx;
798 }
799
800 static inline int armv7_pmnc_disable_intens(int idx)
801 {
802 u32 counter;
803
804 if (!armv7_pmnc_counter_valid(idx)) {
805 pr_err("CPU%u disabling wrong PMNC counter IRQ enable %d\n",
806 smp_processor_id(), idx);
807 return -EINVAL;
808 }
809
810 counter = ARMV7_IDX_TO_COUNTER(idx);
811 asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (BIT(counter)));
812 isb();
813 /* Clear the overflow flag in case an interrupt is pending. */
814 asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (BIT(counter)));
815 isb();
816
817 return idx;
818 }
819
820 static inline u32 armv7_pmnc_getreset_flags(void)
821 {
822 u32 val;
823
824 /* Read */
825 asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
826
827 /* Write to clear flags */
828 val &= ARMV7_FLAG_MASK;
829 asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (val));
830
831 return val;
832 }
833
834 #ifdef DEBUG
835 static void armv7_pmnc_dump_regs(void)
836 {
837 u32 val;
838 unsigned int cnt;
839
840 printk(KERN_INFO "PMNC registers dump:\n");
841
842 asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val));
843 printk(KERN_INFO "PMNC =0x%08x\n", val);
844
845 asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r" (val));
846 printk(KERN_INFO "CNTENS=0x%08x\n", val);
847
848 asm volatile("mrc p15, 0, %0, c9, c14, 1" : "=r" (val));
849 printk(KERN_INFO "INTENS=0x%08x\n", val);
850
851 asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
852 printk(KERN_INFO "FLAGS =0x%08x\n", val);
853
854 asm volatile("mrc p15, 0, %0, c9, c12, 5" : "=r" (val));
855 printk(KERN_INFO "SELECT=0x%08x\n", val);
856
857 asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val));
858 printk(KERN_INFO "CCNT =0x%08x\n", val);
859
860 for (cnt = ARMV7_IDX_COUNTER0; cnt <= ARMV7_IDX_COUNTER_LAST; cnt++) {
861 armv7_pmnc_select_counter(cnt);
862 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val));
863 printk(KERN_INFO "CNT[%d] count =0x%08x\n",
864 ARMV7_IDX_TO_COUNTER(cnt), val);
865 asm volatile("mrc p15, 0, %0, c9, c13, 1" : "=r" (val));
866 printk(KERN_INFO "CNT[%d] evtsel=0x%08x\n",
867 ARMV7_IDX_TO_COUNTER(cnt), val);
868 }
869 }
870 #endif
871
872 static void armv7pmu_enable_event(struct hw_perf_event *hwc, int idx)
873 {
874 unsigned long flags;
875 struct pmu_hw_events *events = cpu_pmu->get_hw_events();
876
877 /*
878 * Enable counter and interrupt, and set the counter to count
879 * the event that we're interested in.
880 */
881 raw_spin_lock_irqsave(&events->pmu_lock, flags);
882
883 /*
884 * Disable counter
885 */
886 armv7_pmnc_disable_counter(idx);
887
888 /*
889 * Set event (if destined for PMNx counters)
890 * We only need to set the event for the cycle counter if we
891 * have the ability to perform event filtering.
892 */
893 if (armv7pmu.set_event_filter || idx != ARMV7_IDX_CYCLE_COUNTER)
894 armv7_pmnc_write_evtsel(idx, hwc->config_base);
895
896 /*
897 * Enable interrupt for this counter
898 */
899 armv7_pmnc_enable_intens(idx);
900
901 /*
902 * Enable counter
903 */
904 armv7_pmnc_enable_counter(idx);
905
906 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
907 }
908
909 static void armv7pmu_disable_event(struct hw_perf_event *hwc, int idx)
910 {
911 unsigned long flags;
912 struct pmu_hw_events *events = cpu_pmu->get_hw_events();
913
914 /*
915 * Disable counter and interrupt
916 */
917 raw_spin_lock_irqsave(&events->pmu_lock, flags);
918
919 /*
920 * Disable counter
921 */
922 armv7_pmnc_disable_counter(idx);
923
924 /*
925 * Disable interrupt for this counter
926 */
927 armv7_pmnc_disable_intens(idx);
928
929 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
930 }
931
932 static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev)
933 {
934 u32 pmnc;
935 struct perf_sample_data data;
936 struct pmu_hw_events *cpuc;
937 struct pt_regs *regs;
938 int idx;
939
940 /*
941 * Get and reset the IRQ flags
942 */
943 pmnc = armv7_pmnc_getreset_flags();
944
945 /*
946 * Did an overflow occur?
947 */
948 if (!armv7_pmnc_has_overflowed(pmnc))
949 return IRQ_NONE;
950
951 /*
952 * Handle the counter(s) overflow(s)
953 */
954 regs = get_irq_regs();
955
956 perf_sample_data_init(&data, 0);
957
958 cpuc = &__get_cpu_var(cpu_hw_events);
959 for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
960 struct perf_event *event = cpuc->events[idx];
961 struct hw_perf_event *hwc;
962
963 /* Ignore if we don't have an event. */
964 if (!event)
965 continue;
966
967 /*
968 * We have a single interrupt for all counters. Check that
969 * each counter has overflowed before we process it.
970 */
971 if (!armv7_pmnc_counter_has_overflowed(pmnc, idx))
972 continue;
973
974 hwc = &event->hw;
975 armpmu_event_update(event, hwc, idx);
976 data.period = event->hw.last_period;
977 if (!armpmu_event_set_period(event, hwc, idx))
978 continue;
979
980 if (perf_event_overflow(event, &data, regs))
981 cpu_pmu->disable(hwc, idx);
982 }
983
984 /*
985 * Handle the pending perf events.
986 *
987 * Note: this call *must* be run with interrupts disabled. For
988 * platforms that can have the PMU interrupts raised as an NMI, this
989 * will not work.
990 */
991 irq_work_run();
992
993 return IRQ_HANDLED;
994 }
995
996 static void armv7pmu_start(void)
997 {
998 unsigned long flags;
999 struct pmu_hw_events *events = cpu_pmu->get_hw_events();
1000
1001 raw_spin_lock_irqsave(&events->pmu_lock, flags);
1002 /* Enable all counters */
1003 armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E);
1004 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1005 }
1006
1007 static void armv7pmu_stop(void)
1008 {
1009 unsigned long flags;
1010 struct pmu_hw_events *events = cpu_pmu->get_hw_events();
1011
1012 raw_spin_lock_irqsave(&events->pmu_lock, flags);
1013 /* Disable all counters */
1014 armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E);
1015 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1016 }
1017
1018 static int armv7pmu_get_event_idx(struct pmu_hw_events *cpuc,
1019 struct hw_perf_event *event)
1020 {
1021 int idx;
1022 unsigned long evtype = event->config_base & ARMV7_EVTYPE_EVENT;
1023
1024 /* Always place a cycle counter into the cycle counter. */
1025 if (evtype == ARMV7_PERFCTR_CPU_CYCLES) {
1026 if (test_and_set_bit(ARMV7_IDX_CYCLE_COUNTER, cpuc->used_mask))
1027 return -EAGAIN;
1028
1029 return ARMV7_IDX_CYCLE_COUNTER;
1030 }
1031
1032 /*
1033 * For anything other than a cycle counter, try and use
1034 * the events counters
1035 */
1036 for (idx = ARMV7_IDX_COUNTER0; idx < cpu_pmu->num_events; ++idx) {
1037 if (!test_and_set_bit(idx, cpuc->used_mask))
1038 return idx;
1039 }
1040
1041 /* The counters are all in use. */
1042 return -EAGAIN;
1043 }
1044
1045 /*
1046 * Add an event filter to a given event. This will only work for PMUv2 PMUs.
1047 */
1048 static int armv7pmu_set_event_filter(struct hw_perf_event *event,
1049 struct perf_event_attr *attr)
1050 {
1051 unsigned long config_base = 0;
1052
1053 if (attr->exclude_idle)
1054 return -EPERM;
1055 if (attr->exclude_user)
1056 config_base |= ARMV7_EXCLUDE_USER;
1057 if (attr->exclude_kernel)
1058 config_base |= ARMV7_EXCLUDE_PL1;
1059 if (!attr->exclude_hv)
1060 config_base |= ARMV7_INCLUDE_HYP;
1061
1062 /*
1063 * Install the filter into config_base as this is used to
1064 * construct the event type.
1065 */
1066 event->config_base = config_base;
1067
1068 return 0;
1069 }
1070
1071 static void armv7pmu_reset(void *info)
1072 {
1073 u32 idx, nb_cnt = cpu_pmu->num_events;
1074
1075 /* The counter and interrupt enable registers are unknown at reset. */
1076 for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx)
1077 armv7pmu_disable_event(NULL, idx);
1078
1079 /* Initialize & Reset PMNC: C and P bits */
1080 armv7_pmnc_write(ARMV7_PMNC_P | ARMV7_PMNC_C);
1081 }
1082
1083 static int armv7_a8_map_event(struct perf_event *event)
1084 {
1085 return map_cpu_event(event, &armv7_a8_perf_map,
1086 &armv7_a8_perf_cache_map, 0xFF);
1087 }
1088
1089 static int armv7_a9_map_event(struct perf_event *event)
1090 {
1091 return map_cpu_event(event, &armv7_a9_perf_map,
1092 &armv7_a9_perf_cache_map, 0xFF);
1093 }
1094
1095 static int armv7_a5_map_event(struct perf_event *event)
1096 {
1097 return map_cpu_event(event, &armv7_a5_perf_map,
1098 &armv7_a5_perf_cache_map, 0xFF);
1099 }
1100
1101 static int armv7_a15_map_event(struct perf_event *event)
1102 {
1103 return map_cpu_event(event, &armv7_a15_perf_map,
1104 &armv7_a15_perf_cache_map, 0xFF);
1105 }
1106
1107 static struct arm_pmu armv7pmu = {
1108 .handle_irq = armv7pmu_handle_irq,
1109 .enable = armv7pmu_enable_event,
1110 .disable = armv7pmu_disable_event,
1111 .read_counter = armv7pmu_read_counter,
1112 .write_counter = armv7pmu_write_counter,
1113 .get_event_idx = armv7pmu_get_event_idx,
1114 .start = armv7pmu_start,
1115 .stop = armv7pmu_stop,
1116 .reset = armv7pmu_reset,
1117 .max_period = (1LLU << 32) - 1,
1118 };
1119
1120 static u32 __init armv7_read_num_pmnc_events(void)
1121 {
1122 u32 nb_cnt;
1123
1124 /* Read the nb of CNTx counters supported from PMNC */
1125 nb_cnt = (armv7_pmnc_read() >> ARMV7_PMNC_N_SHIFT) & ARMV7_PMNC_N_MASK;
1126
1127 /* Add the CPU cycles counter and return */
1128 return nb_cnt + 1;
1129 }
1130
1131 static struct arm_pmu *__init armv7_a8_pmu_init(void)
1132 {
1133 armv7pmu.id = ARM_PERF_PMU_ID_CA8;
1134 armv7pmu.name = "ARMv7 Cortex-A8";
1135 armv7pmu.map_event = armv7_a8_map_event;
1136 armv7pmu.num_events = armv7_read_num_pmnc_events();
1137 return &armv7pmu;
1138 }
1139
1140 static struct arm_pmu *__init armv7_a9_pmu_init(void)
1141 {
1142 armv7pmu.id = ARM_PERF_PMU_ID_CA9;
1143 armv7pmu.name = "ARMv7 Cortex-A9";
1144 armv7pmu.map_event = armv7_a9_map_event;
1145 armv7pmu.num_events = armv7_read_num_pmnc_events();
1146 return &armv7pmu;
1147 }
1148
1149 static struct arm_pmu *__init armv7_a5_pmu_init(void)
1150 {
1151 armv7pmu.id = ARM_PERF_PMU_ID_CA5;
1152 armv7pmu.name = "ARMv7 Cortex-A5";
1153 armv7pmu.map_event = armv7_a5_map_event;
1154 armv7pmu.num_events = armv7_read_num_pmnc_events();
1155 return &armv7pmu;
1156 }
1157
1158 static struct arm_pmu *__init armv7_a15_pmu_init(void)
1159 {
1160 armv7pmu.id = ARM_PERF_PMU_ID_CA15;
1161 armv7pmu.name = "ARMv7 Cortex-A15";
1162 armv7pmu.map_event = armv7_a15_map_event;
1163 armv7pmu.num_events = armv7_read_num_pmnc_events();
1164 armv7pmu.set_event_filter = armv7pmu_set_event_filter;
1165 return &armv7pmu;
1166 }
1167 #else
1168 static struct arm_pmu *__init armv7_a8_pmu_init(void)
1169 {
1170 return NULL;
1171 }
1172
1173 static struct arm_pmu *__init armv7_a9_pmu_init(void)
1174 {
1175 return NULL;
1176 }
1177
1178 static struct arm_pmu *__init armv7_a5_pmu_init(void)
1179 {
1180 return NULL;
1181 }
1182
1183 static struct arm_pmu *__init armv7_a15_pmu_init(void)
1184 {
1185 return NULL;
1186 }
1187 #endif /* CONFIG_CPU_V7 */