?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 4.1.18
[linux/fpc-iii.git] / arch / arm / kernel / perf_event_v7.c
blobf4207a4dcb013ca4b19b42e3d3c2b5ec9d620fde
1 /*
2 * ARMv7 Cortex-A8 and Cortex-A9 Performance Events handling code.
3 *
4 * ARMv7 support: Jean Pihet <jpihet@mvista.com>
5 * 2010 (c) MontaVista Software, LLC.
6 *
7 * Copied from ARMv6 code, with the low level code inspired
8 * by the ARMv7 Oprofile code.
9 *
10 * Cortex-A8 has up to 4 configurable performance counters and
11 * a single cycle counter.
12 * Cortex-A9 has up to 31 configurable performance counters and
13 * a single cycle counter.
14 *
15 * All counters can be enabled/disabled and IRQ masked separately. The cycle
16 * counter and all 4 performance counters together can be reset separately.
17 */
18
19 #ifdef CONFIG_CPU_V7
20
21 #include <asm/cp15.h>
22 #include <asm/vfp.h>
23 #include "../vfp/vfpinstr.h"
24
25 /*
26 * Common ARMv7 event types
27 *
28 * Note: An implementation may not be able to count all of these events
29 * but the encodings are considered to be `reserved' in the case that
30 * they are not available.
31 */
32 enum armv7_perf_types {
33 ARMV7_PERFCTR_PMNC_SW_INCR = 0x00,
34 ARMV7_PERFCTR_L1_ICACHE_REFILL = 0x01,
35 ARMV7_PERFCTR_ITLB_REFILL = 0x02,
36 ARMV7_PERFCTR_L1_DCACHE_REFILL = 0x03,
37 ARMV7_PERFCTR_L1_DCACHE_ACCESS = 0x04,
38 ARMV7_PERFCTR_DTLB_REFILL = 0x05,
39 ARMV7_PERFCTR_MEM_READ = 0x06,
40 ARMV7_PERFCTR_MEM_WRITE = 0x07,
41 ARMV7_PERFCTR_INSTR_EXECUTED = 0x08,
42 ARMV7_PERFCTR_EXC_TAKEN = 0x09,
43 ARMV7_PERFCTR_EXC_EXECUTED = 0x0A,
44 ARMV7_PERFCTR_CID_WRITE = 0x0B,
45
46 /*
47 * ARMV7_PERFCTR_PC_WRITE is equivalent to HW_BRANCH_INSTRUCTIONS.
48 * It counts:
49 * - all (taken) branch instructions,
50 * - instructions that explicitly write the PC,
51 * - exception generating instructions.
52 */
53 ARMV7_PERFCTR_PC_WRITE = 0x0C,
54 ARMV7_PERFCTR_PC_IMM_BRANCH = 0x0D,
55 ARMV7_PERFCTR_PC_PROC_RETURN = 0x0E,
56 ARMV7_PERFCTR_MEM_UNALIGNED_ACCESS = 0x0F,
57 ARMV7_PERFCTR_PC_BRANCH_MIS_PRED = 0x10,
58 ARMV7_PERFCTR_CLOCK_CYCLES = 0x11,
59 ARMV7_PERFCTR_PC_BRANCH_PRED = 0x12,
60
61 /* These events are defined by the PMUv2 supplement (ARM DDI 0457A). */
62 ARMV7_PERFCTR_MEM_ACCESS = 0x13,
63 ARMV7_PERFCTR_L1_ICACHE_ACCESS = 0x14,
64 ARMV7_PERFCTR_L1_DCACHE_WB = 0x15,
65 ARMV7_PERFCTR_L2_CACHE_ACCESS = 0x16,
66 ARMV7_PERFCTR_L2_CACHE_REFILL = 0x17,
67 ARMV7_PERFCTR_L2_CACHE_WB = 0x18,
68 ARMV7_PERFCTR_BUS_ACCESS = 0x19,
69 ARMV7_PERFCTR_MEM_ERROR = 0x1A,
70 ARMV7_PERFCTR_INSTR_SPEC = 0x1B,
71 ARMV7_PERFCTR_TTBR_WRITE = 0x1C,
72 ARMV7_PERFCTR_BUS_CYCLES = 0x1D,
73
74 ARMV7_PERFCTR_CPU_CYCLES = 0xFF
75 };
76
77 /* ARMv7 Cortex-A8 specific event types */
78 enum armv7_a8_perf_types {
79 ARMV7_A8_PERFCTR_L2_CACHE_ACCESS = 0x43,
80 ARMV7_A8_PERFCTR_L2_CACHE_REFILL = 0x44,
81 ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS = 0x50,
82 ARMV7_A8_PERFCTR_STALL_ISIDE = 0x56,
83 };
84
85 /* ARMv7 Cortex-A9 specific event types */
86 enum armv7_a9_perf_types {
87 ARMV7_A9_PERFCTR_INSTR_CORE_RENAME = 0x68,
88 ARMV7_A9_PERFCTR_STALL_ICACHE = 0x60,
89 ARMV7_A9_PERFCTR_STALL_DISPATCH = 0x66,
90 };
91
92 /* ARMv7 Cortex-A5 specific event types */
93 enum armv7_a5_perf_types {
94 ARMV7_A5_PERFCTR_PREFETCH_LINEFILL = 0xc2,
95 ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP = 0xc3,
96 };
97
98 /* ARMv7 Cortex-A15 specific event types */
99 enum armv7_a15_perf_types {
100 ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ = 0x40,
101 ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE = 0x41,
102 ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ = 0x42,
103 ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE = 0x43,
104
105 ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ = 0x4C,
106 ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE = 0x4D,
107
108 ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ = 0x50,
109 ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE = 0x51,
110 ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ = 0x52,
111 ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE = 0x53,
112
113 ARMV7_A15_PERFCTR_PC_WRITE_SPEC = 0x76,
114 };
115
116 /* ARMv7 Cortex-A12 specific event types */
117 enum armv7_a12_perf_types {
118 ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ = 0x40,
119 ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE = 0x41,
120
121 ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ = 0x50,
122 ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE = 0x51,
123
124 ARMV7_A12_PERFCTR_PC_WRITE_SPEC = 0x76,
125
126 ARMV7_A12_PERFCTR_PF_TLB_REFILL = 0xe7,
127 };
128
129 /* ARMv7 Krait specific event types */
130 enum krait_perf_types {
131 KRAIT_PMRESR0_GROUP0 = 0xcc,
132 KRAIT_PMRESR1_GROUP0 = 0xd0,
133 KRAIT_PMRESR2_GROUP0 = 0xd4,
134 KRAIT_VPMRESR0_GROUP0 = 0xd8,
135
136 KRAIT_PERFCTR_L1_ICACHE_ACCESS = 0x10011,
137 KRAIT_PERFCTR_L1_ICACHE_MISS = 0x10010,
138
139 KRAIT_PERFCTR_L1_ITLB_ACCESS = 0x12222,
140 KRAIT_PERFCTR_L1_DTLB_ACCESS = 0x12210,
141 };
142
143 /* ARMv7 Scorpion specific event types */
144 enum scorpion_perf_types {
145 SCORPION_LPM0_GROUP0 = 0x4c,
146 SCORPION_LPM1_GROUP0 = 0x50,
147 SCORPION_LPM2_GROUP0 = 0x54,
148 SCORPION_L2LPM_GROUP0 = 0x58,
149 SCORPION_VLPM_GROUP0 = 0x5c,
150
151 SCORPION_ICACHE_ACCESS = 0x10053,
152 SCORPION_ICACHE_MISS = 0x10052,
153
154 SCORPION_DTLB_ACCESS = 0x12013,
155 SCORPION_DTLB_MISS = 0x12012,
156
157 SCORPION_ITLB_MISS = 0x12021,
158 };
159
160 /*
161 * Cortex-A8 HW events mapping
162 *
163 * The hardware events that we support. We do support cache operations but
164 * we have harvard caches and no way to combine instruction and data
165 * accesses/misses in hardware.
166 */
167 static const unsigned armv7_a8_perf_map[PERF_COUNT_HW_MAX] = {
168 PERF_MAP_ALL_UNSUPPORTED,
169 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
170 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
171 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
172 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
173 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
174 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
175 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A8_PERFCTR_STALL_ISIDE,
176 };
177
178 static const unsigned armv7_a8_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
179 [PERF_COUNT_HW_CACHE_OP_MAX]
180 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
181 PERF_CACHE_MAP_ALL_UNSUPPORTED,
182
183 /*
184 * The performance counters don't differentiate between read and write
185 * accesses/misses so this isn't strictly correct, but it's the best we
186 * can do. Writes and reads get combined.
187 */
188 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
189 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
190 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
191 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
192
193 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS,
194 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
195
196 [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,
197 [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A8_PERFCTR_L2_CACHE_REFILL,
198 [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,
199 [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A8_PERFCTR_L2_CACHE_REFILL,
200
201 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
202 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
203
204 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
205 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
206
207 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
208 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
209 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
210 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
211 };
212
213 /*
214 * Cortex-A9 HW events mapping
215 */
216 static const unsigned armv7_a9_perf_map[PERF_COUNT_HW_MAX] = {
217 PERF_MAP_ALL_UNSUPPORTED,
218 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
219 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_A9_PERFCTR_INSTR_CORE_RENAME,
220 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
221 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
222 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
223 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
224 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A9_PERFCTR_STALL_ICACHE,
225 [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV7_A9_PERFCTR_STALL_DISPATCH,
226 };
227
228 static const unsigned armv7_a9_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
229 [PERF_COUNT_HW_CACHE_OP_MAX]
230 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
231 PERF_CACHE_MAP_ALL_UNSUPPORTED,
232
233 /*
234 * The performance counters don't differentiate between read and write
235 * accesses/misses so this isn't strictly correct, but it's the best we
236 * can do. Writes and reads get combined.
237 */
238 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
239 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
240 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
241 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
242
243 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
244
245 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
246 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
247
248 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
249 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
250
251 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
252 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
253 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
254 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
255 };
256
257 /*
258 * Cortex-A5 HW events mapping
259 */
260 static const unsigned armv7_a5_perf_map[PERF_COUNT_HW_MAX] = {
261 PERF_MAP_ALL_UNSUPPORTED,
262 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
263 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
264 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
265 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
266 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
267 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
268 };
269
270 static const unsigned armv7_a5_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
271 [PERF_COUNT_HW_CACHE_OP_MAX]
272 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
273 PERF_CACHE_MAP_ALL_UNSUPPORTED,
274
275 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
276 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
277 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
278 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
279 [C(L1D)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,
280 [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,
281
282 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
283 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
284 /*
285 * The prefetch counters don't differentiate between the I side and the
286 * D side.
287 */
288 [C(L1I)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,
289 [C(L1I)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,
290
291 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
292 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
293
294 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
295 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
296
297 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
298 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
299 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
300 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
301 };
302
303 /*
304 * Cortex-A15 HW events mapping
305 */
306 static const unsigned armv7_a15_perf_map[PERF_COUNT_HW_MAX] = {
307 PERF_MAP_ALL_UNSUPPORTED,
308 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
309 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
310 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
311 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
312 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_A15_PERFCTR_PC_WRITE_SPEC,
313 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
314 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_BUS_CYCLES,
315 };
316
317 static const unsigned armv7_a15_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
318 [PERF_COUNT_HW_CACHE_OP_MAX]
319 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
320 PERF_CACHE_MAP_ALL_UNSUPPORTED,
321
322 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ,
323 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ,
324 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE,
325 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE,
326
327 /*
328 * Not all performance counters differentiate between read and write
329 * accesses/misses so we're not always strictly correct, but it's the
330 * best we can do. Writes and reads get combined in these cases.
331 */
332 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
333 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
334
335 [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ,
336 [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ,
337 [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE,
338 [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE,
339
340 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ,
341 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE,
342
343 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
344 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
345
346 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
347 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
348 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
349 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
350 };
351
352 /*
353 * Cortex-A7 HW events mapping
354 */
355 static const unsigned armv7_a7_perf_map[PERF_COUNT_HW_MAX] = {
356 PERF_MAP_ALL_UNSUPPORTED,
357 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
358 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
359 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
360 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
361 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
362 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
363 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_BUS_CYCLES,
364 };
365
366 static const unsigned armv7_a7_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
367 [PERF_COUNT_HW_CACHE_OP_MAX]
368 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
369 PERF_CACHE_MAP_ALL_UNSUPPORTED,
370
371 /*
372 * The performance counters don't differentiate between read and write
373 * accesses/misses so this isn't strictly correct, but it's the best we
374 * can do. Writes and reads get combined.
375 */
376 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
377 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
378 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
379 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
380
381 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
382 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
383
384 [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L2_CACHE_ACCESS,
385 [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL,
386 [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L2_CACHE_ACCESS,
387 [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL,
388
389 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
390 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
391
392 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
393 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
394
395 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
396 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
397 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
398 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
399 };
400
401 /*
402 * Cortex-A12 HW events mapping
403 */
404 static const unsigned armv7_a12_perf_map[PERF_COUNT_HW_MAX] = {
405 PERF_MAP_ALL_UNSUPPORTED,
406 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
407 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
408 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
409 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
410 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_A12_PERFCTR_PC_WRITE_SPEC,
411 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
412 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_BUS_CYCLES,
413 };
414
415 static const unsigned armv7_a12_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
416 [PERF_COUNT_HW_CACHE_OP_MAX]
417 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
418 PERF_CACHE_MAP_ALL_UNSUPPORTED,
419
420 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ,
421 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
422 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE,
423 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
424
425 /*
426 * Not all performance counters differentiate between read and write
427 * accesses/misses so we're not always strictly correct, but it's the
428 * best we can do. Writes and reads get combined in these cases.
429 */
430 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
431 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
432
433 [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ,
434 [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL,
435 [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE,
436 [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL,
437
438 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
439 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
440 [C(DTLB)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV7_A12_PERFCTR_PF_TLB_REFILL,
441
442 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
443 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
444
445 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
446 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
447 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
448 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
449 };
450
451 /*
452 * Krait HW events mapping
453 */
454 static const unsigned krait_perf_map[PERF_COUNT_HW_MAX] = {
455 PERF_MAP_ALL_UNSUPPORTED,
456 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
457 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
458 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
459 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
460 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES,
461 };
462
463 static const unsigned krait_perf_map_no_branch[PERF_COUNT_HW_MAX] = {
464 PERF_MAP_ALL_UNSUPPORTED,
465 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
466 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
467 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
468 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES,
469 };
470
471 static const unsigned krait_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
472 [PERF_COUNT_HW_CACHE_OP_MAX]
473 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
474 PERF_CACHE_MAP_ALL_UNSUPPORTED,
475
476 /*
477 * The performance counters don't differentiate between read and write
478 * accesses/misses so this isn't strictly correct, but it's the best we
479 * can do. Writes and reads get combined.
480 */
481 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
482 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
483 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
484 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
485
486 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ICACHE_ACCESS,
487 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = KRAIT_PERFCTR_L1_ICACHE_MISS,
488
489 [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_DTLB_ACCESS,
490 [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_DTLB_ACCESS,
491
492 [C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ITLB_ACCESS,
493 [C(ITLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ITLB_ACCESS,
494
495 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
496 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
497 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
498 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
499 };
500
501 /*
502 * Scorpion HW events mapping
503 */
504 static const unsigned scorpion_perf_map[PERF_COUNT_HW_MAX] = {
505 PERF_MAP_ALL_UNSUPPORTED,
506 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
507 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
508 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
509 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
510 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES,
511 };
512
513 static const unsigned scorpion_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
514 [PERF_COUNT_HW_CACHE_OP_MAX]
515 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
516 PERF_CACHE_MAP_ALL_UNSUPPORTED,
517 /*
518 * The performance counters don't differentiate between read and write
519 * accesses/misses so this isn't strictly correct, but it's the best we
520 * can do. Writes and reads get combined.
521 */
522 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
523 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
524 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
525 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
526 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_ICACHE_ACCESS,
527 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ICACHE_MISS,
528 /*
529 * Only ITLB misses and DTLB refills are supported. If users want the
530 * DTLB refills misses a raw counter must be used.
531 */
532 [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS,
533 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_DTLB_MISS,
534 [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS,
535 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_DTLB_MISS,
536 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ITLB_MISS,
537 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_ITLB_MISS,
538 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
539 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
540 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
541 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
542 };
543
544 /*
545 * Perf Events' indices
546 */
547 #define ARMV7_IDX_CYCLE_COUNTER 0
548 #define ARMV7_IDX_COUNTER0 1
549 #define ARMV7_IDX_COUNTER_LAST(cpu_pmu) \
550 (ARMV7_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1)
551
552 #define ARMV7_MAX_COUNTERS 32
553 #define ARMV7_COUNTER_MASK (ARMV7_MAX_COUNTERS - 1)
554
555 /*
556 * ARMv7 low level PMNC access
557 */
558
559 /*
560 * Perf Event to low level counters mapping
561 */
562 #define ARMV7_IDX_TO_COUNTER(x) \
563 (((x) - ARMV7_IDX_COUNTER0) & ARMV7_COUNTER_MASK)
564
565 /*
566 * Per-CPU PMNC: config reg
567 */
568 #define ARMV7_PMNC_E (1 << 0) /* Enable all counters */
569 #define ARMV7_PMNC_P (1 << 1) /* Reset all counters */
570 #define ARMV7_PMNC_C (1 << 2) /* Cycle counter reset */
571 #define ARMV7_PMNC_D (1 << 3) /* CCNT counts every 64th cpu cycle */
572 #define ARMV7_PMNC_X (1 << 4) /* Export to ETM */
573 #define ARMV7_PMNC_DP (1 << 5) /* Disable CCNT if non-invasive debug*/
574 #define ARMV7_PMNC_N_SHIFT 11 /* Number of counters supported */
575 #define ARMV7_PMNC_N_MASK 0x1f
576 #define ARMV7_PMNC_MASK 0x3f /* Mask for writable bits */
577
578 /*
579 * FLAG: counters overflow flag status reg
580 */
581 #define ARMV7_FLAG_MASK 0xffffffff /* Mask for writable bits */
582 #define ARMV7_OVERFLOWED_MASK ARMV7_FLAG_MASK
583
584 /*
585 * PMXEVTYPER: Event selection reg
586 */
587 #define ARMV7_EVTYPE_MASK 0xc80000ff /* Mask for writable bits */
588 #define ARMV7_EVTYPE_EVENT 0xff /* Mask for EVENT bits */
589
590 /*
591 * Event filters for PMUv2
592 */
593 #define ARMV7_EXCLUDE_PL1 (1 << 31)
594 #define ARMV7_EXCLUDE_USER (1 << 30)
595 #define ARMV7_INCLUDE_HYP (1 << 27)
596
597 static inline u32 armv7_pmnc_read(void)
598 {
599 u32 val;
600 asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r"(val));
601 return val;
602 }
603
604 static inline void armv7_pmnc_write(u32 val)
605 {
606 val &= ARMV7_PMNC_MASK;
607 isb();
608 asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r"(val));
609 }
610
611 static inline int armv7_pmnc_has_overflowed(u32 pmnc)
612 {
613 return pmnc & ARMV7_OVERFLOWED_MASK;
614 }
615
616 static inline int armv7_pmnc_counter_valid(struct arm_pmu *cpu_pmu, int idx)
617 {
618 return idx >= ARMV7_IDX_CYCLE_COUNTER &&
619 idx <= ARMV7_IDX_COUNTER_LAST(cpu_pmu);
620 }
621
622 static inline int armv7_pmnc_counter_has_overflowed(u32 pmnc, int idx)
623 {
624 return pmnc & BIT(ARMV7_IDX_TO_COUNTER(idx));
625 }
626
627 static inline void armv7_pmnc_select_counter(int idx)
628 {
629 u32 counter = ARMV7_IDX_TO_COUNTER(idx);
630 asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (counter));
631 isb();
632 }
633
634 static inline u32 armv7pmu_read_counter(struct perf_event *event)
635 {
636 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
637 struct hw_perf_event *hwc = &event->hw;
638 int idx = hwc->idx;
639 u32 value = 0;
640
641 if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
642 pr_err("CPU%u reading wrong counter %d\n",
643 smp_processor_id(), idx);
644 } else if (idx == ARMV7_IDX_CYCLE_COUNTER) {
645 asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (value));
646 } else {
647 armv7_pmnc_select_counter(idx);
648 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (value));
649 }
650
651 return value;
652 }
653
654 static inline void armv7pmu_write_counter(struct perf_event *event, u32 value)
655 {
656 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
657 struct hw_perf_event *hwc = &event->hw;
658 int idx = hwc->idx;
659
660 if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
661 pr_err("CPU%u writing wrong counter %d\n",
662 smp_processor_id(), idx);
663 } else if (idx == ARMV7_IDX_CYCLE_COUNTER) {
664 asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (value));
665 } else {
666 armv7_pmnc_select_counter(idx);
667 asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" (value));
668 }
669 }
670
671 static inline void armv7_pmnc_write_evtsel(int idx, u32 val)
672 {
673 armv7_pmnc_select_counter(idx);
674 val &= ARMV7_EVTYPE_MASK;
675 asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val));
676 }
677
678 static inline void armv7_pmnc_enable_counter(int idx)
679 {
680 u32 counter = ARMV7_IDX_TO_COUNTER(idx);
681 asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (BIT(counter)));
682 }
683
684 static inline void armv7_pmnc_disable_counter(int idx)
685 {
686 u32 counter = ARMV7_IDX_TO_COUNTER(idx);
687 asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (BIT(counter)));
688 }
689
690 static inline void armv7_pmnc_enable_intens(int idx)
691 {
692 u32 counter = ARMV7_IDX_TO_COUNTER(idx);
693 asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (BIT(counter)));
694 }
695
696 static inline void armv7_pmnc_disable_intens(int idx)
697 {
698 u32 counter = ARMV7_IDX_TO_COUNTER(idx);
699 asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (BIT(counter)));
700 isb();
701 /* Clear the overflow flag in case an interrupt is pending. */
702 asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (BIT(counter)));
703 isb();
704 }
705
706 static inline u32 armv7_pmnc_getreset_flags(void)
707 {
708 u32 val;
709
710 /* Read */
711 asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
712
713 /* Write to clear flags */
714 val &= ARMV7_FLAG_MASK;
715 asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (val));
716
717 return val;
718 }
719
720 #ifdef DEBUG
721 static void armv7_pmnc_dump_regs(struct arm_pmu *cpu_pmu)
722 {
723 u32 val;
724 unsigned int cnt;
725
726 pr_info("PMNC registers dump:\n");
727
728 asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val));
729 pr_info("PMNC =0x%08x\n", val);
730
731 asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r" (val));
732 pr_info("CNTENS=0x%08x\n", val);
733
734 asm volatile("mrc p15, 0, %0, c9, c14, 1" : "=r" (val));
735 pr_info("INTENS=0x%08x\n", val);
736
737 asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
738 pr_info("FLAGS =0x%08x\n", val);
739
740 asm volatile("mrc p15, 0, %0, c9, c12, 5" : "=r" (val));
741 pr_info("SELECT=0x%08x\n", val);
742
743 asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val));
744 pr_info("CCNT =0x%08x\n", val);
745
746 for (cnt = ARMV7_IDX_COUNTER0;
747 cnt <= ARMV7_IDX_COUNTER_LAST(cpu_pmu); cnt++) {
748 armv7_pmnc_select_counter(cnt);
749 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val));
750 pr_info("CNT[%d] count =0x%08x\n",
751 ARMV7_IDX_TO_COUNTER(cnt), val);
752 asm volatile("mrc p15, 0, %0, c9, c13, 1" : "=r" (val));
753 pr_info("CNT[%d] evtsel=0x%08x\n",
754 ARMV7_IDX_TO_COUNTER(cnt), val);
755 }
756 }
757 #endif
758
759 static void armv7pmu_enable_event(struct perf_event *event)
760 {
761 unsigned long flags;
762 struct hw_perf_event *hwc = &event->hw;
763 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
764 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
765 int idx = hwc->idx;
766
767 if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
768 pr_err("CPU%u enabling wrong PMNC counter IRQ enable %d\n",
769 smp_processor_id(), idx);
770 return;
771 }
772
773 /*
774 * Enable counter and interrupt, and set the counter to count
775 * the event that we're interested in.
776 */
777 raw_spin_lock_irqsave(&events->pmu_lock, flags);
778
779 /*
780 * Disable counter
781 */
782 armv7_pmnc_disable_counter(idx);
783
784 /*
785 * Set event (if destined for PMNx counters)
786 * We only need to set the event for the cycle counter if we
787 * have the ability to perform event filtering.
788 */
789 if (cpu_pmu->set_event_filter || idx != ARMV7_IDX_CYCLE_COUNTER)
790 armv7_pmnc_write_evtsel(idx, hwc->config_base);
791
792 /*
793 * Enable interrupt for this counter
794 */
795 armv7_pmnc_enable_intens(idx);
796
797 /*
798 * Enable counter
799 */
800 armv7_pmnc_enable_counter(idx);
801
802 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
803 }
804
805 static void armv7pmu_disable_event(struct perf_event *event)
806 {
807 unsigned long flags;
808 struct hw_perf_event *hwc = &event->hw;
809 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
810 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
811 int idx = hwc->idx;
812
813 if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
814 pr_err("CPU%u disabling wrong PMNC counter IRQ enable %d\n",
815 smp_processor_id(), idx);
816 return;
817 }
818
819 /*
820 * Disable counter and interrupt
821 */
822 raw_spin_lock_irqsave(&events->pmu_lock, flags);
823
824 /*
825 * Disable counter
826 */
827 armv7_pmnc_disable_counter(idx);
828
829 /*
830 * Disable interrupt for this counter
831 */
832 armv7_pmnc_disable_intens(idx);
833
834 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
835 }
836
837 static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev)
838 {
839 u32 pmnc;
840 struct perf_sample_data data;
841 struct arm_pmu *cpu_pmu = (struct arm_pmu *)dev;
842 struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);
843 struct pt_regs *regs;
844 int idx;
845
846 /*
847 * Get and reset the IRQ flags
848 */
849 pmnc = armv7_pmnc_getreset_flags();
850
851 /*
852 * Did an overflow occur?
853 */
854 if (!armv7_pmnc_has_overflowed(pmnc))
855 return IRQ_NONE;
856
857 /*
858 * Handle the counter(s) overflow(s)
859 */
860 regs = get_irq_regs();
861
862 for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
863 struct perf_event *event = cpuc->events[idx];
864 struct hw_perf_event *hwc;
865
866 /* Ignore if we don't have an event. */
867 if (!event)
868 continue;
869
870 /*
871 * We have a single interrupt for all counters. Check that
872 * each counter has overflowed before we process it.
873 */
874 if (!armv7_pmnc_counter_has_overflowed(pmnc, idx))
875 continue;
876
877 hwc = &event->hw;
878 armpmu_event_update(event);
879 perf_sample_data_init(&data, 0, hwc->last_period);
880 if (!armpmu_event_set_period(event))
881 continue;
882
883 if (perf_event_overflow(event, &data, regs))
884 cpu_pmu->disable(event);
885 }
886
887 /*
888 * Handle the pending perf events.
889 *
890 * Note: this call *must* be run with interrupts disabled. For
891 * platforms that can have the PMU interrupts raised as an NMI, this
892 * will not work.
893 */
894 irq_work_run();
895
896 return IRQ_HANDLED;
897 }
898
899 static void armv7pmu_start(struct arm_pmu *cpu_pmu)
900 {
901 unsigned long flags;
902 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
903
904 raw_spin_lock_irqsave(&events->pmu_lock, flags);
905 /* Enable all counters */
906 armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E);
907 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
908 }
909
910 static void armv7pmu_stop(struct arm_pmu *cpu_pmu)
911 {
912 unsigned long flags;
913 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
914
915 raw_spin_lock_irqsave(&events->pmu_lock, flags);
916 /* Disable all counters */
917 armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E);
918 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
919 }
920
921 static int armv7pmu_get_event_idx(struct pmu_hw_events *cpuc,
922 struct perf_event *event)
923 {
924 int idx;
925 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
926 struct hw_perf_event *hwc = &event->hw;
927 unsigned long evtype = hwc->config_base & ARMV7_EVTYPE_EVENT;
928
929 /* Always place a cycle counter into the cycle counter. */
930 if (evtype == ARMV7_PERFCTR_CPU_CYCLES) {
931 if (test_and_set_bit(ARMV7_IDX_CYCLE_COUNTER, cpuc->used_mask))
932 return -EAGAIN;
933
934 return ARMV7_IDX_CYCLE_COUNTER;
935 }
936
937 /*
938 * For anything other than a cycle counter, try and use
939 * the events counters
940 */
941 for (idx = ARMV7_IDX_COUNTER0; idx < cpu_pmu->num_events; ++idx) {
942 if (!test_and_set_bit(idx, cpuc->used_mask))
943 return idx;
944 }
945
946 /* The counters are all in use. */
947 return -EAGAIN;
948 }
949
950 /*
951 * Add an event filter to a given event. This will only work for PMUv2 PMUs.
952 */
953 static int armv7pmu_set_event_filter(struct hw_perf_event *event,
954 struct perf_event_attr *attr)
955 {
956 unsigned long config_base = 0;
957
958 if (attr->exclude_idle)
959 return -EPERM;
960 if (attr->exclude_user)
961 config_base |= ARMV7_EXCLUDE_USER;
962 if (attr->exclude_kernel)
963 config_base |= ARMV7_EXCLUDE_PL1;
964 if (!attr->exclude_hv)
965 config_base |= ARMV7_INCLUDE_HYP;
966
967 /*
968 * Install the filter into config_base as this is used to
969 * construct the event type.
970 */
971 event->config_base = config_base;
972
973 return 0;
974 }
975
976 static void armv7pmu_reset(void *info)
977 {
978 struct arm_pmu *cpu_pmu = (struct arm_pmu *)info;
979 u32 idx, nb_cnt = cpu_pmu->num_events;
980
981 /* The counter and interrupt enable registers are unknown at reset. */
982 for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
983 armv7_pmnc_disable_counter(idx);
984 armv7_pmnc_disable_intens(idx);
985 }
986
987 /* Initialize & Reset PMNC: C and P bits */
988 armv7_pmnc_write(ARMV7_PMNC_P | ARMV7_PMNC_C);
989 }
990
991 static int armv7_a8_map_event(struct perf_event *event)
992 {
993 return armpmu_map_event(event, &armv7_a8_perf_map,
994 &armv7_a8_perf_cache_map, 0xFF);
995 }
996
997 static int armv7_a9_map_event(struct perf_event *event)
998 {
999 return armpmu_map_event(event, &armv7_a9_perf_map,
1000 &armv7_a9_perf_cache_map, 0xFF);
1001 }
1002
1003 static int armv7_a5_map_event(struct perf_event *event)
1004 {
1005 return armpmu_map_event(event, &armv7_a5_perf_map,
1006 &armv7_a5_perf_cache_map, 0xFF);
1007 }
1008
1009 static int armv7_a15_map_event(struct perf_event *event)
1010 {
1011 return armpmu_map_event(event, &armv7_a15_perf_map,
1012 &armv7_a15_perf_cache_map, 0xFF);
1013 }
1014
1015 static int armv7_a7_map_event(struct perf_event *event)
1016 {
1017 return armpmu_map_event(event, &armv7_a7_perf_map,
1018 &armv7_a7_perf_cache_map, 0xFF);
1019 }
1020
1021 static int armv7_a12_map_event(struct perf_event *event)
1022 {
1023 return armpmu_map_event(event, &armv7_a12_perf_map,
1024 &armv7_a12_perf_cache_map, 0xFF);
1025 }
1026
1027 static int krait_map_event(struct perf_event *event)
1028 {
1029 return armpmu_map_event(event, &krait_perf_map,
1030 &krait_perf_cache_map, 0xFFFFF);
1031 }
1032
1033 static int krait_map_event_no_branch(struct perf_event *event)
1034 {
1035 return armpmu_map_event(event, &krait_perf_map_no_branch,
1036 &krait_perf_cache_map, 0xFFFFF);
1037 }
1038
1039 static int scorpion_map_event(struct perf_event *event)
1040 {
1041 return armpmu_map_event(event, &scorpion_perf_map,
1042 &scorpion_perf_cache_map, 0xFFFFF);
1043 }
1044
1045 static void armv7pmu_init(struct arm_pmu *cpu_pmu)
1046 {
1047 cpu_pmu->handle_irq = armv7pmu_handle_irq;
1048 cpu_pmu->enable = armv7pmu_enable_event;
1049 cpu_pmu->disable = armv7pmu_disable_event;
1050 cpu_pmu->read_counter = armv7pmu_read_counter;
1051 cpu_pmu->write_counter = armv7pmu_write_counter;
1052 cpu_pmu->get_event_idx = armv7pmu_get_event_idx;
1053 cpu_pmu->start = armv7pmu_start;
1054 cpu_pmu->stop = armv7pmu_stop;
1055 cpu_pmu->reset = armv7pmu_reset;
1056 cpu_pmu->max_period = (1LLU << 32) - 1;
1057 };
1058
1059 static u32 armv7_read_num_pmnc_events(void)
1060 {
1061 u32 nb_cnt;
1062
1063 /* Read the nb of CNTx counters supported from PMNC */
1064 nb_cnt = (armv7_pmnc_read() >> ARMV7_PMNC_N_SHIFT) & ARMV7_PMNC_N_MASK;
1065
1066 /* Add the CPU cycles counter and return */
1067 return nb_cnt + 1;
1068 }
1069
1070 static int armv7_a8_pmu_init(struct arm_pmu *cpu_pmu)
1071 {
1072 armv7pmu_init(cpu_pmu);
1073 cpu_pmu->name = "armv7_cortex_a8";
1074 cpu_pmu->map_event = armv7_a8_map_event;
1075 cpu_pmu->num_events = armv7_read_num_pmnc_events();
1076 return 0;
1077 }
1078
1079 static int armv7_a9_pmu_init(struct arm_pmu *cpu_pmu)
1080 {
1081 armv7pmu_init(cpu_pmu);
1082 cpu_pmu->name = "armv7_cortex_a9";
1083 cpu_pmu->map_event = armv7_a9_map_event;
1084 cpu_pmu->num_events = armv7_read_num_pmnc_events();
1085 return 0;
1086 }
1087
1088 static int armv7_a5_pmu_init(struct arm_pmu *cpu_pmu)
1089 {
1090 armv7pmu_init(cpu_pmu);
1091 cpu_pmu->name = "armv7_cortex_a5";
1092 cpu_pmu->map_event = armv7_a5_map_event;
1093 cpu_pmu->num_events = armv7_read_num_pmnc_events();
1094 return 0;
1095 }
1096
1097 static int armv7_a15_pmu_init(struct arm_pmu *cpu_pmu)
1098 {
1099 armv7pmu_init(cpu_pmu);
1100 cpu_pmu->name = "armv7_cortex_a15";
1101 cpu_pmu->map_event = armv7_a15_map_event;
1102 cpu_pmu->num_events = armv7_read_num_pmnc_events();
1103 cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1104 return 0;
1105 }
1106
1107 static int armv7_a7_pmu_init(struct arm_pmu *cpu_pmu)
1108 {
1109 armv7pmu_init(cpu_pmu);
1110 cpu_pmu->name = "armv7_cortex_a7";
1111 cpu_pmu->map_event = armv7_a7_map_event;
1112 cpu_pmu->num_events = armv7_read_num_pmnc_events();
1113 cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1114 return 0;
1115 }
1116
1117 static int armv7_a12_pmu_init(struct arm_pmu *cpu_pmu)
1118 {
1119 armv7pmu_init(cpu_pmu);
1120 cpu_pmu->name = "armv7_cortex_a12";
1121 cpu_pmu->map_event = armv7_a12_map_event;
1122 cpu_pmu->num_events = armv7_read_num_pmnc_events();
1123 cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1124 return 0;
1125 }
1126
1127 static int armv7_a17_pmu_init(struct arm_pmu *cpu_pmu)
1128 {
1129 armv7_a12_pmu_init(cpu_pmu);
1130 cpu_pmu->name = "armv7_cortex_a17";
1131 return 0;
1132 }
1133
1134 /*
1135 * Krait Performance Monitor Region Event Selection Register (PMRESRn)
1136 *
1137 * 31 30 24 16 8 0
1138 * +--------------------------------+
1139 * PMRESR0 | EN | CC | CC | CC | CC | N = 1, R = 0
1140 * +--------------------------------+
1141 * PMRESR1 | EN | CC | CC | CC | CC | N = 1, R = 1
1142 * +--------------------------------+
1143 * PMRESR2 | EN | CC | CC | CC | CC | N = 1, R = 2
1144 * +--------------------------------+
1145 * VPMRESR0 | EN | CC | CC | CC | CC | N = 2, R = ?
1146 * +--------------------------------+
1147 * EN | G=3 | G=2 | G=1 | G=0
1148 *
1149 * Event Encoding:
1150 *
1151 * hwc->config_base = 0xNRCCG
1152 *
1153 * N = prefix, 1 for Krait CPU (PMRESRn), 2 for Venum VFP (VPMRESR)
1154 * R = region register
1155 * CC = class of events the group G is choosing from
1156 * G = group or particular event
1157 *
1158 * Example: 0x12021 is a Krait CPU event in PMRESR2's group 1 with code 2
1159 *
1160 * A region (R) corresponds to a piece of the CPU (execution unit, instruction
1161 * unit, etc.) while the event code (CC) corresponds to a particular class of
1162 * events (interrupts for example). An event code is broken down into
1163 * groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for
1164 * example).
1165 */
1166
1167 #define KRAIT_EVENT (1 << 16)
1168 #define VENUM_EVENT (2 << 16)
1169 #define KRAIT_EVENT_MASK (KRAIT_EVENT | VENUM_EVENT)
1170 #define PMRESRn_EN BIT(31)
1171
1172 #define EVENT_REGION(event) (((event) >> 12) & 0xf) /* R */
1173 #define EVENT_GROUP(event) ((event) & 0xf) /* G */
1174 #define EVENT_CODE(event) (((event) >> 4) & 0xff) /* CC */
1175 #define EVENT_VENUM(event) (!!(event & VENUM_EVENT)) /* N=2 */
1176 #define EVENT_CPU(event) (!!(event & KRAIT_EVENT)) /* N=1 */
1177
1178 static u32 krait_read_pmresrn(int n)
1179 {
1180 u32 val;
1181
1182 switch (n) {
1183 case 0:
1184 asm volatile("mrc p15, 1, %0, c9, c15, 0" : "=r" (val));
1185 break;
1186 case 1:
1187 asm volatile("mrc p15, 1, %0, c9, c15, 1" : "=r" (val));
1188 break;
1189 case 2:
1190 asm volatile("mrc p15, 1, %0, c9, c15, 2" : "=r" (val));
1191 break;
1192 default:
1193 BUG(); /* Should be validated in krait_pmu_get_event_idx() */
1194 }
1195
1196 return val;
1197 }
1198
1199 static void krait_write_pmresrn(int n, u32 val)
1200 {
1201 switch (n) {
1202 case 0:
1203 asm volatile("mcr p15, 1, %0, c9, c15, 0" : : "r" (val));
1204 break;
1205 case 1:
1206 asm volatile("mcr p15, 1, %0, c9, c15, 1" : : "r" (val));
1207 break;
1208 case 2:
1209 asm volatile("mcr p15, 1, %0, c9, c15, 2" : : "r" (val));
1210 break;
1211 default:
1212 BUG(); /* Should be validated in krait_pmu_get_event_idx() */
1213 }
1214 }
1215
1216 static u32 venum_read_pmresr(void)
1217 {
1218 u32 val;
1219 asm volatile("mrc p10, 7, %0, c11, c0, 0" : "=r" (val));
1220 return val;
1221 }
1222
1223 static void venum_write_pmresr(u32 val)
1224 {
1225 asm volatile("mcr p10, 7, %0, c11, c0, 0" : : "r" (val));
1226 }
1227
1228 static void venum_pre_pmresr(u32 *venum_orig_val, u32 *fp_orig_val)
1229 {
1230 u32 venum_new_val;
1231 u32 fp_new_val;
1232
1233 BUG_ON(preemptible());
1234 /* CPACR Enable CP10 and CP11 access */
1235 *venum_orig_val = get_copro_access();
1236 venum_new_val = *venum_orig_val | CPACC_SVC(10) | CPACC_SVC(11);
1237 set_copro_access(venum_new_val);
1238
1239 /* Enable FPEXC */
1240 *fp_orig_val = fmrx(FPEXC);
1241 fp_new_val = *fp_orig_val | FPEXC_EN;
1242 fmxr(FPEXC, fp_new_val);
1243 }
1244
1245 static void venum_post_pmresr(u32 venum_orig_val, u32 fp_orig_val)
1246 {
1247 BUG_ON(preemptible());
1248 /* Restore FPEXC */
1249 fmxr(FPEXC, fp_orig_val);
1250 isb();
1251 /* Restore CPACR */
1252 set_copro_access(venum_orig_val);
1253 }
1254
1255 static u32 krait_get_pmresrn_event(unsigned int region)
1256 {
1257 static const u32 pmresrn_table[] = { KRAIT_PMRESR0_GROUP0,
1258 KRAIT_PMRESR1_GROUP0,
1259 KRAIT_PMRESR2_GROUP0 };
1260 return pmresrn_table[region];
1261 }
1262
1263 static void krait_evt_setup(int idx, u32 config_base)
1264 {
1265 u32 val;
1266 u32 mask;
1267 u32 vval, fval;
1268 unsigned int region = EVENT_REGION(config_base);
1269 unsigned int group = EVENT_GROUP(config_base);
1270 unsigned int code = EVENT_CODE(config_base);
1271 unsigned int group_shift;
1272 bool venum_event = EVENT_VENUM(config_base);
1273
1274 group_shift = group * 8;
1275 mask = 0xff << group_shift;
1276
1277 /* Configure evtsel for the region and group */
1278 if (venum_event)
1279 val = KRAIT_VPMRESR0_GROUP0;
1280 else
1281 val = krait_get_pmresrn_event(region);
1282 val += group;
1283 /* Mix in mode-exclusion bits */
1284 val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1);
1285 armv7_pmnc_write_evtsel(idx, val);
1286
1287 if (venum_event) {
1288 venum_pre_pmresr(&vval, &fval);
1289 val = venum_read_pmresr();
1290 val &= ~mask;
1291 val |= code << group_shift;
1292 val |= PMRESRn_EN;
1293 venum_write_pmresr(val);
1294 venum_post_pmresr(vval, fval);
1295 } else {
1296 val = krait_read_pmresrn(region);
1297 val &= ~mask;
1298 val |= code << group_shift;
1299 val |= PMRESRn_EN;
1300 krait_write_pmresrn(region, val);
1301 }
1302 }
1303
1304 static u32 clear_pmresrn_group(u32 val, int group)
1305 {
1306 u32 mask;
1307 int group_shift;
1308
1309 group_shift = group * 8;
1310 mask = 0xff << group_shift;
1311 val &= ~mask;
1312
1313 /* Don't clear enable bit if entire region isn't disabled */
1314 if (val & ~PMRESRn_EN)
1315 return val |= PMRESRn_EN;
1316
1317 return 0;
1318 }
1319
1320 static void krait_clearpmu(u32 config_base)
1321 {
1322 u32 val;
1323 u32 vval, fval;
1324 unsigned int region = EVENT_REGION(config_base);
1325 unsigned int group = EVENT_GROUP(config_base);
1326 bool venum_event = EVENT_VENUM(config_base);
1327
1328 if (venum_event) {
1329 venum_pre_pmresr(&vval, &fval);
1330 val = venum_read_pmresr();
1331 val = clear_pmresrn_group(val, group);
1332 venum_write_pmresr(val);
1333 venum_post_pmresr(vval, fval);
1334 } else {
1335 val = krait_read_pmresrn(region);
1336 val = clear_pmresrn_group(val, group);
1337 krait_write_pmresrn(region, val);
1338 }
1339 }
1340
1341 static void krait_pmu_disable_event(struct perf_event *event)
1342 {
1343 unsigned long flags;
1344 struct hw_perf_event *hwc = &event->hw;
1345 int idx = hwc->idx;
1346 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1347 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1348
1349 /* Disable counter and interrupt */
1350 raw_spin_lock_irqsave(&events->pmu_lock, flags);
1351
1352 /* Disable counter */
1353 armv7_pmnc_disable_counter(idx);
1354
1355 /*
1356 * Clear pmresr code (if destined for PMNx counters)
1357 */
1358 if (hwc->config_base & KRAIT_EVENT_MASK)
1359 krait_clearpmu(hwc->config_base);
1360
1361 /* Disable interrupt for this counter */
1362 armv7_pmnc_disable_intens(idx);
1363
1364 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1365 }
1366
1367 static void krait_pmu_enable_event(struct perf_event *event)
1368 {
1369 unsigned long flags;
1370 struct hw_perf_event *hwc = &event->hw;
1371 int idx = hwc->idx;
1372 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1373 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1374
1375 /*
1376 * Enable counter and interrupt, and set the counter to count
1377 * the event that we're interested in.
1378 */
1379 raw_spin_lock_irqsave(&events->pmu_lock, flags);
1380
1381 /* Disable counter */
1382 armv7_pmnc_disable_counter(idx);
1383
1384 /*
1385 * Set event (if destined for PMNx counters)
1386 * We set the event for the cycle counter because we
1387 * have the ability to perform event filtering.
1388 */
1389 if (hwc->config_base & KRAIT_EVENT_MASK)
1390 krait_evt_setup(idx, hwc->config_base);
1391 else
1392 armv7_pmnc_write_evtsel(idx, hwc->config_base);
1393
1394 /* Enable interrupt for this counter */
1395 armv7_pmnc_enable_intens(idx);
1396
1397 /* Enable counter */
1398 armv7_pmnc_enable_counter(idx);
1399
1400 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1401 }
1402
1403 static void krait_pmu_reset(void *info)
1404 {
1405 u32 vval, fval;
1406 struct arm_pmu *cpu_pmu = info;
1407 u32 idx, nb_cnt = cpu_pmu->num_events;
1408
1409 armv7pmu_reset(info);
1410
1411 /* Clear all pmresrs */
1412 krait_write_pmresrn(0, 0);
1413 krait_write_pmresrn(1, 0);
1414 krait_write_pmresrn(2, 0);
1415
1416 venum_pre_pmresr(&vval, &fval);
1417 venum_write_pmresr(0);
1418 venum_post_pmresr(vval, fval);
1419
1420 /* Reset PMxEVNCTCR to sane default */
1421 for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
1422 armv7_pmnc_select_counter(idx);
1423 asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1424 }
1425
1426 }
1427
1428 static int krait_event_to_bit(struct perf_event *event, unsigned int region,
1429 unsigned int group)
1430 {
1431 int bit;
1432 struct hw_perf_event *hwc = &event->hw;
1433 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1434
1435 if (hwc->config_base & VENUM_EVENT)
1436 bit = KRAIT_VPMRESR0_GROUP0;
1437 else
1438 bit = krait_get_pmresrn_event(region);
1439 bit -= krait_get_pmresrn_event(0);
1440 bit += group;
1441 /*
1442 * Lower bits are reserved for use by the counters (see
1443 * armv7pmu_get_event_idx() for more info)
1444 */
1445 bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1;
1446
1447 return bit;
1448 }
1449
1450 /*
1451 * We check for column exclusion constraints here.
1452 * Two events cant use the same group within a pmresr register.
1453 */
1454 static int krait_pmu_get_event_idx(struct pmu_hw_events *cpuc,
1455 struct perf_event *event)
1456 {
1457 int idx;
1458 int bit = -1;
1459 struct hw_perf_event *hwc = &event->hw;
1460 unsigned int region = EVENT_REGION(hwc->config_base);
1461 unsigned int code = EVENT_CODE(hwc->config_base);
1462 unsigned int group = EVENT_GROUP(hwc->config_base);
1463 bool venum_event = EVENT_VENUM(hwc->config_base);
1464 bool krait_event = EVENT_CPU(hwc->config_base);
1465
1466 if (venum_event || krait_event) {
1467 /* Ignore invalid events */
1468 if (group > 3 || region > 2)
1469 return -EINVAL;
1470 if (venum_event && (code & 0xe0))
1471 return -EINVAL;
1472
1473 bit = krait_event_to_bit(event, region, group);
1474 if (test_and_set_bit(bit, cpuc->used_mask))
1475 return -EAGAIN;
1476 }
1477
1478 idx = armv7pmu_get_event_idx(cpuc, event);
1479 if (idx < 0 && bit >= 0)
1480 clear_bit(bit, cpuc->used_mask);
1481
1482 return idx;
1483 }
1484
1485 static void krait_pmu_clear_event_idx(struct pmu_hw_events *cpuc,
1486 struct perf_event *event)
1487 {
1488 int bit;
1489 struct hw_perf_event *hwc = &event->hw;
1490 unsigned int region = EVENT_REGION(hwc->config_base);
1491 unsigned int group = EVENT_GROUP(hwc->config_base);
1492 bool venum_event = EVENT_VENUM(hwc->config_base);
1493 bool krait_event = EVENT_CPU(hwc->config_base);
1494
1495 if (venum_event || krait_event) {
1496 bit = krait_event_to_bit(event, region, group);
1497 clear_bit(bit, cpuc->used_mask);
1498 }
1499 }
1500
1501 static int krait_pmu_init(struct arm_pmu *cpu_pmu)
1502 {
1503 armv7pmu_init(cpu_pmu);
1504 cpu_pmu->name = "armv7_krait";
1505 /* Some early versions of Krait don't support PC write events */
1506 if (of_property_read_bool(cpu_pmu->plat_device->dev.of_node,
1507 "qcom,no-pc-write"))
1508 cpu_pmu->map_event = krait_map_event_no_branch;
1509 else
1510 cpu_pmu->map_event = krait_map_event;
1511 cpu_pmu->num_events = armv7_read_num_pmnc_events();
1512 cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1513 cpu_pmu->reset = krait_pmu_reset;
1514 cpu_pmu->enable = krait_pmu_enable_event;
1515 cpu_pmu->disable = krait_pmu_disable_event;
1516 cpu_pmu->get_event_idx = krait_pmu_get_event_idx;
1517 cpu_pmu->clear_event_idx = krait_pmu_clear_event_idx;
1518 return 0;
1519 }
1520
1521 /*
1522 * Scorpion Local Performance Monitor Register (LPMn)
1523 *
1524 * 31 30 24 16 8 0
1525 * +--------------------------------+
1526 * LPM0 | EN | CC | CC | CC | CC | N = 1, R = 0
1527 * +--------------------------------+
1528 * LPM1 | EN | CC | CC | CC | CC | N = 1, R = 1
1529 * +--------------------------------+
1530 * LPM2 | EN | CC | CC | CC | CC | N = 1, R = 2
1531 * +--------------------------------+
1532 * L2LPM | EN | CC | CC | CC | CC | N = 1, R = 3
1533 * +--------------------------------+
1534 * VLPM | EN | CC | CC | CC | CC | N = 2, R = ?
1535 * +--------------------------------+
1536 * EN | G=3 | G=2 | G=1 | G=0
1537 *
1538 *
1539 * Event Encoding:
1540 *
1541 * hwc->config_base = 0xNRCCG
1542 *
1543 * N = prefix, 1 for Scorpion CPU (LPMn/L2LPM), 2 for Venum VFP (VLPM)
1544 * R = region register
1545 * CC = class of events the group G is choosing from
1546 * G = group or particular event
1547 *
1548 * Example: 0x12021 is a Scorpion CPU event in LPM2's group 1 with code 2
1549 *
1550 * A region (R) corresponds to a piece of the CPU (execution unit, instruction
1551 * unit, etc.) while the event code (CC) corresponds to a particular class of
1552 * events (interrupts for example). An event code is broken down into
1553 * groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for
1554 * example).
1555 */
1556
1557 static u32 scorpion_read_pmresrn(int n)
1558 {
1559 u32 val;
1560
1561 switch (n) {
1562 case 0:
1563 asm volatile("mrc p15, 0, %0, c15, c0, 0" : "=r" (val));
1564 break;
1565 case 1:
1566 asm volatile("mrc p15, 1, %0, c15, c0, 0" : "=r" (val));
1567 break;
1568 case 2:
1569 asm volatile("mrc p15, 2, %0, c15, c0, 0" : "=r" (val));
1570 break;
1571 case 3:
1572 asm volatile("mrc p15, 3, %0, c15, c2, 0" : "=r" (val));
1573 break;
1574 default:
1575 BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */
1576 }
1577
1578 return val;
1579 }
1580
1581 static void scorpion_write_pmresrn(int n, u32 val)
1582 {
1583 switch (n) {
1584 case 0:
1585 asm volatile("mcr p15, 0, %0, c15, c0, 0" : : "r" (val));
1586 break;
1587 case 1:
1588 asm volatile("mcr p15, 1, %0, c15, c0, 0" : : "r" (val));
1589 break;
1590 case 2:
1591 asm volatile("mcr p15, 2, %0, c15, c0, 0" : : "r" (val));
1592 break;
1593 case 3:
1594 asm volatile("mcr p15, 3, %0, c15, c2, 0" : : "r" (val));
1595 break;
1596 default:
1597 BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */
1598 }
1599 }
1600
1601 static u32 scorpion_get_pmresrn_event(unsigned int region)
1602 {
1603 static const u32 pmresrn_table[] = { SCORPION_LPM0_GROUP0,
1604 SCORPION_LPM1_GROUP0,
1605 SCORPION_LPM2_GROUP0,
1606 SCORPION_L2LPM_GROUP0 };
1607 return pmresrn_table[region];
1608 }
1609
1610 static void scorpion_evt_setup(int idx, u32 config_base)
1611 {
1612 u32 val;
1613 u32 mask;
1614 u32 vval, fval;
1615 unsigned int region = EVENT_REGION(config_base);
1616 unsigned int group = EVENT_GROUP(config_base);
1617 unsigned int code = EVENT_CODE(config_base);
1618 unsigned int group_shift;
1619 bool venum_event = EVENT_VENUM(config_base);
1620
1621 group_shift = group * 8;
1622 mask = 0xff << group_shift;
1623
1624 /* Configure evtsel for the region and group */
1625 if (venum_event)
1626 val = SCORPION_VLPM_GROUP0;
1627 else
1628 val = scorpion_get_pmresrn_event(region);
1629 val += group;
1630 /* Mix in mode-exclusion bits */
1631 val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1);
1632 armv7_pmnc_write_evtsel(idx, val);
1633
1634 asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1635
1636 if (venum_event) {
1637 venum_pre_pmresr(&vval, &fval);
1638 val = venum_read_pmresr();
1639 val &= ~mask;
1640 val |= code << group_shift;
1641 val |= PMRESRn_EN;
1642 venum_write_pmresr(val);
1643 venum_post_pmresr(vval, fval);
1644 } else {
1645 val = scorpion_read_pmresrn(region);
1646 val &= ~mask;
1647 val |= code << group_shift;
1648 val |= PMRESRn_EN;
1649 scorpion_write_pmresrn(region, val);
1650 }
1651 }
1652
1653 static void scorpion_clearpmu(u32 config_base)
1654 {
1655 u32 val;
1656 u32 vval, fval;
1657 unsigned int region = EVENT_REGION(config_base);
1658 unsigned int group = EVENT_GROUP(config_base);
1659 bool venum_event = EVENT_VENUM(config_base);
1660
1661 if (venum_event) {
1662 venum_pre_pmresr(&vval, &fval);
1663 val = venum_read_pmresr();
1664 val = clear_pmresrn_group(val, group);
1665 venum_write_pmresr(val);
1666 venum_post_pmresr(vval, fval);
1667 } else {
1668 val = scorpion_read_pmresrn(region);
1669 val = clear_pmresrn_group(val, group);
1670 scorpion_write_pmresrn(region, val);
1671 }
1672 }
1673
1674 static void scorpion_pmu_disable_event(struct perf_event *event)
1675 {
1676 unsigned long flags;
1677 struct hw_perf_event *hwc = &event->hw;
1678 int idx = hwc->idx;
1679 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1680 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1681
1682 /* Disable counter and interrupt */
1683 raw_spin_lock_irqsave(&events->pmu_lock, flags);
1684
1685 /* Disable counter */
1686 armv7_pmnc_disable_counter(idx);
1687
1688 /*
1689 * Clear pmresr code (if destined for PMNx counters)
1690 */
1691 if (hwc->config_base & KRAIT_EVENT_MASK)
1692 scorpion_clearpmu(hwc->config_base);
1693
1694 /* Disable interrupt for this counter */
1695 armv7_pmnc_disable_intens(idx);
1696
1697 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1698 }
1699
1700 static void scorpion_pmu_enable_event(struct perf_event *event)
1701 {
1702 unsigned long flags;
1703 struct hw_perf_event *hwc = &event->hw;
1704 int idx = hwc->idx;
1705 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1706 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1707
1708 /*
1709 * Enable counter and interrupt, and set the counter to count
1710 * the event that we're interested in.
1711 */
1712 raw_spin_lock_irqsave(&events->pmu_lock, flags);
1713
1714 /* Disable counter */
1715 armv7_pmnc_disable_counter(idx);
1716
1717 /*
1718 * Set event (if destined for PMNx counters)
1719 * We don't set the event for the cycle counter because we
1720 * don't have the ability to perform event filtering.
1721 */
1722 if (hwc->config_base & KRAIT_EVENT_MASK)
1723 scorpion_evt_setup(idx, hwc->config_base);
1724 else if (idx != ARMV7_IDX_CYCLE_COUNTER)
1725 armv7_pmnc_write_evtsel(idx, hwc->config_base);
1726
1727 /* Enable interrupt for this counter */
1728 armv7_pmnc_enable_intens(idx);
1729
1730 /* Enable counter */
1731 armv7_pmnc_enable_counter(idx);
1732
1733 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1734 }
1735
1736 static void scorpion_pmu_reset(void *info)
1737 {
1738 u32 vval, fval;
1739 struct arm_pmu *cpu_pmu = info;
1740 u32 idx, nb_cnt = cpu_pmu->num_events;
1741
1742 armv7pmu_reset(info);
1743
1744 /* Clear all pmresrs */
1745 scorpion_write_pmresrn(0, 0);
1746 scorpion_write_pmresrn(1, 0);
1747 scorpion_write_pmresrn(2, 0);
1748 scorpion_write_pmresrn(3, 0);
1749
1750 venum_pre_pmresr(&vval, &fval);
1751 venum_write_pmresr(0);
1752 venum_post_pmresr(vval, fval);
1753
1754 /* Reset PMxEVNCTCR to sane default */
1755 for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
1756 armv7_pmnc_select_counter(idx);
1757 asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1758 }
1759 }
1760
1761 static int scorpion_event_to_bit(struct perf_event *event, unsigned int region,
1762 unsigned int group)
1763 {
1764 int bit;
1765 struct hw_perf_event *hwc = &event->hw;
1766 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1767
1768 if (hwc->config_base & VENUM_EVENT)
1769 bit = SCORPION_VLPM_GROUP0;
1770 else
1771 bit = scorpion_get_pmresrn_event(region);
1772 bit -= scorpion_get_pmresrn_event(0);
1773 bit += group;
1774 /*
1775 * Lower bits are reserved for use by the counters (see
1776 * armv7pmu_get_event_idx() for more info)
1777 */
1778 bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1;
1779
1780 return bit;
1781 }
1782
1783 /*
1784 * We check for column exclusion constraints here.
1785 * Two events cant use the same group within a pmresr register.
1786 */
1787 static int scorpion_pmu_get_event_idx(struct pmu_hw_events *cpuc,
1788 struct perf_event *event)
1789 {
1790 int idx;
1791 int bit = -1;
1792 struct hw_perf_event *hwc = &event->hw;
1793 unsigned int region = EVENT_REGION(hwc->config_base);
1794 unsigned int group = EVENT_GROUP(hwc->config_base);
1795 bool venum_event = EVENT_VENUM(hwc->config_base);
1796 bool scorpion_event = EVENT_CPU(hwc->config_base);
1797
1798 if (venum_event || scorpion_event) {
1799 /* Ignore invalid events */
1800 if (group > 3 || region > 3)
1801 return -EINVAL;
1802
1803 bit = scorpion_event_to_bit(event, region, group);
1804 if (test_and_set_bit(bit, cpuc->used_mask))
1805 return -EAGAIN;
1806 }
1807
1808 idx = armv7pmu_get_event_idx(cpuc, event);
1809 if (idx < 0 && bit >= 0)
1810 clear_bit(bit, cpuc->used_mask);
1811
1812 return idx;
1813 }
1814
1815 static void scorpion_pmu_clear_event_idx(struct pmu_hw_events *cpuc,
1816 struct perf_event *event)
1817 {
1818 int bit;
1819 struct hw_perf_event *hwc = &event->hw;
1820 unsigned int region = EVENT_REGION(hwc->config_base);
1821 unsigned int group = EVENT_GROUP(hwc->config_base);
1822 bool venum_event = EVENT_VENUM(hwc->config_base);
1823 bool scorpion_event = EVENT_CPU(hwc->config_base);
1824
1825 if (venum_event || scorpion_event) {
1826 bit = scorpion_event_to_bit(event, region, group);
1827 clear_bit(bit, cpuc->used_mask);
1828 }
1829 }
1830
1831 static int scorpion_pmu_init(struct arm_pmu *cpu_pmu)
1832 {
1833 armv7pmu_init(cpu_pmu);
1834 cpu_pmu->name = "armv7_scorpion";
1835 cpu_pmu->map_event = scorpion_map_event;
1836 cpu_pmu->num_events = armv7_read_num_pmnc_events();
1837 cpu_pmu->reset = scorpion_pmu_reset;
1838 cpu_pmu->enable = scorpion_pmu_enable_event;
1839 cpu_pmu->disable = scorpion_pmu_disable_event;
1840 cpu_pmu->get_event_idx = scorpion_pmu_get_event_idx;
1841 cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx;
1842 return 0;
1843 }
1844
1845 static int scorpion_mp_pmu_init(struct arm_pmu *cpu_pmu)
1846 {
1847 armv7pmu_init(cpu_pmu);
1848 cpu_pmu->name = "armv7_scorpion_mp";
1849 cpu_pmu->map_event = scorpion_map_event;
1850 cpu_pmu->num_events = armv7_read_num_pmnc_events();
1851 cpu_pmu->reset = scorpion_pmu_reset;
1852 cpu_pmu->enable = scorpion_pmu_enable_event;
1853 cpu_pmu->disable = scorpion_pmu_disable_event;
1854 cpu_pmu->get_event_idx = scorpion_pmu_get_event_idx;
1855 cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx;
1856 return 0;
1857 }
1858 #else
1859 static inline int armv7_a8_pmu_init(struct arm_pmu *cpu_pmu)
1860 {
1861 return -ENODEV;
1862 }
1863
1864 static inline int armv7_a9_pmu_init(struct arm_pmu *cpu_pmu)
1865 {
1866 return -ENODEV;
1867 }
1868
1869 static inline int armv7_a5_pmu_init(struct arm_pmu *cpu_pmu)
1870 {
1871 return -ENODEV;
1872 }
1873
1874 static inline int armv7_a15_pmu_init(struct arm_pmu *cpu_pmu)
1875 {
1876 return -ENODEV;
1877 }
1878
1879 static inline int armv7_a7_pmu_init(struct arm_pmu *cpu_pmu)
1880 {
1881 return -ENODEV;
1882 }
1883
1884 static inline int armv7_a12_pmu_init(struct arm_pmu *cpu_pmu)
1885 {
1886 return -ENODEV;
1887 }
1888
1889 static inline int armv7_a17_pmu_init(struct arm_pmu *cpu_pmu)
1890 {
1891 return -ENODEV;
1892 }
1893
1894 static inline int krait_pmu_init(struct arm_pmu *cpu_pmu)
1895 {
1896 return -ENODEV;
1897 }
1898
1899 static inline int scorpion_pmu_init(struct arm_pmu *cpu_pmu)
1900 {
1901 return -ENODEV;
1902 }
1903
1904 static inline int scorpion_mp_pmu_init(struct arm_pmu *cpu_pmu)
1905 {
1906 return -ENODEV;
1907 }
1908 #endif /* CONFIG_CPU_V7 */
Linux with added FPC-III support