?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
blk: rq_data_dir() should not return a boolean
[cris-mirror.git] / arch / arm / kernel / perf_event_v7.c
blob126dc679b2308a2b846252bbde1e513389d71cdd
1 /*
2 * ARMv7 Cortex-A8 and Cortex-A9 Performance Events handling code.
3 *
4 * ARMv7 support: Jean Pihet <jpihet@mvista.com>
5 * 2010 (c) MontaVista Software, LLC.
6 *
7 * Copied from ARMv6 code, with the low level code inspired
8 * by the ARMv7 Oprofile code.
9 *
10 * Cortex-A8 has up to 4 configurable performance counters and
11 * a single cycle counter.
12 * Cortex-A9 has up to 31 configurable performance counters and
13 * a single cycle counter.
14 *
15 * All counters can be enabled/disabled and IRQ masked separately. The cycle
16 * counter and all 4 performance counters together can be reset separately.
17 */
18
19 #ifdef CONFIG_CPU_V7
20
21 #include <asm/cp15.h>
22 #include <asm/cputype.h>
23 #include <asm/irq_regs.h>
24 #include <asm/vfp.h>
25 #include "../vfp/vfpinstr.h"
26
27 #include <linux/of.h>
28 #include <linux/perf/arm_pmu.h>
29 #include <linux/platform_device.h>
30
31 /*
32 * Common ARMv7 event types
33 *
34 * Note: An implementation may not be able to count all of these events
35 * but the encodings are considered to be `reserved' in the case that
36 * they are not available.
37 */
38 enum armv7_perf_types {
39 ARMV7_PERFCTR_PMNC_SW_INCR = 0x00,
40 ARMV7_PERFCTR_L1_ICACHE_REFILL = 0x01,
41 ARMV7_PERFCTR_ITLB_REFILL = 0x02,
42 ARMV7_PERFCTR_L1_DCACHE_REFILL = 0x03,
43 ARMV7_PERFCTR_L1_DCACHE_ACCESS = 0x04,
44 ARMV7_PERFCTR_DTLB_REFILL = 0x05,
45 ARMV7_PERFCTR_MEM_READ = 0x06,
46 ARMV7_PERFCTR_MEM_WRITE = 0x07,
47 ARMV7_PERFCTR_INSTR_EXECUTED = 0x08,
48 ARMV7_PERFCTR_EXC_TAKEN = 0x09,
49 ARMV7_PERFCTR_EXC_EXECUTED = 0x0A,
50 ARMV7_PERFCTR_CID_WRITE = 0x0B,
51
52 /*
53 * ARMV7_PERFCTR_PC_WRITE is equivalent to HW_BRANCH_INSTRUCTIONS.
54 * It counts:
55 * - all (taken) branch instructions,
56 * - instructions that explicitly write the PC,
57 * - exception generating instructions.
58 */
59 ARMV7_PERFCTR_PC_WRITE = 0x0C,
60 ARMV7_PERFCTR_PC_IMM_BRANCH = 0x0D,
61 ARMV7_PERFCTR_PC_PROC_RETURN = 0x0E,
62 ARMV7_PERFCTR_MEM_UNALIGNED_ACCESS = 0x0F,
63 ARMV7_PERFCTR_PC_BRANCH_MIS_PRED = 0x10,
64 ARMV7_PERFCTR_CLOCK_CYCLES = 0x11,
65 ARMV7_PERFCTR_PC_BRANCH_PRED = 0x12,
66
67 /* These events are defined by the PMUv2 supplement (ARM DDI 0457A). */
68 ARMV7_PERFCTR_MEM_ACCESS = 0x13,
69 ARMV7_PERFCTR_L1_ICACHE_ACCESS = 0x14,
70 ARMV7_PERFCTR_L1_DCACHE_WB = 0x15,
71 ARMV7_PERFCTR_L2_CACHE_ACCESS = 0x16,
72 ARMV7_PERFCTR_L2_CACHE_REFILL = 0x17,
73 ARMV7_PERFCTR_L2_CACHE_WB = 0x18,
74 ARMV7_PERFCTR_BUS_ACCESS = 0x19,
75 ARMV7_PERFCTR_MEM_ERROR = 0x1A,
76 ARMV7_PERFCTR_INSTR_SPEC = 0x1B,
77 ARMV7_PERFCTR_TTBR_WRITE = 0x1C,
78 ARMV7_PERFCTR_BUS_CYCLES = 0x1D,
79
80 ARMV7_PERFCTR_CPU_CYCLES = 0xFF
81 };
82
83 /* ARMv7 Cortex-A8 specific event types */
84 enum armv7_a8_perf_types {
85 ARMV7_A8_PERFCTR_L2_CACHE_ACCESS = 0x43,
86 ARMV7_A8_PERFCTR_L2_CACHE_REFILL = 0x44,
87 ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS = 0x50,
88 ARMV7_A8_PERFCTR_STALL_ISIDE = 0x56,
89 };
90
91 /* ARMv7 Cortex-A9 specific event types */
92 enum armv7_a9_perf_types {
93 ARMV7_A9_PERFCTR_INSTR_CORE_RENAME = 0x68,
94 ARMV7_A9_PERFCTR_STALL_ICACHE = 0x60,
95 ARMV7_A9_PERFCTR_STALL_DISPATCH = 0x66,
96 };
97
98 /* ARMv7 Cortex-A5 specific event types */
99 enum armv7_a5_perf_types {
100 ARMV7_A5_PERFCTR_PREFETCH_LINEFILL = 0xc2,
101 ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP = 0xc3,
102 };
103
104 /* ARMv7 Cortex-A15 specific event types */
105 enum armv7_a15_perf_types {
106 ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ = 0x40,
107 ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE = 0x41,
108 ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ = 0x42,
109 ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE = 0x43,
110
111 ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ = 0x4C,
112 ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE = 0x4D,
113
114 ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ = 0x50,
115 ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE = 0x51,
116 ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ = 0x52,
117 ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE = 0x53,
118
119 ARMV7_A15_PERFCTR_PC_WRITE_SPEC = 0x76,
120 };
121
122 /* ARMv7 Cortex-A12 specific event types */
123 enum armv7_a12_perf_types {
124 ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ = 0x40,
125 ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE = 0x41,
126
127 ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ = 0x50,
128 ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE = 0x51,
129
130 ARMV7_A12_PERFCTR_PC_WRITE_SPEC = 0x76,
131
132 ARMV7_A12_PERFCTR_PF_TLB_REFILL = 0xe7,
133 };
134
135 /* ARMv7 Krait specific event types */
136 enum krait_perf_types {
137 KRAIT_PMRESR0_GROUP0 = 0xcc,
138 KRAIT_PMRESR1_GROUP0 = 0xd0,
139 KRAIT_PMRESR2_GROUP0 = 0xd4,
140 KRAIT_VPMRESR0_GROUP0 = 0xd8,
141
142 KRAIT_PERFCTR_L1_ICACHE_ACCESS = 0x10011,
143 KRAIT_PERFCTR_L1_ICACHE_MISS = 0x10010,
144
145 KRAIT_PERFCTR_L1_ITLB_ACCESS = 0x12222,
146 KRAIT_PERFCTR_L1_DTLB_ACCESS = 0x12210,
147 };
148
149 /* ARMv7 Scorpion specific event types */
150 enum scorpion_perf_types {
151 SCORPION_LPM0_GROUP0 = 0x4c,
152 SCORPION_LPM1_GROUP0 = 0x50,
153 SCORPION_LPM2_GROUP0 = 0x54,
154 SCORPION_L2LPM_GROUP0 = 0x58,
155 SCORPION_VLPM_GROUP0 = 0x5c,
156
157 SCORPION_ICACHE_ACCESS = 0x10053,
158 SCORPION_ICACHE_MISS = 0x10052,
159
160 SCORPION_DTLB_ACCESS = 0x12013,
161 SCORPION_DTLB_MISS = 0x12012,
162
163 SCORPION_ITLB_MISS = 0x12021,
164 };
165
166 /*
167 * Cortex-A8 HW events mapping
168 *
169 * The hardware events that we support. We do support cache operations but
170 * we have harvard caches and no way to combine instruction and data
171 * accesses/misses in hardware.
172 */
173 static const unsigned armv7_a8_perf_map[PERF_COUNT_HW_MAX] = {
174 PERF_MAP_ALL_UNSUPPORTED,
175 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
176 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
177 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
178 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
179 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
180 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
181 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A8_PERFCTR_STALL_ISIDE,
182 };
183
184 static const unsigned armv7_a8_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
185 [PERF_COUNT_HW_CACHE_OP_MAX]
186 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
187 PERF_CACHE_MAP_ALL_UNSUPPORTED,
188
189 /*
190 * The performance counters don't differentiate between read and write
191 * accesses/misses so this isn't strictly correct, but it's the best we
192 * can do. Writes and reads get combined.
193 */
194 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
195 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
196 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
197 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
198
199 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS,
200 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
201
202 [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,
203 [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A8_PERFCTR_L2_CACHE_REFILL,
204 [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,
205 [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A8_PERFCTR_L2_CACHE_REFILL,
206
207 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
208 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
209
210 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
211 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
212
213 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
214 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
215 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
216 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
217 };
218
219 /*
220 * Cortex-A9 HW events mapping
221 */
222 static const unsigned armv7_a9_perf_map[PERF_COUNT_HW_MAX] = {
223 PERF_MAP_ALL_UNSUPPORTED,
224 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
225 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_A9_PERFCTR_INSTR_CORE_RENAME,
226 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
227 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
228 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
229 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
230 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A9_PERFCTR_STALL_ICACHE,
231 [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV7_A9_PERFCTR_STALL_DISPATCH,
232 };
233
234 static const unsigned armv7_a9_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
235 [PERF_COUNT_HW_CACHE_OP_MAX]
236 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
237 PERF_CACHE_MAP_ALL_UNSUPPORTED,
238
239 /*
240 * The performance counters don't differentiate between read and write
241 * accesses/misses so this isn't strictly correct, but it's the best we
242 * can do. Writes and reads get combined.
243 */
244 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
245 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
246 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
247 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
248
249 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
250
251 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
252 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
253
254 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
255 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
256
257 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
258 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
259 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
260 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
261 };
262
263 /*
264 * Cortex-A5 HW events mapping
265 */
266 static const unsigned armv7_a5_perf_map[PERF_COUNT_HW_MAX] = {
267 PERF_MAP_ALL_UNSUPPORTED,
268 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
269 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
270 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
271 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
272 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
273 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
274 };
275
276 static const unsigned armv7_a5_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
277 [PERF_COUNT_HW_CACHE_OP_MAX]
278 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
279 PERF_CACHE_MAP_ALL_UNSUPPORTED,
280
281 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
282 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
283 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
284 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
285 [C(L1D)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,
286 [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,
287
288 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
289 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
290 /*
291 * The prefetch counters don't differentiate between the I side and the
292 * D side.
293 */
294 [C(L1I)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,
295 [C(L1I)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,
296
297 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
298 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
299
300 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
301 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
302
303 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
304 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
305 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
306 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
307 };
308
309 /*
310 * Cortex-A15 HW events mapping
311 */
312 static const unsigned armv7_a15_perf_map[PERF_COUNT_HW_MAX] = {
313 PERF_MAP_ALL_UNSUPPORTED,
314 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
315 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
316 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
317 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
318 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_A15_PERFCTR_PC_WRITE_SPEC,
319 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
320 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_BUS_CYCLES,
321 };
322
323 static const unsigned armv7_a15_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
324 [PERF_COUNT_HW_CACHE_OP_MAX]
325 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
326 PERF_CACHE_MAP_ALL_UNSUPPORTED,
327
328 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ,
329 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ,
330 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE,
331 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE,
332
333 /*
334 * Not all performance counters differentiate between read and write
335 * accesses/misses so we're not always strictly correct, but it's the
336 * best we can do. Writes and reads get combined in these cases.
337 */
338 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
339 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
340
341 [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ,
342 [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ,
343 [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE,
344 [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE,
345
346 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ,
347 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE,
348
349 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
350 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
351
352 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
353 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
354 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
355 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
356 };
357
358 /*
359 * Cortex-A7 HW events mapping
360 */
361 static const unsigned armv7_a7_perf_map[PERF_COUNT_HW_MAX] = {
362 PERF_MAP_ALL_UNSUPPORTED,
363 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
364 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
365 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
366 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
367 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
368 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
369 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_BUS_CYCLES,
370 };
371
372 static const unsigned armv7_a7_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
373 [PERF_COUNT_HW_CACHE_OP_MAX]
374 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
375 PERF_CACHE_MAP_ALL_UNSUPPORTED,
376
377 /*
378 * The performance counters don't differentiate between read and write
379 * accesses/misses so this isn't strictly correct, but it's the best we
380 * can do. Writes and reads get combined.
381 */
382 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
383 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
384 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
385 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
386
387 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
388 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
389
390 [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L2_CACHE_ACCESS,
391 [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL,
392 [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L2_CACHE_ACCESS,
393 [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL,
394
395 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
396 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
397
398 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
399 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
400
401 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
402 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
403 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
404 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
405 };
406
407 /*
408 * Cortex-A12 HW events mapping
409 */
410 static const unsigned armv7_a12_perf_map[PERF_COUNT_HW_MAX] = {
411 PERF_MAP_ALL_UNSUPPORTED,
412 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
413 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
414 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
415 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
416 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_A12_PERFCTR_PC_WRITE_SPEC,
417 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
418 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_BUS_CYCLES,
419 };
420
421 static const unsigned armv7_a12_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
422 [PERF_COUNT_HW_CACHE_OP_MAX]
423 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
424 PERF_CACHE_MAP_ALL_UNSUPPORTED,
425
426 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ,
427 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
428 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE,
429 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
430
431 /*
432 * Not all performance counters differentiate between read and write
433 * accesses/misses so we're not always strictly correct, but it's the
434 * best we can do. Writes and reads get combined in these cases.
435 */
436 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
437 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
438
439 [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ,
440 [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL,
441 [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE,
442 [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL,
443
444 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
445 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
446 [C(DTLB)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV7_A12_PERFCTR_PF_TLB_REFILL,
447
448 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
449 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
450
451 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
452 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
453 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
454 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
455 };
456
457 /*
458 * Krait HW events mapping
459 */
460 static const unsigned krait_perf_map[PERF_COUNT_HW_MAX] = {
461 PERF_MAP_ALL_UNSUPPORTED,
462 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
463 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
464 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
465 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
466 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES,
467 };
468
469 static const unsigned krait_perf_map_no_branch[PERF_COUNT_HW_MAX] = {
470 PERF_MAP_ALL_UNSUPPORTED,
471 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
472 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
473 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
474 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES,
475 };
476
477 static const unsigned krait_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
478 [PERF_COUNT_HW_CACHE_OP_MAX]
479 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
480 PERF_CACHE_MAP_ALL_UNSUPPORTED,
481
482 /*
483 * The performance counters don't differentiate between read and write
484 * accesses/misses so this isn't strictly correct, but it's the best we
485 * can do. Writes and reads get combined.
486 */
487 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
488 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
489 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
490 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
491
492 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ICACHE_ACCESS,
493 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = KRAIT_PERFCTR_L1_ICACHE_MISS,
494
495 [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_DTLB_ACCESS,
496 [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_DTLB_ACCESS,
497
498 [C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ITLB_ACCESS,
499 [C(ITLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ITLB_ACCESS,
500
501 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
502 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
503 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
504 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
505 };
506
507 /*
508 * Scorpion HW events mapping
509 */
510 static const unsigned scorpion_perf_map[PERF_COUNT_HW_MAX] = {
511 PERF_MAP_ALL_UNSUPPORTED,
512 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
513 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
514 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
515 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
516 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES,
517 };
518
519 static const unsigned scorpion_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
520 [PERF_COUNT_HW_CACHE_OP_MAX]
521 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
522 PERF_CACHE_MAP_ALL_UNSUPPORTED,
523 /*
524 * The performance counters don't differentiate between read and write
525 * accesses/misses so this isn't strictly correct, but it's the best we
526 * can do. Writes and reads get combined.
527 */
528 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
529 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
530 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
531 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
532 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_ICACHE_ACCESS,
533 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ICACHE_MISS,
534 /*
535 * Only ITLB misses and DTLB refills are supported. If users want the
536 * DTLB refills misses a raw counter must be used.
537 */
538 [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS,
539 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_DTLB_MISS,
540 [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS,
541 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_DTLB_MISS,
542 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ITLB_MISS,
543 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_ITLB_MISS,
544 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
545 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
546 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
547 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
548 };
549
550 /*
551 * Perf Events' indices
552 */
553 #define ARMV7_IDX_CYCLE_COUNTER 0
554 #define ARMV7_IDX_COUNTER0 1
555 #define ARMV7_IDX_COUNTER_LAST(cpu_pmu) \
556 (ARMV7_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1)
557
558 #define ARMV7_MAX_COUNTERS 32
559 #define ARMV7_COUNTER_MASK (ARMV7_MAX_COUNTERS - 1)
560
561 /*
562 * ARMv7 low level PMNC access
563 */
564
565 /*
566 * Perf Event to low level counters mapping
567 */
568 #define ARMV7_IDX_TO_COUNTER(x) \
569 (((x) - ARMV7_IDX_COUNTER0) & ARMV7_COUNTER_MASK)
570
571 /*
572 * Per-CPU PMNC: config reg
573 */
574 #define ARMV7_PMNC_E (1 << 0) /* Enable all counters */
575 #define ARMV7_PMNC_P (1 << 1) /* Reset all counters */
576 #define ARMV7_PMNC_C (1 << 2) /* Cycle counter reset */
577 #define ARMV7_PMNC_D (1 << 3) /* CCNT counts every 64th cpu cycle */
578 #define ARMV7_PMNC_X (1 << 4) /* Export to ETM */
579 #define ARMV7_PMNC_DP (1 << 5) /* Disable CCNT if non-invasive debug*/
580 #define ARMV7_PMNC_N_SHIFT 11 /* Number of counters supported */
581 #define ARMV7_PMNC_N_MASK 0x1f
582 #define ARMV7_PMNC_MASK 0x3f /* Mask for writable bits */
583
584 /*
585 * FLAG: counters overflow flag status reg
586 */
587 #define ARMV7_FLAG_MASK 0xffffffff /* Mask for writable bits */
588 #define ARMV7_OVERFLOWED_MASK ARMV7_FLAG_MASK
589
590 /*
591 * PMXEVTYPER: Event selection reg
592 */
593 #define ARMV7_EVTYPE_MASK 0xc80000ff /* Mask for writable bits */
594 #define ARMV7_EVTYPE_EVENT 0xff /* Mask for EVENT bits */
595
596 /*
597 * Event filters for PMUv2
598 */
599 #define ARMV7_EXCLUDE_PL1 (1 << 31)
600 #define ARMV7_EXCLUDE_USER (1 << 30)
601 #define ARMV7_INCLUDE_HYP (1 << 27)
602
603 static inline u32 armv7_pmnc_read(void)
604 {
605 u32 val;
606 asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r"(val));
607 return val;
608 }
609
610 static inline void armv7_pmnc_write(u32 val)
611 {
612 val &= ARMV7_PMNC_MASK;
613 isb();
614 asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r"(val));
615 }
616
617 static inline int armv7_pmnc_has_overflowed(u32 pmnc)
618 {
619 return pmnc & ARMV7_OVERFLOWED_MASK;
620 }
621
622 static inline int armv7_pmnc_counter_valid(struct arm_pmu *cpu_pmu, int idx)
623 {
624 return idx >= ARMV7_IDX_CYCLE_COUNTER &&
625 idx <= ARMV7_IDX_COUNTER_LAST(cpu_pmu);
626 }
627
628 static inline int armv7_pmnc_counter_has_overflowed(u32 pmnc, int idx)
629 {
630 return pmnc & BIT(ARMV7_IDX_TO_COUNTER(idx));
631 }
632
633 static inline void armv7_pmnc_select_counter(int idx)
634 {
635 u32 counter = ARMV7_IDX_TO_COUNTER(idx);
636 asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (counter));
637 isb();
638 }
639
640 static inline u32 armv7pmu_read_counter(struct perf_event *event)
641 {
642 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
643 struct hw_perf_event *hwc = &event->hw;
644 int idx = hwc->idx;
645 u32 value = 0;
646
647 if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
648 pr_err("CPU%u reading wrong counter %d\n",
649 smp_processor_id(), idx);
650 } else if (idx == ARMV7_IDX_CYCLE_COUNTER) {
651 asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (value));
652 } else {
653 armv7_pmnc_select_counter(idx);
654 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (value));
655 }
656
657 return value;
658 }
659
660 static inline void armv7pmu_write_counter(struct perf_event *event, u32 value)
661 {
662 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
663 struct hw_perf_event *hwc = &event->hw;
664 int idx = hwc->idx;
665
666 if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
667 pr_err("CPU%u writing wrong counter %d\n",
668 smp_processor_id(), idx);
669 } else if (idx == ARMV7_IDX_CYCLE_COUNTER) {
670 asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (value));
671 } else {
672 armv7_pmnc_select_counter(idx);
673 asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" (value));
674 }
675 }
676
677 static inline void armv7_pmnc_write_evtsel(int idx, u32 val)
678 {
679 armv7_pmnc_select_counter(idx);
680 val &= ARMV7_EVTYPE_MASK;
681 asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val));
682 }
683
684 static inline void armv7_pmnc_enable_counter(int idx)
685 {
686 u32 counter = ARMV7_IDX_TO_COUNTER(idx);
687 asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (BIT(counter)));
688 }
689
690 static inline void armv7_pmnc_disable_counter(int idx)
691 {
692 u32 counter = ARMV7_IDX_TO_COUNTER(idx);
693 asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (BIT(counter)));
694 }
695
696 static inline void armv7_pmnc_enable_intens(int idx)
697 {
698 u32 counter = ARMV7_IDX_TO_COUNTER(idx);
699 asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (BIT(counter)));
700 }
701
702 static inline void armv7_pmnc_disable_intens(int idx)
703 {
704 u32 counter = ARMV7_IDX_TO_COUNTER(idx);
705 asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (BIT(counter)));
706 isb();
707 /* Clear the overflow flag in case an interrupt is pending. */
708 asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (BIT(counter)));
709 isb();
710 }
711
712 static inline u32 armv7_pmnc_getreset_flags(void)
713 {
714 u32 val;
715
716 /* Read */
717 asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
718
719 /* Write to clear flags */
720 val &= ARMV7_FLAG_MASK;
721 asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (val));
722
723 return val;
724 }
725
726 #ifdef DEBUG
727 static void armv7_pmnc_dump_regs(struct arm_pmu *cpu_pmu)
728 {
729 u32 val;
730 unsigned int cnt;
731
732 pr_info("PMNC registers dump:\n");
733
734 asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val));
735 pr_info("PMNC =0x%08x\n", val);
736
737 asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r" (val));
738 pr_info("CNTENS=0x%08x\n", val);
739
740 asm volatile("mrc p15, 0, %0, c9, c14, 1" : "=r" (val));
741 pr_info("INTENS=0x%08x\n", val);
742
743 asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
744 pr_info("FLAGS =0x%08x\n", val);
745
746 asm volatile("mrc p15, 0, %0, c9, c12, 5" : "=r" (val));
747 pr_info("SELECT=0x%08x\n", val);
748
749 asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val));
750 pr_info("CCNT =0x%08x\n", val);
751
752 for (cnt = ARMV7_IDX_COUNTER0;
753 cnt <= ARMV7_IDX_COUNTER_LAST(cpu_pmu); cnt++) {
754 armv7_pmnc_select_counter(cnt);
755 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val));
756 pr_info("CNT[%d] count =0x%08x\n",
757 ARMV7_IDX_TO_COUNTER(cnt), val);
758 asm volatile("mrc p15, 0, %0, c9, c13, 1" : "=r" (val));
759 pr_info("CNT[%d] evtsel=0x%08x\n",
760 ARMV7_IDX_TO_COUNTER(cnt), val);
761 }
762 }
763 #endif
764
765 static void armv7pmu_enable_event(struct perf_event *event)
766 {
767 unsigned long flags;
768 struct hw_perf_event *hwc = &event->hw;
769 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
770 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
771 int idx = hwc->idx;
772
773 if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
774 pr_err("CPU%u enabling wrong PMNC counter IRQ enable %d\n",
775 smp_processor_id(), idx);
776 return;
777 }
778
779 /*
780 * Enable counter and interrupt, and set the counter to count
781 * the event that we're interested in.
782 */
783 raw_spin_lock_irqsave(&events->pmu_lock, flags);
784
785 /*
786 * Disable counter
787 */
788 armv7_pmnc_disable_counter(idx);
789
790 /*
791 * Set event (if destined for PMNx counters)
792 * We only need to set the event for the cycle counter if we
793 * have the ability to perform event filtering.
794 */
795 if (cpu_pmu->set_event_filter || idx != ARMV7_IDX_CYCLE_COUNTER)
796 armv7_pmnc_write_evtsel(idx, hwc->config_base);
797
798 /*
799 * Enable interrupt for this counter
800 */
801 armv7_pmnc_enable_intens(idx);
802
803 /*
804 * Enable counter
805 */
806 armv7_pmnc_enable_counter(idx);
807
808 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
809 }
810
811 static void armv7pmu_disable_event(struct perf_event *event)
812 {
813 unsigned long flags;
814 struct hw_perf_event *hwc = &event->hw;
815 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
816 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
817 int idx = hwc->idx;
818
819 if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
820 pr_err("CPU%u disabling wrong PMNC counter IRQ enable %d\n",
821 smp_processor_id(), idx);
822 return;
823 }
824
825 /*
826 * Disable counter and interrupt
827 */
828 raw_spin_lock_irqsave(&events->pmu_lock, flags);
829
830 /*
831 * Disable counter
832 */
833 armv7_pmnc_disable_counter(idx);
834
835 /*
836 * Disable interrupt for this counter
837 */
838 armv7_pmnc_disable_intens(idx);
839
840 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
841 }
842
843 static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev)
844 {
845 u32 pmnc;
846 struct perf_sample_data data;
847 struct arm_pmu *cpu_pmu = (struct arm_pmu *)dev;
848 struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);
849 struct pt_regs *regs;
850 int idx;
851
852 /*
853 * Get and reset the IRQ flags
854 */
855 pmnc = armv7_pmnc_getreset_flags();
856
857 /*
858 * Did an overflow occur?
859 */
860 if (!armv7_pmnc_has_overflowed(pmnc))
861 return IRQ_NONE;
862
863 /*
864 * Handle the counter(s) overflow(s)
865 */
866 regs = get_irq_regs();
867
868 for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
869 struct perf_event *event = cpuc->events[idx];
870 struct hw_perf_event *hwc;
871
872 /* Ignore if we don't have an event. */
873 if (!event)
874 continue;
875
876 /*
877 * We have a single interrupt for all counters. Check that
878 * each counter has overflowed before we process it.
879 */
880 if (!armv7_pmnc_counter_has_overflowed(pmnc, idx))
881 continue;
882
883 hwc = &event->hw;
884 armpmu_event_update(event);
885 perf_sample_data_init(&data, 0, hwc->last_period);
886 if (!armpmu_event_set_period(event))
887 continue;
888
889 if (perf_event_overflow(event, &data, regs))
890 cpu_pmu->disable(event);
891 }
892
893 /*
894 * Handle the pending perf events.
895 *
896 * Note: this call *must* be run with interrupts disabled. For
897 * platforms that can have the PMU interrupts raised as an NMI, this
898 * will not work.
899 */
900 irq_work_run();
901
902 return IRQ_HANDLED;
903 }
904
905 static void armv7pmu_start(struct arm_pmu *cpu_pmu)
906 {
907 unsigned long flags;
908 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
909
910 raw_spin_lock_irqsave(&events->pmu_lock, flags);
911 /* Enable all counters */
912 armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E);
913 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
914 }
915
916 static void armv7pmu_stop(struct arm_pmu *cpu_pmu)
917 {
918 unsigned long flags;
919 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
920
921 raw_spin_lock_irqsave(&events->pmu_lock, flags);
922 /* Disable all counters */
923 armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E);
924 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
925 }
926
927 static int armv7pmu_get_event_idx(struct pmu_hw_events *cpuc,
928 struct perf_event *event)
929 {
930 int idx;
931 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
932 struct hw_perf_event *hwc = &event->hw;
933 unsigned long evtype = hwc->config_base & ARMV7_EVTYPE_EVENT;
934
935 /* Always place a cycle counter into the cycle counter. */
936 if (evtype == ARMV7_PERFCTR_CPU_CYCLES) {
937 if (test_and_set_bit(ARMV7_IDX_CYCLE_COUNTER, cpuc->used_mask))
938 return -EAGAIN;
939
940 return ARMV7_IDX_CYCLE_COUNTER;
941 }
942
943 /*
944 * For anything other than a cycle counter, try and use
945 * the events counters
946 */
947 for (idx = ARMV7_IDX_COUNTER0; idx < cpu_pmu->num_events; ++idx) {
948 if (!test_and_set_bit(idx, cpuc->used_mask))
949 return idx;
950 }
951
952 /* The counters are all in use. */
953 return -EAGAIN;
954 }
955
956 /*
957 * Add an event filter to a given event. This will only work for PMUv2 PMUs.
958 */
959 static int armv7pmu_set_event_filter(struct hw_perf_event *event,
960 struct perf_event_attr *attr)
961 {
962 unsigned long config_base = 0;
963
964 if (attr->exclude_idle)
965 return -EPERM;
966 if (attr->exclude_user)
967 config_base |= ARMV7_EXCLUDE_USER;
968 if (attr->exclude_kernel)
969 config_base |= ARMV7_EXCLUDE_PL1;
970 if (!attr->exclude_hv)
971 config_base |= ARMV7_INCLUDE_HYP;
972
973 /*
974 * Install the filter into config_base as this is used to
975 * construct the event type.
976 */
977 event->config_base = config_base;
978
979 return 0;
980 }
981
982 static void armv7pmu_reset(void *info)
983 {
984 struct arm_pmu *cpu_pmu = (struct arm_pmu *)info;
985 u32 idx, nb_cnt = cpu_pmu->num_events;
986
987 /* The counter and interrupt enable registers are unknown at reset. */
988 for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
989 armv7_pmnc_disable_counter(idx);
990 armv7_pmnc_disable_intens(idx);
991 }
992
993 /* Initialize & Reset PMNC: C and P bits */
994 armv7_pmnc_write(ARMV7_PMNC_P | ARMV7_PMNC_C);
995 }
996
997 static int armv7_a8_map_event(struct perf_event *event)
998 {
999 return armpmu_map_event(event, &armv7_a8_perf_map,
1000 &armv7_a8_perf_cache_map, 0xFF);
1001 }
1002
1003 static int armv7_a9_map_event(struct perf_event *event)
1004 {
1005 return armpmu_map_event(event, &armv7_a9_perf_map,
1006 &armv7_a9_perf_cache_map, 0xFF);
1007 }
1008
1009 static int armv7_a5_map_event(struct perf_event *event)
1010 {
1011 return armpmu_map_event(event, &armv7_a5_perf_map,
1012 &armv7_a5_perf_cache_map, 0xFF);
1013 }
1014
1015 static int armv7_a15_map_event(struct perf_event *event)
1016 {
1017 return armpmu_map_event(event, &armv7_a15_perf_map,
1018 &armv7_a15_perf_cache_map, 0xFF);
1019 }
1020
1021 static int armv7_a7_map_event(struct perf_event *event)
1022 {
1023 return armpmu_map_event(event, &armv7_a7_perf_map,
1024 &armv7_a7_perf_cache_map, 0xFF);
1025 }
1026
1027 static int armv7_a12_map_event(struct perf_event *event)
1028 {
1029 return armpmu_map_event(event, &armv7_a12_perf_map,
1030 &armv7_a12_perf_cache_map, 0xFF);
1031 }
1032
1033 static int krait_map_event(struct perf_event *event)
1034 {
1035 return armpmu_map_event(event, &krait_perf_map,
1036 &krait_perf_cache_map, 0xFFFFF);
1037 }
1038
1039 static int krait_map_event_no_branch(struct perf_event *event)
1040 {
1041 return armpmu_map_event(event, &krait_perf_map_no_branch,
1042 &krait_perf_cache_map, 0xFFFFF);
1043 }
1044
1045 static int scorpion_map_event(struct perf_event *event)
1046 {
1047 return armpmu_map_event(event, &scorpion_perf_map,
1048 &scorpion_perf_cache_map, 0xFFFFF);
1049 }
1050
1051 static void armv7pmu_init(struct arm_pmu *cpu_pmu)
1052 {
1053 cpu_pmu->handle_irq = armv7pmu_handle_irq;
1054 cpu_pmu->enable = armv7pmu_enable_event;
1055 cpu_pmu->disable = armv7pmu_disable_event;
1056 cpu_pmu->read_counter = armv7pmu_read_counter;
1057 cpu_pmu->write_counter = armv7pmu_write_counter;
1058 cpu_pmu->get_event_idx = armv7pmu_get_event_idx;
1059 cpu_pmu->start = armv7pmu_start;
1060 cpu_pmu->stop = armv7pmu_stop;
1061 cpu_pmu->reset = armv7pmu_reset;
1062 cpu_pmu->max_period = (1LLU << 32) - 1;
1063 };
1064
1065 static void armv7_read_num_pmnc_events(void *info)
1066 {
1067 int *nb_cnt = info;
1068
1069 /* Read the nb of CNTx counters supported from PMNC */
1070 *nb_cnt = (armv7_pmnc_read() >> ARMV7_PMNC_N_SHIFT) & ARMV7_PMNC_N_MASK;
1071
1072 /* Add the CPU cycles counter */
1073 *nb_cnt += 1;
1074 }
1075
1076 static int armv7_probe_num_events(struct arm_pmu *arm_pmu)
1077 {
1078 return smp_call_function_any(&arm_pmu->supported_cpus,
1079 armv7_read_num_pmnc_events,
1080 &arm_pmu->num_events, 1);
1081 }
1082
1083 static int armv7_a8_pmu_init(struct arm_pmu *cpu_pmu)
1084 {
1085 armv7pmu_init(cpu_pmu);
1086 cpu_pmu->name = "armv7_cortex_a8";
1087 cpu_pmu->map_event = armv7_a8_map_event;
1088 return armv7_probe_num_events(cpu_pmu);
1089 }
1090
1091 static int armv7_a9_pmu_init(struct arm_pmu *cpu_pmu)
1092 {
1093 armv7pmu_init(cpu_pmu);
1094 cpu_pmu->name = "armv7_cortex_a9";
1095 cpu_pmu->map_event = armv7_a9_map_event;
1096 return armv7_probe_num_events(cpu_pmu);
1097 }
1098
1099 static int armv7_a5_pmu_init(struct arm_pmu *cpu_pmu)
1100 {
1101 armv7pmu_init(cpu_pmu);
1102 cpu_pmu->name = "armv7_cortex_a5";
1103 cpu_pmu->map_event = armv7_a5_map_event;
1104 return armv7_probe_num_events(cpu_pmu);
1105 }
1106
1107 static int armv7_a15_pmu_init(struct arm_pmu *cpu_pmu)
1108 {
1109 armv7pmu_init(cpu_pmu);
1110 cpu_pmu->name = "armv7_cortex_a15";
1111 cpu_pmu->map_event = armv7_a15_map_event;
1112 cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1113 return armv7_probe_num_events(cpu_pmu);
1114 }
1115
1116 static int armv7_a7_pmu_init(struct arm_pmu *cpu_pmu)
1117 {
1118 armv7pmu_init(cpu_pmu);
1119 cpu_pmu->name = "armv7_cortex_a7";
1120 cpu_pmu->map_event = armv7_a7_map_event;
1121 cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1122 return armv7_probe_num_events(cpu_pmu);
1123 }
1124
1125 static int armv7_a12_pmu_init(struct arm_pmu *cpu_pmu)
1126 {
1127 armv7pmu_init(cpu_pmu);
1128 cpu_pmu->name = "armv7_cortex_a12";
1129 cpu_pmu->map_event = armv7_a12_map_event;
1130 cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1131 return armv7_probe_num_events(cpu_pmu);
1132 }
1133
1134 static int armv7_a17_pmu_init(struct arm_pmu *cpu_pmu)
1135 {
1136 int ret = armv7_a12_pmu_init(cpu_pmu);
1137 cpu_pmu->name = "armv7_cortex_a17";
1138 return ret;
1139 }
1140
1141 /*
1142 * Krait Performance Monitor Region Event Selection Register (PMRESRn)
1143 *
1144 * 31 30 24 16 8 0
1145 * +--------------------------------+
1146 * PMRESR0 | EN | CC | CC | CC | CC | N = 1, R = 0
1147 * +--------------------------------+
1148 * PMRESR1 | EN | CC | CC | CC | CC | N = 1, R = 1
1149 * +--------------------------------+
1150 * PMRESR2 | EN | CC | CC | CC | CC | N = 1, R = 2
1151 * +--------------------------------+
1152 * VPMRESR0 | EN | CC | CC | CC | CC | N = 2, R = ?
1153 * +--------------------------------+
1154 * EN | G=3 | G=2 | G=1 | G=0
1155 *
1156 * Event Encoding:
1157 *
1158 * hwc->config_base = 0xNRCCG
1159 *
1160 * N = prefix, 1 for Krait CPU (PMRESRn), 2 for Venum VFP (VPMRESR)
1161 * R = region register
1162 * CC = class of events the group G is choosing from
1163 * G = group or particular event
1164 *
1165 * Example: 0x12021 is a Krait CPU event in PMRESR2's group 1 with code 2
1166 *
1167 * A region (R) corresponds to a piece of the CPU (execution unit, instruction
1168 * unit, etc.) while the event code (CC) corresponds to a particular class of
1169 * events (interrupts for example). An event code is broken down into
1170 * groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for
1171 * example).
1172 */
1173
1174 #define KRAIT_EVENT (1 << 16)
1175 #define VENUM_EVENT (2 << 16)
1176 #define KRAIT_EVENT_MASK (KRAIT_EVENT | VENUM_EVENT)
1177 #define PMRESRn_EN BIT(31)
1178
1179 #define EVENT_REGION(event) (((event) >> 12) & 0xf) /* R */
1180 #define EVENT_GROUP(event) ((event) & 0xf) /* G */
1181 #define EVENT_CODE(event) (((event) >> 4) & 0xff) /* CC */
1182 #define EVENT_VENUM(event) (!!(event & VENUM_EVENT)) /* N=2 */
1183 #define EVENT_CPU(event) (!!(event & KRAIT_EVENT)) /* N=1 */
1184
1185 static u32 krait_read_pmresrn(int n)
1186 {
1187 u32 val;
1188
1189 switch (n) {
1190 case 0:
1191 asm volatile("mrc p15, 1, %0, c9, c15, 0" : "=r" (val));
1192 break;
1193 case 1:
1194 asm volatile("mrc p15, 1, %0, c9, c15, 1" : "=r" (val));
1195 break;
1196 case 2:
1197 asm volatile("mrc p15, 1, %0, c9, c15, 2" : "=r" (val));
1198 break;
1199 default:
1200 BUG(); /* Should be validated in krait_pmu_get_event_idx() */
1201 }
1202
1203 return val;
1204 }
1205
1206 static void krait_write_pmresrn(int n, u32 val)
1207 {
1208 switch (n) {
1209 case 0:
1210 asm volatile("mcr p15, 1, %0, c9, c15, 0" : : "r" (val));
1211 break;
1212 case 1:
1213 asm volatile("mcr p15, 1, %0, c9, c15, 1" : : "r" (val));
1214 break;
1215 case 2:
1216 asm volatile("mcr p15, 1, %0, c9, c15, 2" : : "r" (val));
1217 break;
1218 default:
1219 BUG(); /* Should be validated in krait_pmu_get_event_idx() */
1220 }
1221 }
1222
1223 static u32 venum_read_pmresr(void)
1224 {
1225 u32 val;
1226 asm volatile("mrc p10, 7, %0, c11, c0, 0" : "=r" (val));
1227 return val;
1228 }
1229
1230 static void venum_write_pmresr(u32 val)
1231 {
1232 asm volatile("mcr p10, 7, %0, c11, c0, 0" : : "r" (val));
1233 }
1234
1235 static void venum_pre_pmresr(u32 *venum_orig_val, u32 *fp_orig_val)
1236 {
1237 u32 venum_new_val;
1238 u32 fp_new_val;
1239
1240 BUG_ON(preemptible());
1241 /* CPACR Enable CP10 and CP11 access */
1242 *venum_orig_val = get_copro_access();
1243 venum_new_val = *venum_orig_val | CPACC_SVC(10) | CPACC_SVC(11);
1244 set_copro_access(venum_new_val);
1245
1246 /* Enable FPEXC */
1247 *fp_orig_val = fmrx(FPEXC);
1248 fp_new_val = *fp_orig_val | FPEXC_EN;
1249 fmxr(FPEXC, fp_new_val);
1250 }
1251
1252 static void venum_post_pmresr(u32 venum_orig_val, u32 fp_orig_val)
1253 {
1254 BUG_ON(preemptible());
1255 /* Restore FPEXC */
1256 fmxr(FPEXC, fp_orig_val);
1257 isb();
1258 /* Restore CPACR */
1259 set_copro_access(venum_orig_val);
1260 }
1261
1262 static u32 krait_get_pmresrn_event(unsigned int region)
1263 {
1264 static const u32 pmresrn_table[] = { KRAIT_PMRESR0_GROUP0,
1265 KRAIT_PMRESR1_GROUP0,
1266 KRAIT_PMRESR2_GROUP0 };
1267 return pmresrn_table[region];
1268 }
1269
1270 static void krait_evt_setup(int idx, u32 config_base)
1271 {
1272 u32 val;
1273 u32 mask;
1274 u32 vval, fval;
1275 unsigned int region = EVENT_REGION(config_base);
1276 unsigned int group = EVENT_GROUP(config_base);
1277 unsigned int code = EVENT_CODE(config_base);
1278 unsigned int group_shift;
1279 bool venum_event = EVENT_VENUM(config_base);
1280
1281 group_shift = group * 8;
1282 mask = 0xff << group_shift;
1283
1284 /* Configure evtsel for the region and group */
1285 if (venum_event)
1286 val = KRAIT_VPMRESR0_GROUP0;
1287 else
1288 val = krait_get_pmresrn_event(region);
1289 val += group;
1290 /* Mix in mode-exclusion bits */
1291 val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1);
1292 armv7_pmnc_write_evtsel(idx, val);
1293
1294 if (venum_event) {
1295 venum_pre_pmresr(&vval, &fval);
1296 val = venum_read_pmresr();
1297 val &= ~mask;
1298 val |= code << group_shift;
1299 val |= PMRESRn_EN;
1300 venum_write_pmresr(val);
1301 venum_post_pmresr(vval, fval);
1302 } else {
1303 val = krait_read_pmresrn(region);
1304 val &= ~mask;
1305 val |= code << group_shift;
1306 val |= PMRESRn_EN;
1307 krait_write_pmresrn(region, val);
1308 }
1309 }
1310
1311 static u32 clear_pmresrn_group(u32 val, int group)
1312 {
1313 u32 mask;
1314 int group_shift;
1315
1316 group_shift = group * 8;
1317 mask = 0xff << group_shift;
1318 val &= ~mask;
1319
1320 /* Don't clear enable bit if entire region isn't disabled */
1321 if (val & ~PMRESRn_EN)
1322 return val |= PMRESRn_EN;
1323
1324 return 0;
1325 }
1326
1327 static void krait_clearpmu(u32 config_base)
1328 {
1329 u32 val;
1330 u32 vval, fval;
1331 unsigned int region = EVENT_REGION(config_base);
1332 unsigned int group = EVENT_GROUP(config_base);
1333 bool venum_event = EVENT_VENUM(config_base);
1334
1335 if (venum_event) {
1336 venum_pre_pmresr(&vval, &fval);
1337 val = venum_read_pmresr();
1338 val = clear_pmresrn_group(val, group);
1339 venum_write_pmresr(val);
1340 venum_post_pmresr(vval, fval);
1341 } else {
1342 val = krait_read_pmresrn(region);
1343 val = clear_pmresrn_group(val, group);
1344 krait_write_pmresrn(region, val);
1345 }
1346 }
1347
1348 static void krait_pmu_disable_event(struct perf_event *event)
1349 {
1350 unsigned long flags;
1351 struct hw_perf_event *hwc = &event->hw;
1352 int idx = hwc->idx;
1353 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1354 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1355
1356 /* Disable counter and interrupt */
1357 raw_spin_lock_irqsave(&events->pmu_lock, flags);
1358
1359 /* Disable counter */
1360 armv7_pmnc_disable_counter(idx);
1361
1362 /*
1363 * Clear pmresr code (if destined for PMNx counters)
1364 */
1365 if (hwc->config_base & KRAIT_EVENT_MASK)
1366 krait_clearpmu(hwc->config_base);
1367
1368 /* Disable interrupt for this counter */
1369 armv7_pmnc_disable_intens(idx);
1370
1371 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1372 }
1373
1374 static void krait_pmu_enable_event(struct perf_event *event)
1375 {
1376 unsigned long flags;
1377 struct hw_perf_event *hwc = &event->hw;
1378 int idx = hwc->idx;
1379 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1380 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1381
1382 /*
1383 * Enable counter and interrupt, and set the counter to count
1384 * the event that we're interested in.
1385 */
1386 raw_spin_lock_irqsave(&events->pmu_lock, flags);
1387
1388 /* Disable counter */
1389 armv7_pmnc_disable_counter(idx);
1390
1391 /*
1392 * Set event (if destined for PMNx counters)
1393 * We set the event for the cycle counter because we
1394 * have the ability to perform event filtering.
1395 */
1396 if (hwc->config_base & KRAIT_EVENT_MASK)
1397 krait_evt_setup(idx, hwc->config_base);
1398 else
1399 armv7_pmnc_write_evtsel(idx, hwc->config_base);
1400
1401 /* Enable interrupt for this counter */
1402 armv7_pmnc_enable_intens(idx);
1403
1404 /* Enable counter */
1405 armv7_pmnc_enable_counter(idx);
1406
1407 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1408 }
1409
1410 static void krait_pmu_reset(void *info)
1411 {
1412 u32 vval, fval;
1413 struct arm_pmu *cpu_pmu = info;
1414 u32 idx, nb_cnt = cpu_pmu->num_events;
1415
1416 armv7pmu_reset(info);
1417
1418 /* Clear all pmresrs */
1419 krait_write_pmresrn(0, 0);
1420 krait_write_pmresrn(1, 0);
1421 krait_write_pmresrn(2, 0);
1422
1423 venum_pre_pmresr(&vval, &fval);
1424 venum_write_pmresr(0);
1425 venum_post_pmresr(vval, fval);
1426
1427 /* Reset PMxEVNCTCR to sane default */
1428 for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
1429 armv7_pmnc_select_counter(idx);
1430 asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1431 }
1432
1433 }
1434
1435 static int krait_event_to_bit(struct perf_event *event, unsigned int region,
1436 unsigned int group)
1437 {
1438 int bit;
1439 struct hw_perf_event *hwc = &event->hw;
1440 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1441
1442 if (hwc->config_base & VENUM_EVENT)
1443 bit = KRAIT_VPMRESR0_GROUP0;
1444 else
1445 bit = krait_get_pmresrn_event(region);
1446 bit -= krait_get_pmresrn_event(0);
1447 bit += group;
1448 /*
1449 * Lower bits are reserved for use by the counters (see
1450 * armv7pmu_get_event_idx() for more info)
1451 */
1452 bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1;
1453
1454 return bit;
1455 }
1456
1457 /*
1458 * We check for column exclusion constraints here.
1459 * Two events cant use the same group within a pmresr register.
1460 */
1461 static int krait_pmu_get_event_idx(struct pmu_hw_events *cpuc,
1462 struct perf_event *event)
1463 {
1464 int idx;
1465 int bit = -1;
1466 struct hw_perf_event *hwc = &event->hw;
1467 unsigned int region = EVENT_REGION(hwc->config_base);
1468 unsigned int code = EVENT_CODE(hwc->config_base);
1469 unsigned int group = EVENT_GROUP(hwc->config_base);
1470 bool venum_event = EVENT_VENUM(hwc->config_base);
1471 bool krait_event = EVENT_CPU(hwc->config_base);
1472
1473 if (venum_event || krait_event) {
1474 /* Ignore invalid events */
1475 if (group > 3 || region > 2)
1476 return -EINVAL;
1477 if (venum_event && (code & 0xe0))
1478 return -EINVAL;
1479
1480 bit = krait_event_to_bit(event, region, group);
1481 if (test_and_set_bit(bit, cpuc->used_mask))
1482 return -EAGAIN;
1483 }
1484
1485 idx = armv7pmu_get_event_idx(cpuc, event);
1486 if (idx < 0 && bit >= 0)
1487 clear_bit(bit, cpuc->used_mask);
1488
1489 return idx;
1490 }
1491
1492 static void krait_pmu_clear_event_idx(struct pmu_hw_events *cpuc,
1493 struct perf_event *event)
1494 {
1495 int bit;
1496 struct hw_perf_event *hwc = &event->hw;
1497 unsigned int region = EVENT_REGION(hwc->config_base);
1498 unsigned int group = EVENT_GROUP(hwc->config_base);
1499 bool venum_event = EVENT_VENUM(hwc->config_base);
1500 bool krait_event = EVENT_CPU(hwc->config_base);
1501
1502 if (venum_event || krait_event) {
1503 bit = krait_event_to_bit(event, region, group);
1504 clear_bit(bit, cpuc->used_mask);
1505 }
1506 }
1507
1508 static int krait_pmu_init(struct arm_pmu *cpu_pmu)
1509 {
1510 armv7pmu_init(cpu_pmu);
1511 cpu_pmu->name = "armv7_krait";
1512 /* Some early versions of Krait don't support PC write events */
1513 if (of_property_read_bool(cpu_pmu->plat_device->dev.of_node,
1514 "qcom,no-pc-write"))
1515 cpu_pmu->map_event = krait_map_event_no_branch;
1516 else
1517 cpu_pmu->map_event = krait_map_event;
1518 cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1519 cpu_pmu->reset = krait_pmu_reset;
1520 cpu_pmu->enable = krait_pmu_enable_event;
1521 cpu_pmu->disable = krait_pmu_disable_event;
1522 cpu_pmu->get_event_idx = krait_pmu_get_event_idx;
1523 cpu_pmu->clear_event_idx = krait_pmu_clear_event_idx;
1524 return armv7_probe_num_events(cpu_pmu);
1525 }
1526
1527 /*
1528 * Scorpion Local Performance Monitor Register (LPMn)
1529 *
1530 * 31 30 24 16 8 0
1531 * +--------------------------------+
1532 * LPM0 | EN | CC | CC | CC | CC | N = 1, R = 0
1533 * +--------------------------------+
1534 * LPM1 | EN | CC | CC | CC | CC | N = 1, R = 1
1535 * +--------------------------------+
1536 * LPM2 | EN | CC | CC | CC | CC | N = 1, R = 2
1537 * +--------------------------------+
1538 * L2LPM | EN | CC | CC | CC | CC | N = 1, R = 3
1539 * +--------------------------------+
1540 * VLPM | EN | CC | CC | CC | CC | N = 2, R = ?
1541 * +--------------------------------+
1542 * EN | G=3 | G=2 | G=1 | G=0
1543 *
1544 *
1545 * Event Encoding:
1546 *
1547 * hwc->config_base = 0xNRCCG
1548 *
1549 * N = prefix, 1 for Scorpion CPU (LPMn/L2LPM), 2 for Venum VFP (VLPM)
1550 * R = region register
1551 * CC = class of events the group G is choosing from
1552 * G = group or particular event
1553 *
1554 * Example: 0x12021 is a Scorpion CPU event in LPM2's group 1 with code 2
1555 *
1556 * A region (R) corresponds to a piece of the CPU (execution unit, instruction
1557 * unit, etc.) while the event code (CC) corresponds to a particular class of
1558 * events (interrupts for example). An event code is broken down into
1559 * groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for
1560 * example).
1561 */
1562
1563 static u32 scorpion_read_pmresrn(int n)
1564 {
1565 u32 val;
1566
1567 switch (n) {
1568 case 0:
1569 asm volatile("mrc p15, 0, %0, c15, c0, 0" : "=r" (val));
1570 break;
1571 case 1:
1572 asm volatile("mrc p15, 1, %0, c15, c0, 0" : "=r" (val));
1573 break;
1574 case 2:
1575 asm volatile("mrc p15, 2, %0, c15, c0, 0" : "=r" (val));
1576 break;
1577 case 3:
1578 asm volatile("mrc p15, 3, %0, c15, c2, 0" : "=r" (val));
1579 break;
1580 default:
1581 BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */
1582 }
1583
1584 return val;
1585 }
1586
1587 static void scorpion_write_pmresrn(int n, u32 val)
1588 {
1589 switch (n) {
1590 case 0:
1591 asm volatile("mcr p15, 0, %0, c15, c0, 0" : : "r" (val));
1592 break;
1593 case 1:
1594 asm volatile("mcr p15, 1, %0, c15, c0, 0" : : "r" (val));
1595 break;
1596 case 2:
1597 asm volatile("mcr p15, 2, %0, c15, c0, 0" : : "r" (val));
1598 break;
1599 case 3:
1600 asm volatile("mcr p15, 3, %0, c15, c2, 0" : : "r" (val));
1601 break;
1602 default:
1603 BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */
1604 }
1605 }
1606
1607 static u32 scorpion_get_pmresrn_event(unsigned int region)
1608 {
1609 static const u32 pmresrn_table[] = { SCORPION_LPM0_GROUP0,
1610 SCORPION_LPM1_GROUP0,
1611 SCORPION_LPM2_GROUP0,
1612 SCORPION_L2LPM_GROUP0 };
1613 return pmresrn_table[region];
1614 }
1615
1616 static void scorpion_evt_setup(int idx, u32 config_base)
1617 {
1618 u32 val;
1619 u32 mask;
1620 u32 vval, fval;
1621 unsigned int region = EVENT_REGION(config_base);
1622 unsigned int group = EVENT_GROUP(config_base);
1623 unsigned int code = EVENT_CODE(config_base);
1624 unsigned int group_shift;
1625 bool venum_event = EVENT_VENUM(config_base);
1626
1627 group_shift = group * 8;
1628 mask = 0xff << group_shift;
1629
1630 /* Configure evtsel for the region and group */
1631 if (venum_event)
1632 val = SCORPION_VLPM_GROUP0;
1633 else
1634 val = scorpion_get_pmresrn_event(region);
1635 val += group;
1636 /* Mix in mode-exclusion bits */
1637 val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1);
1638 armv7_pmnc_write_evtsel(idx, val);
1639
1640 asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1641
1642 if (venum_event) {
1643 venum_pre_pmresr(&vval, &fval);
1644 val = venum_read_pmresr();
1645 val &= ~mask;
1646 val |= code << group_shift;
1647 val |= PMRESRn_EN;
1648 venum_write_pmresr(val);
1649 venum_post_pmresr(vval, fval);
1650 } else {
1651 val = scorpion_read_pmresrn(region);
1652 val &= ~mask;
1653 val |= code << group_shift;
1654 val |= PMRESRn_EN;
1655 scorpion_write_pmresrn(region, val);
1656 }
1657 }
1658
1659 static void scorpion_clearpmu(u32 config_base)
1660 {
1661 u32 val;
1662 u32 vval, fval;
1663 unsigned int region = EVENT_REGION(config_base);
1664 unsigned int group = EVENT_GROUP(config_base);
1665 bool venum_event = EVENT_VENUM(config_base);
1666
1667 if (venum_event) {
1668 venum_pre_pmresr(&vval, &fval);
1669 val = venum_read_pmresr();
1670 val = clear_pmresrn_group(val, group);
1671 venum_write_pmresr(val);
1672 venum_post_pmresr(vval, fval);
1673 } else {
1674 val = scorpion_read_pmresrn(region);
1675 val = clear_pmresrn_group(val, group);
1676 scorpion_write_pmresrn(region, val);
1677 }
1678 }
1679
1680 static void scorpion_pmu_disable_event(struct perf_event *event)
1681 {
1682 unsigned long flags;
1683 struct hw_perf_event *hwc = &event->hw;
1684 int idx = hwc->idx;
1685 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1686 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1687
1688 /* Disable counter and interrupt */
1689 raw_spin_lock_irqsave(&events->pmu_lock, flags);
1690
1691 /* Disable counter */
1692 armv7_pmnc_disable_counter(idx);
1693
1694 /*
1695 * Clear pmresr code (if destined for PMNx counters)
1696 */
1697 if (hwc->config_base & KRAIT_EVENT_MASK)
1698 scorpion_clearpmu(hwc->config_base);
1699
1700 /* Disable interrupt for this counter */
1701 armv7_pmnc_disable_intens(idx);
1702
1703 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1704 }
1705
1706 static void scorpion_pmu_enable_event(struct perf_event *event)
1707 {
1708 unsigned long flags;
1709 struct hw_perf_event *hwc = &event->hw;
1710 int idx = hwc->idx;
1711 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1712 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1713
1714 /*
1715 * Enable counter and interrupt, and set the counter to count
1716 * the event that we're interested in.
1717 */
1718 raw_spin_lock_irqsave(&events->pmu_lock, flags);
1719
1720 /* Disable counter */
1721 armv7_pmnc_disable_counter(idx);
1722
1723 /*
1724 * Set event (if destined for PMNx counters)
1725 * We don't set the event for the cycle counter because we
1726 * don't have the ability to perform event filtering.
1727 */
1728 if (hwc->config_base & KRAIT_EVENT_MASK)
1729 scorpion_evt_setup(idx, hwc->config_base);
1730 else if (idx != ARMV7_IDX_CYCLE_COUNTER)
1731 armv7_pmnc_write_evtsel(idx, hwc->config_base);
1732
1733 /* Enable interrupt for this counter */
1734 armv7_pmnc_enable_intens(idx);
1735
1736 /* Enable counter */
1737 armv7_pmnc_enable_counter(idx);
1738
1739 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1740 }
1741
1742 static void scorpion_pmu_reset(void *info)
1743 {
1744 u32 vval, fval;
1745 struct arm_pmu *cpu_pmu = info;
1746 u32 idx, nb_cnt = cpu_pmu->num_events;
1747
1748 armv7pmu_reset(info);
1749
1750 /* Clear all pmresrs */
1751 scorpion_write_pmresrn(0, 0);
1752 scorpion_write_pmresrn(1, 0);
1753 scorpion_write_pmresrn(2, 0);
1754 scorpion_write_pmresrn(3, 0);
1755
1756 venum_pre_pmresr(&vval, &fval);
1757 venum_write_pmresr(0);
1758 venum_post_pmresr(vval, fval);
1759
1760 /* Reset PMxEVNCTCR to sane default */
1761 for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
1762 armv7_pmnc_select_counter(idx);
1763 asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1764 }
1765 }
1766
1767 static int scorpion_event_to_bit(struct perf_event *event, unsigned int region,
1768 unsigned int group)
1769 {
1770 int bit;
1771 struct hw_perf_event *hwc = &event->hw;
1772 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1773
1774 if (hwc->config_base & VENUM_EVENT)
1775 bit = SCORPION_VLPM_GROUP0;
1776 else
1777 bit = scorpion_get_pmresrn_event(region);
1778 bit -= scorpion_get_pmresrn_event(0);
1779 bit += group;
1780 /*
1781 * Lower bits are reserved for use by the counters (see
1782 * armv7pmu_get_event_idx() for more info)
1783 */
1784 bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1;
1785
1786 return bit;
1787 }
1788
1789 /*
1790 * We check for column exclusion constraints here.
1791 * Two events cant use the same group within a pmresr register.
1792 */
1793 static int scorpion_pmu_get_event_idx(struct pmu_hw_events *cpuc,
1794 struct perf_event *event)
1795 {
1796 int idx;
1797 int bit = -1;
1798 struct hw_perf_event *hwc = &event->hw;
1799 unsigned int region = EVENT_REGION(hwc->config_base);
1800 unsigned int group = EVENT_GROUP(hwc->config_base);
1801 bool venum_event = EVENT_VENUM(hwc->config_base);
1802 bool scorpion_event = EVENT_CPU(hwc->config_base);
1803
1804 if (venum_event || scorpion_event) {
1805 /* Ignore invalid events */
1806 if (group > 3 || region > 3)
1807 return -EINVAL;
1808
1809 bit = scorpion_event_to_bit(event, region, group);
1810 if (test_and_set_bit(bit, cpuc->used_mask))
1811 return -EAGAIN;
1812 }
1813
1814 idx = armv7pmu_get_event_idx(cpuc, event);
1815 if (idx < 0 && bit >= 0)
1816 clear_bit(bit, cpuc->used_mask);
1817
1818 return idx;
1819 }
1820
1821 static void scorpion_pmu_clear_event_idx(struct pmu_hw_events *cpuc,
1822 struct perf_event *event)
1823 {
1824 int bit;
1825 struct hw_perf_event *hwc = &event->hw;
1826 unsigned int region = EVENT_REGION(hwc->config_base);
1827 unsigned int group = EVENT_GROUP(hwc->config_base);
1828 bool venum_event = EVENT_VENUM(hwc->config_base);
1829 bool scorpion_event = EVENT_CPU(hwc->config_base);
1830
1831 if (venum_event || scorpion_event) {
1832 bit = scorpion_event_to_bit(event, region, group);
1833 clear_bit(bit, cpuc->used_mask);
1834 }
1835 }
1836
1837 static int scorpion_pmu_init(struct arm_pmu *cpu_pmu)
1838 {
1839 armv7pmu_init(cpu_pmu);
1840 cpu_pmu->name = "armv7_scorpion";
1841 cpu_pmu->map_event = scorpion_map_event;
1842 cpu_pmu->reset = scorpion_pmu_reset;
1843 cpu_pmu->enable = scorpion_pmu_enable_event;
1844 cpu_pmu->disable = scorpion_pmu_disable_event;
1845 cpu_pmu->get_event_idx = scorpion_pmu_get_event_idx;
1846 cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx;
1847 return armv7_probe_num_events(cpu_pmu);
1848 }
1849
1850 static int scorpion_mp_pmu_init(struct arm_pmu *cpu_pmu)
1851 {
1852 armv7pmu_init(cpu_pmu);
1853 cpu_pmu->name = "armv7_scorpion_mp";
1854 cpu_pmu->map_event = scorpion_map_event;
1855 cpu_pmu->reset = scorpion_pmu_reset;
1856 cpu_pmu->enable = scorpion_pmu_enable_event;
1857 cpu_pmu->disable = scorpion_pmu_disable_event;
1858 cpu_pmu->get_event_idx = scorpion_pmu_get_event_idx;
1859 cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx;
1860 return armv7_probe_num_events(cpu_pmu);
1861 }
1862
1863 static const struct of_device_id armv7_pmu_of_device_ids[] = {
1864 {.compatible = "arm,cortex-a17-pmu", .data = armv7_a17_pmu_init},
1865 {.compatible = "arm,cortex-a15-pmu", .data = armv7_a15_pmu_init},
1866 {.compatible = "arm,cortex-a12-pmu", .data = armv7_a12_pmu_init},
1867 {.compatible = "arm,cortex-a9-pmu", .data = armv7_a9_pmu_init},
1868 {.compatible = "arm,cortex-a8-pmu", .data = armv7_a8_pmu_init},
1869 {.compatible = "arm,cortex-a7-pmu", .data = armv7_a7_pmu_init},
1870 {.compatible = "arm,cortex-a5-pmu", .data = armv7_a5_pmu_init},
1871 {.compatible = "qcom,krait-pmu", .data = krait_pmu_init},
1872 {.compatible = "qcom,scorpion-pmu", .data = scorpion_pmu_init},
1873 {.compatible = "qcom,scorpion-mp-pmu", .data = scorpion_mp_pmu_init},
1874 {},
1875 };
1876
1877 static const struct pmu_probe_info armv7_pmu_probe_table[] = {
1878 ARM_PMU_PROBE(ARM_CPU_PART_CORTEX_A8, armv7_a8_pmu_init),
1879 ARM_PMU_PROBE(ARM_CPU_PART_CORTEX_A9, armv7_a9_pmu_init),
1880 { /* sentinel value */ }
1881 };
1882
1883
1884 static int armv7_pmu_device_probe(struct platform_device *pdev)
1885 {
1886 return arm_pmu_device_probe(pdev, armv7_pmu_of_device_ids,
1887 armv7_pmu_probe_table);
1888 }
1889
1890 static struct platform_driver armv7_pmu_driver = {
1891 .driver = {
1892 .name = "armv7-pmu",
1893 .of_match_table = armv7_pmu_of_device_ids,
1894 },
1895 .probe = armv7_pmu_device_probe,
1896 };
1897
1898 static int __init register_armv7_pmu_driver(void)
1899 {
1900 return platform_driver_register(&armv7_pmu_driver);
1901 }
1902 device_initcall(register_armv7_pmu_driver);
1903 #endif /* CONFIG_CPU_V7 */
CRIS linux kernel tree