?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
init from v2.6.32.60
[mach-moxart.git] / arch / x86 / oprofile / op_model_amd.c
blob1ed963d2e9b6730c99517cbc1a44a0eaf3e47e8c
1 /*
2 * @file op_model_amd.c
3 * athlon / K7 / K8 / Family 10h model-specific MSR operations
4 *
5 * @remark Copyright 2002-2009 OProfile authors
6 * @remark Read the file COPYING
7 *
8 * @author John Levon
9 * @author Philippe Elie
10 * @author Graydon Hoare
11 * @author Robert Richter <robert.richter@amd.com>
12 * @author Barry Kasindorf <barry.kasindorf@amd.com>
13 * @author Jason Yeh <jason.yeh@amd.com>
14 * @author Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
15 */
16
17 #include <linux/oprofile.h>
18 #include <linux/device.h>
19 #include <linux/pci.h>
20 #include <linux/percpu.h>
21
22 #include <asm/ptrace.h>
23 #include <asm/msr.h>
24 #include <asm/nmi.h>
25
26 #include "op_x86_model.h"
27 #include "op_counter.h"
28
29 #define NUM_COUNTERS 4
30 #define NUM_CONTROLS 4
31 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
32 #define NUM_VIRT_COUNTERS 32
33 #define NUM_VIRT_CONTROLS 32
34 #else
35 #define NUM_VIRT_COUNTERS NUM_COUNTERS
36 #define NUM_VIRT_CONTROLS NUM_CONTROLS
37 #endif
38
39 #define OP_EVENT_MASK 0x0FFF
40 #define OP_CTR_OVERFLOW (1ULL<<31)
41
42 #define MSR_AMD_EVENTSEL_RESERVED ((0xFFFFFCF0ULL<<32)|(1ULL<<21))
43
44 static unsigned long reset_value[NUM_VIRT_COUNTERS];
45
46 #ifdef CONFIG_OPROFILE_IBS
47
48 /* IbsFetchCtl bits/masks */
49 #define IBS_FETCH_RAND_EN (1ULL<<57)
50 #define IBS_FETCH_VAL (1ULL<<49)
51 #define IBS_FETCH_ENABLE (1ULL<<48)
52 #define IBS_FETCH_CNT_MASK 0xFFFF0000ULL
53
54 /*IbsOpCtl bits */
55 #define IBS_OP_CNT_CTL (1ULL<<19)
56 #define IBS_OP_VAL (1ULL<<18)
57 #define IBS_OP_ENABLE (1ULL<<17)
58
59 #define IBS_FETCH_SIZE 6
60 #define IBS_OP_SIZE 12
61
62 static int has_ibs; /* AMD Family10h and later */
63
64 struct op_ibs_config {
65 unsigned long op_enabled;
66 unsigned long fetch_enabled;
67 unsigned long max_cnt_fetch;
68 unsigned long max_cnt_op;
69 unsigned long rand_en;
70 unsigned long dispatched_ops;
71 };
72
73 static struct op_ibs_config ibs_config;
74
75 #endif
76
77 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
78
79 static void op_mux_switch_ctrl(struct op_x86_model_spec const *model,
80 struct op_msrs const * const msrs)
81 {
82 u64 val;
83 int i;
84
85 /* enable active counters */
86 for (i = 0; i < NUM_COUNTERS; ++i) {
87 int virt = op_x86_phys_to_virt(i);
88 if (!reset_value[virt])
89 continue;
90 rdmsrl(msrs->controls[i].addr, val);
91 val &= model->reserved;
92 val |= op_x86_get_ctrl(model, &counter_config[virt]);
93 wrmsrl(msrs->controls[i].addr, val);
94 }
95 }
96
97 #endif
98
99 /* functions for op_amd_spec */
100
101 static void op_amd_fill_in_addresses(struct op_msrs * const msrs)
102 {
103 int i;
104
105 for (i = 0; i < NUM_COUNTERS; i++) {
106 if (reserve_perfctr_nmi(MSR_K7_PERFCTR0 + i))
107 msrs->counters[i].addr = MSR_K7_PERFCTR0 + i;
108 }
109
110 for (i = 0; i < NUM_CONTROLS; i++) {
111 if (reserve_evntsel_nmi(MSR_K7_EVNTSEL0 + i))
112 msrs->controls[i].addr = MSR_K7_EVNTSEL0 + i;
113 }
114 }
115
116 static void op_amd_setup_ctrs(struct op_x86_model_spec const *model,
117 struct op_msrs const * const msrs)
118 {
119 u64 val;
120 int i;
121
122 /* setup reset_value */
123 for (i = 0; i < NUM_VIRT_COUNTERS; ++i) {
124 if (counter_config[i].enabled
125 && msrs->counters[op_x86_virt_to_phys(i)].addr)
126 reset_value[i] = counter_config[i].count;
127 else
128 reset_value[i] = 0;
129 }
130
131 /* clear all counters */
132 for (i = 0; i < NUM_CONTROLS; ++i) {
133 if (unlikely(!msrs->controls[i].addr))
134 continue;
135 rdmsrl(msrs->controls[i].addr, val);
136 val &= model->reserved;
137 wrmsrl(msrs->controls[i].addr, val);
138 }
139
140 /* avoid a false detection of ctr overflows in NMI handler */
141 for (i = 0; i < NUM_COUNTERS; ++i) {
142 if (unlikely(!msrs->counters[i].addr))
143 continue;
144 wrmsrl(msrs->counters[i].addr, -1LL);
145 }
146
147 /* enable active counters */
148 for (i = 0; i < NUM_COUNTERS; ++i) {
149 int virt = op_x86_phys_to_virt(i);
150 if (!reset_value[virt])
151 continue;
152
153 /* setup counter registers */
154 wrmsrl(msrs->counters[i].addr, -(u64)reset_value[virt]);
155
156 /* setup control registers */
157 rdmsrl(msrs->controls[i].addr, val);
158 val &= model->reserved;
159 val |= op_x86_get_ctrl(model, &counter_config[virt]);
160 wrmsrl(msrs->controls[i].addr, val);
161 }
162 }
163
164 #ifdef CONFIG_OPROFILE_IBS
165
166 static inline void
167 op_amd_handle_ibs(struct pt_regs * const regs,
168 struct op_msrs const * const msrs)
169 {
170 u64 val, ctl;
171 struct op_entry entry;
172
173 if (!has_ibs)
174 return;
175
176 if (ibs_config.fetch_enabled) {
177 rdmsrl(MSR_AMD64_IBSFETCHCTL, ctl);
178 if (ctl & IBS_FETCH_VAL) {
179 rdmsrl(MSR_AMD64_IBSFETCHLINAD, val);
180 oprofile_write_reserve(&entry, regs, val,
181 IBS_FETCH_CODE, IBS_FETCH_SIZE);
182 oprofile_add_data64(&entry, val);
183 oprofile_add_data64(&entry, ctl);
184 rdmsrl(MSR_AMD64_IBSFETCHPHYSAD, val);
185 oprofile_add_data64(&entry, val);
186 oprofile_write_commit(&entry);
187
188 /* reenable the IRQ */
189 ctl &= ~(IBS_FETCH_VAL | IBS_FETCH_CNT_MASK);
190 ctl |= IBS_FETCH_ENABLE;
191 wrmsrl(MSR_AMD64_IBSFETCHCTL, ctl);
192 }
193 }
194
195 if (ibs_config.op_enabled) {
196 rdmsrl(MSR_AMD64_IBSOPCTL, ctl);
197 if (ctl & IBS_OP_VAL) {
198 rdmsrl(MSR_AMD64_IBSOPRIP, val);
199 oprofile_write_reserve(&entry, regs, val,
200 IBS_OP_CODE, IBS_OP_SIZE);
201 oprofile_add_data64(&entry, val);
202 rdmsrl(MSR_AMD64_IBSOPDATA, val);
203 oprofile_add_data64(&entry, val);
204 rdmsrl(MSR_AMD64_IBSOPDATA2, val);
205 oprofile_add_data64(&entry, val);
206 rdmsrl(MSR_AMD64_IBSOPDATA3, val);
207 oprofile_add_data64(&entry, val);
208 rdmsrl(MSR_AMD64_IBSDCLINAD, val);
209 oprofile_add_data64(&entry, val);
210 rdmsrl(MSR_AMD64_IBSDCPHYSAD, val);
211 oprofile_add_data64(&entry, val);
212 oprofile_write_commit(&entry);
213
214 /* reenable the IRQ */
215 ctl &= ~IBS_OP_VAL & 0xFFFFFFFF;
216 ctl |= IBS_OP_ENABLE;
217 wrmsrl(MSR_AMD64_IBSOPCTL, ctl);
218 }
219 }
220 }
221
222 static inline void op_amd_start_ibs(void)
223 {
224 u64 val;
225 if (has_ibs && ibs_config.fetch_enabled) {
226 val = (ibs_config.max_cnt_fetch >> 4) & 0xFFFF;
227 val |= ibs_config.rand_en ? IBS_FETCH_RAND_EN : 0;
228 val |= IBS_FETCH_ENABLE;
229 wrmsrl(MSR_AMD64_IBSFETCHCTL, val);
230 }
231
232 if (has_ibs && ibs_config.op_enabled) {
233 val = (ibs_config.max_cnt_op >> 4) & 0xFFFF;
234 val |= ibs_config.dispatched_ops ? IBS_OP_CNT_CTL : 0;
235 val |= IBS_OP_ENABLE;
236 wrmsrl(MSR_AMD64_IBSOPCTL, val);
237 }
238 }
239
240 static void op_amd_stop_ibs(void)
241 {
242 if (has_ibs && ibs_config.fetch_enabled)
243 /* clear max count and enable */
244 wrmsrl(MSR_AMD64_IBSFETCHCTL, 0);
245
246 if (has_ibs && ibs_config.op_enabled)
247 /* clear max count and enable */
248 wrmsrl(MSR_AMD64_IBSOPCTL, 0);
249 }
250
251 #else
252
253 static inline void op_amd_handle_ibs(struct pt_regs * const regs,
254 struct op_msrs const * const msrs) { }
255 static inline void op_amd_start_ibs(void) { }
256 static inline void op_amd_stop_ibs(void) { }
257
258 #endif
259
260 static int op_amd_check_ctrs(struct pt_regs * const regs,
261 struct op_msrs const * const msrs)
262 {
263 u64 val;
264 int i;
265
266 for (i = 0; i < NUM_COUNTERS; ++i) {
267 int virt = op_x86_phys_to_virt(i);
268 if (!reset_value[virt])
269 continue;
270 rdmsrl(msrs->counters[i].addr, val);
271 /* bit is clear if overflowed: */
272 if (val & OP_CTR_OVERFLOW)
273 continue;
274 oprofile_add_sample(regs, virt);
275 wrmsrl(msrs->counters[i].addr, -(u64)reset_value[virt]);
276 }
277
278 op_amd_handle_ibs(regs, msrs);
279
280 /* See op_model_ppro.c */
281 return 1;
282 }
283
284 static void op_amd_start(struct op_msrs const * const msrs)
285 {
286 u64 val;
287 int i;
288
289 for (i = 0; i < NUM_COUNTERS; ++i) {
290 if (!reset_value[op_x86_phys_to_virt(i)])
291 continue;
292 rdmsrl(msrs->controls[i].addr, val);
293 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
294 wrmsrl(msrs->controls[i].addr, val);
295 }
296
297 op_amd_start_ibs();
298 }
299
300 static void op_amd_stop(struct op_msrs const * const msrs)
301 {
302 u64 val;
303 int i;
304
305 /*
306 * Subtle: stop on all counters to avoid race with setting our
307 * pm callback
308 */
309 for (i = 0; i < NUM_COUNTERS; ++i) {
310 if (!reset_value[op_x86_phys_to_virt(i)])
311 continue;
312 rdmsrl(msrs->controls[i].addr, val);
313 val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
314 wrmsrl(msrs->controls[i].addr, val);
315 }
316
317 op_amd_stop_ibs();
318 }
319
320 static void op_amd_shutdown(struct op_msrs const * const msrs)
321 {
322 int i;
323
324 for (i = 0; i < NUM_COUNTERS; ++i) {
325 if (msrs->counters[i].addr)
326 release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
327 }
328 for (i = 0; i < NUM_CONTROLS; ++i) {
329 if (msrs->controls[i].addr)
330 release_evntsel_nmi(MSR_K7_EVNTSEL0 + i);
331 }
332 }
333
334 #ifdef CONFIG_OPROFILE_IBS
335
336 static u8 ibs_eilvt_off;
337
338 static inline void apic_init_ibs_nmi_per_cpu(void *arg)
339 {
340 ibs_eilvt_off = setup_APIC_eilvt_ibs(0, APIC_EILVT_MSG_NMI, 0);
341 }
342
343 static inline void apic_clear_ibs_nmi_per_cpu(void *arg)
344 {
345 setup_APIC_eilvt_ibs(0, APIC_EILVT_MSG_FIX, 1);
346 }
347
348 static int init_ibs_nmi(void)
349 {
350 #define IBSCTL_LVTOFFSETVAL (1 << 8)
351 #define IBSCTL 0x1cc
352 struct pci_dev *cpu_cfg;
353 int nodes;
354 u32 value = 0;
355
356 /* per CPU setup */
357 on_each_cpu(apic_init_ibs_nmi_per_cpu, NULL, 1);
358
359 nodes = 0;
360 cpu_cfg = NULL;
361 do {
362 cpu_cfg = pci_get_device(PCI_VENDOR_ID_AMD,
363 PCI_DEVICE_ID_AMD_10H_NB_MISC,
364 cpu_cfg);
365 if (!cpu_cfg)
366 break;
367 ++nodes;
368 pci_write_config_dword(cpu_cfg, IBSCTL, ibs_eilvt_off
369 | IBSCTL_LVTOFFSETVAL);
370 pci_read_config_dword(cpu_cfg, IBSCTL, &value);
371 if (value != (ibs_eilvt_off | IBSCTL_LVTOFFSETVAL)) {
372 pci_dev_put(cpu_cfg);
373 printk(KERN_DEBUG "Failed to setup IBS LVT offset, "
374 "IBSCTL = 0x%08x", value);
375 return 1;
376 }
377 } while (1);
378
379 if (!nodes) {
380 printk(KERN_DEBUG "No CPU node configured for IBS");
381 return 1;
382 }
383
384 return 0;
385 }
386
387 /* uninitialize the APIC for the IBS interrupts if needed */
388 static void clear_ibs_nmi(void)
389 {
390 if (has_ibs)
391 on_each_cpu(apic_clear_ibs_nmi_per_cpu, NULL, 1);
392 }
393
394 /* initialize the APIC for the IBS interrupts if available */
395 static void ibs_init(void)
396 {
397 has_ibs = boot_cpu_has(X86_FEATURE_IBS);
398
399 if (!has_ibs)
400 return;
401
402 if (init_ibs_nmi()) {
403 has_ibs = 0;
404 return;
405 }
406
407 printk(KERN_INFO "oprofile: AMD IBS detected\n");
408 }
409
410 static void ibs_exit(void)
411 {
412 if (!has_ibs)
413 return;
414
415 clear_ibs_nmi();
416 }
417
418 static int (*create_arch_files)(struct super_block *sb, struct dentry *root);
419
420 static int setup_ibs_files(struct super_block *sb, struct dentry *root)
421 {
422 struct dentry *dir;
423 int ret = 0;
424
425 /* architecture specific files */
426 if (create_arch_files)
427 ret = create_arch_files(sb, root);
428
429 if (ret)
430 return ret;
431
432 if (!has_ibs)
433 return ret;
434
435 /* model specific files */
436
437 /* setup some reasonable defaults */
438 ibs_config.max_cnt_fetch = 250000;
439 ibs_config.fetch_enabled = 0;
440 ibs_config.max_cnt_op = 250000;
441 ibs_config.op_enabled = 0;
442 ibs_config.dispatched_ops = 1;
443
444 dir = oprofilefs_mkdir(sb, root, "ibs_fetch");
445 oprofilefs_create_ulong(sb, dir, "enable",
446 &ibs_config.fetch_enabled);
447 oprofilefs_create_ulong(sb, dir, "max_count",
448 &ibs_config.max_cnt_fetch);
449 oprofilefs_create_ulong(sb, dir, "rand_enable",
450 &ibs_config.rand_en);
451
452 dir = oprofilefs_mkdir(sb, root, "ibs_op");
453 oprofilefs_create_ulong(sb, dir, "enable",
454 &ibs_config.op_enabled);
455 oprofilefs_create_ulong(sb, dir, "max_count",
456 &ibs_config.max_cnt_op);
457 oprofilefs_create_ulong(sb, dir, "dispatched_ops",
458 &ibs_config.dispatched_ops);
459
460 return 0;
461 }
462
463 static int op_amd_init(struct oprofile_operations *ops)
464 {
465 ibs_init();
466 create_arch_files = ops->create_files;
467 ops->create_files = setup_ibs_files;
468 return 0;
469 }
470
471 static void op_amd_exit(void)
472 {
473 ibs_exit();
474 }
475
476 #else
477
478 /* no IBS support */
479
480 static int op_amd_init(struct oprofile_operations *ops)
481 {
482 return 0;
483 }
484
485 static void op_amd_exit(void) {}
486
487 #endif /* CONFIG_OPROFILE_IBS */
488
489 struct op_x86_model_spec op_amd_spec = {
490 .num_counters = NUM_COUNTERS,
491 .num_controls = NUM_CONTROLS,
492 .num_virt_counters = NUM_VIRT_COUNTERS,
493 .reserved = MSR_AMD_EVENTSEL_RESERVED,
494 .event_mask = OP_EVENT_MASK,
495 .init = op_amd_init,
496 .exit = op_amd_exit,
497 .fill_in_addresses = &op_amd_fill_in_addresses,
498 .setup_ctrs = &op_amd_setup_ctrs,
499 .check_ctrs = &op_amd_check_ctrs,
500 .start = &op_amd_start,
501 .stop = &op_amd_stop,
502 .shutdown = &op_amd_shutdown,
503 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
504 .switch_ctrl = &op_mux_switch_ctrl,
505 #endif
506 };
Linux MOXA ART SoC ARM port