3 * athlon / K7 / K8 / Family 10h model-specific MSR operations
5 * @remark Copyright 2002-2009 OProfile authors
6 * @remark Read the file COPYING
9 * @author Philippe Elie
10 * @author Graydon Hoare
11 * @author Robert Richter <robert.richter@amd.com>
12 * @author Barry Kasindorf <barry.kasindorf@amd.com>
13 * @author Jason Yeh <jason.yeh@amd.com>
14 * @author Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
17 #include <linux/oprofile.h>
18 #include <linux/device.h>
19 #include <linux/pci.h>
20 #include <linux/percpu.h>
22 #include <asm/ptrace.h>
26 #include <asm/processor.h>
27 #include <asm/cpufeature.h>
29 #include "op_x86_model.h"
30 #include "op_counter.h"
32 #define NUM_COUNTERS 4
33 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
34 #define NUM_VIRT_COUNTERS 32
36 #define NUM_VIRT_COUNTERS NUM_COUNTERS
39 #define OP_EVENT_MASK 0x0FFF
40 #define OP_CTR_OVERFLOW (1ULL<<31)
42 #define MSR_AMD_EVENTSEL_RESERVED ((0xFFFFFCF0ULL<<32)|(1ULL<<21))
44 static unsigned long reset_value
[NUM_VIRT_COUNTERS
];
46 #define IBS_FETCH_SIZE 6
47 #define IBS_OP_SIZE 12
52 unsigned long op_enabled
;
53 unsigned long fetch_enabled
;
54 unsigned long max_cnt_fetch
;
55 unsigned long max_cnt_op
;
56 unsigned long rand_en
;
57 unsigned long dispatched_ops
;
58 unsigned long branch_target
;
64 unsigned long sample_size
;
67 static struct ibs_config ibs_config
;
68 static struct ibs_state ibs_state
;
71 * IBS cpuid feature detection
74 #define IBS_CPUID_FEATURES 0x8000001b
77 * Same bit mask as for IBS cpuid feature flags (Fn8000_001B_EAX), but
78 * bit 0 is used to indicate the existence of IBS.
80 #define IBS_CAPS_AVAIL (1U<<0)
81 #define IBS_CAPS_FETCHSAM (1U<<1)
82 #define IBS_CAPS_OPSAM (1U<<2)
83 #define IBS_CAPS_RDWROPCNT (1U<<3)
84 #define IBS_CAPS_OPCNT (1U<<4)
85 #define IBS_CAPS_BRNTRGT (1U<<5)
86 #define IBS_CAPS_OPCNTEXT (1U<<6)
88 #define IBS_CAPS_DEFAULT (IBS_CAPS_AVAIL \
96 #define IBSCTL_LVT_OFFSET_VALID (1ULL<<8)
97 #define IBSCTL_LVT_OFFSET_MASK 0x0F
100 * IBS randomization macros
102 #define IBS_RANDOM_BITS 12
103 #define IBS_RANDOM_MASK ((1ULL << IBS_RANDOM_BITS) - 1)
104 #define IBS_RANDOM_MAXCNT_OFFSET (1ULL << (IBS_RANDOM_BITS - 5))
106 static u32
get_ibs_caps(void)
109 unsigned int max_level
;
111 if (!boot_cpu_has(X86_FEATURE_IBS
))
114 /* check IBS cpuid feature flags */
115 max_level
= cpuid_eax(0x80000000);
116 if (max_level
< IBS_CPUID_FEATURES
)
117 return IBS_CAPS_DEFAULT
;
119 ibs_caps
= cpuid_eax(IBS_CPUID_FEATURES
);
120 if (!(ibs_caps
& IBS_CAPS_AVAIL
))
121 /* cpuid flags not valid */
122 return IBS_CAPS_DEFAULT
;
128 * 16-bit Linear Feedback Shift Register (LFSR)
131 * Feedback polynomial = X + X + X + X + 1
133 static unsigned int lfsr_random(void)
135 static unsigned int lfsr_value
= 0xF00D;
138 /* Compute next bit to shift in */
139 bit
= ((lfsr_value
>> 0) ^
142 (lfsr_value
>> 5)) & 0x0001;
144 /* Advance to next register value */
145 lfsr_value
= (lfsr_value
>> 1) | (bit
<< 15);
151 * IBS software randomization
153 * The IBS periodic op counter is randomized in software. The lower 12
154 * bits of the 20 bit counter are randomized. IbsOpCurCnt is
155 * initialized with a 12 bit random value.
157 static inline u64
op_amd_randomize_ibs_op(u64 val
)
159 unsigned int random
= lfsr_random();
161 if (!(ibs_caps
& IBS_CAPS_RDWROPCNT
))
163 * Work around if the hw can not write to IbsOpCurCnt
165 * Randomize the lower 8 bits of the 16 bit
166 * IbsOpMaxCnt [15:0] value in the range of -128 to
167 * +127 by adding/subtracting an offset to the
168 * maximum count (IbsOpMaxCnt).
170 * To avoid over or underflows and protect upper bits
171 * starting at bit 16, the initial value for
172 * IbsOpMaxCnt must fit in the range from 0x0081 to
175 val
+= (s8
)(random
>> 4);
177 val
|= (u64
)(random
& IBS_RANDOM_MASK
) << 32;
183 op_amd_handle_ibs(struct pt_regs
* const regs
,
184 struct op_msrs
const * const msrs
)
187 struct op_entry entry
;
192 if (ibs_config
.fetch_enabled
) {
193 rdmsrl(MSR_AMD64_IBSFETCHCTL
, ctl
);
194 if (ctl
& IBS_FETCH_VAL
) {
195 rdmsrl(MSR_AMD64_IBSFETCHLINAD
, val
);
196 oprofile_write_reserve(&entry
, regs
, val
,
197 IBS_FETCH_CODE
, IBS_FETCH_SIZE
);
198 oprofile_add_data64(&entry
, val
);
199 oprofile_add_data64(&entry
, ctl
);
200 rdmsrl(MSR_AMD64_IBSFETCHPHYSAD
, val
);
201 oprofile_add_data64(&entry
, val
);
202 oprofile_write_commit(&entry
);
204 /* reenable the IRQ */
205 ctl
&= ~(IBS_FETCH_VAL
| IBS_FETCH_CNT
);
206 ctl
|= IBS_FETCH_ENABLE
;
207 wrmsrl(MSR_AMD64_IBSFETCHCTL
, ctl
);
211 if (ibs_config
.op_enabled
) {
212 rdmsrl(MSR_AMD64_IBSOPCTL
, ctl
);
213 if (ctl
& IBS_OP_VAL
) {
214 rdmsrl(MSR_AMD64_IBSOPRIP
, val
);
215 oprofile_write_reserve(&entry
, regs
, val
, IBS_OP_CODE
,
216 ibs_state
.sample_size
);
217 oprofile_add_data64(&entry
, val
);
218 rdmsrl(MSR_AMD64_IBSOPDATA
, val
);
219 oprofile_add_data64(&entry
, val
);
220 rdmsrl(MSR_AMD64_IBSOPDATA2
, val
);
221 oprofile_add_data64(&entry
, val
);
222 rdmsrl(MSR_AMD64_IBSOPDATA3
, val
);
223 oprofile_add_data64(&entry
, val
);
224 rdmsrl(MSR_AMD64_IBSDCLINAD
, val
);
225 oprofile_add_data64(&entry
, val
);
226 rdmsrl(MSR_AMD64_IBSDCPHYSAD
, val
);
227 oprofile_add_data64(&entry
, val
);
228 if (ibs_state
.branch_target
) {
229 rdmsrl(MSR_AMD64_IBSBRTARGET
, val
);
230 oprofile_add_data(&entry
, (unsigned long)val
);
232 oprofile_write_commit(&entry
);
234 /* reenable the IRQ */
235 ctl
= op_amd_randomize_ibs_op(ibs_state
.ibs_op_ctl
);
236 wrmsrl(MSR_AMD64_IBSOPCTL
, ctl
);
241 static inline void op_amd_start_ibs(void)
248 memset(&ibs_state
, 0, sizeof(ibs_state
));
251 * Note: Since the max count settings may out of range we
252 * write back the actual used values so that userland can read
256 if (ibs_config
.fetch_enabled
) {
257 val
= ibs_config
.max_cnt_fetch
>> 4;
258 val
= min(val
, IBS_FETCH_MAX_CNT
);
259 ibs_config
.max_cnt_fetch
= val
<< 4;
260 val
|= ibs_config
.rand_en
? IBS_FETCH_RAND_EN
: 0;
261 val
|= IBS_FETCH_ENABLE
;
262 wrmsrl(MSR_AMD64_IBSFETCHCTL
, val
);
265 if (ibs_config
.op_enabled
) {
266 val
= ibs_config
.max_cnt_op
>> 4;
267 if (!(ibs_caps
& IBS_CAPS_RDWROPCNT
)) {
269 * IbsOpCurCnt not supported. See
270 * op_amd_randomize_ibs_op() for details.
272 val
= clamp(val
, 0x0081ULL
, 0xFF80ULL
);
273 ibs_config
.max_cnt_op
= val
<< 4;
276 * The start value is randomized with a
277 * positive offset, we need to compensate it
278 * with the half of the randomized range. Also
281 val
+= IBS_RANDOM_MAXCNT_OFFSET
;
282 if (ibs_caps
& IBS_CAPS_OPCNTEXT
)
283 val
= min(val
, IBS_OP_MAX_CNT_EXT
);
285 val
= min(val
, IBS_OP_MAX_CNT
);
286 ibs_config
.max_cnt_op
=
287 (val
- IBS_RANDOM_MAXCNT_OFFSET
) << 4;
289 val
= ((val
& ~IBS_OP_MAX_CNT
) << 4) | (val
& IBS_OP_MAX_CNT
);
290 val
|= ibs_config
.dispatched_ops
? IBS_OP_CNT_CTL
: 0;
291 val
|= IBS_OP_ENABLE
;
292 ibs_state
.ibs_op_ctl
= val
;
293 ibs_state
.sample_size
= IBS_OP_SIZE
;
294 if (ibs_config
.branch_target
) {
295 ibs_state
.branch_target
= 1;
296 ibs_state
.sample_size
++;
298 val
= op_amd_randomize_ibs_op(ibs_state
.ibs_op_ctl
);
299 wrmsrl(MSR_AMD64_IBSOPCTL
, val
);
303 static void op_amd_stop_ibs(void)
308 if (ibs_config
.fetch_enabled
)
309 /* clear max count and enable */
310 wrmsrl(MSR_AMD64_IBSFETCHCTL
, 0);
312 if (ibs_config
.op_enabled
)
313 /* clear max count and enable */
314 wrmsrl(MSR_AMD64_IBSOPCTL
, 0);
317 static inline int eilvt_is_available(int offset
)
319 /* check if we may assign a vector */
320 return !setup_APIC_eilvt(offset
, 0, APIC_EILVT_MSG_NMI
, 1);
323 static inline int ibs_eilvt_valid(void)
328 rdmsrl(MSR_AMD64_IBSCTL
, val
);
329 offset
= val
& IBSCTL_LVT_OFFSET_MASK
;
331 if (!(val
& IBSCTL_LVT_OFFSET_VALID
)) {
332 pr_err(FW_BUG
"cpu %d, invalid IBS interrupt offset %d (MSR%08X=0x%016llx)\n",
333 smp_processor_id(), offset
, MSR_AMD64_IBSCTL
, val
);
337 if (!eilvt_is_available(offset
)) {
338 pr_err(FW_BUG
"cpu %d, IBS interrupt offset %d not available (MSR%08X=0x%016llx)\n",
339 smp_processor_id(), offset
, MSR_AMD64_IBSCTL
, val
);
346 static inline int get_ibs_offset(void)
350 rdmsrl(MSR_AMD64_IBSCTL
, val
);
351 if (!(val
& IBSCTL_LVT_OFFSET_VALID
))
354 return val
& IBSCTL_LVT_OFFSET_MASK
;
357 static void setup_APIC_ibs(void)
361 offset
= get_ibs_offset();
365 if (!setup_APIC_eilvt(offset
, 0, APIC_EILVT_MSG_NMI
, 0))
368 pr_warn("oprofile: IBS APIC setup failed on cpu #%d\n",
372 static void clear_APIC_ibs(void)
376 offset
= get_ibs_offset();
378 setup_APIC_eilvt(offset
, 0, APIC_EILVT_MSG_FIX
, 1);
381 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
383 static void op_mux_switch_ctrl(struct op_x86_model_spec
const *model
,
384 struct op_msrs
const * const msrs
)
389 /* enable active counters */
390 for (i
= 0; i
< NUM_COUNTERS
; ++i
) {
391 int virt
= op_x86_phys_to_virt(i
);
392 if (!reset_value
[virt
])
394 rdmsrl(msrs
->controls
[i
].addr
, val
);
395 val
&= model
->reserved
;
396 val
|= op_x86_get_ctrl(model
, &counter_config
[virt
]);
397 wrmsrl(msrs
->controls
[i
].addr
, val
);
403 /* functions for op_amd_spec */
405 static void op_amd_shutdown(struct op_msrs
const * const msrs
)
409 for (i
= 0; i
< NUM_COUNTERS
; ++i
) {
410 if (!msrs
->counters
[i
].addr
)
412 release_perfctr_nmi(MSR_K7_PERFCTR0
+ i
);
413 release_evntsel_nmi(MSR_K7_EVNTSEL0
+ i
);
417 static int op_amd_fill_in_addresses(struct op_msrs
* const msrs
)
421 for (i
= 0; i
< NUM_COUNTERS
; i
++) {
422 if (!reserve_perfctr_nmi(MSR_K7_PERFCTR0
+ i
))
424 if (!reserve_evntsel_nmi(MSR_K7_EVNTSEL0
+ i
)) {
425 release_perfctr_nmi(MSR_K7_PERFCTR0
+ i
);
428 /* both registers must be reserved */
429 msrs
->counters
[i
].addr
= MSR_K7_PERFCTR0
+ i
;
430 msrs
->controls
[i
].addr
= MSR_K7_EVNTSEL0
+ i
;
433 if (!counter_config
[i
].enabled
)
435 op_x86_warn_reserved(i
);
436 op_amd_shutdown(msrs
);
443 static void op_amd_setup_ctrs(struct op_x86_model_spec
const *model
,
444 struct op_msrs
const * const msrs
)
449 /* setup reset_value */
450 for (i
= 0; i
< NUM_VIRT_COUNTERS
; ++i
) {
451 if (counter_config
[i
].enabled
452 && msrs
->counters
[op_x86_virt_to_phys(i
)].addr
)
453 reset_value
[i
] = counter_config
[i
].count
;
458 /* clear all counters */
459 for (i
= 0; i
< NUM_COUNTERS
; ++i
) {
460 if (!msrs
->controls
[i
].addr
)
462 rdmsrl(msrs
->controls
[i
].addr
, val
);
463 if (val
& ARCH_PERFMON_EVENTSEL_ENABLE
)
464 op_x86_warn_in_use(i
);
465 val
&= model
->reserved
;
466 wrmsrl(msrs
->controls
[i
].addr
, val
);
468 * avoid a false detection of ctr overflows in NMI
471 wrmsrl(msrs
->counters
[i
].addr
, -1LL);
474 /* enable active counters */
475 for (i
= 0; i
< NUM_COUNTERS
; ++i
) {
476 int virt
= op_x86_phys_to_virt(i
);
477 if (!reset_value
[virt
])
480 /* setup counter registers */
481 wrmsrl(msrs
->counters
[i
].addr
, -(u64
)reset_value
[virt
]);
483 /* setup control registers */
484 rdmsrl(msrs
->controls
[i
].addr
, val
);
485 val
&= model
->reserved
;
486 val
|= op_x86_get_ctrl(model
, &counter_config
[virt
]);
487 wrmsrl(msrs
->controls
[i
].addr
, val
);
494 static void op_amd_cpu_shutdown(void)
500 static int op_amd_check_ctrs(struct pt_regs
* const regs
,
501 struct op_msrs
const * const msrs
)
506 for (i
= 0; i
< NUM_COUNTERS
; ++i
) {
507 int virt
= op_x86_phys_to_virt(i
);
508 if (!reset_value
[virt
])
510 rdmsrl(msrs
->counters
[i
].addr
, val
);
511 /* bit is clear if overflowed: */
512 if (val
& OP_CTR_OVERFLOW
)
514 oprofile_add_sample(regs
, virt
);
515 wrmsrl(msrs
->counters
[i
].addr
, -(u64
)reset_value
[virt
]);
518 op_amd_handle_ibs(regs
, msrs
);
520 /* See op_model_ppro.c */
524 static void op_amd_start(struct op_msrs
const * const msrs
)
529 for (i
= 0; i
< NUM_COUNTERS
; ++i
) {
530 if (!reset_value
[op_x86_phys_to_virt(i
)])
532 rdmsrl(msrs
->controls
[i
].addr
, val
);
533 val
|= ARCH_PERFMON_EVENTSEL_ENABLE
;
534 wrmsrl(msrs
->controls
[i
].addr
, val
);
540 static void op_amd_stop(struct op_msrs
const * const msrs
)
546 * Subtle: stop on all counters to avoid race with setting our
549 for (i
= 0; i
< NUM_COUNTERS
; ++i
) {
550 if (!reset_value
[op_x86_phys_to_virt(i
)])
552 rdmsrl(msrs
->controls
[i
].addr
, val
);
553 val
&= ~ARCH_PERFMON_EVENTSEL_ENABLE
;
554 wrmsrl(msrs
->controls
[i
].addr
, val
);
560 static int setup_ibs_ctl(int ibs_eilvt_off
)
562 struct pci_dev
*cpu_cfg
;
569 cpu_cfg
= pci_get_device(PCI_VENDOR_ID_AMD
,
570 PCI_DEVICE_ID_AMD_10H_NB_MISC
,
575 pci_write_config_dword(cpu_cfg
, IBSCTL
, ibs_eilvt_off
576 | IBSCTL_LVT_OFFSET_VALID
);
577 pci_read_config_dword(cpu_cfg
, IBSCTL
, &value
);
578 if (value
!= (ibs_eilvt_off
| IBSCTL_LVT_OFFSET_VALID
)) {
579 pci_dev_put(cpu_cfg
);
580 printk(KERN_DEBUG
"Failed to setup IBS LVT offset, "
581 "IBSCTL = 0x%08x\n", value
);
587 printk(KERN_DEBUG
"No CPU node configured for IBS\n");
594 static int force_ibs_eilvt_setup(void)
599 /* find the next free available EILVT entry */
600 for (i
= 1; i
< 4; i
++) {
601 if (!eilvt_is_available(i
))
603 ret
= setup_ibs_ctl(i
);
609 printk(KERN_DEBUG
"No EILVT entry available\n");
614 static int __init_ibs_nmi(void)
618 if (ibs_eilvt_valid())
621 ret
= force_ibs_eilvt_setup();
625 if (!ibs_eilvt_valid())
628 pr_err(FW_BUG
"workaround enabled for IBS LVT offset\n");
633 /* initialize the APIC for the IBS interrupts if available */
634 static void init_ibs(void)
636 ibs_caps
= get_ibs_caps();
641 if (__init_ibs_nmi()) {
646 printk(KERN_INFO
"oprofile: AMD IBS detected (0x%08x)\n",
650 static int (*create_arch_files
)(struct super_block
*sb
, struct dentry
*root
);
652 static int setup_ibs_files(struct super_block
*sb
, struct dentry
*root
)
657 /* architecture specific files */
658 if (create_arch_files
)
659 ret
= create_arch_files(sb
, root
);
667 /* model specific files */
669 /* setup some reasonable defaults */
670 memset(&ibs_config
, 0, sizeof(ibs_config
));
671 ibs_config
.max_cnt_fetch
= 250000;
672 ibs_config
.max_cnt_op
= 250000;
674 if (ibs_caps
& IBS_CAPS_FETCHSAM
) {
675 dir
= oprofilefs_mkdir(sb
, root
, "ibs_fetch");
676 oprofilefs_create_ulong(sb
, dir
, "enable",
677 &ibs_config
.fetch_enabled
);
678 oprofilefs_create_ulong(sb
, dir
, "max_count",
679 &ibs_config
.max_cnt_fetch
);
680 oprofilefs_create_ulong(sb
, dir
, "rand_enable",
681 &ibs_config
.rand_en
);
684 if (ibs_caps
& IBS_CAPS_OPSAM
) {
685 dir
= oprofilefs_mkdir(sb
, root
, "ibs_op");
686 oprofilefs_create_ulong(sb
, dir
, "enable",
687 &ibs_config
.op_enabled
);
688 oprofilefs_create_ulong(sb
, dir
, "max_count",
689 &ibs_config
.max_cnt_op
);
690 if (ibs_caps
& IBS_CAPS_OPCNT
)
691 oprofilefs_create_ulong(sb
, dir
, "dispatched_ops",
692 &ibs_config
.dispatched_ops
);
693 if (ibs_caps
& IBS_CAPS_BRNTRGT
)
694 oprofilefs_create_ulong(sb
, dir
, "branch_target",
695 &ibs_config
.branch_target
);
701 static int op_amd_init(struct oprofile_operations
*ops
)
704 create_arch_files
= ops
->create_files
;
705 ops
->create_files
= setup_ibs_files
;
709 struct op_x86_model_spec op_amd_spec
= {
710 .num_counters
= NUM_COUNTERS
,
711 .num_controls
= NUM_COUNTERS
,
712 .num_virt_counters
= NUM_VIRT_COUNTERS
,
713 .reserved
= MSR_AMD_EVENTSEL_RESERVED
,
714 .event_mask
= OP_EVENT_MASK
,
716 .fill_in_addresses
= &op_amd_fill_in_addresses
,
717 .setup_ctrs
= &op_amd_setup_ctrs
,
718 .cpu_down
= &op_amd_cpu_shutdown
,
719 .check_ctrs
= &op_amd_check_ctrs
,
720 .start
= &op_amd_start
,
721 .stop
= &op_amd_stop
,
722 .shutdown
= &op_amd_shutdown
,
723 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
724 .switch_ctrl
= &op_mux_switch_ctrl
,