?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
perf/x86/intel: Fix event update for auto-reload
[linux/fpc-iii.git] / arch / x86 / events / core.c
blob5a3ccd1715e2a14933427f6aafe803edc2c270fb
1 /*
2 * Performance events x86 architecture code
3 *
4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2009 Jaswinder Singh Rajput
7 * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
8 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra
9 * Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
10 * Copyright (C) 2009 Google, Inc., Stephane Eranian
11 *
12 * For licencing details see kernel-base/COPYING
13 */
14
15 #include <linux/perf_event.h>
16 #include <linux/capability.h>
17 #include <linux/notifier.h>
18 #include <linux/hardirq.h>
19 #include <linux/kprobes.h>
20 #include <linux/export.h>
21 #include <linux/init.h>
22 #include <linux/kdebug.h>
23 #include <linux/sched/mm.h>
24 #include <linux/sched/clock.h>
25 #include <linux/uaccess.h>
26 #include <linux/slab.h>
27 #include <linux/cpu.h>
28 #include <linux/bitops.h>
29 #include <linux/device.h>
30
31 #include <asm/apic.h>
32 #include <asm/stacktrace.h>
33 #include <asm/nmi.h>
34 #include <asm/smp.h>
35 #include <asm/alternative.h>
36 #include <asm/mmu_context.h>
37 #include <asm/tlbflush.h>
38 #include <asm/timer.h>
39 #include <asm/desc.h>
40 #include <asm/ldt.h>
41 #include <asm/unwind.h>
42
43 #include "perf_event.h"
44
45 struct x86_pmu x86_pmu __read_mostly;
46
47 DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
48 .enabled = 1,
49 };
50
51 struct static_key rdpmc_always_available = STATIC_KEY_INIT_FALSE;
52
53 u64 __read_mostly hw_cache_event_ids
54 [PERF_COUNT_HW_CACHE_MAX]
55 [PERF_COUNT_HW_CACHE_OP_MAX]
56 [PERF_COUNT_HW_CACHE_RESULT_MAX];
57 u64 __read_mostly hw_cache_extra_regs
58 [PERF_COUNT_HW_CACHE_MAX]
59 [PERF_COUNT_HW_CACHE_OP_MAX]
60 [PERF_COUNT_HW_CACHE_RESULT_MAX];
61
62 /*
63 * Propagate event elapsed time into the generic event.
64 * Can only be executed on the CPU where the event is active.
65 * Returns the delta events processed.
66 */
67 u64 x86_perf_event_update(struct perf_event *event)
68 {
69 struct hw_perf_event *hwc = &event->hw;
70 int shift = 64 - x86_pmu.cntval_bits;
71 u64 prev_raw_count, new_raw_count;
72 int idx = hwc->idx;
73 u64 delta;
74
75 if (idx == INTEL_PMC_IDX_FIXED_BTS)
76 return 0;
77
78 /*
79 * Careful: an NMI might modify the previous event value.
80 *
81 * Our tactic to handle this is to first atomically read and
82 * exchange a new raw count - then add that new-prev delta
83 * count to the generic event atomically:
84 */
85 again:
86 prev_raw_count = local64_read(&hwc->prev_count);
87 rdpmcl(hwc->event_base_rdpmc, new_raw_count);
88
89 if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
90 new_raw_count) != prev_raw_count)
91 goto again;
92
93 /*
94 * Now we have the new raw value and have updated the prev
95 * timestamp already. We can now calculate the elapsed delta
96 * (event-)time and add that to the generic event.
97 *
98 * Careful, not all hw sign-extends above the physical width
99 * of the count.
100 */
101 delta = (new_raw_count << shift) - (prev_raw_count << shift);
102 delta >>= shift;
103
104 local64_add(delta, &event->count);
105 local64_sub(delta, &hwc->period_left);
106
107 return new_raw_count;
108 }
109
110 /*
111 * Find and validate any extra registers to set up.
112 */
113 static int x86_pmu_extra_regs(u64 config, struct perf_event *event)
114 {
115 struct hw_perf_event_extra *reg;
116 struct extra_reg *er;
117
118 reg = &event->hw.extra_reg;
119
120 if (!x86_pmu.extra_regs)
121 return 0;
122
123 for (er = x86_pmu.extra_regs; er->msr; er++) {
124 if (er->event != (config & er->config_mask))
125 continue;
126 if (event->attr.config1 & ~er->valid_mask)
127 return -EINVAL;
128 /* Check if the extra msrs can be safely accessed*/
129 if (!er->extra_msr_access)
130 return -ENXIO;
131
132 reg->idx = er->idx;
133 reg->config = event->attr.config1;
134 reg->reg = er->msr;
135 break;
136 }
137 return 0;
138 }
139
140 static atomic_t active_events;
141 static atomic_t pmc_refcount;
142 static DEFINE_MUTEX(pmc_reserve_mutex);
143
144 #ifdef CONFIG_X86_LOCAL_APIC
145
146 static bool reserve_pmc_hardware(void)
147 {
148 int i;
149
150 for (i = 0; i < x86_pmu.num_counters; i++) {
151 if (!reserve_perfctr_nmi(x86_pmu_event_addr(i)))
152 goto perfctr_fail;
153 }
154
155 for (i = 0; i < x86_pmu.num_counters; i++) {
156 if (!reserve_evntsel_nmi(x86_pmu_config_addr(i)))
157 goto eventsel_fail;
158 }
159
160 return true;
161
162 eventsel_fail:
163 for (i--; i >= 0; i--)
164 release_evntsel_nmi(x86_pmu_config_addr(i));
165
166 i = x86_pmu.num_counters;
167
168 perfctr_fail:
169 for (i--; i >= 0; i--)
170 release_perfctr_nmi(x86_pmu_event_addr(i));
171
172 return false;
173 }
174
175 static void release_pmc_hardware(void)
176 {
177 int i;
178
179 for (i = 0; i < x86_pmu.num_counters; i++) {
180 release_perfctr_nmi(x86_pmu_event_addr(i));
181 release_evntsel_nmi(x86_pmu_config_addr(i));
182 }
183 }
184
185 #else
186
187 static bool reserve_pmc_hardware(void) { return true; }
188 static void release_pmc_hardware(void) {}
189
190 #endif
191
192 static bool check_hw_exists(void)
193 {
194 u64 val, val_fail = -1, val_new= ~0;
195 int i, reg, reg_fail = -1, ret = 0;
196 int bios_fail = 0;
197 int reg_safe = -1;
198
199 /*
200 * Check to see if the BIOS enabled any of the counters, if so
201 * complain and bail.
202 */
203 for (i = 0; i < x86_pmu.num_counters; i++) {
204 reg = x86_pmu_config_addr(i);
205 ret = rdmsrl_safe(reg, &val);
206 if (ret)
207 goto msr_fail;
208 if (val & ARCH_PERFMON_EVENTSEL_ENABLE) {
209 bios_fail = 1;
210 val_fail = val;
211 reg_fail = reg;
212 } else {
213 reg_safe = i;
214 }
215 }
216
217 if (x86_pmu.num_counters_fixed) {
218 reg = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
219 ret = rdmsrl_safe(reg, &val);
220 if (ret)
221 goto msr_fail;
222 for (i = 0; i < x86_pmu.num_counters_fixed; i++) {
223 if (val & (0x03 << i*4)) {
224 bios_fail = 1;
225 val_fail = val;
226 reg_fail = reg;
227 }
228 }
229 }
230
231 /*
232 * If all the counters are enabled, the below test will always
233 * fail. The tools will also become useless in this scenario.
234 * Just fail and disable the hardware counters.
235 */
236
237 if (reg_safe == -1) {
238 reg = reg_safe;
239 goto msr_fail;
240 }
241
242 /*
243 * Read the current value, change it and read it back to see if it
244 * matches, this is needed to detect certain hardware emulators
245 * (qemu/kvm) that don't trap on the MSR access and always return 0s.
246 */
247 reg = x86_pmu_event_addr(reg_safe);
248 if (rdmsrl_safe(reg, &val))
249 goto msr_fail;
250 val ^= 0xffffUL;
251 ret = wrmsrl_safe(reg, val);
252 ret |= rdmsrl_safe(reg, &val_new);
253 if (ret || val != val_new)
254 goto msr_fail;
255
256 /*
257 * We still allow the PMU driver to operate:
258 */
259 if (bios_fail) {
260 pr_cont("Broken BIOS detected, complain to your hardware vendor.\n");
261 pr_err(FW_BUG "the BIOS has corrupted hw-PMU resources (MSR %x is %Lx)\n",
262 reg_fail, val_fail);
263 }
264
265 return true;
266
267 msr_fail:
268 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
269 pr_cont("PMU not available due to virtualization, using software events only.\n");
270 } else {
271 pr_cont("Broken PMU hardware detected, using software events only.\n");
272 pr_err("Failed to access perfctr msr (MSR %x is %Lx)\n",
273 reg, val_new);
274 }
275
276 return false;
277 }
278
279 static void hw_perf_event_destroy(struct perf_event *event)
280 {
281 x86_release_hardware();
282 atomic_dec(&active_events);
283 }
284
285 void hw_perf_lbr_event_destroy(struct perf_event *event)
286 {
287 hw_perf_event_destroy(event);
288
289 /* undo the lbr/bts event accounting */
290 x86_del_exclusive(x86_lbr_exclusive_lbr);
291 }
292
293 static inline int x86_pmu_initialized(void)
294 {
295 return x86_pmu.handle_irq != NULL;
296 }
297
298 static inline int
299 set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event *event)
300 {
301 struct perf_event_attr *attr = &event->attr;
302 unsigned int cache_type, cache_op, cache_result;
303 u64 config, val;
304
305 config = attr->config;
306
307 cache_type = (config >> 0) & 0xff;
308 if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
309 return -EINVAL;
310
311 cache_op = (config >> 8) & 0xff;
312 if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
313 return -EINVAL;
314
315 cache_result = (config >> 16) & 0xff;
316 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
317 return -EINVAL;
318
319 val = hw_cache_event_ids[cache_type][cache_op][cache_result];
320
321 if (val == 0)
322 return -ENOENT;
323
324 if (val == -1)
325 return -EINVAL;
326
327 hwc->config |= val;
328 attr->config1 = hw_cache_extra_regs[cache_type][cache_op][cache_result];
329 return x86_pmu_extra_regs(val, event);
330 }
331
332 int x86_reserve_hardware(void)
333 {
334 int err = 0;
335
336 if (!atomic_inc_not_zero(&pmc_refcount)) {
337 mutex_lock(&pmc_reserve_mutex);
338 if (atomic_read(&pmc_refcount) == 0) {
339 if (!reserve_pmc_hardware())
340 err = -EBUSY;
341 else
342 reserve_ds_buffers();
343 }
344 if (!err)
345 atomic_inc(&pmc_refcount);
346 mutex_unlock(&pmc_reserve_mutex);
347 }
348
349 return err;
350 }
351
352 void x86_release_hardware(void)
353 {
354 if (atomic_dec_and_mutex_lock(&pmc_refcount, &pmc_reserve_mutex)) {
355 release_pmc_hardware();
356 release_ds_buffers();
357 mutex_unlock(&pmc_reserve_mutex);
358 }
359 }
360
361 /*
362 * Check if we can create event of a certain type (that no conflicting events
363 * are present).
364 */
365 int x86_add_exclusive(unsigned int what)
366 {
367 int i;
368
369 /*
370 * When lbr_pt_coexist we allow PT to coexist with either LBR or BTS.
371 * LBR and BTS are still mutually exclusive.
372 */
373 if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt)
374 return 0;
375
376 if (!atomic_inc_not_zero(&x86_pmu.lbr_exclusive[what])) {
377 mutex_lock(&pmc_reserve_mutex);
378 for (i = 0; i < ARRAY_SIZE(x86_pmu.lbr_exclusive); i++) {
379 if (i != what && atomic_read(&x86_pmu.lbr_exclusive[i]))
380 goto fail_unlock;
381 }
382 atomic_inc(&x86_pmu.lbr_exclusive[what]);
383 mutex_unlock(&pmc_reserve_mutex);
384 }
385
386 atomic_inc(&active_events);
387 return 0;
388
389 fail_unlock:
390 mutex_unlock(&pmc_reserve_mutex);
391 return -EBUSY;
392 }
393
394 void x86_del_exclusive(unsigned int what)
395 {
396 if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt)
397 return;
398
399 atomic_dec(&x86_pmu.lbr_exclusive[what]);
400 atomic_dec(&active_events);
401 }
402
403 int x86_setup_perfctr(struct perf_event *event)
404 {
405 struct perf_event_attr *attr = &event->attr;
406 struct hw_perf_event *hwc = &event->hw;
407 u64 config;
408
409 if (!is_sampling_event(event)) {
410 hwc->sample_period = x86_pmu.max_period;
411 hwc->last_period = hwc->sample_period;
412 local64_set(&hwc->period_left, hwc->sample_period);
413 }
414
415 if (attr->type == PERF_TYPE_RAW)
416 return x86_pmu_extra_regs(event->attr.config, event);
417
418 if (attr->type == PERF_TYPE_HW_CACHE)
419 return set_ext_hw_attr(hwc, event);
420
421 if (attr->config >= x86_pmu.max_events)
422 return -EINVAL;
423
424 /*
425 * The generic map:
426 */
427 config = x86_pmu.event_map(attr->config);
428
429 if (config == 0)
430 return -ENOENT;
431
432 if (config == -1LL)
433 return -EINVAL;
434
435 /*
436 * Branch tracing:
437 */
438 if (attr->config == PERF_COUNT_HW_BRANCH_INSTRUCTIONS &&
439 !attr->freq && hwc->sample_period == 1) {
440 /* BTS is not supported by this architecture. */
441 if (!x86_pmu.bts_active)
442 return -EOPNOTSUPP;
443
444 /* BTS is currently only allowed for user-mode. */
445 if (!attr->exclude_kernel)
446 return -EOPNOTSUPP;
447
448 /* disallow bts if conflicting events are present */
449 if (x86_add_exclusive(x86_lbr_exclusive_lbr))
450 return -EBUSY;
451
452 event->destroy = hw_perf_lbr_event_destroy;
453 }
454
455 hwc->config |= config;
456
457 return 0;
458 }
459
460 /*
461 * check that branch_sample_type is compatible with
462 * settings needed for precise_ip > 1 which implies
463 * using the LBR to capture ALL taken branches at the
464 * priv levels of the measurement
465 */
466 static inline int precise_br_compat(struct perf_event *event)
467 {
468 u64 m = event->attr.branch_sample_type;
469 u64 b = 0;
470
471 /* must capture all branches */
472 if (!(m & PERF_SAMPLE_BRANCH_ANY))
473 return 0;
474
475 m &= PERF_SAMPLE_BRANCH_KERNEL | PERF_SAMPLE_BRANCH_USER;
476
477 if (!event->attr.exclude_user)
478 b |= PERF_SAMPLE_BRANCH_USER;
479
480 if (!event->attr.exclude_kernel)
481 b |= PERF_SAMPLE_BRANCH_KERNEL;
482
483 /*
484 * ignore PERF_SAMPLE_BRANCH_HV, not supported on x86
485 */
486
487 return m == b;
488 }
489
490 int x86_pmu_max_precise(void)
491 {
492 int precise = 0;
493
494 /* Support for constant skid */
495 if (x86_pmu.pebs_active && !x86_pmu.pebs_broken) {
496 precise++;
497
498 /* Support for IP fixup */
499 if (x86_pmu.lbr_nr || x86_pmu.intel_cap.pebs_format >= 2)
500 precise++;
501
502 if (x86_pmu.pebs_prec_dist)
503 precise++;
504 }
505 return precise;
506 }
507
508 int x86_pmu_hw_config(struct perf_event *event)
509 {
510 if (event->attr.precise_ip) {
511 int precise = x86_pmu_max_precise();
512
513 if (event->attr.precise_ip > precise)
514 return -EOPNOTSUPP;
515
516 /* There's no sense in having PEBS for non sampling events: */
517 if (!is_sampling_event(event))
518 return -EINVAL;
519 }
520 /*
521 * check that PEBS LBR correction does not conflict with
522 * whatever the user is asking with attr->branch_sample_type
523 */
524 if (event->attr.precise_ip > 1 && x86_pmu.intel_cap.pebs_format < 2) {
525 u64 *br_type = &event->attr.branch_sample_type;
526
527 if (has_branch_stack(event)) {
528 if (!precise_br_compat(event))
529 return -EOPNOTSUPP;
530
531 /* branch_sample_type is compatible */
532
533 } else {
534 /*
535 * user did not specify branch_sample_type
536 *
537 * For PEBS fixups, we capture all
538 * the branches at the priv level of the
539 * event.
540 */
541 *br_type = PERF_SAMPLE_BRANCH_ANY;
542
543 if (!event->attr.exclude_user)
544 *br_type |= PERF_SAMPLE_BRANCH_USER;
545
546 if (!event->attr.exclude_kernel)
547 *br_type |= PERF_SAMPLE_BRANCH_KERNEL;
548 }
549 }
550
551 if (event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_CALL_STACK)
552 event->attach_state |= PERF_ATTACH_TASK_DATA;
553
554 /*
555 * Generate PMC IRQs:
556 * (keep 'enabled' bit clear for now)
557 */
558 event->hw.config = ARCH_PERFMON_EVENTSEL_INT;
559
560 /*
561 * Count user and OS events unless requested not to
562 */
563 if (!event->attr.exclude_user)
564 event->hw.config |= ARCH_PERFMON_EVENTSEL_USR;
565 if (!event->attr.exclude_kernel)
566 event->hw.config |= ARCH_PERFMON_EVENTSEL_OS;
567
568 if (event->attr.type == PERF_TYPE_RAW)
569 event->hw.config |= event->attr.config & X86_RAW_EVENT_MASK;
570
571 if (event->attr.sample_period && x86_pmu.limit_period) {
572 if (x86_pmu.limit_period(event, event->attr.sample_period) >
573 event->attr.sample_period)
574 return -EINVAL;
575 }
576
577 return x86_setup_perfctr(event);
578 }
579
580 /*
581 * Setup the hardware configuration for a given attr_type
582 */
583 static int __x86_pmu_event_init(struct perf_event *event)
584 {
585 int err;
586
587 if (!x86_pmu_initialized())
588 return -ENODEV;
589
590 err = x86_reserve_hardware();
591 if (err)
592 return err;
593
594 atomic_inc(&active_events);
595 event->destroy = hw_perf_event_destroy;
596
597 event->hw.idx = -1;
598 event->hw.last_cpu = -1;
599 event->hw.last_tag = ~0ULL;
600
601 /* mark unused */
602 event->hw.extra_reg.idx = EXTRA_REG_NONE;
603 event->hw.branch_reg.idx = EXTRA_REG_NONE;
604
605 return x86_pmu.hw_config(event);
606 }
607
608 void x86_pmu_disable_all(void)
609 {
610 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
611 int idx;
612
613 for (idx = 0; idx < x86_pmu.num_counters; idx++) {
614 u64 val;
615
616 if (!test_bit(idx, cpuc->active_mask))
617 continue;
618 rdmsrl(x86_pmu_config_addr(idx), val);
619 if (!(val & ARCH_PERFMON_EVENTSEL_ENABLE))
620 continue;
621 val &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
622 wrmsrl(x86_pmu_config_addr(idx), val);
623 }
624 }
625
626 /*
627 * There may be PMI landing after enabled=0. The PMI hitting could be before or
628 * after disable_all.
629 *
630 * If PMI hits before disable_all, the PMU will be disabled in the NMI handler.
631 * It will not be re-enabled in the NMI handler again, because enabled=0. After
632 * handling the NMI, disable_all will be called, which will not change the
633 * state either. If PMI hits after disable_all, the PMU is already disabled
634 * before entering NMI handler. The NMI handler will not change the state
635 * either.
636 *
637 * So either situation is harmless.
638 */
639 static void x86_pmu_disable(struct pmu *pmu)
640 {
641 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
642
643 if (!x86_pmu_initialized())
644 return;
645
646 if (!cpuc->enabled)
647 return;
648
649 cpuc->n_added = 0;
650 cpuc->enabled = 0;
651 barrier();
652
653 x86_pmu.disable_all();
654 }
655
656 void x86_pmu_enable_all(int added)
657 {
658 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
659 int idx;
660
661 for (idx = 0; idx < x86_pmu.num_counters; idx++) {
662 struct hw_perf_event *hwc = &cpuc->events[idx]->hw;
663
664 if (!test_bit(idx, cpuc->active_mask))
665 continue;
666
667 __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
668 }
669 }
670
671 static struct pmu pmu;
672
673 static inline int is_x86_event(struct perf_event *event)
674 {
675 return event->pmu == &pmu;
676 }
677
678 /*
679 * Event scheduler state:
680 *
681 * Assign events iterating over all events and counters, beginning
682 * with events with least weights first. Keep the current iterator
683 * state in struct sched_state.
684 */
685 struct sched_state {
686 int weight;
687 int event; /* event index */
688 int counter; /* counter index */
689 int unassigned; /* number of events to be assigned left */
690 int nr_gp; /* number of GP counters used */
691 unsigned long used[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
692 };
693
694 /* Total max is X86_PMC_IDX_MAX, but we are O(n!) limited */
695 #define SCHED_STATES_MAX 2
696
697 struct perf_sched {
698 int max_weight;
699 int max_events;
700 int max_gp;
701 int saved_states;
702 struct event_constraint **constraints;
703 struct sched_state state;
704 struct sched_state saved[SCHED_STATES_MAX];
705 };
706
707 /*
708 * Initialize interator that runs through all events and counters.
709 */
710 static void perf_sched_init(struct perf_sched *sched, struct event_constraint **constraints,
711 int num, int wmin, int wmax, int gpmax)
712 {
713 int idx;
714
715 memset(sched, 0, sizeof(*sched));
716 sched->max_events = num;
717 sched->max_weight = wmax;
718 sched->max_gp = gpmax;
719 sched->constraints = constraints;
720
721 for (idx = 0; idx < num; idx++) {
722 if (constraints[idx]->weight == wmin)
723 break;
724 }
725
726 sched->state.event = idx; /* start with min weight */
727 sched->state.weight = wmin;
728 sched->state.unassigned = num;
729 }
730
731 static void perf_sched_save_state(struct perf_sched *sched)
732 {
733 if (WARN_ON_ONCE(sched->saved_states >= SCHED_STATES_MAX))
734 return;
735
736 sched->saved[sched->saved_states] = sched->state;
737 sched->saved_states++;
738 }
739
740 static bool perf_sched_restore_state(struct perf_sched *sched)
741 {
742 if (!sched->saved_states)
743 return false;
744
745 sched->saved_states--;
746 sched->state = sched->saved[sched->saved_states];
747
748 /* continue with next counter: */
749 clear_bit(sched->state.counter++, sched->state.used);
750
751 return true;
752 }
753
754 /*
755 * Select a counter for the current event to schedule. Return true on
756 * success.
757 */
758 static bool __perf_sched_find_counter(struct perf_sched *sched)
759 {
760 struct event_constraint *c;
761 int idx;
762
763 if (!sched->state.unassigned)
764 return false;
765
766 if (sched->state.event >= sched->max_events)
767 return false;
768
769 c = sched->constraints[sched->state.event];
770 /* Prefer fixed purpose counters */
771 if (c->idxmsk64 & (~0ULL << INTEL_PMC_IDX_FIXED)) {
772 idx = INTEL_PMC_IDX_FIXED;
773 for_each_set_bit_from(idx, c->idxmsk, X86_PMC_IDX_MAX) {
774 if (!__test_and_set_bit(idx, sched->state.used))
775 goto done;
776 }
777 }
778
779 /* Grab the first unused counter starting with idx */
780 idx = sched->state.counter;
781 for_each_set_bit_from(idx, c->idxmsk, INTEL_PMC_IDX_FIXED) {
782 if (!__test_and_set_bit(idx, sched->state.used)) {
783 if (sched->state.nr_gp++ >= sched->max_gp)
784 return false;
785
786 goto done;
787 }
788 }
789
790 return false;
791
792 done:
793 sched->state.counter = idx;
794
795 if (c->overlap)
796 perf_sched_save_state(sched);
797
798 return true;
799 }
800
801 static bool perf_sched_find_counter(struct perf_sched *sched)
802 {
803 while (!__perf_sched_find_counter(sched)) {
804 if (!perf_sched_restore_state(sched))
805 return false;
806 }
807
808 return true;
809 }
810
811 /*
812 * Go through all unassigned events and find the next one to schedule.
813 * Take events with the least weight first. Return true on success.
814 */
815 static bool perf_sched_next_event(struct perf_sched *sched)
816 {
817 struct event_constraint *c;
818
819 if (!sched->state.unassigned || !--sched->state.unassigned)
820 return false;
821
822 do {
823 /* next event */
824 sched->state.event++;
825 if (sched->state.event >= sched->max_events) {
826 /* next weight */
827 sched->state.event = 0;
828 sched->state.weight++;
829 if (sched->state.weight > sched->max_weight)
830 return false;
831 }
832 c = sched->constraints[sched->state.event];
833 } while (c->weight != sched->state.weight);
834
835 sched->state.counter = 0; /* start with first counter */
836
837 return true;
838 }
839
840 /*
841 * Assign a counter for each event.
842 */
843 int perf_assign_events(struct event_constraint **constraints, int n,
844 int wmin, int wmax, int gpmax, int *assign)
845 {
846 struct perf_sched sched;
847
848 perf_sched_init(&sched, constraints, n, wmin, wmax, gpmax);
849
850 do {
851 if (!perf_sched_find_counter(&sched))
852 break; /* failed */
853 if (assign)
854 assign[sched.state.event] = sched.state.counter;
855 } while (perf_sched_next_event(&sched));
856
857 return sched.state.unassigned;
858 }
859 EXPORT_SYMBOL_GPL(perf_assign_events);
860
861 int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)
862 {
863 struct event_constraint *c;
864 unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
865 struct perf_event *e;
866 int i, wmin, wmax, unsched = 0;
867 struct hw_perf_event *hwc;
868
869 bitmap_zero(used_mask, X86_PMC_IDX_MAX);
870
871 if (x86_pmu.start_scheduling)
872 x86_pmu.start_scheduling(cpuc);
873
874 for (i = 0, wmin = X86_PMC_IDX_MAX, wmax = 0; i < n; i++) {
875 cpuc->event_constraint[i] = NULL;
876 c = x86_pmu.get_event_constraints(cpuc, i, cpuc->event_list[i]);
877 cpuc->event_constraint[i] = c;
878
879 wmin = min(wmin, c->weight);
880 wmax = max(wmax, c->weight);
881 }
882
883 /*
884 * fastpath, try to reuse previous register
885 */
886 for (i = 0; i < n; i++) {
887 hwc = &cpuc->event_list[i]->hw;
888 c = cpuc->event_constraint[i];
889
890 /* never assigned */
891 if (hwc->idx == -1)
892 break;
893
894 /* constraint still honored */
895 if (!test_bit(hwc->idx, c->idxmsk))
896 break;
897
898 /* not already used */
899 if (test_bit(hwc->idx, used_mask))
900 break;
901
902 __set_bit(hwc->idx, used_mask);
903 if (assign)
904 assign[i] = hwc->idx;
905 }
906
907 /* slow path */
908 if (i != n) {
909 int gpmax = x86_pmu.num_counters;
910
911 /*
912 * Do not allow scheduling of more than half the available
913 * generic counters.
914 *
915 * This helps avoid counter starvation of sibling thread by
916 * ensuring at most half the counters cannot be in exclusive
917 * mode. There is no designated counters for the limits. Any
918 * N/2 counters can be used. This helps with events with
919 * specific counter constraints.
920 */
921 if (is_ht_workaround_enabled() && !cpuc->is_fake &&
922 READ_ONCE(cpuc->excl_cntrs->exclusive_present))
923 gpmax /= 2;
924
925 unsched = perf_assign_events(cpuc->event_constraint, n, wmin,
926 wmax, gpmax, assign);
927 }
928
929 /*
930 * In case of success (unsched = 0), mark events as committed,
931 * so we do not put_constraint() in case new events are added
932 * and fail to be scheduled
933 *
934 * We invoke the lower level commit callback to lock the resource
935 *
936 * We do not need to do all of this in case we are called to
937 * validate an event group (assign == NULL)
938 */
939 if (!unsched && assign) {
940 for (i = 0; i < n; i++) {
941 e = cpuc->event_list[i];
942 e->hw.flags |= PERF_X86_EVENT_COMMITTED;
943 if (x86_pmu.commit_scheduling)
944 x86_pmu.commit_scheduling(cpuc, i, assign[i]);
945 }
946 } else {
947 for (i = 0; i < n; i++) {
948 e = cpuc->event_list[i];
949 /*
950 * do not put_constraint() on comitted events,
951 * because they are good to go
952 */
953 if ((e->hw.flags & PERF_X86_EVENT_COMMITTED))
954 continue;
955
956 /*
957 * release events that failed scheduling
958 */
959 if (x86_pmu.put_event_constraints)
960 x86_pmu.put_event_constraints(cpuc, e);
961 }
962 }
963
964 if (x86_pmu.stop_scheduling)
965 x86_pmu.stop_scheduling(cpuc);
966
967 return unsched ? -EINVAL : 0;
968 }
969
970 /*
971 * dogrp: true if must collect siblings events (group)
972 * returns total number of events and error code
973 */
974 static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp)
975 {
976 struct perf_event *event;
977 int n, max_count;
978
979 max_count = x86_pmu.num_counters + x86_pmu.num_counters_fixed;
980
981 /* current number of events already accepted */
982 n = cpuc->n_events;
983
984 if (is_x86_event(leader)) {
985 if (n >= max_count)
986 return -EINVAL;
987 cpuc->event_list[n] = leader;
988 n++;
989 }
990 if (!dogrp)
991 return n;
992
993 list_for_each_entry(event, &leader->sibling_list, group_entry) {
994 if (!is_x86_event(event) ||
995 event->state <= PERF_EVENT_STATE_OFF)
996 continue;
997
998 if (n >= max_count)
999 return -EINVAL;
1000
1001 cpuc->event_list[n] = event;
1002 n++;
1003 }
1004 return n;
1005 }
1006
1007 static inline void x86_assign_hw_event(struct perf_event *event,
1008 struct cpu_hw_events *cpuc, int i)
1009 {
1010 struct hw_perf_event *hwc = &event->hw;
1011
1012 hwc->idx = cpuc->assign[i];
1013 hwc->last_cpu = smp_processor_id();
1014 hwc->last_tag = ++cpuc->tags[i];
1015
1016 if (hwc->idx == INTEL_PMC_IDX_FIXED_BTS) {
1017 hwc->config_base = 0;
1018 hwc->event_base = 0;
1019 } else if (hwc->idx >= INTEL_PMC_IDX_FIXED) {
1020 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
1021 hwc->event_base = MSR_ARCH_PERFMON_FIXED_CTR0 + (hwc->idx - INTEL_PMC_IDX_FIXED);
1022 hwc->event_base_rdpmc = (hwc->idx - INTEL_PMC_IDX_FIXED) | 1<<30;
1023 } else {
1024 hwc->config_base = x86_pmu_config_addr(hwc->idx);
1025 hwc->event_base = x86_pmu_event_addr(hwc->idx);
1026 hwc->event_base_rdpmc = x86_pmu_rdpmc_index(hwc->idx);
1027 }
1028 }
1029
1030 static inline int match_prev_assignment(struct hw_perf_event *hwc,
1031 struct cpu_hw_events *cpuc,
1032 int i)
1033 {
1034 return hwc->idx == cpuc->assign[i] &&
1035 hwc->last_cpu == smp_processor_id() &&
1036 hwc->last_tag == cpuc->tags[i];
1037 }
1038
1039 static void x86_pmu_start(struct perf_event *event, int flags);
1040
1041 static void x86_pmu_enable(struct pmu *pmu)
1042 {
1043 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1044 struct perf_event *event;
1045 struct hw_perf_event *hwc;
1046 int i, added = cpuc->n_added;
1047
1048 if (!x86_pmu_initialized())
1049 return;
1050
1051 if (cpuc->enabled)
1052 return;
1053
1054 if (cpuc->n_added) {
1055 int n_running = cpuc->n_events - cpuc->n_added;
1056 /*
1057 * apply assignment obtained either from
1058 * hw_perf_group_sched_in() or x86_pmu_enable()
1059 *
1060 * step1: save events moving to new counters
1061 */
1062 for (i = 0; i < n_running; i++) {
1063 event = cpuc->event_list[i];
1064 hwc = &event->hw;
1065
1066 /*
1067 * we can avoid reprogramming counter if:
1068 * - assigned same counter as last time
1069 * - running on same CPU as last time
1070 * - no other event has used the counter since
1071 */
1072 if (hwc->idx == -1 ||
1073 match_prev_assignment(hwc, cpuc, i))
1074 continue;
1075
1076 /*
1077 * Ensure we don't accidentally enable a stopped
1078 * counter simply because we rescheduled.
1079 */
1080 if (hwc->state & PERF_HES_STOPPED)
1081 hwc->state |= PERF_HES_ARCH;
1082
1083 x86_pmu_stop(event, PERF_EF_UPDATE);
1084 }
1085
1086 /*
1087 * step2: reprogram moved events into new counters
1088 */
1089 for (i = 0; i < cpuc->n_events; i++) {
1090 event = cpuc->event_list[i];
1091 hwc = &event->hw;
1092
1093 if (!match_prev_assignment(hwc, cpuc, i))
1094 x86_assign_hw_event(event, cpuc, i);
1095 else if (i < n_running)
1096 continue;
1097
1098 if (hwc->state & PERF_HES_ARCH)
1099 continue;
1100
1101 x86_pmu_start(event, PERF_EF_RELOAD);
1102 }
1103 cpuc->n_added = 0;
1104 perf_events_lapic_init();
1105 }
1106
1107 cpuc->enabled = 1;
1108 barrier();
1109
1110 x86_pmu.enable_all(added);
1111 }
1112
1113 static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
1114
1115 /*
1116 * Set the next IRQ period, based on the hwc->period_left value.
1117 * To be called with the event disabled in hw:
1118 */
1119 int x86_perf_event_set_period(struct perf_event *event)
1120 {
1121 struct hw_perf_event *hwc = &event->hw;
1122 s64 left = local64_read(&hwc->period_left);
1123 s64 period = hwc->sample_period;
1124 int ret = 0, idx = hwc->idx;
1125
1126 if (idx == INTEL_PMC_IDX_FIXED_BTS)
1127 return 0;
1128
1129 /*
1130 * If we are way outside a reasonable range then just skip forward:
1131 */
1132 if (unlikely(left <= -period)) {
1133 left = period;
1134 local64_set(&hwc->period_left, left);
1135 hwc->last_period = period;
1136 ret = 1;
1137 }
1138
1139 if (unlikely(left <= 0)) {
1140 left += period;
1141 local64_set(&hwc->period_left, left);
1142 hwc->last_period = period;
1143 ret = 1;
1144 }
1145 /*
1146 * Quirk: certain CPUs dont like it if just 1 hw_event is left:
1147 */
1148 if (unlikely(left < 2))
1149 left = 2;
1150
1151 if (left > x86_pmu.max_period)
1152 left = x86_pmu.max_period;
1153
1154 if (x86_pmu.limit_period)
1155 left = x86_pmu.limit_period(event, left);
1156
1157 per_cpu(pmc_prev_left[idx], smp_processor_id()) = left;
1158
1159 /*
1160 * The hw event starts counting from this event offset,
1161 * mark it to be able to extra future deltas:
1162 */
1163 local64_set(&hwc->prev_count, (u64)-left);
1164
1165 wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask);
1166
1167 /*
1168 * Due to erratum on certan cpu we need
1169 * a second write to be sure the register
1170 * is updated properly
1171 */
1172 if (x86_pmu.perfctr_second_write) {
1173 wrmsrl(hwc->event_base,
1174 (u64)(-left) & x86_pmu.cntval_mask);
1175 }
1176
1177 perf_event_update_userpage(event);
1178
1179 return ret;
1180 }
1181
1182 void x86_pmu_enable_event(struct perf_event *event)
1183 {
1184 if (__this_cpu_read(cpu_hw_events.enabled))
1185 __x86_pmu_enable_event(&event->hw,
1186 ARCH_PERFMON_EVENTSEL_ENABLE);
1187 }
1188
1189 /*
1190 * Add a single event to the PMU.
1191 *
1192 * The event is added to the group of enabled events
1193 * but only if it can be scehduled with existing events.
1194 */
1195 static int x86_pmu_add(struct perf_event *event, int flags)
1196 {
1197 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1198 struct hw_perf_event *hwc;
1199 int assign[X86_PMC_IDX_MAX];
1200 int n, n0, ret;
1201
1202 hwc = &event->hw;
1203
1204 n0 = cpuc->n_events;
1205 ret = n = collect_events(cpuc, event, false);
1206 if (ret < 0)
1207 goto out;
1208
1209 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
1210 if (!(flags & PERF_EF_START))
1211 hwc->state |= PERF_HES_ARCH;
1212
1213 /*
1214 * If group events scheduling transaction was started,
1215 * skip the schedulability test here, it will be performed
1216 * at commit time (->commit_txn) as a whole.
1217 *
1218 * If commit fails, we'll call ->del() on all events
1219 * for which ->add() was called.
1220 */
1221 if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
1222 goto done_collect;
1223
1224 ret = x86_pmu.schedule_events(cpuc, n, assign);
1225 if (ret)
1226 goto out;
1227 /*
1228 * copy new assignment, now we know it is possible
1229 * will be used by hw_perf_enable()
1230 */
1231 memcpy(cpuc->assign, assign, n*sizeof(int));
1232
1233 done_collect:
1234 /*
1235 * Commit the collect_events() state. See x86_pmu_del() and
1236 * x86_pmu_*_txn().
1237 */
1238 cpuc->n_events = n;
1239 cpuc->n_added += n - n0;
1240 cpuc->n_txn += n - n0;
1241
1242 if (x86_pmu.add) {
1243 /*
1244 * This is before x86_pmu_enable() will call x86_pmu_start(),
1245 * so we enable LBRs before an event needs them etc..
1246 */
1247 x86_pmu.add(event);
1248 }
1249
1250 ret = 0;
1251 out:
1252 return ret;
1253 }
1254
1255 static void x86_pmu_start(struct perf_event *event, int flags)
1256 {
1257 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1258 int idx = event->hw.idx;
1259
1260 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
1261 return;
1262
1263 if (WARN_ON_ONCE(idx == -1))
1264 return;
1265
1266 if (flags & PERF_EF_RELOAD) {
1267 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
1268 x86_perf_event_set_period(event);
1269 }
1270
1271 event->hw.state = 0;
1272
1273 cpuc->events[idx] = event;
1274 __set_bit(idx, cpuc->active_mask);
1275 __set_bit(idx, cpuc->running);
1276 x86_pmu.enable(event);
1277 perf_event_update_userpage(event);
1278 }
1279
1280 void perf_event_print_debug(void)
1281 {
1282 u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
1283 u64 pebs, debugctl;
1284 struct cpu_hw_events *cpuc;
1285 unsigned long flags;
1286 int cpu, idx;
1287
1288 if (!x86_pmu.num_counters)
1289 return;
1290
1291 local_irq_save(flags);
1292
1293 cpu = smp_processor_id();
1294 cpuc = &per_cpu(cpu_hw_events, cpu);
1295
1296 if (x86_pmu.version >= 2) {
1297 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
1298 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
1299 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
1300 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
1301
1302 pr_info("\n");
1303 pr_info("CPU#%d: ctrl: %016llx\n", cpu, ctrl);
1304 pr_info("CPU#%d: status: %016llx\n", cpu, status);
1305 pr_info("CPU#%d: overflow: %016llx\n", cpu, overflow);
1306 pr_info("CPU#%d: fixed: %016llx\n", cpu, fixed);
1307 if (x86_pmu.pebs_constraints) {
1308 rdmsrl(MSR_IA32_PEBS_ENABLE, pebs);
1309 pr_info("CPU#%d: pebs: %016llx\n", cpu, pebs);
1310 }
1311 if (x86_pmu.lbr_nr) {
1312 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
1313 pr_info("CPU#%d: debugctl: %016llx\n", cpu, debugctl);
1314 }
1315 }
1316 pr_info("CPU#%d: active: %016llx\n", cpu, *(u64 *)cpuc->active_mask);
1317
1318 for (idx = 0; idx < x86_pmu.num_counters; idx++) {
1319 rdmsrl(x86_pmu_config_addr(idx), pmc_ctrl);
1320 rdmsrl(x86_pmu_event_addr(idx), pmc_count);
1321
1322 prev_left = per_cpu(pmc_prev_left[idx], cpu);
1323
1324 pr_info("CPU#%d: gen-PMC%d ctrl: %016llx\n",
1325 cpu, idx, pmc_ctrl);
1326 pr_info("CPU#%d: gen-PMC%d count: %016llx\n",
1327 cpu, idx, pmc_count);
1328 pr_info("CPU#%d: gen-PMC%d left: %016llx\n",
1329 cpu, idx, prev_left);
1330 }
1331 for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
1332 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
1333
1334 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
1335 cpu, idx, pmc_count);
1336 }
1337 local_irq_restore(flags);
1338 }
1339
1340 void x86_pmu_stop(struct perf_event *event, int flags)
1341 {
1342 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1343 struct hw_perf_event *hwc = &event->hw;
1344
1345 if (__test_and_clear_bit(hwc->idx, cpuc->active_mask)) {
1346 x86_pmu.disable(event);
1347 cpuc->events[hwc->idx] = NULL;
1348 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
1349 hwc->state |= PERF_HES_STOPPED;
1350 }
1351
1352 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
1353 /*
1354 * Drain the remaining delta count out of a event
1355 * that we are disabling:
1356 */
1357 x86_perf_event_update(event);
1358 hwc->state |= PERF_HES_UPTODATE;
1359 }
1360 }
1361
1362 static void x86_pmu_del(struct perf_event *event, int flags)
1363 {
1364 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1365 int i;
1366
1367 /*
1368 * event is descheduled
1369 */
1370 event->hw.flags &= ~PERF_X86_EVENT_COMMITTED;
1371
1372 /*
1373 * If we're called during a txn, we only need to undo x86_pmu.add.
1374 * The events never got scheduled and ->cancel_txn will truncate
1375 * the event_list.
1376 *
1377 * XXX assumes any ->del() called during a TXN will only be on
1378 * an event added during that same TXN.
1379 */
1380 if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
1381 goto do_del;
1382
1383 /*
1384 * Not a TXN, therefore cleanup properly.
1385 */
1386 x86_pmu_stop(event, PERF_EF_UPDATE);
1387
1388 for (i = 0; i < cpuc->n_events; i++) {
1389 if (event == cpuc->event_list[i])
1390 break;
1391 }
1392
1393 if (WARN_ON_ONCE(i == cpuc->n_events)) /* called ->del() without ->add() ? */
1394 return;
1395
1396 /* If we have a newly added event; make sure to decrease n_added. */
1397 if (i >= cpuc->n_events - cpuc->n_added)
1398 --cpuc->n_added;
1399
1400 if (x86_pmu.put_event_constraints)
1401 x86_pmu.put_event_constraints(cpuc, event);
1402
1403 /* Delete the array entry. */
1404 while (++i < cpuc->n_events) {
1405 cpuc->event_list[i-1] = cpuc->event_list[i];
1406 cpuc->event_constraint[i-1] = cpuc->event_constraint[i];
1407 }
1408 --cpuc->n_events;
1409
1410 perf_event_update_userpage(event);
1411
1412 do_del:
1413 if (x86_pmu.del) {
1414 /*
1415 * This is after x86_pmu_stop(); so we disable LBRs after any
1416 * event can need them etc..
1417 */
1418 x86_pmu.del(event);
1419 }
1420 }
1421
1422 int x86_pmu_handle_irq(struct pt_regs *regs)
1423 {
1424 struct perf_sample_data data;
1425 struct cpu_hw_events *cpuc;
1426 struct perf_event *event;
1427 int idx, handled = 0;
1428 u64 val;
1429
1430 cpuc = this_cpu_ptr(&cpu_hw_events);
1431
1432 /*
1433 * Some chipsets need to unmask the LVTPC in a particular spot
1434 * inside the nmi handler. As a result, the unmasking was pushed
1435 * into all the nmi handlers.
1436 *
1437 * This generic handler doesn't seem to have any issues where the
1438 * unmasking occurs so it was left at the top.
1439 */
1440 apic_write(APIC_LVTPC, APIC_DM_NMI);
1441
1442 for (idx = 0; idx < x86_pmu.num_counters; idx++) {
1443 if (!test_bit(idx, cpuc->active_mask)) {
1444 /*
1445 * Though we deactivated the counter some cpus
1446 * might still deliver spurious interrupts still
1447 * in flight. Catch them:
1448 */
1449 if (__test_and_clear_bit(idx, cpuc->running))
1450 handled++;
1451 continue;
1452 }
1453
1454 event = cpuc->events[idx];
1455
1456 val = x86_perf_event_update(event);
1457 if (val & (1ULL << (x86_pmu.cntval_bits - 1)))
1458 continue;
1459
1460 /*
1461 * event overflow
1462 */
1463 handled++;
1464 perf_sample_data_init(&data, 0, event->hw.last_period);
1465
1466 if (!x86_perf_event_set_period(event))
1467 continue;
1468
1469 if (perf_event_overflow(event, &data, regs))
1470 x86_pmu_stop(event, 0);
1471 }
1472
1473 if (handled)
1474 inc_irq_stat(apic_perf_irqs);
1475
1476 return handled;
1477 }
1478
1479 void perf_events_lapic_init(void)
1480 {
1481 if (!x86_pmu.apic || !x86_pmu_initialized())
1482 return;
1483
1484 /*
1485 * Always use NMI for PMU
1486 */
1487 apic_write(APIC_LVTPC, APIC_DM_NMI);
1488 }
1489
1490 static int
1491 perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs)
1492 {
1493 u64 start_clock;
1494 u64 finish_clock;
1495 int ret;
1496
1497 /*
1498 * All PMUs/events that share this PMI handler should make sure to
1499 * increment active_events for their events.
1500 */
1501 if (!atomic_read(&active_events))
1502 return NMI_DONE;
1503
1504 start_clock = sched_clock();
1505 ret = x86_pmu.handle_irq(regs);
1506 finish_clock = sched_clock();
1507
1508 perf_sample_event_took(finish_clock - start_clock);
1509
1510 return ret;
1511 }
1512 NOKPROBE_SYMBOL(perf_event_nmi_handler);
1513
1514 struct event_constraint emptyconstraint;
1515 struct event_constraint unconstrained;
1516
1517 static int x86_pmu_prepare_cpu(unsigned int cpu)
1518 {
1519 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
1520 int i;
1521
1522 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++)
1523 cpuc->kfree_on_online[i] = NULL;
1524 if (x86_pmu.cpu_prepare)
1525 return x86_pmu.cpu_prepare(cpu);
1526 return 0;
1527 }
1528
1529 static int x86_pmu_dead_cpu(unsigned int cpu)
1530 {
1531 if (x86_pmu.cpu_dead)
1532 x86_pmu.cpu_dead(cpu);
1533 return 0;
1534 }
1535
1536 static int x86_pmu_online_cpu(unsigned int cpu)
1537 {
1538 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
1539 int i;
1540
1541 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++) {
1542 kfree(cpuc->kfree_on_online[i]);
1543 cpuc->kfree_on_online[i] = NULL;
1544 }
1545 return 0;
1546 }
1547
1548 static int x86_pmu_starting_cpu(unsigned int cpu)
1549 {
1550 if (x86_pmu.cpu_starting)
1551 x86_pmu.cpu_starting(cpu);
1552 return 0;
1553 }
1554
1555 static int x86_pmu_dying_cpu(unsigned int cpu)
1556 {
1557 if (x86_pmu.cpu_dying)
1558 x86_pmu.cpu_dying(cpu);
1559 return 0;
1560 }
1561
1562 static void __init pmu_check_apic(void)
1563 {
1564 if (boot_cpu_has(X86_FEATURE_APIC))
1565 return;
1566
1567 x86_pmu.apic = 0;
1568 pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n");
1569 pr_info("no hardware sampling interrupt available.\n");
1570
1571 /*
1572 * If we have a PMU initialized but no APIC
1573 * interrupts, we cannot sample hardware
1574 * events (user-space has to fall back and
1575 * sample via a hrtimer based software event):
1576 */
1577 pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT;
1578
1579 }
1580
1581 static struct attribute_group x86_pmu_format_group = {
1582 .name = "format",
1583 .attrs = NULL,
1584 };
1585
1586 /*
1587 * Remove all undefined events (x86_pmu.event_map(id) == 0)
1588 * out of events_attr attributes.
1589 */
1590 static void __init filter_events(struct attribute **attrs)
1591 {
1592 struct device_attribute *d;
1593 struct perf_pmu_events_attr *pmu_attr;
1594 int offset = 0;
1595 int i, j;
1596
1597 for (i = 0; attrs[i]; i++) {
1598 d = (struct device_attribute *)attrs[i];
1599 pmu_attr = container_of(d, struct perf_pmu_events_attr, attr);
1600 /* str trumps id */
1601 if (pmu_attr->event_str)
1602 continue;
1603 if (x86_pmu.event_map(i + offset))
1604 continue;
1605
1606 for (j = i; attrs[j]; j++)
1607 attrs[j] = attrs[j + 1];
1608
1609 /* Check the shifted attr. */
1610 i--;
1611
1612 /*
1613 * event_map() is index based, the attrs array is organized
1614 * by increasing event index. If we shift the events, then
1615 * we need to compensate for the event_map(), otherwise
1616 * we are looking up the wrong event in the map
1617 */
1618 offset++;
1619 }
1620 }
1621
1622 /* Merge two pointer arrays */
1623 __init struct attribute **merge_attr(struct attribute **a, struct attribute **b)
1624 {
1625 struct attribute **new;
1626 int j, i;
1627
1628 for (j = 0; a[j]; j++)
1629 ;
1630 for (i = 0; b[i]; i++)
1631 j++;
1632 j++;
1633
1634 new = kmalloc(sizeof(struct attribute *) * j, GFP_KERNEL);
1635 if (!new)
1636 return NULL;
1637
1638 j = 0;
1639 for (i = 0; a[i]; i++)
1640 new[j++] = a[i];
1641 for (i = 0; b[i]; i++)
1642 new[j++] = b[i];
1643 new[j] = NULL;
1644
1645 return new;
1646 }
1647
1648 ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr, char *page)
1649 {
1650 struct perf_pmu_events_attr *pmu_attr = \
1651 container_of(attr, struct perf_pmu_events_attr, attr);
1652 u64 config = x86_pmu.event_map(pmu_attr->id);
1653
1654 /* string trumps id */
1655 if (pmu_attr->event_str)
1656 return sprintf(page, "%s", pmu_attr->event_str);
1657
1658 return x86_pmu.events_sysfs_show(page, config);
1659 }
1660 EXPORT_SYMBOL_GPL(events_sysfs_show);
1661
1662 ssize_t events_ht_sysfs_show(struct device *dev, struct device_attribute *attr,
1663 char *page)
1664 {
1665 struct perf_pmu_events_ht_attr *pmu_attr =
1666 container_of(attr, struct perf_pmu_events_ht_attr, attr);
1667
1668 /*
1669 * Report conditional events depending on Hyper-Threading.
1670 *
1671 * This is overly conservative as usually the HT special
1672 * handling is not needed if the other CPU thread is idle.
1673 *
1674 * Note this does not (and cannot) handle the case when thread
1675 * siblings are invisible, for example with virtualization
1676 * if they are owned by some other guest. The user tool
1677 * has to re-read when a thread sibling gets onlined later.
1678 */
1679 return sprintf(page, "%s",
1680 topology_max_smt_threads() > 1 ?
1681 pmu_attr->event_str_ht :
1682 pmu_attr->event_str_noht);
1683 }
1684
1685 EVENT_ATTR(cpu-cycles, CPU_CYCLES );
1686 EVENT_ATTR(instructions, INSTRUCTIONS );
1687 EVENT_ATTR(cache-references, CACHE_REFERENCES );
1688 EVENT_ATTR(cache-misses, CACHE_MISSES );
1689 EVENT_ATTR(branch-instructions, BRANCH_INSTRUCTIONS );
1690 EVENT_ATTR(branch-misses, BRANCH_MISSES );
1691 EVENT_ATTR(bus-cycles, BUS_CYCLES );
1692 EVENT_ATTR(stalled-cycles-frontend, STALLED_CYCLES_FRONTEND );
1693 EVENT_ATTR(stalled-cycles-backend, STALLED_CYCLES_BACKEND );
1694 EVENT_ATTR(ref-cycles, REF_CPU_CYCLES );
1695
1696 static struct attribute *empty_attrs;
1697
1698 static struct attribute *events_attr[] = {
1699 EVENT_PTR(CPU_CYCLES),
1700 EVENT_PTR(INSTRUCTIONS),
1701 EVENT_PTR(CACHE_REFERENCES),
1702 EVENT_PTR(CACHE_MISSES),
1703 EVENT_PTR(BRANCH_INSTRUCTIONS),
1704 EVENT_PTR(BRANCH_MISSES),
1705 EVENT_PTR(BUS_CYCLES),
1706 EVENT_PTR(STALLED_CYCLES_FRONTEND),
1707 EVENT_PTR(STALLED_CYCLES_BACKEND),
1708 EVENT_PTR(REF_CPU_CYCLES),
1709 NULL,
1710 };
1711
1712 static struct attribute_group x86_pmu_events_group = {
1713 .name = "events",
1714 .attrs = events_attr,
1715 };
1716
1717 ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event)
1718 {
1719 u64 umask = (config & ARCH_PERFMON_EVENTSEL_UMASK) >> 8;
1720 u64 cmask = (config & ARCH_PERFMON_EVENTSEL_CMASK) >> 24;
1721 bool edge = (config & ARCH_PERFMON_EVENTSEL_EDGE);
1722 bool pc = (config & ARCH_PERFMON_EVENTSEL_PIN_CONTROL);
1723 bool any = (config & ARCH_PERFMON_EVENTSEL_ANY);
1724 bool inv = (config & ARCH_PERFMON_EVENTSEL_INV);
1725 ssize_t ret;
1726
1727 /*
1728 * We have whole page size to spend and just little data
1729 * to write, so we can safely use sprintf.
1730 */
1731 ret = sprintf(page, "event=0x%02llx", event);
1732
1733 if (umask)
1734 ret += sprintf(page + ret, ",umask=0x%02llx", umask);
1735
1736 if (edge)
1737 ret += sprintf(page + ret, ",edge");
1738
1739 if (pc)
1740 ret += sprintf(page + ret, ",pc");
1741
1742 if (any)
1743 ret += sprintf(page + ret, ",any");
1744
1745 if (inv)
1746 ret += sprintf(page + ret, ",inv");
1747
1748 if (cmask)
1749 ret += sprintf(page + ret, ",cmask=0x%02llx", cmask);
1750
1751 ret += sprintf(page + ret, "\n");
1752
1753 return ret;
1754 }
1755
1756 static struct attribute_group x86_pmu_attr_group;
1757 static struct attribute_group x86_pmu_caps_group;
1758
1759 static int __init init_hw_perf_events(void)
1760 {
1761 struct x86_pmu_quirk *quirk;
1762 int err;
1763
1764 pr_info("Performance Events: ");
1765
1766 switch (boot_cpu_data.x86_vendor) {
1767 case X86_VENDOR_INTEL:
1768 err = intel_pmu_init();
1769 break;
1770 case X86_VENDOR_AMD:
1771 err = amd_pmu_init();
1772 break;
1773 default:
1774 err = -ENOTSUPP;
1775 }
1776 if (err != 0) {
1777 pr_cont("no PMU driver, software events only.\n");
1778 return 0;
1779 }
1780
1781 pmu_check_apic();
1782
1783 /* sanity check that the hardware exists or is emulated */
1784 if (!check_hw_exists())
1785 return 0;
1786
1787 pr_cont("%s PMU driver.\n", x86_pmu.name);
1788
1789 x86_pmu.attr_rdpmc = 1; /* enable userspace RDPMC usage by default */
1790
1791 for (quirk = x86_pmu.quirks; quirk; quirk = quirk->next)
1792 quirk->func();
1793
1794 if (!x86_pmu.intel_ctrl)
1795 x86_pmu.intel_ctrl = (1 << x86_pmu.num_counters) - 1;
1796
1797 perf_events_lapic_init();
1798 register_nmi_handler(NMI_LOCAL, perf_event_nmi_handler, 0, "PMI");
1799
1800 unconstrained = (struct event_constraint)
1801 __EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_counters) - 1,
1802 0, x86_pmu.num_counters, 0, 0);
1803
1804 x86_pmu_format_group.attrs = x86_pmu.format_attrs;
1805
1806 if (x86_pmu.caps_attrs) {
1807 struct attribute **tmp;
1808
1809 tmp = merge_attr(x86_pmu_caps_group.attrs, x86_pmu.caps_attrs);
1810 if (!WARN_ON(!tmp))
1811 x86_pmu_caps_group.attrs = tmp;
1812 }
1813
1814 if (x86_pmu.event_attrs)
1815 x86_pmu_events_group.attrs = x86_pmu.event_attrs;
1816
1817 if (!x86_pmu.events_sysfs_show)
1818 x86_pmu_events_group.attrs = &empty_attrs;
1819 else
1820 filter_events(x86_pmu_events_group.attrs);
1821
1822 if (x86_pmu.cpu_events) {
1823 struct attribute **tmp;
1824
1825 tmp = merge_attr(x86_pmu_events_group.attrs, x86_pmu.cpu_events);
1826 if (!WARN_ON(!tmp))
1827 x86_pmu_events_group.attrs = tmp;
1828 }
1829
1830 if (x86_pmu.attrs) {
1831 struct attribute **tmp;
1832
1833 tmp = merge_attr(x86_pmu_attr_group.attrs, x86_pmu.attrs);
1834 if (!WARN_ON(!tmp))
1835 x86_pmu_attr_group.attrs = tmp;
1836 }
1837
1838 pr_info("... version: %d\n", x86_pmu.version);
1839 pr_info("... bit width: %d\n", x86_pmu.cntval_bits);
1840 pr_info("... generic registers: %d\n", x86_pmu.num_counters);
1841 pr_info("... value mask: %016Lx\n", x86_pmu.cntval_mask);
1842 pr_info("... max period: %016Lx\n", x86_pmu.max_period);
1843 pr_info("... fixed-purpose events: %d\n", x86_pmu.num_counters_fixed);
1844 pr_info("... event mask: %016Lx\n", x86_pmu.intel_ctrl);
1845
1846 /*
1847 * Install callbacks. Core will call them for each online
1848 * cpu.
1849 */
1850 err = cpuhp_setup_state(CPUHP_PERF_X86_PREPARE, "perf/x86:prepare",
1851 x86_pmu_prepare_cpu, x86_pmu_dead_cpu);
1852 if (err)
1853 return err;
1854
1855 err = cpuhp_setup_state(CPUHP_AP_PERF_X86_STARTING,
1856 "perf/x86:starting", x86_pmu_starting_cpu,
1857 x86_pmu_dying_cpu);
1858 if (err)
1859 goto out;
1860
1861 err = cpuhp_setup_state(CPUHP_AP_PERF_X86_ONLINE, "perf/x86:online",
1862 x86_pmu_online_cpu, NULL);
1863 if (err)
1864 goto out1;
1865
1866 err = perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW);
1867 if (err)
1868 goto out2;
1869
1870 return 0;
1871
1872 out2:
1873 cpuhp_remove_state(CPUHP_AP_PERF_X86_ONLINE);
1874 out1:
1875 cpuhp_remove_state(CPUHP_AP_PERF_X86_STARTING);
1876 out:
1877 cpuhp_remove_state(CPUHP_PERF_X86_PREPARE);
1878 return err;
1879 }
1880 early_initcall(init_hw_perf_events);
1881
1882 static inline void x86_pmu_read(struct perf_event *event)
1883 {
1884 x86_perf_event_update(event);
1885 }
1886
1887 /*
1888 * Start group events scheduling transaction
1889 * Set the flag to make pmu::enable() not perform the
1890 * schedulability test, it will be performed at commit time
1891 *
1892 * We only support PERF_PMU_TXN_ADD transactions. Save the
1893 * transaction flags but otherwise ignore non-PERF_PMU_TXN_ADD
1894 * transactions.
1895 */
1896 static void x86_pmu_start_txn(struct pmu *pmu, unsigned int txn_flags)
1897 {
1898 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1899
1900 WARN_ON_ONCE(cpuc->txn_flags); /* txn already in flight */
1901
1902 cpuc->txn_flags = txn_flags;
1903 if (txn_flags & ~PERF_PMU_TXN_ADD)
1904 return;
1905
1906 perf_pmu_disable(pmu);
1907 __this_cpu_write(cpu_hw_events.n_txn, 0);
1908 }
1909
1910 /*
1911 * Stop group events scheduling transaction
1912 * Clear the flag and pmu::enable() will perform the
1913 * schedulability test.
1914 */
1915 static void x86_pmu_cancel_txn(struct pmu *pmu)
1916 {
1917 unsigned int txn_flags;
1918 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1919
1920 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */
1921
1922 txn_flags = cpuc->txn_flags;
1923 cpuc->txn_flags = 0;
1924 if (txn_flags & ~PERF_PMU_TXN_ADD)
1925 return;
1926
1927 /*
1928 * Truncate collected array by the number of events added in this
1929 * transaction. See x86_pmu_add() and x86_pmu_*_txn().
1930 */
1931 __this_cpu_sub(cpu_hw_events.n_added, __this_cpu_read(cpu_hw_events.n_txn));
1932 __this_cpu_sub(cpu_hw_events.n_events, __this_cpu_read(cpu_hw_events.n_txn));
1933 perf_pmu_enable(pmu);
1934 }
1935
1936 /*
1937 * Commit group events scheduling transaction
1938 * Perform the group schedulability test as a whole
1939 * Return 0 if success
1940 *
1941 * Does not cancel the transaction on failure; expects the caller to do this.
1942 */
1943 static int x86_pmu_commit_txn(struct pmu *pmu)
1944 {
1945 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1946 int assign[X86_PMC_IDX_MAX];
1947 int n, ret;
1948
1949 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */
1950
1951 if (cpuc->txn_flags & ~PERF_PMU_TXN_ADD) {
1952 cpuc->txn_flags = 0;
1953 return 0;
1954 }
1955
1956 n = cpuc->n_events;
1957
1958 if (!x86_pmu_initialized())
1959 return -EAGAIN;
1960
1961 ret = x86_pmu.schedule_events(cpuc, n, assign);
1962 if (ret)
1963 return ret;
1964
1965 /*
1966 * copy new assignment, now we know it is possible
1967 * will be used by hw_perf_enable()
1968 */
1969 memcpy(cpuc->assign, assign, n*sizeof(int));
1970
1971 cpuc->txn_flags = 0;
1972 perf_pmu_enable(pmu);
1973 return 0;
1974 }
1975 /*
1976 * a fake_cpuc is used to validate event groups. Due to
1977 * the extra reg logic, we need to also allocate a fake
1978 * per_core and per_cpu structure. Otherwise, group events
1979 * using extra reg may conflict without the kernel being
1980 * able to catch this when the last event gets added to
1981 * the group.
1982 */
1983 static void free_fake_cpuc(struct cpu_hw_events *cpuc)
1984 {
1985 kfree(cpuc->shared_regs);
1986 kfree(cpuc);
1987 }
1988
1989 static struct cpu_hw_events *allocate_fake_cpuc(void)
1990 {
1991 struct cpu_hw_events *cpuc;
1992 int cpu = raw_smp_processor_id();
1993
1994 cpuc = kzalloc(sizeof(*cpuc), GFP_KERNEL);
1995 if (!cpuc)
1996 return ERR_PTR(-ENOMEM);
1997
1998 /* only needed, if we have extra_regs */
1999 if (x86_pmu.extra_regs) {
2000 cpuc->shared_regs = allocate_shared_regs(cpu);
2001 if (!cpuc->shared_regs)
2002 goto error;
2003 }
2004 cpuc->is_fake = 1;
2005 return cpuc;
2006 error:
2007 free_fake_cpuc(cpuc);
2008 return ERR_PTR(-ENOMEM);
2009 }
2010
2011 /*
2012 * validate that we can schedule this event
2013 */
2014 static int validate_event(struct perf_event *event)
2015 {
2016 struct cpu_hw_events *fake_cpuc;
2017 struct event_constraint *c;
2018 int ret = 0;
2019
2020 fake_cpuc = allocate_fake_cpuc();
2021 if (IS_ERR(fake_cpuc))
2022 return PTR_ERR(fake_cpuc);
2023
2024 c = x86_pmu.get_event_constraints(fake_cpuc, -1, event);
2025
2026 if (!c || !c->weight)
2027 ret = -EINVAL;
2028
2029 if (x86_pmu.put_event_constraints)
2030 x86_pmu.put_event_constraints(fake_cpuc, event);
2031
2032 free_fake_cpuc(fake_cpuc);
2033
2034 return ret;
2035 }
2036
2037 /*
2038 * validate a single event group
2039 *
2040 * validation include:
2041 * - check events are compatible which each other
2042 * - events do not compete for the same counter
2043 * - number of events <= number of counters
2044 *
2045 * validation ensures the group can be loaded onto the
2046 * PMU if it was the only group available.
2047 */
2048 static int validate_group(struct perf_event *event)
2049 {
2050 struct perf_event *leader = event->group_leader;
2051 struct cpu_hw_events *fake_cpuc;
2052 int ret = -EINVAL, n;
2053
2054 fake_cpuc = allocate_fake_cpuc();
2055 if (IS_ERR(fake_cpuc))
2056 return PTR_ERR(fake_cpuc);
2057 /*
2058 * the event is not yet connected with its
2059 * siblings therefore we must first collect
2060 * existing siblings, then add the new event
2061 * before we can simulate the scheduling
2062 */
2063 n = collect_events(fake_cpuc, leader, true);
2064 if (n < 0)
2065 goto out;
2066
2067 fake_cpuc->n_events = n;
2068 n = collect_events(fake_cpuc, event, false);
2069 if (n < 0)
2070 goto out;
2071
2072 fake_cpuc->n_events = n;
2073
2074 ret = x86_pmu.schedule_events(fake_cpuc, n, NULL);
2075
2076 out:
2077 free_fake_cpuc(fake_cpuc);
2078 return ret;
2079 }
2080
2081 static int x86_pmu_event_init(struct perf_event *event)
2082 {
2083 struct pmu *tmp;
2084 int err;
2085
2086 switch (event->attr.type) {
2087 case PERF_TYPE_RAW:
2088 case PERF_TYPE_HARDWARE:
2089 case PERF_TYPE_HW_CACHE:
2090 break;
2091
2092 default:
2093 return -ENOENT;
2094 }
2095
2096 err = __x86_pmu_event_init(event);
2097 if (!err) {
2098 /*
2099 * we temporarily connect event to its pmu
2100 * such that validate_group() can classify
2101 * it as an x86 event using is_x86_event()
2102 */
2103 tmp = event->pmu;
2104 event->pmu = &pmu;
2105
2106 if (event->group_leader != event)
2107 err = validate_group(event);
2108 else
2109 err = validate_event(event);
2110
2111 event->pmu = tmp;
2112 }
2113 if (err) {
2114 if (event->destroy)
2115 event->destroy(event);
2116 }
2117
2118 if (READ_ONCE(x86_pmu.attr_rdpmc))
2119 event->hw.flags |= PERF_X86_EVENT_RDPMC_ALLOWED;
2120
2121 return err;
2122 }
2123
2124 static void refresh_pce(void *ignored)
2125 {
2126 load_mm_cr4(this_cpu_read(cpu_tlbstate.loaded_mm));
2127 }
2128
2129 static void x86_pmu_event_mapped(struct perf_event *event, struct mm_struct *mm)
2130 {
2131 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED))
2132 return;
2133
2134 /*
2135 * This function relies on not being called concurrently in two
2136 * tasks in the same mm. Otherwise one task could observe
2137 * perf_rdpmc_allowed > 1 and return all the way back to
2138 * userspace with CR4.PCE clear while another task is still
2139 * doing on_each_cpu_mask() to propagate CR4.PCE.
2140 *
2141 * For now, this can't happen because all callers hold mmap_sem
2142 * for write. If this changes, we'll need a different solution.
2143 */
2144 lockdep_assert_held_exclusive(&mm->mmap_sem);
2145
2146 if (atomic_inc_return(&mm->context.perf_rdpmc_allowed) == 1)
2147 on_each_cpu_mask(mm_cpumask(mm), refresh_pce, NULL, 1);
2148 }
2149
2150 static void x86_pmu_event_unmapped(struct perf_event *event, struct mm_struct *mm)
2151 {
2152
2153 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED))
2154 return;
2155
2156 if (atomic_dec_and_test(&mm->context.perf_rdpmc_allowed))
2157 on_each_cpu_mask(mm_cpumask(mm), refresh_pce, NULL, 1);
2158 }
2159
2160 static int x86_pmu_event_idx(struct perf_event *event)
2161 {
2162 int idx = event->hw.idx;
2163
2164 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED))
2165 return 0;
2166
2167 if (x86_pmu.num_counters_fixed && idx >= INTEL_PMC_IDX_FIXED) {
2168 idx -= INTEL_PMC_IDX_FIXED;
2169 idx |= 1 << 30;
2170 }
2171
2172 return idx + 1;
2173 }
2174
2175 static ssize_t get_attr_rdpmc(struct device *cdev,
2176 struct device_attribute *attr,
2177 char *buf)
2178 {
2179 return snprintf(buf, 40, "%d\n", x86_pmu.attr_rdpmc);
2180 }
2181
2182 static ssize_t set_attr_rdpmc(struct device *cdev,
2183 struct device_attribute *attr,
2184 const char *buf, size_t count)
2185 {
2186 unsigned long val;
2187 ssize_t ret;
2188
2189 ret = kstrtoul(buf, 0, &val);
2190 if (ret)
2191 return ret;
2192
2193 if (val > 2)
2194 return -EINVAL;
2195
2196 if (x86_pmu.attr_rdpmc_broken)
2197 return -ENOTSUPP;
2198
2199 if ((val == 2) != (x86_pmu.attr_rdpmc == 2)) {
2200 /*
2201 * Changing into or out of always available, aka
2202 * perf-event-bypassing mode. This path is extremely slow,
2203 * but only root can trigger it, so it's okay.
2204 */
2205 if (val == 2)
2206 static_key_slow_inc(&rdpmc_always_available);
2207 else
2208 static_key_slow_dec(&rdpmc_always_available);
2209 on_each_cpu(refresh_pce, NULL, 1);
2210 }
2211
2212 x86_pmu.attr_rdpmc = val;
2213
2214 return count;
2215 }
2216
2217 static DEVICE_ATTR(rdpmc, S_IRUSR | S_IWUSR, get_attr_rdpmc, set_attr_rdpmc);
2218
2219 static struct attribute *x86_pmu_attrs[] = {
2220 &dev_attr_rdpmc.attr,
2221 NULL,
2222 };
2223
2224 static struct attribute_group x86_pmu_attr_group = {
2225 .attrs = x86_pmu_attrs,
2226 };
2227
2228 static ssize_t max_precise_show(struct device *cdev,
2229 struct device_attribute *attr,
2230 char *buf)
2231 {
2232 return snprintf(buf, PAGE_SIZE, "%d\n", x86_pmu_max_precise());
2233 }
2234
2235 static DEVICE_ATTR_RO(max_precise);
2236
2237 static struct attribute *x86_pmu_caps_attrs[] = {
2238 &dev_attr_max_precise.attr,
2239 NULL
2240 };
2241
2242 static struct attribute_group x86_pmu_caps_group = {
2243 .name = "caps",
2244 .attrs = x86_pmu_caps_attrs,
2245 };
2246
2247 static const struct attribute_group *x86_pmu_attr_groups[] = {
2248 &x86_pmu_attr_group,
2249 &x86_pmu_format_group,
2250 &x86_pmu_events_group,
2251 &x86_pmu_caps_group,
2252 NULL,
2253 };
2254
2255 static void x86_pmu_sched_task(struct perf_event_context *ctx, bool sched_in)
2256 {
2257 if (x86_pmu.sched_task)
2258 x86_pmu.sched_task(ctx, sched_in);
2259 }
2260
2261 void perf_check_microcode(void)
2262 {
2263 if (x86_pmu.check_microcode)
2264 x86_pmu.check_microcode();
2265 }
2266
2267 static struct pmu pmu = {
2268 .pmu_enable = x86_pmu_enable,
2269 .pmu_disable = x86_pmu_disable,
2270
2271 .attr_groups = x86_pmu_attr_groups,
2272
2273 .event_init = x86_pmu_event_init,
2274
2275 .event_mapped = x86_pmu_event_mapped,
2276 .event_unmapped = x86_pmu_event_unmapped,
2277
2278 .add = x86_pmu_add,
2279 .del = x86_pmu_del,
2280 .start = x86_pmu_start,
2281 .stop = x86_pmu_stop,
2282 .read = x86_pmu_read,
2283
2284 .start_txn = x86_pmu_start_txn,
2285 .cancel_txn = x86_pmu_cancel_txn,
2286 .commit_txn = x86_pmu_commit_txn,
2287
2288 .event_idx = x86_pmu_event_idx,
2289 .sched_task = x86_pmu_sched_task,
2290 .task_ctx_size = sizeof(struct x86_perf_task_context),
2291 };
2292
2293 void arch_perf_update_userpage(struct perf_event *event,
2294 struct perf_event_mmap_page *userpg, u64 now)
2295 {
2296 struct cyc2ns_data data;
2297 u64 offset;
2298
2299 userpg->cap_user_time = 0;
2300 userpg->cap_user_time_zero = 0;
2301 userpg->cap_user_rdpmc =
2302 !!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED);
2303 userpg->pmc_width = x86_pmu.cntval_bits;
2304
2305 if (!using_native_sched_clock() || !sched_clock_stable())
2306 return;
2307
2308 cyc2ns_read_begin(&data);
2309
2310 offset = data.cyc2ns_offset + __sched_clock_offset;
2311
2312 /*
2313 * Internal timekeeping for enabled/running/stopped times
2314 * is always in the local_clock domain.
2315 */
2316 userpg->cap_user_time = 1;
2317 userpg->time_mult = data.cyc2ns_mul;
2318 userpg->time_shift = data.cyc2ns_shift;
2319 userpg->time_offset = offset - now;
2320
2321 /*
2322 * cap_user_time_zero doesn't make sense when we're using a different
2323 * time base for the records.
2324 */
2325 if (!event->attr.use_clockid) {
2326 userpg->cap_user_time_zero = 1;
2327 userpg->time_zero = offset;
2328 }
2329
2330 cyc2ns_read_end();
2331 }
2332
2333 void
2334 perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs)
2335 {
2336 struct unwind_state state;
2337 unsigned long addr;
2338
2339 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
2340 /* TODO: We don't support guest os callchain now */
2341 return;
2342 }
2343
2344 if (perf_callchain_store(entry, regs->ip))
2345 return;
2346
2347 for (unwind_start(&state, current, regs, NULL); !unwind_done(&state);
2348 unwind_next_frame(&state)) {
2349 addr = unwind_get_return_address(&state);
2350 if (!addr || perf_callchain_store(entry, addr))
2351 return;
2352 }
2353 }
2354
2355 static inline int
2356 valid_user_frame(const void __user *fp, unsigned long size)
2357 {
2358 return (__range_not_ok(fp, size, TASK_SIZE) == 0);
2359 }
2360
2361 static unsigned long get_segment_base(unsigned int segment)
2362 {
2363 struct desc_struct *desc;
2364 unsigned int idx = segment >> 3;
2365
2366 if ((segment & SEGMENT_TI_MASK) == SEGMENT_LDT) {
2367 #ifdef CONFIG_MODIFY_LDT_SYSCALL
2368 struct ldt_struct *ldt;
2369
2370 /* IRQs are off, so this synchronizes with smp_store_release */
2371 ldt = READ_ONCE(current->active_mm->context.ldt);
2372 if (!ldt || idx >= ldt->nr_entries)
2373 return 0;
2374
2375 desc = &ldt->entries[idx];
2376 #else
2377 return 0;
2378 #endif
2379 } else {
2380 if (idx >= GDT_ENTRIES)
2381 return 0;
2382
2383 desc = raw_cpu_ptr(gdt_page.gdt) + idx;
2384 }
2385
2386 return get_desc_base(desc);
2387 }
2388
2389 #ifdef CONFIG_IA32_EMULATION
2390
2391 #include <asm/compat.h>
2392
2393 static inline int
2394 perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry)
2395 {
2396 /* 32-bit process in 64-bit kernel. */
2397 unsigned long ss_base, cs_base;
2398 struct stack_frame_ia32 frame;
2399 const void __user *fp;
2400
2401 if (!test_thread_flag(TIF_IA32))
2402 return 0;
2403
2404 cs_base = get_segment_base(regs->cs);
2405 ss_base = get_segment_base(regs->ss);
2406
2407 fp = compat_ptr(ss_base + regs->bp);
2408 pagefault_disable();
2409 while (entry->nr < entry->max_stack) {
2410 unsigned long bytes;
2411 frame.next_frame = 0;
2412 frame.return_address = 0;
2413
2414 if (!valid_user_frame(fp, sizeof(frame)))
2415 break;
2416
2417 bytes = __copy_from_user_nmi(&frame.next_frame, fp, 4);
2418 if (bytes != 0)
2419 break;
2420 bytes = __copy_from_user_nmi(&frame.return_address, fp+4, 4);
2421 if (bytes != 0)
2422 break;
2423
2424 perf_callchain_store(entry, cs_base + frame.return_address);
2425 fp = compat_ptr(ss_base + frame.next_frame);
2426 }
2427 pagefault_enable();
2428 return 1;
2429 }
2430 #else
2431 static inline int
2432 perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry)
2433 {
2434 return 0;
2435 }
2436 #endif
2437
2438 void
2439 perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs)
2440 {
2441 struct stack_frame frame;
2442 const unsigned long __user *fp;
2443
2444 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
2445 /* TODO: We don't support guest os callchain now */
2446 return;
2447 }
2448
2449 /*
2450 * We don't know what to do with VM86 stacks.. ignore them for now.
2451 */
2452 if (regs->flags & (X86_VM_MASK | PERF_EFLAGS_VM))
2453 return;
2454
2455 fp = (unsigned long __user *)regs->bp;
2456
2457 perf_callchain_store(entry, regs->ip);
2458
2459 if (!current->mm)
2460 return;
2461
2462 if (perf_callchain_user32(regs, entry))
2463 return;
2464
2465 pagefault_disable();
2466 while (entry->nr < entry->max_stack) {
2467 unsigned long bytes;
2468
2469 frame.next_frame = NULL;
2470 frame.return_address = 0;
2471
2472 if (!valid_user_frame(fp, sizeof(frame)))
2473 break;
2474
2475 bytes = __copy_from_user_nmi(&frame.next_frame, fp, sizeof(*fp));
2476 if (bytes != 0)
2477 break;
2478 bytes = __copy_from_user_nmi(&frame.return_address, fp + 1, sizeof(*fp));
2479 if (bytes != 0)
2480 break;
2481
2482 perf_callchain_store(entry, frame.return_address);
2483 fp = (void __user *)frame.next_frame;
2484 }
2485 pagefault_enable();
2486 }
2487
2488 /*
2489 * Deal with code segment offsets for the various execution modes:
2490 *
2491 * VM86 - the good olde 16 bit days, where the linear address is
2492 * 20 bits and we use regs->ip + 0x10 * regs->cs.
2493 *
2494 * IA32 - Where we need to look at GDT/LDT segment descriptor tables
2495 * to figure out what the 32bit base address is.
2496 *
2497 * X32 - has TIF_X32 set, but is running in x86_64
2498 *
2499 * X86_64 - CS,DS,SS,ES are all zero based.
2500 */
2501 static unsigned long code_segment_base(struct pt_regs *regs)
2502 {
2503 /*
2504 * For IA32 we look at the GDT/LDT segment base to convert the
2505 * effective IP to a linear address.
2506 */
2507
2508 #ifdef CONFIG_X86_32
2509 /*
2510 * If we are in VM86 mode, add the segment offset to convert to a
2511 * linear address.
2512 */
2513 if (regs->flags & X86_VM_MASK)
2514 return 0x10 * regs->cs;
2515
2516 if (user_mode(regs) && regs->cs != __USER_CS)
2517 return get_segment_base(regs->cs);
2518 #else
2519 if (user_mode(regs) && !user_64bit_mode(regs) &&
2520 regs->cs != __USER32_CS)
2521 return get_segment_base(regs->cs);
2522 #endif
2523 return 0;
2524 }
2525
2526 unsigned long perf_instruction_pointer(struct pt_regs *regs)
2527 {
2528 if (perf_guest_cbs && perf_guest_cbs->is_in_guest())
2529 return perf_guest_cbs->get_guest_ip();
2530
2531 return regs->ip + code_segment_base(regs);
2532 }
2533
2534 unsigned long perf_misc_flags(struct pt_regs *regs)
2535 {
2536 int misc = 0;
2537
2538 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
2539 if (perf_guest_cbs->is_user_mode())
2540 misc |= PERF_RECORD_MISC_GUEST_USER;
2541 else
2542 misc |= PERF_RECORD_MISC_GUEST_KERNEL;
2543 } else {
2544 if (user_mode(regs))
2545 misc |= PERF_RECORD_MISC_USER;
2546 else
2547 misc |= PERF_RECORD_MISC_KERNEL;
2548 }
2549
2550 if (regs->flags & PERF_EFLAGS_EXACT)
2551 misc |= PERF_RECORD_MISC_EXACT_IP;
2552
2553 return misc;
2554 }
2555
2556 void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap)
2557 {
2558 cap->version = x86_pmu.version;
2559 cap->num_counters_gp = x86_pmu.num_counters;
2560 cap->num_counters_fixed = x86_pmu.num_counters_fixed;
2561 cap->bit_width_gp = x86_pmu.cntval_bits;
2562 cap->bit_width_fixed = x86_pmu.cntval_bits;
2563 cap->events_mask = (unsigned int)x86_pmu.events_maskl;
2564 cap->events_mask_len = x86_pmu.events_mask_len;
2565 }
2566 EXPORT_SYMBOL_GPL(perf_get_x86_pmu_capability);
Linux with added FPC-III support