?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
iwlwifi: mvm: fix version check for GEO_TX_POWER_LIMIT support
[linux/fpc-iii.git] / arch / x86 / events / core.c
blobc9625bff43282c8a4a125458ef0d13ca005e3c50
1 /*
2 * Performance events x86 architecture code
3 *
4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2009 Jaswinder Singh Rajput
7 * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
8 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra
9 * Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
10 * Copyright (C) 2009 Google, Inc., Stephane Eranian
11 *
12 * For licencing details see kernel-base/COPYING
13 */
14
15 #include <linux/perf_event.h>
16 #include <linux/capability.h>
17 #include <linux/notifier.h>
18 #include <linux/hardirq.h>
19 #include <linux/kprobes.h>
20 #include <linux/export.h>
21 #include <linux/init.h>
22 #include <linux/kdebug.h>
23 #include <linux/sched/mm.h>
24 #include <linux/sched/clock.h>
25 #include <linux/uaccess.h>
26 #include <linux/slab.h>
27 #include <linux/cpu.h>
28 #include <linux/bitops.h>
29 #include <linux/device.h>
30 #include <linux/nospec.h>
31
32 #include <asm/apic.h>
33 #include <asm/stacktrace.h>
34 #include <asm/nmi.h>
35 #include <asm/smp.h>
36 #include <asm/alternative.h>
37 #include <asm/mmu_context.h>
38 #include <asm/tlbflush.h>
39 #include <asm/timer.h>
40 #include <asm/desc.h>
41 #include <asm/ldt.h>
42 #include <asm/unwind.h>
43
44 #include "perf_event.h"
45
46 struct x86_pmu x86_pmu __read_mostly;
47
48 DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
49 .enabled = 1,
50 };
51
52 DEFINE_STATIC_KEY_FALSE(rdpmc_always_available_key);
53
54 u64 __read_mostly hw_cache_event_ids
55 [PERF_COUNT_HW_CACHE_MAX]
56 [PERF_COUNT_HW_CACHE_OP_MAX]
57 [PERF_COUNT_HW_CACHE_RESULT_MAX];
58 u64 __read_mostly hw_cache_extra_regs
59 [PERF_COUNT_HW_CACHE_MAX]
60 [PERF_COUNT_HW_CACHE_OP_MAX]
61 [PERF_COUNT_HW_CACHE_RESULT_MAX];
62
63 /*
64 * Propagate event elapsed time into the generic event.
65 * Can only be executed on the CPU where the event is active.
66 * Returns the delta events processed.
67 */
68 u64 x86_perf_event_update(struct perf_event *event)
69 {
70 struct hw_perf_event *hwc = &event->hw;
71 int shift = 64 - x86_pmu.cntval_bits;
72 u64 prev_raw_count, new_raw_count;
73 int idx = hwc->idx;
74 u64 delta;
75
76 if (idx == INTEL_PMC_IDX_FIXED_BTS)
77 return 0;
78
79 /*
80 * Careful: an NMI might modify the previous event value.
81 *
82 * Our tactic to handle this is to first atomically read and
83 * exchange a new raw count - then add that new-prev delta
84 * count to the generic event atomically:
85 */
86 again:
87 prev_raw_count = local64_read(&hwc->prev_count);
88 rdpmcl(hwc->event_base_rdpmc, new_raw_count);
89
90 if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
91 new_raw_count) != prev_raw_count)
92 goto again;
93
94 /*
95 * Now we have the new raw value and have updated the prev
96 * timestamp already. We can now calculate the elapsed delta
97 * (event-)time and add that to the generic event.
98 *
99 * Careful, not all hw sign-extends above the physical width
100 * of the count.
101 */
102 delta = (new_raw_count << shift) - (prev_raw_count << shift);
103 delta >>= shift;
104
105 local64_add(delta, &event->count);
106 local64_sub(delta, &hwc->period_left);
107
108 return new_raw_count;
109 }
110
111 /*
112 * Find and validate any extra registers to set up.
113 */
114 static int x86_pmu_extra_regs(u64 config, struct perf_event *event)
115 {
116 struct hw_perf_event_extra *reg;
117 struct extra_reg *er;
118
119 reg = &event->hw.extra_reg;
120
121 if (!x86_pmu.extra_regs)
122 return 0;
123
124 for (er = x86_pmu.extra_regs; er->msr; er++) {
125 if (er->event != (config & er->config_mask))
126 continue;
127 if (event->attr.config1 & ~er->valid_mask)
128 return -EINVAL;
129 /* Check if the extra msrs can be safely accessed*/
130 if (!er->extra_msr_access)
131 return -ENXIO;
132
133 reg->idx = er->idx;
134 reg->config = event->attr.config1;
135 reg->reg = er->msr;
136 break;
137 }
138 return 0;
139 }
140
141 static atomic_t active_events;
142 static atomic_t pmc_refcount;
143 static DEFINE_MUTEX(pmc_reserve_mutex);
144
145 #ifdef CONFIG_X86_LOCAL_APIC
146
147 static bool reserve_pmc_hardware(void)
148 {
149 int i;
150
151 for (i = 0; i < x86_pmu.num_counters; i++) {
152 if (!reserve_perfctr_nmi(x86_pmu_event_addr(i)))
153 goto perfctr_fail;
154 }
155
156 for (i = 0; i < x86_pmu.num_counters; i++) {
157 if (!reserve_evntsel_nmi(x86_pmu_config_addr(i)))
158 goto eventsel_fail;
159 }
160
161 return true;
162
163 eventsel_fail:
164 for (i--; i >= 0; i--)
165 release_evntsel_nmi(x86_pmu_config_addr(i));
166
167 i = x86_pmu.num_counters;
168
169 perfctr_fail:
170 for (i--; i >= 0; i--)
171 release_perfctr_nmi(x86_pmu_event_addr(i));
172
173 return false;
174 }
175
176 static void release_pmc_hardware(void)
177 {
178 int i;
179
180 for (i = 0; i < x86_pmu.num_counters; i++) {
181 release_perfctr_nmi(x86_pmu_event_addr(i));
182 release_evntsel_nmi(x86_pmu_config_addr(i));
183 }
184 }
185
186 #else
187
188 static bool reserve_pmc_hardware(void) { return true; }
189 static void release_pmc_hardware(void) {}
190
191 #endif
192
193 static bool check_hw_exists(void)
194 {
195 u64 val, val_fail = -1, val_new= ~0;
196 int i, reg, reg_fail = -1, ret = 0;
197 int bios_fail = 0;
198 int reg_safe = -1;
199
200 /*
201 * Check to see if the BIOS enabled any of the counters, if so
202 * complain and bail.
203 */
204 for (i = 0; i < x86_pmu.num_counters; i++) {
205 reg = x86_pmu_config_addr(i);
206 ret = rdmsrl_safe(reg, &val);
207 if (ret)
208 goto msr_fail;
209 if (val & ARCH_PERFMON_EVENTSEL_ENABLE) {
210 bios_fail = 1;
211 val_fail = val;
212 reg_fail = reg;
213 } else {
214 reg_safe = i;
215 }
216 }
217
218 if (x86_pmu.num_counters_fixed) {
219 reg = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
220 ret = rdmsrl_safe(reg, &val);
221 if (ret)
222 goto msr_fail;
223 for (i = 0; i < x86_pmu.num_counters_fixed; i++) {
224 if (val & (0x03 << i*4)) {
225 bios_fail = 1;
226 val_fail = val;
227 reg_fail = reg;
228 }
229 }
230 }
231
232 /*
233 * If all the counters are enabled, the below test will always
234 * fail. The tools will also become useless in this scenario.
235 * Just fail and disable the hardware counters.
236 */
237
238 if (reg_safe == -1) {
239 reg = reg_safe;
240 goto msr_fail;
241 }
242
243 /*
244 * Read the current value, change it and read it back to see if it
245 * matches, this is needed to detect certain hardware emulators
246 * (qemu/kvm) that don't trap on the MSR access and always return 0s.
247 */
248 reg = x86_pmu_event_addr(reg_safe);
249 if (rdmsrl_safe(reg, &val))
250 goto msr_fail;
251 val ^= 0xffffUL;
252 ret = wrmsrl_safe(reg, val);
253 ret |= rdmsrl_safe(reg, &val_new);
254 if (ret || val != val_new)
255 goto msr_fail;
256
257 /*
258 * We still allow the PMU driver to operate:
259 */
260 if (bios_fail) {
261 pr_cont("Broken BIOS detected, complain to your hardware vendor.\n");
262 pr_err(FW_BUG "the BIOS has corrupted hw-PMU resources (MSR %x is %Lx)\n",
263 reg_fail, val_fail);
264 }
265
266 return true;
267
268 msr_fail:
269 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
270 pr_cont("PMU not available due to virtualization, using software events only.\n");
271 } else {
272 pr_cont("Broken PMU hardware detected, using software events only.\n");
273 pr_err("Failed to access perfctr msr (MSR %x is %Lx)\n",
274 reg, val_new);
275 }
276
277 return false;
278 }
279
280 static void hw_perf_event_destroy(struct perf_event *event)
281 {
282 x86_release_hardware();
283 atomic_dec(&active_events);
284 }
285
286 void hw_perf_lbr_event_destroy(struct perf_event *event)
287 {
288 hw_perf_event_destroy(event);
289
290 /* undo the lbr/bts event accounting */
291 x86_del_exclusive(x86_lbr_exclusive_lbr);
292 }
293
294 static inline int x86_pmu_initialized(void)
295 {
296 return x86_pmu.handle_irq != NULL;
297 }
298
299 static inline int
300 set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event *event)
301 {
302 struct perf_event_attr *attr = &event->attr;
303 unsigned int cache_type, cache_op, cache_result;
304 u64 config, val;
305
306 config = attr->config;
307
308 cache_type = (config >> 0) & 0xff;
309 if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
310 return -EINVAL;
311 cache_type = array_index_nospec(cache_type, PERF_COUNT_HW_CACHE_MAX);
312
313 cache_op = (config >> 8) & 0xff;
314 if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
315 return -EINVAL;
316 cache_op = array_index_nospec(cache_op, PERF_COUNT_HW_CACHE_OP_MAX);
317
318 cache_result = (config >> 16) & 0xff;
319 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
320 return -EINVAL;
321 cache_result = array_index_nospec(cache_result, PERF_COUNT_HW_CACHE_RESULT_MAX);
322
323 val = hw_cache_event_ids[cache_type][cache_op][cache_result];
324
325 if (val == 0)
326 return -ENOENT;
327
328 if (val == -1)
329 return -EINVAL;
330
331 hwc->config |= val;
332 attr->config1 = hw_cache_extra_regs[cache_type][cache_op][cache_result];
333 return x86_pmu_extra_regs(val, event);
334 }
335
336 int x86_reserve_hardware(void)
337 {
338 int err = 0;
339
340 if (!atomic_inc_not_zero(&pmc_refcount)) {
341 mutex_lock(&pmc_reserve_mutex);
342 if (atomic_read(&pmc_refcount) == 0) {
343 if (!reserve_pmc_hardware())
344 err = -EBUSY;
345 else
346 reserve_ds_buffers();
347 }
348 if (!err)
349 atomic_inc(&pmc_refcount);
350 mutex_unlock(&pmc_reserve_mutex);
351 }
352
353 return err;
354 }
355
356 void x86_release_hardware(void)
357 {
358 if (atomic_dec_and_mutex_lock(&pmc_refcount, &pmc_reserve_mutex)) {
359 release_pmc_hardware();
360 release_ds_buffers();
361 mutex_unlock(&pmc_reserve_mutex);
362 }
363 }
364
365 /*
366 * Check if we can create event of a certain type (that no conflicting events
367 * are present).
368 */
369 int x86_add_exclusive(unsigned int what)
370 {
371 int i;
372
373 /*
374 * When lbr_pt_coexist we allow PT to coexist with either LBR or BTS.
375 * LBR and BTS are still mutually exclusive.
376 */
377 if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt)
378 return 0;
379
380 if (!atomic_inc_not_zero(&x86_pmu.lbr_exclusive[what])) {
381 mutex_lock(&pmc_reserve_mutex);
382 for (i = 0; i < ARRAY_SIZE(x86_pmu.lbr_exclusive); i++) {
383 if (i != what && atomic_read(&x86_pmu.lbr_exclusive[i]))
384 goto fail_unlock;
385 }
386 atomic_inc(&x86_pmu.lbr_exclusive[what]);
387 mutex_unlock(&pmc_reserve_mutex);
388 }
389
390 atomic_inc(&active_events);
391 return 0;
392
393 fail_unlock:
394 mutex_unlock(&pmc_reserve_mutex);
395 return -EBUSY;
396 }
397
398 void x86_del_exclusive(unsigned int what)
399 {
400 if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt)
401 return;
402
403 atomic_dec(&x86_pmu.lbr_exclusive[what]);
404 atomic_dec(&active_events);
405 }
406
407 int x86_setup_perfctr(struct perf_event *event)
408 {
409 struct perf_event_attr *attr = &event->attr;
410 struct hw_perf_event *hwc = &event->hw;
411 u64 config;
412
413 if (!is_sampling_event(event)) {
414 hwc->sample_period = x86_pmu.max_period;
415 hwc->last_period = hwc->sample_period;
416 local64_set(&hwc->period_left, hwc->sample_period);
417 }
418
419 if (attr->type == PERF_TYPE_RAW)
420 return x86_pmu_extra_regs(event->attr.config, event);
421
422 if (attr->type == PERF_TYPE_HW_CACHE)
423 return set_ext_hw_attr(hwc, event);
424
425 if (attr->config >= x86_pmu.max_events)
426 return -EINVAL;
427
428 attr->config = array_index_nospec((unsigned long)attr->config, x86_pmu.max_events);
429
430 /*
431 * The generic map:
432 */
433 config = x86_pmu.event_map(attr->config);
434
435 if (config == 0)
436 return -ENOENT;
437
438 if (config == -1LL)
439 return -EINVAL;
440
441 hwc->config |= config;
442
443 return 0;
444 }
445
446 /*
447 * check that branch_sample_type is compatible with
448 * settings needed for precise_ip > 1 which implies
449 * using the LBR to capture ALL taken branches at the
450 * priv levels of the measurement
451 */
452 static inline int precise_br_compat(struct perf_event *event)
453 {
454 u64 m = event->attr.branch_sample_type;
455 u64 b = 0;
456
457 /* must capture all branches */
458 if (!(m & PERF_SAMPLE_BRANCH_ANY))
459 return 0;
460
461 m &= PERF_SAMPLE_BRANCH_KERNEL | PERF_SAMPLE_BRANCH_USER;
462
463 if (!event->attr.exclude_user)
464 b |= PERF_SAMPLE_BRANCH_USER;
465
466 if (!event->attr.exclude_kernel)
467 b |= PERF_SAMPLE_BRANCH_KERNEL;
468
469 /*
470 * ignore PERF_SAMPLE_BRANCH_HV, not supported on x86
471 */
472
473 return m == b;
474 }
475
476 int x86_pmu_max_precise(void)
477 {
478 int precise = 0;
479
480 /* Support for constant skid */
481 if (x86_pmu.pebs_active && !x86_pmu.pebs_broken) {
482 precise++;
483
484 /* Support for IP fixup */
485 if (x86_pmu.lbr_nr || x86_pmu.intel_cap.pebs_format >= 2)
486 precise++;
487
488 if (x86_pmu.pebs_prec_dist)
489 precise++;
490 }
491 return precise;
492 }
493
494 int x86_pmu_hw_config(struct perf_event *event)
495 {
496 if (event->attr.precise_ip) {
497 int precise = x86_pmu_max_precise();
498
499 if (event->attr.precise_ip > precise)
500 return -EOPNOTSUPP;
501
502 /* There's no sense in having PEBS for non sampling events: */
503 if (!is_sampling_event(event))
504 return -EINVAL;
505 }
506 /*
507 * check that PEBS LBR correction does not conflict with
508 * whatever the user is asking with attr->branch_sample_type
509 */
510 if (event->attr.precise_ip > 1 && x86_pmu.intel_cap.pebs_format < 2) {
511 u64 *br_type = &event->attr.branch_sample_type;
512
513 if (has_branch_stack(event)) {
514 if (!precise_br_compat(event))
515 return -EOPNOTSUPP;
516
517 /* branch_sample_type is compatible */
518
519 } else {
520 /*
521 * user did not specify branch_sample_type
522 *
523 * For PEBS fixups, we capture all
524 * the branches at the priv level of the
525 * event.
526 */
527 *br_type = PERF_SAMPLE_BRANCH_ANY;
528
529 if (!event->attr.exclude_user)
530 *br_type |= PERF_SAMPLE_BRANCH_USER;
531
532 if (!event->attr.exclude_kernel)
533 *br_type |= PERF_SAMPLE_BRANCH_KERNEL;
534 }
535 }
536
537 if (event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_CALL_STACK)
538 event->attach_state |= PERF_ATTACH_TASK_DATA;
539
540 /*
541 * Generate PMC IRQs:
542 * (keep 'enabled' bit clear for now)
543 */
544 event->hw.config = ARCH_PERFMON_EVENTSEL_INT;
545
546 /*
547 * Count user and OS events unless requested not to
548 */
549 if (!event->attr.exclude_user)
550 event->hw.config |= ARCH_PERFMON_EVENTSEL_USR;
551 if (!event->attr.exclude_kernel)
552 event->hw.config |= ARCH_PERFMON_EVENTSEL_OS;
553
554 if (event->attr.type == PERF_TYPE_RAW)
555 event->hw.config |= event->attr.config & X86_RAW_EVENT_MASK;
556
557 if (event->attr.sample_period && x86_pmu.limit_period) {
558 if (x86_pmu.limit_period(event, event->attr.sample_period) >
559 event->attr.sample_period)
560 return -EINVAL;
561 }
562
563 return x86_setup_perfctr(event);
564 }
565
566 /*
567 * Setup the hardware configuration for a given attr_type
568 */
569 static int __x86_pmu_event_init(struct perf_event *event)
570 {
571 int err;
572
573 if (!x86_pmu_initialized())
574 return -ENODEV;
575
576 err = x86_reserve_hardware();
577 if (err)
578 return err;
579
580 atomic_inc(&active_events);
581 event->destroy = hw_perf_event_destroy;
582
583 event->hw.idx = -1;
584 event->hw.last_cpu = -1;
585 event->hw.last_tag = ~0ULL;
586
587 /* mark unused */
588 event->hw.extra_reg.idx = EXTRA_REG_NONE;
589 event->hw.branch_reg.idx = EXTRA_REG_NONE;
590
591 return x86_pmu.hw_config(event);
592 }
593
594 void x86_pmu_disable_all(void)
595 {
596 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
597 int idx;
598
599 for (idx = 0; idx < x86_pmu.num_counters; idx++) {
600 u64 val;
601
602 if (!test_bit(idx, cpuc->active_mask))
603 continue;
604 rdmsrl(x86_pmu_config_addr(idx), val);
605 if (!(val & ARCH_PERFMON_EVENTSEL_ENABLE))
606 continue;
607 val &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
608 wrmsrl(x86_pmu_config_addr(idx), val);
609 }
610 }
611
612 /*
613 * There may be PMI landing after enabled=0. The PMI hitting could be before or
614 * after disable_all.
615 *
616 * If PMI hits before disable_all, the PMU will be disabled in the NMI handler.
617 * It will not be re-enabled in the NMI handler again, because enabled=0. After
618 * handling the NMI, disable_all will be called, which will not change the
619 * state either. If PMI hits after disable_all, the PMU is already disabled
620 * before entering NMI handler. The NMI handler will not change the state
621 * either.
622 *
623 * So either situation is harmless.
624 */
625 static void x86_pmu_disable(struct pmu *pmu)
626 {
627 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
628
629 if (!x86_pmu_initialized())
630 return;
631
632 if (!cpuc->enabled)
633 return;
634
635 cpuc->n_added = 0;
636 cpuc->enabled = 0;
637 barrier();
638
639 x86_pmu.disable_all();
640 }
641
642 void x86_pmu_enable_all(int added)
643 {
644 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
645 int idx;
646
647 for (idx = 0; idx < x86_pmu.num_counters; idx++) {
648 struct hw_perf_event *hwc = &cpuc->events[idx]->hw;
649
650 if (!test_bit(idx, cpuc->active_mask))
651 continue;
652
653 __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
654 }
655 }
656
657 static struct pmu pmu;
658
659 static inline int is_x86_event(struct perf_event *event)
660 {
661 return event->pmu == &pmu;
662 }
663
664 /*
665 * Event scheduler state:
666 *
667 * Assign events iterating over all events and counters, beginning
668 * with events with least weights first. Keep the current iterator
669 * state in struct sched_state.
670 */
671 struct sched_state {
672 int weight;
673 int event; /* event index */
674 int counter; /* counter index */
675 int unassigned; /* number of events to be assigned left */
676 int nr_gp; /* number of GP counters used */
677 unsigned long used[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
678 };
679
680 /* Total max is X86_PMC_IDX_MAX, but we are O(n!) limited */
681 #define SCHED_STATES_MAX 2
682
683 struct perf_sched {
684 int max_weight;
685 int max_events;
686 int max_gp;
687 int saved_states;
688 struct event_constraint **constraints;
689 struct sched_state state;
690 struct sched_state saved[SCHED_STATES_MAX];
691 };
692
693 /*
694 * Initialize interator that runs through all events and counters.
695 */
696 static void perf_sched_init(struct perf_sched *sched, struct event_constraint **constraints,
697 int num, int wmin, int wmax, int gpmax)
698 {
699 int idx;
700
701 memset(sched, 0, sizeof(*sched));
702 sched->max_events = num;
703 sched->max_weight = wmax;
704 sched->max_gp = gpmax;
705 sched->constraints = constraints;
706
707 for (idx = 0; idx < num; idx++) {
708 if (constraints[idx]->weight == wmin)
709 break;
710 }
711
712 sched->state.event = idx; /* start with min weight */
713 sched->state.weight = wmin;
714 sched->state.unassigned = num;
715 }
716
717 static void perf_sched_save_state(struct perf_sched *sched)
718 {
719 if (WARN_ON_ONCE(sched->saved_states >= SCHED_STATES_MAX))
720 return;
721
722 sched->saved[sched->saved_states] = sched->state;
723 sched->saved_states++;
724 }
725
726 static bool perf_sched_restore_state(struct perf_sched *sched)
727 {
728 if (!sched->saved_states)
729 return false;
730
731 sched->saved_states--;
732 sched->state = sched->saved[sched->saved_states];
733
734 /* continue with next counter: */
735 clear_bit(sched->state.counter++, sched->state.used);
736
737 return true;
738 }
739
740 /*
741 * Select a counter for the current event to schedule. Return true on
742 * success.
743 */
744 static bool __perf_sched_find_counter(struct perf_sched *sched)
745 {
746 struct event_constraint *c;
747 int idx;
748
749 if (!sched->state.unassigned)
750 return false;
751
752 if (sched->state.event >= sched->max_events)
753 return false;
754
755 c = sched->constraints[sched->state.event];
756 /* Prefer fixed purpose counters */
757 if (c->idxmsk64 & (~0ULL << INTEL_PMC_IDX_FIXED)) {
758 idx = INTEL_PMC_IDX_FIXED;
759 for_each_set_bit_from(idx, c->idxmsk, X86_PMC_IDX_MAX) {
760 if (!__test_and_set_bit(idx, sched->state.used))
761 goto done;
762 }
763 }
764
765 /* Grab the first unused counter starting with idx */
766 idx = sched->state.counter;
767 for_each_set_bit_from(idx, c->idxmsk, INTEL_PMC_IDX_FIXED) {
768 if (!__test_and_set_bit(idx, sched->state.used)) {
769 if (sched->state.nr_gp++ >= sched->max_gp)
770 return false;
771
772 goto done;
773 }
774 }
775
776 return false;
777
778 done:
779 sched->state.counter = idx;
780
781 if (c->overlap)
782 perf_sched_save_state(sched);
783
784 return true;
785 }
786
787 static bool perf_sched_find_counter(struct perf_sched *sched)
788 {
789 while (!__perf_sched_find_counter(sched)) {
790 if (!perf_sched_restore_state(sched))
791 return false;
792 }
793
794 return true;
795 }
796
797 /*
798 * Go through all unassigned events and find the next one to schedule.
799 * Take events with the least weight first. Return true on success.
800 */
801 static bool perf_sched_next_event(struct perf_sched *sched)
802 {
803 struct event_constraint *c;
804
805 if (!sched->state.unassigned || !--sched->state.unassigned)
806 return false;
807
808 do {
809 /* next event */
810 sched->state.event++;
811 if (sched->state.event >= sched->max_events) {
812 /* next weight */
813 sched->state.event = 0;
814 sched->state.weight++;
815 if (sched->state.weight > sched->max_weight)
816 return false;
817 }
818 c = sched->constraints[sched->state.event];
819 } while (c->weight != sched->state.weight);
820
821 sched->state.counter = 0; /* start with first counter */
822
823 return true;
824 }
825
826 /*
827 * Assign a counter for each event.
828 */
829 int perf_assign_events(struct event_constraint **constraints, int n,
830 int wmin, int wmax, int gpmax, int *assign)
831 {
832 struct perf_sched sched;
833
834 perf_sched_init(&sched, constraints, n, wmin, wmax, gpmax);
835
836 do {
837 if (!perf_sched_find_counter(&sched))
838 break; /* failed */
839 if (assign)
840 assign[sched.state.event] = sched.state.counter;
841 } while (perf_sched_next_event(&sched));
842
843 return sched.state.unassigned;
844 }
845 EXPORT_SYMBOL_GPL(perf_assign_events);
846
847 int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)
848 {
849 struct event_constraint *c;
850 unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
851 struct perf_event *e;
852 int i, wmin, wmax, unsched = 0;
853 struct hw_perf_event *hwc;
854
855 bitmap_zero(used_mask, X86_PMC_IDX_MAX);
856
857 if (x86_pmu.start_scheduling)
858 x86_pmu.start_scheduling(cpuc);
859
860 for (i = 0, wmin = X86_PMC_IDX_MAX, wmax = 0; i < n; i++) {
861 cpuc->event_constraint[i] = NULL;
862 c = x86_pmu.get_event_constraints(cpuc, i, cpuc->event_list[i]);
863 cpuc->event_constraint[i] = c;
864
865 wmin = min(wmin, c->weight);
866 wmax = max(wmax, c->weight);
867 }
868
869 /*
870 * fastpath, try to reuse previous register
871 */
872 for (i = 0; i < n; i++) {
873 hwc = &cpuc->event_list[i]->hw;
874 c = cpuc->event_constraint[i];
875
876 /* never assigned */
877 if (hwc->idx == -1)
878 break;
879
880 /* constraint still honored */
881 if (!test_bit(hwc->idx, c->idxmsk))
882 break;
883
884 /* not already used */
885 if (test_bit(hwc->idx, used_mask))
886 break;
887
888 __set_bit(hwc->idx, used_mask);
889 if (assign)
890 assign[i] = hwc->idx;
891 }
892
893 /* slow path */
894 if (i != n) {
895 int gpmax = x86_pmu.num_counters;
896
897 /*
898 * Do not allow scheduling of more than half the available
899 * generic counters.
900 *
901 * This helps avoid counter starvation of sibling thread by
902 * ensuring at most half the counters cannot be in exclusive
903 * mode. There is no designated counters for the limits. Any
904 * N/2 counters can be used. This helps with events with
905 * specific counter constraints.
906 */
907 if (is_ht_workaround_enabled() && !cpuc->is_fake &&
908 READ_ONCE(cpuc->excl_cntrs->exclusive_present))
909 gpmax /= 2;
910
911 unsched = perf_assign_events(cpuc->event_constraint, n, wmin,
912 wmax, gpmax, assign);
913 }
914
915 /*
916 * In case of success (unsched = 0), mark events as committed,
917 * so we do not put_constraint() in case new events are added
918 * and fail to be scheduled
919 *
920 * We invoke the lower level commit callback to lock the resource
921 *
922 * We do not need to do all of this in case we are called to
923 * validate an event group (assign == NULL)
924 */
925 if (!unsched && assign) {
926 for (i = 0; i < n; i++) {
927 e = cpuc->event_list[i];
928 e->hw.flags |= PERF_X86_EVENT_COMMITTED;
929 if (x86_pmu.commit_scheduling)
930 x86_pmu.commit_scheduling(cpuc, i, assign[i]);
931 }
932 } else {
933 for (i = 0; i < n; i++) {
934 e = cpuc->event_list[i];
935 /*
936 * do not put_constraint() on comitted events,
937 * because they are good to go
938 */
939 if ((e->hw.flags & PERF_X86_EVENT_COMMITTED))
940 continue;
941
942 /*
943 * release events that failed scheduling
944 */
945 if (x86_pmu.put_event_constraints)
946 x86_pmu.put_event_constraints(cpuc, e);
947 }
948 }
949
950 if (x86_pmu.stop_scheduling)
951 x86_pmu.stop_scheduling(cpuc);
952
953 return unsched ? -EINVAL : 0;
954 }
955
956 /*
957 * dogrp: true if must collect siblings events (group)
958 * returns total number of events and error code
959 */
960 static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp)
961 {
962 struct perf_event *event;
963 int n, max_count;
964
965 max_count = x86_pmu.num_counters + x86_pmu.num_counters_fixed;
966
967 /* current number of events already accepted */
968 n = cpuc->n_events;
969
970 if (is_x86_event(leader)) {
971 if (n >= max_count)
972 return -EINVAL;
973 cpuc->event_list[n] = leader;
974 n++;
975 }
976 if (!dogrp)
977 return n;
978
979 for_each_sibling_event(event, leader) {
980 if (!is_x86_event(event) ||
981 event->state <= PERF_EVENT_STATE_OFF)
982 continue;
983
984 if (n >= max_count)
985 return -EINVAL;
986
987 cpuc->event_list[n] = event;
988 n++;
989 }
990 return n;
991 }
992
993 static inline void x86_assign_hw_event(struct perf_event *event,
994 struct cpu_hw_events *cpuc, int i)
995 {
996 struct hw_perf_event *hwc = &event->hw;
997
998 hwc->idx = cpuc->assign[i];
999 hwc->last_cpu = smp_processor_id();
1000 hwc->last_tag = ++cpuc->tags[i];
1001
1002 if (hwc->idx == INTEL_PMC_IDX_FIXED_BTS) {
1003 hwc->config_base = 0;
1004 hwc->event_base = 0;
1005 } else if (hwc->idx >= INTEL_PMC_IDX_FIXED) {
1006 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
1007 hwc->event_base = MSR_ARCH_PERFMON_FIXED_CTR0 + (hwc->idx - INTEL_PMC_IDX_FIXED);
1008 hwc->event_base_rdpmc = (hwc->idx - INTEL_PMC_IDX_FIXED) | 1<<30;
1009 } else {
1010 hwc->config_base = x86_pmu_config_addr(hwc->idx);
1011 hwc->event_base = x86_pmu_event_addr(hwc->idx);
1012 hwc->event_base_rdpmc = x86_pmu_rdpmc_index(hwc->idx);
1013 }
1014 }
1015
1016 static inline int match_prev_assignment(struct hw_perf_event *hwc,
1017 struct cpu_hw_events *cpuc,
1018 int i)
1019 {
1020 return hwc->idx == cpuc->assign[i] &&
1021 hwc->last_cpu == smp_processor_id() &&
1022 hwc->last_tag == cpuc->tags[i];
1023 }
1024
1025 static void x86_pmu_start(struct perf_event *event, int flags);
1026
1027 static void x86_pmu_enable(struct pmu *pmu)
1028 {
1029 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1030 struct perf_event *event;
1031 struct hw_perf_event *hwc;
1032 int i, added = cpuc->n_added;
1033
1034 if (!x86_pmu_initialized())
1035 return;
1036
1037 if (cpuc->enabled)
1038 return;
1039
1040 if (cpuc->n_added) {
1041 int n_running = cpuc->n_events - cpuc->n_added;
1042 /*
1043 * apply assignment obtained either from
1044 * hw_perf_group_sched_in() or x86_pmu_enable()
1045 *
1046 * step1: save events moving to new counters
1047 */
1048 for (i = 0; i < n_running; i++) {
1049 event = cpuc->event_list[i];
1050 hwc = &event->hw;
1051
1052 /*
1053 * we can avoid reprogramming counter if:
1054 * - assigned same counter as last time
1055 * - running on same CPU as last time
1056 * - no other event has used the counter since
1057 */
1058 if (hwc->idx == -1 ||
1059 match_prev_assignment(hwc, cpuc, i))
1060 continue;
1061
1062 /*
1063 * Ensure we don't accidentally enable a stopped
1064 * counter simply because we rescheduled.
1065 */
1066 if (hwc->state & PERF_HES_STOPPED)
1067 hwc->state |= PERF_HES_ARCH;
1068
1069 x86_pmu_stop(event, PERF_EF_UPDATE);
1070 }
1071
1072 /*
1073 * step2: reprogram moved events into new counters
1074 */
1075 for (i = 0; i < cpuc->n_events; i++) {
1076 event = cpuc->event_list[i];
1077 hwc = &event->hw;
1078
1079 if (!match_prev_assignment(hwc, cpuc, i))
1080 x86_assign_hw_event(event, cpuc, i);
1081 else if (i < n_running)
1082 continue;
1083
1084 if (hwc->state & PERF_HES_ARCH)
1085 continue;
1086
1087 x86_pmu_start(event, PERF_EF_RELOAD);
1088 }
1089 cpuc->n_added = 0;
1090 perf_events_lapic_init();
1091 }
1092
1093 cpuc->enabled = 1;
1094 barrier();
1095
1096 x86_pmu.enable_all(added);
1097 }
1098
1099 static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
1100
1101 /*
1102 * Set the next IRQ period, based on the hwc->period_left value.
1103 * To be called with the event disabled in hw:
1104 */
1105 int x86_perf_event_set_period(struct perf_event *event)
1106 {
1107 struct hw_perf_event *hwc = &event->hw;
1108 s64 left = local64_read(&hwc->period_left);
1109 s64 period = hwc->sample_period;
1110 int ret = 0, idx = hwc->idx;
1111
1112 if (idx == INTEL_PMC_IDX_FIXED_BTS)
1113 return 0;
1114
1115 /*
1116 * If we are way outside a reasonable range then just skip forward:
1117 */
1118 if (unlikely(left <= -period)) {
1119 left = period;
1120 local64_set(&hwc->period_left, left);
1121 hwc->last_period = period;
1122 ret = 1;
1123 }
1124
1125 if (unlikely(left <= 0)) {
1126 left += period;
1127 local64_set(&hwc->period_left, left);
1128 hwc->last_period = period;
1129 ret = 1;
1130 }
1131 /*
1132 * Quirk: certain CPUs dont like it if just 1 hw_event is left:
1133 */
1134 if (unlikely(left < 2))
1135 left = 2;
1136
1137 if (left > x86_pmu.max_period)
1138 left = x86_pmu.max_period;
1139
1140 if (x86_pmu.limit_period)
1141 left = x86_pmu.limit_period(event, left);
1142
1143 per_cpu(pmc_prev_left[idx], smp_processor_id()) = left;
1144
1145 /*
1146 * The hw event starts counting from this event offset,
1147 * mark it to be able to extra future deltas:
1148 */
1149 local64_set(&hwc->prev_count, (u64)-left);
1150
1151 wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask);
1152
1153 /*
1154 * Due to erratum on certan cpu we need
1155 * a second write to be sure the register
1156 * is updated properly
1157 */
1158 if (x86_pmu.perfctr_second_write) {
1159 wrmsrl(hwc->event_base,
1160 (u64)(-left) & x86_pmu.cntval_mask);
1161 }
1162
1163 perf_event_update_userpage(event);
1164
1165 return ret;
1166 }
1167
1168 void x86_pmu_enable_event(struct perf_event *event)
1169 {
1170 if (__this_cpu_read(cpu_hw_events.enabled))
1171 __x86_pmu_enable_event(&event->hw,
1172 ARCH_PERFMON_EVENTSEL_ENABLE);
1173 }
1174
1175 /*
1176 * Add a single event to the PMU.
1177 *
1178 * The event is added to the group of enabled events
1179 * but only if it can be scehduled with existing events.
1180 */
1181 static int x86_pmu_add(struct perf_event *event, int flags)
1182 {
1183 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1184 struct hw_perf_event *hwc;
1185 int assign[X86_PMC_IDX_MAX];
1186 int n, n0, ret;
1187
1188 hwc = &event->hw;
1189
1190 n0 = cpuc->n_events;
1191 ret = n = collect_events(cpuc, event, false);
1192 if (ret < 0)
1193 goto out;
1194
1195 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
1196 if (!(flags & PERF_EF_START))
1197 hwc->state |= PERF_HES_ARCH;
1198
1199 /*
1200 * If group events scheduling transaction was started,
1201 * skip the schedulability test here, it will be performed
1202 * at commit time (->commit_txn) as a whole.
1203 *
1204 * If commit fails, we'll call ->del() on all events
1205 * for which ->add() was called.
1206 */
1207 if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
1208 goto done_collect;
1209
1210 ret = x86_pmu.schedule_events(cpuc, n, assign);
1211 if (ret)
1212 goto out;
1213 /*
1214 * copy new assignment, now we know it is possible
1215 * will be used by hw_perf_enable()
1216 */
1217 memcpy(cpuc->assign, assign, n*sizeof(int));
1218
1219 done_collect:
1220 /*
1221 * Commit the collect_events() state. See x86_pmu_del() and
1222 * x86_pmu_*_txn().
1223 */
1224 cpuc->n_events = n;
1225 cpuc->n_added += n - n0;
1226 cpuc->n_txn += n - n0;
1227
1228 if (x86_pmu.add) {
1229 /*
1230 * This is before x86_pmu_enable() will call x86_pmu_start(),
1231 * so we enable LBRs before an event needs them etc..
1232 */
1233 x86_pmu.add(event);
1234 }
1235
1236 ret = 0;
1237 out:
1238 return ret;
1239 }
1240
1241 static void x86_pmu_start(struct perf_event *event, int flags)
1242 {
1243 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1244 int idx = event->hw.idx;
1245
1246 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
1247 return;
1248
1249 if (WARN_ON_ONCE(idx == -1))
1250 return;
1251
1252 if (flags & PERF_EF_RELOAD) {
1253 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
1254 x86_perf_event_set_period(event);
1255 }
1256
1257 event->hw.state = 0;
1258
1259 cpuc->events[idx] = event;
1260 __set_bit(idx, cpuc->active_mask);
1261 __set_bit(idx, cpuc->running);
1262 x86_pmu.enable(event);
1263 perf_event_update_userpage(event);
1264 }
1265
1266 void perf_event_print_debug(void)
1267 {
1268 u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
1269 u64 pebs, debugctl;
1270 struct cpu_hw_events *cpuc;
1271 unsigned long flags;
1272 int cpu, idx;
1273
1274 if (!x86_pmu.num_counters)
1275 return;
1276
1277 local_irq_save(flags);
1278
1279 cpu = smp_processor_id();
1280 cpuc = &per_cpu(cpu_hw_events, cpu);
1281
1282 if (x86_pmu.version >= 2) {
1283 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
1284 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
1285 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
1286 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
1287
1288 pr_info("\n");
1289 pr_info("CPU#%d: ctrl: %016llx\n", cpu, ctrl);
1290 pr_info("CPU#%d: status: %016llx\n", cpu, status);
1291 pr_info("CPU#%d: overflow: %016llx\n", cpu, overflow);
1292 pr_info("CPU#%d: fixed: %016llx\n", cpu, fixed);
1293 if (x86_pmu.pebs_constraints) {
1294 rdmsrl(MSR_IA32_PEBS_ENABLE, pebs);
1295 pr_info("CPU#%d: pebs: %016llx\n", cpu, pebs);
1296 }
1297 if (x86_pmu.lbr_nr) {
1298 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
1299 pr_info("CPU#%d: debugctl: %016llx\n", cpu, debugctl);
1300 }
1301 }
1302 pr_info("CPU#%d: active: %016llx\n", cpu, *(u64 *)cpuc->active_mask);
1303
1304 for (idx = 0; idx < x86_pmu.num_counters; idx++) {
1305 rdmsrl(x86_pmu_config_addr(idx), pmc_ctrl);
1306 rdmsrl(x86_pmu_event_addr(idx), pmc_count);
1307
1308 prev_left = per_cpu(pmc_prev_left[idx], cpu);
1309
1310 pr_info("CPU#%d: gen-PMC%d ctrl: %016llx\n",
1311 cpu, idx, pmc_ctrl);
1312 pr_info("CPU#%d: gen-PMC%d count: %016llx\n",
1313 cpu, idx, pmc_count);
1314 pr_info("CPU#%d: gen-PMC%d left: %016llx\n",
1315 cpu, idx, prev_left);
1316 }
1317 for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
1318 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
1319
1320 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
1321 cpu, idx, pmc_count);
1322 }
1323 local_irq_restore(flags);
1324 }
1325
1326 void x86_pmu_stop(struct perf_event *event, int flags)
1327 {
1328 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1329 struct hw_perf_event *hwc = &event->hw;
1330
1331 if (test_bit(hwc->idx, cpuc->active_mask)) {
1332 x86_pmu.disable(event);
1333 __clear_bit(hwc->idx, cpuc->active_mask);
1334 cpuc->events[hwc->idx] = NULL;
1335 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
1336 hwc->state |= PERF_HES_STOPPED;
1337 }
1338
1339 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
1340 /*
1341 * Drain the remaining delta count out of a event
1342 * that we are disabling:
1343 */
1344 x86_perf_event_update(event);
1345 hwc->state |= PERF_HES_UPTODATE;
1346 }
1347 }
1348
1349 static void x86_pmu_del(struct perf_event *event, int flags)
1350 {
1351 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1352 int i;
1353
1354 /*
1355 * event is descheduled
1356 */
1357 event->hw.flags &= ~PERF_X86_EVENT_COMMITTED;
1358
1359 /*
1360 * If we're called during a txn, we only need to undo x86_pmu.add.
1361 * The events never got scheduled and ->cancel_txn will truncate
1362 * the event_list.
1363 *
1364 * XXX assumes any ->del() called during a TXN will only be on
1365 * an event added during that same TXN.
1366 */
1367 if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
1368 goto do_del;
1369
1370 /*
1371 * Not a TXN, therefore cleanup properly.
1372 */
1373 x86_pmu_stop(event, PERF_EF_UPDATE);
1374
1375 for (i = 0; i < cpuc->n_events; i++) {
1376 if (event == cpuc->event_list[i])
1377 break;
1378 }
1379
1380 if (WARN_ON_ONCE(i == cpuc->n_events)) /* called ->del() without ->add() ? */
1381 return;
1382
1383 /* If we have a newly added event; make sure to decrease n_added. */
1384 if (i >= cpuc->n_events - cpuc->n_added)
1385 --cpuc->n_added;
1386
1387 if (x86_pmu.put_event_constraints)
1388 x86_pmu.put_event_constraints(cpuc, event);
1389
1390 /* Delete the array entry. */
1391 while (++i < cpuc->n_events) {
1392 cpuc->event_list[i-1] = cpuc->event_list[i];
1393 cpuc->event_constraint[i-1] = cpuc->event_constraint[i];
1394 }
1395 --cpuc->n_events;
1396
1397 perf_event_update_userpage(event);
1398
1399 do_del:
1400 if (x86_pmu.del) {
1401 /*
1402 * This is after x86_pmu_stop(); so we disable LBRs after any
1403 * event can need them etc..
1404 */
1405 x86_pmu.del(event);
1406 }
1407 }
1408
1409 int x86_pmu_handle_irq(struct pt_regs *regs)
1410 {
1411 struct perf_sample_data data;
1412 struct cpu_hw_events *cpuc;
1413 struct perf_event *event;
1414 int idx, handled = 0;
1415 u64 val;
1416
1417 cpuc = this_cpu_ptr(&cpu_hw_events);
1418
1419 /*
1420 * Some chipsets need to unmask the LVTPC in a particular spot
1421 * inside the nmi handler. As a result, the unmasking was pushed
1422 * into all the nmi handlers.
1423 *
1424 * This generic handler doesn't seem to have any issues where the
1425 * unmasking occurs so it was left at the top.
1426 */
1427 apic_write(APIC_LVTPC, APIC_DM_NMI);
1428
1429 for (idx = 0; idx < x86_pmu.num_counters; idx++) {
1430 if (!test_bit(idx, cpuc->active_mask))
1431 continue;
1432
1433 event = cpuc->events[idx];
1434
1435 val = x86_perf_event_update(event);
1436 if (val & (1ULL << (x86_pmu.cntval_bits - 1)))
1437 continue;
1438
1439 /*
1440 * event overflow
1441 */
1442 handled++;
1443 perf_sample_data_init(&data, 0, event->hw.last_period);
1444
1445 if (!x86_perf_event_set_period(event))
1446 continue;
1447
1448 if (perf_event_overflow(event, &data, regs))
1449 x86_pmu_stop(event, 0);
1450 }
1451
1452 if (handled)
1453 inc_irq_stat(apic_perf_irqs);
1454
1455 return handled;
1456 }
1457
1458 void perf_events_lapic_init(void)
1459 {
1460 if (!x86_pmu.apic || !x86_pmu_initialized())
1461 return;
1462
1463 /*
1464 * Always use NMI for PMU
1465 */
1466 apic_write(APIC_LVTPC, APIC_DM_NMI);
1467 }
1468
1469 static int
1470 perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs)
1471 {
1472 u64 start_clock;
1473 u64 finish_clock;
1474 int ret;
1475
1476 /*
1477 * All PMUs/events that share this PMI handler should make sure to
1478 * increment active_events for their events.
1479 */
1480 if (!atomic_read(&active_events))
1481 return NMI_DONE;
1482
1483 start_clock = sched_clock();
1484 ret = x86_pmu.handle_irq(regs);
1485 finish_clock = sched_clock();
1486
1487 perf_sample_event_took(finish_clock - start_clock);
1488
1489 return ret;
1490 }
1491 NOKPROBE_SYMBOL(perf_event_nmi_handler);
1492
1493 struct event_constraint emptyconstraint;
1494 struct event_constraint unconstrained;
1495
1496 static int x86_pmu_prepare_cpu(unsigned int cpu)
1497 {
1498 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
1499 int i;
1500
1501 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++)
1502 cpuc->kfree_on_online[i] = NULL;
1503 if (x86_pmu.cpu_prepare)
1504 return x86_pmu.cpu_prepare(cpu);
1505 return 0;
1506 }
1507
1508 static int x86_pmu_dead_cpu(unsigned int cpu)
1509 {
1510 if (x86_pmu.cpu_dead)
1511 x86_pmu.cpu_dead(cpu);
1512 return 0;
1513 }
1514
1515 static int x86_pmu_online_cpu(unsigned int cpu)
1516 {
1517 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
1518 int i;
1519
1520 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++) {
1521 kfree(cpuc->kfree_on_online[i]);
1522 cpuc->kfree_on_online[i] = NULL;
1523 }
1524 return 0;
1525 }
1526
1527 static int x86_pmu_starting_cpu(unsigned int cpu)
1528 {
1529 if (x86_pmu.cpu_starting)
1530 x86_pmu.cpu_starting(cpu);
1531 return 0;
1532 }
1533
1534 static int x86_pmu_dying_cpu(unsigned int cpu)
1535 {
1536 if (x86_pmu.cpu_dying)
1537 x86_pmu.cpu_dying(cpu);
1538 return 0;
1539 }
1540
1541 static void __init pmu_check_apic(void)
1542 {
1543 if (boot_cpu_has(X86_FEATURE_APIC))
1544 return;
1545
1546 x86_pmu.apic = 0;
1547 pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n");
1548 pr_info("no hardware sampling interrupt available.\n");
1549
1550 /*
1551 * If we have a PMU initialized but no APIC
1552 * interrupts, we cannot sample hardware
1553 * events (user-space has to fall back and
1554 * sample via a hrtimer based software event):
1555 */
1556 pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT;
1557
1558 }
1559
1560 static struct attribute_group x86_pmu_format_group = {
1561 .name = "format",
1562 .attrs = NULL,
1563 };
1564
1565 /*
1566 * Remove all undefined events (x86_pmu.event_map(id) == 0)
1567 * out of events_attr attributes.
1568 */
1569 static void __init filter_events(struct attribute **attrs)
1570 {
1571 struct device_attribute *d;
1572 struct perf_pmu_events_attr *pmu_attr;
1573 int offset = 0;
1574 int i, j;
1575
1576 for (i = 0; attrs[i]; i++) {
1577 d = (struct device_attribute *)attrs[i];
1578 pmu_attr = container_of(d, struct perf_pmu_events_attr, attr);
1579 /* str trumps id */
1580 if (pmu_attr->event_str)
1581 continue;
1582 if (x86_pmu.event_map(i + offset))
1583 continue;
1584
1585 for (j = i; attrs[j]; j++)
1586 attrs[j] = attrs[j + 1];
1587
1588 /* Check the shifted attr. */
1589 i--;
1590
1591 /*
1592 * event_map() is index based, the attrs array is organized
1593 * by increasing event index. If we shift the events, then
1594 * we need to compensate for the event_map(), otherwise
1595 * we are looking up the wrong event in the map
1596 */
1597 offset++;
1598 }
1599 }
1600
1601 /* Merge two pointer arrays */
1602 __init struct attribute **merge_attr(struct attribute **a, struct attribute **b)
1603 {
1604 struct attribute **new;
1605 int j, i;
1606
1607 for (j = 0; a[j]; j++)
1608 ;
1609 for (i = 0; b[i]; i++)
1610 j++;
1611 j++;
1612
1613 new = kmalloc_array(j, sizeof(struct attribute *), GFP_KERNEL);
1614 if (!new)
1615 return NULL;
1616
1617 j = 0;
1618 for (i = 0; a[i]; i++)
1619 new[j++] = a[i];
1620 for (i = 0; b[i]; i++)
1621 new[j++] = b[i];
1622 new[j] = NULL;
1623
1624 return new;
1625 }
1626
1627 ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr, char *page)
1628 {
1629 struct perf_pmu_events_attr *pmu_attr = \
1630 container_of(attr, struct perf_pmu_events_attr, attr);
1631 u64 config = x86_pmu.event_map(pmu_attr->id);
1632
1633 /* string trumps id */
1634 if (pmu_attr->event_str)
1635 return sprintf(page, "%s", pmu_attr->event_str);
1636
1637 return x86_pmu.events_sysfs_show(page, config);
1638 }
1639 EXPORT_SYMBOL_GPL(events_sysfs_show);
1640
1641 ssize_t events_ht_sysfs_show(struct device *dev, struct device_attribute *attr,
1642 char *page)
1643 {
1644 struct perf_pmu_events_ht_attr *pmu_attr =
1645 container_of(attr, struct perf_pmu_events_ht_attr, attr);
1646
1647 /*
1648 * Report conditional events depending on Hyper-Threading.
1649 *
1650 * This is overly conservative as usually the HT special
1651 * handling is not needed if the other CPU thread is idle.
1652 *
1653 * Note this does not (and cannot) handle the case when thread
1654 * siblings are invisible, for example with virtualization
1655 * if they are owned by some other guest. The user tool
1656 * has to re-read when a thread sibling gets onlined later.
1657 */
1658 return sprintf(page, "%s",
1659 topology_max_smt_threads() > 1 ?
1660 pmu_attr->event_str_ht :
1661 pmu_attr->event_str_noht);
1662 }
1663
1664 EVENT_ATTR(cpu-cycles, CPU_CYCLES );
1665 EVENT_ATTR(instructions, INSTRUCTIONS );
1666 EVENT_ATTR(cache-references, CACHE_REFERENCES );
1667 EVENT_ATTR(cache-misses, CACHE_MISSES );
1668 EVENT_ATTR(branch-instructions, BRANCH_INSTRUCTIONS );
1669 EVENT_ATTR(branch-misses, BRANCH_MISSES );
1670 EVENT_ATTR(bus-cycles, BUS_CYCLES );
1671 EVENT_ATTR(stalled-cycles-frontend, STALLED_CYCLES_FRONTEND );
1672 EVENT_ATTR(stalled-cycles-backend, STALLED_CYCLES_BACKEND );
1673 EVENT_ATTR(ref-cycles, REF_CPU_CYCLES );
1674
1675 static struct attribute *empty_attrs;
1676
1677 static struct attribute *events_attr[] = {
1678 EVENT_PTR(CPU_CYCLES),
1679 EVENT_PTR(INSTRUCTIONS),
1680 EVENT_PTR(CACHE_REFERENCES),
1681 EVENT_PTR(CACHE_MISSES),
1682 EVENT_PTR(BRANCH_INSTRUCTIONS),
1683 EVENT_PTR(BRANCH_MISSES),
1684 EVENT_PTR(BUS_CYCLES),
1685 EVENT_PTR(STALLED_CYCLES_FRONTEND),
1686 EVENT_PTR(STALLED_CYCLES_BACKEND),
1687 EVENT_PTR(REF_CPU_CYCLES),
1688 NULL,
1689 };
1690
1691 static struct attribute_group x86_pmu_events_group = {
1692 .name = "events",
1693 .attrs = events_attr,
1694 };
1695
1696 ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event)
1697 {
1698 u64 umask = (config & ARCH_PERFMON_EVENTSEL_UMASK) >> 8;
1699 u64 cmask = (config & ARCH_PERFMON_EVENTSEL_CMASK) >> 24;
1700 bool edge = (config & ARCH_PERFMON_EVENTSEL_EDGE);
1701 bool pc = (config & ARCH_PERFMON_EVENTSEL_PIN_CONTROL);
1702 bool any = (config & ARCH_PERFMON_EVENTSEL_ANY);
1703 bool inv = (config & ARCH_PERFMON_EVENTSEL_INV);
1704 ssize_t ret;
1705
1706 /*
1707 * We have whole page size to spend and just little data
1708 * to write, so we can safely use sprintf.
1709 */
1710 ret = sprintf(page, "event=0x%02llx", event);
1711
1712 if (umask)
1713 ret += sprintf(page + ret, ",umask=0x%02llx", umask);
1714
1715 if (edge)
1716 ret += sprintf(page + ret, ",edge");
1717
1718 if (pc)
1719 ret += sprintf(page + ret, ",pc");
1720
1721 if (any)
1722 ret += sprintf(page + ret, ",any");
1723
1724 if (inv)
1725 ret += sprintf(page + ret, ",inv");
1726
1727 if (cmask)
1728 ret += sprintf(page + ret, ",cmask=0x%02llx", cmask);
1729
1730 ret += sprintf(page + ret, "\n");
1731
1732 return ret;
1733 }
1734
1735 static struct attribute_group x86_pmu_attr_group;
1736 static struct attribute_group x86_pmu_caps_group;
1737
1738 static int __init init_hw_perf_events(void)
1739 {
1740 struct x86_pmu_quirk *quirk;
1741 int err;
1742
1743 pr_info("Performance Events: ");
1744
1745 switch (boot_cpu_data.x86_vendor) {
1746 case X86_VENDOR_INTEL:
1747 err = intel_pmu_init();
1748 break;
1749 case X86_VENDOR_AMD:
1750 err = amd_pmu_init();
1751 break;
1752 default:
1753 err = -ENOTSUPP;
1754 }
1755 if (err != 0) {
1756 pr_cont("no PMU driver, software events only.\n");
1757 return 0;
1758 }
1759
1760 pmu_check_apic();
1761
1762 /* sanity check that the hardware exists or is emulated */
1763 if (!check_hw_exists())
1764 return 0;
1765
1766 pr_cont("%s PMU driver.\n", x86_pmu.name);
1767
1768 x86_pmu.attr_rdpmc = 1; /* enable userspace RDPMC usage by default */
1769
1770 for (quirk = x86_pmu.quirks; quirk; quirk = quirk->next)
1771 quirk->func();
1772
1773 if (!x86_pmu.intel_ctrl)
1774 x86_pmu.intel_ctrl = (1 << x86_pmu.num_counters) - 1;
1775
1776 perf_events_lapic_init();
1777 register_nmi_handler(NMI_LOCAL, perf_event_nmi_handler, 0, "PMI");
1778
1779 unconstrained = (struct event_constraint)
1780 __EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_counters) - 1,
1781 0, x86_pmu.num_counters, 0, 0);
1782
1783 x86_pmu_format_group.attrs = x86_pmu.format_attrs;
1784
1785 if (x86_pmu.caps_attrs) {
1786 struct attribute **tmp;
1787
1788 tmp = merge_attr(x86_pmu_caps_group.attrs, x86_pmu.caps_attrs);
1789 if (!WARN_ON(!tmp))
1790 x86_pmu_caps_group.attrs = tmp;
1791 }
1792
1793 if (x86_pmu.event_attrs)
1794 x86_pmu_events_group.attrs = x86_pmu.event_attrs;
1795
1796 if (!x86_pmu.events_sysfs_show)
1797 x86_pmu_events_group.attrs = &empty_attrs;
1798 else
1799 filter_events(x86_pmu_events_group.attrs);
1800
1801 if (x86_pmu.cpu_events) {
1802 struct attribute **tmp;
1803
1804 tmp = merge_attr(x86_pmu_events_group.attrs, x86_pmu.cpu_events);
1805 if (!WARN_ON(!tmp))
1806 x86_pmu_events_group.attrs = tmp;
1807 }
1808
1809 if (x86_pmu.attrs) {
1810 struct attribute **tmp;
1811
1812 tmp = merge_attr(x86_pmu_attr_group.attrs, x86_pmu.attrs);
1813 if (!WARN_ON(!tmp))
1814 x86_pmu_attr_group.attrs = tmp;
1815 }
1816
1817 pr_info("... version: %d\n", x86_pmu.version);
1818 pr_info("... bit width: %d\n", x86_pmu.cntval_bits);
1819 pr_info("... generic registers: %d\n", x86_pmu.num_counters);
1820 pr_info("... value mask: %016Lx\n", x86_pmu.cntval_mask);
1821 pr_info("... max period: %016Lx\n", x86_pmu.max_period);
1822 pr_info("... fixed-purpose events: %d\n", x86_pmu.num_counters_fixed);
1823 pr_info("... event mask: %016Lx\n", x86_pmu.intel_ctrl);
1824
1825 /*
1826 * Install callbacks. Core will call them for each online
1827 * cpu.
1828 */
1829 err = cpuhp_setup_state(CPUHP_PERF_X86_PREPARE, "perf/x86:prepare",
1830 x86_pmu_prepare_cpu, x86_pmu_dead_cpu);
1831 if (err)
1832 return err;
1833
1834 err = cpuhp_setup_state(CPUHP_AP_PERF_X86_STARTING,
1835 "perf/x86:starting", x86_pmu_starting_cpu,
1836 x86_pmu_dying_cpu);
1837 if (err)
1838 goto out;
1839
1840 err = cpuhp_setup_state(CPUHP_AP_PERF_X86_ONLINE, "perf/x86:online",
1841 x86_pmu_online_cpu, NULL);
1842 if (err)
1843 goto out1;
1844
1845 err = perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW);
1846 if (err)
1847 goto out2;
1848
1849 return 0;
1850
1851 out2:
1852 cpuhp_remove_state(CPUHP_AP_PERF_X86_ONLINE);
1853 out1:
1854 cpuhp_remove_state(CPUHP_AP_PERF_X86_STARTING);
1855 out:
1856 cpuhp_remove_state(CPUHP_PERF_X86_PREPARE);
1857 return err;
1858 }
1859 early_initcall(init_hw_perf_events);
1860
1861 static inline void x86_pmu_read(struct perf_event *event)
1862 {
1863 if (x86_pmu.read)
1864 return x86_pmu.read(event);
1865 x86_perf_event_update(event);
1866 }
1867
1868 /*
1869 * Start group events scheduling transaction
1870 * Set the flag to make pmu::enable() not perform the
1871 * schedulability test, it will be performed at commit time
1872 *
1873 * We only support PERF_PMU_TXN_ADD transactions. Save the
1874 * transaction flags but otherwise ignore non-PERF_PMU_TXN_ADD
1875 * transactions.
1876 */
1877 static void x86_pmu_start_txn(struct pmu *pmu, unsigned int txn_flags)
1878 {
1879 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1880
1881 WARN_ON_ONCE(cpuc->txn_flags); /* txn already in flight */
1882
1883 cpuc->txn_flags = txn_flags;
1884 if (txn_flags & ~PERF_PMU_TXN_ADD)
1885 return;
1886
1887 perf_pmu_disable(pmu);
1888 __this_cpu_write(cpu_hw_events.n_txn, 0);
1889 }
1890
1891 /*
1892 * Stop group events scheduling transaction
1893 * Clear the flag and pmu::enable() will perform the
1894 * schedulability test.
1895 */
1896 static void x86_pmu_cancel_txn(struct pmu *pmu)
1897 {
1898 unsigned int txn_flags;
1899 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1900
1901 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */
1902
1903 txn_flags = cpuc->txn_flags;
1904 cpuc->txn_flags = 0;
1905 if (txn_flags & ~PERF_PMU_TXN_ADD)
1906 return;
1907
1908 /*
1909 * Truncate collected array by the number of events added in this
1910 * transaction. See x86_pmu_add() and x86_pmu_*_txn().
1911 */
1912 __this_cpu_sub(cpu_hw_events.n_added, __this_cpu_read(cpu_hw_events.n_txn));
1913 __this_cpu_sub(cpu_hw_events.n_events, __this_cpu_read(cpu_hw_events.n_txn));
1914 perf_pmu_enable(pmu);
1915 }
1916
1917 /*
1918 * Commit group events scheduling transaction
1919 * Perform the group schedulability test as a whole
1920 * Return 0 if success
1921 *
1922 * Does not cancel the transaction on failure; expects the caller to do this.
1923 */
1924 static int x86_pmu_commit_txn(struct pmu *pmu)
1925 {
1926 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1927 int assign[X86_PMC_IDX_MAX];
1928 int n, ret;
1929
1930 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */
1931
1932 if (cpuc->txn_flags & ~PERF_PMU_TXN_ADD) {
1933 cpuc->txn_flags = 0;
1934 return 0;
1935 }
1936
1937 n = cpuc->n_events;
1938
1939 if (!x86_pmu_initialized())
1940 return -EAGAIN;
1941
1942 ret = x86_pmu.schedule_events(cpuc, n, assign);
1943 if (ret)
1944 return ret;
1945
1946 /*
1947 * copy new assignment, now we know it is possible
1948 * will be used by hw_perf_enable()
1949 */
1950 memcpy(cpuc->assign, assign, n*sizeof(int));
1951
1952 cpuc->txn_flags = 0;
1953 perf_pmu_enable(pmu);
1954 return 0;
1955 }
1956 /*
1957 * a fake_cpuc is used to validate event groups. Due to
1958 * the extra reg logic, we need to also allocate a fake
1959 * per_core and per_cpu structure. Otherwise, group events
1960 * using extra reg may conflict without the kernel being
1961 * able to catch this when the last event gets added to
1962 * the group.
1963 */
1964 static void free_fake_cpuc(struct cpu_hw_events *cpuc)
1965 {
1966 intel_cpuc_finish(cpuc);
1967 kfree(cpuc);
1968 }
1969
1970 static struct cpu_hw_events *allocate_fake_cpuc(void)
1971 {
1972 struct cpu_hw_events *cpuc;
1973 int cpu = raw_smp_processor_id();
1974
1975 cpuc = kzalloc(sizeof(*cpuc), GFP_KERNEL);
1976 if (!cpuc)
1977 return ERR_PTR(-ENOMEM);
1978 cpuc->is_fake = 1;
1979
1980 if (intel_cpuc_prepare(cpuc, cpu))
1981 goto error;
1982
1983 return cpuc;
1984 error:
1985 free_fake_cpuc(cpuc);
1986 return ERR_PTR(-ENOMEM);
1987 }
1988
1989 /*
1990 * validate that we can schedule this event
1991 */
1992 static int validate_event(struct perf_event *event)
1993 {
1994 struct cpu_hw_events *fake_cpuc;
1995 struct event_constraint *c;
1996 int ret = 0;
1997
1998 fake_cpuc = allocate_fake_cpuc();
1999 if (IS_ERR(fake_cpuc))
2000 return PTR_ERR(fake_cpuc);
2001
2002 c = x86_pmu.get_event_constraints(fake_cpuc, -1, event);
2003
2004 if (!c || !c->weight)
2005 ret = -EINVAL;
2006
2007 if (x86_pmu.put_event_constraints)
2008 x86_pmu.put_event_constraints(fake_cpuc, event);
2009
2010 free_fake_cpuc(fake_cpuc);
2011
2012 return ret;
2013 }
2014
2015 /*
2016 * validate a single event group
2017 *
2018 * validation include:
2019 * - check events are compatible which each other
2020 * - events do not compete for the same counter
2021 * - number of events <= number of counters
2022 *
2023 * validation ensures the group can be loaded onto the
2024 * PMU if it was the only group available.
2025 */
2026 static int validate_group(struct perf_event *event)
2027 {
2028 struct perf_event *leader = event->group_leader;
2029 struct cpu_hw_events *fake_cpuc;
2030 int ret = -EINVAL, n;
2031
2032 fake_cpuc = allocate_fake_cpuc();
2033 if (IS_ERR(fake_cpuc))
2034 return PTR_ERR(fake_cpuc);
2035 /*
2036 * the event is not yet connected with its
2037 * siblings therefore we must first collect
2038 * existing siblings, then add the new event
2039 * before we can simulate the scheduling
2040 */
2041 n = collect_events(fake_cpuc, leader, true);
2042 if (n < 0)
2043 goto out;
2044
2045 fake_cpuc->n_events = n;
2046 n = collect_events(fake_cpuc, event, false);
2047 if (n < 0)
2048 goto out;
2049
2050 fake_cpuc->n_events = n;
2051
2052 ret = x86_pmu.schedule_events(fake_cpuc, n, NULL);
2053
2054 out:
2055 free_fake_cpuc(fake_cpuc);
2056 return ret;
2057 }
2058
2059 static int x86_pmu_event_init(struct perf_event *event)
2060 {
2061 struct pmu *tmp;
2062 int err;
2063
2064 switch (event->attr.type) {
2065 case PERF_TYPE_RAW:
2066 case PERF_TYPE_HARDWARE:
2067 case PERF_TYPE_HW_CACHE:
2068 break;
2069
2070 default:
2071 return -ENOENT;
2072 }
2073
2074 err = __x86_pmu_event_init(event);
2075 if (!err) {
2076 /*
2077 * we temporarily connect event to its pmu
2078 * such that validate_group() can classify
2079 * it as an x86 event using is_x86_event()
2080 */
2081 tmp = event->pmu;
2082 event->pmu = &pmu;
2083
2084 if (event->group_leader != event)
2085 err = validate_group(event);
2086 else
2087 err = validate_event(event);
2088
2089 event->pmu = tmp;
2090 }
2091 if (err) {
2092 if (event->destroy)
2093 event->destroy(event);
2094 }
2095
2096 if (READ_ONCE(x86_pmu.attr_rdpmc) &&
2097 !(event->hw.flags & PERF_X86_EVENT_LARGE_PEBS))
2098 event->hw.flags |= PERF_X86_EVENT_RDPMC_ALLOWED;
2099
2100 return err;
2101 }
2102
2103 static void refresh_pce(void *ignored)
2104 {
2105 load_mm_cr4(this_cpu_read(cpu_tlbstate.loaded_mm));
2106 }
2107
2108 static void x86_pmu_event_mapped(struct perf_event *event, struct mm_struct *mm)
2109 {
2110 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED))
2111 return;
2112
2113 /*
2114 * This function relies on not being called concurrently in two
2115 * tasks in the same mm. Otherwise one task could observe
2116 * perf_rdpmc_allowed > 1 and return all the way back to
2117 * userspace with CR4.PCE clear while another task is still
2118 * doing on_each_cpu_mask() to propagate CR4.PCE.
2119 *
2120 * For now, this can't happen because all callers hold mmap_sem
2121 * for write. If this changes, we'll need a different solution.
2122 */
2123 lockdep_assert_held_exclusive(&mm->mmap_sem);
2124
2125 if (atomic_inc_return(&mm->context.perf_rdpmc_allowed) == 1)
2126 on_each_cpu_mask(mm_cpumask(mm), refresh_pce, NULL, 1);
2127 }
2128
2129 static void x86_pmu_event_unmapped(struct perf_event *event, struct mm_struct *mm)
2130 {
2131
2132 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED))
2133 return;
2134
2135 if (atomic_dec_and_test(&mm->context.perf_rdpmc_allowed))
2136 on_each_cpu_mask(mm_cpumask(mm), refresh_pce, NULL, 1);
2137 }
2138
2139 static int x86_pmu_event_idx(struct perf_event *event)
2140 {
2141 int idx = event->hw.idx;
2142
2143 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED))
2144 return 0;
2145
2146 if (x86_pmu.num_counters_fixed && idx >= INTEL_PMC_IDX_FIXED) {
2147 idx -= INTEL_PMC_IDX_FIXED;
2148 idx |= 1 << 30;
2149 }
2150
2151 return idx + 1;
2152 }
2153
2154 static ssize_t get_attr_rdpmc(struct device *cdev,
2155 struct device_attribute *attr,
2156 char *buf)
2157 {
2158 return snprintf(buf, 40, "%d\n", x86_pmu.attr_rdpmc);
2159 }
2160
2161 static ssize_t set_attr_rdpmc(struct device *cdev,
2162 struct device_attribute *attr,
2163 const char *buf, size_t count)
2164 {
2165 unsigned long val;
2166 ssize_t ret;
2167
2168 ret = kstrtoul(buf, 0, &val);
2169 if (ret)
2170 return ret;
2171
2172 if (val > 2)
2173 return -EINVAL;
2174
2175 if (x86_pmu.attr_rdpmc_broken)
2176 return -ENOTSUPP;
2177
2178 if ((val == 2) != (x86_pmu.attr_rdpmc == 2)) {
2179 /*
2180 * Changing into or out of always available, aka
2181 * perf-event-bypassing mode. This path is extremely slow,
2182 * but only root can trigger it, so it's okay.
2183 */
2184 if (val == 2)
2185 static_branch_inc(&rdpmc_always_available_key);
2186 else
2187 static_branch_dec(&rdpmc_always_available_key);
2188 on_each_cpu(refresh_pce, NULL, 1);
2189 }
2190
2191 x86_pmu.attr_rdpmc = val;
2192
2193 return count;
2194 }
2195
2196 static DEVICE_ATTR(rdpmc, S_IRUSR | S_IWUSR, get_attr_rdpmc, set_attr_rdpmc);
2197
2198 static struct attribute *x86_pmu_attrs[] = {
2199 &dev_attr_rdpmc.attr,
2200 NULL,
2201 };
2202
2203 static struct attribute_group x86_pmu_attr_group = {
2204 .attrs = x86_pmu_attrs,
2205 };
2206
2207 static ssize_t max_precise_show(struct device *cdev,
2208 struct device_attribute *attr,
2209 char *buf)
2210 {
2211 return snprintf(buf, PAGE_SIZE, "%d\n", x86_pmu_max_precise());
2212 }
2213
2214 static DEVICE_ATTR_RO(max_precise);
2215
2216 static struct attribute *x86_pmu_caps_attrs[] = {
2217 &dev_attr_max_precise.attr,
2218 NULL
2219 };
2220
2221 static struct attribute_group x86_pmu_caps_group = {
2222 .name = "caps",
2223 .attrs = x86_pmu_caps_attrs,
2224 };
2225
2226 static const struct attribute_group *x86_pmu_attr_groups[] = {
2227 &x86_pmu_attr_group,
2228 &x86_pmu_format_group,
2229 &x86_pmu_events_group,
2230 &x86_pmu_caps_group,
2231 NULL,
2232 };
2233
2234 static void x86_pmu_sched_task(struct perf_event_context *ctx, bool sched_in)
2235 {
2236 if (x86_pmu.sched_task)
2237 x86_pmu.sched_task(ctx, sched_in);
2238 }
2239
2240 void perf_check_microcode(void)
2241 {
2242 if (x86_pmu.check_microcode)
2243 x86_pmu.check_microcode();
2244 }
2245
2246 static int x86_pmu_check_period(struct perf_event *event, u64 value)
2247 {
2248 if (x86_pmu.check_period && x86_pmu.check_period(event, value))
2249 return -EINVAL;
2250
2251 if (value && x86_pmu.limit_period) {
2252 if (x86_pmu.limit_period(event, value) > value)
2253 return -EINVAL;
2254 }
2255
2256 return 0;
2257 }
2258
2259 static struct pmu pmu = {
2260 .pmu_enable = x86_pmu_enable,
2261 .pmu_disable = x86_pmu_disable,
2262
2263 .attr_groups = x86_pmu_attr_groups,
2264
2265 .event_init = x86_pmu_event_init,
2266
2267 .event_mapped = x86_pmu_event_mapped,
2268 .event_unmapped = x86_pmu_event_unmapped,
2269
2270 .add = x86_pmu_add,
2271 .del = x86_pmu_del,
2272 .start = x86_pmu_start,
2273 .stop = x86_pmu_stop,
2274 .read = x86_pmu_read,
2275
2276 .start_txn = x86_pmu_start_txn,
2277 .cancel_txn = x86_pmu_cancel_txn,
2278 .commit_txn = x86_pmu_commit_txn,
2279
2280 .event_idx = x86_pmu_event_idx,
2281 .sched_task = x86_pmu_sched_task,
2282 .task_ctx_size = sizeof(struct x86_perf_task_context),
2283 .check_period = x86_pmu_check_period,
2284 };
2285
2286 void arch_perf_update_userpage(struct perf_event *event,
2287 struct perf_event_mmap_page *userpg, u64 now)
2288 {
2289 struct cyc2ns_data data;
2290 u64 offset;
2291
2292 userpg->cap_user_time = 0;
2293 userpg->cap_user_time_zero = 0;
2294 userpg->cap_user_rdpmc =
2295 !!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED);
2296 userpg->pmc_width = x86_pmu.cntval_bits;
2297
2298 if (!using_native_sched_clock() || !sched_clock_stable())
2299 return;
2300
2301 cyc2ns_read_begin(&data);
2302
2303 offset = data.cyc2ns_offset + __sched_clock_offset;
2304
2305 /*
2306 * Internal timekeeping for enabled/running/stopped times
2307 * is always in the local_clock domain.
2308 */
2309 userpg->cap_user_time = 1;
2310 userpg->time_mult = data.cyc2ns_mul;
2311 userpg->time_shift = data.cyc2ns_shift;
2312 userpg->time_offset = offset - now;
2313
2314 /*
2315 * cap_user_time_zero doesn't make sense when we're using a different
2316 * time base for the records.
2317 */
2318 if (!event->attr.use_clockid) {
2319 userpg->cap_user_time_zero = 1;
2320 userpg->time_zero = offset;
2321 }
2322
2323 cyc2ns_read_end();
2324 }
2325
2326 void
2327 perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs)
2328 {
2329 struct unwind_state state;
2330 unsigned long addr;
2331
2332 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
2333 /* TODO: We don't support guest os callchain now */
2334 return;
2335 }
2336
2337 if (perf_callchain_store(entry, regs->ip))
2338 return;
2339
2340 for (unwind_start(&state, current, regs, NULL); !unwind_done(&state);
2341 unwind_next_frame(&state)) {
2342 addr = unwind_get_return_address(&state);
2343 if (!addr || perf_callchain_store(entry, addr))
2344 return;
2345 }
2346 }
2347
2348 static inline int
2349 valid_user_frame(const void __user *fp, unsigned long size)
2350 {
2351 return (__range_not_ok(fp, size, TASK_SIZE) == 0);
2352 }
2353
2354 static unsigned long get_segment_base(unsigned int segment)
2355 {
2356 struct desc_struct *desc;
2357 unsigned int idx = segment >> 3;
2358
2359 if ((segment & SEGMENT_TI_MASK) == SEGMENT_LDT) {
2360 #ifdef CONFIG_MODIFY_LDT_SYSCALL
2361 struct ldt_struct *ldt;
2362
2363 /* IRQs are off, so this synchronizes with smp_store_release */
2364 ldt = READ_ONCE(current->active_mm->context.ldt);
2365 if (!ldt || idx >= ldt->nr_entries)
2366 return 0;
2367
2368 desc = &ldt->entries[idx];
2369 #else
2370 return 0;
2371 #endif
2372 } else {
2373 if (idx >= GDT_ENTRIES)
2374 return 0;
2375
2376 desc = raw_cpu_ptr(gdt_page.gdt) + idx;
2377 }
2378
2379 return get_desc_base(desc);
2380 }
2381
2382 #ifdef CONFIG_IA32_EMULATION
2383
2384 #include <linux/compat.h>
2385
2386 static inline int
2387 perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry)
2388 {
2389 /* 32-bit process in 64-bit kernel. */
2390 unsigned long ss_base, cs_base;
2391 struct stack_frame_ia32 frame;
2392 const void __user *fp;
2393
2394 if (!test_thread_flag(TIF_IA32))
2395 return 0;
2396
2397 cs_base = get_segment_base(regs->cs);
2398 ss_base = get_segment_base(regs->ss);
2399
2400 fp = compat_ptr(ss_base + regs->bp);
2401 pagefault_disable();
2402 while (entry->nr < entry->max_stack) {
2403 unsigned long bytes;
2404 frame.next_frame = 0;
2405 frame.return_address = 0;
2406
2407 if (!valid_user_frame(fp, sizeof(frame)))
2408 break;
2409
2410 bytes = __copy_from_user_nmi(&frame.next_frame, fp, 4);
2411 if (bytes != 0)
2412 break;
2413 bytes = __copy_from_user_nmi(&frame.return_address, fp+4, 4);
2414 if (bytes != 0)
2415 break;
2416
2417 perf_callchain_store(entry, cs_base + frame.return_address);
2418 fp = compat_ptr(ss_base + frame.next_frame);
2419 }
2420 pagefault_enable();
2421 return 1;
2422 }
2423 #else
2424 static inline int
2425 perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry)
2426 {
2427 return 0;
2428 }
2429 #endif
2430
2431 void
2432 perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs)
2433 {
2434 struct stack_frame frame;
2435 const unsigned long __user *fp;
2436
2437 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
2438 /* TODO: We don't support guest os callchain now */
2439 return;
2440 }
2441
2442 /*
2443 * We don't know what to do with VM86 stacks.. ignore them for now.
2444 */
2445 if (regs->flags & (X86_VM_MASK | PERF_EFLAGS_VM))
2446 return;
2447
2448 fp = (unsigned long __user *)regs->bp;
2449
2450 perf_callchain_store(entry, regs->ip);
2451
2452 if (!nmi_uaccess_okay())
2453 return;
2454
2455 if (perf_callchain_user32(regs, entry))
2456 return;
2457
2458 pagefault_disable();
2459 while (entry->nr < entry->max_stack) {
2460 unsigned long bytes;
2461
2462 frame.next_frame = NULL;
2463 frame.return_address = 0;
2464
2465 if (!valid_user_frame(fp, sizeof(frame)))
2466 break;
2467
2468 bytes = __copy_from_user_nmi(&frame.next_frame, fp, sizeof(*fp));
2469 if (bytes != 0)
2470 break;
2471 bytes = __copy_from_user_nmi(&frame.return_address, fp + 1, sizeof(*fp));
2472 if (bytes != 0)
2473 break;
2474
2475 perf_callchain_store(entry, frame.return_address);
2476 fp = (void __user *)frame.next_frame;
2477 }
2478 pagefault_enable();
2479 }
2480
2481 /*
2482 * Deal with code segment offsets for the various execution modes:
2483 *
2484 * VM86 - the good olde 16 bit days, where the linear address is
2485 * 20 bits and we use regs->ip + 0x10 * regs->cs.
2486 *
2487 * IA32 - Where we need to look at GDT/LDT segment descriptor tables
2488 * to figure out what the 32bit base address is.
2489 *
2490 * X32 - has TIF_X32 set, but is running in x86_64
2491 *
2492 * X86_64 - CS,DS,SS,ES are all zero based.
2493 */
2494 static unsigned long code_segment_base(struct pt_regs *regs)
2495 {
2496 /*
2497 * For IA32 we look at the GDT/LDT segment base to convert the
2498 * effective IP to a linear address.
2499 */
2500
2501 #ifdef CONFIG_X86_32
2502 /*
2503 * If we are in VM86 mode, add the segment offset to convert to a
2504 * linear address.
2505 */
2506 if (regs->flags & X86_VM_MASK)
2507 return 0x10 * regs->cs;
2508
2509 if (user_mode(regs) && regs->cs != __USER_CS)
2510 return get_segment_base(regs->cs);
2511 #else
2512 if (user_mode(regs) && !user_64bit_mode(regs) &&
2513 regs->cs != __USER32_CS)
2514 return get_segment_base(regs->cs);
2515 #endif
2516 return 0;
2517 }
2518
2519 unsigned long perf_instruction_pointer(struct pt_regs *regs)
2520 {
2521 if (perf_guest_cbs && perf_guest_cbs->is_in_guest())
2522 return perf_guest_cbs->get_guest_ip();
2523
2524 return regs->ip + code_segment_base(regs);
2525 }
2526
2527 unsigned long perf_misc_flags(struct pt_regs *regs)
2528 {
2529 int misc = 0;
2530
2531 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
2532 if (perf_guest_cbs->is_user_mode())
2533 misc |= PERF_RECORD_MISC_GUEST_USER;
2534 else
2535 misc |= PERF_RECORD_MISC_GUEST_KERNEL;
2536 } else {
2537 if (user_mode(regs))
2538 misc |= PERF_RECORD_MISC_USER;
2539 else
2540 misc |= PERF_RECORD_MISC_KERNEL;
2541 }
2542
2543 if (regs->flags & PERF_EFLAGS_EXACT)
2544 misc |= PERF_RECORD_MISC_EXACT_IP;
2545
2546 return misc;
2547 }
2548
2549 void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap)
2550 {
2551 cap->version = x86_pmu.version;
2552 cap->num_counters_gp = x86_pmu.num_counters;
2553 cap->num_counters_fixed = x86_pmu.num_counters_fixed;
2554 cap->bit_width_gp = x86_pmu.cntval_bits;
2555 cap->bit_width_fixed = x86_pmu.cntval_bits;
2556 cap->events_mask = (unsigned int)x86_pmu.events_maskl;
2557 cap->events_mask_len = x86_pmu.events_mask_len;
2558 }
2559 EXPORT_SYMBOL_GPL(perf_get_x86_pmu_capability);
Linux with added FPC-III support