arch/x86/events/perf_event.h

   1 /*
   2  * Performance events x86 architecture header
   3  *
   4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
   5  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
   6  *  Copyright (C) 2009 Jaswinder Singh Rajput
   7  *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
   8  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra
   9  *  Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
  10  *  Copyright (C) 2009 Google, Inc., Stephane Eranian
  11  *
  12  *  For licencing details see kernel-base/COPYING
  13  */
  14
  15 #include <linux/perf_event.h>
  16
  17 #include <asm/fpu/xstate.h>
  18 #include <asm/intel_ds.h>
  19 #include <asm/cpu.h>
  20
  21 /* To enable MSR tracing please use the generic trace points. */
  22
  23 /*
  24  *          |   NHM/WSM    |      SNB     |
  25  * register -------------------------------
  26  *          |  HT  | no HT |  HT  | no HT |
  27  *-----------------------------------------
  28  * offcore  | core | core  | cpu  | core  |
  29  * lbr_sel  | core | core  | cpu  | core  |
  30  * ld_lat   | cpu  | core  | cpu  | core  |
  31  *-----------------------------------------
  32  *
  33  * Given that there is a small number of shared regs,
  34  * we can pre-allocate their slot in the per-cpu
  35  * per-core reg tables.
  36  */
  37 enum extra_reg_type {
  38         EXTRA_REG_NONE          = -1, /* not used */
  39
  40         EXTRA_REG_RSP_0         = 0,  /* offcore_response_0 */
  41         EXTRA_REG_RSP_1         = 1,  /* offcore_response_1 */
  42         EXTRA_REG_LBR           = 2,  /* lbr_select */
  43         EXTRA_REG_LDLAT         = 3,  /* ld_lat_threshold */
  44         EXTRA_REG_FE            = 4,  /* fe_* */
  45         EXTRA_REG_SNOOP_0       = 5,  /* snoop response 0 */
  46         EXTRA_REG_SNOOP_1       = 6,  /* snoop response 1 */
  47
  48         EXTRA_REG_MAX                 /* number of entries needed */
  49 };
  50
  51 struct event_constraint {
  52         union {
  53                 unsigned long   idxmsk[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  54                 u64             idxmsk64;
  55         };
  56         u64             code;
  57         u64             cmask;
  58         int             weight;
  59         int             overlap;
  60         int             flags;
  61         unsigned int    size;
  62 };
  63
  64 static inline bool constraint_match(struct event_constraint *c, u64 ecode)
  65 {
  66         return ((ecode & c->cmask) - c->code) <= (u64)c->size;
  67 }
  68
  69 #define PERF_ARCH(name, val)    \
  70         PERF_X86_EVENT_##name = val,
  71
  72 /*
  73  * struct hw_perf_event.flags flags
  74  */
  75 enum {
  76 #include "perf_event_flags.h"
  77 };
  78
  79 #undef PERF_ARCH
  80
  81 #define PERF_ARCH(name, val)                                            \
  82         static_assert((PERF_X86_EVENT_##name & PERF_EVENT_FLAG_ARCH) == \
  83                       PERF_X86_EVENT_##name);
  84
  85 #include "perf_event_flags.h"
  86
  87 #undef PERF_ARCH
  88
  89 static inline bool is_topdown_count(struct perf_event *event)
  90 {
  91         return event->hw.flags & PERF_X86_EVENT_TOPDOWN;
  92 }
  93
  94 static inline bool is_metric_event(struct perf_event *event)
  95 {
  96         u64 config = event->attr.config;
  97
  98         return ((config & ARCH_PERFMON_EVENTSEL_EVENT) == 0) &&
  99                 ((config & INTEL_ARCH_EVENT_MASK) >= INTEL_TD_METRIC_RETIRING)  &&
 100                 ((config & INTEL_ARCH_EVENT_MASK) <= INTEL_TD_METRIC_MAX);
 101 }
 102
 103 static inline bool is_slots_event(struct perf_event *event)
 104 {
 105         return (event->attr.config & INTEL_ARCH_EVENT_MASK) == INTEL_TD_SLOTS;
 106 }
 107
 108 static inline bool is_topdown_event(struct perf_event *event)
 109 {
 110         return is_metric_event(event) || is_slots_event(event);
 111 }
 112
 113 static inline bool is_branch_counters_group(struct perf_event *event)
 114 {
 115         return event->group_leader->hw.flags & PERF_X86_EVENT_BRANCH_COUNTERS;
 116 }
 117
 118 struct amd_nb {
 119         int nb_id;  /* NorthBridge id */
 120         int refcnt; /* reference count */
 121         struct perf_event *owners[X86_PMC_IDX_MAX];
 122         struct event_constraint event_constraints[X86_PMC_IDX_MAX];
 123 };
 124
 125 #define PEBS_COUNTER_MASK       ((1ULL << MAX_PEBS_EVENTS) - 1)
 126 #define PEBS_PMI_AFTER_EACH_RECORD BIT_ULL(60)
 127 #define PEBS_OUTPUT_OFFSET      61
 128 #define PEBS_OUTPUT_MASK        (3ull << PEBS_OUTPUT_OFFSET)
 129 #define PEBS_OUTPUT_PT          (1ull << PEBS_OUTPUT_OFFSET)
 130 #define PEBS_VIA_PT_MASK        (PEBS_OUTPUT_PT | PEBS_PMI_AFTER_EACH_RECORD)
 131
 132 /*
 133  * Flags PEBS can handle without an PMI.
 134  *
 135  * TID can only be handled by flushing at context switch.
 136  * REGS_USER can be handled for events limited to ring 3.
 137  *
 138  */
 139 #define LARGE_PEBS_FLAGS \
 140         (PERF_SAMPLE_IP | PERF_SAMPLE_TID | PERF_SAMPLE_ADDR | \
 141         PERF_SAMPLE_ID | PERF_SAMPLE_CPU | PERF_SAMPLE_STREAM_ID | \
 142         PERF_SAMPLE_DATA_SRC | PERF_SAMPLE_IDENTIFIER | \
 143         PERF_SAMPLE_TRANSACTION | PERF_SAMPLE_PHYS_ADDR | \
 144         PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER | \
 145         PERF_SAMPLE_PERIOD | PERF_SAMPLE_CODE_PAGE_SIZE | \
 146         PERF_SAMPLE_WEIGHT_TYPE)
 147
 148 #define PEBS_GP_REGS                    \
 149         ((1ULL << PERF_REG_X86_AX)    | \
 150          (1ULL << PERF_REG_X86_BX)    | \
 151          (1ULL << PERF_REG_X86_CX)    | \
 152          (1ULL << PERF_REG_X86_DX)    | \
 153          (1ULL << PERF_REG_X86_DI)    | \
 154          (1ULL << PERF_REG_X86_SI)    | \
 155          (1ULL << PERF_REG_X86_SP)    | \
 156          (1ULL << PERF_REG_X86_BP)    | \
 157          (1ULL << PERF_REG_X86_IP)    | \
 158          (1ULL << PERF_REG_X86_FLAGS) | \
 159          (1ULL << PERF_REG_X86_R8)    | \
 160          (1ULL << PERF_REG_X86_R9)    | \
 161          (1ULL << PERF_REG_X86_R10)   | \
 162          (1ULL << PERF_REG_X86_R11)   | \
 163          (1ULL << PERF_REG_X86_R12)   | \
 164          (1ULL << PERF_REG_X86_R13)   | \
 165          (1ULL << PERF_REG_X86_R14)   | \
 166          (1ULL << PERF_REG_X86_R15))
 167
 168 /*
 169  * Per register state.
 170  */
 171 struct er_account {
 172         raw_spinlock_t      lock;       /* per-core: protect structure */
 173         u64                 config;     /* extra MSR config */
 174         u64                 reg;        /* extra MSR number */
 175         atomic_t            ref;        /* reference count */
 176 };
 177
 178 /*
 179  * Per core/cpu state
 180  *
 181  * Used to coordinate shared registers between HT threads or
 182  * among events on a single PMU.
 183  */
 184 struct intel_shared_regs {
 185         struct er_account       regs[EXTRA_REG_MAX];
 186         int                     refcnt;         /* per-core: #HT threads */
 187         unsigned                core_id;        /* per-core: core id */
 188 };
 189
 190 enum intel_excl_state_type {
 191         INTEL_EXCL_UNUSED    = 0, /* counter is unused */
 192         INTEL_EXCL_SHARED    = 1, /* counter can be used by both threads */
 193         INTEL_EXCL_EXCLUSIVE = 2, /* counter can be used by one thread only */
 194 };
 195
 196 struct intel_excl_states {
 197         enum intel_excl_state_type state[X86_PMC_IDX_MAX];
 198         bool sched_started; /* true if scheduling has started */
 199 };
 200
 201 struct intel_excl_cntrs {
 202         raw_spinlock_t  lock;
 203
 204         struct intel_excl_states states[2];
 205
 206         union {
 207                 u16     has_exclusive[2];
 208                 u32     exclusive_present;
 209         };
 210
 211         int             refcnt;         /* per-core: #HT threads */
 212         unsigned        core_id;        /* per-core: core id */
 213 };
 214
 215 struct x86_perf_task_context;
 216 #define MAX_LBR_ENTRIES         32
 217
 218 enum {
 219         LBR_FORMAT_32           = 0x00,
 220         LBR_FORMAT_LIP          = 0x01,
 221         LBR_FORMAT_EIP          = 0x02,
 222         LBR_FORMAT_EIP_FLAGS    = 0x03,
 223         LBR_FORMAT_EIP_FLAGS2   = 0x04,
 224         LBR_FORMAT_INFO         = 0x05,
 225         LBR_FORMAT_TIME         = 0x06,
 226         LBR_FORMAT_INFO2        = 0x07,
 227         LBR_FORMAT_MAX_KNOWN    = LBR_FORMAT_INFO2,
 228 };
 229
 230 enum {
 231         X86_PERF_KFREE_SHARED = 0,
 232         X86_PERF_KFREE_EXCL   = 1,
 233         X86_PERF_KFREE_MAX
 234 };
 235
 236 struct cpu_hw_events {
 237         /*
 238          * Generic x86 PMC bits
 239          */
 240         struct perf_event       *events[X86_PMC_IDX_MAX]; /* in counter order */
 241         unsigned long           active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
 242         unsigned long           dirty[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
 243         int                     enabled;
 244
 245         int                     n_events; /* the # of events in the below arrays */
 246         int                     n_added;  /* the # last events in the below arrays;
 247                                              they've never been enabled yet */
 248         int                     n_txn;    /* the # last events in the below arrays;
 249                                              added in the current transaction */
 250         int                     n_txn_pair;
 251         int                     n_txn_metric;
 252         int                     assign[X86_PMC_IDX_MAX]; /* event to counter assignment */
 253         u64                     tags[X86_PMC_IDX_MAX];
 254
 255         struct perf_event       *event_list[X86_PMC_IDX_MAX]; /* in enabled order */
 256         struct event_constraint *event_constraint[X86_PMC_IDX_MAX];
 257
 258         int                     n_excl; /* the number of exclusive events */
 259
 260         unsigned int            txn_flags;
 261         int                     is_fake;
 262
 263         /*
 264          * Intel DebugStore bits
 265          */
 266         struct debug_store      *ds;
 267         void                    *ds_pebs_vaddr;
 268         void                    *ds_bts_vaddr;
 269         u64                     pebs_enabled;
 270         int                     n_pebs;
 271         int                     n_large_pebs;
 272         int                     n_pebs_via_pt;
 273         int                     pebs_output;
 274
 275         /* Current super set of events hardware configuration */
 276         u64                     pebs_data_cfg;
 277         u64                     active_pebs_data_cfg;
 278         int                     pebs_record_size;
 279
 280         /* Intel Fixed counter configuration */
 281         u64                     fixed_ctrl_val;
 282         u64                     active_fixed_ctrl_val;
 283
 284         /*
 285          * Intel LBR bits
 286          */
 287         int                             lbr_users;
 288         int                             lbr_pebs_users;
 289         struct perf_branch_stack        lbr_stack;
 290         struct perf_branch_entry        lbr_entries[MAX_LBR_ENTRIES];
 291         u64                             lbr_counters[MAX_LBR_ENTRIES]; /* branch stack extra */
 292         union {
 293                 struct er_account               *lbr_sel;
 294                 struct er_account               *lbr_ctl;
 295         };
 296         u64                             br_sel;
 297         void                            *last_task_ctx;
 298         int                             last_log_id;
 299         int                             lbr_select;
 300         void                            *lbr_xsave;
 301
 302         /*
 303          * Intel host/guest exclude bits
 304          */
 305         u64                             intel_ctrl_guest_mask;
 306         u64                             intel_ctrl_host_mask;
 307         struct perf_guest_switch_msr    guest_switch_msrs[X86_PMC_IDX_MAX];
 308
 309         /*
 310          * Intel checkpoint mask
 311          */
 312         u64                             intel_cp_status;
 313
 314         /*
 315          * manage shared (per-core, per-cpu) registers
 316          * used on Intel NHM/WSM/SNB
 317          */
 318         struct intel_shared_regs        *shared_regs;
 319         /*
 320          * manage exclusive counter access between hyperthread
 321          */
 322         struct event_constraint *constraint_list; /* in enable order */
 323         struct intel_excl_cntrs         *excl_cntrs;
 324         int excl_thread_id; /* 0 or 1 */
 325
 326         /*
 327          * SKL TSX_FORCE_ABORT shadow
 328          */
 329         u64                             tfa_shadow;
 330
 331         /*
 332          * Perf Metrics
 333          */
 334         /* number of accepted metrics events */
 335         int                             n_metric;
 336
 337         /*
 338          * AMD specific bits
 339          */
 340         struct amd_nb                   *amd_nb;
 341         int                             brs_active; /* BRS is enabled */
 342
 343         /* Inverted mask of bits to clear in the perf_ctr ctrl registers */
 344         u64                             perf_ctr_virt_mask;
 345         int                             n_pair; /* Large increment events */
 346
 347         void                            *kfree_on_online[X86_PERF_KFREE_MAX];
 348
 349         struct pmu                      *pmu;
 350 };
 351
 352 #define __EVENT_CONSTRAINT_RANGE(c, e, n, m, w, o, f) { \
 353         { .idxmsk64 = (n) },            \
 354         .code = (c),                    \
 355         .size = (e) - (c),              \
 356         .cmask = (m),                   \
 357         .weight = (w),                  \
 358         .overlap = (o),                 \
 359         .flags = f,                     \
 360 }
 361
 362 #define __EVENT_CONSTRAINT(c, n, m, w, o, f) \
 363         __EVENT_CONSTRAINT_RANGE(c, c, n, m, w, o, f)
 364
 365 #define EVENT_CONSTRAINT(c, n, m)       \
 366         __EVENT_CONSTRAINT(c, n, m, HWEIGHT(n), 0, 0)
 367
 368 /*
 369  * The constraint_match() function only works for 'simple' event codes
 370  * and not for extended (AMD64_EVENTSEL_EVENT) events codes.
 371  */
 372 #define EVENT_CONSTRAINT_RANGE(c, e, n, m) \
 373         __EVENT_CONSTRAINT_RANGE(c, e, n, m, HWEIGHT(n), 0, 0)
 374
 375 #define INTEL_EXCLEVT_CONSTRAINT(c, n)  \
 376         __EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT, HWEIGHT(n),\
 377                            0, PERF_X86_EVENT_EXCL)
 378
 379 /*
 380  * The overlap flag marks event constraints with overlapping counter
 381  * masks. This is the case if the counter mask of such an event is not
 382  * a subset of any other counter mask of a constraint with an equal or
 383  * higher weight, e.g.:
 384  *
 385  *  c_overlaps = EVENT_CONSTRAINT_OVERLAP(0, 0x09, 0);
 386  *  c_another1 = EVENT_CONSTRAINT(0, 0x07, 0);
 387  *  c_another2 = EVENT_CONSTRAINT(0, 0x38, 0);
 388  *
 389  * The event scheduler may not select the correct counter in the first
 390  * cycle because it needs to know which subsequent events will be
 391  * scheduled. It may fail to schedule the events then. So we set the
 392  * overlap flag for such constraints to give the scheduler a hint which
 393  * events to select for counter rescheduling.
 394  *
 395  * Care must be taken as the rescheduling algorithm is O(n!) which
 396  * will increase scheduling cycles for an over-committed system
 397  * dramatically.  The number of such EVENT_CONSTRAINT_OVERLAP() macros
 398  * and its counter masks must be kept at a minimum.
 399  */
 400 #define EVENT_CONSTRAINT_OVERLAP(c, n, m)       \
 401         __EVENT_CONSTRAINT(c, n, m, HWEIGHT(n), 1, 0)
 402
 403 /*
 404  * Constraint on the Event code.
 405  */
 406 #define INTEL_EVENT_CONSTRAINT(c, n)    \
 407         EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT)
 408
 409 /*
 410  * Constraint on a range of Event codes
 411  */
 412 #define INTEL_EVENT_CONSTRAINT_RANGE(c, e, n)                   \
 413         EVENT_CONSTRAINT_RANGE(c, e, n, ARCH_PERFMON_EVENTSEL_EVENT)
 414
 415 /*
 416  * Constraint on the Event code + UMask + fixed-mask
 417  *
 418  * filter mask to validate fixed counter events.
 419  * the following filters disqualify for fixed counters:
 420  *  - inv
 421  *  - edge
 422  *  - cnt-mask
 423  *  - in_tx
 424  *  - in_tx_checkpointed
 425  *  The other filters are supported by fixed counters.
 426  *  The any-thread option is supported starting with v3.
 427  */
 428 #define FIXED_EVENT_FLAGS (X86_RAW_EVENT_MASK|HSW_IN_TX|HSW_IN_TX_CHECKPOINTED)
 429 #define FIXED_EVENT_CONSTRAINT(c, n)    \
 430         EVENT_CONSTRAINT(c, (1ULL << (32+n)), FIXED_EVENT_FLAGS)
 431
 432 /*
 433  * The special metric counters do not actually exist. They are calculated from
 434  * the combination of the FxCtr3 + MSR_PERF_METRICS.
 435  *
 436  * The special metric counters are mapped to a dummy offset for the scheduler.
 437  * The sharing between multiple users of the same metric without multiplexing
 438  * is not allowed, even though the hardware supports that in principle.
 439  */
 440
 441 #define METRIC_EVENT_CONSTRAINT(c, n)                                   \
 442         EVENT_CONSTRAINT(c, (1ULL << (INTEL_PMC_IDX_METRIC_BASE + n)),  \
 443                          INTEL_ARCH_EVENT_MASK)
 444
 445 /*
 446  * Constraint on the Event code + UMask
 447  */
 448 #define INTEL_UEVENT_CONSTRAINT(c, n)   \
 449         EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK)
 450
 451 /* Constraint on specific umask bit only + event */
 452 #define INTEL_UBIT_EVENT_CONSTRAINT(c, n)       \
 453         EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT|(c))
 454
 455 /* Like UEVENT_CONSTRAINT, but match flags too */
 456 #define INTEL_FLAGS_UEVENT_CONSTRAINT(c, n)     \
 457         EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS)
 458
 459 #define INTEL_EXCLUEVT_CONSTRAINT(c, n) \
 460         __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK, \
 461                            HWEIGHT(n), 0, PERF_X86_EVENT_EXCL)
 462
 463 #define INTEL_PLD_CONSTRAINT(c, n)      \
 464         __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
 465                            HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LDLAT)
 466
 467 #define INTEL_PSD_CONSTRAINT(c, n)      \
 468         __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
 469                            HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_STLAT)
 470
 471 #define INTEL_PST_CONSTRAINT(c, n)      \
 472         __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
 473                           HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST)
 474
 475 #define INTEL_HYBRID_LAT_CONSTRAINT(c, n)       \
 476         __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
 477                           HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LAT_HYBRID)
 478
 479 #define INTEL_HYBRID_LDLAT_CONSTRAINT(c, n)     \
 480         __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
 481                           HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LAT_HYBRID|PERF_X86_EVENT_PEBS_LD_HSW)
 482
 483 #define INTEL_HYBRID_STLAT_CONSTRAINT(c, n)     \
 484         __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
 485                           HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LAT_HYBRID|PERF_X86_EVENT_PEBS_ST_HSW)
 486
 487 /* Event constraint, but match on all event flags too. */
 488 #define INTEL_FLAGS_EVENT_CONSTRAINT(c, n) \
 489         EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS)
 490
 491 #define INTEL_FLAGS_EVENT_CONSTRAINT_RANGE(c, e, n)                     \
 492         EVENT_CONSTRAINT_RANGE(c, e, n, ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS)
 493
 494 /* Check only flags, but allow all event/umask */
 495 #define INTEL_ALL_EVENT_CONSTRAINT(code, n)     \
 496         EVENT_CONSTRAINT(code, n, X86_ALL_EVENT_FLAGS)
 497
 498 /* Check flags and event code, and set the HSW store flag */
 499 #define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_ST(code, n) \
 500         __EVENT_CONSTRAINT(code, n,                     \
 501                           ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
 502                           HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST_HSW)
 503
 504 /* Check flags and event code, and set the HSW load flag */
 505 #define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(code, n) \
 506         __EVENT_CONSTRAINT(code, n,                     \
 507                           ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
 508                           HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LD_HSW)
 509
 510 #define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD_RANGE(code, end, n) \
 511         __EVENT_CONSTRAINT_RANGE(code, end, n,                          \
 512                           ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
 513                           HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LD_HSW)
 514
 515 #define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_XLD(code, n) \
 516         __EVENT_CONSTRAINT(code, n,                     \
 517                           ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
 518                           HWEIGHT(n), 0, \
 519                           PERF_X86_EVENT_PEBS_LD_HSW|PERF_X86_EVENT_EXCL)
 520
 521 /* Check flags and event code/umask, and set the HSW store flag */
 522 #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(code, n) \
 523         __EVENT_CONSTRAINT(code, n,                     \
 524                           INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
 525                           HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST_HSW)
 526
 527 #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XST(code, n) \
 528         __EVENT_CONSTRAINT(code, n,                     \
 529                           INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
 530                           HWEIGHT(n), 0, \
 531                           PERF_X86_EVENT_PEBS_ST_HSW|PERF_X86_EVENT_EXCL)
 532
 533 /* Check flags and event code/umask, and set the HSW load flag */
 534 #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(code, n) \
 535         __EVENT_CONSTRAINT(code, n,                     \
 536                           INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
 537                           HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LD_HSW)
 538
 539 #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XLD(code, n) \
 540         __EVENT_CONSTRAINT(code, n,                     \
 541                           INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
 542                           HWEIGHT(n), 0, \
 543                           PERF_X86_EVENT_PEBS_LD_HSW|PERF_X86_EVENT_EXCL)
 544
 545 /* Check flags and event code/umask, and set the HSW N/A flag */
 546 #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_NA(code, n) \
 547         __EVENT_CONSTRAINT(code, n,                     \
 548                           INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
 549                           HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_NA_HSW)
 550
 551
 552 /*
 553  * We define the end marker as having a weight of -1
 554  * to enable blacklisting of events using a counter bitmask
 555  * of zero and thus a weight of zero.
 556  * The end marker has a weight that cannot possibly be
 557  * obtained from counting the bits in the bitmask.
 558  */
 559 #define EVENT_CONSTRAINT_END { .weight = -1 }
 560
 561 /*
 562  * Check for end marker with weight == -1
 563  */
 564 #define for_each_event_constraint(e, c) \
 565         for ((e) = (c); (e)->weight != -1; (e)++)
 566
 567 /*
 568  * Extra registers for specific events.
 569  *
 570  * Some events need large masks and require external MSRs.
 571  * Those extra MSRs end up being shared for all events on
 572  * a PMU and sometimes between PMU of sibling HT threads.
 573  * In either case, the kernel needs to handle conflicting
 574  * accesses to those extra, shared, regs. The data structure
 575  * to manage those registers is stored in cpu_hw_event.
 576  */
 577 struct extra_reg {
 578         unsigned int            event;
 579         unsigned int            msr;
 580         u64                     config_mask;
 581         u64                     valid_mask;
 582         int                     idx;  /* per_xxx->regs[] reg index */
 583         bool                    extra_msr_access;
 584 };
 585
 586 #define EVENT_EXTRA_REG(e, ms, m, vm, i) {      \
 587         .event = (e),                   \
 588         .msr = (ms),                    \
 589         .config_mask = (m),             \
 590         .valid_mask = (vm),             \
 591         .idx = EXTRA_REG_##i,           \
 592         .extra_msr_access = true,       \
 593         }
 594
 595 #define INTEL_EVENT_EXTRA_REG(event, msr, vm, idx)      \
 596         EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT, vm, idx)
 597
 598 #define INTEL_UEVENT_EXTRA_REG(event, msr, vm, idx) \
 599         EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT | \
 600                         ARCH_PERFMON_EVENTSEL_UMASK, vm, idx)
 601
 602 #define INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(c) \
 603         INTEL_UEVENT_EXTRA_REG(c, \
 604                                MSR_PEBS_LD_LAT_THRESHOLD, \
 605                                0xffff, \
 606                                LDLAT)
 607
 608 #define EVENT_EXTRA_END EVENT_EXTRA_REG(0, 0, 0, 0, RSP_0)
 609
 610 union perf_capabilities {
 611         struct {
 612                 u64     lbr_format:6;
 613                 u64     pebs_trap:1;
 614                 u64     pebs_arch_reg:1;
 615                 u64     pebs_format:4;
 616                 u64     smm_freeze:1;
 617                 /*
 618                  * PMU supports separate counter range for writing
 619                  * values > 32bit.
 620                  */
 621                 u64     full_width_write:1;
 622                 u64     pebs_baseline:1;
 623                 u64     perf_metrics:1;
 624                 u64     pebs_output_pt_available:1;
 625                 u64     pebs_timing_info:1;
 626                 u64     anythread_deprecated:1;
 627         };
 628         u64     capabilities;
 629 };
 630
 631 struct x86_pmu_quirk {
 632         struct x86_pmu_quirk *next;
 633         void (*func)(void);
 634 };
 635
 636 union x86_pmu_config {
 637         struct {
 638                 u64 event:8,
 639                     umask:8,
 640                     usr:1,
 641                     os:1,
 642                     edge:1,
 643                     pc:1,
 644                     interrupt:1,
 645                     __reserved1:1,
 646                     en:1,
 647                     inv:1,
 648                     cmask:8,
 649                     event2:4,
 650                     __reserved2:4,
 651                     go:1,
 652                     ho:1;
 653         } bits;
 654         u64 value;
 655 };
 656
 657 #define X86_CONFIG(args...) ((union x86_pmu_config){.bits = {args}}).value
 658
 659 enum {
 660         x86_lbr_exclusive_lbr,
 661         x86_lbr_exclusive_bts,
 662         x86_lbr_exclusive_pt,
 663         x86_lbr_exclusive_max,
 664 };
 665
 666 #define PERF_PEBS_DATA_SOURCE_MAX       0x100
 667 #define PERF_PEBS_DATA_SOURCE_MASK      (PERF_PEBS_DATA_SOURCE_MAX - 1)
 668 #define PERF_PEBS_DATA_SOURCE_GRT_MAX   0x10
 669 #define PERF_PEBS_DATA_SOURCE_GRT_MASK  (PERF_PEBS_DATA_SOURCE_GRT_MAX - 1)
 670
 671 enum hybrid_cpu_type {
 672         HYBRID_INTEL_NONE,
 673         HYBRID_INTEL_ATOM       = 0x20,
 674         HYBRID_INTEL_CORE       = 0x40,
 675 };
 676
 677 enum hybrid_pmu_type {
 678         not_hybrid,
 679         hybrid_small            = BIT(0),
 680         hybrid_big              = BIT(1),
 681
 682         hybrid_big_small        = hybrid_big | hybrid_small, /* only used for matching */
 683 };
 684
 685 #define X86_HYBRID_PMU_ATOM_IDX         0
 686 #define X86_HYBRID_PMU_CORE_IDX         1
 687
 688 #define X86_HYBRID_NUM_PMUS             2
 689
 690 struct x86_hybrid_pmu {
 691         struct pmu                      pmu;
 692         const char                      *name;
 693         enum hybrid_pmu_type            pmu_type;
 694         cpumask_t                       supported_cpus;
 695         union perf_capabilities         intel_cap;
 696         u64                             intel_ctrl;
 697         u64                             pebs_events_mask;
 698         u64                             config_mask;
 699         union {
 700                         u64             cntr_mask64;
 701                         unsigned long   cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
 702         };
 703         union {
 704                         u64             fixed_cntr_mask64;
 705                         unsigned long   fixed_cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
 706         };
 707         struct event_constraint         unconstrained;
 708
 709         u64                             hw_cache_event_ids
 710                                         [PERF_COUNT_HW_CACHE_MAX]
 711                                         [PERF_COUNT_HW_CACHE_OP_MAX]
 712                                         [PERF_COUNT_HW_CACHE_RESULT_MAX];
 713         u64                             hw_cache_extra_regs
 714                                         [PERF_COUNT_HW_CACHE_MAX]
 715                                         [PERF_COUNT_HW_CACHE_OP_MAX]
 716                                         [PERF_COUNT_HW_CACHE_RESULT_MAX];
 717         struct event_constraint         *event_constraints;
 718         struct event_constraint         *pebs_constraints;
 719         struct extra_reg                *extra_regs;
 720
 721         unsigned int                    late_ack        :1,
 722                                         mid_ack         :1,
 723                                         enabled_ack     :1;
 724
 725         u64                             pebs_data_source[PERF_PEBS_DATA_SOURCE_MAX];
 726 };
 727
 728 static __always_inline struct x86_hybrid_pmu *hybrid_pmu(struct pmu *pmu)
 729 {
 730         return container_of(pmu, struct x86_hybrid_pmu, pmu);
 731 }
 732
 733 extern struct static_key_false perf_is_hybrid;
 734 #define is_hybrid()             static_branch_unlikely(&perf_is_hybrid)
 735
 736 #define hybrid(_pmu, _field)                            \
 737 (*({                                                    \
 738         typeof(&x86_pmu._field) __Fp = &x86_pmu._field; \
 739                                                         \
 740         if (is_hybrid() && (_pmu))                      \
 741                 __Fp = &hybrid_pmu(_pmu)->_field;       \
 742                                                         \
 743         __Fp;                                           \
 744 }))
 745
 746 #define hybrid_var(_pmu, _var)                          \
 747 (*({                                                    \
 748         typeof(&_var) __Fp = &_var;                     \
 749                                                         \
 750         if (is_hybrid() && (_pmu))                      \
 751                 __Fp = &hybrid_pmu(_pmu)->_var;         \
 752                                                         \
 753         __Fp;                                           \
 754 }))
 755
 756 #define hybrid_bit(_pmu, _field)                        \
 757 ({                                                      \
 758         bool __Fp = x86_pmu._field;                     \
 759                                                         \
 760         if (is_hybrid() && (_pmu))                      \
 761                 __Fp = hybrid_pmu(_pmu)->_field;        \
 762                                                         \
 763         __Fp;                                           \
 764 })
 765
 766 /*
 767  * struct x86_pmu - generic x86 pmu
 768  */
 769 struct x86_pmu {
 770         /*
 771          * Generic x86 PMC bits
 772          */
 773         const char      *name;
 774         int             version;
 775         int             (*handle_irq)(struct pt_regs *);
 776         void            (*disable_all)(void);
 777         void            (*enable_all)(int added);
 778         void            (*enable)(struct perf_event *);
 779         void            (*disable)(struct perf_event *);
 780         void            (*assign)(struct perf_event *event, int idx);
 781         void            (*add)(struct perf_event *);
 782         void            (*del)(struct perf_event *);
 783         void            (*read)(struct perf_event *event);
 784         int             (*set_period)(struct perf_event *event);
 785         u64             (*update)(struct perf_event *event);
 786         int             (*hw_config)(struct perf_event *event);
 787         int             (*schedule_events)(struct cpu_hw_events *cpuc, int n, int *assign);
 788         unsigned        eventsel;
 789         unsigned        perfctr;
 790         unsigned        fixedctr;
 791         int             (*addr_offset)(int index, bool eventsel);
 792         int             (*rdpmc_index)(int index);
 793         u64             (*event_map)(int);
 794         int             max_events;
 795         u64             config_mask;
 796         union {
 797                         u64             cntr_mask64;
 798                         unsigned long   cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
 799         };
 800         union {
 801                         u64             fixed_cntr_mask64;
 802                         unsigned long   fixed_cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
 803         };
 804         int             cntval_bits;
 805         u64             cntval_mask;
 806         union {
 807                         unsigned long events_maskl;
 808                         unsigned long events_mask[BITS_TO_LONGS(ARCH_PERFMON_EVENTS_COUNT)];
 809         };
 810         int             events_mask_len;
 811         int             apic;
 812         u64             max_period;
 813         struct event_constraint *
 814                         (*get_event_constraints)(struct cpu_hw_events *cpuc,
 815                                                  int idx,
 816                                                  struct perf_event *event);
 817
 818         void            (*put_event_constraints)(struct cpu_hw_events *cpuc,
 819                                                  struct perf_event *event);
 820
 821         void            (*start_scheduling)(struct cpu_hw_events *cpuc);
 822
 823         void            (*commit_scheduling)(struct cpu_hw_events *cpuc, int idx, int cntr);
 824
 825         void            (*stop_scheduling)(struct cpu_hw_events *cpuc);
 826
 827         struct event_constraint *event_constraints;
 828         struct x86_pmu_quirk *quirks;
 829         void            (*limit_period)(struct perf_event *event, s64 *l);
 830
 831         /* PMI handler bits */
 832         unsigned int    late_ack                :1,
 833                         mid_ack                 :1,
 834                         enabled_ack             :1;
 835         /*
 836          * sysfs attrs
 837          */
 838         int             attr_rdpmc_broken;
 839         int             attr_rdpmc;
 840         struct attribute **format_attrs;
 841
 842         ssize_t         (*events_sysfs_show)(char *page, u64 config);
 843         const struct attribute_group **attr_update;
 844
 845         unsigned long   attr_freeze_on_smi;
 846
 847         /*
 848          * CPU Hotplug hooks
 849          */
 850         int             (*cpu_prepare)(int cpu);
 851         void            (*cpu_starting)(int cpu);
 852         void            (*cpu_dying)(int cpu);
 853         void            (*cpu_dead)(int cpu);
 854
 855         void            (*check_microcode)(void);
 856         void            (*sched_task)(struct perf_event_pmu_context *pmu_ctx,
 857                                       bool sched_in);
 858
 859         /*
 860          * Intel Arch Perfmon v2+
 861          */
 862         u64                     intel_ctrl;
 863         union perf_capabilities intel_cap;
 864
 865         /*
 866          * Intel DebugStore bits
 867          */
 868         unsigned int    bts                     :1,
 869                         bts_active              :1,
 870                         pebs                    :1,
 871                         pebs_active             :1,
 872                         pebs_broken             :1,
 873                         pebs_prec_dist          :1,
 874                         pebs_no_tlb             :1,
 875                         pebs_no_isolation       :1,
 876                         pebs_block              :1,
 877                         pebs_ept                :1;
 878         int             pebs_record_size;
 879         int             pebs_buffer_size;
 880         u64             pebs_events_mask;
 881         void            (*drain_pebs)(struct pt_regs *regs, struct perf_sample_data *data);
 882         struct event_constraint *pebs_constraints;
 883         void            (*pebs_aliases)(struct perf_event *event);
 884         u64             (*pebs_latency_data)(struct perf_event *event, u64 status);
 885         unsigned long   large_pebs_flags;
 886         u64             rtm_abort_event;
 887         u64             pebs_capable;
 888
 889         /*
 890          * Intel LBR
 891          */
 892         unsigned int    lbr_tos, lbr_from, lbr_to,
 893                         lbr_info, lbr_nr;          /* LBR base regs and size */
 894         union {
 895                 u64     lbr_sel_mask;              /* LBR_SELECT valid bits */
 896                 u64     lbr_ctl_mask;              /* LBR_CTL valid bits */
 897         };
 898         union {
 899                 const int       *lbr_sel_map;      /* lbr_select mappings */
 900                 int             *lbr_ctl_map;      /* LBR_CTL mappings */
 901         };
 902         bool            lbr_double_abort;          /* duplicated lbr aborts */
 903         bool            lbr_pt_coexist;            /* (LBR|BTS) may coexist with PT */
 904
 905         unsigned int    lbr_has_info:1;
 906         unsigned int    lbr_has_tsx:1;
 907         unsigned int    lbr_from_flags:1;
 908         unsigned int    lbr_to_cycles:1;
 909
 910         /*
 911          * Intel Architectural LBR CPUID Enumeration
 912          */
 913         unsigned int    lbr_depth_mask:8;
 914         unsigned int    lbr_deep_c_reset:1;
 915         unsigned int    lbr_lip:1;
 916         unsigned int    lbr_cpl:1;
 917         unsigned int    lbr_filter:1;
 918         unsigned int    lbr_call_stack:1;
 919         unsigned int    lbr_mispred:1;
 920         unsigned int    lbr_timed_lbr:1;
 921         unsigned int    lbr_br_type:1;
 922         unsigned int    lbr_counters:4;
 923
 924         void            (*lbr_reset)(void);
 925         void            (*lbr_read)(struct cpu_hw_events *cpuc);
 926         void            (*lbr_save)(void *ctx);
 927         void            (*lbr_restore)(void *ctx);
 928
 929         /*
 930          * Intel PT/LBR/BTS are exclusive
 931          */
 932         atomic_t        lbr_exclusive[x86_lbr_exclusive_max];
 933
 934         /*
 935          * Intel perf metrics
 936          */
 937         int             num_topdown_events;
 938
 939         /*
 940          * perf task context (i.e. struct perf_event_pmu_context::task_ctx_data)
 941          * switch helper to bridge calls from perf/core to perf/x86.
 942          * See struct pmu::swap_task_ctx() usage for examples;
 943          */
 944         void            (*swap_task_ctx)(struct perf_event_pmu_context *prev_epc,
 945                                          struct perf_event_pmu_context *next_epc);
 946
 947         /*
 948          * AMD bits
 949          */
 950         unsigned int    amd_nb_constraints : 1;
 951         u64             perf_ctr_pair_en;
 952
 953         /*
 954          * Extra registers for events
 955          */
 956         struct extra_reg *extra_regs;
 957         unsigned int flags;
 958
 959         /*
 960          * Intel host/guest support (KVM)
 961          */
 962         struct perf_guest_switch_msr *(*guest_get_msrs)(int *nr, void *data);
 963
 964         /*
 965          * Check period value for PERF_EVENT_IOC_PERIOD ioctl.
 966          */
 967         int (*check_period) (struct perf_event *event, u64 period);
 968
 969         int (*aux_output_match) (struct perf_event *event);
 970
 971         void (*filter)(struct pmu *pmu, int cpu, bool *ret);
 972         /*
 973          * Hybrid support
 974          *
 975          * Most PMU capabilities are the same among different hybrid PMUs.
 976          * The global x86_pmu saves the architecture capabilities, which
 977          * are available for all PMUs. The hybrid_pmu only includes the
 978          * unique capabilities.
 979          */
 980         int                             num_hybrid_pmus;
 981         struct x86_hybrid_pmu           *hybrid_pmu;
 982         enum hybrid_cpu_type (*get_hybrid_cpu_type)     (void);
 983 };
 984
 985 struct x86_perf_task_context_opt {
 986         int lbr_callstack_users;
 987         int lbr_stack_state;
 988         int log_id;
 989 };
 990
 991 struct x86_perf_task_context {
 992         u64 lbr_sel;
 993         int tos;
 994         int valid_lbrs;
 995         struct x86_perf_task_context_opt opt;
 996         struct lbr_entry lbr[MAX_LBR_ENTRIES];
 997 };
 998
 999 struct x86_perf_task_context_arch_lbr {
1000         struct x86_perf_task_context_opt opt;
1001         struct lbr_entry entries[];
1002 };
1003
1004 /*
1005  * Add padding to guarantee the 64-byte alignment of the state buffer.
1006  *
1007  * The structure is dynamically allocated. The size of the LBR state may vary
1008  * based on the number of LBR registers.
1009  *
1010  * Do not put anything after the LBR state.
1011  */
1012 struct x86_perf_task_context_arch_lbr_xsave {
1013         struct x86_perf_task_context_opt                opt;
1014
1015         union {
1016                 struct xregs_state                      xsave;
1017                 struct {
1018                         struct fxregs_state             i387;
1019                         struct xstate_header            header;
1020                         struct arch_lbr_state           lbr;
1021                 } __attribute__ ((packed, aligned (XSAVE_ALIGNMENT)));
1022         };
1023 };
1024
1025 #define x86_add_quirk(func_)                                            \
1026 do {                                                                    \
1027         static struct x86_pmu_quirk __quirk __initdata = {              \
1028                 .func = func_,                                          \
1029         };                                                              \
1030         __quirk.next = x86_pmu.quirks;                                  \
1031         x86_pmu.quirks = &__quirk;                                      \
1032 } while (0)
1033
1034 /*
1035  * x86_pmu flags
1036  */
1037 #define PMU_FL_NO_HT_SHARING    0x1 /* no hyper-threading resource sharing */
1038 #define PMU_FL_HAS_RSP_1        0x2 /* has 2 equivalent offcore_rsp regs   */
1039 #define PMU_FL_EXCL_CNTRS       0x4 /* has exclusive counter requirements  */
1040 #define PMU_FL_EXCL_ENABLED     0x8 /* exclusive counter active */
1041 #define PMU_FL_PEBS_ALL         0x10 /* all events are valid PEBS events */
1042 #define PMU_FL_TFA              0x20 /* deal with TSX force abort */
1043 #define PMU_FL_PAIR             0x40 /* merge counters for large incr. events */
1044 #define PMU_FL_INSTR_LATENCY    0x80 /* Support Instruction Latency in PEBS Memory Info Record */
1045 #define PMU_FL_MEM_LOADS_AUX    0x100 /* Require an auxiliary event for the complete memory info */
1046 #define PMU_FL_RETIRE_LATENCY   0x200 /* Support Retire Latency in PEBS */
1047 #define PMU_FL_BR_CNTR          0x400 /* Support branch counter logging */
1048
1049 #define EVENT_VAR(_id)  event_attr_##_id
1050 #define EVENT_PTR(_id) &event_attr_##_id.attr.attr
1051
1052 #define EVENT_ATTR(_name, _id)                                          \
1053 static struct perf_pmu_events_attr EVENT_VAR(_id) = {                   \
1054         .attr           = __ATTR(_name, 0444, events_sysfs_show, NULL), \
1055         .id             = PERF_COUNT_HW_##_id,                          \
1056         .event_str      = NULL,                                         \
1057 };
1058
1059 #define EVENT_ATTR_STR(_name, v, str)                                   \
1060 static struct perf_pmu_events_attr event_attr_##v = {                   \
1061         .attr           = __ATTR(_name, 0444, events_sysfs_show, NULL), \
1062         .id             = 0,                                            \
1063         .event_str      = str,                                          \
1064 };
1065
1066 #define EVENT_ATTR_STR_HT(_name, v, noht, ht)                           \
1067 static struct perf_pmu_events_ht_attr event_attr_##v = {                \
1068         .attr           = __ATTR(_name, 0444, events_ht_sysfs_show, NULL),\
1069         .id             = 0,                                            \
1070         .event_str_noht = noht,                                         \
1071         .event_str_ht   = ht,                                           \
1072 }
1073
1074 #define EVENT_ATTR_STR_HYBRID(_name, v, str, _pmu)                      \
1075 static struct perf_pmu_events_hybrid_attr event_attr_##v = {            \
1076         .attr           = __ATTR(_name, 0444, events_hybrid_sysfs_show, NULL),\
1077         .id             = 0,                                            \
1078         .event_str      = str,                                          \
1079         .pmu_type       = _pmu,                                         \
1080 }
1081
1082 #define FORMAT_HYBRID_PTR(_id) (&format_attr_hybrid_##_id.attr.attr)
1083
1084 #define FORMAT_ATTR_HYBRID(_name, _pmu)                                 \
1085 static struct perf_pmu_format_hybrid_attr format_attr_hybrid_##_name = {\
1086         .attr           = __ATTR_RO(_name),                             \
1087         .pmu_type       = _pmu,                                         \
1088 }
1089
1090 struct pmu *x86_get_pmu(unsigned int cpu);
1091 extern struct x86_pmu x86_pmu __read_mostly;
1092
1093 DECLARE_STATIC_CALL(x86_pmu_set_period, *x86_pmu.set_period);
1094 DECLARE_STATIC_CALL(x86_pmu_update,     *x86_pmu.update);
1095
1096 static __always_inline struct x86_perf_task_context_opt *task_context_opt(void *ctx)
1097 {
1098         if (static_cpu_has(X86_FEATURE_ARCH_LBR))
1099                 return &((struct x86_perf_task_context_arch_lbr *)ctx)->opt;
1100
1101         return &((struct x86_perf_task_context *)ctx)->opt;
1102 }
1103
1104 static inline bool x86_pmu_has_lbr_callstack(void)
1105 {
1106         return  x86_pmu.lbr_sel_map &&
1107                 x86_pmu.lbr_sel_map[PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT] > 0;
1108 }
1109
1110 DECLARE_PER_CPU(struct cpu_hw_events, cpu_hw_events);
1111 DECLARE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
1112
1113 int x86_perf_event_set_period(struct perf_event *event);
1114
1115 /*
1116  * Generalized hw caching related hw_event table, filled
1117  * in on a per model basis. A value of 0 means
1118  * 'not supported', -1 means 'hw_event makes no sense on
1119  * this CPU', any other value means the raw hw_event
1120  * ID.
1121  */
1122
1123 #define C(x) PERF_COUNT_HW_CACHE_##x
1124
1125 extern u64 __read_mostly hw_cache_event_ids
1126                                 [PERF_COUNT_HW_CACHE_MAX]
1127                                 [PERF_COUNT_HW_CACHE_OP_MAX]
1128                                 [PERF_COUNT_HW_CACHE_RESULT_MAX];
1129 extern u64 __read_mostly hw_cache_extra_regs
1130                                 [PERF_COUNT_HW_CACHE_MAX]
1131                                 [PERF_COUNT_HW_CACHE_OP_MAX]
1132                                 [PERF_COUNT_HW_CACHE_RESULT_MAX];
1133
1134 u64 x86_perf_event_update(struct perf_event *event);
1135
1136 static inline unsigned int x86_pmu_config_addr(int index)
1137 {
1138         return x86_pmu.eventsel + (x86_pmu.addr_offset ?
1139                                    x86_pmu.addr_offset(index, true) : index);
1140 }
1141
1142 static inline unsigned int x86_pmu_event_addr(int index)
1143 {
1144         return x86_pmu.perfctr + (x86_pmu.addr_offset ?
1145                                   x86_pmu.addr_offset(index, false) : index);
1146 }
1147
1148 static inline unsigned int x86_pmu_fixed_ctr_addr(int index)
1149 {
1150         return x86_pmu.fixedctr + (x86_pmu.addr_offset ?
1151                                    x86_pmu.addr_offset(index, false) : index);
1152 }
1153
1154 static inline int x86_pmu_rdpmc_index(int index)
1155 {
1156         return x86_pmu.rdpmc_index ? x86_pmu.rdpmc_index(index) : index;
1157 }
1158
1159 bool check_hw_exists(struct pmu *pmu, unsigned long *cntr_mask,
1160                      unsigned long *fixed_cntr_mask);
1161
1162 int x86_add_exclusive(unsigned int what);
1163
1164 void x86_del_exclusive(unsigned int what);
1165
1166 int x86_reserve_hardware(void);
1167
1168 void x86_release_hardware(void);
1169
1170 int x86_pmu_max_precise(void);
1171
1172 void hw_perf_lbr_event_destroy(struct perf_event *event);
1173
1174 int x86_setup_perfctr(struct perf_event *event);
1175
1176 int x86_pmu_hw_config(struct perf_event *event);
1177
1178 void x86_pmu_disable_all(void);
1179
1180 static inline bool has_amd_brs(struct hw_perf_event *hwc)
1181 {
1182         return hwc->flags & PERF_X86_EVENT_AMD_BRS;
1183 }
1184
1185 static inline bool is_counter_pair(struct hw_perf_event *hwc)
1186 {
1187         return hwc->flags & PERF_X86_EVENT_PAIR;
1188 }
1189
1190 static inline void __x86_pmu_enable_event(struct hw_perf_event *hwc,
1191                                           u64 enable_mask)
1192 {
1193         u64 disable_mask = __this_cpu_read(cpu_hw_events.perf_ctr_virt_mask);
1194
1195         if (hwc->extra_reg.reg)
1196                 wrmsrl(hwc->extra_reg.reg, hwc->extra_reg.config);
1197
1198         /*
1199          * Add enabled Merge event on next counter
1200          * if large increment event being enabled on this counter
1201          */
1202         if (is_counter_pair(hwc))
1203                 wrmsrl(x86_pmu_config_addr(hwc->idx + 1), x86_pmu.perf_ctr_pair_en);
1204
1205         wrmsrl(hwc->config_base, (hwc->config | enable_mask) & ~disable_mask);
1206 }
1207
1208 void x86_pmu_enable_all(int added);
1209
1210 int perf_assign_events(struct event_constraint **constraints, int n,
1211                         int wmin, int wmax, int gpmax, int *assign);
1212 int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign);
1213
1214 void x86_pmu_stop(struct perf_event *event, int flags);
1215
1216 static inline void x86_pmu_disable_event(struct perf_event *event)
1217 {
1218         u64 disable_mask = __this_cpu_read(cpu_hw_events.perf_ctr_virt_mask);
1219         struct hw_perf_event *hwc = &event->hw;
1220
1221         wrmsrl(hwc->config_base, hwc->config & ~disable_mask);
1222
1223         if (is_counter_pair(hwc))
1224                 wrmsrl(x86_pmu_config_addr(hwc->idx + 1), 0);
1225 }
1226
1227 void x86_pmu_enable_event(struct perf_event *event);
1228
1229 int x86_pmu_handle_irq(struct pt_regs *regs);
1230
1231 void x86_pmu_show_pmu_cap(struct pmu *pmu);
1232
1233 static inline int x86_pmu_num_counters(struct pmu *pmu)
1234 {
1235         return hweight64(hybrid(pmu, cntr_mask64));
1236 }
1237
1238 static inline int x86_pmu_max_num_counters(struct pmu *pmu)
1239 {
1240         return fls64(hybrid(pmu, cntr_mask64));
1241 }
1242
1243 static inline int x86_pmu_num_counters_fixed(struct pmu *pmu)
1244 {
1245         return hweight64(hybrid(pmu, fixed_cntr_mask64));
1246 }
1247
1248 static inline int x86_pmu_max_num_counters_fixed(struct pmu *pmu)
1249 {
1250         return fls64(hybrid(pmu, fixed_cntr_mask64));
1251 }
1252
1253 static inline u64 x86_pmu_get_event_config(struct perf_event *event)
1254 {
1255         return event->attr.config & hybrid(event->pmu, config_mask);
1256 }
1257
1258 extern struct event_constraint emptyconstraint;
1259
1260 extern struct event_constraint unconstrained;
1261
1262 static inline bool kernel_ip(unsigned long ip)
1263 {
1264 #ifdef CONFIG_X86_32
1265         return ip > PAGE_OFFSET;
1266 #else
1267         return (long)ip < 0;
1268 #endif
1269 }
1270
1271 /*
1272  * Not all PMUs provide the right context information to place the reported IP
1273  * into full context. Specifically segment registers are typically not
1274  * supplied.
1275  *
1276  * Assuming the address is a linear address (it is for IBS), we fake the CS and
1277  * vm86 mode using the known zero-based code segment and 'fix up' the registers
1278  * to reflect this.
1279  *
1280  * Intel PEBS/LBR appear to typically provide the effective address, nothing
1281  * much we can do about that but pray and treat it like a linear address.
1282  */
1283 static inline void set_linear_ip(struct pt_regs *regs, unsigned long ip)
1284 {
1285         regs->cs = kernel_ip(ip) ? __KERNEL_CS : __USER_CS;
1286         if (regs->flags & X86_VM_MASK)
1287                 regs->flags ^= (PERF_EFLAGS_VM | X86_VM_MASK);
1288         regs->ip = ip;
1289 }
1290
1291 /*
1292  * x86control flow change classification
1293  * x86control flow changes include branches, interrupts, traps, faults
1294  */
1295 enum {
1296         X86_BR_NONE             = 0,      /* unknown */
1297
1298         X86_BR_USER             = 1 << 0, /* branch target is user */
1299         X86_BR_KERNEL           = 1 << 1, /* branch target is kernel */
1300
1301         X86_BR_CALL             = 1 << 2, /* call */
1302         X86_BR_RET              = 1 << 3, /* return */
1303         X86_BR_SYSCALL          = 1 << 4, /* syscall */
1304         X86_BR_SYSRET           = 1 << 5, /* syscall return */
1305         X86_BR_INT              = 1 << 6, /* sw interrupt */
1306         X86_BR_IRET             = 1 << 7, /* return from interrupt */
1307         X86_BR_JCC              = 1 << 8, /* conditional */
1308         X86_BR_JMP              = 1 << 9, /* jump */
1309         X86_BR_IRQ              = 1 << 10,/* hw interrupt or trap or fault */
1310         X86_BR_IND_CALL         = 1 << 11,/* indirect calls */
1311         X86_BR_ABORT            = 1 << 12,/* transaction abort */
1312         X86_BR_IN_TX            = 1 << 13,/* in transaction */
1313         X86_BR_NO_TX            = 1 << 14,/* not in transaction */
1314         X86_BR_ZERO_CALL        = 1 << 15,/* zero length call */
1315         X86_BR_CALL_STACK       = 1 << 16,/* call stack */
1316         X86_BR_IND_JMP          = 1 << 17,/* indirect jump */
1317
1318         X86_BR_TYPE_SAVE        = 1 << 18,/* indicate to save branch type */
1319
1320 };
1321
1322 #define X86_BR_PLM (X86_BR_USER | X86_BR_KERNEL)
1323 #define X86_BR_ANYTX (X86_BR_NO_TX | X86_BR_IN_TX)
1324
1325 #define X86_BR_ANY       \
1326         (X86_BR_CALL    |\
1327          X86_BR_RET     |\
1328          X86_BR_SYSCALL |\
1329          X86_BR_SYSRET  |\
1330          X86_BR_INT     |\
1331          X86_BR_IRET    |\
1332          X86_BR_JCC     |\
1333          X86_BR_JMP      |\
1334          X86_BR_IRQ      |\
1335          X86_BR_ABORT    |\
1336          X86_BR_IND_CALL |\
1337          X86_BR_IND_JMP  |\
1338          X86_BR_ZERO_CALL)
1339
1340 #define X86_BR_ALL (X86_BR_PLM | X86_BR_ANY)
1341
1342 #define X86_BR_ANY_CALL          \
1343         (X86_BR_CALL            |\
1344          X86_BR_IND_CALL        |\
1345          X86_BR_ZERO_CALL       |\
1346          X86_BR_SYSCALL         |\
1347          X86_BR_IRQ             |\
1348          X86_BR_INT)
1349
1350 int common_branch_type(int type);
1351 int branch_type(unsigned long from, unsigned long to, int abort);
1352 int branch_type_fused(unsigned long from, unsigned long to, int abort,
1353                       int *offset);
1354
1355 ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event);
1356 ssize_t intel_event_sysfs_show(char *page, u64 config);
1357
1358 ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr,
1359                           char *page);
1360 ssize_t events_ht_sysfs_show(struct device *dev, struct device_attribute *attr,
1361                           char *page);
1362 ssize_t events_hybrid_sysfs_show(struct device *dev,
1363                                  struct device_attribute *attr,
1364                                  char *page);
1365
1366 static inline bool fixed_counter_disabled(int i, struct pmu *pmu)
1367 {
1368         u64 intel_ctrl = hybrid(pmu, intel_ctrl);
1369
1370         return !(intel_ctrl >> (i + INTEL_PMC_IDX_FIXED));
1371 }
1372
1373 #ifdef CONFIG_CPU_SUP_AMD
1374
1375 int amd_pmu_init(void);
1376
1377 int amd_pmu_lbr_init(void);
1378 void amd_pmu_lbr_reset(void);
1379 void amd_pmu_lbr_read(void);
1380 void amd_pmu_lbr_add(struct perf_event *event);
1381 void amd_pmu_lbr_del(struct perf_event *event);
1382 void amd_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in);
1383 void amd_pmu_lbr_enable_all(void);
1384 void amd_pmu_lbr_disable_all(void);
1385 int amd_pmu_lbr_hw_config(struct perf_event *event);
1386
1387 static __always_inline void __amd_pmu_lbr_disable(void)
1388 {
1389         u64 dbg_ctl, dbg_extn_cfg;
1390
1391         rdmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg);
1392         wrmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg & ~DBG_EXTN_CFG_LBRV2EN);
1393
1394         if (cpu_feature_enabled(X86_FEATURE_AMD_LBR_PMC_FREEZE)) {
1395                 rdmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl);
1396                 wrmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl & ~DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
1397         }
1398 }
1399
1400 #ifdef CONFIG_PERF_EVENTS_AMD_BRS
1401
1402 #define AMD_FAM19H_BRS_EVENT 0xc4 /* RETIRED_TAKEN_BRANCH_INSTRUCTIONS */
1403
1404 int amd_brs_init(void);
1405 void amd_brs_disable(void);
1406 void amd_brs_enable(void);
1407 void amd_brs_enable_all(void);
1408 void amd_brs_disable_all(void);
1409 void amd_brs_drain(void);
1410 void amd_brs_lopwr_init(void);
1411 int amd_brs_hw_config(struct perf_event *event);
1412 void amd_brs_reset(void);
1413
1414 static inline void amd_pmu_brs_add(struct perf_event *event)
1415 {
1416         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1417
1418         perf_sched_cb_inc(event->pmu);
1419         cpuc->lbr_users++;
1420         /*
1421          * No need to reset BRS because it is reset
1422          * on brs_enable() and it is saturating
1423          */
1424 }
1425
1426 static inline void amd_pmu_brs_del(struct perf_event *event)
1427 {
1428         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1429
1430         cpuc->lbr_users--;
1431         WARN_ON_ONCE(cpuc->lbr_users < 0);
1432
1433         perf_sched_cb_dec(event->pmu);
1434 }
1435
1436 void amd_pmu_brs_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in);
1437 #else
1438 static inline int amd_brs_init(void)
1439 {
1440         return 0;
1441 }
1442 static inline void amd_brs_disable(void) {}
1443 static inline void amd_brs_enable(void) {}
1444 static inline void amd_brs_drain(void) {}
1445 static inline void amd_brs_lopwr_init(void) {}
1446 static inline void amd_brs_disable_all(void) {}
1447 static inline int amd_brs_hw_config(struct perf_event *event)
1448 {
1449         return 0;
1450 }
1451 static inline void amd_brs_reset(void) {}
1452
1453 static inline void amd_pmu_brs_add(struct perf_event *event)
1454 {
1455 }
1456
1457 static inline void amd_pmu_brs_del(struct perf_event *event)
1458 {
1459 }
1460
1461 static inline void amd_pmu_brs_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in)
1462 {
1463 }
1464
1465 static inline void amd_brs_enable_all(void)
1466 {
1467 }
1468
1469 #endif
1470
1471 #else /* CONFIG_CPU_SUP_AMD */
1472
1473 static inline int amd_pmu_init(void)
1474 {
1475         return 0;
1476 }
1477
1478 static inline int amd_brs_init(void)
1479 {
1480         return -EOPNOTSUPP;
1481 }
1482
1483 static inline void amd_brs_drain(void)
1484 {
1485 }
1486
1487 static inline void amd_brs_enable_all(void)
1488 {
1489 }
1490
1491 static inline void amd_brs_disable_all(void)
1492 {
1493 }
1494 #endif /* CONFIG_CPU_SUP_AMD */
1495
1496 static inline int is_pebs_pt(struct perf_event *event)
1497 {
1498         return !!(event->hw.flags & PERF_X86_EVENT_PEBS_VIA_PT);
1499 }
1500
1501 #ifdef CONFIG_CPU_SUP_INTEL
1502
1503 static inline bool intel_pmu_has_bts_period(struct perf_event *event, u64 period)
1504 {
1505         struct hw_perf_event *hwc = &event->hw;
1506         unsigned int hw_event, bts_event;
1507
1508         if (event->attr.freq)
1509                 return false;
1510
1511         hw_event = hwc->config & INTEL_ARCH_EVENT_MASK;
1512         bts_event = x86_pmu.event_map(PERF_COUNT_HW_BRANCH_INSTRUCTIONS);
1513
1514         return hw_event == bts_event && period == 1;
1515 }
1516
1517 static inline bool intel_pmu_has_bts(struct perf_event *event)
1518 {
1519         struct hw_perf_event *hwc = &event->hw;
1520
1521         return intel_pmu_has_bts_period(event, hwc->sample_period);
1522 }
1523
1524 static __always_inline void __intel_pmu_pebs_disable_all(void)
1525 {
1526         wrmsrl(MSR_IA32_PEBS_ENABLE, 0);
1527 }
1528
1529 static __always_inline void __intel_pmu_arch_lbr_disable(void)
1530 {
1531         wrmsrl(MSR_ARCH_LBR_CTL, 0);
1532 }
1533
1534 static __always_inline void __intel_pmu_lbr_disable(void)
1535 {
1536         u64 debugctl;
1537
1538         rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
1539         debugctl &= ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
1540         wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
1541 }
1542
1543 int intel_pmu_save_and_restart(struct perf_event *event);
1544
1545 struct event_constraint *
1546 x86_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
1547                           struct perf_event *event);
1548
1549 extern int intel_cpuc_prepare(struct cpu_hw_events *cpuc, int cpu);
1550 extern void intel_cpuc_finish(struct cpu_hw_events *cpuc);
1551
1552 int intel_pmu_init(void);
1553
1554 void init_debug_store_on_cpu(int cpu);
1555
1556 void fini_debug_store_on_cpu(int cpu);
1557
1558 void release_ds_buffers(void);
1559
1560 void reserve_ds_buffers(void);
1561
1562 void release_lbr_buffers(void);
1563
1564 void reserve_lbr_buffers(void);
1565
1566 extern struct event_constraint bts_constraint;
1567 extern struct event_constraint vlbr_constraint;
1568
1569 void intel_pmu_enable_bts(u64 config);
1570
1571 void intel_pmu_disable_bts(void);
1572
1573 int intel_pmu_drain_bts_buffer(void);
1574
1575 u64 grt_latency_data(struct perf_event *event, u64 status);
1576
1577 u64 cmt_latency_data(struct perf_event *event, u64 status);
1578
1579 u64 lnl_latency_data(struct perf_event *event, u64 status);
1580
1581 extern struct event_constraint intel_core2_pebs_event_constraints[];
1582
1583 extern struct event_constraint intel_atom_pebs_event_constraints[];
1584
1585 extern struct event_constraint intel_slm_pebs_event_constraints[];
1586
1587 extern struct event_constraint intel_glm_pebs_event_constraints[];
1588
1589 extern struct event_constraint intel_glp_pebs_event_constraints[];
1590
1591 extern struct event_constraint intel_grt_pebs_event_constraints[];
1592
1593 extern struct event_constraint intel_nehalem_pebs_event_constraints[];
1594
1595 extern struct event_constraint intel_westmere_pebs_event_constraints[];
1596
1597 extern struct event_constraint intel_snb_pebs_event_constraints[];
1598
1599 extern struct event_constraint intel_ivb_pebs_event_constraints[];
1600
1601 extern struct event_constraint intel_hsw_pebs_event_constraints[];
1602
1603 extern struct event_constraint intel_bdw_pebs_event_constraints[];
1604
1605 extern struct event_constraint intel_skl_pebs_event_constraints[];
1606
1607 extern struct event_constraint intel_icl_pebs_event_constraints[];
1608
1609 extern struct event_constraint intel_glc_pebs_event_constraints[];
1610
1611 extern struct event_constraint intel_lnc_pebs_event_constraints[];
1612
1613 struct event_constraint *intel_pebs_constraints(struct perf_event *event);
1614
1615 void intel_pmu_pebs_add(struct perf_event *event);
1616
1617 void intel_pmu_pebs_del(struct perf_event *event);
1618
1619 void intel_pmu_pebs_enable(struct perf_event *event);
1620
1621 void intel_pmu_pebs_disable(struct perf_event *event);
1622
1623 void intel_pmu_pebs_enable_all(void);
1624
1625 void intel_pmu_pebs_disable_all(void);
1626
1627 void intel_pmu_pebs_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in);
1628
1629 void intel_pmu_auto_reload_read(struct perf_event *event);
1630
1631 void intel_pmu_store_pebs_lbrs(struct lbr_entry *lbr);
1632
1633 void intel_ds_init(void);
1634
1635 void intel_pmu_lbr_save_brstack(struct perf_sample_data *data,
1636                                 struct cpu_hw_events *cpuc,
1637                                 struct perf_event *event);
1638
1639 void intel_pmu_lbr_swap_task_ctx(struct perf_event_pmu_context *prev_epc,
1640                                  struct perf_event_pmu_context *next_epc);
1641
1642 void intel_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in);
1643
1644 u64 lbr_from_signext_quirk_wr(u64 val);
1645
1646 void intel_pmu_lbr_reset(void);
1647
1648 void intel_pmu_lbr_reset_32(void);
1649
1650 void intel_pmu_lbr_reset_64(void);
1651
1652 void intel_pmu_lbr_add(struct perf_event *event);
1653
1654 void intel_pmu_lbr_del(struct perf_event *event);
1655
1656 void intel_pmu_lbr_enable_all(bool pmi);
1657
1658 void intel_pmu_lbr_disable_all(void);
1659
1660 void intel_pmu_lbr_read(void);
1661
1662 void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc);
1663
1664 void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc);
1665
1666 void intel_pmu_lbr_save(void *ctx);
1667
1668 void intel_pmu_lbr_restore(void *ctx);
1669
1670 void intel_pmu_lbr_init_core(void);
1671
1672 void intel_pmu_lbr_init_nhm(void);
1673
1674 void intel_pmu_lbr_init_atom(void);
1675
1676 void intel_pmu_lbr_init_slm(void);
1677
1678 void intel_pmu_lbr_init_snb(void);
1679
1680 void intel_pmu_lbr_init_hsw(void);
1681
1682 void intel_pmu_lbr_init_skl(void);
1683
1684 void intel_pmu_lbr_init_knl(void);
1685
1686 void intel_pmu_lbr_init(void);
1687
1688 void intel_pmu_arch_lbr_init(void);
1689
1690 void intel_pmu_pebs_data_source_nhm(void);
1691
1692 void intel_pmu_pebs_data_source_skl(bool pmem);
1693
1694 void intel_pmu_pebs_data_source_adl(void);
1695
1696 void intel_pmu_pebs_data_source_grt(void);
1697
1698 void intel_pmu_pebs_data_source_mtl(void);
1699
1700 void intel_pmu_pebs_data_source_cmt(void);
1701
1702 void intel_pmu_pebs_data_source_lnl(void);
1703
1704 int intel_pmu_setup_lbr_filter(struct perf_event *event);
1705
1706 void intel_pt_interrupt(void);
1707
1708 int intel_bts_interrupt(void);
1709
1710 void intel_bts_enable_local(void);
1711
1712 void intel_bts_disable_local(void);
1713
1714 int p4_pmu_init(void);
1715
1716 int p6_pmu_init(void);
1717
1718 int knc_pmu_init(void);
1719
1720 static inline int is_ht_workaround_enabled(void)
1721 {
1722         return !!(x86_pmu.flags & PMU_FL_EXCL_ENABLED);
1723 }
1724
1725 static inline u64 intel_pmu_pebs_mask(u64 cntr_mask)
1726 {
1727         return MAX_PEBS_EVENTS_MASK & cntr_mask;
1728 }
1729
1730 static inline int intel_pmu_max_num_pebs(struct pmu *pmu)
1731 {
1732         static_assert(MAX_PEBS_EVENTS == 32);
1733         return fls((u32)hybrid(pmu, pebs_events_mask));
1734 }
1735
1736 #else /* CONFIG_CPU_SUP_INTEL */
1737
1738 static inline void reserve_ds_buffers(void)
1739 {
1740 }
1741
1742 static inline void release_ds_buffers(void)
1743 {
1744 }
1745
1746 static inline void release_lbr_buffers(void)
1747 {
1748 }
1749
1750 static inline void reserve_lbr_buffers(void)
1751 {
1752 }
1753
1754 static inline int intel_pmu_init(void)
1755 {
1756         return 0;
1757 }
1758
1759 static inline int intel_cpuc_prepare(struct cpu_hw_events *cpuc, int cpu)
1760 {
1761         return 0;
1762 }
1763
1764 static inline void intel_cpuc_finish(struct cpu_hw_events *cpuc)
1765 {
1766 }
1767
1768 static inline int is_ht_workaround_enabled(void)
1769 {
1770         return 0;
1771 }
1772 #endif /* CONFIG_CPU_SUP_INTEL */
1773
1774 #if ((defined CONFIG_CPU_SUP_CENTAUR) || (defined CONFIG_CPU_SUP_ZHAOXIN))
1775 int zhaoxin_pmu_init(void);
1776 #else
1777 static inline int zhaoxin_pmu_init(void)
1778 {
1779         return 0;
1780 }
1781 #endif /*CONFIG_CPU_SUP_CENTAUR or CONFIG_CPU_SUP_ZHAOXIN*/