arch/x86/kernel/cpu/perf_event_intel_lbr.c

   1 #include <linux/perf_event.h>
   2 #include <linux/types.h>
   3
   4 #include <asm/perf_event.h>
   5 #include <asm/msr.h>
   6
   7 #include "perf_event.h"
   8
   9 enum {
  10         LBR_FORMAT_32           = 0x00,
  11         LBR_FORMAT_LIP          = 0x01,
  12         LBR_FORMAT_EIP          = 0x02,
  13         LBR_FORMAT_EIP_FLAGS    = 0x03,
  14 };
  15
  16 /*
  17  * We only support LBR implementations that have FREEZE_LBRS_ON_PMI
  18  * otherwise it becomes near impossible to get a reliable stack.
  19  */
  20
  21 static void __intel_pmu_lbr_enable(void)
  22 {
  23         u64 debugctl;
  24
  25         rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
  26         debugctl |= (DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
  27         wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
  28 }
  29
  30 static void __intel_pmu_lbr_disable(void)
  31 {
  32         u64 debugctl;
  33
  34         rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
  35         debugctl &= ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
  36         wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
  37 }
  38
  39 static void intel_pmu_lbr_reset_32(void)
  40 {
  41         int i;
  42
  43         for (i = 0; i < x86_pmu.lbr_nr; i++)
  44                 wrmsrl(x86_pmu.lbr_from + i, 0);
  45 }
  46
  47 static void intel_pmu_lbr_reset_64(void)
  48 {
  49         int i;
  50
  51         for (i = 0; i < x86_pmu.lbr_nr; i++) {
  52                 wrmsrl(x86_pmu.lbr_from + i, 0);
  53                 wrmsrl(x86_pmu.lbr_to   + i, 0);
  54         }
  55 }
  56
  57 void intel_pmu_lbr_reset(void)
  58 {
  59         if (!x86_pmu.lbr_nr)
  60                 return;
  61
  62         if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32)
  63                 intel_pmu_lbr_reset_32();
  64         else
  65                 intel_pmu_lbr_reset_64();
  66 }
  67
  68 void intel_pmu_lbr_enable(struct perf_event *event)
  69 {
  70         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  71
  72         if (!x86_pmu.lbr_nr)
  73                 return;
  74
  75         /*
  76          * Reset the LBR stack if we changed task context to
  77          * avoid data leaks.
  78          */
  79
  80         if (event->ctx->task && cpuc->lbr_context != event->ctx) {
  81                 intel_pmu_lbr_reset();
  82                 cpuc->lbr_context = event->ctx;
  83         }
  84
  85         cpuc->lbr_users++;
  86 }
  87
  88 void intel_pmu_lbr_disable(struct perf_event *event)
  89 {
  90         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  91
  92         if (!x86_pmu.lbr_nr)
  93                 return;
  94
  95         cpuc->lbr_users--;
  96         WARN_ON_ONCE(cpuc->lbr_users < 0);
  97
  98         if (cpuc->enabled && !cpuc->lbr_users)
  99                 __intel_pmu_lbr_disable();
 100 }
 101
 102 void intel_pmu_lbr_enable_all(void)
 103 {
 104         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 105
 106         if (cpuc->lbr_users)
 107                 __intel_pmu_lbr_enable();
 108 }
 109
 110 void intel_pmu_lbr_disable_all(void)
 111 {
 112         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 113
 114         if (cpuc->lbr_users)
 115                 __intel_pmu_lbr_disable();
 116 }
 117
 118 static inline u64 intel_pmu_lbr_tos(void)
 119 {
 120         u64 tos;
 121
 122         rdmsrl(x86_pmu.lbr_tos, tos);
 123
 124         return tos;
 125 }
 126
 127 static void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)
 128 {
 129         unsigned long mask = x86_pmu.lbr_nr - 1;
 130         u64 tos = intel_pmu_lbr_tos();
 131         int i;
 132
 133         for (i = 0; i < x86_pmu.lbr_nr; i++) {
 134                 unsigned long lbr_idx = (tos - i) & mask;
 135                 union {
 136                         struct {
 137                                 u32 from;
 138                                 u32 to;
 139                         };
 140                         u64     lbr;
 141                 } msr_lastbranch;
 142
 143                 rdmsrl(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr);
 144
 145                 cpuc->lbr_entries[i].from  = msr_lastbranch.from;
 146                 cpuc->lbr_entries[i].to    = msr_lastbranch.to;
 147                 cpuc->lbr_entries[i].flags = 0;
 148         }
 149         cpuc->lbr_stack.nr = i;
 150 }
 151
 152 #define LBR_FROM_FLAG_MISPRED  (1ULL << 63)
 153
 154 /*
 155  * Due to lack of segmentation in Linux the effective address (offset)
 156  * is the same as the linear address, allowing us to merge the LIP and EIP
 157  * LBR formats.
 158  */
 159 static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)
 160 {
 161         unsigned long mask = x86_pmu.lbr_nr - 1;
 162         int lbr_format = x86_pmu.intel_cap.lbr_format;
 163         u64 tos = intel_pmu_lbr_tos();
 164         int i;
 165
 166         for (i = 0; i < x86_pmu.lbr_nr; i++) {
 167                 unsigned long lbr_idx = (tos - i) & mask;
 168                 u64 from, to, flags = 0;
 169
 170                 rdmsrl(x86_pmu.lbr_from + lbr_idx, from);
 171                 rdmsrl(x86_pmu.lbr_to   + lbr_idx, to);
 172
 173                 if (lbr_format == LBR_FORMAT_EIP_FLAGS) {
 174                         flags = !!(from & LBR_FROM_FLAG_MISPRED);
 175                         from = (u64)((((s64)from) << 1) >> 1);
 176                 }
 177
 178                 cpuc->lbr_entries[i].from  = from;
 179                 cpuc->lbr_entries[i].to    = to;
 180                 cpuc->lbr_entries[i].flags = flags;
 181         }
 182         cpuc->lbr_stack.nr = i;
 183 }
 184
 185 void intel_pmu_lbr_read(void)
 186 {
 187         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 188
 189         if (!cpuc->lbr_users)
 190                 return;
 191
 192         if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32)
 193                 intel_pmu_lbr_read_32(cpuc);
 194         else
 195                 intel_pmu_lbr_read_64(cpuc);
 196 }
 197
 198 void intel_pmu_lbr_init_core(void)
 199 {
 200         x86_pmu.lbr_nr     = 4;
 201         x86_pmu.lbr_tos    = 0x01c9;
 202         x86_pmu.lbr_from   = 0x40;
 203         x86_pmu.lbr_to     = 0x60;
 204 }
 205
 206 void intel_pmu_lbr_init_nhm(void)
 207 {
 208         x86_pmu.lbr_nr     = 16;
 209         x86_pmu.lbr_tos    = 0x01c9;
 210         x86_pmu.lbr_from   = 0x680;
 211         x86_pmu.lbr_to     = 0x6c0;
 212 }
 213
 214 void intel_pmu_lbr_init_atom(void)
 215 {
 216         x86_pmu.lbr_nr     = 8;
 217         x86_pmu.lbr_tos    = 0x01c9;
 218         x86_pmu.lbr_from   = 0x40;
 219         x86_pmu.lbr_to     = 0x60;
 220 }