arch/x86/xen/pmu.c

   1 #include <linux/types.h>
   2 #include <linux/interrupt.h>
   3
   4 #include <asm/xen/hypercall.h>
   5 #include <xen/page.h>
   6 #include <xen/interface/xen.h>
   7 #include <xen/interface/vcpu.h>
   8 #include <xen/interface/xenpmu.h>
   9
  10 #include "xen-ops.h"
  11 #include "pmu.h"
  12
  13 /* x86_pmu.handle_irq definition */
  14 #include "../events/perf_event.h"
  15
  16 #define XENPMU_IRQ_PROCESSING    1
  17 struct xenpmu {
  18         /* Shared page between hypervisor and domain */
  19         struct xen_pmu_data *xenpmu_data;
  20
  21         uint8_t flags;
  22 };
  23 static DEFINE_PER_CPU(struct xenpmu, xenpmu_shared);
  24 #define get_xenpmu_data()    (this_cpu_ptr(&xenpmu_shared)->xenpmu_data)
  25 #define get_xenpmu_flags()   (this_cpu_ptr(&xenpmu_shared)->flags)
  26
  27 /* Macro for computing address of a PMU MSR bank */
  28 #define field_offset(ctxt, field) ((void *)((uintptr_t)ctxt + \
  29                                             (uintptr_t)ctxt->field))
  30
  31 /* AMD PMU */
  32 #define F15H_NUM_COUNTERS   6
  33 #define F10H_NUM_COUNTERS   4
  34
  35 static __read_mostly uint32_t amd_counters_base;
  36 static __read_mostly uint32_t amd_ctrls_base;
  37 static __read_mostly int amd_msr_step;
  38 static __read_mostly int k7_counters_mirrored;
  39 static __read_mostly int amd_num_counters;
  40
  41 /* Intel PMU */
  42 #define MSR_TYPE_COUNTER            0
  43 #define MSR_TYPE_CTRL               1
  44 #define MSR_TYPE_GLOBAL             2
  45 #define MSR_TYPE_ARCH_COUNTER       3
  46 #define MSR_TYPE_ARCH_CTRL          4
  47
  48 /* Number of general pmu registers (CPUID.EAX[0xa].EAX[8..15]) */
  49 #define PMU_GENERAL_NR_SHIFT        8
  50 #define PMU_GENERAL_NR_BITS         8
  51 #define PMU_GENERAL_NR_MASK         (((1 << PMU_GENERAL_NR_BITS) - 1) \
  52                                      << PMU_GENERAL_NR_SHIFT)
  53
  54 /* Number of fixed pmu registers (CPUID.EDX[0xa].EDX[0..4]) */
  55 #define PMU_FIXED_NR_SHIFT          0
  56 #define PMU_FIXED_NR_BITS           5
  57 #define PMU_FIXED_NR_MASK           (((1 << PMU_FIXED_NR_BITS) - 1) \
  58                                      << PMU_FIXED_NR_SHIFT)
  59
  60 /* Alias registers (0x4c1) for full-width writes to PMCs */
  61 #define MSR_PMC_ALIAS_MASK          (~(MSR_IA32_PERFCTR0 ^ MSR_IA32_PMC0))
  62
  63 #define INTEL_PMC_TYPE_SHIFT        30
  64
  65 static __read_mostly int intel_num_arch_counters, intel_num_fixed_counters;
  66
  67
  68 static void xen_pmu_arch_init(void)
  69 {
  70         if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
  71
  72                 switch (boot_cpu_data.x86) {
  73                 case 0x15:
  74                         amd_num_counters = F15H_NUM_COUNTERS;
  75                         amd_counters_base = MSR_F15H_PERF_CTR;
  76                         amd_ctrls_base = MSR_F15H_PERF_CTL;
  77                         amd_msr_step = 2;
  78                         k7_counters_mirrored = 1;
  79                         break;
  80                 case 0x10:
  81                 case 0x12:
  82                 case 0x14:
  83                 case 0x16:
  84                 default:
  85                         amd_num_counters = F10H_NUM_COUNTERS;
  86                         amd_counters_base = MSR_K7_PERFCTR0;
  87                         amd_ctrls_base = MSR_K7_EVNTSEL0;
  88                         amd_msr_step = 1;
  89                         k7_counters_mirrored = 0;
  90                         break;
  91                 }
  92         } else {
  93                 uint32_t eax, ebx, ecx, edx;
  94
  95                 cpuid(0xa, &eax, &ebx, &ecx, &edx);
  96
  97                 intel_num_arch_counters = (eax & PMU_GENERAL_NR_MASK) >>
  98                         PMU_GENERAL_NR_SHIFT;
  99                 intel_num_fixed_counters = (edx & PMU_FIXED_NR_MASK) >>
 100                         PMU_FIXED_NR_SHIFT;
 101         }
 102 }
 103
 104 static inline uint32_t get_fam15h_addr(u32 addr)
 105 {
 106         switch (addr) {
 107         case MSR_K7_PERFCTR0:
 108         case MSR_K7_PERFCTR1:
 109         case MSR_K7_PERFCTR2:
 110         case MSR_K7_PERFCTR3:
 111                 return MSR_F15H_PERF_CTR + (addr - MSR_K7_PERFCTR0);
 112         case MSR_K7_EVNTSEL0:
 113         case MSR_K7_EVNTSEL1:
 114         case MSR_K7_EVNTSEL2:
 115         case MSR_K7_EVNTSEL3:
 116                 return MSR_F15H_PERF_CTL + (addr - MSR_K7_EVNTSEL0);
 117         default:
 118                 break;
 119         }
 120
 121         return addr;
 122 }
 123
 124 static inline bool is_amd_pmu_msr(unsigned int msr)
 125 {
 126         if ((msr >= MSR_F15H_PERF_CTL &&
 127              msr < MSR_F15H_PERF_CTR + (amd_num_counters * 2)) ||
 128             (msr >= MSR_K7_EVNTSEL0 &&
 129              msr < MSR_K7_PERFCTR0 + amd_num_counters))
 130                 return true;
 131
 132         return false;
 133 }
 134
 135 static int is_intel_pmu_msr(u32 msr_index, int *type, int *index)
 136 {
 137         u32 msr_index_pmc;
 138
 139         switch (msr_index) {
 140         case MSR_CORE_PERF_FIXED_CTR_CTRL:
 141         case MSR_IA32_DS_AREA:
 142         case MSR_IA32_PEBS_ENABLE:
 143                 *type = MSR_TYPE_CTRL;
 144                 return true;
 145
 146         case MSR_CORE_PERF_GLOBAL_CTRL:
 147         case MSR_CORE_PERF_GLOBAL_STATUS:
 148         case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
 149                 *type = MSR_TYPE_GLOBAL;
 150                 return true;
 151
 152         default:
 153
 154                 if ((msr_index >= MSR_CORE_PERF_FIXED_CTR0) &&
 155                     (msr_index < MSR_CORE_PERF_FIXED_CTR0 +
 156                                  intel_num_fixed_counters)) {
 157                         *index = msr_index - MSR_CORE_PERF_FIXED_CTR0;
 158                         *type = MSR_TYPE_COUNTER;
 159                         return true;
 160                 }
 161
 162                 if ((msr_index >= MSR_P6_EVNTSEL0) &&
 163                     (msr_index < MSR_P6_EVNTSEL0 +  intel_num_arch_counters)) {
 164                         *index = msr_index - MSR_P6_EVNTSEL0;
 165                         *type = MSR_TYPE_ARCH_CTRL;
 166                         return true;
 167                 }
 168
 169                 msr_index_pmc = msr_index & MSR_PMC_ALIAS_MASK;
 170                 if ((msr_index_pmc >= MSR_IA32_PERFCTR0) &&
 171                     (msr_index_pmc < MSR_IA32_PERFCTR0 +
 172                                      intel_num_arch_counters)) {
 173                         *type = MSR_TYPE_ARCH_COUNTER;
 174                         *index = msr_index_pmc - MSR_IA32_PERFCTR0;
 175                         return true;
 176                 }
 177                 return false;
 178         }
 179 }
 180
 181 static bool xen_intel_pmu_emulate(unsigned int msr, u64 *val, int type,
 182                                   int index, bool is_read)
 183 {
 184         uint64_t *reg = NULL;
 185         struct xen_pmu_intel_ctxt *ctxt;
 186         uint64_t *fix_counters;
 187         struct xen_pmu_cntr_pair *arch_cntr_pair;
 188         struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
 189         uint8_t xenpmu_flags = get_xenpmu_flags();
 190
 191
 192         if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING))
 193                 return false;
 194
 195         ctxt = &xenpmu_data->pmu.c.intel;
 196
 197         switch (msr) {
 198         case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
 199                 reg = &ctxt->global_ovf_ctrl;
 200                 break;
 201         case MSR_CORE_PERF_GLOBAL_STATUS:
 202                 reg = &ctxt->global_status;
 203                 break;
 204         case MSR_CORE_PERF_GLOBAL_CTRL:
 205                 reg = &ctxt->global_ctrl;
 206                 break;
 207         case MSR_CORE_PERF_FIXED_CTR_CTRL:
 208                 reg = &ctxt->fixed_ctrl;
 209                 break;
 210         default:
 211                 switch (type) {
 212                 case MSR_TYPE_COUNTER:
 213                         fix_counters = field_offset(ctxt, fixed_counters);
 214                         reg = &fix_counters[index];
 215                         break;
 216                 case MSR_TYPE_ARCH_COUNTER:
 217                         arch_cntr_pair = field_offset(ctxt, arch_counters);
 218                         reg = &arch_cntr_pair[index].counter;
 219                         break;
 220                 case MSR_TYPE_ARCH_CTRL:
 221                         arch_cntr_pair = field_offset(ctxt, arch_counters);
 222                         reg = &arch_cntr_pair[index].control;
 223                         break;
 224                 default:
 225                         return false;
 226                 }
 227         }
 228
 229         if (reg) {
 230                 if (is_read)
 231                         *val = *reg;
 232                 else {
 233                         *reg = *val;
 234
 235                         if (msr == MSR_CORE_PERF_GLOBAL_OVF_CTRL)
 236                                 ctxt->global_status &= (~(*val));
 237                 }
 238                 return true;
 239         }
 240
 241         return false;
 242 }
 243
 244 static bool xen_amd_pmu_emulate(unsigned int msr, u64 *val, bool is_read)
 245 {
 246         uint64_t *reg = NULL;
 247         int i, off = 0;
 248         struct xen_pmu_amd_ctxt *ctxt;
 249         uint64_t *counter_regs, *ctrl_regs;
 250         struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
 251         uint8_t xenpmu_flags = get_xenpmu_flags();
 252
 253         if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING))
 254                 return false;
 255
 256         if (k7_counters_mirrored &&
 257             ((msr >= MSR_K7_EVNTSEL0) && (msr <= MSR_K7_PERFCTR3)))
 258                 msr = get_fam15h_addr(msr);
 259
 260         ctxt = &xenpmu_data->pmu.c.amd;
 261         for (i = 0; i < amd_num_counters; i++) {
 262                 if (msr == amd_ctrls_base + off) {
 263                         ctrl_regs = field_offset(ctxt, ctrls);
 264                         reg = &ctrl_regs[i];
 265                         break;
 266                 } else if (msr == amd_counters_base + off) {
 267                         counter_regs = field_offset(ctxt, counters);
 268                         reg = &counter_regs[i];
 269                         break;
 270                 }
 271                 off += amd_msr_step;
 272         }
 273
 274         if (reg) {
 275                 if (is_read)
 276                         *val = *reg;
 277                 else
 278                         *reg = *val;
 279
 280                 return true;
 281         }
 282         return false;
 283 }
 284
 285 bool pmu_msr_read(unsigned int msr, uint64_t *val, int *err)
 286 {
 287         if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
 288                 if (is_amd_pmu_msr(msr)) {
 289                         if (!xen_amd_pmu_emulate(msr, val, 1))
 290                                 *val = native_read_msr_safe(msr, err);
 291                         return true;
 292                 }
 293         } else {
 294                 int type, index;
 295
 296                 if (is_intel_pmu_msr(msr, &type, &index)) {
 297                         if (!xen_intel_pmu_emulate(msr, val, type, index, 1))
 298                                 *val = native_read_msr_safe(msr, err);
 299                         return true;
 300                 }
 301         }
 302
 303         return false;
 304 }
 305
 306 bool pmu_msr_write(unsigned int msr, uint32_t low, uint32_t high, int *err)
 307 {
 308         uint64_t val = ((uint64_t)high << 32) | low;
 309
 310         if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
 311                 if (is_amd_pmu_msr(msr)) {
 312                         if (!xen_amd_pmu_emulate(msr, &val, 0))
 313                                 *err = native_write_msr_safe(msr, low, high);
 314                         return true;
 315                 }
 316         } else {
 317                 int type, index;
 318
 319                 if (is_intel_pmu_msr(msr, &type, &index)) {
 320                         if (!xen_intel_pmu_emulate(msr, &val, type, index, 0))
 321                                 *err = native_write_msr_safe(msr, low, high);
 322                         return true;
 323                 }
 324         }
 325
 326         return false;
 327 }
 328
 329 static unsigned long long xen_amd_read_pmc(int counter)
 330 {
 331         struct xen_pmu_amd_ctxt *ctxt;
 332         uint64_t *counter_regs;
 333         struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
 334         uint8_t xenpmu_flags = get_xenpmu_flags();
 335
 336         if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING)) {
 337                 uint32_t msr;
 338                 int err;
 339
 340                 msr = amd_counters_base + (counter * amd_msr_step);
 341                 return native_read_msr_safe(msr, &err);
 342         }
 343
 344         ctxt = &xenpmu_data->pmu.c.amd;
 345         counter_regs = field_offset(ctxt, counters);
 346         return counter_regs[counter];
 347 }
 348
 349 static unsigned long long xen_intel_read_pmc(int counter)
 350 {
 351         struct xen_pmu_intel_ctxt *ctxt;
 352         uint64_t *fixed_counters;
 353         struct xen_pmu_cntr_pair *arch_cntr_pair;
 354         struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
 355         uint8_t xenpmu_flags = get_xenpmu_flags();
 356
 357         if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING)) {
 358                 uint32_t msr;
 359                 int err;
 360
 361                 if (counter & (1 << INTEL_PMC_TYPE_SHIFT))
 362                         msr = MSR_CORE_PERF_FIXED_CTR0 + (counter & 0xffff);
 363                 else
 364                         msr = MSR_IA32_PERFCTR0 + counter;
 365
 366                 return native_read_msr_safe(msr, &err);
 367         }
 368
 369         ctxt = &xenpmu_data->pmu.c.intel;
 370         if (counter & (1 << INTEL_PMC_TYPE_SHIFT)) {
 371                 fixed_counters = field_offset(ctxt, fixed_counters);
 372                 return fixed_counters[counter & 0xffff];
 373         }
 374
 375         arch_cntr_pair = field_offset(ctxt, arch_counters);
 376         return arch_cntr_pair[counter].counter;
 377 }
 378
 379 unsigned long long xen_read_pmc(int counter)
 380 {
 381         if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
 382                 return xen_amd_read_pmc(counter);
 383         else
 384                 return xen_intel_read_pmc(counter);
 385 }
 386
 387 int pmu_apic_update(uint32_t val)
 388 {
 389         int ret;
 390         struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
 391
 392         if (!xenpmu_data) {
 393                 pr_warn_once("%s: pmudata not initialized\n", __func__);
 394                 return -EINVAL;
 395         }
 396
 397         xenpmu_data->pmu.l.lapic_lvtpc = val;
 398
 399         if (get_xenpmu_flags() & XENPMU_IRQ_PROCESSING)
 400                 return 0;
 401
 402         ret = HYPERVISOR_xenpmu_op(XENPMU_lvtpc_set, NULL);
 403
 404         return ret;
 405 }
 406
 407 /* perf callbacks */
 408 static int xen_is_in_guest(void)
 409 {
 410         const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
 411
 412         if (!xenpmu_data) {
 413                 pr_warn_once("%s: pmudata not initialized\n", __func__);
 414                 return 0;
 415         }
 416
 417         if (!xen_initial_domain() || (xenpmu_data->domain_id >= DOMID_SELF))
 418                 return 0;
 419
 420         return 1;
 421 }
 422
 423 static int xen_is_user_mode(void)
 424 {
 425         const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
 426
 427         if (!xenpmu_data) {
 428                 pr_warn_once("%s: pmudata not initialized\n", __func__);
 429                 return 0;
 430         }
 431
 432         if (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_PV)
 433                 return (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_USER);
 434         else
 435                 return !!(xenpmu_data->pmu.r.regs.cpl & 3);
 436 }
 437
 438 static unsigned long xen_get_guest_ip(void)
 439 {
 440         const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
 441
 442         if (!xenpmu_data) {
 443                 pr_warn_once("%s: pmudata not initialized\n", __func__);
 444                 return 0;
 445         }
 446
 447         return xenpmu_data->pmu.r.regs.ip;
 448 }
 449
 450 static struct perf_guest_info_callbacks xen_guest_cbs = {
 451         .is_in_guest            = xen_is_in_guest,
 452         .is_user_mode           = xen_is_user_mode,
 453         .get_guest_ip           = xen_get_guest_ip,
 454 };
 455
 456 /* Convert registers from Xen's format to Linux' */
 457 static void xen_convert_regs(const struct xen_pmu_regs *xen_regs,
 458                              struct pt_regs *regs, uint64_t pmu_flags)
 459 {
 460         regs->ip = xen_regs->ip;
 461         regs->cs = xen_regs->cs;
 462         regs->sp = xen_regs->sp;
 463
 464         if (pmu_flags & PMU_SAMPLE_PV) {
 465                 if (pmu_flags & PMU_SAMPLE_USER)
 466                         regs->cs |= 3;
 467                 else
 468                         regs->cs &= ~3;
 469         } else {
 470                 if (xen_regs->cpl)
 471                         regs->cs |= 3;
 472                 else
 473                         regs->cs &= ~3;
 474         }
 475 }
 476
 477 irqreturn_t xen_pmu_irq_handler(int irq, void *dev_id)
 478 {
 479         int err, ret = IRQ_NONE;
 480         struct pt_regs regs;
 481         const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
 482         uint8_t xenpmu_flags = get_xenpmu_flags();
 483
 484         if (!xenpmu_data) {
 485                 pr_warn_once("%s: pmudata not initialized\n", __func__);
 486                 return ret;
 487         }
 488
 489         this_cpu_ptr(&xenpmu_shared)->flags =
 490                 xenpmu_flags | XENPMU_IRQ_PROCESSING;
 491         xen_convert_regs(&xenpmu_data->pmu.r.regs, &regs,
 492                          xenpmu_data->pmu.pmu_flags);
 493         if (x86_pmu.handle_irq(&regs))
 494                 ret = IRQ_HANDLED;
 495
 496         /* Write out cached context to HW */
 497         err = HYPERVISOR_xenpmu_op(XENPMU_flush, NULL);
 498         this_cpu_ptr(&xenpmu_shared)->flags = xenpmu_flags;
 499         if (err) {
 500                 pr_warn_once("%s: failed hypercall, err: %d\n", __func__, err);
 501                 return IRQ_NONE;
 502         }
 503
 504         return ret;
 505 }
 506
 507 bool is_xen_pmu(int cpu)
 508 {
 509         return (get_xenpmu_data() != NULL);
 510 }
 511
 512 void xen_pmu_init(int cpu)
 513 {
 514         int err;
 515         struct xen_pmu_params xp;
 516         unsigned long pfn;
 517         struct xen_pmu_data *xenpmu_data;
 518
 519         BUILD_BUG_ON(sizeof(struct xen_pmu_data) > PAGE_SIZE);
 520
 521         if (xen_hvm_domain())
 522                 return;
 523
 524         xenpmu_data = (struct xen_pmu_data *)get_zeroed_page(GFP_KERNEL);
 525         if (!xenpmu_data) {
 526                 pr_err("VPMU init: No memory\n");
 527                 return;
 528         }
 529         pfn = virt_to_pfn(xenpmu_data);
 530
 531         xp.val = pfn_to_mfn(pfn);
 532         xp.vcpu = cpu;
 533         xp.version.maj = XENPMU_VER_MAJ;
 534         xp.version.min = XENPMU_VER_MIN;
 535         err = HYPERVISOR_xenpmu_op(XENPMU_init, &xp);
 536         if (err)
 537                 goto fail;
 538
 539         per_cpu(xenpmu_shared, cpu).xenpmu_data = xenpmu_data;
 540         per_cpu(xenpmu_shared, cpu).flags = 0;
 541
 542         if (cpu == 0) {
 543                 perf_register_guest_info_callbacks(&xen_guest_cbs);
 544                 xen_pmu_arch_init();
 545         }
 546
 547         return;
 548
 549 fail:
 550         if (err == -EOPNOTSUPP || err == -ENOSYS)
 551                 pr_info_once("VPMU disabled by hypervisor.\n");
 552         else
 553                 pr_info_once("Could not initialize VPMU for cpu %d, error %d\n",
 554                         cpu, err);
 555         free_pages((unsigned long)xenpmu_data, 0);
 556 }
 557
 558 void xen_pmu_finish(int cpu)
 559 {
 560         struct xen_pmu_params xp;
 561
 562         if (xen_hvm_domain())
 563                 return;
 564
 565         xp.vcpu = cpu;
 566         xp.version.maj = XENPMU_VER_MAJ;
 567         xp.version.min = XENPMU_VER_MIN;
 568
 569         (void)HYPERVISOR_xenpmu_op(XENPMU_finish, &xp);
 570
 571         free_pages((unsigned long)per_cpu(xenpmu_shared, cpu).xenpmu_data, 0);
 572         per_cpu(xenpmu_shared, cpu).xenpmu_data = NULL;
 573 }