arch/x86/events/intel/uncore_discovery.c

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * Support Intel uncore PerfMon discovery mechanism.
   4  * Copyright(c) 2021 Intel Corporation.
   5  */
   6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   7
   8 #include "uncore.h"
   9 #include "uncore_discovery.h"
  10
  11 static struct rb_root discovery_tables = RB_ROOT;
  12 static int num_discovered_types[UNCORE_ACCESS_MAX];
  13
  14 static bool has_generic_discovery_table(void)
  15 {
  16         struct pci_dev *dev;
  17         int dvsec;
  18
  19         dev = pci_get_device(PCI_VENDOR_ID_INTEL, UNCORE_DISCOVERY_TABLE_DEVICE, NULL);
  20         if (!dev)
  21                 return false;
  22
  23         /* A discovery table device has the unique capability ID. */
  24         dvsec = pci_find_next_ext_capability(dev, 0, UNCORE_EXT_CAP_ID_DISCOVERY);
  25         pci_dev_put(dev);
  26         if (dvsec)
  27                 return true;
  28
  29         return false;
  30 }
  31
  32 static int logical_die_id;
  33
  34 static int get_device_die_id(struct pci_dev *dev)
  35 {
  36         int node = pcibus_to_node(dev->bus);
  37
  38         /*
  39          * If the NUMA info is not available, assume that the logical die id is
  40          * continuous in the order in which the discovery table devices are
  41          * detected.
  42          */
  43         if (node < 0)
  44                 return logical_die_id++;
  45
  46         return uncore_device_to_die(dev);
  47 }
  48
  49 #define __node_2_type(cur)      \
  50         rb_entry((cur), struct intel_uncore_discovery_type, node)
  51
  52 static inline int __type_cmp(const void *key, const struct rb_node *b)
  53 {
  54         struct intel_uncore_discovery_type *type_b = __node_2_type(b);
  55         const u16 *type_id = key;
  56
  57         if (type_b->type > *type_id)
  58                 return -1;
  59         else if (type_b->type < *type_id)
  60                 return 1;
  61
  62         return 0;
  63 }
  64
  65 static inline struct intel_uncore_discovery_type *
  66 search_uncore_discovery_type(u16 type_id)
  67 {
  68         struct rb_node *node = rb_find(&type_id, &discovery_tables, __type_cmp);
  69
  70         return (node) ? __node_2_type(node) : NULL;
  71 }
  72
  73 static inline bool __type_less(struct rb_node *a, const struct rb_node *b)
  74 {
  75         return (__node_2_type(a)->type < __node_2_type(b)->type);
  76 }
  77
  78 static struct intel_uncore_discovery_type *
  79 add_uncore_discovery_type(struct uncore_unit_discovery *unit)
  80 {
  81         struct intel_uncore_discovery_type *type;
  82
  83         if (unit->access_type >= UNCORE_ACCESS_MAX) {
  84                 pr_warn("Unsupported access type %d\n", unit->access_type);
  85                 return NULL;
  86         }
  87
  88         type = kzalloc(sizeof(struct intel_uncore_discovery_type), GFP_KERNEL);
  89         if (!type)
  90                 return NULL;
  91
  92         type->units = RB_ROOT;
  93
  94         type->access_type = unit->access_type;
  95         num_discovered_types[type->access_type]++;
  96         type->type = unit->box_type;
  97
  98         rb_add(&type->node, &discovery_tables, __type_less);
  99
 100         return type;
 101 }
 102
 103 static struct intel_uncore_discovery_type *
 104 get_uncore_discovery_type(struct uncore_unit_discovery *unit)
 105 {
 106         struct intel_uncore_discovery_type *type;
 107
 108         type = search_uncore_discovery_type(unit->box_type);
 109         if (type)
 110                 return type;
 111
 112         return add_uncore_discovery_type(unit);
 113 }
 114
 115 static inline int pmu_idx_cmp(const void *key, const struct rb_node *b)
 116 {
 117         struct intel_uncore_discovery_unit *unit;
 118         const unsigned int *id = key;
 119
 120         unit = rb_entry(b, struct intel_uncore_discovery_unit, node);
 121
 122         if (unit->pmu_idx > *id)
 123                 return -1;
 124         else if (unit->pmu_idx < *id)
 125                 return 1;
 126
 127         return 0;
 128 }
 129
 130 static struct intel_uncore_discovery_unit *
 131 intel_uncore_find_discovery_unit(struct rb_root *units, int die,
 132                                  unsigned int pmu_idx)
 133 {
 134         struct intel_uncore_discovery_unit *unit;
 135         struct rb_node *pos;
 136
 137         if (!units)
 138                 return NULL;
 139
 140         pos = rb_find_first(&pmu_idx, units, pmu_idx_cmp);
 141         if (!pos)
 142                 return NULL;
 143         unit = rb_entry(pos, struct intel_uncore_discovery_unit, node);
 144
 145         if (die < 0)
 146                 return unit;
 147
 148         for (; pos; pos = rb_next(pos)) {
 149                 unit = rb_entry(pos, struct intel_uncore_discovery_unit, node);
 150
 151                 if (unit->pmu_idx != pmu_idx)
 152                         break;
 153
 154                 if (unit->die == die)
 155                         return unit;
 156         }
 157
 158         return NULL;
 159 }
 160
 161 int intel_uncore_find_discovery_unit_id(struct rb_root *units, int die,
 162                                         unsigned int pmu_idx)
 163 {
 164         struct intel_uncore_discovery_unit *unit;
 165
 166         unit = intel_uncore_find_discovery_unit(units, die, pmu_idx);
 167         if (unit)
 168                 return unit->id;
 169
 170         return -1;
 171 }
 172
 173 static inline bool unit_less(struct rb_node *a, const struct rb_node *b)
 174 {
 175         struct intel_uncore_discovery_unit *a_node, *b_node;
 176
 177         a_node = rb_entry(a, struct intel_uncore_discovery_unit, node);
 178         b_node = rb_entry(b, struct intel_uncore_discovery_unit, node);
 179
 180         if (a_node->pmu_idx < b_node->pmu_idx)
 181                 return true;
 182         if (a_node->pmu_idx > b_node->pmu_idx)
 183                 return false;
 184
 185         if (a_node->die < b_node->die)
 186                 return true;
 187         if (a_node->die > b_node->die)
 188                 return false;
 189
 190         return 0;
 191 }
 192
 193 static inline struct intel_uncore_discovery_unit *
 194 uncore_find_unit(struct rb_root *root, unsigned int id)
 195 {
 196         struct intel_uncore_discovery_unit *unit;
 197         struct rb_node *node;
 198
 199         for (node = rb_first(root); node; node = rb_next(node)) {
 200                 unit = rb_entry(node, struct intel_uncore_discovery_unit, node);
 201                 if (unit->id == id)
 202                         return unit;
 203         }
 204
 205         return NULL;
 206 }
 207
 208 void uncore_find_add_unit(struct intel_uncore_discovery_unit *node,
 209                           struct rb_root *root, u16 *num_units)
 210 {
 211         struct intel_uncore_discovery_unit *unit = uncore_find_unit(root, node->id);
 212
 213         if (unit)
 214                 node->pmu_idx = unit->pmu_idx;
 215         else if (num_units)
 216                 node->pmu_idx = (*num_units)++;
 217
 218         rb_add(&node->node, root, unit_less);
 219 }
 220
 221 static void
 222 uncore_insert_box_info(struct uncore_unit_discovery *unit,
 223                        int die)
 224 {
 225         struct intel_uncore_discovery_unit *node;
 226         struct intel_uncore_discovery_type *type;
 227
 228         if (!unit->ctl || !unit->ctl_offset || !unit->ctr_offset) {
 229                 pr_info("Invalid address is detected for uncore type %d box %d, "
 230                         "Disable the uncore unit.\n",
 231                         unit->box_type, unit->box_id);
 232                 return;
 233         }
 234
 235         node = kzalloc(sizeof(*node), GFP_KERNEL);
 236         if (!node)
 237                 return;
 238
 239         node->die = die;
 240         node->id = unit->box_id;
 241         node->addr = unit->ctl;
 242
 243         type = get_uncore_discovery_type(unit);
 244         if (!type) {
 245                 kfree(node);
 246                 return;
 247         }
 248
 249         uncore_find_add_unit(node, &type->units, &type->num_units);
 250
 251         /* Store generic information for the first box */
 252         if (type->num_units == 1) {
 253                 type->num_counters = unit->num_regs;
 254                 type->counter_width = unit->bit_width;
 255                 type->ctl_offset = unit->ctl_offset;
 256                 type->ctr_offset = unit->ctr_offset;
 257         }
 258 }
 259
 260 static bool
 261 uncore_ignore_unit(struct uncore_unit_discovery *unit, int *ignore)
 262 {
 263         int i;
 264
 265         if (!ignore)
 266                 return false;
 267
 268         for (i = 0; ignore[i] != UNCORE_IGNORE_END ; i++) {
 269                 if (unit->box_type == ignore[i])
 270                         return true;
 271         }
 272
 273         return false;
 274 }
 275
 276 static int parse_discovery_table(struct pci_dev *dev, int die,
 277                                  u32 bar_offset, bool *parsed,
 278                                  int *ignore)
 279 {
 280         struct uncore_global_discovery global;
 281         struct uncore_unit_discovery unit;
 282         void __iomem *io_addr;
 283         resource_size_t addr;
 284         unsigned long size;
 285         u32 val;
 286         int i;
 287
 288         pci_read_config_dword(dev, bar_offset, &val);
 289
 290         if (val & ~PCI_BASE_ADDRESS_MEM_MASK & ~PCI_BASE_ADDRESS_MEM_TYPE_64)
 291                 return -EINVAL;
 292
 293         addr = (resource_size_t)(val & PCI_BASE_ADDRESS_MEM_MASK);
 294 #ifdef CONFIG_PHYS_ADDR_T_64BIT
 295         if ((val & PCI_BASE_ADDRESS_MEM_TYPE_MASK) == PCI_BASE_ADDRESS_MEM_TYPE_64) {
 296                 u32 val2;
 297
 298                 pci_read_config_dword(dev, bar_offset + 4, &val2);
 299                 addr |= ((resource_size_t)val2) << 32;
 300         }
 301 #endif
 302         size = UNCORE_DISCOVERY_GLOBAL_MAP_SIZE;
 303         io_addr = ioremap(addr, size);
 304         if (!io_addr)
 305                 return -ENOMEM;
 306
 307         /* Read Global Discovery State */
 308         memcpy_fromio(&global, io_addr, sizeof(struct uncore_global_discovery));
 309         if (uncore_discovery_invalid_unit(global)) {
 310                 pr_info("Invalid Global Discovery State: 0x%llx 0x%llx 0x%llx\n",
 311                         global.table1, global.ctl, global.table3);
 312                 iounmap(io_addr);
 313                 return -EINVAL;
 314         }
 315         iounmap(io_addr);
 316
 317         size = (1 + global.max_units) * global.stride * 8;
 318         io_addr = ioremap(addr, size);
 319         if (!io_addr)
 320                 return -ENOMEM;
 321
 322         /* Parsing Unit Discovery State */
 323         for (i = 0; i < global.max_units; i++) {
 324                 memcpy_fromio(&unit, io_addr + (i + 1) * (global.stride * 8),
 325                               sizeof(struct uncore_unit_discovery));
 326
 327                 if (uncore_discovery_invalid_unit(unit))
 328                         continue;
 329
 330                 if (unit.access_type >= UNCORE_ACCESS_MAX)
 331                         continue;
 332
 333                 if (uncore_ignore_unit(&unit, ignore))
 334                         continue;
 335
 336                 uncore_insert_box_info(&unit, die);
 337         }
 338
 339         *parsed = true;
 340         iounmap(io_addr);
 341         return 0;
 342 }
 343
 344 bool intel_uncore_has_discovery_tables(int *ignore)
 345 {
 346         u32 device, val, entry_id, bar_offset;
 347         int die, dvsec = 0, ret = true;
 348         struct pci_dev *dev = NULL;
 349         bool parsed = false;
 350
 351         if (has_generic_discovery_table())
 352                 device = UNCORE_DISCOVERY_TABLE_DEVICE;
 353         else
 354                 device = PCI_ANY_ID;
 355
 356         /*
 357          * Start a new search and iterates through the list of
 358          * the discovery table devices.
 359          */
 360         while ((dev = pci_get_device(PCI_VENDOR_ID_INTEL, device, dev)) != NULL) {
 361                 while ((dvsec = pci_find_next_ext_capability(dev, dvsec, UNCORE_EXT_CAP_ID_DISCOVERY))) {
 362                         pci_read_config_dword(dev, dvsec + UNCORE_DISCOVERY_DVSEC_OFFSET, &val);
 363                         entry_id = val & UNCORE_DISCOVERY_DVSEC_ID_MASK;
 364                         if (entry_id != UNCORE_DISCOVERY_DVSEC_ID_PMON)
 365                                 continue;
 366
 367                         pci_read_config_dword(dev, dvsec + UNCORE_DISCOVERY_DVSEC2_OFFSET, &val);
 368
 369                         if (val & ~UNCORE_DISCOVERY_DVSEC2_BIR_MASK) {
 370                                 ret = false;
 371                                 goto err;
 372                         }
 373                         bar_offset = UNCORE_DISCOVERY_BIR_BASE +
 374                                      (val & UNCORE_DISCOVERY_DVSEC2_BIR_MASK) * UNCORE_DISCOVERY_BIR_STEP;
 375
 376                         die = get_device_die_id(dev);
 377                         if (die < 0)
 378                                 continue;
 379
 380                         parse_discovery_table(dev, die, bar_offset, &parsed, ignore);
 381                 }
 382         }
 383
 384         /* None of the discovery tables are available */
 385         if (!parsed)
 386                 ret = false;
 387 err:
 388         pci_dev_put(dev);
 389
 390         return ret;
 391 }
 392
 393 void intel_uncore_clear_discovery_tables(void)
 394 {
 395         struct intel_uncore_discovery_type *type, *next;
 396         struct intel_uncore_discovery_unit *pos;
 397         struct rb_node *node;
 398
 399         rbtree_postorder_for_each_entry_safe(type, next, &discovery_tables, node) {
 400                 while (!RB_EMPTY_ROOT(&type->units)) {
 401                         node = rb_first(&type->units);
 402                         pos = rb_entry(node, struct intel_uncore_discovery_unit, node);
 403                         rb_erase(node, &type->units);
 404                         kfree(pos);
 405                 }
 406                 kfree(type);
 407         }
 408 }
 409
 410 DEFINE_UNCORE_FORMAT_ATTR(event, event, "config:0-7");
 411 DEFINE_UNCORE_FORMAT_ATTR(umask, umask, "config:8-15");
 412 DEFINE_UNCORE_FORMAT_ATTR(edge, edge, "config:18");
 413 DEFINE_UNCORE_FORMAT_ATTR(inv, inv, "config:23");
 414 DEFINE_UNCORE_FORMAT_ATTR(thresh, thresh, "config:24-31");
 415
 416 static struct attribute *generic_uncore_formats_attr[] = {
 417         &format_attr_event.attr,
 418         &format_attr_umask.attr,
 419         &format_attr_edge.attr,
 420         &format_attr_inv.attr,
 421         &format_attr_thresh.attr,
 422         NULL,
 423 };
 424
 425 static const struct attribute_group generic_uncore_format_group = {
 426         .name = "format",
 427         .attrs = generic_uncore_formats_attr,
 428 };
 429
 430 static u64 intel_generic_uncore_box_ctl(struct intel_uncore_box *box)
 431 {
 432         struct intel_uncore_discovery_unit *unit;
 433
 434         unit = intel_uncore_find_discovery_unit(box->pmu->type->boxes,
 435                                                 -1, box->pmu->pmu_idx);
 436         if (WARN_ON_ONCE(!unit))
 437                 return 0;
 438
 439         return unit->addr;
 440 }
 441
 442 void intel_generic_uncore_msr_init_box(struct intel_uncore_box *box)
 443 {
 444         wrmsrl(intel_generic_uncore_box_ctl(box), GENERIC_PMON_BOX_CTL_INT);
 445 }
 446
 447 void intel_generic_uncore_msr_disable_box(struct intel_uncore_box *box)
 448 {
 449         wrmsrl(intel_generic_uncore_box_ctl(box), GENERIC_PMON_BOX_CTL_FRZ);
 450 }
 451
 452 void intel_generic_uncore_msr_enable_box(struct intel_uncore_box *box)
 453 {
 454         wrmsrl(intel_generic_uncore_box_ctl(box), 0);
 455 }
 456
 457 static void intel_generic_uncore_msr_enable_event(struct intel_uncore_box *box,
 458                                             struct perf_event *event)
 459 {
 460         struct hw_perf_event *hwc = &event->hw;
 461
 462         wrmsrl(hwc->config_base, hwc->config);
 463 }
 464
 465 static void intel_generic_uncore_msr_disable_event(struct intel_uncore_box *box,
 466                                              struct perf_event *event)
 467 {
 468         struct hw_perf_event *hwc = &event->hw;
 469
 470         wrmsrl(hwc->config_base, 0);
 471 }
 472
 473 static struct intel_uncore_ops generic_uncore_msr_ops = {
 474         .init_box               = intel_generic_uncore_msr_init_box,
 475         .disable_box            = intel_generic_uncore_msr_disable_box,
 476         .enable_box             = intel_generic_uncore_msr_enable_box,
 477         .disable_event          = intel_generic_uncore_msr_disable_event,
 478         .enable_event           = intel_generic_uncore_msr_enable_event,
 479         .read_counter           = uncore_msr_read_counter,
 480 };
 481
 482 bool intel_generic_uncore_assign_hw_event(struct perf_event *event,
 483                                           struct intel_uncore_box *box)
 484 {
 485         struct hw_perf_event *hwc = &event->hw;
 486         u64 box_ctl;
 487
 488         if (!box->pmu->type->boxes)
 489                 return false;
 490
 491         if (box->io_addr) {
 492                 hwc->config_base = uncore_pci_event_ctl(box, hwc->idx);
 493                 hwc->event_base  = uncore_pci_perf_ctr(box, hwc->idx);
 494                 return true;
 495         }
 496
 497         box_ctl = intel_generic_uncore_box_ctl(box);
 498         if (!box_ctl)
 499                 return false;
 500
 501         if (box->pci_dev) {
 502                 box_ctl = UNCORE_DISCOVERY_PCI_BOX_CTRL(box_ctl);
 503                 hwc->config_base = box_ctl + uncore_pci_event_ctl(box, hwc->idx);
 504                 hwc->event_base  = box_ctl + uncore_pci_perf_ctr(box, hwc->idx);
 505                 return true;
 506         }
 507
 508         hwc->config_base = box_ctl + box->pmu->type->event_ctl + hwc->idx;
 509         hwc->event_base  = box_ctl + box->pmu->type->perf_ctr + hwc->idx;
 510
 511         return true;
 512 }
 513
 514 static inline int intel_pci_uncore_box_ctl(struct intel_uncore_box *box)
 515 {
 516         return UNCORE_DISCOVERY_PCI_BOX_CTRL(intel_generic_uncore_box_ctl(box));
 517 }
 518
 519 void intel_generic_uncore_pci_init_box(struct intel_uncore_box *box)
 520 {
 521         struct pci_dev *pdev = box->pci_dev;
 522         int box_ctl = intel_pci_uncore_box_ctl(box);
 523
 524         __set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags);
 525         pci_write_config_dword(pdev, box_ctl, GENERIC_PMON_BOX_CTL_INT);
 526 }
 527
 528 void intel_generic_uncore_pci_disable_box(struct intel_uncore_box *box)
 529 {
 530         struct pci_dev *pdev = box->pci_dev;
 531         int box_ctl = intel_pci_uncore_box_ctl(box);
 532
 533         pci_write_config_dword(pdev, box_ctl, GENERIC_PMON_BOX_CTL_FRZ);
 534 }
 535
 536 void intel_generic_uncore_pci_enable_box(struct intel_uncore_box *box)
 537 {
 538         struct pci_dev *pdev = box->pci_dev;
 539         int box_ctl = intel_pci_uncore_box_ctl(box);
 540
 541         pci_write_config_dword(pdev, box_ctl, 0);
 542 }
 543
 544 static void intel_generic_uncore_pci_enable_event(struct intel_uncore_box *box,
 545                                             struct perf_event *event)
 546 {
 547         struct pci_dev *pdev = box->pci_dev;
 548         struct hw_perf_event *hwc = &event->hw;
 549
 550         pci_write_config_dword(pdev, hwc->config_base, hwc->config);
 551 }
 552
 553 void intel_generic_uncore_pci_disable_event(struct intel_uncore_box *box,
 554                                             struct perf_event *event)
 555 {
 556         struct pci_dev *pdev = box->pci_dev;
 557         struct hw_perf_event *hwc = &event->hw;
 558
 559         pci_write_config_dword(pdev, hwc->config_base, 0);
 560 }
 561
 562 u64 intel_generic_uncore_pci_read_counter(struct intel_uncore_box *box,
 563                                           struct perf_event *event)
 564 {
 565         struct pci_dev *pdev = box->pci_dev;
 566         struct hw_perf_event *hwc = &event->hw;
 567         u64 count = 0;
 568
 569         pci_read_config_dword(pdev, hwc->event_base, (u32 *)&count);
 570         pci_read_config_dword(pdev, hwc->event_base + 4, (u32 *)&count + 1);
 571
 572         return count;
 573 }
 574
 575 static struct intel_uncore_ops generic_uncore_pci_ops = {
 576         .init_box       = intel_generic_uncore_pci_init_box,
 577         .disable_box    = intel_generic_uncore_pci_disable_box,
 578         .enable_box     = intel_generic_uncore_pci_enable_box,
 579         .disable_event  = intel_generic_uncore_pci_disable_event,
 580         .enable_event   = intel_generic_uncore_pci_enable_event,
 581         .read_counter   = intel_generic_uncore_pci_read_counter,
 582 };
 583
 584 #define UNCORE_GENERIC_MMIO_SIZE                0x4000
 585
 586 void intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box)
 587 {
 588         static struct intel_uncore_discovery_unit *unit;
 589         struct intel_uncore_type *type = box->pmu->type;
 590         resource_size_t addr;
 591
 592         unit = intel_uncore_find_discovery_unit(type->boxes, box->dieid, box->pmu->pmu_idx);
 593         if (!unit) {
 594                 pr_warn("Uncore type %d id %d: Cannot find box control address.\n",
 595                         type->type_id, box->pmu->pmu_idx);
 596                 return;
 597         }
 598
 599         if (!unit->addr) {
 600                 pr_warn("Uncore type %d box %d: Invalid box control address.\n",
 601                         type->type_id, unit->id);
 602                 return;
 603         }
 604
 605         addr = unit->addr;
 606         box->io_addr = ioremap(addr, UNCORE_GENERIC_MMIO_SIZE);
 607         if (!box->io_addr) {
 608                 pr_warn("Uncore type %d box %d: ioremap error for 0x%llx.\n",
 609                         type->type_id, unit->id, (unsigned long long)addr);
 610                 return;
 611         }
 612
 613         writel(GENERIC_PMON_BOX_CTL_INT, box->io_addr);
 614 }
 615
 616 void intel_generic_uncore_mmio_disable_box(struct intel_uncore_box *box)
 617 {
 618         if (!box->io_addr)
 619                 return;
 620
 621         writel(GENERIC_PMON_BOX_CTL_FRZ, box->io_addr);
 622 }
 623
 624 void intel_generic_uncore_mmio_enable_box(struct intel_uncore_box *box)
 625 {
 626         if (!box->io_addr)
 627                 return;
 628
 629         writel(0, box->io_addr);
 630 }
 631
 632 void intel_generic_uncore_mmio_enable_event(struct intel_uncore_box *box,
 633                                             struct perf_event *event)
 634 {
 635         struct hw_perf_event *hwc = &event->hw;
 636
 637         if (!box->io_addr)
 638                 return;
 639
 640         writel(hwc->config, box->io_addr + hwc->config_base);
 641 }
 642
 643 void intel_generic_uncore_mmio_disable_event(struct intel_uncore_box *box,
 644                                              struct perf_event *event)
 645 {
 646         struct hw_perf_event *hwc = &event->hw;
 647
 648         if (!box->io_addr)
 649                 return;
 650
 651         writel(0, box->io_addr + hwc->config_base);
 652 }
 653
 654 static struct intel_uncore_ops generic_uncore_mmio_ops = {
 655         .init_box       = intel_generic_uncore_mmio_init_box,
 656         .exit_box       = uncore_mmio_exit_box,
 657         .disable_box    = intel_generic_uncore_mmio_disable_box,
 658         .enable_box     = intel_generic_uncore_mmio_enable_box,
 659         .disable_event  = intel_generic_uncore_mmio_disable_event,
 660         .enable_event   = intel_generic_uncore_mmio_enable_event,
 661         .read_counter   = uncore_mmio_read_counter,
 662 };
 663
 664 static bool uncore_update_uncore_type(enum uncore_access_type type_id,
 665                                       struct intel_uncore_type *uncore,
 666                                       struct intel_uncore_discovery_type *type)
 667 {
 668         uncore->type_id = type->type;
 669         uncore->num_counters = type->num_counters;
 670         uncore->perf_ctr_bits = type->counter_width;
 671         uncore->perf_ctr = (unsigned int)type->ctr_offset;
 672         uncore->event_ctl = (unsigned int)type->ctl_offset;
 673         uncore->boxes = &type->units;
 674         uncore->num_boxes = type->num_units;
 675
 676         switch (type_id) {
 677         case UNCORE_ACCESS_MSR:
 678                 uncore->ops = &generic_uncore_msr_ops;
 679                 break;
 680         case UNCORE_ACCESS_PCI:
 681                 uncore->ops = &generic_uncore_pci_ops;
 682                 break;
 683         case UNCORE_ACCESS_MMIO:
 684                 uncore->ops = &generic_uncore_mmio_ops;
 685                 uncore->mmio_map_size = UNCORE_GENERIC_MMIO_SIZE;
 686                 break;
 687         default:
 688                 return false;
 689         }
 690
 691         return true;
 692 }
 693
 694 struct intel_uncore_type **
 695 intel_uncore_generic_init_uncores(enum uncore_access_type type_id, int num_extra)
 696 {
 697         struct intel_uncore_discovery_type *type;
 698         struct intel_uncore_type **uncores;
 699         struct intel_uncore_type *uncore;
 700         struct rb_node *node;
 701         int i = 0;
 702
 703         uncores = kcalloc(num_discovered_types[type_id] + num_extra + 1,
 704                           sizeof(struct intel_uncore_type *), GFP_KERNEL);
 705         if (!uncores)
 706                 return empty_uncore;
 707
 708         for (node = rb_first(&discovery_tables); node; node = rb_next(node)) {
 709                 type = rb_entry(node, struct intel_uncore_discovery_type, node);
 710                 if (type->access_type != type_id)
 711                         continue;
 712
 713                 uncore = kzalloc(sizeof(struct intel_uncore_type), GFP_KERNEL);
 714                 if (!uncore)
 715                         break;
 716
 717                 uncore->event_mask = GENERIC_PMON_RAW_EVENT_MASK;
 718                 uncore->format_group = &generic_uncore_format_group;
 719
 720                 if (!uncore_update_uncore_type(type_id, uncore, type)) {
 721                         kfree(uncore);
 722                         continue;
 723                 }
 724                 uncores[i++] = uncore;
 725         }
 726
 727         return uncores;
 728 }
 729
 730 void intel_uncore_generic_uncore_cpu_init(void)
 731 {
 732         uncore_msr_uncores = intel_uncore_generic_init_uncores(UNCORE_ACCESS_MSR, 0);
 733 }
 734
 735 int intel_uncore_generic_uncore_pci_init(void)
 736 {
 737         uncore_pci_uncores = intel_uncore_generic_init_uncores(UNCORE_ACCESS_PCI, 0);
 738
 739         return 0;
 740 }
 741
 742 void intel_uncore_generic_uncore_mmio_init(void)
 743 {
 744         uncore_mmio_uncores = intel_uncore_generic_init_uncores(UNCORE_ACCESS_MMIO, 0);
 745 }