drivers/perf/arm-ni.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 // Copyright (C) 2022-2024 Arm Limited
   3 // NI-700 Network-on-Chip PMU driver
   4
   5 #include <linux/acpi.h>
   6 #include <linux/bitfield.h>
   7 #include <linux/interrupt.h>
   8 #include <linux/io.h>
   9 #include <linux/io-64-nonatomic-lo-hi.h>
  10 #include <linux/kernel.h>
  11 #include <linux/module.h>
  12 #include <linux/of.h>
  13 #include <linux/perf_event.h>
  14 #include <linux/platform_device.h>
  15 #include <linux/slab.h>
  16
  17 /* Common registers */
  18 #define NI_NODE_TYPE            0x000
  19 #define NI_NODE_TYPE_NODE_ID    GENMASK(31, 16)
  20 #define NI_NODE_TYPE_NODE_TYPE  GENMASK(15, 0)
  21
  22 #define NI_CHILD_NODE_INFO      0x004
  23 #define NI_CHILD_PTR(n)         (0x008 + (n) * 4)
  24
  25 #define NI700_PMUSELA           0x00c
  26
  27 /* Config node */
  28 #define NI_PERIPHERAL_ID0       0xfe0
  29 #define NI_PIDR0_PART_7_0       GENMASK(7, 0)
  30 #define NI_PERIPHERAL_ID1       0xfe4
  31 #define NI_PIDR1_PART_11_8      GENMASK(3, 0)
  32 #define NI_PERIPHERAL_ID2       0xfe8
  33 #define NI_PIDR2_VERSION        GENMASK(7, 4)
  34
  35 /* PMU node */
  36 #define NI_PMEVCNTR(n)          (0x008 + (n) * 8)
  37 #define NI_PMCCNTR_L            0x0f8
  38 #define NI_PMCCNTR_U            0x0fc
  39 #define NI_PMEVTYPER(n)         (0x400 + (n) * 4)
  40 #define NI_PMEVTYPER_NODE_TYPE  GENMASK(12, 9)
  41 #define NI_PMEVTYPER_NODE_ID    GENMASK(8, 0)
  42 #define NI_PMCNTENSET           0xc00
  43 #define NI_PMCNTENCLR           0xc20
  44 #define NI_PMINTENSET           0xc40
  45 #define NI_PMINTENCLR           0xc60
  46 #define NI_PMOVSCLR             0xc80
  47 #define NI_PMOVSSET             0xcc0
  48 #define NI_PMCFGR               0xe00
  49 #define NI_PMCR                 0xe04
  50 #define NI_PMCR_RESET_CCNT      BIT(2)
  51 #define NI_PMCR_RESET_EVCNT     BIT(1)
  52 #define NI_PMCR_ENABLE          BIT(0)
  53
  54 #define NI_NUM_COUNTERS         8
  55 #define NI_CCNT_IDX             31
  56
  57 /* Event attributes */
  58 #define NI_CONFIG_TYPE          GENMASK_ULL(15, 0)
  59 #define NI_CONFIG_NODEID        GENMASK_ULL(31, 16)
  60 #define NI_CONFIG_EVENTID       GENMASK_ULL(47, 32)
  61
  62 #define NI_EVENT_TYPE(event)    FIELD_GET(NI_CONFIG_TYPE, (event)->attr.config)
  63 #define NI_EVENT_NODEID(event)  FIELD_GET(NI_CONFIG_NODEID, (event)->attr.config)
  64 #define NI_EVENT_EVENTID(event) FIELD_GET(NI_CONFIG_EVENTID, (event)->attr.config)
  65
  66 enum ni_part {
  67         PART_NI_700 = 0x43b,
  68         PART_NI_710AE = 0x43d,
  69 };
  70
  71 enum ni_node_type {
  72         NI_GLOBAL,
  73         NI_VOLTAGE,
  74         NI_POWER,
  75         NI_CLOCK,
  76         NI_ASNI,
  77         NI_AMNI,
  78         NI_PMU,
  79         NI_HSNI,
  80         NI_HMNI,
  81         NI_PMNI,
  82 };
  83
  84 struct arm_ni_node {
  85         void __iomem *base;
  86         enum ni_node_type type;
  87         u16 id;
  88         u32 num_components;
  89 };
  90
  91 struct arm_ni_unit {
  92         void __iomem *pmusela;
  93         enum ni_node_type type;
  94         u16 id;
  95         bool ns;
  96         union {
  97                 __le64 pmusel;
  98                 u8 event[8];
  99         };
 100 };
 101
 102 struct arm_ni_cd {
 103         void __iomem *pmu_base;
 104         u16 id;
 105         int num_units;
 106         int irq;
 107         int cpu;
 108         struct hlist_node cpuhp_node;
 109         struct pmu pmu;
 110         struct arm_ni_unit *units;
 111         struct perf_event *evcnt[NI_NUM_COUNTERS];
 112         struct perf_event *ccnt;
 113 };
 114
 115 struct arm_ni {
 116         struct device *dev;
 117         void __iomem *base;
 118         enum ni_part part;
 119         int id;
 120         int num_cds;
 121         struct arm_ni_cd cds[] __counted_by(num_cds);
 122 };
 123
 124 #define cd_to_ni(cd) container_of((cd), struct arm_ni, cds[(cd)->id])
 125 #define pmu_to_cd(p) container_of((p), struct arm_ni_cd, pmu)
 126
 127 #define cd_for_each_unit(cd, u) \
 128         for (struct arm_ni_unit *u = cd->units; u < cd->units + cd->num_units; u++)
 129
 130 static int arm_ni_hp_state;
 131
 132 struct arm_ni_event_attr {
 133         struct device_attribute attr;
 134         enum ni_node_type type;
 135 };
 136
 137 #define NI_EVENT_ATTR(_name, _type)                                     \
 138         (&((struct arm_ni_event_attr[]) {{                              \
 139                 .attr = __ATTR(_name, 0444, arm_ni_event_show, NULL),   \
 140                 .type = _type,                                          \
 141         }})[0].attr.attr)
 142
 143 static ssize_t arm_ni_event_show(struct device *dev,
 144                                  struct device_attribute *attr, char *buf)
 145 {
 146         struct arm_ni_event_attr *eattr = container_of(attr, typeof(*eattr), attr);
 147
 148         if (eattr->type == NI_PMU)
 149                 return sysfs_emit(buf, "type=0x%x\n", eattr->type);
 150
 151         return sysfs_emit(buf, "type=0x%x,eventid=?,nodeid=?\n", eattr->type);
 152 }
 153
 154 static umode_t arm_ni_event_attr_is_visible(struct kobject *kobj,
 155                                             struct attribute *attr, int unused)
 156 {
 157         struct device *dev = kobj_to_dev(kobj);
 158         struct arm_ni_cd *cd = pmu_to_cd(dev_get_drvdata(dev));
 159         struct arm_ni_event_attr *eattr;
 160
 161         eattr = container_of(attr, typeof(*eattr), attr.attr);
 162
 163         cd_for_each_unit(cd, unit) {
 164                 if (unit->type == eattr->type && unit->ns)
 165                         return attr->mode;
 166         }
 167
 168         return 0;
 169 }
 170
 171 static struct attribute *arm_ni_event_attrs[] = {
 172         NI_EVENT_ATTR(asni, NI_ASNI),
 173         NI_EVENT_ATTR(amni, NI_AMNI),
 174         NI_EVENT_ATTR(cycles, NI_PMU),
 175         NI_EVENT_ATTR(hsni, NI_HSNI),
 176         NI_EVENT_ATTR(hmni, NI_HMNI),
 177         NI_EVENT_ATTR(pmni, NI_PMNI),
 178         NULL
 179 };
 180
 181 static const struct attribute_group arm_ni_event_attrs_group = {
 182         .name = "events",
 183         .attrs = arm_ni_event_attrs,
 184         .is_visible = arm_ni_event_attr_is_visible,
 185 };
 186
 187 struct arm_ni_format_attr {
 188         struct device_attribute attr;
 189         u64 field;
 190 };
 191
 192 #define NI_FORMAT_ATTR(_name, _fld)                                     \
 193         (&((struct arm_ni_format_attr[]) {{                             \
 194                 .attr = __ATTR(_name, 0444, arm_ni_format_show, NULL),  \
 195                 .field = _fld,                                          \
 196         }})[0].attr.attr)
 197
 198 static ssize_t arm_ni_format_show(struct device *dev,
 199                                   struct device_attribute *attr, char *buf)
 200 {
 201         struct arm_ni_format_attr *fmt = container_of(attr, typeof(*fmt), attr);
 202
 203         return sysfs_emit(buf, "config:%*pbl\n", 64, &fmt->field);
 204 }
 205
 206 static struct attribute *arm_ni_format_attrs[] = {
 207         NI_FORMAT_ATTR(type, NI_CONFIG_TYPE),
 208         NI_FORMAT_ATTR(nodeid, NI_CONFIG_NODEID),
 209         NI_FORMAT_ATTR(eventid, NI_CONFIG_EVENTID),
 210         NULL
 211 };
 212
 213 static const struct attribute_group arm_ni_format_attrs_group = {
 214         .name = "format",
 215         .attrs = arm_ni_format_attrs,
 216 };
 217
 218 static ssize_t arm_ni_cpumask_show(struct device *dev,
 219                                    struct device_attribute *attr, char *buf)
 220 {
 221         struct arm_ni_cd *cd = pmu_to_cd(dev_get_drvdata(dev));
 222
 223         return cpumap_print_to_pagebuf(true, buf, cpumask_of(cd->cpu));
 224 }
 225
 226 static struct device_attribute arm_ni_cpumask_attr =
 227                 __ATTR(cpumask, 0444, arm_ni_cpumask_show, NULL);
 228
 229 static ssize_t arm_ni_identifier_show(struct device *dev,
 230                                       struct device_attribute *attr, char *buf)
 231 {
 232         struct arm_ni *ni = cd_to_ni(pmu_to_cd(dev_get_drvdata(dev)));
 233         u32 reg = readl_relaxed(ni->base + NI_PERIPHERAL_ID2);
 234         int version = FIELD_GET(NI_PIDR2_VERSION, reg);
 235
 236         return sysfs_emit(buf, "%03x%02x\n", ni->part, version);
 237 }
 238
 239 static struct device_attribute arm_ni_identifier_attr =
 240                 __ATTR(identifier, 0444, arm_ni_identifier_show, NULL);
 241
 242 static struct attribute *arm_ni_other_attrs[] = {
 243         &arm_ni_cpumask_attr.attr,
 244         &arm_ni_identifier_attr.attr,
 245         NULL
 246 };
 247
 248 static const struct attribute_group arm_ni_other_attr_group = {
 249         .attrs = arm_ni_other_attrs,
 250         NULL
 251 };
 252
 253 static const struct attribute_group *arm_ni_attr_groups[] = {
 254         &arm_ni_event_attrs_group,
 255         &arm_ni_format_attrs_group,
 256         &arm_ni_other_attr_group,
 257         NULL
 258 };
 259
 260 static void arm_ni_pmu_enable(struct pmu *pmu)
 261 {
 262         writel_relaxed(NI_PMCR_ENABLE, pmu_to_cd(pmu)->pmu_base + NI_PMCR);
 263 }
 264
 265 static void arm_ni_pmu_disable(struct pmu *pmu)
 266 {
 267         writel_relaxed(0, pmu_to_cd(pmu)->pmu_base + NI_PMCR);
 268 }
 269
 270 struct arm_ni_val {
 271         unsigned int evcnt;
 272         unsigned int ccnt;
 273 };
 274
 275 static bool arm_ni_val_count_event(struct perf_event *evt, struct arm_ni_val *val)
 276 {
 277         if (is_software_event(evt))
 278                 return true;
 279
 280         if (NI_EVENT_TYPE(evt) == NI_PMU) {
 281                 val->ccnt++;
 282                 return val->ccnt <= 1;
 283         }
 284
 285         val->evcnt++;
 286         return val->evcnt <= NI_NUM_COUNTERS;
 287 }
 288
 289 static int arm_ni_validate_group(struct perf_event *event)
 290 {
 291         struct perf_event *sibling, *leader = event->group_leader;
 292         struct arm_ni_val val = { 0 };
 293
 294         if (leader == event)
 295                 return 0;
 296
 297         arm_ni_val_count_event(event, &val);
 298         if (!arm_ni_val_count_event(leader, &val))
 299                 return -EINVAL;
 300
 301         for_each_sibling_event(sibling, leader) {
 302                 if (!arm_ni_val_count_event(sibling, &val))
 303                         return -EINVAL;
 304         }
 305         return 0;
 306 }
 307
 308 static int arm_ni_event_init(struct perf_event *event)
 309 {
 310         struct arm_ni_cd *cd = pmu_to_cd(event->pmu);
 311
 312         if (event->attr.type != event->pmu->type)
 313                 return -ENOENT;
 314
 315         if (is_sampling_event(event))
 316                 return -EINVAL;
 317
 318         event->cpu = cd->cpu;
 319         if (NI_EVENT_TYPE(event) == NI_PMU)
 320                 return arm_ni_validate_group(event);
 321
 322         cd_for_each_unit(cd, unit) {
 323                 if (unit->type == NI_EVENT_TYPE(event) &&
 324                     unit->id == NI_EVENT_NODEID(event) && unit->ns) {
 325                         event->hw.config_base = (unsigned long)unit;
 326                         return arm_ni_validate_group(event);
 327                 }
 328         }
 329         return -EINVAL;
 330 }
 331
 332 static u64 arm_ni_read_ccnt(struct arm_ni_cd *cd)
 333 {
 334         u64 l, u_old, u_new;
 335         int retries = 3; /* 1st time unlucky, 2nd improbable, 3rd just broken */
 336
 337         u_new = readl_relaxed(cd->pmu_base + NI_PMCCNTR_U);
 338         do {
 339                 u_old = u_new;
 340                 l = readl_relaxed(cd->pmu_base + NI_PMCCNTR_L);
 341                 u_new = readl_relaxed(cd->pmu_base + NI_PMCCNTR_U);
 342         } while (u_new != u_old && --retries);
 343         WARN_ON(!retries);
 344
 345         return (u_new << 32) | l;
 346 }
 347
 348 static void arm_ni_event_read(struct perf_event *event)
 349 {
 350         struct arm_ni_cd *cd = pmu_to_cd(event->pmu);
 351         struct hw_perf_event *hw = &event->hw;
 352         u64 count, prev;
 353         bool ccnt = hw->idx == NI_CCNT_IDX;
 354
 355         do {
 356                 prev = local64_read(&hw->prev_count);
 357                 if (ccnt)
 358                         count = arm_ni_read_ccnt(cd);
 359                 else
 360                         count = readl_relaxed(cd->pmu_base + NI_PMEVCNTR(hw->idx));
 361         } while (local64_cmpxchg(&hw->prev_count, prev, count) != prev);
 362
 363         count -= prev;
 364         if (!ccnt)
 365                 count = (u32)count;
 366         local64_add(count, &event->count);
 367 }
 368
 369 static void arm_ni_event_start(struct perf_event *event, int flags)
 370 {
 371         struct arm_ni_cd *cd = pmu_to_cd(event->pmu);
 372
 373         writel_relaxed(1U << event->hw.idx, cd->pmu_base + NI_PMCNTENSET);
 374 }
 375
 376 static void arm_ni_event_stop(struct perf_event *event, int flags)
 377 {
 378         struct arm_ni_cd *cd = pmu_to_cd(event->pmu);
 379
 380         writel_relaxed(1U << event->hw.idx, cd->pmu_base + NI_PMCNTENCLR);
 381         if (flags & PERF_EF_UPDATE)
 382                 arm_ni_event_read(event);
 383 }
 384
 385 static void arm_ni_init_ccnt(struct arm_ni_cd *cd)
 386 {
 387         local64_set(&cd->ccnt->hw.prev_count, S64_MIN);
 388         lo_hi_writeq_relaxed(S64_MIN, cd->pmu_base + NI_PMCCNTR_L);
 389 }
 390
 391 static void arm_ni_init_evcnt(struct arm_ni_cd *cd, int idx)
 392 {
 393         local64_set(&cd->evcnt[idx]->hw.prev_count, S32_MIN);
 394         writel_relaxed(S32_MIN, cd->pmu_base + NI_PMEVCNTR(idx));
 395 }
 396
 397 static int arm_ni_event_add(struct perf_event *event, int flags)
 398 {
 399         struct arm_ni_cd *cd = pmu_to_cd(event->pmu);
 400         struct hw_perf_event *hw = &event->hw;
 401         struct arm_ni_unit *unit;
 402         enum ni_node_type type = NI_EVENT_TYPE(event);
 403         u32 reg;
 404
 405         if (type == NI_PMU) {
 406                 if (cd->ccnt)
 407                         return -ENOSPC;
 408                 hw->idx = NI_CCNT_IDX;
 409                 cd->ccnt = event;
 410                 arm_ni_init_ccnt(cd);
 411         } else {
 412                 hw->idx = 0;
 413                 while (cd->evcnt[hw->idx]) {
 414                         if (++hw->idx == NI_NUM_COUNTERS)
 415                                 return -ENOSPC;
 416                 }
 417                 cd->evcnt[hw->idx] = event;
 418                 unit = (void *)hw->config_base;
 419                 unit->event[hw->idx] = NI_EVENT_EVENTID(event);
 420                 arm_ni_init_evcnt(cd, hw->idx);
 421                 lo_hi_writeq_relaxed(le64_to_cpu(unit->pmusel), unit->pmusela);
 422
 423                 reg = FIELD_PREP(NI_PMEVTYPER_NODE_TYPE, type) |
 424                       FIELD_PREP(NI_PMEVTYPER_NODE_ID, NI_EVENT_NODEID(event));
 425                 writel_relaxed(reg, cd->pmu_base + NI_PMEVTYPER(hw->idx));
 426         }
 427         if (flags & PERF_EF_START)
 428                 arm_ni_event_start(event, 0);
 429         return 0;
 430 }
 431
 432 static void arm_ni_event_del(struct perf_event *event, int flags)
 433 {
 434         struct arm_ni_cd *cd = pmu_to_cd(event->pmu);
 435         struct hw_perf_event *hw = &event->hw;
 436
 437         arm_ni_event_stop(event, PERF_EF_UPDATE);
 438
 439         if (hw->idx == NI_CCNT_IDX)
 440                 cd->ccnt = NULL;
 441         else
 442                 cd->evcnt[hw->idx] = NULL;
 443 }
 444
 445 static irqreturn_t arm_ni_handle_irq(int irq, void *dev_id)
 446 {
 447         struct arm_ni_cd *cd = dev_id;
 448         irqreturn_t ret = IRQ_NONE;
 449         u32 reg = readl_relaxed(cd->pmu_base + NI_PMOVSCLR);
 450
 451         if (reg & (1U << NI_CCNT_IDX)) {
 452                 ret = IRQ_HANDLED;
 453                 if (!(WARN_ON(!cd->ccnt))) {
 454                         arm_ni_event_read(cd->ccnt);
 455                         arm_ni_init_ccnt(cd);
 456                 }
 457         }
 458         for (int i = 0; i < NI_NUM_COUNTERS; i++) {
 459                 if (!(reg & (1U << i)))
 460                         continue;
 461                 ret = IRQ_HANDLED;
 462                 if (!(WARN_ON(!cd->evcnt[i]))) {
 463                         arm_ni_event_read(cd->evcnt[i]);
 464                         arm_ni_init_evcnt(cd, i);
 465                 }
 466         }
 467         writel_relaxed(reg, cd->pmu_base + NI_PMOVSCLR);
 468         return ret;
 469 }
 470
 471 static int arm_ni_init_cd(struct arm_ni *ni, struct arm_ni_node *node, u64 res_start)
 472 {
 473         struct arm_ni_cd *cd = ni->cds + node->id;
 474         const char *name;
 475         int err;
 476
 477         cd->id = node->id;
 478         cd->num_units = node->num_components;
 479         cd->units = devm_kcalloc(ni->dev, cd->num_units, sizeof(*(cd->units)), GFP_KERNEL);
 480         if (!cd->units)
 481                 return -ENOMEM;
 482
 483         for (int i = 0; i < cd->num_units; i++) {
 484                 u32 reg = readl_relaxed(node->base + NI_CHILD_PTR(i));
 485                 void __iomem *unit_base = ni->base + reg;
 486                 struct arm_ni_unit *unit = cd->units + i;
 487
 488                 reg = readl_relaxed(unit_base + NI_NODE_TYPE);
 489                 unit->type = FIELD_GET(NI_NODE_TYPE_NODE_TYPE, reg);
 490                 unit->id = FIELD_GET(NI_NODE_TYPE_NODE_ID, reg);
 491
 492                 switch (unit->type) {
 493                 case NI_PMU:
 494                         reg = readl_relaxed(unit_base + NI_PMCFGR);
 495                         if (!reg) {
 496                                 dev_info(ni->dev, "No access to PMU %d\n", cd->id);
 497                                 devm_kfree(ni->dev, cd->units);
 498                                 return 0;
 499                         }
 500                         unit->ns = true;
 501                         cd->pmu_base = unit_base;
 502                         break;
 503                 case NI_ASNI:
 504                 case NI_AMNI:
 505                 case NI_HSNI:
 506                 case NI_HMNI:
 507                 case NI_PMNI:
 508                         unit->pmusela = unit_base + NI700_PMUSELA;
 509                         writel_relaxed(1, unit->pmusela);
 510                         if (readl_relaxed(unit->pmusela) != 1)
 511                                 dev_info(ni->dev, "No access to node 0x%04x%04x\n", unit->id, unit->type);
 512                         else
 513                                 unit->ns = true;
 514                         break;
 515                 default:
 516                         /*
 517                          * e.g. FMU - thankfully bits 3:2 of FMU_ERR_FR0 are RES0 so
 518                          * can't alias any of the leaf node types we're looking for.
 519                          */
 520                         dev_dbg(ni->dev, "Mystery node 0x%04x%04x\n", unit->id, unit->type);
 521                         break;
 522                 }
 523         }
 524
 525         res_start += cd->pmu_base - ni->base;
 526         if (!devm_request_mem_region(ni->dev, res_start, SZ_4K, dev_name(ni->dev))) {
 527                 dev_err(ni->dev, "Failed to request PMU region 0x%llx\n", res_start);
 528                 return -EBUSY;
 529         }
 530
 531         writel_relaxed(NI_PMCR_RESET_CCNT | NI_PMCR_RESET_EVCNT,
 532                        cd->pmu_base + NI_PMCR);
 533         writel_relaxed(U32_MAX, cd->pmu_base + NI_PMCNTENCLR);
 534         writel_relaxed(U32_MAX, cd->pmu_base + NI_PMOVSCLR);
 535         writel_relaxed(U32_MAX, cd->pmu_base + NI_PMINTENSET);
 536
 537         cd->irq = platform_get_irq(to_platform_device(ni->dev), cd->id);
 538         if (cd->irq < 0)
 539                 return cd->irq;
 540
 541         err = devm_request_irq(ni->dev, cd->irq, arm_ni_handle_irq,
 542                                IRQF_NOBALANCING | IRQF_NO_THREAD,
 543                                dev_name(ni->dev), cd);
 544         if (err)
 545                 return err;
 546
 547         cd->cpu = cpumask_local_spread(0, dev_to_node(ni->dev));
 548         cd->pmu = (struct pmu) {
 549                 .module = THIS_MODULE,
 550                 .parent = ni->dev,
 551                 .attr_groups = arm_ni_attr_groups,
 552                 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
 553                 .task_ctx_nr = perf_invalid_context,
 554                 .pmu_enable = arm_ni_pmu_enable,
 555                 .pmu_disable = arm_ni_pmu_disable,
 556                 .event_init = arm_ni_event_init,
 557                 .add = arm_ni_event_add,
 558                 .del = arm_ni_event_del,
 559                 .start = arm_ni_event_start,
 560                 .stop = arm_ni_event_stop,
 561                 .read = arm_ni_event_read,
 562         };
 563
 564         name = devm_kasprintf(ni->dev, GFP_KERNEL, "arm_ni_%d_cd_%d", ni->id, cd->id);
 565         if (!name)
 566                 return -ENOMEM;
 567
 568         err = cpuhp_state_add_instance_nocalls(arm_ni_hp_state, &cd->cpuhp_node);
 569         if (err)
 570                 return err;
 571
 572         err = perf_pmu_register(&cd->pmu, name, -1);
 573         if (err)
 574                 cpuhp_state_remove_instance_nocalls(arm_ni_hp_state, &cd->cpuhp_node);
 575
 576         return err;
 577 }
 578
 579 static void arm_ni_probe_domain(void __iomem *base, struct arm_ni_node *node)
 580 {
 581         u32 reg = readl_relaxed(base + NI_NODE_TYPE);
 582
 583         node->base = base;
 584         node->type = FIELD_GET(NI_NODE_TYPE_NODE_TYPE, reg);
 585         node->id = FIELD_GET(NI_NODE_TYPE_NODE_ID, reg);
 586         node->num_components = readl_relaxed(base + NI_CHILD_NODE_INFO);
 587 }
 588
 589 static int arm_ni_probe(struct platform_device *pdev)
 590 {
 591         struct arm_ni_node cfg, vd, pd, cd;
 592         struct arm_ni *ni;
 593         struct resource *res;
 594         void __iomem *base;
 595         static atomic_t id;
 596         int num_cds;
 597         u32 reg, part;
 598
 599         /*
 600          * We want to map the whole configuration space for ease of discovery,
 601          * but the PMU pages are the only ones for which we can honestly claim
 602          * exclusive ownership, so we'll request them explicitly once found.
 603          */
 604         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 605         base = devm_ioremap(&pdev->dev, res->start, resource_size(res));
 606         if (!base)
 607                 return -ENOMEM;
 608
 609         arm_ni_probe_domain(base, &cfg);
 610         if (cfg.type != NI_GLOBAL)
 611                 return -ENODEV;
 612
 613         reg = readl_relaxed(cfg.base + NI_PERIPHERAL_ID0);
 614         part = FIELD_GET(NI_PIDR0_PART_7_0, reg);
 615         reg = readl_relaxed(cfg.base + NI_PERIPHERAL_ID1);
 616         part |= FIELD_GET(NI_PIDR1_PART_11_8, reg) << 8;
 617
 618         switch (part) {
 619         case PART_NI_700:
 620         case PART_NI_710AE:
 621                 break;
 622         default:
 623                 dev_WARN(&pdev->dev, "Unknown part number: 0x%03x, this may go badly\n", part);
 624                 break;
 625         }
 626
 627         num_cds = 0;
 628         for (int v = 0; v < cfg.num_components; v++) {
 629                 reg = readl_relaxed(cfg.base + NI_CHILD_PTR(v));
 630                 arm_ni_probe_domain(base + reg, &vd);
 631                 for (int p = 0; p < vd.num_components; p++) {
 632                         reg = readl_relaxed(vd.base + NI_CHILD_PTR(p));
 633                         arm_ni_probe_domain(base + reg, &pd);
 634                         num_cds += pd.num_components;
 635                 }
 636         }
 637
 638         ni = devm_kzalloc(&pdev->dev, struct_size(ni, cds, num_cds), GFP_KERNEL);
 639         if (!ni)
 640                 return -ENOMEM;
 641
 642         ni->dev = &pdev->dev;
 643         ni->base = base;
 644         ni->num_cds = num_cds;
 645         ni->part = part;
 646         ni->id = atomic_fetch_inc(&id);
 647
 648         for (int v = 0; v < cfg.num_components; v++) {
 649                 reg = readl_relaxed(cfg.base + NI_CHILD_PTR(v));
 650                 arm_ni_probe_domain(base + reg, &vd);
 651                 for (int p = 0; p < vd.num_components; p++) {
 652                         reg = readl_relaxed(vd.base + NI_CHILD_PTR(p));
 653                         arm_ni_probe_domain(base + reg, &pd);
 654                         for (int c = 0; c < pd.num_components; c++) {
 655                                 int ret;
 656
 657                                 reg = readl_relaxed(pd.base + NI_CHILD_PTR(c));
 658                                 arm_ni_probe_domain(base + reg, &cd);
 659                                 ret = arm_ni_init_cd(ni, &cd, res->start);
 660                                 if (ret)
 661                                         return ret;
 662                         }
 663                 }
 664         }
 665
 666         return 0;
 667 }
 668
 669 static void arm_ni_remove(struct platform_device *pdev)
 670 {
 671         struct arm_ni *ni = platform_get_drvdata(pdev);
 672
 673         for (int i = 0; i < ni->num_cds; i++) {
 674                 struct arm_ni_cd *cd = ni->cds + i;
 675
 676                 if (!cd->pmu_base)
 677                         continue;
 678
 679                 writel_relaxed(0, cd->pmu_base + NI_PMCR);
 680                 writel_relaxed(U32_MAX, cd->pmu_base + NI_PMINTENCLR);
 681                 perf_pmu_unregister(&cd->pmu);
 682                 cpuhp_state_remove_instance_nocalls(arm_ni_hp_state, &cd->cpuhp_node);
 683         }
 684 }
 685
 686 #ifdef CONFIG_OF
 687 static const struct of_device_id arm_ni_of_match[] = {
 688         { .compatible = "arm,ni-700" },
 689         {}
 690 };
 691 MODULE_DEVICE_TABLE(of, arm_ni_of_match);
 692 #endif
 693
 694 #ifdef CONFIG_ACPI
 695 static const struct acpi_device_id arm_ni_acpi_match[] = {
 696         { "ARMHCB70" },
 697         {}
 698 };
 699 MODULE_DEVICE_TABLE(acpi, arm_ni_acpi_match);
 700 #endif
 701
 702 static struct platform_driver arm_ni_driver = {
 703         .driver = {
 704                 .name = "arm-ni",
 705                 .of_match_table = of_match_ptr(arm_ni_of_match),
 706                 .acpi_match_table = ACPI_PTR(arm_ni_acpi_match),
 707         },
 708         .probe = arm_ni_probe,
 709         .remove = arm_ni_remove,
 710 };
 711
 712 static void arm_ni_pmu_migrate(struct arm_ni_cd *cd, unsigned int cpu)
 713 {
 714         perf_pmu_migrate_context(&cd->pmu, cd->cpu, cpu);
 715         irq_set_affinity(cd->irq, cpumask_of(cpu));
 716         cd->cpu = cpu;
 717 }
 718
 719 static int arm_ni_pmu_online_cpu(unsigned int cpu, struct hlist_node *cpuhp_node)
 720 {
 721         struct arm_ni_cd *cd;
 722         int node;
 723
 724         cd = hlist_entry_safe(cpuhp_node, struct arm_ni_cd, cpuhp_node);
 725         node = dev_to_node(cd_to_ni(cd)->dev);
 726         if (cpu_to_node(cd->cpu) != node && cpu_to_node(cpu) == node)
 727                 arm_ni_pmu_migrate(cd, cpu);
 728         return 0;
 729 }
 730
 731 static int arm_ni_pmu_offline_cpu(unsigned int cpu, struct hlist_node *cpuhp_node)
 732 {
 733         struct arm_ni_cd *cd;
 734         unsigned int target;
 735         int node;
 736
 737         cd = hlist_entry_safe(cpuhp_node, struct arm_ni_cd, cpuhp_node);
 738         if (cpu != cd->cpu)
 739                 return 0;
 740
 741         node = dev_to_node(cd_to_ni(cd)->dev);
 742         target = cpumask_any_and_but(cpumask_of_node(node), cpu_online_mask, cpu);
 743         if (target >= nr_cpu_ids)
 744                 target = cpumask_any_but(cpu_online_mask, cpu);
 745
 746         if (target < nr_cpu_ids)
 747                 arm_ni_pmu_migrate(cd, target);
 748         return 0;
 749 }
 750
 751 static int __init arm_ni_init(void)
 752 {
 753         int ret;
 754
 755         ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
 756                                       "perf/arm/ni:online",
 757                                       arm_ni_pmu_online_cpu,
 758                                       arm_ni_pmu_offline_cpu);
 759         if (ret < 0)
 760                 return ret;
 761
 762         arm_ni_hp_state = ret;
 763
 764         ret = platform_driver_register(&arm_ni_driver);
 765         if (ret)
 766                 cpuhp_remove_multi_state(arm_ni_hp_state);
 767         return ret;
 768 }
 769
 770 static void __exit arm_ni_exit(void)
 771 {
 772         platform_driver_unregister(&arm_ni_driver);
 773         cpuhp_remove_multi_state(arm_ni_hp_state);
 774 }
 775
 776 module_init(arm_ni_init);
 777 module_exit(arm_ni_exit);
 778
 779 MODULE_AUTHOR("Robin Murphy <robin.murphy@arm.com>");
 780 MODULE_DESCRIPTION("Arm NI-700 PMU driver");
 781 MODULE_LICENSE("GPL v2");