drivers/perf/arm_dsu_pmu.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * ARM DynamIQ Shared Unit (DSU) PMU driver
   4  *
   5  * Copyright (C) ARM Limited, 2017.
   6  *
   7  * Based on ARM CCI-PMU, ARMv8 PMU-v3 drivers.
   8  */
   9
  10 #define PMUNAME         "arm_dsu"
  11 #define DRVNAME         PMUNAME "_pmu"
  12 #define pr_fmt(fmt)     DRVNAME ": " fmt
  13
  14 #include <linux/acpi.h>
  15 #include <linux/bitmap.h>
  16 #include <linux/bitops.h>
  17 #include <linux/bug.h>
  18 #include <linux/cpumask.h>
  19 #include <linux/device.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/kernel.h>
  22 #include <linux/module.h>
  23 #include <linux/of_device.h>
  24 #include <linux/perf_event.h>
  25 #include <linux/platform_device.h>
  26 #include <linux/spinlock.h>
  27 #include <linux/smp.h>
  28 #include <linux/sysfs.h>
  29 #include <linux/types.h>
  30
  31 #include <asm/arm_dsu_pmu.h>
  32 #include <asm/local64.h>
  33
  34 /* PMU event codes */
  35 #define DSU_PMU_EVT_CYCLES              0x11
  36 #define DSU_PMU_EVT_CHAIN               0x1e
  37
  38 #define DSU_PMU_MAX_COMMON_EVENTS       0x40
  39
  40 #define DSU_PMU_MAX_HW_CNTRS            32
  41 #define DSU_PMU_HW_COUNTER_MASK         (DSU_PMU_MAX_HW_CNTRS - 1)
  42
  43 #define CLUSTERPMCR_E                   BIT(0)
  44 #define CLUSTERPMCR_P                   BIT(1)
  45 #define CLUSTERPMCR_C                   BIT(2)
  46 #define CLUSTERPMCR_N_SHIFT             11
  47 #define CLUSTERPMCR_N_MASK              0x1f
  48 #define CLUSTERPMCR_IDCODE_SHIFT        16
  49 #define CLUSTERPMCR_IDCODE_MASK         0xff
  50 #define CLUSTERPMCR_IMP_SHIFT           24
  51 #define CLUSTERPMCR_IMP_MASK            0xff
  52 #define CLUSTERPMCR_RES_MASK            0x7e8
  53 #define CLUSTERPMCR_RES_VAL             0x40
  54
  55 #define DSU_ACTIVE_CPU_MASK             0x0
  56 #define DSU_ASSOCIATED_CPU_MASK         0x1
  57
  58 /*
  59  * We use the index of the counters as they appear in the counter
  60  * bit maps in the PMU registers (e.g CLUSTERPMSELR).
  61  * i.e,
  62  *      counter 0       - Bit 0
  63  *      counter 1       - Bit 1
  64  *      ...
  65  *      Cycle counter   - Bit 31
  66  */
  67 #define DSU_PMU_IDX_CYCLE_COUNTER       31
  68
  69 /* All event counters are 32bit, with a 64bit Cycle counter */
  70 #define DSU_PMU_COUNTER_WIDTH(idx)      \
  71         (((idx) == DSU_PMU_IDX_CYCLE_COUNTER) ? 64 : 32)
  72
  73 #define DSU_PMU_COUNTER_MASK(idx)       \
  74         GENMASK_ULL((DSU_PMU_COUNTER_WIDTH((idx)) - 1), 0)
  75
  76 #define DSU_EXT_ATTR(_name, _func, _config)             \
  77         (&((struct dev_ext_attribute[]) {                               \
  78                 {                                                       \
  79                         .attr = __ATTR(_name, 0444, _func, NULL),       \
  80                         .var = (void *)_config                          \
  81                 }                                                       \
  82         })[0].attr.attr)
  83
  84 #define DSU_EVENT_ATTR(_name, _config)          \
  85         DSU_EXT_ATTR(_name, dsu_pmu_sysfs_event_show, (unsigned long)_config)
  86
  87 #define DSU_FORMAT_ATTR(_name, _config)         \
  88         DSU_EXT_ATTR(_name, dsu_pmu_sysfs_format_show, (char *)_config)
  89
  90 #define DSU_CPUMASK_ATTR(_name, _config)        \
  91         DSU_EXT_ATTR(_name, dsu_pmu_cpumask_show, (unsigned long)_config)
  92
  93 struct dsu_hw_events {
  94         DECLARE_BITMAP(used_mask, DSU_PMU_MAX_HW_CNTRS);
  95         struct perf_event       *events[DSU_PMU_MAX_HW_CNTRS];
  96 };
  97
  98 /*
  99  * struct dsu_pmu       - DSU PMU descriptor
 100  *
 101  * @pmu_lock            : Protects accesses to DSU PMU register from normal vs
 102  *                        interrupt handler contexts.
 103  * @hw_events           : Holds the event counter state.
 104  * @associated_cpus     : CPUs attached to the DSU.
 105  * @active_cpu          : CPU to which the PMU is bound for accesses.
 106  * @cpuhp_node          : Node for CPU hotplug notifier link.
 107  * @num_counters        : Number of event counters implemented by the PMU,
 108  *                        excluding the cycle counter.
 109  * @irq                 : Interrupt line for counter overflow.
 110  * @cpmceid_bitmap      : Bitmap for the availability of architected common
 111  *                        events (event_code < 0x40).
 112  */
 113 struct dsu_pmu {
 114         struct pmu                      pmu;
 115         struct device                   *dev;
 116         raw_spinlock_t                  pmu_lock;
 117         struct dsu_hw_events            hw_events;
 118         cpumask_t                       associated_cpus;
 119         cpumask_t                       active_cpu;
 120         struct hlist_node               cpuhp_node;
 121         s8                              num_counters;
 122         int                             irq;
 123         DECLARE_BITMAP(cpmceid_bitmap, DSU_PMU_MAX_COMMON_EVENTS);
 124 };
 125
 126 static unsigned long dsu_pmu_cpuhp_state;
 127
 128 static inline struct dsu_pmu *to_dsu_pmu(struct pmu *pmu)
 129 {
 130         return container_of(pmu, struct dsu_pmu, pmu);
 131 }
 132
 133 static ssize_t dsu_pmu_sysfs_event_show(struct device *dev,
 134                                         struct device_attribute *attr,
 135                                         char *buf)
 136 {
 137         struct dev_ext_attribute *eattr = container_of(attr,
 138                                         struct dev_ext_attribute, attr);
 139         return snprintf(buf, PAGE_SIZE, "event=0x%lx\n",
 140                                          (unsigned long)eattr->var);
 141 }
 142
 143 static ssize_t dsu_pmu_sysfs_format_show(struct device *dev,
 144                                          struct device_attribute *attr,
 145                                          char *buf)
 146 {
 147         struct dev_ext_attribute *eattr = container_of(attr,
 148                                         struct dev_ext_attribute, attr);
 149         return snprintf(buf, PAGE_SIZE, "%s\n", (char *)eattr->var);
 150 }
 151
 152 static ssize_t dsu_pmu_cpumask_show(struct device *dev,
 153                                     struct device_attribute *attr,
 154                                     char *buf)
 155 {
 156         struct pmu *pmu = dev_get_drvdata(dev);
 157         struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu);
 158         struct dev_ext_attribute *eattr = container_of(attr,
 159                                         struct dev_ext_attribute, attr);
 160         unsigned long mask_id = (unsigned long)eattr->var;
 161         const cpumask_t *cpumask;
 162
 163         switch (mask_id) {
 164         case DSU_ACTIVE_CPU_MASK:
 165                 cpumask = &dsu_pmu->active_cpu;
 166                 break;
 167         case DSU_ASSOCIATED_CPU_MASK:
 168                 cpumask = &dsu_pmu->associated_cpus;
 169                 break;
 170         default:
 171                 return 0;
 172         }
 173         return cpumap_print_to_pagebuf(true, buf, cpumask);
 174 }
 175
 176 static struct attribute *dsu_pmu_format_attrs[] = {
 177         DSU_FORMAT_ATTR(event, "config:0-31"),
 178         NULL,
 179 };
 180
 181 static const struct attribute_group dsu_pmu_format_attr_group = {
 182         .name = "format",
 183         .attrs = dsu_pmu_format_attrs,
 184 };
 185
 186 static struct attribute *dsu_pmu_event_attrs[] = {
 187         DSU_EVENT_ATTR(cycles, 0x11),
 188         DSU_EVENT_ATTR(bus_access, 0x19),
 189         DSU_EVENT_ATTR(memory_error, 0x1a),
 190         DSU_EVENT_ATTR(bus_cycles, 0x1d),
 191         DSU_EVENT_ATTR(l3d_cache_allocate, 0x29),
 192         DSU_EVENT_ATTR(l3d_cache_refill, 0x2a),
 193         DSU_EVENT_ATTR(l3d_cache, 0x2b),
 194         DSU_EVENT_ATTR(l3d_cache_wb, 0x2c),
 195         NULL,
 196 };
 197
 198 static umode_t
 199 dsu_pmu_event_attr_is_visible(struct kobject *kobj, struct attribute *attr,
 200                                 int unused)
 201 {
 202         struct pmu *pmu = dev_get_drvdata(kobj_to_dev(kobj));
 203         struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu);
 204         struct dev_ext_attribute *eattr = container_of(attr,
 205                                         struct dev_ext_attribute, attr.attr);
 206         unsigned long evt = (unsigned long)eattr->var;
 207
 208         return test_bit(evt, dsu_pmu->cpmceid_bitmap) ? attr->mode : 0;
 209 }
 210
 211 static const struct attribute_group dsu_pmu_events_attr_group = {
 212         .name = "events",
 213         .attrs = dsu_pmu_event_attrs,
 214         .is_visible = dsu_pmu_event_attr_is_visible,
 215 };
 216
 217 static struct attribute *dsu_pmu_cpumask_attrs[] = {
 218         DSU_CPUMASK_ATTR(cpumask, DSU_ACTIVE_CPU_MASK),
 219         DSU_CPUMASK_ATTR(associated_cpus, DSU_ASSOCIATED_CPU_MASK),
 220         NULL,
 221 };
 222
 223 static const struct attribute_group dsu_pmu_cpumask_attr_group = {
 224         .attrs = dsu_pmu_cpumask_attrs,
 225 };
 226
 227 static const struct attribute_group *dsu_pmu_attr_groups[] = {
 228         &dsu_pmu_cpumask_attr_group,
 229         &dsu_pmu_events_attr_group,
 230         &dsu_pmu_format_attr_group,
 231         NULL,
 232 };
 233
 234 static int dsu_pmu_get_online_cpu_any_but(struct dsu_pmu *dsu_pmu, int cpu)
 235 {
 236         struct cpumask online_supported;
 237
 238         cpumask_and(&online_supported,
 239                          &dsu_pmu->associated_cpus, cpu_online_mask);
 240         return cpumask_any_but(&online_supported, cpu);
 241 }
 242
 243 static inline bool dsu_pmu_counter_valid(struct dsu_pmu *dsu_pmu, u32 idx)
 244 {
 245         return (idx < dsu_pmu->num_counters) ||
 246                (idx == DSU_PMU_IDX_CYCLE_COUNTER);
 247 }
 248
 249 static inline u64 dsu_pmu_read_counter(struct perf_event *event)
 250 {
 251         u64 val;
 252         unsigned long flags;
 253         struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
 254         int idx = event->hw.idx;
 255
 256         if (WARN_ON(!cpumask_test_cpu(smp_processor_id(),
 257                                  &dsu_pmu->associated_cpus)))
 258                 return 0;
 259
 260         if (!dsu_pmu_counter_valid(dsu_pmu, idx)) {
 261                 dev_err(event->pmu->dev,
 262                         "Trying reading invalid counter %d\n", idx);
 263                 return 0;
 264         }
 265
 266         raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);
 267         if (idx == DSU_PMU_IDX_CYCLE_COUNTER)
 268                 val = __dsu_pmu_read_pmccntr();
 269         else
 270                 val = __dsu_pmu_read_counter(idx);
 271         raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);
 272
 273         return val;
 274 }
 275
 276 static void dsu_pmu_write_counter(struct perf_event *event, u64 val)
 277 {
 278         unsigned long flags;
 279         struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
 280         int idx = event->hw.idx;
 281
 282         if (WARN_ON(!cpumask_test_cpu(smp_processor_id(),
 283                          &dsu_pmu->associated_cpus)))
 284                 return;
 285
 286         if (!dsu_pmu_counter_valid(dsu_pmu, idx)) {
 287                 dev_err(event->pmu->dev,
 288                         "writing to invalid counter %d\n", idx);
 289                 return;
 290         }
 291
 292         raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);
 293         if (idx == DSU_PMU_IDX_CYCLE_COUNTER)
 294                 __dsu_pmu_write_pmccntr(val);
 295         else
 296                 __dsu_pmu_write_counter(idx, val);
 297         raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);
 298 }
 299
 300 static int dsu_pmu_get_event_idx(struct dsu_hw_events *hw_events,
 301                                  struct perf_event *event)
 302 {
 303         int idx;
 304         unsigned long evtype = event->attr.config;
 305         struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
 306         unsigned long *used_mask = hw_events->used_mask;
 307
 308         if (evtype == DSU_PMU_EVT_CYCLES) {
 309                 if (test_and_set_bit(DSU_PMU_IDX_CYCLE_COUNTER, used_mask))
 310                         return -EAGAIN;
 311                 return DSU_PMU_IDX_CYCLE_COUNTER;
 312         }
 313
 314         idx = find_first_zero_bit(used_mask, dsu_pmu->num_counters);
 315         if (idx >= dsu_pmu->num_counters)
 316                 return -EAGAIN;
 317         set_bit(idx, hw_events->used_mask);
 318         return idx;
 319 }
 320
 321 static void dsu_pmu_enable_counter(struct dsu_pmu *dsu_pmu, int idx)
 322 {
 323         __dsu_pmu_counter_interrupt_enable(idx);
 324         __dsu_pmu_enable_counter(idx);
 325 }
 326
 327 static void dsu_pmu_disable_counter(struct dsu_pmu *dsu_pmu, int idx)
 328 {
 329         __dsu_pmu_disable_counter(idx);
 330         __dsu_pmu_counter_interrupt_disable(idx);
 331 }
 332
 333 static inline void dsu_pmu_set_event(struct dsu_pmu *dsu_pmu,
 334                                         struct perf_event *event)
 335 {
 336         int idx = event->hw.idx;
 337         unsigned long flags;
 338
 339         if (!dsu_pmu_counter_valid(dsu_pmu, idx)) {
 340                 dev_err(event->pmu->dev,
 341                         "Trying to set invalid counter %d\n", idx);
 342                 return;
 343         }
 344
 345         raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);
 346         __dsu_pmu_set_event(idx, event->hw.config_base);
 347         raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);
 348 }
 349
 350 static void dsu_pmu_event_update(struct perf_event *event)
 351 {
 352         struct hw_perf_event *hwc = &event->hw;
 353         u64 delta, prev_count, new_count;
 354
 355         do {
 356                 /* We may also be called from the irq handler */
 357                 prev_count = local64_read(&hwc->prev_count);
 358                 new_count = dsu_pmu_read_counter(event);
 359         } while (local64_cmpxchg(&hwc->prev_count, prev_count, new_count) !=
 360                         prev_count);
 361         delta = (new_count - prev_count) & DSU_PMU_COUNTER_MASK(hwc->idx);
 362         local64_add(delta, &event->count);
 363 }
 364
 365 static void dsu_pmu_read(struct perf_event *event)
 366 {
 367         dsu_pmu_event_update(event);
 368 }
 369
 370 static inline u32 dsu_pmu_get_reset_overflow(void)
 371 {
 372         return __dsu_pmu_get_reset_overflow();
 373 }
 374
 375 /**
 376  * dsu_pmu_set_event_period: Set the period for the counter.
 377  *
 378  * All DSU PMU event counters, except the cycle counter are 32bit
 379  * counters. To handle cases of extreme interrupt latency, we program
 380  * the counter with half of the max count for the counters.
 381  */
 382 static void dsu_pmu_set_event_period(struct perf_event *event)
 383 {
 384         int idx = event->hw.idx;
 385         u64 val = DSU_PMU_COUNTER_MASK(idx) >> 1;
 386
 387         local64_set(&event->hw.prev_count, val);
 388         dsu_pmu_write_counter(event, val);
 389 }
 390
 391 static irqreturn_t dsu_pmu_handle_irq(int irq_num, void *dev)
 392 {
 393         int i;
 394         bool handled = false;
 395         struct dsu_pmu *dsu_pmu = dev;
 396         struct dsu_hw_events *hw_events = &dsu_pmu->hw_events;
 397         unsigned long overflow;
 398
 399         overflow = dsu_pmu_get_reset_overflow();
 400         if (!overflow)
 401                 return IRQ_NONE;
 402
 403         for_each_set_bit(i, &overflow, DSU_PMU_MAX_HW_CNTRS) {
 404                 struct perf_event *event = hw_events->events[i];
 405
 406                 if (!event)
 407                         continue;
 408                 dsu_pmu_event_update(event);
 409                 dsu_pmu_set_event_period(event);
 410                 handled = true;
 411         }
 412
 413         return IRQ_RETVAL(handled);
 414 }
 415
 416 static void dsu_pmu_start(struct perf_event *event, int pmu_flags)
 417 {
 418         struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
 419
 420         /* We always reprogram the counter */
 421         if (pmu_flags & PERF_EF_RELOAD)
 422                 WARN_ON(!(event->hw.state & PERF_HES_UPTODATE));
 423         dsu_pmu_set_event_period(event);
 424         if (event->hw.idx != DSU_PMU_IDX_CYCLE_COUNTER)
 425                 dsu_pmu_set_event(dsu_pmu, event);
 426         event->hw.state = 0;
 427         dsu_pmu_enable_counter(dsu_pmu, event->hw.idx);
 428 }
 429
 430 static void dsu_pmu_stop(struct perf_event *event, int pmu_flags)
 431 {
 432         struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
 433
 434         if (event->hw.state & PERF_HES_STOPPED)
 435                 return;
 436         dsu_pmu_disable_counter(dsu_pmu, event->hw.idx);
 437         dsu_pmu_event_update(event);
 438         event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
 439 }
 440
 441 static int dsu_pmu_add(struct perf_event *event, int flags)
 442 {
 443         struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
 444         struct dsu_hw_events *hw_events = &dsu_pmu->hw_events;
 445         struct hw_perf_event *hwc = &event->hw;
 446         int idx;
 447
 448         if (WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(),
 449                                            &dsu_pmu->associated_cpus)))
 450                 return -ENOENT;
 451
 452         idx = dsu_pmu_get_event_idx(hw_events, event);
 453         if (idx < 0)
 454                 return idx;
 455
 456         hwc->idx = idx;
 457         hw_events->events[idx] = event;
 458         hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
 459
 460         if (flags & PERF_EF_START)
 461                 dsu_pmu_start(event, PERF_EF_RELOAD);
 462
 463         perf_event_update_userpage(event);
 464         return 0;
 465 }
 466
 467 static void dsu_pmu_del(struct perf_event *event, int flags)
 468 {
 469         struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
 470         struct dsu_hw_events *hw_events = &dsu_pmu->hw_events;
 471         struct hw_perf_event *hwc = &event->hw;
 472         int idx = hwc->idx;
 473
 474         dsu_pmu_stop(event, PERF_EF_UPDATE);
 475         hw_events->events[idx] = NULL;
 476         clear_bit(idx, hw_events->used_mask);
 477         perf_event_update_userpage(event);
 478 }
 479
 480 static void dsu_pmu_enable(struct pmu *pmu)
 481 {
 482         u32 pmcr;
 483         unsigned long flags;
 484         struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu);
 485
 486         /* If no counters are added, skip enabling the PMU */
 487         if (bitmap_empty(dsu_pmu->hw_events.used_mask, DSU_PMU_MAX_HW_CNTRS))
 488                 return;
 489
 490         raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);
 491         pmcr = __dsu_pmu_read_pmcr();
 492         pmcr |= CLUSTERPMCR_E;
 493         __dsu_pmu_write_pmcr(pmcr);
 494         raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);
 495 }
 496
 497 static void dsu_pmu_disable(struct pmu *pmu)
 498 {
 499         u32 pmcr;
 500         unsigned long flags;
 501         struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu);
 502
 503         raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);
 504         pmcr = __dsu_pmu_read_pmcr();
 505         pmcr &= ~CLUSTERPMCR_E;
 506         __dsu_pmu_write_pmcr(pmcr);
 507         raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);
 508 }
 509
 510 static bool dsu_pmu_validate_event(struct pmu *pmu,
 511                                   struct dsu_hw_events *hw_events,
 512                                   struct perf_event *event)
 513 {
 514         if (is_software_event(event))
 515                 return true;
 516         /* Reject groups spanning multiple HW PMUs. */
 517         if (event->pmu != pmu)
 518                 return false;
 519         return dsu_pmu_get_event_idx(hw_events, event) >= 0;
 520 }
 521
 522 /*
 523  * Make sure the group of events can be scheduled at once
 524  * on the PMU.
 525  */
 526 static bool dsu_pmu_validate_group(struct perf_event *event)
 527 {
 528         struct perf_event *sibling, *leader = event->group_leader;
 529         struct dsu_hw_events fake_hw;
 530
 531         if (event->group_leader == event)
 532                 return true;
 533
 534         memset(fake_hw.used_mask, 0, sizeof(fake_hw.used_mask));
 535         if (!dsu_pmu_validate_event(event->pmu, &fake_hw, leader))
 536                 return false;
 537         for_each_sibling_event(sibling, leader) {
 538                 if (!dsu_pmu_validate_event(event->pmu, &fake_hw, sibling))
 539                         return false;
 540         }
 541         return dsu_pmu_validate_event(event->pmu, &fake_hw, event);
 542 }
 543
 544 static int dsu_pmu_event_init(struct perf_event *event)
 545 {
 546         struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
 547
 548         if (event->attr.type != event->pmu->type)
 549                 return -ENOENT;
 550
 551         /* We don't support sampling */
 552         if (is_sampling_event(event)) {
 553                 dev_dbg(dsu_pmu->pmu.dev, "Can't support sampling events\n");
 554                 return -EOPNOTSUPP;
 555         }
 556
 557         /* We cannot support task bound events */
 558         if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK) {
 559                 dev_dbg(dsu_pmu->pmu.dev, "Can't support per-task counters\n");
 560                 return -EINVAL;
 561         }
 562
 563         if (has_branch_stack(event)) {
 564                 dev_dbg(dsu_pmu->pmu.dev, "Can't support filtering\n");
 565                 return -EINVAL;
 566         }
 567
 568         if (!cpumask_test_cpu(event->cpu, &dsu_pmu->associated_cpus)) {
 569                 dev_dbg(dsu_pmu->pmu.dev,
 570                          "Requested cpu is not associated with the DSU\n");
 571                 return -EINVAL;
 572         }
 573         /*
 574          * Choose the current active CPU to read the events. We don't want
 575          * to migrate the event contexts, irq handling etc to the requested
 576          * CPU. As long as the requested CPU is within the same DSU, we
 577          * are fine.
 578          */
 579         event->cpu = cpumask_first(&dsu_pmu->active_cpu);
 580         if (event->cpu >= nr_cpu_ids)
 581                 return -EINVAL;
 582         if (!dsu_pmu_validate_group(event))
 583                 return -EINVAL;
 584
 585         event->hw.config_base = event->attr.config;
 586         return 0;
 587 }
 588
 589 static struct dsu_pmu *dsu_pmu_alloc(struct platform_device *pdev)
 590 {
 591         struct dsu_pmu *dsu_pmu;
 592
 593         dsu_pmu = devm_kzalloc(&pdev->dev, sizeof(*dsu_pmu), GFP_KERNEL);
 594         if (!dsu_pmu)
 595                 return ERR_PTR(-ENOMEM);
 596
 597         raw_spin_lock_init(&dsu_pmu->pmu_lock);
 598         /*
 599          * Initialise the number of counters to -1, until we probe
 600          * the real number on a connected CPU.
 601          */
 602         dsu_pmu->num_counters = -1;
 603         return dsu_pmu;
 604 }
 605
 606 /**
 607  * dsu_pmu_dt_get_cpus: Get the list of CPUs in the cluster
 608  * from device tree.
 609  */
 610 static int dsu_pmu_dt_get_cpus(struct device *dev, cpumask_t *mask)
 611 {
 612         int i = 0, n, cpu;
 613         struct device_node *cpu_node;
 614
 615         n = of_count_phandle_with_args(dev->of_node, "cpus", NULL);
 616         if (n <= 0)
 617                 return -ENODEV;
 618         for (; i < n; i++) {
 619                 cpu_node = of_parse_phandle(dev->of_node, "cpus", i);
 620                 if (!cpu_node)
 621                         break;
 622                 cpu = of_cpu_node_to_id(cpu_node);
 623                 of_node_put(cpu_node);
 624                 /*
 625                  * We have to ignore the failures here and continue scanning
 626                  * the list to handle cases where the nr_cpus could be capped
 627                  * in the running kernel.
 628                  */
 629                 if (cpu < 0)
 630                         continue;
 631                 cpumask_set_cpu(cpu, mask);
 632         }
 633         return 0;
 634 }
 635
 636 /**
 637  * dsu_pmu_acpi_get_cpus: Get the list of CPUs in the cluster
 638  * from ACPI.
 639  */
 640 static int dsu_pmu_acpi_get_cpus(struct device *dev, cpumask_t *mask)
 641 {
 642 #ifdef CONFIG_ACPI
 643         int cpu;
 644
 645         /*
 646          * A dsu pmu node is inside a cluster parent node along with cpu nodes.
 647          * We need to find out all cpus that have the same parent with this pmu.
 648          */
 649         for_each_possible_cpu(cpu) {
 650                 struct acpi_device *acpi_dev;
 651                 struct device *cpu_dev = get_cpu_device(cpu);
 652
 653                 if (!cpu_dev)
 654                         continue;
 655
 656                 acpi_dev = ACPI_COMPANION(cpu_dev);
 657                 if (acpi_dev &&
 658                         acpi_dev->parent == ACPI_COMPANION(dev)->parent)
 659                         cpumask_set_cpu(cpu, mask);
 660         }
 661 #endif
 662
 663         return 0;
 664 }
 665
 666 /*
 667  * dsu_pmu_probe_pmu: Probe the PMU details on a CPU in the cluster.
 668  */
 669 static void dsu_pmu_probe_pmu(struct dsu_pmu *dsu_pmu)
 670 {
 671         u64 num_counters;
 672         u32 cpmceid[2];
 673
 674         num_counters = (__dsu_pmu_read_pmcr() >> CLUSTERPMCR_N_SHIFT) &
 675                                                 CLUSTERPMCR_N_MASK;
 676         /* We can only support up to 31 independent counters */
 677         if (WARN_ON(num_counters > 31))
 678                 num_counters = 31;
 679         dsu_pmu->num_counters = num_counters;
 680         if (!dsu_pmu->num_counters)
 681                 return;
 682         cpmceid[0] = __dsu_pmu_read_pmceid(0);
 683         cpmceid[1] = __dsu_pmu_read_pmceid(1);
 684         bitmap_from_arr32(dsu_pmu->cpmceid_bitmap, cpmceid,
 685                           DSU_PMU_MAX_COMMON_EVENTS);
 686 }
 687
 688 static void dsu_pmu_set_active_cpu(int cpu, struct dsu_pmu *dsu_pmu)
 689 {
 690         cpumask_set_cpu(cpu, &dsu_pmu->active_cpu);
 691         if (irq_set_affinity_hint(dsu_pmu->irq, &dsu_pmu->active_cpu))
 692                 pr_warn("Failed to set irq affinity to %d\n", cpu);
 693 }
 694
 695 /*
 696  * dsu_pmu_init_pmu: Initialise the DSU PMU configurations if
 697  * we haven't done it already.
 698  */
 699 static void dsu_pmu_init_pmu(struct dsu_pmu *dsu_pmu)
 700 {
 701         if (dsu_pmu->num_counters == -1)
 702                 dsu_pmu_probe_pmu(dsu_pmu);
 703         /* Reset the interrupt overflow mask */
 704         dsu_pmu_get_reset_overflow();
 705 }
 706
 707 static int dsu_pmu_device_probe(struct platform_device *pdev)
 708 {
 709         int irq, rc;
 710         struct dsu_pmu *dsu_pmu;
 711         struct fwnode_handle *fwnode = dev_fwnode(&pdev->dev);
 712         char *name;
 713         static atomic_t pmu_idx = ATOMIC_INIT(-1);
 714
 715         dsu_pmu = dsu_pmu_alloc(pdev);
 716         if (IS_ERR(dsu_pmu))
 717                 return PTR_ERR(dsu_pmu);
 718
 719         if (is_of_node(fwnode))
 720                 rc = dsu_pmu_dt_get_cpus(&pdev->dev, &dsu_pmu->associated_cpus);
 721         else if (is_acpi_device_node(fwnode))
 722                 rc = dsu_pmu_acpi_get_cpus(&pdev->dev, &dsu_pmu->associated_cpus);
 723         else
 724                 return -ENOENT;
 725
 726         if (rc) {
 727                 dev_warn(&pdev->dev, "Failed to parse the CPUs\n");
 728                 return rc;
 729         }
 730
 731         irq = platform_get_irq(pdev, 0);
 732         if (irq < 0)
 733                 return -EINVAL;
 734
 735         name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s_%d",
 736                                 PMUNAME, atomic_inc_return(&pmu_idx));
 737         if (!name)
 738                 return -ENOMEM;
 739         rc = devm_request_irq(&pdev->dev, irq, dsu_pmu_handle_irq,
 740                               IRQF_NOBALANCING, name, dsu_pmu);
 741         if (rc) {
 742                 dev_warn(&pdev->dev, "Failed to request IRQ %d\n", irq);
 743                 return rc;
 744         }
 745
 746         dsu_pmu->irq = irq;
 747         platform_set_drvdata(pdev, dsu_pmu);
 748         rc = cpuhp_state_add_instance(dsu_pmu_cpuhp_state,
 749                                                 &dsu_pmu->cpuhp_node);
 750         if (rc)
 751                 return rc;
 752
 753         dsu_pmu->pmu = (struct pmu) {
 754                 .task_ctx_nr    = perf_invalid_context,
 755                 .module         = THIS_MODULE,
 756                 .pmu_enable     = dsu_pmu_enable,
 757                 .pmu_disable    = dsu_pmu_disable,
 758                 .event_init     = dsu_pmu_event_init,
 759                 .add            = dsu_pmu_add,
 760                 .del            = dsu_pmu_del,
 761                 .start          = dsu_pmu_start,
 762                 .stop           = dsu_pmu_stop,
 763                 .read           = dsu_pmu_read,
 764
 765                 .attr_groups    = dsu_pmu_attr_groups,
 766                 .capabilities   = PERF_PMU_CAP_NO_EXCLUDE,
 767         };
 768
 769         rc = perf_pmu_register(&dsu_pmu->pmu, name, -1);
 770         if (rc) {
 771                 cpuhp_state_remove_instance(dsu_pmu_cpuhp_state,
 772                                                  &dsu_pmu->cpuhp_node);
 773                 irq_set_affinity_hint(dsu_pmu->irq, NULL);
 774         }
 775
 776         return rc;
 777 }
 778
 779 static int dsu_pmu_device_remove(struct platform_device *pdev)
 780 {
 781         struct dsu_pmu *dsu_pmu = platform_get_drvdata(pdev);
 782
 783         perf_pmu_unregister(&dsu_pmu->pmu);
 784         cpuhp_state_remove_instance(dsu_pmu_cpuhp_state, &dsu_pmu->cpuhp_node);
 785         irq_set_affinity_hint(dsu_pmu->irq, NULL);
 786
 787         return 0;
 788 }
 789
 790 static const struct of_device_id dsu_pmu_of_match[] = {
 791         { .compatible = "arm,dsu-pmu", },
 792         {},
 793 };
 794 MODULE_DEVICE_TABLE(of, dsu_pmu_of_match);
 795
 796 #ifdef CONFIG_ACPI
 797 static const struct acpi_device_id dsu_pmu_acpi_match[] = {
 798         { "ARMHD500", 0},
 799         {},
 800 };
 801 MODULE_DEVICE_TABLE(acpi, dsu_pmu_acpi_match);
 802 #endif
 803
 804 static struct platform_driver dsu_pmu_driver = {
 805         .driver = {
 806                 .name   = DRVNAME,
 807                 .of_match_table = of_match_ptr(dsu_pmu_of_match),
 808                 .acpi_match_table = ACPI_PTR(dsu_pmu_acpi_match),
 809                 .suppress_bind_attrs = true,
 810         },
 811         .probe = dsu_pmu_device_probe,
 812         .remove = dsu_pmu_device_remove,
 813 };
 814
 815 static int dsu_pmu_cpu_online(unsigned int cpu, struct hlist_node *node)
 816 {
 817         struct dsu_pmu *dsu_pmu = hlist_entry_safe(node, struct dsu_pmu,
 818                                                    cpuhp_node);
 819
 820         if (!cpumask_test_cpu(cpu, &dsu_pmu->associated_cpus))
 821                 return 0;
 822
 823         /* If the PMU is already managed, there is nothing to do */
 824         if (!cpumask_empty(&dsu_pmu->active_cpu))
 825                 return 0;
 826
 827         dsu_pmu_init_pmu(dsu_pmu);
 828         dsu_pmu_set_active_cpu(cpu, dsu_pmu);
 829
 830         return 0;
 831 }
 832
 833 static int dsu_pmu_cpu_teardown(unsigned int cpu, struct hlist_node *node)
 834 {
 835         int dst;
 836         struct dsu_pmu *dsu_pmu = hlist_entry_safe(node, struct dsu_pmu,
 837                                                    cpuhp_node);
 838
 839         if (!cpumask_test_and_clear_cpu(cpu, &dsu_pmu->active_cpu))
 840                 return 0;
 841
 842         dst = dsu_pmu_get_online_cpu_any_but(dsu_pmu, cpu);
 843         /* If there are no active CPUs in the DSU, leave IRQ disabled */
 844         if (dst >= nr_cpu_ids) {
 845                 irq_set_affinity_hint(dsu_pmu->irq, NULL);
 846                 return 0;
 847         }
 848
 849         perf_pmu_migrate_context(&dsu_pmu->pmu, cpu, dst);
 850         dsu_pmu_set_active_cpu(dst, dsu_pmu);
 851
 852         return 0;
 853 }
 854
 855 static int __init dsu_pmu_init(void)
 856 {
 857         int ret;
 858
 859         ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
 860                                         DRVNAME,
 861                                         dsu_pmu_cpu_online,
 862                                         dsu_pmu_cpu_teardown);
 863         if (ret < 0)
 864                 return ret;
 865         dsu_pmu_cpuhp_state = ret;
 866         return platform_driver_register(&dsu_pmu_driver);
 867 }
 868
 869 static void __exit dsu_pmu_exit(void)
 870 {
 871         platform_driver_unregister(&dsu_pmu_driver);
 872         cpuhp_remove_multi_state(dsu_pmu_cpuhp_state);
 873 }
 874
 875 module_init(dsu_pmu_init);
 876 module_exit(dsu_pmu_exit);
 877
 878 MODULE_DESCRIPTION("Perf driver for ARM DynamIQ Shared Unit");
 879 MODULE_AUTHOR("Suzuki K Poulose <suzuki.poulose@arm.com>");
 880 MODULE_LICENSE("GPL v2");