drivers/perf/arm_dsu_pmu.c

   1 /*
   2  * ARM DynamIQ Shared Unit (DSU) PMU driver
   3  *
   4  * Copyright (C) ARM Limited, 2017.
   5  *
   6  * Based on ARM CCI-PMU, ARMv8 PMU-v3 drivers.
   7  *
   8  * This program is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU General Public License
  10  * version 2 as published by the Free Software Foundation.
  11  */
  12
  13 #define PMUNAME         "arm_dsu"
  14 #define DRVNAME         PMUNAME "_pmu"
  15 #define pr_fmt(fmt)     DRVNAME ": " fmt
  16
  17 #include <linux/bitmap.h>
  18 #include <linux/bitops.h>
  19 #include <linux/bug.h>
  20 #include <linux/cpumask.h>
  21 #include <linux/device.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/kernel.h>
  24 #include <linux/module.h>
  25 #include <linux/of_device.h>
  26 #include <linux/perf_event.h>
  27 #include <linux/platform_device.h>
  28 #include <linux/spinlock.h>
  29 #include <linux/smp.h>
  30 #include <linux/sysfs.h>
  31 #include <linux/types.h>
  32
  33 #include <asm/arm_dsu_pmu.h>
  34 #include <asm/local64.h>
  35
  36 /* PMU event codes */
  37 #define DSU_PMU_EVT_CYCLES              0x11
  38 #define DSU_PMU_EVT_CHAIN               0x1e
  39
  40 #define DSU_PMU_MAX_COMMON_EVENTS       0x40
  41
  42 #define DSU_PMU_MAX_HW_CNTRS            32
  43 #define DSU_PMU_HW_COUNTER_MASK         (DSU_PMU_MAX_HW_CNTRS - 1)
  44
  45 #define CLUSTERPMCR_E                   BIT(0)
  46 #define CLUSTERPMCR_P                   BIT(1)
  47 #define CLUSTERPMCR_C                   BIT(2)
  48 #define CLUSTERPMCR_N_SHIFT             11
  49 #define CLUSTERPMCR_N_MASK              0x1f
  50 #define CLUSTERPMCR_IDCODE_SHIFT        16
  51 #define CLUSTERPMCR_IDCODE_MASK         0xff
  52 #define CLUSTERPMCR_IMP_SHIFT           24
  53 #define CLUSTERPMCR_IMP_MASK            0xff
  54 #define CLUSTERPMCR_RES_MASK            0x7e8
  55 #define CLUSTERPMCR_RES_VAL             0x40
  56
  57 #define DSU_ACTIVE_CPU_MASK             0x0
  58 #define DSU_ASSOCIATED_CPU_MASK         0x1
  59
  60 /*
  61  * We use the index of the counters as they appear in the counter
  62  * bit maps in the PMU registers (e.g CLUSTERPMSELR).
  63  * i.e,
  64  *      counter 0       - Bit 0
  65  *      counter 1       - Bit 1
  66  *      ...
  67  *      Cycle counter   - Bit 31
  68  */
  69 #define DSU_PMU_IDX_CYCLE_COUNTER       31
  70
  71 /* All event counters are 32bit, with a 64bit Cycle counter */
  72 #define DSU_PMU_COUNTER_WIDTH(idx)      \
  73         (((idx) == DSU_PMU_IDX_CYCLE_COUNTER) ? 64 : 32)
  74
  75 #define DSU_PMU_COUNTER_MASK(idx)       \
  76         GENMASK_ULL((DSU_PMU_COUNTER_WIDTH((idx)) - 1), 0)
  77
  78 #define DSU_EXT_ATTR(_name, _func, _config)             \
  79         (&((struct dev_ext_attribute[]) {                               \
  80                 {                                                       \
  81                         .attr = __ATTR(_name, 0444, _func, NULL),       \
  82                         .var = (void *)_config                          \
  83                 }                                                       \
  84         })[0].attr.attr)
  85
  86 #define DSU_EVENT_ATTR(_name, _config)          \
  87         DSU_EXT_ATTR(_name, dsu_pmu_sysfs_event_show, (unsigned long)_config)
  88
  89 #define DSU_FORMAT_ATTR(_name, _config)         \
  90         DSU_EXT_ATTR(_name, dsu_pmu_sysfs_format_show, (char *)_config)
  91
  92 #define DSU_CPUMASK_ATTR(_name, _config)        \
  93         DSU_EXT_ATTR(_name, dsu_pmu_cpumask_show, (unsigned long)_config)
  94
  95 struct dsu_hw_events {
  96         DECLARE_BITMAP(used_mask, DSU_PMU_MAX_HW_CNTRS);
  97         struct perf_event       *events[DSU_PMU_MAX_HW_CNTRS];
  98 };
  99
 100 /*
 101  * struct dsu_pmu       - DSU PMU descriptor
 102  *
 103  * @pmu_lock            : Protects accesses to DSU PMU register from normal vs
 104  *                        interrupt handler contexts.
 105  * @hw_events           : Holds the event counter state.
 106  * @associated_cpus     : CPUs attached to the DSU.
 107  * @active_cpu          : CPU to which the PMU is bound for accesses.
 108  * @cpuhp_node          : Node for CPU hotplug notifier link.
 109  * @num_counters        : Number of event counters implemented by the PMU,
 110  *                        excluding the cycle counter.
 111  * @irq                 : Interrupt line for counter overflow.
 112  * @cpmceid_bitmap      : Bitmap for the availability of architected common
 113  *                        events (event_code < 0x40).
 114  */
 115 struct dsu_pmu {
 116         struct pmu                      pmu;
 117         struct device                   *dev;
 118         raw_spinlock_t                  pmu_lock;
 119         struct dsu_hw_events            hw_events;
 120         cpumask_t                       associated_cpus;
 121         cpumask_t                       active_cpu;
 122         struct hlist_node               cpuhp_node;
 123         s8                              num_counters;
 124         int                             irq;
 125         DECLARE_BITMAP(cpmceid_bitmap, DSU_PMU_MAX_COMMON_EVENTS);
 126 };
 127
 128 static unsigned long dsu_pmu_cpuhp_state;
 129
 130 static inline struct dsu_pmu *to_dsu_pmu(struct pmu *pmu)
 131 {
 132         return container_of(pmu, struct dsu_pmu, pmu);
 133 }
 134
 135 static ssize_t dsu_pmu_sysfs_event_show(struct device *dev,
 136                                         struct device_attribute *attr,
 137                                         char *buf)
 138 {
 139         struct dev_ext_attribute *eattr = container_of(attr,
 140                                         struct dev_ext_attribute, attr);
 141         return snprintf(buf, PAGE_SIZE, "event=0x%lx\n",
 142                                          (unsigned long)eattr->var);
 143 }
 144
 145 static ssize_t dsu_pmu_sysfs_format_show(struct device *dev,
 146                                          struct device_attribute *attr,
 147                                          char *buf)
 148 {
 149         struct dev_ext_attribute *eattr = container_of(attr,
 150                                         struct dev_ext_attribute, attr);
 151         return snprintf(buf, PAGE_SIZE, "%s\n", (char *)eattr->var);
 152 }
 153
 154 static ssize_t dsu_pmu_cpumask_show(struct device *dev,
 155                                     struct device_attribute *attr,
 156                                     char *buf)
 157 {
 158         struct pmu *pmu = dev_get_drvdata(dev);
 159         struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu);
 160         struct dev_ext_attribute *eattr = container_of(attr,
 161                                         struct dev_ext_attribute, attr);
 162         unsigned long mask_id = (unsigned long)eattr->var;
 163         const cpumask_t *cpumask;
 164
 165         switch (mask_id) {
 166         case DSU_ACTIVE_CPU_MASK:
 167                 cpumask = &dsu_pmu->active_cpu;
 168                 break;
 169         case DSU_ASSOCIATED_CPU_MASK:
 170                 cpumask = &dsu_pmu->associated_cpus;
 171                 break;
 172         default:
 173                 return 0;
 174         }
 175         return cpumap_print_to_pagebuf(true, buf, cpumask);
 176 }
 177
 178 static struct attribute *dsu_pmu_format_attrs[] = {
 179         DSU_FORMAT_ATTR(event, "config:0-31"),
 180         NULL,
 181 };
 182
 183 static const struct attribute_group dsu_pmu_format_attr_group = {
 184         .name = "format",
 185         .attrs = dsu_pmu_format_attrs,
 186 };
 187
 188 static struct attribute *dsu_pmu_event_attrs[] = {
 189         DSU_EVENT_ATTR(cycles, 0x11),
 190         DSU_EVENT_ATTR(bus_access, 0x19),
 191         DSU_EVENT_ATTR(memory_error, 0x1a),
 192         DSU_EVENT_ATTR(bus_cycles, 0x1d),
 193         DSU_EVENT_ATTR(l3d_cache_allocate, 0x29),
 194         DSU_EVENT_ATTR(l3d_cache_refill, 0x2a),
 195         DSU_EVENT_ATTR(l3d_cache, 0x2b),
 196         DSU_EVENT_ATTR(l3d_cache_wb, 0x2c),
 197         NULL,
 198 };
 199
 200 static umode_t
 201 dsu_pmu_event_attr_is_visible(struct kobject *kobj, struct attribute *attr,
 202                                 int unused)
 203 {
 204         struct pmu *pmu = dev_get_drvdata(kobj_to_dev(kobj));
 205         struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu);
 206         struct dev_ext_attribute *eattr = container_of(attr,
 207                                         struct dev_ext_attribute, attr.attr);
 208         unsigned long evt = (unsigned long)eattr->var;
 209
 210         return test_bit(evt, dsu_pmu->cpmceid_bitmap) ? attr->mode : 0;
 211 }
 212
 213 static const struct attribute_group dsu_pmu_events_attr_group = {
 214         .name = "events",
 215         .attrs = dsu_pmu_event_attrs,
 216         .is_visible = dsu_pmu_event_attr_is_visible,
 217 };
 218
 219 static struct attribute *dsu_pmu_cpumask_attrs[] = {
 220         DSU_CPUMASK_ATTR(cpumask, DSU_ACTIVE_CPU_MASK),
 221         DSU_CPUMASK_ATTR(associated_cpus, DSU_ASSOCIATED_CPU_MASK),
 222         NULL,
 223 };
 224
 225 static const struct attribute_group dsu_pmu_cpumask_attr_group = {
 226         .attrs = dsu_pmu_cpumask_attrs,
 227 };
 228
 229 static const struct attribute_group *dsu_pmu_attr_groups[] = {
 230         &dsu_pmu_cpumask_attr_group,
 231         &dsu_pmu_events_attr_group,
 232         &dsu_pmu_format_attr_group,
 233         NULL,
 234 };
 235
 236 static int dsu_pmu_get_online_cpu_any_but(struct dsu_pmu *dsu_pmu, int cpu)
 237 {
 238         struct cpumask online_supported;
 239
 240         cpumask_and(&online_supported,
 241                          &dsu_pmu->associated_cpus, cpu_online_mask);
 242         return cpumask_any_but(&online_supported, cpu);
 243 }
 244
 245 static inline bool dsu_pmu_counter_valid(struct dsu_pmu *dsu_pmu, u32 idx)
 246 {
 247         return (idx < dsu_pmu->num_counters) ||
 248                (idx == DSU_PMU_IDX_CYCLE_COUNTER);
 249 }
 250
 251 static inline u64 dsu_pmu_read_counter(struct perf_event *event)
 252 {
 253         u64 val;
 254         unsigned long flags;
 255         struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
 256         int idx = event->hw.idx;
 257
 258         if (WARN_ON(!cpumask_test_cpu(smp_processor_id(),
 259                                  &dsu_pmu->associated_cpus)))
 260                 return 0;
 261
 262         if (!dsu_pmu_counter_valid(dsu_pmu, idx)) {
 263                 dev_err(event->pmu->dev,
 264                         "Trying reading invalid counter %d\n", idx);
 265                 return 0;
 266         }
 267
 268         raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);
 269         if (idx == DSU_PMU_IDX_CYCLE_COUNTER)
 270                 val = __dsu_pmu_read_pmccntr();
 271         else
 272                 val = __dsu_pmu_read_counter(idx);
 273         raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);
 274
 275         return val;
 276 }
 277
 278 static void dsu_pmu_write_counter(struct perf_event *event, u64 val)
 279 {
 280         unsigned long flags;
 281         struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
 282         int idx = event->hw.idx;
 283
 284         if (WARN_ON(!cpumask_test_cpu(smp_processor_id(),
 285                          &dsu_pmu->associated_cpus)))
 286                 return;
 287
 288         if (!dsu_pmu_counter_valid(dsu_pmu, idx)) {
 289                 dev_err(event->pmu->dev,
 290                         "writing to invalid counter %d\n", idx);
 291                 return;
 292         }
 293
 294         raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);
 295         if (idx == DSU_PMU_IDX_CYCLE_COUNTER)
 296                 __dsu_pmu_write_pmccntr(val);
 297         else
 298                 __dsu_pmu_write_counter(idx, val);
 299         raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);
 300 }
 301
 302 static int dsu_pmu_get_event_idx(struct dsu_hw_events *hw_events,
 303                                  struct perf_event *event)
 304 {
 305         int idx;
 306         unsigned long evtype = event->attr.config;
 307         struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
 308         unsigned long *used_mask = hw_events->used_mask;
 309
 310         if (evtype == DSU_PMU_EVT_CYCLES) {
 311                 if (test_and_set_bit(DSU_PMU_IDX_CYCLE_COUNTER, used_mask))
 312                         return -EAGAIN;
 313                 return DSU_PMU_IDX_CYCLE_COUNTER;
 314         }
 315
 316         idx = find_first_zero_bit(used_mask, dsu_pmu->num_counters);
 317         if (idx >= dsu_pmu->num_counters)
 318                 return -EAGAIN;
 319         set_bit(idx, hw_events->used_mask);
 320         return idx;
 321 }
 322
 323 static void dsu_pmu_enable_counter(struct dsu_pmu *dsu_pmu, int idx)
 324 {
 325         __dsu_pmu_counter_interrupt_enable(idx);
 326         __dsu_pmu_enable_counter(idx);
 327 }
 328
 329 static void dsu_pmu_disable_counter(struct dsu_pmu *dsu_pmu, int idx)
 330 {
 331         __dsu_pmu_disable_counter(idx);
 332         __dsu_pmu_counter_interrupt_disable(idx);
 333 }
 334
 335 static inline void dsu_pmu_set_event(struct dsu_pmu *dsu_pmu,
 336                                         struct perf_event *event)
 337 {
 338         int idx = event->hw.idx;
 339         unsigned long flags;
 340
 341         if (!dsu_pmu_counter_valid(dsu_pmu, idx)) {
 342                 dev_err(event->pmu->dev,
 343                         "Trying to set invalid counter %d\n", idx);
 344                 return;
 345         }
 346
 347         raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);
 348         __dsu_pmu_set_event(idx, event->hw.config_base);
 349         raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);
 350 }
 351
 352 static void dsu_pmu_event_update(struct perf_event *event)
 353 {
 354         struct hw_perf_event *hwc = &event->hw;
 355         u64 delta, prev_count, new_count;
 356
 357         do {
 358                 /* We may also be called from the irq handler */
 359                 prev_count = local64_read(&hwc->prev_count);
 360                 new_count = dsu_pmu_read_counter(event);
 361         } while (local64_cmpxchg(&hwc->prev_count, prev_count, new_count) !=
 362                         prev_count);
 363         delta = (new_count - prev_count) & DSU_PMU_COUNTER_MASK(hwc->idx);
 364         local64_add(delta, &event->count);
 365 }
 366
 367 static void dsu_pmu_read(struct perf_event *event)
 368 {
 369         dsu_pmu_event_update(event);
 370 }
 371
 372 static inline u32 dsu_pmu_get_reset_overflow(void)
 373 {
 374         return __dsu_pmu_get_reset_overflow();
 375 }
 376
 377 /**
 378  * dsu_pmu_set_event_period: Set the period for the counter.
 379  *
 380  * All DSU PMU event counters, except the cycle counter are 32bit
 381  * counters. To handle cases of extreme interrupt latency, we program
 382  * the counter with half of the max count for the counters.
 383  */
 384 static void dsu_pmu_set_event_period(struct perf_event *event)
 385 {
 386         int idx = event->hw.idx;
 387         u64 val = DSU_PMU_COUNTER_MASK(idx) >> 1;
 388
 389         local64_set(&event->hw.prev_count, val);
 390         dsu_pmu_write_counter(event, val);
 391 }
 392
 393 static irqreturn_t dsu_pmu_handle_irq(int irq_num, void *dev)
 394 {
 395         int i;
 396         bool handled = false;
 397         struct dsu_pmu *dsu_pmu = dev;
 398         struct dsu_hw_events *hw_events = &dsu_pmu->hw_events;
 399         unsigned long overflow;
 400
 401         overflow = dsu_pmu_get_reset_overflow();
 402         if (!overflow)
 403                 return IRQ_NONE;
 404
 405         for_each_set_bit(i, &overflow, DSU_PMU_MAX_HW_CNTRS) {
 406                 struct perf_event *event = hw_events->events[i];
 407
 408                 if (!event)
 409                         continue;
 410                 dsu_pmu_event_update(event);
 411                 dsu_pmu_set_event_period(event);
 412                 handled = true;
 413         }
 414
 415         return IRQ_RETVAL(handled);
 416 }
 417
 418 static void dsu_pmu_start(struct perf_event *event, int pmu_flags)
 419 {
 420         struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
 421
 422         /* We always reprogram the counter */
 423         if (pmu_flags & PERF_EF_RELOAD)
 424                 WARN_ON(!(event->hw.state & PERF_HES_UPTODATE));
 425         dsu_pmu_set_event_period(event);
 426         if (event->hw.idx != DSU_PMU_IDX_CYCLE_COUNTER)
 427                 dsu_pmu_set_event(dsu_pmu, event);
 428         event->hw.state = 0;
 429         dsu_pmu_enable_counter(dsu_pmu, event->hw.idx);
 430 }
 431
 432 static void dsu_pmu_stop(struct perf_event *event, int pmu_flags)
 433 {
 434         struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
 435
 436         if (event->hw.state & PERF_HES_STOPPED)
 437                 return;
 438         dsu_pmu_disable_counter(dsu_pmu, event->hw.idx);
 439         dsu_pmu_event_update(event);
 440         event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
 441 }
 442
 443 static int dsu_pmu_add(struct perf_event *event, int flags)
 444 {
 445         struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
 446         struct dsu_hw_events *hw_events = &dsu_pmu->hw_events;
 447         struct hw_perf_event *hwc = &event->hw;
 448         int idx;
 449
 450         if (WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(),
 451                                            &dsu_pmu->associated_cpus)))
 452                 return -ENOENT;
 453
 454         idx = dsu_pmu_get_event_idx(hw_events, event);
 455         if (idx < 0)
 456                 return idx;
 457
 458         hwc->idx = idx;
 459         hw_events->events[idx] = event;
 460         hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
 461
 462         if (flags & PERF_EF_START)
 463                 dsu_pmu_start(event, PERF_EF_RELOAD);
 464
 465         perf_event_update_userpage(event);
 466         return 0;
 467 }
 468
 469 static void dsu_pmu_del(struct perf_event *event, int flags)
 470 {
 471         struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
 472         struct dsu_hw_events *hw_events = &dsu_pmu->hw_events;
 473         struct hw_perf_event *hwc = &event->hw;
 474         int idx = hwc->idx;
 475
 476         dsu_pmu_stop(event, PERF_EF_UPDATE);
 477         hw_events->events[idx] = NULL;
 478         clear_bit(idx, hw_events->used_mask);
 479         perf_event_update_userpage(event);
 480 }
 481
 482 static void dsu_pmu_enable(struct pmu *pmu)
 483 {
 484         u32 pmcr;
 485         unsigned long flags;
 486         struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu);
 487
 488         /* If no counters are added, skip enabling the PMU */
 489         if (bitmap_empty(dsu_pmu->hw_events.used_mask, DSU_PMU_MAX_HW_CNTRS))
 490                 return;
 491
 492         raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);
 493         pmcr = __dsu_pmu_read_pmcr();
 494         pmcr |= CLUSTERPMCR_E;
 495         __dsu_pmu_write_pmcr(pmcr);
 496         raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);
 497 }
 498
 499 static void dsu_pmu_disable(struct pmu *pmu)
 500 {
 501         u32 pmcr;
 502         unsigned long flags;
 503         struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu);
 504
 505         raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags);
 506         pmcr = __dsu_pmu_read_pmcr();
 507         pmcr &= ~CLUSTERPMCR_E;
 508         __dsu_pmu_write_pmcr(pmcr);
 509         raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags);
 510 }
 511
 512 static bool dsu_pmu_validate_event(struct pmu *pmu,
 513                                   struct dsu_hw_events *hw_events,
 514                                   struct perf_event *event)
 515 {
 516         if (is_software_event(event))
 517                 return true;
 518         /* Reject groups spanning multiple HW PMUs. */
 519         if (event->pmu != pmu)
 520                 return false;
 521         return dsu_pmu_get_event_idx(hw_events, event) >= 0;
 522 }
 523
 524 /*
 525  * Make sure the group of events can be scheduled at once
 526  * on the PMU.
 527  */
 528 static bool dsu_pmu_validate_group(struct perf_event *event)
 529 {
 530         struct perf_event *sibling, *leader = event->group_leader;
 531         struct dsu_hw_events fake_hw;
 532
 533         if (event->group_leader == event)
 534                 return true;
 535
 536         memset(fake_hw.used_mask, 0, sizeof(fake_hw.used_mask));
 537         if (!dsu_pmu_validate_event(event->pmu, &fake_hw, leader))
 538                 return false;
 539         list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
 540                 if (!dsu_pmu_validate_event(event->pmu, &fake_hw, sibling))
 541                         return false;
 542         }
 543         return dsu_pmu_validate_event(event->pmu, &fake_hw, event);
 544 }
 545
 546 static int dsu_pmu_event_init(struct perf_event *event)
 547 {
 548         struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu);
 549
 550         if (event->attr.type != event->pmu->type)
 551                 return -ENOENT;
 552
 553         /* We don't support sampling */
 554         if (is_sampling_event(event)) {
 555                 dev_dbg(dsu_pmu->pmu.dev, "Can't support sampling events\n");
 556                 return -EOPNOTSUPP;
 557         }
 558
 559         /* We cannot support task bound events */
 560         if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK) {
 561                 dev_dbg(dsu_pmu->pmu.dev, "Can't support per-task counters\n");
 562                 return -EINVAL;
 563         }
 564
 565         if (has_branch_stack(event) ||
 566             event->attr.exclude_user ||
 567             event->attr.exclude_kernel ||
 568             event->attr.exclude_hv ||
 569             event->attr.exclude_idle ||
 570             event->attr.exclude_host ||
 571             event->attr.exclude_guest) {
 572                 dev_dbg(dsu_pmu->pmu.dev, "Can't support filtering\n");
 573                 return -EINVAL;
 574         }
 575
 576         if (!cpumask_test_cpu(event->cpu, &dsu_pmu->associated_cpus)) {
 577                 dev_dbg(dsu_pmu->pmu.dev,
 578                          "Requested cpu is not associated with the DSU\n");
 579                 return -EINVAL;
 580         }
 581         /*
 582          * Choose the current active CPU to read the events. We don't want
 583          * to migrate the event contexts, irq handling etc to the requested
 584          * CPU. As long as the requested CPU is within the same DSU, we
 585          * are fine.
 586          */
 587         event->cpu = cpumask_first(&dsu_pmu->active_cpu);
 588         if (event->cpu >= nr_cpu_ids)
 589                 return -EINVAL;
 590         if (!dsu_pmu_validate_group(event))
 591                 return -EINVAL;
 592
 593         event->hw.config_base = event->attr.config;
 594         return 0;
 595 }
 596
 597 static struct dsu_pmu *dsu_pmu_alloc(struct platform_device *pdev)
 598 {
 599         struct dsu_pmu *dsu_pmu;
 600
 601         dsu_pmu = devm_kzalloc(&pdev->dev, sizeof(*dsu_pmu), GFP_KERNEL);
 602         if (!dsu_pmu)
 603                 return ERR_PTR(-ENOMEM);
 604
 605         raw_spin_lock_init(&dsu_pmu->pmu_lock);
 606         /*
 607          * Initialise the number of counters to -1, until we probe
 608          * the real number on a connected CPU.
 609          */
 610         dsu_pmu->num_counters = -1;
 611         return dsu_pmu;
 612 }
 613
 614 /**
 615  * dsu_pmu_dt_get_cpus: Get the list of CPUs in the cluster.
 616  */
 617 static int dsu_pmu_dt_get_cpus(struct device_node *dev, cpumask_t *mask)
 618 {
 619         int i = 0, n, cpu;
 620         struct device_node *cpu_node;
 621
 622         n = of_count_phandle_with_args(dev, "cpus", NULL);
 623         if (n <= 0)
 624                 return -ENODEV;
 625         for (; i < n; i++) {
 626                 cpu_node = of_parse_phandle(dev, "cpus", i);
 627                 if (!cpu_node)
 628                         break;
 629                 cpu = of_cpu_node_to_id(cpu_node);
 630                 of_node_put(cpu_node);
 631                 /*
 632                  * We have to ignore the failures here and continue scanning
 633                  * the list to handle cases where the nr_cpus could be capped
 634                  * in the running kernel.
 635                  */
 636                 if (cpu < 0)
 637                         continue;
 638                 cpumask_set_cpu(cpu, mask);
 639         }
 640         return 0;
 641 }
 642
 643 /*
 644  * dsu_pmu_probe_pmu: Probe the PMU details on a CPU in the cluster.
 645  */
 646 static void dsu_pmu_probe_pmu(struct dsu_pmu *dsu_pmu)
 647 {
 648         u64 num_counters;
 649         u32 cpmceid[2];
 650
 651         num_counters = (__dsu_pmu_read_pmcr() >> CLUSTERPMCR_N_SHIFT) &
 652                                                 CLUSTERPMCR_N_MASK;
 653         /* We can only support up to 31 independent counters */
 654         if (WARN_ON(num_counters > 31))
 655                 num_counters = 31;
 656         dsu_pmu->num_counters = num_counters;
 657         if (!dsu_pmu->num_counters)
 658                 return;
 659         cpmceid[0] = __dsu_pmu_read_pmceid(0);
 660         cpmceid[1] = __dsu_pmu_read_pmceid(1);
 661         bitmap_from_arr32(dsu_pmu->cpmceid_bitmap, cpmceid,
 662                           DSU_PMU_MAX_COMMON_EVENTS);
 663 }
 664
 665 static void dsu_pmu_set_active_cpu(int cpu, struct dsu_pmu *dsu_pmu)
 666 {
 667         cpumask_set_cpu(cpu, &dsu_pmu->active_cpu);
 668         if (irq_set_affinity_hint(dsu_pmu->irq, &dsu_pmu->active_cpu))
 669                 pr_warn("Failed to set irq affinity to %d\n", cpu);
 670 }
 671
 672 /*
 673  * dsu_pmu_init_pmu: Initialise the DSU PMU configurations if
 674  * we haven't done it already.
 675  */
 676 static void dsu_pmu_init_pmu(struct dsu_pmu *dsu_pmu)
 677 {
 678         if (dsu_pmu->num_counters == -1)
 679                 dsu_pmu_probe_pmu(dsu_pmu);
 680         /* Reset the interrupt overflow mask */
 681         dsu_pmu_get_reset_overflow();
 682 }
 683
 684 static int dsu_pmu_device_probe(struct platform_device *pdev)
 685 {
 686         int irq, rc;
 687         struct dsu_pmu *dsu_pmu;
 688         char *name;
 689         static atomic_t pmu_idx = ATOMIC_INIT(-1);
 690
 691         dsu_pmu = dsu_pmu_alloc(pdev);
 692         if (IS_ERR(dsu_pmu))
 693                 return PTR_ERR(dsu_pmu);
 694
 695         rc = dsu_pmu_dt_get_cpus(pdev->dev.of_node, &dsu_pmu->associated_cpus);
 696         if (rc) {
 697                 dev_warn(&pdev->dev, "Failed to parse the CPUs\n");
 698                 return rc;
 699         }
 700
 701         irq = platform_get_irq(pdev, 0);
 702         if (irq < 0) {
 703                 dev_warn(&pdev->dev, "Failed to find IRQ\n");
 704                 return -EINVAL;
 705         }
 706
 707         name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s_%d",
 708                                 PMUNAME, atomic_inc_return(&pmu_idx));
 709         if (!name)
 710                 return -ENOMEM;
 711         rc = devm_request_irq(&pdev->dev, irq, dsu_pmu_handle_irq,
 712                               IRQF_NOBALANCING, name, dsu_pmu);
 713         if (rc) {
 714                 dev_warn(&pdev->dev, "Failed to request IRQ %d\n", irq);
 715                 return rc;
 716         }
 717
 718         dsu_pmu->irq = irq;
 719         platform_set_drvdata(pdev, dsu_pmu);
 720         rc = cpuhp_state_add_instance(dsu_pmu_cpuhp_state,
 721                                                 &dsu_pmu->cpuhp_node);
 722         if (rc)
 723                 return rc;
 724
 725         dsu_pmu->pmu = (struct pmu) {
 726                 .task_ctx_nr    = perf_invalid_context,
 727                 .module         = THIS_MODULE,
 728                 .pmu_enable     = dsu_pmu_enable,
 729                 .pmu_disable    = dsu_pmu_disable,
 730                 .event_init     = dsu_pmu_event_init,
 731                 .add            = dsu_pmu_add,
 732                 .del            = dsu_pmu_del,
 733                 .start          = dsu_pmu_start,
 734                 .stop           = dsu_pmu_stop,
 735                 .read           = dsu_pmu_read,
 736
 737                 .attr_groups    = dsu_pmu_attr_groups,
 738         };
 739
 740         rc = perf_pmu_register(&dsu_pmu->pmu, name, -1);
 741         if (rc) {
 742                 cpuhp_state_remove_instance(dsu_pmu_cpuhp_state,
 743                                                  &dsu_pmu->cpuhp_node);
 744                 irq_set_affinity_hint(dsu_pmu->irq, NULL);
 745         }
 746
 747         return rc;
 748 }
 749
 750 static int dsu_pmu_device_remove(struct platform_device *pdev)
 751 {
 752         struct dsu_pmu *dsu_pmu = platform_get_drvdata(pdev);
 753
 754         perf_pmu_unregister(&dsu_pmu->pmu);
 755         cpuhp_state_remove_instance(dsu_pmu_cpuhp_state, &dsu_pmu->cpuhp_node);
 756         irq_set_affinity_hint(dsu_pmu->irq, NULL);
 757
 758         return 0;
 759 }
 760
 761 static const struct of_device_id dsu_pmu_of_match[] = {
 762         { .compatible = "arm,dsu-pmu", },
 763         {},
 764 };
 765
 766 static struct platform_driver dsu_pmu_driver = {
 767         .driver = {
 768                 .name   = DRVNAME,
 769                 .of_match_table = of_match_ptr(dsu_pmu_of_match),
 770         },
 771         .probe = dsu_pmu_device_probe,
 772         .remove = dsu_pmu_device_remove,
 773 };
 774
 775 static int dsu_pmu_cpu_online(unsigned int cpu, struct hlist_node *node)
 776 {
 777         struct dsu_pmu *dsu_pmu = hlist_entry_safe(node, struct dsu_pmu,
 778                                                    cpuhp_node);
 779
 780         if (!cpumask_test_cpu(cpu, &dsu_pmu->associated_cpus))
 781                 return 0;
 782
 783         /* If the PMU is already managed, there is nothing to do */
 784         if (!cpumask_empty(&dsu_pmu->active_cpu))
 785                 return 0;
 786
 787         dsu_pmu_init_pmu(dsu_pmu);
 788         dsu_pmu_set_active_cpu(cpu, dsu_pmu);
 789
 790         return 0;
 791 }
 792
 793 static int dsu_pmu_cpu_teardown(unsigned int cpu, struct hlist_node *node)
 794 {
 795         int dst;
 796         struct dsu_pmu *dsu_pmu = hlist_entry_safe(node, struct dsu_pmu,
 797                                                    cpuhp_node);
 798
 799         if (!cpumask_test_and_clear_cpu(cpu, &dsu_pmu->active_cpu))
 800                 return 0;
 801
 802         dst = dsu_pmu_get_online_cpu_any_but(dsu_pmu, cpu);
 803         /* If there are no active CPUs in the DSU, leave IRQ disabled */
 804         if (dst >= nr_cpu_ids) {
 805                 irq_set_affinity_hint(dsu_pmu->irq, NULL);
 806                 return 0;
 807         }
 808
 809         perf_pmu_migrate_context(&dsu_pmu->pmu, cpu, dst);
 810         dsu_pmu_set_active_cpu(dst, dsu_pmu);
 811
 812         return 0;
 813 }
 814
 815 static int __init dsu_pmu_init(void)
 816 {
 817         int ret;
 818
 819         ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
 820                                         DRVNAME,
 821                                         dsu_pmu_cpu_online,
 822                                         dsu_pmu_cpu_teardown);
 823         if (ret < 0)
 824                 return ret;
 825         dsu_pmu_cpuhp_state = ret;
 826         return platform_driver_register(&dsu_pmu_driver);
 827 }
 828
 829 static void __exit dsu_pmu_exit(void)
 830 {
 831         platform_driver_unregister(&dsu_pmu_driver);
 832         cpuhp_remove_multi_state(dsu_pmu_cpuhp_state);
 833 }
 834
 835 module_init(dsu_pmu_init);
 836 module_exit(dsu_pmu_exit);
 837
 838 MODULE_DEVICE_TABLE(of, dsu_pmu_of_match);
 839 MODULE_DESCRIPTION("Perf driver for ARM DynamIQ Shared Unit");
 840 MODULE_AUTHOR("Suzuki K Poulose <suzuki.poulose@arm.com>");
 841 MODULE_LICENSE("GPL v2");