arch/arm/mm/cache-l2x0-pmu.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * L220/L310 cache controller support
   4  *
   5  * Copyright (C) 2016 ARM Limited
   6  */
   7 #include <linux/errno.h>
   8 #include <linux/hrtimer.h>
   9 #include <linux/io.h>
  10 #include <linux/list.h>
  11 #include <linux/perf_event.h>
  12 #include <linux/printk.h>
  13 #include <linux/slab.h>
  14 #include <linux/types.h>
  15
  16 #include <asm/hardware/cache-l2x0.h>
  17
  18 #define PMU_NR_COUNTERS 2
  19
  20 static void __iomem *l2x0_base;
  21 static struct pmu *l2x0_pmu;
  22 static cpumask_t pmu_cpu;
  23
  24 static const char *l2x0_name;
  25
  26 static ktime_t l2x0_pmu_poll_period;
  27 static struct hrtimer l2x0_pmu_hrtimer;
  28
  29 /*
  30  * The L220/PL310 PMU has two equivalent counters, Counter1 and Counter0.
  31  * Registers controlling these are laid out in pairs, in descending order, i.e.
  32  * the register for Counter1 comes first, followed by the register for
  33  * Counter0.
  34  * We ensure that idx 0 -> Counter0, and idx1 -> Counter1.
  35  */
  36 static struct perf_event *events[PMU_NR_COUNTERS];
  37
  38 /* Find an unused counter */
  39 static int l2x0_pmu_find_idx(void)
  40 {
  41         int i;
  42
  43         for (i = 0; i < PMU_NR_COUNTERS; i++) {
  44                 if (!events[i])
  45                         return i;
  46         }
  47
  48         return -1;
  49 }
  50
  51 /* How many counters are allocated? */
  52 static int l2x0_pmu_num_active_counters(void)
  53 {
  54         int i, cnt = 0;
  55
  56         for (i = 0; i < PMU_NR_COUNTERS; i++) {
  57                 if (events[i])
  58                         cnt++;
  59         }
  60
  61         return cnt;
  62 }
  63
  64 static void l2x0_pmu_counter_config_write(int idx, u32 val)
  65 {
  66         writel_relaxed(val, l2x0_base + L2X0_EVENT_CNT0_CFG - 4 * idx);
  67 }
  68
  69 static u32 l2x0_pmu_counter_read(int idx)
  70 {
  71         return readl_relaxed(l2x0_base + L2X0_EVENT_CNT0_VAL - 4 * idx);
  72 }
  73
  74 static void l2x0_pmu_counter_write(int idx, u32 val)
  75 {
  76         writel_relaxed(val, l2x0_base + L2X0_EVENT_CNT0_VAL - 4 * idx);
  77 }
  78
  79 static void __l2x0_pmu_enable(void)
  80 {
  81         u32 val = readl_relaxed(l2x0_base + L2X0_EVENT_CNT_CTRL);
  82         val |= L2X0_EVENT_CNT_CTRL_ENABLE;
  83         writel_relaxed(val, l2x0_base + L2X0_EVENT_CNT_CTRL);
  84 }
  85
  86 static void __l2x0_pmu_disable(void)
  87 {
  88         u32 val = readl_relaxed(l2x0_base + L2X0_EVENT_CNT_CTRL);
  89         val &= ~L2X0_EVENT_CNT_CTRL_ENABLE;
  90         writel_relaxed(val, l2x0_base + L2X0_EVENT_CNT_CTRL);
  91 }
  92
  93 static void l2x0_pmu_enable(struct pmu *pmu)
  94 {
  95         if (l2x0_pmu_num_active_counters() == 0)
  96                 return;
  97
  98         __l2x0_pmu_enable();
  99 }
 100
 101 static void l2x0_pmu_disable(struct pmu *pmu)
 102 {
 103         if (l2x0_pmu_num_active_counters() == 0)
 104                 return;
 105
 106         __l2x0_pmu_disable();
 107 }
 108
 109 static void warn_if_saturated(u32 count)
 110 {
 111         if (count != 0xffffffff)
 112                 return;
 113
 114         pr_warn_ratelimited("L2X0 counter saturated. Poll period too long\n");
 115 }
 116
 117 static void l2x0_pmu_event_read(struct perf_event *event)
 118 {
 119         struct hw_perf_event *hw = &event->hw;
 120         u64 prev_count, new_count, mask;
 121
 122         do {
 123                  prev_count = local64_read(&hw->prev_count);
 124                  new_count = l2x0_pmu_counter_read(hw->idx);
 125         } while (local64_xchg(&hw->prev_count, new_count) != prev_count);
 126
 127         mask = GENMASK_ULL(31, 0);
 128         local64_add((new_count - prev_count) & mask, &event->count);
 129
 130         warn_if_saturated(new_count);
 131 }
 132
 133 static void l2x0_pmu_event_configure(struct perf_event *event)
 134 {
 135         struct hw_perf_event *hw = &event->hw;
 136
 137         /*
 138          * The L2X0 counters saturate at 0xffffffff rather than wrapping, so we
 139          * will *always* lose some number of events when a counter saturates,
 140          * and have no way of detecting how many were lost.
 141          *
 142          * To minimize the impact of this, we try to maximize the period by
 143          * always starting counters at zero. To ensure that group ratios are
 144          * representative, we poll periodically to avoid counters saturating.
 145          * See l2x0_pmu_poll().
 146          */
 147         local64_set(&hw->prev_count, 0);
 148         l2x0_pmu_counter_write(hw->idx, 0);
 149 }
 150
 151 static enum hrtimer_restart l2x0_pmu_poll(struct hrtimer *hrtimer)
 152 {
 153         unsigned long flags;
 154         int i;
 155
 156         local_irq_save(flags);
 157         __l2x0_pmu_disable();
 158
 159         for (i = 0; i < PMU_NR_COUNTERS; i++) {
 160                 struct perf_event *event = events[i];
 161
 162                 if (!event)
 163                         continue;
 164
 165                 l2x0_pmu_event_read(event);
 166                 l2x0_pmu_event_configure(event);
 167         }
 168
 169         __l2x0_pmu_enable();
 170         local_irq_restore(flags);
 171
 172         hrtimer_forward_now(hrtimer, l2x0_pmu_poll_period);
 173         return HRTIMER_RESTART;
 174 }
 175
 176
 177 static void __l2x0_pmu_event_enable(int idx, u32 event)
 178 {
 179         u32 val;
 180
 181         val = event << L2X0_EVENT_CNT_CFG_SRC_SHIFT;
 182         val |= L2X0_EVENT_CNT_CFG_INT_DISABLED;
 183         l2x0_pmu_counter_config_write(idx, val);
 184 }
 185
 186 static void l2x0_pmu_event_start(struct perf_event *event, int flags)
 187 {
 188         struct hw_perf_event *hw = &event->hw;
 189
 190         if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
 191                 return;
 192
 193         if (flags & PERF_EF_RELOAD) {
 194                 WARN_ON_ONCE(!(hw->state & PERF_HES_UPTODATE));
 195                 l2x0_pmu_event_configure(event);
 196         }
 197
 198         hw->state = 0;
 199
 200         __l2x0_pmu_event_enable(hw->idx, hw->config_base);
 201 }
 202
 203 static void __l2x0_pmu_event_disable(int idx)
 204 {
 205         u32 val;
 206
 207         val = L2X0_EVENT_CNT_CFG_SRC_DISABLED << L2X0_EVENT_CNT_CFG_SRC_SHIFT;
 208         val |= L2X0_EVENT_CNT_CFG_INT_DISABLED;
 209         l2x0_pmu_counter_config_write(idx, val);
 210 }
 211
 212 static void l2x0_pmu_event_stop(struct perf_event *event, int flags)
 213 {
 214         struct hw_perf_event *hw = &event->hw;
 215
 216         if (WARN_ON_ONCE(event->hw.state & PERF_HES_STOPPED))
 217                 return;
 218
 219         __l2x0_pmu_event_disable(hw->idx);
 220
 221         hw->state |= PERF_HES_STOPPED;
 222
 223         if (flags & PERF_EF_UPDATE) {
 224                 l2x0_pmu_event_read(event);
 225                 hw->state |= PERF_HES_UPTODATE;
 226         }
 227 }
 228
 229 static int l2x0_pmu_event_add(struct perf_event *event, int flags)
 230 {
 231         struct hw_perf_event *hw = &event->hw;
 232         int idx = l2x0_pmu_find_idx();
 233
 234         if (idx == -1)
 235                 return -EAGAIN;
 236
 237         /*
 238          * Pin the timer, so that the overflows are handled by the chosen
 239          * event->cpu (this is the same one as presented in "cpumask"
 240          * attribute).
 241          */
 242         if (l2x0_pmu_num_active_counters() == 0)
 243                 hrtimer_start(&l2x0_pmu_hrtimer, l2x0_pmu_poll_period,
 244                               HRTIMER_MODE_REL_PINNED);
 245
 246         events[idx] = event;
 247         hw->idx = idx;
 248
 249         l2x0_pmu_event_configure(event);
 250
 251         hw->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
 252
 253         if (flags & PERF_EF_START)
 254                 l2x0_pmu_event_start(event, 0);
 255
 256         return 0;
 257 }
 258
 259 static void l2x0_pmu_event_del(struct perf_event *event, int flags)
 260 {
 261         struct hw_perf_event *hw = &event->hw;
 262
 263         l2x0_pmu_event_stop(event, PERF_EF_UPDATE);
 264
 265         events[hw->idx] = NULL;
 266         hw->idx = -1;
 267
 268         if (l2x0_pmu_num_active_counters() == 0)
 269                 hrtimer_cancel(&l2x0_pmu_hrtimer);
 270 }
 271
 272 static bool l2x0_pmu_group_is_valid(struct perf_event *event)
 273 {
 274         struct pmu *pmu = event->pmu;
 275         struct perf_event *leader = event->group_leader;
 276         struct perf_event *sibling;
 277         int num_hw = 0;
 278
 279         if (leader->pmu == pmu)
 280                 num_hw++;
 281         else if (!is_software_event(leader))
 282                 return false;
 283
 284         for_each_sibling_event(sibling, leader) {
 285                 if (sibling->pmu == pmu)
 286                         num_hw++;
 287                 else if (!is_software_event(sibling))
 288                         return false;
 289         }
 290
 291         return num_hw <= PMU_NR_COUNTERS;
 292 }
 293
 294 static int l2x0_pmu_event_init(struct perf_event *event)
 295 {
 296         struct hw_perf_event *hw = &event->hw;
 297
 298         if (event->attr.type != l2x0_pmu->type)
 299                 return -ENOENT;
 300
 301         if (is_sampling_event(event) ||
 302             event->attach_state & PERF_ATTACH_TASK)
 303                 return -EINVAL;
 304
 305         if (event->cpu < 0)
 306                 return -EINVAL;
 307
 308         if (event->attr.config & ~L2X0_EVENT_CNT_CFG_SRC_MASK)
 309                 return -EINVAL;
 310
 311         hw->config_base = event->attr.config;
 312
 313         if (!l2x0_pmu_group_is_valid(event))
 314                 return -EINVAL;
 315
 316         event->cpu = cpumask_first(&pmu_cpu);
 317
 318         return 0;
 319 }
 320
 321 struct l2x0_event_attribute {
 322         struct device_attribute attr;
 323         unsigned int config;
 324         bool pl310_only;
 325 };
 326
 327 #define L2X0_EVENT_ATTR(_name, _config, _pl310_only)                            \
 328         (&((struct l2x0_event_attribute[]) {{                                   \
 329                 .attr = __ATTR(_name, S_IRUGO, l2x0_pmu_event_show, NULL),      \
 330                 .config = _config,                                              \
 331                 .pl310_only = _pl310_only,                                      \
 332         }})[0].attr.attr)
 333
 334 #define L220_PLUS_EVENT_ATTR(_name, _config)                                    \
 335         L2X0_EVENT_ATTR(_name, _config, false)
 336
 337 #define PL310_EVENT_ATTR(_name, _config)                                        \
 338         L2X0_EVENT_ATTR(_name, _config, true)
 339
 340 static ssize_t l2x0_pmu_event_show(struct device *dev,
 341                                    struct device_attribute *attr, char *buf)
 342 {
 343         struct l2x0_event_attribute *lattr;
 344
 345         lattr = container_of(attr, typeof(*lattr), attr);
 346         return snprintf(buf, PAGE_SIZE, "config=0x%x\n", lattr->config);
 347 }
 348
 349 static umode_t l2x0_pmu_event_attr_is_visible(struct kobject *kobj,
 350                                               struct attribute *attr,
 351                                               int unused)
 352 {
 353         struct device *dev = kobj_to_dev(kobj);
 354         struct pmu *pmu = dev_get_drvdata(dev);
 355         struct l2x0_event_attribute *lattr;
 356
 357         lattr = container_of(attr, typeof(*lattr), attr.attr);
 358
 359         if (!lattr->pl310_only || strcmp("l2c_310", pmu->name) == 0)
 360                 return attr->mode;
 361
 362         return 0;
 363 }
 364
 365 static struct attribute *l2x0_pmu_event_attrs[] = {
 366         L220_PLUS_EVENT_ATTR(co,        0x1),
 367         L220_PLUS_EVENT_ATTR(drhit,     0x2),
 368         L220_PLUS_EVENT_ATTR(drreq,     0x3),
 369         L220_PLUS_EVENT_ATTR(dwhit,     0x4),
 370         L220_PLUS_EVENT_ATTR(dwreq,     0x5),
 371         L220_PLUS_EVENT_ATTR(dwtreq,    0x6),
 372         L220_PLUS_EVENT_ATTR(irhit,     0x7),
 373         L220_PLUS_EVENT_ATTR(irreq,     0x8),
 374         L220_PLUS_EVENT_ATTR(wa,        0x9),
 375         PL310_EVENT_ATTR(ipfalloc,      0xa),
 376         PL310_EVENT_ATTR(epfhit,        0xb),
 377         PL310_EVENT_ATTR(epfalloc,      0xc),
 378         PL310_EVENT_ATTR(srrcvd,        0xd),
 379         PL310_EVENT_ATTR(srconf,        0xe),
 380         PL310_EVENT_ATTR(epfrcvd,       0xf),
 381         NULL
 382 };
 383
 384 static struct attribute_group l2x0_pmu_event_attrs_group = {
 385         .name = "events",
 386         .attrs = l2x0_pmu_event_attrs,
 387         .is_visible = l2x0_pmu_event_attr_is_visible,
 388 };
 389
 390 static ssize_t l2x0_pmu_cpumask_show(struct device *dev,
 391                                      struct device_attribute *attr, char *buf)
 392 {
 393         return cpumap_print_to_pagebuf(true, buf, &pmu_cpu);
 394 }
 395
 396 static struct device_attribute l2x0_pmu_cpumask_attr =
 397                 __ATTR(cpumask, S_IRUGO, l2x0_pmu_cpumask_show, NULL);
 398
 399 static struct attribute *l2x0_pmu_cpumask_attrs[] = {
 400         &l2x0_pmu_cpumask_attr.attr,
 401         NULL,
 402 };
 403
 404 static struct attribute_group l2x0_pmu_cpumask_attr_group = {
 405         .attrs = l2x0_pmu_cpumask_attrs,
 406 };
 407
 408 static const struct attribute_group *l2x0_pmu_attr_groups[] = {
 409         &l2x0_pmu_event_attrs_group,
 410         &l2x0_pmu_cpumask_attr_group,
 411         NULL,
 412 };
 413
 414 static void l2x0_pmu_reset(void)
 415 {
 416         int i;
 417
 418         __l2x0_pmu_disable();
 419
 420         for (i = 0; i < PMU_NR_COUNTERS; i++)
 421                 __l2x0_pmu_event_disable(i);
 422 }
 423
 424 static int l2x0_pmu_offline_cpu(unsigned int cpu)
 425 {
 426         unsigned int target;
 427
 428         if (!cpumask_test_and_clear_cpu(cpu, &pmu_cpu))
 429                 return 0;
 430
 431         target = cpumask_any_but(cpu_online_mask, cpu);
 432         if (target >= nr_cpu_ids)
 433                 return 0;
 434
 435         perf_pmu_migrate_context(l2x0_pmu, cpu, target);
 436         cpumask_set_cpu(target, &pmu_cpu);
 437
 438         return 0;
 439 }
 440
 441 void l2x0_pmu_suspend(void)
 442 {
 443         int i;
 444
 445         if (!l2x0_pmu)
 446                 return;
 447
 448         l2x0_pmu_disable(l2x0_pmu);
 449
 450         for (i = 0; i < PMU_NR_COUNTERS; i++) {
 451                 if (events[i])
 452                         l2x0_pmu_event_stop(events[i], PERF_EF_UPDATE);
 453         }
 454
 455 }
 456
 457 void l2x0_pmu_resume(void)
 458 {
 459         int i;
 460
 461         if (!l2x0_pmu)
 462                 return;
 463
 464         l2x0_pmu_reset();
 465
 466         for (i = 0; i < PMU_NR_COUNTERS; i++) {
 467                 if (events[i])
 468                         l2x0_pmu_event_start(events[i], PERF_EF_RELOAD);
 469         }
 470
 471         l2x0_pmu_enable(l2x0_pmu);
 472 }
 473
 474 void __init l2x0_pmu_register(void __iomem *base, u32 part)
 475 {
 476         /*
 477          * Determine whether we support the PMU, and choose the name for sysfs.
 478          * This is also used by l2x0_pmu_event_attr_is_visible to determine
 479          * which events to display, as the PL310 PMU supports a superset of
 480          * L220 events.
 481          *
 482          * The L210 PMU has a different programmer's interface, and is not
 483          * supported by this driver.
 484          *
 485          * We must defer registering the PMU until the perf subsystem is up and
 486          * running, so just stash the name and base, and leave that to another
 487          * initcall.
 488          */
 489         switch (part & L2X0_CACHE_ID_PART_MASK) {
 490         case L2X0_CACHE_ID_PART_L220:
 491                 l2x0_name = "l2c_220";
 492                 break;
 493         case L2X0_CACHE_ID_PART_L310:
 494                 l2x0_name = "l2c_310";
 495                 break;
 496         default:
 497                 return;
 498         }
 499
 500         l2x0_base = base;
 501 }
 502
 503 static __init int l2x0_pmu_init(void)
 504 {
 505         int ret;
 506
 507         if (!l2x0_base)
 508                 return 0;
 509
 510         l2x0_pmu = kzalloc(sizeof(*l2x0_pmu), GFP_KERNEL);
 511         if (!l2x0_pmu) {
 512                 pr_warn("Unable to allocate L2x0 PMU\n");
 513                 return -ENOMEM;
 514         }
 515
 516         *l2x0_pmu = (struct pmu) {
 517                 .task_ctx_nr = perf_invalid_context,
 518                 .pmu_enable = l2x0_pmu_enable,
 519                 .pmu_disable = l2x0_pmu_disable,
 520                 .read = l2x0_pmu_event_read,
 521                 .start = l2x0_pmu_event_start,
 522                 .stop = l2x0_pmu_event_stop,
 523                 .add = l2x0_pmu_event_add,
 524                 .del = l2x0_pmu_event_del,
 525                 .event_init = l2x0_pmu_event_init,
 526                 .attr_groups = l2x0_pmu_attr_groups,
 527                 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
 528         };
 529
 530         l2x0_pmu_reset();
 531
 532         /*
 533          * We always use a hrtimer rather than an interrupt.
 534          * See comments in l2x0_pmu_event_configure and l2x0_pmu_poll.
 535          *
 536          * Polling once a second allows the counters to fill up to 1/128th on a
 537          * quad-core test chip with cores clocked at 400MHz. Hopefully this
 538          * leaves sufficient headroom to avoid overflow on production silicon
 539          * at higher frequencies.
 540          */
 541         l2x0_pmu_poll_period = ms_to_ktime(1000);
 542         hrtimer_init(&l2x0_pmu_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 543         l2x0_pmu_hrtimer.function = l2x0_pmu_poll;
 544
 545         cpumask_set_cpu(0, &pmu_cpu);
 546         ret = cpuhp_setup_state_nocalls(CPUHP_AP_PERF_ARM_L2X0_ONLINE,
 547                                         "perf/arm/l2x0:online", NULL,
 548                                         l2x0_pmu_offline_cpu);
 549         if (ret)
 550                 goto out_pmu;
 551
 552         ret = perf_pmu_register(l2x0_pmu, l2x0_name, -1);
 553         if (ret)
 554                 goto out_cpuhp;
 555
 556         return 0;
 557
 558 out_cpuhp:
 559         cpuhp_remove_state_nocalls(CPUHP_AP_PERF_ARM_L2X0_ONLINE);
 560 out_pmu:
 561         kfree(l2x0_pmu);
 562         l2x0_pmu = NULL;
 563         return ret;
 564 }
 565 device_initcall(l2x0_pmu_init);