drivers/dma/idxd/perfmon.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2020 Intel Corporation. All rights rsvd. */
   3
   4 #include <linux/sched/task.h>
   5 #include <linux/io-64-nonatomic-lo-hi.h>
   6 #include "idxd.h"
   7 #include "perfmon.h"
   8
   9 /*
  10  * These attributes specify the bits in the config word that the perf
  11  * syscall uses to pass the event ids and categories to perfmon.
  12  */
  13 DEFINE_PERFMON_FORMAT_ATTR(event_category, "config:0-3");
  14 DEFINE_PERFMON_FORMAT_ATTR(event, "config:4-31");
  15
  16 /*
  17  * These attributes specify the bits in the config1 word that the perf
  18  * syscall uses to pass filter data to perfmon.
  19  */
  20 DEFINE_PERFMON_FORMAT_ATTR(filter_wq, "config1:0-31");
  21 DEFINE_PERFMON_FORMAT_ATTR(filter_tc, "config1:32-39");
  22 DEFINE_PERFMON_FORMAT_ATTR(filter_pgsz, "config1:40-43");
  23 DEFINE_PERFMON_FORMAT_ATTR(filter_sz, "config1:44-51");
  24 DEFINE_PERFMON_FORMAT_ATTR(filter_eng, "config1:52-59");
  25
  26 #define PERFMON_FILTERS_START   2
  27 #define PERFMON_FILTERS_MAX     5
  28
  29 static struct attribute *perfmon_format_attrs[] = {
  30         &format_attr_idxd_event_category.attr,
  31         &format_attr_idxd_event.attr,
  32         &format_attr_idxd_filter_wq.attr,
  33         &format_attr_idxd_filter_tc.attr,
  34         &format_attr_idxd_filter_pgsz.attr,
  35         &format_attr_idxd_filter_sz.attr,
  36         &format_attr_idxd_filter_eng.attr,
  37         NULL,
  38 };
  39
  40 static struct attribute_group perfmon_format_attr_group = {
  41         .name = "format",
  42         .attrs = perfmon_format_attrs,
  43 };
  44
  45 static const struct attribute_group *perfmon_attr_groups[] = {
  46         &perfmon_format_attr_group,
  47         NULL,
  48 };
  49
  50 static bool is_idxd_event(struct idxd_pmu *idxd_pmu, struct perf_event *event)
  51 {
  52         return &idxd_pmu->pmu == event->pmu;
  53 }
  54
  55 static int perfmon_collect_events(struct idxd_pmu *idxd_pmu,
  56                                   struct perf_event *leader,
  57                                   bool do_grp)
  58 {
  59         struct perf_event *event;
  60         int n, max_count;
  61
  62         max_count = idxd_pmu->n_counters;
  63         n = idxd_pmu->n_events;
  64
  65         if (n >= max_count)
  66                 return -EINVAL;
  67
  68         if (is_idxd_event(idxd_pmu, leader)) {
  69                 idxd_pmu->event_list[n] = leader;
  70                 idxd_pmu->event_list[n]->hw.idx = n;
  71                 n++;
  72         }
  73
  74         if (!do_grp)
  75                 return n;
  76
  77         for_each_sibling_event(event, leader) {
  78                 if (!is_idxd_event(idxd_pmu, event) ||
  79                     event->state <= PERF_EVENT_STATE_OFF)
  80                         continue;
  81
  82                 if (n >= max_count)
  83                         return -EINVAL;
  84
  85                 idxd_pmu->event_list[n] = event;
  86                 idxd_pmu->event_list[n]->hw.idx = n;
  87                 n++;
  88         }
  89
  90         return n;
  91 }
  92
  93 static void perfmon_assign_hw_event(struct idxd_pmu *idxd_pmu,
  94                                     struct perf_event *event, int idx)
  95 {
  96         struct idxd_device *idxd = idxd_pmu->idxd;
  97         struct hw_perf_event *hwc = &event->hw;
  98
  99         hwc->idx = idx;
 100         hwc->config_base = ioread64(CNTRCFG_REG(idxd, idx));
 101         hwc->event_base = ioread64(CNTRCFG_REG(idxd, idx));
 102 }
 103
 104 static int perfmon_assign_event(struct idxd_pmu *idxd_pmu,
 105                                 struct perf_event *event)
 106 {
 107         int i;
 108
 109         for (i = 0; i < IDXD_PMU_EVENT_MAX; i++)
 110                 if (!test_and_set_bit(i, idxd_pmu->used_mask))
 111                         return i;
 112
 113         return -EINVAL;
 114 }
 115
 116 /*
 117  * Check whether there are enough counters to satisfy that all the
 118  * events in the group can actually be scheduled at the same time.
 119  *
 120  * To do this, create a fake idxd_pmu object so the event collection
 121  * and assignment functions can be used without affecting the internal
 122  * state of the real idxd_pmu object.
 123  */
 124 static int perfmon_validate_group(struct idxd_pmu *pmu,
 125                                   struct perf_event *event)
 126 {
 127         struct perf_event *leader = event->group_leader;
 128         struct idxd_pmu *fake_pmu;
 129         int i, ret = 0, n, idx;
 130
 131         fake_pmu = kzalloc(sizeof(*fake_pmu), GFP_KERNEL);
 132         if (!fake_pmu)
 133                 return -ENOMEM;
 134
 135         fake_pmu->pmu.name = pmu->pmu.name;
 136         fake_pmu->n_counters = pmu->n_counters;
 137
 138         n = perfmon_collect_events(fake_pmu, leader, true);
 139         if (n < 0) {
 140                 ret = n;
 141                 goto out;
 142         }
 143
 144         fake_pmu->n_events = n;
 145         n = perfmon_collect_events(fake_pmu, event, false);
 146         if (n < 0) {
 147                 ret = n;
 148                 goto out;
 149         }
 150
 151         fake_pmu->n_events = n;
 152
 153         for (i = 0; i < n; i++) {
 154                 event = fake_pmu->event_list[i];
 155
 156                 idx = perfmon_assign_event(fake_pmu, event);
 157                 if (idx < 0) {
 158                         ret = idx;
 159                         goto out;
 160                 }
 161         }
 162 out:
 163         kfree(fake_pmu);
 164
 165         return ret;
 166 }
 167
 168 static int perfmon_pmu_event_init(struct perf_event *event)
 169 {
 170         struct idxd_device *idxd;
 171         int ret = 0;
 172
 173         idxd = event_to_idxd(event);
 174         event->hw.idx = -1;
 175
 176         if (event->attr.type != event->pmu->type)
 177                 return -ENOENT;
 178
 179         /* sampling not supported */
 180         if (event->attr.sample_period)
 181                 return -EINVAL;
 182
 183         if (event->cpu < 0)
 184                 return -EINVAL;
 185
 186         if (event->pmu != &idxd->idxd_pmu->pmu)
 187                 return -EINVAL;
 188
 189         event->hw.event_base = ioread64(PERFMON_TABLE_OFFSET(idxd));
 190         event->hw.config = event->attr.config;
 191
 192         if (event->group_leader != event)
 193                  /* non-group events have themselves as leader */
 194                 ret = perfmon_validate_group(idxd->idxd_pmu, event);
 195
 196         return ret;
 197 }
 198
 199 static inline u64 perfmon_pmu_read_counter(struct perf_event *event)
 200 {
 201         struct hw_perf_event *hwc = &event->hw;
 202         struct idxd_device *idxd;
 203         int cntr = hwc->idx;
 204
 205         idxd = event_to_idxd(event);
 206
 207         return ioread64(CNTRDATA_REG(idxd, cntr));
 208 }
 209
 210 static void perfmon_pmu_event_update(struct perf_event *event)
 211 {
 212         struct idxd_device *idxd = event_to_idxd(event);
 213         u64 prev_raw_count, new_raw_count, delta, p, n;
 214         int shift = 64 - idxd->idxd_pmu->counter_width;
 215         struct hw_perf_event *hwc = &event->hw;
 216
 217         prev_raw_count = local64_read(&hwc->prev_count);
 218         do {
 219                 new_raw_count = perfmon_pmu_read_counter(event);
 220         } while (!local64_try_cmpxchg(&hwc->prev_count,
 221                                       &prev_raw_count, new_raw_count));
 222         n = (new_raw_count << shift);
 223         p = (prev_raw_count << shift);
 224
 225         delta = ((n - p) >> shift);
 226
 227         local64_add(delta, &event->count);
 228 }
 229
 230 void perfmon_counter_overflow(struct idxd_device *idxd)
 231 {
 232         int i, n_counters, max_loop = OVERFLOW_SIZE;
 233         struct perf_event *event;
 234         unsigned long ovfstatus;
 235
 236         n_counters = min(idxd->idxd_pmu->n_counters, OVERFLOW_SIZE);
 237
 238         ovfstatus = ioread32(OVFSTATUS_REG(idxd));
 239
 240         /*
 241          * While updating overflowed counters, other counters behind
 242          * them could overflow and be missed in a given pass.
 243          * Normally this could happen at most n_counters times, but in
 244          * theory a tiny counter width could result in continual
 245          * overflows and endless looping.  max_loop provides a
 246          * failsafe in that highly unlikely case.
 247          */
 248         while (ovfstatus && max_loop--) {
 249                 /* Figure out which counter(s) overflowed */
 250                 for_each_set_bit(i, &ovfstatus, n_counters) {
 251                         unsigned long ovfstatus_clear = 0;
 252
 253                         /* Update event->count for overflowed counter */
 254                         event = idxd->idxd_pmu->event_list[i];
 255                         perfmon_pmu_event_update(event);
 256                         /* Writing 1 to OVFSTATUS bit clears it */
 257                         set_bit(i, &ovfstatus_clear);
 258                         iowrite32(ovfstatus_clear, OVFSTATUS_REG(idxd));
 259                 }
 260
 261                 ovfstatus = ioread32(OVFSTATUS_REG(idxd));
 262         }
 263
 264         /*
 265          * Should never happen.  If so, it means a counter(s) looped
 266          * around twice while this handler was running.
 267          */
 268         WARN_ON_ONCE(ovfstatus);
 269 }
 270
 271 static inline void perfmon_reset_config(struct idxd_device *idxd)
 272 {
 273         iowrite32(CONFIG_RESET, PERFRST_REG(idxd));
 274         iowrite32(0, OVFSTATUS_REG(idxd));
 275         iowrite32(0, PERFFRZ_REG(idxd));
 276 }
 277
 278 static inline void perfmon_reset_counters(struct idxd_device *idxd)
 279 {
 280         iowrite32(CNTR_RESET, PERFRST_REG(idxd));
 281 }
 282
 283 static inline void perfmon_reset(struct idxd_device *idxd)
 284 {
 285         perfmon_reset_config(idxd);
 286         perfmon_reset_counters(idxd);
 287 }
 288
 289 static void perfmon_pmu_event_start(struct perf_event *event, int mode)
 290 {
 291         u32 flt_wq, flt_tc, flt_pg_sz, flt_xfer_sz, flt_eng = 0;
 292         u64 cntr_cfg, cntrdata, event_enc, event_cat = 0;
 293         struct hw_perf_event *hwc = &event->hw;
 294         union filter_cfg flt_cfg;
 295         union event_cfg event_cfg;
 296         struct idxd_device *idxd;
 297         int cntr;
 298
 299         idxd = event_to_idxd(event);
 300
 301         event->hw.idx = hwc->idx;
 302         cntr = hwc->idx;
 303
 304         /* Obtain event category and event value from user space */
 305         event_cfg.val = event->attr.config;
 306         flt_cfg.val = event->attr.config1;
 307         event_cat = event_cfg.event_cat;
 308         event_enc = event_cfg.event_enc;
 309
 310         /* Obtain filter configuration from user space */
 311         flt_wq = flt_cfg.wq;
 312         flt_tc = flt_cfg.tc;
 313         flt_pg_sz = flt_cfg.pg_sz;
 314         flt_xfer_sz = flt_cfg.xfer_sz;
 315         flt_eng = flt_cfg.eng;
 316
 317         if (flt_wq && test_bit(FLT_WQ, &idxd->idxd_pmu->supported_filters))
 318                 iowrite32(flt_wq, FLTCFG_REG(idxd, cntr, FLT_WQ));
 319         if (flt_tc && test_bit(FLT_TC, &idxd->idxd_pmu->supported_filters))
 320                 iowrite32(flt_tc, FLTCFG_REG(idxd, cntr, FLT_TC));
 321         if (flt_pg_sz && test_bit(FLT_PG_SZ, &idxd->idxd_pmu->supported_filters))
 322                 iowrite32(flt_pg_sz, FLTCFG_REG(idxd, cntr, FLT_PG_SZ));
 323         if (flt_xfer_sz && test_bit(FLT_XFER_SZ, &idxd->idxd_pmu->supported_filters))
 324                 iowrite32(flt_xfer_sz, FLTCFG_REG(idxd, cntr, FLT_XFER_SZ));
 325         if (flt_eng && test_bit(FLT_ENG, &idxd->idxd_pmu->supported_filters))
 326                 iowrite32(flt_eng, FLTCFG_REG(idxd, cntr, FLT_ENG));
 327
 328         /* Read the start value */
 329         cntrdata = ioread64(CNTRDATA_REG(idxd, cntr));
 330         local64_set(&event->hw.prev_count, cntrdata);
 331
 332         /* Set counter to event/category */
 333         cntr_cfg = event_cat << CNTRCFG_CATEGORY_SHIFT;
 334         cntr_cfg |= event_enc << CNTRCFG_EVENT_SHIFT;
 335         /* Set interrupt on overflow and counter enable bits */
 336         cntr_cfg |= (CNTRCFG_IRQ_OVERFLOW | CNTRCFG_ENABLE);
 337
 338         iowrite64(cntr_cfg, CNTRCFG_REG(idxd, cntr));
 339 }
 340
 341 static void perfmon_pmu_event_stop(struct perf_event *event, int mode)
 342 {
 343         struct hw_perf_event *hwc = &event->hw;
 344         struct idxd_device *idxd;
 345         int i, cntr = hwc->idx;
 346         u64 cntr_cfg;
 347
 348         idxd = event_to_idxd(event);
 349
 350         /* remove this event from event list */
 351         for (i = 0; i < idxd->idxd_pmu->n_events; i++) {
 352                 if (event != idxd->idxd_pmu->event_list[i])
 353                         continue;
 354
 355                 for (++i; i < idxd->idxd_pmu->n_events; i++)
 356                         idxd->idxd_pmu->event_list[i - 1] = idxd->idxd_pmu->event_list[i];
 357                 --idxd->idxd_pmu->n_events;
 358                 break;
 359         }
 360
 361         cntr_cfg = ioread64(CNTRCFG_REG(idxd, cntr));
 362         cntr_cfg &= ~CNTRCFG_ENABLE;
 363         iowrite64(cntr_cfg, CNTRCFG_REG(idxd, cntr));
 364
 365         if (mode == PERF_EF_UPDATE)
 366                 perfmon_pmu_event_update(event);
 367
 368         event->hw.idx = -1;
 369         clear_bit(cntr, idxd->idxd_pmu->used_mask);
 370 }
 371
 372 static void perfmon_pmu_event_del(struct perf_event *event, int mode)
 373 {
 374         perfmon_pmu_event_stop(event, PERF_EF_UPDATE);
 375 }
 376
 377 static int perfmon_pmu_event_add(struct perf_event *event, int flags)
 378 {
 379         struct idxd_device *idxd = event_to_idxd(event);
 380         struct idxd_pmu *idxd_pmu = idxd->idxd_pmu;
 381         struct hw_perf_event *hwc = &event->hw;
 382         int idx, n;
 383
 384         n = perfmon_collect_events(idxd_pmu, event, false);
 385         if (n < 0)
 386                 return n;
 387
 388         hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
 389         if (!(flags & PERF_EF_START))
 390                 hwc->state |= PERF_HES_ARCH;
 391
 392         idx = perfmon_assign_event(idxd_pmu, event);
 393         if (idx < 0)
 394                 return idx;
 395
 396         perfmon_assign_hw_event(idxd_pmu, event, idx);
 397
 398         if (flags & PERF_EF_START)
 399                 perfmon_pmu_event_start(event, 0);
 400
 401         idxd_pmu->n_events = n;
 402
 403         return 0;
 404 }
 405
 406 static void enable_perfmon_pmu(struct idxd_device *idxd)
 407 {
 408         iowrite32(COUNTER_UNFREEZE, PERFFRZ_REG(idxd));
 409 }
 410
 411 static void disable_perfmon_pmu(struct idxd_device *idxd)
 412 {
 413         iowrite32(COUNTER_FREEZE, PERFFRZ_REG(idxd));
 414 }
 415
 416 static void perfmon_pmu_enable(struct pmu *pmu)
 417 {
 418         struct idxd_device *idxd = pmu_to_idxd(pmu);
 419
 420         enable_perfmon_pmu(idxd);
 421 }
 422
 423 static void perfmon_pmu_disable(struct pmu *pmu)
 424 {
 425         struct idxd_device *idxd = pmu_to_idxd(pmu);
 426
 427         disable_perfmon_pmu(idxd);
 428 }
 429
 430 static void skip_filter(int i)
 431 {
 432         int j;
 433
 434         for (j = i; j < PERFMON_FILTERS_MAX; j++)
 435                 perfmon_format_attrs[PERFMON_FILTERS_START + j] =
 436                         perfmon_format_attrs[PERFMON_FILTERS_START + j + 1];
 437 }
 438
 439 static void idxd_pmu_init(struct idxd_pmu *idxd_pmu)
 440 {
 441         int i;
 442
 443         for (i = 0 ; i < PERFMON_FILTERS_MAX; i++) {
 444                 if (!test_bit(i, &idxd_pmu->supported_filters))
 445                         skip_filter(i);
 446         }
 447
 448         idxd_pmu->pmu.name              = idxd_pmu->name;
 449         idxd_pmu->pmu.attr_groups       = perfmon_attr_groups;
 450         idxd_pmu->pmu.task_ctx_nr       = perf_invalid_context;
 451         idxd_pmu->pmu.event_init        = perfmon_pmu_event_init;
 452         idxd_pmu->pmu.pmu_enable        = perfmon_pmu_enable;
 453         idxd_pmu->pmu.pmu_disable       = perfmon_pmu_disable;
 454         idxd_pmu->pmu.add               = perfmon_pmu_event_add;
 455         idxd_pmu->pmu.del               = perfmon_pmu_event_del;
 456         idxd_pmu->pmu.start             = perfmon_pmu_event_start;
 457         idxd_pmu->pmu.stop              = perfmon_pmu_event_stop;
 458         idxd_pmu->pmu.read              = perfmon_pmu_event_update;
 459         idxd_pmu->pmu.capabilities      = PERF_PMU_CAP_NO_EXCLUDE;
 460         idxd_pmu->pmu.scope             = PERF_PMU_SCOPE_SYS_WIDE;
 461         idxd_pmu->pmu.module            = THIS_MODULE;
 462 }
 463
 464 void perfmon_pmu_remove(struct idxd_device *idxd)
 465 {
 466         if (!idxd->idxd_pmu)
 467                 return;
 468
 469         perf_pmu_unregister(&idxd->idxd_pmu->pmu);
 470         kfree(idxd->idxd_pmu);
 471         idxd->idxd_pmu = NULL;
 472 }
 473
 474 int perfmon_pmu_init(struct idxd_device *idxd)
 475 {
 476         union idxd_perfcap perfcap;
 477         struct idxd_pmu *idxd_pmu;
 478         int rc = -ENODEV;
 479
 480         /*
 481          * If perfmon_offset or num_counters is 0, it means perfmon is
 482          * not supported on this hardware.
 483          */
 484         if (idxd->perfmon_offset == 0)
 485                 return -ENODEV;
 486
 487         idxd_pmu = kzalloc(sizeof(*idxd_pmu), GFP_KERNEL);
 488         if (!idxd_pmu)
 489                 return -ENOMEM;
 490
 491         idxd_pmu->idxd = idxd;
 492         idxd->idxd_pmu = idxd_pmu;
 493
 494         if (idxd->data->type == IDXD_TYPE_DSA) {
 495                 rc = sprintf(idxd_pmu->name, "dsa%d", idxd->id);
 496                 if (rc < 0)
 497                         goto free;
 498         } else if (idxd->data->type == IDXD_TYPE_IAX) {
 499                 rc = sprintf(idxd_pmu->name, "iax%d", idxd->id);
 500                 if (rc < 0)
 501                         goto free;
 502         } else {
 503                 goto free;
 504         }
 505
 506         perfmon_reset(idxd);
 507
 508         perfcap.bits = ioread64(PERFCAP_REG(idxd));
 509
 510         /*
 511          * If total perf counter is 0, stop further registration.
 512          * This is necessary in order to support driver running on
 513          * guest which does not have pmon support.
 514          */
 515         if (perfcap.num_perf_counter == 0)
 516                 goto free;
 517
 518         /* A counter width of 0 means it can't count */
 519         if (perfcap.counter_width == 0)
 520                 goto free;
 521
 522         /* Overflow interrupt and counter freeze support must be available */
 523         if (!perfcap.overflow_interrupt || !perfcap.counter_freeze)
 524                 goto free;
 525
 526         /* Number of event categories cannot be 0 */
 527         if (perfcap.num_event_category == 0)
 528                 goto free;
 529
 530         /*
 531          * We don't support per-counter capabilities for now.
 532          */
 533         if (perfcap.cap_per_counter)
 534                 goto free;
 535
 536         idxd_pmu->n_event_categories = perfcap.num_event_category;
 537         idxd_pmu->supported_event_categories = perfcap.global_event_category;
 538         idxd_pmu->per_counter_caps_supported = perfcap.cap_per_counter;
 539
 540         /* check filter capability.  If 0, then filters are not supported */
 541         idxd_pmu->supported_filters = perfcap.filter;
 542         if (perfcap.filter)
 543                 idxd_pmu->n_filters = hweight8(perfcap.filter);
 544
 545         /* Store the total number of counters categories, and counter width */
 546         idxd_pmu->n_counters = perfcap.num_perf_counter;
 547         idxd_pmu->counter_width = perfcap.counter_width;
 548
 549         idxd_pmu_init(idxd_pmu);
 550
 551         rc = perf_pmu_register(&idxd_pmu->pmu, idxd_pmu->name, -1);
 552         if (rc)
 553                 goto free;
 554
 555 out:
 556         return rc;
 557 free:
 558         kfree(idxd_pmu);
 559         idxd->idxd_pmu = NULL;
 560
 561         goto out;
 562 }