drivers/iommu/intel/perfmon.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Support Intel IOMMU PerfMon
   4  * Copyright(c) 2023 Intel Corporation.
   5  */
   6 #define pr_fmt(fmt)     "DMAR: " fmt
   7 #define dev_fmt(fmt)    pr_fmt(fmt)
   8
   9 #include <linux/dmar.h>
  10 #include "iommu.h"
  11 #include "perfmon.h"
  12
  13 PMU_FORMAT_ATTR(event,          "config:0-27");         /* ES: Events Select */
  14 PMU_FORMAT_ATTR(event_group,    "config:28-31");        /* EGI: Event Group Index */
  15
  16 static struct attribute *iommu_pmu_format_attrs[] = {
  17         &format_attr_event_group.attr,
  18         &format_attr_event.attr,
  19         NULL
  20 };
  21
  22 static struct attribute_group iommu_pmu_format_attr_group = {
  23         .name = "format",
  24         .attrs = iommu_pmu_format_attrs,
  25 };
  26
  27 /* The available events are added in attr_update later */
  28 static struct attribute *attrs_empty[] = {
  29         NULL
  30 };
  31
  32 static struct attribute_group iommu_pmu_events_attr_group = {
  33         .name = "events",
  34         .attrs = attrs_empty,
  35 };
  36
  37 static const struct attribute_group *iommu_pmu_attr_groups[] = {
  38         &iommu_pmu_format_attr_group,
  39         &iommu_pmu_events_attr_group,
  40         NULL
  41 };
  42
  43 static inline struct iommu_pmu *dev_to_iommu_pmu(struct device *dev)
  44 {
  45         /*
  46          * The perf_event creates its own dev for each PMU.
  47          * See pmu_dev_alloc()
  48          */
  49         return container_of(dev_get_drvdata(dev), struct iommu_pmu, pmu);
  50 }
  51
  52 #define IOMMU_PMU_ATTR(_name, _format, _filter)                         \
  53         PMU_FORMAT_ATTR(_name, _format);                                \
  54                                                                         \
  55 static struct attribute *_name##_attr[] = {                             \
  56         &format_attr_##_name.attr,                                      \
  57         NULL                                                            \
  58 };                                                                      \
  59                                                                         \
  60 static umode_t                                                          \
  61 _name##_is_visible(struct kobject *kobj, struct attribute *attr, int i) \
  62 {                                                                       \
  63         struct device *dev = kobj_to_dev(kobj);                         \
  64         struct iommu_pmu *iommu_pmu = dev_to_iommu_pmu(dev);            \
  65                                                                         \
  66         if (!iommu_pmu)                                                 \
  67                 return 0;                                               \
  68         return (iommu_pmu->filter & _filter) ? attr->mode : 0;          \
  69 }                                                                       \
  70                                                                         \
  71 static struct attribute_group _name = {                                 \
  72         .name           = "format",                                     \
  73         .attrs          = _name##_attr,                                 \
  74         .is_visible     = _name##_is_visible,                           \
  75 };
  76
  77 IOMMU_PMU_ATTR(filter_requester_id_en,  "config1:0",            IOMMU_PMU_FILTER_REQUESTER_ID);
  78 IOMMU_PMU_ATTR(filter_domain_en,        "config1:1",            IOMMU_PMU_FILTER_DOMAIN);
  79 IOMMU_PMU_ATTR(filter_pasid_en,         "config1:2",            IOMMU_PMU_FILTER_PASID);
  80 IOMMU_PMU_ATTR(filter_ats_en,           "config1:3",            IOMMU_PMU_FILTER_ATS);
  81 IOMMU_PMU_ATTR(filter_page_table_en,    "config1:4",            IOMMU_PMU_FILTER_PAGE_TABLE);
  82 IOMMU_PMU_ATTR(filter_requester_id,     "config1:16-31",        IOMMU_PMU_FILTER_REQUESTER_ID);
  83 IOMMU_PMU_ATTR(filter_domain,           "config1:32-47",        IOMMU_PMU_FILTER_DOMAIN);
  84 IOMMU_PMU_ATTR(filter_pasid,            "config2:0-21",         IOMMU_PMU_FILTER_PASID);
  85 IOMMU_PMU_ATTR(filter_ats,              "config2:24-28",        IOMMU_PMU_FILTER_ATS);
  86 IOMMU_PMU_ATTR(filter_page_table,       "config2:32-36",        IOMMU_PMU_FILTER_PAGE_TABLE);
  87
  88 #define iommu_pmu_en_requester_id(e)            ((e) & 0x1)
  89 #define iommu_pmu_en_domain(e)                  (((e) >> 1) & 0x1)
  90 #define iommu_pmu_en_pasid(e)                   (((e) >> 2) & 0x1)
  91 #define iommu_pmu_en_ats(e)                     (((e) >> 3) & 0x1)
  92 #define iommu_pmu_en_page_table(e)              (((e) >> 4) & 0x1)
  93 #define iommu_pmu_get_requester_id(filter)      (((filter) >> 16) & 0xffff)
  94 #define iommu_pmu_get_domain(filter)            (((filter) >> 32) & 0xffff)
  95 #define iommu_pmu_get_pasid(filter)             ((filter) & 0x3fffff)
  96 #define iommu_pmu_get_ats(filter)               (((filter) >> 24) & 0x1f)
  97 #define iommu_pmu_get_page_table(filter)        (((filter) >> 32) & 0x1f)
  98
  99 #define iommu_pmu_set_filter(_name, _config, _filter, _idx, _econfig)           \
 100 {                                                                               \
 101         if ((iommu_pmu->filter & _filter) && iommu_pmu_en_##_name(_econfig)) {  \
 102                 dmar_writel(iommu_pmu->cfg_reg + _idx * IOMMU_PMU_CFG_OFFSET +  \
 103                             IOMMU_PMU_CFG_SIZE +                                \
 104                             (ffs(_filter) - 1) * IOMMU_PMU_CFG_FILTERS_OFFSET,  \
 105                             iommu_pmu_get_##_name(_config) | IOMMU_PMU_FILTER_EN);\
 106         }                                                                       \
 107 }
 108
 109 #define iommu_pmu_clear_filter(_filter, _idx)                                   \
 110 {                                                                               \
 111         if (iommu_pmu->filter & _filter) {                                      \
 112                 dmar_writel(iommu_pmu->cfg_reg + _idx * IOMMU_PMU_CFG_OFFSET +  \
 113                             IOMMU_PMU_CFG_SIZE +                                \
 114                             (ffs(_filter) - 1) * IOMMU_PMU_CFG_FILTERS_OFFSET,  \
 115                             0);                                                 \
 116         }                                                                       \
 117 }
 118
 119 /*
 120  * Define the event attr related functions
 121  * Input: _name: event attr name
 122  *        _string: string of the event in sysfs
 123  *        _g_idx: event group encoding
 124  *        _event: event encoding
 125  */
 126 #define IOMMU_PMU_EVENT_ATTR(_name, _string, _g_idx, _event)                    \
 127         PMU_EVENT_ATTR_STRING(_name, event_attr_##_name, _string)               \
 128                                                                                 \
 129 static struct attribute *_name##_attr[] = {                                     \
 130         &event_attr_##_name.attr.attr,                                          \
 131         NULL                                                                    \
 132 };                                                                              \
 133                                                                                 \
 134 static umode_t                                                                  \
 135 _name##_is_visible(struct kobject *kobj, struct attribute *attr, int i)         \
 136 {                                                                               \
 137         struct device *dev = kobj_to_dev(kobj);                                 \
 138         struct iommu_pmu *iommu_pmu = dev_to_iommu_pmu(dev);                    \
 139                                                                                 \
 140         if (!iommu_pmu)                                                         \
 141                 return 0;                                                       \
 142         return (iommu_pmu->evcap[_g_idx] & _event) ? attr->mode : 0;            \
 143 }                                                                               \
 144                                                                                 \
 145 static struct attribute_group _name = {                                         \
 146         .name           = "events",                                             \
 147         .attrs          = _name##_attr,                                         \
 148         .is_visible     = _name##_is_visible,                                   \
 149 };
 150
 151 IOMMU_PMU_EVENT_ATTR(iommu_clocks,              "event_group=0x0,event=0x001", 0x0, 0x001)
 152 IOMMU_PMU_EVENT_ATTR(iommu_requests,            "event_group=0x0,event=0x002", 0x0, 0x002)
 153 IOMMU_PMU_EVENT_ATTR(pw_occupancy,              "event_group=0x0,event=0x004", 0x0, 0x004)
 154 IOMMU_PMU_EVENT_ATTR(ats_blocked,               "event_group=0x0,event=0x008", 0x0, 0x008)
 155 IOMMU_PMU_EVENT_ATTR(iommu_mrds,                "event_group=0x1,event=0x001", 0x1, 0x001)
 156 IOMMU_PMU_EVENT_ATTR(iommu_mem_blocked,         "event_group=0x1,event=0x020", 0x1, 0x020)
 157 IOMMU_PMU_EVENT_ATTR(pg_req_posted,             "event_group=0x1,event=0x040", 0x1, 0x040)
 158 IOMMU_PMU_EVENT_ATTR(ctxt_cache_lookup,         "event_group=0x2,event=0x001", 0x2, 0x001)
 159 IOMMU_PMU_EVENT_ATTR(ctxt_cache_hit,            "event_group=0x2,event=0x002", 0x2, 0x002)
 160 IOMMU_PMU_EVENT_ATTR(pasid_cache_lookup,        "event_group=0x2,event=0x004", 0x2, 0x004)
 161 IOMMU_PMU_EVENT_ATTR(pasid_cache_hit,           "event_group=0x2,event=0x008", 0x2, 0x008)
 162 IOMMU_PMU_EVENT_ATTR(ss_nonleaf_lookup,         "event_group=0x2,event=0x010", 0x2, 0x010)
 163 IOMMU_PMU_EVENT_ATTR(ss_nonleaf_hit,            "event_group=0x2,event=0x020", 0x2, 0x020)
 164 IOMMU_PMU_EVENT_ATTR(fs_nonleaf_lookup,         "event_group=0x2,event=0x040", 0x2, 0x040)
 165 IOMMU_PMU_EVENT_ATTR(fs_nonleaf_hit,            "event_group=0x2,event=0x080", 0x2, 0x080)
 166 IOMMU_PMU_EVENT_ATTR(hpt_nonleaf_lookup,        "event_group=0x2,event=0x100", 0x2, 0x100)
 167 IOMMU_PMU_EVENT_ATTR(hpt_nonleaf_hit,           "event_group=0x2,event=0x200", 0x2, 0x200)
 168 IOMMU_PMU_EVENT_ATTR(iotlb_lookup,              "event_group=0x3,event=0x001", 0x3, 0x001)
 169 IOMMU_PMU_EVENT_ATTR(iotlb_hit,                 "event_group=0x3,event=0x002", 0x3, 0x002)
 170 IOMMU_PMU_EVENT_ATTR(hpt_leaf_lookup,           "event_group=0x3,event=0x004", 0x3, 0x004)
 171 IOMMU_PMU_EVENT_ATTR(hpt_leaf_hit,              "event_group=0x3,event=0x008", 0x3, 0x008)
 172 IOMMU_PMU_EVENT_ATTR(int_cache_lookup,          "event_group=0x4,event=0x001", 0x4, 0x001)
 173 IOMMU_PMU_EVENT_ATTR(int_cache_hit_nonposted,   "event_group=0x4,event=0x002", 0x4, 0x002)
 174 IOMMU_PMU_EVENT_ATTR(int_cache_hit_posted,      "event_group=0x4,event=0x004", 0x4, 0x004)
 175
 176 static const struct attribute_group *iommu_pmu_attr_update[] = {
 177         &filter_requester_id_en,
 178         &filter_domain_en,
 179         &filter_pasid_en,
 180         &filter_ats_en,
 181         &filter_page_table_en,
 182         &filter_requester_id,
 183         &filter_domain,
 184         &filter_pasid,
 185         &filter_ats,
 186         &filter_page_table,
 187         &iommu_clocks,
 188         &iommu_requests,
 189         &pw_occupancy,
 190         &ats_blocked,
 191         &iommu_mrds,
 192         &iommu_mem_blocked,
 193         &pg_req_posted,
 194         &ctxt_cache_lookup,
 195         &ctxt_cache_hit,
 196         &pasid_cache_lookup,
 197         &pasid_cache_hit,
 198         &ss_nonleaf_lookup,
 199         &ss_nonleaf_hit,
 200         &fs_nonleaf_lookup,
 201         &fs_nonleaf_hit,
 202         &hpt_nonleaf_lookup,
 203         &hpt_nonleaf_hit,
 204         &iotlb_lookup,
 205         &iotlb_hit,
 206         &hpt_leaf_lookup,
 207         &hpt_leaf_hit,
 208         &int_cache_lookup,
 209         &int_cache_hit_nonposted,
 210         &int_cache_hit_posted,
 211         NULL
 212 };
 213
 214 static inline void __iomem *
 215 iommu_event_base(struct iommu_pmu *iommu_pmu, int idx)
 216 {
 217         return iommu_pmu->cntr_reg + idx * iommu_pmu->cntr_stride;
 218 }
 219
 220 static inline void __iomem *
 221 iommu_config_base(struct iommu_pmu *iommu_pmu, int idx)
 222 {
 223         return iommu_pmu->cfg_reg + idx * IOMMU_PMU_CFG_OFFSET;
 224 }
 225
 226 static inline struct iommu_pmu *iommu_event_to_pmu(struct perf_event *event)
 227 {
 228         return container_of(event->pmu, struct iommu_pmu, pmu);
 229 }
 230
 231 static inline u64 iommu_event_config(struct perf_event *event)
 232 {
 233         u64 config = event->attr.config;
 234
 235         return (iommu_event_select(config) << IOMMU_EVENT_CFG_ES_SHIFT) |
 236                (iommu_event_group(config) << IOMMU_EVENT_CFG_EGI_SHIFT) |
 237                IOMMU_EVENT_CFG_INT;
 238 }
 239
 240 static inline bool is_iommu_pmu_event(struct iommu_pmu *iommu_pmu,
 241                                       struct perf_event *event)
 242 {
 243         return event->pmu == &iommu_pmu->pmu;
 244 }
 245
 246 static int iommu_pmu_validate_event(struct perf_event *event)
 247 {
 248         struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event);
 249         u32 event_group = iommu_event_group(event->attr.config);
 250
 251         if (event_group >= iommu_pmu->num_eg)
 252                 return -EINVAL;
 253
 254         return 0;
 255 }
 256
 257 static int iommu_pmu_validate_group(struct perf_event *event)
 258 {
 259         struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event);
 260         struct perf_event *sibling;
 261         int nr = 0;
 262
 263         /*
 264          * All events in a group must be scheduled simultaneously.
 265          * Check whether there is enough counters for all the events.
 266          */
 267         for_each_sibling_event(sibling, event->group_leader) {
 268                 if (!is_iommu_pmu_event(iommu_pmu, sibling) ||
 269                     sibling->state <= PERF_EVENT_STATE_OFF)
 270                         continue;
 271
 272                 if (++nr > iommu_pmu->num_cntr)
 273                         return -EINVAL;
 274         }
 275
 276         return 0;
 277 }
 278
 279 static int iommu_pmu_event_init(struct perf_event *event)
 280 {
 281         struct hw_perf_event *hwc = &event->hw;
 282
 283         if (event->attr.type != event->pmu->type)
 284                 return -ENOENT;
 285
 286         /* sampling not supported */
 287         if (event->attr.sample_period)
 288                 return -EINVAL;
 289
 290         if (event->cpu < 0)
 291                 return -EINVAL;
 292
 293         if (iommu_pmu_validate_event(event))
 294                 return -EINVAL;
 295
 296         hwc->config = iommu_event_config(event);
 297
 298         return iommu_pmu_validate_group(event);
 299 }
 300
 301 static void iommu_pmu_event_update(struct perf_event *event)
 302 {
 303         struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event);
 304         struct hw_perf_event *hwc = &event->hw;
 305         u64 prev_count, new_count, delta;
 306         int shift = 64 - iommu_pmu->cntr_width;
 307
 308 again:
 309         prev_count = local64_read(&hwc->prev_count);
 310         new_count = dmar_readq(iommu_event_base(iommu_pmu, hwc->idx));
 311         if (local64_xchg(&hwc->prev_count, new_count) != prev_count)
 312                 goto again;
 313
 314         /*
 315          * The counter width is enumerated. Always shift the counter
 316          * before using it.
 317          */
 318         delta = (new_count << shift) - (prev_count << shift);
 319         delta >>= shift;
 320
 321         local64_add(delta, &event->count);
 322 }
 323
 324 static void iommu_pmu_start(struct perf_event *event, int flags)
 325 {
 326         struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event);
 327         struct intel_iommu *iommu = iommu_pmu->iommu;
 328         struct hw_perf_event *hwc = &event->hw;
 329         u64 count;
 330
 331         if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
 332                 return;
 333
 334         if (WARN_ON_ONCE(hwc->idx < 0 || hwc->idx >= IOMMU_PMU_IDX_MAX))
 335                 return;
 336
 337         if (flags & PERF_EF_RELOAD)
 338                 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
 339
 340         hwc->state = 0;
 341
 342         /* Always reprogram the period */
 343         count = dmar_readq(iommu_event_base(iommu_pmu, hwc->idx));
 344         local64_set((&hwc->prev_count), count);
 345
 346         /*
 347          * The error of ecmd will be ignored.
 348          * - The existing perf_event subsystem doesn't handle the error.
 349          *   Only IOMMU PMU returns runtime HW error. We don't want to
 350          *   change the existing generic interfaces for the specific case.
 351          * - It's a corner case caused by HW, which is very unlikely to
 352          *   happen. There is nothing SW can do.
 353          * - The worst case is that the user will get <not count> with
 354          *   perf command, which can give the user some hints.
 355          */
 356         ecmd_submit_sync(iommu, DMA_ECMD_ENABLE, hwc->idx, 0);
 357
 358         perf_event_update_userpage(event);
 359 }
 360
 361 static void iommu_pmu_stop(struct perf_event *event, int flags)
 362 {
 363         struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event);
 364         struct intel_iommu *iommu = iommu_pmu->iommu;
 365         struct hw_perf_event *hwc = &event->hw;
 366
 367         if (!(hwc->state & PERF_HES_STOPPED)) {
 368                 ecmd_submit_sync(iommu, DMA_ECMD_DISABLE, hwc->idx, 0);
 369
 370                 iommu_pmu_event_update(event);
 371
 372                 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
 373         }
 374 }
 375
 376 static inline int
 377 iommu_pmu_validate_per_cntr_event(struct iommu_pmu *iommu_pmu,
 378                                   int idx, struct perf_event *event)
 379 {
 380         u32 event_group = iommu_event_group(event->attr.config);
 381         u32 select = iommu_event_select(event->attr.config);
 382
 383         if (!(iommu_pmu->cntr_evcap[idx][event_group] & select))
 384                 return -EINVAL;
 385
 386         return 0;
 387 }
 388
 389 static int iommu_pmu_assign_event(struct iommu_pmu *iommu_pmu,
 390                                   struct perf_event *event)
 391 {
 392         struct hw_perf_event *hwc = &event->hw;
 393         int idx;
 394
 395         /*
 396          * The counters which support limited events are usually at the end.
 397          * Schedule them first to accommodate more events.
 398          */
 399         for (idx = iommu_pmu->num_cntr - 1; idx >= 0; idx--) {
 400                 if (test_and_set_bit(idx, iommu_pmu->used_mask))
 401                         continue;
 402                 /* Check per-counter event capabilities */
 403                 if (!iommu_pmu_validate_per_cntr_event(iommu_pmu, idx, event))
 404                         break;
 405                 clear_bit(idx, iommu_pmu->used_mask);
 406         }
 407         if (idx < 0)
 408                 return -EINVAL;
 409
 410         iommu_pmu->event_list[idx] = event;
 411         hwc->idx = idx;
 412
 413         /* config events */
 414         dmar_writeq(iommu_config_base(iommu_pmu, idx), hwc->config);
 415
 416         iommu_pmu_set_filter(requester_id, event->attr.config1,
 417                              IOMMU_PMU_FILTER_REQUESTER_ID, idx,
 418                              event->attr.config1);
 419         iommu_pmu_set_filter(domain, event->attr.config1,
 420                              IOMMU_PMU_FILTER_DOMAIN, idx,
 421                              event->attr.config1);
 422         iommu_pmu_set_filter(pasid, event->attr.config2,
 423                              IOMMU_PMU_FILTER_PASID, idx,
 424                              event->attr.config1);
 425         iommu_pmu_set_filter(ats, event->attr.config2,
 426                              IOMMU_PMU_FILTER_ATS, idx,
 427                              event->attr.config1);
 428         iommu_pmu_set_filter(page_table, event->attr.config2,
 429                              IOMMU_PMU_FILTER_PAGE_TABLE, idx,
 430                              event->attr.config1);
 431
 432         return 0;
 433 }
 434
 435 static int iommu_pmu_add(struct perf_event *event, int flags)
 436 {
 437         struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event);
 438         struct hw_perf_event *hwc = &event->hw;
 439         int ret;
 440
 441         ret = iommu_pmu_assign_event(iommu_pmu, event);
 442         if (ret < 0)
 443                 return ret;
 444
 445         hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
 446
 447         if (flags & PERF_EF_START)
 448                 iommu_pmu_start(event, 0);
 449
 450         return 0;
 451 }
 452
 453 static void iommu_pmu_del(struct perf_event *event, int flags)
 454 {
 455         struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event);
 456         int idx = event->hw.idx;
 457
 458         iommu_pmu_stop(event, PERF_EF_UPDATE);
 459
 460         iommu_pmu_clear_filter(IOMMU_PMU_FILTER_REQUESTER_ID, idx);
 461         iommu_pmu_clear_filter(IOMMU_PMU_FILTER_DOMAIN, idx);
 462         iommu_pmu_clear_filter(IOMMU_PMU_FILTER_PASID, idx);
 463         iommu_pmu_clear_filter(IOMMU_PMU_FILTER_ATS, idx);
 464         iommu_pmu_clear_filter(IOMMU_PMU_FILTER_PAGE_TABLE, idx);
 465
 466         iommu_pmu->event_list[idx] = NULL;
 467         event->hw.idx = -1;
 468         clear_bit(idx, iommu_pmu->used_mask);
 469
 470         perf_event_update_userpage(event);
 471 }
 472
 473 static void iommu_pmu_enable(struct pmu *pmu)
 474 {
 475         struct iommu_pmu *iommu_pmu = container_of(pmu, struct iommu_pmu, pmu);
 476         struct intel_iommu *iommu = iommu_pmu->iommu;
 477
 478         ecmd_submit_sync(iommu, DMA_ECMD_UNFREEZE, 0, 0);
 479 }
 480
 481 static void iommu_pmu_disable(struct pmu *pmu)
 482 {
 483         struct iommu_pmu *iommu_pmu = container_of(pmu, struct iommu_pmu, pmu);
 484         struct intel_iommu *iommu = iommu_pmu->iommu;
 485
 486         ecmd_submit_sync(iommu, DMA_ECMD_FREEZE, 0, 0);
 487 }
 488
 489 static void iommu_pmu_counter_overflow(struct iommu_pmu *iommu_pmu)
 490 {
 491         struct perf_event *event;
 492         u64 status;
 493         int i;
 494
 495         /*
 496          * Two counters may be overflowed very close. Always check
 497          * whether there are more to handle.
 498          */
 499         while ((status = dmar_readq(iommu_pmu->overflow))) {
 500                 for_each_set_bit(i, (unsigned long *)&status, iommu_pmu->num_cntr) {
 501                         /*
 502                          * Find the assigned event of the counter.
 503                          * Accumulate the value into the event->count.
 504                          */
 505                         event = iommu_pmu->event_list[i];
 506                         if (!event) {
 507                                 pr_warn_once("Cannot find the assigned event for counter %d\n", i);
 508                                 continue;
 509                         }
 510                         iommu_pmu_event_update(event);
 511                 }
 512
 513                 dmar_writeq(iommu_pmu->overflow, status);
 514         }
 515 }
 516
 517 static irqreturn_t iommu_pmu_irq_handler(int irq, void *dev_id)
 518 {
 519         struct intel_iommu *iommu = dev_id;
 520
 521         if (!dmar_readl(iommu->reg + DMAR_PERFINTRSTS_REG))
 522                 return IRQ_NONE;
 523
 524         iommu_pmu_counter_overflow(iommu->pmu);
 525
 526         /* Clear the status bit */
 527         dmar_writel(iommu->reg + DMAR_PERFINTRSTS_REG, DMA_PERFINTRSTS_PIS);
 528
 529         return IRQ_HANDLED;
 530 }
 531
 532 static int __iommu_pmu_register(struct intel_iommu *iommu)
 533 {
 534         struct iommu_pmu *iommu_pmu = iommu->pmu;
 535
 536         iommu_pmu->pmu.name             = iommu->name;
 537         iommu_pmu->pmu.task_ctx_nr      = perf_invalid_context;
 538         iommu_pmu->pmu.event_init       = iommu_pmu_event_init;
 539         iommu_pmu->pmu.pmu_enable       = iommu_pmu_enable;
 540         iommu_pmu->pmu.pmu_disable      = iommu_pmu_disable;
 541         iommu_pmu->pmu.add              = iommu_pmu_add;
 542         iommu_pmu->pmu.del              = iommu_pmu_del;
 543         iommu_pmu->pmu.start            = iommu_pmu_start;
 544         iommu_pmu->pmu.stop             = iommu_pmu_stop;
 545         iommu_pmu->pmu.read             = iommu_pmu_event_update;
 546         iommu_pmu->pmu.attr_groups      = iommu_pmu_attr_groups;
 547         iommu_pmu->pmu.attr_update      = iommu_pmu_attr_update;
 548         iommu_pmu->pmu.capabilities     = PERF_PMU_CAP_NO_EXCLUDE;
 549         iommu_pmu->pmu.scope            = PERF_PMU_SCOPE_SYS_WIDE;
 550         iommu_pmu->pmu.module           = THIS_MODULE;
 551
 552         return perf_pmu_register(&iommu_pmu->pmu, iommu_pmu->pmu.name, -1);
 553 }
 554
 555 static inline void __iomem *
 556 get_perf_reg_address(struct intel_iommu *iommu, u32 offset)
 557 {
 558         u32 off = dmar_readl(iommu->reg + offset);
 559
 560         return iommu->reg + off;
 561 }
 562
 563 int alloc_iommu_pmu(struct intel_iommu *iommu)
 564 {
 565         struct iommu_pmu *iommu_pmu;
 566         int i, j, ret;
 567         u64 perfcap;
 568         u32 cap;
 569
 570         if (!ecap_pms(iommu->ecap))
 571                 return 0;
 572
 573         /* The IOMMU PMU requires the ECMD support as well */
 574         if (!cap_ecmds(iommu->cap))
 575                 return -ENODEV;
 576
 577         perfcap = dmar_readq(iommu->reg + DMAR_PERFCAP_REG);
 578         /* The performance monitoring is not supported. */
 579         if (!perfcap)
 580                 return -ENODEV;
 581
 582         /* Sanity check for the number of the counters and event groups */
 583         if (!pcap_num_cntr(perfcap) || !pcap_num_event_group(perfcap))
 584                 return -ENODEV;
 585
 586         /* The interrupt on overflow is required */
 587         if (!pcap_interrupt(perfcap))
 588                 return -ENODEV;
 589
 590         /* Check required Enhanced Command Capability */
 591         if (!ecmd_has_pmu_essential(iommu))
 592                 return -ENODEV;
 593
 594         iommu_pmu = kzalloc(sizeof(*iommu_pmu), GFP_KERNEL);
 595         if (!iommu_pmu)
 596                 return -ENOMEM;
 597
 598         iommu_pmu->num_cntr = pcap_num_cntr(perfcap);
 599         if (iommu_pmu->num_cntr > IOMMU_PMU_IDX_MAX) {
 600                 pr_warn_once("The number of IOMMU counters %d > max(%d), clipping!",
 601                              iommu_pmu->num_cntr, IOMMU_PMU_IDX_MAX);
 602                 iommu_pmu->num_cntr = IOMMU_PMU_IDX_MAX;
 603         }
 604
 605         iommu_pmu->cntr_width = pcap_cntr_width(perfcap);
 606         iommu_pmu->filter = pcap_filters_mask(perfcap);
 607         iommu_pmu->cntr_stride = pcap_cntr_stride(perfcap);
 608         iommu_pmu->num_eg = pcap_num_event_group(perfcap);
 609
 610         iommu_pmu->evcap = kcalloc(iommu_pmu->num_eg, sizeof(u64), GFP_KERNEL);
 611         if (!iommu_pmu->evcap) {
 612                 ret = -ENOMEM;
 613                 goto free_pmu;
 614         }
 615
 616         /* Parse event group capabilities */
 617         for (i = 0; i < iommu_pmu->num_eg; i++) {
 618                 u64 pcap;
 619
 620                 pcap = dmar_readq(iommu->reg + DMAR_PERFEVNTCAP_REG +
 621                                   i * IOMMU_PMU_CAP_REGS_STEP);
 622                 iommu_pmu->evcap[i] = pecap_es(pcap);
 623         }
 624
 625         iommu_pmu->cntr_evcap = kcalloc(iommu_pmu->num_cntr, sizeof(u32 *), GFP_KERNEL);
 626         if (!iommu_pmu->cntr_evcap) {
 627                 ret = -ENOMEM;
 628                 goto free_pmu_evcap;
 629         }
 630         for (i = 0; i < iommu_pmu->num_cntr; i++) {
 631                 iommu_pmu->cntr_evcap[i] = kcalloc(iommu_pmu->num_eg, sizeof(u32), GFP_KERNEL);
 632                 if (!iommu_pmu->cntr_evcap[i]) {
 633                         ret = -ENOMEM;
 634                         goto free_pmu_cntr_evcap;
 635                 }
 636                 /*
 637                  * Set to the global capabilities, will adjust according
 638                  * to per-counter capabilities later.
 639                  */
 640                 for (j = 0; j < iommu_pmu->num_eg; j++)
 641                         iommu_pmu->cntr_evcap[i][j] = (u32)iommu_pmu->evcap[j];
 642         }
 643
 644         iommu_pmu->cfg_reg = get_perf_reg_address(iommu, DMAR_PERFCFGOFF_REG);
 645         iommu_pmu->cntr_reg = get_perf_reg_address(iommu, DMAR_PERFCNTROFF_REG);
 646         iommu_pmu->overflow = get_perf_reg_address(iommu, DMAR_PERFOVFOFF_REG);
 647
 648         /*
 649          * Check per-counter capabilities. All counters should have the
 650          * same capabilities on Interrupt on Overflow Support and Counter
 651          * Width.
 652          */
 653         for (i = 0; i < iommu_pmu->num_cntr; i++) {
 654                 cap = dmar_readl(iommu_pmu->cfg_reg +
 655                                  i * IOMMU_PMU_CFG_OFFSET +
 656                                  IOMMU_PMU_CFG_CNTRCAP_OFFSET);
 657                 if (!iommu_cntrcap_pcc(cap))
 658                         continue;
 659
 660                 /*
 661                  * It's possible that some counters have a different
 662                  * capability because of e.g., HW bug. Check the corner
 663                  * case here and simply drop those counters.
 664                  */
 665                 if ((iommu_cntrcap_cw(cap) != iommu_pmu->cntr_width) ||
 666                     !iommu_cntrcap_ios(cap)) {
 667                         iommu_pmu->num_cntr = i;
 668                         pr_warn("PMU counter capability inconsistent, counter number reduced to %d\n",
 669                                 iommu_pmu->num_cntr);
 670                 }
 671
 672                 /* Clear the pre-defined events group */
 673                 for (j = 0; j < iommu_pmu->num_eg; j++)
 674                         iommu_pmu->cntr_evcap[i][j] = 0;
 675
 676                 /* Override with per-counter event capabilities */
 677                 for (j = 0; j < iommu_cntrcap_egcnt(cap); j++) {
 678                         cap = dmar_readl(iommu_pmu->cfg_reg + i * IOMMU_PMU_CFG_OFFSET +
 679                                          IOMMU_PMU_CFG_CNTREVCAP_OFFSET +
 680                                          (j * IOMMU_PMU_OFF_REGS_STEP));
 681                         iommu_pmu->cntr_evcap[i][iommu_event_group(cap)] = iommu_event_select(cap);
 682                         /*
 683                          * Some events may only be supported by a specific counter.
 684                          * Track them in the evcap as well.
 685                          */
 686                         iommu_pmu->evcap[iommu_event_group(cap)] |= iommu_event_select(cap);
 687                 }
 688         }
 689
 690         iommu_pmu->iommu = iommu;
 691         iommu->pmu = iommu_pmu;
 692
 693         return 0;
 694
 695 free_pmu_cntr_evcap:
 696         for (i = 0; i < iommu_pmu->num_cntr; i++)
 697                 kfree(iommu_pmu->cntr_evcap[i]);
 698         kfree(iommu_pmu->cntr_evcap);
 699 free_pmu_evcap:
 700         kfree(iommu_pmu->evcap);
 701 free_pmu:
 702         kfree(iommu_pmu);
 703
 704         return ret;
 705 }
 706
 707 void free_iommu_pmu(struct intel_iommu *iommu)
 708 {
 709         struct iommu_pmu *iommu_pmu = iommu->pmu;
 710
 711         if (!iommu_pmu)
 712                 return;
 713
 714         if (iommu_pmu->evcap) {
 715                 int i;
 716
 717                 for (i = 0; i < iommu_pmu->num_cntr; i++)
 718                         kfree(iommu_pmu->cntr_evcap[i]);
 719                 kfree(iommu_pmu->cntr_evcap);
 720         }
 721         kfree(iommu_pmu->evcap);
 722         kfree(iommu_pmu);
 723         iommu->pmu = NULL;
 724 }
 725
 726 static int iommu_pmu_set_interrupt(struct intel_iommu *iommu)
 727 {
 728         struct iommu_pmu *iommu_pmu = iommu->pmu;
 729         int irq, ret;
 730
 731         irq = dmar_alloc_hwirq(IOMMU_IRQ_ID_OFFSET_PERF + iommu->seq_id, iommu->node, iommu);
 732         if (irq <= 0)
 733                 return -EINVAL;
 734
 735         snprintf(iommu_pmu->irq_name, sizeof(iommu_pmu->irq_name), "dmar%d-perf", iommu->seq_id);
 736
 737         iommu->perf_irq = irq;
 738         ret = request_threaded_irq(irq, NULL, iommu_pmu_irq_handler,
 739                                    IRQF_ONESHOT, iommu_pmu->irq_name, iommu);
 740         if (ret) {
 741                 dmar_free_hwirq(irq);
 742                 iommu->perf_irq = 0;
 743                 return ret;
 744         }
 745         return 0;
 746 }
 747
 748 static void iommu_pmu_unset_interrupt(struct intel_iommu *iommu)
 749 {
 750         if (!iommu->perf_irq)
 751                 return;
 752
 753         free_irq(iommu->perf_irq, iommu);
 754         dmar_free_hwirq(iommu->perf_irq);
 755         iommu->perf_irq = 0;
 756 }
 757
 758 void iommu_pmu_register(struct intel_iommu *iommu)
 759 {
 760         struct iommu_pmu *iommu_pmu = iommu->pmu;
 761
 762         if (!iommu_pmu)
 763                 return;
 764
 765         if (__iommu_pmu_register(iommu))
 766                 goto err;
 767
 768         /* Set interrupt for overflow */
 769         if (iommu_pmu_set_interrupt(iommu))
 770                 goto unregister;
 771
 772         return;
 773
 774 unregister:
 775         perf_pmu_unregister(&iommu_pmu->pmu);
 776 err:
 777         pr_err("Failed to register PMU for iommu (seq_id = %d)\n", iommu->seq_id);
 778         free_iommu_pmu(iommu);
 779 }
 780
 781 void iommu_pmu_unregister(struct intel_iommu *iommu)
 782 {
 783         struct iommu_pmu *iommu_pmu = iommu->pmu;
 784
 785         if (!iommu_pmu)
 786                 return;
 787
 788         iommu_pmu_unset_interrupt(iommu);
 789         perf_pmu_unregister(&iommu_pmu->pmu);
 790 }