drivers/perf/hisilicon/hisi_uncore_pmu.c

   1 /*
   2  * HiSilicon SoC Hardware event counters support
   3  *
   4  * Copyright (C) 2017 Hisilicon Limited
   5  * Author: Anurup M <anurup.m@huawei.com>
   6  *         Shaokun Zhang <zhangshaokun@hisilicon.com>
   7  *
   8  * This code is based on the uncore PMUs like arm-cci and arm-ccn.
   9  *
  10  * This program is free software; you can redistribute it and/or modify
  11  * it under the terms of the GNU General Public License version 2 as
  12  * published by the Free Software Foundation.
  13  */
  14 #include <linux/bitmap.h>
  15 #include <linux/bitops.h>
  16 #include <linux/bug.h>
  17 #include <linux/err.h>
  18 #include <linux/errno.h>
  19 #include <linux/interrupt.h>
  20
  21 #include <asm/local64.h>
  22
  23 #include "hisi_uncore_pmu.h"
  24
  25 #define HISI_GET_EVENTID(ev) (ev->hw.config_base & 0xff)
  26 #define HISI_MAX_PERIOD(nr) (BIT_ULL(nr) - 1)
  27
  28 /*
  29  * PMU format attributes
  30  */
  31 ssize_t hisi_format_sysfs_show(struct device *dev,
  32                                struct device_attribute *attr, char *buf)
  33 {
  34         struct dev_ext_attribute *eattr;
  35
  36         eattr = container_of(attr, struct dev_ext_attribute, attr);
  37
  38         return sprintf(buf, "%s\n", (char *)eattr->var);
  39 }
  40
  41 /*
  42  * PMU event attributes
  43  */
  44 ssize_t hisi_event_sysfs_show(struct device *dev,
  45                               struct device_attribute *attr, char *page)
  46 {
  47         struct dev_ext_attribute *eattr;
  48
  49         eattr = container_of(attr, struct dev_ext_attribute, attr);
  50
  51         return sprintf(page, "config=0x%lx\n", (unsigned long)eattr->var);
  52 }
  53
  54 /*
  55  * sysfs cpumask attributes. For uncore PMU, we only have a single CPU to show
  56  */
  57 ssize_t hisi_cpumask_sysfs_show(struct device *dev,
  58                                 struct device_attribute *attr, char *buf)
  59 {
  60         struct hisi_pmu *hisi_pmu = to_hisi_pmu(dev_get_drvdata(dev));
  61
  62         return sprintf(buf, "%d\n", hisi_pmu->on_cpu);
  63 }
  64
  65 static bool hisi_validate_event_group(struct perf_event *event)
  66 {
  67         struct perf_event *sibling, *leader = event->group_leader;
  68         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
  69         /* Include count for the event */
  70         int counters = 1;
  71
  72         if (!is_software_event(leader)) {
  73                 /*
  74                  * We must NOT create groups containing mixed PMUs, although
  75                  * software events are acceptable
  76                  */
  77                 if (leader->pmu != event->pmu)
  78                         return false;
  79
  80                 /* Increment counter for the leader */
  81                 if (leader != event)
  82                         counters++;
  83         }
  84
  85         for_each_sibling_event(sibling, event->group_leader) {
  86                 if (is_software_event(sibling))
  87                         continue;
  88                 if (sibling->pmu != event->pmu)
  89                         return false;
  90                 /* Increment counter for each sibling */
  91                 counters++;
  92         }
  93
  94         /* The group can not count events more than the counters in the HW */
  95         return counters <= hisi_pmu->num_counters;
  96 }
  97
  98 int hisi_uncore_pmu_counter_valid(struct hisi_pmu *hisi_pmu, int idx)
  99 {
 100         return idx >= 0 && idx < hisi_pmu->num_counters;
 101 }
 102
 103 int hisi_uncore_pmu_get_event_idx(struct perf_event *event)
 104 {
 105         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
 106         unsigned long *used_mask = hisi_pmu->pmu_events.used_mask;
 107         u32 num_counters = hisi_pmu->num_counters;
 108         int idx;
 109
 110         idx = find_first_zero_bit(used_mask, num_counters);
 111         if (idx == num_counters)
 112                 return -EAGAIN;
 113
 114         set_bit(idx, used_mask);
 115
 116         return idx;
 117 }
 118
 119 static void hisi_uncore_pmu_clear_event_idx(struct hisi_pmu *hisi_pmu, int idx)
 120 {
 121         if (!hisi_uncore_pmu_counter_valid(hisi_pmu, idx)) {
 122                 dev_err(hisi_pmu->dev, "Unsupported event index:%d!\n", idx);
 123                 return;
 124         }
 125
 126         clear_bit(idx, hisi_pmu->pmu_events.used_mask);
 127 }
 128
 129 int hisi_uncore_pmu_event_init(struct perf_event *event)
 130 {
 131         struct hw_perf_event *hwc = &event->hw;
 132         struct hisi_pmu *hisi_pmu;
 133
 134         if (event->attr.type != event->pmu->type)
 135                 return -ENOENT;
 136
 137         /*
 138          * We do not support sampling as the counters are all
 139          * shared by all CPU cores in a CPU die(SCCL). Also we
 140          * do not support attach to a task(per-process mode)
 141          */
 142         if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
 143                 return -EOPNOTSUPP;
 144
 145         /* counters do not have these bits */
 146         if (event->attr.exclude_user    ||
 147             event->attr.exclude_kernel  ||
 148             event->attr.exclude_host    ||
 149             event->attr.exclude_guest   ||
 150             event->attr.exclude_hv      ||
 151             event->attr.exclude_idle)
 152                 return -EINVAL;
 153
 154         /*
 155          *  The uncore counters not specific to any CPU, so cannot
 156          *  support per-task
 157          */
 158         if (event->cpu < 0)
 159                 return -EINVAL;
 160
 161         /*
 162          * Validate if the events in group does not exceed the
 163          * available counters in hardware.
 164          */
 165         if (!hisi_validate_event_group(event))
 166                 return -EINVAL;
 167
 168         hisi_pmu = to_hisi_pmu(event->pmu);
 169         if (event->attr.config > hisi_pmu->check_event)
 170                 return -EINVAL;
 171
 172         if (hisi_pmu->on_cpu == -1)
 173                 return -EINVAL;
 174         /*
 175          * We don't assign an index until we actually place the event onto
 176          * hardware. Use -1 to signify that we haven't decided where to put it
 177          * yet.
 178          */
 179         hwc->idx                = -1;
 180         hwc->config_base        = event->attr.config;
 181
 182         /* Enforce to use the same CPU for all events in this PMU */
 183         event->cpu = hisi_pmu->on_cpu;
 184
 185         return 0;
 186 }
 187
 188 /*
 189  * Set the counter to count the event that we're interested in,
 190  * and enable interrupt and counter.
 191  */
 192 static void hisi_uncore_pmu_enable_event(struct perf_event *event)
 193 {
 194         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
 195         struct hw_perf_event *hwc = &event->hw;
 196
 197         hisi_pmu->ops->write_evtype(hisi_pmu, hwc->idx,
 198                                     HISI_GET_EVENTID(event));
 199
 200         hisi_pmu->ops->enable_counter_int(hisi_pmu, hwc);
 201         hisi_pmu->ops->enable_counter(hisi_pmu, hwc);
 202 }
 203
 204 /*
 205  * Disable counter and interrupt.
 206  */
 207 static void hisi_uncore_pmu_disable_event(struct perf_event *event)
 208 {
 209         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
 210         struct hw_perf_event *hwc = &event->hw;
 211
 212         hisi_pmu->ops->disable_counter(hisi_pmu, hwc);
 213         hisi_pmu->ops->disable_counter_int(hisi_pmu, hwc);
 214 }
 215
 216 void hisi_uncore_pmu_set_event_period(struct perf_event *event)
 217 {
 218         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
 219         struct hw_perf_event *hwc = &event->hw;
 220
 221         /*
 222          * The HiSilicon PMU counters support 32 bits or 48 bits, depending on
 223          * the PMU. We reduce it to 2^(counter_bits - 1) to account for the
 224          * extreme interrupt latency. So we could hopefully handle the overflow
 225          * interrupt before another 2^(counter_bits - 1) events occur and the
 226          * counter overtakes its previous value.
 227          */
 228         u64 val = BIT_ULL(hisi_pmu->counter_bits - 1);
 229
 230         local64_set(&hwc->prev_count, val);
 231         /* Write start value to the hardware event counter */
 232         hisi_pmu->ops->write_counter(hisi_pmu, hwc, val);
 233 }
 234
 235 void hisi_uncore_pmu_event_update(struct perf_event *event)
 236 {
 237         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
 238         struct hw_perf_event *hwc = &event->hw;
 239         u64 delta, prev_raw_count, new_raw_count;
 240
 241         do {
 242                 /* Read the count from the counter register */
 243                 new_raw_count = hisi_pmu->ops->read_counter(hisi_pmu, hwc);
 244                 prev_raw_count = local64_read(&hwc->prev_count);
 245         } while (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
 246                                  new_raw_count) != prev_raw_count);
 247         /*
 248          * compute the delta
 249          */
 250         delta = (new_raw_count - prev_raw_count) &
 251                 HISI_MAX_PERIOD(hisi_pmu->counter_bits);
 252         local64_add(delta, &event->count);
 253 }
 254
 255 void hisi_uncore_pmu_start(struct perf_event *event, int flags)
 256 {
 257         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
 258         struct hw_perf_event *hwc = &event->hw;
 259
 260         if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
 261                 return;
 262
 263         WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
 264         hwc->state = 0;
 265         hisi_uncore_pmu_set_event_period(event);
 266
 267         if (flags & PERF_EF_RELOAD) {
 268                 u64 prev_raw_count =  local64_read(&hwc->prev_count);
 269
 270                 hisi_pmu->ops->write_counter(hisi_pmu, hwc, prev_raw_count);
 271         }
 272
 273         hisi_uncore_pmu_enable_event(event);
 274         perf_event_update_userpage(event);
 275 }
 276
 277 void hisi_uncore_pmu_stop(struct perf_event *event, int flags)
 278 {
 279         struct hw_perf_event *hwc = &event->hw;
 280
 281         hisi_uncore_pmu_disable_event(event);
 282         WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
 283         hwc->state |= PERF_HES_STOPPED;
 284
 285         if (hwc->state & PERF_HES_UPTODATE)
 286                 return;
 287
 288         /* Read hardware counter and update the perf counter statistics */
 289         hisi_uncore_pmu_event_update(event);
 290         hwc->state |= PERF_HES_UPTODATE;
 291 }
 292
 293 int hisi_uncore_pmu_add(struct perf_event *event, int flags)
 294 {
 295         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
 296         struct hw_perf_event *hwc = &event->hw;
 297         int idx;
 298
 299         hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
 300
 301         /* Get an available counter index for counting */
 302         idx = hisi_pmu->ops->get_event_idx(event);
 303         if (idx < 0)
 304                 return idx;
 305
 306         event->hw.idx = idx;
 307         hisi_pmu->pmu_events.hw_events[idx] = event;
 308
 309         if (flags & PERF_EF_START)
 310                 hisi_uncore_pmu_start(event, PERF_EF_RELOAD);
 311
 312         return 0;
 313 }
 314
 315 void hisi_uncore_pmu_del(struct perf_event *event, int flags)
 316 {
 317         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
 318         struct hw_perf_event *hwc = &event->hw;
 319
 320         hisi_uncore_pmu_stop(event, PERF_EF_UPDATE);
 321         hisi_uncore_pmu_clear_event_idx(hisi_pmu, hwc->idx);
 322         perf_event_update_userpage(event);
 323         hisi_pmu->pmu_events.hw_events[hwc->idx] = NULL;
 324 }
 325
 326 void hisi_uncore_pmu_read(struct perf_event *event)
 327 {
 328         /* Read hardware counter and update the perf counter statistics */
 329         hisi_uncore_pmu_event_update(event);
 330 }
 331
 332 void hisi_uncore_pmu_enable(struct pmu *pmu)
 333 {
 334         struct hisi_pmu *hisi_pmu = to_hisi_pmu(pmu);
 335         int enabled = bitmap_weight(hisi_pmu->pmu_events.used_mask,
 336                                     hisi_pmu->num_counters);
 337
 338         if (!enabled)
 339                 return;
 340
 341         hisi_pmu->ops->start_counters(hisi_pmu);
 342 }
 343
 344 void hisi_uncore_pmu_disable(struct pmu *pmu)
 345 {
 346         struct hisi_pmu *hisi_pmu = to_hisi_pmu(pmu);
 347
 348         hisi_pmu->ops->stop_counters(hisi_pmu);
 349 }
 350
 351 /*
 352  * Read Super CPU cluster and CPU cluster ID from MPIDR_EL1.
 353  * If multi-threading is supported, CCL_ID is the low 3-bits in MPIDR[Aff2]
 354  * and SCCL_ID is the upper 5-bits of Aff2 field; if not, SCCL_ID
 355  * is in MPIDR[Aff2] and CCL_ID is in MPIDR[Aff1].
 356  */
 357 static void hisi_read_sccl_and_ccl_id(int *sccl_id, int *ccl_id)
 358 {
 359         u64 mpidr = read_cpuid_mpidr();
 360
 361         if (mpidr & MPIDR_MT_BITMASK) {
 362                 int aff2 = MPIDR_AFFINITY_LEVEL(mpidr, 2);
 363
 364                 if (sccl_id)
 365                         *sccl_id = aff2 >> 3;
 366                 if (ccl_id)
 367                         *ccl_id = aff2 & 0x7;
 368         } else {
 369                 if (sccl_id)
 370                         *sccl_id = MPIDR_AFFINITY_LEVEL(mpidr, 2);
 371                 if (ccl_id)
 372                         *ccl_id = MPIDR_AFFINITY_LEVEL(mpidr, 1);
 373         }
 374 }
 375
 376 /*
 377  * Check whether the CPU is associated with this uncore PMU
 378  */
 379 static bool hisi_pmu_cpu_is_associated_pmu(struct hisi_pmu *hisi_pmu)
 380 {
 381         int sccl_id, ccl_id;
 382
 383         if (hisi_pmu->ccl_id == -1) {
 384                 /* If CCL_ID is -1, the PMU only shares the same SCCL */
 385                 hisi_read_sccl_and_ccl_id(&sccl_id, NULL);
 386
 387                 return sccl_id == hisi_pmu->sccl_id;
 388         }
 389
 390         hisi_read_sccl_and_ccl_id(&sccl_id, &ccl_id);
 391
 392         return sccl_id == hisi_pmu->sccl_id && ccl_id == hisi_pmu->ccl_id;
 393 }
 394
 395 int hisi_uncore_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
 396 {
 397         struct hisi_pmu *hisi_pmu = hlist_entry_safe(node, struct hisi_pmu,
 398                                                      node);
 399
 400         if (!hisi_pmu_cpu_is_associated_pmu(hisi_pmu))
 401                 return 0;
 402
 403         cpumask_set_cpu(cpu, &hisi_pmu->associated_cpus);
 404
 405         /* If another CPU is already managing this PMU, simply return. */
 406         if (hisi_pmu->on_cpu != -1)
 407                 return 0;
 408
 409         /* Use this CPU in cpumask for event counting */
 410         hisi_pmu->on_cpu = cpu;
 411
 412         /* Overflow interrupt also should use the same CPU */
 413         WARN_ON(irq_set_affinity(hisi_pmu->irq, cpumask_of(cpu)));
 414
 415         return 0;
 416 }
 417
 418 int hisi_uncore_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
 419 {
 420         struct hisi_pmu *hisi_pmu = hlist_entry_safe(node, struct hisi_pmu,
 421                                                      node);
 422         cpumask_t pmu_online_cpus;
 423         unsigned int target;
 424
 425         if (!cpumask_test_and_clear_cpu(cpu, &hisi_pmu->associated_cpus))
 426                 return 0;
 427
 428         /* Nothing to do if this CPU doesn't own the PMU */
 429         if (hisi_pmu->on_cpu != cpu)
 430                 return 0;
 431
 432         /* Give up ownership of the PMU */
 433         hisi_pmu->on_cpu = -1;
 434
 435         /* Choose a new CPU to migrate ownership of the PMU to */
 436         cpumask_and(&pmu_online_cpus, &hisi_pmu->associated_cpus,
 437                     cpu_online_mask);
 438         target = cpumask_any_but(&pmu_online_cpus, cpu);
 439         if (target >= nr_cpu_ids)
 440                 return 0;
 441
 442         perf_pmu_migrate_context(&hisi_pmu->pmu, cpu, target);
 443         /* Use this CPU for event counting */
 444         hisi_pmu->on_cpu = target;
 445         WARN_ON(irq_set_affinity(hisi_pmu->irq, cpumask_of(target)));
 446
 447         return 0;
 448 }