drivers/perf/hisilicon/hisi_uncore_pmu.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * HiSilicon SoC Hardware event counters support
   4  *
   5  * Copyright (C) 2017 Hisilicon Limited
   6  * Author: Anurup M <anurup.m@huawei.com>
   7  *         Shaokun Zhang <zhangshaokun@hisilicon.com>
   8  *
   9  * This code is based on the uncore PMUs like arm-cci and arm-ccn.
  10  */
  11 #include <linux/bitmap.h>
  12 #include <linux/bitops.h>
  13 #include <linux/bug.h>
  14 #include <linux/err.h>
  15 #include <linux/errno.h>
  16 #include <linux/interrupt.h>
  17
  18 #include <asm/cputype.h>
  19 #include <asm/local64.h>
  20
  21 #include "hisi_uncore_pmu.h"
  22
  23 #define HISI_GET_EVENTID(ev) (ev->hw.config_base & 0xff)
  24 #define HISI_MAX_PERIOD(nr) (BIT_ULL(nr) - 1)
  25
  26 /*
  27  * PMU format attributes
  28  */
  29 ssize_t hisi_format_sysfs_show(struct device *dev,
  30                                struct device_attribute *attr, char *buf)
  31 {
  32         struct dev_ext_attribute *eattr;
  33
  34         eattr = container_of(attr, struct dev_ext_attribute, attr);
  35
  36         return sprintf(buf, "%s\n", (char *)eattr->var);
  37 }
  38 EXPORT_SYMBOL_GPL(hisi_format_sysfs_show);
  39
  40 /*
  41  * PMU event attributes
  42  */
  43 ssize_t hisi_event_sysfs_show(struct device *dev,
  44                               struct device_attribute *attr, char *page)
  45 {
  46         struct dev_ext_attribute *eattr;
  47
  48         eattr = container_of(attr, struct dev_ext_attribute, attr);
  49
  50         return sprintf(page, "config=0x%lx\n", (unsigned long)eattr->var);
  51 }
  52 EXPORT_SYMBOL_GPL(hisi_event_sysfs_show);
  53
  54 /*
  55  * sysfs cpumask attributes. For uncore PMU, we only have a single CPU to show
  56  */
  57 ssize_t hisi_cpumask_sysfs_show(struct device *dev,
  58                                 struct device_attribute *attr, char *buf)
  59 {
  60         struct hisi_pmu *hisi_pmu = to_hisi_pmu(dev_get_drvdata(dev));
  61
  62         return sprintf(buf, "%d\n", hisi_pmu->on_cpu);
  63 }
  64 EXPORT_SYMBOL_GPL(hisi_cpumask_sysfs_show);
  65
  66 static bool hisi_validate_event_group(struct perf_event *event)
  67 {
  68         struct perf_event *sibling, *leader = event->group_leader;
  69         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
  70         /* Include count for the event */
  71         int counters = 1;
  72
  73         if (!is_software_event(leader)) {
  74                 /*
  75                  * We must NOT create groups containing mixed PMUs, although
  76                  * software events are acceptable
  77                  */
  78                 if (leader->pmu != event->pmu)
  79                         return false;
  80
  81                 /* Increment counter for the leader */
  82                 if (leader != event)
  83                         counters++;
  84         }
  85
  86         for_each_sibling_event(sibling, event->group_leader) {
  87                 if (is_software_event(sibling))
  88                         continue;
  89                 if (sibling->pmu != event->pmu)
  90                         return false;
  91                 /* Increment counter for each sibling */
  92                 counters++;
  93         }
  94
  95         /* The group can not count events more than the counters in the HW */
  96         return counters <= hisi_pmu->num_counters;
  97 }
  98
  99 int hisi_uncore_pmu_counter_valid(struct hisi_pmu *hisi_pmu, int idx)
 100 {
 101         return idx >= 0 && idx < hisi_pmu->num_counters;
 102 }
 103 EXPORT_SYMBOL_GPL(hisi_uncore_pmu_counter_valid);
 104
 105 int hisi_uncore_pmu_get_event_idx(struct perf_event *event)
 106 {
 107         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
 108         unsigned long *used_mask = hisi_pmu->pmu_events.used_mask;
 109         u32 num_counters = hisi_pmu->num_counters;
 110         int idx;
 111
 112         idx = find_first_zero_bit(used_mask, num_counters);
 113         if (idx == num_counters)
 114                 return -EAGAIN;
 115
 116         set_bit(idx, used_mask);
 117
 118         return idx;
 119 }
 120 EXPORT_SYMBOL_GPL(hisi_uncore_pmu_get_event_idx);
 121
 122 ssize_t hisi_uncore_pmu_identifier_attr_show(struct device *dev,
 123                                              struct device_attribute *attr,
 124                                              char *page)
 125 {
 126         struct hisi_pmu *hisi_pmu = to_hisi_pmu(dev_get_drvdata(dev));
 127
 128         return snprintf(page, PAGE_SIZE, "0x%08x\n", hisi_pmu->identifier);
 129 }
 130 EXPORT_SYMBOL_GPL(hisi_uncore_pmu_identifier_attr_show);
 131
 132 static void hisi_uncore_pmu_clear_event_idx(struct hisi_pmu *hisi_pmu, int idx)
 133 {
 134         if (!hisi_uncore_pmu_counter_valid(hisi_pmu, idx)) {
 135                 dev_err(hisi_pmu->dev, "Unsupported event index:%d!\n", idx);
 136                 return;
 137         }
 138
 139         clear_bit(idx, hisi_pmu->pmu_events.used_mask);
 140 }
 141
 142 int hisi_uncore_pmu_event_init(struct perf_event *event)
 143 {
 144         struct hw_perf_event *hwc = &event->hw;
 145         struct hisi_pmu *hisi_pmu;
 146
 147         if (event->attr.type != event->pmu->type)
 148                 return -ENOENT;
 149
 150         /*
 151          * We do not support sampling as the counters are all
 152          * shared by all CPU cores in a CPU die(SCCL). Also we
 153          * do not support attach to a task(per-process mode)
 154          */
 155         if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
 156                 return -EOPNOTSUPP;
 157
 158         /*
 159          *  The uncore counters not specific to any CPU, so cannot
 160          *  support per-task
 161          */
 162         if (event->cpu < 0)
 163                 return -EINVAL;
 164
 165         /*
 166          * Validate if the events in group does not exceed the
 167          * available counters in hardware.
 168          */
 169         if (!hisi_validate_event_group(event))
 170                 return -EINVAL;
 171
 172         hisi_pmu = to_hisi_pmu(event->pmu);
 173         if (event->attr.config > hisi_pmu->check_event)
 174                 return -EINVAL;
 175
 176         if (hisi_pmu->on_cpu == -1)
 177                 return -EINVAL;
 178         /*
 179          * We don't assign an index until we actually place the event onto
 180          * hardware. Use -1 to signify that we haven't decided where to put it
 181          * yet.
 182          */
 183         hwc->idx                = -1;
 184         hwc->config_base        = event->attr.config;
 185
 186         /* Enforce to use the same CPU for all events in this PMU */
 187         event->cpu = hisi_pmu->on_cpu;
 188
 189         return 0;
 190 }
 191 EXPORT_SYMBOL_GPL(hisi_uncore_pmu_event_init);
 192
 193 /*
 194  * Set the counter to count the event that we're interested in,
 195  * and enable interrupt and counter.
 196  */
 197 static void hisi_uncore_pmu_enable_event(struct perf_event *event)
 198 {
 199         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
 200         struct hw_perf_event *hwc = &event->hw;
 201
 202         hisi_pmu->ops->write_evtype(hisi_pmu, hwc->idx,
 203                                     HISI_GET_EVENTID(event));
 204
 205         hisi_pmu->ops->enable_counter_int(hisi_pmu, hwc);
 206         hisi_pmu->ops->enable_counter(hisi_pmu, hwc);
 207 }
 208
 209 /*
 210  * Disable counter and interrupt.
 211  */
 212 static void hisi_uncore_pmu_disable_event(struct perf_event *event)
 213 {
 214         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
 215         struct hw_perf_event *hwc = &event->hw;
 216
 217         hisi_pmu->ops->disable_counter(hisi_pmu, hwc);
 218         hisi_pmu->ops->disable_counter_int(hisi_pmu, hwc);
 219 }
 220
 221 void hisi_uncore_pmu_set_event_period(struct perf_event *event)
 222 {
 223         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
 224         struct hw_perf_event *hwc = &event->hw;
 225
 226         /*
 227          * The HiSilicon PMU counters support 32 bits or 48 bits, depending on
 228          * the PMU. We reduce it to 2^(counter_bits - 1) to account for the
 229          * extreme interrupt latency. So we could hopefully handle the overflow
 230          * interrupt before another 2^(counter_bits - 1) events occur and the
 231          * counter overtakes its previous value.
 232          */
 233         u64 val = BIT_ULL(hisi_pmu->counter_bits - 1);
 234
 235         local64_set(&hwc->prev_count, val);
 236         /* Write start value to the hardware event counter */
 237         hisi_pmu->ops->write_counter(hisi_pmu, hwc, val);
 238 }
 239 EXPORT_SYMBOL_GPL(hisi_uncore_pmu_set_event_period);
 240
 241 void hisi_uncore_pmu_event_update(struct perf_event *event)
 242 {
 243         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
 244         struct hw_perf_event *hwc = &event->hw;
 245         u64 delta, prev_raw_count, new_raw_count;
 246
 247         do {
 248                 /* Read the count from the counter register */
 249                 new_raw_count = hisi_pmu->ops->read_counter(hisi_pmu, hwc);
 250                 prev_raw_count = local64_read(&hwc->prev_count);
 251         } while (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
 252                                  new_raw_count) != prev_raw_count);
 253         /*
 254          * compute the delta
 255          */
 256         delta = (new_raw_count - prev_raw_count) &
 257                 HISI_MAX_PERIOD(hisi_pmu->counter_bits);
 258         local64_add(delta, &event->count);
 259 }
 260 EXPORT_SYMBOL_GPL(hisi_uncore_pmu_event_update);
 261
 262 void hisi_uncore_pmu_start(struct perf_event *event, int flags)
 263 {
 264         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
 265         struct hw_perf_event *hwc = &event->hw;
 266
 267         if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
 268                 return;
 269
 270         WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
 271         hwc->state = 0;
 272         hisi_uncore_pmu_set_event_period(event);
 273
 274         if (flags & PERF_EF_RELOAD) {
 275                 u64 prev_raw_count =  local64_read(&hwc->prev_count);
 276
 277                 hisi_pmu->ops->write_counter(hisi_pmu, hwc, prev_raw_count);
 278         }
 279
 280         hisi_uncore_pmu_enable_event(event);
 281         perf_event_update_userpage(event);
 282 }
 283 EXPORT_SYMBOL_GPL(hisi_uncore_pmu_start);
 284
 285 void hisi_uncore_pmu_stop(struct perf_event *event, int flags)
 286 {
 287         struct hw_perf_event *hwc = &event->hw;
 288
 289         hisi_uncore_pmu_disable_event(event);
 290         WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
 291         hwc->state |= PERF_HES_STOPPED;
 292
 293         if (hwc->state & PERF_HES_UPTODATE)
 294                 return;
 295
 296         /* Read hardware counter and update the perf counter statistics */
 297         hisi_uncore_pmu_event_update(event);
 298         hwc->state |= PERF_HES_UPTODATE;
 299 }
 300 EXPORT_SYMBOL_GPL(hisi_uncore_pmu_stop);
 301
 302 int hisi_uncore_pmu_add(struct perf_event *event, int flags)
 303 {
 304         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
 305         struct hw_perf_event *hwc = &event->hw;
 306         int idx;
 307
 308         hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
 309
 310         /* Get an available counter index for counting */
 311         idx = hisi_pmu->ops->get_event_idx(event);
 312         if (idx < 0)
 313                 return idx;
 314
 315         event->hw.idx = idx;
 316         hisi_pmu->pmu_events.hw_events[idx] = event;
 317
 318         if (flags & PERF_EF_START)
 319                 hisi_uncore_pmu_start(event, PERF_EF_RELOAD);
 320
 321         return 0;
 322 }
 323 EXPORT_SYMBOL_GPL(hisi_uncore_pmu_add);
 324
 325 void hisi_uncore_pmu_del(struct perf_event *event, int flags)
 326 {
 327         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
 328         struct hw_perf_event *hwc = &event->hw;
 329
 330         hisi_uncore_pmu_stop(event, PERF_EF_UPDATE);
 331         hisi_uncore_pmu_clear_event_idx(hisi_pmu, hwc->idx);
 332         perf_event_update_userpage(event);
 333         hisi_pmu->pmu_events.hw_events[hwc->idx] = NULL;
 334 }
 335 EXPORT_SYMBOL_GPL(hisi_uncore_pmu_del);
 336
 337 void hisi_uncore_pmu_read(struct perf_event *event)
 338 {
 339         /* Read hardware counter and update the perf counter statistics */
 340         hisi_uncore_pmu_event_update(event);
 341 }
 342 EXPORT_SYMBOL_GPL(hisi_uncore_pmu_read);
 343
 344 void hisi_uncore_pmu_enable(struct pmu *pmu)
 345 {
 346         struct hisi_pmu *hisi_pmu = to_hisi_pmu(pmu);
 347         int enabled = bitmap_weight(hisi_pmu->pmu_events.used_mask,
 348                                     hisi_pmu->num_counters);
 349
 350         if (!enabled)
 351                 return;
 352
 353         hisi_pmu->ops->start_counters(hisi_pmu);
 354 }
 355 EXPORT_SYMBOL_GPL(hisi_uncore_pmu_enable);
 356
 357 void hisi_uncore_pmu_disable(struct pmu *pmu)
 358 {
 359         struct hisi_pmu *hisi_pmu = to_hisi_pmu(pmu);
 360
 361         hisi_pmu->ops->stop_counters(hisi_pmu);
 362 }
 363 EXPORT_SYMBOL_GPL(hisi_uncore_pmu_disable);
 364
 365
 366 /*
 367  * The Super CPU Cluster (SCCL) and CPU Cluster (CCL) IDs can be
 368  * determined from the MPIDR_EL1, but the encoding varies by CPU:
 369  *
 370  * - For MT variants of TSV110:
 371  *   SCCL is Aff2[7:3], CCL is Aff2[2:0]
 372  *
 373  * - For other MT parts:
 374  *   SCCL is Aff3[7:0], CCL is Aff2[7:0]
 375  *
 376  * - For non-MT parts:
 377  *   SCCL is Aff2[7:0], CCL is Aff1[7:0]
 378  */
 379 static void hisi_read_sccl_and_ccl_id(int *scclp, int *cclp)
 380 {
 381         u64 mpidr = read_cpuid_mpidr();
 382         int aff3 = MPIDR_AFFINITY_LEVEL(mpidr, 3);
 383         int aff2 = MPIDR_AFFINITY_LEVEL(mpidr, 2);
 384         int aff1 = MPIDR_AFFINITY_LEVEL(mpidr, 1);
 385         bool mt = mpidr & MPIDR_MT_BITMASK;
 386         int sccl, ccl;
 387
 388         if (mt && read_cpuid_part_number() == HISI_CPU_PART_TSV110) {
 389                 sccl = aff2 >> 3;
 390                 ccl = aff2 & 0x7;
 391         } else if (mt) {
 392                 sccl = aff3;
 393                 ccl = aff2;
 394         } else {
 395                 sccl = aff2;
 396                 ccl = aff1;
 397         }
 398
 399         if (scclp)
 400                 *scclp = sccl;
 401         if (cclp)
 402                 *cclp = ccl;
 403 }
 404
 405 /*
 406  * Check whether the CPU is associated with this uncore PMU
 407  */
 408 static bool hisi_pmu_cpu_is_associated_pmu(struct hisi_pmu *hisi_pmu)
 409 {
 410         int sccl_id, ccl_id;
 411
 412         if (hisi_pmu->ccl_id == -1) {
 413                 /* If CCL_ID is -1, the PMU only shares the same SCCL */
 414                 hisi_read_sccl_and_ccl_id(&sccl_id, NULL);
 415
 416                 return sccl_id == hisi_pmu->sccl_id;
 417         }
 418
 419         hisi_read_sccl_and_ccl_id(&sccl_id, &ccl_id);
 420
 421         return sccl_id == hisi_pmu->sccl_id && ccl_id == hisi_pmu->ccl_id;
 422 }
 423
 424 int hisi_uncore_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
 425 {
 426         struct hisi_pmu *hisi_pmu = hlist_entry_safe(node, struct hisi_pmu,
 427                                                      node);
 428
 429         if (!hisi_pmu_cpu_is_associated_pmu(hisi_pmu))
 430                 return 0;
 431
 432         cpumask_set_cpu(cpu, &hisi_pmu->associated_cpus);
 433
 434         /* If another CPU is already managing this PMU, simply return. */
 435         if (hisi_pmu->on_cpu != -1)
 436                 return 0;
 437
 438         /* Use this CPU in cpumask for event counting */
 439         hisi_pmu->on_cpu = cpu;
 440
 441         /* Overflow interrupt also should use the same CPU */
 442         WARN_ON(irq_set_affinity_hint(hisi_pmu->irq, cpumask_of(cpu)));
 443
 444         return 0;
 445 }
 446 EXPORT_SYMBOL_GPL(hisi_uncore_pmu_online_cpu);
 447
 448 int hisi_uncore_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
 449 {
 450         struct hisi_pmu *hisi_pmu = hlist_entry_safe(node, struct hisi_pmu,
 451                                                      node);
 452         cpumask_t pmu_online_cpus;
 453         unsigned int target;
 454
 455         if (!cpumask_test_and_clear_cpu(cpu, &hisi_pmu->associated_cpus))
 456                 return 0;
 457
 458         /* Nothing to do if this CPU doesn't own the PMU */
 459         if (hisi_pmu->on_cpu != cpu)
 460                 return 0;
 461
 462         /* Give up ownership of the PMU */
 463         hisi_pmu->on_cpu = -1;
 464
 465         /* Choose a new CPU to migrate ownership of the PMU to */
 466         cpumask_and(&pmu_online_cpus, &hisi_pmu->associated_cpus,
 467                     cpu_online_mask);
 468         target = cpumask_any_but(&pmu_online_cpus, cpu);
 469         if (target >= nr_cpu_ids)
 470                 return 0;
 471
 472         perf_pmu_migrate_context(&hisi_pmu->pmu, cpu, target);
 473         /* Use this CPU for event counting */
 474         hisi_pmu->on_cpu = target;
 475         WARN_ON(irq_set_affinity_hint(hisi_pmu->irq, cpumask_of(target)));
 476
 477         return 0;
 478 }
 479 EXPORT_SYMBOL_GPL(hisi_uncore_pmu_offline_cpu);
 480
 481 MODULE_LICENSE("GPL v2");