drivers/perf/hisilicon/hisi_uncore_pmu.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * HiSilicon SoC Hardware event counters support
   4  *
   5  * Copyright (C) 2017 Hisilicon Limited
   6  * Author: Anurup M <anurup.m@huawei.com>
   7  *         Shaokun Zhang <zhangshaokun@hisilicon.com>
   8  *
   9  * This code is based on the uncore PMUs like arm-cci and arm-ccn.
  10  */
  11 #include <linux/bitmap.h>
  12 #include <linux/bitops.h>
  13 #include <linux/bug.h>
  14 #include <linux/err.h>
  15 #include <linux/errno.h>
  16 #include <linux/interrupt.h>
  17
  18 #include <asm/cputype.h>
  19 #include <asm/local64.h>
  20
  21 #include "hisi_uncore_pmu.h"
  22
  23 #define HISI_GET_EVENTID(ev) (ev->hw.config_base & 0xff)
  24 #define HISI_MAX_PERIOD(nr) (BIT_ULL(nr) - 1)
  25
  26 /*
  27  * PMU format attributes
  28  */
  29 ssize_t hisi_format_sysfs_show(struct device *dev,
  30                                struct device_attribute *attr, char *buf)
  31 {
  32         struct dev_ext_attribute *eattr;
  33
  34         eattr = container_of(attr, struct dev_ext_attribute, attr);
  35
  36         return sprintf(buf, "%s\n", (char *)eattr->var);
  37 }
  38
  39 /*
  40  * PMU event attributes
  41  */
  42 ssize_t hisi_event_sysfs_show(struct device *dev,
  43                               struct device_attribute *attr, char *page)
  44 {
  45         struct dev_ext_attribute *eattr;
  46
  47         eattr = container_of(attr, struct dev_ext_attribute, attr);
  48
  49         return sprintf(page, "config=0x%lx\n", (unsigned long)eattr->var);
  50 }
  51
  52 /*
  53  * sysfs cpumask attributes. For uncore PMU, we only have a single CPU to show
  54  */
  55 ssize_t hisi_cpumask_sysfs_show(struct device *dev,
  56                                 struct device_attribute *attr, char *buf)
  57 {
  58         struct hisi_pmu *hisi_pmu = to_hisi_pmu(dev_get_drvdata(dev));
  59
  60         return sprintf(buf, "%d\n", hisi_pmu->on_cpu);
  61 }
  62
  63 static bool hisi_validate_event_group(struct perf_event *event)
  64 {
  65         struct perf_event *sibling, *leader = event->group_leader;
  66         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
  67         /* Include count for the event */
  68         int counters = 1;
  69
  70         if (!is_software_event(leader)) {
  71                 /*
  72                  * We must NOT create groups containing mixed PMUs, although
  73                  * software events are acceptable
  74                  */
  75                 if (leader->pmu != event->pmu)
  76                         return false;
  77
  78                 /* Increment counter for the leader */
  79                 if (leader != event)
  80                         counters++;
  81         }
  82
  83         for_each_sibling_event(sibling, event->group_leader) {
  84                 if (is_software_event(sibling))
  85                         continue;
  86                 if (sibling->pmu != event->pmu)
  87                         return false;
  88                 /* Increment counter for each sibling */
  89                 counters++;
  90         }
  91
  92         /* The group can not count events more than the counters in the HW */
  93         return counters <= hisi_pmu->num_counters;
  94 }
  95
  96 int hisi_uncore_pmu_counter_valid(struct hisi_pmu *hisi_pmu, int idx)
  97 {
  98         return idx >= 0 && idx < hisi_pmu->num_counters;
  99 }
 100
 101 int hisi_uncore_pmu_get_event_idx(struct perf_event *event)
 102 {
 103         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
 104         unsigned long *used_mask = hisi_pmu->pmu_events.used_mask;
 105         u32 num_counters = hisi_pmu->num_counters;
 106         int idx;
 107
 108         idx = find_first_zero_bit(used_mask, num_counters);
 109         if (idx == num_counters)
 110                 return -EAGAIN;
 111
 112         set_bit(idx, used_mask);
 113
 114         return idx;
 115 }
 116
 117 static void hisi_uncore_pmu_clear_event_idx(struct hisi_pmu *hisi_pmu, int idx)
 118 {
 119         if (!hisi_uncore_pmu_counter_valid(hisi_pmu, idx)) {
 120                 dev_err(hisi_pmu->dev, "Unsupported event index:%d!\n", idx);
 121                 return;
 122         }
 123
 124         clear_bit(idx, hisi_pmu->pmu_events.used_mask);
 125 }
 126
 127 int hisi_uncore_pmu_event_init(struct perf_event *event)
 128 {
 129         struct hw_perf_event *hwc = &event->hw;
 130         struct hisi_pmu *hisi_pmu;
 131
 132         if (event->attr.type != event->pmu->type)
 133                 return -ENOENT;
 134
 135         /*
 136          * We do not support sampling as the counters are all
 137          * shared by all CPU cores in a CPU die(SCCL). Also we
 138          * do not support attach to a task(per-process mode)
 139          */
 140         if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
 141                 return -EOPNOTSUPP;
 142
 143         /*
 144          *  The uncore counters not specific to any CPU, so cannot
 145          *  support per-task
 146          */
 147         if (event->cpu < 0)
 148                 return -EINVAL;
 149
 150         /*
 151          * Validate if the events in group does not exceed the
 152          * available counters in hardware.
 153          */
 154         if (!hisi_validate_event_group(event))
 155                 return -EINVAL;
 156
 157         hisi_pmu = to_hisi_pmu(event->pmu);
 158         if (event->attr.config > hisi_pmu->check_event)
 159                 return -EINVAL;
 160
 161         if (hisi_pmu->on_cpu == -1)
 162                 return -EINVAL;
 163         /*
 164          * We don't assign an index until we actually place the event onto
 165          * hardware. Use -1 to signify that we haven't decided where to put it
 166          * yet.
 167          */
 168         hwc->idx                = -1;
 169         hwc->config_base        = event->attr.config;
 170
 171         /* Enforce to use the same CPU for all events in this PMU */
 172         event->cpu = hisi_pmu->on_cpu;
 173
 174         return 0;
 175 }
 176
 177 /*
 178  * Set the counter to count the event that we're interested in,
 179  * and enable interrupt and counter.
 180  */
 181 static void hisi_uncore_pmu_enable_event(struct perf_event *event)
 182 {
 183         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
 184         struct hw_perf_event *hwc = &event->hw;
 185
 186         hisi_pmu->ops->write_evtype(hisi_pmu, hwc->idx,
 187                                     HISI_GET_EVENTID(event));
 188
 189         hisi_pmu->ops->enable_counter_int(hisi_pmu, hwc);
 190         hisi_pmu->ops->enable_counter(hisi_pmu, hwc);
 191 }
 192
 193 /*
 194  * Disable counter and interrupt.
 195  */
 196 static void hisi_uncore_pmu_disable_event(struct perf_event *event)
 197 {
 198         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
 199         struct hw_perf_event *hwc = &event->hw;
 200
 201         hisi_pmu->ops->disable_counter(hisi_pmu, hwc);
 202         hisi_pmu->ops->disable_counter_int(hisi_pmu, hwc);
 203 }
 204
 205 void hisi_uncore_pmu_set_event_period(struct perf_event *event)
 206 {
 207         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
 208         struct hw_perf_event *hwc = &event->hw;
 209
 210         /*
 211          * The HiSilicon PMU counters support 32 bits or 48 bits, depending on
 212          * the PMU. We reduce it to 2^(counter_bits - 1) to account for the
 213          * extreme interrupt latency. So we could hopefully handle the overflow
 214          * interrupt before another 2^(counter_bits - 1) events occur and the
 215          * counter overtakes its previous value.
 216          */
 217         u64 val = BIT_ULL(hisi_pmu->counter_bits - 1);
 218
 219         local64_set(&hwc->prev_count, val);
 220         /* Write start value to the hardware event counter */
 221         hisi_pmu->ops->write_counter(hisi_pmu, hwc, val);
 222 }
 223
 224 void hisi_uncore_pmu_event_update(struct perf_event *event)
 225 {
 226         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
 227         struct hw_perf_event *hwc = &event->hw;
 228         u64 delta, prev_raw_count, new_raw_count;
 229
 230         do {
 231                 /* Read the count from the counter register */
 232                 new_raw_count = hisi_pmu->ops->read_counter(hisi_pmu, hwc);
 233                 prev_raw_count = local64_read(&hwc->prev_count);
 234         } while (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
 235                                  new_raw_count) != prev_raw_count);
 236         /*
 237          * compute the delta
 238          */
 239         delta = (new_raw_count - prev_raw_count) &
 240                 HISI_MAX_PERIOD(hisi_pmu->counter_bits);
 241         local64_add(delta, &event->count);
 242 }
 243
 244 void hisi_uncore_pmu_start(struct perf_event *event, int flags)
 245 {
 246         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
 247         struct hw_perf_event *hwc = &event->hw;
 248
 249         if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
 250                 return;
 251
 252         WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
 253         hwc->state = 0;
 254         hisi_uncore_pmu_set_event_period(event);
 255
 256         if (flags & PERF_EF_RELOAD) {
 257                 u64 prev_raw_count =  local64_read(&hwc->prev_count);
 258
 259                 hisi_pmu->ops->write_counter(hisi_pmu, hwc, prev_raw_count);
 260         }
 261
 262         hisi_uncore_pmu_enable_event(event);
 263         perf_event_update_userpage(event);
 264 }
 265
 266 void hisi_uncore_pmu_stop(struct perf_event *event, int flags)
 267 {
 268         struct hw_perf_event *hwc = &event->hw;
 269
 270         hisi_uncore_pmu_disable_event(event);
 271         WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
 272         hwc->state |= PERF_HES_STOPPED;
 273
 274         if (hwc->state & PERF_HES_UPTODATE)
 275                 return;
 276
 277         /* Read hardware counter and update the perf counter statistics */
 278         hisi_uncore_pmu_event_update(event);
 279         hwc->state |= PERF_HES_UPTODATE;
 280 }
 281
 282 int hisi_uncore_pmu_add(struct perf_event *event, int flags)
 283 {
 284         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
 285         struct hw_perf_event *hwc = &event->hw;
 286         int idx;
 287
 288         hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
 289
 290         /* Get an available counter index for counting */
 291         idx = hisi_pmu->ops->get_event_idx(event);
 292         if (idx < 0)
 293                 return idx;
 294
 295         event->hw.idx = idx;
 296         hisi_pmu->pmu_events.hw_events[idx] = event;
 297
 298         if (flags & PERF_EF_START)
 299                 hisi_uncore_pmu_start(event, PERF_EF_RELOAD);
 300
 301         return 0;
 302 }
 303
 304 void hisi_uncore_pmu_del(struct perf_event *event, int flags)
 305 {
 306         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
 307         struct hw_perf_event *hwc = &event->hw;
 308
 309         hisi_uncore_pmu_stop(event, PERF_EF_UPDATE);
 310         hisi_uncore_pmu_clear_event_idx(hisi_pmu, hwc->idx);
 311         perf_event_update_userpage(event);
 312         hisi_pmu->pmu_events.hw_events[hwc->idx] = NULL;
 313 }
 314
 315 void hisi_uncore_pmu_read(struct perf_event *event)
 316 {
 317         /* Read hardware counter and update the perf counter statistics */
 318         hisi_uncore_pmu_event_update(event);
 319 }
 320
 321 void hisi_uncore_pmu_enable(struct pmu *pmu)
 322 {
 323         struct hisi_pmu *hisi_pmu = to_hisi_pmu(pmu);
 324         int enabled = bitmap_weight(hisi_pmu->pmu_events.used_mask,
 325                                     hisi_pmu->num_counters);
 326
 327         if (!enabled)
 328                 return;
 329
 330         hisi_pmu->ops->start_counters(hisi_pmu);
 331 }
 332
 333 void hisi_uncore_pmu_disable(struct pmu *pmu)
 334 {
 335         struct hisi_pmu *hisi_pmu = to_hisi_pmu(pmu);
 336
 337         hisi_pmu->ops->stop_counters(hisi_pmu);
 338 }
 339
 340
 341 /*
 342  * The Super CPU Cluster (SCCL) and CPU Cluster (CCL) IDs can be
 343  * determined from the MPIDR_EL1, but the encoding varies by CPU:
 344  *
 345  * - For MT variants of TSV110:
 346  *   SCCL is Aff2[7:3], CCL is Aff2[2:0]
 347  *
 348  * - For other MT parts:
 349  *   SCCL is Aff3[7:0], CCL is Aff2[7:0]
 350  *
 351  * - For non-MT parts:
 352  *   SCCL is Aff2[7:0], CCL is Aff1[7:0]
 353  */
 354 static void hisi_read_sccl_and_ccl_id(int *scclp, int *cclp)
 355 {
 356         u64 mpidr = read_cpuid_mpidr();
 357         int aff3 = MPIDR_AFFINITY_LEVEL(mpidr, 3);
 358         int aff2 = MPIDR_AFFINITY_LEVEL(mpidr, 2);
 359         int aff1 = MPIDR_AFFINITY_LEVEL(mpidr, 1);
 360         bool mt = mpidr & MPIDR_MT_BITMASK;
 361         int sccl, ccl;
 362
 363         if (mt && read_cpuid_part_number() == HISI_CPU_PART_TSV110) {
 364                 sccl = aff2 >> 3;
 365                 ccl = aff2 & 0x7;
 366         } else if (mt) {
 367                 sccl = aff3;
 368                 ccl = aff2;
 369         } else {
 370                 sccl = aff2;
 371                 ccl = aff1;
 372         }
 373
 374         if (scclp)
 375                 *scclp = sccl;
 376         if (cclp)
 377                 *cclp = ccl;
 378 }
 379
 380 /*
 381  * Check whether the CPU is associated with this uncore PMU
 382  */
 383 static bool hisi_pmu_cpu_is_associated_pmu(struct hisi_pmu *hisi_pmu)
 384 {
 385         int sccl_id, ccl_id;
 386
 387         if (hisi_pmu->ccl_id == -1) {
 388                 /* If CCL_ID is -1, the PMU only shares the same SCCL */
 389                 hisi_read_sccl_and_ccl_id(&sccl_id, NULL);
 390
 391                 return sccl_id == hisi_pmu->sccl_id;
 392         }
 393
 394         hisi_read_sccl_and_ccl_id(&sccl_id, &ccl_id);
 395
 396         return sccl_id == hisi_pmu->sccl_id && ccl_id == hisi_pmu->ccl_id;
 397 }
 398
 399 int hisi_uncore_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
 400 {
 401         struct hisi_pmu *hisi_pmu = hlist_entry_safe(node, struct hisi_pmu,
 402                                                      node);
 403
 404         if (!hisi_pmu_cpu_is_associated_pmu(hisi_pmu))
 405                 return 0;
 406
 407         cpumask_set_cpu(cpu, &hisi_pmu->associated_cpus);
 408
 409         /* If another CPU is already managing this PMU, simply return. */
 410         if (hisi_pmu->on_cpu != -1)
 411                 return 0;
 412
 413         /* Use this CPU in cpumask for event counting */
 414         hisi_pmu->on_cpu = cpu;
 415
 416         /* Overflow interrupt also should use the same CPU */
 417         WARN_ON(irq_set_affinity(hisi_pmu->irq, cpumask_of(cpu)));
 418
 419         return 0;
 420 }
 421
 422 int hisi_uncore_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
 423 {
 424         struct hisi_pmu *hisi_pmu = hlist_entry_safe(node, struct hisi_pmu,
 425                                                      node);
 426         cpumask_t pmu_online_cpus;
 427         unsigned int target;
 428
 429         if (!cpumask_test_and_clear_cpu(cpu, &hisi_pmu->associated_cpus))
 430                 return 0;
 431
 432         /* Nothing to do if this CPU doesn't own the PMU */
 433         if (hisi_pmu->on_cpu != cpu)
 434                 return 0;
 435
 436         /* Give up ownership of the PMU */
 437         hisi_pmu->on_cpu = -1;
 438
 439         /* Choose a new CPU to migrate ownership of the PMU to */
 440         cpumask_and(&pmu_online_cpus, &hisi_pmu->associated_cpus,
 441                     cpu_online_mask);
 442         target = cpumask_any_but(&pmu_online_cpus, cpu);
 443         if (target >= nr_cpu_ids)
 444                 return 0;
 445
 446         perf_pmu_migrate_context(&hisi_pmu->pmu, cpu, target);
 447         /* Use this CPU for event counting */
 448         hisi_pmu->on_cpu = target;
 449         WARN_ON(irq_set_affinity(hisi_pmu->irq, cpumask_of(target)));
 450
 451         return 0;
 452 }