drivers/perf/hisilicon/hisi_uncore_pmu.c

   1 /*
   2  * HiSilicon SoC Hardware event counters support
   3  *
   4  * Copyright (C) 2017 Hisilicon Limited
   5  * Author: Anurup M <anurup.m@huawei.com>
   6  *         Shaokun Zhang <zhangshaokun@hisilicon.com>
   7  *
   8  * This code is based on the uncore PMUs like arm-cci and arm-ccn.
   9  *
  10  * This program is free software; you can redistribute it and/or modify
  11  * it under the terms of the GNU General Public License version 2 as
  12  * published by the Free Software Foundation.
  13  */
  14 #include <linux/bitmap.h>
  15 #include <linux/bitops.h>
  16 #include <linux/bug.h>
  17 #include <linux/err.h>
  18 #include <linux/errno.h>
  19 #include <linux/interrupt.h>
  20
  21 #include <asm/local64.h>
  22
  23 #include "hisi_uncore_pmu.h"
  24
  25 #define HISI_GET_EVENTID(ev) (ev->hw.config_base & 0xff)
  26 #define HISI_MAX_PERIOD(nr) (BIT_ULL(nr) - 1)
  27
  28 /*
  29  * PMU format attributes
  30  */
  31 ssize_t hisi_format_sysfs_show(struct device *dev,
  32                                struct device_attribute *attr, char *buf)
  33 {
  34         struct dev_ext_attribute *eattr;
  35
  36         eattr = container_of(attr, struct dev_ext_attribute, attr);
  37
  38         return sprintf(buf, "%s\n", (char *)eattr->var);
  39 }
  40
  41 /*
  42  * PMU event attributes
  43  */
  44 ssize_t hisi_event_sysfs_show(struct device *dev,
  45                               struct device_attribute *attr, char *page)
  46 {
  47         struct dev_ext_attribute *eattr;
  48
  49         eattr = container_of(attr, struct dev_ext_attribute, attr);
  50
  51         return sprintf(page, "config=0x%lx\n", (unsigned long)eattr->var);
  52 }
  53
  54 /*
  55  * sysfs cpumask attributes. For uncore PMU, we only have a single CPU to show
  56  */
  57 ssize_t hisi_cpumask_sysfs_show(struct device *dev,
  58                                 struct device_attribute *attr, char *buf)
  59 {
  60         struct hisi_pmu *hisi_pmu = to_hisi_pmu(dev_get_drvdata(dev));
  61
  62         return sprintf(buf, "%d\n", hisi_pmu->on_cpu);
  63 }
  64
  65 static bool hisi_validate_event_group(struct perf_event *event)
  66 {
  67         struct perf_event *sibling, *leader = event->group_leader;
  68         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
  69         /* Include count for the event */
  70         int counters = 1;
  71
  72         if (!is_software_event(leader)) {
  73                 /*
  74                  * We must NOT create groups containing mixed PMUs, although
  75                  * software events are acceptable
  76                  */
  77                 if (leader->pmu != event->pmu)
  78                         return false;
  79
  80                 /* Increment counter for the leader */
  81                 if (leader != event)
  82                         counters++;
  83         }
  84
  85         list_for_each_entry(sibling, &event->group_leader->sibling_list,
  86                             group_entry) {
  87                 if (is_software_event(sibling))
  88                         continue;
  89                 if (sibling->pmu != event->pmu)
  90                         return false;
  91                 /* Increment counter for each sibling */
  92                 counters++;
  93         }
  94
  95         /* The group can not count events more than the counters in the HW */
  96         return counters <= hisi_pmu->num_counters;
  97 }
  98
  99 int hisi_uncore_pmu_counter_valid(struct hisi_pmu *hisi_pmu, int idx)
 100 {
 101         return idx >= 0 && idx < hisi_pmu->num_counters;
 102 }
 103
 104 int hisi_uncore_pmu_get_event_idx(struct perf_event *event)
 105 {
 106         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
 107         unsigned long *used_mask = hisi_pmu->pmu_events.used_mask;
 108         u32 num_counters = hisi_pmu->num_counters;
 109         int idx;
 110
 111         idx = find_first_zero_bit(used_mask, num_counters);
 112         if (idx == num_counters)
 113                 return -EAGAIN;
 114
 115         set_bit(idx, used_mask);
 116
 117         return idx;
 118 }
 119
 120 static void hisi_uncore_pmu_clear_event_idx(struct hisi_pmu *hisi_pmu, int idx)
 121 {
 122         if (!hisi_uncore_pmu_counter_valid(hisi_pmu, idx)) {
 123                 dev_err(hisi_pmu->dev, "Unsupported event index:%d!\n", idx);
 124                 return;
 125         }
 126
 127         clear_bit(idx, hisi_pmu->pmu_events.used_mask);
 128 }
 129
 130 int hisi_uncore_pmu_event_init(struct perf_event *event)
 131 {
 132         struct hw_perf_event *hwc = &event->hw;
 133         struct hisi_pmu *hisi_pmu;
 134
 135         if (event->attr.type != event->pmu->type)
 136                 return -ENOENT;
 137
 138         /*
 139          * We do not support sampling as the counters are all
 140          * shared by all CPU cores in a CPU die(SCCL). Also we
 141          * do not support attach to a task(per-process mode)
 142          */
 143         if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
 144                 return -EOPNOTSUPP;
 145
 146         /* counters do not have these bits */
 147         if (event->attr.exclude_user    ||
 148             event->attr.exclude_kernel  ||
 149             event->attr.exclude_host    ||
 150             event->attr.exclude_guest   ||
 151             event->attr.exclude_hv      ||
 152             event->attr.exclude_idle)
 153                 return -EINVAL;
 154
 155         /*
 156          *  The uncore counters not specific to any CPU, so cannot
 157          *  support per-task
 158          */
 159         if (event->cpu < 0)
 160                 return -EINVAL;
 161
 162         /*
 163          * Validate if the events in group does not exceed the
 164          * available counters in hardware.
 165          */
 166         if (!hisi_validate_event_group(event))
 167                 return -EINVAL;
 168
 169         hisi_pmu = to_hisi_pmu(event->pmu);
 170         if (event->attr.config > hisi_pmu->check_event)
 171                 return -EINVAL;
 172
 173         if (hisi_pmu->on_cpu == -1)
 174                 return -EINVAL;
 175         /*
 176          * We don't assign an index until we actually place the event onto
 177          * hardware. Use -1 to signify that we haven't decided where to put it
 178          * yet.
 179          */
 180         hwc->idx                = -1;
 181         hwc->config_base        = event->attr.config;
 182
 183         /* Enforce to use the same CPU for all events in this PMU */
 184         event->cpu = hisi_pmu->on_cpu;
 185
 186         return 0;
 187 }
 188
 189 /*
 190  * Set the counter to count the event that we're interested in,
 191  * and enable interrupt and counter.
 192  */
 193 static void hisi_uncore_pmu_enable_event(struct perf_event *event)
 194 {
 195         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
 196         struct hw_perf_event *hwc = &event->hw;
 197
 198         hisi_pmu->ops->write_evtype(hisi_pmu, hwc->idx,
 199                                     HISI_GET_EVENTID(event));
 200
 201         hisi_pmu->ops->enable_counter_int(hisi_pmu, hwc);
 202         hisi_pmu->ops->enable_counter(hisi_pmu, hwc);
 203 }
 204
 205 /*
 206  * Disable counter and interrupt.
 207  */
 208 static void hisi_uncore_pmu_disable_event(struct perf_event *event)
 209 {
 210         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
 211         struct hw_perf_event *hwc = &event->hw;
 212
 213         hisi_pmu->ops->disable_counter(hisi_pmu, hwc);
 214         hisi_pmu->ops->disable_counter_int(hisi_pmu, hwc);
 215 }
 216
 217 void hisi_uncore_pmu_set_event_period(struct perf_event *event)
 218 {
 219         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
 220         struct hw_perf_event *hwc = &event->hw;
 221
 222         /*
 223          * The HiSilicon PMU counters support 32 bits or 48 bits, depending on
 224          * the PMU. We reduce it to 2^(counter_bits - 1) to account for the
 225          * extreme interrupt latency. So we could hopefully handle the overflow
 226          * interrupt before another 2^(counter_bits - 1) events occur and the
 227          * counter overtakes its previous value.
 228          */
 229         u64 val = BIT_ULL(hisi_pmu->counter_bits - 1);
 230
 231         local64_set(&hwc->prev_count, val);
 232         /* Write start value to the hardware event counter */
 233         hisi_pmu->ops->write_counter(hisi_pmu, hwc, val);
 234 }
 235
 236 void hisi_uncore_pmu_event_update(struct perf_event *event)
 237 {
 238         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
 239         struct hw_perf_event *hwc = &event->hw;
 240         u64 delta, prev_raw_count, new_raw_count;
 241
 242         do {
 243                 /* Read the count from the counter register */
 244                 new_raw_count = hisi_pmu->ops->read_counter(hisi_pmu, hwc);
 245                 prev_raw_count = local64_read(&hwc->prev_count);
 246         } while (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
 247                                  new_raw_count) != prev_raw_count);
 248         /*
 249          * compute the delta
 250          */
 251         delta = (new_raw_count - prev_raw_count) &
 252                 HISI_MAX_PERIOD(hisi_pmu->counter_bits);
 253         local64_add(delta, &event->count);
 254 }
 255
 256 void hisi_uncore_pmu_start(struct perf_event *event, int flags)
 257 {
 258         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
 259         struct hw_perf_event *hwc = &event->hw;
 260
 261         if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
 262                 return;
 263
 264         WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
 265         hwc->state = 0;
 266         hisi_uncore_pmu_set_event_period(event);
 267
 268         if (flags & PERF_EF_RELOAD) {
 269                 u64 prev_raw_count =  local64_read(&hwc->prev_count);
 270
 271                 hisi_pmu->ops->write_counter(hisi_pmu, hwc, prev_raw_count);
 272         }
 273
 274         hisi_uncore_pmu_enable_event(event);
 275         perf_event_update_userpage(event);
 276 }
 277
 278 void hisi_uncore_pmu_stop(struct perf_event *event, int flags)
 279 {
 280         struct hw_perf_event *hwc = &event->hw;
 281
 282         hisi_uncore_pmu_disable_event(event);
 283         WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
 284         hwc->state |= PERF_HES_STOPPED;
 285
 286         if (hwc->state & PERF_HES_UPTODATE)
 287                 return;
 288
 289         /* Read hardware counter and update the perf counter statistics */
 290         hisi_uncore_pmu_event_update(event);
 291         hwc->state |= PERF_HES_UPTODATE;
 292 }
 293
 294 int hisi_uncore_pmu_add(struct perf_event *event, int flags)
 295 {
 296         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
 297         struct hw_perf_event *hwc = &event->hw;
 298         int idx;
 299
 300         hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
 301
 302         /* Get an available counter index for counting */
 303         idx = hisi_pmu->ops->get_event_idx(event);
 304         if (idx < 0)
 305                 return idx;
 306
 307         event->hw.idx = idx;
 308         hisi_pmu->pmu_events.hw_events[idx] = event;
 309
 310         if (flags & PERF_EF_START)
 311                 hisi_uncore_pmu_start(event, PERF_EF_RELOAD);
 312
 313         return 0;
 314 }
 315
 316 void hisi_uncore_pmu_del(struct perf_event *event, int flags)
 317 {
 318         struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
 319         struct hw_perf_event *hwc = &event->hw;
 320
 321         hisi_uncore_pmu_stop(event, PERF_EF_UPDATE);
 322         hisi_uncore_pmu_clear_event_idx(hisi_pmu, hwc->idx);
 323         perf_event_update_userpage(event);
 324         hisi_pmu->pmu_events.hw_events[hwc->idx] = NULL;
 325 }
 326
 327 void hisi_uncore_pmu_read(struct perf_event *event)
 328 {
 329         /* Read hardware counter and update the perf counter statistics */
 330         hisi_uncore_pmu_event_update(event);
 331 }
 332
 333 void hisi_uncore_pmu_enable(struct pmu *pmu)
 334 {
 335         struct hisi_pmu *hisi_pmu = to_hisi_pmu(pmu);
 336         int enabled = bitmap_weight(hisi_pmu->pmu_events.used_mask,
 337                                     hisi_pmu->num_counters);
 338
 339         if (!enabled)
 340                 return;
 341
 342         hisi_pmu->ops->start_counters(hisi_pmu);
 343 }
 344
 345 void hisi_uncore_pmu_disable(struct pmu *pmu)
 346 {
 347         struct hisi_pmu *hisi_pmu = to_hisi_pmu(pmu);
 348
 349         hisi_pmu->ops->stop_counters(hisi_pmu);
 350 }
 351
 352 /*
 353  * Read Super CPU cluster and CPU cluster ID from MPIDR_EL1.
 354  * If multi-threading is supported, SCCL_ID is in MPIDR[aff3] and CCL_ID
 355  * is in MPIDR[aff2]; if not, SCCL_ID is in MPIDR[aff2] and CCL_ID is
 356  * in MPIDR[aff1]. If this changes in future, this shall be updated.
 357  */
 358 static void hisi_read_sccl_and_ccl_id(int *sccl_id, int *ccl_id)
 359 {
 360         u64 mpidr = read_cpuid_mpidr();
 361
 362         if (mpidr & MPIDR_MT_BITMASK) {
 363                 if (sccl_id)
 364                         *sccl_id = MPIDR_AFFINITY_LEVEL(mpidr, 3);
 365                 if (ccl_id)
 366                         *ccl_id = MPIDR_AFFINITY_LEVEL(mpidr, 2);
 367         } else {
 368                 if (sccl_id)
 369                         *sccl_id = MPIDR_AFFINITY_LEVEL(mpidr, 2);
 370                 if (ccl_id)
 371                         *ccl_id = MPIDR_AFFINITY_LEVEL(mpidr, 1);
 372         }
 373 }
 374
 375 /*
 376  * Check whether the CPU is associated with this uncore PMU
 377  */
 378 static bool hisi_pmu_cpu_is_associated_pmu(struct hisi_pmu *hisi_pmu)
 379 {
 380         int sccl_id, ccl_id;
 381
 382         if (hisi_pmu->ccl_id == -1) {
 383                 /* If CCL_ID is -1, the PMU only shares the same SCCL */
 384                 hisi_read_sccl_and_ccl_id(&sccl_id, NULL);
 385
 386                 return sccl_id == hisi_pmu->sccl_id;
 387         }
 388
 389         hisi_read_sccl_and_ccl_id(&sccl_id, &ccl_id);
 390
 391         return sccl_id == hisi_pmu->sccl_id && ccl_id == hisi_pmu->ccl_id;
 392 }
 393
 394 int hisi_uncore_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
 395 {
 396         struct hisi_pmu *hisi_pmu = hlist_entry_safe(node, struct hisi_pmu,
 397                                                      node);
 398
 399         if (!hisi_pmu_cpu_is_associated_pmu(hisi_pmu))
 400                 return 0;
 401
 402         cpumask_set_cpu(cpu, &hisi_pmu->associated_cpus);
 403
 404         /* If another CPU is already managing this PMU, simply return. */
 405         if (hisi_pmu->on_cpu != -1)
 406                 return 0;
 407
 408         /* Use this CPU in cpumask for event counting */
 409         hisi_pmu->on_cpu = cpu;
 410
 411         /* Overflow interrupt also should use the same CPU */
 412         WARN_ON(irq_set_affinity(hisi_pmu->irq, cpumask_of(cpu)));
 413
 414         return 0;
 415 }
 416
 417 int hisi_uncore_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
 418 {
 419         struct hisi_pmu *hisi_pmu = hlist_entry_safe(node, struct hisi_pmu,
 420                                                      node);
 421         cpumask_t pmu_online_cpus;
 422         unsigned int target;
 423
 424         if (!cpumask_test_and_clear_cpu(cpu, &hisi_pmu->associated_cpus))
 425                 return 0;
 426
 427         /* Nothing to do if this CPU doesn't own the PMU */
 428         if (hisi_pmu->on_cpu != cpu)
 429                 return 0;
 430
 431         /* Give up ownership of the PMU */
 432         hisi_pmu->on_cpu = -1;
 433
 434         /* Choose a new CPU to migrate ownership of the PMU to */
 435         cpumask_and(&pmu_online_cpus, &hisi_pmu->associated_cpus,
 436                     cpu_online_mask);
 437         target = cpumask_any_but(&pmu_online_cpus, cpu);
 438         if (target >= nr_cpu_ids)
 439                 return 0;
 440
 441         perf_pmu_migrate_context(&hisi_pmu->pmu, cpu, target);
 442         /* Use this CPU for event counting */
 443         hisi_pmu->on_cpu = target;
 444         WARN_ON(irq_set_affinity(hisi_pmu->irq, cpumask_of(target)));
 445
 446         return 0;
 447 }