1 /* Copyright (c) 2015-2017 The Linux Foundation. All rights reserved.
3 * This program is free software; you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License version 2 and
5 * only version 2 as published by the Free Software Foundation.
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
12 #include <linux/acpi.h>
13 #include <linux/bitops.h>
14 #include <linux/bug.h>
15 #include <linux/cpuhotplug.h>
16 #include <linux/cpumask.h>
17 #include <linux/device.h>
18 #include <linux/errno.h>
19 #include <linux/interrupt.h>
20 #include <linux/irq.h>
21 #include <linux/kernel.h>
22 #include <linux/list.h>
23 #include <linux/percpu.h>
24 #include <linux/perf_event.h>
25 #include <linux/platform_device.h>
26 #include <linux/smp.h>
27 #include <linux/spinlock.h>
28 #include <linux/sysfs.h>
29 #include <linux/types.h>
31 #include <asm/barrier.h>
32 #include <asm/local64.h>
33 #include <asm/sysreg.h>
37 #define L2PMCR_NUM_EV_SHIFT 11
38 #define L2PMCR_NUM_EV_MASK 0x1F
41 #define L2PMCNTENCLR 0x403
42 #define L2PMCNTENSET 0x404
43 #define L2PMINTENCLR 0x405
44 #define L2PMINTENSET 0x406
45 #define L2PMOVSCLR 0x407
46 #define L2PMOVSSET 0x408
47 #define L2PMCCNTCR 0x409
48 #define L2PMCCNTR 0x40A
49 #define L2PMCCNTSR 0x40C
50 #define L2PMRESR 0x410
51 #define IA_L2PMXEVCNTCR_BASE 0x420
52 #define IA_L2PMXEVCNTR_BASE 0x421
53 #define IA_L2PMXEVFILTER_BASE 0x423
54 #define IA_L2PMXEVTYPER_BASE 0x424
56 #define IA_L2_REG_OFFSET 0x10
58 #define L2PMXEVFILTER_SUFILTER_ALL 0x000E0000
59 #define L2PMXEVFILTER_ORGFILTER_IDINDEP 0x00000004
60 #define L2PMXEVFILTER_ORGFILTER_ALL 0x00000003
62 #define L2EVTYPER_REG_SHIFT 3
64 #define L2PMRESR_GROUP_BITS 8
65 #define L2PMRESR_GROUP_MASK GENMASK(7, 0)
67 #define L2CYCLE_CTR_BIT 31
68 #define L2CYCLE_CTR_RAW_CODE 0xFE
70 #define L2PMCR_RESET_ALL 0x6
71 #define L2PMCR_COUNTERS_ENABLE 0x1
72 #define L2PMCR_COUNTERS_DISABLE 0x0
74 #define L2PMRESR_EN BIT_ULL(63)
76 #define L2_EVT_MASK 0x00000FFF
77 #define L2_EVT_CODE_MASK 0x00000FF0
78 #define L2_EVT_GRP_MASK 0x0000000F
79 #define L2_EVT_CODE_SHIFT 4
80 #define L2_EVT_GRP_SHIFT 0
82 #define L2_EVT_CODE(event) (((event) & L2_EVT_CODE_MASK) >> L2_EVT_CODE_SHIFT)
83 #define L2_EVT_GROUP(event) (((event) & L2_EVT_GRP_MASK) >> L2_EVT_GRP_SHIFT)
85 #define L2_EVT_GROUP_MAX 7
87 #define L2_COUNTER_RELOAD BIT_ULL(31)
88 #define L2_CYCLE_COUNTER_RELOAD BIT_ULL(63)
90 #define L2CPUSRSELR_EL1 sys_reg(3, 3, 15, 0, 6)
91 #define L2CPUSRDR_EL1 sys_reg(3, 3, 15, 0, 7)
93 #define reg_idx(reg, i) (((i) * IA_L2_REG_OFFSET) + reg##_BASE)
98 #define L2_EVENT_CYCLES 0xfe
99 #define L2_EVENT_DCACHE_OPS 0x400
100 #define L2_EVENT_ICACHE_OPS 0x401
101 #define L2_EVENT_TLBI 0x402
102 #define L2_EVENT_BARRIERS 0x403
103 #define L2_EVENT_TOTAL_READS 0x405
104 #define L2_EVENT_TOTAL_WRITES 0x406
105 #define L2_EVENT_TOTAL_REQUESTS 0x407
106 #define L2_EVENT_LDREX 0x420
107 #define L2_EVENT_STREX 0x421
108 #define L2_EVENT_CLREX 0x422
110 static DEFINE_RAW_SPINLOCK(l2_access_lock
);
113 * set_l2_indirect_reg: write value to an L2 register
114 * @reg: Address of L2 register.
115 * @value: Value to be written to register.
117 * Use architecturally required barriers for ordering between system register
120 static void set_l2_indirect_reg(u64 reg
, u64 val
)
124 raw_spin_lock_irqsave(&l2_access_lock
, flags
);
125 write_sysreg_s(reg
, L2CPUSRSELR_EL1
);
127 write_sysreg_s(val
, L2CPUSRDR_EL1
);
129 raw_spin_unlock_irqrestore(&l2_access_lock
, flags
);
133 * get_l2_indirect_reg: read an L2 register value
134 * @reg: Address of L2 register.
136 * Use architecturally required barriers for ordering between system register
139 static u64
get_l2_indirect_reg(u64 reg
)
144 raw_spin_lock_irqsave(&l2_access_lock
, flags
);
145 write_sysreg_s(reg
, L2CPUSRSELR_EL1
);
147 val
= read_sysreg_s(L2CPUSRDR_EL1
);
148 raw_spin_unlock_irqrestore(&l2_access_lock
, flags
);
156 * Aggregate PMU. Implements the core pmu functions and manages
160 struct hlist_node node
;
165 struct platform_device
*pdev
;
166 struct cluster_pmu
* __percpu
*pmu_cluster
;
167 struct list_head clusters
;
171 * The cache is made up of one or more clusters, each cluster has its own PMU.
172 * Each cluster is associated with one or more CPUs.
173 * This structure represents one of the hardware PMUs.
175 * Events can be envisioned as a 2-dimensional array. Each column represents
176 * a group of events. There are 8 groups. Only one entry from each
177 * group can be in use at a time.
179 * Events are specified as 0xCCG, where CC is 2 hex digits specifying
180 * the code (array row) and G specifies the group (column).
182 * In addition there is a cycle counter event specified by L2CYCLE_CTR_RAW_CODE
183 * which is outside the above scheme.
186 struct list_head next
;
187 struct perf_event
*events
[MAX_L2_CTRS
];
188 struct l2cache_pmu
*l2cache_pmu
;
189 DECLARE_BITMAP(used_counters
, MAX_L2_CTRS
);
190 DECLARE_BITMAP(used_groups
, L2_EVT_GROUP_MAX
+ 1);
193 /* The CPU that is used for collecting events on this cluster */
195 /* All the CPUs associated with this cluster */
196 cpumask_t cluster_cpus
;
200 #define to_l2cache_pmu(p) (container_of(p, struct l2cache_pmu, pmu))
202 static u32 l2_cycle_ctr_idx
;
203 static u32 l2_counter_present_mask
;
205 static inline u32
idx_to_reg_bit(u32 idx
)
207 if (idx
== l2_cycle_ctr_idx
)
208 return BIT(L2CYCLE_CTR_BIT
);
213 static inline struct cluster_pmu
*get_cluster_pmu(
214 struct l2cache_pmu
*l2cache_pmu
, int cpu
)
216 return *per_cpu_ptr(l2cache_pmu
->pmu_cluster
, cpu
);
219 static void cluster_pmu_reset(void)
221 /* Reset all counters */
222 set_l2_indirect_reg(L2PMCR
, L2PMCR_RESET_ALL
);
223 set_l2_indirect_reg(L2PMCNTENCLR
, l2_counter_present_mask
);
224 set_l2_indirect_reg(L2PMINTENCLR
, l2_counter_present_mask
);
225 set_l2_indirect_reg(L2PMOVSCLR
, l2_counter_present_mask
);
228 static inline void cluster_pmu_enable(void)
230 set_l2_indirect_reg(L2PMCR
, L2PMCR_COUNTERS_ENABLE
);
233 static inline void cluster_pmu_disable(void)
235 set_l2_indirect_reg(L2PMCR
, L2PMCR_COUNTERS_DISABLE
);
238 static inline void cluster_pmu_counter_set_value(u32 idx
, u64 value
)
240 if (idx
== l2_cycle_ctr_idx
)
241 set_l2_indirect_reg(L2PMCCNTR
, value
);
243 set_l2_indirect_reg(reg_idx(IA_L2PMXEVCNTR
, idx
), value
);
246 static inline u64
cluster_pmu_counter_get_value(u32 idx
)
250 if (idx
== l2_cycle_ctr_idx
)
251 value
= get_l2_indirect_reg(L2PMCCNTR
);
253 value
= get_l2_indirect_reg(reg_idx(IA_L2PMXEVCNTR
, idx
));
258 static inline void cluster_pmu_counter_enable(u32 idx
)
260 set_l2_indirect_reg(L2PMCNTENSET
, idx_to_reg_bit(idx
));
263 static inline void cluster_pmu_counter_disable(u32 idx
)
265 set_l2_indirect_reg(L2PMCNTENCLR
, idx_to_reg_bit(idx
));
268 static inline void cluster_pmu_counter_enable_interrupt(u32 idx
)
270 set_l2_indirect_reg(L2PMINTENSET
, idx_to_reg_bit(idx
));
273 static inline void cluster_pmu_counter_disable_interrupt(u32 idx
)
275 set_l2_indirect_reg(L2PMINTENCLR
, idx_to_reg_bit(idx
));
278 static inline void cluster_pmu_set_evccntcr(u32 val
)
280 set_l2_indirect_reg(L2PMCCNTCR
, val
);
283 static inline void cluster_pmu_set_evcntcr(u32 ctr
, u32 val
)
285 set_l2_indirect_reg(reg_idx(IA_L2PMXEVCNTCR
, ctr
), val
);
288 static inline void cluster_pmu_set_evtyper(u32 ctr
, u32 val
)
290 set_l2_indirect_reg(reg_idx(IA_L2PMXEVTYPER
, ctr
), val
);
293 static void cluster_pmu_set_resr(struct cluster_pmu
*cluster
,
294 u32 event_group
, u32 event_cc
)
301 shift
= L2PMRESR_GROUP_BITS
* event_group
;
302 field
= ((u64
)(event_cc
& L2PMRESR_GROUP_MASK
) << shift
);
304 spin_lock_irqsave(&cluster
->pmu_lock
, flags
);
306 resr_val
= get_l2_indirect_reg(L2PMRESR
);
307 resr_val
&= ~(L2PMRESR_GROUP_MASK
<< shift
);
309 resr_val
|= L2PMRESR_EN
;
310 set_l2_indirect_reg(L2PMRESR
, resr_val
);
312 spin_unlock_irqrestore(&cluster
->pmu_lock
, flags
);
316 * Hardware allows filtering of events based on the originating
317 * CPU. Turn this off by setting filter bits to allow events from
318 * all CPUS, subunits and ID independent events in this cluster.
320 static inline void cluster_pmu_set_evfilter_sys_mode(u32 ctr
)
322 u32 val
= L2PMXEVFILTER_SUFILTER_ALL
|
323 L2PMXEVFILTER_ORGFILTER_IDINDEP
|
324 L2PMXEVFILTER_ORGFILTER_ALL
;
326 set_l2_indirect_reg(reg_idx(IA_L2PMXEVFILTER
, ctr
), val
);
329 static inline u32
cluster_pmu_getreset_ovsr(void)
331 u32 result
= get_l2_indirect_reg(L2PMOVSSET
);
333 set_l2_indirect_reg(L2PMOVSCLR
, result
);
337 static inline bool cluster_pmu_has_overflowed(u32 ovsr
)
339 return !!(ovsr
& l2_counter_present_mask
);
342 static inline bool cluster_pmu_counter_has_overflowed(u32 ovsr
, u32 idx
)
344 return !!(ovsr
& idx_to_reg_bit(idx
));
347 static void l2_cache_event_update(struct perf_event
*event
)
349 struct hw_perf_event
*hwc
= &event
->hw
;
350 u64 delta
, prev
, now
;
354 prev
= local64_read(&hwc
->prev_count
);
355 now
= cluster_pmu_counter_get_value(idx
);
356 } while (local64_cmpxchg(&hwc
->prev_count
, prev
, now
) != prev
);
359 * The cycle counter is 64-bit, but all other counters are
360 * 32-bit, and we must handle 32-bit overflow explicitly.
363 if (idx
!= l2_cycle_ctr_idx
)
366 local64_add(delta
, &event
->count
);
369 static void l2_cache_cluster_set_period(struct cluster_pmu
*cluster
,
370 struct hw_perf_event
*hwc
)
376 * We limit the max period to half the max counter value so
377 * that even in the case of extreme interrupt latency the
378 * counter will (hopefully) not wrap past its initial value.
380 if (idx
== l2_cycle_ctr_idx
)
381 new = L2_CYCLE_COUNTER_RELOAD
;
383 new = L2_COUNTER_RELOAD
;
385 local64_set(&hwc
->prev_count
, new);
386 cluster_pmu_counter_set_value(idx
, new);
389 static int l2_cache_get_event_idx(struct cluster_pmu
*cluster
,
390 struct perf_event
*event
)
392 struct hw_perf_event
*hwc
= &event
->hw
;
394 int num_ctrs
= cluster
->l2cache_pmu
->num_counters
- 1;
397 if (hwc
->config_base
== L2CYCLE_CTR_RAW_CODE
) {
398 if (test_and_set_bit(l2_cycle_ctr_idx
, cluster
->used_counters
))
401 return l2_cycle_ctr_idx
;
404 idx
= find_first_zero_bit(cluster
->used_counters
, num_ctrs
);
406 /* The counters are all in use. */
410 * Check for column exclusion: event column already in use by another
411 * event. This is for events which are not in the same group.
412 * Conflicting events in the same group are detected in event_init.
414 group
= L2_EVT_GROUP(hwc
->config_base
);
415 if (test_bit(group
, cluster
->used_groups
))
418 set_bit(idx
, cluster
->used_counters
);
419 set_bit(group
, cluster
->used_groups
);
424 static void l2_cache_clear_event_idx(struct cluster_pmu
*cluster
,
425 struct perf_event
*event
)
427 struct hw_perf_event
*hwc
= &event
->hw
;
430 clear_bit(idx
, cluster
->used_counters
);
431 if (hwc
->config_base
!= L2CYCLE_CTR_RAW_CODE
)
432 clear_bit(L2_EVT_GROUP(hwc
->config_base
), cluster
->used_groups
);
435 static irqreturn_t
l2_cache_handle_irq(int irq_num
, void *data
)
437 struct cluster_pmu
*cluster
= data
;
438 int num_counters
= cluster
->l2cache_pmu
->num_counters
;
442 ovsr
= cluster_pmu_getreset_ovsr();
443 if (!cluster_pmu_has_overflowed(ovsr
))
446 for_each_set_bit(idx
, cluster
->used_counters
, num_counters
) {
447 struct perf_event
*event
= cluster
->events
[idx
];
448 struct hw_perf_event
*hwc
;
450 if (WARN_ON_ONCE(!event
))
453 if (!cluster_pmu_counter_has_overflowed(ovsr
, idx
))
456 l2_cache_event_update(event
);
459 l2_cache_cluster_set_period(cluster
, hwc
);
466 * Implementation of abstract pmu functionality required by
467 * the core perf events code.
470 static void l2_cache_pmu_enable(struct pmu
*pmu
)
473 * Although there is only one PMU (per socket) controlling multiple
474 * physical PMUs (per cluster), because we do not support per-task mode
475 * each event is associated with a CPU. Each event has pmu_enable
476 * called on its CPU, so here it is only necessary to enable the
477 * counters for the current CPU.
480 cluster_pmu_enable();
483 static void l2_cache_pmu_disable(struct pmu
*pmu
)
485 cluster_pmu_disable();
488 static int l2_cache_event_init(struct perf_event
*event
)
490 struct hw_perf_event
*hwc
= &event
->hw
;
491 struct cluster_pmu
*cluster
;
492 struct perf_event
*sibling
;
493 struct l2cache_pmu
*l2cache_pmu
;
495 if (event
->attr
.type
!= event
->pmu
->type
)
498 l2cache_pmu
= to_l2cache_pmu(event
->pmu
);
500 if (hwc
->sample_period
) {
501 dev_dbg_ratelimited(&l2cache_pmu
->pdev
->dev
,
502 "Sampling not supported\n");
506 if (event
->cpu
< 0) {
507 dev_dbg_ratelimited(&l2cache_pmu
->pdev
->dev
,
508 "Per-task mode not supported\n");
512 /* We cannot filter accurately so we just don't allow it. */
513 if (event
->attr
.exclude_user
|| event
->attr
.exclude_kernel
||
514 event
->attr
.exclude_hv
|| event
->attr
.exclude_idle
) {
515 dev_dbg_ratelimited(&l2cache_pmu
->pdev
->dev
,
516 "Can't exclude execution levels\n");
520 if (((L2_EVT_GROUP(event
->attr
.config
) > L2_EVT_GROUP_MAX
) ||
521 ((event
->attr
.config
& ~L2_EVT_MASK
) != 0)) &&
522 (event
->attr
.config
!= L2CYCLE_CTR_RAW_CODE
)) {
523 dev_dbg_ratelimited(&l2cache_pmu
->pdev
->dev
,
524 "Invalid config %llx\n",
529 /* Don't allow groups with mixed PMUs, except for s/w events */
530 if (event
->group_leader
->pmu
!= event
->pmu
&&
531 !is_software_event(event
->group_leader
)) {
532 dev_dbg_ratelimited(&l2cache_pmu
->pdev
->dev
,
533 "Can't create mixed PMU group\n");
537 list_for_each_entry(sibling
, &event
->group_leader
->sibling_list
,
539 if (sibling
->pmu
!= event
->pmu
&&
540 !is_software_event(sibling
)) {
541 dev_dbg_ratelimited(&l2cache_pmu
->pdev
->dev
,
542 "Can't create mixed PMU group\n");
546 cluster
= get_cluster_pmu(l2cache_pmu
, event
->cpu
);
548 /* CPU has not been initialised */
549 dev_dbg_ratelimited(&l2cache_pmu
->pdev
->dev
,
550 "CPU%d not associated with L2 cluster\n", event
->cpu
);
554 /* Ensure all events in a group are on the same cpu */
555 if ((event
->group_leader
!= event
) &&
556 (cluster
->on_cpu
!= event
->group_leader
->cpu
)) {
557 dev_dbg_ratelimited(&l2cache_pmu
->pdev
->dev
,
558 "Can't create group on CPUs %d and %d",
559 event
->cpu
, event
->group_leader
->cpu
);
563 if ((event
!= event
->group_leader
) &&
564 !is_software_event(event
->group_leader
) &&
565 (L2_EVT_GROUP(event
->group_leader
->attr
.config
) ==
566 L2_EVT_GROUP(event
->attr
.config
))) {
567 dev_dbg_ratelimited(&l2cache_pmu
->pdev
->dev
,
568 "Column exclusion: conflicting events %llx %llx\n",
569 event
->group_leader
->attr
.config
,
574 list_for_each_entry(sibling
, &event
->group_leader
->sibling_list
,
576 if ((sibling
!= event
) &&
577 !is_software_event(sibling
) &&
578 (L2_EVT_GROUP(sibling
->attr
.config
) ==
579 L2_EVT_GROUP(event
->attr
.config
))) {
580 dev_dbg_ratelimited(&l2cache_pmu
->pdev
->dev
,
581 "Column exclusion: conflicting events %llx %llx\n",
582 sibling
->attr
.config
,
589 hwc
->config_base
= event
->attr
.config
;
592 * Ensure all events are on the same cpu so all events are in the
593 * same cpu context, to avoid races on pmu_enable etc.
595 event
->cpu
= cluster
->on_cpu
;
600 static void l2_cache_event_start(struct perf_event
*event
, int flags
)
602 struct cluster_pmu
*cluster
;
603 struct hw_perf_event
*hwc
= &event
->hw
;
606 u32 event_cc
, event_group
;
610 cluster
= get_cluster_pmu(to_l2cache_pmu(event
->pmu
), event
->cpu
);
612 l2_cache_cluster_set_period(cluster
, hwc
);
614 if (hwc
->config_base
== L2CYCLE_CTR_RAW_CODE
) {
615 cluster_pmu_set_evccntcr(0);
617 config
= hwc
->config_base
;
618 event_cc
= L2_EVT_CODE(config
);
619 event_group
= L2_EVT_GROUP(config
);
621 cluster_pmu_set_evcntcr(idx
, 0);
622 cluster_pmu_set_evtyper(idx
, event_group
);
623 cluster_pmu_set_resr(cluster
, event_group
, event_cc
);
624 cluster_pmu_set_evfilter_sys_mode(idx
);
627 cluster_pmu_counter_enable_interrupt(idx
);
628 cluster_pmu_counter_enable(idx
);
631 static void l2_cache_event_stop(struct perf_event
*event
, int flags
)
633 struct hw_perf_event
*hwc
= &event
->hw
;
636 if (hwc
->state
& PERF_HES_STOPPED
)
639 cluster_pmu_counter_disable_interrupt(idx
);
640 cluster_pmu_counter_disable(idx
);
642 if (flags
& PERF_EF_UPDATE
)
643 l2_cache_event_update(event
);
644 hwc
->state
|= PERF_HES_STOPPED
| PERF_HES_UPTODATE
;
647 static int l2_cache_event_add(struct perf_event
*event
, int flags
)
649 struct hw_perf_event
*hwc
= &event
->hw
;
652 struct cluster_pmu
*cluster
;
654 cluster
= get_cluster_pmu(to_l2cache_pmu(event
->pmu
), event
->cpu
);
656 idx
= l2_cache_get_event_idx(cluster
, event
);
661 hwc
->state
= PERF_HES_STOPPED
| PERF_HES_UPTODATE
;
662 cluster
->events
[idx
] = event
;
663 local64_set(&hwc
->prev_count
, 0);
665 if (flags
& PERF_EF_START
)
666 l2_cache_event_start(event
, flags
);
668 /* Propagate changes to the userspace mapping. */
669 perf_event_update_userpage(event
);
674 static void l2_cache_event_del(struct perf_event
*event
, int flags
)
676 struct hw_perf_event
*hwc
= &event
->hw
;
677 struct cluster_pmu
*cluster
;
680 cluster
= get_cluster_pmu(to_l2cache_pmu(event
->pmu
), event
->cpu
);
682 l2_cache_event_stop(event
, flags
| PERF_EF_UPDATE
);
683 cluster
->events
[idx
] = NULL
;
684 l2_cache_clear_event_idx(cluster
, event
);
686 perf_event_update_userpage(event
);
689 static void l2_cache_event_read(struct perf_event
*event
)
691 l2_cache_event_update(event
);
694 static ssize_t
l2_cache_pmu_cpumask_show(struct device
*dev
,
695 struct device_attribute
*attr
,
698 struct l2cache_pmu
*l2cache_pmu
= to_l2cache_pmu(dev_get_drvdata(dev
));
700 return cpumap_print_to_pagebuf(true, buf
, &l2cache_pmu
->cpumask
);
703 static struct device_attribute l2_cache_pmu_cpumask_attr
=
704 __ATTR(cpumask
, S_IRUGO
, l2_cache_pmu_cpumask_show
, NULL
);
706 static struct attribute
*l2_cache_pmu_cpumask_attrs
[] = {
707 &l2_cache_pmu_cpumask_attr
.attr
,
711 static struct attribute_group l2_cache_pmu_cpumask_group
= {
712 .attrs
= l2_cache_pmu_cpumask_attrs
,
715 /* CCG format for perf RAW codes. */
716 PMU_FORMAT_ATTR(l2_code
, "config:4-11");
717 PMU_FORMAT_ATTR(l2_group
, "config:0-3");
718 PMU_FORMAT_ATTR(event
, "config:0-11");
720 static struct attribute
*l2_cache_pmu_formats
[] = {
721 &format_attr_l2_code
.attr
,
722 &format_attr_l2_group
.attr
,
723 &format_attr_event
.attr
,
727 static struct attribute_group l2_cache_pmu_format_group
= {
729 .attrs
= l2_cache_pmu_formats
,
732 static ssize_t
l2cache_pmu_event_show(struct device
*dev
,
733 struct device_attribute
*attr
, char *page
)
735 struct perf_pmu_events_attr
*pmu_attr
;
737 pmu_attr
= container_of(attr
, struct perf_pmu_events_attr
, attr
);
738 return sprintf(page
, "event=0x%02llx\n", pmu_attr
->id
);
741 #define L2CACHE_EVENT_ATTR(_name, _id) \
742 (&((struct perf_pmu_events_attr[]) { \
743 { .attr = __ATTR(_name, 0444, l2cache_pmu_event_show, NULL), \
747 static struct attribute
*l2_cache_pmu_events
[] = {
748 L2CACHE_EVENT_ATTR(cycles
, L2_EVENT_CYCLES
),
749 L2CACHE_EVENT_ATTR(dcache
-ops
, L2_EVENT_DCACHE_OPS
),
750 L2CACHE_EVENT_ATTR(icache
-ops
, L2_EVENT_ICACHE_OPS
),
751 L2CACHE_EVENT_ATTR(tlbi
, L2_EVENT_TLBI
),
752 L2CACHE_EVENT_ATTR(barriers
, L2_EVENT_BARRIERS
),
753 L2CACHE_EVENT_ATTR(total
-reads
, L2_EVENT_TOTAL_READS
),
754 L2CACHE_EVENT_ATTR(total
-writes
, L2_EVENT_TOTAL_WRITES
),
755 L2CACHE_EVENT_ATTR(total
-requests
, L2_EVENT_TOTAL_REQUESTS
),
756 L2CACHE_EVENT_ATTR(ldrex
, L2_EVENT_LDREX
),
757 L2CACHE_EVENT_ATTR(strex
, L2_EVENT_STREX
),
758 L2CACHE_EVENT_ATTR(clrex
, L2_EVENT_CLREX
),
762 static struct attribute_group l2_cache_pmu_events_group
= {
764 .attrs
= l2_cache_pmu_events
,
767 static const struct attribute_group
*l2_cache_pmu_attr_grps
[] = {
768 &l2_cache_pmu_format_group
,
769 &l2_cache_pmu_cpumask_group
,
770 &l2_cache_pmu_events_group
,
775 * Generic device handlers
778 static const struct acpi_device_id l2_cache_pmu_acpi_match
[] = {
783 static int get_num_counters(void)
787 val
= get_l2_indirect_reg(L2PMCR
);
790 * Read number of counters from L2PMCR and add 1
791 * for the cycle counter.
793 return ((val
>> L2PMCR_NUM_EV_SHIFT
) & L2PMCR_NUM_EV_MASK
) + 1;
796 static struct cluster_pmu
*l2_cache_associate_cpu_with_cluster(
797 struct l2cache_pmu
*l2cache_pmu
, int cpu
)
801 struct cluster_pmu
*cluster
= NULL
;
804 * This assumes that the cluster_id is in MPIDR[aff1] for
805 * single-threaded cores, and MPIDR[aff2] for multi-threaded
806 * cores. This logic will have to be updated if this changes.
808 mpidr
= read_cpuid_mpidr();
809 if (mpidr
& MPIDR_MT_BITMASK
)
810 cpu_cluster_id
= MPIDR_AFFINITY_LEVEL(mpidr
, 2);
812 cpu_cluster_id
= MPIDR_AFFINITY_LEVEL(mpidr
, 1);
814 list_for_each_entry(cluster
, &l2cache_pmu
->clusters
, next
) {
815 if (cluster
->cluster_id
!= cpu_cluster_id
)
818 dev_info(&l2cache_pmu
->pdev
->dev
,
819 "CPU%d associated with cluster %d\n", cpu
,
820 cluster
->cluster_id
);
821 cpumask_set_cpu(cpu
, &cluster
->cluster_cpus
);
822 *per_cpu_ptr(l2cache_pmu
->pmu_cluster
, cpu
) = cluster
;
829 static int l2cache_pmu_online_cpu(unsigned int cpu
, struct hlist_node
*node
)
831 struct cluster_pmu
*cluster
;
832 struct l2cache_pmu
*l2cache_pmu
;
834 l2cache_pmu
= hlist_entry_safe(node
, struct l2cache_pmu
, node
);
835 cluster
= get_cluster_pmu(l2cache_pmu
, cpu
);
837 /* First time this CPU has come online */
838 cluster
= l2_cache_associate_cpu_with_cluster(l2cache_pmu
, cpu
);
840 /* Only if broken firmware doesn't list every cluster */
841 WARN_ONCE(1, "No L2 cache cluster for CPU%d\n", cpu
);
846 /* If another CPU is managing this cluster, we're done */
847 if (cluster
->on_cpu
!= -1)
851 * All CPUs on this cluster were down, use this one.
852 * Reset to put it into sane state.
854 cluster
->on_cpu
= cpu
;
855 cpumask_set_cpu(cpu
, &l2cache_pmu
->cpumask
);
858 WARN_ON(irq_set_affinity(cluster
->irq
, cpumask_of(cpu
)));
859 enable_irq(cluster
->irq
);
864 static int l2cache_pmu_offline_cpu(unsigned int cpu
, struct hlist_node
*node
)
866 struct cluster_pmu
*cluster
;
867 struct l2cache_pmu
*l2cache_pmu
;
868 cpumask_t cluster_online_cpus
;
871 l2cache_pmu
= hlist_entry_safe(node
, struct l2cache_pmu
, node
);
872 cluster
= get_cluster_pmu(l2cache_pmu
, cpu
);
876 /* If this CPU is not managing the cluster, we're done */
877 if (cluster
->on_cpu
!= cpu
)
880 /* Give up ownership of cluster */
881 cpumask_clear_cpu(cpu
, &l2cache_pmu
->cpumask
);
882 cluster
->on_cpu
= -1;
884 /* Any other CPU for this cluster which is still online */
885 cpumask_and(&cluster_online_cpus
, &cluster
->cluster_cpus
,
887 target
= cpumask_any_but(&cluster_online_cpus
, cpu
);
888 if (target
>= nr_cpu_ids
) {
889 disable_irq(cluster
->irq
);
893 perf_pmu_migrate_context(&l2cache_pmu
->pmu
, cpu
, target
);
894 cluster
->on_cpu
= target
;
895 cpumask_set_cpu(target
, &l2cache_pmu
->cpumask
);
896 WARN_ON(irq_set_affinity(cluster
->irq
, cpumask_of(target
)));
901 static int l2_cache_pmu_probe_cluster(struct device
*dev
, void *data
)
903 struct platform_device
*pdev
= to_platform_device(dev
->parent
);
904 struct platform_device
*sdev
= to_platform_device(dev
);
905 struct l2cache_pmu
*l2cache_pmu
= data
;
906 struct cluster_pmu
*cluster
;
907 struct acpi_device
*device
;
908 unsigned long fw_cluster_id
;
912 if (acpi_bus_get_device(ACPI_HANDLE(dev
), &device
))
915 if (kstrtoul(device
->pnp
.unique_id
, 10, &fw_cluster_id
) < 0) {
916 dev_err(&pdev
->dev
, "unable to read ACPI uid\n");
920 cluster
= devm_kzalloc(&pdev
->dev
, sizeof(*cluster
), GFP_KERNEL
);
924 INIT_LIST_HEAD(&cluster
->next
);
925 list_add(&cluster
->next
, &l2cache_pmu
->clusters
);
926 cluster
->cluster_id
= fw_cluster_id
;
928 irq
= platform_get_irq(sdev
, 0);
931 "Failed to get valid irq for cluster %ld\n",
935 irq_set_status_flags(irq
, IRQ_NOAUTOEN
);
938 cluster
->l2cache_pmu
= l2cache_pmu
;
939 cluster
->on_cpu
= -1;
941 err
= devm_request_irq(&pdev
->dev
, irq
, l2_cache_handle_irq
,
942 IRQF_NOBALANCING
| IRQF_NO_THREAD
,
943 "l2-cache-pmu", cluster
);
946 "Unable to request IRQ%d for L2 PMU counters\n", irq
);
951 "Registered L2 cache PMU cluster %ld\n", fw_cluster_id
);
953 spin_lock_init(&cluster
->pmu_lock
);
955 l2cache_pmu
->num_pmus
++;
960 static int l2_cache_pmu_probe(struct platform_device
*pdev
)
963 struct l2cache_pmu
*l2cache_pmu
;
966 devm_kzalloc(&pdev
->dev
, sizeof(*l2cache_pmu
), GFP_KERNEL
);
970 INIT_LIST_HEAD(&l2cache_pmu
->clusters
);
972 platform_set_drvdata(pdev
, l2cache_pmu
);
973 l2cache_pmu
->pmu
= (struct pmu
) {
974 /* suffix is instance id for future use with multiple sockets */
976 .task_ctx_nr
= perf_invalid_context
,
977 .pmu_enable
= l2_cache_pmu_enable
,
978 .pmu_disable
= l2_cache_pmu_disable
,
979 .event_init
= l2_cache_event_init
,
980 .add
= l2_cache_event_add
,
981 .del
= l2_cache_event_del
,
982 .start
= l2_cache_event_start
,
983 .stop
= l2_cache_event_stop
,
984 .read
= l2_cache_event_read
,
985 .attr_groups
= l2_cache_pmu_attr_grps
,
988 l2cache_pmu
->num_counters
= get_num_counters();
989 l2cache_pmu
->pdev
= pdev
;
990 l2cache_pmu
->pmu_cluster
= devm_alloc_percpu(&pdev
->dev
,
991 struct cluster_pmu
*);
992 if (!l2cache_pmu
->pmu_cluster
)
995 l2_cycle_ctr_idx
= l2cache_pmu
->num_counters
- 1;
996 l2_counter_present_mask
= GENMASK(l2cache_pmu
->num_counters
- 2, 0) |
997 BIT(L2CYCLE_CTR_BIT
);
999 cpumask_clear(&l2cache_pmu
->cpumask
);
1001 /* Read cluster info and initialize each cluster */
1002 err
= device_for_each_child(&pdev
->dev
, l2cache_pmu
,
1003 l2_cache_pmu_probe_cluster
);
1007 if (l2cache_pmu
->num_pmus
== 0) {
1008 dev_err(&pdev
->dev
, "No hardware L2 cache PMUs found\n");
1012 err
= cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE
,
1013 &l2cache_pmu
->node
);
1015 dev_err(&pdev
->dev
, "Error %d registering hotplug", err
);
1019 err
= perf_pmu_register(&l2cache_pmu
->pmu
, l2cache_pmu
->pmu
.name
, -1);
1021 dev_err(&pdev
->dev
, "Error %d registering L2 cache PMU\n", err
);
1022 goto out_unregister
;
1025 dev_info(&pdev
->dev
, "Registered L2 cache PMU using %d HW PMUs\n",
1026 l2cache_pmu
->num_pmus
);
1031 cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE
,
1032 &l2cache_pmu
->node
);
1036 static int l2_cache_pmu_remove(struct platform_device
*pdev
)
1038 struct l2cache_pmu
*l2cache_pmu
=
1039 to_l2cache_pmu(platform_get_drvdata(pdev
));
1041 perf_pmu_unregister(&l2cache_pmu
->pmu
);
1042 cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE
,
1043 &l2cache_pmu
->node
);
1047 static struct platform_driver l2_cache_pmu_driver
= {
1049 .name
= "qcom-l2cache-pmu",
1050 .acpi_match_table
= ACPI_PTR(l2_cache_pmu_acpi_match
),
1052 .probe
= l2_cache_pmu_probe
,
1053 .remove
= l2_cache_pmu_remove
,
1056 static int __init
register_l2_cache_pmu_driver(void)
1060 err
= cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE
,
1061 "AP_PERF_ARM_QCOM_L2_ONLINE",
1062 l2cache_pmu_online_cpu
,
1063 l2cache_pmu_offline_cpu
);
1067 return platform_driver_register(&l2_cache_pmu_driver
);
1069 device_initcall(register_l2_cache_pmu_driver
);