1 // SPDX-License-Identifier: GPL-2.0
3 * Performance events callchain code, extracted from core.c:
5 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
6 * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
7 * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
8 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
11 #include <linux/perf_event.h>
12 #include <linux/slab.h>
13 #include <linux/sched/task_stack.h>
17 struct callchain_cpus_entries
{
18 struct rcu_head rcu_head
;
19 struct perf_callchain_entry
*cpu_entries
[];
22 int sysctl_perf_event_max_stack __read_mostly
= PERF_MAX_STACK_DEPTH
;
23 int sysctl_perf_event_max_contexts_per_stack __read_mostly
= PERF_MAX_CONTEXTS_PER_STACK
;
25 static inline size_t perf_callchain_entry__sizeof(void)
27 return (sizeof(struct perf_callchain_entry
) +
28 sizeof(__u64
) * (sysctl_perf_event_max_stack
+
29 sysctl_perf_event_max_contexts_per_stack
));
32 static DEFINE_PER_CPU(int, callchain_recursion
[PERF_NR_CONTEXTS
]);
33 static atomic_t nr_callchain_events
;
34 static DEFINE_MUTEX(callchain_mutex
);
35 static struct callchain_cpus_entries
*callchain_cpus_entries
;
38 __weak
void perf_callchain_kernel(struct perf_callchain_entry_ctx
*entry
,
43 __weak
void perf_callchain_user(struct perf_callchain_entry_ctx
*entry
,
48 static void release_callchain_buffers_rcu(struct rcu_head
*head
)
50 struct callchain_cpus_entries
*entries
;
53 entries
= container_of(head
, struct callchain_cpus_entries
, rcu_head
);
55 for_each_possible_cpu(cpu
)
56 kfree(entries
->cpu_entries
[cpu
]);
61 static void release_callchain_buffers(void)
63 struct callchain_cpus_entries
*entries
;
65 entries
= callchain_cpus_entries
;
66 RCU_INIT_POINTER(callchain_cpus_entries
, NULL
);
67 call_rcu(&entries
->rcu_head
, release_callchain_buffers_rcu
);
70 static int alloc_callchain_buffers(void)
74 struct callchain_cpus_entries
*entries
;
77 * We can't use the percpu allocation API for data that can be
78 * accessed from NMI. Use a temporary manual per cpu allocation
79 * until that gets sorted out.
81 size
= offsetof(struct callchain_cpus_entries
, cpu_entries
[nr_cpu_ids
]);
83 entries
= kzalloc(size
, GFP_KERNEL
);
87 size
= perf_callchain_entry__sizeof() * PERF_NR_CONTEXTS
;
89 for_each_possible_cpu(cpu
) {
90 entries
->cpu_entries
[cpu
] = kmalloc_node(size
, GFP_KERNEL
,
92 if (!entries
->cpu_entries
[cpu
])
96 rcu_assign_pointer(callchain_cpus_entries
, entries
);
101 for_each_possible_cpu(cpu
)
102 kfree(entries
->cpu_entries
[cpu
]);
108 int get_callchain_buffers(int event_max_stack
)
113 mutex_lock(&callchain_mutex
);
115 count
= atomic_inc_return(&nr_callchain_events
);
116 if (WARN_ON_ONCE(count
< 1)) {
122 * If requesting per event more than the global cap,
123 * return a different error to help userspace figure
126 * And also do it here so that we have &callchain_mutex held.
128 if (event_max_stack
> sysctl_perf_event_max_stack
) {
134 err
= alloc_callchain_buffers();
137 atomic_dec(&nr_callchain_events
);
139 mutex_unlock(&callchain_mutex
);
144 void put_callchain_buffers(void)
146 if (atomic_dec_and_mutex_lock(&nr_callchain_events
, &callchain_mutex
)) {
147 release_callchain_buffers();
148 mutex_unlock(&callchain_mutex
);
152 struct perf_callchain_entry
*get_callchain_entry(int *rctx
)
155 struct callchain_cpus_entries
*entries
;
157 *rctx
= get_recursion_context(this_cpu_ptr(callchain_recursion
));
161 entries
= rcu_dereference(callchain_cpus_entries
);
163 put_recursion_context(this_cpu_ptr(callchain_recursion
), *rctx
);
167 cpu
= smp_processor_id();
169 return (((void *)entries
->cpu_entries
[cpu
]) +
170 (*rctx
* perf_callchain_entry__sizeof()));
174 put_callchain_entry(int rctx
)
176 put_recursion_context(this_cpu_ptr(callchain_recursion
), rctx
);
179 struct perf_callchain_entry
*
180 get_perf_callchain(struct pt_regs
*regs
, u32 init_nr
, bool kernel
, bool user
,
181 u32 max_stack
, bool crosstask
, bool add_mark
)
183 struct perf_callchain_entry
*entry
;
184 struct perf_callchain_entry_ctx ctx
;
187 entry
= get_callchain_entry(&rctx
);
192 ctx
.max_stack
= max_stack
;
193 ctx
.nr
= entry
->nr
= init_nr
;
195 ctx
.contexts_maxed
= false;
197 if (kernel
&& !user_mode(regs
)) {
199 perf_callchain_store_context(&ctx
, PERF_CONTEXT_KERNEL
);
200 perf_callchain_kernel(&ctx
, regs
);
204 if (!user_mode(regs
)) {
206 regs
= task_pt_regs(current
);
218 perf_callchain_store_context(&ctx
, PERF_CONTEXT_USER
);
220 fs
= force_uaccess_begin();
221 perf_callchain_user(&ctx
, regs
);
222 force_uaccess_end(fs
);
227 put_callchain_entry(rctx
);
233 * Used for sysctl_perf_event_max_stack and
234 * sysctl_perf_event_max_contexts_per_stack.
236 int perf_event_max_stack_handler(struct ctl_table
*table
, int write
,
237 void *buffer
, size_t *lenp
, loff_t
*ppos
)
239 int *value
= table
->data
;
240 int new_value
= *value
, ret
;
241 struct ctl_table new_table
= *table
;
243 new_table
.data
= &new_value
;
244 ret
= proc_dointvec_minmax(&new_table
, write
, buffer
, lenp
, ppos
);
248 mutex_lock(&callchain_mutex
);
249 if (atomic_read(&nr_callchain_events
))
254 mutex_unlock(&callchain_mutex
);