1 /* SPDX-License-Identifier: GPL-2.0-only */
2 #ifndef _LINUX_TRACEPOINT_H
3 #define _LINUX_TRACEPOINT_H
6 * Kernel Tracepoint API.
8 * See Documentation/trace/tracepoints.rst.
10 * Copyright (C) 2008-2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
12 * Heavily inspired from the Linux Kernel Markers.
15 #include <linux/smp.h>
16 #include <linux/srcu.h>
17 #include <linux/errno.h>
18 #include <linux/types.h>
19 #include <linux/rcupdate.h>
20 #include <linux/rcupdate_trace.h>
21 #include <linux/tracepoint-defs.h>
22 #include <linux/static_call.h>
26 struct notifier_block
;
28 struct trace_eval_map
{
30 const char *eval_string
;
31 unsigned long eval_value
;
34 #define TRACEPOINT_DEFAULT_PRIO 10
37 tracepoint_probe_register(struct tracepoint
*tp
, void *probe
, void *data
);
39 tracepoint_probe_register_prio(struct tracepoint
*tp
, void *probe
, void *data
,
42 tracepoint_probe_register_prio_may_exist(struct tracepoint
*tp
, void *probe
, void *data
,
45 tracepoint_probe_unregister(struct tracepoint
*tp
, void *probe
, void *data
);
47 tracepoint_probe_register_may_exist(struct tracepoint
*tp
, void *probe
,
50 return tracepoint_probe_register_prio_may_exist(tp
, probe
, data
,
51 TRACEPOINT_DEFAULT_PRIO
);
54 for_each_kernel_tracepoint(void (*fct
)(struct tracepoint
*tp
, void *priv
),
59 struct list_head list
;
63 bool trace_module_has_bad_taint(struct module
*mod
);
64 extern int register_tracepoint_module_notifier(struct notifier_block
*nb
);
65 extern int unregister_tracepoint_module_notifier(struct notifier_block
*nb
);
66 void for_each_module_tracepoint(void (*fct
)(struct tracepoint
*,
67 struct module
*, void *),
69 void for_each_tracepoint_in_module(struct module
*,
70 void (*fct
)(struct tracepoint
*,
71 struct module
*, void *),
74 static inline bool trace_module_has_bad_taint(struct module
*mod
)
79 int register_tracepoint_module_notifier(struct notifier_block
*nb
)
84 int unregister_tracepoint_module_notifier(struct notifier_block
*nb
)
89 void for_each_module_tracepoint(void (*fct
)(struct tracepoint
*,
90 struct module
*, void *),
95 void for_each_tracepoint_in_module(struct module
*mod
,
96 void (*fct
)(struct tracepoint
*,
97 struct module
*, void *),
101 #endif /* CONFIG_MODULES */
104 * tracepoint_synchronize_unregister must be called between the last tracepoint
105 * probe unregistration and the end of module exit to make sure there is no
106 * caller executing a probe when it is freed.
108 * An alternative is to use the following for batch reclaim associated
109 * with a given tracepoint:
111 * - tracepoint_is_faultable() == false: call_rcu()
112 * - tracepoint_is_faultable() == true: call_rcu_tasks_trace()
114 #ifdef CONFIG_TRACEPOINTS
115 static inline void tracepoint_synchronize_unregister(void)
117 synchronize_rcu_tasks_trace();
120 static inline bool tracepoint_is_faultable(struct tracepoint
*tp
)
122 return tp
->ext
&& tp
->ext
->faultable
;
125 static inline void tracepoint_synchronize_unregister(void)
127 static inline bool tracepoint_is_faultable(struct tracepoint
*tp
)
133 #ifdef CONFIG_HAVE_SYSCALL_TRACEPOINTS
134 extern int syscall_regfunc(void);
135 extern void syscall_unregfunc(void);
136 #endif /* CONFIG_HAVE_SYSCALL_TRACEPOINTS */
139 #define PARAMS(args...) args
142 #define TRACE_DEFINE_ENUM(x)
143 #define TRACE_DEFINE_SIZEOF(x)
145 #ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS
146 static inline struct tracepoint
*tracepoint_ptr_deref(tracepoint_ptr_t
*p
)
148 return offset_to_ptr(p
);
151 #define __TRACEPOINT_ENTRY(name) \
152 asm(" .section \"__tracepoints_ptrs\", \"a\" \n" \
154 " .long __tracepoint_" #name " - . \n" \
157 static inline struct tracepoint
*tracepoint_ptr_deref(tracepoint_ptr_t
*p
)
162 #define __TRACEPOINT_ENTRY(name) \
163 static tracepoint_ptr_t __tracepoint_ptr_##name __used \
164 __section("__tracepoints_ptrs") = &__tracepoint_##name
167 #endif /* _LINUX_TRACEPOINT_H */
170 * Note: we keep the TRACE_EVENT and DECLARE_TRACE outside the include
171 * file ifdef protection.
172 * This is due to the way trace events work. If a file includes two
173 * trace event headers under one "CREATE_TRACE_POINTS" the first include
174 * will override the TRACE_EVENT and break the second include.
177 #ifndef DECLARE_TRACE
179 #define TP_PROTO(args...) args
180 #define TP_ARGS(args...) args
181 #define TP_CONDITION(args...) args
184 * Individual subsystem my have a separate configuration to
185 * enable their tracepoints. By default, this file will create
186 * the tracepoints if CONFIG_TRACEPOINTS is defined. If a subsystem
187 * wants to be able to disable its tracepoints from being created
188 * it can define NOTRACE before including the tracepoint headers.
190 #if defined(CONFIG_TRACEPOINTS) && !defined(NOTRACE)
191 #define TRACEPOINTS_ENABLED
194 #ifdef TRACEPOINTS_ENABLED
196 #ifdef CONFIG_HAVE_STATIC_CALL
197 #define __DO_TRACE_CALL(name, args) \
199 struct tracepoint_func *it_func_ptr; \
202 rcu_dereference_raw((&__tracepoint_##name)->funcs); \
204 __data = (it_func_ptr)->data; \
205 static_call(tp_func_##name)(__data, args); \
209 #define __DO_TRACE_CALL(name, args) __traceiter_##name(NULL, args)
210 #endif /* CONFIG_HAVE_STATIC_CALL */
213 * Make sure the alignment of the structure in the __tracepoints section will
214 * not add unwanted padding between the beginning of the section and the
215 * structure. Force alignment to the same alignment as the section start.
217 * When lockdep is enabled, we make sure to always test if RCU is
218 * "watching" regardless if the tracepoint is enabled or not. Tracepoints
219 * require RCU to be active, and it should always warn at the tracepoint
220 * site if it is not watching, as it will need to be active when the
221 * tracepoint is enabled.
223 #define __DECLARE_TRACE_COMMON(name, proto, args, cond, data_proto) \
224 extern int __traceiter_##name(data_proto); \
225 DECLARE_STATIC_CALL(tp_func_##name, __traceiter_##name); \
226 extern struct tracepoint __tracepoint_##name; \
228 register_trace_##name(void (*probe)(data_proto), void *data) \
230 return tracepoint_probe_register(&__tracepoint_##name, \
231 (void *)probe, data); \
234 register_trace_prio_##name(void (*probe)(data_proto), void *data,\
237 return tracepoint_probe_register_prio(&__tracepoint_##name, \
238 (void *)probe, data, prio); \
241 unregister_trace_##name(void (*probe)(data_proto), void *data) \
243 return tracepoint_probe_unregister(&__tracepoint_##name,\
244 (void *)probe, data); \
247 check_trace_callback_type_##name(void (*cb)(data_proto)) \
251 trace_##name##_enabled(void) \
253 return static_branch_unlikely(&__tracepoint_##name.key);\
256 #define __DECLARE_TRACE(name, proto, args, cond, data_proto) \
257 __DECLARE_TRACE_COMMON(name, PARAMS(proto), PARAMS(args), cond, PARAMS(data_proto)) \
258 static inline void trace_##name(proto) \
260 if (static_branch_unlikely(&__tracepoint_##name.key)) { \
262 scoped_guard(preempt_notrace) \
263 __DO_TRACE_CALL(name, TP_ARGS(args)); \
266 if (IS_ENABLED(CONFIG_LOCKDEP) && (cond)) { \
267 WARN_ONCE(!rcu_is_watching(), \
268 "RCU not watching for tracepoint"); \
272 #define __DECLARE_TRACE_SYSCALL(name, proto, args, cond, data_proto) \
273 __DECLARE_TRACE_COMMON(name, PARAMS(proto), PARAMS(args), cond, PARAMS(data_proto)) \
274 static inline void trace_##name(proto) \
277 if (static_branch_unlikely(&__tracepoint_##name.key)) { \
279 scoped_guard(rcu_tasks_trace) \
280 __DO_TRACE_CALL(name, TP_ARGS(args)); \
283 if (IS_ENABLED(CONFIG_LOCKDEP) && (cond)) { \
284 WARN_ONCE(!rcu_is_watching(), \
285 "RCU not watching for tracepoint"); \
290 * We have no guarantee that gcc and the linker won't up-align the tracepoint
291 * structures, so we create an array of pointers that will be used for iteration
292 * on the tracepoints.
294 * it_func[0] is never NULL because there is at least one element in the array
295 * when the array itself is non NULL.
297 #define __DEFINE_TRACE_EXT(_name, _ext, proto, args) \
298 static const char __tpstrtab_##_name[] \
299 __section("__tracepoints_strings") = #_name; \
300 extern struct static_call_key STATIC_CALL_KEY(tp_func_##_name); \
301 int __traceiter_##_name(void *__data, proto); \
302 void __probestub_##_name(void *__data, proto); \
303 struct tracepoint __tracepoint_##_name __used \
304 __section("__tracepoints") = { \
305 .name = __tpstrtab_##_name, \
306 .key = STATIC_KEY_FALSE_INIT, \
307 .static_call_key = &STATIC_CALL_KEY(tp_func_##_name), \
308 .static_call_tramp = STATIC_CALL_TRAMP_ADDR(tp_func_##_name), \
309 .iterator = &__traceiter_##_name, \
310 .probestub = &__probestub_##_name, \
314 __TRACEPOINT_ENTRY(_name); \
315 int __traceiter_##_name(void *__data, proto) \
317 struct tracepoint_func *it_func_ptr; \
321 rcu_dereference_raw((&__tracepoint_##_name)->funcs); \
324 it_func = READ_ONCE((it_func_ptr)->func); \
325 __data = (it_func_ptr)->data; \
326 ((void(*)(void *, proto))(it_func))(__data, args); \
327 } while ((++it_func_ptr)->func); \
331 void __probestub_##_name(void *__data, proto) \
334 DEFINE_STATIC_CALL(tp_func_##_name, __traceiter_##_name);
336 #define DEFINE_TRACE_FN(_name, _reg, _unreg, _proto, _args) \
337 static struct tracepoint_ext __tracepoint_ext_##_name = { \
339 .unregfunc = _unreg, \
340 .faultable = false, \
342 __DEFINE_TRACE_EXT(_name, &__tracepoint_ext_##_name, PARAMS(_proto), PARAMS(_args));
344 #define DEFINE_TRACE_SYSCALL(_name, _reg, _unreg, _proto, _args) \
345 static struct tracepoint_ext __tracepoint_ext_##_name = { \
347 .unregfunc = _unreg, \
350 __DEFINE_TRACE_EXT(_name, &__tracepoint_ext_##_name, PARAMS(_proto), PARAMS(_args));
352 #define DEFINE_TRACE(_name, _proto, _args) \
353 __DEFINE_TRACE_EXT(_name, NULL, PARAMS(_proto), PARAMS(_args));
355 #define EXPORT_TRACEPOINT_SYMBOL_GPL(name) \
356 EXPORT_SYMBOL_GPL(__tracepoint_##name); \
357 EXPORT_SYMBOL_GPL(__traceiter_##name); \
358 EXPORT_STATIC_CALL_GPL(tp_func_##name)
359 #define EXPORT_TRACEPOINT_SYMBOL(name) \
360 EXPORT_SYMBOL(__tracepoint_##name); \
361 EXPORT_SYMBOL(__traceiter_##name); \
362 EXPORT_STATIC_CALL(tp_func_##name)
365 #else /* !TRACEPOINTS_ENABLED */
366 #define __DECLARE_TRACE(name, proto, args, cond, data_proto) \
367 static inline void trace_##name(proto) \
370 register_trace_##name(void (*probe)(data_proto), \
376 unregister_trace_##name(void (*probe)(data_proto), \
381 static inline void check_trace_callback_type_##name(void (*cb)(data_proto)) \
385 trace_##name##_enabled(void) \
390 #define __DECLARE_TRACE_SYSCALL __DECLARE_TRACE
392 #define DEFINE_TRACE_FN(name, reg, unreg, proto, args)
393 #define DEFINE_TRACE_SYSCALL(name, reg, unreg, proto, args)
394 #define DEFINE_TRACE(name, proto, args)
395 #define EXPORT_TRACEPOINT_SYMBOL_GPL(name)
396 #define EXPORT_TRACEPOINT_SYMBOL(name)
398 #endif /* TRACEPOINTS_ENABLED */
400 #ifdef CONFIG_TRACING
402 * tracepoint_string - register constant persistent string to trace system
403 * @str - a constant persistent string that will be referenced in tracepoints
405 * If constant strings are being used in tracepoints, it is faster and
406 * more efficient to just save the pointer to the string and reference
407 * that with a printf "%s" instead of saving the string in the ring buffer
408 * and wasting space and time.
410 * The problem with the above approach is that userspace tools that read
411 * the binary output of the trace buffers do not have access to the string.
412 * Instead they just show the address of the string which is not very
415 * With tracepoint_string(), the string will be registered to the tracing
416 * system and exported to userspace via the debugfs/tracing/printk_formats
417 * file that maps the string address to the string text. This way userspace
418 * tools that read the binary buffers have a way to map the pointers to
419 * the ASCII strings they represent.
421 * The @str used must be a constant string and persistent as it would not
422 * make sense to show a string that no longer exists. But it is still fine
423 * to be used with modules, because when modules are unloaded, if they
424 * had tracepoints, the ring buffers are cleared too. As long as the string
425 * does not change during the life of the module, it is fine to use
426 * tracepoint_string() within a module.
428 #define tracepoint_string(str) \
430 static const char *___tp_str __tracepoint_string = str; \
433 #define __tracepoint_string __used __section("__tracepoint_str")
436 * tracepoint_string() is used to save the string address for userspace
437 * tracing tools. When tracing isn't configured, there's no need to save
440 # define tracepoint_string(str) str
441 # define __tracepoint_string
444 #define DECLARE_TRACE(name, proto, args) \
445 __DECLARE_TRACE(name, PARAMS(proto), PARAMS(args), \
446 cpu_online(raw_smp_processor_id()), \
447 PARAMS(void *__data, proto))
449 #define DECLARE_TRACE_CONDITION(name, proto, args, cond) \
450 __DECLARE_TRACE(name, PARAMS(proto), PARAMS(args), \
451 cpu_online(raw_smp_processor_id()) && (PARAMS(cond)), \
452 PARAMS(void *__data, proto))
454 #define DECLARE_TRACE_SYSCALL(name, proto, args) \
455 __DECLARE_TRACE_SYSCALL(name, PARAMS(proto), PARAMS(args), \
456 cpu_online(raw_smp_processor_id()), \
457 PARAMS(void *__data, proto))
459 #define TRACE_EVENT_FLAGS(event, flag)
461 #define TRACE_EVENT_PERF_PERM(event, expr...)
463 #endif /* DECLARE_TRACE */
467 * For use with the TRACE_EVENT macro:
469 * We define a tracepoint, its arguments, its printk format
470 * and its 'fast binary record' layout.
472 * Firstly, name your tracepoint via TRACE_EVENT(name : the
473 * 'subsystem_event' notation is fine.
475 * Think about this whole construct as the
476 * 'trace_sched_switch() function' from now on.
479 * TRACE_EVENT(sched_switch,
482 * * A function has a regular function arguments
483 * * prototype, declare it via TP_PROTO():
486 * TP_PROTO(struct rq *rq, struct task_struct *prev,
487 * struct task_struct *next),
490 * * Define the call signature of the 'function'.
491 * * (Design sidenote: we use this instead of a
492 * * TP_PROTO1/TP_PROTO2/TP_PROTO3 ugliness.)
495 * TP_ARGS(rq, prev, next),
498 * * Fast binary tracing: define the trace record via
499 * * TP_STRUCT__entry(). You can think about it like a
500 * * regular C structure local variable definition.
502 * * This is how the trace record is structured and will
503 * * be saved into the ring buffer. These are the fields
504 * * that will be exposed to user-space in
505 * * /sys/kernel/tracing/events/<*>/format.
507 * * The declared 'local variable' is called '__entry'
509 * * __field(pid_t, prev_pid) is equivalent to a standard declaration:
513 * * __array(char, prev_comm, TASK_COMM_LEN) is equivalent to:
515 * * char prev_comm[TASK_COMM_LEN];
519 * __array( char, prev_comm, TASK_COMM_LEN )
520 * __field( pid_t, prev_pid )
521 * __field( int, prev_prio )
522 * __array( char, next_comm, TASK_COMM_LEN )
523 * __field( pid_t, next_pid )
524 * __field( int, next_prio )
528 * * Assign the entry into the trace record, by embedding
529 * * a full C statement block into TP_fast_assign(). You
530 * * can refer to the trace record as '__entry' -
531 * * otherwise you can put arbitrary C code in here.
533 * * Note: this C code will execute every time a trace event
534 * * happens, on an active tracepoint.
538 * memcpy(__entry->next_comm, next->comm, TASK_COMM_LEN);
539 * __entry->prev_pid = prev->pid;
540 * __entry->prev_prio = prev->prio;
541 * memcpy(__entry->prev_comm, prev->comm, TASK_COMM_LEN);
542 * __entry->next_pid = next->pid;
543 * __entry->next_prio = next->prio;
547 * * Formatted output of a trace record via TP_printk().
548 * * This is how the tracepoint will appear under ftrace
549 * * plugins that make use of this tracepoint.
551 * * (raw-binary tracing wont actually perform this step.)
554 * TP_printk("task %s:%d [%d] ==> %s:%d [%d]",
555 * __entry->prev_comm, __entry->prev_pid, __entry->prev_prio,
556 * __entry->next_comm, __entry->next_pid, __entry->next_prio),
560 * This macro construct is thus used for the regular printk format
561 * tracing setup, it is used to construct a function pointer based
562 * tracepoint callback (this is used by programmatic plugins and
563 * can also by used by generic instrumentation like SystemTap), and
564 * it is also used to expose a structured trace record in
565 * /sys/kernel/tracing/events/.
567 * A set of (un)registration functions can be passed to the variant
568 * TRACE_EVENT_FN to perform any (un)registration work.
571 #define DECLARE_EVENT_CLASS(name, proto, args, tstruct, assign, print)
572 #define DEFINE_EVENT(template, name, proto, args) \
573 DECLARE_TRACE(name, PARAMS(proto), PARAMS(args))
574 #define DEFINE_EVENT_FN(template, name, proto, args, reg, unreg)\
575 DECLARE_TRACE(name, PARAMS(proto), PARAMS(args))
576 #define DEFINE_EVENT_PRINT(template, name, proto, args, print) \
577 DECLARE_TRACE(name, PARAMS(proto), PARAMS(args))
578 #define DEFINE_EVENT_CONDITION(template, name, proto, \
580 DECLARE_TRACE_CONDITION(name, PARAMS(proto), \
581 PARAMS(args), PARAMS(cond))
583 #define TRACE_EVENT(name, proto, args, struct, assign, print) \
584 DECLARE_TRACE(name, PARAMS(proto), PARAMS(args))
585 #define TRACE_EVENT_FN(name, proto, args, struct, \
586 assign, print, reg, unreg) \
587 DECLARE_TRACE(name, PARAMS(proto), PARAMS(args))
588 #define TRACE_EVENT_FN_COND(name, proto, args, cond, struct, \
589 assign, print, reg, unreg) \
590 DECLARE_TRACE_CONDITION(name, PARAMS(proto), \
591 PARAMS(args), PARAMS(cond))
592 #define TRACE_EVENT_CONDITION(name, proto, args, cond, \
593 struct, assign, print) \
594 DECLARE_TRACE_CONDITION(name, PARAMS(proto), \
595 PARAMS(args), PARAMS(cond))
596 #define TRACE_EVENT_SYSCALL(name, proto, args, struct, assign, \
598 DECLARE_TRACE_SYSCALL(name, PARAMS(proto), PARAMS(args))
600 #define TRACE_EVENT_FLAGS(event, flag)
602 #define TRACE_EVENT_PERF_PERM(event, expr...)
604 #define DECLARE_EVENT_NOP(name, proto, args) \
605 static inline void trace_##name(proto) \
607 static inline bool trace_##name##_enabled(void) \
612 #define TRACE_EVENT_NOP(name, proto, args, struct, assign, print) \
613 DECLARE_EVENT_NOP(name, PARAMS(proto), PARAMS(args))
615 #define DECLARE_EVENT_CLASS_NOP(name, proto, args, tstruct, assign, print)
616 #define DEFINE_EVENT_NOP(template, name, proto, args) \
617 DECLARE_EVENT_NOP(name, PARAMS(proto), PARAMS(args))
619 #endif /* ifdef TRACE_EVENT (see note above) */