2 * Kprobes-based tracing events
4 * Created by Masami Hiramatsu <mhiramat@redhat.com>
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 #include <linux/module.h>
21 #include <linux/uaccess.h>
23 #include "trace_probe.h"
25 #define KPROBE_EVENT_SYSTEM "kprobes"
28 * Kprobe event core functions
31 struct list_head list
;
32 struct kretprobe rp
; /* Use rp.kp for kprobe use */
34 unsigned int flags
; /* For TP_FLAG_* */
35 const char *symbol
; /* symbol name */
36 struct ftrace_event_class
class;
37 struct ftrace_event_call call
;
38 struct list_head files
;
39 ssize_t size
; /* trace entry size */
41 struct probe_arg args
[];
44 struct event_file_link
{
45 struct ftrace_event_file
*file
;
46 struct list_head list
;
49 #define SIZEOF_TRACE_PROBE(n) \
50 (offsetof(struct trace_probe, args) + \
51 (sizeof(struct probe_arg) * (n)))
54 static __kprobes
bool trace_probe_is_return(struct trace_probe
*tp
)
56 return tp
->rp
.handler
!= NULL
;
59 static __kprobes
const char *trace_probe_symbol(struct trace_probe
*tp
)
61 return tp
->symbol
? tp
->symbol
: "unknown";
64 static __kprobes
unsigned long trace_probe_offset(struct trace_probe
*tp
)
66 return tp
->rp
.kp
.offset
;
69 static __kprobes
bool trace_probe_is_enabled(struct trace_probe
*tp
)
71 return !!(tp
->flags
& (TP_FLAG_TRACE
| TP_FLAG_PROFILE
));
74 static __kprobes
bool trace_probe_is_registered(struct trace_probe
*tp
)
76 return !!(tp
->flags
& TP_FLAG_REGISTERED
);
79 static __kprobes
bool trace_probe_has_gone(struct trace_probe
*tp
)
81 return !!(kprobe_gone(&tp
->rp
.kp
));
84 static __kprobes
bool trace_probe_within_module(struct trace_probe
*tp
,
87 int len
= strlen(mod
->name
);
88 const char *name
= trace_probe_symbol(tp
);
89 return strncmp(mod
->name
, name
, len
) == 0 && name
[len
] == ':';
92 static __kprobes
bool trace_probe_is_on_module(struct trace_probe
*tp
)
94 return !!strchr(trace_probe_symbol(tp
), ':');
97 static int register_probe_event(struct trace_probe
*tp
);
98 static int unregister_probe_event(struct trace_probe
*tp
);
100 static DEFINE_MUTEX(probe_lock
);
101 static LIST_HEAD(probe_list
);
103 static int kprobe_dispatcher(struct kprobe
*kp
, struct pt_regs
*regs
);
104 static int kretprobe_dispatcher(struct kretprobe_instance
*ri
,
105 struct pt_regs
*regs
);
108 * Allocate new trace_probe and initialize it (including kprobes).
110 static struct trace_probe
*alloc_trace_probe(const char *group
,
115 int nargs
, bool is_return
)
117 struct trace_probe
*tp
;
120 tp
= kzalloc(SIZEOF_TRACE_PROBE(nargs
), GFP_KERNEL
);
125 tp
->symbol
= kstrdup(symbol
, GFP_KERNEL
);
128 tp
->rp
.kp
.symbol_name
= tp
->symbol
;
129 tp
->rp
.kp
.offset
= offs
;
131 tp
->rp
.kp
.addr
= addr
;
134 tp
->rp
.handler
= kretprobe_dispatcher
;
136 tp
->rp
.kp
.pre_handler
= kprobe_dispatcher
;
138 if (!event
|| !is_good_name(event
)) {
143 tp
->call
.class = &tp
->class;
144 tp
->call
.name
= kstrdup(event
, GFP_KERNEL
);
148 if (!group
|| !is_good_name(group
)) {
153 tp
->class.system
= kstrdup(group
, GFP_KERNEL
);
154 if (!tp
->class.system
)
157 INIT_LIST_HEAD(&tp
->list
);
158 INIT_LIST_HEAD(&tp
->files
);
161 kfree(tp
->call
.name
);
167 static void free_trace_probe(struct trace_probe
*tp
)
171 for (i
= 0; i
< tp
->nr_args
; i
++)
172 traceprobe_free_probe_arg(&tp
->args
[i
]);
174 kfree(tp
->call
.class->system
);
175 kfree(tp
->call
.name
);
180 static struct trace_probe
*find_trace_probe(const char *event
,
183 struct trace_probe
*tp
;
185 list_for_each_entry(tp
, &probe_list
, list
)
186 if (strcmp(tp
->call
.name
, event
) == 0 &&
187 strcmp(tp
->call
.class->system
, group
) == 0)
194 * if the file is NULL, enable "perf" handler, or enable "trace" handler.
197 enable_trace_probe(struct trace_probe
*tp
, struct ftrace_event_file
*file
)
202 struct event_file_link
*link
;
204 link
= kmalloc(sizeof(*link
), GFP_KERNEL
);
211 list_add_tail_rcu(&link
->list
, &tp
->files
);
213 tp
->flags
|= TP_FLAG_TRACE
;
215 tp
->flags
|= TP_FLAG_PROFILE
;
217 if (trace_probe_is_registered(tp
) && !trace_probe_has_gone(tp
)) {
218 if (trace_probe_is_return(tp
))
219 ret
= enable_kretprobe(&tp
->rp
);
221 ret
= enable_kprobe(&tp
->rp
.kp
);
227 static struct event_file_link
*
228 find_event_file_link(struct trace_probe
*tp
, struct ftrace_event_file
*file
)
230 struct event_file_link
*link
;
232 list_for_each_entry(link
, &tp
->files
, list
)
233 if (link
->file
== file
)
240 * Disable trace_probe
241 * if the file is NULL, disable "perf" handler, or disable "trace" handler.
244 disable_trace_probe(struct trace_probe
*tp
, struct ftrace_event_file
*file
)
246 struct event_file_link
*link
= NULL
;
251 link
= find_event_file_link(tp
, file
);
257 list_del_rcu(&link
->list
);
259 if (!list_empty(&tp
->files
))
262 tp
->flags
&= ~TP_FLAG_TRACE
;
264 tp
->flags
&= ~TP_FLAG_PROFILE
;
266 if (!trace_probe_is_enabled(tp
) && trace_probe_is_registered(tp
)) {
267 if (trace_probe_is_return(tp
))
268 disable_kretprobe(&tp
->rp
);
270 disable_kprobe(&tp
->rp
.kp
);
276 * Synchronize with kprobe_trace_func/kretprobe_trace_func
277 * to ensure disabled (all running handlers are finished).
278 * This is not only for kfree(), but also the caller,
279 * trace_remove_event_call() supposes it for releasing
280 * event_call related objects, which will be accessed in
281 * the kprobe_trace_func/kretprobe_trace_func.
284 kfree(link
); /* Ignored if link == NULL */
290 /* Internal register function - just handle k*probes and flags */
291 static int __register_trace_probe(struct trace_probe
*tp
)
295 if (trace_probe_is_registered(tp
))
298 for (i
= 0; i
< tp
->nr_args
; i
++)
299 traceprobe_update_arg(&tp
->args
[i
]);
301 /* Set/clear disabled flag according to tp->flag */
302 if (trace_probe_is_enabled(tp
))
303 tp
->rp
.kp
.flags
&= ~KPROBE_FLAG_DISABLED
;
305 tp
->rp
.kp
.flags
|= KPROBE_FLAG_DISABLED
;
307 if (trace_probe_is_return(tp
))
308 ret
= register_kretprobe(&tp
->rp
);
310 ret
= register_kprobe(&tp
->rp
.kp
);
313 tp
->flags
|= TP_FLAG_REGISTERED
;
315 pr_warning("Could not insert probe at %s+%lu: %d\n",
316 trace_probe_symbol(tp
), trace_probe_offset(tp
), ret
);
317 if (ret
== -ENOENT
&& trace_probe_is_on_module(tp
)) {
318 pr_warning("This probe might be able to register after"
319 "target module is loaded. Continue.\n");
321 } else if (ret
== -EILSEQ
) {
322 pr_warning("Probing address(0x%p) is not an "
323 "instruction boundary.\n",
332 /* Internal unregister function - just handle k*probes and flags */
333 static void __unregister_trace_probe(struct trace_probe
*tp
)
335 if (trace_probe_is_registered(tp
)) {
336 if (trace_probe_is_return(tp
))
337 unregister_kretprobe(&tp
->rp
);
339 unregister_kprobe(&tp
->rp
.kp
);
340 tp
->flags
&= ~TP_FLAG_REGISTERED
;
341 /* Cleanup kprobe for reuse */
342 if (tp
->rp
.kp
.symbol_name
)
343 tp
->rp
.kp
.addr
= NULL
;
347 /* Unregister a trace_probe and probe_event: call with locking probe_lock */
348 static int unregister_trace_probe(struct trace_probe
*tp
)
350 /* Enabled event can not be unregistered */
351 if (trace_probe_is_enabled(tp
))
354 /* Will fail if probe is being used by ftrace or perf */
355 if (unregister_probe_event(tp
))
358 __unregister_trace_probe(tp
);
364 /* Register a trace_probe and probe_event */
365 static int register_trace_probe(struct trace_probe
*tp
)
367 struct trace_probe
*old_tp
;
370 mutex_lock(&probe_lock
);
372 /* Delete old (same name) event if exist */
373 old_tp
= find_trace_probe(tp
->call
.name
, tp
->call
.class->system
);
375 ret
= unregister_trace_probe(old_tp
);
378 free_trace_probe(old_tp
);
381 /* Register new event */
382 ret
= register_probe_event(tp
);
384 pr_warning("Failed to register probe event(%d)\n", ret
);
388 /* Register k*probe */
389 ret
= __register_trace_probe(tp
);
391 unregister_probe_event(tp
);
393 list_add_tail(&tp
->list
, &probe_list
);
396 mutex_unlock(&probe_lock
);
400 /* Module notifier call back, checking event on the module */
401 static int trace_probe_module_callback(struct notifier_block
*nb
,
402 unsigned long val
, void *data
)
404 struct module
*mod
= data
;
405 struct trace_probe
*tp
;
408 if (val
!= MODULE_STATE_COMING
)
411 /* Update probes on coming module */
412 mutex_lock(&probe_lock
);
413 list_for_each_entry(tp
, &probe_list
, list
) {
414 if (trace_probe_within_module(tp
, mod
)) {
415 /* Don't need to check busy - this should have gone. */
416 __unregister_trace_probe(tp
);
417 ret
= __register_trace_probe(tp
);
419 pr_warning("Failed to re-register probe %s on"
421 tp
->call
.name
, mod
->name
, ret
);
424 mutex_unlock(&probe_lock
);
429 static struct notifier_block trace_probe_module_nb
= {
430 .notifier_call
= trace_probe_module_callback
,
431 .priority
= 1 /* Invoked after kprobe module callback */
434 static int create_trace_probe(int argc
, char **argv
)
438 * - Add kprobe: p[:[GRP/]EVENT] [MOD:]KSYM[+OFFS]|KADDR [FETCHARGS]
439 * - Add kretprobe: r[:[GRP/]EVENT] [MOD:]KSYM[+0] [FETCHARGS]
441 * $retval : fetch return value
442 * $stack : fetch stack address
443 * $stackN : fetch Nth of stack (N:0-)
444 * @ADDR : fetch memory at ADDR (ADDR should be in kernel)
445 * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol)
446 * %REG : fetch register REG
447 * Dereferencing memory fetch:
448 * +|-offs(ARG) : fetch memory at ARG +|- offs address.
449 * Alias name of args:
450 * NAME=FETCHARG : set NAME as alias of FETCHARG.
452 * FETCHARG:TYPE : use TYPE instead of unsigned long.
454 struct trace_probe
*tp
;
456 bool is_return
= false, is_delete
= false;
457 char *symbol
= NULL
, *event
= NULL
, *group
= NULL
;
459 unsigned long offset
= 0;
461 char buf
[MAX_EVENT_NAME_LEN
];
463 /* argc must be >= 1 */
464 if (argv
[0][0] == 'p')
466 else if (argv
[0][0] == 'r')
468 else if (argv
[0][0] == '-')
471 pr_info("Probe definition must be started with 'p', 'r' or"
476 if (argv
[0][1] == ':') {
478 if (strchr(event
, '/')) {
480 event
= strchr(group
, '/') + 1;
482 if (strlen(group
) == 0) {
483 pr_info("Group name is not specified\n");
487 if (strlen(event
) == 0) {
488 pr_info("Event name is not specified\n");
493 group
= KPROBE_EVENT_SYSTEM
;
497 pr_info("Delete command needs an event name.\n");
500 mutex_lock(&probe_lock
);
501 tp
= find_trace_probe(event
, group
);
503 mutex_unlock(&probe_lock
);
504 pr_info("Event %s/%s doesn't exist.\n", group
, event
);
507 /* delete an event */
508 ret
= unregister_trace_probe(tp
);
510 free_trace_probe(tp
);
511 mutex_unlock(&probe_lock
);
516 pr_info("Probe point is not specified.\n");
519 if (isdigit(argv
[1][0])) {
521 pr_info("Return probe point must be a symbol.\n");
524 /* an address specified */
525 ret
= kstrtoul(&argv
[1][0], 0, (unsigned long *)&addr
);
527 pr_info("Failed to parse address.\n");
531 /* a symbol specified */
533 /* TODO: support .init module functions */
534 ret
= traceprobe_split_symbol_offset(symbol
, &offset
);
536 pr_info("Failed to parse symbol.\n");
539 if (offset
&& is_return
) {
540 pr_info("Return probe must be used without offset.\n");
544 argc
-= 2; argv
+= 2;
548 /* Make a new event name */
550 snprintf(buf
, MAX_EVENT_NAME_LEN
, "%c_%s_%ld",
551 is_return
? 'r' : 'p', symbol
, offset
);
553 snprintf(buf
, MAX_EVENT_NAME_LEN
, "%c_0x%p",
554 is_return
? 'r' : 'p', addr
);
557 tp
= alloc_trace_probe(group
, event
, addr
, symbol
, offset
, argc
,
560 pr_info("Failed to allocate trace_probe.(%d)\n",
565 /* parse arguments */
567 for (i
= 0; i
< argc
&& i
< MAX_TRACE_ARGS
; i
++) {
568 /* Increment count for freeing args in error case */
571 /* Parse argument name */
572 arg
= strchr(argv
[i
], '=');
575 tp
->args
[i
].name
= kstrdup(argv
[i
], GFP_KERNEL
);
578 /* If argument name is omitted, set "argN" */
579 snprintf(buf
, MAX_EVENT_NAME_LEN
, "arg%d", i
+ 1);
580 tp
->args
[i
].name
= kstrdup(buf
, GFP_KERNEL
);
583 if (!tp
->args
[i
].name
) {
584 pr_info("Failed to allocate argument[%d] name.\n", i
);
589 if (!is_good_name(tp
->args
[i
].name
)) {
590 pr_info("Invalid argument[%d] name: %s\n",
591 i
, tp
->args
[i
].name
);
596 if (traceprobe_conflict_field_name(tp
->args
[i
].name
,
598 pr_info("Argument[%d] name '%s' conflicts with "
599 "another field.\n", i
, argv
[i
]);
604 /* Parse fetch argument */
605 ret
= traceprobe_parse_probe_arg(arg
, &tp
->size
, &tp
->args
[i
],
608 pr_info("Parse error at argument[%d]. (%d)\n", i
, ret
);
613 ret
= register_trace_probe(tp
);
619 free_trace_probe(tp
);
623 static int release_all_trace_probes(void)
625 struct trace_probe
*tp
;
628 mutex_lock(&probe_lock
);
629 /* Ensure no probe is in use. */
630 list_for_each_entry(tp
, &probe_list
, list
)
631 if (trace_probe_is_enabled(tp
)) {
635 /* TODO: Use batch unregistration */
636 while (!list_empty(&probe_list
)) {
637 tp
= list_entry(probe_list
.next
, struct trace_probe
, list
);
638 ret
= unregister_trace_probe(tp
);
641 free_trace_probe(tp
);
645 mutex_unlock(&probe_lock
);
650 /* Probes listing interfaces */
651 static void *probes_seq_start(struct seq_file
*m
, loff_t
*pos
)
653 mutex_lock(&probe_lock
);
654 return seq_list_start(&probe_list
, *pos
);
657 static void *probes_seq_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
659 return seq_list_next(v
, &probe_list
, pos
);
662 static void probes_seq_stop(struct seq_file
*m
, void *v
)
664 mutex_unlock(&probe_lock
);
667 static int probes_seq_show(struct seq_file
*m
, void *v
)
669 struct trace_probe
*tp
= v
;
672 seq_printf(m
, "%c", trace_probe_is_return(tp
) ? 'r' : 'p');
673 seq_printf(m
, ":%s/%s", tp
->call
.class->system
, tp
->call
.name
);
676 seq_printf(m
, " 0x%p", tp
->rp
.kp
.addr
);
677 else if (tp
->rp
.kp
.offset
)
678 seq_printf(m
, " %s+%u", trace_probe_symbol(tp
),
681 seq_printf(m
, " %s", trace_probe_symbol(tp
));
683 for (i
= 0; i
< tp
->nr_args
; i
++)
684 seq_printf(m
, " %s=%s", tp
->args
[i
].name
, tp
->args
[i
].comm
);
690 static const struct seq_operations probes_seq_op
= {
691 .start
= probes_seq_start
,
692 .next
= probes_seq_next
,
693 .stop
= probes_seq_stop
,
694 .show
= probes_seq_show
697 static int probes_open(struct inode
*inode
, struct file
*file
)
701 if ((file
->f_mode
& FMODE_WRITE
) && (file
->f_flags
& O_TRUNC
)) {
702 ret
= release_all_trace_probes();
707 return seq_open(file
, &probes_seq_op
);
710 static ssize_t
probes_write(struct file
*file
, const char __user
*buffer
,
711 size_t count
, loff_t
*ppos
)
713 return traceprobe_probes_write(file
, buffer
, count
, ppos
,
717 static const struct file_operations kprobe_events_ops
= {
718 .owner
= THIS_MODULE
,
722 .release
= seq_release
,
723 .write
= probes_write
,
726 /* Probes profiling interfaces */
727 static int probes_profile_seq_show(struct seq_file
*m
, void *v
)
729 struct trace_probe
*tp
= v
;
731 seq_printf(m
, " %-44s %15lu %15lu\n", tp
->call
.name
, tp
->nhit
,
737 static const struct seq_operations profile_seq_op
= {
738 .start
= probes_seq_start
,
739 .next
= probes_seq_next
,
740 .stop
= probes_seq_stop
,
741 .show
= probes_profile_seq_show
744 static int profile_open(struct inode
*inode
, struct file
*file
)
746 return seq_open(file
, &profile_seq_op
);
749 static const struct file_operations kprobe_profile_ops
= {
750 .owner
= THIS_MODULE
,
751 .open
= profile_open
,
754 .release
= seq_release
,
757 /* Sum up total data length for dynamic arraies (strings) */
758 static __kprobes
int __get_data_size(struct trace_probe
*tp
,
759 struct pt_regs
*regs
)
764 for (i
= 0; i
< tp
->nr_args
; i
++)
765 if (unlikely(tp
->args
[i
].fetch_size
.fn
)) {
766 call_fetch(&tp
->args
[i
].fetch_size
, regs
, &len
);
773 /* Store the value of each argument */
774 static __kprobes
void store_trace_args(int ent_size
, struct trace_probe
*tp
,
775 struct pt_regs
*regs
,
776 u8
*data
, int maxlen
)
780 u32
*dl
; /* Data (relative) location */
782 for (i
= 0; i
< tp
->nr_args
; i
++) {
783 if (unlikely(tp
->args
[i
].fetch_size
.fn
)) {
785 * First, we set the relative location and
786 * maximum data length to *dl
788 dl
= (u32
*)(data
+ tp
->args
[i
].offset
);
789 *dl
= make_data_rloc(maxlen
, end
- tp
->args
[i
].offset
);
790 /* Then try to fetch string or dynamic array data */
791 call_fetch(&tp
->args
[i
].fetch
, regs
, dl
);
792 /* Reduce maximum length */
793 end
+= get_rloc_len(*dl
);
794 maxlen
-= get_rloc_len(*dl
);
795 /* Trick here, convert data_rloc to data_loc */
796 *dl
= convert_rloc_to_loc(*dl
,
797 ent_size
+ tp
->args
[i
].offset
);
799 /* Just fetching data normally */
800 call_fetch(&tp
->args
[i
].fetch
, regs
,
801 data
+ tp
->args
[i
].offset
);
806 static __kprobes
void
807 __kprobe_trace_func(struct trace_probe
*tp
, struct pt_regs
*regs
,
808 struct ftrace_event_file
*ftrace_file
)
810 struct kprobe_trace_entry_head
*entry
;
811 struct ring_buffer_event
*event
;
812 struct ring_buffer
*buffer
;
814 unsigned long irq_flags
;
815 struct ftrace_event_call
*call
= &tp
->call
;
817 WARN_ON(call
!= ftrace_file
->event_call
);
819 if (test_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT
, &ftrace_file
->flags
))
822 local_save_flags(irq_flags
);
823 pc
= preempt_count();
825 dsize
= __get_data_size(tp
, regs
);
826 size
= sizeof(*entry
) + tp
->size
+ dsize
;
828 event
= trace_event_buffer_lock_reserve(&buffer
, ftrace_file
,
830 size
, irq_flags
, pc
);
834 entry
= ring_buffer_event_data(event
);
835 entry
->ip
= (unsigned long)tp
->rp
.kp
.addr
;
836 store_trace_args(sizeof(*entry
), tp
, regs
, (u8
*)&entry
[1], dsize
);
838 if (!filter_current_check_discard(buffer
, call
, entry
, event
))
839 trace_buffer_unlock_commit_regs(buffer
, event
,
840 irq_flags
, pc
, regs
);
843 static __kprobes
void
844 kprobe_trace_func(struct trace_probe
*tp
, struct pt_regs
*regs
)
846 struct event_file_link
*link
;
848 list_for_each_entry_rcu(link
, &tp
->files
, list
)
849 __kprobe_trace_func(tp
, regs
, link
->file
);
852 /* Kretprobe handler */
853 static __kprobes
void
854 __kretprobe_trace_func(struct trace_probe
*tp
, struct kretprobe_instance
*ri
,
855 struct pt_regs
*regs
,
856 struct ftrace_event_file
*ftrace_file
)
858 struct kretprobe_trace_entry_head
*entry
;
859 struct ring_buffer_event
*event
;
860 struct ring_buffer
*buffer
;
862 unsigned long irq_flags
;
863 struct ftrace_event_call
*call
= &tp
->call
;
865 WARN_ON(call
!= ftrace_file
->event_call
);
867 if (test_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT
, &ftrace_file
->flags
))
870 local_save_flags(irq_flags
);
871 pc
= preempt_count();
873 dsize
= __get_data_size(tp
, regs
);
874 size
= sizeof(*entry
) + tp
->size
+ dsize
;
876 event
= trace_event_buffer_lock_reserve(&buffer
, ftrace_file
,
878 size
, irq_flags
, pc
);
882 entry
= ring_buffer_event_data(event
);
883 entry
->func
= (unsigned long)tp
->rp
.kp
.addr
;
884 entry
->ret_ip
= (unsigned long)ri
->ret_addr
;
885 store_trace_args(sizeof(*entry
), tp
, regs
, (u8
*)&entry
[1], dsize
);
887 if (!filter_current_check_discard(buffer
, call
, entry
, event
))
888 trace_buffer_unlock_commit_regs(buffer
, event
,
889 irq_flags
, pc
, regs
);
892 static __kprobes
void
893 kretprobe_trace_func(struct trace_probe
*tp
, struct kretprobe_instance
*ri
,
894 struct pt_regs
*regs
)
896 struct event_file_link
*link
;
898 list_for_each_entry_rcu(link
, &tp
->files
, list
)
899 __kretprobe_trace_func(tp
, ri
, regs
, link
->file
);
902 /* Event entry printers */
903 static enum print_line_t
904 print_kprobe_event(struct trace_iterator
*iter
, int flags
,
905 struct trace_event
*event
)
907 struct kprobe_trace_entry_head
*field
;
908 struct trace_seq
*s
= &iter
->seq
;
909 struct trace_probe
*tp
;
913 field
= (struct kprobe_trace_entry_head
*)iter
->ent
;
914 tp
= container_of(event
, struct trace_probe
, call
.event
);
916 if (!trace_seq_printf(s
, "%s: (", tp
->call
.name
))
919 if (!seq_print_ip_sym(s
, field
->ip
, flags
| TRACE_ITER_SYM_OFFSET
))
922 if (!trace_seq_puts(s
, ")"))
925 data
= (u8
*)&field
[1];
926 for (i
= 0; i
< tp
->nr_args
; i
++)
927 if (!tp
->args
[i
].type
->print(s
, tp
->args
[i
].name
,
928 data
+ tp
->args
[i
].offset
, field
))
931 if (!trace_seq_puts(s
, "\n"))
934 return TRACE_TYPE_HANDLED
;
936 return TRACE_TYPE_PARTIAL_LINE
;
939 static enum print_line_t
940 print_kretprobe_event(struct trace_iterator
*iter
, int flags
,
941 struct trace_event
*event
)
943 struct kretprobe_trace_entry_head
*field
;
944 struct trace_seq
*s
= &iter
->seq
;
945 struct trace_probe
*tp
;
949 field
= (struct kretprobe_trace_entry_head
*)iter
->ent
;
950 tp
= container_of(event
, struct trace_probe
, call
.event
);
952 if (!trace_seq_printf(s
, "%s: (", tp
->call
.name
))
955 if (!seq_print_ip_sym(s
, field
->ret_ip
, flags
| TRACE_ITER_SYM_OFFSET
))
958 if (!trace_seq_puts(s
, " <- "))
961 if (!seq_print_ip_sym(s
, field
->func
, flags
& ~TRACE_ITER_SYM_OFFSET
))
964 if (!trace_seq_puts(s
, ")"))
967 data
= (u8
*)&field
[1];
968 for (i
= 0; i
< tp
->nr_args
; i
++)
969 if (!tp
->args
[i
].type
->print(s
, tp
->args
[i
].name
,
970 data
+ tp
->args
[i
].offset
, field
))
973 if (!trace_seq_puts(s
, "\n"))
976 return TRACE_TYPE_HANDLED
;
978 return TRACE_TYPE_PARTIAL_LINE
;
982 static int kprobe_event_define_fields(struct ftrace_event_call
*event_call
)
985 struct kprobe_trace_entry_head field
;
986 struct trace_probe
*tp
= (struct trace_probe
*)event_call
->data
;
988 DEFINE_FIELD(unsigned long, ip
, FIELD_STRING_IP
, 0);
989 /* Set argument names as fields */
990 for (i
= 0; i
< tp
->nr_args
; i
++) {
991 ret
= trace_define_field(event_call
, tp
->args
[i
].type
->fmttype
,
993 sizeof(field
) + tp
->args
[i
].offset
,
994 tp
->args
[i
].type
->size
,
995 tp
->args
[i
].type
->is_signed
,
1003 static int kretprobe_event_define_fields(struct ftrace_event_call
*event_call
)
1006 struct kretprobe_trace_entry_head field
;
1007 struct trace_probe
*tp
= (struct trace_probe
*)event_call
->data
;
1009 DEFINE_FIELD(unsigned long, func
, FIELD_STRING_FUNC
, 0);
1010 DEFINE_FIELD(unsigned long, ret_ip
, FIELD_STRING_RETIP
, 0);
1011 /* Set argument names as fields */
1012 for (i
= 0; i
< tp
->nr_args
; i
++) {
1013 ret
= trace_define_field(event_call
, tp
->args
[i
].type
->fmttype
,
1015 sizeof(field
) + tp
->args
[i
].offset
,
1016 tp
->args
[i
].type
->size
,
1017 tp
->args
[i
].type
->is_signed
,
1025 static int __set_print_fmt(struct trace_probe
*tp
, char *buf
, int len
)
1030 const char *fmt
, *arg
;
1032 if (!trace_probe_is_return(tp
)) {
1034 arg
= "REC->" FIELD_STRING_IP
;
1036 fmt
= "(%lx <- %lx)";
1037 arg
= "REC->" FIELD_STRING_FUNC
", REC->" FIELD_STRING_RETIP
;
1040 /* When len=0, we just calculate the needed length */
1041 #define LEN_OR_ZERO (len ? len - pos : 0)
1043 pos
+= snprintf(buf
+ pos
, LEN_OR_ZERO
, "\"%s", fmt
);
1045 for (i
= 0; i
< tp
->nr_args
; i
++) {
1046 pos
+= snprintf(buf
+ pos
, LEN_OR_ZERO
, " %s=%s",
1047 tp
->args
[i
].name
, tp
->args
[i
].type
->fmt
);
1050 pos
+= snprintf(buf
+ pos
, LEN_OR_ZERO
, "\", %s", arg
);
1052 for (i
= 0; i
< tp
->nr_args
; i
++) {
1053 if (strcmp(tp
->args
[i
].type
->name
, "string") == 0)
1054 pos
+= snprintf(buf
+ pos
, LEN_OR_ZERO
,
1058 pos
+= snprintf(buf
+ pos
, LEN_OR_ZERO
, ", REC->%s",
1064 /* return the length of print_fmt */
1068 static int set_print_fmt(struct trace_probe
*tp
)
1073 /* First: called with 0 length to calculate the needed length */
1074 len
= __set_print_fmt(tp
, NULL
, 0);
1075 print_fmt
= kmalloc(len
+ 1, GFP_KERNEL
);
1079 /* Second: actually write the @print_fmt */
1080 __set_print_fmt(tp
, print_fmt
, len
+ 1);
1081 tp
->call
.print_fmt
= print_fmt
;
1086 #ifdef CONFIG_PERF_EVENTS
1088 /* Kprobe profile handler */
1089 static __kprobes
void
1090 kprobe_perf_func(struct trace_probe
*tp
, struct pt_regs
*regs
)
1092 struct ftrace_event_call
*call
= &tp
->call
;
1093 struct kprobe_trace_entry_head
*entry
;
1094 struct hlist_head
*head
;
1095 int size
, __size
, dsize
;
1098 head
= this_cpu_ptr(call
->perf_events
);
1099 if (hlist_empty(head
))
1102 dsize
= __get_data_size(tp
, regs
);
1103 __size
= sizeof(*entry
) + tp
->size
+ dsize
;
1104 size
= ALIGN(__size
+ sizeof(u32
), sizeof(u64
));
1105 size
-= sizeof(u32
);
1107 entry
= perf_trace_buf_prepare(size
, call
->event
.type
, regs
, &rctx
);
1111 entry
->ip
= (unsigned long)tp
->rp
.kp
.addr
;
1112 memset(&entry
[1], 0, dsize
);
1113 store_trace_args(sizeof(*entry
), tp
, regs
, (u8
*)&entry
[1], dsize
);
1114 perf_trace_buf_submit(entry
, size
, rctx
, 0, 1, regs
, head
, NULL
);
1117 /* Kretprobe profile handler */
1118 static __kprobes
void
1119 kretprobe_perf_func(struct trace_probe
*tp
, struct kretprobe_instance
*ri
,
1120 struct pt_regs
*regs
)
1122 struct ftrace_event_call
*call
= &tp
->call
;
1123 struct kretprobe_trace_entry_head
*entry
;
1124 struct hlist_head
*head
;
1125 int size
, __size
, dsize
;
1128 head
= this_cpu_ptr(call
->perf_events
);
1129 if (hlist_empty(head
))
1132 dsize
= __get_data_size(tp
, regs
);
1133 __size
= sizeof(*entry
) + tp
->size
+ dsize
;
1134 size
= ALIGN(__size
+ sizeof(u32
), sizeof(u64
));
1135 size
-= sizeof(u32
);
1137 entry
= perf_trace_buf_prepare(size
, call
->event
.type
, regs
, &rctx
);
1141 entry
->func
= (unsigned long)tp
->rp
.kp
.addr
;
1142 entry
->ret_ip
= (unsigned long)ri
->ret_addr
;
1143 store_trace_args(sizeof(*entry
), tp
, regs
, (u8
*)&entry
[1], dsize
);
1144 perf_trace_buf_submit(entry
, size
, rctx
, 0, 1, regs
, head
, NULL
);
1146 #endif /* CONFIG_PERF_EVENTS */
1149 * called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex.
1151 * kprobe_trace_self_tests_init() does enable_trace_probe/disable_trace_probe
1152 * lockless, but we can't race with this __init function.
1155 int kprobe_register(struct ftrace_event_call
*event
,
1156 enum trace_reg type
, void *data
)
1158 struct trace_probe
*tp
= (struct trace_probe
*)event
->data
;
1159 struct ftrace_event_file
*file
= data
;
1162 case TRACE_REG_REGISTER
:
1163 return enable_trace_probe(tp
, file
);
1164 case TRACE_REG_UNREGISTER
:
1165 return disable_trace_probe(tp
, file
);
1167 #ifdef CONFIG_PERF_EVENTS
1168 case TRACE_REG_PERF_REGISTER
:
1169 return enable_trace_probe(tp
, NULL
);
1170 case TRACE_REG_PERF_UNREGISTER
:
1171 return disable_trace_probe(tp
, NULL
);
1172 case TRACE_REG_PERF_OPEN
:
1173 case TRACE_REG_PERF_CLOSE
:
1174 case TRACE_REG_PERF_ADD
:
1175 case TRACE_REG_PERF_DEL
:
1183 int kprobe_dispatcher(struct kprobe
*kp
, struct pt_regs
*regs
)
1185 struct trace_probe
*tp
= container_of(kp
, struct trace_probe
, rp
.kp
);
1189 if (tp
->flags
& TP_FLAG_TRACE
)
1190 kprobe_trace_func(tp
, regs
);
1191 #ifdef CONFIG_PERF_EVENTS
1192 if (tp
->flags
& TP_FLAG_PROFILE
)
1193 kprobe_perf_func(tp
, regs
);
1195 return 0; /* We don't tweek kernel, so just return 0 */
1199 int kretprobe_dispatcher(struct kretprobe_instance
*ri
, struct pt_regs
*regs
)
1201 struct trace_probe
*tp
= container_of(ri
->rp
, struct trace_probe
, rp
);
1205 if (tp
->flags
& TP_FLAG_TRACE
)
1206 kretprobe_trace_func(tp
, ri
, regs
);
1207 #ifdef CONFIG_PERF_EVENTS
1208 if (tp
->flags
& TP_FLAG_PROFILE
)
1209 kretprobe_perf_func(tp
, ri
, regs
);
1211 return 0; /* We don't tweek kernel, so just return 0 */
1214 static struct trace_event_functions kretprobe_funcs
= {
1215 .trace
= print_kretprobe_event
1218 static struct trace_event_functions kprobe_funcs
= {
1219 .trace
= print_kprobe_event
1222 static int register_probe_event(struct trace_probe
*tp
)
1224 struct ftrace_event_call
*call
= &tp
->call
;
1227 /* Initialize ftrace_event_call */
1228 INIT_LIST_HEAD(&call
->class->fields
);
1229 if (trace_probe_is_return(tp
)) {
1230 call
->event
.funcs
= &kretprobe_funcs
;
1231 call
->class->define_fields
= kretprobe_event_define_fields
;
1233 call
->event
.funcs
= &kprobe_funcs
;
1234 call
->class->define_fields
= kprobe_event_define_fields
;
1236 if (set_print_fmt(tp
) < 0)
1238 ret
= register_ftrace_event(&call
->event
);
1240 kfree(call
->print_fmt
);
1244 call
->class->reg
= kprobe_register
;
1246 ret
= trace_add_event_call(call
);
1248 pr_info("Failed to register kprobe event: %s\n", call
->name
);
1249 kfree(call
->print_fmt
);
1250 unregister_ftrace_event(&call
->event
);
1255 static int unregister_probe_event(struct trace_probe
*tp
)
1259 /* tp->event is unregistered in trace_remove_event_call() */
1260 ret
= trace_remove_event_call(&tp
->call
);
1262 kfree(tp
->call
.print_fmt
);
1266 /* Make a debugfs interface for controlling probe points */
1267 static __init
int init_kprobe_trace(void)
1269 struct dentry
*d_tracer
;
1270 struct dentry
*entry
;
1272 if (register_module_notifier(&trace_probe_module_nb
))
1275 d_tracer
= tracing_init_dentry();
1279 entry
= debugfs_create_file("kprobe_events", 0644, d_tracer
,
1280 NULL
, &kprobe_events_ops
);
1282 /* Event list interface */
1284 pr_warning("Could not create debugfs "
1285 "'kprobe_events' entry\n");
1287 /* Profile interface */
1288 entry
= debugfs_create_file("kprobe_profile", 0444, d_tracer
,
1289 NULL
, &kprobe_profile_ops
);
1292 pr_warning("Could not create debugfs "
1293 "'kprobe_profile' entry\n");
1296 fs_initcall(init_kprobe_trace
);
1299 #ifdef CONFIG_FTRACE_STARTUP_TEST
1302 * The "__used" keeps gcc from removing the function symbol
1303 * from the kallsyms table.
1305 static __used
int kprobe_trace_selftest_target(int a1
, int a2
, int a3
,
1306 int a4
, int a5
, int a6
)
1308 return a1
+ a2
+ a3
+ a4
+ a5
+ a6
;
1311 static struct ftrace_event_file
*
1312 find_trace_probe_file(struct trace_probe
*tp
, struct trace_array
*tr
)
1314 struct ftrace_event_file
*file
;
1316 list_for_each_entry(file
, &tr
->events
, list
)
1317 if (file
->event_call
== &tp
->call
)
1324 * Nobody but us can call enable_trace_probe/disable_trace_probe at this
1325 * stage, we can do this lockless.
1327 static __init
int kprobe_trace_self_tests_init(void)
1330 int (*target
)(int, int, int, int, int, int);
1331 struct trace_probe
*tp
;
1332 struct ftrace_event_file
*file
;
1334 target
= kprobe_trace_selftest_target
;
1336 pr_info("Testing kprobe tracing: ");
1338 ret
= traceprobe_command("p:testprobe kprobe_trace_selftest_target "
1339 "$stack $stack0 +0($stack)",
1340 create_trace_probe
);
1341 if (WARN_ON_ONCE(ret
)) {
1342 pr_warn("error on probing function entry.\n");
1345 /* Enable trace point */
1346 tp
= find_trace_probe("testprobe", KPROBE_EVENT_SYSTEM
);
1347 if (WARN_ON_ONCE(tp
== NULL
)) {
1348 pr_warn("error on getting new probe.\n");
1351 file
= find_trace_probe_file(tp
, top_trace_array());
1352 if (WARN_ON_ONCE(file
== NULL
)) {
1353 pr_warn("error on getting probe file.\n");
1356 enable_trace_probe(tp
, file
);
1360 ret
= traceprobe_command("r:testprobe2 kprobe_trace_selftest_target "
1361 "$retval", create_trace_probe
);
1362 if (WARN_ON_ONCE(ret
)) {
1363 pr_warn("error on probing function return.\n");
1366 /* Enable trace point */
1367 tp
= find_trace_probe("testprobe2", KPROBE_EVENT_SYSTEM
);
1368 if (WARN_ON_ONCE(tp
== NULL
)) {
1369 pr_warn("error on getting 2nd new probe.\n");
1372 file
= find_trace_probe_file(tp
, top_trace_array());
1373 if (WARN_ON_ONCE(file
== NULL
)) {
1374 pr_warn("error on getting probe file.\n");
1377 enable_trace_probe(tp
, file
);
1384 ret
= target(1, 2, 3, 4, 5, 6);
1386 /* Disable trace points before removing it */
1387 tp
= find_trace_probe("testprobe", KPROBE_EVENT_SYSTEM
);
1388 if (WARN_ON_ONCE(tp
== NULL
)) {
1389 pr_warn("error on getting test probe.\n");
1392 file
= find_trace_probe_file(tp
, top_trace_array());
1393 if (WARN_ON_ONCE(file
== NULL
)) {
1394 pr_warn("error on getting probe file.\n");
1397 disable_trace_probe(tp
, file
);
1400 tp
= find_trace_probe("testprobe2", KPROBE_EVENT_SYSTEM
);
1401 if (WARN_ON_ONCE(tp
== NULL
)) {
1402 pr_warn("error on getting 2nd test probe.\n");
1405 file
= find_trace_probe_file(tp
, top_trace_array());
1406 if (WARN_ON_ONCE(file
== NULL
)) {
1407 pr_warn("error on getting probe file.\n");
1410 disable_trace_probe(tp
, file
);
1413 ret
= traceprobe_command("-:testprobe", create_trace_probe
);
1414 if (WARN_ON_ONCE(ret
)) {
1415 pr_warn("error on deleting a probe.\n");
1419 ret
= traceprobe_command("-:testprobe2", create_trace_probe
);
1420 if (WARN_ON_ONCE(ret
)) {
1421 pr_warn("error on deleting a probe.\n");
1426 release_all_trace_probes();
1428 pr_cont("NG: Some tests are failed. Please check them.\n");
1434 late_initcall(kprobe_trace_self_tests_init
);