3 * Function graph tracer.
4 * Copyright (c) 2008-2009 Frederic Weisbecker <fweisbec@gmail.com>
5 * Mostly borrowed from function tracer which
6 * is Copyright (c) Steven Rostedt <srostedt@redhat.com>
9 #include <linux/debugfs.h>
10 #include <linux/uaccess.h>
11 #include <linux/ftrace.h>
12 #include <linux/slab.h>
16 #include "trace_output.h"
18 /* When set, irq functions will be ignored */
19 static int ftrace_graph_skip_irqs
;
21 struct fgraph_cpu_data
{
26 unsigned long enter_funcs
[FTRACE_RETFUNC_DEPTH
];
30 struct fgraph_cpu_data __percpu
*cpu_data
;
32 /* Place to preserve last processed entry. */
33 struct ftrace_graph_ent_entry ent
;
34 struct ftrace_graph_ret_entry ret
;
39 #define TRACE_GRAPH_INDENT 2
42 #define TRACE_GRAPH_PRINT_OVERRUN 0x1
43 #define TRACE_GRAPH_PRINT_CPU 0x2
44 #define TRACE_GRAPH_PRINT_OVERHEAD 0x4
45 #define TRACE_GRAPH_PRINT_PROC 0x8
46 #define TRACE_GRAPH_PRINT_DURATION 0x10
47 #define TRACE_GRAPH_PRINT_ABS_TIME 0x20
48 #define TRACE_GRAPH_PRINT_IRQS 0x40
50 static unsigned int max_depth
;
52 static struct tracer_opt trace_opts
[] = {
53 /* Display overruns? (for self-debug purpose) */
54 { TRACER_OPT(funcgraph
-overrun
, TRACE_GRAPH_PRINT_OVERRUN
) },
56 { TRACER_OPT(funcgraph
-cpu
, TRACE_GRAPH_PRINT_CPU
) },
57 /* Display Overhead ? */
58 { TRACER_OPT(funcgraph
-overhead
, TRACE_GRAPH_PRINT_OVERHEAD
) },
59 /* Display proc name/pid */
60 { TRACER_OPT(funcgraph
-proc
, TRACE_GRAPH_PRINT_PROC
) },
61 /* Display duration of execution */
62 { TRACER_OPT(funcgraph
-duration
, TRACE_GRAPH_PRINT_DURATION
) },
63 /* Display absolute time of an entry */
64 { TRACER_OPT(funcgraph
-abstime
, TRACE_GRAPH_PRINT_ABS_TIME
) },
65 /* Display interrupts */
66 { TRACER_OPT(funcgraph
-irqs
, TRACE_GRAPH_PRINT_IRQS
) },
70 static struct tracer_flags tracer_flags
= {
71 /* Don't display overruns and proc by default */
72 .val
= TRACE_GRAPH_PRINT_CPU
| TRACE_GRAPH_PRINT_OVERHEAD
|
73 TRACE_GRAPH_PRINT_DURATION
| TRACE_GRAPH_PRINT_IRQS
,
77 static struct trace_array
*graph_array
;
80 * DURATION column is being also used to display IRQ signs,
81 * following values are used by print_graph_irq and others
82 * to fill in space into DURATION column.
85 DURATION_FILL_FULL
= -1,
86 DURATION_FILL_START
= -2,
87 DURATION_FILL_END
= -3,
90 static enum print_line_t
91 print_graph_duration(unsigned long long duration
, struct trace_seq
*s
,
94 /* Add a function return address to the trace stack on thread info.*/
96 ftrace_push_return_trace(unsigned long ret
, unsigned long func
, int *depth
,
97 unsigned long frame_pointer
)
99 unsigned long long calltime
;
102 if (!current
->ret_stack
)
106 * We must make sure the ret_stack is tested before we read
111 /* The return trace stack is full */
112 if (current
->curr_ret_stack
== FTRACE_RETFUNC_DEPTH
- 1) {
113 atomic_inc(¤t
->trace_overrun
);
117 calltime
= trace_clock_local();
119 index
= ++current
->curr_ret_stack
;
121 current
->ret_stack
[index
].ret
= ret
;
122 current
->ret_stack
[index
].func
= func
;
123 current
->ret_stack
[index
].calltime
= calltime
;
124 current
->ret_stack
[index
].subtime
= 0;
125 current
->ret_stack
[index
].fp
= frame_pointer
;
131 /* Retrieve a function return address to the trace stack on thread info.*/
133 ftrace_pop_return_trace(struct ftrace_graph_ret
*trace
, unsigned long *ret
,
134 unsigned long frame_pointer
)
138 index
= current
->curr_ret_stack
;
140 if (unlikely(index
< 0)) {
143 /* Might as well panic, otherwise we have no where to go */
144 *ret
= (unsigned long)panic
;
148 #if defined(CONFIG_HAVE_FUNCTION_GRAPH_FP_TEST) && !defined(CC_USING_FENTRY)
150 * The arch may choose to record the frame pointer used
151 * and check it here to make sure that it is what we expect it
152 * to be. If gcc does not set the place holder of the return
153 * address in the frame pointer, and does a copy instead, then
154 * the function graph trace will fail. This test detects this
157 * Currently, x86_32 with optimize for size (-Os) makes the latest
160 * Note, -mfentry does not use frame pointers, and this test
161 * is not needed if CC_USING_FENTRY is set.
163 if (unlikely(current
->ret_stack
[index
].fp
!= frame_pointer
)) {
165 WARN(1, "Bad frame pointer: expected %lx, received %lx\n"
166 " from func %ps return to %lx\n",
167 current
->ret_stack
[index
].fp
,
169 (void *)current
->ret_stack
[index
].func
,
170 current
->ret_stack
[index
].ret
);
171 *ret
= (unsigned long)panic
;
176 *ret
= current
->ret_stack
[index
].ret
;
177 trace
->func
= current
->ret_stack
[index
].func
;
178 trace
->calltime
= current
->ret_stack
[index
].calltime
;
179 trace
->overrun
= atomic_read(¤t
->trace_overrun
);
180 trace
->depth
= index
;
184 * Send the trace to the ring-buffer.
185 * @return the original return address.
187 unsigned long ftrace_return_to_handler(unsigned long frame_pointer
)
189 struct ftrace_graph_ret trace
;
192 ftrace_pop_return_trace(&trace
, &ret
, frame_pointer
);
193 trace
.rettime
= trace_clock_local();
195 current
->curr_ret_stack
--;
198 * The trace should run after decrementing the ret counter
199 * in case an interrupt were to come in. We don't want to
200 * lose the interrupt if max_depth is set.
202 ftrace_graph_return(&trace
);
204 if (unlikely(!ret
)) {
207 /* Might as well panic. What else to do? */
208 ret
= (unsigned long)panic
;
214 int __trace_graph_entry(struct trace_array
*tr
,
215 struct ftrace_graph_ent
*trace
,
219 struct ftrace_event_call
*call
= &event_funcgraph_entry
;
220 struct ring_buffer_event
*event
;
221 struct ring_buffer
*buffer
= tr
->trace_buffer
.buffer
;
222 struct ftrace_graph_ent_entry
*entry
;
224 if (unlikely(__this_cpu_read(ftrace_cpu_disabled
)))
227 event
= trace_buffer_lock_reserve(buffer
, TRACE_GRAPH_ENT
,
228 sizeof(*entry
), flags
, pc
);
231 entry
= ring_buffer_event_data(event
);
232 entry
->graph_ent
= *trace
;
233 if (!filter_current_check_discard(buffer
, call
, entry
, event
))
234 __buffer_unlock_commit(buffer
, event
);
239 static inline int ftrace_graph_ignore_irqs(void)
241 if (!ftrace_graph_skip_irqs
|| trace_recursion_test(TRACE_IRQ_BIT
))
247 int trace_graph_entry(struct ftrace_graph_ent
*trace
)
249 struct trace_array
*tr
= graph_array
;
250 struct trace_array_cpu
*data
;
257 if (!ftrace_trace_task(current
))
260 /* trace it when it is-nested-in or is a function enabled. */
261 if ((!(trace
->depth
|| ftrace_graph_addr(trace
->func
)) ||
262 ftrace_graph_ignore_irqs()) ||
263 (max_depth
&& trace
->depth
>= max_depth
))
266 local_irq_save(flags
);
267 cpu
= raw_smp_processor_id();
268 data
= per_cpu_ptr(tr
->trace_buffer
.data
, cpu
);
269 disabled
= atomic_inc_return(&data
->disabled
);
270 if (likely(disabled
== 1)) {
271 pc
= preempt_count();
272 ret
= __trace_graph_entry(tr
, trace
, flags
, pc
);
277 atomic_dec(&data
->disabled
);
278 local_irq_restore(flags
);
283 int trace_graph_thresh_entry(struct ftrace_graph_ent
*trace
)
288 return trace_graph_entry(trace
);
292 __trace_graph_function(struct trace_array
*tr
,
293 unsigned long ip
, unsigned long flags
, int pc
)
295 u64 time
= trace_clock_local();
296 struct ftrace_graph_ent ent
= {
300 struct ftrace_graph_ret ret
= {
307 __trace_graph_entry(tr
, &ent
, flags
, pc
);
308 __trace_graph_return(tr
, &ret
, flags
, pc
);
312 trace_graph_function(struct trace_array
*tr
,
313 unsigned long ip
, unsigned long parent_ip
,
314 unsigned long flags
, int pc
)
316 __trace_graph_function(tr
, ip
, flags
, pc
);
319 void __trace_graph_return(struct trace_array
*tr
,
320 struct ftrace_graph_ret
*trace
,
324 struct ftrace_event_call
*call
= &event_funcgraph_exit
;
325 struct ring_buffer_event
*event
;
326 struct ring_buffer
*buffer
= tr
->trace_buffer
.buffer
;
327 struct ftrace_graph_ret_entry
*entry
;
329 if (unlikely(__this_cpu_read(ftrace_cpu_disabled
)))
332 event
= trace_buffer_lock_reserve(buffer
, TRACE_GRAPH_RET
,
333 sizeof(*entry
), flags
, pc
);
336 entry
= ring_buffer_event_data(event
);
338 if (!filter_current_check_discard(buffer
, call
, entry
, event
))
339 __buffer_unlock_commit(buffer
, event
);
342 void trace_graph_return(struct ftrace_graph_ret
*trace
)
344 struct trace_array
*tr
= graph_array
;
345 struct trace_array_cpu
*data
;
351 local_irq_save(flags
);
352 cpu
= raw_smp_processor_id();
353 data
= per_cpu_ptr(tr
->trace_buffer
.data
, cpu
);
354 disabled
= atomic_inc_return(&data
->disabled
);
355 if (likely(disabled
== 1)) {
356 pc
= preempt_count();
357 __trace_graph_return(tr
, trace
, flags
, pc
);
359 atomic_dec(&data
->disabled
);
360 local_irq_restore(flags
);
363 void set_graph_array(struct trace_array
*tr
)
367 /* Make graph_array visible before we start tracing */
372 void trace_graph_thresh_return(struct ftrace_graph_ret
*trace
)
374 if (tracing_thresh
&&
375 (trace
->rettime
- trace
->calltime
< tracing_thresh
))
378 trace_graph_return(trace
);
381 static int graph_trace_init(struct trace_array
*tr
)
387 ret
= register_ftrace_graph(&trace_graph_thresh_return
,
388 &trace_graph_thresh_entry
);
390 ret
= register_ftrace_graph(&trace_graph_return
,
394 tracing_start_cmdline_record();
399 static void graph_trace_reset(struct trace_array
*tr
)
401 tracing_stop_cmdline_record();
402 unregister_ftrace_graph();
405 static int max_bytes_for_cpu
;
407 static enum print_line_t
408 print_graph_cpu(struct trace_seq
*s
, int cpu
)
413 * Start with a space character - to make it stand out
414 * to the right a bit when trace output is pasted into
417 ret
= trace_seq_printf(s
, " %*d) ", max_bytes_for_cpu
, cpu
);
419 return TRACE_TYPE_PARTIAL_LINE
;
421 return TRACE_TYPE_HANDLED
;
424 #define TRACE_GRAPH_PROCINFO_LENGTH 14
426 static enum print_line_t
427 print_graph_proc(struct trace_seq
*s
, pid_t pid
)
429 char comm
[TASK_COMM_LEN
];
430 /* sign + log10(MAX_INT) + '\0' */
437 trace_find_cmdline(pid
, comm
);
439 sprintf(pid_str
, "%d", pid
);
441 /* 1 stands for the "-" character */
442 len
= strlen(comm
) + strlen(pid_str
) + 1;
444 if (len
< TRACE_GRAPH_PROCINFO_LENGTH
)
445 spaces
= TRACE_GRAPH_PROCINFO_LENGTH
- len
;
447 /* First spaces to align center */
448 for (i
= 0; i
< spaces
/ 2; i
++) {
449 ret
= trace_seq_putc(s
, ' ');
451 return TRACE_TYPE_PARTIAL_LINE
;
454 ret
= trace_seq_printf(s
, "%s-%s", comm
, pid_str
);
456 return TRACE_TYPE_PARTIAL_LINE
;
458 /* Last spaces to align center */
459 for (i
= 0; i
< spaces
- (spaces
/ 2); i
++) {
460 ret
= trace_seq_putc(s
, ' ');
462 return TRACE_TYPE_PARTIAL_LINE
;
464 return TRACE_TYPE_HANDLED
;
468 static enum print_line_t
469 print_graph_lat_fmt(struct trace_seq
*s
, struct trace_entry
*entry
)
471 if (!trace_seq_putc(s
, ' '))
474 return trace_print_lat_fmt(s
, entry
);
477 /* If the pid changed since the last trace, output this event */
478 static enum print_line_t
479 verif_pid(struct trace_seq
*s
, pid_t pid
, int cpu
, struct fgraph_data
*data
)
486 return TRACE_TYPE_HANDLED
;
488 last_pid
= &(per_cpu_ptr(data
->cpu_data
, cpu
)->last_pid
);
490 if (*last_pid
== pid
)
491 return TRACE_TYPE_HANDLED
;
493 prev_pid
= *last_pid
;
497 return TRACE_TYPE_HANDLED
;
499 * Context-switch trace line:
501 ------------------------------------------
502 | 1) migration/0--1 => sshd-1755
503 ------------------------------------------
506 ret
= trace_seq_puts(s
,
507 " ------------------------------------------\n");
509 return TRACE_TYPE_PARTIAL_LINE
;
511 ret
= print_graph_cpu(s
, cpu
);
512 if (ret
== TRACE_TYPE_PARTIAL_LINE
)
513 return TRACE_TYPE_PARTIAL_LINE
;
515 ret
= print_graph_proc(s
, prev_pid
);
516 if (ret
== TRACE_TYPE_PARTIAL_LINE
)
517 return TRACE_TYPE_PARTIAL_LINE
;
519 ret
= trace_seq_puts(s
, " => ");
521 return TRACE_TYPE_PARTIAL_LINE
;
523 ret
= print_graph_proc(s
, pid
);
524 if (ret
== TRACE_TYPE_PARTIAL_LINE
)
525 return TRACE_TYPE_PARTIAL_LINE
;
527 ret
= trace_seq_puts(s
,
528 "\n ------------------------------------------\n\n");
530 return TRACE_TYPE_PARTIAL_LINE
;
532 return TRACE_TYPE_HANDLED
;
535 static struct ftrace_graph_ret_entry
*
536 get_return_for_leaf(struct trace_iterator
*iter
,
537 struct ftrace_graph_ent_entry
*curr
)
539 struct fgraph_data
*data
= iter
->private;
540 struct ring_buffer_iter
*ring_iter
= NULL
;
541 struct ring_buffer_event
*event
;
542 struct ftrace_graph_ret_entry
*next
;
545 * If the previous output failed to write to the seq buffer,
546 * then we just reuse the data from before.
548 if (data
&& data
->failed
) {
553 ring_iter
= trace_buffer_iter(iter
, iter
->cpu
);
555 /* First peek to compare current entry and the next one */
557 event
= ring_buffer_iter_peek(ring_iter
, NULL
);
560 * We need to consume the current entry to see
563 ring_buffer_consume(iter
->trace_buffer
->buffer
, iter
->cpu
,
565 event
= ring_buffer_peek(iter
->trace_buffer
->buffer
, iter
->cpu
,
572 next
= ring_buffer_event_data(event
);
576 * Save current and next entries for later reference
577 * if the output fails.
581 * If the next event is not a return type, then
582 * we only care about what type it is. Otherwise we can
583 * safely copy the entire event.
585 if (next
->ent
.type
== TRACE_GRAPH_RET
)
588 data
->ret
.ent
.type
= next
->ent
.type
;
592 if (next
->ent
.type
!= TRACE_GRAPH_RET
)
595 if (curr
->ent
.pid
!= next
->ent
.pid
||
596 curr
->graph_ent
.func
!= next
->ret
.func
)
599 /* this is a leaf, now advance the iterator */
601 ring_buffer_read(ring_iter
, NULL
);
606 static int print_graph_abs_time(u64 t
, struct trace_seq
*s
)
608 unsigned long usecs_rem
;
610 usecs_rem
= do_div(t
, NSEC_PER_SEC
);
613 return trace_seq_printf(s
, "%5lu.%06lu | ",
614 (unsigned long)t
, usecs_rem
);
617 static enum print_line_t
618 print_graph_irq(struct trace_iterator
*iter
, unsigned long addr
,
619 enum trace_type type
, int cpu
, pid_t pid
, u32 flags
)
622 struct trace_seq
*s
= &iter
->seq
;
624 if (addr
< (unsigned long)__irqentry_text_start
||
625 addr
>= (unsigned long)__irqentry_text_end
)
626 return TRACE_TYPE_UNHANDLED
;
628 if (trace_flags
& TRACE_ITER_CONTEXT_INFO
) {
630 if (flags
& TRACE_GRAPH_PRINT_ABS_TIME
) {
631 ret
= print_graph_abs_time(iter
->ts
, s
);
633 return TRACE_TYPE_PARTIAL_LINE
;
637 if (flags
& TRACE_GRAPH_PRINT_CPU
) {
638 ret
= print_graph_cpu(s
, cpu
);
639 if (ret
== TRACE_TYPE_PARTIAL_LINE
)
640 return TRACE_TYPE_PARTIAL_LINE
;
644 if (flags
& TRACE_GRAPH_PRINT_PROC
) {
645 ret
= print_graph_proc(s
, pid
);
646 if (ret
== TRACE_TYPE_PARTIAL_LINE
)
647 return TRACE_TYPE_PARTIAL_LINE
;
648 ret
= trace_seq_puts(s
, " | ");
650 return TRACE_TYPE_PARTIAL_LINE
;
655 ret
= print_graph_duration(DURATION_FILL_START
, s
, flags
);
656 if (ret
!= TRACE_TYPE_HANDLED
)
659 if (type
== TRACE_GRAPH_ENT
)
660 ret
= trace_seq_puts(s
, "==========>");
662 ret
= trace_seq_puts(s
, "<==========");
665 return TRACE_TYPE_PARTIAL_LINE
;
667 ret
= print_graph_duration(DURATION_FILL_END
, s
, flags
);
668 if (ret
!= TRACE_TYPE_HANDLED
)
671 ret
= trace_seq_putc(s
, '\n');
674 return TRACE_TYPE_PARTIAL_LINE
;
675 return TRACE_TYPE_HANDLED
;
679 trace_print_graph_duration(unsigned long long duration
, struct trace_seq
*s
)
681 unsigned long nsecs_rem
= do_div(duration
, 1000);
682 /* log10(ULONG_MAX) + '\0' */
688 sprintf(msecs_str
, "%lu", (unsigned long) duration
);
691 ret
= trace_seq_printf(s
, "%s", msecs_str
);
693 return TRACE_TYPE_PARTIAL_LINE
;
695 len
= strlen(msecs_str
);
697 /* Print nsecs (we don't want to exceed 7 numbers) */
699 size_t slen
= min_t(size_t, sizeof(nsecs_str
), 8UL - len
);
701 snprintf(nsecs_str
, slen
, "%03lu", nsecs_rem
);
702 ret
= trace_seq_printf(s
, ".%s", nsecs_str
);
704 return TRACE_TYPE_PARTIAL_LINE
;
705 len
+= strlen(nsecs_str
);
708 ret
= trace_seq_puts(s
, " us ");
710 return TRACE_TYPE_PARTIAL_LINE
;
712 /* Print remaining spaces to fit the row's width */
713 for (i
= len
; i
< 7; i
++) {
714 ret
= trace_seq_putc(s
, ' ');
716 return TRACE_TYPE_PARTIAL_LINE
;
718 return TRACE_TYPE_HANDLED
;
721 static enum print_line_t
722 print_graph_duration(unsigned long long duration
, struct trace_seq
*s
,
727 if (!(flags
& TRACE_GRAPH_PRINT_DURATION
) ||
728 !(trace_flags
& TRACE_ITER_CONTEXT_INFO
))
729 return TRACE_TYPE_HANDLED
;
731 /* No real adata, just filling the column with spaces */
733 case DURATION_FILL_FULL
:
734 ret
= trace_seq_puts(s
, " | ");
735 return ret
? TRACE_TYPE_HANDLED
: TRACE_TYPE_PARTIAL_LINE
;
736 case DURATION_FILL_START
:
737 ret
= trace_seq_puts(s
, " ");
738 return ret
? TRACE_TYPE_HANDLED
: TRACE_TYPE_PARTIAL_LINE
;
739 case DURATION_FILL_END
:
740 ret
= trace_seq_puts(s
, " |");
741 return ret
? TRACE_TYPE_HANDLED
: TRACE_TYPE_PARTIAL_LINE
;
744 /* Signal a overhead of time execution to the output */
745 if (flags
& TRACE_GRAPH_PRINT_OVERHEAD
) {
746 /* Duration exceeded 100 msecs */
747 if (duration
> 100000ULL)
748 ret
= trace_seq_puts(s
, "! ");
749 /* Duration exceeded 10 msecs */
750 else if (duration
> 10000ULL)
751 ret
= trace_seq_puts(s
, "+ ");
755 * The -1 means we either did not exceed the duration tresholds
756 * or we dont want to print out the overhead. Either way we need
757 * to fill out the space.
760 ret
= trace_seq_puts(s
, " ");
762 /* Catching here any failure happenned above */
764 return TRACE_TYPE_PARTIAL_LINE
;
766 ret
= trace_print_graph_duration(duration
, s
);
767 if (ret
!= TRACE_TYPE_HANDLED
)
770 ret
= trace_seq_puts(s
, "| ");
772 return TRACE_TYPE_PARTIAL_LINE
;
774 return TRACE_TYPE_HANDLED
;
777 /* Case of a leaf function on its call entry */
778 static enum print_line_t
779 print_graph_entry_leaf(struct trace_iterator
*iter
,
780 struct ftrace_graph_ent_entry
*entry
,
781 struct ftrace_graph_ret_entry
*ret_entry
,
782 struct trace_seq
*s
, u32 flags
)
784 struct fgraph_data
*data
= iter
->private;
785 struct ftrace_graph_ret
*graph_ret
;
786 struct ftrace_graph_ent
*call
;
787 unsigned long long duration
;
791 graph_ret
= &ret_entry
->ret
;
792 call
= &entry
->graph_ent
;
793 duration
= graph_ret
->rettime
- graph_ret
->calltime
;
796 struct fgraph_cpu_data
*cpu_data
;
799 cpu_data
= per_cpu_ptr(data
->cpu_data
, cpu
);
802 * Comments display at + 1 to depth. Since
803 * this is a leaf function, keep the comments
804 * equal to this depth.
806 cpu_data
->depth
= call
->depth
- 1;
808 /* No need to keep this function around for this depth */
809 if (call
->depth
< FTRACE_RETFUNC_DEPTH
)
810 cpu_data
->enter_funcs
[call
->depth
] = 0;
813 /* Overhead and duration */
814 ret
= print_graph_duration(duration
, s
, flags
);
815 if (ret
== TRACE_TYPE_PARTIAL_LINE
)
816 return TRACE_TYPE_PARTIAL_LINE
;
819 for (i
= 0; i
< call
->depth
* TRACE_GRAPH_INDENT
; i
++) {
820 ret
= trace_seq_putc(s
, ' ');
822 return TRACE_TYPE_PARTIAL_LINE
;
825 ret
= trace_seq_printf(s
, "%ps();\n", (void *)call
->func
);
827 return TRACE_TYPE_PARTIAL_LINE
;
829 return TRACE_TYPE_HANDLED
;
832 static enum print_line_t
833 print_graph_entry_nested(struct trace_iterator
*iter
,
834 struct ftrace_graph_ent_entry
*entry
,
835 struct trace_seq
*s
, int cpu
, u32 flags
)
837 struct ftrace_graph_ent
*call
= &entry
->graph_ent
;
838 struct fgraph_data
*data
= iter
->private;
843 struct fgraph_cpu_data
*cpu_data
;
846 cpu_data
= per_cpu_ptr(data
->cpu_data
, cpu
);
847 cpu_data
->depth
= call
->depth
;
849 /* Save this function pointer to see if the exit matches */
850 if (call
->depth
< FTRACE_RETFUNC_DEPTH
)
851 cpu_data
->enter_funcs
[call
->depth
] = call
->func
;
855 ret
= print_graph_duration(DURATION_FILL_FULL
, s
, flags
);
856 if (ret
!= TRACE_TYPE_HANDLED
)
860 for (i
= 0; i
< call
->depth
* TRACE_GRAPH_INDENT
; i
++) {
861 ret
= trace_seq_putc(s
, ' ');
863 return TRACE_TYPE_PARTIAL_LINE
;
866 ret
= trace_seq_printf(s
, "%ps() {\n", (void *)call
->func
);
868 return TRACE_TYPE_PARTIAL_LINE
;
871 * we already consumed the current entry to check the next one
872 * and see if this is a leaf.
874 return TRACE_TYPE_NO_CONSUME
;
877 static enum print_line_t
878 print_graph_prologue(struct trace_iterator
*iter
, struct trace_seq
*s
,
879 int type
, unsigned long addr
, u32 flags
)
881 struct fgraph_data
*data
= iter
->private;
882 struct trace_entry
*ent
= iter
->ent
;
887 if (verif_pid(s
, ent
->pid
, cpu
, data
) == TRACE_TYPE_PARTIAL_LINE
)
888 return TRACE_TYPE_PARTIAL_LINE
;
892 ret
= print_graph_irq(iter
, addr
, type
, cpu
, ent
->pid
, flags
);
893 if (ret
== TRACE_TYPE_PARTIAL_LINE
)
894 return TRACE_TYPE_PARTIAL_LINE
;
897 if (!(trace_flags
& TRACE_ITER_CONTEXT_INFO
))
901 if (flags
& TRACE_GRAPH_PRINT_ABS_TIME
) {
902 ret
= print_graph_abs_time(iter
->ts
, s
);
904 return TRACE_TYPE_PARTIAL_LINE
;
908 if (flags
& TRACE_GRAPH_PRINT_CPU
) {
909 ret
= print_graph_cpu(s
, cpu
);
910 if (ret
== TRACE_TYPE_PARTIAL_LINE
)
911 return TRACE_TYPE_PARTIAL_LINE
;
915 if (flags
& TRACE_GRAPH_PRINT_PROC
) {
916 ret
= print_graph_proc(s
, ent
->pid
);
917 if (ret
== TRACE_TYPE_PARTIAL_LINE
)
918 return TRACE_TYPE_PARTIAL_LINE
;
920 ret
= trace_seq_puts(s
, " | ");
922 return TRACE_TYPE_PARTIAL_LINE
;
926 if (trace_flags
& TRACE_ITER_LATENCY_FMT
) {
927 ret
= print_graph_lat_fmt(s
, ent
);
928 if (ret
== TRACE_TYPE_PARTIAL_LINE
)
929 return TRACE_TYPE_PARTIAL_LINE
;
936 * Entry check for irq code
939 * - we are inside irq code
940 * - we just entered irq code
943 * - funcgraph-interrupts option is set
944 * - we are not inside irq code
947 check_irq_entry(struct trace_iterator
*iter
, u32 flags
,
948 unsigned long addr
, int depth
)
952 struct fgraph_data
*data
= iter
->private;
955 * If we are either displaying irqs, or we got called as
956 * a graph event and private data does not exist,
957 * then we bypass the irq check.
959 if ((flags
& TRACE_GRAPH_PRINT_IRQS
) ||
963 depth_irq
= &(per_cpu_ptr(data
->cpu_data
, cpu
)->depth_irq
);
966 * We are inside the irq code
971 if ((addr
< (unsigned long)__irqentry_text_start
) ||
972 (addr
>= (unsigned long)__irqentry_text_end
))
976 * We are entering irq code.
983 * Return check for irq code
986 * - we are inside irq code
987 * - we just left irq code
990 * - funcgraph-interrupts option is set
991 * - we are not inside irq code
994 check_irq_return(struct trace_iterator
*iter
, u32 flags
, int depth
)
998 struct fgraph_data
*data
= iter
->private;
1001 * If we are either displaying irqs, or we got called as
1002 * a graph event and private data does not exist,
1003 * then we bypass the irq check.
1005 if ((flags
& TRACE_GRAPH_PRINT_IRQS
) ||
1009 depth_irq
= &(per_cpu_ptr(data
->cpu_data
, cpu
)->depth_irq
);
1012 * We are not inside the irq code.
1014 if (*depth_irq
== -1)
1018 * We are inside the irq code, and this is returning entry.
1019 * Let's not trace it and clear the entry depth, since
1020 * we are out of irq code.
1022 * This condition ensures that we 'leave the irq code' once
1023 * we are out of the entry depth. Thus protecting us from
1024 * the RETURN entry loss.
1026 if (*depth_irq
>= depth
) {
1032 * We are inside the irq code, and this is not the entry.
1037 static enum print_line_t
1038 print_graph_entry(struct ftrace_graph_ent_entry
*field
, struct trace_seq
*s
,
1039 struct trace_iterator
*iter
, u32 flags
)
1041 struct fgraph_data
*data
= iter
->private;
1042 struct ftrace_graph_ent
*call
= &field
->graph_ent
;
1043 struct ftrace_graph_ret_entry
*leaf_ret
;
1044 static enum print_line_t ret
;
1045 int cpu
= iter
->cpu
;
1047 if (check_irq_entry(iter
, flags
, call
->func
, call
->depth
))
1048 return TRACE_TYPE_HANDLED
;
1050 if (print_graph_prologue(iter
, s
, TRACE_GRAPH_ENT
, call
->func
, flags
))
1051 return TRACE_TYPE_PARTIAL_LINE
;
1053 leaf_ret
= get_return_for_leaf(iter
, field
);
1055 ret
= print_graph_entry_leaf(iter
, field
, leaf_ret
, s
, flags
);
1057 ret
= print_graph_entry_nested(iter
, field
, s
, cpu
, flags
);
1061 * If we failed to write our output, then we need to make
1062 * note of it. Because we already consumed our entry.
1074 static enum print_line_t
1075 print_graph_return(struct ftrace_graph_ret
*trace
, struct trace_seq
*s
,
1076 struct trace_entry
*ent
, struct trace_iterator
*iter
,
1079 unsigned long long duration
= trace
->rettime
- trace
->calltime
;
1080 struct fgraph_data
*data
= iter
->private;
1081 pid_t pid
= ent
->pid
;
1082 int cpu
= iter
->cpu
;
1087 if (check_irq_return(iter
, flags
, trace
->depth
))
1088 return TRACE_TYPE_HANDLED
;
1091 struct fgraph_cpu_data
*cpu_data
;
1092 int cpu
= iter
->cpu
;
1094 cpu_data
= per_cpu_ptr(data
->cpu_data
, cpu
);
1097 * Comments display at + 1 to depth. This is the
1098 * return from a function, we now want the comments
1099 * to display at the same level of the bracket.
1101 cpu_data
->depth
= trace
->depth
- 1;
1103 if (trace
->depth
< FTRACE_RETFUNC_DEPTH
) {
1104 if (cpu_data
->enter_funcs
[trace
->depth
] != trace
->func
)
1106 cpu_data
->enter_funcs
[trace
->depth
] = 0;
1110 if (print_graph_prologue(iter
, s
, 0, 0, flags
))
1111 return TRACE_TYPE_PARTIAL_LINE
;
1113 /* Overhead and duration */
1114 ret
= print_graph_duration(duration
, s
, flags
);
1115 if (ret
== TRACE_TYPE_PARTIAL_LINE
)
1116 return TRACE_TYPE_PARTIAL_LINE
;
1119 for (i
= 0; i
< trace
->depth
* TRACE_GRAPH_INDENT
; i
++) {
1120 ret
= trace_seq_putc(s
, ' ');
1122 return TRACE_TYPE_PARTIAL_LINE
;
1126 * If the return function does not have a matching entry,
1127 * then the entry was lost. Instead of just printing
1128 * the '}' and letting the user guess what function this
1129 * belongs to, write out the function name.
1132 ret
= trace_seq_puts(s
, "}\n");
1134 return TRACE_TYPE_PARTIAL_LINE
;
1136 ret
= trace_seq_printf(s
, "} /* %ps */\n", (void *)trace
->func
);
1138 return TRACE_TYPE_PARTIAL_LINE
;
1142 if (flags
& TRACE_GRAPH_PRINT_OVERRUN
) {
1143 ret
= trace_seq_printf(s
, " (Overruns: %lu)\n",
1146 return TRACE_TYPE_PARTIAL_LINE
;
1149 ret
= print_graph_irq(iter
, trace
->func
, TRACE_GRAPH_RET
,
1151 if (ret
== TRACE_TYPE_PARTIAL_LINE
)
1152 return TRACE_TYPE_PARTIAL_LINE
;
1154 return TRACE_TYPE_HANDLED
;
1157 static enum print_line_t
1158 print_graph_comment(struct trace_seq
*s
, struct trace_entry
*ent
,
1159 struct trace_iterator
*iter
, u32 flags
)
1161 unsigned long sym_flags
= (trace_flags
& TRACE_ITER_SYM_MASK
);
1162 struct fgraph_data
*data
= iter
->private;
1163 struct trace_event
*event
;
1169 depth
= per_cpu_ptr(data
->cpu_data
, iter
->cpu
)->depth
;
1171 if (print_graph_prologue(iter
, s
, 0, 0, flags
))
1172 return TRACE_TYPE_PARTIAL_LINE
;
1175 ret
= print_graph_duration(DURATION_FILL_FULL
, s
, flags
);
1176 if (ret
!= TRACE_TYPE_HANDLED
)
1181 for (i
= 0; i
< (depth
+ 1) * TRACE_GRAPH_INDENT
; i
++) {
1182 ret
= trace_seq_putc(s
, ' ');
1184 return TRACE_TYPE_PARTIAL_LINE
;
1188 ret
= trace_seq_puts(s
, "/* ");
1190 return TRACE_TYPE_PARTIAL_LINE
;
1192 switch (iter
->ent
->type
) {
1194 ret
= trace_print_bprintk_msg_only(iter
);
1195 if (ret
!= TRACE_TYPE_HANDLED
)
1199 ret
= trace_print_printk_msg_only(iter
);
1200 if (ret
!= TRACE_TYPE_HANDLED
)
1204 event
= ftrace_find_event(ent
->type
);
1206 return TRACE_TYPE_UNHANDLED
;
1208 ret
= event
->funcs
->trace(iter
, sym_flags
, event
);
1209 if (ret
!= TRACE_TYPE_HANDLED
)
1213 /* Strip ending newline */
1214 if (s
->buffer
[s
->len
- 1] == '\n') {
1215 s
->buffer
[s
->len
- 1] = '\0';
1219 ret
= trace_seq_puts(s
, " */\n");
1221 return TRACE_TYPE_PARTIAL_LINE
;
1223 return TRACE_TYPE_HANDLED
;
1228 print_graph_function_flags(struct trace_iterator
*iter
, u32 flags
)
1230 struct ftrace_graph_ent_entry
*field
;
1231 struct fgraph_data
*data
= iter
->private;
1232 struct trace_entry
*entry
= iter
->ent
;
1233 struct trace_seq
*s
= &iter
->seq
;
1234 int cpu
= iter
->cpu
;
1237 if (data
&& per_cpu_ptr(data
->cpu_data
, cpu
)->ignore
) {
1238 per_cpu_ptr(data
->cpu_data
, cpu
)->ignore
= 0;
1239 return TRACE_TYPE_HANDLED
;
1243 * If the last output failed, there's a possibility we need
1244 * to print out the missing entry which would never go out.
1246 if (data
&& data
->failed
) {
1248 iter
->cpu
= data
->cpu
;
1249 ret
= print_graph_entry(field
, s
, iter
, flags
);
1250 if (ret
== TRACE_TYPE_HANDLED
&& iter
->cpu
!= cpu
) {
1251 per_cpu_ptr(data
->cpu_data
, iter
->cpu
)->ignore
= 1;
1252 ret
= TRACE_TYPE_NO_CONSUME
;
1258 switch (entry
->type
) {
1259 case TRACE_GRAPH_ENT
: {
1261 * print_graph_entry() may consume the current event,
1262 * thus @field may become invalid, so we need to save it.
1263 * sizeof(struct ftrace_graph_ent_entry) is very small,
1264 * it can be safely saved at the stack.
1266 struct ftrace_graph_ent_entry saved
;
1267 trace_assign_type(field
, entry
);
1269 return print_graph_entry(&saved
, s
, iter
, flags
);
1271 case TRACE_GRAPH_RET
: {
1272 struct ftrace_graph_ret_entry
*field
;
1273 trace_assign_type(field
, entry
);
1274 return print_graph_return(&field
->ret
, s
, entry
, iter
, flags
);
1278 /* dont trace stack and functions as comments */
1279 return TRACE_TYPE_UNHANDLED
;
1282 return print_graph_comment(s
, entry
, iter
, flags
);
1285 return TRACE_TYPE_HANDLED
;
1288 static enum print_line_t
1289 print_graph_function(struct trace_iterator
*iter
)
1291 return print_graph_function_flags(iter
, tracer_flags
.val
);
1294 static enum print_line_t
1295 print_graph_function_event(struct trace_iterator
*iter
, int flags
,
1296 struct trace_event
*event
)
1298 return print_graph_function(iter
);
1301 static void print_lat_header(struct seq_file
*s
, u32 flags
)
1303 static const char spaces
[] = " " /* 16 spaces */
1305 " "; /* 17 spaces */
1308 if (flags
& TRACE_GRAPH_PRINT_ABS_TIME
)
1310 if (flags
& TRACE_GRAPH_PRINT_CPU
)
1312 if (flags
& TRACE_GRAPH_PRINT_PROC
)
1315 seq_printf(s
, "#%.*s _-----=> irqs-off \n", size
, spaces
);
1316 seq_printf(s
, "#%.*s / _----=> need-resched \n", size
, spaces
);
1317 seq_printf(s
, "#%.*s| / _---=> hardirq/softirq \n", size
, spaces
);
1318 seq_printf(s
, "#%.*s|| / _--=> preempt-depth \n", size
, spaces
);
1319 seq_printf(s
, "#%.*s||| / \n", size
, spaces
);
1322 static void __print_graph_headers_flags(struct seq_file
*s
, u32 flags
)
1324 int lat
= trace_flags
& TRACE_ITER_LATENCY_FMT
;
1327 print_lat_header(s
, flags
);
1331 if (flags
& TRACE_GRAPH_PRINT_ABS_TIME
)
1332 seq_printf(s
, " TIME ");
1333 if (flags
& TRACE_GRAPH_PRINT_CPU
)
1334 seq_printf(s
, " CPU");
1335 if (flags
& TRACE_GRAPH_PRINT_PROC
)
1336 seq_printf(s
, " TASK/PID ");
1338 seq_printf(s
, "||||");
1339 if (flags
& TRACE_GRAPH_PRINT_DURATION
)
1340 seq_printf(s
, " DURATION ");
1341 seq_printf(s
, " FUNCTION CALLS\n");
1345 if (flags
& TRACE_GRAPH_PRINT_ABS_TIME
)
1346 seq_printf(s
, " | ");
1347 if (flags
& TRACE_GRAPH_PRINT_CPU
)
1348 seq_printf(s
, " | ");
1349 if (flags
& TRACE_GRAPH_PRINT_PROC
)
1350 seq_printf(s
, " | | ");
1352 seq_printf(s
, "||||");
1353 if (flags
& TRACE_GRAPH_PRINT_DURATION
)
1354 seq_printf(s
, " | | ");
1355 seq_printf(s
, " | | | |\n");
1358 void print_graph_headers(struct seq_file
*s
)
1360 print_graph_headers_flags(s
, tracer_flags
.val
);
1363 void print_graph_headers_flags(struct seq_file
*s
, u32 flags
)
1365 struct trace_iterator
*iter
= s
->private;
1367 if (!(trace_flags
& TRACE_ITER_CONTEXT_INFO
))
1370 if (trace_flags
& TRACE_ITER_LATENCY_FMT
) {
1371 /* print nothing if the buffers are empty */
1372 if (trace_empty(iter
))
1375 print_trace_header(s
, iter
);
1378 __print_graph_headers_flags(s
, flags
);
1381 void graph_trace_open(struct trace_iterator
*iter
)
1383 /* pid and depth on the last trace processed */
1384 struct fgraph_data
*data
;
1387 iter
->private = NULL
;
1389 data
= kzalloc(sizeof(*data
), GFP_KERNEL
);
1393 data
->cpu_data
= alloc_percpu(struct fgraph_cpu_data
);
1394 if (!data
->cpu_data
)
1397 for_each_possible_cpu(cpu
) {
1398 pid_t
*pid
= &(per_cpu_ptr(data
->cpu_data
, cpu
)->last_pid
);
1399 int *depth
= &(per_cpu_ptr(data
->cpu_data
, cpu
)->depth
);
1400 int *ignore
= &(per_cpu_ptr(data
->cpu_data
, cpu
)->ignore
);
1401 int *depth_irq
= &(per_cpu_ptr(data
->cpu_data
, cpu
)->depth_irq
);
1409 iter
->private = data
;
1416 pr_warning("function graph tracer: not enough memory\n");
1419 void graph_trace_close(struct trace_iterator
*iter
)
1421 struct fgraph_data
*data
= iter
->private;
1424 free_percpu(data
->cpu_data
);
1429 static int func_graph_set_flag(u32 old_flags
, u32 bit
, int set
)
1431 if (bit
== TRACE_GRAPH_PRINT_IRQS
)
1432 ftrace_graph_skip_irqs
= !set
;
1437 static struct trace_event_functions graph_functions
= {
1438 .trace
= print_graph_function_event
,
1441 static struct trace_event graph_trace_entry_event
= {
1442 .type
= TRACE_GRAPH_ENT
,
1443 .funcs
= &graph_functions
,
1446 static struct trace_event graph_trace_ret_event
= {
1447 .type
= TRACE_GRAPH_RET
,
1448 .funcs
= &graph_functions
1451 static struct tracer graph_trace __tracer_data
= {
1452 .name
= "function_graph",
1453 .open
= graph_trace_open
,
1454 .pipe_open
= graph_trace_open
,
1455 .close
= graph_trace_close
,
1456 .pipe_close
= graph_trace_close
,
1457 .wait_pipe
= poll_wait_pipe
,
1458 .init
= graph_trace_init
,
1459 .reset
= graph_trace_reset
,
1460 .print_line
= print_graph_function
,
1461 .print_header
= print_graph_headers
,
1462 .flags
= &tracer_flags
,
1463 .set_flag
= func_graph_set_flag
,
1464 #ifdef CONFIG_FTRACE_SELFTEST
1465 .selftest
= trace_selftest_startup_function_graph
,
1471 graph_depth_write(struct file
*filp
, const char __user
*ubuf
, size_t cnt
,
1477 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
1489 graph_depth_read(struct file
*filp
, char __user
*ubuf
, size_t cnt
,
1492 char buf
[15]; /* More than enough to hold UINT_MAX + "\n"*/
1495 n
= sprintf(buf
, "%d\n", max_depth
);
1497 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, n
);
1500 static const struct file_operations graph_depth_fops
= {
1501 .open
= tracing_open_generic
,
1502 .write
= graph_depth_write
,
1503 .read
= graph_depth_read
,
1504 .llseek
= generic_file_llseek
,
1507 static __init
int init_graph_debugfs(void)
1509 struct dentry
*d_tracer
;
1511 d_tracer
= tracing_init_dentry();
1515 trace_create_file("max_graph_depth", 0644, d_tracer
,
1516 NULL
, &graph_depth_fops
);
1520 fs_initcall(init_graph_debugfs
);
1522 static __init
int init_graph_trace(void)
1524 max_bytes_for_cpu
= snprintf(NULL
, 0, "%d", nr_cpu_ids
- 1);
1526 if (!register_ftrace_event(&graph_trace_entry_event
)) {
1527 pr_warning("Warning: could not register graph trace events\n");
1531 if (!register_ftrace_event(&graph_trace_ret_event
)) {
1532 pr_warning("Warning: could not register graph trace events\n");
1536 return register_tracer(&graph_trace
);
1539 core_initcall(init_graph_trace
);