kernel/trace/trace_sched_wakeup.c

   1 /*
   2  * trace task wakeup timings
   3  *
   4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5  * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
   6  *
   7  * Based on code from the latency_tracer, that is:
   8  *
   9  *  Copyright (C) 2004-2006 Ingo Molnar
  10  *  Copyright (C) 2004 Nadia Yvette Chambers
  11  */
  12 #include <linux/module.h>
  13 #include <linux/fs.h>
  14 #include <linux/debugfs.h>
  15 #include <linux/kallsyms.h>
  16 #include <linux/uaccess.h>
  17 #include <linux/ftrace.h>
  18 #include <linux/sched/rt.h>
  19 #include <trace/events/sched.h>
  20 #include "trace.h"
  21
  22 static struct trace_array       *wakeup_trace;
  23 static int __read_mostly        tracer_enabled;
  24
  25 static struct task_struct       *wakeup_task;
  26 static int                      wakeup_cpu;
  27 static int                      wakeup_current_cpu;
  28 static unsigned                 wakeup_prio = -1;
  29 static int                      wakeup_rt;
  30
  31 static arch_spinlock_t wakeup_lock =
  32         (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
  33
  34 static void wakeup_reset(struct trace_array *tr);
  35 static void __wakeup_reset(struct trace_array *tr);
  36 static int wakeup_graph_entry(struct ftrace_graph_ent *trace);
  37 static void wakeup_graph_return(struct ftrace_graph_ret *trace);
  38
  39 static int save_flags;
  40 static bool function_enabled;
  41
  42 #define TRACE_DISPLAY_GRAPH     1
  43
  44 static struct tracer_opt trace_opts[] = {
  45 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  46         /* display latency trace as call graph */
  47         { TRACER_OPT(display-graph, TRACE_DISPLAY_GRAPH) },
  48 #endif
  49         { } /* Empty entry */
  50 };
  51
  52 static struct tracer_flags tracer_flags = {
  53         .val  = 0,
  54         .opts = trace_opts,
  55 };
  56
  57 #define is_graph() (tracer_flags.val & TRACE_DISPLAY_GRAPH)
  58
  59 #ifdef CONFIG_FUNCTION_TRACER
  60
  61 /*
  62  * Prologue for the wakeup function tracers.
  63  *
  64  * Returns 1 if it is OK to continue, and preemption
  65  *            is disabled and data->disabled is incremented.
  66  *         0 if the trace is to be ignored, and preemption
  67  *            is not disabled and data->disabled is
  68  *            kept the same.
  69  *
  70  * Note, this function is also used outside this ifdef but
  71  *  inside the #ifdef of the function graph tracer below.
  72  *  This is OK, since the function graph tracer is
  73  *  dependent on the function tracer.
  74  */
  75 static int
  76 func_prolog_preempt_disable(struct trace_array *tr,
  77                             struct trace_array_cpu **data,
  78                             int *pc)
  79 {
  80         long disabled;
  81         int cpu;
  82
  83         if (likely(!wakeup_task))
  84                 return 0;
  85
  86         *pc = preempt_count();
  87         preempt_disable_notrace();
  88
  89         cpu = raw_smp_processor_id();
  90         if (cpu != wakeup_current_cpu)
  91                 goto out_enable;
  92
  93         *data = per_cpu_ptr(tr->trace_buffer.data, cpu);
  94         disabled = atomic_inc_return(&(*data)->disabled);
  95         if (unlikely(disabled != 1))
  96                 goto out;
  97
  98         return 1;
  99
 100 out:
 101         atomic_dec(&(*data)->disabled);
 102
 103 out_enable:
 104         preempt_enable_notrace();
 105         return 0;
 106 }
 107
 108 /*
 109  * wakeup uses its own tracer function to keep the overhead down:
 110  */
 111 static void
 112 wakeup_tracer_call(unsigned long ip, unsigned long parent_ip,
 113                    struct ftrace_ops *op, struct pt_regs *pt_regs)
 114 {
 115         struct trace_array *tr = wakeup_trace;
 116         struct trace_array_cpu *data;
 117         unsigned long flags;
 118         int pc;
 119
 120         if (!func_prolog_preempt_disable(tr, &data, &pc))
 121                 return;
 122
 123         local_irq_save(flags);
 124         trace_function(tr, ip, parent_ip, flags, pc);
 125         local_irq_restore(flags);
 126
 127         atomic_dec(&data->disabled);
 128         preempt_enable_notrace();
 129 }
 130
 131 static struct ftrace_ops trace_ops __read_mostly =
 132 {
 133         .func = wakeup_tracer_call,
 134         .flags = FTRACE_OPS_FL_GLOBAL | FTRACE_OPS_FL_RECURSION_SAFE,
 135 };
 136 #endif /* CONFIG_FUNCTION_TRACER */
 137
 138 static int register_wakeup_function(int graph, int set)
 139 {
 140         int ret;
 141
 142         /* 'set' is set if TRACE_ITER_FUNCTION is about to be set */
 143         if (function_enabled || (!set && !(trace_flags & TRACE_ITER_FUNCTION)))
 144                 return 0;
 145
 146         if (graph)
 147                 ret = register_ftrace_graph(&wakeup_graph_return,
 148                                             &wakeup_graph_entry);
 149         else
 150                 ret = register_ftrace_function(&trace_ops);
 151
 152         if (!ret)
 153                 function_enabled = true;
 154
 155         return ret;
 156 }
 157
 158 static void unregister_wakeup_function(int graph)
 159 {
 160         if (!function_enabled)
 161                 return;
 162
 163         if (graph)
 164                 unregister_ftrace_graph();
 165         else
 166                 unregister_ftrace_function(&trace_ops);
 167
 168         function_enabled = false;
 169 }
 170
 171 static void wakeup_function_set(int set)
 172 {
 173         if (set)
 174                 register_wakeup_function(is_graph(), 1);
 175         else
 176                 unregister_wakeup_function(is_graph());
 177 }
 178
 179 static int wakeup_flag_changed(struct tracer *tracer, u32 mask, int set)
 180 {
 181         if (mask & TRACE_ITER_FUNCTION)
 182                 wakeup_function_set(set);
 183
 184         return trace_keep_overwrite(tracer, mask, set);
 185 }
 186
 187 static int start_func_tracer(int graph)
 188 {
 189         int ret;
 190
 191         ret = register_wakeup_function(graph, 0);
 192
 193         if (!ret && tracing_is_enabled())
 194                 tracer_enabled = 1;
 195         else
 196                 tracer_enabled = 0;
 197
 198         return ret;
 199 }
 200
 201 static void stop_func_tracer(int graph)
 202 {
 203         tracer_enabled = 0;
 204
 205         unregister_wakeup_function(graph);
 206 }
 207
 208 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 209 static int wakeup_set_flag(u32 old_flags, u32 bit, int set)
 210 {
 211
 212         if (!(bit & TRACE_DISPLAY_GRAPH))
 213                 return -EINVAL;
 214
 215         if (!(is_graph() ^ set))
 216                 return 0;
 217
 218         stop_func_tracer(!set);
 219
 220         wakeup_reset(wakeup_trace);
 221         tracing_max_latency = 0;
 222
 223         return start_func_tracer(set);
 224 }
 225
 226 static int wakeup_graph_entry(struct ftrace_graph_ent *trace)
 227 {
 228         struct trace_array *tr = wakeup_trace;
 229         struct trace_array_cpu *data;
 230         unsigned long flags;
 231         int pc, ret = 0;
 232
 233         if (!func_prolog_preempt_disable(tr, &data, &pc))
 234                 return 0;
 235
 236         local_save_flags(flags);
 237         ret = __trace_graph_entry(tr, trace, flags, pc);
 238         atomic_dec(&data->disabled);
 239         preempt_enable_notrace();
 240
 241         return ret;
 242 }
 243
 244 static void wakeup_graph_return(struct ftrace_graph_ret *trace)
 245 {
 246         struct trace_array *tr = wakeup_trace;
 247         struct trace_array_cpu *data;
 248         unsigned long flags;
 249         int pc;
 250
 251         if (!func_prolog_preempt_disable(tr, &data, &pc))
 252                 return;
 253
 254         local_save_flags(flags);
 255         __trace_graph_return(tr, trace, flags, pc);
 256         atomic_dec(&data->disabled);
 257
 258         preempt_enable_notrace();
 259         return;
 260 }
 261
 262 static void wakeup_trace_open(struct trace_iterator *iter)
 263 {
 264         if (is_graph())
 265                 graph_trace_open(iter);
 266 }
 267
 268 static void wakeup_trace_close(struct trace_iterator *iter)
 269 {
 270         if (iter->private)
 271                 graph_trace_close(iter);
 272 }
 273
 274 #define GRAPH_TRACER_FLAGS (TRACE_GRAPH_PRINT_PROC | \
 275                             TRACE_GRAPH_PRINT_ABS_TIME | \
 276                             TRACE_GRAPH_PRINT_DURATION)
 277
 278 static enum print_line_t wakeup_print_line(struct trace_iterator *iter)
 279 {
 280         /*
 281          * In graph mode call the graph tracer output function,
 282          * otherwise go with the TRACE_FN event handler
 283          */
 284         if (is_graph())
 285                 return print_graph_function_flags(iter, GRAPH_TRACER_FLAGS);
 286
 287         return TRACE_TYPE_UNHANDLED;
 288 }
 289
 290 static void wakeup_print_header(struct seq_file *s)
 291 {
 292         if (is_graph())
 293                 print_graph_headers_flags(s, GRAPH_TRACER_FLAGS);
 294         else
 295                 trace_default_header(s);
 296 }
 297
 298 static void
 299 __trace_function(struct trace_array *tr,
 300                  unsigned long ip, unsigned long parent_ip,
 301                  unsigned long flags, int pc)
 302 {
 303         if (is_graph())
 304                 trace_graph_function(tr, ip, parent_ip, flags, pc);
 305         else
 306                 trace_function(tr, ip, parent_ip, flags, pc);
 307 }
 308 #else
 309 #define __trace_function trace_function
 310
 311 static int wakeup_set_flag(u32 old_flags, u32 bit, int set)
 312 {
 313         return -EINVAL;
 314 }
 315
 316 static int wakeup_graph_entry(struct ftrace_graph_ent *trace)
 317 {
 318         return -1;
 319 }
 320
 321 static enum print_line_t wakeup_print_line(struct trace_iterator *iter)
 322 {
 323         return TRACE_TYPE_UNHANDLED;
 324 }
 325
 326 static void wakeup_graph_return(struct ftrace_graph_ret *trace) { }
 327 static void wakeup_trace_open(struct trace_iterator *iter) { }
 328 static void wakeup_trace_close(struct trace_iterator *iter) { }
 329
 330 #ifdef CONFIG_FUNCTION_TRACER
 331 static void wakeup_print_header(struct seq_file *s)
 332 {
 333         trace_default_header(s);
 334 }
 335 #else
 336 static void wakeup_print_header(struct seq_file *s)
 337 {
 338         trace_latency_header(s);
 339 }
 340 #endif /* CONFIG_FUNCTION_TRACER */
 341 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
 342
 343 /*
 344  * Should this new latency be reported/recorded?
 345  */
 346 static int report_latency(cycle_t delta)
 347 {
 348         if (tracing_thresh) {
 349                 if (delta < tracing_thresh)
 350                         return 0;
 351         } else {
 352                 if (delta <= tracing_max_latency)
 353                         return 0;
 354         }
 355         return 1;
 356 }
 357
 358 static void
 359 probe_wakeup_migrate_task(void *ignore, struct task_struct *task, int cpu)
 360 {
 361         if (task != wakeup_task)
 362                 return;
 363
 364         wakeup_current_cpu = cpu;
 365 }
 366
 367 static void notrace
 368 probe_wakeup_sched_switch(void *ignore,
 369                           struct task_struct *prev, struct task_struct *next)
 370 {
 371         struct trace_array_cpu *data;
 372         cycle_t T0, T1, delta;
 373         unsigned long flags;
 374         long disabled;
 375         int cpu;
 376         int pc;
 377
 378         tracing_record_cmdline(prev);
 379
 380         if (unlikely(!tracer_enabled))
 381                 return;
 382
 383         /*
 384          * When we start a new trace, we set wakeup_task to NULL
 385          * and then set tracer_enabled = 1. We want to make sure
 386          * that another CPU does not see the tracer_enabled = 1
 387          * and the wakeup_task with an older task, that might
 388          * actually be the same as next.
 389          */
 390         smp_rmb();
 391
 392         if (next != wakeup_task)
 393                 return;
 394
 395         pc = preempt_count();
 396
 397         /* disable local data, not wakeup_cpu data */
 398         cpu = raw_smp_processor_id();
 399         disabled = atomic_inc_return(&per_cpu_ptr(wakeup_trace->trace_buffer.data, cpu)->disabled);
 400         if (likely(disabled != 1))
 401                 goto out;
 402
 403         local_irq_save(flags);
 404         arch_spin_lock(&wakeup_lock);
 405
 406         /* We could race with grabbing wakeup_lock */
 407         if (unlikely(!tracer_enabled || next != wakeup_task))
 408                 goto out_unlock;
 409
 410         /* The task we are waiting for is waking up */
 411         data = per_cpu_ptr(wakeup_trace->trace_buffer.data, wakeup_cpu);
 412
 413         __trace_function(wakeup_trace, CALLER_ADDR0, CALLER_ADDR1, flags, pc);
 414         tracing_sched_switch_trace(wakeup_trace, prev, next, flags, pc);
 415
 416         T0 = data->preempt_timestamp;
 417         T1 = ftrace_now(cpu);
 418         delta = T1-T0;
 419
 420         if (!report_latency(delta))
 421                 goto out_unlock;
 422
 423         if (likely(!is_tracing_stopped())) {
 424                 tracing_max_latency = delta;
 425                 update_max_tr(wakeup_trace, wakeup_task, wakeup_cpu);
 426         }
 427
 428 out_unlock:
 429         __wakeup_reset(wakeup_trace);
 430         arch_spin_unlock(&wakeup_lock);
 431         local_irq_restore(flags);
 432 out:
 433         atomic_dec(&per_cpu_ptr(wakeup_trace->trace_buffer.data, cpu)->disabled);
 434 }
 435
 436 static void __wakeup_reset(struct trace_array *tr)
 437 {
 438         wakeup_cpu = -1;
 439         wakeup_prio = -1;
 440
 441         if (wakeup_task)
 442                 put_task_struct(wakeup_task);
 443
 444         wakeup_task = NULL;
 445 }
 446
 447 static void wakeup_reset(struct trace_array *tr)
 448 {
 449         unsigned long flags;
 450
 451         tracing_reset_online_cpus(&tr->trace_buffer);
 452
 453         local_irq_save(flags);
 454         arch_spin_lock(&wakeup_lock);
 455         __wakeup_reset(tr);
 456         arch_spin_unlock(&wakeup_lock);
 457         local_irq_restore(flags);
 458 }
 459
 460 static void
 461 probe_wakeup(void *ignore, struct task_struct *p, int success)
 462 {
 463         struct trace_array_cpu *data;
 464         int cpu = smp_processor_id();
 465         unsigned long flags;
 466         long disabled;
 467         int pc;
 468
 469         if (likely(!tracer_enabled))
 470                 return;
 471
 472         tracing_record_cmdline(p);
 473         tracing_record_cmdline(current);
 474
 475         if ((wakeup_rt && !rt_task(p)) ||
 476                         p->prio >= wakeup_prio ||
 477                         p->prio >= current->prio)
 478                 return;
 479
 480         pc = preempt_count();
 481         disabled = atomic_inc_return(&per_cpu_ptr(wakeup_trace->trace_buffer.data, cpu)->disabled);
 482         if (unlikely(disabled != 1))
 483                 goto out;
 484
 485         /* interrupts should be off from try_to_wake_up */
 486         arch_spin_lock(&wakeup_lock);
 487
 488         /* check for races. */
 489         if (!tracer_enabled || p->prio >= wakeup_prio)
 490                 goto out_locked;
 491
 492         /* reset the trace */
 493         __wakeup_reset(wakeup_trace);
 494
 495         wakeup_cpu = task_cpu(p);
 496         wakeup_current_cpu = wakeup_cpu;
 497         wakeup_prio = p->prio;
 498
 499         wakeup_task = p;
 500         get_task_struct(wakeup_task);
 501
 502         local_save_flags(flags);
 503
 504         data = per_cpu_ptr(wakeup_trace->trace_buffer.data, wakeup_cpu);
 505         data->preempt_timestamp = ftrace_now(cpu);
 506         tracing_sched_wakeup_trace(wakeup_trace, p, current, flags, pc);
 507
 508         /*
 509          * We must be careful in using CALLER_ADDR2. But since wake_up
 510          * is not called by an assembly function  (where as schedule is)
 511          * it should be safe to use it here.
 512          */
 513         __trace_function(wakeup_trace, CALLER_ADDR1, CALLER_ADDR2, flags, pc);
 514
 515 out_locked:
 516         arch_spin_unlock(&wakeup_lock);
 517 out:
 518         atomic_dec(&per_cpu_ptr(wakeup_trace->trace_buffer.data, cpu)->disabled);
 519 }
 520
 521 static void start_wakeup_tracer(struct trace_array *tr)
 522 {
 523         int ret;
 524
 525         ret = register_trace_sched_wakeup(probe_wakeup, NULL);
 526         if (ret) {
 527                 pr_info("wakeup trace: Couldn't activate tracepoint"
 528                         " probe to kernel_sched_wakeup\n");
 529                 return;
 530         }
 531
 532         ret = register_trace_sched_wakeup_new(probe_wakeup, NULL);
 533         if (ret) {
 534                 pr_info("wakeup trace: Couldn't activate tracepoint"
 535                         " probe to kernel_sched_wakeup_new\n");
 536                 goto fail_deprobe;
 537         }
 538
 539         ret = register_trace_sched_switch(probe_wakeup_sched_switch, NULL);
 540         if (ret) {
 541                 pr_info("sched trace: Couldn't activate tracepoint"
 542                         " probe to kernel_sched_switch\n");
 543                 goto fail_deprobe_wake_new;
 544         }
 545
 546         ret = register_trace_sched_migrate_task(probe_wakeup_migrate_task, NULL);
 547         if (ret) {
 548                 pr_info("wakeup trace: Couldn't activate tracepoint"
 549                         " probe to kernel_sched_migrate_task\n");
 550                 return;
 551         }
 552
 553         wakeup_reset(tr);
 554
 555         /*
 556          * Don't let the tracer_enabled = 1 show up before
 557          * the wakeup_task is reset. This may be overkill since
 558          * wakeup_reset does a spin_unlock after setting the
 559          * wakeup_task to NULL, but I want to be safe.
 560          * This is a slow path anyway.
 561          */
 562         smp_wmb();
 563
 564         if (start_func_tracer(is_graph()))
 565                 printk(KERN_ERR "failed to start wakeup tracer\n");
 566
 567         return;
 568 fail_deprobe_wake_new:
 569         unregister_trace_sched_wakeup_new(probe_wakeup, NULL);
 570 fail_deprobe:
 571         unregister_trace_sched_wakeup(probe_wakeup, NULL);
 572 }
 573
 574 static void stop_wakeup_tracer(struct trace_array *tr)
 575 {
 576         tracer_enabled = 0;
 577         stop_func_tracer(is_graph());
 578         unregister_trace_sched_switch(probe_wakeup_sched_switch, NULL);
 579         unregister_trace_sched_wakeup_new(probe_wakeup, NULL);
 580         unregister_trace_sched_wakeup(probe_wakeup, NULL);
 581         unregister_trace_sched_migrate_task(probe_wakeup_migrate_task, NULL);
 582 }
 583
 584 static int __wakeup_tracer_init(struct trace_array *tr)
 585 {
 586         save_flags = trace_flags;
 587
 588         /* non overwrite screws up the latency tracers */
 589         set_tracer_flag(tr, TRACE_ITER_OVERWRITE, 1);
 590         set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, 1);
 591
 592         tracing_max_latency = 0;
 593         wakeup_trace = tr;
 594         start_wakeup_tracer(tr);
 595         return 0;
 596 }
 597
 598 static int wakeup_tracer_init(struct trace_array *tr)
 599 {
 600         wakeup_rt = 0;
 601         return __wakeup_tracer_init(tr);
 602 }
 603
 604 static int wakeup_rt_tracer_init(struct trace_array *tr)
 605 {
 606         wakeup_rt = 1;
 607         return __wakeup_tracer_init(tr);
 608 }
 609
 610 static void wakeup_tracer_reset(struct trace_array *tr)
 611 {
 612         int lat_flag = save_flags & TRACE_ITER_LATENCY_FMT;
 613         int overwrite_flag = save_flags & TRACE_ITER_OVERWRITE;
 614
 615         stop_wakeup_tracer(tr);
 616         /* make sure we put back any tasks we are tracing */
 617         wakeup_reset(tr);
 618
 619         set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, lat_flag);
 620         set_tracer_flag(tr, TRACE_ITER_OVERWRITE, overwrite_flag);
 621 }
 622
 623 static void wakeup_tracer_start(struct trace_array *tr)
 624 {
 625         wakeup_reset(tr);
 626         tracer_enabled = 1;
 627 }
 628
 629 static void wakeup_tracer_stop(struct trace_array *tr)
 630 {
 631         tracer_enabled = 0;
 632 }
 633
 634 static struct tracer wakeup_tracer __read_mostly =
 635 {
 636         .name           = "wakeup",
 637         .init           = wakeup_tracer_init,
 638         .reset          = wakeup_tracer_reset,
 639         .start          = wakeup_tracer_start,
 640         .stop           = wakeup_tracer_stop,
 641         .print_max      = true,
 642         .print_header   = wakeup_print_header,
 643         .print_line     = wakeup_print_line,
 644         .flags          = &tracer_flags,
 645         .set_flag       = wakeup_set_flag,
 646         .flag_changed   = wakeup_flag_changed,
 647 #ifdef CONFIG_FTRACE_SELFTEST
 648         .selftest    = trace_selftest_startup_wakeup,
 649 #endif
 650         .open           = wakeup_trace_open,
 651         .close          = wakeup_trace_close,
 652         .use_max_tr     = true,
 653 };
 654
 655 static struct tracer wakeup_rt_tracer __read_mostly =
 656 {
 657         .name           = "wakeup_rt",
 658         .init           = wakeup_rt_tracer_init,
 659         .reset          = wakeup_tracer_reset,
 660         .start          = wakeup_tracer_start,
 661         .stop           = wakeup_tracer_stop,
 662         .wait_pipe      = poll_wait_pipe,
 663         .print_max      = true,
 664         .print_header   = wakeup_print_header,
 665         .print_line     = wakeup_print_line,
 666         .flags          = &tracer_flags,
 667         .set_flag       = wakeup_set_flag,
 668         .flag_changed   = wakeup_flag_changed,
 669 #ifdef CONFIG_FTRACE_SELFTEST
 670         .selftest    = trace_selftest_startup_wakeup,
 671 #endif
 672         .open           = wakeup_trace_open,
 673         .close          = wakeup_trace_close,
 674         .use_max_tr     = true,
 675 };
 676
 677 __init static int init_wakeup_tracer(void)
 678 {
 679         int ret;
 680
 681         ret = register_tracer(&wakeup_tracer);
 682         if (ret)
 683                 return ret;
 684
 685         ret = register_tracer(&wakeup_rt_tracer);
 686         if (ret)
 687                 return ret;
 688
 689         return 0;
 690 }
 691 core_initcall(init_wakeup_tracer);