kernel/trace/trace_functions.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * ring buffer based function tracer
   4  *
   5  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   6  * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
   7  *
   8  * Based on code from the latency_tracer, that is:
   9  *
  10  *  Copyright (C) 2004-2006 Ingo Molnar
  11  *  Copyright (C) 2004 Nadia Yvette Chambers
  12  */
  13 #include <linux/ring_buffer.h>
  14 #include <linux/debugfs.h>
  15 #include <linux/uaccess.h>
  16 #include <linux/ftrace.h>
  17 #include <linux/slab.h>
  18 #include <linux/fs.h>
  19
  20 #include "trace.h"
  21
  22 static void tracing_start_function_trace(struct trace_array *tr);
  23 static void tracing_stop_function_trace(struct trace_array *tr);
  24 static void
  25 function_trace_call(unsigned long ip, unsigned long parent_ip,
  26                     struct ftrace_ops *op, struct ftrace_regs *fregs);
  27 static void
  28 function_stack_trace_call(unsigned long ip, unsigned long parent_ip,
  29                           struct ftrace_ops *op, struct ftrace_regs *fregs);
  30 static void
  31 function_no_repeats_trace_call(unsigned long ip, unsigned long parent_ip,
  32                                struct ftrace_ops *op, struct ftrace_regs *fregs);
  33 static void
  34 function_stack_no_repeats_trace_call(unsigned long ip, unsigned long parent_ip,
  35                                      struct ftrace_ops *op,
  36                                      struct ftrace_regs *fregs);
  37 static struct tracer_flags func_flags;
  38
  39 /* Our option */
  40 enum {
  41
  42         TRACE_FUNC_NO_OPTS              = 0x0, /* No flags set. */
  43         TRACE_FUNC_OPT_STACK            = 0x1,
  44         TRACE_FUNC_OPT_NO_REPEATS       = 0x2,
  45
  46         /* Update this to next highest bit. */
  47         TRACE_FUNC_OPT_HIGHEST_BIT      = 0x4
  48 };
  49
  50 #define TRACE_FUNC_OPT_MASK     (TRACE_FUNC_OPT_HIGHEST_BIT - 1)
  51
  52 int ftrace_allocate_ftrace_ops(struct trace_array *tr)
  53 {
  54         struct ftrace_ops *ops;
  55
  56         /* The top level array uses the "global_ops" */
  57         if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
  58                 return 0;
  59
  60         ops = kzalloc(sizeof(*ops), GFP_KERNEL);
  61         if (!ops)
  62                 return -ENOMEM;
  63
  64         /* Currently only the non stack version is supported */
  65         ops->func = function_trace_call;
  66         ops->flags = FTRACE_OPS_FL_PID;
  67
  68         tr->ops = ops;
  69         ops->private = tr;
  70
  71         return 0;
  72 }
  73
  74 void ftrace_free_ftrace_ops(struct trace_array *tr)
  75 {
  76         kfree(tr->ops);
  77         tr->ops = NULL;
  78 }
  79
  80 int ftrace_create_function_files(struct trace_array *tr,
  81                                  struct dentry *parent)
  82 {
  83         int ret;
  84         /*
  85          * The top level array uses the "global_ops", and the files are
  86          * created on boot up.
  87          */
  88         if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
  89                 return 0;
  90
  91         if (!tr->ops)
  92                 return -EINVAL;
  93
  94         ret = allocate_fgraph_ops(tr, tr->ops);
  95         if (ret) {
  96                 kfree(tr->ops);
  97                 return ret;
  98         }
  99
 100         ftrace_create_filter_files(tr->ops, parent);
 101
 102         return 0;
 103 }
 104
 105 void ftrace_destroy_function_files(struct trace_array *tr)
 106 {
 107         ftrace_destroy_filter_files(tr->ops);
 108         ftrace_free_ftrace_ops(tr);
 109         free_fgraph_ops(tr);
 110 }
 111
 112 static ftrace_func_t select_trace_function(u32 flags_val)
 113 {
 114         switch (flags_val & TRACE_FUNC_OPT_MASK) {
 115         case TRACE_FUNC_NO_OPTS:
 116                 return function_trace_call;
 117         case TRACE_FUNC_OPT_STACK:
 118                 return function_stack_trace_call;
 119         case TRACE_FUNC_OPT_NO_REPEATS:
 120                 return function_no_repeats_trace_call;
 121         case TRACE_FUNC_OPT_STACK | TRACE_FUNC_OPT_NO_REPEATS:
 122                 return function_stack_no_repeats_trace_call;
 123         default:
 124                 return NULL;
 125         }
 126 }
 127
 128 static bool handle_func_repeats(struct trace_array *tr, u32 flags_val)
 129 {
 130         if (!tr->last_func_repeats &&
 131             (flags_val & TRACE_FUNC_OPT_NO_REPEATS)) {
 132                 tr->last_func_repeats = alloc_percpu(struct trace_func_repeats);
 133                 if (!tr->last_func_repeats)
 134                         return false;
 135         }
 136
 137         return true;
 138 }
 139
 140 static int function_trace_init(struct trace_array *tr)
 141 {
 142         ftrace_func_t func;
 143         /*
 144          * Instance trace_arrays get their ops allocated
 145          * at instance creation. Unless it failed
 146          * the allocation.
 147          */
 148         if (!tr->ops)
 149                 return -ENOMEM;
 150
 151         func = select_trace_function(func_flags.val);
 152         if (!func)
 153                 return -EINVAL;
 154
 155         if (!handle_func_repeats(tr, func_flags.val))
 156                 return -ENOMEM;
 157
 158         ftrace_init_array_ops(tr, func);
 159
 160         tr->array_buffer.cpu = raw_smp_processor_id();
 161
 162         tracing_start_cmdline_record();
 163         tracing_start_function_trace(tr);
 164         return 0;
 165 }
 166
 167 static void function_trace_reset(struct trace_array *tr)
 168 {
 169         tracing_stop_function_trace(tr);
 170         tracing_stop_cmdline_record();
 171         ftrace_reset_array_ops(tr);
 172 }
 173
 174 static void function_trace_start(struct trace_array *tr)
 175 {
 176         tracing_reset_online_cpus(&tr->array_buffer);
 177 }
 178
 179 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 180 static __always_inline unsigned long
 181 function_get_true_parent_ip(unsigned long parent_ip, struct ftrace_regs *fregs)
 182 {
 183         unsigned long true_parent_ip;
 184         int idx = 0;
 185
 186         true_parent_ip = parent_ip;
 187         if (unlikely(parent_ip == (unsigned long)&return_to_handler) && fregs)
 188                 true_parent_ip = ftrace_graph_ret_addr(current, &idx, parent_ip,
 189                                 (unsigned long *)ftrace_regs_get_stack_pointer(fregs));
 190         return true_parent_ip;
 191 }
 192 #else
 193 static __always_inline unsigned long
 194 function_get_true_parent_ip(unsigned long parent_ip, struct ftrace_regs *fregs)
 195 {
 196         return parent_ip;
 197 }
 198 #endif
 199
 200 static void
 201 function_trace_call(unsigned long ip, unsigned long parent_ip,
 202                     struct ftrace_ops *op, struct ftrace_regs *fregs)
 203 {
 204         struct trace_array *tr = op->private;
 205         struct trace_array_cpu *data;
 206         unsigned int trace_ctx;
 207         int bit;
 208
 209         if (unlikely(!tr->function_enabled))
 210                 return;
 211
 212         bit = ftrace_test_recursion_trylock(ip, parent_ip);
 213         if (bit < 0)
 214                 return;
 215
 216         parent_ip = function_get_true_parent_ip(parent_ip, fregs);
 217
 218         trace_ctx = tracing_gen_ctx();
 219
 220         data = this_cpu_ptr(tr->array_buffer.data);
 221         if (!atomic_read(&data->disabled))
 222                 trace_function(tr, ip, parent_ip, trace_ctx);
 223
 224         ftrace_test_recursion_unlock(bit);
 225 }
 226
 227 #ifdef CONFIG_UNWINDER_ORC
 228 /*
 229  * Skip 2:
 230  *
 231  *   function_stack_trace_call()
 232  *   ftrace_call()
 233  */
 234 #define STACK_SKIP 2
 235 #else
 236 /*
 237  * Skip 3:
 238  *   __trace_stack()
 239  *   function_stack_trace_call()
 240  *   ftrace_call()
 241  */
 242 #define STACK_SKIP 3
 243 #endif
 244
 245 static void
 246 function_stack_trace_call(unsigned long ip, unsigned long parent_ip,
 247                           struct ftrace_ops *op, struct ftrace_regs *fregs)
 248 {
 249         struct trace_array *tr = op->private;
 250         struct trace_array_cpu *data;
 251         unsigned long flags;
 252         long disabled;
 253         int cpu;
 254         unsigned int trace_ctx;
 255         int skip = STACK_SKIP;
 256
 257         if (unlikely(!tr->function_enabled))
 258                 return;
 259
 260         /*
 261          * Need to use raw, since this must be called before the
 262          * recursive protection is performed.
 263          */
 264         local_irq_save(flags);
 265         parent_ip = function_get_true_parent_ip(parent_ip, fregs);
 266         cpu = raw_smp_processor_id();
 267         data = per_cpu_ptr(tr->array_buffer.data, cpu);
 268         disabled = atomic_inc_return(&data->disabled);
 269
 270         if (likely(disabled == 1)) {
 271                 trace_ctx = tracing_gen_ctx_flags(flags);
 272                 trace_function(tr, ip, parent_ip, trace_ctx);
 273 #ifdef CONFIG_UNWINDER_FRAME_POINTER
 274                 if (ftrace_pids_enabled(op))
 275                         skip++;
 276 #endif
 277                 __trace_stack(tr, trace_ctx, skip);
 278         }
 279
 280         atomic_dec(&data->disabled);
 281         local_irq_restore(flags);
 282 }
 283
 284 static inline bool is_repeat_check(struct trace_array *tr,
 285                                    struct trace_func_repeats *last_info,
 286                                    unsigned long ip, unsigned long parent_ip)
 287 {
 288         if (last_info->ip == ip &&
 289             last_info->parent_ip == parent_ip &&
 290             last_info->count < U16_MAX) {
 291                 last_info->ts_last_call =
 292                         ring_buffer_time_stamp(tr->array_buffer.buffer);
 293                 last_info->count++;
 294                 return true;
 295         }
 296
 297         return false;
 298 }
 299
 300 static inline void process_repeats(struct trace_array *tr,
 301                                    unsigned long ip, unsigned long parent_ip,
 302                                    struct trace_func_repeats *last_info,
 303                                    unsigned int trace_ctx)
 304 {
 305         if (last_info->count) {
 306                 trace_last_func_repeats(tr, last_info, trace_ctx);
 307                 last_info->count = 0;
 308         }
 309
 310         last_info->ip = ip;
 311         last_info->parent_ip = parent_ip;
 312 }
 313
 314 static void
 315 function_no_repeats_trace_call(unsigned long ip, unsigned long parent_ip,
 316                                struct ftrace_ops *op,
 317                                struct ftrace_regs *fregs)
 318 {
 319         struct trace_func_repeats *last_info;
 320         struct trace_array *tr = op->private;
 321         struct trace_array_cpu *data;
 322         unsigned int trace_ctx;
 323         unsigned long flags;
 324         int bit;
 325
 326         if (unlikely(!tr->function_enabled))
 327                 return;
 328
 329         bit = ftrace_test_recursion_trylock(ip, parent_ip);
 330         if (bit < 0)
 331                 return;
 332
 333         parent_ip = function_get_true_parent_ip(parent_ip, fregs);
 334         data = this_cpu_ptr(tr->array_buffer.data);
 335         if (atomic_read(&data->disabled))
 336                 goto out;
 337
 338         /*
 339          * An interrupt may happen at any place here. But as far as I can see,
 340          * the only damage that this can cause is to mess up the repetition
 341          * counter without valuable data being lost.
 342          * TODO: think about a solution that is better than just hoping to be
 343          * lucky.
 344          */
 345         last_info = this_cpu_ptr(tr->last_func_repeats);
 346         if (is_repeat_check(tr, last_info, ip, parent_ip))
 347                 goto out;
 348
 349         local_save_flags(flags);
 350         trace_ctx = tracing_gen_ctx_flags(flags);
 351         process_repeats(tr, ip, parent_ip, last_info, trace_ctx);
 352
 353         trace_function(tr, ip, parent_ip, trace_ctx);
 354
 355 out:
 356         ftrace_test_recursion_unlock(bit);
 357 }
 358
 359 static void
 360 function_stack_no_repeats_trace_call(unsigned long ip, unsigned long parent_ip,
 361                                      struct ftrace_ops *op,
 362                                      struct ftrace_regs *fregs)
 363 {
 364         struct trace_func_repeats *last_info;
 365         struct trace_array *tr = op->private;
 366         struct trace_array_cpu *data;
 367         unsigned long flags;
 368         long disabled;
 369         int cpu;
 370         unsigned int trace_ctx;
 371
 372         if (unlikely(!tr->function_enabled))
 373                 return;
 374
 375         /*
 376          * Need to use raw, since this must be called before the
 377          * recursive protection is performed.
 378          */
 379         local_irq_save(flags);
 380         parent_ip = function_get_true_parent_ip(parent_ip, fregs);
 381         cpu = raw_smp_processor_id();
 382         data = per_cpu_ptr(tr->array_buffer.data, cpu);
 383         disabled = atomic_inc_return(&data->disabled);
 384
 385         if (likely(disabled == 1)) {
 386                 last_info = per_cpu_ptr(tr->last_func_repeats, cpu);
 387                 if (is_repeat_check(tr, last_info, ip, parent_ip))
 388                         goto out;
 389
 390                 trace_ctx = tracing_gen_ctx_flags(flags);
 391                 process_repeats(tr, ip, parent_ip, last_info, trace_ctx);
 392
 393                 trace_function(tr, ip, parent_ip, trace_ctx);
 394                 __trace_stack(tr, trace_ctx, STACK_SKIP);
 395         }
 396
 397  out:
 398         atomic_dec(&data->disabled);
 399         local_irq_restore(flags);
 400 }
 401
 402 static struct tracer_opt func_opts[] = {
 403 #ifdef CONFIG_STACKTRACE
 404         { TRACER_OPT(func_stack_trace, TRACE_FUNC_OPT_STACK) },
 405 #endif
 406         { TRACER_OPT(func-no-repeats, TRACE_FUNC_OPT_NO_REPEATS) },
 407         { } /* Always set a last empty entry */
 408 };
 409
 410 static struct tracer_flags func_flags = {
 411         .val = TRACE_FUNC_NO_OPTS, /* By default: all flags disabled */
 412         .opts = func_opts
 413 };
 414
 415 static void tracing_start_function_trace(struct trace_array *tr)
 416 {
 417         tr->function_enabled = 0;
 418         register_ftrace_function(tr->ops);
 419         tr->function_enabled = 1;
 420 }
 421
 422 static void tracing_stop_function_trace(struct trace_array *tr)
 423 {
 424         tr->function_enabled = 0;
 425         unregister_ftrace_function(tr->ops);
 426 }
 427
 428 static struct tracer function_trace;
 429
 430 static int
 431 func_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
 432 {
 433         ftrace_func_t func;
 434         u32 new_flags;
 435
 436         /* Do nothing if already set. */
 437         if (!!set == !!(func_flags.val & bit))
 438                 return 0;
 439
 440         /* We can change this flag only when not running. */
 441         if (tr->current_trace != &function_trace)
 442                 return 0;
 443
 444         new_flags = (func_flags.val & ~bit) | (set ? bit : 0);
 445         func = select_trace_function(new_flags);
 446         if (!func)
 447                 return -EINVAL;
 448
 449         /* Check if there's anything to change. */
 450         if (tr->ops->func == func)
 451                 return 0;
 452
 453         if (!handle_func_repeats(tr, new_flags))
 454                 return -ENOMEM;
 455
 456         unregister_ftrace_function(tr->ops);
 457         tr->ops->func = func;
 458         register_ftrace_function(tr->ops);
 459
 460         return 0;
 461 }
 462
 463 static struct tracer function_trace __tracer_data =
 464 {
 465         .name           = "function",
 466         .init           = function_trace_init,
 467         .reset          = function_trace_reset,
 468         .start          = function_trace_start,
 469         .flags          = &func_flags,
 470         .set_flag       = func_set_flag,
 471         .allow_instances = true,
 472 #ifdef CONFIG_FTRACE_SELFTEST
 473         .selftest       = trace_selftest_startup_function,
 474 #endif
 475 };
 476
 477 #ifdef CONFIG_DYNAMIC_FTRACE
 478 static void update_traceon_count(struct ftrace_probe_ops *ops,
 479                                  unsigned long ip,
 480                                  struct trace_array *tr, bool on,
 481                                  void *data)
 482 {
 483         struct ftrace_func_mapper *mapper = data;
 484         long *count;
 485         long old_count;
 486
 487         /*
 488          * Tracing gets disabled (or enabled) once per count.
 489          * This function can be called at the same time on multiple CPUs.
 490          * It is fine if both disable (or enable) tracing, as disabling
 491          * (or enabling) the second time doesn't do anything as the
 492          * state of the tracer is already disabled (or enabled).
 493          * What needs to be synchronized in this case is that the count
 494          * only gets decremented once, even if the tracer is disabled
 495          * (or enabled) twice, as the second one is really a nop.
 496          *
 497          * The memory barriers guarantee that we only decrement the
 498          * counter once. First the count is read to a local variable
 499          * and a read barrier is used to make sure that it is loaded
 500          * before checking if the tracer is in the state we want.
 501          * If the tracer is not in the state we want, then the count
 502          * is guaranteed to be the old count.
 503          *
 504          * Next the tracer is set to the state we want (disabled or enabled)
 505          * then a write memory barrier is used to make sure that
 506          * the new state is visible before changing the counter by
 507          * one minus the old counter. This guarantees that another CPU
 508          * executing this code will see the new state before seeing
 509          * the new counter value, and would not do anything if the new
 510          * counter is seen.
 511          *
 512          * Note, there is no synchronization between this and a user
 513          * setting the tracing_on file. But we currently don't care
 514          * about that.
 515          */
 516         count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
 517         old_count = *count;
 518
 519         if (old_count <= 0)
 520                 return;
 521
 522         /* Make sure we see count before checking tracing state */
 523         smp_rmb();
 524
 525         if (on == !!tracer_tracing_is_on(tr))
 526                 return;
 527
 528         if (on)
 529                 tracer_tracing_on(tr);
 530         else
 531                 tracer_tracing_off(tr);
 532
 533         /* Make sure tracing state is visible before updating count */
 534         smp_wmb();
 535
 536         *count = old_count - 1;
 537 }
 538
 539 static void
 540 ftrace_traceon_count(unsigned long ip, unsigned long parent_ip,
 541                      struct trace_array *tr, struct ftrace_probe_ops *ops,
 542                      void *data)
 543 {
 544         update_traceon_count(ops, ip, tr, 1, data);
 545 }
 546
 547 static void
 548 ftrace_traceoff_count(unsigned long ip, unsigned long parent_ip,
 549                       struct trace_array *tr, struct ftrace_probe_ops *ops,
 550                       void *data)
 551 {
 552         update_traceon_count(ops, ip, tr, 0, data);
 553 }
 554
 555 static void
 556 ftrace_traceon(unsigned long ip, unsigned long parent_ip,
 557                struct trace_array *tr, struct ftrace_probe_ops *ops,
 558                void *data)
 559 {
 560         if (tracer_tracing_is_on(tr))
 561                 return;
 562
 563         tracer_tracing_on(tr);
 564 }
 565
 566 static void
 567 ftrace_traceoff(unsigned long ip, unsigned long parent_ip,
 568                 struct trace_array *tr, struct ftrace_probe_ops *ops,
 569                 void *data)
 570 {
 571         if (!tracer_tracing_is_on(tr))
 572                 return;
 573
 574         tracer_tracing_off(tr);
 575 }
 576
 577 #ifdef CONFIG_UNWINDER_ORC
 578 /*
 579  * Skip 3:
 580  *
 581  *   function_trace_probe_call()
 582  *   ftrace_ops_assist_func()
 583  *   ftrace_call()
 584  */
 585 #define FTRACE_STACK_SKIP 3
 586 #else
 587 /*
 588  * Skip 5:
 589  *
 590  *   __trace_stack()
 591  *   ftrace_stacktrace()
 592  *   function_trace_probe_call()
 593  *   ftrace_ops_assist_func()
 594  *   ftrace_call()
 595  */
 596 #define FTRACE_STACK_SKIP 5
 597 #endif
 598
 599 static __always_inline void trace_stack(struct trace_array *tr)
 600 {
 601         unsigned int trace_ctx;
 602
 603         trace_ctx = tracing_gen_ctx();
 604
 605         __trace_stack(tr, trace_ctx, FTRACE_STACK_SKIP);
 606 }
 607
 608 static void
 609 ftrace_stacktrace(unsigned long ip, unsigned long parent_ip,
 610                   struct trace_array *tr, struct ftrace_probe_ops *ops,
 611                   void *data)
 612 {
 613         trace_stack(tr);
 614 }
 615
 616 static void
 617 ftrace_stacktrace_count(unsigned long ip, unsigned long parent_ip,
 618                         struct trace_array *tr, struct ftrace_probe_ops *ops,
 619                         void *data)
 620 {
 621         struct ftrace_func_mapper *mapper = data;
 622         long *count;
 623         long old_count;
 624         long new_count;
 625
 626         if (!tracing_is_on())
 627                 return;
 628
 629         /* unlimited? */
 630         if (!mapper) {
 631                 trace_stack(tr);
 632                 return;
 633         }
 634
 635         count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
 636
 637         /*
 638          * Stack traces should only execute the number of times the
 639          * user specified in the counter.
 640          */
 641         do {
 642                 old_count = *count;
 643
 644                 if (!old_count)
 645                         return;
 646
 647                 new_count = old_count - 1;
 648                 new_count = cmpxchg(count, old_count, new_count);
 649                 if (new_count == old_count)
 650                         trace_stack(tr);
 651
 652                 if (!tracing_is_on())
 653                         return;
 654
 655         } while (new_count != old_count);
 656 }
 657
 658 static int update_count(struct ftrace_probe_ops *ops, unsigned long ip,
 659                         void *data)
 660 {
 661         struct ftrace_func_mapper *mapper = data;
 662         long *count = NULL;
 663
 664         if (mapper)
 665                 count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
 666
 667         if (count) {
 668                 if (*count <= 0)
 669                         return 0;
 670                 (*count)--;
 671         }
 672
 673         return 1;
 674 }
 675
 676 static void
 677 ftrace_dump_probe(unsigned long ip, unsigned long parent_ip,
 678                   struct trace_array *tr, struct ftrace_probe_ops *ops,
 679                   void *data)
 680 {
 681         if (update_count(ops, ip, data))
 682                 ftrace_dump(DUMP_ALL);
 683 }
 684
 685 /* Only dump the current CPU buffer. */
 686 static void
 687 ftrace_cpudump_probe(unsigned long ip, unsigned long parent_ip,
 688                      struct trace_array *tr, struct ftrace_probe_ops *ops,
 689                      void *data)
 690 {
 691         if (update_count(ops, ip, data))
 692                 ftrace_dump(DUMP_ORIG);
 693 }
 694
 695 static int
 696 ftrace_probe_print(const char *name, struct seq_file *m,
 697                    unsigned long ip, struct ftrace_probe_ops *ops,
 698                    void *data)
 699 {
 700         struct ftrace_func_mapper *mapper = data;
 701         long *count = NULL;
 702
 703         seq_printf(m, "%ps:%s", (void *)ip, name);
 704
 705         if (mapper)
 706                 count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
 707
 708         if (count)
 709                 seq_printf(m, ":count=%ld\n", *count);
 710         else
 711                 seq_puts(m, ":unlimited\n");
 712
 713         return 0;
 714 }
 715
 716 static int
 717 ftrace_traceon_print(struct seq_file *m, unsigned long ip,
 718                      struct ftrace_probe_ops *ops,
 719                      void *data)
 720 {
 721         return ftrace_probe_print("traceon", m, ip, ops, data);
 722 }
 723
 724 static int
 725 ftrace_traceoff_print(struct seq_file *m, unsigned long ip,
 726                          struct ftrace_probe_ops *ops, void *data)
 727 {
 728         return ftrace_probe_print("traceoff", m, ip, ops, data);
 729 }
 730
 731 static int
 732 ftrace_stacktrace_print(struct seq_file *m, unsigned long ip,
 733                         struct ftrace_probe_ops *ops, void *data)
 734 {
 735         return ftrace_probe_print("stacktrace", m, ip, ops, data);
 736 }
 737
 738 static int
 739 ftrace_dump_print(struct seq_file *m, unsigned long ip,
 740                         struct ftrace_probe_ops *ops, void *data)
 741 {
 742         return ftrace_probe_print("dump", m, ip, ops, data);
 743 }
 744
 745 static int
 746 ftrace_cpudump_print(struct seq_file *m, unsigned long ip,
 747                         struct ftrace_probe_ops *ops, void *data)
 748 {
 749         return ftrace_probe_print("cpudump", m, ip, ops, data);
 750 }
 751
 752
 753 static int
 754 ftrace_count_init(struct ftrace_probe_ops *ops, struct trace_array *tr,
 755                   unsigned long ip, void *init_data, void **data)
 756 {
 757         struct ftrace_func_mapper *mapper = *data;
 758
 759         if (!mapper) {
 760                 mapper = allocate_ftrace_func_mapper();
 761                 if (!mapper)
 762                         return -ENOMEM;
 763                 *data = mapper;
 764         }
 765
 766         return ftrace_func_mapper_add_ip(mapper, ip, init_data);
 767 }
 768
 769 static void
 770 ftrace_count_free(struct ftrace_probe_ops *ops, struct trace_array *tr,
 771                   unsigned long ip, void *data)
 772 {
 773         struct ftrace_func_mapper *mapper = data;
 774
 775         if (!ip) {
 776                 free_ftrace_func_mapper(mapper, NULL);
 777                 return;
 778         }
 779
 780         ftrace_func_mapper_remove_ip(mapper, ip);
 781 }
 782
 783 static struct ftrace_probe_ops traceon_count_probe_ops = {
 784         .func                   = ftrace_traceon_count,
 785         .print                  = ftrace_traceon_print,
 786         .init                   = ftrace_count_init,
 787         .free                   = ftrace_count_free,
 788 };
 789
 790 static struct ftrace_probe_ops traceoff_count_probe_ops = {
 791         .func                   = ftrace_traceoff_count,
 792         .print                  = ftrace_traceoff_print,
 793         .init                   = ftrace_count_init,
 794         .free                   = ftrace_count_free,
 795 };
 796
 797 static struct ftrace_probe_ops stacktrace_count_probe_ops = {
 798         .func                   = ftrace_stacktrace_count,
 799         .print                  = ftrace_stacktrace_print,
 800         .init                   = ftrace_count_init,
 801         .free                   = ftrace_count_free,
 802 };
 803
 804 static struct ftrace_probe_ops dump_probe_ops = {
 805         .func                   = ftrace_dump_probe,
 806         .print                  = ftrace_dump_print,
 807         .init                   = ftrace_count_init,
 808         .free                   = ftrace_count_free,
 809 };
 810
 811 static struct ftrace_probe_ops cpudump_probe_ops = {
 812         .func                   = ftrace_cpudump_probe,
 813         .print                  = ftrace_cpudump_print,
 814 };
 815
 816 static struct ftrace_probe_ops traceon_probe_ops = {
 817         .func                   = ftrace_traceon,
 818         .print                  = ftrace_traceon_print,
 819 };
 820
 821 static struct ftrace_probe_ops traceoff_probe_ops = {
 822         .func                   = ftrace_traceoff,
 823         .print                  = ftrace_traceoff_print,
 824 };
 825
 826 static struct ftrace_probe_ops stacktrace_probe_ops = {
 827         .func                   = ftrace_stacktrace,
 828         .print                  = ftrace_stacktrace_print,
 829 };
 830
 831 static int
 832 ftrace_trace_probe_callback(struct trace_array *tr,
 833                             struct ftrace_probe_ops *ops,
 834                             struct ftrace_hash *hash, char *glob,
 835                             char *cmd, char *param, int enable)
 836 {
 837         void *count = (void *)-1;
 838         char *number;
 839         int ret;
 840
 841         /* hash funcs only work with set_ftrace_filter */
 842         if (!enable)
 843                 return -EINVAL;
 844
 845         if (glob[0] == '!')
 846                 return unregister_ftrace_function_probe_func(glob+1, tr, ops);
 847
 848         if (!param)
 849                 goto out_reg;
 850
 851         number = strsep(&param, ":");
 852
 853         if (!strlen(number))
 854                 goto out_reg;
 855
 856         /*
 857          * We use the callback data field (which is a pointer)
 858          * as our counter.
 859          */
 860         ret = kstrtoul(number, 0, (unsigned long *)&count);
 861         if (ret)
 862                 return ret;
 863
 864  out_reg:
 865         ret = register_ftrace_function_probe(glob, tr, ops, count);
 866
 867         return ret < 0 ? ret : 0;
 868 }
 869
 870 static int
 871 ftrace_trace_onoff_callback(struct trace_array *tr, struct ftrace_hash *hash,
 872                             char *glob, char *cmd, char *param, int enable)
 873 {
 874         struct ftrace_probe_ops *ops;
 875
 876         if (!tr)
 877                 return -ENODEV;
 878
 879         /* we register both traceon and traceoff to this callback */
 880         if (strcmp(cmd, "traceon") == 0)
 881                 ops = param ? &traceon_count_probe_ops : &traceon_probe_ops;
 882         else
 883                 ops = param ? &traceoff_count_probe_ops : &traceoff_probe_ops;
 884
 885         return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd,
 886                                            param, enable);
 887 }
 888
 889 static int
 890 ftrace_stacktrace_callback(struct trace_array *tr, struct ftrace_hash *hash,
 891                            char *glob, char *cmd, char *param, int enable)
 892 {
 893         struct ftrace_probe_ops *ops;
 894
 895         if (!tr)
 896                 return -ENODEV;
 897
 898         ops = param ? &stacktrace_count_probe_ops : &stacktrace_probe_ops;
 899
 900         return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd,
 901                                            param, enable);
 902 }
 903
 904 static int
 905 ftrace_dump_callback(struct trace_array *tr, struct ftrace_hash *hash,
 906                            char *glob, char *cmd, char *param, int enable)
 907 {
 908         struct ftrace_probe_ops *ops;
 909
 910         if (!tr)
 911                 return -ENODEV;
 912
 913         ops = &dump_probe_ops;
 914
 915         /* Only dump once. */
 916         return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd,
 917                                            "1", enable);
 918 }
 919
 920 static int
 921 ftrace_cpudump_callback(struct trace_array *tr, struct ftrace_hash *hash,
 922                            char *glob, char *cmd, char *param, int enable)
 923 {
 924         struct ftrace_probe_ops *ops;
 925
 926         if (!tr)
 927                 return -ENODEV;
 928
 929         ops = &cpudump_probe_ops;
 930
 931         /* Only dump once. */
 932         return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd,
 933                                            "1", enable);
 934 }
 935
 936 static struct ftrace_func_command ftrace_traceon_cmd = {
 937         .name                   = "traceon",
 938         .func                   = ftrace_trace_onoff_callback,
 939 };
 940
 941 static struct ftrace_func_command ftrace_traceoff_cmd = {
 942         .name                   = "traceoff",
 943         .func                   = ftrace_trace_onoff_callback,
 944 };
 945
 946 static struct ftrace_func_command ftrace_stacktrace_cmd = {
 947         .name                   = "stacktrace",
 948         .func                   = ftrace_stacktrace_callback,
 949 };
 950
 951 static struct ftrace_func_command ftrace_dump_cmd = {
 952         .name                   = "dump",
 953         .func                   = ftrace_dump_callback,
 954 };
 955
 956 static struct ftrace_func_command ftrace_cpudump_cmd = {
 957         .name                   = "cpudump",
 958         .func                   = ftrace_cpudump_callback,
 959 };
 960
 961 static int __init init_func_cmd_traceon(void)
 962 {
 963         int ret;
 964
 965         ret = register_ftrace_command(&ftrace_traceoff_cmd);
 966         if (ret)
 967                 return ret;
 968
 969         ret = register_ftrace_command(&ftrace_traceon_cmd);
 970         if (ret)
 971                 goto out_free_traceoff;
 972
 973         ret = register_ftrace_command(&ftrace_stacktrace_cmd);
 974         if (ret)
 975                 goto out_free_traceon;
 976
 977         ret = register_ftrace_command(&ftrace_dump_cmd);
 978         if (ret)
 979                 goto out_free_stacktrace;
 980
 981         ret = register_ftrace_command(&ftrace_cpudump_cmd);
 982         if (ret)
 983                 goto out_free_dump;
 984
 985         return 0;
 986
 987  out_free_dump:
 988         unregister_ftrace_command(&ftrace_dump_cmd);
 989  out_free_stacktrace:
 990         unregister_ftrace_command(&ftrace_stacktrace_cmd);
 991  out_free_traceon:
 992         unregister_ftrace_command(&ftrace_traceon_cmd);
 993  out_free_traceoff:
 994         unregister_ftrace_command(&ftrace_traceoff_cmd);
 995
 996         return ret;
 997 }
 998 #else
 999 static inline int init_func_cmd_traceon(void)
1000 {
1001         return 0;
1002 }
1003 #endif /* CONFIG_DYNAMIC_FTRACE */
1004
1005 __init int init_function_trace(void)
1006 {
1007         init_func_cmd_traceon();
1008         return register_tracer(&function_trace);
1009 }