arch/powerpc/kernel/ftrace.c

   1 /*
   2  * Code for replacing ftrace calls with jumps.
   3  *
   4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5  *
   6  * Thanks goes out to P.A. Semi, Inc for supplying me with a PPC64 box.
   7  *
   8  * Added function graph tracer code, taken from x86 that was written
   9  * by Frederic Weisbecker, and ported to PPC by Steven Rostedt.
  10  *
  11  */
  12
  13 #include <linux/spinlock.h>
  14 #include <linux/hardirq.h>
  15 #include <linux/uaccess.h>
  16 #include <linux/module.h>
  17 #include <linux/ftrace.h>
  18 #include <linux/percpu.h>
  19 #include <linux/init.h>
  20 #include <linux/list.h>
  21
  22 #include <asm/cacheflush.h>
  23 #include <asm/code-patching.h>
  24 #include <asm/ftrace.h>
  25
  26
  27 #ifdef CONFIG_DYNAMIC_FTRACE
  28 static unsigned int
  29 ftrace_call_replace(unsigned long ip, unsigned long addr, int link)
  30 {
  31         unsigned int op;
  32
  33         addr = ppc_function_entry((void *)addr);
  34
  35         /* if (link) set op to 'bl' else 'b' */
  36         op = create_branch((unsigned int *)ip, addr, link ? 1 : 0);
  37
  38         return op;
  39 }
  40
  41 static int
  42 ftrace_modify_code(unsigned long ip, unsigned int old, unsigned int new)
  43 {
  44         unsigned int replaced;
  45
  46         /*
  47          * Note: Due to modules and __init, code can
  48          *  disappear and change, we need to protect against faulting
  49          *  as well as code changing. We do this by using the
  50          *  probe_kernel_* functions.
  51          *
  52          * No real locking needed, this code is run through
  53          * kstop_machine, or before SMP starts.
  54          */
  55
  56         /* read the text we want to modify */
  57         if (probe_kernel_read(&replaced, (void *)ip, MCOUNT_INSN_SIZE))
  58                 return -EFAULT;
  59
  60         /* Make sure it is what we expect it to be */
  61         if (replaced != old)
  62                 return -EINVAL;
  63
  64         /* replace the text with the new text */
  65         if (probe_kernel_write((void *)ip, &new, MCOUNT_INSN_SIZE))
  66                 return -EPERM;
  67
  68         flush_icache_range(ip, ip + 8);
  69
  70         return 0;
  71 }
  72
  73 /*
  74  * Helper functions that are the same for both PPC64 and PPC32.
  75  */
  76 static int test_24bit_addr(unsigned long ip, unsigned long addr)
  77 {
  78
  79         /* use the create_branch to verify that this offset can be branched */
  80         return create_branch((unsigned int *)ip, addr, 0);
  81 }
  82
  83 #ifdef CONFIG_MODULES
  84
  85 static int is_bl_op(unsigned int op)
  86 {
  87         return (op & 0xfc000003) == 0x48000001;
  88 }
  89
  90 static unsigned long find_bl_target(unsigned long ip, unsigned int op)
  91 {
  92         static int offset;
  93
  94         offset = (op & 0x03fffffc);
  95         /* make it signed */
  96         if (offset & 0x02000000)
  97                 offset |= 0xfe000000;
  98
  99         return ip + (long)offset;
 100 }
 101
 102 #ifdef CONFIG_PPC64
 103 static int
 104 __ftrace_make_nop(struct module *mod,
 105                   struct dyn_ftrace *rec, unsigned long addr)
 106 {
 107         unsigned int op;
 108         unsigned int jmp[5];
 109         unsigned long ptr;
 110         unsigned long ip = rec->ip;
 111         unsigned long tramp;
 112         int offset;
 113
 114         /* read where this goes */
 115         if (probe_kernel_read(&op, (void *)ip, sizeof(int)))
 116                 return -EFAULT;
 117
 118         /* Make sure that that this is still a 24bit jump */
 119         if (!is_bl_op(op)) {
 120                 printk(KERN_ERR "Not expected bl: opcode is %x\n", op);
 121                 return -EINVAL;
 122         }
 123
 124         /* lets find where the pointer goes */
 125         tramp = find_bl_target(ip, op);
 126
 127         /*
 128          * On PPC64 the trampoline looks like:
 129          * 0x3d, 0x82, 0x00, 0x00,    addis   r12,r2, <high>
 130          * 0x39, 0x8c, 0x00, 0x00,    addi    r12,r12, <low>
 131          *   Where the bytes 2,3,6 and 7 make up the 32bit offset
 132          *   to the TOC that holds the pointer.
 133          *   to jump to.
 134          * 0xf8, 0x41, 0x00, 0x28,    std     r2,40(r1)
 135          * 0xe9, 0x6c, 0x00, 0x20,    ld      r11,32(r12)
 136          *   The actually address is 32 bytes from the offset
 137          *   into the TOC.
 138          * 0xe8, 0x4c, 0x00, 0x28,    ld      r2,40(r12)
 139          */
 140
 141         pr_devel("ip:%lx jumps to %lx r2: %lx", ip, tramp, mod->arch.toc);
 142
 143         /* Find where the trampoline jumps to */
 144         if (probe_kernel_read(jmp, (void *)tramp, sizeof(jmp))) {
 145                 printk(KERN_ERR "Failed to read %lx\n", tramp);
 146                 return -EFAULT;
 147         }
 148
 149         pr_devel(" %08x %08x", jmp[0], jmp[1]);
 150
 151         /* verify that this is what we expect it to be */
 152         if (((jmp[0] & 0xffff0000) != 0x3d820000) ||
 153             ((jmp[1] & 0xffff0000) != 0x398c0000) ||
 154             (jmp[2] != 0xf8410028) ||
 155             (jmp[3] != 0xe96c0020) ||
 156             (jmp[4] != 0xe84c0028)) {
 157                 printk(KERN_ERR "Not a trampoline\n");
 158                 return -EINVAL;
 159         }
 160
 161         /* The bottom half is signed extended */
 162         offset = ((unsigned)((unsigned short)jmp[0]) << 16) +
 163                 (int)((short)jmp[1]);
 164
 165         pr_devel(" %x ", offset);
 166
 167         /* get the address this jumps too */
 168         tramp = mod->arch.toc + offset + 32;
 169         pr_devel("toc: %lx", tramp);
 170
 171         if (probe_kernel_read(jmp, (void *)tramp, 8)) {
 172                 printk(KERN_ERR "Failed to read %lx\n", tramp);
 173                 return -EFAULT;
 174         }
 175
 176         pr_devel(" %08x %08x\n", jmp[0], jmp[1]);
 177
 178         ptr = ((unsigned long)jmp[0] << 32) + jmp[1];
 179
 180         /* This should match what was called */
 181         if (ptr != ppc_function_entry((void *)addr)) {
 182                 printk(KERN_ERR "addr does not match %lx\n", ptr);
 183                 return -EINVAL;
 184         }
 185
 186         /*
 187          * We want to nop the line, but the next line is
 188          *  0xe8, 0x41, 0x00, 0x28   ld r2,40(r1)
 189          * This needs to be turned to a nop too.
 190          */
 191         if (probe_kernel_read(&op, (void *)(ip+4), MCOUNT_INSN_SIZE))
 192                 return -EFAULT;
 193
 194         if (op != 0xe8410028) {
 195                 printk(KERN_ERR "Next line is not ld! (%08x)\n", op);
 196                 return -EINVAL;
 197         }
 198
 199         /*
 200          * Milton Miller pointed out that we can not blindly do nops.
 201          * If a task was preempted when calling a trace function,
 202          * the nops will remove the way to restore the TOC in r2
 203          * and the r2 TOC will get corrupted.
 204          */
 205
 206         /*
 207          * Replace:
 208          *   bl <tramp>  <==== will be replaced with "b 1f"
 209          *   ld r2,40(r1)
 210          *  1:
 211          */
 212         op = 0x48000008;        /* b +8 */
 213
 214         if (probe_kernel_write((void *)ip, &op, MCOUNT_INSN_SIZE))
 215                 return -EPERM;
 216
 217
 218         flush_icache_range(ip, ip + 8);
 219
 220         return 0;
 221 }
 222
 223 #else /* !PPC64 */
 224 static int
 225 __ftrace_make_nop(struct module *mod,
 226                   struct dyn_ftrace *rec, unsigned long addr)
 227 {
 228         unsigned int op;
 229         unsigned int jmp[4];
 230         unsigned long ip = rec->ip;
 231         unsigned long tramp;
 232
 233         if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
 234                 return -EFAULT;
 235
 236         /* Make sure that that this is still a 24bit jump */
 237         if (!is_bl_op(op)) {
 238                 printk(KERN_ERR "Not expected bl: opcode is %x\n", op);
 239                 return -EINVAL;
 240         }
 241
 242         /* lets find where the pointer goes */
 243         tramp = find_bl_target(ip, op);
 244
 245         /*
 246          * On PPC32 the trampoline looks like:
 247          *  0x3d, 0x60, 0x00, 0x00  lis r11,sym@ha
 248          *  0x39, 0x6b, 0x00, 0x00  addi r11,r11,sym@l
 249          *  0x7d, 0x69, 0x03, 0xa6  mtctr r11
 250          *  0x4e, 0x80, 0x04, 0x20  bctr
 251          */
 252
 253         pr_devel("ip:%lx jumps to %lx", ip, tramp);
 254
 255         /* Find where the trampoline jumps to */
 256         if (probe_kernel_read(jmp, (void *)tramp, sizeof(jmp))) {
 257                 printk(KERN_ERR "Failed to read %lx\n", tramp);
 258                 return -EFAULT;
 259         }
 260
 261         pr_devel(" %08x %08x ", jmp[0], jmp[1]);
 262
 263         /* verify that this is what we expect it to be */
 264         if (((jmp[0] & 0xffff0000) != 0x3d600000) ||
 265             ((jmp[1] & 0xffff0000) != 0x396b0000) ||
 266             (jmp[2] != 0x7d6903a6) ||
 267             (jmp[3] != 0x4e800420)) {
 268                 printk(KERN_ERR "Not a trampoline\n");
 269                 return -EINVAL;
 270         }
 271
 272         tramp = (jmp[1] & 0xffff) |
 273                 ((jmp[0] & 0xffff) << 16);
 274         if (tramp & 0x8000)
 275                 tramp -= 0x10000;
 276
 277         pr_devel(" %lx ", tramp);
 278
 279         if (tramp != addr) {
 280                 printk(KERN_ERR
 281                        "Trampoline location %08lx does not match addr\n",
 282                        tramp);
 283                 return -EINVAL;
 284         }
 285
 286         op = PPC_INST_NOP;
 287
 288         if (probe_kernel_write((void *)ip, &op, MCOUNT_INSN_SIZE))
 289                 return -EPERM;
 290
 291         flush_icache_range(ip, ip + 8);
 292
 293         return 0;
 294 }
 295 #endif /* PPC64 */
 296 #endif /* CONFIG_MODULES */
 297
 298 int ftrace_make_nop(struct module *mod,
 299                     struct dyn_ftrace *rec, unsigned long addr)
 300 {
 301         unsigned long ip = rec->ip;
 302         unsigned int old, new;
 303
 304         /*
 305          * If the calling address is more that 24 bits away,
 306          * then we had to use a trampoline to make the call.
 307          * Otherwise just update the call site.
 308          */
 309         if (test_24bit_addr(ip, addr)) {
 310                 /* within range */
 311                 old = ftrace_call_replace(ip, addr, 1);
 312                 new = PPC_INST_NOP;
 313                 return ftrace_modify_code(ip, old, new);
 314         }
 315
 316 #ifdef CONFIG_MODULES
 317         /*
 318          * Out of range jumps are called from modules.
 319          * We should either already have a pointer to the module
 320          * or it has been passed in.
 321          */
 322         if (!rec->arch.mod) {
 323                 if (!mod) {
 324                         printk(KERN_ERR "No module loaded addr=%lx\n",
 325                                addr);
 326                         return -EFAULT;
 327                 }
 328                 rec->arch.mod = mod;
 329         } else if (mod) {
 330                 if (mod != rec->arch.mod) {
 331                         printk(KERN_ERR
 332                                "Record mod %p not equal to passed in mod %p\n",
 333                                rec->arch.mod, mod);
 334                         return -EINVAL;
 335                 }
 336                 /* nothing to do if mod == rec->arch.mod */
 337         } else
 338                 mod = rec->arch.mod;
 339
 340         return __ftrace_make_nop(mod, rec, addr);
 341 #else
 342         /* We should not get here without modules */
 343         return -EINVAL;
 344 #endif /* CONFIG_MODULES */
 345 }
 346
 347 #ifdef CONFIG_MODULES
 348 #ifdef CONFIG_PPC64
 349 static int
 350 __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
 351 {
 352         unsigned int op[2];
 353         unsigned long ip = rec->ip;
 354
 355         /* read where this goes */
 356         if (probe_kernel_read(op, (void *)ip, MCOUNT_INSN_SIZE * 2))
 357                 return -EFAULT;
 358
 359         /*
 360          * It should be pointing to two nops or
 361          *  b +8; ld r2,40(r1)
 362          */
 363         if (((op[0] != 0x48000008) || (op[1] != 0xe8410028)) &&
 364             ((op[0] != PPC_INST_NOP) || (op[1] != PPC_INST_NOP))) {
 365                 printk(KERN_ERR "Expected NOPs but have %x %x\n", op[0], op[1]);
 366                 return -EINVAL;
 367         }
 368
 369         /* If we never set up a trampoline to ftrace_caller, then bail */
 370         if (!rec->arch.mod->arch.tramp) {
 371                 printk(KERN_ERR "No ftrace trampoline\n");
 372                 return -EINVAL;
 373         }
 374
 375         /* create the branch to the trampoline */
 376         op[0] = create_branch((unsigned int *)ip,
 377                               rec->arch.mod->arch.tramp, BRANCH_SET_LINK);
 378         if (!op[0]) {
 379                 printk(KERN_ERR "REL24 out of range!\n");
 380                 return -EINVAL;
 381         }
 382
 383         /* ld r2,40(r1) */
 384         op[1] = 0xe8410028;
 385
 386         pr_devel("write to %lx\n", rec->ip);
 387
 388         if (probe_kernel_write((void *)ip, op, MCOUNT_INSN_SIZE * 2))
 389                 return -EPERM;
 390
 391         flush_icache_range(ip, ip + 8);
 392
 393         return 0;
 394 }
 395 #else
 396 static int
 397 __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
 398 {
 399         unsigned int op;
 400         unsigned long ip = rec->ip;
 401
 402         /* read where this goes */
 403         if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
 404                 return -EFAULT;
 405
 406         /* It should be pointing to a nop */
 407         if (op != PPC_INST_NOP) {
 408                 printk(KERN_ERR "Expected NOP but have %x\n", op);
 409                 return -EINVAL;
 410         }
 411
 412         /* If we never set up a trampoline to ftrace_caller, then bail */
 413         if (!rec->arch.mod->arch.tramp) {
 414                 printk(KERN_ERR "No ftrace trampoline\n");
 415                 return -EINVAL;
 416         }
 417
 418         /* create the branch to the trampoline */
 419         op = create_branch((unsigned int *)ip,
 420                            rec->arch.mod->arch.tramp, BRANCH_SET_LINK);
 421         if (!op) {
 422                 printk(KERN_ERR "REL24 out of range!\n");
 423                 return -EINVAL;
 424         }
 425
 426         pr_devel("write to %lx\n", rec->ip);
 427
 428         if (probe_kernel_write((void *)ip, &op, MCOUNT_INSN_SIZE))
 429                 return -EPERM;
 430
 431         flush_icache_range(ip, ip + 8);
 432
 433         return 0;
 434 }
 435 #endif /* CONFIG_PPC64 */
 436 #endif /* CONFIG_MODULES */
 437
 438 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
 439 {
 440         unsigned long ip = rec->ip;
 441         unsigned int old, new;
 442
 443         /*
 444          * If the calling address is more that 24 bits away,
 445          * then we had to use a trampoline to make the call.
 446          * Otherwise just update the call site.
 447          */
 448         if (test_24bit_addr(ip, addr)) {
 449                 /* within range */
 450                 old = PPC_INST_NOP;
 451                 new = ftrace_call_replace(ip, addr, 1);
 452                 return ftrace_modify_code(ip, old, new);
 453         }
 454
 455 #ifdef CONFIG_MODULES
 456         /*
 457          * Out of range jumps are called from modules.
 458          * Being that we are converting from nop, it had better
 459          * already have a module defined.
 460          */
 461         if (!rec->arch.mod) {
 462                 printk(KERN_ERR "No module loaded\n");
 463                 return -EINVAL;
 464         }
 465
 466         return __ftrace_make_call(rec, addr);
 467 #else
 468         /* We should not get here without modules */
 469         return -EINVAL;
 470 #endif /* CONFIG_MODULES */
 471 }
 472
 473 int ftrace_update_ftrace_func(ftrace_func_t func)
 474 {
 475         unsigned long ip = (unsigned long)(&ftrace_call);
 476         unsigned int old, new;
 477         int ret;
 478
 479         old = *(unsigned int *)&ftrace_call;
 480         new = ftrace_call_replace(ip, (unsigned long)func, 1);
 481         ret = ftrace_modify_code(ip, old, new);
 482
 483         return ret;
 484 }
 485
 486 int __init ftrace_dyn_arch_init(void *data)
 487 {
 488         /* caller expects data to be zero */
 489         unsigned long *p = data;
 490
 491         *p = 0;
 492
 493         return 0;
 494 }
 495 #endif /* CONFIG_DYNAMIC_FTRACE */
 496
 497 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 498
 499 #ifdef CONFIG_DYNAMIC_FTRACE
 500 extern void ftrace_graph_call(void);
 501 extern void ftrace_graph_stub(void);
 502
 503 int ftrace_enable_ftrace_graph_caller(void)
 504 {
 505         unsigned long ip = (unsigned long)(&ftrace_graph_call);
 506         unsigned long addr = (unsigned long)(&ftrace_graph_caller);
 507         unsigned long stub = (unsigned long)(&ftrace_graph_stub);
 508         unsigned int old, new;
 509
 510         old = ftrace_call_replace(ip, stub, 0);
 511         new = ftrace_call_replace(ip, addr, 0);
 512
 513         return ftrace_modify_code(ip, old, new);
 514 }
 515
 516 int ftrace_disable_ftrace_graph_caller(void)
 517 {
 518         unsigned long ip = (unsigned long)(&ftrace_graph_call);
 519         unsigned long addr = (unsigned long)(&ftrace_graph_caller);
 520         unsigned long stub = (unsigned long)(&ftrace_graph_stub);
 521         unsigned int old, new;
 522
 523         old = ftrace_call_replace(ip, addr, 0);
 524         new = ftrace_call_replace(ip, stub, 0);
 525
 526         return ftrace_modify_code(ip, old, new);
 527 }
 528 #endif /* CONFIG_DYNAMIC_FTRACE */
 529
 530 #ifdef CONFIG_PPC64
 531 extern void mod_return_to_handler(void);
 532 #endif
 533
 534 /*
 535  * Hook the return address and push it in the stack of return addrs
 536  * in current thread info.
 537  */
 538 void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr)
 539 {
 540         unsigned long old;
 541         int faulted;
 542         struct ftrace_graph_ent trace;
 543         unsigned long return_hooker = (unsigned long)&return_to_handler;
 544
 545         if (unlikely(atomic_read(&current->tracing_graph_pause)))
 546                 return;
 547
 548 #ifdef CONFIG_PPC64
 549         /* non core kernel code needs to save and restore the TOC */
 550         if (REGION_ID(self_addr) != KERNEL_REGION_ID)
 551                 return_hooker = (unsigned long)&mod_return_to_handler;
 552 #endif
 553
 554         return_hooker = ppc_function_entry((void *)return_hooker);
 555
 556         /*
 557          * Protect against fault, even if it shouldn't
 558          * happen. This tool is too much intrusive to
 559          * ignore such a protection.
 560          */
 561         asm volatile(
 562                 "1: " PPC_LL "%[old], 0(%[parent])\n"
 563                 "2: " PPC_STL "%[return_hooker], 0(%[parent])\n"
 564                 "   li %[faulted], 0\n"
 565                 "3:\n"
 566
 567                 ".section .fixup, \"ax\"\n"
 568                 "4: li %[faulted], 1\n"
 569                 "   b 3b\n"
 570                 ".previous\n"
 571
 572                 ".section __ex_table,\"a\"\n"
 573                         PPC_LONG_ALIGN "\n"
 574                         PPC_LONG "1b,4b\n"
 575                         PPC_LONG "2b,4b\n"
 576                 ".previous"
 577
 578                 : [old] "=&r" (old), [faulted] "=r" (faulted)
 579                 : [parent] "r" (parent), [return_hooker] "r" (return_hooker)
 580                 : "memory"
 581         );
 582
 583         if (unlikely(faulted)) {
 584                 ftrace_graph_stop();
 585                 WARN_ON(1);
 586                 return;
 587         }
 588
 589         if (ftrace_push_return_trace(old, self_addr, &trace.depth, 0) == -EBUSY) {
 590                 *parent = old;
 591                 return;
 592         }
 593
 594         trace.func = self_addr;
 595
 596         /* Only trace if the calling function expects to */
 597         if (!ftrace_graph_entry(&trace)) {
 598                 current->curr_ret_stack--;
 599                 *parent = old;
 600         }
 601 }
 602 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */