arch/powerpc/kernel/trace/ftrace.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Code for replacing ftrace calls with jumps.
   4  *
   5  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   6  *
   7  * Thanks goes out to P.A. Semi, Inc for supplying me with a PPC64 box.
   8  *
   9  * Added function graph tracer code, taken from x86 that was written
  10  * by Frederic Weisbecker, and ported to PPC by Steven Rostedt.
  11  *
  12  */
  13
  14 #define pr_fmt(fmt) "ftrace-powerpc: " fmt
  15
  16 #include <linux/spinlock.h>
  17 #include <linux/hardirq.h>
  18 #include <linux/uaccess.h>
  19 #include <linux/module.h>
  20 #include <linux/ftrace.h>
  21 #include <linux/percpu.h>
  22 #include <linux/init.h>
  23 #include <linux/list.h>
  24
  25 #include <asm/asm-prototypes.h>
  26 #include <asm/cacheflush.h>
  27 #include <asm/code-patching.h>
  28 #include <asm/ftrace.h>
  29 #include <asm/syscall.h>
  30
  31
  32 #ifdef CONFIG_DYNAMIC_FTRACE
  33
  34 /*
  35  * We generally only have a single long_branch tramp and at most 2 or 3 plt
  36  * tramps generated. But, we don't use the plt tramps currently. We also allot
  37  * 2 tramps after .text and .init.text. So, we only end up with around 3 usable
  38  * tramps in total. Set aside 8 just to be sure.
  39  */
  40 #define NUM_FTRACE_TRAMPS       8
  41 static unsigned long ftrace_tramps[NUM_FTRACE_TRAMPS];
  42
  43 static unsigned int
  44 ftrace_call_replace(unsigned long ip, unsigned long addr, int link)
  45 {
  46         unsigned int op;
  47
  48         addr = ppc_function_entry((void *)addr);
  49
  50         /* if (link) set op to 'bl' else 'b' */
  51         op = create_branch((unsigned int *)ip, addr, link ? 1 : 0);
  52
  53         return op;
  54 }
  55
  56 static int
  57 ftrace_modify_code(unsigned long ip, unsigned int old, unsigned int new)
  58 {
  59         unsigned int replaced;
  60
  61         /*
  62          * Note:
  63          * We are paranoid about modifying text, as if a bug was to happen, it
  64          * could cause us to read or write to someplace that could cause harm.
  65          * Carefully read and modify the code with probe_kernel_*(), and make
  66          * sure what we read is what we expected it to be before modifying it.
  67          */
  68
  69         /* read the text we want to modify */
  70         if (probe_kernel_read(&replaced, (void *)ip, MCOUNT_INSN_SIZE))
  71                 return -EFAULT;
  72
  73         /* Make sure it is what we expect it to be */
  74         if (replaced != old) {
  75                 pr_err("%p: replaced (%#x) != old (%#x)",
  76                 (void *)ip, replaced, old);
  77                 return -EINVAL;
  78         }
  79
  80         /* replace the text with the new text */
  81         if (patch_instruction((unsigned int *)ip, new))
  82                 return -EPERM;
  83
  84         return 0;
  85 }
  86
  87 /*
  88  * Helper functions that are the same for both PPC64 and PPC32.
  89  */
  90 static int test_24bit_addr(unsigned long ip, unsigned long addr)
  91 {
  92         addr = ppc_function_entry((void *)addr);
  93
  94         /* use the create_branch to verify that this offset can be branched */
  95         return create_branch((unsigned int *)ip, addr, 0);
  96 }
  97
  98 static int is_bl_op(unsigned int op)
  99 {
 100         return (op & 0xfc000003) == 0x48000001;
 101 }
 102
 103 static int is_b_op(unsigned int op)
 104 {
 105         return (op & 0xfc000003) == 0x48000000;
 106 }
 107
 108 static unsigned long find_bl_target(unsigned long ip, unsigned int op)
 109 {
 110         int offset;
 111
 112         offset = (op & 0x03fffffc);
 113         /* make it signed */
 114         if (offset & 0x02000000)
 115                 offset |= 0xfe000000;
 116
 117         return ip + (long)offset;
 118 }
 119
 120 #ifdef CONFIG_MODULES
 121 #ifdef CONFIG_PPC64
 122 static int
 123 __ftrace_make_nop(struct module *mod,
 124                   struct dyn_ftrace *rec, unsigned long addr)
 125 {
 126         unsigned long entry, ptr, tramp;
 127         unsigned long ip = rec->ip;
 128         unsigned int op, pop;
 129
 130         /* read where this goes */
 131         if (probe_kernel_read(&op, (void *)ip, sizeof(int))) {
 132                 pr_err("Fetching opcode failed.\n");
 133                 return -EFAULT;
 134         }
 135
 136         /* Make sure that that this is still a 24bit jump */
 137         if (!is_bl_op(op)) {
 138                 pr_err("Not expected bl: opcode is %x\n", op);
 139                 return -EINVAL;
 140         }
 141
 142         /* lets find where the pointer goes */
 143         tramp = find_bl_target(ip, op);
 144
 145         pr_devel("ip:%lx jumps to %lx", ip, tramp);
 146
 147         if (module_trampoline_target(mod, tramp, &ptr)) {
 148                 pr_err("Failed to get trampoline target\n");
 149                 return -EFAULT;
 150         }
 151
 152         pr_devel("trampoline target %lx", ptr);
 153
 154         entry = ppc_global_function_entry((void *)addr);
 155         /* This should match what was called */
 156         if (ptr != entry) {
 157                 pr_err("addr %lx does not match expected %lx\n", ptr, entry);
 158                 return -EINVAL;
 159         }
 160
 161 #ifdef CONFIG_MPROFILE_KERNEL
 162         /* When using -mkernel_profile there is no load to jump over */
 163         pop = PPC_INST_NOP;
 164
 165         if (probe_kernel_read(&op, (void *)(ip - 4), 4)) {
 166                 pr_err("Fetching instruction at %lx failed.\n", ip - 4);
 167                 return -EFAULT;
 168         }
 169
 170         /* We expect either a mflr r0, or a std r0, LRSAVE(r1) */
 171         if (op != PPC_INST_MFLR && op != PPC_INST_STD_LR) {
 172                 pr_err("Unexpected instruction %08x around bl _mcount\n", op);
 173                 return -EINVAL;
 174         }
 175 #else
 176         /*
 177          * Our original call site looks like:
 178          *
 179          * bl <tramp>
 180          * ld r2,XX(r1)
 181          *
 182          * Milton Miller pointed out that we can not simply nop the branch.
 183          * If a task was preempted when calling a trace function, the nops
 184          * will remove the way to restore the TOC in r2 and the r2 TOC will
 185          * get corrupted.
 186          *
 187          * Use a b +8 to jump over the load.
 188          */
 189
 190         pop = PPC_INST_BRANCH | 8;      /* b +8 */
 191
 192         /*
 193          * Check what is in the next instruction. We can see ld r2,40(r1), but
 194          * on first pass after boot we will see mflr r0.
 195          */
 196         if (probe_kernel_read(&op, (void *)(ip+4), MCOUNT_INSN_SIZE)) {
 197                 pr_err("Fetching op failed.\n");
 198                 return -EFAULT;
 199         }
 200
 201         if (op != PPC_INST_LD_TOC) {
 202                 pr_err("Expected %08x found %08x\n", PPC_INST_LD_TOC, op);
 203                 return -EINVAL;
 204         }
 205 #endif /* CONFIG_MPROFILE_KERNEL */
 206
 207         if (patch_instruction((unsigned int *)ip, pop)) {
 208                 pr_err("Patching NOP failed.\n");
 209                 return -EPERM;
 210         }
 211
 212         return 0;
 213 }
 214
 215 #else /* !PPC64 */
 216 static int
 217 __ftrace_make_nop(struct module *mod,
 218                   struct dyn_ftrace *rec, unsigned long addr)
 219 {
 220         unsigned int op;
 221         unsigned int jmp[4];
 222         unsigned long ip = rec->ip;
 223         unsigned long tramp;
 224
 225         if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
 226                 return -EFAULT;
 227
 228         /* Make sure that that this is still a 24bit jump */
 229         if (!is_bl_op(op)) {
 230                 pr_err("Not expected bl: opcode is %x\n", op);
 231                 return -EINVAL;
 232         }
 233
 234         /* lets find where the pointer goes */
 235         tramp = find_bl_target(ip, op);
 236
 237         /*
 238          * On PPC32 the trampoline looks like:
 239          *  0x3d, 0x80, 0x00, 0x00  lis r12,sym@ha
 240          *  0x39, 0x8c, 0x00, 0x00  addi r12,r12,sym@l
 241          *  0x7d, 0x89, 0x03, 0xa6  mtctr r12
 242          *  0x4e, 0x80, 0x04, 0x20  bctr
 243          */
 244
 245         pr_devel("ip:%lx jumps to %lx", ip, tramp);
 246
 247         /* Find where the trampoline jumps to */
 248         if (probe_kernel_read(jmp, (void *)tramp, sizeof(jmp))) {
 249                 pr_err("Failed to read %lx\n", tramp);
 250                 return -EFAULT;
 251         }
 252
 253         pr_devel(" %08x %08x ", jmp[0], jmp[1]);
 254
 255         /* verify that this is what we expect it to be */
 256         if (((jmp[0] & 0xffff0000) != 0x3d800000) ||
 257             ((jmp[1] & 0xffff0000) != 0x398c0000) ||
 258             (jmp[2] != 0x7d8903a6) ||
 259             (jmp[3] != 0x4e800420)) {
 260                 pr_err("Not a trampoline\n");
 261                 return -EINVAL;
 262         }
 263
 264         tramp = (jmp[1] & 0xffff) |
 265                 ((jmp[0] & 0xffff) << 16);
 266         if (tramp & 0x8000)
 267                 tramp -= 0x10000;
 268
 269         pr_devel(" %lx ", tramp);
 270
 271         if (tramp != addr) {
 272                 pr_err("Trampoline location %08lx does not match addr\n",
 273                        tramp);
 274                 return -EINVAL;
 275         }
 276
 277         op = PPC_INST_NOP;
 278
 279         if (patch_instruction((unsigned int *)ip, op))
 280                 return -EPERM;
 281
 282         return 0;
 283 }
 284 #endif /* PPC64 */
 285 #endif /* CONFIG_MODULES */
 286
 287 static unsigned long find_ftrace_tramp(unsigned long ip)
 288 {
 289         int i;
 290
 291         /*
 292          * We have the compiler generated long_branch tramps at the end
 293          * and we prefer those
 294          */
 295         for (i = NUM_FTRACE_TRAMPS - 1; i >= 0; i--)
 296                 if (!ftrace_tramps[i])
 297                         continue;
 298                 else if (create_branch((void *)ip, ftrace_tramps[i], 0))
 299                         return ftrace_tramps[i];
 300
 301         return 0;
 302 }
 303
 304 static int add_ftrace_tramp(unsigned long tramp)
 305 {
 306         int i;
 307
 308         for (i = 0; i < NUM_FTRACE_TRAMPS; i++)
 309                 if (!ftrace_tramps[i]) {
 310                         ftrace_tramps[i] = tramp;
 311                         return 0;
 312                 }
 313
 314         return -1;
 315 }
 316
 317 /*
 318  * If this is a compiler generated long_branch trampoline (essentially, a
 319  * trampoline that has a branch to _mcount()), we re-write the branch to
 320  * instead go to ftrace_[regs_]caller() and note down the location of this
 321  * trampoline.
 322  */
 323 static int setup_mcount_compiler_tramp(unsigned long tramp)
 324 {
 325         int i, op;
 326         unsigned long ptr;
 327         static unsigned long ftrace_plt_tramps[NUM_FTRACE_TRAMPS];
 328
 329         /* Is this a known long jump tramp? */
 330         for (i = 0; i < NUM_FTRACE_TRAMPS; i++)
 331                 if (!ftrace_tramps[i])
 332                         break;
 333                 else if (ftrace_tramps[i] == tramp)
 334                         return 0;
 335
 336         /* Is this a known plt tramp? */
 337         for (i = 0; i < NUM_FTRACE_TRAMPS; i++)
 338                 if (!ftrace_plt_tramps[i])
 339                         break;
 340                 else if (ftrace_plt_tramps[i] == tramp)
 341                         return -1;
 342
 343         /* New trampoline -- read where this goes */
 344         if (probe_kernel_read(&op, (void *)tramp, sizeof(int))) {
 345                 pr_debug("Fetching opcode failed.\n");
 346                 return -1;
 347         }
 348
 349         /* Is this a 24 bit branch? */
 350         if (!is_b_op(op)) {
 351                 pr_debug("Trampoline is not a long branch tramp.\n");
 352                 return -1;
 353         }
 354
 355         /* lets find where the pointer goes */
 356         ptr = find_bl_target(tramp, op);
 357
 358         if (ptr != ppc_global_function_entry((void *)_mcount)) {
 359                 pr_debug("Trampoline target %p is not _mcount\n", (void *)ptr);
 360                 return -1;
 361         }
 362
 363         /* Let's re-write the tramp to go to ftrace_[regs_]caller */
 364 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
 365         ptr = ppc_global_function_entry((void *)ftrace_regs_caller);
 366 #else
 367         ptr = ppc_global_function_entry((void *)ftrace_caller);
 368 #endif
 369         if (!create_branch((void *)tramp, ptr, 0)) {
 370                 pr_debug("%ps is not reachable from existing mcount tramp\n",
 371                                 (void *)ptr);
 372                 return -1;
 373         }
 374
 375         if (patch_branch((unsigned int *)tramp, ptr, 0)) {
 376                 pr_debug("REL24 out of range!\n");
 377                 return -1;
 378         }
 379
 380         if (add_ftrace_tramp(tramp)) {
 381                 pr_debug("No tramp locations left\n");
 382                 return -1;
 383         }
 384
 385         return 0;
 386 }
 387
 388 static int __ftrace_make_nop_kernel(struct dyn_ftrace *rec, unsigned long addr)
 389 {
 390         unsigned long tramp, ip = rec->ip;
 391         unsigned int op;
 392
 393         /* Read where this goes */
 394         if (probe_kernel_read(&op, (void *)ip, sizeof(int))) {
 395                 pr_err("Fetching opcode failed.\n");
 396                 return -EFAULT;
 397         }
 398
 399         /* Make sure that that this is still a 24bit jump */
 400         if (!is_bl_op(op)) {
 401                 pr_err("Not expected bl: opcode is %x\n", op);
 402                 return -EINVAL;
 403         }
 404
 405         /* Let's find where the pointer goes */
 406         tramp = find_bl_target(ip, op);
 407
 408         pr_devel("ip:%lx jumps to %lx", ip, tramp);
 409
 410         if (setup_mcount_compiler_tramp(tramp)) {
 411                 /* Are other trampolines reachable? */
 412                 if (!find_ftrace_tramp(ip)) {
 413                         pr_err("No ftrace trampolines reachable from %ps\n",
 414                                         (void *)ip);
 415                         return -EINVAL;
 416                 }
 417         }
 418
 419         if (patch_instruction((unsigned int *)ip, PPC_INST_NOP)) {
 420                 pr_err("Patching NOP failed.\n");
 421                 return -EPERM;
 422         }
 423
 424         return 0;
 425 }
 426
 427 int ftrace_make_nop(struct module *mod,
 428                     struct dyn_ftrace *rec, unsigned long addr)
 429 {
 430         unsigned long ip = rec->ip;
 431         unsigned int old, new;
 432
 433         /*
 434          * If the calling address is more that 24 bits away,
 435          * then we had to use a trampoline to make the call.
 436          * Otherwise just update the call site.
 437          */
 438         if (test_24bit_addr(ip, addr)) {
 439                 /* within range */
 440                 old = ftrace_call_replace(ip, addr, 1);
 441                 new = PPC_INST_NOP;
 442                 return ftrace_modify_code(ip, old, new);
 443         } else if (core_kernel_text(ip))
 444                 return __ftrace_make_nop_kernel(rec, addr);
 445
 446 #ifdef CONFIG_MODULES
 447         /*
 448          * Out of range jumps are called from modules.
 449          * We should either already have a pointer to the module
 450          * or it has been passed in.
 451          */
 452         if (!rec->arch.mod) {
 453                 if (!mod) {
 454                         pr_err("No module loaded addr=%lx\n", addr);
 455                         return -EFAULT;
 456                 }
 457                 rec->arch.mod = mod;
 458         } else if (mod) {
 459                 if (mod != rec->arch.mod) {
 460                         pr_err("Record mod %p not equal to passed in mod %p\n",
 461                                rec->arch.mod, mod);
 462                         return -EINVAL;
 463                 }
 464                 /* nothing to do if mod == rec->arch.mod */
 465         } else
 466                 mod = rec->arch.mod;
 467
 468         return __ftrace_make_nop(mod, rec, addr);
 469 #else
 470         /* We should not get here without modules */
 471         return -EINVAL;
 472 #endif /* CONFIG_MODULES */
 473 }
 474
 475 #ifdef CONFIG_MODULES
 476 #ifdef CONFIG_PPC64
 477 /*
 478  * Examine the existing instructions for __ftrace_make_call.
 479  * They should effectively be a NOP, and follow formal constraints,
 480  * depending on the ABI. Return false if they don't.
 481  */
 482 #ifndef CONFIG_MPROFILE_KERNEL
 483 static int
 484 expected_nop_sequence(void *ip, unsigned int op0, unsigned int op1)
 485 {
 486         /*
 487          * We expect to see:
 488          *
 489          * b +8
 490          * ld r2,XX(r1)
 491          *
 492          * The load offset is different depending on the ABI. For simplicity
 493          * just mask it out when doing the compare.
 494          */
 495         if ((op0 != 0x48000008) || ((op1 & 0xffff0000) != 0xe8410000))
 496                 return 0;
 497         return 1;
 498 }
 499 #else
 500 static int
 501 expected_nop_sequence(void *ip, unsigned int op0, unsigned int op1)
 502 {
 503         /* look for patched "NOP" on ppc64 with -mprofile-kernel */
 504         if (op0 != PPC_INST_NOP)
 505                 return 0;
 506         return 1;
 507 }
 508 #endif
 509
 510 static int
 511 __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
 512 {
 513         unsigned int op[2];
 514         void *ip = (void *)rec->ip;
 515         unsigned long entry, ptr, tramp;
 516         struct module *mod = rec->arch.mod;
 517
 518         /* read where this goes */
 519         if (probe_kernel_read(op, ip, sizeof(op)))
 520                 return -EFAULT;
 521
 522         if (!expected_nop_sequence(ip, op[0], op[1])) {
 523                 pr_err("Unexpected call sequence at %p: %x %x\n",
 524                 ip, op[0], op[1]);
 525                 return -EINVAL;
 526         }
 527
 528         /* If we never set up ftrace trampoline(s), then bail */
 529 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
 530         if (!mod->arch.tramp || !mod->arch.tramp_regs) {
 531 #else
 532         if (!mod->arch.tramp) {
 533 #endif
 534                 pr_err("No ftrace trampoline\n");
 535                 return -EINVAL;
 536         }
 537
 538 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
 539         if (rec->flags & FTRACE_FL_REGS)
 540                 tramp = mod->arch.tramp_regs;
 541         else
 542 #endif
 543                 tramp = mod->arch.tramp;
 544
 545         if (module_trampoline_target(mod, tramp, &ptr)) {
 546                 pr_err("Failed to get trampoline target\n");
 547                 return -EFAULT;
 548         }
 549
 550         pr_devel("trampoline target %lx", ptr);
 551
 552         entry = ppc_global_function_entry((void *)addr);
 553         /* This should match what was called */
 554         if (ptr != entry) {
 555                 pr_err("addr %lx does not match expected %lx\n", ptr, entry);
 556                 return -EINVAL;
 557         }
 558
 559         /* Ensure branch is within 24 bits */
 560         if (!create_branch(ip, tramp, BRANCH_SET_LINK)) {
 561                 pr_err("Branch out of range\n");
 562                 return -EINVAL;
 563         }
 564
 565         if (patch_branch(ip, tramp, BRANCH_SET_LINK)) {
 566                 pr_err("REL24 out of range!\n");
 567                 return -EINVAL;
 568         }
 569
 570         return 0;
 571 }
 572
 573 #else  /* !CONFIG_PPC64: */
 574 static int
 575 __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
 576 {
 577         unsigned int op;
 578         unsigned long ip = rec->ip;
 579
 580         /* read where this goes */
 581         if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
 582                 return -EFAULT;
 583
 584         /* It should be pointing to a nop */
 585         if (op != PPC_INST_NOP) {
 586                 pr_err("Expected NOP but have %x\n", op);
 587                 return -EINVAL;
 588         }
 589
 590         /* If we never set up a trampoline to ftrace_caller, then bail */
 591         if (!rec->arch.mod->arch.tramp) {
 592                 pr_err("No ftrace trampoline\n");
 593                 return -EINVAL;
 594         }
 595
 596         /* create the branch to the trampoline */
 597         op = create_branch((unsigned int *)ip,
 598                            rec->arch.mod->arch.tramp, BRANCH_SET_LINK);
 599         if (!op) {
 600                 pr_err("REL24 out of range!\n");
 601                 return -EINVAL;
 602         }
 603
 604         pr_devel("write to %lx\n", rec->ip);
 605
 606         if (patch_instruction((unsigned int *)ip, op))
 607                 return -EPERM;
 608
 609         return 0;
 610 }
 611 #endif /* CONFIG_PPC64 */
 612 #endif /* CONFIG_MODULES */
 613
 614 static int __ftrace_make_call_kernel(struct dyn_ftrace *rec, unsigned long addr)
 615 {
 616         unsigned int op;
 617         void *ip = (void *)rec->ip;
 618         unsigned long tramp, entry, ptr;
 619
 620         /* Make sure we're being asked to patch branch to a known ftrace addr */
 621         entry = ppc_global_function_entry((void *)ftrace_caller);
 622         ptr = ppc_global_function_entry((void *)addr);
 623
 624         if (ptr != entry) {
 625 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
 626                 entry = ppc_global_function_entry((void *)ftrace_regs_caller);
 627                 if (ptr != entry) {
 628 #endif
 629                         pr_err("Unknown ftrace addr to patch: %ps\n", (void *)ptr);
 630                         return -EINVAL;
 631 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
 632                 }
 633 #endif
 634         }
 635
 636         /* Make sure we have a nop */
 637         if (probe_kernel_read(&op, ip, sizeof(op))) {
 638                 pr_err("Unable to read ftrace location %p\n", ip);
 639                 return -EFAULT;
 640         }
 641
 642         if (op != PPC_INST_NOP) {
 643                 pr_err("Unexpected call sequence at %p: %x\n", ip, op);
 644                 return -EINVAL;
 645         }
 646
 647         tramp = find_ftrace_tramp((unsigned long)ip);
 648         if (!tramp) {
 649                 pr_err("No ftrace trampolines reachable from %ps\n", ip);
 650                 return -EINVAL;
 651         }
 652
 653         if (patch_branch(ip, tramp, BRANCH_SET_LINK)) {
 654                 pr_err("Error patching branch to ftrace tramp!\n");
 655                 return -EINVAL;
 656         }
 657
 658         return 0;
 659 }
 660
 661 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
 662 {
 663         unsigned long ip = rec->ip;
 664         unsigned int old, new;
 665
 666         /*
 667          * If the calling address is more that 24 bits away,
 668          * then we had to use a trampoline to make the call.
 669          * Otherwise just update the call site.
 670          */
 671         if (test_24bit_addr(ip, addr)) {
 672                 /* within range */
 673                 old = PPC_INST_NOP;
 674                 new = ftrace_call_replace(ip, addr, 1);
 675                 return ftrace_modify_code(ip, old, new);
 676         } else if (core_kernel_text(ip))
 677                 return __ftrace_make_call_kernel(rec, addr);
 678
 679 #ifdef CONFIG_MODULES
 680         /*
 681          * Out of range jumps are called from modules.
 682          * Being that we are converting from nop, it had better
 683          * already have a module defined.
 684          */
 685         if (!rec->arch.mod) {
 686                 pr_err("No module loaded\n");
 687                 return -EINVAL;
 688         }
 689
 690         return __ftrace_make_call(rec, addr);
 691 #else
 692         /* We should not get here without modules */
 693         return -EINVAL;
 694 #endif /* CONFIG_MODULES */
 695 }
 696
 697 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
 698 #ifdef CONFIG_MODULES
 699 static int
 700 __ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
 701                                         unsigned long addr)
 702 {
 703         unsigned int op;
 704         unsigned long ip = rec->ip;
 705         unsigned long entry, ptr, tramp;
 706         struct module *mod = rec->arch.mod;
 707
 708         /* If we never set up ftrace trampolines, then bail */
 709         if (!mod->arch.tramp || !mod->arch.tramp_regs) {
 710                 pr_err("No ftrace trampoline\n");
 711                 return -EINVAL;
 712         }
 713
 714         /* read where this goes */
 715         if (probe_kernel_read(&op, (void *)ip, sizeof(int))) {
 716                 pr_err("Fetching opcode failed.\n");
 717                 return -EFAULT;
 718         }
 719
 720         /* Make sure that that this is still a 24bit jump */
 721         if (!is_bl_op(op)) {
 722                 pr_err("Not expected bl: opcode is %x\n", op);
 723                 return -EINVAL;
 724         }
 725
 726         /* lets find where the pointer goes */
 727         tramp = find_bl_target(ip, op);
 728         entry = ppc_global_function_entry((void *)old_addr);
 729
 730         pr_devel("ip:%lx jumps to %lx", ip, tramp);
 731
 732         if (tramp != entry) {
 733                 /* old_addr is not within range, so we must have used a trampoline */
 734                 if (module_trampoline_target(mod, tramp, &ptr)) {
 735                         pr_err("Failed to get trampoline target\n");
 736                         return -EFAULT;
 737                 }
 738
 739                 pr_devel("trampoline target %lx", ptr);
 740
 741                 /* This should match what was called */
 742                 if (ptr != entry) {
 743                         pr_err("addr %lx does not match expected %lx\n", ptr, entry);
 744                         return -EINVAL;
 745                 }
 746         }
 747
 748         /* The new target may be within range */
 749         if (test_24bit_addr(ip, addr)) {
 750                 /* within range */
 751                 if (patch_branch((unsigned int *)ip, addr, BRANCH_SET_LINK)) {
 752                         pr_err("REL24 out of range!\n");
 753                         return -EINVAL;
 754                 }
 755
 756                 return 0;
 757         }
 758
 759         if (rec->flags & FTRACE_FL_REGS)
 760                 tramp = mod->arch.tramp_regs;
 761         else
 762                 tramp = mod->arch.tramp;
 763
 764         if (module_trampoline_target(mod, tramp, &ptr)) {
 765                 pr_err("Failed to get trampoline target\n");
 766                 return -EFAULT;
 767         }
 768
 769         pr_devel("trampoline target %lx", ptr);
 770
 771         entry = ppc_global_function_entry((void *)addr);
 772         /* This should match what was called */
 773         if (ptr != entry) {
 774                 pr_err("addr %lx does not match expected %lx\n", ptr, entry);
 775                 return -EINVAL;
 776         }
 777
 778         /* Ensure branch is within 24 bits */
 779         if (!create_branch((unsigned int *)ip, tramp, BRANCH_SET_LINK)) {
 780                 pr_err("Branch out of range\n");
 781                 return -EINVAL;
 782         }
 783
 784         if (patch_branch((unsigned int *)ip, tramp, BRANCH_SET_LINK)) {
 785                 pr_err("REL24 out of range!\n");
 786                 return -EINVAL;
 787         }
 788
 789         return 0;
 790 }
 791 #endif
 792
 793 int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
 794                         unsigned long addr)
 795 {
 796         unsigned long ip = rec->ip;
 797         unsigned int old, new;
 798
 799         /*
 800          * If the calling address is more that 24 bits away,
 801          * then we had to use a trampoline to make the call.
 802          * Otherwise just update the call site.
 803          */
 804         if (test_24bit_addr(ip, addr) && test_24bit_addr(ip, old_addr)) {
 805                 /* within range */
 806                 old = ftrace_call_replace(ip, old_addr, 1);
 807                 new = ftrace_call_replace(ip, addr, 1);
 808                 return ftrace_modify_code(ip, old, new);
 809         } else if (core_kernel_text(ip)) {
 810                 /*
 811                  * We always patch out of range locations to go to the regs
 812                  * variant, so there is nothing to do here
 813                  */
 814                 return 0;
 815         }
 816
 817 #ifdef CONFIG_MODULES
 818         /*
 819          * Out of range jumps are called from modules.
 820          */
 821         if (!rec->arch.mod) {
 822                 pr_err("No module loaded\n");
 823                 return -EINVAL;
 824         }
 825
 826         return __ftrace_modify_call(rec, old_addr, addr);
 827 #else
 828         /* We should not get here without modules */
 829         return -EINVAL;
 830 #endif /* CONFIG_MODULES */
 831 }
 832 #endif
 833
 834 int ftrace_update_ftrace_func(ftrace_func_t func)
 835 {
 836         unsigned long ip = (unsigned long)(&ftrace_call);
 837         unsigned int old, new;
 838         int ret;
 839
 840         old = *(unsigned int *)&ftrace_call;
 841         new = ftrace_call_replace(ip, (unsigned long)func, 1);
 842         ret = ftrace_modify_code(ip, old, new);
 843
 844 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
 845         /* Also update the regs callback function */
 846         if (!ret) {
 847                 ip = (unsigned long)(&ftrace_regs_call);
 848                 old = *(unsigned int *)&ftrace_regs_call;
 849                 new = ftrace_call_replace(ip, (unsigned long)func, 1);
 850                 ret = ftrace_modify_code(ip, old, new);
 851         }
 852 #endif
 853
 854         return ret;
 855 }
 856
 857 /*
 858  * Use the default ftrace_modify_all_code, but without
 859  * stop_machine().
 860  */
 861 void arch_ftrace_update_code(int command)
 862 {
 863         ftrace_modify_all_code(command);
 864 }
 865
 866 #ifdef CONFIG_PPC64
 867 #define PACATOC offsetof(struct paca_struct, kernel_toc)
 868
 869 extern unsigned int ftrace_tramp_text[], ftrace_tramp_init[];
 870
 871 int __init ftrace_dyn_arch_init(void)
 872 {
 873         int i;
 874         unsigned int *tramp[] = { ftrace_tramp_text, ftrace_tramp_init };
 875         u32 stub_insns[] = {
 876                 0xe98d0000 | PACATOC,   /* ld      r12,PACATOC(r13)     */
 877                 0x3d8c0000,             /* addis   r12,r12,<high>       */
 878                 0x398c0000,             /* addi    r12,r12,<low>        */
 879                 0x7d8903a6,             /* mtctr   r12                  */
 880                 0x4e800420,             /* bctr                         */
 881         };
 882 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
 883         unsigned long addr = ppc_global_function_entry((void *)ftrace_regs_caller);
 884 #else
 885         unsigned long addr = ppc_global_function_entry((void *)ftrace_caller);
 886 #endif
 887         long reladdr = addr - kernel_toc_addr();
 888
 889         if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) {
 890                 pr_err("Address of %ps out of range of kernel_toc.\n",
 891                                 (void *)addr);
 892                 return -1;
 893         }
 894
 895         for (i = 0; i < 2; i++) {
 896                 memcpy(tramp[i], stub_insns, sizeof(stub_insns));
 897                 tramp[i][1] |= PPC_HA(reladdr);
 898                 tramp[i][2] |= PPC_LO(reladdr);
 899                 add_ftrace_tramp((unsigned long)tramp[i]);
 900         }
 901
 902         return 0;
 903 }
 904 #else
 905 int __init ftrace_dyn_arch_init(void)
 906 {
 907         return 0;
 908 }
 909 #endif
 910 #endif /* CONFIG_DYNAMIC_FTRACE */
 911
 912 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 913
 914 extern void ftrace_graph_call(void);
 915 extern void ftrace_graph_stub(void);
 916
 917 int ftrace_enable_ftrace_graph_caller(void)
 918 {
 919         unsigned long ip = (unsigned long)(&ftrace_graph_call);
 920         unsigned long addr = (unsigned long)(&ftrace_graph_caller);
 921         unsigned long stub = (unsigned long)(&ftrace_graph_stub);
 922         unsigned int old, new;
 923
 924         old = ftrace_call_replace(ip, stub, 0);
 925         new = ftrace_call_replace(ip, addr, 0);
 926
 927         return ftrace_modify_code(ip, old, new);
 928 }
 929
 930 int ftrace_disable_ftrace_graph_caller(void)
 931 {
 932         unsigned long ip = (unsigned long)(&ftrace_graph_call);
 933         unsigned long addr = (unsigned long)(&ftrace_graph_caller);
 934         unsigned long stub = (unsigned long)(&ftrace_graph_stub);
 935         unsigned int old, new;
 936
 937         old = ftrace_call_replace(ip, addr, 0);
 938         new = ftrace_call_replace(ip, stub, 0);
 939
 940         return ftrace_modify_code(ip, old, new);
 941 }
 942
 943 /*
 944  * Hook the return address and push it in the stack of return addrs
 945  * in current thread info. Return the address we want to divert to.
 946  */
 947 unsigned long prepare_ftrace_return(unsigned long parent, unsigned long ip,
 948                                                 unsigned long sp)
 949 {
 950         unsigned long return_hooker;
 951
 952         if (unlikely(ftrace_graph_is_dead()))
 953                 goto out;
 954
 955         if (unlikely(atomic_read(&current->tracing_graph_pause)))
 956                 goto out;
 957
 958         return_hooker = ppc_function_entry(return_to_handler);
 959
 960         if (!function_graph_enter(parent, ip, 0, (unsigned long *)sp))
 961                 parent = return_hooker;
 962 out:
 963         return parent;
 964 }
 965 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
 966
 967 #ifdef PPC64_ELF_ABI_v1
 968 char *arch_ftrace_match_adjust(char *str, const char *search)
 969 {
 970         if (str[0] == '.' && search[0] != '.')
 971                 return str + 1;
 972         else
 973                 return str;
 974 }
 975 #endif /* PPC64_ELF_ABI_v1 */