arch/powerpc/kernel/trace/ftrace.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Code for replacing ftrace calls with jumps.
   4  *
   5  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   6  *
   7  * Thanks goes out to P.A. Semi, Inc for supplying me with a PPC64 box.
   8  *
   9  * Added function graph tracer code, taken from x86 that was written
  10  * by Frederic Weisbecker, and ported to PPC by Steven Rostedt.
  11  *
  12  */
  13
  14 #define pr_fmt(fmt) "ftrace-powerpc: " fmt
  15
  16 #include <linux/spinlock.h>
  17 #include <linux/hardirq.h>
  18 #include <linux/uaccess.h>
  19 #include <linux/module.h>
  20 #include <linux/ftrace.h>
  21 #include <linux/percpu.h>
  22 #include <linux/init.h>
  23 #include <linux/list.h>
  24
  25 #include <asm/asm-prototypes.h>
  26 #include <asm/cacheflush.h>
  27 #include <asm/code-patching.h>
  28 #include <asm/ftrace.h>
  29 #include <asm/syscall.h>
  30 #include <asm/inst.h>
  31
  32
  33 #ifdef CONFIG_DYNAMIC_FTRACE
  34
  35 /*
  36  * We generally only have a single long_branch tramp and at most 2 or 3 plt
  37  * tramps generated. But, we don't use the plt tramps currently. We also allot
  38  * 2 tramps after .text and .init.text. So, we only end up with around 3 usable
  39  * tramps in total. Set aside 8 just to be sure.
  40  */
  41 #define NUM_FTRACE_TRAMPS       8
  42 static unsigned long ftrace_tramps[NUM_FTRACE_TRAMPS];
  43
  44 static struct ppc_inst
  45 ftrace_call_replace(unsigned long ip, unsigned long addr, int link)
  46 {
  47         struct ppc_inst op;
  48
  49         addr = ppc_function_entry((void *)addr);
  50
  51         /* if (link) set op to 'bl' else 'b' */
  52         create_branch(&op, (struct ppc_inst *)ip, addr, link ? 1 : 0);
  53
  54         return op;
  55 }
  56
  57 static int
  58 ftrace_modify_code(unsigned long ip, struct ppc_inst old, struct ppc_inst new)
  59 {
  60         struct ppc_inst replaced;
  61
  62         /*
  63          * Note:
  64          * We are paranoid about modifying text, as if a bug was to happen, it
  65          * could cause us to read or write to someplace that could cause harm.
  66          * Carefully read and modify the code with probe_kernel_*(), and make
  67          * sure what we read is what we expected it to be before modifying it.
  68          */
  69
  70         /* read the text we want to modify */
  71         if (probe_kernel_read_inst(&replaced, (void *)ip))
  72                 return -EFAULT;
  73
  74         /* Make sure it is what we expect it to be */
  75         if (!ppc_inst_equal(replaced, old)) {
  76                 pr_err("%p: replaced (%s) != old (%s)",
  77                 (void *)ip, ppc_inst_as_str(replaced), ppc_inst_as_str(old));
  78                 return -EINVAL;
  79         }
  80
  81         /* replace the text with the new text */
  82         if (patch_instruction((struct ppc_inst *)ip, new))
  83                 return -EPERM;
  84
  85         return 0;
  86 }
  87
  88 /*
  89  * Helper functions that are the same for both PPC64 and PPC32.
  90  */
  91 static int test_24bit_addr(unsigned long ip, unsigned long addr)
  92 {
  93         struct ppc_inst op;
  94         addr = ppc_function_entry((void *)addr);
  95
  96         /* use the create_branch to verify that this offset can be branched */
  97         return create_branch(&op, (struct ppc_inst *)ip, addr, 0) == 0;
  98 }
  99
 100 static int is_bl_op(struct ppc_inst op)
 101 {
 102         return (ppc_inst_val(op) & 0xfc000003) == 0x48000001;
 103 }
 104
 105 static int is_b_op(struct ppc_inst op)
 106 {
 107         return (ppc_inst_val(op) & 0xfc000003) == 0x48000000;
 108 }
 109
 110 static unsigned long find_bl_target(unsigned long ip, struct ppc_inst op)
 111 {
 112         int offset;
 113
 114         offset = (ppc_inst_val(op) & 0x03fffffc);
 115         /* make it signed */
 116         if (offset & 0x02000000)
 117                 offset |= 0xfe000000;
 118
 119         return ip + (long)offset;
 120 }
 121
 122 #ifdef CONFIG_MODULES
 123 #ifdef CONFIG_PPC64
 124 static int
 125 __ftrace_make_nop(struct module *mod,
 126                   struct dyn_ftrace *rec, unsigned long addr)
 127 {
 128         unsigned long entry, ptr, tramp;
 129         unsigned long ip = rec->ip;
 130         struct ppc_inst op, pop;
 131
 132         /* read where this goes */
 133         if (probe_kernel_read_inst(&op, (void *)ip)) {
 134                 pr_err("Fetching opcode failed.\n");
 135                 return -EFAULT;
 136         }
 137
 138         /* Make sure that that this is still a 24bit jump */
 139         if (!is_bl_op(op)) {
 140                 pr_err("Not expected bl: opcode is %s\n", ppc_inst_as_str(op));
 141                 return -EINVAL;
 142         }
 143
 144         /* lets find where the pointer goes */
 145         tramp = find_bl_target(ip, op);
 146
 147         pr_devel("ip:%lx jumps to %lx", ip, tramp);
 148
 149         if (module_trampoline_target(mod, tramp, &ptr)) {
 150                 pr_err("Failed to get trampoline target\n");
 151                 return -EFAULT;
 152         }
 153
 154         pr_devel("trampoline target %lx", ptr);
 155
 156         entry = ppc_global_function_entry((void *)addr);
 157         /* This should match what was called */
 158         if (ptr != entry) {
 159                 pr_err("addr %lx does not match expected %lx\n", ptr, entry);
 160                 return -EINVAL;
 161         }
 162
 163 #ifdef CONFIG_MPROFILE_KERNEL
 164         /* When using -mkernel_profile there is no load to jump over */
 165         pop = ppc_inst(PPC_INST_NOP);
 166
 167         if (probe_kernel_read_inst(&op, (void *)(ip - 4))) {
 168                 pr_err("Fetching instruction at %lx failed.\n", ip - 4);
 169                 return -EFAULT;
 170         }
 171
 172         /* We expect either a mflr r0, or a std r0, LRSAVE(r1) */
 173         if (!ppc_inst_equal(op, ppc_inst(PPC_INST_MFLR)) &&
 174             !ppc_inst_equal(op, ppc_inst(PPC_INST_STD_LR))) {
 175                 pr_err("Unexpected instruction %s around bl _mcount\n",
 176                        ppc_inst_as_str(op));
 177                 return -EINVAL;
 178         }
 179 #else
 180         /*
 181          * Our original call site looks like:
 182          *
 183          * bl <tramp>
 184          * ld r2,XX(r1)
 185          *
 186          * Milton Miller pointed out that we can not simply nop the branch.
 187          * If a task was preempted when calling a trace function, the nops
 188          * will remove the way to restore the TOC in r2 and the r2 TOC will
 189          * get corrupted.
 190          *
 191          * Use a b +8 to jump over the load.
 192          */
 193
 194         pop = ppc_inst(PPC_INST_BRANCH | 8);    /* b +8 */
 195
 196         /*
 197          * Check what is in the next instruction. We can see ld r2,40(r1), but
 198          * on first pass after boot we will see mflr r0.
 199          */
 200         if (probe_kernel_read_inst(&op, (void *)(ip + 4))) {
 201                 pr_err("Fetching op failed.\n");
 202                 return -EFAULT;
 203         }
 204
 205         if (!ppc_inst_equal(op,  ppc_inst(PPC_INST_LD_TOC))) {
 206                 pr_err("Expected %08x found %s\n", PPC_INST_LD_TOC, ppc_inst_as_str(op));
 207                 return -EINVAL;
 208         }
 209 #endif /* CONFIG_MPROFILE_KERNEL */
 210
 211         if (patch_instruction((struct ppc_inst *)ip, pop)) {
 212                 pr_err("Patching NOP failed.\n");
 213                 return -EPERM;
 214         }
 215
 216         return 0;
 217 }
 218
 219 #else /* !PPC64 */
 220 static int
 221 __ftrace_make_nop(struct module *mod,
 222                   struct dyn_ftrace *rec, unsigned long addr)
 223 {
 224         struct ppc_inst op;
 225         unsigned int jmp[4];
 226         unsigned long ip = rec->ip;
 227         unsigned long tramp;
 228
 229         if (copy_from_kernel_nofault(&op, (void *)ip, MCOUNT_INSN_SIZE))
 230                 return -EFAULT;
 231
 232         /* Make sure that that this is still a 24bit jump */
 233         if (!is_bl_op(op)) {
 234                 pr_err("Not expected bl: opcode is %s\n", ppc_inst_as_str(op));
 235                 return -EINVAL;
 236         }
 237
 238         /* lets find where the pointer goes */
 239         tramp = find_bl_target(ip, op);
 240
 241         /*
 242          * On PPC32 the trampoline looks like:
 243          *  0x3d, 0x80, 0x00, 0x00  lis r12,sym@ha
 244          *  0x39, 0x8c, 0x00, 0x00  addi r12,r12,sym@l
 245          *  0x7d, 0x89, 0x03, 0xa6  mtctr r12
 246          *  0x4e, 0x80, 0x04, 0x20  bctr
 247          */
 248
 249         pr_devel("ip:%lx jumps to %lx", ip, tramp);
 250
 251         /* Find where the trampoline jumps to */
 252         if (copy_from_kernel_nofault(jmp, (void *)tramp, sizeof(jmp))) {
 253                 pr_err("Failed to read %lx\n", tramp);
 254                 return -EFAULT;
 255         }
 256
 257         pr_devel(" %08x %08x ", jmp[0], jmp[1]);
 258
 259         /* verify that this is what we expect it to be */
 260         if (((jmp[0] & 0xffff0000) != 0x3d800000) ||
 261             ((jmp[1] & 0xffff0000) != 0x398c0000) ||
 262             (jmp[2] != 0x7d8903a6) ||
 263             (jmp[3] != 0x4e800420)) {
 264                 pr_err("Not a trampoline\n");
 265                 return -EINVAL;
 266         }
 267
 268         tramp = (jmp[1] & 0xffff) |
 269                 ((jmp[0] & 0xffff) << 16);
 270         if (tramp & 0x8000)
 271                 tramp -= 0x10000;
 272
 273         pr_devel(" %lx ", tramp);
 274
 275         if (tramp != addr) {
 276                 pr_err("Trampoline location %08lx does not match addr\n",
 277                        tramp);
 278                 return -EINVAL;
 279         }
 280
 281         op = ppc_inst(PPC_INST_NOP);
 282
 283         if (patch_instruction((struct ppc_inst *)ip, op))
 284                 return -EPERM;
 285
 286         return 0;
 287 }
 288 #endif /* PPC64 */
 289 #endif /* CONFIG_MODULES */
 290
 291 static unsigned long find_ftrace_tramp(unsigned long ip)
 292 {
 293         int i;
 294         struct ppc_inst instr;
 295
 296         /*
 297          * We have the compiler generated long_branch tramps at the end
 298          * and we prefer those
 299          */
 300         for (i = NUM_FTRACE_TRAMPS - 1; i >= 0; i--)
 301                 if (!ftrace_tramps[i])
 302                         continue;
 303                 else if (create_branch(&instr, (void *)ip,
 304                                        ftrace_tramps[i], 0) == 0)
 305                         return ftrace_tramps[i];
 306
 307         return 0;
 308 }
 309
 310 static int add_ftrace_tramp(unsigned long tramp)
 311 {
 312         int i;
 313
 314         for (i = 0; i < NUM_FTRACE_TRAMPS; i++)
 315                 if (!ftrace_tramps[i]) {
 316                         ftrace_tramps[i] = tramp;
 317                         return 0;
 318                 }
 319
 320         return -1;
 321 }
 322
 323 /*
 324  * If this is a compiler generated long_branch trampoline (essentially, a
 325  * trampoline that has a branch to _mcount()), we re-write the branch to
 326  * instead go to ftrace_[regs_]caller() and note down the location of this
 327  * trampoline.
 328  */
 329 static int setup_mcount_compiler_tramp(unsigned long tramp)
 330 {
 331         int i;
 332         struct ppc_inst op;
 333         unsigned long ptr;
 334         struct ppc_inst instr;
 335         static unsigned long ftrace_plt_tramps[NUM_FTRACE_TRAMPS];
 336
 337         /* Is this a known long jump tramp? */
 338         for (i = 0; i < NUM_FTRACE_TRAMPS; i++)
 339                 if (!ftrace_tramps[i])
 340                         break;
 341                 else if (ftrace_tramps[i] == tramp)
 342                         return 0;
 343
 344         /* Is this a known plt tramp? */
 345         for (i = 0; i < NUM_FTRACE_TRAMPS; i++)
 346                 if (!ftrace_plt_tramps[i])
 347                         break;
 348                 else if (ftrace_plt_tramps[i] == tramp)
 349                         return -1;
 350
 351         /* New trampoline -- read where this goes */
 352         if (probe_kernel_read_inst(&op, (void *)tramp)) {
 353                 pr_debug("Fetching opcode failed.\n");
 354                 return -1;
 355         }
 356
 357         /* Is this a 24 bit branch? */
 358         if (!is_b_op(op)) {
 359                 pr_debug("Trampoline is not a long branch tramp.\n");
 360                 return -1;
 361         }
 362
 363         /* lets find where the pointer goes */
 364         ptr = find_bl_target(tramp, op);
 365
 366         if (ptr != ppc_global_function_entry((void *)_mcount)) {
 367                 pr_debug("Trampoline target %p is not _mcount\n", (void *)ptr);
 368                 return -1;
 369         }
 370
 371         /* Let's re-write the tramp to go to ftrace_[regs_]caller */
 372 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
 373         ptr = ppc_global_function_entry((void *)ftrace_regs_caller);
 374 #else
 375         ptr = ppc_global_function_entry((void *)ftrace_caller);
 376 #endif
 377         if (create_branch(&instr, (void *)tramp, ptr, 0)) {
 378                 pr_debug("%ps is not reachable from existing mcount tramp\n",
 379                                 (void *)ptr);
 380                 return -1;
 381         }
 382
 383         if (patch_branch((struct ppc_inst *)tramp, ptr, 0)) {
 384                 pr_debug("REL24 out of range!\n");
 385                 return -1;
 386         }
 387
 388         if (add_ftrace_tramp(tramp)) {
 389                 pr_debug("No tramp locations left\n");
 390                 return -1;
 391         }
 392
 393         return 0;
 394 }
 395
 396 static int __ftrace_make_nop_kernel(struct dyn_ftrace *rec, unsigned long addr)
 397 {
 398         unsigned long tramp, ip = rec->ip;
 399         struct ppc_inst op;
 400
 401         /* Read where this goes */
 402         if (probe_kernel_read_inst(&op, (void *)ip)) {
 403                 pr_err("Fetching opcode failed.\n");
 404                 return -EFAULT;
 405         }
 406
 407         /* Make sure that that this is still a 24bit jump */
 408         if (!is_bl_op(op)) {
 409                 pr_err("Not expected bl: opcode is %s\n", ppc_inst_as_str(op));
 410                 return -EINVAL;
 411         }
 412
 413         /* Let's find where the pointer goes */
 414         tramp = find_bl_target(ip, op);
 415
 416         pr_devel("ip:%lx jumps to %lx", ip, tramp);
 417
 418         if (setup_mcount_compiler_tramp(tramp)) {
 419                 /* Are other trampolines reachable? */
 420                 if (!find_ftrace_tramp(ip)) {
 421                         pr_err("No ftrace trampolines reachable from %ps\n",
 422                                         (void *)ip);
 423                         return -EINVAL;
 424                 }
 425         }
 426
 427         if (patch_instruction((struct ppc_inst *)ip, ppc_inst(PPC_INST_NOP))) {
 428                 pr_err("Patching NOP failed.\n");
 429                 return -EPERM;
 430         }
 431
 432         return 0;
 433 }
 434
 435 int ftrace_make_nop(struct module *mod,
 436                     struct dyn_ftrace *rec, unsigned long addr)
 437 {
 438         unsigned long ip = rec->ip;
 439         struct ppc_inst old, new;
 440
 441         /*
 442          * If the calling address is more that 24 bits away,
 443          * then we had to use a trampoline to make the call.
 444          * Otherwise just update the call site.
 445          */
 446         if (test_24bit_addr(ip, addr)) {
 447                 /* within range */
 448                 old = ftrace_call_replace(ip, addr, 1);
 449                 new = ppc_inst(PPC_INST_NOP);
 450                 return ftrace_modify_code(ip, old, new);
 451         } else if (core_kernel_text(ip))
 452                 return __ftrace_make_nop_kernel(rec, addr);
 453
 454 #ifdef CONFIG_MODULES
 455         /*
 456          * Out of range jumps are called from modules.
 457          * We should either already have a pointer to the module
 458          * or it has been passed in.
 459          */
 460         if (!rec->arch.mod) {
 461                 if (!mod) {
 462                         pr_err("No module loaded addr=%lx\n", addr);
 463                         return -EFAULT;
 464                 }
 465                 rec->arch.mod = mod;
 466         } else if (mod) {
 467                 if (mod != rec->arch.mod) {
 468                         pr_err("Record mod %p not equal to passed in mod %p\n",
 469                                rec->arch.mod, mod);
 470                         return -EINVAL;
 471                 }
 472                 /* nothing to do if mod == rec->arch.mod */
 473         } else
 474                 mod = rec->arch.mod;
 475
 476         return __ftrace_make_nop(mod, rec, addr);
 477 #else
 478         /* We should not get here without modules */
 479         return -EINVAL;
 480 #endif /* CONFIG_MODULES */
 481 }
 482
 483 #ifdef CONFIG_MODULES
 484 #ifdef CONFIG_PPC64
 485 /*
 486  * Examine the existing instructions for __ftrace_make_call.
 487  * They should effectively be a NOP, and follow formal constraints,
 488  * depending on the ABI. Return false if they don't.
 489  */
 490 #ifndef CONFIG_MPROFILE_KERNEL
 491 static int
 492 expected_nop_sequence(void *ip, struct ppc_inst op0, struct ppc_inst op1)
 493 {
 494         /*
 495          * We expect to see:
 496          *
 497          * b +8
 498          * ld r2,XX(r1)
 499          *
 500          * The load offset is different depending on the ABI. For simplicity
 501          * just mask it out when doing the compare.
 502          */
 503         if (!ppc_inst_equal(op0, ppc_inst(0x48000008)) ||
 504             (ppc_inst_val(op1) & 0xffff0000) != 0xe8410000)
 505                 return 0;
 506         return 1;
 507 }
 508 #else
 509 static int
 510 expected_nop_sequence(void *ip, struct ppc_inst op0, struct ppc_inst op1)
 511 {
 512         /* look for patched "NOP" on ppc64 with -mprofile-kernel */
 513         if (!ppc_inst_equal(op0, ppc_inst(PPC_INST_NOP)))
 514                 return 0;
 515         return 1;
 516 }
 517 #endif
 518
 519 static int
 520 __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
 521 {
 522         struct ppc_inst op[2];
 523         struct ppc_inst instr;
 524         void *ip = (void *)rec->ip;
 525         unsigned long entry, ptr, tramp;
 526         struct module *mod = rec->arch.mod;
 527
 528         /* read where this goes */
 529         if (probe_kernel_read_inst(op, ip))
 530                 return -EFAULT;
 531
 532         if (probe_kernel_read_inst(op + 1, ip + 4))
 533                 return -EFAULT;
 534
 535         if (!expected_nop_sequence(ip, op[0], op[1])) {
 536                 pr_err("Unexpected call sequence at %p: %s %s\n",
 537                 ip, ppc_inst_as_str(op[0]), ppc_inst_as_str(op[1]));
 538                 return -EINVAL;
 539         }
 540
 541         /* If we never set up ftrace trampoline(s), then bail */
 542 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
 543         if (!mod->arch.tramp || !mod->arch.tramp_regs) {
 544 #else
 545         if (!mod->arch.tramp) {
 546 #endif
 547                 pr_err("No ftrace trampoline\n");
 548                 return -EINVAL;
 549         }
 550
 551 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
 552         if (rec->flags & FTRACE_FL_REGS)
 553                 tramp = mod->arch.tramp_regs;
 554         else
 555 #endif
 556                 tramp = mod->arch.tramp;
 557
 558         if (module_trampoline_target(mod, tramp, &ptr)) {
 559                 pr_err("Failed to get trampoline target\n");
 560                 return -EFAULT;
 561         }
 562
 563         pr_devel("trampoline target %lx", ptr);
 564
 565         entry = ppc_global_function_entry((void *)addr);
 566         /* This should match what was called */
 567         if (ptr != entry) {
 568                 pr_err("addr %lx does not match expected %lx\n", ptr, entry);
 569                 return -EINVAL;
 570         }
 571
 572         /* Ensure branch is within 24 bits */
 573         if (create_branch(&instr, ip, tramp, BRANCH_SET_LINK)) {
 574                 pr_err("Branch out of range\n");
 575                 return -EINVAL;
 576         }
 577
 578         if (patch_branch(ip, tramp, BRANCH_SET_LINK)) {
 579                 pr_err("REL24 out of range!\n");
 580                 return -EINVAL;
 581         }
 582
 583         return 0;
 584 }
 585
 586 #else  /* !CONFIG_PPC64: */
 587 static int
 588 __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
 589 {
 590         int err;
 591         struct ppc_inst op;
 592         unsigned long ip = rec->ip;
 593
 594         /* read where this goes */
 595         if (probe_kernel_read_inst(&op, (void *)ip))
 596                 return -EFAULT;
 597
 598         /* It should be pointing to a nop */
 599         if (!ppc_inst_equal(op,  ppc_inst(PPC_INST_NOP))) {
 600                 pr_err("Expected NOP but have %s\n", ppc_inst_as_str(op));
 601                 return -EINVAL;
 602         }
 603
 604         /* If we never set up a trampoline to ftrace_caller, then bail */
 605         if (!rec->arch.mod->arch.tramp) {
 606                 pr_err("No ftrace trampoline\n");
 607                 return -EINVAL;
 608         }
 609
 610         /* create the branch to the trampoline */
 611         err = create_branch(&op, (struct ppc_inst *)ip,
 612                             rec->arch.mod->arch.tramp, BRANCH_SET_LINK);
 613         if (err) {
 614                 pr_err("REL24 out of range!\n");
 615                 return -EINVAL;
 616         }
 617
 618         pr_devel("write to %lx\n", rec->ip);
 619
 620         if (patch_instruction((struct ppc_inst *)ip, op))
 621                 return -EPERM;
 622
 623         return 0;
 624 }
 625 #endif /* CONFIG_PPC64 */
 626 #endif /* CONFIG_MODULES */
 627
 628 static int __ftrace_make_call_kernel(struct dyn_ftrace *rec, unsigned long addr)
 629 {
 630         struct ppc_inst op;
 631         void *ip = (void *)rec->ip;
 632         unsigned long tramp, entry, ptr;
 633
 634         /* Make sure we're being asked to patch branch to a known ftrace addr */
 635         entry = ppc_global_function_entry((void *)ftrace_caller);
 636         ptr = ppc_global_function_entry((void *)addr);
 637
 638         if (ptr != entry) {
 639 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
 640                 entry = ppc_global_function_entry((void *)ftrace_regs_caller);
 641                 if (ptr != entry) {
 642 #endif
 643                         pr_err("Unknown ftrace addr to patch: %ps\n", (void *)ptr);
 644                         return -EINVAL;
 645 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
 646                 }
 647 #endif
 648         }
 649
 650         /* Make sure we have a nop */
 651         if (probe_kernel_read_inst(&op, ip)) {
 652                 pr_err("Unable to read ftrace location %p\n", ip);
 653                 return -EFAULT;
 654         }
 655
 656         if (!ppc_inst_equal(op, ppc_inst(PPC_INST_NOP))) {
 657                 pr_err("Unexpected call sequence at %p: %s\n", ip, ppc_inst_as_str(op));
 658                 return -EINVAL;
 659         }
 660
 661         tramp = find_ftrace_tramp((unsigned long)ip);
 662         if (!tramp) {
 663                 pr_err("No ftrace trampolines reachable from %ps\n", ip);
 664                 return -EINVAL;
 665         }
 666
 667         if (patch_branch(ip, tramp, BRANCH_SET_LINK)) {
 668                 pr_err("Error patching branch to ftrace tramp!\n");
 669                 return -EINVAL;
 670         }
 671
 672         return 0;
 673 }
 674
 675 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
 676 {
 677         unsigned long ip = rec->ip;
 678         struct ppc_inst old, new;
 679
 680         /*
 681          * If the calling address is more that 24 bits away,
 682          * then we had to use a trampoline to make the call.
 683          * Otherwise just update the call site.
 684          */
 685         if (test_24bit_addr(ip, addr)) {
 686                 /* within range */
 687                 old = ppc_inst(PPC_INST_NOP);
 688                 new = ftrace_call_replace(ip, addr, 1);
 689                 return ftrace_modify_code(ip, old, new);
 690         } else if (core_kernel_text(ip))
 691                 return __ftrace_make_call_kernel(rec, addr);
 692
 693 #ifdef CONFIG_MODULES
 694         /*
 695          * Out of range jumps are called from modules.
 696          * Being that we are converting from nop, it had better
 697          * already have a module defined.
 698          */
 699         if (!rec->arch.mod) {
 700                 pr_err("No module loaded\n");
 701                 return -EINVAL;
 702         }
 703
 704         return __ftrace_make_call(rec, addr);
 705 #else
 706         /* We should not get here without modules */
 707         return -EINVAL;
 708 #endif /* CONFIG_MODULES */
 709 }
 710
 711 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
 712 #ifdef CONFIG_MODULES
 713 static int
 714 __ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
 715                                         unsigned long addr)
 716 {
 717         struct ppc_inst op;
 718         unsigned long ip = rec->ip;
 719         unsigned long entry, ptr, tramp;
 720         struct module *mod = rec->arch.mod;
 721
 722         /* If we never set up ftrace trampolines, then bail */
 723         if (!mod->arch.tramp || !mod->arch.tramp_regs) {
 724                 pr_err("No ftrace trampoline\n");
 725                 return -EINVAL;
 726         }
 727
 728         /* read where this goes */
 729         if (probe_kernel_read_inst(&op, (void *)ip)) {
 730                 pr_err("Fetching opcode failed.\n");
 731                 return -EFAULT;
 732         }
 733
 734         /* Make sure that that this is still a 24bit jump */
 735         if (!is_bl_op(op)) {
 736                 pr_err("Not expected bl: opcode is %s\n", ppc_inst_as_str(op));
 737                 return -EINVAL;
 738         }
 739
 740         /* lets find where the pointer goes */
 741         tramp = find_bl_target(ip, op);
 742         entry = ppc_global_function_entry((void *)old_addr);
 743
 744         pr_devel("ip:%lx jumps to %lx", ip, tramp);
 745
 746         if (tramp != entry) {
 747                 /* old_addr is not within range, so we must have used a trampoline */
 748                 if (module_trampoline_target(mod, tramp, &ptr)) {
 749                         pr_err("Failed to get trampoline target\n");
 750                         return -EFAULT;
 751                 }
 752
 753                 pr_devel("trampoline target %lx", ptr);
 754
 755                 /* This should match what was called */
 756                 if (ptr != entry) {
 757                         pr_err("addr %lx does not match expected %lx\n", ptr, entry);
 758                         return -EINVAL;
 759                 }
 760         }
 761
 762         /* The new target may be within range */
 763         if (test_24bit_addr(ip, addr)) {
 764                 /* within range */
 765                 if (patch_branch((struct ppc_inst *)ip, addr, BRANCH_SET_LINK)) {
 766                         pr_err("REL24 out of range!\n");
 767                         return -EINVAL;
 768                 }
 769
 770                 return 0;
 771         }
 772
 773         if (rec->flags & FTRACE_FL_REGS)
 774                 tramp = mod->arch.tramp_regs;
 775         else
 776                 tramp = mod->arch.tramp;
 777
 778         if (module_trampoline_target(mod, tramp, &ptr)) {
 779                 pr_err("Failed to get trampoline target\n");
 780                 return -EFAULT;
 781         }
 782
 783         pr_devel("trampoline target %lx", ptr);
 784
 785         entry = ppc_global_function_entry((void *)addr);
 786         /* This should match what was called */
 787         if (ptr != entry) {
 788                 pr_err("addr %lx does not match expected %lx\n", ptr, entry);
 789                 return -EINVAL;
 790         }
 791
 792         /* Ensure branch is within 24 bits */
 793         if (create_branch(&op, (struct ppc_inst *)ip, tramp, BRANCH_SET_LINK)) {
 794                 pr_err("Branch out of range\n");
 795                 return -EINVAL;
 796         }
 797
 798         if (patch_branch((struct ppc_inst *)ip, tramp, BRANCH_SET_LINK)) {
 799                 pr_err("REL24 out of range!\n");
 800                 return -EINVAL;
 801         }
 802
 803         return 0;
 804 }
 805 #endif
 806
 807 int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
 808                         unsigned long addr)
 809 {
 810         unsigned long ip = rec->ip;
 811         struct ppc_inst old, new;
 812
 813         /*
 814          * If the calling address is more that 24 bits away,
 815          * then we had to use a trampoline to make the call.
 816          * Otherwise just update the call site.
 817          */
 818         if (test_24bit_addr(ip, addr) && test_24bit_addr(ip, old_addr)) {
 819                 /* within range */
 820                 old = ftrace_call_replace(ip, old_addr, 1);
 821                 new = ftrace_call_replace(ip, addr, 1);
 822                 return ftrace_modify_code(ip, old, new);
 823         } else if (core_kernel_text(ip)) {
 824                 /*
 825                  * We always patch out of range locations to go to the regs
 826                  * variant, so there is nothing to do here
 827                  */
 828                 return 0;
 829         }
 830
 831 #ifdef CONFIG_MODULES
 832         /*
 833          * Out of range jumps are called from modules.
 834          */
 835         if (!rec->arch.mod) {
 836                 pr_err("No module loaded\n");
 837                 return -EINVAL;
 838         }
 839
 840         return __ftrace_modify_call(rec, old_addr, addr);
 841 #else
 842         /* We should not get here without modules */
 843         return -EINVAL;
 844 #endif /* CONFIG_MODULES */
 845 }
 846 #endif
 847
 848 int ftrace_update_ftrace_func(ftrace_func_t func)
 849 {
 850         unsigned long ip = (unsigned long)(&ftrace_call);
 851         struct ppc_inst old, new;
 852         int ret;
 853
 854         old = ppc_inst_read((struct ppc_inst *)&ftrace_call);
 855         new = ftrace_call_replace(ip, (unsigned long)func, 1);
 856         ret = ftrace_modify_code(ip, old, new);
 857
 858 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
 859         /* Also update the regs callback function */
 860         if (!ret) {
 861                 ip = (unsigned long)(&ftrace_regs_call);
 862                 old = ppc_inst_read((struct ppc_inst *)&ftrace_regs_call);
 863                 new = ftrace_call_replace(ip, (unsigned long)func, 1);
 864                 ret = ftrace_modify_code(ip, old, new);
 865         }
 866 #endif
 867
 868         return ret;
 869 }
 870
 871 /*
 872  * Use the default ftrace_modify_all_code, but without
 873  * stop_machine().
 874  */
 875 void arch_ftrace_update_code(int command)
 876 {
 877         ftrace_modify_all_code(command);
 878 }
 879
 880 #ifdef CONFIG_PPC64
 881 #define PACATOC offsetof(struct paca_struct, kernel_toc)
 882
 883 extern unsigned int ftrace_tramp_text[], ftrace_tramp_init[];
 884
 885 int __init ftrace_dyn_arch_init(void)
 886 {
 887         int i;
 888         unsigned int *tramp[] = { ftrace_tramp_text, ftrace_tramp_init };
 889         u32 stub_insns[] = {
 890                 0xe98d0000 | PACATOC,   /* ld      r12,PACATOC(r13)     */
 891                 0x3d8c0000,             /* addis   r12,r12,<high>       */
 892                 0x398c0000,             /* addi    r12,r12,<low>        */
 893                 0x7d8903a6,             /* mtctr   r12                  */
 894                 0x4e800420,             /* bctr                         */
 895         };
 896 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
 897         unsigned long addr = ppc_global_function_entry((void *)ftrace_regs_caller);
 898 #else
 899         unsigned long addr = ppc_global_function_entry((void *)ftrace_caller);
 900 #endif
 901         long reladdr = addr - kernel_toc_addr();
 902
 903         if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) {
 904                 pr_err("Address of %ps out of range of kernel_toc.\n",
 905                                 (void *)addr);
 906                 return -1;
 907         }
 908
 909         for (i = 0; i < 2; i++) {
 910                 memcpy(tramp[i], stub_insns, sizeof(stub_insns));
 911                 tramp[i][1] |= PPC_HA(reladdr);
 912                 tramp[i][2] |= PPC_LO(reladdr);
 913                 add_ftrace_tramp((unsigned long)tramp[i]);
 914         }
 915
 916         return 0;
 917 }
 918 #else
 919 int __init ftrace_dyn_arch_init(void)
 920 {
 921         return 0;
 922 }
 923 #endif
 924 #endif /* CONFIG_DYNAMIC_FTRACE */
 925
 926 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 927
 928 extern void ftrace_graph_call(void);
 929 extern void ftrace_graph_stub(void);
 930
 931 int ftrace_enable_ftrace_graph_caller(void)
 932 {
 933         unsigned long ip = (unsigned long)(&ftrace_graph_call);
 934         unsigned long addr = (unsigned long)(&ftrace_graph_caller);
 935         unsigned long stub = (unsigned long)(&ftrace_graph_stub);
 936         struct ppc_inst old, new;
 937
 938         old = ftrace_call_replace(ip, stub, 0);
 939         new = ftrace_call_replace(ip, addr, 0);
 940
 941         return ftrace_modify_code(ip, old, new);
 942 }
 943
 944 int ftrace_disable_ftrace_graph_caller(void)
 945 {
 946         unsigned long ip = (unsigned long)(&ftrace_graph_call);
 947         unsigned long addr = (unsigned long)(&ftrace_graph_caller);
 948         unsigned long stub = (unsigned long)(&ftrace_graph_stub);
 949         struct ppc_inst old, new;
 950
 951         old = ftrace_call_replace(ip, addr, 0);
 952         new = ftrace_call_replace(ip, stub, 0);
 953
 954         return ftrace_modify_code(ip, old, new);
 955 }
 956
 957 /*
 958  * Hook the return address and push it in the stack of return addrs
 959  * in current thread info. Return the address we want to divert to.
 960  */
 961 unsigned long prepare_ftrace_return(unsigned long parent, unsigned long ip,
 962                                                 unsigned long sp)
 963 {
 964         unsigned long return_hooker;
 965
 966         if (unlikely(ftrace_graph_is_dead()))
 967                 goto out;
 968
 969         if (unlikely(atomic_read(&current->tracing_graph_pause)))
 970                 goto out;
 971
 972         return_hooker = ppc_function_entry(return_to_handler);
 973
 974         if (!function_graph_enter(parent, ip, 0, (unsigned long *)sp))
 975                 parent = return_hooker;
 976 out:
 977         return parent;
 978 }
 979 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
 980
 981 #ifdef PPC64_ELF_ABI_v1
 982 char *arch_ftrace_match_adjust(char *str, const char *search)
 983 {
 984         if (str[0] == '.' && search[0] != '.')
 985                 return str + 1;
 986         else
 987                 return str;
 988 }
 989 #endif /* PPC64_ELF_ABI_v1 */