2 * Code for replacing ftrace calls with jumps.
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
6 * Thanks goes out to P.A. Semi, Inc for supplying me with a PPC64 box.
10 #include <linux/spinlock.h>
11 #include <linux/hardirq.h>
12 #include <linux/uaccess.h>
13 #include <linux/module.h>
14 #include <linux/ftrace.h>
15 #include <linux/percpu.h>
16 #include <linux/init.h>
17 #include <linux/list.h>
19 #include <asm/cacheflush.h>
20 #include <asm/code-patching.h>
21 #include <asm/ftrace.h>
26 #define DEBUGP(fmt , ...) do { } while (0)
29 static unsigned int ftrace_nop
= PPC_NOP_INSTR
;
32 # define GET_ADDR(addr) addr
34 /* PowerPC64's functions are data that points to the functions */
35 # define GET_ADDR(addr) (*(unsigned long *)addr)
39 static unsigned int ftrace_calc_offset(long ip
, long addr
)
41 return (int)(addr
- ip
);
44 static unsigned char *ftrace_nop_replace(void)
46 return (char *)&ftrace_nop
;
49 static unsigned char *ftrace_call_replace(unsigned long ip
, unsigned long addr
)
51 static unsigned int op
;
54 * It would be nice to just use create_function_call, but that will
55 * update the code itself. Here we need to just return the
56 * instruction that is going to be modified, without modifying the
59 addr
= GET_ADDR(addr
);
61 /* Set to "bl addr" */
62 op
= 0x48000001 | (ftrace_calc_offset(ip
, addr
) & 0x03fffffc);
65 * No locking needed, this must be called via kstop_machine
66 * which in essence is like running on a uniprocessor machine.
68 return (unsigned char *)&op
;
72 # define _ASM_ALIGN " .align 3 "
73 # define _ASM_PTR " .llong "
75 # define _ASM_ALIGN " .align 2 "
76 # define _ASM_PTR " .long "
80 ftrace_modify_code(unsigned long ip
, unsigned char *old_code
,
81 unsigned char *new_code
)
83 unsigned char replaced
[MCOUNT_INSN_SIZE
];
86 * Note: Due to modules and __init, code can
87 * disappear and change, we need to protect against faulting
88 * as well as code changing. We do this by using the
89 * probe_kernel_* functions.
91 * No real locking needed, this code is run through
92 * kstop_machine, or before SMP starts.
95 /* read the text we want to modify */
96 if (probe_kernel_read(replaced
, (void *)ip
, MCOUNT_INSN_SIZE
))
99 /* Make sure it is what we expect it to be */
100 if (memcmp(replaced
, old_code
, MCOUNT_INSN_SIZE
) != 0)
103 /* replace the text with the new text */
104 if (probe_kernel_write((void *)ip
, new_code
, MCOUNT_INSN_SIZE
))
107 flush_icache_range(ip
, ip
+ 8);
113 * Helper functions that are the same for both PPC64 and PPC32.
115 static int test_24bit_addr(unsigned long ip
, unsigned long addr
)
118 /* use the create_branch to verify that this offset can be branched */
119 return create_branch((unsigned int *)ip
, addr
, 0);
122 static int is_bl_op(unsigned int op
)
124 return (op
& 0xfc000003) == 0x48000001;
127 static unsigned long find_bl_target(unsigned long ip
, unsigned int op
)
131 offset
= (op
& 0x03fffffc);
133 if (offset
& 0x02000000)
134 offset
|= 0xfe000000;
136 return ip
+ (long)offset
;
141 __ftrace_make_nop(struct module
*mod
,
142 struct dyn_ftrace
*rec
, unsigned long addr
)
147 unsigned long ip
= rec
->ip
;
151 /* read where this goes */
152 if (probe_kernel_read(&op
, (void *)ip
, sizeof(int)))
155 /* Make sure that that this is still a 24bit jump */
157 printk(KERN_ERR
"Not expected bl: opcode is %x\n", op
);
161 /* lets find where the pointer goes */
162 tramp
= find_bl_target(ip
, op
);
165 * On PPC64 the trampoline looks like:
166 * 0x3d, 0x82, 0x00, 0x00, addis r12,r2, <high>
167 * 0x39, 0x8c, 0x00, 0x00, addi r12,r12, <low>
168 * Where the bytes 2,3,6 and 7 make up the 32bit offset
169 * to the TOC that holds the pointer.
171 * 0xf8, 0x41, 0x00, 0x28, std r2,40(r1)
172 * 0xe9, 0x6c, 0x00, 0x20, ld r11,32(r12)
173 * The actually address is 32 bytes from the offset
175 * 0xe8, 0x4c, 0x00, 0x28, ld r2,40(r12)
178 DEBUGP("ip:%lx jumps to %lx r2: %lx", ip
, tramp
, mod
->arch
.toc
);
180 /* Find where the trampoline jumps to */
181 if (probe_kernel_read(jmp
, (void *)tramp
, sizeof(jmp
))) {
182 printk(KERN_ERR
"Failed to read %lx\n", tramp
);
186 DEBUGP(" %08x %08x", jmp
[0], jmp
[1]);
188 /* verify that this is what we expect it to be */
189 if (((jmp
[0] & 0xffff0000) != 0x3d820000) ||
190 ((jmp
[1] & 0xffff0000) != 0x398c0000) ||
191 (jmp
[2] != 0xf8410028) ||
192 (jmp
[3] != 0xe96c0020) ||
193 (jmp
[4] != 0xe84c0028)) {
194 printk(KERN_ERR
"Not a trampoline\n");
198 offset
= (unsigned)((unsigned short)jmp
[0]) << 16 |
199 (unsigned)((unsigned short)jmp
[1]);
201 DEBUGP(" %x ", offset
);
203 /* get the address this jumps too */
204 tramp
= mod
->arch
.toc
+ offset
+ 32;
205 DEBUGP("toc: %lx", tramp
);
207 if (probe_kernel_read(jmp
, (void *)tramp
, 8)) {
208 printk(KERN_ERR
"Failed to read %lx\n", tramp
);
212 DEBUGP(" %08x %08x\n", jmp
[0], jmp
[1]);
214 ptr
= ((unsigned long)jmp
[0] << 32) + jmp
[1];
216 /* This should match what was called */
217 if (ptr
!= GET_ADDR(addr
)) {
218 printk(KERN_ERR
"addr does not match %lx\n", ptr
);
223 * We want to nop the line, but the next line is
224 * 0xe8, 0x41, 0x00, 0x28 ld r2,40(r1)
225 * This needs to be turned to a nop too.
227 if (probe_kernel_read(&op
, (void *)(ip
+4), MCOUNT_INSN_SIZE
))
230 if (op
!= 0xe8410028) {
231 printk(KERN_ERR
"Next line is not ld! (%08x)\n", op
);
236 * Milton Miller pointed out that we can not blindly do nops.
237 * If a task was preempted when calling a trace function,
238 * the nops will remove the way to restore the TOC in r2
239 * and the r2 TOC will get corrupted.
244 * bl <tramp> <==== will be replaced with "b 1f"
248 op
= 0x48000008; /* b +8 */
250 if (probe_kernel_write((void *)ip
, &op
, MCOUNT_INSN_SIZE
))
254 flush_icache_range(ip
, ip
+ 8);
261 __ftrace_make_nop(struct module
*mod
,
262 struct dyn_ftrace
*rec
, unsigned long addr
)
266 unsigned long ip
= rec
->ip
;
269 if (probe_kernel_read(&op
, (void *)ip
, MCOUNT_INSN_SIZE
))
272 /* Make sure that that this is still a 24bit jump */
274 printk(KERN_ERR
"Not expected bl: opcode is %x\n", op
);
278 /* lets find where the pointer goes */
279 tramp
= find_bl_target(ip
, op
);
282 * On PPC32 the trampoline looks like:
283 * 0x3d, 0x60, 0x00, 0x00 lis r11,sym@ha
284 * 0x39, 0x6b, 0x00, 0x00 addi r11,r11,sym@l
285 * 0x7d, 0x69, 0x03, 0xa6 mtctr r11
286 * 0x4e, 0x80, 0x04, 0x20 bctr
289 DEBUGP("ip:%lx jumps to %lx", ip
, tramp
);
291 /* Find where the trampoline jumps to */
292 if (probe_kernel_read(jmp
, (void *)tramp
, sizeof(jmp
))) {
293 printk(KERN_ERR
"Failed to read %lx\n", tramp
);
297 DEBUGP(" %08x %08x ", jmp
[0], jmp
[1]);
299 /* verify that this is what we expect it to be */
300 if (((jmp
[0] & 0xffff0000) != 0x3d600000) ||
301 ((jmp
[1] & 0xffff0000) != 0x396b0000) ||
302 (jmp
[2] != 0x7d6903a6) ||
303 (jmp
[3] != 0x4e800420)) {
304 printk(KERN_ERR
"Not a trampoline\n");
308 tramp
= (jmp
[1] & 0xffff) |
309 ((jmp
[0] & 0xffff) << 16);
313 DEBUGP(" %x ", tramp
);
317 "Trampoline location %08lx does not match addr\n",
324 if (probe_kernel_write((void *)ip
, &op
, MCOUNT_INSN_SIZE
))
327 flush_icache_range(ip
, ip
+ 8);
333 int ftrace_make_nop(struct module
*mod
,
334 struct dyn_ftrace
*rec
, unsigned long addr
)
336 unsigned char *old
, *new;
337 unsigned long ip
= rec
->ip
;
340 * If the calling address is more that 24 bits away,
341 * then we had to use a trampoline to make the call.
342 * Otherwise just update the call site.
344 if (test_24bit_addr(ip
, addr
)) {
346 old
= ftrace_call_replace(ip
, addr
);
347 new = ftrace_nop_replace();
348 return ftrace_modify_code(ip
, old
, new);
352 * Out of range jumps are called from modules.
353 * We should either already have a pointer to the module
354 * or it has been passed in.
356 if (!rec
->arch
.mod
) {
358 printk(KERN_ERR
"No module loaded addr=%lx\n",
364 if (mod
!= rec
->arch
.mod
) {
366 "Record mod %p not equal to passed in mod %p\n",
370 /* nothing to do if mod == rec->arch.mod */
374 return __ftrace_make_nop(mod
, rec
, addr
);
380 __ftrace_make_call(struct dyn_ftrace
*rec
, unsigned long addr
)
383 unsigned long ip
= rec
->ip
;
385 /* read where this goes */
386 if (probe_kernel_read(op
, (void *)ip
, MCOUNT_INSN_SIZE
* 2))
390 * It should be pointing to two nops or
393 if (((op
[0] != 0x48000008) || (op
[1] != 0xe8410028)) &&
394 ((op
[0] != PPC_NOP_INSTR
) || (op
[1] != PPC_NOP_INSTR
))) {
395 printk(KERN_ERR
"Expected NOPs but have %x %x\n", op
[0], op
[1]);
399 /* If we never set up a trampoline to ftrace_caller, then bail */
400 if (!rec
->arch
.mod
->arch
.tramp
) {
401 printk(KERN_ERR
"No ftrace trampoline\n");
405 /* create the branch to the trampoline */
406 op
[0] = create_branch((unsigned int *)ip
,
407 rec
->arch
.mod
->arch
.tramp
, BRANCH_SET_LINK
);
409 printk(KERN_ERR
"REL24 out of range!\n");
416 DEBUGP("write to %lx\n", rec
->ip
);
418 if (probe_kernel_write((void *)ip
, op
, MCOUNT_INSN_SIZE
* 2))
421 flush_icache_range(ip
, ip
+ 8);
427 __ftrace_make_call(struct dyn_ftrace
*rec
, unsigned long addr
)
430 unsigned long ip
= rec
->ip
;
432 /* read where this goes */
433 if (probe_kernel_read(&op
, (void *)ip
, MCOUNT_INSN_SIZE
))
436 /* It should be pointing to a nop */
437 if (op
!= PPC_NOP_INSTR
) {
438 printk(KERN_ERR
"Expected NOP but have %x\n", op
);
442 /* If we never set up a trampoline to ftrace_caller, then bail */
443 if (!rec
->arch
.mod
->arch
.tramp
) {
444 printk(KERN_ERR
"No ftrace trampoline\n");
448 /* create the branch to the trampoline */
449 op
= create_branch((unsigned int *)ip
,
450 rec
->arch
.mod
->arch
.tramp
, BRANCH_SET_LINK
);
452 printk(KERN_ERR
"REL24 out of range!\n");
456 DEBUGP("write to %lx\n", rec
->ip
);
458 if (probe_kernel_write((void *)ip
, &op
, MCOUNT_INSN_SIZE
))
461 flush_icache_range(ip
, ip
+ 8);
465 #endif /* CONFIG_PPC64 */
467 int ftrace_make_call(struct dyn_ftrace
*rec
, unsigned long addr
)
469 unsigned char *old
, *new;
470 unsigned long ip
= rec
->ip
;
473 * If the calling address is more that 24 bits away,
474 * then we had to use a trampoline to make the call.
475 * Otherwise just update the call site.
477 if (test_24bit_addr(ip
, addr
)) {
479 old
= ftrace_nop_replace();
480 new = ftrace_call_replace(ip
, addr
);
481 return ftrace_modify_code(ip
, old
, new);
485 * Out of range jumps are called from modules.
486 * Being that we are converting from nop, it had better
487 * already have a module defined.
489 if (!rec
->arch
.mod
) {
490 printk(KERN_ERR
"No module loaded\n");
494 return __ftrace_make_call(rec
, addr
);
497 int ftrace_update_ftrace_func(ftrace_func_t func
)
499 unsigned long ip
= (unsigned long)(&ftrace_call
);
500 unsigned char old
[MCOUNT_INSN_SIZE
], *new;
503 memcpy(old
, &ftrace_call
, MCOUNT_INSN_SIZE
);
504 new = ftrace_call_replace(ip
, (unsigned long)func
);
505 ret
= ftrace_modify_code(ip
, old
, new);
510 int __init
ftrace_dyn_arch_init(void *data
)
512 /* caller expects data to be zero */
513 unsigned long *p
= data
;