2 * Kernel Probes (KProbes)
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18 * Copyright (C) IBM Corporation, 2002, 2004
20 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
21 * Probes initial implementation ( includes contributions from
23 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
24 * interface to access function arguments.
25 * 2004-Nov Ananth N Mavinakayanahalli <ananth@in.ibm.com> kprobes port
29 #include <linux/kprobes.h>
30 #include <linux/ptrace.h>
31 #include <linux/preempt.h>
32 #include <linux/module.h>
33 #include <linux/kdebug.h>
34 #include <linux/slab.h>
35 #include <asm/cacheflush.h>
36 #include <asm/sstep.h>
37 #include <asm/uaccess.h>
39 DEFINE_PER_CPU(struct kprobe
*, current_kprobe
) = NULL
;
40 DEFINE_PER_CPU(struct kprobe_ctlblk
, kprobe_ctlblk
);
42 struct kretprobe_blackpoint kretprobe_blacklist
[] = {{NULL
, NULL
}};
44 int __kprobes
arch_prepare_kprobe(struct kprobe
*p
)
47 kprobe_opcode_t insn
= *p
->addr
;
49 if ((unsigned long)p
->addr
& 0x03) {
50 printk("Attempt to register kprobe at an unaligned address\n");
52 } else if (IS_MTMSRD(insn
) || IS_RFID(insn
) || IS_RFI(insn
)) {
53 printk("Cannot register a kprobe on rfi/rfid or mtmsr[d]\n");
57 /* insn must be on a special executable page on ppc64. This is
58 * not explicitly required on ppc32 (right now), but it doesn't hurt */
60 p
->ainsn
.insn
= get_insn_slot();
66 memcpy(p
->ainsn
.insn
, p
->addr
,
67 MAX_INSN_SIZE
* sizeof(kprobe_opcode_t
));
69 flush_icache_range((unsigned long)p
->ainsn
.insn
,
70 (unsigned long)p
->ainsn
.insn
+ sizeof(kprobe_opcode_t
));
73 p
->ainsn
.boostable
= 0;
77 void __kprobes
arch_arm_kprobe(struct kprobe
*p
)
79 *p
->addr
= BREAKPOINT_INSTRUCTION
;
80 flush_icache_range((unsigned long) p
->addr
,
81 (unsigned long) p
->addr
+ sizeof(kprobe_opcode_t
));
84 void __kprobes
arch_disarm_kprobe(struct kprobe
*p
)
87 flush_icache_range((unsigned long) p
->addr
,
88 (unsigned long) p
->addr
+ sizeof(kprobe_opcode_t
));
91 void __kprobes
arch_remove_kprobe(struct kprobe
*p
)
94 free_insn_slot(p
->ainsn
.insn
, 0);
99 static void __kprobes
prepare_singlestep(struct kprobe
*p
, struct pt_regs
*regs
)
101 enable_single_step(regs
);
104 * On powerpc we should single step on the original
105 * instruction even if the probed insn is a trap
106 * variant as values in regs could play a part in
107 * if the trap is taken or not
109 regs
->nip
= (unsigned long)p
->ainsn
.insn
;
112 static void __kprobes
save_previous_kprobe(struct kprobe_ctlblk
*kcb
)
114 kcb
->prev_kprobe
.kp
= kprobe_running();
115 kcb
->prev_kprobe
.status
= kcb
->kprobe_status
;
116 kcb
->prev_kprobe
.saved_msr
= kcb
->kprobe_saved_msr
;
119 static void __kprobes
restore_previous_kprobe(struct kprobe_ctlblk
*kcb
)
121 __get_cpu_var(current_kprobe
) = kcb
->prev_kprobe
.kp
;
122 kcb
->kprobe_status
= kcb
->prev_kprobe
.status
;
123 kcb
->kprobe_saved_msr
= kcb
->prev_kprobe
.saved_msr
;
126 static void __kprobes
set_current_kprobe(struct kprobe
*p
, struct pt_regs
*regs
,
127 struct kprobe_ctlblk
*kcb
)
129 __get_cpu_var(current_kprobe
) = p
;
130 kcb
->kprobe_saved_msr
= regs
->msr
;
133 void __kprobes
arch_prepare_kretprobe(struct kretprobe_instance
*ri
,
134 struct pt_regs
*regs
)
136 ri
->ret_addr
= (kprobe_opcode_t
*)regs
->link
;
138 /* Replace the return addr with trampoline addr */
139 regs
->link
= (unsigned long)kretprobe_trampoline
;
142 static int __kprobes
kprobe_handler(struct pt_regs
*regs
)
146 unsigned int *addr
= (unsigned int *)regs
->nip
;
147 struct kprobe_ctlblk
*kcb
;
150 * We don't want to be preempted for the entire
151 * duration of kprobe processing
154 kcb
= get_kprobe_ctlblk();
156 /* Check we're not actually recursing */
157 if (kprobe_running()) {
158 p
= get_kprobe(addr
);
160 kprobe_opcode_t insn
= *p
->ainsn
.insn
;
161 if (kcb
->kprobe_status
== KPROBE_HIT_SS
&&
163 /* Turn off 'trace' bits */
164 regs
->msr
&= ~MSR_SINGLESTEP
;
165 regs
->msr
|= kcb
->kprobe_saved_msr
;
168 /* We have reentered the kprobe_handler(), since
169 * another probe was hit while within the handler.
170 * We here save the original kprobes variables and
171 * just single step on the instruction of the new probe
172 * without calling any user handlers.
174 save_previous_kprobe(kcb
);
175 set_current_kprobe(p
, regs
, kcb
);
176 kcb
->kprobe_saved_msr
= regs
->msr
;
177 kprobes_inc_nmissed_count(p
);
178 prepare_singlestep(p
, regs
);
179 kcb
->kprobe_status
= KPROBE_REENTER
;
182 if (*addr
!= BREAKPOINT_INSTRUCTION
) {
183 /* If trap variant, then it belongs not to us */
184 kprobe_opcode_t cur_insn
= *addr
;
185 if (is_trap(cur_insn
))
187 /* The breakpoint instruction was removed by
188 * another cpu right after we hit, no further
189 * handling of this interrupt is appropriate
194 p
= __get_cpu_var(current_kprobe
);
195 if (p
->break_handler
&& p
->break_handler(p
, regs
)) {
202 p
= get_kprobe(addr
);
204 if (*addr
!= BREAKPOINT_INSTRUCTION
) {
206 * PowerPC has multiple variants of the "trap"
207 * instruction. If the current instruction is a
208 * trap variant, it could belong to someone else
210 kprobe_opcode_t cur_insn
= *addr
;
211 if (is_trap(cur_insn
))
214 * The breakpoint instruction was removed right
215 * after we hit it. Another cpu has removed
216 * either a probepoint or a debugger breakpoint
217 * at this address. In either case, no further
218 * handling of this interrupt is appropriate.
222 /* Not one of ours: let kernel handle it */
226 kcb
->kprobe_status
= KPROBE_HIT_ACTIVE
;
227 set_current_kprobe(p
, regs
, kcb
);
228 if (p
->pre_handler
&& p
->pre_handler(p
, regs
))
229 /* handler has already set things up, so skip ss setup */
233 if (p
->ainsn
.boostable
>= 0) {
234 unsigned int insn
= *p
->ainsn
.insn
;
236 /* regs->nip is also adjusted if emulate_step returns 1 */
237 ret
= emulate_step(regs
, insn
);
240 * Once this instruction has been boosted
241 * successfully, set the boostable flag
243 if (unlikely(p
->ainsn
.boostable
== 0))
244 p
->ainsn
.boostable
= 1;
247 p
->post_handler(p
, regs
, 0);
249 kcb
->kprobe_status
= KPROBE_HIT_SSDONE
;
250 reset_current_kprobe();
251 preempt_enable_no_resched();
253 } else if (ret
< 0) {
255 * We don't allow kprobes on mtmsr(d)/rfi(d), etc.
256 * So, we should never get here... but, its still
257 * good to catch them, just in case...
259 printk("Can't step on instruction %x\n", insn
);
262 /* This instruction can't be boosted */
263 p
->ainsn
.boostable
= -1;
265 prepare_singlestep(p
, regs
);
266 kcb
->kprobe_status
= KPROBE_HIT_SS
;
270 preempt_enable_no_resched();
275 * Function return probe trampoline:
276 * - init_kprobes() establishes a probepoint here
277 * - When the probed function returns, this probe
278 * causes the handlers to fire
280 static void __used
kretprobe_trampoline_holder(void)
282 asm volatile(".global kretprobe_trampoline\n"
283 "kretprobe_trampoline:\n"
288 * Called when the probe at kretprobe trampoline is hit
290 static int __kprobes
trampoline_probe_handler(struct kprobe
*p
,
291 struct pt_regs
*regs
)
293 struct kretprobe_instance
*ri
= NULL
;
294 struct hlist_head
*head
, empty_rp
;
295 struct hlist_node
*tmp
;
296 unsigned long flags
, orig_ret_address
= 0;
297 unsigned long trampoline_address
=(unsigned long)&kretprobe_trampoline
;
299 INIT_HLIST_HEAD(&empty_rp
);
300 kretprobe_hash_lock(current
, &head
, &flags
);
303 * It is possible to have multiple instances associated with a given
304 * task either because an multiple functions in the call path
305 * have a return probe installed on them, and/or more than one return
306 * return probe was registered for a target function.
308 * We can handle this because:
309 * - instances are always inserted at the head of the list
310 * - when multiple return probes are registered for the same
311 * function, the first instance's ret_addr will point to the
312 * real return address, and all the rest will point to
313 * kretprobe_trampoline
315 hlist_for_each_entry_safe(ri
, tmp
, head
, hlist
) {
316 if (ri
->task
!= current
)
317 /* another task is sharing our hash bucket */
320 if (ri
->rp
&& ri
->rp
->handler
)
321 ri
->rp
->handler(ri
, regs
);
323 orig_ret_address
= (unsigned long)ri
->ret_addr
;
324 recycle_rp_inst(ri
, &empty_rp
);
326 if (orig_ret_address
!= trampoline_address
)
328 * This is the real return address. Any other
329 * instances associated with this task are for
330 * other calls deeper on the call stack
335 kretprobe_assert(ri
, orig_ret_address
, trampoline_address
);
336 regs
->nip
= orig_ret_address
;
338 reset_current_kprobe();
339 kretprobe_hash_unlock(current
, &flags
);
340 preempt_enable_no_resched();
342 hlist_for_each_entry_safe(ri
, tmp
, &empty_rp
, hlist
) {
343 hlist_del(&ri
->hlist
);
347 * By returning a non-zero value, we are telling
348 * kprobe_handler() that we don't want the post_handler
349 * to run (and have re-enabled preemption)
355 * Called after single-stepping. p->addr is the address of the
356 * instruction whose first byte has been replaced by the "breakpoint"
357 * instruction. To avoid the SMP problems that can occur when we
358 * temporarily put back the original opcode to single-step, we
359 * single-stepped a copy of the instruction. The address of this
360 * copy is p->ainsn.insn.
362 static int __kprobes
post_kprobe_handler(struct pt_regs
*regs
)
364 struct kprobe
*cur
= kprobe_running();
365 struct kprobe_ctlblk
*kcb
= get_kprobe_ctlblk();
370 /* make sure we got here for instruction we have a kprobe on */
371 if (((unsigned long)cur
->ainsn
.insn
+ 4) != regs
->nip
)
374 if ((kcb
->kprobe_status
!= KPROBE_REENTER
) && cur
->post_handler
) {
375 kcb
->kprobe_status
= KPROBE_HIT_SSDONE
;
376 cur
->post_handler(cur
, regs
, 0);
379 /* Adjust nip to after the single-stepped instruction */
380 regs
->nip
= (unsigned long)cur
->addr
+ 4;
381 regs
->msr
|= kcb
->kprobe_saved_msr
;
383 /*Restore back the original saved kprobes variables and continue. */
384 if (kcb
->kprobe_status
== KPROBE_REENTER
) {
385 restore_previous_kprobe(kcb
);
388 reset_current_kprobe();
390 preempt_enable_no_resched();
393 * if somebody else is singlestepping across a probe point, msr
394 * will have DE/SE set, in which case, continue the remaining processing
395 * of do_debug, as if this is not a probe hit.
397 if (regs
->msr
& MSR_SINGLESTEP
)
403 int __kprobes
kprobe_fault_handler(struct pt_regs
*regs
, int trapnr
)
405 struct kprobe
*cur
= kprobe_running();
406 struct kprobe_ctlblk
*kcb
= get_kprobe_ctlblk();
407 const struct exception_table_entry
*entry
;
409 switch(kcb
->kprobe_status
) {
413 * We are here because the instruction being single
414 * stepped caused a page fault. We reset the current
415 * kprobe and the nip points back to the probe address
416 * and allow the page fault handler to continue as a
419 regs
->nip
= (unsigned long)cur
->addr
;
420 regs
->msr
&= ~MSR_SINGLESTEP
; /* Turn off 'trace' bits */
421 regs
->msr
|= kcb
->kprobe_saved_msr
;
422 if (kcb
->kprobe_status
== KPROBE_REENTER
)
423 restore_previous_kprobe(kcb
);
425 reset_current_kprobe();
426 preempt_enable_no_resched();
428 case KPROBE_HIT_ACTIVE
:
429 case KPROBE_HIT_SSDONE
:
431 * We increment the nmissed count for accounting,
432 * we can also use npre/npostfault count for accounting
433 * these specific fault cases.
435 kprobes_inc_nmissed_count(cur
);
438 * We come here because instructions in the pre/post
439 * handler caused the page_fault, this could happen
440 * if handler tries to access user space by
441 * copy_from_user(), get_user() etc. Let the
442 * user-specified handler try to fix it first.
444 if (cur
->fault_handler
&& cur
->fault_handler(cur
, regs
, trapnr
))
448 * In case the user-specified fault handler returned
449 * zero, try to fix up.
451 if ((entry
= search_exception_tables(regs
->nip
)) != NULL
) {
452 regs
->nip
= entry
->fixup
;
457 * fixup_exception() could not handle it,
458 * Let do_page_fault() fix it.
468 * Wrapper routine to for handling exceptions.
470 int __kprobes
kprobe_exceptions_notify(struct notifier_block
*self
,
471 unsigned long val
, void *data
)
473 struct die_args
*args
= (struct die_args
*)data
;
474 int ret
= NOTIFY_DONE
;
476 if (args
->regs
&& user_mode(args
->regs
))
481 if (kprobe_handler(args
->regs
))
485 if (post_kprobe_handler(args
->regs
))
495 unsigned long arch_deref_entry_point(void *entry
)
497 return ((func_descr_t
*)entry
)->entry
;
501 int __kprobes
setjmp_pre_handler(struct kprobe
*p
, struct pt_regs
*regs
)
503 struct jprobe
*jp
= container_of(p
, struct jprobe
, kp
);
504 struct kprobe_ctlblk
*kcb
= get_kprobe_ctlblk();
506 memcpy(&kcb
->jprobe_saved_regs
, regs
, sizeof(struct pt_regs
));
508 /* setup return addr to the jprobe handler routine */
509 regs
->nip
= arch_deref_entry_point(jp
->entry
);
511 regs
->gpr
[2] = (unsigned long)(((func_descr_t
*)jp
->entry
)->toc
);
517 void __used __kprobes
jprobe_return(void)
519 asm volatile("trap" ::: "memory");
522 static void __used __kprobes
jprobe_return_end(void)
526 int __kprobes
longjmp_break_handler(struct kprobe
*p
, struct pt_regs
*regs
)
528 struct kprobe_ctlblk
*kcb
= get_kprobe_ctlblk();
531 * FIXME - we should ideally be validating that we got here 'cos
532 * of the "trap" in jprobe_return() above, before restoring the
535 memcpy(regs
, &kcb
->jprobe_saved_regs
, sizeof(struct pt_regs
));
536 preempt_enable_no_resched();
540 static struct kprobe trampoline_p
= {
541 .addr
= (kprobe_opcode_t
*) &kretprobe_trampoline
,
542 .pre_handler
= trampoline_probe_handler
545 int __init
arch_init_kprobes(void)
547 return register_kprobe(&trampoline_p
);
550 int __kprobes
arch_trampoline_kprobe(struct kprobe
*p
)
552 if (p
->addr
== (kprobe_opcode_t
*)&kretprobe_trampoline
)