| blob | 9916d150b28c151bac0920aeb822545a41c40738 |
1 /*
2 * PowerPC version
3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 * Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
5 * Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
6 * Adapted for Power Macintosh by Paul Mackerras.
7 * Low-level exception handlers and MMU support
8 * rewritten by Paul Mackerras.
9 * Copyright (C) 1996 Paul Mackerras.
10 * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
11 *
12 * This file contains the system call entry code, context switch
13 * code, and exception/interrupt return code for PowerPC.
14 *
15 * This program is free software; you can redistribute it and/or
16 * modify it under the terms of the GNU General Public License
17 * as published by the Free Software Foundation; either version
18 * 2 of the License, or (at your option) any later version.
19 */
21 #include <linux/errno.h>
22 #include <linux/err.h>
23 #include <asm/unistd.h>
24 #include <asm/processor.h>
25 #include <asm/page.h>
26 #include <asm/mmu.h>
27 #include <asm/thread_info.h>
28 #include <asm/ppc_asm.h>
29 #include <asm/asm-offsets.h>
30 #include <asm/cputable.h>
31 #include <asm/firmware.h>
32 #include <asm/bug.h>
33 #include <asm/ptrace.h>
34 #include <asm/irqflags.h>
35 #include <asm/ftrace.h>
36 #include <asm/hw_irq.h>
37 #include <asm/context_tracking.h>
38 #include <asm/tm.h>
40 /*
41 * System calls.
42 */
43 .section ".toc","aw"
44 SYS_CALL_TABLE:
45 .tc sys_call_table[TC],sys_call_table
47 /* This value is used to mark exception frames on the stack. */
48 exception_marker:
49 .tc ID_EXC_MARKER[TC],STACK_FRAME_REGS_MARKER
51 .section ".text"
52 .align 7
54 .globl system_call_common
55 system_call_common:
56 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
57 BEGIN_FTR_SECTION
58 extrdi. r10, r12, 1, (63-MSR_TS_T_LG) /* transaction active? */
59 bne tabort_syscall
60 END_FTR_SECTION_IFSET(CPU_FTR_TM)
61 #endif
62 andi. r10,r12,MSR_PR
63 mr r10,r1
64 addi r1,r1,-INT_FRAME_SIZE
65 beq- 1f
66 ld r1,PACAKSAVE(r13)
67 1: std r10,0(r1)
68 std r11,_NIP(r1)
69 std r12,_MSR(r1)
70 std r0,GPR0(r1)
71 std r10,GPR1(r1)
72 beq 2f /* if from kernel mode */
73 ACCOUNT_CPU_USER_ENTRY(r10, r11)
74 2: std r2,GPR2(r1)
75 std r3,GPR3(r1)
76 mfcr r2
77 std r4,GPR4(r1)
78 std r5,GPR5(r1)
79 std r6,GPR6(r1)
80 std r7,GPR7(r1)
81 std r8,GPR8(r1)
82 li r11,0
83 std r11,GPR9(r1)
84 std r11,GPR10(r1)
85 std r11,GPR11(r1)
86 std r11,GPR12(r1)
87 std r11,_XER(r1)
88 std r11,_CTR(r1)
89 std r9,GPR13(r1)
90 mflr r10
91 /*
92 * This clears CR0.SO (bit 28), which is the error indication on
93 * return from this system call.
94 */
95 rldimi r2,r11,28,(63-28)
96 li r11,0xc01
97 std r10,_LINK(r1)
98 std r11,_TRAP(r1)
99 std r3,ORIG_GPR3(r1)
100 std r2,_CCR(r1)
101 ld r2,PACATOC(r13)
102 addi r9,r1,STACK_FRAME_OVERHEAD
103 ld r11,exception_marker@toc(r2)
104 std r11,-16(r9) /* "regshere" marker */
105 #if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) && defined(CONFIG_PPC_SPLPAR)
106 BEGIN_FW_FTR_SECTION
107 beq 33f
108 /* if from user, see if there are any DTL entries to process */
109 ld r10,PACALPPACAPTR(r13) /* get ptr to VPA */
110 ld r11,PACA_DTL_RIDX(r13) /* get log read index */
111 addi r10,r10,LPPACA_DTLIDX
112 LDX_BE r10,0,r10 /* get log write index */
113 cmpd cr1,r11,r10
114 beq+ cr1,33f
115 bl accumulate_stolen_time
116 REST_GPR(0,r1)
117 REST_4GPRS(3,r1)
118 REST_2GPRS(7,r1)
119 addi r9,r1,STACK_FRAME_OVERHEAD
120 33:
121 END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR)
122 #endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE && CONFIG_PPC_SPLPAR */
124 /*
125 * A syscall should always be called with interrupts enabled
126 * so we just unconditionally hard-enable here. When some kind
127 * of irq tracing is used, we additionally check that condition
128 * is correct
129 */
130 #if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_BUG)
131 lbz r10,PACASOFTIRQEN(r13)
132 xori r10,r10,1
133 1: tdnei r10,0
134 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
135 #endif
137 #ifdef CONFIG_PPC_BOOK3E
138 wrteei 1
139 #else
140 ld r11,PACAKMSR(r13)
141 ori r11,r11,MSR_EE
142 mtmsrd r11,1
143 #endif /* CONFIG_PPC_BOOK3E */
145 /* We do need to set SOFTE in the stack frame or the return
146 * from interrupt will be painful
147 */
148 li r10,1
149 std r10,SOFTE(r1)
151 CURRENT_THREAD_INFO(r11, r1)
152 ld r10,TI_FLAGS(r11)
153 andi. r11,r10,_TIF_SYSCALL_DOTRACE
154 bne syscall_dotrace /* does not return */
155 cmpldi 0,r0,NR_syscalls
156 bge- syscall_enosys
158 system_call: /* label this so stack traces look sane */
159 /*
160 * Need to vector to 32 Bit or default sys_call_table here,
161 * based on caller's run-mode / personality.
162 */
163 ld r11,SYS_CALL_TABLE@toc(2)
164 andi. r10,r10,_TIF_32BIT
165 beq 15f
166 addi r11,r11,8 /* use 32-bit syscall entries */
167 clrldi r3,r3,32
168 clrldi r4,r4,32
169 clrldi r5,r5,32
170 clrldi r6,r6,32
171 clrldi r7,r7,32
172 clrldi r8,r8,32
173 15:
174 slwi r0,r0,4
175 ldx r12,r11,r0 /* Fetch system call handler [ptr] */
176 mtctr r12
177 bctrl /* Call handler */
179 .Lsyscall_exit:
180 std r3,RESULT(r1)
181 CURRENT_THREAD_INFO(r12, r1)
183 ld r8,_MSR(r1)
184 #ifdef CONFIG_PPC_BOOK3S
185 /* No MSR:RI on BookE */
186 andi. r10,r8,MSR_RI
187 beq- unrecov_restore
188 #endif
189 /*
190 * Disable interrupts so current_thread_info()->flags can't change,
191 * and so that we don't get interrupted after loading SRR0/1.
192 */
193 #ifdef CONFIG_PPC_BOOK3E
194 wrteei 0
195 #else
196 ld r10,PACAKMSR(r13)
197 /*
198 * For performance reasons we clear RI the same time that we
199 * clear EE. We only need to clear RI just before we restore r13
200 * below, but batching it with EE saves us one expensive mtmsrd call.
201 * We have to be careful to restore RI if we branch anywhere from
202 * here (eg syscall_exit_work).
203 */
204 li r9,MSR_RI
205 andc r11,r10,r9
206 mtmsrd r11,1
207 #endif /* CONFIG_PPC_BOOK3E */
209 ld r9,TI_FLAGS(r12)
210 li r11,-MAX_ERRNO
211 andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
212 bne- syscall_exit_work
214 andi. r0,r8,MSR_FP
215 beq 2f
216 #ifdef CONFIG_ALTIVEC
217 andis. r0,r8,MSR_VEC@h
218 bne 3f
219 #endif
220 2: addi r3,r1,STACK_FRAME_OVERHEAD
221 #ifdef CONFIG_PPC_BOOK3S
222 mtmsrd r10,1 /* Restore RI */
223 #endif
224 bl restore_math
225 #ifdef CONFIG_PPC_BOOK3S
226 ld r10,PACAKMSR(r13)
227 li r9,MSR_RI
228 andc r11,r10,r9 /* Re-clear RI */
229 mtmsrd r11,1
230 #endif
231 ld r8,_MSR(r1)
232 ld r3,RESULT(r1)
233 li r11,-MAX_ERRNO
235 3: cmpld r3,r11
236 ld r5,_CCR(r1)
237 bge- syscall_error
238 .Lsyscall_error_cont:
239 ld r7,_NIP(r1)
240 BEGIN_FTR_SECTION
241 stdcx. r0,0,r1 /* to clear the reservation */
242 END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
243 andi. r6,r8,MSR_PR
244 ld r4,_LINK(r1)
246 beq- 1f
247 ACCOUNT_CPU_USER_EXIT(r11, r12)
249 BEGIN_FTR_SECTION
250 HMT_MEDIUM_LOW
251 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
253 ld r13,GPR13(r1) /* only restore r13 if returning to usermode */
254 1: ld r2,GPR2(r1)
255 ld r1,GPR1(r1)
256 mtlr r4
257 mtcr r5
258 mtspr SPRN_SRR0,r7
259 mtspr SPRN_SRR1,r8
260 RFI
261 b . /* prevent speculative execution */
263 syscall_error:
264 oris r5,r5,0x1000 /* Set SO bit in CR */
265 neg r3,r3
266 std r5,_CCR(r1)
267 b .Lsyscall_error_cont
269 /* Traced system call support */
270 syscall_dotrace:
271 bl save_nvgprs
272 addi r3,r1,STACK_FRAME_OVERHEAD
273 bl do_syscall_trace_enter
275 /*
276 * We use the return value of do_syscall_trace_enter() as the syscall
277 * number. If the syscall was rejected for any reason do_syscall_trace_enter()
278 * returns an invalid syscall number and the test below against
279 * NR_syscalls will fail.
280 */
281 mr r0,r3
283 /* Restore argument registers just clobbered and/or possibly changed. */
284 ld r3,GPR3(r1)
285 ld r4,GPR4(r1)
286 ld r5,GPR5(r1)
287 ld r6,GPR6(r1)
288 ld r7,GPR7(r1)
289 ld r8,GPR8(r1)
291 /* Repopulate r9 and r10 for the system_call path */
292 addi r9,r1,STACK_FRAME_OVERHEAD
293 CURRENT_THREAD_INFO(r10, r1)
294 ld r10,TI_FLAGS(r10)
296 cmpldi r0,NR_syscalls
297 blt+ system_call
299 /* Return code is already in r3 thanks to do_syscall_trace_enter() */
300 b .Lsyscall_exit
303 syscall_enosys:
304 li r3,-ENOSYS
305 b .Lsyscall_exit
307 syscall_exit_work:
308 #ifdef CONFIG_PPC_BOOK3S
309 mtmsrd r10,1 /* Restore RI */
310 #endif
311 /* If TIF_RESTOREALL is set, don't scribble on either r3 or ccr.
312 If TIF_NOERROR is set, just save r3 as it is. */
314 andi. r0,r9,_TIF_RESTOREALL
315 beq+ 0f
316 REST_NVGPRS(r1)
317 b 2f
318 0: cmpld r3,r11 /* r11 is -MAX_ERRNO */
319 blt+ 1f
320 andi. r0,r9,_TIF_NOERROR
321 bne- 1f
322 ld r5,_CCR(r1)
323 neg r3,r3
324 oris r5,r5,0x1000 /* Set SO bit in CR */
325 std r5,_CCR(r1)
326 1: std r3,GPR3(r1)
327 2: andi. r0,r9,(_TIF_PERSYSCALL_MASK)
328 beq 4f
330 /* Clear per-syscall TIF flags if any are set. */
332 li r11,_TIF_PERSYSCALL_MASK
333 addi r12,r12,TI_FLAGS
334 3: ldarx r10,0,r12
335 andc r10,r10,r11
336 stdcx. r10,0,r12
337 bne- 3b
338 subi r12,r12,TI_FLAGS
340 4: /* Anything else left to do? */
341 BEGIN_FTR_SECTION
342 lis r3,INIT_PPR@highest /* Set thread.ppr = 3 */
343 ld r10,PACACURRENT(r13)
344 sldi r3,r3,32 /* bits 11-13 are used for ppr */
345 std r3,TASKTHREADPPR(r10)
346 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
348 andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP)
349 beq ret_from_except_lite
351 /* Re-enable interrupts */
352 #ifdef CONFIG_PPC_BOOK3E
353 wrteei 1
354 #else
355 ld r10,PACAKMSR(r13)
356 ori r10,r10,MSR_EE
357 mtmsrd r10,1
358 #endif /* CONFIG_PPC_BOOK3E */
360 bl save_nvgprs
361 addi r3,r1,STACK_FRAME_OVERHEAD
362 bl do_syscall_trace_leave
363 b ret_from_except
365 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
366 tabort_syscall:
367 /* Firstly we need to enable TM in the kernel */
368 mfmsr r10
369 li r13, 1
370 rldimi r10, r13, MSR_TM_LG, 63-MSR_TM_LG
371 mtmsrd r10, 0
373 /* tabort, this dooms the transaction, nothing else */
374 li r13, (TM_CAUSE_SYSCALL|TM_CAUSE_PERSISTENT)
375 TABORT(R13)
377 /*
378 * Return directly to userspace. We have corrupted user register state,
379 * but userspace will never see that register state. Execution will
380 * resume after the tbegin of the aborted transaction with the
381 * checkpointed register state.
382 */
383 li r13, MSR_RI
384 andc r10, r10, r13
385 mtmsrd r10, 1
386 mtspr SPRN_SRR0, r11
387 mtspr SPRN_SRR1, r12
389 rfid
390 b . /* prevent speculative execution */
391 #endif
393 /* Save non-volatile GPRs, if not already saved. */
394 _GLOBAL(save_nvgprs)
395 ld r11,_TRAP(r1)
396 andi. r0,r11,1
397 beqlr-
398 SAVE_NVGPRS(r1)
399 clrrdi r0,r11,1
400 std r0,_TRAP(r1)
401 blr
404 /*
405 * The sigsuspend and rt_sigsuspend system calls can call do_signal
406 * and thus put the process into the stopped state where we might
407 * want to examine its user state with ptrace. Therefore we need
408 * to save all the nonvolatile registers (r14 - r31) before calling
409 * the C code. Similarly, fork, vfork and clone need the full
410 * register state on the stack so that it can be copied to the child.
411 */
413 _GLOBAL(ppc_fork)
414 bl save_nvgprs
415 bl sys_fork
416 b .Lsyscall_exit
418 _GLOBAL(ppc_vfork)
419 bl save_nvgprs
420 bl sys_vfork
421 b .Lsyscall_exit
423 _GLOBAL(ppc_clone)
424 bl save_nvgprs
425 bl sys_clone
426 b .Lsyscall_exit
428 _GLOBAL(ppc32_swapcontext)
429 bl save_nvgprs
430 bl compat_sys_swapcontext
431 b .Lsyscall_exit
433 _GLOBAL(ppc64_swapcontext)
434 bl save_nvgprs
435 bl sys_swapcontext
436 b .Lsyscall_exit
438 _GLOBAL(ppc_switch_endian)
439 bl save_nvgprs
440 bl sys_switch_endian
441 b .Lsyscall_exit
443 _GLOBAL(ret_from_fork)
444 bl schedule_tail
445 REST_NVGPRS(r1)
446 li r3,0
447 b .Lsyscall_exit
449 _GLOBAL(ret_from_kernel_thread)
450 bl schedule_tail
451 REST_NVGPRS(r1)
452 mtlr r14
453 mr r3,r15
454 #if defined(_CALL_ELF) && _CALL_ELF == 2
455 mr r12,r14
456 #endif
457 blrl
458 li r3,0
459 b .Lsyscall_exit
461 /*
462 * This routine switches between two different tasks. The process
463 * state of one is saved on its kernel stack. Then the state
464 * of the other is restored from its kernel stack. The memory
465 * management hardware is updated to the second process's state.
466 * Finally, we can return to the second process, via ret_from_except.
467 * On entry, r3 points to the THREAD for the current task, r4
468 * points to the THREAD for the new task.
469 *
470 * Note: there are two ways to get to the "going out" portion
471 * of this code; either by coming in via the entry (_switch)
472 * or via "fork" which must set up an environment equivalent
473 * to the "_switch" path. If you change this you'll have to change
474 * the fork code also.
475 *
476 * The code which creates the new task context is in 'copy_thread'
477 * in arch/powerpc/kernel/process.c
478 */
479 .align 7
480 _GLOBAL(_switch)
481 mflr r0
482 std r0,16(r1)
483 stdu r1,-SWITCH_FRAME_SIZE(r1)
484 /* r3-r13 are caller saved -- Cort */
485 SAVE_8GPRS(14, r1)
486 SAVE_10GPRS(22, r1)
487 std r0,_NIP(r1) /* Return to switch caller */
488 mfcr r23
489 std r23,_CCR(r1)
490 std r1,KSP(r3) /* Set old stack pointer */
492 #ifdef CONFIG_SMP
493 /* We need a sync somewhere here to make sure that if the
494 * previous task gets rescheduled on another CPU, it sees all
495 * stores it has performed on this one.
496 */
497 sync
498 #endif /* CONFIG_SMP */
500 /*
501 * If we optimise away the clear of the reservation in system
502 * calls because we know the CPU tracks the address of the
503 * reservation, then we need to clear it here to cover the
504 * case that the kernel context switch path has no larx
505 * instructions.
506 */
507 BEGIN_FTR_SECTION
508 ldarx r6,0,r1
509 END_FTR_SECTION_IFSET(CPU_FTR_STCX_CHECKS_ADDRESS)
511 #ifdef CONFIG_PPC_BOOK3S
512 /* Cancel all explict user streams as they will have no use after context
513 * switch and will stop the HW from creating streams itself
514 */
515 DCBT_STOP_ALL_STREAM_IDS(r6)
516 #endif
518 addi r6,r4,-THREAD /* Convert THREAD to 'current' */
519 std r6,PACACURRENT(r13) /* Set new 'current' */
521 ld r8,KSP(r4) /* new stack pointer */
522 #ifdef CONFIG_PPC_BOOK3S
523 BEGIN_FTR_SECTION
524 clrrdi r6,r8,28 /* get its ESID */
525 clrrdi r9,r1,28 /* get current sp ESID */
526 FTR_SECTION_ELSE
527 clrrdi r6,r8,40 /* get its 1T ESID */
528 clrrdi r9,r1,40 /* get current sp 1T ESID */
529 ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_1T_SEGMENT)
530 clrldi. r0,r6,2 /* is new ESID c00000000? */
531 cmpd cr1,r6,r9 /* or is new ESID the same as current ESID? */
532 cror eq,4*cr1+eq,eq
533 beq 2f /* if yes, don't slbie it */
535 /* Bolt in the new stack SLB entry */
536 ld r7,KSP_VSID(r4) /* Get new stack's VSID */
537 oris r0,r6,(SLB_ESID_V)@h
538 ori r0,r0,(SLB_NUM_BOLTED-1)@l
539 BEGIN_FTR_SECTION
540 li r9,MMU_SEGSIZE_1T /* insert B field */
541 oris r6,r6,(MMU_SEGSIZE_1T << SLBIE_SSIZE_SHIFT)@h
542 rldimi r7,r9,SLB_VSID_SSIZE_SHIFT,0
543 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
545 /* Update the last bolted SLB. No write barriers are needed
546 * here, provided we only update the current CPU's SLB shadow
547 * buffer.
548 */
549 ld r9,PACA_SLBSHADOWPTR(r13)
550 li r12,0
551 std r12,SLBSHADOW_STACKESID(r9) /* Clear ESID */
552 li r12,SLBSHADOW_STACKVSID
553 STDX_BE r7,r12,r9 /* Save VSID */
554 li r12,SLBSHADOW_STACKESID
555 STDX_BE r0,r12,r9 /* Save ESID */
557 /* No need to check for MMU_FTR_NO_SLBIE_B here, since when
558 * we have 1TB segments, the only CPUs known to have the errata
559 * only support less than 1TB of system memory and we'll never
560 * actually hit this code path.
561 */
563 slbie r6
564 slbie r6 /* Workaround POWER5 < DD2.1 issue */
565 slbmte r7,r0
566 isync
567 2:
568 #endif /* !CONFIG_PPC_BOOK3S */
570 CURRENT_THREAD_INFO(r7, r8) /* base of new stack */
571 /* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE
572 because we don't need to leave the 288-byte ABI gap at the
573 top of the kernel stack. */
574 addi r7,r7,THREAD_SIZE-SWITCH_FRAME_SIZE
576 mr r1,r8 /* start using new stack pointer */
577 std r7,PACAKSAVE(r13)
579 ld r6,_CCR(r1)
580 mtcrf 0xFF,r6
582 /* r3-r13 are destroyed -- Cort */
583 REST_8GPRS(14, r1)
584 REST_10GPRS(22, r1)
586 /* convert old thread to its task_struct for return value */
587 addi r3,r3,-THREAD
588 ld r7,_NIP(r1) /* Return to _switch caller in new task */
589 mtlr r7
590 addi r1,r1,SWITCH_FRAME_SIZE
591 blr
593 .align 7
594 _GLOBAL(ret_from_except)
595 ld r11,_TRAP(r1)
596 andi. r0,r11,1
597 bne ret_from_except_lite
598 REST_NVGPRS(r1)
600 _GLOBAL(ret_from_except_lite)
601 /*
602 * Disable interrupts so that current_thread_info()->flags
603 * can't change between when we test it and when we return
604 * from the interrupt.
605 */
606 #ifdef CONFIG_PPC_BOOK3E
607 wrteei 0
608 #else
609 ld r10,PACAKMSR(r13) /* Get kernel MSR without EE */
610 mtmsrd r10,1 /* Update machine state */
611 #endif /* CONFIG_PPC_BOOK3E */
613 CURRENT_THREAD_INFO(r9, r1)
614 ld r3,_MSR(r1)
615 #ifdef CONFIG_PPC_BOOK3E
616 ld r10,PACACURRENT(r13)
617 #endif /* CONFIG_PPC_BOOK3E */
618 ld r4,TI_FLAGS(r9)
619 andi. r3,r3,MSR_PR
620 beq resume_kernel
621 #ifdef CONFIG_PPC_BOOK3E
622 lwz r3,(THREAD+THREAD_DBCR0)(r10)
623 #endif /* CONFIG_PPC_BOOK3E */
625 /* Check current_thread_info()->flags */
626 andi. r0,r4,_TIF_USER_WORK_MASK
627 bne 1f
628 #ifdef CONFIG_PPC_BOOK3E
629 /*
630 * Check to see if the dbcr0 register is set up to debug.
631 * Use the internal debug mode bit to do this.
632 */
633 andis. r0,r3,DBCR0_IDM@h
634 beq restore
635 mfmsr r0
636 rlwinm r0,r0,0,~MSR_DE /* Clear MSR.DE */
637 mtmsr r0
638 mtspr SPRN_DBCR0,r3
639 li r10, -1
640 mtspr SPRN_DBSR,r10
641 b restore
642 #else
643 addi r3,r1,STACK_FRAME_OVERHEAD
644 bl restore_math
645 b restore
646 #endif
647 1: andi. r0,r4,_TIF_NEED_RESCHED
648 beq 2f
649 bl restore_interrupts
650 SCHEDULE_USER
651 b ret_from_except_lite
652 2:
653 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
654 andi. r0,r4,_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM
655 bne 3f /* only restore TM if nothing else to do */
656 addi r3,r1,STACK_FRAME_OVERHEAD
657 bl restore_tm_state
658 b restore
659 3:
660 #endif
661 bl save_nvgprs
662 /*
663 * Use a non volatile GPR to save and restore our thread_info flags
664 * across the call to restore_interrupts.
665 */
666 mr r30,r4
667 bl restore_interrupts
668 mr r4,r30
669 addi r3,r1,STACK_FRAME_OVERHEAD
670 bl do_notify_resume
671 b ret_from_except
673 resume_kernel:
674 /* check current_thread_info, _TIF_EMULATE_STACK_STORE */
675 andis. r8,r4,_TIF_EMULATE_STACK_STORE@h
676 beq+ 1f
678 addi r8,r1,INT_FRAME_SIZE /* Get the kprobed function entry */
680 lwz r3,GPR1(r1)
681 subi r3,r3,INT_FRAME_SIZE /* dst: Allocate a trampoline exception frame */
682 mr r4,r1 /* src: current exception frame */
683 mr r1,r3 /* Reroute the trampoline frame to r1 */
685 /* Copy from the original to the trampoline. */
686 li r5,INT_FRAME_SIZE/8 /* size: INT_FRAME_SIZE */
687 li r6,0 /* start offset: 0 */
688 mtctr r5
689 2: ldx r0,r6,r4
690 stdx r0,r6,r3
691 addi r6,r6,8
692 bdnz 2b
694 /* Do real store operation to complete stwu */
695 lwz r5,GPR1(r1)
696 std r8,0(r5)
698 /* Clear _TIF_EMULATE_STACK_STORE flag */
699 lis r11,_TIF_EMULATE_STACK_STORE@h
700 addi r5,r9,TI_FLAGS
701 0: ldarx r4,0,r5
702 andc r4,r4,r11
703 stdcx. r4,0,r5
704 bne- 0b
705 1:
707 #ifdef CONFIG_PREEMPT
708 /* Check if we need to preempt */
709 andi. r0,r4,_TIF_NEED_RESCHED
710 beq+ restore
711 /* Check that preempt_count() == 0 and interrupts are enabled */
712 lwz r8,TI_PREEMPT(r9)
713 cmpwi cr1,r8,0
714 ld r0,SOFTE(r1)
715 cmpdi r0,0
716 crandc eq,cr1*4+eq,eq
717 bne restore
719 /*
720 * Here we are preempting the current task. We want to make
721 * sure we are soft-disabled first and reconcile irq state.
722 */
723 RECONCILE_IRQ_STATE(r3,r4)
724 1: bl preempt_schedule_irq
726 /* Re-test flags and eventually loop */
727 CURRENT_THREAD_INFO(r9, r1)
728 ld r4,TI_FLAGS(r9)
729 andi. r0,r4,_TIF_NEED_RESCHED
730 bne 1b
732 /*
733 * arch_local_irq_restore() from preempt_schedule_irq above may
734 * enable hard interrupt but we really should disable interrupts
735 * when we return from the interrupt, and so that we don't get
736 * interrupted after loading SRR0/1.
737 */
738 #ifdef CONFIG_PPC_BOOK3E
739 wrteei 0
740 #else
741 ld r10,PACAKMSR(r13) /* Get kernel MSR without EE */
742 mtmsrd r10,1 /* Update machine state */
743 #endif /* CONFIG_PPC_BOOK3E */
744 #endif /* CONFIG_PREEMPT */
746 .globl fast_exc_return_irq
747 fast_exc_return_irq:
748 restore:
749 /*
750 * This is the main kernel exit path. First we check if we
751 * are about to re-enable interrupts
752 */
753 ld r5,SOFTE(r1)
754 lbz r6,PACASOFTIRQEN(r13)
755 cmpwi cr0,r5,0
756 beq restore_irq_off
758 /* We are enabling, were we already enabled ? Yes, just return */
759 cmpwi cr0,r6,1
760 beq cr0,do_restore
762 /*
763 * We are about to soft-enable interrupts (we are hard disabled
764 * at this point). We check if there's anything that needs to
765 * be replayed first.
766 */
767 lbz r0,PACAIRQHAPPENED(r13)
768 cmpwi cr0,r0,0
769 bne- restore_check_irq_replay
771 /*
772 * Get here when nothing happened while soft-disabled, just
773 * soft-enable and move-on. We will hard-enable as a side
774 * effect of rfi
775 */
776 restore_no_replay:
777 TRACE_ENABLE_INTS
778 li r0,1
779 stb r0,PACASOFTIRQEN(r13);
781 /*
782 * Final return path. BookE is handled in a different file
783 */
784 do_restore:
785 #ifdef CONFIG_PPC_BOOK3E
786 b exception_return_book3e
787 #else
788 /*
789 * Clear the reservation. If we know the CPU tracks the address of
790 * the reservation then we can potentially save some cycles and use
791 * a larx. On POWER6 and POWER7 this is significantly faster.
792 */
793 BEGIN_FTR_SECTION
794 stdcx. r0,0,r1 /* to clear the reservation */
795 FTR_SECTION_ELSE
796 ldarx r4,0,r1
797 ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
799 /*
800 * Some code path such as load_up_fpu or altivec return directly
801 * here. They run entirely hard disabled and do not alter the
802 * interrupt state. They also don't use lwarx/stwcx. and thus
803 * are known not to leave dangling reservations.
804 */
805 .globl fast_exception_return
806 fast_exception_return:
807 ld r3,_MSR(r1)
808 ld r4,_CTR(r1)
809 ld r0,_LINK(r1)
810 mtctr r4
811 mtlr r0
812 ld r4,_XER(r1)
813 mtspr SPRN_XER,r4
815 REST_8GPRS(5, r1)
817 andi. r0,r3,MSR_RI
818 beq- unrecov_restore
820 /* Load PPR from thread struct before we clear MSR:RI */
821 BEGIN_FTR_SECTION
822 ld r2,PACACURRENT(r13)
823 ld r2,TASKTHREADPPR(r2)
824 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
826 /*
827 * Clear RI before restoring r13. If we are returning to
828 * userspace and we take an exception after restoring r13,
829 * we end up corrupting the userspace r13 value.
830 */
831 ld r4,PACAKMSR(r13) /* Get kernel MSR without EE */
832 andc r4,r4,r0 /* r0 contains MSR_RI here */
833 mtmsrd r4,1
835 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
836 /* TM debug */
837 std r3, PACATMSCRATCH(r13) /* Stash returned-to MSR */
838 #endif
839 /*
840 * r13 is our per cpu area, only restore it if we are returning to
841 * userspace the value stored in the stack frame may belong to
842 * another CPU.
843 */
844 andi. r0,r3,MSR_PR
845 beq 1f
846 BEGIN_FTR_SECTION
847 mtspr SPRN_PPR,r2 /* Restore PPR */
848 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
849 ACCOUNT_CPU_USER_EXIT(r2, r4)
850 REST_GPR(13, r1)
851 1:
852 mtspr SPRN_SRR1,r3
854 ld r2,_CCR(r1)
855 mtcrf 0xFF,r2
856 ld r2,_NIP(r1)
857 mtspr SPRN_SRR0,r2
859 ld r0,GPR0(r1)
860 ld r2,GPR2(r1)
861 ld r3,GPR3(r1)
862 ld r4,GPR4(r1)
863 ld r1,GPR1(r1)
865 rfid
866 b . /* prevent speculative execution */
868 #endif /* CONFIG_PPC_BOOK3E */
870 /*
871 * We are returning to a context with interrupts soft disabled.
872 *
873 * However, we may also about to hard enable, so we need to
874 * make sure that in this case, we also clear PACA_IRQ_HARD_DIS
875 * or that bit can get out of sync and bad things will happen
876 */
877 restore_irq_off:
878 ld r3,_MSR(r1)
879 lbz r7,PACAIRQHAPPENED(r13)
880 andi. r0,r3,MSR_EE
881 beq 1f
882 rlwinm r7,r7,0,~PACA_IRQ_HARD_DIS
883 stb r7,PACAIRQHAPPENED(r13)
884 1: li r0,0
885 stb r0,PACASOFTIRQEN(r13);
886 TRACE_DISABLE_INTS
887 b do_restore
889 /*
890 * Something did happen, check if a re-emit is needed
891 * (this also clears paca->irq_happened)
892 */
893 restore_check_irq_replay:
894 /* XXX: We could implement a fast path here where we check
895 * for irq_happened being just 0x01, in which case we can
896 * clear it and return. That means that we would potentially
897 * miss a decrementer having wrapped all the way around.
898 *
899 * Still, this might be useful for things like hash_page
900 */
901 bl __check_irq_replay
902 cmpwi cr0,r3,0
903 beq restore_no_replay
905 /*
906 * We need to re-emit an interrupt. We do so by re-using our
907 * existing exception frame. We first change the trap value,
908 * but we need to ensure we preserve the low nibble of it
909 */
910 ld r4,_TRAP(r1)
911 clrldi r4,r4,60
912 or r4,r4,r3
913 std r4,_TRAP(r1)
915 /*
916 * Then find the right handler and call it. Interrupts are
917 * still soft-disabled and we keep them that way.
918 */
919 cmpwi cr0,r3,0x500
920 bne 1f
921 addi r3,r1,STACK_FRAME_OVERHEAD;
922 bl do_IRQ
923 b ret_from_except
924 1: cmpwi cr0,r3,0xe60
925 bne 1f
926 addi r3,r1,STACK_FRAME_OVERHEAD;
927 bl handle_hmi_exception
928 b ret_from_except
929 1: cmpwi cr0,r3,0x900
930 bne 1f
931 addi r3,r1,STACK_FRAME_OVERHEAD;
932 bl timer_interrupt
933 b ret_from_except
934 #ifdef CONFIG_PPC_DOORBELL
935 1:
936 #ifdef CONFIG_PPC_BOOK3E
937 cmpwi cr0,r3,0x280
938 #else
939 BEGIN_FTR_SECTION
940 cmpwi cr0,r3,0xe80
941 FTR_SECTION_ELSE
942 cmpwi cr0,r3,0xa00
943 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
944 #endif /* CONFIG_PPC_BOOK3E */
945 bne 1f
946 addi r3,r1,STACK_FRAME_OVERHEAD;
947 bl doorbell_exception
948 b ret_from_except
949 #endif /* CONFIG_PPC_DOORBELL */
950 1: b ret_from_except /* What else to do here ? */
952 unrecov_restore:
953 addi r3,r1,STACK_FRAME_OVERHEAD
954 bl unrecoverable_exception
955 b unrecov_restore
957 #ifdef CONFIG_PPC_RTAS
958 /*
959 * On CHRP, the Run-Time Abstraction Services (RTAS) have to be
960 * called with the MMU off.
961 *
962 * In addition, we need to be in 32b mode, at least for now.
963 *
964 * Note: r3 is an input parameter to rtas, so don't trash it...
965 */
966 _GLOBAL(enter_rtas)
967 mflr r0
968 std r0,16(r1)
969 stdu r1,-RTAS_FRAME_SIZE(r1) /* Save SP and create stack space. */
971 /* Because RTAS is running in 32b mode, it clobbers the high order half
972 * of all registers that it saves. We therefore save those registers
973 * RTAS might touch to the stack. (r0, r3-r13 are caller saved)
974 */
975 SAVE_GPR(2, r1) /* Save the TOC */
976 SAVE_GPR(13, r1) /* Save paca */
977 SAVE_8GPRS(14, r1) /* Save the non-volatiles */
978 SAVE_10GPRS(22, r1) /* ditto */
980 mfcr r4
981 std r4,_CCR(r1)
982 mfctr r5
983 std r5,_CTR(r1)
984 mfspr r6,SPRN_XER
985 std r6,_XER(r1)
986 mfdar r7
987 std r7,_DAR(r1)
988 mfdsisr r8
989 std r8,_DSISR(r1)
991 /* Temporary workaround to clear CR until RTAS can be modified to
992 * ignore all bits.
993 */
994 li r0,0
995 mtcr r0
997 #ifdef CONFIG_BUG
998 /* There is no way it is acceptable to get here with interrupts enabled,
999 * check it with the asm equivalent of WARN_ON
1000 */
1001 lbz r0,PACASOFTIRQEN(r13)
1002 1: tdnei r0,0
1003 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
1004 #endif
1006 /* Hard-disable interrupts */
1007 mfmsr r6
1008 rldicl r7,r6,48,1
1009 rotldi r7,r7,16
1010 mtmsrd r7,1
1012 /* Unfortunately, the stack pointer and the MSR are also clobbered,
1013 * so they are saved in the PACA which allows us to restore
1014 * our original state after RTAS returns.
1015 */
1016 std r1,PACAR1(r13)
1017 std r6,PACASAVEDMSR(r13)
1019 /* Setup our real return addr */
1020 LOAD_REG_ADDR(r4,rtas_return_loc)
1021 clrldi r4,r4,2 /* convert to realmode address */
1022 mtlr r4
1024 li r0,0
1025 ori r0,r0,MSR_EE|MSR_SE|MSR_BE|MSR_RI
1026 andc r0,r6,r0
1028 li r9,1
1029 rldicr r9,r9,MSR_SF_LG,(63-MSR_SF_LG)
1030 ori r9,r9,MSR_IR|MSR_DR|MSR_FE0|MSR_FE1|MSR_FP|MSR_RI|MSR_LE
1031 andc r6,r0,r9
1032 sync /* disable interrupts so SRR0/1 */
1033 mtmsrd r0 /* don't get trashed */
1035 LOAD_REG_ADDR(r4, rtas)
1036 ld r5,RTASENTRY(r4) /* get the rtas->entry value */
1037 ld r4,RTASBASE(r4) /* get the rtas->base value */
1039 mtspr SPRN_SRR0,r5
1040 mtspr SPRN_SRR1,r6
1041 rfid
1042 b . /* prevent speculative execution */
1044 rtas_return_loc:
1045 FIXUP_ENDIAN
1047 /* relocation is off at this point */
1048 GET_PACA(r4)
1049 clrldi r4,r4,2 /* convert to realmode address */
1051 bcl 20,31,$+4
1052 0: mflr r3
1053 ld r3,(1f-0b)(r3) /* get &rtas_restore_regs */
1055 mfmsr r6
1056 li r0,MSR_RI
1057 andc r6,r6,r0
1058 sync
1059 mtmsrd r6
1061 ld r1,PACAR1(r4) /* Restore our SP */
1062 ld r4,PACASAVEDMSR(r4) /* Restore our MSR */
1064 mtspr SPRN_SRR0,r3
1065 mtspr SPRN_SRR1,r4
1066 rfid
1067 b . /* prevent speculative execution */
1069 .align 3
1070 1: .llong rtas_restore_regs
1072 rtas_restore_regs:
1073 /* relocation is on at this point */
1074 REST_GPR(2, r1) /* Restore the TOC */
1075 REST_GPR(13, r1) /* Restore paca */
1076 REST_8GPRS(14, r1) /* Restore the non-volatiles */
1077 REST_10GPRS(22, r1) /* ditto */
1079 GET_PACA(r13)
1081 ld r4,_CCR(r1)
1082 mtcr r4
1083 ld r5,_CTR(r1)
1084 mtctr r5
1085 ld r6,_XER(r1)
1086 mtspr SPRN_XER,r6
1087 ld r7,_DAR(r1)
1088 mtdar r7
1089 ld r8,_DSISR(r1)
1090 mtdsisr r8
1092 addi r1,r1,RTAS_FRAME_SIZE /* Unstack our frame */
1093 ld r0,16(r1) /* get return address */
1095 mtlr r0
1096 blr /* return to caller */
1098 #endif /* CONFIG_PPC_RTAS */
1100 _GLOBAL(enter_prom)
1101 mflr r0
1102 std r0,16(r1)
1103 stdu r1,-PROM_FRAME_SIZE(r1) /* Save SP and create stack space */
1105 /* Because PROM is running in 32b mode, it clobbers the high order half
1106 * of all registers that it saves. We therefore save those registers
1107 * PROM might touch to the stack. (r0, r3-r13 are caller saved)
1108 */
1109 SAVE_GPR(2, r1)
1110 SAVE_GPR(13, r1)
1111 SAVE_8GPRS(14, r1)
1112 SAVE_10GPRS(22, r1)
1113 mfcr r10
1114 mfmsr r11
1115 std r10,_CCR(r1)
1116 std r11,_MSR(r1)
1118 /* Put PROM address in SRR0 */
1119 mtsrr0 r4
1121 /* Setup our trampoline return addr in LR */
1122 bcl 20,31,$+4
1123 0: mflr r4
1124 addi r4,r4,(1f - 0b)
1125 mtlr r4
1127 /* Prepare a 32-bit mode big endian MSR
1128 */
1129 #ifdef CONFIG_PPC_BOOK3E
1130 rlwinm r11,r11,0,1,31
1131 mtsrr1 r11
1132 rfi
1133 #else /* CONFIG_PPC_BOOK3E */
1134 LOAD_REG_IMMEDIATE(r12, MSR_SF | MSR_ISF | MSR_LE)
1135 andc r11,r11,r12
1136 mtsrr1 r11
1137 rfid
1138 #endif /* CONFIG_PPC_BOOK3E */
1140 1: /* Return from OF */
1141 FIXUP_ENDIAN
1143 /* Just make sure that r1 top 32 bits didn't get
1144 * corrupt by OF
1145 */
1146 rldicl r1,r1,0,32
1148 /* Restore the MSR (back to 64 bits) */
1149 ld r0,_MSR(r1)
1150 MTMSRD(r0)
1151 isync
1153 /* Restore other registers */
1154 REST_GPR(2, r1)
1155 REST_GPR(13, r1)
1156 REST_8GPRS(14, r1)
1157 REST_10GPRS(22, r1)
1158 ld r4,_CCR(r1)
1159 mtcr r4
1161 addi r1,r1,PROM_FRAME_SIZE
1162 ld r0,16(r1)
1163 mtlr r0
1164 blr
1166 #ifdef CONFIG_FUNCTION_TRACER
1167 #ifdef CONFIG_DYNAMIC_FTRACE
1168 _GLOBAL(mcount)
1169 _GLOBAL(_mcount)
1170 mflr r12
1171 mtctr r12
1172 mtlr r0
1173 bctr
1175 #ifndef CC_USING_MPROFILE_KERNEL
1176 _GLOBAL_TOC(ftrace_caller)
1177 /* Taken from output of objdump from lib64/glibc */
1178 mflr r3
1179 ld r11, 0(r1)
1180 stdu r1, -112(r1)
1181 std r3, 128(r1)
1182 ld r4, 16(r11)
1183 subi r3, r3, MCOUNT_INSN_SIZE
1184 .globl ftrace_call
1185 ftrace_call:
1186 bl ftrace_stub
1187 nop
1188 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1189 .globl ftrace_graph_call
1190 ftrace_graph_call:
1191 b ftrace_graph_stub
1192 _GLOBAL(ftrace_graph_stub)
1193 #endif
1194 ld r0, 128(r1)
1195 mtlr r0
1196 addi r1, r1, 112
1198 #else /* CC_USING_MPROFILE_KERNEL */
1199 /*
1200 *
1201 * ftrace_caller() is the function that replaces _mcount() when ftrace is
1202 * active.
1203 *
1204 * We arrive here after a function A calls function B, and we are the trace
1205 * function for B. When we enter r1 points to A's stack frame, B has not yet
1206 * had a chance to allocate one yet.
1207 *
1208 * Additionally r2 may point either to the TOC for A, or B, depending on
1209 * whether B did a TOC setup sequence before calling us.
1210 *
1211 * On entry the LR points back to the _mcount() call site, and r0 holds the
1212 * saved LR as it was on entry to B, ie. the original return address at the
1213 * call site in A.
1214 *
1215 * Our job is to save the register state into a struct pt_regs (on the stack)
1216 * and then arrange for the ftrace function to be called.
1217 */
1218 _GLOBAL(ftrace_caller)
1219 /* Save the original return address in A's stack frame */
1220 std r0,LRSAVE(r1)
1222 /* Create our stack frame + pt_regs */
1223 stdu r1,-SWITCH_FRAME_SIZE(r1)
1225 /* Save all gprs to pt_regs */
1226 SAVE_8GPRS(0,r1)
1227 SAVE_8GPRS(8,r1)
1228 SAVE_8GPRS(16,r1)
1229 SAVE_8GPRS(24,r1)
1231 /* Load special regs for save below */
1232 mfmsr r8
1233 mfctr r9
1234 mfxer r10
1235 mfcr r11
1237 /* Get the _mcount() call site out of LR */
1238 mflr r7
1239 /* Save it as pt_regs->nip & pt_regs->link */
1240 std r7, _NIP(r1)
1241 std r7, _LINK(r1)
1243 /* Save callee's TOC in the ABI compliant location */
1244 std r2, 24(r1)
1245 ld r2,PACATOC(r13) /* get kernel TOC in r2 */
1247 addis r3,r2,function_trace_op@toc@ha
1248 addi r3,r3,function_trace_op@toc@l
1249 ld r5,0(r3)
1251 /* Calculate ip from nip-4 into r3 for call below */
1252 subi r3, r7, MCOUNT_INSN_SIZE
1254 /* Put the original return address in r4 as parent_ip */
1255 mr r4, r0
1257 /* Save special regs */
1258 std r8, _MSR(r1)
1259 std r9, _CTR(r1)
1260 std r10, _XER(r1)
1261 std r11, _CCR(r1)
1263 /* Load &pt_regs in r6 for call below */
1264 addi r6, r1 ,STACK_FRAME_OVERHEAD
1266 /* ftrace_call(r3, r4, r5, r6) */
1267 .globl ftrace_call
1268 ftrace_call:
1269 bl ftrace_stub
1270 nop
1272 /* Load ctr with the possibly modified NIP */
1273 ld r3, _NIP(r1)
1274 mtctr r3
1276 /* Restore gprs */
1277 REST_8GPRS(0,r1)
1278 REST_8GPRS(8,r1)
1279 REST_8GPRS(16,r1)
1280 REST_8GPRS(24,r1)
1282 /* Restore callee's TOC */
1283 ld r2, 24(r1)
1285 /* Pop our stack frame */
1286 addi r1, r1, SWITCH_FRAME_SIZE
1288 /* Restore original LR for return to B */
1289 ld r0, LRSAVE(r1)
1290 mtlr r0
1292 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1293 stdu r1, -112(r1)
1294 .globl ftrace_graph_call
1295 ftrace_graph_call:
1296 b ftrace_graph_stub
1297 _GLOBAL(ftrace_graph_stub)
1298 addi r1, r1, 112
1299 #endif
1301 ld r0,LRSAVE(r1) /* restore callee's lr at _mcount site */
1302 mtlr r0
1303 bctr /* jump after _mcount site */
1304 #endif /* CC_USING_MPROFILE_KERNEL */
1306 _GLOBAL(ftrace_stub)
1307 blr
1308 #else
1309 _GLOBAL_TOC(_mcount)
1310 /* Taken from output of objdump from lib64/glibc */
1311 mflr r3
1312 ld r11, 0(r1)
1313 stdu r1, -112(r1)
1314 std r3, 128(r1)
1315 ld r4, 16(r11)
1317 subi r3, r3, MCOUNT_INSN_SIZE
1318 LOAD_REG_ADDR(r5,ftrace_trace_function)
1319 ld r5,0(r5)
1320 ld r5,0(r5)
1321 mtctr r5
1322 bctrl
1323 nop
1326 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1327 b ftrace_graph_caller
1328 #endif
1329 ld r0, 128(r1)
1330 mtlr r0
1331 addi r1, r1, 112
1332 _GLOBAL(ftrace_stub)
1333 blr
1335 #endif /* CONFIG_DYNAMIC_FTRACE */
1337 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1338 #ifndef CC_USING_MPROFILE_KERNEL
1339 _GLOBAL(ftrace_graph_caller)
1340 /* load r4 with local address */
1341 ld r4, 128(r1)
1342 subi r4, r4, MCOUNT_INSN_SIZE
1344 /* Grab the LR out of the caller stack frame */
1345 ld r11, 112(r1)
1346 ld r3, 16(r11)
1348 bl prepare_ftrace_return
1349 nop
1351 /*
1352 * prepare_ftrace_return gives us the address we divert to.
1353 * Change the LR in the callers stack frame to this.
1354 */
1355 ld r11, 112(r1)
1356 std r3, 16(r11)
1358 ld r0, 128(r1)
1359 mtlr r0
1360 addi r1, r1, 112
1361 blr
1363 #else /* CC_USING_MPROFILE_KERNEL */
1364 _GLOBAL(ftrace_graph_caller)
1365 /* with -mprofile-kernel, parameter regs are still alive at _mcount */
1366 std r10, 104(r1)
1367 std r9, 96(r1)
1368 std r8, 88(r1)
1369 std r7, 80(r1)
1370 std r6, 72(r1)
1371 std r5, 64(r1)
1372 std r4, 56(r1)
1373 std r3, 48(r1)
1375 /* Save callee's TOC in the ABI compliant location */
1376 std r2, 24(r1)
1377 ld r2, PACATOC(r13) /* get kernel TOC in r2 */
1379 mfctr r4 /* ftrace_caller has moved local addr here */
1380 std r4, 40(r1)
1381 mflr r3 /* ftrace_caller has restored LR from stack */
1382 subi r4, r4, MCOUNT_INSN_SIZE
1384 bl prepare_ftrace_return
1385 nop
1387 /*
1388 * prepare_ftrace_return gives us the address we divert to.
1389 * Change the LR to this.
1390 */
1391 mtlr r3
1393 ld r0, 40(r1)
1394 mtctr r0
1395 ld r10, 104(r1)
1396 ld r9, 96(r1)
1397 ld r8, 88(r1)
1398 ld r7, 80(r1)
1399 ld r6, 72(r1)
1400 ld r5, 64(r1)
1401 ld r4, 56(r1)
1402 ld r3, 48(r1)
1404 /* Restore callee's TOC */
1405 ld r2, 24(r1)
1407 addi r1, r1, 112
1408 mflr r0
1409 std r0, LRSAVE(r1)
1410 bctr
1411 #endif /* CC_USING_MPROFILE_KERNEL */
1413 _GLOBAL(return_to_handler)
1414 /* need to save return values */
1415 std r4, -32(r1)
1416 std r3, -24(r1)
1417 /* save TOC */
1418 std r2, -16(r1)
1419 std r31, -8(r1)
1420 mr r31, r1
1421 stdu r1, -112(r1)
1423 /*
1424 * We might be called from a module.
1425 * Switch to our TOC to run inside the core kernel.
1426 */
1427 ld r2, PACATOC(r13)
1429 bl ftrace_return_to_handler
1430 nop
1432 /* return value has real return address */
1433 mtlr r3
1435 ld r1, 0(r1)
1436 ld r4, -32(r1)
1437 ld r3, -24(r1)
1438 ld r2, -16(r1)
1439 ld r31, -8(r1)
1441 /* Jump back to real return address */
1442 blr
1443 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1444 #endif /* CONFIG_FUNCTION_TRACER */