| blob | a94f155db78e83d67fca2e967ba43585f18aeda8 |
1 /*
2 * PowerPC version
3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 * Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
5 * Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
6 * Adapted for Power Macintosh by Paul Mackerras.
7 * Low-level exception handlers and MMU support
8 * rewritten by Paul Mackerras.
9 * Copyright (C) 1996 Paul Mackerras.
10 * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
11 *
12 * This file contains the system call entry code, context switch
13 * code, and exception/interrupt return code for PowerPC.
14 *
15 * This program is free software; you can redistribute it and/or
16 * modify it under the terms of the GNU General Public License
17 * as published by the Free Software Foundation; either version
18 * 2 of the License, or (at your option) any later version.
19 */
21 #include <linux/errno.h>
22 #include <linux/err.h>
23 #include <asm/unistd.h>
24 #include <asm/processor.h>
25 #include <asm/page.h>
26 #include <asm/mmu.h>
27 #include <asm/thread_info.h>
28 #include <asm/ppc_asm.h>
29 #include <asm/asm-offsets.h>
30 #include <asm/cputable.h>
31 #include <asm/firmware.h>
32 #include <asm/bug.h>
33 #include <asm/ptrace.h>
34 #include <asm/irqflags.h>
35 #include <asm/ftrace.h>
36 #include <asm/hw_irq.h>
37 #include <asm/context_tracking.h>
38 #include <asm/tm.h>
40 /*
41 * System calls.
42 */
43 .section ".toc","aw"
44 SYS_CALL_TABLE:
45 .tc sys_call_table[TC],sys_call_table
47 /* This value is used to mark exception frames on the stack. */
48 exception_marker:
49 .tc ID_EXC_MARKER[TC],STACK_FRAME_REGS_MARKER
51 .section ".text"
52 .align 7
54 .globl system_call_common
55 system_call_common:
56 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
57 BEGIN_FTR_SECTION
58 extrdi. r10, r12, 1, (63-MSR_TS_T_LG) /* transaction active? */
59 bne tabort_syscall
60 END_FTR_SECTION_IFSET(CPU_FTR_TM)
61 #endif
62 andi. r10,r12,MSR_PR
63 mr r10,r1
64 addi r1,r1,-INT_FRAME_SIZE
65 beq- 1f
66 ld r1,PACAKSAVE(r13)
67 1: std r10,0(r1)
68 std r11,_NIP(r1)
69 std r12,_MSR(r1)
70 std r0,GPR0(r1)
71 std r10,GPR1(r1)
72 beq 2f /* if from kernel mode */
73 ACCOUNT_CPU_USER_ENTRY(r10, r11)
74 2: std r2,GPR2(r1)
75 std r3,GPR3(r1)
76 mfcr r2
77 std r4,GPR4(r1)
78 std r5,GPR5(r1)
79 std r6,GPR6(r1)
80 std r7,GPR7(r1)
81 std r8,GPR8(r1)
82 li r11,0
83 std r11,GPR9(r1)
84 std r11,GPR10(r1)
85 std r11,GPR11(r1)
86 std r11,GPR12(r1)
87 std r11,_XER(r1)
88 std r11,_CTR(r1)
89 std r9,GPR13(r1)
90 mflr r10
91 /*
92 * This clears CR0.SO (bit 28), which is the error indication on
93 * return from this system call.
94 */
95 rldimi r2,r11,28,(63-28)
96 li r11,0xc01
97 std r10,_LINK(r1)
98 std r11,_TRAP(r1)
99 std r3,ORIG_GPR3(r1)
100 std r2,_CCR(r1)
101 ld r2,PACATOC(r13)
102 addi r9,r1,STACK_FRAME_OVERHEAD
103 ld r11,exception_marker@toc(r2)
104 std r11,-16(r9) /* "regshere" marker */
105 #if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) && defined(CONFIG_PPC_SPLPAR)
106 BEGIN_FW_FTR_SECTION
107 beq 33f
108 /* if from user, see if there are any DTL entries to process */
109 ld r10,PACALPPACAPTR(r13) /* get ptr to VPA */
110 ld r11,PACA_DTL_RIDX(r13) /* get log read index */
111 addi r10,r10,LPPACA_DTLIDX
112 LDX_BE r10,0,r10 /* get log write index */
113 cmpd cr1,r11,r10
114 beq+ cr1,33f
115 bl accumulate_stolen_time
116 REST_GPR(0,r1)
117 REST_4GPRS(3,r1)
118 REST_2GPRS(7,r1)
119 addi r9,r1,STACK_FRAME_OVERHEAD
120 33:
121 END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR)
122 #endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE && CONFIG_PPC_SPLPAR */
124 /*
125 * A syscall should always be called with interrupts enabled
126 * so we just unconditionally hard-enable here. When some kind
127 * of irq tracing is used, we additionally check that condition
128 * is correct
129 */
130 #if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_BUG)
131 lbz r10,PACASOFTIRQEN(r13)
132 xori r10,r10,1
133 1: tdnei r10,0
134 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
135 #endif
137 #ifdef CONFIG_PPC_BOOK3E
138 wrteei 1
139 #else
140 ld r11,PACAKMSR(r13)
141 ori r11,r11,MSR_EE
142 mtmsrd r11,1
143 #endif /* CONFIG_PPC_BOOK3E */
145 /* We do need to set SOFTE in the stack frame or the return
146 * from interrupt will be painful
147 */
148 li r10,1
149 std r10,SOFTE(r1)
151 CURRENT_THREAD_INFO(r11, r1)
152 ld r10,TI_FLAGS(r11)
153 andi. r11,r10,_TIF_SYSCALL_DOTRACE
154 bne syscall_dotrace /* does not return */
155 cmpldi 0,r0,NR_syscalls
156 bge- syscall_enosys
158 system_call: /* label this so stack traces look sane */
159 /*
160 * Need to vector to 32 Bit or default sys_call_table here,
161 * based on caller's run-mode / personality.
162 */
163 ld r11,SYS_CALL_TABLE@toc(2)
164 andi. r10,r10,_TIF_32BIT
165 beq 15f
166 addi r11,r11,8 /* use 32-bit syscall entries */
167 clrldi r3,r3,32
168 clrldi r4,r4,32
169 clrldi r5,r5,32
170 clrldi r6,r6,32
171 clrldi r7,r7,32
172 clrldi r8,r8,32
173 15:
174 slwi r0,r0,4
175 ldx r12,r11,r0 /* Fetch system call handler [ptr] */
176 mtctr r12
177 bctrl /* Call handler */
179 .Lsyscall_exit:
180 std r3,RESULT(r1)
181 CURRENT_THREAD_INFO(r12, r1)
183 ld r8,_MSR(r1)
184 #ifdef CONFIG_PPC_BOOK3S
185 /* No MSR:RI on BookE */
186 andi. r10,r8,MSR_RI
187 beq- unrecov_restore
188 #endif
189 /*
190 * Disable interrupts so current_thread_info()->flags can't change,
191 * and so that we don't get interrupted after loading SRR0/1.
192 */
193 #ifdef CONFIG_PPC_BOOK3E
194 wrteei 0
195 #else
196 ld r10,PACAKMSR(r13)
197 /*
198 * For performance reasons we clear RI the same time that we
199 * clear EE. We only need to clear RI just before we restore r13
200 * below, but batching it with EE saves us one expensive mtmsrd call.
201 * We have to be careful to restore RI if we branch anywhere from
202 * here (eg syscall_exit_work).
203 */
204 li r9,MSR_RI
205 andc r11,r10,r9
206 mtmsrd r11,1
207 #endif /* CONFIG_PPC_BOOK3E */
209 ld r9,TI_FLAGS(r12)
210 li r11,-MAX_ERRNO
211 andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
212 bne- syscall_exit_work
213 cmpld r3,r11
214 ld r5,_CCR(r1)
215 bge- syscall_error
216 .Lsyscall_error_cont:
217 ld r7,_NIP(r1)
218 BEGIN_FTR_SECTION
219 stdcx. r0,0,r1 /* to clear the reservation */
220 END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
221 andi. r6,r8,MSR_PR
222 ld r4,_LINK(r1)
224 beq- 1f
225 ACCOUNT_CPU_USER_EXIT(r11, r12)
226 HMT_MEDIUM_LOW_HAS_PPR
227 ld r13,GPR13(r1) /* only restore r13 if returning to usermode */
228 1: ld r2,GPR2(r1)
229 ld r1,GPR1(r1)
230 mtlr r4
231 mtcr r5
232 mtspr SPRN_SRR0,r7
233 mtspr SPRN_SRR1,r8
234 RFI
235 b . /* prevent speculative execution */
237 syscall_error:
238 oris r5,r5,0x1000 /* Set SO bit in CR */
239 neg r3,r3
240 std r5,_CCR(r1)
241 b .Lsyscall_error_cont
243 /* Traced system call support */
244 syscall_dotrace:
245 bl save_nvgprs
246 addi r3,r1,STACK_FRAME_OVERHEAD
247 bl do_syscall_trace_enter
249 /*
250 * We use the return value of do_syscall_trace_enter() as the syscall
251 * number. If the syscall was rejected for any reason do_syscall_trace_enter()
252 * returns an invalid syscall number and the test below against
253 * NR_syscalls will fail.
254 */
255 mr r0,r3
257 /* Restore argument registers just clobbered and/or possibly changed. */
258 ld r3,GPR3(r1)
259 ld r4,GPR4(r1)
260 ld r5,GPR5(r1)
261 ld r6,GPR6(r1)
262 ld r7,GPR7(r1)
263 ld r8,GPR8(r1)
265 /* Repopulate r9 and r10 for the system_call path */
266 addi r9,r1,STACK_FRAME_OVERHEAD
267 CURRENT_THREAD_INFO(r10, r1)
268 ld r10,TI_FLAGS(r10)
270 cmpldi r0,NR_syscalls
271 blt+ system_call
273 /* Return code is already in r3 thanks to do_syscall_trace_enter() */
274 b .Lsyscall_exit
277 syscall_enosys:
278 li r3,-ENOSYS
279 b .Lsyscall_exit
281 syscall_exit_work:
282 #ifdef CONFIG_PPC_BOOK3S
283 mtmsrd r10,1 /* Restore RI */
284 #endif
285 /* If TIF_RESTOREALL is set, don't scribble on either r3 or ccr.
286 If TIF_NOERROR is set, just save r3 as it is. */
288 andi. r0,r9,_TIF_RESTOREALL
289 beq+ 0f
290 REST_NVGPRS(r1)
291 b 2f
292 0: cmpld r3,r11 /* r11 is -MAX_ERRNO */
293 blt+ 1f
294 andi. r0,r9,_TIF_NOERROR
295 bne- 1f
296 ld r5,_CCR(r1)
297 neg r3,r3
298 oris r5,r5,0x1000 /* Set SO bit in CR */
299 std r5,_CCR(r1)
300 1: std r3,GPR3(r1)
301 2: andi. r0,r9,(_TIF_PERSYSCALL_MASK)
302 beq 4f
304 /* Clear per-syscall TIF flags if any are set. */
306 li r11,_TIF_PERSYSCALL_MASK
307 addi r12,r12,TI_FLAGS
308 3: ldarx r10,0,r12
309 andc r10,r10,r11
310 stdcx. r10,0,r12
311 bne- 3b
312 subi r12,r12,TI_FLAGS
314 4: /* Anything else left to do? */
315 SET_DEFAULT_THREAD_PPR(r3, r10) /* Set thread.ppr = 3 */
316 andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP)
317 beq ret_from_except_lite
319 /* Re-enable interrupts */
320 #ifdef CONFIG_PPC_BOOK3E
321 wrteei 1
322 #else
323 ld r10,PACAKMSR(r13)
324 ori r10,r10,MSR_EE
325 mtmsrd r10,1
326 #endif /* CONFIG_PPC_BOOK3E */
328 bl save_nvgprs
329 addi r3,r1,STACK_FRAME_OVERHEAD
330 bl do_syscall_trace_leave
331 b ret_from_except
333 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
334 tabort_syscall:
335 /* Firstly we need to enable TM in the kernel */
336 mfmsr r10
337 li r13, 1
338 rldimi r10, r13, MSR_TM_LG, 63-MSR_TM_LG
339 mtmsrd r10, 0
341 /* tabort, this dooms the transaction, nothing else */
342 li r13, (TM_CAUSE_SYSCALL|TM_CAUSE_PERSISTENT)
343 TABORT(R13)
345 /*
346 * Return directly to userspace. We have corrupted user register state,
347 * but userspace will never see that register state. Execution will
348 * resume after the tbegin of the aborted transaction with the
349 * checkpointed register state.
350 */
351 li r13, MSR_RI
352 andc r10, r10, r13
353 mtmsrd r10, 1
354 mtspr SPRN_SRR0, r11
355 mtspr SPRN_SRR1, r12
357 rfid
358 b . /* prevent speculative execution */
359 #endif
361 /* Save non-volatile GPRs, if not already saved. */
362 _GLOBAL(save_nvgprs)
363 ld r11,_TRAP(r1)
364 andi. r0,r11,1
365 beqlr-
366 SAVE_NVGPRS(r1)
367 clrrdi r0,r11,1
368 std r0,_TRAP(r1)
369 blr
372 /*
373 * The sigsuspend and rt_sigsuspend system calls can call do_signal
374 * and thus put the process into the stopped state where we might
375 * want to examine its user state with ptrace. Therefore we need
376 * to save all the nonvolatile registers (r14 - r31) before calling
377 * the C code. Similarly, fork, vfork and clone need the full
378 * register state on the stack so that it can be copied to the child.
379 */
381 _GLOBAL(ppc_fork)
382 bl save_nvgprs
383 bl sys_fork
384 b .Lsyscall_exit
386 _GLOBAL(ppc_vfork)
387 bl save_nvgprs
388 bl sys_vfork
389 b .Lsyscall_exit
391 _GLOBAL(ppc_clone)
392 bl save_nvgprs
393 bl sys_clone
394 b .Lsyscall_exit
396 _GLOBAL(ppc32_swapcontext)
397 bl save_nvgprs
398 bl compat_sys_swapcontext
399 b .Lsyscall_exit
401 _GLOBAL(ppc64_swapcontext)
402 bl save_nvgprs
403 bl sys_swapcontext
404 b .Lsyscall_exit
406 _GLOBAL(ppc_switch_endian)
407 bl save_nvgprs
408 bl sys_switch_endian
409 b .Lsyscall_exit
411 _GLOBAL(ret_from_fork)
412 bl schedule_tail
413 REST_NVGPRS(r1)
414 li r3,0
415 b .Lsyscall_exit
417 _GLOBAL(ret_from_kernel_thread)
418 bl schedule_tail
419 REST_NVGPRS(r1)
420 mtlr r14
421 mr r3,r15
422 #if defined(_CALL_ELF) && _CALL_ELF == 2
423 mr r12,r14
424 #endif
425 blrl
426 li r3,0
427 b .Lsyscall_exit
429 /*
430 * This routine switches between two different tasks. The process
431 * state of one is saved on its kernel stack. Then the state
432 * of the other is restored from its kernel stack. The memory
433 * management hardware is updated to the second process's state.
434 * Finally, we can return to the second process, via ret_from_except.
435 * On entry, r3 points to the THREAD for the current task, r4
436 * points to the THREAD for the new task.
437 *
438 * Note: there are two ways to get to the "going out" portion
439 * of this code; either by coming in via the entry (_switch)
440 * or via "fork" which must set up an environment equivalent
441 * to the "_switch" path. If you change this you'll have to change
442 * the fork code also.
443 *
444 * The code which creates the new task context is in 'copy_thread'
445 * in arch/powerpc/kernel/process.c
446 */
447 .align 7
448 _GLOBAL(_switch)
449 mflr r0
450 std r0,16(r1)
451 stdu r1,-SWITCH_FRAME_SIZE(r1)
452 /* r3-r13 are caller saved -- Cort */
453 SAVE_8GPRS(14, r1)
454 SAVE_10GPRS(22, r1)
455 mflr r20 /* Return to switch caller */
456 mfmsr r22
457 li r0, MSR_FP
458 #ifdef CONFIG_VSX
459 BEGIN_FTR_SECTION
460 oris r0,r0,MSR_VSX@h /* Disable VSX */
461 END_FTR_SECTION_IFSET(CPU_FTR_VSX)
462 #endif /* CONFIG_VSX */
463 #ifdef CONFIG_ALTIVEC
464 BEGIN_FTR_SECTION
465 oris r0,r0,MSR_VEC@h /* Disable altivec */
466 mfspr r24,SPRN_VRSAVE /* save vrsave register value */
467 std r24,THREAD_VRSAVE(r3)
468 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
469 #endif /* CONFIG_ALTIVEC */
470 and. r0,r0,r22
471 beq+ 1f
472 andc r22,r22,r0
473 MTMSRD(r22)
474 isync
475 1: std r20,_NIP(r1)
476 mfcr r23
477 std r23,_CCR(r1)
478 std r1,KSP(r3) /* Set old stack pointer */
480 #ifdef CONFIG_PPC_BOOK3S_64
481 BEGIN_FTR_SECTION
482 /* Event based branch registers */
483 mfspr r0, SPRN_BESCR
484 std r0, THREAD_BESCR(r3)
485 mfspr r0, SPRN_EBBHR
486 std r0, THREAD_EBBHR(r3)
487 mfspr r0, SPRN_EBBRR
488 std r0, THREAD_EBBRR(r3)
489 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
490 #endif
492 #ifdef CONFIG_SMP
493 /* We need a sync somewhere here to make sure that if the
494 * previous task gets rescheduled on another CPU, it sees all
495 * stores it has performed on this one.
496 */
497 sync
498 #endif /* CONFIG_SMP */
500 /*
501 * If we optimise away the clear of the reservation in system
502 * calls because we know the CPU tracks the address of the
503 * reservation, then we need to clear it here to cover the
504 * case that the kernel context switch path has no larx
505 * instructions.
506 */
507 BEGIN_FTR_SECTION
508 ldarx r6,0,r1
509 END_FTR_SECTION_IFSET(CPU_FTR_STCX_CHECKS_ADDRESS)
511 #ifdef CONFIG_PPC_BOOK3S
512 /* Cancel all explict user streams as they will have no use after context
513 * switch and will stop the HW from creating streams itself
514 */
515 DCBT_STOP_ALL_STREAM_IDS(r6)
516 #endif
518 addi r6,r4,-THREAD /* Convert THREAD to 'current' */
519 std r6,PACACURRENT(r13) /* Set new 'current' */
521 ld r8,KSP(r4) /* new stack pointer */
522 #ifdef CONFIG_PPC_BOOK3S
523 BEGIN_FTR_SECTION
524 clrrdi r6,r8,28 /* get its ESID */
525 clrrdi r9,r1,28 /* get current sp ESID */
526 FTR_SECTION_ELSE
527 clrrdi r6,r8,40 /* get its 1T ESID */
528 clrrdi r9,r1,40 /* get current sp 1T ESID */
529 ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_1T_SEGMENT)
530 clrldi. r0,r6,2 /* is new ESID c00000000? */
531 cmpd cr1,r6,r9 /* or is new ESID the same as current ESID? */
532 cror eq,4*cr1+eq,eq
533 beq 2f /* if yes, don't slbie it */
535 /* Bolt in the new stack SLB entry */
536 ld r7,KSP_VSID(r4) /* Get new stack's VSID */
537 oris r0,r6,(SLB_ESID_V)@h
538 ori r0,r0,(SLB_NUM_BOLTED-1)@l
539 BEGIN_FTR_SECTION
540 li r9,MMU_SEGSIZE_1T /* insert B field */
541 oris r6,r6,(MMU_SEGSIZE_1T << SLBIE_SSIZE_SHIFT)@h
542 rldimi r7,r9,SLB_VSID_SSIZE_SHIFT,0
543 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
545 /* Update the last bolted SLB. No write barriers are needed
546 * here, provided we only update the current CPU's SLB shadow
547 * buffer.
548 */
549 ld r9,PACA_SLBSHADOWPTR(r13)
550 li r12,0
551 std r12,SLBSHADOW_STACKESID(r9) /* Clear ESID */
552 li r12,SLBSHADOW_STACKVSID
553 STDX_BE r7,r12,r9 /* Save VSID */
554 li r12,SLBSHADOW_STACKESID
555 STDX_BE r0,r12,r9 /* Save ESID */
557 /* No need to check for MMU_FTR_NO_SLBIE_B here, since when
558 * we have 1TB segments, the only CPUs known to have the errata
559 * only support less than 1TB of system memory and we'll never
560 * actually hit this code path.
561 */
563 slbie r6
564 slbie r6 /* Workaround POWER5 < DD2.1 issue */
565 slbmte r7,r0
566 isync
567 2:
568 #endif /* !CONFIG_PPC_BOOK3S */
570 CURRENT_THREAD_INFO(r7, r8) /* base of new stack */
571 /* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE
572 because we don't need to leave the 288-byte ABI gap at the
573 top of the kernel stack. */
574 addi r7,r7,THREAD_SIZE-SWITCH_FRAME_SIZE
576 mr r1,r8 /* start using new stack pointer */
577 std r7,PACAKSAVE(r13)
579 #ifdef CONFIG_PPC_BOOK3S_64
580 BEGIN_FTR_SECTION
581 /* Event based branch registers */
582 ld r0, THREAD_BESCR(r4)
583 mtspr SPRN_BESCR, r0
584 ld r0, THREAD_EBBHR(r4)
585 mtspr SPRN_EBBHR, r0
586 ld r0, THREAD_EBBRR(r4)
587 mtspr SPRN_EBBRR, r0
589 ld r0,THREAD_TAR(r4)
590 mtspr SPRN_TAR,r0
591 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
592 #endif
594 #ifdef CONFIG_ALTIVEC
595 BEGIN_FTR_SECTION
596 ld r0,THREAD_VRSAVE(r4)
597 mtspr SPRN_VRSAVE,r0 /* if G4, restore VRSAVE reg */
598 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
599 #endif /* CONFIG_ALTIVEC */
600 #ifdef CONFIG_PPC64
601 BEGIN_FTR_SECTION
602 lwz r6,THREAD_DSCR_INHERIT(r4)
603 ld r0,THREAD_DSCR(r4)
604 cmpwi r6,0
605 bne 1f
606 ld r0,PACA_DSCR_DEFAULT(r13)
607 1:
608 BEGIN_FTR_SECTION_NESTED(70)
609 mfspr r8, SPRN_FSCR
610 rldimi r8, r6, FSCR_DSCR_LG, (63 - FSCR_DSCR_LG)
611 mtspr SPRN_FSCR, r8
612 END_FTR_SECTION_NESTED(CPU_FTR_ARCH_207S, CPU_FTR_ARCH_207S, 70)
613 cmpd r0,r25
614 beq 2f
615 mtspr SPRN_DSCR,r0
616 2:
617 END_FTR_SECTION_IFSET(CPU_FTR_DSCR)
618 #endif
620 ld r6,_CCR(r1)
621 mtcrf 0xFF,r6
623 /* r3-r13 are destroyed -- Cort */
624 REST_8GPRS(14, r1)
625 REST_10GPRS(22, r1)
627 /* convert old thread to its task_struct for return value */
628 addi r3,r3,-THREAD
629 ld r7,_NIP(r1) /* Return to _switch caller in new task */
630 mtlr r7
631 addi r1,r1,SWITCH_FRAME_SIZE
632 blr
634 .align 7
635 _GLOBAL(ret_from_except)
636 ld r11,_TRAP(r1)
637 andi. r0,r11,1
638 bne ret_from_except_lite
639 REST_NVGPRS(r1)
641 _GLOBAL(ret_from_except_lite)
642 /*
643 * Disable interrupts so that current_thread_info()->flags
644 * can't change between when we test it and when we return
645 * from the interrupt.
646 */
647 #ifdef CONFIG_PPC_BOOK3E
648 wrteei 0
649 #else
650 ld r10,PACAKMSR(r13) /* Get kernel MSR without EE */
651 mtmsrd r10,1 /* Update machine state */
652 #endif /* CONFIG_PPC_BOOK3E */
654 CURRENT_THREAD_INFO(r9, r1)
655 ld r3,_MSR(r1)
656 #ifdef CONFIG_PPC_BOOK3E
657 ld r10,PACACURRENT(r13)
658 #endif /* CONFIG_PPC_BOOK3E */
659 ld r4,TI_FLAGS(r9)
660 andi. r3,r3,MSR_PR
661 beq resume_kernel
662 #ifdef CONFIG_PPC_BOOK3E
663 lwz r3,(THREAD+THREAD_DBCR0)(r10)
664 #endif /* CONFIG_PPC_BOOK3E */
666 /* Check current_thread_info()->flags */
667 andi. r0,r4,_TIF_USER_WORK_MASK
668 #ifdef CONFIG_PPC_BOOK3E
669 bne 1f
670 /*
671 * Check to see if the dbcr0 register is set up to debug.
672 * Use the internal debug mode bit to do this.
673 */
674 andis. r0,r3,DBCR0_IDM@h
675 beq restore
676 mfmsr r0
677 rlwinm r0,r0,0,~MSR_DE /* Clear MSR.DE */
678 mtmsr r0
679 mtspr SPRN_DBCR0,r3
680 li r10, -1
681 mtspr SPRN_DBSR,r10
682 b restore
683 #else
684 beq restore
685 #endif
686 1: andi. r0,r4,_TIF_NEED_RESCHED
687 beq 2f
688 bl restore_interrupts
689 SCHEDULE_USER
690 b ret_from_except_lite
691 2:
692 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
693 andi. r0,r4,_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM
694 bne 3f /* only restore TM if nothing else to do */
695 addi r3,r1,STACK_FRAME_OVERHEAD
696 bl restore_tm_state
697 b restore
698 3:
699 #endif
700 bl save_nvgprs
701 /*
702 * Use a non volatile GPR to save and restore our thread_info flags
703 * across the call to restore_interrupts.
704 */
705 mr r30,r4
706 bl restore_interrupts
707 mr r4,r30
708 addi r3,r1,STACK_FRAME_OVERHEAD
709 bl do_notify_resume
710 b ret_from_except
712 resume_kernel:
713 /* check current_thread_info, _TIF_EMULATE_STACK_STORE */
714 andis. r8,r4,_TIF_EMULATE_STACK_STORE@h
715 beq+ 1f
717 addi r8,r1,INT_FRAME_SIZE /* Get the kprobed function entry */
719 lwz r3,GPR1(r1)
720 subi r3,r3,INT_FRAME_SIZE /* dst: Allocate a trampoline exception frame */
721 mr r4,r1 /* src: current exception frame */
722 mr r1,r3 /* Reroute the trampoline frame to r1 */
724 /* Copy from the original to the trampoline. */
725 li r5,INT_FRAME_SIZE/8 /* size: INT_FRAME_SIZE */
726 li r6,0 /* start offset: 0 */
727 mtctr r5
728 2: ldx r0,r6,r4
729 stdx r0,r6,r3
730 addi r6,r6,8
731 bdnz 2b
733 /* Do real store operation to complete stwu */
734 lwz r5,GPR1(r1)
735 std r8,0(r5)
737 /* Clear _TIF_EMULATE_STACK_STORE flag */
738 lis r11,_TIF_EMULATE_STACK_STORE@h
739 addi r5,r9,TI_FLAGS
740 0: ldarx r4,0,r5
741 andc r4,r4,r11
742 stdcx. r4,0,r5
743 bne- 0b
744 1:
746 #ifdef CONFIG_PREEMPT
747 /* Check if we need to preempt */
748 andi. r0,r4,_TIF_NEED_RESCHED
749 beq+ restore
750 /* Check that preempt_count() == 0 and interrupts are enabled */
751 lwz r8,TI_PREEMPT(r9)
752 cmpwi cr1,r8,0
753 ld r0,SOFTE(r1)
754 cmpdi r0,0
755 crandc eq,cr1*4+eq,eq
756 bne restore
758 /*
759 * Here we are preempting the current task. We want to make
760 * sure we are soft-disabled first and reconcile irq state.
761 */
762 RECONCILE_IRQ_STATE(r3,r4)
763 1: bl preempt_schedule_irq
765 /* Re-test flags and eventually loop */
766 CURRENT_THREAD_INFO(r9, r1)
767 ld r4,TI_FLAGS(r9)
768 andi. r0,r4,_TIF_NEED_RESCHED
769 bne 1b
771 /*
772 * arch_local_irq_restore() from preempt_schedule_irq above may
773 * enable hard interrupt but we really should disable interrupts
774 * when we return from the interrupt, and so that we don't get
775 * interrupted after loading SRR0/1.
776 */
777 #ifdef CONFIG_PPC_BOOK3E
778 wrteei 0
779 #else
780 ld r10,PACAKMSR(r13) /* Get kernel MSR without EE */
781 mtmsrd r10,1 /* Update machine state */
782 #endif /* CONFIG_PPC_BOOK3E */
783 #endif /* CONFIG_PREEMPT */
785 .globl fast_exc_return_irq
786 fast_exc_return_irq:
787 restore:
788 /*
789 * This is the main kernel exit path. First we check if we
790 * are about to re-enable interrupts
791 */
792 ld r5,SOFTE(r1)
793 lbz r6,PACASOFTIRQEN(r13)
794 cmpwi cr0,r5,0
795 beq restore_irq_off
797 /* We are enabling, were we already enabled ? Yes, just return */
798 cmpwi cr0,r6,1
799 beq cr0,do_restore
801 /*
802 * We are about to soft-enable interrupts (we are hard disabled
803 * at this point). We check if there's anything that needs to
804 * be replayed first.
805 */
806 lbz r0,PACAIRQHAPPENED(r13)
807 cmpwi cr0,r0,0
808 bne- restore_check_irq_replay
810 /*
811 * Get here when nothing happened while soft-disabled, just
812 * soft-enable and move-on. We will hard-enable as a side
813 * effect of rfi
814 */
815 restore_no_replay:
816 TRACE_ENABLE_INTS
817 li r0,1
818 stb r0,PACASOFTIRQEN(r13);
820 /*
821 * Final return path. BookE is handled in a different file
822 */
823 do_restore:
824 #ifdef CONFIG_PPC_BOOK3E
825 b exception_return_book3e
826 #else
827 /*
828 * Clear the reservation. If we know the CPU tracks the address of
829 * the reservation then we can potentially save some cycles and use
830 * a larx. On POWER6 and POWER7 this is significantly faster.
831 */
832 BEGIN_FTR_SECTION
833 stdcx. r0,0,r1 /* to clear the reservation */
834 FTR_SECTION_ELSE
835 ldarx r4,0,r1
836 ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
838 /*
839 * Some code path such as load_up_fpu or altivec return directly
840 * here. They run entirely hard disabled and do not alter the
841 * interrupt state. They also don't use lwarx/stwcx. and thus
842 * are known not to leave dangling reservations.
843 */
844 .globl fast_exception_return
845 fast_exception_return:
846 ld r3,_MSR(r1)
847 ld r4,_CTR(r1)
848 ld r0,_LINK(r1)
849 mtctr r4
850 mtlr r0
851 ld r4,_XER(r1)
852 mtspr SPRN_XER,r4
854 REST_8GPRS(5, r1)
856 andi. r0,r3,MSR_RI
857 beq- unrecov_restore
859 /* Load PPR from thread struct before we clear MSR:RI */
860 BEGIN_FTR_SECTION
861 ld r2,PACACURRENT(r13)
862 ld r2,TASKTHREADPPR(r2)
863 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
865 /*
866 * Clear RI before restoring r13. If we are returning to
867 * userspace and we take an exception after restoring r13,
868 * we end up corrupting the userspace r13 value.
869 */
870 ld r4,PACAKMSR(r13) /* Get kernel MSR without EE */
871 andc r4,r4,r0 /* r0 contains MSR_RI here */
872 mtmsrd r4,1
874 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
875 /* TM debug */
876 std r3, PACATMSCRATCH(r13) /* Stash returned-to MSR */
877 #endif
878 /*
879 * r13 is our per cpu area, only restore it if we are returning to
880 * userspace the value stored in the stack frame may belong to
881 * another CPU.
882 */
883 andi. r0,r3,MSR_PR
884 beq 1f
885 BEGIN_FTR_SECTION
886 mtspr SPRN_PPR,r2 /* Restore PPR */
887 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
888 ACCOUNT_CPU_USER_EXIT(r2, r4)
889 REST_GPR(13, r1)
890 1:
891 mtspr SPRN_SRR1,r3
893 ld r2,_CCR(r1)
894 mtcrf 0xFF,r2
895 ld r2,_NIP(r1)
896 mtspr SPRN_SRR0,r2
898 ld r0,GPR0(r1)
899 ld r2,GPR2(r1)
900 ld r3,GPR3(r1)
901 ld r4,GPR4(r1)
902 ld r1,GPR1(r1)
904 rfid
905 b . /* prevent speculative execution */
907 #endif /* CONFIG_PPC_BOOK3E */
909 /*
910 * We are returning to a context with interrupts soft disabled.
911 *
912 * However, we may also about to hard enable, so we need to
913 * make sure that in this case, we also clear PACA_IRQ_HARD_DIS
914 * or that bit can get out of sync and bad things will happen
915 */
916 restore_irq_off:
917 ld r3,_MSR(r1)
918 lbz r7,PACAIRQHAPPENED(r13)
919 andi. r0,r3,MSR_EE
920 beq 1f
921 rlwinm r7,r7,0,~PACA_IRQ_HARD_DIS
922 stb r7,PACAIRQHAPPENED(r13)
923 1: li r0,0
924 stb r0,PACASOFTIRQEN(r13);
925 TRACE_DISABLE_INTS
926 b do_restore
928 /*
929 * Something did happen, check if a re-emit is needed
930 * (this also clears paca->irq_happened)
931 */
932 restore_check_irq_replay:
933 /* XXX: We could implement a fast path here where we check
934 * for irq_happened being just 0x01, in which case we can
935 * clear it and return. That means that we would potentially
936 * miss a decrementer having wrapped all the way around.
937 *
938 * Still, this might be useful for things like hash_page
939 */
940 bl __check_irq_replay
941 cmpwi cr0,r3,0
942 beq restore_no_replay
944 /*
945 * We need to re-emit an interrupt. We do so by re-using our
946 * existing exception frame. We first change the trap value,
947 * but we need to ensure we preserve the low nibble of it
948 */
949 ld r4,_TRAP(r1)
950 clrldi r4,r4,60
951 or r4,r4,r3
952 std r4,_TRAP(r1)
954 /*
955 * Then find the right handler and call it. Interrupts are
956 * still soft-disabled and we keep them that way.
957 */
958 cmpwi cr0,r3,0x500
959 bne 1f
960 addi r3,r1,STACK_FRAME_OVERHEAD;
961 bl do_IRQ
962 b ret_from_except
963 1: cmpwi cr0,r3,0xe60
964 bne 1f
965 addi r3,r1,STACK_FRAME_OVERHEAD;
966 bl handle_hmi_exception
967 b ret_from_except
968 1: cmpwi cr0,r3,0x900
969 bne 1f
970 addi r3,r1,STACK_FRAME_OVERHEAD;
971 bl timer_interrupt
972 b ret_from_except
973 #ifdef CONFIG_PPC_DOORBELL
974 1:
975 #ifdef CONFIG_PPC_BOOK3E
976 cmpwi cr0,r3,0x280
977 #else
978 BEGIN_FTR_SECTION
979 cmpwi cr0,r3,0xe80
980 FTR_SECTION_ELSE
981 cmpwi cr0,r3,0xa00
982 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
983 #endif /* CONFIG_PPC_BOOK3E */
984 bne 1f
985 addi r3,r1,STACK_FRAME_OVERHEAD;
986 bl doorbell_exception
987 b ret_from_except
988 #endif /* CONFIG_PPC_DOORBELL */
989 1: b ret_from_except /* What else to do here ? */
991 unrecov_restore:
992 addi r3,r1,STACK_FRAME_OVERHEAD
993 bl unrecoverable_exception
994 b unrecov_restore
996 #ifdef CONFIG_PPC_RTAS
997 /*
998 * On CHRP, the Run-Time Abstraction Services (RTAS) have to be
999 * called with the MMU off.
1000 *
1001 * In addition, we need to be in 32b mode, at least for now.
1002 *
1003 * Note: r3 is an input parameter to rtas, so don't trash it...
1004 */
1005 _GLOBAL(enter_rtas)
1006 mflr r0
1007 std r0,16(r1)
1008 stdu r1,-RTAS_FRAME_SIZE(r1) /* Save SP and create stack space. */
1010 /* Because RTAS is running in 32b mode, it clobbers the high order half
1011 * of all registers that it saves. We therefore save those registers
1012 * RTAS might touch to the stack. (r0, r3-r13 are caller saved)
1013 */
1014 SAVE_GPR(2, r1) /* Save the TOC */
1015 SAVE_GPR(13, r1) /* Save paca */
1016 SAVE_8GPRS(14, r1) /* Save the non-volatiles */
1017 SAVE_10GPRS(22, r1) /* ditto */
1019 mfcr r4
1020 std r4,_CCR(r1)
1021 mfctr r5
1022 std r5,_CTR(r1)
1023 mfspr r6,SPRN_XER
1024 std r6,_XER(r1)
1025 mfdar r7
1026 std r7,_DAR(r1)
1027 mfdsisr r8
1028 std r8,_DSISR(r1)
1030 /* Temporary workaround to clear CR until RTAS can be modified to
1031 * ignore all bits.
1032 */
1033 li r0,0
1034 mtcr r0
1036 #ifdef CONFIG_BUG
1037 /* There is no way it is acceptable to get here with interrupts enabled,
1038 * check it with the asm equivalent of WARN_ON
1039 */
1040 lbz r0,PACASOFTIRQEN(r13)
1041 1: tdnei r0,0
1042 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
1043 #endif
1045 /* Hard-disable interrupts */
1046 mfmsr r6
1047 rldicl r7,r6,48,1
1048 rotldi r7,r7,16
1049 mtmsrd r7,1
1051 /* Unfortunately, the stack pointer and the MSR are also clobbered,
1052 * so they are saved in the PACA which allows us to restore
1053 * our original state after RTAS returns.
1054 */
1055 std r1,PACAR1(r13)
1056 std r6,PACASAVEDMSR(r13)
1058 /* Setup our real return addr */
1059 LOAD_REG_ADDR(r4,rtas_return_loc)
1060 clrldi r4,r4,2 /* convert to realmode address */
1061 mtlr r4
1063 li r0,0
1064 ori r0,r0,MSR_EE|MSR_SE|MSR_BE|MSR_RI
1065 andc r0,r6,r0
1067 li r9,1
1068 rldicr r9,r9,MSR_SF_LG,(63-MSR_SF_LG)
1069 ori r9,r9,MSR_IR|MSR_DR|MSR_FE0|MSR_FE1|MSR_FP|MSR_RI|MSR_LE
1070 andc r6,r0,r9
1071 sync /* disable interrupts so SRR0/1 */
1072 mtmsrd r0 /* don't get trashed */
1074 LOAD_REG_ADDR(r4, rtas)
1075 ld r5,RTASENTRY(r4) /* get the rtas->entry value */
1076 ld r4,RTASBASE(r4) /* get the rtas->base value */
1078 mtspr SPRN_SRR0,r5
1079 mtspr SPRN_SRR1,r6
1080 rfid
1081 b . /* prevent speculative execution */
1083 rtas_return_loc:
1084 FIXUP_ENDIAN
1086 /* relocation is off at this point */
1087 GET_PACA(r4)
1088 clrldi r4,r4,2 /* convert to realmode address */
1090 bcl 20,31,$+4
1091 0: mflr r3
1092 ld r3,(1f-0b)(r3) /* get &rtas_restore_regs */
1094 mfmsr r6
1095 li r0,MSR_RI
1096 andc r6,r6,r0
1097 sync
1098 mtmsrd r6
1100 ld r1,PACAR1(r4) /* Restore our SP */
1101 ld r4,PACASAVEDMSR(r4) /* Restore our MSR */
1103 mtspr SPRN_SRR0,r3
1104 mtspr SPRN_SRR1,r4
1105 rfid
1106 b . /* prevent speculative execution */
1108 .align 3
1109 1: .llong rtas_restore_regs
1111 rtas_restore_regs:
1112 /* relocation is on at this point */
1113 REST_GPR(2, r1) /* Restore the TOC */
1114 REST_GPR(13, r1) /* Restore paca */
1115 REST_8GPRS(14, r1) /* Restore the non-volatiles */
1116 REST_10GPRS(22, r1) /* ditto */
1118 GET_PACA(r13)
1120 ld r4,_CCR(r1)
1121 mtcr r4
1122 ld r5,_CTR(r1)
1123 mtctr r5
1124 ld r6,_XER(r1)
1125 mtspr SPRN_XER,r6
1126 ld r7,_DAR(r1)
1127 mtdar r7
1128 ld r8,_DSISR(r1)
1129 mtdsisr r8
1131 addi r1,r1,RTAS_FRAME_SIZE /* Unstack our frame */
1132 ld r0,16(r1) /* get return address */
1134 mtlr r0
1135 blr /* return to caller */
1137 #endif /* CONFIG_PPC_RTAS */
1139 _GLOBAL(enter_prom)
1140 mflr r0
1141 std r0,16(r1)
1142 stdu r1,-PROM_FRAME_SIZE(r1) /* Save SP and create stack space */
1144 /* Because PROM is running in 32b mode, it clobbers the high order half
1145 * of all registers that it saves. We therefore save those registers
1146 * PROM might touch to the stack. (r0, r3-r13 are caller saved)
1147 */
1148 SAVE_GPR(2, r1)
1149 SAVE_GPR(13, r1)
1150 SAVE_8GPRS(14, r1)
1151 SAVE_10GPRS(22, r1)
1152 mfcr r10
1153 mfmsr r11
1154 std r10,_CCR(r1)
1155 std r11,_MSR(r1)
1157 /* Put PROM address in SRR0 */
1158 mtsrr0 r4
1160 /* Setup our trampoline return addr in LR */
1161 bcl 20,31,$+4
1162 0: mflr r4
1163 addi r4,r4,(1f - 0b)
1164 mtlr r4
1166 /* Prepare a 32-bit mode big endian MSR
1167 */
1168 #ifdef CONFIG_PPC_BOOK3E
1169 rlwinm r11,r11,0,1,31
1170 mtsrr1 r11
1171 rfi
1172 #else /* CONFIG_PPC_BOOK3E */
1173 LOAD_REG_IMMEDIATE(r12, MSR_SF | MSR_ISF | MSR_LE)
1174 andc r11,r11,r12
1175 mtsrr1 r11
1176 rfid
1177 #endif /* CONFIG_PPC_BOOK3E */
1179 1: /* Return from OF */
1180 FIXUP_ENDIAN
1182 /* Just make sure that r1 top 32 bits didn't get
1183 * corrupt by OF
1184 */
1185 rldicl r1,r1,0,32
1187 /* Restore the MSR (back to 64 bits) */
1188 ld r0,_MSR(r1)
1189 MTMSRD(r0)
1190 isync
1192 /* Restore other registers */
1193 REST_GPR(2, r1)
1194 REST_GPR(13, r1)
1195 REST_8GPRS(14, r1)
1196 REST_10GPRS(22, r1)
1197 ld r4,_CCR(r1)
1198 mtcr r4
1200 addi r1,r1,PROM_FRAME_SIZE
1201 ld r0,16(r1)
1202 mtlr r0
1203 blr
1205 #ifdef CONFIG_FUNCTION_TRACER
1206 #ifdef CONFIG_DYNAMIC_FTRACE
1207 _GLOBAL(mcount)
1208 _GLOBAL(_mcount)
1209 blr
1211 _GLOBAL_TOC(ftrace_caller)
1212 /* Taken from output of objdump from lib64/glibc */
1213 mflr r3
1214 ld r11, 0(r1)
1215 stdu r1, -112(r1)
1216 std r3, 128(r1)
1217 ld r4, 16(r11)
1218 subi r3, r3, MCOUNT_INSN_SIZE
1219 .globl ftrace_call
1220 ftrace_call:
1221 bl ftrace_stub
1222 nop
1223 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1224 .globl ftrace_graph_call
1225 ftrace_graph_call:
1226 b ftrace_graph_stub
1227 _GLOBAL(ftrace_graph_stub)
1228 #endif
1229 ld r0, 128(r1)
1230 mtlr r0
1231 addi r1, r1, 112
1232 _GLOBAL(ftrace_stub)
1233 blr
1234 #else
1235 _GLOBAL_TOC(_mcount)
1236 /* Taken from output of objdump from lib64/glibc */
1237 mflr r3
1238 ld r11, 0(r1)
1239 stdu r1, -112(r1)
1240 std r3, 128(r1)
1241 ld r4, 16(r11)
1243 subi r3, r3, MCOUNT_INSN_SIZE
1244 LOAD_REG_ADDR(r5,ftrace_trace_function)
1245 ld r5,0(r5)
1246 ld r5,0(r5)
1247 mtctr r5
1248 bctrl
1249 nop
1252 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1253 b ftrace_graph_caller
1254 #endif
1255 ld r0, 128(r1)
1256 mtlr r0
1257 addi r1, r1, 112
1258 _GLOBAL(ftrace_stub)
1259 blr
1261 #endif /* CONFIG_DYNAMIC_FTRACE */
1263 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1264 _GLOBAL(ftrace_graph_caller)
1265 /* load r4 with local address */
1266 ld r4, 128(r1)
1267 subi r4, r4, MCOUNT_INSN_SIZE
1269 /* Grab the LR out of the caller stack frame */
1270 ld r11, 112(r1)
1271 ld r3, 16(r11)
1273 bl prepare_ftrace_return
1274 nop
1276 /*
1277 * prepare_ftrace_return gives us the address we divert to.
1278 * Change the LR in the callers stack frame to this.
1279 */
1280 ld r11, 112(r1)
1281 std r3, 16(r11)
1283 ld r0, 128(r1)
1284 mtlr r0
1285 addi r1, r1, 112
1286 blr
1288 _GLOBAL(return_to_handler)
1289 /* need to save return values */
1290 std r4, -32(r1)
1291 std r3, -24(r1)
1292 /* save TOC */
1293 std r2, -16(r1)
1294 std r31, -8(r1)
1295 mr r31, r1
1296 stdu r1, -112(r1)
1298 /*
1299 * We might be called from a module.
1300 * Switch to our TOC to run inside the core kernel.
1301 */
1302 ld r2, PACATOC(r13)
1304 bl ftrace_return_to_handler
1305 nop
1307 /* return value has real return address */
1308 mtlr r3
1310 ld r1, 0(r1)
1311 ld r4, -32(r1)
1312 ld r3, -24(r1)
1313 ld r2, -16(r1)
1314 ld r31, -8(r1)
1316 /* Jump back to real return address */
1317 blr
1318 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1319 #endif /* CONFIG_FUNCTION_TRACER */