| blob | 6528c5e2cc44800a201802858c23d34e92162041 |
1 /*
2 * PowerPC version
3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 * Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
5 * Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
6 * Adapted for Power Macintosh by Paul Mackerras.
7 * Low-level exception handlers and MMU support
8 * rewritten by Paul Mackerras.
9 * Copyright (C) 1996 Paul Mackerras.
10 * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
11 *
12 * This file contains the system call entry code, context switch
13 * code, and exception/interrupt return code for PowerPC.
14 *
15 * This program is free software; you can redistribute it and/or
16 * modify it under the terms of the GNU General Public License
17 * as published by the Free Software Foundation; either version
18 * 2 of the License, or (at your option) any later version.
19 */
21 #include <linux/errno.h>
22 #include <asm/unistd.h>
23 #include <asm/processor.h>
24 #include <asm/page.h>
25 #include <asm/mmu.h>
26 #include <asm/thread_info.h>
27 #include <asm/ppc_asm.h>
28 #include <asm/asm-offsets.h>
29 #include <asm/cputable.h>
30 #include <asm/firmware.h>
31 #include <asm/bug.h>
32 #include <asm/ptrace.h>
33 #include <asm/irqflags.h>
34 #include <asm/ftrace.h>
35 #include <asm/hw_irq.h>
36 #include <asm/context_tracking.h>
38 /*
39 * System calls.
40 */
41 .section ".toc","aw"
42 SYS_CALL_TABLE:
43 .tc sys_call_table[TC],sys_call_table
45 /* This value is used to mark exception frames on the stack. */
46 exception_marker:
47 .tc ID_EXC_MARKER[TC],STACK_FRAME_REGS_MARKER
49 .section ".text"
50 .align 7
52 #undef SHOW_SYSCALLS
54 .globl system_call_common
55 system_call_common:
56 andi. r10,r12,MSR_PR
57 mr r10,r1
58 addi r1,r1,-INT_FRAME_SIZE
59 beq- 1f
60 ld r1,PACAKSAVE(r13)
61 1: std r10,0(r1)
62 std r11,_NIP(r1)
63 std r12,_MSR(r1)
64 std r0,GPR0(r1)
65 std r10,GPR1(r1)
66 beq 2f /* if from kernel mode */
67 ACCOUNT_CPU_USER_ENTRY(r10, r11)
68 2: std r2,GPR2(r1)
69 std r3,GPR3(r1)
70 mfcr r2
71 std r4,GPR4(r1)
72 std r5,GPR5(r1)
73 std r6,GPR6(r1)
74 std r7,GPR7(r1)
75 std r8,GPR8(r1)
76 li r11,0
77 std r11,GPR9(r1)
78 std r11,GPR10(r1)
79 std r11,GPR11(r1)
80 std r11,GPR12(r1)
81 std r11,_XER(r1)
82 std r11,_CTR(r1)
83 std r9,GPR13(r1)
84 mflr r10
85 /*
86 * This clears CR0.SO (bit 28), which is the error indication on
87 * return from this system call.
88 */
89 rldimi r2,r11,28,(63-28)
90 li r11,0xc01
91 std r10,_LINK(r1)
92 std r11,_TRAP(r1)
93 std r3,ORIG_GPR3(r1)
94 std r2,_CCR(r1)
95 ld r2,PACATOC(r13)
96 addi r9,r1,STACK_FRAME_OVERHEAD
97 ld r11,exception_marker@toc(r2)
98 std r11,-16(r9) /* "regshere" marker */
99 #if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) && defined(CONFIG_PPC_SPLPAR)
100 BEGIN_FW_FTR_SECTION
101 beq 33f
102 /* if from user, see if there are any DTL entries to process */
103 ld r10,PACALPPACAPTR(r13) /* get ptr to VPA */
104 ld r11,PACA_DTL_RIDX(r13) /* get log read index */
105 addi r10,r10,LPPACA_DTLIDX
106 LDX_BE r10,0,r10 /* get log write index */
107 cmpd cr1,r11,r10
108 beq+ cr1,33f
109 bl accumulate_stolen_time
110 REST_GPR(0,r1)
111 REST_4GPRS(3,r1)
112 REST_2GPRS(7,r1)
113 addi r9,r1,STACK_FRAME_OVERHEAD
114 33:
115 END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR)
116 #endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE && CONFIG_PPC_SPLPAR */
118 /*
119 * A syscall should always be called with interrupts enabled
120 * so we just unconditionally hard-enable here. When some kind
121 * of irq tracing is used, we additionally check that condition
122 * is correct
123 */
124 #if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_BUG)
125 lbz r10,PACASOFTIRQEN(r13)
126 xori r10,r10,1
127 1: tdnei r10,0
128 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
129 #endif
131 #ifdef CONFIG_PPC_BOOK3E
132 wrteei 1
133 #else
134 ld r11,PACAKMSR(r13)
135 ori r11,r11,MSR_EE
136 mtmsrd r11,1
137 #endif /* CONFIG_PPC_BOOK3E */
139 /* We do need to set SOFTE in the stack frame or the return
140 * from interrupt will be painful
141 */
142 li r10,1
143 std r10,SOFTE(r1)
145 #ifdef SHOW_SYSCALLS
146 bl do_show_syscall
147 REST_GPR(0,r1)
148 REST_4GPRS(3,r1)
149 REST_2GPRS(7,r1)
150 addi r9,r1,STACK_FRAME_OVERHEAD
151 #endif
152 CURRENT_THREAD_INFO(r11, r1)
153 ld r10,TI_FLAGS(r11)
154 andi. r11,r10,_TIF_SYSCALL_T_OR_A
155 bne syscall_dotrace
156 .Lsyscall_dotrace_cont:
157 cmpldi 0,r0,NR_syscalls
158 bge- syscall_enosys
160 system_call: /* label this so stack traces look sane */
161 /*
162 * Need to vector to 32 Bit or default sys_call_table here,
163 * based on caller's run-mode / personality.
164 */
165 ld r11,SYS_CALL_TABLE@toc(2)
166 andi. r10,r10,_TIF_32BIT
167 beq 15f
168 addi r11,r11,8 /* use 32-bit syscall entries */
169 clrldi r3,r3,32
170 clrldi r4,r4,32
171 clrldi r5,r5,32
172 clrldi r6,r6,32
173 clrldi r7,r7,32
174 clrldi r8,r8,32
175 15:
176 slwi r0,r0,4
177 ldx r12,r11,r0 /* Fetch system call handler [ptr] */
178 mtctr r12
179 bctrl /* Call handler */
181 syscall_exit:
182 std r3,RESULT(r1)
183 #ifdef SHOW_SYSCALLS
184 bl do_show_syscall_exit
185 ld r3,RESULT(r1)
186 #endif
187 CURRENT_THREAD_INFO(r12, r1)
189 ld r8,_MSR(r1)
190 #ifdef CONFIG_PPC_BOOK3S
191 /* No MSR:RI on BookE */
192 andi. r10,r8,MSR_RI
193 beq- unrecov_restore
194 #endif
195 /*
196 * Disable interrupts so current_thread_info()->flags can't change,
197 * and so that we don't get interrupted after loading SRR0/1.
198 */
199 #ifdef CONFIG_PPC_BOOK3E
200 wrteei 0
201 #else
202 ld r10,PACAKMSR(r13)
203 /*
204 * For performance reasons we clear RI the same time that we
205 * clear EE. We only need to clear RI just before we restore r13
206 * below, but batching it with EE saves us one expensive mtmsrd call.
207 * We have to be careful to restore RI if we branch anywhere from
208 * here (eg syscall_exit_work).
209 */
210 li r9,MSR_RI
211 andc r11,r10,r9
212 mtmsrd r11,1
213 #endif /* CONFIG_PPC_BOOK3E */
215 ld r9,TI_FLAGS(r12)
216 li r11,-_LAST_ERRNO
217 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
218 bne- syscall_exit_work
219 cmpld r3,r11
220 ld r5,_CCR(r1)
221 bge- syscall_error
222 .Lsyscall_error_cont:
223 ld r7,_NIP(r1)
224 BEGIN_FTR_SECTION
225 stdcx. r0,0,r1 /* to clear the reservation */
226 END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
227 andi. r6,r8,MSR_PR
228 ld r4,_LINK(r1)
230 beq- 1f
231 ACCOUNT_CPU_USER_EXIT(r11, r12)
232 HMT_MEDIUM_LOW_HAS_PPR
233 ld r13,GPR13(r1) /* only restore r13 if returning to usermode */
234 1: ld r2,GPR2(r1)
235 ld r1,GPR1(r1)
236 mtlr r4
237 mtcr r5
238 mtspr SPRN_SRR0,r7
239 mtspr SPRN_SRR1,r8
240 RFI
241 b . /* prevent speculative execution */
243 syscall_error:
244 oris r5,r5,0x1000 /* Set SO bit in CR */
245 neg r3,r3
246 std r5,_CCR(r1)
247 b .Lsyscall_error_cont
249 /* Traced system call support */
250 syscall_dotrace:
251 bl save_nvgprs
252 addi r3,r1,STACK_FRAME_OVERHEAD
253 bl do_syscall_trace_enter
254 /*
255 * Restore argument registers possibly just changed.
256 * We use the return value of do_syscall_trace_enter
257 * for the call number to look up in the table (r0).
258 */
259 mr r0,r3
260 ld r3,GPR3(r1)
261 ld r4,GPR4(r1)
262 ld r5,GPR5(r1)
263 ld r6,GPR6(r1)
264 ld r7,GPR7(r1)
265 ld r8,GPR8(r1)
266 addi r9,r1,STACK_FRAME_OVERHEAD
267 CURRENT_THREAD_INFO(r10, r1)
268 ld r10,TI_FLAGS(r10)
269 b .Lsyscall_dotrace_cont
271 syscall_enosys:
272 li r3,-ENOSYS
273 b syscall_exit
275 syscall_exit_work:
276 #ifdef CONFIG_PPC_BOOK3S
277 mtmsrd r10,1 /* Restore RI */
278 #endif
279 /* If TIF_RESTOREALL is set, don't scribble on either r3 or ccr.
280 If TIF_NOERROR is set, just save r3 as it is. */
282 andi. r0,r9,_TIF_RESTOREALL
283 beq+ 0f
284 REST_NVGPRS(r1)
285 b 2f
286 0: cmpld r3,r11 /* r10 is -LAST_ERRNO */
287 blt+ 1f
288 andi. r0,r9,_TIF_NOERROR
289 bne- 1f
290 ld r5,_CCR(r1)
291 neg r3,r3
292 oris r5,r5,0x1000 /* Set SO bit in CR */
293 std r5,_CCR(r1)
294 1: std r3,GPR3(r1)
295 2: andi. r0,r9,(_TIF_PERSYSCALL_MASK)
296 beq 4f
298 /* Clear per-syscall TIF flags if any are set. */
300 li r11,_TIF_PERSYSCALL_MASK
301 addi r12,r12,TI_FLAGS
302 3: ldarx r10,0,r12
303 andc r10,r10,r11
304 stdcx. r10,0,r12
305 bne- 3b
306 subi r12,r12,TI_FLAGS
308 4: /* Anything else left to do? */
309 SET_DEFAULT_THREAD_PPR(r3, r10) /* Set thread.ppr = 3 */
310 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP)
311 beq ret_from_except_lite
313 /* Re-enable interrupts */
314 #ifdef CONFIG_PPC_BOOK3E
315 wrteei 1
316 #else
317 ld r10,PACAKMSR(r13)
318 ori r10,r10,MSR_EE
319 mtmsrd r10,1
320 #endif /* CONFIG_PPC_BOOK3E */
322 bl save_nvgprs
323 addi r3,r1,STACK_FRAME_OVERHEAD
324 bl do_syscall_trace_leave
325 b ret_from_except
327 /* Save non-volatile GPRs, if not already saved. */
328 _GLOBAL(save_nvgprs)
329 ld r11,_TRAP(r1)
330 andi. r0,r11,1
331 beqlr-
332 SAVE_NVGPRS(r1)
333 clrrdi r0,r11,1
334 std r0,_TRAP(r1)
335 blr
338 /*
339 * The sigsuspend and rt_sigsuspend system calls can call do_signal
340 * and thus put the process into the stopped state where we might
341 * want to examine its user state with ptrace. Therefore we need
342 * to save all the nonvolatile registers (r14 - r31) before calling
343 * the C code. Similarly, fork, vfork and clone need the full
344 * register state on the stack so that it can be copied to the child.
345 */
347 _GLOBAL(ppc_fork)
348 bl save_nvgprs
349 bl sys_fork
350 b syscall_exit
352 _GLOBAL(ppc_vfork)
353 bl save_nvgprs
354 bl sys_vfork
355 b syscall_exit
357 _GLOBAL(ppc_clone)
358 bl save_nvgprs
359 bl sys_clone
360 b syscall_exit
362 _GLOBAL(ppc32_swapcontext)
363 bl save_nvgprs
364 bl compat_sys_swapcontext
365 b syscall_exit
367 _GLOBAL(ppc64_swapcontext)
368 bl save_nvgprs
369 bl sys_swapcontext
370 b syscall_exit
372 _GLOBAL(ret_from_fork)
373 bl schedule_tail
374 REST_NVGPRS(r1)
375 li r3,0
376 b syscall_exit
378 _GLOBAL(ret_from_kernel_thread)
379 bl schedule_tail
380 REST_NVGPRS(r1)
381 mtlr r14
382 mr r3,r15
383 #if defined(_CALL_ELF) && _CALL_ELF == 2
384 mr r12,r14
385 #endif
386 blrl
387 li r3,0
388 b syscall_exit
390 /*
391 * This routine switches between two different tasks. The process
392 * state of one is saved on its kernel stack. Then the state
393 * of the other is restored from its kernel stack. The memory
394 * management hardware is updated to the second process's state.
395 * Finally, we can return to the second process, via ret_from_except.
396 * On entry, r3 points to the THREAD for the current task, r4
397 * points to the THREAD for the new task.
398 *
399 * Note: there are two ways to get to the "going out" portion
400 * of this code; either by coming in via the entry (_switch)
401 * or via "fork" which must set up an environment equivalent
402 * to the "_switch" path. If you change this you'll have to change
403 * the fork code also.
404 *
405 * The code which creates the new task context is in 'copy_thread'
406 * in arch/powerpc/kernel/process.c
407 */
408 .align 7
409 _GLOBAL(_switch)
410 mflr r0
411 std r0,16(r1)
412 stdu r1,-SWITCH_FRAME_SIZE(r1)
413 /* r3-r13 are caller saved -- Cort */
414 SAVE_8GPRS(14, r1)
415 SAVE_10GPRS(22, r1)
416 mflr r20 /* Return to switch caller */
417 mfmsr r22
418 li r0, MSR_FP
419 #ifdef CONFIG_VSX
420 BEGIN_FTR_SECTION
421 oris r0,r0,MSR_VSX@h /* Disable VSX */
422 END_FTR_SECTION_IFSET(CPU_FTR_VSX)
423 #endif /* CONFIG_VSX */
424 #ifdef CONFIG_ALTIVEC
425 BEGIN_FTR_SECTION
426 oris r0,r0,MSR_VEC@h /* Disable altivec */
427 mfspr r24,SPRN_VRSAVE /* save vrsave register value */
428 std r24,THREAD_VRSAVE(r3)
429 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
430 #endif /* CONFIG_ALTIVEC */
431 and. r0,r0,r22
432 beq+ 1f
433 andc r22,r22,r0
434 MTMSRD(r22)
435 isync
436 1: std r20,_NIP(r1)
437 mfcr r23
438 std r23,_CCR(r1)
439 std r1,KSP(r3) /* Set old stack pointer */
441 #ifdef CONFIG_PPC_BOOK3S_64
442 BEGIN_FTR_SECTION
443 /* Event based branch registers */
444 mfspr r0, SPRN_BESCR
445 std r0, THREAD_BESCR(r3)
446 mfspr r0, SPRN_EBBHR
447 std r0, THREAD_EBBHR(r3)
448 mfspr r0, SPRN_EBBRR
449 std r0, THREAD_EBBRR(r3)
450 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
451 #endif
453 #ifdef CONFIG_SMP
454 /* We need a sync somewhere here to make sure that if the
455 * previous task gets rescheduled on another CPU, it sees all
456 * stores it has performed on this one.
457 */
458 sync
459 #endif /* CONFIG_SMP */
461 /*
462 * If we optimise away the clear of the reservation in system
463 * calls because we know the CPU tracks the address of the
464 * reservation, then we need to clear it here to cover the
465 * case that the kernel context switch path has no larx
466 * instructions.
467 */
468 BEGIN_FTR_SECTION
469 ldarx r6,0,r1
470 END_FTR_SECTION_IFSET(CPU_FTR_STCX_CHECKS_ADDRESS)
472 #ifdef CONFIG_PPC_BOOK3S
473 /* Cancel all explict user streams as they will have no use after context
474 * switch and will stop the HW from creating streams itself
475 */
476 DCBT_STOP_ALL_STREAM_IDS(r6)
477 #endif
479 addi r6,r4,-THREAD /* Convert THREAD to 'current' */
480 std r6,PACACURRENT(r13) /* Set new 'current' */
482 ld r8,KSP(r4) /* new stack pointer */
483 #ifdef CONFIG_PPC_BOOK3S
484 BEGIN_FTR_SECTION
485 BEGIN_FTR_SECTION_NESTED(95)
486 clrrdi r6,r8,28 /* get its ESID */
487 clrrdi r9,r1,28 /* get current sp ESID */
488 FTR_SECTION_ELSE_NESTED(95)
489 clrrdi r6,r8,40 /* get its 1T ESID */
490 clrrdi r9,r1,40 /* get current sp 1T ESID */
491 ALT_MMU_FTR_SECTION_END_NESTED_IFCLR(MMU_FTR_1T_SEGMENT, 95)
492 FTR_SECTION_ELSE
493 b 2f
494 ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_SLB)
495 clrldi. r0,r6,2 /* is new ESID c00000000? */
496 cmpd cr1,r6,r9 /* or is new ESID the same as current ESID? */
497 cror eq,4*cr1+eq,eq
498 beq 2f /* if yes, don't slbie it */
500 /* Bolt in the new stack SLB entry */
501 ld r7,KSP_VSID(r4) /* Get new stack's VSID */
502 oris r0,r6,(SLB_ESID_V)@h
503 ori r0,r0,(SLB_NUM_BOLTED-1)@l
504 BEGIN_FTR_SECTION
505 li r9,MMU_SEGSIZE_1T /* insert B field */
506 oris r6,r6,(MMU_SEGSIZE_1T << SLBIE_SSIZE_SHIFT)@h
507 rldimi r7,r9,SLB_VSID_SSIZE_SHIFT,0
508 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
510 /* Update the last bolted SLB. No write barriers are needed
511 * here, provided we only update the current CPU's SLB shadow
512 * buffer.
513 */
514 ld r9,PACA_SLBSHADOWPTR(r13)
515 li r12,0
516 std r12,SLBSHADOW_STACKESID(r9) /* Clear ESID */
517 li r12,SLBSHADOW_STACKVSID
518 STDX_BE r7,r12,r9 /* Save VSID */
519 li r12,SLBSHADOW_STACKESID
520 STDX_BE r0,r12,r9 /* Save ESID */
522 /* No need to check for MMU_FTR_NO_SLBIE_B here, since when
523 * we have 1TB segments, the only CPUs known to have the errata
524 * only support less than 1TB of system memory and we'll never
525 * actually hit this code path.
526 */
528 slbie r6
529 slbie r6 /* Workaround POWER5 < DD2.1 issue */
530 slbmte r7,r0
531 isync
532 2:
533 #endif /* !CONFIG_PPC_BOOK3S */
535 CURRENT_THREAD_INFO(r7, r8) /* base of new stack */
536 /* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE
537 because we don't need to leave the 288-byte ABI gap at the
538 top of the kernel stack. */
539 addi r7,r7,THREAD_SIZE-SWITCH_FRAME_SIZE
541 mr r1,r8 /* start using new stack pointer */
542 std r7,PACAKSAVE(r13)
544 #ifdef CONFIG_PPC_BOOK3S_64
545 BEGIN_FTR_SECTION
546 /* Event based branch registers */
547 ld r0, THREAD_BESCR(r4)
548 mtspr SPRN_BESCR, r0
549 ld r0, THREAD_EBBHR(r4)
550 mtspr SPRN_EBBHR, r0
551 ld r0, THREAD_EBBRR(r4)
552 mtspr SPRN_EBBRR, r0
554 ld r0,THREAD_TAR(r4)
555 mtspr SPRN_TAR,r0
556 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
557 #endif
559 #ifdef CONFIG_ALTIVEC
560 BEGIN_FTR_SECTION
561 ld r0,THREAD_VRSAVE(r4)
562 mtspr SPRN_VRSAVE,r0 /* if G4, restore VRSAVE reg */
563 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
564 #endif /* CONFIG_ALTIVEC */
565 #ifdef CONFIG_PPC64
566 BEGIN_FTR_SECTION
567 lwz r6,THREAD_DSCR_INHERIT(r4)
568 ld r0,THREAD_DSCR(r4)
569 cmpwi r6,0
570 bne 1f
571 ld r0,PACA_DSCR(r13)
572 1:
573 BEGIN_FTR_SECTION_NESTED(70)
574 mfspr r8, SPRN_FSCR
575 rldimi r8, r6, FSCR_DSCR_LG, (63 - FSCR_DSCR_LG)
576 mtspr SPRN_FSCR, r8
577 END_FTR_SECTION_NESTED(CPU_FTR_ARCH_207S, CPU_FTR_ARCH_207S, 70)
578 cmpd r0,r25
579 beq 2f
580 mtspr SPRN_DSCR,r0
581 2:
582 END_FTR_SECTION_IFSET(CPU_FTR_DSCR)
583 #endif
585 ld r6,_CCR(r1)
586 mtcrf 0xFF,r6
588 /* r3-r13 are destroyed -- Cort */
589 REST_8GPRS(14, r1)
590 REST_10GPRS(22, r1)
592 /* convert old thread to its task_struct for return value */
593 addi r3,r3,-THREAD
594 ld r7,_NIP(r1) /* Return to _switch caller in new task */
595 mtlr r7
596 addi r1,r1,SWITCH_FRAME_SIZE
597 blr
599 .align 7
600 _GLOBAL(ret_from_except)
601 ld r11,_TRAP(r1)
602 andi. r0,r11,1
603 bne ret_from_except_lite
604 REST_NVGPRS(r1)
606 _GLOBAL(ret_from_except_lite)
607 /*
608 * Disable interrupts so that current_thread_info()->flags
609 * can't change between when we test it and when we return
610 * from the interrupt.
611 */
612 #ifdef CONFIG_PPC_BOOK3E
613 wrteei 0
614 #else
615 ld r10,PACAKMSR(r13) /* Get kernel MSR without EE */
616 mtmsrd r10,1 /* Update machine state */
617 #endif /* CONFIG_PPC_BOOK3E */
619 CURRENT_THREAD_INFO(r9, r1)
620 ld r3,_MSR(r1)
621 #ifdef CONFIG_PPC_BOOK3E
622 ld r10,PACACURRENT(r13)
623 #endif /* CONFIG_PPC_BOOK3E */
624 ld r4,TI_FLAGS(r9)
625 andi. r3,r3,MSR_PR
626 beq resume_kernel
627 #ifdef CONFIG_PPC_BOOK3E
628 lwz r3,(THREAD+THREAD_DBCR0)(r10)
629 #endif /* CONFIG_PPC_BOOK3E */
631 /* Check current_thread_info()->flags */
632 andi. r0,r4,_TIF_USER_WORK_MASK
633 #ifdef CONFIG_PPC_BOOK3E
634 bne 1f
635 /*
636 * Check to see if the dbcr0 register is set up to debug.
637 * Use the internal debug mode bit to do this.
638 */
639 andis. r0,r3,DBCR0_IDM@h
640 beq restore
641 mfmsr r0
642 rlwinm r0,r0,0,~MSR_DE /* Clear MSR.DE */
643 mtmsr r0
644 mtspr SPRN_DBCR0,r3
645 li r10, -1
646 mtspr SPRN_DBSR,r10
647 b restore
648 #else
649 beq restore
650 #endif
651 1: andi. r0,r4,_TIF_NEED_RESCHED
652 beq 2f
653 bl restore_interrupts
654 SCHEDULE_USER
655 b ret_from_except_lite
656 2:
657 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
658 andi. r0,r4,_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM
659 bne 3f /* only restore TM if nothing else to do */
660 addi r3,r1,STACK_FRAME_OVERHEAD
661 bl restore_tm_state
662 b restore
663 3:
664 #endif
665 bl save_nvgprs
666 bl restore_interrupts
667 addi r3,r1,STACK_FRAME_OVERHEAD
668 bl do_notify_resume
669 b ret_from_except
671 resume_kernel:
672 /* check current_thread_info, _TIF_EMULATE_STACK_STORE */
673 andis. r8,r4,_TIF_EMULATE_STACK_STORE@h
674 beq+ 1f
676 addi r8,r1,INT_FRAME_SIZE /* Get the kprobed function entry */
678 lwz r3,GPR1(r1)
679 subi r3,r3,INT_FRAME_SIZE /* dst: Allocate a trampoline exception frame */
680 mr r4,r1 /* src: current exception frame */
681 mr r1,r3 /* Reroute the trampoline frame to r1 */
683 /* Copy from the original to the trampoline. */
684 li r5,INT_FRAME_SIZE/8 /* size: INT_FRAME_SIZE */
685 li r6,0 /* start offset: 0 */
686 mtctr r5
687 2: ldx r0,r6,r4
688 stdx r0,r6,r3
689 addi r6,r6,8
690 bdnz 2b
692 /* Do real store operation to complete stwu */
693 lwz r5,GPR1(r1)
694 std r8,0(r5)
696 /* Clear _TIF_EMULATE_STACK_STORE flag */
697 lis r11,_TIF_EMULATE_STACK_STORE@h
698 addi r5,r9,TI_FLAGS
699 0: ldarx r4,0,r5
700 andc r4,r4,r11
701 stdcx. r4,0,r5
702 bne- 0b
703 1:
705 #ifdef CONFIG_PREEMPT
706 /* Check if we need to preempt */
707 andi. r0,r4,_TIF_NEED_RESCHED
708 beq+ restore
709 /* Check that preempt_count() == 0 and interrupts are enabled */
710 lwz r8,TI_PREEMPT(r9)
711 cmpwi cr1,r8,0
712 ld r0,SOFTE(r1)
713 cmpdi r0,0
714 crandc eq,cr1*4+eq,eq
715 bne restore
717 /*
718 * Here we are preempting the current task. We want to make
719 * sure we are soft-disabled first and reconcile irq state.
720 */
721 RECONCILE_IRQ_STATE(r3,r4)
722 1: bl preempt_schedule_irq
724 /* Re-test flags and eventually loop */
725 CURRENT_THREAD_INFO(r9, r1)
726 ld r4,TI_FLAGS(r9)
727 andi. r0,r4,_TIF_NEED_RESCHED
728 bne 1b
730 /*
731 * arch_local_irq_restore() from preempt_schedule_irq above may
732 * enable hard interrupt but we really should disable interrupts
733 * when we return from the interrupt, and so that we don't get
734 * interrupted after loading SRR0/1.
735 */
736 #ifdef CONFIG_PPC_BOOK3E
737 wrteei 0
738 #else
739 ld r10,PACAKMSR(r13) /* Get kernel MSR without EE */
740 mtmsrd r10,1 /* Update machine state */
741 #endif /* CONFIG_PPC_BOOK3E */
742 #endif /* CONFIG_PREEMPT */
744 .globl fast_exc_return_irq
745 fast_exc_return_irq:
746 restore:
747 /*
748 * This is the main kernel exit path. First we check if we
749 * are about to re-enable interrupts
750 */
751 ld r5,SOFTE(r1)
752 lbz r6,PACASOFTIRQEN(r13)
753 cmpwi cr0,r5,0
754 beq restore_irq_off
756 /* We are enabling, were we already enabled ? Yes, just return */
757 cmpwi cr0,r6,1
758 beq cr0,do_restore
760 /*
761 * We are about to soft-enable interrupts (we are hard disabled
762 * at this point). We check if there's anything that needs to
763 * be replayed first.
764 */
765 lbz r0,PACAIRQHAPPENED(r13)
766 cmpwi cr0,r0,0
767 bne- restore_check_irq_replay
769 /*
770 * Get here when nothing happened while soft-disabled, just
771 * soft-enable and move-on. We will hard-enable as a side
772 * effect of rfi
773 */
774 restore_no_replay:
775 TRACE_ENABLE_INTS
776 li r0,1
777 stb r0,PACASOFTIRQEN(r13);
779 /*
780 * Final return path. BookE is handled in a different file
781 */
782 do_restore:
783 #ifdef CONFIG_PPC_BOOK3E
784 b exception_return_book3e
785 #else
786 /*
787 * Clear the reservation. If we know the CPU tracks the address of
788 * the reservation then we can potentially save some cycles and use
789 * a larx. On POWER6 and POWER7 this is significantly faster.
790 */
791 BEGIN_FTR_SECTION
792 stdcx. r0,0,r1 /* to clear the reservation */
793 FTR_SECTION_ELSE
794 ldarx r4,0,r1
795 ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
797 /*
798 * Some code path such as load_up_fpu or altivec return directly
799 * here. They run entirely hard disabled and do not alter the
800 * interrupt state. They also don't use lwarx/stwcx. and thus
801 * are known not to leave dangling reservations.
802 */
803 .globl fast_exception_return
804 fast_exception_return:
805 ld r3,_MSR(r1)
806 ld r4,_CTR(r1)
807 ld r0,_LINK(r1)
808 mtctr r4
809 mtlr r0
810 ld r4,_XER(r1)
811 mtspr SPRN_XER,r4
813 REST_8GPRS(5, r1)
815 andi. r0,r3,MSR_RI
816 beq- unrecov_restore
818 /* Load PPR from thread struct before we clear MSR:RI */
819 BEGIN_FTR_SECTION
820 ld r2,PACACURRENT(r13)
821 ld r2,TASKTHREADPPR(r2)
822 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
824 /*
825 * Clear RI before restoring r13. If we are returning to
826 * userspace and we take an exception after restoring r13,
827 * we end up corrupting the userspace r13 value.
828 */
829 ld r4,PACAKMSR(r13) /* Get kernel MSR without EE */
830 andc r4,r4,r0 /* r0 contains MSR_RI here */
831 mtmsrd r4,1
833 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
834 /* TM debug */
835 std r3, PACATMSCRATCH(r13) /* Stash returned-to MSR */
836 #endif
837 /*
838 * r13 is our per cpu area, only restore it if we are returning to
839 * userspace the value stored in the stack frame may belong to
840 * another CPU.
841 */
842 andi. r0,r3,MSR_PR
843 beq 1f
844 BEGIN_FTR_SECTION
845 mtspr SPRN_PPR,r2 /* Restore PPR */
846 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
847 ACCOUNT_CPU_USER_EXIT(r2, r4)
848 REST_GPR(13, r1)
849 1:
850 mtspr SPRN_SRR1,r3
852 ld r2,_CCR(r1)
853 mtcrf 0xFF,r2
854 ld r2,_NIP(r1)
855 mtspr SPRN_SRR0,r2
857 ld r0,GPR0(r1)
858 ld r2,GPR2(r1)
859 ld r3,GPR3(r1)
860 ld r4,GPR4(r1)
861 ld r1,GPR1(r1)
863 rfid
864 b . /* prevent speculative execution */
866 #endif /* CONFIG_PPC_BOOK3E */
868 /*
869 * We are returning to a context with interrupts soft disabled.
870 *
871 * However, we may also about to hard enable, so we need to
872 * make sure that in this case, we also clear PACA_IRQ_HARD_DIS
873 * or that bit can get out of sync and bad things will happen
874 */
875 restore_irq_off:
876 ld r3,_MSR(r1)
877 lbz r7,PACAIRQHAPPENED(r13)
878 andi. r0,r3,MSR_EE
879 beq 1f
880 rlwinm r7,r7,0,~PACA_IRQ_HARD_DIS
881 stb r7,PACAIRQHAPPENED(r13)
882 1: li r0,0
883 stb r0,PACASOFTIRQEN(r13);
884 TRACE_DISABLE_INTS
885 b do_restore
887 /*
888 * Something did happen, check if a re-emit is needed
889 * (this also clears paca->irq_happened)
890 */
891 restore_check_irq_replay:
892 /* XXX: We could implement a fast path here where we check
893 * for irq_happened being just 0x01, in which case we can
894 * clear it and return. That means that we would potentially
895 * miss a decrementer having wrapped all the way around.
896 *
897 * Still, this might be useful for things like hash_page
898 */
899 bl __check_irq_replay
900 cmpwi cr0,r3,0
901 beq restore_no_replay
903 /*
904 * We need to re-emit an interrupt. We do so by re-using our
905 * existing exception frame. We first change the trap value,
906 * but we need to ensure we preserve the low nibble of it
907 */
908 ld r4,_TRAP(r1)
909 clrldi r4,r4,60
910 or r4,r4,r3
911 std r4,_TRAP(r1)
913 /*
914 * Then find the right handler and call it. Interrupts are
915 * still soft-disabled and we keep them that way.
916 */
917 cmpwi cr0,r3,0x500
918 bne 1f
919 addi r3,r1,STACK_FRAME_OVERHEAD;
920 bl do_IRQ
921 b ret_from_except
922 1: cmpwi cr0,r3,0x900
923 bne 1f
924 addi r3,r1,STACK_FRAME_OVERHEAD;
925 bl timer_interrupt
926 b ret_from_except
927 #ifdef CONFIG_PPC_DOORBELL
928 1:
929 #ifdef CONFIG_PPC_BOOK3E
930 cmpwi cr0,r3,0x280
931 #else
932 BEGIN_FTR_SECTION
933 cmpwi cr0,r3,0xe80
934 FTR_SECTION_ELSE
935 cmpwi cr0,r3,0xa00
936 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
937 #endif /* CONFIG_PPC_BOOK3E */
938 bne 1f
939 addi r3,r1,STACK_FRAME_OVERHEAD;
940 bl doorbell_exception
941 b ret_from_except
942 #endif /* CONFIG_PPC_DOORBELL */
943 1: b ret_from_except /* What else to do here ? */
945 unrecov_restore:
946 addi r3,r1,STACK_FRAME_OVERHEAD
947 bl unrecoverable_exception
948 b unrecov_restore
950 #ifdef CONFIG_PPC_RTAS
951 /*
952 * On CHRP, the Run-Time Abstraction Services (RTAS) have to be
953 * called with the MMU off.
954 *
955 * In addition, we need to be in 32b mode, at least for now.
956 *
957 * Note: r3 is an input parameter to rtas, so don't trash it...
958 */
959 _GLOBAL(enter_rtas)
960 mflr r0
961 std r0,16(r1)
962 stdu r1,-RTAS_FRAME_SIZE(r1) /* Save SP and create stack space. */
964 /* Because RTAS is running in 32b mode, it clobbers the high order half
965 * of all registers that it saves. We therefore save those registers
966 * RTAS might touch to the stack. (r0, r3-r13 are caller saved)
967 */
968 SAVE_GPR(2, r1) /* Save the TOC */
969 SAVE_GPR(13, r1) /* Save paca */
970 SAVE_8GPRS(14, r1) /* Save the non-volatiles */
971 SAVE_10GPRS(22, r1) /* ditto */
973 mfcr r4
974 std r4,_CCR(r1)
975 mfctr r5
976 std r5,_CTR(r1)
977 mfspr r6,SPRN_XER
978 std r6,_XER(r1)
979 mfdar r7
980 std r7,_DAR(r1)
981 mfdsisr r8
982 std r8,_DSISR(r1)
984 /* Temporary workaround to clear CR until RTAS can be modified to
985 * ignore all bits.
986 */
987 li r0,0
988 mtcr r0
990 #ifdef CONFIG_BUG
991 /* There is no way it is acceptable to get here with interrupts enabled,
992 * check it with the asm equivalent of WARN_ON
993 */
994 lbz r0,PACASOFTIRQEN(r13)
995 1: tdnei r0,0
996 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
997 #endif
999 /* Hard-disable interrupts */
1000 mfmsr r6
1001 rldicl r7,r6,48,1
1002 rotldi r7,r7,16
1003 mtmsrd r7,1
1005 /* Unfortunately, the stack pointer and the MSR are also clobbered,
1006 * so they are saved in the PACA which allows us to restore
1007 * our original state after RTAS returns.
1008 */
1009 std r1,PACAR1(r13)
1010 std r6,PACASAVEDMSR(r13)
1012 /* Setup our real return addr */
1013 LOAD_REG_ADDR(r4,rtas_return_loc)
1014 clrldi r4,r4,2 /* convert to realmode address */
1015 mtlr r4
1017 li r0,0
1018 ori r0,r0,MSR_EE|MSR_SE|MSR_BE|MSR_RI
1019 andc r0,r6,r0
1021 li r9,1
1022 rldicr r9,r9,MSR_SF_LG,(63-MSR_SF_LG)
1023 ori r9,r9,MSR_IR|MSR_DR|MSR_FE0|MSR_FE1|MSR_FP|MSR_RI|MSR_LE
1024 andc r6,r0,r9
1025 sync /* disable interrupts so SRR0/1 */
1026 mtmsrd r0 /* don't get trashed */
1028 LOAD_REG_ADDR(r4, rtas)
1029 ld r5,RTASENTRY(r4) /* get the rtas->entry value */
1030 ld r4,RTASBASE(r4) /* get the rtas->base value */
1032 mtspr SPRN_SRR0,r5
1033 mtspr SPRN_SRR1,r6
1034 rfid
1035 b . /* prevent speculative execution */
1037 rtas_return_loc:
1038 FIXUP_ENDIAN
1040 /* relocation is off at this point */
1041 GET_PACA(r4)
1042 clrldi r4,r4,2 /* convert to realmode address */
1044 bcl 20,31,$+4
1045 0: mflr r3
1046 ld r3,(1f-0b)(r3) /* get &rtas_restore_regs */
1048 mfmsr r6
1049 li r0,MSR_RI
1050 andc r6,r6,r0
1051 sync
1052 mtmsrd r6
1054 ld r1,PACAR1(r4) /* Restore our SP */
1055 ld r4,PACASAVEDMSR(r4) /* Restore our MSR */
1057 mtspr SPRN_SRR0,r3
1058 mtspr SPRN_SRR1,r4
1059 rfid
1060 b . /* prevent speculative execution */
1062 .align 3
1063 1: .llong rtas_restore_regs
1065 rtas_restore_regs:
1066 /* relocation is on at this point */
1067 REST_GPR(2, r1) /* Restore the TOC */
1068 REST_GPR(13, r1) /* Restore paca */
1069 REST_8GPRS(14, r1) /* Restore the non-volatiles */
1070 REST_10GPRS(22, r1) /* ditto */
1072 GET_PACA(r13)
1074 ld r4,_CCR(r1)
1075 mtcr r4
1076 ld r5,_CTR(r1)
1077 mtctr r5
1078 ld r6,_XER(r1)
1079 mtspr SPRN_XER,r6
1080 ld r7,_DAR(r1)
1081 mtdar r7
1082 ld r8,_DSISR(r1)
1083 mtdsisr r8
1085 addi r1,r1,RTAS_FRAME_SIZE /* Unstack our frame */
1086 ld r0,16(r1) /* get return address */
1088 mtlr r0
1089 blr /* return to caller */
1091 #endif /* CONFIG_PPC_RTAS */
1093 _GLOBAL(enter_prom)
1094 mflr r0
1095 std r0,16(r1)
1096 stdu r1,-PROM_FRAME_SIZE(r1) /* Save SP and create stack space */
1098 /* Because PROM is running in 32b mode, it clobbers the high order half
1099 * of all registers that it saves. We therefore save those registers
1100 * PROM might touch to the stack. (r0, r3-r13 are caller saved)
1101 */
1102 SAVE_GPR(2, r1)
1103 SAVE_GPR(13, r1)
1104 SAVE_8GPRS(14, r1)
1105 SAVE_10GPRS(22, r1)
1106 mfcr r10
1107 mfmsr r11
1108 std r10,_CCR(r1)
1109 std r11,_MSR(r1)
1111 /* Put PROM address in SRR0 */
1112 mtsrr0 r4
1114 /* Setup our trampoline return addr in LR */
1115 bcl 20,31,$+4
1116 0: mflr r4
1117 addi r4,r4,(1f - 0b)
1118 mtlr r4
1120 /* Prepare a 32-bit mode big endian MSR
1121 */
1122 #ifdef CONFIG_PPC_BOOK3E
1123 rlwinm r11,r11,0,1,31
1124 mtsrr1 r11
1125 rfi
1126 #else /* CONFIG_PPC_BOOK3E */
1127 LOAD_REG_IMMEDIATE(r12, MSR_SF | MSR_ISF | MSR_LE)
1128 andc r11,r11,r12
1129 mtsrr1 r11
1130 rfid
1131 #endif /* CONFIG_PPC_BOOK3E */
1133 1: /* Return from OF */
1134 FIXUP_ENDIAN
1136 /* Just make sure that r1 top 32 bits didn't get
1137 * corrupt by OF
1138 */
1139 rldicl r1,r1,0,32
1141 /* Restore the MSR (back to 64 bits) */
1142 ld r0,_MSR(r1)
1143 MTMSRD(r0)
1144 isync
1146 /* Restore other registers */
1147 REST_GPR(2, r1)
1148 REST_GPR(13, r1)
1149 REST_8GPRS(14, r1)
1150 REST_10GPRS(22, r1)
1151 ld r4,_CCR(r1)
1152 mtcr r4
1154 addi r1,r1,PROM_FRAME_SIZE
1155 ld r0,16(r1)
1156 mtlr r0
1157 blr
1159 #ifdef CONFIG_FUNCTION_TRACER
1160 #ifdef CONFIG_DYNAMIC_FTRACE
1161 _GLOBAL(mcount)
1162 _GLOBAL(_mcount)
1163 blr
1165 _GLOBAL_TOC(ftrace_caller)
1166 /* Taken from output of objdump from lib64/glibc */
1167 mflr r3
1168 ld r11, 0(r1)
1169 stdu r1, -112(r1)
1170 std r3, 128(r1)
1171 ld r4, 16(r11)
1172 subi r3, r3, MCOUNT_INSN_SIZE
1173 .globl ftrace_call
1174 ftrace_call:
1175 bl ftrace_stub
1176 nop
1177 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1178 .globl ftrace_graph_call
1179 ftrace_graph_call:
1180 b ftrace_graph_stub
1181 _GLOBAL(ftrace_graph_stub)
1182 #endif
1183 ld r0, 128(r1)
1184 mtlr r0
1185 addi r1, r1, 112
1186 _GLOBAL(ftrace_stub)
1187 blr
1188 #else
1189 _GLOBAL_TOC(_mcount)
1190 /* Taken from output of objdump from lib64/glibc */
1191 mflr r3
1192 ld r11, 0(r1)
1193 stdu r1, -112(r1)
1194 std r3, 128(r1)
1195 ld r4, 16(r11)
1197 subi r3, r3, MCOUNT_INSN_SIZE
1198 LOAD_REG_ADDR(r5,ftrace_trace_function)
1199 ld r5,0(r5)
1200 ld r5,0(r5)
1201 mtctr r5
1202 bctrl
1203 nop
1206 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1207 b ftrace_graph_caller
1208 #endif
1209 ld r0, 128(r1)
1210 mtlr r0
1211 addi r1, r1, 112
1212 _GLOBAL(ftrace_stub)
1213 blr
1215 #endif /* CONFIG_DYNAMIC_FTRACE */
1217 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1218 _GLOBAL(ftrace_graph_caller)
1219 /* load r4 with local address */
1220 ld r4, 128(r1)
1221 subi r4, r4, MCOUNT_INSN_SIZE
1223 /* get the parent address */
1224 ld r11, 112(r1)
1225 addi r3, r11, 16
1227 bl prepare_ftrace_return
1228 nop
1230 ld r0, 128(r1)
1231 mtlr r0
1232 addi r1, r1, 112
1233 blr
1235 _GLOBAL(return_to_handler)
1236 /* need to save return values */
1237 std r4, -24(r1)
1238 std r3, -16(r1)
1239 std r31, -8(r1)
1240 mr r31, r1
1241 stdu r1, -112(r1)
1243 bl ftrace_return_to_handler
1244 nop
1246 /* return value has real return address */
1247 mtlr r3
1249 ld r1, 0(r1)
1250 ld r4, -24(r1)
1251 ld r3, -16(r1)
1252 ld r31, -8(r1)
1254 /* Jump back to real return address */
1255 blr
1257 _GLOBAL(mod_return_to_handler)
1258 /* need to save return values */
1259 std r4, -32(r1)
1260 std r3, -24(r1)
1261 /* save TOC */
1262 std r2, -16(r1)
1263 std r31, -8(r1)
1264 mr r31, r1
1265 stdu r1, -112(r1)
1267 /*
1268 * We are in a module using the module's TOC.
1269 * Switch to our TOC to run inside the core kernel.
1270 */
1271 ld r2, PACATOC(r13)
1273 bl ftrace_return_to_handler
1274 nop
1276 /* return value has real return address */
1277 mtlr r3
1279 ld r1, 0(r1)
1280 ld r4, -32(r1)
1281 ld r3, -24(r1)
1282 ld r2, -16(r1)
1283 ld r31, -8(r1)
1285 /* Jump back to real return address */
1286 blr
1287 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1288 #endif /* CONFIG_FUNCTION_TRACER */