| blob | 246b11c4fe7eda803a4c076c078c5afdcff04590 |
1 /*
2 * PowerPC version
3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 * Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
5 * Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
6 * Adapted for Power Macintosh by Paul Mackerras.
7 * Low-level exception handlers and MMU support
8 * rewritten by Paul Mackerras.
9 * Copyright (C) 1996 Paul Mackerras.
10 * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
11 *
12 * This file contains the system call entry code, context switch
13 * code, and exception/interrupt return code for PowerPC.
14 *
15 * This program is free software; you can redistribute it and/or
16 * modify it under the terms of the GNU General Public License
17 * as published by the Free Software Foundation; either version
18 * 2 of the License, or (at your option) any later version.
19 */
21 #include <linux/errno.h>
22 #include <asm/unistd.h>
23 #include <asm/processor.h>
24 #include <asm/page.h>
25 #include <asm/mmu.h>
26 #include <asm/thread_info.h>
27 #include <asm/ppc_asm.h>
28 #include <asm/asm-offsets.h>
29 #include <asm/cputable.h>
30 #include <asm/firmware.h>
31 #include <asm/bug.h>
32 #include <asm/ptrace.h>
33 #include <asm/irqflags.h>
34 #include <asm/ftrace.h>
35 #include <asm/hw_irq.h>
36 #include <asm/context_tracking.h>
38 /*
39 * System calls.
40 */
41 .section ".toc","aw"
42 .SYS_CALL_TABLE:
43 .tc .sys_call_table[TC],.sys_call_table
45 /* This value is used to mark exception frames on the stack. */
46 exception_marker:
47 .tc ID_EXC_MARKER[TC],STACK_FRAME_REGS_MARKER
49 .section ".text"
50 .align 7
52 #undef SHOW_SYSCALLS
54 .globl system_call_common
55 system_call_common:
56 andi. r10,r12,MSR_PR
57 mr r10,r1
58 addi r1,r1,-INT_FRAME_SIZE
59 beq- 1f
60 ld r1,PACAKSAVE(r13)
61 1: std r10,0(r1)
62 std r11,_NIP(r1)
63 std r12,_MSR(r1)
64 std r0,GPR0(r1)
65 std r10,GPR1(r1)
66 beq 2f /* if from kernel mode */
67 ACCOUNT_CPU_USER_ENTRY(r10, r11)
68 2: std r2,GPR2(r1)
69 std r3,GPR3(r1)
70 mfcr r2
71 std r4,GPR4(r1)
72 std r5,GPR5(r1)
73 std r6,GPR6(r1)
74 std r7,GPR7(r1)
75 std r8,GPR8(r1)
76 li r11,0
77 std r11,GPR9(r1)
78 std r11,GPR10(r1)
79 std r11,GPR11(r1)
80 std r11,GPR12(r1)
81 std r11,_XER(r1)
82 std r11,_CTR(r1)
83 std r9,GPR13(r1)
84 mflr r10
85 /*
86 * This clears CR0.SO (bit 28), which is the error indication on
87 * return from this system call.
88 */
89 rldimi r2,r11,28,(63-28)
90 li r11,0xc01
91 std r10,_LINK(r1)
92 std r11,_TRAP(r1)
93 std r3,ORIG_GPR3(r1)
94 std r2,_CCR(r1)
95 ld r2,PACATOC(r13)
96 addi r9,r1,STACK_FRAME_OVERHEAD
97 ld r11,exception_marker@toc(r2)
98 std r11,-16(r9) /* "regshere" marker */
99 #if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) && defined(CONFIG_PPC_SPLPAR)
100 BEGIN_FW_FTR_SECTION
101 beq 33f
102 /* if from user, see if there are any DTL entries to process */
103 ld r10,PACALPPACAPTR(r13) /* get ptr to VPA */
104 ld r11,PACA_DTL_RIDX(r13) /* get log read index */
105 ld r10,LPPACA_DTLIDX(r10) /* get log write index */
106 cmpd cr1,r11,r10
107 beq+ cr1,33f
108 bl .accumulate_stolen_time
109 REST_GPR(0,r1)
110 REST_4GPRS(3,r1)
111 REST_2GPRS(7,r1)
112 addi r9,r1,STACK_FRAME_OVERHEAD
113 33:
114 END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR)
115 #endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE && CONFIG_PPC_SPLPAR */
117 /*
118 * A syscall should always be called with interrupts enabled
119 * so we just unconditionally hard-enable here. When some kind
120 * of irq tracing is used, we additionally check that condition
121 * is correct
122 */
123 #if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_BUG)
124 lbz r10,PACASOFTIRQEN(r13)
125 xori r10,r10,1
126 1: tdnei r10,0
127 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
128 #endif
130 #ifdef CONFIG_PPC_BOOK3E
131 wrteei 1
132 #else
133 ld r11,PACAKMSR(r13)
134 ori r11,r11,MSR_EE
135 mtmsrd r11,1
136 #endif /* CONFIG_PPC_BOOK3E */
138 /* We do need to set SOFTE in the stack frame or the return
139 * from interrupt will be painful
140 */
141 li r10,1
142 std r10,SOFTE(r1)
144 #ifdef SHOW_SYSCALLS
145 bl .do_show_syscall
146 REST_GPR(0,r1)
147 REST_4GPRS(3,r1)
148 REST_2GPRS(7,r1)
149 addi r9,r1,STACK_FRAME_OVERHEAD
150 #endif
151 CURRENT_THREAD_INFO(r11, r1)
152 ld r10,TI_FLAGS(r11)
153 andi. r11,r10,_TIF_SYSCALL_T_OR_A
154 bne syscall_dotrace
155 .Lsyscall_dotrace_cont:
156 cmpldi 0,r0,NR_syscalls
157 bge- syscall_enosys
159 system_call: /* label this so stack traces look sane */
160 /*
161 * Need to vector to 32 Bit or default sys_call_table here,
162 * based on caller's run-mode / personality.
163 */
164 ld r11,.SYS_CALL_TABLE@toc(2)
165 andi. r10,r10,_TIF_32BIT
166 beq 15f
167 addi r11,r11,8 /* use 32-bit syscall entries */
168 clrldi r3,r3,32
169 clrldi r4,r4,32
170 clrldi r5,r5,32
171 clrldi r6,r6,32
172 clrldi r7,r7,32
173 clrldi r8,r8,32
174 15:
175 slwi r0,r0,4
176 ldx r10,r11,r0 /* Fetch system call handler [ptr] */
177 mtctr r10
178 bctrl /* Call handler */
180 syscall_exit:
181 std r3,RESULT(r1)
182 #ifdef SHOW_SYSCALLS
183 bl .do_show_syscall_exit
184 ld r3,RESULT(r1)
185 #endif
186 CURRENT_THREAD_INFO(r12, r1)
188 ld r8,_MSR(r1)
189 #ifdef CONFIG_PPC_BOOK3S
190 /* No MSR:RI on BookE */
191 andi. r10,r8,MSR_RI
192 beq- unrecov_restore
193 #endif
194 /*
195 * Disable interrupts so current_thread_info()->flags can't change,
196 * and so that we don't get interrupted after loading SRR0/1.
197 */
198 #ifdef CONFIG_PPC_BOOK3E
199 wrteei 0
200 #else
201 ld r10,PACAKMSR(r13)
202 /*
203 * For performance reasons we clear RI the same time that we
204 * clear EE. We only need to clear RI just before we restore r13
205 * below, but batching it with EE saves us one expensive mtmsrd call.
206 * We have to be careful to restore RI if we branch anywhere from
207 * here (eg syscall_exit_work).
208 */
209 li r9,MSR_RI
210 andc r11,r10,r9
211 mtmsrd r11,1
212 #endif /* CONFIG_PPC_BOOK3E */
214 ld r9,TI_FLAGS(r12)
215 li r11,-_LAST_ERRNO
216 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
217 bne- syscall_exit_work
218 cmpld r3,r11
219 ld r5,_CCR(r1)
220 bge- syscall_error
221 .Lsyscall_error_cont:
222 ld r7,_NIP(r1)
223 BEGIN_FTR_SECTION
224 stdcx. r0,0,r1 /* to clear the reservation */
225 END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
226 andi. r6,r8,MSR_PR
227 ld r4,_LINK(r1)
229 beq- 1f
230 ACCOUNT_CPU_USER_EXIT(r11, r12)
231 HMT_MEDIUM_LOW_HAS_PPR
232 ld r13,GPR13(r1) /* only restore r13 if returning to usermode */
233 1: ld r2,GPR2(r1)
234 ld r1,GPR1(r1)
235 mtlr r4
236 mtcr r5
237 mtspr SPRN_SRR0,r7
238 mtspr SPRN_SRR1,r8
239 RFI
240 b . /* prevent speculative execution */
242 syscall_error:
243 oris r5,r5,0x1000 /* Set SO bit in CR */
244 neg r3,r3
245 std r5,_CCR(r1)
246 b .Lsyscall_error_cont
248 /* Traced system call support */
249 syscall_dotrace:
250 bl .save_nvgprs
251 addi r3,r1,STACK_FRAME_OVERHEAD
252 bl .do_syscall_trace_enter
253 /*
254 * Restore argument registers possibly just changed.
255 * We use the return value of do_syscall_trace_enter
256 * for the call number to look up in the table (r0).
257 */
258 mr r0,r3
259 ld r3,GPR3(r1)
260 ld r4,GPR4(r1)
261 ld r5,GPR5(r1)
262 ld r6,GPR6(r1)
263 ld r7,GPR7(r1)
264 ld r8,GPR8(r1)
265 addi r9,r1,STACK_FRAME_OVERHEAD
266 CURRENT_THREAD_INFO(r10, r1)
267 ld r10,TI_FLAGS(r10)
268 b .Lsyscall_dotrace_cont
270 syscall_enosys:
271 li r3,-ENOSYS
272 b syscall_exit
274 syscall_exit_work:
275 #ifdef CONFIG_PPC_BOOK3S
276 mtmsrd r10,1 /* Restore RI */
277 #endif
278 /* If TIF_RESTOREALL is set, don't scribble on either r3 or ccr.
279 If TIF_NOERROR is set, just save r3 as it is. */
281 andi. r0,r9,_TIF_RESTOREALL
282 beq+ 0f
283 REST_NVGPRS(r1)
284 b 2f
285 0: cmpld r3,r11 /* r10 is -LAST_ERRNO */
286 blt+ 1f
287 andi. r0,r9,_TIF_NOERROR
288 bne- 1f
289 ld r5,_CCR(r1)
290 neg r3,r3
291 oris r5,r5,0x1000 /* Set SO bit in CR */
292 std r5,_CCR(r1)
293 1: std r3,GPR3(r1)
294 2: andi. r0,r9,(_TIF_PERSYSCALL_MASK)
295 beq 4f
297 /* Clear per-syscall TIF flags if any are set. */
299 li r11,_TIF_PERSYSCALL_MASK
300 addi r12,r12,TI_FLAGS
301 3: ldarx r10,0,r12
302 andc r10,r10,r11
303 stdcx. r10,0,r12
304 bne- 3b
305 subi r12,r12,TI_FLAGS
307 4: /* Anything else left to do? */
308 SET_DEFAULT_THREAD_PPR(r3, r10) /* Set thread.ppr = 3 */
309 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP)
310 beq .ret_from_except_lite
312 /* Re-enable interrupts */
313 #ifdef CONFIG_PPC_BOOK3E
314 wrteei 1
315 #else
316 ld r10,PACAKMSR(r13)
317 ori r10,r10,MSR_EE
318 mtmsrd r10,1
319 #endif /* CONFIG_PPC_BOOK3E */
321 bl .save_nvgprs
322 addi r3,r1,STACK_FRAME_OVERHEAD
323 bl .do_syscall_trace_leave
324 b .ret_from_except
326 /* Save non-volatile GPRs, if not already saved. */
327 _GLOBAL(save_nvgprs)
328 ld r11,_TRAP(r1)
329 andi. r0,r11,1
330 beqlr-
331 SAVE_NVGPRS(r1)
332 clrrdi r0,r11,1
333 std r0,_TRAP(r1)
334 blr
337 /*
338 * The sigsuspend and rt_sigsuspend system calls can call do_signal
339 * and thus put the process into the stopped state where we might
340 * want to examine its user state with ptrace. Therefore we need
341 * to save all the nonvolatile registers (r14 - r31) before calling
342 * the C code. Similarly, fork, vfork and clone need the full
343 * register state on the stack so that it can be copied to the child.
344 */
346 _GLOBAL(ppc_fork)
347 bl .save_nvgprs
348 bl .sys_fork
349 b syscall_exit
351 _GLOBAL(ppc_vfork)
352 bl .save_nvgprs
353 bl .sys_vfork
354 b syscall_exit
356 _GLOBAL(ppc_clone)
357 bl .save_nvgprs
358 bl .sys_clone
359 b syscall_exit
361 _GLOBAL(ppc32_swapcontext)
362 bl .save_nvgprs
363 bl .compat_sys_swapcontext
364 b syscall_exit
366 _GLOBAL(ppc64_swapcontext)
367 bl .save_nvgprs
368 bl .sys_swapcontext
369 b syscall_exit
371 _GLOBAL(ret_from_fork)
372 bl .schedule_tail
373 REST_NVGPRS(r1)
374 li r3,0
375 b syscall_exit
377 _GLOBAL(ret_from_kernel_thread)
378 bl .schedule_tail
379 REST_NVGPRS(r1)
380 ld r14, 0(r14)
381 mtlr r14
382 mr r3,r15
383 blrl
384 li r3,0
385 b syscall_exit
387 .section ".toc","aw"
388 DSCR_DEFAULT:
389 .tc dscr_default[TC],dscr_default
391 .section ".text"
393 /*
394 * This routine switches between two different tasks. The process
395 * state of one is saved on its kernel stack. Then the state
396 * of the other is restored from its kernel stack. The memory
397 * management hardware is updated to the second process's state.
398 * Finally, we can return to the second process, via ret_from_except.
399 * On entry, r3 points to the THREAD for the current task, r4
400 * points to the THREAD for the new task.
401 *
402 * Note: there are two ways to get to the "going out" portion
403 * of this code; either by coming in via the entry (_switch)
404 * or via "fork" which must set up an environment equivalent
405 * to the "_switch" path. If you change this you'll have to change
406 * the fork code also.
407 *
408 * The code which creates the new task context is in 'copy_thread'
409 * in arch/powerpc/kernel/process.c
410 */
411 .align 7
412 _GLOBAL(_switch)
413 mflr r0
414 std r0,16(r1)
415 stdu r1,-SWITCH_FRAME_SIZE(r1)
416 /* r3-r13 are caller saved -- Cort */
417 SAVE_8GPRS(14, r1)
418 SAVE_10GPRS(22, r1)
419 mflr r20 /* Return to switch caller */
420 mfmsr r22
421 li r0, MSR_FP
422 #ifdef CONFIG_VSX
423 BEGIN_FTR_SECTION
424 oris r0,r0,MSR_VSX@h /* Disable VSX */
425 END_FTR_SECTION_IFSET(CPU_FTR_VSX)
426 #endif /* CONFIG_VSX */
427 #ifdef CONFIG_ALTIVEC
428 BEGIN_FTR_SECTION
429 oris r0,r0,MSR_VEC@h /* Disable altivec */
430 mfspr r24,SPRN_VRSAVE /* save vrsave register value */
431 std r24,THREAD_VRSAVE(r3)
432 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
433 #endif /* CONFIG_ALTIVEC */
434 #ifdef CONFIG_PPC64
435 BEGIN_FTR_SECTION
436 mfspr r25,SPRN_DSCR
437 std r25,THREAD_DSCR(r3)
438 END_FTR_SECTION_IFSET(CPU_FTR_DSCR)
439 #endif
440 and. r0,r0,r22
441 beq+ 1f
442 andc r22,r22,r0
443 MTMSRD(r22)
444 isync
445 1: std r20,_NIP(r1)
446 mfcr r23
447 std r23,_CCR(r1)
448 std r1,KSP(r3) /* Set old stack pointer */
450 #ifdef CONFIG_PPC_BOOK3S_64
451 BEGIN_FTR_SECTION
452 /*
453 * Back up the TAR across context switches. Note that the TAR is not
454 * available for use in the kernel. (To provide this, the TAR should
455 * be backed up/restored on exception entry/exit instead, and be in
456 * pt_regs. FIXME, this should be in pt_regs anyway (for debug).)
457 */
458 mfspr r0,SPRN_TAR
459 std r0,THREAD_TAR(r3)
461 /* Event based branch registers */
462 mfspr r0, SPRN_BESCR
463 std r0, THREAD_BESCR(r3)
464 mfspr r0, SPRN_EBBHR
465 std r0, THREAD_EBBHR(r3)
466 mfspr r0, SPRN_EBBRR
467 std r0, THREAD_EBBRR(r3)
469 /* PMU registers made user read/(write) by EBB */
470 mfspr r0, SPRN_SIAR
471 std r0, THREAD_SIAR(r3)
472 mfspr r0, SPRN_SDAR
473 std r0, THREAD_SDAR(r3)
474 mfspr r0, SPRN_SIER
475 std r0, THREAD_SIER(r3)
476 mfspr r0, SPRN_MMCR0
477 std r0, THREAD_MMCR0(r3)
478 mfspr r0, SPRN_MMCR2
479 std r0, THREAD_MMCR2(r3)
480 mfspr r0, SPRN_MMCRA
481 std r0, THREAD_MMCRA(r3)
482 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
483 #endif
485 #ifdef CONFIG_SMP
486 /* We need a sync somewhere here to make sure that if the
487 * previous task gets rescheduled on another CPU, it sees all
488 * stores it has performed on this one.
489 */
490 sync
491 #endif /* CONFIG_SMP */
493 /*
494 * If we optimise away the clear of the reservation in system
495 * calls because we know the CPU tracks the address of the
496 * reservation, then we need to clear it here to cover the
497 * case that the kernel context switch path has no larx
498 * instructions.
499 */
500 BEGIN_FTR_SECTION
501 ldarx r6,0,r1
502 END_FTR_SECTION_IFSET(CPU_FTR_STCX_CHECKS_ADDRESS)
504 #ifdef CONFIG_PPC_BOOK3S
505 /* Cancel all explict user streams as they will have no use after context
506 * switch and will stop the HW from creating streams itself
507 */
508 DCBT_STOP_ALL_STREAM_IDS(r6)
509 #endif
511 addi r6,r4,-THREAD /* Convert THREAD to 'current' */
512 std r6,PACACURRENT(r13) /* Set new 'current' */
514 ld r8,KSP(r4) /* new stack pointer */
515 #ifdef CONFIG_PPC_BOOK3S
516 BEGIN_FTR_SECTION
517 BEGIN_FTR_SECTION_NESTED(95)
518 clrrdi r6,r8,28 /* get its ESID */
519 clrrdi r9,r1,28 /* get current sp ESID */
520 FTR_SECTION_ELSE_NESTED(95)
521 clrrdi r6,r8,40 /* get its 1T ESID */
522 clrrdi r9,r1,40 /* get current sp 1T ESID */
523 ALT_MMU_FTR_SECTION_END_NESTED_IFCLR(MMU_FTR_1T_SEGMENT, 95)
524 FTR_SECTION_ELSE
525 b 2f
526 ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_SLB)
527 clrldi. r0,r6,2 /* is new ESID c00000000? */
528 cmpd cr1,r6,r9 /* or is new ESID the same as current ESID? */
529 cror eq,4*cr1+eq,eq
530 beq 2f /* if yes, don't slbie it */
532 /* Bolt in the new stack SLB entry */
533 ld r7,KSP_VSID(r4) /* Get new stack's VSID */
534 oris r0,r6,(SLB_ESID_V)@h
535 ori r0,r0,(SLB_NUM_BOLTED-1)@l
536 BEGIN_FTR_SECTION
537 li r9,MMU_SEGSIZE_1T /* insert B field */
538 oris r6,r6,(MMU_SEGSIZE_1T << SLBIE_SSIZE_SHIFT)@h
539 rldimi r7,r9,SLB_VSID_SSIZE_SHIFT,0
540 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
542 /* Update the last bolted SLB. No write barriers are needed
543 * here, provided we only update the current CPU's SLB shadow
544 * buffer.
545 */
546 ld r9,PACA_SLBSHADOWPTR(r13)
547 li r12,0
548 std r12,SLBSHADOW_STACKESID(r9) /* Clear ESID */
549 std r7,SLBSHADOW_STACKVSID(r9) /* Save VSID */
550 std r0,SLBSHADOW_STACKESID(r9) /* Save ESID */
552 /* No need to check for MMU_FTR_NO_SLBIE_B here, since when
553 * we have 1TB segments, the only CPUs known to have the errata
554 * only support less than 1TB of system memory and we'll never
555 * actually hit this code path.
556 */
558 slbie r6
559 slbie r6 /* Workaround POWER5 < DD2.1 issue */
560 slbmte r7,r0
561 isync
562 2:
563 #endif /* !CONFIG_PPC_BOOK3S */
565 CURRENT_THREAD_INFO(r7, r8) /* base of new stack */
566 /* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE
567 because we don't need to leave the 288-byte ABI gap at the
568 top of the kernel stack. */
569 addi r7,r7,THREAD_SIZE-SWITCH_FRAME_SIZE
571 mr r1,r8 /* start using new stack pointer */
572 std r7,PACAKSAVE(r13)
574 #ifdef CONFIG_PPC_BOOK3S_64
575 BEGIN_FTR_SECTION
576 /* Event based branch registers */
577 ld r0, THREAD_BESCR(r4)
578 mtspr SPRN_BESCR, r0
579 ld r0, THREAD_EBBHR(r4)
580 mtspr SPRN_EBBHR, r0
581 ld r0, THREAD_EBBRR(r4)
582 mtspr SPRN_EBBRR, r0
584 /* PMU registers made user read/(write) by EBB */
585 ld r0, THREAD_SIAR(r4)
586 mtspr SPRN_SIAR, r0
587 ld r0, THREAD_SDAR(r4)
588 mtspr SPRN_SDAR, r0
589 ld r0, THREAD_SIER(r4)
590 mtspr SPRN_SIER, r0
591 ld r0, THREAD_MMCR0(r4)
592 mtspr SPRN_MMCR0, r0
593 ld r0, THREAD_MMCR2(r4)
594 mtspr SPRN_MMCR2, r0
595 ld r0, THREAD_MMCRA(r4)
596 mtspr SPRN_MMCRA, r0
598 ld r0,THREAD_TAR(r4)
599 mtspr SPRN_TAR,r0
600 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
601 #endif
603 #ifdef CONFIG_ALTIVEC
604 BEGIN_FTR_SECTION
605 ld r0,THREAD_VRSAVE(r4)
606 mtspr SPRN_VRSAVE,r0 /* if G4, restore VRSAVE reg */
607 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
608 #endif /* CONFIG_ALTIVEC */
609 #ifdef CONFIG_PPC64
610 BEGIN_FTR_SECTION
611 lwz r6,THREAD_DSCR_INHERIT(r4)
612 ld r7,DSCR_DEFAULT@toc(2)
613 ld r0,THREAD_DSCR(r4)
614 cmpwi r6,0
615 bne 1f
616 ld r0,0(r7)
617 1: cmpd r0,r25
618 beq 2f
619 mtspr SPRN_DSCR,r0
620 2:
621 END_FTR_SECTION_IFSET(CPU_FTR_DSCR)
622 #endif
624 ld r6,_CCR(r1)
625 mtcrf 0xFF,r6
627 /* r3-r13 are destroyed -- Cort */
628 REST_8GPRS(14, r1)
629 REST_10GPRS(22, r1)
631 /* convert old thread to its task_struct for return value */
632 addi r3,r3,-THREAD
633 ld r7,_NIP(r1) /* Return to _switch caller in new task */
634 mtlr r7
635 addi r1,r1,SWITCH_FRAME_SIZE
636 blr
638 .align 7
639 _GLOBAL(ret_from_except)
640 ld r11,_TRAP(r1)
641 andi. r0,r11,1
642 bne .ret_from_except_lite
643 REST_NVGPRS(r1)
645 _GLOBAL(ret_from_except_lite)
646 /*
647 * Disable interrupts so that current_thread_info()->flags
648 * can't change between when we test it and when we return
649 * from the interrupt.
650 */
651 #ifdef CONFIG_PPC_BOOK3E
652 wrteei 0
653 #else
654 ld r10,PACAKMSR(r13) /* Get kernel MSR without EE */
655 mtmsrd r10,1 /* Update machine state */
656 #endif /* CONFIG_PPC_BOOK3E */
658 CURRENT_THREAD_INFO(r9, r1)
659 ld r3,_MSR(r1)
660 ld r4,TI_FLAGS(r9)
661 andi. r3,r3,MSR_PR
662 beq resume_kernel
664 /* Check current_thread_info()->flags */
665 andi. r0,r4,_TIF_USER_WORK_MASK
666 beq restore
668 andi. r0,r4,_TIF_NEED_RESCHED
669 beq 1f
670 bl .restore_interrupts
671 SCHEDULE_USER
672 b .ret_from_except_lite
674 1: bl .save_nvgprs
675 bl .restore_interrupts
676 addi r3,r1,STACK_FRAME_OVERHEAD
677 bl .do_notify_resume
678 b .ret_from_except
680 resume_kernel:
681 /* check current_thread_info, _TIF_EMULATE_STACK_STORE */
682 CURRENT_THREAD_INFO(r9, r1)
683 ld r8,TI_FLAGS(r9)
684 andis. r8,r8,_TIF_EMULATE_STACK_STORE@h
685 beq+ 1f
687 addi r8,r1,INT_FRAME_SIZE /* Get the kprobed function entry */
689 lwz r3,GPR1(r1)
690 subi r3,r3,INT_FRAME_SIZE /* dst: Allocate a trampoline exception frame */
691 mr r4,r1 /* src: current exception frame */
692 mr r1,r3 /* Reroute the trampoline frame to r1 */
694 /* Copy from the original to the trampoline. */
695 li r5,INT_FRAME_SIZE/8 /* size: INT_FRAME_SIZE */
696 li r6,0 /* start offset: 0 */
697 mtctr r5
698 2: ldx r0,r6,r4
699 stdx r0,r6,r3
700 addi r6,r6,8
701 bdnz 2b
703 /* Do real store operation to complete stwu */
704 lwz r5,GPR1(r1)
705 std r8,0(r5)
707 /* Clear _TIF_EMULATE_STACK_STORE flag */
708 lis r11,_TIF_EMULATE_STACK_STORE@h
709 addi r5,r9,TI_FLAGS
710 0: ldarx r4,0,r5
711 andc r4,r4,r11
712 stdcx. r4,0,r5
713 bne- 0b
714 1:
716 #ifdef CONFIG_PREEMPT
717 /* Check if we need to preempt */
718 andi. r0,r4,_TIF_NEED_RESCHED
719 beq+ restore
720 /* Check that preempt_count() == 0 and interrupts are enabled */
721 lwz r8,TI_PREEMPT(r9)
722 cmpwi cr1,r8,0
723 ld r0,SOFTE(r1)
724 cmpdi r0,0
725 crandc eq,cr1*4+eq,eq
726 bne restore
728 /*
729 * Here we are preempting the current task. We want to make
730 * sure we are soft-disabled first
731 */
732 SOFT_DISABLE_INTS(r3,r4)
733 1: bl .preempt_schedule_irq
735 /* Re-test flags and eventually loop */
736 CURRENT_THREAD_INFO(r9, r1)
737 ld r4,TI_FLAGS(r9)
738 andi. r0,r4,_TIF_NEED_RESCHED
739 bne 1b
741 /*
742 * arch_local_irq_restore() from preempt_schedule_irq above may
743 * enable hard interrupt but we really should disable interrupts
744 * when we return from the interrupt, and so that we don't get
745 * interrupted after loading SRR0/1.
746 */
747 #ifdef CONFIG_PPC_BOOK3E
748 wrteei 0
749 #else
750 ld r10,PACAKMSR(r13) /* Get kernel MSR without EE */
751 mtmsrd r10,1 /* Update machine state */
752 #endif /* CONFIG_PPC_BOOK3E */
753 #endif /* CONFIG_PREEMPT */
755 .globl fast_exc_return_irq
756 fast_exc_return_irq:
757 restore:
758 /*
759 * This is the main kernel exit path. First we check if we
760 * are about to re-enable interrupts
761 */
762 ld r5,SOFTE(r1)
763 lbz r6,PACASOFTIRQEN(r13)
764 cmpwi cr0,r5,0
765 beq restore_irq_off
767 /* We are enabling, were we already enabled ? Yes, just return */
768 cmpwi cr0,r6,1
769 beq cr0,do_restore
771 /*
772 * We are about to soft-enable interrupts (we are hard disabled
773 * at this point). We check if there's anything that needs to
774 * be replayed first.
775 */
776 lbz r0,PACAIRQHAPPENED(r13)
777 cmpwi cr0,r0,0
778 bne- restore_check_irq_replay
780 /*
781 * Get here when nothing happened while soft-disabled, just
782 * soft-enable and move-on. We will hard-enable as a side
783 * effect of rfi
784 */
785 restore_no_replay:
786 TRACE_ENABLE_INTS
787 li r0,1
788 stb r0,PACASOFTIRQEN(r13);
790 /*
791 * Final return path. BookE is handled in a different file
792 */
793 do_restore:
794 #ifdef CONFIG_PPC_BOOK3E
795 b .exception_return_book3e
796 #else
797 /*
798 * Clear the reservation. If we know the CPU tracks the address of
799 * the reservation then we can potentially save some cycles and use
800 * a larx. On POWER6 and POWER7 this is significantly faster.
801 */
802 BEGIN_FTR_SECTION
803 stdcx. r0,0,r1 /* to clear the reservation */
804 FTR_SECTION_ELSE
805 ldarx r4,0,r1
806 ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
808 /*
809 * Some code path such as load_up_fpu or altivec return directly
810 * here. They run entirely hard disabled and do not alter the
811 * interrupt state. They also don't use lwarx/stwcx. and thus
812 * are known not to leave dangling reservations.
813 */
814 .globl fast_exception_return
815 fast_exception_return:
816 ld r3,_MSR(r1)
817 ld r4,_CTR(r1)
818 ld r0,_LINK(r1)
819 mtctr r4
820 mtlr r0
821 ld r4,_XER(r1)
822 mtspr SPRN_XER,r4
824 REST_8GPRS(5, r1)
826 andi. r0,r3,MSR_RI
827 beq- unrecov_restore
829 /*
830 * Clear RI before restoring r13. If we are returning to
831 * userspace and we take an exception after restoring r13,
832 * we end up corrupting the userspace r13 value.
833 */
834 ld r4,PACAKMSR(r13) /* Get kernel MSR without EE */
835 andc r4,r4,r0 /* r0 contains MSR_RI here */
836 mtmsrd r4,1
838 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
839 /* TM debug */
840 std r3, PACATMSCRATCH(r13) /* Stash returned-to MSR */
841 #endif
842 /*
843 * r13 is our per cpu area, only restore it if we are returning to
844 * userspace the value stored in the stack frame may belong to
845 * another CPU.
846 */
847 andi. r0,r3,MSR_PR
848 beq 1f
849 ACCOUNT_CPU_USER_EXIT(r2, r4)
850 RESTORE_PPR(r2, r4)
851 REST_GPR(13, r1)
852 1:
853 mtspr SPRN_SRR1,r3
855 ld r2,_CCR(r1)
856 mtcrf 0xFF,r2
857 ld r2,_NIP(r1)
858 mtspr SPRN_SRR0,r2
860 ld r0,GPR0(r1)
861 ld r2,GPR2(r1)
862 ld r3,GPR3(r1)
863 ld r4,GPR4(r1)
864 ld r1,GPR1(r1)
866 rfid
867 b . /* prevent speculative execution */
869 #endif /* CONFIG_PPC_BOOK3E */
871 /*
872 * We are returning to a context with interrupts soft disabled.
873 *
874 * However, we may also about to hard enable, so we need to
875 * make sure that in this case, we also clear PACA_IRQ_HARD_DIS
876 * or that bit can get out of sync and bad things will happen
877 */
878 restore_irq_off:
879 ld r3,_MSR(r1)
880 lbz r7,PACAIRQHAPPENED(r13)
881 andi. r0,r3,MSR_EE
882 beq 1f
883 rlwinm r7,r7,0,~PACA_IRQ_HARD_DIS
884 stb r7,PACAIRQHAPPENED(r13)
885 1: li r0,0
886 stb r0,PACASOFTIRQEN(r13);
887 TRACE_DISABLE_INTS
888 b do_restore
890 /*
891 * Something did happen, check if a re-emit is needed
892 * (this also clears paca->irq_happened)
893 */
894 restore_check_irq_replay:
895 /* XXX: We could implement a fast path here where we check
896 * for irq_happened being just 0x01, in which case we can
897 * clear it and return. That means that we would potentially
898 * miss a decrementer having wrapped all the way around.
899 *
900 * Still, this might be useful for things like hash_page
901 */
902 bl .__check_irq_replay
903 cmpwi cr0,r3,0
904 beq restore_no_replay
906 /*
907 * We need to re-emit an interrupt. We do so by re-using our
908 * existing exception frame. We first change the trap value,
909 * but we need to ensure we preserve the low nibble of it
910 */
911 ld r4,_TRAP(r1)
912 clrldi r4,r4,60
913 or r4,r4,r3
914 std r4,_TRAP(r1)
916 /*
917 * Then find the right handler and call it. Interrupts are
918 * still soft-disabled and we keep them that way.
919 */
920 cmpwi cr0,r3,0x500
921 bne 1f
922 addi r3,r1,STACK_FRAME_OVERHEAD;
923 bl .do_IRQ
924 b .ret_from_except
925 1: cmpwi cr0,r3,0x900
926 bne 1f
927 addi r3,r1,STACK_FRAME_OVERHEAD;
928 bl .timer_interrupt
929 b .ret_from_except
930 #ifdef CONFIG_PPC_DOORBELL
931 1:
932 #ifdef CONFIG_PPC_BOOK3E
933 cmpwi cr0,r3,0x280
934 #else
935 BEGIN_FTR_SECTION
936 cmpwi cr0,r3,0xe80
937 FTR_SECTION_ELSE
938 cmpwi cr0,r3,0xa00
939 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
940 #endif /* CONFIG_PPC_BOOK3E */
941 bne 1f
942 addi r3,r1,STACK_FRAME_OVERHEAD;
943 bl .doorbell_exception
944 b .ret_from_except
945 #endif /* CONFIG_PPC_DOORBELL */
946 1: b .ret_from_except /* What else to do here ? */
948 unrecov_restore:
949 addi r3,r1,STACK_FRAME_OVERHEAD
950 bl .unrecoverable_exception
951 b unrecov_restore
953 #ifdef CONFIG_PPC_RTAS
954 /*
955 * On CHRP, the Run-Time Abstraction Services (RTAS) have to be
956 * called with the MMU off.
957 *
958 * In addition, we need to be in 32b mode, at least for now.
959 *
960 * Note: r3 is an input parameter to rtas, so don't trash it...
961 */
962 _GLOBAL(enter_rtas)
963 mflr r0
964 std r0,16(r1)
965 stdu r1,-RTAS_FRAME_SIZE(r1) /* Save SP and create stack space. */
967 /* Because RTAS is running in 32b mode, it clobbers the high order half
968 * of all registers that it saves. We therefore save those registers
969 * RTAS might touch to the stack. (r0, r3-r13 are caller saved)
970 */
971 SAVE_GPR(2, r1) /* Save the TOC */
972 SAVE_GPR(13, r1) /* Save paca */
973 SAVE_8GPRS(14, r1) /* Save the non-volatiles */
974 SAVE_10GPRS(22, r1) /* ditto */
976 mfcr r4
977 std r4,_CCR(r1)
978 mfctr r5
979 std r5,_CTR(r1)
980 mfspr r6,SPRN_XER
981 std r6,_XER(r1)
982 mfdar r7
983 std r7,_DAR(r1)
984 mfdsisr r8
985 std r8,_DSISR(r1)
987 /* Temporary workaround to clear CR until RTAS can be modified to
988 * ignore all bits.
989 */
990 li r0,0
991 mtcr r0
993 #ifdef CONFIG_BUG
994 /* There is no way it is acceptable to get here with interrupts enabled,
995 * check it with the asm equivalent of WARN_ON
996 */
997 lbz r0,PACASOFTIRQEN(r13)
998 1: tdnei r0,0
999 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
1000 #endif
1002 /* Hard-disable interrupts */
1003 mfmsr r6
1004 rldicl r7,r6,48,1
1005 rotldi r7,r7,16
1006 mtmsrd r7,1
1008 /* Unfortunately, the stack pointer and the MSR are also clobbered,
1009 * so they are saved in the PACA which allows us to restore
1010 * our original state after RTAS returns.
1011 */
1012 std r1,PACAR1(r13)
1013 std r6,PACASAVEDMSR(r13)
1015 /* Setup our real return addr */
1016 LOAD_REG_ADDR(r4,.rtas_return_loc)
1017 clrldi r4,r4,2 /* convert to realmode address */
1018 mtlr r4
1020 li r0,0
1021 ori r0,r0,MSR_EE|MSR_SE|MSR_BE|MSR_RI
1022 andc r0,r6,r0
1024 li r9,1
1025 rldicr r9,r9,MSR_SF_LG,(63-MSR_SF_LG)
1026 ori r9,r9,MSR_IR|MSR_DR|MSR_FE0|MSR_FE1|MSR_FP|MSR_RI
1027 andc r6,r0,r9
1028 sync /* disable interrupts so SRR0/1 */
1029 mtmsrd r0 /* don't get trashed */
1031 LOAD_REG_ADDR(r4, rtas)
1032 ld r5,RTASENTRY(r4) /* get the rtas->entry value */
1033 ld r4,RTASBASE(r4) /* get the rtas->base value */
1035 mtspr SPRN_SRR0,r5
1036 mtspr SPRN_SRR1,r6
1037 rfid
1038 b . /* prevent speculative execution */
1040 _STATIC(rtas_return_loc)
1041 /* relocation is off at this point */
1042 GET_PACA(r4)
1043 clrldi r4,r4,2 /* convert to realmode address */
1045 bcl 20,31,$+4
1046 0: mflr r3
1047 ld r3,(1f-0b)(r3) /* get &.rtas_restore_regs */
1049 mfmsr r6
1050 li r0,MSR_RI
1051 andc r6,r6,r0
1052 sync
1053 mtmsrd r6
1055 ld r1,PACAR1(r4) /* Restore our SP */
1056 ld r4,PACASAVEDMSR(r4) /* Restore our MSR */
1058 mtspr SPRN_SRR0,r3
1059 mtspr SPRN_SRR1,r4
1060 rfid
1061 b . /* prevent speculative execution */
1063 .align 3
1064 1: .llong .rtas_restore_regs
1066 _STATIC(rtas_restore_regs)
1067 /* relocation is on at this point */
1068 REST_GPR(2, r1) /* Restore the TOC */
1069 REST_GPR(13, r1) /* Restore paca */
1070 REST_8GPRS(14, r1) /* Restore the non-volatiles */
1071 REST_10GPRS(22, r1) /* ditto */
1073 GET_PACA(r13)
1075 ld r4,_CCR(r1)
1076 mtcr r4
1077 ld r5,_CTR(r1)
1078 mtctr r5
1079 ld r6,_XER(r1)
1080 mtspr SPRN_XER,r6
1081 ld r7,_DAR(r1)
1082 mtdar r7
1083 ld r8,_DSISR(r1)
1084 mtdsisr r8
1086 addi r1,r1,RTAS_FRAME_SIZE /* Unstack our frame */
1087 ld r0,16(r1) /* get return address */
1089 mtlr r0
1090 blr /* return to caller */
1092 #endif /* CONFIG_PPC_RTAS */
1094 _GLOBAL(enter_prom)
1095 mflr r0
1096 std r0,16(r1)
1097 stdu r1,-PROM_FRAME_SIZE(r1) /* Save SP and create stack space */
1099 /* Because PROM is running in 32b mode, it clobbers the high order half
1100 * of all registers that it saves. We therefore save those registers
1101 * PROM might touch to the stack. (r0, r3-r13 are caller saved)
1102 */
1103 SAVE_GPR(2, r1)
1104 SAVE_GPR(13, r1)
1105 SAVE_8GPRS(14, r1)
1106 SAVE_10GPRS(22, r1)
1107 mfcr r10
1108 mfmsr r11
1109 std r10,_CCR(r1)
1110 std r11,_MSR(r1)
1112 /* Get the PROM entrypoint */
1113 mtlr r4
1115 /* Switch MSR to 32 bits mode
1116 */
1117 #ifdef CONFIG_PPC_BOOK3E
1118 rlwinm r11,r11,0,1,31
1119 mtmsr r11
1120 #else /* CONFIG_PPC_BOOK3E */
1121 mfmsr r11
1122 li r12,1
1123 rldicr r12,r12,MSR_SF_LG,(63-MSR_SF_LG)
1124 andc r11,r11,r12
1125 li r12,1
1126 rldicr r12,r12,MSR_ISF_LG,(63-MSR_ISF_LG)
1127 andc r11,r11,r12
1128 mtmsrd r11
1129 #endif /* CONFIG_PPC_BOOK3E */
1130 isync
1132 /* Enter PROM here... */
1133 blrl
1135 /* Just make sure that r1 top 32 bits didn't get
1136 * corrupt by OF
1137 */
1138 rldicl r1,r1,0,32
1140 /* Restore the MSR (back to 64 bits) */
1141 ld r0,_MSR(r1)
1142 MTMSRD(r0)
1143 isync
1145 /* Restore other registers */
1146 REST_GPR(2, r1)
1147 REST_GPR(13, r1)
1148 REST_8GPRS(14, r1)
1149 REST_10GPRS(22, r1)
1150 ld r4,_CCR(r1)
1151 mtcr r4
1153 addi r1,r1,PROM_FRAME_SIZE
1154 ld r0,16(r1)
1155 mtlr r0
1156 blr
1158 #ifdef CONFIG_FUNCTION_TRACER
1159 #ifdef CONFIG_DYNAMIC_FTRACE
1160 _GLOBAL(mcount)
1161 _GLOBAL(_mcount)
1162 blr
1164 _GLOBAL(ftrace_caller)
1165 /* Taken from output of objdump from lib64/glibc */
1166 mflr r3
1167 ld r11, 0(r1)
1168 stdu r1, -112(r1)
1169 std r3, 128(r1)
1170 ld r4, 16(r11)
1171 subi r3, r3, MCOUNT_INSN_SIZE
1172 .globl ftrace_call
1173 ftrace_call:
1174 bl ftrace_stub
1175 nop
1176 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1177 .globl ftrace_graph_call
1178 ftrace_graph_call:
1179 b ftrace_graph_stub
1180 _GLOBAL(ftrace_graph_stub)
1181 #endif
1182 ld r0, 128(r1)
1183 mtlr r0
1184 addi r1, r1, 112
1185 _GLOBAL(ftrace_stub)
1186 blr
1187 #else
1188 _GLOBAL(mcount)
1189 blr
1191 _GLOBAL(_mcount)
1192 /* Taken from output of objdump from lib64/glibc */
1193 mflr r3
1194 ld r11, 0(r1)
1195 stdu r1, -112(r1)
1196 std r3, 128(r1)
1197 ld r4, 16(r11)
1199 subi r3, r3, MCOUNT_INSN_SIZE
1200 LOAD_REG_ADDR(r5,ftrace_trace_function)
1201 ld r5,0(r5)
1202 ld r5,0(r5)
1203 mtctr r5
1204 bctrl
1205 nop
1208 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1209 b ftrace_graph_caller
1210 #endif
1211 ld r0, 128(r1)
1212 mtlr r0
1213 addi r1, r1, 112
1214 _GLOBAL(ftrace_stub)
1215 blr
1217 #endif /* CONFIG_DYNAMIC_FTRACE */
1219 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1220 _GLOBAL(ftrace_graph_caller)
1221 /* load r4 with local address */
1222 ld r4, 128(r1)
1223 subi r4, r4, MCOUNT_INSN_SIZE
1225 /* get the parent address */
1226 ld r11, 112(r1)
1227 addi r3, r11, 16
1229 bl .prepare_ftrace_return
1230 nop
1232 ld r0, 128(r1)
1233 mtlr r0
1234 addi r1, r1, 112
1235 blr
1237 _GLOBAL(return_to_handler)
1238 /* need to save return values */
1239 std r4, -24(r1)
1240 std r3, -16(r1)
1241 std r31, -8(r1)
1242 mr r31, r1
1243 stdu r1, -112(r1)
1245 bl .ftrace_return_to_handler
1246 nop
1248 /* return value has real return address */
1249 mtlr r3
1251 ld r1, 0(r1)
1252 ld r4, -24(r1)
1253 ld r3, -16(r1)
1254 ld r31, -8(r1)
1256 /* Jump back to real return address */
1257 blr
1259 _GLOBAL(mod_return_to_handler)
1260 /* need to save return values */
1261 std r4, -32(r1)
1262 std r3, -24(r1)
1263 /* save TOC */
1264 std r2, -16(r1)
1265 std r31, -8(r1)
1266 mr r31, r1
1267 stdu r1, -112(r1)
1269 /*
1270 * We are in a module using the module's TOC.
1271 * Switch to our TOC to run inside the core kernel.
1272 */
1273 ld r2, PACATOC(r13)
1275 bl .ftrace_return_to_handler
1276 nop
1278 /* return value has real return address */
1279 mtlr r3
1281 ld r1, 0(r1)
1282 ld r4, -32(r1)
1283 ld r3, -24(r1)
1284 ld r2, -16(r1)
1285 ld r31, -8(r1)
1287 /* Jump back to real return address */
1288 blr
1289 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1290 #endif /* CONFIG_FUNCTION_TRACER */