| blob | 917cebc2f262b2cd2b3d1f125759611cefd5d7ae |
1 /*
2 * PowerPC version
3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 * Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
5 * Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
6 * Adapted for Power Macintosh by Paul Mackerras.
7 * Low-level exception handlers and MMU support
8 * rewritten by Paul Mackerras.
9 * Copyright (C) 1996 Paul Mackerras.
10 * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
11 *
12 * This file contains the system call entry code, context switch
13 * code, and exception/interrupt return code for PowerPC.
14 *
15 * This program is free software; you can redistribute it and/or
16 * modify it under the terms of the GNU General Public License
17 * as published by the Free Software Foundation; either version
18 * 2 of the License, or (at your option) any later version.
19 */
21 #include <linux/errno.h>
22 #include <asm/unistd.h>
23 #include <asm/processor.h>
24 #include <asm/page.h>
25 #include <asm/mmu.h>
26 #include <asm/thread_info.h>
27 #include <asm/ppc_asm.h>
28 #include <asm/asm-offsets.h>
29 #include <asm/cputable.h>
30 #include <asm/firmware.h>
31 #include <asm/bug.h>
32 #include <asm/ptrace.h>
33 #include <asm/irqflags.h>
34 #include <asm/ftrace.h>
36 /*
37 * System calls.
38 */
39 .section ".toc","aw"
40 .SYS_CALL_TABLE:
41 .tc .sys_call_table[TC],.sys_call_table
43 /* This value is used to mark exception frames on the stack. */
44 exception_marker:
45 .tc ID_EXC_MARKER[TC],STACK_FRAME_REGS_MARKER
47 .section ".text"
48 .align 7
50 #undef SHOW_SYSCALLS
52 .globl system_call_common
53 system_call_common:
54 andi. r10,r12,MSR_PR
55 mr r10,r1
56 addi r1,r1,-INT_FRAME_SIZE
57 beq- 1f
58 ld r1,PACAKSAVE(r13)
59 1: std r10,0(r1)
60 std r11,_NIP(r1)
61 std r12,_MSR(r1)
62 std r0,GPR0(r1)
63 std r10,GPR1(r1)
64 ACCOUNT_CPU_USER_ENTRY(r10, r11)
65 /*
66 * This "crclr so" clears CR0.SO, which is the error indication on
67 * return from this system call. There must be no cmp instruction
68 * between it and the "mfcr r9" below, otherwise if XER.SO is set,
69 * CR0.SO will get set, causing all system calls to appear to fail.
70 */
71 crclr so
72 std r2,GPR2(r1)
73 std r3,GPR3(r1)
74 std r4,GPR4(r1)
75 std r5,GPR5(r1)
76 std r6,GPR6(r1)
77 std r7,GPR7(r1)
78 std r8,GPR8(r1)
79 li r11,0
80 std r11,GPR9(r1)
81 std r11,GPR10(r1)
82 std r11,GPR11(r1)
83 std r11,GPR12(r1)
84 std r9,GPR13(r1)
85 mfcr r9
86 mflr r10
87 li r11,0xc01
88 std r9,_CCR(r1)
89 std r10,_LINK(r1)
90 std r11,_TRAP(r1)
91 mfxer r9
92 mfctr r10
93 std r9,_XER(r1)
94 std r10,_CTR(r1)
95 std r3,ORIG_GPR3(r1)
96 ld r2,PACATOC(r13)
97 addi r9,r1,STACK_FRAME_OVERHEAD
98 ld r11,exception_marker@toc(r2)
99 std r11,-16(r9) /* "regshere" marker */
100 #ifdef CONFIG_TRACE_IRQFLAGS
101 bl .trace_hardirqs_on
102 REST_GPR(0,r1)
103 REST_4GPRS(3,r1)
104 REST_2GPRS(7,r1)
105 addi r9,r1,STACK_FRAME_OVERHEAD
106 ld r12,_MSR(r1)
107 #endif /* CONFIG_TRACE_IRQFLAGS */
108 li r10,1
109 stb r10,PACASOFTIRQEN(r13)
110 stb r10,PACAHARDIRQEN(r13)
111 std r10,SOFTE(r1)
112 #ifdef CONFIG_PPC_ISERIES
113 BEGIN_FW_FTR_SECTION
114 /* Hack for handling interrupts when soft-enabling on iSeries */
115 cmpdi cr1,r0,0x5555 /* syscall 0x5555 */
116 andi. r10,r12,MSR_PR /* from kernel */
117 crand 4*cr0+eq,4*cr1+eq,4*cr0+eq
118 bne 2f
119 b hardware_interrupt_entry
120 2:
121 END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
122 #endif /* CONFIG_PPC_ISERIES */
124 /* Hard enable interrupts */
125 #ifdef CONFIG_PPC_BOOK3E
126 wrteei 1
127 #else
128 mfmsr r11
129 ori r11,r11,MSR_EE
130 mtmsrd r11,1
131 #endif /* CONFIG_PPC_BOOK3E */
133 #ifdef SHOW_SYSCALLS
134 bl .do_show_syscall
135 REST_GPR(0,r1)
136 REST_4GPRS(3,r1)
137 REST_2GPRS(7,r1)
138 addi r9,r1,STACK_FRAME_OVERHEAD
139 #endif
140 clrrdi r11,r1,THREAD_SHIFT
141 ld r10,TI_FLAGS(r11)
142 andi. r11,r10,_TIF_SYSCALL_T_OR_A
143 bne- syscall_dotrace
144 syscall_dotrace_cont:
145 cmpldi 0,r0,NR_syscalls
146 bge- syscall_enosys
148 system_call: /* label this so stack traces look sane */
149 /*
150 * Need to vector to 32 Bit or default sys_call_table here,
151 * based on caller's run-mode / personality.
152 */
153 ld r11,.SYS_CALL_TABLE@toc(2)
154 andi. r10,r10,_TIF_32BIT
155 beq 15f
156 addi r11,r11,8 /* use 32-bit syscall entries */
157 clrldi r3,r3,32
158 clrldi r4,r4,32
159 clrldi r5,r5,32
160 clrldi r6,r6,32
161 clrldi r7,r7,32
162 clrldi r8,r8,32
163 15:
164 slwi r0,r0,4
165 ldx r10,r11,r0 /* Fetch system call handler [ptr] */
166 mtctr r10
167 bctrl /* Call handler */
169 syscall_exit:
170 std r3,RESULT(r1)
171 #ifdef SHOW_SYSCALLS
172 bl .do_show_syscall_exit
173 ld r3,RESULT(r1)
174 #endif
175 clrrdi r12,r1,THREAD_SHIFT
177 ld r8,_MSR(r1)
178 #ifdef CONFIG_PPC_BOOK3S
179 /* No MSR:RI on BookE */
180 andi. r10,r8,MSR_RI
181 beq- unrecov_restore
182 #endif
184 /* Disable interrupts so current_thread_info()->flags can't change,
185 * and so that we don't get interrupted after loading SRR0/1.
186 */
187 #ifdef CONFIG_PPC_BOOK3E
188 wrteei 0
189 #else
190 mfmsr r10
191 rldicl r10,r10,48,1
192 rotldi r10,r10,16
193 mtmsrd r10,1
194 #endif /* CONFIG_PPC_BOOK3E */
196 ld r9,TI_FLAGS(r12)
197 li r11,-_LAST_ERRNO
198 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
199 bne- syscall_exit_work
200 cmpld r3,r11
201 ld r5,_CCR(r1)
202 bge- syscall_error
203 syscall_error_cont:
204 ld r7,_NIP(r1)
205 stdcx. r0,0,r1 /* to clear the reservation */
206 andi. r6,r8,MSR_PR
207 ld r4,_LINK(r1)
208 /*
209 * Clear RI before restoring r13. If we are returning to
210 * userspace and we take an exception after restoring r13,
211 * we end up corrupting the userspace r13 value.
212 */
213 #ifdef CONFIG_PPC_BOOK3S
214 /* No MSR:RI on BookE */
215 li r12,MSR_RI
216 andc r11,r10,r12
217 mtmsrd r11,1 /* clear MSR.RI */
218 #endif /* CONFIG_PPC_BOOK3S */
220 beq- 1f
221 ACCOUNT_CPU_USER_EXIT(r11, r12)
222 ld r13,GPR13(r1) /* only restore r13 if returning to usermode */
223 1: ld r2,GPR2(r1)
224 ld r1,GPR1(r1)
225 mtlr r4
226 mtcr r5
227 mtspr SPRN_SRR0,r7
228 mtspr SPRN_SRR1,r8
229 RFI
230 b . /* prevent speculative execution */
232 syscall_error:
233 oris r5,r5,0x1000 /* Set SO bit in CR */
234 neg r3,r3
235 std r5,_CCR(r1)
236 b syscall_error_cont
238 /* Traced system call support */
239 syscall_dotrace:
240 bl .save_nvgprs
241 addi r3,r1,STACK_FRAME_OVERHEAD
242 bl .do_syscall_trace_enter
243 /*
244 * Restore argument registers possibly just changed.
245 * We use the return value of do_syscall_trace_enter
246 * for the call number to look up in the table (r0).
247 */
248 mr r0,r3
249 ld r3,GPR3(r1)
250 ld r4,GPR4(r1)
251 ld r5,GPR5(r1)
252 ld r6,GPR6(r1)
253 ld r7,GPR7(r1)
254 ld r8,GPR8(r1)
255 addi r9,r1,STACK_FRAME_OVERHEAD
256 clrrdi r10,r1,THREAD_SHIFT
257 ld r10,TI_FLAGS(r10)
258 b syscall_dotrace_cont
260 syscall_enosys:
261 li r3,-ENOSYS
262 b syscall_exit
264 syscall_exit_work:
265 /* If TIF_RESTOREALL is set, don't scribble on either r3 or ccr.
266 If TIF_NOERROR is set, just save r3 as it is. */
268 andi. r0,r9,_TIF_RESTOREALL
269 beq+ 0f
270 REST_NVGPRS(r1)
271 b 2f
272 0: cmpld r3,r11 /* r10 is -LAST_ERRNO */
273 blt+ 1f
274 andi. r0,r9,_TIF_NOERROR
275 bne- 1f
276 ld r5,_CCR(r1)
277 neg r3,r3
278 oris r5,r5,0x1000 /* Set SO bit in CR */
279 std r5,_CCR(r1)
280 1: std r3,GPR3(r1)
281 2: andi. r0,r9,(_TIF_PERSYSCALL_MASK)
282 beq 4f
284 /* Clear per-syscall TIF flags if any are set. */
286 li r11,_TIF_PERSYSCALL_MASK
287 addi r12,r12,TI_FLAGS
288 3: ldarx r10,0,r12
289 andc r10,r10,r11
290 stdcx. r10,0,r12
291 bne- 3b
292 subi r12,r12,TI_FLAGS
294 4: /* Anything else left to do? */
295 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP)
296 beq .ret_from_except_lite
298 /* Re-enable interrupts */
299 #ifdef CONFIG_PPC_BOOK3E
300 wrteei 1
301 #else
302 mfmsr r10
303 ori r10,r10,MSR_EE
304 mtmsrd r10,1
305 #endif /* CONFIG_PPC_BOOK3E */
307 bl .save_nvgprs
308 addi r3,r1,STACK_FRAME_OVERHEAD
309 bl .do_syscall_trace_leave
310 b .ret_from_except
312 /* Save non-volatile GPRs, if not already saved. */
313 _GLOBAL(save_nvgprs)
314 ld r11,_TRAP(r1)
315 andi. r0,r11,1
316 beqlr-
317 SAVE_NVGPRS(r1)
318 clrrdi r0,r11,1
319 std r0,_TRAP(r1)
320 blr
323 /*
324 * The sigsuspend and rt_sigsuspend system calls can call do_signal
325 * and thus put the process into the stopped state where we might
326 * want to examine its user state with ptrace. Therefore we need
327 * to save all the nonvolatile registers (r14 - r31) before calling
328 * the C code. Similarly, fork, vfork and clone need the full
329 * register state on the stack so that it can be copied to the child.
330 */
332 _GLOBAL(ppc_fork)
333 bl .save_nvgprs
334 bl .sys_fork
335 b syscall_exit
337 _GLOBAL(ppc_vfork)
338 bl .save_nvgprs
339 bl .sys_vfork
340 b syscall_exit
342 _GLOBAL(ppc_clone)
343 bl .save_nvgprs
344 bl .sys_clone
345 b syscall_exit
347 _GLOBAL(ppc32_swapcontext)
348 bl .save_nvgprs
349 bl .compat_sys_swapcontext
350 b syscall_exit
352 _GLOBAL(ppc64_swapcontext)
353 bl .save_nvgprs
354 bl .sys_swapcontext
355 b syscall_exit
357 _GLOBAL(ret_from_fork)
358 bl .schedule_tail
359 REST_NVGPRS(r1)
360 li r3,0
361 b syscall_exit
363 /*
364 * This routine switches between two different tasks. The process
365 * state of one is saved on its kernel stack. Then the state
366 * of the other is restored from its kernel stack. The memory
367 * management hardware is updated to the second process's state.
368 * Finally, we can return to the second process, via ret_from_except.
369 * On entry, r3 points to the THREAD for the current task, r4
370 * points to the THREAD for the new task.
371 *
372 * Note: there are two ways to get to the "going out" portion
373 * of this code; either by coming in via the entry (_switch)
374 * or via "fork" which must set up an environment equivalent
375 * to the "_switch" path. If you change this you'll have to change
376 * the fork code also.
377 *
378 * The code which creates the new task context is in 'copy_thread'
379 * in arch/powerpc/kernel/process.c
380 */
381 .align 7
382 _GLOBAL(_switch)
383 mflr r0
384 std r0,16(r1)
385 stdu r1,-SWITCH_FRAME_SIZE(r1)
386 /* r3-r13 are caller saved -- Cort */
387 SAVE_8GPRS(14, r1)
388 SAVE_10GPRS(22, r1)
389 mflr r20 /* Return to switch caller */
390 mfmsr r22
391 li r0, MSR_FP
392 #ifdef CONFIG_VSX
393 BEGIN_FTR_SECTION
394 oris r0,r0,MSR_VSX@h /* Disable VSX */
395 END_FTR_SECTION_IFSET(CPU_FTR_VSX)
396 #endif /* CONFIG_VSX */
397 #ifdef CONFIG_ALTIVEC
398 BEGIN_FTR_SECTION
399 oris r0,r0,MSR_VEC@h /* Disable altivec */
400 mfspr r24,SPRN_VRSAVE /* save vrsave register value */
401 std r24,THREAD_VRSAVE(r3)
402 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
403 #endif /* CONFIG_ALTIVEC */
404 and. r0,r0,r22
405 beq+ 1f
406 andc r22,r22,r0
407 MTMSRD(r22)
408 isync
409 1: std r20,_NIP(r1)
410 mfcr r23
411 std r23,_CCR(r1)
412 std r1,KSP(r3) /* Set old stack pointer */
414 #ifdef CONFIG_SMP
415 /* We need a sync somewhere here to make sure that if the
416 * previous task gets rescheduled on another CPU, it sees all
417 * stores it has performed on this one.
418 */
419 sync
420 #endif /* CONFIG_SMP */
422 addi r6,r4,-THREAD /* Convert THREAD to 'current' */
423 std r6,PACACURRENT(r13) /* Set new 'current' */
425 ld r8,KSP(r4) /* new stack pointer */
426 #ifdef CONFIG_PPC_BOOK3S
427 BEGIN_FTR_SECTION
428 BEGIN_FTR_SECTION_NESTED(95)
429 clrrdi r6,r8,28 /* get its ESID */
430 clrrdi r9,r1,28 /* get current sp ESID */
431 FTR_SECTION_ELSE_NESTED(95)
432 clrrdi r6,r8,40 /* get its 1T ESID */
433 clrrdi r9,r1,40 /* get current sp 1T ESID */
434 ALT_FTR_SECTION_END_NESTED_IFCLR(CPU_FTR_1T_SEGMENT, 95)
435 FTR_SECTION_ELSE
436 b 2f
437 ALT_FTR_SECTION_END_IFSET(CPU_FTR_SLB)
438 clrldi. r0,r6,2 /* is new ESID c00000000? */
439 cmpd cr1,r6,r9 /* or is new ESID the same as current ESID? */
440 cror eq,4*cr1+eq,eq
441 beq 2f /* if yes, don't slbie it */
443 /* Bolt in the new stack SLB entry */
444 ld r7,KSP_VSID(r4) /* Get new stack's VSID */
445 oris r0,r6,(SLB_ESID_V)@h
446 ori r0,r0,(SLB_NUM_BOLTED-1)@l
447 BEGIN_FTR_SECTION
448 li r9,MMU_SEGSIZE_1T /* insert B field */
449 oris r6,r6,(MMU_SEGSIZE_1T << SLBIE_SSIZE_SHIFT)@h
450 rldimi r7,r9,SLB_VSID_SSIZE_SHIFT,0
451 END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
453 /* Update the last bolted SLB. No write barriers are needed
454 * here, provided we only update the current CPU's SLB shadow
455 * buffer.
456 */
457 ld r9,PACA_SLBSHADOWPTR(r13)
458 li r12,0
459 std r12,SLBSHADOW_STACKESID(r9) /* Clear ESID */
460 std r7,SLBSHADOW_STACKVSID(r9) /* Save VSID */
461 std r0,SLBSHADOW_STACKESID(r9) /* Save ESID */
463 /* No need to check for CPU_FTR_NO_SLBIE_B here, since when
464 * we have 1TB segments, the only CPUs known to have the errata
465 * only support less than 1TB of system memory and we'll never
466 * actually hit this code path.
467 */
469 slbie r6
470 slbie r6 /* Workaround POWER5 < DD2.1 issue */
471 slbmte r7,r0
472 isync
473 2:
474 #endif /* !CONFIG_PPC_BOOK3S */
476 clrrdi r7,r8,THREAD_SHIFT /* base of new stack */
477 /* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE
478 because we don't need to leave the 288-byte ABI gap at the
479 top of the kernel stack. */
480 addi r7,r7,THREAD_SIZE-SWITCH_FRAME_SIZE
482 mr r1,r8 /* start using new stack pointer */
483 std r7,PACAKSAVE(r13)
485 ld r6,_CCR(r1)
486 mtcrf 0xFF,r6
488 #ifdef CONFIG_ALTIVEC
489 BEGIN_FTR_SECTION
490 ld r0,THREAD_VRSAVE(r4)
491 mtspr SPRN_VRSAVE,r0 /* if G4, restore VRSAVE reg */
492 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
493 #endif /* CONFIG_ALTIVEC */
495 /* r3-r13 are destroyed -- Cort */
496 REST_8GPRS(14, r1)
497 REST_10GPRS(22, r1)
499 /* convert old thread to its task_struct for return value */
500 addi r3,r3,-THREAD
501 ld r7,_NIP(r1) /* Return to _switch caller in new task */
502 mtlr r7
503 addi r1,r1,SWITCH_FRAME_SIZE
504 blr
506 .align 7
507 _GLOBAL(ret_from_except)
508 ld r11,_TRAP(r1)
509 andi. r0,r11,1
510 bne .ret_from_except_lite
511 REST_NVGPRS(r1)
513 _GLOBAL(ret_from_except_lite)
514 /*
515 * Disable interrupts so that current_thread_info()->flags
516 * can't change between when we test it and when we return
517 * from the interrupt.
518 */
519 #ifdef CONFIG_PPC_BOOK3E
520 wrteei 0
521 #else
522 mfmsr r10 /* Get current interrupt state */
523 rldicl r9,r10,48,1 /* clear MSR_EE */
524 rotldi r9,r9,16
525 mtmsrd r9,1 /* Update machine state */
526 #endif /* CONFIG_PPC_BOOK3E */
528 #ifdef CONFIG_PREEMPT
529 clrrdi r9,r1,THREAD_SHIFT /* current_thread_info() */
530 li r0,_TIF_NEED_RESCHED /* bits to check */
531 ld r3,_MSR(r1)
532 ld r4,TI_FLAGS(r9)
533 /* Move MSR_PR bit in r3 to _TIF_SIGPENDING position in r0 */
534 rlwimi r0,r3,32+TIF_SIGPENDING-MSR_PR_LG,_TIF_SIGPENDING
535 and. r0,r4,r0 /* check NEED_RESCHED and maybe SIGPENDING */
536 bne do_work
538 #else /* !CONFIG_PREEMPT */
539 ld r3,_MSR(r1) /* Returning to user mode? */
540 andi. r3,r3,MSR_PR
541 beq restore /* if not, just restore regs and return */
543 /* Check current_thread_info()->flags */
544 clrrdi r9,r1,THREAD_SHIFT
545 ld r4,TI_FLAGS(r9)
546 andi. r0,r4,_TIF_USER_WORK_MASK
547 bne do_work
548 #endif
550 restore:
551 BEGIN_FW_FTR_SECTION
552 ld r5,SOFTE(r1)
553 FW_FTR_SECTION_ELSE
554 b iseries_check_pending_irqs
555 ALT_FW_FTR_SECTION_END_IFCLR(FW_FEATURE_ISERIES)
556 2:
557 TRACE_AND_RESTORE_IRQ(r5);
559 /* extract EE bit and use it to restore paca->hard_enabled */
560 ld r3,_MSR(r1)
561 rldicl r4,r3,49,63 /* r0 = (r3 >> 15) & 1 */
562 stb r4,PACAHARDIRQEN(r13)
564 #ifdef CONFIG_PPC_BOOK3E
565 b .exception_return_book3e
566 #else
567 ld r4,_CTR(r1)
568 ld r0,_LINK(r1)
569 mtctr r4
570 mtlr r0
571 ld r4,_XER(r1)
572 mtspr SPRN_XER,r4
574 REST_8GPRS(5, r1)
576 andi. r0,r3,MSR_RI
577 beq- unrecov_restore
579 stdcx. r0,0,r1 /* to clear the reservation */
581 /*
582 * Clear RI before restoring r13. If we are returning to
583 * userspace and we take an exception after restoring r13,
584 * we end up corrupting the userspace r13 value.
585 */
586 mfmsr r4
587 andc r4,r4,r0 /* r0 contains MSR_RI here */
588 mtmsrd r4,1
590 /*
591 * r13 is our per cpu area, only restore it if we are returning to
592 * userspace
593 */
594 andi. r0,r3,MSR_PR
595 beq 1f
596 ACCOUNT_CPU_USER_EXIT(r2, r4)
597 REST_GPR(13, r1)
598 1:
599 mtspr SPRN_SRR1,r3
601 ld r2,_CCR(r1)
602 mtcrf 0xFF,r2
603 ld r2,_NIP(r1)
604 mtspr SPRN_SRR0,r2
606 ld r0,GPR0(r1)
607 ld r2,GPR2(r1)
608 ld r3,GPR3(r1)
609 ld r4,GPR4(r1)
610 ld r1,GPR1(r1)
612 rfid
613 b . /* prevent speculative execution */
615 #endif /* CONFIG_PPC_BOOK3E */
617 iseries_check_pending_irqs:
618 #ifdef CONFIG_PPC_ISERIES
619 ld r5,SOFTE(r1)
620 cmpdi 0,r5,0
621 beq 2b
622 /* Check for pending interrupts (iSeries) */
623 ld r3,PACALPPACAPTR(r13)
624 ld r3,LPPACAANYINT(r3)
625 cmpdi r3,0
626 beq+ 2b /* skip do_IRQ if no interrupts */
628 li r3,0
629 stb r3,PACASOFTIRQEN(r13) /* ensure we are soft-disabled */
630 #ifdef CONFIG_TRACE_IRQFLAGS
631 bl .trace_hardirqs_off
632 mfmsr r10
633 #endif
634 ori r10,r10,MSR_EE
635 mtmsrd r10 /* hard-enable again */
636 addi r3,r1,STACK_FRAME_OVERHEAD
637 bl .do_IRQ
638 b .ret_from_except_lite /* loop back and handle more */
639 #endif
641 do_work:
642 #ifdef CONFIG_PREEMPT
643 andi. r0,r3,MSR_PR /* Returning to user mode? */
644 bne user_work
645 /* Check that preempt_count() == 0 and interrupts are enabled */
646 lwz r8,TI_PREEMPT(r9)
647 cmpwi cr1,r8,0
648 ld r0,SOFTE(r1)
649 cmpdi r0,0
650 crandc eq,cr1*4+eq,eq
651 bne restore
653 /* Here we are preempting the current task.
654 *
655 * Ensure interrupts are soft-disabled. We also properly mark
656 * the PACA to reflect the fact that they are hard-disabled
657 * and trace the change
658 */
659 li r0,0
660 stb r0,PACASOFTIRQEN(r13)
661 stb r0,PACAHARDIRQEN(r13)
662 TRACE_DISABLE_INTS
664 /* Call the scheduler with soft IRQs off */
665 1: bl .preempt_schedule_irq
667 /* Hard-disable interrupts again (and update PACA) */
668 #ifdef CONFIG_PPC_BOOK3E
669 wrteei 0
670 #else
671 mfmsr r10
672 rldicl r10,r10,48,1
673 rotldi r10,r10,16
674 mtmsrd r10,1
675 #endif /* CONFIG_PPC_BOOK3E */
676 li r0,0
677 stb r0,PACAHARDIRQEN(r13)
679 /* Re-test flags and eventually loop */
680 clrrdi r9,r1,THREAD_SHIFT
681 ld r4,TI_FLAGS(r9)
682 andi. r0,r4,_TIF_NEED_RESCHED
683 bne 1b
684 b restore
686 user_work:
687 #endif /* CONFIG_PREEMPT */
689 /* Enable interrupts */
690 #ifdef CONFIG_PPC_BOOK3E
691 wrteei 1
692 #else
693 ori r10,r10,MSR_EE
694 mtmsrd r10,1
695 #endif /* CONFIG_PPC_BOOK3E */
697 andi. r0,r4,_TIF_NEED_RESCHED
698 beq 1f
699 bl .schedule
700 b .ret_from_except_lite
702 1: bl .save_nvgprs
703 addi r3,r1,STACK_FRAME_OVERHEAD
704 bl .do_signal
705 b .ret_from_except
707 unrecov_restore:
708 addi r3,r1,STACK_FRAME_OVERHEAD
709 bl .unrecoverable_exception
710 b unrecov_restore
712 #ifdef CONFIG_PPC_RTAS
713 /*
714 * On CHRP, the Run-Time Abstraction Services (RTAS) have to be
715 * called with the MMU off.
716 *
717 * In addition, we need to be in 32b mode, at least for now.
718 *
719 * Note: r3 is an input parameter to rtas, so don't trash it...
720 */
721 _GLOBAL(enter_rtas)
722 mflr r0
723 std r0,16(r1)
724 stdu r1,-RTAS_FRAME_SIZE(r1) /* Save SP and create stack space. */
726 /* Because RTAS is running in 32b mode, it clobbers the high order half
727 * of all registers that it saves. We therefore save those registers
728 * RTAS might touch to the stack. (r0, r3-r13 are caller saved)
729 */
730 SAVE_GPR(2, r1) /* Save the TOC */
731 SAVE_GPR(13, r1) /* Save paca */
732 SAVE_8GPRS(14, r1) /* Save the non-volatiles */
733 SAVE_10GPRS(22, r1) /* ditto */
735 mfcr r4
736 std r4,_CCR(r1)
737 mfctr r5
738 std r5,_CTR(r1)
739 mfspr r6,SPRN_XER
740 std r6,_XER(r1)
741 mfdar r7
742 std r7,_DAR(r1)
743 mfdsisr r8
744 std r8,_DSISR(r1)
746 /* Temporary workaround to clear CR until RTAS can be modified to
747 * ignore all bits.
748 */
749 li r0,0
750 mtcr r0
752 #ifdef CONFIG_BUG
753 /* There is no way it is acceptable to get here with interrupts enabled,
754 * check it with the asm equivalent of WARN_ON
755 */
756 lbz r0,PACASOFTIRQEN(r13)
757 1: tdnei r0,0
758 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
759 #endif
761 /* Hard-disable interrupts */
762 mfmsr r6
763 rldicl r7,r6,48,1
764 rotldi r7,r7,16
765 mtmsrd r7,1
767 /* Unfortunately, the stack pointer and the MSR are also clobbered,
768 * so they are saved in the PACA which allows us to restore
769 * our original state after RTAS returns.
770 */
771 std r1,PACAR1(r13)
772 std r6,PACASAVEDMSR(r13)
774 /* Setup our real return addr */
775 LOAD_REG_ADDR(r4,.rtas_return_loc)
776 clrldi r4,r4,2 /* convert to realmode address */
777 mtlr r4
779 li r0,0
780 ori r0,r0,MSR_EE|MSR_SE|MSR_BE|MSR_RI
781 andc r0,r6,r0
783 li r9,1
784 rldicr r9,r9,MSR_SF_LG,(63-MSR_SF_LG)
785 ori r9,r9,MSR_IR|MSR_DR|MSR_FE0|MSR_FE1|MSR_FP
786 andc r6,r0,r9
787 ori r6,r6,MSR_RI
788 sync /* disable interrupts so SRR0/1 */
789 mtmsrd r0 /* don't get trashed */
791 LOAD_REG_ADDR(r4, rtas)
792 ld r5,RTASENTRY(r4) /* get the rtas->entry value */
793 ld r4,RTASBASE(r4) /* get the rtas->base value */
795 mtspr SPRN_SRR0,r5
796 mtspr SPRN_SRR1,r6
797 rfid
798 b . /* prevent speculative execution */
800 _STATIC(rtas_return_loc)
801 /* relocation is off at this point */
802 mfspr r4,SPRN_SPRG_PACA /* Get PACA */
803 clrldi r4,r4,2 /* convert to realmode address */
805 bcl 20,31,$+4
806 0: mflr r3
807 ld r3,(1f-0b)(r3) /* get &.rtas_restore_regs */
809 mfmsr r6
810 li r0,MSR_RI
811 andc r6,r6,r0
812 sync
813 mtmsrd r6
815 ld r1,PACAR1(r4) /* Restore our SP */
816 ld r4,PACASAVEDMSR(r4) /* Restore our MSR */
818 mtspr SPRN_SRR0,r3
819 mtspr SPRN_SRR1,r4
820 rfid
821 b . /* prevent speculative execution */
823 .align 3
824 1: .llong .rtas_restore_regs
826 _STATIC(rtas_restore_regs)
827 /* relocation is on at this point */
828 REST_GPR(2, r1) /* Restore the TOC */
829 REST_GPR(13, r1) /* Restore paca */
830 REST_8GPRS(14, r1) /* Restore the non-volatiles */
831 REST_10GPRS(22, r1) /* ditto */
833 mfspr r13,SPRN_SPRG_PACA
835 ld r4,_CCR(r1)
836 mtcr r4
837 ld r5,_CTR(r1)
838 mtctr r5
839 ld r6,_XER(r1)
840 mtspr SPRN_XER,r6
841 ld r7,_DAR(r1)
842 mtdar r7
843 ld r8,_DSISR(r1)
844 mtdsisr r8
846 addi r1,r1,RTAS_FRAME_SIZE /* Unstack our frame */
847 ld r0,16(r1) /* get return address */
849 mtlr r0
850 blr /* return to caller */
852 #endif /* CONFIG_PPC_RTAS */
854 _GLOBAL(enter_prom)
855 mflr r0
856 std r0,16(r1)
857 stdu r1,-PROM_FRAME_SIZE(r1) /* Save SP and create stack space */
859 /* Because PROM is running in 32b mode, it clobbers the high order half
860 * of all registers that it saves. We therefore save those registers
861 * PROM might touch to the stack. (r0, r3-r13 are caller saved)
862 */
863 SAVE_GPR(2, r1)
864 SAVE_GPR(13, r1)
865 SAVE_8GPRS(14, r1)
866 SAVE_10GPRS(22, r1)
867 mfcr r10
868 mfmsr r11
869 std r10,_CCR(r1)
870 std r11,_MSR(r1)
872 /* Get the PROM entrypoint */
873 mtlr r4
875 /* Switch MSR to 32 bits mode
876 */
877 #ifdef CONFIG_PPC_BOOK3E
878 rlwinm r11,r11,0,1,31
879 mtmsr r11
880 #else /* CONFIG_PPC_BOOK3E */
881 mfmsr r11
882 li r12,1
883 rldicr r12,r12,MSR_SF_LG,(63-MSR_SF_LG)
884 andc r11,r11,r12
885 li r12,1
886 rldicr r12,r12,MSR_ISF_LG,(63-MSR_ISF_LG)
887 andc r11,r11,r12
888 mtmsrd r11
889 #endif /* CONFIG_PPC_BOOK3E */
890 isync
892 /* Enter PROM here... */
893 blrl
895 /* Just make sure that r1 top 32 bits didn't get
896 * corrupt by OF
897 */
898 rldicl r1,r1,0,32
900 /* Restore the MSR (back to 64 bits) */
901 ld r0,_MSR(r1)
902 MTMSRD(r0)
903 isync
905 /* Restore other registers */
906 REST_GPR(2, r1)
907 REST_GPR(13, r1)
908 REST_8GPRS(14, r1)
909 REST_10GPRS(22, r1)
910 ld r4,_CCR(r1)
911 mtcr r4
913 addi r1,r1,PROM_FRAME_SIZE
914 ld r0,16(r1)
915 mtlr r0
916 blr
918 #ifdef CONFIG_FUNCTION_TRACER
919 #ifdef CONFIG_DYNAMIC_FTRACE
920 _GLOBAL(mcount)
921 _GLOBAL(_mcount)
922 blr
924 _GLOBAL(ftrace_caller)
925 /* Taken from output of objdump from lib64/glibc */
926 mflr r3
927 ld r11, 0(r1)
928 stdu r1, -112(r1)
929 std r3, 128(r1)
930 ld r4, 16(r11)
931 subi r3, r3, MCOUNT_INSN_SIZE
932 .globl ftrace_call
933 ftrace_call:
934 bl ftrace_stub
935 nop
936 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
937 .globl ftrace_graph_call
938 ftrace_graph_call:
939 b ftrace_graph_stub
940 _GLOBAL(ftrace_graph_stub)
941 #endif
942 ld r0, 128(r1)
943 mtlr r0
944 addi r1, r1, 112
945 _GLOBAL(ftrace_stub)
946 blr
947 #else
948 _GLOBAL(mcount)
949 blr
951 _GLOBAL(_mcount)
952 /* Taken from output of objdump from lib64/glibc */
953 mflr r3
954 ld r11, 0(r1)
955 stdu r1, -112(r1)
956 std r3, 128(r1)
957 ld r4, 16(r11)
959 subi r3, r3, MCOUNT_INSN_SIZE
960 LOAD_REG_ADDR(r5,ftrace_trace_function)
961 ld r5,0(r5)
962 ld r5,0(r5)
963 mtctr r5
964 bctrl
965 nop
968 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
969 b ftrace_graph_caller
970 #endif
971 ld r0, 128(r1)
972 mtlr r0
973 addi r1, r1, 112
974 _GLOBAL(ftrace_stub)
975 blr
977 #endif /* CONFIG_DYNAMIC_FTRACE */
979 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
980 _GLOBAL(ftrace_graph_caller)
981 /* load r4 with local address */
982 ld r4, 128(r1)
983 subi r4, r4, MCOUNT_INSN_SIZE
985 /* get the parent address */
986 ld r11, 112(r1)
987 addi r3, r11, 16
989 bl .prepare_ftrace_return
990 nop
992 ld r0, 128(r1)
993 mtlr r0
994 addi r1, r1, 112
995 blr
997 _GLOBAL(return_to_handler)
998 /* need to save return values */
999 std r4, -24(r1)
1000 std r3, -16(r1)
1001 std r31, -8(r1)
1002 mr r31, r1
1003 stdu r1, -112(r1)
1005 bl .ftrace_return_to_handler
1006 nop
1008 /* return value has real return address */
1009 mtlr r3
1011 ld r1, 0(r1)
1012 ld r4, -24(r1)
1013 ld r3, -16(r1)
1014 ld r31, -8(r1)
1016 /* Jump back to real return address */
1017 blr
1019 _GLOBAL(mod_return_to_handler)
1020 /* need to save return values */
1021 std r4, -32(r1)
1022 std r3, -24(r1)
1023 /* save TOC */
1024 std r2, -16(r1)
1025 std r31, -8(r1)
1026 mr r31, r1
1027 stdu r1, -112(r1)
1029 /*
1030 * We are in a module using the module's TOC.
1031 * Switch to our TOC to run inside the core kernel.
1032 */
1033 ld r2, PACATOC(r13)
1035 bl .ftrace_return_to_handler
1036 nop
1038 /* return value has real return address */
1039 mtlr r3
1041 ld r1, 0(r1)
1042 ld r4, -32(r1)
1043 ld r3, -24(r1)
1044 ld r2, -16(r1)
1045 ld r31, -8(r1)
1047 /* Jump back to real return address */
1048 blr
1049 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1050 #endif /* CONFIG_FUNCTION_TRACER */