| blob | f9fd54bfcc84b67a98992f3527235dc8243f314b |
1 /*
2 * PowerPC version
3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 * Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
5 * Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
6 * Adapted for Power Macintosh by Paul Mackerras.
7 * Low-level exception handlers and MMU support
8 * rewritten by Paul Mackerras.
9 * Copyright (C) 1996 Paul Mackerras.
10 * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
11 *
12 * This file contains the system call entry code, context switch
13 * code, and exception/interrupt return code for PowerPC.
14 *
15 * This program is free software; you can redistribute it and/or
16 * modify it under the terms of the GNU General Public License
17 * as published by the Free Software Foundation; either version
18 * 2 of the License, or (at your option) any later version.
19 */
21 #include <linux/errno.h>
22 #include <asm/unistd.h>
23 #include <asm/processor.h>
24 #include <asm/page.h>
25 #include <asm/mmu.h>
26 #include <asm/thread_info.h>
27 #include <asm/ppc_asm.h>
28 #include <asm/asm-offsets.h>
29 #include <asm/cputable.h>
30 #include <asm/firmware.h>
31 #include <asm/bug.h>
32 #include <asm/ptrace.h>
33 #include <asm/irqflags.h>
34 #include <asm/ftrace.h>
36 /*
37 * System calls.
38 */
39 .section ".toc","aw"
40 .SYS_CALL_TABLE:
41 .tc .sys_call_table[TC],.sys_call_table
43 /* This value is used to mark exception frames on the stack. */
44 exception_marker:
45 .tc ID_EXC_MARKER[TC],STACK_FRAME_REGS_MARKER
47 .section ".text"
48 .align 7
50 #undef SHOW_SYSCALLS
52 .globl system_call_common
53 system_call_common:
54 andi. r10,r12,MSR_PR
55 mr r10,r1
56 addi r1,r1,-INT_FRAME_SIZE
57 beq- 1f
58 ld r1,PACAKSAVE(r13)
59 1: std r10,0(r1)
60 std r11,_NIP(r1)
61 std r12,_MSR(r1)
62 std r0,GPR0(r1)
63 std r10,GPR1(r1)
64 ACCOUNT_CPU_USER_ENTRY(r10, r11)
65 /*
66 * This "crclr so" clears CR0.SO, which is the error indication on
67 * return from this system call. There must be no cmp instruction
68 * between it and the "mfcr r9" below, otherwise if XER.SO is set,
69 * CR0.SO will get set, causing all system calls to appear to fail.
70 */
71 crclr so
72 std r2,GPR2(r1)
73 std r3,GPR3(r1)
74 std r4,GPR4(r1)
75 std r5,GPR5(r1)
76 std r6,GPR6(r1)
77 std r7,GPR7(r1)
78 std r8,GPR8(r1)
79 li r11,0
80 std r11,GPR9(r1)
81 std r11,GPR10(r1)
82 std r11,GPR11(r1)
83 std r11,GPR12(r1)
84 std r9,GPR13(r1)
85 mfcr r9
86 mflr r10
87 li r11,0xc01
88 std r9,_CCR(r1)
89 std r10,_LINK(r1)
90 std r11,_TRAP(r1)
91 mfxer r9
92 mfctr r10
93 std r9,_XER(r1)
94 std r10,_CTR(r1)
95 std r3,ORIG_GPR3(r1)
96 ld r2,PACATOC(r13)
97 addi r9,r1,STACK_FRAME_OVERHEAD
98 ld r11,exception_marker@toc(r2)
99 std r11,-16(r9) /* "regshere" marker */
100 #ifdef CONFIG_TRACE_IRQFLAGS
101 bl .trace_hardirqs_on
102 REST_GPR(0,r1)
103 REST_4GPRS(3,r1)
104 REST_2GPRS(7,r1)
105 addi r9,r1,STACK_FRAME_OVERHEAD
106 ld r12,_MSR(r1)
107 #endif /* CONFIG_TRACE_IRQFLAGS */
108 li r10,1
109 stb r10,PACASOFTIRQEN(r13)
110 stb r10,PACAHARDIRQEN(r13)
111 std r10,SOFTE(r1)
112 #ifdef CONFIG_PPC_ISERIES
113 BEGIN_FW_FTR_SECTION
114 /* Hack for handling interrupts when soft-enabling on iSeries */
115 cmpdi cr1,r0,0x5555 /* syscall 0x5555 */
116 andi. r10,r12,MSR_PR /* from kernel */
117 crand 4*cr0+eq,4*cr1+eq,4*cr0+eq
118 bne 2f
119 b hardware_interrupt_entry
120 2:
121 END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
122 #endif /* CONFIG_PPC_ISERIES */
124 /* Hard enable interrupts */
125 #ifdef CONFIG_PPC_BOOK3E
126 wrteei 1
127 #else
128 mfmsr r11
129 ori r11,r11,MSR_EE
130 mtmsrd r11,1
131 #endif /* CONFIG_PPC_BOOK3E */
133 #ifdef SHOW_SYSCALLS
134 bl .do_show_syscall
135 REST_GPR(0,r1)
136 REST_4GPRS(3,r1)
137 REST_2GPRS(7,r1)
138 addi r9,r1,STACK_FRAME_OVERHEAD
139 #endif
140 clrrdi r11,r1,THREAD_SHIFT
141 ld r10,TI_FLAGS(r11)
142 andi. r11,r10,_TIF_SYSCALL_T_OR_A
143 bne- syscall_dotrace
144 syscall_dotrace_cont:
145 cmpldi 0,r0,NR_syscalls
146 bge- syscall_enosys
148 system_call: /* label this so stack traces look sane */
149 /*
150 * Need to vector to 32 Bit or default sys_call_table here,
151 * based on caller's run-mode / personality.
152 */
153 ld r11,.SYS_CALL_TABLE@toc(2)
154 andi. r10,r10,_TIF_32BIT
155 beq 15f
156 addi r11,r11,8 /* use 32-bit syscall entries */
157 clrldi r3,r3,32
158 clrldi r4,r4,32
159 clrldi r5,r5,32
160 clrldi r6,r6,32
161 clrldi r7,r7,32
162 clrldi r8,r8,32
163 15:
164 slwi r0,r0,4
165 ldx r10,r11,r0 /* Fetch system call handler [ptr] */
166 mtctr r10
167 bctrl /* Call handler */
169 syscall_exit:
170 std r3,RESULT(r1)
171 #ifdef SHOW_SYSCALLS
172 bl .do_show_syscall_exit
173 ld r3,RESULT(r1)
174 #endif
175 clrrdi r12,r1,THREAD_SHIFT
177 ld r8,_MSR(r1)
178 #ifdef CONFIG_PPC_BOOK3S
179 /* No MSR:RI on BookE */
180 andi. r10,r8,MSR_RI
181 beq- unrecov_restore
182 #endif
184 /* Disable interrupts so current_thread_info()->flags can't change,
185 * and so that we don't get interrupted after loading SRR0/1.
186 */
187 #ifdef CONFIG_PPC_BOOK3E
188 wrteei 0
189 #else
190 mfmsr r10
191 rldicl r10,r10,48,1
192 rotldi r10,r10,16
193 mtmsrd r10,1
194 #endif /* CONFIG_PPC_BOOK3E */
196 ld r9,TI_FLAGS(r12)
197 li r11,-_LAST_ERRNO
198 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
199 bne- syscall_exit_work
200 cmpld r3,r11
201 ld r5,_CCR(r1)
202 bge- syscall_error
203 syscall_error_cont:
204 ld r7,_NIP(r1)
205 stdcx. r0,0,r1 /* to clear the reservation */
206 andi. r6,r8,MSR_PR
207 ld r4,_LINK(r1)
208 /*
209 * Clear RI before restoring r13. If we are returning to
210 * userspace and we take an exception after restoring r13,
211 * we end up corrupting the userspace r13 value.
212 */
213 #ifdef CONFIG_PPC_BOOK3S
214 /* No MSR:RI on BookE */
215 li r12,MSR_RI
216 andc r11,r10,r12
217 mtmsrd r11,1 /* clear MSR.RI */
218 #endif /* CONFIG_PPC_BOOK3S */
220 beq- 1f
221 ACCOUNT_CPU_USER_EXIT(r11, r12)
222 ld r13,GPR13(r1) /* only restore r13 if returning to usermode */
223 1: ld r2,GPR2(r1)
224 ld r1,GPR1(r1)
225 mtlr r4
226 mtcr r5
227 mtspr SPRN_SRR0,r7
228 mtspr SPRN_SRR1,r8
229 RFI
230 b . /* prevent speculative execution */
232 syscall_error:
233 oris r5,r5,0x1000 /* Set SO bit in CR */
234 neg r3,r3
235 std r5,_CCR(r1)
236 b syscall_error_cont
238 /* Traced system call support */
239 syscall_dotrace:
240 bl .save_nvgprs
241 addi r3,r1,STACK_FRAME_OVERHEAD
242 bl .do_syscall_trace_enter
243 /*
244 * Restore argument registers possibly just changed.
245 * We use the return value of do_syscall_trace_enter
246 * for the call number to look up in the table (r0).
247 */
248 mr r0,r3
249 ld r3,GPR3(r1)
250 ld r4,GPR4(r1)
251 ld r5,GPR5(r1)
252 ld r6,GPR6(r1)
253 ld r7,GPR7(r1)
254 ld r8,GPR8(r1)
255 addi r9,r1,STACK_FRAME_OVERHEAD
256 clrrdi r10,r1,THREAD_SHIFT
257 ld r10,TI_FLAGS(r10)
258 b syscall_dotrace_cont
260 syscall_enosys:
261 li r3,-ENOSYS
262 b syscall_exit
264 syscall_exit_work:
265 /* If TIF_RESTOREALL is set, don't scribble on either r3 or ccr.
266 If TIF_NOERROR is set, just save r3 as it is. */
268 andi. r0,r9,_TIF_RESTOREALL
269 beq+ 0f
270 REST_NVGPRS(r1)
271 b 2f
272 0: cmpld r3,r11 /* r10 is -LAST_ERRNO */
273 blt+ 1f
274 andi. r0,r9,_TIF_NOERROR
275 bne- 1f
276 ld r5,_CCR(r1)
277 neg r3,r3
278 oris r5,r5,0x1000 /* Set SO bit in CR */
279 std r5,_CCR(r1)
280 1: std r3,GPR3(r1)
281 2: andi. r0,r9,(_TIF_PERSYSCALL_MASK)
282 beq 4f
284 /* Clear per-syscall TIF flags if any are set. */
286 li r11,_TIF_PERSYSCALL_MASK
287 addi r12,r12,TI_FLAGS
288 3: ldarx r10,0,r12
289 andc r10,r10,r11
290 stdcx. r10,0,r12
291 bne- 3b
292 subi r12,r12,TI_FLAGS
294 4: /* Anything else left to do? */
295 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP)
296 beq .ret_from_except_lite
298 /* Re-enable interrupts */
299 #ifdef CONFIG_PPC_BOOK3E
300 wrteei 1
301 #else
302 mfmsr r10
303 ori r10,r10,MSR_EE
304 mtmsrd r10,1
305 #endif /* CONFIG_PPC_BOOK3E */
307 bl .save_nvgprs
308 addi r3,r1,STACK_FRAME_OVERHEAD
309 bl .do_syscall_trace_leave
310 b .ret_from_except
312 /* Save non-volatile GPRs, if not already saved. */
313 _GLOBAL(save_nvgprs)
314 ld r11,_TRAP(r1)
315 andi. r0,r11,1
316 beqlr-
317 SAVE_NVGPRS(r1)
318 clrrdi r0,r11,1
319 std r0,_TRAP(r1)
320 blr
323 /*
324 * The sigsuspend and rt_sigsuspend system calls can call do_signal
325 * and thus put the process into the stopped state where we might
326 * want to examine its user state with ptrace. Therefore we need
327 * to save all the nonvolatile registers (r14 - r31) before calling
328 * the C code. Similarly, fork, vfork and clone need the full
329 * register state on the stack so that it can be copied to the child.
330 */
332 _GLOBAL(ppc_fork)
333 bl .save_nvgprs
334 bl .sys_fork
335 b syscall_exit
337 _GLOBAL(ppc_vfork)
338 bl .save_nvgprs
339 bl .sys_vfork
340 b syscall_exit
342 _GLOBAL(ppc_clone)
343 bl .save_nvgprs
344 bl .sys_clone
345 b syscall_exit
347 _GLOBAL(ppc32_swapcontext)
348 bl .save_nvgprs
349 bl .compat_sys_swapcontext
350 b syscall_exit
352 _GLOBAL(ppc64_swapcontext)
353 bl .save_nvgprs
354 bl .sys_swapcontext
355 b syscall_exit
357 _GLOBAL(ret_from_fork)
358 bl .schedule_tail
359 REST_NVGPRS(r1)
360 li r3,0
361 b syscall_exit
363 /*
364 * This routine switches between two different tasks. The process
365 * state of one is saved on its kernel stack. Then the state
366 * of the other is restored from its kernel stack. The memory
367 * management hardware is updated to the second process's state.
368 * Finally, we can return to the second process, via ret_from_except.
369 * On entry, r3 points to the THREAD for the current task, r4
370 * points to the THREAD for the new task.
371 *
372 * Note: there are two ways to get to the "going out" portion
373 * of this code; either by coming in via the entry (_switch)
374 * or via "fork" which must set up an environment equivalent
375 * to the "_switch" path. If you change this you'll have to change
376 * the fork code also.
377 *
378 * The code which creates the new task context is in 'copy_thread'
379 * in arch/powerpc/kernel/process.c
380 */
381 .align 7
382 _GLOBAL(_switch)
383 mflr r0
384 std r0,16(r1)
385 stdu r1,-SWITCH_FRAME_SIZE(r1)
386 /* r3-r13 are caller saved -- Cort */
387 SAVE_8GPRS(14, r1)
388 SAVE_10GPRS(22, r1)
389 mflr r20 /* Return to switch caller */
390 mfmsr r22
391 li r0, MSR_FP
392 #ifdef CONFIG_VSX
393 BEGIN_FTR_SECTION
394 oris r0,r0,MSR_VSX@h /* Disable VSX */
395 END_FTR_SECTION_IFSET(CPU_FTR_VSX)
396 #endif /* CONFIG_VSX */
397 #ifdef CONFIG_ALTIVEC
398 BEGIN_FTR_SECTION
399 oris r0,r0,MSR_VEC@h /* Disable altivec */
400 mfspr r24,SPRN_VRSAVE /* save vrsave register value */
401 std r24,THREAD_VRSAVE(r3)
402 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
403 #endif /* CONFIG_ALTIVEC */
404 and. r0,r0,r22
405 beq+ 1f
406 andc r22,r22,r0
407 MTMSRD(r22)
408 isync
409 1: std r20,_NIP(r1)
410 mfcr r23
411 std r23,_CCR(r1)
412 std r1,KSP(r3) /* Set old stack pointer */
414 #ifdef CONFIG_SMP
415 /* We need a sync somewhere here to make sure that if the
416 * previous task gets rescheduled on another CPU, it sees all
417 * stores it has performed on this one.
418 */
419 sync
420 #endif /* CONFIG_SMP */
422 addi r6,r4,-THREAD /* Convert THREAD to 'current' */
423 std r6,PACACURRENT(r13) /* Set new 'current' */
425 ld r8,KSP(r4) /* new stack pointer */
426 #ifdef CONFIG_PPC_BOOK3S
427 BEGIN_FTR_SECTION
428 BEGIN_FTR_SECTION_NESTED(95)
429 clrrdi r6,r8,28 /* get its ESID */
430 clrrdi r9,r1,28 /* get current sp ESID */
431 FTR_SECTION_ELSE_NESTED(95)
432 clrrdi r6,r8,40 /* get its 1T ESID */
433 clrrdi r9,r1,40 /* get current sp 1T ESID */
434 ALT_FTR_SECTION_END_NESTED_IFCLR(CPU_FTR_1T_SEGMENT, 95)
435 FTR_SECTION_ELSE
436 b 2f
437 ALT_FTR_SECTION_END_IFSET(CPU_FTR_SLB)
438 clrldi. r0,r6,2 /* is new ESID c00000000? */
439 cmpd cr1,r6,r9 /* or is new ESID the same as current ESID? */
440 cror eq,4*cr1+eq,eq
441 beq 2f /* if yes, don't slbie it */
443 /* Bolt in the new stack SLB entry */
444 ld r7,KSP_VSID(r4) /* Get new stack's VSID */
445 oris r0,r6,(SLB_ESID_V)@h
446 ori r0,r0,(SLB_NUM_BOLTED-1)@l
447 BEGIN_FTR_SECTION
448 li r9,MMU_SEGSIZE_1T /* insert B field */
449 oris r6,r6,(MMU_SEGSIZE_1T << SLBIE_SSIZE_SHIFT)@h
450 rldimi r7,r9,SLB_VSID_SSIZE_SHIFT,0
451 END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
453 /* Update the last bolted SLB. No write barriers are needed
454 * here, provided we only update the current CPU's SLB shadow
455 * buffer.
456 */
457 ld r9,PACA_SLBSHADOWPTR(r13)
458 li r12,0
459 std r12,SLBSHADOW_STACKESID(r9) /* Clear ESID */
460 std r7,SLBSHADOW_STACKVSID(r9) /* Save VSID */
461 std r0,SLBSHADOW_STACKESID(r9) /* Save ESID */
463 /* No need to check for CPU_FTR_NO_SLBIE_B here, since when
464 * we have 1TB segments, the only CPUs known to have the errata
465 * only support less than 1TB of system memory and we'll never
466 * actually hit this code path.
467 */
469 slbie r6
470 slbie r6 /* Workaround POWER5 < DD2.1 issue */
471 slbmte r7,r0
472 isync
473 2:
474 #endif /* !CONFIG_PPC_BOOK3S */
476 clrrdi r7,r8,THREAD_SHIFT /* base of new stack */
477 /* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE
478 because we don't need to leave the 288-byte ABI gap at the
479 top of the kernel stack. */
480 addi r7,r7,THREAD_SIZE-SWITCH_FRAME_SIZE
482 mr r1,r8 /* start using new stack pointer */
483 std r7,PACAKSAVE(r13)
485 ld r6,_CCR(r1)
486 mtcrf 0xFF,r6
488 #ifdef CONFIG_ALTIVEC
489 BEGIN_FTR_SECTION
490 ld r0,THREAD_VRSAVE(r4)
491 mtspr SPRN_VRSAVE,r0 /* if G4, restore VRSAVE reg */
492 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
493 #endif /* CONFIG_ALTIVEC */
495 /* r3-r13 are destroyed -- Cort */
496 REST_8GPRS(14, r1)
497 REST_10GPRS(22, r1)
499 /* convert old thread to its task_struct for return value */
500 addi r3,r3,-THREAD
501 ld r7,_NIP(r1) /* Return to _switch caller in new task */
502 mtlr r7
503 addi r1,r1,SWITCH_FRAME_SIZE
504 blr
506 .align 7
507 _GLOBAL(ret_from_except)
508 ld r11,_TRAP(r1)
509 andi. r0,r11,1
510 bne .ret_from_except_lite
511 REST_NVGPRS(r1)
513 _GLOBAL(ret_from_except_lite)
514 /*
515 * Disable interrupts so that current_thread_info()->flags
516 * can't change between when we test it and when we return
517 * from the interrupt.
518 */
519 #ifdef CONFIG_PPC_BOOK3E
520 wrteei 0
521 #else
522 mfmsr r10 /* Get current interrupt state */
523 rldicl r9,r10,48,1 /* clear MSR_EE */
524 rotldi r9,r9,16
525 mtmsrd r9,1 /* Update machine state */
526 #endif /* CONFIG_PPC_BOOK3E */
528 #ifdef CONFIG_PREEMPT
529 clrrdi r9,r1,THREAD_SHIFT /* current_thread_info() */
530 li r0,_TIF_NEED_RESCHED /* bits to check */
531 ld r3,_MSR(r1)
532 ld r4,TI_FLAGS(r9)
533 /* Move MSR_PR bit in r3 to _TIF_SIGPENDING position in r0 */
534 rlwimi r0,r3,32+TIF_SIGPENDING-MSR_PR_LG,_TIF_SIGPENDING
535 and. r0,r4,r0 /* check NEED_RESCHED and maybe SIGPENDING */
536 bne do_work
538 #else /* !CONFIG_PREEMPT */
539 ld r3,_MSR(r1) /* Returning to user mode? */
540 andi. r3,r3,MSR_PR
541 beq restore /* if not, just restore regs and return */
543 /* Check current_thread_info()->flags */
544 clrrdi r9,r1,THREAD_SHIFT
545 ld r4,TI_FLAGS(r9)
546 andi. r0,r4,_TIF_USER_WORK_MASK
547 bne do_work
548 #endif
550 restore:
551 BEGIN_FW_FTR_SECTION
552 ld r5,SOFTE(r1)
553 FW_FTR_SECTION_ELSE
554 b iseries_check_pending_irqs
555 ALT_FW_FTR_SECTION_END_IFCLR(FW_FEATURE_ISERIES)
556 2:
557 TRACE_AND_RESTORE_IRQ(r5);
559 #ifdef CONFIG_PERF_EVENTS
560 /* check paca->perf_event_pending if we're enabling ints */
561 lbz r3,PACAPERFPEND(r13)
562 and. r3,r3,r5
563 beq 27f
564 bl .perf_event_do_pending
565 27:
566 #endif /* CONFIG_PERF_EVENTS */
568 /* extract EE bit and use it to restore paca->hard_enabled */
569 ld r3,_MSR(r1)
570 rldicl r4,r3,49,63 /* r0 = (r3 >> 15) & 1 */
571 stb r4,PACAHARDIRQEN(r13)
573 #ifdef CONFIG_PPC_BOOK3E
574 b .exception_return_book3e
575 #else
576 ld r4,_CTR(r1)
577 ld r0,_LINK(r1)
578 mtctr r4
579 mtlr r0
580 ld r4,_XER(r1)
581 mtspr SPRN_XER,r4
583 REST_8GPRS(5, r1)
585 andi. r0,r3,MSR_RI
586 beq- unrecov_restore
588 stdcx. r0,0,r1 /* to clear the reservation */
590 /*
591 * Clear RI before restoring r13. If we are returning to
592 * userspace and we take an exception after restoring r13,
593 * we end up corrupting the userspace r13 value.
594 */
595 mfmsr r4
596 andc r4,r4,r0 /* r0 contains MSR_RI here */
597 mtmsrd r4,1
599 /*
600 * r13 is our per cpu area, only restore it if we are returning to
601 * userspace
602 */
603 andi. r0,r3,MSR_PR
604 beq 1f
605 ACCOUNT_CPU_USER_EXIT(r2, r4)
606 REST_GPR(13, r1)
607 1:
608 mtspr SPRN_SRR1,r3
610 ld r2,_CCR(r1)
611 mtcrf 0xFF,r2
612 ld r2,_NIP(r1)
613 mtspr SPRN_SRR0,r2
615 ld r0,GPR0(r1)
616 ld r2,GPR2(r1)
617 ld r3,GPR3(r1)
618 ld r4,GPR4(r1)
619 ld r1,GPR1(r1)
621 rfid
622 b . /* prevent speculative execution */
624 #endif /* CONFIG_PPC_BOOK3E */
626 iseries_check_pending_irqs:
627 #ifdef CONFIG_PPC_ISERIES
628 ld r5,SOFTE(r1)
629 cmpdi 0,r5,0
630 beq 2b
631 /* Check for pending interrupts (iSeries) */
632 ld r3,PACALPPACAPTR(r13)
633 ld r3,LPPACAANYINT(r3)
634 cmpdi r3,0
635 beq+ 2b /* skip do_IRQ if no interrupts */
637 li r3,0
638 stb r3,PACASOFTIRQEN(r13) /* ensure we are soft-disabled */
639 #ifdef CONFIG_TRACE_IRQFLAGS
640 bl .trace_hardirqs_off
641 mfmsr r10
642 #endif
643 ori r10,r10,MSR_EE
644 mtmsrd r10 /* hard-enable again */
645 addi r3,r1,STACK_FRAME_OVERHEAD
646 bl .do_IRQ
647 b .ret_from_except_lite /* loop back and handle more */
648 #endif
650 do_work:
651 #ifdef CONFIG_PREEMPT
652 andi. r0,r3,MSR_PR /* Returning to user mode? */
653 bne user_work
654 /* Check that preempt_count() == 0 and interrupts are enabled */
655 lwz r8,TI_PREEMPT(r9)
656 cmpwi cr1,r8,0
657 ld r0,SOFTE(r1)
658 cmpdi r0,0
659 crandc eq,cr1*4+eq,eq
660 bne restore
661 /* here we are preempting the current task */
662 1:
663 #ifdef CONFIG_TRACE_IRQFLAGS
664 bl .trace_hardirqs_on
665 /* Note: we just clobbered r10 which used to contain the previous
666 * MSR before the hard-disabling done by the caller of do_work.
667 * We don't have that value anymore, but it doesn't matter as
668 * we will hard-enable unconditionally, we can just reload the
669 * current MSR into r10
670 */
671 mfmsr r10
672 #endif /* CONFIG_TRACE_IRQFLAGS */
673 li r0,1
674 stb r0,PACASOFTIRQEN(r13)
675 stb r0,PACAHARDIRQEN(r13)
676 #ifdef CONFIG_PPC_BOOK3E
677 wrteei 1
678 bl .preempt_schedule
679 wrteei 0
680 #else
681 ori r10,r10,MSR_EE
682 mtmsrd r10,1 /* reenable interrupts */
683 bl .preempt_schedule
684 mfmsr r10
685 clrrdi r9,r1,THREAD_SHIFT
686 rldicl r10,r10,48,1 /* disable interrupts again */
687 rotldi r10,r10,16
688 mtmsrd r10,1
689 #endif /* CONFIG_PPC_BOOK3E */
690 ld r4,TI_FLAGS(r9)
691 andi. r0,r4,_TIF_NEED_RESCHED
692 bne 1b
693 b restore
695 user_work:
696 #endif
697 /* Enable interrupts */
698 #ifdef CONFIG_PPC_BOOK3E
699 wrteei 1
700 #else
701 ori r10,r10,MSR_EE
702 mtmsrd r10,1
703 #endif /* CONFIG_PPC_BOOK3E */
705 andi. r0,r4,_TIF_NEED_RESCHED
706 beq 1f
707 bl .schedule
708 b .ret_from_except_lite
710 1: bl .save_nvgprs
711 addi r3,r1,STACK_FRAME_OVERHEAD
712 bl .do_signal
713 b .ret_from_except
715 unrecov_restore:
716 addi r3,r1,STACK_FRAME_OVERHEAD
717 bl .unrecoverable_exception
718 b unrecov_restore
720 #ifdef CONFIG_PPC_RTAS
721 /*
722 * On CHRP, the Run-Time Abstraction Services (RTAS) have to be
723 * called with the MMU off.
724 *
725 * In addition, we need to be in 32b mode, at least for now.
726 *
727 * Note: r3 is an input parameter to rtas, so don't trash it...
728 */
729 _GLOBAL(enter_rtas)
730 mflr r0
731 std r0,16(r1)
732 stdu r1,-RTAS_FRAME_SIZE(r1) /* Save SP and create stack space. */
734 /* Because RTAS is running in 32b mode, it clobbers the high order half
735 * of all registers that it saves. We therefore save those registers
736 * RTAS might touch to the stack. (r0, r3-r13 are caller saved)
737 */
738 SAVE_GPR(2, r1) /* Save the TOC */
739 SAVE_GPR(13, r1) /* Save paca */
740 SAVE_8GPRS(14, r1) /* Save the non-volatiles */
741 SAVE_10GPRS(22, r1) /* ditto */
743 mfcr r4
744 std r4,_CCR(r1)
745 mfctr r5
746 std r5,_CTR(r1)
747 mfspr r6,SPRN_XER
748 std r6,_XER(r1)
749 mfdar r7
750 std r7,_DAR(r1)
751 mfdsisr r8
752 std r8,_DSISR(r1)
754 /* Temporary workaround to clear CR until RTAS can be modified to
755 * ignore all bits.
756 */
757 li r0,0
758 mtcr r0
760 #ifdef CONFIG_BUG
761 /* There is no way it is acceptable to get here with interrupts enabled,
762 * check it with the asm equivalent of WARN_ON
763 */
764 lbz r0,PACASOFTIRQEN(r13)
765 1: tdnei r0,0
766 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
767 #endif
769 /* Hard-disable interrupts */
770 mfmsr r6
771 rldicl r7,r6,48,1
772 rotldi r7,r7,16
773 mtmsrd r7,1
775 /* Unfortunately, the stack pointer and the MSR are also clobbered,
776 * so they are saved in the PACA which allows us to restore
777 * our original state after RTAS returns.
778 */
779 std r1,PACAR1(r13)
780 std r6,PACASAVEDMSR(r13)
782 /* Setup our real return addr */
783 LOAD_REG_ADDR(r4,.rtas_return_loc)
784 clrldi r4,r4,2 /* convert to realmode address */
785 mtlr r4
787 li r0,0
788 ori r0,r0,MSR_EE|MSR_SE|MSR_BE|MSR_RI
789 andc r0,r6,r0
791 li r9,1
792 rldicr r9,r9,MSR_SF_LG,(63-MSR_SF_LG)
793 ori r9,r9,MSR_IR|MSR_DR|MSR_FE0|MSR_FE1|MSR_FP
794 andc r6,r0,r9
795 ori r6,r6,MSR_RI
796 sync /* disable interrupts so SRR0/1 */
797 mtmsrd r0 /* don't get trashed */
799 LOAD_REG_ADDR(r4, rtas)
800 ld r5,RTASENTRY(r4) /* get the rtas->entry value */
801 ld r4,RTASBASE(r4) /* get the rtas->base value */
803 mtspr SPRN_SRR0,r5
804 mtspr SPRN_SRR1,r6
805 rfid
806 b . /* prevent speculative execution */
808 _STATIC(rtas_return_loc)
809 /* relocation is off at this point */
810 mfspr r4,SPRN_SPRG_PACA /* Get PACA */
811 clrldi r4,r4,2 /* convert to realmode address */
813 bcl 20,31,$+4
814 0: mflr r3
815 ld r3,(1f-0b)(r3) /* get &.rtas_restore_regs */
817 mfmsr r6
818 li r0,MSR_RI
819 andc r6,r6,r0
820 sync
821 mtmsrd r6
823 ld r1,PACAR1(r4) /* Restore our SP */
824 ld r4,PACASAVEDMSR(r4) /* Restore our MSR */
826 mtspr SPRN_SRR0,r3
827 mtspr SPRN_SRR1,r4
828 rfid
829 b . /* prevent speculative execution */
831 .align 3
832 1: .llong .rtas_restore_regs
834 _STATIC(rtas_restore_regs)
835 /* relocation is on at this point */
836 REST_GPR(2, r1) /* Restore the TOC */
837 REST_GPR(13, r1) /* Restore paca */
838 REST_8GPRS(14, r1) /* Restore the non-volatiles */
839 REST_10GPRS(22, r1) /* ditto */
841 mfspr r13,SPRN_SPRG_PACA
843 ld r4,_CCR(r1)
844 mtcr r4
845 ld r5,_CTR(r1)
846 mtctr r5
847 ld r6,_XER(r1)
848 mtspr SPRN_XER,r6
849 ld r7,_DAR(r1)
850 mtdar r7
851 ld r8,_DSISR(r1)
852 mtdsisr r8
854 addi r1,r1,RTAS_FRAME_SIZE /* Unstack our frame */
855 ld r0,16(r1) /* get return address */
857 mtlr r0
858 blr /* return to caller */
860 #endif /* CONFIG_PPC_RTAS */
862 _GLOBAL(enter_prom)
863 mflr r0
864 std r0,16(r1)
865 stdu r1,-PROM_FRAME_SIZE(r1) /* Save SP and create stack space */
867 /* Because PROM is running in 32b mode, it clobbers the high order half
868 * of all registers that it saves. We therefore save those registers
869 * PROM might touch to the stack. (r0, r3-r13 are caller saved)
870 */
871 SAVE_GPR(2, r1)
872 SAVE_GPR(13, r1)
873 SAVE_8GPRS(14, r1)
874 SAVE_10GPRS(22, r1)
875 mfcr r10
876 mfmsr r11
877 std r10,_CCR(r1)
878 std r11,_MSR(r1)
880 /* Get the PROM entrypoint */
881 mtlr r4
883 /* Switch MSR to 32 bits mode
884 */
885 #ifdef CONFIG_PPC_BOOK3E
886 rlwinm r11,r11,0,1,31
887 mtmsr r11
888 #else /* CONFIG_PPC_BOOK3E */
889 mfmsr r11
890 li r12,1
891 rldicr r12,r12,MSR_SF_LG,(63-MSR_SF_LG)
892 andc r11,r11,r12
893 li r12,1
894 rldicr r12,r12,MSR_ISF_LG,(63-MSR_ISF_LG)
895 andc r11,r11,r12
896 mtmsrd r11
897 #endif /* CONFIG_PPC_BOOK3E */
898 isync
900 /* Enter PROM here... */
901 blrl
903 /* Just make sure that r1 top 32 bits didn't get
904 * corrupt by OF
905 */
906 rldicl r1,r1,0,32
908 /* Restore the MSR (back to 64 bits) */
909 ld r0,_MSR(r1)
910 MTMSRD(r0)
911 isync
913 /* Restore other registers */
914 REST_GPR(2, r1)
915 REST_GPR(13, r1)
916 REST_8GPRS(14, r1)
917 REST_10GPRS(22, r1)
918 ld r4,_CCR(r1)
919 mtcr r4
921 addi r1,r1,PROM_FRAME_SIZE
922 ld r0,16(r1)
923 mtlr r0
924 blr
926 #ifdef CONFIG_FUNCTION_TRACER
927 #ifdef CONFIG_DYNAMIC_FTRACE
928 _GLOBAL(mcount)
929 _GLOBAL(_mcount)
930 blr
932 _GLOBAL(ftrace_caller)
933 /* Taken from output of objdump from lib64/glibc */
934 mflr r3
935 ld r11, 0(r1)
936 stdu r1, -112(r1)
937 std r3, 128(r1)
938 ld r4, 16(r11)
939 subi r3, r3, MCOUNT_INSN_SIZE
940 .globl ftrace_call
941 ftrace_call:
942 bl ftrace_stub
943 nop
944 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
945 .globl ftrace_graph_call
946 ftrace_graph_call:
947 b ftrace_graph_stub
948 _GLOBAL(ftrace_graph_stub)
949 #endif
950 ld r0, 128(r1)
951 mtlr r0
952 addi r1, r1, 112
953 _GLOBAL(ftrace_stub)
954 blr
955 #else
956 _GLOBAL(mcount)
957 blr
959 _GLOBAL(_mcount)
960 /* Taken from output of objdump from lib64/glibc */
961 mflr r3
962 ld r11, 0(r1)
963 stdu r1, -112(r1)
964 std r3, 128(r1)
965 ld r4, 16(r11)
967 subi r3, r3, MCOUNT_INSN_SIZE
968 LOAD_REG_ADDR(r5,ftrace_trace_function)
969 ld r5,0(r5)
970 ld r5,0(r5)
971 mtctr r5
972 bctrl
973 nop
976 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
977 b ftrace_graph_caller
978 #endif
979 ld r0, 128(r1)
980 mtlr r0
981 addi r1, r1, 112
982 _GLOBAL(ftrace_stub)
983 blr
985 #endif /* CONFIG_DYNAMIC_FTRACE */
987 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
988 _GLOBAL(ftrace_graph_caller)
989 /* load r4 with local address */
990 ld r4, 128(r1)
991 subi r4, r4, MCOUNT_INSN_SIZE
993 /* get the parent address */
994 ld r11, 112(r1)
995 addi r3, r11, 16
997 bl .prepare_ftrace_return
998 nop
1000 ld r0, 128(r1)
1001 mtlr r0
1002 addi r1, r1, 112
1003 blr
1005 _GLOBAL(return_to_handler)
1006 /* need to save return values */
1007 std r4, -24(r1)
1008 std r3, -16(r1)
1009 std r31, -8(r1)
1010 mr r31, r1
1011 stdu r1, -112(r1)
1013 bl .ftrace_return_to_handler
1014 nop
1016 /* return value has real return address */
1017 mtlr r3
1019 ld r1, 0(r1)
1020 ld r4, -24(r1)
1021 ld r3, -16(r1)
1022 ld r31, -8(r1)
1024 /* Jump back to real return address */
1025 blr
1027 _GLOBAL(mod_return_to_handler)
1028 /* need to save return values */
1029 std r4, -32(r1)
1030 std r3, -24(r1)
1031 /* save TOC */
1032 std r2, -16(r1)
1033 std r31, -8(r1)
1034 mr r31, r1
1035 stdu r1, -112(r1)
1037 /*
1038 * We are in a module using the module's TOC.
1039 * Switch to our TOC to run inside the core kernel.
1040 */
1041 ld r2, PACATOC(r13)
1043 bl .ftrace_return_to_handler
1044 nop
1046 /* return value has real return address */
1047 mtlr r3
1049 ld r1, 0(r1)
1050 ld r4, -32(r1)
1051 ld r3, -24(r1)
1052 ld r2, -16(r1)
1053 ld r31, -8(r1)
1055 /* Jump back to real return address */
1056 blr
1057 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1058 #endif /* CONFIG_FUNCTION_TRACER */