| blob | 56212bc0ab087c4b40bb21df3496c89849a01f06 |
1 /*
2 * PowerPC version
3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 * Rewritten by Cort Dougan (cort@fsmlabs.com) for PReP
5 * Copyright (C) 1996 Cort Dougan <cort@fsmlabs.com>
6 * Adapted for Power Macintosh by Paul Mackerras.
7 * Low-level exception handlers and MMU support
8 * rewritten by Paul Mackerras.
9 * Copyright (C) 1996 Paul Mackerras.
10 * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
11 *
12 * This file contains the system call entry code, context switch
13 * code, and exception/interrupt return code for PowerPC.
14 *
15 * This program is free software; you can redistribute it and/or
16 * modify it under the terms of the GNU General Public License
17 * as published by the Free Software Foundation; either version
18 * 2 of the License, or (at your option) any later version.
19 *
20 */
22 #include <linux/errno.h>
23 #include <linux/sys.h>
24 #include <linux/threads.h>
25 #include <asm/reg.h>
26 #include <asm/page.h>
27 #include <asm/mmu.h>
28 #include <asm/cputable.h>
29 #include <asm/thread_info.h>
30 #include <asm/ppc_asm.h>
31 #include <asm/asm-offsets.h>
32 #include <asm/unistd.h>
33 #include <asm/ftrace.h>
34 #include <asm/ptrace.h>
36 #undef SHOW_SYSCALLS
37 #undef SHOW_SYSCALLS_TASK
39 /*
40 * MSR_KERNEL is > 0x10000 on 4xx/Book-E since it include MSR_CE.
41 */
42 #if MSR_KERNEL >= 0x10000
43 #define LOAD_MSR_KERNEL(r, x) lis r,(x)@h; ori r,r,(x)@l
44 #else
45 #define LOAD_MSR_KERNEL(r, x) li r,(x)
46 #endif
48 #ifdef CONFIG_BOOKE
49 .globl mcheck_transfer_to_handler
50 mcheck_transfer_to_handler:
51 mfspr r0,SPRN_DSRR0
52 stw r0,_DSRR0(r11)
53 mfspr r0,SPRN_DSRR1
54 stw r0,_DSRR1(r11)
55 /* fall through */
57 .globl debug_transfer_to_handler
58 debug_transfer_to_handler:
59 mfspr r0,SPRN_CSRR0
60 stw r0,_CSRR0(r11)
61 mfspr r0,SPRN_CSRR1
62 stw r0,_CSRR1(r11)
63 /* fall through */
65 .globl crit_transfer_to_handler
66 crit_transfer_to_handler:
67 #ifdef CONFIG_PPC_BOOK3E_MMU
68 mfspr r0,SPRN_MAS0
69 stw r0,MAS0(r11)
70 mfspr r0,SPRN_MAS1
71 stw r0,MAS1(r11)
72 mfspr r0,SPRN_MAS2
73 stw r0,MAS2(r11)
74 mfspr r0,SPRN_MAS3
75 stw r0,MAS3(r11)
76 mfspr r0,SPRN_MAS6
77 stw r0,MAS6(r11)
78 #ifdef CONFIG_PHYS_64BIT
79 mfspr r0,SPRN_MAS7
80 stw r0,MAS7(r11)
81 #endif /* CONFIG_PHYS_64BIT */
82 #endif /* CONFIG_PPC_BOOK3E_MMU */
83 #ifdef CONFIG_44x
84 mfspr r0,SPRN_MMUCR
85 stw r0,MMUCR(r11)
86 #endif
87 mfspr r0,SPRN_SRR0
88 stw r0,_SRR0(r11)
89 mfspr r0,SPRN_SRR1
90 stw r0,_SRR1(r11)
92 mfspr r8,SPRN_SPRG_THREAD
93 lwz r0,KSP_LIMIT(r8)
94 stw r0,SAVED_KSP_LIMIT(r11)
95 rlwimi r0,r1,0,0,(31-THREAD_SHIFT)
96 stw r0,KSP_LIMIT(r8)
97 /* fall through */
98 #endif
100 #ifdef CONFIG_40x
101 .globl crit_transfer_to_handler
102 crit_transfer_to_handler:
103 lwz r0,crit_r10@l(0)
104 stw r0,GPR10(r11)
105 lwz r0,crit_r11@l(0)
106 stw r0,GPR11(r11)
107 mfspr r0,SPRN_SRR0
108 stw r0,crit_srr0@l(0)
109 mfspr r0,SPRN_SRR1
110 stw r0,crit_srr1@l(0)
112 mfspr r8,SPRN_SPRG_THREAD
113 lwz r0,KSP_LIMIT(r8)
114 stw r0,saved_ksp_limit@l(0)
115 rlwimi r0,r1,0,0,(31-THREAD_SHIFT)
116 stw r0,KSP_LIMIT(r8)
117 /* fall through */
118 #endif
120 /*
121 * This code finishes saving the registers to the exception frame
122 * and jumps to the appropriate handler for the exception, turning
123 * on address translation.
124 * Note that we rely on the caller having set cr0.eq iff the exception
125 * occurred in kernel mode (i.e. MSR:PR = 0).
126 */
127 .globl transfer_to_handler_full
128 transfer_to_handler_full:
129 SAVE_NVGPRS(r11)
130 /* fall through */
132 .globl transfer_to_handler
133 transfer_to_handler:
134 stw r2,GPR2(r11)
135 stw r12,_NIP(r11)
136 stw r9,_MSR(r11)
137 andi. r2,r9,MSR_PR
138 mfctr r12
139 mfspr r2,SPRN_XER
140 stw r12,_CTR(r11)
141 stw r2,_XER(r11)
142 mfspr r12,SPRN_SPRG_THREAD
143 addi r2,r12,-THREAD
144 tovirt(r2,r2) /* set r2 to current */
145 beq 2f /* if from user, fix up THREAD.regs */
146 addi r11,r1,STACK_FRAME_OVERHEAD
147 stw r11,PT_REGS(r12)
148 #if defined(CONFIG_40x) || defined(CONFIG_BOOKE)
149 /* Check to see if the dbcr0 register is set up to debug. Use the
150 internal debug mode bit to do this. */
151 lwz r12,THREAD_DBCR0(r12)
152 andis. r12,r12,DBCR0_IDM@h
153 beq+ 3f
154 /* From user and task is ptraced - load up global dbcr0 */
155 li r12,-1 /* clear all pending debug events */
156 mtspr SPRN_DBSR,r12
157 lis r11,global_dbcr0@ha
158 tophys(r11,r11)
159 addi r11,r11,global_dbcr0@l
160 #ifdef CONFIG_SMP
161 rlwinm r9,r1,0,0,(31-THREAD_SHIFT)
162 lwz r9,TI_CPU(r9)
163 slwi r9,r9,3
164 add r11,r11,r9
165 #endif
166 lwz r12,0(r11)
167 mtspr SPRN_DBCR0,r12
168 lwz r12,4(r11)
169 addi r12,r12,-1
170 stw r12,4(r11)
171 #endif
172 b 3f
174 2: /* if from kernel, check interrupted DOZE/NAP mode and
175 * check for stack overflow
176 */
177 lwz r9,KSP_LIMIT(r12)
178 cmplw r1,r9 /* if r1 <= ksp_limit */
179 ble- stack_ovf /* then the kernel stack overflowed */
180 5:
181 #if defined(CONFIG_6xx) || defined(CONFIG_E500)
182 rlwinm r9,r1,0,0,31-THREAD_SHIFT
183 tophys(r9,r9) /* check local flags */
184 lwz r12,TI_LOCAL_FLAGS(r9)
185 mtcrf 0x01,r12
186 bt- 31-TLF_NAPPING,4f
187 bt- 31-TLF_SLEEPING,7f
188 #endif /* CONFIG_6xx || CONFIG_E500 */
189 .globl transfer_to_handler_cont
190 transfer_to_handler_cont:
191 3:
192 mflr r9
193 lwz r11,0(r9) /* virtual address of handler */
194 lwz r9,4(r9) /* where to go when done */
195 #ifdef CONFIG_TRACE_IRQFLAGS
196 lis r12,reenable_mmu@h
197 ori r12,r12,reenable_mmu@l
198 mtspr SPRN_SRR0,r12
199 mtspr SPRN_SRR1,r10
200 SYNC
201 RFI
202 reenable_mmu: /* re-enable mmu so we can */
203 mfmsr r10
204 lwz r12,_MSR(r1)
205 xor r10,r10,r12
206 andi. r10,r10,MSR_EE /* Did EE change? */
207 beq 1f
209 /* Save handler and return address into the 2 unused words
210 * of the STACK_FRAME_OVERHEAD (sneak sneak sneak). Everything
211 * else can be recovered from the pt_regs except r3 which for
212 * normal interrupts has been set to pt_regs and for syscalls
213 * is an argument, so we temporarily use ORIG_GPR3 to save it
214 */
215 stw r9,8(r1)
216 stw r11,12(r1)
217 stw r3,ORIG_GPR3(r1)
218 bl trace_hardirqs_off
219 lwz r0,GPR0(r1)
220 lwz r3,ORIG_GPR3(r1)
221 lwz r4,GPR4(r1)
222 lwz r5,GPR5(r1)
223 lwz r6,GPR6(r1)
224 lwz r7,GPR7(r1)
225 lwz r8,GPR8(r1)
226 lwz r9,8(r1)
227 lwz r11,12(r1)
228 1: mtctr r11
229 mtlr r9
230 bctr /* jump to handler */
231 #else /* CONFIG_TRACE_IRQFLAGS */
232 mtspr SPRN_SRR0,r11
233 mtspr SPRN_SRR1,r10
234 mtlr r9
235 SYNC
236 RFI /* jump to handler, enable MMU */
237 #endif /* CONFIG_TRACE_IRQFLAGS */
239 #if defined (CONFIG_6xx) || defined(CONFIG_E500)
240 4: rlwinm r12,r12,0,~_TLF_NAPPING
241 stw r12,TI_LOCAL_FLAGS(r9)
242 b power_save_ppc32_restore
244 7: rlwinm r12,r12,0,~_TLF_SLEEPING
245 stw r12,TI_LOCAL_FLAGS(r9)
246 lwz r9,_MSR(r11) /* if sleeping, clear MSR.EE */
247 rlwinm r9,r9,0,~MSR_EE
248 lwz r12,_LINK(r11) /* and return to address in LR */
249 b fast_exception_return
250 #endif
252 /*
253 * On kernel stack overflow, load up an initial stack pointer
254 * and call StackOverflow(regs), which should not return.
255 */
256 stack_ovf:
257 /* sometimes we use a statically-allocated stack, which is OK. */
258 lis r12,_end@h
259 ori r12,r12,_end@l
260 cmplw r1,r12
261 ble 5b /* r1 <= &_end is OK */
262 SAVE_NVGPRS(r11)
263 addi r3,r1,STACK_FRAME_OVERHEAD
264 lis r1,init_thread_union@ha
265 addi r1,r1,init_thread_union@l
266 addi r1,r1,THREAD_SIZE-STACK_FRAME_OVERHEAD
267 lis r9,StackOverflow@ha
268 addi r9,r9,StackOverflow@l
269 LOAD_MSR_KERNEL(r10,MSR_KERNEL)
270 FIX_SRR1(r10,r12)
271 mtspr SPRN_SRR0,r9
272 mtspr SPRN_SRR1,r10
273 SYNC
274 RFI
276 /*
277 * Handle a system call.
278 */
279 .stabs "arch/powerpc/kernel/",N_SO,0,0,0f
280 .stabs "entry_32.S",N_SO,0,0,0f
281 0:
283 _GLOBAL(DoSyscall)
284 stw r3,ORIG_GPR3(r1)
285 li r12,0
286 stw r12,RESULT(r1)
287 lwz r11,_CCR(r1) /* Clear SO bit in CR */
288 rlwinm r11,r11,0,4,2
289 stw r11,_CCR(r1)
290 #ifdef SHOW_SYSCALLS
291 bl do_show_syscall
292 #endif /* SHOW_SYSCALLS */
293 #ifdef CONFIG_TRACE_IRQFLAGS
294 /* Return from syscalls can (and generally will) hard enable
295 * interrupts. You aren't supposed to call a syscall with
296 * interrupts disabled in the first place. However, to ensure
297 * that we get it right vs. lockdep if it happens, we force
298 * that hard enable here with appropriate tracing if we see
299 * that we have been called with interrupts off
300 */
301 mfmsr r11
302 andi. r12,r11,MSR_EE
303 bne+ 1f
304 /* We came in with interrupts disabled, we enable them now */
305 bl trace_hardirqs_on
306 mfmsr r11
307 lwz r0,GPR0(r1)
308 lwz r3,GPR3(r1)
309 lwz r4,GPR4(r1)
310 ori r11,r11,MSR_EE
311 lwz r5,GPR5(r1)
312 lwz r6,GPR6(r1)
313 lwz r7,GPR7(r1)
314 lwz r8,GPR8(r1)
315 mtmsr r11
316 1:
317 #endif /* CONFIG_TRACE_IRQFLAGS */
318 rlwinm r10,r1,0,0,(31-THREAD_SHIFT) /* current_thread_info() */
319 lwz r11,TI_FLAGS(r10)
320 andi. r11,r11,_TIF_SYSCALL_T_OR_A
321 bne- syscall_dotrace
322 syscall_dotrace_cont:
323 cmplwi 0,r0,NR_syscalls
324 lis r10,sys_call_table@h
325 ori r10,r10,sys_call_table@l
326 slwi r0,r0,2
327 bge- 66f
328 lwzx r10,r10,r0 /* Fetch system call handler [ptr] */
329 mtlr r10
330 addi r9,r1,STACK_FRAME_OVERHEAD
331 PPC440EP_ERR42
332 blrl /* Call handler */
333 .globl ret_from_syscall
334 ret_from_syscall:
335 #ifdef SHOW_SYSCALLS
336 bl do_show_syscall_exit
337 #endif
338 mr r6,r3
339 rlwinm r12,r1,0,0,(31-THREAD_SHIFT) /* current_thread_info() */
340 /* disable interrupts so current_thread_info()->flags can't change */
341 LOAD_MSR_KERNEL(r10,MSR_KERNEL) /* doesn't include MSR_EE */
342 /* Note: We don't bother telling lockdep about it */
343 SYNC
344 MTMSRD(r10)
345 lwz r9,TI_FLAGS(r12)
346 li r8,-_LAST_ERRNO
347 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
348 bne- syscall_exit_work
349 cmplw 0,r3,r8
350 blt+ syscall_exit_cont
351 lwz r11,_CCR(r1) /* Load CR */
352 neg r3,r3
353 oris r11,r11,0x1000 /* Set SO bit in CR */
354 stw r11,_CCR(r1)
355 syscall_exit_cont:
356 lwz r8,_MSR(r1)
357 #ifdef CONFIG_TRACE_IRQFLAGS
358 /* If we are going to return from the syscall with interrupts
359 * off, we trace that here. It shouldn't happen though but we
360 * want to catch the bugger if it does right ?
361 */
362 andi. r10,r8,MSR_EE
363 bne+ 1f
364 stw r3,GPR3(r1)
365 bl trace_hardirqs_off
366 lwz r3,GPR3(r1)
367 1:
368 #endif /* CONFIG_TRACE_IRQFLAGS */
369 #if defined(CONFIG_4xx) || defined(CONFIG_BOOKE)
370 /* If the process has its own DBCR0 value, load it up. The internal
371 debug mode bit tells us that dbcr0 should be loaded. */
372 lwz r0,THREAD+THREAD_DBCR0(r2)
373 andis. r10,r0,DBCR0_IDM@h
374 bnel- load_dbcr0
375 #endif
376 #ifdef CONFIG_44x
377 BEGIN_MMU_FTR_SECTION
378 lis r4,icache_44x_need_flush@ha
379 lwz r5,icache_44x_need_flush@l(r4)
380 cmplwi cr0,r5,0
381 bne- 2f
382 1:
383 END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_47x)
384 #endif /* CONFIG_44x */
385 BEGIN_FTR_SECTION
386 lwarx r7,0,r1
387 END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX)
388 stwcx. r0,0,r1 /* to clear the reservation */
389 lwz r4,_LINK(r1)
390 lwz r5,_CCR(r1)
391 mtlr r4
392 mtcr r5
393 lwz r7,_NIP(r1)
394 FIX_SRR1(r8, r0)
395 lwz r2,GPR2(r1)
396 lwz r1,GPR1(r1)
397 mtspr SPRN_SRR0,r7
398 mtspr SPRN_SRR1,r8
399 SYNC
400 RFI
401 #ifdef CONFIG_44x
402 2: li r7,0
403 iccci r0,r0
404 stw r7,icache_44x_need_flush@l(r4)
405 b 1b
406 #endif /* CONFIG_44x */
408 66: li r3,-ENOSYS
409 b ret_from_syscall
411 .globl ret_from_fork
412 ret_from_fork:
413 REST_NVGPRS(r1)
414 bl schedule_tail
415 li r3,0
416 b ret_from_syscall
418 /* Traced system call support */
419 syscall_dotrace:
420 SAVE_NVGPRS(r1)
421 li r0,0xc00
422 stw r0,_TRAP(r1)
423 addi r3,r1,STACK_FRAME_OVERHEAD
424 bl do_syscall_trace_enter
425 /*
426 * Restore argument registers possibly just changed.
427 * We use the return value of do_syscall_trace_enter
428 * for call number to look up in the table (r0).
429 */
430 mr r0,r3
431 lwz r3,GPR3(r1)
432 lwz r4,GPR4(r1)
433 lwz r5,GPR5(r1)
434 lwz r6,GPR6(r1)
435 lwz r7,GPR7(r1)
436 lwz r8,GPR8(r1)
437 REST_NVGPRS(r1)
438 b syscall_dotrace_cont
440 syscall_exit_work:
441 andi. r0,r9,_TIF_RESTOREALL
442 beq+ 0f
443 REST_NVGPRS(r1)
444 b 2f
445 0: cmplw 0,r3,r8
446 blt+ 1f
447 andi. r0,r9,_TIF_NOERROR
448 bne- 1f
449 lwz r11,_CCR(r1) /* Load CR */
450 neg r3,r3
451 oris r11,r11,0x1000 /* Set SO bit in CR */
452 stw r11,_CCR(r1)
454 1: stw r6,RESULT(r1) /* Save result */
455 stw r3,GPR3(r1) /* Update return value */
456 2: andi. r0,r9,(_TIF_PERSYSCALL_MASK)
457 beq 4f
459 /* Clear per-syscall TIF flags if any are set. */
461 li r11,_TIF_PERSYSCALL_MASK
462 addi r12,r12,TI_FLAGS
463 3: lwarx r8,0,r12
464 andc r8,r8,r11
465 #ifdef CONFIG_IBM405_ERR77
466 dcbt 0,r12
467 #endif
468 stwcx. r8,0,r12
469 bne- 3b
470 subi r12,r12,TI_FLAGS
472 4: /* Anything which requires enabling interrupts? */
473 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP)
474 beq ret_from_except
476 /* Re-enable interrupts. There is no need to trace that with
477 * lockdep as we are supposed to have IRQs on at this point
478 */
479 ori r10,r10,MSR_EE
480 SYNC
481 MTMSRD(r10)
483 /* Save NVGPRS if they're not saved already */
484 lwz r4,_TRAP(r1)
485 andi. r4,r4,1
486 beq 5f
487 SAVE_NVGPRS(r1)
488 li r4,0xc00
489 stw r4,_TRAP(r1)
490 5:
491 addi r3,r1,STACK_FRAME_OVERHEAD
492 bl do_syscall_trace_leave
493 b ret_from_except_full
495 #ifdef SHOW_SYSCALLS
496 do_show_syscall:
497 #ifdef SHOW_SYSCALLS_TASK
498 lis r11,show_syscalls_task@ha
499 lwz r11,show_syscalls_task@l(r11)
500 cmp 0,r2,r11
501 bnelr
502 #endif
503 stw r31,GPR31(r1)
504 mflr r31
505 lis r3,7f@ha
506 addi r3,r3,7f@l
507 lwz r4,GPR0(r1)
508 lwz r5,GPR3(r1)
509 lwz r6,GPR4(r1)
510 lwz r7,GPR5(r1)
511 lwz r8,GPR6(r1)
512 lwz r9,GPR7(r1)
513 bl printk
514 lis r3,77f@ha
515 addi r3,r3,77f@l
516 lwz r4,GPR8(r1)
517 mr r5,r2
518 bl printk
519 lwz r0,GPR0(r1)
520 lwz r3,GPR3(r1)
521 lwz r4,GPR4(r1)
522 lwz r5,GPR5(r1)
523 lwz r6,GPR6(r1)
524 lwz r7,GPR7(r1)
525 lwz r8,GPR8(r1)
526 mtlr r31
527 lwz r31,GPR31(r1)
528 blr
530 do_show_syscall_exit:
531 #ifdef SHOW_SYSCALLS_TASK
532 lis r11,show_syscalls_task@ha
533 lwz r11,show_syscalls_task@l(r11)
534 cmp 0,r2,r11
535 bnelr
536 #endif
537 stw r31,GPR31(r1)
538 mflr r31
539 stw r3,RESULT(r1) /* Save result */
540 mr r4,r3
541 lis r3,79f@ha
542 addi r3,r3,79f@l
543 bl printk
544 lwz r3,RESULT(r1)
545 mtlr r31
546 lwz r31,GPR31(r1)
547 blr
549 7: .string "syscall %d(%x, %x, %x, %x, %x, "
550 77: .string "%x), current=%p\n"
551 79: .string " -> %x\n"
552 .align 2,0
554 #ifdef SHOW_SYSCALLS_TASK
555 .data
556 .globl show_syscalls_task
557 show_syscalls_task:
558 .long -1
559 .text
560 #endif
561 #endif /* SHOW_SYSCALLS */
563 /*
564 * The fork/clone functions need to copy the full register set into
565 * the child process. Therefore we need to save all the nonvolatile
566 * registers (r13 - r31) before calling the C code.
567 */
568 .globl ppc_fork
569 ppc_fork:
570 SAVE_NVGPRS(r1)
571 lwz r0,_TRAP(r1)
572 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
573 stw r0,_TRAP(r1) /* register set saved */
574 b sys_fork
576 .globl ppc_vfork
577 ppc_vfork:
578 SAVE_NVGPRS(r1)
579 lwz r0,_TRAP(r1)
580 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
581 stw r0,_TRAP(r1) /* register set saved */
582 b sys_vfork
584 .globl ppc_clone
585 ppc_clone:
586 SAVE_NVGPRS(r1)
587 lwz r0,_TRAP(r1)
588 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
589 stw r0,_TRAP(r1) /* register set saved */
590 b sys_clone
592 .globl ppc_swapcontext
593 ppc_swapcontext:
594 SAVE_NVGPRS(r1)
595 lwz r0,_TRAP(r1)
596 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
597 stw r0,_TRAP(r1) /* register set saved */
598 b sys_swapcontext
600 /*
601 * Top-level page fault handling.
602 * This is in assembler because if do_page_fault tells us that
603 * it is a bad kernel page fault, we want to save the non-volatile
604 * registers before calling bad_page_fault.
605 */
606 .globl handle_page_fault
607 handle_page_fault:
608 stw r4,_DAR(r1)
609 addi r3,r1,STACK_FRAME_OVERHEAD
610 bl do_page_fault
611 cmpwi r3,0
612 beq+ ret_from_except
613 SAVE_NVGPRS(r1)
614 lwz r0,_TRAP(r1)
615 clrrwi r0,r0,1
616 stw r0,_TRAP(r1)
617 mr r5,r3
618 addi r3,r1,STACK_FRAME_OVERHEAD
619 lwz r4,_DAR(r1)
620 bl bad_page_fault
621 b ret_from_except_full
623 /*
624 * This routine switches between two different tasks. The process
625 * state of one is saved on its kernel stack. Then the state
626 * of the other is restored from its kernel stack. The memory
627 * management hardware is updated to the second process's state.
628 * Finally, we can return to the second process.
629 * On entry, r3 points to the THREAD for the current task, r4
630 * points to the THREAD for the new task.
631 *
632 * This routine is always called with interrupts disabled.
633 *
634 * Note: there are two ways to get to the "going out" portion
635 * of this code; either by coming in via the entry (_switch)
636 * or via "fork" which must set up an environment equivalent
637 * to the "_switch" path. If you change this , you'll have to
638 * change the fork code also.
639 *
640 * The code which creates the new task context is in 'copy_thread'
641 * in arch/ppc/kernel/process.c
642 */
643 _GLOBAL(_switch)
644 stwu r1,-INT_FRAME_SIZE(r1)
645 mflr r0
646 stw r0,INT_FRAME_SIZE+4(r1)
647 /* r3-r12 are caller saved -- Cort */
648 SAVE_NVGPRS(r1)
649 stw r0,_NIP(r1) /* Return to switch caller */
650 mfmsr r11
651 li r0,MSR_FP /* Disable floating-point */
652 #ifdef CONFIG_ALTIVEC
653 BEGIN_FTR_SECTION
654 oris r0,r0,MSR_VEC@h /* Disable altivec */
655 mfspr r12,SPRN_VRSAVE /* save vrsave register value */
656 stw r12,THREAD+THREAD_VRSAVE(r2)
657 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
658 #endif /* CONFIG_ALTIVEC */
659 #ifdef CONFIG_SPE
660 BEGIN_FTR_SECTION
661 oris r0,r0,MSR_SPE@h /* Disable SPE */
662 mfspr r12,SPRN_SPEFSCR /* save spefscr register value */
663 stw r12,THREAD+THREAD_SPEFSCR(r2)
664 END_FTR_SECTION_IFSET(CPU_FTR_SPE)
665 #endif /* CONFIG_SPE */
666 and. r0,r0,r11 /* FP or altivec or SPE enabled? */
667 beq+ 1f
668 andc r11,r11,r0
669 MTMSRD(r11)
670 isync
671 1: stw r11,_MSR(r1)
672 mfcr r10
673 stw r10,_CCR(r1)
674 stw r1,KSP(r3) /* Set old stack pointer */
676 #ifdef CONFIG_SMP
677 /* We need a sync somewhere here to make sure that if the
678 * previous task gets rescheduled on another CPU, it sees all
679 * stores it has performed on this one.
680 */
681 sync
682 #endif /* CONFIG_SMP */
684 tophys(r0,r4)
685 CLR_TOP32(r0)
686 mtspr SPRN_SPRG_THREAD,r0 /* Update current THREAD phys addr */
687 lwz r1,KSP(r4) /* Load new stack pointer */
689 /* save the old current 'last' for return value */
690 mr r3,r2
691 addi r2,r4,-THREAD /* Update current */
693 #ifdef CONFIG_ALTIVEC
694 BEGIN_FTR_SECTION
695 lwz r0,THREAD+THREAD_VRSAVE(r2)
696 mtspr SPRN_VRSAVE,r0 /* if G4, restore VRSAVE reg */
697 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
698 #endif /* CONFIG_ALTIVEC */
699 #ifdef CONFIG_SPE
700 BEGIN_FTR_SECTION
701 lwz r0,THREAD+THREAD_SPEFSCR(r2)
702 mtspr SPRN_SPEFSCR,r0 /* restore SPEFSCR reg */
703 END_FTR_SECTION_IFSET(CPU_FTR_SPE)
704 #endif /* CONFIG_SPE */
706 lwz r0,_CCR(r1)
707 mtcrf 0xFF,r0
708 /* r3-r12 are destroyed -- Cort */
709 REST_NVGPRS(r1)
711 lwz r4,_NIP(r1) /* Return to _switch caller in new task */
712 mtlr r4
713 addi r1,r1,INT_FRAME_SIZE
714 blr
716 .globl fast_exception_return
717 fast_exception_return:
718 #if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
719 andi. r10,r9,MSR_RI /* check for recoverable interrupt */
720 beq 1f /* if not, we've got problems */
721 #endif
723 2: REST_4GPRS(3, r11)
724 lwz r10,_CCR(r11)
725 REST_GPR(1, r11)
726 mtcr r10
727 lwz r10,_LINK(r11)
728 mtlr r10
729 REST_GPR(10, r11)
730 mtspr SPRN_SRR1,r9
731 mtspr SPRN_SRR0,r12
732 REST_GPR(9, r11)
733 REST_GPR(12, r11)
734 lwz r11,GPR11(r11)
735 SYNC
736 RFI
738 #if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
739 /* check if the exception happened in a restartable section */
740 1: lis r3,exc_exit_restart_end@ha
741 addi r3,r3,exc_exit_restart_end@l
742 cmplw r12,r3
743 bge 3f
744 lis r4,exc_exit_restart@ha
745 addi r4,r4,exc_exit_restart@l
746 cmplw r12,r4
747 blt 3f
748 lis r3,fee_restarts@ha
749 tophys(r3,r3)
750 lwz r5,fee_restarts@l(r3)
751 addi r5,r5,1
752 stw r5,fee_restarts@l(r3)
753 mr r12,r4 /* restart at exc_exit_restart */
754 b 2b
756 .section .bss
757 .align 2
758 fee_restarts:
759 .space 4
760 .previous
762 /* aargh, a nonrecoverable interrupt, panic */
763 /* aargh, we don't know which trap this is */
764 /* but the 601 doesn't implement the RI bit, so assume it's OK */
765 3:
766 BEGIN_FTR_SECTION
767 b 2b
768 END_FTR_SECTION_IFSET(CPU_FTR_601)
769 li r10,-1
770 stw r10,_TRAP(r11)
771 addi r3,r1,STACK_FRAME_OVERHEAD
772 lis r10,MSR_KERNEL@h
773 ori r10,r10,MSR_KERNEL@l
774 bl transfer_to_handler_full
775 .long nonrecoverable_exception
776 .long ret_from_except
777 #endif
779 .globl ret_from_except_full
780 ret_from_except_full:
781 REST_NVGPRS(r1)
782 /* fall through */
784 .globl ret_from_except
785 ret_from_except:
786 /* Hard-disable interrupts so that current_thread_info()->flags
787 * can't change between when we test it and when we return
788 * from the interrupt. */
789 /* Note: We don't bother telling lockdep about it */
790 LOAD_MSR_KERNEL(r10,MSR_KERNEL)
791 SYNC /* Some chip revs have problems here... */
792 MTMSRD(r10) /* disable interrupts */
794 lwz r3,_MSR(r1) /* Returning to user mode? */
795 andi. r0,r3,MSR_PR
796 beq resume_kernel
798 user_exc_return: /* r10 contains MSR_KERNEL here */
799 /* Check current_thread_info()->flags */
800 rlwinm r9,r1,0,0,(31-THREAD_SHIFT)
801 lwz r9,TI_FLAGS(r9)
802 andi. r0,r9,_TIF_USER_WORK_MASK
803 bne do_work
805 restore_user:
806 #if defined(CONFIG_4xx) || defined(CONFIG_BOOKE)
807 /* Check whether this process has its own DBCR0 value. The internal
808 debug mode bit tells us that dbcr0 should be loaded. */
809 lwz r0,THREAD+THREAD_DBCR0(r2)
810 andis. r10,r0,DBCR0_IDM@h
811 bnel- load_dbcr0
812 #endif
814 #ifdef CONFIG_PREEMPT
815 b restore
817 /* N.B. the only way to get here is from the beq following ret_from_except. */
818 resume_kernel:
819 /* check current_thread_info->preempt_count */
820 rlwinm r9,r1,0,0,(31-THREAD_SHIFT)
821 lwz r0,TI_PREEMPT(r9)
822 cmpwi 0,r0,0 /* if non-zero, just restore regs and return */
823 bne restore
824 lwz r0,TI_FLAGS(r9)
825 andi. r0,r0,_TIF_NEED_RESCHED
826 beq+ restore
827 andi. r0,r3,MSR_EE /* interrupts off? */
828 beq restore /* don't schedule if so */
829 #ifdef CONFIG_TRACE_IRQFLAGS
830 /* Lockdep thinks irqs are enabled, we need to call
831 * preempt_schedule_irq with IRQs off, so we inform lockdep
832 * now that we -did- turn them off already
833 */
834 bl trace_hardirqs_off
835 #endif
836 1: bl preempt_schedule_irq
837 rlwinm r9,r1,0,0,(31-THREAD_SHIFT)
838 lwz r3,TI_FLAGS(r9)
839 andi. r0,r3,_TIF_NEED_RESCHED
840 bne- 1b
841 #ifdef CONFIG_TRACE_IRQFLAGS
842 /* And now, to properly rebalance the above, we tell lockdep they
843 * are being turned back on, which will happen when we return
844 */
845 bl trace_hardirqs_on
846 #endif
847 #else
848 resume_kernel:
849 #endif /* CONFIG_PREEMPT */
851 /* interrupts are hard-disabled at this point */
852 restore:
853 #ifdef CONFIG_44x
854 BEGIN_MMU_FTR_SECTION
855 b 1f
856 END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_47x)
857 lis r4,icache_44x_need_flush@ha
858 lwz r5,icache_44x_need_flush@l(r4)
859 cmplwi cr0,r5,0
860 beq+ 1f
861 li r6,0
862 iccci r0,r0
863 stw r6,icache_44x_need_flush@l(r4)
864 1:
865 #endif /* CONFIG_44x */
867 lwz r9,_MSR(r1)
868 #ifdef CONFIG_TRACE_IRQFLAGS
869 /* Lockdep doesn't know about the fact that IRQs are temporarily turned
870 * off in this assembly code while peeking at TI_FLAGS() and such. However
871 * we need to inform it if the exception turned interrupts off, and we
872 * are about to trun them back on.
873 *
874 * The problem here sadly is that we don't know whether the exceptions was
875 * one that turned interrupts off or not. So we always tell lockdep about
876 * turning them on here when we go back to wherever we came from with EE
877 * on, even if that may meen some redudant calls being tracked. Maybe later
878 * we could encode what the exception did somewhere or test the exception
879 * type in the pt_regs but that sounds overkill
880 */
881 andi. r10,r9,MSR_EE
882 beq 1f
883 /*
884 * Since the ftrace irqsoff latency trace checks CALLER_ADDR1,
885 * which is the stack frame here, we need to force a stack frame
886 * in case we came from user space.
887 */
888 stwu r1,-32(r1)
889 mflr r0
890 stw r0,4(r1)
891 stwu r1,-32(r1)
892 bl trace_hardirqs_on
893 lwz r1,0(r1)
894 lwz r1,0(r1)
895 lwz r9,_MSR(r1)
896 1:
897 #endif /* CONFIG_TRACE_IRQFLAGS */
899 lwz r0,GPR0(r1)
900 lwz r2,GPR2(r1)
901 REST_4GPRS(3, r1)
902 REST_2GPRS(7, r1)
904 lwz r10,_XER(r1)
905 lwz r11,_CTR(r1)
906 mtspr SPRN_XER,r10
907 mtctr r11
909 PPC405_ERR77(0,r1)
910 BEGIN_FTR_SECTION
911 lwarx r11,0,r1
912 END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX)
913 stwcx. r0,0,r1 /* to clear the reservation */
915 #if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
916 andi. r10,r9,MSR_RI /* check if this exception occurred */
917 beql nonrecoverable /* at a bad place (MSR:RI = 0) */
919 lwz r10,_CCR(r1)
920 lwz r11,_LINK(r1)
921 mtcrf 0xFF,r10
922 mtlr r11
924 /*
925 * Once we put values in SRR0 and SRR1, we are in a state
926 * where exceptions are not recoverable, since taking an
927 * exception will trash SRR0 and SRR1. Therefore we clear the
928 * MSR:RI bit to indicate this. If we do take an exception,
929 * we can't return to the point of the exception but we
930 * can restart the exception exit path at the label
931 * exc_exit_restart below. -- paulus
932 */
933 LOAD_MSR_KERNEL(r10,MSR_KERNEL & ~MSR_RI)
934 SYNC
935 MTMSRD(r10) /* clear the RI bit */
936 .globl exc_exit_restart
937 exc_exit_restart:
938 lwz r12,_NIP(r1)
939 FIX_SRR1(r9,r10)
940 mtspr SPRN_SRR0,r12
941 mtspr SPRN_SRR1,r9
942 REST_4GPRS(9, r1)
943 lwz r1,GPR1(r1)
944 .globl exc_exit_restart_end
945 exc_exit_restart_end:
946 SYNC
947 RFI
949 #else /* !(CONFIG_4xx || CONFIG_BOOKE) */
950 /*
951 * This is a bit different on 4xx/Book-E because it doesn't have
952 * the RI bit in the MSR.
953 * The TLB miss handler checks if we have interrupted
954 * the exception exit path and restarts it if so
955 * (well maybe one day it will... :).
956 */
957 lwz r11,_LINK(r1)
958 mtlr r11
959 lwz r10,_CCR(r1)
960 mtcrf 0xff,r10
961 REST_2GPRS(9, r1)
962 .globl exc_exit_restart
963 exc_exit_restart:
964 lwz r11,_NIP(r1)
965 lwz r12,_MSR(r1)
966 exc_exit_start:
967 mtspr SPRN_SRR0,r11
968 mtspr SPRN_SRR1,r12
969 REST_2GPRS(11, r1)
970 lwz r1,GPR1(r1)
971 .globl exc_exit_restart_end
972 exc_exit_restart_end:
973 PPC405_ERR77_SYNC
974 rfi
975 b . /* prevent prefetch past rfi */
977 /*
978 * Returning from a critical interrupt in user mode doesn't need
979 * to be any different from a normal exception. For a critical
980 * interrupt in the kernel, we just return (without checking for
981 * preemption) since the interrupt may have happened at some crucial
982 * place (e.g. inside the TLB miss handler), and because we will be
983 * running with r1 pointing into critical_stack, not the current
984 * process's kernel stack (and therefore current_thread_info() will
985 * give the wrong answer).
986 * We have to restore various SPRs that may have been in use at the
987 * time of the critical interrupt.
988 *
989 */
990 #ifdef CONFIG_40x
991 #define PPC_40x_TURN_OFF_MSR_DR \
992 /* avoid any possible TLB misses here by turning off MSR.DR, we \
993 * assume the instructions here are mapped by a pinned TLB entry */ \
994 li r10,MSR_IR; \
995 mtmsr r10; \
996 isync; \
997 tophys(r1, r1);
998 #else
999 #define PPC_40x_TURN_OFF_MSR_DR
1000 #endif
1002 #define RET_FROM_EXC_LEVEL(exc_lvl_srr0, exc_lvl_srr1, exc_lvl_rfi) \
1003 REST_NVGPRS(r1); \
1004 lwz r3,_MSR(r1); \
1005 andi. r3,r3,MSR_PR; \
1006 LOAD_MSR_KERNEL(r10,MSR_KERNEL); \
1007 bne user_exc_return; \
1008 lwz r0,GPR0(r1); \
1009 lwz r2,GPR2(r1); \
1010 REST_4GPRS(3, r1); \
1011 REST_2GPRS(7, r1); \
1012 lwz r10,_XER(r1); \
1013 lwz r11,_CTR(r1); \
1014 mtspr SPRN_XER,r10; \
1015 mtctr r11; \
1016 PPC405_ERR77(0,r1); \
1017 stwcx. r0,0,r1; /* to clear the reservation */ \
1018 lwz r11,_LINK(r1); \
1019 mtlr r11; \
1020 lwz r10,_CCR(r1); \
1021 mtcrf 0xff,r10; \
1022 PPC_40x_TURN_OFF_MSR_DR; \
1023 lwz r9,_DEAR(r1); \
1024 lwz r10,_ESR(r1); \
1025 mtspr SPRN_DEAR,r9; \
1026 mtspr SPRN_ESR,r10; \
1027 lwz r11,_NIP(r1); \
1028 lwz r12,_MSR(r1); \
1029 mtspr exc_lvl_srr0,r11; \
1030 mtspr exc_lvl_srr1,r12; \
1031 lwz r9,GPR9(r1); \
1032 lwz r12,GPR12(r1); \
1033 lwz r10,GPR10(r1); \
1034 lwz r11,GPR11(r1); \
1035 lwz r1,GPR1(r1); \
1036 PPC405_ERR77_SYNC; \
1037 exc_lvl_rfi; \
1038 b .; /* prevent prefetch past exc_lvl_rfi */
1040 #define RESTORE_xSRR(exc_lvl_srr0, exc_lvl_srr1) \
1041 lwz r9,_##exc_lvl_srr0(r1); \
1042 lwz r10,_##exc_lvl_srr1(r1); \
1043 mtspr SPRN_##exc_lvl_srr0,r9; \
1044 mtspr SPRN_##exc_lvl_srr1,r10;
1046 #if defined(CONFIG_PPC_BOOK3E_MMU)
1047 #ifdef CONFIG_PHYS_64BIT
1048 #define RESTORE_MAS7 \
1049 lwz r11,MAS7(r1); \
1050 mtspr SPRN_MAS7,r11;
1051 #else
1052 #define RESTORE_MAS7
1053 #endif /* CONFIG_PHYS_64BIT */
1054 #define RESTORE_MMU_REGS \
1055 lwz r9,MAS0(r1); \
1056 lwz r10,MAS1(r1); \
1057 lwz r11,MAS2(r1); \
1058 mtspr SPRN_MAS0,r9; \
1059 lwz r9,MAS3(r1); \
1060 mtspr SPRN_MAS1,r10; \
1061 lwz r10,MAS6(r1); \
1062 mtspr SPRN_MAS2,r11; \
1063 mtspr SPRN_MAS3,r9; \
1064 mtspr SPRN_MAS6,r10; \
1065 RESTORE_MAS7;
1066 #elif defined(CONFIG_44x)
1067 #define RESTORE_MMU_REGS \
1068 lwz r9,MMUCR(r1); \
1069 mtspr SPRN_MMUCR,r9;
1070 #else
1071 #define RESTORE_MMU_REGS
1072 #endif
1074 #ifdef CONFIG_40x
1075 .globl ret_from_crit_exc
1076 ret_from_crit_exc:
1077 mfspr r9,SPRN_SPRG_THREAD
1078 lis r10,saved_ksp_limit@ha;
1079 lwz r10,saved_ksp_limit@l(r10);
1080 tovirt(r9,r9);
1081 stw r10,KSP_LIMIT(r9)
1082 lis r9,crit_srr0@ha;
1083 lwz r9,crit_srr0@l(r9);
1084 lis r10,crit_srr1@ha;
1085 lwz r10,crit_srr1@l(r10);
1086 mtspr SPRN_SRR0,r9;
1087 mtspr SPRN_SRR1,r10;
1088 RET_FROM_EXC_LEVEL(SPRN_CSRR0, SPRN_CSRR1, PPC_RFCI)
1089 #endif /* CONFIG_40x */
1091 #ifdef CONFIG_BOOKE
1092 .globl ret_from_crit_exc
1093 ret_from_crit_exc:
1094 mfspr r9,SPRN_SPRG_THREAD
1095 lwz r10,SAVED_KSP_LIMIT(r1)
1096 stw r10,KSP_LIMIT(r9)
1097 RESTORE_xSRR(SRR0,SRR1);
1098 RESTORE_MMU_REGS;
1099 RET_FROM_EXC_LEVEL(SPRN_CSRR0, SPRN_CSRR1, PPC_RFCI)
1101 .globl ret_from_debug_exc
1102 ret_from_debug_exc:
1103 mfspr r9,SPRN_SPRG_THREAD
1104 lwz r10,SAVED_KSP_LIMIT(r1)
1105 stw r10,KSP_LIMIT(r9)
1106 lwz r9,THREAD_INFO-THREAD(r9)
1107 rlwinm r10,r1,0,0,(31-THREAD_SHIFT)
1108 lwz r10,TI_PREEMPT(r10)
1109 stw r10,TI_PREEMPT(r9)
1110 RESTORE_xSRR(SRR0,SRR1);
1111 RESTORE_xSRR(CSRR0,CSRR1);
1112 RESTORE_MMU_REGS;
1113 RET_FROM_EXC_LEVEL(SPRN_DSRR0, SPRN_DSRR1, PPC_RFDI)
1115 .globl ret_from_mcheck_exc
1116 ret_from_mcheck_exc:
1117 mfspr r9,SPRN_SPRG_THREAD
1118 lwz r10,SAVED_KSP_LIMIT(r1)
1119 stw r10,KSP_LIMIT(r9)
1120 RESTORE_xSRR(SRR0,SRR1);
1121 RESTORE_xSRR(CSRR0,CSRR1);
1122 RESTORE_xSRR(DSRR0,DSRR1);
1123 RESTORE_MMU_REGS;
1124 RET_FROM_EXC_LEVEL(SPRN_MCSRR0, SPRN_MCSRR1, PPC_RFMCI)
1125 #endif /* CONFIG_BOOKE */
1127 /*
1128 * Load the DBCR0 value for a task that is being ptraced,
1129 * having first saved away the global DBCR0. Note that r0
1130 * has the dbcr0 value to set upon entry to this.
1131 */
1132 load_dbcr0:
1133 mfmsr r10 /* first disable debug exceptions */
1134 rlwinm r10,r10,0,~MSR_DE
1135 mtmsr r10
1136 isync
1137 mfspr r10,SPRN_DBCR0
1138 lis r11,global_dbcr0@ha
1139 addi r11,r11,global_dbcr0@l
1140 #ifdef CONFIG_SMP
1141 rlwinm r9,r1,0,0,(31-THREAD_SHIFT)
1142 lwz r9,TI_CPU(r9)
1143 slwi r9,r9,3
1144 add r11,r11,r9
1145 #endif
1146 stw r10,0(r11)
1147 mtspr SPRN_DBCR0,r0
1148 lwz r10,4(r11)
1149 addi r10,r10,1
1150 stw r10,4(r11)
1151 li r11,-1
1152 mtspr SPRN_DBSR,r11 /* clear all pending debug events */
1153 blr
1155 .section .bss
1156 .align 4
1157 global_dbcr0:
1158 .space 8*NR_CPUS
1159 .previous
1160 #endif /* !(CONFIG_4xx || CONFIG_BOOKE) */
1162 do_work: /* r10 contains MSR_KERNEL here */
1163 andi. r0,r9,_TIF_NEED_RESCHED
1164 beq do_user_signal
1166 do_resched: /* r10 contains MSR_KERNEL here */
1167 /* Note: We don't need to inform lockdep that we are enabling
1168 * interrupts here. As far as it knows, they are already enabled
1169 */
1170 ori r10,r10,MSR_EE
1171 SYNC
1172 MTMSRD(r10) /* hard-enable interrupts */
1173 bl schedule
1174 recheck:
1175 /* Note: And we don't tell it we are disabling them again
1176 * neither. Those disable/enable cycles used to peek at
1177 * TI_FLAGS aren't advertised.
1178 */
1179 LOAD_MSR_KERNEL(r10,MSR_KERNEL)
1180 SYNC
1181 MTMSRD(r10) /* disable interrupts */
1182 rlwinm r9,r1,0,0,(31-THREAD_SHIFT)
1183 lwz r9,TI_FLAGS(r9)
1184 andi. r0,r9,_TIF_NEED_RESCHED
1185 bne- do_resched
1186 andi. r0,r9,_TIF_USER_WORK_MASK
1187 beq restore_user
1188 do_user_signal: /* r10 contains MSR_KERNEL here */
1189 ori r10,r10,MSR_EE
1190 SYNC
1191 MTMSRD(r10) /* hard-enable interrupts */
1192 /* save r13-r31 in the exception frame, if not already done */
1193 lwz r3,_TRAP(r1)
1194 andi. r0,r3,1
1195 beq 2f
1196 SAVE_NVGPRS(r1)
1197 rlwinm r3,r3,0,0,30
1198 stw r3,_TRAP(r1)
1199 2: addi r3,r1,STACK_FRAME_OVERHEAD
1200 mr r4,r9
1201 bl do_signal
1202 REST_NVGPRS(r1)
1203 b recheck
1205 /*
1206 * We come here when we are at the end of handling an exception
1207 * that occurred at a place where taking an exception will lose
1208 * state information, such as the contents of SRR0 and SRR1.
1209 */
1210 nonrecoverable:
1211 lis r10,exc_exit_restart_end@ha
1212 addi r10,r10,exc_exit_restart_end@l
1213 cmplw r12,r10
1214 bge 3f
1215 lis r11,exc_exit_restart@ha
1216 addi r11,r11,exc_exit_restart@l
1217 cmplw r12,r11
1218 blt 3f
1219 lis r10,ee_restarts@ha
1220 lwz r12,ee_restarts@l(r10)
1221 addi r12,r12,1
1222 stw r12,ee_restarts@l(r10)
1223 mr r12,r11 /* restart at exc_exit_restart */
1224 blr
1225 3: /* OK, we can't recover, kill this process */
1226 /* but the 601 doesn't implement the RI bit, so assume it's OK */
1227 BEGIN_FTR_SECTION
1228 blr
1229 END_FTR_SECTION_IFSET(CPU_FTR_601)
1230 lwz r3,_TRAP(r1)
1231 andi. r0,r3,1
1232 beq 4f
1233 SAVE_NVGPRS(r1)
1234 rlwinm r3,r3,0,0,30
1235 stw r3,_TRAP(r1)
1236 4: addi r3,r1,STACK_FRAME_OVERHEAD
1237 bl nonrecoverable_exception
1238 /* shouldn't return */
1239 b 4b
1241 .section .bss
1242 .align 2
1243 ee_restarts:
1244 .space 4
1245 .previous
1247 /*
1248 * PROM code for specific machines follows. Put it
1249 * here so it's easy to add arch-specific sections later.
1250 * -- Cort
1251 */
1252 #ifdef CONFIG_PPC_RTAS
1253 /*
1254 * On CHRP, the Run-Time Abstraction Services (RTAS) have to be
1255 * called with the MMU off.
1256 */
1257 _GLOBAL(enter_rtas)
1258 stwu r1,-INT_FRAME_SIZE(r1)
1259 mflr r0
1260 stw r0,INT_FRAME_SIZE+4(r1)
1261 LOAD_REG_ADDR(r4, rtas)
1262 lis r6,1f@ha /* physical return address for rtas */
1263 addi r6,r6,1f@l
1264 tophys(r6,r6)
1265 tophys(r7,r1)
1266 lwz r8,RTASENTRY(r4)
1267 lwz r4,RTASBASE(r4)
1268 mfmsr r9
1269 stw r9,8(r1)
1270 LOAD_MSR_KERNEL(r0,MSR_KERNEL)
1271 SYNC /* disable interrupts so SRR0/1 */
1272 MTMSRD(r0) /* don't get trashed */
1273 li r9,MSR_KERNEL & ~(MSR_IR|MSR_DR)
1274 mtlr r6
1275 mtspr SPRN_SPRG_RTAS,r7
1276 mtspr SPRN_SRR0,r8
1277 mtspr SPRN_SRR1,r9
1278 RFI
1279 1: tophys(r9,r1)
1280 lwz r8,INT_FRAME_SIZE+4(r9) /* get return address */
1281 lwz r9,8(r9) /* original msr value */
1282 FIX_SRR1(r9,r0)
1283 addi r1,r1,INT_FRAME_SIZE
1284 li r0,0
1285 mtspr SPRN_SPRG_RTAS,r0
1286 mtspr SPRN_SRR0,r8
1287 mtspr SPRN_SRR1,r9
1288 RFI /* return to caller */
1290 .globl machine_check_in_rtas
1291 machine_check_in_rtas:
1292 twi 31,0,0
1293 /* XXX load up BATs and panic */
1295 #endif /* CONFIG_PPC_RTAS */
1297 #ifdef CONFIG_FUNCTION_TRACER
1298 #ifdef CONFIG_DYNAMIC_FTRACE
1299 _GLOBAL(mcount)
1300 _GLOBAL(_mcount)
1301 /*
1302 * It is required that _mcount on PPC32 must preserve the
1303 * link register. But we have r0 to play with. We use r0
1304 * to push the return address back to the caller of mcount
1305 * into the ctr register, restore the link register and
1306 * then jump back using the ctr register.
1307 */
1308 mflr r0
1309 mtctr r0
1310 lwz r0, 4(r1)
1311 mtlr r0
1312 bctr
1314 _GLOBAL(ftrace_caller)
1315 MCOUNT_SAVE_FRAME
1316 /* r3 ends up with link register */
1317 subi r3, r3, MCOUNT_INSN_SIZE
1318 .globl ftrace_call
1319 ftrace_call:
1320 bl ftrace_stub
1321 nop
1322 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1323 .globl ftrace_graph_call
1324 ftrace_graph_call:
1325 b ftrace_graph_stub
1326 _GLOBAL(ftrace_graph_stub)
1327 #endif
1328 MCOUNT_RESTORE_FRAME
1329 /* old link register ends up in ctr reg */
1330 bctr
1331 #else
1332 _GLOBAL(mcount)
1333 _GLOBAL(_mcount)
1335 MCOUNT_SAVE_FRAME
1337 subi r3, r3, MCOUNT_INSN_SIZE
1338 LOAD_REG_ADDR(r5, ftrace_trace_function)
1339 lwz r5,0(r5)
1341 mtctr r5
1342 bctrl
1343 nop
1345 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1346 b ftrace_graph_caller
1347 #endif
1348 MCOUNT_RESTORE_FRAME
1349 bctr
1350 #endif
1352 _GLOBAL(ftrace_stub)
1353 blr
1355 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1356 _GLOBAL(ftrace_graph_caller)
1357 /* load r4 with local address */
1358 lwz r4, 44(r1)
1359 subi r4, r4, MCOUNT_INSN_SIZE
1361 /* get the parent address */
1362 addi r3, r1, 52
1364 bl prepare_ftrace_return
1365 nop
1367 MCOUNT_RESTORE_FRAME
1368 /* old link register ends up in ctr reg */
1369 bctr
1371 _GLOBAL(return_to_handler)
1372 /* need to save return values */
1373 stwu r1, -32(r1)
1374 stw r3, 20(r1)
1375 stw r4, 16(r1)
1376 stw r31, 12(r1)
1377 mr r31, r1
1379 bl ftrace_return_to_handler
1380 nop
1382 /* return value has real return address */
1383 mtlr r3
1385 lwz r3, 20(r1)
1386 lwz r4, 16(r1)
1387 lwz r31,12(r1)
1388 lwz r1, 0(r1)
1390 /* Jump back to real return address */
1391 blr
1392 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1394 #endif /* CONFIG_MCOUNT */