| blob | 46fc0f4d8982c64c18f2046bab96e25ff5c1649c |
1 /*
2 * PowerPC version
3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 * Rewritten by Cort Dougan (cort@fsmlabs.com) for PReP
5 * Copyright (C) 1996 Cort Dougan <cort@fsmlabs.com>
6 * Adapted for Power Macintosh by Paul Mackerras.
7 * Low-level exception handlers and MMU support
8 * rewritten by Paul Mackerras.
9 * Copyright (C) 1996 Paul Mackerras.
10 * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
11 *
12 * This file contains the system call entry code, context switch
13 * code, and exception/interrupt return code for PowerPC.
14 *
15 * This program is free software; you can redistribute it and/or
16 * modify it under the terms of the GNU General Public License
17 * as published by the Free Software Foundation; either version
18 * 2 of the License, or (at your option) any later version.
19 *
20 */
22 #include <linux/errno.h>
23 #include <linux/sys.h>
24 #include <linux/threads.h>
25 #include <asm/reg.h>
26 #include <asm/page.h>
27 #include <asm/mmu.h>
28 #include <asm/cputable.h>
29 #include <asm/thread_info.h>
30 #include <asm/ppc_asm.h>
31 #include <asm/asm-offsets.h>
32 #include <asm/unistd.h>
33 #include <asm/ftrace.h>
34 #include <asm/ptrace.h>
36 /*
37 * MSR_KERNEL is > 0x10000 on 4xx/Book-E since it include MSR_CE.
38 */
39 #if MSR_KERNEL >= 0x10000
40 #define LOAD_MSR_KERNEL(r, x) lis r,(x)@h; ori r,r,(x)@l
41 #else
42 #define LOAD_MSR_KERNEL(r, x) li r,(x)
43 #endif
45 #ifdef CONFIG_BOOKE
46 .globl mcheck_transfer_to_handler
47 mcheck_transfer_to_handler:
48 mfspr r0,SPRN_DSRR0
49 stw r0,_DSRR0(r11)
50 mfspr r0,SPRN_DSRR1
51 stw r0,_DSRR1(r11)
52 /* fall through */
54 .globl debug_transfer_to_handler
55 debug_transfer_to_handler:
56 mfspr r0,SPRN_CSRR0
57 stw r0,_CSRR0(r11)
58 mfspr r0,SPRN_CSRR1
59 stw r0,_CSRR1(r11)
60 /* fall through */
62 .globl crit_transfer_to_handler
63 crit_transfer_to_handler:
64 #ifdef CONFIG_PPC_BOOK3E_MMU
65 mfspr r0,SPRN_MAS0
66 stw r0,MAS0(r11)
67 mfspr r0,SPRN_MAS1
68 stw r0,MAS1(r11)
69 mfspr r0,SPRN_MAS2
70 stw r0,MAS2(r11)
71 mfspr r0,SPRN_MAS3
72 stw r0,MAS3(r11)
73 mfspr r0,SPRN_MAS6
74 stw r0,MAS6(r11)
75 #ifdef CONFIG_PHYS_64BIT
76 mfspr r0,SPRN_MAS7
77 stw r0,MAS7(r11)
78 #endif /* CONFIG_PHYS_64BIT */
79 #endif /* CONFIG_PPC_BOOK3E_MMU */
80 #ifdef CONFIG_44x
81 mfspr r0,SPRN_MMUCR
82 stw r0,MMUCR(r11)
83 #endif
84 mfspr r0,SPRN_SRR0
85 stw r0,_SRR0(r11)
86 mfspr r0,SPRN_SRR1
87 stw r0,_SRR1(r11)
89 /* set the stack limit to the current stack
90 * and set the limit to protect the thread_info
91 * struct
92 */
93 mfspr r8,SPRN_SPRG_THREAD
94 lwz r0,KSP_LIMIT(r8)
95 stw r0,SAVED_KSP_LIMIT(r11)
96 rlwimi r0,r1,0,0,(31-THREAD_SHIFT)
97 stw r0,KSP_LIMIT(r8)
98 /* fall through */
99 #endif
101 #ifdef CONFIG_40x
102 .globl crit_transfer_to_handler
103 crit_transfer_to_handler:
104 lwz r0,crit_r10@l(0)
105 stw r0,GPR10(r11)
106 lwz r0,crit_r11@l(0)
107 stw r0,GPR11(r11)
108 mfspr r0,SPRN_SRR0
109 stw r0,crit_srr0@l(0)
110 mfspr r0,SPRN_SRR1
111 stw r0,crit_srr1@l(0)
113 /* set the stack limit to the current stack
114 * and set the limit to protect the thread_info
115 * struct
116 */
117 mfspr r8,SPRN_SPRG_THREAD
118 lwz r0,KSP_LIMIT(r8)
119 stw r0,saved_ksp_limit@l(0)
120 rlwimi r0,r1,0,0,(31-THREAD_SHIFT)
121 stw r0,KSP_LIMIT(r8)
122 /* fall through */
123 #endif
125 /*
126 * This code finishes saving the registers to the exception frame
127 * and jumps to the appropriate handler for the exception, turning
128 * on address translation.
129 * Note that we rely on the caller having set cr0.eq iff the exception
130 * occurred in kernel mode (i.e. MSR:PR = 0).
131 */
132 .globl transfer_to_handler_full
133 transfer_to_handler_full:
134 SAVE_NVGPRS(r11)
135 /* fall through */
137 .globl transfer_to_handler
138 transfer_to_handler:
139 stw r2,GPR2(r11)
140 stw r12,_NIP(r11)
141 stw r9,_MSR(r11)
142 andi. r2,r9,MSR_PR
143 mfctr r12
144 mfspr r2,SPRN_XER
145 stw r12,_CTR(r11)
146 stw r2,_XER(r11)
147 mfspr r12,SPRN_SPRG_THREAD
148 addi r2,r12,-THREAD
149 tovirt(r2,r2) /* set r2 to current */
150 beq 2f /* if from user, fix up THREAD.regs */
151 addi r11,r1,STACK_FRAME_OVERHEAD
152 stw r11,PT_REGS(r12)
153 #if defined(CONFIG_40x) || defined(CONFIG_BOOKE)
154 /* Check to see if the dbcr0 register is set up to debug. Use the
155 internal debug mode bit to do this. */
156 lwz r12,THREAD_DBCR0(r12)
157 andis. r12,r12,DBCR0_IDM@h
158 beq+ 3f
159 /* From user and task is ptraced - load up global dbcr0 */
160 li r12,-1 /* clear all pending debug events */
161 mtspr SPRN_DBSR,r12
162 lis r11,global_dbcr0@ha
163 tophys(r11,r11)
164 addi r11,r11,global_dbcr0@l
165 #ifdef CONFIG_SMP
166 CURRENT_THREAD_INFO(r9, r1)
167 lwz r9,TI_CPU(r9)
168 slwi r9,r9,3
169 add r11,r11,r9
170 #endif
171 lwz r12,0(r11)
172 mtspr SPRN_DBCR0,r12
173 lwz r12,4(r11)
174 addi r12,r12,-1
175 stw r12,4(r11)
176 #endif
177 b 3f
179 2: /* if from kernel, check interrupted DOZE/NAP mode and
180 * check for stack overflow
181 */
182 lwz r9,KSP_LIMIT(r12)
183 cmplw r1,r9 /* if r1 <= ksp_limit */
184 ble- stack_ovf /* then the kernel stack overflowed */
185 5:
186 #if defined(CONFIG_6xx) || defined(CONFIG_E500)
187 CURRENT_THREAD_INFO(r9, r1)
188 tophys(r9,r9) /* check local flags */
189 lwz r12,TI_LOCAL_FLAGS(r9)
190 mtcrf 0x01,r12
191 bt- 31-TLF_NAPPING,4f
192 bt- 31-TLF_SLEEPING,7f
193 #endif /* CONFIG_6xx || CONFIG_E500 */
194 .globl transfer_to_handler_cont
195 transfer_to_handler_cont:
196 3:
197 mflr r9
198 lwz r11,0(r9) /* virtual address of handler */
199 lwz r9,4(r9) /* where to go when done */
200 #ifdef CONFIG_TRACE_IRQFLAGS
201 lis r12,reenable_mmu@h
202 ori r12,r12,reenable_mmu@l
203 mtspr SPRN_SRR0,r12
204 mtspr SPRN_SRR1,r10
205 SYNC
206 RFI
207 reenable_mmu: /* re-enable mmu so we can */
208 mfmsr r10
209 lwz r12,_MSR(r1)
210 xor r10,r10,r12
211 andi. r10,r10,MSR_EE /* Did EE change? */
212 beq 1f
214 /*
215 * The trace_hardirqs_off will use CALLER_ADDR0 and CALLER_ADDR1.
216 * If from user mode there is only one stack frame on the stack, and
217 * accessing CALLER_ADDR1 will cause oops. So we need create a dummy
218 * stack frame to make trace_hardirqs_off happy.
219 *
220 * This is handy because we also need to save a bunch of GPRs,
221 * r3 can be different from GPR3(r1) at this point, r9 and r11
222 * contains the old MSR and handler address respectively,
223 * r4 & r5 can contain page fault arguments that need to be passed
224 * along as well. r12, CCR, CTR, XER etc... are left clobbered as
225 * they aren't useful past this point (aren't syscall arguments),
226 * the rest is restored from the exception frame.
227 */
228 stwu r1,-32(r1)
229 stw r9,8(r1)
230 stw r11,12(r1)
231 stw r3,16(r1)
232 stw r4,20(r1)
233 stw r5,24(r1)
234 bl trace_hardirqs_off
235 lwz r5,24(r1)
236 lwz r4,20(r1)
237 lwz r3,16(r1)
238 lwz r11,12(r1)
239 lwz r9,8(r1)
240 addi r1,r1,32
241 lwz r0,GPR0(r1)
242 lwz r6,GPR6(r1)
243 lwz r7,GPR7(r1)
244 lwz r8,GPR8(r1)
245 1: mtctr r11
246 mtlr r9
247 bctr /* jump to handler */
248 #else /* CONFIG_TRACE_IRQFLAGS */
249 mtspr SPRN_SRR0,r11
250 mtspr SPRN_SRR1,r10
251 mtlr r9
252 SYNC
253 RFI /* jump to handler, enable MMU */
254 #endif /* CONFIG_TRACE_IRQFLAGS */
256 #if defined (CONFIG_6xx) || defined(CONFIG_E500)
257 4: rlwinm r12,r12,0,~_TLF_NAPPING
258 stw r12,TI_LOCAL_FLAGS(r9)
259 b power_save_ppc32_restore
261 7: rlwinm r12,r12,0,~_TLF_SLEEPING
262 stw r12,TI_LOCAL_FLAGS(r9)
263 lwz r9,_MSR(r11) /* if sleeping, clear MSR.EE */
264 rlwinm r9,r9,0,~MSR_EE
265 lwz r12,_LINK(r11) /* and return to address in LR */
266 b fast_exception_return
267 #endif
269 /*
270 * On kernel stack overflow, load up an initial stack pointer
271 * and call StackOverflow(regs), which should not return.
272 */
273 stack_ovf:
274 /* sometimes we use a statically-allocated stack, which is OK. */
275 lis r12,_end@h
276 ori r12,r12,_end@l
277 cmplw r1,r12
278 ble 5b /* r1 <= &_end is OK */
279 SAVE_NVGPRS(r11)
280 addi r3,r1,STACK_FRAME_OVERHEAD
281 lis r1,init_thread_union@ha
282 addi r1,r1,init_thread_union@l
283 addi r1,r1,THREAD_SIZE-STACK_FRAME_OVERHEAD
284 lis r9,StackOverflow@ha
285 addi r9,r9,StackOverflow@l
286 LOAD_MSR_KERNEL(r10,MSR_KERNEL)
287 FIX_SRR1(r10,r12)
288 mtspr SPRN_SRR0,r9
289 mtspr SPRN_SRR1,r10
290 SYNC
291 RFI
293 /*
294 * Handle a system call.
295 */
296 .stabs "arch/powerpc/kernel/",N_SO,0,0,0f
297 .stabs "entry_32.S",N_SO,0,0,0f
298 0:
300 _GLOBAL(DoSyscall)
301 stw r3,ORIG_GPR3(r1)
302 li r12,0
303 stw r12,RESULT(r1)
304 lwz r11,_CCR(r1) /* Clear SO bit in CR */
305 rlwinm r11,r11,0,4,2
306 stw r11,_CCR(r1)
307 #ifdef CONFIG_TRACE_IRQFLAGS
308 /* Return from syscalls can (and generally will) hard enable
309 * interrupts. You aren't supposed to call a syscall with
310 * interrupts disabled in the first place. However, to ensure
311 * that we get it right vs. lockdep if it happens, we force
312 * that hard enable here with appropriate tracing if we see
313 * that we have been called with interrupts off
314 */
315 mfmsr r11
316 andi. r12,r11,MSR_EE
317 bne+ 1f
318 /* We came in with interrupts disabled, we enable them now */
319 bl trace_hardirqs_on
320 mfmsr r11
321 lwz r0,GPR0(r1)
322 lwz r3,GPR3(r1)
323 lwz r4,GPR4(r1)
324 ori r11,r11,MSR_EE
325 lwz r5,GPR5(r1)
326 lwz r6,GPR6(r1)
327 lwz r7,GPR7(r1)
328 lwz r8,GPR8(r1)
329 mtmsr r11
330 1:
331 #endif /* CONFIG_TRACE_IRQFLAGS */
332 CURRENT_THREAD_INFO(r10, r1)
333 lwz r11,TI_FLAGS(r10)
334 andi. r11,r11,_TIF_SYSCALL_DOTRACE
335 bne- syscall_dotrace
336 syscall_dotrace_cont:
337 cmplwi 0,r0,NR_syscalls
338 lis r10,sys_call_table@h
339 ori r10,r10,sys_call_table@l
340 slwi r0,r0,2
341 bge- 66f
342 lwzx r10,r10,r0 /* Fetch system call handler [ptr] */
343 mtlr r10
344 addi r9,r1,STACK_FRAME_OVERHEAD
345 PPC440EP_ERR42
346 blrl /* Call handler */
347 .globl ret_from_syscall
348 ret_from_syscall:
349 mr r6,r3
350 CURRENT_THREAD_INFO(r12, r1)
351 /* disable interrupts so current_thread_info()->flags can't change */
352 LOAD_MSR_KERNEL(r10,MSR_KERNEL) /* doesn't include MSR_EE */
353 /* Note: We don't bother telling lockdep about it */
354 SYNC
355 MTMSRD(r10)
356 lwz r9,TI_FLAGS(r12)
357 li r8,-_LAST_ERRNO
358 andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
359 bne- syscall_exit_work
360 cmplw 0,r3,r8
361 blt+ syscall_exit_cont
362 lwz r11,_CCR(r1) /* Load CR */
363 neg r3,r3
364 oris r11,r11,0x1000 /* Set SO bit in CR */
365 stw r11,_CCR(r1)
366 syscall_exit_cont:
367 lwz r8,_MSR(r1)
368 #ifdef CONFIG_TRACE_IRQFLAGS
369 /* If we are going to return from the syscall with interrupts
370 * off, we trace that here. It shouldn't happen though but we
371 * want to catch the bugger if it does right ?
372 */
373 andi. r10,r8,MSR_EE
374 bne+ 1f
375 stw r3,GPR3(r1)
376 bl trace_hardirqs_off
377 lwz r3,GPR3(r1)
378 1:
379 #endif /* CONFIG_TRACE_IRQFLAGS */
380 #if defined(CONFIG_4xx) || defined(CONFIG_BOOKE)
381 /* If the process has its own DBCR0 value, load it up. The internal
382 debug mode bit tells us that dbcr0 should be loaded. */
383 lwz r0,THREAD+THREAD_DBCR0(r2)
384 andis. r10,r0,DBCR0_IDM@h
385 bnel- load_dbcr0
386 #endif
387 #ifdef CONFIG_44x
388 BEGIN_MMU_FTR_SECTION
389 lis r4,icache_44x_need_flush@ha
390 lwz r5,icache_44x_need_flush@l(r4)
391 cmplwi cr0,r5,0
392 bne- 2f
393 1:
394 END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_47x)
395 #endif /* CONFIG_44x */
396 BEGIN_FTR_SECTION
397 lwarx r7,0,r1
398 END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX)
399 stwcx. r0,0,r1 /* to clear the reservation */
400 lwz r4,_LINK(r1)
401 lwz r5,_CCR(r1)
402 mtlr r4
403 mtcr r5
404 lwz r7,_NIP(r1)
405 FIX_SRR1(r8, r0)
406 lwz r2,GPR2(r1)
407 lwz r1,GPR1(r1)
408 mtspr SPRN_SRR0,r7
409 mtspr SPRN_SRR1,r8
410 SYNC
411 RFI
412 #ifdef CONFIG_44x
413 2: li r7,0
414 iccci r0,r0
415 stw r7,icache_44x_need_flush@l(r4)
416 b 1b
417 #endif /* CONFIG_44x */
419 66: li r3,-ENOSYS
420 b ret_from_syscall
422 .globl ret_from_fork
423 ret_from_fork:
424 REST_NVGPRS(r1)
425 bl schedule_tail
426 li r3,0
427 b ret_from_syscall
429 .globl ret_from_kernel_thread
430 ret_from_kernel_thread:
431 REST_NVGPRS(r1)
432 bl schedule_tail
433 mtlr r14
434 mr r3,r15
435 PPC440EP_ERR42
436 blrl
437 li r3,0
438 b ret_from_syscall
440 /* Traced system call support */
441 syscall_dotrace:
442 SAVE_NVGPRS(r1)
443 li r0,0xc00
444 stw r0,_TRAP(r1)
445 addi r3,r1,STACK_FRAME_OVERHEAD
446 bl do_syscall_trace_enter
447 /*
448 * Restore argument registers possibly just changed.
449 * We use the return value of do_syscall_trace_enter
450 * for call number to look up in the table (r0).
451 */
452 mr r0,r3
453 lwz r3,GPR3(r1)
454 lwz r4,GPR4(r1)
455 lwz r5,GPR5(r1)
456 lwz r6,GPR6(r1)
457 lwz r7,GPR7(r1)
458 lwz r8,GPR8(r1)
459 REST_NVGPRS(r1)
460 b syscall_dotrace_cont
462 syscall_exit_work:
463 andi. r0,r9,_TIF_RESTOREALL
464 beq+ 0f
465 REST_NVGPRS(r1)
466 b 2f
467 0: cmplw 0,r3,r8
468 blt+ 1f
469 andi. r0,r9,_TIF_NOERROR
470 bne- 1f
471 lwz r11,_CCR(r1) /* Load CR */
472 neg r3,r3
473 oris r11,r11,0x1000 /* Set SO bit in CR */
474 stw r11,_CCR(r1)
476 1: stw r6,RESULT(r1) /* Save result */
477 stw r3,GPR3(r1) /* Update return value */
478 2: andi. r0,r9,(_TIF_PERSYSCALL_MASK)
479 beq 4f
481 /* Clear per-syscall TIF flags if any are set. */
483 li r11,_TIF_PERSYSCALL_MASK
484 addi r12,r12,TI_FLAGS
485 3: lwarx r8,0,r12
486 andc r8,r8,r11
487 #ifdef CONFIG_IBM405_ERR77
488 dcbt 0,r12
489 #endif
490 stwcx. r8,0,r12
491 bne- 3b
492 subi r12,r12,TI_FLAGS
494 4: /* Anything which requires enabling interrupts? */
495 andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP)
496 beq ret_from_except
498 /* Re-enable interrupts. There is no need to trace that with
499 * lockdep as we are supposed to have IRQs on at this point
500 */
501 ori r10,r10,MSR_EE
502 SYNC
503 MTMSRD(r10)
505 /* Save NVGPRS if they're not saved already */
506 lwz r4,_TRAP(r1)
507 andi. r4,r4,1
508 beq 5f
509 SAVE_NVGPRS(r1)
510 li r4,0xc00
511 stw r4,_TRAP(r1)
512 5:
513 addi r3,r1,STACK_FRAME_OVERHEAD
514 bl do_syscall_trace_leave
515 b ret_from_except_full
517 /*
518 * The fork/clone functions need to copy the full register set into
519 * the child process. Therefore we need to save all the nonvolatile
520 * registers (r13 - r31) before calling the C code.
521 */
522 .globl ppc_fork
523 ppc_fork:
524 SAVE_NVGPRS(r1)
525 lwz r0,_TRAP(r1)
526 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
527 stw r0,_TRAP(r1) /* register set saved */
528 b sys_fork
530 .globl ppc_vfork
531 ppc_vfork:
532 SAVE_NVGPRS(r1)
533 lwz r0,_TRAP(r1)
534 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
535 stw r0,_TRAP(r1) /* register set saved */
536 b sys_vfork
538 .globl ppc_clone
539 ppc_clone:
540 SAVE_NVGPRS(r1)
541 lwz r0,_TRAP(r1)
542 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
543 stw r0,_TRAP(r1) /* register set saved */
544 b sys_clone
546 .globl ppc_swapcontext
547 ppc_swapcontext:
548 SAVE_NVGPRS(r1)
549 lwz r0,_TRAP(r1)
550 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
551 stw r0,_TRAP(r1) /* register set saved */
552 b sys_swapcontext
554 /*
555 * Top-level page fault handling.
556 * This is in assembler because if do_page_fault tells us that
557 * it is a bad kernel page fault, we want to save the non-volatile
558 * registers before calling bad_page_fault.
559 */
560 .globl handle_page_fault
561 handle_page_fault:
562 stw r4,_DAR(r1)
563 addi r3,r1,STACK_FRAME_OVERHEAD
564 bl do_page_fault
565 cmpwi r3,0
566 beq+ ret_from_except
567 SAVE_NVGPRS(r1)
568 lwz r0,_TRAP(r1)
569 clrrwi r0,r0,1
570 stw r0,_TRAP(r1)
571 mr r5,r3
572 addi r3,r1,STACK_FRAME_OVERHEAD
573 lwz r4,_DAR(r1)
574 bl bad_page_fault
575 b ret_from_except_full
577 /*
578 * This routine switches between two different tasks. The process
579 * state of one is saved on its kernel stack. Then the state
580 * of the other is restored from its kernel stack. The memory
581 * management hardware is updated to the second process's state.
582 * Finally, we can return to the second process.
583 * On entry, r3 points to the THREAD for the current task, r4
584 * points to the THREAD for the new task.
585 *
586 * This routine is always called with interrupts disabled.
587 *
588 * Note: there are two ways to get to the "going out" portion
589 * of this code; either by coming in via the entry (_switch)
590 * or via "fork" which must set up an environment equivalent
591 * to the "_switch" path. If you change this , you'll have to
592 * change the fork code also.
593 *
594 * The code which creates the new task context is in 'copy_thread'
595 * in arch/ppc/kernel/process.c
596 */
597 _GLOBAL(_switch)
598 stwu r1,-INT_FRAME_SIZE(r1)
599 mflr r0
600 stw r0,INT_FRAME_SIZE+4(r1)
601 /* r3-r12 are caller saved -- Cort */
602 SAVE_NVGPRS(r1)
603 stw r0,_NIP(r1) /* Return to switch caller */
604 mfmsr r11
605 li r0,MSR_FP /* Disable floating-point */
606 #ifdef CONFIG_ALTIVEC
607 BEGIN_FTR_SECTION
608 oris r0,r0,MSR_VEC@h /* Disable altivec */
609 mfspr r12,SPRN_VRSAVE /* save vrsave register value */
610 stw r12,THREAD+THREAD_VRSAVE(r2)
611 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
612 #endif /* CONFIG_ALTIVEC */
613 #ifdef CONFIG_SPE
614 BEGIN_FTR_SECTION
615 oris r0,r0,MSR_SPE@h /* Disable SPE */
616 mfspr r12,SPRN_SPEFSCR /* save spefscr register value */
617 stw r12,THREAD+THREAD_SPEFSCR(r2)
618 END_FTR_SECTION_IFSET(CPU_FTR_SPE)
619 #endif /* CONFIG_SPE */
620 and. r0,r0,r11 /* FP or altivec or SPE enabled? */
621 beq+ 1f
622 andc r11,r11,r0
623 MTMSRD(r11)
624 isync
625 1: stw r11,_MSR(r1)
626 mfcr r10
627 stw r10,_CCR(r1)
628 stw r1,KSP(r3) /* Set old stack pointer */
630 #ifdef CONFIG_SMP
631 /* We need a sync somewhere here to make sure that if the
632 * previous task gets rescheduled on another CPU, it sees all
633 * stores it has performed on this one.
634 */
635 sync
636 #endif /* CONFIG_SMP */
638 tophys(r0,r4)
639 CLR_TOP32(r0)
640 mtspr SPRN_SPRG_THREAD,r0 /* Update current THREAD phys addr */
641 lwz r1,KSP(r4) /* Load new stack pointer */
643 /* save the old current 'last' for return value */
644 mr r3,r2
645 addi r2,r4,-THREAD /* Update current */
647 #ifdef CONFIG_ALTIVEC
648 BEGIN_FTR_SECTION
649 lwz r0,THREAD+THREAD_VRSAVE(r2)
650 mtspr SPRN_VRSAVE,r0 /* if G4, restore VRSAVE reg */
651 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
652 #endif /* CONFIG_ALTIVEC */
653 #ifdef CONFIG_SPE
654 BEGIN_FTR_SECTION
655 lwz r0,THREAD+THREAD_SPEFSCR(r2)
656 mtspr SPRN_SPEFSCR,r0 /* restore SPEFSCR reg */
657 END_FTR_SECTION_IFSET(CPU_FTR_SPE)
658 #endif /* CONFIG_SPE */
660 lwz r0,_CCR(r1)
661 mtcrf 0xFF,r0
662 /* r3-r12 are destroyed -- Cort */
663 REST_NVGPRS(r1)
665 lwz r4,_NIP(r1) /* Return to _switch caller in new task */
666 mtlr r4
667 addi r1,r1,INT_FRAME_SIZE
668 blr
670 .globl fast_exception_return
671 fast_exception_return:
672 #if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
673 andi. r10,r9,MSR_RI /* check for recoverable interrupt */
674 beq 1f /* if not, we've got problems */
675 #endif
677 2: REST_4GPRS(3, r11)
678 lwz r10,_CCR(r11)
679 REST_GPR(1, r11)
680 mtcr r10
681 lwz r10,_LINK(r11)
682 mtlr r10
683 REST_GPR(10, r11)
684 mtspr SPRN_SRR1,r9
685 mtspr SPRN_SRR0,r12
686 REST_GPR(9, r11)
687 REST_GPR(12, r11)
688 lwz r11,GPR11(r11)
689 SYNC
690 RFI
692 #if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
693 /* check if the exception happened in a restartable section */
694 1: lis r3,exc_exit_restart_end@ha
695 addi r3,r3,exc_exit_restart_end@l
696 cmplw r12,r3
697 bge 3f
698 lis r4,exc_exit_restart@ha
699 addi r4,r4,exc_exit_restart@l
700 cmplw r12,r4
701 blt 3f
702 lis r3,fee_restarts@ha
703 tophys(r3,r3)
704 lwz r5,fee_restarts@l(r3)
705 addi r5,r5,1
706 stw r5,fee_restarts@l(r3)
707 mr r12,r4 /* restart at exc_exit_restart */
708 b 2b
710 .section .bss
711 .align 2
712 fee_restarts:
713 .space 4
714 .previous
716 /* aargh, a nonrecoverable interrupt, panic */
717 /* aargh, we don't know which trap this is */
718 /* but the 601 doesn't implement the RI bit, so assume it's OK */
719 3:
720 BEGIN_FTR_SECTION
721 b 2b
722 END_FTR_SECTION_IFSET(CPU_FTR_601)
723 li r10,-1
724 stw r10,_TRAP(r11)
725 addi r3,r1,STACK_FRAME_OVERHEAD
726 lis r10,MSR_KERNEL@h
727 ori r10,r10,MSR_KERNEL@l
728 bl transfer_to_handler_full
729 .long nonrecoverable_exception
730 .long ret_from_except
731 #endif
733 .globl ret_from_except_full
734 ret_from_except_full:
735 REST_NVGPRS(r1)
736 /* fall through */
738 .globl ret_from_except
739 ret_from_except:
740 /* Hard-disable interrupts so that current_thread_info()->flags
741 * can't change between when we test it and when we return
742 * from the interrupt. */
743 /* Note: We don't bother telling lockdep about it */
744 LOAD_MSR_KERNEL(r10,MSR_KERNEL)
745 SYNC /* Some chip revs have problems here... */
746 MTMSRD(r10) /* disable interrupts */
748 lwz r3,_MSR(r1) /* Returning to user mode? */
749 andi. r0,r3,MSR_PR
750 beq resume_kernel
752 user_exc_return: /* r10 contains MSR_KERNEL here */
753 /* Check current_thread_info()->flags */
754 CURRENT_THREAD_INFO(r9, r1)
755 lwz r9,TI_FLAGS(r9)
756 andi. r0,r9,_TIF_USER_WORK_MASK
757 bne do_work
759 restore_user:
760 #if defined(CONFIG_4xx) || defined(CONFIG_BOOKE)
761 /* Check whether this process has its own DBCR0 value. The internal
762 debug mode bit tells us that dbcr0 should be loaded. */
763 lwz r0,THREAD+THREAD_DBCR0(r2)
764 andis. r10,r0,DBCR0_IDM@h
765 bnel- load_dbcr0
766 #endif
768 b restore
770 /* N.B. the only way to get here is from the beq following ret_from_except. */
771 resume_kernel:
772 /* check current_thread_info, _TIF_EMULATE_STACK_STORE */
773 CURRENT_THREAD_INFO(r9, r1)
774 lwz r8,TI_FLAGS(r9)
775 andis. r0,r8,_TIF_EMULATE_STACK_STORE@h
776 beq+ 1f
778 addi r8,r1,INT_FRAME_SIZE /* Get the kprobed function entry */
780 lwz r3,GPR1(r1)
781 subi r3,r3,INT_FRAME_SIZE /* dst: Allocate a trampoline exception frame */
782 mr r4,r1 /* src: current exception frame */
783 mr r1,r3 /* Reroute the trampoline frame to r1 */
785 /* Copy from the original to the trampoline. */
786 li r5,INT_FRAME_SIZE/4 /* size: INT_FRAME_SIZE */
787 li r6,0 /* start offset: 0 */
788 mtctr r5
789 2: lwzx r0,r6,r4
790 stwx r0,r6,r3
791 addi r6,r6,4
792 bdnz 2b
794 /* Do real store operation to complete stwu */
795 lwz r5,GPR1(r1)
796 stw r8,0(r5)
798 /* Clear _TIF_EMULATE_STACK_STORE flag */
799 lis r11,_TIF_EMULATE_STACK_STORE@h
800 addi r5,r9,TI_FLAGS
801 0: lwarx r8,0,r5
802 andc r8,r8,r11
803 #ifdef CONFIG_IBM405_ERR77
804 dcbt 0,r5
805 #endif
806 stwcx. r8,0,r5
807 bne- 0b
808 1:
810 #ifdef CONFIG_PREEMPT
811 /* check current_thread_info->preempt_count */
812 lwz r0,TI_PREEMPT(r9)
813 cmpwi 0,r0,0 /* if non-zero, just restore regs and return */
814 bne restore
815 andi. r8,r8,_TIF_NEED_RESCHED
816 beq+ restore
817 lwz r3,_MSR(r1)
818 andi. r0,r3,MSR_EE /* interrupts off? */
819 beq restore /* don't schedule if so */
820 #ifdef CONFIG_TRACE_IRQFLAGS
821 /* Lockdep thinks irqs are enabled, we need to call
822 * preempt_schedule_irq with IRQs off, so we inform lockdep
823 * now that we -did- turn them off already
824 */
825 bl trace_hardirqs_off
826 #endif
827 1: bl preempt_schedule_irq
828 CURRENT_THREAD_INFO(r9, r1)
829 lwz r3,TI_FLAGS(r9)
830 andi. r0,r3,_TIF_NEED_RESCHED
831 bne- 1b
832 #ifdef CONFIG_TRACE_IRQFLAGS
833 /* And now, to properly rebalance the above, we tell lockdep they
834 * are being turned back on, which will happen when we return
835 */
836 bl trace_hardirqs_on
837 #endif
838 #endif /* CONFIG_PREEMPT */
840 /* interrupts are hard-disabled at this point */
841 restore:
842 #ifdef CONFIG_44x
843 BEGIN_MMU_FTR_SECTION
844 b 1f
845 END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_47x)
846 lis r4,icache_44x_need_flush@ha
847 lwz r5,icache_44x_need_flush@l(r4)
848 cmplwi cr0,r5,0
849 beq+ 1f
850 li r6,0
851 iccci r0,r0
852 stw r6,icache_44x_need_flush@l(r4)
853 1:
854 #endif /* CONFIG_44x */
856 lwz r9,_MSR(r1)
857 #ifdef CONFIG_TRACE_IRQFLAGS
858 /* Lockdep doesn't know about the fact that IRQs are temporarily turned
859 * off in this assembly code while peeking at TI_FLAGS() and such. However
860 * we need to inform it if the exception turned interrupts off, and we
861 * are about to trun them back on.
862 *
863 * The problem here sadly is that we don't know whether the exceptions was
864 * one that turned interrupts off or not. So we always tell lockdep about
865 * turning them on here when we go back to wherever we came from with EE
866 * on, even if that may meen some redudant calls being tracked. Maybe later
867 * we could encode what the exception did somewhere or test the exception
868 * type in the pt_regs but that sounds overkill
869 */
870 andi. r10,r9,MSR_EE
871 beq 1f
872 /*
873 * Since the ftrace irqsoff latency trace checks CALLER_ADDR1,
874 * which is the stack frame here, we need to force a stack frame
875 * in case we came from user space.
876 */
877 stwu r1,-32(r1)
878 mflr r0
879 stw r0,4(r1)
880 stwu r1,-32(r1)
881 bl trace_hardirqs_on
882 lwz r1,0(r1)
883 lwz r1,0(r1)
884 lwz r9,_MSR(r1)
885 1:
886 #endif /* CONFIG_TRACE_IRQFLAGS */
888 lwz r0,GPR0(r1)
889 lwz r2,GPR2(r1)
890 REST_4GPRS(3, r1)
891 REST_2GPRS(7, r1)
893 lwz r10,_XER(r1)
894 lwz r11,_CTR(r1)
895 mtspr SPRN_XER,r10
896 mtctr r11
898 PPC405_ERR77(0,r1)
899 BEGIN_FTR_SECTION
900 lwarx r11,0,r1
901 END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX)
902 stwcx. r0,0,r1 /* to clear the reservation */
904 #if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
905 andi. r10,r9,MSR_RI /* check if this exception occurred */
906 beql nonrecoverable /* at a bad place (MSR:RI = 0) */
908 lwz r10,_CCR(r1)
909 lwz r11,_LINK(r1)
910 mtcrf 0xFF,r10
911 mtlr r11
913 /*
914 * Once we put values in SRR0 and SRR1, we are in a state
915 * where exceptions are not recoverable, since taking an
916 * exception will trash SRR0 and SRR1. Therefore we clear the
917 * MSR:RI bit to indicate this. If we do take an exception,
918 * we can't return to the point of the exception but we
919 * can restart the exception exit path at the label
920 * exc_exit_restart below. -- paulus
921 */
922 LOAD_MSR_KERNEL(r10,MSR_KERNEL & ~MSR_RI)
923 SYNC
924 MTMSRD(r10) /* clear the RI bit */
925 .globl exc_exit_restart
926 exc_exit_restart:
927 lwz r12,_NIP(r1)
928 FIX_SRR1(r9,r10)
929 mtspr SPRN_SRR0,r12
930 mtspr SPRN_SRR1,r9
931 REST_4GPRS(9, r1)
932 lwz r1,GPR1(r1)
933 .globl exc_exit_restart_end
934 exc_exit_restart_end:
935 SYNC
936 RFI
938 #else /* !(CONFIG_4xx || CONFIG_BOOKE) */
939 /*
940 * This is a bit different on 4xx/Book-E because it doesn't have
941 * the RI bit in the MSR.
942 * The TLB miss handler checks if we have interrupted
943 * the exception exit path and restarts it if so
944 * (well maybe one day it will... :).
945 */
946 lwz r11,_LINK(r1)
947 mtlr r11
948 lwz r10,_CCR(r1)
949 mtcrf 0xff,r10
950 REST_2GPRS(9, r1)
951 .globl exc_exit_restart
952 exc_exit_restart:
953 lwz r11,_NIP(r1)
954 lwz r12,_MSR(r1)
955 exc_exit_start:
956 mtspr SPRN_SRR0,r11
957 mtspr SPRN_SRR1,r12
958 REST_2GPRS(11, r1)
959 lwz r1,GPR1(r1)
960 .globl exc_exit_restart_end
961 exc_exit_restart_end:
962 PPC405_ERR77_SYNC
963 rfi
964 b . /* prevent prefetch past rfi */
966 /*
967 * Returning from a critical interrupt in user mode doesn't need
968 * to be any different from a normal exception. For a critical
969 * interrupt in the kernel, we just return (without checking for
970 * preemption) since the interrupt may have happened at some crucial
971 * place (e.g. inside the TLB miss handler), and because we will be
972 * running with r1 pointing into critical_stack, not the current
973 * process's kernel stack (and therefore current_thread_info() will
974 * give the wrong answer).
975 * We have to restore various SPRs that may have been in use at the
976 * time of the critical interrupt.
977 *
978 */
979 #ifdef CONFIG_40x
980 #define PPC_40x_TURN_OFF_MSR_DR \
981 /* avoid any possible TLB misses here by turning off MSR.DR, we \
982 * assume the instructions here are mapped by a pinned TLB entry */ \
983 li r10,MSR_IR; \
984 mtmsr r10; \
985 isync; \
986 tophys(r1, r1);
987 #else
988 #define PPC_40x_TURN_OFF_MSR_DR
989 #endif
991 #define RET_FROM_EXC_LEVEL(exc_lvl_srr0, exc_lvl_srr1, exc_lvl_rfi) \
992 REST_NVGPRS(r1); \
993 lwz r3,_MSR(r1); \
994 andi. r3,r3,MSR_PR; \
995 LOAD_MSR_KERNEL(r10,MSR_KERNEL); \
996 bne user_exc_return; \
997 lwz r0,GPR0(r1); \
998 lwz r2,GPR2(r1); \
999 REST_4GPRS(3, r1); \
1000 REST_2GPRS(7, r1); \
1001 lwz r10,_XER(r1); \
1002 lwz r11,_CTR(r1); \
1003 mtspr SPRN_XER,r10; \
1004 mtctr r11; \
1005 PPC405_ERR77(0,r1); \
1006 stwcx. r0,0,r1; /* to clear the reservation */ \
1007 lwz r11,_LINK(r1); \
1008 mtlr r11; \
1009 lwz r10,_CCR(r1); \
1010 mtcrf 0xff,r10; \
1011 PPC_40x_TURN_OFF_MSR_DR; \
1012 lwz r9,_DEAR(r1); \
1013 lwz r10,_ESR(r1); \
1014 mtspr SPRN_DEAR,r9; \
1015 mtspr SPRN_ESR,r10; \
1016 lwz r11,_NIP(r1); \
1017 lwz r12,_MSR(r1); \
1018 mtspr exc_lvl_srr0,r11; \
1019 mtspr exc_lvl_srr1,r12; \
1020 lwz r9,GPR9(r1); \
1021 lwz r12,GPR12(r1); \
1022 lwz r10,GPR10(r1); \
1023 lwz r11,GPR11(r1); \
1024 lwz r1,GPR1(r1); \
1025 PPC405_ERR77_SYNC; \
1026 exc_lvl_rfi; \
1027 b .; /* prevent prefetch past exc_lvl_rfi */
1029 #define RESTORE_xSRR(exc_lvl_srr0, exc_lvl_srr1) \
1030 lwz r9,_##exc_lvl_srr0(r1); \
1031 lwz r10,_##exc_lvl_srr1(r1); \
1032 mtspr SPRN_##exc_lvl_srr0,r9; \
1033 mtspr SPRN_##exc_lvl_srr1,r10;
1035 #if defined(CONFIG_PPC_BOOK3E_MMU)
1036 #ifdef CONFIG_PHYS_64BIT
1037 #define RESTORE_MAS7 \
1038 lwz r11,MAS7(r1); \
1039 mtspr SPRN_MAS7,r11;
1040 #else
1041 #define RESTORE_MAS7
1042 #endif /* CONFIG_PHYS_64BIT */
1043 #define RESTORE_MMU_REGS \
1044 lwz r9,MAS0(r1); \
1045 lwz r10,MAS1(r1); \
1046 lwz r11,MAS2(r1); \
1047 mtspr SPRN_MAS0,r9; \
1048 lwz r9,MAS3(r1); \
1049 mtspr SPRN_MAS1,r10; \
1050 lwz r10,MAS6(r1); \
1051 mtspr SPRN_MAS2,r11; \
1052 mtspr SPRN_MAS3,r9; \
1053 mtspr SPRN_MAS6,r10; \
1054 RESTORE_MAS7;
1055 #elif defined(CONFIG_44x)
1056 #define RESTORE_MMU_REGS \
1057 lwz r9,MMUCR(r1); \
1058 mtspr SPRN_MMUCR,r9;
1059 #else
1060 #define RESTORE_MMU_REGS
1061 #endif
1063 #ifdef CONFIG_40x
1064 .globl ret_from_crit_exc
1065 ret_from_crit_exc:
1066 mfspr r9,SPRN_SPRG_THREAD
1067 lis r10,saved_ksp_limit@ha;
1068 lwz r10,saved_ksp_limit@l(r10);
1069 tovirt(r9,r9);
1070 stw r10,KSP_LIMIT(r9)
1071 lis r9,crit_srr0@ha;
1072 lwz r9,crit_srr0@l(r9);
1073 lis r10,crit_srr1@ha;
1074 lwz r10,crit_srr1@l(r10);
1075 mtspr SPRN_SRR0,r9;
1076 mtspr SPRN_SRR1,r10;
1077 RET_FROM_EXC_LEVEL(SPRN_CSRR0, SPRN_CSRR1, PPC_RFCI)
1078 #endif /* CONFIG_40x */
1080 #ifdef CONFIG_BOOKE
1081 .globl ret_from_crit_exc
1082 ret_from_crit_exc:
1083 mfspr r9,SPRN_SPRG_THREAD
1084 lwz r10,SAVED_KSP_LIMIT(r1)
1085 stw r10,KSP_LIMIT(r9)
1086 RESTORE_xSRR(SRR0,SRR1);
1087 RESTORE_MMU_REGS;
1088 RET_FROM_EXC_LEVEL(SPRN_CSRR0, SPRN_CSRR1, PPC_RFCI)
1090 .globl ret_from_debug_exc
1091 ret_from_debug_exc:
1092 mfspr r9,SPRN_SPRG_THREAD
1093 lwz r10,SAVED_KSP_LIMIT(r1)
1094 stw r10,KSP_LIMIT(r9)
1095 lwz r9,THREAD_INFO-THREAD(r9)
1096 CURRENT_THREAD_INFO(r10, r1)
1097 lwz r10,TI_PREEMPT(r10)
1098 stw r10,TI_PREEMPT(r9)
1099 RESTORE_xSRR(SRR0,SRR1);
1100 RESTORE_xSRR(CSRR0,CSRR1);
1101 RESTORE_MMU_REGS;
1102 RET_FROM_EXC_LEVEL(SPRN_DSRR0, SPRN_DSRR1, PPC_RFDI)
1104 .globl ret_from_mcheck_exc
1105 ret_from_mcheck_exc:
1106 mfspr r9,SPRN_SPRG_THREAD
1107 lwz r10,SAVED_KSP_LIMIT(r1)
1108 stw r10,KSP_LIMIT(r9)
1109 RESTORE_xSRR(SRR0,SRR1);
1110 RESTORE_xSRR(CSRR0,CSRR1);
1111 RESTORE_xSRR(DSRR0,DSRR1);
1112 RESTORE_MMU_REGS;
1113 RET_FROM_EXC_LEVEL(SPRN_MCSRR0, SPRN_MCSRR1, PPC_RFMCI)
1114 #endif /* CONFIG_BOOKE */
1116 /*
1117 * Load the DBCR0 value for a task that is being ptraced,
1118 * having first saved away the global DBCR0. Note that r0
1119 * has the dbcr0 value to set upon entry to this.
1120 */
1121 load_dbcr0:
1122 mfmsr r10 /* first disable debug exceptions */
1123 rlwinm r10,r10,0,~MSR_DE
1124 mtmsr r10
1125 isync
1126 mfspr r10,SPRN_DBCR0
1127 lis r11,global_dbcr0@ha
1128 addi r11,r11,global_dbcr0@l
1129 #ifdef CONFIG_SMP
1130 CURRENT_THREAD_INFO(r9, r1)
1131 lwz r9,TI_CPU(r9)
1132 slwi r9,r9,3
1133 add r11,r11,r9
1134 #endif
1135 stw r10,0(r11)
1136 mtspr SPRN_DBCR0,r0
1137 lwz r10,4(r11)
1138 addi r10,r10,1
1139 stw r10,4(r11)
1140 li r11,-1
1141 mtspr SPRN_DBSR,r11 /* clear all pending debug events */
1142 blr
1144 .section .bss
1145 .align 4
1146 global_dbcr0:
1147 .space 8*NR_CPUS
1148 .previous
1149 #endif /* !(CONFIG_4xx || CONFIG_BOOKE) */
1151 do_work: /* r10 contains MSR_KERNEL here */
1152 andi. r0,r9,_TIF_NEED_RESCHED
1153 beq do_user_signal
1155 do_resched: /* r10 contains MSR_KERNEL here */
1156 /* Note: We don't need to inform lockdep that we are enabling
1157 * interrupts here. As far as it knows, they are already enabled
1158 */
1159 ori r10,r10,MSR_EE
1160 SYNC
1161 MTMSRD(r10) /* hard-enable interrupts */
1162 bl schedule
1163 recheck:
1164 /* Note: And we don't tell it we are disabling them again
1165 * neither. Those disable/enable cycles used to peek at
1166 * TI_FLAGS aren't advertised.
1167 */
1168 LOAD_MSR_KERNEL(r10,MSR_KERNEL)
1169 SYNC
1170 MTMSRD(r10) /* disable interrupts */
1171 CURRENT_THREAD_INFO(r9, r1)
1172 lwz r9,TI_FLAGS(r9)
1173 andi. r0,r9,_TIF_NEED_RESCHED
1174 bne- do_resched
1175 andi. r0,r9,_TIF_USER_WORK_MASK
1176 beq restore_user
1177 do_user_signal: /* r10 contains MSR_KERNEL here */
1178 ori r10,r10,MSR_EE
1179 SYNC
1180 MTMSRD(r10) /* hard-enable interrupts */
1181 /* save r13-r31 in the exception frame, if not already done */
1182 lwz r3,_TRAP(r1)
1183 andi. r0,r3,1
1184 beq 2f
1185 SAVE_NVGPRS(r1)
1186 rlwinm r3,r3,0,0,30
1187 stw r3,_TRAP(r1)
1188 2: addi r3,r1,STACK_FRAME_OVERHEAD
1189 mr r4,r9
1190 bl do_notify_resume
1191 REST_NVGPRS(r1)
1192 b recheck
1194 /*
1195 * We come here when we are at the end of handling an exception
1196 * that occurred at a place where taking an exception will lose
1197 * state information, such as the contents of SRR0 and SRR1.
1198 */
1199 nonrecoverable:
1200 lis r10,exc_exit_restart_end@ha
1201 addi r10,r10,exc_exit_restart_end@l
1202 cmplw r12,r10
1203 bge 3f
1204 lis r11,exc_exit_restart@ha
1205 addi r11,r11,exc_exit_restart@l
1206 cmplw r12,r11
1207 blt 3f
1208 lis r10,ee_restarts@ha
1209 lwz r12,ee_restarts@l(r10)
1210 addi r12,r12,1
1211 stw r12,ee_restarts@l(r10)
1212 mr r12,r11 /* restart at exc_exit_restart */
1213 blr
1214 3: /* OK, we can't recover, kill this process */
1215 /* but the 601 doesn't implement the RI bit, so assume it's OK */
1216 BEGIN_FTR_SECTION
1217 blr
1218 END_FTR_SECTION_IFSET(CPU_FTR_601)
1219 lwz r3,_TRAP(r1)
1220 andi. r0,r3,1
1221 beq 4f
1222 SAVE_NVGPRS(r1)
1223 rlwinm r3,r3,0,0,30
1224 stw r3,_TRAP(r1)
1225 4: addi r3,r1,STACK_FRAME_OVERHEAD
1226 bl nonrecoverable_exception
1227 /* shouldn't return */
1228 b 4b
1230 .section .bss
1231 .align 2
1232 ee_restarts:
1233 .space 4
1234 .previous
1236 /*
1237 * PROM code for specific machines follows. Put it
1238 * here so it's easy to add arch-specific sections later.
1239 * -- Cort
1240 */
1241 #ifdef CONFIG_PPC_RTAS
1242 /*
1243 * On CHRP, the Run-Time Abstraction Services (RTAS) have to be
1244 * called with the MMU off.
1245 */
1246 _GLOBAL(enter_rtas)
1247 stwu r1,-INT_FRAME_SIZE(r1)
1248 mflr r0
1249 stw r0,INT_FRAME_SIZE+4(r1)
1250 LOAD_REG_ADDR(r4, rtas)
1251 lis r6,1f@ha /* physical return address for rtas */
1252 addi r6,r6,1f@l
1253 tophys(r6,r6)
1254 tophys(r7,r1)
1255 lwz r8,RTASENTRY(r4)
1256 lwz r4,RTASBASE(r4)
1257 mfmsr r9
1258 stw r9,8(r1)
1259 LOAD_MSR_KERNEL(r0,MSR_KERNEL)
1260 SYNC /* disable interrupts so SRR0/1 */
1261 MTMSRD(r0) /* don't get trashed */
1262 li r9,MSR_KERNEL & ~(MSR_IR|MSR_DR)
1263 mtlr r6
1264 mtspr SPRN_SPRG_RTAS,r7
1265 mtspr SPRN_SRR0,r8
1266 mtspr SPRN_SRR1,r9
1267 RFI
1268 1: tophys(r9,r1)
1269 lwz r8,INT_FRAME_SIZE+4(r9) /* get return address */
1270 lwz r9,8(r9) /* original msr value */
1271 FIX_SRR1(r9,r0)
1272 addi r1,r1,INT_FRAME_SIZE
1273 li r0,0
1274 mtspr SPRN_SPRG_RTAS,r0
1275 mtspr SPRN_SRR0,r8
1276 mtspr SPRN_SRR1,r9
1277 RFI /* return to caller */
1279 .globl machine_check_in_rtas
1280 machine_check_in_rtas:
1281 twi 31,0,0
1282 /* XXX load up BATs and panic */
1284 #endif /* CONFIG_PPC_RTAS */
1286 #ifdef CONFIG_FUNCTION_TRACER
1287 #ifdef CONFIG_DYNAMIC_FTRACE
1288 _GLOBAL(mcount)
1289 _GLOBAL(_mcount)
1290 /*
1291 * It is required that _mcount on PPC32 must preserve the
1292 * link register. But we have r0 to play with. We use r0
1293 * to push the return address back to the caller of mcount
1294 * into the ctr register, restore the link register and
1295 * then jump back using the ctr register.
1296 */
1297 mflr r0
1298 mtctr r0
1299 lwz r0, 4(r1)
1300 mtlr r0
1301 bctr
1303 _GLOBAL(ftrace_caller)
1304 MCOUNT_SAVE_FRAME
1305 /* r3 ends up with link register */
1306 subi r3, r3, MCOUNT_INSN_SIZE
1307 .globl ftrace_call
1308 ftrace_call:
1309 bl ftrace_stub
1310 nop
1311 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1312 .globl ftrace_graph_call
1313 ftrace_graph_call:
1314 b ftrace_graph_stub
1315 _GLOBAL(ftrace_graph_stub)
1316 #endif
1317 MCOUNT_RESTORE_FRAME
1318 /* old link register ends up in ctr reg */
1319 bctr
1320 #else
1321 _GLOBAL(mcount)
1322 _GLOBAL(_mcount)
1324 MCOUNT_SAVE_FRAME
1326 subi r3, r3, MCOUNT_INSN_SIZE
1327 LOAD_REG_ADDR(r5, ftrace_trace_function)
1328 lwz r5,0(r5)
1330 mtctr r5
1331 bctrl
1332 nop
1334 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1335 b ftrace_graph_caller
1336 #endif
1337 MCOUNT_RESTORE_FRAME
1338 bctr
1339 #endif
1341 _GLOBAL(ftrace_stub)
1342 blr
1344 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1345 _GLOBAL(ftrace_graph_caller)
1346 /* load r4 with local address */
1347 lwz r4, 44(r1)
1348 subi r4, r4, MCOUNT_INSN_SIZE
1350 /* Grab the LR out of the caller stack frame */
1351 lwz r3,52(r1)
1353 bl prepare_ftrace_return
1354 nop
1356 /*
1357 * prepare_ftrace_return gives us the address we divert to.
1358 * Change the LR in the callers stack frame to this.
1359 */
1360 stw r3,52(r1)
1362 MCOUNT_RESTORE_FRAME
1363 /* old link register ends up in ctr reg */
1364 bctr
1366 _GLOBAL(return_to_handler)
1367 /* need to save return values */
1368 stwu r1, -32(r1)
1369 stw r3, 20(r1)
1370 stw r4, 16(r1)
1371 stw r31, 12(r1)
1372 mr r31, r1
1374 bl ftrace_return_to_handler
1375 nop
1377 /* return value has real return address */
1378 mtlr r3
1380 lwz r3, 20(r1)
1381 lwz r4, 16(r1)
1382 lwz r31,12(r1)
1383 lwz r1, 0(r1)
1385 /* Jump back to real return address */
1386 blr
1387 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1389 #endif /* CONFIG_FUNCTION_TRACER */