| blob | 22b45a4955cd8f03a26c89a7fb2ab79252f2b6d4 |
1 /*
2 * PowerPC version
3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 * Rewritten by Cort Dougan (cort@fsmlabs.com) for PReP
5 * Copyright (C) 1996 Cort Dougan <cort@fsmlabs.com>
6 * Adapted for Power Macintosh by Paul Mackerras.
7 * Low-level exception handlers and MMU support
8 * rewritten by Paul Mackerras.
9 * Copyright (C) 1996 Paul Mackerras.
10 * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
11 *
12 * This file contains the system call entry code, context switch
13 * code, and exception/interrupt return code for PowerPC.
14 *
15 * This program is free software; you can redistribute it and/or
16 * modify it under the terms of the GNU General Public License
17 * as published by the Free Software Foundation; either version
18 * 2 of the License, or (at your option) any later version.
19 *
20 */
22 #include <linux/errno.h>
23 #include <linux/sys.h>
24 #include <linux/threads.h>
25 #include <asm/reg.h>
26 #include <asm/page.h>
27 #include <asm/mmu.h>
28 #include <asm/cputable.h>
29 #include <asm/thread_info.h>
30 #include <asm/ppc_asm.h>
31 #include <asm/asm-offsets.h>
32 #include <asm/unistd.h>
33 #include <asm/ftrace.h>
34 #include <asm/ptrace.h>
36 #undef SHOW_SYSCALLS
37 #undef SHOW_SYSCALLS_TASK
39 /*
40 * MSR_KERNEL is > 0x10000 on 4xx/Book-E since it include MSR_CE.
41 */
42 #if MSR_KERNEL >= 0x10000
43 #define LOAD_MSR_KERNEL(r, x) lis r,(x)@h; ori r,r,(x)@l
44 #else
45 #define LOAD_MSR_KERNEL(r, x) li r,(x)
46 #endif
48 #ifdef CONFIG_BOOKE
49 .globl mcheck_transfer_to_handler
50 mcheck_transfer_to_handler:
51 mfspr r0,SPRN_DSRR0
52 stw r0,_DSRR0(r11)
53 mfspr r0,SPRN_DSRR1
54 stw r0,_DSRR1(r11)
55 /* fall through */
57 .globl debug_transfer_to_handler
58 debug_transfer_to_handler:
59 mfspr r0,SPRN_CSRR0
60 stw r0,_CSRR0(r11)
61 mfspr r0,SPRN_CSRR1
62 stw r0,_CSRR1(r11)
63 /* fall through */
65 .globl crit_transfer_to_handler
66 crit_transfer_to_handler:
67 #ifdef CONFIG_PPC_BOOK3E_MMU
68 mfspr r0,SPRN_MAS0
69 stw r0,MAS0(r11)
70 mfspr r0,SPRN_MAS1
71 stw r0,MAS1(r11)
72 mfspr r0,SPRN_MAS2
73 stw r0,MAS2(r11)
74 mfspr r0,SPRN_MAS3
75 stw r0,MAS3(r11)
76 mfspr r0,SPRN_MAS6
77 stw r0,MAS6(r11)
78 #ifdef CONFIG_PHYS_64BIT
79 mfspr r0,SPRN_MAS7
80 stw r0,MAS7(r11)
81 #endif /* CONFIG_PHYS_64BIT */
82 #endif /* CONFIG_PPC_BOOK3E_MMU */
83 #ifdef CONFIG_44x
84 mfspr r0,SPRN_MMUCR
85 stw r0,MMUCR(r11)
86 #endif
87 mfspr r0,SPRN_SRR0
88 stw r0,_SRR0(r11)
89 mfspr r0,SPRN_SRR1
90 stw r0,_SRR1(r11)
92 /* set the stack limit to the current stack
93 * and set the limit to protect the thread_info
94 * struct
95 */
96 mfspr r8,SPRN_SPRG_THREAD
97 lwz r0,KSP_LIMIT(r8)
98 stw r0,SAVED_KSP_LIMIT(r11)
99 rlwimi r0,r1,0,0,(31-THREAD_SHIFT)
100 stw r0,KSP_LIMIT(r8)
101 /* fall through */
102 #endif
104 #ifdef CONFIG_40x
105 .globl crit_transfer_to_handler
106 crit_transfer_to_handler:
107 lwz r0,crit_r10@l(0)
108 stw r0,GPR10(r11)
109 lwz r0,crit_r11@l(0)
110 stw r0,GPR11(r11)
111 mfspr r0,SPRN_SRR0
112 stw r0,crit_srr0@l(0)
113 mfspr r0,SPRN_SRR1
114 stw r0,crit_srr1@l(0)
116 /* set the stack limit to the current stack
117 * and set the limit to protect the thread_info
118 * struct
119 */
120 mfspr r8,SPRN_SPRG_THREAD
121 lwz r0,KSP_LIMIT(r8)
122 stw r0,saved_ksp_limit@l(0)
123 rlwimi r0,r1,0,0,(31-THREAD_SHIFT)
124 stw r0,KSP_LIMIT(r8)
125 /* fall through */
126 #endif
128 /*
129 * This code finishes saving the registers to the exception frame
130 * and jumps to the appropriate handler for the exception, turning
131 * on address translation.
132 * Note that we rely on the caller having set cr0.eq iff the exception
133 * occurred in kernel mode (i.e. MSR:PR = 0).
134 */
135 .globl transfer_to_handler_full
136 transfer_to_handler_full:
137 SAVE_NVGPRS(r11)
138 /* fall through */
140 .globl transfer_to_handler
141 transfer_to_handler:
142 stw r2,GPR2(r11)
143 stw r12,_NIP(r11)
144 stw r9,_MSR(r11)
145 andi. r2,r9,MSR_PR
146 mfctr r12
147 mfspr r2,SPRN_XER
148 stw r12,_CTR(r11)
149 stw r2,_XER(r11)
150 mfspr r12,SPRN_SPRG_THREAD
151 addi r2,r12,-THREAD
152 tovirt(r2,r2) /* set r2 to current */
153 beq 2f /* if from user, fix up THREAD.regs */
154 addi r11,r1,STACK_FRAME_OVERHEAD
155 stw r11,PT_REGS(r12)
156 #if defined(CONFIG_40x) || defined(CONFIG_BOOKE)
157 /* Check to see if the dbcr0 register is set up to debug. Use the
158 internal debug mode bit to do this. */
159 lwz r12,THREAD_DBCR0(r12)
160 andis. r12,r12,DBCR0_IDM@h
161 beq+ 3f
162 /* From user and task is ptraced - load up global dbcr0 */
163 li r12,-1 /* clear all pending debug events */
164 mtspr SPRN_DBSR,r12
165 lis r11,global_dbcr0@ha
166 tophys(r11,r11)
167 addi r11,r11,global_dbcr0@l
168 #ifdef CONFIG_SMP
169 CURRENT_THREAD_INFO(r9, r1)
170 lwz r9,TI_CPU(r9)
171 slwi r9,r9,3
172 add r11,r11,r9
173 #endif
174 lwz r12,0(r11)
175 mtspr SPRN_DBCR0,r12
176 lwz r12,4(r11)
177 addi r12,r12,-1
178 stw r12,4(r11)
179 #endif
180 b 3f
182 2: /* if from kernel, check interrupted DOZE/NAP mode and
183 * check for stack overflow
184 */
185 lwz r9,KSP_LIMIT(r12)
186 cmplw r1,r9 /* if r1 <= ksp_limit */
187 ble- stack_ovf /* then the kernel stack overflowed */
188 5:
189 #if defined(CONFIG_6xx) || defined(CONFIG_E500)
190 CURRENT_THREAD_INFO(r9, r1)
191 tophys(r9,r9) /* check local flags */
192 lwz r12,TI_LOCAL_FLAGS(r9)
193 mtcrf 0x01,r12
194 bt- 31-TLF_NAPPING,4f
195 bt- 31-TLF_SLEEPING,7f
196 #endif /* CONFIG_6xx || CONFIG_E500 */
197 .globl transfer_to_handler_cont
198 transfer_to_handler_cont:
199 3:
200 mflr r9
201 lwz r11,0(r9) /* virtual address of handler */
202 lwz r9,4(r9) /* where to go when done */
203 #ifdef CONFIG_TRACE_IRQFLAGS
204 lis r12,reenable_mmu@h
205 ori r12,r12,reenable_mmu@l
206 mtspr SPRN_SRR0,r12
207 mtspr SPRN_SRR1,r10
208 SYNC
209 RFI
210 reenable_mmu: /* re-enable mmu so we can */
211 mfmsr r10
212 lwz r12,_MSR(r1)
213 xor r10,r10,r12
214 andi. r10,r10,MSR_EE /* Did EE change? */
215 beq 1f
217 /*
218 * The trace_hardirqs_off will use CALLER_ADDR0 and CALLER_ADDR1.
219 * If from user mode there is only one stack frame on the stack, and
220 * accessing CALLER_ADDR1 will cause oops. So we need create a dummy
221 * stack frame to make trace_hardirqs_off happy.
222 *
223 * This is handy because we also need to save a bunch of GPRs,
224 * r3 can be different from GPR3(r1) at this point, r9 and r11
225 * contains the old MSR and handler address respectively,
226 * r4 & r5 can contain page fault arguments that need to be passed
227 * along as well. r12, CCR, CTR, XER etc... are left clobbered as
228 * they aren't useful past this point (aren't syscall arguments),
229 * the rest is restored from the exception frame.
230 */
231 stwu r1,-32(r1)
232 stw r9,8(r1)
233 stw r11,12(r1)
234 stw r3,16(r1)
235 stw r4,20(r1)
236 stw r5,24(r1)
237 bl trace_hardirqs_off
238 lwz r5,24(r1)
239 lwz r4,20(r1)
240 lwz r3,16(r1)
241 lwz r11,12(r1)
242 lwz r9,8(r1)
243 addi r1,r1,32
244 lwz r0,GPR0(r1)
245 lwz r6,GPR6(r1)
246 lwz r7,GPR7(r1)
247 lwz r8,GPR8(r1)
248 1: mtctr r11
249 mtlr r9
250 bctr /* jump to handler */
251 #else /* CONFIG_TRACE_IRQFLAGS */
252 mtspr SPRN_SRR0,r11
253 mtspr SPRN_SRR1,r10
254 mtlr r9
255 SYNC
256 RFI /* jump to handler, enable MMU */
257 #endif /* CONFIG_TRACE_IRQFLAGS */
259 #if defined (CONFIG_6xx) || defined(CONFIG_E500)
260 4: rlwinm r12,r12,0,~_TLF_NAPPING
261 stw r12,TI_LOCAL_FLAGS(r9)
262 b power_save_ppc32_restore
264 7: rlwinm r12,r12,0,~_TLF_SLEEPING
265 stw r12,TI_LOCAL_FLAGS(r9)
266 lwz r9,_MSR(r11) /* if sleeping, clear MSR.EE */
267 rlwinm r9,r9,0,~MSR_EE
268 lwz r12,_LINK(r11) /* and return to address in LR */
269 b fast_exception_return
270 #endif
272 /*
273 * On kernel stack overflow, load up an initial stack pointer
274 * and call StackOverflow(regs), which should not return.
275 */
276 stack_ovf:
277 /* sometimes we use a statically-allocated stack, which is OK. */
278 lis r12,_end@h
279 ori r12,r12,_end@l
280 cmplw r1,r12
281 ble 5b /* r1 <= &_end is OK */
282 SAVE_NVGPRS(r11)
283 addi r3,r1,STACK_FRAME_OVERHEAD
284 lis r1,init_thread_union@ha
285 addi r1,r1,init_thread_union@l
286 addi r1,r1,THREAD_SIZE-STACK_FRAME_OVERHEAD
287 lis r9,StackOverflow@ha
288 addi r9,r9,StackOverflow@l
289 LOAD_MSR_KERNEL(r10,MSR_KERNEL)
290 FIX_SRR1(r10,r12)
291 mtspr SPRN_SRR0,r9
292 mtspr SPRN_SRR1,r10
293 SYNC
294 RFI
296 /*
297 * Handle a system call.
298 */
299 .stabs "arch/powerpc/kernel/",N_SO,0,0,0f
300 .stabs "entry_32.S",N_SO,0,0,0f
301 0:
303 _GLOBAL(DoSyscall)
304 stw r3,ORIG_GPR3(r1)
305 li r12,0
306 stw r12,RESULT(r1)
307 lwz r11,_CCR(r1) /* Clear SO bit in CR */
308 rlwinm r11,r11,0,4,2
309 stw r11,_CCR(r1)
310 #ifdef SHOW_SYSCALLS
311 bl do_show_syscall
312 #endif /* SHOW_SYSCALLS */
313 #ifdef CONFIG_TRACE_IRQFLAGS
314 /* Return from syscalls can (and generally will) hard enable
315 * interrupts. You aren't supposed to call a syscall with
316 * interrupts disabled in the first place. However, to ensure
317 * that we get it right vs. lockdep if it happens, we force
318 * that hard enable here with appropriate tracing if we see
319 * that we have been called with interrupts off
320 */
321 mfmsr r11
322 andi. r12,r11,MSR_EE
323 bne+ 1f
324 /* We came in with interrupts disabled, we enable them now */
325 bl trace_hardirqs_on
326 mfmsr r11
327 lwz r0,GPR0(r1)
328 lwz r3,GPR3(r1)
329 lwz r4,GPR4(r1)
330 ori r11,r11,MSR_EE
331 lwz r5,GPR5(r1)
332 lwz r6,GPR6(r1)
333 lwz r7,GPR7(r1)
334 lwz r8,GPR8(r1)
335 mtmsr r11
336 1:
337 #endif /* CONFIG_TRACE_IRQFLAGS */
338 CURRENT_THREAD_INFO(r10, r1)
339 lwz r11,TI_FLAGS(r10)
340 andi. r11,r11,_TIF_SYSCALL_T_OR_A
341 bne- syscall_dotrace
342 syscall_dotrace_cont:
343 cmplwi 0,r0,NR_syscalls
344 lis r10,sys_call_table@h
345 ori r10,r10,sys_call_table@l
346 slwi r0,r0,2
347 bge- 66f
348 lwzx r10,r10,r0 /* Fetch system call handler [ptr] */
349 mtlr r10
350 addi r9,r1,STACK_FRAME_OVERHEAD
351 PPC440EP_ERR42
352 blrl /* Call handler */
353 .globl ret_from_syscall
354 ret_from_syscall:
355 #ifdef SHOW_SYSCALLS
356 bl do_show_syscall_exit
357 #endif
358 mr r6,r3
359 CURRENT_THREAD_INFO(r12, r1)
360 /* disable interrupts so current_thread_info()->flags can't change */
361 LOAD_MSR_KERNEL(r10,MSR_KERNEL) /* doesn't include MSR_EE */
362 /* Note: We don't bother telling lockdep about it */
363 SYNC
364 MTMSRD(r10)
365 lwz r9,TI_FLAGS(r12)
366 li r8,-_LAST_ERRNO
367 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
368 bne- syscall_exit_work
369 cmplw 0,r3,r8
370 blt+ syscall_exit_cont
371 lwz r11,_CCR(r1) /* Load CR */
372 neg r3,r3
373 oris r11,r11,0x1000 /* Set SO bit in CR */
374 stw r11,_CCR(r1)
375 syscall_exit_cont:
376 lwz r8,_MSR(r1)
377 #ifdef CONFIG_TRACE_IRQFLAGS
378 /* If we are going to return from the syscall with interrupts
379 * off, we trace that here. It shouldn't happen though but we
380 * want to catch the bugger if it does right ?
381 */
382 andi. r10,r8,MSR_EE
383 bne+ 1f
384 stw r3,GPR3(r1)
385 bl trace_hardirqs_off
386 lwz r3,GPR3(r1)
387 1:
388 #endif /* CONFIG_TRACE_IRQFLAGS */
389 #if defined(CONFIG_4xx) || defined(CONFIG_BOOKE)
390 /* If the process has its own DBCR0 value, load it up. The internal
391 debug mode bit tells us that dbcr0 should be loaded. */
392 lwz r0,THREAD+THREAD_DBCR0(r2)
393 andis. r10,r0,DBCR0_IDM@h
394 bnel- load_dbcr0
395 #endif
396 #ifdef CONFIG_44x
397 BEGIN_MMU_FTR_SECTION
398 lis r4,icache_44x_need_flush@ha
399 lwz r5,icache_44x_need_flush@l(r4)
400 cmplwi cr0,r5,0
401 bne- 2f
402 1:
403 END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_47x)
404 #endif /* CONFIG_44x */
405 BEGIN_FTR_SECTION
406 lwarx r7,0,r1
407 END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX)
408 stwcx. r0,0,r1 /* to clear the reservation */
409 lwz r4,_LINK(r1)
410 lwz r5,_CCR(r1)
411 mtlr r4
412 mtcr r5
413 lwz r7,_NIP(r1)
414 FIX_SRR1(r8, r0)
415 lwz r2,GPR2(r1)
416 lwz r1,GPR1(r1)
417 mtspr SPRN_SRR0,r7
418 mtspr SPRN_SRR1,r8
419 SYNC
420 RFI
421 #ifdef CONFIG_44x
422 2: li r7,0
423 iccci r0,r0
424 stw r7,icache_44x_need_flush@l(r4)
425 b 1b
426 #endif /* CONFIG_44x */
428 66: li r3,-ENOSYS
429 b ret_from_syscall
431 .globl ret_from_fork
432 ret_from_fork:
433 REST_NVGPRS(r1)
434 bl schedule_tail
435 li r3,0
436 b ret_from_syscall
438 .globl ret_from_kernel_thread
439 ret_from_kernel_thread:
440 REST_NVGPRS(r1)
441 bl schedule_tail
442 mtlr r14
443 mr r3,r15
444 PPC440EP_ERR42
445 blrl
446 li r3,0
447 b ret_from_syscall
449 /* Traced system call support */
450 syscall_dotrace:
451 SAVE_NVGPRS(r1)
452 li r0,0xc00
453 stw r0,_TRAP(r1)
454 addi r3,r1,STACK_FRAME_OVERHEAD
455 bl do_syscall_trace_enter
456 /*
457 * Restore argument registers possibly just changed.
458 * We use the return value of do_syscall_trace_enter
459 * for call number to look up in the table (r0).
460 */
461 mr r0,r3
462 lwz r3,GPR3(r1)
463 lwz r4,GPR4(r1)
464 lwz r5,GPR5(r1)
465 lwz r6,GPR6(r1)
466 lwz r7,GPR7(r1)
467 lwz r8,GPR8(r1)
468 REST_NVGPRS(r1)
469 b syscall_dotrace_cont
471 syscall_exit_work:
472 andi. r0,r9,_TIF_RESTOREALL
473 beq+ 0f
474 REST_NVGPRS(r1)
475 b 2f
476 0: cmplw 0,r3,r8
477 blt+ 1f
478 andi. r0,r9,_TIF_NOERROR
479 bne- 1f
480 lwz r11,_CCR(r1) /* Load CR */
481 neg r3,r3
482 oris r11,r11,0x1000 /* Set SO bit in CR */
483 stw r11,_CCR(r1)
485 1: stw r6,RESULT(r1) /* Save result */
486 stw r3,GPR3(r1) /* Update return value */
487 2: andi. r0,r9,(_TIF_PERSYSCALL_MASK)
488 beq 4f
490 /* Clear per-syscall TIF flags if any are set. */
492 li r11,_TIF_PERSYSCALL_MASK
493 addi r12,r12,TI_FLAGS
494 3: lwarx r8,0,r12
495 andc r8,r8,r11
496 #ifdef CONFIG_IBM405_ERR77
497 dcbt 0,r12
498 #endif
499 stwcx. r8,0,r12
500 bne- 3b
501 subi r12,r12,TI_FLAGS
503 4: /* Anything which requires enabling interrupts? */
504 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP)
505 beq ret_from_except
507 /* Re-enable interrupts. There is no need to trace that with
508 * lockdep as we are supposed to have IRQs on at this point
509 */
510 ori r10,r10,MSR_EE
511 SYNC
512 MTMSRD(r10)
514 /* Save NVGPRS if they're not saved already */
515 lwz r4,_TRAP(r1)
516 andi. r4,r4,1
517 beq 5f
518 SAVE_NVGPRS(r1)
519 li r4,0xc00
520 stw r4,_TRAP(r1)
521 5:
522 addi r3,r1,STACK_FRAME_OVERHEAD
523 bl do_syscall_trace_leave
524 b ret_from_except_full
526 #ifdef SHOW_SYSCALLS
527 do_show_syscall:
528 #ifdef SHOW_SYSCALLS_TASK
529 lis r11,show_syscalls_task@ha
530 lwz r11,show_syscalls_task@l(r11)
531 cmp 0,r2,r11
532 bnelr
533 #endif
534 stw r31,GPR31(r1)
535 mflr r31
536 lis r3,7f@ha
537 addi r3,r3,7f@l
538 lwz r4,GPR0(r1)
539 lwz r5,GPR3(r1)
540 lwz r6,GPR4(r1)
541 lwz r7,GPR5(r1)
542 lwz r8,GPR6(r1)
543 lwz r9,GPR7(r1)
544 bl printk
545 lis r3,77f@ha
546 addi r3,r3,77f@l
547 lwz r4,GPR8(r1)
548 mr r5,r2
549 bl printk
550 lwz r0,GPR0(r1)
551 lwz r3,GPR3(r1)
552 lwz r4,GPR4(r1)
553 lwz r5,GPR5(r1)
554 lwz r6,GPR6(r1)
555 lwz r7,GPR7(r1)
556 lwz r8,GPR8(r1)
557 mtlr r31
558 lwz r31,GPR31(r1)
559 blr
561 do_show_syscall_exit:
562 #ifdef SHOW_SYSCALLS_TASK
563 lis r11,show_syscalls_task@ha
564 lwz r11,show_syscalls_task@l(r11)
565 cmp 0,r2,r11
566 bnelr
567 #endif
568 stw r31,GPR31(r1)
569 mflr r31
570 stw r3,RESULT(r1) /* Save result */
571 mr r4,r3
572 lis r3,79f@ha
573 addi r3,r3,79f@l
574 bl printk
575 lwz r3,RESULT(r1)
576 mtlr r31
577 lwz r31,GPR31(r1)
578 blr
580 7: .string "syscall %d(%x, %x, %x, %x, %x, "
581 77: .string "%x), current=%p\n"
582 79: .string " -> %x\n"
583 .align 2,0
585 #ifdef SHOW_SYSCALLS_TASK
586 .data
587 .globl show_syscalls_task
588 show_syscalls_task:
589 .long -1
590 .text
591 #endif
592 #endif /* SHOW_SYSCALLS */
594 /*
595 * The fork/clone functions need to copy the full register set into
596 * the child process. Therefore we need to save all the nonvolatile
597 * registers (r13 - r31) before calling the C code.
598 */
599 .globl ppc_fork
600 ppc_fork:
601 SAVE_NVGPRS(r1)
602 lwz r0,_TRAP(r1)
603 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
604 stw r0,_TRAP(r1) /* register set saved */
605 b sys_fork
607 .globl ppc_vfork
608 ppc_vfork:
609 SAVE_NVGPRS(r1)
610 lwz r0,_TRAP(r1)
611 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
612 stw r0,_TRAP(r1) /* register set saved */
613 b sys_vfork
615 .globl ppc_clone
616 ppc_clone:
617 SAVE_NVGPRS(r1)
618 lwz r0,_TRAP(r1)
619 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
620 stw r0,_TRAP(r1) /* register set saved */
621 b sys_clone
623 .globl ppc_swapcontext
624 ppc_swapcontext:
625 SAVE_NVGPRS(r1)
626 lwz r0,_TRAP(r1)
627 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
628 stw r0,_TRAP(r1) /* register set saved */
629 b sys_swapcontext
631 /*
632 * Top-level page fault handling.
633 * This is in assembler because if do_page_fault tells us that
634 * it is a bad kernel page fault, we want to save the non-volatile
635 * registers before calling bad_page_fault.
636 */
637 .globl handle_page_fault
638 handle_page_fault:
639 stw r4,_DAR(r1)
640 addi r3,r1,STACK_FRAME_OVERHEAD
641 bl do_page_fault
642 cmpwi r3,0
643 beq+ ret_from_except
644 SAVE_NVGPRS(r1)
645 lwz r0,_TRAP(r1)
646 clrrwi r0,r0,1
647 stw r0,_TRAP(r1)
648 mr r5,r3
649 addi r3,r1,STACK_FRAME_OVERHEAD
650 lwz r4,_DAR(r1)
651 bl bad_page_fault
652 b ret_from_except_full
654 /*
655 * This routine switches between two different tasks. The process
656 * state of one is saved on its kernel stack. Then the state
657 * of the other is restored from its kernel stack. The memory
658 * management hardware is updated to the second process's state.
659 * Finally, we can return to the second process.
660 * On entry, r3 points to the THREAD for the current task, r4
661 * points to the THREAD for the new task.
662 *
663 * This routine is always called with interrupts disabled.
664 *
665 * Note: there are two ways to get to the "going out" portion
666 * of this code; either by coming in via the entry (_switch)
667 * or via "fork" which must set up an environment equivalent
668 * to the "_switch" path. If you change this , you'll have to
669 * change the fork code also.
670 *
671 * The code which creates the new task context is in 'copy_thread'
672 * in arch/ppc/kernel/process.c
673 */
674 _GLOBAL(_switch)
675 stwu r1,-INT_FRAME_SIZE(r1)
676 mflr r0
677 stw r0,INT_FRAME_SIZE+4(r1)
678 /* r3-r12 are caller saved -- Cort */
679 SAVE_NVGPRS(r1)
680 stw r0,_NIP(r1) /* Return to switch caller */
681 mfmsr r11
682 li r0,MSR_FP /* Disable floating-point */
683 #ifdef CONFIG_ALTIVEC
684 BEGIN_FTR_SECTION
685 oris r0,r0,MSR_VEC@h /* Disable altivec */
686 mfspr r12,SPRN_VRSAVE /* save vrsave register value */
687 stw r12,THREAD+THREAD_VRSAVE(r2)
688 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
689 #endif /* CONFIG_ALTIVEC */
690 #ifdef CONFIG_SPE
691 BEGIN_FTR_SECTION
692 oris r0,r0,MSR_SPE@h /* Disable SPE */
693 mfspr r12,SPRN_SPEFSCR /* save spefscr register value */
694 stw r12,THREAD+THREAD_SPEFSCR(r2)
695 END_FTR_SECTION_IFSET(CPU_FTR_SPE)
696 #endif /* CONFIG_SPE */
697 and. r0,r0,r11 /* FP or altivec or SPE enabled? */
698 beq+ 1f
699 andc r11,r11,r0
700 MTMSRD(r11)
701 isync
702 1: stw r11,_MSR(r1)
703 mfcr r10
704 stw r10,_CCR(r1)
705 stw r1,KSP(r3) /* Set old stack pointer */
707 #ifdef CONFIG_SMP
708 /* We need a sync somewhere here to make sure that if the
709 * previous task gets rescheduled on another CPU, it sees all
710 * stores it has performed on this one.
711 */
712 sync
713 #endif /* CONFIG_SMP */
715 tophys(r0,r4)
716 CLR_TOP32(r0)
717 mtspr SPRN_SPRG_THREAD,r0 /* Update current THREAD phys addr */
718 lwz r1,KSP(r4) /* Load new stack pointer */
720 /* save the old current 'last' for return value */
721 mr r3,r2
722 addi r2,r4,-THREAD /* Update current */
724 #ifdef CONFIG_ALTIVEC
725 BEGIN_FTR_SECTION
726 lwz r0,THREAD+THREAD_VRSAVE(r2)
727 mtspr SPRN_VRSAVE,r0 /* if G4, restore VRSAVE reg */
728 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
729 #endif /* CONFIG_ALTIVEC */
730 #ifdef CONFIG_SPE
731 BEGIN_FTR_SECTION
732 lwz r0,THREAD+THREAD_SPEFSCR(r2)
733 mtspr SPRN_SPEFSCR,r0 /* restore SPEFSCR reg */
734 END_FTR_SECTION_IFSET(CPU_FTR_SPE)
735 #endif /* CONFIG_SPE */
737 lwz r0,_CCR(r1)
738 mtcrf 0xFF,r0
739 /* r3-r12 are destroyed -- Cort */
740 REST_NVGPRS(r1)
742 lwz r4,_NIP(r1) /* Return to _switch caller in new task */
743 mtlr r4
744 addi r1,r1,INT_FRAME_SIZE
745 blr
747 .globl fast_exception_return
748 fast_exception_return:
749 #if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
750 andi. r10,r9,MSR_RI /* check for recoverable interrupt */
751 beq 1f /* if not, we've got problems */
752 #endif
754 2: REST_4GPRS(3, r11)
755 lwz r10,_CCR(r11)
756 REST_GPR(1, r11)
757 mtcr r10
758 lwz r10,_LINK(r11)
759 mtlr r10
760 REST_GPR(10, r11)
761 mtspr SPRN_SRR1,r9
762 mtspr SPRN_SRR0,r12
763 REST_GPR(9, r11)
764 REST_GPR(12, r11)
765 lwz r11,GPR11(r11)
766 SYNC
767 RFI
769 #if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
770 /* check if the exception happened in a restartable section */
771 1: lis r3,exc_exit_restart_end@ha
772 addi r3,r3,exc_exit_restart_end@l
773 cmplw r12,r3
774 bge 3f
775 lis r4,exc_exit_restart@ha
776 addi r4,r4,exc_exit_restart@l
777 cmplw r12,r4
778 blt 3f
779 lis r3,fee_restarts@ha
780 tophys(r3,r3)
781 lwz r5,fee_restarts@l(r3)
782 addi r5,r5,1
783 stw r5,fee_restarts@l(r3)
784 mr r12,r4 /* restart at exc_exit_restart */
785 b 2b
787 .section .bss
788 .align 2
789 fee_restarts:
790 .space 4
791 .previous
793 /* aargh, a nonrecoverable interrupt, panic */
794 /* aargh, we don't know which trap this is */
795 /* but the 601 doesn't implement the RI bit, so assume it's OK */
796 3:
797 BEGIN_FTR_SECTION
798 b 2b
799 END_FTR_SECTION_IFSET(CPU_FTR_601)
800 li r10,-1
801 stw r10,_TRAP(r11)
802 addi r3,r1,STACK_FRAME_OVERHEAD
803 lis r10,MSR_KERNEL@h
804 ori r10,r10,MSR_KERNEL@l
805 bl transfer_to_handler_full
806 .long nonrecoverable_exception
807 .long ret_from_except
808 #endif
810 .globl ret_from_except_full
811 ret_from_except_full:
812 REST_NVGPRS(r1)
813 /* fall through */
815 .globl ret_from_except
816 ret_from_except:
817 /* Hard-disable interrupts so that current_thread_info()->flags
818 * can't change between when we test it and when we return
819 * from the interrupt. */
820 /* Note: We don't bother telling lockdep about it */
821 LOAD_MSR_KERNEL(r10,MSR_KERNEL)
822 SYNC /* Some chip revs have problems here... */
823 MTMSRD(r10) /* disable interrupts */
825 lwz r3,_MSR(r1) /* Returning to user mode? */
826 andi. r0,r3,MSR_PR
827 beq resume_kernel
829 user_exc_return: /* r10 contains MSR_KERNEL here */
830 /* Check current_thread_info()->flags */
831 CURRENT_THREAD_INFO(r9, r1)
832 lwz r9,TI_FLAGS(r9)
833 andi. r0,r9,_TIF_USER_WORK_MASK
834 bne do_work
836 restore_user:
837 #if defined(CONFIG_4xx) || defined(CONFIG_BOOKE)
838 /* Check whether this process has its own DBCR0 value. The internal
839 debug mode bit tells us that dbcr0 should be loaded. */
840 lwz r0,THREAD+THREAD_DBCR0(r2)
841 andis. r10,r0,DBCR0_IDM@h
842 bnel- load_dbcr0
843 #endif
845 b restore
847 /* N.B. the only way to get here is from the beq following ret_from_except. */
848 resume_kernel:
849 /* check current_thread_info, _TIF_EMULATE_STACK_STORE */
850 CURRENT_THREAD_INFO(r9, r1)
851 lwz r8,TI_FLAGS(r9)
852 andis. r0,r8,_TIF_EMULATE_STACK_STORE@h
853 beq+ 1f
855 addi r8,r1,INT_FRAME_SIZE /* Get the kprobed function entry */
857 lwz r3,GPR1(r1)
858 subi r3,r3,INT_FRAME_SIZE /* dst: Allocate a trampoline exception frame */
859 mr r4,r1 /* src: current exception frame */
860 mr r1,r3 /* Reroute the trampoline frame to r1 */
862 /* Copy from the original to the trampoline. */
863 li r5,INT_FRAME_SIZE/4 /* size: INT_FRAME_SIZE */
864 li r6,0 /* start offset: 0 */
865 mtctr r5
866 2: lwzx r0,r6,r4
867 stwx r0,r6,r3
868 addi r6,r6,4
869 bdnz 2b
871 /* Do real store operation to complete stwu */
872 lwz r5,GPR1(r1)
873 stw r8,0(r5)
875 /* Clear _TIF_EMULATE_STACK_STORE flag */
876 lis r11,_TIF_EMULATE_STACK_STORE@h
877 addi r5,r9,TI_FLAGS
878 0: lwarx r8,0,r5
879 andc r8,r8,r11
880 #ifdef CONFIG_IBM405_ERR77
881 dcbt 0,r5
882 #endif
883 stwcx. r8,0,r5
884 bne- 0b
885 1:
887 #ifdef CONFIG_PREEMPT
888 /* check current_thread_info->preempt_count */
889 lwz r0,TI_PREEMPT(r9)
890 cmpwi 0,r0,0 /* if non-zero, just restore regs and return */
891 bne restore
892 andi. r8,r8,_TIF_NEED_RESCHED
893 beq+ restore
894 lwz r3,_MSR(r1)
895 andi. r0,r3,MSR_EE /* interrupts off? */
896 beq restore /* don't schedule if so */
897 #ifdef CONFIG_TRACE_IRQFLAGS
898 /* Lockdep thinks irqs are enabled, we need to call
899 * preempt_schedule_irq with IRQs off, so we inform lockdep
900 * now that we -did- turn them off already
901 */
902 bl trace_hardirqs_off
903 #endif
904 1: bl preempt_schedule_irq
905 CURRENT_THREAD_INFO(r9, r1)
906 lwz r3,TI_FLAGS(r9)
907 andi. r0,r3,_TIF_NEED_RESCHED
908 bne- 1b
909 #ifdef CONFIG_TRACE_IRQFLAGS
910 /* And now, to properly rebalance the above, we tell lockdep they
911 * are being turned back on, which will happen when we return
912 */
913 bl trace_hardirqs_on
914 #endif
915 #endif /* CONFIG_PREEMPT */
917 /* interrupts are hard-disabled at this point */
918 restore:
919 #ifdef CONFIG_44x
920 BEGIN_MMU_FTR_SECTION
921 b 1f
922 END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_47x)
923 lis r4,icache_44x_need_flush@ha
924 lwz r5,icache_44x_need_flush@l(r4)
925 cmplwi cr0,r5,0
926 beq+ 1f
927 li r6,0
928 iccci r0,r0
929 stw r6,icache_44x_need_flush@l(r4)
930 1:
931 #endif /* CONFIG_44x */
933 lwz r9,_MSR(r1)
934 #ifdef CONFIG_TRACE_IRQFLAGS
935 /* Lockdep doesn't know about the fact that IRQs are temporarily turned
936 * off in this assembly code while peeking at TI_FLAGS() and such. However
937 * we need to inform it if the exception turned interrupts off, and we
938 * are about to trun them back on.
939 *
940 * The problem here sadly is that we don't know whether the exceptions was
941 * one that turned interrupts off or not. So we always tell lockdep about
942 * turning them on here when we go back to wherever we came from with EE
943 * on, even if that may meen some redudant calls being tracked. Maybe later
944 * we could encode what the exception did somewhere or test the exception
945 * type in the pt_regs but that sounds overkill
946 */
947 andi. r10,r9,MSR_EE
948 beq 1f
949 /*
950 * Since the ftrace irqsoff latency trace checks CALLER_ADDR1,
951 * which is the stack frame here, we need to force a stack frame
952 * in case we came from user space.
953 */
954 stwu r1,-32(r1)
955 mflr r0
956 stw r0,4(r1)
957 stwu r1,-32(r1)
958 bl trace_hardirqs_on
959 lwz r1,0(r1)
960 lwz r1,0(r1)
961 lwz r9,_MSR(r1)
962 1:
963 #endif /* CONFIG_TRACE_IRQFLAGS */
965 lwz r0,GPR0(r1)
966 lwz r2,GPR2(r1)
967 REST_4GPRS(3, r1)
968 REST_2GPRS(7, r1)
970 lwz r10,_XER(r1)
971 lwz r11,_CTR(r1)
972 mtspr SPRN_XER,r10
973 mtctr r11
975 PPC405_ERR77(0,r1)
976 BEGIN_FTR_SECTION
977 lwarx r11,0,r1
978 END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX)
979 stwcx. r0,0,r1 /* to clear the reservation */
981 #if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
982 andi. r10,r9,MSR_RI /* check if this exception occurred */
983 beql nonrecoverable /* at a bad place (MSR:RI = 0) */
985 lwz r10,_CCR(r1)
986 lwz r11,_LINK(r1)
987 mtcrf 0xFF,r10
988 mtlr r11
990 /*
991 * Once we put values in SRR0 and SRR1, we are in a state
992 * where exceptions are not recoverable, since taking an
993 * exception will trash SRR0 and SRR1. Therefore we clear the
994 * MSR:RI bit to indicate this. If we do take an exception,
995 * we can't return to the point of the exception but we
996 * can restart the exception exit path at the label
997 * exc_exit_restart below. -- paulus
998 */
999 LOAD_MSR_KERNEL(r10,MSR_KERNEL & ~MSR_RI)
1000 SYNC
1001 MTMSRD(r10) /* clear the RI bit */
1002 .globl exc_exit_restart
1003 exc_exit_restart:
1004 lwz r12,_NIP(r1)
1005 FIX_SRR1(r9,r10)
1006 mtspr SPRN_SRR0,r12
1007 mtspr SPRN_SRR1,r9
1008 REST_4GPRS(9, r1)
1009 lwz r1,GPR1(r1)
1010 .globl exc_exit_restart_end
1011 exc_exit_restart_end:
1012 SYNC
1013 RFI
1015 #else /* !(CONFIG_4xx || CONFIG_BOOKE) */
1016 /*
1017 * This is a bit different on 4xx/Book-E because it doesn't have
1018 * the RI bit in the MSR.
1019 * The TLB miss handler checks if we have interrupted
1020 * the exception exit path and restarts it if so
1021 * (well maybe one day it will... :).
1022 */
1023 lwz r11,_LINK(r1)
1024 mtlr r11
1025 lwz r10,_CCR(r1)
1026 mtcrf 0xff,r10
1027 REST_2GPRS(9, r1)
1028 .globl exc_exit_restart
1029 exc_exit_restart:
1030 lwz r11,_NIP(r1)
1031 lwz r12,_MSR(r1)
1032 exc_exit_start:
1033 mtspr SPRN_SRR0,r11
1034 mtspr SPRN_SRR1,r12
1035 REST_2GPRS(11, r1)
1036 lwz r1,GPR1(r1)
1037 .globl exc_exit_restart_end
1038 exc_exit_restart_end:
1039 PPC405_ERR77_SYNC
1040 rfi
1041 b . /* prevent prefetch past rfi */
1043 /*
1044 * Returning from a critical interrupt in user mode doesn't need
1045 * to be any different from a normal exception. For a critical
1046 * interrupt in the kernel, we just return (without checking for
1047 * preemption) since the interrupt may have happened at some crucial
1048 * place (e.g. inside the TLB miss handler), and because we will be
1049 * running with r1 pointing into critical_stack, not the current
1050 * process's kernel stack (and therefore current_thread_info() will
1051 * give the wrong answer).
1052 * We have to restore various SPRs that may have been in use at the
1053 * time of the critical interrupt.
1054 *
1055 */
1056 #ifdef CONFIG_40x
1057 #define PPC_40x_TURN_OFF_MSR_DR \
1058 /* avoid any possible TLB misses here by turning off MSR.DR, we \
1059 * assume the instructions here are mapped by a pinned TLB entry */ \
1060 li r10,MSR_IR; \
1061 mtmsr r10; \
1062 isync; \
1063 tophys(r1, r1);
1064 #else
1065 #define PPC_40x_TURN_OFF_MSR_DR
1066 #endif
1068 #define RET_FROM_EXC_LEVEL(exc_lvl_srr0, exc_lvl_srr1, exc_lvl_rfi) \
1069 REST_NVGPRS(r1); \
1070 lwz r3,_MSR(r1); \
1071 andi. r3,r3,MSR_PR; \
1072 LOAD_MSR_KERNEL(r10,MSR_KERNEL); \
1073 bne user_exc_return; \
1074 lwz r0,GPR0(r1); \
1075 lwz r2,GPR2(r1); \
1076 REST_4GPRS(3, r1); \
1077 REST_2GPRS(7, r1); \
1078 lwz r10,_XER(r1); \
1079 lwz r11,_CTR(r1); \
1080 mtspr SPRN_XER,r10; \
1081 mtctr r11; \
1082 PPC405_ERR77(0,r1); \
1083 stwcx. r0,0,r1; /* to clear the reservation */ \
1084 lwz r11,_LINK(r1); \
1085 mtlr r11; \
1086 lwz r10,_CCR(r1); \
1087 mtcrf 0xff,r10; \
1088 PPC_40x_TURN_OFF_MSR_DR; \
1089 lwz r9,_DEAR(r1); \
1090 lwz r10,_ESR(r1); \
1091 mtspr SPRN_DEAR,r9; \
1092 mtspr SPRN_ESR,r10; \
1093 lwz r11,_NIP(r1); \
1094 lwz r12,_MSR(r1); \
1095 mtspr exc_lvl_srr0,r11; \
1096 mtspr exc_lvl_srr1,r12; \
1097 lwz r9,GPR9(r1); \
1098 lwz r12,GPR12(r1); \
1099 lwz r10,GPR10(r1); \
1100 lwz r11,GPR11(r1); \
1101 lwz r1,GPR1(r1); \
1102 PPC405_ERR77_SYNC; \
1103 exc_lvl_rfi; \
1104 b .; /* prevent prefetch past exc_lvl_rfi */
1106 #define RESTORE_xSRR(exc_lvl_srr0, exc_lvl_srr1) \
1107 lwz r9,_##exc_lvl_srr0(r1); \
1108 lwz r10,_##exc_lvl_srr1(r1); \
1109 mtspr SPRN_##exc_lvl_srr0,r9; \
1110 mtspr SPRN_##exc_lvl_srr1,r10;
1112 #if defined(CONFIG_PPC_BOOK3E_MMU)
1113 #ifdef CONFIG_PHYS_64BIT
1114 #define RESTORE_MAS7 \
1115 lwz r11,MAS7(r1); \
1116 mtspr SPRN_MAS7,r11;
1117 #else
1118 #define RESTORE_MAS7
1119 #endif /* CONFIG_PHYS_64BIT */
1120 #define RESTORE_MMU_REGS \
1121 lwz r9,MAS0(r1); \
1122 lwz r10,MAS1(r1); \
1123 lwz r11,MAS2(r1); \
1124 mtspr SPRN_MAS0,r9; \
1125 lwz r9,MAS3(r1); \
1126 mtspr SPRN_MAS1,r10; \
1127 lwz r10,MAS6(r1); \
1128 mtspr SPRN_MAS2,r11; \
1129 mtspr SPRN_MAS3,r9; \
1130 mtspr SPRN_MAS6,r10; \
1131 RESTORE_MAS7;
1132 #elif defined(CONFIG_44x)
1133 #define RESTORE_MMU_REGS \
1134 lwz r9,MMUCR(r1); \
1135 mtspr SPRN_MMUCR,r9;
1136 #else
1137 #define RESTORE_MMU_REGS
1138 #endif
1140 #ifdef CONFIG_40x
1141 .globl ret_from_crit_exc
1142 ret_from_crit_exc:
1143 mfspr r9,SPRN_SPRG_THREAD
1144 lis r10,saved_ksp_limit@ha;
1145 lwz r10,saved_ksp_limit@l(r10);
1146 tovirt(r9,r9);
1147 stw r10,KSP_LIMIT(r9)
1148 lis r9,crit_srr0@ha;
1149 lwz r9,crit_srr0@l(r9);
1150 lis r10,crit_srr1@ha;
1151 lwz r10,crit_srr1@l(r10);
1152 mtspr SPRN_SRR0,r9;
1153 mtspr SPRN_SRR1,r10;
1154 RET_FROM_EXC_LEVEL(SPRN_CSRR0, SPRN_CSRR1, PPC_RFCI)
1155 #endif /* CONFIG_40x */
1157 #ifdef CONFIG_BOOKE
1158 .globl ret_from_crit_exc
1159 ret_from_crit_exc:
1160 mfspr r9,SPRN_SPRG_THREAD
1161 lwz r10,SAVED_KSP_LIMIT(r1)
1162 stw r10,KSP_LIMIT(r9)
1163 RESTORE_xSRR(SRR0,SRR1);
1164 RESTORE_MMU_REGS;
1165 RET_FROM_EXC_LEVEL(SPRN_CSRR0, SPRN_CSRR1, PPC_RFCI)
1167 .globl ret_from_debug_exc
1168 ret_from_debug_exc:
1169 mfspr r9,SPRN_SPRG_THREAD
1170 lwz r10,SAVED_KSP_LIMIT(r1)
1171 stw r10,KSP_LIMIT(r9)
1172 lwz r9,THREAD_INFO-THREAD(r9)
1173 CURRENT_THREAD_INFO(r10, r1)
1174 lwz r10,TI_PREEMPT(r10)
1175 stw r10,TI_PREEMPT(r9)
1176 RESTORE_xSRR(SRR0,SRR1);
1177 RESTORE_xSRR(CSRR0,CSRR1);
1178 RESTORE_MMU_REGS;
1179 RET_FROM_EXC_LEVEL(SPRN_DSRR0, SPRN_DSRR1, PPC_RFDI)
1181 .globl ret_from_mcheck_exc
1182 ret_from_mcheck_exc:
1183 mfspr r9,SPRN_SPRG_THREAD
1184 lwz r10,SAVED_KSP_LIMIT(r1)
1185 stw r10,KSP_LIMIT(r9)
1186 RESTORE_xSRR(SRR0,SRR1);
1187 RESTORE_xSRR(CSRR0,CSRR1);
1188 RESTORE_xSRR(DSRR0,DSRR1);
1189 RESTORE_MMU_REGS;
1190 RET_FROM_EXC_LEVEL(SPRN_MCSRR0, SPRN_MCSRR1, PPC_RFMCI)
1191 #endif /* CONFIG_BOOKE */
1193 /*
1194 * Load the DBCR0 value for a task that is being ptraced,
1195 * having first saved away the global DBCR0. Note that r0
1196 * has the dbcr0 value to set upon entry to this.
1197 */
1198 load_dbcr0:
1199 mfmsr r10 /* first disable debug exceptions */
1200 rlwinm r10,r10,0,~MSR_DE
1201 mtmsr r10
1202 isync
1203 mfspr r10,SPRN_DBCR0
1204 lis r11,global_dbcr0@ha
1205 addi r11,r11,global_dbcr0@l
1206 #ifdef CONFIG_SMP
1207 CURRENT_THREAD_INFO(r9, r1)
1208 lwz r9,TI_CPU(r9)
1209 slwi r9,r9,3
1210 add r11,r11,r9
1211 #endif
1212 stw r10,0(r11)
1213 mtspr SPRN_DBCR0,r0
1214 lwz r10,4(r11)
1215 addi r10,r10,1
1216 stw r10,4(r11)
1217 li r11,-1
1218 mtspr SPRN_DBSR,r11 /* clear all pending debug events */
1219 blr
1221 .section .bss
1222 .align 4
1223 global_dbcr0:
1224 .space 8*NR_CPUS
1225 .previous
1226 #endif /* !(CONFIG_4xx || CONFIG_BOOKE) */
1228 do_work: /* r10 contains MSR_KERNEL here */
1229 andi. r0,r9,_TIF_NEED_RESCHED
1230 beq do_user_signal
1232 do_resched: /* r10 contains MSR_KERNEL here */
1233 /* Note: We don't need to inform lockdep that we are enabling
1234 * interrupts here. As far as it knows, they are already enabled
1235 */
1236 ori r10,r10,MSR_EE
1237 SYNC
1238 MTMSRD(r10) /* hard-enable interrupts */
1239 bl schedule
1240 recheck:
1241 /* Note: And we don't tell it we are disabling them again
1242 * neither. Those disable/enable cycles used to peek at
1243 * TI_FLAGS aren't advertised.
1244 */
1245 LOAD_MSR_KERNEL(r10,MSR_KERNEL)
1246 SYNC
1247 MTMSRD(r10) /* disable interrupts */
1248 CURRENT_THREAD_INFO(r9, r1)
1249 lwz r9,TI_FLAGS(r9)
1250 andi. r0,r9,_TIF_NEED_RESCHED
1251 bne- do_resched
1252 andi. r0,r9,_TIF_USER_WORK_MASK
1253 beq restore_user
1254 do_user_signal: /* r10 contains MSR_KERNEL here */
1255 ori r10,r10,MSR_EE
1256 SYNC
1257 MTMSRD(r10) /* hard-enable interrupts */
1258 /* save r13-r31 in the exception frame, if not already done */
1259 lwz r3,_TRAP(r1)
1260 andi. r0,r3,1
1261 beq 2f
1262 SAVE_NVGPRS(r1)
1263 rlwinm r3,r3,0,0,30
1264 stw r3,_TRAP(r1)
1265 2: addi r3,r1,STACK_FRAME_OVERHEAD
1266 mr r4,r9
1267 bl do_notify_resume
1268 REST_NVGPRS(r1)
1269 b recheck
1271 /*
1272 * We come here when we are at the end of handling an exception
1273 * that occurred at a place where taking an exception will lose
1274 * state information, such as the contents of SRR0 and SRR1.
1275 */
1276 nonrecoverable:
1277 lis r10,exc_exit_restart_end@ha
1278 addi r10,r10,exc_exit_restart_end@l
1279 cmplw r12,r10
1280 bge 3f
1281 lis r11,exc_exit_restart@ha
1282 addi r11,r11,exc_exit_restart@l
1283 cmplw r12,r11
1284 blt 3f
1285 lis r10,ee_restarts@ha
1286 lwz r12,ee_restarts@l(r10)
1287 addi r12,r12,1
1288 stw r12,ee_restarts@l(r10)
1289 mr r12,r11 /* restart at exc_exit_restart */
1290 blr
1291 3: /* OK, we can't recover, kill this process */
1292 /* but the 601 doesn't implement the RI bit, so assume it's OK */
1293 BEGIN_FTR_SECTION
1294 blr
1295 END_FTR_SECTION_IFSET(CPU_FTR_601)
1296 lwz r3,_TRAP(r1)
1297 andi. r0,r3,1
1298 beq 4f
1299 SAVE_NVGPRS(r1)
1300 rlwinm r3,r3,0,0,30
1301 stw r3,_TRAP(r1)
1302 4: addi r3,r1,STACK_FRAME_OVERHEAD
1303 bl nonrecoverable_exception
1304 /* shouldn't return */
1305 b 4b
1307 .section .bss
1308 .align 2
1309 ee_restarts:
1310 .space 4
1311 .previous
1313 /*
1314 * PROM code for specific machines follows. Put it
1315 * here so it's easy to add arch-specific sections later.
1316 * -- Cort
1317 */
1318 #ifdef CONFIG_PPC_RTAS
1319 /*
1320 * On CHRP, the Run-Time Abstraction Services (RTAS) have to be
1321 * called with the MMU off.
1322 */
1323 _GLOBAL(enter_rtas)
1324 stwu r1,-INT_FRAME_SIZE(r1)
1325 mflr r0
1326 stw r0,INT_FRAME_SIZE+4(r1)
1327 LOAD_REG_ADDR(r4, rtas)
1328 lis r6,1f@ha /* physical return address for rtas */
1329 addi r6,r6,1f@l
1330 tophys(r6,r6)
1331 tophys(r7,r1)
1332 lwz r8,RTASENTRY(r4)
1333 lwz r4,RTASBASE(r4)
1334 mfmsr r9
1335 stw r9,8(r1)
1336 LOAD_MSR_KERNEL(r0,MSR_KERNEL)
1337 SYNC /* disable interrupts so SRR0/1 */
1338 MTMSRD(r0) /* don't get trashed */
1339 li r9,MSR_KERNEL & ~(MSR_IR|MSR_DR)
1340 mtlr r6
1341 mtspr SPRN_SPRG_RTAS,r7
1342 mtspr SPRN_SRR0,r8
1343 mtspr SPRN_SRR1,r9
1344 RFI
1345 1: tophys(r9,r1)
1346 lwz r8,INT_FRAME_SIZE+4(r9) /* get return address */
1347 lwz r9,8(r9) /* original msr value */
1348 FIX_SRR1(r9,r0)
1349 addi r1,r1,INT_FRAME_SIZE
1350 li r0,0
1351 mtspr SPRN_SPRG_RTAS,r0
1352 mtspr SPRN_SRR0,r8
1353 mtspr SPRN_SRR1,r9
1354 RFI /* return to caller */
1356 .globl machine_check_in_rtas
1357 machine_check_in_rtas:
1358 twi 31,0,0
1359 /* XXX load up BATs and panic */
1361 #endif /* CONFIG_PPC_RTAS */
1363 #ifdef CONFIG_FUNCTION_TRACER
1364 #ifdef CONFIG_DYNAMIC_FTRACE
1365 _GLOBAL(mcount)
1366 _GLOBAL(_mcount)
1367 /*
1368 * It is required that _mcount on PPC32 must preserve the
1369 * link register. But we have r0 to play with. We use r0
1370 * to push the return address back to the caller of mcount
1371 * into the ctr register, restore the link register and
1372 * then jump back using the ctr register.
1373 */
1374 mflr r0
1375 mtctr r0
1376 lwz r0, 4(r1)
1377 mtlr r0
1378 bctr
1380 _GLOBAL(ftrace_caller)
1381 MCOUNT_SAVE_FRAME
1382 /* r3 ends up with link register */
1383 subi r3, r3, MCOUNT_INSN_SIZE
1384 .globl ftrace_call
1385 ftrace_call:
1386 bl ftrace_stub
1387 nop
1388 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1389 .globl ftrace_graph_call
1390 ftrace_graph_call:
1391 b ftrace_graph_stub
1392 _GLOBAL(ftrace_graph_stub)
1393 #endif
1394 MCOUNT_RESTORE_FRAME
1395 /* old link register ends up in ctr reg */
1396 bctr
1397 #else
1398 _GLOBAL(mcount)
1399 _GLOBAL(_mcount)
1401 MCOUNT_SAVE_FRAME
1403 subi r3, r3, MCOUNT_INSN_SIZE
1404 LOAD_REG_ADDR(r5, ftrace_trace_function)
1405 lwz r5,0(r5)
1407 mtctr r5
1408 bctrl
1409 nop
1411 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1412 b ftrace_graph_caller
1413 #endif
1414 MCOUNT_RESTORE_FRAME
1415 bctr
1416 #endif
1418 _GLOBAL(ftrace_stub)
1419 blr
1421 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1422 _GLOBAL(ftrace_graph_caller)
1423 /* load r4 with local address */
1424 lwz r4, 44(r1)
1425 subi r4, r4, MCOUNT_INSN_SIZE
1427 /* get the parent address */
1428 addi r3, r1, 52
1430 bl prepare_ftrace_return
1431 nop
1433 MCOUNT_RESTORE_FRAME
1434 /* old link register ends up in ctr reg */
1435 bctr
1437 _GLOBAL(return_to_handler)
1438 /* need to save return values */
1439 stwu r1, -32(r1)
1440 stw r3, 20(r1)
1441 stw r4, 16(r1)
1442 stw r31, 12(r1)
1443 mr r31, r1
1445 bl ftrace_return_to_handler
1446 nop
1448 /* return value has real return address */
1449 mtlr r3
1451 lwz r3, 20(r1)
1452 lwz r4, 16(r1)
1453 lwz r31,12(r1)
1454 lwz r1, 0(r1)
1456 /* Jump back to real return address */
1457 blr
1458 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1460 #endif /* CONFIG_MCOUNT */