| blob | e514de57a125333a4ce174cb399070b6cc62e3b4 |
1 /*
2 * PowerPC version
3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 * Rewritten by Cort Dougan (cort@fsmlabs.com) for PReP
5 * Copyright (C) 1996 Cort Dougan <cort@fsmlabs.com>
6 * Adapted for Power Macintosh by Paul Mackerras.
7 * Low-level exception handlers and MMU support
8 * rewritten by Paul Mackerras.
9 * Copyright (C) 1996 Paul Mackerras.
10 * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
11 *
12 * This file contains the system call entry code, context switch
13 * code, and exception/interrupt return code for PowerPC.
14 *
15 * This program is free software; you can redistribute it and/or
16 * modify it under the terms of the GNU General Public License
17 * as published by the Free Software Foundation; either version
18 * 2 of the License, or (at your option) any later version.
19 *
20 */
22 #include <linux/errno.h>
23 #include <linux/sys.h>
24 #include <linux/threads.h>
25 #include <asm/reg.h>
26 #include <asm/page.h>
27 #include <asm/mmu.h>
28 #include <asm/cputable.h>
29 #include <asm/thread_info.h>
30 #include <asm/ppc_asm.h>
31 #include <asm/asm-offsets.h>
32 #include <asm/unistd.h>
33 #include <asm/ftrace.h>
34 #include <asm/ptrace.h>
36 #undef SHOW_SYSCALLS
37 #undef SHOW_SYSCALLS_TASK
39 /*
40 * MSR_KERNEL is > 0x10000 on 4xx/Book-E since it include MSR_CE.
41 */
42 #if MSR_KERNEL >= 0x10000
43 #define LOAD_MSR_KERNEL(r, x) lis r,(x)@h; ori r,r,(x)@l
44 #else
45 #define LOAD_MSR_KERNEL(r, x) li r,(x)
46 #endif
48 #ifdef CONFIG_BOOKE
49 .globl mcheck_transfer_to_handler
50 mcheck_transfer_to_handler:
51 mfspr r0,SPRN_DSRR0
52 stw r0,_DSRR0(r11)
53 mfspr r0,SPRN_DSRR1
54 stw r0,_DSRR1(r11)
55 /* fall through */
57 .globl debug_transfer_to_handler
58 debug_transfer_to_handler:
59 mfspr r0,SPRN_CSRR0
60 stw r0,_CSRR0(r11)
61 mfspr r0,SPRN_CSRR1
62 stw r0,_CSRR1(r11)
63 /* fall through */
65 .globl crit_transfer_to_handler
66 crit_transfer_to_handler:
67 #ifdef CONFIG_PPC_BOOK3E_MMU
68 mfspr r0,SPRN_MAS0
69 stw r0,MAS0(r11)
70 mfspr r0,SPRN_MAS1
71 stw r0,MAS1(r11)
72 mfspr r0,SPRN_MAS2
73 stw r0,MAS2(r11)
74 mfspr r0,SPRN_MAS3
75 stw r0,MAS3(r11)
76 mfspr r0,SPRN_MAS6
77 stw r0,MAS6(r11)
78 #ifdef CONFIG_PHYS_64BIT
79 mfspr r0,SPRN_MAS7
80 stw r0,MAS7(r11)
81 #endif /* CONFIG_PHYS_64BIT */
82 #endif /* CONFIG_PPC_BOOK3E_MMU */
83 #ifdef CONFIG_44x
84 mfspr r0,SPRN_MMUCR
85 stw r0,MMUCR(r11)
86 #endif
87 mfspr r0,SPRN_SRR0
88 stw r0,_SRR0(r11)
89 mfspr r0,SPRN_SRR1
90 stw r0,_SRR1(r11)
92 /* set the stack limit to the current stack
93 * and set the limit to protect the thread_info
94 * struct
95 */
96 mfspr r8,SPRN_SPRG_THREAD
97 lwz r0,KSP_LIMIT(r8)
98 stw r0,SAVED_KSP_LIMIT(r11)
99 rlwimi r0,r1,0,0,(31-THREAD_SHIFT)
100 stw r0,KSP_LIMIT(r8)
101 /* fall through */
102 #endif
104 #ifdef CONFIG_40x
105 .globl crit_transfer_to_handler
106 crit_transfer_to_handler:
107 lwz r0,crit_r10@l(0)
108 stw r0,GPR10(r11)
109 lwz r0,crit_r11@l(0)
110 stw r0,GPR11(r11)
111 mfspr r0,SPRN_SRR0
112 stw r0,crit_srr0@l(0)
113 mfspr r0,SPRN_SRR1
114 stw r0,crit_srr1@l(0)
116 /* set the stack limit to the current stack
117 * and set the limit to protect the thread_info
118 * struct
119 */
120 mfspr r8,SPRN_SPRG_THREAD
121 lwz r0,KSP_LIMIT(r8)
122 stw r0,saved_ksp_limit@l(0)
123 rlwimi r0,r1,0,0,(31-THREAD_SHIFT)
124 stw r0,KSP_LIMIT(r8)
125 /* fall through */
126 #endif
128 /*
129 * This code finishes saving the registers to the exception frame
130 * and jumps to the appropriate handler for the exception, turning
131 * on address translation.
132 * Note that we rely on the caller having set cr0.eq iff the exception
133 * occurred in kernel mode (i.e. MSR:PR = 0).
134 */
135 .globl transfer_to_handler_full
136 transfer_to_handler_full:
137 SAVE_NVGPRS(r11)
138 /* fall through */
140 .globl transfer_to_handler
141 transfer_to_handler:
142 stw r2,GPR2(r11)
143 stw r12,_NIP(r11)
144 stw r9,_MSR(r11)
145 andi. r2,r9,MSR_PR
146 mfctr r12
147 mfspr r2,SPRN_XER
148 stw r12,_CTR(r11)
149 stw r2,_XER(r11)
150 mfspr r12,SPRN_SPRG_THREAD
151 addi r2,r12,-THREAD
152 tovirt(r2,r2) /* set r2 to current */
153 beq 2f /* if from user, fix up THREAD.regs */
154 addi r11,r1,STACK_FRAME_OVERHEAD
155 stw r11,PT_REGS(r12)
156 #if defined(CONFIG_40x) || defined(CONFIG_BOOKE)
157 /* Check to see if the dbcr0 register is set up to debug. Use the
158 internal debug mode bit to do this. */
159 lwz r12,THREAD_DBCR0(r12)
160 andis. r12,r12,DBCR0_IDM@h
161 beq+ 3f
162 /* From user and task is ptraced - load up global dbcr0 */
163 li r12,-1 /* clear all pending debug events */
164 mtspr SPRN_DBSR,r12
165 lis r11,global_dbcr0@ha
166 tophys(r11,r11)
167 addi r11,r11,global_dbcr0@l
168 #ifdef CONFIG_SMP
169 CURRENT_THREAD_INFO(r9, r1)
170 lwz r9,TI_CPU(r9)
171 slwi r9,r9,3
172 add r11,r11,r9
173 #endif
174 lwz r12,0(r11)
175 mtspr SPRN_DBCR0,r12
176 lwz r12,4(r11)
177 addi r12,r12,-1
178 stw r12,4(r11)
179 #endif
180 b 3f
182 2: /* if from kernel, check interrupted DOZE/NAP mode and
183 * check for stack overflow
184 */
185 lwz r9,KSP_LIMIT(r12)
186 cmplw r1,r9 /* if r1 <= ksp_limit */
187 ble- stack_ovf /* then the kernel stack overflowed */
188 5:
189 #if defined(CONFIG_6xx) || defined(CONFIG_E500)
190 CURRENT_THREAD_INFO(r9, r1)
191 tophys(r9,r9) /* check local flags */
192 lwz r12,TI_LOCAL_FLAGS(r9)
193 mtcrf 0x01,r12
194 bt- 31-TLF_NAPPING,4f
195 bt- 31-TLF_SLEEPING,7f
196 #endif /* CONFIG_6xx || CONFIG_E500 */
197 .globl transfer_to_handler_cont
198 transfer_to_handler_cont:
199 3:
200 mflr r9
201 lwz r11,0(r9) /* virtual address of handler */
202 lwz r9,4(r9) /* where to go when done */
203 #ifdef CONFIG_TRACE_IRQFLAGS
204 lis r12,reenable_mmu@h
205 ori r12,r12,reenable_mmu@l
206 mtspr SPRN_SRR0,r12
207 mtspr SPRN_SRR1,r10
208 SYNC
209 RFI
210 reenable_mmu: /* re-enable mmu so we can */
211 mfmsr r10
212 lwz r12,_MSR(r1)
213 xor r10,r10,r12
214 andi. r10,r10,MSR_EE /* Did EE change? */
215 beq 1f
217 /*
218 * The trace_hardirqs_off will use CALLER_ADDR0 and CALLER_ADDR1.
219 * If from user mode there is only one stack frame on the stack, and
220 * accessing CALLER_ADDR1 will cause oops. So we need create a dummy
221 * stack frame to make trace_hardirqs_off happy.
222 *
223 * This is handy because we also need to save a bunch of GPRs,
224 * r3 can be different from GPR3(r1) at this point, r9 and r11
225 * contains the old MSR and handler address respectively,
226 * r4 & r5 can contain page fault arguments that need to be passed
227 * along as well. r12, CCR, CTR, XER etc... are left clobbered as
228 * they aren't useful past this point (aren't syscall arguments),
229 * the rest is restored from the exception frame.
230 */
231 stwu r1,-32(r1)
232 stw r9,8(r1)
233 stw r11,12(r1)
234 stw r3,16(r1)
235 stw r4,20(r1)
236 stw r5,24(r1)
237 bl trace_hardirqs_off
238 lwz r5,24(r1)
239 lwz r4,20(r1)
240 lwz r3,16(r1)
241 lwz r11,12(r1)
242 lwz r9,8(r1)
243 addi r1,r1,32
244 lwz r0,GPR0(r1)
245 lwz r6,GPR6(r1)
246 lwz r7,GPR7(r1)
247 lwz r8,GPR8(r1)
248 1: mtctr r11
249 mtlr r9
250 bctr /* jump to handler */
251 #else /* CONFIG_TRACE_IRQFLAGS */
252 mtspr SPRN_SRR0,r11
253 mtspr SPRN_SRR1,r10
254 mtlr r9
255 SYNC
256 RFI /* jump to handler, enable MMU */
257 #endif /* CONFIG_TRACE_IRQFLAGS */
259 #if defined (CONFIG_6xx) || defined(CONFIG_E500)
260 4: rlwinm r12,r12,0,~_TLF_NAPPING
261 stw r12,TI_LOCAL_FLAGS(r9)
262 b power_save_ppc32_restore
264 7: rlwinm r12,r12,0,~_TLF_SLEEPING
265 stw r12,TI_LOCAL_FLAGS(r9)
266 lwz r9,_MSR(r11) /* if sleeping, clear MSR.EE */
267 rlwinm r9,r9,0,~MSR_EE
268 lwz r12,_LINK(r11) /* and return to address in LR */
269 b fast_exception_return
270 #endif
272 /*
273 * On kernel stack overflow, load up an initial stack pointer
274 * and call StackOverflow(regs), which should not return.
275 */
276 stack_ovf:
277 /* sometimes we use a statically-allocated stack, which is OK. */
278 lis r12,_end@h
279 ori r12,r12,_end@l
280 cmplw r1,r12
281 ble 5b /* r1 <= &_end is OK */
282 SAVE_NVGPRS(r11)
283 addi r3,r1,STACK_FRAME_OVERHEAD
284 lis r1,init_thread_union@ha
285 addi r1,r1,init_thread_union@l
286 addi r1,r1,THREAD_SIZE-STACK_FRAME_OVERHEAD
287 lis r9,StackOverflow@ha
288 addi r9,r9,StackOverflow@l
289 LOAD_MSR_KERNEL(r10,MSR_KERNEL)
290 FIX_SRR1(r10,r12)
291 mtspr SPRN_SRR0,r9
292 mtspr SPRN_SRR1,r10
293 SYNC
294 RFI
296 /*
297 * Handle a system call.
298 */
299 .stabs "arch/powerpc/kernel/",N_SO,0,0,0f
300 .stabs "entry_32.S",N_SO,0,0,0f
301 0:
303 _GLOBAL(DoSyscall)
304 stw r3,ORIG_GPR3(r1)
305 li r12,0
306 stw r12,RESULT(r1)
307 lwz r11,_CCR(r1) /* Clear SO bit in CR */
308 rlwinm r11,r11,0,4,2
309 stw r11,_CCR(r1)
310 #ifdef SHOW_SYSCALLS
311 bl do_show_syscall
312 #endif /* SHOW_SYSCALLS */
313 #ifdef CONFIG_TRACE_IRQFLAGS
314 /* Return from syscalls can (and generally will) hard enable
315 * interrupts. You aren't supposed to call a syscall with
316 * interrupts disabled in the first place. However, to ensure
317 * that we get it right vs. lockdep if it happens, we force
318 * that hard enable here with appropriate tracing if we see
319 * that we have been called with interrupts off
320 */
321 mfmsr r11
322 andi. r12,r11,MSR_EE
323 bne+ 1f
324 /* We came in with interrupts disabled, we enable them now */
325 bl trace_hardirqs_on
326 mfmsr r11
327 lwz r0,GPR0(r1)
328 lwz r3,GPR3(r1)
329 lwz r4,GPR4(r1)
330 ori r11,r11,MSR_EE
331 lwz r5,GPR5(r1)
332 lwz r6,GPR6(r1)
333 lwz r7,GPR7(r1)
334 lwz r8,GPR8(r1)
335 mtmsr r11
336 1:
337 #endif /* CONFIG_TRACE_IRQFLAGS */
338 CURRENT_THREAD_INFO(r10, r1)
339 lwz r11,TI_FLAGS(r10)
340 andi. r11,r11,_TIF_SYSCALL_T_OR_A
341 bne- syscall_dotrace
342 syscall_dotrace_cont:
343 cmplwi 0,r0,NR_syscalls
344 lis r10,sys_call_table@h
345 ori r10,r10,sys_call_table@l
346 slwi r0,r0,2
347 bge- 66f
348 lwzx r10,r10,r0 /* Fetch system call handler [ptr] */
349 mtlr r10
350 addi r9,r1,STACK_FRAME_OVERHEAD
351 PPC440EP_ERR42
352 blrl /* Call handler */
353 .globl ret_from_syscall
354 ret_from_syscall:
355 #ifdef SHOW_SYSCALLS
356 bl do_show_syscall_exit
357 #endif
358 mr r6,r3
359 CURRENT_THREAD_INFO(r12, r1)
360 /* disable interrupts so current_thread_info()->flags can't change */
361 LOAD_MSR_KERNEL(r10,MSR_KERNEL) /* doesn't include MSR_EE */
362 /* Note: We don't bother telling lockdep about it */
363 SYNC
364 MTMSRD(r10)
365 lwz r9,TI_FLAGS(r12)
366 li r8,-_LAST_ERRNO
367 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
368 bne- syscall_exit_work
369 cmplw 0,r3,r8
370 blt+ syscall_exit_cont
371 lwz r11,_CCR(r1) /* Load CR */
372 neg r3,r3
373 oris r11,r11,0x1000 /* Set SO bit in CR */
374 stw r11,_CCR(r1)
375 syscall_exit_cont:
376 lwz r8,_MSR(r1)
377 #ifdef CONFIG_TRACE_IRQFLAGS
378 /* If we are going to return from the syscall with interrupts
379 * off, we trace that here. It shouldn't happen though but we
380 * want to catch the bugger if it does right ?
381 */
382 andi. r10,r8,MSR_EE
383 bne+ 1f
384 stw r3,GPR3(r1)
385 bl trace_hardirqs_off
386 lwz r3,GPR3(r1)
387 1:
388 #endif /* CONFIG_TRACE_IRQFLAGS */
389 #if defined(CONFIG_4xx) || defined(CONFIG_BOOKE)
390 /* If the process has its own DBCR0 value, load it up. The internal
391 debug mode bit tells us that dbcr0 should be loaded. */
392 lwz r0,THREAD+THREAD_DBCR0(r2)
393 andis. r10,r0,DBCR0_IDM@h
394 bnel- load_dbcr0
395 #endif
396 #ifdef CONFIG_44x
397 BEGIN_MMU_FTR_SECTION
398 lis r4,icache_44x_need_flush@ha
399 lwz r5,icache_44x_need_flush@l(r4)
400 cmplwi cr0,r5,0
401 bne- 2f
402 1:
403 END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_47x)
404 #endif /* CONFIG_44x */
405 BEGIN_FTR_SECTION
406 lwarx r7,0,r1
407 END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX)
408 stwcx. r0,0,r1 /* to clear the reservation */
409 lwz r4,_LINK(r1)
410 lwz r5,_CCR(r1)
411 mtlr r4
412 mtcr r5
413 lwz r7,_NIP(r1)
414 FIX_SRR1(r8, r0)
415 lwz r2,GPR2(r1)
416 lwz r1,GPR1(r1)
417 mtspr SPRN_SRR0,r7
418 mtspr SPRN_SRR1,r8
419 SYNC
420 RFI
421 #ifdef CONFIG_44x
422 2: li r7,0
423 iccci r0,r0
424 stw r7,icache_44x_need_flush@l(r4)
425 b 1b
426 #endif /* CONFIG_44x */
428 66: li r3,-ENOSYS
429 b ret_from_syscall
431 .globl ret_from_fork
432 ret_from_fork:
433 REST_NVGPRS(r1)
434 bl schedule_tail
435 li r3,0
436 b ret_from_syscall
438 .globl ret_from_kernel_thread
439 ret_from_kernel_thread:
440 REST_NVGPRS(r1)
441 bl schedule_tail
442 li r3,0
443 stw r3,0(r1)
444 mtlr r14
445 mr r3,r15
446 PPC440EP_ERR42
447 blrl
448 li r3,0
449 b ret_from_syscall
451 /* Traced system call support */
452 syscall_dotrace:
453 SAVE_NVGPRS(r1)
454 li r0,0xc00
455 stw r0,_TRAP(r1)
456 addi r3,r1,STACK_FRAME_OVERHEAD
457 bl do_syscall_trace_enter
458 /*
459 * Restore argument registers possibly just changed.
460 * We use the return value of do_syscall_trace_enter
461 * for call number to look up in the table (r0).
462 */
463 mr r0,r3
464 lwz r3,GPR3(r1)
465 lwz r4,GPR4(r1)
466 lwz r5,GPR5(r1)
467 lwz r6,GPR6(r1)
468 lwz r7,GPR7(r1)
469 lwz r8,GPR8(r1)
470 REST_NVGPRS(r1)
471 b syscall_dotrace_cont
473 syscall_exit_work:
474 andi. r0,r9,_TIF_RESTOREALL
475 beq+ 0f
476 REST_NVGPRS(r1)
477 b 2f
478 0: cmplw 0,r3,r8
479 blt+ 1f
480 andi. r0,r9,_TIF_NOERROR
481 bne- 1f
482 lwz r11,_CCR(r1) /* Load CR */
483 neg r3,r3
484 oris r11,r11,0x1000 /* Set SO bit in CR */
485 stw r11,_CCR(r1)
487 1: stw r6,RESULT(r1) /* Save result */
488 stw r3,GPR3(r1) /* Update return value */
489 2: andi. r0,r9,(_TIF_PERSYSCALL_MASK)
490 beq 4f
492 /* Clear per-syscall TIF flags if any are set. */
494 li r11,_TIF_PERSYSCALL_MASK
495 addi r12,r12,TI_FLAGS
496 3: lwarx r8,0,r12
497 andc r8,r8,r11
498 #ifdef CONFIG_IBM405_ERR77
499 dcbt 0,r12
500 #endif
501 stwcx. r8,0,r12
502 bne- 3b
503 subi r12,r12,TI_FLAGS
505 4: /* Anything which requires enabling interrupts? */
506 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP)
507 beq ret_from_except
509 /* Re-enable interrupts. There is no need to trace that with
510 * lockdep as we are supposed to have IRQs on at this point
511 */
512 ori r10,r10,MSR_EE
513 SYNC
514 MTMSRD(r10)
516 /* Save NVGPRS if they're not saved already */
517 lwz r4,_TRAP(r1)
518 andi. r4,r4,1
519 beq 5f
520 SAVE_NVGPRS(r1)
521 li r4,0xc00
522 stw r4,_TRAP(r1)
523 5:
524 addi r3,r1,STACK_FRAME_OVERHEAD
525 bl do_syscall_trace_leave
526 b ret_from_except_full
528 #ifdef SHOW_SYSCALLS
529 do_show_syscall:
530 #ifdef SHOW_SYSCALLS_TASK
531 lis r11,show_syscalls_task@ha
532 lwz r11,show_syscalls_task@l(r11)
533 cmp 0,r2,r11
534 bnelr
535 #endif
536 stw r31,GPR31(r1)
537 mflr r31
538 lis r3,7f@ha
539 addi r3,r3,7f@l
540 lwz r4,GPR0(r1)
541 lwz r5,GPR3(r1)
542 lwz r6,GPR4(r1)
543 lwz r7,GPR5(r1)
544 lwz r8,GPR6(r1)
545 lwz r9,GPR7(r1)
546 bl printk
547 lis r3,77f@ha
548 addi r3,r3,77f@l
549 lwz r4,GPR8(r1)
550 mr r5,r2
551 bl printk
552 lwz r0,GPR0(r1)
553 lwz r3,GPR3(r1)
554 lwz r4,GPR4(r1)
555 lwz r5,GPR5(r1)
556 lwz r6,GPR6(r1)
557 lwz r7,GPR7(r1)
558 lwz r8,GPR8(r1)
559 mtlr r31
560 lwz r31,GPR31(r1)
561 blr
563 do_show_syscall_exit:
564 #ifdef SHOW_SYSCALLS_TASK
565 lis r11,show_syscalls_task@ha
566 lwz r11,show_syscalls_task@l(r11)
567 cmp 0,r2,r11
568 bnelr
569 #endif
570 stw r31,GPR31(r1)
571 mflr r31
572 stw r3,RESULT(r1) /* Save result */
573 mr r4,r3
574 lis r3,79f@ha
575 addi r3,r3,79f@l
576 bl printk
577 lwz r3,RESULT(r1)
578 mtlr r31
579 lwz r31,GPR31(r1)
580 blr
582 7: .string "syscall %d(%x, %x, %x, %x, %x, "
583 77: .string "%x), current=%p\n"
584 79: .string " -> %x\n"
585 .align 2,0
587 #ifdef SHOW_SYSCALLS_TASK
588 .data
589 .globl show_syscalls_task
590 show_syscalls_task:
591 .long -1
592 .text
593 #endif
594 #endif /* SHOW_SYSCALLS */
596 /*
597 * The fork/clone functions need to copy the full register set into
598 * the child process. Therefore we need to save all the nonvolatile
599 * registers (r13 - r31) before calling the C code.
600 */
601 .globl ppc_fork
602 ppc_fork:
603 SAVE_NVGPRS(r1)
604 lwz r0,_TRAP(r1)
605 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
606 stw r0,_TRAP(r1) /* register set saved */
607 b sys_fork
609 .globl ppc_vfork
610 ppc_vfork:
611 SAVE_NVGPRS(r1)
612 lwz r0,_TRAP(r1)
613 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
614 stw r0,_TRAP(r1) /* register set saved */
615 b sys_vfork
617 .globl ppc_clone
618 ppc_clone:
619 SAVE_NVGPRS(r1)
620 lwz r0,_TRAP(r1)
621 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
622 stw r0,_TRAP(r1) /* register set saved */
623 b sys_clone
625 .globl ppc_swapcontext
626 ppc_swapcontext:
627 SAVE_NVGPRS(r1)
628 lwz r0,_TRAP(r1)
629 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
630 stw r0,_TRAP(r1) /* register set saved */
631 b sys_swapcontext
633 /*
634 * Top-level page fault handling.
635 * This is in assembler because if do_page_fault tells us that
636 * it is a bad kernel page fault, we want to save the non-volatile
637 * registers before calling bad_page_fault.
638 */
639 .globl handle_page_fault
640 handle_page_fault:
641 stw r4,_DAR(r1)
642 addi r3,r1,STACK_FRAME_OVERHEAD
643 bl do_page_fault
644 cmpwi r3,0
645 beq+ ret_from_except
646 SAVE_NVGPRS(r1)
647 lwz r0,_TRAP(r1)
648 clrrwi r0,r0,1
649 stw r0,_TRAP(r1)
650 mr r5,r3
651 addi r3,r1,STACK_FRAME_OVERHEAD
652 lwz r4,_DAR(r1)
653 bl bad_page_fault
654 b ret_from_except_full
656 /*
657 * This routine switches between two different tasks. The process
658 * state of one is saved on its kernel stack. Then the state
659 * of the other is restored from its kernel stack. The memory
660 * management hardware is updated to the second process's state.
661 * Finally, we can return to the second process.
662 * On entry, r3 points to the THREAD for the current task, r4
663 * points to the THREAD for the new task.
664 *
665 * This routine is always called with interrupts disabled.
666 *
667 * Note: there are two ways to get to the "going out" portion
668 * of this code; either by coming in via the entry (_switch)
669 * or via "fork" which must set up an environment equivalent
670 * to the "_switch" path. If you change this , you'll have to
671 * change the fork code also.
672 *
673 * The code which creates the new task context is in 'copy_thread'
674 * in arch/ppc/kernel/process.c
675 */
676 _GLOBAL(_switch)
677 stwu r1,-INT_FRAME_SIZE(r1)
678 mflr r0
679 stw r0,INT_FRAME_SIZE+4(r1)
680 /* r3-r12 are caller saved -- Cort */
681 SAVE_NVGPRS(r1)
682 stw r0,_NIP(r1) /* Return to switch caller */
683 mfmsr r11
684 li r0,MSR_FP /* Disable floating-point */
685 #ifdef CONFIG_ALTIVEC
686 BEGIN_FTR_SECTION
687 oris r0,r0,MSR_VEC@h /* Disable altivec */
688 mfspr r12,SPRN_VRSAVE /* save vrsave register value */
689 stw r12,THREAD+THREAD_VRSAVE(r2)
690 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
691 #endif /* CONFIG_ALTIVEC */
692 #ifdef CONFIG_SPE
693 BEGIN_FTR_SECTION
694 oris r0,r0,MSR_SPE@h /* Disable SPE */
695 mfspr r12,SPRN_SPEFSCR /* save spefscr register value */
696 stw r12,THREAD+THREAD_SPEFSCR(r2)
697 END_FTR_SECTION_IFSET(CPU_FTR_SPE)
698 #endif /* CONFIG_SPE */
699 and. r0,r0,r11 /* FP or altivec or SPE enabled? */
700 beq+ 1f
701 andc r11,r11,r0
702 MTMSRD(r11)
703 isync
704 1: stw r11,_MSR(r1)
705 mfcr r10
706 stw r10,_CCR(r1)
707 stw r1,KSP(r3) /* Set old stack pointer */
709 #ifdef CONFIG_SMP
710 /* We need a sync somewhere here to make sure that if the
711 * previous task gets rescheduled on another CPU, it sees all
712 * stores it has performed on this one.
713 */
714 sync
715 #endif /* CONFIG_SMP */
717 tophys(r0,r4)
718 CLR_TOP32(r0)
719 mtspr SPRN_SPRG_THREAD,r0 /* Update current THREAD phys addr */
720 lwz r1,KSP(r4) /* Load new stack pointer */
722 /* save the old current 'last' for return value */
723 mr r3,r2
724 addi r2,r4,-THREAD /* Update current */
726 #ifdef CONFIG_ALTIVEC
727 BEGIN_FTR_SECTION
728 lwz r0,THREAD+THREAD_VRSAVE(r2)
729 mtspr SPRN_VRSAVE,r0 /* if G4, restore VRSAVE reg */
730 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
731 #endif /* CONFIG_ALTIVEC */
732 #ifdef CONFIG_SPE
733 BEGIN_FTR_SECTION
734 lwz r0,THREAD+THREAD_SPEFSCR(r2)
735 mtspr SPRN_SPEFSCR,r0 /* restore SPEFSCR reg */
736 END_FTR_SECTION_IFSET(CPU_FTR_SPE)
737 #endif /* CONFIG_SPE */
739 lwz r0,_CCR(r1)
740 mtcrf 0xFF,r0
741 /* r3-r12 are destroyed -- Cort */
742 REST_NVGPRS(r1)
744 lwz r4,_NIP(r1) /* Return to _switch caller in new task */
745 mtlr r4
746 addi r1,r1,INT_FRAME_SIZE
747 blr
749 .globl fast_exception_return
750 fast_exception_return:
751 #if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
752 andi. r10,r9,MSR_RI /* check for recoverable interrupt */
753 beq 1f /* if not, we've got problems */
754 #endif
756 2: REST_4GPRS(3, r11)
757 lwz r10,_CCR(r11)
758 REST_GPR(1, r11)
759 mtcr r10
760 lwz r10,_LINK(r11)
761 mtlr r10
762 REST_GPR(10, r11)
763 mtspr SPRN_SRR1,r9
764 mtspr SPRN_SRR0,r12
765 REST_GPR(9, r11)
766 REST_GPR(12, r11)
767 lwz r11,GPR11(r11)
768 SYNC
769 RFI
771 #if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
772 /* check if the exception happened in a restartable section */
773 1: lis r3,exc_exit_restart_end@ha
774 addi r3,r3,exc_exit_restart_end@l
775 cmplw r12,r3
776 bge 3f
777 lis r4,exc_exit_restart@ha
778 addi r4,r4,exc_exit_restart@l
779 cmplw r12,r4
780 blt 3f
781 lis r3,fee_restarts@ha
782 tophys(r3,r3)
783 lwz r5,fee_restarts@l(r3)
784 addi r5,r5,1
785 stw r5,fee_restarts@l(r3)
786 mr r12,r4 /* restart at exc_exit_restart */
787 b 2b
789 .section .bss
790 .align 2
791 fee_restarts:
792 .space 4
793 .previous
795 /* aargh, a nonrecoverable interrupt, panic */
796 /* aargh, we don't know which trap this is */
797 /* but the 601 doesn't implement the RI bit, so assume it's OK */
798 3:
799 BEGIN_FTR_SECTION
800 b 2b
801 END_FTR_SECTION_IFSET(CPU_FTR_601)
802 li r10,-1
803 stw r10,_TRAP(r11)
804 addi r3,r1,STACK_FRAME_OVERHEAD
805 lis r10,MSR_KERNEL@h
806 ori r10,r10,MSR_KERNEL@l
807 bl transfer_to_handler_full
808 .long nonrecoverable_exception
809 .long ret_from_except
810 #endif
812 .globl ret_from_except_full
813 ret_from_except_full:
814 REST_NVGPRS(r1)
815 /* fall through */
817 .globl ret_from_except
818 ret_from_except:
819 /* Hard-disable interrupts so that current_thread_info()->flags
820 * can't change between when we test it and when we return
821 * from the interrupt. */
822 /* Note: We don't bother telling lockdep about it */
823 LOAD_MSR_KERNEL(r10,MSR_KERNEL)
824 SYNC /* Some chip revs have problems here... */
825 MTMSRD(r10) /* disable interrupts */
827 lwz r3,_MSR(r1) /* Returning to user mode? */
828 andi. r0,r3,MSR_PR
829 beq resume_kernel
831 user_exc_return: /* r10 contains MSR_KERNEL here */
832 /* Check current_thread_info()->flags */
833 CURRENT_THREAD_INFO(r9, r1)
834 lwz r9,TI_FLAGS(r9)
835 andi. r0,r9,_TIF_USER_WORK_MASK
836 bne do_work
838 restore_user:
839 #if defined(CONFIG_4xx) || defined(CONFIG_BOOKE)
840 /* Check whether this process has its own DBCR0 value. The internal
841 debug mode bit tells us that dbcr0 should be loaded. */
842 lwz r0,THREAD+THREAD_DBCR0(r2)
843 andis. r10,r0,DBCR0_IDM@h
844 bnel- load_dbcr0
845 #endif
847 b restore
849 /* N.B. the only way to get here is from the beq following ret_from_except. */
850 resume_kernel:
851 /* check current_thread_info, _TIF_EMULATE_STACK_STORE */
852 CURRENT_THREAD_INFO(r9, r1)
853 lwz r8,TI_FLAGS(r9)
854 andis. r8,r8,_TIF_EMULATE_STACK_STORE@h
855 beq+ 1f
857 addi r8,r1,INT_FRAME_SIZE /* Get the kprobed function entry */
859 lwz r3,GPR1(r1)
860 subi r3,r3,INT_FRAME_SIZE /* dst: Allocate a trampoline exception frame */
861 mr r4,r1 /* src: current exception frame */
862 mr r1,r3 /* Reroute the trampoline frame to r1 */
864 /* Copy from the original to the trampoline. */
865 li r5,INT_FRAME_SIZE/4 /* size: INT_FRAME_SIZE */
866 li r6,0 /* start offset: 0 */
867 mtctr r5
868 2: lwzx r0,r6,r4
869 stwx r0,r6,r3
870 addi r6,r6,4
871 bdnz 2b
873 /* Do real store operation to complete stwu */
874 lwz r5,GPR1(r1)
875 stw r8,0(r5)
877 /* Clear _TIF_EMULATE_STACK_STORE flag */
878 lis r11,_TIF_EMULATE_STACK_STORE@h
879 addi r5,r9,TI_FLAGS
880 0: lwarx r8,0,r5
881 andc r8,r8,r11
882 #ifdef CONFIG_IBM405_ERR77
883 dcbt 0,r5
884 #endif
885 stwcx. r8,0,r5
886 bne- 0b
887 1:
889 #ifdef CONFIG_PREEMPT
890 /* check current_thread_info->preempt_count */
891 lwz r0,TI_PREEMPT(r9)
892 cmpwi 0,r0,0 /* if non-zero, just restore regs and return */
893 bne restore
894 andi. r8,r8,_TIF_NEED_RESCHED
895 beq+ restore
896 lwz r3,_MSR(r1)
897 andi. r0,r3,MSR_EE /* interrupts off? */
898 beq restore /* don't schedule if so */
899 #ifdef CONFIG_TRACE_IRQFLAGS
900 /* Lockdep thinks irqs are enabled, we need to call
901 * preempt_schedule_irq with IRQs off, so we inform lockdep
902 * now that we -did- turn them off already
903 */
904 bl trace_hardirqs_off
905 #endif
906 1: bl preempt_schedule_irq
907 CURRENT_THREAD_INFO(r9, r1)
908 lwz r3,TI_FLAGS(r9)
909 andi. r0,r3,_TIF_NEED_RESCHED
910 bne- 1b
911 #ifdef CONFIG_TRACE_IRQFLAGS
912 /* And now, to properly rebalance the above, we tell lockdep they
913 * are being turned back on, which will happen when we return
914 */
915 bl trace_hardirqs_on
916 #endif
917 #endif /* CONFIG_PREEMPT */
919 /* interrupts are hard-disabled at this point */
920 restore:
921 #ifdef CONFIG_44x
922 BEGIN_MMU_FTR_SECTION
923 b 1f
924 END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_47x)
925 lis r4,icache_44x_need_flush@ha
926 lwz r5,icache_44x_need_flush@l(r4)
927 cmplwi cr0,r5,0
928 beq+ 1f
929 li r6,0
930 iccci r0,r0
931 stw r6,icache_44x_need_flush@l(r4)
932 1:
933 #endif /* CONFIG_44x */
935 lwz r9,_MSR(r1)
936 #ifdef CONFIG_TRACE_IRQFLAGS
937 /* Lockdep doesn't know about the fact that IRQs are temporarily turned
938 * off in this assembly code while peeking at TI_FLAGS() and such. However
939 * we need to inform it if the exception turned interrupts off, and we
940 * are about to trun them back on.
941 *
942 * The problem here sadly is that we don't know whether the exceptions was
943 * one that turned interrupts off or not. So we always tell lockdep about
944 * turning them on here when we go back to wherever we came from with EE
945 * on, even if that may meen some redudant calls being tracked. Maybe later
946 * we could encode what the exception did somewhere or test the exception
947 * type in the pt_regs but that sounds overkill
948 */
949 andi. r10,r9,MSR_EE
950 beq 1f
951 /*
952 * Since the ftrace irqsoff latency trace checks CALLER_ADDR1,
953 * which is the stack frame here, we need to force a stack frame
954 * in case we came from user space.
955 */
956 stwu r1,-32(r1)
957 mflr r0
958 stw r0,4(r1)
959 stwu r1,-32(r1)
960 bl trace_hardirqs_on
961 lwz r1,0(r1)
962 lwz r1,0(r1)
963 lwz r9,_MSR(r1)
964 1:
965 #endif /* CONFIG_TRACE_IRQFLAGS */
967 lwz r0,GPR0(r1)
968 lwz r2,GPR2(r1)
969 REST_4GPRS(3, r1)
970 REST_2GPRS(7, r1)
972 lwz r10,_XER(r1)
973 lwz r11,_CTR(r1)
974 mtspr SPRN_XER,r10
975 mtctr r11
977 PPC405_ERR77(0,r1)
978 BEGIN_FTR_SECTION
979 lwarx r11,0,r1
980 END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX)
981 stwcx. r0,0,r1 /* to clear the reservation */
983 #if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
984 andi. r10,r9,MSR_RI /* check if this exception occurred */
985 beql nonrecoverable /* at a bad place (MSR:RI = 0) */
987 lwz r10,_CCR(r1)
988 lwz r11,_LINK(r1)
989 mtcrf 0xFF,r10
990 mtlr r11
992 /*
993 * Once we put values in SRR0 and SRR1, we are in a state
994 * where exceptions are not recoverable, since taking an
995 * exception will trash SRR0 and SRR1. Therefore we clear the
996 * MSR:RI bit to indicate this. If we do take an exception,
997 * we can't return to the point of the exception but we
998 * can restart the exception exit path at the label
999 * exc_exit_restart below. -- paulus
1000 */
1001 LOAD_MSR_KERNEL(r10,MSR_KERNEL & ~MSR_RI)
1002 SYNC
1003 MTMSRD(r10) /* clear the RI bit */
1004 .globl exc_exit_restart
1005 exc_exit_restart:
1006 lwz r12,_NIP(r1)
1007 FIX_SRR1(r9,r10)
1008 mtspr SPRN_SRR0,r12
1009 mtspr SPRN_SRR1,r9
1010 REST_4GPRS(9, r1)
1011 lwz r1,GPR1(r1)
1012 .globl exc_exit_restart_end
1013 exc_exit_restart_end:
1014 SYNC
1015 RFI
1017 #else /* !(CONFIG_4xx || CONFIG_BOOKE) */
1018 /*
1019 * This is a bit different on 4xx/Book-E because it doesn't have
1020 * the RI bit in the MSR.
1021 * The TLB miss handler checks if we have interrupted
1022 * the exception exit path and restarts it if so
1023 * (well maybe one day it will... :).
1024 */
1025 lwz r11,_LINK(r1)
1026 mtlr r11
1027 lwz r10,_CCR(r1)
1028 mtcrf 0xff,r10
1029 REST_2GPRS(9, r1)
1030 .globl exc_exit_restart
1031 exc_exit_restart:
1032 lwz r11,_NIP(r1)
1033 lwz r12,_MSR(r1)
1034 exc_exit_start:
1035 mtspr SPRN_SRR0,r11
1036 mtspr SPRN_SRR1,r12
1037 REST_2GPRS(11, r1)
1038 lwz r1,GPR1(r1)
1039 .globl exc_exit_restart_end
1040 exc_exit_restart_end:
1041 PPC405_ERR77_SYNC
1042 rfi
1043 b . /* prevent prefetch past rfi */
1045 /*
1046 * Returning from a critical interrupt in user mode doesn't need
1047 * to be any different from a normal exception. For a critical
1048 * interrupt in the kernel, we just return (without checking for
1049 * preemption) since the interrupt may have happened at some crucial
1050 * place (e.g. inside the TLB miss handler), and because we will be
1051 * running with r1 pointing into critical_stack, not the current
1052 * process's kernel stack (and therefore current_thread_info() will
1053 * give the wrong answer).
1054 * We have to restore various SPRs that may have been in use at the
1055 * time of the critical interrupt.
1056 *
1057 */
1058 #ifdef CONFIG_40x
1059 #define PPC_40x_TURN_OFF_MSR_DR \
1060 /* avoid any possible TLB misses here by turning off MSR.DR, we \
1061 * assume the instructions here are mapped by a pinned TLB entry */ \
1062 li r10,MSR_IR; \
1063 mtmsr r10; \
1064 isync; \
1065 tophys(r1, r1);
1066 #else
1067 #define PPC_40x_TURN_OFF_MSR_DR
1068 #endif
1070 #define RET_FROM_EXC_LEVEL(exc_lvl_srr0, exc_lvl_srr1, exc_lvl_rfi) \
1071 REST_NVGPRS(r1); \
1072 lwz r3,_MSR(r1); \
1073 andi. r3,r3,MSR_PR; \
1074 LOAD_MSR_KERNEL(r10,MSR_KERNEL); \
1075 bne user_exc_return; \
1076 lwz r0,GPR0(r1); \
1077 lwz r2,GPR2(r1); \
1078 REST_4GPRS(3, r1); \
1079 REST_2GPRS(7, r1); \
1080 lwz r10,_XER(r1); \
1081 lwz r11,_CTR(r1); \
1082 mtspr SPRN_XER,r10; \
1083 mtctr r11; \
1084 PPC405_ERR77(0,r1); \
1085 stwcx. r0,0,r1; /* to clear the reservation */ \
1086 lwz r11,_LINK(r1); \
1087 mtlr r11; \
1088 lwz r10,_CCR(r1); \
1089 mtcrf 0xff,r10; \
1090 PPC_40x_TURN_OFF_MSR_DR; \
1091 lwz r9,_DEAR(r1); \
1092 lwz r10,_ESR(r1); \
1093 mtspr SPRN_DEAR,r9; \
1094 mtspr SPRN_ESR,r10; \
1095 lwz r11,_NIP(r1); \
1096 lwz r12,_MSR(r1); \
1097 mtspr exc_lvl_srr0,r11; \
1098 mtspr exc_lvl_srr1,r12; \
1099 lwz r9,GPR9(r1); \
1100 lwz r12,GPR12(r1); \
1101 lwz r10,GPR10(r1); \
1102 lwz r11,GPR11(r1); \
1103 lwz r1,GPR1(r1); \
1104 PPC405_ERR77_SYNC; \
1105 exc_lvl_rfi; \
1106 b .; /* prevent prefetch past exc_lvl_rfi */
1108 #define RESTORE_xSRR(exc_lvl_srr0, exc_lvl_srr1) \
1109 lwz r9,_##exc_lvl_srr0(r1); \
1110 lwz r10,_##exc_lvl_srr1(r1); \
1111 mtspr SPRN_##exc_lvl_srr0,r9; \
1112 mtspr SPRN_##exc_lvl_srr1,r10;
1114 #if defined(CONFIG_PPC_BOOK3E_MMU)
1115 #ifdef CONFIG_PHYS_64BIT
1116 #define RESTORE_MAS7 \
1117 lwz r11,MAS7(r1); \
1118 mtspr SPRN_MAS7,r11;
1119 #else
1120 #define RESTORE_MAS7
1121 #endif /* CONFIG_PHYS_64BIT */
1122 #define RESTORE_MMU_REGS \
1123 lwz r9,MAS0(r1); \
1124 lwz r10,MAS1(r1); \
1125 lwz r11,MAS2(r1); \
1126 mtspr SPRN_MAS0,r9; \
1127 lwz r9,MAS3(r1); \
1128 mtspr SPRN_MAS1,r10; \
1129 lwz r10,MAS6(r1); \
1130 mtspr SPRN_MAS2,r11; \
1131 mtspr SPRN_MAS3,r9; \
1132 mtspr SPRN_MAS6,r10; \
1133 RESTORE_MAS7;
1134 #elif defined(CONFIG_44x)
1135 #define RESTORE_MMU_REGS \
1136 lwz r9,MMUCR(r1); \
1137 mtspr SPRN_MMUCR,r9;
1138 #else
1139 #define RESTORE_MMU_REGS
1140 #endif
1142 #ifdef CONFIG_40x
1143 .globl ret_from_crit_exc
1144 ret_from_crit_exc:
1145 mfspr r9,SPRN_SPRG_THREAD
1146 lis r10,saved_ksp_limit@ha;
1147 lwz r10,saved_ksp_limit@l(r10);
1148 tovirt(r9,r9);
1149 stw r10,KSP_LIMIT(r9)
1150 lis r9,crit_srr0@ha;
1151 lwz r9,crit_srr0@l(r9);
1152 lis r10,crit_srr1@ha;
1153 lwz r10,crit_srr1@l(r10);
1154 mtspr SPRN_SRR0,r9;
1155 mtspr SPRN_SRR1,r10;
1156 RET_FROM_EXC_LEVEL(SPRN_CSRR0, SPRN_CSRR1, PPC_RFCI)
1157 #endif /* CONFIG_40x */
1159 #ifdef CONFIG_BOOKE
1160 .globl ret_from_crit_exc
1161 ret_from_crit_exc:
1162 mfspr r9,SPRN_SPRG_THREAD
1163 lwz r10,SAVED_KSP_LIMIT(r1)
1164 stw r10,KSP_LIMIT(r9)
1165 RESTORE_xSRR(SRR0,SRR1);
1166 RESTORE_MMU_REGS;
1167 RET_FROM_EXC_LEVEL(SPRN_CSRR0, SPRN_CSRR1, PPC_RFCI)
1169 .globl ret_from_debug_exc
1170 ret_from_debug_exc:
1171 mfspr r9,SPRN_SPRG_THREAD
1172 lwz r10,SAVED_KSP_LIMIT(r1)
1173 stw r10,KSP_LIMIT(r9)
1174 lwz r9,THREAD_INFO-THREAD(r9)
1175 CURRENT_THREAD_INFO(r10, r1)
1176 lwz r10,TI_PREEMPT(r10)
1177 stw r10,TI_PREEMPT(r9)
1178 RESTORE_xSRR(SRR0,SRR1);
1179 RESTORE_xSRR(CSRR0,CSRR1);
1180 RESTORE_MMU_REGS;
1181 RET_FROM_EXC_LEVEL(SPRN_DSRR0, SPRN_DSRR1, PPC_RFDI)
1183 .globl ret_from_mcheck_exc
1184 ret_from_mcheck_exc:
1185 mfspr r9,SPRN_SPRG_THREAD
1186 lwz r10,SAVED_KSP_LIMIT(r1)
1187 stw r10,KSP_LIMIT(r9)
1188 RESTORE_xSRR(SRR0,SRR1);
1189 RESTORE_xSRR(CSRR0,CSRR1);
1190 RESTORE_xSRR(DSRR0,DSRR1);
1191 RESTORE_MMU_REGS;
1192 RET_FROM_EXC_LEVEL(SPRN_MCSRR0, SPRN_MCSRR1, PPC_RFMCI)
1193 #endif /* CONFIG_BOOKE */
1195 /*
1196 * Load the DBCR0 value for a task that is being ptraced,
1197 * having first saved away the global DBCR0. Note that r0
1198 * has the dbcr0 value to set upon entry to this.
1199 */
1200 load_dbcr0:
1201 mfmsr r10 /* first disable debug exceptions */
1202 rlwinm r10,r10,0,~MSR_DE
1203 mtmsr r10
1204 isync
1205 mfspr r10,SPRN_DBCR0
1206 lis r11,global_dbcr0@ha
1207 addi r11,r11,global_dbcr0@l
1208 #ifdef CONFIG_SMP
1209 CURRENT_THREAD_INFO(r9, r1)
1210 lwz r9,TI_CPU(r9)
1211 slwi r9,r9,3
1212 add r11,r11,r9
1213 #endif
1214 stw r10,0(r11)
1215 mtspr SPRN_DBCR0,r0
1216 lwz r10,4(r11)
1217 addi r10,r10,1
1218 stw r10,4(r11)
1219 li r11,-1
1220 mtspr SPRN_DBSR,r11 /* clear all pending debug events */
1221 blr
1223 .section .bss
1224 .align 4
1225 global_dbcr0:
1226 .space 8*NR_CPUS
1227 .previous
1228 #endif /* !(CONFIG_4xx || CONFIG_BOOKE) */
1230 do_work: /* r10 contains MSR_KERNEL here */
1231 andi. r0,r9,_TIF_NEED_RESCHED
1232 beq do_user_signal
1234 do_resched: /* r10 contains MSR_KERNEL here */
1235 /* Note: We don't need to inform lockdep that we are enabling
1236 * interrupts here. As far as it knows, they are already enabled
1237 */
1238 ori r10,r10,MSR_EE
1239 SYNC
1240 MTMSRD(r10) /* hard-enable interrupts */
1241 bl schedule
1242 recheck:
1243 /* Note: And we don't tell it we are disabling them again
1244 * neither. Those disable/enable cycles used to peek at
1245 * TI_FLAGS aren't advertised.
1246 */
1247 LOAD_MSR_KERNEL(r10,MSR_KERNEL)
1248 SYNC
1249 MTMSRD(r10) /* disable interrupts */
1250 CURRENT_THREAD_INFO(r9, r1)
1251 lwz r9,TI_FLAGS(r9)
1252 andi. r0,r9,_TIF_NEED_RESCHED
1253 bne- do_resched
1254 andi. r0,r9,_TIF_USER_WORK_MASK
1255 beq restore_user
1256 do_user_signal: /* r10 contains MSR_KERNEL here */
1257 ori r10,r10,MSR_EE
1258 SYNC
1259 MTMSRD(r10) /* hard-enable interrupts */
1260 /* save r13-r31 in the exception frame, if not already done */
1261 lwz r3,_TRAP(r1)
1262 andi. r0,r3,1
1263 beq 2f
1264 SAVE_NVGPRS(r1)
1265 rlwinm r3,r3,0,0,30
1266 stw r3,_TRAP(r1)
1267 2: addi r3,r1,STACK_FRAME_OVERHEAD
1268 mr r4,r9
1269 bl do_notify_resume
1270 REST_NVGPRS(r1)
1271 b recheck
1273 /*
1274 * We come here when we are at the end of handling an exception
1275 * that occurred at a place where taking an exception will lose
1276 * state information, such as the contents of SRR0 and SRR1.
1277 */
1278 nonrecoverable:
1279 lis r10,exc_exit_restart_end@ha
1280 addi r10,r10,exc_exit_restart_end@l
1281 cmplw r12,r10
1282 bge 3f
1283 lis r11,exc_exit_restart@ha
1284 addi r11,r11,exc_exit_restart@l
1285 cmplw r12,r11
1286 blt 3f
1287 lis r10,ee_restarts@ha
1288 lwz r12,ee_restarts@l(r10)
1289 addi r12,r12,1
1290 stw r12,ee_restarts@l(r10)
1291 mr r12,r11 /* restart at exc_exit_restart */
1292 blr
1293 3: /* OK, we can't recover, kill this process */
1294 /* but the 601 doesn't implement the RI bit, so assume it's OK */
1295 BEGIN_FTR_SECTION
1296 blr
1297 END_FTR_SECTION_IFSET(CPU_FTR_601)
1298 lwz r3,_TRAP(r1)
1299 andi. r0,r3,1
1300 beq 4f
1301 SAVE_NVGPRS(r1)
1302 rlwinm r3,r3,0,0,30
1303 stw r3,_TRAP(r1)
1304 4: addi r3,r1,STACK_FRAME_OVERHEAD
1305 bl nonrecoverable_exception
1306 /* shouldn't return */
1307 b 4b
1309 .section .bss
1310 .align 2
1311 ee_restarts:
1312 .space 4
1313 .previous
1315 /*
1316 * PROM code for specific machines follows. Put it
1317 * here so it's easy to add arch-specific sections later.
1318 * -- Cort
1319 */
1320 #ifdef CONFIG_PPC_RTAS
1321 /*
1322 * On CHRP, the Run-Time Abstraction Services (RTAS) have to be
1323 * called with the MMU off.
1324 */
1325 _GLOBAL(enter_rtas)
1326 stwu r1,-INT_FRAME_SIZE(r1)
1327 mflr r0
1328 stw r0,INT_FRAME_SIZE+4(r1)
1329 LOAD_REG_ADDR(r4, rtas)
1330 lis r6,1f@ha /* physical return address for rtas */
1331 addi r6,r6,1f@l
1332 tophys(r6,r6)
1333 tophys(r7,r1)
1334 lwz r8,RTASENTRY(r4)
1335 lwz r4,RTASBASE(r4)
1336 mfmsr r9
1337 stw r9,8(r1)
1338 LOAD_MSR_KERNEL(r0,MSR_KERNEL)
1339 SYNC /* disable interrupts so SRR0/1 */
1340 MTMSRD(r0) /* don't get trashed */
1341 li r9,MSR_KERNEL & ~(MSR_IR|MSR_DR)
1342 mtlr r6
1343 mtspr SPRN_SPRG_RTAS,r7
1344 mtspr SPRN_SRR0,r8
1345 mtspr SPRN_SRR1,r9
1346 RFI
1347 1: tophys(r9,r1)
1348 lwz r8,INT_FRAME_SIZE+4(r9) /* get return address */
1349 lwz r9,8(r9) /* original msr value */
1350 FIX_SRR1(r9,r0)
1351 addi r1,r1,INT_FRAME_SIZE
1352 li r0,0
1353 mtspr SPRN_SPRG_RTAS,r0
1354 mtspr SPRN_SRR0,r8
1355 mtspr SPRN_SRR1,r9
1356 RFI /* return to caller */
1358 .globl machine_check_in_rtas
1359 machine_check_in_rtas:
1360 twi 31,0,0
1361 /* XXX load up BATs and panic */
1363 #endif /* CONFIG_PPC_RTAS */
1365 #ifdef CONFIG_FUNCTION_TRACER
1366 #ifdef CONFIG_DYNAMIC_FTRACE
1367 _GLOBAL(mcount)
1368 _GLOBAL(_mcount)
1369 /*
1370 * It is required that _mcount on PPC32 must preserve the
1371 * link register. But we have r0 to play with. We use r0
1372 * to push the return address back to the caller of mcount
1373 * into the ctr register, restore the link register and
1374 * then jump back using the ctr register.
1375 */
1376 mflr r0
1377 mtctr r0
1378 lwz r0, 4(r1)
1379 mtlr r0
1380 bctr
1382 _GLOBAL(ftrace_caller)
1383 MCOUNT_SAVE_FRAME
1384 /* r3 ends up with link register */
1385 subi r3, r3, MCOUNT_INSN_SIZE
1386 .globl ftrace_call
1387 ftrace_call:
1388 bl ftrace_stub
1389 nop
1390 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1391 .globl ftrace_graph_call
1392 ftrace_graph_call:
1393 b ftrace_graph_stub
1394 _GLOBAL(ftrace_graph_stub)
1395 #endif
1396 MCOUNT_RESTORE_FRAME
1397 /* old link register ends up in ctr reg */
1398 bctr
1399 #else
1400 _GLOBAL(mcount)
1401 _GLOBAL(_mcount)
1403 MCOUNT_SAVE_FRAME
1405 subi r3, r3, MCOUNT_INSN_SIZE
1406 LOAD_REG_ADDR(r5, ftrace_trace_function)
1407 lwz r5,0(r5)
1409 mtctr r5
1410 bctrl
1411 nop
1413 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1414 b ftrace_graph_caller
1415 #endif
1416 MCOUNT_RESTORE_FRAME
1417 bctr
1418 #endif
1420 _GLOBAL(ftrace_stub)
1421 blr
1423 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1424 _GLOBAL(ftrace_graph_caller)
1425 /* load r4 with local address */
1426 lwz r4, 44(r1)
1427 subi r4, r4, MCOUNT_INSN_SIZE
1429 /* get the parent address */
1430 addi r3, r1, 52
1432 bl prepare_ftrace_return
1433 nop
1435 MCOUNT_RESTORE_FRAME
1436 /* old link register ends up in ctr reg */
1437 bctr
1439 _GLOBAL(return_to_handler)
1440 /* need to save return values */
1441 stwu r1, -32(r1)
1442 stw r3, 20(r1)
1443 stw r4, 16(r1)
1444 stw r31, 12(r1)
1445 mr r31, r1
1447 bl ftrace_return_to_handler
1448 nop
1450 /* return value has real return address */
1451 mtlr r3
1453 lwz r3, 20(r1)
1454 lwz r4, 16(r1)
1455 lwz r31,12(r1)
1456 lwz r1, 0(r1)
1458 /* Jump back to real return address */
1459 blr
1460 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1462 #endif /* CONFIG_MCOUNT */