| blob | 59e77eb63338de4ea3bab18462967a6e16d4b208 |
1 /*
2 * PowerPC version
3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 * Rewritten by Cort Dougan (cort@fsmlabs.com) for PReP
5 * Copyright (C) 1996 Cort Dougan <cort@fsmlabs.com>
6 * Adapted for Power Macintosh by Paul Mackerras.
7 * Low-level exception handlers and MMU support
8 * rewritten by Paul Mackerras.
9 * Copyright (C) 1996 Paul Mackerras.
10 * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
11 *
12 * This file contains the system call entry code, context switch
13 * code, and exception/interrupt return code for PowerPC.
14 *
15 * This program is free software; you can redistribute it and/or
16 * modify it under the terms of the GNU General Public License
17 * as published by the Free Software Foundation; either version
18 * 2 of the License, or (at your option) any later version.
19 *
20 */
22 #include <linux/errno.h>
23 #include <linux/sys.h>
24 #include <linux/threads.h>
25 #include <asm/processor.h>
26 #include <asm/page.h>
27 #include <asm/mmu.h>
28 #include <asm/cputable.h>
29 #include <asm/thread_info.h>
30 #include <asm/ppc_asm.h>
31 #include <asm/asm-offsets.h>
32 #include <asm/unistd.h>
34 #undef SHOW_SYSCALLS
35 #undef SHOW_SYSCALLS_TASK
37 /*
38 * MSR_KERNEL is > 0x10000 on 4xx/Book-E since it include MSR_CE.
39 */
40 #if MSR_KERNEL >= 0x10000
41 #define LOAD_MSR_KERNEL(r, x) lis r,(x)@h; ori r,r,(x)@l
42 #else
43 #define LOAD_MSR_KERNEL(r, x) li r,(x)
44 #endif
46 #ifdef CONFIG_BOOKE
47 #include "head_booke.h"
48 #define TRANSFER_TO_HANDLER_EXC_LEVEL(exc_level) \
49 mtspr exc_level##_SPRG,r8; \
50 BOOKE_LOAD_EXC_LEVEL_STACK(exc_level); \
51 lwz r0,GPR10-INT_FRAME_SIZE(r8); \
52 stw r0,GPR10(r11); \
53 lwz r0,GPR11-INT_FRAME_SIZE(r8); \
54 stw r0,GPR11(r11); \
55 mfspr r8,exc_level##_SPRG
57 .globl mcheck_transfer_to_handler
58 mcheck_transfer_to_handler:
59 TRANSFER_TO_HANDLER_EXC_LEVEL(MCHECK)
60 b transfer_to_handler_full
62 .globl debug_transfer_to_handler
63 debug_transfer_to_handler:
64 TRANSFER_TO_HANDLER_EXC_LEVEL(DEBUG)
65 b transfer_to_handler_full
67 .globl crit_transfer_to_handler
68 crit_transfer_to_handler:
69 TRANSFER_TO_HANDLER_EXC_LEVEL(CRIT)
70 /* fall through */
71 #endif
73 #ifdef CONFIG_40x
74 .globl crit_transfer_to_handler
75 crit_transfer_to_handler:
76 lwz r0,crit_r10@l(0)
77 stw r0,GPR10(r11)
78 lwz r0,crit_r11@l(0)
79 stw r0,GPR11(r11)
80 /* fall through */
81 #endif
83 /*
84 * This code finishes saving the registers to the exception frame
85 * and jumps to the appropriate handler for the exception, turning
86 * on address translation.
87 * Note that we rely on the caller having set cr0.eq iff the exception
88 * occurred in kernel mode (i.e. MSR:PR = 0).
89 */
90 .globl transfer_to_handler_full
91 transfer_to_handler_full:
92 SAVE_NVGPRS(r11)
93 /* fall through */
95 .globl transfer_to_handler
96 transfer_to_handler:
97 stw r2,GPR2(r11)
98 stw r12,_NIP(r11)
99 stw r9,_MSR(r11)
100 andi. r2,r9,MSR_PR
101 mfctr r12
102 mfspr r2,SPRN_XER
103 stw r12,_CTR(r11)
104 stw r2,_XER(r11)
105 mfspr r12,SPRN_SPRG3
106 addi r2,r12,-THREAD
107 tovirt(r2,r2) /* set r2 to current */
108 beq 2f /* if from user, fix up THREAD.regs */
109 addi r11,r1,STACK_FRAME_OVERHEAD
110 stw r11,PT_REGS(r12)
111 #if defined(CONFIG_40x) || defined(CONFIG_BOOKE)
112 /* Check to see if the dbcr0 register is set up to debug. Use the
113 single-step bit to do this. */
114 lwz r12,THREAD_DBCR0(r12)
115 andis. r12,r12,DBCR0_IC@h
116 beq+ 3f
117 /* From user and task is ptraced - load up global dbcr0 */
118 li r12,-1 /* clear all pending debug events */
119 mtspr SPRN_DBSR,r12
120 lis r11,global_dbcr0@ha
121 tophys(r11,r11)
122 addi r11,r11,global_dbcr0@l
123 lwz r12,0(r11)
124 mtspr SPRN_DBCR0,r12
125 lwz r12,4(r11)
126 addi r12,r12,-1
127 stw r12,4(r11)
128 #endif
129 b 3f
131 2: /* if from kernel, check interrupted DOZE/NAP mode and
132 * check for stack overflow
133 */
134 lwz r9,THREAD_INFO-THREAD(r12)
135 cmplw r1,r9 /* if r1 <= current->thread_info */
136 ble- stack_ovf /* then the kernel stack overflowed */
137 5:
138 #ifdef CONFIG_6xx
139 tophys(r9,r9) /* check local flags */
140 lwz r12,TI_LOCAL_FLAGS(r9)
141 mtcrf 0x01,r12
142 bt- 31-TLF_NAPPING,4f
143 #endif /* CONFIG_6xx */
144 .globl transfer_to_handler_cont
145 transfer_to_handler_cont:
146 3:
147 mflr r9
148 lwz r11,0(r9) /* virtual address of handler */
149 lwz r9,4(r9) /* where to go when done */
150 mtspr SPRN_SRR0,r11
151 mtspr SPRN_SRR1,r10
152 mtlr r9
153 SYNC
154 RFI /* jump to handler, enable MMU */
156 #ifdef CONFIG_6xx
157 4: rlwinm r12,r12,0,~_TLF_NAPPING
158 stw r12,TI_LOCAL_FLAGS(r9)
159 b power_save_6xx_restore
160 #endif
162 /*
163 * On kernel stack overflow, load up an initial stack pointer
164 * and call StackOverflow(regs), which should not return.
165 */
166 stack_ovf:
167 /* sometimes we use a statically-allocated stack, which is OK. */
168 lis r12,_end@h
169 ori r12,r12,_end@l
170 cmplw r1,r12
171 ble 5b /* r1 <= &_end is OK */
172 SAVE_NVGPRS(r11)
173 addi r3,r1,STACK_FRAME_OVERHEAD
174 lis r1,init_thread_union@ha
175 addi r1,r1,init_thread_union@l
176 addi r1,r1,THREAD_SIZE-STACK_FRAME_OVERHEAD
177 lis r9,StackOverflow@ha
178 addi r9,r9,StackOverflow@l
179 LOAD_MSR_KERNEL(r10,MSR_KERNEL)
180 FIX_SRR1(r10,r12)
181 mtspr SPRN_SRR0,r9
182 mtspr SPRN_SRR1,r10
183 SYNC
184 RFI
186 /*
187 * Handle a system call.
188 */
189 .stabs "arch/ppc/kernel/",N_SO,0,0,0f
190 .stabs "entry.S",N_SO,0,0,0f
191 0:
193 _GLOBAL(DoSyscall)
194 stw r3,ORIG_GPR3(r1)
195 li r12,0
196 stw r12,RESULT(r1)
197 lwz r11,_CCR(r1) /* Clear SO bit in CR */
198 rlwinm r11,r11,0,4,2
199 stw r11,_CCR(r1)
200 #ifdef SHOW_SYSCALLS
201 bl do_show_syscall
202 #endif /* SHOW_SYSCALLS */
203 rlwinm r10,r1,0,0,18 /* current_thread_info() */
204 lwz r11,TI_FLAGS(r10)
205 andi. r11,r11,_TIF_SYSCALL_T_OR_A
206 bne- syscall_dotrace
207 syscall_dotrace_cont:
208 cmplwi 0,r0,NR_syscalls
209 lis r10,sys_call_table@h
210 ori r10,r10,sys_call_table@l
211 slwi r0,r0,2
212 bge- 66f
213 lwzx r10,r10,r0 /* Fetch system call handler [ptr] */
214 mtlr r10
215 addi r9,r1,STACK_FRAME_OVERHEAD
216 PPC440EP_ERR42
217 blrl /* Call handler */
218 .globl ret_from_syscall
219 ret_from_syscall:
220 #ifdef SHOW_SYSCALLS
221 bl do_show_syscall_exit
222 #endif
223 mr r6,r3
224 rlwinm r12,r1,0,0,18 /* current_thread_info() */
225 /* disable interrupts so current_thread_info()->flags can't change */
226 LOAD_MSR_KERNEL(r10,MSR_KERNEL) /* doesn't include MSR_EE */
227 SYNC
228 MTMSRD(r10)
229 lwz r9,TI_FLAGS(r12)
230 li r8,-_LAST_ERRNO
231 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
232 bne- syscall_exit_work
233 cmplw 0,r3,r8
234 blt+ syscall_exit_cont
235 lwz r11,_CCR(r1) /* Load CR */
236 neg r3,r3
237 oris r11,r11,0x1000 /* Set SO bit in CR */
238 stw r11,_CCR(r1)
239 syscall_exit_cont:
240 #if defined(CONFIG_4xx) || defined(CONFIG_BOOKE)
241 /* If the process has its own DBCR0 value, load it up. The single
242 step bit tells us that dbcr0 should be loaded. */
243 lwz r0,THREAD+THREAD_DBCR0(r2)
244 andis. r10,r0,DBCR0_IC@h
245 bnel- load_dbcr0
246 #endif
247 #ifdef CONFIG_44x
248 lis r4,icache_44x_need_flush@ha
249 lwz r5,icache_44x_need_flush@l(r4)
250 cmplwi cr0,r5,0
251 bne- 2f
252 1:
253 #endif /* CONFIG_44x */
254 BEGIN_FTR_SECTION
255 lwarx r7,0,r1
256 END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX)
257 stwcx. r0,0,r1 /* to clear the reservation */
258 lwz r4,_LINK(r1)
259 lwz r5,_CCR(r1)
260 mtlr r4
261 mtcr r5
262 lwz r7,_NIP(r1)
263 lwz r8,_MSR(r1)
264 FIX_SRR1(r8, r0)
265 lwz r2,GPR2(r1)
266 lwz r1,GPR1(r1)
267 mtspr SPRN_SRR0,r7
268 mtspr SPRN_SRR1,r8
269 SYNC
270 RFI
271 #ifdef CONFIG_44x
272 2: li r7,0
273 iccci r0,r0
274 stw r7,icache_44x_need_flush@l(r4)
275 b 1b
276 #endif /* CONFIG_44x */
278 66: li r3,-ENOSYS
279 b ret_from_syscall
281 .globl ret_from_fork
282 ret_from_fork:
283 REST_NVGPRS(r1)
284 bl schedule_tail
285 li r3,0
286 b ret_from_syscall
288 /* Traced system call support */
289 syscall_dotrace:
290 SAVE_NVGPRS(r1)
291 li r0,0xc00
292 stw r0,TRAP(r1)
293 addi r3,r1,STACK_FRAME_OVERHEAD
294 bl do_syscall_trace_enter
295 lwz r0,GPR0(r1) /* Restore original registers */
296 lwz r3,GPR3(r1)
297 lwz r4,GPR4(r1)
298 lwz r5,GPR5(r1)
299 lwz r6,GPR6(r1)
300 lwz r7,GPR7(r1)
301 lwz r8,GPR8(r1)
302 REST_NVGPRS(r1)
303 b syscall_dotrace_cont
305 syscall_exit_work:
306 andi. r0,r9,_TIF_RESTOREALL
307 beq+ 0f
308 REST_NVGPRS(r1)
309 b 2f
310 0: cmplw 0,r3,r8
311 blt+ 1f
312 andi. r0,r9,_TIF_NOERROR
313 bne- 1f
314 lwz r11,_CCR(r1) /* Load CR */
315 neg r3,r3
316 oris r11,r11,0x1000 /* Set SO bit in CR */
317 stw r11,_CCR(r1)
319 1: stw r6,RESULT(r1) /* Save result */
320 stw r3,GPR3(r1) /* Update return value */
321 2: andi. r0,r9,(_TIF_PERSYSCALL_MASK)
322 beq 4f
324 /* Clear per-syscall TIF flags if any are set. */
326 li r11,_TIF_PERSYSCALL_MASK
327 addi r12,r12,TI_FLAGS
328 3: lwarx r8,0,r12
329 andc r8,r8,r11
330 #ifdef CONFIG_IBM405_ERR77
331 dcbt 0,r12
332 #endif
333 stwcx. r8,0,r12
334 bne- 3b
335 subi r12,r12,TI_FLAGS
337 4: /* Anything which requires enabling interrupts? */
338 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP)
339 beq ret_from_except
341 /* Re-enable interrupts */
342 ori r10,r10,MSR_EE
343 SYNC
344 MTMSRD(r10)
346 /* Save NVGPRS if they're not saved already */
347 lwz r4,TRAP(r1)
348 andi. r4,r4,1
349 beq 5f
350 SAVE_NVGPRS(r1)
351 li r4,0xc00
352 stw r4,TRAP(r1)
353 5:
354 addi r3,r1,STACK_FRAME_OVERHEAD
355 bl do_syscall_trace_leave
356 b ret_from_except_full
358 #ifdef SHOW_SYSCALLS
359 do_show_syscall:
360 #ifdef SHOW_SYSCALLS_TASK
361 lis r11,show_syscalls_task@ha
362 lwz r11,show_syscalls_task@l(r11)
363 cmp 0,r2,r11
364 bnelr
365 #endif
366 stw r31,GPR31(r1)
367 mflr r31
368 lis r3,7f@ha
369 addi r3,r3,7f@l
370 lwz r4,GPR0(r1)
371 lwz r5,GPR3(r1)
372 lwz r6,GPR4(r1)
373 lwz r7,GPR5(r1)
374 lwz r8,GPR6(r1)
375 lwz r9,GPR7(r1)
376 bl printk
377 lis r3,77f@ha
378 addi r3,r3,77f@l
379 lwz r4,GPR8(r1)
380 mr r5,r2
381 bl printk
382 lwz r0,GPR0(r1)
383 lwz r3,GPR3(r1)
384 lwz r4,GPR4(r1)
385 lwz r5,GPR5(r1)
386 lwz r6,GPR6(r1)
387 lwz r7,GPR7(r1)
388 lwz r8,GPR8(r1)
389 mtlr r31
390 lwz r31,GPR31(r1)
391 blr
393 do_show_syscall_exit:
394 #ifdef SHOW_SYSCALLS_TASK
395 lis r11,show_syscalls_task@ha
396 lwz r11,show_syscalls_task@l(r11)
397 cmp 0,r2,r11
398 bnelr
399 #endif
400 stw r31,GPR31(r1)
401 mflr r31
402 stw r3,RESULT(r1) /* Save result */
403 mr r4,r3
404 lis r3,79f@ha
405 addi r3,r3,79f@l
406 bl printk
407 lwz r3,RESULT(r1)
408 mtlr r31
409 lwz r31,GPR31(r1)
410 blr
412 7: .string "syscall %d(%x, %x, %x, %x, %x, "
413 77: .string "%x), current=%p\n"
414 79: .string " -> %x\n"
415 .align 2,0
417 #ifdef SHOW_SYSCALLS_TASK
418 .data
419 .globl show_syscalls_task
420 show_syscalls_task:
421 .long -1
422 .text
423 #endif
424 #endif /* SHOW_SYSCALLS */
426 /*
427 * The fork/clone functions need to copy the full register set into
428 * the child process. Therefore we need to save all the nonvolatile
429 * registers (r13 - r31) before calling the C code.
430 */
431 .globl ppc_fork
432 ppc_fork:
433 SAVE_NVGPRS(r1)
434 lwz r0,TRAP(r1)
435 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
436 stw r0,TRAP(r1) /* register set saved */
437 b sys_fork
439 .globl ppc_vfork
440 ppc_vfork:
441 SAVE_NVGPRS(r1)
442 lwz r0,TRAP(r1)
443 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
444 stw r0,TRAP(r1) /* register set saved */
445 b sys_vfork
447 .globl ppc_clone
448 ppc_clone:
449 SAVE_NVGPRS(r1)
450 lwz r0,TRAP(r1)
451 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
452 stw r0,TRAP(r1) /* register set saved */
453 b sys_clone
455 .globl ppc_swapcontext
456 ppc_swapcontext:
457 SAVE_NVGPRS(r1)
458 lwz r0,TRAP(r1)
459 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
460 stw r0,TRAP(r1) /* register set saved */
461 b sys_swapcontext
463 /*
464 * Top-level page fault handling.
465 * This is in assembler because if do_page_fault tells us that
466 * it is a bad kernel page fault, we want to save the non-volatile
467 * registers before calling bad_page_fault.
468 */
469 .globl handle_page_fault
470 handle_page_fault:
471 stw r4,_DAR(r1)
472 addi r3,r1,STACK_FRAME_OVERHEAD
473 bl do_page_fault
474 cmpwi r3,0
475 beq+ ret_from_except
476 SAVE_NVGPRS(r1)
477 lwz r0,TRAP(r1)
478 clrrwi r0,r0,1
479 stw r0,TRAP(r1)
480 mr r5,r3
481 addi r3,r1,STACK_FRAME_OVERHEAD
482 lwz r4,_DAR(r1)
483 bl bad_page_fault
484 b ret_from_except_full
486 /*
487 * This routine switches between two different tasks. The process
488 * state of one is saved on its kernel stack. Then the state
489 * of the other is restored from its kernel stack. The memory
490 * management hardware is updated to the second process's state.
491 * Finally, we can return to the second process.
492 * On entry, r3 points to the THREAD for the current task, r4
493 * points to the THREAD for the new task.
494 *
495 * This routine is always called with interrupts disabled.
496 *
497 * Note: there are two ways to get to the "going out" portion
498 * of this code; either by coming in via the entry (_switch)
499 * or via "fork" which must set up an environment equivalent
500 * to the "_switch" path. If you change this , you'll have to
501 * change the fork code also.
502 *
503 * The code which creates the new task context is in 'copy_thread'
504 * in arch/ppc/kernel/process.c
505 */
506 _GLOBAL(_switch)
507 stwu r1,-INT_FRAME_SIZE(r1)
508 mflr r0
509 stw r0,INT_FRAME_SIZE+4(r1)
510 /* r3-r12 are caller saved -- Cort */
511 SAVE_NVGPRS(r1)
512 stw r0,_NIP(r1) /* Return to switch caller */
513 mfmsr r11
514 li r0,MSR_FP /* Disable floating-point */
515 #ifdef CONFIG_ALTIVEC
516 BEGIN_FTR_SECTION
517 oris r0,r0,MSR_VEC@h /* Disable altivec */
518 mfspr r12,SPRN_VRSAVE /* save vrsave register value */
519 stw r12,THREAD+THREAD_VRSAVE(r2)
520 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
521 #endif /* CONFIG_ALTIVEC */
522 #ifdef CONFIG_SPE
523 oris r0,r0,MSR_SPE@h /* Disable SPE */
524 mfspr r12,SPRN_SPEFSCR /* save spefscr register value */
525 stw r12,THREAD+THREAD_SPEFSCR(r2)
526 #endif /* CONFIG_SPE */
527 and. r0,r0,r11 /* FP or altivec or SPE enabled? */
528 beq+ 1f
529 andc r11,r11,r0
530 MTMSRD(r11)
531 isync
532 1: stw r11,_MSR(r1)
533 mfcr r10
534 stw r10,_CCR(r1)
535 stw r1,KSP(r3) /* Set old stack pointer */
537 #ifdef CONFIG_SMP
538 /* We need a sync somewhere here to make sure that if the
539 * previous task gets rescheduled on another CPU, it sees all
540 * stores it has performed on this one.
541 */
542 sync
543 #endif /* CONFIG_SMP */
545 tophys(r0,r4)
546 CLR_TOP32(r0)
547 mtspr SPRN_SPRG3,r0 /* Update current THREAD phys addr */
548 lwz r1,KSP(r4) /* Load new stack pointer */
550 /* save the old current 'last' for return value */
551 mr r3,r2
552 addi r2,r4,-THREAD /* Update current */
554 #ifdef CONFIG_ALTIVEC
555 BEGIN_FTR_SECTION
556 lwz r0,THREAD+THREAD_VRSAVE(r2)
557 mtspr SPRN_VRSAVE,r0 /* if G4, restore VRSAVE reg */
558 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
559 #endif /* CONFIG_ALTIVEC */
560 #ifdef CONFIG_SPE
561 lwz r0,THREAD+THREAD_SPEFSCR(r2)
562 mtspr SPRN_SPEFSCR,r0 /* restore SPEFSCR reg */
563 #endif /* CONFIG_SPE */
565 lwz r0,_CCR(r1)
566 mtcrf 0xFF,r0
567 /* r3-r12 are destroyed -- Cort */
568 REST_NVGPRS(r1)
570 lwz r4,_NIP(r1) /* Return to _switch caller in new task */
571 mtlr r4
572 addi r1,r1,INT_FRAME_SIZE
573 blr
575 .globl fast_exception_return
576 fast_exception_return:
577 #if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
578 andi. r10,r9,MSR_RI /* check for recoverable interrupt */
579 beq 1f /* if not, we've got problems */
580 #endif
582 2: REST_4GPRS(3, r11)
583 lwz r10,_CCR(r11)
584 REST_GPR(1, r11)
585 mtcr r10
586 lwz r10,_LINK(r11)
587 mtlr r10
588 REST_GPR(10, r11)
589 mtspr SPRN_SRR1,r9
590 mtspr SPRN_SRR0,r12
591 REST_GPR(9, r11)
592 REST_GPR(12, r11)
593 lwz r11,GPR11(r11)
594 SYNC
595 RFI
597 #if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
598 /* check if the exception happened in a restartable section */
599 1: lis r3,exc_exit_restart_end@ha
600 addi r3,r3,exc_exit_restart_end@l
601 cmplw r12,r3
602 bge 3f
603 lis r4,exc_exit_restart@ha
604 addi r4,r4,exc_exit_restart@l
605 cmplw r12,r4
606 blt 3f
607 lis r3,fee_restarts@ha
608 tophys(r3,r3)
609 lwz r5,fee_restarts@l(r3)
610 addi r5,r5,1
611 stw r5,fee_restarts@l(r3)
612 mr r12,r4 /* restart at exc_exit_restart */
613 b 2b
615 .section .bss
616 .align 2
617 fee_restarts:
618 .space 4
619 .previous
621 /* aargh, a nonrecoverable interrupt, panic */
622 /* aargh, we don't know which trap this is */
623 /* but the 601 doesn't implement the RI bit, so assume it's OK */
624 3:
625 BEGIN_FTR_SECTION
626 b 2b
627 END_FTR_SECTION_IFSET(CPU_FTR_601)
628 li r10,-1
629 stw r10,TRAP(r11)
630 addi r3,r1,STACK_FRAME_OVERHEAD
631 lis r10,MSR_KERNEL@h
632 ori r10,r10,MSR_KERNEL@l
633 bl transfer_to_handler_full
634 .long nonrecoverable_exception
635 .long ret_from_except
636 #endif
638 .globl ret_from_except_full
639 ret_from_except_full:
640 REST_NVGPRS(r1)
641 /* fall through */
643 .globl ret_from_except
644 ret_from_except:
645 /* Hard-disable interrupts so that current_thread_info()->flags
646 * can't change between when we test it and when we return
647 * from the interrupt. */
648 LOAD_MSR_KERNEL(r10,MSR_KERNEL)
649 SYNC /* Some chip revs have problems here... */
650 MTMSRD(r10) /* disable interrupts */
652 lwz r3,_MSR(r1) /* Returning to user mode? */
653 andi. r0,r3,MSR_PR
654 beq resume_kernel
656 user_exc_return: /* r10 contains MSR_KERNEL here */
657 /* Check current_thread_info()->flags */
658 rlwinm r9,r1,0,0,18
659 lwz r9,TI_FLAGS(r9)
660 andi. r0,r9,(_TIF_SIGPENDING|_TIF_RESTORE_SIGMASK|_TIF_NEED_RESCHED)
661 bne do_work
663 restore_user:
664 #if defined(CONFIG_4xx) || defined(CONFIG_BOOKE)
665 /* Check whether this process has its own DBCR0 value. The single
666 step bit tells us that dbcr0 should be loaded. */
667 lwz r0,THREAD+THREAD_DBCR0(r2)
668 andis. r10,r0,DBCR0_IC@h
669 bnel- load_dbcr0
670 #endif
672 #ifdef CONFIG_PREEMPT
673 b restore
675 /* N.B. the only way to get here is from the beq following ret_from_except. */
676 resume_kernel:
677 /* check current_thread_info->preempt_count */
678 rlwinm r9,r1,0,0,18
679 lwz r0,TI_PREEMPT(r9)
680 cmpwi 0,r0,0 /* if non-zero, just restore regs and return */
681 bne restore
682 lwz r0,TI_FLAGS(r9)
683 andi. r0,r0,_TIF_NEED_RESCHED
684 beq+ restore
685 andi. r0,r3,MSR_EE /* interrupts off? */
686 beq restore /* don't schedule if so */
687 1: bl preempt_schedule_irq
688 rlwinm r9,r1,0,0,18
689 lwz r3,TI_FLAGS(r9)
690 andi. r0,r3,_TIF_NEED_RESCHED
691 bne- 1b
692 #else
693 resume_kernel:
694 #endif /* CONFIG_PREEMPT */
696 /* interrupts are hard-disabled at this point */
697 restore:
698 #ifdef CONFIG_44x
699 lis r4,icache_44x_need_flush@ha
700 lwz r5,icache_44x_need_flush@l(r4)
701 cmplwi cr0,r5,0
702 beq+ 1f
703 li r6,0
704 iccci r0,r0
705 stw r6,icache_44x_need_flush@l(r4)
706 1:
707 #endif /* CONFIG_44x */
708 lwz r0,GPR0(r1)
709 lwz r2,GPR2(r1)
710 REST_4GPRS(3, r1)
711 REST_2GPRS(7, r1)
713 lwz r10,_XER(r1)
714 lwz r11,_CTR(r1)
715 mtspr SPRN_XER,r10
716 mtctr r11
718 PPC405_ERR77(0,r1)
719 BEGIN_FTR_SECTION
720 lwarx r11,0,r1
721 END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX)
722 stwcx. r0,0,r1 /* to clear the reservation */
724 #if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
725 lwz r9,_MSR(r1)
726 andi. r10,r9,MSR_RI /* check if this exception occurred */
727 beql nonrecoverable /* at a bad place (MSR:RI = 0) */
729 lwz r10,_CCR(r1)
730 lwz r11,_LINK(r1)
731 mtcrf 0xFF,r10
732 mtlr r11
734 /*
735 * Once we put values in SRR0 and SRR1, we are in a state
736 * where exceptions are not recoverable, since taking an
737 * exception will trash SRR0 and SRR1. Therefore we clear the
738 * MSR:RI bit to indicate this. If we do take an exception,
739 * we can't return to the point of the exception but we
740 * can restart the exception exit path at the label
741 * exc_exit_restart below. -- paulus
742 */
743 LOAD_MSR_KERNEL(r10,MSR_KERNEL & ~MSR_RI)
744 SYNC
745 MTMSRD(r10) /* clear the RI bit */
746 .globl exc_exit_restart
747 exc_exit_restart:
748 lwz r9,_MSR(r1)
749 lwz r12,_NIP(r1)
750 FIX_SRR1(r9,r10)
751 mtspr SPRN_SRR0,r12
752 mtspr SPRN_SRR1,r9
753 REST_4GPRS(9, r1)
754 lwz r1,GPR1(r1)
755 .globl exc_exit_restart_end
756 exc_exit_restart_end:
757 SYNC
758 RFI
760 #else /* !(CONFIG_4xx || CONFIG_BOOKE) */
761 /*
762 * This is a bit different on 4xx/Book-E because it doesn't have
763 * the RI bit in the MSR.
764 * The TLB miss handler checks if we have interrupted
765 * the exception exit path and restarts it if so
766 * (well maybe one day it will... :).
767 */
768 lwz r11,_LINK(r1)
769 mtlr r11
770 lwz r10,_CCR(r1)
771 mtcrf 0xff,r10
772 REST_2GPRS(9, r1)
773 .globl exc_exit_restart
774 exc_exit_restart:
775 lwz r11,_NIP(r1)
776 lwz r12,_MSR(r1)
777 exc_exit_start:
778 mtspr SPRN_SRR0,r11
779 mtspr SPRN_SRR1,r12
780 REST_2GPRS(11, r1)
781 lwz r1,GPR1(r1)
782 .globl exc_exit_restart_end
783 exc_exit_restart_end:
784 PPC405_ERR77_SYNC
785 rfi
786 b . /* prevent prefetch past rfi */
788 /*
789 * Returning from a critical interrupt in user mode doesn't need
790 * to be any different from a normal exception. For a critical
791 * interrupt in the kernel, we just return (without checking for
792 * preemption) since the interrupt may have happened at some crucial
793 * place (e.g. inside the TLB miss handler), and because we will be
794 * running with r1 pointing into critical_stack, not the current
795 * process's kernel stack (and therefore current_thread_info() will
796 * give the wrong answer).
797 * We have to restore various SPRs that may have been in use at the
798 * time of the critical interrupt.
799 *
800 */
801 #ifdef CONFIG_40x
802 #define PPC_40x_TURN_OFF_MSR_DR \
803 /* avoid any possible TLB misses here by turning off MSR.DR, we \
804 * assume the instructions here are mapped by a pinned TLB entry */ \
805 li r10,MSR_IR; \
806 mtmsr r10; \
807 isync; \
808 tophys(r1, r1);
809 #else
810 #define PPC_40x_TURN_OFF_MSR_DR
811 #endif
813 #define RET_FROM_EXC_LEVEL(exc_lvl_srr0, exc_lvl_srr1, exc_lvl_rfi) \
814 REST_NVGPRS(r1); \
815 lwz r3,_MSR(r1); \
816 andi. r3,r3,MSR_PR; \
817 LOAD_MSR_KERNEL(r10,MSR_KERNEL); \
818 bne user_exc_return; \
819 lwz r0,GPR0(r1); \
820 lwz r2,GPR2(r1); \
821 REST_4GPRS(3, r1); \
822 REST_2GPRS(7, r1); \
823 lwz r10,_XER(r1); \
824 lwz r11,_CTR(r1); \
825 mtspr SPRN_XER,r10; \
826 mtctr r11; \
827 PPC405_ERR77(0,r1); \
828 stwcx. r0,0,r1; /* to clear the reservation */ \
829 lwz r11,_LINK(r1); \
830 mtlr r11; \
831 lwz r10,_CCR(r1); \
832 mtcrf 0xff,r10; \
833 PPC_40x_TURN_OFF_MSR_DR; \
834 lwz r9,_DEAR(r1); \
835 lwz r10,_ESR(r1); \
836 mtspr SPRN_DEAR,r9; \
837 mtspr SPRN_ESR,r10; \
838 lwz r11,_NIP(r1); \
839 lwz r12,_MSR(r1); \
840 mtspr exc_lvl_srr0,r11; \
841 mtspr exc_lvl_srr1,r12; \
842 lwz r9,GPR9(r1); \
843 lwz r12,GPR12(r1); \
844 lwz r10,GPR10(r1); \
845 lwz r11,GPR11(r1); \
846 lwz r1,GPR1(r1); \
847 PPC405_ERR77_SYNC; \
848 exc_lvl_rfi; \
849 b .; /* prevent prefetch past exc_lvl_rfi */
851 .globl ret_from_crit_exc
852 ret_from_crit_exc:
853 RET_FROM_EXC_LEVEL(SPRN_CSRR0, SPRN_CSRR1, RFCI)
855 #ifdef CONFIG_BOOKE
856 .globl ret_from_debug_exc
857 ret_from_debug_exc:
858 RET_FROM_EXC_LEVEL(SPRN_DSRR0, SPRN_DSRR1, RFDI)
860 .globl ret_from_mcheck_exc
861 ret_from_mcheck_exc:
862 RET_FROM_EXC_LEVEL(SPRN_MCSRR0, SPRN_MCSRR1, RFMCI)
863 #endif /* CONFIG_BOOKE */
865 /*
866 * Load the DBCR0 value for a task that is being ptraced,
867 * having first saved away the global DBCR0. Note that r0
868 * has the dbcr0 value to set upon entry to this.
869 */
870 load_dbcr0:
871 mfmsr r10 /* first disable debug exceptions */
872 rlwinm r10,r10,0,~MSR_DE
873 mtmsr r10
874 isync
875 mfspr r10,SPRN_DBCR0
876 lis r11,global_dbcr0@ha
877 addi r11,r11,global_dbcr0@l
878 stw r10,0(r11)
879 mtspr SPRN_DBCR0,r0
880 lwz r10,4(r11)
881 addi r10,r10,1
882 stw r10,4(r11)
883 li r11,-1
884 mtspr SPRN_DBSR,r11 /* clear all pending debug events */
885 blr
887 .section .bss
888 .align 4
889 global_dbcr0:
890 .space 8
891 .previous
892 #endif /* !(CONFIG_4xx || CONFIG_BOOKE) */
894 do_work: /* r10 contains MSR_KERNEL here */
895 andi. r0,r9,_TIF_NEED_RESCHED
896 beq do_user_signal
898 do_resched: /* r10 contains MSR_KERNEL here */
899 ori r10,r10,MSR_EE
900 SYNC
901 MTMSRD(r10) /* hard-enable interrupts */
902 bl schedule
903 recheck:
904 LOAD_MSR_KERNEL(r10,MSR_KERNEL)
905 SYNC
906 MTMSRD(r10) /* disable interrupts */
907 rlwinm r9,r1,0,0,18
908 lwz r9,TI_FLAGS(r9)
909 andi. r0,r9,_TIF_NEED_RESCHED
910 bne- do_resched
911 andi. r0,r9,_TIF_SIGPENDING
912 beq restore_user
913 do_user_signal: /* r10 contains MSR_KERNEL here */
914 ori r10,r10,MSR_EE
915 SYNC
916 MTMSRD(r10) /* hard-enable interrupts */
917 /* save r13-r31 in the exception frame, if not already done */
918 lwz r3,TRAP(r1)
919 andi. r0,r3,1
920 beq 2f
921 SAVE_NVGPRS(r1)
922 rlwinm r3,r3,0,0,30
923 stw r3,TRAP(r1)
924 2: li r3,0
925 addi r4,r1,STACK_FRAME_OVERHEAD
926 bl do_signal
927 REST_NVGPRS(r1)
928 b recheck
930 /*
931 * We come here when we are at the end of handling an exception
932 * that occurred at a place where taking an exception will lose
933 * state information, such as the contents of SRR0 and SRR1.
934 */
935 nonrecoverable:
936 lis r10,exc_exit_restart_end@ha
937 addi r10,r10,exc_exit_restart_end@l
938 cmplw r12,r10
939 bge 3f
940 lis r11,exc_exit_restart@ha
941 addi r11,r11,exc_exit_restart@l
942 cmplw r12,r11
943 blt 3f
944 lis r10,ee_restarts@ha
945 lwz r12,ee_restarts@l(r10)
946 addi r12,r12,1
947 stw r12,ee_restarts@l(r10)
948 mr r12,r11 /* restart at exc_exit_restart */
949 blr
950 3: /* OK, we can't recover, kill this process */
951 /* but the 601 doesn't implement the RI bit, so assume it's OK */
952 BEGIN_FTR_SECTION
953 blr
954 END_FTR_SECTION_IFSET(CPU_FTR_601)
955 lwz r3,TRAP(r1)
956 andi. r0,r3,1
957 beq 4f
958 SAVE_NVGPRS(r1)
959 rlwinm r3,r3,0,0,30
960 stw r3,TRAP(r1)
961 4: addi r3,r1,STACK_FRAME_OVERHEAD
962 bl nonrecoverable_exception
963 /* shouldn't return */
964 b 4b
966 .section .bss
967 .align 2
968 ee_restarts:
969 .space 4
970 .previous