| blob | 2cb5109a7ea3d3d4959fa0d9360248218b56d688 |
1 /*
2 * PowerPC version
3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 * Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
5 * Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
6 * Adapted for Power Macintosh by Paul Mackerras.
7 * Low-level exception handlers and MMU support
8 * rewritten by Paul Mackerras.
9 * Copyright (C) 1996 Paul Mackerras.
10 * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
11 *
12 * This file contains the system call entry code, context switch
13 * code, and exception/interrupt return code for PowerPC.
14 *
15 * This program is free software; you can redistribute it and/or
16 * modify it under the terms of the GNU General Public License
17 * as published by the Free Software Foundation; either version
18 * 2 of the License, or (at your option) any later version.
19 */
21 #include <linux/errno.h>
22 #include <linux/err.h>
23 #include <asm/unistd.h>
24 #include <asm/processor.h>
25 #include <asm/page.h>
26 #include <asm/mmu.h>
27 #include <asm/thread_info.h>
28 #include <asm/ppc_asm.h>
29 #include <asm/asm-offsets.h>
30 #include <asm/cputable.h>
31 #include <asm/firmware.h>
32 #include <asm/bug.h>
33 #include <asm/ptrace.h>
34 #include <asm/irqflags.h>
35 #include <asm/hw_irq.h>
36 #include <asm/context_tracking.h>
37 #include <asm/tm.h>
38 #include <asm/ppc-opcode.h>
39 #include <asm/export.h>
40 #ifdef CONFIG_PPC_BOOK3S
41 #include <asm/exception-64s.h>
42 #else
43 #include <asm/exception-64e.h>
44 #endif
46 /*
47 * System calls.
48 */
49 .section ".toc","aw"
50 SYS_CALL_TABLE:
51 .tc sys_call_table[TC],sys_call_table
53 /* This value is used to mark exception frames on the stack. */
54 exception_marker:
55 .tc ID_EXC_MARKER[TC],STACK_FRAME_REGS_MARKER
57 .section ".text"
58 .align 7
60 .globl system_call_common
61 system_call_common:
62 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
63 BEGIN_FTR_SECTION
64 extrdi. r10, r12, 1, (63-MSR_TS_T_LG) /* transaction active? */
65 bne .Ltabort_syscall
66 END_FTR_SECTION_IFSET(CPU_FTR_TM)
67 #endif
68 andi. r10,r12,MSR_PR
69 mr r10,r1
70 addi r1,r1,-INT_FRAME_SIZE
71 beq- 1f
72 ld r1,PACAKSAVE(r13)
73 1: std r10,0(r1)
74 std r11,_NIP(r1)
75 std r12,_MSR(r1)
76 std r0,GPR0(r1)
77 std r10,GPR1(r1)
78 beq 2f /* if from kernel mode */
79 ACCOUNT_CPU_USER_ENTRY(r13, r10, r11)
80 2: std r2,GPR2(r1)
81 std r3,GPR3(r1)
82 mfcr r2
83 std r4,GPR4(r1)
84 std r5,GPR5(r1)
85 std r6,GPR6(r1)
86 std r7,GPR7(r1)
87 std r8,GPR8(r1)
88 li r11,0
89 std r11,GPR9(r1)
90 std r11,GPR10(r1)
91 std r11,GPR11(r1)
92 std r11,GPR12(r1)
93 std r11,_XER(r1)
94 std r11,_CTR(r1)
95 std r9,GPR13(r1)
96 mflr r10
97 /*
98 * This clears CR0.SO (bit 28), which is the error indication on
99 * return from this system call.
100 */
101 rldimi r2,r11,28,(63-28)
102 li r11,0xc01
103 std r10,_LINK(r1)
104 std r11,_TRAP(r1)
105 std r3,ORIG_GPR3(r1)
106 std r2,_CCR(r1)
107 ld r2,PACATOC(r13)
108 addi r9,r1,STACK_FRAME_OVERHEAD
109 ld r11,exception_marker@toc(r2)
110 std r11,-16(r9) /* "regshere" marker */
111 #if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) && defined(CONFIG_PPC_SPLPAR)
112 BEGIN_FW_FTR_SECTION
113 beq 33f
114 /* if from user, see if there are any DTL entries to process */
115 ld r10,PACALPPACAPTR(r13) /* get ptr to VPA */
116 ld r11,PACA_DTL_RIDX(r13) /* get log read index */
117 addi r10,r10,LPPACA_DTLIDX
118 LDX_BE r10,0,r10 /* get log write index */
119 cmpd cr1,r11,r10
120 beq+ cr1,33f
121 bl accumulate_stolen_time
122 REST_GPR(0,r1)
123 REST_4GPRS(3,r1)
124 REST_2GPRS(7,r1)
125 addi r9,r1,STACK_FRAME_OVERHEAD
126 33:
127 END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR)
128 #endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE && CONFIG_PPC_SPLPAR */
130 /*
131 * A syscall should always be called with interrupts enabled
132 * so we just unconditionally hard-enable here. When some kind
133 * of irq tracing is used, we additionally check that condition
134 * is correct
135 */
136 #if defined(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG) && defined(CONFIG_BUG)
137 lbz r10,PACAIRQSOFTMASK(r13)
138 1: tdnei r10,IRQS_ENABLED
139 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
140 #endif
142 #ifdef CONFIG_PPC_BOOK3E
143 wrteei 1
144 #else
145 li r11,MSR_RI
146 ori r11,r11,MSR_EE
147 mtmsrd r11,1
148 #endif /* CONFIG_PPC_BOOK3E */
150 system_call: /* label this so stack traces look sane */
151 /* We do need to set SOFTE in the stack frame or the return
152 * from interrupt will be painful
153 */
154 li r10,IRQS_ENABLED
155 std r10,SOFTE(r1)
157 CURRENT_THREAD_INFO(r11, r1)
158 ld r10,TI_FLAGS(r11)
159 andi. r11,r10,_TIF_SYSCALL_DOTRACE
160 bne .Lsyscall_dotrace /* does not return */
161 cmpldi 0,r0,NR_syscalls
162 bge- .Lsyscall_enosys
164 .Lsyscall:
165 /*
166 * Need to vector to 32 Bit or default sys_call_table here,
167 * based on caller's run-mode / personality.
168 */
169 ld r11,SYS_CALL_TABLE@toc(2)
170 andi. r10,r10,_TIF_32BIT
171 beq 15f
172 addi r11,r11,8 /* use 32-bit syscall entries */
173 clrldi r3,r3,32
174 clrldi r4,r4,32
175 clrldi r5,r5,32
176 clrldi r6,r6,32
177 clrldi r7,r7,32
178 clrldi r8,r8,32
179 15:
180 slwi r0,r0,4
181 ldx r12,r11,r0 /* Fetch system call handler [ptr] */
182 mtctr r12
183 bctrl /* Call handler */
185 .Lsyscall_exit:
186 std r3,RESULT(r1)
187 CURRENT_THREAD_INFO(r12, r1)
189 ld r8,_MSR(r1)
190 #ifdef CONFIG_PPC_BOOK3S
191 /* No MSR:RI on BookE */
192 andi. r10,r8,MSR_RI
193 beq- .Lunrecov_restore
194 #endif
196 /*
197 * This is a few instructions into the actual syscall exit path (which actually
198 * starts at .Lsyscall_exit) to cater to kprobe blacklisting and to reduce the
199 * number of visible symbols for profiling purposes.
200 *
201 * We can probe from system_call until this point as MSR_RI is set. But once it
202 * is cleared below, we won't be able to take a trap.
203 *
204 * This is blacklisted from kprobes further below with _ASM_NOKPROBE_SYMBOL().
205 */
206 system_call_exit:
207 /*
208 * Disable interrupts so current_thread_info()->flags can't change,
209 * and so that we don't get interrupted after loading SRR0/1.
210 */
211 #ifdef CONFIG_PPC_BOOK3E
212 wrteei 0
213 #else
214 /*
215 * For performance reasons we clear RI the same time that we
216 * clear EE. We only need to clear RI just before we restore r13
217 * below, but batching it with EE saves us one expensive mtmsrd call.
218 * We have to be careful to restore RI if we branch anywhere from
219 * here (eg syscall_exit_work).
220 */
221 li r11,0
222 mtmsrd r11,1
223 #endif /* CONFIG_PPC_BOOK3E */
225 ld r9,TI_FLAGS(r12)
226 li r11,-MAX_ERRNO
227 andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
228 bne- .Lsyscall_exit_work
230 andi. r0,r8,MSR_FP
231 beq 2f
232 #ifdef CONFIG_ALTIVEC
233 andis. r0,r8,MSR_VEC@h
234 bne 3f
235 #endif
236 2: addi r3,r1,STACK_FRAME_OVERHEAD
237 #ifdef CONFIG_PPC_BOOK3S
238 li r10,MSR_RI
239 mtmsrd r10,1 /* Restore RI */
240 #endif
241 bl restore_math
242 #ifdef CONFIG_PPC_BOOK3S
243 li r11,0
244 mtmsrd r11,1
245 #endif
246 ld r8,_MSR(r1)
247 ld r3,RESULT(r1)
248 li r11,-MAX_ERRNO
250 3: cmpld r3,r11
251 ld r5,_CCR(r1)
252 bge- .Lsyscall_error
253 .Lsyscall_error_cont:
254 ld r7,_NIP(r1)
255 BEGIN_FTR_SECTION
256 stdcx. r0,0,r1 /* to clear the reservation */
257 END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
258 andi. r6,r8,MSR_PR
259 ld r4,_LINK(r1)
261 beq- 1f
262 ACCOUNT_CPU_USER_EXIT(r13, r11, r12)
264 BEGIN_FTR_SECTION
265 HMT_MEDIUM_LOW
266 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
268 ld r13,GPR13(r1) /* only restore r13 if returning to usermode */
269 ld r2,GPR2(r1)
270 ld r1,GPR1(r1)
271 mtlr r4
272 mtcr r5
273 mtspr SPRN_SRR0,r7
274 mtspr SPRN_SRR1,r8
275 RFI_TO_USER
276 b . /* prevent speculative execution */
278 /* exit to kernel */
279 1: ld r2,GPR2(r1)
280 ld r1,GPR1(r1)
281 mtlr r4
282 mtcr r5
283 mtspr SPRN_SRR0,r7
284 mtspr SPRN_SRR1,r8
285 RFI_TO_KERNEL
286 b . /* prevent speculative execution */
288 .Lsyscall_error:
289 oris r5,r5,0x1000 /* Set SO bit in CR */
290 neg r3,r3
291 std r5,_CCR(r1)
292 b .Lsyscall_error_cont
294 /* Traced system call support */
295 .Lsyscall_dotrace:
296 bl save_nvgprs
297 addi r3,r1,STACK_FRAME_OVERHEAD
298 bl do_syscall_trace_enter
300 /*
301 * We use the return value of do_syscall_trace_enter() as the syscall
302 * number. If the syscall was rejected for any reason do_syscall_trace_enter()
303 * returns an invalid syscall number and the test below against
304 * NR_syscalls will fail.
305 */
306 mr r0,r3
308 /* Restore argument registers just clobbered and/or possibly changed. */
309 ld r3,GPR3(r1)
310 ld r4,GPR4(r1)
311 ld r5,GPR5(r1)
312 ld r6,GPR6(r1)
313 ld r7,GPR7(r1)
314 ld r8,GPR8(r1)
316 /* Repopulate r9 and r10 for the syscall path */
317 addi r9,r1,STACK_FRAME_OVERHEAD
318 CURRENT_THREAD_INFO(r10, r1)
319 ld r10,TI_FLAGS(r10)
321 cmpldi r0,NR_syscalls
322 blt+ .Lsyscall
324 /* Return code is already in r3 thanks to do_syscall_trace_enter() */
325 b .Lsyscall_exit
328 .Lsyscall_enosys:
329 li r3,-ENOSYS
330 b .Lsyscall_exit
332 .Lsyscall_exit_work:
333 #ifdef CONFIG_PPC_BOOK3S
334 li r10,MSR_RI
335 mtmsrd r10,1 /* Restore RI */
336 #endif
337 /* If TIF_RESTOREALL is set, don't scribble on either r3 or ccr.
338 If TIF_NOERROR is set, just save r3 as it is. */
340 andi. r0,r9,_TIF_RESTOREALL
341 beq+ 0f
342 REST_NVGPRS(r1)
343 b 2f
344 0: cmpld r3,r11 /* r11 is -MAX_ERRNO */
345 blt+ 1f
346 andi. r0,r9,_TIF_NOERROR
347 bne- 1f
348 ld r5,_CCR(r1)
349 neg r3,r3
350 oris r5,r5,0x1000 /* Set SO bit in CR */
351 std r5,_CCR(r1)
352 1: std r3,GPR3(r1)
353 2: andi. r0,r9,(_TIF_PERSYSCALL_MASK)
354 beq 4f
356 /* Clear per-syscall TIF flags if any are set. */
358 li r11,_TIF_PERSYSCALL_MASK
359 addi r12,r12,TI_FLAGS
360 3: ldarx r10,0,r12
361 andc r10,r10,r11
362 stdcx. r10,0,r12
363 bne- 3b
364 subi r12,r12,TI_FLAGS
366 4: /* Anything else left to do? */
367 BEGIN_FTR_SECTION
368 lis r3,INIT_PPR@highest /* Set thread.ppr = 3 */
369 ld r10,PACACURRENT(r13)
370 sldi r3,r3,32 /* bits 11-13 are used for ppr */
371 std r3,TASKTHREADPPR(r10)
372 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
374 andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP)
375 beq ret_from_except_lite
377 /* Re-enable interrupts */
378 #ifdef CONFIG_PPC_BOOK3E
379 wrteei 1
380 #else
381 li r10,MSR_RI
382 ori r10,r10,MSR_EE
383 mtmsrd r10,1
384 #endif /* CONFIG_PPC_BOOK3E */
386 bl save_nvgprs
387 addi r3,r1,STACK_FRAME_OVERHEAD
388 bl do_syscall_trace_leave
389 b ret_from_except
391 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
392 .Ltabort_syscall:
393 /* Firstly we need to enable TM in the kernel */
394 mfmsr r10
395 li r9, 1
396 rldimi r10, r9, MSR_TM_LG, 63-MSR_TM_LG
397 mtmsrd r10, 0
399 /* tabort, this dooms the transaction, nothing else */
400 li r9, (TM_CAUSE_SYSCALL|TM_CAUSE_PERSISTENT)
401 TABORT(R9)
403 /*
404 * Return directly to userspace. We have corrupted user register state,
405 * but userspace will never see that register state. Execution will
406 * resume after the tbegin of the aborted transaction with the
407 * checkpointed register state.
408 */
409 li r9, MSR_RI
410 andc r10, r10, r9
411 mtmsrd r10, 1
412 mtspr SPRN_SRR0, r11
413 mtspr SPRN_SRR1, r12
414 RFI_TO_USER
415 b . /* prevent speculative execution */
416 #endif
417 _ASM_NOKPROBE_SYMBOL(system_call_common);
418 _ASM_NOKPROBE_SYMBOL(system_call_exit);
420 /* Save non-volatile GPRs, if not already saved. */
421 _GLOBAL(save_nvgprs)
422 ld r11,_TRAP(r1)
423 andi. r0,r11,1
424 beqlr-
425 SAVE_NVGPRS(r1)
426 clrrdi r0,r11,1
427 std r0,_TRAP(r1)
428 blr
429 _ASM_NOKPROBE_SYMBOL(save_nvgprs);
432 /*
433 * The sigsuspend and rt_sigsuspend system calls can call do_signal
434 * and thus put the process into the stopped state where we might
435 * want to examine its user state with ptrace. Therefore we need
436 * to save all the nonvolatile registers (r14 - r31) before calling
437 * the C code. Similarly, fork, vfork and clone need the full
438 * register state on the stack so that it can be copied to the child.
439 */
441 _GLOBAL(ppc_fork)
442 bl save_nvgprs
443 bl sys_fork
444 b .Lsyscall_exit
446 _GLOBAL(ppc_vfork)
447 bl save_nvgprs
448 bl sys_vfork
449 b .Lsyscall_exit
451 _GLOBAL(ppc_clone)
452 bl save_nvgprs
453 bl sys_clone
454 b .Lsyscall_exit
456 _GLOBAL(ppc32_swapcontext)
457 bl save_nvgprs
458 bl compat_sys_swapcontext
459 b .Lsyscall_exit
461 _GLOBAL(ppc64_swapcontext)
462 bl save_nvgprs
463 bl sys_swapcontext
464 b .Lsyscall_exit
466 _GLOBAL(ppc_switch_endian)
467 bl save_nvgprs
468 bl sys_switch_endian
469 b .Lsyscall_exit
471 _GLOBAL(ret_from_fork)
472 bl schedule_tail
473 REST_NVGPRS(r1)
474 li r3,0
475 b .Lsyscall_exit
477 _GLOBAL(ret_from_kernel_thread)
478 bl schedule_tail
479 REST_NVGPRS(r1)
480 mtlr r14
481 mr r3,r15
482 #ifdef PPC64_ELF_ABI_v2
483 mr r12,r14
484 #endif
485 blrl
486 li r3,0
487 b .Lsyscall_exit
489 /*
490 * This routine switches between two different tasks. The process
491 * state of one is saved on its kernel stack. Then the state
492 * of the other is restored from its kernel stack. The memory
493 * management hardware is updated to the second process's state.
494 * Finally, we can return to the second process, via ret_from_except.
495 * On entry, r3 points to the THREAD for the current task, r4
496 * points to the THREAD for the new task.
497 *
498 * Note: there are two ways to get to the "going out" portion
499 * of this code; either by coming in via the entry (_switch)
500 * or via "fork" which must set up an environment equivalent
501 * to the "_switch" path. If you change this you'll have to change
502 * the fork code also.
503 *
504 * The code which creates the new task context is in 'copy_thread'
505 * in arch/powerpc/kernel/process.c
506 */
507 .align 7
508 _GLOBAL(_switch)
509 mflr r0
510 std r0,16(r1)
511 stdu r1,-SWITCH_FRAME_SIZE(r1)
512 /* r3-r13 are caller saved -- Cort */
513 SAVE_8GPRS(14, r1)
514 SAVE_10GPRS(22, r1)
515 std r0,_NIP(r1) /* Return to switch caller */
516 mfcr r23
517 std r23,_CCR(r1)
518 std r1,KSP(r3) /* Set old stack pointer */
520 /*
521 * On SMP kernels, care must be taken because a task may be
522 * scheduled off CPUx and on to CPUy. Memory ordering must be
523 * considered.
524 *
525 * Cacheable stores on CPUx will be visible when the task is
526 * scheduled on CPUy by virtue of the core scheduler barriers
527 * (see "Notes on Program-Order guarantees on SMP systems." in
528 * kernel/sched/core.c).
529 *
530 * Uncacheable stores in the case of involuntary preemption must
531 * be taken care of. The smp_mb__before_spin_lock() in __schedule()
532 * is implemented as hwsync on powerpc, which orders MMIO too. So
533 * long as there is an hwsync in the context switch path, it will
534 * be executed on the source CPU after the task has performed
535 * all MMIO ops on that CPU, and on the destination CPU before the
536 * task performs any MMIO ops there.
537 */
539 /*
540 * The kernel context switch path must contain a spin_lock,
541 * which contains larx/stcx, which will clear any reservation
542 * of the task being switched.
543 */
544 #ifdef CONFIG_PPC_BOOK3S
545 /* Cancel all explict user streams as they will have no use after context
546 * switch and will stop the HW from creating streams itself
547 */
548 DCBT_STOP_ALL_STREAM_IDS(r6)
549 #endif
551 addi r6,r4,-THREAD /* Convert THREAD to 'current' */
552 std r6,PACACURRENT(r13) /* Set new 'current' */
554 ld r8,KSP(r4) /* new stack pointer */
555 #ifdef CONFIG_PPC_BOOK3S_64
556 BEGIN_MMU_FTR_SECTION
557 b 2f
558 END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX)
559 BEGIN_FTR_SECTION
560 clrrdi r6,r8,28 /* get its ESID */
561 clrrdi r9,r1,28 /* get current sp ESID */
562 FTR_SECTION_ELSE
563 clrrdi r6,r8,40 /* get its 1T ESID */
564 clrrdi r9,r1,40 /* get current sp 1T ESID */
565 ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_1T_SEGMENT)
566 clrldi. r0,r6,2 /* is new ESID c00000000? */
567 cmpd cr1,r6,r9 /* or is new ESID the same as current ESID? */
568 cror eq,4*cr1+eq,eq
569 beq 2f /* if yes, don't slbie it */
571 /* Bolt in the new stack SLB entry */
572 ld r7,KSP_VSID(r4) /* Get new stack's VSID */
573 oris r0,r6,(SLB_ESID_V)@h
574 ori r0,r0,(SLB_NUM_BOLTED-1)@l
575 BEGIN_FTR_SECTION
576 li r9,MMU_SEGSIZE_1T /* insert B field */
577 oris r6,r6,(MMU_SEGSIZE_1T << SLBIE_SSIZE_SHIFT)@h
578 rldimi r7,r9,SLB_VSID_SSIZE_SHIFT,0
579 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
581 /* Update the last bolted SLB. No write barriers are needed
582 * here, provided we only update the current CPU's SLB shadow
583 * buffer.
584 */
585 ld r9,PACA_SLBSHADOWPTR(r13)
586 li r12,0
587 std r12,SLBSHADOW_STACKESID(r9) /* Clear ESID */
588 li r12,SLBSHADOW_STACKVSID
589 STDX_BE r7,r12,r9 /* Save VSID */
590 li r12,SLBSHADOW_STACKESID
591 STDX_BE r0,r12,r9 /* Save ESID */
593 /* No need to check for MMU_FTR_NO_SLBIE_B here, since when
594 * we have 1TB segments, the only CPUs known to have the errata
595 * only support less than 1TB of system memory and we'll never
596 * actually hit this code path.
597 */
599 slbie r6
600 slbie r6 /* Workaround POWER5 < DD2.1 issue */
601 slbmte r7,r0
602 isync
603 2:
604 #endif /* CONFIG_PPC_BOOK3S_64 */
606 CURRENT_THREAD_INFO(r7, r8) /* base of new stack */
607 /* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE
608 because we don't need to leave the 288-byte ABI gap at the
609 top of the kernel stack. */
610 addi r7,r7,THREAD_SIZE-SWITCH_FRAME_SIZE
612 /*
613 * PMU interrupts in radix may come in here. They will use r1, not
614 * PACAKSAVE, so this stack switch will not cause a problem. They
615 * will store to the process stack, which may then be migrated to
616 * another CPU. However the rq lock release on this CPU paired with
617 * the rq lock acquire on the new CPU before the stack becomes
618 * active on the new CPU, will order those stores.
619 */
620 mr r1,r8 /* start using new stack pointer */
621 std r7,PACAKSAVE(r13)
623 ld r6,_CCR(r1)
624 mtcrf 0xFF,r6
626 /* r3-r13 are destroyed -- Cort */
627 REST_8GPRS(14, r1)
628 REST_10GPRS(22, r1)
630 /* convert old thread to its task_struct for return value */
631 addi r3,r3,-THREAD
632 ld r7,_NIP(r1) /* Return to _switch caller in new task */
633 mtlr r7
634 addi r1,r1,SWITCH_FRAME_SIZE
635 blr
637 .align 7
638 _GLOBAL(ret_from_except)
639 ld r11,_TRAP(r1)
640 andi. r0,r11,1
641 bne ret_from_except_lite
642 REST_NVGPRS(r1)
644 _GLOBAL(ret_from_except_lite)
645 /*
646 * Disable interrupts so that current_thread_info()->flags
647 * can't change between when we test it and when we return
648 * from the interrupt.
649 */
650 #ifdef CONFIG_PPC_BOOK3E
651 wrteei 0
652 #else
653 li r10,MSR_RI
654 mtmsrd r10,1 /* Update machine state */
655 #endif /* CONFIG_PPC_BOOK3E */
657 CURRENT_THREAD_INFO(r9, r1)
658 ld r3,_MSR(r1)
659 #ifdef CONFIG_PPC_BOOK3E
660 ld r10,PACACURRENT(r13)
661 #endif /* CONFIG_PPC_BOOK3E */
662 ld r4,TI_FLAGS(r9)
663 andi. r3,r3,MSR_PR
664 beq resume_kernel
665 #ifdef CONFIG_PPC_BOOK3E
666 lwz r3,(THREAD+THREAD_DBCR0)(r10)
667 #endif /* CONFIG_PPC_BOOK3E */
669 /* Check current_thread_info()->flags */
670 andi. r0,r4,_TIF_USER_WORK_MASK
671 bne 1f
672 #ifdef CONFIG_PPC_BOOK3E
673 /*
674 * Check to see if the dbcr0 register is set up to debug.
675 * Use the internal debug mode bit to do this.
676 */
677 andis. r0,r3,DBCR0_IDM@h
678 beq restore
679 mfmsr r0
680 rlwinm r0,r0,0,~MSR_DE /* Clear MSR.DE */
681 mtmsr r0
682 mtspr SPRN_DBCR0,r3
683 li r10, -1
684 mtspr SPRN_DBSR,r10
685 b restore
686 #else
687 addi r3,r1,STACK_FRAME_OVERHEAD
688 bl restore_math
689 b restore
690 #endif
691 1: andi. r0,r4,_TIF_NEED_RESCHED
692 beq 2f
693 bl restore_interrupts
694 SCHEDULE_USER
695 b ret_from_except_lite
696 2:
697 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
698 andi. r0,r4,_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM
699 bne 3f /* only restore TM if nothing else to do */
700 addi r3,r1,STACK_FRAME_OVERHEAD
701 bl restore_tm_state
702 b restore
703 3:
704 #endif
705 bl save_nvgprs
706 /*
707 * Use a non volatile GPR to save and restore our thread_info flags
708 * across the call to restore_interrupts.
709 */
710 mr r30,r4
711 bl restore_interrupts
712 mr r4,r30
713 addi r3,r1,STACK_FRAME_OVERHEAD
714 bl do_notify_resume
715 b ret_from_except
717 resume_kernel:
718 /* check current_thread_info, _TIF_EMULATE_STACK_STORE */
719 andis. r8,r4,_TIF_EMULATE_STACK_STORE@h
720 beq+ 1f
722 addi r8,r1,INT_FRAME_SIZE /* Get the kprobed function entry */
724 ld r3,GPR1(r1)
725 subi r3,r3,INT_FRAME_SIZE /* dst: Allocate a trampoline exception frame */
726 mr r4,r1 /* src: current exception frame */
727 mr r1,r3 /* Reroute the trampoline frame to r1 */
729 /* Copy from the original to the trampoline. */
730 li r5,INT_FRAME_SIZE/8 /* size: INT_FRAME_SIZE */
731 li r6,0 /* start offset: 0 */
732 mtctr r5
733 2: ldx r0,r6,r4
734 stdx r0,r6,r3
735 addi r6,r6,8
736 bdnz 2b
738 /* Do real store operation to complete stdu */
739 ld r5,GPR1(r1)
740 std r8,0(r5)
742 /* Clear _TIF_EMULATE_STACK_STORE flag */
743 lis r11,_TIF_EMULATE_STACK_STORE@h
744 addi r5,r9,TI_FLAGS
745 0: ldarx r4,0,r5
746 andc r4,r4,r11
747 stdcx. r4,0,r5
748 bne- 0b
749 1:
751 #ifdef CONFIG_PREEMPT
752 /* Check if we need to preempt */
753 andi. r0,r4,_TIF_NEED_RESCHED
754 beq+ restore
755 /* Check that preempt_count() == 0 and interrupts are enabled */
756 lwz r8,TI_PREEMPT(r9)
757 cmpwi cr0,r8,0
758 bne restore
759 ld r0,SOFTE(r1)
760 andi. r0,r0,IRQS_DISABLED
761 bne restore
763 /*
764 * Here we are preempting the current task. We want to make
765 * sure we are soft-disabled first and reconcile irq state.
766 */
767 RECONCILE_IRQ_STATE(r3,r4)
768 1: bl preempt_schedule_irq
770 /* Re-test flags and eventually loop */
771 CURRENT_THREAD_INFO(r9, r1)
772 ld r4,TI_FLAGS(r9)
773 andi. r0,r4,_TIF_NEED_RESCHED
774 bne 1b
776 /*
777 * arch_local_irq_restore() from preempt_schedule_irq above may
778 * enable hard interrupt but we really should disable interrupts
779 * when we return from the interrupt, and so that we don't get
780 * interrupted after loading SRR0/1.
781 */
782 #ifdef CONFIG_PPC_BOOK3E
783 wrteei 0
784 #else
785 li r10,MSR_RI
786 mtmsrd r10,1 /* Update machine state */
787 #endif /* CONFIG_PPC_BOOK3E */
788 #endif /* CONFIG_PREEMPT */
790 .globl fast_exc_return_irq
791 fast_exc_return_irq:
792 restore:
793 /*
794 * This is the main kernel exit path. First we check if we
795 * are about to re-enable interrupts
796 */
797 ld r5,SOFTE(r1)
798 lbz r6,PACAIRQSOFTMASK(r13)
799 andi. r5,r5,IRQS_DISABLED
800 bne .Lrestore_irq_off
802 /* We are enabling, were we already enabled ? Yes, just return */
803 andi. r6,r6,IRQS_DISABLED
804 beq cr0,.Ldo_restore
806 /*
807 * We are about to soft-enable interrupts (we are hard disabled
808 * at this point). We check if there's anything that needs to
809 * be replayed first.
810 */
811 lbz r0,PACAIRQHAPPENED(r13)
812 cmpwi cr0,r0,0
813 bne- .Lrestore_check_irq_replay
815 /*
816 * Get here when nothing happened while soft-disabled, just
817 * soft-enable and move-on. We will hard-enable as a side
818 * effect of rfi
819 */
820 .Lrestore_no_replay:
821 TRACE_ENABLE_INTS
822 li r0,IRQS_ENABLED
823 stb r0,PACAIRQSOFTMASK(r13);
825 /*
826 * Final return path. BookE is handled in a different file
827 */
828 .Ldo_restore:
829 #ifdef CONFIG_PPC_BOOK3E
830 b exception_return_book3e
831 #else
832 /*
833 * Clear the reservation. If we know the CPU tracks the address of
834 * the reservation then we can potentially save some cycles and use
835 * a larx. On POWER6 and POWER7 this is significantly faster.
836 */
837 BEGIN_FTR_SECTION
838 stdcx. r0,0,r1 /* to clear the reservation */
839 FTR_SECTION_ELSE
840 ldarx r4,0,r1
841 ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
843 /*
844 * Some code path such as load_up_fpu or altivec return directly
845 * here. They run entirely hard disabled and do not alter the
846 * interrupt state. They also don't use lwarx/stwcx. and thus
847 * are known not to leave dangling reservations.
848 */
849 .globl fast_exception_return
850 fast_exception_return:
851 ld r3,_MSR(r1)
852 ld r4,_CTR(r1)
853 ld r0,_LINK(r1)
854 mtctr r4
855 mtlr r0
856 ld r4,_XER(r1)
857 mtspr SPRN_XER,r4
859 REST_8GPRS(5, r1)
861 andi. r0,r3,MSR_RI
862 beq- .Lunrecov_restore
864 /* Load PPR from thread struct before we clear MSR:RI */
865 BEGIN_FTR_SECTION
866 ld r2,PACACURRENT(r13)
867 ld r2,TASKTHREADPPR(r2)
868 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
870 /*
871 * Clear RI before restoring r13. If we are returning to
872 * userspace and we take an exception after restoring r13,
873 * we end up corrupting the userspace r13 value.
874 */
875 li r4,0
876 mtmsrd r4,1
878 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
879 /* TM debug */
880 std r3, PACATMSCRATCH(r13) /* Stash returned-to MSR */
881 #endif
882 /*
883 * r13 is our per cpu area, only restore it if we are returning to
884 * userspace the value stored in the stack frame may belong to
885 * another CPU.
886 */
887 andi. r0,r3,MSR_PR
888 beq 1f
889 BEGIN_FTR_SECTION
890 mtspr SPRN_PPR,r2 /* Restore PPR */
891 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
892 ACCOUNT_CPU_USER_EXIT(r13, r2, r4)
893 REST_GPR(13, r1)
895 mtspr SPRN_SRR1,r3
897 ld r2,_CCR(r1)
898 mtcrf 0xFF,r2
899 ld r2,_NIP(r1)
900 mtspr SPRN_SRR0,r2
902 ld r0,GPR0(r1)
903 ld r2,GPR2(r1)
904 ld r3,GPR3(r1)
905 ld r4,GPR4(r1)
906 ld r1,GPR1(r1)
907 RFI_TO_USER
908 b . /* prevent speculative execution */
910 1: mtspr SPRN_SRR1,r3
912 ld r2,_CCR(r1)
913 mtcrf 0xFF,r2
914 ld r2,_NIP(r1)
915 mtspr SPRN_SRR0,r2
917 ld r0,GPR0(r1)
918 ld r2,GPR2(r1)
919 ld r3,GPR3(r1)
920 ld r4,GPR4(r1)
921 ld r1,GPR1(r1)
922 RFI_TO_KERNEL
923 b . /* prevent speculative execution */
925 #endif /* CONFIG_PPC_BOOK3E */
927 /*
928 * We are returning to a context with interrupts soft disabled.
929 *
930 * However, we may also about to hard enable, so we need to
931 * make sure that in this case, we also clear PACA_IRQ_HARD_DIS
932 * or that bit can get out of sync and bad things will happen
933 */
934 .Lrestore_irq_off:
935 ld r3,_MSR(r1)
936 lbz r7,PACAIRQHAPPENED(r13)
937 andi. r0,r3,MSR_EE
938 beq 1f
939 rlwinm r7,r7,0,~PACA_IRQ_HARD_DIS
940 stb r7,PACAIRQHAPPENED(r13)
941 1:
942 #if defined(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG) && defined(CONFIG_BUG)
943 /* The interrupt should not have soft enabled. */
944 lbz r7,PACAIRQSOFTMASK(r13)
945 1: tdeqi r7,IRQS_ENABLED
946 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
947 #endif
948 b .Ldo_restore
950 /*
951 * Something did happen, check if a re-emit is needed
952 * (this also clears paca->irq_happened)
953 */
954 .Lrestore_check_irq_replay:
955 /* XXX: We could implement a fast path here where we check
956 * for irq_happened being just 0x01, in which case we can
957 * clear it and return. That means that we would potentially
958 * miss a decrementer having wrapped all the way around.
959 *
960 * Still, this might be useful for things like hash_page
961 */
962 bl __check_irq_replay
963 cmpwi cr0,r3,0
964 beq .Lrestore_no_replay
966 /*
967 * We need to re-emit an interrupt. We do so by re-using our
968 * existing exception frame. We first change the trap value,
969 * but we need to ensure we preserve the low nibble of it
970 */
971 ld r4,_TRAP(r1)
972 clrldi r4,r4,60
973 or r4,r4,r3
974 std r4,_TRAP(r1)
976 /*
977 * Then find the right handler and call it. Interrupts are
978 * still soft-disabled and we keep them that way.
979 */
980 cmpwi cr0,r3,0x500
981 bne 1f
982 addi r3,r1,STACK_FRAME_OVERHEAD;
983 bl do_IRQ
984 b ret_from_except
985 1: cmpwi cr0,r3,0xf00
986 bne 1f
987 addi r3,r1,STACK_FRAME_OVERHEAD;
988 bl performance_monitor_exception
989 b ret_from_except
990 1: cmpwi cr0,r3,0xe60
991 bne 1f
992 addi r3,r1,STACK_FRAME_OVERHEAD;
993 bl handle_hmi_exception
994 b ret_from_except
995 1: cmpwi cr0,r3,0x900
996 bne 1f
997 addi r3,r1,STACK_FRAME_OVERHEAD;
998 bl timer_interrupt
999 b ret_from_except
1000 #ifdef CONFIG_PPC_DOORBELL
1001 1:
1002 #ifdef CONFIG_PPC_BOOK3E
1003 cmpwi cr0,r3,0x280
1004 #else
1005 cmpwi cr0,r3,0xa00
1006 #endif /* CONFIG_PPC_BOOK3E */
1007 bne 1f
1008 addi r3,r1,STACK_FRAME_OVERHEAD;
1009 bl doorbell_exception
1010 #endif /* CONFIG_PPC_DOORBELL */
1011 1: b ret_from_except /* What else to do here ? */
1013 .Lunrecov_restore:
1014 addi r3,r1,STACK_FRAME_OVERHEAD
1015 bl unrecoverable_exception
1016 b .Lunrecov_restore
1018 _ASM_NOKPROBE_SYMBOL(ret_from_except);
1019 _ASM_NOKPROBE_SYMBOL(ret_from_except_lite);
1020 _ASM_NOKPROBE_SYMBOL(resume_kernel);
1021 _ASM_NOKPROBE_SYMBOL(fast_exc_return_irq);
1022 _ASM_NOKPROBE_SYMBOL(restore);
1023 _ASM_NOKPROBE_SYMBOL(fast_exception_return);
1026 #ifdef CONFIG_PPC_RTAS
1027 /*
1028 * On CHRP, the Run-Time Abstraction Services (RTAS) have to be
1029 * called with the MMU off.
1030 *
1031 * In addition, we need to be in 32b mode, at least for now.
1032 *
1033 * Note: r3 is an input parameter to rtas, so don't trash it...
1034 */
1035 _GLOBAL(enter_rtas)
1036 mflr r0
1037 std r0,16(r1)
1038 stdu r1,-RTAS_FRAME_SIZE(r1) /* Save SP and create stack space. */
1040 /* Because RTAS is running in 32b mode, it clobbers the high order half
1041 * of all registers that it saves. We therefore save those registers
1042 * RTAS might touch to the stack. (r0, r3-r13 are caller saved)
1043 */
1044 SAVE_GPR(2, r1) /* Save the TOC */
1045 SAVE_GPR(13, r1) /* Save paca */
1046 SAVE_8GPRS(14, r1) /* Save the non-volatiles */
1047 SAVE_10GPRS(22, r1) /* ditto */
1049 mfcr r4
1050 std r4,_CCR(r1)
1051 mfctr r5
1052 std r5,_CTR(r1)
1053 mfspr r6,SPRN_XER
1054 std r6,_XER(r1)
1055 mfdar r7
1056 std r7,_DAR(r1)
1057 mfdsisr r8
1058 std r8,_DSISR(r1)
1060 /* Temporary workaround to clear CR until RTAS can be modified to
1061 * ignore all bits.
1062 */
1063 li r0,0
1064 mtcr r0
1066 #ifdef CONFIG_BUG
1067 /* There is no way it is acceptable to get here with interrupts enabled,
1068 * check it with the asm equivalent of WARN_ON
1069 */
1070 lbz r0,PACAIRQSOFTMASK(r13)
1071 1: tdeqi r0,IRQS_ENABLED
1072 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
1073 #endif
1075 /* Hard-disable interrupts */
1076 mfmsr r6
1077 rldicl r7,r6,48,1
1078 rotldi r7,r7,16
1079 mtmsrd r7,1
1081 /* Unfortunately, the stack pointer and the MSR are also clobbered,
1082 * so they are saved in the PACA which allows us to restore
1083 * our original state after RTAS returns.
1084 */
1085 std r1,PACAR1(r13)
1086 std r6,PACASAVEDMSR(r13)
1088 /* Setup our real return addr */
1089 LOAD_REG_ADDR(r4,rtas_return_loc)
1090 clrldi r4,r4,2 /* convert to realmode address */
1091 mtlr r4
1093 li r0,0
1094 ori r0,r0,MSR_EE|MSR_SE|MSR_BE|MSR_RI
1095 andc r0,r6,r0
1097 li r9,1
1098 rldicr r9,r9,MSR_SF_LG,(63-MSR_SF_LG)
1099 ori r9,r9,MSR_IR|MSR_DR|MSR_FE0|MSR_FE1|MSR_FP|MSR_RI|MSR_LE
1100 andc r6,r0,r9
1102 __enter_rtas:
1103 sync /* disable interrupts so SRR0/1 */
1104 mtmsrd r0 /* don't get trashed */
1106 LOAD_REG_ADDR(r4, rtas)
1107 ld r5,RTASENTRY(r4) /* get the rtas->entry value */
1108 ld r4,RTASBASE(r4) /* get the rtas->base value */
1110 mtspr SPRN_SRR0,r5
1111 mtspr SPRN_SRR1,r6
1112 RFI_TO_KERNEL
1113 b . /* prevent speculative execution */
1115 rtas_return_loc:
1116 FIXUP_ENDIAN
1118 /*
1119 * Clear RI and set SF before anything.
1120 */
1121 mfmsr r6
1122 li r0,MSR_RI
1123 andc r6,r6,r0
1124 sldi r0,r0,(MSR_SF_LG - MSR_RI_LG)
1125 or r6,r6,r0
1126 sync
1127 mtmsrd r6
1129 /* relocation is off at this point */
1130 GET_PACA(r4)
1131 clrldi r4,r4,2 /* convert to realmode address */
1133 bcl 20,31,$+4
1134 0: mflr r3
1135 ld r3,(1f-0b)(r3) /* get &rtas_restore_regs */
1137 ld r1,PACAR1(r4) /* Restore our SP */
1138 ld r4,PACASAVEDMSR(r4) /* Restore our MSR */
1140 mtspr SPRN_SRR0,r3
1141 mtspr SPRN_SRR1,r4
1142 RFI_TO_KERNEL
1143 b . /* prevent speculative execution */
1144 _ASM_NOKPROBE_SYMBOL(__enter_rtas)
1145 _ASM_NOKPROBE_SYMBOL(rtas_return_loc)
1147 .align 3
1148 1: .8byte rtas_restore_regs
1150 rtas_restore_regs:
1151 /* relocation is on at this point */
1152 REST_GPR(2, r1) /* Restore the TOC */
1153 REST_GPR(13, r1) /* Restore paca */
1154 REST_8GPRS(14, r1) /* Restore the non-volatiles */
1155 REST_10GPRS(22, r1) /* ditto */
1157 GET_PACA(r13)
1159 ld r4,_CCR(r1)
1160 mtcr r4
1161 ld r5,_CTR(r1)
1162 mtctr r5
1163 ld r6,_XER(r1)
1164 mtspr SPRN_XER,r6
1165 ld r7,_DAR(r1)
1166 mtdar r7
1167 ld r8,_DSISR(r1)
1168 mtdsisr r8
1170 addi r1,r1,RTAS_FRAME_SIZE /* Unstack our frame */
1171 ld r0,16(r1) /* get return address */
1173 mtlr r0
1174 blr /* return to caller */
1176 #endif /* CONFIG_PPC_RTAS */
1178 _GLOBAL(enter_prom)
1179 mflr r0
1180 std r0,16(r1)
1181 stdu r1,-PROM_FRAME_SIZE(r1) /* Save SP and create stack space */
1183 /* Because PROM is running in 32b mode, it clobbers the high order half
1184 * of all registers that it saves. We therefore save those registers
1185 * PROM might touch to the stack. (r0, r3-r13 are caller saved)
1186 */
1187 SAVE_GPR(2, r1)
1188 SAVE_GPR(13, r1)
1189 SAVE_8GPRS(14, r1)
1190 SAVE_10GPRS(22, r1)
1191 mfcr r10
1192 mfmsr r11
1193 std r10,_CCR(r1)
1194 std r11,_MSR(r1)
1196 /* Put PROM address in SRR0 */
1197 mtsrr0 r4
1199 /* Setup our trampoline return addr in LR */
1200 bcl 20,31,$+4
1201 0: mflr r4
1202 addi r4,r4,(1f - 0b)
1203 mtlr r4
1205 /* Prepare a 32-bit mode big endian MSR
1206 */
1207 #ifdef CONFIG_PPC_BOOK3E
1208 rlwinm r11,r11,0,1,31
1209 mtsrr1 r11
1210 rfi
1211 #else /* CONFIG_PPC_BOOK3E */
1212 LOAD_REG_IMMEDIATE(r12, MSR_SF | MSR_ISF | MSR_LE)
1213 andc r11,r11,r12
1214 mtsrr1 r11
1215 RFI_TO_KERNEL
1216 #endif /* CONFIG_PPC_BOOK3E */
1218 1: /* Return from OF */
1219 FIXUP_ENDIAN
1221 /* Just make sure that r1 top 32 bits didn't get
1222 * corrupt by OF
1223 */
1224 rldicl r1,r1,0,32
1226 /* Restore the MSR (back to 64 bits) */
1227 ld r0,_MSR(r1)
1228 MTMSRD(r0)
1229 isync
1231 /* Restore other registers */
1232 REST_GPR(2, r1)
1233 REST_GPR(13, r1)
1234 REST_8GPRS(14, r1)
1235 REST_10GPRS(22, r1)
1236 ld r4,_CCR(r1)
1237 mtcr r4
1239 addi r1,r1,PROM_FRAME_SIZE
1240 ld r0,16(r1)
1241 mtlr r0
1242 blr