| blob | 4a0fd4f4024533c473f5cfe6f015924e29ed4acc |
1 /*
2 * PowerPC version
3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 * Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
5 * Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
6 * Adapted for Power Macintosh by Paul Mackerras.
7 * Low-level exception handlers and MMU support
8 * rewritten by Paul Mackerras.
9 * Copyright (C) 1996 Paul Mackerras.
10 * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
11 *
12 * This file contains the system call entry code, context switch
13 * code, and exception/interrupt return code for PowerPC.
14 *
15 * This program is free software; you can redistribute it and/or
16 * modify it under the terms of the GNU General Public License
17 * as published by the Free Software Foundation; either version
18 * 2 of the License, or (at your option) any later version.
19 */
21 #include <linux/errno.h>
22 #include <linux/err.h>
23 #include <asm/unistd.h>
24 #include <asm/processor.h>
25 #include <asm/page.h>
26 #include <asm/mmu.h>
27 #include <asm/thread_info.h>
28 #include <asm/ppc_asm.h>
29 #include <asm/asm-offsets.h>
30 #include <asm/cputable.h>
31 #include <asm/firmware.h>
32 #include <asm/bug.h>
33 #include <asm/ptrace.h>
34 #include <asm/irqflags.h>
35 #include <asm/hw_irq.h>
36 #include <asm/context_tracking.h>
37 #include <asm/tm.h>
38 #include <asm/ppc-opcode.h>
39 #include <asm/export.h>
41 /*
42 * System calls.
43 */
44 .section ".toc","aw"
45 SYS_CALL_TABLE:
46 .tc sys_call_table[TC],sys_call_table
48 /* This value is used to mark exception frames on the stack. */
49 exception_marker:
50 .tc ID_EXC_MARKER[TC],STACK_FRAME_REGS_MARKER
52 .section ".text"
53 .align 7
55 .globl system_call_common
56 system_call_common:
57 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
58 BEGIN_FTR_SECTION
59 extrdi. r10, r12, 1, (63-MSR_TS_T_LG) /* transaction active? */
60 bne .Ltabort_syscall
61 END_FTR_SECTION_IFSET(CPU_FTR_TM)
62 #endif
63 andi. r10,r12,MSR_PR
64 mr r10,r1
65 addi r1,r1,-INT_FRAME_SIZE
66 beq- 1f
67 ld r1,PACAKSAVE(r13)
68 1: std r10,0(r1)
69 std r11,_NIP(r1)
70 std r12,_MSR(r1)
71 std r0,GPR0(r1)
72 std r10,GPR1(r1)
73 beq 2f /* if from kernel mode */
74 ACCOUNT_CPU_USER_ENTRY(r13, r10, r11)
75 2: std r2,GPR2(r1)
76 std r3,GPR3(r1)
77 mfcr r2
78 std r4,GPR4(r1)
79 std r5,GPR5(r1)
80 std r6,GPR6(r1)
81 std r7,GPR7(r1)
82 std r8,GPR8(r1)
83 li r11,0
84 std r11,GPR9(r1)
85 std r11,GPR10(r1)
86 std r11,GPR11(r1)
87 std r11,GPR12(r1)
88 std r11,_XER(r1)
89 std r11,_CTR(r1)
90 std r9,GPR13(r1)
91 mflr r10
92 /*
93 * This clears CR0.SO (bit 28), which is the error indication on
94 * return from this system call.
95 */
96 rldimi r2,r11,28,(63-28)
97 li r11,0xc01
98 std r10,_LINK(r1)
99 std r11,_TRAP(r1)
100 std r3,ORIG_GPR3(r1)
101 std r2,_CCR(r1)
102 ld r2,PACATOC(r13)
103 addi r9,r1,STACK_FRAME_OVERHEAD
104 ld r11,exception_marker@toc(r2)
105 std r11,-16(r9) /* "regshere" marker */
106 #if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) && defined(CONFIG_PPC_SPLPAR)
107 BEGIN_FW_FTR_SECTION
108 beq 33f
109 /* if from user, see if there are any DTL entries to process */
110 ld r10,PACALPPACAPTR(r13) /* get ptr to VPA */
111 ld r11,PACA_DTL_RIDX(r13) /* get log read index */
112 addi r10,r10,LPPACA_DTLIDX
113 LDX_BE r10,0,r10 /* get log write index */
114 cmpd cr1,r11,r10
115 beq+ cr1,33f
116 bl accumulate_stolen_time
117 REST_GPR(0,r1)
118 REST_4GPRS(3,r1)
119 REST_2GPRS(7,r1)
120 addi r9,r1,STACK_FRAME_OVERHEAD
121 33:
122 END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR)
123 #endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE && CONFIG_PPC_SPLPAR */
125 /*
126 * A syscall should always be called with interrupts enabled
127 * so we just unconditionally hard-enable here. When some kind
128 * of irq tracing is used, we additionally check that condition
129 * is correct
130 */
131 #if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_BUG)
132 lbz r10,PACASOFTIRQEN(r13)
133 xori r10,r10,1
134 1: tdnei r10,0
135 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
136 #endif
138 #ifdef CONFIG_PPC_BOOK3E
139 wrteei 1
140 #else
141 li r11,MSR_RI
142 ori r11,r11,MSR_EE
143 mtmsrd r11,1
144 #endif /* CONFIG_PPC_BOOK3E */
146 system_call: /* label this so stack traces look sane */
147 /* We do need to set SOFTE in the stack frame or the return
148 * from interrupt will be painful
149 */
150 li r10,1
151 std r10,SOFTE(r1)
153 CURRENT_THREAD_INFO(r11, r1)
154 ld r10,TI_FLAGS(r11)
155 andi. r11,r10,_TIF_SYSCALL_DOTRACE
156 bne .Lsyscall_dotrace /* does not return */
157 cmpldi 0,r0,NR_syscalls
158 bge- .Lsyscall_enosys
160 .Lsyscall:
161 /*
162 * Need to vector to 32 Bit or default sys_call_table here,
163 * based on caller's run-mode / personality.
164 */
165 ld r11,SYS_CALL_TABLE@toc(2)
166 andi. r10,r10,_TIF_32BIT
167 beq 15f
168 addi r11,r11,8 /* use 32-bit syscall entries */
169 clrldi r3,r3,32
170 clrldi r4,r4,32
171 clrldi r5,r5,32
172 clrldi r6,r6,32
173 clrldi r7,r7,32
174 clrldi r8,r8,32
175 15:
176 slwi r0,r0,4
177 ldx r12,r11,r0 /* Fetch system call handler [ptr] */
178 mtctr r12
179 bctrl /* Call handler */
181 .Lsyscall_exit:
182 std r3,RESULT(r1)
183 CURRENT_THREAD_INFO(r12, r1)
185 ld r8,_MSR(r1)
186 #ifdef CONFIG_PPC_BOOK3S
187 /* No MSR:RI on BookE */
188 andi. r10,r8,MSR_RI
189 beq- .Lunrecov_restore
190 #endif
192 /*
193 * This is a few instructions into the actual syscall exit path (which actually
194 * starts at .Lsyscall_exit) to cater to kprobe blacklisting and to reduce the
195 * number of visible symbols for profiling purposes.
196 *
197 * We can probe from system_call until this point as MSR_RI is set. But once it
198 * is cleared below, we won't be able to take a trap.
199 *
200 * This is blacklisted from kprobes further below with _ASM_NOKPROBE_SYMBOL().
201 */
202 system_call_exit:
203 /*
204 * Disable interrupts so current_thread_info()->flags can't change,
205 * and so that we don't get interrupted after loading SRR0/1.
206 */
207 #ifdef CONFIG_PPC_BOOK3E
208 wrteei 0
209 #else
210 /*
211 * For performance reasons we clear RI the same time that we
212 * clear EE. We only need to clear RI just before we restore r13
213 * below, but batching it with EE saves us one expensive mtmsrd call.
214 * We have to be careful to restore RI if we branch anywhere from
215 * here (eg syscall_exit_work).
216 */
217 li r11,0
218 mtmsrd r11,1
219 #endif /* CONFIG_PPC_BOOK3E */
221 ld r9,TI_FLAGS(r12)
222 li r11,-MAX_ERRNO
223 andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
224 bne- .Lsyscall_exit_work
226 andi. r0,r8,MSR_FP
227 beq 2f
228 #ifdef CONFIG_ALTIVEC
229 andis. r0,r8,MSR_VEC@h
230 bne 3f
231 #endif
232 2: addi r3,r1,STACK_FRAME_OVERHEAD
233 #ifdef CONFIG_PPC_BOOK3S
234 li r10,MSR_RI
235 mtmsrd r10,1 /* Restore RI */
236 #endif
237 bl restore_math
238 #ifdef CONFIG_PPC_BOOK3S
239 li r11,0
240 mtmsrd r11,1
241 #endif
242 ld r8,_MSR(r1)
243 ld r3,RESULT(r1)
244 li r11,-MAX_ERRNO
246 3: cmpld r3,r11
247 ld r5,_CCR(r1)
248 bge- .Lsyscall_error
249 .Lsyscall_error_cont:
250 ld r7,_NIP(r1)
251 BEGIN_FTR_SECTION
252 stdcx. r0,0,r1 /* to clear the reservation */
253 END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
254 andi. r6,r8,MSR_PR
255 ld r4,_LINK(r1)
257 beq- 1f
258 ACCOUNT_CPU_USER_EXIT(r13, r11, r12)
260 BEGIN_FTR_SECTION
261 HMT_MEDIUM_LOW
262 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
264 ld r13,GPR13(r1) /* only restore r13 if returning to usermode */
265 1: ld r2,GPR2(r1)
266 ld r1,GPR1(r1)
267 mtlr r4
268 mtcr r5
269 mtspr SPRN_SRR0,r7
270 mtspr SPRN_SRR1,r8
271 RFI
272 b . /* prevent speculative execution */
274 .Lsyscall_error:
275 oris r5,r5,0x1000 /* Set SO bit in CR */
276 neg r3,r3
277 std r5,_CCR(r1)
278 b .Lsyscall_error_cont
280 /* Traced system call support */
281 .Lsyscall_dotrace:
282 bl save_nvgprs
283 addi r3,r1,STACK_FRAME_OVERHEAD
284 bl do_syscall_trace_enter
286 /*
287 * We use the return value of do_syscall_trace_enter() as the syscall
288 * number. If the syscall was rejected for any reason do_syscall_trace_enter()
289 * returns an invalid syscall number and the test below against
290 * NR_syscalls will fail.
291 */
292 mr r0,r3
294 /* Restore argument registers just clobbered and/or possibly changed. */
295 ld r3,GPR3(r1)
296 ld r4,GPR4(r1)
297 ld r5,GPR5(r1)
298 ld r6,GPR6(r1)
299 ld r7,GPR7(r1)
300 ld r8,GPR8(r1)
302 /* Repopulate r9 and r10 for the syscall path */
303 addi r9,r1,STACK_FRAME_OVERHEAD
304 CURRENT_THREAD_INFO(r10, r1)
305 ld r10,TI_FLAGS(r10)
307 cmpldi r0,NR_syscalls
308 blt+ .Lsyscall
310 /* Return code is already in r3 thanks to do_syscall_trace_enter() */
311 b .Lsyscall_exit
314 .Lsyscall_enosys:
315 li r3,-ENOSYS
316 b .Lsyscall_exit
318 .Lsyscall_exit_work:
319 #ifdef CONFIG_PPC_BOOK3S
320 li r10,MSR_RI
321 mtmsrd r10,1 /* Restore RI */
322 #endif
323 /* If TIF_RESTOREALL is set, don't scribble on either r3 or ccr.
324 If TIF_NOERROR is set, just save r3 as it is. */
326 andi. r0,r9,_TIF_RESTOREALL
327 beq+ 0f
328 REST_NVGPRS(r1)
329 b 2f
330 0: cmpld r3,r11 /* r11 is -MAX_ERRNO */
331 blt+ 1f
332 andi. r0,r9,_TIF_NOERROR
333 bne- 1f
334 ld r5,_CCR(r1)
335 neg r3,r3
336 oris r5,r5,0x1000 /* Set SO bit in CR */
337 std r5,_CCR(r1)
338 1: std r3,GPR3(r1)
339 2: andi. r0,r9,(_TIF_PERSYSCALL_MASK)
340 beq 4f
342 /* Clear per-syscall TIF flags if any are set. */
344 li r11,_TIF_PERSYSCALL_MASK
345 addi r12,r12,TI_FLAGS
346 3: ldarx r10,0,r12
347 andc r10,r10,r11
348 stdcx. r10,0,r12
349 bne- 3b
350 subi r12,r12,TI_FLAGS
352 4: /* Anything else left to do? */
353 BEGIN_FTR_SECTION
354 lis r3,INIT_PPR@highest /* Set thread.ppr = 3 */
355 ld r10,PACACURRENT(r13)
356 sldi r3,r3,32 /* bits 11-13 are used for ppr */
357 std r3,TASKTHREADPPR(r10)
358 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
360 andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP)
361 beq ret_from_except_lite
363 /* Re-enable interrupts */
364 #ifdef CONFIG_PPC_BOOK3E
365 wrteei 1
366 #else
367 li r10,MSR_RI
368 ori r10,r10,MSR_EE
369 mtmsrd r10,1
370 #endif /* CONFIG_PPC_BOOK3E */
372 bl save_nvgprs
373 addi r3,r1,STACK_FRAME_OVERHEAD
374 bl do_syscall_trace_leave
375 b ret_from_except
377 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
378 .Ltabort_syscall:
379 /* Firstly we need to enable TM in the kernel */
380 mfmsr r10
381 li r9, 1
382 rldimi r10, r9, MSR_TM_LG, 63-MSR_TM_LG
383 mtmsrd r10, 0
385 /* tabort, this dooms the transaction, nothing else */
386 li r9, (TM_CAUSE_SYSCALL|TM_CAUSE_PERSISTENT)
387 TABORT(R9)
389 /*
390 * Return directly to userspace. We have corrupted user register state,
391 * but userspace will never see that register state. Execution will
392 * resume after the tbegin of the aborted transaction with the
393 * checkpointed register state.
394 */
395 li r9, MSR_RI
396 andc r10, r10, r9
397 mtmsrd r10, 1
398 mtspr SPRN_SRR0, r11
399 mtspr SPRN_SRR1, r12
401 rfid
402 b . /* prevent speculative execution */
403 #endif
404 _ASM_NOKPROBE_SYMBOL(system_call_common);
405 _ASM_NOKPROBE_SYMBOL(system_call_exit);
407 /* Save non-volatile GPRs, if not already saved. */
408 _GLOBAL(save_nvgprs)
409 ld r11,_TRAP(r1)
410 andi. r0,r11,1
411 beqlr-
412 SAVE_NVGPRS(r1)
413 clrrdi r0,r11,1
414 std r0,_TRAP(r1)
415 blr
416 _ASM_NOKPROBE_SYMBOL(save_nvgprs);
419 /*
420 * The sigsuspend and rt_sigsuspend system calls can call do_signal
421 * and thus put the process into the stopped state where we might
422 * want to examine its user state with ptrace. Therefore we need
423 * to save all the nonvolatile registers (r14 - r31) before calling
424 * the C code. Similarly, fork, vfork and clone need the full
425 * register state on the stack so that it can be copied to the child.
426 */
428 _GLOBAL(ppc_fork)
429 bl save_nvgprs
430 bl sys_fork
431 b .Lsyscall_exit
433 _GLOBAL(ppc_vfork)
434 bl save_nvgprs
435 bl sys_vfork
436 b .Lsyscall_exit
438 _GLOBAL(ppc_clone)
439 bl save_nvgprs
440 bl sys_clone
441 b .Lsyscall_exit
443 _GLOBAL(ppc32_swapcontext)
444 bl save_nvgprs
445 bl compat_sys_swapcontext
446 b .Lsyscall_exit
448 _GLOBAL(ppc64_swapcontext)
449 bl save_nvgprs
450 bl sys_swapcontext
451 b .Lsyscall_exit
453 _GLOBAL(ppc_switch_endian)
454 bl save_nvgprs
455 bl sys_switch_endian
456 b .Lsyscall_exit
458 _GLOBAL(ret_from_fork)
459 bl schedule_tail
460 REST_NVGPRS(r1)
461 li r3,0
462 b .Lsyscall_exit
464 _GLOBAL(ret_from_kernel_thread)
465 bl schedule_tail
466 REST_NVGPRS(r1)
467 mtlr r14
468 mr r3,r15
469 #ifdef PPC64_ELF_ABI_v2
470 mr r12,r14
471 #endif
472 blrl
473 li r3,0
474 b .Lsyscall_exit
476 /*
477 * This routine switches between two different tasks. The process
478 * state of one is saved on its kernel stack. Then the state
479 * of the other is restored from its kernel stack. The memory
480 * management hardware is updated to the second process's state.
481 * Finally, we can return to the second process, via ret_from_except.
482 * On entry, r3 points to the THREAD for the current task, r4
483 * points to the THREAD for the new task.
484 *
485 * Note: there are two ways to get to the "going out" portion
486 * of this code; either by coming in via the entry (_switch)
487 * or via "fork" which must set up an environment equivalent
488 * to the "_switch" path. If you change this you'll have to change
489 * the fork code also.
490 *
491 * The code which creates the new task context is in 'copy_thread'
492 * in arch/powerpc/kernel/process.c
493 */
494 .align 7
495 _GLOBAL(_switch)
496 mflr r0
497 std r0,16(r1)
498 stdu r1,-SWITCH_FRAME_SIZE(r1)
499 /* r3-r13 are caller saved -- Cort */
500 SAVE_8GPRS(14, r1)
501 SAVE_10GPRS(22, r1)
502 std r0,_NIP(r1) /* Return to switch caller */
503 mfcr r23
504 std r23,_CCR(r1)
505 std r1,KSP(r3) /* Set old stack pointer */
507 /*
508 * On SMP kernels, care must be taken because a task may be
509 * scheduled off CPUx and on to CPUy. Memory ordering must be
510 * considered.
511 *
512 * Cacheable stores on CPUx will be visible when the task is
513 * scheduled on CPUy by virtue of the core scheduler barriers
514 * (see "Notes on Program-Order guarantees on SMP systems." in
515 * kernel/sched/core.c).
516 *
517 * Uncacheable stores in the case of involuntary preemption must
518 * be taken care of. The smp_mb__before_spin_lock() in __schedule()
519 * is implemented as hwsync on powerpc, which orders MMIO too. So
520 * long as there is an hwsync in the context switch path, it will
521 * be executed on the source CPU after the task has performed
522 * all MMIO ops on that CPU, and on the destination CPU before the
523 * task performs any MMIO ops there.
524 */
526 /*
527 * The kernel context switch path must contain a spin_lock,
528 * which contains larx/stcx, which will clear any reservation
529 * of the task being switched.
530 */
531 #ifdef CONFIG_PPC_BOOK3S
532 /* Cancel all explict user streams as they will have no use after context
533 * switch and will stop the HW from creating streams itself
534 */
535 DCBT_STOP_ALL_STREAM_IDS(r6)
536 #endif
538 addi r6,r4,-THREAD /* Convert THREAD to 'current' */
539 std r6,PACACURRENT(r13) /* Set new 'current' */
541 ld r8,KSP(r4) /* new stack pointer */
542 #ifdef CONFIG_PPC_STD_MMU_64
543 BEGIN_MMU_FTR_SECTION
544 b 2f
545 END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX)
546 BEGIN_FTR_SECTION
547 clrrdi r6,r8,28 /* get its ESID */
548 clrrdi r9,r1,28 /* get current sp ESID */
549 FTR_SECTION_ELSE
550 clrrdi r6,r8,40 /* get its 1T ESID */
551 clrrdi r9,r1,40 /* get current sp 1T ESID */
552 ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_1T_SEGMENT)
553 clrldi. r0,r6,2 /* is new ESID c00000000? */
554 cmpd cr1,r6,r9 /* or is new ESID the same as current ESID? */
555 cror eq,4*cr1+eq,eq
556 beq 2f /* if yes, don't slbie it */
558 /* Bolt in the new stack SLB entry */
559 ld r7,KSP_VSID(r4) /* Get new stack's VSID */
560 oris r0,r6,(SLB_ESID_V)@h
561 ori r0,r0,(SLB_NUM_BOLTED-1)@l
562 BEGIN_FTR_SECTION
563 li r9,MMU_SEGSIZE_1T /* insert B field */
564 oris r6,r6,(MMU_SEGSIZE_1T << SLBIE_SSIZE_SHIFT)@h
565 rldimi r7,r9,SLB_VSID_SSIZE_SHIFT,0
566 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
568 /* Update the last bolted SLB. No write barriers are needed
569 * here, provided we only update the current CPU's SLB shadow
570 * buffer.
571 */
572 ld r9,PACA_SLBSHADOWPTR(r13)
573 li r12,0
574 std r12,SLBSHADOW_STACKESID(r9) /* Clear ESID */
575 li r12,SLBSHADOW_STACKVSID
576 STDX_BE r7,r12,r9 /* Save VSID */
577 li r12,SLBSHADOW_STACKESID
578 STDX_BE r0,r12,r9 /* Save ESID */
580 /* No need to check for MMU_FTR_NO_SLBIE_B here, since when
581 * we have 1TB segments, the only CPUs known to have the errata
582 * only support less than 1TB of system memory and we'll never
583 * actually hit this code path.
584 */
586 slbie r6
587 slbie r6 /* Workaround POWER5 < DD2.1 issue */
588 slbmte r7,r0
589 isync
590 2:
591 #endif /* CONFIG_PPC_STD_MMU_64 */
593 CURRENT_THREAD_INFO(r7, r8) /* base of new stack */
594 /* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE
595 because we don't need to leave the 288-byte ABI gap at the
596 top of the kernel stack. */
597 addi r7,r7,THREAD_SIZE-SWITCH_FRAME_SIZE
599 /*
600 * PMU interrupts in radix may come in here. They will use r1, not
601 * PACAKSAVE, so this stack switch will not cause a problem. They
602 * will store to the process stack, which may then be migrated to
603 * another CPU. However the rq lock release on this CPU paired with
604 * the rq lock acquire on the new CPU before the stack becomes
605 * active on the new CPU, will order those stores.
606 */
607 mr r1,r8 /* start using new stack pointer */
608 std r7,PACAKSAVE(r13)
610 ld r6,_CCR(r1)
611 mtcrf 0xFF,r6
613 /* r3-r13 are destroyed -- Cort */
614 REST_8GPRS(14, r1)
615 REST_10GPRS(22, r1)
617 /* convert old thread to its task_struct for return value */
618 addi r3,r3,-THREAD
619 ld r7,_NIP(r1) /* Return to _switch caller in new task */
620 mtlr r7
621 addi r1,r1,SWITCH_FRAME_SIZE
622 blr
624 .align 7
625 _GLOBAL(ret_from_except)
626 ld r11,_TRAP(r1)
627 andi. r0,r11,1
628 bne ret_from_except_lite
629 REST_NVGPRS(r1)
631 _GLOBAL(ret_from_except_lite)
632 /*
633 * Disable interrupts so that current_thread_info()->flags
634 * can't change between when we test it and when we return
635 * from the interrupt.
636 */
637 #ifdef CONFIG_PPC_BOOK3E
638 wrteei 0
639 #else
640 li r10,MSR_RI
641 mtmsrd r10,1 /* Update machine state */
642 #endif /* CONFIG_PPC_BOOK3E */
644 CURRENT_THREAD_INFO(r9, r1)
645 ld r3,_MSR(r1)
646 #ifdef CONFIG_PPC_BOOK3E
647 ld r10,PACACURRENT(r13)
648 #endif /* CONFIG_PPC_BOOK3E */
649 ld r4,TI_FLAGS(r9)
650 andi. r3,r3,MSR_PR
651 beq resume_kernel
652 #ifdef CONFIG_PPC_BOOK3E
653 lwz r3,(THREAD+THREAD_DBCR0)(r10)
654 #endif /* CONFIG_PPC_BOOK3E */
656 /* Check current_thread_info()->flags */
657 andi. r0,r4,_TIF_USER_WORK_MASK
658 bne 1f
659 #ifdef CONFIG_PPC_BOOK3E
660 /*
661 * Check to see if the dbcr0 register is set up to debug.
662 * Use the internal debug mode bit to do this.
663 */
664 andis. r0,r3,DBCR0_IDM@h
665 beq restore
666 mfmsr r0
667 rlwinm r0,r0,0,~MSR_DE /* Clear MSR.DE */
668 mtmsr r0
669 mtspr SPRN_DBCR0,r3
670 li r10, -1
671 mtspr SPRN_DBSR,r10
672 b restore
673 #else
674 addi r3,r1,STACK_FRAME_OVERHEAD
675 bl restore_math
676 b restore
677 #endif
678 1: andi. r0,r4,_TIF_NEED_RESCHED
679 beq 2f
680 bl restore_interrupts
681 SCHEDULE_USER
682 b ret_from_except_lite
683 2:
684 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
685 andi. r0,r4,_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM
686 bne 3f /* only restore TM if nothing else to do */
687 addi r3,r1,STACK_FRAME_OVERHEAD
688 bl restore_tm_state
689 b restore
690 3:
691 #endif
692 bl save_nvgprs
693 /*
694 * Use a non volatile GPR to save and restore our thread_info flags
695 * across the call to restore_interrupts.
696 */
697 mr r30,r4
698 bl restore_interrupts
699 mr r4,r30
700 addi r3,r1,STACK_FRAME_OVERHEAD
701 bl do_notify_resume
702 b ret_from_except
704 resume_kernel:
705 /* check current_thread_info, _TIF_EMULATE_STACK_STORE */
706 andis. r8,r4,_TIF_EMULATE_STACK_STORE@h
707 beq+ 1f
709 addi r8,r1,INT_FRAME_SIZE /* Get the kprobed function entry */
711 ld r3,GPR1(r1)
712 subi r3,r3,INT_FRAME_SIZE /* dst: Allocate a trampoline exception frame */
713 mr r4,r1 /* src: current exception frame */
714 mr r1,r3 /* Reroute the trampoline frame to r1 */
716 /* Copy from the original to the trampoline. */
717 li r5,INT_FRAME_SIZE/8 /* size: INT_FRAME_SIZE */
718 li r6,0 /* start offset: 0 */
719 mtctr r5
720 2: ldx r0,r6,r4
721 stdx r0,r6,r3
722 addi r6,r6,8
723 bdnz 2b
725 /* Do real store operation to complete stdu */
726 ld r5,GPR1(r1)
727 std r8,0(r5)
729 /* Clear _TIF_EMULATE_STACK_STORE flag */
730 lis r11,_TIF_EMULATE_STACK_STORE@h
731 addi r5,r9,TI_FLAGS
732 0: ldarx r4,0,r5
733 andc r4,r4,r11
734 stdcx. r4,0,r5
735 bne- 0b
736 1:
738 #ifdef CONFIG_PREEMPT
739 /* Check if we need to preempt */
740 andi. r0,r4,_TIF_NEED_RESCHED
741 beq+ restore
742 /* Check that preempt_count() == 0 and interrupts are enabled */
743 lwz r8,TI_PREEMPT(r9)
744 cmpwi cr1,r8,0
745 ld r0,SOFTE(r1)
746 cmpdi r0,0
747 crandc eq,cr1*4+eq,eq
748 bne restore
750 /*
751 * Here we are preempting the current task. We want to make
752 * sure we are soft-disabled first and reconcile irq state.
753 */
754 RECONCILE_IRQ_STATE(r3,r4)
755 1: bl preempt_schedule_irq
757 /* Re-test flags and eventually loop */
758 CURRENT_THREAD_INFO(r9, r1)
759 ld r4,TI_FLAGS(r9)
760 andi. r0,r4,_TIF_NEED_RESCHED
761 bne 1b
763 /*
764 * arch_local_irq_restore() from preempt_schedule_irq above may
765 * enable hard interrupt but we really should disable interrupts
766 * when we return from the interrupt, and so that we don't get
767 * interrupted after loading SRR0/1.
768 */
769 #ifdef CONFIG_PPC_BOOK3E
770 wrteei 0
771 #else
772 li r10,MSR_RI
773 mtmsrd r10,1 /* Update machine state */
774 #endif /* CONFIG_PPC_BOOK3E */
775 #endif /* CONFIG_PREEMPT */
777 .globl fast_exc_return_irq
778 fast_exc_return_irq:
779 restore:
780 /*
781 * This is the main kernel exit path. First we check if we
782 * are about to re-enable interrupts
783 */
784 ld r5,SOFTE(r1)
785 lbz r6,PACASOFTIRQEN(r13)
786 cmpwi cr0,r5,0
787 beq .Lrestore_irq_off
789 /* We are enabling, were we already enabled ? Yes, just return */
790 cmpwi cr0,r6,1
791 beq cr0,.Ldo_restore
793 /*
794 * We are about to soft-enable interrupts (we are hard disabled
795 * at this point). We check if there's anything that needs to
796 * be replayed first.
797 */
798 lbz r0,PACAIRQHAPPENED(r13)
799 cmpwi cr0,r0,0
800 bne- .Lrestore_check_irq_replay
802 /*
803 * Get here when nothing happened while soft-disabled, just
804 * soft-enable and move-on. We will hard-enable as a side
805 * effect of rfi
806 */
807 .Lrestore_no_replay:
808 TRACE_ENABLE_INTS
809 li r0,1
810 stb r0,PACASOFTIRQEN(r13);
812 /*
813 * Final return path. BookE is handled in a different file
814 */
815 .Ldo_restore:
816 #ifdef CONFIG_PPC_BOOK3E
817 b exception_return_book3e
818 #else
819 /*
820 * Clear the reservation. If we know the CPU tracks the address of
821 * the reservation then we can potentially save some cycles and use
822 * a larx. On POWER6 and POWER7 this is significantly faster.
823 */
824 BEGIN_FTR_SECTION
825 stdcx. r0,0,r1 /* to clear the reservation */
826 FTR_SECTION_ELSE
827 ldarx r4,0,r1
828 ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
830 /*
831 * Some code path such as load_up_fpu or altivec return directly
832 * here. They run entirely hard disabled and do not alter the
833 * interrupt state. They also don't use lwarx/stwcx. and thus
834 * are known not to leave dangling reservations.
835 */
836 .globl fast_exception_return
837 fast_exception_return:
838 ld r3,_MSR(r1)
839 ld r4,_CTR(r1)
840 ld r0,_LINK(r1)
841 mtctr r4
842 mtlr r0
843 ld r4,_XER(r1)
844 mtspr SPRN_XER,r4
846 REST_8GPRS(5, r1)
848 andi. r0,r3,MSR_RI
849 beq- .Lunrecov_restore
851 /* Load PPR from thread struct before we clear MSR:RI */
852 BEGIN_FTR_SECTION
853 ld r2,PACACURRENT(r13)
854 ld r2,TASKTHREADPPR(r2)
855 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
857 /*
858 * Clear RI before restoring r13. If we are returning to
859 * userspace and we take an exception after restoring r13,
860 * we end up corrupting the userspace r13 value.
861 */
862 li r4,0
863 mtmsrd r4,1
865 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
866 /* TM debug */
867 std r3, PACATMSCRATCH(r13) /* Stash returned-to MSR */
868 #endif
869 /*
870 * r13 is our per cpu area, only restore it if we are returning to
871 * userspace the value stored in the stack frame may belong to
872 * another CPU.
873 */
874 andi. r0,r3,MSR_PR
875 beq 1f
876 BEGIN_FTR_SECTION
877 mtspr SPRN_PPR,r2 /* Restore PPR */
878 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
879 ACCOUNT_CPU_USER_EXIT(r13, r2, r4)
880 REST_GPR(13, r1)
881 1:
882 mtspr SPRN_SRR1,r3
884 ld r2,_CCR(r1)
885 mtcrf 0xFF,r2
886 ld r2,_NIP(r1)
887 mtspr SPRN_SRR0,r2
889 ld r0,GPR0(r1)
890 ld r2,GPR2(r1)
891 ld r3,GPR3(r1)
892 ld r4,GPR4(r1)
893 ld r1,GPR1(r1)
895 rfid
896 b . /* prevent speculative execution */
898 #endif /* CONFIG_PPC_BOOK3E */
900 /*
901 * We are returning to a context with interrupts soft disabled.
902 *
903 * However, we may also about to hard enable, so we need to
904 * make sure that in this case, we also clear PACA_IRQ_HARD_DIS
905 * or that bit can get out of sync and bad things will happen
906 */
907 .Lrestore_irq_off:
908 ld r3,_MSR(r1)
909 lbz r7,PACAIRQHAPPENED(r13)
910 andi. r0,r3,MSR_EE
911 beq 1f
912 rlwinm r7,r7,0,~PACA_IRQ_HARD_DIS
913 stb r7,PACAIRQHAPPENED(r13)
914 1: li r0,0
915 stb r0,PACASOFTIRQEN(r13);
916 TRACE_DISABLE_INTS
917 b .Ldo_restore
919 /*
920 * Something did happen, check if a re-emit is needed
921 * (this also clears paca->irq_happened)
922 */
923 .Lrestore_check_irq_replay:
924 /* XXX: We could implement a fast path here where we check
925 * for irq_happened being just 0x01, in which case we can
926 * clear it and return. That means that we would potentially
927 * miss a decrementer having wrapped all the way around.
928 *
929 * Still, this might be useful for things like hash_page
930 */
931 bl __check_irq_replay
932 cmpwi cr0,r3,0
933 beq .Lrestore_no_replay
935 /*
936 * We need to re-emit an interrupt. We do so by re-using our
937 * existing exception frame. We first change the trap value,
938 * but we need to ensure we preserve the low nibble of it
939 */
940 ld r4,_TRAP(r1)
941 clrldi r4,r4,60
942 or r4,r4,r3
943 std r4,_TRAP(r1)
945 /*
946 * Then find the right handler and call it. Interrupts are
947 * still soft-disabled and we keep them that way.
948 */
949 cmpwi cr0,r3,0x500
950 bne 1f
951 addi r3,r1,STACK_FRAME_OVERHEAD;
952 bl do_IRQ
953 b ret_from_except
954 1: cmpwi cr0,r3,0xe60
955 bne 1f
956 addi r3,r1,STACK_FRAME_OVERHEAD;
957 bl handle_hmi_exception
958 b ret_from_except
959 1: cmpwi cr0,r3,0x900
960 bne 1f
961 addi r3,r1,STACK_FRAME_OVERHEAD;
962 bl timer_interrupt
963 b ret_from_except
964 #ifdef CONFIG_PPC_DOORBELL
965 1:
966 #ifdef CONFIG_PPC_BOOK3E
967 cmpwi cr0,r3,0x280
968 #else
969 cmpwi cr0,r3,0xa00
970 #endif /* CONFIG_PPC_BOOK3E */
971 bne 1f
972 addi r3,r1,STACK_FRAME_OVERHEAD;
973 bl doorbell_exception
974 #endif /* CONFIG_PPC_DOORBELL */
975 1: b ret_from_except /* What else to do here ? */
977 .Lunrecov_restore:
978 addi r3,r1,STACK_FRAME_OVERHEAD
979 bl unrecoverable_exception
980 b .Lunrecov_restore
982 _ASM_NOKPROBE_SYMBOL(ret_from_except);
983 _ASM_NOKPROBE_SYMBOL(ret_from_except_lite);
984 _ASM_NOKPROBE_SYMBOL(resume_kernel);
985 _ASM_NOKPROBE_SYMBOL(fast_exc_return_irq);
986 _ASM_NOKPROBE_SYMBOL(restore);
987 _ASM_NOKPROBE_SYMBOL(fast_exception_return);
990 #ifdef CONFIG_PPC_RTAS
991 /*
992 * On CHRP, the Run-Time Abstraction Services (RTAS) have to be
993 * called with the MMU off.
994 *
995 * In addition, we need to be in 32b mode, at least for now.
996 *
997 * Note: r3 is an input parameter to rtas, so don't trash it...
998 */
999 _GLOBAL(enter_rtas)
1000 mflr r0
1001 std r0,16(r1)
1002 stdu r1,-RTAS_FRAME_SIZE(r1) /* Save SP and create stack space. */
1004 /* Because RTAS is running in 32b mode, it clobbers the high order half
1005 * of all registers that it saves. We therefore save those registers
1006 * RTAS might touch to the stack. (r0, r3-r13 are caller saved)
1007 */
1008 SAVE_GPR(2, r1) /* Save the TOC */
1009 SAVE_GPR(13, r1) /* Save paca */
1010 SAVE_8GPRS(14, r1) /* Save the non-volatiles */
1011 SAVE_10GPRS(22, r1) /* ditto */
1013 mfcr r4
1014 std r4,_CCR(r1)
1015 mfctr r5
1016 std r5,_CTR(r1)
1017 mfspr r6,SPRN_XER
1018 std r6,_XER(r1)
1019 mfdar r7
1020 std r7,_DAR(r1)
1021 mfdsisr r8
1022 std r8,_DSISR(r1)
1024 /* Temporary workaround to clear CR until RTAS can be modified to
1025 * ignore all bits.
1026 */
1027 li r0,0
1028 mtcr r0
1030 #ifdef CONFIG_BUG
1031 /* There is no way it is acceptable to get here with interrupts enabled,
1032 * check it with the asm equivalent of WARN_ON
1033 */
1034 lbz r0,PACASOFTIRQEN(r13)
1035 1: tdnei r0,0
1036 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
1037 #endif
1039 /* Hard-disable interrupts */
1040 mfmsr r6
1041 rldicl r7,r6,48,1
1042 rotldi r7,r7,16
1043 mtmsrd r7,1
1045 /* Unfortunately, the stack pointer and the MSR are also clobbered,
1046 * so they are saved in the PACA which allows us to restore
1047 * our original state after RTAS returns.
1048 */
1049 std r1,PACAR1(r13)
1050 std r6,PACASAVEDMSR(r13)
1052 /* Setup our real return addr */
1053 LOAD_REG_ADDR(r4,rtas_return_loc)
1054 clrldi r4,r4,2 /* convert to realmode address */
1055 mtlr r4
1057 li r0,0
1058 ori r0,r0,MSR_EE|MSR_SE|MSR_BE|MSR_RI
1059 andc r0,r6,r0
1061 li r9,1
1062 rldicr r9,r9,MSR_SF_LG,(63-MSR_SF_LG)
1063 ori r9,r9,MSR_IR|MSR_DR|MSR_FE0|MSR_FE1|MSR_FP|MSR_RI|MSR_LE
1064 andc r6,r0,r9
1066 __enter_rtas:
1067 sync /* disable interrupts so SRR0/1 */
1068 mtmsrd r0 /* don't get trashed */
1070 LOAD_REG_ADDR(r4, rtas)
1071 ld r5,RTASENTRY(r4) /* get the rtas->entry value */
1072 ld r4,RTASBASE(r4) /* get the rtas->base value */
1074 mtspr SPRN_SRR0,r5
1075 mtspr SPRN_SRR1,r6
1076 rfid
1077 b . /* prevent speculative execution */
1079 rtas_return_loc:
1080 FIXUP_ENDIAN
1082 /* relocation is off at this point */
1083 GET_PACA(r4)
1084 clrldi r4,r4,2 /* convert to realmode address */
1086 bcl 20,31,$+4
1087 0: mflr r3
1088 ld r3,(1f-0b)(r3) /* get &rtas_restore_regs */
1090 mfmsr r6
1091 li r0,MSR_RI
1092 andc r6,r6,r0
1093 sync
1094 mtmsrd r6
1096 ld r1,PACAR1(r4) /* Restore our SP */
1097 ld r4,PACASAVEDMSR(r4) /* Restore our MSR */
1099 mtspr SPRN_SRR0,r3
1100 mtspr SPRN_SRR1,r4
1101 rfid
1102 b . /* prevent speculative execution */
1103 _ASM_NOKPROBE_SYMBOL(__enter_rtas)
1104 _ASM_NOKPROBE_SYMBOL(rtas_return_loc)
1106 .align 3
1107 1: .8byte rtas_restore_regs
1109 rtas_restore_regs:
1110 /* relocation is on at this point */
1111 REST_GPR(2, r1) /* Restore the TOC */
1112 REST_GPR(13, r1) /* Restore paca */
1113 REST_8GPRS(14, r1) /* Restore the non-volatiles */
1114 REST_10GPRS(22, r1) /* ditto */
1116 GET_PACA(r13)
1118 ld r4,_CCR(r1)
1119 mtcr r4
1120 ld r5,_CTR(r1)
1121 mtctr r5
1122 ld r6,_XER(r1)
1123 mtspr SPRN_XER,r6
1124 ld r7,_DAR(r1)
1125 mtdar r7
1126 ld r8,_DSISR(r1)
1127 mtdsisr r8
1129 addi r1,r1,RTAS_FRAME_SIZE /* Unstack our frame */
1130 ld r0,16(r1) /* get return address */
1132 mtlr r0
1133 blr /* return to caller */
1135 #endif /* CONFIG_PPC_RTAS */
1137 _GLOBAL(enter_prom)
1138 mflr r0
1139 std r0,16(r1)
1140 stdu r1,-PROM_FRAME_SIZE(r1) /* Save SP and create stack space */
1142 /* Because PROM is running in 32b mode, it clobbers the high order half
1143 * of all registers that it saves. We therefore save those registers
1144 * PROM might touch to the stack. (r0, r3-r13 are caller saved)
1145 */
1146 SAVE_GPR(2, r1)
1147 SAVE_GPR(13, r1)
1148 SAVE_8GPRS(14, r1)
1149 SAVE_10GPRS(22, r1)
1150 mfcr r10
1151 mfmsr r11
1152 std r10,_CCR(r1)
1153 std r11,_MSR(r1)
1155 /* Put PROM address in SRR0 */
1156 mtsrr0 r4
1158 /* Setup our trampoline return addr in LR */
1159 bcl 20,31,$+4
1160 0: mflr r4
1161 addi r4,r4,(1f - 0b)
1162 mtlr r4
1164 /* Prepare a 32-bit mode big endian MSR
1165 */
1166 #ifdef CONFIG_PPC_BOOK3E
1167 rlwinm r11,r11,0,1,31
1168 mtsrr1 r11
1169 rfi
1170 #else /* CONFIG_PPC_BOOK3E */
1171 LOAD_REG_IMMEDIATE(r12, MSR_SF | MSR_ISF | MSR_LE)
1172 andc r11,r11,r12
1173 mtsrr1 r11
1174 rfid
1175 #endif /* CONFIG_PPC_BOOK3E */
1177 1: /* Return from OF */
1178 FIXUP_ENDIAN
1180 /* Just make sure that r1 top 32 bits didn't get
1181 * corrupt by OF
1182 */
1183 rldicl r1,r1,0,32
1185 /* Restore the MSR (back to 64 bits) */
1186 ld r0,_MSR(r1)
1187 MTMSRD(r0)
1188 isync
1190 /* Restore other registers */
1191 REST_GPR(2, r1)
1192 REST_GPR(13, r1)
1193 REST_8GPRS(14, r1)
1194 REST_10GPRS(22, r1)
1195 ld r4,_CCR(r1)
1196 mtcr r4
1198 addi r1,r1,PROM_FRAME_SIZE
1199 ld r0,16(r1)
1200 mtlr r0
1201 blr