| blob | aa1af139d947297fdac16126fdb7b48d4d189a2b |
1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3 * PowerPC version
4 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
5 * Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
6 * Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
7 * Adapted for Power Macintosh by Paul Mackerras.
8 * Low-level exception handlers and MMU support
9 * rewritten by Paul Mackerras.
10 * Copyright (C) 1996 Paul Mackerras.
11 * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
12 *
13 * This file contains the system call entry code, context switch
14 * code, and exception/interrupt return code for PowerPC.
15 */
17 #include <linux/errno.h>
18 #include <linux/err.h>
19 #include <asm/cache.h>
20 #include <asm/unistd.h>
21 #include <asm/processor.h>
22 #include <asm/page.h>
23 #include <asm/mmu.h>
24 #include <asm/thread_info.h>
25 #include <asm/code-patching-asm.h>
26 #include <asm/ppc_asm.h>
27 #include <asm/asm-offsets.h>
28 #include <asm/cputable.h>
29 #include <asm/firmware.h>
30 #include <asm/bug.h>
31 #include <asm/ptrace.h>
32 #include <asm/irqflags.h>
33 #include <asm/hw_irq.h>
34 #include <asm/context_tracking.h>
35 #include <asm/tm.h>
36 #include <asm/ppc-opcode.h>
37 #include <asm/barrier.h>
38 #include <asm/export.h>
39 #include <asm/asm-compat.h>
40 #ifdef CONFIG_PPC_BOOK3S
41 #include <asm/exception-64s.h>
42 #else
43 #include <asm/exception-64e.h>
44 #endif
45 #include <asm/feature-fixups.h>
46 #include <asm/kup.h>
48 /*
49 * System calls.
50 */
51 .section ".toc","aw"
52 SYS_CALL_TABLE:
53 .tc sys_call_table[TC],sys_call_table
55 #ifdef CONFIG_COMPAT
56 COMPAT_SYS_CALL_TABLE:
57 .tc compat_sys_call_table[TC],compat_sys_call_table
58 #endif
60 /* This value is used to mark exception frames on the stack. */
61 exception_marker:
62 .tc ID_EXC_MARKER[TC],STACK_FRAME_REGS_MARKER
64 .section ".text"
65 .align 7
67 #ifdef CONFIG_PPC_BOOK3S
68 .macro system_call_vectored name trapnr
69 .globl system_call_vectored_\name
70 system_call_vectored_\name:
71 _ASM_NOKPROBE_SYMBOL(system_call_vectored_\name)
72 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
73 BEGIN_FTR_SECTION
74 extrdi. r10, r12, 1, (63-MSR_TS_T_LG) /* transaction active? */
75 bne .Ltabort_syscall
76 END_FTR_SECTION_IFSET(CPU_FTR_TM)
77 #endif
78 INTERRUPT_TO_KERNEL
79 mr r10,r1
80 ld r1,PACAKSAVE(r13)
81 std r10,0(r1)
82 std r11,_NIP(r1)
83 std r12,_MSR(r1)
84 std r0,GPR0(r1)
85 std r10,GPR1(r1)
86 std r2,GPR2(r1)
87 ld r2,PACATOC(r13)
88 mfcr r12
89 li r11,0
90 /* Can we avoid saving r3-r8 in common case? */
91 std r3,GPR3(r1)
92 std r4,GPR4(r1)
93 std r5,GPR5(r1)
94 std r6,GPR6(r1)
95 std r7,GPR7(r1)
96 std r8,GPR8(r1)
97 /* Zero r9-r12, this should only be required when restoring all GPRs */
98 std r11,GPR9(r1)
99 std r11,GPR10(r1)
100 std r11,GPR11(r1)
101 std r11,GPR12(r1)
102 std r9,GPR13(r1)
103 SAVE_NVGPRS(r1)
104 std r11,_XER(r1)
105 std r11,_LINK(r1)
106 std r11,_CTR(r1)
108 li r11,\trapnr
109 std r11,_TRAP(r1)
110 std r12,_CCR(r1)
111 std r3,ORIG_GPR3(r1)
112 addi r10,r1,STACK_FRAME_OVERHEAD
113 ld r11,exception_marker@toc(r2)
114 std r11,-16(r10) /* "regshere" marker */
116 BEGIN_FTR_SECTION
117 HMT_MEDIUM
118 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
120 /*
121 * RECONCILE_IRQ_STATE without calling trace_hardirqs_off(), which
122 * would clobber syscall parameters. Also we always enter with IRQs
123 * enabled and nothing pending. system_call_exception() will call
124 * trace_hardirqs_off().
125 *
126 * scv enters with MSR[EE]=1, so don't set PACA_IRQ_HARD_DIS. The
127 * entry vector already sets PACAIRQSOFTMASK to IRQS_ALL_DISABLED.
128 */
130 /* Calling convention has r9 = orig r0, r10 = regs */
131 mr r9,r0
132 bl system_call_exception
134 .Lsyscall_vectored_\name\()_exit:
135 addi r4,r1,STACK_FRAME_OVERHEAD
136 li r5,1 /* scv */
137 bl syscall_exit_prepare
139 ld r2,_CCR(r1)
140 ld r4,_NIP(r1)
141 ld r5,_MSR(r1)
143 BEGIN_FTR_SECTION
144 stdcx. r0,0,r1 /* to clear the reservation */
145 END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
147 BEGIN_FTR_SECTION
148 HMT_MEDIUM_LOW
149 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
151 cmpdi r3,0
152 bne .Lsyscall_vectored_\name\()_restore_regs
154 /* rfscv returns with LR->NIA and CTR->MSR */
155 mtlr r4
156 mtctr r5
158 /* Could zero these as per ABI, but we may consider a stricter ABI
159 * which preserves these if libc implementations can benefit, so
160 * restore them for now until further measurement is done. */
161 ld r0,GPR0(r1)
162 ld r4,GPR4(r1)
163 ld r5,GPR5(r1)
164 ld r6,GPR6(r1)
165 ld r7,GPR7(r1)
166 ld r8,GPR8(r1)
167 /* Zero volatile regs that may contain sensitive kernel data */
168 li r9,0
169 li r10,0
170 li r11,0
171 li r12,0
172 mtspr SPRN_XER,r0
174 /*
175 * We don't need to restore AMR on the way back to userspace for KUAP.
176 * The value of AMR only matters while we're in the kernel.
177 */
178 mtcr r2
179 ld r2,GPR2(r1)
180 ld r3,GPR3(r1)
181 ld r13,GPR13(r1)
182 ld r1,GPR1(r1)
183 RFSCV_TO_USER
184 b . /* prevent speculative execution */
186 .Lsyscall_vectored_\name\()_restore_regs:
187 li r3,0
188 mtmsrd r3,1
189 mtspr SPRN_SRR0,r4
190 mtspr SPRN_SRR1,r5
192 ld r3,_CTR(r1)
193 ld r4,_LINK(r1)
194 ld r5,_XER(r1)
196 REST_NVGPRS(r1)
197 ld r0,GPR0(r1)
198 mtcr r2
199 mtctr r3
200 mtlr r4
201 mtspr SPRN_XER,r5
202 REST_10GPRS(2, r1)
203 REST_2GPRS(12, r1)
204 ld r1,GPR1(r1)
205 RFI_TO_USER
206 .endm
208 system_call_vectored common 0x3000
209 /*
210 * We instantiate another entry copy for the SIGILL variant, with TRAP=0x7ff0
211 * which is tested by system_call_exception when r0 is -1 (as set by vector
212 * entry code).
213 */
214 system_call_vectored sigill 0x7ff0
217 /*
218 * Entered via kernel return set up by kernel/sstep.c, must match entry regs
219 */
220 .globl system_call_vectored_emulate
221 system_call_vectored_emulate:
222 _ASM_NOKPROBE_SYMBOL(system_call_vectored_emulate)
223 li r10,IRQS_ALL_DISABLED
224 stb r10,PACAIRQSOFTMASK(r13)
225 b system_call_vectored_common
226 #endif
228 .balign IFETCH_ALIGN_BYTES
229 .globl system_call_common
230 system_call_common:
231 _ASM_NOKPROBE_SYMBOL(system_call_common)
232 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
233 BEGIN_FTR_SECTION
234 extrdi. r10, r12, 1, (63-MSR_TS_T_LG) /* transaction active? */
235 bne .Ltabort_syscall
236 END_FTR_SECTION_IFSET(CPU_FTR_TM)
237 #endif
238 mr r10,r1
239 ld r1,PACAKSAVE(r13)
240 std r10,0(r1)
241 std r11,_NIP(r1)
242 std r12,_MSR(r1)
243 std r0,GPR0(r1)
244 std r10,GPR1(r1)
245 std r2,GPR2(r1)
246 #ifdef CONFIG_PPC_FSL_BOOK3E
247 START_BTB_FLUSH_SECTION
248 BTB_FLUSH(r10)
249 END_BTB_FLUSH_SECTION
250 #endif
251 ld r2,PACATOC(r13)
252 mfcr r12
253 li r11,0
254 /* Can we avoid saving r3-r8 in common case? */
255 std r3,GPR3(r1)
256 std r4,GPR4(r1)
257 std r5,GPR5(r1)
258 std r6,GPR6(r1)
259 std r7,GPR7(r1)
260 std r8,GPR8(r1)
261 /* Zero r9-r12, this should only be required when restoring all GPRs */
262 std r11,GPR9(r1)
263 std r11,GPR10(r1)
264 std r11,GPR11(r1)
265 std r11,GPR12(r1)
266 std r9,GPR13(r1)
267 SAVE_NVGPRS(r1)
268 std r11,_XER(r1)
269 std r11,_CTR(r1)
270 mflr r10
272 /*
273 * This clears CR0.SO (bit 28), which is the error indication on
274 * return from this system call.
275 */
276 rldimi r12,r11,28,(63-28)
277 li r11,0xc00
278 std r10,_LINK(r1)
279 std r11,_TRAP(r1)
280 std r12,_CCR(r1)
281 std r3,ORIG_GPR3(r1)
282 addi r10,r1,STACK_FRAME_OVERHEAD
283 ld r11,exception_marker@toc(r2)
284 std r11,-16(r10) /* "regshere" marker */
286 /*
287 * RECONCILE_IRQ_STATE without calling trace_hardirqs_off(), which
288 * would clobber syscall parameters. Also we always enter with IRQs
289 * enabled and nothing pending. system_call_exception() will call
290 * trace_hardirqs_off().
291 */
292 li r11,IRQS_ALL_DISABLED
293 li r12,PACA_IRQ_HARD_DIS
294 stb r11,PACAIRQSOFTMASK(r13)
295 stb r12,PACAIRQHAPPENED(r13)
297 /* Calling convention has r9 = orig r0, r10 = regs */
298 mr r9,r0
299 bl system_call_exception
301 .Lsyscall_exit:
302 addi r4,r1,STACK_FRAME_OVERHEAD
303 li r5,0 /* !scv */
304 bl syscall_exit_prepare
306 ld r2,_CCR(r1)
307 ld r4,_NIP(r1)
308 ld r5,_MSR(r1)
309 ld r6,_LINK(r1)
311 BEGIN_FTR_SECTION
312 stdcx. r0,0,r1 /* to clear the reservation */
313 END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
315 mtspr SPRN_SRR0,r4
316 mtspr SPRN_SRR1,r5
317 mtlr r6
319 cmpdi r3,0
320 bne .Lsyscall_restore_regs
321 /* Zero volatile regs that may contain sensitive kernel data */
322 li r0,0
323 li r4,0
324 li r5,0
325 li r6,0
326 li r7,0
327 li r8,0
328 li r9,0
329 li r10,0
330 li r11,0
331 li r12,0
332 mtctr r0
333 mtspr SPRN_XER,r0
334 .Lsyscall_restore_regs_cont:
336 BEGIN_FTR_SECTION
337 HMT_MEDIUM_LOW
338 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
340 /*
341 * We don't need to restore AMR on the way back to userspace for KUAP.
342 * The value of AMR only matters while we're in the kernel.
343 */
344 mtcr r2
345 ld r2,GPR2(r1)
346 ld r3,GPR3(r1)
347 ld r13,GPR13(r1)
348 ld r1,GPR1(r1)
349 RFI_TO_USER
350 b . /* prevent speculative execution */
352 .Lsyscall_restore_regs:
353 ld r3,_CTR(r1)
354 ld r4,_XER(r1)
355 REST_NVGPRS(r1)
356 mtctr r3
357 mtspr SPRN_XER,r4
358 ld r0,GPR0(r1)
359 REST_8GPRS(4, r1)
360 ld r12,GPR12(r1)
361 b .Lsyscall_restore_regs_cont
363 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
364 .Ltabort_syscall:
365 /* Firstly we need to enable TM in the kernel */
366 mfmsr r10
367 li r9, 1
368 rldimi r10, r9, MSR_TM_LG, 63-MSR_TM_LG
369 mtmsrd r10, 0
371 /* tabort, this dooms the transaction, nothing else */
372 li r9, (TM_CAUSE_SYSCALL|TM_CAUSE_PERSISTENT)
373 TABORT(R9)
375 /*
376 * Return directly to userspace. We have corrupted user register state,
377 * but userspace will never see that register state. Execution will
378 * resume after the tbegin of the aborted transaction with the
379 * checkpointed register state.
380 */
381 li r9, MSR_RI
382 andc r10, r10, r9
383 mtmsrd r10, 1
384 mtspr SPRN_SRR0, r11
385 mtspr SPRN_SRR1, r12
386 RFI_TO_USER
387 b . /* prevent speculative execution */
388 #endif
390 #ifdef CONFIG_PPC_BOOK3S
391 _GLOBAL(ret_from_fork_scv)
392 bl schedule_tail
393 REST_NVGPRS(r1)
394 li r3,0 /* fork() return value */
395 b .Lsyscall_vectored_common_exit
396 #endif
398 _GLOBAL(ret_from_fork)
399 bl schedule_tail
400 REST_NVGPRS(r1)
401 li r3,0 /* fork() return value */
402 b .Lsyscall_exit
404 _GLOBAL(ret_from_kernel_thread)
405 bl schedule_tail
406 REST_NVGPRS(r1)
407 mtctr r14
408 mr r3,r15
409 #ifdef PPC64_ELF_ABI_v2
410 mr r12,r14
411 #endif
412 bctrl
413 li r3,0
414 b .Lsyscall_exit
416 #ifdef CONFIG_PPC_BOOK3E
417 /* Save non-volatile GPRs, if not already saved. */
418 _GLOBAL(save_nvgprs)
419 ld r11,_TRAP(r1)
420 andi. r0,r11,1
421 beqlr-
422 SAVE_NVGPRS(r1)
423 clrrdi r0,r11,1
424 std r0,_TRAP(r1)
425 blr
426 _ASM_NOKPROBE_SYMBOL(save_nvgprs);
427 #endif
429 #ifdef CONFIG_PPC_BOOK3S_64
431 #define FLUSH_COUNT_CACHE \
432 1: nop; \
433 patch_site 1b, patch__call_flush_branch_caches1; \
434 1: nop; \
435 patch_site 1b, patch__call_flush_branch_caches2; \
436 1: nop; \
437 patch_site 1b, patch__call_flush_branch_caches3
439 .macro nops number
440 .rept \number
441 nop
442 .endr
443 .endm
445 .balign 32
446 .global flush_branch_caches
447 flush_branch_caches:
448 /* Save LR into r9 */
449 mflr r9
451 // Flush the link stack
452 .rept 64
453 bl .+4
454 .endr
455 b 1f
456 nops 6
458 .balign 32
459 /* Restore LR */
460 1: mtlr r9
462 // If we're just flushing the link stack, return here
463 3: nop
464 patch_site 3b patch__flush_link_stack_return
466 li r9,0x7fff
467 mtctr r9
469 PPC_BCCTR_FLUSH
471 2: nop
472 patch_site 2b patch__flush_count_cache_return
474 nops 3
476 .rept 278
477 .balign 32
478 PPC_BCCTR_FLUSH
479 nops 7
480 .endr
482 blr
483 #else
484 #define FLUSH_COUNT_CACHE
485 #endif /* CONFIG_PPC_BOOK3S_64 */
487 /*
488 * This routine switches between two different tasks. The process
489 * state of one is saved on its kernel stack. Then the state
490 * of the other is restored from its kernel stack. The memory
491 * management hardware is updated to the second process's state.
492 * Finally, we can return to the second process, via interrupt_return.
493 * On entry, r3 points to the THREAD for the current task, r4
494 * points to the THREAD for the new task.
495 *
496 * Note: there are two ways to get to the "going out" portion
497 * of this code; either by coming in via the entry (_switch)
498 * or via "fork" which must set up an environment equivalent
499 * to the "_switch" path. If you change this you'll have to change
500 * the fork code also.
501 *
502 * The code which creates the new task context is in 'copy_thread'
503 * in arch/powerpc/kernel/process.c
504 */
505 .align 7
506 _GLOBAL(_switch)
507 mflr r0
508 std r0,16(r1)
509 stdu r1,-SWITCH_FRAME_SIZE(r1)
510 /* r3-r13 are caller saved -- Cort */
511 SAVE_NVGPRS(r1)
512 std r0,_NIP(r1) /* Return to switch caller */
513 mfcr r23
514 std r23,_CCR(r1)
515 std r1,KSP(r3) /* Set old stack pointer */
517 kuap_check_amr r9, r10
519 FLUSH_COUNT_CACHE /* Clobbers r9, ctr */
521 /*
522 * On SMP kernels, care must be taken because a task may be
523 * scheduled off CPUx and on to CPUy. Memory ordering must be
524 * considered.
525 *
526 * Cacheable stores on CPUx will be visible when the task is
527 * scheduled on CPUy by virtue of the core scheduler barriers
528 * (see "Notes on Program-Order guarantees on SMP systems." in
529 * kernel/sched/core.c).
530 *
531 * Uncacheable stores in the case of involuntary preemption must
532 * be taken care of. The smp_mb__after_spinlock() in __schedule()
533 * is implemented as hwsync on powerpc, which orders MMIO too. So
534 * long as there is an hwsync in the context switch path, it will
535 * be executed on the source CPU after the task has performed
536 * all MMIO ops on that CPU, and on the destination CPU before the
537 * task performs any MMIO ops there.
538 */
540 /*
541 * The kernel context switch path must contain a spin_lock,
542 * which contains larx/stcx, which will clear any reservation
543 * of the task being switched.
544 */
545 #ifdef CONFIG_PPC_BOOK3S
546 /* Cancel all explict user streams as they will have no use after context
547 * switch and will stop the HW from creating streams itself
548 */
549 DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r6)
550 #endif
552 addi r6,r4,-THREAD /* Convert THREAD to 'current' */
553 std r6,PACACURRENT(r13) /* Set new 'current' */
554 #if defined(CONFIG_STACKPROTECTOR)
555 ld r6, TASK_CANARY(r6)
556 std r6, PACA_CANARY(r13)
557 #endif
559 ld r8,KSP(r4) /* new stack pointer */
560 #ifdef CONFIG_PPC_BOOK3S_64
561 BEGIN_MMU_FTR_SECTION
562 b 2f
563 END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX)
564 BEGIN_FTR_SECTION
565 clrrdi r6,r8,28 /* get its ESID */
566 clrrdi r9,r1,28 /* get current sp ESID */
567 FTR_SECTION_ELSE
568 clrrdi r6,r8,40 /* get its 1T ESID */
569 clrrdi r9,r1,40 /* get current sp 1T ESID */
570 ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_1T_SEGMENT)
571 clrldi. r0,r6,2 /* is new ESID c00000000? */
572 cmpd cr1,r6,r9 /* or is new ESID the same as current ESID? */
573 cror eq,4*cr1+eq,eq
574 beq 2f /* if yes, don't slbie it */
576 /* Bolt in the new stack SLB entry */
577 ld r7,KSP_VSID(r4) /* Get new stack's VSID */
578 oris r0,r6,(SLB_ESID_V)@h
579 ori r0,r0,(SLB_NUM_BOLTED-1)@l
580 BEGIN_FTR_SECTION
581 li r9,MMU_SEGSIZE_1T /* insert B field */
582 oris r6,r6,(MMU_SEGSIZE_1T << SLBIE_SSIZE_SHIFT)@h
583 rldimi r7,r9,SLB_VSID_SSIZE_SHIFT,0
584 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
586 /* Update the last bolted SLB. No write barriers are needed
587 * here, provided we only update the current CPU's SLB shadow
588 * buffer.
589 */
590 ld r9,PACA_SLBSHADOWPTR(r13)
591 li r12,0
592 std r12,SLBSHADOW_STACKESID(r9) /* Clear ESID */
593 li r12,SLBSHADOW_STACKVSID
594 STDX_BE r7,r12,r9 /* Save VSID */
595 li r12,SLBSHADOW_STACKESID
596 STDX_BE r0,r12,r9 /* Save ESID */
598 /* No need to check for MMU_FTR_NO_SLBIE_B here, since when
599 * we have 1TB segments, the only CPUs known to have the errata
600 * only support less than 1TB of system memory and we'll never
601 * actually hit this code path.
602 */
604 isync
605 slbie r6
606 BEGIN_FTR_SECTION
607 slbie r6 /* Workaround POWER5 < DD2.1 issue */
608 END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
609 slbmte r7,r0
610 isync
611 2:
612 #endif /* CONFIG_PPC_BOOK3S_64 */
614 clrrdi r7, r8, THREAD_SHIFT /* base of new stack */
615 /* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE
616 because we don't need to leave the 288-byte ABI gap at the
617 top of the kernel stack. */
618 addi r7,r7,THREAD_SIZE-SWITCH_FRAME_SIZE
620 /*
621 * PMU interrupts in radix may come in here. They will use r1, not
622 * PACAKSAVE, so this stack switch will not cause a problem. They
623 * will store to the process stack, which may then be migrated to
624 * another CPU. However the rq lock release on this CPU paired with
625 * the rq lock acquire on the new CPU before the stack becomes
626 * active on the new CPU, will order those stores.
627 */
628 mr r1,r8 /* start using new stack pointer */
629 std r7,PACAKSAVE(r13)
631 ld r6,_CCR(r1)
632 mtcrf 0xFF,r6
634 /* r3-r13 are destroyed -- Cort */
635 REST_NVGPRS(r1)
637 /* convert old thread to its task_struct for return value */
638 addi r3,r3,-THREAD
639 ld r7,_NIP(r1) /* Return to _switch caller in new task */
640 mtlr r7
641 addi r1,r1,SWITCH_FRAME_SIZE
642 blr
644 #ifdef CONFIG_PPC_BOOK3S
645 /*
646 * If MSR EE/RI was never enabled, IRQs not reconciled, NVGPRs not
647 * touched, no exit work created, then this can be used.
648 */
649 .balign IFETCH_ALIGN_BYTES
650 .globl fast_interrupt_return
651 fast_interrupt_return:
652 _ASM_NOKPROBE_SYMBOL(fast_interrupt_return)
653 kuap_check_amr r3, r4
654 ld r5,_MSR(r1)
655 andi. r0,r5,MSR_PR
656 bne .Lfast_user_interrupt_return_amr
657 kuap_kernel_restore r3, r4
658 andi. r0,r5,MSR_RI
659 li r3,0 /* 0 return value, no EMULATE_STACK_STORE */
660 bne+ .Lfast_kernel_interrupt_return
661 addi r3,r1,STACK_FRAME_OVERHEAD
662 bl unrecoverable_exception
663 b . /* should not get here */
665 .balign IFETCH_ALIGN_BYTES
666 .globl interrupt_return
667 interrupt_return:
668 _ASM_NOKPROBE_SYMBOL(interrupt_return)
669 ld r4,_MSR(r1)
670 andi. r0,r4,MSR_PR
671 beq .Lkernel_interrupt_return
672 addi r3,r1,STACK_FRAME_OVERHEAD
673 bl interrupt_exit_user_prepare
674 cmpdi r3,0
675 bne- .Lrestore_nvgprs
677 .Lfast_user_interrupt_return_amr:
678 kuap_user_restore r3, r4
679 .Lfast_user_interrupt_return:
680 ld r11,_NIP(r1)
681 ld r12,_MSR(r1)
682 BEGIN_FTR_SECTION
683 ld r10,_PPR(r1)
684 mtspr SPRN_PPR,r10
685 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
686 mtspr SPRN_SRR0,r11
687 mtspr SPRN_SRR1,r12
689 BEGIN_FTR_SECTION
690 stdcx. r0,0,r1 /* to clear the reservation */
691 FTR_SECTION_ELSE
692 ldarx r0,0,r1
693 ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
695 ld r3,_CCR(r1)
696 ld r4,_LINK(r1)
697 ld r5,_CTR(r1)
698 ld r6,_XER(r1)
699 li r0,0
701 REST_4GPRS(7, r1)
702 REST_2GPRS(11, r1)
703 REST_GPR(13, r1)
705 mtcr r3
706 mtlr r4
707 mtctr r5
708 mtspr SPRN_XER,r6
710 REST_4GPRS(2, r1)
711 REST_GPR(6, r1)
712 REST_GPR(0, r1)
713 REST_GPR(1, r1)
714 RFI_TO_USER
715 b . /* prevent speculative execution */
717 .Lrestore_nvgprs:
718 REST_NVGPRS(r1)
719 b .Lfast_user_interrupt_return
721 .balign IFETCH_ALIGN_BYTES
722 .Lkernel_interrupt_return:
723 addi r3,r1,STACK_FRAME_OVERHEAD
724 bl interrupt_exit_kernel_prepare
726 .Lfast_kernel_interrupt_return:
727 cmpdi cr1,r3,0
728 ld r11,_NIP(r1)
729 ld r12,_MSR(r1)
730 mtspr SPRN_SRR0,r11
731 mtspr SPRN_SRR1,r12
733 BEGIN_FTR_SECTION
734 stdcx. r0,0,r1 /* to clear the reservation */
735 FTR_SECTION_ELSE
736 ldarx r0,0,r1
737 ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
739 ld r3,_LINK(r1)
740 ld r4,_CTR(r1)
741 ld r5,_XER(r1)
742 ld r6,_CCR(r1)
743 li r0,0
745 REST_4GPRS(7, r1)
746 REST_2GPRS(11, r1)
748 mtlr r3
749 mtctr r4
750 mtspr SPRN_XER,r5
752 /*
753 * Leaving a stale exception_marker on the stack can confuse
754 * the reliable stack unwinder later on. Clear it.
755 */
756 std r0,STACK_FRAME_OVERHEAD-16(r1)
758 REST_4GPRS(2, r1)
760 bne- cr1,1f /* emulate stack store */
761 mtcr r6
762 REST_GPR(6, r1)
763 REST_GPR(0, r1)
764 REST_GPR(1, r1)
765 RFI_TO_KERNEL
766 b . /* prevent speculative execution */
768 1: /*
769 * Emulate stack store with update. New r1 value was already calculated
770 * and updated in our interrupt regs by emulate_loadstore, but we can't
771 * store the previous value of r1 to the stack before re-loading our
772 * registers from it, otherwise they could be clobbered. Use
773 * PACA_EXGEN as temporary storage to hold the store data, as
774 * interrupts are disabled here so it won't be clobbered.
775 */
776 mtcr r6
777 std r9,PACA_EXGEN+0(r13)
778 addi r9,r1,INT_FRAME_SIZE /* get original r1 */
779 REST_GPR(6, r1)
780 REST_GPR(0, r1)
781 REST_GPR(1, r1)
782 std r9,0(r1) /* perform store component of stdu */
783 ld r9,PACA_EXGEN+0(r13)
785 RFI_TO_KERNEL
786 b . /* prevent speculative execution */
787 #endif /* CONFIG_PPC_BOOK3S */
789 #ifdef CONFIG_PPC_RTAS
790 /*
791 * On CHRP, the Run-Time Abstraction Services (RTAS) have to be
792 * called with the MMU off.
793 *
794 * In addition, we need to be in 32b mode, at least for now.
795 *
796 * Note: r3 is an input parameter to rtas, so don't trash it...
797 */
798 _GLOBAL(enter_rtas)
799 mflr r0
800 std r0,16(r1)
801 stdu r1,-SWITCH_FRAME_SIZE(r1) /* Save SP and create stack space. */
803 /* Because RTAS is running in 32b mode, it clobbers the high order half
804 * of all registers that it saves. We therefore save those registers
805 * RTAS might touch to the stack. (r0, r3-r13 are caller saved)
806 */
807 SAVE_GPR(2, r1) /* Save the TOC */
808 SAVE_GPR(13, r1) /* Save paca */
809 SAVE_NVGPRS(r1) /* Save the non-volatiles */
811 mfcr r4
812 std r4,_CCR(r1)
813 mfctr r5
814 std r5,_CTR(r1)
815 mfspr r6,SPRN_XER
816 std r6,_XER(r1)
817 mfdar r7
818 std r7,_DAR(r1)
819 mfdsisr r8
820 std r8,_DSISR(r1)
822 /* Temporary workaround to clear CR until RTAS can be modified to
823 * ignore all bits.
824 */
825 li r0,0
826 mtcr r0
828 #ifdef CONFIG_BUG
829 /* There is no way it is acceptable to get here with interrupts enabled,
830 * check it with the asm equivalent of WARN_ON
831 */
832 lbz r0,PACAIRQSOFTMASK(r13)
833 1: tdeqi r0,IRQS_ENABLED
834 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
835 #endif
837 /* Hard-disable interrupts */
838 mfmsr r6
839 rldicl r7,r6,48,1
840 rotldi r7,r7,16
841 mtmsrd r7,1
843 /* Unfortunately, the stack pointer and the MSR are also clobbered,
844 * so they are saved in the PACA which allows us to restore
845 * our original state after RTAS returns.
846 */
847 std r1,PACAR1(r13)
848 std r6,PACASAVEDMSR(r13)
850 /* Setup our real return addr */
851 LOAD_REG_ADDR(r4,rtas_return_loc)
852 clrldi r4,r4,2 /* convert to realmode address */
853 mtlr r4
855 li r0,0
856 ori r0,r0,MSR_EE|MSR_SE|MSR_BE|MSR_RI
857 andc r0,r6,r0
859 li r9,1
860 rldicr r9,r9,MSR_SF_LG,(63-MSR_SF_LG)
861 ori r9,r9,MSR_IR|MSR_DR|MSR_FE0|MSR_FE1|MSR_FP|MSR_RI|MSR_LE
862 andc r6,r0,r9
864 __enter_rtas:
865 sync /* disable interrupts so SRR0/1 */
866 mtmsrd r0 /* don't get trashed */
868 LOAD_REG_ADDR(r4, rtas)
869 ld r5,RTASENTRY(r4) /* get the rtas->entry value */
870 ld r4,RTASBASE(r4) /* get the rtas->base value */
872 mtspr SPRN_SRR0,r5
873 mtspr SPRN_SRR1,r6
874 RFI_TO_KERNEL
875 b . /* prevent speculative execution */
877 rtas_return_loc:
878 FIXUP_ENDIAN
880 /*
881 * Clear RI and set SF before anything.
882 */
883 mfmsr r6
884 li r0,MSR_RI
885 andc r6,r6,r0
886 sldi r0,r0,(MSR_SF_LG - MSR_RI_LG)
887 or r6,r6,r0
888 sync
889 mtmsrd r6
891 /* relocation is off at this point */
892 GET_PACA(r4)
893 clrldi r4,r4,2 /* convert to realmode address */
895 bcl 20,31,$+4
896 0: mflr r3
897 ld r3,(1f-0b)(r3) /* get &rtas_restore_regs */
899 ld r1,PACAR1(r4) /* Restore our SP */
900 ld r4,PACASAVEDMSR(r4) /* Restore our MSR */
902 mtspr SPRN_SRR0,r3
903 mtspr SPRN_SRR1,r4
904 RFI_TO_KERNEL
905 b . /* prevent speculative execution */
906 _ASM_NOKPROBE_SYMBOL(__enter_rtas)
907 _ASM_NOKPROBE_SYMBOL(rtas_return_loc)
909 .align 3
910 1: .8byte rtas_restore_regs
912 rtas_restore_regs:
913 /* relocation is on at this point */
914 REST_GPR(2, r1) /* Restore the TOC */
915 REST_GPR(13, r1) /* Restore paca */
916 REST_NVGPRS(r1) /* Restore the non-volatiles */
918 GET_PACA(r13)
920 ld r4,_CCR(r1)
921 mtcr r4
922 ld r5,_CTR(r1)
923 mtctr r5
924 ld r6,_XER(r1)
925 mtspr SPRN_XER,r6
926 ld r7,_DAR(r1)
927 mtdar r7
928 ld r8,_DSISR(r1)
929 mtdsisr r8
931 addi r1,r1,SWITCH_FRAME_SIZE /* Unstack our frame */
932 ld r0,16(r1) /* get return address */
934 mtlr r0
935 blr /* return to caller */
937 #endif /* CONFIG_PPC_RTAS */
939 _GLOBAL(enter_prom)
940 mflr r0
941 std r0,16(r1)
942 stdu r1,-SWITCH_FRAME_SIZE(r1) /* Save SP and create stack space */
944 /* Because PROM is running in 32b mode, it clobbers the high order half
945 * of all registers that it saves. We therefore save those registers
946 * PROM might touch to the stack. (r0, r3-r13 are caller saved)
947 */
948 SAVE_GPR(2, r1)
949 SAVE_GPR(13, r1)
950 SAVE_NVGPRS(r1)
951 mfcr r10
952 mfmsr r11
953 std r10,_CCR(r1)
954 std r11,_MSR(r1)
956 /* Put PROM address in SRR0 */
957 mtsrr0 r4
959 /* Setup our trampoline return addr in LR */
960 bcl 20,31,$+4
961 0: mflr r4
962 addi r4,r4,(1f - 0b)
963 mtlr r4
965 /* Prepare a 32-bit mode big endian MSR
966 */
967 #ifdef CONFIG_PPC_BOOK3E
968 rlwinm r11,r11,0,1,31
969 mtsrr1 r11
970 rfi
971 #else /* CONFIG_PPC_BOOK3E */
972 LOAD_REG_IMMEDIATE(r12, MSR_SF | MSR_LE)
973 andc r11,r11,r12
974 mtsrr1 r11
975 RFI_TO_KERNEL
976 #endif /* CONFIG_PPC_BOOK3E */
978 1: /* Return from OF */
979 FIXUP_ENDIAN
981 /* Just make sure that r1 top 32 bits didn't get
982 * corrupt by OF
983 */
984 rldicl r1,r1,0,32
986 /* Restore the MSR (back to 64 bits) */
987 ld r0,_MSR(r1)
988 MTMSRD(r0)
989 isync
991 /* Restore other registers */
992 REST_GPR(2, r1)
993 REST_GPR(13, r1)
994 REST_NVGPRS(r1)
995 ld r4,_CCR(r1)
996 mtcr r4
998 addi r1,r1,SWITCH_FRAME_SIZE
999 ld r0,16(r1)
1000 mtlr r0
1001 blr