| blob | 6ba675b0cf7da92a6d6b7c22cabd29e9c7aa012d |
1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3 * PowerPC version
4 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
5 * Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
6 * Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
7 * Adapted for Power Macintosh by Paul Mackerras.
8 * Low-level exception handlers and MMU support
9 * rewritten by Paul Mackerras.
10 * Copyright (C) 1996 Paul Mackerras.
11 * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
12 *
13 * This file contains the system call entry code, context switch
14 * code, and exception/interrupt return code for PowerPC.
15 */
17 #include <linux/errno.h>
18 #include <linux/err.h>
19 #include <asm/unistd.h>
20 #include <asm/processor.h>
21 #include <asm/page.h>
22 #include <asm/mmu.h>
23 #include <asm/thread_info.h>
24 #include <asm/code-patching-asm.h>
25 #include <asm/ppc_asm.h>
26 #include <asm/asm-offsets.h>
27 #include <asm/cputable.h>
28 #include <asm/firmware.h>
29 #include <asm/bug.h>
30 #include <asm/ptrace.h>
31 #include <asm/irqflags.h>
32 #include <asm/hw_irq.h>
33 #include <asm/context_tracking.h>
34 #include <asm/tm.h>
35 #include <asm/ppc-opcode.h>
36 #include <asm/barrier.h>
37 #include <asm/export.h>
38 #include <asm/asm-compat.h>
39 #ifdef CONFIG_PPC_BOOK3S
40 #include <asm/exception-64s.h>
41 #else
42 #include <asm/exception-64e.h>
43 #endif
44 #include <asm/feature-fixups.h>
45 #include <asm/kup.h>
47 /*
48 * System calls.
49 */
50 .section ".toc","aw"
51 SYS_CALL_TABLE:
52 .tc sys_call_table[TC],sys_call_table
54 COMPAT_SYS_CALL_TABLE:
55 .tc compat_sys_call_table[TC],compat_sys_call_table
57 /* This value is used to mark exception frames on the stack. */
58 exception_marker:
59 .tc ID_EXC_MARKER[TC],STACK_FRAME_REGS_MARKER
61 .section ".text"
62 .align 7
64 .globl system_call_common
65 system_call_common:
66 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
67 BEGIN_FTR_SECTION
68 extrdi. r10, r12, 1, (63-MSR_TS_T_LG) /* transaction active? */
69 bne .Ltabort_syscall
70 END_FTR_SECTION_IFSET(CPU_FTR_TM)
71 #endif
72 mr r10,r1
73 ld r1,PACAKSAVE(r13)
74 std r10,0(r1)
75 std r11,_NIP(r1)
76 std r12,_MSR(r1)
77 std r0,GPR0(r1)
78 std r10,GPR1(r1)
79 #ifdef CONFIG_PPC_FSL_BOOK3E
80 START_BTB_FLUSH_SECTION
81 BTB_FLUSH(r10)
82 END_BTB_FLUSH_SECTION
83 #endif
84 ACCOUNT_CPU_USER_ENTRY(r13, r10, r11)
85 std r2,GPR2(r1)
86 std r3,GPR3(r1)
87 mfcr r2
88 std r4,GPR4(r1)
89 std r5,GPR5(r1)
90 std r6,GPR6(r1)
91 std r7,GPR7(r1)
92 std r8,GPR8(r1)
93 li r11,0
94 std r11,GPR9(r1)
95 std r11,GPR10(r1)
96 std r11,GPR11(r1)
97 std r11,GPR12(r1)
98 std r11,_XER(r1)
99 std r11,_CTR(r1)
100 std r9,GPR13(r1)
101 mflr r10
102 /*
103 * This clears CR0.SO (bit 28), which is the error indication on
104 * return from this system call.
105 */
106 rldimi r2,r11,28,(63-28)
107 li r11,0xc01
108 std r10,_LINK(r1)
109 std r11,_TRAP(r1)
110 std r3,ORIG_GPR3(r1)
111 std r2,_CCR(r1)
112 ld r2,PACATOC(r13)
113 addi r9,r1,STACK_FRAME_OVERHEAD
114 ld r11,exception_marker@toc(r2)
115 std r11,-16(r9) /* "regshere" marker */
117 kuap_check_amr r10, r11
119 #if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) && defined(CONFIG_PPC_SPLPAR)
120 BEGIN_FW_FTR_SECTION
121 /* see if there are any DTL entries to process */
122 ld r10,PACALPPACAPTR(r13) /* get ptr to VPA */
123 ld r11,PACA_DTL_RIDX(r13) /* get log read index */
124 addi r10,r10,LPPACA_DTLIDX
125 LDX_BE r10,0,r10 /* get log write index */
126 cmpd r11,r10
127 beq+ 33f
128 bl accumulate_stolen_time
129 REST_GPR(0,r1)
130 REST_4GPRS(3,r1)
131 REST_2GPRS(7,r1)
132 addi r9,r1,STACK_FRAME_OVERHEAD
133 33:
134 END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR)
135 #endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE && CONFIG_PPC_SPLPAR */
137 /*
138 * A syscall should always be called with interrupts enabled
139 * so we just unconditionally hard-enable here. When some kind
140 * of irq tracing is used, we additionally check that condition
141 * is correct
142 */
143 #if defined(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG) && defined(CONFIG_BUG)
144 lbz r10,PACAIRQSOFTMASK(r13)
145 1: tdnei r10,IRQS_ENABLED
146 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
147 #endif
149 #ifdef CONFIG_PPC_BOOK3E
150 wrteei 1
151 #else
152 li r11,MSR_RI
153 ori r11,r11,MSR_EE
154 mtmsrd r11,1
155 #endif /* CONFIG_PPC_BOOK3E */
157 system_call: /* label this so stack traces look sane */
158 /* We do need to set SOFTE in the stack frame or the return
159 * from interrupt will be painful
160 */
161 li r10,IRQS_ENABLED
162 std r10,SOFTE(r1)
164 ld r11, PACA_THREAD_INFO(r13)
165 ld r10,TI_FLAGS(r11)
166 andi. r11,r10,_TIF_SYSCALL_DOTRACE
167 bne .Lsyscall_dotrace /* does not return */
168 cmpldi 0,r0,NR_syscalls
169 bge- .Lsyscall_enosys
171 .Lsyscall:
172 /*
173 * Need to vector to 32 Bit or default sys_call_table here,
174 * based on caller's run-mode / personality.
175 */
176 ld r11,SYS_CALL_TABLE@toc(2)
177 andis. r10,r10,_TIF_32BIT@h
178 beq 15f
179 ld r11,COMPAT_SYS_CALL_TABLE@toc(2)
180 clrldi r3,r3,32
181 clrldi r4,r4,32
182 clrldi r5,r5,32
183 clrldi r6,r6,32
184 clrldi r7,r7,32
185 clrldi r8,r8,32
186 15:
187 slwi r0,r0,3
189 barrier_nospec_asm
190 /*
191 * Prevent the load of the handler below (based on the user-passed
192 * system call number) being speculatively executed until the test
193 * against NR_syscalls and branch to .Lsyscall_enosys above has
194 * committed.
195 */
197 ldx r12,r11,r0 /* Fetch system call handler [ptr] */
198 mtctr r12
199 bctrl /* Call handler */
201 /* syscall_exit can exit to kernel mode, via ret_from_kernel_thread */
202 .Lsyscall_exit:
203 std r3,RESULT(r1)
205 #ifdef CONFIG_DEBUG_RSEQ
206 /* Check whether the syscall is issued inside a restartable sequence */
207 addi r3,r1,STACK_FRAME_OVERHEAD
208 bl rseq_syscall
209 ld r3,RESULT(r1)
210 #endif
212 ld r12, PACA_THREAD_INFO(r13)
214 ld r8,_MSR(r1)
216 /*
217 * This is a few instructions into the actual syscall exit path (which actually
218 * starts at .Lsyscall_exit) to cater to kprobe blacklisting and to reduce the
219 * number of visible symbols for profiling purposes.
220 *
221 * We can probe from system_call until this point as MSR_RI is set. But once it
222 * is cleared below, we won't be able to take a trap.
223 *
224 * This is blacklisted from kprobes further below with _ASM_NOKPROBE_SYMBOL().
225 */
226 system_call_exit:
227 /*
228 * Disable interrupts so current_thread_info()->flags can't change,
229 * and so that we don't get interrupted after loading SRR0/1.
230 *
231 * Leave MSR_RI enabled for now, because with THREAD_INFO_IN_TASK we
232 * could fault on the load of the TI_FLAGS below.
233 */
234 #ifdef CONFIG_PPC_BOOK3E
235 wrteei 0
236 #else
237 li r11,MSR_RI
238 mtmsrd r11,1
239 #endif /* CONFIG_PPC_BOOK3E */
241 ld r9,TI_FLAGS(r12)
242 li r11,-MAX_ERRNO
243 andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
244 bne- .Lsyscall_exit_work
246 andi. r0,r8,MSR_FP
247 beq 2f
248 #ifdef CONFIG_ALTIVEC
249 andis. r0,r8,MSR_VEC@h
250 bne 3f
251 #endif
252 2: addi r3,r1,STACK_FRAME_OVERHEAD
253 bl restore_math
254 ld r8,_MSR(r1)
255 ld r3,RESULT(r1)
256 li r11,-MAX_ERRNO
258 3: cmpld r3,r11
259 ld r5,_CCR(r1)
260 bge- .Lsyscall_error
261 .Lsyscall_error_cont:
262 ld r7,_NIP(r1)
263 BEGIN_FTR_SECTION
264 stdcx. r0,0,r1 /* to clear the reservation */
265 END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
266 andi. r6,r8,MSR_PR
267 ld r4,_LINK(r1)
269 kuap_check_amr r10, r11
271 #ifdef CONFIG_PPC_BOOK3S
272 /*
273 * Clear MSR_RI, MSR_EE is already and remains disabled. We could do
274 * this later, but testing shows that doing it here causes less slow
275 * down than doing it closer to the rfid.
276 */
277 li r11,0
278 mtmsrd r11,1
279 #endif
281 beq- 1f
282 ACCOUNT_CPU_USER_EXIT(r13, r11, r12)
284 BEGIN_FTR_SECTION
285 HMT_MEDIUM_LOW
286 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
288 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
289 std r8, PACATMSCRATCH(r13)
290 #endif
292 /*
293 * We don't need to restore AMR on the way back to userspace for KUAP.
294 * The value of AMR only matters while we're in the kernel.
295 */
296 ld r13,GPR13(r1) /* only restore r13 if returning to usermode */
297 ld r2,GPR2(r1)
298 ld r1,GPR1(r1)
299 mtlr r4
300 mtcr r5
301 mtspr SPRN_SRR0,r7
302 mtspr SPRN_SRR1,r8
303 RFI_TO_USER
304 b . /* prevent speculative execution */
306 1: /* exit to kernel */
307 kuap_restore_amr r2
309 ld r2,GPR2(r1)
310 ld r1,GPR1(r1)
311 mtlr r4
312 mtcr r5
313 mtspr SPRN_SRR0,r7
314 mtspr SPRN_SRR1,r8
315 RFI_TO_KERNEL
316 b . /* prevent speculative execution */
318 .Lsyscall_error:
319 oris r5,r5,0x1000 /* Set SO bit in CR */
320 neg r3,r3
321 std r5,_CCR(r1)
322 b .Lsyscall_error_cont
324 /* Traced system call support */
325 .Lsyscall_dotrace:
326 bl save_nvgprs
327 addi r3,r1,STACK_FRAME_OVERHEAD
328 bl do_syscall_trace_enter
330 /*
331 * We use the return value of do_syscall_trace_enter() as the syscall
332 * number. If the syscall was rejected for any reason do_syscall_trace_enter()
333 * returns an invalid syscall number and the test below against
334 * NR_syscalls will fail.
335 */
336 mr r0,r3
338 /* Restore argument registers just clobbered and/or possibly changed. */
339 ld r3,GPR3(r1)
340 ld r4,GPR4(r1)
341 ld r5,GPR5(r1)
342 ld r6,GPR6(r1)
343 ld r7,GPR7(r1)
344 ld r8,GPR8(r1)
346 /* Repopulate r9 and r10 for the syscall path */
347 addi r9,r1,STACK_FRAME_OVERHEAD
348 ld r10, PACA_THREAD_INFO(r13)
349 ld r10,TI_FLAGS(r10)
351 cmpldi r0,NR_syscalls
352 blt+ .Lsyscall
354 /* Return code is already in r3 thanks to do_syscall_trace_enter() */
355 b .Lsyscall_exit
358 .Lsyscall_enosys:
359 li r3,-ENOSYS
360 b .Lsyscall_exit
362 .Lsyscall_exit_work:
363 /* If TIF_RESTOREALL is set, don't scribble on either r3 or ccr.
364 If TIF_NOERROR is set, just save r3 as it is. */
366 andi. r0,r9,_TIF_RESTOREALL
367 beq+ 0f
368 REST_NVGPRS(r1)
369 b 2f
370 0: cmpld r3,r11 /* r11 is -MAX_ERRNO */
371 blt+ 1f
372 andi. r0,r9,_TIF_NOERROR
373 bne- 1f
374 ld r5,_CCR(r1)
375 neg r3,r3
376 oris r5,r5,0x1000 /* Set SO bit in CR */
377 std r5,_CCR(r1)
378 1: std r3,GPR3(r1)
379 2: andi. r0,r9,(_TIF_PERSYSCALL_MASK)
380 beq 4f
382 /* Clear per-syscall TIF flags if any are set. */
384 li r11,_TIF_PERSYSCALL_MASK
385 addi r12,r12,TI_FLAGS
386 3: ldarx r10,0,r12
387 andc r10,r10,r11
388 stdcx. r10,0,r12
389 bne- 3b
390 subi r12,r12,TI_FLAGS
392 4: /* Anything else left to do? */
393 BEGIN_FTR_SECTION
394 lis r3,DEFAULT_PPR@highest /* Set default PPR */
395 sldi r3,r3,32 /* bits 11-13 are used for ppr */
396 std r3,_PPR(r1)
397 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
399 andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP)
400 beq ret_from_except_lite
402 /* Re-enable interrupts */
403 #ifdef CONFIG_PPC_BOOK3E
404 wrteei 1
405 #else
406 li r10,MSR_RI
407 ori r10,r10,MSR_EE
408 mtmsrd r10,1
409 #endif /* CONFIG_PPC_BOOK3E */
411 bl save_nvgprs
412 addi r3,r1,STACK_FRAME_OVERHEAD
413 bl do_syscall_trace_leave
414 b ret_from_except
416 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
417 .Ltabort_syscall:
418 /* Firstly we need to enable TM in the kernel */
419 mfmsr r10
420 li r9, 1
421 rldimi r10, r9, MSR_TM_LG, 63-MSR_TM_LG
422 mtmsrd r10, 0
424 /* tabort, this dooms the transaction, nothing else */
425 li r9, (TM_CAUSE_SYSCALL|TM_CAUSE_PERSISTENT)
426 TABORT(R9)
428 /*
429 * Return directly to userspace. We have corrupted user register state,
430 * but userspace will never see that register state. Execution will
431 * resume after the tbegin of the aborted transaction with the
432 * checkpointed register state.
433 */
434 li r9, MSR_RI
435 andc r10, r10, r9
436 mtmsrd r10, 1
437 mtspr SPRN_SRR0, r11
438 mtspr SPRN_SRR1, r12
439 RFI_TO_USER
440 b . /* prevent speculative execution */
441 #endif
442 _ASM_NOKPROBE_SYMBOL(system_call_common);
443 _ASM_NOKPROBE_SYMBOL(system_call_exit);
445 /* Save non-volatile GPRs, if not already saved. */
446 _GLOBAL(save_nvgprs)
447 ld r11,_TRAP(r1)
448 andi. r0,r11,1
449 beqlr-
450 SAVE_NVGPRS(r1)
451 clrrdi r0,r11,1
452 std r0,_TRAP(r1)
453 blr
454 _ASM_NOKPROBE_SYMBOL(save_nvgprs);
457 /*
458 * The sigsuspend and rt_sigsuspend system calls can call do_signal
459 * and thus put the process into the stopped state where we might
460 * want to examine its user state with ptrace. Therefore we need
461 * to save all the nonvolatile registers (r14 - r31) before calling
462 * the C code. Similarly, fork, vfork and clone need the full
463 * register state on the stack so that it can be copied to the child.
464 */
466 _GLOBAL(ppc_fork)
467 bl save_nvgprs
468 bl sys_fork
469 b .Lsyscall_exit
471 _GLOBAL(ppc_vfork)
472 bl save_nvgprs
473 bl sys_vfork
474 b .Lsyscall_exit
476 _GLOBAL(ppc_clone)
477 bl save_nvgprs
478 bl sys_clone
479 b .Lsyscall_exit
481 _GLOBAL(ppc_clone3)
482 bl save_nvgprs
483 bl sys_clone3
484 b .Lsyscall_exit
486 _GLOBAL(ppc32_swapcontext)
487 bl save_nvgprs
488 bl compat_sys_swapcontext
489 b .Lsyscall_exit
491 _GLOBAL(ppc64_swapcontext)
492 bl save_nvgprs
493 bl sys_swapcontext
494 b .Lsyscall_exit
496 _GLOBAL(ppc_switch_endian)
497 bl save_nvgprs
498 bl sys_switch_endian
499 b .Lsyscall_exit
501 _GLOBAL(ret_from_fork)
502 bl schedule_tail
503 REST_NVGPRS(r1)
504 li r3,0
505 b .Lsyscall_exit
507 _GLOBAL(ret_from_kernel_thread)
508 bl schedule_tail
509 REST_NVGPRS(r1)
510 mtlr r14
511 mr r3,r15
512 #ifdef PPC64_ELF_ABI_v2
513 mr r12,r14
514 #endif
515 blrl
516 li r3,0
517 b .Lsyscall_exit
519 #ifdef CONFIG_PPC_BOOK3S_64
521 #define FLUSH_COUNT_CACHE \
522 1: nop; \
523 patch_site 1b, patch__call_flush_count_cache
526 #define BCCTR_FLUSH .long 0x4c400420
528 .macro nops number
529 .rept \number
530 nop
531 .endr
532 .endm
534 .balign 32
535 .global flush_count_cache
536 flush_count_cache:
537 /* Save LR into r9 */
538 mflr r9
540 // Flush the link stack
541 .rept 64
542 bl .+4
543 .endr
544 b 1f
545 nops 6
547 .balign 32
548 /* Restore LR */
549 1: mtlr r9
551 // If we're just flushing the link stack, return here
552 3: nop
553 patch_site 3b patch__flush_link_stack_return
555 li r9,0x7fff
556 mtctr r9
558 BCCTR_FLUSH
560 2: nop
561 patch_site 2b patch__flush_count_cache_return
563 nops 3
565 .rept 278
566 .balign 32
567 BCCTR_FLUSH
568 nops 7
569 .endr
571 blr
572 #else
573 #define FLUSH_COUNT_CACHE
574 #endif /* CONFIG_PPC_BOOK3S_64 */
576 /*
577 * This routine switches between two different tasks. The process
578 * state of one is saved on its kernel stack. Then the state
579 * of the other is restored from its kernel stack. The memory
580 * management hardware is updated to the second process's state.
581 * Finally, we can return to the second process, via ret_from_except.
582 * On entry, r3 points to the THREAD for the current task, r4
583 * points to the THREAD for the new task.
584 *
585 * Note: there are two ways to get to the "going out" portion
586 * of this code; either by coming in via the entry (_switch)
587 * or via "fork" which must set up an environment equivalent
588 * to the "_switch" path. If you change this you'll have to change
589 * the fork code also.
590 *
591 * The code which creates the new task context is in 'copy_thread'
592 * in arch/powerpc/kernel/process.c
593 */
594 .align 7
595 _GLOBAL(_switch)
596 mflr r0
597 std r0,16(r1)
598 stdu r1,-SWITCH_FRAME_SIZE(r1)
599 /* r3-r13 are caller saved -- Cort */
600 SAVE_NVGPRS(r1)
601 std r0,_NIP(r1) /* Return to switch caller */
602 mfcr r23
603 std r23,_CCR(r1)
604 std r1,KSP(r3) /* Set old stack pointer */
606 kuap_check_amr r9, r10
608 FLUSH_COUNT_CACHE
610 /*
611 * On SMP kernels, care must be taken because a task may be
612 * scheduled off CPUx and on to CPUy. Memory ordering must be
613 * considered.
614 *
615 * Cacheable stores on CPUx will be visible when the task is
616 * scheduled on CPUy by virtue of the core scheduler barriers
617 * (see "Notes on Program-Order guarantees on SMP systems." in
618 * kernel/sched/core.c).
619 *
620 * Uncacheable stores in the case of involuntary preemption must
621 * be taken care of. The smp_mb__before_spin_lock() in __schedule()
622 * is implemented as hwsync on powerpc, which orders MMIO too. So
623 * long as there is an hwsync in the context switch path, it will
624 * be executed on the source CPU after the task has performed
625 * all MMIO ops on that CPU, and on the destination CPU before the
626 * task performs any MMIO ops there.
627 */
629 /*
630 * The kernel context switch path must contain a spin_lock,
631 * which contains larx/stcx, which will clear any reservation
632 * of the task being switched.
633 */
634 #ifdef CONFIG_PPC_BOOK3S
635 /* Cancel all explict user streams as they will have no use after context
636 * switch and will stop the HW from creating streams itself
637 */
638 DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r6)
639 #endif
641 addi r6,r4,-THREAD /* Convert THREAD to 'current' */
642 std r6,PACACURRENT(r13) /* Set new 'current' */
643 #if defined(CONFIG_STACKPROTECTOR)
644 ld r6, TASK_CANARY(r6)
645 std r6, PACA_CANARY(r13)
646 #endif
648 ld r8,KSP(r4) /* new stack pointer */
649 #ifdef CONFIG_PPC_BOOK3S_64
650 BEGIN_MMU_FTR_SECTION
651 b 2f
652 END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX)
653 BEGIN_FTR_SECTION
654 clrrdi r6,r8,28 /* get its ESID */
655 clrrdi r9,r1,28 /* get current sp ESID */
656 FTR_SECTION_ELSE
657 clrrdi r6,r8,40 /* get its 1T ESID */
658 clrrdi r9,r1,40 /* get current sp 1T ESID */
659 ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_1T_SEGMENT)
660 clrldi. r0,r6,2 /* is new ESID c00000000? */
661 cmpd cr1,r6,r9 /* or is new ESID the same as current ESID? */
662 cror eq,4*cr1+eq,eq
663 beq 2f /* if yes, don't slbie it */
665 /* Bolt in the new stack SLB entry */
666 ld r7,KSP_VSID(r4) /* Get new stack's VSID */
667 oris r0,r6,(SLB_ESID_V)@h
668 ori r0,r0,(SLB_NUM_BOLTED-1)@l
669 BEGIN_FTR_SECTION
670 li r9,MMU_SEGSIZE_1T /* insert B field */
671 oris r6,r6,(MMU_SEGSIZE_1T << SLBIE_SSIZE_SHIFT)@h
672 rldimi r7,r9,SLB_VSID_SSIZE_SHIFT,0
673 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
675 /* Update the last bolted SLB. No write barriers are needed
676 * here, provided we only update the current CPU's SLB shadow
677 * buffer.
678 */
679 ld r9,PACA_SLBSHADOWPTR(r13)
680 li r12,0
681 std r12,SLBSHADOW_STACKESID(r9) /* Clear ESID */
682 li r12,SLBSHADOW_STACKVSID
683 STDX_BE r7,r12,r9 /* Save VSID */
684 li r12,SLBSHADOW_STACKESID
685 STDX_BE r0,r12,r9 /* Save ESID */
687 /* No need to check for MMU_FTR_NO_SLBIE_B here, since when
688 * we have 1TB segments, the only CPUs known to have the errata
689 * only support less than 1TB of system memory and we'll never
690 * actually hit this code path.
691 */
693 isync
694 slbie r6
695 BEGIN_FTR_SECTION
696 slbie r6 /* Workaround POWER5 < DD2.1 issue */
697 END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
698 slbmte r7,r0
699 isync
700 2:
701 #endif /* CONFIG_PPC_BOOK3S_64 */
703 clrrdi r7, r8, THREAD_SHIFT /* base of new stack */
704 /* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE
705 because we don't need to leave the 288-byte ABI gap at the
706 top of the kernel stack. */
707 addi r7,r7,THREAD_SIZE-SWITCH_FRAME_SIZE
709 /*
710 * PMU interrupts in radix may come in here. They will use r1, not
711 * PACAKSAVE, so this stack switch will not cause a problem. They
712 * will store to the process stack, which may then be migrated to
713 * another CPU. However the rq lock release on this CPU paired with
714 * the rq lock acquire on the new CPU before the stack becomes
715 * active on the new CPU, will order those stores.
716 */
717 mr r1,r8 /* start using new stack pointer */
718 std r7,PACAKSAVE(r13)
720 ld r6,_CCR(r1)
721 mtcrf 0xFF,r6
723 /* r3-r13 are destroyed -- Cort */
724 REST_NVGPRS(r1)
726 /* convert old thread to its task_struct for return value */
727 addi r3,r3,-THREAD
728 ld r7,_NIP(r1) /* Return to _switch caller in new task */
729 mtlr r7
730 addi r1,r1,SWITCH_FRAME_SIZE
731 blr
733 .align 7
734 _GLOBAL(ret_from_except)
735 ld r11,_TRAP(r1)
736 andi. r0,r11,1
737 bne ret_from_except_lite
738 REST_NVGPRS(r1)
740 _GLOBAL(ret_from_except_lite)
741 /*
742 * Disable interrupts so that current_thread_info()->flags
743 * can't change between when we test it and when we return
744 * from the interrupt.
745 */
746 #ifdef CONFIG_PPC_BOOK3E
747 wrteei 0
748 #else
749 li r10,MSR_RI
750 mtmsrd r10,1 /* Update machine state */
751 #endif /* CONFIG_PPC_BOOK3E */
753 ld r9, PACA_THREAD_INFO(r13)
754 ld r3,_MSR(r1)
755 #ifdef CONFIG_PPC_BOOK3E
756 ld r10,PACACURRENT(r13)
757 #endif /* CONFIG_PPC_BOOK3E */
758 ld r4,TI_FLAGS(r9)
759 andi. r3,r3,MSR_PR
760 beq resume_kernel
761 #ifdef CONFIG_PPC_BOOK3E
762 lwz r3,(THREAD+THREAD_DBCR0)(r10)
763 #endif /* CONFIG_PPC_BOOK3E */
765 /* Check current_thread_info()->flags */
766 andi. r0,r4,_TIF_USER_WORK_MASK
767 bne 1f
768 #ifdef CONFIG_PPC_BOOK3E
769 /*
770 * Check to see if the dbcr0 register is set up to debug.
771 * Use the internal debug mode bit to do this.
772 */
773 andis. r0,r3,DBCR0_IDM@h
774 beq restore
775 mfmsr r0
776 rlwinm r0,r0,0,~MSR_DE /* Clear MSR.DE */
777 mtmsr r0
778 mtspr SPRN_DBCR0,r3
779 li r10, -1
780 mtspr SPRN_DBSR,r10
781 b restore
782 #else
783 addi r3,r1,STACK_FRAME_OVERHEAD
784 bl restore_math
785 b restore
786 #endif
787 1: andi. r0,r4,_TIF_NEED_RESCHED
788 beq 2f
789 bl restore_interrupts
790 SCHEDULE_USER
791 b ret_from_except_lite
792 2:
793 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
794 andi. r0,r4,_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM
795 bne 3f /* only restore TM if nothing else to do */
796 addi r3,r1,STACK_FRAME_OVERHEAD
797 bl restore_tm_state
798 b restore
799 3:
800 #endif
801 bl save_nvgprs
802 /*
803 * Use a non volatile GPR to save and restore our thread_info flags
804 * across the call to restore_interrupts.
805 */
806 mr r30,r4
807 bl restore_interrupts
808 mr r4,r30
809 addi r3,r1,STACK_FRAME_OVERHEAD
810 bl do_notify_resume
811 b ret_from_except
813 resume_kernel:
814 /* check current_thread_info, _TIF_EMULATE_STACK_STORE */
815 andis. r8,r4,_TIF_EMULATE_STACK_STORE@h
816 beq+ 1f
818 addi r8,r1,INT_FRAME_SIZE /* Get the kprobed function entry */
820 ld r3,GPR1(r1)
821 subi r3,r3,INT_FRAME_SIZE /* dst: Allocate a trampoline exception frame */
822 mr r4,r1 /* src: current exception frame */
823 mr r1,r3 /* Reroute the trampoline frame to r1 */
825 /* Copy from the original to the trampoline. */
826 li r5,INT_FRAME_SIZE/8 /* size: INT_FRAME_SIZE */
827 li r6,0 /* start offset: 0 */
828 mtctr r5
829 2: ldx r0,r6,r4
830 stdx r0,r6,r3
831 addi r6,r6,8
832 bdnz 2b
834 /* Do real store operation to complete stdu */
835 ld r5,GPR1(r1)
836 std r8,0(r5)
838 /* Clear _TIF_EMULATE_STACK_STORE flag */
839 lis r11,_TIF_EMULATE_STACK_STORE@h
840 addi r5,r9,TI_FLAGS
841 0: ldarx r4,0,r5
842 andc r4,r4,r11
843 stdcx. r4,0,r5
844 bne- 0b
845 1:
847 #ifdef CONFIG_PREEMPTION
848 /* Check if we need to preempt */
849 andi. r0,r4,_TIF_NEED_RESCHED
850 beq+ restore
851 /* Check that preempt_count() == 0 and interrupts are enabled */
852 lwz r8,TI_PREEMPT(r9)
853 cmpwi cr0,r8,0
854 bne restore
855 ld r0,SOFTE(r1)
856 andi. r0,r0,IRQS_DISABLED
857 bne restore
859 /*
860 * Here we are preempting the current task. We want to make
861 * sure we are soft-disabled first and reconcile irq state.
862 */
863 RECONCILE_IRQ_STATE(r3,r4)
864 bl preempt_schedule_irq
866 /*
867 * arch_local_irq_restore() from preempt_schedule_irq above may
868 * enable hard interrupt but we really should disable interrupts
869 * when we return from the interrupt, and so that we don't get
870 * interrupted after loading SRR0/1.
871 */
872 #ifdef CONFIG_PPC_BOOK3E
873 wrteei 0
874 #else
875 li r10,MSR_RI
876 mtmsrd r10,1 /* Update machine state */
877 #endif /* CONFIG_PPC_BOOK3E */
878 #endif /* CONFIG_PREEMPTION */
880 .globl fast_exc_return_irq
881 fast_exc_return_irq:
882 restore:
883 /*
884 * This is the main kernel exit path. First we check if we
885 * are about to re-enable interrupts
886 */
887 ld r5,SOFTE(r1)
888 lbz r6,PACAIRQSOFTMASK(r13)
889 andi. r5,r5,IRQS_DISABLED
890 bne .Lrestore_irq_off
892 /* We are enabling, were we already enabled ? Yes, just return */
893 andi. r6,r6,IRQS_DISABLED
894 beq cr0,.Ldo_restore
896 /*
897 * We are about to soft-enable interrupts (we are hard disabled
898 * at this point). We check if there's anything that needs to
899 * be replayed first.
900 */
901 lbz r0,PACAIRQHAPPENED(r13)
902 cmpwi cr0,r0,0
903 bne- .Lrestore_check_irq_replay
905 /*
906 * Get here when nothing happened while soft-disabled, just
907 * soft-enable and move-on. We will hard-enable as a side
908 * effect of rfi
909 */
910 .Lrestore_no_replay:
911 TRACE_ENABLE_INTS
912 li r0,IRQS_ENABLED
913 stb r0,PACAIRQSOFTMASK(r13);
915 /*
916 * Final return path. BookE is handled in a different file
917 */
918 .Ldo_restore:
919 #ifdef CONFIG_PPC_BOOK3E
920 b exception_return_book3e
921 #else
922 /*
923 * Clear the reservation. If we know the CPU tracks the address of
924 * the reservation then we can potentially save some cycles and use
925 * a larx. On POWER6 and POWER7 this is significantly faster.
926 */
927 BEGIN_FTR_SECTION
928 stdcx. r0,0,r1 /* to clear the reservation */
929 FTR_SECTION_ELSE
930 ldarx r4,0,r1
931 ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
933 /*
934 * Some code path such as load_up_fpu or altivec return directly
935 * here. They run entirely hard disabled and do not alter the
936 * interrupt state. They also don't use lwarx/stwcx. and thus
937 * are known not to leave dangling reservations.
938 */
939 .globl fast_exception_return
940 fast_exception_return:
941 ld r3,_MSR(r1)
942 ld r4,_CTR(r1)
943 ld r0,_LINK(r1)
944 mtctr r4
945 mtlr r0
946 ld r4,_XER(r1)
947 mtspr SPRN_XER,r4
949 kuap_check_amr r5, r6
951 REST_8GPRS(5, r1)
953 andi. r0,r3,MSR_RI
954 beq- .Lunrecov_restore
956 /*
957 * Clear RI before restoring r13. If we are returning to
958 * userspace and we take an exception after restoring r13,
959 * we end up corrupting the userspace r13 value.
960 */
961 li r4,0
962 mtmsrd r4,1
964 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
965 /* TM debug */
966 std r3, PACATMSCRATCH(r13) /* Stash returned-to MSR */
967 #endif
968 /*
969 * r13 is our per cpu area, only restore it if we are returning to
970 * userspace the value stored in the stack frame may belong to
971 * another CPU.
972 */
973 andi. r0,r3,MSR_PR
974 beq 1f
975 BEGIN_FTR_SECTION
976 /* Restore PPR */
977 ld r2,_PPR(r1)
978 mtspr SPRN_PPR,r2
979 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
980 ACCOUNT_CPU_USER_EXIT(r13, r2, r4)
981 REST_GPR(13, r1)
983 /*
984 * We don't need to restore AMR on the way back to userspace for KUAP.
985 * The value of AMR only matters while we're in the kernel.
986 */
987 mtspr SPRN_SRR1,r3
989 ld r2,_CCR(r1)
990 mtcrf 0xFF,r2
991 ld r2,_NIP(r1)
992 mtspr SPRN_SRR0,r2
994 ld r0,GPR0(r1)
995 ld r2,GPR2(r1)
996 ld r3,GPR3(r1)
997 ld r4,GPR4(r1)
998 ld r1,GPR1(r1)
999 RFI_TO_USER
1000 b . /* prevent speculative execution */
1002 1: mtspr SPRN_SRR1,r3
1004 ld r2,_CCR(r1)
1005 mtcrf 0xFF,r2
1006 ld r2,_NIP(r1)
1007 mtspr SPRN_SRR0,r2
1009 /*
1010 * Leaving a stale exception_marker on the stack can confuse
1011 * the reliable stack unwinder later on. Clear it.
1012 */
1013 li r2,0
1014 std r2,STACK_FRAME_OVERHEAD-16(r1)
1016 ld r0,GPR0(r1)
1017 ld r2,GPR2(r1)
1018 ld r3,GPR3(r1)
1020 kuap_restore_amr r4
1022 ld r4,GPR4(r1)
1023 ld r1,GPR1(r1)
1024 RFI_TO_KERNEL
1025 b . /* prevent speculative execution */
1027 #endif /* CONFIG_PPC_BOOK3E */
1029 /*
1030 * We are returning to a context with interrupts soft disabled.
1031 *
1032 * However, we may also about to hard enable, so we need to
1033 * make sure that in this case, we also clear PACA_IRQ_HARD_DIS
1034 * or that bit can get out of sync and bad things will happen
1035 */
1036 .Lrestore_irq_off:
1037 ld r3,_MSR(r1)
1038 lbz r7,PACAIRQHAPPENED(r13)
1039 andi. r0,r3,MSR_EE
1040 beq 1f
1041 rlwinm r7,r7,0,~PACA_IRQ_HARD_DIS
1042 stb r7,PACAIRQHAPPENED(r13)
1043 1:
1044 #if defined(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG) && defined(CONFIG_BUG)
1045 /* The interrupt should not have soft enabled. */
1046 lbz r7,PACAIRQSOFTMASK(r13)
1047 1: tdeqi r7,IRQS_ENABLED
1048 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
1049 #endif
1050 b .Ldo_restore
1052 /*
1053 * Something did happen, check if a re-emit is needed
1054 * (this also clears paca->irq_happened)
1055 */
1056 .Lrestore_check_irq_replay:
1057 /* XXX: We could implement a fast path here where we check
1058 * for irq_happened being just 0x01, in which case we can
1059 * clear it and return. That means that we would potentially
1060 * miss a decrementer having wrapped all the way around.
1061 *
1062 * Still, this might be useful for things like hash_page
1063 */
1064 bl __check_irq_replay
1065 cmpwi cr0,r3,0
1066 beq .Lrestore_no_replay
1068 /*
1069 * We need to re-emit an interrupt. We do so by re-using our
1070 * existing exception frame. We first change the trap value,
1071 * but we need to ensure we preserve the low nibble of it
1072 */
1073 ld r4,_TRAP(r1)
1074 clrldi r4,r4,60
1075 or r4,r4,r3
1076 std r4,_TRAP(r1)
1078 /*
1079 * PACA_IRQ_HARD_DIS won't always be set here, so set it now
1080 * to reconcile the IRQ state. Tracing is already accounted for.
1081 */
1082 lbz r4,PACAIRQHAPPENED(r13)
1083 ori r4,r4,PACA_IRQ_HARD_DIS
1084 stb r4,PACAIRQHAPPENED(r13)
1086 /*
1087 * Then find the right handler and call it. Interrupts are
1088 * still soft-disabled and we keep them that way.
1089 */
1090 cmpwi cr0,r3,0x500
1091 bne 1f
1092 addi r3,r1,STACK_FRAME_OVERHEAD;
1093 bl do_IRQ
1094 b ret_from_except
1095 1: cmpwi cr0,r3,0xf00
1096 bne 1f
1097 addi r3,r1,STACK_FRAME_OVERHEAD;
1098 bl performance_monitor_exception
1099 b ret_from_except
1100 1: cmpwi cr0,r3,0xe60
1101 bne 1f
1102 addi r3,r1,STACK_FRAME_OVERHEAD;
1103 bl handle_hmi_exception
1104 b ret_from_except
1105 1: cmpwi cr0,r3,0x900
1106 bne 1f
1107 addi r3,r1,STACK_FRAME_OVERHEAD;
1108 bl timer_interrupt
1109 b ret_from_except
1110 #ifdef CONFIG_PPC_DOORBELL
1111 1:
1112 #ifdef CONFIG_PPC_BOOK3E
1113 cmpwi cr0,r3,0x280
1114 #else
1115 cmpwi cr0,r3,0xa00
1116 #endif /* CONFIG_PPC_BOOK3E */
1117 bne 1f
1118 addi r3,r1,STACK_FRAME_OVERHEAD;
1119 bl doorbell_exception
1120 #endif /* CONFIG_PPC_DOORBELL */
1121 1: b ret_from_except /* What else to do here ? */
1123 .Lunrecov_restore:
1124 addi r3,r1,STACK_FRAME_OVERHEAD
1125 bl unrecoverable_exception
1126 b .Lunrecov_restore
1128 _ASM_NOKPROBE_SYMBOL(ret_from_except);
1129 _ASM_NOKPROBE_SYMBOL(ret_from_except_lite);
1130 _ASM_NOKPROBE_SYMBOL(resume_kernel);
1131 _ASM_NOKPROBE_SYMBOL(fast_exc_return_irq);
1132 _ASM_NOKPROBE_SYMBOL(restore);
1133 _ASM_NOKPROBE_SYMBOL(fast_exception_return);
1136 #ifdef CONFIG_PPC_RTAS
1137 /*
1138 * On CHRP, the Run-Time Abstraction Services (RTAS) have to be
1139 * called with the MMU off.
1140 *
1141 * In addition, we need to be in 32b mode, at least for now.
1142 *
1143 * Note: r3 is an input parameter to rtas, so don't trash it...
1144 */
1145 _GLOBAL(enter_rtas)
1146 mflr r0
1147 std r0,16(r1)
1148 stdu r1,-SWITCH_FRAME_SIZE(r1) /* Save SP and create stack space. */
1150 /* Because RTAS is running in 32b mode, it clobbers the high order half
1151 * of all registers that it saves. We therefore save those registers
1152 * RTAS might touch to the stack. (r0, r3-r13 are caller saved)
1153 */
1154 SAVE_GPR(2, r1) /* Save the TOC */
1155 SAVE_GPR(13, r1) /* Save paca */
1156 SAVE_NVGPRS(r1) /* Save the non-volatiles */
1158 mfcr r4
1159 std r4,_CCR(r1)
1160 mfctr r5
1161 std r5,_CTR(r1)
1162 mfspr r6,SPRN_XER
1163 std r6,_XER(r1)
1164 mfdar r7
1165 std r7,_DAR(r1)
1166 mfdsisr r8
1167 std r8,_DSISR(r1)
1169 /* Temporary workaround to clear CR until RTAS can be modified to
1170 * ignore all bits.
1171 */
1172 li r0,0
1173 mtcr r0
1175 #ifdef CONFIG_BUG
1176 /* There is no way it is acceptable to get here with interrupts enabled,
1177 * check it with the asm equivalent of WARN_ON
1178 */
1179 lbz r0,PACAIRQSOFTMASK(r13)
1180 1: tdeqi r0,IRQS_ENABLED
1181 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
1182 #endif
1184 /* Hard-disable interrupts */
1185 mfmsr r6
1186 rldicl r7,r6,48,1
1187 rotldi r7,r7,16
1188 mtmsrd r7,1
1190 /* Unfortunately, the stack pointer and the MSR are also clobbered,
1191 * so they are saved in the PACA which allows us to restore
1192 * our original state after RTAS returns.
1193 */
1194 std r1,PACAR1(r13)
1195 std r6,PACASAVEDMSR(r13)
1197 /* Setup our real return addr */
1198 LOAD_REG_ADDR(r4,rtas_return_loc)
1199 clrldi r4,r4,2 /* convert to realmode address */
1200 mtlr r4
1202 li r0,0
1203 ori r0,r0,MSR_EE|MSR_SE|MSR_BE|MSR_RI
1204 andc r0,r6,r0
1206 li r9,1
1207 rldicr r9,r9,MSR_SF_LG,(63-MSR_SF_LG)
1208 ori r9,r9,MSR_IR|MSR_DR|MSR_FE0|MSR_FE1|MSR_FP|MSR_RI|MSR_LE
1209 andc r6,r0,r9
1211 __enter_rtas:
1212 sync /* disable interrupts so SRR0/1 */
1213 mtmsrd r0 /* don't get trashed */
1215 LOAD_REG_ADDR(r4, rtas)
1216 ld r5,RTASENTRY(r4) /* get the rtas->entry value */
1217 ld r4,RTASBASE(r4) /* get the rtas->base value */
1219 mtspr SPRN_SRR0,r5
1220 mtspr SPRN_SRR1,r6
1221 RFI_TO_KERNEL
1222 b . /* prevent speculative execution */
1224 rtas_return_loc:
1225 FIXUP_ENDIAN
1227 /*
1228 * Clear RI and set SF before anything.
1229 */
1230 mfmsr r6
1231 li r0,MSR_RI
1232 andc r6,r6,r0
1233 sldi r0,r0,(MSR_SF_LG - MSR_RI_LG)
1234 or r6,r6,r0
1235 sync
1236 mtmsrd r6
1238 /* relocation is off at this point */
1239 GET_PACA(r4)
1240 clrldi r4,r4,2 /* convert to realmode address */
1242 bcl 20,31,$+4
1243 0: mflr r3
1244 ld r3,(1f-0b)(r3) /* get &rtas_restore_regs */
1246 ld r1,PACAR1(r4) /* Restore our SP */
1247 ld r4,PACASAVEDMSR(r4) /* Restore our MSR */
1249 mtspr SPRN_SRR0,r3
1250 mtspr SPRN_SRR1,r4
1251 RFI_TO_KERNEL
1252 b . /* prevent speculative execution */
1253 _ASM_NOKPROBE_SYMBOL(__enter_rtas)
1254 _ASM_NOKPROBE_SYMBOL(rtas_return_loc)
1256 .align 3
1257 1: .8byte rtas_restore_regs
1259 rtas_restore_regs:
1260 /* relocation is on at this point */
1261 REST_GPR(2, r1) /* Restore the TOC */
1262 REST_GPR(13, r1) /* Restore paca */
1263 REST_NVGPRS(r1) /* Restore the non-volatiles */
1265 GET_PACA(r13)
1267 ld r4,_CCR(r1)
1268 mtcr r4
1269 ld r5,_CTR(r1)
1270 mtctr r5
1271 ld r6,_XER(r1)
1272 mtspr SPRN_XER,r6
1273 ld r7,_DAR(r1)
1274 mtdar r7
1275 ld r8,_DSISR(r1)
1276 mtdsisr r8
1278 addi r1,r1,SWITCH_FRAME_SIZE /* Unstack our frame */
1279 ld r0,16(r1) /* get return address */
1281 mtlr r0
1282 blr /* return to caller */
1284 #endif /* CONFIG_PPC_RTAS */
1286 _GLOBAL(enter_prom)
1287 mflr r0
1288 std r0,16(r1)
1289 stdu r1,-SWITCH_FRAME_SIZE(r1) /* Save SP and create stack space */
1291 /* Because PROM is running in 32b mode, it clobbers the high order half
1292 * of all registers that it saves. We therefore save those registers
1293 * PROM might touch to the stack. (r0, r3-r13 are caller saved)
1294 */
1295 SAVE_GPR(2, r1)
1296 SAVE_GPR(13, r1)
1297 SAVE_NVGPRS(r1)
1298 mfcr r10
1299 mfmsr r11
1300 std r10,_CCR(r1)
1301 std r11,_MSR(r1)
1303 /* Put PROM address in SRR0 */
1304 mtsrr0 r4
1306 /* Setup our trampoline return addr in LR */
1307 bcl 20,31,$+4
1308 0: mflr r4
1309 addi r4,r4,(1f - 0b)
1310 mtlr r4
1312 /* Prepare a 32-bit mode big endian MSR
1313 */
1314 #ifdef CONFIG_PPC_BOOK3E
1315 rlwinm r11,r11,0,1,31
1316 mtsrr1 r11
1317 rfi
1318 #else /* CONFIG_PPC_BOOK3E */
1319 LOAD_REG_IMMEDIATE(r12, MSR_SF | MSR_ISF | MSR_LE)
1320 andc r11,r11,r12
1321 mtsrr1 r11
1322 RFI_TO_KERNEL
1323 #endif /* CONFIG_PPC_BOOK3E */
1325 1: /* Return from OF */
1326 FIXUP_ENDIAN
1328 /* Just make sure that r1 top 32 bits didn't get
1329 * corrupt by OF
1330 */
1331 rldicl r1,r1,0,32
1333 /* Restore the MSR (back to 64 bits) */
1334 ld r0,_MSR(r1)
1335 MTMSRD(r0)
1336 isync
1338 /* Restore other registers */
1339 REST_GPR(2, r1)
1340 REST_GPR(13, r1)
1341 REST_NVGPRS(r1)
1342 ld r4,_CCR(r1)
1343 mtcr r4
1345 addi r1,r1,SWITCH_FRAME_SIZE
1346 ld r0,16(r1)
1347 mtlr r0
1348 blr