| blob | 0a0399c2af119c1c63efe094b1404f3250045769 |
1 /*
2 * This file contains the 64-bit "server" PowerPC variant
3 * of the low level exception handling including exception
4 * vectors, exception return, part of the slb and stab
5 * handling and other fixed offset specific things.
6 *
7 * This file is meant to be #included from head_64.S due to
8 * position dependent assembly.
9 *
10 * Most of this originates from head_64.S and thus has the same
11 * copyright history.
12 *
13 */
15 #include <asm/hw_irq.h>
16 #include <asm/exception-64s.h>
17 #include <asm/ptrace.h>
18 #include <asm/cpuidle.h>
20 /*
21 * We layout physical memory as follows:
22 * 0x0000 - 0x00ff : Secondary processor spin code
23 * 0x0100 - 0x17ff : pSeries Interrupt prologs
24 * 0x1800 - 0x4000 : interrupt support common interrupt prologs
25 * 0x4000 - 0x5fff : pSeries interrupts with IR=1,DR=1
26 * 0x6000 - 0x6fff : more interrupt support including for IR=1,DR=1
27 * 0x7000 - 0x7fff : FWNMI data area
28 * 0x8000 - 0x8fff : Initial (CPU0) segment table
29 * 0x9000 - : Early init and support code
30 */
31 /* Syscall routine is used twice, in reloc-off and reloc-on paths */
32 #define SYSCALL_PSERIES_1 \
33 BEGIN_FTR_SECTION \
34 cmpdi r0,0x1ebe ; \
35 beq- 1f ; \
36 END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE) \
37 mr r9,r13 ; \
38 GET_PACA(r13) ; \
39 mfspr r11,SPRN_SRR0 ; \
40 0:
42 #define SYSCALL_PSERIES_2_RFID \
43 mfspr r12,SPRN_SRR1 ; \
44 ld r10,PACAKBASE(r13) ; \
45 LOAD_HANDLER(r10, system_call_entry) ; \
46 mtspr SPRN_SRR0,r10 ; \
47 ld r10,PACAKMSR(r13) ; \
48 mtspr SPRN_SRR1,r10 ; \
49 rfid ; \
50 b . ; /* prevent speculative execution */
52 #define SYSCALL_PSERIES_3 \
53 /* Fast LE/BE switch system call */ \
54 1: mfspr r12,SPRN_SRR1 ; \
55 xori r12,r12,MSR_LE ; \
56 mtspr SPRN_SRR1,r12 ; \
57 rfid ; /* return to userspace */ \
58 b . ; /* prevent speculative execution */
60 #if defined(CONFIG_RELOCATABLE)
61 /*
62 * We can't branch directly so we do it via the CTR which
63 * is volatile across system calls.
64 */
65 #define SYSCALL_PSERIES_2_DIRECT \
66 mflr r10 ; \
67 ld r12,PACAKBASE(r13) ; \
68 LOAD_HANDLER(r12, system_call_entry) ; \
69 mtctr r12 ; \
70 mfspr r12,SPRN_SRR1 ; \
71 /* Re-use of r13... No spare regs to do this */ \
72 li r13,MSR_RI ; \
73 mtmsrd r13,1 ; \
74 GET_PACA(r13) ; /* get r13 back */ \
75 bctr ;
76 #else
77 /* We can branch directly */
78 #define SYSCALL_PSERIES_2_DIRECT \
79 mfspr r12,SPRN_SRR1 ; \
80 li r10,MSR_RI ; \
81 mtmsrd r10,1 ; /* Set RI (EE=0) */ \
82 b system_call_common ;
83 #endif
85 /*
86 * This is the start of the interrupt handlers for pSeries
87 * This code runs with relocation off.
88 * Code from here to __end_interrupts gets copied down to real
89 * address 0x100 when we are running a relocatable kernel.
90 * Therefore any relative branches in this section must only
91 * branch to labels in this section.
92 */
93 . = 0x100
94 .globl __start_interrupts
95 __start_interrupts:
97 .globl system_reset_pSeries;
98 system_reset_pSeries:
99 HMT_MEDIUM_PPR_DISCARD
100 SET_SCRATCH0(r13)
101 #ifdef CONFIG_PPC_P7_NAP
102 BEGIN_FTR_SECTION
103 /* Running native on arch 2.06 or later, check if we are
104 * waking up from nap/sleep/winkle.
105 */
106 mfspr r13,SPRN_SRR1
107 rlwinm. r13,r13,47-31,30,31
108 beq 9f
110 cmpwi cr3,r13,2
112 /*
113 * Check if last bit of HSPGR0 is set. This indicates whether we are
114 * waking up from winkle.
115 */
116 GET_PACA(r13)
117 clrldi r5,r13,63
118 clrrdi r13,r13,1
119 cmpwi cr4,r5,1
120 mtspr SPRN_HSPRG0,r13
122 lbz r0,PACA_THREAD_IDLE_STATE(r13)
123 cmpwi cr2,r0,PNV_THREAD_NAP
124 bgt cr2,8f /* Either sleep or Winkle */
126 /* Waking up from nap should not cause hypervisor state loss */
127 bgt cr3,.
129 /* Waking up from nap */
130 li r0,PNV_THREAD_RUNNING
131 stb r0,PACA_THREAD_IDLE_STATE(r13) /* Clear thread state */
133 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
134 li r0,KVM_HWTHREAD_IN_KERNEL
135 stb r0,HSTATE_HWTHREAD_STATE(r13)
136 /* Order setting hwthread_state vs. testing hwthread_req */
137 sync
138 lbz r0,HSTATE_HWTHREAD_REQ(r13)
139 cmpwi r0,0
140 beq 1f
141 b kvm_start_guest
142 1:
143 #endif
145 /* Return SRR1 from power7_nap() */
146 mfspr r3,SPRN_SRR1
147 beq cr3,2f
148 b power7_wakeup_noloss
149 2: b power7_wakeup_loss
151 /* Fast Sleep wakeup on PowerNV */
152 8: GET_PACA(r13)
153 b power7_wakeup_tb_loss
155 9:
156 END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
157 #endif /* CONFIG_PPC_P7_NAP */
158 EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common, EXC_STD,
159 NOTEST, 0x100)
161 . = 0x200
162 machine_check_pSeries_1:
163 /* This is moved out of line as it can be patched by FW, but
164 * some code path might still want to branch into the original
165 * vector
166 */
167 HMT_MEDIUM_PPR_DISCARD
168 SET_SCRATCH0(r13) /* save r13 */
169 #ifdef CONFIG_PPC_P7_NAP
170 BEGIN_FTR_SECTION
171 /* Running native on arch 2.06 or later, check if we are
172 * waking up from nap. We only handle no state loss and
173 * supervisor state loss. We do -not- handle hypervisor
174 * state loss at this time.
175 */
176 mfspr r13,SPRN_SRR1
177 rlwinm. r13,r13,47-31,30,31
178 OPT_GET_SPR(r13, SPRN_CFAR, CPU_FTR_CFAR)
179 beq 9f
181 mfspr r13,SPRN_SRR1
182 rlwinm. r13,r13,47-31,30,31
183 /* waking up from powersave (nap) state */
184 cmpwi cr1,r13,2
185 /* Total loss of HV state is fatal. let's just stay stuck here */
186 OPT_GET_SPR(r13, SPRN_CFAR, CPU_FTR_CFAR)
187 bgt cr1,.
188 9:
189 OPT_SET_SPR(r13, SPRN_CFAR, CPU_FTR_CFAR)
190 END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
191 #endif /* CONFIG_PPC_P7_NAP */
192 EXCEPTION_PROLOG_0(PACA_EXMC)
193 BEGIN_FTR_SECTION
194 b machine_check_pSeries_early
195 FTR_SECTION_ELSE
196 b machine_check_pSeries_0
197 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
199 . = 0x300
200 .globl data_access_pSeries
201 data_access_pSeries:
202 HMT_MEDIUM_PPR_DISCARD
203 SET_SCRATCH0(r13)
204 EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, data_access_common, EXC_STD,
205 KVMTEST, 0x300)
207 . = 0x380
208 .globl data_access_slb_pSeries
209 data_access_slb_pSeries:
210 HMT_MEDIUM_PPR_DISCARD
211 SET_SCRATCH0(r13)
212 EXCEPTION_PROLOG_0(PACA_EXSLB)
213 EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST, 0x380)
214 std r3,PACA_EXSLB+EX_R3(r13)
215 mfspr r3,SPRN_DAR
216 #ifdef __DISABLED__
217 /* Keep that around for when we re-implement dynamic VSIDs */
218 cmpdi r3,0
219 bge slb_miss_user_pseries
220 #endif /* __DISABLED__ */
221 mfspr r12,SPRN_SRR1
222 #ifndef CONFIG_RELOCATABLE
223 b slb_miss_realmode
224 #else
225 /*
226 * We can't just use a direct branch to slb_miss_realmode
227 * because the distance from here to there depends on where
228 * the kernel ends up being put.
229 */
230 mfctr r11
231 ld r10,PACAKBASE(r13)
232 LOAD_HANDLER(r10, slb_miss_realmode)
233 mtctr r10
234 bctr
235 #endif
237 STD_EXCEPTION_PSERIES(0x400, 0x400, instruction_access)
239 . = 0x480
240 .globl instruction_access_slb_pSeries
241 instruction_access_slb_pSeries:
242 HMT_MEDIUM_PPR_DISCARD
243 SET_SCRATCH0(r13)
244 EXCEPTION_PROLOG_0(PACA_EXSLB)
245 EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST_PR, 0x480)
246 std r3,PACA_EXSLB+EX_R3(r13)
247 mfspr r3,SPRN_SRR0 /* SRR0 is faulting address */
248 #ifdef __DISABLED__
249 /* Keep that around for when we re-implement dynamic VSIDs */
250 cmpdi r3,0
251 bge slb_miss_user_pseries
252 #endif /* __DISABLED__ */
253 mfspr r12,SPRN_SRR1
254 #ifndef CONFIG_RELOCATABLE
255 b slb_miss_realmode
256 #else
257 mfctr r11
258 ld r10,PACAKBASE(r13)
259 LOAD_HANDLER(r10, slb_miss_realmode)
260 mtctr r10
261 bctr
262 #endif
264 /* We open code these as we can't have a ". = x" (even with
265 * x = "." within a feature section
266 */
267 . = 0x500;
268 .globl hardware_interrupt_pSeries;
269 .globl hardware_interrupt_hv;
270 hardware_interrupt_pSeries:
271 hardware_interrupt_hv:
272 HMT_MEDIUM_PPR_DISCARD
273 BEGIN_FTR_SECTION
274 _MASKABLE_EXCEPTION_PSERIES(0x502, hardware_interrupt,
275 EXC_HV, SOFTEN_TEST_HV)
276 KVM_HANDLER(PACA_EXGEN, EXC_HV, 0x502)
277 FTR_SECTION_ELSE
278 _MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt,
279 EXC_STD, SOFTEN_TEST_HV_201)
280 KVM_HANDLER(PACA_EXGEN, EXC_STD, 0x500)
281 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
283 STD_EXCEPTION_PSERIES(0x600, 0x600, alignment)
284 KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x600)
286 STD_EXCEPTION_PSERIES(0x700, 0x700, program_check)
287 KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x700)
289 STD_EXCEPTION_PSERIES(0x800, 0x800, fp_unavailable)
290 KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x800)
292 . = 0x900
293 .globl decrementer_pSeries
294 decrementer_pSeries:
295 _MASKABLE_EXCEPTION_PSERIES(0x900, decrementer, EXC_STD, SOFTEN_TEST_PR)
297 STD_EXCEPTION_HV(0x980, 0x982, hdecrementer)
299 MASKABLE_EXCEPTION_PSERIES(0xa00, 0xa00, doorbell_super)
300 KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xa00)
302 STD_EXCEPTION_PSERIES(0xb00, 0xb00, trap_0b)
303 KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xb00)
305 . = 0xc00
306 .globl system_call_pSeries
307 system_call_pSeries:
308 /*
309 * If CONFIG_KVM_BOOK3S_64_HANDLER is set, save the PPR (on systems
310 * that support it) before changing to HMT_MEDIUM. That allows the KVM
311 * code to save that value into the guest state (it is the guest's PPR
312 * value). Otherwise just change to HMT_MEDIUM as userspace has
313 * already saved the PPR.
314 */
315 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
316 SET_SCRATCH0(r13)
317 GET_PACA(r13)
318 std r9,PACA_EXGEN+EX_R9(r13)
319 OPT_GET_SPR(r9, SPRN_PPR, CPU_FTR_HAS_PPR);
320 HMT_MEDIUM;
321 std r10,PACA_EXGEN+EX_R10(r13)
322 OPT_SAVE_REG_TO_PACA(PACA_EXGEN+EX_PPR, r9, CPU_FTR_HAS_PPR);
323 mfcr r9
324 KVMTEST(0xc00)
325 GET_SCRATCH0(r13)
326 #else
327 HMT_MEDIUM;
328 #endif
329 SYSCALL_PSERIES_1
330 SYSCALL_PSERIES_2_RFID
331 SYSCALL_PSERIES_3
332 KVM_HANDLER(PACA_EXGEN, EXC_STD, 0xc00)
334 STD_EXCEPTION_PSERIES(0xd00, 0xd00, single_step)
335 KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xd00)
337 /* At 0xe??? we have a bunch of hypervisor exceptions, we branch
338 * out of line to handle them
339 */
340 . = 0xe00
341 hv_data_storage_trampoline:
342 SET_SCRATCH0(r13)
343 EXCEPTION_PROLOG_0(PACA_EXGEN)
344 b h_data_storage_hv
346 . = 0xe20
347 hv_instr_storage_trampoline:
348 SET_SCRATCH0(r13)
349 EXCEPTION_PROLOG_0(PACA_EXGEN)
350 b h_instr_storage_hv
352 . = 0xe40
353 emulation_assist_trampoline:
354 SET_SCRATCH0(r13)
355 EXCEPTION_PROLOG_0(PACA_EXGEN)
356 b emulation_assist_hv
358 . = 0xe60
359 hv_exception_trampoline:
360 SET_SCRATCH0(r13)
361 EXCEPTION_PROLOG_0(PACA_EXGEN)
362 b hmi_exception_early
364 . = 0xe80
365 hv_doorbell_trampoline:
366 SET_SCRATCH0(r13)
367 EXCEPTION_PROLOG_0(PACA_EXGEN)
368 b h_doorbell_hv
370 /* We need to deal with the Altivec unavailable exception
371 * here which is at 0xf20, thus in the middle of the
372 * prolog code of the PerformanceMonitor one. A little
373 * trickery is thus necessary
374 */
375 . = 0xf00
376 performance_monitor_pseries_trampoline:
377 SET_SCRATCH0(r13)
378 EXCEPTION_PROLOG_0(PACA_EXGEN)
379 b performance_monitor_pSeries
381 . = 0xf20
382 altivec_unavailable_pseries_trampoline:
383 SET_SCRATCH0(r13)
384 EXCEPTION_PROLOG_0(PACA_EXGEN)
385 b altivec_unavailable_pSeries
387 . = 0xf40
388 vsx_unavailable_pseries_trampoline:
389 SET_SCRATCH0(r13)
390 EXCEPTION_PROLOG_0(PACA_EXGEN)
391 b vsx_unavailable_pSeries
393 . = 0xf60
394 facility_unavailable_trampoline:
395 SET_SCRATCH0(r13)
396 EXCEPTION_PROLOG_0(PACA_EXGEN)
397 b facility_unavailable_pSeries
399 . = 0xf80
400 hv_facility_unavailable_trampoline:
401 SET_SCRATCH0(r13)
402 EXCEPTION_PROLOG_0(PACA_EXGEN)
403 b facility_unavailable_hv
405 #ifdef CONFIG_CBE_RAS
406 STD_EXCEPTION_HV(0x1200, 0x1202, cbe_system_error)
407 KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0x1202)
408 #endif /* CONFIG_CBE_RAS */
410 STD_EXCEPTION_PSERIES(0x1300, 0x1300, instruction_breakpoint)
411 KVM_HANDLER_PR_SKIP(PACA_EXGEN, EXC_STD, 0x1300)
413 . = 0x1500
414 .global denorm_exception_hv
415 denorm_exception_hv:
416 HMT_MEDIUM_PPR_DISCARD
417 mtspr SPRN_SPRG_HSCRATCH0,r13
418 EXCEPTION_PROLOG_0(PACA_EXGEN)
419 EXCEPTION_PROLOG_1(PACA_EXGEN, NOTEST, 0x1500)
421 #ifdef CONFIG_PPC_DENORMALISATION
422 mfspr r10,SPRN_HSRR1
423 mfspr r11,SPRN_HSRR0 /* save HSRR0 */
424 andis. r10,r10,(HSRR1_DENORM)@h /* denorm? */
425 addi r11,r11,-4 /* HSRR0 is next instruction */
426 bne+ denorm_assist
427 #endif
429 KVMTEST(0x1500)
430 EXCEPTION_PROLOG_PSERIES_1(denorm_common, EXC_HV)
431 KVM_HANDLER_SKIP(PACA_EXGEN, EXC_STD, 0x1500)
433 #ifdef CONFIG_CBE_RAS
434 STD_EXCEPTION_HV(0x1600, 0x1602, cbe_maintenance)
435 KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0x1602)
436 #endif /* CONFIG_CBE_RAS */
438 STD_EXCEPTION_PSERIES(0x1700, 0x1700, altivec_assist)
439 KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x1700)
441 #ifdef CONFIG_CBE_RAS
442 STD_EXCEPTION_HV(0x1800, 0x1802, cbe_thermal)
443 KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0x1802)
444 #else
445 . = 0x1800
446 #endif /* CONFIG_CBE_RAS */
449 /*** Out of line interrupts support ***/
451 .align 7
452 /* moved from 0x200 */
453 machine_check_pSeries_early:
454 BEGIN_FTR_SECTION
455 EXCEPTION_PROLOG_1(PACA_EXMC, NOTEST, 0x200)
456 /*
457 * Register contents:
458 * R13 = PACA
459 * R9 = CR
460 * Original R9 to R13 is saved on PACA_EXMC
461 *
462 * Switch to mc_emergency stack and handle re-entrancy (we limit
463 * the nested MCE upto level 4 to avoid stack overflow).
464 * Save MCE registers srr1, srr0, dar and dsisr and then set ME=1
465 *
466 * We use paca->in_mce to check whether this is the first entry or
467 * nested machine check. We increment paca->in_mce to track nested
468 * machine checks.
469 *
470 * If this is the first entry then set stack pointer to
471 * paca->mc_emergency_sp, otherwise r1 is already pointing to
472 * stack frame on mc_emergency stack.
473 *
474 * NOTE: We are here with MSR_ME=0 (off), which means we risk a
475 * checkstop if we get another machine check exception before we do
476 * rfid with MSR_ME=1.
477 */
478 mr r11,r1 /* Save r1 */
479 lhz r10,PACA_IN_MCE(r13)
480 cmpwi r10,0 /* Are we in nested machine check */
481 bne 0f /* Yes, we are. */
482 /* First machine check entry */
483 ld r1,PACAMCEMERGSP(r13) /* Use MC emergency stack */
484 0: subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */
485 addi r10,r10,1 /* increment paca->in_mce */
486 sth r10,PACA_IN_MCE(r13)
487 /* Limit nested MCE to level 4 to avoid stack overflow */
488 cmpwi r10,4
489 bgt 2f /* Check if we hit limit of 4 */
490 std r11,GPR1(r1) /* Save r1 on the stack. */
491 std r11,0(r1) /* make stack chain pointer */
492 mfspr r11,SPRN_SRR0 /* Save SRR0 */
493 std r11,_NIP(r1)
494 mfspr r11,SPRN_SRR1 /* Save SRR1 */
495 std r11,_MSR(r1)
496 mfspr r11,SPRN_DAR /* Save DAR */
497 std r11,_DAR(r1)
498 mfspr r11,SPRN_DSISR /* Save DSISR */
499 std r11,_DSISR(r1)
500 std r9,_CCR(r1) /* Save CR in stackframe */
501 /* Save r9 through r13 from EXMC save area to stack frame. */
502 EXCEPTION_PROLOG_COMMON_2(PACA_EXMC)
503 mfmsr r11 /* get MSR value */
504 ori r11,r11,MSR_ME /* turn on ME bit */
505 ori r11,r11,MSR_RI /* turn on RI bit */
506 ld r12,PACAKBASE(r13) /* get high part of &label */
507 LOAD_HANDLER(r12, machine_check_handle_early)
508 1: mtspr SPRN_SRR0,r12
509 mtspr SPRN_SRR1,r11
510 rfid
511 b . /* prevent speculative execution */
512 2:
513 /* Stack overflow. Stay on emergency stack and panic.
514 * Keep the ME bit off while panic-ing, so that if we hit
515 * another machine check we checkstop.
516 */
517 addi r1,r1,INT_FRAME_SIZE /* go back to previous stack frame */
518 ld r11,PACAKMSR(r13)
519 ld r12,PACAKBASE(r13)
520 LOAD_HANDLER(r12, unrecover_mce)
521 li r10,MSR_ME
522 andc r11,r11,r10 /* Turn off MSR_ME */
523 b 1b
524 b . /* prevent speculative execution */
525 END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
527 machine_check_pSeries:
528 .globl machine_check_fwnmi
529 machine_check_fwnmi:
530 HMT_MEDIUM_PPR_DISCARD
531 SET_SCRATCH0(r13) /* save r13 */
532 EXCEPTION_PROLOG_0(PACA_EXMC)
533 machine_check_pSeries_0:
534 EXCEPTION_PROLOG_1(PACA_EXMC, KVMTEST, 0x200)
535 EXCEPTION_PROLOG_PSERIES_1(machine_check_common, EXC_STD)
536 KVM_HANDLER_SKIP(PACA_EXMC, EXC_STD, 0x200)
537 KVM_HANDLER_SKIP(PACA_EXGEN, EXC_STD, 0x300)
538 KVM_HANDLER_SKIP(PACA_EXSLB, EXC_STD, 0x380)
539 KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x400)
540 KVM_HANDLER_PR(PACA_EXSLB, EXC_STD, 0x480)
541 KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x900)
542 KVM_HANDLER(PACA_EXGEN, EXC_HV, 0x982)
544 #ifdef CONFIG_PPC_DENORMALISATION
545 denorm_assist:
546 BEGIN_FTR_SECTION
547 /*
548 * To denormalise we need to move a copy of the register to itself.
549 * For POWER6 do that here for all FP regs.
550 */
551 mfmsr r10
552 ori r10,r10,(MSR_FP|MSR_FE0|MSR_FE1)
553 xori r10,r10,(MSR_FE0|MSR_FE1)
554 mtmsrd r10
555 sync
557 #define FMR2(n) fmr (n), (n) ; fmr n+1, n+1
558 #define FMR4(n) FMR2(n) ; FMR2(n+2)
559 #define FMR8(n) FMR4(n) ; FMR4(n+4)
560 #define FMR16(n) FMR8(n) ; FMR8(n+8)
561 #define FMR32(n) FMR16(n) ; FMR16(n+16)
562 FMR32(0)
564 FTR_SECTION_ELSE
565 /*
566 * To denormalise we need to move a copy of the register to itself.
567 * For POWER7 do that here for the first 32 VSX registers only.
568 */
569 mfmsr r10
570 oris r10,r10,MSR_VSX@h
571 mtmsrd r10
572 sync
574 #define XVCPSGNDP2(n) XVCPSGNDP(n,n,n) ; XVCPSGNDP(n+1,n+1,n+1)
575 #define XVCPSGNDP4(n) XVCPSGNDP2(n) ; XVCPSGNDP2(n+2)
576 #define XVCPSGNDP8(n) XVCPSGNDP4(n) ; XVCPSGNDP4(n+4)
577 #define XVCPSGNDP16(n) XVCPSGNDP8(n) ; XVCPSGNDP8(n+8)
578 #define XVCPSGNDP32(n) XVCPSGNDP16(n) ; XVCPSGNDP16(n+16)
579 XVCPSGNDP32(0)
581 ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_206)
583 BEGIN_FTR_SECTION
584 b denorm_done
585 END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
586 /*
587 * To denormalise we need to move a copy of the register to itself.
588 * For POWER8 we need to do that for all 64 VSX registers
589 */
590 XVCPSGNDP32(32)
591 denorm_done:
592 mtspr SPRN_HSRR0,r11
593 mtcrf 0x80,r9
594 ld r9,PACA_EXGEN+EX_R9(r13)
595 RESTORE_PPR_PACA(PACA_EXGEN, r10)
596 BEGIN_FTR_SECTION
597 ld r10,PACA_EXGEN+EX_CFAR(r13)
598 mtspr SPRN_CFAR,r10
599 END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
600 ld r10,PACA_EXGEN+EX_R10(r13)
601 ld r11,PACA_EXGEN+EX_R11(r13)
602 ld r12,PACA_EXGEN+EX_R12(r13)
603 ld r13,PACA_EXGEN+EX_R13(r13)
604 HRFID
605 b .
606 #endif
608 .align 7
609 /* moved from 0xe00 */
610 STD_EXCEPTION_HV_OOL(0xe02, h_data_storage)
611 KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0xe02)
612 STD_EXCEPTION_HV_OOL(0xe22, h_instr_storage)
613 KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe22)
614 STD_EXCEPTION_HV_OOL(0xe42, emulation_assist)
615 KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe42)
616 MASKABLE_EXCEPTION_HV_OOL(0xe62, hmi_exception)
617 KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe62)
619 MASKABLE_EXCEPTION_HV_OOL(0xe82, h_doorbell)
620 KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe82)
622 /* moved from 0xf00 */
623 STD_EXCEPTION_PSERIES_OOL(0xf00, performance_monitor)
624 KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xf00)
625 STD_EXCEPTION_PSERIES_OOL(0xf20, altivec_unavailable)
626 KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xf20)
627 STD_EXCEPTION_PSERIES_OOL(0xf40, vsx_unavailable)
628 KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xf40)
629 STD_EXCEPTION_PSERIES_OOL(0xf60, facility_unavailable)
630 KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xf60)
631 STD_EXCEPTION_HV_OOL(0xf82, facility_unavailable)
632 KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xf82)
634 /*
635 * An interrupt came in while soft-disabled. We set paca->irq_happened, then:
636 * - If it was a decrementer interrupt, we bump the dec to max and and return.
637 * - If it was a doorbell we return immediately since doorbells are edge
638 * triggered and won't automatically refire.
639 * - If it was a HMI we return immediately since we handled it in realmode
640 * and it won't refire.
641 * - else we hard disable and return.
642 * This is called with r10 containing the value to OR to the paca field.
643 */
644 #define MASKED_INTERRUPT(_H) \
645 masked_##_H##interrupt: \
646 std r11,PACA_EXGEN+EX_R11(r13); \
647 lbz r11,PACAIRQHAPPENED(r13); \
648 or r11,r11,r10; \
649 stb r11,PACAIRQHAPPENED(r13); \
650 cmpwi r10,PACA_IRQ_DEC; \
651 bne 1f; \
652 lis r10,0x7fff; \
653 ori r10,r10,0xffff; \
654 mtspr SPRN_DEC,r10; \
655 b 2f; \
656 1: cmpwi r10,PACA_IRQ_DBELL; \
657 beq 2f; \
658 cmpwi r10,PACA_IRQ_HMI; \
659 beq 2f; \
660 mfspr r10,SPRN_##_H##SRR1; \
661 rldicl r10,r10,48,1; /* clear MSR_EE */ \
662 rotldi r10,r10,16; \
663 mtspr SPRN_##_H##SRR1,r10; \
664 2: mtcrf 0x80,r9; \
665 ld r9,PACA_EXGEN+EX_R9(r13); \
666 ld r10,PACA_EXGEN+EX_R10(r13); \
667 ld r11,PACA_EXGEN+EX_R11(r13); \
668 GET_SCRATCH0(r13); \
669 ##_H##rfid; \
670 b .
672 MASKED_INTERRUPT()
673 MASKED_INTERRUPT(H)
675 /*
676 * Called from arch_local_irq_enable when an interrupt needs
677 * to be resent. r3 contains 0x500, 0x900, 0xa00 or 0xe80 to indicate
678 * which kind of interrupt. MSR:EE is already off. We generate a
679 * stackframe like if a real interrupt had happened.
680 *
681 * Note: While MSR:EE is off, we need to make sure that _MSR
682 * in the generated frame has EE set to 1 or the exception
683 * handler will not properly re-enable them.
684 */
685 _GLOBAL(__replay_interrupt)
686 /* We are going to jump to the exception common code which
687 * will retrieve various register values from the PACA which
688 * we don't give a damn about, so we don't bother storing them.
689 */
690 mfmsr r12
691 mflr r11
692 mfcr r9
693 ori r12,r12,MSR_EE
694 cmpwi r3,0x900
695 beq decrementer_common
696 cmpwi r3,0x500
697 beq hardware_interrupt_common
698 BEGIN_FTR_SECTION
699 cmpwi r3,0xe80
700 beq h_doorbell_common
701 FTR_SECTION_ELSE
702 cmpwi r3,0xa00
703 beq doorbell_super_common
704 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
705 blr
707 #ifdef CONFIG_PPC_PSERIES
708 /*
709 * Vectors for the FWNMI option. Share common code.
710 */
711 .globl system_reset_fwnmi
712 .align 7
713 system_reset_fwnmi:
714 HMT_MEDIUM_PPR_DISCARD
715 SET_SCRATCH0(r13) /* save r13 */
716 EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common, EXC_STD,
717 NOTEST, 0x100)
719 #endif /* CONFIG_PPC_PSERIES */
721 #ifdef __DISABLED__
722 /*
723 * This is used for when the SLB miss handler has to go virtual,
724 * which doesn't happen for now anymore but will once we re-implement
725 * dynamic VSIDs for shared page tables
726 */
727 slb_miss_user_pseries:
728 std r10,PACA_EXGEN+EX_R10(r13)
729 std r11,PACA_EXGEN+EX_R11(r13)
730 std r12,PACA_EXGEN+EX_R12(r13)
731 GET_SCRATCH0(r10)
732 ld r11,PACA_EXSLB+EX_R9(r13)
733 ld r12,PACA_EXSLB+EX_R3(r13)
734 std r10,PACA_EXGEN+EX_R13(r13)
735 std r11,PACA_EXGEN+EX_R9(r13)
736 std r12,PACA_EXGEN+EX_R3(r13)
737 clrrdi r12,r13,32
738 mfmsr r10
739 mfspr r11,SRR0 /* save SRR0 */
740 ori r12,r12,slb_miss_user_common@l /* virt addr of handler */
741 ori r10,r10,MSR_IR|MSR_DR|MSR_RI
742 mtspr SRR0,r12
743 mfspr r12,SRR1 /* and SRR1 */
744 mtspr SRR1,r10
745 rfid
746 b . /* prevent spec. execution */
747 #endif /* __DISABLED__ */
749 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
750 kvmppc_skip_interrupt:
751 /*
752 * Here all GPRs are unchanged from when the interrupt happened
753 * except for r13, which is saved in SPRG_SCRATCH0.
754 */
755 mfspr r13, SPRN_SRR0
756 addi r13, r13, 4
757 mtspr SPRN_SRR0, r13
758 GET_SCRATCH0(r13)
759 rfid
760 b .
762 kvmppc_skip_Hinterrupt:
763 /*
764 * Here all GPRs are unchanged from when the interrupt happened
765 * except for r13, which is saved in SPRG_SCRATCH0.
766 */
767 mfspr r13, SPRN_HSRR0
768 addi r13, r13, 4
769 mtspr SPRN_HSRR0, r13
770 GET_SCRATCH0(r13)
771 hrfid
772 b .
773 #endif
775 /*
776 * Code from here down to __end_handlers is invoked from the
777 * exception prologs above. Because the prologs assemble the
778 * addresses of these handlers using the LOAD_HANDLER macro,
779 * which uses an ori instruction, these handlers must be in
780 * the first 64k of the kernel image.
781 */
783 /*** Common interrupt handlers ***/
785 STD_EXCEPTION_COMMON(0x100, system_reset, system_reset_exception)
787 STD_EXCEPTION_COMMON_ASYNC(0x500, hardware_interrupt, do_IRQ)
788 STD_EXCEPTION_COMMON_ASYNC(0x900, decrementer, timer_interrupt)
789 STD_EXCEPTION_COMMON(0x980, hdecrementer, hdec_interrupt)
790 #ifdef CONFIG_PPC_DOORBELL
791 STD_EXCEPTION_COMMON_ASYNC(0xa00, doorbell_super, doorbell_exception)
792 #else
793 STD_EXCEPTION_COMMON_ASYNC(0xa00, doorbell_super, unknown_exception)
794 #endif
795 STD_EXCEPTION_COMMON(0xb00, trap_0b, unknown_exception)
796 STD_EXCEPTION_COMMON(0xd00, single_step, single_step_exception)
797 STD_EXCEPTION_COMMON(0xe00, trap_0e, unknown_exception)
798 STD_EXCEPTION_COMMON(0xe40, emulation_assist, emulation_assist_interrupt)
799 STD_EXCEPTION_COMMON_ASYNC(0xe60, hmi_exception, handle_hmi_exception)
800 #ifdef CONFIG_PPC_DOORBELL
801 STD_EXCEPTION_COMMON_ASYNC(0xe80, h_doorbell, doorbell_exception)
802 #else
803 STD_EXCEPTION_COMMON_ASYNC(0xe80, h_doorbell, unknown_exception)
804 #endif
805 STD_EXCEPTION_COMMON_ASYNC(0xf00, performance_monitor, performance_monitor_exception)
806 STD_EXCEPTION_COMMON(0x1300, instruction_breakpoint, instruction_breakpoint_exception)
807 STD_EXCEPTION_COMMON(0x1502, denorm, unknown_exception)
808 #ifdef CONFIG_ALTIVEC
809 STD_EXCEPTION_COMMON(0x1700, altivec_assist, altivec_assist_exception)
810 #else
811 STD_EXCEPTION_COMMON(0x1700, altivec_assist, unknown_exception)
812 #endif
813 #ifdef CONFIG_CBE_RAS
814 STD_EXCEPTION_COMMON(0x1200, cbe_system_error, cbe_system_error_exception)
815 STD_EXCEPTION_COMMON(0x1600, cbe_maintenance, cbe_maintenance_exception)
816 STD_EXCEPTION_COMMON(0x1800, cbe_thermal, cbe_thermal_exception)
817 #endif /* CONFIG_CBE_RAS */
819 /*
820 * Relocation-on interrupts: A subset of the interrupts can be delivered
821 * with IR=1/DR=1, if AIL==2 and MSR.HV won't be changed by delivering
822 * it. Addresses are the same as the original interrupt addresses, but
823 * offset by 0xc000000000004000.
824 * It's impossible to receive interrupts below 0x300 via this mechanism.
825 * KVM: None of these traps are from the guest ; anything that escalated
826 * to HV=1 from HV=0 is delivered via real mode handlers.
827 */
829 /*
830 * This uses the standard macro, since the original 0x300 vector
831 * only has extra guff for STAB-based processors -- which never
832 * come here.
833 */
834 STD_RELON_EXCEPTION_PSERIES(0x4300, 0x300, data_access)
835 . = 0x4380
836 .globl data_access_slb_relon_pSeries
837 data_access_slb_relon_pSeries:
838 SET_SCRATCH0(r13)
839 EXCEPTION_PROLOG_0(PACA_EXSLB)
840 EXCEPTION_PROLOG_1(PACA_EXSLB, NOTEST, 0x380)
841 std r3,PACA_EXSLB+EX_R3(r13)
842 mfspr r3,SPRN_DAR
843 mfspr r12,SPRN_SRR1
844 #ifndef CONFIG_RELOCATABLE
845 b slb_miss_realmode
846 #else
847 /*
848 * We can't just use a direct branch to slb_miss_realmode
849 * because the distance from here to there depends on where
850 * the kernel ends up being put.
851 */
852 mfctr r11
853 ld r10,PACAKBASE(r13)
854 LOAD_HANDLER(r10, slb_miss_realmode)
855 mtctr r10
856 bctr
857 #endif
859 STD_RELON_EXCEPTION_PSERIES(0x4400, 0x400, instruction_access)
860 . = 0x4480
861 .globl instruction_access_slb_relon_pSeries
862 instruction_access_slb_relon_pSeries:
863 SET_SCRATCH0(r13)
864 EXCEPTION_PROLOG_0(PACA_EXSLB)
865 EXCEPTION_PROLOG_1(PACA_EXSLB, NOTEST, 0x480)
866 std r3,PACA_EXSLB+EX_R3(r13)
867 mfspr r3,SPRN_SRR0 /* SRR0 is faulting address */
868 mfspr r12,SPRN_SRR1
869 #ifndef CONFIG_RELOCATABLE
870 b slb_miss_realmode
871 #else
872 mfctr r11
873 ld r10,PACAKBASE(r13)
874 LOAD_HANDLER(r10, slb_miss_realmode)
875 mtctr r10
876 bctr
877 #endif
879 . = 0x4500
880 .globl hardware_interrupt_relon_pSeries;
881 .globl hardware_interrupt_relon_hv;
882 hardware_interrupt_relon_pSeries:
883 hardware_interrupt_relon_hv:
884 BEGIN_FTR_SECTION
885 _MASKABLE_RELON_EXCEPTION_PSERIES(0x502, hardware_interrupt, EXC_HV, SOFTEN_TEST_HV)
886 FTR_SECTION_ELSE
887 _MASKABLE_RELON_EXCEPTION_PSERIES(0x500, hardware_interrupt, EXC_STD, SOFTEN_TEST_PR)
888 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
889 STD_RELON_EXCEPTION_PSERIES(0x4600, 0x600, alignment)
890 STD_RELON_EXCEPTION_PSERIES(0x4700, 0x700, program_check)
891 STD_RELON_EXCEPTION_PSERIES(0x4800, 0x800, fp_unavailable)
892 MASKABLE_RELON_EXCEPTION_PSERIES(0x4900, 0x900, decrementer)
893 STD_RELON_EXCEPTION_HV(0x4980, 0x982, hdecrementer)
894 MASKABLE_RELON_EXCEPTION_PSERIES(0x4a00, 0xa00, doorbell_super)
895 STD_RELON_EXCEPTION_PSERIES(0x4b00, 0xb00, trap_0b)
897 . = 0x4c00
898 .globl system_call_relon_pSeries
899 system_call_relon_pSeries:
900 HMT_MEDIUM
901 SYSCALL_PSERIES_1
902 SYSCALL_PSERIES_2_DIRECT
903 SYSCALL_PSERIES_3
905 STD_RELON_EXCEPTION_PSERIES(0x4d00, 0xd00, single_step)
907 . = 0x4e00
908 b . /* Can't happen, see v2.07 Book III-S section 6.5 */
910 . = 0x4e20
911 b . /* Can't happen, see v2.07 Book III-S section 6.5 */
913 . = 0x4e40
914 emulation_assist_relon_trampoline:
915 SET_SCRATCH0(r13)
916 EXCEPTION_PROLOG_0(PACA_EXGEN)
917 b emulation_assist_relon_hv
919 . = 0x4e60
920 b . /* Can't happen, see v2.07 Book III-S section 6.5 */
922 . = 0x4e80
923 h_doorbell_relon_trampoline:
924 SET_SCRATCH0(r13)
925 EXCEPTION_PROLOG_0(PACA_EXGEN)
926 b h_doorbell_relon_hv
928 . = 0x4f00
929 performance_monitor_relon_pseries_trampoline:
930 SET_SCRATCH0(r13)
931 EXCEPTION_PROLOG_0(PACA_EXGEN)
932 b performance_monitor_relon_pSeries
934 . = 0x4f20
935 altivec_unavailable_relon_pseries_trampoline:
936 SET_SCRATCH0(r13)
937 EXCEPTION_PROLOG_0(PACA_EXGEN)
938 b altivec_unavailable_relon_pSeries
940 . = 0x4f40
941 vsx_unavailable_relon_pseries_trampoline:
942 SET_SCRATCH0(r13)
943 EXCEPTION_PROLOG_0(PACA_EXGEN)
944 b vsx_unavailable_relon_pSeries
946 . = 0x4f60
947 facility_unavailable_relon_trampoline:
948 SET_SCRATCH0(r13)
949 EXCEPTION_PROLOG_0(PACA_EXGEN)
950 b facility_unavailable_relon_pSeries
952 . = 0x4f80
953 hv_facility_unavailable_relon_trampoline:
954 SET_SCRATCH0(r13)
955 EXCEPTION_PROLOG_0(PACA_EXGEN)
956 b hv_facility_unavailable_relon_hv
958 STD_RELON_EXCEPTION_PSERIES(0x5300, 0x1300, instruction_breakpoint)
959 #ifdef CONFIG_PPC_DENORMALISATION
960 . = 0x5500
961 b denorm_exception_hv
962 #endif
963 STD_RELON_EXCEPTION_PSERIES(0x5700, 0x1700, altivec_assist)
965 /* Other future vectors */
966 .align 7
967 .globl __end_interrupts
968 __end_interrupts:
970 .align 7
971 system_call_entry:
972 b system_call_common
974 ppc64_runlatch_on_trampoline:
975 b __ppc64_runlatch_on
977 /*
978 * Here r13 points to the paca, r9 contains the saved CR,
979 * SRR0 and SRR1 are saved in r11 and r12,
980 * r9 - r13 are saved in paca->exgen.
981 */
982 .align 7
983 .globl data_access_common
984 data_access_common:
985 mfspr r10,SPRN_DAR
986 std r10,PACA_EXGEN+EX_DAR(r13)
987 mfspr r10,SPRN_DSISR
988 stw r10,PACA_EXGEN+EX_DSISR(r13)
989 EXCEPTION_PROLOG_COMMON(0x300, PACA_EXGEN)
990 RECONCILE_IRQ_STATE(r10, r11)
991 ld r12,_MSR(r1)
992 ld r3,PACA_EXGEN+EX_DAR(r13)
993 lwz r4,PACA_EXGEN+EX_DSISR(r13)
994 li r5,0x300
995 b do_hash_page /* Try to handle as hpte fault */
997 .align 7
998 .globl h_data_storage_common
999 h_data_storage_common:
1000 mfspr r10,SPRN_HDAR
1001 std r10,PACA_EXGEN+EX_DAR(r13)
1002 mfspr r10,SPRN_HDSISR
1003 stw r10,PACA_EXGEN+EX_DSISR(r13)
1004 EXCEPTION_PROLOG_COMMON(0xe00, PACA_EXGEN)
1005 bl save_nvgprs
1006 RECONCILE_IRQ_STATE(r10, r11)
1007 addi r3,r1,STACK_FRAME_OVERHEAD
1008 bl unknown_exception
1009 b ret_from_except
1011 .align 7
1012 .globl instruction_access_common
1013 instruction_access_common:
1014 EXCEPTION_PROLOG_COMMON(0x400, PACA_EXGEN)
1015 RECONCILE_IRQ_STATE(r10, r11)
1016 ld r12,_MSR(r1)
1017 ld r3,_NIP(r1)
1018 andis. r4,r12,0x5820
1019 li r5,0x400
1020 b do_hash_page /* Try to handle as hpte fault */
1022 STD_EXCEPTION_COMMON(0xe20, h_instr_storage, unknown_exception)
1024 /*
1025 * Here is the common SLB miss user that is used when going to virtual
1026 * mode for SLB misses, that is currently not used
1027 */
1028 #ifdef __DISABLED__
1029 .align 7
1030 .globl slb_miss_user_common
1031 slb_miss_user_common:
1032 mflr r10
1033 std r3,PACA_EXGEN+EX_DAR(r13)
1034 stw r9,PACA_EXGEN+EX_CCR(r13)
1035 std r10,PACA_EXGEN+EX_LR(r13)
1036 std r11,PACA_EXGEN+EX_SRR0(r13)
1037 bl slb_allocate_user
1039 ld r10,PACA_EXGEN+EX_LR(r13)
1040 ld r3,PACA_EXGEN+EX_R3(r13)
1041 lwz r9,PACA_EXGEN+EX_CCR(r13)
1042 ld r11,PACA_EXGEN+EX_SRR0(r13)
1043 mtlr r10
1044 beq- slb_miss_fault
1046 andi. r10,r12,MSR_RI /* check for unrecoverable exception */
1047 beq- unrecov_user_slb
1048 mfmsr r10
1050 .machine push
1051 .machine "power4"
1052 mtcrf 0x80,r9
1053 .machine pop
1055 clrrdi r10,r10,2 /* clear RI before setting SRR0/1 */
1056 mtmsrd r10,1
1058 mtspr SRR0,r11
1059 mtspr SRR1,r12
1061 ld r9,PACA_EXGEN+EX_R9(r13)
1062 ld r10,PACA_EXGEN+EX_R10(r13)
1063 ld r11,PACA_EXGEN+EX_R11(r13)
1064 ld r12,PACA_EXGEN+EX_R12(r13)
1065 ld r13,PACA_EXGEN+EX_R13(r13)
1066 rfid
1067 b .
1069 slb_miss_fault:
1070 EXCEPTION_PROLOG_COMMON(0x380, PACA_EXGEN)
1071 ld r4,PACA_EXGEN+EX_DAR(r13)
1072 li r5,0
1073 std r4,_DAR(r1)
1074 std r5,_DSISR(r1)
1075 b handle_page_fault
1077 unrecov_user_slb:
1078 EXCEPTION_PROLOG_COMMON(0x4200, PACA_EXGEN)
1079 RECONCILE_IRQ_STATE(r10, r11)
1080 bl save_nvgprs
1081 1: addi r3,r1,STACK_FRAME_OVERHEAD
1082 bl unrecoverable_exception
1083 b 1b
1085 #endif /* __DISABLED__ */
1088 /*
1089 * Machine check is different because we use a different
1090 * save area: PACA_EXMC instead of PACA_EXGEN.
1091 */
1092 .align 7
1093 .globl machine_check_common
1094 machine_check_common:
1096 mfspr r10,SPRN_DAR
1097 std r10,PACA_EXGEN+EX_DAR(r13)
1098 mfspr r10,SPRN_DSISR
1099 stw r10,PACA_EXGEN+EX_DSISR(r13)
1100 EXCEPTION_PROLOG_COMMON(0x200, PACA_EXMC)
1101 FINISH_NAP
1102 RECONCILE_IRQ_STATE(r10, r11)
1103 ld r3,PACA_EXGEN+EX_DAR(r13)
1104 lwz r4,PACA_EXGEN+EX_DSISR(r13)
1105 std r3,_DAR(r1)
1106 std r4,_DSISR(r1)
1107 bl save_nvgprs
1108 addi r3,r1,STACK_FRAME_OVERHEAD
1109 bl machine_check_exception
1110 b ret_from_except
1112 .align 7
1113 .globl alignment_common
1114 alignment_common:
1115 mfspr r10,SPRN_DAR
1116 std r10,PACA_EXGEN+EX_DAR(r13)
1117 mfspr r10,SPRN_DSISR
1118 stw r10,PACA_EXGEN+EX_DSISR(r13)
1119 EXCEPTION_PROLOG_COMMON(0x600, PACA_EXGEN)
1120 ld r3,PACA_EXGEN+EX_DAR(r13)
1121 lwz r4,PACA_EXGEN+EX_DSISR(r13)
1122 std r3,_DAR(r1)
1123 std r4,_DSISR(r1)
1124 bl save_nvgprs
1125 RECONCILE_IRQ_STATE(r10, r11)
1126 addi r3,r1,STACK_FRAME_OVERHEAD
1127 bl alignment_exception
1128 b ret_from_except
1130 .align 7
1131 .globl program_check_common
1132 program_check_common:
1133 EXCEPTION_PROLOG_COMMON(0x700, PACA_EXGEN)
1134 bl save_nvgprs
1135 RECONCILE_IRQ_STATE(r10, r11)
1136 addi r3,r1,STACK_FRAME_OVERHEAD
1137 bl program_check_exception
1138 b ret_from_except
1140 .align 7
1141 .globl fp_unavailable_common
1142 fp_unavailable_common:
1143 EXCEPTION_PROLOG_COMMON(0x800, PACA_EXGEN)
1144 bne 1f /* if from user, just load it up */
1145 bl save_nvgprs
1146 RECONCILE_IRQ_STATE(r10, r11)
1147 addi r3,r1,STACK_FRAME_OVERHEAD
1148 bl kernel_fp_unavailable_exception
1149 BUG_OPCODE
1150 1:
1151 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1152 BEGIN_FTR_SECTION
1153 /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in
1154 * transaction), go do TM stuff
1155 */
1156 rldicl. r0, r12, (64-MSR_TS_LG), (64-2)
1157 bne- 2f
1158 END_FTR_SECTION_IFSET(CPU_FTR_TM)
1159 #endif
1160 bl load_up_fpu
1161 b fast_exception_return
1162 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1163 2: /* User process was in a transaction */
1164 bl save_nvgprs
1165 RECONCILE_IRQ_STATE(r10, r11)
1166 addi r3,r1,STACK_FRAME_OVERHEAD
1167 bl fp_unavailable_tm
1168 b ret_from_except
1169 #endif
1170 .align 7
1171 .globl altivec_unavailable_common
1172 altivec_unavailable_common:
1173 EXCEPTION_PROLOG_COMMON(0xf20, PACA_EXGEN)
1174 #ifdef CONFIG_ALTIVEC
1175 BEGIN_FTR_SECTION
1176 beq 1f
1177 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1178 BEGIN_FTR_SECTION_NESTED(69)
1179 /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in
1180 * transaction), go do TM stuff
1181 */
1182 rldicl. r0, r12, (64-MSR_TS_LG), (64-2)
1183 bne- 2f
1184 END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69)
1185 #endif
1186 bl load_up_altivec
1187 b fast_exception_return
1188 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1189 2: /* User process was in a transaction */
1190 bl save_nvgprs
1191 RECONCILE_IRQ_STATE(r10, r11)
1192 addi r3,r1,STACK_FRAME_OVERHEAD
1193 bl altivec_unavailable_tm
1194 b ret_from_except
1195 #endif
1196 1:
1197 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
1198 #endif
1199 bl save_nvgprs
1200 RECONCILE_IRQ_STATE(r10, r11)
1201 addi r3,r1,STACK_FRAME_OVERHEAD
1202 bl altivec_unavailable_exception
1203 b ret_from_except
1205 .align 7
1206 .globl vsx_unavailable_common
1207 vsx_unavailable_common:
1208 EXCEPTION_PROLOG_COMMON(0xf40, PACA_EXGEN)
1209 #ifdef CONFIG_VSX
1210 BEGIN_FTR_SECTION
1211 beq 1f
1212 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1213 BEGIN_FTR_SECTION_NESTED(69)
1214 /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in
1215 * transaction), go do TM stuff
1216 */
1217 rldicl. r0, r12, (64-MSR_TS_LG), (64-2)
1218 bne- 2f
1219 END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69)
1220 #endif
1221 b load_up_vsx
1222 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1223 2: /* User process was in a transaction */
1224 bl save_nvgprs
1225 RECONCILE_IRQ_STATE(r10, r11)
1226 addi r3,r1,STACK_FRAME_OVERHEAD
1227 bl vsx_unavailable_tm
1228 b ret_from_except
1229 #endif
1230 1:
1231 END_FTR_SECTION_IFSET(CPU_FTR_VSX)
1232 #endif
1233 bl save_nvgprs
1234 RECONCILE_IRQ_STATE(r10, r11)
1235 addi r3,r1,STACK_FRAME_OVERHEAD
1236 bl vsx_unavailable_exception
1237 b ret_from_except
1239 STD_EXCEPTION_COMMON(0xf60, facility_unavailable, facility_unavailable_exception)
1240 STD_EXCEPTION_COMMON(0xf80, hv_facility_unavailable, facility_unavailable_exception)
1242 .align 7
1243 .globl __end_handlers
1244 __end_handlers:
1246 /* Equivalents to the above handlers for relocation-on interrupt vectors */
1247 STD_RELON_EXCEPTION_HV_OOL(0xe40, emulation_assist)
1248 MASKABLE_RELON_EXCEPTION_HV_OOL(0xe80, h_doorbell)
1250 STD_RELON_EXCEPTION_PSERIES_OOL(0xf00, performance_monitor)
1251 STD_RELON_EXCEPTION_PSERIES_OOL(0xf20, altivec_unavailable)
1252 STD_RELON_EXCEPTION_PSERIES_OOL(0xf40, vsx_unavailable)
1253 STD_RELON_EXCEPTION_PSERIES_OOL(0xf60, facility_unavailable)
1254 STD_RELON_EXCEPTION_HV_OOL(0xf80, hv_facility_unavailable)
1256 #if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV)
1257 /*
1258 * Data area reserved for FWNMI option.
1259 * This address (0x7000) is fixed by the RPA.
1260 */
1261 .= 0x7000
1262 .globl fwnmi_data_area
1263 fwnmi_data_area:
1265 /* pseries and powernv need to keep the whole page from
1266 * 0x7000 to 0x8000 free for use by the firmware
1267 */
1268 . = 0x8000
1269 #endif /* defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV) */
1271 .globl hmi_exception_early
1272 hmi_exception_early:
1273 EXCEPTION_PROLOG_1(PACA_EXGEN, NOTEST, 0xe60)
1274 mr r10,r1 /* Save r1 */
1275 ld r1,PACAEMERGSP(r13) /* Use emergency stack */
1276 subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */
1277 std r9,_CCR(r1) /* save CR in stackframe */
1278 mfspr r11,SPRN_HSRR0 /* Save HSRR0 */
1279 std r11,_NIP(r1) /* save HSRR0 in stackframe */
1280 mfspr r12,SPRN_HSRR1 /* Save SRR1 */
1281 std r12,_MSR(r1) /* save SRR1 in stackframe */
1282 std r10,0(r1) /* make stack chain pointer */
1283 std r0,GPR0(r1) /* save r0 in stackframe */
1284 std r10,GPR1(r1) /* save r1 in stackframe */
1285 EXCEPTION_PROLOG_COMMON_2(PACA_EXGEN)
1286 EXCEPTION_PROLOG_COMMON_3(0xe60)
1287 addi r3,r1,STACK_FRAME_OVERHEAD
1288 bl hmi_exception_realmode
1289 /* Windup the stack. */
1290 /* Move original HSRR0 and HSRR1 into the respective regs */
1291 ld r9,_MSR(r1)
1292 mtspr SPRN_HSRR1,r9
1293 ld r3,_NIP(r1)
1294 mtspr SPRN_HSRR0,r3
1295 ld r9,_CTR(r1)
1296 mtctr r9
1297 ld r9,_XER(r1)
1298 mtxer r9
1299 ld r9,_LINK(r1)
1300 mtlr r9
1301 REST_GPR(0, r1)
1302 REST_8GPRS(2, r1)
1303 REST_GPR(10, r1)
1304 ld r11,_CCR(r1)
1305 mtcr r11
1306 REST_GPR(11, r1)
1307 REST_2GPRS(12, r1)
1308 /* restore original r1. */
1309 ld r1,GPR1(r1)
1311 /*
1312 * Go to virtual mode and pull the HMI event information from
1313 * firmware.
1314 */
1315 .globl hmi_exception_after_realmode
1316 hmi_exception_after_realmode:
1317 SET_SCRATCH0(r13)
1318 EXCEPTION_PROLOG_0(PACA_EXGEN)
1319 b hmi_exception_hv
1322 #define MACHINE_CHECK_HANDLER_WINDUP \
1323 /* Clear MSR_RI before setting SRR0 and SRR1. */\
1324 li r0,MSR_RI; \
1325 mfmsr r9; /* get MSR value */ \
1326 andc r9,r9,r0; \
1327 mtmsrd r9,1; /* Clear MSR_RI */ \
1328 /* Move original SRR0 and SRR1 into the respective regs */ \
1329 ld r9,_MSR(r1); \
1330 mtspr SPRN_SRR1,r9; \
1331 ld r3,_NIP(r1); \
1332 mtspr SPRN_SRR0,r3; \
1333 ld r9,_CTR(r1); \
1334 mtctr r9; \
1335 ld r9,_XER(r1); \
1336 mtxer r9; \
1337 ld r9,_LINK(r1); \
1338 mtlr r9; \
1339 REST_GPR(0, r1); \
1340 REST_8GPRS(2, r1); \
1341 REST_GPR(10, r1); \
1342 ld r11,_CCR(r1); \
1343 mtcr r11; \
1344 /* Decrement paca->in_mce. */ \
1345 lhz r12,PACA_IN_MCE(r13); \
1346 subi r12,r12,1; \
1347 sth r12,PACA_IN_MCE(r13); \
1348 REST_GPR(11, r1); \
1349 REST_2GPRS(12, r1); \
1350 /* restore original r1. */ \
1351 ld r1,GPR1(r1)
1353 /*
1354 * Handle machine check early in real mode. We come here with
1355 * ME=1, MMU (IR=0 and DR=0) off and using MC emergency stack.
1356 */
1357 .align 7
1358 .globl machine_check_handle_early
1359 machine_check_handle_early:
1360 std r0,GPR0(r1) /* Save r0 */
1361 EXCEPTION_PROLOG_COMMON_3(0x200)
1362 bl save_nvgprs
1363 addi r3,r1,STACK_FRAME_OVERHEAD
1364 bl machine_check_early
1365 std r3,RESULT(r1) /* Save result */
1366 ld r12,_MSR(r1)
1367 #ifdef CONFIG_PPC_P7_NAP
1368 /*
1369 * Check if thread was in power saving mode. We come here when any
1370 * of the following is true:
1371 * a. thread wasn't in power saving mode
1372 * b. thread was in power saving mode with no state loss or
1373 * supervisor state loss
1374 *
1375 * Go back to nap again if (b) is true.
1376 */
1377 rlwinm. r11,r12,47-31,30,31 /* Was it in power saving mode? */
1378 beq 4f /* No, it wasn;t */
1379 /* Thread was in power saving mode. Go back to nap again. */
1380 cmpwi r11,2
1381 bne 3f
1382 /* Supervisor state loss */
1383 li r0,1
1384 stb r0,PACA_NAPSTATELOST(r13)
1385 3: bl machine_check_queue_event
1386 MACHINE_CHECK_HANDLER_WINDUP
1387 GET_PACA(r13)
1388 ld r1,PACAR1(r13)
1389 li r3,PNV_THREAD_NAP
1390 b power7_enter_nap_mode
1391 4:
1392 #endif
1393 /*
1394 * Check if we are coming from hypervisor userspace. If yes then we
1395 * continue in host kernel in V mode to deliver the MC event.
1396 */
1397 rldicl. r11,r12,4,63 /* See if MC hit while in HV mode. */
1398 beq 5f
1399 andi. r11,r12,MSR_PR /* See if coming from user. */
1400 bne 9f /* continue in V mode if we are. */
1402 5:
1403 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
1404 /*
1405 * We are coming from kernel context. Check if we are coming from
1406 * guest. if yes, then we can continue. We will fall through
1407 * do_kvm_200->kvmppc_interrupt to deliver the MC event to guest.
1408 */
1409 lbz r11,HSTATE_IN_GUEST(r13)
1410 cmpwi r11,0 /* Check if coming from guest */
1411 bne 9f /* continue if we are. */
1412 #endif
1413 /*
1414 * At this point we are not sure about what context we come from.
1415 * Queue up the MCE event and return from the interrupt.
1416 * But before that, check if this is an un-recoverable exception.
1417 * If yes, then stay on emergency stack and panic.
1418 */
1419 andi. r11,r12,MSR_RI
1420 bne 2f
1421 1: mfspr r11,SPRN_SRR0
1422 ld r10,PACAKBASE(r13)
1423 LOAD_HANDLER(r10,unrecover_mce)
1424 mtspr SPRN_SRR0,r10
1425 ld r10,PACAKMSR(r13)
1426 /*
1427 * We are going down. But there are chances that we might get hit by
1428 * another MCE during panic path and we may run into unstable state
1429 * with no way out. Hence, turn ME bit off while going down, so that
1430 * when another MCE is hit during panic path, system will checkstop
1431 * and hypervisor will get restarted cleanly by SP.
1432 */
1433 li r3,MSR_ME
1434 andc r10,r10,r3 /* Turn off MSR_ME */
1435 mtspr SPRN_SRR1,r10
1436 rfid
1437 b .
1438 2:
1439 /*
1440 * Check if we have successfully handled/recovered from error, if not
1441 * then stay on emergency stack and panic.
1442 */
1443 ld r3,RESULT(r1) /* Load result */
1444 cmpdi r3,0 /* see if we handled MCE successfully */
1446 beq 1b /* if !handled then panic */
1447 /*
1448 * Return from MC interrupt.
1449 * Queue up the MCE event so that we can log it later, while
1450 * returning from kernel or opal call.
1451 */
1452 bl machine_check_queue_event
1453 MACHINE_CHECK_HANDLER_WINDUP
1454 rfid
1455 9:
1456 /* Deliver the machine check to host kernel in V mode. */
1457 MACHINE_CHECK_HANDLER_WINDUP
1458 b machine_check_pSeries
1460 unrecover_mce:
1461 /* Invoke machine_check_exception to print MCE event and panic. */
1462 addi r3,r1,STACK_FRAME_OVERHEAD
1463 bl machine_check_exception
1464 /*
1465 * We will not reach here. Even if we did, there is no way out. Call
1466 * unrecoverable_exception and die.
1467 */
1468 1: addi r3,r1,STACK_FRAME_OVERHEAD
1469 bl unrecoverable_exception
1470 b 1b
1471 /*
1472 * r13 points to the PACA, r9 contains the saved CR,
1473 * r12 contain the saved SRR1, SRR0 is still ready for return
1474 * r3 has the faulting address
1475 * r9 - r13 are saved in paca->exslb.
1476 * r3 is saved in paca->slb_r3
1477 * We assume we aren't going to take any exceptions during this procedure.
1478 */
1479 slb_miss_realmode:
1480 mflr r10
1481 #ifdef CONFIG_RELOCATABLE
1482 mtctr r11
1483 #endif
1485 stw r9,PACA_EXSLB+EX_CCR(r13) /* save CR in exc. frame */
1486 std r10,PACA_EXSLB+EX_LR(r13) /* save LR */
1488 bl slb_allocate_realmode
1490 /* All done -- return from exception. */
1492 ld r10,PACA_EXSLB+EX_LR(r13)
1493 ld r3,PACA_EXSLB+EX_R3(r13)
1494 lwz r9,PACA_EXSLB+EX_CCR(r13) /* get saved CR */
1496 mtlr r10
1498 andi. r10,r12,MSR_RI /* check for unrecoverable exception */
1499 beq- 2f
1501 .machine push
1502 .machine "power4"
1503 mtcrf 0x80,r9
1504 mtcrf 0x01,r9 /* slb_allocate uses cr0 and cr7 */
1505 .machine pop
1507 RESTORE_PPR_PACA(PACA_EXSLB, r9)
1508 ld r9,PACA_EXSLB+EX_R9(r13)
1509 ld r10,PACA_EXSLB+EX_R10(r13)
1510 ld r11,PACA_EXSLB+EX_R11(r13)
1511 ld r12,PACA_EXSLB+EX_R12(r13)
1512 ld r13,PACA_EXSLB+EX_R13(r13)
1513 rfid
1514 b . /* prevent speculative execution */
1516 2: mfspr r11,SPRN_SRR0
1517 ld r10,PACAKBASE(r13)
1518 LOAD_HANDLER(r10,unrecov_slb)
1519 mtspr SPRN_SRR0,r10
1520 ld r10,PACAKMSR(r13)
1521 mtspr SPRN_SRR1,r10
1522 rfid
1523 b .
1525 unrecov_slb:
1526 EXCEPTION_PROLOG_COMMON(0x4100, PACA_EXSLB)
1527 RECONCILE_IRQ_STATE(r10, r11)
1528 bl save_nvgprs
1529 1: addi r3,r1,STACK_FRAME_OVERHEAD
1530 bl unrecoverable_exception
1531 b 1b
1534 #ifdef CONFIG_PPC_970_NAP
1535 power4_fixup_nap:
1536 andc r9,r9,r10
1537 std r9,TI_LOCAL_FLAGS(r11)
1538 ld r10,_LINK(r1) /* make idle task do the */
1539 std r10,_NIP(r1) /* equivalent of a blr */
1540 blr
1541 #endif
1543 /*
1544 * Hash table stuff
1545 */
1546 .align 7
1547 do_hash_page:
1548 std r3,_DAR(r1)
1549 std r4,_DSISR(r1)
1551 andis. r0,r4,0xa410 /* weird error? */
1552 bne- handle_page_fault /* if not, try to insert a HPTE */
1553 andis. r0,r4,DSISR_DABRMATCH@h
1554 bne- handle_dabr_fault
1555 CURRENT_THREAD_INFO(r11, r1)
1556 lwz r0,TI_PREEMPT(r11) /* If we're in an "NMI" */
1557 andis. r0,r0,NMI_MASK@h /* (i.e. an irq when soft-disabled) */
1558 bne 77f /* then don't call hash_page now */
1559 /*
1560 * We need to set the _PAGE_USER bit if MSR_PR is set or if we are
1561 * accessing a userspace segment (even from the kernel). We assume
1562 * kernel addresses always have the high bit set.
1563 */
1564 rlwinm r4,r4,32-25+9,31-9,31-9 /* DSISR_STORE -> _PAGE_RW */
1565 rotldi r0,r3,15 /* Move high bit into MSR_PR posn */
1566 orc r0,r12,r0 /* MSR_PR | ~high_bit */
1567 rlwimi r4,r0,32-13,30,30 /* becomes _PAGE_USER access bit */
1568 ori r4,r4,1 /* add _PAGE_PRESENT */
1569 rlwimi r4,r5,22+2,31-2,31-2 /* Set _PAGE_EXEC if trap is 0x400 */
1571 /*
1572 * r3 contains the faulting address
1573 * r4 contains the required access permissions
1574 * r5 contains the trap number
1575 * r6 contains dsisr
1576 *
1577 * at return r3 = 0 for success, 1 for page fault, negative for error
1578 */
1579 ld r6,_DSISR(r1)
1580 bl hash_page /* build HPTE if possible */
1581 cmpdi r3,0 /* see if hash_page succeeded */
1583 /* Success */
1584 beq fast_exc_return_irq /* Return from exception on success */
1586 /* Error */
1587 blt- 13f
1589 /* Here we have a page fault that hash_page can't handle. */
1590 handle_page_fault:
1591 11: ld r4,_DAR(r1)
1592 ld r5,_DSISR(r1)
1593 addi r3,r1,STACK_FRAME_OVERHEAD
1594 bl do_page_fault
1595 cmpdi r3,0
1596 beq+ 12f
1597 bl save_nvgprs
1598 mr r5,r3
1599 addi r3,r1,STACK_FRAME_OVERHEAD
1600 lwz r4,_DAR(r1)
1601 bl bad_page_fault
1602 b ret_from_except
1604 /* We have a data breakpoint exception - handle it */
1605 handle_dabr_fault:
1606 bl save_nvgprs
1607 ld r4,_DAR(r1)
1608 ld r5,_DSISR(r1)
1609 addi r3,r1,STACK_FRAME_OVERHEAD
1610 bl do_break
1611 12: b ret_from_except_lite
1614 /* We have a page fault that hash_page could handle but HV refused
1615 * the PTE insertion
1616 */
1617 13: bl save_nvgprs
1618 mr r5,r3
1619 addi r3,r1,STACK_FRAME_OVERHEAD
1620 ld r4,_DAR(r1)
1621 bl low_hash_fault
1622 b ret_from_except
1624 /*
1625 * We come here as a result of a DSI at a point where we don't want
1626 * to call hash_page, such as when we are accessing memory (possibly
1627 * user memory) inside a PMU interrupt that occurred while interrupts
1628 * were soft-disabled. We want to invoke the exception handler for
1629 * the access, or panic if there isn't a handler.
1630 */
1631 77: bl save_nvgprs
1632 mr r4,r3
1633 addi r3,r1,STACK_FRAME_OVERHEAD
1634 li r5,SIGSEGV
1635 bl bad_page_fault
1636 b ret_from_except
1638 /*
1639 * Here we have detected that the kernel stack pointer is bad.
1640 * R9 contains the saved CR, r13 points to the paca,
1641 * r10 contains the (bad) kernel stack pointer,
1642 * r11 and r12 contain the saved SRR0 and SRR1.
1643 * We switch to using an emergency stack, save the registers there,
1644 * and call kernel_bad_stack(), which panics.
1645 */
1646 bad_stack:
1647 ld r1,PACAEMERGSP(r13)
1648 subi r1,r1,64+INT_FRAME_SIZE
1649 std r9,_CCR(r1)
1650 std r10,GPR1(r1)
1651 std r11,_NIP(r1)
1652 std r12,_MSR(r1)
1653 mfspr r11,SPRN_DAR
1654 mfspr r12,SPRN_DSISR
1655 std r11,_DAR(r1)
1656 std r12,_DSISR(r1)
1657 mflr r10
1658 mfctr r11
1659 mfxer r12
1660 std r10,_LINK(r1)
1661 std r11,_CTR(r1)
1662 std r12,_XER(r1)
1663 SAVE_GPR(0,r1)
1664 SAVE_GPR(2,r1)
1665 ld r10,EX_R3(r3)
1666 std r10,GPR3(r1)
1667 SAVE_GPR(4,r1)
1668 SAVE_4GPRS(5,r1)
1669 ld r9,EX_R9(r3)
1670 ld r10,EX_R10(r3)
1671 SAVE_2GPRS(9,r1)
1672 ld r9,EX_R11(r3)
1673 ld r10,EX_R12(r3)
1674 ld r11,EX_R13(r3)
1675 std r9,GPR11(r1)
1676 std r10,GPR12(r1)
1677 std r11,GPR13(r1)
1678 BEGIN_FTR_SECTION
1679 ld r10,EX_CFAR(r3)
1680 std r10,ORIG_GPR3(r1)
1681 END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
1682 SAVE_8GPRS(14,r1)
1683 SAVE_10GPRS(22,r1)
1684 lhz r12,PACA_TRAP_SAVE(r13)
1685 std r12,_TRAP(r1)
1686 addi r11,r1,INT_FRAME_SIZE
1687 std r11,0(r1)
1688 li r12,0
1689 std r12,0(r11)
1690 ld r2,PACATOC(r13)
1691 ld r11,exception_marker@toc(r2)
1692 std r12,RESULT(r1)
1693 std r11,STACK_FRAME_OVERHEAD-16(r1)
1694 1: addi r3,r1,STACK_FRAME_OVERHEAD
1695 bl kernel_bad_stack
1696 b 1b