| blob | 3c6badcd53efced649af69ff9a4adf823b28d9a0 |
1 /*
2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License, version 2, as
4 * published by the Free Software Foundation.
5 *
6 * This program is distributed in the hope that it will be useful,
7 * but WITHOUT ANY WARRANTY; without even the implied warranty of
8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
9 * GNU General Public License for more details.
10 *
11 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
12 *
13 * Derived from book3s_rmhandlers.S and other files, which are:
14 *
15 * Copyright SUSE Linux Products GmbH 2009
16 *
17 * Authors: Alexander Graf <agraf@suse.de>
18 */
20 #include <asm/ppc_asm.h>
21 #include <asm/kvm_asm.h>
22 #include <asm/reg.h>
23 #include <asm/mmu.h>
24 #include <asm/page.h>
25 #include <asm/ptrace.h>
26 #include <asm/hvcall.h>
27 #include <asm/asm-offsets.h>
28 #include <asm/exception-64s.h>
29 #include <asm/kvm_book3s_asm.h>
30 #include <asm/mmu-hash64.h>
31 #include <asm/tm.h>
33 #define VCPU_GPRS_TM(reg) (((reg) * ULONG_SIZE) + VCPU_GPR_TM)
35 /* Values in HSTATE_NAPPING(r13) */
36 #define NAPPING_CEDE 1
37 #define NAPPING_NOVCPU 2
39 /*
40 * Call kvmppc_hv_entry in real mode.
41 * Must be called with interrupts hard-disabled.
42 *
43 * Input Registers:
44 *
45 * LR = return address to continue at after eventually re-enabling MMU
46 */
47 _GLOBAL_TOC(kvmppc_hv_entry_trampoline)
48 mflr r0
49 std r0, PPC_LR_STKOFF(r1)
50 stdu r1, -112(r1)
51 mfmsr r10
52 LOAD_REG_ADDR(r5, kvmppc_call_hv_entry)
53 li r0,MSR_RI
54 andc r0,r10,r0
55 li r6,MSR_IR | MSR_DR
56 andc r6,r10,r6
57 mtmsrd r0,1 /* clear RI in MSR */
58 mtsrr0 r5
59 mtsrr1 r6
60 RFI
62 kvmppc_call_hv_entry:
63 ld r4, HSTATE_KVM_VCPU(r13)
64 bl kvmppc_hv_entry
66 /* Back from guest - restore host state and return to caller */
68 BEGIN_FTR_SECTION
69 /* Restore host DABR and DABRX */
70 ld r5,HSTATE_DABR(r13)
71 li r6,7
72 mtspr SPRN_DABR,r5
73 mtspr SPRN_DABRX,r6
74 END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
76 /* Restore SPRG3 */
77 ld r3,PACA_SPRG_VDSO(r13)
78 mtspr SPRN_SPRG_VDSO_WRITE,r3
80 /* Reload the host's PMU registers */
81 ld r3, PACALPPACAPTR(r13) /* is the host using the PMU? */
82 lbz r4, LPPACA_PMCINUSE(r3)
83 cmpwi r4, 0
84 beq 23f /* skip if not */
85 BEGIN_FTR_SECTION
86 ld r3, HSTATE_MMCR0(r13)
87 andi. r4, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO
88 cmpwi r4, MMCR0_PMAO
89 beql kvmppc_fix_pmao
90 END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG)
91 lwz r3, HSTATE_PMC1(r13)
92 lwz r4, HSTATE_PMC2(r13)
93 lwz r5, HSTATE_PMC3(r13)
94 lwz r6, HSTATE_PMC4(r13)
95 lwz r8, HSTATE_PMC5(r13)
96 lwz r9, HSTATE_PMC6(r13)
97 mtspr SPRN_PMC1, r3
98 mtspr SPRN_PMC2, r4
99 mtspr SPRN_PMC3, r5
100 mtspr SPRN_PMC4, r6
101 mtspr SPRN_PMC5, r8
102 mtspr SPRN_PMC6, r9
103 ld r3, HSTATE_MMCR0(r13)
104 ld r4, HSTATE_MMCR1(r13)
105 ld r5, HSTATE_MMCRA(r13)
106 ld r6, HSTATE_SIAR(r13)
107 ld r7, HSTATE_SDAR(r13)
108 mtspr SPRN_MMCR1, r4
109 mtspr SPRN_MMCRA, r5
110 mtspr SPRN_SIAR, r6
111 mtspr SPRN_SDAR, r7
112 BEGIN_FTR_SECTION
113 ld r8, HSTATE_MMCR2(r13)
114 ld r9, HSTATE_SIER(r13)
115 mtspr SPRN_MMCR2, r8
116 mtspr SPRN_SIER, r9
117 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
118 mtspr SPRN_MMCR0, r3
119 isync
120 23:
122 /*
123 * Reload DEC. HDEC interrupts were disabled when
124 * we reloaded the host's LPCR value.
125 */
126 ld r3, HSTATE_DECEXP(r13)
127 mftb r4
128 subf r4, r4, r3
129 mtspr SPRN_DEC, r4
131 /* hwthread_req may have got set by cede or no vcpu, so clear it */
132 li r0, 0
133 stb r0, HSTATE_HWTHREAD_REQ(r13)
135 /*
136 * For external and machine check interrupts, we need
137 * to call the Linux handler to process the interrupt.
138 * We do that by jumping to absolute address 0x500 for
139 * external interrupts, or the machine_check_fwnmi label
140 * for machine checks (since firmware might have patched
141 * the vector area at 0x200). The [h]rfid at the end of the
142 * handler will return to the book3s_hv_interrupts.S code.
143 * For other interrupts we do the rfid to get back
144 * to the book3s_hv_interrupts.S code here.
145 */
146 ld r8, 112+PPC_LR_STKOFF(r1)
147 addi r1, r1, 112
148 ld r7, HSTATE_HOST_MSR(r13)
150 cmpwi cr1, r12, BOOK3S_INTERRUPT_MACHINE_CHECK
151 cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL
152 beq 11f
153 cmpwi r12, BOOK3S_INTERRUPT_H_DOORBELL
154 beq 15f /* Invoke the H_DOORBELL handler */
155 cmpwi cr2, r12, BOOK3S_INTERRUPT_HMI
156 beq cr2, 14f /* HMI check */
158 /* RFI into the highmem handler, or branch to interrupt handler */
159 mfmsr r6
160 li r0, MSR_RI
161 andc r6, r6, r0
162 mtmsrd r6, 1 /* Clear RI in MSR */
163 mtsrr0 r8
164 mtsrr1 r7
165 beq cr1, 13f /* machine check */
166 RFI
168 /* On POWER7, we have external interrupts set to use HSRR0/1 */
169 11: mtspr SPRN_HSRR0, r8
170 mtspr SPRN_HSRR1, r7
171 ba 0x500
173 13: b machine_check_fwnmi
175 14: mtspr SPRN_HSRR0, r8
176 mtspr SPRN_HSRR1, r7
177 b hmi_exception_after_realmode
179 15: mtspr SPRN_HSRR0, r8
180 mtspr SPRN_HSRR1, r7
181 ba 0xe80
183 kvmppc_primary_no_guest:
184 /* We handle this much like a ceded vcpu */
185 /* put the HDEC into the DEC, since HDEC interrupts don't wake us */
186 mfspr r3, SPRN_HDEC
187 mtspr SPRN_DEC, r3
188 /*
189 * Make sure the primary has finished the MMU switch.
190 * We should never get here on a secondary thread, but
191 * check it for robustness' sake.
192 */
193 ld r5, HSTATE_KVM_VCORE(r13)
194 65: lbz r0, VCORE_IN_GUEST(r5)
195 cmpwi r0, 0
196 beq 65b
197 /* Set LPCR. */
198 ld r8,VCORE_LPCR(r5)
199 mtspr SPRN_LPCR,r8
200 isync
201 /* set our bit in napping_threads */
202 ld r5, HSTATE_KVM_VCORE(r13)
203 lbz r7, HSTATE_PTID(r13)
204 li r0, 1
205 sld r0, r0, r7
206 addi r6, r5, VCORE_NAPPING_THREADS
207 1: lwarx r3, 0, r6
208 or r3, r3, r0
209 stwcx. r3, 0, r6
210 bne 1b
211 /* order napping_threads update vs testing entry_exit_map */
212 isync
213 li r12, 0
214 lwz r7, VCORE_ENTRY_EXIT(r5)
215 cmpwi r7, 0x100
216 bge kvm_novcpu_exit /* another thread already exiting */
217 li r3, NAPPING_NOVCPU
218 stb r3, HSTATE_NAPPING(r13)
220 li r3, 0 /* Don't wake on privileged (OS) doorbell */
221 b kvm_do_nap
223 kvm_novcpu_wakeup:
224 ld r1, HSTATE_HOST_R1(r13)
225 ld r5, HSTATE_KVM_VCORE(r13)
226 li r0, 0
227 stb r0, HSTATE_NAPPING(r13)
229 /* check the wake reason */
230 bl kvmppc_check_wake_reason
232 /* see if any other thread is already exiting */
233 lwz r0, VCORE_ENTRY_EXIT(r5)
234 cmpwi r0, 0x100
235 bge kvm_novcpu_exit
237 /* clear our bit in napping_threads */
238 lbz r7, HSTATE_PTID(r13)
239 li r0, 1
240 sld r0, r0, r7
241 addi r6, r5, VCORE_NAPPING_THREADS
242 4: lwarx r7, 0, r6
243 andc r7, r7, r0
244 stwcx. r7, 0, r6
245 bne 4b
247 /* See if the wake reason means we need to exit */
248 cmpdi r3, 0
249 bge kvm_novcpu_exit
251 /* See if our timeslice has expired (HDEC is negative) */
252 mfspr r0, SPRN_HDEC
253 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER
254 cmpwi r0, 0
255 blt kvm_novcpu_exit
257 /* Got an IPI but other vcpus aren't yet exiting, must be a latecomer */
258 ld r4, HSTATE_KVM_VCPU(r13)
259 cmpdi r4, 0
260 beq kvmppc_primary_no_guest
262 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
263 addi r3, r4, VCPU_TB_RMENTRY
264 bl kvmhv_start_timing
265 #endif
266 b kvmppc_got_guest
268 kvm_novcpu_exit:
269 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
270 ld r4, HSTATE_KVM_VCPU(r13)
271 cmpdi r4, 0
272 beq 13f
273 addi r3, r4, VCPU_TB_RMEXIT
274 bl kvmhv_accumulate_time
275 #endif
276 13: mr r3, r12
277 stw r12, 112-4(r1)
278 bl kvmhv_commence_exit
279 nop
280 lwz r12, 112-4(r1)
281 b kvmhv_switch_to_host
283 /*
284 * We come in here when wakened from nap mode.
285 * Relocation is off and most register values are lost.
286 * r13 points to the PACA.
287 */
288 .globl kvm_start_guest
289 kvm_start_guest:
291 /* Set runlatch bit the minute you wake up from nap */
292 mfspr r0, SPRN_CTRLF
293 ori r0, r0, 1
294 mtspr SPRN_CTRLT, r0
296 ld r2,PACATOC(r13)
298 li r0,KVM_HWTHREAD_IN_KVM
299 stb r0,HSTATE_HWTHREAD_STATE(r13)
301 /* NV GPR values from power7_idle() will no longer be valid */
302 li r0,1
303 stb r0,PACA_NAPSTATELOST(r13)
305 /* were we napping due to cede? */
306 lbz r0,HSTATE_NAPPING(r13)
307 cmpwi r0,NAPPING_CEDE
308 beq kvm_end_cede
309 cmpwi r0,NAPPING_NOVCPU
310 beq kvm_novcpu_wakeup
312 ld r1,PACAEMERGSP(r13)
313 subi r1,r1,STACK_FRAME_OVERHEAD
315 /*
316 * We weren't napping due to cede, so this must be a secondary
317 * thread being woken up to run a guest, or being woken up due
318 * to a stray IPI. (Or due to some machine check or hypervisor
319 * maintenance interrupt while the core is in KVM.)
320 */
322 /* Check the wake reason in SRR1 to see why we got here */
323 bl kvmppc_check_wake_reason
324 cmpdi r3, 0
325 bge kvm_no_guest
327 /* get vcore pointer, NULL if we have nothing to run */
328 ld r5,HSTATE_KVM_VCORE(r13)
329 cmpdi r5,0
330 /* if we have no vcore to run, go back to sleep */
331 beq kvm_no_guest
333 kvm_secondary_got_guest:
335 /* Set HSTATE_DSCR(r13) to something sensible */
336 ld r6, PACA_DSCR_DEFAULT(r13)
337 std r6, HSTATE_DSCR(r13)
339 /* On thread 0 of a subcore, set HDEC to max */
340 lbz r4, HSTATE_PTID(r13)
341 cmpwi r4, 0
342 bne 63f
343 lis r6, 0x7fff
344 ori r6, r6, 0xffff
345 mtspr SPRN_HDEC, r6
346 /* and set per-LPAR registers, if doing dynamic micro-threading */
347 ld r6, HSTATE_SPLIT_MODE(r13)
348 cmpdi r6, 0
349 beq 63f
350 ld r0, KVM_SPLIT_RPR(r6)
351 mtspr SPRN_RPR, r0
352 ld r0, KVM_SPLIT_PMMAR(r6)
353 mtspr SPRN_PMMAR, r0
354 ld r0, KVM_SPLIT_LDBAR(r6)
355 mtspr SPRN_LDBAR, r0
356 isync
357 63:
358 /* Order load of vcpu after load of vcore */
359 lwsync
360 ld r4, HSTATE_KVM_VCPU(r13)
361 bl kvmppc_hv_entry
363 /* Back from the guest, go back to nap */
364 /* Clear our vcpu and vcore pointers so we don't come back in early */
365 li r0, 0
366 std r0, HSTATE_KVM_VCPU(r13)
367 /*
368 * Once we clear HSTATE_KVM_VCORE(r13), the code in
369 * kvmppc_run_core() is going to assume that all our vcpu
370 * state is visible in memory. This lwsync makes sure
371 * that that is true.
372 */
373 lwsync
374 std r0, HSTATE_KVM_VCORE(r13)
376 /*
377 * At this point we have finished executing in the guest.
378 * We need to wait for hwthread_req to become zero, since
379 * we may not turn on the MMU while hwthread_req is non-zero.
380 * While waiting we also need to check if we get given a vcpu to run.
381 */
382 kvm_no_guest:
383 lbz r3, HSTATE_HWTHREAD_REQ(r13)
384 cmpwi r3, 0
385 bne 53f
386 HMT_MEDIUM
387 li r0, KVM_HWTHREAD_IN_KERNEL
388 stb r0, HSTATE_HWTHREAD_STATE(r13)
389 /* need to recheck hwthread_req after a barrier, to avoid race */
390 sync
391 lbz r3, HSTATE_HWTHREAD_REQ(r13)
392 cmpwi r3, 0
393 bne 54f
394 /*
395 * We jump to power7_wakeup_loss, which will return to the caller
396 * of power7_nap in the powernv cpu offline loop. The value we
397 * put in r3 becomes the return value for power7_nap.
398 */
399 li r3, LPCR_PECE0
400 mfspr r4, SPRN_LPCR
401 rlwimi r4, r3, 0, LPCR_PECE0 | LPCR_PECE1
402 mtspr SPRN_LPCR, r4
403 li r3, 0
404 b power7_wakeup_loss
406 53: HMT_LOW
407 ld r5, HSTATE_KVM_VCORE(r13)
408 cmpdi r5, 0
409 bne 60f
410 ld r3, HSTATE_SPLIT_MODE(r13)
411 cmpdi r3, 0
412 beq kvm_no_guest
413 lbz r0, KVM_SPLIT_DO_NAP(r3)
414 cmpwi r0, 0
415 beq kvm_no_guest
416 HMT_MEDIUM
417 b kvm_unsplit_nap
418 60: HMT_MEDIUM
419 b kvm_secondary_got_guest
421 54: li r0, KVM_HWTHREAD_IN_KVM
422 stb r0, HSTATE_HWTHREAD_STATE(r13)
423 b kvm_no_guest
425 /*
426 * Here the primary thread is trying to return the core to
427 * whole-core mode, so we need to nap.
428 */
429 kvm_unsplit_nap:
430 /*
431 * Ensure that secondary doesn't nap when it has
432 * its vcore pointer set.
433 */
434 sync /* matches smp_mb() before setting split_info.do_nap */
435 ld r0, HSTATE_KVM_VCORE(r13)
436 cmpdi r0, 0
437 bne kvm_no_guest
438 /* clear any pending message */
439 BEGIN_FTR_SECTION
440 lis r6, (PPC_DBELL_SERVER << (63-36))@h
441 PPC_MSGCLR(6)
442 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
443 /* Set kvm_split_mode.napped[tid] = 1 */
444 ld r3, HSTATE_SPLIT_MODE(r13)
445 li r0, 1
446 lhz r4, PACAPACAINDEX(r13)
447 clrldi r4, r4, 61 /* micro-threading => P8 => 8 threads/core */
448 addi r4, r4, KVM_SPLIT_NAPPED
449 stbx r0, r3, r4
450 /* Check the do_nap flag again after setting napped[] */
451 sync
452 lbz r0, KVM_SPLIT_DO_NAP(r3)
453 cmpwi r0, 0
454 beq 57f
455 li r3, (LPCR_PECEDH | LPCR_PECE0) >> 4
456 mfspr r4, SPRN_LPCR
457 rlwimi r4, r3, 4, (LPCR_PECEDP | LPCR_PECEDH | LPCR_PECE0 | LPCR_PECE1)
458 mtspr SPRN_LPCR, r4
459 isync
460 std r0, HSTATE_SCRATCH0(r13)
461 ptesync
462 ld r0, HSTATE_SCRATCH0(r13)
463 1: cmpd r0, r0
464 bne 1b
465 nap
466 b .
468 57: li r0, 0
469 stbx r0, r3, r4
470 b kvm_no_guest
472 /******************************************************************************
473 * *
474 * Entry code *
475 * *
476 *****************************************************************************/
478 .global kvmppc_hv_entry
479 kvmppc_hv_entry:
481 /* Required state:
482 *
483 * R4 = vcpu pointer (or NULL)
484 * MSR = ~IR|DR
485 * R13 = PACA
486 * R1 = host R1
487 * R2 = TOC
488 * all other volatile GPRS = free
489 */
490 mflr r0
491 std r0, PPC_LR_STKOFF(r1)
492 stdu r1, -112(r1)
494 /* Save R1 in the PACA */
495 std r1, HSTATE_HOST_R1(r13)
497 li r6, KVM_GUEST_MODE_HOST_HV
498 stb r6, HSTATE_IN_GUEST(r13)
500 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
501 /* Store initial timestamp */
502 cmpdi r4, 0
503 beq 1f
504 addi r3, r4, VCPU_TB_RMENTRY
505 bl kvmhv_start_timing
506 1:
507 #endif
508 /* Clear out SLB */
509 li r6,0
510 slbmte r6,r6
511 slbia
512 ptesync
514 /*
515 * POWER7/POWER8 host -> guest partition switch code.
516 * We don't have to lock against concurrent tlbies,
517 * but we do have to coordinate across hardware threads.
518 */
519 /* Set bit in entry map iff exit map is zero. */
520 ld r5, HSTATE_KVM_VCORE(r13)
521 li r7, 1
522 lbz r6, HSTATE_PTID(r13)
523 sld r7, r7, r6
524 addi r9, r5, VCORE_ENTRY_EXIT
525 21: lwarx r3, 0, r9
526 cmpwi r3, 0x100 /* any threads starting to exit? */
527 bge secondary_too_late /* if so we're too late to the party */
528 or r3, r3, r7
529 stwcx. r3, 0, r9
530 bne 21b
532 /* Primary thread switches to guest partition. */
533 ld r9,VCORE_KVM(r5) /* pointer to struct kvm */
534 cmpwi r6,0
535 bne 10f
536 ld r6,KVM_SDR1(r9)
537 lwz r7,KVM_LPID(r9)
538 li r0,LPID_RSVD /* switch to reserved LPID */
539 mtspr SPRN_LPID,r0
540 ptesync
541 mtspr SPRN_SDR1,r6 /* switch to partition page table */
542 mtspr SPRN_LPID,r7
543 isync
545 /* See if we need to flush the TLB */
546 lhz r6,PACAPACAINDEX(r13) /* test_bit(cpu, need_tlb_flush) */
547 clrldi r7,r6,64-6 /* extract bit number (6 bits) */
548 srdi r6,r6,6 /* doubleword number */
549 sldi r6,r6,3 /* address offset */
550 add r6,r6,r9
551 addi r6,r6,KVM_NEED_FLUSH /* dword in kvm->arch.need_tlb_flush */
552 li r0,1
553 sld r0,r0,r7
554 ld r7,0(r6)
555 and. r7,r7,r0
556 beq 22f
557 23: ldarx r7,0,r6 /* if set, clear the bit */
558 andc r7,r7,r0
559 stdcx. r7,0,r6
560 bne 23b
561 /* Flush the TLB of any entries for this LPID */
562 /* use arch 2.07S as a proxy for POWER8 */
563 BEGIN_FTR_SECTION
564 li r6,512 /* POWER8 has 512 sets */
565 FTR_SECTION_ELSE
566 li r6,128 /* POWER7 has 128 sets */
567 ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S)
568 mtctr r6
569 li r7,0x800 /* IS field = 0b10 */
570 ptesync
571 28: tlbiel r7
572 addi r7,r7,0x1000
573 bdnz 28b
574 ptesync
576 /* Add timebase offset onto timebase */
577 22: ld r8,VCORE_TB_OFFSET(r5)
578 cmpdi r8,0
579 beq 37f
580 mftb r6 /* current host timebase */
581 add r8,r8,r6
582 mtspr SPRN_TBU40,r8 /* update upper 40 bits */
583 mftb r7 /* check if lower 24 bits overflowed */
584 clrldi r6,r6,40
585 clrldi r7,r7,40
586 cmpld r7,r6
587 bge 37f
588 addis r8,r8,0x100 /* if so, increment upper 40 bits */
589 mtspr SPRN_TBU40,r8
591 /* Load guest PCR value to select appropriate compat mode */
592 37: ld r7, VCORE_PCR(r5)
593 cmpdi r7, 0
594 beq 38f
595 mtspr SPRN_PCR, r7
596 38:
598 BEGIN_FTR_SECTION
599 /* DPDES is shared between threads */
600 ld r8, VCORE_DPDES(r5)
601 mtspr SPRN_DPDES, r8
602 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
604 li r0,1
605 stb r0,VCORE_IN_GUEST(r5) /* signal secondaries to continue */
607 /* Do we have a guest vcpu to run? */
608 10: cmpdi r4, 0
609 beq kvmppc_primary_no_guest
610 kvmppc_got_guest:
612 /* Load up guest SLB entries */
613 lwz r5,VCPU_SLB_MAX(r4)
614 cmpwi r5,0
615 beq 9f
616 mtctr r5
617 addi r6,r4,VCPU_SLB
618 1: ld r8,VCPU_SLB_E(r6)
619 ld r9,VCPU_SLB_V(r6)
620 slbmte r9,r8
621 addi r6,r6,VCPU_SLB_SIZE
622 bdnz 1b
623 9:
624 /* Increment yield count if they have a VPA */
625 ld r3, VCPU_VPA(r4)
626 cmpdi r3, 0
627 beq 25f
628 li r6, LPPACA_YIELDCOUNT
629 LWZX_BE r5, r3, r6
630 addi r5, r5, 1
631 STWX_BE r5, r3, r6
632 li r6, 1
633 stb r6, VCPU_VPA_DIRTY(r4)
634 25:
636 /* Save purr/spurr */
637 mfspr r5,SPRN_PURR
638 mfspr r6,SPRN_SPURR
639 std r5,HSTATE_PURR(r13)
640 std r6,HSTATE_SPURR(r13)
641 ld r7,VCPU_PURR(r4)
642 ld r8,VCPU_SPURR(r4)
643 mtspr SPRN_PURR,r7
644 mtspr SPRN_SPURR,r8
646 BEGIN_FTR_SECTION
647 /* Set partition DABR */
648 /* Do this before re-enabling PMU to avoid P7 DABR corruption bug */
649 lwz r5,VCPU_DABRX(r4)
650 ld r6,VCPU_DABR(r4)
651 mtspr SPRN_DABRX,r5
652 mtspr SPRN_DABR,r6
653 isync
654 END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
656 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
657 BEGIN_FTR_SECTION
658 b skip_tm
659 END_FTR_SECTION_IFCLR(CPU_FTR_TM)
661 /* Turn on TM/FP/VSX/VMX so we can restore them. */
662 mfmsr r5
663 li r6, MSR_TM >> 32
664 sldi r6, r6, 32
665 or r5, r5, r6
666 ori r5, r5, MSR_FP
667 oris r5, r5, (MSR_VEC | MSR_VSX)@h
668 mtmsrd r5
670 /*
671 * The user may change these outside of a transaction, so they must
672 * always be context switched.
673 */
674 ld r5, VCPU_TFHAR(r4)
675 ld r6, VCPU_TFIAR(r4)
676 ld r7, VCPU_TEXASR(r4)
677 mtspr SPRN_TFHAR, r5
678 mtspr SPRN_TFIAR, r6
679 mtspr SPRN_TEXASR, r7
681 ld r5, VCPU_MSR(r4)
682 rldicl. r5, r5, 64 - MSR_TS_S_LG, 62
683 beq skip_tm /* TM not active in guest */
685 /* Make sure the failure summary is set, otherwise we'll program check
686 * when we trechkpt. It's possible that this might have been not set
687 * on a kvmppc_set_one_reg() call but we shouldn't let this crash the
688 * host.
689 */
690 oris r7, r7, (TEXASR_FS)@h
691 mtspr SPRN_TEXASR, r7
693 /*
694 * We need to load up the checkpointed state for the guest.
695 * We need to do this early as it will blow away any GPRs, VSRs and
696 * some SPRs.
697 */
699 mr r31, r4
700 addi r3, r31, VCPU_FPRS_TM
701 bl load_fp_state
702 addi r3, r31, VCPU_VRS_TM
703 bl load_vr_state
704 mr r4, r31
705 lwz r7, VCPU_VRSAVE_TM(r4)
706 mtspr SPRN_VRSAVE, r7
708 ld r5, VCPU_LR_TM(r4)
709 lwz r6, VCPU_CR_TM(r4)
710 ld r7, VCPU_CTR_TM(r4)
711 ld r8, VCPU_AMR_TM(r4)
712 ld r9, VCPU_TAR_TM(r4)
713 mtlr r5
714 mtcr r6
715 mtctr r7
716 mtspr SPRN_AMR, r8
717 mtspr SPRN_TAR, r9
719 /*
720 * Load up PPR and DSCR values but don't put them in the actual SPRs
721 * till the last moment to avoid running with userspace PPR and DSCR for
722 * too long.
723 */
724 ld r29, VCPU_DSCR_TM(r4)
725 ld r30, VCPU_PPR_TM(r4)
727 std r2, PACATMSCRATCH(r13) /* Save TOC */
729 /* Clear the MSR RI since r1, r13 are all going to be foobar. */
730 li r5, 0
731 mtmsrd r5, 1
733 /* Load GPRs r0-r28 */
734 reg = 0
735 .rept 29
736 ld reg, VCPU_GPRS_TM(reg)(r31)
737 reg = reg + 1
738 .endr
740 mtspr SPRN_DSCR, r29
741 mtspr SPRN_PPR, r30
743 /* Load final GPRs */
744 ld 29, VCPU_GPRS_TM(29)(r31)
745 ld 30, VCPU_GPRS_TM(30)(r31)
746 ld 31, VCPU_GPRS_TM(31)(r31)
748 /* TM checkpointed state is now setup. All GPRs are now volatile. */
749 TRECHKPT
751 /* Now let's get back the state we need. */
752 HMT_MEDIUM
753 GET_PACA(r13)
754 ld r29, HSTATE_DSCR(r13)
755 mtspr SPRN_DSCR, r29
756 ld r4, HSTATE_KVM_VCPU(r13)
757 ld r1, HSTATE_HOST_R1(r13)
758 ld r2, PACATMSCRATCH(r13)
760 /* Set the MSR RI since we have our registers back. */
761 li r5, MSR_RI
762 mtmsrd r5, 1
763 skip_tm:
764 #endif
766 /* Load guest PMU registers */
767 /* R4 is live here (vcpu pointer) */
768 li r3, 1
769 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */
770 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */
771 isync
772 BEGIN_FTR_SECTION
773 ld r3, VCPU_MMCR(r4)
774 andi. r5, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO
775 cmpwi r5, MMCR0_PMAO
776 beql kvmppc_fix_pmao
777 END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG)
778 lwz r3, VCPU_PMC(r4) /* always load up guest PMU registers */
779 lwz r5, VCPU_PMC + 4(r4) /* to prevent information leak */
780 lwz r6, VCPU_PMC + 8(r4)
781 lwz r7, VCPU_PMC + 12(r4)
782 lwz r8, VCPU_PMC + 16(r4)
783 lwz r9, VCPU_PMC + 20(r4)
784 mtspr SPRN_PMC1, r3
785 mtspr SPRN_PMC2, r5
786 mtspr SPRN_PMC3, r6
787 mtspr SPRN_PMC4, r7
788 mtspr SPRN_PMC5, r8
789 mtspr SPRN_PMC6, r9
790 ld r3, VCPU_MMCR(r4)
791 ld r5, VCPU_MMCR + 8(r4)
792 ld r6, VCPU_MMCR + 16(r4)
793 ld r7, VCPU_SIAR(r4)
794 ld r8, VCPU_SDAR(r4)
795 mtspr SPRN_MMCR1, r5
796 mtspr SPRN_MMCRA, r6
797 mtspr SPRN_SIAR, r7
798 mtspr SPRN_SDAR, r8
799 BEGIN_FTR_SECTION
800 ld r5, VCPU_MMCR + 24(r4)
801 ld r6, VCPU_SIER(r4)
802 lwz r7, VCPU_PMC + 24(r4)
803 lwz r8, VCPU_PMC + 28(r4)
804 ld r9, VCPU_MMCR + 32(r4)
805 mtspr SPRN_MMCR2, r5
806 mtspr SPRN_SIER, r6
807 mtspr SPRN_SPMC1, r7
808 mtspr SPRN_SPMC2, r8
809 mtspr SPRN_MMCRS, r9
810 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
811 mtspr SPRN_MMCR0, r3
812 isync
814 /* Load up FP, VMX and VSX registers */
815 bl kvmppc_load_fp
817 ld r14, VCPU_GPR(R14)(r4)
818 ld r15, VCPU_GPR(R15)(r4)
819 ld r16, VCPU_GPR(R16)(r4)
820 ld r17, VCPU_GPR(R17)(r4)
821 ld r18, VCPU_GPR(R18)(r4)
822 ld r19, VCPU_GPR(R19)(r4)
823 ld r20, VCPU_GPR(R20)(r4)
824 ld r21, VCPU_GPR(R21)(r4)
825 ld r22, VCPU_GPR(R22)(r4)
826 ld r23, VCPU_GPR(R23)(r4)
827 ld r24, VCPU_GPR(R24)(r4)
828 ld r25, VCPU_GPR(R25)(r4)
829 ld r26, VCPU_GPR(R26)(r4)
830 ld r27, VCPU_GPR(R27)(r4)
831 ld r28, VCPU_GPR(R28)(r4)
832 ld r29, VCPU_GPR(R29)(r4)
833 ld r30, VCPU_GPR(R30)(r4)
834 ld r31, VCPU_GPR(R31)(r4)
836 /* Switch DSCR to guest value */
837 ld r5, VCPU_DSCR(r4)
838 mtspr SPRN_DSCR, r5
840 BEGIN_FTR_SECTION
841 /* Skip next section on POWER7 */
842 b 8f
843 END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
844 /* Turn on TM so we can access TFHAR/TFIAR/TEXASR */
845 mfmsr r8
846 li r0, 1
847 rldimi r8, r0, MSR_TM_LG, 63-MSR_TM_LG
848 mtmsrd r8
850 /* Load up POWER8-specific registers */
851 ld r5, VCPU_IAMR(r4)
852 lwz r6, VCPU_PSPB(r4)
853 ld r7, VCPU_FSCR(r4)
854 mtspr SPRN_IAMR, r5
855 mtspr SPRN_PSPB, r6
856 mtspr SPRN_FSCR, r7
857 ld r5, VCPU_DAWR(r4)
858 ld r6, VCPU_DAWRX(r4)
859 ld r7, VCPU_CIABR(r4)
860 ld r8, VCPU_TAR(r4)
861 mtspr SPRN_DAWR, r5
862 mtspr SPRN_DAWRX, r6
863 mtspr SPRN_CIABR, r7
864 mtspr SPRN_TAR, r8
865 ld r5, VCPU_IC(r4)
866 ld r6, VCPU_VTB(r4)
867 mtspr SPRN_IC, r5
868 mtspr SPRN_VTB, r6
869 ld r8, VCPU_EBBHR(r4)
870 mtspr SPRN_EBBHR, r8
871 ld r5, VCPU_EBBRR(r4)
872 ld r6, VCPU_BESCR(r4)
873 ld r7, VCPU_CSIGR(r4)
874 ld r8, VCPU_TACR(r4)
875 mtspr SPRN_EBBRR, r5
876 mtspr SPRN_BESCR, r6
877 mtspr SPRN_CSIGR, r7
878 mtspr SPRN_TACR, r8
879 ld r5, VCPU_TCSCR(r4)
880 ld r6, VCPU_ACOP(r4)
881 lwz r7, VCPU_GUEST_PID(r4)
882 ld r8, VCPU_WORT(r4)
883 mtspr SPRN_TCSCR, r5
884 mtspr SPRN_ACOP, r6
885 mtspr SPRN_PID, r7
886 mtspr SPRN_WORT, r8
887 8:
889 /*
890 * Set the decrementer to the guest decrementer.
891 */
892 ld r8,VCPU_DEC_EXPIRES(r4)
893 /* r8 is a host timebase value here, convert to guest TB */
894 ld r5,HSTATE_KVM_VCORE(r13)
895 ld r6,VCORE_TB_OFFSET(r5)
896 add r8,r8,r6
897 mftb r7
898 subf r3,r7,r8
899 mtspr SPRN_DEC,r3
900 stw r3,VCPU_DEC(r4)
902 ld r5, VCPU_SPRG0(r4)
903 ld r6, VCPU_SPRG1(r4)
904 ld r7, VCPU_SPRG2(r4)
905 ld r8, VCPU_SPRG3(r4)
906 mtspr SPRN_SPRG0, r5
907 mtspr SPRN_SPRG1, r6
908 mtspr SPRN_SPRG2, r7
909 mtspr SPRN_SPRG3, r8
911 /* Load up DAR and DSISR */
912 ld r5, VCPU_DAR(r4)
913 lwz r6, VCPU_DSISR(r4)
914 mtspr SPRN_DAR, r5
915 mtspr SPRN_DSISR, r6
917 /* Restore AMR and UAMOR, set AMOR to all 1s */
918 ld r5,VCPU_AMR(r4)
919 ld r6,VCPU_UAMOR(r4)
920 li r7,-1
921 mtspr SPRN_AMR,r5
922 mtspr SPRN_UAMOR,r6
923 mtspr SPRN_AMOR,r7
925 /* Restore state of CTRL run bit; assume 1 on entry */
926 lwz r5,VCPU_CTRL(r4)
927 andi. r5,r5,1
928 bne 4f
929 mfspr r6,SPRN_CTRLF
930 clrrdi r6,r6,1
931 mtspr SPRN_CTRLT,r6
932 4:
933 /* Secondary threads wait for primary to have done partition switch */
934 ld r5, HSTATE_KVM_VCORE(r13)
935 lbz r6, HSTATE_PTID(r13)
936 cmpwi r6, 0
937 beq 21f
938 lbz r0, VCORE_IN_GUEST(r5)
939 cmpwi r0, 0
940 bne 21f
941 HMT_LOW
942 20: lwz r3, VCORE_ENTRY_EXIT(r5)
943 cmpwi r3, 0x100
944 bge no_switch_exit
945 lbz r0, VCORE_IN_GUEST(r5)
946 cmpwi r0, 0
947 beq 20b
948 HMT_MEDIUM
949 21:
950 /* Set LPCR. */
951 ld r8,VCORE_LPCR(r5)
952 mtspr SPRN_LPCR,r8
953 isync
955 /* Check if HDEC expires soon */
956 mfspr r3, SPRN_HDEC
957 cmpwi r3, 512 /* 1 microsecond */
958 blt hdec_soon
960 ld r6, VCPU_CTR(r4)
961 ld r7, VCPU_XER(r4)
963 mtctr r6
964 mtxer r7
966 kvmppc_cede_reentry: /* r4 = vcpu, r13 = paca */
967 ld r10, VCPU_PC(r4)
968 ld r11, VCPU_MSR(r4)
969 ld r6, VCPU_SRR0(r4)
970 ld r7, VCPU_SRR1(r4)
971 mtspr SPRN_SRR0, r6
972 mtspr SPRN_SRR1, r7
974 deliver_guest_interrupt:
975 /* r11 = vcpu->arch.msr & ~MSR_HV */
976 rldicl r11, r11, 63 - MSR_HV_LG, 1
977 rotldi r11, r11, 1 + MSR_HV_LG
978 ori r11, r11, MSR_ME
980 /* Check if we can deliver an external or decrementer interrupt now */
981 ld r0, VCPU_PENDING_EXC(r4)
982 rldicl r0, r0, 64 - BOOK3S_IRQPRIO_EXTERNAL_LEVEL, 63
983 cmpdi cr1, r0, 0
984 andi. r8, r11, MSR_EE
985 mfspr r8, SPRN_LPCR
986 /* Insert EXTERNAL_LEVEL bit into LPCR at the MER bit position */
987 rldimi r8, r0, LPCR_MER_SH, 63 - LPCR_MER_SH
988 mtspr SPRN_LPCR, r8
989 isync
990 beq 5f
991 li r0, BOOK3S_INTERRUPT_EXTERNAL
992 bne cr1, 12f
993 mfspr r0, SPRN_DEC
994 cmpwi r0, 0
995 li r0, BOOK3S_INTERRUPT_DECREMENTER
996 bge 5f
998 12: mtspr SPRN_SRR0, r10
999 mr r10,r0
1000 mtspr SPRN_SRR1, r11
1001 mr r9, r4
1002 bl kvmppc_msr_interrupt
1003 5:
1005 /*
1006 * Required state:
1007 * R4 = vcpu
1008 * R10: value for HSRR0
1009 * R11: value for HSRR1
1010 * R13 = PACA
1011 */
1012 fast_guest_return:
1013 li r0,0
1014 stb r0,VCPU_CEDED(r4) /* cancel cede */
1015 mtspr SPRN_HSRR0,r10
1016 mtspr SPRN_HSRR1,r11
1018 /* Activate guest mode, so faults get handled by KVM */
1019 li r9, KVM_GUEST_MODE_GUEST_HV
1020 stb r9, HSTATE_IN_GUEST(r13)
1022 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1023 /* Accumulate timing */
1024 addi r3, r4, VCPU_TB_GUEST
1025 bl kvmhv_accumulate_time
1026 #endif
1028 /* Enter guest */
1030 BEGIN_FTR_SECTION
1031 ld r5, VCPU_CFAR(r4)
1032 mtspr SPRN_CFAR, r5
1033 END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
1034 BEGIN_FTR_SECTION
1035 ld r0, VCPU_PPR(r4)
1036 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
1038 ld r5, VCPU_LR(r4)
1039 lwz r6, VCPU_CR(r4)
1040 mtlr r5
1041 mtcr r6
1043 ld r1, VCPU_GPR(R1)(r4)
1044 ld r2, VCPU_GPR(R2)(r4)
1045 ld r3, VCPU_GPR(R3)(r4)
1046 ld r5, VCPU_GPR(R5)(r4)
1047 ld r6, VCPU_GPR(R6)(r4)
1048 ld r7, VCPU_GPR(R7)(r4)
1049 ld r8, VCPU_GPR(R8)(r4)
1050 ld r9, VCPU_GPR(R9)(r4)
1051 ld r10, VCPU_GPR(R10)(r4)
1052 ld r11, VCPU_GPR(R11)(r4)
1053 ld r12, VCPU_GPR(R12)(r4)
1054 ld r13, VCPU_GPR(R13)(r4)
1056 BEGIN_FTR_SECTION
1057 mtspr SPRN_PPR, r0
1058 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
1059 ld r0, VCPU_GPR(R0)(r4)
1060 ld r4, VCPU_GPR(R4)(r4)
1062 hrfid
1063 b .
1065 secondary_too_late:
1066 li r12, 0
1067 cmpdi r4, 0
1068 beq 11f
1069 stw r12, VCPU_TRAP(r4)
1070 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1071 addi r3, r4, VCPU_TB_RMEXIT
1072 bl kvmhv_accumulate_time
1073 #endif
1074 11: b kvmhv_switch_to_host
1076 no_switch_exit:
1077 HMT_MEDIUM
1078 li r12, 0
1079 b 12f
1080 hdec_soon:
1081 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER
1082 12: stw r12, VCPU_TRAP(r4)
1083 mr r9, r4
1084 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1085 addi r3, r4, VCPU_TB_RMEXIT
1086 bl kvmhv_accumulate_time
1087 #endif
1088 b guest_exit_cont
1090 /******************************************************************************
1091 * *
1092 * Exit code *
1093 * *
1094 *****************************************************************************/
1096 /*
1097 * We come here from the first-level interrupt handlers.
1098 */
1099 .globl kvmppc_interrupt_hv
1100 kvmppc_interrupt_hv:
1101 /*
1102 * Register contents:
1103 * R12 = interrupt vector
1104 * R13 = PACA
1105 * guest CR, R12 saved in shadow VCPU SCRATCH1/0
1106 * guest R13 saved in SPRN_SCRATCH0
1107 */
1108 std r9, HSTATE_SCRATCH2(r13)
1110 lbz r9, HSTATE_IN_GUEST(r13)
1111 cmpwi r9, KVM_GUEST_MODE_HOST_HV
1112 beq kvmppc_bad_host_intr
1113 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
1114 cmpwi r9, KVM_GUEST_MODE_GUEST
1115 ld r9, HSTATE_SCRATCH2(r13)
1116 beq kvmppc_interrupt_pr
1117 #endif
1118 /* We're now back in the host but in guest MMU context */
1119 li r9, KVM_GUEST_MODE_HOST_HV
1120 stb r9, HSTATE_IN_GUEST(r13)
1122 ld r9, HSTATE_KVM_VCPU(r13)
1124 /* Save registers */
1126 std r0, VCPU_GPR(R0)(r9)
1127 std r1, VCPU_GPR(R1)(r9)
1128 std r2, VCPU_GPR(R2)(r9)
1129 std r3, VCPU_GPR(R3)(r9)
1130 std r4, VCPU_GPR(R4)(r9)
1131 std r5, VCPU_GPR(R5)(r9)
1132 std r6, VCPU_GPR(R6)(r9)
1133 std r7, VCPU_GPR(R7)(r9)
1134 std r8, VCPU_GPR(R8)(r9)
1135 ld r0, HSTATE_SCRATCH2(r13)
1136 std r0, VCPU_GPR(R9)(r9)
1137 std r10, VCPU_GPR(R10)(r9)
1138 std r11, VCPU_GPR(R11)(r9)
1139 ld r3, HSTATE_SCRATCH0(r13)
1140 lwz r4, HSTATE_SCRATCH1(r13)
1141 std r3, VCPU_GPR(R12)(r9)
1142 stw r4, VCPU_CR(r9)
1143 BEGIN_FTR_SECTION
1144 ld r3, HSTATE_CFAR(r13)
1145 std r3, VCPU_CFAR(r9)
1146 END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
1147 BEGIN_FTR_SECTION
1148 ld r4, HSTATE_PPR(r13)
1149 std r4, VCPU_PPR(r9)
1150 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
1152 /* Restore R1/R2 so we can handle faults */
1153 ld r1, HSTATE_HOST_R1(r13)
1154 ld r2, PACATOC(r13)
1156 mfspr r10, SPRN_SRR0
1157 mfspr r11, SPRN_SRR1
1158 std r10, VCPU_SRR0(r9)
1159 std r11, VCPU_SRR1(r9)
1160 andi. r0, r12, 2 /* need to read HSRR0/1? */
1161 beq 1f
1162 mfspr r10, SPRN_HSRR0
1163 mfspr r11, SPRN_HSRR1
1164 clrrdi r12, r12, 2
1165 1: std r10, VCPU_PC(r9)
1166 std r11, VCPU_MSR(r9)
1168 GET_SCRATCH0(r3)
1169 mflr r4
1170 std r3, VCPU_GPR(R13)(r9)
1171 std r4, VCPU_LR(r9)
1173 stw r12,VCPU_TRAP(r9)
1175 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1176 addi r3, r9, VCPU_TB_RMINTR
1177 mr r4, r9
1178 bl kvmhv_accumulate_time
1179 ld r5, VCPU_GPR(R5)(r9)
1180 ld r6, VCPU_GPR(R6)(r9)
1181 ld r7, VCPU_GPR(R7)(r9)
1182 ld r8, VCPU_GPR(R8)(r9)
1183 #endif
1185 /* Save HEIR (HV emulation assist reg) in emul_inst
1186 if this is an HEI (HV emulation interrupt, e40) */
1187 li r3,KVM_INST_FETCH_FAILED
1188 stw r3,VCPU_LAST_INST(r9)
1189 cmpwi r12,BOOK3S_INTERRUPT_H_EMUL_ASSIST
1190 bne 11f
1191 mfspr r3,SPRN_HEIR
1192 11: stw r3,VCPU_HEIR(r9)
1194 /* these are volatile across C function calls */
1195 mfctr r3
1196 mfxer r4
1197 std r3, VCPU_CTR(r9)
1198 std r4, VCPU_XER(r9)
1200 /* If this is a page table miss then see if it's theirs or ours */
1201 cmpwi r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
1202 beq kvmppc_hdsi
1203 cmpwi r12, BOOK3S_INTERRUPT_H_INST_STORAGE
1204 beq kvmppc_hisi
1206 /* See if this is a leftover HDEC interrupt */
1207 cmpwi r12,BOOK3S_INTERRUPT_HV_DECREMENTER
1208 bne 2f
1209 mfspr r3,SPRN_HDEC
1210 cmpwi r3,0
1211 mr r4,r9
1212 bge fast_guest_return
1213 2:
1214 /* See if this is an hcall we can handle in real mode */
1215 cmpwi r12,BOOK3S_INTERRUPT_SYSCALL
1216 beq hcall_try_real_mode
1218 /* Hypervisor doorbell - exit only if host IPI flag set */
1219 cmpwi r12, BOOK3S_INTERRUPT_H_DOORBELL
1220 bne 3f
1221 lbz r0, HSTATE_HOST_IPI(r13)
1222 cmpwi r0, 0
1223 beq 4f
1224 b guest_exit_cont
1225 3:
1226 /* External interrupt ? */
1227 cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL
1228 bne+ guest_exit_cont
1230 /* External interrupt, first check for host_ipi. If this is
1231 * set, we know the host wants us out so let's do it now
1232 */
1233 bl kvmppc_read_intr
1234 cmpdi r3, 0
1235 bgt guest_exit_cont
1237 /* Check if any CPU is heading out to the host, if so head out too */
1238 4: ld r5, HSTATE_KVM_VCORE(r13)
1239 lwz r0, VCORE_ENTRY_EXIT(r5)
1240 cmpwi r0, 0x100
1241 mr r4, r9
1242 blt deliver_guest_interrupt
1244 guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */
1245 /* Save more register state */
1246 mfdar r6
1247 mfdsisr r7
1248 std r6, VCPU_DAR(r9)
1249 stw r7, VCPU_DSISR(r9)
1250 /* don't overwrite fault_dar/fault_dsisr if HDSI */
1251 cmpwi r12,BOOK3S_INTERRUPT_H_DATA_STORAGE
1252 beq mc_cont
1253 std r6, VCPU_FAULT_DAR(r9)
1254 stw r7, VCPU_FAULT_DSISR(r9)
1256 /* See if it is a machine check */
1257 cmpwi r12, BOOK3S_INTERRUPT_MACHINE_CHECK
1258 beq machine_check_realmode
1259 mc_cont:
1260 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1261 addi r3, r9, VCPU_TB_RMEXIT
1262 mr r4, r9
1263 bl kvmhv_accumulate_time
1264 #endif
1266 mr r3, r12
1267 /* Increment exit count, poke other threads to exit */
1268 bl kvmhv_commence_exit
1269 nop
1270 ld r9, HSTATE_KVM_VCPU(r13)
1271 lwz r12, VCPU_TRAP(r9)
1273 /* Stop others sending VCPU interrupts to this physical CPU */
1274 li r0, -1
1275 stw r0, VCPU_CPU(r9)
1276 stw r0, VCPU_THREAD_CPU(r9)
1278 /* Save guest CTRL register, set runlatch to 1 */
1279 mfspr r6,SPRN_CTRLF
1280 stw r6,VCPU_CTRL(r9)
1281 andi. r0,r6,1
1282 bne 4f
1283 ori r6,r6,1
1284 mtspr SPRN_CTRLT,r6
1285 4:
1286 /* Read the guest SLB and save it away */
1287 lwz r0,VCPU_SLB_NR(r9) /* number of entries in SLB */
1288 mtctr r0
1289 li r6,0
1290 addi r7,r9,VCPU_SLB
1291 li r5,0
1292 1: slbmfee r8,r6
1293 andis. r0,r8,SLB_ESID_V@h
1294 beq 2f
1295 add r8,r8,r6 /* put index in */
1296 slbmfev r3,r6
1297 std r8,VCPU_SLB_E(r7)
1298 std r3,VCPU_SLB_V(r7)
1299 addi r7,r7,VCPU_SLB_SIZE
1300 addi r5,r5,1
1301 2: addi r6,r6,1
1302 bdnz 1b
1303 stw r5,VCPU_SLB_MAX(r9)
1305 /*
1306 * Save the guest PURR/SPURR
1307 */
1308 mfspr r5,SPRN_PURR
1309 mfspr r6,SPRN_SPURR
1310 ld r7,VCPU_PURR(r9)
1311 ld r8,VCPU_SPURR(r9)
1312 std r5,VCPU_PURR(r9)
1313 std r6,VCPU_SPURR(r9)
1314 subf r5,r7,r5
1315 subf r6,r8,r6
1317 /*
1318 * Restore host PURR/SPURR and add guest times
1319 * so that the time in the guest gets accounted.
1320 */
1321 ld r3,HSTATE_PURR(r13)
1322 ld r4,HSTATE_SPURR(r13)
1323 add r3,r3,r5
1324 add r4,r4,r6
1325 mtspr SPRN_PURR,r3
1326 mtspr SPRN_SPURR,r4
1328 /* Save DEC */
1329 mfspr r5,SPRN_DEC
1330 mftb r6
1331 extsw r5,r5
1332 add r5,r5,r6
1333 /* r5 is a guest timebase value here, convert to host TB */
1334 ld r3,HSTATE_KVM_VCORE(r13)
1335 ld r4,VCORE_TB_OFFSET(r3)
1336 subf r5,r4,r5
1337 std r5,VCPU_DEC_EXPIRES(r9)
1339 BEGIN_FTR_SECTION
1340 b 8f
1341 END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
1342 /* Save POWER8-specific registers */
1343 mfspr r5, SPRN_IAMR
1344 mfspr r6, SPRN_PSPB
1345 mfspr r7, SPRN_FSCR
1346 std r5, VCPU_IAMR(r9)
1347 stw r6, VCPU_PSPB(r9)
1348 std r7, VCPU_FSCR(r9)
1349 mfspr r5, SPRN_IC
1350 mfspr r6, SPRN_VTB
1351 mfspr r7, SPRN_TAR
1352 std r5, VCPU_IC(r9)
1353 std r6, VCPU_VTB(r9)
1354 std r7, VCPU_TAR(r9)
1355 mfspr r8, SPRN_EBBHR
1356 std r8, VCPU_EBBHR(r9)
1357 mfspr r5, SPRN_EBBRR
1358 mfspr r6, SPRN_BESCR
1359 mfspr r7, SPRN_CSIGR
1360 mfspr r8, SPRN_TACR
1361 std r5, VCPU_EBBRR(r9)
1362 std r6, VCPU_BESCR(r9)
1363 std r7, VCPU_CSIGR(r9)
1364 std r8, VCPU_TACR(r9)
1365 mfspr r5, SPRN_TCSCR
1366 mfspr r6, SPRN_ACOP
1367 mfspr r7, SPRN_PID
1368 mfspr r8, SPRN_WORT
1369 std r5, VCPU_TCSCR(r9)
1370 std r6, VCPU_ACOP(r9)
1371 stw r7, VCPU_GUEST_PID(r9)
1372 std r8, VCPU_WORT(r9)
1373 8:
1375 /* Save and reset AMR and UAMOR before turning on the MMU */
1376 mfspr r5,SPRN_AMR
1377 mfspr r6,SPRN_UAMOR
1378 std r5,VCPU_AMR(r9)
1379 std r6,VCPU_UAMOR(r9)
1380 li r6,0
1381 mtspr SPRN_AMR,r6
1383 /* Switch DSCR back to host value */
1384 mfspr r8, SPRN_DSCR
1385 ld r7, HSTATE_DSCR(r13)
1386 std r8, VCPU_DSCR(r9)
1387 mtspr SPRN_DSCR, r7
1389 /* Save non-volatile GPRs */
1390 std r14, VCPU_GPR(R14)(r9)
1391 std r15, VCPU_GPR(R15)(r9)
1392 std r16, VCPU_GPR(R16)(r9)
1393 std r17, VCPU_GPR(R17)(r9)
1394 std r18, VCPU_GPR(R18)(r9)
1395 std r19, VCPU_GPR(R19)(r9)
1396 std r20, VCPU_GPR(R20)(r9)
1397 std r21, VCPU_GPR(R21)(r9)
1398 std r22, VCPU_GPR(R22)(r9)
1399 std r23, VCPU_GPR(R23)(r9)
1400 std r24, VCPU_GPR(R24)(r9)
1401 std r25, VCPU_GPR(R25)(r9)
1402 std r26, VCPU_GPR(R26)(r9)
1403 std r27, VCPU_GPR(R27)(r9)
1404 std r28, VCPU_GPR(R28)(r9)
1405 std r29, VCPU_GPR(R29)(r9)
1406 std r30, VCPU_GPR(R30)(r9)
1407 std r31, VCPU_GPR(R31)(r9)
1409 /* Save SPRGs */
1410 mfspr r3, SPRN_SPRG0
1411 mfspr r4, SPRN_SPRG1
1412 mfspr r5, SPRN_SPRG2
1413 mfspr r6, SPRN_SPRG3
1414 std r3, VCPU_SPRG0(r9)
1415 std r4, VCPU_SPRG1(r9)
1416 std r5, VCPU_SPRG2(r9)
1417 std r6, VCPU_SPRG3(r9)
1419 /* save FP state */
1420 mr r3, r9
1421 bl kvmppc_save_fp
1423 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1424 BEGIN_FTR_SECTION
1425 b 2f
1426 END_FTR_SECTION_IFCLR(CPU_FTR_TM)
1427 /* Turn on TM. */
1428 mfmsr r8
1429 li r0, 1
1430 rldimi r8, r0, MSR_TM_LG, 63-MSR_TM_LG
1431 mtmsrd r8
1433 ld r5, VCPU_MSR(r9)
1434 rldicl. r5, r5, 64 - MSR_TS_S_LG, 62
1435 beq 1f /* TM not active in guest. */
1437 li r3, TM_CAUSE_KVM_RESCHED
1439 /* Clear the MSR RI since r1, r13 are all going to be foobar. */
1440 li r5, 0
1441 mtmsrd r5, 1
1443 /* All GPRs are volatile at this point. */
1444 TRECLAIM(R3)
1446 /* Temporarily store r13 and r9 so we have some regs to play with */
1447 SET_SCRATCH0(r13)
1448 GET_PACA(r13)
1449 std r9, PACATMSCRATCH(r13)
1450 ld r9, HSTATE_KVM_VCPU(r13)
1452 /* Get a few more GPRs free. */
1453 std r29, VCPU_GPRS_TM(29)(r9)
1454 std r30, VCPU_GPRS_TM(30)(r9)
1455 std r31, VCPU_GPRS_TM(31)(r9)
1457 /* Save away PPR and DSCR soon so don't run with user values. */
1458 mfspr r31, SPRN_PPR
1459 HMT_MEDIUM
1460 mfspr r30, SPRN_DSCR
1461 ld r29, HSTATE_DSCR(r13)
1462 mtspr SPRN_DSCR, r29
1464 /* Save all but r9, r13 & r29-r31 */
1465 reg = 0
1466 .rept 29
1467 .if (reg != 9) && (reg != 13)
1468 std reg, VCPU_GPRS_TM(reg)(r9)
1469 .endif
1470 reg = reg + 1
1471 .endr
1472 /* ... now save r13 */
1473 GET_SCRATCH0(r4)
1474 std r4, VCPU_GPRS_TM(13)(r9)
1475 /* ... and save r9 */
1476 ld r4, PACATMSCRATCH(r13)
1477 std r4, VCPU_GPRS_TM(9)(r9)
1479 /* Reload stack pointer and TOC. */
1480 ld r1, HSTATE_HOST_R1(r13)
1481 ld r2, PACATOC(r13)
1483 /* Set MSR RI now we have r1 and r13 back. */
1484 li r5, MSR_RI
1485 mtmsrd r5, 1
1487 /* Save away checkpinted SPRs. */
1488 std r31, VCPU_PPR_TM(r9)
1489 std r30, VCPU_DSCR_TM(r9)
1490 mflr r5
1491 mfcr r6
1492 mfctr r7
1493 mfspr r8, SPRN_AMR
1494 mfspr r10, SPRN_TAR
1495 std r5, VCPU_LR_TM(r9)
1496 stw r6, VCPU_CR_TM(r9)
1497 std r7, VCPU_CTR_TM(r9)
1498 std r8, VCPU_AMR_TM(r9)
1499 std r10, VCPU_TAR_TM(r9)
1501 /* Restore r12 as trap number. */
1502 lwz r12, VCPU_TRAP(r9)
1504 /* Save FP/VSX. */
1505 addi r3, r9, VCPU_FPRS_TM
1506 bl store_fp_state
1507 addi r3, r9, VCPU_VRS_TM
1508 bl store_vr_state
1509 mfspr r6, SPRN_VRSAVE
1510 stw r6, VCPU_VRSAVE_TM(r9)
1511 1:
1512 /*
1513 * We need to save these SPRs after the treclaim so that the software
1514 * error code is recorded correctly in the TEXASR. Also the user may
1515 * change these outside of a transaction, so they must always be
1516 * context switched.
1517 */
1518 mfspr r5, SPRN_TFHAR
1519 mfspr r6, SPRN_TFIAR
1520 mfspr r7, SPRN_TEXASR
1521 std r5, VCPU_TFHAR(r9)
1522 std r6, VCPU_TFIAR(r9)
1523 std r7, VCPU_TEXASR(r9)
1524 2:
1525 #endif
1527 /* Increment yield count if they have a VPA */
1528 ld r8, VCPU_VPA(r9) /* do they have a VPA? */
1529 cmpdi r8, 0
1530 beq 25f
1531 li r4, LPPACA_YIELDCOUNT
1532 LWZX_BE r3, r8, r4
1533 addi r3, r3, 1
1534 STWX_BE r3, r8, r4
1535 li r3, 1
1536 stb r3, VCPU_VPA_DIRTY(r9)
1537 25:
1538 /* Save PMU registers if requested */
1539 /* r8 and cr0.eq are live here */
1540 BEGIN_FTR_SECTION
1541 /*
1542 * POWER8 seems to have a hardware bug where setting
1543 * MMCR0[PMAE] along with MMCR0[PMC1CE] and/or MMCR0[PMCjCE]
1544 * when some counters are already negative doesn't seem
1545 * to cause a performance monitor alert (and hence interrupt).
1546 * The effect of this is that when saving the PMU state,
1547 * if there is no PMU alert pending when we read MMCR0
1548 * before freezing the counters, but one becomes pending
1549 * before we read the counters, we lose it.
1550 * To work around this, we need a way to freeze the counters
1551 * before reading MMCR0. Normally, freezing the counters
1552 * is done by writing MMCR0 (to set MMCR0[FC]) which
1553 * unavoidably writes MMCR0[PMA0] as well. On POWER8,
1554 * we can also freeze the counters using MMCR2, by writing
1555 * 1s to all the counter freeze condition bits (there are
1556 * 9 bits each for 6 counters).
1557 */
1558 li r3, -1 /* set all freeze bits */
1559 clrrdi r3, r3, 10
1560 mfspr r10, SPRN_MMCR2
1561 mtspr SPRN_MMCR2, r3
1562 isync
1563 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1564 li r3, 1
1565 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */
1566 mfspr r4, SPRN_MMCR0 /* save MMCR0 */
1567 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */
1568 mfspr r6, SPRN_MMCRA
1569 /* Clear MMCRA in order to disable SDAR updates */
1570 li r7, 0
1571 mtspr SPRN_MMCRA, r7
1572 isync
1573 beq 21f /* if no VPA, save PMU stuff anyway */
1574 lbz r7, LPPACA_PMCINUSE(r8)
1575 cmpwi r7, 0 /* did they ask for PMU stuff to be saved? */
1576 bne 21f
1577 std r3, VCPU_MMCR(r9) /* if not, set saved MMCR0 to FC */
1578 b 22f
1579 21: mfspr r5, SPRN_MMCR1
1580 mfspr r7, SPRN_SIAR
1581 mfspr r8, SPRN_SDAR
1582 std r4, VCPU_MMCR(r9)
1583 std r5, VCPU_MMCR + 8(r9)
1584 std r6, VCPU_MMCR + 16(r9)
1585 BEGIN_FTR_SECTION
1586 std r10, VCPU_MMCR + 24(r9)
1587 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1588 std r7, VCPU_SIAR(r9)
1589 std r8, VCPU_SDAR(r9)
1590 mfspr r3, SPRN_PMC1
1591 mfspr r4, SPRN_PMC2
1592 mfspr r5, SPRN_PMC3
1593 mfspr r6, SPRN_PMC4
1594 mfspr r7, SPRN_PMC5
1595 mfspr r8, SPRN_PMC6
1596 stw r3, VCPU_PMC(r9)
1597 stw r4, VCPU_PMC + 4(r9)
1598 stw r5, VCPU_PMC + 8(r9)
1599 stw r6, VCPU_PMC + 12(r9)
1600 stw r7, VCPU_PMC + 16(r9)
1601 stw r8, VCPU_PMC + 20(r9)
1602 BEGIN_FTR_SECTION
1603 mfspr r5, SPRN_SIER
1604 mfspr r6, SPRN_SPMC1
1605 mfspr r7, SPRN_SPMC2
1606 mfspr r8, SPRN_MMCRS
1607 std r5, VCPU_SIER(r9)
1608 stw r6, VCPU_PMC + 24(r9)
1609 stw r7, VCPU_PMC + 28(r9)
1610 std r8, VCPU_MMCR + 32(r9)
1611 lis r4, 0x8000
1612 mtspr SPRN_MMCRS, r4
1613 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1614 22:
1615 /* Clear out SLB */
1616 li r5,0
1617 slbmte r5,r5
1618 slbia
1619 ptesync
1621 /*
1622 * POWER7/POWER8 guest -> host partition switch code.
1623 * We don't have to lock against tlbies but we do
1624 * have to coordinate the hardware threads.
1625 */
1626 kvmhv_switch_to_host:
1627 /* Secondary threads wait for primary to do partition switch */
1628 ld r5,HSTATE_KVM_VCORE(r13)
1629 ld r4,VCORE_KVM(r5) /* pointer to struct kvm */
1630 lbz r3,HSTATE_PTID(r13)
1631 cmpwi r3,0
1632 beq 15f
1633 HMT_LOW
1634 13: lbz r3,VCORE_IN_GUEST(r5)
1635 cmpwi r3,0
1636 bne 13b
1637 HMT_MEDIUM
1638 b 16f
1640 /* Primary thread waits for all the secondaries to exit guest */
1641 15: lwz r3,VCORE_ENTRY_EXIT(r5)
1642 rlwinm r0,r3,32-8,0xff
1643 clrldi r3,r3,56
1644 cmpw r3,r0
1645 bne 15b
1646 isync
1648 /* Did we actually switch to the guest at all? */
1649 lbz r6, VCORE_IN_GUEST(r5)
1650 cmpwi r6, 0
1651 beq 19f
1653 /* Primary thread switches back to host partition */
1654 ld r6,KVM_HOST_SDR1(r4)
1655 lwz r7,KVM_HOST_LPID(r4)
1656 li r8,LPID_RSVD /* switch to reserved LPID */
1657 mtspr SPRN_LPID,r8
1658 ptesync
1659 mtspr SPRN_SDR1,r6 /* switch to partition page table */
1660 mtspr SPRN_LPID,r7
1661 isync
1663 BEGIN_FTR_SECTION
1664 /* DPDES is shared between threads */
1665 mfspr r7, SPRN_DPDES
1666 std r7, VCORE_DPDES(r5)
1667 /* clear DPDES so we don't get guest doorbells in the host */
1668 li r8, 0
1669 mtspr SPRN_DPDES, r8
1670 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1672 /* Subtract timebase offset from timebase */
1673 ld r8,VCORE_TB_OFFSET(r5)
1674 cmpdi r8,0
1675 beq 17f
1676 mftb r6 /* current guest timebase */
1677 subf r8,r8,r6
1678 mtspr SPRN_TBU40,r8 /* update upper 40 bits */
1679 mftb r7 /* check if lower 24 bits overflowed */
1680 clrldi r6,r6,40
1681 clrldi r7,r7,40
1682 cmpld r7,r6
1683 bge 17f
1684 addis r8,r8,0x100 /* if so, increment upper 40 bits */
1685 mtspr SPRN_TBU40,r8
1687 /* Reset PCR */
1688 17: ld r0, VCORE_PCR(r5)
1689 cmpdi r0, 0
1690 beq 18f
1691 li r0, 0
1692 mtspr SPRN_PCR, r0
1693 18:
1694 /* Signal secondary CPUs to continue */
1695 stb r0,VCORE_IN_GUEST(r5)
1696 19: lis r8,0x7fff /* MAX_INT@h */
1697 mtspr SPRN_HDEC,r8
1699 16: ld r8,KVM_HOST_LPCR(r4)
1700 mtspr SPRN_LPCR,r8
1701 isync
1703 /* load host SLB entries */
1704 ld r8,PACA_SLBSHADOWPTR(r13)
1706 .rept SLB_NUM_BOLTED
1707 li r3, SLBSHADOW_SAVEAREA
1708 LDX_BE r5, r8, r3
1709 addi r3, r3, 8
1710 LDX_BE r6, r8, r3
1711 andis. r7,r5,SLB_ESID_V@h
1712 beq 1f
1713 slbmte r6,r5
1714 1: addi r8,r8,16
1715 .endr
1717 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1718 /* Finish timing, if we have a vcpu */
1719 ld r4, HSTATE_KVM_VCPU(r13)
1720 cmpdi r4, 0
1721 li r3, 0
1722 beq 2f
1723 bl kvmhv_accumulate_time
1724 2:
1725 #endif
1726 /* Unset guest mode */
1727 li r0, KVM_GUEST_MODE_NONE
1728 stb r0, HSTATE_IN_GUEST(r13)
1730 ld r0, 112+PPC_LR_STKOFF(r1)
1731 addi r1, r1, 112
1732 mtlr r0
1733 blr
1735 /*
1736 * Check whether an HDSI is an HPTE not found fault or something else.
1737 * If it is an HPTE not found fault that is due to the guest accessing
1738 * a page that they have mapped but which we have paged out, then
1739 * we continue on with the guest exit path. In all other cases,
1740 * reflect the HDSI to the guest as a DSI.
1741 */
1742 kvmppc_hdsi:
1743 mfspr r4, SPRN_HDAR
1744 mfspr r6, SPRN_HDSISR
1745 /* HPTE not found fault or protection fault? */
1746 andis. r0, r6, (DSISR_NOHPTE | DSISR_PROTFAULT)@h
1747 beq 1f /* if not, send it to the guest */
1748 andi. r0, r11, MSR_DR /* data relocation enabled? */
1749 beq 3f
1750 clrrdi r0, r4, 28
1751 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */
1752 li r0, BOOK3S_INTERRUPT_DATA_SEGMENT
1753 bne 7f /* if no SLB entry found */
1754 4: std r4, VCPU_FAULT_DAR(r9)
1755 stw r6, VCPU_FAULT_DSISR(r9)
1757 /* Search the hash table. */
1758 mr r3, r9 /* vcpu pointer */
1759 li r7, 1 /* data fault */
1760 bl kvmppc_hpte_hv_fault
1761 ld r9, HSTATE_KVM_VCPU(r13)
1762 ld r10, VCPU_PC(r9)
1763 ld r11, VCPU_MSR(r9)
1764 li r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
1765 cmpdi r3, 0 /* retry the instruction */
1766 beq 6f
1767 cmpdi r3, -1 /* handle in kernel mode */
1768 beq guest_exit_cont
1769 cmpdi r3, -2 /* MMIO emulation; need instr word */
1770 beq 2f
1772 /* Synthesize a DSI (or DSegI) for the guest */
1773 ld r4, VCPU_FAULT_DAR(r9)
1774 mr r6, r3
1775 1: li r0, BOOK3S_INTERRUPT_DATA_STORAGE
1776 mtspr SPRN_DSISR, r6
1777 7: mtspr SPRN_DAR, r4
1778 mtspr SPRN_SRR0, r10
1779 mtspr SPRN_SRR1, r11
1780 mr r10, r0
1781 bl kvmppc_msr_interrupt
1782 fast_interrupt_c_return:
1783 6: ld r7, VCPU_CTR(r9)
1784 ld r8, VCPU_XER(r9)
1785 mtctr r7
1786 mtxer r8
1787 mr r4, r9
1788 b fast_guest_return
1790 3: ld r5, VCPU_KVM(r9) /* not relocated, use VRMA */
1791 ld r5, KVM_VRMA_SLB_V(r5)
1792 b 4b
1794 /* If this is for emulated MMIO, load the instruction word */
1795 2: li r8, KVM_INST_FETCH_FAILED /* In case lwz faults */
1797 /* Set guest mode to 'jump over instruction' so if lwz faults
1798 * we'll just continue at the next IP. */
1799 li r0, KVM_GUEST_MODE_SKIP
1800 stb r0, HSTATE_IN_GUEST(r13)
1802 /* Do the access with MSR:DR enabled */
1803 mfmsr r3
1804 ori r4, r3, MSR_DR /* Enable paging for data */
1805 mtmsrd r4
1806 lwz r8, 0(r10)
1807 mtmsrd r3
1809 /* Store the result */
1810 stw r8, VCPU_LAST_INST(r9)
1812 /* Unset guest mode. */
1813 li r0, KVM_GUEST_MODE_HOST_HV
1814 stb r0, HSTATE_IN_GUEST(r13)
1815 b guest_exit_cont
1817 /*
1818 * Similarly for an HISI, reflect it to the guest as an ISI unless
1819 * it is an HPTE not found fault for a page that we have paged out.
1820 */
1821 kvmppc_hisi:
1822 andis. r0, r11, SRR1_ISI_NOPT@h
1823 beq 1f
1824 andi. r0, r11, MSR_IR /* instruction relocation enabled? */
1825 beq 3f
1826 clrrdi r0, r10, 28
1827 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */
1828 li r0, BOOK3S_INTERRUPT_INST_SEGMENT
1829 bne 7f /* if no SLB entry found */
1830 4:
1831 /* Search the hash table. */
1832 mr r3, r9 /* vcpu pointer */
1833 mr r4, r10
1834 mr r6, r11
1835 li r7, 0 /* instruction fault */
1836 bl kvmppc_hpte_hv_fault
1837 ld r9, HSTATE_KVM_VCPU(r13)
1838 ld r10, VCPU_PC(r9)
1839 ld r11, VCPU_MSR(r9)
1840 li r12, BOOK3S_INTERRUPT_H_INST_STORAGE
1841 cmpdi r3, 0 /* retry the instruction */
1842 beq fast_interrupt_c_return
1843 cmpdi r3, -1 /* handle in kernel mode */
1844 beq guest_exit_cont
1846 /* Synthesize an ISI (or ISegI) for the guest */
1847 mr r11, r3
1848 1: li r0, BOOK3S_INTERRUPT_INST_STORAGE
1849 7: mtspr SPRN_SRR0, r10
1850 mtspr SPRN_SRR1, r11
1851 mr r10, r0
1852 bl kvmppc_msr_interrupt
1853 b fast_interrupt_c_return
1855 3: ld r6, VCPU_KVM(r9) /* not relocated, use VRMA */
1856 ld r5, KVM_VRMA_SLB_V(r6)
1857 b 4b
1859 /*
1860 * Try to handle an hcall in real mode.
1861 * Returns to the guest if we handle it, or continues on up to
1862 * the kernel if we can't (i.e. if we don't have a handler for
1863 * it, or if the handler returns H_TOO_HARD).
1864 *
1865 * r5 - r8 contain hcall args,
1866 * r9 = vcpu, r10 = pc, r11 = msr, r12 = trap, r13 = paca
1867 */
1868 hcall_try_real_mode:
1869 ld r3,VCPU_GPR(R3)(r9)
1870 andi. r0,r11,MSR_PR
1871 /* sc 1 from userspace - reflect to guest syscall */
1872 bne sc_1_fast_return
1873 clrrdi r3,r3,2
1874 cmpldi r3,hcall_real_table_end - hcall_real_table
1875 bge guest_exit_cont
1876 /* See if this hcall is enabled for in-kernel handling */
1877 ld r4, VCPU_KVM(r9)
1878 srdi r0, r3, 8 /* r0 = (r3 / 4) >> 6 */
1879 sldi r0, r0, 3 /* index into kvm->arch.enabled_hcalls[] */
1880 add r4, r4, r0
1881 ld r0, KVM_ENABLED_HCALLS(r4)
1882 rlwinm r4, r3, 32-2, 0x3f /* r4 = (r3 / 4) & 0x3f */
1883 srd r0, r0, r4
1884 andi. r0, r0, 1
1885 beq guest_exit_cont
1886 /* Get pointer to handler, if any, and call it */
1887 LOAD_REG_ADDR(r4, hcall_real_table)
1888 lwax r3,r3,r4
1889 cmpwi r3,0
1890 beq guest_exit_cont
1891 add r12,r3,r4
1892 mtctr r12
1893 mr r3,r9 /* get vcpu pointer */
1894 ld r4,VCPU_GPR(R4)(r9)
1895 bctrl
1896 cmpdi r3,H_TOO_HARD
1897 beq hcall_real_fallback
1898 ld r4,HSTATE_KVM_VCPU(r13)
1899 std r3,VCPU_GPR(R3)(r4)
1900 ld r10,VCPU_PC(r4)
1901 ld r11,VCPU_MSR(r4)
1902 b fast_guest_return
1904 sc_1_fast_return:
1905 mtspr SPRN_SRR0,r10
1906 mtspr SPRN_SRR1,r11
1907 li r10, BOOK3S_INTERRUPT_SYSCALL
1908 bl kvmppc_msr_interrupt
1909 mr r4,r9
1910 b fast_guest_return
1912 /* We've attempted a real mode hcall, but it's punted it back
1913 * to userspace. We need to restore some clobbered volatiles
1914 * before resuming the pass-it-to-qemu path */
1915 hcall_real_fallback:
1916 li r12,BOOK3S_INTERRUPT_SYSCALL
1917 ld r9, HSTATE_KVM_VCPU(r13)
1919 b guest_exit_cont
1921 .globl hcall_real_table
1922 hcall_real_table:
1923 .long 0 /* 0 - unused */
1924 .long DOTSYM(kvmppc_h_remove) - hcall_real_table
1925 .long DOTSYM(kvmppc_h_enter) - hcall_real_table
1926 .long DOTSYM(kvmppc_h_read) - hcall_real_table
1927 .long DOTSYM(kvmppc_h_clear_mod) - hcall_real_table
1928 .long DOTSYM(kvmppc_h_clear_ref) - hcall_real_table
1929 .long DOTSYM(kvmppc_h_protect) - hcall_real_table
1930 .long DOTSYM(kvmppc_h_get_tce) - hcall_real_table
1931 .long DOTSYM(kvmppc_h_put_tce) - hcall_real_table
1932 .long 0 /* 0x24 - H_SET_SPRG0 */
1933 .long DOTSYM(kvmppc_h_set_dabr) - hcall_real_table
1934 .long 0 /* 0x2c */
1935 .long 0 /* 0x30 */
1936 .long 0 /* 0x34 */
1937 .long 0 /* 0x38 */
1938 .long 0 /* 0x3c */
1939 .long 0 /* 0x40 */
1940 .long 0 /* 0x44 */
1941 .long 0 /* 0x48 */
1942 .long 0 /* 0x4c */
1943 .long 0 /* 0x50 */
1944 .long 0 /* 0x54 */
1945 .long 0 /* 0x58 */
1946 .long 0 /* 0x5c */
1947 .long 0 /* 0x60 */
1948 #ifdef CONFIG_KVM_XICS
1949 .long DOTSYM(kvmppc_rm_h_eoi) - hcall_real_table
1950 .long DOTSYM(kvmppc_rm_h_cppr) - hcall_real_table
1951 .long DOTSYM(kvmppc_rm_h_ipi) - hcall_real_table
1952 .long 0 /* 0x70 - H_IPOLL */
1953 .long DOTSYM(kvmppc_rm_h_xirr) - hcall_real_table
1954 #else
1955 .long 0 /* 0x64 - H_EOI */
1956 .long 0 /* 0x68 - H_CPPR */
1957 .long 0 /* 0x6c - H_IPI */
1958 .long 0 /* 0x70 - H_IPOLL */
1959 .long 0 /* 0x74 - H_XIRR */
1960 #endif
1961 .long 0 /* 0x78 */
1962 .long 0 /* 0x7c */
1963 .long 0 /* 0x80 */
1964 .long 0 /* 0x84 */
1965 .long 0 /* 0x88 */
1966 .long 0 /* 0x8c */
1967 .long 0 /* 0x90 */
1968 .long 0 /* 0x94 */
1969 .long 0 /* 0x98 */
1970 .long 0 /* 0x9c */
1971 .long 0 /* 0xa0 */
1972 .long 0 /* 0xa4 */
1973 .long 0 /* 0xa8 */
1974 .long 0 /* 0xac */
1975 .long 0 /* 0xb0 */
1976 .long 0 /* 0xb4 */
1977 .long 0 /* 0xb8 */
1978 .long 0 /* 0xbc */
1979 .long 0 /* 0xc0 */
1980 .long 0 /* 0xc4 */
1981 .long 0 /* 0xc8 */
1982 .long 0 /* 0xcc */
1983 .long 0 /* 0xd0 */
1984 .long 0 /* 0xd4 */
1985 .long 0 /* 0xd8 */
1986 .long 0 /* 0xdc */
1987 .long DOTSYM(kvmppc_h_cede) - hcall_real_table
1988 .long DOTSYM(kvmppc_rm_h_confer) - hcall_real_table
1989 .long 0 /* 0xe8 */
1990 .long 0 /* 0xec */
1991 .long 0 /* 0xf0 */
1992 .long 0 /* 0xf4 */
1993 .long 0 /* 0xf8 */
1994 .long 0 /* 0xfc */
1995 .long 0 /* 0x100 */
1996 .long 0 /* 0x104 */
1997 .long 0 /* 0x108 */
1998 .long 0 /* 0x10c */
1999 .long 0 /* 0x110 */
2000 .long 0 /* 0x114 */
2001 .long 0 /* 0x118 */
2002 .long 0 /* 0x11c */
2003 .long 0 /* 0x120 */
2004 .long DOTSYM(kvmppc_h_bulk_remove) - hcall_real_table
2005 .long 0 /* 0x128 */
2006 .long 0 /* 0x12c */
2007 .long 0 /* 0x130 */
2008 .long DOTSYM(kvmppc_h_set_xdabr) - hcall_real_table
2009 .long 0 /* 0x138 */
2010 .long 0 /* 0x13c */
2011 .long 0 /* 0x140 */
2012 .long 0 /* 0x144 */
2013 .long 0 /* 0x148 */
2014 .long 0 /* 0x14c */
2015 .long 0 /* 0x150 */
2016 .long 0 /* 0x154 */
2017 .long 0 /* 0x158 */
2018 .long 0 /* 0x15c */
2019 .long 0 /* 0x160 */
2020 .long 0 /* 0x164 */
2021 .long 0 /* 0x168 */
2022 .long 0 /* 0x16c */
2023 .long 0 /* 0x170 */
2024 .long 0 /* 0x174 */
2025 .long 0 /* 0x178 */
2026 .long 0 /* 0x17c */
2027 .long 0 /* 0x180 */
2028 .long 0 /* 0x184 */
2029 .long 0 /* 0x188 */
2030 .long 0 /* 0x18c */
2031 .long 0 /* 0x190 */
2032 .long 0 /* 0x194 */
2033 .long 0 /* 0x198 */
2034 .long 0 /* 0x19c */
2035 .long 0 /* 0x1a0 */
2036 .long 0 /* 0x1a4 */
2037 .long 0 /* 0x1a8 */
2038 .long 0 /* 0x1ac */
2039 .long 0 /* 0x1b0 */
2040 .long 0 /* 0x1b4 */
2041 .long 0 /* 0x1b8 */
2042 .long 0 /* 0x1bc */
2043 .long 0 /* 0x1c0 */
2044 .long 0 /* 0x1c4 */
2045 .long 0 /* 0x1c8 */
2046 .long 0 /* 0x1cc */
2047 .long 0 /* 0x1d0 */
2048 .long 0 /* 0x1d4 */
2049 .long 0 /* 0x1d8 */
2050 .long 0 /* 0x1dc */
2051 .long 0 /* 0x1e0 */
2052 .long 0 /* 0x1e4 */
2053 .long 0 /* 0x1e8 */
2054 .long 0 /* 0x1ec */
2055 .long 0 /* 0x1f0 */
2056 .long 0 /* 0x1f4 */
2057 .long 0 /* 0x1f8 */
2058 .long 0 /* 0x1fc */
2059 .long 0 /* 0x200 */
2060 .long 0 /* 0x204 */
2061 .long 0 /* 0x208 */
2062 .long 0 /* 0x20c */
2063 .long 0 /* 0x210 */
2064 .long 0 /* 0x214 */
2065 .long 0 /* 0x218 */
2066 .long 0 /* 0x21c */
2067 .long 0 /* 0x220 */
2068 .long 0 /* 0x224 */
2069 .long 0 /* 0x228 */
2070 .long 0 /* 0x22c */
2071 .long 0 /* 0x230 */
2072 .long 0 /* 0x234 */
2073 .long 0 /* 0x238 */
2074 .long 0 /* 0x23c */
2075 .long 0 /* 0x240 */
2076 .long 0 /* 0x244 */
2077 .long 0 /* 0x248 */
2078 .long 0 /* 0x24c */
2079 .long 0 /* 0x250 */
2080 .long 0 /* 0x254 */
2081 .long 0 /* 0x258 */
2082 .long 0 /* 0x25c */
2083 .long 0 /* 0x260 */
2084 .long 0 /* 0x264 */
2085 .long 0 /* 0x268 */
2086 .long 0 /* 0x26c */
2087 .long 0 /* 0x270 */
2088 .long 0 /* 0x274 */
2089 .long 0 /* 0x278 */
2090 .long 0 /* 0x27c */
2091 .long 0 /* 0x280 */
2092 .long 0 /* 0x284 */
2093 .long 0 /* 0x288 */
2094 .long 0 /* 0x28c */
2095 .long 0 /* 0x290 */
2096 .long 0 /* 0x294 */
2097 .long 0 /* 0x298 */
2098 .long 0 /* 0x29c */
2099 .long 0 /* 0x2a0 */
2100 .long 0 /* 0x2a4 */
2101 .long 0 /* 0x2a8 */
2102 .long 0 /* 0x2ac */
2103 .long 0 /* 0x2b0 */
2104 .long 0 /* 0x2b4 */
2105 .long 0 /* 0x2b8 */
2106 .long 0 /* 0x2bc */
2107 .long 0 /* 0x2c0 */
2108 .long 0 /* 0x2c4 */
2109 .long 0 /* 0x2c8 */
2110 .long 0 /* 0x2cc */
2111 .long 0 /* 0x2d0 */
2112 .long 0 /* 0x2d4 */
2113 .long 0 /* 0x2d8 */
2114 .long 0 /* 0x2dc */
2115 .long 0 /* 0x2e0 */
2116 .long 0 /* 0x2e4 */
2117 .long 0 /* 0x2e8 */
2118 .long 0 /* 0x2ec */
2119 .long 0 /* 0x2f0 */
2120 .long 0 /* 0x2f4 */
2121 .long 0 /* 0x2f8 */
2122 .long 0 /* 0x2fc */
2123 .long DOTSYM(kvmppc_h_random) - hcall_real_table
2124 .globl hcall_real_table_end
2125 hcall_real_table_end:
2127 _GLOBAL(kvmppc_h_set_xdabr)
2128 andi. r0, r5, DABRX_USER | DABRX_KERNEL
2129 beq 6f
2130 li r0, DABRX_USER | DABRX_KERNEL | DABRX_BTI
2131 andc. r0, r5, r0
2132 beq 3f
2133 6: li r3, H_PARAMETER
2134 blr
2136 _GLOBAL(kvmppc_h_set_dabr)
2137 li r5, DABRX_USER | DABRX_KERNEL
2138 3:
2139 BEGIN_FTR_SECTION
2140 b 2f
2141 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2142 std r4,VCPU_DABR(r3)
2143 stw r5, VCPU_DABRX(r3)
2144 mtspr SPRN_DABRX, r5
2145 /* Work around P7 bug where DABR can get corrupted on mtspr */
2146 1: mtspr SPRN_DABR,r4
2147 mfspr r5, SPRN_DABR
2148 cmpd r4, r5
2149 bne 1b
2150 isync
2151 li r3,0
2152 blr
2154 /* Emulate H_SET_DABR/X on P8 for the sake of compat mode guests */
2155 2: rlwimi r5, r4, 5, DAWRX_DR | DAWRX_DW
2156 rlwimi r5, r4, 1, DAWRX_WT
2157 clrrdi r4, r4, 3
2158 std r4, VCPU_DAWR(r3)
2159 std r5, VCPU_DAWRX(r3)
2160 mtspr SPRN_DAWR, r4
2161 mtspr SPRN_DAWRX, r5
2162 li r3, 0
2163 blr
2165 _GLOBAL(kvmppc_h_cede) /* r3 = vcpu pointer, r11 = msr, r13 = paca */
2166 ori r11,r11,MSR_EE
2167 std r11,VCPU_MSR(r3)
2168 li r0,1
2169 stb r0,VCPU_CEDED(r3)
2170 sync /* order setting ceded vs. testing prodded */
2171 lbz r5,VCPU_PRODDED(r3)
2172 cmpwi r5,0
2173 bne kvm_cede_prodded
2174 li r12,0 /* set trap to 0 to say hcall is handled */
2175 stw r12,VCPU_TRAP(r3)
2176 li r0,H_SUCCESS
2177 std r0,VCPU_GPR(R3)(r3)
2179 /*
2180 * Set our bit in the bitmask of napping threads unless all the
2181 * other threads are already napping, in which case we send this
2182 * up to the host.
2183 */
2184 ld r5,HSTATE_KVM_VCORE(r13)
2185 lbz r6,HSTATE_PTID(r13)
2186 lwz r8,VCORE_ENTRY_EXIT(r5)
2187 clrldi r8,r8,56
2188 li r0,1
2189 sld r0,r0,r6
2190 addi r6,r5,VCORE_NAPPING_THREADS
2191 31: lwarx r4,0,r6
2192 or r4,r4,r0
2193 cmpw r4,r8
2194 beq kvm_cede_exit
2195 stwcx. r4,0,r6
2196 bne 31b
2197 /* order napping_threads update vs testing entry_exit_map */
2198 isync
2199 li r0,NAPPING_CEDE
2200 stb r0,HSTATE_NAPPING(r13)
2201 lwz r7,VCORE_ENTRY_EXIT(r5)
2202 cmpwi r7,0x100
2203 bge 33f /* another thread already exiting */
2205 /*
2206 * Although not specifically required by the architecture, POWER7
2207 * preserves the following registers in nap mode, even if an SMT mode
2208 * switch occurs: SLB entries, PURR, SPURR, AMOR, UAMOR, AMR, SPRG0-3,
2209 * DAR, DSISR, DABR, DABRX, DSCR, PMCx, MMCRx, SIAR, SDAR.
2210 */
2211 /* Save non-volatile GPRs */
2212 std r14, VCPU_GPR(R14)(r3)
2213 std r15, VCPU_GPR(R15)(r3)
2214 std r16, VCPU_GPR(R16)(r3)
2215 std r17, VCPU_GPR(R17)(r3)
2216 std r18, VCPU_GPR(R18)(r3)
2217 std r19, VCPU_GPR(R19)(r3)
2218 std r20, VCPU_GPR(R20)(r3)
2219 std r21, VCPU_GPR(R21)(r3)
2220 std r22, VCPU_GPR(R22)(r3)
2221 std r23, VCPU_GPR(R23)(r3)
2222 std r24, VCPU_GPR(R24)(r3)
2223 std r25, VCPU_GPR(R25)(r3)
2224 std r26, VCPU_GPR(R26)(r3)
2225 std r27, VCPU_GPR(R27)(r3)
2226 std r28, VCPU_GPR(R28)(r3)
2227 std r29, VCPU_GPR(R29)(r3)
2228 std r30, VCPU_GPR(R30)(r3)
2229 std r31, VCPU_GPR(R31)(r3)
2231 /* save FP state */
2232 bl kvmppc_save_fp
2234 /*
2235 * Set DEC to the smaller of DEC and HDEC, so that we wake
2236 * no later than the end of our timeslice (HDEC interrupts
2237 * don't wake us from nap).
2238 */
2239 mfspr r3, SPRN_DEC
2240 mfspr r4, SPRN_HDEC
2241 mftb r5
2242 cmpw r3, r4
2243 ble 67f
2244 mtspr SPRN_DEC, r4
2245 67:
2246 /* save expiry time of guest decrementer */
2247 extsw r3, r3
2248 add r3, r3, r5
2249 ld r4, HSTATE_KVM_VCPU(r13)
2250 ld r5, HSTATE_KVM_VCORE(r13)
2251 ld r6, VCORE_TB_OFFSET(r5)
2252 subf r3, r6, r3 /* convert to host TB value */
2253 std r3, VCPU_DEC_EXPIRES(r4)
2255 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
2256 ld r4, HSTATE_KVM_VCPU(r13)
2257 addi r3, r4, VCPU_TB_CEDE
2258 bl kvmhv_accumulate_time
2259 #endif
2261 lis r3, LPCR_PECEDP@h /* Do wake on privileged doorbell */
2263 /*
2264 * Take a nap until a decrementer or external or doobell interrupt
2265 * occurs, with PECE1 and PECE0 set in LPCR.
2266 * On POWER8, set PECEDH, and if we are ceding, also set PECEDP.
2267 * Also clear the runlatch bit before napping.
2268 */
2269 kvm_do_nap:
2270 mfspr r0, SPRN_CTRLF
2271 clrrdi r0, r0, 1
2272 mtspr SPRN_CTRLT, r0
2274 li r0,1
2275 stb r0,HSTATE_HWTHREAD_REQ(r13)
2276 mfspr r5,SPRN_LPCR
2277 ori r5,r5,LPCR_PECE0 | LPCR_PECE1
2278 BEGIN_FTR_SECTION
2279 ori r5, r5, LPCR_PECEDH
2280 rlwimi r5, r3, 0, LPCR_PECEDP
2281 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2282 mtspr SPRN_LPCR,r5
2283 isync
2284 li r0, 0
2285 std r0, HSTATE_SCRATCH0(r13)
2286 ptesync
2287 ld r0, HSTATE_SCRATCH0(r13)
2288 1: cmpd r0, r0
2289 bne 1b
2290 nap
2291 b .
2293 33: mr r4, r3
2294 li r3, 0
2295 li r12, 0
2296 b 34f
2298 kvm_end_cede:
2299 /* get vcpu pointer */
2300 ld r4, HSTATE_KVM_VCPU(r13)
2302 /* Woken by external or decrementer interrupt */
2303 ld r1, HSTATE_HOST_R1(r13)
2305 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
2306 addi r3, r4, VCPU_TB_RMINTR
2307 bl kvmhv_accumulate_time
2308 #endif
2310 /* load up FP state */
2311 bl kvmppc_load_fp
2313 /* Restore guest decrementer */
2314 ld r3, VCPU_DEC_EXPIRES(r4)
2315 ld r5, HSTATE_KVM_VCORE(r13)
2316 ld r6, VCORE_TB_OFFSET(r5)
2317 add r3, r3, r6 /* convert host TB to guest TB value */
2318 mftb r7
2319 subf r3, r7, r3
2320 mtspr SPRN_DEC, r3
2322 /* Load NV GPRS */
2323 ld r14, VCPU_GPR(R14)(r4)
2324 ld r15, VCPU_GPR(R15)(r4)
2325 ld r16, VCPU_GPR(R16)(r4)
2326 ld r17, VCPU_GPR(R17)(r4)
2327 ld r18, VCPU_GPR(R18)(r4)
2328 ld r19, VCPU_GPR(R19)(r4)
2329 ld r20, VCPU_GPR(R20)(r4)
2330 ld r21, VCPU_GPR(R21)(r4)
2331 ld r22, VCPU_GPR(R22)(r4)
2332 ld r23, VCPU_GPR(R23)(r4)
2333 ld r24, VCPU_GPR(R24)(r4)
2334 ld r25, VCPU_GPR(R25)(r4)
2335 ld r26, VCPU_GPR(R26)(r4)
2336 ld r27, VCPU_GPR(R27)(r4)
2337 ld r28, VCPU_GPR(R28)(r4)
2338 ld r29, VCPU_GPR(R29)(r4)
2339 ld r30, VCPU_GPR(R30)(r4)
2340 ld r31, VCPU_GPR(R31)(r4)
2342 /* Check the wake reason in SRR1 to see why we got here */
2343 bl kvmppc_check_wake_reason
2345 /* clear our bit in vcore->napping_threads */
2346 34: ld r5,HSTATE_KVM_VCORE(r13)
2347 lbz r7,HSTATE_PTID(r13)
2348 li r0,1
2349 sld r0,r0,r7
2350 addi r6,r5,VCORE_NAPPING_THREADS
2351 32: lwarx r7,0,r6
2352 andc r7,r7,r0
2353 stwcx. r7,0,r6
2354 bne 32b
2355 li r0,0
2356 stb r0,HSTATE_NAPPING(r13)
2358 /* See if the wake reason means we need to exit */
2359 stw r12, VCPU_TRAP(r4)
2360 mr r9, r4
2361 cmpdi r3, 0
2362 bgt guest_exit_cont
2364 /* see if any other thread is already exiting */
2365 lwz r0,VCORE_ENTRY_EXIT(r5)
2366 cmpwi r0,0x100
2367 bge guest_exit_cont
2369 b kvmppc_cede_reentry /* if not go back to guest */
2371 /* cede when already previously prodded case */
2372 kvm_cede_prodded:
2373 li r0,0
2374 stb r0,VCPU_PRODDED(r3)
2375 sync /* order testing prodded vs. clearing ceded */
2376 stb r0,VCPU_CEDED(r3)
2377 li r3,H_SUCCESS
2378 blr
2380 /* we've ceded but we want to give control to the host */
2381 kvm_cede_exit:
2382 ld r9, HSTATE_KVM_VCPU(r13)
2383 b guest_exit_cont
2385 /* Try to handle a machine check in real mode */
2386 machine_check_realmode:
2387 mr r3, r9 /* get vcpu pointer */
2388 bl kvmppc_realmode_machine_check
2389 nop
2390 ld r9, HSTATE_KVM_VCPU(r13)
2391 li r12, BOOK3S_INTERRUPT_MACHINE_CHECK
2392 /*
2393 * Deliver unhandled/fatal (e.g. UE) MCE errors to guest through
2394 * machine check interrupt (set HSRR0 to 0x200). And for handled
2395 * errors (no-fatal), just go back to guest execution with current
2396 * HSRR0 instead of exiting guest. This new approach will inject
2397 * machine check to guest for fatal error causing guest to crash.
2398 *
2399 * The old code used to return to host for unhandled errors which
2400 * was causing guest to hang with soft lockups inside guest and
2401 * makes it difficult to recover guest instance.
2402 *
2403 * if we receive machine check with MSR(RI=0) then deliver it to
2404 * guest as machine check causing guest to crash.
2405 */
2406 ld r11, VCPU_MSR(r9)
2407 andi. r10, r11, MSR_RI /* check for unrecoverable exception */
2408 beq 1f /* Deliver a machine check to guest */
2409 ld r10, VCPU_PC(r9)
2410 cmpdi r3, 0 /* Did we handle MCE ? */
2411 bne 2f /* Continue guest execution. */
2412 /* If not, deliver a machine check. SRR0/1 are already set */
2413 1: li r10, BOOK3S_INTERRUPT_MACHINE_CHECK
2414 bl kvmppc_msr_interrupt
2415 2: b fast_interrupt_c_return
2417 /*
2418 * Check the reason we woke from nap, and take appropriate action.
2419 * Returns (in r3):
2420 * 0 if nothing needs to be done
2421 * 1 if something happened that needs to be handled by the host
2422 * -1 if there was a guest wakeup (IPI or msgsnd)
2423 *
2424 * Also sets r12 to the interrupt vector for any interrupt that needs
2425 * to be handled now by the host (0x500 for external interrupt), or zero.
2426 * Modifies r0, r6, r7, r8.
2427 */
2428 kvmppc_check_wake_reason:
2429 mfspr r6, SPRN_SRR1
2430 BEGIN_FTR_SECTION
2431 rlwinm r6, r6, 45-31, 0xf /* extract wake reason field (P8) */
2432 FTR_SECTION_ELSE
2433 rlwinm r6, r6, 45-31, 0xe /* P7 wake reason field is 3 bits */
2434 ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S)
2435 cmpwi r6, 8 /* was it an external interrupt? */
2436 li r12, BOOK3S_INTERRUPT_EXTERNAL
2437 beq kvmppc_read_intr /* if so, see what it was */
2438 li r3, 0
2439 li r12, 0
2440 cmpwi r6, 6 /* was it the decrementer? */
2441 beq 0f
2442 BEGIN_FTR_SECTION
2443 cmpwi r6, 5 /* privileged doorbell? */
2444 beq 0f
2445 cmpwi r6, 3 /* hypervisor doorbell? */
2446 beq 3f
2447 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2448 li r3, 1 /* anything else, return 1 */
2449 0: blr
2451 /* hypervisor doorbell */
2452 3: li r12, BOOK3S_INTERRUPT_H_DOORBELL
2454 /*
2455 * Clear the doorbell as we will invoke the handler
2456 * explicitly in the guest exit path.
2457 */
2458 lis r6, (PPC_DBELL_SERVER << (63-36))@h
2459 PPC_MSGCLR(6)
2460 /* see if it's a host IPI */
2461 li r3, 1
2462 lbz r0, HSTATE_HOST_IPI(r13)
2463 cmpwi r0, 0
2464 bnelr
2465 /* if not, return -1 */
2466 li r3, -1
2467 blr
2469 /*
2470 * Determine what sort of external interrupt is pending (if any).
2471 * Returns:
2472 * 0 if no interrupt is pending
2473 * 1 if an interrupt is pending that needs to be handled by the host
2474 * -1 if there was a guest wakeup IPI (which has now been cleared)
2475 * Modifies r0, r6, r7, r8, returns value in r3.
2476 */
2477 kvmppc_read_intr:
2478 /* see if a host IPI is pending */
2479 li r3, 1
2480 lbz r0, HSTATE_HOST_IPI(r13)
2481 cmpwi r0, 0
2482 bne 1f
2484 /* Now read the interrupt from the ICP */
2485 ld r6, HSTATE_XICS_PHYS(r13)
2486 li r7, XICS_XIRR
2487 cmpdi r6, 0
2488 beq- 1f
2489 lwzcix r0, r6, r7
2490 /*
2491 * Save XIRR for later. Since we get in in reverse endian on LE
2492 * systems, save it byte reversed and fetch it back in host endian.
2493 */
2494 li r3, HSTATE_SAVED_XIRR
2495 STWX_BE r0, r3, r13
2496 #ifdef __LITTLE_ENDIAN__
2497 lwz r3, HSTATE_SAVED_XIRR(r13)
2498 #else
2499 mr r3, r0
2500 #endif
2501 rlwinm. r3, r3, 0, 0xffffff
2502 sync
2503 beq 1f /* if nothing pending in the ICP */
2505 /* We found something in the ICP...
2506 *
2507 * If it's not an IPI, stash it in the PACA and return to
2508 * the host, we don't (yet) handle directing real external
2509 * interrupts directly to the guest
2510 */
2511 cmpwi r3, XICS_IPI /* if there is, is it an IPI? */
2512 bne 42f
2514 /* It's an IPI, clear the MFRR and EOI it */
2515 li r3, 0xff
2516 li r8, XICS_MFRR
2517 stbcix r3, r6, r8 /* clear the IPI */
2518 stwcix r0, r6, r7 /* EOI it */
2519 sync
2521 /* We need to re-check host IPI now in case it got set in the
2522 * meantime. If it's clear, we bounce the interrupt to the
2523 * guest
2524 */
2525 lbz r0, HSTATE_HOST_IPI(r13)
2526 cmpwi r0, 0
2527 bne- 43f
2529 /* OK, it's an IPI for us */
2530 li r12, 0
2531 li r3, -1
2532 1: blr
2534 42: /* It's not an IPI and it's for the host. We saved a copy of XIRR in
2535 * the PACA earlier, it will be picked up by the host ICP driver
2536 */
2537 li r3, 1
2538 b 1b
2540 43: /* We raced with the host, we need to resend that IPI, bummer */
2541 li r0, IPI_PRIORITY
2542 stbcix r0, r6, r8 /* set the IPI */
2543 sync
2544 li r3, 1
2545 b 1b
2547 /*
2548 * Save away FP, VMX and VSX registers.
2549 * r3 = vcpu pointer
2550 * N.B. r30 and r31 are volatile across this function,
2551 * thus it is not callable from C.
2552 */
2553 kvmppc_save_fp:
2554 mflr r30
2555 mr r31,r3
2556 mfmsr r5
2557 ori r8,r5,MSR_FP
2558 #ifdef CONFIG_ALTIVEC
2559 BEGIN_FTR_SECTION
2560 oris r8,r8,MSR_VEC@h
2561 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2562 #endif
2563 #ifdef CONFIG_VSX
2564 BEGIN_FTR_SECTION
2565 oris r8,r8,MSR_VSX@h
2566 END_FTR_SECTION_IFSET(CPU_FTR_VSX)
2567 #endif
2568 mtmsrd r8
2569 addi r3,r3,VCPU_FPRS
2570 bl store_fp_state
2571 #ifdef CONFIG_ALTIVEC
2572 BEGIN_FTR_SECTION
2573 addi r3,r31,VCPU_VRS
2574 bl store_vr_state
2575 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2576 #endif
2577 mfspr r6,SPRN_VRSAVE
2578 stw r6,VCPU_VRSAVE(r31)
2579 mtlr r30
2580 blr
2582 /*
2583 * Load up FP, VMX and VSX registers
2584 * r4 = vcpu pointer
2585 * N.B. r30 and r31 are volatile across this function,
2586 * thus it is not callable from C.
2587 */
2588 kvmppc_load_fp:
2589 mflr r30
2590 mr r31,r4
2591 mfmsr r9
2592 ori r8,r9,MSR_FP
2593 #ifdef CONFIG_ALTIVEC
2594 BEGIN_FTR_SECTION
2595 oris r8,r8,MSR_VEC@h
2596 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2597 #endif
2598 #ifdef CONFIG_VSX
2599 BEGIN_FTR_SECTION
2600 oris r8,r8,MSR_VSX@h
2601 END_FTR_SECTION_IFSET(CPU_FTR_VSX)
2602 #endif
2603 mtmsrd r8
2604 addi r3,r4,VCPU_FPRS
2605 bl load_fp_state
2606 #ifdef CONFIG_ALTIVEC
2607 BEGIN_FTR_SECTION
2608 addi r3,r31,VCPU_VRS
2609 bl load_vr_state
2610 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2611 #endif
2612 lwz r7,VCPU_VRSAVE(r31)
2613 mtspr SPRN_VRSAVE,r7
2614 mtlr r30
2615 mr r4,r31
2616 blr
2618 /*
2619 * We come here if we get any exception or interrupt while we are
2620 * executing host real mode code while in guest MMU context.
2621 * For now just spin, but we should do something better.
2622 */
2623 kvmppc_bad_host_intr:
2624 b .
2626 /*
2627 * This mimics the MSR transition on IRQ delivery. The new guest MSR is taken
2628 * from VCPU_INTR_MSR and is modified based on the required TM state changes.
2629 * r11 has the guest MSR value (in/out)
2630 * r9 has a vcpu pointer (in)
2631 * r0 is used as a scratch register
2632 */
2633 kvmppc_msr_interrupt:
2634 rldicl r0, r11, 64 - MSR_TS_S_LG, 62
2635 cmpwi r0, 2 /* Check if we are in transactional state.. */
2636 ld r11, VCPU_INTR_MSR(r9)
2637 bne 1f
2638 /* ... if transactional, change to suspended */
2639 li r0, 1
2640 1: rldimi r11, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG
2641 blr
2643 /*
2644 * This works around a hardware bug on POWER8E processors, where
2645 * writing a 1 to the MMCR0[PMAO] bit doesn't generate a
2646 * performance monitor interrupt. Instead, when we need to have
2647 * an interrupt pending, we have to arrange for a counter to overflow.
2648 */
2649 kvmppc_fix_pmao:
2650 li r3, 0
2651 mtspr SPRN_MMCR2, r3
2652 lis r3, (MMCR0_PMXE | MMCR0_FCECE)@h
2653 ori r3, r3, MMCR0_PMCjCE | MMCR0_C56RUN
2654 mtspr SPRN_MMCR0, r3
2655 lis r3, 0x7fff
2656 ori r3, r3, 0xffff
2657 mtspr SPRN_PMC6, r3
2658 isync
2659 blr
2661 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
2662 /*
2663 * Start timing an activity
2664 * r3 = pointer to time accumulation struct, r4 = vcpu
2665 */
2666 kvmhv_start_timing:
2667 ld r5, HSTATE_KVM_VCORE(r13)
2668 lbz r6, VCORE_IN_GUEST(r5)
2669 cmpwi r6, 0
2670 beq 5f /* if in guest, need to */
2671 ld r6, VCORE_TB_OFFSET(r5) /* subtract timebase offset */
2672 5: mftb r5
2673 subf r5, r6, r5
2674 std r3, VCPU_CUR_ACTIVITY(r4)
2675 std r5, VCPU_ACTIVITY_START(r4)
2676 blr
2678 /*
2679 * Accumulate time to one activity and start another.
2680 * r3 = pointer to new time accumulation struct, r4 = vcpu
2681 */
2682 kvmhv_accumulate_time:
2683 ld r5, HSTATE_KVM_VCORE(r13)
2684 lbz r8, VCORE_IN_GUEST(r5)
2685 cmpwi r8, 0
2686 beq 4f /* if in guest, need to */
2687 ld r8, VCORE_TB_OFFSET(r5) /* subtract timebase offset */
2688 4: ld r5, VCPU_CUR_ACTIVITY(r4)
2689 ld r6, VCPU_ACTIVITY_START(r4)
2690 std r3, VCPU_CUR_ACTIVITY(r4)
2691 mftb r7
2692 subf r7, r8, r7
2693 std r7, VCPU_ACTIVITY_START(r4)
2694 cmpdi r5, 0
2695 beqlr
2696 subf r3, r6, r7
2697 ld r8, TAS_SEQCOUNT(r5)
2698 cmpdi r8, 0
2699 addi r8, r8, 1
2700 std r8, TAS_SEQCOUNT(r5)
2701 lwsync
2702 ld r7, TAS_TOTAL(r5)
2703 add r7, r7, r3
2704 std r7, TAS_TOTAL(r5)
2705 ld r6, TAS_MIN(r5)
2706 ld r7, TAS_MAX(r5)
2707 beq 3f
2708 cmpd r3, r6
2709 bge 1f
2710 3: std r3, TAS_MIN(r5)
2711 1: cmpd r3, r7
2712 ble 2f
2713 std r3, TAS_MAX(r5)
2714 2: lwsync
2715 addi r8, r8, 1
2716 std r8, TAS_SEQCOUNT(r5)
2717 blr
2718 #endif