| blob | e571ad277398fd6625499c05ae03cde47a46d20a |
1 /*
2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License, version 2, as
4 * published by the Free Software Foundation.
5 *
6 * This program is distributed in the hope that it will be useful,
7 * but WITHOUT ANY WARRANTY; without even the implied warranty of
8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
9 * GNU General Public License for more details.
10 *
11 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
12 *
13 * Derived from book3s_rmhandlers.S and other files, which are:
14 *
15 * Copyright SUSE Linux Products GmbH 2009
16 *
17 * Authors: Alexander Graf <agraf@suse.de>
18 */
20 #include <asm/ppc_asm.h>
21 #include <asm/kvm_asm.h>
22 #include <asm/reg.h>
23 #include <asm/mmu.h>
24 #include <asm/page.h>
25 #include <asm/ptrace.h>
26 #include <asm/hvcall.h>
27 #include <asm/asm-offsets.h>
28 #include <asm/exception-64s.h>
29 #include <asm/kvm_book3s_asm.h>
30 #include <asm/book3s/64/mmu-hash.h>
31 #include <asm/tm.h>
33 #define VCPU_GPRS_TM(reg) (((reg) * ULONG_SIZE) + VCPU_GPR_TM)
35 /* Values in HSTATE_NAPPING(r13) */
36 #define NAPPING_CEDE 1
37 #define NAPPING_NOVCPU 2
39 /*
40 * Call kvmppc_hv_entry in real mode.
41 * Must be called with interrupts hard-disabled.
42 *
43 * Input Registers:
44 *
45 * LR = return address to continue at after eventually re-enabling MMU
46 */
47 _GLOBAL_TOC(kvmppc_hv_entry_trampoline)
48 mflr r0
49 std r0, PPC_LR_STKOFF(r1)
50 stdu r1, -112(r1)
51 mfmsr r10
52 LOAD_REG_ADDR(r5, kvmppc_call_hv_entry)
53 li r0,MSR_RI
54 andc r0,r10,r0
55 li r6,MSR_IR | MSR_DR
56 andc r6,r10,r6
57 mtmsrd r0,1 /* clear RI in MSR */
58 mtsrr0 r5
59 mtsrr1 r6
60 RFI
62 kvmppc_call_hv_entry:
63 ld r4, HSTATE_KVM_VCPU(r13)
64 bl kvmppc_hv_entry
66 /* Back from guest - restore host state and return to caller */
68 BEGIN_FTR_SECTION
69 /* Restore host DABR and DABRX */
70 ld r5,HSTATE_DABR(r13)
71 li r6,7
72 mtspr SPRN_DABR,r5
73 mtspr SPRN_DABRX,r6
74 END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
76 /* Restore SPRG3 */
77 ld r3,PACA_SPRG_VDSO(r13)
78 mtspr SPRN_SPRG_VDSO_WRITE,r3
80 /* Reload the host's PMU registers */
81 ld r3, PACALPPACAPTR(r13) /* is the host using the PMU? */
82 lbz r4, LPPACA_PMCINUSE(r3)
83 cmpwi r4, 0
84 beq 23f /* skip if not */
85 BEGIN_FTR_SECTION
86 ld r3, HSTATE_MMCR0(r13)
87 andi. r4, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO
88 cmpwi r4, MMCR0_PMAO
89 beql kvmppc_fix_pmao
90 END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG)
91 lwz r3, HSTATE_PMC1(r13)
92 lwz r4, HSTATE_PMC2(r13)
93 lwz r5, HSTATE_PMC3(r13)
94 lwz r6, HSTATE_PMC4(r13)
95 lwz r8, HSTATE_PMC5(r13)
96 lwz r9, HSTATE_PMC6(r13)
97 mtspr SPRN_PMC1, r3
98 mtspr SPRN_PMC2, r4
99 mtspr SPRN_PMC3, r5
100 mtspr SPRN_PMC4, r6
101 mtspr SPRN_PMC5, r8
102 mtspr SPRN_PMC6, r9
103 ld r3, HSTATE_MMCR0(r13)
104 ld r4, HSTATE_MMCR1(r13)
105 ld r5, HSTATE_MMCRA(r13)
106 ld r6, HSTATE_SIAR(r13)
107 ld r7, HSTATE_SDAR(r13)
108 mtspr SPRN_MMCR1, r4
109 mtspr SPRN_MMCRA, r5
110 mtspr SPRN_SIAR, r6
111 mtspr SPRN_SDAR, r7
112 BEGIN_FTR_SECTION
113 ld r8, HSTATE_MMCR2(r13)
114 ld r9, HSTATE_SIER(r13)
115 mtspr SPRN_MMCR2, r8
116 mtspr SPRN_SIER, r9
117 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
118 mtspr SPRN_MMCR0, r3
119 isync
120 23:
122 /*
123 * Reload DEC. HDEC interrupts were disabled when
124 * we reloaded the host's LPCR value.
125 */
126 ld r3, HSTATE_DECEXP(r13)
127 mftb r4
128 subf r4, r4, r3
129 mtspr SPRN_DEC, r4
131 /* hwthread_req may have got set by cede or no vcpu, so clear it */
132 li r0, 0
133 stb r0, HSTATE_HWTHREAD_REQ(r13)
135 /*
136 * For external and machine check interrupts, we need
137 * to call the Linux handler to process the interrupt.
138 * We do that by jumping to absolute address 0x500 for
139 * external interrupts, or the machine_check_fwnmi label
140 * for machine checks (since firmware might have patched
141 * the vector area at 0x200). The [h]rfid at the end of the
142 * handler will return to the book3s_hv_interrupts.S code.
143 * For other interrupts we do the rfid to get back
144 * to the book3s_hv_interrupts.S code here.
145 */
146 ld r8, 112+PPC_LR_STKOFF(r1)
147 addi r1, r1, 112
148 ld r7, HSTATE_HOST_MSR(r13)
150 cmpwi cr1, r12, BOOK3S_INTERRUPT_MACHINE_CHECK
151 cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL
152 beq 11f
153 cmpwi r12, BOOK3S_INTERRUPT_H_DOORBELL
154 beq 15f /* Invoke the H_DOORBELL handler */
155 cmpwi cr2, r12, BOOK3S_INTERRUPT_HMI
156 beq cr2, 14f /* HMI check */
158 /* RFI into the highmem handler, or branch to interrupt handler */
159 mfmsr r6
160 li r0, MSR_RI
161 andc r6, r6, r0
162 mtmsrd r6, 1 /* Clear RI in MSR */
163 mtsrr0 r8
164 mtsrr1 r7
165 beq cr1, 13f /* machine check */
166 RFI
168 /* On POWER7, we have external interrupts set to use HSRR0/1 */
169 11: mtspr SPRN_HSRR0, r8
170 mtspr SPRN_HSRR1, r7
171 ba 0x500
173 13: b machine_check_fwnmi
175 14: mtspr SPRN_HSRR0, r8
176 mtspr SPRN_HSRR1, r7
177 b hmi_exception_after_realmode
179 15: mtspr SPRN_HSRR0, r8
180 mtspr SPRN_HSRR1, r7
181 ba 0xe80
183 kvmppc_primary_no_guest:
184 /* We handle this much like a ceded vcpu */
185 /* put the HDEC into the DEC, since HDEC interrupts don't wake us */
186 mfspr r3, SPRN_HDEC
187 mtspr SPRN_DEC, r3
188 /*
189 * Make sure the primary has finished the MMU switch.
190 * We should never get here on a secondary thread, but
191 * check it for robustness' sake.
192 */
193 ld r5, HSTATE_KVM_VCORE(r13)
194 65: lbz r0, VCORE_IN_GUEST(r5)
195 cmpwi r0, 0
196 beq 65b
197 /* Set LPCR. */
198 ld r8,VCORE_LPCR(r5)
199 mtspr SPRN_LPCR,r8
200 isync
201 /* set our bit in napping_threads */
202 ld r5, HSTATE_KVM_VCORE(r13)
203 lbz r7, HSTATE_PTID(r13)
204 li r0, 1
205 sld r0, r0, r7
206 addi r6, r5, VCORE_NAPPING_THREADS
207 1: lwarx r3, 0, r6
208 or r3, r3, r0
209 stwcx. r3, 0, r6
210 bne 1b
211 /* order napping_threads update vs testing entry_exit_map */
212 isync
213 li r12, 0
214 lwz r7, VCORE_ENTRY_EXIT(r5)
215 cmpwi r7, 0x100
216 bge kvm_novcpu_exit /* another thread already exiting */
217 li r3, NAPPING_NOVCPU
218 stb r3, HSTATE_NAPPING(r13)
220 li r3, 0 /* Don't wake on privileged (OS) doorbell */
221 b kvm_do_nap
223 kvm_novcpu_wakeup:
224 ld r1, HSTATE_HOST_R1(r13)
225 ld r5, HSTATE_KVM_VCORE(r13)
226 li r0, 0
227 stb r0, HSTATE_NAPPING(r13)
229 /* check the wake reason */
230 bl kvmppc_check_wake_reason
232 /* see if any other thread is already exiting */
233 lwz r0, VCORE_ENTRY_EXIT(r5)
234 cmpwi r0, 0x100
235 bge kvm_novcpu_exit
237 /* clear our bit in napping_threads */
238 lbz r7, HSTATE_PTID(r13)
239 li r0, 1
240 sld r0, r0, r7
241 addi r6, r5, VCORE_NAPPING_THREADS
242 4: lwarx r7, 0, r6
243 andc r7, r7, r0
244 stwcx. r7, 0, r6
245 bne 4b
247 /* See if the wake reason means we need to exit */
248 cmpdi r3, 0
249 bge kvm_novcpu_exit
251 /* See if our timeslice has expired (HDEC is negative) */
252 mfspr r0, SPRN_HDEC
253 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER
254 cmpwi r0, 0
255 blt kvm_novcpu_exit
257 /* Got an IPI but other vcpus aren't yet exiting, must be a latecomer */
258 ld r4, HSTATE_KVM_VCPU(r13)
259 cmpdi r4, 0
260 beq kvmppc_primary_no_guest
262 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
263 addi r3, r4, VCPU_TB_RMENTRY
264 bl kvmhv_start_timing
265 #endif
266 b kvmppc_got_guest
268 kvm_novcpu_exit:
269 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
270 ld r4, HSTATE_KVM_VCPU(r13)
271 cmpdi r4, 0
272 beq 13f
273 addi r3, r4, VCPU_TB_RMEXIT
274 bl kvmhv_accumulate_time
275 #endif
276 13: mr r3, r12
277 stw r12, 112-4(r1)
278 bl kvmhv_commence_exit
279 nop
280 lwz r12, 112-4(r1)
281 b kvmhv_switch_to_host
283 /*
284 * We come in here when wakened from nap mode.
285 * Relocation is off and most register values are lost.
286 * r13 points to the PACA.
287 */
288 .globl kvm_start_guest
289 kvm_start_guest:
291 /* Set runlatch bit the minute you wake up from nap */
292 mfspr r0, SPRN_CTRLF
293 ori r0, r0, 1
294 mtspr SPRN_CTRLT, r0
296 ld r2,PACATOC(r13)
298 li r0,KVM_HWTHREAD_IN_KVM
299 stb r0,HSTATE_HWTHREAD_STATE(r13)
301 /* NV GPR values from power7_idle() will no longer be valid */
302 li r0,1
303 stb r0,PACA_NAPSTATELOST(r13)
305 /* were we napping due to cede? */
306 lbz r0,HSTATE_NAPPING(r13)
307 cmpwi r0,NAPPING_CEDE
308 beq kvm_end_cede
309 cmpwi r0,NAPPING_NOVCPU
310 beq kvm_novcpu_wakeup
312 ld r1,PACAEMERGSP(r13)
313 subi r1,r1,STACK_FRAME_OVERHEAD
315 /*
316 * We weren't napping due to cede, so this must be a secondary
317 * thread being woken up to run a guest, or being woken up due
318 * to a stray IPI. (Or due to some machine check or hypervisor
319 * maintenance interrupt while the core is in KVM.)
320 */
322 /* Check the wake reason in SRR1 to see why we got here */
323 bl kvmppc_check_wake_reason
324 cmpdi r3, 0
325 bge kvm_no_guest
327 /* get vcore pointer, NULL if we have nothing to run */
328 ld r5,HSTATE_KVM_VCORE(r13)
329 cmpdi r5,0
330 /* if we have no vcore to run, go back to sleep */
331 beq kvm_no_guest
333 kvm_secondary_got_guest:
335 /* Set HSTATE_DSCR(r13) to something sensible */
336 ld r6, PACA_DSCR_DEFAULT(r13)
337 std r6, HSTATE_DSCR(r13)
339 /* On thread 0 of a subcore, set HDEC to max */
340 lbz r4, HSTATE_PTID(r13)
341 cmpwi r4, 0
342 bne 63f
343 lis r6, 0x7fff
344 ori r6, r6, 0xffff
345 mtspr SPRN_HDEC, r6
346 /* and set per-LPAR registers, if doing dynamic micro-threading */
347 ld r6, HSTATE_SPLIT_MODE(r13)
348 cmpdi r6, 0
349 beq 63f
350 ld r0, KVM_SPLIT_RPR(r6)
351 mtspr SPRN_RPR, r0
352 ld r0, KVM_SPLIT_PMMAR(r6)
353 mtspr SPRN_PMMAR, r0
354 ld r0, KVM_SPLIT_LDBAR(r6)
355 mtspr SPRN_LDBAR, r0
356 isync
357 63:
358 /* Order load of vcpu after load of vcore */
359 lwsync
360 ld r4, HSTATE_KVM_VCPU(r13)
361 bl kvmppc_hv_entry
363 /* Back from the guest, go back to nap */
364 /* Clear our vcpu and vcore pointers so we don't come back in early */
365 li r0, 0
366 std r0, HSTATE_KVM_VCPU(r13)
367 /*
368 * Once we clear HSTATE_KVM_VCORE(r13), the code in
369 * kvmppc_run_core() is going to assume that all our vcpu
370 * state is visible in memory. This lwsync makes sure
371 * that that is true.
372 */
373 lwsync
374 std r0, HSTATE_KVM_VCORE(r13)
376 /*
377 * At this point we have finished executing in the guest.
378 * We need to wait for hwthread_req to become zero, since
379 * we may not turn on the MMU while hwthread_req is non-zero.
380 * While waiting we also need to check if we get given a vcpu to run.
381 */
382 kvm_no_guest:
383 lbz r3, HSTATE_HWTHREAD_REQ(r13)
384 cmpwi r3, 0
385 bne 53f
386 HMT_MEDIUM
387 li r0, KVM_HWTHREAD_IN_KERNEL
388 stb r0, HSTATE_HWTHREAD_STATE(r13)
389 /* need to recheck hwthread_req after a barrier, to avoid race */
390 sync
391 lbz r3, HSTATE_HWTHREAD_REQ(r13)
392 cmpwi r3, 0
393 bne 54f
394 /*
395 * We jump to power7_wakeup_loss, which will return to the caller
396 * of power7_nap in the powernv cpu offline loop. The value we
397 * put in r3 becomes the return value for power7_nap.
398 */
399 li r3, LPCR_PECE0
400 mfspr r4, SPRN_LPCR
401 rlwimi r4, r3, 0, LPCR_PECE0 | LPCR_PECE1
402 mtspr SPRN_LPCR, r4
403 li r3, 0
404 b power7_wakeup_loss
406 53: HMT_LOW
407 ld r5, HSTATE_KVM_VCORE(r13)
408 cmpdi r5, 0
409 bne 60f
410 ld r3, HSTATE_SPLIT_MODE(r13)
411 cmpdi r3, 0
412 beq kvm_no_guest
413 lbz r0, KVM_SPLIT_DO_NAP(r3)
414 cmpwi r0, 0
415 beq kvm_no_guest
416 HMT_MEDIUM
417 b kvm_unsplit_nap
418 60: HMT_MEDIUM
419 b kvm_secondary_got_guest
421 54: li r0, KVM_HWTHREAD_IN_KVM
422 stb r0, HSTATE_HWTHREAD_STATE(r13)
423 b kvm_no_guest
425 /*
426 * Here the primary thread is trying to return the core to
427 * whole-core mode, so we need to nap.
428 */
429 kvm_unsplit_nap:
430 /*
431 * Ensure that secondary doesn't nap when it has
432 * its vcore pointer set.
433 */
434 sync /* matches smp_mb() before setting split_info.do_nap */
435 ld r0, HSTATE_KVM_VCORE(r13)
436 cmpdi r0, 0
437 bne kvm_no_guest
438 /* clear any pending message */
439 BEGIN_FTR_SECTION
440 lis r6, (PPC_DBELL_SERVER << (63-36))@h
441 PPC_MSGCLR(6)
442 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
443 /* Set kvm_split_mode.napped[tid] = 1 */
444 ld r3, HSTATE_SPLIT_MODE(r13)
445 li r0, 1
446 lhz r4, PACAPACAINDEX(r13)
447 clrldi r4, r4, 61 /* micro-threading => P8 => 8 threads/core */
448 addi r4, r4, KVM_SPLIT_NAPPED
449 stbx r0, r3, r4
450 /* Check the do_nap flag again after setting napped[] */
451 sync
452 lbz r0, KVM_SPLIT_DO_NAP(r3)
453 cmpwi r0, 0
454 beq 57f
455 li r3, (LPCR_PECEDH | LPCR_PECE0) >> 4
456 mfspr r4, SPRN_LPCR
457 rlwimi r4, r3, 4, (LPCR_PECEDP | LPCR_PECEDH | LPCR_PECE0 | LPCR_PECE1)
458 mtspr SPRN_LPCR, r4
459 isync
460 std r0, HSTATE_SCRATCH0(r13)
461 ptesync
462 ld r0, HSTATE_SCRATCH0(r13)
463 1: cmpd r0, r0
464 bne 1b
465 nap
466 b .
468 57: li r0, 0
469 stbx r0, r3, r4
470 b kvm_no_guest
472 /******************************************************************************
473 * *
474 * Entry code *
475 * *
476 *****************************************************************************/
478 .global kvmppc_hv_entry
479 kvmppc_hv_entry:
481 /* Required state:
482 *
483 * R4 = vcpu pointer (or NULL)
484 * MSR = ~IR|DR
485 * R13 = PACA
486 * R1 = host R1
487 * R2 = TOC
488 * all other volatile GPRS = free
489 */
490 mflr r0
491 std r0, PPC_LR_STKOFF(r1)
492 stdu r1, -112(r1)
494 /* Save R1 in the PACA */
495 std r1, HSTATE_HOST_R1(r13)
497 li r6, KVM_GUEST_MODE_HOST_HV
498 stb r6, HSTATE_IN_GUEST(r13)
500 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
501 /* Store initial timestamp */
502 cmpdi r4, 0
503 beq 1f
504 addi r3, r4, VCPU_TB_RMENTRY
505 bl kvmhv_start_timing
506 1:
507 #endif
508 /* Clear out SLB */
509 li r6,0
510 slbmte r6,r6
511 slbia
512 ptesync
514 /*
515 * POWER7/POWER8 host -> guest partition switch code.
516 * We don't have to lock against concurrent tlbies,
517 * but we do have to coordinate across hardware threads.
518 */
519 /* Set bit in entry map iff exit map is zero. */
520 ld r5, HSTATE_KVM_VCORE(r13)
521 li r7, 1
522 lbz r6, HSTATE_PTID(r13)
523 sld r7, r7, r6
524 addi r9, r5, VCORE_ENTRY_EXIT
525 21: lwarx r3, 0, r9
526 cmpwi r3, 0x100 /* any threads starting to exit? */
527 bge secondary_too_late /* if so we're too late to the party */
528 or r3, r3, r7
529 stwcx. r3, 0, r9
530 bne 21b
532 /* Primary thread switches to guest partition. */
533 ld r9,VCORE_KVM(r5) /* pointer to struct kvm */
534 cmpwi r6,0
535 bne 10f
536 ld r6,KVM_SDR1(r9)
537 lwz r7,KVM_LPID(r9)
538 li r0,LPID_RSVD /* switch to reserved LPID */
539 mtspr SPRN_LPID,r0
540 ptesync
541 mtspr SPRN_SDR1,r6 /* switch to partition page table */
542 mtspr SPRN_LPID,r7
543 isync
545 /* See if we need to flush the TLB */
546 lhz r6,PACAPACAINDEX(r13) /* test_bit(cpu, need_tlb_flush) */
547 clrldi r7,r6,64-6 /* extract bit number (6 bits) */
548 srdi r6,r6,6 /* doubleword number */
549 sldi r6,r6,3 /* address offset */
550 add r6,r6,r9
551 addi r6,r6,KVM_NEED_FLUSH /* dword in kvm->arch.need_tlb_flush */
552 li r0,1
553 sld r0,r0,r7
554 ld r7,0(r6)
555 and. r7,r7,r0
556 beq 22f
557 23: ldarx r7,0,r6 /* if set, clear the bit */
558 andc r7,r7,r0
559 stdcx. r7,0,r6
560 bne 23b
561 /* Flush the TLB of any entries for this LPID */
562 /* use arch 2.07S as a proxy for POWER8 */
563 BEGIN_FTR_SECTION
564 li r6,512 /* POWER8 has 512 sets */
565 FTR_SECTION_ELSE
566 li r6,128 /* POWER7 has 128 sets */
567 ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S)
568 mtctr r6
569 li r7,0x800 /* IS field = 0b10 */
570 ptesync
571 28: tlbiel r7
572 addi r7,r7,0x1000
573 bdnz 28b
574 ptesync
576 /* Add timebase offset onto timebase */
577 22: ld r8,VCORE_TB_OFFSET(r5)
578 cmpdi r8,0
579 beq 37f
580 mftb r6 /* current host timebase */
581 add r8,r8,r6
582 mtspr SPRN_TBU40,r8 /* update upper 40 bits */
583 mftb r7 /* check if lower 24 bits overflowed */
584 clrldi r6,r6,40
585 clrldi r7,r7,40
586 cmpld r7,r6
587 bge 37f
588 addis r8,r8,0x100 /* if so, increment upper 40 bits */
589 mtspr SPRN_TBU40,r8
591 /* Load guest PCR value to select appropriate compat mode */
592 37: ld r7, VCORE_PCR(r5)
593 cmpdi r7, 0
594 beq 38f
595 mtspr SPRN_PCR, r7
596 38:
598 BEGIN_FTR_SECTION
599 /* DPDES is shared between threads */
600 ld r8, VCORE_DPDES(r5)
601 mtspr SPRN_DPDES, r8
602 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
604 li r0,1
605 stb r0,VCORE_IN_GUEST(r5) /* signal secondaries to continue */
607 /* Do we have a guest vcpu to run? */
608 10: cmpdi r4, 0
609 beq kvmppc_primary_no_guest
610 kvmppc_got_guest:
612 /* Load up guest SLB entries */
613 lwz r5,VCPU_SLB_MAX(r4)
614 cmpwi r5,0
615 beq 9f
616 mtctr r5
617 addi r6,r4,VCPU_SLB
618 1: ld r8,VCPU_SLB_E(r6)
619 ld r9,VCPU_SLB_V(r6)
620 slbmte r9,r8
621 addi r6,r6,VCPU_SLB_SIZE
622 bdnz 1b
623 9:
624 /* Increment yield count if they have a VPA */
625 ld r3, VCPU_VPA(r4)
626 cmpdi r3, 0
627 beq 25f
628 li r6, LPPACA_YIELDCOUNT
629 LWZX_BE r5, r3, r6
630 addi r5, r5, 1
631 STWX_BE r5, r3, r6
632 li r6, 1
633 stb r6, VCPU_VPA_DIRTY(r4)
634 25:
636 /* Save purr/spurr */
637 mfspr r5,SPRN_PURR
638 mfspr r6,SPRN_SPURR
639 std r5,HSTATE_PURR(r13)
640 std r6,HSTATE_SPURR(r13)
641 ld r7,VCPU_PURR(r4)
642 ld r8,VCPU_SPURR(r4)
643 mtspr SPRN_PURR,r7
644 mtspr SPRN_SPURR,r8
646 BEGIN_FTR_SECTION
647 /* Set partition DABR */
648 /* Do this before re-enabling PMU to avoid P7 DABR corruption bug */
649 lwz r5,VCPU_DABRX(r4)
650 ld r6,VCPU_DABR(r4)
651 mtspr SPRN_DABRX,r5
652 mtspr SPRN_DABR,r6
653 isync
654 END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
656 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
657 BEGIN_FTR_SECTION
658 b skip_tm
659 END_FTR_SECTION_IFCLR(CPU_FTR_TM)
661 /* Turn on TM/FP/VSX/VMX so we can restore them. */
662 mfmsr r5
663 li r6, MSR_TM >> 32
664 sldi r6, r6, 32
665 or r5, r5, r6
666 ori r5, r5, MSR_FP
667 oris r5, r5, (MSR_VEC | MSR_VSX)@h
668 mtmsrd r5
670 /*
671 * The user may change these outside of a transaction, so they must
672 * always be context switched.
673 */
674 ld r5, VCPU_TFHAR(r4)
675 ld r6, VCPU_TFIAR(r4)
676 ld r7, VCPU_TEXASR(r4)
677 mtspr SPRN_TFHAR, r5
678 mtspr SPRN_TFIAR, r6
679 mtspr SPRN_TEXASR, r7
681 ld r5, VCPU_MSR(r4)
682 rldicl. r5, r5, 64 - MSR_TS_S_LG, 62
683 beq skip_tm /* TM not active in guest */
685 /* Make sure the failure summary is set, otherwise we'll program check
686 * when we trechkpt. It's possible that this might have been not set
687 * on a kvmppc_set_one_reg() call but we shouldn't let this crash the
688 * host.
689 */
690 oris r7, r7, (TEXASR_FS)@h
691 mtspr SPRN_TEXASR, r7
693 /*
694 * We need to load up the checkpointed state for the guest.
695 * We need to do this early as it will blow away any GPRs, VSRs and
696 * some SPRs.
697 */
699 mr r31, r4
700 addi r3, r31, VCPU_FPRS_TM
701 bl load_fp_state
702 addi r3, r31, VCPU_VRS_TM
703 bl load_vr_state
704 mr r4, r31
705 lwz r7, VCPU_VRSAVE_TM(r4)
706 mtspr SPRN_VRSAVE, r7
708 ld r5, VCPU_LR_TM(r4)
709 lwz r6, VCPU_CR_TM(r4)
710 ld r7, VCPU_CTR_TM(r4)
711 ld r8, VCPU_AMR_TM(r4)
712 ld r9, VCPU_TAR_TM(r4)
713 mtlr r5
714 mtcr r6
715 mtctr r7
716 mtspr SPRN_AMR, r8
717 mtspr SPRN_TAR, r9
719 /*
720 * Load up PPR and DSCR values but don't put them in the actual SPRs
721 * till the last moment to avoid running with userspace PPR and DSCR for
722 * too long.
723 */
724 ld r29, VCPU_DSCR_TM(r4)
725 ld r30, VCPU_PPR_TM(r4)
727 std r2, PACATMSCRATCH(r13) /* Save TOC */
729 /* Clear the MSR RI since r1, r13 are all going to be foobar. */
730 li r5, 0
731 mtmsrd r5, 1
733 /* Load GPRs r0-r28 */
734 reg = 0
735 .rept 29
736 ld reg, VCPU_GPRS_TM(reg)(r31)
737 reg = reg + 1
738 .endr
740 mtspr SPRN_DSCR, r29
741 mtspr SPRN_PPR, r30
743 /* Load final GPRs */
744 ld 29, VCPU_GPRS_TM(29)(r31)
745 ld 30, VCPU_GPRS_TM(30)(r31)
746 ld 31, VCPU_GPRS_TM(31)(r31)
748 /* TM checkpointed state is now setup. All GPRs are now volatile. */
749 TRECHKPT
751 /* Now let's get back the state we need. */
752 HMT_MEDIUM
753 GET_PACA(r13)
754 ld r29, HSTATE_DSCR(r13)
755 mtspr SPRN_DSCR, r29
756 ld r4, HSTATE_KVM_VCPU(r13)
757 ld r1, HSTATE_HOST_R1(r13)
758 ld r2, PACATMSCRATCH(r13)
760 /* Set the MSR RI since we have our registers back. */
761 li r5, MSR_RI
762 mtmsrd r5, 1
763 skip_tm:
764 #endif
766 /* Load guest PMU registers */
767 /* R4 is live here (vcpu pointer) */
768 li r3, 1
769 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */
770 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */
771 isync
772 BEGIN_FTR_SECTION
773 ld r3, VCPU_MMCR(r4)
774 andi. r5, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO
775 cmpwi r5, MMCR0_PMAO
776 beql kvmppc_fix_pmao
777 END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG)
778 lwz r3, VCPU_PMC(r4) /* always load up guest PMU registers */
779 lwz r5, VCPU_PMC + 4(r4) /* to prevent information leak */
780 lwz r6, VCPU_PMC + 8(r4)
781 lwz r7, VCPU_PMC + 12(r4)
782 lwz r8, VCPU_PMC + 16(r4)
783 lwz r9, VCPU_PMC + 20(r4)
784 mtspr SPRN_PMC1, r3
785 mtspr SPRN_PMC2, r5
786 mtspr SPRN_PMC3, r6
787 mtspr SPRN_PMC4, r7
788 mtspr SPRN_PMC5, r8
789 mtspr SPRN_PMC6, r9
790 ld r3, VCPU_MMCR(r4)
791 ld r5, VCPU_MMCR + 8(r4)
792 ld r6, VCPU_MMCR + 16(r4)
793 ld r7, VCPU_SIAR(r4)
794 ld r8, VCPU_SDAR(r4)
795 mtspr SPRN_MMCR1, r5
796 mtspr SPRN_MMCRA, r6
797 mtspr SPRN_SIAR, r7
798 mtspr SPRN_SDAR, r8
799 BEGIN_FTR_SECTION
800 ld r5, VCPU_MMCR + 24(r4)
801 ld r6, VCPU_SIER(r4)
802 lwz r7, VCPU_PMC + 24(r4)
803 lwz r8, VCPU_PMC + 28(r4)
804 ld r9, VCPU_MMCR + 32(r4)
805 mtspr SPRN_MMCR2, r5
806 mtspr SPRN_SIER, r6
807 mtspr SPRN_SPMC1, r7
808 mtspr SPRN_SPMC2, r8
809 mtspr SPRN_MMCRS, r9
810 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
811 mtspr SPRN_MMCR0, r3
812 isync
814 /* Load up FP, VMX and VSX registers */
815 bl kvmppc_load_fp
817 ld r14, VCPU_GPR(R14)(r4)
818 ld r15, VCPU_GPR(R15)(r4)
819 ld r16, VCPU_GPR(R16)(r4)
820 ld r17, VCPU_GPR(R17)(r4)
821 ld r18, VCPU_GPR(R18)(r4)
822 ld r19, VCPU_GPR(R19)(r4)
823 ld r20, VCPU_GPR(R20)(r4)
824 ld r21, VCPU_GPR(R21)(r4)
825 ld r22, VCPU_GPR(R22)(r4)
826 ld r23, VCPU_GPR(R23)(r4)
827 ld r24, VCPU_GPR(R24)(r4)
828 ld r25, VCPU_GPR(R25)(r4)
829 ld r26, VCPU_GPR(R26)(r4)
830 ld r27, VCPU_GPR(R27)(r4)
831 ld r28, VCPU_GPR(R28)(r4)
832 ld r29, VCPU_GPR(R29)(r4)
833 ld r30, VCPU_GPR(R30)(r4)
834 ld r31, VCPU_GPR(R31)(r4)
836 /* Switch DSCR to guest value */
837 ld r5, VCPU_DSCR(r4)
838 mtspr SPRN_DSCR, r5
840 BEGIN_FTR_SECTION
841 /* Skip next section on POWER7 */
842 b 8f
843 END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
844 /* Turn on TM so we can access TFHAR/TFIAR/TEXASR */
845 mfmsr r8
846 li r0, 1
847 rldimi r8, r0, MSR_TM_LG, 63-MSR_TM_LG
848 mtmsrd r8
850 /* Load up POWER8-specific registers */
851 ld r5, VCPU_IAMR(r4)
852 lwz r6, VCPU_PSPB(r4)
853 ld r7, VCPU_FSCR(r4)
854 mtspr SPRN_IAMR, r5
855 mtspr SPRN_PSPB, r6
856 mtspr SPRN_FSCR, r7
857 ld r5, VCPU_DAWR(r4)
858 ld r6, VCPU_DAWRX(r4)
859 ld r7, VCPU_CIABR(r4)
860 ld r8, VCPU_TAR(r4)
861 mtspr SPRN_DAWR, r5
862 mtspr SPRN_DAWRX, r6
863 mtspr SPRN_CIABR, r7
864 mtspr SPRN_TAR, r8
865 ld r5, VCPU_IC(r4)
866 ld r6, VCPU_VTB(r4)
867 mtspr SPRN_IC, r5
868 mtspr SPRN_VTB, r6
869 ld r8, VCPU_EBBHR(r4)
870 mtspr SPRN_EBBHR, r8
871 ld r5, VCPU_EBBRR(r4)
872 ld r6, VCPU_BESCR(r4)
873 ld r7, VCPU_CSIGR(r4)
874 ld r8, VCPU_TACR(r4)
875 mtspr SPRN_EBBRR, r5
876 mtspr SPRN_BESCR, r6
877 mtspr SPRN_CSIGR, r7
878 mtspr SPRN_TACR, r8
879 ld r5, VCPU_TCSCR(r4)
880 ld r6, VCPU_ACOP(r4)
881 lwz r7, VCPU_GUEST_PID(r4)
882 ld r8, VCPU_WORT(r4)
883 mtspr SPRN_TCSCR, r5
884 mtspr SPRN_ACOP, r6
885 mtspr SPRN_PID, r7
886 mtspr SPRN_WORT, r8
887 8:
889 /*
890 * Set the decrementer to the guest decrementer.
891 */
892 ld r8,VCPU_DEC_EXPIRES(r4)
893 /* r8 is a host timebase value here, convert to guest TB */
894 ld r5,HSTATE_KVM_VCORE(r13)
895 ld r6,VCORE_TB_OFFSET(r5)
896 add r8,r8,r6
897 mftb r7
898 subf r3,r7,r8
899 mtspr SPRN_DEC,r3
900 stw r3,VCPU_DEC(r4)
902 ld r5, VCPU_SPRG0(r4)
903 ld r6, VCPU_SPRG1(r4)
904 ld r7, VCPU_SPRG2(r4)
905 ld r8, VCPU_SPRG3(r4)
906 mtspr SPRN_SPRG0, r5
907 mtspr SPRN_SPRG1, r6
908 mtspr SPRN_SPRG2, r7
909 mtspr SPRN_SPRG3, r8
911 /* Load up DAR and DSISR */
912 ld r5, VCPU_DAR(r4)
913 lwz r6, VCPU_DSISR(r4)
914 mtspr SPRN_DAR, r5
915 mtspr SPRN_DSISR, r6
917 /* Restore AMR and UAMOR, set AMOR to all 1s */
918 ld r5,VCPU_AMR(r4)
919 ld r6,VCPU_UAMOR(r4)
920 li r7,-1
921 mtspr SPRN_AMR,r5
922 mtspr SPRN_UAMOR,r6
923 mtspr SPRN_AMOR,r7
925 /* Restore state of CTRL run bit; assume 1 on entry */
926 lwz r5,VCPU_CTRL(r4)
927 andi. r5,r5,1
928 bne 4f
929 mfspr r6,SPRN_CTRLF
930 clrrdi r6,r6,1
931 mtspr SPRN_CTRLT,r6
932 4:
933 /* Secondary threads wait for primary to have done partition switch */
934 ld r5, HSTATE_KVM_VCORE(r13)
935 lbz r6, HSTATE_PTID(r13)
936 cmpwi r6, 0
937 beq 21f
938 lbz r0, VCORE_IN_GUEST(r5)
939 cmpwi r0, 0
940 bne 21f
941 HMT_LOW
942 20: lwz r3, VCORE_ENTRY_EXIT(r5)
943 cmpwi r3, 0x100
944 bge no_switch_exit
945 lbz r0, VCORE_IN_GUEST(r5)
946 cmpwi r0, 0
947 beq 20b
948 HMT_MEDIUM
949 21:
950 /* Set LPCR. */
951 ld r8,VCORE_LPCR(r5)
952 mtspr SPRN_LPCR,r8
953 isync
955 /* Check if HDEC expires soon */
956 mfspr r3, SPRN_HDEC
957 cmpwi r3, 512 /* 1 microsecond */
958 blt hdec_soon
960 ld r6, VCPU_CTR(r4)
961 ld r7, VCPU_XER(r4)
963 mtctr r6
964 mtxer r7
966 kvmppc_cede_reentry: /* r4 = vcpu, r13 = paca */
967 ld r10, VCPU_PC(r4)
968 ld r11, VCPU_MSR(r4)
969 ld r6, VCPU_SRR0(r4)
970 ld r7, VCPU_SRR1(r4)
971 mtspr SPRN_SRR0, r6
972 mtspr SPRN_SRR1, r7
974 deliver_guest_interrupt:
975 /* r11 = vcpu->arch.msr & ~MSR_HV */
976 rldicl r11, r11, 63 - MSR_HV_LG, 1
977 rotldi r11, r11, 1 + MSR_HV_LG
978 ori r11, r11, MSR_ME
980 /* Check if we can deliver an external or decrementer interrupt now */
981 ld r0, VCPU_PENDING_EXC(r4)
982 rldicl r0, r0, 64 - BOOK3S_IRQPRIO_EXTERNAL_LEVEL, 63
983 cmpdi cr1, r0, 0
984 andi. r8, r11, MSR_EE
985 mfspr r8, SPRN_LPCR
986 /* Insert EXTERNAL_LEVEL bit into LPCR at the MER bit position */
987 rldimi r8, r0, LPCR_MER_SH, 63 - LPCR_MER_SH
988 mtspr SPRN_LPCR, r8
989 isync
990 beq 5f
991 li r0, BOOK3S_INTERRUPT_EXTERNAL
992 bne cr1, 12f
993 mfspr r0, SPRN_DEC
994 cmpwi r0, 0
995 li r0, BOOK3S_INTERRUPT_DECREMENTER
996 bge 5f
998 12: mtspr SPRN_SRR0, r10
999 mr r10,r0
1000 mtspr SPRN_SRR1, r11
1001 mr r9, r4
1002 bl kvmppc_msr_interrupt
1003 5:
1005 /*
1006 * Required state:
1007 * R4 = vcpu
1008 * R10: value for HSRR0
1009 * R11: value for HSRR1
1010 * R13 = PACA
1011 */
1012 fast_guest_return:
1013 li r0,0
1014 stb r0,VCPU_CEDED(r4) /* cancel cede */
1015 mtspr SPRN_HSRR0,r10
1016 mtspr SPRN_HSRR1,r11
1018 /* Activate guest mode, so faults get handled by KVM */
1019 li r9, KVM_GUEST_MODE_GUEST_HV
1020 stb r9, HSTATE_IN_GUEST(r13)
1022 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1023 /* Accumulate timing */
1024 addi r3, r4, VCPU_TB_GUEST
1025 bl kvmhv_accumulate_time
1026 #endif
1028 /* Enter guest */
1030 BEGIN_FTR_SECTION
1031 ld r5, VCPU_CFAR(r4)
1032 mtspr SPRN_CFAR, r5
1033 END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
1034 BEGIN_FTR_SECTION
1035 ld r0, VCPU_PPR(r4)
1036 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
1038 ld r5, VCPU_LR(r4)
1039 lwz r6, VCPU_CR(r4)
1040 mtlr r5
1041 mtcr r6
1043 ld r1, VCPU_GPR(R1)(r4)
1044 ld r2, VCPU_GPR(R2)(r4)
1045 ld r3, VCPU_GPR(R3)(r4)
1046 ld r5, VCPU_GPR(R5)(r4)
1047 ld r6, VCPU_GPR(R6)(r4)
1048 ld r7, VCPU_GPR(R7)(r4)
1049 ld r8, VCPU_GPR(R8)(r4)
1050 ld r9, VCPU_GPR(R9)(r4)
1051 ld r10, VCPU_GPR(R10)(r4)
1052 ld r11, VCPU_GPR(R11)(r4)
1053 ld r12, VCPU_GPR(R12)(r4)
1054 ld r13, VCPU_GPR(R13)(r4)
1056 BEGIN_FTR_SECTION
1057 mtspr SPRN_PPR, r0
1058 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
1059 ld r0, VCPU_GPR(R0)(r4)
1060 ld r4, VCPU_GPR(R4)(r4)
1062 hrfid
1063 b .
1065 secondary_too_late:
1066 li r12, 0
1067 cmpdi r4, 0
1068 beq 11f
1069 stw r12, VCPU_TRAP(r4)
1070 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1071 addi r3, r4, VCPU_TB_RMEXIT
1072 bl kvmhv_accumulate_time
1073 #endif
1074 11: b kvmhv_switch_to_host
1076 no_switch_exit:
1077 HMT_MEDIUM
1078 li r12, 0
1079 b 12f
1080 hdec_soon:
1081 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER
1082 12: stw r12, VCPU_TRAP(r4)
1083 mr r9, r4
1084 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1085 addi r3, r4, VCPU_TB_RMEXIT
1086 bl kvmhv_accumulate_time
1087 #endif
1088 b guest_exit_cont
1090 /******************************************************************************
1091 * *
1092 * Exit code *
1093 * *
1094 *****************************************************************************/
1096 /*
1097 * We come here from the first-level interrupt handlers.
1098 */
1099 .globl kvmppc_interrupt_hv
1100 kvmppc_interrupt_hv:
1101 /*
1102 * Register contents:
1103 * R12 = interrupt vector
1104 * R13 = PACA
1105 * guest CR, R12 saved in shadow VCPU SCRATCH1/0
1106 * guest R13 saved in SPRN_SCRATCH0
1107 */
1108 std r9, HSTATE_SCRATCH2(r13)
1110 lbz r9, HSTATE_IN_GUEST(r13)
1111 cmpwi r9, KVM_GUEST_MODE_HOST_HV
1112 beq kvmppc_bad_host_intr
1113 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
1114 cmpwi r9, KVM_GUEST_MODE_GUEST
1115 ld r9, HSTATE_SCRATCH2(r13)
1116 beq kvmppc_interrupt_pr
1117 #endif
1118 /* We're now back in the host but in guest MMU context */
1119 li r9, KVM_GUEST_MODE_HOST_HV
1120 stb r9, HSTATE_IN_GUEST(r13)
1122 ld r9, HSTATE_KVM_VCPU(r13)
1124 /* Save registers */
1126 std r0, VCPU_GPR(R0)(r9)
1127 std r1, VCPU_GPR(R1)(r9)
1128 std r2, VCPU_GPR(R2)(r9)
1129 std r3, VCPU_GPR(R3)(r9)
1130 std r4, VCPU_GPR(R4)(r9)
1131 std r5, VCPU_GPR(R5)(r9)
1132 std r6, VCPU_GPR(R6)(r9)
1133 std r7, VCPU_GPR(R7)(r9)
1134 std r8, VCPU_GPR(R8)(r9)
1135 ld r0, HSTATE_SCRATCH2(r13)
1136 std r0, VCPU_GPR(R9)(r9)
1137 std r10, VCPU_GPR(R10)(r9)
1138 std r11, VCPU_GPR(R11)(r9)
1139 ld r3, HSTATE_SCRATCH0(r13)
1140 lwz r4, HSTATE_SCRATCH1(r13)
1141 std r3, VCPU_GPR(R12)(r9)
1142 stw r4, VCPU_CR(r9)
1143 BEGIN_FTR_SECTION
1144 ld r3, HSTATE_CFAR(r13)
1145 std r3, VCPU_CFAR(r9)
1146 END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
1147 BEGIN_FTR_SECTION
1148 ld r4, HSTATE_PPR(r13)
1149 std r4, VCPU_PPR(r9)
1150 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
1152 /* Restore R1/R2 so we can handle faults */
1153 ld r1, HSTATE_HOST_R1(r13)
1154 ld r2, PACATOC(r13)
1156 mfspr r10, SPRN_SRR0
1157 mfspr r11, SPRN_SRR1
1158 std r10, VCPU_SRR0(r9)
1159 std r11, VCPU_SRR1(r9)
1160 andi. r0, r12, 2 /* need to read HSRR0/1? */
1161 beq 1f
1162 mfspr r10, SPRN_HSRR0
1163 mfspr r11, SPRN_HSRR1
1164 clrrdi r12, r12, 2
1165 1: std r10, VCPU_PC(r9)
1166 std r11, VCPU_MSR(r9)
1168 GET_SCRATCH0(r3)
1169 mflr r4
1170 std r3, VCPU_GPR(R13)(r9)
1171 std r4, VCPU_LR(r9)
1173 stw r12,VCPU_TRAP(r9)
1175 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1176 addi r3, r9, VCPU_TB_RMINTR
1177 mr r4, r9
1178 bl kvmhv_accumulate_time
1179 ld r5, VCPU_GPR(R5)(r9)
1180 ld r6, VCPU_GPR(R6)(r9)
1181 ld r7, VCPU_GPR(R7)(r9)
1182 ld r8, VCPU_GPR(R8)(r9)
1183 #endif
1185 /* Save HEIR (HV emulation assist reg) in emul_inst
1186 if this is an HEI (HV emulation interrupt, e40) */
1187 li r3,KVM_INST_FETCH_FAILED
1188 stw r3,VCPU_LAST_INST(r9)
1189 cmpwi r12,BOOK3S_INTERRUPT_H_EMUL_ASSIST
1190 bne 11f
1191 mfspr r3,SPRN_HEIR
1192 11: stw r3,VCPU_HEIR(r9)
1194 /* these are volatile across C function calls */
1195 mfctr r3
1196 mfxer r4
1197 std r3, VCPU_CTR(r9)
1198 std r4, VCPU_XER(r9)
1200 /* If this is a page table miss then see if it's theirs or ours */
1201 cmpwi r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
1202 beq kvmppc_hdsi
1203 cmpwi r12, BOOK3S_INTERRUPT_H_INST_STORAGE
1204 beq kvmppc_hisi
1206 /* See if this is a leftover HDEC interrupt */
1207 cmpwi r12,BOOK3S_INTERRUPT_HV_DECREMENTER
1208 bne 2f
1209 mfspr r3,SPRN_HDEC
1210 cmpwi r3,0
1211 mr r4,r9
1212 bge fast_guest_return
1213 2:
1214 /* See if this is an hcall we can handle in real mode */
1215 cmpwi r12,BOOK3S_INTERRUPT_SYSCALL
1216 beq hcall_try_real_mode
1218 /* Hypervisor doorbell - exit only if host IPI flag set */
1219 cmpwi r12, BOOK3S_INTERRUPT_H_DOORBELL
1220 bne 3f
1221 lbz r0, HSTATE_HOST_IPI(r13)
1222 cmpwi r0, 0
1223 beq 4f
1224 b guest_exit_cont
1225 3:
1226 /* External interrupt ? */
1227 cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL
1228 bne+ guest_exit_cont
1230 /* External interrupt, first check for host_ipi. If this is
1231 * set, we know the host wants us out so let's do it now
1232 */
1233 bl kvmppc_read_intr
1234 cmpdi r3, 0
1235 bgt guest_exit_cont
1237 /* Check if any CPU is heading out to the host, if so head out too */
1238 4: ld r5, HSTATE_KVM_VCORE(r13)
1239 lwz r0, VCORE_ENTRY_EXIT(r5)
1240 cmpwi r0, 0x100
1241 mr r4, r9
1242 blt deliver_guest_interrupt
1244 guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */
1245 /* Save more register state */
1246 mfdar r6
1247 mfdsisr r7
1248 std r6, VCPU_DAR(r9)
1249 stw r7, VCPU_DSISR(r9)
1250 /* don't overwrite fault_dar/fault_dsisr if HDSI */
1251 cmpwi r12,BOOK3S_INTERRUPT_H_DATA_STORAGE
1252 beq mc_cont
1253 std r6, VCPU_FAULT_DAR(r9)
1254 stw r7, VCPU_FAULT_DSISR(r9)
1256 /* See if it is a machine check */
1257 cmpwi r12, BOOK3S_INTERRUPT_MACHINE_CHECK
1258 beq machine_check_realmode
1259 mc_cont:
1260 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1261 addi r3, r9, VCPU_TB_RMEXIT
1262 mr r4, r9
1263 bl kvmhv_accumulate_time
1264 #endif
1266 mr r3, r12
1267 /* Increment exit count, poke other threads to exit */
1268 bl kvmhv_commence_exit
1269 nop
1270 ld r9, HSTATE_KVM_VCPU(r13)
1271 lwz r12, VCPU_TRAP(r9)
1273 /* Stop others sending VCPU interrupts to this physical CPU */
1274 li r0, -1
1275 stw r0, VCPU_CPU(r9)
1276 stw r0, VCPU_THREAD_CPU(r9)
1278 /* Save guest CTRL register, set runlatch to 1 */
1279 mfspr r6,SPRN_CTRLF
1280 stw r6,VCPU_CTRL(r9)
1281 andi. r0,r6,1
1282 bne 4f
1283 ori r6,r6,1
1284 mtspr SPRN_CTRLT,r6
1285 4:
1286 /* Read the guest SLB and save it away */
1287 lwz r0,VCPU_SLB_NR(r9) /* number of entries in SLB */
1288 mtctr r0
1289 li r6,0
1290 addi r7,r9,VCPU_SLB
1291 li r5,0
1292 1: slbmfee r8,r6
1293 andis. r0,r8,SLB_ESID_V@h
1294 beq 2f
1295 add r8,r8,r6 /* put index in */
1296 slbmfev r3,r6
1297 std r8,VCPU_SLB_E(r7)
1298 std r3,VCPU_SLB_V(r7)
1299 addi r7,r7,VCPU_SLB_SIZE
1300 addi r5,r5,1
1301 2: addi r6,r6,1
1302 bdnz 1b
1303 stw r5,VCPU_SLB_MAX(r9)
1305 /*
1306 * Save the guest PURR/SPURR
1307 */
1308 mfspr r5,SPRN_PURR
1309 mfspr r6,SPRN_SPURR
1310 ld r7,VCPU_PURR(r9)
1311 ld r8,VCPU_SPURR(r9)
1312 std r5,VCPU_PURR(r9)
1313 std r6,VCPU_SPURR(r9)
1314 subf r5,r7,r5
1315 subf r6,r8,r6
1317 /*
1318 * Restore host PURR/SPURR and add guest times
1319 * so that the time in the guest gets accounted.
1320 */
1321 ld r3,HSTATE_PURR(r13)
1322 ld r4,HSTATE_SPURR(r13)
1323 add r3,r3,r5
1324 add r4,r4,r6
1325 mtspr SPRN_PURR,r3
1326 mtspr SPRN_SPURR,r4
1328 /* Save DEC */
1329 mfspr r5,SPRN_DEC
1330 mftb r6
1331 extsw r5,r5
1332 add r5,r5,r6
1333 /* r5 is a guest timebase value here, convert to host TB */
1334 ld r3,HSTATE_KVM_VCORE(r13)
1335 ld r4,VCORE_TB_OFFSET(r3)
1336 subf r5,r4,r5
1337 std r5,VCPU_DEC_EXPIRES(r9)
1339 BEGIN_FTR_SECTION
1340 b 8f
1341 END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
1342 /* Save POWER8-specific registers */
1343 mfspr r5, SPRN_IAMR
1344 mfspr r6, SPRN_PSPB
1345 mfspr r7, SPRN_FSCR
1346 std r5, VCPU_IAMR(r9)
1347 stw r6, VCPU_PSPB(r9)
1348 std r7, VCPU_FSCR(r9)
1349 mfspr r5, SPRN_IC
1350 mfspr r6, SPRN_VTB
1351 mfspr r7, SPRN_TAR
1352 std r5, VCPU_IC(r9)
1353 std r6, VCPU_VTB(r9)
1354 std r7, VCPU_TAR(r9)
1355 mfspr r8, SPRN_EBBHR
1356 std r8, VCPU_EBBHR(r9)
1357 mfspr r5, SPRN_EBBRR
1358 mfspr r6, SPRN_BESCR
1359 mfspr r7, SPRN_CSIGR
1360 mfspr r8, SPRN_TACR
1361 std r5, VCPU_EBBRR(r9)
1362 std r6, VCPU_BESCR(r9)
1363 std r7, VCPU_CSIGR(r9)
1364 std r8, VCPU_TACR(r9)
1365 mfspr r5, SPRN_TCSCR
1366 mfspr r6, SPRN_ACOP
1367 mfspr r7, SPRN_PID
1368 mfspr r8, SPRN_WORT
1369 std r5, VCPU_TCSCR(r9)
1370 std r6, VCPU_ACOP(r9)
1371 stw r7, VCPU_GUEST_PID(r9)
1372 std r8, VCPU_WORT(r9)
1373 /*
1374 * Restore various registers to 0, where non-zero values
1375 * set by the guest could disrupt the host.
1376 */
1377 li r0, 0
1378 mtspr SPRN_IAMR, r0
1379 mtspr SPRN_CIABR, r0
1380 mtspr SPRN_DAWRX, r0
1381 mtspr SPRN_TCSCR, r0
1382 mtspr SPRN_WORT, r0
1383 /* Set MMCRS to 1<<31 to freeze and disable the SPMC counters */
1384 li r0, 1
1385 sldi r0, r0, 31
1386 mtspr SPRN_MMCRS, r0
1387 8:
1389 /* Save and reset AMR and UAMOR before turning on the MMU */
1390 mfspr r5,SPRN_AMR
1391 mfspr r6,SPRN_UAMOR
1392 std r5,VCPU_AMR(r9)
1393 std r6,VCPU_UAMOR(r9)
1394 li r6,0
1395 mtspr SPRN_AMR,r6
1397 /* Switch DSCR back to host value */
1398 mfspr r8, SPRN_DSCR
1399 ld r7, HSTATE_DSCR(r13)
1400 std r8, VCPU_DSCR(r9)
1401 mtspr SPRN_DSCR, r7
1403 /* Save non-volatile GPRs */
1404 std r14, VCPU_GPR(R14)(r9)
1405 std r15, VCPU_GPR(R15)(r9)
1406 std r16, VCPU_GPR(R16)(r9)
1407 std r17, VCPU_GPR(R17)(r9)
1408 std r18, VCPU_GPR(R18)(r9)
1409 std r19, VCPU_GPR(R19)(r9)
1410 std r20, VCPU_GPR(R20)(r9)
1411 std r21, VCPU_GPR(R21)(r9)
1412 std r22, VCPU_GPR(R22)(r9)
1413 std r23, VCPU_GPR(R23)(r9)
1414 std r24, VCPU_GPR(R24)(r9)
1415 std r25, VCPU_GPR(R25)(r9)
1416 std r26, VCPU_GPR(R26)(r9)
1417 std r27, VCPU_GPR(R27)(r9)
1418 std r28, VCPU_GPR(R28)(r9)
1419 std r29, VCPU_GPR(R29)(r9)
1420 std r30, VCPU_GPR(R30)(r9)
1421 std r31, VCPU_GPR(R31)(r9)
1423 /* Save SPRGs */
1424 mfspr r3, SPRN_SPRG0
1425 mfspr r4, SPRN_SPRG1
1426 mfspr r5, SPRN_SPRG2
1427 mfspr r6, SPRN_SPRG3
1428 std r3, VCPU_SPRG0(r9)
1429 std r4, VCPU_SPRG1(r9)
1430 std r5, VCPU_SPRG2(r9)
1431 std r6, VCPU_SPRG3(r9)
1433 /* save FP state */
1434 mr r3, r9
1435 bl kvmppc_save_fp
1437 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1438 BEGIN_FTR_SECTION
1439 b 2f
1440 END_FTR_SECTION_IFCLR(CPU_FTR_TM)
1441 /* Turn on TM. */
1442 mfmsr r8
1443 li r0, 1
1444 rldimi r8, r0, MSR_TM_LG, 63-MSR_TM_LG
1445 mtmsrd r8
1447 ld r5, VCPU_MSR(r9)
1448 rldicl. r5, r5, 64 - MSR_TS_S_LG, 62
1449 beq 1f /* TM not active in guest. */
1451 li r3, TM_CAUSE_KVM_RESCHED
1453 /* Clear the MSR RI since r1, r13 are all going to be foobar. */
1454 li r5, 0
1455 mtmsrd r5, 1
1457 /* All GPRs are volatile at this point. */
1458 TRECLAIM(R3)
1460 /* Temporarily store r13 and r9 so we have some regs to play with */
1461 SET_SCRATCH0(r13)
1462 GET_PACA(r13)
1463 std r9, PACATMSCRATCH(r13)
1464 ld r9, HSTATE_KVM_VCPU(r13)
1466 /* Get a few more GPRs free. */
1467 std r29, VCPU_GPRS_TM(29)(r9)
1468 std r30, VCPU_GPRS_TM(30)(r9)
1469 std r31, VCPU_GPRS_TM(31)(r9)
1471 /* Save away PPR and DSCR soon so don't run with user values. */
1472 mfspr r31, SPRN_PPR
1473 HMT_MEDIUM
1474 mfspr r30, SPRN_DSCR
1475 ld r29, HSTATE_DSCR(r13)
1476 mtspr SPRN_DSCR, r29
1478 /* Save all but r9, r13 & r29-r31 */
1479 reg = 0
1480 .rept 29
1481 .if (reg != 9) && (reg != 13)
1482 std reg, VCPU_GPRS_TM(reg)(r9)
1483 .endif
1484 reg = reg + 1
1485 .endr
1486 /* ... now save r13 */
1487 GET_SCRATCH0(r4)
1488 std r4, VCPU_GPRS_TM(13)(r9)
1489 /* ... and save r9 */
1490 ld r4, PACATMSCRATCH(r13)
1491 std r4, VCPU_GPRS_TM(9)(r9)
1493 /* Reload stack pointer and TOC. */
1494 ld r1, HSTATE_HOST_R1(r13)
1495 ld r2, PACATOC(r13)
1497 /* Set MSR RI now we have r1 and r13 back. */
1498 li r5, MSR_RI
1499 mtmsrd r5, 1
1501 /* Save away checkpinted SPRs. */
1502 std r31, VCPU_PPR_TM(r9)
1503 std r30, VCPU_DSCR_TM(r9)
1504 mflr r5
1505 mfcr r6
1506 mfctr r7
1507 mfspr r8, SPRN_AMR
1508 mfspr r10, SPRN_TAR
1509 std r5, VCPU_LR_TM(r9)
1510 stw r6, VCPU_CR_TM(r9)
1511 std r7, VCPU_CTR_TM(r9)
1512 std r8, VCPU_AMR_TM(r9)
1513 std r10, VCPU_TAR_TM(r9)
1515 /* Restore r12 as trap number. */
1516 lwz r12, VCPU_TRAP(r9)
1518 /* Save FP/VSX. */
1519 addi r3, r9, VCPU_FPRS_TM
1520 bl store_fp_state
1521 addi r3, r9, VCPU_VRS_TM
1522 bl store_vr_state
1523 mfspr r6, SPRN_VRSAVE
1524 stw r6, VCPU_VRSAVE_TM(r9)
1525 1:
1526 /*
1527 * We need to save these SPRs after the treclaim so that the software
1528 * error code is recorded correctly in the TEXASR. Also the user may
1529 * change these outside of a transaction, so they must always be
1530 * context switched.
1531 */
1532 mfspr r5, SPRN_TFHAR
1533 mfspr r6, SPRN_TFIAR
1534 mfspr r7, SPRN_TEXASR
1535 std r5, VCPU_TFHAR(r9)
1536 std r6, VCPU_TFIAR(r9)
1537 std r7, VCPU_TEXASR(r9)
1538 2:
1539 #endif
1541 /* Increment yield count if they have a VPA */
1542 ld r8, VCPU_VPA(r9) /* do they have a VPA? */
1543 cmpdi r8, 0
1544 beq 25f
1545 li r4, LPPACA_YIELDCOUNT
1546 LWZX_BE r3, r8, r4
1547 addi r3, r3, 1
1548 STWX_BE r3, r8, r4
1549 li r3, 1
1550 stb r3, VCPU_VPA_DIRTY(r9)
1551 25:
1552 /* Save PMU registers if requested */
1553 /* r8 and cr0.eq are live here */
1554 BEGIN_FTR_SECTION
1555 /*
1556 * POWER8 seems to have a hardware bug where setting
1557 * MMCR0[PMAE] along with MMCR0[PMC1CE] and/or MMCR0[PMCjCE]
1558 * when some counters are already negative doesn't seem
1559 * to cause a performance monitor alert (and hence interrupt).
1560 * The effect of this is that when saving the PMU state,
1561 * if there is no PMU alert pending when we read MMCR0
1562 * before freezing the counters, but one becomes pending
1563 * before we read the counters, we lose it.
1564 * To work around this, we need a way to freeze the counters
1565 * before reading MMCR0. Normally, freezing the counters
1566 * is done by writing MMCR0 (to set MMCR0[FC]) which
1567 * unavoidably writes MMCR0[PMA0] as well. On POWER8,
1568 * we can also freeze the counters using MMCR2, by writing
1569 * 1s to all the counter freeze condition bits (there are
1570 * 9 bits each for 6 counters).
1571 */
1572 li r3, -1 /* set all freeze bits */
1573 clrrdi r3, r3, 10
1574 mfspr r10, SPRN_MMCR2
1575 mtspr SPRN_MMCR2, r3
1576 isync
1577 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1578 li r3, 1
1579 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */
1580 mfspr r4, SPRN_MMCR0 /* save MMCR0 */
1581 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */
1582 mfspr r6, SPRN_MMCRA
1583 /* Clear MMCRA in order to disable SDAR updates */
1584 li r7, 0
1585 mtspr SPRN_MMCRA, r7
1586 isync
1587 beq 21f /* if no VPA, save PMU stuff anyway */
1588 lbz r7, LPPACA_PMCINUSE(r8)
1589 cmpwi r7, 0 /* did they ask for PMU stuff to be saved? */
1590 bne 21f
1591 std r3, VCPU_MMCR(r9) /* if not, set saved MMCR0 to FC */
1592 b 22f
1593 21: mfspr r5, SPRN_MMCR1
1594 mfspr r7, SPRN_SIAR
1595 mfspr r8, SPRN_SDAR
1596 std r4, VCPU_MMCR(r9)
1597 std r5, VCPU_MMCR + 8(r9)
1598 std r6, VCPU_MMCR + 16(r9)
1599 BEGIN_FTR_SECTION
1600 std r10, VCPU_MMCR + 24(r9)
1601 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1602 std r7, VCPU_SIAR(r9)
1603 std r8, VCPU_SDAR(r9)
1604 mfspr r3, SPRN_PMC1
1605 mfspr r4, SPRN_PMC2
1606 mfspr r5, SPRN_PMC3
1607 mfspr r6, SPRN_PMC4
1608 mfspr r7, SPRN_PMC5
1609 mfspr r8, SPRN_PMC6
1610 stw r3, VCPU_PMC(r9)
1611 stw r4, VCPU_PMC + 4(r9)
1612 stw r5, VCPU_PMC + 8(r9)
1613 stw r6, VCPU_PMC + 12(r9)
1614 stw r7, VCPU_PMC + 16(r9)
1615 stw r8, VCPU_PMC + 20(r9)
1616 BEGIN_FTR_SECTION
1617 mfspr r5, SPRN_SIER
1618 mfspr r6, SPRN_SPMC1
1619 mfspr r7, SPRN_SPMC2
1620 mfspr r8, SPRN_MMCRS
1621 std r5, VCPU_SIER(r9)
1622 stw r6, VCPU_PMC + 24(r9)
1623 stw r7, VCPU_PMC + 28(r9)
1624 std r8, VCPU_MMCR + 32(r9)
1625 lis r4, 0x8000
1626 mtspr SPRN_MMCRS, r4
1627 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1628 22:
1629 /* Clear out SLB */
1630 li r5,0
1631 slbmte r5,r5
1632 slbia
1633 ptesync
1635 /*
1636 * POWER7/POWER8 guest -> host partition switch code.
1637 * We don't have to lock against tlbies but we do
1638 * have to coordinate the hardware threads.
1639 */
1640 kvmhv_switch_to_host:
1641 /* Secondary threads wait for primary to do partition switch */
1642 ld r5,HSTATE_KVM_VCORE(r13)
1643 ld r4,VCORE_KVM(r5) /* pointer to struct kvm */
1644 lbz r3,HSTATE_PTID(r13)
1645 cmpwi r3,0
1646 beq 15f
1647 HMT_LOW
1648 13: lbz r3,VCORE_IN_GUEST(r5)
1649 cmpwi r3,0
1650 bne 13b
1651 HMT_MEDIUM
1652 b 16f
1654 /* Primary thread waits for all the secondaries to exit guest */
1655 15: lwz r3,VCORE_ENTRY_EXIT(r5)
1656 rlwinm r0,r3,32-8,0xff
1657 clrldi r3,r3,56
1658 cmpw r3,r0
1659 bne 15b
1660 isync
1662 /* Did we actually switch to the guest at all? */
1663 lbz r6, VCORE_IN_GUEST(r5)
1664 cmpwi r6, 0
1665 beq 19f
1667 /* Primary thread switches back to host partition */
1668 ld r6,KVM_HOST_SDR1(r4)
1669 lwz r7,KVM_HOST_LPID(r4)
1670 li r8,LPID_RSVD /* switch to reserved LPID */
1671 mtspr SPRN_LPID,r8
1672 ptesync
1673 mtspr SPRN_SDR1,r6 /* switch to partition page table */
1674 mtspr SPRN_LPID,r7
1675 isync
1677 BEGIN_FTR_SECTION
1678 /* DPDES is shared between threads */
1679 mfspr r7, SPRN_DPDES
1680 std r7, VCORE_DPDES(r5)
1681 /* clear DPDES so we don't get guest doorbells in the host */
1682 li r8, 0
1683 mtspr SPRN_DPDES, r8
1684 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1686 /* Subtract timebase offset from timebase */
1687 ld r8,VCORE_TB_OFFSET(r5)
1688 cmpdi r8,0
1689 beq 17f
1690 mftb r6 /* current guest timebase */
1691 subf r8,r8,r6
1692 mtspr SPRN_TBU40,r8 /* update upper 40 bits */
1693 mftb r7 /* check if lower 24 bits overflowed */
1694 clrldi r6,r6,40
1695 clrldi r7,r7,40
1696 cmpld r7,r6
1697 bge 17f
1698 addis r8,r8,0x100 /* if so, increment upper 40 bits */
1699 mtspr SPRN_TBU40,r8
1701 /* Reset PCR */
1702 17: ld r0, VCORE_PCR(r5)
1703 cmpdi r0, 0
1704 beq 18f
1705 li r0, 0
1706 mtspr SPRN_PCR, r0
1707 18:
1708 /* Signal secondary CPUs to continue */
1709 stb r0,VCORE_IN_GUEST(r5)
1710 19: lis r8,0x7fff /* MAX_INT@h */
1711 mtspr SPRN_HDEC,r8
1713 16: ld r8,KVM_HOST_LPCR(r4)
1714 mtspr SPRN_LPCR,r8
1715 isync
1717 /* load host SLB entries */
1718 ld r8,PACA_SLBSHADOWPTR(r13)
1720 .rept SLB_NUM_BOLTED
1721 li r3, SLBSHADOW_SAVEAREA
1722 LDX_BE r5, r8, r3
1723 addi r3, r3, 8
1724 LDX_BE r6, r8, r3
1725 andis. r7,r5,SLB_ESID_V@h
1726 beq 1f
1727 slbmte r6,r5
1728 1: addi r8,r8,16
1729 .endr
1731 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1732 /* Finish timing, if we have a vcpu */
1733 ld r4, HSTATE_KVM_VCPU(r13)
1734 cmpdi r4, 0
1735 li r3, 0
1736 beq 2f
1737 bl kvmhv_accumulate_time
1738 2:
1739 #endif
1740 /* Unset guest mode */
1741 li r0, KVM_GUEST_MODE_NONE
1742 stb r0, HSTATE_IN_GUEST(r13)
1744 ld r0, 112+PPC_LR_STKOFF(r1)
1745 addi r1, r1, 112
1746 mtlr r0
1747 blr
1749 /*
1750 * Check whether an HDSI is an HPTE not found fault or something else.
1751 * If it is an HPTE not found fault that is due to the guest accessing
1752 * a page that they have mapped but which we have paged out, then
1753 * we continue on with the guest exit path. In all other cases,
1754 * reflect the HDSI to the guest as a DSI.
1755 */
1756 kvmppc_hdsi:
1757 mfspr r4, SPRN_HDAR
1758 mfspr r6, SPRN_HDSISR
1759 /* HPTE not found fault or protection fault? */
1760 andis. r0, r6, (DSISR_NOHPTE | DSISR_PROTFAULT)@h
1761 beq 1f /* if not, send it to the guest */
1762 andi. r0, r11, MSR_DR /* data relocation enabled? */
1763 beq 3f
1764 clrrdi r0, r4, 28
1765 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */
1766 li r0, BOOK3S_INTERRUPT_DATA_SEGMENT
1767 bne 7f /* if no SLB entry found */
1768 4: std r4, VCPU_FAULT_DAR(r9)
1769 stw r6, VCPU_FAULT_DSISR(r9)
1771 /* Search the hash table. */
1772 mr r3, r9 /* vcpu pointer */
1773 li r7, 1 /* data fault */
1774 bl kvmppc_hpte_hv_fault
1775 ld r9, HSTATE_KVM_VCPU(r13)
1776 ld r10, VCPU_PC(r9)
1777 ld r11, VCPU_MSR(r9)
1778 li r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
1779 cmpdi r3, 0 /* retry the instruction */
1780 beq 6f
1781 cmpdi r3, -1 /* handle in kernel mode */
1782 beq guest_exit_cont
1783 cmpdi r3, -2 /* MMIO emulation; need instr word */
1784 beq 2f
1786 /* Synthesize a DSI (or DSegI) for the guest */
1787 ld r4, VCPU_FAULT_DAR(r9)
1788 mr r6, r3
1789 1: li r0, BOOK3S_INTERRUPT_DATA_STORAGE
1790 mtspr SPRN_DSISR, r6
1791 7: mtspr SPRN_DAR, r4
1792 mtspr SPRN_SRR0, r10
1793 mtspr SPRN_SRR1, r11
1794 mr r10, r0
1795 bl kvmppc_msr_interrupt
1796 fast_interrupt_c_return:
1797 6: ld r7, VCPU_CTR(r9)
1798 ld r8, VCPU_XER(r9)
1799 mtctr r7
1800 mtxer r8
1801 mr r4, r9
1802 b fast_guest_return
1804 3: ld r5, VCPU_KVM(r9) /* not relocated, use VRMA */
1805 ld r5, KVM_VRMA_SLB_V(r5)
1806 b 4b
1808 /* If this is for emulated MMIO, load the instruction word */
1809 2: li r8, KVM_INST_FETCH_FAILED /* In case lwz faults */
1811 /* Set guest mode to 'jump over instruction' so if lwz faults
1812 * we'll just continue at the next IP. */
1813 li r0, KVM_GUEST_MODE_SKIP
1814 stb r0, HSTATE_IN_GUEST(r13)
1816 /* Do the access with MSR:DR enabled */
1817 mfmsr r3
1818 ori r4, r3, MSR_DR /* Enable paging for data */
1819 mtmsrd r4
1820 lwz r8, 0(r10)
1821 mtmsrd r3
1823 /* Store the result */
1824 stw r8, VCPU_LAST_INST(r9)
1826 /* Unset guest mode. */
1827 li r0, KVM_GUEST_MODE_HOST_HV
1828 stb r0, HSTATE_IN_GUEST(r13)
1829 b guest_exit_cont
1831 /*
1832 * Similarly for an HISI, reflect it to the guest as an ISI unless
1833 * it is an HPTE not found fault for a page that we have paged out.
1834 */
1835 kvmppc_hisi:
1836 andis. r0, r11, SRR1_ISI_NOPT@h
1837 beq 1f
1838 andi. r0, r11, MSR_IR /* instruction relocation enabled? */
1839 beq 3f
1840 clrrdi r0, r10, 28
1841 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */
1842 li r0, BOOK3S_INTERRUPT_INST_SEGMENT
1843 bne 7f /* if no SLB entry found */
1844 4:
1845 /* Search the hash table. */
1846 mr r3, r9 /* vcpu pointer */
1847 mr r4, r10
1848 mr r6, r11
1849 li r7, 0 /* instruction fault */
1850 bl kvmppc_hpte_hv_fault
1851 ld r9, HSTATE_KVM_VCPU(r13)
1852 ld r10, VCPU_PC(r9)
1853 ld r11, VCPU_MSR(r9)
1854 li r12, BOOK3S_INTERRUPT_H_INST_STORAGE
1855 cmpdi r3, 0 /* retry the instruction */
1856 beq fast_interrupt_c_return
1857 cmpdi r3, -1 /* handle in kernel mode */
1858 beq guest_exit_cont
1860 /* Synthesize an ISI (or ISegI) for the guest */
1861 mr r11, r3
1862 1: li r0, BOOK3S_INTERRUPT_INST_STORAGE
1863 7: mtspr SPRN_SRR0, r10
1864 mtspr SPRN_SRR1, r11
1865 mr r10, r0
1866 bl kvmppc_msr_interrupt
1867 b fast_interrupt_c_return
1869 3: ld r6, VCPU_KVM(r9) /* not relocated, use VRMA */
1870 ld r5, KVM_VRMA_SLB_V(r6)
1871 b 4b
1873 /*
1874 * Try to handle an hcall in real mode.
1875 * Returns to the guest if we handle it, or continues on up to
1876 * the kernel if we can't (i.e. if we don't have a handler for
1877 * it, or if the handler returns H_TOO_HARD).
1878 *
1879 * r5 - r8 contain hcall args,
1880 * r9 = vcpu, r10 = pc, r11 = msr, r12 = trap, r13 = paca
1881 */
1882 hcall_try_real_mode:
1883 ld r3,VCPU_GPR(R3)(r9)
1884 andi. r0,r11,MSR_PR
1885 /* sc 1 from userspace - reflect to guest syscall */
1886 bne sc_1_fast_return
1887 clrrdi r3,r3,2
1888 cmpldi r3,hcall_real_table_end - hcall_real_table
1889 bge guest_exit_cont
1890 /* See if this hcall is enabled for in-kernel handling */
1891 ld r4, VCPU_KVM(r9)
1892 srdi r0, r3, 8 /* r0 = (r3 / 4) >> 6 */
1893 sldi r0, r0, 3 /* index into kvm->arch.enabled_hcalls[] */
1894 add r4, r4, r0
1895 ld r0, KVM_ENABLED_HCALLS(r4)
1896 rlwinm r4, r3, 32-2, 0x3f /* r4 = (r3 / 4) & 0x3f */
1897 srd r0, r0, r4
1898 andi. r0, r0, 1
1899 beq guest_exit_cont
1900 /* Get pointer to handler, if any, and call it */
1901 LOAD_REG_ADDR(r4, hcall_real_table)
1902 lwax r3,r3,r4
1903 cmpwi r3,0
1904 beq guest_exit_cont
1905 add r12,r3,r4
1906 mtctr r12
1907 mr r3,r9 /* get vcpu pointer */
1908 ld r4,VCPU_GPR(R4)(r9)
1909 bctrl
1910 cmpdi r3,H_TOO_HARD
1911 beq hcall_real_fallback
1912 ld r4,HSTATE_KVM_VCPU(r13)
1913 std r3,VCPU_GPR(R3)(r4)
1914 ld r10,VCPU_PC(r4)
1915 ld r11,VCPU_MSR(r4)
1916 b fast_guest_return
1918 sc_1_fast_return:
1919 mtspr SPRN_SRR0,r10
1920 mtspr SPRN_SRR1,r11
1921 li r10, BOOK3S_INTERRUPT_SYSCALL
1922 bl kvmppc_msr_interrupt
1923 mr r4,r9
1924 b fast_guest_return
1926 /* We've attempted a real mode hcall, but it's punted it back
1927 * to userspace. We need to restore some clobbered volatiles
1928 * before resuming the pass-it-to-qemu path */
1929 hcall_real_fallback:
1930 li r12,BOOK3S_INTERRUPT_SYSCALL
1931 ld r9, HSTATE_KVM_VCPU(r13)
1933 b guest_exit_cont
1935 .globl hcall_real_table
1936 hcall_real_table:
1937 .long 0 /* 0 - unused */
1938 .long DOTSYM(kvmppc_h_remove) - hcall_real_table
1939 .long DOTSYM(kvmppc_h_enter) - hcall_real_table
1940 .long DOTSYM(kvmppc_h_read) - hcall_real_table
1941 .long DOTSYM(kvmppc_h_clear_mod) - hcall_real_table
1942 .long DOTSYM(kvmppc_h_clear_ref) - hcall_real_table
1943 .long DOTSYM(kvmppc_h_protect) - hcall_real_table
1944 .long DOTSYM(kvmppc_h_get_tce) - hcall_real_table
1945 .long DOTSYM(kvmppc_rm_h_put_tce) - hcall_real_table
1946 .long 0 /* 0x24 - H_SET_SPRG0 */
1947 .long DOTSYM(kvmppc_h_set_dabr) - hcall_real_table
1948 .long 0 /* 0x2c */
1949 .long 0 /* 0x30 */
1950 .long 0 /* 0x34 */
1951 .long 0 /* 0x38 */
1952 .long 0 /* 0x3c */
1953 .long 0 /* 0x40 */
1954 .long 0 /* 0x44 */
1955 .long 0 /* 0x48 */
1956 .long 0 /* 0x4c */
1957 .long 0 /* 0x50 */
1958 .long 0 /* 0x54 */
1959 .long 0 /* 0x58 */
1960 .long 0 /* 0x5c */
1961 .long 0 /* 0x60 */
1962 #ifdef CONFIG_KVM_XICS
1963 .long DOTSYM(kvmppc_rm_h_eoi) - hcall_real_table
1964 .long DOTSYM(kvmppc_rm_h_cppr) - hcall_real_table
1965 .long DOTSYM(kvmppc_rm_h_ipi) - hcall_real_table
1966 .long 0 /* 0x70 - H_IPOLL */
1967 .long DOTSYM(kvmppc_rm_h_xirr) - hcall_real_table
1968 #else
1969 .long 0 /* 0x64 - H_EOI */
1970 .long 0 /* 0x68 - H_CPPR */
1971 .long 0 /* 0x6c - H_IPI */
1972 .long 0 /* 0x70 - H_IPOLL */
1973 .long 0 /* 0x74 - H_XIRR */
1974 #endif
1975 .long 0 /* 0x78 */
1976 .long 0 /* 0x7c */
1977 .long 0 /* 0x80 */
1978 .long 0 /* 0x84 */
1979 .long 0 /* 0x88 */
1980 .long 0 /* 0x8c */
1981 .long 0 /* 0x90 */
1982 .long 0 /* 0x94 */
1983 .long 0 /* 0x98 */
1984 .long 0 /* 0x9c */
1985 .long 0 /* 0xa0 */
1986 .long 0 /* 0xa4 */
1987 .long 0 /* 0xa8 */
1988 .long 0 /* 0xac */
1989 .long 0 /* 0xb0 */
1990 .long 0 /* 0xb4 */
1991 .long 0 /* 0xb8 */
1992 .long 0 /* 0xbc */
1993 .long 0 /* 0xc0 */
1994 .long 0 /* 0xc4 */
1995 .long 0 /* 0xc8 */
1996 .long 0 /* 0xcc */
1997 .long 0 /* 0xd0 */
1998 .long 0 /* 0xd4 */
1999 .long 0 /* 0xd8 */
2000 .long 0 /* 0xdc */
2001 .long DOTSYM(kvmppc_h_cede) - hcall_real_table
2002 .long DOTSYM(kvmppc_rm_h_confer) - hcall_real_table
2003 .long 0 /* 0xe8 */
2004 .long 0 /* 0xec */
2005 .long 0 /* 0xf0 */
2006 .long 0 /* 0xf4 */
2007 .long 0 /* 0xf8 */
2008 .long 0 /* 0xfc */
2009 .long 0 /* 0x100 */
2010 .long 0 /* 0x104 */
2011 .long 0 /* 0x108 */
2012 .long 0 /* 0x10c */
2013 .long 0 /* 0x110 */
2014 .long 0 /* 0x114 */
2015 .long 0 /* 0x118 */
2016 .long 0 /* 0x11c */
2017 .long 0 /* 0x120 */
2018 .long DOTSYM(kvmppc_h_bulk_remove) - hcall_real_table
2019 .long 0 /* 0x128 */
2020 .long 0 /* 0x12c */
2021 .long 0 /* 0x130 */
2022 .long DOTSYM(kvmppc_h_set_xdabr) - hcall_real_table
2023 .long DOTSYM(kvmppc_rm_h_stuff_tce) - hcall_real_table
2024 .long DOTSYM(kvmppc_rm_h_put_tce_indirect) - hcall_real_table
2025 .long 0 /* 0x140 */
2026 .long 0 /* 0x144 */
2027 .long 0 /* 0x148 */
2028 .long 0 /* 0x14c */
2029 .long 0 /* 0x150 */
2030 .long 0 /* 0x154 */
2031 .long 0 /* 0x158 */
2032 .long 0 /* 0x15c */
2033 .long 0 /* 0x160 */
2034 .long 0 /* 0x164 */
2035 .long 0 /* 0x168 */
2036 .long 0 /* 0x16c */
2037 .long 0 /* 0x170 */
2038 .long 0 /* 0x174 */
2039 .long 0 /* 0x178 */
2040 .long 0 /* 0x17c */
2041 .long 0 /* 0x180 */
2042 .long 0 /* 0x184 */
2043 .long 0 /* 0x188 */
2044 .long 0 /* 0x18c */
2045 .long 0 /* 0x190 */
2046 .long 0 /* 0x194 */
2047 .long 0 /* 0x198 */
2048 .long 0 /* 0x19c */
2049 .long 0 /* 0x1a0 */
2050 .long 0 /* 0x1a4 */
2051 .long 0 /* 0x1a8 */
2052 .long 0 /* 0x1ac */
2053 .long 0 /* 0x1b0 */
2054 .long 0 /* 0x1b4 */
2055 .long 0 /* 0x1b8 */
2056 .long 0 /* 0x1bc */
2057 .long 0 /* 0x1c0 */
2058 .long 0 /* 0x1c4 */
2059 .long 0 /* 0x1c8 */
2060 .long 0 /* 0x1cc */
2061 .long 0 /* 0x1d0 */
2062 .long 0 /* 0x1d4 */
2063 .long 0 /* 0x1d8 */
2064 .long 0 /* 0x1dc */
2065 .long 0 /* 0x1e0 */
2066 .long 0 /* 0x1e4 */
2067 .long 0 /* 0x1e8 */
2068 .long 0 /* 0x1ec */
2069 .long 0 /* 0x1f0 */
2070 .long 0 /* 0x1f4 */
2071 .long 0 /* 0x1f8 */
2072 .long 0 /* 0x1fc */
2073 .long 0 /* 0x200 */
2074 .long 0 /* 0x204 */
2075 .long 0 /* 0x208 */
2076 .long 0 /* 0x20c */
2077 .long 0 /* 0x210 */
2078 .long 0 /* 0x214 */
2079 .long 0 /* 0x218 */
2080 .long 0 /* 0x21c */
2081 .long 0 /* 0x220 */
2082 .long 0 /* 0x224 */
2083 .long 0 /* 0x228 */
2084 .long 0 /* 0x22c */
2085 .long 0 /* 0x230 */
2086 .long 0 /* 0x234 */
2087 .long 0 /* 0x238 */
2088 .long 0 /* 0x23c */
2089 .long 0 /* 0x240 */
2090 .long 0 /* 0x244 */
2091 .long 0 /* 0x248 */
2092 .long 0 /* 0x24c */
2093 .long 0 /* 0x250 */
2094 .long 0 /* 0x254 */
2095 .long 0 /* 0x258 */
2096 .long 0 /* 0x25c */
2097 .long 0 /* 0x260 */
2098 .long 0 /* 0x264 */
2099 .long 0 /* 0x268 */
2100 .long 0 /* 0x26c */
2101 .long 0 /* 0x270 */
2102 .long 0 /* 0x274 */
2103 .long 0 /* 0x278 */
2104 .long 0 /* 0x27c */
2105 .long 0 /* 0x280 */
2106 .long 0 /* 0x284 */
2107 .long 0 /* 0x288 */
2108 .long 0 /* 0x28c */
2109 .long 0 /* 0x290 */
2110 .long 0 /* 0x294 */
2111 .long 0 /* 0x298 */
2112 .long 0 /* 0x29c */
2113 .long 0 /* 0x2a0 */
2114 .long 0 /* 0x2a4 */
2115 .long 0 /* 0x2a8 */
2116 .long 0 /* 0x2ac */
2117 .long 0 /* 0x2b0 */
2118 .long 0 /* 0x2b4 */
2119 .long 0 /* 0x2b8 */
2120 .long 0 /* 0x2bc */
2121 .long 0 /* 0x2c0 */
2122 .long 0 /* 0x2c4 */
2123 .long 0 /* 0x2c8 */
2124 .long 0 /* 0x2cc */
2125 .long 0 /* 0x2d0 */
2126 .long 0 /* 0x2d4 */
2127 .long 0 /* 0x2d8 */
2128 .long 0 /* 0x2dc */
2129 .long 0 /* 0x2e0 */
2130 .long 0 /* 0x2e4 */
2131 .long 0 /* 0x2e8 */
2132 .long 0 /* 0x2ec */
2133 .long 0 /* 0x2f0 */
2134 .long 0 /* 0x2f4 */
2135 .long 0 /* 0x2f8 */
2136 .long 0 /* 0x2fc */
2137 .long DOTSYM(kvmppc_h_random) - hcall_real_table
2138 .globl hcall_real_table_end
2139 hcall_real_table_end:
2141 _GLOBAL(kvmppc_h_set_xdabr)
2142 andi. r0, r5, DABRX_USER | DABRX_KERNEL
2143 beq 6f
2144 li r0, DABRX_USER | DABRX_KERNEL | DABRX_BTI
2145 andc. r0, r5, r0
2146 beq 3f
2147 6: li r3, H_PARAMETER
2148 blr
2150 _GLOBAL(kvmppc_h_set_dabr)
2151 li r5, DABRX_USER | DABRX_KERNEL
2152 3:
2153 BEGIN_FTR_SECTION
2154 b 2f
2155 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2156 std r4,VCPU_DABR(r3)
2157 stw r5, VCPU_DABRX(r3)
2158 mtspr SPRN_DABRX, r5
2159 /* Work around P7 bug where DABR can get corrupted on mtspr */
2160 1: mtspr SPRN_DABR,r4
2161 mfspr r5, SPRN_DABR
2162 cmpd r4, r5
2163 bne 1b
2164 isync
2165 li r3,0
2166 blr
2168 /* Emulate H_SET_DABR/X on P8 for the sake of compat mode guests */
2169 2: rlwimi r5, r4, 5, DAWRX_DR | DAWRX_DW
2170 rlwimi r5, r4, 2, DAWRX_WT
2171 clrrdi r4, r4, 3
2172 std r4, VCPU_DAWR(r3)
2173 std r5, VCPU_DAWRX(r3)
2174 mtspr SPRN_DAWR, r4
2175 mtspr SPRN_DAWRX, r5
2176 li r3, 0
2177 blr
2179 _GLOBAL(kvmppc_h_cede) /* r3 = vcpu pointer, r11 = msr, r13 = paca */
2180 ori r11,r11,MSR_EE
2181 std r11,VCPU_MSR(r3)
2182 li r0,1
2183 stb r0,VCPU_CEDED(r3)
2184 sync /* order setting ceded vs. testing prodded */
2185 lbz r5,VCPU_PRODDED(r3)
2186 cmpwi r5,0
2187 bne kvm_cede_prodded
2188 li r12,0 /* set trap to 0 to say hcall is handled */
2189 stw r12,VCPU_TRAP(r3)
2190 li r0,H_SUCCESS
2191 std r0,VCPU_GPR(R3)(r3)
2193 /*
2194 * Set our bit in the bitmask of napping threads unless all the
2195 * other threads are already napping, in which case we send this
2196 * up to the host.
2197 */
2198 ld r5,HSTATE_KVM_VCORE(r13)
2199 lbz r6,HSTATE_PTID(r13)
2200 lwz r8,VCORE_ENTRY_EXIT(r5)
2201 clrldi r8,r8,56
2202 li r0,1
2203 sld r0,r0,r6
2204 addi r6,r5,VCORE_NAPPING_THREADS
2205 31: lwarx r4,0,r6
2206 or r4,r4,r0
2207 cmpw r4,r8
2208 beq kvm_cede_exit
2209 stwcx. r4,0,r6
2210 bne 31b
2211 /* order napping_threads update vs testing entry_exit_map */
2212 isync
2213 li r0,NAPPING_CEDE
2214 stb r0,HSTATE_NAPPING(r13)
2215 lwz r7,VCORE_ENTRY_EXIT(r5)
2216 cmpwi r7,0x100
2217 bge 33f /* another thread already exiting */
2219 /*
2220 * Although not specifically required by the architecture, POWER7
2221 * preserves the following registers in nap mode, even if an SMT mode
2222 * switch occurs: SLB entries, PURR, SPURR, AMOR, UAMOR, AMR, SPRG0-3,
2223 * DAR, DSISR, DABR, DABRX, DSCR, PMCx, MMCRx, SIAR, SDAR.
2224 */
2225 /* Save non-volatile GPRs */
2226 std r14, VCPU_GPR(R14)(r3)
2227 std r15, VCPU_GPR(R15)(r3)
2228 std r16, VCPU_GPR(R16)(r3)
2229 std r17, VCPU_GPR(R17)(r3)
2230 std r18, VCPU_GPR(R18)(r3)
2231 std r19, VCPU_GPR(R19)(r3)
2232 std r20, VCPU_GPR(R20)(r3)
2233 std r21, VCPU_GPR(R21)(r3)
2234 std r22, VCPU_GPR(R22)(r3)
2235 std r23, VCPU_GPR(R23)(r3)
2236 std r24, VCPU_GPR(R24)(r3)
2237 std r25, VCPU_GPR(R25)(r3)
2238 std r26, VCPU_GPR(R26)(r3)
2239 std r27, VCPU_GPR(R27)(r3)
2240 std r28, VCPU_GPR(R28)(r3)
2241 std r29, VCPU_GPR(R29)(r3)
2242 std r30, VCPU_GPR(R30)(r3)
2243 std r31, VCPU_GPR(R31)(r3)
2245 /* save FP state */
2246 bl kvmppc_save_fp
2248 /*
2249 * Set DEC to the smaller of DEC and HDEC, so that we wake
2250 * no later than the end of our timeslice (HDEC interrupts
2251 * don't wake us from nap).
2252 */
2253 mfspr r3, SPRN_DEC
2254 mfspr r4, SPRN_HDEC
2255 mftb r5
2256 cmpw r3, r4
2257 ble 67f
2258 mtspr SPRN_DEC, r4
2259 67:
2260 /* save expiry time of guest decrementer */
2261 extsw r3, r3
2262 add r3, r3, r5
2263 ld r4, HSTATE_KVM_VCPU(r13)
2264 ld r5, HSTATE_KVM_VCORE(r13)
2265 ld r6, VCORE_TB_OFFSET(r5)
2266 subf r3, r6, r3 /* convert to host TB value */
2267 std r3, VCPU_DEC_EXPIRES(r4)
2269 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
2270 ld r4, HSTATE_KVM_VCPU(r13)
2271 addi r3, r4, VCPU_TB_CEDE
2272 bl kvmhv_accumulate_time
2273 #endif
2275 lis r3, LPCR_PECEDP@h /* Do wake on privileged doorbell */
2277 /*
2278 * Take a nap until a decrementer or external or doobell interrupt
2279 * occurs, with PECE1 and PECE0 set in LPCR.
2280 * On POWER8, set PECEDH, and if we are ceding, also set PECEDP.
2281 * Also clear the runlatch bit before napping.
2282 */
2283 kvm_do_nap:
2284 mfspr r0, SPRN_CTRLF
2285 clrrdi r0, r0, 1
2286 mtspr SPRN_CTRLT, r0
2288 li r0,1
2289 stb r0,HSTATE_HWTHREAD_REQ(r13)
2290 mfspr r5,SPRN_LPCR
2291 ori r5,r5,LPCR_PECE0 | LPCR_PECE1
2292 BEGIN_FTR_SECTION
2293 ori r5, r5, LPCR_PECEDH
2294 rlwimi r5, r3, 0, LPCR_PECEDP
2295 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2296 mtspr SPRN_LPCR,r5
2297 isync
2298 li r0, 0
2299 std r0, HSTATE_SCRATCH0(r13)
2300 ptesync
2301 ld r0, HSTATE_SCRATCH0(r13)
2302 1: cmpd r0, r0
2303 bne 1b
2304 nap
2305 b .
2307 33: mr r4, r3
2308 li r3, 0
2309 li r12, 0
2310 b 34f
2312 kvm_end_cede:
2313 /* get vcpu pointer */
2314 ld r4, HSTATE_KVM_VCPU(r13)
2316 /* Woken by external or decrementer interrupt */
2317 ld r1, HSTATE_HOST_R1(r13)
2319 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
2320 addi r3, r4, VCPU_TB_RMINTR
2321 bl kvmhv_accumulate_time
2322 #endif
2324 /* load up FP state */
2325 bl kvmppc_load_fp
2327 /* Restore guest decrementer */
2328 ld r3, VCPU_DEC_EXPIRES(r4)
2329 ld r5, HSTATE_KVM_VCORE(r13)
2330 ld r6, VCORE_TB_OFFSET(r5)
2331 add r3, r3, r6 /* convert host TB to guest TB value */
2332 mftb r7
2333 subf r3, r7, r3
2334 mtspr SPRN_DEC, r3
2336 /* Load NV GPRS */
2337 ld r14, VCPU_GPR(R14)(r4)
2338 ld r15, VCPU_GPR(R15)(r4)
2339 ld r16, VCPU_GPR(R16)(r4)
2340 ld r17, VCPU_GPR(R17)(r4)
2341 ld r18, VCPU_GPR(R18)(r4)
2342 ld r19, VCPU_GPR(R19)(r4)
2343 ld r20, VCPU_GPR(R20)(r4)
2344 ld r21, VCPU_GPR(R21)(r4)
2345 ld r22, VCPU_GPR(R22)(r4)
2346 ld r23, VCPU_GPR(R23)(r4)
2347 ld r24, VCPU_GPR(R24)(r4)
2348 ld r25, VCPU_GPR(R25)(r4)
2349 ld r26, VCPU_GPR(R26)(r4)
2350 ld r27, VCPU_GPR(R27)(r4)
2351 ld r28, VCPU_GPR(R28)(r4)
2352 ld r29, VCPU_GPR(R29)(r4)
2353 ld r30, VCPU_GPR(R30)(r4)
2354 ld r31, VCPU_GPR(R31)(r4)
2356 /* Check the wake reason in SRR1 to see why we got here */
2357 bl kvmppc_check_wake_reason
2359 /* clear our bit in vcore->napping_threads */
2360 34: ld r5,HSTATE_KVM_VCORE(r13)
2361 lbz r7,HSTATE_PTID(r13)
2362 li r0,1
2363 sld r0,r0,r7
2364 addi r6,r5,VCORE_NAPPING_THREADS
2365 32: lwarx r7,0,r6
2366 andc r7,r7,r0
2367 stwcx. r7,0,r6
2368 bne 32b
2369 li r0,0
2370 stb r0,HSTATE_NAPPING(r13)
2372 /* See if the wake reason means we need to exit */
2373 stw r12, VCPU_TRAP(r4)
2374 mr r9, r4
2375 cmpdi r3, 0
2376 bgt guest_exit_cont
2378 /* see if any other thread is already exiting */
2379 lwz r0,VCORE_ENTRY_EXIT(r5)
2380 cmpwi r0,0x100
2381 bge guest_exit_cont
2383 b kvmppc_cede_reentry /* if not go back to guest */
2385 /* cede when already previously prodded case */
2386 kvm_cede_prodded:
2387 li r0,0
2388 stb r0,VCPU_PRODDED(r3)
2389 sync /* order testing prodded vs. clearing ceded */
2390 stb r0,VCPU_CEDED(r3)
2391 li r3,H_SUCCESS
2392 blr
2394 /* we've ceded but we want to give control to the host */
2395 kvm_cede_exit:
2396 ld r9, HSTATE_KVM_VCPU(r13)
2397 b guest_exit_cont
2399 /* Try to handle a machine check in real mode */
2400 machine_check_realmode:
2401 mr r3, r9 /* get vcpu pointer */
2402 bl kvmppc_realmode_machine_check
2403 nop
2404 ld r9, HSTATE_KVM_VCPU(r13)
2405 li r12, BOOK3S_INTERRUPT_MACHINE_CHECK
2406 /*
2407 * Deliver unhandled/fatal (e.g. UE) MCE errors to guest through
2408 * machine check interrupt (set HSRR0 to 0x200). And for handled
2409 * errors (no-fatal), just go back to guest execution with current
2410 * HSRR0 instead of exiting guest. This new approach will inject
2411 * machine check to guest for fatal error causing guest to crash.
2412 *
2413 * The old code used to return to host for unhandled errors which
2414 * was causing guest to hang with soft lockups inside guest and
2415 * makes it difficult to recover guest instance.
2416 *
2417 * if we receive machine check with MSR(RI=0) then deliver it to
2418 * guest as machine check causing guest to crash.
2419 */
2420 ld r11, VCPU_MSR(r9)
2421 rldicl. r0, r11, 64-MSR_HV_LG, 63 /* check if it happened in HV mode */
2422 bne mc_cont /* if so, exit to host */
2423 andi. r10, r11, MSR_RI /* check for unrecoverable exception */
2424 beq 1f /* Deliver a machine check to guest */
2425 ld r10, VCPU_PC(r9)
2426 cmpdi r3, 0 /* Did we handle MCE ? */
2427 bne 2f /* Continue guest execution. */
2428 /* If not, deliver a machine check. SRR0/1 are already set */
2429 1: li r10, BOOK3S_INTERRUPT_MACHINE_CHECK
2430 bl kvmppc_msr_interrupt
2431 2: b fast_interrupt_c_return
2433 /*
2434 * Check the reason we woke from nap, and take appropriate action.
2435 * Returns (in r3):
2436 * 0 if nothing needs to be done
2437 * 1 if something happened that needs to be handled by the host
2438 * -1 if there was a guest wakeup (IPI or msgsnd)
2439 *
2440 * Also sets r12 to the interrupt vector for any interrupt that needs
2441 * to be handled now by the host (0x500 for external interrupt), or zero.
2442 * Modifies r0, r6, r7, r8.
2443 */
2444 kvmppc_check_wake_reason:
2445 mfspr r6, SPRN_SRR1
2446 BEGIN_FTR_SECTION
2447 rlwinm r6, r6, 45-31, 0xf /* extract wake reason field (P8) */
2448 FTR_SECTION_ELSE
2449 rlwinm r6, r6, 45-31, 0xe /* P7 wake reason field is 3 bits */
2450 ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S)
2451 cmpwi r6, 8 /* was it an external interrupt? */
2452 li r12, BOOK3S_INTERRUPT_EXTERNAL
2453 beq kvmppc_read_intr /* if so, see what it was */
2454 li r3, 0
2455 li r12, 0
2456 cmpwi r6, 6 /* was it the decrementer? */
2457 beq 0f
2458 BEGIN_FTR_SECTION
2459 cmpwi r6, 5 /* privileged doorbell? */
2460 beq 0f
2461 cmpwi r6, 3 /* hypervisor doorbell? */
2462 beq 3f
2463 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2464 li r3, 1 /* anything else, return 1 */
2465 0: blr
2467 /* hypervisor doorbell */
2468 3: li r12, BOOK3S_INTERRUPT_H_DOORBELL
2470 /*
2471 * Clear the doorbell as we will invoke the handler
2472 * explicitly in the guest exit path.
2473 */
2474 lis r6, (PPC_DBELL_SERVER << (63-36))@h
2475 PPC_MSGCLR(6)
2476 /* see if it's a host IPI */
2477 li r3, 1
2478 lbz r0, HSTATE_HOST_IPI(r13)
2479 cmpwi r0, 0
2480 bnelr
2481 /* if not, return -1 */
2482 li r3, -1
2483 blr
2485 /*
2486 * Determine what sort of external interrupt is pending (if any).
2487 * Returns:
2488 * 0 if no interrupt is pending
2489 * 1 if an interrupt is pending that needs to be handled by the host
2490 * -1 if there was a guest wakeup IPI (which has now been cleared)
2491 * Modifies r0, r6, r7, r8, returns value in r3.
2492 */
2493 kvmppc_read_intr:
2494 /* see if a host IPI is pending */
2495 li r3, 1
2496 lbz r0, HSTATE_HOST_IPI(r13)
2497 cmpwi r0, 0
2498 bne 1f
2500 /* Now read the interrupt from the ICP */
2501 ld r6, HSTATE_XICS_PHYS(r13)
2502 li r7, XICS_XIRR
2503 cmpdi r6, 0
2504 beq- 1f
2505 lwzcix r0, r6, r7
2506 /*
2507 * Save XIRR for later. Since we get in in reverse endian on LE
2508 * systems, save it byte reversed and fetch it back in host endian.
2509 */
2510 li r3, HSTATE_SAVED_XIRR
2511 STWX_BE r0, r3, r13
2512 #ifdef __LITTLE_ENDIAN__
2513 lwz r3, HSTATE_SAVED_XIRR(r13)
2514 #else
2515 mr r3, r0
2516 #endif
2517 rlwinm. r3, r3, 0, 0xffffff
2518 sync
2519 beq 1f /* if nothing pending in the ICP */
2521 /* We found something in the ICP...
2522 *
2523 * If it's not an IPI, stash it in the PACA and return to
2524 * the host, we don't (yet) handle directing real external
2525 * interrupts directly to the guest
2526 */
2527 cmpwi r3, XICS_IPI /* if there is, is it an IPI? */
2528 bne 42f
2530 /* It's an IPI, clear the MFRR and EOI it */
2531 li r3, 0xff
2532 li r8, XICS_MFRR
2533 stbcix r3, r6, r8 /* clear the IPI */
2534 stwcix r0, r6, r7 /* EOI it */
2535 sync
2537 /* We need to re-check host IPI now in case it got set in the
2538 * meantime. If it's clear, we bounce the interrupt to the
2539 * guest
2540 */
2541 lbz r0, HSTATE_HOST_IPI(r13)
2542 cmpwi r0, 0
2543 bne- 43f
2545 /* OK, it's an IPI for us */
2546 li r12, 0
2547 li r3, -1
2548 1: blr
2550 42: /* It's not an IPI and it's for the host. We saved a copy of XIRR in
2551 * the PACA earlier, it will be picked up by the host ICP driver
2552 */
2553 li r3, 1
2554 b 1b
2556 43: /* We raced with the host, we need to resend that IPI, bummer */
2557 li r0, IPI_PRIORITY
2558 stbcix r0, r6, r8 /* set the IPI */
2559 sync
2560 li r3, 1
2561 b 1b
2563 /*
2564 * Save away FP, VMX and VSX registers.
2565 * r3 = vcpu pointer
2566 * N.B. r30 and r31 are volatile across this function,
2567 * thus it is not callable from C.
2568 */
2569 kvmppc_save_fp:
2570 mflr r30
2571 mr r31,r3
2572 mfmsr r5
2573 ori r8,r5,MSR_FP
2574 #ifdef CONFIG_ALTIVEC
2575 BEGIN_FTR_SECTION
2576 oris r8,r8,MSR_VEC@h
2577 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2578 #endif
2579 #ifdef CONFIG_VSX
2580 BEGIN_FTR_SECTION
2581 oris r8,r8,MSR_VSX@h
2582 END_FTR_SECTION_IFSET(CPU_FTR_VSX)
2583 #endif
2584 mtmsrd r8
2585 addi r3,r3,VCPU_FPRS
2586 bl store_fp_state
2587 #ifdef CONFIG_ALTIVEC
2588 BEGIN_FTR_SECTION
2589 addi r3,r31,VCPU_VRS
2590 bl store_vr_state
2591 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2592 #endif
2593 mfspr r6,SPRN_VRSAVE
2594 stw r6,VCPU_VRSAVE(r31)
2595 mtlr r30
2596 blr
2598 /*
2599 * Load up FP, VMX and VSX registers
2600 * r4 = vcpu pointer
2601 * N.B. r30 and r31 are volatile across this function,
2602 * thus it is not callable from C.
2603 */
2604 kvmppc_load_fp:
2605 mflr r30
2606 mr r31,r4
2607 mfmsr r9
2608 ori r8,r9,MSR_FP
2609 #ifdef CONFIG_ALTIVEC
2610 BEGIN_FTR_SECTION
2611 oris r8,r8,MSR_VEC@h
2612 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2613 #endif
2614 #ifdef CONFIG_VSX
2615 BEGIN_FTR_SECTION
2616 oris r8,r8,MSR_VSX@h
2617 END_FTR_SECTION_IFSET(CPU_FTR_VSX)
2618 #endif
2619 mtmsrd r8
2620 addi r3,r4,VCPU_FPRS
2621 bl load_fp_state
2622 #ifdef CONFIG_ALTIVEC
2623 BEGIN_FTR_SECTION
2624 addi r3,r31,VCPU_VRS
2625 bl load_vr_state
2626 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2627 #endif
2628 lwz r7,VCPU_VRSAVE(r31)
2629 mtspr SPRN_VRSAVE,r7
2630 mtlr r30
2631 mr r4,r31
2632 blr
2634 /*
2635 * We come here if we get any exception or interrupt while we are
2636 * executing host real mode code while in guest MMU context.
2637 * For now just spin, but we should do something better.
2638 */
2639 kvmppc_bad_host_intr:
2640 b .
2642 /*
2643 * This mimics the MSR transition on IRQ delivery. The new guest MSR is taken
2644 * from VCPU_INTR_MSR and is modified based on the required TM state changes.
2645 * r11 has the guest MSR value (in/out)
2646 * r9 has a vcpu pointer (in)
2647 * r0 is used as a scratch register
2648 */
2649 kvmppc_msr_interrupt:
2650 rldicl r0, r11, 64 - MSR_TS_S_LG, 62
2651 cmpwi r0, 2 /* Check if we are in transactional state.. */
2652 ld r11, VCPU_INTR_MSR(r9)
2653 bne 1f
2654 /* ... if transactional, change to suspended */
2655 li r0, 1
2656 1: rldimi r11, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG
2657 blr
2659 /*
2660 * This works around a hardware bug on POWER8E processors, where
2661 * writing a 1 to the MMCR0[PMAO] bit doesn't generate a
2662 * performance monitor interrupt. Instead, when we need to have
2663 * an interrupt pending, we have to arrange for a counter to overflow.
2664 */
2665 kvmppc_fix_pmao:
2666 li r3, 0
2667 mtspr SPRN_MMCR2, r3
2668 lis r3, (MMCR0_PMXE | MMCR0_FCECE)@h
2669 ori r3, r3, MMCR0_PMCjCE | MMCR0_C56RUN
2670 mtspr SPRN_MMCR0, r3
2671 lis r3, 0x7fff
2672 ori r3, r3, 0xffff
2673 mtspr SPRN_PMC6, r3
2674 isync
2675 blr
2677 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
2678 /*
2679 * Start timing an activity
2680 * r3 = pointer to time accumulation struct, r4 = vcpu
2681 */
2682 kvmhv_start_timing:
2683 ld r5, HSTATE_KVM_VCORE(r13)
2684 lbz r6, VCORE_IN_GUEST(r5)
2685 cmpwi r6, 0
2686 beq 5f /* if in guest, need to */
2687 ld r6, VCORE_TB_OFFSET(r5) /* subtract timebase offset */
2688 5: mftb r5
2689 subf r5, r6, r5
2690 std r3, VCPU_CUR_ACTIVITY(r4)
2691 std r5, VCPU_ACTIVITY_START(r4)
2692 blr
2694 /*
2695 * Accumulate time to one activity and start another.
2696 * r3 = pointer to new time accumulation struct, r4 = vcpu
2697 */
2698 kvmhv_accumulate_time:
2699 ld r5, HSTATE_KVM_VCORE(r13)
2700 lbz r8, VCORE_IN_GUEST(r5)
2701 cmpwi r8, 0
2702 beq 4f /* if in guest, need to */
2703 ld r8, VCORE_TB_OFFSET(r5) /* subtract timebase offset */
2704 4: ld r5, VCPU_CUR_ACTIVITY(r4)
2705 ld r6, VCPU_ACTIVITY_START(r4)
2706 std r3, VCPU_CUR_ACTIVITY(r4)
2707 mftb r7
2708 subf r7, r8, r7
2709 std r7, VCPU_ACTIVITY_START(r4)
2710 cmpdi r5, 0
2711 beqlr
2712 subf r3, r6, r7
2713 ld r8, TAS_SEQCOUNT(r5)
2714 cmpdi r8, 0
2715 addi r8, r8, 1
2716 std r8, TAS_SEQCOUNT(r5)
2717 lwsync
2718 ld r7, TAS_TOTAL(r5)
2719 add r7, r7, r3
2720 std r7, TAS_TOTAL(r5)
2721 ld r6, TAS_MIN(r5)
2722 ld r7, TAS_MAX(r5)
2723 beq 3f
2724 cmpd r3, r6
2725 bge 1f
2726 3: std r3, TAS_MIN(r5)
2727 1: cmpd r3, r7
2728 ble 2f
2729 std r3, TAS_MAX(r5)
2730 2: lwsync
2731 addi r8, r8, 1
2732 std r8, TAS_SEQCOUNT(r5)
2733 blr
2734 #endif