1 // SPDX-License-Identifier: GPL-2.0-only
4 * Copyright 2010-2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
7 #include <linux/types.h>
8 #include <linux/string.h>
10 #include <linux/kvm_host.h>
11 #include <linux/hugetlb.h>
12 #include <linux/module.h>
13 #include <linux/log2.h>
14 #include <linux/sizes.h>
16 #include <asm/trace.h>
17 #include <asm/kvm_ppc.h>
18 #include <asm/kvm_book3s.h>
19 #include <asm/book3s/64/mmu-hash.h>
20 #include <asm/hvcall.h>
21 #include <asm/synch.h>
22 #include <asm/ppc-opcode.h>
23 #include <asm/pte-walk.h>
25 /* Translate address of a vmalloc'd thing to a linear map address */
26 static void *real_vmalloc_addr(void *x
)
28 unsigned long addr
= (unsigned long) x
;
31 * assume we don't have huge pages in vmalloc space...
32 * So don't worry about THP collapse/split. Called
33 * Only in realmode with MSR_EE = 0, hence won't need irq_save/restore.
35 p
= find_init_mm_pte(addr
, NULL
);
36 if (!p
|| !pte_present(*p
))
38 addr
= (pte_pfn(*p
) << PAGE_SHIFT
) | (addr
& ~PAGE_MASK
);
42 /* Return 1 if we need to do a global tlbie, 0 if we can use tlbiel */
43 static int global_invalidates(struct kvm
*kvm
)
49 * If there is only one vcore, and it's currently running,
50 * as indicated by local_paca->kvm_hstate.kvm_vcpu being set,
51 * we can use tlbiel as long as we mark all other physical
52 * cores as potentially having stale TLB entries for this lpid.
53 * Otherwise, don't use tlbiel.
55 if (kvm
->arch
.online_vcores
== 1 && local_paca
->kvm_hstate
.kvm_vcpu
)
61 /* any other core might now have stale TLB entries... */
63 cpumask_setall(&kvm
->arch
.need_tlb_flush
);
64 cpu
= local_paca
->kvm_hstate
.kvm_vcore
->pcpu
;
66 * On POWER9, threads are independent but the TLB is shared,
67 * so use the bit for the first thread to represent the core.
69 if (cpu_has_feature(CPU_FTR_ARCH_300
))
70 cpu
= cpu_first_thread_sibling(cpu
);
71 cpumask_clear_cpu(cpu
, &kvm
->arch
.need_tlb_flush
);
78 * Add this HPTE into the chain for the real page.
79 * Must be called with the chain locked; it unlocks the chain.
81 void kvmppc_add_revmap_chain(struct kvm
*kvm
, struct revmap_entry
*rev
,
82 unsigned long *rmap
, long pte_index
, int realmode
)
84 struct revmap_entry
*head
, *tail
;
87 if (*rmap
& KVMPPC_RMAP_PRESENT
) {
88 i
= *rmap
& KVMPPC_RMAP_INDEX
;
89 head
= &kvm
->arch
.hpt
.rev
[i
];
91 head
= real_vmalloc_addr(head
);
92 tail
= &kvm
->arch
.hpt
.rev
[head
->back
];
94 tail
= real_vmalloc_addr(tail
);
96 rev
->back
= head
->back
;
97 tail
->forw
= pte_index
;
98 head
->back
= pte_index
;
100 rev
->forw
= rev
->back
= pte_index
;
101 *rmap
= (*rmap
& ~KVMPPC_RMAP_INDEX
) |
102 pte_index
| KVMPPC_RMAP_PRESENT
| KVMPPC_RMAP_HPT
;
106 EXPORT_SYMBOL_GPL(kvmppc_add_revmap_chain
);
108 /* Update the dirty bitmap of a memslot */
109 void kvmppc_update_dirty_map(const struct kvm_memory_slot
*memslot
,
110 unsigned long gfn
, unsigned long psize
)
112 unsigned long npages
;
114 if (!psize
|| !memslot
->dirty_bitmap
)
116 npages
= (psize
+ PAGE_SIZE
- 1) / PAGE_SIZE
;
117 gfn
-= memslot
->base_gfn
;
118 set_dirty_bits_atomic(memslot
->dirty_bitmap
, gfn
, npages
);
120 EXPORT_SYMBOL_GPL(kvmppc_update_dirty_map
);
122 static void kvmppc_set_dirty_from_hpte(struct kvm
*kvm
,
123 unsigned long hpte_v
, unsigned long hpte_gr
)
125 struct kvm_memory_slot
*memslot
;
129 psize
= kvmppc_actual_pgsz(hpte_v
, hpte_gr
);
130 gfn
= hpte_rpn(hpte_gr
, psize
);
131 memslot
= __gfn_to_memslot(kvm_memslots_raw(kvm
), gfn
);
132 if (memslot
&& memslot
->dirty_bitmap
)
133 kvmppc_update_dirty_map(memslot
, gfn
, psize
);
136 /* Returns a pointer to the revmap entry for the page mapped by a HPTE */
137 static unsigned long *revmap_for_hpte(struct kvm
*kvm
, unsigned long hpte_v
,
138 unsigned long hpte_gr
,
139 struct kvm_memory_slot
**memslotp
,
142 struct kvm_memory_slot
*memslot
;
146 gfn
= hpte_rpn(hpte_gr
, kvmppc_actual_pgsz(hpte_v
, hpte_gr
));
147 memslot
= __gfn_to_memslot(kvm_memslots_raw(kvm
), gfn
);
155 rmap
= real_vmalloc_addr(&memslot
->arch
.rmap
[gfn
- memslot
->base_gfn
]);
159 /* Remove this HPTE from the chain for a real page */
160 static void remove_revmap_chain(struct kvm
*kvm
, long pte_index
,
161 struct revmap_entry
*rev
,
162 unsigned long hpte_v
, unsigned long hpte_r
)
164 struct revmap_entry
*next
, *prev
;
165 unsigned long ptel
, head
;
167 unsigned long rcbits
;
168 struct kvm_memory_slot
*memslot
;
171 rcbits
= hpte_r
& (HPTE_R_R
| HPTE_R_C
);
172 ptel
= rev
->guest_rpte
|= rcbits
;
173 rmap
= revmap_for_hpte(kvm
, hpte_v
, ptel
, &memslot
, &gfn
);
178 head
= *rmap
& KVMPPC_RMAP_INDEX
;
179 next
= real_vmalloc_addr(&kvm
->arch
.hpt
.rev
[rev
->forw
]);
180 prev
= real_vmalloc_addr(&kvm
->arch
.hpt
.rev
[rev
->back
]);
181 next
->back
= rev
->back
;
182 prev
->forw
= rev
->forw
;
183 if (head
== pte_index
) {
185 if (head
== pte_index
)
186 *rmap
&= ~(KVMPPC_RMAP_PRESENT
| KVMPPC_RMAP_INDEX
);
188 *rmap
= (*rmap
& ~KVMPPC_RMAP_INDEX
) | head
;
190 *rmap
|= rcbits
<< KVMPPC_RMAP_RC_SHIFT
;
191 if (rcbits
& HPTE_R_C
)
192 kvmppc_update_dirty_map(memslot
, gfn
,
193 kvmppc_actual_pgsz(hpte_v
, hpte_r
));
197 long kvmppc_do_h_enter(struct kvm
*kvm
, unsigned long flags
,
198 long pte_index
, unsigned long pteh
, unsigned long ptel
,
199 pgd_t
*pgdir
, bool realmode
, unsigned long *pte_idx_ret
)
201 unsigned long i
, pa
, gpa
, gfn
, psize
;
202 unsigned long slot_fn
, hva
;
204 struct revmap_entry
*rev
;
205 unsigned long g_ptel
;
206 struct kvm_memory_slot
*memslot
;
207 unsigned hpage_shift
;
211 unsigned int writing
;
212 unsigned long mmu_seq
;
213 unsigned long rcbits
, irq_flags
= 0;
215 if (kvm_is_radix(kvm
))
217 psize
= kvmppc_actual_pgsz(pteh
, ptel
);
220 writing
= hpte_is_writable(ptel
);
221 pteh
&= ~(HPTE_V_HVLOCK
| HPTE_V_ABSENT
| HPTE_V_VALID
);
222 ptel
&= ~HPTE_GR_RESERVED
;
225 /* used later to detect if we might have been invalidated */
226 mmu_seq
= kvm
->mmu_notifier_seq
;
229 /* Find the memslot (if any) for this address */
230 gpa
= (ptel
& HPTE_R_RPN
) & ~(psize
- 1);
231 gfn
= gpa
>> PAGE_SHIFT
;
232 memslot
= __gfn_to_memslot(kvm_memslots_raw(kvm
), gfn
);
236 if (!(memslot
&& !(memslot
->flags
& KVM_MEMSLOT_INVALID
))) {
237 /* Emulated MMIO - mark this with key=31 */
238 pteh
|= HPTE_V_ABSENT
;
239 ptel
|= HPTE_R_KEY_HI
| HPTE_R_KEY_LO
;
243 /* Check if the requested page fits entirely in the memslot. */
244 if (!slot_is_aligned(memslot
, psize
))
246 slot_fn
= gfn
- memslot
->base_gfn
;
247 rmap
= &memslot
->arch
.rmap
[slot_fn
];
249 /* Translate to host virtual address */
250 hva
= __gfn_to_hva_memslot(memslot
, gfn
);
252 * If we had a page table table change after lookup, we would
253 * retry via mmu_notifier_retry.
256 local_irq_save(irq_flags
);
258 * If called in real mode we have MSR_EE = 0. Otherwise
259 * we disable irq above.
261 ptep
= __find_linux_pte(pgdir
, hva
, NULL
, &hpage_shift
);
264 unsigned int host_pte_size
;
267 host_pte_size
= 1ul << hpage_shift
;
269 host_pte_size
= PAGE_SIZE
;
271 * We should always find the guest page size
272 * to <= host page size, if host is using hugepage
274 if (host_pte_size
< psize
) {
276 local_irq_restore(flags
);
279 pte
= kvmppc_read_update_linux_pte(ptep
, writing
);
280 if (pte_present(pte
) && !pte_protnone(pte
)) {
281 if (writing
&& !__pte_write(pte
))
282 /* make the actual HPTE be read-only */
283 ptel
= hpte_make_readonly(ptel
);
285 pa
= pte_pfn(pte
) << PAGE_SHIFT
;
286 pa
|= hva
& (host_pte_size
- 1);
287 pa
|= gpa
& ~PAGE_MASK
;
291 local_irq_restore(irq_flags
);
293 ptel
&= HPTE_R_KEY
| HPTE_R_PP0
| (psize
-1);
297 pteh
|= HPTE_V_VALID
;
299 pteh
|= HPTE_V_ABSENT
;
300 ptel
&= ~(HPTE_R_KEY_HI
| HPTE_R_KEY_LO
);
303 /*If we had host pte mapping then Check WIMG */
304 if (ptep
&& !hpte_cache_flags_ok(ptel
, is_ci
)) {
308 * Allow guest to map emulated device memory as
309 * uncacheable, but actually make it cacheable.
311 ptel
&= ~(HPTE_R_W
|HPTE_R_I
|HPTE_R_G
);
315 /* Find and lock the HPTEG slot to use */
317 if (pte_index
>= kvmppc_hpt_npte(&kvm
->arch
.hpt
))
319 if (likely((flags
& H_EXACT
) == 0)) {
321 hpte
= (__be64
*)(kvm
->arch
.hpt
.virt
+ (pte_index
<< 4));
322 for (i
= 0; i
< 8; ++i
) {
323 if ((be64_to_cpu(*hpte
) & HPTE_V_VALID
) == 0 &&
324 try_lock_hpte(hpte
, HPTE_V_HVLOCK
| HPTE_V_VALID
|
331 * Since try_lock_hpte doesn't retry (not even stdcx.
332 * failures), it could be that there is a free slot
333 * but we transiently failed to lock it. Try again,
334 * actually locking each slot and checking it.
337 for (i
= 0; i
< 8; ++i
) {
339 while (!try_lock_hpte(hpte
, HPTE_V_HVLOCK
))
341 pte
= be64_to_cpu(hpte
[0]);
342 if (!(pte
& (HPTE_V_VALID
| HPTE_V_ABSENT
)))
344 __unlock_hpte(hpte
, pte
);
352 hpte
= (__be64
*)(kvm
->arch
.hpt
.virt
+ (pte_index
<< 4));
353 if (!try_lock_hpte(hpte
, HPTE_V_HVLOCK
| HPTE_V_VALID
|
355 /* Lock the slot and check again */
358 while (!try_lock_hpte(hpte
, HPTE_V_HVLOCK
))
360 pte
= be64_to_cpu(hpte
[0]);
361 if (pte
& (HPTE_V_VALID
| HPTE_V_ABSENT
)) {
362 __unlock_hpte(hpte
, pte
);
368 /* Save away the guest's idea of the second HPTE dword */
369 rev
= &kvm
->arch
.hpt
.rev
[pte_index
];
371 rev
= real_vmalloc_addr(rev
);
373 rev
->guest_rpte
= g_ptel
;
374 note_hpte_modification(kvm
, rev
);
377 /* Link HPTE into reverse-map chain */
378 if (pteh
& HPTE_V_VALID
) {
380 rmap
= real_vmalloc_addr(rmap
);
382 /* Check for pending invalidations under the rmap chain lock */
383 if (mmu_notifier_retry(kvm
, mmu_seq
)) {
384 /* inval in progress, write a non-present HPTE */
385 pteh
|= HPTE_V_ABSENT
;
386 pteh
&= ~HPTE_V_VALID
;
387 ptel
&= ~(HPTE_R_KEY_HI
| HPTE_R_KEY_LO
);
390 kvmppc_add_revmap_chain(kvm
, rev
, rmap
, pte_index
,
392 /* Only set R/C in real HPTE if already set in *rmap */
393 rcbits
= *rmap
>> KVMPPC_RMAP_RC_SHIFT
;
394 ptel
&= rcbits
| ~(HPTE_R_R
| HPTE_R_C
);
398 /* Convert to new format on P9 */
399 if (cpu_has_feature(CPU_FTR_ARCH_300
)) {
400 ptel
= hpte_old_to_new_r(pteh
, ptel
);
401 pteh
= hpte_old_to_new_v(pteh
);
403 hpte
[1] = cpu_to_be64(ptel
);
405 /* Write the first HPTE dword, unlocking the HPTE and making it valid */
407 __unlock_hpte(hpte
, pteh
);
408 asm volatile("ptesync" : : : "memory");
410 *pte_idx_ret
= pte_index
;
413 EXPORT_SYMBOL_GPL(kvmppc_do_h_enter
);
415 long kvmppc_h_enter(struct kvm_vcpu
*vcpu
, unsigned long flags
,
416 long pte_index
, unsigned long pteh
, unsigned long ptel
)
418 return kvmppc_do_h_enter(vcpu
->kvm
, flags
, pte_index
, pteh
, ptel
,
419 vcpu
->arch
.pgdir
, true,
420 &vcpu
->arch
.regs
.gpr
[4]);
423 #ifdef __BIG_ENDIAN__
424 #define LOCK_TOKEN (*(u32 *)(&get_paca()->lock_token))
426 #define LOCK_TOKEN (*(u32 *)(&get_paca()->paca_index))
429 static inline int is_mmio_hpte(unsigned long v
, unsigned long r
)
431 return ((v
& HPTE_V_ABSENT
) &&
432 (r
& (HPTE_R_KEY_HI
| HPTE_R_KEY_LO
)) ==
433 (HPTE_R_KEY_HI
| HPTE_R_KEY_LO
));
436 static inline void fixup_tlbie_lpid(unsigned long rb_value
, unsigned long lpid
)
439 if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG
)) {
440 /* Radix flush for a hash guest */
442 unsigned long rb
,rs
,prs
,r
,ric
;
444 rb
= PPC_BIT(52); /* IS = 2 */
445 rs
= 0; /* lpid = 0 */
446 prs
= 0; /* partition scoped */
447 r
= 1; /* radix format */
448 ric
= 0; /* RIC_FLSUH_TLB */
451 * Need the extra ptesync to make sure we don't
454 asm volatile("ptesync": : :"memory");
455 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
456 : : "r"(rb
), "i"(r
), "i"(prs
),
457 "i"(ric
), "r"(rs
) : "memory");
460 if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG
)) {
461 asm volatile("ptesync": : :"memory");
462 asm volatile(PPC_TLBIE_5(%0,%1,0,0,0) : :
463 "r" (rb_value
), "r" (lpid
));
467 static void do_tlbies(struct kvm
*kvm
, unsigned long *rbvalues
,
468 long npages
, int global
, bool need_sync
)
473 * We use the POWER9 5-operand versions of tlbie and tlbiel here.
474 * Since we are using RIC=0 PRS=0 R=0, and P7/P8 tlbiel ignores
475 * the RS field, this is backwards-compatible with P7 and P8.
479 asm volatile("ptesync" : : : "memory");
480 for (i
= 0; i
< npages
; ++i
) {
481 asm volatile(PPC_TLBIE_5(%0,%1,0,0,0) : :
482 "r" (rbvalues
[i
]), "r" (kvm
->arch
.lpid
));
485 fixup_tlbie_lpid(rbvalues
[i
- 1], kvm
->arch
.lpid
);
486 asm volatile("eieio; tlbsync; ptesync" : : : "memory");
489 asm volatile("ptesync" : : : "memory");
490 for (i
= 0; i
< npages
; ++i
) {
491 asm volatile(PPC_TLBIEL(%0,%1,0,0,0) : :
492 "r" (rbvalues
[i
]), "r" (0));
494 asm volatile("ptesync" : : : "memory");
498 long kvmppc_do_h_remove(struct kvm
*kvm
, unsigned long flags
,
499 unsigned long pte_index
, unsigned long avpn
,
500 unsigned long *hpret
)
503 unsigned long v
, r
, rb
;
504 struct revmap_entry
*rev
;
505 u64 pte
, orig_pte
, pte_r
;
507 if (kvm_is_radix(kvm
))
509 if (pte_index
>= kvmppc_hpt_npte(&kvm
->arch
.hpt
))
511 hpte
= (__be64
*)(kvm
->arch
.hpt
.virt
+ (pte_index
<< 4));
512 while (!try_lock_hpte(hpte
, HPTE_V_HVLOCK
))
514 pte
= orig_pte
= be64_to_cpu(hpte
[0]);
515 pte_r
= be64_to_cpu(hpte
[1]);
516 if (cpu_has_feature(CPU_FTR_ARCH_300
)) {
517 pte
= hpte_new_to_old_v(pte
, pte_r
);
518 pte_r
= hpte_new_to_old_r(pte_r
);
520 if ((pte
& (HPTE_V_ABSENT
| HPTE_V_VALID
)) == 0 ||
521 ((flags
& H_AVPN
) && (pte
& ~0x7fUL
) != avpn
) ||
522 ((flags
& H_ANDCOND
) && (pte
& avpn
) != 0)) {
523 __unlock_hpte(hpte
, orig_pte
);
527 rev
= real_vmalloc_addr(&kvm
->arch
.hpt
.rev
[pte_index
]);
528 v
= pte
& ~HPTE_V_HVLOCK
;
529 if (v
& HPTE_V_VALID
) {
530 hpte
[0] &= ~cpu_to_be64(HPTE_V_VALID
);
531 rb
= compute_tlbie_rb(v
, pte_r
, pte_index
);
532 do_tlbies(kvm
, &rb
, 1, global_invalidates(kvm
), true);
534 * The reference (R) and change (C) bits in a HPT
535 * entry can be set by hardware at any time up until
536 * the HPTE is invalidated and the TLB invalidation
537 * sequence has completed. This means that when
538 * removing a HPTE, we need to re-read the HPTE after
539 * the invalidation sequence has completed in order to
540 * obtain reliable values of R and C.
542 remove_revmap_chain(kvm
, pte_index
, rev
, v
,
543 be64_to_cpu(hpte
[1]));
545 r
= rev
->guest_rpte
& ~HPTE_GR_RESERVED
;
546 note_hpte_modification(kvm
, rev
);
547 unlock_hpte(hpte
, 0);
549 if (is_mmio_hpte(v
, pte_r
))
550 atomic64_inc(&kvm
->arch
.mmio_update
);
552 if (v
& HPTE_V_ABSENT
)
553 v
= (v
& ~HPTE_V_ABSENT
) | HPTE_V_VALID
;
558 EXPORT_SYMBOL_GPL(kvmppc_do_h_remove
);
560 long kvmppc_h_remove(struct kvm_vcpu
*vcpu
, unsigned long flags
,
561 unsigned long pte_index
, unsigned long avpn
)
563 return kvmppc_do_h_remove(vcpu
->kvm
, flags
, pte_index
, avpn
,
564 &vcpu
->arch
.regs
.gpr
[4]);
567 long kvmppc_h_bulk_remove(struct kvm_vcpu
*vcpu
)
569 struct kvm
*kvm
= vcpu
->kvm
;
570 unsigned long *args
= &vcpu
->arch
.regs
.gpr
[4];
571 __be64
*hp
, *hptes
[4];
572 unsigned long tlbrb
[4];
573 long int i
, j
, k
, n
, found
, indexes
[4];
574 unsigned long flags
, req
, pte_index
, rcbits
;
576 long int ret
= H_SUCCESS
;
577 struct revmap_entry
*rev
, *revs
[4];
580 if (kvm_is_radix(kvm
))
582 global
= global_invalidates(kvm
);
583 for (i
= 0; i
< 4 && ret
== H_SUCCESS
; ) {
588 flags
= pte_index
>> 56;
589 pte_index
&= ((1ul << 56) - 1);
592 if (req
== 3) { /* no more requests */
596 if (req
!= 1 || flags
== 3 ||
597 pte_index
>= kvmppc_hpt_npte(&kvm
->arch
.hpt
)) {
598 /* parameter error */
599 args
[j
] = ((0xa0 | flags
) << 56) + pte_index
;
603 hp
= (__be64
*) (kvm
->arch
.hpt
.virt
+ (pte_index
<< 4));
604 /* to avoid deadlock, don't spin except for first */
605 if (!try_lock_hpte(hp
, HPTE_V_HVLOCK
)) {
608 while (!try_lock_hpte(hp
, HPTE_V_HVLOCK
))
612 hp0
= be64_to_cpu(hp
[0]);
613 hp1
= be64_to_cpu(hp
[1]);
614 if (cpu_has_feature(CPU_FTR_ARCH_300
)) {
615 hp0
= hpte_new_to_old_v(hp0
, hp1
);
616 hp1
= hpte_new_to_old_r(hp1
);
618 if (hp0
& (HPTE_V_ABSENT
| HPTE_V_VALID
)) {
620 case 0: /* absolute */
623 case 1: /* andcond */
624 if (!(hp0
& args
[j
+ 1]))
628 if ((hp0
& ~0x7fUL
) == args
[j
+ 1])
634 hp
[0] &= ~cpu_to_be64(HPTE_V_HVLOCK
);
635 args
[j
] = ((0x90 | flags
) << 56) + pte_index
;
639 args
[j
] = ((0x80 | flags
) << 56) + pte_index
;
640 rev
= real_vmalloc_addr(&kvm
->arch
.hpt
.rev
[pte_index
]);
641 note_hpte_modification(kvm
, rev
);
643 if (!(hp0
& HPTE_V_VALID
)) {
644 /* insert R and C bits from PTE */
645 rcbits
= rev
->guest_rpte
& (HPTE_R_R
|HPTE_R_C
);
646 args
[j
] |= rcbits
<< (56 - 5);
648 if (is_mmio_hpte(hp0
, hp1
))
649 atomic64_inc(&kvm
->arch
.mmio_update
);
653 /* leave it locked */
654 hp
[0] &= ~cpu_to_be64(HPTE_V_VALID
);
655 tlbrb
[n
] = compute_tlbie_rb(hp0
, hp1
, pte_index
);
665 /* Now that we've collected a batch, do the tlbies */
666 do_tlbies(kvm
, tlbrb
, n
, global
, true);
668 /* Read PTE low words after tlbie to get final R/C values */
669 for (k
= 0; k
< n
; ++k
) {
671 pte_index
= args
[j
] & ((1ul << 56) - 1);
674 remove_revmap_chain(kvm
, pte_index
, rev
,
675 be64_to_cpu(hp
[0]), be64_to_cpu(hp
[1]));
676 rcbits
= rev
->guest_rpte
& (HPTE_R_R
|HPTE_R_C
);
677 args
[j
] |= rcbits
<< (56 - 5);
678 __unlock_hpte(hp
, 0);
685 long kvmppc_h_protect(struct kvm_vcpu
*vcpu
, unsigned long flags
,
686 unsigned long pte_index
, unsigned long avpn
,
689 struct kvm
*kvm
= vcpu
->kvm
;
691 struct revmap_entry
*rev
;
692 unsigned long v
, r
, rb
, mask
, bits
;
695 if (kvm_is_radix(kvm
))
697 if (pte_index
>= kvmppc_hpt_npte(&kvm
->arch
.hpt
))
700 hpte
= (__be64
*)(kvm
->arch
.hpt
.virt
+ (pte_index
<< 4));
701 while (!try_lock_hpte(hpte
, HPTE_V_HVLOCK
))
703 v
= pte_v
= be64_to_cpu(hpte
[0]);
704 if (cpu_has_feature(CPU_FTR_ARCH_300
))
705 v
= hpte_new_to_old_v(v
, be64_to_cpu(hpte
[1]));
706 if ((v
& (HPTE_V_ABSENT
| HPTE_V_VALID
)) == 0 ||
707 ((flags
& H_AVPN
) && (v
& ~0x7fUL
) != avpn
)) {
708 __unlock_hpte(hpte
, pte_v
);
712 pte_r
= be64_to_cpu(hpte
[1]);
713 bits
= (flags
<< 55) & HPTE_R_PP0
;
714 bits
|= (flags
<< 48) & HPTE_R_KEY_HI
;
715 bits
|= flags
& (HPTE_R_PP
| HPTE_R_N
| HPTE_R_KEY_LO
);
717 /* Update guest view of 2nd HPTE dword */
718 mask
= HPTE_R_PP0
| HPTE_R_PP
| HPTE_R_N
|
719 HPTE_R_KEY_HI
| HPTE_R_KEY_LO
;
720 rev
= real_vmalloc_addr(&kvm
->arch
.hpt
.rev
[pte_index
]);
722 r
= (rev
->guest_rpte
& ~mask
) | bits
;
724 note_hpte_modification(kvm
, rev
);
728 if (v
& HPTE_V_VALID
) {
730 * If the page is valid, don't let it transition from
731 * readonly to writable. If it should be writable, we'll
732 * take a trap and let the page fault code sort it out.
734 r
= (pte_r
& ~mask
) | bits
;
735 if (hpte_is_writable(r
) && !hpte_is_writable(pte_r
))
736 r
= hpte_make_readonly(r
);
737 /* If the PTE is changing, invalidate it first */
739 rb
= compute_tlbie_rb(v
, r
, pte_index
);
740 hpte
[0] = cpu_to_be64((pte_v
& ~HPTE_V_VALID
) |
742 do_tlbies(kvm
, &rb
, 1, global_invalidates(kvm
), true);
743 /* Don't lose R/C bit updates done by hardware */
744 r
|= be64_to_cpu(hpte
[1]) & (HPTE_R_R
| HPTE_R_C
);
745 hpte
[1] = cpu_to_be64(r
);
748 unlock_hpte(hpte
, pte_v
& ~HPTE_V_HVLOCK
);
749 asm volatile("ptesync" : : : "memory");
750 if (is_mmio_hpte(v
, pte_r
))
751 atomic64_inc(&kvm
->arch
.mmio_update
);
756 long kvmppc_h_read(struct kvm_vcpu
*vcpu
, unsigned long flags
,
757 unsigned long pte_index
)
759 struct kvm
*kvm
= vcpu
->kvm
;
763 struct revmap_entry
*rev
= NULL
;
765 if (kvm_is_radix(kvm
))
767 if (pte_index
>= kvmppc_hpt_npte(&kvm
->arch
.hpt
))
769 if (flags
& H_READ_4
) {
773 rev
= real_vmalloc_addr(&kvm
->arch
.hpt
.rev
[pte_index
]);
774 for (i
= 0; i
< n
; ++i
, ++pte_index
) {
775 hpte
= (__be64
*)(kvm
->arch
.hpt
.virt
+ (pte_index
<< 4));
776 v
= be64_to_cpu(hpte
[0]) & ~HPTE_V_HVLOCK
;
777 r
= be64_to_cpu(hpte
[1]);
778 if (cpu_has_feature(CPU_FTR_ARCH_300
)) {
779 v
= hpte_new_to_old_v(v
, r
);
780 r
= hpte_new_to_old_r(r
);
782 if (v
& HPTE_V_ABSENT
) {
786 if (v
& HPTE_V_VALID
) {
787 r
= rev
[i
].guest_rpte
| (r
& (HPTE_R_R
| HPTE_R_C
));
788 r
&= ~HPTE_GR_RESERVED
;
790 vcpu
->arch
.regs
.gpr
[4 + i
* 2] = v
;
791 vcpu
->arch
.regs
.gpr
[5 + i
* 2] = r
;
796 long kvmppc_h_clear_ref(struct kvm_vcpu
*vcpu
, unsigned long flags
,
797 unsigned long pte_index
)
799 struct kvm
*kvm
= vcpu
->kvm
;
801 unsigned long v
, r
, gr
;
802 struct revmap_entry
*rev
;
804 long ret
= H_NOT_FOUND
;
806 if (kvm_is_radix(kvm
))
808 if (pte_index
>= kvmppc_hpt_npte(&kvm
->arch
.hpt
))
811 rev
= real_vmalloc_addr(&kvm
->arch
.hpt
.rev
[pte_index
]);
812 hpte
= (__be64
*)(kvm
->arch
.hpt
.virt
+ (pte_index
<< 4));
813 while (!try_lock_hpte(hpte
, HPTE_V_HVLOCK
))
815 v
= be64_to_cpu(hpte
[0]);
816 r
= be64_to_cpu(hpte
[1]);
817 if (!(v
& (HPTE_V_VALID
| HPTE_V_ABSENT
)))
820 gr
= rev
->guest_rpte
;
821 if (rev
->guest_rpte
& HPTE_R_R
) {
822 rev
->guest_rpte
&= ~HPTE_R_R
;
823 note_hpte_modification(kvm
, rev
);
825 if (v
& HPTE_V_VALID
) {
826 gr
|= r
& (HPTE_R_R
| HPTE_R_C
);
828 kvmppc_clear_ref_hpte(kvm
, hpte
, pte_index
);
829 rmap
= revmap_for_hpte(kvm
, v
, gr
, NULL
, NULL
);
832 *rmap
|= KVMPPC_RMAP_REFERENCED
;
837 vcpu
->arch
.regs
.gpr
[4] = gr
;
840 unlock_hpte(hpte
, v
& ~HPTE_V_HVLOCK
);
844 long kvmppc_h_clear_mod(struct kvm_vcpu
*vcpu
, unsigned long flags
,
845 unsigned long pte_index
)
847 struct kvm
*kvm
= vcpu
->kvm
;
849 unsigned long v
, r
, gr
;
850 struct revmap_entry
*rev
;
851 long ret
= H_NOT_FOUND
;
853 if (kvm_is_radix(kvm
))
855 if (pte_index
>= kvmppc_hpt_npte(&kvm
->arch
.hpt
))
858 rev
= real_vmalloc_addr(&kvm
->arch
.hpt
.rev
[pte_index
]);
859 hpte
= (__be64
*)(kvm
->arch
.hpt
.virt
+ (pte_index
<< 4));
860 while (!try_lock_hpte(hpte
, HPTE_V_HVLOCK
))
862 v
= be64_to_cpu(hpte
[0]);
863 r
= be64_to_cpu(hpte
[1]);
864 if (!(v
& (HPTE_V_VALID
| HPTE_V_ABSENT
)))
867 gr
= rev
->guest_rpte
;
869 rev
->guest_rpte
&= ~HPTE_R_C
;
870 note_hpte_modification(kvm
, rev
);
872 if (v
& HPTE_V_VALID
) {
873 /* need to make it temporarily absent so C is stable */
874 hpte
[0] |= cpu_to_be64(HPTE_V_ABSENT
);
875 kvmppc_invalidate_hpte(kvm
, hpte
, pte_index
);
876 r
= be64_to_cpu(hpte
[1]);
877 gr
|= r
& (HPTE_R_R
| HPTE_R_C
);
879 hpte
[1] = cpu_to_be64(r
& ~HPTE_R_C
);
881 kvmppc_set_dirty_from_hpte(kvm
, v
, gr
);
884 vcpu
->arch
.regs
.gpr
[4] = gr
;
887 unlock_hpte(hpte
, v
& ~HPTE_V_HVLOCK
);
891 static int kvmppc_get_hpa(struct kvm_vcpu
*vcpu
, unsigned long gpa
,
892 int writing
, unsigned long *hpa
,
893 struct kvm_memory_slot
**memslot_p
)
895 struct kvm
*kvm
= vcpu
->kvm
;
896 struct kvm_memory_slot
*memslot
;
897 unsigned long gfn
, hva
, pa
, psize
= PAGE_SHIFT
;
901 /* Find the memslot for this address */
902 gfn
= gpa
>> PAGE_SHIFT
;
903 memslot
= __gfn_to_memslot(kvm_memslots_raw(kvm
), gfn
);
904 if (!memslot
|| (memslot
->flags
& KVM_MEMSLOT_INVALID
))
907 /* Translate to host virtual address */
908 hva
= __gfn_to_hva_memslot(memslot
, gfn
);
910 /* Try to find the host pte for that virtual address */
911 ptep
= __find_linux_pte(vcpu
->arch
.pgdir
, hva
, NULL
, &shift
);
914 pte
= kvmppc_read_update_linux_pte(ptep
, writing
);
915 if (!pte_present(pte
))
918 /* Convert to a physical address */
920 psize
= 1UL << shift
;
921 pa
= pte_pfn(pte
) << PAGE_SHIFT
;
922 pa
|= hva
& (psize
- 1);
923 pa
|= gpa
& ~PAGE_MASK
;
928 *memslot_p
= memslot
;
933 static long kvmppc_do_h_page_init_zero(struct kvm_vcpu
*vcpu
,
936 struct kvm_memory_slot
*memslot
;
937 struct kvm
*kvm
= vcpu
->kvm
;
938 unsigned long pa
, mmu_seq
;
939 long ret
= H_SUCCESS
;
942 /* Used later to detect if we might have been invalidated */
943 mmu_seq
= kvm
->mmu_notifier_seq
;
946 ret
= kvmppc_get_hpa(vcpu
, dest
, 1, &pa
, &memslot
);
947 if (ret
!= H_SUCCESS
)
950 /* Check if we've been invalidated */
951 raw_spin_lock(&kvm
->mmu_lock
.rlock
);
952 if (mmu_notifier_retry(kvm
, mmu_seq
)) {
958 for (i
= 0; i
< SZ_4K
; i
+= L1_CACHE_BYTES
, pa
+= L1_CACHE_BYTES
)
960 kvmppc_update_dirty_map(memslot
, dest
>> PAGE_SHIFT
, PAGE_SIZE
);
963 raw_spin_unlock(&kvm
->mmu_lock
.rlock
);
967 static long kvmppc_do_h_page_init_copy(struct kvm_vcpu
*vcpu
,
968 unsigned long dest
, unsigned long src
)
970 unsigned long dest_pa
, src_pa
, mmu_seq
;
971 struct kvm_memory_slot
*dest_memslot
;
972 struct kvm
*kvm
= vcpu
->kvm
;
973 long ret
= H_SUCCESS
;
975 /* Used later to detect if we might have been invalidated */
976 mmu_seq
= kvm
->mmu_notifier_seq
;
979 ret
= kvmppc_get_hpa(vcpu
, dest
, 1, &dest_pa
, &dest_memslot
);
980 if (ret
!= H_SUCCESS
)
982 ret
= kvmppc_get_hpa(vcpu
, src
, 0, &src_pa
, NULL
);
983 if (ret
!= H_SUCCESS
)
986 /* Check if we've been invalidated */
987 raw_spin_lock(&kvm
->mmu_lock
.rlock
);
988 if (mmu_notifier_retry(kvm
, mmu_seq
)) {
994 memcpy((void *)dest_pa
, (void *)src_pa
, SZ_4K
);
996 kvmppc_update_dirty_map(dest_memslot
, dest
>> PAGE_SHIFT
, PAGE_SIZE
);
999 raw_spin_unlock(&kvm
->mmu_lock
.rlock
);
1003 long kvmppc_rm_h_page_init(struct kvm_vcpu
*vcpu
, unsigned long flags
,
1004 unsigned long dest
, unsigned long src
)
1006 struct kvm
*kvm
= vcpu
->kvm
;
1007 u64 pg_mask
= SZ_4K
- 1; /* 4K page size */
1008 long ret
= H_SUCCESS
;
1010 /* Don't handle radix mode here, go up to the virtual mode handler */
1011 if (kvm_is_radix(kvm
))
1014 /* Check for invalid flags (H_PAGE_SET_LOANED covers all CMO flags) */
1015 if (flags
& ~(H_ICACHE_INVALIDATE
| H_ICACHE_SYNCHRONIZE
|
1016 H_ZERO_PAGE
| H_COPY_PAGE
| H_PAGE_SET_LOANED
))
1019 /* dest (and src if copy_page flag set) must be page aligned */
1020 if ((dest
& pg_mask
) || ((flags
& H_COPY_PAGE
) && (src
& pg_mask
)))
1023 /* zero and/or copy the page as determined by the flags */
1024 if (flags
& H_COPY_PAGE
)
1025 ret
= kvmppc_do_h_page_init_copy(vcpu
, dest
, src
);
1026 else if (flags
& H_ZERO_PAGE
)
1027 ret
= kvmppc_do_h_page_init_zero(vcpu
, dest
);
1029 /* We can ignore the other flags */
1034 void kvmppc_invalidate_hpte(struct kvm
*kvm
, __be64
*hptep
,
1035 unsigned long pte_index
)
1040 hptep
[0] &= ~cpu_to_be64(HPTE_V_VALID
);
1041 hp0
= be64_to_cpu(hptep
[0]);
1042 hp1
= be64_to_cpu(hptep
[1]);
1043 if (cpu_has_feature(CPU_FTR_ARCH_300
)) {
1044 hp0
= hpte_new_to_old_v(hp0
, hp1
);
1045 hp1
= hpte_new_to_old_r(hp1
);
1047 rb
= compute_tlbie_rb(hp0
, hp1
, pte_index
);
1048 do_tlbies(kvm
, &rb
, 1, 1, true);
1050 EXPORT_SYMBOL_GPL(kvmppc_invalidate_hpte
);
1052 void kvmppc_clear_ref_hpte(struct kvm
*kvm
, __be64
*hptep
,
1053 unsigned long pte_index
)
1056 unsigned char rbyte
;
1059 hp0
= be64_to_cpu(hptep
[0]);
1060 hp1
= be64_to_cpu(hptep
[1]);
1061 if (cpu_has_feature(CPU_FTR_ARCH_300
)) {
1062 hp0
= hpte_new_to_old_v(hp0
, hp1
);
1063 hp1
= hpte_new_to_old_r(hp1
);
1065 rb
= compute_tlbie_rb(hp0
, hp1
, pte_index
);
1066 rbyte
= (be64_to_cpu(hptep
[1]) & ~HPTE_R_R
) >> 8;
1067 /* modify only the second-last byte, which contains the ref bit */
1068 *((char *)hptep
+ 14) = rbyte
;
1069 do_tlbies(kvm
, &rb
, 1, 1, false);
1071 EXPORT_SYMBOL_GPL(kvmppc_clear_ref_hpte
);
1073 static int slb_base_page_shift
[4] = {
1077 20, /* 1M, unsupported */
1080 static struct mmio_hpte_cache_entry
*mmio_cache_search(struct kvm_vcpu
*vcpu
,
1081 unsigned long eaddr
, unsigned long slb_v
, long mmio_update
)
1083 struct mmio_hpte_cache_entry
*entry
= NULL
;
1084 unsigned int pshift
;
1087 for (i
= 0; i
< MMIO_HPTE_CACHE_SIZE
; i
++) {
1088 entry
= &vcpu
->arch
.mmio_cache
.entry
[i
];
1089 if (entry
->mmio_update
== mmio_update
) {
1090 pshift
= entry
->slb_base_pshift
;
1091 if ((entry
->eaddr
>> pshift
) == (eaddr
>> pshift
) &&
1092 entry
->slb_v
== slb_v
)
1099 static struct mmio_hpte_cache_entry
*
1100 next_mmio_cache_entry(struct kvm_vcpu
*vcpu
)
1102 unsigned int index
= vcpu
->arch
.mmio_cache
.index
;
1104 vcpu
->arch
.mmio_cache
.index
++;
1105 if (vcpu
->arch
.mmio_cache
.index
== MMIO_HPTE_CACHE_SIZE
)
1106 vcpu
->arch
.mmio_cache
.index
= 0;
1108 return &vcpu
->arch
.mmio_cache
.entry
[index
];
1111 /* When called from virtmode, this func should be protected by
1112 * preempt_disable(), otherwise, the holding of HPTE_V_HVLOCK
1113 * can trigger deadlock issue.
1115 long kvmppc_hv_find_lock_hpte(struct kvm
*kvm
, gva_t eaddr
, unsigned long slb_v
,
1116 unsigned long valid
)
1119 unsigned int pshift
;
1120 unsigned long somask
;
1121 unsigned long vsid
, hash
;
1124 unsigned long mask
, val
;
1125 unsigned long v
, r
, orig_v
;
1127 /* Get page shift, work out hash and AVPN etc. */
1128 mask
= SLB_VSID_B
| HPTE_V_AVPN
| HPTE_V_SECONDARY
;
1131 if (slb_v
& SLB_VSID_L
) {
1132 mask
|= HPTE_V_LARGE
;
1133 val
|= HPTE_V_LARGE
;
1134 pshift
= slb_base_page_shift
[(slb_v
& SLB_VSID_LP
) >> 4];
1136 if (slb_v
& SLB_VSID_B_1T
) {
1137 somask
= (1UL << 40) - 1;
1138 vsid
= (slb_v
& ~SLB_VSID_B
) >> SLB_VSID_SHIFT_1T
;
1141 somask
= (1UL << 28) - 1;
1142 vsid
= (slb_v
& ~SLB_VSID_B
) >> SLB_VSID_SHIFT
;
1144 hash
= (vsid
^ ((eaddr
& somask
) >> pshift
)) & kvmppc_hpt_mask(&kvm
->arch
.hpt
);
1145 avpn
= slb_v
& ~(somask
>> 16); /* also includes B */
1146 avpn
|= (eaddr
& somask
) >> 16;
1149 avpn
&= ~((1UL << (pshift
- 16)) - 1);
1155 hpte
= (__be64
*)(kvm
->arch
.hpt
.virt
+ (hash
<< 7));
1157 for (i
= 0; i
< 16; i
+= 2) {
1158 /* Read the PTE racily */
1159 v
= be64_to_cpu(hpte
[i
]) & ~HPTE_V_HVLOCK
;
1160 if (cpu_has_feature(CPU_FTR_ARCH_300
))
1161 v
= hpte_new_to_old_v(v
, be64_to_cpu(hpte
[i
+1]));
1163 /* Check valid/absent, hash, segment size and AVPN */
1164 if (!(v
& valid
) || (v
& mask
) != val
)
1167 /* Lock the PTE and read it under the lock */
1168 while (!try_lock_hpte(&hpte
[i
], HPTE_V_HVLOCK
))
1170 v
= orig_v
= be64_to_cpu(hpte
[i
]) & ~HPTE_V_HVLOCK
;
1171 r
= be64_to_cpu(hpte
[i
+1]);
1172 if (cpu_has_feature(CPU_FTR_ARCH_300
)) {
1173 v
= hpte_new_to_old_v(v
, r
);
1174 r
= hpte_new_to_old_r(r
);
1178 * Check the HPTE again, including base page size
1180 if ((v
& valid
) && (v
& mask
) == val
&&
1181 kvmppc_hpte_base_page_shift(v
, r
) == pshift
)
1182 /* Return with the HPTE still locked */
1183 return (hash
<< 3) + (i
>> 1);
1185 __unlock_hpte(&hpte
[i
], orig_v
);
1188 if (val
& HPTE_V_SECONDARY
)
1190 val
|= HPTE_V_SECONDARY
;
1191 hash
= hash
^ kvmppc_hpt_mask(&kvm
->arch
.hpt
);
1195 EXPORT_SYMBOL(kvmppc_hv_find_lock_hpte
);
1198 * Called in real mode to check whether an HPTE not found fault
1199 * is due to accessing a paged-out page or an emulated MMIO page,
1200 * or if a protection fault is due to accessing a page that the
1201 * guest wanted read/write access to but which we made read-only.
1202 * Returns a possibly modified status (DSISR) value if not
1203 * (i.e. pass the interrupt to the guest),
1204 * -1 to pass the fault up to host kernel mode code, -2 to do that
1205 * and also load the instruction word (for MMIO emulation),
1206 * or 0 if we should make the guest retry the access.
1208 long kvmppc_hpte_hv_fault(struct kvm_vcpu
*vcpu
, unsigned long addr
,
1209 unsigned long slb_v
, unsigned int status
, bool data
)
1211 struct kvm
*kvm
= vcpu
->kvm
;
1213 unsigned long v
, r
, gr
, orig_v
;
1215 unsigned long valid
;
1216 struct revmap_entry
*rev
;
1217 unsigned long pp
, key
;
1218 struct mmio_hpte_cache_entry
*cache_entry
= NULL
;
1219 long mmio_update
= 0;
1221 /* For protection fault, expect to find a valid HPTE */
1222 valid
= HPTE_V_VALID
;
1223 if (status
& DSISR_NOHPTE
) {
1224 valid
|= HPTE_V_ABSENT
;
1225 mmio_update
= atomic64_read(&kvm
->arch
.mmio_update
);
1226 cache_entry
= mmio_cache_search(vcpu
, addr
, slb_v
, mmio_update
);
1229 index
= cache_entry
->pte_index
;
1230 v
= cache_entry
->hpte_v
;
1231 r
= cache_entry
->hpte_r
;
1232 gr
= cache_entry
->rpte
;
1234 index
= kvmppc_hv_find_lock_hpte(kvm
, addr
, slb_v
, valid
);
1236 if (status
& DSISR_NOHPTE
)
1237 return status
; /* there really was no HPTE */
1238 return 0; /* for prot fault, HPTE disappeared */
1240 hpte
= (__be64
*)(kvm
->arch
.hpt
.virt
+ (index
<< 4));
1241 v
= orig_v
= be64_to_cpu(hpte
[0]) & ~HPTE_V_HVLOCK
;
1242 r
= be64_to_cpu(hpte
[1]);
1243 if (cpu_has_feature(CPU_FTR_ARCH_300
)) {
1244 v
= hpte_new_to_old_v(v
, r
);
1245 r
= hpte_new_to_old_r(r
);
1247 rev
= real_vmalloc_addr(&kvm
->arch
.hpt
.rev
[index
]);
1248 gr
= rev
->guest_rpte
;
1250 unlock_hpte(hpte
, orig_v
);
1253 /* For not found, if the HPTE is valid by now, retry the instruction */
1254 if ((status
& DSISR_NOHPTE
) && (v
& HPTE_V_VALID
))
1257 /* Check access permissions to the page */
1258 pp
= gr
& (HPTE_R_PP0
| HPTE_R_PP
);
1259 key
= (vcpu
->arch
.shregs
.msr
& MSR_PR
) ? SLB_VSID_KP
: SLB_VSID_KS
;
1260 status
&= ~DSISR_NOHPTE
; /* DSISR_NOHPTE == SRR1_ISI_NOPT */
1262 if (gr
& (HPTE_R_N
| HPTE_R_G
))
1263 return status
| SRR1_ISI_N_OR_G
;
1264 if (!hpte_read_permission(pp
, slb_v
& key
))
1265 return status
| SRR1_ISI_PROT
;
1266 } else if (status
& DSISR_ISSTORE
) {
1267 /* check write permission */
1268 if (!hpte_write_permission(pp
, slb_v
& key
))
1269 return status
| DSISR_PROTFAULT
;
1271 if (!hpte_read_permission(pp
, slb_v
& key
))
1272 return status
| DSISR_PROTFAULT
;
1275 /* Check storage key, if applicable */
1276 if (data
&& (vcpu
->arch
.shregs
.msr
& MSR_DR
)) {
1277 unsigned int perm
= hpte_get_skey_perm(gr
, vcpu
->arch
.amr
);
1278 if (status
& DSISR_ISSTORE
)
1281 return status
| DSISR_KEYFAULT
;
1284 /* Save HPTE info for virtual-mode handler */
1285 vcpu
->arch
.pgfault_addr
= addr
;
1286 vcpu
->arch
.pgfault_index
= index
;
1287 vcpu
->arch
.pgfault_hpte
[0] = v
;
1288 vcpu
->arch
.pgfault_hpte
[1] = r
;
1289 vcpu
->arch
.pgfault_cache
= cache_entry
;
1291 /* Check the storage key to see if it is possibly emulated MMIO */
1292 if ((r
& (HPTE_R_KEY_HI
| HPTE_R_KEY_LO
)) ==
1293 (HPTE_R_KEY_HI
| HPTE_R_KEY_LO
)) {
1295 unsigned int pshift
= 12;
1296 unsigned int pshift_index
;
1298 if (slb_v
& SLB_VSID_L
) {
1299 pshift_index
= ((slb_v
& SLB_VSID_LP
) >> 4);
1300 pshift
= slb_base_page_shift
[pshift_index
];
1302 cache_entry
= next_mmio_cache_entry(vcpu
);
1303 cache_entry
->eaddr
= addr
;
1304 cache_entry
->slb_base_pshift
= pshift
;
1305 cache_entry
->pte_index
= index
;
1306 cache_entry
->hpte_v
= v
;
1307 cache_entry
->hpte_r
= r
;
1308 cache_entry
->rpte
= gr
;
1309 cache_entry
->slb_v
= slb_v
;
1310 cache_entry
->mmio_update
= mmio_update
;
1312 if (data
&& (vcpu
->arch
.shregs
.msr
& MSR_IR
))
1313 return -2; /* MMIO emulation - load instr word */
1316 return -1; /* send fault up to host kernel mode */