2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License, version 2, as
4 * published by the Free Software Foundation.
6 * This program is distributed in the hope that it will be useful,
7 * but WITHOUT ANY WARRANTY; without even the implied warranty of
8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
9 * GNU General Public License for more details.
11 * You should have received a copy of the GNU General Public License
12 * along with this program; if not, write to the Free Software
13 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
15 * Copyright SUSE Linux Products GmbH 2009
17 * Authors: Alexander Graf <agraf@suse.de>
20 #include <linux/types.h>
21 #include <linux/string.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/highmem.h>
26 #include <asm/tlbflush.h>
27 #include <asm/kvm_ppc.h>
28 #include <asm/kvm_book3s.h>
29 #include <asm/mmu-hash64.h>
31 /* #define DEBUG_MMU */
34 #define dprintk(X...) printk(KERN_INFO X)
36 #define dprintk(X...) do { } while(0)
39 static void kvmppc_mmu_book3s_64_reset_msr(struct kvm_vcpu
*vcpu
)
41 kvmppc_set_msr(vcpu
, MSR_SF
);
44 static struct kvmppc_slb
*kvmppc_mmu_book3s_64_find_slbe(
45 struct kvm_vcpu
*vcpu
,
49 u64 esid
= GET_ESID(eaddr
);
50 u64 esid_1t
= GET_ESID_1T(eaddr
);
52 for (i
= 0; i
< vcpu
->arch
.slb_nr
; i
++) {
55 if (!vcpu
->arch
.slb
[i
].valid
)
58 if (vcpu
->arch
.slb
[i
].tb
)
61 if (vcpu
->arch
.slb
[i
].esid
== cmp_esid
)
62 return &vcpu
->arch
.slb
[i
];
65 dprintk("KVM: No SLB entry found for 0x%lx [%llx | %llx]\n",
66 eaddr
, esid
, esid_1t
);
67 for (i
= 0; i
< vcpu
->arch
.slb_nr
; i
++) {
68 if (vcpu
->arch
.slb
[i
].vsid
)
69 dprintk(" %d: %c%c%c %llx %llx\n", i
,
70 vcpu
->arch
.slb
[i
].valid
? 'v' : ' ',
71 vcpu
->arch
.slb
[i
].large
? 'l' : ' ',
72 vcpu
->arch
.slb
[i
].tb
? 't' : ' ',
73 vcpu
->arch
.slb
[i
].esid
,
74 vcpu
->arch
.slb
[i
].vsid
);
80 static int kvmppc_slb_sid_shift(struct kvmppc_slb
*slbe
)
82 return slbe
->tb
? SID_SHIFT_1T
: SID_SHIFT
;
85 static u64
kvmppc_slb_offset_mask(struct kvmppc_slb
*slbe
)
87 return (1ul << kvmppc_slb_sid_shift(slbe
)) - 1;
90 static u64
kvmppc_slb_calc_vpn(struct kvmppc_slb
*slb
, gva_t eaddr
)
92 eaddr
&= kvmppc_slb_offset_mask(slb
);
94 return (eaddr
>> VPN_SHIFT
) |
95 ((slb
->vsid
) << (kvmppc_slb_sid_shift(slb
) - VPN_SHIFT
));
98 static u64
kvmppc_mmu_book3s_64_ea_to_vp(struct kvm_vcpu
*vcpu
, gva_t eaddr
,
101 struct kvmppc_slb
*slb
;
103 slb
= kvmppc_mmu_book3s_64_find_slbe(vcpu
, eaddr
);
107 return kvmppc_slb_calc_vpn(slb
, eaddr
);
110 static int mmu_pagesize(int mmu_pg
)
121 static int kvmppc_mmu_book3s_64_get_pagesize(struct kvmppc_slb
*slbe
)
123 return mmu_pagesize(slbe
->base_page_size
);
126 static u32
kvmppc_mmu_book3s_64_get_page(struct kvmppc_slb
*slbe
, gva_t eaddr
)
128 int p
= kvmppc_mmu_book3s_64_get_pagesize(slbe
);
130 return ((eaddr
& kvmppc_slb_offset_mask(slbe
)) >> p
);
133 static hva_t
kvmppc_mmu_book3s_64_get_pteg(struct kvm_vcpu
*vcpu
,
134 struct kvmppc_slb
*slbe
, gva_t eaddr
,
137 struct kvmppc_vcpu_book3s
*vcpu_book3s
= to_book3s(vcpu
);
138 u64 hash
, pteg
, htabsize
;
143 htabsize
= ((1 << ((vcpu_book3s
->sdr1
& 0x1f) + 11)) - 1);
145 vpn
= kvmppc_slb_calc_vpn(slbe
, eaddr
);
146 ssize
= slbe
->tb
? MMU_SEGSIZE_1T
: MMU_SEGSIZE_256M
;
147 hash
= hpt_hash(vpn
, kvmppc_mmu_book3s_64_get_pagesize(slbe
), ssize
);
150 hash
&= ((1ULL << 39ULL) - 1ULL);
154 pteg
= vcpu_book3s
->sdr1
& 0xfffffffffffc0000ULL
;
157 dprintk("MMU: page=0x%x sdr1=0x%llx pteg=0x%llx vsid=0x%llx\n",
158 page
, vcpu_book3s
->sdr1
, pteg
, slbe
->vsid
);
160 /* When running a PAPR guest, SDR1 contains a HVA address instead
162 if (vcpu
->arch
.papr_enabled
)
165 r
= gfn_to_hva(vcpu
->kvm
, pteg
>> PAGE_SHIFT
);
167 if (kvm_is_error_hva(r
))
169 return r
| (pteg
& ~PAGE_MASK
);
172 static u64
kvmppc_mmu_book3s_64_get_avpn(struct kvmppc_slb
*slbe
, gva_t eaddr
)
174 int p
= kvmppc_mmu_book3s_64_get_pagesize(slbe
);
177 avpn
= kvmppc_mmu_book3s_64_get_page(slbe
, eaddr
);
178 avpn
|= slbe
->vsid
<< (kvmppc_slb_sid_shift(slbe
) - p
);
181 avpn
>>= ((80 - p
) - 56) - 8; /* 16 - p */
189 * Return page size encoded in the second word of a HPTE, or
190 * -1 for an invalid encoding for the base page size indicated by
191 * the SLB entry. This doesn't handle mixed pagesize segments yet.
193 static int decode_pagesize(struct kvmppc_slb
*slbe
, u64 r
)
195 switch (slbe
->base_page_size
) {
197 if ((r
& 0xf000) == 0x1000)
201 if ((r
& 0xff000) == 0)
208 static int kvmppc_mmu_book3s_64_xlate(struct kvm_vcpu
*vcpu
, gva_t eaddr
,
209 struct kvmppc_pte
*gpte
, bool data
,
212 struct kvmppc_slb
*slbe
;
224 ulong mp_ea
= vcpu
->arch
.magic_page_ea
;
226 /* Magic page override */
227 if (unlikely(mp_ea
) &&
228 unlikely((eaddr
& ~0xfffULL
) == (mp_ea
& ~0xfffULL
)) &&
229 !(vcpu
->arch
.shared
->msr
& MSR_PR
)) {
231 gpte
->vpage
= kvmppc_mmu_book3s_64_ea_to_vp(vcpu
, eaddr
, data
);
232 gpte
->raddr
= vcpu
->arch
.magic_page_pa
| (gpte
->raddr
& 0xfff);
233 gpte
->raddr
&= KVM_PAM
;
234 gpte
->may_execute
= true;
235 gpte
->may_read
= true;
236 gpte
->may_write
= true;
237 gpte
->page_size
= MMU_PAGE_4K
;
242 slbe
= kvmppc_mmu_book3s_64_find_slbe(vcpu
, eaddr
);
246 avpn
= kvmppc_mmu_book3s_64_get_avpn(slbe
, eaddr
);
247 v_val
= avpn
& HPTE_V_AVPN
;
250 v_val
|= SLB_VSID_B_1T
;
252 v_val
|= HPTE_V_LARGE
;
253 v_val
|= HPTE_V_VALID
;
255 v_mask
= SLB_VSID_B
| HPTE_V_AVPN
| HPTE_V_LARGE
| HPTE_V_VALID
|
258 pgsize
= slbe
->large
? MMU_PAGE_16M
: MMU_PAGE_4K
;
260 mutex_lock(&vcpu
->kvm
->arch
.hpt_mutex
);
263 ptegp
= kvmppc_mmu_book3s_64_get_pteg(vcpu
, slbe
, eaddr
, second
);
264 if (kvm_is_error_hva(ptegp
))
267 if(copy_from_user(pteg
, (void __user
*)ptegp
, sizeof(pteg
))) {
268 printk(KERN_ERR
"KVM can't copy data from 0x%lx!\n", ptegp
);
272 if ((vcpu
->arch
.shared
->msr
& MSR_PR
) && slbe
->Kp
)
274 else if (!(vcpu
->arch
.shared
->msr
& MSR_PR
) && slbe
->Ks
)
277 for (i
=0; i
<16; i
+=2) {
278 /* Check all relevant fields of 1st dword */
279 if ((pteg
[i
] & v_mask
) == v_val
) {
280 /* If large page bit is set, check pgsize encoding */
282 (vcpu
->arch
.hflags
& BOOK3S_HFLAG_MULTI_PGSIZE
)) {
283 pgsize
= decode_pagesize(slbe
, pteg
[i
+1]);
295 v_val
|= HPTE_V_SECONDARY
;
302 pp
= (r
& HPTE_R_PP
) | key
;
307 gpte
->vpage
= kvmppc_mmu_book3s_64_ea_to_vp(vcpu
, eaddr
, data
);
309 eaddr_mask
= (1ull << mmu_pagesize(pgsize
)) - 1;
310 gpte
->raddr
= (r
& HPTE_R_RPN
& ~eaddr_mask
) | (eaddr
& eaddr_mask
);
311 gpte
->page_size
= pgsize
;
312 gpte
->may_execute
= ((r
& HPTE_R_N
) ? false : true);
313 gpte
->may_read
= false;
314 gpte
->may_write
= false;
321 gpte
->may_write
= true;
327 gpte
->may_read
= true;
331 dprintk("KVM MMU: Translated 0x%lx [0x%llx] -> 0x%llx "
333 eaddr
, avpn
, gpte
->vpage
, gpte
->raddr
);
335 /* Update PTE R and C bits, so the guest's swapper knows we used the
337 if (gpte
->may_read
&& !(r
& HPTE_R_R
)) {
339 * Set the accessed flag.
340 * We have to write this back with a single byte write
341 * because another vcpu may be accessing this on
342 * non-PAPR platforms such as mac99, and this is
343 * what real hardware does.
345 char __user
*addr
= (char __user
*) &pteg
[i
+1];
347 put_user(r
>> 8, addr
+ 6);
349 if (iswrite
&& gpte
->may_write
&& !(r
& HPTE_R_C
)) {
350 /* Set the dirty flag */
351 /* Use a single byte write */
352 char __user
*addr
= (char __user
*) &pteg
[i
+1];
354 put_user(r
, addr
+ 7);
357 mutex_unlock(&vcpu
->kvm
->arch
.hpt_mutex
);
359 if (!gpte
->may_read
|| (iswrite
&& !gpte
->may_write
))
364 mutex_unlock(&vcpu
->kvm
->arch
.hpt_mutex
);
368 dprintk("KVM MMU: Trigger segment fault\n");
372 static void kvmppc_mmu_book3s_64_slbmte(struct kvm_vcpu
*vcpu
, u64 rs
, u64 rb
)
374 struct kvmppc_vcpu_book3s
*vcpu_book3s
;
377 struct kvmppc_slb
*slbe
;
379 dprintk("KVM MMU: slbmte(0x%llx, 0x%llx)\n", rs
, rb
);
381 vcpu_book3s
= to_book3s(vcpu
);
384 esid_1t
= GET_ESID_1T(rb
);
387 if (slb_nr
> vcpu
->arch
.slb_nr
)
390 slbe
= &vcpu
->arch
.slb
[slb_nr
];
392 slbe
->large
= (rs
& SLB_VSID_L
) ? 1 : 0;
393 slbe
->tb
= (rs
& SLB_VSID_B_1T
) ? 1 : 0;
394 slbe
->esid
= slbe
->tb
? esid_1t
: esid
;
395 slbe
->vsid
= (rs
& ~SLB_VSID_B
) >> (kvmppc_slb_sid_shift(slbe
) - 16);
396 slbe
->valid
= (rb
& SLB_ESID_V
) ? 1 : 0;
397 slbe
->Ks
= (rs
& SLB_VSID_KS
) ? 1 : 0;
398 slbe
->Kp
= (rs
& SLB_VSID_KP
) ? 1 : 0;
399 slbe
->nx
= (rs
& SLB_VSID_N
) ? 1 : 0;
400 slbe
->class = (rs
& SLB_VSID_C
) ? 1 : 0;
402 slbe
->base_page_size
= MMU_PAGE_4K
;
404 if (vcpu
->arch
.hflags
& BOOK3S_HFLAG_MULTI_PGSIZE
) {
405 switch (rs
& SLB_VSID_LP
) {
407 slbe
->base_page_size
= MMU_PAGE_16M
;
410 slbe
->base_page_size
= MMU_PAGE_64K
;
414 slbe
->base_page_size
= MMU_PAGE_16M
;
417 slbe
->orige
= rb
& (ESID_MASK
| SLB_ESID_V
);
420 /* Map the new segment */
421 kvmppc_mmu_map_segment(vcpu
, esid
<< SID_SHIFT
);
424 static u64
kvmppc_mmu_book3s_64_slbmfee(struct kvm_vcpu
*vcpu
, u64 slb_nr
)
426 struct kvmppc_slb
*slbe
;
428 if (slb_nr
> vcpu
->arch
.slb_nr
)
431 slbe
= &vcpu
->arch
.slb
[slb_nr
];
436 static u64
kvmppc_mmu_book3s_64_slbmfev(struct kvm_vcpu
*vcpu
, u64 slb_nr
)
438 struct kvmppc_slb
*slbe
;
440 if (slb_nr
> vcpu
->arch
.slb_nr
)
443 slbe
= &vcpu
->arch
.slb
[slb_nr
];
448 static void kvmppc_mmu_book3s_64_slbie(struct kvm_vcpu
*vcpu
, u64 ea
)
450 struct kvmppc_slb
*slbe
;
453 dprintk("KVM MMU: slbie(0x%llx)\n", ea
);
455 slbe
= kvmppc_mmu_book3s_64_find_slbe(vcpu
, ea
);
460 dprintk("KVM MMU: slbie(0x%llx, 0x%llx)\n", ea
, slbe
->esid
);
466 seg_size
= 1ull << kvmppc_slb_sid_shift(slbe
);
467 kvmppc_mmu_flush_segment(vcpu
, ea
& ~(seg_size
- 1), seg_size
);
470 static void kvmppc_mmu_book3s_64_slbia(struct kvm_vcpu
*vcpu
)
474 dprintk("KVM MMU: slbia()\n");
476 for (i
= 1; i
< vcpu
->arch
.slb_nr
; i
++) {
477 vcpu
->arch
.slb
[i
].valid
= false;
478 vcpu
->arch
.slb
[i
].orige
= 0;
479 vcpu
->arch
.slb
[i
].origv
= 0;
482 if (vcpu
->arch
.shared
->msr
& MSR_IR
) {
483 kvmppc_mmu_flush_segments(vcpu
);
484 kvmppc_mmu_map_segment(vcpu
, kvmppc_get_pc(vcpu
));
488 static void kvmppc_mmu_book3s_64_mtsrin(struct kvm_vcpu
*vcpu
, u32 srnum
,
494 * According to Book3 2.01 mtsrin is implemented as:
496 * The SLB entry specified by (RB)32:35 is loaded from register
499 * SLBE Bit Source SLB Field
501 * 0:31 0x0000_0000 ESID-0:31
502 * 32:35 (RB)32:35 ESID-32:35
504 * 37:61 0x00_0000|| 0b0 VSID-0:24
505 * 62:88 (RS)37:63 VSID-25:51
506 * 89:91 (RS)33:35 Ks Kp N
507 * 92 (RS)36 L ((RS)36 must be 0b0)
511 dprintk("KVM MMU: mtsrin(0x%x, 0x%lx)\n", srnum
, value
);
514 rb
|= (srnum
& 0xf) << 28;
515 /* Set the valid bit */
521 rs
|= (value
& 0xfffffff) << 12;
523 rs
|= ((value
>> 28) & 0x7) << 9;
525 kvmppc_mmu_book3s_64_slbmte(vcpu
, rs
, rb
);
528 static void kvmppc_mmu_book3s_64_tlbie(struct kvm_vcpu
*vcpu
, ulong va
,
531 u64 mask
= 0xFFFFFFFFFULL
;
535 dprintk("KVM MMU: tlbie(0x%lx)\n", va
);
538 * The tlbie instruction changed behaviour starting with
539 * POWER6. POWER6 and later don't have the large page flag
540 * in the instruction but in the RB value, along with bits
541 * indicating page and segment sizes.
543 if (vcpu
->arch
.hflags
& BOOK3S_HFLAG_NEW_TLBIE
) {
544 /* POWER6 or later */
545 if (va
& 1) { /* L bit */
546 if ((va
& 0xf000) == 0x1000)
547 mask
= 0xFFFFFFFF0ULL
; /* 64k page */
549 mask
= 0xFFFFFF000ULL
; /* 16M page */
552 /* older processors, e.g. PPC970 */
554 mask
= 0xFFFFFF000ULL
;
556 /* flush this VA on all vcpus */
557 kvm_for_each_vcpu(i
, v
, vcpu
->kvm
)
558 kvmppc_mmu_pte_vflush(v
, va
>> 12, mask
);
561 #ifdef CONFIG_PPC_64K_PAGES
562 static int segment_contains_magic_page(struct kvm_vcpu
*vcpu
, ulong esid
)
564 ulong mp_ea
= vcpu
->arch
.magic_page_ea
;
566 return mp_ea
&& !(vcpu
->arch
.shared
->msr
& MSR_PR
) &&
567 (mp_ea
>> SID_SHIFT
) == esid
;
571 static int kvmppc_mmu_book3s_64_esid_to_vsid(struct kvm_vcpu
*vcpu
, ulong esid
,
574 ulong ea
= esid
<< SID_SHIFT
;
575 struct kvmppc_slb
*slb
;
577 ulong mp_ea
= vcpu
->arch
.magic_page_ea
;
578 int pagesize
= MMU_PAGE_64K
;
580 if (vcpu
->arch
.shared
->msr
& (MSR_DR
|MSR_IR
)) {
581 slb
= kvmppc_mmu_book3s_64_find_slbe(vcpu
, ea
);
584 pagesize
= slb
->base_page_size
;
586 gvsid
<<= SID_SHIFT_1T
- SID_SHIFT
;
587 gvsid
|= esid
& ((1ul << (SID_SHIFT_1T
- SID_SHIFT
)) - 1);
593 switch (vcpu
->arch
.shared
->msr
& (MSR_DR
|MSR_IR
)) {
595 gvsid
= VSID_REAL
| esid
;
598 gvsid
|= VSID_REAL_IR
;
601 gvsid
|= VSID_REAL_DR
;
613 #ifdef CONFIG_PPC_64K_PAGES
615 * Mark this as a 64k segment if the host is using
616 * 64k pages, the host MMU supports 64k pages and
617 * the guest segment page size is >= 64k,
618 * but not if this segment contains the magic page.
620 if (pagesize
>= MMU_PAGE_64K
&&
621 mmu_psize_defs
[MMU_PAGE_64K
].shift
&&
622 !segment_contains_magic_page(vcpu
, esid
))
626 if (vcpu
->arch
.shared
->msr
& MSR_PR
)
633 /* Catch magic page case */
634 if (unlikely(mp_ea
) &&
635 unlikely(esid
== (mp_ea
>> SID_SHIFT
)) &&
636 !(vcpu
->arch
.shared
->msr
& MSR_PR
)) {
637 *vsid
= VSID_REAL
| esid
;
644 static bool kvmppc_mmu_book3s_64_is_dcbz32(struct kvm_vcpu
*vcpu
)
646 return (to_book3s(vcpu
)->hid
[5] & 0x80);
649 void kvmppc_mmu_book3s_64_init(struct kvm_vcpu
*vcpu
)
651 struct kvmppc_mmu
*mmu
= &vcpu
->arch
.mmu
;
654 mmu
->mtsrin
= kvmppc_mmu_book3s_64_mtsrin
;
655 mmu
->slbmte
= kvmppc_mmu_book3s_64_slbmte
;
656 mmu
->slbmfee
= kvmppc_mmu_book3s_64_slbmfee
;
657 mmu
->slbmfev
= kvmppc_mmu_book3s_64_slbmfev
;
658 mmu
->slbie
= kvmppc_mmu_book3s_64_slbie
;
659 mmu
->slbia
= kvmppc_mmu_book3s_64_slbia
;
660 mmu
->xlate
= kvmppc_mmu_book3s_64_xlate
;
661 mmu
->reset_msr
= kvmppc_mmu_book3s_64_reset_msr
;
662 mmu
->tlbie
= kvmppc_mmu_book3s_64_tlbie
;
663 mmu
->esid_to_vsid
= kvmppc_mmu_book3s_64_esid_to_vsid
;
664 mmu
->ea_to_vp
= kvmppc_mmu_book3s_64_ea_to_vp
;
665 mmu
->is_dcbz32
= kvmppc_mmu_book3s_64_is_dcbz32
;
667 vcpu
->arch
.hflags
|= BOOK3S_HFLAG_SLB
;