?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
xen: cleancache shim to Xen Transcendent Memory
[linux-2.6/next.git] / arch / ia64 / kvm / vtlb.c
blob4332f7ee5203f27179db08b5a95d372c6509e681
1 /*
2 * vtlb.c: guest virtual tlb handling module.
3 * Copyright (c) 2004, Intel Corporation.
4 * Yaozu Dong (Eddie Dong) <Eddie.dong@intel.com>
5 * Xuefei Xu (Anthony Xu) <anthony.xu@intel.com>
6 *
7 * Copyright (c) 2007, Intel Corporation.
8 * Xuefei Xu (Anthony Xu) <anthony.xu@intel.com>
9 * Xiantao Zhang <xiantao.zhang@intel.com>
10 *
11 * This program is free software; you can redistribute it and/or modify it
12 * under the terms and conditions of the GNU General Public License,
13 * version 2, as published by the Free Software Foundation.
14 *
15 * This program is distributed in the hope it will be useful, but WITHOUT
16 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
17 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
18 * more details.
19 *
20 * You should have received a copy of the GNU General Public License along with
21 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
22 * Place - Suite 330, Boston, MA 02111-1307 USA.
23 *
24 */
25
26 #include "vcpu.h"
27
28 #include <linux/rwsem.h>
29
30 #include <asm/tlb.h>
31
32 /*
33 * Check to see if the address rid:va is translated by the TLB
34 */
35
36 static int __is_tr_translated(struct thash_data *trp, u64 rid, u64 va)
37 {
38 return ((trp->p) && (trp->rid == rid)
39 && ((va-trp->vadr) < PSIZE(trp->ps)));
40 }
41
42 /*
43 * Only for GUEST TR format.
44 */
45 static int __is_tr_overlap(struct thash_data *trp, u64 rid, u64 sva, u64 eva)
46 {
47 u64 sa1, ea1;
48
49 if (!trp->p || trp->rid != rid)
50 return 0;
51
52 sa1 = trp->vadr;
53 ea1 = sa1 + PSIZE(trp->ps) - 1;
54 eva -= 1;
55 if ((sva > ea1) || (sa1 > eva))
56 return 0;
57 else
58 return 1;
59
60 }
61
62 void machine_tlb_purge(u64 va, u64 ps)
63 {
64 ia64_ptcl(va, ps << 2);
65 }
66
67 void local_flush_tlb_all(void)
68 {
69 int i, j;
70 unsigned long flags, count0, count1;
71 unsigned long stride0, stride1, addr;
72
73 addr = current_vcpu->arch.ptce_base;
74 count0 = current_vcpu->arch.ptce_count[0];
75 count1 = current_vcpu->arch.ptce_count[1];
76 stride0 = current_vcpu->arch.ptce_stride[0];
77 stride1 = current_vcpu->arch.ptce_stride[1];
78
79 local_irq_save(flags);
80 for (i = 0; i < count0; ++i) {
81 for (j = 0; j < count1; ++j) {
82 ia64_ptce(addr);
83 addr += stride1;
84 }
85 addr += stride0;
86 }
87 local_irq_restore(flags);
88 ia64_srlz_i(); /* srlz.i implies srlz.d */
89 }
90
91 int vhpt_enabled(struct kvm_vcpu *vcpu, u64 vadr, enum vhpt_ref ref)
92 {
93 union ia64_rr vrr;
94 union ia64_pta vpta;
95 struct ia64_psr vpsr;
96
97 vpsr = *(struct ia64_psr *)&VCPU(vcpu, vpsr);
98 vrr.val = vcpu_get_rr(vcpu, vadr);
99 vpta.val = vcpu_get_pta(vcpu);
100
101 if (vrr.ve & vpta.ve) {
102 switch (ref) {
103 case DATA_REF:
104 case NA_REF:
105 return vpsr.dt;
106 case INST_REF:
107 return vpsr.dt && vpsr.it && vpsr.ic;
108 case RSE_REF:
109 return vpsr.dt && vpsr.rt;
110
111 }
112 }
113 return 0;
114 }
115
116 struct thash_data *vsa_thash(union ia64_pta vpta, u64 va, u64 vrr, u64 *tag)
117 {
118 u64 index, pfn, rid, pfn_bits;
119
120 pfn_bits = vpta.size - 5 - 8;
121 pfn = REGION_OFFSET(va) >> _REGION_PAGE_SIZE(vrr);
122 rid = _REGION_ID(vrr);
123 index = ((rid & 0xff) << pfn_bits)|(pfn & ((1UL << pfn_bits) - 1));
124 *tag = ((rid >> 8) & 0xffff) | ((pfn >> pfn_bits) << 16);
125
126 return (struct thash_data *)((vpta.base << PTA_BASE_SHIFT) +
127 (index << 5));
128 }
129
130 struct thash_data *__vtr_lookup(struct kvm_vcpu *vcpu, u64 va, int type)
131 {
132
133 struct thash_data *trp;
134 int i;
135 u64 rid;
136
137 rid = vcpu_get_rr(vcpu, va);
138 rid = rid & RR_RID_MASK;
139 if (type == D_TLB) {
140 if (vcpu_quick_region_check(vcpu->arch.dtr_regions, va)) {
141 for (trp = (struct thash_data *)&vcpu->arch.dtrs, i = 0;
142 i < NDTRS; i++, trp++) {
143 if (__is_tr_translated(trp, rid, va))
144 return trp;
145 }
146 }
147 } else {
148 if (vcpu_quick_region_check(vcpu->arch.itr_regions, va)) {
149 for (trp = (struct thash_data *)&vcpu->arch.itrs, i = 0;
150 i < NITRS; i++, trp++) {
151 if (__is_tr_translated(trp, rid, va))
152 return trp;
153 }
154 }
155 }
156
157 return NULL;
158 }
159
160 static void vhpt_insert(u64 pte, u64 itir, u64 ifa, u64 gpte)
161 {
162 union ia64_rr rr;
163 struct thash_data *head;
164 unsigned long ps, gpaddr;
165
166 ps = itir_ps(itir);
167 rr.val = ia64_get_rr(ifa);
168
169 gpaddr = ((gpte & _PAGE_PPN_MASK) >> ps << ps) |
170 (ifa & ((1UL << ps) - 1));
171
172 head = (struct thash_data *)ia64_thash(ifa);
173 head->etag = INVALID_TI_TAG;
174 ia64_mf();
175 head->page_flags = pte & ~PAGE_FLAGS_RV_MASK;
176 head->itir = rr.ps << 2;
177 head->etag = ia64_ttag(ifa);
178 head->gpaddr = gpaddr;
179 }
180
181 void mark_pages_dirty(struct kvm_vcpu *v, u64 pte, u64 ps)
182 {
183 u64 i, dirty_pages = 1;
184 u64 base_gfn = (pte&_PAGE_PPN_MASK) >> PAGE_SHIFT;
185 vmm_spinlock_t *lock = __kvm_va(v->arch.dirty_log_lock_pa);
186 void *dirty_bitmap = (void *)KVM_MEM_DIRTY_LOG_BASE;
187
188 dirty_pages <<= ps <= PAGE_SHIFT ? 0 : ps - PAGE_SHIFT;
189
190 vmm_spin_lock(lock);
191 for (i = 0; i < dirty_pages; i++) {
192 /* avoid RMW */
193 if (!test_bit(base_gfn + i, dirty_bitmap))
194 set_bit(base_gfn + i , dirty_bitmap);
195 }
196 vmm_spin_unlock(lock);
197 }
198
199 void thash_vhpt_insert(struct kvm_vcpu *v, u64 pte, u64 itir, u64 va, int type)
200 {
201 u64 phy_pte, psr;
202 union ia64_rr mrr;
203
204 mrr.val = ia64_get_rr(va);
205 phy_pte = translate_phy_pte(&pte, itir, va);
206
207 if (itir_ps(itir) >= mrr.ps) {
208 vhpt_insert(phy_pte, itir, va, pte);
209 } else {
210 phy_pte &= ~PAGE_FLAGS_RV_MASK;
211 psr = ia64_clear_ic();
212 ia64_itc(type, va, phy_pte, itir_ps(itir));
213 paravirt_dv_serialize_data();
214 ia64_set_psr(psr);
215 }
216
217 if (!(pte&VTLB_PTE_IO))
218 mark_pages_dirty(v, pte, itir_ps(itir));
219 }
220
221 /*
222 * vhpt lookup
223 */
224 struct thash_data *vhpt_lookup(u64 va)
225 {
226 struct thash_data *head;
227 u64 tag;
228
229 head = (struct thash_data *)ia64_thash(va);
230 tag = ia64_ttag(va);
231 if (head->etag == tag)
232 return head;
233 return NULL;
234 }
235
236 u64 guest_vhpt_lookup(u64 iha, u64 *pte)
237 {
238 u64 ret;
239 struct thash_data *data;
240
241 data = __vtr_lookup(current_vcpu, iha, D_TLB);
242 if (data != NULL)
243 thash_vhpt_insert(current_vcpu, data->page_flags,
244 data->itir, iha, D_TLB);
245
246 asm volatile ("rsm psr.ic|psr.i;;"
247 "srlz.d;;"
248 "ld8.s r9=[%1];;"
249 "tnat.nz p6,p7=r9;;"
250 "(p6) mov %0=1;"
251 "(p6) mov r9=r0;"
252 "(p7) extr.u r9=r9,0,53;;"
253 "(p7) mov %0=r0;"
254 "(p7) st8 [%2]=r9;;"
255 "ssm psr.ic;;"
256 "srlz.d;;"
257 "ssm psr.i;;"
258 "srlz.d;;"
259 : "=r"(ret) : "r"(iha), "r"(pte):"memory");
260
261 return ret;
262 }
263
264 /*
265 * purge software guest tlb
266 */
267
268 static void vtlb_purge(struct kvm_vcpu *v, u64 va, u64 ps)
269 {
270 struct thash_data *cur;
271 u64 start, curadr, size, psbits, tag, rr_ps, num;
272 union ia64_rr vrr;
273 struct thash_cb *hcb = &v->arch.vtlb;
274
275 vrr.val = vcpu_get_rr(v, va);
276 psbits = VMX(v, psbits[(va >> 61)]);
277 start = va & ~((1UL << ps) - 1);
278 while (psbits) {
279 curadr = start;
280 rr_ps = __ffs(psbits);
281 psbits &= ~(1UL << rr_ps);
282 num = 1UL << ((ps < rr_ps) ? 0 : (ps - rr_ps));
283 size = PSIZE(rr_ps);
284 vrr.ps = rr_ps;
285 while (num) {
286 cur = vsa_thash(hcb->pta, curadr, vrr.val, &tag);
287 if (cur->etag == tag && cur->ps == rr_ps)
288 cur->etag = INVALID_TI_TAG;
289 curadr += size;
290 num--;
291 }
292 }
293 }
294
295
296 /*
297 * purge VHPT and machine TLB
298 */
299 static void vhpt_purge(struct kvm_vcpu *v, u64 va, u64 ps)
300 {
301 struct thash_data *cur;
302 u64 start, size, tag, num;
303 union ia64_rr rr;
304
305 start = va & ~((1UL << ps) - 1);
306 rr.val = ia64_get_rr(va);
307 size = PSIZE(rr.ps);
308 num = 1UL << ((ps < rr.ps) ? 0 : (ps - rr.ps));
309 while (num) {
310 cur = (struct thash_data *)ia64_thash(start);
311 tag = ia64_ttag(start);
312 if (cur->etag == tag)
313 cur->etag = INVALID_TI_TAG;
314 start += size;
315 num--;
316 }
317 machine_tlb_purge(va, ps);
318 }
319
320 /*
321 * Insert an entry into hash TLB or VHPT.
322 * NOTES:
323 * 1: When inserting VHPT to thash, "va" is a must covered
324 * address by the inserted machine VHPT entry.
325 * 2: The format of entry is always in TLB.
326 * 3: The caller need to make sure the new entry will not overlap
327 * with any existed entry.
328 */
329 void vtlb_insert(struct kvm_vcpu *v, u64 pte, u64 itir, u64 va)
330 {
331 struct thash_data *head;
332 union ia64_rr vrr;
333 u64 tag;
334 struct thash_cb *hcb = &v->arch.vtlb;
335
336 vrr.val = vcpu_get_rr(v, va);
337 vrr.ps = itir_ps(itir);
338 VMX(v, psbits[va >> 61]) |= (1UL << vrr.ps);
339 head = vsa_thash(hcb->pta, va, vrr.val, &tag);
340 head->page_flags = pte;
341 head->itir = itir;
342 head->etag = tag;
343 }
344
345 int vtr_find_overlap(struct kvm_vcpu *vcpu, u64 va, u64 ps, int type)
346 {
347 struct thash_data *trp;
348 int i;
349 u64 end, rid;
350
351 rid = vcpu_get_rr(vcpu, va);
352 rid = rid & RR_RID_MASK;
353 end = va + PSIZE(ps);
354 if (type == D_TLB) {
355 if (vcpu_quick_region_check(vcpu->arch.dtr_regions, va)) {
356 for (trp = (struct thash_data *)&vcpu->arch.dtrs, i = 0;
357 i < NDTRS; i++, trp++) {
358 if (__is_tr_overlap(trp, rid, va, end))
359 return i;
360 }
361 }
362 } else {
363 if (vcpu_quick_region_check(vcpu->arch.itr_regions, va)) {
364 for (trp = (struct thash_data *)&vcpu->arch.itrs, i = 0;
365 i < NITRS; i++, trp++) {
366 if (__is_tr_overlap(trp, rid, va, end))
367 return i;
368 }
369 }
370 }
371 return -1;
372 }
373
374 /*
375 * Purge entries in VTLB and VHPT
376 */
377 void thash_purge_entries(struct kvm_vcpu *v, u64 va, u64 ps)
378 {
379 if (vcpu_quick_region_check(v->arch.tc_regions, va))
380 vtlb_purge(v, va, ps);
381 vhpt_purge(v, va, ps);
382 }
383
384 void thash_purge_entries_remote(struct kvm_vcpu *v, u64 va, u64 ps)
385 {
386 u64 old_va = va;
387 va = REGION_OFFSET(va);
388 if (vcpu_quick_region_check(v->arch.tc_regions, old_va))
389 vtlb_purge(v, va, ps);
390 vhpt_purge(v, va, ps);
391 }
392
393 u64 translate_phy_pte(u64 *pte, u64 itir, u64 va)
394 {
395 u64 ps, ps_mask, paddr, maddr, io_mask;
396 union pte_flags phy_pte;
397
398 ps = itir_ps(itir);
399 ps_mask = ~((1UL << ps) - 1);
400 phy_pte.val = *pte;
401 paddr = *pte;
402 paddr = ((paddr & _PAGE_PPN_MASK) & ps_mask) | (va & ~ps_mask);
403 maddr = kvm_get_mpt_entry(paddr >> PAGE_SHIFT);
404 io_mask = maddr & GPFN_IO_MASK;
405 if (io_mask && (io_mask != GPFN_PHYS_MMIO)) {
406 *pte |= VTLB_PTE_IO;
407 return -1;
408 }
409 maddr = ((maddr & _PAGE_PPN_MASK) & PAGE_MASK) |
410 (paddr & ~PAGE_MASK);
411 phy_pte.ppn = maddr >> ARCH_PAGE_SHIFT;
412 return phy_pte.val;
413 }
414
415 /*
416 * Purge overlap TCs and then insert the new entry to emulate itc ops.
417 * Notes: Only TC entry can purge and insert.
418 */
419 void thash_purge_and_insert(struct kvm_vcpu *v, u64 pte, u64 itir,
420 u64 ifa, int type)
421 {
422 u64 ps;
423 u64 phy_pte, io_mask, index;
424 union ia64_rr vrr, mrr;
425
426 ps = itir_ps(itir);
427 vrr.val = vcpu_get_rr(v, ifa);
428 mrr.val = ia64_get_rr(ifa);
429
430 index = (pte & _PAGE_PPN_MASK) >> PAGE_SHIFT;
431 io_mask = kvm_get_mpt_entry(index) & GPFN_IO_MASK;
432 phy_pte = translate_phy_pte(&pte, itir, ifa);
433
434 /* Ensure WB attribute if pte is related to a normal mem page,
435 * which is required by vga acceleration since qemu maps shared
436 * vram buffer with WB.
437 */
438 if (!(pte & VTLB_PTE_IO) && ((pte & _PAGE_MA_MASK) != _PAGE_MA_NAT) &&
439 io_mask != GPFN_PHYS_MMIO) {
440 pte &= ~_PAGE_MA_MASK;
441 phy_pte &= ~_PAGE_MA_MASK;
442 }
443
444 vtlb_purge(v, ifa, ps);
445 vhpt_purge(v, ifa, ps);
446
447 if ((ps != mrr.ps) || (pte & VTLB_PTE_IO)) {
448 vtlb_insert(v, pte, itir, ifa);
449 vcpu_quick_region_set(VMX(v, tc_regions), ifa);
450 }
451 if (pte & VTLB_PTE_IO)
452 return;
453
454 if (ps >= mrr.ps)
455 vhpt_insert(phy_pte, itir, ifa, pte);
456 else {
457 u64 psr;
458 phy_pte &= ~PAGE_FLAGS_RV_MASK;
459 psr = ia64_clear_ic();
460 ia64_itc(type, ifa, phy_pte, ps);
461 paravirt_dv_serialize_data();
462 ia64_set_psr(psr);
463 }
464 if (!(pte&VTLB_PTE_IO))
465 mark_pages_dirty(v, pte, ps);
466
467 }
468
469 /*
470 * Purge all TCs or VHPT entries including those in Hash table.
471 *
472 */
473
474 void thash_purge_all(struct kvm_vcpu *v)
475 {
476 int i;
477 struct thash_data *head;
478 struct thash_cb *vtlb, *vhpt;
479 vtlb = &v->arch.vtlb;
480 vhpt = &v->arch.vhpt;
481
482 for (i = 0; i < 8; i++)
483 VMX(v, psbits[i]) = 0;
484
485 head = vtlb->hash;
486 for (i = 0; i < vtlb->num; i++) {
487 head->page_flags = 0;
488 head->etag = INVALID_TI_TAG;
489 head->itir = 0;
490 head->next = 0;
491 head++;
492 };
493
494 head = vhpt->hash;
495 for (i = 0; i < vhpt->num; i++) {
496 head->page_flags = 0;
497 head->etag = INVALID_TI_TAG;
498 head->itir = 0;
499 head->next = 0;
500 head++;
501 };
502
503 local_flush_tlb_all();
504 }
505
506 /*
507 * Lookup the hash table and its collision chain to find an entry
508 * covering this address rid:va or the entry.
509 *
510 * INPUT:
511 * in: TLB format for both VHPT & TLB.
512 */
513 struct thash_data *vtlb_lookup(struct kvm_vcpu *v, u64 va, int is_data)
514 {
515 struct thash_data *cch;
516 u64 psbits, ps, tag;
517 union ia64_rr vrr;
518
519 struct thash_cb *hcb = &v->arch.vtlb;
520
521 cch = __vtr_lookup(v, va, is_data);
522 if (cch)
523 return cch;
524
525 if (vcpu_quick_region_check(v->arch.tc_regions, va) == 0)
526 return NULL;
527
528 psbits = VMX(v, psbits[(va >> 61)]);
529 vrr.val = vcpu_get_rr(v, va);
530 while (psbits) {
531 ps = __ffs(psbits);
532 psbits &= ~(1UL << ps);
533 vrr.ps = ps;
534 cch = vsa_thash(hcb->pta, va, vrr.val, &tag);
535 if (cch->etag == tag && cch->ps == ps)
536 return cch;
537 }
538
539 return NULL;
540 }
541
542 /*
543 * Initialize internal control data before service.
544 */
545 void thash_init(struct thash_cb *hcb, u64 sz)
546 {
547 int i;
548 struct thash_data *head;
549
550 hcb->pta.val = (unsigned long)hcb->hash;
551 hcb->pta.vf = 1;
552 hcb->pta.ve = 1;
553 hcb->pta.size = sz;
554 head = hcb->hash;
555 for (i = 0; i < hcb->num; i++) {
556 head->page_flags = 0;
557 head->itir = 0;
558 head->etag = INVALID_TI_TAG;
559 head->next = 0;
560 head++;
561 }
562 }
563
564 u64 kvm_get_mpt_entry(u64 gpfn)
565 {
566 u64 *base = (u64 *) KVM_P2M_BASE;
567
568 if (gpfn >= (KVM_P2M_SIZE >> 3))
569 panic_vm(current_vcpu, "Invalid gpfn =%lx\n", gpfn);
570
571 return *(base + gpfn);
572 }
573
574 u64 kvm_lookup_mpa(u64 gpfn)
575 {
576 u64 maddr;
577 maddr = kvm_get_mpt_entry(gpfn);
578 return maddr&_PAGE_PPN_MASK;
579 }
580
581 u64 kvm_gpa_to_mpa(u64 gpa)
582 {
583 u64 pte = kvm_lookup_mpa(gpa >> PAGE_SHIFT);
584 return (pte >> PAGE_SHIFT << PAGE_SHIFT) | (gpa & ~PAGE_MASK);
585 }
586
587 /*
588 * Fetch guest bundle code.
589 * INPUT:
590 * gip: guest ip
591 * pbundle: used to return fetched bundle.
592 */
593 int fetch_code(struct kvm_vcpu *vcpu, u64 gip, IA64_BUNDLE *pbundle)
594 {
595 u64 gpip = 0; /* guest physical IP*/
596 u64 *vpa;
597 struct thash_data *tlb;
598 u64 maddr;
599
600 if (!(VCPU(vcpu, vpsr) & IA64_PSR_IT)) {
601 /* I-side physical mode */
602 gpip = gip;
603 } else {
604 tlb = vtlb_lookup(vcpu, gip, I_TLB);
605 if (tlb)
606 gpip = (tlb->ppn >> (tlb->ps - 12) << tlb->ps) |
607 (gip & (PSIZE(tlb->ps) - 1));
608 }
609 if (gpip) {
610 maddr = kvm_gpa_to_mpa(gpip);
611 } else {
612 tlb = vhpt_lookup(gip);
613 if (tlb == NULL) {
614 ia64_ptcl(gip, ARCH_PAGE_SHIFT << 2);
615 return IA64_FAULT;
616 }
617 maddr = (tlb->ppn >> (tlb->ps - 12) << tlb->ps)
618 | (gip & (PSIZE(tlb->ps) - 1));
619 }
620 vpa = (u64 *)__kvm_va(maddr);
621
622 pbundle->i64[0] = *vpa++;
623 pbundle->i64[1] = *vpa;
624
625 return IA64_NO_FAULT;
626 }
627
628 void kvm_init_vhpt(struct kvm_vcpu *v)
629 {
630 v->arch.vhpt.num = VHPT_NUM_ENTRIES;
631 thash_init(&v->arch.vhpt, VHPT_SHIFT);
632 ia64_set_pta(v->arch.vhpt.pta.val);
633 /*Enable VHPT here?*/
634 }
635
636 void kvm_init_vtlb(struct kvm_vcpu *v)
637 {
638 v->arch.vtlb.num = VTLB_NUM_ENTRIES;
639 thash_init(&v->arch.vtlb, VTLB_SHIFT);
640 }
linux-next