?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
announce overlapping skbs merge
[cor_2_6_31.git] / arch / x86 / kvm / mmu.c
blob7030b5f911bff0df9019a1c080bf3b5561086780
1 /*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
6 *
7 * MMU support
8 *
9 * Copyright (C) 2006 Qumranet, Inc.
10 *
11 * Authors:
12 * Yaniv Kamay <yaniv@qumranet.com>
13 * Avi Kivity <avi@qumranet.com>
14 *
15 * This work is licensed under the terms of the GNU GPL, version 2. See
16 * the COPYING file in the top-level directory.
17 *
18 */
19
20 #include "mmu.h"
21
22 #include <linux/kvm_host.h>
23 #include <linux/types.h>
24 #include <linux/string.h>
25 #include <linux/mm.h>
26 #include <linux/highmem.h>
27 #include <linux/module.h>
28 #include <linux/swap.h>
29 #include <linux/hugetlb.h>
30 #include <linux/compiler.h>
31
32 #include <asm/page.h>
33 #include <asm/cmpxchg.h>
34 #include <asm/io.h>
35 #include <asm/vmx.h>
36
37 /*
38 * When setting this variable to true it enables Two-Dimensional-Paging
39 * where the hardware walks 2 page tables:
40 * 1. the guest-virtual to guest-physical
41 * 2. while doing 1. it walks guest-physical to host-physical
42 * If the hardware supports that we don't need to do shadow paging.
43 */
44 bool tdp_enabled = false;
45
46 #undef MMU_DEBUG
47
48 #undef AUDIT
49
50 #ifdef AUDIT
51 static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg);
52 #else
53 static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg) {}
54 #endif
55
56 #ifdef MMU_DEBUG
57
58 #define pgprintk(x...) do { if (dbg) printk(x); } while (0)
59 #define rmap_printk(x...) do { if (dbg) printk(x); } while (0)
60
61 #else
62
63 #define pgprintk(x...) do { } while (0)
64 #define rmap_printk(x...) do { } while (0)
65
66 #endif
67
68 #if defined(MMU_DEBUG) || defined(AUDIT)
69 static int dbg = 0;
70 module_param(dbg, bool, 0644);
71 #endif
72
73 static int oos_shadow = 1;
74 module_param(oos_shadow, bool, 0644);
75
76 #ifndef MMU_DEBUG
77 #define ASSERT(x) do { } while (0)
78 #else
79 #define ASSERT(x) \
80 if (!(x)) { \
81 printk(KERN_WARNING "assertion failed %s:%d: %s\n", \
82 __FILE__, __LINE__, #x); \
83 }
84 #endif
85
86 #define PT_FIRST_AVAIL_BITS_SHIFT 9
87 #define PT64_SECOND_AVAIL_BITS_SHIFT 52
88
89 #define VALID_PAGE(x) ((x) != INVALID_PAGE)
90
91 #define PT64_LEVEL_BITS 9
92
93 #define PT64_LEVEL_SHIFT(level) \
94 (PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)
95
96 #define PT64_LEVEL_MASK(level) \
97 (((1ULL << PT64_LEVEL_BITS) - 1) << PT64_LEVEL_SHIFT(level))
98
99 #define PT64_INDEX(address, level)\
100 (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))
101
102
103 #define PT32_LEVEL_BITS 10
104
105 #define PT32_LEVEL_SHIFT(level) \
106 (PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)
107
108 #define PT32_LEVEL_MASK(level) \
109 (((1ULL << PT32_LEVEL_BITS) - 1) << PT32_LEVEL_SHIFT(level))
110
111 #define PT32_INDEX(address, level)\
112 (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
113
114
115 #define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
116 #define PT64_DIR_BASE_ADDR_MASK \
117 (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
118
119 #define PT32_BASE_ADDR_MASK PAGE_MASK
120 #define PT32_DIR_BASE_ADDR_MASK \
121 (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
122
123 #define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK \
124 | PT64_NX_MASK)
125
126 #define PFERR_PRESENT_MASK (1U << 0)
127 #define PFERR_WRITE_MASK (1U << 1)
128 #define PFERR_USER_MASK (1U << 2)
129 #define PFERR_RSVD_MASK (1U << 3)
130 #define PFERR_FETCH_MASK (1U << 4)
131
132 #define PT_DIRECTORY_LEVEL 2
133 #define PT_PAGE_TABLE_LEVEL 1
134
135 #define RMAP_EXT 4
136
137 #define ACC_EXEC_MASK 1
138 #define ACC_WRITE_MASK PT_WRITABLE_MASK
139 #define ACC_USER_MASK PT_USER_MASK
140 #define ACC_ALL (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)
141
142 #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)
143
144 struct kvm_rmap_desc {
145 u64 *shadow_ptes[RMAP_EXT];
146 struct kvm_rmap_desc *more;
147 };
148
149 struct kvm_shadow_walk_iterator {
150 u64 addr;
151 hpa_t shadow_addr;
152 int level;
153 u64 *sptep;
154 unsigned index;
155 };
156
157 #define for_each_shadow_entry(_vcpu, _addr, _walker) \
158 for (shadow_walk_init(&(_walker), _vcpu, _addr); \
159 shadow_walk_okay(&(_walker)); \
160 shadow_walk_next(&(_walker)))
161
162
163 struct kvm_unsync_walk {
164 int (*entry) (struct kvm_mmu_page *sp, struct kvm_unsync_walk *walk);
165 };
166
167 typedef int (*mmu_parent_walk_fn) (struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp);
168
169 static struct kmem_cache *pte_chain_cache;
170 static struct kmem_cache *rmap_desc_cache;
171 static struct kmem_cache *mmu_page_header_cache;
172
173 static u64 __read_mostly shadow_trap_nonpresent_pte;
174 static u64 __read_mostly shadow_notrap_nonpresent_pte;
175 static u64 __read_mostly shadow_base_present_pte;
176 static u64 __read_mostly shadow_nx_mask;
177 static u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */
178 static u64 __read_mostly shadow_user_mask;
179 static u64 __read_mostly shadow_accessed_mask;
180 static u64 __read_mostly shadow_dirty_mask;
181
182 static inline u64 rsvd_bits(int s, int e)
183 {
184 return ((1ULL << (e - s + 1)) - 1) << s;
185 }
186
187 void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte)
188 {
189 shadow_trap_nonpresent_pte = trap_pte;
190 shadow_notrap_nonpresent_pte = notrap_pte;
191 }
192 EXPORT_SYMBOL_GPL(kvm_mmu_set_nonpresent_ptes);
193
194 void kvm_mmu_set_base_ptes(u64 base_pte)
195 {
196 shadow_base_present_pte = base_pte;
197 }
198 EXPORT_SYMBOL_GPL(kvm_mmu_set_base_ptes);
199
200 void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
201 u64 dirty_mask, u64 nx_mask, u64 x_mask)
202 {
203 shadow_user_mask = user_mask;
204 shadow_accessed_mask = accessed_mask;
205 shadow_dirty_mask = dirty_mask;
206 shadow_nx_mask = nx_mask;
207 shadow_x_mask = x_mask;
208 }
209 EXPORT_SYMBOL_GPL(kvm_mmu_set_mask_ptes);
210
211 static int is_write_protection(struct kvm_vcpu *vcpu)
212 {
213 return vcpu->arch.cr0 & X86_CR0_WP;
214 }
215
216 static int is_cpuid_PSE36(void)
217 {
218 return 1;
219 }
220
221 static int is_nx(struct kvm_vcpu *vcpu)
222 {
223 return vcpu->arch.shadow_efer & EFER_NX;
224 }
225
226 static int is_shadow_present_pte(u64 pte)
227 {
228 return pte != shadow_trap_nonpresent_pte
229 && pte != shadow_notrap_nonpresent_pte;
230 }
231
232 static int is_large_pte(u64 pte)
233 {
234 return pte & PT_PAGE_SIZE_MASK;
235 }
236
237 static int is_writeble_pte(unsigned long pte)
238 {
239 return pte & PT_WRITABLE_MASK;
240 }
241
242 static int is_dirty_pte(unsigned long pte)
243 {
244 return pte & shadow_dirty_mask;
245 }
246
247 static int is_rmap_pte(u64 pte)
248 {
249 return is_shadow_present_pte(pte);
250 }
251
252 static pfn_t spte_to_pfn(u64 pte)
253 {
254 return (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
255 }
256
257 static gfn_t pse36_gfn_delta(u32 gpte)
258 {
259 int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT;
260
261 return (gpte & PT32_DIR_PSE36_MASK) << shift;
262 }
263
264 static void set_shadow_pte(u64 *sptep, u64 spte)
265 {
266 #ifdef CONFIG_X86_64
267 set_64bit((unsigned long *)sptep, spte);
268 #else
269 set_64bit((unsigned long long *)sptep, spte);
270 #endif
271 }
272
273 static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
274 struct kmem_cache *base_cache, int min)
275 {
276 void *obj;
277
278 if (cache->nobjs >= min)
279 return 0;
280 while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
281 obj = kmem_cache_zalloc(base_cache, GFP_KERNEL);
282 if (!obj)
283 return -ENOMEM;
284 cache->objects[cache->nobjs++] = obj;
285 }
286 return 0;
287 }
288
289 static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc)
290 {
291 while (mc->nobjs)
292 kfree(mc->objects[--mc->nobjs]);
293 }
294
295 static int mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache,
296 int min)
297 {
298 struct page *page;
299
300 if (cache->nobjs >= min)
301 return 0;
302 while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
303 page = alloc_page(GFP_KERNEL);
304 if (!page)
305 return -ENOMEM;
306 set_page_private(page, 0);
307 cache->objects[cache->nobjs++] = page_address(page);
308 }
309 return 0;
310 }
311
312 static void mmu_free_memory_cache_page(struct kvm_mmu_memory_cache *mc)
313 {
314 while (mc->nobjs)
315 free_page((unsigned long)mc->objects[--mc->nobjs]);
316 }
317
318 static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
319 {
320 int r;
321
322 r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_chain_cache,
323 pte_chain_cache, 4);
324 if (r)
325 goto out;
326 r = mmu_topup_memory_cache(&vcpu->arch.mmu_rmap_desc_cache,
327 rmap_desc_cache, 4);
328 if (r)
329 goto out;
330 r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8);
331 if (r)
332 goto out;
333 r = mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache,
334 mmu_page_header_cache, 4);
335 out:
336 return r;
337 }
338
339 static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
340 {
341 mmu_free_memory_cache(&vcpu->arch.mmu_pte_chain_cache);
342 mmu_free_memory_cache(&vcpu->arch.mmu_rmap_desc_cache);
343 mmu_free_memory_cache_page(&vcpu->arch.mmu_page_cache);
344 mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache);
345 }
346
347 static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc,
348 size_t size)
349 {
350 void *p;
351
352 BUG_ON(!mc->nobjs);
353 p = mc->objects[--mc->nobjs];
354 return p;
355 }
356
357 static struct kvm_pte_chain *mmu_alloc_pte_chain(struct kvm_vcpu *vcpu)
358 {
359 return mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_chain_cache,
360 sizeof(struct kvm_pte_chain));
361 }
362
363 static void mmu_free_pte_chain(struct kvm_pte_chain *pc)
364 {
365 kfree(pc);
366 }
367
368 static struct kvm_rmap_desc *mmu_alloc_rmap_desc(struct kvm_vcpu *vcpu)
369 {
370 return mmu_memory_cache_alloc(&vcpu->arch.mmu_rmap_desc_cache,
371 sizeof(struct kvm_rmap_desc));
372 }
373
374 static void mmu_free_rmap_desc(struct kvm_rmap_desc *rd)
375 {
376 kfree(rd);
377 }
378
379 /*
380 * Return the pointer to the largepage write count for a given
381 * gfn, handling slots that are not large page aligned.
382 */
383 static int *slot_largepage_idx(gfn_t gfn, struct kvm_memory_slot *slot)
384 {
385 unsigned long idx;
386
387 idx = (gfn / KVM_PAGES_PER_HPAGE) -
388 (slot->base_gfn / KVM_PAGES_PER_HPAGE);
389 return &slot->lpage_info[idx].write_count;
390 }
391
392 static void account_shadowed(struct kvm *kvm, gfn_t gfn)
393 {
394 int *write_count;
395
396 gfn = unalias_gfn(kvm, gfn);
397 write_count = slot_largepage_idx(gfn,
398 gfn_to_memslot_unaliased(kvm, gfn));
399 *write_count += 1;
400 }
401
402 static void unaccount_shadowed(struct kvm *kvm, gfn_t gfn)
403 {
404 int *write_count;
405
406 gfn = unalias_gfn(kvm, gfn);
407 write_count = slot_largepage_idx(gfn,
408 gfn_to_memslot_unaliased(kvm, gfn));
409 *write_count -= 1;
410 WARN_ON(*write_count < 0);
411 }
412
413 static int has_wrprotected_page(struct kvm *kvm, gfn_t gfn)
414 {
415 struct kvm_memory_slot *slot;
416 int *largepage_idx;
417
418 gfn = unalias_gfn(kvm, gfn);
419 slot = gfn_to_memslot_unaliased(kvm, gfn);
420 if (slot) {
421 largepage_idx = slot_largepage_idx(gfn, slot);
422 return *largepage_idx;
423 }
424
425 return 1;
426 }
427
428 static int host_largepage_backed(struct kvm *kvm, gfn_t gfn)
429 {
430 struct vm_area_struct *vma;
431 unsigned long addr;
432 int ret = 0;
433
434 addr = gfn_to_hva(kvm, gfn);
435 if (kvm_is_error_hva(addr))
436 return ret;
437
438 down_read(&current->mm->mmap_sem);
439 vma = find_vma(current->mm, addr);
440 if (vma && is_vm_hugetlb_page(vma))
441 ret = 1;
442 up_read(&current->mm->mmap_sem);
443
444 return ret;
445 }
446
447 static int is_largepage_backed(struct kvm_vcpu *vcpu, gfn_t large_gfn)
448 {
449 struct kvm_memory_slot *slot;
450
451 if (has_wrprotected_page(vcpu->kvm, large_gfn))
452 return 0;
453
454 if (!host_largepage_backed(vcpu->kvm, large_gfn))
455 return 0;
456
457 slot = gfn_to_memslot(vcpu->kvm, large_gfn);
458 if (slot && slot->dirty_bitmap)
459 return 0;
460
461 return 1;
462 }
463
464 /*
465 * Take gfn and return the reverse mapping to it.
466 * Note: gfn must be unaliased before this function get called
467 */
468
469 static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, int lpage)
470 {
471 struct kvm_memory_slot *slot;
472 unsigned long idx;
473
474 slot = gfn_to_memslot(kvm, gfn);
475 if (!lpage)
476 return &slot->rmap[gfn - slot->base_gfn];
477
478 idx = (gfn / KVM_PAGES_PER_HPAGE) -
479 (slot->base_gfn / KVM_PAGES_PER_HPAGE);
480
481 return &slot->lpage_info[idx].rmap_pde;
482 }
483
484 /*
485 * Reverse mapping data structures:
486 *
487 * If rmapp bit zero is zero, then rmapp point to the shadw page table entry
488 * that points to page_address(page).
489 *
490 * If rmapp bit zero is one, (then rmap & ~1) points to a struct kvm_rmap_desc
491 * containing more mappings.
492 */
493 static void rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn, int lpage)
494 {
495 struct kvm_mmu_page *sp;
496 struct kvm_rmap_desc *desc;
497 unsigned long *rmapp;
498 int i;
499
500 if (!is_rmap_pte(*spte))
501 return;
502 gfn = unalias_gfn(vcpu->kvm, gfn);
503 sp = page_header(__pa(spte));
504 sp->gfns[spte - sp->spt] = gfn;
505 rmapp = gfn_to_rmap(vcpu->kvm, gfn, lpage);
506 if (!*rmapp) {
507 rmap_printk("rmap_add: %p %llx 0->1\n", spte, *spte);
508 *rmapp = (unsigned long)spte;
509 } else if (!(*rmapp & 1)) {
510 rmap_printk("rmap_add: %p %llx 1->many\n", spte, *spte);
511 desc = mmu_alloc_rmap_desc(vcpu);
512 desc->shadow_ptes[0] = (u64 *)*rmapp;
513 desc->shadow_ptes[1] = spte;
514 *rmapp = (unsigned long)desc | 1;
515 } else {
516 rmap_printk("rmap_add: %p %llx many->many\n", spte, *spte);
517 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
518 while (desc->shadow_ptes[RMAP_EXT-1] && desc->more)
519 desc = desc->more;
520 if (desc->shadow_ptes[RMAP_EXT-1]) {
521 desc->more = mmu_alloc_rmap_desc(vcpu);
522 desc = desc->more;
523 }
524 for (i = 0; desc->shadow_ptes[i]; ++i)
525 ;
526 desc->shadow_ptes[i] = spte;
527 }
528 }
529
530 static void rmap_desc_remove_entry(unsigned long *rmapp,
531 struct kvm_rmap_desc *desc,
532 int i,
533 struct kvm_rmap_desc *prev_desc)
534 {
535 int j;
536
537 for (j = RMAP_EXT - 1; !desc->shadow_ptes[j] && j > i; --j)
538 ;
539 desc->shadow_ptes[i] = desc->shadow_ptes[j];
540 desc->shadow_ptes[j] = NULL;
541 if (j != 0)
542 return;
543 if (!prev_desc && !desc->more)
544 *rmapp = (unsigned long)desc->shadow_ptes[0];
545 else
546 if (prev_desc)
547 prev_desc->more = desc->more;
548 else
549 *rmapp = (unsigned long)desc->more | 1;
550 mmu_free_rmap_desc(desc);
551 }
552
553 static void rmap_remove(struct kvm *kvm, u64 *spte)
554 {
555 struct kvm_rmap_desc *desc;
556 struct kvm_rmap_desc *prev_desc;
557 struct kvm_mmu_page *sp;
558 pfn_t pfn;
559 unsigned long *rmapp;
560 int i;
561
562 if (!is_rmap_pte(*spte))
563 return;
564 sp = page_header(__pa(spte));
565 pfn = spte_to_pfn(*spte);
566 if (*spte & shadow_accessed_mask)
567 kvm_set_pfn_accessed(pfn);
568 if (is_writeble_pte(*spte))
569 kvm_release_pfn_dirty(pfn);
570 else
571 kvm_release_pfn_clean(pfn);
572 rmapp = gfn_to_rmap(kvm, sp->gfns[spte - sp->spt], is_large_pte(*spte));
573 if (!*rmapp) {
574 printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte);
575 BUG();
576 } else if (!(*rmapp & 1)) {
577 rmap_printk("rmap_remove: %p %llx 1->0\n", spte, *spte);
578 if ((u64 *)*rmapp != spte) {
579 printk(KERN_ERR "rmap_remove: %p %llx 1->BUG\n",
580 spte, *spte);
581 BUG();
582 }
583 *rmapp = 0;
584 } else {
585 rmap_printk("rmap_remove: %p %llx many->many\n", spte, *spte);
586 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
587 prev_desc = NULL;
588 while (desc) {
589 for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i)
590 if (desc->shadow_ptes[i] == spte) {
591 rmap_desc_remove_entry(rmapp,
592 desc, i,
593 prev_desc);
594 return;
595 }
596 prev_desc = desc;
597 desc = desc->more;
598 }
599 BUG();
600 }
601 }
602
603 static u64 *rmap_next(struct kvm *kvm, unsigned long *rmapp, u64 *spte)
604 {
605 struct kvm_rmap_desc *desc;
606 struct kvm_rmap_desc *prev_desc;
607 u64 *prev_spte;
608 int i;
609
610 if (!*rmapp)
611 return NULL;
612 else if (!(*rmapp & 1)) {
613 if (!spte)
614 return (u64 *)*rmapp;
615 return NULL;
616 }
617 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
618 prev_desc = NULL;
619 prev_spte = NULL;
620 while (desc) {
621 for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i) {
622 if (prev_spte == spte)
623 return desc->shadow_ptes[i];
624 prev_spte = desc->shadow_ptes[i];
625 }
626 desc = desc->more;
627 }
628 return NULL;
629 }
630
631 static int rmap_write_protect(struct kvm *kvm, u64 gfn)
632 {
633 unsigned long *rmapp;
634 u64 *spte;
635 int write_protected = 0;
636
637 gfn = unalias_gfn(kvm, gfn);
638 rmapp = gfn_to_rmap(kvm, gfn, 0);
639
640 spte = rmap_next(kvm, rmapp, NULL);
641 while (spte) {
642 BUG_ON(!spte);
643 BUG_ON(!(*spte & PT_PRESENT_MASK));
644 rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte);
645 if (is_writeble_pte(*spte)) {
646 set_shadow_pte(spte, *spte & ~PT_WRITABLE_MASK);
647 write_protected = 1;
648 }
649 spte = rmap_next(kvm, rmapp, spte);
650 }
651 if (write_protected) {
652 pfn_t pfn;
653
654 spte = rmap_next(kvm, rmapp, NULL);
655 pfn = spte_to_pfn(*spte);
656 kvm_set_pfn_dirty(pfn);
657 }
658
659 /* check for huge page mappings */
660 rmapp = gfn_to_rmap(kvm, gfn, 1);
661 spte = rmap_next(kvm, rmapp, NULL);
662 while (spte) {
663 BUG_ON(!spte);
664 BUG_ON(!(*spte & PT_PRESENT_MASK));
665 BUG_ON((*spte & (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK)) != (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK));
666 pgprintk("rmap_write_protect(large): spte %p %llx %lld\n", spte, *spte, gfn);
667 if (is_writeble_pte(*spte)) {
668 rmap_remove(kvm, spte);
669 --kvm->stat.lpages;
670 set_shadow_pte(spte, shadow_trap_nonpresent_pte);
671 spte = NULL;
672 write_protected = 1;
673 }
674 spte = rmap_next(kvm, rmapp, spte);
675 }
676
677 return write_protected;
678 }
679
680 static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp)
681 {
682 u64 *spte;
683 int need_tlb_flush = 0;
684
685 while ((spte = rmap_next(kvm, rmapp, NULL))) {
686 BUG_ON(!(*spte & PT_PRESENT_MASK));
687 rmap_printk("kvm_rmap_unmap_hva: spte %p %llx\n", spte, *spte);
688 rmap_remove(kvm, spte);
689 set_shadow_pte(spte, shadow_trap_nonpresent_pte);
690 need_tlb_flush = 1;
691 }
692 return need_tlb_flush;
693 }
694
695 static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
696 int (*handler)(struct kvm *kvm, unsigned long *rmapp))
697 {
698 int i;
699 int retval = 0;
700
701 /*
702 * If mmap_sem isn't taken, we can look the memslots with only
703 * the mmu_lock by skipping over the slots with userspace_addr == 0.
704 */
705 for (i = 0; i < kvm->nmemslots; i++) {
706 struct kvm_memory_slot *memslot = &kvm->memslots[i];
707 unsigned long start = memslot->userspace_addr;
708 unsigned long end;
709
710 /* mmu_lock protects userspace_addr */
711 if (!start)
712 continue;
713
714 end = start + (memslot->npages << PAGE_SHIFT);
715 if (hva >= start && hva < end) {
716 gfn_t gfn_offset = (hva - start) >> PAGE_SHIFT;
717 retval |= handler(kvm, &memslot->rmap[gfn_offset]);
718 retval |= handler(kvm,
719 &memslot->lpage_info[
720 gfn_offset /
721 KVM_PAGES_PER_HPAGE].rmap_pde);
722 }
723 }
724
725 return retval;
726 }
727
728 int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
729 {
730 return kvm_handle_hva(kvm, hva, kvm_unmap_rmapp);
731 }
732
733 static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp)
734 {
735 u64 *spte;
736 int young = 0;
737
738 /* always return old for EPT */
739 if (!shadow_accessed_mask)
740 return 0;
741
742 spte = rmap_next(kvm, rmapp, NULL);
743 while (spte) {
744 int _young;
745 u64 _spte = *spte;
746 BUG_ON(!(_spte & PT_PRESENT_MASK));
747 _young = _spte & PT_ACCESSED_MASK;
748 if (_young) {
749 young = 1;
750 clear_bit(PT_ACCESSED_SHIFT, (unsigned long *)spte);
751 }
752 spte = rmap_next(kvm, rmapp, spte);
753 }
754 return young;
755 }
756
757 int kvm_age_hva(struct kvm *kvm, unsigned long hva)
758 {
759 return kvm_handle_hva(kvm, hva, kvm_age_rmapp);
760 }
761
762 #ifdef MMU_DEBUG
763 static int is_empty_shadow_page(u64 *spt)
764 {
765 u64 *pos;
766 u64 *end;
767
768 for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
769 if (is_shadow_present_pte(*pos)) {
770 printk(KERN_ERR "%s: %p %llx\n", __func__,
771 pos, *pos);
772 return 0;
773 }
774 return 1;
775 }
776 #endif
777
778 static void kvm_mmu_free_page(struct kvm *kvm, struct kvm_mmu_page *sp)
779 {
780 ASSERT(is_empty_shadow_page(sp->spt));
781 list_del(&sp->link);
782 __free_page(virt_to_page(sp->spt));
783 __free_page(virt_to_page(sp->gfns));
784 kfree(sp);
785 ++kvm->arch.n_free_mmu_pages;
786 }
787
788 static unsigned kvm_page_table_hashfn(gfn_t gfn)
789 {
790 return gfn & ((1 << KVM_MMU_HASH_SHIFT) - 1);
791 }
792
793 static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
794 u64 *parent_pte)
795 {
796 struct kvm_mmu_page *sp;
797
798 sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache, sizeof *sp);
799 sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE);
800 sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE);
801 set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
802 list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);
803 INIT_LIST_HEAD(&sp->oos_link);
804 bitmap_zero(sp->slot_bitmap, KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS);
805 sp->multimapped = 0;
806 sp->parent_pte = parent_pte;
807 --vcpu->kvm->arch.n_free_mmu_pages;
808 return sp;
809 }
810
811 static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
812 struct kvm_mmu_page *sp, u64 *parent_pte)
813 {
814 struct kvm_pte_chain *pte_chain;
815 struct hlist_node *node;
816 int i;
817
818 if (!parent_pte)
819 return;
820 if (!sp->multimapped) {
821 u64 *old = sp->parent_pte;
822
823 if (!old) {
824 sp->parent_pte = parent_pte;
825 return;
826 }
827 sp->multimapped = 1;
828 pte_chain = mmu_alloc_pte_chain(vcpu);
829 INIT_HLIST_HEAD(&sp->parent_ptes);
830 hlist_add_head(&pte_chain->link, &sp->parent_ptes);
831 pte_chain->parent_ptes[0] = old;
832 }
833 hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link) {
834 if (pte_chain->parent_ptes[NR_PTE_CHAIN_ENTRIES-1])
835 continue;
836 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i)
837 if (!pte_chain->parent_ptes[i]) {
838 pte_chain->parent_ptes[i] = parent_pte;
839 return;
840 }
841 }
842 pte_chain = mmu_alloc_pte_chain(vcpu);
843 BUG_ON(!pte_chain);
844 hlist_add_head(&pte_chain->link, &sp->parent_ptes);
845 pte_chain->parent_ptes[0] = parent_pte;
846 }
847
848 static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp,
849 u64 *parent_pte)
850 {
851 struct kvm_pte_chain *pte_chain;
852 struct hlist_node *node;
853 int i;
854
855 if (!sp->multimapped) {
856 BUG_ON(sp->parent_pte != parent_pte);
857 sp->parent_pte = NULL;
858 return;
859 }
860 hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link)
861 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
862 if (!pte_chain->parent_ptes[i])
863 break;
864 if (pte_chain->parent_ptes[i] != parent_pte)
865 continue;
866 while (i + 1 < NR_PTE_CHAIN_ENTRIES
867 && pte_chain->parent_ptes[i + 1]) {
868 pte_chain->parent_ptes[i]
869 = pte_chain->parent_ptes[i + 1];
870 ++i;
871 }
872 pte_chain->parent_ptes[i] = NULL;
873 if (i == 0) {
874 hlist_del(&pte_chain->link);
875 mmu_free_pte_chain(pte_chain);
876 if (hlist_empty(&sp->parent_ptes)) {
877 sp->multimapped = 0;
878 sp->parent_pte = NULL;
879 }
880 }
881 return;
882 }
883 BUG();
884 }
885
886
887 static void mmu_parent_walk(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
888 mmu_parent_walk_fn fn)
889 {
890 struct kvm_pte_chain *pte_chain;
891 struct hlist_node *node;
892 struct kvm_mmu_page *parent_sp;
893 int i;
894
895 if (!sp->multimapped && sp->parent_pte) {
896 parent_sp = page_header(__pa(sp->parent_pte));
897 fn(vcpu, parent_sp);
898 mmu_parent_walk(vcpu, parent_sp, fn);
899 return;
900 }
901 hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link)
902 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
903 if (!pte_chain->parent_ptes[i])
904 break;
905 parent_sp = page_header(__pa(pte_chain->parent_ptes[i]));
906 fn(vcpu, parent_sp);
907 mmu_parent_walk(vcpu, parent_sp, fn);
908 }
909 }
910
911 static void kvm_mmu_update_unsync_bitmap(u64 *spte)
912 {
913 unsigned int index;
914 struct kvm_mmu_page *sp = page_header(__pa(spte));
915
916 index = spte - sp->spt;
917 if (!__test_and_set_bit(index, sp->unsync_child_bitmap))
918 sp->unsync_children++;
919 WARN_ON(!sp->unsync_children);
920 }
921
922 static void kvm_mmu_update_parents_unsync(struct kvm_mmu_page *sp)
923 {
924 struct kvm_pte_chain *pte_chain;
925 struct hlist_node *node;
926 int i;
927
928 if (!sp->parent_pte)
929 return;
930
931 if (!sp->multimapped) {
932 kvm_mmu_update_unsync_bitmap(sp->parent_pte);
933 return;
934 }
935
936 hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link)
937 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
938 if (!pte_chain->parent_ptes[i])
939 break;
940 kvm_mmu_update_unsync_bitmap(pte_chain->parent_ptes[i]);
941 }
942 }
943
944 static int unsync_walk_fn(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
945 {
946 kvm_mmu_update_parents_unsync(sp);
947 return 1;
948 }
949
950 static void kvm_mmu_mark_parents_unsync(struct kvm_vcpu *vcpu,
951 struct kvm_mmu_page *sp)
952 {
953 mmu_parent_walk(vcpu, sp, unsync_walk_fn);
954 kvm_mmu_update_parents_unsync(sp);
955 }
956
957 static void nonpaging_prefetch_page(struct kvm_vcpu *vcpu,
958 struct kvm_mmu_page *sp)
959 {
960 int i;
961
962 for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
963 sp->spt[i] = shadow_trap_nonpresent_pte;
964 }
965
966 static int nonpaging_sync_page(struct kvm_vcpu *vcpu,
967 struct kvm_mmu_page *sp)
968 {
969 return 1;
970 }
971
972 static void nonpaging_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
973 {
974 }
975
976 #define KVM_PAGE_ARRAY_NR 16
977
978 struct kvm_mmu_pages {
979 struct mmu_page_and_offset {
980 struct kvm_mmu_page *sp;
981 unsigned int idx;
982 } page[KVM_PAGE_ARRAY_NR];
983 unsigned int nr;
984 };
985
986 #define for_each_unsync_children(bitmap, idx) \
987 for (idx = find_first_bit(bitmap, 512); \
988 idx < 512; \
989 idx = find_next_bit(bitmap, 512, idx+1))
990
991 static int mmu_pages_add(struct kvm_mmu_pages *pvec, struct kvm_mmu_page *sp,
992 int idx)
993 {
994 int i;
995
996 if (sp->unsync)
997 for (i=0; i < pvec->nr; i++)
998 if (pvec->page[i].sp == sp)
999 return 0;
1000
1001 pvec->page[pvec->nr].sp = sp;
1002 pvec->page[pvec->nr].idx = idx;
1003 pvec->nr++;
1004 return (pvec->nr == KVM_PAGE_ARRAY_NR);
1005 }
1006
1007 static int __mmu_unsync_walk(struct kvm_mmu_page *sp,
1008 struct kvm_mmu_pages *pvec)
1009 {
1010 int i, ret, nr_unsync_leaf = 0;
1011
1012 for_each_unsync_children(sp->unsync_child_bitmap, i) {
1013 u64 ent = sp->spt[i];
1014
1015 if (is_shadow_present_pte(ent) && !is_large_pte(ent)) {
1016 struct kvm_mmu_page *child;
1017 child = page_header(ent & PT64_BASE_ADDR_MASK);
1018
1019 if (child->unsync_children) {
1020 if (mmu_pages_add(pvec, child, i))
1021 return -ENOSPC;
1022
1023 ret = __mmu_unsync_walk(child, pvec);
1024 if (!ret)
1025 __clear_bit(i, sp->unsync_child_bitmap);
1026 else if (ret > 0)
1027 nr_unsync_leaf += ret;
1028 else
1029 return ret;
1030 }
1031
1032 if (child->unsync) {
1033 nr_unsync_leaf++;
1034 if (mmu_pages_add(pvec, child, i))
1035 return -ENOSPC;
1036 }
1037 }
1038 }
1039
1040 if (find_first_bit(sp->unsync_child_bitmap, 512) == 512)
1041 sp->unsync_children = 0;
1042
1043 return nr_unsync_leaf;
1044 }
1045
1046 static int mmu_unsync_walk(struct kvm_mmu_page *sp,
1047 struct kvm_mmu_pages *pvec)
1048 {
1049 if (!sp->unsync_children)
1050 return 0;
1051
1052 mmu_pages_add(pvec, sp, 0);
1053 return __mmu_unsync_walk(sp, pvec);
1054 }
1055
1056 static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm *kvm, gfn_t gfn)
1057 {
1058 unsigned index;
1059 struct hlist_head *bucket;
1060 struct kvm_mmu_page *sp;
1061 struct hlist_node *node;
1062
1063 pgprintk("%s: looking for gfn %lx\n", __func__, gfn);
1064 index = kvm_page_table_hashfn(gfn);
1065 bucket = &kvm->arch.mmu_page_hash[index];
1066 hlist_for_each_entry(sp, node, bucket, hash_link)
1067 if (sp->gfn == gfn && !sp->role.direct
1068 && !sp->role.invalid) {
1069 pgprintk("%s: found role %x\n",
1070 __func__, sp->role.word);
1071 return sp;
1072 }
1073 return NULL;
1074 }
1075
1076 static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
1077 {
1078 WARN_ON(!sp->unsync);
1079 sp->unsync = 0;
1080 --kvm->stat.mmu_unsync;
1081 }
1082
1083 static int kvm_mmu_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp);
1084
1085 static int kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
1086 {
1087 if (sp->role.glevels != vcpu->arch.mmu.root_level) {
1088 kvm_mmu_zap_page(vcpu->kvm, sp);
1089 return 1;
1090 }
1091
1092 if (rmap_write_protect(vcpu->kvm, sp->gfn))
1093 kvm_flush_remote_tlbs(vcpu->kvm);
1094 kvm_unlink_unsync_page(vcpu->kvm, sp);
1095 if (vcpu->arch.mmu.sync_page(vcpu, sp)) {
1096 kvm_mmu_zap_page(vcpu->kvm, sp);
1097 return 1;
1098 }
1099
1100 kvm_mmu_flush_tlb(vcpu);
1101 return 0;
1102 }
1103
1104 struct mmu_page_path {
1105 struct kvm_mmu_page *parent[PT64_ROOT_LEVEL-1];
1106 unsigned int idx[PT64_ROOT_LEVEL-1];
1107 };
1108
1109 #define for_each_sp(pvec, sp, parents, i) \
1110 for (i = mmu_pages_next(&pvec, &parents, -1), \
1111 sp = pvec.page[i].sp; \
1112 i < pvec.nr && ({ sp = pvec.page[i].sp; 1;}); \
1113 i = mmu_pages_next(&pvec, &parents, i))
1114
1115 static int mmu_pages_next(struct kvm_mmu_pages *pvec,
1116 struct mmu_page_path *parents,
1117 int i)
1118 {
1119 int n;
1120
1121 for (n = i+1; n < pvec->nr; n++) {
1122 struct kvm_mmu_page *sp = pvec->page[n].sp;
1123
1124 if (sp->role.level == PT_PAGE_TABLE_LEVEL) {
1125 parents->idx[0] = pvec->page[n].idx;
1126 return n;
1127 }
1128
1129 parents->parent[sp->role.level-2] = sp;
1130 parents->idx[sp->role.level-1] = pvec->page[n].idx;
1131 }
1132
1133 return n;
1134 }
1135
1136 static void mmu_pages_clear_parents(struct mmu_page_path *parents)
1137 {
1138 struct kvm_mmu_page *sp;
1139 unsigned int level = 0;
1140
1141 do {
1142 unsigned int idx = parents->idx[level];
1143
1144 sp = parents->parent[level];
1145 if (!sp)
1146 return;
1147
1148 --sp->unsync_children;
1149 WARN_ON((int)sp->unsync_children < 0);
1150 __clear_bit(idx, sp->unsync_child_bitmap);
1151 level++;
1152 } while (level < PT64_ROOT_LEVEL-1 && !sp->unsync_children);
1153 }
1154
1155 static void kvm_mmu_pages_init(struct kvm_mmu_page *parent,
1156 struct mmu_page_path *parents,
1157 struct kvm_mmu_pages *pvec)
1158 {
1159 parents->parent[parent->role.level-1] = NULL;
1160 pvec->nr = 0;
1161 }
1162
1163 static void mmu_sync_children(struct kvm_vcpu *vcpu,
1164 struct kvm_mmu_page *parent)
1165 {
1166 int i;
1167 struct kvm_mmu_page *sp;
1168 struct mmu_page_path parents;
1169 struct kvm_mmu_pages pages;
1170
1171 kvm_mmu_pages_init(parent, &parents, &pages);
1172 while (mmu_unsync_walk(parent, &pages)) {
1173 int protected = 0;
1174
1175 for_each_sp(pages, sp, parents, i)
1176 protected |= rmap_write_protect(vcpu->kvm, sp->gfn);
1177
1178 if (protected)
1179 kvm_flush_remote_tlbs(vcpu->kvm);
1180
1181 for_each_sp(pages, sp, parents, i) {
1182 kvm_sync_page(vcpu, sp);
1183 mmu_pages_clear_parents(&parents);
1184 }
1185 cond_resched_lock(&vcpu->kvm->mmu_lock);
1186 kvm_mmu_pages_init(parent, &parents, &pages);
1187 }
1188 }
1189
1190 static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
1191 gfn_t gfn,
1192 gva_t gaddr,
1193 unsigned level,
1194 int direct,
1195 unsigned access,
1196 u64 *parent_pte)
1197 {
1198 union kvm_mmu_page_role role;
1199 unsigned index;
1200 unsigned quadrant;
1201 struct hlist_head *bucket;
1202 struct kvm_mmu_page *sp;
1203 struct hlist_node *node, *tmp;
1204
1205 role = vcpu->arch.mmu.base_role;
1206 role.level = level;
1207 role.direct = direct;
1208 role.access = access;
1209 if (vcpu->arch.mmu.root_level <= PT32_ROOT_LEVEL) {
1210 quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
1211 quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
1212 role.quadrant = quadrant;
1213 }
1214 pgprintk("%s: looking gfn %lx role %x\n", __func__,
1215 gfn, role.word);
1216 index = kvm_page_table_hashfn(gfn);
1217 bucket = &vcpu->kvm->arch.mmu_page_hash[index];
1218 hlist_for_each_entry_safe(sp, node, tmp, bucket, hash_link)
1219 if (sp->gfn == gfn) {
1220 if (sp->unsync)
1221 if (kvm_sync_page(vcpu, sp))
1222 continue;
1223
1224 if (sp->role.word != role.word)
1225 continue;
1226
1227 mmu_page_add_parent_pte(vcpu, sp, parent_pte);
1228 if (sp->unsync_children) {
1229 set_bit(KVM_REQ_MMU_SYNC, &vcpu->requests);
1230 kvm_mmu_mark_parents_unsync(vcpu, sp);
1231 }
1232 pgprintk("%s: found\n", __func__);
1233 return sp;
1234 }
1235 ++vcpu->kvm->stat.mmu_cache_miss;
1236 sp = kvm_mmu_alloc_page(vcpu, parent_pte);
1237 if (!sp)
1238 return sp;
1239 pgprintk("%s: adding gfn %lx role %x\n", __func__, gfn, role.word);
1240 sp->gfn = gfn;
1241 sp->role = role;
1242 hlist_add_head(&sp->hash_link, bucket);
1243 if (!direct) {
1244 if (rmap_write_protect(vcpu->kvm, gfn))
1245 kvm_flush_remote_tlbs(vcpu->kvm);
1246 account_shadowed(vcpu->kvm, gfn);
1247 }
1248 if (shadow_trap_nonpresent_pte != shadow_notrap_nonpresent_pte)
1249 vcpu->arch.mmu.prefetch_page(vcpu, sp);
1250 else
1251 nonpaging_prefetch_page(vcpu, sp);
1252 return sp;
1253 }
1254
1255 static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator,
1256 struct kvm_vcpu *vcpu, u64 addr)
1257 {
1258 iterator->addr = addr;
1259 iterator->shadow_addr = vcpu->arch.mmu.root_hpa;
1260 iterator->level = vcpu->arch.mmu.shadow_root_level;
1261 if (iterator->level == PT32E_ROOT_LEVEL) {
1262 iterator->shadow_addr
1263 = vcpu->arch.mmu.pae_root[(addr >> 30) & 3];
1264 iterator->shadow_addr &= PT64_BASE_ADDR_MASK;
1265 --iterator->level;
1266 if (!iterator->shadow_addr)
1267 iterator->level = 0;
1268 }
1269 }
1270
1271 static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator)
1272 {
1273 if (iterator->level < PT_PAGE_TABLE_LEVEL)
1274 return false;
1275 iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level);
1276 iterator->sptep = ((u64 *)__va(iterator->shadow_addr)) + iterator->index;
1277 return true;
1278 }
1279
1280 static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator)
1281 {
1282 iterator->shadow_addr = *iterator->sptep & PT64_BASE_ADDR_MASK;
1283 --iterator->level;
1284 }
1285
1286 static void kvm_mmu_page_unlink_children(struct kvm *kvm,
1287 struct kvm_mmu_page *sp)
1288 {
1289 unsigned i;
1290 u64 *pt;
1291 u64 ent;
1292
1293 pt = sp->spt;
1294
1295 if (sp->role.level == PT_PAGE_TABLE_LEVEL) {
1296 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
1297 if (is_shadow_present_pte(pt[i]))
1298 rmap_remove(kvm, &pt[i]);
1299 pt[i] = shadow_trap_nonpresent_pte;
1300 }
1301 return;
1302 }
1303
1304 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
1305 ent = pt[i];
1306
1307 if (is_shadow_present_pte(ent)) {
1308 if (!is_large_pte(ent)) {
1309 ent &= PT64_BASE_ADDR_MASK;
1310 mmu_page_remove_parent_pte(page_header(ent),
1311 &pt[i]);
1312 } else {
1313 --kvm->stat.lpages;
1314 rmap_remove(kvm, &pt[i]);
1315 }
1316 }
1317 pt[i] = shadow_trap_nonpresent_pte;
1318 }
1319 }
1320
1321 static void kvm_mmu_put_page(struct kvm_mmu_page *sp, u64 *parent_pte)
1322 {
1323 mmu_page_remove_parent_pte(sp, parent_pte);
1324 }
1325
1326 static void kvm_mmu_reset_last_pte_updated(struct kvm *kvm)
1327 {
1328 int i;
1329
1330 for (i = 0; i < KVM_MAX_VCPUS; ++i)
1331 if (kvm->vcpus[i])
1332 kvm->vcpus[i]->arch.last_pte_updated = NULL;
1333 }
1334
1335 static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
1336 {
1337 u64 *parent_pte;
1338
1339 while (sp->multimapped || sp->parent_pte) {
1340 if (!sp->multimapped)
1341 parent_pte = sp->parent_pte;
1342 else {
1343 struct kvm_pte_chain *chain;
1344
1345 chain = container_of(sp->parent_ptes.first,
1346 struct kvm_pte_chain, link);
1347 parent_pte = chain->parent_ptes[0];
1348 }
1349 BUG_ON(!parent_pte);
1350 kvm_mmu_put_page(sp, parent_pte);
1351 set_shadow_pte(parent_pte, shadow_trap_nonpresent_pte);
1352 }
1353 }
1354
1355 static int mmu_zap_unsync_children(struct kvm *kvm,
1356 struct kvm_mmu_page *parent)
1357 {
1358 int i, zapped = 0;
1359 struct mmu_page_path parents;
1360 struct kvm_mmu_pages pages;
1361
1362 if (parent->role.level == PT_PAGE_TABLE_LEVEL)
1363 return 0;
1364
1365 kvm_mmu_pages_init(parent, &parents, &pages);
1366 while (mmu_unsync_walk(parent, &pages)) {
1367 struct kvm_mmu_page *sp;
1368
1369 for_each_sp(pages, sp, parents, i) {
1370 kvm_mmu_zap_page(kvm, sp);
1371 mmu_pages_clear_parents(&parents);
1372 }
1373 zapped += pages.nr;
1374 kvm_mmu_pages_init(parent, &parents, &pages);
1375 }
1376
1377 return zapped;
1378 }
1379
1380 static int kvm_mmu_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp)
1381 {
1382 int ret;
1383 ++kvm->stat.mmu_shadow_zapped;
1384 ret = mmu_zap_unsync_children(kvm, sp);
1385 kvm_mmu_page_unlink_children(kvm, sp);
1386 kvm_mmu_unlink_parents(kvm, sp);
1387 kvm_flush_remote_tlbs(kvm);
1388 if (!sp->role.invalid && !sp->role.direct)
1389 unaccount_shadowed(kvm, sp->gfn);
1390 if (sp->unsync)
1391 kvm_unlink_unsync_page(kvm, sp);
1392 if (!sp->root_count) {
1393 hlist_del(&sp->hash_link);
1394 kvm_mmu_free_page(kvm, sp);
1395 } else {
1396 sp->role.invalid = 1;
1397 list_move(&sp->link, &kvm->arch.active_mmu_pages);
1398 kvm_reload_remote_mmus(kvm);
1399 }
1400 kvm_mmu_reset_last_pte_updated(kvm);
1401 return ret;
1402 }
1403
1404 /*
1405 * Changing the number of mmu pages allocated to the vm
1406 * Note: if kvm_nr_mmu_pages is too small, you will get dead lock
1407 */
1408 void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages)
1409 {
1410 /*
1411 * If we set the number of mmu pages to be smaller be than the
1412 * number of actived pages , we must to free some mmu pages before we
1413 * change the value
1414 */
1415
1416 if ((kvm->arch.n_alloc_mmu_pages - kvm->arch.n_free_mmu_pages) >
1417 kvm_nr_mmu_pages) {
1418 int n_used_mmu_pages = kvm->arch.n_alloc_mmu_pages
1419 - kvm->arch.n_free_mmu_pages;
1420
1421 while (n_used_mmu_pages > kvm_nr_mmu_pages) {
1422 struct kvm_mmu_page *page;
1423
1424 page = container_of(kvm->arch.active_mmu_pages.prev,
1425 struct kvm_mmu_page, link);
1426 kvm_mmu_zap_page(kvm, page);
1427 n_used_mmu_pages--;
1428 }
1429 kvm->arch.n_free_mmu_pages = 0;
1430 }
1431 else
1432 kvm->arch.n_free_mmu_pages += kvm_nr_mmu_pages
1433 - kvm->arch.n_alloc_mmu_pages;
1434
1435 kvm->arch.n_alloc_mmu_pages = kvm_nr_mmu_pages;
1436 }
1437
1438 static int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
1439 {
1440 unsigned index;
1441 struct hlist_head *bucket;
1442 struct kvm_mmu_page *sp;
1443 struct hlist_node *node, *n;
1444 int r;
1445
1446 pgprintk("%s: looking for gfn %lx\n", __func__, gfn);
1447 r = 0;
1448 index = kvm_page_table_hashfn(gfn);
1449 bucket = &kvm->arch.mmu_page_hash[index];
1450 hlist_for_each_entry_safe(sp, node, n, bucket, hash_link)
1451 if (sp->gfn == gfn && !sp->role.direct) {
1452 pgprintk("%s: gfn %lx role %x\n", __func__, gfn,
1453 sp->role.word);
1454 r = 1;
1455 if (kvm_mmu_zap_page(kvm, sp))
1456 n = bucket->first;
1457 }
1458 return r;
1459 }
1460
1461 static void mmu_unshadow(struct kvm *kvm, gfn_t gfn)
1462 {
1463 unsigned index;
1464 struct hlist_head *bucket;
1465 struct kvm_mmu_page *sp;
1466 struct hlist_node *node, *nn;
1467
1468 index = kvm_page_table_hashfn(gfn);
1469 bucket = &kvm->arch.mmu_page_hash[index];
1470 hlist_for_each_entry_safe(sp, node, nn, bucket, hash_link) {
1471 if (sp->gfn == gfn && !sp->role.direct
1472 && !sp->role.invalid) {
1473 pgprintk("%s: zap %lx %x\n",
1474 __func__, gfn, sp->role.word);
1475 kvm_mmu_zap_page(kvm, sp);
1476 }
1477 }
1478 }
1479
1480 static void page_header_update_slot(struct kvm *kvm, void *pte, gfn_t gfn)
1481 {
1482 int slot = memslot_id(kvm, gfn_to_memslot(kvm, gfn));
1483 struct kvm_mmu_page *sp = page_header(__pa(pte));
1484
1485 __set_bit(slot, sp->slot_bitmap);
1486 }
1487
1488 static void mmu_convert_notrap(struct kvm_mmu_page *sp)
1489 {
1490 int i;
1491 u64 *pt = sp->spt;
1492
1493 if (shadow_trap_nonpresent_pte == shadow_notrap_nonpresent_pte)
1494 return;
1495
1496 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
1497 if (pt[i] == shadow_notrap_nonpresent_pte)
1498 set_shadow_pte(&pt[i], shadow_trap_nonpresent_pte);
1499 }
1500 }
1501
1502 struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva)
1503 {
1504 struct page *page;
1505
1506 gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gva);
1507
1508 if (gpa == UNMAPPED_GVA)
1509 return NULL;
1510
1511 page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
1512
1513 return page;
1514 }
1515
1516 /*
1517 * The function is based on mtrr_type_lookup() in
1518 * arch/x86/kernel/cpu/mtrr/generic.c
1519 */
1520 static int get_mtrr_type(struct mtrr_state_type *mtrr_state,
1521 u64 start, u64 end)
1522 {
1523 int i;
1524 u64 base, mask;
1525 u8 prev_match, curr_match;
1526 int num_var_ranges = KVM_NR_VAR_MTRR;
1527
1528 if (!mtrr_state->enabled)
1529 return 0xFF;
1530
1531 /* Make end inclusive end, instead of exclusive */
1532 end--;
1533
1534 /* Look in fixed ranges. Just return the type as per start */
1535 if (mtrr_state->have_fixed && (start < 0x100000)) {
1536 int idx;
1537
1538 if (start < 0x80000) {
1539 idx = 0;
1540 idx += (start >> 16);
1541 return mtrr_state->fixed_ranges[idx];
1542 } else if (start < 0xC0000) {
1543 idx = 1 * 8;
1544 idx += ((start - 0x80000) >> 14);
1545 return mtrr_state->fixed_ranges[idx];
1546 } else if (start < 0x1000000) {
1547 idx = 3 * 8;
1548 idx += ((start - 0xC0000) >> 12);
1549 return mtrr_state->fixed_ranges[idx];
1550 }
1551 }
1552
1553 /*
1554 * Look in variable ranges
1555 * Look of multiple ranges matching this address and pick type
1556 * as per MTRR precedence
1557 */
1558 if (!(mtrr_state->enabled & 2))
1559 return mtrr_state->def_type;
1560
1561 prev_match = 0xFF;
1562 for (i = 0; i < num_var_ranges; ++i) {
1563 unsigned short start_state, end_state;
1564
1565 if (!(mtrr_state->var_ranges[i].mask_lo & (1 << 11)))
1566 continue;
1567
1568 base = (((u64)mtrr_state->var_ranges[i].base_hi) << 32) +
1569 (mtrr_state->var_ranges[i].base_lo & PAGE_MASK);
1570 mask = (((u64)mtrr_state->var_ranges[i].mask_hi) << 32) +
1571 (mtrr_state->var_ranges[i].mask_lo & PAGE_MASK);
1572
1573 start_state = ((start & mask) == (base & mask));
1574 end_state = ((end & mask) == (base & mask));
1575 if (start_state != end_state)
1576 return 0xFE;
1577
1578 if ((start & mask) != (base & mask))
1579 continue;
1580
1581 curr_match = mtrr_state->var_ranges[i].base_lo & 0xff;
1582 if (prev_match == 0xFF) {
1583 prev_match = curr_match;
1584 continue;
1585 }
1586
1587 if (prev_match == MTRR_TYPE_UNCACHABLE ||
1588 curr_match == MTRR_TYPE_UNCACHABLE)
1589 return MTRR_TYPE_UNCACHABLE;
1590
1591 if ((prev_match == MTRR_TYPE_WRBACK &&
1592 curr_match == MTRR_TYPE_WRTHROUGH) ||
1593 (prev_match == MTRR_TYPE_WRTHROUGH &&
1594 curr_match == MTRR_TYPE_WRBACK)) {
1595 prev_match = MTRR_TYPE_WRTHROUGH;
1596 curr_match = MTRR_TYPE_WRTHROUGH;
1597 }
1598
1599 if (prev_match != curr_match)
1600 return MTRR_TYPE_UNCACHABLE;
1601 }
1602
1603 if (prev_match != 0xFF)
1604 return prev_match;
1605
1606 return mtrr_state->def_type;
1607 }
1608
1609 u8 kvm_get_guest_memory_type(struct kvm_vcpu *vcpu, gfn_t gfn)
1610 {
1611 u8 mtrr;
1612
1613 mtrr = get_mtrr_type(&vcpu->arch.mtrr_state, gfn << PAGE_SHIFT,
1614 (gfn << PAGE_SHIFT) + PAGE_SIZE);
1615 if (mtrr == 0xfe || mtrr == 0xff)
1616 mtrr = MTRR_TYPE_WRBACK;
1617 return mtrr;
1618 }
1619 EXPORT_SYMBOL_GPL(kvm_get_guest_memory_type);
1620
1621 static int kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
1622 {
1623 unsigned index;
1624 struct hlist_head *bucket;
1625 struct kvm_mmu_page *s;
1626 struct hlist_node *node, *n;
1627
1628 index = kvm_page_table_hashfn(sp->gfn);
1629 bucket = &vcpu->kvm->arch.mmu_page_hash[index];
1630 /* don't unsync if pagetable is shadowed with multiple roles */
1631 hlist_for_each_entry_safe(s, node, n, bucket, hash_link) {
1632 if (s->gfn != sp->gfn || s->role.direct)
1633 continue;
1634 if (s->role.word != sp->role.word)
1635 return 1;
1636 }
1637 ++vcpu->kvm->stat.mmu_unsync;
1638 sp->unsync = 1;
1639
1640 kvm_mmu_mark_parents_unsync(vcpu, sp);
1641
1642 mmu_convert_notrap(sp);
1643 return 0;
1644 }
1645
1646 static int mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn,
1647 bool can_unsync)
1648 {
1649 struct kvm_mmu_page *shadow;
1650
1651 shadow = kvm_mmu_lookup_page(vcpu->kvm, gfn);
1652 if (shadow) {
1653 if (shadow->role.level != PT_PAGE_TABLE_LEVEL)
1654 return 1;
1655 if (shadow->unsync)
1656 return 0;
1657 if (can_unsync && oos_shadow)
1658 return kvm_unsync_page(vcpu, shadow);
1659 return 1;
1660 }
1661 return 0;
1662 }
1663
1664 static int set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte,
1665 unsigned pte_access, int user_fault,
1666 int write_fault, int dirty, int largepage,
1667 gfn_t gfn, pfn_t pfn, bool speculative,
1668 bool can_unsync)
1669 {
1670 u64 spte;
1671 int ret = 0;
1672
1673 /*
1674 * We don't set the accessed bit, since we sometimes want to see
1675 * whether the guest actually used the pte (in order to detect
1676 * demand paging).
1677 */
1678 spte = shadow_base_present_pte | shadow_dirty_mask;
1679 if (!speculative)
1680 spte |= shadow_accessed_mask;
1681 if (!dirty)
1682 pte_access &= ~ACC_WRITE_MASK;
1683 if (pte_access & ACC_EXEC_MASK)
1684 spte |= shadow_x_mask;
1685 else
1686 spte |= shadow_nx_mask;
1687 if (pte_access & ACC_USER_MASK)
1688 spte |= shadow_user_mask;
1689 if (largepage)
1690 spte |= PT_PAGE_SIZE_MASK;
1691 if (tdp_enabled)
1692 spte |= kvm_x86_ops->get_mt_mask(vcpu, gfn,
1693 kvm_is_mmio_pfn(pfn));
1694
1695 spte |= (u64)pfn << PAGE_SHIFT;
1696
1697 if ((pte_access & ACC_WRITE_MASK)
1698 || (write_fault && !is_write_protection(vcpu) && !user_fault)) {
1699
1700 if (largepage && has_wrprotected_page(vcpu->kvm, gfn)) {
1701 ret = 1;
1702 spte = shadow_trap_nonpresent_pte;
1703 goto set_pte;
1704 }
1705
1706 spte |= PT_WRITABLE_MASK;
1707
1708 /*
1709 * Optimization: for pte sync, if spte was writable the hash
1710 * lookup is unnecessary (and expensive). Write protection
1711 * is responsibility of mmu_get_page / kvm_sync_page.
1712 * Same reasoning can be applied to dirty page accounting.
1713 */
1714 if (!can_unsync && is_writeble_pte(*shadow_pte))
1715 goto set_pte;
1716
1717 if (mmu_need_write_protect(vcpu, gfn, can_unsync)) {
1718 pgprintk("%s: found shadow page for %lx, marking ro\n",
1719 __func__, gfn);
1720 ret = 1;
1721 pte_access &= ~ACC_WRITE_MASK;
1722 if (is_writeble_pte(spte))
1723 spte &= ~PT_WRITABLE_MASK;
1724 }
1725 }
1726
1727 if (pte_access & ACC_WRITE_MASK)
1728 mark_page_dirty(vcpu->kvm, gfn);
1729
1730 set_pte:
1731 set_shadow_pte(shadow_pte, spte);
1732 return ret;
1733 }
1734
1735 static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte,
1736 unsigned pt_access, unsigned pte_access,
1737 int user_fault, int write_fault, int dirty,
1738 int *ptwrite, int largepage, gfn_t gfn,
1739 pfn_t pfn, bool speculative)
1740 {
1741 int was_rmapped = 0;
1742 int was_writeble = is_writeble_pte(*shadow_pte);
1743
1744 pgprintk("%s: spte %llx access %x write_fault %d"
1745 " user_fault %d gfn %lx\n",
1746 __func__, *shadow_pte, pt_access,
1747 write_fault, user_fault, gfn);
1748
1749 if (is_rmap_pte(*shadow_pte)) {
1750 /*
1751 * If we overwrite a PTE page pointer with a 2MB PMD, unlink
1752 * the parent of the now unreachable PTE.
1753 */
1754 if (largepage && !is_large_pte(*shadow_pte)) {
1755 struct kvm_mmu_page *child;
1756 u64 pte = *shadow_pte;
1757
1758 child = page_header(pte & PT64_BASE_ADDR_MASK);
1759 mmu_page_remove_parent_pte(child, shadow_pte);
1760 } else if (pfn != spte_to_pfn(*shadow_pte)) {
1761 pgprintk("hfn old %lx new %lx\n",
1762 spte_to_pfn(*shadow_pte), pfn);
1763 rmap_remove(vcpu->kvm, shadow_pte);
1764 } else
1765 was_rmapped = 1;
1766 }
1767 if (set_spte(vcpu, shadow_pte, pte_access, user_fault, write_fault,
1768 dirty, largepage, gfn, pfn, speculative, true)) {
1769 if (write_fault)
1770 *ptwrite = 1;
1771 kvm_x86_ops->tlb_flush(vcpu);
1772 }
1773
1774 pgprintk("%s: setting spte %llx\n", __func__, *shadow_pte);
1775 pgprintk("instantiating %s PTE (%s) at %ld (%llx) addr %p\n",
1776 is_large_pte(*shadow_pte)? "2MB" : "4kB",
1777 is_present_pte(*shadow_pte)?"RW":"R", gfn,
1778 *shadow_pte, shadow_pte);
1779 if (!was_rmapped && is_large_pte(*shadow_pte))
1780 ++vcpu->kvm->stat.lpages;
1781
1782 page_header_update_slot(vcpu->kvm, shadow_pte, gfn);
1783 if (!was_rmapped) {
1784 rmap_add(vcpu, shadow_pte, gfn, largepage);
1785 if (!is_rmap_pte(*shadow_pte))
1786 kvm_release_pfn_clean(pfn);
1787 } else {
1788 if (was_writeble)
1789 kvm_release_pfn_dirty(pfn);
1790 else
1791 kvm_release_pfn_clean(pfn);
1792 }
1793 if (speculative) {
1794 vcpu->arch.last_pte_updated = shadow_pte;
1795 vcpu->arch.last_pte_gfn = gfn;
1796 }
1797 }
1798
1799 static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
1800 {
1801 }
1802
1803 static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
1804 int largepage, gfn_t gfn, pfn_t pfn)
1805 {
1806 struct kvm_shadow_walk_iterator iterator;
1807 struct kvm_mmu_page *sp;
1808 int pt_write = 0;
1809 gfn_t pseudo_gfn;
1810
1811 for_each_shadow_entry(vcpu, (u64)gfn << PAGE_SHIFT, iterator) {
1812 if (iterator.level == PT_PAGE_TABLE_LEVEL
1813 || (largepage && iterator.level == PT_DIRECTORY_LEVEL)) {
1814 mmu_set_spte(vcpu, iterator.sptep, ACC_ALL, ACC_ALL,
1815 0, write, 1, &pt_write,
1816 largepage, gfn, pfn, false);
1817 ++vcpu->stat.pf_fixed;
1818 break;
1819 }
1820
1821 if (*iterator.sptep == shadow_trap_nonpresent_pte) {
1822 pseudo_gfn = (iterator.addr & PT64_DIR_BASE_ADDR_MASK) >> PAGE_SHIFT;
1823 sp = kvm_mmu_get_page(vcpu, pseudo_gfn, iterator.addr,
1824 iterator.level - 1,
1825 1, ACC_ALL, iterator.sptep);
1826 if (!sp) {
1827 pgprintk("nonpaging_map: ENOMEM\n");
1828 kvm_release_pfn_clean(pfn);
1829 return -ENOMEM;
1830 }
1831
1832 set_shadow_pte(iterator.sptep,
1833 __pa(sp->spt)
1834 | PT_PRESENT_MASK | PT_WRITABLE_MASK
1835 | shadow_user_mask | shadow_x_mask);
1836 }
1837 }
1838 return pt_write;
1839 }
1840
1841 static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn)
1842 {
1843 int r;
1844 int largepage = 0;
1845 pfn_t pfn;
1846 unsigned long mmu_seq;
1847
1848 if (is_largepage_backed(vcpu, gfn & ~(KVM_PAGES_PER_HPAGE-1))) {
1849 gfn &= ~(KVM_PAGES_PER_HPAGE-1);
1850 largepage = 1;
1851 }
1852
1853 mmu_seq = vcpu->kvm->mmu_notifier_seq;
1854 smp_rmb();
1855 pfn = gfn_to_pfn(vcpu->kvm, gfn);
1856
1857 /* mmio */
1858 if (is_error_pfn(pfn)) {
1859 kvm_release_pfn_clean(pfn);
1860 return 1;
1861 }
1862
1863 spin_lock(&vcpu->kvm->mmu_lock);
1864 if (mmu_notifier_retry(vcpu, mmu_seq))
1865 goto out_unlock;
1866 kvm_mmu_free_some_pages(vcpu);
1867 r = __direct_map(vcpu, v, write, largepage, gfn, pfn);
1868 spin_unlock(&vcpu->kvm->mmu_lock);
1869
1870
1871 return r;
1872
1873 out_unlock:
1874 spin_unlock(&vcpu->kvm->mmu_lock);
1875 kvm_release_pfn_clean(pfn);
1876 return 0;
1877 }
1878
1879
1880 static void mmu_free_roots(struct kvm_vcpu *vcpu)
1881 {
1882 int i;
1883 struct kvm_mmu_page *sp;
1884
1885 if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
1886 return;
1887 spin_lock(&vcpu->kvm->mmu_lock);
1888 if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
1889 hpa_t root = vcpu->arch.mmu.root_hpa;
1890
1891 sp = page_header(root);
1892 --sp->root_count;
1893 if (!sp->root_count && sp->role.invalid)
1894 kvm_mmu_zap_page(vcpu->kvm, sp);
1895 vcpu->arch.mmu.root_hpa = INVALID_PAGE;
1896 spin_unlock(&vcpu->kvm->mmu_lock);
1897 return;
1898 }
1899 for (i = 0; i < 4; ++i) {
1900 hpa_t root = vcpu->arch.mmu.pae_root[i];
1901
1902 if (root) {
1903 root &= PT64_BASE_ADDR_MASK;
1904 sp = page_header(root);
1905 --sp->root_count;
1906 if (!sp->root_count && sp->role.invalid)
1907 kvm_mmu_zap_page(vcpu->kvm, sp);
1908 }
1909 vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
1910 }
1911 spin_unlock(&vcpu->kvm->mmu_lock);
1912 vcpu->arch.mmu.root_hpa = INVALID_PAGE;
1913 }
1914
1915 static int mmu_check_root(struct kvm_vcpu *vcpu, gfn_t root_gfn)
1916 {
1917 int ret = 0;
1918
1919 if (!kvm_is_visible_gfn(vcpu->kvm, root_gfn)) {
1920 set_bit(KVM_REQ_TRIPLE_FAULT, &vcpu->requests);
1921 ret = 1;
1922 }
1923
1924 return ret;
1925 }
1926
1927 static int mmu_alloc_roots(struct kvm_vcpu *vcpu)
1928 {
1929 int i;
1930 gfn_t root_gfn;
1931 struct kvm_mmu_page *sp;
1932 int direct = 0;
1933
1934 root_gfn = vcpu->arch.cr3 >> PAGE_SHIFT;
1935
1936 if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
1937 hpa_t root = vcpu->arch.mmu.root_hpa;
1938
1939 ASSERT(!VALID_PAGE(root));
1940 if (tdp_enabled)
1941 direct = 1;
1942 if (mmu_check_root(vcpu, root_gfn))
1943 return 1;
1944 sp = kvm_mmu_get_page(vcpu, root_gfn, 0,
1945 PT64_ROOT_LEVEL, direct,
1946 ACC_ALL, NULL);
1947 root = __pa(sp->spt);
1948 ++sp->root_count;
1949 vcpu->arch.mmu.root_hpa = root;
1950 return 0;
1951 }
1952 direct = !is_paging(vcpu);
1953 if (tdp_enabled)
1954 direct = 1;
1955 for (i = 0; i < 4; ++i) {
1956 hpa_t root = vcpu->arch.mmu.pae_root[i];
1957
1958 ASSERT(!VALID_PAGE(root));
1959 if (vcpu->arch.mmu.root_level == PT32E_ROOT_LEVEL) {
1960 if (!is_present_pte(vcpu->arch.pdptrs[i])) {
1961 vcpu->arch.mmu.pae_root[i] = 0;
1962 continue;
1963 }
1964 root_gfn = vcpu->arch.pdptrs[i] >> PAGE_SHIFT;
1965 } else if (vcpu->arch.mmu.root_level == 0)
1966 root_gfn = 0;
1967 if (mmu_check_root(vcpu, root_gfn))
1968 return 1;
1969 sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
1970 PT32_ROOT_LEVEL, direct,
1971 ACC_ALL, NULL);
1972 root = __pa(sp->spt);
1973 ++sp->root_count;
1974 vcpu->arch.mmu.pae_root[i] = root | PT_PRESENT_MASK;
1975 }
1976 vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
1977 return 0;
1978 }
1979
1980 static void mmu_sync_roots(struct kvm_vcpu *vcpu)
1981 {
1982 int i;
1983 struct kvm_mmu_page *sp;
1984
1985 if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
1986 return;
1987 if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
1988 hpa_t root = vcpu->arch.mmu.root_hpa;
1989 sp = page_header(root);
1990 mmu_sync_children(vcpu, sp);
1991 return;
1992 }
1993 for (i = 0; i < 4; ++i) {
1994 hpa_t root = vcpu->arch.mmu.pae_root[i];
1995
1996 if (root && VALID_PAGE(root)) {
1997 root &= PT64_BASE_ADDR_MASK;
1998 sp = page_header(root);
1999 mmu_sync_children(vcpu, sp);
2000 }
2001 }
2002 }
2003
2004 void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
2005 {
2006 spin_lock(&vcpu->kvm->mmu_lock);
2007 mmu_sync_roots(vcpu);
2008 spin_unlock(&vcpu->kvm->mmu_lock);
2009 }
2010
2011 static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr)
2012 {
2013 return vaddr;
2014 }
2015
2016 static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
2017 u32 error_code)
2018 {
2019 gfn_t gfn;
2020 int r;
2021
2022 pgprintk("%s: gva %lx error %x\n", __func__, gva, error_code);
2023 r = mmu_topup_memory_caches(vcpu);
2024 if (r)
2025 return r;
2026
2027 ASSERT(vcpu);
2028 ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));
2029
2030 gfn = gva >> PAGE_SHIFT;
2031
2032 return nonpaging_map(vcpu, gva & PAGE_MASK,
2033 error_code & PFERR_WRITE_MASK, gfn);
2034 }
2035
2036 static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa,
2037 u32 error_code)
2038 {
2039 pfn_t pfn;
2040 int r;
2041 int largepage = 0;
2042 gfn_t gfn = gpa >> PAGE_SHIFT;
2043 unsigned long mmu_seq;
2044
2045 ASSERT(vcpu);
2046 ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));
2047
2048 r = mmu_topup_memory_caches(vcpu);
2049 if (r)
2050 return r;
2051
2052 if (is_largepage_backed(vcpu, gfn & ~(KVM_PAGES_PER_HPAGE-1))) {
2053 gfn &= ~(KVM_PAGES_PER_HPAGE-1);
2054 largepage = 1;
2055 }
2056 mmu_seq = vcpu->kvm->mmu_notifier_seq;
2057 smp_rmb();
2058 pfn = gfn_to_pfn(vcpu->kvm, gfn);
2059 if (is_error_pfn(pfn)) {
2060 kvm_release_pfn_clean(pfn);
2061 return 1;
2062 }
2063 spin_lock(&vcpu->kvm->mmu_lock);
2064 if (mmu_notifier_retry(vcpu, mmu_seq))
2065 goto out_unlock;
2066 kvm_mmu_free_some_pages(vcpu);
2067 r = __direct_map(vcpu, gpa, error_code & PFERR_WRITE_MASK,
2068 largepage, gfn, pfn);
2069 spin_unlock(&vcpu->kvm->mmu_lock);
2070
2071 return r;
2072
2073 out_unlock:
2074 spin_unlock(&vcpu->kvm->mmu_lock);
2075 kvm_release_pfn_clean(pfn);
2076 return 0;
2077 }
2078
2079 static void nonpaging_free(struct kvm_vcpu *vcpu)
2080 {
2081 mmu_free_roots(vcpu);
2082 }
2083
2084 static int nonpaging_init_context(struct kvm_vcpu *vcpu)
2085 {
2086 struct kvm_mmu *context = &vcpu->arch.mmu;
2087
2088 context->new_cr3 = nonpaging_new_cr3;
2089 context->page_fault = nonpaging_page_fault;
2090 context->gva_to_gpa = nonpaging_gva_to_gpa;
2091 context->free = nonpaging_free;
2092 context->prefetch_page = nonpaging_prefetch_page;
2093 context->sync_page = nonpaging_sync_page;
2094 context->invlpg = nonpaging_invlpg;
2095 context->root_level = 0;
2096 context->shadow_root_level = PT32E_ROOT_LEVEL;
2097 context->root_hpa = INVALID_PAGE;
2098 return 0;
2099 }
2100
2101 void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)
2102 {
2103 ++vcpu->stat.tlb_flush;
2104 kvm_x86_ops->tlb_flush(vcpu);
2105 }
2106
2107 static void paging_new_cr3(struct kvm_vcpu *vcpu)
2108 {
2109 pgprintk("%s: cr3 %lx\n", __func__, vcpu->arch.cr3);
2110 mmu_free_roots(vcpu);
2111 }
2112
2113 static void inject_page_fault(struct kvm_vcpu *vcpu,
2114 u64 addr,
2115 u32 err_code)
2116 {
2117 kvm_inject_page_fault(vcpu, addr, err_code);
2118 }
2119
2120 static void paging_free(struct kvm_vcpu *vcpu)
2121 {
2122 nonpaging_free(vcpu);
2123 }
2124
2125 static bool is_rsvd_bits_set(struct kvm_vcpu *vcpu, u64 gpte, int level)
2126 {
2127 int bit7;
2128
2129 bit7 = (gpte >> 7) & 1;
2130 return (gpte & vcpu->arch.mmu.rsvd_bits_mask[bit7][level-1]) != 0;
2131 }
2132
2133 #define PTTYPE 64
2134 #include "paging_tmpl.h"
2135 #undef PTTYPE
2136
2137 #define PTTYPE 32
2138 #include "paging_tmpl.h"
2139 #undef PTTYPE
2140
2141 static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu, int level)
2142 {
2143 struct kvm_mmu *context = &vcpu->arch.mmu;
2144 int maxphyaddr = cpuid_maxphyaddr(vcpu);
2145 u64 exb_bit_rsvd = 0;
2146
2147 if (!is_nx(vcpu))
2148 exb_bit_rsvd = rsvd_bits(63, 63);
2149 switch (level) {
2150 case PT32_ROOT_LEVEL:
2151 /* no rsvd bits for 2 level 4K page table entries */
2152 context->rsvd_bits_mask[0][1] = 0;
2153 context->rsvd_bits_mask[0][0] = 0;
2154 if (is_cpuid_PSE36())
2155 /* 36bits PSE 4MB page */
2156 context->rsvd_bits_mask[1][1] = rsvd_bits(17, 21);
2157 else
2158 /* 32 bits PSE 4MB page */
2159 context->rsvd_bits_mask[1][1] = rsvd_bits(13, 21);
2160 context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[1][0];
2161 break;
2162 case PT32E_ROOT_LEVEL:
2163 context->rsvd_bits_mask[0][2] =
2164 rsvd_bits(maxphyaddr, 63) |
2165 rsvd_bits(7, 8) | rsvd_bits(1, 2); /* PDPTE */
2166 context->rsvd_bits_mask[0][1] = exb_bit_rsvd |
2167 rsvd_bits(maxphyaddr, 62); /* PDE */
2168 context->rsvd_bits_mask[0][0] = exb_bit_rsvd |
2169 rsvd_bits(maxphyaddr, 62); /* PTE */
2170 context->rsvd_bits_mask[1][1] = exb_bit_rsvd |
2171 rsvd_bits(maxphyaddr, 62) |
2172 rsvd_bits(13, 20); /* large page */
2173 context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[1][0];
2174 break;
2175 case PT64_ROOT_LEVEL:
2176 context->rsvd_bits_mask[0][3] = exb_bit_rsvd |
2177 rsvd_bits(maxphyaddr, 51) | rsvd_bits(7, 8);
2178 context->rsvd_bits_mask[0][2] = exb_bit_rsvd |
2179 rsvd_bits(maxphyaddr, 51) | rsvd_bits(7, 8);
2180 context->rsvd_bits_mask[0][1] = exb_bit_rsvd |
2181 rsvd_bits(maxphyaddr, 51);
2182 context->rsvd_bits_mask[0][0] = exb_bit_rsvd |
2183 rsvd_bits(maxphyaddr, 51);
2184 context->rsvd_bits_mask[1][3] = context->rsvd_bits_mask[0][3];
2185 context->rsvd_bits_mask[1][2] = context->rsvd_bits_mask[0][2];
2186 context->rsvd_bits_mask[1][1] = exb_bit_rsvd |
2187 rsvd_bits(maxphyaddr, 51) |
2188 rsvd_bits(13, 20); /* large page */
2189 context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[1][0];
2190 break;
2191 }
2192 }
2193
2194 static int paging64_init_context_common(struct kvm_vcpu *vcpu, int level)
2195 {
2196 struct kvm_mmu *context = &vcpu->arch.mmu;
2197
2198 ASSERT(is_pae(vcpu));
2199 context->new_cr3 = paging_new_cr3;
2200 context->page_fault = paging64_page_fault;
2201 context->gva_to_gpa = paging64_gva_to_gpa;
2202 context->prefetch_page = paging64_prefetch_page;
2203 context->sync_page = paging64_sync_page;
2204 context->invlpg = paging64_invlpg;
2205 context->free = paging_free;
2206 context->root_level = level;
2207 context->shadow_root_level = level;
2208 context->root_hpa = INVALID_PAGE;
2209 return 0;
2210 }
2211
2212 static int paging64_init_context(struct kvm_vcpu *vcpu)
2213 {
2214 reset_rsvds_bits_mask(vcpu, PT64_ROOT_LEVEL);
2215 return paging64_init_context_common(vcpu, PT64_ROOT_LEVEL);
2216 }
2217
2218 static int paging32_init_context(struct kvm_vcpu *vcpu)
2219 {
2220 struct kvm_mmu *context = &vcpu->arch.mmu;
2221
2222 reset_rsvds_bits_mask(vcpu, PT32_ROOT_LEVEL);
2223 context->new_cr3 = paging_new_cr3;
2224 context->page_fault = paging32_page_fault;
2225 context->gva_to_gpa = paging32_gva_to_gpa;
2226 context->free = paging_free;
2227 context->prefetch_page = paging32_prefetch_page;
2228 context->sync_page = paging32_sync_page;
2229 context->invlpg = paging32_invlpg;
2230 context->root_level = PT32_ROOT_LEVEL;
2231 context->shadow_root_level = PT32E_ROOT_LEVEL;
2232 context->root_hpa = INVALID_PAGE;
2233 return 0;
2234 }
2235
2236 static int paging32E_init_context(struct kvm_vcpu *vcpu)
2237 {
2238 reset_rsvds_bits_mask(vcpu, PT32E_ROOT_LEVEL);
2239 return paging64_init_context_common(vcpu, PT32E_ROOT_LEVEL);
2240 }
2241
2242 static int init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
2243 {
2244 struct kvm_mmu *context = &vcpu->arch.mmu;
2245
2246 context->new_cr3 = nonpaging_new_cr3;
2247 context->page_fault = tdp_page_fault;
2248 context->free = nonpaging_free;
2249 context->prefetch_page = nonpaging_prefetch_page;
2250 context->sync_page = nonpaging_sync_page;
2251 context->invlpg = nonpaging_invlpg;
2252 context->shadow_root_level = kvm_x86_ops->get_tdp_level();
2253 context->root_hpa = INVALID_PAGE;
2254
2255 if (!is_paging(vcpu)) {
2256 context->gva_to_gpa = nonpaging_gva_to_gpa;
2257 context->root_level = 0;
2258 } else if (is_long_mode(vcpu)) {
2259 reset_rsvds_bits_mask(vcpu, PT64_ROOT_LEVEL);
2260 context->gva_to_gpa = paging64_gva_to_gpa;
2261 context->root_level = PT64_ROOT_LEVEL;
2262 } else if (is_pae(vcpu)) {
2263 reset_rsvds_bits_mask(vcpu, PT32E_ROOT_LEVEL);
2264 context->gva_to_gpa = paging64_gva_to_gpa;
2265 context->root_level = PT32E_ROOT_LEVEL;
2266 } else {
2267 reset_rsvds_bits_mask(vcpu, PT32_ROOT_LEVEL);
2268 context->gva_to_gpa = paging32_gva_to_gpa;
2269 context->root_level = PT32_ROOT_LEVEL;
2270 }
2271
2272 return 0;
2273 }
2274
2275 static int init_kvm_softmmu(struct kvm_vcpu *vcpu)
2276 {
2277 int r;
2278
2279 ASSERT(vcpu);
2280 ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
2281
2282 if (!is_paging(vcpu))
2283 r = nonpaging_init_context(vcpu);
2284 else if (is_long_mode(vcpu))
2285 r = paging64_init_context(vcpu);
2286 else if (is_pae(vcpu))
2287 r = paging32E_init_context(vcpu);
2288 else
2289 r = paging32_init_context(vcpu);
2290
2291 vcpu->arch.mmu.base_role.glevels = vcpu->arch.mmu.root_level;
2292
2293 return r;
2294 }
2295
2296 static int init_kvm_mmu(struct kvm_vcpu *vcpu)
2297 {
2298 vcpu->arch.update_pte.pfn = bad_pfn;
2299
2300 if (tdp_enabled)
2301 return init_kvm_tdp_mmu(vcpu);
2302 else
2303 return init_kvm_softmmu(vcpu);
2304 }
2305
2306 static void destroy_kvm_mmu(struct kvm_vcpu *vcpu)
2307 {
2308 ASSERT(vcpu);
2309 if (VALID_PAGE(vcpu->arch.mmu.root_hpa)) {
2310 vcpu->arch.mmu.free(vcpu);
2311 vcpu->arch.mmu.root_hpa = INVALID_PAGE;
2312 }
2313 }
2314
2315 int kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
2316 {
2317 destroy_kvm_mmu(vcpu);
2318 return init_kvm_mmu(vcpu);
2319 }
2320 EXPORT_SYMBOL_GPL(kvm_mmu_reset_context);
2321
2322 int kvm_mmu_load(struct kvm_vcpu *vcpu)
2323 {
2324 int r;
2325
2326 r = mmu_topup_memory_caches(vcpu);
2327 if (r)
2328 goto out;
2329 spin_lock(&vcpu->kvm->mmu_lock);
2330 kvm_mmu_free_some_pages(vcpu);
2331 r = mmu_alloc_roots(vcpu);
2332 mmu_sync_roots(vcpu);
2333 spin_unlock(&vcpu->kvm->mmu_lock);
2334 if (r)
2335 goto out;
2336 kvm_x86_ops->set_cr3(vcpu, vcpu->arch.mmu.root_hpa);
2337 kvm_mmu_flush_tlb(vcpu);
2338 out:
2339 return r;
2340 }
2341 EXPORT_SYMBOL_GPL(kvm_mmu_load);
2342
2343 void kvm_mmu_unload(struct kvm_vcpu *vcpu)
2344 {
2345 mmu_free_roots(vcpu);
2346 }
2347
2348 static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu,
2349 struct kvm_mmu_page *sp,
2350 u64 *spte)
2351 {
2352 u64 pte;
2353 struct kvm_mmu_page *child;
2354
2355 pte = *spte;
2356 if (is_shadow_present_pte(pte)) {
2357 if (sp->role.level == PT_PAGE_TABLE_LEVEL ||
2358 is_large_pte(pte))
2359 rmap_remove(vcpu->kvm, spte);
2360 else {
2361 child = page_header(pte & PT64_BASE_ADDR_MASK);
2362 mmu_page_remove_parent_pte(child, spte);
2363 }
2364 }
2365 set_shadow_pte(spte, shadow_trap_nonpresent_pte);
2366 if (is_large_pte(pte))
2367 --vcpu->kvm->stat.lpages;
2368 }
2369
2370 static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
2371 struct kvm_mmu_page *sp,
2372 u64 *spte,
2373 const void *new)
2374 {
2375 if (sp->role.level != PT_PAGE_TABLE_LEVEL) {
2376 if (!vcpu->arch.update_pte.largepage ||
2377 sp->role.glevels == PT32_ROOT_LEVEL) {
2378 ++vcpu->kvm->stat.mmu_pde_zapped;
2379 return;
2380 }
2381 }
2382
2383 ++vcpu->kvm->stat.mmu_pte_updated;
2384 if (sp->role.glevels == PT32_ROOT_LEVEL)
2385 paging32_update_pte(vcpu, sp, spte, new);
2386 else
2387 paging64_update_pte(vcpu, sp, spte, new);
2388 }
2389
2390 static bool need_remote_flush(u64 old, u64 new)
2391 {
2392 if (!is_shadow_present_pte(old))
2393 return false;
2394 if (!is_shadow_present_pte(new))
2395 return true;
2396 if ((old ^ new) & PT64_BASE_ADDR_MASK)
2397 return true;
2398 old ^= PT64_NX_MASK;
2399 new ^= PT64_NX_MASK;
2400 return (old & ~new & PT64_PERM_MASK) != 0;
2401 }
2402
2403 static void mmu_pte_write_flush_tlb(struct kvm_vcpu *vcpu, u64 old, u64 new)
2404 {
2405 if (need_remote_flush(old, new))
2406 kvm_flush_remote_tlbs(vcpu->kvm);
2407 else
2408 kvm_mmu_flush_tlb(vcpu);
2409 }
2410
2411 static bool last_updated_pte_accessed(struct kvm_vcpu *vcpu)
2412 {
2413 u64 *spte = vcpu->arch.last_pte_updated;
2414
2415 return !!(spte && (*spte & shadow_accessed_mask));
2416 }
2417
2418 static void mmu_guess_page_from_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
2419 const u8 *new, int bytes)
2420 {
2421 gfn_t gfn;
2422 int r;
2423 u64 gpte = 0;
2424 pfn_t pfn;
2425
2426 vcpu->arch.update_pte.largepage = 0;
2427
2428 if (bytes != 4 && bytes != 8)
2429 return;
2430
2431 /*
2432 * Assume that the pte write on a page table of the same type
2433 * as the current vcpu paging mode. This is nearly always true
2434 * (might be false while changing modes). Note it is verified later
2435 * by update_pte().
2436 */
2437 if (is_pae(vcpu)) {
2438 /* Handle a 32-bit guest writing two halves of a 64-bit gpte */
2439 if ((bytes == 4) && (gpa % 4 == 0)) {
2440 r = kvm_read_guest(vcpu->kvm, gpa & ~(u64)7, &gpte, 8);
2441 if (r)
2442 return;
2443 memcpy((void *)&gpte + (gpa % 8), new, 4);
2444 } else if ((bytes == 8) && (gpa % 8 == 0)) {
2445 memcpy((void *)&gpte, new, 8);
2446 }
2447 } else {
2448 if ((bytes == 4) && (gpa % 4 == 0))
2449 memcpy((void *)&gpte, new, 4);
2450 }
2451 if (!is_present_pte(gpte))
2452 return;
2453 gfn = (gpte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
2454
2455 if (is_large_pte(gpte) && is_largepage_backed(vcpu, gfn)) {
2456 gfn &= ~(KVM_PAGES_PER_HPAGE-1);
2457 vcpu->arch.update_pte.largepage = 1;
2458 }
2459 vcpu->arch.update_pte.mmu_seq = vcpu->kvm->mmu_notifier_seq;
2460 smp_rmb();
2461 pfn = gfn_to_pfn(vcpu->kvm, gfn);
2462
2463 if (is_error_pfn(pfn)) {
2464 kvm_release_pfn_clean(pfn);
2465 return;
2466 }
2467 vcpu->arch.update_pte.gfn = gfn;
2468 vcpu->arch.update_pte.pfn = pfn;
2469 }
2470
2471 static void kvm_mmu_access_page(struct kvm_vcpu *vcpu, gfn_t gfn)
2472 {
2473 u64 *spte = vcpu->arch.last_pte_updated;
2474
2475 if (spte
2476 && vcpu->arch.last_pte_gfn == gfn
2477 && shadow_accessed_mask
2478 && !(*spte & shadow_accessed_mask)
2479 && is_shadow_present_pte(*spte))
2480 set_bit(PT_ACCESSED_SHIFT, (unsigned long *)spte);
2481 }
2482
2483 void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
2484 const u8 *new, int bytes,
2485 bool guest_initiated)
2486 {
2487 gfn_t gfn = gpa >> PAGE_SHIFT;
2488 struct kvm_mmu_page *sp;
2489 struct hlist_node *node, *n;
2490 struct hlist_head *bucket;
2491 unsigned index;
2492 u64 entry, gentry;
2493 u64 *spte;
2494 unsigned offset = offset_in_page(gpa);
2495 unsigned pte_size;
2496 unsigned page_offset;
2497 unsigned misaligned;
2498 unsigned quadrant;
2499 int level;
2500 int flooded = 0;
2501 int npte;
2502 int r;
2503
2504 pgprintk("%s: gpa %llx bytes %d\n", __func__, gpa, bytes);
2505 mmu_guess_page_from_pte_write(vcpu, gpa, new, bytes);
2506 spin_lock(&vcpu->kvm->mmu_lock);
2507 kvm_mmu_access_page(vcpu, gfn);
2508 kvm_mmu_free_some_pages(vcpu);
2509 ++vcpu->kvm->stat.mmu_pte_write;
2510 kvm_mmu_audit(vcpu, "pre pte write");
2511 if (guest_initiated) {
2512 if (gfn == vcpu->arch.last_pt_write_gfn
2513 && !last_updated_pte_accessed(vcpu)) {
2514 ++vcpu->arch.last_pt_write_count;
2515 if (vcpu->arch.last_pt_write_count >= 3)
2516 flooded = 1;
2517 } else {
2518 vcpu->arch.last_pt_write_gfn = gfn;
2519 vcpu->arch.last_pt_write_count = 1;
2520 vcpu->arch.last_pte_updated = NULL;
2521 }
2522 }
2523 index = kvm_page_table_hashfn(gfn);
2524 bucket = &vcpu->kvm->arch.mmu_page_hash[index];
2525 hlist_for_each_entry_safe(sp, node, n, bucket, hash_link) {
2526 if (sp->gfn != gfn || sp->role.direct || sp->role.invalid)
2527 continue;
2528 pte_size = sp->role.glevels == PT32_ROOT_LEVEL ? 4 : 8;
2529 misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
2530 misaligned |= bytes < 4;
2531 if (misaligned || flooded) {
2532 /*
2533 * Misaligned accesses are too much trouble to fix
2534 * up; also, they usually indicate a page is not used
2535 * as a page table.
2536 *
2537 * If we're seeing too many writes to a page,
2538 * it may no longer be a page table, or we may be
2539 * forking, in which case it is better to unmap the
2540 * page.
2541 */
2542 pgprintk("misaligned: gpa %llx bytes %d role %x\n",
2543 gpa, bytes, sp->role.word);
2544 if (kvm_mmu_zap_page(vcpu->kvm, sp))
2545 n = bucket->first;
2546 ++vcpu->kvm->stat.mmu_flooded;
2547 continue;
2548 }
2549 page_offset = offset;
2550 level = sp->role.level;
2551 npte = 1;
2552 if (sp->role.glevels == PT32_ROOT_LEVEL) {
2553 page_offset <<= 1; /* 32->64 */
2554 /*
2555 * A 32-bit pde maps 4MB while the shadow pdes map
2556 * only 2MB. So we need to double the offset again
2557 * and zap two pdes instead of one.
2558 */
2559 if (level == PT32_ROOT_LEVEL) {
2560 page_offset &= ~7; /* kill rounding error */
2561 page_offset <<= 1;
2562 npte = 2;
2563 }
2564 quadrant = page_offset >> PAGE_SHIFT;
2565 page_offset &= ~PAGE_MASK;
2566 if (quadrant != sp->role.quadrant)
2567 continue;
2568 }
2569 spte = &sp->spt[page_offset / sizeof(*spte)];
2570 if ((gpa & (pte_size - 1)) || (bytes < pte_size)) {
2571 gentry = 0;
2572 r = kvm_read_guest_atomic(vcpu->kvm,
2573 gpa & ~(u64)(pte_size - 1),
2574 &gentry, pte_size);
2575 new = (const void *)&gentry;
2576 if (r < 0)
2577 new = NULL;
2578 }
2579 while (npte--) {
2580 entry = *spte;
2581 mmu_pte_write_zap_pte(vcpu, sp, spte);
2582 if (new)
2583 mmu_pte_write_new_pte(vcpu, sp, spte, new);
2584 mmu_pte_write_flush_tlb(vcpu, entry, *spte);
2585 ++spte;
2586 }
2587 }
2588 kvm_mmu_audit(vcpu, "post pte write");
2589 spin_unlock(&vcpu->kvm->mmu_lock);
2590 if (!is_error_pfn(vcpu->arch.update_pte.pfn)) {
2591 kvm_release_pfn_clean(vcpu->arch.update_pte.pfn);
2592 vcpu->arch.update_pte.pfn = bad_pfn;
2593 }
2594 }
2595
2596 int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
2597 {
2598 gpa_t gpa;
2599 int r;
2600
2601 gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gva);
2602
2603 spin_lock(&vcpu->kvm->mmu_lock);
2604 r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
2605 spin_unlock(&vcpu->kvm->mmu_lock);
2606 return r;
2607 }
2608 EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page_virt);
2609
2610 void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu)
2611 {
2612 while (vcpu->kvm->arch.n_free_mmu_pages < KVM_REFILL_PAGES) {
2613 struct kvm_mmu_page *sp;
2614
2615 sp = container_of(vcpu->kvm->arch.active_mmu_pages.prev,
2616 struct kvm_mmu_page, link);
2617 kvm_mmu_zap_page(vcpu->kvm, sp);
2618 ++vcpu->kvm->stat.mmu_recycled;
2619 }
2620 }
2621
2622 int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code)
2623 {
2624 int r;
2625 enum emulation_result er;
2626
2627 r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code);
2628 if (r < 0)
2629 goto out;
2630
2631 if (!r) {
2632 r = 1;
2633 goto out;
2634 }
2635
2636 r = mmu_topup_memory_caches(vcpu);
2637 if (r)
2638 goto out;
2639
2640 er = emulate_instruction(vcpu, vcpu->run, cr2, error_code, 0);
2641
2642 switch (er) {
2643 case EMULATE_DONE:
2644 return 1;
2645 case EMULATE_DO_MMIO:
2646 ++vcpu->stat.mmio_exits;
2647 return 0;
2648 case EMULATE_FAIL:
2649 kvm_report_emulation_failure(vcpu, "pagetable");
2650 return 1;
2651 default:
2652 BUG();
2653 }
2654 out:
2655 return r;
2656 }
2657 EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);
2658
2659 void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
2660 {
2661 vcpu->arch.mmu.invlpg(vcpu, gva);
2662 kvm_mmu_flush_tlb(vcpu);
2663 ++vcpu->stat.invlpg;
2664 }
2665 EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);
2666
2667 void kvm_enable_tdp(void)
2668 {
2669 tdp_enabled = true;
2670 }
2671 EXPORT_SYMBOL_GPL(kvm_enable_tdp);
2672
2673 void kvm_disable_tdp(void)
2674 {
2675 tdp_enabled = false;
2676 }
2677 EXPORT_SYMBOL_GPL(kvm_disable_tdp);
2678
2679 static void free_mmu_pages(struct kvm_vcpu *vcpu)
2680 {
2681 free_page((unsigned long)vcpu->arch.mmu.pae_root);
2682 }
2683
2684 static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
2685 {
2686 struct page *page;
2687 int i;
2688
2689 ASSERT(vcpu);
2690
2691 if (vcpu->kvm->arch.n_requested_mmu_pages)
2692 vcpu->kvm->arch.n_free_mmu_pages =
2693 vcpu->kvm->arch.n_requested_mmu_pages;
2694 else
2695 vcpu->kvm->arch.n_free_mmu_pages =
2696 vcpu->kvm->arch.n_alloc_mmu_pages;
2697 /*
2698 * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
2699 * Therefore we need to allocate shadow page tables in the first
2700 * 4GB of memory, which happens to fit the DMA32 zone.
2701 */
2702 page = alloc_page(GFP_KERNEL | __GFP_DMA32);
2703 if (!page)
2704 goto error_1;
2705 vcpu->arch.mmu.pae_root = page_address(page);
2706 for (i = 0; i < 4; ++i)
2707 vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
2708
2709 return 0;
2710
2711 error_1:
2712 free_mmu_pages(vcpu);
2713 return -ENOMEM;
2714 }
2715
2716 int kvm_mmu_create(struct kvm_vcpu *vcpu)
2717 {
2718 ASSERT(vcpu);
2719 ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
2720
2721 return alloc_mmu_pages(vcpu);
2722 }
2723
2724 int kvm_mmu_setup(struct kvm_vcpu *vcpu)
2725 {
2726 ASSERT(vcpu);
2727 ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
2728
2729 return init_kvm_mmu(vcpu);
2730 }
2731
2732 void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
2733 {
2734 ASSERT(vcpu);
2735
2736 destroy_kvm_mmu(vcpu);
2737 free_mmu_pages(vcpu);
2738 mmu_free_memory_caches(vcpu);
2739 }
2740
2741 void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
2742 {
2743 struct kvm_mmu_page *sp;
2744
2745 list_for_each_entry(sp, &kvm->arch.active_mmu_pages, link) {
2746 int i;
2747 u64 *pt;
2748
2749 if (!test_bit(slot, sp->slot_bitmap))
2750 continue;
2751
2752 pt = sp->spt;
2753 for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
2754 /* avoid RMW */
2755 if (pt[i] & PT_WRITABLE_MASK)
2756 pt[i] &= ~PT_WRITABLE_MASK;
2757 }
2758 kvm_flush_remote_tlbs(kvm);
2759 }
2760
2761 void kvm_mmu_zap_all(struct kvm *kvm)
2762 {
2763 struct kvm_mmu_page *sp, *node;
2764
2765 spin_lock(&kvm->mmu_lock);
2766 list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link)
2767 if (kvm_mmu_zap_page(kvm, sp))
2768 node = container_of(kvm->arch.active_mmu_pages.next,
2769 struct kvm_mmu_page, link);
2770 spin_unlock(&kvm->mmu_lock);
2771
2772 kvm_flush_remote_tlbs(kvm);
2773 }
2774
2775 static void kvm_mmu_remove_one_alloc_mmu_page(struct kvm *kvm)
2776 {
2777 struct kvm_mmu_page *page;
2778
2779 page = container_of(kvm->arch.active_mmu_pages.prev,
2780 struct kvm_mmu_page, link);
2781 kvm_mmu_zap_page(kvm, page);
2782 }
2783
2784 static int mmu_shrink(int nr_to_scan, gfp_t gfp_mask)
2785 {
2786 struct kvm *kvm;
2787 struct kvm *kvm_freed = NULL;
2788 int cache_count = 0;
2789
2790 spin_lock(&kvm_lock);
2791
2792 list_for_each_entry(kvm, &vm_list, vm_list) {
2793 int npages;
2794
2795 if (!down_read_trylock(&kvm->slots_lock))
2796 continue;
2797 spin_lock(&kvm->mmu_lock);
2798 npages = kvm->arch.n_alloc_mmu_pages -
2799 kvm->arch.n_free_mmu_pages;
2800 cache_count += npages;
2801 if (!kvm_freed && nr_to_scan > 0 && npages > 0) {
2802 kvm_mmu_remove_one_alloc_mmu_page(kvm);
2803 cache_count--;
2804 kvm_freed = kvm;
2805 }
2806 nr_to_scan--;
2807
2808 spin_unlock(&kvm->mmu_lock);
2809 up_read(&kvm->slots_lock);
2810 }
2811 if (kvm_freed)
2812 list_move_tail(&kvm_freed->vm_list, &vm_list);
2813
2814 spin_unlock(&kvm_lock);
2815
2816 return cache_count;
2817 }
2818
2819 static struct shrinker mmu_shrinker = {
2820 .shrink = mmu_shrink,
2821 .seeks = DEFAULT_SEEKS * 10,
2822 };
2823
2824 static void mmu_destroy_caches(void)
2825 {
2826 if (pte_chain_cache)
2827 kmem_cache_destroy(pte_chain_cache);
2828 if (rmap_desc_cache)
2829 kmem_cache_destroy(rmap_desc_cache);
2830 if (mmu_page_header_cache)
2831 kmem_cache_destroy(mmu_page_header_cache);
2832 }
2833
2834 void kvm_mmu_module_exit(void)
2835 {
2836 mmu_destroy_caches();
2837 unregister_shrinker(&mmu_shrinker);
2838 }
2839
2840 int kvm_mmu_module_init(void)
2841 {
2842 pte_chain_cache = kmem_cache_create("kvm_pte_chain",
2843 sizeof(struct kvm_pte_chain),
2844 0, 0, NULL);
2845 if (!pte_chain_cache)
2846 goto nomem;
2847 rmap_desc_cache = kmem_cache_create("kvm_rmap_desc",
2848 sizeof(struct kvm_rmap_desc),
2849 0, 0, NULL);
2850 if (!rmap_desc_cache)
2851 goto nomem;
2852
2853 mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
2854 sizeof(struct kvm_mmu_page),
2855 0, 0, NULL);
2856 if (!mmu_page_header_cache)
2857 goto nomem;
2858
2859 register_shrinker(&mmu_shrinker);
2860
2861 return 0;
2862
2863 nomem:
2864 mmu_destroy_caches();
2865 return -ENOMEM;
2866 }
2867
2868 /*
2869 * Caculate mmu pages needed for kvm.
2870 */
2871 unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm)
2872 {
2873 int i;
2874 unsigned int nr_mmu_pages;
2875 unsigned int nr_pages = 0;
2876
2877 for (i = 0; i < kvm->nmemslots; i++)
2878 nr_pages += kvm->memslots[i].npages;
2879
2880 nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000;
2881 nr_mmu_pages = max(nr_mmu_pages,
2882 (unsigned int) KVM_MIN_ALLOC_MMU_PAGES);
2883
2884 return nr_mmu_pages;
2885 }
2886
2887 static void *pv_mmu_peek_buffer(struct kvm_pv_mmu_op_buffer *buffer,
2888 unsigned len)
2889 {
2890 if (len > buffer->len)
2891 return NULL;
2892 return buffer->ptr;
2893 }
2894
2895 static void *pv_mmu_read_buffer(struct kvm_pv_mmu_op_buffer *buffer,
2896 unsigned len)
2897 {
2898 void *ret;
2899
2900 ret = pv_mmu_peek_buffer(buffer, len);
2901 if (!ret)
2902 return ret;
2903 buffer->ptr += len;
2904 buffer->len -= len;
2905 buffer->processed += len;
2906 return ret;
2907 }
2908
2909 static int kvm_pv_mmu_write(struct kvm_vcpu *vcpu,
2910 gpa_t addr, gpa_t value)
2911 {
2912 int bytes = 8;
2913 int r;
2914
2915 if (!is_long_mode(vcpu) && !is_pae(vcpu))
2916 bytes = 4;
2917
2918 r = mmu_topup_memory_caches(vcpu);
2919 if (r)
2920 return r;
2921
2922 if (!emulator_write_phys(vcpu, addr, &value, bytes))
2923 return -EFAULT;
2924
2925 return 1;
2926 }
2927
2928 static int kvm_pv_mmu_flush_tlb(struct kvm_vcpu *vcpu)
2929 {
2930 kvm_set_cr3(vcpu, vcpu->arch.cr3);
2931 return 1;
2932 }
2933
2934 static int kvm_pv_mmu_release_pt(struct kvm_vcpu *vcpu, gpa_t addr)
2935 {
2936 spin_lock(&vcpu->kvm->mmu_lock);
2937 mmu_unshadow(vcpu->kvm, addr >> PAGE_SHIFT);
2938 spin_unlock(&vcpu->kvm->mmu_lock);
2939 return 1;
2940 }
2941
2942 static int kvm_pv_mmu_op_one(struct kvm_vcpu *vcpu,
2943 struct kvm_pv_mmu_op_buffer *buffer)
2944 {
2945 struct kvm_mmu_op_header *header;
2946
2947 header = pv_mmu_peek_buffer(buffer, sizeof *header);
2948 if (!header)
2949 return 0;
2950 switch (header->op) {
2951 case KVM_MMU_OP_WRITE_PTE: {
2952 struct kvm_mmu_op_write_pte *wpte;
2953
2954 wpte = pv_mmu_read_buffer(buffer, sizeof *wpte);
2955 if (!wpte)
2956 return 0;
2957 return kvm_pv_mmu_write(vcpu, wpte->pte_phys,
2958 wpte->pte_val);
2959 }
2960 case KVM_MMU_OP_FLUSH_TLB: {
2961 struct kvm_mmu_op_flush_tlb *ftlb;
2962
2963 ftlb = pv_mmu_read_buffer(buffer, sizeof *ftlb);
2964 if (!ftlb)
2965 return 0;
2966 return kvm_pv_mmu_flush_tlb(vcpu);
2967 }
2968 case KVM_MMU_OP_RELEASE_PT: {
2969 struct kvm_mmu_op_release_pt *rpt;
2970
2971 rpt = pv_mmu_read_buffer(buffer, sizeof *rpt);
2972 if (!rpt)
2973 return 0;
2974 return kvm_pv_mmu_release_pt(vcpu, rpt->pt_phys);
2975 }
2976 default: return 0;
2977 }
2978 }
2979
2980 int kvm_pv_mmu_op(struct kvm_vcpu *vcpu, unsigned long bytes,
2981 gpa_t addr, unsigned long *ret)
2982 {
2983 int r;
2984 struct kvm_pv_mmu_op_buffer *buffer = &vcpu->arch.mmu_op_buffer;
2985
2986 buffer->ptr = buffer->buf;
2987 buffer->len = min_t(unsigned long, bytes, sizeof buffer->buf);
2988 buffer->processed = 0;
2989
2990 r = kvm_read_guest(vcpu->kvm, addr, buffer->buf, buffer->len);
2991 if (r)
2992 goto out;
2993
2994 while (buffer->len) {
2995 r = kvm_pv_mmu_op_one(vcpu, buffer);
2996 if (r < 0)
2997 goto out;
2998 if (r == 0)
2999 break;
3000 }
3001
3002 r = 1;
3003 out:
3004 *ret = buffer->processed;
3005 return r;
3006 }
3007
3008 #ifdef AUDIT
3009
3010 static const char *audit_msg;
3011
3012 static gva_t canonicalize(gva_t gva)
3013 {
3014 #ifdef CONFIG_X86_64
3015 gva = (long long)(gva << 16) >> 16;
3016 #endif
3017 return gva;
3018 }
3019
3020 static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte,
3021 gva_t va, int level)
3022 {
3023 u64 *pt = __va(page_pte & PT64_BASE_ADDR_MASK);
3024 int i;
3025 gva_t va_delta = 1ul << (PAGE_SHIFT + 9 * (level - 1));
3026
3027 for (i = 0; i < PT64_ENT_PER_PAGE; ++i, va += va_delta) {
3028 u64 ent = pt[i];
3029
3030 if (ent == shadow_trap_nonpresent_pte)
3031 continue;
3032
3033 va = canonicalize(va);
3034 if (level > 1) {
3035 if (ent == shadow_notrap_nonpresent_pte)
3036 printk(KERN_ERR "audit: (%s) nontrapping pte"
3037 " in nonleaf level: levels %d gva %lx"
3038 " level %d pte %llx\n", audit_msg,
3039 vcpu->arch.mmu.root_level, va, level, ent);
3040 else
3041 audit_mappings_page(vcpu, ent, va, level - 1);
3042 } else {
3043 gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, va);
3044 gfn_t gfn = gpa >> PAGE_SHIFT;
3045 pfn_t pfn = gfn_to_pfn(vcpu->kvm, gfn);
3046 hpa_t hpa = (hpa_t)pfn << PAGE_SHIFT;
3047
3048 if (is_shadow_present_pte(ent)
3049 && (ent & PT64_BASE_ADDR_MASK) != hpa)
3050 printk(KERN_ERR "xx audit error: (%s) levels %d"
3051 " gva %lx gpa %llx hpa %llx ent %llx %d\n",
3052 audit_msg, vcpu->arch.mmu.root_level,
3053 va, gpa, hpa, ent,
3054 is_shadow_present_pte(ent));
3055 else if (ent == shadow_notrap_nonpresent_pte
3056 && !is_error_hpa(hpa))
3057 printk(KERN_ERR "audit: (%s) notrap shadow,"
3058 " valid guest gva %lx\n", audit_msg, va);
3059 kvm_release_pfn_clean(pfn);
3060
3061 }
3062 }
3063 }
3064
3065 static void audit_mappings(struct kvm_vcpu *vcpu)
3066 {
3067 unsigned i;
3068
3069 if (vcpu->arch.mmu.root_level == 4)
3070 audit_mappings_page(vcpu, vcpu->arch.mmu.root_hpa, 0, 4);
3071 else
3072 for (i = 0; i < 4; ++i)
3073 if (vcpu->arch.mmu.pae_root[i] & PT_PRESENT_MASK)
3074 audit_mappings_page(vcpu,
3075 vcpu->arch.mmu.pae_root[i],
3076 i << 30,
3077 2);
3078 }
3079
3080 static int count_rmaps(struct kvm_vcpu *vcpu)
3081 {
3082 int nmaps = 0;
3083 int i, j, k;
3084
3085 for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
3086 struct kvm_memory_slot *m = &vcpu->kvm->memslots[i];
3087 struct kvm_rmap_desc *d;
3088
3089 for (j = 0; j < m->npages; ++j) {
3090 unsigned long *rmapp = &m->rmap[j];
3091
3092 if (!*rmapp)
3093 continue;
3094 if (!(*rmapp & 1)) {
3095 ++nmaps;
3096 continue;
3097 }
3098 d = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
3099 while (d) {
3100 for (k = 0; k < RMAP_EXT; ++k)
3101 if (d->shadow_ptes[k])
3102 ++nmaps;
3103 else
3104 break;
3105 d = d->more;
3106 }
3107 }
3108 }
3109 return nmaps;
3110 }
3111
3112 static int count_writable_mappings(struct kvm_vcpu *vcpu)
3113 {
3114 int nmaps = 0;
3115 struct kvm_mmu_page *sp;
3116 int i;
3117
3118 list_for_each_entry(sp, &vcpu->kvm->arch.active_mmu_pages, link) {
3119 u64 *pt = sp->spt;
3120
3121 if (sp->role.level != PT_PAGE_TABLE_LEVEL)
3122 continue;
3123
3124 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
3125 u64 ent = pt[i];
3126
3127 if (!(ent & PT_PRESENT_MASK))
3128 continue;
3129 if (!(ent & PT_WRITABLE_MASK))
3130 continue;
3131 ++nmaps;
3132 }
3133 }
3134 return nmaps;
3135 }
3136
3137 static void audit_rmap(struct kvm_vcpu *vcpu)
3138 {
3139 int n_rmap = count_rmaps(vcpu);
3140 int n_actual = count_writable_mappings(vcpu);
3141
3142 if (n_rmap != n_actual)
3143 printk(KERN_ERR "%s: (%s) rmap %d actual %d\n",
3144 __func__, audit_msg, n_rmap, n_actual);
3145 }
3146
3147 static void audit_write_protection(struct kvm_vcpu *vcpu)
3148 {
3149 struct kvm_mmu_page *sp;
3150 struct kvm_memory_slot *slot;
3151 unsigned long *rmapp;
3152 gfn_t gfn;
3153
3154 list_for_each_entry(sp, &vcpu->kvm->arch.active_mmu_pages, link) {
3155 if (sp->role.direct)
3156 continue;
3157
3158 gfn = unalias_gfn(vcpu->kvm, sp->gfn);
3159 slot = gfn_to_memslot_unaliased(vcpu->kvm, sp->gfn);
3160 rmapp = &slot->rmap[gfn - slot->base_gfn];
3161 if (*rmapp)
3162 printk(KERN_ERR "%s: (%s) shadow page has writable"
3163 " mappings: gfn %lx role %x\n",
3164 __func__, audit_msg, sp->gfn,
3165 sp->role.word);
3166 }
3167 }
3168
3169 static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg)
3170 {
3171 int olddbg = dbg;
3172
3173 dbg = 0;
3174 audit_msg = msg;
3175 audit_rmap(vcpu);
3176 audit_write_protection(vcpu);
3177 audit_mappings(vcpu);
3178 dbg = olddbg;
3179 }
3180
3181 #endif
connection oriented routing - ARCHIVE NO UPDATES HERE