Linux 4.8.3
[linux/fpc-iii.git] / virt / kvm / kvm_main.c
blob195078225aa5d0c3b3214fd40e0b5f441c5d7518
1 /*
2 * Kernel-based Virtual Machine driver for Linux
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
7 * Copyright (C) 2006 Qumranet, Inc.
8 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
10 * Authors:
11 * Avi Kivity <avi@qumranet.com>
12 * Yaniv Kamay <yaniv@qumranet.com>
14 * This work is licensed under the terms of the GNU GPL, version 2. See
15 * the COPYING file in the top-level directory.
19 #include <kvm/iodev.h>
21 #include <linux/kvm_host.h>
22 #include <linux/kvm.h>
23 #include <linux/module.h>
24 #include <linux/errno.h>
25 #include <linux/percpu.h>
26 #include <linux/mm.h>
27 #include <linux/miscdevice.h>
28 #include <linux/vmalloc.h>
29 #include <linux/reboot.h>
30 #include <linux/debugfs.h>
31 #include <linux/highmem.h>
32 #include <linux/file.h>
33 #include <linux/syscore_ops.h>
34 #include <linux/cpu.h>
35 #include <linux/sched.h>
36 #include <linux/cpumask.h>
37 #include <linux/smp.h>
38 #include <linux/anon_inodes.h>
39 #include <linux/profile.h>
40 #include <linux/kvm_para.h>
41 #include <linux/pagemap.h>
42 #include <linux/mman.h>
43 #include <linux/swap.h>
44 #include <linux/bitops.h>
45 #include <linux/spinlock.h>
46 #include <linux/compat.h>
47 #include <linux/srcu.h>
48 #include <linux/hugetlb.h>
49 #include <linux/slab.h>
50 #include <linux/sort.h>
51 #include <linux/bsearch.h>
53 #include <asm/processor.h>
54 #include <asm/io.h>
55 #include <asm/ioctl.h>
56 #include <asm/uaccess.h>
57 #include <asm/pgtable.h>
59 #include "coalesced_mmio.h"
60 #include "async_pf.h"
61 #include "vfio.h"
63 #define CREATE_TRACE_POINTS
64 #include <trace/events/kvm.h>
66 /* Worst case buffer size needed for holding an integer. */
67 #define ITOA_MAX_LEN 12
69 MODULE_AUTHOR("Qumranet");
70 MODULE_LICENSE("GPL");
72 /* Architectures should define their poll value according to the halt latency */
73 static unsigned int halt_poll_ns = KVM_HALT_POLL_NS_DEFAULT;
74 module_param(halt_poll_ns, uint, S_IRUGO | S_IWUSR);
76 /* Default doubles per-vcpu halt_poll_ns. */
77 static unsigned int halt_poll_ns_grow = 2;
78 module_param(halt_poll_ns_grow, uint, S_IRUGO | S_IWUSR);
80 /* Default resets per-vcpu halt_poll_ns . */
81 static unsigned int halt_poll_ns_shrink;
82 module_param(halt_poll_ns_shrink, uint, S_IRUGO | S_IWUSR);
85 * Ordering of locks:
87 * kvm->lock --> kvm->slots_lock --> kvm->irq_lock
90 DEFINE_SPINLOCK(kvm_lock);
91 static DEFINE_RAW_SPINLOCK(kvm_count_lock);
92 LIST_HEAD(vm_list);
94 static cpumask_var_t cpus_hardware_enabled;
95 static int kvm_usage_count;
96 static atomic_t hardware_enable_failed;
98 struct kmem_cache *kvm_vcpu_cache;
99 EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
101 static __read_mostly struct preempt_ops kvm_preempt_ops;
103 struct dentry *kvm_debugfs_dir;
104 EXPORT_SYMBOL_GPL(kvm_debugfs_dir);
106 static int kvm_debugfs_num_entries;
107 static const struct file_operations *stat_fops_per_vm[];
109 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
110 unsigned long arg);
111 #ifdef CONFIG_KVM_COMPAT
112 static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl,
113 unsigned long arg);
114 #endif
115 static int hardware_enable_all(void);
116 static void hardware_disable_all(void);
118 static void kvm_io_bus_destroy(struct kvm_io_bus *bus);
120 static void kvm_release_pfn_dirty(kvm_pfn_t pfn);
121 static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, gfn_t gfn);
123 __visible bool kvm_rebooting;
124 EXPORT_SYMBOL_GPL(kvm_rebooting);
126 static bool largepages_enabled = true;
128 bool kvm_is_reserved_pfn(kvm_pfn_t pfn)
130 if (pfn_valid(pfn))
131 return PageReserved(pfn_to_page(pfn));
133 return true;
137 * Switches to specified vcpu, until a matching vcpu_put()
139 int vcpu_load(struct kvm_vcpu *vcpu)
141 int cpu;
143 if (mutex_lock_killable(&vcpu->mutex))
144 return -EINTR;
145 cpu = get_cpu();
146 preempt_notifier_register(&vcpu->preempt_notifier);
147 kvm_arch_vcpu_load(vcpu, cpu);
148 put_cpu();
149 return 0;
151 EXPORT_SYMBOL_GPL(vcpu_load);
153 void vcpu_put(struct kvm_vcpu *vcpu)
155 preempt_disable();
156 kvm_arch_vcpu_put(vcpu);
157 preempt_notifier_unregister(&vcpu->preempt_notifier);
158 preempt_enable();
159 mutex_unlock(&vcpu->mutex);
161 EXPORT_SYMBOL_GPL(vcpu_put);
163 static void ack_flush(void *_completed)
167 bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req)
169 int i, cpu, me;
170 cpumask_var_t cpus;
171 bool called = true;
172 struct kvm_vcpu *vcpu;
174 zalloc_cpumask_var(&cpus, GFP_ATOMIC);
176 me = get_cpu();
177 kvm_for_each_vcpu(i, vcpu, kvm) {
178 kvm_make_request(req, vcpu);
179 cpu = vcpu->cpu;
181 /* Set ->requests bit before we read ->mode. */
182 smp_mb__after_atomic();
184 if (cpus != NULL && cpu != -1 && cpu != me &&
185 kvm_vcpu_exiting_guest_mode(vcpu) != OUTSIDE_GUEST_MODE)
186 cpumask_set_cpu(cpu, cpus);
188 if (unlikely(cpus == NULL))
189 smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1);
190 else if (!cpumask_empty(cpus))
191 smp_call_function_many(cpus, ack_flush, NULL, 1);
192 else
193 called = false;
194 put_cpu();
195 free_cpumask_var(cpus);
196 return called;
199 #ifndef CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL
200 void kvm_flush_remote_tlbs(struct kvm *kvm)
203 * Read tlbs_dirty before setting KVM_REQ_TLB_FLUSH in
204 * kvm_make_all_cpus_request.
206 long dirty_count = smp_load_acquire(&kvm->tlbs_dirty);
209 * We want to publish modifications to the page tables before reading
210 * mode. Pairs with a memory barrier in arch-specific code.
211 * - x86: smp_mb__after_srcu_read_unlock in vcpu_enter_guest
212 * and smp_mb in walk_shadow_page_lockless_begin/end.
213 * - powerpc: smp_mb in kvmppc_prepare_to_enter.
215 * There is already an smp_mb__after_atomic() before
216 * kvm_make_all_cpus_request() reads vcpu->mode. We reuse that
217 * barrier here.
219 if (kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
220 ++kvm->stat.remote_tlb_flush;
221 cmpxchg(&kvm->tlbs_dirty, dirty_count, 0);
223 EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs);
224 #endif
226 void kvm_reload_remote_mmus(struct kvm *kvm)
228 kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
231 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
233 struct page *page;
234 int r;
236 mutex_init(&vcpu->mutex);
237 vcpu->cpu = -1;
238 vcpu->kvm = kvm;
239 vcpu->vcpu_id = id;
240 vcpu->pid = NULL;
241 init_swait_queue_head(&vcpu->wq);
242 kvm_async_pf_vcpu_init(vcpu);
244 vcpu->pre_pcpu = -1;
245 INIT_LIST_HEAD(&vcpu->blocked_vcpu_list);
247 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
248 if (!page) {
249 r = -ENOMEM;
250 goto fail;
252 vcpu->run = page_address(page);
254 kvm_vcpu_set_in_spin_loop(vcpu, false);
255 kvm_vcpu_set_dy_eligible(vcpu, false);
256 vcpu->preempted = false;
258 r = kvm_arch_vcpu_init(vcpu);
259 if (r < 0)
260 goto fail_free_run;
261 return 0;
263 fail_free_run:
264 free_page((unsigned long)vcpu->run);
265 fail:
266 return r;
268 EXPORT_SYMBOL_GPL(kvm_vcpu_init);
270 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
272 put_pid(vcpu->pid);
273 kvm_arch_vcpu_uninit(vcpu);
274 free_page((unsigned long)vcpu->run);
276 EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);
278 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
279 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
281 return container_of(mn, struct kvm, mmu_notifier);
284 static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
285 struct mm_struct *mm,
286 unsigned long address)
288 struct kvm *kvm = mmu_notifier_to_kvm(mn);
289 int need_tlb_flush, idx;
292 * When ->invalidate_page runs, the linux pte has been zapped
293 * already but the page is still allocated until
294 * ->invalidate_page returns. So if we increase the sequence
295 * here the kvm page fault will notice if the spte can't be
296 * established because the page is going to be freed. If
297 * instead the kvm page fault establishes the spte before
298 * ->invalidate_page runs, kvm_unmap_hva will release it
299 * before returning.
301 * The sequence increase only need to be seen at spin_unlock
302 * time, and not at spin_lock time.
304 * Increasing the sequence after the spin_unlock would be
305 * unsafe because the kvm page fault could then establish the
306 * pte after kvm_unmap_hva returned, without noticing the page
307 * is going to be freed.
309 idx = srcu_read_lock(&kvm->srcu);
310 spin_lock(&kvm->mmu_lock);
312 kvm->mmu_notifier_seq++;
313 need_tlb_flush = kvm_unmap_hva(kvm, address) | kvm->tlbs_dirty;
314 /* we've to flush the tlb before the pages can be freed */
315 if (need_tlb_flush)
316 kvm_flush_remote_tlbs(kvm);
318 spin_unlock(&kvm->mmu_lock);
320 kvm_arch_mmu_notifier_invalidate_page(kvm, address);
322 srcu_read_unlock(&kvm->srcu, idx);
325 static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
326 struct mm_struct *mm,
327 unsigned long address,
328 pte_t pte)
330 struct kvm *kvm = mmu_notifier_to_kvm(mn);
331 int idx;
333 idx = srcu_read_lock(&kvm->srcu);
334 spin_lock(&kvm->mmu_lock);
335 kvm->mmu_notifier_seq++;
336 kvm_set_spte_hva(kvm, address, pte);
337 spin_unlock(&kvm->mmu_lock);
338 srcu_read_unlock(&kvm->srcu, idx);
341 static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
342 struct mm_struct *mm,
343 unsigned long start,
344 unsigned long end)
346 struct kvm *kvm = mmu_notifier_to_kvm(mn);
347 int need_tlb_flush = 0, idx;
349 idx = srcu_read_lock(&kvm->srcu);
350 spin_lock(&kvm->mmu_lock);
352 * The count increase must become visible at unlock time as no
353 * spte can be established without taking the mmu_lock and
354 * count is also read inside the mmu_lock critical section.
356 kvm->mmu_notifier_count++;
357 need_tlb_flush = kvm_unmap_hva_range(kvm, start, end);
358 need_tlb_flush |= kvm->tlbs_dirty;
359 /* we've to flush the tlb before the pages can be freed */
360 if (need_tlb_flush)
361 kvm_flush_remote_tlbs(kvm);
363 spin_unlock(&kvm->mmu_lock);
364 srcu_read_unlock(&kvm->srcu, idx);
367 static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
368 struct mm_struct *mm,
369 unsigned long start,
370 unsigned long end)
372 struct kvm *kvm = mmu_notifier_to_kvm(mn);
374 spin_lock(&kvm->mmu_lock);
376 * This sequence increase will notify the kvm page fault that
377 * the page that is going to be mapped in the spte could have
378 * been freed.
380 kvm->mmu_notifier_seq++;
381 smp_wmb();
383 * The above sequence increase must be visible before the
384 * below count decrease, which is ensured by the smp_wmb above
385 * in conjunction with the smp_rmb in mmu_notifier_retry().
387 kvm->mmu_notifier_count--;
388 spin_unlock(&kvm->mmu_lock);
390 BUG_ON(kvm->mmu_notifier_count < 0);
393 static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
394 struct mm_struct *mm,
395 unsigned long start,
396 unsigned long end)
398 struct kvm *kvm = mmu_notifier_to_kvm(mn);
399 int young, idx;
401 idx = srcu_read_lock(&kvm->srcu);
402 spin_lock(&kvm->mmu_lock);
404 young = kvm_age_hva(kvm, start, end);
405 if (young)
406 kvm_flush_remote_tlbs(kvm);
408 spin_unlock(&kvm->mmu_lock);
409 srcu_read_unlock(&kvm->srcu, idx);
411 return young;
414 static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn,
415 struct mm_struct *mm,
416 unsigned long start,
417 unsigned long end)
419 struct kvm *kvm = mmu_notifier_to_kvm(mn);
420 int young, idx;
422 idx = srcu_read_lock(&kvm->srcu);
423 spin_lock(&kvm->mmu_lock);
425 * Even though we do not flush TLB, this will still adversely
426 * affect performance on pre-Haswell Intel EPT, where there is
427 * no EPT Access Bit to clear so that we have to tear down EPT
428 * tables instead. If we find this unacceptable, we can always
429 * add a parameter to kvm_age_hva so that it effectively doesn't
430 * do anything on clear_young.
432 * Also note that currently we never issue secondary TLB flushes
433 * from clear_young, leaving this job up to the regular system
434 * cadence. If we find this inaccurate, we might come up with a
435 * more sophisticated heuristic later.
437 young = kvm_age_hva(kvm, start, end);
438 spin_unlock(&kvm->mmu_lock);
439 srcu_read_unlock(&kvm->srcu, idx);
441 return young;
444 static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn,
445 struct mm_struct *mm,
446 unsigned long address)
448 struct kvm *kvm = mmu_notifier_to_kvm(mn);
449 int young, idx;
451 idx = srcu_read_lock(&kvm->srcu);
452 spin_lock(&kvm->mmu_lock);
453 young = kvm_test_age_hva(kvm, address);
454 spin_unlock(&kvm->mmu_lock);
455 srcu_read_unlock(&kvm->srcu, idx);
457 return young;
460 static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
461 struct mm_struct *mm)
463 struct kvm *kvm = mmu_notifier_to_kvm(mn);
464 int idx;
466 idx = srcu_read_lock(&kvm->srcu);
467 kvm_arch_flush_shadow_all(kvm);
468 srcu_read_unlock(&kvm->srcu, idx);
471 static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
472 .invalidate_page = kvm_mmu_notifier_invalidate_page,
473 .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start,
474 .invalidate_range_end = kvm_mmu_notifier_invalidate_range_end,
475 .clear_flush_young = kvm_mmu_notifier_clear_flush_young,
476 .clear_young = kvm_mmu_notifier_clear_young,
477 .test_young = kvm_mmu_notifier_test_young,
478 .change_pte = kvm_mmu_notifier_change_pte,
479 .release = kvm_mmu_notifier_release,
482 static int kvm_init_mmu_notifier(struct kvm *kvm)
484 kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
485 return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
488 #else /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */
490 static int kvm_init_mmu_notifier(struct kvm *kvm)
492 return 0;
495 #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */
497 static struct kvm_memslots *kvm_alloc_memslots(void)
499 int i;
500 struct kvm_memslots *slots;
502 slots = kvm_kvzalloc(sizeof(struct kvm_memslots));
503 if (!slots)
504 return NULL;
507 * Init kvm generation close to the maximum to easily test the
508 * code of handling generation number wrap-around.
510 slots->generation = -150;
511 for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
512 slots->id_to_index[i] = slots->memslots[i].id = i;
514 return slots;
517 static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
519 if (!memslot->dirty_bitmap)
520 return;
522 kvfree(memslot->dirty_bitmap);
523 memslot->dirty_bitmap = NULL;
527 * Free any memory in @free but not in @dont.
529 static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
530 struct kvm_memory_slot *dont)
532 if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
533 kvm_destroy_dirty_bitmap(free);
535 kvm_arch_free_memslot(kvm, free, dont);
537 free->npages = 0;
540 static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots)
542 struct kvm_memory_slot *memslot;
544 if (!slots)
545 return;
547 kvm_for_each_memslot(memslot, slots)
548 kvm_free_memslot(kvm, memslot, NULL);
550 kvfree(slots);
553 static void kvm_destroy_vm_debugfs(struct kvm *kvm)
555 int i;
557 if (!kvm->debugfs_dentry)
558 return;
560 debugfs_remove_recursive(kvm->debugfs_dentry);
562 for (i = 0; i < kvm_debugfs_num_entries; i++)
563 kfree(kvm->debugfs_stat_data[i]);
564 kfree(kvm->debugfs_stat_data);
567 static int kvm_create_vm_debugfs(struct kvm *kvm, int fd)
569 char dir_name[ITOA_MAX_LEN * 2];
570 struct kvm_stat_data *stat_data;
571 struct kvm_stats_debugfs_item *p;
573 if (!debugfs_initialized())
574 return 0;
576 snprintf(dir_name, sizeof(dir_name), "%d-%d", task_pid_nr(current), fd);
577 kvm->debugfs_dentry = debugfs_create_dir(dir_name,
578 kvm_debugfs_dir);
579 if (!kvm->debugfs_dentry)
580 return -ENOMEM;
582 kvm->debugfs_stat_data = kcalloc(kvm_debugfs_num_entries,
583 sizeof(*kvm->debugfs_stat_data),
584 GFP_KERNEL);
585 if (!kvm->debugfs_stat_data)
586 return -ENOMEM;
588 for (p = debugfs_entries; p->name; p++) {
589 stat_data = kzalloc(sizeof(*stat_data), GFP_KERNEL);
590 if (!stat_data)
591 return -ENOMEM;
593 stat_data->kvm = kvm;
594 stat_data->offset = p->offset;
595 kvm->debugfs_stat_data[p - debugfs_entries] = stat_data;
596 if (!debugfs_create_file(p->name, 0444,
597 kvm->debugfs_dentry,
598 stat_data,
599 stat_fops_per_vm[p->kind]))
600 return -ENOMEM;
602 return 0;
605 static struct kvm *kvm_create_vm(unsigned long type)
607 int r, i;
608 struct kvm *kvm = kvm_arch_alloc_vm();
610 if (!kvm)
611 return ERR_PTR(-ENOMEM);
613 spin_lock_init(&kvm->mmu_lock);
614 atomic_inc(&current->mm->mm_count);
615 kvm->mm = current->mm;
616 kvm_eventfd_init(kvm);
617 mutex_init(&kvm->lock);
618 mutex_init(&kvm->irq_lock);
619 mutex_init(&kvm->slots_lock);
620 atomic_set(&kvm->users_count, 1);
621 INIT_LIST_HEAD(&kvm->devices);
623 r = kvm_arch_init_vm(kvm, type);
624 if (r)
625 goto out_err_no_disable;
627 r = hardware_enable_all();
628 if (r)
629 goto out_err_no_disable;
631 #ifdef CONFIG_HAVE_KVM_IRQFD
632 INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
633 #endif
635 BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX);
637 r = -ENOMEM;
638 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
639 kvm->memslots[i] = kvm_alloc_memslots();
640 if (!kvm->memslots[i])
641 goto out_err_no_srcu;
644 if (init_srcu_struct(&kvm->srcu))
645 goto out_err_no_srcu;
646 if (init_srcu_struct(&kvm->irq_srcu))
647 goto out_err_no_irq_srcu;
648 for (i = 0; i < KVM_NR_BUSES; i++) {
649 kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus),
650 GFP_KERNEL);
651 if (!kvm->buses[i])
652 goto out_err;
655 r = kvm_init_mmu_notifier(kvm);
656 if (r)
657 goto out_err;
659 spin_lock(&kvm_lock);
660 list_add(&kvm->vm_list, &vm_list);
661 spin_unlock(&kvm_lock);
663 preempt_notifier_inc();
665 return kvm;
667 out_err:
668 cleanup_srcu_struct(&kvm->irq_srcu);
669 out_err_no_irq_srcu:
670 cleanup_srcu_struct(&kvm->srcu);
671 out_err_no_srcu:
672 hardware_disable_all();
673 out_err_no_disable:
674 for (i = 0; i < KVM_NR_BUSES; i++)
675 kfree(kvm->buses[i]);
676 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
677 kvm_free_memslots(kvm, kvm->memslots[i]);
678 kvm_arch_free_vm(kvm);
679 mmdrop(current->mm);
680 return ERR_PTR(r);
684 * Avoid using vmalloc for a small buffer.
685 * Should not be used when the size is statically known.
687 void *kvm_kvzalloc(unsigned long size)
689 if (size > PAGE_SIZE)
690 return vzalloc(size);
691 else
692 return kzalloc(size, GFP_KERNEL);
695 static void kvm_destroy_devices(struct kvm *kvm)
697 struct kvm_device *dev, *tmp;
700 * We do not need to take the kvm->lock here, because nobody else
701 * has a reference to the struct kvm at this point and therefore
702 * cannot access the devices list anyhow.
704 list_for_each_entry_safe(dev, tmp, &kvm->devices, vm_node) {
705 list_del(&dev->vm_node);
706 dev->ops->destroy(dev);
710 static void kvm_destroy_vm(struct kvm *kvm)
712 int i;
713 struct mm_struct *mm = kvm->mm;
715 kvm_destroy_vm_debugfs(kvm);
716 kvm_arch_sync_events(kvm);
717 spin_lock(&kvm_lock);
718 list_del(&kvm->vm_list);
719 spin_unlock(&kvm_lock);
720 kvm_free_irq_routing(kvm);
721 for (i = 0; i < KVM_NR_BUSES; i++)
722 kvm_io_bus_destroy(kvm->buses[i]);
723 kvm_coalesced_mmio_free(kvm);
724 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
725 mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
726 #else
727 kvm_arch_flush_shadow_all(kvm);
728 #endif
729 kvm_arch_destroy_vm(kvm);
730 kvm_destroy_devices(kvm);
731 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
732 kvm_free_memslots(kvm, kvm->memslots[i]);
733 cleanup_srcu_struct(&kvm->irq_srcu);
734 cleanup_srcu_struct(&kvm->srcu);
735 kvm_arch_free_vm(kvm);
736 preempt_notifier_dec();
737 hardware_disable_all();
738 mmdrop(mm);
741 void kvm_get_kvm(struct kvm *kvm)
743 atomic_inc(&kvm->users_count);
745 EXPORT_SYMBOL_GPL(kvm_get_kvm);
747 void kvm_put_kvm(struct kvm *kvm)
749 if (atomic_dec_and_test(&kvm->users_count))
750 kvm_destroy_vm(kvm);
752 EXPORT_SYMBOL_GPL(kvm_put_kvm);
755 static int kvm_vm_release(struct inode *inode, struct file *filp)
757 struct kvm *kvm = filp->private_data;
759 kvm_irqfd_release(kvm);
761 kvm_put_kvm(kvm);
762 return 0;
766 * Allocation size is twice as large as the actual dirty bitmap size.
767 * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed.
769 static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot)
771 unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot);
773 memslot->dirty_bitmap = kvm_kvzalloc(dirty_bytes);
774 if (!memslot->dirty_bitmap)
775 return -ENOMEM;
777 return 0;
781 * Insert memslot and re-sort memslots based on their GFN,
782 * so binary search could be used to lookup GFN.
783 * Sorting algorithm takes advantage of having initially
784 * sorted array and known changed memslot position.
786 static void update_memslots(struct kvm_memslots *slots,
787 struct kvm_memory_slot *new)
789 int id = new->id;
790 int i = slots->id_to_index[id];
791 struct kvm_memory_slot *mslots = slots->memslots;
793 WARN_ON(mslots[i].id != id);
794 if (!new->npages) {
795 WARN_ON(!mslots[i].npages);
796 if (mslots[i].npages)
797 slots->used_slots--;
798 } else {
799 if (!mslots[i].npages)
800 slots->used_slots++;
803 while (i < KVM_MEM_SLOTS_NUM - 1 &&
804 new->base_gfn <= mslots[i + 1].base_gfn) {
805 if (!mslots[i + 1].npages)
806 break;
807 mslots[i] = mslots[i + 1];
808 slots->id_to_index[mslots[i].id] = i;
809 i++;
813 * The ">=" is needed when creating a slot with base_gfn == 0,
814 * so that it moves before all those with base_gfn == npages == 0.
816 * On the other hand, if new->npages is zero, the above loop has
817 * already left i pointing to the beginning of the empty part of
818 * mslots, and the ">=" would move the hole backwards in this
819 * case---which is wrong. So skip the loop when deleting a slot.
821 if (new->npages) {
822 while (i > 0 &&
823 new->base_gfn >= mslots[i - 1].base_gfn) {
824 mslots[i] = mslots[i - 1];
825 slots->id_to_index[mslots[i].id] = i;
826 i--;
828 } else
829 WARN_ON_ONCE(i != slots->used_slots);
831 mslots[i] = *new;
832 slots->id_to_index[mslots[i].id] = i;
835 static int check_memory_region_flags(const struct kvm_userspace_memory_region *mem)
837 u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES;
839 #ifdef __KVM_HAVE_READONLY_MEM
840 valid_flags |= KVM_MEM_READONLY;
841 #endif
843 if (mem->flags & ~valid_flags)
844 return -EINVAL;
846 return 0;
849 static struct kvm_memslots *install_new_memslots(struct kvm *kvm,
850 int as_id, struct kvm_memslots *slots)
852 struct kvm_memslots *old_memslots = __kvm_memslots(kvm, as_id);
855 * Set the low bit in the generation, which disables SPTE caching
856 * until the end of synchronize_srcu_expedited.
858 WARN_ON(old_memslots->generation & 1);
859 slots->generation = old_memslots->generation + 1;
861 rcu_assign_pointer(kvm->memslots[as_id], slots);
862 synchronize_srcu_expedited(&kvm->srcu);
865 * Increment the new memslot generation a second time. This prevents
866 * vm exits that race with memslot updates from caching a memslot
867 * generation that will (potentially) be valid forever.
869 slots->generation++;
871 kvm_arch_memslots_updated(kvm, slots);
873 return old_memslots;
877 * Allocate some memory and give it an address in the guest physical address
878 * space.
880 * Discontiguous memory is allowed, mostly for framebuffers.
882 * Must be called holding kvm->slots_lock for write.
884 int __kvm_set_memory_region(struct kvm *kvm,
885 const struct kvm_userspace_memory_region *mem)
887 int r;
888 gfn_t base_gfn;
889 unsigned long npages;
890 struct kvm_memory_slot *slot;
891 struct kvm_memory_slot old, new;
892 struct kvm_memslots *slots = NULL, *old_memslots;
893 int as_id, id;
894 enum kvm_mr_change change;
896 r = check_memory_region_flags(mem);
897 if (r)
898 goto out;
900 r = -EINVAL;
901 as_id = mem->slot >> 16;
902 id = (u16)mem->slot;
904 /* General sanity checks */
905 if (mem->memory_size & (PAGE_SIZE - 1))
906 goto out;
907 if (mem->guest_phys_addr & (PAGE_SIZE - 1))
908 goto out;
909 /* We can read the guest memory with __xxx_user() later on. */
910 if ((id < KVM_USER_MEM_SLOTS) &&
911 ((mem->userspace_addr & (PAGE_SIZE - 1)) ||
912 !access_ok(VERIFY_WRITE,
913 (void __user *)(unsigned long)mem->userspace_addr,
914 mem->memory_size)))
915 goto out;
916 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_MEM_SLOTS_NUM)
917 goto out;
918 if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
919 goto out;
921 slot = id_to_memslot(__kvm_memslots(kvm, as_id), id);
922 base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
923 npages = mem->memory_size >> PAGE_SHIFT;
925 if (npages > KVM_MEM_MAX_NR_PAGES)
926 goto out;
928 new = old = *slot;
930 new.id = id;
931 new.base_gfn = base_gfn;
932 new.npages = npages;
933 new.flags = mem->flags;
935 if (npages) {
936 if (!old.npages)
937 change = KVM_MR_CREATE;
938 else { /* Modify an existing slot. */
939 if ((mem->userspace_addr != old.userspace_addr) ||
940 (npages != old.npages) ||
941 ((new.flags ^ old.flags) & KVM_MEM_READONLY))
942 goto out;
944 if (base_gfn != old.base_gfn)
945 change = KVM_MR_MOVE;
946 else if (new.flags != old.flags)
947 change = KVM_MR_FLAGS_ONLY;
948 else { /* Nothing to change. */
949 r = 0;
950 goto out;
953 } else {
954 if (!old.npages)
955 goto out;
957 change = KVM_MR_DELETE;
958 new.base_gfn = 0;
959 new.flags = 0;
962 if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
963 /* Check for overlaps */
964 r = -EEXIST;
965 kvm_for_each_memslot(slot, __kvm_memslots(kvm, as_id)) {
966 if ((slot->id >= KVM_USER_MEM_SLOTS) ||
967 (slot->id == id))
968 continue;
969 if (!((base_gfn + npages <= slot->base_gfn) ||
970 (base_gfn >= slot->base_gfn + slot->npages)))
971 goto out;
975 /* Free page dirty bitmap if unneeded */
976 if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
977 new.dirty_bitmap = NULL;
979 r = -ENOMEM;
980 if (change == KVM_MR_CREATE) {
981 new.userspace_addr = mem->userspace_addr;
983 if (kvm_arch_create_memslot(kvm, &new, npages))
984 goto out_free;
987 /* Allocate page dirty bitmap if needed */
988 if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
989 if (kvm_create_dirty_bitmap(&new) < 0)
990 goto out_free;
993 slots = kvm_kvzalloc(sizeof(struct kvm_memslots));
994 if (!slots)
995 goto out_free;
996 memcpy(slots, __kvm_memslots(kvm, as_id), sizeof(struct kvm_memslots));
998 if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) {
999 slot = id_to_memslot(slots, id);
1000 slot->flags |= KVM_MEMSLOT_INVALID;
1002 old_memslots = install_new_memslots(kvm, as_id, slots);
1004 /* slot was deleted or moved, clear iommu mapping */
1005 kvm_iommu_unmap_pages(kvm, &old);
1006 /* From this point no new shadow pages pointing to a deleted,
1007 * or moved, memslot will be created.
1009 * validation of sp->gfn happens in:
1010 * - gfn_to_hva (kvm_read_guest, gfn_to_pfn)
1011 * - kvm_is_visible_gfn (mmu_check_roots)
1013 kvm_arch_flush_shadow_memslot(kvm, slot);
1016 * We can re-use the old_memslots from above, the only difference
1017 * from the currently installed memslots is the invalid flag. This
1018 * will get overwritten by update_memslots anyway.
1020 slots = old_memslots;
1023 r = kvm_arch_prepare_memory_region(kvm, &new, mem, change);
1024 if (r)
1025 goto out_slots;
1027 /* actual memory is freed via old in kvm_free_memslot below */
1028 if (change == KVM_MR_DELETE) {
1029 new.dirty_bitmap = NULL;
1030 memset(&new.arch, 0, sizeof(new.arch));
1033 update_memslots(slots, &new);
1034 old_memslots = install_new_memslots(kvm, as_id, slots);
1036 kvm_arch_commit_memory_region(kvm, mem, &old, &new, change);
1038 kvm_free_memslot(kvm, &old, &new);
1039 kvfree(old_memslots);
1042 * IOMMU mapping: New slots need to be mapped. Old slots need to be
1043 * un-mapped and re-mapped if their base changes. Since base change
1044 * unmapping is handled above with slot deletion, mapping alone is
1045 * needed here. Anything else the iommu might care about for existing
1046 * slots (size changes, userspace addr changes and read-only flag
1047 * changes) is disallowed above, so any other attribute changes getting
1048 * here can be skipped.
1050 if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
1051 r = kvm_iommu_map_pages(kvm, &new);
1052 return r;
1055 return 0;
1057 out_slots:
1058 kvfree(slots);
1059 out_free:
1060 kvm_free_memslot(kvm, &new, &old);
1061 out:
1062 return r;
1064 EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
1066 int kvm_set_memory_region(struct kvm *kvm,
1067 const struct kvm_userspace_memory_region *mem)
1069 int r;
1071 mutex_lock(&kvm->slots_lock);
1072 r = __kvm_set_memory_region(kvm, mem);
1073 mutex_unlock(&kvm->slots_lock);
1074 return r;
1076 EXPORT_SYMBOL_GPL(kvm_set_memory_region);
1078 static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
1079 struct kvm_userspace_memory_region *mem)
1081 if ((u16)mem->slot >= KVM_USER_MEM_SLOTS)
1082 return -EINVAL;
1084 return kvm_set_memory_region(kvm, mem);
1087 int kvm_get_dirty_log(struct kvm *kvm,
1088 struct kvm_dirty_log *log, int *is_dirty)
1090 struct kvm_memslots *slots;
1091 struct kvm_memory_slot *memslot;
1092 int r, i, as_id, id;
1093 unsigned long n;
1094 unsigned long any = 0;
1096 r = -EINVAL;
1097 as_id = log->slot >> 16;
1098 id = (u16)log->slot;
1099 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
1100 goto out;
1102 slots = __kvm_memslots(kvm, as_id);
1103 memslot = id_to_memslot(slots, id);
1104 r = -ENOENT;
1105 if (!memslot->dirty_bitmap)
1106 goto out;
1108 n = kvm_dirty_bitmap_bytes(memslot);
1110 for (i = 0; !any && i < n/sizeof(long); ++i)
1111 any = memslot->dirty_bitmap[i];
1113 r = -EFAULT;
1114 if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
1115 goto out;
1117 if (any)
1118 *is_dirty = 1;
1120 r = 0;
1121 out:
1122 return r;
1124 EXPORT_SYMBOL_GPL(kvm_get_dirty_log);
1126 #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
1128 * kvm_get_dirty_log_protect - get a snapshot of dirty pages, and if any pages
1129 * are dirty write protect them for next write.
1130 * @kvm: pointer to kvm instance
1131 * @log: slot id and address to which we copy the log
1132 * @is_dirty: flag set if any page is dirty
1134 * We need to keep it in mind that VCPU threads can write to the bitmap
1135 * concurrently. So, to avoid losing track of dirty pages we keep the
1136 * following order:
1138 * 1. Take a snapshot of the bit and clear it if needed.
1139 * 2. Write protect the corresponding page.
1140 * 3. Copy the snapshot to the userspace.
1141 * 4. Upon return caller flushes TLB's if needed.
1143 * Between 2 and 4, the guest may write to the page using the remaining TLB
1144 * entry. This is not a problem because the page is reported dirty using
1145 * the snapshot taken before and step 4 ensures that writes done after
1146 * exiting to userspace will be logged for the next call.
1149 int kvm_get_dirty_log_protect(struct kvm *kvm,
1150 struct kvm_dirty_log *log, bool *is_dirty)
1152 struct kvm_memslots *slots;
1153 struct kvm_memory_slot *memslot;
1154 int r, i, as_id, id;
1155 unsigned long n;
1156 unsigned long *dirty_bitmap;
1157 unsigned long *dirty_bitmap_buffer;
1159 r = -EINVAL;
1160 as_id = log->slot >> 16;
1161 id = (u16)log->slot;
1162 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
1163 goto out;
1165 slots = __kvm_memslots(kvm, as_id);
1166 memslot = id_to_memslot(slots, id);
1168 dirty_bitmap = memslot->dirty_bitmap;
1169 r = -ENOENT;
1170 if (!dirty_bitmap)
1171 goto out;
1173 n = kvm_dirty_bitmap_bytes(memslot);
1175 dirty_bitmap_buffer = dirty_bitmap + n / sizeof(long);
1176 memset(dirty_bitmap_buffer, 0, n);
1178 spin_lock(&kvm->mmu_lock);
1179 *is_dirty = false;
1180 for (i = 0; i < n / sizeof(long); i++) {
1181 unsigned long mask;
1182 gfn_t offset;
1184 if (!dirty_bitmap[i])
1185 continue;
1187 *is_dirty = true;
1189 mask = xchg(&dirty_bitmap[i], 0);
1190 dirty_bitmap_buffer[i] = mask;
1192 if (mask) {
1193 offset = i * BITS_PER_LONG;
1194 kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
1195 offset, mask);
1199 spin_unlock(&kvm->mmu_lock);
1201 r = -EFAULT;
1202 if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n))
1203 goto out;
1205 r = 0;
1206 out:
1207 return r;
1209 EXPORT_SYMBOL_GPL(kvm_get_dirty_log_protect);
1210 #endif
1212 bool kvm_largepages_enabled(void)
1214 return largepages_enabled;
1217 void kvm_disable_largepages(void)
1219 largepages_enabled = false;
1221 EXPORT_SYMBOL_GPL(kvm_disable_largepages);
1223 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
1225 return __gfn_to_memslot(kvm_memslots(kvm), gfn);
1227 EXPORT_SYMBOL_GPL(gfn_to_memslot);
1229 struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn)
1231 return __gfn_to_memslot(kvm_vcpu_memslots(vcpu), gfn);
1234 bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
1236 struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);
1238 if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS ||
1239 memslot->flags & KVM_MEMSLOT_INVALID)
1240 return false;
1242 return true;
1244 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);
1246 unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn)
1248 struct vm_area_struct *vma;
1249 unsigned long addr, size;
1251 size = PAGE_SIZE;
1253 addr = gfn_to_hva(kvm, gfn);
1254 if (kvm_is_error_hva(addr))
1255 return PAGE_SIZE;
1257 down_read(&current->mm->mmap_sem);
1258 vma = find_vma(current->mm, addr);
1259 if (!vma)
1260 goto out;
1262 size = vma_kernel_pagesize(vma);
1264 out:
1265 up_read(&current->mm->mmap_sem);
1267 return size;
1270 static bool memslot_is_readonly(struct kvm_memory_slot *slot)
1272 return slot->flags & KVM_MEM_READONLY;
1275 static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
1276 gfn_t *nr_pages, bool write)
1278 if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
1279 return KVM_HVA_ERR_BAD;
1281 if (memslot_is_readonly(slot) && write)
1282 return KVM_HVA_ERR_RO_BAD;
1284 if (nr_pages)
1285 *nr_pages = slot->npages - (gfn - slot->base_gfn);
1287 return __gfn_to_hva_memslot(slot, gfn);
1290 static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
1291 gfn_t *nr_pages)
1293 return __gfn_to_hva_many(slot, gfn, nr_pages, true);
1296 unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot,
1297 gfn_t gfn)
1299 return gfn_to_hva_many(slot, gfn, NULL);
1301 EXPORT_SYMBOL_GPL(gfn_to_hva_memslot);
1303 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
1305 return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
1307 EXPORT_SYMBOL_GPL(gfn_to_hva);
1309 unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn)
1311 return gfn_to_hva_many(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, NULL);
1313 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_hva);
1316 * If writable is set to false, the hva returned by this function is only
1317 * allowed to be read.
1319 unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot,
1320 gfn_t gfn, bool *writable)
1322 unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false);
1324 if (!kvm_is_error_hva(hva) && writable)
1325 *writable = !memslot_is_readonly(slot);
1327 return hva;
1330 unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable)
1332 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1334 return gfn_to_hva_memslot_prot(slot, gfn, writable);
1337 unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable)
1339 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1341 return gfn_to_hva_memslot_prot(slot, gfn, writable);
1344 static int get_user_page_nowait(unsigned long start, int write,
1345 struct page **page)
1347 int flags = FOLL_TOUCH | FOLL_NOWAIT | FOLL_HWPOISON | FOLL_GET;
1349 if (write)
1350 flags |= FOLL_WRITE;
1352 return __get_user_pages(current, current->mm, start, 1, flags, page,
1353 NULL, NULL);
1356 static inline int check_user_page_hwpoison(unsigned long addr)
1358 int rc, flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_WRITE;
1360 rc = __get_user_pages(current, current->mm, addr, 1,
1361 flags, NULL, NULL, NULL);
1362 return rc == -EHWPOISON;
1366 * The atomic path to get the writable pfn which will be stored in @pfn,
1367 * true indicates success, otherwise false is returned.
1369 static bool hva_to_pfn_fast(unsigned long addr, bool atomic, bool *async,
1370 bool write_fault, bool *writable, kvm_pfn_t *pfn)
1372 struct page *page[1];
1373 int npages;
1375 if (!(async || atomic))
1376 return false;
1379 * Fast pin a writable pfn only if it is a write fault request
1380 * or the caller allows to map a writable pfn for a read fault
1381 * request.
1383 if (!(write_fault || writable))
1384 return false;
1386 npages = __get_user_pages_fast(addr, 1, 1, page);
1387 if (npages == 1) {
1388 *pfn = page_to_pfn(page[0]);
1390 if (writable)
1391 *writable = true;
1392 return true;
1395 return false;
1399 * The slow path to get the pfn of the specified host virtual address,
1400 * 1 indicates success, -errno is returned if error is detected.
1402 static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault,
1403 bool *writable, kvm_pfn_t *pfn)
1405 struct page *page[1];
1406 int npages = 0;
1408 might_sleep();
1410 if (writable)
1411 *writable = write_fault;
1413 if (async) {
1414 down_read(&current->mm->mmap_sem);
1415 npages = get_user_page_nowait(addr, write_fault, page);
1416 up_read(&current->mm->mmap_sem);
1417 } else
1418 npages = __get_user_pages_unlocked(current, current->mm, addr, 1,
1419 write_fault, 0, page,
1420 FOLL_TOUCH|FOLL_HWPOISON);
1421 if (npages != 1)
1422 return npages;
1424 /* map read fault as writable if possible */
1425 if (unlikely(!write_fault) && writable) {
1426 struct page *wpage[1];
1428 npages = __get_user_pages_fast(addr, 1, 1, wpage);
1429 if (npages == 1) {
1430 *writable = true;
1431 put_page(page[0]);
1432 page[0] = wpage[0];
1435 npages = 1;
1437 *pfn = page_to_pfn(page[0]);
1438 return npages;
1441 static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault)
1443 if (unlikely(!(vma->vm_flags & VM_READ)))
1444 return false;
1446 if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE))))
1447 return false;
1449 return true;
1452 static int hva_to_pfn_remapped(struct vm_area_struct *vma,
1453 unsigned long addr, bool *async,
1454 bool write_fault, kvm_pfn_t *p_pfn)
1456 unsigned long pfn;
1457 int r;
1459 r = follow_pfn(vma, addr, &pfn);
1460 if (r) {
1462 * get_user_pages fails for VM_IO and VM_PFNMAP vmas and does
1463 * not call the fault handler, so do it here.
1465 bool unlocked = false;
1466 r = fixup_user_fault(current, current->mm, addr,
1467 (write_fault ? FAULT_FLAG_WRITE : 0),
1468 &unlocked);
1469 if (unlocked)
1470 return -EAGAIN;
1471 if (r)
1472 return r;
1474 r = follow_pfn(vma, addr, &pfn);
1475 if (r)
1476 return r;
1482 * Get a reference here because callers of *hva_to_pfn* and
1483 * *gfn_to_pfn* ultimately call kvm_release_pfn_clean on the
1484 * returned pfn. This is only needed if the VMA has VM_MIXEDMAP
1485 * set, but the kvm_get_pfn/kvm_release_pfn_clean pair will
1486 * simply do nothing for reserved pfns.
1488 * Whoever called remap_pfn_range is also going to call e.g.
1489 * unmap_mapping_range before the underlying pages are freed,
1490 * causing a call to our MMU notifier.
1492 kvm_get_pfn(pfn);
1494 *p_pfn = pfn;
1495 return 0;
1499 * Pin guest page in memory and return its pfn.
1500 * @addr: host virtual address which maps memory to the guest
1501 * @atomic: whether this function can sleep
1502 * @async: whether this function need to wait IO complete if the
1503 * host page is not in the memory
1504 * @write_fault: whether we should get a writable host page
1505 * @writable: whether it allows to map a writable host page for !@write_fault
1507 * The function will map a writable host page for these two cases:
1508 * 1): @write_fault = true
1509 * 2): @write_fault = false && @writable, @writable will tell the caller
1510 * whether the mapping is writable.
1512 static kvm_pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
1513 bool write_fault, bool *writable)
1515 struct vm_area_struct *vma;
1516 kvm_pfn_t pfn = 0;
1517 int npages, r;
1519 /* we can do it either atomically or asynchronously, not both */
1520 BUG_ON(atomic && async);
1522 if (hva_to_pfn_fast(addr, atomic, async, write_fault, writable, &pfn))
1523 return pfn;
1525 if (atomic)
1526 return KVM_PFN_ERR_FAULT;
1528 npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn);
1529 if (npages == 1)
1530 return pfn;
1532 down_read(&current->mm->mmap_sem);
1533 if (npages == -EHWPOISON ||
1534 (!async && check_user_page_hwpoison(addr))) {
1535 pfn = KVM_PFN_ERR_HWPOISON;
1536 goto exit;
1539 retry:
1540 vma = find_vma_intersection(current->mm, addr, addr + 1);
1542 if (vma == NULL)
1543 pfn = KVM_PFN_ERR_FAULT;
1544 else if (vma->vm_flags & (VM_IO | VM_PFNMAP)) {
1545 r = hva_to_pfn_remapped(vma, addr, async, write_fault, &pfn);
1546 if (r == -EAGAIN)
1547 goto retry;
1548 if (r < 0)
1549 pfn = KVM_PFN_ERR_FAULT;
1550 } else {
1551 if (async && vma_is_valid(vma, write_fault))
1552 *async = true;
1553 pfn = KVM_PFN_ERR_FAULT;
1555 exit:
1556 up_read(&current->mm->mmap_sem);
1557 return pfn;
1560 kvm_pfn_t __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn,
1561 bool atomic, bool *async, bool write_fault,
1562 bool *writable)
1564 unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault);
1566 if (addr == KVM_HVA_ERR_RO_BAD) {
1567 if (writable)
1568 *writable = false;
1569 return KVM_PFN_ERR_RO_FAULT;
1572 if (kvm_is_error_hva(addr)) {
1573 if (writable)
1574 *writable = false;
1575 return KVM_PFN_NOSLOT;
1578 /* Do not map writable pfn in the readonly memslot. */
1579 if (writable && memslot_is_readonly(slot)) {
1580 *writable = false;
1581 writable = NULL;
1584 return hva_to_pfn(addr, atomic, async, write_fault,
1585 writable);
1587 EXPORT_SYMBOL_GPL(__gfn_to_pfn_memslot);
1589 kvm_pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
1590 bool *writable)
1592 return __gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn, false, NULL,
1593 write_fault, writable);
1595 EXPORT_SYMBOL_GPL(gfn_to_pfn_prot);
1597 kvm_pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
1599 return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL);
1601 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot);
1603 kvm_pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn)
1605 return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL);
1607 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic);
1609 kvm_pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn)
1611 return gfn_to_pfn_memslot_atomic(gfn_to_memslot(kvm, gfn), gfn);
1613 EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic);
1615 kvm_pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn)
1617 return gfn_to_pfn_memslot_atomic(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn);
1619 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn_atomic);
1621 kvm_pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
1623 return gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn);
1625 EXPORT_SYMBOL_GPL(gfn_to_pfn);
1627 kvm_pfn_t kvm_vcpu_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn)
1629 return gfn_to_pfn_memslot(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn);
1631 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn);
1633 int gfn_to_page_many_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
1634 struct page **pages, int nr_pages)
1636 unsigned long addr;
1637 gfn_t entry;
1639 addr = gfn_to_hva_many(slot, gfn, &entry);
1640 if (kvm_is_error_hva(addr))
1641 return -1;
1643 if (entry < nr_pages)
1644 return 0;
1646 return __get_user_pages_fast(addr, nr_pages, 1, pages);
1648 EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic);
1650 static struct page *kvm_pfn_to_page(kvm_pfn_t pfn)
1652 if (is_error_noslot_pfn(pfn))
1653 return KVM_ERR_PTR_BAD_PAGE;
1655 if (kvm_is_reserved_pfn(pfn)) {
1656 WARN_ON(1);
1657 return KVM_ERR_PTR_BAD_PAGE;
1660 return pfn_to_page(pfn);
1663 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
1665 kvm_pfn_t pfn;
1667 pfn = gfn_to_pfn(kvm, gfn);
1669 return kvm_pfn_to_page(pfn);
1671 EXPORT_SYMBOL_GPL(gfn_to_page);
1673 struct page *kvm_vcpu_gfn_to_page(struct kvm_vcpu *vcpu, gfn_t gfn)
1675 kvm_pfn_t pfn;
1677 pfn = kvm_vcpu_gfn_to_pfn(vcpu, gfn);
1679 return kvm_pfn_to_page(pfn);
1681 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_page);
1683 void kvm_release_page_clean(struct page *page)
1685 WARN_ON(is_error_page(page));
1687 kvm_release_pfn_clean(page_to_pfn(page));
1689 EXPORT_SYMBOL_GPL(kvm_release_page_clean);
1691 void kvm_release_pfn_clean(kvm_pfn_t pfn)
1693 if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn))
1694 put_page(pfn_to_page(pfn));
1696 EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);
1698 void kvm_release_page_dirty(struct page *page)
1700 WARN_ON(is_error_page(page));
1702 kvm_release_pfn_dirty(page_to_pfn(page));
1704 EXPORT_SYMBOL_GPL(kvm_release_page_dirty);
1706 static void kvm_release_pfn_dirty(kvm_pfn_t pfn)
1708 kvm_set_pfn_dirty(pfn);
1709 kvm_release_pfn_clean(pfn);
1712 void kvm_set_pfn_dirty(kvm_pfn_t pfn)
1714 if (!kvm_is_reserved_pfn(pfn)) {
1715 struct page *page = pfn_to_page(pfn);
1717 if (!PageReserved(page))
1718 SetPageDirty(page);
1721 EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);
1723 void kvm_set_pfn_accessed(kvm_pfn_t pfn)
1725 if (!kvm_is_reserved_pfn(pfn))
1726 mark_page_accessed(pfn_to_page(pfn));
1728 EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);
1730 void kvm_get_pfn(kvm_pfn_t pfn)
1732 if (!kvm_is_reserved_pfn(pfn))
1733 get_page(pfn_to_page(pfn));
1735 EXPORT_SYMBOL_GPL(kvm_get_pfn);
1737 static int next_segment(unsigned long len, int offset)
1739 if (len > PAGE_SIZE - offset)
1740 return PAGE_SIZE - offset;
1741 else
1742 return len;
1745 static int __kvm_read_guest_page(struct kvm_memory_slot *slot, gfn_t gfn,
1746 void *data, int offset, int len)
1748 int r;
1749 unsigned long addr;
1751 addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
1752 if (kvm_is_error_hva(addr))
1753 return -EFAULT;
1754 r = __copy_from_user(data, (void __user *)addr + offset, len);
1755 if (r)
1756 return -EFAULT;
1757 return 0;
1760 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
1761 int len)
1763 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1765 return __kvm_read_guest_page(slot, gfn, data, offset, len);
1767 EXPORT_SYMBOL_GPL(kvm_read_guest_page);
1769 int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data,
1770 int offset, int len)
1772 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1774 return __kvm_read_guest_page(slot, gfn, data, offset, len);
1776 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_page);
1778 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
1780 gfn_t gfn = gpa >> PAGE_SHIFT;
1781 int seg;
1782 int offset = offset_in_page(gpa);
1783 int ret;
1785 while ((seg = next_segment(len, offset)) != 0) {
1786 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
1787 if (ret < 0)
1788 return ret;
1789 offset = 0;
1790 len -= seg;
1791 data += seg;
1792 ++gfn;
1794 return 0;
1796 EXPORT_SYMBOL_GPL(kvm_read_guest);
1798 int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, unsigned long len)
1800 gfn_t gfn = gpa >> PAGE_SHIFT;
1801 int seg;
1802 int offset = offset_in_page(gpa);
1803 int ret;
1805 while ((seg = next_segment(len, offset)) != 0) {
1806 ret = kvm_vcpu_read_guest_page(vcpu, gfn, data, offset, seg);
1807 if (ret < 0)
1808 return ret;
1809 offset = 0;
1810 len -= seg;
1811 data += seg;
1812 ++gfn;
1814 return 0;
1816 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest);
1818 static int __kvm_read_guest_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
1819 void *data, int offset, unsigned long len)
1821 int r;
1822 unsigned long addr;
1824 addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
1825 if (kvm_is_error_hva(addr))
1826 return -EFAULT;
1827 pagefault_disable();
1828 r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
1829 pagefault_enable();
1830 if (r)
1831 return -EFAULT;
1832 return 0;
1835 int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
1836 unsigned long len)
1838 gfn_t gfn = gpa >> PAGE_SHIFT;
1839 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1840 int offset = offset_in_page(gpa);
1842 return __kvm_read_guest_atomic(slot, gfn, data, offset, len);
1844 EXPORT_SYMBOL_GPL(kvm_read_guest_atomic);
1846 int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa,
1847 void *data, unsigned long len)
1849 gfn_t gfn = gpa >> PAGE_SHIFT;
1850 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1851 int offset = offset_in_page(gpa);
1853 return __kvm_read_guest_atomic(slot, gfn, data, offset, len);
1855 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_atomic);
1857 static int __kvm_write_guest_page(struct kvm_memory_slot *memslot, gfn_t gfn,
1858 const void *data, int offset, int len)
1860 int r;
1861 unsigned long addr;
1863 addr = gfn_to_hva_memslot(memslot, gfn);
1864 if (kvm_is_error_hva(addr))
1865 return -EFAULT;
1866 r = __copy_to_user((void __user *)addr + offset, data, len);
1867 if (r)
1868 return -EFAULT;
1869 mark_page_dirty_in_slot(memslot, gfn);
1870 return 0;
1873 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn,
1874 const void *data, int offset, int len)
1876 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1878 return __kvm_write_guest_page(slot, gfn, data, offset, len);
1880 EXPORT_SYMBOL_GPL(kvm_write_guest_page);
1882 int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,
1883 const void *data, int offset, int len)
1885 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1887 return __kvm_write_guest_page(slot, gfn, data, offset, len);
1889 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest_page);
1891 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
1892 unsigned long len)
1894 gfn_t gfn = gpa >> PAGE_SHIFT;
1895 int seg;
1896 int offset = offset_in_page(gpa);
1897 int ret;
1899 while ((seg = next_segment(len, offset)) != 0) {
1900 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
1901 if (ret < 0)
1902 return ret;
1903 offset = 0;
1904 len -= seg;
1905 data += seg;
1906 ++gfn;
1908 return 0;
1910 EXPORT_SYMBOL_GPL(kvm_write_guest);
1912 int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data,
1913 unsigned long len)
1915 gfn_t gfn = gpa >> PAGE_SHIFT;
1916 int seg;
1917 int offset = offset_in_page(gpa);
1918 int ret;
1920 while ((seg = next_segment(len, offset)) != 0) {
1921 ret = kvm_vcpu_write_guest_page(vcpu, gfn, data, offset, seg);
1922 if (ret < 0)
1923 return ret;
1924 offset = 0;
1925 len -= seg;
1926 data += seg;
1927 ++gfn;
1929 return 0;
1931 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest);
1933 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1934 gpa_t gpa, unsigned long len)
1936 struct kvm_memslots *slots = kvm_memslots(kvm);
1937 int offset = offset_in_page(gpa);
1938 gfn_t start_gfn = gpa >> PAGE_SHIFT;
1939 gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT;
1940 gfn_t nr_pages_needed = end_gfn - start_gfn + 1;
1941 gfn_t nr_pages_avail;
1943 ghc->gpa = gpa;
1944 ghc->generation = slots->generation;
1945 ghc->len = len;
1946 ghc->memslot = gfn_to_memslot(kvm, start_gfn);
1947 ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, NULL);
1948 if (!kvm_is_error_hva(ghc->hva) && nr_pages_needed <= 1) {
1949 ghc->hva += offset;
1950 } else {
1952 * If the requested region crosses two memslots, we still
1953 * verify that the entire region is valid here.
1955 while (start_gfn <= end_gfn) {
1956 ghc->memslot = gfn_to_memslot(kvm, start_gfn);
1957 ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn,
1958 &nr_pages_avail);
1959 if (kvm_is_error_hva(ghc->hva))
1960 return -EFAULT;
1961 start_gfn += nr_pages_avail;
1963 /* Use the slow path for cross page reads and writes. */
1964 ghc->memslot = NULL;
1966 return 0;
1968 EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init);
1970 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1971 void *data, unsigned long len)
1973 struct kvm_memslots *slots = kvm_memslots(kvm);
1974 int r;
1976 BUG_ON(len > ghc->len);
1978 if (slots->generation != ghc->generation)
1979 kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa, ghc->len);
1981 if (unlikely(!ghc->memslot))
1982 return kvm_write_guest(kvm, ghc->gpa, data, len);
1984 if (kvm_is_error_hva(ghc->hva))
1985 return -EFAULT;
1987 r = __copy_to_user((void __user *)ghc->hva, data, len);
1988 if (r)
1989 return -EFAULT;
1990 mark_page_dirty_in_slot(ghc->memslot, ghc->gpa >> PAGE_SHIFT);
1992 return 0;
1994 EXPORT_SYMBOL_GPL(kvm_write_guest_cached);
1996 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1997 void *data, unsigned long len)
1999 struct kvm_memslots *slots = kvm_memslots(kvm);
2000 int r;
2002 BUG_ON(len > ghc->len);
2004 if (slots->generation != ghc->generation)
2005 kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa, ghc->len);
2007 if (unlikely(!ghc->memslot))
2008 return kvm_read_guest(kvm, ghc->gpa, data, len);
2010 if (kvm_is_error_hva(ghc->hva))
2011 return -EFAULT;
2013 r = __copy_from_user(data, (void __user *)ghc->hva, len);
2014 if (r)
2015 return -EFAULT;
2017 return 0;
2019 EXPORT_SYMBOL_GPL(kvm_read_guest_cached);
2021 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
2023 const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0)));
2025 return kvm_write_guest_page(kvm, gfn, zero_page, offset, len);
2027 EXPORT_SYMBOL_GPL(kvm_clear_guest_page);
2029 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
2031 gfn_t gfn = gpa >> PAGE_SHIFT;
2032 int seg;
2033 int offset = offset_in_page(gpa);
2034 int ret;
2036 while ((seg = next_segment(len, offset)) != 0) {
2037 ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
2038 if (ret < 0)
2039 return ret;
2040 offset = 0;
2041 len -= seg;
2042 ++gfn;
2044 return 0;
2046 EXPORT_SYMBOL_GPL(kvm_clear_guest);
2048 static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot,
2049 gfn_t gfn)
2051 if (memslot && memslot->dirty_bitmap) {
2052 unsigned long rel_gfn = gfn - memslot->base_gfn;
2054 set_bit_le(rel_gfn, memslot->dirty_bitmap);
2058 void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
2060 struct kvm_memory_slot *memslot;
2062 memslot = gfn_to_memslot(kvm, gfn);
2063 mark_page_dirty_in_slot(memslot, gfn);
2065 EXPORT_SYMBOL_GPL(mark_page_dirty);
2067 void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn)
2069 struct kvm_memory_slot *memslot;
2071 memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
2072 mark_page_dirty_in_slot(memslot, gfn);
2074 EXPORT_SYMBOL_GPL(kvm_vcpu_mark_page_dirty);
2076 static void grow_halt_poll_ns(struct kvm_vcpu *vcpu)
2078 unsigned int old, val, grow;
2080 old = val = vcpu->halt_poll_ns;
2081 grow = READ_ONCE(halt_poll_ns_grow);
2082 /* 10us base */
2083 if (val == 0 && grow)
2084 val = 10000;
2085 else
2086 val *= grow;
2088 if (val > halt_poll_ns)
2089 val = halt_poll_ns;
2091 vcpu->halt_poll_ns = val;
2092 trace_kvm_halt_poll_ns_grow(vcpu->vcpu_id, val, old);
2095 static void shrink_halt_poll_ns(struct kvm_vcpu *vcpu)
2097 unsigned int old, val, shrink;
2099 old = val = vcpu->halt_poll_ns;
2100 shrink = READ_ONCE(halt_poll_ns_shrink);
2101 if (shrink == 0)
2102 val = 0;
2103 else
2104 val /= shrink;
2106 vcpu->halt_poll_ns = val;
2107 trace_kvm_halt_poll_ns_shrink(vcpu->vcpu_id, val, old);
2110 static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu)
2112 if (kvm_arch_vcpu_runnable(vcpu)) {
2113 kvm_make_request(KVM_REQ_UNHALT, vcpu);
2114 return -EINTR;
2116 if (kvm_cpu_has_pending_timer(vcpu))
2117 return -EINTR;
2118 if (signal_pending(current))
2119 return -EINTR;
2121 return 0;
2125 * The vCPU has executed a HLT instruction with in-kernel mode enabled.
2127 void kvm_vcpu_block(struct kvm_vcpu *vcpu)
2129 ktime_t start, cur;
2130 DECLARE_SWAITQUEUE(wait);
2131 bool waited = false;
2132 u64 block_ns;
2134 start = cur = ktime_get();
2135 if (vcpu->halt_poll_ns) {
2136 ktime_t stop = ktime_add_ns(ktime_get(), vcpu->halt_poll_ns);
2138 ++vcpu->stat.halt_attempted_poll;
2139 do {
2141 * This sets KVM_REQ_UNHALT if an interrupt
2142 * arrives.
2144 if (kvm_vcpu_check_block(vcpu) < 0) {
2145 ++vcpu->stat.halt_successful_poll;
2146 if (!vcpu_valid_wakeup(vcpu))
2147 ++vcpu->stat.halt_poll_invalid;
2148 goto out;
2150 cur = ktime_get();
2151 } while (single_task_running() && ktime_before(cur, stop));
2154 kvm_arch_vcpu_blocking(vcpu);
2156 for (;;) {
2157 prepare_to_swait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
2159 if (kvm_vcpu_check_block(vcpu) < 0)
2160 break;
2162 waited = true;
2163 schedule();
2166 finish_swait(&vcpu->wq, &wait);
2167 cur = ktime_get();
2169 kvm_arch_vcpu_unblocking(vcpu);
2170 out:
2171 block_ns = ktime_to_ns(cur) - ktime_to_ns(start);
2173 if (!vcpu_valid_wakeup(vcpu))
2174 shrink_halt_poll_ns(vcpu);
2175 else if (halt_poll_ns) {
2176 if (block_ns <= vcpu->halt_poll_ns)
2178 /* we had a long block, shrink polling */
2179 else if (vcpu->halt_poll_ns && block_ns > halt_poll_ns)
2180 shrink_halt_poll_ns(vcpu);
2181 /* we had a short halt and our poll time is too small */
2182 else if (vcpu->halt_poll_ns < halt_poll_ns &&
2183 block_ns < halt_poll_ns)
2184 grow_halt_poll_ns(vcpu);
2185 } else
2186 vcpu->halt_poll_ns = 0;
2188 trace_kvm_vcpu_wakeup(block_ns, waited, vcpu_valid_wakeup(vcpu));
2189 kvm_arch_vcpu_block_finish(vcpu);
2191 EXPORT_SYMBOL_GPL(kvm_vcpu_block);
2193 #ifndef CONFIG_S390
2194 void kvm_vcpu_wake_up(struct kvm_vcpu *vcpu)
2196 struct swait_queue_head *wqp;
2198 wqp = kvm_arch_vcpu_wq(vcpu);
2199 if (swait_active(wqp)) {
2200 swake_up(wqp);
2201 ++vcpu->stat.halt_wakeup;
2205 EXPORT_SYMBOL_GPL(kvm_vcpu_wake_up);
2208 * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode.
2210 void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
2212 int me;
2213 int cpu = vcpu->cpu;
2215 kvm_vcpu_wake_up(vcpu);
2216 me = get_cpu();
2217 if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
2218 if (kvm_arch_vcpu_should_kick(vcpu))
2219 smp_send_reschedule(cpu);
2220 put_cpu();
2222 EXPORT_SYMBOL_GPL(kvm_vcpu_kick);
2223 #endif /* !CONFIG_S390 */
2225 int kvm_vcpu_yield_to(struct kvm_vcpu *target)
2227 struct pid *pid;
2228 struct task_struct *task = NULL;
2229 int ret = 0;
2231 rcu_read_lock();
2232 pid = rcu_dereference(target->pid);
2233 if (pid)
2234 task = get_pid_task(pid, PIDTYPE_PID);
2235 rcu_read_unlock();
2236 if (!task)
2237 return ret;
2238 ret = yield_to(task, 1);
2239 put_task_struct(task);
2241 return ret;
2243 EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to);
2246 * Helper that checks whether a VCPU is eligible for directed yield.
2247 * Most eligible candidate to yield is decided by following heuristics:
2249 * (a) VCPU which has not done pl-exit or cpu relax intercepted recently
2250 * (preempted lock holder), indicated by @in_spin_loop.
2251 * Set at the beiginning and cleared at the end of interception/PLE handler.
2253 * (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get
2254 * chance last time (mostly it has become eligible now since we have probably
2255 * yielded to lockholder in last iteration. This is done by toggling
2256 * @dy_eligible each time a VCPU checked for eligibility.)
2258 * Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding
2259 * to preempted lock-holder could result in wrong VCPU selection and CPU
2260 * burning. Giving priority for a potential lock-holder increases lock
2261 * progress.
2263 * Since algorithm is based on heuristics, accessing another VCPU data without
2264 * locking does not harm. It may result in trying to yield to same VCPU, fail
2265 * and continue with next VCPU and so on.
2267 static bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
2269 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
2270 bool eligible;
2272 eligible = !vcpu->spin_loop.in_spin_loop ||
2273 vcpu->spin_loop.dy_eligible;
2275 if (vcpu->spin_loop.in_spin_loop)
2276 kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible);
2278 return eligible;
2279 #else
2280 return true;
2281 #endif
2284 void kvm_vcpu_on_spin(struct kvm_vcpu *me)
2286 struct kvm *kvm = me->kvm;
2287 struct kvm_vcpu *vcpu;
2288 int last_boosted_vcpu = me->kvm->last_boosted_vcpu;
2289 int yielded = 0;
2290 int try = 3;
2291 int pass;
2292 int i;
2294 kvm_vcpu_set_in_spin_loop(me, true);
2296 * We boost the priority of a VCPU that is runnable but not
2297 * currently running, because it got preempted by something
2298 * else and called schedule in __vcpu_run. Hopefully that
2299 * VCPU is holding the lock that we need and will release it.
2300 * We approximate round-robin by starting at the last boosted VCPU.
2302 for (pass = 0; pass < 2 && !yielded && try; pass++) {
2303 kvm_for_each_vcpu(i, vcpu, kvm) {
2304 if (!pass && i <= last_boosted_vcpu) {
2305 i = last_boosted_vcpu;
2306 continue;
2307 } else if (pass && i > last_boosted_vcpu)
2308 break;
2309 if (!ACCESS_ONCE(vcpu->preempted))
2310 continue;
2311 if (vcpu == me)
2312 continue;
2313 if (swait_active(&vcpu->wq) && !kvm_arch_vcpu_runnable(vcpu))
2314 continue;
2315 if (!kvm_vcpu_eligible_for_directed_yield(vcpu))
2316 continue;
2318 yielded = kvm_vcpu_yield_to(vcpu);
2319 if (yielded > 0) {
2320 kvm->last_boosted_vcpu = i;
2321 break;
2322 } else if (yielded < 0) {
2323 try--;
2324 if (!try)
2325 break;
2329 kvm_vcpu_set_in_spin_loop(me, false);
2331 /* Ensure vcpu is not eligible during next spinloop */
2332 kvm_vcpu_set_dy_eligible(me, false);
2334 EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);
2336 static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2338 struct kvm_vcpu *vcpu = vma->vm_file->private_data;
2339 struct page *page;
2341 if (vmf->pgoff == 0)
2342 page = virt_to_page(vcpu->run);
2343 #ifdef CONFIG_X86
2344 else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
2345 page = virt_to_page(vcpu->arch.pio_data);
2346 #endif
2347 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
2348 else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
2349 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
2350 #endif
2351 else
2352 return kvm_arch_vcpu_fault(vcpu, vmf);
2353 get_page(page);
2354 vmf->page = page;
2355 return 0;
2358 static const struct vm_operations_struct kvm_vcpu_vm_ops = {
2359 .fault = kvm_vcpu_fault,
2362 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
2364 vma->vm_ops = &kvm_vcpu_vm_ops;
2365 return 0;
2368 static int kvm_vcpu_release(struct inode *inode, struct file *filp)
2370 struct kvm_vcpu *vcpu = filp->private_data;
2372 kvm_put_kvm(vcpu->kvm);
2373 return 0;
2376 static struct file_operations kvm_vcpu_fops = {
2377 .release = kvm_vcpu_release,
2378 .unlocked_ioctl = kvm_vcpu_ioctl,
2379 #ifdef CONFIG_KVM_COMPAT
2380 .compat_ioctl = kvm_vcpu_compat_ioctl,
2381 #endif
2382 .mmap = kvm_vcpu_mmap,
2383 .llseek = noop_llseek,
2387 * Allocates an inode for the vcpu.
2389 static int create_vcpu_fd(struct kvm_vcpu *vcpu)
2391 return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC);
2395 * Creates some virtual cpus. Good luck creating more than one.
2397 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
2399 int r;
2400 struct kvm_vcpu *vcpu;
2402 if (id >= KVM_MAX_VCPU_ID)
2403 return -EINVAL;
2405 mutex_lock(&kvm->lock);
2406 if (kvm->created_vcpus == KVM_MAX_VCPUS) {
2407 mutex_unlock(&kvm->lock);
2408 return -EINVAL;
2411 kvm->created_vcpus++;
2412 mutex_unlock(&kvm->lock);
2414 vcpu = kvm_arch_vcpu_create(kvm, id);
2415 if (IS_ERR(vcpu)) {
2416 r = PTR_ERR(vcpu);
2417 goto vcpu_decrement;
2420 preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
2422 r = kvm_arch_vcpu_setup(vcpu);
2423 if (r)
2424 goto vcpu_destroy;
2426 mutex_lock(&kvm->lock);
2427 if (kvm_get_vcpu_by_id(kvm, id)) {
2428 r = -EEXIST;
2429 goto unlock_vcpu_destroy;
2432 BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]);
2434 /* Now it's all set up, let userspace reach it */
2435 kvm_get_kvm(kvm);
2436 r = create_vcpu_fd(vcpu);
2437 if (r < 0) {
2438 kvm_put_kvm(kvm);
2439 goto unlock_vcpu_destroy;
2442 kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu;
2445 * Pairs with smp_rmb() in kvm_get_vcpu. Write kvm->vcpus
2446 * before kvm->online_vcpu's incremented value.
2448 smp_wmb();
2449 atomic_inc(&kvm->online_vcpus);
2451 mutex_unlock(&kvm->lock);
2452 kvm_arch_vcpu_postcreate(vcpu);
2453 return r;
2455 unlock_vcpu_destroy:
2456 mutex_unlock(&kvm->lock);
2457 vcpu_destroy:
2458 kvm_arch_vcpu_destroy(vcpu);
2459 vcpu_decrement:
2460 mutex_lock(&kvm->lock);
2461 kvm->created_vcpus--;
2462 mutex_unlock(&kvm->lock);
2463 return r;
2466 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
2468 if (sigset) {
2469 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
2470 vcpu->sigset_active = 1;
2471 vcpu->sigset = *sigset;
2472 } else
2473 vcpu->sigset_active = 0;
2474 return 0;
2477 static long kvm_vcpu_ioctl(struct file *filp,
2478 unsigned int ioctl, unsigned long arg)
2480 struct kvm_vcpu *vcpu = filp->private_data;
2481 void __user *argp = (void __user *)arg;
2482 int r;
2483 struct kvm_fpu *fpu = NULL;
2484 struct kvm_sregs *kvm_sregs = NULL;
2486 if (vcpu->kvm->mm != current->mm)
2487 return -EIO;
2489 if (unlikely(_IOC_TYPE(ioctl) != KVMIO))
2490 return -EINVAL;
2492 #if defined(CONFIG_S390) || defined(CONFIG_PPC) || defined(CONFIG_MIPS)
2494 * Special cases: vcpu ioctls that are asynchronous to vcpu execution,
2495 * so vcpu_load() would break it.
2497 if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_S390_IRQ || ioctl == KVM_INTERRUPT)
2498 return kvm_arch_vcpu_ioctl(filp, ioctl, arg);
2499 #endif
2502 r = vcpu_load(vcpu);
2503 if (r)
2504 return r;
2505 switch (ioctl) {
2506 case KVM_RUN:
2507 r = -EINVAL;
2508 if (arg)
2509 goto out;
2510 if (unlikely(vcpu->pid != current->pids[PIDTYPE_PID].pid)) {
2511 /* The thread running this VCPU changed. */
2512 struct pid *oldpid = vcpu->pid;
2513 struct pid *newpid = get_task_pid(current, PIDTYPE_PID);
2515 rcu_assign_pointer(vcpu->pid, newpid);
2516 if (oldpid)
2517 synchronize_rcu();
2518 put_pid(oldpid);
2520 r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
2521 trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
2522 break;
2523 case KVM_GET_REGS: {
2524 struct kvm_regs *kvm_regs;
2526 r = -ENOMEM;
2527 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
2528 if (!kvm_regs)
2529 goto out;
2530 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
2531 if (r)
2532 goto out_free1;
2533 r = -EFAULT;
2534 if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
2535 goto out_free1;
2536 r = 0;
2537 out_free1:
2538 kfree(kvm_regs);
2539 break;
2541 case KVM_SET_REGS: {
2542 struct kvm_regs *kvm_regs;
2544 r = -ENOMEM;
2545 kvm_regs = memdup_user(argp, sizeof(*kvm_regs));
2546 if (IS_ERR(kvm_regs)) {
2547 r = PTR_ERR(kvm_regs);
2548 goto out;
2550 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
2551 kfree(kvm_regs);
2552 break;
2554 case KVM_GET_SREGS: {
2555 kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
2556 r = -ENOMEM;
2557 if (!kvm_sregs)
2558 goto out;
2559 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
2560 if (r)
2561 goto out;
2562 r = -EFAULT;
2563 if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
2564 goto out;
2565 r = 0;
2566 break;
2568 case KVM_SET_SREGS: {
2569 kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs));
2570 if (IS_ERR(kvm_sregs)) {
2571 r = PTR_ERR(kvm_sregs);
2572 kvm_sregs = NULL;
2573 goto out;
2575 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
2576 break;
2578 case KVM_GET_MP_STATE: {
2579 struct kvm_mp_state mp_state;
2581 r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
2582 if (r)
2583 goto out;
2584 r = -EFAULT;
2585 if (copy_to_user(argp, &mp_state, sizeof(mp_state)))
2586 goto out;
2587 r = 0;
2588 break;
2590 case KVM_SET_MP_STATE: {
2591 struct kvm_mp_state mp_state;
2593 r = -EFAULT;
2594 if (copy_from_user(&mp_state, argp, sizeof(mp_state)))
2595 goto out;
2596 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
2597 break;
2599 case KVM_TRANSLATE: {
2600 struct kvm_translation tr;
2602 r = -EFAULT;
2603 if (copy_from_user(&tr, argp, sizeof(tr)))
2604 goto out;
2605 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
2606 if (r)
2607 goto out;
2608 r = -EFAULT;
2609 if (copy_to_user(argp, &tr, sizeof(tr)))
2610 goto out;
2611 r = 0;
2612 break;
2614 case KVM_SET_GUEST_DEBUG: {
2615 struct kvm_guest_debug dbg;
2617 r = -EFAULT;
2618 if (copy_from_user(&dbg, argp, sizeof(dbg)))
2619 goto out;
2620 r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
2621 break;
2623 case KVM_SET_SIGNAL_MASK: {
2624 struct kvm_signal_mask __user *sigmask_arg = argp;
2625 struct kvm_signal_mask kvm_sigmask;
2626 sigset_t sigset, *p;
2628 p = NULL;
2629 if (argp) {
2630 r = -EFAULT;
2631 if (copy_from_user(&kvm_sigmask, argp,
2632 sizeof(kvm_sigmask)))
2633 goto out;
2634 r = -EINVAL;
2635 if (kvm_sigmask.len != sizeof(sigset))
2636 goto out;
2637 r = -EFAULT;
2638 if (copy_from_user(&sigset, sigmask_arg->sigset,
2639 sizeof(sigset)))
2640 goto out;
2641 p = &sigset;
2643 r = kvm_vcpu_ioctl_set_sigmask(vcpu, p);
2644 break;
2646 case KVM_GET_FPU: {
2647 fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
2648 r = -ENOMEM;
2649 if (!fpu)
2650 goto out;
2651 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
2652 if (r)
2653 goto out;
2654 r = -EFAULT;
2655 if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
2656 goto out;
2657 r = 0;
2658 break;
2660 case KVM_SET_FPU: {
2661 fpu = memdup_user(argp, sizeof(*fpu));
2662 if (IS_ERR(fpu)) {
2663 r = PTR_ERR(fpu);
2664 fpu = NULL;
2665 goto out;
2667 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
2668 break;
2670 default:
2671 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
2673 out:
2674 vcpu_put(vcpu);
2675 kfree(fpu);
2676 kfree(kvm_sregs);
2677 return r;
2680 #ifdef CONFIG_KVM_COMPAT
2681 static long kvm_vcpu_compat_ioctl(struct file *filp,
2682 unsigned int ioctl, unsigned long arg)
2684 struct kvm_vcpu *vcpu = filp->private_data;
2685 void __user *argp = compat_ptr(arg);
2686 int r;
2688 if (vcpu->kvm->mm != current->mm)
2689 return -EIO;
2691 switch (ioctl) {
2692 case KVM_SET_SIGNAL_MASK: {
2693 struct kvm_signal_mask __user *sigmask_arg = argp;
2694 struct kvm_signal_mask kvm_sigmask;
2695 compat_sigset_t csigset;
2696 sigset_t sigset;
2698 if (argp) {
2699 r = -EFAULT;
2700 if (copy_from_user(&kvm_sigmask, argp,
2701 sizeof(kvm_sigmask)))
2702 goto out;
2703 r = -EINVAL;
2704 if (kvm_sigmask.len != sizeof(csigset))
2705 goto out;
2706 r = -EFAULT;
2707 if (copy_from_user(&csigset, sigmask_arg->sigset,
2708 sizeof(csigset)))
2709 goto out;
2710 sigset_from_compat(&sigset, &csigset);
2711 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
2712 } else
2713 r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL);
2714 break;
2716 default:
2717 r = kvm_vcpu_ioctl(filp, ioctl, arg);
2720 out:
2721 return r;
2723 #endif
2725 static int kvm_device_ioctl_attr(struct kvm_device *dev,
2726 int (*accessor)(struct kvm_device *dev,
2727 struct kvm_device_attr *attr),
2728 unsigned long arg)
2730 struct kvm_device_attr attr;
2732 if (!accessor)
2733 return -EPERM;
2735 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2736 return -EFAULT;
2738 return accessor(dev, &attr);
2741 static long kvm_device_ioctl(struct file *filp, unsigned int ioctl,
2742 unsigned long arg)
2744 struct kvm_device *dev = filp->private_data;
2746 switch (ioctl) {
2747 case KVM_SET_DEVICE_ATTR:
2748 return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg);
2749 case KVM_GET_DEVICE_ATTR:
2750 return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg);
2751 case KVM_HAS_DEVICE_ATTR:
2752 return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg);
2753 default:
2754 if (dev->ops->ioctl)
2755 return dev->ops->ioctl(dev, ioctl, arg);
2757 return -ENOTTY;
2761 static int kvm_device_release(struct inode *inode, struct file *filp)
2763 struct kvm_device *dev = filp->private_data;
2764 struct kvm *kvm = dev->kvm;
2766 kvm_put_kvm(kvm);
2767 return 0;
2770 static const struct file_operations kvm_device_fops = {
2771 .unlocked_ioctl = kvm_device_ioctl,
2772 #ifdef CONFIG_KVM_COMPAT
2773 .compat_ioctl = kvm_device_ioctl,
2774 #endif
2775 .release = kvm_device_release,
2778 struct kvm_device *kvm_device_from_filp(struct file *filp)
2780 if (filp->f_op != &kvm_device_fops)
2781 return NULL;
2783 return filp->private_data;
2786 static struct kvm_device_ops *kvm_device_ops_table[KVM_DEV_TYPE_MAX] = {
2787 #ifdef CONFIG_KVM_MPIC
2788 [KVM_DEV_TYPE_FSL_MPIC_20] = &kvm_mpic_ops,
2789 [KVM_DEV_TYPE_FSL_MPIC_42] = &kvm_mpic_ops,
2790 #endif
2792 #ifdef CONFIG_KVM_XICS
2793 [KVM_DEV_TYPE_XICS] = &kvm_xics_ops,
2794 #endif
2797 int kvm_register_device_ops(struct kvm_device_ops *ops, u32 type)
2799 if (type >= ARRAY_SIZE(kvm_device_ops_table))
2800 return -ENOSPC;
2802 if (kvm_device_ops_table[type] != NULL)
2803 return -EEXIST;
2805 kvm_device_ops_table[type] = ops;
2806 return 0;
2809 void kvm_unregister_device_ops(u32 type)
2811 if (kvm_device_ops_table[type] != NULL)
2812 kvm_device_ops_table[type] = NULL;
2815 static int kvm_ioctl_create_device(struct kvm *kvm,
2816 struct kvm_create_device *cd)
2818 struct kvm_device_ops *ops = NULL;
2819 struct kvm_device *dev;
2820 bool test = cd->flags & KVM_CREATE_DEVICE_TEST;
2821 int ret;
2823 if (cd->type >= ARRAY_SIZE(kvm_device_ops_table))
2824 return -ENODEV;
2826 ops = kvm_device_ops_table[cd->type];
2827 if (ops == NULL)
2828 return -ENODEV;
2830 if (test)
2831 return 0;
2833 dev = kzalloc(sizeof(*dev), GFP_KERNEL);
2834 if (!dev)
2835 return -ENOMEM;
2837 dev->ops = ops;
2838 dev->kvm = kvm;
2840 mutex_lock(&kvm->lock);
2841 ret = ops->create(dev, cd->type);
2842 if (ret < 0) {
2843 mutex_unlock(&kvm->lock);
2844 kfree(dev);
2845 return ret;
2847 list_add(&dev->vm_node, &kvm->devices);
2848 mutex_unlock(&kvm->lock);
2850 if (ops->init)
2851 ops->init(dev);
2853 ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC);
2854 if (ret < 0) {
2855 ops->destroy(dev);
2856 mutex_lock(&kvm->lock);
2857 list_del(&dev->vm_node);
2858 mutex_unlock(&kvm->lock);
2859 return ret;
2862 kvm_get_kvm(kvm);
2863 cd->fd = ret;
2864 return 0;
2867 static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg)
2869 switch (arg) {
2870 case KVM_CAP_USER_MEMORY:
2871 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
2872 case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
2873 case KVM_CAP_INTERNAL_ERROR_DATA:
2874 #ifdef CONFIG_HAVE_KVM_MSI
2875 case KVM_CAP_SIGNAL_MSI:
2876 #endif
2877 #ifdef CONFIG_HAVE_KVM_IRQFD
2878 case KVM_CAP_IRQFD:
2879 case KVM_CAP_IRQFD_RESAMPLE:
2880 #endif
2881 case KVM_CAP_IOEVENTFD_ANY_LENGTH:
2882 case KVM_CAP_CHECK_EXTENSION_VM:
2883 return 1;
2884 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
2885 case KVM_CAP_IRQ_ROUTING:
2886 return KVM_MAX_IRQ_ROUTES;
2887 #endif
2888 #if KVM_ADDRESS_SPACE_NUM > 1
2889 case KVM_CAP_MULTI_ADDRESS_SPACE:
2890 return KVM_ADDRESS_SPACE_NUM;
2891 #endif
2892 case KVM_CAP_MAX_VCPU_ID:
2893 return KVM_MAX_VCPU_ID;
2894 default:
2895 break;
2897 return kvm_vm_ioctl_check_extension(kvm, arg);
2900 static long kvm_vm_ioctl(struct file *filp,
2901 unsigned int ioctl, unsigned long arg)
2903 struct kvm *kvm = filp->private_data;
2904 void __user *argp = (void __user *)arg;
2905 int r;
2907 if (kvm->mm != current->mm)
2908 return -EIO;
2909 switch (ioctl) {
2910 case KVM_CREATE_VCPU:
2911 r = kvm_vm_ioctl_create_vcpu(kvm, arg);
2912 break;
2913 case KVM_SET_USER_MEMORY_REGION: {
2914 struct kvm_userspace_memory_region kvm_userspace_mem;
2916 r = -EFAULT;
2917 if (copy_from_user(&kvm_userspace_mem, argp,
2918 sizeof(kvm_userspace_mem)))
2919 goto out;
2921 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem);
2922 break;
2924 case KVM_GET_DIRTY_LOG: {
2925 struct kvm_dirty_log log;
2927 r = -EFAULT;
2928 if (copy_from_user(&log, argp, sizeof(log)))
2929 goto out;
2930 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
2931 break;
2933 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
2934 case KVM_REGISTER_COALESCED_MMIO: {
2935 struct kvm_coalesced_mmio_zone zone;
2937 r = -EFAULT;
2938 if (copy_from_user(&zone, argp, sizeof(zone)))
2939 goto out;
2940 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
2941 break;
2943 case KVM_UNREGISTER_COALESCED_MMIO: {
2944 struct kvm_coalesced_mmio_zone zone;
2946 r = -EFAULT;
2947 if (copy_from_user(&zone, argp, sizeof(zone)))
2948 goto out;
2949 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
2950 break;
2952 #endif
2953 case KVM_IRQFD: {
2954 struct kvm_irqfd data;
2956 r = -EFAULT;
2957 if (copy_from_user(&data, argp, sizeof(data)))
2958 goto out;
2959 r = kvm_irqfd(kvm, &data);
2960 break;
2962 case KVM_IOEVENTFD: {
2963 struct kvm_ioeventfd data;
2965 r = -EFAULT;
2966 if (copy_from_user(&data, argp, sizeof(data)))
2967 goto out;
2968 r = kvm_ioeventfd(kvm, &data);
2969 break;
2971 #ifdef CONFIG_HAVE_KVM_MSI
2972 case KVM_SIGNAL_MSI: {
2973 struct kvm_msi msi;
2975 r = -EFAULT;
2976 if (copy_from_user(&msi, argp, sizeof(msi)))
2977 goto out;
2978 r = kvm_send_userspace_msi(kvm, &msi);
2979 break;
2981 #endif
2982 #ifdef __KVM_HAVE_IRQ_LINE
2983 case KVM_IRQ_LINE_STATUS:
2984 case KVM_IRQ_LINE: {
2985 struct kvm_irq_level irq_event;
2987 r = -EFAULT;
2988 if (copy_from_user(&irq_event, argp, sizeof(irq_event)))
2989 goto out;
2991 r = kvm_vm_ioctl_irq_line(kvm, &irq_event,
2992 ioctl == KVM_IRQ_LINE_STATUS);
2993 if (r)
2994 goto out;
2996 r = -EFAULT;
2997 if (ioctl == KVM_IRQ_LINE_STATUS) {
2998 if (copy_to_user(argp, &irq_event, sizeof(irq_event)))
2999 goto out;
3002 r = 0;
3003 break;
3005 #endif
3006 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
3007 case KVM_SET_GSI_ROUTING: {
3008 struct kvm_irq_routing routing;
3009 struct kvm_irq_routing __user *urouting;
3010 struct kvm_irq_routing_entry *entries = NULL;
3012 r = -EFAULT;
3013 if (copy_from_user(&routing, argp, sizeof(routing)))
3014 goto out;
3015 r = -EINVAL;
3016 if (routing.nr > KVM_MAX_IRQ_ROUTES)
3017 goto out;
3018 if (routing.flags)
3019 goto out;
3020 if (routing.nr) {
3021 r = -ENOMEM;
3022 entries = vmalloc(routing.nr * sizeof(*entries));
3023 if (!entries)
3024 goto out;
3025 r = -EFAULT;
3026 urouting = argp;
3027 if (copy_from_user(entries, urouting->entries,
3028 routing.nr * sizeof(*entries)))
3029 goto out_free_irq_routing;
3031 r = kvm_set_irq_routing(kvm, entries, routing.nr,
3032 routing.flags);
3033 out_free_irq_routing:
3034 vfree(entries);
3035 break;
3037 #endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */
3038 case KVM_CREATE_DEVICE: {
3039 struct kvm_create_device cd;
3041 r = -EFAULT;
3042 if (copy_from_user(&cd, argp, sizeof(cd)))
3043 goto out;
3045 r = kvm_ioctl_create_device(kvm, &cd);
3046 if (r)
3047 goto out;
3049 r = -EFAULT;
3050 if (copy_to_user(argp, &cd, sizeof(cd)))
3051 goto out;
3053 r = 0;
3054 break;
3056 case KVM_CHECK_EXTENSION:
3057 r = kvm_vm_ioctl_check_extension_generic(kvm, arg);
3058 break;
3059 default:
3060 r = kvm_arch_vm_ioctl(filp, ioctl, arg);
3062 out:
3063 return r;
3066 #ifdef CONFIG_KVM_COMPAT
3067 struct compat_kvm_dirty_log {
3068 __u32 slot;
3069 __u32 padding1;
3070 union {
3071 compat_uptr_t dirty_bitmap; /* one bit per page */
3072 __u64 padding2;
3076 static long kvm_vm_compat_ioctl(struct file *filp,
3077 unsigned int ioctl, unsigned long arg)
3079 struct kvm *kvm = filp->private_data;
3080 int r;
3082 if (kvm->mm != current->mm)
3083 return -EIO;
3084 switch (ioctl) {
3085 case KVM_GET_DIRTY_LOG: {
3086 struct compat_kvm_dirty_log compat_log;
3087 struct kvm_dirty_log log;
3089 r = -EFAULT;
3090 if (copy_from_user(&compat_log, (void __user *)arg,
3091 sizeof(compat_log)))
3092 goto out;
3093 log.slot = compat_log.slot;
3094 log.padding1 = compat_log.padding1;
3095 log.padding2 = compat_log.padding2;
3096 log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);
3098 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
3099 break;
3101 default:
3102 r = kvm_vm_ioctl(filp, ioctl, arg);
3105 out:
3106 return r;
3108 #endif
3110 static struct file_operations kvm_vm_fops = {
3111 .release = kvm_vm_release,
3112 .unlocked_ioctl = kvm_vm_ioctl,
3113 #ifdef CONFIG_KVM_COMPAT
3114 .compat_ioctl = kvm_vm_compat_ioctl,
3115 #endif
3116 .llseek = noop_llseek,
3119 static int kvm_dev_ioctl_create_vm(unsigned long type)
3121 int r;
3122 struct kvm *kvm;
3123 struct file *file;
3125 kvm = kvm_create_vm(type);
3126 if (IS_ERR(kvm))
3127 return PTR_ERR(kvm);
3128 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
3129 r = kvm_coalesced_mmio_init(kvm);
3130 if (r < 0) {
3131 kvm_put_kvm(kvm);
3132 return r;
3134 #endif
3135 r = get_unused_fd_flags(O_CLOEXEC);
3136 if (r < 0) {
3137 kvm_put_kvm(kvm);
3138 return r;
3140 file = anon_inode_getfile("kvm-vm", &kvm_vm_fops, kvm, O_RDWR);
3141 if (IS_ERR(file)) {
3142 put_unused_fd(r);
3143 kvm_put_kvm(kvm);
3144 return PTR_ERR(file);
3147 if (kvm_create_vm_debugfs(kvm, r) < 0) {
3148 put_unused_fd(r);
3149 fput(file);
3150 return -ENOMEM;
3153 fd_install(r, file);
3154 return r;
3157 static long kvm_dev_ioctl(struct file *filp,
3158 unsigned int ioctl, unsigned long arg)
3160 long r = -EINVAL;
3162 switch (ioctl) {
3163 case KVM_GET_API_VERSION:
3164 if (arg)
3165 goto out;
3166 r = KVM_API_VERSION;
3167 break;
3168 case KVM_CREATE_VM:
3169 r = kvm_dev_ioctl_create_vm(arg);
3170 break;
3171 case KVM_CHECK_EXTENSION:
3172 r = kvm_vm_ioctl_check_extension_generic(NULL, arg);
3173 break;
3174 case KVM_GET_VCPU_MMAP_SIZE:
3175 if (arg)
3176 goto out;
3177 r = PAGE_SIZE; /* struct kvm_run */
3178 #ifdef CONFIG_X86
3179 r += PAGE_SIZE; /* pio data page */
3180 #endif
3181 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
3182 r += PAGE_SIZE; /* coalesced mmio ring page */
3183 #endif
3184 break;
3185 case KVM_TRACE_ENABLE:
3186 case KVM_TRACE_PAUSE:
3187 case KVM_TRACE_DISABLE:
3188 r = -EOPNOTSUPP;
3189 break;
3190 default:
3191 return kvm_arch_dev_ioctl(filp, ioctl, arg);
3193 out:
3194 return r;
3197 static struct file_operations kvm_chardev_ops = {
3198 .unlocked_ioctl = kvm_dev_ioctl,
3199 .compat_ioctl = kvm_dev_ioctl,
3200 .llseek = noop_llseek,
3203 static struct miscdevice kvm_dev = {
3204 KVM_MINOR,
3205 "kvm",
3206 &kvm_chardev_ops,
3209 static void hardware_enable_nolock(void *junk)
3211 int cpu = raw_smp_processor_id();
3212 int r;
3214 if (cpumask_test_cpu(cpu, cpus_hardware_enabled))
3215 return;
3217 cpumask_set_cpu(cpu, cpus_hardware_enabled);
3219 r = kvm_arch_hardware_enable();
3221 if (r) {
3222 cpumask_clear_cpu(cpu, cpus_hardware_enabled);
3223 atomic_inc(&hardware_enable_failed);
3224 pr_info("kvm: enabling virtualization on CPU%d failed\n", cpu);
3228 static int kvm_starting_cpu(unsigned int cpu)
3230 raw_spin_lock(&kvm_count_lock);
3231 if (kvm_usage_count)
3232 hardware_enable_nolock(NULL);
3233 raw_spin_unlock(&kvm_count_lock);
3234 return 0;
3237 static void hardware_disable_nolock(void *junk)
3239 int cpu = raw_smp_processor_id();
3241 if (!cpumask_test_cpu(cpu, cpus_hardware_enabled))
3242 return;
3243 cpumask_clear_cpu(cpu, cpus_hardware_enabled);
3244 kvm_arch_hardware_disable();
3247 static int kvm_dying_cpu(unsigned int cpu)
3249 raw_spin_lock(&kvm_count_lock);
3250 if (kvm_usage_count)
3251 hardware_disable_nolock(NULL);
3252 raw_spin_unlock(&kvm_count_lock);
3253 return 0;
3256 static void hardware_disable_all_nolock(void)
3258 BUG_ON(!kvm_usage_count);
3260 kvm_usage_count--;
3261 if (!kvm_usage_count)
3262 on_each_cpu(hardware_disable_nolock, NULL, 1);
3265 static void hardware_disable_all(void)
3267 raw_spin_lock(&kvm_count_lock);
3268 hardware_disable_all_nolock();
3269 raw_spin_unlock(&kvm_count_lock);
3272 static int hardware_enable_all(void)
3274 int r = 0;
3276 raw_spin_lock(&kvm_count_lock);
3278 kvm_usage_count++;
3279 if (kvm_usage_count == 1) {
3280 atomic_set(&hardware_enable_failed, 0);
3281 on_each_cpu(hardware_enable_nolock, NULL, 1);
3283 if (atomic_read(&hardware_enable_failed)) {
3284 hardware_disable_all_nolock();
3285 r = -EBUSY;
3289 raw_spin_unlock(&kvm_count_lock);
3291 return r;
3294 static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
3295 void *v)
3298 * Some (well, at least mine) BIOSes hang on reboot if
3299 * in vmx root mode.
3301 * And Intel TXT required VMX off for all cpu when system shutdown.
3303 pr_info("kvm: exiting hardware virtualization\n");
3304 kvm_rebooting = true;
3305 on_each_cpu(hardware_disable_nolock, NULL, 1);
3306 return NOTIFY_OK;
3309 static struct notifier_block kvm_reboot_notifier = {
3310 .notifier_call = kvm_reboot,
3311 .priority = 0,
3314 static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
3316 int i;
3318 for (i = 0; i < bus->dev_count; i++) {
3319 struct kvm_io_device *pos = bus->range[i].dev;
3321 kvm_iodevice_destructor(pos);
3323 kfree(bus);
3326 static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1,
3327 const struct kvm_io_range *r2)
3329 gpa_t addr1 = r1->addr;
3330 gpa_t addr2 = r2->addr;
3332 if (addr1 < addr2)
3333 return -1;
3335 /* If r2->len == 0, match the exact address. If r2->len != 0,
3336 * accept any overlapping write. Any order is acceptable for
3337 * overlapping ranges, because kvm_io_bus_get_first_dev ensures
3338 * we process all of them.
3340 if (r2->len) {
3341 addr1 += r1->len;
3342 addr2 += r2->len;
3345 if (addr1 > addr2)
3346 return 1;
3348 return 0;
3351 static int kvm_io_bus_sort_cmp(const void *p1, const void *p2)
3353 return kvm_io_bus_cmp(p1, p2);
3356 static int kvm_io_bus_insert_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev,
3357 gpa_t addr, int len)
3359 bus->range[bus->dev_count++] = (struct kvm_io_range) {
3360 .addr = addr,
3361 .len = len,
3362 .dev = dev,
3365 sort(bus->range, bus->dev_count, sizeof(struct kvm_io_range),
3366 kvm_io_bus_sort_cmp, NULL);
3368 return 0;
3371 static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus,
3372 gpa_t addr, int len)
3374 struct kvm_io_range *range, key;
3375 int off;
3377 key = (struct kvm_io_range) {
3378 .addr = addr,
3379 .len = len,
3382 range = bsearch(&key, bus->range, bus->dev_count,
3383 sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp);
3384 if (range == NULL)
3385 return -ENOENT;
3387 off = range - bus->range;
3389 while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0)
3390 off--;
3392 return off;
3395 static int __kvm_io_bus_write(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
3396 struct kvm_io_range *range, const void *val)
3398 int idx;
3400 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
3401 if (idx < 0)
3402 return -EOPNOTSUPP;
3404 while (idx < bus->dev_count &&
3405 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
3406 if (!kvm_iodevice_write(vcpu, bus->range[idx].dev, range->addr,
3407 range->len, val))
3408 return idx;
3409 idx++;
3412 return -EOPNOTSUPP;
3415 /* kvm_io_bus_write - called under kvm->slots_lock */
3416 int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
3417 int len, const void *val)
3419 struct kvm_io_bus *bus;
3420 struct kvm_io_range range;
3421 int r;
3423 range = (struct kvm_io_range) {
3424 .addr = addr,
3425 .len = len,
3428 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
3429 r = __kvm_io_bus_write(vcpu, bus, &range, val);
3430 return r < 0 ? r : 0;
3433 /* kvm_io_bus_write_cookie - called under kvm->slots_lock */
3434 int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx,
3435 gpa_t addr, int len, const void *val, long cookie)
3437 struct kvm_io_bus *bus;
3438 struct kvm_io_range range;
3440 range = (struct kvm_io_range) {
3441 .addr = addr,
3442 .len = len,
3445 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
3447 /* First try the device referenced by cookie. */
3448 if ((cookie >= 0) && (cookie < bus->dev_count) &&
3449 (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0))
3450 if (!kvm_iodevice_write(vcpu, bus->range[cookie].dev, addr, len,
3451 val))
3452 return cookie;
3455 * cookie contained garbage; fall back to search and return the
3456 * correct cookie value.
3458 return __kvm_io_bus_write(vcpu, bus, &range, val);
3461 static int __kvm_io_bus_read(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
3462 struct kvm_io_range *range, void *val)
3464 int idx;
3466 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
3467 if (idx < 0)
3468 return -EOPNOTSUPP;
3470 while (idx < bus->dev_count &&
3471 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
3472 if (!kvm_iodevice_read(vcpu, bus->range[idx].dev, range->addr,
3473 range->len, val))
3474 return idx;
3475 idx++;
3478 return -EOPNOTSUPP;
3480 EXPORT_SYMBOL_GPL(kvm_io_bus_write);
3482 /* kvm_io_bus_read - called under kvm->slots_lock */
3483 int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
3484 int len, void *val)
3486 struct kvm_io_bus *bus;
3487 struct kvm_io_range range;
3488 int r;
3490 range = (struct kvm_io_range) {
3491 .addr = addr,
3492 .len = len,
3495 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
3496 r = __kvm_io_bus_read(vcpu, bus, &range, val);
3497 return r < 0 ? r : 0;
3501 /* Caller must hold slots_lock. */
3502 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
3503 int len, struct kvm_io_device *dev)
3505 struct kvm_io_bus *new_bus, *bus;
3507 bus = kvm->buses[bus_idx];
3508 /* exclude ioeventfd which is limited by maximum fd */
3509 if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1)
3510 return -ENOSPC;
3512 new_bus = kmalloc(sizeof(*bus) + ((bus->dev_count + 1) *
3513 sizeof(struct kvm_io_range)), GFP_KERNEL);
3514 if (!new_bus)
3515 return -ENOMEM;
3516 memcpy(new_bus, bus, sizeof(*bus) + (bus->dev_count *
3517 sizeof(struct kvm_io_range)));
3518 kvm_io_bus_insert_dev(new_bus, dev, addr, len);
3519 rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
3520 synchronize_srcu_expedited(&kvm->srcu);
3521 kfree(bus);
3523 return 0;
3526 /* Caller must hold slots_lock. */
3527 int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
3528 struct kvm_io_device *dev)
3530 int i, r;
3531 struct kvm_io_bus *new_bus, *bus;
3533 bus = kvm->buses[bus_idx];
3534 r = -ENOENT;
3535 for (i = 0; i < bus->dev_count; i++)
3536 if (bus->range[i].dev == dev) {
3537 r = 0;
3538 break;
3541 if (r)
3542 return r;
3544 new_bus = kmalloc(sizeof(*bus) + ((bus->dev_count - 1) *
3545 sizeof(struct kvm_io_range)), GFP_KERNEL);
3546 if (!new_bus)
3547 return -ENOMEM;
3549 memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
3550 new_bus->dev_count--;
3551 memcpy(new_bus->range + i, bus->range + i + 1,
3552 (new_bus->dev_count - i) * sizeof(struct kvm_io_range));
3554 rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
3555 synchronize_srcu_expedited(&kvm->srcu);
3556 kfree(bus);
3557 return r;
3560 struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx,
3561 gpa_t addr)
3563 struct kvm_io_bus *bus;
3564 int dev_idx, srcu_idx;
3565 struct kvm_io_device *iodev = NULL;
3567 srcu_idx = srcu_read_lock(&kvm->srcu);
3569 bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
3571 dev_idx = kvm_io_bus_get_first_dev(bus, addr, 1);
3572 if (dev_idx < 0)
3573 goto out_unlock;
3575 iodev = bus->range[dev_idx].dev;
3577 out_unlock:
3578 srcu_read_unlock(&kvm->srcu, srcu_idx);
3580 return iodev;
3582 EXPORT_SYMBOL_GPL(kvm_io_bus_get_dev);
3584 static int kvm_debugfs_open(struct inode *inode, struct file *file,
3585 int (*get)(void *, u64 *), int (*set)(void *, u64),
3586 const char *fmt)
3588 struct kvm_stat_data *stat_data = (struct kvm_stat_data *)
3589 inode->i_private;
3591 /* The debugfs files are a reference to the kvm struct which
3592 * is still valid when kvm_destroy_vm is called.
3593 * To avoid the race between open and the removal of the debugfs
3594 * directory we test against the users count.
3596 if (!atomic_add_unless(&stat_data->kvm->users_count, 1, 0))
3597 return -ENOENT;
3599 if (simple_attr_open(inode, file, get, set, fmt)) {
3600 kvm_put_kvm(stat_data->kvm);
3601 return -ENOMEM;
3604 return 0;
3607 static int kvm_debugfs_release(struct inode *inode, struct file *file)
3609 struct kvm_stat_data *stat_data = (struct kvm_stat_data *)
3610 inode->i_private;
3612 simple_attr_release(inode, file);
3613 kvm_put_kvm(stat_data->kvm);
3615 return 0;
3618 static int vm_stat_get_per_vm(void *data, u64 *val)
3620 struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;
3622 *val = *(u32 *)((void *)stat_data->kvm + stat_data->offset);
3624 return 0;
3627 static int vm_stat_get_per_vm_open(struct inode *inode, struct file *file)
3629 __simple_attr_check_format("%llu\n", 0ull);
3630 return kvm_debugfs_open(inode, file, vm_stat_get_per_vm,
3631 NULL, "%llu\n");
3634 static const struct file_operations vm_stat_get_per_vm_fops = {
3635 .owner = THIS_MODULE,
3636 .open = vm_stat_get_per_vm_open,
3637 .release = kvm_debugfs_release,
3638 .read = simple_attr_read,
3639 .write = simple_attr_write,
3640 .llseek = generic_file_llseek,
3643 static int vcpu_stat_get_per_vm(void *data, u64 *val)
3645 int i;
3646 struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;
3647 struct kvm_vcpu *vcpu;
3649 *val = 0;
3651 kvm_for_each_vcpu(i, vcpu, stat_data->kvm)
3652 *val += *(u32 *)((void *)vcpu + stat_data->offset);
3654 return 0;
3657 static int vcpu_stat_get_per_vm_open(struct inode *inode, struct file *file)
3659 __simple_attr_check_format("%llu\n", 0ull);
3660 return kvm_debugfs_open(inode, file, vcpu_stat_get_per_vm,
3661 NULL, "%llu\n");
3664 static const struct file_operations vcpu_stat_get_per_vm_fops = {
3665 .owner = THIS_MODULE,
3666 .open = vcpu_stat_get_per_vm_open,
3667 .release = kvm_debugfs_release,
3668 .read = simple_attr_read,
3669 .write = simple_attr_write,
3670 .llseek = generic_file_llseek,
3673 static const struct file_operations *stat_fops_per_vm[] = {
3674 [KVM_STAT_VCPU] = &vcpu_stat_get_per_vm_fops,
3675 [KVM_STAT_VM] = &vm_stat_get_per_vm_fops,
3678 static int vm_stat_get(void *_offset, u64 *val)
3680 unsigned offset = (long)_offset;
3681 struct kvm *kvm;
3682 struct kvm_stat_data stat_tmp = {.offset = offset};
3683 u64 tmp_val;
3685 *val = 0;
3686 spin_lock(&kvm_lock);
3687 list_for_each_entry(kvm, &vm_list, vm_list) {
3688 stat_tmp.kvm = kvm;
3689 vm_stat_get_per_vm((void *)&stat_tmp, &tmp_val);
3690 *val += tmp_val;
3692 spin_unlock(&kvm_lock);
3693 return 0;
3696 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n");
3698 static int vcpu_stat_get(void *_offset, u64 *val)
3700 unsigned offset = (long)_offset;
3701 struct kvm *kvm;
3702 struct kvm_stat_data stat_tmp = {.offset = offset};
3703 u64 tmp_val;
3705 *val = 0;
3706 spin_lock(&kvm_lock);
3707 list_for_each_entry(kvm, &vm_list, vm_list) {
3708 stat_tmp.kvm = kvm;
3709 vcpu_stat_get_per_vm((void *)&stat_tmp, &tmp_val);
3710 *val += tmp_val;
3712 spin_unlock(&kvm_lock);
3713 return 0;
3716 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n");
3718 static const struct file_operations *stat_fops[] = {
3719 [KVM_STAT_VCPU] = &vcpu_stat_fops,
3720 [KVM_STAT_VM] = &vm_stat_fops,
3723 static int kvm_init_debug(void)
3725 int r = -EEXIST;
3726 struct kvm_stats_debugfs_item *p;
3728 kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
3729 if (kvm_debugfs_dir == NULL)
3730 goto out;
3732 kvm_debugfs_num_entries = 0;
3733 for (p = debugfs_entries; p->name; ++p, kvm_debugfs_num_entries++) {
3734 if (!debugfs_create_file(p->name, 0444, kvm_debugfs_dir,
3735 (void *)(long)p->offset,
3736 stat_fops[p->kind]))
3737 goto out_dir;
3740 return 0;
3742 out_dir:
3743 debugfs_remove_recursive(kvm_debugfs_dir);
3744 out:
3745 return r;
3748 static int kvm_suspend(void)
3750 if (kvm_usage_count)
3751 hardware_disable_nolock(NULL);
3752 return 0;
3755 static void kvm_resume(void)
3757 if (kvm_usage_count) {
3758 WARN_ON(raw_spin_is_locked(&kvm_count_lock));
3759 hardware_enable_nolock(NULL);
3763 static struct syscore_ops kvm_syscore_ops = {
3764 .suspend = kvm_suspend,
3765 .resume = kvm_resume,
3768 static inline
3769 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
3771 return container_of(pn, struct kvm_vcpu, preempt_notifier);
3774 static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
3776 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
3778 if (vcpu->preempted)
3779 vcpu->preempted = false;
3781 kvm_arch_sched_in(vcpu, cpu);
3783 kvm_arch_vcpu_load(vcpu, cpu);
3786 static void kvm_sched_out(struct preempt_notifier *pn,
3787 struct task_struct *next)
3789 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
3791 if (current->state == TASK_RUNNING)
3792 vcpu->preempted = true;
3793 kvm_arch_vcpu_put(vcpu);
3796 int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
3797 struct module *module)
3799 int r;
3800 int cpu;
3802 r = kvm_arch_init(opaque);
3803 if (r)
3804 goto out_fail;
3807 * kvm_arch_init makes sure there's at most one caller
3808 * for architectures that support multiple implementations,
3809 * like intel and amd on x86.
3810 * kvm_arch_init must be called before kvm_irqfd_init to avoid creating
3811 * conflicts in case kvm is already setup for another implementation.
3813 r = kvm_irqfd_init();
3814 if (r)
3815 goto out_irqfd;
3817 if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
3818 r = -ENOMEM;
3819 goto out_free_0;
3822 r = kvm_arch_hardware_setup();
3823 if (r < 0)
3824 goto out_free_0a;
3826 for_each_online_cpu(cpu) {
3827 smp_call_function_single(cpu,
3828 kvm_arch_check_processor_compat,
3829 &r, 1);
3830 if (r < 0)
3831 goto out_free_1;
3834 r = cpuhp_setup_state_nocalls(CPUHP_AP_KVM_STARTING, "AP_KVM_STARTING",
3835 kvm_starting_cpu, kvm_dying_cpu);
3836 if (r)
3837 goto out_free_2;
3838 register_reboot_notifier(&kvm_reboot_notifier);
3840 /* A kmem cache lets us meet the alignment requirements of fx_save. */
3841 if (!vcpu_align)
3842 vcpu_align = __alignof__(struct kvm_vcpu);
3843 kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align,
3844 0, NULL);
3845 if (!kvm_vcpu_cache) {
3846 r = -ENOMEM;
3847 goto out_free_3;
3850 r = kvm_async_pf_init();
3851 if (r)
3852 goto out_free;
3854 kvm_chardev_ops.owner = module;
3855 kvm_vm_fops.owner = module;
3856 kvm_vcpu_fops.owner = module;
3858 r = misc_register(&kvm_dev);
3859 if (r) {
3860 pr_err("kvm: misc device register failed\n");
3861 goto out_unreg;
3864 register_syscore_ops(&kvm_syscore_ops);
3866 kvm_preempt_ops.sched_in = kvm_sched_in;
3867 kvm_preempt_ops.sched_out = kvm_sched_out;
3869 r = kvm_init_debug();
3870 if (r) {
3871 pr_err("kvm: create debugfs files failed\n");
3872 goto out_undebugfs;
3875 r = kvm_vfio_ops_init();
3876 WARN_ON(r);
3878 return 0;
3880 out_undebugfs:
3881 unregister_syscore_ops(&kvm_syscore_ops);
3882 misc_deregister(&kvm_dev);
3883 out_unreg:
3884 kvm_async_pf_deinit();
3885 out_free:
3886 kmem_cache_destroy(kvm_vcpu_cache);
3887 out_free_3:
3888 unregister_reboot_notifier(&kvm_reboot_notifier);
3889 cpuhp_remove_state_nocalls(CPUHP_AP_KVM_STARTING);
3890 out_free_2:
3891 out_free_1:
3892 kvm_arch_hardware_unsetup();
3893 out_free_0a:
3894 free_cpumask_var(cpus_hardware_enabled);
3895 out_free_0:
3896 kvm_irqfd_exit();
3897 out_irqfd:
3898 kvm_arch_exit();
3899 out_fail:
3900 return r;
3902 EXPORT_SYMBOL_GPL(kvm_init);
3904 void kvm_exit(void)
3906 debugfs_remove_recursive(kvm_debugfs_dir);
3907 misc_deregister(&kvm_dev);
3908 kmem_cache_destroy(kvm_vcpu_cache);
3909 kvm_async_pf_deinit();
3910 unregister_syscore_ops(&kvm_syscore_ops);
3911 unregister_reboot_notifier(&kvm_reboot_notifier);
3912 cpuhp_remove_state_nocalls(CPUHP_AP_KVM_STARTING);
3913 on_each_cpu(hardware_disable_nolock, NULL, 1);
3914 kvm_arch_hardware_unsetup();
3915 kvm_arch_exit();
3916 kvm_irqfd_exit();
3917 free_cpumask_var(cpus_hardware_enabled);
3918 kvm_vfio_ops_exit();
3920 EXPORT_SYMBOL_GPL(kvm_exit);