virt/kvm/dirty_ring.c

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * KVM dirty ring implementation
   4  *
   5  * Copyright 2019 Red Hat, Inc.
   6  */
   7 #include <linux/kvm_host.h>
   8 #include <linux/kvm.h>
   9 #include <linux/vmalloc.h>
  10 #include <linux/kvm_dirty_ring.h>
  11 #include <trace/events/kvm.h>
  12
  13 int __weak kvm_cpu_dirty_log_size(void)
  14 {
  15         return 0;
  16 }
  17
  18 u32 kvm_dirty_ring_get_rsvd_entries(void)
  19 {
  20         return KVM_DIRTY_RING_RSVD_ENTRIES + kvm_cpu_dirty_log_size();
  21 }
  22
  23 static u32 kvm_dirty_ring_used(struct kvm_dirty_ring *ring)
  24 {
  25         return READ_ONCE(ring->dirty_index) - READ_ONCE(ring->reset_index);
  26 }
  27
  28 bool kvm_dirty_ring_soft_full(struct kvm_dirty_ring *ring)
  29 {
  30         return kvm_dirty_ring_used(ring) >= ring->soft_limit;
  31 }
  32
  33 static bool kvm_dirty_ring_full(struct kvm_dirty_ring *ring)
  34 {
  35         return kvm_dirty_ring_used(ring) >= ring->size;
  36 }
  37
  38 struct kvm_dirty_ring *kvm_dirty_ring_get(struct kvm *kvm)
  39 {
  40         struct kvm_vcpu *vcpu = kvm_get_running_vcpu();
  41
  42         WARN_ON_ONCE(vcpu->kvm != kvm);
  43
  44         return &vcpu->dirty_ring;
  45 }
  46
  47 static void kvm_reset_dirty_gfn(struct kvm *kvm, u32 slot, u64 offset, u64 mask)
  48 {
  49         struct kvm_memory_slot *memslot;
  50         int as_id, id;
  51
  52         as_id = slot >> 16;
  53         id = (u16)slot;
  54
  55         if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
  56                 return;
  57
  58         memslot = id_to_memslot(__kvm_memslots(kvm, as_id), id);
  59
  60         if (!memslot || (offset + __fls(mask)) >= memslot->npages)
  61                 return;
  62
  63         spin_lock(&kvm->mmu_lock);
  64         kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot, offset, mask);
  65         spin_unlock(&kvm->mmu_lock);
  66 }
  67
  68 int kvm_dirty_ring_alloc(struct kvm_dirty_ring *ring, int index, u32 size)
  69 {
  70         ring->dirty_gfns = vmalloc(size);
  71         if (!ring->dirty_gfns)
  72                 return -ENOMEM;
  73         memset(ring->dirty_gfns, 0, size);
  74
  75         ring->size = size / sizeof(struct kvm_dirty_gfn);
  76         ring->soft_limit = ring->size - kvm_dirty_ring_get_rsvd_entries();
  77         ring->dirty_index = 0;
  78         ring->reset_index = 0;
  79         ring->index = index;
  80
  81         return 0;
  82 }
  83
  84 static inline void kvm_dirty_gfn_set_invalid(struct kvm_dirty_gfn *gfn)
  85 {
  86         gfn->flags = 0;
  87 }
  88
  89 static inline void kvm_dirty_gfn_set_dirtied(struct kvm_dirty_gfn *gfn)
  90 {
  91         gfn->flags = KVM_DIRTY_GFN_F_DIRTY;
  92 }
  93
  94 static inline bool kvm_dirty_gfn_invalid(struct kvm_dirty_gfn *gfn)
  95 {
  96         return gfn->flags == 0;
  97 }
  98
  99 static inline bool kvm_dirty_gfn_harvested(struct kvm_dirty_gfn *gfn)
 100 {
 101         return gfn->flags & KVM_DIRTY_GFN_F_RESET;
 102 }
 103
 104 int kvm_dirty_ring_reset(struct kvm *kvm, struct kvm_dirty_ring *ring)
 105 {
 106         u32 cur_slot, next_slot;
 107         u64 cur_offset, next_offset;
 108         unsigned long mask;
 109         int count = 0;
 110         struct kvm_dirty_gfn *entry;
 111         bool first_round = true;
 112
 113         /* This is only needed to make compilers happy */
 114         cur_slot = cur_offset = mask = 0;
 115
 116         while (true) {
 117                 entry = &ring->dirty_gfns[ring->reset_index & (ring->size - 1)];
 118
 119                 if (!kvm_dirty_gfn_harvested(entry))
 120                         break;
 121
 122                 next_slot = READ_ONCE(entry->slot);
 123                 next_offset = READ_ONCE(entry->offset);
 124
 125                 /* Update the flags to reflect that this GFN is reset */
 126                 kvm_dirty_gfn_set_invalid(entry);
 127
 128                 ring->reset_index++;
 129                 count++;
 130                 /*
 131                  * Try to coalesce the reset operations when the guest is
 132                  * scanning pages in the same slot.
 133                  */
 134                 if (!first_round && next_slot == cur_slot) {
 135                         s64 delta = next_offset - cur_offset;
 136
 137                         if (delta >= 0 && delta < BITS_PER_LONG) {
 138                                 mask |= 1ull << delta;
 139                                 continue;
 140                         }
 141
 142                         /* Backwards visit, careful about overflows!  */
 143                         if (delta > -BITS_PER_LONG && delta < 0 &&
 144                             (mask << -delta >> -delta) == mask) {
 145                                 cur_offset = next_offset;
 146                                 mask = (mask << -delta) | 1;
 147                                 continue;
 148                         }
 149                 }
 150                 kvm_reset_dirty_gfn(kvm, cur_slot, cur_offset, mask);
 151                 cur_slot = next_slot;
 152                 cur_offset = next_offset;
 153                 mask = 1;
 154                 first_round = false;
 155         }
 156
 157         kvm_reset_dirty_gfn(kvm, cur_slot, cur_offset, mask);
 158
 159         trace_kvm_dirty_ring_reset(ring);
 160
 161         return count;
 162 }
 163
 164 void kvm_dirty_ring_push(struct kvm_dirty_ring *ring, u32 slot, u64 offset)
 165 {
 166         struct kvm_dirty_gfn *entry;
 167
 168         /* It should never get full */
 169         WARN_ON_ONCE(kvm_dirty_ring_full(ring));
 170
 171         entry = &ring->dirty_gfns[ring->dirty_index & (ring->size - 1)];
 172
 173         entry->slot = slot;
 174         entry->offset = offset;
 175         /*
 176          * Make sure the data is filled in before we publish this to
 177          * the userspace program.  There's no paired kernel-side reader.
 178          */
 179         smp_wmb();
 180         kvm_dirty_gfn_set_dirtied(entry);
 181         ring->dirty_index++;
 182         trace_kvm_dirty_ring_push(ring, slot, offset);
 183 }
 184
 185 struct page *kvm_dirty_ring_get_page(struct kvm_dirty_ring *ring, u32 offset)
 186 {
 187         return vmalloc_to_page((void *)ring->dirty_gfns + offset * PAGE_SIZE);
 188 }
 189
 190 void kvm_dirty_ring_free(struct kvm_dirty_ring *ring)
 191 {
 192         vfree(ring->dirty_gfns);
 193         ring->dirty_gfns = NULL;
 194 }