arch/x86/hyperv/mmu.c

   1 #define pr_fmt(fmt)  "Hyper-V: " fmt
   2
   3 #include <linux/hyperv.h>
   4 #include <linux/log2.h>
   5 #include <linux/slab.h>
   6 #include <linux/types.h>
   7
   8 #include <asm/fpu/api.h>
   9 #include <asm/mshyperv.h>
  10 #include <asm/msr.h>
  11 #include <asm/tlbflush.h>
  12 #include <asm/tlb.h>
  13
  14 #define CREATE_TRACE_POINTS
  15 #include <asm/trace/hyperv.h>
  16
  17 /* Each gva in gva_list encodes up to 4096 pages to flush */
  18 #define HV_TLB_FLUSH_UNIT (4096 * PAGE_SIZE)
  19
  20 static u64 hyperv_flush_tlb_others_ex(const struct cpumask *cpus,
  21                                       const struct flush_tlb_info *info);
  22
  23 /*
  24  * Fills in gva_list starting from offset. Returns the number of items added.
  25  */
  26 static inline int fill_gva_list(u64 gva_list[], int offset,
  27                                 unsigned long start, unsigned long end)
  28 {
  29         int gva_n = offset;
  30         unsigned long cur = start, diff;
  31
  32         do {
  33                 diff = end > cur ? end - cur : 0;
  34
  35                 gva_list[gva_n] = cur & PAGE_MASK;
  36                 /*
  37                  * Lower 12 bits encode the number of additional
  38                  * pages to flush (in addition to the 'cur' page).
  39                  */
  40                 if (diff >= HV_TLB_FLUSH_UNIT)
  41                         gva_list[gva_n] |= ~PAGE_MASK;
  42                 else if (diff)
  43                         gva_list[gva_n] |= (diff - 1) >> PAGE_SHIFT;
  44
  45                 cur += HV_TLB_FLUSH_UNIT;
  46                 gva_n++;
  47
  48         } while (cur < end);
  49
  50         return gva_n - offset;
  51 }
  52
  53 static void hyperv_flush_tlb_others(const struct cpumask *cpus,
  54                                     const struct flush_tlb_info *info)
  55 {
  56         int cpu, vcpu, gva_n, max_gvas;
  57         struct hv_tlb_flush **flush_pcpu;
  58         struct hv_tlb_flush *flush;
  59         u64 status = U64_MAX;
  60         unsigned long flags;
  61
  62         trace_hyperv_mmu_flush_tlb_others(cpus, info);
  63
  64         if (!hv_hypercall_pg)
  65                 goto do_native;
  66
  67         if (cpumask_empty(cpus))
  68                 return;
  69
  70         local_irq_save(flags);
  71
  72         flush_pcpu = (struct hv_tlb_flush **)
  73                      this_cpu_ptr(hyperv_pcpu_input_arg);
  74
  75         flush = *flush_pcpu;
  76
  77         if (unlikely(!flush)) {
  78                 local_irq_restore(flags);
  79                 goto do_native;
  80         }
  81
  82         if (info->mm) {
  83                 /*
  84                  * AddressSpace argument must match the CR3 with PCID bits
  85                  * stripped out.
  86                  */
  87                 flush->address_space = virt_to_phys(info->mm->pgd);
  88                 flush->address_space &= CR3_ADDR_MASK;
  89                 flush->flags = 0;
  90         } else {
  91                 flush->address_space = 0;
  92                 flush->flags = HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES;
  93         }
  94
  95         flush->processor_mask = 0;
  96         if (cpumask_equal(cpus, cpu_present_mask)) {
  97                 flush->flags |= HV_FLUSH_ALL_PROCESSORS;
  98         } else {
  99                 /*
 100                  * From the supplied CPU set we need to figure out if we can get
 101                  * away with cheaper HVCALL_FLUSH_VIRTUAL_ADDRESS_{LIST,SPACE}
 102                  * hypercalls. This is possible when the highest VP number in
 103                  * the set is < 64. As VP numbers are usually in ascending order
 104                  * and match Linux CPU ids, here is an optimization: we check
 105                  * the VP number for the highest bit in the supplied set first
 106                  * so we can quickly find out if using *_EX hypercalls is a
 107                  * must. We will also check all VP numbers when walking the
 108                  * supplied CPU set to remain correct in all cases.
 109                  */
 110                 if (hv_cpu_number_to_vp_number(cpumask_last(cpus)) >= 64)
 111                         goto do_ex_hypercall;
 112
 113                 for_each_cpu(cpu, cpus) {
 114                         vcpu = hv_cpu_number_to_vp_number(cpu);
 115                         if (vcpu == VP_INVAL) {
 116                                 local_irq_restore(flags);
 117                                 goto do_native;
 118                         }
 119
 120                         if (vcpu >= 64)
 121                                 goto do_ex_hypercall;
 122
 123                         __set_bit(vcpu, (unsigned long *)
 124                                   &flush->processor_mask);
 125                 }
 126         }
 127
 128         /*
 129          * We can flush not more than max_gvas with one hypercall. Flush the
 130          * whole address space if we were asked to do more.
 131          */
 132         max_gvas = (PAGE_SIZE - sizeof(*flush)) / sizeof(flush->gva_list[0]);
 133
 134         if (info->end == TLB_FLUSH_ALL) {
 135                 flush->flags |= HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY;
 136                 status = hv_do_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE,
 137                                          flush, NULL);
 138         } else if (info->end &&
 139                    ((info->end - info->start)/HV_TLB_FLUSH_UNIT) > max_gvas) {
 140                 status = hv_do_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE,
 141                                          flush, NULL);
 142         } else {
 143                 gva_n = fill_gva_list(flush->gva_list, 0,
 144                                       info->start, info->end);
 145                 status = hv_do_rep_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST,
 146                                              gva_n, 0, flush, NULL);
 147         }
 148         goto check_status;
 149
 150 do_ex_hypercall:
 151         status = hyperv_flush_tlb_others_ex(cpus, info);
 152
 153 check_status:
 154         local_irq_restore(flags);
 155
 156         if (!(status & HV_HYPERCALL_RESULT_MASK))
 157                 return;
 158 do_native:
 159         native_flush_tlb_others(cpus, info);
 160 }
 161
 162 static u64 hyperv_flush_tlb_others_ex(const struct cpumask *cpus,
 163                                       const struct flush_tlb_info *info)
 164 {
 165         int nr_bank = 0, max_gvas, gva_n;
 166         struct hv_tlb_flush_ex **flush_pcpu;
 167         struct hv_tlb_flush_ex *flush;
 168         u64 status;
 169
 170         if (!(ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED))
 171                 return U64_MAX;
 172
 173         flush_pcpu = (struct hv_tlb_flush_ex **)
 174                      this_cpu_ptr(hyperv_pcpu_input_arg);
 175
 176         flush = *flush_pcpu;
 177
 178         if (info->mm) {
 179                 /*
 180                  * AddressSpace argument must match the CR3 with PCID bits
 181                  * stripped out.
 182                  */
 183                 flush->address_space = virt_to_phys(info->mm->pgd);
 184                 flush->address_space &= CR3_ADDR_MASK;
 185                 flush->flags = 0;
 186         } else {
 187                 flush->address_space = 0;
 188                 flush->flags = HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES;
 189         }
 190
 191         flush->hv_vp_set.valid_bank_mask = 0;
 192
 193         flush->hv_vp_set.format = HV_GENERIC_SET_SPARSE_4K;
 194         nr_bank = cpumask_to_vpset(&(flush->hv_vp_set), cpus);
 195         if (nr_bank < 0)
 196                 return U64_MAX;
 197
 198         /*
 199          * We can flush not more than max_gvas with one hypercall. Flush the
 200          * whole address space if we were asked to do more.
 201          */
 202         max_gvas =
 203                 (PAGE_SIZE - sizeof(*flush) - nr_bank *
 204                  sizeof(flush->hv_vp_set.bank_contents[0])) /
 205                 sizeof(flush->gva_list[0]);
 206
 207         if (info->end == TLB_FLUSH_ALL) {
 208                 flush->flags |= HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY;
 209                 status = hv_do_rep_hypercall(
 210                         HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX,
 211                         0, nr_bank, flush, NULL);
 212         } else if (info->end &&
 213                    ((info->end - info->start)/HV_TLB_FLUSH_UNIT) > max_gvas) {
 214                 status = hv_do_rep_hypercall(
 215                         HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX,
 216                         0, nr_bank, flush, NULL);
 217         } else {
 218                 gva_n = fill_gva_list(flush->gva_list, nr_bank,
 219                                       info->start, info->end);
 220                 status = hv_do_rep_hypercall(
 221                         HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX,
 222                         gva_n, nr_bank, flush, NULL);
 223         }
 224
 225         return status;
 226 }
 227
 228 void hyperv_setup_mmu_ops(void)
 229 {
 230         if (!(ms_hyperv.hints & HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED))
 231                 return;
 232
 233         pr_info("Using hypercall for remote TLB flush\n");
 234         pv_ops.mmu.flush_tlb_others = hyperv_flush_tlb_others;
 235         pv_ops.mmu.tlb_remove_table = tlb_remove_table;
 236 }