arch/riscv/kvm/tlb.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2022 Ventana Micro Systems Inc.
   4  */
   5
   6 #include <linux/bitmap.h>
   7 #include <linux/cpumask.h>
   8 #include <linux/errno.h>
   9 #include <linux/err.h>
  10 #include <linux/module.h>
  11 #include <linux/smp.h>
  12 #include <linux/kvm_host.h>
  13 #include <asm/cacheflush.h>
  14 #include <asm/csr.h>
  15 #include <asm/cpufeature.h>
  16 #include <asm/insn-def.h>
  17 #include <asm/kvm_nacl.h>
  18
  19 #define has_svinval()   riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL)
  20
  21 void kvm_riscv_local_hfence_gvma_vmid_gpa(unsigned long vmid,
  22                                           gpa_t gpa, gpa_t gpsz,
  23                                           unsigned long order)
  24 {
  25         gpa_t pos;
  26
  27         if (PTRS_PER_PTE < (gpsz >> order)) {
  28                 kvm_riscv_local_hfence_gvma_vmid_all(vmid);
  29                 return;
  30         }
  31
  32         if (has_svinval()) {
  33                 asm volatile (SFENCE_W_INVAL() ::: "memory");
  34                 for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order))
  35                         asm volatile (HINVAL_GVMA(%0, %1)
  36                         : : "r" (pos >> 2), "r" (vmid) : "memory");
  37                 asm volatile (SFENCE_INVAL_IR() ::: "memory");
  38         } else {
  39                 for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order))
  40                         asm volatile (HFENCE_GVMA(%0, %1)
  41                         : : "r" (pos >> 2), "r" (vmid) : "memory");
  42         }
  43 }
  44
  45 void kvm_riscv_local_hfence_gvma_vmid_all(unsigned long vmid)
  46 {
  47         asm volatile(HFENCE_GVMA(zero, %0) : : "r" (vmid) : "memory");
  48 }
  49
  50 void kvm_riscv_local_hfence_gvma_gpa(gpa_t gpa, gpa_t gpsz,
  51                                      unsigned long order)
  52 {
  53         gpa_t pos;
  54
  55         if (PTRS_PER_PTE < (gpsz >> order)) {
  56                 kvm_riscv_local_hfence_gvma_all();
  57                 return;
  58         }
  59
  60         if (has_svinval()) {
  61                 asm volatile (SFENCE_W_INVAL() ::: "memory");
  62                 for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order))
  63                         asm volatile(HINVAL_GVMA(%0, zero)
  64                         : : "r" (pos >> 2) : "memory");
  65                 asm volatile (SFENCE_INVAL_IR() ::: "memory");
  66         } else {
  67                 for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order))
  68                         asm volatile(HFENCE_GVMA(%0, zero)
  69                         : : "r" (pos >> 2) : "memory");
  70         }
  71 }
  72
  73 void kvm_riscv_local_hfence_gvma_all(void)
  74 {
  75         asm volatile(HFENCE_GVMA(zero, zero) : : : "memory");
  76 }
  77
  78 void kvm_riscv_local_hfence_vvma_asid_gva(unsigned long vmid,
  79                                           unsigned long asid,
  80                                           unsigned long gva,
  81                                           unsigned long gvsz,
  82                                           unsigned long order)
  83 {
  84         unsigned long pos, hgatp;
  85
  86         if (PTRS_PER_PTE < (gvsz >> order)) {
  87                 kvm_riscv_local_hfence_vvma_asid_all(vmid, asid);
  88                 return;
  89         }
  90
  91         hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT);
  92
  93         if (has_svinval()) {
  94                 asm volatile (SFENCE_W_INVAL() ::: "memory");
  95                 for (pos = gva; pos < (gva + gvsz); pos += BIT(order))
  96                         asm volatile(HINVAL_VVMA(%0, %1)
  97                         : : "r" (pos), "r" (asid) : "memory");
  98                 asm volatile (SFENCE_INVAL_IR() ::: "memory");
  99         } else {
 100                 for (pos = gva; pos < (gva + gvsz); pos += BIT(order))
 101                         asm volatile(HFENCE_VVMA(%0, %1)
 102                         : : "r" (pos), "r" (asid) : "memory");
 103         }
 104
 105         csr_write(CSR_HGATP, hgatp);
 106 }
 107
 108 void kvm_riscv_local_hfence_vvma_asid_all(unsigned long vmid,
 109                                           unsigned long asid)
 110 {
 111         unsigned long hgatp;
 112
 113         hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT);
 114
 115         asm volatile(HFENCE_VVMA(zero, %0) : : "r" (asid) : "memory");
 116
 117         csr_write(CSR_HGATP, hgatp);
 118 }
 119
 120 void kvm_riscv_local_hfence_vvma_gva(unsigned long vmid,
 121                                      unsigned long gva, unsigned long gvsz,
 122                                      unsigned long order)
 123 {
 124         unsigned long pos, hgatp;
 125
 126         if (PTRS_PER_PTE < (gvsz >> order)) {
 127                 kvm_riscv_local_hfence_vvma_all(vmid);
 128                 return;
 129         }
 130
 131         hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT);
 132
 133         if (has_svinval()) {
 134                 asm volatile (SFENCE_W_INVAL() ::: "memory");
 135                 for (pos = gva; pos < (gva + gvsz); pos += BIT(order))
 136                         asm volatile(HINVAL_VVMA(%0, zero)
 137                         : : "r" (pos) : "memory");
 138                 asm volatile (SFENCE_INVAL_IR() ::: "memory");
 139         } else {
 140                 for (pos = gva; pos < (gva + gvsz); pos += BIT(order))
 141                         asm volatile(HFENCE_VVMA(%0, zero)
 142                         : : "r" (pos) : "memory");
 143         }
 144
 145         csr_write(CSR_HGATP, hgatp);
 146 }
 147
 148 void kvm_riscv_local_hfence_vvma_all(unsigned long vmid)
 149 {
 150         unsigned long hgatp;
 151
 152         hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT);
 153
 154         asm volatile(HFENCE_VVMA(zero, zero) : : : "memory");
 155
 156         csr_write(CSR_HGATP, hgatp);
 157 }
 158
 159 void kvm_riscv_local_tlb_sanitize(struct kvm_vcpu *vcpu)
 160 {
 161         unsigned long vmid;
 162
 163         if (!kvm_riscv_gstage_vmid_bits() ||
 164             vcpu->arch.last_exit_cpu == vcpu->cpu)
 165                 return;
 166
 167         /*
 168          * On RISC-V platforms with hardware VMID support, we share same
 169          * VMID for all VCPUs of a particular Guest/VM. This means we might
 170          * have stale G-stage TLB entries on the current Host CPU due to
 171          * some other VCPU of the same Guest which ran previously on the
 172          * current Host CPU.
 173          *
 174          * To cleanup stale TLB entries, we simply flush all G-stage TLB
 175          * entries by VMID whenever underlying Host CPU changes for a VCPU.
 176          */
 177
 178         vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid);
 179         kvm_riscv_local_hfence_gvma_vmid_all(vmid);
 180 }
 181
 182 void kvm_riscv_fence_i_process(struct kvm_vcpu *vcpu)
 183 {
 184         kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_FENCE_I_RCVD);
 185         local_flush_icache_all();
 186 }
 187
 188 void kvm_riscv_hfence_gvma_vmid_all_process(struct kvm_vcpu *vcpu)
 189 {
 190         struct kvm_vmid *v = &vcpu->kvm->arch.vmid;
 191         unsigned long vmid = READ_ONCE(v->vmid);
 192
 193         if (kvm_riscv_nacl_available())
 194                 nacl_hfence_gvma_vmid_all(nacl_shmem(), vmid);
 195         else
 196                 kvm_riscv_local_hfence_gvma_vmid_all(vmid);
 197 }
 198
 199 void kvm_riscv_hfence_vvma_all_process(struct kvm_vcpu *vcpu)
 200 {
 201         struct kvm_vmid *v = &vcpu->kvm->arch.vmid;
 202         unsigned long vmid = READ_ONCE(v->vmid);
 203
 204         if (kvm_riscv_nacl_available())
 205                 nacl_hfence_vvma_all(nacl_shmem(), vmid);
 206         else
 207                 kvm_riscv_local_hfence_vvma_all(vmid);
 208 }
 209
 210 static bool vcpu_hfence_dequeue(struct kvm_vcpu *vcpu,
 211                                 struct kvm_riscv_hfence *out_data)
 212 {
 213         bool ret = false;
 214         struct kvm_vcpu_arch *varch = &vcpu->arch;
 215
 216         spin_lock(&varch->hfence_lock);
 217
 218         if (varch->hfence_queue[varch->hfence_head].type) {
 219                 memcpy(out_data, &varch->hfence_queue[varch->hfence_head],
 220                        sizeof(*out_data));
 221                 varch->hfence_queue[varch->hfence_head].type = 0;
 222
 223                 varch->hfence_head++;
 224                 if (varch->hfence_head == KVM_RISCV_VCPU_MAX_HFENCE)
 225                         varch->hfence_head = 0;
 226
 227                 ret = true;
 228         }
 229
 230         spin_unlock(&varch->hfence_lock);
 231
 232         return ret;
 233 }
 234
 235 static bool vcpu_hfence_enqueue(struct kvm_vcpu *vcpu,
 236                                 const struct kvm_riscv_hfence *data)
 237 {
 238         bool ret = false;
 239         struct kvm_vcpu_arch *varch = &vcpu->arch;
 240
 241         spin_lock(&varch->hfence_lock);
 242
 243         if (!varch->hfence_queue[varch->hfence_tail].type) {
 244                 memcpy(&varch->hfence_queue[varch->hfence_tail],
 245                        data, sizeof(*data));
 246
 247                 varch->hfence_tail++;
 248                 if (varch->hfence_tail == KVM_RISCV_VCPU_MAX_HFENCE)
 249                         varch->hfence_tail = 0;
 250
 251                 ret = true;
 252         }
 253
 254         spin_unlock(&varch->hfence_lock);
 255
 256         return ret;
 257 }
 258
 259 void kvm_riscv_hfence_process(struct kvm_vcpu *vcpu)
 260 {
 261         unsigned long vmid;
 262         struct kvm_riscv_hfence d = { 0 };
 263         struct kvm_vmid *v = &vcpu->kvm->arch.vmid;
 264
 265         while (vcpu_hfence_dequeue(vcpu, &d)) {
 266                 switch (d.type) {
 267                 case KVM_RISCV_HFENCE_UNKNOWN:
 268                         break;
 269                 case KVM_RISCV_HFENCE_GVMA_VMID_GPA:
 270                         vmid = READ_ONCE(v->vmid);
 271                         if (kvm_riscv_nacl_available())
 272                                 nacl_hfence_gvma_vmid(nacl_shmem(), vmid,
 273                                                       d.addr, d.size, d.order);
 274                         else
 275                                 kvm_riscv_local_hfence_gvma_vmid_gpa(vmid, d.addr,
 276                                                                      d.size, d.order);
 277                         break;
 278                 case KVM_RISCV_HFENCE_VVMA_ASID_GVA:
 279                         kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_ASID_RCVD);
 280                         vmid = READ_ONCE(v->vmid);
 281                         if (kvm_riscv_nacl_available())
 282                                 nacl_hfence_vvma_asid(nacl_shmem(), vmid, d.asid,
 283                                                       d.addr, d.size, d.order);
 284                         else
 285                                 kvm_riscv_local_hfence_vvma_asid_gva(vmid, d.asid, d.addr,
 286                                                                      d.size, d.order);
 287                         break;
 288                 case KVM_RISCV_HFENCE_VVMA_ASID_ALL:
 289                         kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_ASID_RCVD);
 290                         vmid = READ_ONCE(v->vmid);
 291                         if (kvm_riscv_nacl_available())
 292                                 nacl_hfence_vvma_asid_all(nacl_shmem(), vmid, d.asid);
 293                         else
 294                                 kvm_riscv_local_hfence_vvma_asid_all(vmid, d.asid);
 295                         break;
 296                 case KVM_RISCV_HFENCE_VVMA_GVA:
 297                         kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_RCVD);
 298                         vmid = READ_ONCE(v->vmid);
 299                         if (kvm_riscv_nacl_available())
 300                                 nacl_hfence_vvma(nacl_shmem(), vmid,
 301                                                  d.addr, d.size, d.order);
 302                         else
 303                                 kvm_riscv_local_hfence_vvma_gva(vmid, d.addr,
 304                                                                 d.size, d.order);
 305                         break;
 306                 default:
 307                         break;
 308                 }
 309         }
 310 }
 311
 312 static void make_xfence_request(struct kvm *kvm,
 313                                 unsigned long hbase, unsigned long hmask,
 314                                 unsigned int req, unsigned int fallback_req,
 315                                 const struct kvm_riscv_hfence *data)
 316 {
 317         unsigned long i;
 318         struct kvm_vcpu *vcpu;
 319         unsigned int actual_req = req;
 320         DECLARE_BITMAP(vcpu_mask, KVM_MAX_VCPUS);
 321
 322         bitmap_zero(vcpu_mask, KVM_MAX_VCPUS);
 323         kvm_for_each_vcpu(i, vcpu, kvm) {
 324                 if (hbase != -1UL) {
 325                         if (vcpu->vcpu_id < hbase)
 326                                 continue;
 327                         if (!(hmask & (1UL << (vcpu->vcpu_id - hbase))))
 328                                 continue;
 329                 }
 330
 331                 bitmap_set(vcpu_mask, i, 1);
 332
 333                 if (!data || !data->type)
 334                         continue;
 335
 336                 /*
 337                  * Enqueue hfence data to VCPU hfence queue. If we don't
 338                  * have space in the VCPU hfence queue then fallback to
 339                  * a more conservative hfence request.
 340                  */
 341                 if (!vcpu_hfence_enqueue(vcpu, data))
 342                         actual_req = fallback_req;
 343         }
 344
 345         kvm_make_vcpus_request_mask(kvm, actual_req, vcpu_mask);
 346 }
 347
 348 void kvm_riscv_fence_i(struct kvm *kvm,
 349                        unsigned long hbase, unsigned long hmask)
 350 {
 351         make_xfence_request(kvm, hbase, hmask, KVM_REQ_FENCE_I,
 352                             KVM_REQ_FENCE_I, NULL);
 353 }
 354
 355 void kvm_riscv_hfence_gvma_vmid_gpa(struct kvm *kvm,
 356                                     unsigned long hbase, unsigned long hmask,
 357                                     gpa_t gpa, gpa_t gpsz,
 358                                     unsigned long order)
 359 {
 360         struct kvm_riscv_hfence data;
 361
 362         data.type = KVM_RISCV_HFENCE_GVMA_VMID_GPA;
 363         data.asid = 0;
 364         data.addr = gpa;
 365         data.size = gpsz;
 366         data.order = order;
 367         make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE,
 368                             KVM_REQ_HFENCE_GVMA_VMID_ALL, &data);
 369 }
 370
 371 void kvm_riscv_hfence_gvma_vmid_all(struct kvm *kvm,
 372                                     unsigned long hbase, unsigned long hmask)
 373 {
 374         make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE_GVMA_VMID_ALL,
 375                             KVM_REQ_HFENCE_GVMA_VMID_ALL, NULL);
 376 }
 377
 378 void kvm_riscv_hfence_vvma_asid_gva(struct kvm *kvm,
 379                                     unsigned long hbase, unsigned long hmask,
 380                                     unsigned long gva, unsigned long gvsz,
 381                                     unsigned long order, unsigned long asid)
 382 {
 383         struct kvm_riscv_hfence data;
 384
 385         data.type = KVM_RISCV_HFENCE_VVMA_ASID_GVA;
 386         data.asid = asid;
 387         data.addr = gva;
 388         data.size = gvsz;
 389         data.order = order;
 390         make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE,
 391                             KVM_REQ_HFENCE_VVMA_ALL, &data);
 392 }
 393
 394 void kvm_riscv_hfence_vvma_asid_all(struct kvm *kvm,
 395                                     unsigned long hbase, unsigned long hmask,
 396                                     unsigned long asid)
 397 {
 398         struct kvm_riscv_hfence data;
 399
 400         data.type = KVM_RISCV_HFENCE_VVMA_ASID_ALL;
 401         data.asid = asid;
 402         data.addr = data.size = data.order = 0;
 403         make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE,
 404                             KVM_REQ_HFENCE_VVMA_ALL, &data);
 405 }
 406
 407 void kvm_riscv_hfence_vvma_gva(struct kvm *kvm,
 408                                unsigned long hbase, unsigned long hmask,
 409                                unsigned long gva, unsigned long gvsz,
 410                                unsigned long order)
 411 {
 412         struct kvm_riscv_hfence data;
 413
 414         data.type = KVM_RISCV_HFENCE_VVMA_GVA;
 415         data.asid = 0;
 416         data.addr = gva;
 417         data.size = gvsz;
 418         data.order = order;
 419         make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE,
 420                             KVM_REQ_HFENCE_VVMA_ALL, &data);
 421 }
 422
 423 void kvm_riscv_hfence_vvma_all(struct kvm *kvm,
 424                                unsigned long hbase, unsigned long hmask)
 425 {
 426         make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE_VVMA_ALL,
 427                             KVM_REQ_HFENCE_VVMA_ALL, NULL);
 428 }