arch/arm64/kvm/va_layout.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2017 ARM Ltd.
   4  * Author: Marc Zyngier <marc.zyngier@arm.com>
   5  */
   6
   7 #include <linux/kvm_host.h>
   8 #include <linux/random.h>
   9 #include <linux/memblock.h>
  10 #include <asm/alternative.h>
  11 #include <asm/debug-monitors.h>
  12 #include <asm/insn.h>
  13 #include <asm/kvm_mmu.h>
  14
  15 /*
  16  * The LSB of the random hyp VA tag or 0 if no randomization is used.
  17  */
  18 static u8 tag_lsb;
  19 /*
  20  * The random hyp VA tag value with the region bit if hyp randomization is used
  21  */
  22 static u64 tag_val;
  23 static u64 va_mask;
  24
  25 static void compute_layout(void)
  26 {
  27         phys_addr_t idmap_addr = __pa_symbol(__hyp_idmap_text_start);
  28         u64 hyp_va_msb;
  29         int kva_msb;
  30
  31         /* Where is my RAM region? */
  32         hyp_va_msb  = idmap_addr & BIT(vabits_actual - 1);
  33         hyp_va_msb ^= BIT(vabits_actual - 1);
  34
  35         kva_msb = fls64((u64)phys_to_virt(memblock_start_of_DRAM()) ^
  36                         (u64)(high_memory - 1));
  37
  38         if (kva_msb == (vabits_actual - 1)) {
  39                 /*
  40                  * No space in the address, let's compute the mask so
  41                  * that it covers (vabits_actual - 1) bits, and the region
  42                  * bit. The tag stays set to zero.
  43                  */
  44                 va_mask  = BIT(vabits_actual - 1) - 1;
  45                 va_mask |= hyp_va_msb;
  46         } else {
  47                 /*
  48                  * We do have some free bits to insert a random tag.
  49                  * Hyp VAs are now created from kernel linear map VAs
  50                  * using the following formula (with V == vabits_actual):
  51                  *
  52                  *  63 ... V |     V-1    | V-2 .. tag_lsb | tag_lsb - 1 .. 0
  53                  *  ---------------------------------------------------------
  54                  * | 0000000 | hyp_va_msb |    random tag  |  kern linear VA |
  55                  */
  56                 tag_lsb = kva_msb;
  57                 va_mask = GENMASK_ULL(tag_lsb - 1, 0);
  58                 tag_val = get_random_long() & GENMASK_ULL(vabits_actual - 2, tag_lsb);
  59                 tag_val |= hyp_va_msb;
  60                 tag_val >>= tag_lsb;
  61         }
  62 }
  63
  64 static u32 compute_instruction(int n, u32 rd, u32 rn)
  65 {
  66         u32 insn = AARCH64_BREAK_FAULT;
  67
  68         switch (n) {
  69         case 0:
  70                 insn = aarch64_insn_gen_logical_immediate(AARCH64_INSN_LOGIC_AND,
  71                                                           AARCH64_INSN_VARIANT_64BIT,
  72                                                           rn, rd, va_mask);
  73                 break;
  74
  75         case 1:
  76                 /* ROR is a variant of EXTR with Rm = Rn */
  77                 insn = aarch64_insn_gen_extr(AARCH64_INSN_VARIANT_64BIT,
  78                                              rn, rn, rd,
  79                                              tag_lsb);
  80                 break;
  81
  82         case 2:
  83                 insn = aarch64_insn_gen_add_sub_imm(rd, rn,
  84                                                     tag_val & GENMASK(11, 0),
  85                                                     AARCH64_INSN_VARIANT_64BIT,
  86                                                     AARCH64_INSN_ADSB_ADD);
  87                 break;
  88
  89         case 3:
  90                 insn = aarch64_insn_gen_add_sub_imm(rd, rn,
  91                                                     tag_val & GENMASK(23, 12),
  92                                                     AARCH64_INSN_VARIANT_64BIT,
  93                                                     AARCH64_INSN_ADSB_ADD);
  94                 break;
  95
  96         case 4:
  97                 /* ROR is a variant of EXTR with Rm = Rn */
  98                 insn = aarch64_insn_gen_extr(AARCH64_INSN_VARIANT_64BIT,
  99                                              rn, rn, rd, 64 - tag_lsb);
 100                 break;
 101         }
 102
 103         return insn;
 104 }
 105
 106 void __init kvm_update_va_mask(struct alt_instr *alt,
 107                                __le32 *origptr, __le32 *updptr, int nr_inst)
 108 {
 109         int i;
 110
 111         BUG_ON(nr_inst != 5);
 112
 113         if (!has_vhe() && !va_mask)
 114                 compute_layout();
 115
 116         for (i = 0; i < nr_inst; i++) {
 117                 u32 rd, rn, insn, oinsn;
 118
 119                 /*
 120                  * VHE doesn't need any address translation, let's NOP
 121                  * everything.
 122                  *
 123                  * Alternatively, if we don't have any spare bits in
 124                  * the address, NOP everything after masking that
 125                  * kernel VA.
 126                  */
 127                 if (has_vhe() || (!tag_lsb && i > 0)) {
 128                         updptr[i] = cpu_to_le32(aarch64_insn_gen_nop());
 129                         continue;
 130                 }
 131
 132                 oinsn = le32_to_cpu(origptr[i]);
 133                 rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, oinsn);
 134                 rn = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RN, oinsn);
 135
 136                 insn = compute_instruction(i, rd, rn);
 137                 BUG_ON(insn == AARCH64_BREAK_FAULT);
 138
 139                 updptr[i] = cpu_to_le32(insn);
 140         }
 141 }
 142
 143 void *__kvm_bp_vect_base;
 144 int __kvm_harden_el2_vector_slot;
 145
 146 void kvm_patch_vector_branch(struct alt_instr *alt,
 147                              __le32 *origptr, __le32 *updptr, int nr_inst)
 148 {
 149         u64 addr;
 150         u32 insn;
 151
 152         BUG_ON(nr_inst != 5);
 153
 154         if (has_vhe() || !cpus_have_const_cap(ARM64_HARDEN_EL2_VECTORS)) {
 155                 WARN_ON_ONCE(cpus_have_const_cap(ARM64_HARDEN_EL2_VECTORS));
 156                 return;
 157         }
 158
 159         if (!va_mask)
 160                 compute_layout();
 161
 162         /*
 163          * Compute HYP VA by using the same computation as kern_hyp_va()
 164          */
 165         addr = (uintptr_t)kvm_ksym_ref(__kvm_hyp_vector);
 166         addr &= va_mask;
 167         addr |= tag_val << tag_lsb;
 168
 169         /* Use PC[10:7] to branch to the same vector in KVM */
 170         addr |= ((u64)origptr & GENMASK_ULL(10, 7));
 171
 172         /*
 173          * Branch over the preamble in order to avoid the initial store on
 174          * the stack (which we already perform in the hardening vectors).
 175          */
 176         addr += KVM_VECTOR_PREAMBLE;
 177
 178         /* stp x0, x1, [sp, #-16]! */
 179         insn = aarch64_insn_gen_load_store_pair(AARCH64_INSN_REG_0,
 180                                                 AARCH64_INSN_REG_1,
 181                                                 AARCH64_INSN_REG_SP,
 182                                                 -16,
 183                                                 AARCH64_INSN_VARIANT_64BIT,
 184                                                 AARCH64_INSN_LDST_STORE_PAIR_PRE_INDEX);
 185         *updptr++ = cpu_to_le32(insn);
 186
 187         /* movz x0, #(addr & 0xffff) */
 188         insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0,
 189                                          (u16)addr,
 190                                          0,
 191                                          AARCH64_INSN_VARIANT_64BIT,
 192                                          AARCH64_INSN_MOVEWIDE_ZERO);
 193         *updptr++ = cpu_to_le32(insn);
 194
 195         /* movk x0, #((addr >> 16) & 0xffff), lsl #16 */
 196         insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0,
 197                                          (u16)(addr >> 16),
 198                                          16,
 199                                          AARCH64_INSN_VARIANT_64BIT,
 200                                          AARCH64_INSN_MOVEWIDE_KEEP);
 201         *updptr++ = cpu_to_le32(insn);
 202
 203         /* movk x0, #((addr >> 32) & 0xffff), lsl #32 */
 204         insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0,
 205                                          (u16)(addr >> 32),
 206                                          32,
 207                                          AARCH64_INSN_VARIANT_64BIT,
 208                                          AARCH64_INSN_MOVEWIDE_KEEP);
 209         *updptr++ = cpu_to_le32(insn);
 210
 211         /* br x0 */
 212         insn = aarch64_insn_gen_branch_reg(AARCH64_INSN_REG_0,
 213                                            AARCH64_INSN_BRANCH_NOLINK);
 214         *updptr++ = cpu_to_le32(insn);
 215 }