arch/arm64/kvm/va_layout.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2017 ARM Ltd.
   4  * Author: Marc Zyngier <marc.zyngier@arm.com>
   5  */
   6
   7 #include <linux/kvm_host.h>
   8 #include <linux/random.h>
   9 #include <linux/memblock.h>
  10 #include <asm/alternative.h>
  11 #include <asm/debug-monitors.h>
  12 #include <asm/insn.h>
  13 #include <asm/kvm_mmu.h>
  14 #include <asm/memory.h>
  15
  16 /*
  17  * The LSB of the HYP VA tag
  18  */
  19 static u8 tag_lsb;
  20 /*
  21  * The HYP VA tag value with the region bit
  22  */
  23 static u64 tag_val;
  24 static u64 va_mask;
  25
  26 /*
  27  * Compute HYP VA by using the same computation as kern_hyp_va().
  28  */
  29 static u64 __early_kern_hyp_va(u64 addr)
  30 {
  31         addr &= va_mask;
  32         addr |= tag_val << tag_lsb;
  33         return addr;
  34 }
  35
  36 /*
  37  * Store a hyp VA <-> PA offset into a hyp-owned variable.
  38  */
  39 static void init_hyp_physvirt_offset(void)
  40 {
  41         extern s64 kvm_nvhe_sym(hyp_physvirt_offset);
  42         u64 kern_va, hyp_va;
  43
  44         /* Compute the offset from the hyp VA and PA of a random symbol. */
  45         kern_va = (u64)kvm_ksym_ref(__hyp_text_start);
  46         hyp_va = __early_kern_hyp_va(kern_va);
  47         CHOOSE_NVHE_SYM(hyp_physvirt_offset) = (s64)__pa(kern_va) - (s64)hyp_va;
  48 }
  49
  50 /*
  51  * We want to generate a hyp VA with the following format (with V ==
  52  * vabits_actual):
  53  *
  54  *  63 ... V |     V-1    | V-2 .. tag_lsb | tag_lsb - 1 .. 0
  55  *  ---------------------------------------------------------
  56  * | 0000000 | hyp_va_msb |   random tag   |  kern linear VA |
  57  *           |--------- tag_val -----------|----- va_mask ---|
  58  *
  59  * which does not conflict with the idmap regions.
  60  */
  61 __init void kvm_compute_layout(void)
  62 {
  63         phys_addr_t idmap_addr = __pa_symbol(__hyp_idmap_text_start);
  64         u64 hyp_va_msb;
  65
  66         /* Where is my RAM region? */
  67         hyp_va_msb  = idmap_addr & BIT(vabits_actual - 1);
  68         hyp_va_msb ^= BIT(vabits_actual - 1);
  69
  70         tag_lsb = fls64((u64)phys_to_virt(memblock_start_of_DRAM()) ^
  71                         (u64)(high_memory - 1));
  72
  73         va_mask = GENMASK_ULL(tag_lsb - 1, 0);
  74         tag_val = hyp_va_msb;
  75
  76         if (IS_ENABLED(CONFIG_RANDOMIZE_BASE) && tag_lsb != (vabits_actual - 1)) {
  77                 /* We have some free bits to insert a random tag. */
  78                 tag_val |= get_random_long() & GENMASK_ULL(vabits_actual - 2, tag_lsb);
  79         }
  80         tag_val >>= tag_lsb;
  81
  82         init_hyp_physvirt_offset();
  83 }
  84
  85 static u32 compute_instruction(int n, u32 rd, u32 rn)
  86 {
  87         u32 insn = AARCH64_BREAK_FAULT;
  88
  89         switch (n) {
  90         case 0:
  91                 insn = aarch64_insn_gen_logical_immediate(AARCH64_INSN_LOGIC_AND,
  92                                                           AARCH64_INSN_VARIANT_64BIT,
  93                                                           rn, rd, va_mask);
  94                 break;
  95
  96         case 1:
  97                 /* ROR is a variant of EXTR with Rm = Rn */
  98                 insn = aarch64_insn_gen_extr(AARCH64_INSN_VARIANT_64BIT,
  99                                              rn, rn, rd,
 100                                              tag_lsb);
 101                 break;
 102
 103         case 2:
 104                 insn = aarch64_insn_gen_add_sub_imm(rd, rn,
 105                                                     tag_val & GENMASK(11, 0),
 106                                                     AARCH64_INSN_VARIANT_64BIT,
 107                                                     AARCH64_INSN_ADSB_ADD);
 108                 break;
 109
 110         case 3:
 111                 insn = aarch64_insn_gen_add_sub_imm(rd, rn,
 112                                                     tag_val & GENMASK(23, 12),
 113                                                     AARCH64_INSN_VARIANT_64BIT,
 114                                                     AARCH64_INSN_ADSB_ADD);
 115                 break;
 116
 117         case 4:
 118                 /* ROR is a variant of EXTR with Rm = Rn */
 119                 insn = aarch64_insn_gen_extr(AARCH64_INSN_VARIANT_64BIT,
 120                                              rn, rn, rd, 64 - tag_lsb);
 121                 break;
 122         }
 123
 124         return insn;
 125 }
 126
 127 void __init kvm_update_va_mask(struct alt_instr *alt,
 128                                __le32 *origptr, __le32 *updptr, int nr_inst)
 129 {
 130         int i;
 131
 132         BUG_ON(nr_inst != 5);
 133
 134         for (i = 0; i < nr_inst; i++) {
 135                 u32 rd, rn, insn, oinsn;
 136
 137                 /*
 138                  * VHE doesn't need any address translation, let's NOP
 139                  * everything.
 140                  *
 141                  * Alternatively, if the tag is zero (because the layout
 142                  * dictates it and we don't have any spare bits in the
 143                  * address), NOP everything after masking the kernel VA.
 144                  */
 145                 if (has_vhe() || (!tag_val && i > 0)) {
 146                         updptr[i] = cpu_to_le32(aarch64_insn_gen_nop());
 147                         continue;
 148                 }
 149
 150                 oinsn = le32_to_cpu(origptr[i]);
 151                 rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, oinsn);
 152                 rn = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RN, oinsn);
 153
 154                 insn = compute_instruction(i, rd, rn);
 155                 BUG_ON(insn == AARCH64_BREAK_FAULT);
 156
 157                 updptr[i] = cpu_to_le32(insn);
 158         }
 159 }
 160
 161 void kvm_patch_vector_branch(struct alt_instr *alt,
 162                              __le32 *origptr, __le32 *updptr, int nr_inst)
 163 {
 164         u64 addr;
 165         u32 insn;
 166
 167         BUG_ON(nr_inst != 4);
 168
 169         if (!cpus_have_const_cap(ARM64_SPECTRE_V3A) || WARN_ON_ONCE(has_vhe()))
 170                 return;
 171
 172         /*
 173          * Compute HYP VA by using the same computation as kern_hyp_va()
 174          */
 175         addr = __early_kern_hyp_va((u64)kvm_ksym_ref(__kvm_hyp_vector));
 176
 177         /* Use PC[10:7] to branch to the same vector in KVM */
 178         addr |= ((u64)origptr & GENMASK_ULL(10, 7));
 179
 180         /*
 181          * Branch over the preamble in order to avoid the initial store on
 182          * the stack (which we already perform in the hardening vectors).
 183          */
 184         addr += KVM_VECTOR_PREAMBLE;
 185
 186         /* movz x0, #(addr & 0xffff) */
 187         insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0,
 188                                          (u16)addr,
 189                                          0,
 190                                          AARCH64_INSN_VARIANT_64BIT,
 191                                          AARCH64_INSN_MOVEWIDE_ZERO);
 192         *updptr++ = cpu_to_le32(insn);
 193
 194         /* movk x0, #((addr >> 16) & 0xffff), lsl #16 */
 195         insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0,
 196                                          (u16)(addr >> 16),
 197                                          16,
 198                                          AARCH64_INSN_VARIANT_64BIT,
 199                                          AARCH64_INSN_MOVEWIDE_KEEP);
 200         *updptr++ = cpu_to_le32(insn);
 201
 202         /* movk x0, #((addr >> 32) & 0xffff), lsl #32 */
 203         insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0,
 204                                          (u16)(addr >> 32),
 205                                          32,
 206                                          AARCH64_INSN_VARIANT_64BIT,
 207                                          AARCH64_INSN_MOVEWIDE_KEEP);
 208         *updptr++ = cpu_to_le32(insn);
 209
 210         /* br x0 */
 211         insn = aarch64_insn_gen_branch_reg(AARCH64_INSN_REG_0,
 212                                            AARCH64_INSN_BRANCH_NOLINK);
 213         *updptr++ = cpu_to_le32(insn);
 214 }
 215
 216 static void generate_mov_q(u64 val, __le32 *origptr, __le32 *updptr, int nr_inst)
 217 {
 218         u32 insn, oinsn, rd;
 219
 220         BUG_ON(nr_inst != 4);
 221
 222         /* Compute target register */
 223         oinsn = le32_to_cpu(*origptr);
 224         rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, oinsn);
 225
 226         /* movz rd, #(val & 0xffff) */
 227         insn = aarch64_insn_gen_movewide(rd,
 228                                          (u16)val,
 229                                          0,
 230                                          AARCH64_INSN_VARIANT_64BIT,
 231                                          AARCH64_INSN_MOVEWIDE_ZERO);
 232         *updptr++ = cpu_to_le32(insn);
 233
 234         /* movk rd, #((val >> 16) & 0xffff), lsl #16 */
 235         insn = aarch64_insn_gen_movewide(rd,
 236                                          (u16)(val >> 16),
 237                                          16,
 238                                          AARCH64_INSN_VARIANT_64BIT,
 239                                          AARCH64_INSN_MOVEWIDE_KEEP);
 240         *updptr++ = cpu_to_le32(insn);
 241
 242         /* movk rd, #((val >> 32) & 0xffff), lsl #32 */
 243         insn = aarch64_insn_gen_movewide(rd,
 244                                          (u16)(val >> 32),
 245                                          32,
 246                                          AARCH64_INSN_VARIANT_64BIT,
 247                                          AARCH64_INSN_MOVEWIDE_KEEP);
 248         *updptr++ = cpu_to_le32(insn);
 249
 250         /* movk rd, #((val >> 48) & 0xffff), lsl #48 */
 251         insn = aarch64_insn_gen_movewide(rd,
 252                                          (u16)(val >> 48),
 253                                          48,
 254                                          AARCH64_INSN_VARIANT_64BIT,
 255                                          AARCH64_INSN_MOVEWIDE_KEEP);
 256         *updptr++ = cpu_to_le32(insn);
 257 }
 258
 259 void kvm_update_kimg_phys_offset(struct alt_instr *alt,
 260                                  __le32 *origptr, __le32 *updptr, int nr_inst)
 261 {
 262         generate_mov_q(kimage_voffset + PHYS_OFFSET, origptr, updptr, nr_inst);
 263 }
 264
 265 void kvm_get_kimage_voffset(struct alt_instr *alt,
 266                             __le32 *origptr, __le32 *updptr, int nr_inst)
 267 {
 268         generate_mov_q(kimage_voffset, origptr, updptr, nr_inst);
 269 }