arch/arm64/kernel/module-plts.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2014-2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
   4  */
   5
   6 #include <linux/elf.h>
   7 #include <linux/kernel.h>
   8 #include <linux/module.h>
   9 #include <linux/sort.h>
  10
  11 static struct plt_entry __get_adrp_add_pair(u64 dst, u64 pc,
  12                                             enum aarch64_insn_register reg)
  13 {
  14         u32 adrp, add;
  15
  16         adrp = aarch64_insn_gen_adr(pc, dst, reg, AARCH64_INSN_ADR_TYPE_ADRP);
  17         add = aarch64_insn_gen_add_sub_imm(reg, reg, dst % SZ_4K,
  18                                            AARCH64_INSN_VARIANT_64BIT,
  19                                            AARCH64_INSN_ADSB_ADD);
  20
  21         return (struct plt_entry){ cpu_to_le32(adrp), cpu_to_le32(add) };
  22 }
  23
  24 struct plt_entry get_plt_entry(u64 dst, void *pc)
  25 {
  26         struct plt_entry plt;
  27         static u32 br;
  28
  29         if (!br)
  30                 br = aarch64_insn_gen_branch_reg(AARCH64_INSN_REG_16,
  31                                                  AARCH64_INSN_BRANCH_NOLINK);
  32
  33         plt = __get_adrp_add_pair(dst, (u64)pc, AARCH64_INSN_REG_16);
  34         plt.br = cpu_to_le32(br);
  35
  36         return plt;
  37 }
  38
  39 bool plt_entries_equal(const struct plt_entry *a, const struct plt_entry *b)
  40 {
  41         u64 p, q;
  42
  43         /*
  44          * Check whether both entries refer to the same target:
  45          * do the cheapest checks first.
  46          * If the 'add' or 'br' opcodes are different, then the target
  47          * cannot be the same.
  48          */
  49         if (a->add != b->add || a->br != b->br)
  50                 return false;
  51
  52         p = ALIGN_DOWN((u64)a, SZ_4K);
  53         q = ALIGN_DOWN((u64)b, SZ_4K);
  54
  55         /*
  56          * If the 'adrp' opcodes are the same then we just need to check
  57          * that they refer to the same 4k region.
  58          */
  59         if (a->adrp == b->adrp && p == q)
  60                 return true;
  61
  62         return (p + aarch64_insn_adrp_get_offset(le32_to_cpu(a->adrp))) ==
  63                (q + aarch64_insn_adrp_get_offset(le32_to_cpu(b->adrp)));
  64 }
  65
  66 static bool in_init(const struct module *mod, void *loc)
  67 {
  68         return (u64)loc - (u64)mod->init_layout.base < mod->init_layout.size;
  69 }
  70
  71 u64 module_emit_plt_entry(struct module *mod, Elf64_Shdr *sechdrs,
  72                           void *loc, const Elf64_Rela *rela,
  73                           Elf64_Sym *sym)
  74 {
  75         struct mod_plt_sec *pltsec = !in_init(mod, loc) ? &mod->arch.core :
  76                                                           &mod->arch.init;
  77         struct plt_entry *plt = (struct plt_entry *)sechdrs[pltsec->plt_shndx].sh_addr;
  78         int i = pltsec->plt_num_entries;
  79         int j = i - 1;
  80         u64 val = sym->st_value + rela->r_addend;
  81
  82         if (is_forbidden_offset_for_adrp(&plt[i].adrp))
  83                 i++;
  84
  85         plt[i] = get_plt_entry(val, &plt[i]);
  86
  87         /*
  88          * Check if the entry we just created is a duplicate. Given that the
  89          * relocations are sorted, this will be the last entry we allocated.
  90          * (if one exists).
  91          */
  92         if (j >= 0 && plt_entries_equal(plt + i, plt + j))
  93                 return (u64)&plt[j];
  94
  95         pltsec->plt_num_entries += i - j;
  96         if (WARN_ON(pltsec->plt_num_entries > pltsec->plt_max_entries))
  97                 return 0;
  98
  99         return (u64)&plt[i];
 100 }
 101
 102 #ifdef CONFIG_ARM64_ERRATUM_843419
 103 u64 module_emit_veneer_for_adrp(struct module *mod, Elf64_Shdr *sechdrs,
 104                                 void *loc, u64 val)
 105 {
 106         struct mod_plt_sec *pltsec = !in_init(mod, loc) ? &mod->arch.core :
 107                                                           &mod->arch.init;
 108         struct plt_entry *plt = (struct plt_entry *)sechdrs[pltsec->plt_shndx].sh_addr;
 109         int i = pltsec->plt_num_entries++;
 110         u32 br;
 111         int rd;
 112
 113         if (WARN_ON(pltsec->plt_num_entries > pltsec->plt_max_entries))
 114                 return 0;
 115
 116         if (is_forbidden_offset_for_adrp(&plt[i].adrp))
 117                 i = pltsec->plt_num_entries++;
 118
 119         /* get the destination register of the ADRP instruction */
 120         rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD,
 121                                           le32_to_cpup((__le32 *)loc));
 122
 123         br = aarch64_insn_gen_branch_imm((u64)&plt[i].br, (u64)loc + 4,
 124                                          AARCH64_INSN_BRANCH_NOLINK);
 125
 126         plt[i] = __get_adrp_add_pair(val, (u64)&plt[i], rd);
 127         plt[i].br = cpu_to_le32(br);
 128
 129         return (u64)&plt[i];
 130 }
 131 #endif
 132
 133 #define cmp_3way(a,b)   ((a) < (b) ? -1 : (a) > (b))
 134
 135 static int cmp_rela(const void *a, const void *b)
 136 {
 137         const Elf64_Rela *x = a, *y = b;
 138         int i;
 139
 140         /* sort by type, symbol index and addend */
 141         i = cmp_3way(ELF64_R_TYPE(x->r_info), ELF64_R_TYPE(y->r_info));
 142         if (i == 0)
 143                 i = cmp_3way(ELF64_R_SYM(x->r_info), ELF64_R_SYM(y->r_info));
 144         if (i == 0)
 145                 i = cmp_3way(x->r_addend, y->r_addend);
 146         return i;
 147 }
 148
 149 static bool duplicate_rel(const Elf64_Rela *rela, int num)
 150 {
 151         /*
 152          * Entries are sorted by type, symbol index and addend. That means
 153          * that, if a duplicate entry exists, it must be in the preceding
 154          * slot.
 155          */
 156         return num > 0 && cmp_rela(rela + num, rela + num - 1) == 0;
 157 }
 158
 159 static unsigned int count_plts(Elf64_Sym *syms, Elf64_Rela *rela, int num,
 160                                Elf64_Word dstidx, Elf_Shdr *dstsec)
 161 {
 162         unsigned int ret = 0;
 163         Elf64_Sym *s;
 164         int i;
 165
 166         for (i = 0; i < num; i++) {
 167                 u64 min_align;
 168
 169                 switch (ELF64_R_TYPE(rela[i].r_info)) {
 170                 case R_AARCH64_JUMP26:
 171                 case R_AARCH64_CALL26:
 172                         if (!IS_ENABLED(CONFIG_RANDOMIZE_BASE))
 173                                 break;
 174
 175                         /*
 176                          * We only have to consider branch targets that resolve
 177                          * to symbols that are defined in a different section.
 178                          * This is not simply a heuristic, it is a fundamental
 179                          * limitation, since there is no guaranteed way to emit
 180                          * PLT entries sufficiently close to the branch if the
 181                          * section size exceeds the range of a branch
 182                          * instruction. So ignore relocations against defined
 183                          * symbols if they live in the same section as the
 184                          * relocation target.
 185                          */
 186                         s = syms + ELF64_R_SYM(rela[i].r_info);
 187                         if (s->st_shndx == dstidx)
 188                                 break;
 189
 190                         /*
 191                          * Jump relocations with non-zero addends against
 192                          * undefined symbols are supported by the ELF spec, but
 193                          * do not occur in practice (e.g., 'jump n bytes past
 194                          * the entry point of undefined function symbol f').
 195                          * So we need to support them, but there is no need to
 196                          * take them into consideration when trying to optimize
 197                          * this code. So let's only check for duplicates when
 198                          * the addend is zero: this allows us to record the PLT
 199                          * entry address in the symbol table itself, rather than
 200                          * having to search the list for duplicates each time we
 201                          * emit one.
 202                          */
 203                         if (rela[i].r_addend != 0 || !duplicate_rel(rela, i))
 204                                 ret++;
 205                         break;
 206                 case R_AARCH64_ADR_PREL_PG_HI21_NC:
 207                 case R_AARCH64_ADR_PREL_PG_HI21:
 208                         if (!IS_ENABLED(CONFIG_ARM64_ERRATUM_843419) ||
 209                             !cpus_have_const_cap(ARM64_WORKAROUND_843419))
 210                                 break;
 211
 212                         /*
 213                          * Determine the minimal safe alignment for this ADRP
 214                          * instruction: the section alignment at which it is
 215                          * guaranteed not to appear at a vulnerable offset.
 216                          *
 217                          * This comes down to finding the least significant zero
 218                          * bit in bits [11:3] of the section offset, and
 219                          * increasing the section's alignment so that the
 220                          * resulting address of this instruction is guaranteed
 221                          * to equal the offset in that particular bit (as well
 222                          * as all less signficant bits). This ensures that the
 223                          * address modulo 4 KB != 0xfff8 or 0xfffc (which would
 224                          * have all ones in bits [11:3])
 225                          */
 226                         min_align = 2ULL << ffz(rela[i].r_offset | 0x7);
 227
 228                         /*
 229                          * Allocate veneer space for each ADRP that may appear
 230                          * at a vulnerable offset nonetheless. At relocation
 231                          * time, some of these will remain unused since some
 232                          * ADRP instructions can be patched to ADR instructions
 233                          * instead.
 234                          */
 235                         if (min_align > SZ_4K)
 236                                 ret++;
 237                         else
 238                                 dstsec->sh_addralign = max(dstsec->sh_addralign,
 239                                                            min_align);
 240                         break;
 241                 }
 242         }
 243
 244         if (IS_ENABLED(CONFIG_ARM64_ERRATUM_843419) &&
 245             cpus_have_const_cap(ARM64_WORKAROUND_843419))
 246                 /*
 247                  * Add some slack so we can skip PLT slots that may trigger
 248                  * the erratum due to the placement of the ADRP instruction.
 249                  */
 250                 ret += DIV_ROUND_UP(ret, (SZ_4K / sizeof(struct plt_entry)));
 251
 252         return ret;
 253 }
 254
 255 int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs,
 256                               char *secstrings, struct module *mod)
 257 {
 258         unsigned long core_plts = 0;
 259         unsigned long init_plts = 0;
 260         Elf64_Sym *syms = NULL;
 261         Elf_Shdr *pltsec, *tramp = NULL;
 262         int i;
 263
 264         /*
 265          * Find the empty .plt section so we can expand it to store the PLT
 266          * entries. Record the symtab address as well.
 267          */
 268         for (i = 0; i < ehdr->e_shnum; i++) {
 269                 if (!strcmp(secstrings + sechdrs[i].sh_name, ".plt"))
 270                         mod->arch.core.plt_shndx = i;
 271                 else if (!strcmp(secstrings + sechdrs[i].sh_name, ".init.plt"))
 272                         mod->arch.init.plt_shndx = i;
 273                 else if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE) &&
 274                          !strcmp(secstrings + sechdrs[i].sh_name,
 275                                  ".text.ftrace_trampoline"))
 276                         tramp = sechdrs + i;
 277                 else if (sechdrs[i].sh_type == SHT_SYMTAB)
 278                         syms = (Elf64_Sym *)sechdrs[i].sh_addr;
 279         }
 280
 281         if (!mod->arch.core.plt_shndx || !mod->arch.init.plt_shndx) {
 282                 pr_err("%s: module PLT section(s) missing\n", mod->name);
 283                 return -ENOEXEC;
 284         }
 285         if (!syms) {
 286                 pr_err("%s: module symtab section missing\n", mod->name);
 287                 return -ENOEXEC;
 288         }
 289
 290         for (i = 0; i < ehdr->e_shnum; i++) {
 291                 Elf64_Rela *rels = (void *)ehdr + sechdrs[i].sh_offset;
 292                 int numrels = sechdrs[i].sh_size / sizeof(Elf64_Rela);
 293                 Elf64_Shdr *dstsec = sechdrs + sechdrs[i].sh_info;
 294
 295                 if (sechdrs[i].sh_type != SHT_RELA)
 296                         continue;
 297
 298                 /* ignore relocations that operate on non-exec sections */
 299                 if (!(dstsec->sh_flags & SHF_EXECINSTR))
 300                         continue;
 301
 302                 /* sort by type, symbol index and addend */
 303                 sort(rels, numrels, sizeof(Elf64_Rela), cmp_rela, NULL);
 304
 305                 if (strncmp(secstrings + dstsec->sh_name, ".init", 5) != 0)
 306                         core_plts += count_plts(syms, rels, numrels,
 307                                                 sechdrs[i].sh_info, dstsec);
 308                 else
 309                         init_plts += count_plts(syms, rels, numrels,
 310                                                 sechdrs[i].sh_info, dstsec);
 311         }
 312
 313         pltsec = sechdrs + mod->arch.core.plt_shndx;
 314         pltsec->sh_type = SHT_NOBITS;
 315         pltsec->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
 316         pltsec->sh_addralign = L1_CACHE_BYTES;
 317         pltsec->sh_size = (core_plts  + 1) * sizeof(struct plt_entry);
 318         mod->arch.core.plt_num_entries = 0;
 319         mod->arch.core.plt_max_entries = core_plts;
 320
 321         pltsec = sechdrs + mod->arch.init.plt_shndx;
 322         pltsec->sh_type = SHT_NOBITS;
 323         pltsec->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
 324         pltsec->sh_addralign = L1_CACHE_BYTES;
 325         pltsec->sh_size = (init_plts + 1) * sizeof(struct plt_entry);
 326         mod->arch.init.plt_num_entries = 0;
 327         mod->arch.init.plt_max_entries = init_plts;
 328
 329         if (tramp) {
 330                 tramp->sh_type = SHT_NOBITS;
 331                 tramp->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
 332                 tramp->sh_addralign = __alignof__(struct plt_entry);
 333                 tramp->sh_size = sizeof(struct plt_entry);
 334         }
 335
 336         return 0;
 337 }