arch/arm/kernel/module.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *  linux/arch/arm/kernel/module.c
   4  *
   5  *  Copyright (C) 2002 Russell King.
   6  *  Modified for nommu by Hyok S. Choi
   7  *
   8  * Module allocation method suggested by Andi Kleen.
   9  */
  10 #include <linux/module.h>
  11 #include <linux/moduleloader.h>
  12 #include <linux/kernel.h>
  13 #include <linux/mm.h>
  14 #include <linux/elf.h>
  15 #include <linux/fs.h>
  16 #include <linux/string.h>
  17
  18 #include <asm/sections.h>
  19 #include <asm/smp_plat.h>
  20 #include <asm/unwind.h>
  21 #include <asm/opcodes.h>
  22
  23 bool module_init_section(const char *name)
  24 {
  25         return strstarts(name, ".init") ||
  26                 strstarts(name, ".ARM.extab.init") ||
  27                 strstarts(name, ".ARM.exidx.init");
  28 }
  29
  30 bool module_exit_section(const char *name)
  31 {
  32         return strstarts(name, ".exit") ||
  33                 strstarts(name, ".ARM.extab.exit") ||
  34                 strstarts(name, ".ARM.exidx.exit");
  35 }
  36
  37 #ifdef CONFIG_ARM_HAS_GROUP_RELOCS
  38 /*
  39  * This implements the partitioning algorithm for group relocations as
  40  * documented in the ARM AArch32 ELF psABI (IHI 0044).
  41  *
  42  * A single PC-relative symbol reference is divided in up to 3 add or subtract
  43  * operations, where the final one could be incorporated into a load/store
  44  * instruction with immediate offset. E.g.,
  45  *
  46  *   ADD        Rd, PC, #...            or      ADD     Rd, PC, #...
  47  *   ADD        Rd, Rd, #...                    ADD     Rd, Rd, #...
  48  *   LDR        Rd, [Rd, #...]                  ADD     Rd, Rd, #...
  49  *
  50  * The latter has a guaranteed range of only 16 MiB (3x8 == 24 bits), so it is
  51  * of limited use in the kernel. However, the ADD/ADD/LDR combo has a range of
  52  * -/+ 256 MiB, (2x8 + 12 == 28 bits), which means it has sufficient range for
  53  * any in-kernel symbol reference (unless module PLTs are being used).
  54  *
  55  * The main advantage of this approach over the typical pattern using a literal
  56  * load is that literal loads may miss in the D-cache, and generally lead to
  57  * lower cache efficiency for variables that are referenced often from many
  58  * different places in the code.
  59  */
  60 static u32 get_group_rem(u32 group, u32 *offset)
  61 {
  62         u32 val = *offset;
  63         u32 shift;
  64         do {
  65                 shift = val ? (31 - __fls(val)) & ~1 : 32;
  66                 *offset = val;
  67                 if (!val)
  68                         break;
  69                 val &= 0xffffff >> shift;
  70         } while (group--);
  71         return shift;
  72 }
  73 #endif
  74
  75 int
  76 apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex,
  77                unsigned int relindex, struct module *module)
  78 {
  79         Elf32_Shdr *symsec = sechdrs + symindex;
  80         Elf32_Shdr *relsec = sechdrs + relindex;
  81         Elf32_Shdr *dstsec = sechdrs + relsec->sh_info;
  82         Elf32_Rel *rel = (void *)relsec->sh_addr;
  83         unsigned int i;
  84
  85         for (i = 0; i < relsec->sh_size / sizeof(Elf32_Rel); i++, rel++) {
  86                 unsigned long loc;
  87                 Elf32_Sym *sym;
  88                 const char *symname;
  89 #ifdef CONFIG_ARM_HAS_GROUP_RELOCS
  90                 u32 shift, group = 1;
  91 #endif
  92                 s32 offset;
  93                 u32 tmp;
  94 #ifdef CONFIG_THUMB2_KERNEL
  95                 u32 upper, lower, sign, j1, j2;
  96 #endif
  97
  98                 offset = ELF32_R_SYM(rel->r_info);
  99                 if (offset < 0 || offset > (symsec->sh_size / sizeof(Elf32_Sym))) {
 100                         pr_err("%s: section %u reloc %u: bad relocation sym offset\n",
 101                                 module->name, relindex, i);
 102                         return -ENOEXEC;
 103                 }
 104
 105                 sym = ((Elf32_Sym *)symsec->sh_addr) + offset;
 106                 symname = strtab + sym->st_name;
 107
 108                 if (rel->r_offset < 0 || rel->r_offset > dstsec->sh_size - sizeof(u32)) {
 109                         pr_err("%s: section %u reloc %u sym '%s': out of bounds relocation, offset %d size %u\n",
 110                                module->name, relindex, i, symname,
 111                                rel->r_offset, dstsec->sh_size);
 112                         return -ENOEXEC;
 113                 }
 114
 115                 loc = dstsec->sh_addr + rel->r_offset;
 116
 117                 switch (ELF32_R_TYPE(rel->r_info)) {
 118                 case R_ARM_NONE:
 119                         /* ignore */
 120                         break;
 121
 122                 case R_ARM_ABS32:
 123                 case R_ARM_TARGET1:
 124                         *(u32 *)loc += sym->st_value;
 125                         break;
 126
 127                 case R_ARM_PC24:
 128                 case R_ARM_CALL:
 129                 case R_ARM_JUMP24:
 130                         if (sym->st_value & 3) {
 131                                 pr_err("%s: section %u reloc %u sym '%s': unsupported interworking call (ARM -> Thumb)\n",
 132                                        module->name, relindex, i, symname);
 133                                 return -ENOEXEC;
 134                         }
 135
 136                         offset = __mem_to_opcode_arm(*(u32 *)loc);
 137                         offset = (offset & 0x00ffffff) << 2;
 138                         offset = sign_extend32(offset, 25);
 139
 140                         offset += sym->st_value - loc;
 141
 142                         /*
 143                          * Route through a PLT entry if 'offset' exceeds the
 144                          * supported range. Note that 'offset + loc + 8'
 145                          * contains the absolute jump target, i.e.,
 146                          * @sym + addend, corrected for the +8 PC bias.
 147                          */
 148                         if (IS_ENABLED(CONFIG_ARM_MODULE_PLTS) &&
 149                             (offset <= (s32)0xfe000000 ||
 150                              offset >= (s32)0x02000000))
 151                                 offset = get_module_plt(module, loc,
 152                                                         offset + loc + 8)
 153                                          - loc - 8;
 154
 155                         if (offset <= (s32)0xfe000000 ||
 156                             offset >= (s32)0x02000000) {
 157                                 pr_err("%s: section %u reloc %u sym '%s': relocation %u out of range (%#lx -> %#x)\n",
 158                                        module->name, relindex, i, symname,
 159                                        ELF32_R_TYPE(rel->r_info), loc,
 160                                        sym->st_value);
 161                                 return -ENOEXEC;
 162                         }
 163
 164                         offset >>= 2;
 165                         offset &= 0x00ffffff;
 166
 167                         *(u32 *)loc &= __opcode_to_mem_arm(0xff000000);
 168                         *(u32 *)loc |= __opcode_to_mem_arm(offset);
 169                         break;
 170
 171                case R_ARM_V4BX:
 172                        /* Preserve Rm and the condition code. Alter
 173                         * other bits to re-code instruction as
 174                         * MOV PC,Rm.
 175                         */
 176                        *(u32 *)loc &= __opcode_to_mem_arm(0xf000000f);
 177                        *(u32 *)loc |= __opcode_to_mem_arm(0x01a0f000);
 178                        break;
 179
 180                 case R_ARM_PREL31:
 181                         offset = (*(s32 *)loc << 1) >> 1; /* sign extend */
 182                         offset += sym->st_value - loc;
 183                         if (offset >= 0x40000000 || offset < -0x40000000) {
 184                                 pr_err("%s: section %u reloc %u sym '%s': relocation %u out of range (%#lx -> %#x)\n",
 185                                        module->name, relindex, i, symname,
 186                                        ELF32_R_TYPE(rel->r_info), loc,
 187                                        sym->st_value);
 188                                 return -ENOEXEC;
 189                         }
 190                         *(u32 *)loc &= 0x80000000;
 191                         *(u32 *)loc |= offset & 0x7fffffff;
 192                         break;
 193
 194                 case R_ARM_REL32:
 195                         *(u32 *)loc += sym->st_value - loc;
 196                         break;
 197
 198                 case R_ARM_MOVW_ABS_NC:
 199                 case R_ARM_MOVT_ABS:
 200                 case R_ARM_MOVW_PREL_NC:
 201                 case R_ARM_MOVT_PREL:
 202                         offset = tmp = __mem_to_opcode_arm(*(u32 *)loc);
 203                         offset = ((offset & 0xf0000) >> 4) | (offset & 0xfff);
 204                         offset = sign_extend32(offset, 15);
 205
 206                         offset += sym->st_value;
 207                         if (ELF32_R_TYPE(rel->r_info) == R_ARM_MOVT_PREL ||
 208                             ELF32_R_TYPE(rel->r_info) == R_ARM_MOVW_PREL_NC)
 209                                 offset -= loc;
 210                         if (ELF32_R_TYPE(rel->r_info) == R_ARM_MOVT_ABS ||
 211                             ELF32_R_TYPE(rel->r_info) == R_ARM_MOVT_PREL)
 212                                 offset >>= 16;
 213
 214                         tmp &= 0xfff0f000;
 215                         tmp |= ((offset & 0xf000) << 4) |
 216                                 (offset & 0x0fff);
 217
 218                         *(u32 *)loc = __opcode_to_mem_arm(tmp);
 219                         break;
 220
 221 #ifdef CONFIG_ARM_HAS_GROUP_RELOCS
 222                 case R_ARM_ALU_PC_G0_NC:
 223                         group = 0;
 224                         fallthrough;
 225                 case R_ARM_ALU_PC_G1_NC:
 226                         tmp = __mem_to_opcode_arm(*(u32 *)loc);
 227                         offset = ror32(tmp & 0xff, (tmp & 0xf00) >> 7);
 228                         if (tmp & BIT(22))
 229                                 offset = -offset;
 230                         offset += sym->st_value - loc;
 231                         if (offset < 0) {
 232                                 offset = -offset;
 233                                 tmp = (tmp & ~BIT(23)) | BIT(22); // SUB opcode
 234                         } else {
 235                                 tmp = (tmp & ~BIT(22)) | BIT(23); // ADD opcode
 236                         }
 237
 238                         shift = get_group_rem(group, &offset);
 239                         if (shift < 24) {
 240                                 offset >>= 24 - shift;
 241                                 offset |= (shift + 8) << 7;
 242                         }
 243                         *(u32 *)loc = __opcode_to_mem_arm((tmp & ~0xfff) | offset);
 244                         break;
 245
 246                 case R_ARM_LDR_PC_G2:
 247                         tmp = __mem_to_opcode_arm(*(u32 *)loc);
 248                         offset = tmp & 0xfff;
 249                         if (~tmp & BIT(23))             // U bit cleared?
 250                                 offset = -offset;
 251                         offset += sym->st_value - loc;
 252                         if (offset < 0) {
 253                                 offset = -offset;
 254                                 tmp &= ~BIT(23);        // clear U bit
 255                         } else {
 256                                 tmp |= BIT(23);         // set U bit
 257                         }
 258                         get_group_rem(2, &offset);
 259
 260                         if (offset > 0xfff) {
 261                                 pr_err("%s: section %u reloc %u sym '%s': relocation %u out of range (%#lx -> %#x)\n",
 262                                        module->name, relindex, i, symname,
 263                                        ELF32_R_TYPE(rel->r_info), loc,
 264                                        sym->st_value);
 265                                 return -ENOEXEC;
 266                         }
 267                         *(u32 *)loc = __opcode_to_mem_arm((tmp & ~0xfff) | offset);
 268                         break;
 269 #endif
 270 #ifdef CONFIG_THUMB2_KERNEL
 271                 case R_ARM_THM_CALL:
 272                 case R_ARM_THM_JUMP24:
 273                         /*
 274                          * For function symbols, only Thumb addresses are
 275                          * allowed (no interworking).
 276                          *
 277                          * For non-function symbols, the destination
 278                          * has no specific ARM/Thumb disposition, so
 279                          * the branch is resolved under the assumption
 280                          * that interworking is not required.
 281                          */
 282                         if (ELF32_ST_TYPE(sym->st_info) == STT_FUNC &&
 283                             !(sym->st_value & 1)) {
 284                                 pr_err("%s: section %u reloc %u sym '%s': unsupported interworking call (Thumb -> ARM)\n",
 285                                        module->name, relindex, i, symname);
 286                                 return -ENOEXEC;
 287                         }
 288
 289                         upper = __mem_to_opcode_thumb16(*(u16 *)loc);
 290                         lower = __mem_to_opcode_thumb16(*(u16 *)(loc + 2));
 291
 292                         /*
 293                          * 25 bit signed address range (Thumb-2 BL and B.W
 294                          * instructions):
 295                          *   S:I1:I2:imm10:imm11:0
 296                          * where:
 297                          *   S     = upper[10]   = offset[24]
 298                          *   I1    = ~(J1 ^ S)   = offset[23]
 299                          *   I2    = ~(J2 ^ S)   = offset[22]
 300                          *   imm10 = upper[9:0]  = offset[21:12]
 301                          *   imm11 = lower[10:0] = offset[11:1]
 302                          *   J1    = lower[13]
 303                          *   J2    = lower[11]
 304                          */
 305                         sign = (upper >> 10) & 1;
 306                         j1 = (lower >> 13) & 1;
 307                         j2 = (lower >> 11) & 1;
 308                         offset = (sign << 24) | ((~(j1 ^ sign) & 1) << 23) |
 309                                 ((~(j2 ^ sign) & 1) << 22) |
 310                                 ((upper & 0x03ff) << 12) |
 311                                 ((lower & 0x07ff) << 1);
 312                         offset = sign_extend32(offset, 24);
 313                         offset += sym->st_value - loc;
 314
 315                         /*
 316                          * Route through a PLT entry if 'offset' exceeds the
 317                          * supported range.
 318                          */
 319                         if (IS_ENABLED(CONFIG_ARM_MODULE_PLTS) &&
 320                             (offset <= (s32)0xff000000 ||
 321                              offset >= (s32)0x01000000))
 322                                 offset = get_module_plt(module, loc,
 323                                                         offset + loc + 4)
 324                                          - loc - 4;
 325
 326                         if (offset <= (s32)0xff000000 ||
 327                             offset >= (s32)0x01000000) {
 328                                 pr_err("%s: section %u reloc %u sym '%s': relocation %u out of range (%#lx -> %#x)\n",
 329                                        module->name, relindex, i, symname,
 330                                        ELF32_R_TYPE(rel->r_info), loc,
 331                                        sym->st_value);
 332                                 return -ENOEXEC;
 333                         }
 334
 335                         sign = (offset >> 24) & 1;
 336                         j1 = sign ^ (~(offset >> 23) & 1);
 337                         j2 = sign ^ (~(offset >> 22) & 1);
 338                         upper = (u16)((upper & 0xf800) | (sign << 10) |
 339                                             ((offset >> 12) & 0x03ff));
 340                         lower = (u16)((lower & 0xd000) |
 341                                       (j1 << 13) | (j2 << 11) |
 342                                       ((offset >> 1) & 0x07ff));
 343
 344                         *(u16 *)loc = __opcode_to_mem_thumb16(upper);
 345                         *(u16 *)(loc + 2) = __opcode_to_mem_thumb16(lower);
 346                         break;
 347
 348                 case R_ARM_THM_MOVW_ABS_NC:
 349                 case R_ARM_THM_MOVT_ABS:
 350                 case R_ARM_THM_MOVW_PREL_NC:
 351                 case R_ARM_THM_MOVT_PREL:
 352                         upper = __mem_to_opcode_thumb16(*(u16 *)loc);
 353                         lower = __mem_to_opcode_thumb16(*(u16 *)(loc + 2));
 354
 355                         /*
 356                          * MOVT/MOVW instructions encoding in Thumb-2:
 357                          *
 358                          * i    = upper[10]
 359                          * imm4 = upper[3:0]
 360                          * imm3 = lower[14:12]
 361                          * imm8 = lower[7:0]
 362                          *
 363                          * imm16 = imm4:i:imm3:imm8
 364                          */
 365                         offset = ((upper & 0x000f) << 12) |
 366                                 ((upper & 0x0400) << 1) |
 367                                 ((lower & 0x7000) >> 4) | (lower & 0x00ff);
 368                         offset = sign_extend32(offset, 15);
 369                         offset += sym->st_value;
 370
 371                         if (ELF32_R_TYPE(rel->r_info) == R_ARM_THM_MOVT_PREL ||
 372                             ELF32_R_TYPE(rel->r_info) == R_ARM_THM_MOVW_PREL_NC)
 373                                 offset -= loc;
 374                         if (ELF32_R_TYPE(rel->r_info) == R_ARM_THM_MOVT_ABS ||
 375                             ELF32_R_TYPE(rel->r_info) == R_ARM_THM_MOVT_PREL)
 376                                 offset >>= 16;
 377
 378                         upper = (u16)((upper & 0xfbf0) |
 379                                       ((offset & 0xf000) >> 12) |
 380                                       ((offset & 0x0800) >> 1));
 381                         lower = (u16)((lower & 0x8f00) |
 382                                       ((offset & 0x0700) << 4) |
 383                                       (offset & 0x00ff));
 384                         *(u16 *)loc = __opcode_to_mem_thumb16(upper);
 385                         *(u16 *)(loc + 2) = __opcode_to_mem_thumb16(lower);
 386                         break;
 387 #endif
 388
 389                 default:
 390                         pr_err("%s: unknown relocation: %u\n",
 391                                module->name, ELF32_R_TYPE(rel->r_info));
 392                         return -ENOEXEC;
 393                 }
 394         }
 395         return 0;
 396 }
 397
 398 static const Elf_Shdr *find_mod_section(const Elf32_Ehdr *hdr,
 399         const Elf_Shdr *sechdrs, const char *name)
 400 {
 401         const Elf_Shdr *s, *se;
 402         const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
 403
 404         for (s = sechdrs, se = sechdrs + hdr->e_shnum; s < se; s++)
 405                 if (strcmp(name, secstrs + s->sh_name) == 0)
 406                         return s;
 407
 408         return NULL;
 409 }
 410
 411 extern void fixup_pv_table(const void *, unsigned long);
 412 extern void fixup_smp(const void *, unsigned long);
 413
 414 int module_finalize(const Elf32_Ehdr *hdr, const Elf_Shdr *sechdrs,
 415                     struct module *mod)
 416 {
 417         const Elf_Shdr *s = NULL;
 418 #ifdef CONFIG_ARM_UNWIND
 419         const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
 420         const Elf_Shdr *sechdrs_end = sechdrs + hdr->e_shnum;
 421         struct list_head *unwind_list = &mod->arch.unwind_list;
 422
 423         INIT_LIST_HEAD(unwind_list);
 424         mod->arch.init_table = NULL;
 425
 426         for (s = sechdrs; s < sechdrs_end; s++) {
 427                 const char *secname = secstrs + s->sh_name;
 428                 const char *txtname;
 429                 const Elf_Shdr *txt_sec;
 430
 431                 if (!(s->sh_flags & SHF_ALLOC) ||
 432                     s->sh_type != ELF_SECTION_UNWIND)
 433                         continue;
 434
 435                 if (!strcmp(".ARM.exidx", secname))
 436                         txtname = ".text";
 437                 else
 438                         txtname = secname + strlen(".ARM.exidx");
 439                 txt_sec = find_mod_section(hdr, sechdrs, txtname);
 440
 441                 if (txt_sec) {
 442                         struct unwind_table *table =
 443                                 unwind_table_add(s->sh_addr,
 444                                                 s->sh_size,
 445                                                 txt_sec->sh_addr,
 446                                                 txt_sec->sh_size);
 447
 448                         list_add(&table->mod_list, unwind_list);
 449
 450                         /* save init table for module_arch_freeing_init */
 451                         if (strcmp(".ARM.exidx.init.text", secname) == 0)
 452                                 mod->arch.init_table = table;
 453                 }
 454         }
 455 #endif
 456 #ifdef CONFIG_ARM_PATCH_PHYS_VIRT
 457         s = find_mod_section(hdr, sechdrs, ".pv_table");
 458         if (s)
 459                 fixup_pv_table((void *)s->sh_addr, s->sh_size);
 460 #endif
 461         s = find_mod_section(hdr, sechdrs, ".alt.smp.init");
 462         if (s && !is_smp())
 463 #ifdef CONFIG_SMP_ON_UP
 464                 fixup_smp((void *)s->sh_addr, s->sh_size);
 465 #else
 466                 return -EINVAL;
 467 #endif
 468         return 0;
 469 }
 470
 471 void
 472 module_arch_cleanup(struct module *mod)
 473 {
 474 #ifdef CONFIG_ARM_UNWIND
 475         struct unwind_table *tmp;
 476         struct unwind_table *n;
 477
 478         list_for_each_entry_safe(tmp, n,
 479                         &mod->arch.unwind_list, mod_list) {
 480                 list_del(&tmp->mod_list);
 481                 unwind_table_del(tmp);
 482         }
 483         mod->arch.init_table = NULL;
 484 #endif
 485 }
 486
 487 void __weak module_arch_freeing_init(struct module *mod)
 488 {
 489 #ifdef CONFIG_ARM_UNWIND
 490         struct unwind_table *init = mod->arch.init_table;
 491
 492         if (init) {
 493                 mod->arch.init_table = NULL;
 494                 list_del(&init->mod_list);
 495                 unwind_table_del(init);
 496         }
 497 #endif
 498 }