arch/riscv/kernel/elf_kexec.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Load ELF vmlinux file for the kexec_file_load syscall.
   4  *
   5  * Copyright (C) 2021 Huawei Technologies Co, Ltd.
   6  *
   7  * Author: Liao Chang (liaochang1@huawei.com)
   8  *
   9  * Based on kexec-tools' kexec-elf-riscv.c, heavily modified
  10  * for kernel.
  11  */
  12
  13 #define pr_fmt(fmt)     "kexec_image: " fmt
  14
  15 #include <linux/elf.h>
  16 #include <linux/kexec.h>
  17 #include <linux/slab.h>
  18 #include <linux/of.h>
  19 #include <linux/libfdt.h>
  20 #include <linux/types.h>
  21 #include <linux/memblock.h>
  22 #include <linux/vmalloc.h>
  23 #include <asm/setup.h>
  24
  25 int arch_kimage_file_post_load_cleanup(struct kimage *image)
  26 {
  27         kvfree(image->arch.fdt);
  28         image->arch.fdt = NULL;
  29
  30         vfree(image->elf_headers);
  31         image->elf_headers = NULL;
  32         image->elf_headers_sz = 0;
  33
  34         return kexec_image_post_load_cleanup_default(image);
  35 }
  36
  37 static int riscv_kexec_elf_load(struct kimage *image, struct elfhdr *ehdr,
  38                                 struct kexec_elf_info *elf_info, unsigned long old_pbase,
  39                                 unsigned long new_pbase)
  40 {
  41         int i;
  42         int ret = 0;
  43         size_t size;
  44         struct kexec_buf kbuf;
  45         const struct elf_phdr *phdr;
  46
  47         kbuf.image = image;
  48
  49         for (i = 0; i < ehdr->e_phnum; i++) {
  50                 phdr = &elf_info->proghdrs[i];
  51                 if (phdr->p_type != PT_LOAD)
  52                         continue;
  53
  54                 size = phdr->p_filesz;
  55                 if (size > phdr->p_memsz)
  56                         size = phdr->p_memsz;
  57
  58                 kbuf.buffer = (void *) elf_info->buffer + phdr->p_offset;
  59                 kbuf.bufsz = size;
  60                 kbuf.buf_align = phdr->p_align;
  61                 kbuf.mem = phdr->p_paddr - old_pbase + new_pbase;
  62                 kbuf.memsz = phdr->p_memsz;
  63                 kbuf.top_down = false;
  64                 ret = kexec_add_buffer(&kbuf);
  65                 if (ret)
  66                         break;
  67         }
  68
  69         return ret;
  70 }
  71
  72 /*
  73  * Go through the available phsyical memory regions and find one that hold
  74  * an image of the specified size.
  75  */
  76 static int elf_find_pbase(struct kimage *image, unsigned long kernel_len,
  77                           struct elfhdr *ehdr, struct kexec_elf_info *elf_info,
  78                           unsigned long *old_pbase, unsigned long *new_pbase)
  79 {
  80         int i;
  81         int ret;
  82         struct kexec_buf kbuf;
  83         const struct elf_phdr *phdr;
  84         unsigned long lowest_paddr = ULONG_MAX;
  85         unsigned long lowest_vaddr = ULONG_MAX;
  86
  87         for (i = 0; i < ehdr->e_phnum; i++) {
  88                 phdr = &elf_info->proghdrs[i];
  89                 if (phdr->p_type != PT_LOAD)
  90                         continue;
  91
  92                 if (lowest_paddr > phdr->p_paddr)
  93                         lowest_paddr = phdr->p_paddr;
  94
  95                 if (lowest_vaddr > phdr->p_vaddr)
  96                         lowest_vaddr = phdr->p_vaddr;
  97         }
  98
  99         kbuf.image = image;
 100         kbuf.buf_min = lowest_paddr;
 101         kbuf.buf_max = ULONG_MAX;
 102
 103         /*
 104          * Current riscv boot protocol requires 2MB alignment for
 105          * RV64 and 4MB alignment for RV32
 106          *
 107          */
 108         kbuf.buf_align = PMD_SIZE;
 109         kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
 110         kbuf.memsz = ALIGN(kernel_len, PAGE_SIZE);
 111         kbuf.top_down = false;
 112         ret = arch_kexec_locate_mem_hole(&kbuf);
 113         if (!ret) {
 114                 *old_pbase = lowest_paddr;
 115                 *new_pbase = kbuf.mem;
 116                 image->start = ehdr->e_entry - lowest_vaddr + kbuf.mem;
 117         }
 118         return ret;
 119 }
 120
 121 #ifdef CONFIG_CRASH_DUMP
 122 static int get_nr_ram_ranges_callback(struct resource *res, void *arg)
 123 {
 124         unsigned int *nr_ranges = arg;
 125
 126         (*nr_ranges)++;
 127         return 0;
 128 }
 129
 130 static int prepare_elf64_ram_headers_callback(struct resource *res, void *arg)
 131 {
 132         struct crash_mem *cmem = arg;
 133
 134         cmem->ranges[cmem->nr_ranges].start = res->start;
 135         cmem->ranges[cmem->nr_ranges].end = res->end;
 136         cmem->nr_ranges++;
 137
 138         return 0;
 139 }
 140
 141 static int prepare_elf_headers(void **addr, unsigned long *sz)
 142 {
 143         struct crash_mem *cmem;
 144         unsigned int nr_ranges;
 145         int ret;
 146
 147         nr_ranges = 1; /* For exclusion of crashkernel region */
 148         walk_system_ram_res(0, -1, &nr_ranges, get_nr_ram_ranges_callback);
 149
 150         cmem = kmalloc(struct_size(cmem, ranges, nr_ranges), GFP_KERNEL);
 151         if (!cmem)
 152                 return -ENOMEM;
 153
 154         cmem->max_nr_ranges = nr_ranges;
 155         cmem->nr_ranges = 0;
 156         ret = walk_system_ram_res(0, -1, cmem, prepare_elf64_ram_headers_callback);
 157         if (ret)
 158                 goto out;
 159
 160         /* Exclude crashkernel region */
 161         ret = crash_exclude_mem_range(cmem, crashk_res.start, crashk_res.end);
 162         if (!ret)
 163                 ret = crash_prepare_elf64_headers(cmem, true, addr, sz);
 164
 165 out:
 166         kfree(cmem);
 167         return ret;
 168 }
 169
 170 static char *setup_kdump_cmdline(struct kimage *image, char *cmdline,
 171                                  unsigned long cmdline_len)
 172 {
 173         int elfcorehdr_strlen;
 174         char *cmdline_ptr;
 175
 176         cmdline_ptr = kzalloc(COMMAND_LINE_SIZE, GFP_KERNEL);
 177         if (!cmdline_ptr)
 178                 return NULL;
 179
 180         elfcorehdr_strlen = sprintf(cmdline_ptr, "elfcorehdr=0x%lx ",
 181                 image->elf_load_addr);
 182
 183         if (elfcorehdr_strlen + cmdline_len > COMMAND_LINE_SIZE) {
 184                 pr_err("Appending elfcorehdr=<addr> exceeds cmdline size\n");
 185                 kfree(cmdline_ptr);
 186                 return NULL;
 187         }
 188
 189         memcpy(cmdline_ptr + elfcorehdr_strlen, cmdline, cmdline_len);
 190         /* Ensure it's nul terminated */
 191         cmdline_ptr[COMMAND_LINE_SIZE - 1] = '\0';
 192         return cmdline_ptr;
 193 }
 194 #endif
 195
 196 static void *elf_kexec_load(struct kimage *image, char *kernel_buf,
 197                             unsigned long kernel_len, char *initrd,
 198                             unsigned long initrd_len, char *cmdline,
 199                             unsigned long cmdline_len)
 200 {
 201         int ret;
 202         void *fdt;
 203         unsigned long old_kernel_pbase = ULONG_MAX;
 204         unsigned long new_kernel_pbase = 0UL;
 205         unsigned long initrd_pbase = 0UL;
 206         unsigned long kernel_start;
 207         struct elfhdr ehdr;
 208         struct kexec_buf kbuf;
 209         struct kexec_elf_info elf_info;
 210         char *modified_cmdline = NULL;
 211
 212         ret = kexec_build_elf_info(kernel_buf, kernel_len, &ehdr, &elf_info);
 213         if (ret)
 214                 return ERR_PTR(ret);
 215
 216         ret = elf_find_pbase(image, kernel_len, &ehdr, &elf_info,
 217                              &old_kernel_pbase, &new_kernel_pbase);
 218         if (ret)
 219                 goto out;
 220         kernel_start = image->start;
 221
 222         /* Add the kernel binary to the image */
 223         ret = riscv_kexec_elf_load(image, &ehdr, &elf_info,
 224                                    old_kernel_pbase, new_kernel_pbase);
 225         if (ret)
 226                 goto out;
 227
 228         kbuf.image = image;
 229         kbuf.buf_min = new_kernel_pbase + kernel_len;
 230         kbuf.buf_max = ULONG_MAX;
 231
 232 #ifdef CONFIG_CRASH_DUMP
 233         /* Add elfcorehdr */
 234         if (image->type == KEXEC_TYPE_CRASH) {
 235                 void *headers;
 236                 unsigned long headers_sz;
 237                 ret = prepare_elf_headers(&headers, &headers_sz);
 238                 if (ret) {
 239                         pr_err("Preparing elf core header failed\n");
 240                         goto out;
 241                 }
 242
 243                 kbuf.buffer = headers;
 244                 kbuf.bufsz = headers_sz;
 245                 kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
 246                 kbuf.memsz = headers_sz;
 247                 kbuf.buf_align = ELF_CORE_HEADER_ALIGN;
 248                 kbuf.top_down = true;
 249
 250                 ret = kexec_add_buffer(&kbuf);
 251                 if (ret) {
 252                         vfree(headers);
 253                         goto out;
 254                 }
 255                 image->elf_headers = headers;
 256                 image->elf_load_addr = kbuf.mem;
 257                 image->elf_headers_sz = headers_sz;
 258
 259                 kexec_dprintk("Loaded elf core header at 0x%lx bufsz=0x%lx memsz=0x%lx\n",
 260                               image->elf_load_addr, kbuf.bufsz, kbuf.memsz);
 261
 262                 /* Setup cmdline for kdump kernel case */
 263                 modified_cmdline = setup_kdump_cmdline(image, cmdline,
 264                                                        cmdline_len);
 265                 if (!modified_cmdline) {
 266                         pr_err("Setting up cmdline for kdump kernel failed\n");
 267                         ret = -EINVAL;
 268                         goto out;
 269                 }
 270                 cmdline = modified_cmdline;
 271         }
 272 #endif
 273
 274 #ifdef CONFIG_ARCH_SUPPORTS_KEXEC_PURGATORY
 275         /* Add purgatory to the image */
 276         kbuf.top_down = true;
 277         kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
 278         ret = kexec_load_purgatory(image, &kbuf);
 279         if (ret) {
 280                 pr_err("Error loading purgatory ret=%d\n", ret);
 281                 goto out;
 282         }
 283         kexec_dprintk("Loaded purgatory at 0x%lx\n", kbuf.mem);
 284
 285         ret = kexec_purgatory_get_set_symbol(image, "riscv_kernel_entry",
 286                                              &kernel_start,
 287                                              sizeof(kernel_start), 0);
 288         if (ret)
 289                 pr_err("Error update purgatory ret=%d\n", ret);
 290 #endif /* CONFIG_ARCH_SUPPORTS_KEXEC_PURGATORY */
 291
 292         /* Add the initrd to the image */
 293         if (initrd != NULL) {
 294                 kbuf.buffer = initrd;
 295                 kbuf.bufsz = kbuf.memsz = initrd_len;
 296                 kbuf.buf_align = PAGE_SIZE;
 297                 kbuf.top_down = true;
 298                 kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
 299                 ret = kexec_add_buffer(&kbuf);
 300                 if (ret)
 301                         goto out;
 302                 initrd_pbase = kbuf.mem;
 303                 kexec_dprintk("Loaded initrd at 0x%lx\n", initrd_pbase);
 304         }
 305
 306         /* Add the DTB to the image */
 307         fdt = of_kexec_alloc_and_setup_fdt(image, initrd_pbase,
 308                                            initrd_len, cmdline, 0);
 309         if (!fdt) {
 310                 pr_err("Error setting up the new device tree.\n");
 311                 ret = -EINVAL;
 312                 goto out;
 313         }
 314
 315         fdt_pack(fdt);
 316         kbuf.buffer = fdt;
 317         kbuf.bufsz = kbuf.memsz = fdt_totalsize(fdt);
 318         kbuf.buf_align = PAGE_SIZE;
 319         kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
 320         kbuf.top_down = true;
 321         ret = kexec_add_buffer(&kbuf);
 322         if (ret) {
 323                 pr_err("Error add DTB kbuf ret=%d\n", ret);
 324                 goto out_free_fdt;
 325         }
 326         /* Cache the fdt buffer address for memory cleanup */
 327         image->arch.fdt = fdt;
 328         kexec_dprintk("Loaded device tree at 0x%lx\n", kbuf.mem);
 329         goto out;
 330
 331 out_free_fdt:
 332         kvfree(fdt);
 333 out:
 334         kfree(modified_cmdline);
 335         kexec_free_elf_info(&elf_info);
 336         return ret ? ERR_PTR(ret) : NULL;
 337 }
 338
 339 #define RV_X(x, s, n)  (((x) >> (s)) & ((1 << (n)) - 1))
 340 #define RISCV_IMM_BITS 12
 341 #define RISCV_IMM_REACH (1LL << RISCV_IMM_BITS)
 342 #define RISCV_CONST_HIGH_PART(x) \
 343         (((x) + (RISCV_IMM_REACH >> 1)) & ~(RISCV_IMM_REACH - 1))
 344 #define RISCV_CONST_LOW_PART(x) ((x) - RISCV_CONST_HIGH_PART(x))
 345
 346 #define ENCODE_ITYPE_IMM(x) \
 347         (RV_X(x, 0, 12) << 20)
 348 #define ENCODE_BTYPE_IMM(x) \
 349         ((RV_X(x, 1, 4) << 8) | (RV_X(x, 5, 6) << 25) | \
 350         (RV_X(x, 11, 1) << 7) | (RV_X(x, 12, 1) << 31))
 351 #define ENCODE_UTYPE_IMM(x) \
 352         (RV_X(x, 12, 20) << 12)
 353 #define ENCODE_JTYPE_IMM(x) \
 354         ((RV_X(x, 1, 10) << 21) | (RV_X(x, 11, 1) << 20) | \
 355         (RV_X(x, 12, 8) << 12) | (RV_X(x, 20, 1) << 31))
 356 #define ENCODE_CBTYPE_IMM(x) \
 357         ((RV_X(x, 1, 2) << 3) | (RV_X(x, 3, 2) << 10) | (RV_X(x, 5, 1) << 2) | \
 358         (RV_X(x, 6, 2) << 5) | (RV_X(x, 8, 1) << 12))
 359 #define ENCODE_CJTYPE_IMM(x) \
 360         ((RV_X(x, 1, 3) << 3) | (RV_X(x, 4, 1) << 11) | (RV_X(x, 5, 1) << 2) | \
 361         (RV_X(x, 6, 1) << 7) | (RV_X(x, 7, 1) << 6) | (RV_X(x, 8, 2) << 9) | \
 362         (RV_X(x, 10, 1) << 8) | (RV_X(x, 11, 1) << 12))
 363 #define ENCODE_UJTYPE_IMM(x) \
 364         (ENCODE_UTYPE_IMM(RISCV_CONST_HIGH_PART(x)) | \
 365         (ENCODE_ITYPE_IMM(RISCV_CONST_LOW_PART(x)) << 32))
 366 #define ENCODE_UITYPE_IMM(x) \
 367         (ENCODE_UTYPE_IMM(x) | (ENCODE_ITYPE_IMM(x) << 32))
 368
 369 #define CLEAN_IMM(type, x) \
 370         ((~ENCODE_##type##_IMM((uint64_t)(-1))) & (x))
 371
 372 int arch_kexec_apply_relocations_add(struct purgatory_info *pi,
 373                                      Elf_Shdr *section,
 374                                      const Elf_Shdr *relsec,
 375                                      const Elf_Shdr *symtab)
 376 {
 377         const char *strtab, *name, *shstrtab;
 378         const Elf_Shdr *sechdrs;
 379         Elf64_Rela *relas;
 380         int i, r_type;
 381
 382         /* String & section header string table */
 383         sechdrs = (void *)pi->ehdr + pi->ehdr->e_shoff;
 384         strtab = (char *)pi->ehdr + sechdrs[symtab->sh_link].sh_offset;
 385         shstrtab = (char *)pi->ehdr + sechdrs[pi->ehdr->e_shstrndx].sh_offset;
 386
 387         relas = (void *)pi->ehdr + relsec->sh_offset;
 388
 389         for (i = 0; i < relsec->sh_size / sizeof(*relas); i++) {
 390                 const Elf_Sym *sym;     /* symbol to relocate */
 391                 unsigned long addr;     /* final location after relocation */
 392                 unsigned long val;      /* relocated symbol value */
 393                 unsigned long sec_base; /* relocated symbol value */
 394                 void *loc;              /* tmp location to modify */
 395
 396                 sym = (void *)pi->ehdr + symtab->sh_offset;
 397                 sym += ELF64_R_SYM(relas[i].r_info);
 398
 399                 if (sym->st_name)
 400                         name = strtab + sym->st_name;
 401                 else
 402                         name = shstrtab + sechdrs[sym->st_shndx].sh_name;
 403
 404                 loc = pi->purgatory_buf;
 405                 loc += section->sh_offset;
 406                 loc += relas[i].r_offset;
 407
 408                 if (sym->st_shndx == SHN_ABS)
 409                         sec_base = 0;
 410                 else if (sym->st_shndx >= pi->ehdr->e_shnum) {
 411                         pr_err("Invalid section %d for symbol %s\n",
 412                                sym->st_shndx, name);
 413                         return -ENOEXEC;
 414                 } else
 415                         sec_base = pi->sechdrs[sym->st_shndx].sh_addr;
 416
 417                 val = sym->st_value;
 418                 val += sec_base;
 419                 val += relas[i].r_addend;
 420
 421                 addr = section->sh_addr + relas[i].r_offset;
 422
 423                 r_type = ELF64_R_TYPE(relas[i].r_info);
 424
 425                 switch (r_type) {
 426                 case R_RISCV_BRANCH:
 427                         *(u32 *)loc = CLEAN_IMM(BTYPE, *(u32 *)loc) |
 428                                  ENCODE_BTYPE_IMM(val - addr);
 429                         break;
 430                 case R_RISCV_JAL:
 431                         *(u32 *)loc = CLEAN_IMM(JTYPE, *(u32 *)loc) |
 432                                  ENCODE_JTYPE_IMM(val - addr);
 433                         break;
 434                 /*
 435                  * With no R_RISCV_PCREL_LO12_S, R_RISCV_PCREL_LO12_I
 436                  * sym is expected to be next to R_RISCV_PCREL_HI20
 437                  * in purgatory relsec. Handle it like R_RISCV_CALL
 438                  * sym, instead of searching the whole relsec.
 439                  */
 440                 case R_RISCV_PCREL_HI20:
 441                 case R_RISCV_CALL_PLT:
 442                 case R_RISCV_CALL:
 443                         *(u64 *)loc = CLEAN_IMM(UITYPE, *(u64 *)loc) |
 444                                  ENCODE_UJTYPE_IMM(val - addr);
 445                         break;
 446                 case R_RISCV_RVC_BRANCH:
 447                         *(u32 *)loc = CLEAN_IMM(CBTYPE, *(u32 *)loc) |
 448                                  ENCODE_CBTYPE_IMM(val - addr);
 449                         break;
 450                 case R_RISCV_RVC_JUMP:
 451                         *(u32 *)loc = CLEAN_IMM(CJTYPE, *(u32 *)loc) |
 452                                  ENCODE_CJTYPE_IMM(val - addr);
 453                         break;
 454                 case R_RISCV_ADD16:
 455                         *(u16 *)loc += val;
 456                         break;
 457                 case R_RISCV_SUB16:
 458                         *(u16 *)loc -= val;
 459                         break;
 460                 case R_RISCV_ADD32:
 461                         *(u32 *)loc += val;
 462                         break;
 463                 case R_RISCV_SUB32:
 464                         *(u32 *)loc -= val;
 465                         break;
 466                 /* It has been applied by R_RISCV_PCREL_HI20 sym */
 467                 case R_RISCV_PCREL_LO12_I:
 468                 case R_RISCV_ALIGN:
 469                 case R_RISCV_RELAX:
 470                         break;
 471                 default:
 472                         pr_err("Unknown rela relocation: %d\n", r_type);
 473                         return -ENOEXEC;
 474                 }
 475         }
 476         return 0;
 477 }
 478
 479 const struct kexec_file_ops elf_kexec_ops = {
 480         .probe = kexec_elf_probe,
 481         .load  = elf_kexec_load,
 482 };