tools/objtool/check.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com>
   4  */
   5
   6 #include <string.h>
   7 #include <stdlib.h>
   8 #include <inttypes.h>
   9 #include <sys/mman.h>
  10
  11 #include <objtool/builtin.h>
  12 #include <objtool/cfi.h>
  13 #include <objtool/arch.h>
  14 #include <objtool/check.h>
  15 #include <objtool/special.h>
  16 #include <objtool/warn.h>
  17 #include <objtool/endianness.h>
  18
  19 #include <linux/objtool_types.h>
  20 #include <linux/hashtable.h>
  21 #include <linux/kernel.h>
  22 #include <linux/static_call_types.h>
  23 #include <linux/string.h>
  24
  25 struct alternative {
  26         struct alternative *next;
  27         struct instruction *insn;
  28         bool skip_orig;
  29 };
  30
  31 static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache;
  32
  33 static struct cfi_init_state initial_func_cfi;
  34 static struct cfi_state init_cfi;
  35 static struct cfi_state func_cfi;
  36 static struct cfi_state force_undefined_cfi;
  37
  38 struct instruction *find_insn(struct objtool_file *file,
  39                               struct section *sec, unsigned long offset)
  40 {
  41         struct instruction *insn;
  42
  43         hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) {
  44                 if (insn->sec == sec && insn->offset == offset)
  45                         return insn;
  46         }
  47
  48         return NULL;
  49 }
  50
  51 struct instruction *next_insn_same_sec(struct objtool_file *file,
  52                                        struct instruction *insn)
  53 {
  54         if (insn->idx == INSN_CHUNK_MAX)
  55                 return find_insn(file, insn->sec, insn->offset + insn->len);
  56
  57         insn++;
  58         if (!insn->len)
  59                 return NULL;
  60
  61         return insn;
  62 }
  63
  64 static struct instruction *next_insn_same_func(struct objtool_file *file,
  65                                                struct instruction *insn)
  66 {
  67         struct instruction *next = next_insn_same_sec(file, insn);
  68         struct symbol *func = insn_func(insn);
  69
  70         if (!func)
  71                 return NULL;
  72
  73         if (next && insn_func(next) == func)
  74                 return next;
  75
  76         /* Check if we're already in the subfunction: */
  77         if (func == func->cfunc)
  78                 return NULL;
  79
  80         /* Move to the subfunction: */
  81         return find_insn(file, func->cfunc->sec, func->cfunc->offset);
  82 }
  83
  84 static struct instruction *prev_insn_same_sec(struct objtool_file *file,
  85                                               struct instruction *insn)
  86 {
  87         if (insn->idx == 0) {
  88                 if (insn->prev_len)
  89                         return find_insn(file, insn->sec, insn->offset - insn->prev_len);
  90                 return NULL;
  91         }
  92
  93         return insn - 1;
  94 }
  95
  96 static struct instruction *prev_insn_same_sym(struct objtool_file *file,
  97                                               struct instruction *insn)
  98 {
  99         struct instruction *prev = prev_insn_same_sec(file, insn);
 100
 101         if (prev && insn_func(prev) == insn_func(insn))
 102                 return prev;
 103
 104         return NULL;
 105 }
 106
 107 #define for_each_insn(file, insn)                                       \
 108         for (struct section *__sec, *__fake = (struct section *)1;      \
 109              __fake; __fake = NULL)                                     \
 110                 for_each_sec(file, __sec)                               \
 111                         sec_for_each_insn(file, __sec, insn)
 112
 113 #define func_for_each_insn(file, func, insn)                            \
 114         for (insn = find_insn(file, func->sec, func->offset);           \
 115              insn;                                                      \
 116              insn = next_insn_same_func(file, insn))
 117
 118 #define sym_for_each_insn(file, sym, insn)                              \
 119         for (insn = find_insn(file, sym->sec, sym->offset);             \
 120              insn && insn->offset < sym->offset + sym->len;             \
 121              insn = next_insn_same_sec(file, insn))
 122
 123 #define sym_for_each_insn_continue_reverse(file, sym, insn)             \
 124         for (insn = prev_insn_same_sec(file, insn);                     \
 125              insn && insn->offset >= sym->offset;                       \
 126              insn = prev_insn_same_sec(file, insn))
 127
 128 #define sec_for_each_insn_from(file, insn)                              \
 129         for (; insn; insn = next_insn_same_sec(file, insn))
 130
 131 #define sec_for_each_insn_continue(file, insn)                          \
 132         for (insn = next_insn_same_sec(file, insn); insn;               \
 133              insn = next_insn_same_sec(file, insn))
 134
 135 static inline struct symbol *insn_call_dest(struct instruction *insn)
 136 {
 137         if (insn->type == INSN_JUMP_DYNAMIC ||
 138             insn->type == INSN_CALL_DYNAMIC)
 139                 return NULL;
 140
 141         return insn->_call_dest;
 142 }
 143
 144 static inline struct reloc *insn_jump_table(struct instruction *insn)
 145 {
 146         if (insn->type == INSN_JUMP_DYNAMIC ||
 147             insn->type == INSN_CALL_DYNAMIC)
 148                 return insn->_jump_table;
 149
 150         return NULL;
 151 }
 152
 153 static bool is_jump_table_jump(struct instruction *insn)
 154 {
 155         struct alt_group *alt_group = insn->alt_group;
 156
 157         if (insn_jump_table(insn))
 158                 return true;
 159
 160         /* Retpoline alternative for a jump table? */
 161         return alt_group && alt_group->orig_group &&
 162                insn_jump_table(alt_group->orig_group->first_insn);
 163 }
 164
 165 static bool is_sibling_call(struct instruction *insn)
 166 {
 167         /*
 168          * Assume only STT_FUNC calls have jump-tables.
 169          */
 170         if (insn_func(insn)) {
 171                 /* An indirect jump is either a sibling call or a jump to a table. */
 172                 if (insn->type == INSN_JUMP_DYNAMIC)
 173                         return !is_jump_table_jump(insn);
 174         }
 175
 176         /* add_jump_destinations() sets insn_call_dest(insn) for sibling calls. */
 177         return (is_static_jump(insn) && insn_call_dest(insn));
 178 }
 179
 180 /*
 181  * Checks if a string ends with another.
 182  */
 183 static bool str_ends_with(const char *s, const char *sub)
 184 {
 185         const int slen = strlen(s);
 186         const int sublen = strlen(sub);
 187
 188         if (sublen > slen)
 189                 return 0;
 190
 191         return !memcmp(s + slen - sublen, sub, sublen);
 192 }
 193
 194 /*
 195  * Checks if a function is a Rust "noreturn" one.
 196  */
 197 static bool is_rust_noreturn(const struct symbol *func)
 198 {
 199         /*
 200          * If it does not start with "_R", then it is not a Rust symbol.
 201          */
 202         if (strncmp(func->name, "_R", 2))
 203                 return false;
 204
 205         /*
 206          * These are just heuristics -- we do not control the precise symbol
 207          * name, due to the crate disambiguators (which depend on the compiler)
 208          * as well as changes to the source code itself between versions (since
 209          * these come from the Rust standard library).
 210          */
 211         return str_ends_with(func->name, "_4core5sliceSp15copy_from_slice17len_mismatch_fail")          ||
 212                str_ends_with(func->name, "_4core6option13unwrap_failed")                                ||
 213                str_ends_with(func->name, "_4core6result13unwrap_failed")                                ||
 214                str_ends_with(func->name, "_4core9panicking5panic")                                      ||
 215                str_ends_with(func->name, "_4core9panicking9panic_fmt")                                  ||
 216                str_ends_with(func->name, "_4core9panicking14panic_explicit")                            ||
 217                str_ends_with(func->name, "_4core9panicking14panic_nounwind")                            ||
 218                str_ends_with(func->name, "_4core9panicking18panic_bounds_check")                        ||
 219                str_ends_with(func->name, "_4core9panicking19assert_failed_inner")                       ||
 220                str_ends_with(func->name, "_4core9panicking36panic_misaligned_pointer_dereference")      ||
 221                strstr(func->name, "_4core9panicking11panic_const24panic_const_")                        ||
 222                (strstr(func->name, "_4core5slice5index24slice_") &&
 223                 str_ends_with(func->name, "_fail"));
 224 }
 225
 226 /*
 227  * This checks to see if the given function is a "noreturn" function.
 228  *
 229  * For global functions which are outside the scope of this object file, we
 230  * have to keep a manual list of them.
 231  *
 232  * For local functions, we have to detect them manually by simply looking for
 233  * the lack of a return instruction.
 234  */
 235 static bool __dead_end_function(struct objtool_file *file, struct symbol *func,
 236                                 int recursion)
 237 {
 238         int i;
 239         struct instruction *insn;
 240         bool empty = true;
 241
 242 #define NORETURN(func) __stringify(func),
 243         static const char * const global_noreturns[] = {
 244 #include "noreturns.h"
 245         };
 246 #undef NORETURN
 247
 248         if (!func)
 249                 return false;
 250
 251         if (func->bind == STB_GLOBAL || func->bind == STB_WEAK) {
 252                 if (is_rust_noreturn(func))
 253                         return true;
 254
 255                 for (i = 0; i < ARRAY_SIZE(global_noreturns); i++)
 256                         if (!strcmp(func->name, global_noreturns[i]))
 257                                 return true;
 258         }
 259
 260         if (func->bind == STB_WEAK)
 261                 return false;
 262
 263         if (!func->len)
 264                 return false;
 265
 266         insn = find_insn(file, func->sec, func->offset);
 267         if (!insn || !insn_func(insn))
 268                 return false;
 269
 270         func_for_each_insn(file, func, insn) {
 271                 empty = false;
 272
 273                 if (insn->type == INSN_RETURN)
 274                         return false;
 275         }
 276
 277         if (empty)
 278                 return false;
 279
 280         /*
 281          * A function can have a sibling call instead of a return.  In that
 282          * case, the function's dead-end status depends on whether the target
 283          * of the sibling call returns.
 284          */
 285         func_for_each_insn(file, func, insn) {
 286                 if (is_sibling_call(insn)) {
 287                         struct instruction *dest = insn->jump_dest;
 288
 289                         if (!dest)
 290                                 /* sibling call to another file */
 291                                 return false;
 292
 293                         /* local sibling call */
 294                         if (recursion == 5) {
 295                                 /*
 296                                  * Infinite recursion: two functions have
 297                                  * sibling calls to each other.  This is a very
 298                                  * rare case.  It means they aren't dead ends.
 299                                  */
 300                                 return false;
 301                         }
 302
 303                         return __dead_end_function(file, insn_func(dest), recursion+1);
 304                 }
 305         }
 306
 307         return true;
 308 }
 309
 310 static bool dead_end_function(struct objtool_file *file, struct symbol *func)
 311 {
 312         return __dead_end_function(file, func, 0);
 313 }
 314
 315 static void init_cfi_state(struct cfi_state *cfi)
 316 {
 317         int i;
 318
 319         for (i = 0; i < CFI_NUM_REGS; i++) {
 320                 cfi->regs[i].base = CFI_UNDEFINED;
 321                 cfi->vals[i].base = CFI_UNDEFINED;
 322         }
 323         cfi->cfa.base = CFI_UNDEFINED;
 324         cfi->drap_reg = CFI_UNDEFINED;
 325         cfi->drap_offset = -1;
 326 }
 327
 328 static void init_insn_state(struct objtool_file *file, struct insn_state *state,
 329                             struct section *sec)
 330 {
 331         memset(state, 0, sizeof(*state));
 332         init_cfi_state(&state->cfi);
 333
 334         /*
 335          * We need the full vmlinux for noinstr validation, otherwise we can
 336          * not correctly determine insn_call_dest(insn)->sec (external symbols
 337          * do not have a section).
 338          */
 339         if (opts.link && opts.noinstr && sec)
 340                 state->noinstr = sec->noinstr;
 341 }
 342
 343 static struct cfi_state *cfi_alloc(void)
 344 {
 345         struct cfi_state *cfi = calloc(1, sizeof(struct cfi_state));
 346         if (!cfi) {
 347                 WARN("calloc failed");
 348                 exit(1);
 349         }
 350         nr_cfi++;
 351         return cfi;
 352 }
 353
 354 static int cfi_bits;
 355 static struct hlist_head *cfi_hash;
 356
 357 static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2)
 358 {
 359         return memcmp((void *)cfi1 + sizeof(cfi1->hash),
 360                       (void *)cfi2 + sizeof(cfi2->hash),
 361                       sizeof(struct cfi_state) - sizeof(struct hlist_node));
 362 }
 363
 364 static inline u32 cfi_key(struct cfi_state *cfi)
 365 {
 366         return jhash((void *)cfi + sizeof(cfi->hash),
 367                      sizeof(*cfi) - sizeof(cfi->hash), 0);
 368 }
 369
 370 static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi)
 371 {
 372         struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
 373         struct cfi_state *obj;
 374
 375         hlist_for_each_entry(obj, head, hash) {
 376                 if (!cficmp(cfi, obj)) {
 377                         nr_cfi_cache++;
 378                         return obj;
 379                 }
 380         }
 381
 382         obj = cfi_alloc();
 383         *obj = *cfi;
 384         hlist_add_head(&obj->hash, head);
 385
 386         return obj;
 387 }
 388
 389 static void cfi_hash_add(struct cfi_state *cfi)
 390 {
 391         struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
 392
 393         hlist_add_head(&cfi->hash, head);
 394 }
 395
 396 static void *cfi_hash_alloc(unsigned long size)
 397 {
 398         cfi_bits = max(10, ilog2(size));
 399         cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits,
 400                         PROT_READ|PROT_WRITE,
 401                         MAP_PRIVATE|MAP_ANON, -1, 0);
 402         if (cfi_hash == (void *)-1L) {
 403                 WARN("mmap fail cfi_hash");
 404                 cfi_hash = NULL;
 405         }  else if (opts.stats) {
 406                 printf("cfi_bits: %d\n", cfi_bits);
 407         }
 408
 409         return cfi_hash;
 410 }
 411
 412 static unsigned long nr_insns;
 413 static unsigned long nr_insns_visited;
 414
 415 /*
 416  * Call the arch-specific instruction decoder for all the instructions and add
 417  * them to the global instruction list.
 418  */
 419 static int decode_instructions(struct objtool_file *file)
 420 {
 421         struct section *sec;
 422         struct symbol *func;
 423         unsigned long offset;
 424         struct instruction *insn;
 425         int ret;
 426
 427         for_each_sec(file, sec) {
 428                 struct instruction *insns = NULL;
 429                 u8 prev_len = 0;
 430                 u8 idx = 0;
 431
 432                 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
 433                         continue;
 434
 435                 if (strcmp(sec->name, ".altinstr_replacement") &&
 436                     strcmp(sec->name, ".altinstr_aux") &&
 437                     strncmp(sec->name, ".discard.", 9))
 438                         sec->text = true;
 439
 440                 if (!strcmp(sec->name, ".noinstr.text") ||
 441                     !strcmp(sec->name, ".entry.text") ||
 442                     !strcmp(sec->name, ".cpuidle.text") ||
 443                     !strncmp(sec->name, ".text..__x86.", 13))
 444                         sec->noinstr = true;
 445
 446                 /*
 447                  * .init.text code is ran before userspace and thus doesn't
 448                  * strictly need retpolines, except for modules which are
 449                  * loaded late, they very much do need retpoline in their
 450                  * .init.text
 451                  */
 452                 if (!strcmp(sec->name, ".init.text") && !opts.module)
 453                         sec->init = true;
 454
 455                 for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) {
 456                         if (!insns || idx == INSN_CHUNK_MAX) {
 457                                 insns = calloc(sizeof(*insn), INSN_CHUNK_SIZE);
 458                                 if (!insns) {
 459                                         WARN("malloc failed");
 460                                         return -1;
 461                                 }
 462                                 idx = 0;
 463                         } else {
 464                                 idx++;
 465                         }
 466                         insn = &insns[idx];
 467                         insn->idx = idx;
 468
 469                         INIT_LIST_HEAD(&insn->call_node);
 470                         insn->sec = sec;
 471                         insn->offset = offset;
 472                         insn->prev_len = prev_len;
 473
 474                         ret = arch_decode_instruction(file, sec, offset,
 475                                                       sec->sh.sh_size - offset,
 476                                                       insn);
 477                         if (ret)
 478                                 return ret;
 479
 480                         prev_len = insn->len;
 481
 482                         /*
 483                          * By default, "ud2" is a dead end unless otherwise
 484                          * annotated, because GCC 7 inserts it for certain
 485                          * divide-by-zero cases.
 486                          */
 487                         if (insn->type == INSN_BUG)
 488                                 insn->dead_end = true;
 489
 490                         hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
 491                         nr_insns++;
 492                 }
 493
 494 //              printf("%s: last chunk used: %d\n", sec->name, (int)idx);
 495
 496                 sec_for_each_sym(sec, func) {
 497                         if (func->type != STT_NOTYPE && func->type != STT_FUNC)
 498                                 continue;
 499
 500                         if (func->offset == sec->sh.sh_size) {
 501                                 /* Heuristic: likely an "end" symbol */
 502                                 if (func->type == STT_NOTYPE)
 503                                         continue;
 504                                 WARN("%s(): STT_FUNC at end of section",
 505                                      func->name);
 506                                 return -1;
 507                         }
 508
 509                         if (func->embedded_insn || func->alias != func)
 510                                 continue;
 511
 512                         if (!find_insn(file, sec, func->offset)) {
 513                                 WARN("%s(): can't find starting instruction",
 514                                      func->name);
 515                                 return -1;
 516                         }
 517
 518                         sym_for_each_insn(file, func, insn) {
 519                                 insn->sym = func;
 520                                 if (func->type == STT_FUNC &&
 521                                     insn->type == INSN_ENDBR &&
 522                                     list_empty(&insn->call_node)) {
 523                                         if (insn->offset == func->offset) {
 524                                                 list_add_tail(&insn->call_node, &file->endbr_list);
 525                                                 file->nr_endbr++;
 526                                         } else {
 527                                                 file->nr_endbr_int++;
 528                                         }
 529                                 }
 530                         }
 531                 }
 532         }
 533
 534         if (opts.stats)
 535                 printf("nr_insns: %lu\n", nr_insns);
 536
 537         return 0;
 538 }
 539
 540 /*
 541  * Read the pv_ops[] .data table to find the static initialized values.
 542  */
 543 static int add_pv_ops(struct objtool_file *file, const char *symname)
 544 {
 545         struct symbol *sym, *func;
 546         unsigned long off, end;
 547         struct reloc *reloc;
 548         int idx;
 549
 550         sym = find_symbol_by_name(file->elf, symname);
 551         if (!sym)
 552                 return 0;
 553
 554         off = sym->offset;
 555         end = off + sym->len;
 556         for (;;) {
 557                 reloc = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off);
 558                 if (!reloc)
 559                         break;
 560
 561                 func = reloc->sym;
 562                 if (func->type == STT_SECTION)
 563                         func = find_symbol_by_offset(reloc->sym->sec,
 564                                                      reloc_addend(reloc));
 565
 566                 idx = (reloc_offset(reloc) - sym->offset) / sizeof(unsigned long);
 567
 568                 objtool_pv_add(file, idx, func);
 569
 570                 off = reloc_offset(reloc) + 1;
 571                 if (off > end)
 572                         break;
 573         }
 574
 575         return 0;
 576 }
 577
 578 /*
 579  * Allocate and initialize file->pv_ops[].
 580  */
 581 static int init_pv_ops(struct objtool_file *file)
 582 {
 583         static const char *pv_ops_tables[] = {
 584                 "pv_ops",
 585                 "xen_cpu_ops",
 586                 "xen_irq_ops",
 587                 "xen_mmu_ops",
 588                 NULL,
 589         };
 590         const char *pv_ops;
 591         struct symbol *sym;
 592         int idx, nr;
 593
 594         if (!opts.noinstr)
 595                 return 0;
 596
 597         file->pv_ops = NULL;
 598
 599         sym = find_symbol_by_name(file->elf, "pv_ops");
 600         if (!sym)
 601                 return 0;
 602
 603         nr = sym->len / sizeof(unsigned long);
 604         file->pv_ops = calloc(sizeof(struct pv_state), nr);
 605         if (!file->pv_ops)
 606                 return -1;
 607
 608         for (idx = 0; idx < nr; idx++)
 609                 INIT_LIST_HEAD(&file->pv_ops[idx].targets);
 610
 611         for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++)
 612                 add_pv_ops(file, pv_ops);
 613
 614         return 0;
 615 }
 616
 617 static struct instruction *find_last_insn(struct objtool_file *file,
 618                                           struct section *sec)
 619 {
 620         struct instruction *insn = NULL;
 621         unsigned int offset;
 622         unsigned int end = (sec->sh.sh_size > 10) ? sec->sh.sh_size - 10 : 0;
 623
 624         for (offset = sec->sh.sh_size - 1; offset >= end && !insn; offset--)
 625                 insn = find_insn(file, sec, offset);
 626
 627         return insn;
 628 }
 629
 630 /*
 631  * Mark "ud2" instructions and manually annotated dead ends.
 632  */
 633 static int add_dead_ends(struct objtool_file *file)
 634 {
 635         struct section *rsec;
 636         struct reloc *reloc;
 637         struct instruction *insn;
 638         uint64_t offset;
 639
 640         /*
 641          * Check for manually annotated dead ends.
 642          */
 643         rsec = find_section_by_name(file->elf, ".rela.discard.unreachable");
 644         if (!rsec)
 645                 goto reachable;
 646
 647         for_each_reloc(rsec, reloc) {
 648                 if (reloc->sym->type == STT_SECTION) {
 649                         offset = reloc_addend(reloc);
 650                 } else if (reloc->sym->local_label) {
 651                         offset = reloc->sym->offset;
 652                 } else {
 653                         WARN("unexpected relocation symbol type in %s", rsec->name);
 654                         return -1;
 655                 }
 656
 657                 insn = find_insn(file, reloc->sym->sec, offset);
 658                 if (insn)
 659                         insn = prev_insn_same_sec(file, insn);
 660                 else if (offset == reloc->sym->sec->sh.sh_size) {
 661                         insn = find_last_insn(file, reloc->sym->sec);
 662                         if (!insn) {
 663                                 WARN("can't find unreachable insn at %s+0x%" PRIx64,
 664                                      reloc->sym->sec->name, offset);
 665                                 return -1;
 666                         }
 667                 } else {
 668                         WARN("can't find unreachable insn at %s+0x%" PRIx64,
 669                              reloc->sym->sec->name, offset);
 670                         return -1;
 671                 }
 672
 673                 insn->dead_end = true;
 674         }
 675
 676 reachable:
 677         /*
 678          * These manually annotated reachable checks are needed for GCC 4.4,
 679          * where the Linux unreachable() macro isn't supported.  In that case
 680          * GCC doesn't know the "ud2" is fatal, so it generates code as if it's
 681          * not a dead end.
 682          */
 683         rsec = find_section_by_name(file->elf, ".rela.discard.reachable");
 684         if (!rsec)
 685                 return 0;
 686
 687         for_each_reloc(rsec, reloc) {
 688                 if (reloc->sym->type == STT_SECTION) {
 689                         offset = reloc_addend(reloc);
 690                 } else if (reloc->sym->local_label) {
 691                         offset = reloc->sym->offset;
 692                 } else {
 693                         WARN("unexpected relocation symbol type in %s", rsec->name);
 694                         return -1;
 695                 }
 696
 697                 insn = find_insn(file, reloc->sym->sec, offset);
 698                 if (insn)
 699                         insn = prev_insn_same_sec(file, insn);
 700                 else if (offset == reloc->sym->sec->sh.sh_size) {
 701                         insn = find_last_insn(file, reloc->sym->sec);
 702                         if (!insn) {
 703                                 WARN("can't find reachable insn at %s+0x%" PRIx64,
 704                                      reloc->sym->sec->name, offset);
 705                                 return -1;
 706                         }
 707                 } else {
 708                         WARN("can't find reachable insn at %s+0x%" PRIx64,
 709                              reloc->sym->sec->name, offset);
 710                         return -1;
 711                 }
 712
 713                 insn->dead_end = false;
 714         }
 715
 716         return 0;
 717 }
 718
 719 static int create_static_call_sections(struct objtool_file *file)
 720 {
 721         struct static_call_site *site;
 722         struct section *sec;
 723         struct instruction *insn;
 724         struct symbol *key_sym;
 725         char *key_name, *tmp;
 726         int idx;
 727
 728         sec = find_section_by_name(file->elf, ".static_call_sites");
 729         if (sec) {
 730                 INIT_LIST_HEAD(&file->static_call_list);
 731                 WARN("file already has .static_call_sites section, skipping");
 732                 return 0;
 733         }
 734
 735         if (list_empty(&file->static_call_list))
 736                 return 0;
 737
 738         idx = 0;
 739         list_for_each_entry(insn, &file->static_call_list, call_node)
 740                 idx++;
 741
 742         sec = elf_create_section_pair(file->elf, ".static_call_sites",
 743                                       sizeof(*site), idx, idx * 2);
 744         if (!sec)
 745                 return -1;
 746
 747         /* Allow modules to modify the low bits of static_call_site::key */
 748         sec->sh.sh_flags |= SHF_WRITE;
 749
 750         idx = 0;
 751         list_for_each_entry(insn, &file->static_call_list, call_node) {
 752
 753                 /* populate reloc for 'addr' */
 754                 if (!elf_init_reloc_text_sym(file->elf, sec,
 755                                              idx * sizeof(*site), idx * 2,
 756                                              insn->sec, insn->offset))
 757                         return -1;
 758
 759                 /* find key symbol */
 760                 key_name = strdup(insn_call_dest(insn)->name);
 761                 if (!key_name) {
 762                         perror("strdup");
 763                         return -1;
 764                 }
 765                 if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
 766                             STATIC_CALL_TRAMP_PREFIX_LEN)) {
 767                         WARN("static_call: trampoline name malformed: %s", key_name);
 768                         free(key_name);
 769                         return -1;
 770                 }
 771                 tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
 772                 memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);
 773
 774                 key_sym = find_symbol_by_name(file->elf, tmp);
 775                 if (!key_sym) {
 776                         if (!opts.module) {
 777                                 WARN("static_call: can't find static_call_key symbol: %s", tmp);
 778                                 free(key_name);
 779                                 return -1;
 780                         }
 781
 782                         /*
 783                          * For modules(), the key might not be exported, which
 784                          * means the module can make static calls but isn't
 785                          * allowed to change them.
 786                          *
 787                          * In that case we temporarily set the key to be the
 788                          * trampoline address.  This is fixed up in
 789                          * static_call_add_module().
 790                          */
 791                         key_sym = insn_call_dest(insn);
 792                 }
 793                 free(key_name);
 794
 795                 /* populate reloc for 'key' */
 796                 if (!elf_init_reloc_data_sym(file->elf, sec,
 797                                              idx * sizeof(*site) + 4,
 798                                              (idx * 2) + 1, key_sym,
 799                                              is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
 800                         return -1;
 801
 802                 idx++;
 803         }
 804
 805         return 0;
 806 }
 807
 808 static int create_retpoline_sites_sections(struct objtool_file *file)
 809 {
 810         struct instruction *insn;
 811         struct section *sec;
 812         int idx;
 813
 814         sec = find_section_by_name(file->elf, ".retpoline_sites");
 815         if (sec) {
 816                 WARN("file already has .retpoline_sites, skipping");
 817                 return 0;
 818         }
 819
 820         idx = 0;
 821         list_for_each_entry(insn, &file->retpoline_call_list, call_node)
 822                 idx++;
 823
 824         if (!idx)
 825                 return 0;
 826
 827         sec = elf_create_section_pair(file->elf, ".retpoline_sites",
 828                                       sizeof(int), idx, idx);
 829         if (!sec)
 830                 return -1;
 831
 832         idx = 0;
 833         list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
 834
 835                 if (!elf_init_reloc_text_sym(file->elf, sec,
 836                                              idx * sizeof(int), idx,
 837                                              insn->sec, insn->offset))
 838                         return -1;
 839
 840                 idx++;
 841         }
 842
 843         return 0;
 844 }
 845
 846 static int create_return_sites_sections(struct objtool_file *file)
 847 {
 848         struct instruction *insn;
 849         struct section *sec;
 850         int idx;
 851
 852         sec = find_section_by_name(file->elf, ".return_sites");
 853         if (sec) {
 854                 WARN("file already has .return_sites, skipping");
 855                 return 0;
 856         }
 857
 858         idx = 0;
 859         list_for_each_entry(insn, &file->return_thunk_list, call_node)
 860                 idx++;
 861
 862         if (!idx)
 863                 return 0;
 864
 865         sec = elf_create_section_pair(file->elf, ".return_sites",
 866                                       sizeof(int), idx, idx);
 867         if (!sec)
 868                 return -1;
 869
 870         idx = 0;
 871         list_for_each_entry(insn, &file->return_thunk_list, call_node) {
 872
 873                 if (!elf_init_reloc_text_sym(file->elf, sec,
 874                                              idx * sizeof(int), idx,
 875                                              insn->sec, insn->offset))
 876                         return -1;
 877
 878                 idx++;
 879         }
 880
 881         return 0;
 882 }
 883
 884 static int create_ibt_endbr_seal_sections(struct objtool_file *file)
 885 {
 886         struct instruction *insn;
 887         struct section *sec;
 888         int idx;
 889
 890         sec = find_section_by_name(file->elf, ".ibt_endbr_seal");
 891         if (sec) {
 892                 WARN("file already has .ibt_endbr_seal, skipping");
 893                 return 0;
 894         }
 895
 896         idx = 0;
 897         list_for_each_entry(insn, &file->endbr_list, call_node)
 898                 idx++;
 899
 900         if (opts.stats) {
 901                 printf("ibt: ENDBR at function start: %d\n", file->nr_endbr);
 902                 printf("ibt: ENDBR inside functions:  %d\n", file->nr_endbr_int);
 903                 printf("ibt: superfluous ENDBR:       %d\n", idx);
 904         }
 905
 906         if (!idx)
 907                 return 0;
 908
 909         sec = elf_create_section_pair(file->elf, ".ibt_endbr_seal",
 910                                       sizeof(int), idx, idx);
 911         if (!sec)
 912                 return -1;
 913
 914         idx = 0;
 915         list_for_each_entry(insn, &file->endbr_list, call_node) {
 916
 917                 int *site = (int *)sec->data->d_buf + idx;
 918                 struct symbol *sym = insn->sym;
 919                 *site = 0;
 920
 921                 if (opts.module && sym && sym->type == STT_FUNC &&
 922                     insn->offset == sym->offset &&
 923                     (!strcmp(sym->name, "init_module") ||
 924                      !strcmp(sym->name, "cleanup_module")))
 925                         WARN("%s(): not an indirect call target", sym->name);
 926
 927                 if (!elf_init_reloc_text_sym(file->elf, sec,
 928                                              idx * sizeof(int), idx,
 929                                              insn->sec, insn->offset))
 930                         return -1;
 931
 932                 idx++;
 933         }
 934
 935         return 0;
 936 }
 937
 938 static int create_cfi_sections(struct objtool_file *file)
 939 {
 940         struct section *sec;
 941         struct symbol *sym;
 942         int idx;
 943
 944         sec = find_section_by_name(file->elf, ".cfi_sites");
 945         if (sec) {
 946                 INIT_LIST_HEAD(&file->call_list);
 947                 WARN("file already has .cfi_sites section, skipping");
 948                 return 0;
 949         }
 950
 951         idx = 0;
 952         for_each_sym(file, sym) {
 953                 if (sym->type != STT_FUNC)
 954                         continue;
 955
 956                 if (strncmp(sym->name, "__cfi_", 6))
 957                         continue;
 958
 959                 idx++;
 960         }
 961
 962         sec = elf_create_section_pair(file->elf, ".cfi_sites",
 963                                       sizeof(unsigned int), idx, idx);
 964         if (!sec)
 965                 return -1;
 966
 967         idx = 0;
 968         for_each_sym(file, sym) {
 969                 if (sym->type != STT_FUNC)
 970                         continue;
 971
 972                 if (strncmp(sym->name, "__cfi_", 6))
 973                         continue;
 974
 975                 if (!elf_init_reloc_text_sym(file->elf, sec,
 976                                              idx * sizeof(unsigned int), idx,
 977                                              sym->sec, sym->offset))
 978                         return -1;
 979
 980                 idx++;
 981         }
 982
 983         return 0;
 984 }
 985
 986 static int create_mcount_loc_sections(struct objtool_file *file)
 987 {
 988         size_t addr_size = elf_addr_size(file->elf);
 989         struct instruction *insn;
 990         struct section *sec;
 991         int idx;
 992
 993         sec = find_section_by_name(file->elf, "__mcount_loc");
 994         if (sec) {
 995                 INIT_LIST_HEAD(&file->mcount_loc_list);
 996                 WARN("file already has __mcount_loc section, skipping");
 997                 return 0;
 998         }
 999
1000         if (list_empty(&file->mcount_loc_list))
1001                 return 0;
1002
1003         idx = 0;
1004         list_for_each_entry(insn, &file->mcount_loc_list, call_node)
1005                 idx++;
1006
1007         sec = elf_create_section_pair(file->elf, "__mcount_loc", addr_size,
1008                                       idx, idx);
1009         if (!sec)
1010                 return -1;
1011
1012         sec->sh.sh_addralign = addr_size;
1013
1014         idx = 0;
1015         list_for_each_entry(insn, &file->mcount_loc_list, call_node) {
1016
1017                 struct reloc *reloc;
1018
1019                 reloc = elf_init_reloc_text_sym(file->elf, sec, idx * addr_size, idx,
1020                                                insn->sec, insn->offset);
1021                 if (!reloc)
1022                         return -1;
1023
1024                 set_reloc_type(file->elf, reloc, addr_size == 8 ? R_ABS64 : R_ABS32);
1025
1026                 idx++;
1027         }
1028
1029         return 0;
1030 }
1031
1032 static int create_direct_call_sections(struct objtool_file *file)
1033 {
1034         struct instruction *insn;
1035         struct section *sec;
1036         int idx;
1037
1038         sec = find_section_by_name(file->elf, ".call_sites");
1039         if (sec) {
1040                 INIT_LIST_HEAD(&file->call_list);
1041                 WARN("file already has .call_sites section, skipping");
1042                 return 0;
1043         }
1044
1045         if (list_empty(&file->call_list))
1046                 return 0;
1047
1048         idx = 0;
1049         list_for_each_entry(insn, &file->call_list, call_node)
1050                 idx++;
1051
1052         sec = elf_create_section_pair(file->elf, ".call_sites",
1053                                       sizeof(unsigned int), idx, idx);
1054         if (!sec)
1055                 return -1;
1056
1057         idx = 0;
1058         list_for_each_entry(insn, &file->call_list, call_node) {
1059
1060                 if (!elf_init_reloc_text_sym(file->elf, sec,
1061                                              idx * sizeof(unsigned int), idx,
1062                                              insn->sec, insn->offset))
1063                         return -1;
1064
1065                 idx++;
1066         }
1067
1068         return 0;
1069 }
1070
1071 /*
1072  * Warnings shouldn't be reported for ignored functions.
1073  */
1074 static void add_ignores(struct objtool_file *file)
1075 {
1076         struct instruction *insn;
1077         struct section *rsec;
1078         struct symbol *func;
1079         struct reloc *reloc;
1080
1081         rsec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard");
1082         if (!rsec)
1083                 return;
1084
1085         for_each_reloc(rsec, reloc) {
1086                 switch (reloc->sym->type) {
1087                 case STT_FUNC:
1088                         func = reloc->sym;
1089                         break;
1090
1091                 case STT_SECTION:
1092                         func = find_func_by_offset(reloc->sym->sec, reloc_addend(reloc));
1093                         if (!func)
1094                                 continue;
1095                         break;
1096
1097                 default:
1098                         WARN("unexpected relocation symbol type in %s: %d",
1099                              rsec->name, reloc->sym->type);
1100                         continue;
1101                 }
1102
1103                 func_for_each_insn(file, func, insn)
1104                         insn->ignore = true;
1105         }
1106 }
1107
1108 /*
1109  * This is a whitelist of functions that is allowed to be called with AC set.
1110  * The list is meant to be minimal and only contains compiler instrumentation
1111  * ABI and a few functions used to implement *_{to,from}_user() functions.
1112  *
1113  * These functions must not directly change AC, but may PUSHF/POPF.
1114  */
1115 static const char *uaccess_safe_builtin[] = {
1116         /* KASAN */
1117         "kasan_report",
1118         "kasan_check_range",
1119         /* KASAN out-of-line */
1120         "__asan_loadN_noabort",
1121         "__asan_load1_noabort",
1122         "__asan_load2_noabort",
1123         "__asan_load4_noabort",
1124         "__asan_load8_noabort",
1125         "__asan_load16_noabort",
1126         "__asan_storeN_noabort",
1127         "__asan_store1_noabort",
1128         "__asan_store2_noabort",
1129         "__asan_store4_noabort",
1130         "__asan_store8_noabort",
1131         "__asan_store16_noabort",
1132         "__kasan_check_read",
1133         "__kasan_check_write",
1134         /* KASAN in-line */
1135         "__asan_report_load_n_noabort",
1136         "__asan_report_load1_noabort",
1137         "__asan_report_load2_noabort",
1138         "__asan_report_load4_noabort",
1139         "__asan_report_load8_noabort",
1140         "__asan_report_load16_noabort",
1141         "__asan_report_store_n_noabort",
1142         "__asan_report_store1_noabort",
1143         "__asan_report_store2_noabort",
1144         "__asan_report_store4_noabort",
1145         "__asan_report_store8_noabort",
1146         "__asan_report_store16_noabort",
1147         /* KCSAN */
1148         "__kcsan_check_access",
1149         "__kcsan_mb",
1150         "__kcsan_wmb",
1151         "__kcsan_rmb",
1152         "__kcsan_release",
1153         "kcsan_found_watchpoint",
1154         "kcsan_setup_watchpoint",
1155         "kcsan_check_scoped_accesses",
1156         "kcsan_disable_current",
1157         "kcsan_enable_current_nowarn",
1158         /* KCSAN/TSAN */
1159         "__tsan_func_entry",
1160         "__tsan_func_exit",
1161         "__tsan_read_range",
1162         "__tsan_write_range",
1163         "__tsan_read1",
1164         "__tsan_read2",
1165         "__tsan_read4",
1166         "__tsan_read8",
1167         "__tsan_read16",
1168         "__tsan_write1",
1169         "__tsan_write2",
1170         "__tsan_write4",
1171         "__tsan_write8",
1172         "__tsan_write16",
1173         "__tsan_read_write1",
1174         "__tsan_read_write2",
1175         "__tsan_read_write4",
1176         "__tsan_read_write8",
1177         "__tsan_read_write16",
1178         "__tsan_volatile_read1",
1179         "__tsan_volatile_read2",
1180         "__tsan_volatile_read4",
1181         "__tsan_volatile_read8",
1182         "__tsan_volatile_read16",
1183         "__tsan_volatile_write1",
1184         "__tsan_volatile_write2",
1185         "__tsan_volatile_write4",
1186         "__tsan_volatile_write8",
1187         "__tsan_volatile_write16",
1188         "__tsan_atomic8_load",
1189         "__tsan_atomic16_load",
1190         "__tsan_atomic32_load",
1191         "__tsan_atomic64_load",
1192         "__tsan_atomic8_store",
1193         "__tsan_atomic16_store",
1194         "__tsan_atomic32_store",
1195         "__tsan_atomic64_store",
1196         "__tsan_atomic8_exchange",
1197         "__tsan_atomic16_exchange",
1198         "__tsan_atomic32_exchange",
1199         "__tsan_atomic64_exchange",
1200         "__tsan_atomic8_fetch_add",
1201         "__tsan_atomic16_fetch_add",
1202         "__tsan_atomic32_fetch_add",
1203         "__tsan_atomic64_fetch_add",
1204         "__tsan_atomic8_fetch_sub",
1205         "__tsan_atomic16_fetch_sub",
1206         "__tsan_atomic32_fetch_sub",
1207         "__tsan_atomic64_fetch_sub",
1208         "__tsan_atomic8_fetch_and",
1209         "__tsan_atomic16_fetch_and",
1210         "__tsan_atomic32_fetch_and",
1211         "__tsan_atomic64_fetch_and",
1212         "__tsan_atomic8_fetch_or",
1213         "__tsan_atomic16_fetch_or",
1214         "__tsan_atomic32_fetch_or",
1215         "__tsan_atomic64_fetch_or",
1216         "__tsan_atomic8_fetch_xor",
1217         "__tsan_atomic16_fetch_xor",
1218         "__tsan_atomic32_fetch_xor",
1219         "__tsan_atomic64_fetch_xor",
1220         "__tsan_atomic8_fetch_nand",
1221         "__tsan_atomic16_fetch_nand",
1222         "__tsan_atomic32_fetch_nand",
1223         "__tsan_atomic64_fetch_nand",
1224         "__tsan_atomic8_compare_exchange_strong",
1225         "__tsan_atomic16_compare_exchange_strong",
1226         "__tsan_atomic32_compare_exchange_strong",
1227         "__tsan_atomic64_compare_exchange_strong",
1228         "__tsan_atomic8_compare_exchange_weak",
1229         "__tsan_atomic16_compare_exchange_weak",
1230         "__tsan_atomic32_compare_exchange_weak",
1231         "__tsan_atomic64_compare_exchange_weak",
1232         "__tsan_atomic8_compare_exchange_val",
1233         "__tsan_atomic16_compare_exchange_val",
1234         "__tsan_atomic32_compare_exchange_val",
1235         "__tsan_atomic64_compare_exchange_val",
1236         "__tsan_atomic_thread_fence",
1237         "__tsan_atomic_signal_fence",
1238         "__tsan_unaligned_read16",
1239         "__tsan_unaligned_write16",
1240         /* KCOV */
1241         "write_comp_data",
1242         "check_kcov_mode",
1243         "__sanitizer_cov_trace_pc",
1244         "__sanitizer_cov_trace_const_cmp1",
1245         "__sanitizer_cov_trace_const_cmp2",
1246         "__sanitizer_cov_trace_const_cmp4",
1247         "__sanitizer_cov_trace_const_cmp8",
1248         "__sanitizer_cov_trace_cmp1",
1249         "__sanitizer_cov_trace_cmp2",
1250         "__sanitizer_cov_trace_cmp4",
1251         "__sanitizer_cov_trace_cmp8",
1252         "__sanitizer_cov_trace_switch",
1253         /* KMSAN */
1254         "kmsan_copy_to_user",
1255         "kmsan_disable_current",
1256         "kmsan_enable_current",
1257         "kmsan_report",
1258         "kmsan_unpoison_entry_regs",
1259         "kmsan_unpoison_memory",
1260         "__msan_chain_origin",
1261         "__msan_get_context_state",
1262         "__msan_instrument_asm_store",
1263         "__msan_metadata_ptr_for_load_1",
1264         "__msan_metadata_ptr_for_load_2",
1265         "__msan_metadata_ptr_for_load_4",
1266         "__msan_metadata_ptr_for_load_8",
1267         "__msan_metadata_ptr_for_load_n",
1268         "__msan_metadata_ptr_for_store_1",
1269         "__msan_metadata_ptr_for_store_2",
1270         "__msan_metadata_ptr_for_store_4",
1271         "__msan_metadata_ptr_for_store_8",
1272         "__msan_metadata_ptr_for_store_n",
1273         "__msan_poison_alloca",
1274         "__msan_warning",
1275         /* UBSAN */
1276         "ubsan_type_mismatch_common",
1277         "__ubsan_handle_type_mismatch",
1278         "__ubsan_handle_type_mismatch_v1",
1279         "__ubsan_handle_shift_out_of_bounds",
1280         "__ubsan_handle_load_invalid_value",
1281         /* STACKLEAK */
1282         "stackleak_track_stack",
1283         /* misc */
1284         "csum_partial_copy_generic",
1285         "copy_mc_fragile",
1286         "copy_mc_fragile_handle_tail",
1287         "copy_mc_enhanced_fast_string",
1288         "ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
1289         "rep_stos_alternative",
1290         "rep_movs_alternative",
1291         "__copy_user_nocache",
1292         NULL
1293 };
1294
1295 static void add_uaccess_safe(struct objtool_file *file)
1296 {
1297         struct symbol *func;
1298         const char **name;
1299
1300         if (!opts.uaccess)
1301                 return;
1302
1303         for (name = uaccess_safe_builtin; *name; name++) {
1304                 func = find_symbol_by_name(file->elf, *name);
1305                 if (!func)
1306                         continue;
1307
1308                 func->uaccess_safe = true;
1309         }
1310 }
1311
1312 /*
1313  * FIXME: For now, just ignore any alternatives which add retpolines.  This is
1314  * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
1315  * But it at least allows objtool to understand the control flow *around* the
1316  * retpoline.
1317  */
1318 static int add_ignore_alternatives(struct objtool_file *file)
1319 {
1320         struct section *rsec;
1321         struct reloc *reloc;
1322         struct instruction *insn;
1323
1324         rsec = find_section_by_name(file->elf, ".rela.discard.ignore_alts");
1325         if (!rsec)
1326                 return 0;
1327
1328         for_each_reloc(rsec, reloc) {
1329                 if (reloc->sym->type != STT_SECTION) {
1330                         WARN("unexpected relocation symbol type in %s", rsec->name);
1331                         return -1;
1332                 }
1333
1334                 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
1335                 if (!insn) {
1336                         WARN("bad .discard.ignore_alts entry");
1337                         return -1;
1338                 }
1339
1340                 insn->ignore_alts = true;
1341         }
1342
1343         return 0;
1344 }
1345
1346 /*
1347  * Symbols that replace INSN_CALL_DYNAMIC, every (tail) call to such a symbol
1348  * will be added to the .retpoline_sites section.
1349  */
1350 __weak bool arch_is_retpoline(struct symbol *sym)
1351 {
1352         return false;
1353 }
1354
1355 /*
1356  * Symbols that replace INSN_RETURN, every (tail) call to such a symbol
1357  * will be added to the .return_sites section.
1358  */
1359 __weak bool arch_is_rethunk(struct symbol *sym)
1360 {
1361         return false;
1362 }
1363
1364 /*
1365  * Symbols that are embedded inside other instructions, because sometimes crazy
1366  * code exists. These are mostly ignored for validation purposes.
1367  */
1368 __weak bool arch_is_embedded_insn(struct symbol *sym)
1369 {
1370         return false;
1371 }
1372
1373 static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn)
1374 {
1375         struct reloc *reloc;
1376
1377         if (insn->no_reloc)
1378                 return NULL;
1379
1380         if (!file)
1381                 return NULL;
1382
1383         reloc = find_reloc_by_dest_range(file->elf, insn->sec,
1384                                          insn->offset, insn->len);
1385         if (!reloc) {
1386                 insn->no_reloc = 1;
1387                 return NULL;
1388         }
1389
1390         return reloc;
1391 }
1392
1393 static void remove_insn_ops(struct instruction *insn)
1394 {
1395         struct stack_op *op, *next;
1396
1397         for (op = insn->stack_ops; op; op = next) {
1398                 next = op->next;
1399                 free(op);
1400         }
1401         insn->stack_ops = NULL;
1402 }
1403
1404 static void annotate_call_site(struct objtool_file *file,
1405                                struct instruction *insn, bool sibling)
1406 {
1407         struct reloc *reloc = insn_reloc(file, insn);
1408         struct symbol *sym = insn_call_dest(insn);
1409
1410         if (!sym)
1411                 sym = reloc->sym;
1412
1413         /*
1414          * Alternative replacement code is just template code which is
1415          * sometimes copied to the original instruction. For now, don't
1416          * annotate it. (In the future we might consider annotating the
1417          * original instruction if/when it ever makes sense to do so.)
1418          */
1419         if (!strcmp(insn->sec->name, ".altinstr_replacement"))
1420                 return;
1421
1422         if (sym->static_call_tramp) {
1423                 list_add_tail(&insn->call_node, &file->static_call_list);
1424                 return;
1425         }
1426
1427         if (sym->retpoline_thunk) {
1428                 list_add_tail(&insn->call_node, &file->retpoline_call_list);
1429                 return;
1430         }
1431
1432         /*
1433          * Many compilers cannot disable KCOV or sanitizer calls with a function
1434          * attribute so they need a little help, NOP out any such calls from
1435          * noinstr text.
1436          */
1437         if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) {
1438                 if (reloc)
1439                         set_reloc_type(file->elf, reloc, R_NONE);
1440
1441                 elf_write_insn(file->elf, insn->sec,
1442                                insn->offset, insn->len,
1443                                sibling ? arch_ret_insn(insn->len)
1444                                        : arch_nop_insn(insn->len));
1445
1446                 insn->type = sibling ? INSN_RETURN : INSN_NOP;
1447
1448                 if (sibling) {
1449                         /*
1450                          * We've replaced the tail-call JMP insn by two new
1451                          * insn: RET; INT3, except we only have a single struct
1452                          * insn here. Mark it retpoline_safe to avoid the SLS
1453                          * warning, instead of adding another insn.
1454                          */
1455                         insn->retpoline_safe = true;
1456                 }
1457
1458                 return;
1459         }
1460
1461         if (opts.mcount && sym->fentry) {
1462                 if (sibling)
1463                         WARN_INSN(insn, "tail call to __fentry__ !?!?");
1464                 if (opts.mnop) {
1465                         if (reloc)
1466                                 set_reloc_type(file->elf, reloc, R_NONE);
1467
1468                         elf_write_insn(file->elf, insn->sec,
1469                                        insn->offset, insn->len,
1470                                        arch_nop_insn(insn->len));
1471
1472                         insn->type = INSN_NOP;
1473                 }
1474
1475                 list_add_tail(&insn->call_node, &file->mcount_loc_list);
1476                 return;
1477         }
1478
1479         if (insn->type == INSN_CALL && !insn->sec->init)
1480                 list_add_tail(&insn->call_node, &file->call_list);
1481
1482         if (!sibling && dead_end_function(file, sym))
1483                 insn->dead_end = true;
1484 }
1485
1486 static void add_call_dest(struct objtool_file *file, struct instruction *insn,
1487                           struct symbol *dest, bool sibling)
1488 {
1489         insn->_call_dest = dest;
1490         if (!dest)
1491                 return;
1492
1493         /*
1494          * Whatever stack impact regular CALLs have, should be undone
1495          * by the RETURN of the called function.
1496          *
1497          * Annotated intra-function calls retain the stack_ops but
1498          * are converted to JUMP, see read_intra_function_calls().
1499          */
1500         remove_insn_ops(insn);
1501
1502         annotate_call_site(file, insn, sibling);
1503 }
1504
1505 static void add_retpoline_call(struct objtool_file *file, struct instruction *insn)
1506 {
1507         /*
1508          * Retpoline calls/jumps are really dynamic calls/jumps in disguise,
1509          * so convert them accordingly.
1510          */
1511         switch (insn->type) {
1512         case INSN_CALL:
1513                 insn->type = INSN_CALL_DYNAMIC;
1514                 break;
1515         case INSN_JUMP_UNCONDITIONAL:
1516                 insn->type = INSN_JUMP_DYNAMIC;
1517                 break;
1518         case INSN_JUMP_CONDITIONAL:
1519                 insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
1520                 break;
1521         default:
1522                 return;
1523         }
1524
1525         insn->retpoline_safe = true;
1526
1527         /*
1528          * Whatever stack impact regular CALLs have, should be undone
1529          * by the RETURN of the called function.
1530          *
1531          * Annotated intra-function calls retain the stack_ops but
1532          * are converted to JUMP, see read_intra_function_calls().
1533          */
1534         remove_insn_ops(insn);
1535
1536         annotate_call_site(file, insn, false);
1537 }
1538
1539 static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add)
1540 {
1541         /*
1542          * Return thunk tail calls are really just returns in disguise,
1543          * so convert them accordingly.
1544          */
1545         insn->type = INSN_RETURN;
1546         insn->retpoline_safe = true;
1547
1548         if (add)
1549                 list_add_tail(&insn->call_node, &file->return_thunk_list);
1550 }
1551
1552 static bool is_first_func_insn(struct objtool_file *file,
1553                                struct instruction *insn, struct symbol *sym)
1554 {
1555         if (insn->offset == sym->offset)
1556                 return true;
1557
1558         /* Allow direct CALL/JMP past ENDBR */
1559         if (opts.ibt) {
1560                 struct instruction *prev = prev_insn_same_sym(file, insn);
1561
1562                 if (prev && prev->type == INSN_ENDBR &&
1563                     insn->offset == sym->offset + prev->len)
1564                         return true;
1565         }
1566
1567         return false;
1568 }
1569
1570 /*
1571  * A sibling call is a tail-call to another symbol -- to differentiate from a
1572  * recursive tail-call which is to the same symbol.
1573  */
1574 static bool jump_is_sibling_call(struct objtool_file *file,
1575                                  struct instruction *from, struct instruction *to)
1576 {
1577         struct symbol *fs = from->sym;
1578         struct symbol *ts = to->sym;
1579
1580         /* Not a sibling call if from/to a symbol hole */
1581         if (!fs || !ts)
1582                 return false;
1583
1584         /* Not a sibling call if not targeting the start of a symbol. */
1585         if (!is_first_func_insn(file, to, ts))
1586                 return false;
1587
1588         /* Disallow sibling calls into STT_NOTYPE */
1589         if (ts->type == STT_NOTYPE)
1590                 return false;
1591
1592         /* Must not be self to be a sibling */
1593         return fs->pfunc != ts->pfunc;
1594 }
1595
1596 /*
1597  * Find the destination instructions for all jumps.
1598  */
1599 static int add_jump_destinations(struct objtool_file *file)
1600 {
1601         struct instruction *insn, *jump_dest;
1602         struct reloc *reloc;
1603         struct section *dest_sec;
1604         unsigned long dest_off;
1605
1606         for_each_insn(file, insn) {
1607                 if (insn->jump_dest) {
1608                         /*
1609                          * handle_group_alt() may have previously set
1610                          * 'jump_dest' for some alternatives.
1611                          */
1612                         continue;
1613                 }
1614                 if (!is_static_jump(insn))
1615                         continue;
1616
1617                 reloc = insn_reloc(file, insn);
1618                 if (!reloc) {
1619                         dest_sec = insn->sec;
1620                         dest_off = arch_jump_destination(insn);
1621                 } else if (reloc->sym->type == STT_SECTION) {
1622                         dest_sec = reloc->sym->sec;
1623                         dest_off = arch_dest_reloc_offset(reloc_addend(reloc));
1624                 } else if (reloc->sym->retpoline_thunk) {
1625                         add_retpoline_call(file, insn);
1626                         continue;
1627                 } else if (reloc->sym->return_thunk) {
1628                         add_return_call(file, insn, true);
1629                         continue;
1630                 } else if (insn_func(insn)) {
1631                         /*
1632                          * External sibling call or internal sibling call with
1633                          * STT_FUNC reloc.
1634                          */
1635                         add_call_dest(file, insn, reloc->sym, true);
1636                         continue;
1637                 } else if (reloc->sym->sec->idx) {
1638                         dest_sec = reloc->sym->sec;
1639                         dest_off = reloc->sym->sym.st_value +
1640                                    arch_dest_reloc_offset(reloc_addend(reloc));
1641                 } else {
1642                         /* non-func asm code jumping to another file */
1643                         continue;
1644                 }
1645
1646                 jump_dest = find_insn(file, dest_sec, dest_off);
1647                 if (!jump_dest) {
1648                         struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off);
1649
1650                         /*
1651                          * This is a special case for retbleed_untrain_ret().
1652                          * It jumps to __x86_return_thunk(), but objtool
1653                          * can't find the thunk's starting RET
1654                          * instruction, because the RET is also in the
1655                          * middle of another instruction.  Objtool only
1656                          * knows about the outer instruction.
1657                          */
1658                         if (sym && sym->embedded_insn) {
1659                                 add_return_call(file, insn, false);
1660                                 continue;
1661                         }
1662
1663                         WARN_INSN(insn, "can't find jump dest instruction at %s+0x%lx",
1664                                   dest_sec->name, dest_off);
1665                         return -1;
1666                 }
1667
1668                 /*
1669                  * An intra-TU jump in retpoline.o might not have a relocation
1670                  * for its jump dest, in which case the above
1671                  * add_{retpoline,return}_call() didn't happen.
1672                  */
1673                 if (jump_dest->sym && jump_dest->offset == jump_dest->sym->offset) {
1674                         if (jump_dest->sym->retpoline_thunk) {
1675                                 add_retpoline_call(file, insn);
1676                                 continue;
1677                         }
1678                         if (jump_dest->sym->return_thunk) {
1679                                 add_return_call(file, insn, true);
1680                                 continue;
1681                         }
1682                 }
1683
1684                 /*
1685                  * Cross-function jump.
1686                  */
1687                 if (insn_func(insn) && insn_func(jump_dest) &&
1688                     insn_func(insn) != insn_func(jump_dest)) {
1689
1690                         /*
1691                          * For GCC 8+, create parent/child links for any cold
1692                          * subfunctions.  This is _mostly_ redundant with a
1693                          * similar initialization in read_symbols().
1694                          *
1695                          * If a function has aliases, we want the *first* such
1696                          * function in the symbol table to be the subfunction's
1697                          * parent.  In that case we overwrite the
1698                          * initialization done in read_symbols().
1699                          *
1700                          * However this code can't completely replace the
1701                          * read_symbols() code because this doesn't detect the
1702                          * case where the parent function's only reference to a
1703                          * subfunction is through a jump table.
1704                          */
1705                         if (!strstr(insn_func(insn)->name, ".cold") &&
1706                             strstr(insn_func(jump_dest)->name, ".cold")) {
1707                                 insn_func(insn)->cfunc = insn_func(jump_dest);
1708                                 insn_func(jump_dest)->pfunc = insn_func(insn);
1709                         }
1710                 }
1711
1712                 if (jump_is_sibling_call(file, insn, jump_dest)) {
1713                         /*
1714                          * Internal sibling call without reloc or with
1715                          * STT_SECTION reloc.
1716                          */
1717                         add_call_dest(file, insn, insn_func(jump_dest), true);
1718                         continue;
1719                 }
1720
1721                 insn->jump_dest = jump_dest;
1722         }
1723
1724         return 0;
1725 }
1726
1727 static struct symbol *find_call_destination(struct section *sec, unsigned long offset)
1728 {
1729         struct symbol *call_dest;
1730
1731         call_dest = find_func_by_offset(sec, offset);
1732         if (!call_dest)
1733                 call_dest = find_symbol_by_offset(sec, offset);
1734
1735         return call_dest;
1736 }
1737
1738 /*
1739  * Find the destination instructions for all calls.
1740  */
1741 static int add_call_destinations(struct objtool_file *file)
1742 {
1743         struct instruction *insn;
1744         unsigned long dest_off;
1745         struct symbol *dest;
1746         struct reloc *reloc;
1747
1748         for_each_insn(file, insn) {
1749                 if (insn->type != INSN_CALL)
1750                         continue;
1751
1752                 reloc = insn_reloc(file, insn);
1753                 if (!reloc) {
1754                         dest_off = arch_jump_destination(insn);
1755                         dest = find_call_destination(insn->sec, dest_off);
1756
1757                         add_call_dest(file, insn, dest, false);
1758
1759                         if (insn->ignore)
1760                                 continue;
1761
1762                         if (!insn_call_dest(insn)) {
1763                                 WARN_INSN(insn, "unannotated intra-function call");
1764                                 return -1;
1765                         }
1766
1767                         if (insn_func(insn) && insn_call_dest(insn)->type != STT_FUNC) {
1768                                 WARN_INSN(insn, "unsupported call to non-function");
1769                                 return -1;
1770                         }
1771
1772                 } else if (reloc->sym->type == STT_SECTION) {
1773                         dest_off = arch_dest_reloc_offset(reloc_addend(reloc));
1774                         dest = find_call_destination(reloc->sym->sec, dest_off);
1775                         if (!dest) {
1776                                 WARN_INSN(insn, "can't find call dest symbol at %s+0x%lx",
1777                                           reloc->sym->sec->name, dest_off);
1778                                 return -1;
1779                         }
1780
1781                         add_call_dest(file, insn, dest, false);
1782
1783                 } else if (reloc->sym->retpoline_thunk) {
1784                         add_retpoline_call(file, insn);
1785
1786                 } else
1787                         add_call_dest(file, insn, reloc->sym, false);
1788         }
1789
1790         return 0;
1791 }
1792
1793 /*
1794  * The .alternatives section requires some extra special care over and above
1795  * other special sections because alternatives are patched in place.
1796  */
1797 static int handle_group_alt(struct objtool_file *file,
1798                             struct special_alt *special_alt,
1799                             struct instruction *orig_insn,
1800                             struct instruction **new_insn)
1801 {
1802         struct instruction *last_new_insn = NULL, *insn, *nop = NULL;
1803         struct alt_group *orig_alt_group, *new_alt_group;
1804         unsigned long dest_off;
1805
1806         orig_alt_group = orig_insn->alt_group;
1807         if (!orig_alt_group) {
1808                 struct instruction *last_orig_insn = NULL;
1809
1810                 orig_alt_group = malloc(sizeof(*orig_alt_group));
1811                 if (!orig_alt_group) {
1812                         WARN("malloc failed");
1813                         return -1;
1814                 }
1815                 orig_alt_group->cfi = calloc(special_alt->orig_len,
1816                                              sizeof(struct cfi_state *));
1817                 if (!orig_alt_group->cfi) {
1818                         WARN("calloc failed");
1819                         return -1;
1820                 }
1821
1822                 insn = orig_insn;
1823                 sec_for_each_insn_from(file, insn) {
1824                         if (insn->offset >= special_alt->orig_off + special_alt->orig_len)
1825                                 break;
1826
1827                         insn->alt_group = orig_alt_group;
1828                         last_orig_insn = insn;
1829                 }
1830                 orig_alt_group->orig_group = NULL;
1831                 orig_alt_group->first_insn = orig_insn;
1832                 orig_alt_group->last_insn = last_orig_insn;
1833                 orig_alt_group->nop = NULL;
1834         } else {
1835                 if (orig_alt_group->last_insn->offset + orig_alt_group->last_insn->len -
1836                     orig_alt_group->first_insn->offset != special_alt->orig_len) {
1837                         WARN_INSN(orig_insn, "weirdly overlapping alternative! %ld != %d",
1838                                   orig_alt_group->last_insn->offset +
1839                                   orig_alt_group->last_insn->len -
1840                                   orig_alt_group->first_insn->offset,
1841                                   special_alt->orig_len);
1842                         return -1;
1843                 }
1844         }
1845
1846         new_alt_group = malloc(sizeof(*new_alt_group));
1847         if (!new_alt_group) {
1848                 WARN("malloc failed");
1849                 return -1;
1850         }
1851
1852         if (special_alt->new_len < special_alt->orig_len) {
1853                 /*
1854                  * Insert a fake nop at the end to make the replacement
1855                  * alt_group the same size as the original.  This is needed to
1856                  * allow propagate_alt_cfi() to do its magic.  When the last
1857                  * instruction affects the stack, the instruction after it (the
1858                  * nop) will propagate the new state to the shared CFI array.
1859                  */
1860                 nop = malloc(sizeof(*nop));
1861                 if (!nop) {
1862                         WARN("malloc failed");
1863                         return -1;
1864                 }
1865                 memset(nop, 0, sizeof(*nop));
1866
1867                 nop->sec = special_alt->new_sec;
1868                 nop->offset = special_alt->new_off + special_alt->new_len;
1869                 nop->len = special_alt->orig_len - special_alt->new_len;
1870                 nop->type = INSN_NOP;
1871                 nop->sym = orig_insn->sym;
1872                 nop->alt_group = new_alt_group;
1873                 nop->ignore = orig_insn->ignore_alts;
1874         }
1875
1876         if (!special_alt->new_len) {
1877                 *new_insn = nop;
1878                 goto end;
1879         }
1880
1881         insn = *new_insn;
1882         sec_for_each_insn_from(file, insn) {
1883                 struct reloc *alt_reloc;
1884
1885                 if (insn->offset >= special_alt->new_off + special_alt->new_len)
1886                         break;
1887
1888                 last_new_insn = insn;
1889
1890                 insn->ignore = orig_insn->ignore_alts;
1891                 insn->sym = orig_insn->sym;
1892                 insn->alt_group = new_alt_group;
1893
1894                 /*
1895                  * Since alternative replacement code is copy/pasted by the
1896                  * kernel after applying relocations, generally such code can't
1897                  * have relative-address relocation references to outside the
1898                  * .altinstr_replacement section, unless the arch's
1899                  * alternatives code can adjust the relative offsets
1900                  * accordingly.
1901                  */
1902                 alt_reloc = insn_reloc(file, insn);
1903                 if (alt_reloc && arch_pc_relative_reloc(alt_reloc) &&
1904                     !arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
1905
1906                         WARN_INSN(insn, "unsupported relocation in alternatives section");
1907                         return -1;
1908                 }
1909
1910                 if (!is_static_jump(insn))
1911                         continue;
1912
1913                 if (!insn->immediate)
1914                         continue;
1915
1916                 dest_off = arch_jump_destination(insn);
1917                 if (dest_off == special_alt->new_off + special_alt->new_len) {
1918                         insn->jump_dest = next_insn_same_sec(file, orig_alt_group->last_insn);
1919                         if (!insn->jump_dest) {
1920                                 WARN_INSN(insn, "can't find alternative jump destination");
1921                                 return -1;
1922                         }
1923                 }
1924         }
1925
1926         if (!last_new_insn) {
1927                 WARN_FUNC("can't find last new alternative instruction",
1928                           special_alt->new_sec, special_alt->new_off);
1929                 return -1;
1930         }
1931
1932 end:
1933         new_alt_group->orig_group = orig_alt_group;
1934         new_alt_group->first_insn = *new_insn;
1935         new_alt_group->last_insn = last_new_insn;
1936         new_alt_group->nop = nop;
1937         new_alt_group->cfi = orig_alt_group->cfi;
1938         return 0;
1939 }
1940
1941 /*
1942  * A jump table entry can either convert a nop to a jump or a jump to a nop.
1943  * If the original instruction is a jump, make the alt entry an effective nop
1944  * by just skipping the original instruction.
1945  */
1946 static int handle_jump_alt(struct objtool_file *file,
1947                            struct special_alt *special_alt,
1948                            struct instruction *orig_insn,
1949                            struct instruction **new_insn)
1950 {
1951         if (orig_insn->type != INSN_JUMP_UNCONDITIONAL &&
1952             orig_insn->type != INSN_NOP) {
1953
1954                 WARN_INSN(orig_insn, "unsupported instruction at jump label");
1955                 return -1;
1956         }
1957
1958         if (opts.hack_jump_label && special_alt->key_addend & 2) {
1959                 struct reloc *reloc = insn_reloc(file, orig_insn);
1960
1961                 if (reloc)
1962                         set_reloc_type(file->elf, reloc, R_NONE);
1963                 elf_write_insn(file->elf, orig_insn->sec,
1964                                orig_insn->offset, orig_insn->len,
1965                                arch_nop_insn(orig_insn->len));
1966                 orig_insn->type = INSN_NOP;
1967         }
1968
1969         if (orig_insn->type == INSN_NOP) {
1970                 if (orig_insn->len == 2)
1971                         file->jl_nop_short++;
1972                 else
1973                         file->jl_nop_long++;
1974
1975                 return 0;
1976         }
1977
1978         if (orig_insn->len == 2)
1979                 file->jl_short++;
1980         else
1981                 file->jl_long++;
1982
1983         *new_insn = next_insn_same_sec(file, orig_insn);
1984         return 0;
1985 }
1986
1987 /*
1988  * Read all the special sections which have alternate instructions which can be
1989  * patched in or redirected to at runtime.  Each instruction having alternate
1990  * instruction(s) has them added to its insn->alts list, which will be
1991  * traversed in validate_branch().
1992  */
1993 static int add_special_section_alts(struct objtool_file *file)
1994 {
1995         struct list_head special_alts;
1996         struct instruction *orig_insn, *new_insn;
1997         struct special_alt *special_alt, *tmp;
1998         struct alternative *alt;
1999         int ret;
2000
2001         ret = special_get_alts(file->elf, &special_alts);
2002         if (ret)
2003                 return ret;
2004
2005         list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
2006
2007                 orig_insn = find_insn(file, special_alt->orig_sec,
2008                                       special_alt->orig_off);
2009                 if (!orig_insn) {
2010                         WARN_FUNC("special: can't find orig instruction",
2011                                   special_alt->orig_sec, special_alt->orig_off);
2012                         ret = -1;
2013                         goto out;
2014                 }
2015
2016                 new_insn = NULL;
2017                 if (!special_alt->group || special_alt->new_len) {
2018                         new_insn = find_insn(file, special_alt->new_sec,
2019                                              special_alt->new_off);
2020                         if (!new_insn) {
2021                                 WARN_FUNC("special: can't find new instruction",
2022                                           special_alt->new_sec,
2023                                           special_alt->new_off);
2024                                 ret = -1;
2025                                 goto out;
2026                         }
2027                 }
2028
2029                 if (special_alt->group) {
2030                         if (!special_alt->orig_len) {
2031                                 WARN_INSN(orig_insn, "empty alternative entry");
2032                                 continue;
2033                         }
2034
2035                         ret = handle_group_alt(file, special_alt, orig_insn,
2036                                                &new_insn);
2037                         if (ret)
2038                                 goto out;
2039                 } else if (special_alt->jump_or_nop) {
2040                         ret = handle_jump_alt(file, special_alt, orig_insn,
2041                                               &new_insn);
2042                         if (ret)
2043                                 goto out;
2044                 }
2045
2046                 alt = malloc(sizeof(*alt));
2047                 if (!alt) {
2048                         WARN("malloc failed");
2049                         ret = -1;
2050                         goto out;
2051                 }
2052
2053                 alt->insn = new_insn;
2054                 alt->skip_orig = special_alt->skip_orig;
2055                 orig_insn->ignore_alts |= special_alt->skip_alt;
2056                 alt->next = orig_insn->alts;
2057                 orig_insn->alts = alt;
2058
2059                 list_del(&special_alt->list);
2060                 free(special_alt);
2061         }
2062
2063         if (opts.stats) {
2064                 printf("jl\\\tNOP\tJMP\n");
2065                 printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short);
2066                 printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long);
2067         }
2068
2069 out:
2070         return ret;
2071 }
2072
2073 static int add_jump_table(struct objtool_file *file, struct instruction *insn,
2074                           struct reloc *next_table)
2075 {
2076         struct symbol *pfunc = insn_func(insn)->pfunc;
2077         struct reloc *table = insn_jump_table(insn);
2078         struct instruction *dest_insn;
2079         unsigned int prev_offset = 0;
2080         struct reloc *reloc = table;
2081         struct alternative *alt;
2082
2083         /*
2084          * Each @reloc is a switch table relocation which points to the target
2085          * instruction.
2086          */
2087         for_each_reloc_from(table->sec, reloc) {
2088
2089                 /* Check for the end of the table: */
2090                 if (reloc != table && reloc == next_table)
2091                         break;
2092
2093                 /* Make sure the table entries are consecutive: */
2094                 if (prev_offset && reloc_offset(reloc) != prev_offset + 8)
2095                         break;
2096
2097                 /* Detect function pointers from contiguous objects: */
2098                 if (reloc->sym->sec == pfunc->sec &&
2099                     reloc_addend(reloc) == pfunc->offset)
2100                         break;
2101
2102                 dest_insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2103                 if (!dest_insn)
2104                         break;
2105
2106                 /* Make sure the destination is in the same function: */
2107                 if (!insn_func(dest_insn) || insn_func(dest_insn)->pfunc != pfunc)
2108                         break;
2109
2110                 alt = malloc(sizeof(*alt));
2111                 if (!alt) {
2112                         WARN("malloc failed");
2113                         return -1;
2114                 }
2115
2116                 alt->insn = dest_insn;
2117                 alt->next = insn->alts;
2118                 insn->alts = alt;
2119                 prev_offset = reloc_offset(reloc);
2120         }
2121
2122         if (!prev_offset) {
2123                 WARN_INSN(insn, "can't find switch jump table");
2124                 return -1;
2125         }
2126
2127         return 0;
2128 }
2129
2130 /*
2131  * find_jump_table() - Given a dynamic jump, find the switch jump table
2132  * associated with it.
2133  */
2134 static struct reloc *find_jump_table(struct objtool_file *file,
2135                                       struct symbol *func,
2136                                       struct instruction *insn)
2137 {
2138         struct reloc *table_reloc;
2139         struct instruction *dest_insn, *orig_insn = insn;
2140
2141         /*
2142          * Backward search using the @first_jump_src links, these help avoid
2143          * much of the 'in between' code. Which avoids us getting confused by
2144          * it.
2145          */
2146         for (;
2147              insn && insn_func(insn) && insn_func(insn)->pfunc == func;
2148              insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
2149
2150                 if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
2151                         break;
2152
2153                 /* allow small jumps within the range */
2154                 if (insn->type == INSN_JUMP_UNCONDITIONAL &&
2155                     insn->jump_dest &&
2156                     (insn->jump_dest->offset <= insn->offset ||
2157                      insn->jump_dest->offset > orig_insn->offset))
2158                     break;
2159
2160                 table_reloc = arch_find_switch_table(file, insn);
2161                 if (!table_reloc)
2162                         continue;
2163                 dest_insn = find_insn(file, table_reloc->sym->sec, reloc_addend(table_reloc));
2164                 if (!dest_insn || !insn_func(dest_insn) || insn_func(dest_insn)->pfunc != func)
2165                         continue;
2166
2167                 return table_reloc;
2168         }
2169
2170         return NULL;
2171 }
2172
2173 /*
2174  * First pass: Mark the head of each jump table so that in the next pass,
2175  * we know when a given jump table ends and the next one starts.
2176  */
2177 static void mark_func_jump_tables(struct objtool_file *file,
2178                                     struct symbol *func)
2179 {
2180         struct instruction *insn, *last = NULL;
2181         struct reloc *reloc;
2182
2183         func_for_each_insn(file, func, insn) {
2184                 if (!last)
2185                         last = insn;
2186
2187                 /*
2188                  * Store back-pointers for unconditional forward jumps such
2189                  * that find_jump_table() can back-track using those and
2190                  * avoid some potentially confusing code.
2191                  */
2192                 if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest &&
2193                     insn->offset > last->offset &&
2194                     insn->jump_dest->offset > insn->offset &&
2195                     !insn->jump_dest->first_jump_src) {
2196
2197                         insn->jump_dest->first_jump_src = insn;
2198                         last = insn->jump_dest;
2199                 }
2200
2201                 if (insn->type != INSN_JUMP_DYNAMIC)
2202                         continue;
2203
2204                 reloc = find_jump_table(file, func, insn);
2205                 if (reloc)
2206                         insn->_jump_table = reloc;
2207         }
2208 }
2209
2210 static int add_func_jump_tables(struct objtool_file *file,
2211                                   struct symbol *func)
2212 {
2213         struct instruction *insn, *insn_t1 = NULL, *insn_t2;
2214         int ret = 0;
2215
2216         func_for_each_insn(file, func, insn) {
2217                 if (!insn_jump_table(insn))
2218                         continue;
2219
2220                 if (!insn_t1) {
2221                         insn_t1 = insn;
2222                         continue;
2223                 }
2224
2225                 insn_t2 = insn;
2226
2227                 ret = add_jump_table(file, insn_t1, insn_jump_table(insn_t2));
2228                 if (ret)
2229                         return ret;
2230
2231                 insn_t1 = insn_t2;
2232         }
2233
2234         if (insn_t1)
2235                 ret = add_jump_table(file, insn_t1, NULL);
2236
2237         return ret;
2238 }
2239
2240 /*
2241  * For some switch statements, gcc generates a jump table in the .rodata
2242  * section which contains a list of addresses within the function to jump to.
2243  * This finds these jump tables and adds them to the insn->alts lists.
2244  */
2245 static int add_jump_table_alts(struct objtool_file *file)
2246 {
2247         struct symbol *func;
2248         int ret;
2249
2250         if (!file->rodata)
2251                 return 0;
2252
2253         for_each_sym(file, func) {
2254                 if (func->type != STT_FUNC)
2255                         continue;
2256
2257                 mark_func_jump_tables(file, func);
2258                 ret = add_func_jump_tables(file, func);
2259                 if (ret)
2260                         return ret;
2261         }
2262
2263         return 0;
2264 }
2265
2266 static void set_func_state(struct cfi_state *state)
2267 {
2268         state->cfa = initial_func_cfi.cfa;
2269         memcpy(&state->regs, &initial_func_cfi.regs,
2270                CFI_NUM_REGS * sizeof(struct cfi_reg));
2271         state->stack_size = initial_func_cfi.cfa.offset;
2272         state->type = UNWIND_HINT_TYPE_CALL;
2273 }
2274
2275 static int read_unwind_hints(struct objtool_file *file)
2276 {
2277         struct cfi_state cfi = init_cfi;
2278         struct section *sec;
2279         struct unwind_hint *hint;
2280         struct instruction *insn;
2281         struct reloc *reloc;
2282         unsigned long offset;
2283         int i;
2284
2285         sec = find_section_by_name(file->elf, ".discard.unwind_hints");
2286         if (!sec)
2287                 return 0;
2288
2289         if (!sec->rsec) {
2290                 WARN("missing .rela.discard.unwind_hints section");
2291                 return -1;
2292         }
2293
2294         if (sec->sh.sh_size % sizeof(struct unwind_hint)) {
2295                 WARN("struct unwind_hint size mismatch");
2296                 return -1;
2297         }
2298
2299         file->hints = true;
2300
2301         for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) {
2302                 hint = (struct unwind_hint *)sec->data->d_buf + i;
2303
2304                 reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint));
2305                 if (!reloc) {
2306                         WARN("can't find reloc for unwind_hints[%d]", i);
2307                         return -1;
2308                 }
2309
2310                 if (reloc->sym->type == STT_SECTION) {
2311                         offset = reloc_addend(reloc);
2312                 } else if (reloc->sym->local_label) {
2313                         offset = reloc->sym->offset;
2314                 } else {
2315                         WARN("unexpected relocation symbol type in %s", sec->rsec->name);
2316                         return -1;
2317                 }
2318
2319                 insn = find_insn(file, reloc->sym->sec, offset);
2320                 if (!insn) {
2321                         WARN("can't find insn for unwind_hints[%d]", i);
2322                         return -1;
2323                 }
2324
2325                 insn->hint = true;
2326
2327                 if (hint->type == UNWIND_HINT_TYPE_UNDEFINED) {
2328                         insn->cfi = &force_undefined_cfi;
2329                         continue;
2330                 }
2331
2332                 if (hint->type == UNWIND_HINT_TYPE_SAVE) {
2333                         insn->hint = false;
2334                         insn->save = true;
2335                         continue;
2336                 }
2337
2338                 if (hint->type == UNWIND_HINT_TYPE_RESTORE) {
2339                         insn->restore = true;
2340                         continue;
2341                 }
2342
2343                 if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) {
2344                         struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset);
2345
2346                         if (sym && sym->bind == STB_GLOBAL) {
2347                                 if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) {
2348                                         WARN_INSN(insn, "UNWIND_HINT_IRET_REGS without ENDBR");
2349                                 }
2350                         }
2351                 }
2352
2353                 if (hint->type == UNWIND_HINT_TYPE_FUNC) {
2354                         insn->cfi = &func_cfi;
2355                         continue;
2356                 }
2357
2358                 if (insn->cfi)
2359                         cfi = *(insn->cfi);
2360
2361                 if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) {
2362                         WARN_INSN(insn, "unsupported unwind_hint sp base reg %d", hint->sp_reg);
2363                         return -1;
2364                 }
2365
2366                 cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset);
2367                 cfi.type = hint->type;
2368                 cfi.signal = hint->signal;
2369
2370                 insn->cfi = cfi_hash_find_or_add(&cfi);
2371         }
2372
2373         return 0;
2374 }
2375
2376 static int read_noendbr_hints(struct objtool_file *file)
2377 {
2378         struct instruction *insn;
2379         struct section *rsec;
2380         struct reloc *reloc;
2381
2382         rsec = find_section_by_name(file->elf, ".rela.discard.noendbr");
2383         if (!rsec)
2384                 return 0;
2385
2386         for_each_reloc(rsec, reloc) {
2387                 insn = find_insn(file, reloc->sym->sec,
2388                                  reloc->sym->offset + reloc_addend(reloc));
2389                 if (!insn) {
2390                         WARN("bad .discard.noendbr entry");
2391                         return -1;
2392                 }
2393
2394                 insn->noendbr = 1;
2395         }
2396
2397         return 0;
2398 }
2399
2400 static int read_retpoline_hints(struct objtool_file *file)
2401 {
2402         struct section *rsec;
2403         struct instruction *insn;
2404         struct reloc *reloc;
2405
2406         rsec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe");
2407         if (!rsec)
2408                 return 0;
2409
2410         for_each_reloc(rsec, reloc) {
2411                 if (reloc->sym->type != STT_SECTION) {
2412                         WARN("unexpected relocation symbol type in %s", rsec->name);
2413                         return -1;
2414                 }
2415
2416                 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2417                 if (!insn) {
2418                         WARN("bad .discard.retpoline_safe entry");
2419                         return -1;
2420                 }
2421
2422                 if (insn->type != INSN_JUMP_DYNAMIC &&
2423                     insn->type != INSN_CALL_DYNAMIC &&
2424                     insn->type != INSN_RETURN &&
2425                     insn->type != INSN_NOP) {
2426                         WARN_INSN(insn, "retpoline_safe hint not an indirect jump/call/ret/nop");
2427                         return -1;
2428                 }
2429
2430                 insn->retpoline_safe = true;
2431         }
2432
2433         return 0;
2434 }
2435
2436 static int read_instr_hints(struct objtool_file *file)
2437 {
2438         struct section *rsec;
2439         struct instruction *insn;
2440         struct reloc *reloc;
2441
2442         rsec = find_section_by_name(file->elf, ".rela.discard.instr_end");
2443         if (!rsec)
2444                 return 0;
2445
2446         for_each_reloc(rsec, reloc) {
2447                 if (reloc->sym->type != STT_SECTION) {
2448                         WARN("unexpected relocation symbol type in %s", rsec->name);
2449                         return -1;
2450                 }
2451
2452                 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2453                 if (!insn) {
2454                         WARN("bad .discard.instr_end entry");
2455                         return -1;
2456                 }
2457
2458                 insn->instr--;
2459         }
2460
2461         rsec = find_section_by_name(file->elf, ".rela.discard.instr_begin");
2462         if (!rsec)
2463                 return 0;
2464
2465         for_each_reloc(rsec, reloc) {
2466                 if (reloc->sym->type != STT_SECTION) {
2467                         WARN("unexpected relocation symbol type in %s", rsec->name);
2468                         return -1;
2469                 }
2470
2471                 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2472                 if (!insn) {
2473                         WARN("bad .discard.instr_begin entry");
2474                         return -1;
2475                 }
2476
2477                 insn->instr++;
2478         }
2479
2480         return 0;
2481 }
2482
2483 static int read_validate_unret_hints(struct objtool_file *file)
2484 {
2485         struct section *rsec;
2486         struct instruction *insn;
2487         struct reloc *reloc;
2488
2489         rsec = find_section_by_name(file->elf, ".rela.discard.validate_unret");
2490         if (!rsec)
2491                 return 0;
2492
2493         for_each_reloc(rsec, reloc) {
2494                 if (reloc->sym->type != STT_SECTION) {
2495                         WARN("unexpected relocation symbol type in %s", rsec->name);
2496                         return -1;
2497                 }
2498
2499                 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2500                 if (!insn) {
2501                         WARN("bad .discard.instr_end entry");
2502                         return -1;
2503                 }
2504                 insn->unret = 1;
2505         }
2506
2507         return 0;
2508 }
2509
2510
2511 static int read_intra_function_calls(struct objtool_file *file)
2512 {
2513         struct instruction *insn;
2514         struct section *rsec;
2515         struct reloc *reloc;
2516
2517         rsec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls");
2518         if (!rsec)
2519                 return 0;
2520
2521         for_each_reloc(rsec, reloc) {
2522                 unsigned long dest_off;
2523
2524                 if (reloc->sym->type != STT_SECTION) {
2525                         WARN("unexpected relocation symbol type in %s",
2526                              rsec->name);
2527                         return -1;
2528                 }
2529
2530                 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2531                 if (!insn) {
2532                         WARN("bad .discard.intra_function_call entry");
2533                         return -1;
2534                 }
2535
2536                 if (insn->type != INSN_CALL) {
2537                         WARN_INSN(insn, "intra_function_call not a direct call");
2538                         return -1;
2539                 }
2540
2541                 /*
2542                  * Treat intra-function CALLs as JMPs, but with a stack_op.
2543                  * See add_call_destinations(), which strips stack_ops from
2544                  * normal CALLs.
2545                  */
2546                 insn->type = INSN_JUMP_UNCONDITIONAL;
2547
2548                 dest_off = arch_jump_destination(insn);
2549                 insn->jump_dest = find_insn(file, insn->sec, dest_off);
2550                 if (!insn->jump_dest) {
2551                         WARN_INSN(insn, "can't find call dest at %s+0x%lx",
2552                                   insn->sec->name, dest_off);
2553                         return -1;
2554                 }
2555         }
2556
2557         return 0;
2558 }
2559
2560 /*
2561  * Return true if name matches an instrumentation function, where calls to that
2562  * function from noinstr code can safely be removed, but compilers won't do so.
2563  */
2564 static bool is_profiling_func(const char *name)
2565 {
2566         /*
2567          * Many compilers cannot disable KCOV with a function attribute.
2568          */
2569         if (!strncmp(name, "__sanitizer_cov_", 16))
2570                 return true;
2571
2572         /*
2573          * Some compilers currently do not remove __tsan_func_entry/exit nor
2574          * __tsan_atomic_signal_fence (used for barrier instrumentation) with
2575          * the __no_sanitize_thread attribute, remove them. Once the kernel's
2576          * minimum Clang version is 14.0, this can be removed.
2577          */
2578         if (!strncmp(name, "__tsan_func_", 12) ||
2579             !strcmp(name, "__tsan_atomic_signal_fence"))
2580                 return true;
2581
2582         return false;
2583 }
2584
2585 static int classify_symbols(struct objtool_file *file)
2586 {
2587         struct symbol *func;
2588
2589         for_each_sym(file, func) {
2590                 if (func->type == STT_NOTYPE && strstarts(func->name, ".L"))
2591                         func->local_label = true;
2592
2593                 if (func->bind != STB_GLOBAL)
2594                         continue;
2595
2596                 if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR,
2597                              strlen(STATIC_CALL_TRAMP_PREFIX_STR)))
2598                         func->static_call_tramp = true;
2599
2600                 if (arch_is_retpoline(func))
2601                         func->retpoline_thunk = true;
2602
2603                 if (arch_is_rethunk(func))
2604                         func->return_thunk = true;
2605
2606                 if (arch_is_embedded_insn(func))
2607                         func->embedded_insn = true;
2608
2609                 if (arch_ftrace_match(func->name))
2610                         func->fentry = true;
2611
2612                 if (is_profiling_func(func->name))
2613                         func->profiling_func = true;
2614         }
2615
2616         return 0;
2617 }
2618
2619 static void mark_rodata(struct objtool_file *file)
2620 {
2621         struct section *sec;
2622         bool found = false;
2623
2624         /*
2625          * Search for the following rodata sections, each of which can
2626          * potentially contain jump tables:
2627          *
2628          * - .rodata: can contain GCC switch tables
2629          * - .rodata.<func>: same, if -fdata-sections is being used
2630          * - .rodata..c_jump_table: contains C annotated jump tables
2631          *
2632          * .rodata.str1.* sections are ignored; they don't contain jump tables.
2633          */
2634         for_each_sec(file, sec) {
2635                 if (!strncmp(sec->name, ".rodata", 7) &&
2636                     !strstr(sec->name, ".str1.")) {
2637                         sec->rodata = true;
2638                         found = true;
2639                 }
2640         }
2641
2642         file->rodata = found;
2643 }
2644
2645 static int decode_sections(struct objtool_file *file)
2646 {
2647         int ret;
2648
2649         mark_rodata(file);
2650
2651         ret = init_pv_ops(file);
2652         if (ret)
2653                 return ret;
2654
2655         /*
2656          * Must be before add_{jump_call}_destination.
2657          */
2658         ret = classify_symbols(file);
2659         if (ret)
2660                 return ret;
2661
2662         ret = decode_instructions(file);
2663         if (ret)
2664                 return ret;
2665
2666         add_ignores(file);
2667         add_uaccess_safe(file);
2668
2669         ret = add_ignore_alternatives(file);
2670         if (ret)
2671                 return ret;
2672
2673         /*
2674          * Must be before read_unwind_hints() since that needs insn->noendbr.
2675          */
2676         ret = read_noendbr_hints(file);
2677         if (ret)
2678                 return ret;
2679
2680         /*
2681          * Must be before add_jump_destinations(), which depends on 'func'
2682          * being set for alternatives, to enable proper sibling call detection.
2683          */
2684         if (opts.stackval || opts.orc || opts.uaccess || opts.noinstr) {
2685                 ret = add_special_section_alts(file);
2686                 if (ret)
2687                         return ret;
2688         }
2689
2690         ret = add_jump_destinations(file);
2691         if (ret)
2692                 return ret;
2693
2694         /*
2695          * Must be before add_call_destination(); it changes INSN_CALL to
2696          * INSN_JUMP.
2697          */
2698         ret = read_intra_function_calls(file);
2699         if (ret)
2700                 return ret;
2701
2702         ret = add_call_destinations(file);
2703         if (ret)
2704                 return ret;
2705
2706         /*
2707          * Must be after add_call_destinations() such that it can override
2708          * dead_end_function() marks.
2709          */
2710         ret = add_dead_ends(file);
2711         if (ret)
2712                 return ret;
2713
2714         ret = add_jump_table_alts(file);
2715         if (ret)
2716                 return ret;
2717
2718         ret = read_unwind_hints(file);
2719         if (ret)
2720                 return ret;
2721
2722         ret = read_retpoline_hints(file);
2723         if (ret)
2724                 return ret;
2725
2726         ret = read_instr_hints(file);
2727         if (ret)
2728                 return ret;
2729
2730         ret = read_validate_unret_hints(file);
2731         if (ret)
2732                 return ret;
2733
2734         return 0;
2735 }
2736
2737 static bool is_special_call(struct instruction *insn)
2738 {
2739         if (insn->type == INSN_CALL) {
2740                 struct symbol *dest = insn_call_dest(insn);
2741
2742                 if (!dest)
2743                         return false;
2744
2745                 if (dest->fentry || dest->embedded_insn)
2746                         return true;
2747         }
2748
2749         return false;
2750 }
2751
2752 static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state)
2753 {
2754         struct cfi_state *cfi = &state->cfi;
2755         int i;
2756
2757         if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap)
2758                 return true;
2759
2760         if (cfi->cfa.offset != initial_func_cfi.cfa.offset)
2761                 return true;
2762
2763         if (cfi->stack_size != initial_func_cfi.cfa.offset)
2764                 return true;
2765
2766         for (i = 0; i < CFI_NUM_REGS; i++) {
2767                 if (cfi->regs[i].base != initial_func_cfi.regs[i].base ||
2768                     cfi->regs[i].offset != initial_func_cfi.regs[i].offset)
2769                         return true;
2770         }
2771
2772         return false;
2773 }
2774
2775 static bool check_reg_frame_pos(const struct cfi_reg *reg,
2776                                 int expected_offset)
2777 {
2778         return reg->base == CFI_CFA &&
2779                reg->offset == expected_offset;
2780 }
2781
2782 static bool has_valid_stack_frame(struct insn_state *state)
2783 {
2784         struct cfi_state *cfi = &state->cfi;
2785
2786         if (cfi->cfa.base == CFI_BP &&
2787             check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) &&
2788             check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8))
2789                 return true;
2790
2791         if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP)
2792                 return true;
2793
2794         return false;
2795 }
2796
2797 static int update_cfi_state_regs(struct instruction *insn,
2798                                   struct cfi_state *cfi,
2799                                   struct stack_op *op)
2800 {
2801         struct cfi_reg *cfa = &cfi->cfa;
2802
2803         if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT)
2804                 return 0;
2805
2806         /* push */
2807         if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF)
2808                 cfa->offset += 8;
2809
2810         /* pop */
2811         if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF)
2812                 cfa->offset -= 8;
2813
2814         /* add immediate to sp */
2815         if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD &&
2816             op->dest.reg == CFI_SP && op->src.reg == CFI_SP)
2817                 cfa->offset -= op->src.offset;
2818
2819         return 0;
2820 }
2821
2822 static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset)
2823 {
2824         if (arch_callee_saved_reg(reg) &&
2825             cfi->regs[reg].base == CFI_UNDEFINED) {
2826                 cfi->regs[reg].base = base;
2827                 cfi->regs[reg].offset = offset;
2828         }
2829 }
2830
2831 static void restore_reg(struct cfi_state *cfi, unsigned char reg)
2832 {
2833         cfi->regs[reg].base = initial_func_cfi.regs[reg].base;
2834         cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset;
2835 }
2836
2837 /*
2838  * A note about DRAP stack alignment:
2839  *
2840  * GCC has the concept of a DRAP register, which is used to help keep track of
2841  * the stack pointer when aligning the stack.  r10 or r13 is used as the DRAP
2842  * register.  The typical DRAP pattern is:
2843  *
2844  *   4c 8d 54 24 08             lea    0x8(%rsp),%r10
2845  *   48 83 e4 c0                and    $0xffffffffffffffc0,%rsp
2846  *   41 ff 72 f8                pushq  -0x8(%r10)
2847  *   55                         push   %rbp
2848  *   48 89 e5                   mov    %rsp,%rbp
2849  *                              (more pushes)
2850  *   41 52                      push   %r10
2851  *                              ...
2852  *   41 5a                      pop    %r10
2853  *                              (more pops)
2854  *   5d                         pop    %rbp
2855  *   49 8d 62 f8                lea    -0x8(%r10),%rsp
2856  *   c3                         retq
2857  *
2858  * There are some variations in the epilogues, like:
2859  *
2860  *   5b                         pop    %rbx
2861  *   41 5a                      pop    %r10
2862  *   41 5c                      pop    %r12
2863  *   41 5d                      pop    %r13
2864  *   41 5e                      pop    %r14
2865  *   c9                         leaveq
2866  *   49 8d 62 f8                lea    -0x8(%r10),%rsp
2867  *   c3                         retq
2868  *
2869  * and:
2870  *
2871  *   4c 8b 55 e8                mov    -0x18(%rbp),%r10
2872  *   48 8b 5d e0                mov    -0x20(%rbp),%rbx
2873  *   4c 8b 65 f0                mov    -0x10(%rbp),%r12
2874  *   4c 8b 6d f8                mov    -0x8(%rbp),%r13
2875  *   c9                         leaveq
2876  *   49 8d 62 f8                lea    -0x8(%r10),%rsp
2877  *   c3                         retq
2878  *
2879  * Sometimes r13 is used as the DRAP register, in which case it's saved and
2880  * restored beforehand:
2881  *
2882  *   41 55                      push   %r13
2883  *   4c 8d 6c 24 10             lea    0x10(%rsp),%r13
2884  *   48 83 e4 f0                and    $0xfffffffffffffff0,%rsp
2885  *                              ...
2886  *   49 8d 65 f0                lea    -0x10(%r13),%rsp
2887  *   41 5d                      pop    %r13
2888  *   c3                         retq
2889  */
2890 static int update_cfi_state(struct instruction *insn,
2891                             struct instruction *next_insn,
2892                             struct cfi_state *cfi, struct stack_op *op)
2893 {
2894         struct cfi_reg *cfa = &cfi->cfa;
2895         struct cfi_reg *regs = cfi->regs;
2896
2897         /* ignore UNWIND_HINT_UNDEFINED regions */
2898         if (cfi->force_undefined)
2899                 return 0;
2900
2901         /* stack operations don't make sense with an undefined CFA */
2902         if (cfa->base == CFI_UNDEFINED) {
2903                 if (insn_func(insn)) {
2904                         WARN_INSN(insn, "undefined stack state");
2905                         return -1;
2906                 }
2907                 return 0;
2908         }
2909
2910         if (cfi->type == UNWIND_HINT_TYPE_REGS ||
2911             cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL)
2912                 return update_cfi_state_regs(insn, cfi, op);
2913
2914         switch (op->dest.type) {
2915
2916         case OP_DEST_REG:
2917                 switch (op->src.type) {
2918
2919                 case OP_SRC_REG:
2920                         if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP &&
2921                             cfa->base == CFI_SP &&
2922                             check_reg_frame_pos(&regs[CFI_BP], -cfa->offset)) {
2923
2924                                 /* mov %rsp, %rbp */
2925                                 cfa->base = op->dest.reg;
2926                                 cfi->bp_scratch = false;
2927                         }
2928
2929                         else if (op->src.reg == CFI_SP &&
2930                                  op->dest.reg == CFI_BP && cfi->drap) {
2931
2932                                 /* drap: mov %rsp, %rbp */
2933                                 regs[CFI_BP].base = CFI_BP;
2934                                 regs[CFI_BP].offset = -cfi->stack_size;
2935                                 cfi->bp_scratch = false;
2936                         }
2937
2938                         else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2939
2940                                 /*
2941                                  * mov %rsp, %reg
2942                                  *
2943                                  * This is needed for the rare case where GCC
2944                                  * does:
2945                                  *
2946                                  *   mov    %rsp, %rax
2947                                  *   ...
2948                                  *   mov    %rax, %rsp
2949                                  */
2950                                 cfi->vals[op->dest.reg].base = CFI_CFA;
2951                                 cfi->vals[op->dest.reg].offset = -cfi->stack_size;
2952                         }
2953
2954                         else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP &&
2955                                  (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) {
2956
2957                                 /*
2958                                  * mov %rbp, %rsp
2959                                  *
2960                                  * Restore the original stack pointer (Clang).
2961                                  */
2962                                 cfi->stack_size = -cfi->regs[CFI_BP].offset;
2963                         }
2964
2965                         else if (op->dest.reg == cfa->base) {
2966
2967                                 /* mov %reg, %rsp */
2968                                 if (cfa->base == CFI_SP &&
2969                                     cfi->vals[op->src.reg].base == CFI_CFA) {
2970
2971                                         /*
2972                                          * This is needed for the rare case
2973                                          * where GCC does something dumb like:
2974                                          *
2975                                          *   lea    0x8(%rsp), %rcx
2976                                          *   ...
2977                                          *   mov    %rcx, %rsp
2978                                          */
2979                                         cfa->offset = -cfi->vals[op->src.reg].offset;
2980                                         cfi->stack_size = cfa->offset;
2981
2982                                 } else if (cfa->base == CFI_SP &&
2983                                            cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2984                                            cfi->vals[op->src.reg].offset == cfa->offset) {
2985
2986                                         /*
2987                                          * Stack swizzle:
2988                                          *
2989                                          * 1: mov %rsp, (%[tos])
2990                                          * 2: mov %[tos], %rsp
2991                                          *    ...
2992                                          * 3: pop %rsp
2993                                          *
2994                                          * Where:
2995                                          *
2996                                          * 1 - places a pointer to the previous
2997                                          *     stack at the Top-of-Stack of the
2998                                          *     new stack.
2999                                          *
3000                                          * 2 - switches to the new stack.
3001                                          *
3002                                          * 3 - pops the Top-of-Stack to restore
3003                                          *     the original stack.
3004                                          *
3005                                          * Note: we set base to SP_INDIRECT
3006                                          * here and preserve offset. Therefore
3007                                          * when the unwinder reaches ToS it
3008                                          * will dereference SP and then add the
3009                                          * offset to find the next frame, IOW:
3010                                          * (%rsp) + offset.
3011                                          */
3012                                         cfa->base = CFI_SP_INDIRECT;
3013
3014                                 } else {
3015                                         cfa->base = CFI_UNDEFINED;
3016                                         cfa->offset = 0;
3017                                 }
3018                         }
3019
3020                         else if (op->dest.reg == CFI_SP &&
3021                                  cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
3022                                  cfi->vals[op->src.reg].offset == cfa->offset) {
3023
3024                                 /*
3025                                  * The same stack swizzle case 2) as above. But
3026                                  * because we can't change cfa->base, case 3)
3027                                  * will become a regular POP. Pretend we're a
3028                                  * PUSH so things don't go unbalanced.
3029                                  */
3030                                 cfi->stack_size += 8;
3031                         }
3032
3033
3034                         break;
3035
3036                 case OP_SRC_ADD:
3037                         if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) {
3038
3039                                 /* add imm, %rsp */
3040                                 cfi->stack_size -= op->src.offset;
3041                                 if (cfa->base == CFI_SP)
3042                                         cfa->offset -= op->src.offset;
3043                                 break;
3044                         }
3045
3046                         if (op->dest.reg == CFI_BP && op->src.reg == CFI_SP &&
3047                             insn->sym->frame_pointer) {
3048                                 /* addi.d fp,sp,imm on LoongArch */
3049                                 if (cfa->base == CFI_SP && cfa->offset == op->src.offset) {
3050                                         cfa->base = CFI_BP;
3051                                         cfa->offset = 0;
3052                                 }
3053                                 break;
3054                         }
3055
3056                         if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) {
3057                                 /* addi.d sp,fp,imm on LoongArch */
3058                                 if (cfa->base == CFI_BP && cfa->offset == 0) {
3059                                         if (insn->sym->frame_pointer) {
3060                                                 cfa->base = CFI_SP;
3061                                                 cfa->offset = -op->src.offset;
3062                                         }
3063                                 } else {
3064                                         /* lea disp(%rbp), %rsp */
3065                                         cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset);
3066                                 }
3067                                 break;
3068                         }
3069
3070                         if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
3071
3072                                 /* drap: lea disp(%rsp), %drap */
3073                                 cfi->drap_reg = op->dest.reg;
3074
3075                                 /*
3076                                  * lea disp(%rsp), %reg
3077                                  *
3078                                  * This is needed for the rare case where GCC
3079                                  * does something dumb like:
3080                                  *
3081                                  *   lea    0x8(%rsp), %rcx
3082                                  *   ...
3083                                  *   mov    %rcx, %rsp
3084                                  */
3085                                 cfi->vals[op->dest.reg].base = CFI_CFA;
3086                                 cfi->vals[op->dest.reg].offset = \
3087                                         -cfi->stack_size + op->src.offset;
3088
3089                                 break;
3090                         }
3091
3092                         if (cfi->drap && op->dest.reg == CFI_SP &&
3093                             op->src.reg == cfi->drap_reg) {
3094
3095                                  /* drap: lea disp(%drap), %rsp */
3096                                 cfa->base = CFI_SP;
3097                                 cfa->offset = cfi->stack_size = -op->src.offset;
3098                                 cfi->drap_reg = CFI_UNDEFINED;
3099                                 cfi->drap = false;
3100                                 break;
3101                         }
3102
3103                         if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) {
3104                                 WARN_INSN(insn, "unsupported stack register modification");
3105                                 return -1;
3106                         }
3107
3108                         break;
3109
3110                 case OP_SRC_AND:
3111                         if (op->dest.reg != CFI_SP ||
3112                             (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) ||
3113                             (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) {
3114                                 WARN_INSN(insn, "unsupported stack pointer realignment");
3115                                 return -1;
3116                         }
3117
3118                         if (cfi->drap_reg != CFI_UNDEFINED) {
3119                                 /* drap: and imm, %rsp */
3120                                 cfa->base = cfi->drap_reg;
3121                                 cfa->offset = cfi->stack_size = 0;
3122                                 cfi->drap = true;
3123                         }
3124
3125                         /*
3126                          * Older versions of GCC (4.8ish) realign the stack
3127                          * without DRAP, with a frame pointer.
3128                          */
3129
3130                         break;
3131
3132                 case OP_SRC_POP:
3133                 case OP_SRC_POPF:
3134                         if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) {
3135
3136                                 /* pop %rsp; # restore from a stack swizzle */
3137                                 cfa->base = CFI_SP;
3138                                 break;
3139                         }
3140
3141                         if (!cfi->drap && op->dest.reg == cfa->base) {
3142
3143                                 /* pop %rbp */
3144                                 cfa->base = CFI_SP;
3145                         }
3146
3147                         if (cfi->drap && cfa->base == CFI_BP_INDIRECT &&
3148                             op->dest.reg == cfi->drap_reg &&
3149                             cfi->drap_offset == -cfi->stack_size) {
3150
3151                                 /* drap: pop %drap */
3152                                 cfa->base = cfi->drap_reg;
3153                                 cfa->offset = 0;
3154                                 cfi->drap_offset = -1;
3155
3156                         } else if (cfi->stack_size == -regs[op->dest.reg].offset) {
3157
3158                                 /* pop %reg */
3159                                 restore_reg(cfi, op->dest.reg);
3160                         }
3161
3162                         cfi->stack_size -= 8;
3163                         if (cfa->base == CFI_SP)
3164                                 cfa->offset -= 8;
3165
3166                         break;
3167
3168                 case OP_SRC_REG_INDIRECT:
3169                         if (!cfi->drap && op->dest.reg == cfa->base &&
3170                             op->dest.reg == CFI_BP) {
3171
3172                                 /* mov disp(%rsp), %rbp */
3173                                 cfa->base = CFI_SP;
3174                                 cfa->offset = cfi->stack_size;
3175                         }
3176
3177                         if (cfi->drap && op->src.reg == CFI_BP &&
3178                             op->src.offset == cfi->drap_offset) {
3179
3180                                 /* drap: mov disp(%rbp), %drap */
3181                                 cfa->base = cfi->drap_reg;
3182                                 cfa->offset = 0;
3183                                 cfi->drap_offset = -1;
3184                         }
3185
3186                         if (cfi->drap && op->src.reg == CFI_BP &&
3187                             op->src.offset == regs[op->dest.reg].offset) {
3188
3189                                 /* drap: mov disp(%rbp), %reg */
3190                                 restore_reg(cfi, op->dest.reg);
3191
3192                         } else if (op->src.reg == cfa->base &&
3193                             op->src.offset == regs[op->dest.reg].offset + cfa->offset) {
3194
3195                                 /* mov disp(%rbp), %reg */
3196                                 /* mov disp(%rsp), %reg */
3197                                 restore_reg(cfi, op->dest.reg);
3198
3199                         } else if (op->src.reg == CFI_SP &&
3200                                    op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) {
3201
3202                                 /* mov disp(%rsp), %reg */
3203                                 restore_reg(cfi, op->dest.reg);
3204                         }
3205
3206                         break;
3207
3208                 default:
3209                         WARN_INSN(insn, "unknown stack-related instruction");
3210                         return -1;
3211                 }
3212
3213                 break;
3214
3215         case OP_DEST_PUSH:
3216         case OP_DEST_PUSHF:
3217                 cfi->stack_size += 8;
3218                 if (cfa->base == CFI_SP)
3219                         cfa->offset += 8;
3220
3221                 if (op->src.type != OP_SRC_REG)
3222                         break;
3223
3224                 if (cfi->drap) {
3225                         if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3226
3227                                 /* drap: push %drap */
3228                                 cfa->base = CFI_BP_INDIRECT;
3229                                 cfa->offset = -cfi->stack_size;
3230
3231                                 /* save drap so we know when to restore it */
3232                                 cfi->drap_offset = -cfi->stack_size;
3233
3234                         } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) {
3235
3236                                 /* drap: push %rbp */
3237                                 cfi->stack_size = 0;
3238
3239                         } else {
3240
3241                                 /* drap: push %reg */
3242                                 save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size);
3243                         }
3244
3245                 } else {
3246
3247                         /* push %reg */
3248                         save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size);
3249                 }
3250
3251                 /* detect when asm code uses rbp as a scratch register */
3252                 if (opts.stackval && insn_func(insn) && op->src.reg == CFI_BP &&
3253                     cfa->base != CFI_BP)
3254                         cfi->bp_scratch = true;
3255                 break;
3256
3257         case OP_DEST_REG_INDIRECT:
3258
3259                 if (cfi->drap) {
3260                         if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3261
3262                                 /* drap: mov %drap, disp(%rbp) */
3263                                 cfa->base = CFI_BP_INDIRECT;
3264                                 cfa->offset = op->dest.offset;
3265
3266                                 /* save drap offset so we know when to restore it */
3267                                 cfi->drap_offset = op->dest.offset;
3268                         } else {
3269
3270                                 /* drap: mov reg, disp(%rbp) */
3271                                 save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset);
3272                         }
3273
3274                 } else if (op->dest.reg == cfa->base) {
3275
3276                         /* mov reg, disp(%rbp) */
3277                         /* mov reg, disp(%rsp) */
3278                         save_reg(cfi, op->src.reg, CFI_CFA,
3279                                  op->dest.offset - cfi->cfa.offset);
3280
3281                 } else if (op->dest.reg == CFI_SP) {
3282
3283                         /* mov reg, disp(%rsp) */
3284                         save_reg(cfi, op->src.reg, CFI_CFA,
3285                                  op->dest.offset - cfi->stack_size);
3286
3287                 } else if (op->src.reg == CFI_SP && op->dest.offset == 0) {
3288
3289                         /* mov %rsp, (%reg); # setup a stack swizzle. */
3290                         cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT;
3291                         cfi->vals[op->dest.reg].offset = cfa->offset;
3292                 }
3293
3294                 break;
3295
3296         case OP_DEST_MEM:
3297                 if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) {
3298                         WARN_INSN(insn, "unknown stack-related memory operation");
3299                         return -1;
3300                 }
3301
3302                 /* pop mem */
3303                 cfi->stack_size -= 8;
3304                 if (cfa->base == CFI_SP)
3305                         cfa->offset -= 8;
3306
3307                 break;
3308
3309         default:
3310                 WARN_INSN(insn, "unknown stack-related instruction");
3311                 return -1;
3312         }
3313
3314         return 0;
3315 }
3316
3317 /*
3318  * The stack layouts of alternatives instructions can sometimes diverge when
3319  * they have stack modifications.  That's fine as long as the potential stack
3320  * layouts don't conflict at any given potential instruction boundary.
3321  *
3322  * Flatten the CFIs of the different alternative code streams (both original
3323  * and replacement) into a single shared CFI array which can be used to detect
3324  * conflicts and nicely feed a linear array of ORC entries to the unwinder.
3325  */
3326 static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn)
3327 {
3328         struct cfi_state **alt_cfi;
3329         int group_off;
3330
3331         if (!insn->alt_group)
3332                 return 0;
3333
3334         if (!insn->cfi) {
3335                 WARN("CFI missing");
3336                 return -1;
3337         }
3338
3339         alt_cfi = insn->alt_group->cfi;
3340         group_off = insn->offset - insn->alt_group->first_insn->offset;
3341
3342         if (!alt_cfi[group_off]) {
3343                 alt_cfi[group_off] = insn->cfi;
3344         } else {
3345                 if (cficmp(alt_cfi[group_off], insn->cfi)) {
3346                         struct alt_group *orig_group = insn->alt_group->orig_group ?: insn->alt_group;
3347                         struct instruction *orig = orig_group->first_insn;
3348                         char *where = offstr(insn->sec, insn->offset);
3349                         WARN_INSN(orig, "stack layout conflict in alternatives: %s", where);
3350                         free(where);
3351                         return -1;
3352                 }
3353         }
3354
3355         return 0;
3356 }
3357
3358 static int handle_insn_ops(struct instruction *insn,
3359                            struct instruction *next_insn,
3360                            struct insn_state *state)
3361 {
3362         struct stack_op *op;
3363
3364         for (op = insn->stack_ops; op; op = op->next) {
3365
3366                 if (update_cfi_state(insn, next_insn, &state->cfi, op))
3367                         return 1;
3368
3369                 if (!insn->alt_group)
3370                         continue;
3371
3372                 if (op->dest.type == OP_DEST_PUSHF) {
3373                         if (!state->uaccess_stack) {
3374                                 state->uaccess_stack = 1;
3375                         } else if (state->uaccess_stack >> 31) {
3376                                 WARN_INSN(insn, "PUSHF stack exhausted");
3377                                 return 1;
3378                         }
3379                         state->uaccess_stack <<= 1;
3380                         state->uaccess_stack  |= state->uaccess;
3381                 }
3382
3383                 if (op->src.type == OP_SRC_POPF) {
3384                         if (state->uaccess_stack) {
3385                                 state->uaccess = state->uaccess_stack & 1;
3386                                 state->uaccess_stack >>= 1;
3387                                 if (state->uaccess_stack == 1)
3388                                         state->uaccess_stack = 0;
3389                         }
3390                 }
3391         }
3392
3393         return 0;
3394 }
3395
3396 static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2)
3397 {
3398         struct cfi_state *cfi1 = insn->cfi;
3399         int i;
3400
3401         if (!cfi1) {
3402                 WARN("CFI missing");
3403                 return false;
3404         }
3405
3406         if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) {
3407
3408                 WARN_INSN(insn, "stack state mismatch: cfa1=%d%+d cfa2=%d%+d",
3409                           cfi1->cfa.base, cfi1->cfa.offset,
3410                           cfi2->cfa.base, cfi2->cfa.offset);
3411
3412         } else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) {
3413                 for (i = 0; i < CFI_NUM_REGS; i++) {
3414                         if (!memcmp(&cfi1->regs[i], &cfi2->regs[i],
3415                                     sizeof(struct cfi_reg)))
3416                                 continue;
3417
3418                         WARN_INSN(insn, "stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d",
3419                                   i, cfi1->regs[i].base, cfi1->regs[i].offset,
3420                                   i, cfi2->regs[i].base, cfi2->regs[i].offset);
3421                         break;
3422                 }
3423
3424         } else if (cfi1->type != cfi2->type) {
3425
3426                 WARN_INSN(insn, "stack state mismatch: type1=%d type2=%d",
3427                           cfi1->type, cfi2->type);
3428
3429         } else if (cfi1->drap != cfi2->drap ||
3430                    (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) ||
3431                    (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) {
3432
3433                 WARN_INSN(insn, "stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)",
3434                           cfi1->drap, cfi1->drap_reg, cfi1->drap_offset,
3435                           cfi2->drap, cfi2->drap_reg, cfi2->drap_offset);
3436
3437         } else
3438                 return true;
3439
3440         return false;
3441 }
3442
3443 static inline bool func_uaccess_safe(struct symbol *func)
3444 {
3445         if (func)
3446                 return func->uaccess_safe;
3447
3448         return false;
3449 }
3450
3451 static inline const char *call_dest_name(struct instruction *insn)
3452 {
3453         static char pvname[19];
3454         struct reloc *reloc;
3455         int idx;
3456
3457         if (insn_call_dest(insn))
3458                 return insn_call_dest(insn)->name;
3459
3460         reloc = insn_reloc(NULL, insn);
3461         if (reloc && !strcmp(reloc->sym->name, "pv_ops")) {
3462                 idx = (reloc_addend(reloc) / sizeof(void *));
3463                 snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx);
3464                 return pvname;
3465         }
3466
3467         return "{dynamic}";
3468 }
3469
3470 static bool pv_call_dest(struct objtool_file *file, struct instruction *insn)
3471 {
3472         struct symbol *target;
3473         struct reloc *reloc;
3474         int idx;
3475
3476         reloc = insn_reloc(file, insn);
3477         if (!reloc || strcmp(reloc->sym->name, "pv_ops"))
3478                 return false;
3479
3480         idx = (arch_dest_reloc_offset(reloc_addend(reloc)) / sizeof(void *));
3481
3482         if (file->pv_ops[idx].clean)
3483                 return true;
3484
3485         file->pv_ops[idx].clean = true;
3486
3487         list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) {
3488                 if (!target->sec->noinstr) {
3489                         WARN("pv_ops[%d]: %s", idx, target->name);
3490                         file->pv_ops[idx].clean = false;
3491                 }
3492         }
3493
3494         return file->pv_ops[idx].clean;
3495 }
3496
3497 static inline bool noinstr_call_dest(struct objtool_file *file,
3498                                      struct instruction *insn,
3499                                      struct symbol *func)
3500 {
3501         /*
3502          * We can't deal with indirect function calls at present;
3503          * assume they're instrumented.
3504          */
3505         if (!func) {
3506                 if (file->pv_ops)
3507                         return pv_call_dest(file, insn);
3508
3509                 return false;
3510         }
3511
3512         /*
3513          * If the symbol is from a noinstr section; we good.
3514          */
3515         if (func->sec->noinstr)
3516                 return true;
3517
3518         /*
3519          * If the symbol is a static_call trampoline, we can't tell.
3520          */
3521         if (func->static_call_tramp)
3522                 return true;
3523
3524         /*
3525          * The __ubsan_handle_*() calls are like WARN(), they only happen when
3526          * something 'BAD' happened. At the risk of taking the machine down,
3527          * let them proceed to get the message out.
3528          */
3529         if (!strncmp(func->name, "__ubsan_handle_", 15))
3530                 return true;
3531
3532         return false;
3533 }
3534
3535 static int validate_call(struct objtool_file *file,
3536                          struct instruction *insn,
3537                          struct insn_state *state)
3538 {
3539         if (state->noinstr && state->instr <= 0 &&
3540             !noinstr_call_dest(file, insn, insn_call_dest(insn))) {
3541                 WARN_INSN(insn, "call to %s() leaves .noinstr.text section", call_dest_name(insn));
3542                 return 1;
3543         }
3544
3545         if (state->uaccess && !func_uaccess_safe(insn_call_dest(insn))) {
3546                 WARN_INSN(insn, "call to %s() with UACCESS enabled", call_dest_name(insn));
3547                 return 1;
3548         }
3549
3550         if (state->df) {
3551                 WARN_INSN(insn, "call to %s() with DF set", call_dest_name(insn));
3552                 return 1;
3553         }
3554
3555         return 0;
3556 }
3557
3558 static int validate_sibling_call(struct objtool_file *file,
3559                                  struct instruction *insn,
3560                                  struct insn_state *state)
3561 {
3562         if (insn_func(insn) && has_modified_stack_frame(insn, state)) {
3563                 WARN_INSN(insn, "sibling call from callable instruction with modified stack frame");
3564                 return 1;
3565         }
3566
3567         return validate_call(file, insn, state);
3568 }
3569
3570 static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state)
3571 {
3572         if (state->noinstr && state->instr > 0) {
3573                 WARN_INSN(insn, "return with instrumentation enabled");
3574                 return 1;
3575         }
3576
3577         if (state->uaccess && !func_uaccess_safe(func)) {
3578                 WARN_INSN(insn, "return with UACCESS enabled");
3579                 return 1;
3580         }
3581
3582         if (!state->uaccess && func_uaccess_safe(func)) {
3583                 WARN_INSN(insn, "return with UACCESS disabled from a UACCESS-safe function");
3584                 return 1;
3585         }
3586
3587         if (state->df) {
3588                 WARN_INSN(insn, "return with DF set");
3589                 return 1;
3590         }
3591
3592         if (func && has_modified_stack_frame(insn, state)) {
3593                 WARN_INSN(insn, "return with modified stack frame");
3594                 return 1;
3595         }
3596
3597         if (state->cfi.bp_scratch) {
3598                 WARN_INSN(insn, "BP used as a scratch register");
3599                 return 1;
3600         }
3601
3602         return 0;
3603 }
3604
3605 static struct instruction *next_insn_to_validate(struct objtool_file *file,
3606                                                  struct instruction *insn)
3607 {
3608         struct alt_group *alt_group = insn->alt_group;
3609
3610         /*
3611          * Simulate the fact that alternatives are patched in-place.  When the
3612          * end of a replacement alt_group is reached, redirect objtool flow to
3613          * the end of the original alt_group.
3614          *
3615          * insn->alts->insn -> alt_group->first_insn
3616          *                     ...
3617          *                     alt_group->last_insn
3618          *                     [alt_group->nop]      -> next(orig_group->last_insn)
3619          */
3620         if (alt_group) {
3621                 if (alt_group->nop) {
3622                         /* ->nop implies ->orig_group */
3623                         if (insn == alt_group->last_insn)
3624                                 return alt_group->nop;
3625                         if (insn == alt_group->nop)
3626                                 goto next_orig;
3627                 }
3628                 if (insn == alt_group->last_insn && alt_group->orig_group)
3629                         goto next_orig;
3630         }
3631
3632         return next_insn_same_sec(file, insn);
3633
3634 next_orig:
3635         return next_insn_same_sec(file, alt_group->orig_group->last_insn);
3636 }
3637
3638 /*
3639  * Follow the branch starting at the given instruction, and recursively follow
3640  * any other branches (jumps).  Meanwhile, track the frame pointer state at
3641  * each instruction and validate all the rules described in
3642  * tools/objtool/Documentation/objtool.txt.
3643  */
3644 static int validate_branch(struct objtool_file *file, struct symbol *func,
3645                            struct instruction *insn, struct insn_state state)
3646 {
3647         struct alternative *alt;
3648         struct instruction *next_insn, *prev_insn = NULL;
3649         struct section *sec;
3650         u8 visited;
3651         int ret;
3652
3653         sec = insn->sec;
3654
3655         while (1) {
3656                 next_insn = next_insn_to_validate(file, insn);
3657
3658                 if (func && insn_func(insn) && func != insn_func(insn)->pfunc) {
3659                         /* Ignore KCFI type preambles, which always fall through */
3660                         if (!strncmp(func->name, "__cfi_", 6) ||
3661                             !strncmp(func->name, "__pfx_", 6))
3662                                 return 0;
3663
3664                         WARN("%s() falls through to next function %s()",
3665                              func->name, insn_func(insn)->name);
3666                         return 1;
3667                 }
3668
3669                 if (func && insn->ignore) {
3670                         WARN_INSN(insn, "BUG: why am I validating an ignored function?");
3671                         return 1;
3672                 }
3673
3674                 visited = VISITED_BRANCH << state.uaccess;
3675                 if (insn->visited & VISITED_BRANCH_MASK) {
3676                         if (!insn->hint && !insn_cfi_match(insn, &state.cfi))
3677                                 return 1;
3678
3679                         if (insn->visited & visited)
3680                                 return 0;
3681                 } else {
3682                         nr_insns_visited++;
3683                 }
3684
3685                 if (state.noinstr)
3686                         state.instr += insn->instr;
3687
3688                 if (insn->hint) {
3689                         if (insn->restore) {
3690                                 struct instruction *save_insn, *i;
3691
3692                                 i = insn;
3693                                 save_insn = NULL;
3694
3695                                 sym_for_each_insn_continue_reverse(file, func, i) {
3696                                         if (i->save) {
3697                                                 save_insn = i;
3698                                                 break;
3699                                         }
3700                                 }
3701
3702                                 if (!save_insn) {
3703                                         WARN_INSN(insn, "no corresponding CFI save for CFI restore");
3704                                         return 1;
3705                                 }
3706
3707                                 if (!save_insn->visited) {
3708                                         /*
3709                                          * If the restore hint insn is at the
3710                                          * beginning of a basic block and was
3711                                          * branched to from elsewhere, and the
3712                                          * save insn hasn't been visited yet,
3713                                          * defer following this branch for now.
3714                                          * It will be seen later via the
3715                                          * straight-line path.
3716                                          */
3717                                         if (!prev_insn)
3718                                                 return 0;
3719
3720                                         WARN_INSN(insn, "objtool isn't smart enough to handle this CFI save/restore combo");
3721                                         return 1;
3722                                 }
3723
3724                                 insn->cfi = save_insn->cfi;
3725                                 nr_cfi_reused++;
3726                         }
3727
3728                         state.cfi = *insn->cfi;
3729                 } else {
3730                         /* XXX track if we actually changed state.cfi */
3731
3732                         if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) {
3733                                 insn->cfi = prev_insn->cfi;
3734                                 nr_cfi_reused++;
3735                         } else {
3736                                 insn->cfi = cfi_hash_find_or_add(&state.cfi);
3737                         }
3738                 }
3739
3740                 insn->visited |= visited;
3741
3742                 if (propagate_alt_cfi(file, insn))
3743                         return 1;
3744
3745                 if (!insn->ignore_alts && insn->alts) {
3746                         bool skip_orig = false;
3747
3748                         for (alt = insn->alts; alt; alt = alt->next) {
3749                                 if (alt->skip_orig)
3750                                         skip_orig = true;
3751
3752                                 ret = validate_branch(file, func, alt->insn, state);
3753                                 if (ret) {
3754                                         BT_INSN(insn, "(alt)");
3755                                         return ret;
3756                                 }
3757                         }
3758
3759                         if (skip_orig)
3760                                 return 0;
3761                 }
3762
3763                 if (handle_insn_ops(insn, next_insn, &state))
3764                         return 1;
3765
3766                 switch (insn->type) {
3767
3768                 case INSN_RETURN:
3769                         return validate_return(func, insn, &state);
3770
3771                 case INSN_CALL:
3772                 case INSN_CALL_DYNAMIC:
3773                         ret = validate_call(file, insn, &state);
3774                         if (ret)
3775                                 return ret;
3776
3777                         if (opts.stackval && func && !is_special_call(insn) &&
3778                             !has_valid_stack_frame(&state)) {
3779                                 WARN_INSN(insn, "call without frame pointer save/setup");
3780                                 return 1;
3781                         }
3782
3783                         if (insn->dead_end)
3784                                 return 0;
3785
3786                         break;
3787
3788                 case INSN_JUMP_CONDITIONAL:
3789                 case INSN_JUMP_UNCONDITIONAL:
3790                         if (is_sibling_call(insn)) {
3791                                 ret = validate_sibling_call(file, insn, &state);
3792                                 if (ret)
3793                                         return ret;
3794
3795                         } else if (insn->jump_dest) {
3796                                 ret = validate_branch(file, func,
3797                                                       insn->jump_dest, state);
3798                                 if (ret) {
3799                                         BT_INSN(insn, "(branch)");
3800                                         return ret;
3801                                 }
3802                         }
3803
3804                         if (insn->type == INSN_JUMP_UNCONDITIONAL)
3805                                 return 0;
3806
3807                         break;
3808
3809                 case INSN_JUMP_DYNAMIC:
3810                 case INSN_JUMP_DYNAMIC_CONDITIONAL:
3811                         if (is_sibling_call(insn)) {
3812                                 ret = validate_sibling_call(file, insn, &state);
3813                                 if (ret)
3814                                         return ret;
3815                         }
3816
3817                         if (insn->type == INSN_JUMP_DYNAMIC)
3818                                 return 0;
3819
3820                         break;
3821
3822                 case INSN_CONTEXT_SWITCH:
3823                         if (func && (!next_insn || !next_insn->hint)) {
3824                                 WARN_INSN(insn, "unsupported instruction in callable function");
3825                                 return 1;
3826                         }
3827                         return 0;
3828
3829                 case INSN_STAC:
3830                         if (state.uaccess) {
3831                                 WARN_INSN(insn, "recursive UACCESS enable");
3832                                 return 1;
3833                         }
3834
3835                         state.uaccess = true;
3836                         break;
3837
3838                 case INSN_CLAC:
3839                         if (!state.uaccess && func) {
3840                                 WARN_INSN(insn, "redundant UACCESS disable");
3841                                 return 1;
3842                         }
3843
3844                         if (func_uaccess_safe(func) && !state.uaccess_stack) {
3845                                 WARN_INSN(insn, "UACCESS-safe disables UACCESS");
3846                                 return 1;
3847                         }
3848
3849                         state.uaccess = false;
3850                         break;
3851
3852                 case INSN_STD:
3853                         if (state.df) {
3854                                 WARN_INSN(insn, "recursive STD");
3855                                 return 1;
3856                         }
3857
3858                         state.df = true;
3859                         break;
3860
3861                 case INSN_CLD:
3862                         if (!state.df && func) {
3863                                 WARN_INSN(insn, "redundant CLD");
3864                                 return 1;
3865                         }
3866
3867                         state.df = false;
3868                         break;
3869
3870                 default:
3871                         break;
3872                 }
3873
3874                 if (insn->dead_end)
3875                         return 0;
3876
3877                 if (!next_insn) {
3878                         if (state.cfi.cfa.base == CFI_UNDEFINED)
3879                                 return 0;
3880                         WARN("%s: unexpected end of section", sec->name);
3881                         return 1;
3882                 }
3883
3884                 prev_insn = insn;
3885                 insn = next_insn;
3886         }
3887
3888         return 0;
3889 }
3890
3891 static int validate_unwind_hint(struct objtool_file *file,
3892                                   struct instruction *insn,
3893                                   struct insn_state *state)
3894 {
3895         if (insn->hint && !insn->visited && !insn->ignore) {
3896                 int ret = validate_branch(file, insn_func(insn), insn, *state);
3897                 if (ret)
3898                         BT_INSN(insn, "<=== (hint)");
3899                 return ret;
3900         }
3901
3902         return 0;
3903 }
3904
3905 static int validate_unwind_hints(struct objtool_file *file, struct section *sec)
3906 {
3907         struct instruction *insn;
3908         struct insn_state state;
3909         int warnings = 0;
3910
3911         if (!file->hints)
3912                 return 0;
3913
3914         init_insn_state(file, &state, sec);
3915
3916         if (sec) {
3917                 sec_for_each_insn(file, sec, insn)
3918                         warnings += validate_unwind_hint(file, insn, &state);
3919         } else {
3920                 for_each_insn(file, insn)
3921                         warnings += validate_unwind_hint(file, insn, &state);
3922         }
3923
3924         return warnings;
3925 }
3926
3927 /*
3928  * Validate rethunk entry constraint: must untrain RET before the first RET.
3929  *
3930  * Follow every branch (intra-function) and ensure VALIDATE_UNRET_END comes
3931  * before an actual RET instruction.
3932  */
3933 static int validate_unret(struct objtool_file *file, struct instruction *insn)
3934 {
3935         struct instruction *next, *dest;
3936         int ret;
3937
3938         for (;;) {
3939                 next = next_insn_to_validate(file, insn);
3940
3941                 if (insn->visited & VISITED_UNRET)
3942                         return 0;
3943
3944                 insn->visited |= VISITED_UNRET;
3945
3946                 if (!insn->ignore_alts && insn->alts) {
3947                         struct alternative *alt;
3948                         bool skip_orig = false;
3949
3950                         for (alt = insn->alts; alt; alt = alt->next) {
3951                                 if (alt->skip_orig)
3952                                         skip_orig = true;
3953
3954                                 ret = validate_unret(file, alt->insn);
3955                                 if (ret) {
3956                                         BT_INSN(insn, "(alt)");
3957                                         return ret;
3958                                 }
3959                         }
3960
3961                         if (skip_orig)
3962                                 return 0;
3963                 }
3964
3965                 switch (insn->type) {
3966
3967                 case INSN_CALL_DYNAMIC:
3968                 case INSN_JUMP_DYNAMIC:
3969                 case INSN_JUMP_DYNAMIC_CONDITIONAL:
3970                         WARN_INSN(insn, "early indirect call");
3971                         return 1;
3972
3973                 case INSN_JUMP_UNCONDITIONAL:
3974                 case INSN_JUMP_CONDITIONAL:
3975                         if (!is_sibling_call(insn)) {
3976                                 if (!insn->jump_dest) {
3977                                         WARN_INSN(insn, "unresolved jump target after linking?!?");
3978                                         return -1;
3979                                 }
3980                                 ret = validate_unret(file, insn->jump_dest);
3981                                 if (ret) {
3982                                         BT_INSN(insn, "(branch%s)",
3983                                                 insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : "");
3984                                         return ret;
3985                                 }
3986
3987                                 if (insn->type == INSN_JUMP_UNCONDITIONAL)
3988                                         return 0;
3989
3990                                 break;
3991                         }
3992
3993                         /* fallthrough */
3994                 case INSN_CALL:
3995                         dest = find_insn(file, insn_call_dest(insn)->sec,
3996                                          insn_call_dest(insn)->offset);
3997                         if (!dest) {
3998                                 WARN("Unresolved function after linking!?: %s",
3999                                      insn_call_dest(insn)->name);
4000                                 return -1;
4001                         }
4002
4003                         ret = validate_unret(file, dest);
4004                         if (ret) {
4005                                 BT_INSN(insn, "(call)");
4006                                 return ret;
4007                         }
4008                         /*
4009                          * If a call returns without error, it must have seen UNTRAIN_RET.
4010                          * Therefore any non-error return is a success.
4011                          */
4012                         return 0;
4013
4014                 case INSN_RETURN:
4015                         WARN_INSN(insn, "RET before UNTRAIN");
4016                         return 1;
4017
4018                 case INSN_NOP:
4019                         if (insn->retpoline_safe)
4020                                 return 0;
4021                         break;
4022
4023                 default:
4024                         break;
4025                 }
4026
4027                 if (!next) {
4028                         WARN_INSN(insn, "teh end!");
4029                         return -1;
4030                 }
4031                 insn = next;
4032         }
4033
4034         return 0;
4035 }
4036
4037 /*
4038  * Validate that all branches starting at VALIDATE_UNRET_BEGIN encounter
4039  * VALIDATE_UNRET_END before RET.
4040  */
4041 static int validate_unrets(struct objtool_file *file)
4042 {
4043         struct instruction *insn;
4044         int ret, warnings = 0;
4045
4046         for_each_insn(file, insn) {
4047                 if (!insn->unret)
4048                         continue;
4049
4050                 ret = validate_unret(file, insn);
4051                 if (ret < 0) {
4052                         WARN_INSN(insn, "Failed UNRET validation");
4053                         return ret;
4054                 }
4055                 warnings += ret;
4056         }
4057
4058         return warnings;
4059 }
4060
4061 static int validate_retpoline(struct objtool_file *file)
4062 {
4063         struct instruction *insn;
4064         int warnings = 0;
4065
4066         for_each_insn(file, insn) {
4067                 if (insn->type != INSN_JUMP_DYNAMIC &&
4068                     insn->type != INSN_CALL_DYNAMIC &&
4069                     insn->type != INSN_RETURN)
4070                         continue;
4071
4072                 if (insn->retpoline_safe)
4073                         continue;
4074
4075                 if (insn->sec->init)
4076                         continue;
4077
4078                 if (insn->type == INSN_RETURN) {
4079                         if (opts.rethunk) {
4080                                 WARN_INSN(insn, "'naked' return found in MITIGATION_RETHUNK build");
4081                         } else
4082                                 continue;
4083                 } else {
4084                         WARN_INSN(insn, "indirect %s found in MITIGATION_RETPOLINE build",
4085                                   insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");
4086                 }
4087
4088                 warnings++;
4089         }
4090
4091         return warnings;
4092 }
4093
4094 static bool is_kasan_insn(struct instruction *insn)
4095 {
4096         return (insn->type == INSN_CALL &&
4097                 !strcmp(insn_call_dest(insn)->name, "__asan_handle_no_return"));
4098 }
4099
4100 static bool is_ubsan_insn(struct instruction *insn)
4101 {
4102         return (insn->type == INSN_CALL &&
4103                 !strcmp(insn_call_dest(insn)->name,
4104                         "__ubsan_handle_builtin_unreachable"));
4105 }
4106
4107 static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn)
4108 {
4109         int i;
4110         struct instruction *prev_insn;
4111
4112         if (insn->ignore || insn->type == INSN_NOP || insn->type == INSN_TRAP)
4113                 return true;
4114
4115         /*
4116          * Ignore alternative replacement instructions.  This can happen
4117          * when a whitelisted function uses one of the ALTERNATIVE macros.
4118          */
4119         if (!strcmp(insn->sec->name, ".altinstr_replacement") ||
4120             !strcmp(insn->sec->name, ".altinstr_aux"))
4121                 return true;
4122
4123         /*
4124          * Whole archive runs might encounter dead code from weak symbols.
4125          * This is where the linker will have dropped the weak symbol in
4126          * favour of a regular symbol, but leaves the code in place.
4127          *
4128          * In this case we'll find a piece of code (whole function) that is not
4129          * covered by a !section symbol. Ignore them.
4130          */
4131         if (opts.link && !insn_func(insn)) {
4132                 int size = find_symbol_hole_containing(insn->sec, insn->offset);
4133                 unsigned long end = insn->offset + size;
4134
4135                 if (!size) /* not a hole */
4136                         return false;
4137
4138                 if (size < 0) /* hole until the end */
4139                         return true;
4140
4141                 sec_for_each_insn_continue(file, insn) {
4142                         /*
4143                          * If we reach a visited instruction at or before the
4144                          * end of the hole, ignore the unreachable.
4145                          */
4146                         if (insn->visited)
4147                                 return true;
4148
4149                         if (insn->offset >= end)
4150                                 break;
4151
4152                         /*
4153                          * If this hole jumps to a .cold function, mark it ignore too.
4154                          */
4155                         if (insn->jump_dest && insn_func(insn->jump_dest) &&
4156                             strstr(insn_func(insn->jump_dest)->name, ".cold")) {
4157                                 struct instruction *dest = insn->jump_dest;
4158                                 func_for_each_insn(file, insn_func(dest), dest)
4159                                         dest->ignore = true;
4160                         }
4161                 }
4162
4163                 return false;
4164         }
4165
4166         if (!insn_func(insn))
4167                 return false;
4168
4169         if (insn_func(insn)->static_call_tramp)
4170                 return true;
4171
4172         /*
4173          * CONFIG_UBSAN_TRAP inserts a UD2 when it sees
4174          * __builtin_unreachable().  The BUG() macro has an unreachable() after
4175          * the UD2, which causes GCC's undefined trap logic to emit another UD2
4176          * (or occasionally a JMP to UD2).
4177          *
4178          * It may also insert a UD2 after calling a __noreturn function.
4179          */
4180         prev_insn = prev_insn_same_sec(file, insn);
4181         if (prev_insn->dead_end &&
4182             (insn->type == INSN_BUG ||
4183              (insn->type == INSN_JUMP_UNCONDITIONAL &&
4184               insn->jump_dest && insn->jump_dest->type == INSN_BUG)))
4185                 return true;
4186
4187         /*
4188          * Check if this (or a subsequent) instruction is related to
4189          * CONFIG_UBSAN or CONFIG_KASAN.
4190          *
4191          * End the search at 5 instructions to avoid going into the weeds.
4192          */
4193         for (i = 0; i < 5; i++) {
4194
4195                 if (is_kasan_insn(insn) || is_ubsan_insn(insn))
4196                         return true;
4197
4198                 if (insn->type == INSN_JUMP_UNCONDITIONAL) {
4199                         if (insn->jump_dest &&
4200                             insn_func(insn->jump_dest) == insn_func(insn)) {
4201                                 insn = insn->jump_dest;
4202                                 continue;
4203                         }
4204
4205                         break;
4206                 }
4207
4208                 if (insn->offset + insn->len >= insn_func(insn)->offset + insn_func(insn)->len)
4209                         break;
4210
4211                 insn = next_insn_same_sec(file, insn);
4212         }
4213
4214         return false;
4215 }
4216
4217 static int add_prefix_symbol(struct objtool_file *file, struct symbol *func)
4218 {
4219         struct instruction *insn, *prev;
4220         struct cfi_state *cfi;
4221
4222         insn = find_insn(file, func->sec, func->offset);
4223         if (!insn)
4224                 return -1;
4225
4226         for (prev = prev_insn_same_sec(file, insn);
4227              prev;
4228              prev = prev_insn_same_sec(file, prev)) {
4229                 u64 offset;
4230
4231                 if (prev->type != INSN_NOP)
4232                         return -1;
4233
4234                 offset = func->offset - prev->offset;
4235
4236                 if (offset > opts.prefix)
4237                         return -1;
4238
4239                 if (offset < opts.prefix)
4240                         continue;
4241
4242                 elf_create_prefix_symbol(file->elf, func, opts.prefix);
4243                 break;
4244         }
4245
4246         if (!prev)
4247                 return -1;
4248
4249         if (!insn->cfi) {
4250                 /*
4251                  * This can happen if stack validation isn't enabled or the
4252                  * function is annotated with STACK_FRAME_NON_STANDARD.
4253                  */
4254                 return 0;
4255         }
4256
4257         /* Propagate insn->cfi to the prefix code */
4258         cfi = cfi_hash_find_or_add(insn->cfi);
4259         for (; prev != insn; prev = next_insn_same_sec(file, prev))
4260                 prev->cfi = cfi;
4261
4262         return 0;
4263 }
4264
4265 static int add_prefix_symbols(struct objtool_file *file)
4266 {
4267         struct section *sec;
4268         struct symbol *func;
4269
4270         for_each_sec(file, sec) {
4271                 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4272                         continue;
4273
4274                 sec_for_each_sym(sec, func) {
4275                         if (func->type != STT_FUNC)
4276                                 continue;
4277
4278                         add_prefix_symbol(file, func);
4279                 }
4280         }
4281
4282         return 0;
4283 }
4284
4285 static int validate_symbol(struct objtool_file *file, struct section *sec,
4286                            struct symbol *sym, struct insn_state *state)
4287 {
4288         struct instruction *insn;
4289         int ret;
4290
4291         if (!sym->len) {
4292                 WARN("%s() is missing an ELF size annotation", sym->name);
4293                 return 1;
4294         }
4295
4296         if (sym->pfunc != sym || sym->alias != sym)
4297                 return 0;
4298
4299         insn = find_insn(file, sec, sym->offset);
4300         if (!insn || insn->ignore || insn->visited)
4301                 return 0;
4302
4303         state->uaccess = sym->uaccess_safe;
4304
4305         ret = validate_branch(file, insn_func(insn), insn, *state);
4306         if (ret)
4307                 BT_INSN(insn, "<=== (sym)");
4308         return ret;
4309 }
4310
4311 static int validate_section(struct objtool_file *file, struct section *sec)
4312 {
4313         struct insn_state state;
4314         struct symbol *func;
4315         int warnings = 0;
4316
4317         sec_for_each_sym(sec, func) {
4318                 if (func->type != STT_FUNC)
4319                         continue;
4320
4321                 init_insn_state(file, &state, sec);
4322                 set_func_state(&state.cfi);
4323
4324                 warnings += validate_symbol(file, sec, func, &state);
4325         }
4326
4327         return warnings;
4328 }
4329
4330 static int validate_noinstr_sections(struct objtool_file *file)
4331 {
4332         struct section *sec;
4333         int warnings = 0;
4334
4335         sec = find_section_by_name(file->elf, ".noinstr.text");
4336         if (sec) {
4337                 warnings += validate_section(file, sec);
4338                 warnings += validate_unwind_hints(file, sec);
4339         }
4340
4341         sec = find_section_by_name(file->elf, ".entry.text");
4342         if (sec) {
4343                 warnings += validate_section(file, sec);
4344                 warnings += validate_unwind_hints(file, sec);
4345         }
4346
4347         sec = find_section_by_name(file->elf, ".cpuidle.text");
4348         if (sec) {
4349                 warnings += validate_section(file, sec);
4350                 warnings += validate_unwind_hints(file, sec);
4351         }
4352
4353         return warnings;
4354 }
4355
4356 static int validate_functions(struct objtool_file *file)
4357 {
4358         struct section *sec;
4359         int warnings = 0;
4360
4361         for_each_sec(file, sec) {
4362                 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4363                         continue;
4364
4365                 warnings += validate_section(file, sec);
4366         }
4367
4368         return warnings;
4369 }
4370
4371 static void mark_endbr_used(struct instruction *insn)
4372 {
4373         if (!list_empty(&insn->call_node))
4374                 list_del_init(&insn->call_node);
4375 }
4376
4377 static bool noendbr_range(struct objtool_file *file, struct instruction *insn)
4378 {
4379         struct symbol *sym = find_symbol_containing(insn->sec, insn->offset-1);
4380         struct instruction *first;
4381
4382         if (!sym)
4383                 return false;
4384
4385         first = find_insn(file, sym->sec, sym->offset);
4386         if (!first)
4387                 return false;
4388
4389         if (first->type != INSN_ENDBR && !first->noendbr)
4390                 return false;
4391
4392         return insn->offset == sym->offset + sym->len;
4393 }
4394
4395 static int __validate_ibt_insn(struct objtool_file *file, struct instruction *insn,
4396                                struct instruction *dest)
4397 {
4398         if (dest->type == INSN_ENDBR) {
4399                 mark_endbr_used(dest);
4400                 return 0;
4401         }
4402
4403         if (insn_func(dest) && insn_func(insn) &&
4404             insn_func(dest)->pfunc == insn_func(insn)->pfunc) {
4405                 /*
4406                  * Anything from->to self is either _THIS_IP_ or
4407                  * IRET-to-self.
4408                  *
4409                  * There is no sane way to annotate _THIS_IP_ since the
4410                  * compiler treats the relocation as a constant and is
4411                  * happy to fold in offsets, skewing any annotation we
4412                  * do, leading to vast amounts of false-positives.
4413                  *
4414                  * There's also compiler generated _THIS_IP_ through
4415                  * KCOV and such which we have no hope of annotating.
4416                  *
4417                  * As such, blanket accept self-references without
4418                  * issue.
4419                  */
4420                 return 0;
4421         }
4422
4423         /*
4424          * Accept anything ANNOTATE_NOENDBR.
4425          */
4426         if (dest->noendbr)
4427                 return 0;
4428
4429         /*
4430          * Accept if this is the instruction after a symbol
4431          * that is (no)endbr -- typical code-range usage.
4432          */
4433         if (noendbr_range(file, dest))
4434                 return 0;
4435
4436         WARN_INSN(insn, "relocation to !ENDBR: %s", offstr(dest->sec, dest->offset));
4437         return 1;
4438 }
4439
4440 static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn)
4441 {
4442         struct instruction *dest;
4443         struct reloc *reloc;
4444         unsigned long off;
4445         int warnings = 0;
4446
4447         /*
4448          * Looking for function pointer load relocations.  Ignore
4449          * direct/indirect branches:
4450          */
4451         switch (insn->type) {
4452
4453         case INSN_CALL:
4454         case INSN_CALL_DYNAMIC:
4455         case INSN_JUMP_CONDITIONAL:
4456         case INSN_JUMP_UNCONDITIONAL:
4457         case INSN_JUMP_DYNAMIC:
4458         case INSN_JUMP_DYNAMIC_CONDITIONAL:
4459         case INSN_RETURN:
4460         case INSN_NOP:
4461                 return 0;
4462
4463         case INSN_LEA_RIP:
4464                 if (!insn_reloc(file, insn)) {
4465                         /* local function pointer reference without reloc */
4466
4467                         off = arch_jump_destination(insn);
4468
4469                         dest = find_insn(file, insn->sec, off);
4470                         if (!dest) {
4471                                 WARN_INSN(insn, "corrupt function pointer reference");
4472                                 return 1;
4473                         }
4474
4475                         return __validate_ibt_insn(file, insn, dest);
4476                 }
4477                 break;
4478
4479         default:
4480                 break;
4481         }
4482
4483         for (reloc = insn_reloc(file, insn);
4484              reloc;
4485              reloc = find_reloc_by_dest_range(file->elf, insn->sec,
4486                                               reloc_offset(reloc) + 1,
4487                                               (insn->offset + insn->len) - (reloc_offset(reloc) + 1))) {
4488
4489                 off = reloc->sym->offset;
4490                 if (reloc_type(reloc) == R_X86_64_PC32 ||
4491                     reloc_type(reloc) == R_X86_64_PLT32)
4492                         off += arch_dest_reloc_offset(reloc_addend(reloc));
4493                 else
4494                         off += reloc_addend(reloc);
4495
4496                 dest = find_insn(file, reloc->sym->sec, off);
4497                 if (!dest)
4498                         continue;
4499
4500                 warnings += __validate_ibt_insn(file, insn, dest);
4501         }
4502
4503         return warnings;
4504 }
4505
4506 static int validate_ibt_data_reloc(struct objtool_file *file,
4507                                    struct reloc *reloc)
4508 {
4509         struct instruction *dest;
4510
4511         dest = find_insn(file, reloc->sym->sec,
4512                          reloc->sym->offset + reloc_addend(reloc));
4513         if (!dest)
4514                 return 0;
4515
4516         if (dest->type == INSN_ENDBR) {
4517                 mark_endbr_used(dest);
4518                 return 0;
4519         }
4520
4521         if (dest->noendbr)
4522                 return 0;
4523
4524         WARN_FUNC("data relocation to !ENDBR: %s",
4525                   reloc->sec->base, reloc_offset(reloc),
4526                   offstr(dest->sec, dest->offset));
4527
4528         return 1;
4529 }
4530
4531 /*
4532  * Validate IBT rules and remove used ENDBR instructions from the seal list.
4533  * Unused ENDBR instructions will be annotated for sealing (i.e., replaced with
4534  * NOPs) later, in create_ibt_endbr_seal_sections().
4535  */
4536 static int validate_ibt(struct objtool_file *file)
4537 {
4538         struct section *sec;
4539         struct reloc *reloc;
4540         struct instruction *insn;
4541         int warnings = 0;
4542
4543         for_each_insn(file, insn)
4544                 warnings += validate_ibt_insn(file, insn);
4545
4546         for_each_sec(file, sec) {
4547
4548                 /* Already done by validate_ibt_insn() */
4549                 if (sec->sh.sh_flags & SHF_EXECINSTR)
4550                         continue;
4551
4552                 if (!sec->rsec)
4553                         continue;
4554
4555                 /*
4556                  * These sections can reference text addresses, but not with
4557                  * the intent to indirect branch to them.
4558                  */
4559                 if ((!strncmp(sec->name, ".discard", 8) &&
4560                      strcmp(sec->name, ".discard.ibt_endbr_noseal"))    ||
4561                     !strncmp(sec->name, ".debug", 6)                    ||
4562                     !strcmp(sec->name, ".altinstructions")              ||
4563                     !strcmp(sec->name, ".ibt_endbr_seal")               ||
4564                     !strcmp(sec->name, ".orc_unwind_ip")                ||
4565                     !strcmp(sec->name, ".parainstructions")             ||
4566                     !strcmp(sec->name, ".retpoline_sites")              ||
4567                     !strcmp(sec->name, ".smp_locks")                    ||
4568                     !strcmp(sec->name, ".static_call_sites")            ||
4569                     !strcmp(sec->name, "_error_injection_whitelist")    ||
4570                     !strcmp(sec->name, "_kprobe_blacklist")             ||
4571                     !strcmp(sec->name, "__bug_table")                   ||
4572                     !strcmp(sec->name, "__ex_table")                    ||
4573                     !strcmp(sec->name, "__jump_table")                  ||
4574                     !strcmp(sec->name, "__mcount_loc")                  ||
4575                     !strcmp(sec->name, ".kcfi_traps")                   ||
4576                     !strcmp(sec->name, ".llvm.call-graph-profile")      ||
4577                     !strcmp(sec->name, ".llvm_bb_addr_map")             ||
4578                     !strcmp(sec->name, "__tracepoints")                 ||
4579                     strstr(sec->name, "__patchable_function_entries"))
4580                         continue;
4581
4582                 for_each_reloc(sec->rsec, reloc)
4583                         warnings += validate_ibt_data_reloc(file, reloc);
4584         }
4585
4586         return warnings;
4587 }
4588
4589 static int validate_sls(struct objtool_file *file)
4590 {
4591         struct instruction *insn, *next_insn;
4592         int warnings = 0;
4593
4594         for_each_insn(file, insn) {
4595                 next_insn = next_insn_same_sec(file, insn);
4596
4597                 if (insn->retpoline_safe)
4598                         continue;
4599
4600                 switch (insn->type) {
4601                 case INSN_RETURN:
4602                         if (!next_insn || next_insn->type != INSN_TRAP) {
4603                                 WARN_INSN(insn, "missing int3 after ret");
4604                                 warnings++;
4605                         }
4606
4607                         break;
4608                 case INSN_JUMP_DYNAMIC:
4609                         if (!next_insn || next_insn->type != INSN_TRAP) {
4610                                 WARN_INSN(insn, "missing int3 after indirect jump");
4611                                 warnings++;
4612                         }
4613                         break;
4614                 default:
4615                         break;
4616                 }
4617         }
4618
4619         return warnings;
4620 }
4621
4622 static bool ignore_noreturn_call(struct instruction *insn)
4623 {
4624         struct symbol *call_dest = insn_call_dest(insn);
4625
4626         /*
4627          * FIXME: hack, we need a real noreturn solution
4628          *
4629          * Problem is, exc_double_fault() may or may not return, depending on
4630          * whether CONFIG_X86_ESPFIX64 is set.  But objtool has no visibility
4631          * to the kernel config.
4632          *
4633          * Other potential ways to fix it:
4634          *
4635          *   - have compiler communicate __noreturn functions somehow
4636          *   - remove CONFIG_X86_ESPFIX64
4637          *   - read the .config file
4638          *   - add a cmdline option
4639          *   - create a generic objtool annotation format (vs a bunch of custom
4640          *     formats) and annotate it
4641          */
4642         if (!strcmp(call_dest->name, "exc_double_fault")) {
4643                 /* prevent further unreachable warnings for the caller */
4644                 insn->sym->warned = 1;
4645                 return true;
4646         }
4647
4648         return false;
4649 }
4650
4651 static int validate_reachable_instructions(struct objtool_file *file)
4652 {
4653         struct instruction *insn, *prev_insn;
4654         struct symbol *call_dest;
4655         int warnings = 0;
4656
4657         if (file->ignore_unreachables)
4658                 return 0;
4659
4660         for_each_insn(file, insn) {
4661                 if (insn->visited || ignore_unreachable_insn(file, insn))
4662                         continue;
4663
4664                 prev_insn = prev_insn_same_sec(file, insn);
4665                 if (prev_insn && prev_insn->dead_end) {
4666                         call_dest = insn_call_dest(prev_insn);
4667                         if (call_dest && !ignore_noreturn_call(prev_insn)) {
4668                                 WARN_INSN(insn, "%s() is missing a __noreturn annotation",
4669                                           call_dest->name);
4670                                 warnings++;
4671                                 continue;
4672                         }
4673                 }
4674
4675                 WARN_INSN(insn, "unreachable instruction");
4676                 warnings++;
4677         }
4678
4679         return warnings;
4680 }
4681
4682 /* 'funcs' is a space-separated list of function names */
4683 static int disas_funcs(const char *funcs)
4684 {
4685         const char *objdump_str, *cross_compile;
4686         int size, ret;
4687         char *cmd;
4688
4689         cross_compile = getenv("CROSS_COMPILE");
4690
4691         objdump_str = "%sobjdump -wdr %s | gawk -M -v _funcs='%s' '"
4692                         "BEGIN { split(_funcs, funcs); }"
4693                         "/^$/ { func_match = 0; }"
4694                         "/<.*>:/ { "
4695                                 "f = gensub(/.*<(.*)>:/, \"\\\\1\", 1);"
4696                                 "for (i in funcs) {"
4697                                         "if (funcs[i] == f) {"
4698                                                 "func_match = 1;"
4699                                                 "base = strtonum(\"0x\" $1);"
4700                                                 "break;"
4701                                         "}"
4702                                 "}"
4703                         "}"
4704                         "{"
4705                                 "if (func_match) {"
4706                                         "addr = strtonum(\"0x\" $1);"
4707                                         "printf(\"%%04x \", addr - base);"
4708                                         "print;"
4709                                 "}"
4710                         "}' 1>&2";
4711
4712         /* fake snprintf() to calculate the size */
4713         size = snprintf(NULL, 0, objdump_str, cross_compile, objname, funcs) + 1;
4714         if (size <= 0) {
4715                 WARN("objdump string size calculation failed");
4716                 return -1;
4717         }
4718
4719         cmd = malloc(size);
4720
4721         /* real snprintf() */
4722         snprintf(cmd, size, objdump_str, cross_compile, objname, funcs);
4723         ret = system(cmd);
4724         if (ret) {
4725                 WARN("disassembly failed: %d", ret);
4726                 return -1;
4727         }
4728
4729         return 0;
4730 }
4731
4732 static int disas_warned_funcs(struct objtool_file *file)
4733 {
4734         struct symbol *sym;
4735         char *funcs = NULL, *tmp;
4736
4737         for_each_sym(file, sym) {
4738                 if (sym->warned) {
4739                         if (!funcs) {
4740                                 funcs = malloc(strlen(sym->name) + 1);
4741                                 strcpy(funcs, sym->name);
4742                         } else {
4743                                 tmp = malloc(strlen(funcs) + strlen(sym->name) + 2);
4744                                 sprintf(tmp, "%s %s", funcs, sym->name);
4745                                 free(funcs);
4746                                 funcs = tmp;
4747                         }
4748                 }
4749         }
4750
4751         if (funcs)
4752                 disas_funcs(funcs);
4753
4754         return 0;
4755 }
4756
4757 struct insn_chunk {
4758         void *addr;
4759         struct insn_chunk *next;
4760 };
4761
4762 /*
4763  * Reduce peak RSS usage by freeing insns memory before writing the ELF file,
4764  * which can trigger more allocations for .debug_* sections whose data hasn't
4765  * been read yet.
4766  */
4767 static void free_insns(struct objtool_file *file)
4768 {
4769         struct instruction *insn;
4770         struct insn_chunk *chunks = NULL, *chunk;
4771
4772         for_each_insn(file, insn) {
4773                 if (!insn->idx) {
4774                         chunk = malloc(sizeof(*chunk));
4775                         chunk->addr = insn;
4776                         chunk->next = chunks;
4777                         chunks = chunk;
4778                 }
4779         }
4780
4781         for (chunk = chunks; chunk; chunk = chunk->next)
4782                 free(chunk->addr);
4783 }
4784
4785 int check(struct objtool_file *file)
4786 {
4787         int ret, warnings = 0;
4788
4789         arch_initial_func_cfi_state(&initial_func_cfi);
4790         init_cfi_state(&init_cfi);
4791         init_cfi_state(&func_cfi);
4792         set_func_state(&func_cfi);
4793         init_cfi_state(&force_undefined_cfi);
4794         force_undefined_cfi.force_undefined = true;
4795
4796         if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3)))
4797                 goto out;
4798
4799         cfi_hash_add(&init_cfi);
4800         cfi_hash_add(&func_cfi);
4801
4802         ret = decode_sections(file);
4803         if (ret < 0)
4804                 goto out;
4805
4806         warnings += ret;
4807
4808         if (!nr_insns)
4809                 goto out;
4810
4811         if (opts.retpoline) {
4812                 ret = validate_retpoline(file);
4813                 if (ret < 0)
4814                         return ret;
4815                 warnings += ret;
4816         }
4817
4818         if (opts.stackval || opts.orc || opts.uaccess) {
4819                 ret = validate_functions(file);
4820                 if (ret < 0)
4821                         goto out;
4822                 warnings += ret;
4823
4824                 ret = validate_unwind_hints(file, NULL);
4825                 if (ret < 0)
4826                         goto out;
4827                 warnings += ret;
4828
4829                 if (!warnings) {
4830                         ret = validate_reachable_instructions(file);
4831                         if (ret < 0)
4832                                 goto out;
4833                         warnings += ret;
4834                 }
4835
4836         } else if (opts.noinstr) {
4837                 ret = validate_noinstr_sections(file);
4838                 if (ret < 0)
4839                         goto out;
4840                 warnings += ret;
4841         }
4842
4843         if (opts.unret) {
4844                 /*
4845                  * Must be after validate_branch() and friends, it plays
4846                  * further games with insn->visited.
4847                  */
4848                 ret = validate_unrets(file);
4849                 if (ret < 0)
4850                         return ret;
4851                 warnings += ret;
4852         }
4853
4854         if (opts.ibt) {
4855                 ret = validate_ibt(file);
4856                 if (ret < 0)
4857                         goto out;
4858                 warnings += ret;
4859         }
4860
4861         if (opts.sls) {
4862                 ret = validate_sls(file);
4863                 if (ret < 0)
4864                         goto out;
4865                 warnings += ret;
4866         }
4867
4868         if (opts.static_call) {
4869                 ret = create_static_call_sections(file);
4870                 if (ret < 0)
4871                         goto out;
4872                 warnings += ret;
4873         }
4874
4875         if (opts.retpoline) {
4876                 ret = create_retpoline_sites_sections(file);
4877                 if (ret < 0)
4878                         goto out;
4879                 warnings += ret;
4880         }
4881
4882         if (opts.cfi) {
4883                 ret = create_cfi_sections(file);
4884                 if (ret < 0)
4885                         goto out;
4886                 warnings += ret;
4887         }
4888
4889         if (opts.rethunk) {
4890                 ret = create_return_sites_sections(file);
4891                 if (ret < 0)
4892                         goto out;
4893                 warnings += ret;
4894
4895                 if (opts.hack_skylake) {
4896                         ret = create_direct_call_sections(file);
4897                         if (ret < 0)
4898                                 goto out;
4899                         warnings += ret;
4900                 }
4901         }
4902
4903         if (opts.mcount) {
4904                 ret = create_mcount_loc_sections(file);
4905                 if (ret < 0)
4906                         goto out;
4907                 warnings += ret;
4908         }
4909
4910         if (opts.prefix) {
4911                 ret = add_prefix_symbols(file);
4912                 if (ret < 0)
4913                         return ret;
4914                 warnings += ret;
4915         }
4916
4917         if (opts.ibt) {
4918                 ret = create_ibt_endbr_seal_sections(file);
4919                 if (ret < 0)
4920                         goto out;
4921                 warnings += ret;
4922         }
4923
4924         if (opts.orc && nr_insns) {
4925                 ret = orc_create(file);
4926                 if (ret < 0)
4927                         goto out;
4928                 warnings += ret;
4929         }
4930
4931         free_insns(file);
4932
4933         if (opts.verbose)
4934                 disas_warned_funcs(file);
4935
4936         if (opts.stats) {
4937                 printf("nr_insns_visited: %ld\n", nr_insns_visited);
4938                 printf("nr_cfi: %ld\n", nr_cfi);
4939                 printf("nr_cfi_reused: %ld\n", nr_cfi_reused);
4940                 printf("nr_cfi_cache: %ld\n", nr_cfi_cache);
4941         }
4942
4943 out:
4944         /*
4945          *  For now, don't fail the kernel build on fatal warnings.  These
4946          *  errors are still fairly common due to the growing matrix of
4947          *  supported toolchains and their recent pace of change.
4948          */
4949         return 0;
4950 }