compiler-rt/lib/cfi/cfi.cpp

   1 //===-------- cfi.cpp -----------------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements the runtime support for the cross-DSO CFI.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #include <assert.h>
  14 #include <elf.h>
  15
  16 #include "sanitizer_common/sanitizer_common.h"
  17 #if SANITIZER_FREEBSD
  18 #include <sys/link_elf.h>
  19 #endif
  20 #include <link.h>
  21 #include <string.h>
  22 #include <stdlib.h>
  23 #include <sys/mman.h>
  24
  25 #if SANITIZER_LINUX
  26 typedef ElfW(Phdr) Elf_Phdr;
  27 typedef ElfW(Ehdr) Elf_Ehdr;
  28 typedef ElfW(Addr) Elf_Addr;
  29 typedef ElfW(Sym) Elf_Sym;
  30 typedef ElfW(Dyn) Elf_Dyn;
  31 #elif SANITIZER_FREEBSD
  32 #if SANITIZER_WORDSIZE == 64
  33 #define ElfW64_Dyn Elf_Dyn
  34 #define ElfW64_Sym Elf_Sym
  35 #else
  36 #define ElfW32_Dyn Elf_Dyn
  37 #define ElfW32_Sym Elf_Sym
  38 #endif
  39 #endif
  40
  41 #include "interception/interception.h"
  42 #include "sanitizer_common/sanitizer_flag_parser.h"
  43 #include "ubsan/ubsan_init.h"
  44 #include "ubsan/ubsan_flags.h"
  45
  46 #ifdef CFI_ENABLE_DIAG
  47 #include "ubsan/ubsan_handlers.h"
  48 #endif
  49
  50 using namespace __sanitizer;
  51
  52 namespace __cfi {
  53
  54 #if SANITIZER_LOONGARCH64
  55 #define kCfiShadowLimitsStorageSize 16384 // 16KiB on loongarch64 per page
  56 #else
  57 #define kCfiShadowLimitsStorageSize 4096 // 1 page
  58 #endif
  59 // Lets hope that the data segment is mapped with 4K pages.
  60 // The pointer to the cfi shadow region is stored at the start of this page.
  61 // The rest of the page is unused and re-mapped read-only.
  62 static union {
  63   char space[kCfiShadowLimitsStorageSize];
  64   struct {
  65     uptr start;
  66     uptr size;
  67   } limits;
  68 } cfi_shadow_limits_storage
  69     __attribute__((aligned(kCfiShadowLimitsStorageSize)));
  70 static constexpr uptr kShadowGranularity = 12;
  71 static constexpr uptr kShadowAlign = 1UL << kShadowGranularity; // 4096
  72
  73 static constexpr uint16_t kInvalidShadow = 0;
  74 static constexpr uint16_t kUncheckedShadow = 0xFFFFU;
  75
  76 // Get the start address of the CFI shadow region.
  77 uptr GetShadow() {
  78   return cfi_shadow_limits_storage.limits.start;
  79 }
  80
  81 uptr GetShadowSize() {
  82   return cfi_shadow_limits_storage.limits.size;
  83 }
  84
  85 // This will only work while the shadow is not allocated.
  86 void SetShadowSize(uptr size) {
  87   cfi_shadow_limits_storage.limits.size = size;
  88 }
  89
  90 uptr MemToShadowOffset(uptr x) {
  91   return (x >> kShadowGranularity) << 1;
  92 }
  93
  94 uint16_t *MemToShadow(uptr x, uptr shadow_base) {
  95   return (uint16_t *)(shadow_base + MemToShadowOffset(x));
  96 }
  97
  98 typedef int (*CFICheckFn)(u64, void *, void *);
  99
 100 // This class reads and decodes the shadow contents.
 101 class ShadowValue {
 102   uptr addr;
 103   uint16_t v;
 104   explicit ShadowValue(uptr addr, uint16_t v) : addr(addr), v(v) {}
 105
 106 public:
 107   bool is_invalid() const { return v == kInvalidShadow; }
 108
 109   bool is_unchecked() const { return v == kUncheckedShadow; }
 110
 111   CFICheckFn get_cfi_check() const {
 112     assert(!is_invalid() && !is_unchecked());
 113     uptr aligned_addr = addr & ~(kShadowAlign - 1);
 114     uptr p = aligned_addr - (((uptr)v - 1) << kShadowGranularity);
 115     return reinterpret_cast<CFICheckFn>(p);
 116   }
 117
 118   // Load a shadow value for the given application memory address.
 119   static const ShadowValue load(uptr addr) {
 120     uptr shadow_base = GetShadow();
 121     uptr shadow_offset = MemToShadowOffset(addr);
 122     if (shadow_offset > GetShadowSize())
 123       return ShadowValue(addr, kInvalidShadow);
 124     else
 125       return ShadowValue(
 126           addr, *reinterpret_cast<uint16_t *>(shadow_base + shadow_offset));
 127   }
 128 };
 129
 130 class ShadowBuilder {
 131   uptr shadow_;
 132
 133 public:
 134   // Allocate a new empty shadow (for the entire address space) on the side.
 135   void Start();
 136   // Mark the given address range as unchecked.
 137   // This is used for uninstrumented libraries like libc.
 138   // Any CFI check with a target in that range will pass.
 139   void AddUnchecked(uptr begin, uptr end);
 140   // Mark the given address range as belonging to a library with the given
 141   // cfi_check function.
 142   void Add(uptr begin, uptr end, uptr cfi_check);
 143   // Finish shadow construction. Atomically switch the current active shadow
 144   // region with the newly constructed one and deallocate the former.
 145   void Install();
 146 };
 147
 148 void ShadowBuilder::Start() {
 149   shadow_ = (uptr)MmapNoReserveOrDie(GetShadowSize(), "CFI shadow");
 150   VReport(1, "CFI: shadow at %zx .. %zx\n", shadow_, shadow_ + GetShadowSize());
 151 }
 152
 153 void ShadowBuilder::AddUnchecked(uptr begin, uptr end) {
 154   uint16_t *shadow_begin = MemToShadow(begin, shadow_);
 155   uint16_t *shadow_end = MemToShadow(end - 1, shadow_) + 1;
 156   // memset takes a byte, so our unchecked shadow value requires both bytes to
 157   // be the same. Make sure we're ok during compilation.
 158   static_assert((kUncheckedShadow & 0xff) == ((kUncheckedShadow >> 8) & 0xff),
 159                 "Both bytes of the 16-bit value must be the same!");
 160   memset(shadow_begin, kUncheckedShadow & 0xff,
 161          (shadow_end - shadow_begin) * sizeof(*shadow_begin));
 162 }
 163
 164 void ShadowBuilder::Add(uptr begin, uptr end, uptr cfi_check) {
 165   assert((cfi_check & (kShadowAlign - 1)) == 0);
 166
 167   // Don't fill anything below cfi_check. We can not represent those addresses
 168   // in the shadow, and must make sure at codegen to place all valid call
 169   // targets above cfi_check.
 170   begin = Max(begin, cfi_check);
 171   uint16_t *s = MemToShadow(begin, shadow_);
 172   uint16_t *s_end = MemToShadow(end - 1, shadow_) + 1;
 173   uint16_t sv = ((begin - cfi_check) >> kShadowGranularity) + 1;
 174   for (; s < s_end; s++, sv++)
 175     *s = sv;
 176 }
 177
 178 #if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD
 179 void ShadowBuilder::Install() {
 180   MprotectReadOnly(shadow_, GetShadowSize());
 181   uptr main_shadow = GetShadow();
 182   if (main_shadow) {
 183     // Update.
 184 #if SANITIZER_LINUX
 185     void *res = mremap((void *)shadow_, GetShadowSize(), GetShadowSize(),
 186                        MREMAP_MAYMOVE | MREMAP_FIXED, (void *)main_shadow);
 187     CHECK(res != MAP_FAILED);
 188 #elif SANITIZER_NETBSD
 189     void *res = mremap((void *)shadow_, GetShadowSize(), (void *)main_shadow,
 190                        GetShadowSize(), MAP_FIXED);
 191     CHECK(res != MAP_FAILED);
 192 #else
 193     void *res = MmapFixedOrDie(shadow_, GetShadowSize(), "cfi shadow");
 194     CHECK(res != MAP_FAILED);
 195     ::memcpy(&shadow_, &main_shadow, GetShadowSize());
 196 #endif
 197   } else {
 198     // Initial setup.
 199     CHECK_EQ(kCfiShadowLimitsStorageSize, GetPageSizeCached());
 200     CHECK_EQ(0, GetShadow());
 201     cfi_shadow_limits_storage.limits.start = shadow_;
 202     MprotectReadOnly((uptr)&cfi_shadow_limits_storage,
 203                      sizeof(cfi_shadow_limits_storage));
 204     CHECK_EQ(shadow_, GetShadow());
 205   }
 206 }
 207 #else
 208 #error not implemented
 209 #endif
 210
 211 // This is a workaround for a glibc bug:
 212 // https://sourceware.org/bugzilla/show_bug.cgi?id=15199
 213 // Other platforms can, hopefully, just do
 214 //    dlopen(RTLD_NOLOAD | RTLD_LAZY)
 215 //    dlsym("__cfi_check").
 216 uptr find_cfi_check_in_dso(dl_phdr_info *info) {
 217   const Elf_Dyn *dynamic = nullptr;
 218   for (int i = 0; i < info->dlpi_phnum; ++i) {
 219     if (info->dlpi_phdr[i].p_type == PT_DYNAMIC) {
 220       dynamic =
 221           (const Elf_Dyn *)(info->dlpi_addr + info->dlpi_phdr[i].p_vaddr);
 222       break;
 223     }
 224   }
 225   if (!dynamic) return 0;
 226   uptr strtab = 0, symtab = 0, strsz = 0;
 227   for (const Elf_Dyn *p = dynamic; p->d_tag != PT_NULL; ++p) {
 228     if (p->d_tag == DT_SYMTAB)
 229       symtab = p->d_un.d_ptr;
 230     else if (p->d_tag == DT_STRTAB)
 231       strtab = p->d_un.d_ptr;
 232     else if (p->d_tag == DT_STRSZ)
 233       strsz = p->d_un.d_ptr;
 234   }
 235
 236   if (symtab > strtab) {
 237     VReport(1, "Can not handle: symtab > strtab (%zx > %zx)\n", symtab, strtab);
 238     return 0;
 239   }
 240
 241   // Verify that strtab and symtab are inside of the same LOAD segment.
 242   // This excludes VDSO, which has (very high) bogus strtab and symtab pointers.
 243   int phdr_idx;
 244   for (phdr_idx = 0; phdr_idx < info->dlpi_phnum; phdr_idx++) {
 245     const Elf_Phdr *phdr = &info->dlpi_phdr[phdr_idx];
 246     if (phdr->p_type == PT_LOAD) {
 247       uptr beg = info->dlpi_addr + phdr->p_vaddr;
 248       uptr end = beg + phdr->p_memsz;
 249       if (strtab >= beg && strtab + strsz < end && symtab >= beg &&
 250           symtab < end)
 251         break;
 252     }
 253   }
 254   if (phdr_idx == info->dlpi_phnum) {
 255     // Nope, either different segments or just bogus pointers.
 256     // Can not handle this.
 257     VReport(1, "Can not handle: symtab %zx, strtab %zx\n", symtab, strtab);
 258     return 0;
 259   }
 260
 261   for (const Elf_Sym *p = (const Elf_Sym *)symtab; (Elf_Addr)p < strtab;
 262        ++p) {
 263     // There is no reliable way to find the end of the symbol table. In
 264     // lld-produces files, there are other sections between symtab and strtab.
 265     // Stop looking when the symbol name is not inside strtab.
 266     if (p->st_name >= strsz) break;
 267     char *name = (char*)(strtab + p->st_name);
 268     if (strcmp(name, "__cfi_check") == 0) {
 269       assert(p->st_info == ELF32_ST_INFO(STB_GLOBAL, STT_FUNC) ||
 270              p->st_info == ELF32_ST_INFO(STB_WEAK, STT_FUNC));
 271       uptr addr = info->dlpi_addr + p->st_value;
 272       return addr;
 273     }
 274   }
 275   return 0;
 276 }
 277
 278 int dl_iterate_phdr_cb(dl_phdr_info *info, size_t size, void *data) {
 279   uptr cfi_check = find_cfi_check_in_dso(info);
 280   if (cfi_check)
 281     VReport(1, "Module '%s' __cfi_check %zx\n", info->dlpi_name, cfi_check);
 282
 283   ShadowBuilder *b = reinterpret_cast<ShadowBuilder *>(data);
 284
 285   for (int i = 0; i < info->dlpi_phnum; i++) {
 286     const Elf_Phdr *phdr = &info->dlpi_phdr[i];
 287     if (phdr->p_type == PT_LOAD) {
 288       // Jump tables are in the executable segment.
 289       // VTables are in the non-executable one.
 290       // Need to fill shadow for both.
 291       // FIXME: reject writable if vtables are in the r/o segment. Depend on
 292       // PT_RELRO?
 293       uptr cur_beg = info->dlpi_addr + phdr->p_vaddr;
 294       uptr cur_end = cur_beg + phdr->p_memsz;
 295       if (cfi_check) {
 296         VReport(1, "   %zx .. %zx\n", cur_beg, cur_end);
 297         b->Add(cur_beg, cur_end, cfi_check);
 298       } else {
 299         b->AddUnchecked(cur_beg, cur_end);
 300       }
 301     }
 302   }
 303   return 0;
 304 }
 305
 306 // Init or update shadow for the current set of loaded libraries.
 307 void UpdateShadow() {
 308   ShadowBuilder b;
 309   b.Start();
 310   dl_iterate_phdr(dl_iterate_phdr_cb, &b);
 311   b.Install();
 312 }
 313
 314 void InitShadow() {
 315   CHECK_EQ(0, GetShadow());
 316   CHECK_EQ(0, GetShadowSize());
 317
 318   uptr vma = GetMaxUserVirtualAddress();
 319   // Shadow is 2 -> 2**kShadowGranularity.
 320   SetShadowSize((vma >> (kShadowGranularity - 1)) + 1);
 321   VReport(1, "CFI: VMA size %zx, shadow size %zx\n", vma, GetShadowSize());
 322
 323   UpdateShadow();
 324 }
 325
 326 THREADLOCAL int in_loader;
 327 Mutex shadow_update_lock;
 328
 329 void EnterLoader() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
 330   if (in_loader == 0) {
 331     shadow_update_lock.Lock();
 332   }
 333   ++in_loader;
 334 }
 335
 336 void ExitLoader() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
 337   CHECK(in_loader > 0);
 338   --in_loader;
 339   UpdateShadow();
 340   if (in_loader == 0) {
 341     shadow_update_lock.Unlock();
 342   }
 343 }
 344
 345 ALWAYS_INLINE void CfiSlowPathCommon(u64 CallSiteTypeId, void *Ptr,
 346                                      void *DiagData) {
 347   uptr Addr = (uptr)Ptr;
 348   VReport(3, "__cfi_slowpath: %llx, %p\n", CallSiteTypeId, Ptr);
 349   ShadowValue sv = ShadowValue::load(Addr);
 350   if (sv.is_invalid()) {
 351     VReport(1, "CFI: invalid memory region for a check target: %p\n", Ptr);
 352 #ifdef CFI_ENABLE_DIAG
 353     if (DiagData) {
 354       __ubsan_handle_cfi_check_fail(
 355           reinterpret_cast<__ubsan::CFICheckFailData *>(DiagData), Addr, false);
 356       return;
 357     }
 358 #endif
 359     Trap();
 360   }
 361   if (sv.is_unchecked()) {
 362     VReport(2, "CFI: unchecked call (shadow=FFFF): %p\n", Ptr);
 363     return;
 364   }
 365   CFICheckFn cfi_check = sv.get_cfi_check();
 366   VReport(2, "__cfi_check at %p\n", (void *)cfi_check);
 367   cfi_check(CallSiteTypeId, Ptr, DiagData);
 368 }
 369
 370 void InitializeFlags() {
 371   SetCommonFlagsDefaults();
 372 #ifdef CFI_ENABLE_DIAG
 373   __ubsan::Flags *uf = __ubsan::flags();
 374   uf->SetDefaults();
 375 #endif
 376
 377   FlagParser cfi_parser;
 378   RegisterCommonFlags(&cfi_parser);
 379   cfi_parser.ParseStringFromEnv("CFI_OPTIONS");
 380
 381 #ifdef CFI_ENABLE_DIAG
 382   FlagParser ubsan_parser;
 383   __ubsan::RegisterUbsanFlags(&ubsan_parser, uf);
 384   RegisterCommonFlags(&ubsan_parser);
 385
 386   const char *ubsan_default_options = __ubsan_default_options();
 387   ubsan_parser.ParseString(ubsan_default_options);
 388   ubsan_parser.ParseStringFromEnv("UBSAN_OPTIONS");
 389 #endif
 390
 391   InitializeCommonFlags();
 392
 393   if (Verbosity())
 394     ReportUnrecognizedFlags();
 395
 396   if (common_flags()->help) {
 397     cfi_parser.PrintFlagDescriptions();
 398   }
 399 }
 400
 401 } // namespace __cfi
 402
 403 using namespace __cfi;
 404
 405 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
 406 __cfi_slowpath(u64 CallSiteTypeId, void *Ptr) {
 407   CfiSlowPathCommon(CallSiteTypeId, Ptr, nullptr);
 408 }
 409
 410 #ifdef CFI_ENABLE_DIAG
 411 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
 412 __cfi_slowpath_diag(u64 CallSiteTypeId, void *Ptr, void *DiagData) {
 413   CfiSlowPathCommon(CallSiteTypeId, Ptr, DiagData);
 414 }
 415 #endif
 416
 417 static void EnsureInterceptorsInitialized();
 418
 419 // Setup shadow for dlopen()ed libraries.
 420 // The actual shadow setup happens after dlopen() returns, which means that
 421 // a library can not be a target of any CFI checks while its constructors are
 422 // running. It's unclear how to fix this without some extra help from libc.
 423 // In glibc, mmap inside dlopen is not interceptable.
 424 // Maybe a seccomp-bpf filter?
 425 // We could insert a high-priority constructor into the library, but that would
 426 // not help with the uninstrumented libraries.
 427 INTERCEPTOR(void*, dlopen, const char *filename, int flag) {
 428   EnsureInterceptorsInitialized();
 429   EnterLoader();
 430   void *handle = REAL(dlopen)(filename, flag);
 431   ExitLoader();
 432   return handle;
 433 }
 434
 435 INTERCEPTOR(int, dlclose, void *handle) {
 436   EnsureInterceptorsInitialized();
 437   EnterLoader();
 438   int res = REAL(dlclose)(handle);
 439   ExitLoader();
 440   return res;
 441 }
 442
 443 static Mutex interceptor_init_lock;
 444 static bool interceptors_inited = false;
 445
 446 static void EnsureInterceptorsInitialized() {
 447   Lock lock(&interceptor_init_lock);
 448   if (interceptors_inited)
 449     return;
 450
 451   INTERCEPT_FUNCTION(dlopen);
 452   INTERCEPT_FUNCTION(dlclose);
 453
 454   interceptors_inited = true;
 455 }
 456
 457 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
 458 #if !SANITIZER_CAN_USE_PREINIT_ARRAY
 459 // On ELF platforms, the constructor is invoked using .preinit_array (see below)
 460 __attribute__((constructor(0)))
 461 #endif
 462 void __cfi_init() {
 463   SanitizerToolName = "CFI";
 464   InitializeFlags();
 465   InitShadow();
 466
 467 #ifdef CFI_ENABLE_DIAG
 468   __ubsan::InitAsPlugin();
 469 #endif
 470 }
 471
 472 #if SANITIZER_CAN_USE_PREINIT_ARRAY
 473 // On ELF platforms, run cfi initialization before any other constructors.
 474 // On other platforms we use the constructor attribute to arrange to run our
 475 // initialization early.
 476 extern "C" {
 477 __attribute__((section(".preinit_array"),
 478                used)) void (*__cfi_preinit)(void) = __cfi_init;
 479 }
 480 #endif