compiler-rt/lib/dfsan/dfsan.cpp

   1 //===-- dfsan.cpp ---------------------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file is a part of DataFlowSanitizer.
  10 //
  11 // DataFlowSanitizer runtime.  This file defines the public interface to
  12 // DataFlowSanitizer as well as the definition of certain runtime functions
  13 // called automatically by the compiler (specifically the instrumentation pass
  14 // in llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp).
  15 //
  16 // The public interface is defined in include/sanitizer/dfsan_interface.h whose
  17 // functions are prefixed dfsan_ while the compiler interface functions are
  18 // prefixed __dfsan_.
  19 //===----------------------------------------------------------------------===//
  20
  21 #include "dfsan/dfsan.h"
  22
  23 #include "dfsan/dfsan_chained_origin_depot.h"
  24 #include "dfsan/dfsan_flags.h"
  25 #include "dfsan/dfsan_origin.h"
  26 #include "dfsan/dfsan_thread.h"
  27 #include "sanitizer_common/sanitizer_atomic.h"
  28 #include "sanitizer_common/sanitizer_common.h"
  29 #include "sanitizer_common/sanitizer_file.h"
  30 #include "sanitizer_common/sanitizer_flag_parser.h"
  31 #include "sanitizer_common/sanitizer_flags.h"
  32 #include "sanitizer_common/sanitizer_internal_defs.h"
  33 #include "sanitizer_common/sanitizer_libc.h"
  34 #include "sanitizer_common/sanitizer_report_decorator.h"
  35 #include "sanitizer_common/sanitizer_stacktrace.h"
  36
  37 using namespace __dfsan;
  38
  39 Flags __dfsan::flags_data;
  40
  41 // The size of TLS variables. These constants must be kept in sync with the ones
  42 // in DataFlowSanitizer.cpp.
  43 static const int kDFsanArgTlsSize = 800;
  44 static const int kDFsanRetvalTlsSize = 800;
  45 static const int kDFsanArgOriginTlsSize = 800;
  46
  47 SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL u64
  48     __dfsan_retval_tls[kDFsanRetvalTlsSize / sizeof(u64)];
  49 SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL u32 __dfsan_retval_origin_tls;
  50 SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL u64
  51     __dfsan_arg_tls[kDFsanArgTlsSize / sizeof(u64)];
  52 SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL u32
  53     __dfsan_arg_origin_tls[kDFsanArgOriginTlsSize / sizeof(u32)];
  54
  55 // Instrumented code may set this value in terms of -dfsan-track-origins.
  56 // * undefined or 0: do not track origins.
  57 // * 1: track origins at memory store operations.
  58 // * 2: track origins at memory load and store operations.
  59 //      TODO: track callsites.
  60 extern "C" SANITIZER_WEAK_ATTRIBUTE const int __dfsan_track_origins;
  61
  62 extern "C" SANITIZER_INTERFACE_ATTRIBUTE int dfsan_get_track_origins() {
  63   return &__dfsan_track_origins ? __dfsan_track_origins : 0;
  64 }
  65
  66 // On Linux/x86_64, memory is laid out as follows:
  67 //
  68 //  +--------------------+ 0x800000000000 (top of memory)
  69 //  |    application 3   |
  70 //  +--------------------+ 0x700000000000
  71 //  |      invalid       |
  72 //  +--------------------+ 0x610000000000
  73 //  |      origin 1      |
  74 //  +--------------------+ 0x600000000000
  75 //  |    application 2   |
  76 //  +--------------------+ 0x510000000000
  77 //  |      shadow 1      |
  78 //  +--------------------+ 0x500000000000
  79 //  |      invalid       |
  80 //  +--------------------+ 0x400000000000
  81 //  |      origin 3      |
  82 //  +--------------------+ 0x300000000000
  83 //  |      shadow 3      |
  84 //  +--------------------+ 0x200000000000
  85 //  |      origin 2      |
  86 //  +--------------------+ 0x110000000000
  87 //  |      invalid       |
  88 //  +--------------------+ 0x100000000000
  89 //  |      shadow 2      |
  90 //  +--------------------+ 0x010000000000
  91 //  |    application 1   |
  92 //  +--------------------+ 0x000000000000
  93 //
  94 //  MEM_TO_SHADOW(mem) = mem ^ 0x500000000000
  95 //  SHADOW_TO_ORIGIN(shadow) = shadow + 0x100000000000
  96
  97 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
  98 dfsan_label __dfsan_union_load(const dfsan_label *ls, uptr n) {
  99   dfsan_label label = ls[0];
 100   for (uptr i = 1; i != n; ++i)
 101     label |= ls[i];
 102   return label;
 103 }
 104
 105 // Return the union of all the n labels from addr at the high 32 bit, and the
 106 // origin of the first taint byte at the low 32 bit.
 107 extern "C" SANITIZER_INTERFACE_ATTRIBUTE u64
 108 __dfsan_load_label_and_origin(const void *addr, uptr n) {
 109   dfsan_label label = 0;
 110   u64 ret = 0;
 111   uptr p = (uptr)addr;
 112   dfsan_label *s = shadow_for((void *)p);
 113   for (uptr i = 0; i < n; ++i) {
 114     dfsan_label l = s[i];
 115     if (!l)
 116       continue;
 117     label |= l;
 118     if (!ret)
 119       ret = *(dfsan_origin *)origin_for((void *)(p + i));
 120   }
 121   return ret | (u64)label << 32;
 122 }
 123
 124 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
 125 void __dfsan_unimplemented(char *fname) {
 126   if (flags().warn_unimplemented)
 127     Report("WARNING: DataFlowSanitizer: call to uninstrumented function %s\n",
 128            fname);
 129 }
 130
 131 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __dfsan_wrapper_extern_weak_null(
 132     const void *addr, char *fname) {
 133   if (!addr)
 134     Report(
 135         "ERROR: DataFlowSanitizer: dfsan generated wrapper calling null "
 136         "extern_weak function %s\nIf this only happens with dfsan, the "
 137         "dfsan instrumentation pass may be accidentally optimizing out a "
 138         "null check\n",
 139         fname);
 140 }
 141
 142 // Use '-mllvm -dfsan-debug-nonzero-labels' and break on this function
 143 // to try to figure out where labels are being introduced in a nominally
 144 // label-free program.
 145 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __dfsan_nonzero_label() {
 146   if (flags().warn_nonzero_labels)
 147     Report("WARNING: DataFlowSanitizer: saw nonzero label\n");
 148 }
 149
 150 // Indirect call to an uninstrumented vararg function. We don't have a way of
 151 // handling these at the moment.
 152 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
 153 __dfsan_vararg_wrapper(const char *fname) {
 154   Report("FATAL: DataFlowSanitizer: unsupported indirect call to vararg "
 155          "function %s\n", fname);
 156   Die();
 157 }
 158
 159 // Resolves the union of two labels.
 160 SANITIZER_INTERFACE_ATTRIBUTE dfsan_label
 161 dfsan_union(dfsan_label l1, dfsan_label l2) {
 162   return l1 | l2;
 163 }
 164
 165 static const uptr kOriginAlign = sizeof(dfsan_origin);
 166 static const uptr kOriginAlignMask = ~(kOriginAlign - 1UL);
 167
 168 static uptr OriginAlignUp(uptr u) {
 169   return (u + kOriginAlign - 1) & kOriginAlignMask;
 170 }
 171
 172 static uptr OriginAlignDown(uptr u) { return u & kOriginAlignMask; }
 173
 174 // Return the origin of the first taint byte in the size bytes from the address
 175 // addr.
 176 static dfsan_origin GetOriginIfTainted(uptr addr, uptr size) {
 177   for (uptr i = 0; i < size; ++i, ++addr) {
 178     dfsan_label *s = shadow_for((void *)addr);
 179
 180     if (*s) {
 181       // Validate address region.
 182       CHECK(MEM_IS_SHADOW(s));
 183       return *(dfsan_origin *)origin_for((void *)addr);
 184     }
 185   }
 186   return 0;
 187 }
 188
 189 // For platforms which support slow unwinder only, we need to restrict the store
 190 // context size to 1, basically only storing the current pc, because the slow
 191 // unwinder which is based on libunwind is not async signal safe and causes
 192 // random freezes in forking applications as well as in signal handlers.
 193 // DFSan supports only Linux. So we do not restrict the store context size.
 194 #define GET_STORE_STACK_TRACE_PC_BP(pc, bp) \
 195   BufferedStackTrace stack;                 \
 196   stack.Unwind(pc, bp, nullptr, true, flags().store_context_size);
 197
 198 #define PRINT_CALLER_STACK_TRACE        \
 199   {                                     \
 200     GET_CALLER_PC_BP;                   \
 201     GET_STORE_STACK_TRACE_PC_BP(pc, bp) \
 202     stack.Print();                      \
 203   }
 204
 205 // Return a chain with the previous ID id and the current stack.
 206 // from_init = true if this is the first chain of an origin tracking path.
 207 static u32 ChainOrigin(u32 id, StackTrace *stack, bool from_init = false) {
 208   // StackDepot is not async signal safe. Do not create new chains in a signal
 209   // handler.
 210   DFsanThread *t = GetCurrentThread();
 211   if (t && t->InSignalHandler())
 212     return id;
 213
 214   // As an optimization the origin of an application byte is updated only when
 215   // its shadow is non-zero. Because we are only interested in the origins of
 216   // taint labels, it does not matter what origin a zero label has. This reduces
 217   // memory write cost. MSan does similar optimization. The following invariant
 218   // may not hold because of some bugs. We check the invariant to help debug.
 219   if (!from_init && id == 0 && flags().check_origin_invariant) {
 220     Printf("  DFSan found invalid origin invariant\n");
 221     PRINT_CALLER_STACK_TRACE
 222   }
 223
 224   Origin o = Origin::FromRawId(id);
 225   stack->tag = StackTrace::TAG_UNKNOWN;
 226   Origin chained = Origin::CreateChainedOrigin(o, stack);
 227   return chained.raw_id();
 228 }
 229
 230 static void ChainAndWriteOriginIfTainted(uptr src, uptr size, uptr dst,
 231                                          StackTrace *stack) {
 232   dfsan_origin o = GetOriginIfTainted(src, size);
 233   if (o) {
 234     o = ChainOrigin(o, stack);
 235     *(dfsan_origin *)origin_for((void *)dst) = o;
 236   }
 237 }
 238
 239 // Copy the origins of the size bytes from src to dst. The source and target
 240 // memory ranges cannot be overlapped. This is used by memcpy. stack records the
 241 // stack trace of the memcpy. When dst and src are not 4-byte aligned properly,
 242 // origins at the unaligned address boundaries may be overwritten because four
 243 // contiguous bytes share the same origin.
 244 static void CopyOrigin(const void *dst, const void *src, uptr size,
 245                        StackTrace *stack) {
 246   uptr d = (uptr)dst;
 247   uptr beg = OriginAlignDown(d);
 248   // Copy left unaligned origin if that memory is tainted.
 249   if (beg < d) {
 250     ChainAndWriteOriginIfTainted((uptr)src, beg + kOriginAlign - d, beg, stack);
 251     beg += kOriginAlign;
 252   }
 253
 254   uptr end = OriginAlignDown(d + size);
 255   // If both ends fall into the same 4-byte slot, we are done.
 256   if (end < beg)
 257     return;
 258
 259   // Copy right unaligned origin if that memory is tainted.
 260   if (end < d + size)
 261     ChainAndWriteOriginIfTainted((uptr)src + (end - d), (d + size) - end, end,
 262                                  stack);
 263
 264   if (beg >= end)
 265     return;
 266
 267   // Align src up.
 268   uptr src_a = OriginAlignUp((uptr)src);
 269   dfsan_origin *src_o = origin_for((void *)src_a);
 270   u32 *src_s = (u32 *)shadow_for((void *)src_a);
 271   dfsan_origin *src_end = origin_for((void *)(src_a + (end - beg)));
 272   dfsan_origin *dst_o = origin_for((void *)beg);
 273   dfsan_origin last_src_o = 0;
 274   dfsan_origin last_dst_o = 0;
 275   for (; src_o < src_end; ++src_o, ++src_s, ++dst_o) {
 276     if (!*src_s)
 277       continue;
 278     if (*src_o != last_src_o) {
 279       last_src_o = *src_o;
 280       last_dst_o = ChainOrigin(last_src_o, stack);
 281     }
 282     *dst_o = last_dst_o;
 283   }
 284 }
 285
 286 // Copy the origins of the size bytes from src to dst. The source and target
 287 // memory ranges may be overlapped. So the copy is done in a reverse order.
 288 // This is used by memmove. stack records the stack trace of the memmove.
 289 static void ReverseCopyOrigin(const void *dst, const void *src, uptr size,
 290                               StackTrace *stack) {
 291   uptr d = (uptr)dst;
 292   uptr end = OriginAlignDown(d + size);
 293
 294   // Copy right unaligned origin if that memory is tainted.
 295   if (end < d + size)
 296     ChainAndWriteOriginIfTainted((uptr)src + (end - d), (d + size) - end, end,
 297                                  stack);
 298
 299   uptr beg = OriginAlignDown(d);
 300
 301   if (beg + kOriginAlign < end) {
 302     // Align src up.
 303     uptr src_a = OriginAlignUp((uptr)src);
 304     void *src_end = (void *)(src_a + end - beg - kOriginAlign);
 305     dfsan_origin *src_end_o = origin_for(src_end);
 306     u32 *src_end_s = (u32 *)shadow_for(src_end);
 307     dfsan_origin *src_begin_o = origin_for((void *)src_a);
 308     dfsan_origin *dst = origin_for((void *)(end - kOriginAlign));
 309     dfsan_origin last_src_o = 0;
 310     dfsan_origin last_dst_o = 0;
 311     for (; src_end_o >= src_begin_o; --src_end_o, --src_end_s, --dst) {
 312       if (!*src_end_s)
 313         continue;
 314       if (*src_end_o != last_src_o) {
 315         last_src_o = *src_end_o;
 316         last_dst_o = ChainOrigin(last_src_o, stack);
 317       }
 318       *dst = last_dst_o;
 319     }
 320   }
 321
 322   // Copy left unaligned origin if that memory is tainted.
 323   if (beg < d)
 324     ChainAndWriteOriginIfTainted((uptr)src, beg + kOriginAlign - d, beg, stack);
 325 }
 326
 327 // Copy or move the origins of the len bytes from src to dst. The source and
 328 // target memory ranges may or may not be overlapped. This is used by memory
 329 // transfer operations. stack records the stack trace of the memory transfer
 330 // operation.
 331 static void MoveOrigin(const void *dst, const void *src, uptr size,
 332                        StackTrace *stack) {
 333   // Validate address regions.
 334   if (!MEM_IS_SHADOW(shadow_for(dst)) ||
 335       !MEM_IS_SHADOW(shadow_for((void *)((uptr)dst + size))) ||
 336       !MEM_IS_SHADOW(shadow_for(src)) ||
 337       !MEM_IS_SHADOW(shadow_for((void *)((uptr)src + size)))) {
 338     CHECK(false);
 339     return;
 340   }
 341   // If destination origin range overlaps with source origin range, move
 342   // origins by copying origins in a reverse order; otherwise, copy origins in
 343   // a normal order. The orders of origin transfer are consistent with the
 344   // orders of how memcpy and memmove transfer user data.
 345   uptr src_aligned_beg = OriginAlignDown((uptr)src);
 346   uptr src_aligned_end = OriginAlignDown((uptr)src + size);
 347   uptr dst_aligned_beg = OriginAlignDown((uptr)dst);
 348   if (dst_aligned_beg < src_aligned_end && dst_aligned_beg >= src_aligned_beg)
 349     return ReverseCopyOrigin(dst, src, size, stack);
 350   return CopyOrigin(dst, src, size, stack);
 351 }
 352
 353 // Set the size bytes from the addres dst to be the origin value.
 354 static void SetOrigin(const void *dst, uptr size, u32 origin) {
 355   if (size == 0)
 356     return;
 357
 358   // Origin mapping is 4 bytes per 4 bytes of application memory.
 359   // Here we extend the range such that its left and right bounds are both
 360   // 4 byte aligned.
 361   uptr x = unaligned_origin_for((uptr)dst);
 362   uptr beg = OriginAlignDown(x);
 363   uptr end = OriginAlignUp(x + size);  // align up.
 364   u64 origin64 = ((u64)origin << 32) | origin;
 365   // This is like memset, but the value is 32-bit. We unroll by 2 to write
 366   // 64 bits at once. May want to unroll further to get 128-bit stores.
 367   if (beg & 7ULL) {
 368     if (*(u32 *)beg != origin)
 369       *(u32 *)beg = origin;
 370     beg += 4;
 371   }
 372   for (uptr addr = beg; addr < (end & ~7UL); addr += 8) {
 373     if (*(u64 *)addr == origin64)
 374       continue;
 375     *(u64 *)addr = origin64;
 376   }
 377   if (end & 7ULL)
 378     if (*(u32 *)(end - kOriginAlign) != origin)
 379       *(u32 *)(end - kOriginAlign) = origin;
 380 }
 381
 382 #define RET_CHAIN_ORIGIN(id)           \
 383   GET_CALLER_PC_BP;                    \
 384   GET_STORE_STACK_TRACE_PC_BP(pc, bp); \
 385   return ChainOrigin(id, &stack);
 386
 387 // Return a new origin chain with the previous ID id and the current stack
 388 // trace.
 389 extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_origin
 390 __dfsan_chain_origin(dfsan_origin id) {
 391   RET_CHAIN_ORIGIN(id)
 392 }
 393
 394 // Return a new origin chain with the previous ID id and the current stack
 395 // trace if the label is tainted.
 396 extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_origin
 397 __dfsan_chain_origin_if_tainted(dfsan_label label, dfsan_origin id) {
 398   if (!label)
 399     return id;
 400   RET_CHAIN_ORIGIN(id)
 401 }
 402
 403 // Copy or move the origins of the len bytes from src to dst.
 404 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __dfsan_mem_origin_transfer(
 405     const void *dst, const void *src, uptr len) {
 406   if (src == dst)
 407     return;
 408   GET_CALLER_PC_BP;
 409   GET_STORE_STACK_TRACE_PC_BP(pc, bp);
 410   MoveOrigin(dst, src, len, &stack);
 411 }
 412
 413 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void dfsan_mem_origin_transfer(
 414     const void *dst, const void *src, uptr len) {
 415   __dfsan_mem_origin_transfer(dst, src, len);
 416 }
 417
 418 static void CopyShadow(void *dst, const void *src, uptr len) {
 419   internal_memcpy((void *)__dfsan::shadow_for(dst),
 420                   (const void *)__dfsan::shadow_for(src),
 421                   len * sizeof(dfsan_label));
 422 }
 423
 424 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void dfsan_mem_shadow_transfer(
 425     void *dst, const void *src, uptr len) {
 426   CopyShadow(dst, src, len);
 427 }
 428
 429 // Copy shadow and origins of the len bytes from src to dst.
 430 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
 431 __dfsan_mem_shadow_origin_transfer(void *dst, const void *src, uptr size) {
 432   if (src == dst)
 433     return;
 434   CopyShadow(dst, src, size);
 435   if (dfsan_get_track_origins()) {
 436     // Duplicating code instead of calling __dfsan_mem_origin_transfer
 437     // so that the getting the caller stack frame works correctly.
 438     GET_CALLER_PC_BP;
 439     GET_STORE_STACK_TRACE_PC_BP(pc, bp);
 440     MoveOrigin(dst, src, size, &stack);
 441   }
 442 }
 443
 444 // Copy shadow and origins as per __atomic_compare_exchange.
 445 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
 446 __dfsan_mem_shadow_origin_conditional_exchange(u8 condition, void *target,
 447                                                void *expected,
 448                                                const void *desired, uptr size) {
 449   void *dst;
 450   const void *src;
 451   // condition is result of native call to __atomic_compare_exchange
 452   if (condition) {
 453     // Copy desired into target
 454     dst = target;
 455     src = desired;
 456   } else {
 457     // Copy target into expected
 458     dst = expected;
 459     src = target;
 460   }
 461   if (src == dst)
 462     return;
 463   CopyShadow(dst, src, size);
 464   if (dfsan_get_track_origins()) {
 465     // Duplicating code instead of calling __dfsan_mem_origin_transfer
 466     // so that the getting the caller stack frame works correctly.
 467     GET_CALLER_PC_BP;
 468     GET_STORE_STACK_TRACE_PC_BP(pc, bp);
 469     MoveOrigin(dst, src, size, &stack);
 470   }
 471 }
 472
 473 namespace __dfsan {
 474
 475 bool dfsan_inited = false;
 476 bool dfsan_init_is_running = false;
 477
 478 void dfsan_copy_memory(void *dst, const void *src, uptr size) {
 479   internal_memcpy(dst, src, size);
 480   dfsan_mem_shadow_transfer(dst, src, size);
 481   if (dfsan_get_track_origins())
 482     dfsan_mem_origin_transfer(dst, src, size);
 483 }
 484
 485 // Releases the pages within the origin address range.
 486 static void ReleaseOrigins(void *addr, uptr size) {
 487   const uptr beg_origin_addr = (uptr)__dfsan::origin_for(addr);
 488   const void *end_addr = (void *)((uptr)addr + size);
 489   const uptr end_origin_addr = (uptr)__dfsan::origin_for(end_addr);
 490
 491   if (end_origin_addr - beg_origin_addr <
 492       common_flags()->clear_shadow_mmap_threshold)
 493     return;
 494
 495   const uptr page_size = GetPageSizeCached();
 496   const uptr beg_aligned = RoundUpTo(beg_origin_addr, page_size);
 497   const uptr end_aligned = RoundDownTo(end_origin_addr, page_size);
 498
 499   if (!MmapFixedSuperNoReserve(beg_aligned, end_aligned - beg_aligned))
 500     Die();
 501 }
 502
 503 static void WriteZeroShadowInRange(uptr beg, uptr end) {
 504   // Don't write the label if it is already the value we need it to be.
 505   // In a program where most addresses are not labeled, it is common that
 506   // a page of shadow memory is entirely zeroed.  The Linux copy-on-write
 507   // implementation will share all of the zeroed pages, making a copy of a
 508   // page when any value is written.  The un-sharing will happen even if
 509   // the value written does not change the value in memory.  Avoiding the
 510   // write when both |label| and |*labelp| are zero dramatically reduces
 511   // the amount of real memory used by large programs.
 512   if (!mem_is_zero((const char *)beg, end - beg))
 513     internal_memset((void *)beg, 0, end - beg);
 514 }
 515
 516 // Releases the pages within the shadow address range, and sets
 517 // the shadow addresses not on the pages to be 0.
 518 static void ReleaseOrClearShadows(void *addr, uptr size) {
 519   const uptr beg_shadow_addr = (uptr)__dfsan::shadow_for(addr);
 520   const void *end_addr = (void *)((uptr)addr + size);
 521   const uptr end_shadow_addr = (uptr)__dfsan::shadow_for(end_addr);
 522
 523   if (end_shadow_addr - beg_shadow_addr <
 524       common_flags()->clear_shadow_mmap_threshold) {
 525     WriteZeroShadowInRange(beg_shadow_addr, end_shadow_addr);
 526     return;
 527   }
 528
 529   const uptr page_size = GetPageSizeCached();
 530   const uptr beg_aligned = RoundUpTo(beg_shadow_addr, page_size);
 531   const uptr end_aligned = RoundDownTo(end_shadow_addr, page_size);
 532
 533   if (beg_aligned >= end_aligned) {
 534     WriteZeroShadowInRange(beg_shadow_addr, end_shadow_addr);
 535   } else {
 536     if (beg_aligned != beg_shadow_addr)
 537       WriteZeroShadowInRange(beg_shadow_addr, beg_aligned);
 538     if (end_aligned != end_shadow_addr)
 539       WriteZeroShadowInRange(end_aligned, end_shadow_addr);
 540     if (!MmapFixedSuperNoReserve(beg_aligned, end_aligned - beg_aligned))
 541       Die();
 542   }
 543 }
 544
 545 void SetShadow(dfsan_label label, void *addr, uptr size, dfsan_origin origin) {
 546   if (0 != label) {
 547     const uptr beg_shadow_addr = (uptr)__dfsan::shadow_for(addr);
 548     internal_memset((void *)beg_shadow_addr, label, size);
 549     if (dfsan_get_track_origins())
 550       SetOrigin(addr, size, origin);
 551     return;
 552   }
 553
 554   if (dfsan_get_track_origins())
 555     ReleaseOrigins(addr, size);
 556
 557   ReleaseOrClearShadows(addr, size);
 558 }
 559
 560 }  // namespace __dfsan
 561
 562 // If the label s is tainted, set the size bytes from the address p to be a new
 563 // origin chain with the previous ID o and the current stack trace. This is
 564 // used by instrumentation to reduce code size when too much code is inserted.
 565 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __dfsan_maybe_store_origin(
 566     dfsan_label s, void *p, uptr size, dfsan_origin o) {
 567   if (UNLIKELY(s)) {
 568     GET_CALLER_PC_BP;
 569     GET_STORE_STACK_TRACE_PC_BP(pc, bp);
 570     SetOrigin(p, size, ChainOrigin(o, &stack));
 571   }
 572 }
 573
 574 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __dfsan_set_label(
 575     dfsan_label label, dfsan_origin origin, void *addr, uptr size) {
 576   __dfsan::SetShadow(label, addr, size, origin);
 577 }
 578
 579 SANITIZER_INTERFACE_ATTRIBUTE
 580 void dfsan_set_label(dfsan_label label, void *addr, uptr size) {
 581   dfsan_origin init_origin = 0;
 582   if (label && dfsan_get_track_origins()) {
 583     GET_CALLER_PC_BP;
 584     GET_STORE_STACK_TRACE_PC_BP(pc, bp);
 585     init_origin = ChainOrigin(0, &stack, true);
 586   }
 587   __dfsan::SetShadow(label, addr, size, init_origin);
 588 }
 589
 590 SANITIZER_INTERFACE_ATTRIBUTE
 591 void dfsan_add_label(dfsan_label label, void *addr, uptr size) {
 592   if (0 == label)
 593     return;
 594
 595   if (dfsan_get_track_origins()) {
 596     GET_CALLER_PC_BP;
 597     GET_STORE_STACK_TRACE_PC_BP(pc, bp);
 598     dfsan_origin init_origin = ChainOrigin(0, &stack, true);
 599     SetOrigin(addr, size, init_origin);
 600   }
 601
 602   for (dfsan_label *labelp = shadow_for(addr); size != 0; --size, ++labelp)
 603     *labelp |= label;
 604 }
 605
 606 // Unlike the other dfsan interface functions the behavior of this function
 607 // depends on the label of one of its arguments.  Hence it is implemented as a
 608 // custom function.
 609 extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_label
 610 __dfsw_dfsan_get_label(long data, dfsan_label data_label,
 611                        dfsan_label *ret_label) {
 612   *ret_label = 0;
 613   return data_label;
 614 }
 615
 616 extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_label __dfso_dfsan_get_label(
 617     long data, dfsan_label data_label, dfsan_label *ret_label,
 618     dfsan_origin data_origin, dfsan_origin *ret_origin) {
 619   *ret_label = 0;
 620   *ret_origin = 0;
 621   return data_label;
 622 }
 623
 624 // This function is used if dfsan_get_origin is called when origin tracking is
 625 // off.
 626 extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_origin __dfsw_dfsan_get_origin(
 627     long data, dfsan_label data_label, dfsan_label *ret_label) {
 628   *ret_label = 0;
 629   return 0;
 630 }
 631
 632 extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_origin __dfso_dfsan_get_origin(
 633     long data, dfsan_label data_label, dfsan_label *ret_label,
 634     dfsan_origin data_origin, dfsan_origin *ret_origin) {
 635   *ret_label = 0;
 636   *ret_origin = 0;
 637   return data_origin;
 638 }
 639
 640 SANITIZER_INTERFACE_ATTRIBUTE dfsan_label
 641 dfsan_read_label(const void *addr, uptr size) {
 642   if (size == 0)
 643     return 0;
 644   return __dfsan_union_load(shadow_for(addr), size);
 645 }
 646
 647 SANITIZER_INTERFACE_ATTRIBUTE dfsan_origin
 648 dfsan_read_origin_of_first_taint(const void *addr, uptr size) {
 649   return GetOriginIfTainted((uptr)addr, size);
 650 }
 651
 652 SANITIZER_INTERFACE_ATTRIBUTE void dfsan_set_label_origin(dfsan_label label,
 653                                                           dfsan_origin origin,
 654                                                           void *addr,
 655                                                           uptr size) {
 656   __dfsan_set_label(label, origin, addr, size);
 657 }
 658
 659 extern "C" SANITIZER_INTERFACE_ATTRIBUTE int
 660 dfsan_has_label(dfsan_label label, dfsan_label elem) {
 661   return (label & elem) == elem;
 662 }
 663
 664 namespace __dfsan {
 665
 666 typedef void (*dfsan_conditional_callback_t)(dfsan_label label,
 667                                              dfsan_origin origin);
 668 static dfsan_conditional_callback_t conditional_callback = nullptr;
 669 static dfsan_label labels_in_signal_conditional = 0;
 670
 671 static void ConditionalCallback(dfsan_label label, dfsan_origin origin) {
 672   // Programs have many branches. For efficiency the conditional sink callback
 673   // handler needs to ignore as many as possible as early as possible.
 674   if (label == 0) {
 675     return;
 676   }
 677   if (conditional_callback == nullptr) {
 678     return;
 679   }
 680
 681   // This initial ConditionalCallback handler needs to be in here in dfsan
 682   // runtime (rather than being an entirely user implemented hook) so that it
 683   // has access to dfsan thread information.
 684   DFsanThread *t = GetCurrentThread();
 685   // A callback operation which does useful work (like record the flow) will
 686   // likely be too long executed in a signal handler.
 687   if (t && t->InSignalHandler()) {
 688     // Record set of labels used in signal handler for completeness.
 689     labels_in_signal_conditional |= label;
 690     return;
 691   }
 692
 693   conditional_callback(label, origin);
 694 }
 695
 696 }  // namespace __dfsan
 697
 698 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
 699 __dfsan_conditional_callback_origin(dfsan_label label, dfsan_origin origin) {
 700   __dfsan::ConditionalCallback(label, origin);
 701 }
 702
 703 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __dfsan_conditional_callback(
 704     dfsan_label label) {
 705   __dfsan::ConditionalCallback(label, 0);
 706 }
 707
 708 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void dfsan_set_conditional_callback(
 709     __dfsan::dfsan_conditional_callback_t callback) {
 710   __dfsan::conditional_callback = callback;
 711 }
 712
 713 extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_label
 714 dfsan_get_labels_in_signal_conditional() {
 715   return __dfsan::labels_in_signal_conditional;
 716 }
 717
 718 namespace __dfsan {
 719
 720 typedef void (*dfsan_reaches_function_callback_t)(dfsan_label label,
 721                                                   dfsan_origin origin,
 722                                                   const char *file,
 723                                                   unsigned int line,
 724                                                   const char *function);
 725 static dfsan_reaches_function_callback_t reaches_function_callback = nullptr;
 726 static dfsan_label labels_in_signal_reaches_function = 0;
 727
 728 static void ReachesFunctionCallback(dfsan_label label, dfsan_origin origin,
 729                                     const char *file, unsigned int line,
 730                                     const char *function) {
 731   if (label == 0) {
 732     return;
 733   }
 734   if (reaches_function_callback == nullptr) {
 735     return;
 736   }
 737
 738   // This initial ReachesFunctionCallback handler needs to be in here in dfsan
 739   // runtime (rather than being an entirely user implemented hook) so that it
 740   // has access to dfsan thread information.
 741   DFsanThread *t = GetCurrentThread();
 742   // A callback operation which does useful work (like record the flow) will
 743   // likely be too long executed in a signal handler.
 744   if (t && t->InSignalHandler()) {
 745     // Record set of labels used in signal handler for completeness.
 746     labels_in_signal_reaches_function |= label;
 747     return;
 748   }
 749
 750   reaches_function_callback(label, origin, file, line, function);
 751 }
 752
 753 }  // namespace __dfsan
 754
 755 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
 756 __dfsan_reaches_function_callback_origin(dfsan_label label, dfsan_origin origin,
 757                                          const char *file, unsigned int line,
 758                                          const char *function) {
 759   __dfsan::ReachesFunctionCallback(label, origin, file, line, function);
 760 }
 761
 762 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
 763 __dfsan_reaches_function_callback(dfsan_label label, const char *file,
 764                                   unsigned int line, const char *function) {
 765   __dfsan::ReachesFunctionCallback(label, 0, file, line, function);
 766 }
 767
 768 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
 769 dfsan_set_reaches_function_callback(
 770     __dfsan::dfsan_reaches_function_callback_t callback) {
 771   __dfsan::reaches_function_callback = callback;
 772 }
 773
 774 extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_label
 775 dfsan_get_labels_in_signal_reaches_function() {
 776   return __dfsan::labels_in_signal_reaches_function;
 777 }
 778
 779 class Decorator : public __sanitizer::SanitizerCommonDecorator {
 780  public:
 781   Decorator() : SanitizerCommonDecorator() {}
 782   const char *Origin() const { return Magenta(); }
 783 };
 784
 785 namespace {
 786
 787 void PrintNoOriginTrackingWarning() {
 788   Decorator d;
 789   Printf(
 790       "  %sDFSan: origin tracking is not enabled. Did you specify the "
 791       "-dfsan-track-origins=1 option?%s\n",
 792       d.Warning(), d.Default());
 793 }
 794
 795 void PrintNoTaintWarning(const void *address) {
 796   Decorator d;
 797   Printf("  %sDFSan: no tainted value at %x%s\n", d.Warning(), address,
 798          d.Default());
 799 }
 800
 801 void PrintInvalidOriginWarning(dfsan_label label, const void *address) {
 802   Decorator d;
 803   Printf(
 804       "  %sTaint value 0x%x (at %p) has invalid origin tracking. This can "
 805       "be a DFSan bug.%s\n",
 806       d.Warning(), label, address, d.Default());
 807 }
 808
 809 void PrintInvalidOriginIdWarning(dfsan_origin origin) {
 810   Decorator d;
 811   Printf(
 812       "  %sOrigin Id %d has invalid origin tracking. This can "
 813       "be a DFSan bug.%s\n",
 814       d.Warning(), origin, d.Default());
 815 }
 816
 817 bool PrintOriginTraceFramesToStr(Origin o, InternalScopedString *out) {
 818   Decorator d;
 819   bool found = false;
 820
 821   while (o.isChainedOrigin()) {
 822     StackTrace stack;
 823     dfsan_origin origin_id = o.raw_id();
 824     o = o.getNextChainedOrigin(&stack);
 825     if (o.isChainedOrigin())
 826       out->AppendF(
 827           "  %sOrigin value: 0x%x, Taint value was stored to memory at%s\n",
 828           d.Origin(), origin_id, d.Default());
 829     else
 830       out->AppendF("  %sOrigin value: 0x%x, Taint value was created at%s\n",
 831                    d.Origin(), origin_id, d.Default());
 832
 833     // Includes a trailing newline, so no need to add it again.
 834     stack.PrintTo(out);
 835     found = true;
 836   }
 837
 838   return found;
 839 }
 840
 841 bool PrintOriginTraceToStr(const void *addr, const char *description,
 842                            InternalScopedString *out) {
 843   CHECK(out);
 844   CHECK(dfsan_get_track_origins());
 845   Decorator d;
 846
 847   const dfsan_label label = *__dfsan::shadow_for(addr);
 848   CHECK(label);
 849
 850   const dfsan_origin origin = *__dfsan::origin_for(addr);
 851
 852   out->AppendF("  %sTaint value 0x%x (at %p) origin tracking (%s)%s\n",
 853                d.Origin(), label, addr, description ? description : "",
 854                d.Default());
 855
 856   Origin o = Origin::FromRawId(origin);
 857   return PrintOriginTraceFramesToStr(o, out);
 858 }
 859
 860 }  // namespace
 861
 862 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void dfsan_print_origin_trace(
 863     const void *addr, const char *description) {
 864   if (!dfsan_get_track_origins()) {
 865     PrintNoOriginTrackingWarning();
 866     return;
 867   }
 868
 869   const dfsan_label label = *__dfsan::shadow_for(addr);
 870   if (!label) {
 871     PrintNoTaintWarning(addr);
 872     return;
 873   }
 874
 875   InternalScopedString trace;
 876   bool success = PrintOriginTraceToStr(addr, description, &trace);
 877
 878   if (trace.length())
 879     Printf("%s", trace.data());
 880
 881   if (!success)
 882     PrintInvalidOriginWarning(label, addr);
 883 }
 884
 885 extern "C" SANITIZER_INTERFACE_ATTRIBUTE uptr
 886 dfsan_sprint_origin_trace(const void *addr, const char *description,
 887                           char *out_buf, uptr out_buf_size) {
 888   CHECK(out_buf);
 889
 890   if (!dfsan_get_track_origins()) {
 891     PrintNoOriginTrackingWarning();
 892     return 0;
 893   }
 894
 895   const dfsan_label label = *__dfsan::shadow_for(addr);
 896   if (!label) {
 897     PrintNoTaintWarning(addr);
 898     return 0;
 899   }
 900
 901   InternalScopedString trace;
 902   bool success = PrintOriginTraceToStr(addr, description, &trace);
 903
 904   if (!success) {
 905     PrintInvalidOriginWarning(label, addr);
 906     return 0;
 907   }
 908
 909   if (out_buf_size) {
 910     internal_strncpy(out_buf, trace.data(), out_buf_size - 1);
 911     out_buf[out_buf_size - 1] = '\0';
 912   }
 913
 914   return trace.length();
 915 }
 916
 917 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void dfsan_print_origin_id_trace(
 918     dfsan_origin origin) {
 919   if (!dfsan_get_track_origins()) {
 920     PrintNoOriginTrackingWarning();
 921     return;
 922   }
 923   Origin o = Origin::FromRawId(origin);
 924
 925   InternalScopedString trace;
 926   bool success = PrintOriginTraceFramesToStr(o, &trace);
 927
 928   if (trace.length())
 929     Printf("%s", trace.data());
 930
 931   if (!success)
 932     PrintInvalidOriginIdWarning(origin);
 933 }
 934
 935 extern "C" SANITIZER_INTERFACE_ATTRIBUTE uptr dfsan_sprint_origin_id_trace(
 936     dfsan_origin origin, char *out_buf, uptr out_buf_size) {
 937   CHECK(out_buf);
 938
 939   if (!dfsan_get_track_origins()) {
 940     PrintNoOriginTrackingWarning();
 941     return 0;
 942   }
 943   Origin o = Origin::FromRawId(origin);
 944
 945   InternalScopedString trace;
 946   bool success = PrintOriginTraceFramesToStr(o, &trace);
 947
 948   if (!success) {
 949     PrintInvalidOriginIdWarning(origin);
 950     return 0;
 951   }
 952
 953   if (out_buf_size) {
 954     internal_strncpy(out_buf, trace.data(), out_buf_size - 1);
 955     out_buf[out_buf_size - 1] = '\0';
 956   }
 957
 958   return trace.length();
 959 }
 960
 961 extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_origin
 962 dfsan_get_init_origin(const void *addr) {
 963   if (!dfsan_get_track_origins())
 964     return 0;
 965
 966   const dfsan_label label = *__dfsan::shadow_for(addr);
 967   if (!label)
 968     return 0;
 969
 970   const dfsan_origin origin = *__dfsan::origin_for(addr);
 971
 972   Origin o = Origin::FromRawId(origin);
 973   dfsan_origin origin_id = o.raw_id();
 974   while (o.isChainedOrigin()) {
 975     StackTrace stack;
 976     origin_id = o.raw_id();
 977     o = o.getNextChainedOrigin(&stack);
 978   }
 979   return origin_id;
 980 }
 981
 982 void __sanitizer::BufferedStackTrace::UnwindImpl(uptr pc, uptr bp,
 983                                                  void *context,
 984                                                  bool request_fast,
 985                                                  u32 max_depth) {
 986   using namespace __dfsan;
 987   DFsanThread *t = GetCurrentThread();
 988   if (!t || !StackTrace::WillUseFastUnwind(request_fast)) {
 989     return Unwind(max_depth, pc, bp, context, 0, 0, false);
 990   }
 991   Unwind(max_depth, pc, bp, nullptr, t->stack_top(), t->stack_bottom(), true);
 992 }
 993
 994 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_print_stack_trace() {
 995   GET_CALLER_PC_BP;
 996   GET_STORE_STACK_TRACE_PC_BP(pc, bp);
 997   stack.Print();
 998 }
 999
1000 extern "C" SANITIZER_INTERFACE_ATTRIBUTE uptr
1001 dfsan_sprint_stack_trace(char *out_buf, uptr out_buf_size) {
1002   CHECK(out_buf);
1003   GET_CALLER_PC_BP;
1004   GET_STORE_STACK_TRACE_PC_BP(pc, bp);
1005   return stack.PrintTo(out_buf, out_buf_size);
1006 }
1007
1008 void Flags::SetDefaults() {
1009 #define DFSAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
1010 #include "dfsan_flags.inc"
1011 #undef DFSAN_FLAG
1012 }
1013
1014 static void RegisterDfsanFlags(FlagParser *parser, Flags *f) {
1015 #define DFSAN_FLAG(Type, Name, DefaultValue, Description) \
1016   RegisterFlag(parser, #Name, Description, &f->Name);
1017 #include "dfsan_flags.inc"
1018 #undef DFSAN_FLAG
1019 }
1020
1021 static void InitializeFlags() {
1022   SetCommonFlagsDefaults();
1023   {
1024     CommonFlags cf;
1025     cf.CopyFrom(*common_flags());
1026     cf.intercept_tls_get_addr = true;
1027     OverrideCommonFlags(cf);
1028   }
1029   flags().SetDefaults();
1030
1031   FlagParser parser;
1032   RegisterCommonFlags(&parser);
1033   RegisterDfsanFlags(&parser, &flags());
1034   parser.ParseStringFromEnv("DFSAN_OPTIONS");
1035   InitializeCommonFlags();
1036   if (Verbosity()) ReportUnrecognizedFlags();
1037   if (common_flags()->help) parser.PrintFlagDescriptions();
1038 }
1039
1040 SANITIZER_INTERFACE_ATTRIBUTE
1041 void dfsan_clear_arg_tls(uptr offset, uptr size) {
1042   internal_memset((void *)((uptr)__dfsan_arg_tls + offset), 0, size);
1043 }
1044
1045 SANITIZER_INTERFACE_ATTRIBUTE
1046 void dfsan_clear_thread_local_state() {
1047   internal_memset(__dfsan_arg_tls, 0, sizeof(__dfsan_arg_tls));
1048   internal_memset(__dfsan_retval_tls, 0, sizeof(__dfsan_retval_tls));
1049
1050   if (dfsan_get_track_origins()) {
1051     internal_memset(__dfsan_arg_origin_tls, 0, sizeof(__dfsan_arg_origin_tls));
1052     internal_memset(&__dfsan_retval_origin_tls, 0,
1053                     sizeof(__dfsan_retval_origin_tls));
1054   }
1055 }
1056
1057 SANITIZER_INTERFACE_ATTRIBUTE
1058 void dfsan_set_arg_tls(uptr offset, dfsan_label label) {
1059   // 2x to match ShadowTLSAlignment.
1060   // ShadowTLSAlignment should probably be changed.
1061   // TODO: Consider reducing ShadowTLSAlignment to 1.
1062   // Aligning to 2 bytes is probably a remnant of fast16 mode.
1063   ((dfsan_label *)__dfsan_arg_tls)[offset * 2] = label;
1064 }
1065
1066 SANITIZER_INTERFACE_ATTRIBUTE
1067 void dfsan_set_arg_origin_tls(uptr offset, dfsan_origin o) {
1068   __dfsan_arg_origin_tls[offset] = o;
1069 }
1070
1071 extern "C" void dfsan_flush() {
1072   const uptr maxVirtualAddress = GetMaxUserVirtualAddress();
1073   for (unsigned i = 0; i < kMemoryLayoutSize; ++i) {
1074     uptr start = kMemoryLayout[i].start;
1075     uptr end = kMemoryLayout[i].end;
1076     uptr size = end - start;
1077     MappingDesc::Type type = kMemoryLayout[i].type;
1078
1079     if (type != MappingDesc::SHADOW && type != MappingDesc::ORIGIN)
1080       continue;
1081
1082     // Check if the segment should be mapped based on platform constraints.
1083     if (start >= maxVirtualAddress)
1084       continue;
1085
1086     if (!MmapFixedSuperNoReserve(start, size, kMemoryLayout[i].name)) {
1087       Printf("FATAL: DataFlowSanitizer: failed to clear memory region\n");
1088       Die();
1089     }
1090   }
1091   __dfsan::labels_in_signal_conditional = 0;
1092   __dfsan::labels_in_signal_reaches_function = 0;
1093 }
1094
1095 // TODO: CheckMemoryLayoutSanity is based on msan.
1096 // Consider refactoring these into a shared implementation.
1097 static void CheckMemoryLayoutSanity() {
1098   uptr prev_end = 0;
1099   for (unsigned i = 0; i < kMemoryLayoutSize; ++i) {
1100     uptr start = kMemoryLayout[i].start;
1101     uptr end = kMemoryLayout[i].end;
1102     MappingDesc::Type type = kMemoryLayout[i].type;
1103     CHECK_LT(start, end);
1104     CHECK_EQ(prev_end, start);
1105     CHECK(addr_is_type(start, type));
1106     CHECK(addr_is_type((start + end) / 2, type));
1107     CHECK(addr_is_type(end - 1, type));
1108     if (type == MappingDesc::APP) {
1109       uptr addr = start;
1110       CHECK(MEM_IS_SHADOW(MEM_TO_SHADOW(addr)));
1111       CHECK(MEM_IS_ORIGIN(MEM_TO_ORIGIN(addr)));
1112       CHECK_EQ(MEM_TO_ORIGIN(addr), SHADOW_TO_ORIGIN(MEM_TO_SHADOW(addr)));
1113
1114       addr = (start + end) / 2;
1115       CHECK(MEM_IS_SHADOW(MEM_TO_SHADOW(addr)));
1116       CHECK(MEM_IS_ORIGIN(MEM_TO_ORIGIN(addr)));
1117       CHECK_EQ(MEM_TO_ORIGIN(addr), SHADOW_TO_ORIGIN(MEM_TO_SHADOW(addr)));
1118
1119       addr = end - 1;
1120       CHECK(MEM_IS_SHADOW(MEM_TO_SHADOW(addr)));
1121       CHECK(MEM_IS_ORIGIN(MEM_TO_ORIGIN(addr)));
1122       CHECK_EQ(MEM_TO_ORIGIN(addr), SHADOW_TO_ORIGIN(MEM_TO_SHADOW(addr)));
1123     }
1124     prev_end = end;
1125   }
1126 }
1127
1128 // TODO: CheckMemoryRangeAvailability is based on msan.
1129 // Consider refactoring these into a shared implementation.
1130 static bool CheckMemoryRangeAvailability(uptr beg, uptr size) {
1131   if (size > 0) {
1132     uptr end = beg + size - 1;
1133     if (!MemoryRangeIsAvailable(beg, end)) {
1134       Printf("FATAL: Memory range %p - %p is not available.\n", beg, end);
1135       return false;
1136     }
1137   }
1138   return true;
1139 }
1140
1141 // TODO: ProtectMemoryRange is based on msan.
1142 // Consider refactoring these into a shared implementation.
1143 static bool ProtectMemoryRange(uptr beg, uptr size, const char *name) {
1144   if (size > 0) {
1145     void *addr = MmapFixedNoAccess(beg, size, name);
1146     if (beg == 0 && addr) {
1147       // Depending on the kernel configuration, we may not be able to protect
1148       // the page at address zero.
1149       uptr gap = 16 * GetPageSizeCached();
1150       beg += gap;
1151       size -= gap;
1152       addr = MmapFixedNoAccess(beg, size, name);
1153     }
1154     if ((uptr)addr != beg) {
1155       uptr end = beg + size - 1;
1156       Printf("FATAL: Cannot protect memory range %p - %p (%s).\n", beg, end,
1157              name);
1158       return false;
1159     }
1160   }
1161   return true;
1162 }
1163
1164 // TODO: InitShadow is based on msan.
1165 // Consider refactoring these into a shared implementation.
1166 bool InitShadow(bool init_origins) {
1167   // Let user know mapping parameters first.
1168   VPrintf(1, "dfsan_init %p\n", (void *)&__dfsan::dfsan_init);
1169   for (unsigned i = 0; i < kMemoryLayoutSize; ++i)
1170     VPrintf(1, "%s: %zx - %zx\n", kMemoryLayout[i].name, kMemoryLayout[i].start,
1171             kMemoryLayout[i].end - 1);
1172
1173   CheckMemoryLayoutSanity();
1174
1175   if (!MEM_IS_APP(&__dfsan::dfsan_init)) {
1176     Printf("FATAL: Code %p is out of application range. Non-PIE build?\n",
1177            (uptr)&__dfsan::dfsan_init);
1178     return false;
1179   }
1180
1181   const uptr maxVirtualAddress = GetMaxUserVirtualAddress();
1182
1183   for (unsigned i = 0; i < kMemoryLayoutSize; ++i) {
1184     uptr start = kMemoryLayout[i].start;
1185     uptr end = kMemoryLayout[i].end;
1186     uptr size = end - start;
1187     MappingDesc::Type type = kMemoryLayout[i].type;
1188
1189     // Check if the segment should be mapped based on platform constraints.
1190     if (start >= maxVirtualAddress)
1191       continue;
1192
1193     bool map = type == MappingDesc::SHADOW ||
1194                (init_origins && type == MappingDesc::ORIGIN);
1195     bool protect = type == MappingDesc::INVALID ||
1196                    (!init_origins && type == MappingDesc::ORIGIN);
1197     CHECK(!(map && protect));
1198     if (!map && !protect)
1199       CHECK(type == MappingDesc::APP);
1200     if (map) {
1201       if (!CheckMemoryRangeAvailability(start, size))
1202         return false;
1203       if (!MmapFixedSuperNoReserve(start, size, kMemoryLayout[i].name))
1204         return false;
1205       if (common_flags()->use_madv_dontdump)
1206         DontDumpShadowMemory(start, size);
1207     }
1208     if (protect) {
1209       if (!CheckMemoryRangeAvailability(start, size))
1210         return false;
1211       if (!ProtectMemoryRange(start, size, kMemoryLayout[i].name))
1212         return false;
1213     }
1214   }
1215
1216   return true;
1217 }
1218
1219 static void DFsanInit(int argc, char **argv, char **envp) {
1220   CHECK(!dfsan_init_is_running);
1221   if (dfsan_inited)
1222     return;
1223   dfsan_init_is_running = true;
1224   SanitizerToolName = "DataflowSanitizer";
1225
1226   AvoidCVE_2016_2143();
1227
1228   InitializeFlags();
1229
1230   CheckASLR();
1231
1232   InitShadow(dfsan_get_track_origins());
1233
1234   initialize_interceptors();
1235
1236   // Set up threads
1237   DFsanTSDInit(DFsanTSDDtor);
1238
1239   dfsan_allocator_init();
1240
1241   DFsanThread *main_thread = DFsanThread::Create(nullptr, nullptr);
1242   SetCurrentThread(main_thread);
1243   main_thread->Init();
1244
1245   dfsan_init_is_running = false;
1246   dfsan_inited = true;
1247 }
1248
1249 namespace __dfsan {
1250
1251 void dfsan_init() { DFsanInit(0, nullptr, nullptr); }
1252
1253 }  // namespace __dfsan
1254
1255 #if SANITIZER_CAN_USE_PREINIT_ARRAY
1256 __attribute__((section(".preinit_array"),
1257                used)) static void (*dfsan_init_ptr)(int, char **,
1258                                                     char **) = DFsanInit;
1259 #endif