compiler-rt/lib/tsan/rtl/tsan_rtl.cpp

   1 //===-- tsan_rtl.cpp ------------------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file is a part of ThreadSanitizer (TSan), a race detector.
  10 //
  11 // Main file (entry points) for the TSan run-time.
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "tsan_rtl.h"
  15
  16 #include "sanitizer_common/sanitizer_atomic.h"
  17 #include "sanitizer_common/sanitizer_common.h"
  18 #include "sanitizer_common/sanitizer_file.h"
  19 #include "sanitizer_common/sanitizer_interface_internal.h"
  20 #include "sanitizer_common/sanitizer_libc.h"
  21 #include "sanitizer_common/sanitizer_placement_new.h"
  22 #include "sanitizer_common/sanitizer_stackdepot.h"
  23 #include "sanitizer_common/sanitizer_symbolizer.h"
  24 #include "tsan_defs.h"
  25 #include "tsan_interface.h"
  26 #include "tsan_mman.h"
  27 #include "tsan_platform.h"
  28 #include "tsan_suppressions.h"
  29 #include "tsan_symbolize.h"
  30 #include "ubsan/ubsan_init.h"
  31
  32 volatile int __tsan_resumed = 0;
  33
  34 extern "C" void __tsan_resume() {
  35   __tsan_resumed = 1;
  36 }
  37
  38 #if SANITIZER_APPLE
  39 SANITIZER_WEAK_DEFAULT_IMPL
  40 void __tsan_test_only_on_fork() {}
  41 #endif
  42
  43 namespace __tsan {
  44
  45 #if !SANITIZER_GO
  46 void (*on_initialize)(void);
  47 int (*on_finalize)(int);
  48 #endif
  49
  50 #if !SANITIZER_GO && !SANITIZER_APPLE
  51 alignas(SANITIZER_CACHE_LINE_SIZE) THREADLOCAL __attribute__((tls_model(
  52     "initial-exec"))) char cur_thread_placeholder[sizeof(ThreadState)];
  53 #endif
  54 alignas(SANITIZER_CACHE_LINE_SIZE) static char ctx_placeholder[sizeof(Context)];
  55 Context *ctx;
  56
  57 // Can be overriden by a front-end.
  58 #ifdef TSAN_EXTERNAL_HOOKS
  59 bool OnFinalize(bool failed);
  60 void OnInitialize();
  61 #else
  62 SANITIZER_WEAK_CXX_DEFAULT_IMPL
  63 bool OnFinalize(bool failed) {
  64 #  if !SANITIZER_GO
  65   if (on_finalize)
  66     return on_finalize(failed);
  67 #  endif
  68   return failed;
  69 }
  70
  71 SANITIZER_WEAK_CXX_DEFAULT_IMPL
  72 void OnInitialize() {
  73 #  if !SANITIZER_GO
  74   if (on_initialize)
  75     on_initialize();
  76 #  endif
  77 }
  78 #endif
  79
  80 static TracePart* TracePartAlloc(ThreadState* thr) {
  81   TracePart* part = nullptr;
  82   {
  83     Lock lock(&ctx->slot_mtx);
  84     uptr max_parts = Trace::kMinParts + flags()->history_size;
  85     Trace* trace = &thr->tctx->trace;
  86     if (trace->parts_allocated == max_parts ||
  87         ctx->trace_part_finished_excess) {
  88       part = ctx->trace_part_recycle.PopFront();
  89       DPrintf("#%d: TracePartAlloc: part=%p\n", thr->tid, part);
  90       if (part && part->trace) {
  91         Trace* trace1 = part->trace;
  92         Lock trace_lock(&trace1->mtx);
  93         part->trace = nullptr;
  94         TracePart* part1 = trace1->parts.PopFront();
  95         CHECK_EQ(part, part1);
  96         if (trace1->parts_allocated > trace1->parts.Size()) {
  97           ctx->trace_part_finished_excess +=
  98               trace1->parts_allocated - trace1->parts.Size();
  99           trace1->parts_allocated = trace1->parts.Size();
 100         }
 101       }
 102     }
 103     if (trace->parts_allocated < max_parts) {
 104       trace->parts_allocated++;
 105       if (ctx->trace_part_finished_excess)
 106         ctx->trace_part_finished_excess--;
 107     }
 108     if (!part)
 109       ctx->trace_part_total_allocated++;
 110     else if (ctx->trace_part_recycle_finished)
 111       ctx->trace_part_recycle_finished--;
 112   }
 113   if (!part)
 114     part = new (MmapOrDie(sizeof(*part), "TracePart")) TracePart();
 115   return part;
 116 }
 117
 118 static void TracePartFree(TracePart* part) SANITIZER_REQUIRES(ctx->slot_mtx) {
 119   DCHECK(part->trace);
 120   part->trace = nullptr;
 121   ctx->trace_part_recycle.PushFront(part);
 122 }
 123
 124 void TraceResetForTesting() {
 125   Lock lock(&ctx->slot_mtx);
 126   while (auto* part = ctx->trace_part_recycle.PopFront()) {
 127     if (auto trace = part->trace)
 128       CHECK_EQ(trace->parts.PopFront(), part);
 129     UnmapOrDie(part, sizeof(*part));
 130   }
 131   ctx->trace_part_total_allocated = 0;
 132   ctx->trace_part_recycle_finished = 0;
 133   ctx->trace_part_finished_excess = 0;
 134 }
 135
 136 static void DoResetImpl(uptr epoch) {
 137   ThreadRegistryLock lock0(&ctx->thread_registry);
 138   Lock lock1(&ctx->slot_mtx);
 139   CHECK_EQ(ctx->global_epoch, epoch);
 140   ctx->global_epoch++;
 141   CHECK(!ctx->resetting);
 142   ctx->resetting = true;
 143   for (u32 i = ctx->thread_registry.NumThreadsLocked(); i--;) {
 144     ThreadContext* tctx = (ThreadContext*)ctx->thread_registry.GetThreadLocked(
 145         static_cast<Tid>(i));
 146     // Potentially we could purge all ThreadStatusDead threads from the
 147     // registry. Since we reset all shadow, they can't race with anything
 148     // anymore. However, their tid's can still be stored in some aux places
 149     // (e.g. tid of thread that created something).
 150     auto trace = &tctx->trace;
 151     Lock lock(&trace->mtx);
 152     bool attached = tctx->thr && tctx->thr->slot;
 153     auto parts = &trace->parts;
 154     bool local = false;
 155     while (!parts->Empty()) {
 156       auto part = parts->Front();
 157       local = local || part == trace->local_head;
 158       if (local)
 159         CHECK(!ctx->trace_part_recycle.Queued(part));
 160       else
 161         ctx->trace_part_recycle.Remove(part);
 162       if (attached && parts->Size() == 1) {
 163         // The thread is running and this is the last/current part.
 164         // Set the trace position to the end of the current part
 165         // to force the thread to call SwitchTracePart and re-attach
 166         // to a new slot and allocate a new trace part.
 167         // Note: the thread is concurrently modifying the position as well,
 168         // so this is only best-effort. The thread can only modify position
 169         // within this part, because switching parts is protected by
 170         // slot/trace mutexes that we hold here.
 171         atomic_store_relaxed(
 172             &tctx->thr->trace_pos,
 173             reinterpret_cast<uptr>(&part->events[TracePart::kSize]));
 174         break;
 175       }
 176       parts->Remove(part);
 177       TracePartFree(part);
 178     }
 179     CHECK_LE(parts->Size(), 1);
 180     trace->local_head = parts->Front();
 181     if (tctx->thr && !tctx->thr->slot) {
 182       atomic_store_relaxed(&tctx->thr->trace_pos, 0);
 183       tctx->thr->trace_prev_pc = 0;
 184     }
 185     if (trace->parts_allocated > trace->parts.Size()) {
 186       ctx->trace_part_finished_excess +=
 187           trace->parts_allocated - trace->parts.Size();
 188       trace->parts_allocated = trace->parts.Size();
 189     }
 190   }
 191   while (ctx->slot_queue.PopFront()) {
 192   }
 193   for (auto& slot : ctx->slots) {
 194     slot.SetEpoch(kEpochZero);
 195     slot.journal.Reset();
 196     slot.thr = nullptr;
 197     ctx->slot_queue.PushBack(&slot);
 198   }
 199
 200   DPrintf("Resetting shadow...\n");
 201   auto shadow_begin = ShadowBeg();
 202   auto shadow_end = ShadowEnd();
 203 #if SANITIZER_GO
 204   CHECK_NE(0, ctx->mapped_shadow_begin);
 205   shadow_begin = ctx->mapped_shadow_begin;
 206   shadow_end = ctx->mapped_shadow_end;
 207   VPrintf(2, "shadow_begin-shadow_end: (0x%zx-0x%zx)\n",
 208           shadow_begin, shadow_end);
 209 #endif
 210
 211 #if SANITIZER_WINDOWS
 212   auto resetFailed =
 213       !ZeroMmapFixedRegion(shadow_begin, shadow_end - shadow_begin);
 214 #else
 215   auto resetFailed =
 216       !MmapFixedSuperNoReserve(shadow_begin, shadow_end-shadow_begin, "shadow");
 217 #  if !SANITIZER_GO
 218   DontDumpShadow(shadow_begin, shadow_end - shadow_begin);
 219 #  endif
 220 #endif
 221   if (resetFailed) {
 222     Printf("failed to reset shadow memory\n");
 223     Die();
 224   }
 225   DPrintf("Resetting meta shadow...\n");
 226   ctx->metamap.ResetClocks();
 227   StoreShadow(&ctx->last_spurious_race, Shadow::kEmpty);
 228   ctx->resetting = false;
 229 }
 230
 231 // Clang does not understand locking all slots in the loop:
 232 // error: expecting mutex 'slot.mtx' to be held at start of each loop
 233 void DoReset(ThreadState* thr, uptr epoch) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
 234   for (auto& slot : ctx->slots) {
 235     slot.mtx.Lock();
 236     if (UNLIKELY(epoch == 0))
 237       epoch = ctx->global_epoch;
 238     if (UNLIKELY(epoch != ctx->global_epoch)) {
 239       // Epoch can't change once we've locked the first slot.
 240       CHECK_EQ(slot.sid, 0);
 241       slot.mtx.Unlock();
 242       return;
 243     }
 244   }
 245   DPrintf("#%d: DoReset epoch=%lu\n", thr ? thr->tid : -1, epoch);
 246   DoResetImpl(epoch);
 247   for (auto& slot : ctx->slots) slot.mtx.Unlock();
 248 }
 249
 250 void FlushShadowMemory() { DoReset(nullptr, 0); }
 251
 252 static TidSlot* FindSlotAndLock(ThreadState* thr)
 253     SANITIZER_ACQUIRE(thr->slot->mtx) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
 254   CHECK(!thr->slot);
 255   TidSlot* slot = nullptr;
 256   for (;;) {
 257     uptr epoch;
 258     {
 259       Lock lock(&ctx->slot_mtx);
 260       epoch = ctx->global_epoch;
 261       if (slot) {
 262         // This is an exhausted slot from the previous iteration.
 263         if (ctx->slot_queue.Queued(slot))
 264           ctx->slot_queue.Remove(slot);
 265         thr->slot_locked = false;
 266         slot->mtx.Unlock();
 267       }
 268       for (;;) {
 269         slot = ctx->slot_queue.PopFront();
 270         if (!slot)
 271           break;
 272         if (slot->epoch() != kEpochLast) {
 273           ctx->slot_queue.PushBack(slot);
 274           break;
 275         }
 276       }
 277     }
 278     if (!slot) {
 279       DoReset(thr, epoch);
 280       continue;
 281     }
 282     slot->mtx.Lock();
 283     CHECK(!thr->slot_locked);
 284     thr->slot_locked = true;
 285     if (slot->thr) {
 286       DPrintf("#%d: preempting sid=%d tid=%d\n", thr->tid, (u32)slot->sid,
 287               slot->thr->tid);
 288       slot->SetEpoch(slot->thr->fast_state.epoch());
 289       slot->thr = nullptr;
 290     }
 291     if (slot->epoch() != kEpochLast)
 292       return slot;
 293   }
 294 }
 295
 296 void SlotAttachAndLock(ThreadState* thr) {
 297   TidSlot* slot = FindSlotAndLock(thr);
 298   DPrintf("#%d: SlotAttach: slot=%u\n", thr->tid, static_cast<int>(slot->sid));
 299   CHECK(!slot->thr);
 300   CHECK(!thr->slot);
 301   slot->thr = thr;
 302   thr->slot = slot;
 303   Epoch epoch = EpochInc(slot->epoch());
 304   CHECK(!EpochOverflow(epoch));
 305   slot->SetEpoch(epoch);
 306   thr->fast_state.SetSid(slot->sid);
 307   thr->fast_state.SetEpoch(epoch);
 308   if (thr->slot_epoch != ctx->global_epoch) {
 309     thr->slot_epoch = ctx->global_epoch;
 310     thr->clock.Reset();
 311 #if !SANITIZER_GO
 312     thr->last_sleep_stack_id = kInvalidStackID;
 313     thr->last_sleep_clock.Reset();
 314 #endif
 315   }
 316   thr->clock.Set(slot->sid, epoch);
 317   slot->journal.PushBack({thr->tid, epoch});
 318 }
 319
 320 static void SlotDetachImpl(ThreadState* thr, bool exiting) {
 321   TidSlot* slot = thr->slot;
 322   thr->slot = nullptr;
 323   if (thr != slot->thr) {
 324     slot = nullptr;  // we don't own the slot anymore
 325     if (thr->slot_epoch != ctx->global_epoch) {
 326       TracePart* part = nullptr;
 327       auto* trace = &thr->tctx->trace;
 328       {
 329         Lock l(&trace->mtx);
 330         auto* parts = &trace->parts;
 331         // The trace can be completely empty in an unlikely event
 332         // the thread is preempted right after it acquired the slot
 333         // in ThreadStart and did not trace any events yet.
 334         CHECK_LE(parts->Size(), 1);
 335         part = parts->PopFront();
 336         thr->tctx->trace.local_head = nullptr;
 337         atomic_store_relaxed(&thr->trace_pos, 0);
 338         thr->trace_prev_pc = 0;
 339       }
 340       if (part) {
 341         Lock l(&ctx->slot_mtx);
 342         TracePartFree(part);
 343       }
 344     }
 345     return;
 346   }
 347   CHECK(exiting || thr->fast_state.epoch() == kEpochLast);
 348   slot->SetEpoch(thr->fast_state.epoch());
 349   slot->thr = nullptr;
 350 }
 351
 352 void SlotDetach(ThreadState* thr) {
 353   Lock lock(&thr->slot->mtx);
 354   SlotDetachImpl(thr, true);
 355 }
 356
 357 void SlotLock(ThreadState* thr) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
 358   DCHECK(!thr->slot_locked);
 359 #if SANITIZER_DEBUG
 360   // Check these mutexes are not locked.
 361   // We can call DoReset from SlotAttachAndLock, which will lock
 362   // these mutexes, but it happens only every once in a while.
 363   { ThreadRegistryLock lock(&ctx->thread_registry); }
 364   { Lock lock(&ctx->slot_mtx); }
 365 #endif
 366   TidSlot* slot = thr->slot;
 367   slot->mtx.Lock();
 368   thr->slot_locked = true;
 369   if (LIKELY(thr == slot->thr && thr->fast_state.epoch() != kEpochLast))
 370     return;
 371   SlotDetachImpl(thr, false);
 372   thr->slot_locked = false;
 373   slot->mtx.Unlock();
 374   SlotAttachAndLock(thr);
 375 }
 376
 377 void SlotUnlock(ThreadState* thr) {
 378   DCHECK(thr->slot_locked);
 379   thr->slot_locked = false;
 380   thr->slot->mtx.Unlock();
 381 }
 382
 383 Context::Context()
 384     : initialized(),
 385       report_mtx(MutexTypeReport),
 386       nreported(),
 387       thread_registry([](Tid tid) -> ThreadContextBase* {
 388         return new (Alloc(sizeof(ThreadContext))) ThreadContext(tid);
 389       }),
 390       racy_mtx(MutexTypeRacy),
 391       racy_stacks(),
 392       fired_suppressions_mtx(MutexTypeFired),
 393       slot_mtx(MutexTypeSlots),
 394       resetting() {
 395   fired_suppressions.reserve(8);
 396   for (uptr i = 0; i < ARRAY_SIZE(slots); i++) {
 397     TidSlot* slot = &slots[i];
 398     slot->sid = static_cast<Sid>(i);
 399     slot_queue.PushBack(slot);
 400   }
 401   global_epoch = 1;
 402 }
 403
 404 TidSlot::TidSlot() : mtx(MutexTypeSlot) {}
 405
 406 // The objects are allocated in TLS, so one may rely on zero-initialization.
 407 ThreadState::ThreadState(Tid tid)
 408     // Do not touch these, rely on zero initialization,
 409     // they may be accessed before the ctor.
 410     // ignore_reads_and_writes()
 411     // ignore_interceptors()
 412     : tid(tid) {
 413   CHECK_EQ(reinterpret_cast<uptr>(this) % SANITIZER_CACHE_LINE_SIZE, 0);
 414 #if !SANITIZER_GO
 415   // C/C++ uses fixed size shadow stack.
 416   const int kInitStackSize = kShadowStackSize;
 417   shadow_stack = static_cast<uptr*>(
 418       MmapNoReserveOrDie(kInitStackSize * sizeof(uptr), "shadow stack"));
 419   SetShadowRegionHugePageMode(reinterpret_cast<uptr>(shadow_stack),
 420                               kInitStackSize * sizeof(uptr));
 421 #else
 422   // Go uses malloc-allocated shadow stack with dynamic size.
 423   const int kInitStackSize = 8;
 424   shadow_stack = static_cast<uptr*>(Alloc(kInitStackSize * sizeof(uptr)));
 425 #endif
 426   shadow_stack_pos = shadow_stack;
 427   shadow_stack_end = shadow_stack + kInitStackSize;
 428 }
 429
 430 #if !SANITIZER_GO
 431 void MemoryProfiler(u64 uptime) {
 432   if (ctx->memprof_fd == kInvalidFd)
 433     return;
 434   InternalMmapVector<char> buf(4096);
 435   WriteMemoryProfile(buf.data(), buf.size(), uptime);
 436   WriteToFile(ctx->memprof_fd, buf.data(), internal_strlen(buf.data()));
 437 }
 438
 439 static bool InitializeMemoryProfiler() {
 440   ctx->memprof_fd = kInvalidFd;
 441   const char *fname = flags()->profile_memory;
 442   if (!fname || !fname[0])
 443     return false;
 444   if (internal_strcmp(fname, "stdout") == 0) {
 445     ctx->memprof_fd = 1;
 446   } else if (internal_strcmp(fname, "stderr") == 0) {
 447     ctx->memprof_fd = 2;
 448   } else {
 449     InternalScopedString filename;
 450     filename.AppendF("%s.%d", fname, (int)internal_getpid());
 451     ctx->memprof_fd = OpenFile(filename.data(), WrOnly);
 452     if (ctx->memprof_fd == kInvalidFd) {
 453       Printf("ThreadSanitizer: failed to open memory profile file '%s'\n",
 454              filename.data());
 455       return false;
 456     }
 457   }
 458   MemoryProfiler(0);
 459   return true;
 460 }
 461
 462 static void *BackgroundThread(void *arg) {
 463   // This is a non-initialized non-user thread, nothing to see here.
 464   // We don't use ScopedIgnoreInterceptors, because we want ignores to be
 465   // enabled even when the thread function exits (e.g. during pthread thread
 466   // shutdown code).
 467   cur_thread_init()->ignore_interceptors++;
 468   const u64 kMs2Ns = 1000 * 1000;
 469   const u64 start = NanoTime();
 470
 471   u64 last_flush = start;
 472   uptr last_rss = 0;
 473   while (!atomic_load_relaxed(&ctx->stop_background_thread)) {
 474     SleepForMillis(100);
 475     u64 now = NanoTime();
 476
 477     // Flush memory if requested.
 478     if (flags()->flush_memory_ms > 0) {
 479       if (last_flush + flags()->flush_memory_ms * kMs2Ns < now) {
 480         VReport(1, "ThreadSanitizer: periodic memory flush\n");
 481         FlushShadowMemory();
 482         now = last_flush = NanoTime();
 483       }
 484     }
 485     if (flags()->memory_limit_mb > 0) {
 486       uptr rss = GetRSS();
 487       uptr limit = uptr(flags()->memory_limit_mb) << 20;
 488       VReport(1,
 489               "ThreadSanitizer: memory flush check"
 490               " RSS=%llu LAST=%llu LIMIT=%llu\n",
 491               (u64)rss >> 20, (u64)last_rss >> 20, (u64)limit >> 20);
 492       if (2 * rss > limit + last_rss) {
 493         VReport(1, "ThreadSanitizer: flushing memory due to RSS\n");
 494         FlushShadowMemory();
 495         rss = GetRSS();
 496         now = NanoTime();
 497         VReport(1, "ThreadSanitizer: memory flushed RSS=%llu\n",
 498                 (u64)rss >> 20);
 499       }
 500       last_rss = rss;
 501     }
 502
 503     MemoryProfiler(now - start);
 504
 505     // Flush symbolizer cache if requested.
 506     if (flags()->flush_symbolizer_ms > 0) {
 507       u64 last = atomic_load(&ctx->last_symbolize_time_ns,
 508                              memory_order_relaxed);
 509       if (last != 0 && last + flags()->flush_symbolizer_ms * kMs2Ns < now) {
 510         Lock l(&ctx->report_mtx);
 511         ScopedErrorReportLock l2;
 512         SymbolizeFlush();
 513         atomic_store(&ctx->last_symbolize_time_ns, 0, memory_order_relaxed);
 514       }
 515     }
 516   }
 517   return nullptr;
 518 }
 519
 520 static void StartBackgroundThread() {
 521   ctx->background_thread = internal_start_thread(&BackgroundThread, 0);
 522 }
 523
 524 #ifndef __mips__
 525 static void StopBackgroundThread() {
 526   atomic_store(&ctx->stop_background_thread, 1, memory_order_relaxed);
 527   internal_join_thread(ctx->background_thread);
 528   ctx->background_thread = 0;
 529 }
 530 #endif
 531 #endif
 532
 533 void DontNeedShadowFor(uptr addr, uptr size) {
 534   ReleaseMemoryPagesToOS(reinterpret_cast<uptr>(MemToShadow(addr)),
 535                          reinterpret_cast<uptr>(MemToShadow(addr + size)));
 536 }
 537
 538 #if !SANITIZER_GO
 539 // We call UnmapShadow before the actual munmap, at that point we don't yet
 540 // know if the provided address/size are sane. We can't call UnmapShadow
 541 // after the actual munmap becuase at that point the memory range can
 542 // already be reused for something else, so we can't rely on the munmap
 543 // return value to understand is the values are sane.
 544 // While calling munmap with insane values (non-canonical address, negative
 545 // size, etc) is an error, the kernel won't crash. We must also try to not
 546 // crash as the failure mode is very confusing (paging fault inside of the
 547 // runtime on some derived shadow address).
 548 static bool IsValidMmapRange(uptr addr, uptr size) {
 549   if (size == 0)
 550     return true;
 551   if (static_cast<sptr>(size) < 0)
 552     return false;
 553   if (!IsAppMem(addr) || !IsAppMem(addr + size - 1))
 554     return false;
 555   // Check that if the start of the region belongs to one of app ranges,
 556   // end of the region belongs to the same region.
 557   const uptr ranges[][2] = {
 558       {LoAppMemBeg(), LoAppMemEnd()},
 559       {MidAppMemBeg(), MidAppMemEnd()},
 560       {HiAppMemBeg(), HiAppMemEnd()},
 561   };
 562   for (auto range : ranges) {
 563     if (addr >= range[0] && addr < range[1])
 564       return addr + size <= range[1];
 565   }
 566   return false;
 567 }
 568
 569 void UnmapShadow(ThreadState *thr, uptr addr, uptr size) {
 570   if (size == 0 || !IsValidMmapRange(addr, size))
 571     return;
 572   DontNeedShadowFor(addr, size);
 573   ScopedGlobalProcessor sgp;
 574   SlotLocker locker(thr, true);
 575   ctx->metamap.ResetRange(thr->proc(), addr, size, true);
 576 }
 577 #endif
 578
 579 void MapShadow(uptr addr, uptr size) {
 580   // Ensure thead registry lock held, so as to synchronize
 581   // with DoReset, which also access the mapped_shadow_* ctxt fields.
 582   ThreadRegistryLock lock0(&ctx->thread_registry);
 583   static bool data_mapped = false;
 584
 585 #if !SANITIZER_GO
 586   // Global data is not 64K aligned, but there are no adjacent mappings,
 587   // so we can get away with unaligned mapping.
 588   // CHECK_EQ(addr, addr & ~((64 << 10) - 1));  // windows wants 64K alignment
 589   const uptr kPageSize = GetPageSizeCached();
 590   uptr shadow_begin = RoundDownTo((uptr)MemToShadow(addr), kPageSize);
 591   uptr shadow_end = RoundUpTo((uptr)MemToShadow(addr + size), kPageSize);
 592   if (!MmapFixedNoReserve(shadow_begin, shadow_end - shadow_begin, "shadow"))
 593     Die();
 594 #else
 595   uptr shadow_begin = RoundDownTo((uptr)MemToShadow(addr), (64 << 10));
 596   uptr shadow_end = RoundUpTo((uptr)MemToShadow(addr + size), (64 << 10));
 597   VPrintf(2, "MapShadow for (0x%zx-0x%zx), begin/end: (0x%zx-0x%zx)\n",
 598           addr, addr + size, shadow_begin, shadow_end);
 599
 600   if (!data_mapped) {
 601     // First call maps data+bss.
 602     if (!MmapFixedSuperNoReserve(shadow_begin, shadow_end - shadow_begin, "shadow"))
 603       Die();
 604   } else {
 605     VPrintf(2, "ctx->mapped_shadow_{begin,end} = (0x%zx-0x%zx)\n",
 606             ctx->mapped_shadow_begin, ctx->mapped_shadow_end);
 607     // Second and subsequent calls map heap.
 608     if (shadow_end <= ctx->mapped_shadow_end)
 609       return;
 610     if (!ctx->mapped_shadow_begin || ctx->mapped_shadow_begin > shadow_begin)
 611        ctx->mapped_shadow_begin = shadow_begin;
 612     if (shadow_begin < ctx->mapped_shadow_end)
 613       shadow_begin = ctx->mapped_shadow_end;
 614     VPrintf(2, "MapShadow begin/end = (0x%zx-0x%zx)\n",
 615             shadow_begin, shadow_end);
 616     if (!MmapFixedSuperNoReserve(shadow_begin, shadow_end - shadow_begin,
 617                                  "shadow"))
 618       Die();
 619     ctx->mapped_shadow_end = shadow_end;
 620   }
 621 #endif
 622
 623   // Meta shadow is 2:1, so tread carefully.
 624   static uptr mapped_meta_end = 0;
 625   uptr meta_begin = (uptr)MemToMeta(addr);
 626   uptr meta_end = (uptr)MemToMeta(addr + size);
 627   meta_begin = RoundDownTo(meta_begin, 64 << 10);
 628   meta_end = RoundUpTo(meta_end, 64 << 10);
 629   if (!data_mapped) {
 630     // First call maps data+bss.
 631     data_mapped = true;
 632     if (!MmapFixedSuperNoReserve(meta_begin, meta_end - meta_begin,
 633                                  "meta shadow"))
 634       Die();
 635   } else {
 636     // Mapping continuous heap.
 637     // Windows wants 64K alignment.
 638     meta_begin = RoundDownTo(meta_begin, 64 << 10);
 639     meta_end = RoundUpTo(meta_end, 64 << 10);
 640     CHECK_GT(meta_end, mapped_meta_end);
 641     if (meta_begin < mapped_meta_end)
 642       meta_begin = mapped_meta_end;
 643     if (!MmapFixedSuperNoReserve(meta_begin, meta_end - meta_begin,
 644                                  "meta shadow"))
 645       Die();
 646     mapped_meta_end = meta_end;
 647   }
 648   VPrintf(2, "mapped meta shadow for (0x%zx-0x%zx) at (0x%zx-0x%zx)\n", addr,
 649           addr + size, meta_begin, meta_end);
 650 }
 651
 652 #if !SANITIZER_GO
 653 static void OnStackUnwind(const SignalContext &sig, const void *,
 654                           BufferedStackTrace *stack) {
 655   stack->Unwind(StackTrace::GetNextInstructionPc(sig.pc), sig.bp, sig.context,
 656                 common_flags()->fast_unwind_on_fatal);
 657 }
 658
 659 static void TsanOnDeadlySignal(int signo, void *siginfo, void *context) {
 660   HandleDeadlySignal(siginfo, context, GetTid(), &OnStackUnwind, nullptr);
 661 }
 662 #endif
 663
 664 void CheckUnwind() {
 665   // There is high probability that interceptors will check-fail as well,
 666   // on the other hand there is no sense in processing interceptors
 667   // since we are going to die soon.
 668   ScopedIgnoreInterceptors ignore;
 669 #if !SANITIZER_GO
 670   ThreadState* thr = cur_thread();
 671   thr->nomalloc = false;
 672   thr->ignore_sync++;
 673   thr->ignore_reads_and_writes++;
 674   atomic_store_relaxed(&thr->in_signal_handler, 0);
 675 #endif
 676   PrintCurrentStack(StackTrace::GetCurrentPc(),
 677                     common_flags()->fast_unwind_on_fatal);
 678 }
 679
 680 bool is_initialized;
 681
 682 void Initialize(ThreadState *thr) {
 683   // Thread safe because done before all threads exist.
 684   if (is_initialized)
 685     return;
 686   is_initialized = true;
 687   // We are not ready to handle interceptors yet.
 688   ScopedIgnoreInterceptors ignore;
 689   SanitizerToolName = "ThreadSanitizer";
 690   // Install tool-specific callbacks in sanitizer_common.
 691   SetCheckUnwindCallback(CheckUnwind);
 692
 693   ctx = new(ctx_placeholder) Context;
 694   const char *env_name = SANITIZER_GO ? "GORACE" : "TSAN_OPTIONS";
 695   const char *options = GetEnv(env_name);
 696   CacheBinaryName();
 697   CheckASLR();
 698   InitializeFlags(&ctx->flags, options, env_name);
 699   AvoidCVE_2016_2143();
 700   __sanitizer::InitializePlatformEarly();
 701   __tsan::InitializePlatformEarly();
 702
 703 #if !SANITIZER_GO
 704   InitializeAllocator();
 705   ReplaceSystemMalloc();
 706 #endif
 707   if (common_flags()->detect_deadlocks)
 708     ctx->dd = DDetector::Create(flags());
 709   Processor *proc = ProcCreate();
 710   ProcWire(proc, thr);
 711   InitializeInterceptors();
 712   InitializePlatform();
 713   InitializeDynamicAnnotations();
 714 #if !SANITIZER_GO
 715   InitializeShadowMemory();
 716   InitializeAllocatorLate();
 717   InstallDeadlySignalHandlers(TsanOnDeadlySignal);
 718 #endif
 719   // Setup correct file descriptor for error reports.
 720   __sanitizer_set_report_path(common_flags()->log_path);
 721   InitializeSuppressions();
 722 #if !SANITIZER_GO
 723   InitializeLibIgnore();
 724   Symbolizer::GetOrInit()->AddHooks(EnterSymbolizer, ExitSymbolizer);
 725 #endif
 726
 727   VPrintf(1, "***** Running under ThreadSanitizer v3 (pid %d) *****\n",
 728           (int)internal_getpid());
 729
 730   // Initialize thread 0.
 731   Tid tid = ThreadCreate(nullptr, 0, 0, true);
 732   CHECK_EQ(tid, kMainTid);
 733   ThreadStart(thr, tid, GetTid(), ThreadType::Regular);
 734 #if TSAN_CONTAINS_UBSAN
 735   __ubsan::InitAsPlugin();
 736 #endif
 737
 738 #if !SANITIZER_GO
 739   Symbolizer::LateInitialize();
 740   if (InitializeMemoryProfiler() || flags()->force_background_thread)
 741     MaybeSpawnBackgroundThread();
 742 #endif
 743   ctx->initialized = true;
 744
 745   if (flags()->stop_on_start) {
 746     Printf("ThreadSanitizer is suspended at startup (pid %d)."
 747            " Call __tsan_resume().\n",
 748            (int)internal_getpid());
 749     while (__tsan_resumed == 0) {}
 750   }
 751
 752   OnInitialize();
 753 }
 754
 755 void MaybeSpawnBackgroundThread() {
 756   // On MIPS, TSan initialization is run before
 757   // __pthread_initialize_minimal_internal() is finished, so we can not spawn
 758   // new threads.
 759 #if !SANITIZER_GO && !defined(__mips__)
 760   static atomic_uint32_t bg_thread = {};
 761   if (atomic_load(&bg_thread, memory_order_relaxed) == 0 &&
 762       atomic_exchange(&bg_thread, 1, memory_order_relaxed) == 0) {
 763     StartBackgroundThread();
 764     SetSandboxingCallback(StopBackgroundThread);
 765   }
 766 #endif
 767 }
 768
 769 int Finalize(ThreadState *thr) {
 770   bool failed = false;
 771
 772 #if !SANITIZER_GO
 773   if (common_flags()->print_module_map == 1)
 774     DumpProcessMap();
 775 #endif
 776
 777   if (flags()->atexit_sleep_ms > 0 && ThreadCount(thr) > 1)
 778     internal_usleep(u64(flags()->atexit_sleep_ms) * 1000);
 779
 780   {
 781     // Wait for pending reports.
 782     ScopedErrorReportLock lock;
 783   }
 784
 785 #if !SANITIZER_GO
 786   if (Verbosity()) AllocatorPrintStats();
 787 #endif
 788
 789   ThreadFinalize(thr);
 790
 791   if (ctx->nreported) {
 792     failed = true;
 793 #if !SANITIZER_GO
 794     Printf("ThreadSanitizer: reported %d warnings\n", ctx->nreported);
 795 #else
 796     Printf("Found %d data race(s)\n", ctx->nreported);
 797 #endif
 798   }
 799
 800   if (common_flags()->print_suppressions)
 801     PrintMatchedSuppressions();
 802
 803   failed = OnFinalize(failed);
 804
 805   return failed ? common_flags()->exitcode : 0;
 806 }
 807
 808 #if !SANITIZER_GO
 809 void ForkBefore(ThreadState* thr, uptr pc) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
 810   VReport(2, "BeforeFork tid: %llu\n", GetTid());
 811   GlobalProcessorLock();
 812   // Detaching from the slot makes OnUserFree skip writing to the shadow.
 813   // The slot will be locked so any attempts to use it will deadlock anyway.
 814   SlotDetach(thr);
 815   for (auto& slot : ctx->slots) slot.mtx.Lock();
 816   ctx->thread_registry.Lock();
 817   ctx->slot_mtx.Lock();
 818   ScopedErrorReportLock::Lock();
 819   AllocatorLockBeforeFork();
 820   // Suppress all reports in the pthread_atfork callbacks.
 821   // Reports will deadlock on the report_mtx.
 822   // We could ignore sync operations as well,
 823   // but so far it's unclear if it will do more good or harm.
 824   // Unnecessarily ignoring things can lead to false positives later.
 825   thr->suppress_reports++;
 826   // On OS X, REAL(fork) can call intercepted functions (OSSpinLockLock), and
 827   // we'll assert in CheckNoLocks() unless we ignore interceptors.
 828   // On OS X libSystem_atfork_prepare/parent/child callbacks are called
 829   // after/before our callbacks and they call free.
 830   thr->ignore_interceptors++;
 831   // Disables memory write in OnUserAlloc/Free.
 832   thr->ignore_reads_and_writes++;
 833
 834 #  if SANITIZER_APPLE
 835   __tsan_test_only_on_fork();
 836 #  endif
 837 }
 838
 839 static void ForkAfter(ThreadState* thr,
 840                       bool child) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
 841   thr->suppress_reports--;  // Enabled in ForkBefore.
 842   thr->ignore_interceptors--;
 843   thr->ignore_reads_and_writes--;
 844   AllocatorUnlockAfterFork(child);
 845   ScopedErrorReportLock::Unlock();
 846   ctx->slot_mtx.Unlock();
 847   ctx->thread_registry.Unlock();
 848   for (auto& slot : ctx->slots) slot.mtx.Unlock();
 849   SlotAttachAndLock(thr);
 850   SlotUnlock(thr);
 851   GlobalProcessorUnlock();
 852   VReport(2, "AfterFork tid: %llu\n", GetTid());
 853 }
 854
 855 void ForkParentAfter(ThreadState* thr, uptr pc) { ForkAfter(thr, false); }
 856
 857 void ForkChildAfter(ThreadState* thr, uptr pc, bool start_thread) {
 858   ForkAfter(thr, true);
 859   u32 nthread = ctx->thread_registry.OnFork(thr->tid);
 860   VPrintf(1,
 861           "ThreadSanitizer: forked new process with pid %d,"
 862           " parent had %d threads\n",
 863           (int)internal_getpid(), (int)nthread);
 864   if (nthread == 1) {
 865     if (start_thread)
 866       StartBackgroundThread();
 867   } else {
 868     // We've just forked a multi-threaded process. We cannot reasonably function
 869     // after that (some mutexes may be locked before fork). So just enable
 870     // ignores for everything in the hope that we will exec soon.
 871     ctx->after_multithreaded_fork = true;
 872     thr->ignore_interceptors++;
 873     thr->suppress_reports++;
 874     ThreadIgnoreBegin(thr, pc);
 875     ThreadIgnoreSyncBegin(thr, pc);
 876   }
 877 }
 878 #endif
 879
 880 #if SANITIZER_GO
 881 NOINLINE
 882 void GrowShadowStack(ThreadState *thr) {
 883   const int sz = thr->shadow_stack_end - thr->shadow_stack;
 884   const int newsz = 2 * sz;
 885   auto *newstack = (uptr *)Alloc(newsz * sizeof(uptr));
 886   internal_memcpy(newstack, thr->shadow_stack, sz * sizeof(uptr));
 887   Free(thr->shadow_stack);
 888   thr->shadow_stack = newstack;
 889   thr->shadow_stack_pos = newstack + sz;
 890   thr->shadow_stack_end = newstack + newsz;
 891 }
 892 #endif
 893
 894 StackID CurrentStackId(ThreadState *thr, uptr pc) {
 895 #if !SANITIZER_GO
 896   if (!thr->is_inited)  // May happen during bootstrap.
 897     return kInvalidStackID;
 898 #endif
 899   if (pc != 0) {
 900 #if !SANITIZER_GO
 901     DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
 902 #else
 903     if (thr->shadow_stack_pos == thr->shadow_stack_end)
 904       GrowShadowStack(thr);
 905 #endif
 906     thr->shadow_stack_pos[0] = pc;
 907     thr->shadow_stack_pos++;
 908   }
 909   StackID id = StackDepotPut(
 910       StackTrace(thr->shadow_stack, thr->shadow_stack_pos - thr->shadow_stack));
 911   if (pc != 0)
 912     thr->shadow_stack_pos--;
 913   return id;
 914 }
 915
 916 static bool TraceSkipGap(ThreadState* thr) {
 917   Trace *trace = &thr->tctx->trace;
 918   Event *pos = reinterpret_cast<Event *>(atomic_load_relaxed(&thr->trace_pos));
 919   DCHECK_EQ(reinterpret_cast<uptr>(pos + 1) & TracePart::kAlignment, 0);
 920   auto *part = trace->parts.Back();
 921   DPrintf("#%d: TraceSwitchPart enter trace=%p parts=%p-%p pos=%p\n", thr->tid,
 922           trace, trace->parts.Front(), part, pos);
 923   if (!part)
 924     return false;
 925   // We can get here when we still have space in the current trace part.
 926   // The fast-path check in TraceAcquire has false positives in the middle of
 927   // the part. Check if we are indeed at the end of the current part or not,
 928   // and fill any gaps with NopEvent's.
 929   Event* end = &part->events[TracePart::kSize];
 930   DCHECK_GE(pos, &part->events[0]);
 931   DCHECK_LE(pos, end);
 932   if (pos + 1 < end) {
 933     if ((reinterpret_cast<uptr>(pos) & TracePart::kAlignment) ==
 934         TracePart::kAlignment)
 935       *pos++ = NopEvent;
 936     *pos++ = NopEvent;
 937     DCHECK_LE(pos + 2, end);
 938     atomic_store_relaxed(&thr->trace_pos, reinterpret_cast<uptr>(pos));
 939     return true;
 940   }
 941   // We are indeed at the end.
 942   for (; pos < end; pos++) *pos = NopEvent;
 943   return false;
 944 }
 945
 946 NOINLINE
 947 void TraceSwitchPart(ThreadState* thr) {
 948   if (TraceSkipGap(thr))
 949     return;
 950 #if !SANITIZER_GO
 951   if (ctx->after_multithreaded_fork) {
 952     // We just need to survive till exec.
 953     TracePart* part = thr->tctx->trace.parts.Back();
 954     if (part) {
 955       atomic_store_relaxed(&thr->trace_pos,
 956                            reinterpret_cast<uptr>(&part->events[0]));
 957       return;
 958     }
 959   }
 960 #endif
 961   TraceSwitchPartImpl(thr);
 962 }
 963
 964 void TraceSwitchPartImpl(ThreadState* thr) {
 965   SlotLocker locker(thr, true);
 966   Trace* trace = &thr->tctx->trace;
 967   TracePart* part = TracePartAlloc(thr);
 968   part->trace = trace;
 969   thr->trace_prev_pc = 0;
 970   TracePart* recycle = nullptr;
 971   // Keep roughly half of parts local to the thread
 972   // (not queued into the recycle queue).
 973   uptr local_parts = (Trace::kMinParts + flags()->history_size + 1) / 2;
 974   {
 975     Lock lock(&trace->mtx);
 976     if (trace->parts.Empty())
 977       trace->local_head = part;
 978     if (trace->parts.Size() >= local_parts) {
 979       recycle = trace->local_head;
 980       trace->local_head = trace->parts.Next(recycle);
 981     }
 982     trace->parts.PushBack(part);
 983     atomic_store_relaxed(&thr->trace_pos,
 984                          reinterpret_cast<uptr>(&part->events[0]));
 985   }
 986   // Make this part self-sufficient by restoring the current stack
 987   // and mutex set in the beginning of the trace.
 988   TraceTime(thr);
 989   {
 990     // Pathologically large stacks may not fit into the part.
 991     // In these cases we log only fixed number of top frames.
 992     const uptr kMaxFrames = 1000;
 993     // Check that kMaxFrames won't consume the whole part.
 994     static_assert(kMaxFrames < TracePart::kSize / 2, "kMaxFrames is too big");
 995     uptr* pos = Max(&thr->shadow_stack[0], thr->shadow_stack_pos - kMaxFrames);
 996     for (; pos < thr->shadow_stack_pos; pos++) {
 997       if (TryTraceFunc(thr, *pos))
 998         continue;
 999       CHECK(TraceSkipGap(thr));
1000       CHECK(TryTraceFunc(thr, *pos));
1001     }
1002   }
1003   for (uptr i = 0; i < thr->mset.Size(); i++) {
1004     MutexSet::Desc d = thr->mset.Get(i);
1005     for (uptr i = 0; i < d.count; i++)
1006       TraceMutexLock(thr, d.write ? EventType::kLock : EventType::kRLock, 0,
1007                      d.addr, d.stack_id);
1008   }
1009   // Callers of TraceSwitchPart expect that TraceAcquire will always succeed
1010   // after the call. It's possible that TryTraceFunc/TraceMutexLock above
1011   // filled the trace part exactly up to the TracePart::kAlignment gap
1012   // and the next TraceAcquire won't succeed. Skip the gap to avoid that.
1013   EventFunc *ev;
1014   if (!TraceAcquire(thr, &ev)) {
1015     CHECK(TraceSkipGap(thr));
1016     CHECK(TraceAcquire(thr, &ev));
1017   }
1018   {
1019     Lock lock(&ctx->slot_mtx);
1020     // There is a small chance that the slot may be not queued at this point.
1021     // This can happen if the slot has kEpochLast epoch and another thread
1022     // in FindSlotAndLock discovered that it's exhausted and removed it from
1023     // the slot queue. kEpochLast can happen in 2 cases: (1) if TraceSwitchPart
1024     // was called with the slot locked and epoch already at kEpochLast,
1025     // or (2) if we've acquired a new slot in SlotLock in the beginning
1026     // of the function and the slot was at kEpochLast - 1, so after increment
1027     // in SlotAttachAndLock it become kEpochLast.
1028     if (ctx->slot_queue.Queued(thr->slot)) {
1029       ctx->slot_queue.Remove(thr->slot);
1030       ctx->slot_queue.PushBack(thr->slot);
1031     }
1032     if (recycle)
1033       ctx->trace_part_recycle.PushBack(recycle);
1034   }
1035   DPrintf("#%d: TraceSwitchPart exit parts=%p-%p pos=0x%zx\n", thr->tid,
1036           trace->parts.Front(), trace->parts.Back(),
1037           atomic_load_relaxed(&thr->trace_pos));
1038 }
1039
1040 void ThreadIgnoreBegin(ThreadState* thr, uptr pc) {
1041   DPrintf("#%d: ThreadIgnoreBegin\n", thr->tid);
1042   thr->ignore_reads_and_writes++;
1043   CHECK_GT(thr->ignore_reads_and_writes, 0);
1044   thr->fast_state.SetIgnoreBit();
1045 #if !SANITIZER_GO
1046   if (pc && !ctx->after_multithreaded_fork)
1047     thr->mop_ignore_set.Add(CurrentStackId(thr, pc));
1048 #endif
1049 }
1050
1051 void ThreadIgnoreEnd(ThreadState *thr) {
1052   DPrintf("#%d: ThreadIgnoreEnd\n", thr->tid);
1053   CHECK_GT(thr->ignore_reads_and_writes, 0);
1054   thr->ignore_reads_and_writes--;
1055   if (thr->ignore_reads_and_writes == 0) {
1056     thr->fast_state.ClearIgnoreBit();
1057 #if !SANITIZER_GO
1058     thr->mop_ignore_set.Reset();
1059 #endif
1060   }
1061 }
1062
1063 #if !SANITIZER_GO
1064 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
1065 uptr __tsan_testonly_shadow_stack_current_size() {
1066   ThreadState *thr = cur_thread();
1067   return thr->shadow_stack_pos - thr->shadow_stack;
1068 }
1069 #endif
1070
1071 void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc) {
1072   DPrintf("#%d: ThreadIgnoreSyncBegin\n", thr->tid);
1073   thr->ignore_sync++;
1074   CHECK_GT(thr->ignore_sync, 0);
1075 #if !SANITIZER_GO
1076   if (pc && !ctx->after_multithreaded_fork)
1077     thr->sync_ignore_set.Add(CurrentStackId(thr, pc));
1078 #endif
1079 }
1080
1081 void ThreadIgnoreSyncEnd(ThreadState *thr) {
1082   DPrintf("#%d: ThreadIgnoreSyncEnd\n", thr->tid);
1083   CHECK_GT(thr->ignore_sync, 0);
1084   thr->ignore_sync--;
1085 #if !SANITIZER_GO
1086   if (thr->ignore_sync == 0)
1087     thr->sync_ignore_set.Reset();
1088 #endif
1089 }
1090
1091 bool MD5Hash::operator==(const MD5Hash &other) const {
1092   return hash[0] == other.hash[0] && hash[1] == other.hash[1];
1093 }
1094
1095 #if SANITIZER_DEBUG
1096 void build_consistency_debug() {}
1097 #else
1098 void build_consistency_release() {}
1099 #endif
1100 }  // namespace __tsan
1101
1102 #if SANITIZER_CHECK_DEADLOCKS
1103 namespace __sanitizer {
1104 using namespace __tsan;
1105 MutexMeta mutex_meta[] = {
1106     {MutexInvalid, "Invalid", {}},
1107     {MutexThreadRegistry,
1108      "ThreadRegistry",
1109      {MutexTypeSlots, MutexTypeTrace, MutexTypeReport}},
1110     {MutexTypeReport, "Report", {MutexTypeTrace}},
1111     {MutexTypeSyncVar, "SyncVar", {MutexTypeReport, MutexTypeTrace}},
1112     {MutexTypeAnnotations, "Annotations", {}},
1113     {MutexTypeAtExit, "AtExit", {}},
1114     {MutexTypeFired, "Fired", {MutexLeaf}},
1115     {MutexTypeRacy, "Racy", {MutexLeaf}},
1116     {MutexTypeGlobalProc, "GlobalProc", {MutexTypeSlot, MutexTypeSlots}},
1117     {MutexTypeInternalAlloc, "InternalAlloc", {MutexLeaf}},
1118     {MutexTypeTrace, "Trace", {}},
1119     {MutexTypeSlot,
1120      "Slot",
1121      {MutexMulti, MutexTypeTrace, MutexTypeSyncVar, MutexThreadRegistry,
1122       MutexTypeSlots}},
1123     {MutexTypeSlots, "Slots", {MutexTypeTrace, MutexTypeReport}},
1124     {},
1125 };
1126
1127 void PrintMutexPC(uptr pc) { StackTrace(&pc, 1).Print(); }
1128
1129 }  // namespace __sanitizer
1130 #endif