compiler-rt/lib/tsan/rtl/tsan_rtl.h

   1 //===-- tsan_rtl.h ----------------------------------------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file is a part of ThreadSanitizer (TSan), a race detector.
  10 //
  11 // Main internal TSan header file.
  12 //
  13 // Ground rules:
  14 //   - C++ run-time should not be used (static CTORs, RTTI, exceptions, static
  15 //     function-scope locals)
  16 //   - All functions/classes/etc reside in namespace __tsan, except for those
  17 //     declared in tsan_interface.h.
  18 //   - Platform-specific files should be used instead of ifdefs (*).
  19 //   - No system headers included in header files (*).
  20 //   - Platform specific headres included only into platform-specific files (*).
  21 //
  22 //  (*) Except when inlining is critical for performance.
  23 //===----------------------------------------------------------------------===//
  24
  25 #ifndef TSAN_RTL_H
  26 #define TSAN_RTL_H
  27
  28 #include "sanitizer_common/sanitizer_allocator.h"
  29 #include "sanitizer_common/sanitizer_allocator_internal.h"
  30 #include "sanitizer_common/sanitizer_asm.h"
  31 #include "sanitizer_common/sanitizer_common.h"
  32 #include "sanitizer_common/sanitizer_deadlock_detector_interface.h"
  33 #include "sanitizer_common/sanitizer_libignore.h"
  34 #include "sanitizer_common/sanitizer_suppressions.h"
  35 #include "sanitizer_common/sanitizer_thread_registry.h"
  36 #include "sanitizer_common/sanitizer_vector.h"
  37 #include "tsan_defs.h"
  38 #include "tsan_flags.h"
  39 #include "tsan_ignoreset.h"
  40 #include "tsan_ilist.h"
  41 #include "tsan_mman.h"
  42 #include "tsan_mutexset.h"
  43 #include "tsan_platform.h"
  44 #include "tsan_report.h"
  45 #include "tsan_shadow.h"
  46 #include "tsan_stack_trace.h"
  47 #include "tsan_sync.h"
  48 #include "tsan_trace.h"
  49 #include "tsan_vector_clock.h"
  50
  51 #if SANITIZER_WORDSIZE != 64
  52 # error "ThreadSanitizer is supported only on 64-bit platforms"
  53 #endif
  54
  55 namespace __tsan {
  56
  57 #if !SANITIZER_GO
  58 struct MapUnmapCallback;
  59 #  if defined(__mips64) || defined(__aarch64__) || defined(__loongarch__) || \
  60       defined(__powerpc__) || SANITIZER_RISCV64
  61
  62 struct AP32 {
  63   static const uptr kSpaceBeg = 0;
  64   static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE;
  65   static const uptr kMetadataSize = 0;
  66   typedef __sanitizer::CompactSizeClassMap SizeClassMap;
  67   static const uptr kRegionSizeLog = 20;
  68   using AddressSpaceView = LocalAddressSpaceView;
  69   typedef __tsan::MapUnmapCallback MapUnmapCallback;
  70   static const uptr kFlags = 0;
  71 };
  72 typedef SizeClassAllocator32<AP32> PrimaryAllocator;
  73 #else
  74 struct AP64 {  // Allocator64 parameters. Deliberately using a short name.
  75 #    if defined(__s390x__)
  76   typedef MappingS390x Mapping;
  77 #    else
  78   typedef Mapping48AddressSpace Mapping;
  79 #    endif
  80   static const uptr kSpaceBeg = Mapping::kHeapMemBeg;
  81   static const uptr kSpaceSize = Mapping::kHeapMemEnd - Mapping::kHeapMemBeg;
  82   static const uptr kMetadataSize = 0;
  83   typedef DefaultSizeClassMap SizeClassMap;
  84   typedef __tsan::MapUnmapCallback MapUnmapCallback;
  85   static const uptr kFlags = 0;
  86   using AddressSpaceView = LocalAddressSpaceView;
  87 };
  88 typedef SizeClassAllocator64<AP64> PrimaryAllocator;
  89 #endif
  90 typedef CombinedAllocator<PrimaryAllocator> Allocator;
  91 typedef Allocator::AllocatorCache AllocatorCache;
  92 Allocator *allocator();
  93 #endif
  94
  95 struct ThreadSignalContext;
  96
  97 struct JmpBuf {
  98   uptr sp;
  99   int int_signal_send;
 100   bool in_blocking_func;
 101   uptr in_signal_handler;
 102   uptr *shadow_stack_pos;
 103 };
 104
 105 // A Processor represents a physical thread, or a P for Go.
 106 // It is used to store internal resources like allocate cache, and does not
 107 // participate in race-detection logic (invisible to end user).
 108 // In C++ it is tied to an OS thread just like ThreadState, however ideally
 109 // it should be tied to a CPU (this way we will have fewer allocator caches).
 110 // In Go it is tied to a P, so there are significantly fewer Processor's than
 111 // ThreadState's (which are tied to Gs).
 112 // A ThreadState must be wired with a Processor to handle events.
 113 struct Processor {
 114   ThreadState *thr; // currently wired thread, or nullptr
 115 #if !SANITIZER_GO
 116   AllocatorCache alloc_cache;
 117   InternalAllocatorCache internal_alloc_cache;
 118 #endif
 119   DenseSlabAllocCache block_cache;
 120   DenseSlabAllocCache sync_cache;
 121   DDPhysicalThread *dd_pt;
 122 };
 123
 124 #if !SANITIZER_GO
 125 // ScopedGlobalProcessor temporary setups a global processor for the current
 126 // thread, if it does not have one. Intended for interceptors that can run
 127 // at the very thread end, when we already destroyed the thread processor.
 128 struct ScopedGlobalProcessor {
 129   ScopedGlobalProcessor();
 130   ~ScopedGlobalProcessor();
 131 };
 132 #endif
 133
 134 struct TidEpoch {
 135   Tid tid;
 136   Epoch epoch;
 137 };
 138
 139 struct alignas(SANITIZER_CACHE_LINE_SIZE) TidSlot {
 140   Mutex mtx;
 141   Sid sid;
 142   atomic_uint32_t raw_epoch;
 143   ThreadState *thr;
 144   Vector<TidEpoch> journal;
 145   INode node;
 146
 147   Epoch epoch() const {
 148     return static_cast<Epoch>(atomic_load(&raw_epoch, memory_order_relaxed));
 149   }
 150
 151   void SetEpoch(Epoch v) {
 152     atomic_store(&raw_epoch, static_cast<u32>(v), memory_order_relaxed);
 153   }
 154
 155   TidSlot();
 156 };
 157
 158 // This struct is stored in TLS.
 159 struct alignas(SANITIZER_CACHE_LINE_SIZE) ThreadState {
 160   FastState fast_state;
 161   int ignore_sync;
 162 #if !SANITIZER_GO
 163   int ignore_interceptors;
 164 #endif
 165   uptr *shadow_stack_pos;
 166
 167   // Current position in tctx->trace.Back()->events (Event*).
 168   atomic_uintptr_t trace_pos;
 169   // PC of the last memory access, used to compute PC deltas in the trace.
 170   uptr trace_prev_pc;
 171
 172   // Technically `current` should be a separate THREADLOCAL variable;
 173   // but it is placed here in order to share cache line with previous fields.
 174   ThreadState* current;
 175
 176   atomic_sint32_t pending_signals;
 177
 178   VectorClock clock;
 179
 180   // This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read.
 181   // We do not distinguish beteween ignoring reads and writes
 182   // for better performance.
 183   int ignore_reads_and_writes;
 184   int suppress_reports;
 185   // Go does not support ignores.
 186 #if !SANITIZER_GO
 187   IgnoreSet mop_ignore_set;
 188   IgnoreSet sync_ignore_set;
 189 #endif
 190   uptr *shadow_stack;
 191   uptr *shadow_stack_end;
 192 #if !SANITIZER_GO
 193   Vector<JmpBuf> jmp_bufs;
 194   int in_symbolizer;
 195   atomic_uintptr_t in_blocking_func;
 196   bool in_ignored_lib;
 197   bool is_inited;
 198 #endif
 199   MutexSet mset;
 200   bool is_dead;
 201   const Tid tid;
 202   uptr stk_addr;
 203   uptr stk_size;
 204   uptr tls_addr;
 205   uptr tls_size;
 206   ThreadContext *tctx;
 207
 208   DDLogicalThread *dd_lt;
 209
 210   TidSlot *slot;
 211   uptr slot_epoch;
 212   bool slot_locked;
 213
 214   // Current wired Processor, or nullptr. Required to handle any events.
 215   Processor *proc1;
 216 #if !SANITIZER_GO
 217   Processor *proc() { return proc1; }
 218 #else
 219   Processor *proc();
 220 #endif
 221
 222   atomic_uintptr_t in_signal_handler;
 223   atomic_uintptr_t signal_ctx;
 224
 225 #if !SANITIZER_GO
 226   StackID last_sleep_stack_id;
 227   VectorClock last_sleep_clock;
 228 #endif
 229
 230   // Set in regions of runtime that must be signal-safe and fork-safe.
 231   // If set, malloc must not be called.
 232   int nomalloc;
 233
 234   const ReportDesc *current_report;
 235
 236   explicit ThreadState(Tid tid);
 237 };
 238
 239 #if !SANITIZER_GO
 240 #if SANITIZER_APPLE || SANITIZER_ANDROID
 241 ThreadState *cur_thread();
 242 void set_cur_thread(ThreadState *thr);
 243 void cur_thread_finalize();
 244 inline ThreadState *cur_thread_init() { return cur_thread(); }
 245 #  else
 246 __attribute__((tls_model("initial-exec")))
 247 extern THREADLOCAL char cur_thread_placeholder[];
 248 inline ThreadState *cur_thread() {
 249   return reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current;
 250 }
 251 inline ThreadState *cur_thread_init() {
 252   ThreadState *thr = reinterpret_cast<ThreadState *>(cur_thread_placeholder);
 253   if (UNLIKELY(!thr->current))
 254     thr->current = thr;
 255   return thr->current;
 256 }
 257 inline void set_cur_thread(ThreadState *thr) {
 258   reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current = thr;
 259 }
 260 inline void cur_thread_finalize() { }
 261 #  endif  // SANITIZER_APPLE || SANITIZER_ANDROID
 262 #endif  // SANITIZER_GO
 263
 264 class ThreadContext final : public ThreadContextBase {
 265  public:
 266   explicit ThreadContext(Tid tid);
 267   ~ThreadContext();
 268   ThreadState *thr;
 269   StackID creation_stack_id;
 270   VectorClock *sync;
 271   uptr sync_epoch;
 272   Trace trace;
 273
 274   // Override superclass callbacks.
 275   void OnDead() override;
 276   void OnJoined(void *arg) override;
 277   void OnFinished() override;
 278   void OnStarted(void *arg) override;
 279   void OnCreated(void *arg) override;
 280   void OnReset() override;
 281   void OnDetached(void *arg) override;
 282 };
 283
 284 struct RacyStacks {
 285   MD5Hash hash[2];
 286   bool operator==(const RacyStacks &other) const;
 287 };
 288
 289 struct RacyAddress {
 290   uptr addr_min;
 291   uptr addr_max;
 292 };
 293
 294 struct FiredSuppression {
 295   ReportType type;
 296   uptr pc_or_addr;
 297   Suppression *supp;
 298 };
 299
 300 struct Context {
 301   Context();
 302
 303   bool initialized;
 304 #if !SANITIZER_GO
 305   bool after_multithreaded_fork;
 306 #endif
 307
 308   MetaMap metamap;
 309
 310   Mutex report_mtx;
 311   int nreported;
 312   atomic_uint64_t last_symbolize_time_ns;
 313
 314   void *background_thread;
 315   atomic_uint32_t stop_background_thread;
 316
 317   ThreadRegistry thread_registry;
 318
 319   // This is used to prevent a very unlikely but very pathological behavior.
 320   // Since memory access handling is not synchronized with DoReset,
 321   // a thread running concurrently with DoReset can leave a bogus shadow value
 322   // that will be later falsely detected as a race. For such false races
 323   // RestoreStack will return false and we will not report it.
 324   // However, consider that a thread leaves a whole lot of such bogus values
 325   // and these values are later read by a whole lot of threads.
 326   // This will cause massive amounts of ReportRace calls and lots of
 327   // serialization. In very pathological cases the resulting slowdown
 328   // can be >100x. This is very unlikely, but it was presumably observed
 329   // in practice: https://github.com/google/sanitizers/issues/1552
 330   // If this happens, previous access sid+epoch will be the same for all of
 331   // these false races b/c if the thread will try to increment epoch, it will
 332   // notice that DoReset has happened and will stop producing bogus shadow
 333   // values. So, last_spurious_race is used to remember the last sid+epoch
 334   // for which RestoreStack returned false. Then it is used to filter out
 335   // races with the same sid+epoch very early and quickly.
 336   // It is of course possible that multiple threads left multiple bogus shadow
 337   // values and all of them are read by lots of threads at the same time.
 338   // In such case last_spurious_race will only be able to deduplicate a few
 339   // races from one thread, then few from another and so on. An alternative
 340   // would be to hold an array of such sid+epoch, but we consider such scenario
 341   // as even less likely.
 342   // Note: this can lead to some rare false negatives as well:
 343   // 1. When a legit access with the same sid+epoch participates in a race
 344   // as the "previous" memory access, it will be wrongly filtered out.
 345   // 2. When RestoreStack returns false for a legit memory access because it
 346   // was already evicted from the thread trace, we will still remember it in
 347   // last_spurious_race. Then if there is another racing memory access from
 348   // the same thread that happened in the same epoch, but was stored in the
 349   // next thread trace part (which is still preserved in the thread trace),
 350   // we will also wrongly filter it out while RestoreStack would actually
 351   // succeed for that second memory access.
 352   RawShadow last_spurious_race;
 353
 354   Mutex racy_mtx;
 355   Vector<RacyStacks> racy_stacks;
 356   // Number of fired suppressions may be large enough.
 357   Mutex fired_suppressions_mtx;
 358   InternalMmapVector<FiredSuppression> fired_suppressions;
 359   DDetector *dd;
 360
 361   Flags flags;
 362   fd_t memprof_fd;
 363
 364   // The last slot index (kFreeSid) is used to denote freed memory.
 365   TidSlot slots[kThreadSlotCount - 1];
 366
 367   // Protects global_epoch, slot_queue, trace_part_recycle.
 368   Mutex slot_mtx;
 369   uptr global_epoch;  // guarded by slot_mtx and by all slot mutexes
 370   bool resetting;     // global reset is in progress
 371   IList<TidSlot, &TidSlot::node> slot_queue SANITIZER_GUARDED_BY(slot_mtx);
 372   IList<TraceHeader, &TraceHeader::global, TracePart> trace_part_recycle
 373       SANITIZER_GUARDED_BY(slot_mtx);
 374   uptr trace_part_total_allocated SANITIZER_GUARDED_BY(slot_mtx);
 375   uptr trace_part_recycle_finished SANITIZER_GUARDED_BY(slot_mtx);
 376   uptr trace_part_finished_excess SANITIZER_GUARDED_BY(slot_mtx);
 377 #if SANITIZER_GO
 378   uptr mapped_shadow_begin;
 379   uptr mapped_shadow_end;
 380 #endif
 381 };
 382
 383 extern Context *ctx;  // The one and the only global runtime context.
 384
 385 ALWAYS_INLINE Flags *flags() {
 386   return &ctx->flags;
 387 }
 388
 389 struct ScopedIgnoreInterceptors {
 390   ScopedIgnoreInterceptors() {
 391 #if !SANITIZER_GO
 392     cur_thread()->ignore_interceptors++;
 393 #endif
 394   }
 395
 396   ~ScopedIgnoreInterceptors() {
 397 #if !SANITIZER_GO
 398     cur_thread()->ignore_interceptors--;
 399 #endif
 400   }
 401 };
 402
 403 const char *GetObjectTypeFromTag(uptr tag);
 404 const char *GetReportHeaderFromTag(uptr tag);
 405 uptr TagFromShadowStackFrame(uptr pc);
 406
 407 class ScopedReportBase {
 408  public:
 409   void AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, Tid tid,
 410                        StackTrace stack, const MutexSet *mset);
 411   void AddStack(StackTrace stack, bool suppressable = false);
 412   void AddThread(const ThreadContext *tctx, bool suppressable = false);
 413   void AddThread(Tid tid, bool suppressable = false);
 414   void AddUniqueTid(Tid unique_tid);
 415   int AddMutex(uptr addr, StackID creation_stack_id);
 416   void AddLocation(uptr addr, uptr size);
 417   void AddSleep(StackID stack_id);
 418   void SetCount(int count);
 419   void SetSigNum(int sig);
 420
 421   const ReportDesc *GetReport() const;
 422
 423  protected:
 424   ScopedReportBase(ReportType typ, uptr tag);
 425   ~ScopedReportBase();
 426
 427  private:
 428   ReportDesc *rep_;
 429   // Symbolizer makes lots of intercepted calls. If we try to process them,
 430   // at best it will cause deadlocks on internal mutexes.
 431   ScopedIgnoreInterceptors ignore_interceptors_;
 432
 433   ScopedReportBase(const ScopedReportBase &) = delete;
 434   void operator=(const ScopedReportBase &) = delete;
 435 };
 436
 437 class ScopedReport : public ScopedReportBase {
 438  public:
 439   explicit ScopedReport(ReportType typ, uptr tag = kExternalTagNone);
 440   ~ScopedReport();
 441
 442  private:
 443   ScopedErrorReportLock lock_;
 444 };
 445
 446 bool ShouldReport(ThreadState *thr, ReportType typ);
 447 ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack);
 448
 449 // The stack could look like:
 450 //   <start> | <main> | <foo> | tag | <bar>
 451 // This will extract the tag and keep:
 452 //   <start> | <main> | <foo> | <bar>
 453 template<typename StackTraceTy>
 454 void ExtractTagFromStack(StackTraceTy *stack, uptr *tag = nullptr) {
 455   if (stack->size < 2) return;
 456   uptr possible_tag_pc = stack->trace[stack->size - 2];
 457   uptr possible_tag = TagFromShadowStackFrame(possible_tag_pc);
 458   if (possible_tag == kExternalTagNone) return;
 459   stack->trace_buffer[stack->size - 2] = stack->trace_buffer[stack->size - 1];
 460   stack->size -= 1;
 461   if (tag) *tag = possible_tag;
 462 }
 463
 464 template<typename StackTraceTy>
 465 void ObtainCurrentStack(ThreadState *thr, uptr toppc, StackTraceTy *stack,
 466                         uptr *tag = nullptr) {
 467   uptr size = thr->shadow_stack_pos - thr->shadow_stack;
 468   uptr start = 0;
 469   if (size + !!toppc > kStackTraceMax) {
 470     start = size + !!toppc - kStackTraceMax;
 471     size = kStackTraceMax - !!toppc;
 472   }
 473   stack->Init(&thr->shadow_stack[start], size, toppc);
 474   ExtractTagFromStack(stack, tag);
 475 }
 476
 477 #define GET_STACK_TRACE_FATAL(thr, pc) \
 478   VarSizeStackTrace stack; \
 479   ObtainCurrentStack(thr, pc, &stack); \
 480   stack.ReverseOrder();
 481
 482 void MapShadow(uptr addr, uptr size);
 483 void MapThreadTrace(uptr addr, uptr size, const char *name);
 484 void DontNeedShadowFor(uptr addr, uptr size);
 485 void UnmapShadow(ThreadState *thr, uptr addr, uptr size);
 486 void InitializeShadowMemory();
 487 void DontDumpShadow(uptr addr, uptr size);
 488 void InitializeInterceptors();
 489 void InitializeLibIgnore();
 490 void InitializeDynamicAnnotations();
 491
 492 void ForkBefore(ThreadState *thr, uptr pc);
 493 void ForkParentAfter(ThreadState *thr, uptr pc);
 494 void ForkChildAfter(ThreadState *thr, uptr pc, bool start_thread);
 495
 496 void ReportRace(ThreadState *thr, RawShadow *shadow_mem, Shadow cur, Shadow old,
 497                 AccessType typ);
 498 bool OutputReport(ThreadState *thr, const ScopedReport &srep);
 499 bool IsFiredSuppression(Context *ctx, ReportType type, StackTrace trace);
 500 bool IsExpectedReport(uptr addr, uptr size);
 501
 502 #if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 1
 503 # define DPrintf Printf
 504 #else
 505 # define DPrintf(...)
 506 #endif
 507
 508 #if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 2
 509 # define DPrintf2 Printf
 510 #else
 511 # define DPrintf2(...)
 512 #endif
 513
 514 StackID CurrentStackId(ThreadState *thr, uptr pc);
 515 ReportStack *SymbolizeStackId(StackID stack_id);
 516 void PrintCurrentStack(ThreadState *thr, uptr pc);
 517 void PrintCurrentStack(uptr pc, bool fast);  // may uses libunwind
 518 MBlock *JavaHeapBlock(uptr addr, uptr *start);
 519
 520 void Initialize(ThreadState *thr);
 521 void MaybeSpawnBackgroundThread();
 522 int Finalize(ThreadState *thr);
 523
 524 void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write);
 525 void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write);
 526
 527 void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
 528                   AccessType typ);
 529 void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
 530                            AccessType typ);
 531 // This creates 2 non-inlined specialized versions of MemoryAccessRange.
 532 template <bool is_read>
 533 void MemoryAccessRangeT(ThreadState *thr, uptr pc, uptr addr, uptr size);
 534
 535 ALWAYS_INLINE
 536 void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
 537                        bool is_write) {
 538   if (size == 0)
 539     return;
 540   if (is_write)
 541     MemoryAccessRangeT<false>(thr, pc, addr, size);
 542   else
 543     MemoryAccessRangeT<true>(thr, pc, addr, size);
 544 }
 545
 546 void ShadowSet(RawShadow *p, RawShadow *end, RawShadow v);
 547 void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size);
 548 void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size);
 549 void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size);
 550 void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr,
 551                                          uptr size);
 552
 553 void ThreadIgnoreBegin(ThreadState *thr, uptr pc);
 554 void ThreadIgnoreEnd(ThreadState *thr);
 555 void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc);
 556 void ThreadIgnoreSyncEnd(ThreadState *thr);
 557
 558 Tid ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached);
 559 void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id,
 560                  ThreadType thread_type);
 561 void ThreadFinish(ThreadState *thr);
 562 Tid ThreadConsumeTid(ThreadState *thr, uptr pc, uptr uid);
 563 void ThreadJoin(ThreadState *thr, uptr pc, Tid tid);
 564 void ThreadDetach(ThreadState *thr, uptr pc, Tid tid);
 565 void ThreadFinalize(ThreadState *thr);
 566 void ThreadSetName(ThreadState *thr, const char *name);
 567 int ThreadCount(ThreadState *thr);
 568 void ProcessPendingSignalsImpl(ThreadState *thr);
 569 void ThreadNotJoined(ThreadState *thr, uptr pc, Tid tid, uptr uid);
 570
 571 Processor *ProcCreate();
 572 void ProcDestroy(Processor *proc);
 573 void ProcWire(Processor *proc, ThreadState *thr);
 574 void ProcUnwire(Processor *proc, ThreadState *thr);
 575
 576 // Note: the parameter is called flagz, because flags is already taken
 577 // by the global function that returns flags.
 578 void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
 579 void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
 580 void MutexPreLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
 581 void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0,
 582     int rec = 1);
 583 int  MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
 584 void MutexPreReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
 585 void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
 586 void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr);
 587 void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr);
 588 void MutexRepair(ThreadState *thr, uptr pc, uptr addr);  // call on EOWNERDEAD
 589 void MutexInvalidAccess(ThreadState *thr, uptr pc, uptr addr);
 590
 591 void Acquire(ThreadState *thr, uptr pc, uptr addr);
 592 // AcquireGlobal synchronizes the current thread with all other threads.
 593 // In terms of happens-before relation, it draws a HB edge from all threads
 594 // (where they happen to execute right now) to the current thread. We use it to
 595 // handle Go finalizers. Namely, finalizer goroutine executes AcquireGlobal
 596 // right before executing finalizers. This provides a coarse, but simple
 597 // approximation of the actual required synchronization.
 598 void AcquireGlobal(ThreadState *thr);
 599 void Release(ThreadState *thr, uptr pc, uptr addr);
 600 void ReleaseStoreAcquire(ThreadState *thr, uptr pc, uptr addr);
 601 void ReleaseStore(ThreadState *thr, uptr pc, uptr addr);
 602 void AfterSleep(ThreadState *thr, uptr pc);
 603 void IncrementEpoch(ThreadState *thr);
 604
 605 #if !SANITIZER_GO
 606 uptr ALWAYS_INLINE HeapEnd() {
 607   return HeapMemEnd() + PrimaryAllocator::AdditionalSize();
 608 }
 609 #endif
 610
 611 void SlotAttachAndLock(ThreadState *thr) SANITIZER_ACQUIRE(thr->slot->mtx);
 612 void SlotDetach(ThreadState *thr);
 613 void SlotLock(ThreadState *thr) SANITIZER_ACQUIRE(thr->slot->mtx);
 614 void SlotUnlock(ThreadState *thr) SANITIZER_RELEASE(thr->slot->mtx);
 615 void DoReset(ThreadState *thr, uptr epoch);
 616 void FlushShadowMemory();
 617
 618 ThreadState *FiberCreate(ThreadState *thr, uptr pc, unsigned flags);
 619 void FiberDestroy(ThreadState *thr, uptr pc, ThreadState *fiber);
 620 void FiberSwitch(ThreadState *thr, uptr pc, ThreadState *fiber, unsigned flags);
 621
 622 // These need to match __tsan_switch_to_fiber_* flags defined in
 623 // tsan_interface.h. See documentation there as well.
 624 enum FiberSwitchFlags {
 625   FiberSwitchFlagNoSync = 1 << 0, // __tsan_switch_to_fiber_no_sync
 626 };
 627
 628 class SlotLocker {
 629  public:
 630   ALWAYS_INLINE
 631   SlotLocker(ThreadState *thr, bool recursive = false)
 632       : thr_(thr), locked_(recursive ? thr->slot_locked : false) {
 633 #if !SANITIZER_GO
 634     // We are in trouble if we are here with in_blocking_func set.
 635     // If in_blocking_func is set, all signals will be delivered synchronously,
 636     // which means we can't lock slots since the signal handler will try
 637     // to lock it recursively and deadlock.
 638     DCHECK(!atomic_load(&thr->in_blocking_func, memory_order_relaxed));
 639 #endif
 640     if (!locked_)
 641       SlotLock(thr_);
 642   }
 643
 644   ALWAYS_INLINE
 645   ~SlotLocker() {
 646     if (!locked_)
 647       SlotUnlock(thr_);
 648   }
 649
 650  private:
 651   ThreadState *thr_;
 652   bool locked_;
 653 };
 654
 655 class SlotUnlocker {
 656  public:
 657   SlotUnlocker(ThreadState *thr) : thr_(thr), locked_(thr->slot_locked) {
 658     if (locked_)
 659       SlotUnlock(thr_);
 660   }
 661
 662   ~SlotUnlocker() {
 663     if (locked_)
 664       SlotLock(thr_);
 665   }
 666
 667  private:
 668   ThreadState *thr_;
 669   bool locked_;
 670 };
 671
 672 ALWAYS_INLINE void ProcessPendingSignals(ThreadState *thr) {
 673   if (UNLIKELY(atomic_load_relaxed(&thr->pending_signals)))
 674     ProcessPendingSignalsImpl(thr);
 675 }
 676
 677 extern bool is_initialized;
 678
 679 ALWAYS_INLINE
 680 void LazyInitialize(ThreadState *thr) {
 681   // If we can use .preinit_array, assume that __tsan_init
 682   // called from .preinit_array initializes runtime before
 683   // any instrumented code except when tsan is used as a
 684   // shared library.
 685 #if (!SANITIZER_CAN_USE_PREINIT_ARRAY || defined(SANITIZER_SHARED))
 686   if (UNLIKELY(!is_initialized))
 687     Initialize(thr);
 688 #endif
 689 }
 690
 691 void TraceResetForTesting();
 692 void TraceSwitchPart(ThreadState *thr);
 693 void TraceSwitchPartImpl(ThreadState *thr);
 694 bool RestoreStack(EventType type, Sid sid, Epoch epoch, uptr addr, uptr size,
 695                   AccessType typ, Tid *ptid, VarSizeStackTrace *pstk,
 696                   MutexSet *pmset, uptr *ptag);
 697
 698 template <typename EventT>
 699 ALWAYS_INLINE WARN_UNUSED_RESULT bool TraceAcquire(ThreadState *thr,
 700                                                    EventT **ev) {
 701   // TraceSwitchPart accesses shadow_stack, but it's called infrequently,
 702   // so we check it here proactively.
 703   DCHECK(thr->shadow_stack);
 704   Event *pos = reinterpret_cast<Event *>(atomic_load_relaxed(&thr->trace_pos));
 705 #if SANITIZER_DEBUG
 706   // TraceSwitch acquires these mutexes,
 707   // so we lock them here to detect deadlocks more reliably.
 708   { Lock lock(&ctx->slot_mtx); }
 709   { Lock lock(&thr->tctx->trace.mtx); }
 710   TracePart *current = thr->tctx->trace.parts.Back();
 711   if (current) {
 712     DCHECK_GE(pos, &current->events[0]);
 713     DCHECK_LE(pos, &current->events[TracePart::kSize]);
 714   } else {
 715     DCHECK_EQ(pos, nullptr);
 716   }
 717 #endif
 718   // TracePart is allocated with mmap and is at least 4K aligned.
 719   // So the following check is a faster way to check for part end.
 720   // It may have false positives in the middle of the trace,
 721   // they are filtered out in TraceSwitch.
 722   if (UNLIKELY(((uptr)(pos + 1) & TracePart::kAlignment) == 0))
 723     return false;
 724   *ev = reinterpret_cast<EventT *>(pos);
 725   return true;
 726 }
 727
 728 template <typename EventT>
 729 ALWAYS_INLINE void TraceRelease(ThreadState *thr, EventT *evp) {
 730   DCHECK_LE(evp + 1, &thr->tctx->trace.parts.Back()->events[TracePart::kSize]);
 731   atomic_store_relaxed(&thr->trace_pos, (uptr)(evp + 1));
 732 }
 733
 734 template <typename EventT>
 735 void TraceEvent(ThreadState *thr, EventT ev) {
 736   EventT *evp;
 737   if (!TraceAcquire(thr, &evp)) {
 738     TraceSwitchPart(thr);
 739     UNUSED bool res = TraceAcquire(thr, &evp);
 740     DCHECK(res);
 741   }
 742   *evp = ev;
 743   TraceRelease(thr, evp);
 744 }
 745
 746 ALWAYS_INLINE WARN_UNUSED_RESULT bool TryTraceFunc(ThreadState *thr,
 747                                                    uptr pc = 0) {
 748   if (!kCollectHistory)
 749     return true;
 750   EventFunc *ev;
 751   if (UNLIKELY(!TraceAcquire(thr, &ev)))
 752     return false;
 753   ev->is_access = 0;
 754   ev->is_func = 1;
 755   ev->pc = pc;
 756   TraceRelease(thr, ev);
 757   return true;
 758 }
 759
 760 WARN_UNUSED_RESULT
 761 bool TryTraceMemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
 762                           AccessType typ);
 763 WARN_UNUSED_RESULT
 764 bool TryTraceMemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
 765                                AccessType typ);
 766 void TraceMemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
 767                             AccessType typ);
 768 void TraceFunc(ThreadState *thr, uptr pc = 0);
 769 void TraceMutexLock(ThreadState *thr, EventType type, uptr pc, uptr addr,
 770                     StackID stk);
 771 void TraceMutexUnlock(ThreadState *thr, uptr addr);
 772 void TraceTime(ThreadState *thr);
 773
 774 void TraceRestartFuncExit(ThreadState *thr);
 775 void TraceRestartFuncEntry(ThreadState *thr, uptr pc);
 776
 777 void GrowShadowStack(ThreadState *thr);
 778
 779 ALWAYS_INLINE
 780 void FuncEntry(ThreadState *thr, uptr pc) {
 781   DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.sid(), (void *)pc);
 782   if (UNLIKELY(!TryTraceFunc(thr, pc)))
 783     return TraceRestartFuncEntry(thr, pc);
 784   DCHECK_GE(thr->shadow_stack_pos, thr->shadow_stack);
 785 #if !SANITIZER_GO
 786   DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
 787 #else
 788   if (thr->shadow_stack_pos == thr->shadow_stack_end)
 789     GrowShadowStack(thr);
 790 #endif
 791   thr->shadow_stack_pos[0] = pc;
 792   thr->shadow_stack_pos++;
 793 }
 794
 795 ALWAYS_INLINE
 796 void FuncExit(ThreadState *thr) {
 797   DPrintf2("#%d: FuncExit\n", (int)thr->fast_state.sid());
 798   if (UNLIKELY(!TryTraceFunc(thr, 0)))
 799     return TraceRestartFuncExit(thr);
 800   DCHECK_GT(thr->shadow_stack_pos, thr->shadow_stack);
 801 #if !SANITIZER_GO
 802   DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
 803 #endif
 804   thr->shadow_stack_pos--;
 805 }
 806
 807 #if !SANITIZER_GO
 808 extern void (*on_initialize)(void);
 809 extern int (*on_finalize)(int);
 810 #endif
 811 }  // namespace __tsan
 812
 813 #endif  // TSAN_RTL_H