compiler-rt/lib/tsan/rtl/tsan_rtl.h

   1 //===-- tsan_rtl.h ----------------------------------------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file is a part of ThreadSanitizer (TSan), a race detector.
  10 //
  11 // Main internal TSan header file.
  12 //
  13 // Ground rules:
  14 //   - C++ run-time should not be used (static CTORs, RTTI, exceptions, static
  15 //     function-scope locals)
  16 //   - All functions/classes/etc reside in namespace __tsan, except for those
  17 //     declared in tsan_interface.h.
  18 //   - Platform-specific files should be used instead of ifdefs (*).
  19 //   - No system headers included in header files (*).
  20 //   - Platform specific headres included only into platform-specific files (*).
  21 //
  22 //  (*) Except when inlining is critical for performance.
  23 //===----------------------------------------------------------------------===//
  24
  25 #ifndef TSAN_RTL_H
  26 #define TSAN_RTL_H
  27
  28 #include "sanitizer_common/sanitizer_allocator.h"
  29 #include "sanitizer_common/sanitizer_allocator_internal.h"
  30 #include "sanitizer_common/sanitizer_asm.h"
  31 #include "sanitizer_common/sanitizer_common.h"
  32 #include "sanitizer_common/sanitizer_deadlock_detector_interface.h"
  33 #include "sanitizer_common/sanitizer_libignore.h"
  34 #include "sanitizer_common/sanitizer_suppressions.h"
  35 #include "sanitizer_common/sanitizer_thread_registry.h"
  36 #include "sanitizer_common/sanitizer_vector.h"
  37 #include "tsan_defs.h"
  38 #include "tsan_flags.h"
  39 #include "tsan_ignoreset.h"
  40 #include "tsan_ilist.h"
  41 #include "tsan_mman.h"
  42 #include "tsan_mutexset.h"
  43 #include "tsan_platform.h"
  44 #include "tsan_report.h"
  45 #include "tsan_shadow.h"
  46 #include "tsan_stack_trace.h"
  47 #include "tsan_sync.h"
  48 #include "tsan_trace.h"
  49 #include "tsan_vector_clock.h"
  50
  51 #if SANITIZER_WORDSIZE != 64
  52 # error "ThreadSanitizer is supported only on 64-bit platforms"
  53 #endif
  54
  55 namespace __tsan {
  56
  57 #if !SANITIZER_GO
  58 struct MapUnmapCallback;
  59 #if defined(__mips64) || defined(__aarch64__) || defined(__powerpc__)
  60
  61 struct AP32 {
  62   static const uptr kSpaceBeg = 0;
  63   static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE;
  64   static const uptr kMetadataSize = 0;
  65   typedef __sanitizer::CompactSizeClassMap SizeClassMap;
  66   static const uptr kRegionSizeLog = 20;
  67   using AddressSpaceView = LocalAddressSpaceView;
  68   typedef __tsan::MapUnmapCallback MapUnmapCallback;
  69   static const uptr kFlags = 0;
  70 };
  71 typedef SizeClassAllocator32<AP32> PrimaryAllocator;
  72 #else
  73 struct AP64 {  // Allocator64 parameters. Deliberately using a short name.
  74 #    if defined(__s390x__)
  75   typedef MappingS390x Mapping;
  76 #    else
  77   typedef Mapping48AddressSpace Mapping;
  78 #    endif
  79   static const uptr kSpaceBeg = Mapping::kHeapMemBeg;
  80   static const uptr kSpaceSize = Mapping::kHeapMemEnd - Mapping::kHeapMemBeg;
  81   static const uptr kMetadataSize = 0;
  82   typedef DefaultSizeClassMap SizeClassMap;
  83   typedef __tsan::MapUnmapCallback MapUnmapCallback;
  84   static const uptr kFlags = 0;
  85   using AddressSpaceView = LocalAddressSpaceView;
  86 };
  87 typedef SizeClassAllocator64<AP64> PrimaryAllocator;
  88 #endif
  89 typedef CombinedAllocator<PrimaryAllocator> Allocator;
  90 typedef Allocator::AllocatorCache AllocatorCache;
  91 Allocator *allocator();
  92 #endif
  93
  94 struct ThreadSignalContext;
  95
  96 struct JmpBuf {
  97   uptr sp;
  98   int int_signal_send;
  99   bool in_blocking_func;
 100   uptr in_signal_handler;
 101   uptr *shadow_stack_pos;
 102 };
 103
 104 // A Processor represents a physical thread, or a P for Go.
 105 // It is used to store internal resources like allocate cache, and does not
 106 // participate in race-detection logic (invisible to end user).
 107 // In C++ it is tied to an OS thread just like ThreadState, however ideally
 108 // it should be tied to a CPU (this way we will have fewer allocator caches).
 109 // In Go it is tied to a P, so there are significantly fewer Processor's than
 110 // ThreadState's (which are tied to Gs).
 111 // A ThreadState must be wired with a Processor to handle events.
 112 struct Processor {
 113   ThreadState *thr; // currently wired thread, or nullptr
 114 #if !SANITIZER_GO
 115   AllocatorCache alloc_cache;
 116   InternalAllocatorCache internal_alloc_cache;
 117 #endif
 118   DenseSlabAllocCache block_cache;
 119   DenseSlabAllocCache sync_cache;
 120   DDPhysicalThread *dd_pt;
 121 };
 122
 123 #if !SANITIZER_GO
 124 // ScopedGlobalProcessor temporary setups a global processor for the current
 125 // thread, if it does not have one. Intended for interceptors that can run
 126 // at the very thread end, when we already destroyed the thread processor.
 127 struct ScopedGlobalProcessor {
 128   ScopedGlobalProcessor();
 129   ~ScopedGlobalProcessor();
 130 };
 131 #endif
 132
 133 struct TidEpoch {
 134   Tid tid;
 135   Epoch epoch;
 136 };
 137
 138 struct TidSlot {
 139   Mutex mtx;
 140   Sid sid;
 141   atomic_uint32_t raw_epoch;
 142   ThreadState *thr;
 143   Vector<TidEpoch> journal;
 144   INode node;
 145
 146   Epoch epoch() const {
 147     return static_cast<Epoch>(atomic_load(&raw_epoch, memory_order_relaxed));
 148   }
 149
 150   void SetEpoch(Epoch v) {
 151     atomic_store(&raw_epoch, static_cast<u32>(v), memory_order_relaxed);
 152   }
 153
 154   TidSlot();
 155 } ALIGNED(SANITIZER_CACHE_LINE_SIZE);
 156
 157 // This struct is stored in TLS.
 158 struct ThreadState {
 159   FastState fast_state;
 160   int ignore_sync;
 161 #if !SANITIZER_GO
 162   int ignore_interceptors;
 163 #endif
 164   uptr *shadow_stack_pos;
 165
 166   // Current position in tctx->trace.Back()->events (Event*).
 167   atomic_uintptr_t trace_pos;
 168   // PC of the last memory access, used to compute PC deltas in the trace.
 169   uptr trace_prev_pc;
 170
 171   // Technically `current` should be a separate THREADLOCAL variable;
 172   // but it is placed here in order to share cache line with previous fields.
 173   ThreadState* current;
 174
 175   atomic_sint32_t pending_signals;
 176
 177   VectorClock clock;
 178
 179   // This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read.
 180   // We do not distinguish beteween ignoring reads and writes
 181   // for better performance.
 182   int ignore_reads_and_writes;
 183   int suppress_reports;
 184   // Go does not support ignores.
 185 #if !SANITIZER_GO
 186   IgnoreSet mop_ignore_set;
 187   IgnoreSet sync_ignore_set;
 188 #endif
 189   uptr *shadow_stack;
 190   uptr *shadow_stack_end;
 191 #if !SANITIZER_GO
 192   Vector<JmpBuf> jmp_bufs;
 193   int in_symbolizer;
 194   bool in_ignored_lib;
 195   bool is_inited;
 196 #endif
 197   MutexSet mset;
 198   bool is_dead;
 199   const Tid tid;
 200   uptr stk_addr;
 201   uptr stk_size;
 202   uptr tls_addr;
 203   uptr tls_size;
 204   ThreadContext *tctx;
 205
 206   DDLogicalThread *dd_lt;
 207
 208   TidSlot *slot;
 209   uptr slot_epoch;
 210   bool slot_locked;
 211
 212   // Current wired Processor, or nullptr. Required to handle any events.
 213   Processor *proc1;
 214 #if !SANITIZER_GO
 215   Processor *proc() { return proc1; }
 216 #else
 217   Processor *proc();
 218 #endif
 219
 220   atomic_uintptr_t in_signal_handler;
 221   ThreadSignalContext *signal_ctx;
 222
 223 #if !SANITIZER_GO
 224   StackID last_sleep_stack_id;
 225   VectorClock last_sleep_clock;
 226 #endif
 227
 228   // Set in regions of runtime that must be signal-safe and fork-safe.
 229   // If set, malloc must not be called.
 230   int nomalloc;
 231
 232   const ReportDesc *current_report;
 233
 234   explicit ThreadState(Tid tid);
 235 } ALIGNED(SANITIZER_CACHE_LINE_SIZE);
 236
 237 #if !SANITIZER_GO
 238 #if SANITIZER_MAC || SANITIZER_ANDROID
 239 ThreadState *cur_thread();
 240 void set_cur_thread(ThreadState *thr);
 241 void cur_thread_finalize();
 242 inline ThreadState *cur_thread_init() { return cur_thread(); }
 243 #  else
 244 __attribute__((tls_model("initial-exec")))
 245 extern THREADLOCAL char cur_thread_placeholder[];
 246 inline ThreadState *cur_thread() {
 247   return reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current;
 248 }
 249 inline ThreadState *cur_thread_init() {
 250   ThreadState *thr = reinterpret_cast<ThreadState *>(cur_thread_placeholder);
 251   if (UNLIKELY(!thr->current))
 252     thr->current = thr;
 253   return thr->current;
 254 }
 255 inline void set_cur_thread(ThreadState *thr) {
 256   reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current = thr;
 257 }
 258 inline void cur_thread_finalize() { }
 259 #  endif  // SANITIZER_MAC || SANITIZER_ANDROID
 260 #endif  // SANITIZER_GO
 261
 262 class ThreadContext final : public ThreadContextBase {
 263  public:
 264   explicit ThreadContext(Tid tid);
 265   ~ThreadContext();
 266   ThreadState *thr;
 267   StackID creation_stack_id;
 268   VectorClock *sync;
 269   uptr sync_epoch;
 270   Trace trace;
 271
 272   // Override superclass callbacks.
 273   void OnDead() override;
 274   void OnJoined(void *arg) override;
 275   void OnFinished() override;
 276   void OnStarted(void *arg) override;
 277   void OnCreated(void *arg) override;
 278   void OnReset() override;
 279   void OnDetached(void *arg) override;
 280 };
 281
 282 struct RacyStacks {
 283   MD5Hash hash[2];
 284   bool operator==(const RacyStacks &other) const;
 285 };
 286
 287 struct RacyAddress {
 288   uptr addr_min;
 289   uptr addr_max;
 290 };
 291
 292 struct FiredSuppression {
 293   ReportType type;
 294   uptr pc_or_addr;
 295   Suppression *supp;
 296 };
 297
 298 struct Context {
 299   Context();
 300
 301   bool initialized;
 302 #if !SANITIZER_GO
 303   bool after_multithreaded_fork;
 304 #endif
 305
 306   MetaMap metamap;
 307
 308   Mutex report_mtx;
 309   int nreported;
 310   atomic_uint64_t last_symbolize_time_ns;
 311
 312   void *background_thread;
 313   atomic_uint32_t stop_background_thread;
 314
 315   ThreadRegistry thread_registry;
 316
 317   Mutex racy_mtx;
 318   Vector<RacyStacks> racy_stacks;
 319   Vector<RacyAddress> racy_addresses;
 320   // Number of fired suppressions may be large enough.
 321   Mutex fired_suppressions_mtx;
 322   InternalMmapVector<FiredSuppression> fired_suppressions;
 323   DDetector *dd;
 324
 325   Flags flags;
 326   fd_t memprof_fd;
 327
 328   // The last slot index (kFreeSid) is used to denote freed memory.
 329   TidSlot slots[kThreadSlotCount - 1];
 330
 331   // Protects global_epoch, slot_queue, trace_part_recycle.
 332   Mutex slot_mtx;
 333   uptr global_epoch;  // guarded by slot_mtx and by all slot mutexes
 334   bool resetting;     // global reset is in progress
 335   IList<TidSlot, &TidSlot::node> slot_queue SANITIZER_GUARDED_BY(slot_mtx);
 336   IList<TraceHeader, &TraceHeader::global, TracePart> trace_part_recycle
 337       SANITIZER_GUARDED_BY(slot_mtx);
 338   uptr trace_part_total_allocated SANITIZER_GUARDED_BY(slot_mtx);
 339   uptr trace_part_recycle_finished SANITIZER_GUARDED_BY(slot_mtx);
 340   uptr trace_part_finished_excess SANITIZER_GUARDED_BY(slot_mtx);
 341 };
 342
 343 extern Context *ctx;  // The one and the only global runtime context.
 344
 345 ALWAYS_INLINE Flags *flags() {
 346   return &ctx->flags;
 347 }
 348
 349 struct ScopedIgnoreInterceptors {
 350   ScopedIgnoreInterceptors() {
 351 #if !SANITIZER_GO
 352     cur_thread()->ignore_interceptors++;
 353 #endif
 354   }
 355
 356   ~ScopedIgnoreInterceptors() {
 357 #if !SANITIZER_GO
 358     cur_thread()->ignore_interceptors--;
 359 #endif
 360   }
 361 };
 362
 363 const char *GetObjectTypeFromTag(uptr tag);
 364 const char *GetReportHeaderFromTag(uptr tag);
 365 uptr TagFromShadowStackFrame(uptr pc);
 366
 367 class ScopedReportBase {
 368  public:
 369   void AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, Tid tid,
 370                        StackTrace stack, const MutexSet *mset);
 371   void AddStack(StackTrace stack, bool suppressable = false);
 372   void AddThread(const ThreadContext *tctx, bool suppressable = false);
 373   void AddThread(Tid tid, bool suppressable = false);
 374   void AddUniqueTid(Tid unique_tid);
 375   int AddMutex(uptr addr, StackID creation_stack_id);
 376   void AddLocation(uptr addr, uptr size);
 377   void AddSleep(StackID stack_id);
 378   void SetCount(int count);
 379   void SetSigNum(int sig);
 380
 381   const ReportDesc *GetReport() const;
 382
 383  protected:
 384   ScopedReportBase(ReportType typ, uptr tag);
 385   ~ScopedReportBase();
 386
 387  private:
 388   ReportDesc *rep_;
 389   // Symbolizer makes lots of intercepted calls. If we try to process them,
 390   // at best it will cause deadlocks on internal mutexes.
 391   ScopedIgnoreInterceptors ignore_interceptors_;
 392
 393   ScopedReportBase(const ScopedReportBase &) = delete;
 394   void operator=(const ScopedReportBase &) = delete;
 395 };
 396
 397 class ScopedReport : public ScopedReportBase {
 398  public:
 399   explicit ScopedReport(ReportType typ, uptr tag = kExternalTagNone);
 400   ~ScopedReport();
 401
 402  private:
 403   ScopedErrorReportLock lock_;
 404 };
 405
 406 bool ShouldReport(ThreadState *thr, ReportType typ);
 407 ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack);
 408
 409 // The stack could look like:
 410 //   <start> | <main> | <foo> | tag | <bar>
 411 // This will extract the tag and keep:
 412 //   <start> | <main> | <foo> | <bar>
 413 template<typename StackTraceTy>
 414 void ExtractTagFromStack(StackTraceTy *stack, uptr *tag = nullptr) {
 415   if (stack->size < 2) return;
 416   uptr possible_tag_pc = stack->trace[stack->size - 2];
 417   uptr possible_tag = TagFromShadowStackFrame(possible_tag_pc);
 418   if (possible_tag == kExternalTagNone) return;
 419   stack->trace_buffer[stack->size - 2] = stack->trace_buffer[stack->size - 1];
 420   stack->size -= 1;
 421   if (tag) *tag = possible_tag;
 422 }
 423
 424 template<typename StackTraceTy>
 425 void ObtainCurrentStack(ThreadState *thr, uptr toppc, StackTraceTy *stack,
 426                         uptr *tag = nullptr) {
 427   uptr size = thr->shadow_stack_pos - thr->shadow_stack;
 428   uptr start = 0;
 429   if (size + !!toppc > kStackTraceMax) {
 430     start = size + !!toppc - kStackTraceMax;
 431     size = kStackTraceMax - !!toppc;
 432   }
 433   stack->Init(&thr->shadow_stack[start], size, toppc);
 434   ExtractTagFromStack(stack, tag);
 435 }
 436
 437 #define GET_STACK_TRACE_FATAL(thr, pc) \
 438   VarSizeStackTrace stack; \
 439   ObtainCurrentStack(thr, pc, &stack); \
 440   stack.ReverseOrder();
 441
 442 void MapShadow(uptr addr, uptr size);
 443 void MapThreadTrace(uptr addr, uptr size, const char *name);
 444 void DontNeedShadowFor(uptr addr, uptr size);
 445 void UnmapShadow(ThreadState *thr, uptr addr, uptr size);
 446 void InitializeShadowMemory();
 447 void InitializeInterceptors();
 448 void InitializeLibIgnore();
 449 void InitializeDynamicAnnotations();
 450
 451 void ForkBefore(ThreadState *thr, uptr pc);
 452 void ForkParentAfter(ThreadState *thr, uptr pc);
 453 void ForkChildAfter(ThreadState *thr, uptr pc, bool start_thread);
 454
 455 void ReportRace(ThreadState *thr, RawShadow *shadow_mem, Shadow cur, Shadow old,
 456                 AccessType typ);
 457 bool OutputReport(ThreadState *thr, const ScopedReport &srep);
 458 bool IsFiredSuppression(Context *ctx, ReportType type, StackTrace trace);
 459 bool IsExpectedReport(uptr addr, uptr size);
 460
 461 #if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 1
 462 # define DPrintf Printf
 463 #else
 464 # define DPrintf(...)
 465 #endif
 466
 467 #if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 2
 468 # define DPrintf2 Printf
 469 #else
 470 # define DPrintf2(...)
 471 #endif
 472
 473 StackID CurrentStackId(ThreadState *thr, uptr pc);
 474 ReportStack *SymbolizeStackId(StackID stack_id);
 475 void PrintCurrentStack(ThreadState *thr, uptr pc);
 476 void PrintCurrentStackSlow(uptr pc);  // uses libunwind
 477 MBlock *JavaHeapBlock(uptr addr, uptr *start);
 478
 479 void Initialize(ThreadState *thr);
 480 void MaybeSpawnBackgroundThread();
 481 int Finalize(ThreadState *thr);
 482
 483 void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write);
 484 void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write);
 485
 486 void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
 487                   AccessType typ);
 488 void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
 489                            AccessType typ);
 490 // This creates 2 non-inlined specialized versions of MemoryAccessRange.
 491 template <bool is_read>
 492 void MemoryAccessRangeT(ThreadState *thr, uptr pc, uptr addr, uptr size);
 493
 494 ALWAYS_INLINE
 495 void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
 496                        bool is_write) {
 497   if (size == 0)
 498     return;
 499   if (is_write)
 500     MemoryAccessRangeT<false>(thr, pc, addr, size);
 501   else
 502     MemoryAccessRangeT<true>(thr, pc, addr, size);
 503 }
 504
 505 void ShadowSet(RawShadow *p, RawShadow *end, RawShadow v);
 506 void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size);
 507 void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size);
 508 void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size);
 509 void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr,
 510                                          uptr size);
 511
 512 void ThreadIgnoreBegin(ThreadState *thr, uptr pc);
 513 void ThreadIgnoreEnd(ThreadState *thr);
 514 void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc);
 515 void ThreadIgnoreSyncEnd(ThreadState *thr);
 516
 517 Tid ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached);
 518 void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id,
 519                  ThreadType thread_type);
 520 void ThreadFinish(ThreadState *thr);
 521 Tid ThreadConsumeTid(ThreadState *thr, uptr pc, uptr uid);
 522 void ThreadJoin(ThreadState *thr, uptr pc, Tid tid);
 523 void ThreadDetach(ThreadState *thr, uptr pc, Tid tid);
 524 void ThreadFinalize(ThreadState *thr);
 525 void ThreadSetName(ThreadState *thr, const char *name);
 526 int ThreadCount(ThreadState *thr);
 527 void ProcessPendingSignalsImpl(ThreadState *thr);
 528 void ThreadNotJoined(ThreadState *thr, uptr pc, Tid tid, uptr uid);
 529
 530 Processor *ProcCreate();
 531 void ProcDestroy(Processor *proc);
 532 void ProcWire(Processor *proc, ThreadState *thr);
 533 void ProcUnwire(Processor *proc, ThreadState *thr);
 534
 535 // Note: the parameter is called flagz, because flags is already taken
 536 // by the global function that returns flags.
 537 void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
 538 void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
 539 void MutexPreLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
 540 void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0,
 541     int rec = 1);
 542 int  MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
 543 void MutexPreReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
 544 void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
 545 void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr);
 546 void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr);
 547 void MutexRepair(ThreadState *thr, uptr pc, uptr addr);  // call on EOWNERDEAD
 548 void MutexInvalidAccess(ThreadState *thr, uptr pc, uptr addr);
 549
 550 void Acquire(ThreadState *thr, uptr pc, uptr addr);
 551 // AcquireGlobal synchronizes the current thread with all other threads.
 552 // In terms of happens-before relation, it draws a HB edge from all threads
 553 // (where they happen to execute right now) to the current thread. We use it to
 554 // handle Go finalizers. Namely, finalizer goroutine executes AcquireGlobal
 555 // right before executing finalizers. This provides a coarse, but simple
 556 // approximation of the actual required synchronization.
 557 void AcquireGlobal(ThreadState *thr);
 558 void Release(ThreadState *thr, uptr pc, uptr addr);
 559 void ReleaseStoreAcquire(ThreadState *thr, uptr pc, uptr addr);
 560 void ReleaseStore(ThreadState *thr, uptr pc, uptr addr);
 561 void AfterSleep(ThreadState *thr, uptr pc);
 562 void IncrementEpoch(ThreadState *thr);
 563
 564 #if !SANITIZER_GO
 565 uptr ALWAYS_INLINE HeapEnd() {
 566   return HeapMemEnd() + PrimaryAllocator::AdditionalSize();
 567 }
 568 #endif
 569
 570 void SlotAttachAndLock(ThreadState *thr) SANITIZER_ACQUIRE(thr->slot->mtx);
 571 void SlotDetach(ThreadState *thr);
 572 void SlotLock(ThreadState *thr) SANITIZER_ACQUIRE(thr->slot->mtx);
 573 void SlotUnlock(ThreadState *thr) SANITIZER_RELEASE(thr->slot->mtx);
 574 void DoReset(ThreadState *thr, uptr epoch);
 575 void FlushShadowMemory();
 576
 577 ThreadState *FiberCreate(ThreadState *thr, uptr pc, unsigned flags);
 578 void FiberDestroy(ThreadState *thr, uptr pc, ThreadState *fiber);
 579 void FiberSwitch(ThreadState *thr, uptr pc, ThreadState *fiber, unsigned flags);
 580
 581 // These need to match __tsan_switch_to_fiber_* flags defined in
 582 // tsan_interface.h. See documentation there as well.
 583 enum FiberSwitchFlags {
 584   FiberSwitchFlagNoSync = 1 << 0, // __tsan_switch_to_fiber_no_sync
 585 };
 586
 587 class SlotLocker {
 588  public:
 589   ALWAYS_INLINE
 590   SlotLocker(ThreadState *thr, bool recursive = false)
 591       : thr_(thr), locked_(recursive ? thr->slot_locked : false) {
 592     if (!locked_)
 593       SlotLock(thr_);
 594   }
 595
 596   ALWAYS_INLINE
 597   ~SlotLocker() {
 598     if (!locked_)
 599       SlotUnlock(thr_);
 600   }
 601
 602  private:
 603   ThreadState *thr_;
 604   bool locked_;
 605 };
 606
 607 class SlotUnlocker {
 608  public:
 609   SlotUnlocker(ThreadState *thr) : thr_(thr), locked_(thr->slot_locked) {
 610     if (locked_)
 611       SlotUnlock(thr_);
 612   }
 613
 614   ~SlotUnlocker() {
 615     if (locked_)
 616       SlotLock(thr_);
 617   }
 618
 619  private:
 620   ThreadState *thr_;
 621   bool locked_;
 622 };
 623
 624 ALWAYS_INLINE void ProcessPendingSignals(ThreadState *thr) {
 625   if (UNLIKELY(atomic_load_relaxed(&thr->pending_signals)))
 626     ProcessPendingSignalsImpl(thr);
 627 }
 628
 629 extern bool is_initialized;
 630
 631 ALWAYS_INLINE
 632 void LazyInitialize(ThreadState *thr) {
 633   // If we can use .preinit_array, assume that __tsan_init
 634   // called from .preinit_array initializes runtime before
 635   // any instrumented code.
 636 #if !SANITIZER_CAN_USE_PREINIT_ARRAY
 637   if (UNLIKELY(!is_initialized))
 638     Initialize(thr);
 639 #endif
 640 }
 641
 642 void TraceResetForTesting();
 643 void TraceSwitchPart(ThreadState *thr);
 644 void TraceSwitchPartImpl(ThreadState *thr);
 645 bool RestoreStack(EventType type, Sid sid, Epoch epoch, uptr addr, uptr size,
 646                   AccessType typ, Tid *ptid, VarSizeStackTrace *pstk,
 647                   MutexSet *pmset, uptr *ptag);
 648
 649 template <typename EventT>
 650 ALWAYS_INLINE WARN_UNUSED_RESULT bool TraceAcquire(ThreadState *thr,
 651                                                    EventT **ev) {
 652   // TraceSwitchPart accesses shadow_stack, but it's called infrequently,
 653   // so we check it here proactively.
 654   DCHECK(thr->shadow_stack);
 655   Event *pos = reinterpret_cast<Event *>(atomic_load_relaxed(&thr->trace_pos));
 656 #if SANITIZER_DEBUG
 657   // TraceSwitch acquires these mutexes,
 658   // so we lock them here to detect deadlocks more reliably.
 659   { Lock lock(&ctx->slot_mtx); }
 660   { Lock lock(&thr->tctx->trace.mtx); }
 661   TracePart *current = thr->tctx->trace.parts.Back();
 662   if (current) {
 663     DCHECK_GE(pos, &current->events[0]);
 664     DCHECK_LE(pos, &current->events[TracePart::kSize]);
 665   } else {
 666     DCHECK_EQ(pos, nullptr);
 667   }
 668 #endif
 669   // TracePart is allocated with mmap and is at least 4K aligned.
 670   // So the following check is a faster way to check for part end.
 671   // It may have false positives in the middle of the trace,
 672   // they are filtered out in TraceSwitch.
 673   if (UNLIKELY(((uptr)(pos + 1) & TracePart::kAlignment) == 0))
 674     return false;
 675   *ev = reinterpret_cast<EventT *>(pos);
 676   return true;
 677 }
 678
 679 template <typename EventT>
 680 ALWAYS_INLINE void TraceRelease(ThreadState *thr, EventT *evp) {
 681   DCHECK_LE(evp + 1, &thr->tctx->trace.parts.Back()->events[TracePart::kSize]);
 682   atomic_store_relaxed(&thr->trace_pos, (uptr)(evp + 1));
 683 }
 684
 685 template <typename EventT>
 686 void TraceEvent(ThreadState *thr, EventT ev) {
 687   EventT *evp;
 688   if (!TraceAcquire(thr, &evp)) {
 689     TraceSwitchPart(thr);
 690     UNUSED bool res = TraceAcquire(thr, &evp);
 691     DCHECK(res);
 692   }
 693   *evp = ev;
 694   TraceRelease(thr, evp);
 695 }
 696
 697 ALWAYS_INLINE WARN_UNUSED_RESULT bool TryTraceFunc(ThreadState *thr,
 698                                                    uptr pc = 0) {
 699   if (!kCollectHistory)
 700     return true;
 701   EventFunc *ev;
 702   if (UNLIKELY(!TraceAcquire(thr, &ev)))
 703     return false;
 704   ev->is_access = 0;
 705   ev->is_func = 1;
 706   ev->pc = pc;
 707   TraceRelease(thr, ev);
 708   return true;
 709 }
 710
 711 WARN_UNUSED_RESULT
 712 bool TryTraceMemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
 713                           AccessType typ);
 714 WARN_UNUSED_RESULT
 715 bool TryTraceMemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
 716                                AccessType typ);
 717 void TraceMemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
 718                             AccessType typ);
 719 void TraceFunc(ThreadState *thr, uptr pc = 0);
 720 void TraceMutexLock(ThreadState *thr, EventType type, uptr pc, uptr addr,
 721                     StackID stk);
 722 void TraceMutexUnlock(ThreadState *thr, uptr addr);
 723 void TraceTime(ThreadState *thr);
 724
 725 void TraceRestartFuncExit(ThreadState *thr);
 726 void TraceRestartFuncEntry(ThreadState *thr, uptr pc);
 727
 728 void GrowShadowStack(ThreadState *thr);
 729
 730 ALWAYS_INLINE
 731 void FuncEntry(ThreadState *thr, uptr pc) {
 732   DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.sid(), (void *)pc);
 733   if (UNLIKELY(!TryTraceFunc(thr, pc)))
 734     return TraceRestartFuncEntry(thr, pc);
 735   DCHECK_GE(thr->shadow_stack_pos, thr->shadow_stack);
 736 #if !SANITIZER_GO
 737   DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
 738 #else
 739   if (thr->shadow_stack_pos == thr->shadow_stack_end)
 740     GrowShadowStack(thr);
 741 #endif
 742   thr->shadow_stack_pos[0] = pc;
 743   thr->shadow_stack_pos++;
 744 }
 745
 746 ALWAYS_INLINE
 747 void FuncExit(ThreadState *thr) {
 748   DPrintf2("#%d: FuncExit\n", (int)thr->fast_state.sid());
 749   if (UNLIKELY(!TryTraceFunc(thr, 0)))
 750     return TraceRestartFuncExit(thr);
 751   DCHECK_GT(thr->shadow_stack_pos, thr->shadow_stack);
 752 #if !SANITIZER_GO
 753   DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
 754 #endif
 755   thr->shadow_stack_pos--;
 756 }
 757
 758 #if !SANITIZER_GO
 759 extern void (*on_initialize)(void);
 760 extern int (*on_finalize)(int);
 761 #endif
 762 }  // namespace __tsan
 763
 764 #endif  // TSAN_RTL_H