compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.cpp

   1 //===-- sanitizer_stackdepot.cpp ------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file is shared between AddressSanitizer and ThreadSanitizer
  10 // run-time libraries.
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "sanitizer_stackdepot.h"
  14
  15 #include "sanitizer_atomic.h"
  16 #include "sanitizer_common.h"
  17 #include "sanitizer_hash.h"
  18 #include "sanitizer_mutex.h"
  19 #include "sanitizer_stack_store.h"
  20 #include "sanitizer_stackdepotbase.h"
  21
  22 namespace __sanitizer {
  23
  24 struct StackDepotNode {
  25   using hash_type = u64;
  26   hash_type stack_hash;
  27   u32 link;
  28   StackStore::Id store_id;
  29
  30   static const u32 kTabSizeLog = SANITIZER_ANDROID ? 16 : 20;
  31
  32   typedef StackTrace args_type;
  33   bool eq(hash_type hash, const args_type &args) const {
  34     return hash == stack_hash;
  35   }
  36   static uptr allocated();
  37   static hash_type hash(const args_type &args) {
  38     MurMur2Hash64Builder H(args.size * sizeof(uptr));
  39     for (uptr i = 0; i < args.size; i++) H.add(args.trace[i]);
  40     H.add(args.tag);
  41     return H.get();
  42   }
  43   static bool is_valid(const args_type &args) {
  44     return args.size > 0 && args.trace;
  45   }
  46   void store(u32 id, const args_type &args, hash_type hash);
  47   args_type load(u32 id) const;
  48   static StackDepotHandle get_handle(u32 id);
  49
  50   typedef StackDepotHandle handle_type;
  51 };
  52
  53 static StackStore stackStore;
  54
  55 // FIXME(dvyukov): this single reserved bit is used in TSan.
  56 typedef StackDepotBase<StackDepotNode, 1, StackDepotNode::kTabSizeLog>
  57     StackDepot;
  58 static StackDepot theDepot;
  59 // Keep mutable data out of frequently access nodes to improve caching
  60 // efficiency.
  61 static TwoLevelMap<atomic_uint32_t, StackDepot::kNodesSize1,
  62                    StackDepot::kNodesSize2>
  63     useCounts;
  64
  65 int StackDepotHandle::use_count() const {
  66   return atomic_load_relaxed(&useCounts[id_]);
  67 }
  68
  69 void StackDepotHandle::inc_use_count_unsafe() {
  70   atomic_fetch_add(&useCounts[id_], 1, memory_order_relaxed);
  71 }
  72
  73 uptr StackDepotNode::allocated() {
  74   return stackStore.Allocated() + useCounts.MemoryUsage();
  75 }
  76
  77 static void CompressStackStore() {
  78   u64 start = Verbosity() >= 1 ? MonotonicNanoTime() : 0;
  79   uptr diff = stackStore.Pack(static_cast<StackStore::Compression>(
  80       Abs(common_flags()->compress_stack_depot)));
  81   if (!diff)
  82     return;
  83   if (Verbosity() >= 1) {
  84     u64 finish = MonotonicNanoTime();
  85     uptr total_before = theDepot.GetStats().allocated + diff;
  86     VPrintf(1, "%s: StackDepot released %zu KiB out of %zu KiB in %llu ms\n",
  87             SanitizerToolName, diff >> 10, total_before >> 10,
  88             (finish - start) / 1000000);
  89   }
  90 }
  91
  92 namespace {
  93
  94 class CompressThread {
  95  public:
  96   constexpr CompressThread() = default;
  97   void NewWorkNotify();
  98   void Stop();
  99   void LockAndStop() SANITIZER_NO_THREAD_SAFETY_ANALYSIS;
 100   void Unlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS;
 101
 102  private:
 103   enum class State {
 104     NotStarted = 0,
 105     Started,
 106     Failed,
 107     Stopped,
 108   };
 109
 110   void Run();
 111
 112   bool WaitForWork() {
 113     semaphore_.Wait();
 114     return atomic_load(&run_, memory_order_acquire);
 115   }
 116
 117   Semaphore semaphore_ = {};
 118   StaticSpinMutex mutex_ = {};
 119   State state_ SANITIZER_GUARDED_BY(mutex_) = State::NotStarted;
 120   void *thread_ SANITIZER_GUARDED_BY(mutex_) = nullptr;
 121   atomic_uint8_t run_ = {};
 122 };
 123
 124 static CompressThread compress_thread;
 125
 126 void CompressThread::NewWorkNotify() {
 127   int compress = common_flags()->compress_stack_depot;
 128   if (!compress)
 129     return;
 130   if (compress > 0 /* for testing or debugging */) {
 131     SpinMutexLock l(&mutex_);
 132     if (state_ == State::NotStarted) {
 133       atomic_store(&run_, 1, memory_order_release);
 134       CHECK_EQ(nullptr, thread_);
 135       thread_ = internal_start_thread(
 136           [](void *arg) -> void * {
 137             reinterpret_cast<CompressThread *>(arg)->Run();
 138             return nullptr;
 139           },
 140           this);
 141       state_ = thread_ ? State::Started : State::Failed;
 142     }
 143     if (state_ == State::Started) {
 144       semaphore_.Post();
 145       return;
 146     }
 147   }
 148   CompressStackStore();
 149 }
 150
 151 void CompressThread::Run() {
 152   VPrintf(1, "%s: StackDepot compression thread started\n", SanitizerToolName);
 153   while (WaitForWork()) CompressStackStore();
 154   VPrintf(1, "%s: StackDepot compression thread stopped\n", SanitizerToolName);
 155 }
 156
 157 void CompressThread::Stop() {
 158   void *t = nullptr;
 159   {
 160     SpinMutexLock l(&mutex_);
 161     if (state_ != State::Started)
 162       return;
 163     state_ = State::Stopped;
 164     CHECK_NE(nullptr, thread_);
 165     t = thread_;
 166     thread_ = nullptr;
 167   }
 168   atomic_store(&run_, 0, memory_order_release);
 169   semaphore_.Post();
 170   internal_join_thread(t);
 171 }
 172
 173 void CompressThread::LockAndStop() {
 174   mutex_.Lock();
 175   if (state_ != State::Started)
 176     return;
 177   CHECK_NE(nullptr, thread_);
 178
 179   atomic_store(&run_, 0, memory_order_release);
 180   semaphore_.Post();
 181   internal_join_thread(thread_);
 182   // Allow to restart after Unlock() if needed.
 183   state_ = State::NotStarted;
 184   thread_ = nullptr;
 185 }
 186
 187 void CompressThread::Unlock() { mutex_.Unlock(); }
 188
 189 }  // namespace
 190
 191 void StackDepotNode::store(u32 id, const args_type &args, hash_type hash) {
 192   stack_hash = hash;
 193   uptr pack = 0;
 194   store_id = stackStore.Store(args, &pack);
 195   if (LIKELY(!pack))
 196     return;
 197   compress_thread.NewWorkNotify();
 198 }
 199
 200 StackDepotNode::args_type StackDepotNode::load(u32 id) const {
 201   if (!store_id)
 202     return {};
 203   return stackStore.Load(store_id);
 204 }
 205
 206 StackDepotStats StackDepotGetStats() { return theDepot.GetStats(); }
 207
 208 u32 StackDepotPut(StackTrace stack) { return theDepot.Put(stack); }
 209
 210 StackDepotHandle StackDepotPut_WithHandle(StackTrace stack) {
 211   return StackDepotNode::get_handle(theDepot.Put(stack));
 212 }
 213
 214 StackTrace StackDepotGet(u32 id) {
 215   return theDepot.Get(id);
 216 }
 217
 218 void StackDepotLockBeforeFork() {
 219   theDepot.LockBeforeFork();
 220   compress_thread.LockAndStop();
 221   stackStore.LockAll();
 222 }
 223
 224 void StackDepotUnlockAfterFork(bool fork_child) {
 225   stackStore.UnlockAll();
 226   compress_thread.Unlock();
 227   theDepot.UnlockAfterFork(fork_child);
 228 }
 229
 230 void StackDepotPrintAll() {
 231 #if !SANITIZER_GO
 232   theDepot.PrintAll();
 233 #endif
 234 }
 235
 236 void StackDepotStopBackgroundThread() { compress_thread.Stop(); }
 237
 238 StackDepotHandle StackDepotNode::get_handle(u32 id) {
 239   return StackDepotHandle(&theDepot.nodes[id], id);
 240 }
 241
 242 void StackDepotTestOnlyUnmap() {
 243   theDepot.TestOnlyUnmap();
 244   stackStore.TestOnlyUnmap();
 245 }
 246
 247 } // namespace __sanitizer