compiler-rt/lib/scudo/standalone/tsd_exclusive.h

   1 //===-- tsd_exclusive.h -----------------------------------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #ifndef SCUDO_TSD_EXCLUSIVE_H_
  10 #define SCUDO_TSD_EXCLUSIVE_H_
  11
  12 #include "tsd.h"
  13
  14 #include "string_utils.h"
  15
  16 namespace scudo {
  17
  18 struct ThreadState {
  19   bool DisableMemInit : 1;
  20   enum : unsigned {
  21     NotInitialized = 0,
  22     Initialized,
  23     TornDown,
  24   } InitState : 2;
  25 };
  26
  27 template <class Allocator> void teardownThread(void *Ptr);
  28
  29 template <class Allocator> struct TSDRegistryExT {
  30   void init(Allocator *Instance) REQUIRES(Mutex) {
  31     DCHECK(!Initialized);
  32     Instance->init();
  33     CHECK_EQ(pthread_key_create(&PThreadKey, teardownThread<Allocator>), 0);
  34     FallbackTSD.init(Instance);
  35     Initialized = true;
  36   }
  37
  38   void initOnceMaybe(Allocator *Instance) EXCLUDES(Mutex) {
  39     ScopedLock L(Mutex);
  40     if (LIKELY(Initialized))
  41       return;
  42     init(Instance); // Sets Initialized.
  43   }
  44
  45   void unmapTestOnly(Allocator *Instance) EXCLUDES(Mutex) {
  46     DCHECK(Instance);
  47     if (reinterpret_cast<Allocator *>(pthread_getspecific(PThreadKey))) {
  48       DCHECK_EQ(reinterpret_cast<Allocator *>(pthread_getspecific(PThreadKey)),
  49                 Instance);
  50       ThreadTSD.commitBack(Instance);
  51       ThreadTSD = {};
  52     }
  53     CHECK_EQ(pthread_key_delete(PThreadKey), 0);
  54     PThreadKey = {};
  55     FallbackTSD.commitBack(Instance);
  56     FallbackTSD = {};
  57     State = {};
  58     ScopedLock L(Mutex);
  59     Initialized = false;
  60   }
  61
  62   void drainCaches(Allocator *Instance) {
  63     // We don't have a way to iterate all thread local `ThreadTSD`s. Simply
  64     // drain the `ThreadTSD` of current thread and `FallbackTSD`.
  65     Instance->drainCache(&ThreadTSD);
  66     FallbackTSD.lock();
  67     Instance->drainCache(&FallbackTSD);
  68     FallbackTSD.unlock();
  69   }
  70
  71   ALWAYS_INLINE void initThreadMaybe(Allocator *Instance, bool MinimalInit) {
  72     if (LIKELY(State.InitState != ThreadState::NotInitialized))
  73       return;
  74     initThread(Instance, MinimalInit);
  75   }
  76
  77   // TODO(chiahungduan): Consider removing the argument `UnlockRequired` by
  78   // embedding the logic into TSD or always locking the TSD. It will enable us
  79   // to properly mark thread annotation here and adding proper runtime
  80   // assertions in the member functions of TSD. For example, assert the lock is
  81   // acquired before calling TSD::commitBack().
  82   ALWAYS_INLINE TSD<Allocator> *
  83   getTSDAndLock(bool *UnlockRequired) NO_THREAD_SAFETY_ANALYSIS {
  84     if (LIKELY(State.InitState == ThreadState::Initialized &&
  85                !atomic_load(&Disabled, memory_order_acquire))) {
  86       *UnlockRequired = false;
  87       return &ThreadTSD;
  88     }
  89     FallbackTSD.lock();
  90     *UnlockRequired = true;
  91     return &FallbackTSD;
  92   }
  93
  94   // To disable the exclusive TSD registry, we effectively lock the fallback TSD
  95   // and force all threads to attempt to use it instead of their local one.
  96   void disable() NO_THREAD_SAFETY_ANALYSIS {
  97     Mutex.lock();
  98     FallbackTSD.lock();
  99     atomic_store(&Disabled, 1U, memory_order_release);
 100   }
 101
 102   void enable() NO_THREAD_SAFETY_ANALYSIS {
 103     atomic_store(&Disabled, 0U, memory_order_release);
 104     FallbackTSD.unlock();
 105     Mutex.unlock();
 106   }
 107
 108   bool setOption(Option O, sptr Value) {
 109     if (O == Option::ThreadDisableMemInit)
 110       State.DisableMemInit = Value;
 111     if (O == Option::MaxTSDsCount)
 112       return false;
 113     return true;
 114   }
 115
 116   bool getDisableMemInit() { return State.DisableMemInit; }
 117
 118   void getStats(ScopedString *Str) {
 119     // We don't have a way to iterate all thread local `ThreadTSD`s. Instead of
 120     // printing only self `ThreadTSD` which may mislead the usage, we just skip
 121     // it.
 122     Str->append("Exclusive TSD don't support iterating each TSD\n");
 123   }
 124
 125 private:
 126   // Using minimal initialization allows for global initialization while keeping
 127   // the thread specific structure untouched. The fallback structure will be
 128   // used instead.
 129   NOINLINE void initThread(Allocator *Instance, bool MinimalInit) {
 130     initOnceMaybe(Instance);
 131     if (UNLIKELY(MinimalInit))
 132       return;
 133     CHECK_EQ(
 134         pthread_setspecific(PThreadKey, reinterpret_cast<void *>(Instance)), 0);
 135     ThreadTSD.init(Instance);
 136     State.InitState = ThreadState::Initialized;
 137     Instance->callPostInitCallback();
 138   }
 139
 140   pthread_key_t PThreadKey = {};
 141   bool Initialized GUARDED_BY(Mutex) = false;
 142   atomic_u8 Disabled = {};
 143   TSD<Allocator> FallbackTSD;
 144   HybridMutex Mutex;
 145   static thread_local ThreadState State;
 146   static thread_local TSD<Allocator> ThreadTSD;
 147
 148   friend void teardownThread<Allocator>(void *Ptr);
 149 };
 150
 151 template <class Allocator>
 152 thread_local TSD<Allocator> TSDRegistryExT<Allocator>::ThreadTSD;
 153 template <class Allocator>
 154 thread_local ThreadState TSDRegistryExT<Allocator>::State;
 155
 156 template <class Allocator>
 157 void teardownThread(void *Ptr) NO_THREAD_SAFETY_ANALYSIS {
 158   typedef TSDRegistryExT<Allocator> TSDRegistryT;
 159   Allocator *Instance = reinterpret_cast<Allocator *>(Ptr);
 160   // The glibc POSIX thread-local-storage deallocation routine calls user
 161   // provided destructors in a loop of PTHREAD_DESTRUCTOR_ITERATIONS.
 162   // We want to be called last since other destructors might call free and the
 163   // like, so we wait until PTHREAD_DESTRUCTOR_ITERATIONS before draining the
 164   // quarantine and swallowing the cache.
 165   if (TSDRegistryT::ThreadTSD.DestructorIterations > 1) {
 166     TSDRegistryT::ThreadTSD.DestructorIterations--;
 167     // If pthread_setspecific fails, we will go ahead with the teardown.
 168     if (LIKELY(pthread_setspecific(Instance->getTSDRegistry()->PThreadKey,
 169                                    Ptr) == 0))
 170       return;
 171   }
 172   TSDRegistryT::ThreadTSD.commitBack(Instance);
 173   TSDRegistryT::State.InitState = ThreadState::TornDown;
 174 }
 175
 176 } // namespace scudo
 177
 178 #endif // SCUDO_TSD_EXCLUSIVE_H_