libcxxabi/src/cxa_guard_impl.h

   1 //===----------------------------------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #ifndef LIBCXXABI_SRC_INCLUDE_CXA_GUARD_IMPL_H
  10 #define LIBCXXABI_SRC_INCLUDE_CXA_GUARD_IMPL_H
  11
  12 /* cxa_guard_impl.h - Implements the C++ runtime support for function local
  13  * static guards.
  14  * The layout of the guard object is the same across ARM and Itanium.
  15  *
  16  * The first "guard byte" (which is checked by the compiler) is set only upon
  17  * the completion of cxa release.
  18  *
  19  * The second "init byte" does the rest of the bookkeeping. It tracks if
  20  * initialization is complete or pending, and if there are waiting threads.
  21  *
  22  * If the guard variable is 64-bits and the platforms supplies a 32-bit thread
  23  * identifier, it is used to detect recursive initialization. The thread ID of
  24  * the thread currently performing initialization is stored in the second word.
  25  *
  26  *  Guard Object Layout:
  27  * ---------------------------------------------------------------------------
  28  * | a+0: guard byte | a+1: init byte | a+2: unused ... | a+4: thread-id ... |
  29  * ---------------------------------------------------------------------------
  30  *
  31  * Note that we don't do what the ABI docs suggest (put a mutex in the guard
  32  * object which we acquire in cxa_guard_acquire and release in
  33  * cxa_guard_release). Instead we use the init byte to imitate that behaviour,
  34  * but without actually holding anything mutex related between aquire and
  35  * release/abort.
  36  *
  37  *  Access Protocol:
  38  *    For each implementation the guard byte is checked and set before accessing
  39  *    the init byte.
  40  *
  41  *  Overall Design:
  42  *    The implementation was designed to allow each implementation to be tested
  43  *    independent of the C++ runtime or platform support.
  44  *
  45  */
  46
  47 #include "__cxxabi_config.h"
  48 #include "include/atomic_support.h" // from libc++
  49 #if defined(__has_include)
  50 #  if __has_include(<sys/syscall.h>)
  51 #    include <sys/syscall.h>
  52 #  endif
  53 #  if __has_include(<unistd.h>)
  54 #    include <unistd.h>
  55 #  endif
  56 #endif
  57
  58 #include <__threading_support>
  59 #include <cstdint>
  60 #include <cstring>
  61 #include <limits.h>
  62 #include <stdlib.h>
  63
  64 #ifndef _LIBCXXABI_HAS_NO_THREADS
  65 #  if defined(__ELF__) && defined(_LIBCXXABI_LINK_PTHREAD_LIB)
  66 #    pragma comment(lib, "pthread")
  67 #  endif
  68 #endif
  69
  70 #if defined(__clang__)
  71 #  pragma clang diagnostic push
  72 #  pragma clang diagnostic ignored "-Wtautological-pointer-compare"
  73 #elif defined(__GNUC__)
  74 #  pragma GCC diagnostic push
  75 #  pragma GCC diagnostic ignored "-Waddress"
  76 #endif
  77
  78 // To make testing possible, this header is included from both cxa_guard.cpp
  79 // and a number of tests.
  80 //
  81 // For this reason we place everything in an anonymous namespace -- even though
  82 // we're in a header. We want the actual implementation and the tests to have
  83 // unique definitions of the types in this header (since the tests may depend
  84 // on function local statics).
  85 //
  86 // To enforce this either `BUILDING_CXA_GUARD` or `TESTING_CXA_GUARD` must be
  87 // defined when including this file. Only `src/cxa_guard.cpp` should define
  88 // the former.
  89 #ifdef BUILDING_CXA_GUARD
  90 #  include "abort_message.h"
  91 #  define ABORT_WITH_MESSAGE(...) ::abort_message(__VA_ARGS__)
  92 #elif defined(TESTING_CXA_GUARD)
  93 #  define ABORT_WITH_MESSAGE(...) ::abort()
  94 #else
  95 #  error "Either BUILDING_CXA_GUARD or TESTING_CXA_GUARD must be defined"
  96 #endif
  97
  98 #if __has_feature(thread_sanitizer)
  99 extern "C" void __tsan_acquire(void*);
 100 extern "C" void __tsan_release(void*);
 101 #else
 102 #  define __tsan_acquire(addr) ((void)0)
 103 #  define __tsan_release(addr) ((void)0)
 104 #endif
 105
 106 namespace __cxxabiv1 {
 107 // Use an anonymous namespace to ensure that the tests and actual implementation
 108 // have unique definitions of these symbols.
 109 namespace {
 110
 111 //===----------------------------------------------------------------------===//
 112 //                          Misc Utilities
 113 //===----------------------------------------------------------------------===//
 114
 115 template <class T, T (*Init)()>
 116 struct LazyValue {
 117   LazyValue() : is_init(false) {}
 118
 119   T& get() {
 120     if (!is_init) {
 121       value = Init();
 122       is_init = true;
 123     }
 124     return value;
 125   }
 126
 127 private:
 128   T value;
 129   bool is_init = false;
 130 };
 131
 132 template <class IntType>
 133 class AtomicInt {
 134 public:
 135   using MemoryOrder = std::__libcpp_atomic_order;
 136
 137   explicit AtomicInt(IntType* b) : b_(b) {}
 138   AtomicInt(AtomicInt const&) = delete;
 139   AtomicInt& operator=(AtomicInt const&) = delete;
 140
 141   IntType load(MemoryOrder ord) { return std::__libcpp_atomic_load(b_, ord); }
 142   void store(IntType val, MemoryOrder ord) { std::__libcpp_atomic_store(b_, val, ord); }
 143   IntType exchange(IntType new_val, MemoryOrder ord) { return std::__libcpp_atomic_exchange(b_, new_val, ord); }
 144   bool compare_exchange(IntType* expected, IntType desired, MemoryOrder ord_success, MemoryOrder ord_failure) {
 145     return std::__libcpp_atomic_compare_exchange(b_, expected, desired, ord_success, ord_failure);
 146   }
 147
 148 private:
 149   IntType* b_;
 150 };
 151
 152 //===----------------------------------------------------------------------===//
 153 //                       PlatformGetThreadID
 154 //===----------------------------------------------------------------------===//
 155
 156 #if defined(__APPLE__) && defined(_LIBCPP_HAS_THREAD_API_PTHREAD)
 157 uint32_t PlatformThreadID() {
 158   static_assert(sizeof(mach_port_t) == sizeof(uint32_t), "");
 159   return static_cast<uint32_t>(pthread_mach_thread_np(std::__libcpp_thread_get_current_id()));
 160 }
 161 #elif defined(SYS_gettid) && defined(_LIBCPP_HAS_THREAD_API_PTHREAD)
 162 uint32_t PlatformThreadID() {
 163   static_assert(sizeof(pid_t) == sizeof(uint32_t), "");
 164   return static_cast<uint32_t>(syscall(SYS_gettid));
 165 }
 166 #else
 167 constexpr uint32_t (*PlatformThreadID)() = nullptr;
 168 #endif
 169
 170 //===----------------------------------------------------------------------===//
 171 //                          GuardByte
 172 //===----------------------------------------------------------------------===//
 173
 174 static constexpr uint8_t UNSET = 0;
 175 static constexpr uint8_t COMPLETE_BIT = (1 << 0);
 176 static constexpr uint8_t PENDING_BIT = (1 << 1);
 177 static constexpr uint8_t WAITING_BIT = (1 << 2);
 178
 179 /// Manages reads and writes to the guard byte.
 180 struct GuardByte {
 181   GuardByte() = delete;
 182   GuardByte(GuardByte const&) = delete;
 183   GuardByte& operator=(GuardByte const&) = delete;
 184
 185   explicit GuardByte(uint8_t* const guard_byte_address) : guard_byte(guard_byte_address) {}
 186
 187 public:
 188   /// The guard byte portion of cxa_guard_acquire. Returns true if
 189   /// initialization has already been completed.
 190   bool acquire() {
 191     // if guard_byte is non-zero, we have already completed initialization
 192     // (i.e. release has been called)
 193     return guard_byte.load(std::_AO_Acquire) != UNSET;
 194   }
 195
 196   /// The guard byte portion of cxa_guard_release.
 197   void release() { guard_byte.store(COMPLETE_BIT, std::_AO_Release); }
 198
 199   /// The guard byte portion of cxa_guard_abort.
 200   void abort() {} // Nothing to do
 201
 202 private:
 203   AtomicInt<uint8_t> guard_byte;
 204 };
 205
 206 //===----------------------------------------------------------------------===//
 207 //                       InitByte Implementations
 208 //===----------------------------------------------------------------------===//
 209 //
 210 // Each initialization byte implementation supports the following methods:
 211 //
 212 //  InitByte(uint8_t* _init_byte_address, uint32_t* _thread_id_address)
 213 //    Construct the InitByte object, initializing our member variables
 214 //
 215 //  bool acquire()
 216 //    Called before we start the initialization. Check if someone else has already started, and if
 217 //    not to signal our intent to start it ourselves. We determine the current status from the init
 218 //    byte, which is one of 4 possible values:
 219 //      COMPLETE:           Initialization was finished by somebody else. Return true.
 220 //      PENDING:            Somebody has started the initialization already, set the WAITING bit,
 221 //                          then wait for the init byte to get updated with a new value.
 222 //      (PENDING|WAITING):  Somebody has started the initialization already, and we're not the
 223 //                          first one waiting. Wait for the init byte to get updated.
 224 //      UNSET:              Initialization hasn't successfully completed, and nobody is currently
 225 //                          performing the initialization. Set the PENDING bit to indicate our
 226 //                          intention to start the initialization, and return false.
 227 //    The return value indicates whether initialization has already been completed.
 228 //
 229 //  void release()
 230 //    Called after successfully completing the initialization. Update the init byte to reflect
 231 //    that, then if anybody else is waiting, wake them up.
 232 //
 233 //  void abort()
 234 //    Called after an error is thrown during the initialization. Reset the init byte to UNSET to
 235 //    indicate that we're no longer performing the initialization, then if anybody is waiting, wake
 236 //    them up so they can try performing the initialization.
 237 //
 238
 239 //===----------------------------------------------------------------------===//
 240 //                    Single Threaded Implementation
 241 //===----------------------------------------------------------------------===//
 242
 243 /// InitByteNoThreads - Doesn't use any inter-thread synchronization when
 244 /// managing reads and writes to the init byte.
 245 struct InitByteNoThreads {
 246   InitByteNoThreads() = delete;
 247   InitByteNoThreads(InitByteNoThreads const&) = delete;
 248   InitByteNoThreads& operator=(InitByteNoThreads const&) = delete;
 249
 250   explicit InitByteNoThreads(uint8_t* _init_byte_address, uint32_t*) : init_byte_address(_init_byte_address) {}
 251
 252   /// The init byte portion of cxa_guard_acquire. Returns true if
 253   /// initialization has already been completed.
 254   bool acquire() {
 255     if (*init_byte_address == COMPLETE_BIT)
 256       return true;
 257     if (*init_byte_address & PENDING_BIT)
 258       ABORT_WITH_MESSAGE("__cxa_guard_acquire detected recursive initialization: do you have a function-local static variable whose initialization depends on that function?");
 259     *init_byte_address = PENDING_BIT;
 260     return false;
 261   }
 262
 263   /// The init byte portion of cxa_guard_release.
 264   void release() { *init_byte_address = COMPLETE_BIT; }
 265   /// The init byte portion of cxa_guard_abort.
 266   void abort() { *init_byte_address = UNSET; }
 267
 268 private:
 269   /// The address of the byte used during initialization.
 270   uint8_t* const init_byte_address;
 271 };
 272
 273 //===----------------------------------------------------------------------===//
 274 //                     Global Mutex Implementation
 275 //===----------------------------------------------------------------------===//
 276
 277 struct LibcppMutex;
 278 struct LibcppCondVar;
 279
 280 #ifndef _LIBCXXABI_HAS_NO_THREADS
 281 struct LibcppMutex {
 282   LibcppMutex() = default;
 283   LibcppMutex(LibcppMutex const&) = delete;
 284   LibcppMutex& operator=(LibcppMutex const&) = delete;
 285
 286   bool lock() { return std::__libcpp_mutex_lock(&mutex); }
 287   bool unlock() { return std::__libcpp_mutex_unlock(&mutex); }
 288
 289 private:
 290   friend struct LibcppCondVar;
 291   std::__libcpp_mutex_t mutex = _LIBCPP_MUTEX_INITIALIZER;
 292 };
 293
 294 struct LibcppCondVar {
 295   LibcppCondVar() = default;
 296   LibcppCondVar(LibcppCondVar const&) = delete;
 297   LibcppCondVar& operator=(LibcppCondVar const&) = delete;
 298
 299   bool wait(LibcppMutex& mut) { return std::__libcpp_condvar_wait(&cond, &mut.mutex); }
 300   bool broadcast() { return std::__libcpp_condvar_broadcast(&cond); }
 301
 302 private:
 303   std::__libcpp_condvar_t cond = _LIBCPP_CONDVAR_INITIALIZER;
 304 };
 305 #else
 306 struct LibcppMutex {};
 307 struct LibcppCondVar {};
 308 #endif // !defined(_LIBCXXABI_HAS_NO_THREADS)
 309
 310 /// InitByteGlobalMutex - Uses a global mutex and condition variable (common to
 311 /// all static local variables) to manage reads and writes to the init byte.
 312 template <class Mutex, class CondVar, Mutex& global_mutex, CondVar& global_cond,
 313           uint32_t (*GetThreadID)() = PlatformThreadID>
 314 struct InitByteGlobalMutex {
 315
 316   explicit InitByteGlobalMutex(uint8_t* _init_byte_address, uint32_t* _thread_id_address)
 317       : init_byte_address(_init_byte_address), thread_id_address(_thread_id_address),
 318         has_thread_id_support(_thread_id_address != nullptr && GetThreadID != nullptr) {}
 319
 320 public:
 321   /// The init byte portion of cxa_guard_acquire. Returns true if
 322   /// initialization has already been completed.
 323   bool acquire() {
 324     LockGuard g("__cxa_guard_acquire");
 325     // Check for possible recursive initialization.
 326     if (has_thread_id_support && (*init_byte_address & PENDING_BIT)) {
 327       if (*thread_id_address == current_thread_id.get())
 328         ABORT_WITH_MESSAGE("__cxa_guard_acquire detected recursive initialization: do you have a function-local static variable whose initialization depends on that function?");
 329     }
 330
 331     // Wait until the pending bit is not set.
 332     while (*init_byte_address & PENDING_BIT) {
 333       *init_byte_address |= WAITING_BIT;
 334       global_cond.wait(global_mutex);
 335     }
 336
 337     if (*init_byte_address == COMPLETE_BIT)
 338       return true;
 339
 340     if (has_thread_id_support)
 341       *thread_id_address = current_thread_id.get();
 342
 343     *init_byte_address = PENDING_BIT;
 344     return false;
 345   }
 346
 347   /// The init byte portion of cxa_guard_release.
 348   void release() {
 349     bool has_waiting;
 350     {
 351       LockGuard g("__cxa_guard_release");
 352       has_waiting = *init_byte_address & WAITING_BIT;
 353       *init_byte_address = COMPLETE_BIT;
 354     }
 355     if (has_waiting) {
 356       if (global_cond.broadcast()) {
 357         ABORT_WITH_MESSAGE("%s failed to broadcast", "__cxa_guard_release");
 358       }
 359     }
 360   }
 361
 362   /// The init byte portion of cxa_guard_abort.
 363   void abort() {
 364     bool has_waiting;
 365     {
 366       LockGuard g("__cxa_guard_abort");
 367       if (has_thread_id_support)
 368         *thread_id_address = 0;
 369       has_waiting = *init_byte_address & WAITING_BIT;
 370       *init_byte_address = UNSET;
 371     }
 372     if (has_waiting) {
 373       if (global_cond.broadcast()) {
 374         ABORT_WITH_MESSAGE("%s failed to broadcast", "__cxa_guard_abort");
 375       }
 376     }
 377   }
 378
 379 private:
 380   /// The address of the byte used during initialization.
 381   uint8_t* const init_byte_address;
 382   /// An optional address storing an identifier for the thread performing initialization.
 383   /// It's used to detect recursive initialization.
 384   uint32_t* const thread_id_address;
 385
 386   const bool has_thread_id_support;
 387   LazyValue<uint32_t, GetThreadID> current_thread_id;
 388
 389 private:
 390   struct LockGuard {
 391     LockGuard() = delete;
 392     LockGuard(LockGuard const&) = delete;
 393     LockGuard& operator=(LockGuard const&) = delete;
 394
 395     explicit LockGuard(const char* calling_func) : calling_func_(calling_func) {
 396       if (global_mutex.lock())
 397         ABORT_WITH_MESSAGE("%s failed to acquire mutex", calling_func_);
 398     }
 399
 400     ~LockGuard() {
 401       if (global_mutex.unlock())
 402         ABORT_WITH_MESSAGE("%s failed to release mutex", calling_func_);
 403     }
 404
 405   private:
 406     const char* const calling_func_;
 407   };
 408 };
 409
 410 //===----------------------------------------------------------------------===//
 411 //                         Futex Implementation
 412 //===----------------------------------------------------------------------===//
 413
 414 #if defined(SYS_futex)
 415 void PlatformFutexWait(int* addr, int expect) {
 416   constexpr int WAIT = 0;
 417   syscall(SYS_futex, addr, WAIT, expect, 0);
 418   __tsan_acquire(addr);
 419 }
 420 void PlatformFutexWake(int* addr) {
 421   constexpr int WAKE = 1;
 422   __tsan_release(addr);
 423   syscall(SYS_futex, addr, WAKE, INT_MAX);
 424 }
 425 #else
 426 constexpr void (*PlatformFutexWait)(int*, int) = nullptr;
 427 constexpr void (*PlatformFutexWake)(int*) = nullptr;
 428 #endif
 429
 430 constexpr bool PlatformSupportsFutex() { return +PlatformFutexWait != nullptr; }
 431
 432 /// InitByteFutex - Uses a futex to manage reads and writes to the init byte.
 433 template <void (*Wait)(int*, int) = PlatformFutexWait, void (*Wake)(int*) = PlatformFutexWake,
 434           uint32_t (*GetThreadIDArg)() = PlatformThreadID>
 435 struct InitByteFutex {
 436
 437   explicit InitByteFutex(uint8_t* _init_byte_address, uint32_t* _thread_id_address)
 438       : init_byte(_init_byte_address),
 439         has_thread_id_support(_thread_id_address != nullptr && GetThreadIDArg != nullptr),
 440         thread_id(_thread_id_address),
 441         base_address(reinterpret_cast<int*>(/*_init_byte_address & ~0x3*/ _init_byte_address - 1)) {}
 442
 443 public:
 444   /// The init byte portion of cxa_guard_acquire. Returns true if
 445   /// initialization has already been completed.
 446   bool acquire() {
 447     while (true) {
 448       uint8_t last_val = UNSET;
 449       if (init_byte.compare_exchange(&last_val, PENDING_BIT, std::_AO_Acq_Rel, std::_AO_Acquire)) {
 450         if (has_thread_id_support) {
 451           thread_id.store(current_thread_id.get(), std::_AO_Relaxed);
 452         }
 453         return false;
 454       }
 455
 456       if (last_val == COMPLETE_BIT)
 457         return true;
 458
 459       if (last_val & PENDING_BIT) {
 460
 461         // Check for recursive initialization
 462         if (has_thread_id_support && thread_id.load(std::_AO_Relaxed) == current_thread_id.get()) {
 463           ABORT_WITH_MESSAGE("__cxa_guard_acquire detected recursive initialization: do you have a function-local static variable whose initialization depends on that function?");
 464         }
 465
 466         if ((last_val & WAITING_BIT) == 0) {
 467           // This compare exchange can fail for several reasons
 468           // (1) another thread finished the whole thing before we got here
 469           // (2) another thread set the waiting bit we were trying to thread
 470           // (3) another thread had an exception and failed to finish
 471           if (!init_byte.compare_exchange(&last_val, PENDING_BIT | WAITING_BIT, std::_AO_Acq_Rel, std::_AO_Release)) {
 472             // (1) success, via someone else's work!
 473             if (last_val == COMPLETE_BIT)
 474               return true;
 475
 476             // (3) someone else, bailed on doing the work, retry from the start!
 477             if (last_val == UNSET)
 478               continue;
 479
 480             // (2) the waiting bit got set, so we are happy to keep waiting
 481           }
 482         }
 483         wait_on_initialization();
 484       }
 485     }
 486   }
 487
 488   /// The init byte portion of cxa_guard_release.
 489   void release() {
 490     uint8_t old = init_byte.exchange(COMPLETE_BIT, std::_AO_Acq_Rel);
 491     if (old & WAITING_BIT)
 492       wake_all();
 493   }
 494
 495   /// The init byte portion of cxa_guard_abort.
 496   void abort() {
 497     if (has_thread_id_support)
 498       thread_id.store(0, std::_AO_Relaxed);
 499
 500     uint8_t old = init_byte.exchange(UNSET, std::_AO_Acq_Rel);
 501     if (old & WAITING_BIT)
 502       wake_all();
 503   }
 504
 505 private:
 506   /// Use the futex to wait on the current guard variable. Futex expects a
 507   /// 32-bit 4-byte aligned address as the first argument, so we use the 4-byte
 508   /// aligned address that encompasses the init byte (i.e. the address of the
 509   /// raw guard object that was passed to __cxa_guard_acquire/release/abort).
 510   void wait_on_initialization() { Wait(base_address, expected_value_for_futex(PENDING_BIT | WAITING_BIT)); }
 511   void wake_all() { Wake(base_address); }
 512
 513 private:
 514   AtomicInt<uint8_t> init_byte;
 515
 516   const bool has_thread_id_support;
 517   // Unsafe to use unless has_thread_id_support
 518   AtomicInt<uint32_t> thread_id;
 519   LazyValue<uint32_t, GetThreadIDArg> current_thread_id;
 520
 521   /// the 4-byte-aligned address that encompasses the init byte (i.e. the
 522   /// address of the raw guard object).
 523   int* const base_address;
 524
 525   /// Create the expected integer value for futex `wait(int* addr, int expected)`.
 526   /// We pass the base address as the first argument, So this function creates
 527   /// an zero-initialized integer  with `b` copied at the correct offset.
 528   static int expected_value_for_futex(uint8_t b) {
 529     int dest_val = 0;
 530     std::memcpy(reinterpret_cast<char*>(&dest_val) + 1, &b, 1);
 531     return dest_val;
 532   }
 533
 534   static_assert(Wait != nullptr && Wake != nullptr, "");
 535 };
 536
 537 //===----------------------------------------------------------------------===//
 538 //                          GuardObject
 539 //===----------------------------------------------------------------------===//
 540
 541 enum class AcquireResult {
 542   INIT_IS_DONE,
 543   INIT_IS_PENDING,
 544 };
 545 constexpr AcquireResult INIT_IS_DONE = AcquireResult::INIT_IS_DONE;
 546 constexpr AcquireResult INIT_IS_PENDING = AcquireResult::INIT_IS_PENDING;
 547
 548 /// Co-ordinates between GuardByte and InitByte.
 549 template <class InitByteT>
 550 struct GuardObject {
 551   GuardObject() = delete;
 552   GuardObject(GuardObject const&) = delete;
 553   GuardObject& operator=(GuardObject const&) = delete;
 554
 555 private:
 556   GuardByte guard_byte;
 557   InitByteT init_byte;
 558
 559 public:
 560   /// ARM Constructor
 561   explicit GuardObject(uint32_t* raw_guard_object)
 562       : guard_byte(reinterpret_cast<uint8_t*>(raw_guard_object)),
 563         init_byte(reinterpret_cast<uint8_t*>(raw_guard_object) + 1, nullptr) {}
 564
 565   /// Itanium Constructor
 566   explicit GuardObject(uint64_t* raw_guard_object)
 567       : guard_byte(reinterpret_cast<uint8_t*>(raw_guard_object)),
 568         init_byte(reinterpret_cast<uint8_t*>(raw_guard_object) + 1, reinterpret_cast<uint32_t*>(raw_guard_object) + 1) {
 569   }
 570
 571   /// Implements __cxa_guard_acquire.
 572   AcquireResult cxa_guard_acquire() {
 573     // Use short-circuit evaluation to avoid calling init_byte.acquire when
 574     // guard_byte.acquire returns true. (i.e. don't call it when we know from
 575     // the guard byte that initialization has already been completed)
 576     if (guard_byte.acquire() || init_byte.acquire())
 577       return INIT_IS_DONE;
 578     return INIT_IS_PENDING;
 579   }
 580
 581   /// Implements __cxa_guard_release.
 582   void cxa_guard_release() {
 583     // Update guard byte first, so if somebody is woken up by init_byte.release
 584     // and comes all the way back around to __cxa_guard_acquire again, they see
 585     // it as having completed initialization.
 586     guard_byte.release();
 587     init_byte.release();
 588   }
 589
 590   /// Implements __cxa_guard_abort.
 591   void cxa_guard_abort() {
 592     guard_byte.abort();
 593     init_byte.abort();
 594   }
 595 };
 596
 597 //===----------------------------------------------------------------------===//
 598 //                          Convenience Classes
 599 //===----------------------------------------------------------------------===//
 600
 601 /// NoThreadsGuard - Manages initialization without performing any inter-thread
 602 /// synchronization.
 603 using NoThreadsGuard = GuardObject<InitByteNoThreads>;
 604
 605 /// GlobalMutexGuard - Manages initialization using a global mutex and
 606 /// condition variable.
 607 template <class Mutex, class CondVar, Mutex& global_mutex, CondVar& global_cond,
 608           uint32_t (*GetThreadID)() = PlatformThreadID>
 609 using GlobalMutexGuard = GuardObject<InitByteGlobalMutex<Mutex, CondVar, global_mutex, global_cond, GetThreadID>>;
 610
 611 /// FutexGuard - Manages initialization using atomics and the futex syscall for
 612 /// waiting and waking.
 613 template <void (*Wait)(int*, int) = PlatformFutexWait, void (*Wake)(int*) = PlatformFutexWake,
 614           uint32_t (*GetThreadIDArg)() = PlatformThreadID>
 615 using FutexGuard = GuardObject<InitByteFutex<Wait, Wake, GetThreadIDArg>>;
 616
 617 //===----------------------------------------------------------------------===//
 618 //
 619 //===----------------------------------------------------------------------===//
 620
 621 template <class T>
 622 struct GlobalStatic {
 623   static T instance;
 624 };
 625 template <class T>
 626 _LIBCPP_CONSTINIT T GlobalStatic<T>::instance = {};
 627
 628 enum class Implementation { NoThreads, GlobalMutex, Futex };
 629
 630 template <Implementation Impl>
 631 struct SelectImplementation;
 632
 633 template <>
 634 struct SelectImplementation<Implementation::NoThreads> {
 635   using type = NoThreadsGuard;
 636 };
 637
 638 template <>
 639 struct SelectImplementation<Implementation::GlobalMutex> {
 640   using type = GlobalMutexGuard<LibcppMutex, LibcppCondVar, GlobalStatic<LibcppMutex>::instance,
 641                                 GlobalStatic<LibcppCondVar>::instance, PlatformThreadID>;
 642 };
 643
 644 template <>
 645 struct SelectImplementation<Implementation::Futex> {
 646   using type = FutexGuard<PlatformFutexWait, PlatformFutexWake, PlatformThreadID>;
 647 };
 648
 649 // TODO(EricWF): We should prefer the futex implementation when available. But
 650 // it should be done in a separate step from adding the implementation.
 651 constexpr Implementation CurrentImplementation =
 652 #if defined(_LIBCXXABI_HAS_NO_THREADS)
 653     Implementation::NoThreads;
 654 #elif defined(_LIBCXXABI_USE_FUTEX)
 655     Implementation::Futex;
 656 #else
 657     Implementation::GlobalMutex;
 658 #endif
 659
 660 static_assert(CurrentImplementation != Implementation::Futex || PlatformSupportsFutex(),
 661               "Futex selected but not supported");
 662
 663 using SelectedImplementation = SelectImplementation<CurrentImplementation>::type;
 664
 665 } // end namespace
 666 } // end namespace __cxxabiv1
 667
 668 #if defined(__clang__)
 669 #  pragma clang diagnostic pop
 670 #elif defined(__GNUC__)
 671 #  pragma GCC diagnostic pop
 672 #endif
 673
 674 #endif // LIBCXXABI_SRC_INCLUDE_CXA_GUARD_IMPL_H