libcxx/src/memory.cpp

   1 //===----------------------------------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include <__config>
  10 #ifdef _LIBCPP_DEPRECATED_ABI_LEGACY_LIBRARY_DEFINITIONS_FOR_INLINE_FUNCTIONS
  11 #  define _LIBCPP_SHARED_PTR_DEFINE_LEGACY_INLINE_FUNCTIONS
  12 #endif
  13
  14 #include <memory>
  15
  16 #ifndef _LIBCPP_HAS_NO_THREADS
  17 #  include <mutex>
  18 #  include <thread>
  19 #  if defined(__ELF__) && defined(_LIBCPP_LINK_PTHREAD_LIB)
  20 #    pragma comment(lib, "pthread")
  21 #  endif
  22 #endif
  23
  24 #include "include/atomic_support.h"
  25
  26 _LIBCPP_BEGIN_NAMESPACE_STD
  27
  28 bad_weak_ptr::~bad_weak_ptr() noexcept {}
  29
  30 const char* bad_weak_ptr::what() const noexcept { return "bad_weak_ptr"; }
  31
  32 __shared_count::~__shared_count() {}
  33
  34 __shared_weak_count::~__shared_weak_count() {}
  35
  36 #if defined(_LIBCPP_SHARED_PTR_DEFINE_LEGACY_INLINE_FUNCTIONS)
  37 void __shared_count::__add_shared() noexcept { __libcpp_atomic_refcount_increment(__shared_owners_); }
  38
  39 bool __shared_count::__release_shared() noexcept {
  40   if (__libcpp_atomic_refcount_decrement(__shared_owners_) == -1) {
  41     __on_zero_shared();
  42     return true;
  43   }
  44   return false;
  45 }
  46
  47 void __shared_weak_count::__add_shared() noexcept { __shared_count::__add_shared(); }
  48
  49 void __shared_weak_count::__add_weak() noexcept { __libcpp_atomic_refcount_increment(__shared_weak_owners_); }
  50
  51 void __shared_weak_count::__release_shared() noexcept {
  52   if (__shared_count::__release_shared())
  53     __release_weak();
  54 }
  55 #endif // _LIBCPP_SHARED_PTR_DEFINE_LEGACY_INLINE_FUNCTIONS
  56
  57 void __shared_weak_count::__release_weak() noexcept {
  58   // NOTE: The acquire load here is an optimization of the very
  59   // common case where a shared pointer is being destructed while
  60   // having no other contended references.
  61   //
  62   // BENEFIT: We avoid expensive atomic stores like XADD and STREX
  63   // in a common case.  Those instructions are slow and do nasty
  64   // things to caches.
  65   //
  66   // IS THIS SAFE?  Yes.  During weak destruction, if we see that we
  67   // are the last reference, we know that no-one else is accessing
  68   // us. If someone were accessing us, then they would be doing so
  69   // while the last shared / weak_ptr was being destructed, and
  70   // that's undefined anyway.
  71   //
  72   // If we see anything other than a 0, then we have possible
  73   // contention, and need to use an atomicrmw primitive.
  74   // The same arguments don't apply for increment, where it is legal
  75   // (though inadvisable) to share shared_ptr references between
  76   // threads, and have them all get copied at once.  The argument
  77   // also doesn't apply for __release_shared, because an outstanding
  78   // weak_ptr::lock() could read / modify the shared count.
  79   if (__libcpp_atomic_load(&__shared_weak_owners_, _AO_Acquire) == 0) {
  80     // no need to do this store, because we are about
  81     // to destroy everything.
  82     //__libcpp_atomic_store(&__shared_weak_owners_, -1, _AO_Release);
  83     __on_zero_shared_weak();
  84   } else if (__libcpp_atomic_refcount_decrement(__shared_weak_owners_) == -1)
  85     __on_zero_shared_weak();
  86 }
  87
  88 __shared_weak_count* __shared_weak_count::lock() noexcept {
  89   long object_owners = __libcpp_atomic_load(&__shared_owners_);
  90   while (object_owners != -1) {
  91     if (__libcpp_atomic_compare_exchange(&__shared_owners_, &object_owners, object_owners + 1))
  92       return this;
  93   }
  94   return nullptr;
  95 }
  96
  97 const void* __shared_weak_count::__get_deleter(const type_info&) const noexcept { return nullptr; }
  98
  99 #if !defined(_LIBCPP_HAS_NO_THREADS)
 100
 101 static constexpr std::size_t __sp_mut_count                = 32;
 102 static constinit __libcpp_mutex_t mut_back[__sp_mut_count] = {
 103     _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER,
 104     _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER,
 105     _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER,
 106     _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER,
 107     _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER,
 108     _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER,
 109     _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER,
 110     _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER};
 111
 112 constexpr __sp_mut::__sp_mut(void* p) noexcept : __lx_(p) {}
 113
 114 void __sp_mut::lock() noexcept {
 115   auto m = static_cast<__libcpp_mutex_t*>(__lx_);
 116   __libcpp_mutex_lock(m);
 117 }
 118
 119 void __sp_mut::unlock() noexcept { __libcpp_mutex_unlock(static_cast<__libcpp_mutex_t*>(__lx_)); }
 120
 121 __sp_mut& __get_sp_mut(const void* p) {
 122   static constinit __sp_mut muts[__sp_mut_count] = {
 123       &mut_back[0],  &mut_back[1],  &mut_back[2],  &mut_back[3],  &mut_back[4],  &mut_back[5],  &mut_back[6],
 124       &mut_back[7],  &mut_back[8],  &mut_back[9],  &mut_back[10], &mut_back[11], &mut_back[12], &mut_back[13],
 125       &mut_back[14], &mut_back[15], &mut_back[16], &mut_back[17], &mut_back[18], &mut_back[19], &mut_back[20],
 126       &mut_back[21], &mut_back[22], &mut_back[23], &mut_back[24], &mut_back[25], &mut_back[26], &mut_back[27],
 127       &mut_back[28], &mut_back[29], &mut_back[30], &mut_back[31]};
 128   return muts[hash<const void*>()(p) & (__sp_mut_count - 1)];
 129 }
 130
 131 #endif // !defined(_LIBCPP_HAS_NO_THREADS)
 132
 133 void* align(size_t alignment, size_t size, void*& ptr, size_t& space) {
 134   void* r = nullptr;
 135   if (size <= space) {
 136     char* p1 = static_cast<char*>(ptr);
 137     char* p2 = reinterpret_cast<char*>(reinterpret_cast<uintptr_t>(p1 + (alignment - 1)) & -alignment);
 138     size_t d = static_cast<size_t>(p2 - p1);
 139     if (d <= space - size) {
 140       r   = p2;
 141       ptr = r;
 142       space -= d;
 143     }
 144   }
 145   return r;
 146 }
 147
 148 _LIBCPP_END_NAMESPACE_STD