libcxx/src/memory.cpp

   1 //===----------------------------------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include <__config>
  10 #ifdef _LIBCPP_DEPRECATED_ABI_LEGACY_LIBRARY_DEFINITIONS_FOR_INLINE_FUNCTIONS
  11 #   define _LIBCPP_SHARED_PTR_DEFINE_LEGACY_INLINE_FUNCTIONS
  12 #endif
  13
  14 #include <memory>
  15
  16 #ifndef _LIBCPP_HAS_NO_THREADS
  17 #  include <mutex>
  18 #  include <thread>
  19 #  if defined(__ELF__) && defined(_LIBCPP_LINK_PTHREAD_LIB)
  20 #    pragma comment(lib, "pthread")
  21 #  endif
  22 #endif
  23
  24 #include "include/atomic_support.h"
  25
  26 _LIBCPP_BEGIN_NAMESPACE_STD
  27
  28 bad_weak_ptr::~bad_weak_ptr() noexcept {}
  29
  30 const char*
  31 bad_weak_ptr::what() const noexcept
  32 {
  33     return "bad_weak_ptr";
  34 }
  35
  36 __shared_count::~__shared_count()
  37 {
  38 }
  39
  40 __shared_weak_count::~__shared_weak_count()
  41 {
  42 }
  43
  44 #if defined(_LIBCPP_SHARED_PTR_DEFINE_LEGACY_INLINE_FUNCTIONS)
  45 void
  46 __shared_count::__add_shared() noexcept
  47 {
  48     __libcpp_atomic_refcount_increment(__shared_owners_);
  49 }
  50
  51 bool
  52 __shared_count::__release_shared() noexcept
  53 {
  54     if (__libcpp_atomic_refcount_decrement(__shared_owners_) == -1)
  55     {
  56         __on_zero_shared();
  57         return true;
  58     }
  59     return false;
  60 }
  61
  62 void
  63 __shared_weak_count::__add_shared() noexcept
  64 {
  65     __shared_count::__add_shared();
  66 }
  67
  68 void
  69 __shared_weak_count::__add_weak() noexcept
  70 {
  71     __libcpp_atomic_refcount_increment(__shared_weak_owners_);
  72 }
  73
  74 void
  75 __shared_weak_count::__release_shared() noexcept
  76 {
  77     if (__shared_count::__release_shared())
  78         __release_weak();
  79 }
  80 #endif // _LIBCPP_SHARED_PTR_DEFINE_LEGACY_INLINE_FUNCTIONS
  81
  82 void
  83 __shared_weak_count::__release_weak() noexcept
  84 {
  85     // NOTE: The acquire load here is an optimization of the very
  86     // common case where a shared pointer is being destructed while
  87     // having no other contended references.
  88     //
  89     // BENEFIT: We avoid expensive atomic stores like XADD and STREX
  90     // in a common case.  Those instructions are slow and do nasty
  91     // things to caches.
  92     //
  93     // IS THIS SAFE?  Yes.  During weak destruction, if we see that we
  94     // are the last reference, we know that no-one else is accessing
  95     // us. If someone were accessing us, then they would be doing so
  96     // while the last shared / weak_ptr was being destructed, and
  97     // that's undefined anyway.
  98     //
  99     // If we see anything other than a 0, then we have possible
 100     // contention, and need to use an atomicrmw primitive.
 101     // The same arguments don't apply for increment, where it is legal
 102     // (though inadvisable) to share shared_ptr references between
 103     // threads, and have them all get copied at once.  The argument
 104     // also doesn't apply for __release_shared, because an outstanding
 105     // weak_ptr::lock() could read / modify the shared count.
 106     if (__libcpp_atomic_load(&__shared_weak_owners_, _AO_Acquire) == 0)
 107     {
 108         // no need to do this store, because we are about
 109         // to destroy everything.
 110         //__libcpp_atomic_store(&__shared_weak_owners_, -1, _AO_Release);
 111         __on_zero_shared_weak();
 112     }
 113     else if (__libcpp_atomic_refcount_decrement(__shared_weak_owners_) == -1)
 114         __on_zero_shared_weak();
 115 }
 116
 117 __shared_weak_count*
 118 __shared_weak_count::lock() noexcept
 119 {
 120     long object_owners = __libcpp_atomic_load(&__shared_owners_);
 121     while (object_owners != -1)
 122     {
 123         if (__libcpp_atomic_compare_exchange(&__shared_owners_,
 124                                              &object_owners,
 125                                              object_owners+1))
 126             return this;
 127     }
 128     return nullptr;
 129 }
 130
 131 const void*
 132 __shared_weak_count::__get_deleter(const type_info&) const noexcept
 133 {
 134     return nullptr;
 135 }
 136
 137 #if !defined(_LIBCPP_HAS_NO_THREADS)
 138
 139 static constexpr std::size_t __sp_mut_count = 32;
 140 static constinit __libcpp_mutex_t mut_back[__sp_mut_count] =
 141 {
 142     _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER,
 143     _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER,
 144     _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER,
 145     _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER,
 146     _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER,
 147     _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER,
 148     _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER,
 149     _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER
 150 };
 151
 152 constexpr __sp_mut::__sp_mut(void* p) noexcept
 153    : __lx_(p)
 154 {
 155 }
 156
 157 void
 158 __sp_mut::lock() noexcept
 159 {
 160     auto m = static_cast<__libcpp_mutex_t*>(__lx_);
 161     __libcpp_mutex_lock(m);
 162 }
 163
 164 void
 165 __sp_mut::unlock() noexcept
 166 {
 167     __libcpp_mutex_unlock(static_cast<__libcpp_mutex_t*>(__lx_));
 168 }
 169
 170 __sp_mut&
 171 __get_sp_mut(const void* p)
 172 {
 173     static constinit __sp_mut muts[__sp_mut_count] = {
 174         &mut_back[ 0], &mut_back[ 1], &mut_back[ 2], &mut_back[ 3],
 175         &mut_back[ 4], &mut_back[ 5], &mut_back[ 6], &mut_back[ 7],
 176         &mut_back[ 8], &mut_back[ 9], &mut_back[10], &mut_back[11],
 177         &mut_back[12], &mut_back[13], &mut_back[14], &mut_back[15],
 178         &mut_back[16], &mut_back[17], &mut_back[18], &mut_back[19],
 179         &mut_back[20], &mut_back[21], &mut_back[22], &mut_back[23],
 180         &mut_back[24], &mut_back[25], &mut_back[26], &mut_back[27],
 181         &mut_back[28], &mut_back[29], &mut_back[30], &mut_back[31]
 182     };
 183     return muts[hash<const void*>()(p) & (__sp_mut_count-1)];
 184 }
 185
 186 #endif // !defined(_LIBCPP_HAS_NO_THREADS)
 187
 188 void*
 189 align(size_t alignment, size_t size, void*& ptr, size_t& space)
 190 {
 191     void* r = nullptr;
 192     if (size <= space)
 193     {
 194         char* p1 = static_cast<char*>(ptr);
 195         char* p2 = reinterpret_cast<char*>(reinterpret_cast<uintptr_t>(p1 + (alignment - 1)) & -alignment);
 196         size_t d = static_cast<size_t>(p2 - p1);
 197         if (d <= space - size)
 198         {
 199             r = p2;
 200             ptr = r;
 201             space -= d;
 202         }
 203     }
 204     return r;
 205 }
 206
 207 _LIBCPP_END_NAMESPACE_STD