libcxx/include/__atomic/cxx_atomic_impl.h

   1 //===----------------------------------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #ifndef _LIBCPP___ATOMIC_CXX_ATOMIC_IMPL_H
  10 #define _LIBCPP___ATOMIC_CXX_ATOMIC_IMPL_H
  11
  12 #include <__atomic/is_always_lock_free.h>
  13 #include <__atomic/memory_order.h>
  14 #include <__config>
  15 #include <__memory/addressof.h>
  16 #include <__type_traits/conditional.h>
  17 #include <__type_traits/is_assignable.h>
  18 #include <__type_traits/is_trivially_copyable.h>
  19 #include <__type_traits/remove_const.h>
  20 #include <cstddef>
  21 #include <cstring>
  22
  23 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
  24 #  pragma GCC system_header
  25 #endif
  26
  27 _LIBCPP_BEGIN_NAMESPACE_STD
  28
  29 #if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) || defined(_LIBCPP_ATOMIC_ONLY_USE_BUILTINS)
  30
  31 // [atomics.types.generic]p1 guarantees _Tp is trivially copyable. Because
  32 // the default operator= in an object is not volatile, a byte-by-byte copy
  33 // is required.
  34 template <typename _Tp, typename _Tv, __enable_if_t<is_assignable<_Tp&, _Tv>::value, int> = 0>
  35 _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_assign_volatile(_Tp& __a_value, _Tv const& __val) {
  36   __a_value = __val;
  37 }
  38 template <typename _Tp, typename _Tv, __enable_if_t<is_assignable<_Tp&, _Tv>::value, int> = 0>
  39 _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_assign_volatile(_Tp volatile& __a_value, _Tv volatile const& __val) {
  40   volatile char* __to         = reinterpret_cast<volatile char*>(std::addressof(__a_value));
  41   volatile char* __end        = __to + sizeof(_Tp);
  42   volatile const char* __from = reinterpret_cast<volatile const char*>(std::addressof(__val));
  43   while (__to != __end)
  44     *__to++ = *__from++;
  45 }
  46
  47 #endif
  48
  49 #if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP)
  50
  51 template <typename _Tp>
  52 struct __cxx_atomic_base_impl {
  53   _LIBCPP_HIDE_FROM_ABI
  54 #  ifndef _LIBCPP_CXX03_LANG
  55   __cxx_atomic_base_impl() _NOEXCEPT = default;
  56 #  else
  57   __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {
  58   }
  59 #  endif // _LIBCPP_CXX03_LANG
  60   _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT : __a_value(value) {}
  61   _Tp __a_value;
  62 };
  63
  64 _LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR int __to_gcc_order(memory_order __order) {
  65   // Avoid switch statement to make this a constexpr.
  66   return __order == memory_order_relaxed
  67            ? __ATOMIC_RELAXED
  68            : (__order == memory_order_acquire
  69                   ? __ATOMIC_ACQUIRE
  70                   : (__order == memory_order_release
  71                          ? __ATOMIC_RELEASE
  72                          : (__order == memory_order_seq_cst
  73                                 ? __ATOMIC_SEQ_CST
  74                                 : (__order == memory_order_acq_rel ? __ATOMIC_ACQ_REL : __ATOMIC_CONSUME))));
  75 }
  76
  77 _LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR int __to_gcc_failure_order(memory_order __order) {
  78   // Avoid switch statement to make this a constexpr.
  79   return __order == memory_order_relaxed
  80            ? __ATOMIC_RELAXED
  81            : (__order == memory_order_acquire
  82                   ? __ATOMIC_ACQUIRE
  83                   : (__order == memory_order_release
  84                          ? __ATOMIC_RELAXED
  85                          : (__order == memory_order_seq_cst
  86                                 ? __ATOMIC_SEQ_CST
  87                                 : (__order == memory_order_acq_rel ? __ATOMIC_ACQUIRE : __ATOMIC_CONSUME))));
  88 }
  89
  90 template <typename _Tp>
  91 _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val) {
  92   __cxx_atomic_assign_volatile(__a->__a_value, __val);
  93 }
  94
  95 template <typename _Tp>
  96 _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) {
  97   __a->__a_value = __val;
  98 }
  99
 100 _LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_thread_fence(memory_order __order) {
 101   __atomic_thread_fence(__to_gcc_order(__order));
 102 }
 103
 104 _LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_signal_fence(memory_order __order) {
 105   __atomic_signal_fence(__to_gcc_order(__order));
 106 }
 107
 108 template <typename _Tp>
 109 _LIBCPP_HIDE_FROM_ABI void
 110 __cxx_atomic_store(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) {
 111   __atomic_store(std::addressof(__a->__a_value), std::addressof(__val), __to_gcc_order(__order));
 112 }
 113
 114 template <typename _Tp>
 115 _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) {
 116   __atomic_store(std::addressof(__a->__a_value), std::addressof(__val), __to_gcc_order(__order));
 117 }
 118
 119 template <typename _Tp>
 120 _LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(const volatile __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) {
 121   _Tp __ret;
 122   __atomic_load(std::addressof(__a->__a_value), std::addressof(__ret), __to_gcc_order(__order));
 123   return __ret;
 124 }
 125
 126 template <typename _Tp>
 127 _LIBCPP_HIDE_FROM_ABI void
 128 __cxx_atomic_load_inplace(const volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __dst, memory_order __order) {
 129   __atomic_load(std::addressof(__a->__a_value), __dst, __to_gcc_order(__order));
 130 }
 131
 132 template <typename _Tp>
 133 _LIBCPP_HIDE_FROM_ABI void
 134 __cxx_atomic_load_inplace(const __cxx_atomic_base_impl<_Tp>* __a, _Tp* __dst, memory_order __order) {
 135   __atomic_load(std::addressof(__a->__a_value), __dst, __to_gcc_order(__order));
 136 }
 137
 138 template <typename _Tp>
 139 _LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(const __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) {
 140   _Tp __ret;
 141   __atomic_load(std::addressof(__a->__a_value), std::addressof(__ret), __to_gcc_order(__order));
 142   return __ret;
 143 }
 144
 145 template <typename _Tp>
 146 _LIBCPP_HIDE_FROM_ABI _Tp
 147 __cxx_atomic_exchange(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) {
 148   _Tp __ret;
 149   __atomic_exchange(
 150       std::addressof(__a->__a_value), std::addressof(__value), std::addressof(__ret), __to_gcc_order(__order));
 151   return __ret;
 152 }
 153
 154 template <typename _Tp>
 155 _LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) {
 156   _Tp __ret;
 157   __atomic_exchange(
 158       std::addressof(__a->__a_value), std::addressof(__value), std::addressof(__ret), __to_gcc_order(__order));
 159   return __ret;
 160 }
 161
 162 template <typename _Tp>
 163 _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
 164     volatile __cxx_atomic_base_impl<_Tp>* __a,
 165     _Tp* __expected,
 166     _Tp __value,
 167     memory_order __success,
 168     memory_order __failure) {
 169   return __atomic_compare_exchange(
 170       std::addressof(__a->__a_value),
 171       __expected,
 172       std::addressof(__value),
 173       false,
 174       __to_gcc_order(__success),
 175       __to_gcc_failure_order(__failure));
 176 }
 177
 178 template <typename _Tp>
 179 _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
 180     __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) {
 181   return __atomic_compare_exchange(
 182       std::addressof(__a->__a_value),
 183       __expected,
 184       std::addressof(__value),
 185       false,
 186       __to_gcc_order(__success),
 187       __to_gcc_failure_order(__failure));
 188 }
 189
 190 template <typename _Tp>
 191 _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
 192     volatile __cxx_atomic_base_impl<_Tp>* __a,
 193     _Tp* __expected,
 194     _Tp __value,
 195     memory_order __success,
 196     memory_order __failure) {
 197   return __atomic_compare_exchange(
 198       std::addressof(__a->__a_value),
 199       __expected,
 200       std::addressof(__value),
 201       true,
 202       __to_gcc_order(__success),
 203       __to_gcc_failure_order(__failure));
 204 }
 205
 206 template <typename _Tp>
 207 _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
 208     __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) {
 209   return __atomic_compare_exchange(
 210       std::addressof(__a->__a_value),
 211       __expected,
 212       std::addressof(__value),
 213       true,
 214       __to_gcc_order(__success),
 215       __to_gcc_failure_order(__failure));
 216 }
 217
 218 template <typename _Tp>
 219 struct __skip_amt {
 220   enum { value = 1 };
 221 };
 222
 223 template <typename _Tp>
 224 struct __skip_amt<_Tp*> {
 225   enum { value = sizeof(_Tp) };
 226 };
 227
 228 // FIXME: Haven't figured out what the spec says about using arrays with
 229 // atomic_fetch_add. Force a failure rather than creating bad behavior.
 230 template <typename _Tp>
 231 struct __skip_amt<_Tp[]> {};
 232 template <typename _Tp, int n>
 233 struct __skip_amt<_Tp[n]> {};
 234
 235 template <typename _Tp, typename _Td>
 236 _LIBCPP_HIDE_FROM_ABI _Tp
 237 __cxx_atomic_fetch_add(volatile __cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) {
 238   return __atomic_fetch_add(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order));
 239 }
 240
 241 template <typename _Tp, typename _Td>
 242 _LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) {
 243   return __atomic_fetch_add(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order));
 244 }
 245
 246 template <typename _Tp, typename _Td>
 247 _LIBCPP_HIDE_FROM_ABI _Tp
 248 __cxx_atomic_fetch_sub(volatile __cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) {
 249   return __atomic_fetch_sub(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order));
 250 }
 251
 252 template <typename _Tp, typename _Td>
 253 _LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) {
 254   return __atomic_fetch_sub(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order));
 255 }
 256
 257 template <typename _Tp>
 258 _LIBCPP_HIDE_FROM_ABI _Tp
 259 __cxx_atomic_fetch_and(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
 260   return __atomic_fetch_and(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
 261 }
 262
 263 template <typename _Tp>
 264 _LIBCPP_HIDE_FROM_ABI _Tp
 265 __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
 266   return __atomic_fetch_and(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
 267 }
 268
 269 template <typename _Tp>
 270 _LIBCPP_HIDE_FROM_ABI _Tp
 271 __cxx_atomic_fetch_or(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
 272   return __atomic_fetch_or(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
 273 }
 274
 275 template <typename _Tp>
 276 _LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
 277   return __atomic_fetch_or(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
 278 }
 279
 280 template <typename _Tp>
 281 _LIBCPP_HIDE_FROM_ABI _Tp
 282 __cxx_atomic_fetch_xor(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
 283   return __atomic_fetch_xor(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
 284 }
 285
 286 template <typename _Tp>
 287 _LIBCPP_HIDE_FROM_ABI _Tp
 288 __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
 289   return __atomic_fetch_xor(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
 290 }
 291
 292 #  define __cxx_atomic_is_lock_free(__s) __atomic_is_lock_free(__s, 0)
 293
 294 #elif defined(_LIBCPP_HAS_C_ATOMIC_IMP)
 295
 296 template <typename _Tp>
 297 struct __cxx_atomic_base_impl {
 298   _LIBCPP_HIDE_FROM_ABI
 299 #  ifndef _LIBCPP_CXX03_LANG
 300   __cxx_atomic_base_impl() _NOEXCEPT = default;
 301 #  else
 302   __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {
 303   }
 304 #  endif // _LIBCPP_CXX03_LANG
 305   _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp __value) _NOEXCEPT : __a_value(__value) {}
 306   _LIBCPP_DISABLE_EXTENSION_WARNING _Atomic(_Tp) __a_value;
 307 };
 308
 309 #  define __cxx_atomic_is_lock_free(__s) __c11_atomic_is_lock_free(__s)
 310
 311 _LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_thread_fence(memory_order __order) _NOEXCEPT {
 312   __c11_atomic_thread_fence(static_cast<__memory_order_underlying_t>(__order));
 313 }
 314
 315 _LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_signal_fence(memory_order __order) _NOEXCEPT {
 316   __c11_atomic_signal_fence(static_cast<__memory_order_underlying_t>(__order));
 317 }
 318
 319 template <class _Tp>
 320 _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val) _NOEXCEPT {
 321   __c11_atomic_init(std::addressof(__a->__a_value), __val);
 322 }
 323 template <class _Tp>
 324 _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) _NOEXCEPT {
 325   __c11_atomic_init(std::addressof(__a->__a_value), __val);
 326 }
 327
 328 template <class _Tp>
 329 _LIBCPP_HIDE_FROM_ABI void
 330 __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT {
 331   __c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
 332 }
 333 template <class _Tp>
 334 _LIBCPP_HIDE_FROM_ABI void
 335 __cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) _NOEXCEPT {
 336   __c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
 337 }
 338
 339 template <class _Tp>
 340 _LIBCPP_HIDE_FROM_ABI _Tp
 341 __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const volatile* __a, memory_order __order) _NOEXCEPT {
 342   using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
 343   return __c11_atomic_load(
 344       const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
 345 }
 346 template <class _Tp>
 347 _LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const* __a, memory_order __order) _NOEXCEPT {
 348   using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
 349   return __c11_atomic_load(
 350       const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
 351 }
 352
 353 template <class _Tp>
 354 _LIBCPP_HIDE_FROM_ABI void
 355 __cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const volatile* __a, _Tp* __dst, memory_order __order) _NOEXCEPT {
 356   using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
 357   *__dst           = __c11_atomic_load(
 358       const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
 359 }
 360 template <class _Tp>
 361 _LIBCPP_HIDE_FROM_ABI void
 362 __cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const* __a, _Tp* __dst, memory_order __order) _NOEXCEPT {
 363   using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
 364   *__dst           = __c11_atomic_load(
 365       const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
 366 }
 367
 368 template <class _Tp>
 369 _LIBCPP_HIDE_FROM_ABI _Tp
 370 __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT {
 371   return __c11_atomic_exchange(
 372       std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order));
 373 }
 374 template <class _Tp>
 375 _LIBCPP_HIDE_FROM_ABI _Tp
 376 __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) _NOEXCEPT {
 377   return __c11_atomic_exchange(
 378       std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order));
 379 }
 380
 381 _LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR memory_order __to_failure_order(memory_order __order) {
 382   // Avoid switch statement to make this a constexpr.
 383   return __order == memory_order_release
 384            ? memory_order_relaxed
 385            : (__order == memory_order_acq_rel ? memory_order_acquire : __order);
 386 }
 387
 388 template <class _Tp>
 389 _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
 390     __cxx_atomic_base_impl<_Tp> volatile* __a,
 391     _Tp* __expected,
 392     _Tp __value,
 393     memory_order __success,
 394     memory_order __failure) _NOEXCEPT {
 395   return __c11_atomic_compare_exchange_strong(
 396       std::addressof(__a->__a_value),
 397       __expected,
 398       __value,
 399       static_cast<__memory_order_underlying_t>(__success),
 400       static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
 401 }
 402 template <class _Tp>
 403 _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
 404     __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure)
 405     _NOEXCEPT {
 406   return __c11_atomic_compare_exchange_strong(
 407       std::addressof(__a->__a_value),
 408       __expected,
 409       __value,
 410       static_cast<__memory_order_underlying_t>(__success),
 411       static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
 412 }
 413
 414 template <class _Tp>
 415 _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
 416     __cxx_atomic_base_impl<_Tp> volatile* __a,
 417     _Tp* __expected,
 418     _Tp __value,
 419     memory_order __success,
 420     memory_order __failure) _NOEXCEPT {
 421   return __c11_atomic_compare_exchange_weak(
 422       std::addressof(__a->__a_value),
 423       __expected,
 424       __value,
 425       static_cast<__memory_order_underlying_t>(__success),
 426       static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
 427 }
 428 template <class _Tp>
 429 _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
 430     __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure)
 431     _NOEXCEPT {
 432   return __c11_atomic_compare_exchange_weak(
 433       std::addressof(__a->__a_value),
 434       __expected,
 435       __value,
 436       static_cast<__memory_order_underlying_t>(__success),
 437       static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
 438 }
 439
 440 template <class _Tp>
 441 _LIBCPP_HIDE_FROM_ABI _Tp
 442 __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
 443   return __c11_atomic_fetch_add(
 444       std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
 445 }
 446 template <class _Tp>
 447 _LIBCPP_HIDE_FROM_ABI _Tp
 448 __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
 449   return __c11_atomic_fetch_add(
 450       std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
 451 }
 452
 453 template <class _Tp>
 454 _LIBCPP_HIDE_FROM_ABI _Tp*
 455 __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
 456   return __c11_atomic_fetch_add(
 457       std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
 458 }
 459 template <class _Tp>
 460 _LIBCPP_HIDE_FROM_ABI _Tp*
 461 __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*>* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
 462   return __c11_atomic_fetch_add(
 463       std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
 464 }
 465
 466 template <class _Tp>
 467 _LIBCPP_HIDE_FROM_ABI _Tp
 468 __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
 469   return __c11_atomic_fetch_sub(
 470       std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
 471 }
 472 template <class _Tp>
 473 _LIBCPP_HIDE_FROM_ABI _Tp
 474 __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
 475   return __c11_atomic_fetch_sub(
 476       std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
 477 }
 478 template <class _Tp>
 479 _LIBCPP_HIDE_FROM_ABI _Tp*
 480 __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
 481   return __c11_atomic_fetch_sub(
 482       std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
 483 }
 484 template <class _Tp>
 485 _LIBCPP_HIDE_FROM_ABI _Tp*
 486 __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*>* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
 487   return __c11_atomic_fetch_sub(
 488       std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
 489 }
 490
 491 template <class _Tp>
 492 _LIBCPP_HIDE_FROM_ABI _Tp
 493 __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
 494   return __c11_atomic_fetch_and(
 495       std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
 496 }
 497 template <class _Tp>
 498 _LIBCPP_HIDE_FROM_ABI _Tp
 499 __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
 500   return __c11_atomic_fetch_and(
 501       std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
 502 }
 503
 504 template <class _Tp>
 505 _LIBCPP_HIDE_FROM_ABI _Tp
 506 __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
 507   return __c11_atomic_fetch_or(
 508       std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
 509 }
 510 template <class _Tp>
 511 _LIBCPP_HIDE_FROM_ABI _Tp
 512 __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
 513   return __c11_atomic_fetch_or(
 514       std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
 515 }
 516
 517 template <class _Tp>
 518 _LIBCPP_HIDE_FROM_ABI _Tp
 519 __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
 520   return __c11_atomic_fetch_xor(
 521       std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
 522 }
 523 template <class _Tp>
 524 _LIBCPP_HIDE_FROM_ABI _Tp
 525 __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
 526   return __c11_atomic_fetch_xor(
 527       std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
 528 }
 529
 530 #endif // _LIBCPP_HAS_GCC_ATOMIC_IMP, _LIBCPP_HAS_C_ATOMIC_IMP
 531
 532 #ifdef _LIBCPP_ATOMIC_ONLY_USE_BUILTINS
 533
 534 template <typename _Tp>
 535 struct __cxx_atomic_lock_impl {
 536   _LIBCPP_HIDE_FROM_ABI __cxx_atomic_lock_impl() _NOEXCEPT : __a_value(), __a_lock(0) {}
 537   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __cxx_atomic_lock_impl(_Tp value) _NOEXCEPT
 538       : __a_value(value),
 539         __a_lock(0) {}
 540
 541   _Tp __a_value;
 542   mutable __cxx_atomic_base_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_lock;
 543
 544   _LIBCPP_HIDE_FROM_ABI void __lock() const volatile {
 545     while (1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire))
 546       /*spin*/;
 547   }
 548   _LIBCPP_HIDE_FROM_ABI void __lock() const {
 549     while (1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire))
 550       /*spin*/;
 551   }
 552   _LIBCPP_HIDE_FROM_ABI void __unlock() const volatile {
 553     __cxx_atomic_store(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(false), memory_order_release);
 554   }
 555   _LIBCPP_HIDE_FROM_ABI void __unlock() const {
 556     __cxx_atomic_store(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(false), memory_order_release);
 557   }
 558   _LIBCPP_HIDE_FROM_ABI _Tp __read() const volatile {
 559     __lock();
 560     _Tp __old;
 561     __cxx_atomic_assign_volatile(__old, __a_value);
 562     __unlock();
 563     return __old;
 564   }
 565   _LIBCPP_HIDE_FROM_ABI _Tp __read() const {
 566     __lock();
 567     _Tp __old = __a_value;
 568     __unlock();
 569     return __old;
 570   }
 571   _LIBCPP_HIDE_FROM_ABI void __read_inplace(_Tp* __dst) const volatile {
 572     __lock();
 573     __cxx_atomic_assign_volatile(*__dst, __a_value);
 574     __unlock();
 575   }
 576   _LIBCPP_HIDE_FROM_ABI void __read_inplace(_Tp* __dst) const {
 577     __lock();
 578     *__dst = __a_value;
 579     __unlock();
 580   }
 581 };
 582
 583 template <typename _Tp>
 584 _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __val) {
 585   __cxx_atomic_assign_volatile(__a->__a_value, __val);
 586 }
 587 template <typename _Tp>
 588 _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __val) {
 589   __a->__a_value = __val;
 590 }
 591
 592 template <typename _Tp>
 593 _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_store(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __val, memory_order) {
 594   __a->__lock();
 595   __cxx_atomic_assign_volatile(__a->__a_value, __val);
 596   __a->__unlock();
 597 }
 598 template <typename _Tp>
 599 _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_store(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __val, memory_order) {
 600   __a->__lock();
 601   __a->__a_value = __val;
 602   __a->__unlock();
 603 }
 604
 605 template <typename _Tp>
 606 _LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(const volatile __cxx_atomic_lock_impl<_Tp>* __a, memory_order) {
 607   return __a->__read();
 608 }
 609 template <typename _Tp>
 610 _LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(const __cxx_atomic_lock_impl<_Tp>* __a, memory_order) {
 611   return __a->__read();
 612 }
 613
 614 template <typename _Tp>
 615 _LIBCPP_HIDE_FROM_ABI void
 616 __cxx_atomic_load(const volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp* __dst, memory_order) {
 617   __a->__read_inplace(__dst);
 618 }
 619 template <typename _Tp>
 620 _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_load(const __cxx_atomic_lock_impl<_Tp>* __a, _Tp* __dst, memory_order) {
 621   __a->__read_inplace(__dst);
 622 }
 623
 624 template <typename _Tp>
 625 _LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_exchange(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) {
 626   __a->__lock();
 627   _Tp __old;
 628   __cxx_atomic_assign_volatile(__old, __a->__a_value);
 629   __cxx_atomic_assign_volatile(__a->__a_value, __value);
 630   __a->__unlock();
 631   return __old;
 632 }
 633 template <typename _Tp>
 634 _LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_exchange(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) {
 635   __a->__lock();
 636   _Tp __old      = __a->__a_value;
 637   __a->__a_value = __value;
 638   __a->__unlock();
 639   return __old;
 640 }
 641
 642 template <typename _Tp>
 643 _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
 644     volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order, memory_order) {
 645   _Tp __temp;
 646   __a->__lock();
 647   __cxx_atomic_assign_volatile(__temp, __a->__a_value);
 648   bool __ret = (std::memcmp(&__temp, __expected, sizeof(_Tp)) == 0);
 649   if (__ret)
 650     __cxx_atomic_assign_volatile(__a->__a_value, __value);
 651   else
 652     __cxx_atomic_assign_volatile(*__expected, __a->__a_value);
 653   __a->__unlock();
 654   return __ret;
 655 }
 656 template <typename _Tp>
 657 _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
 658     __cxx_atomic_lock_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order, memory_order) {
 659   __a->__lock();
 660   bool __ret = (std::memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0);
 661   if (__ret)
 662     std::memcpy(&__a->__a_value, &__value, sizeof(_Tp));
 663   else
 664     std::memcpy(__expected, &__a->__a_value, sizeof(_Tp));
 665   __a->__unlock();
 666   return __ret;
 667 }
 668
 669 template <typename _Tp>
 670 _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
 671     volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order, memory_order) {
 672   _Tp __temp;
 673   __a->__lock();
 674   __cxx_atomic_assign_volatile(__temp, __a->__a_value);
 675   bool __ret = (std::memcmp(&__temp, __expected, sizeof(_Tp)) == 0);
 676   if (__ret)
 677     __cxx_atomic_assign_volatile(__a->__a_value, __value);
 678   else
 679     __cxx_atomic_assign_volatile(*__expected, __a->__a_value);
 680   __a->__unlock();
 681   return __ret;
 682 }
 683 template <typename _Tp>
 684 _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
 685     __cxx_atomic_lock_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order, memory_order) {
 686   __a->__lock();
 687   bool __ret = (std::memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0);
 688   if (__ret)
 689     std::memcpy(&__a->__a_value, &__value, sizeof(_Tp));
 690   else
 691     std::memcpy(__expected, &__a->__a_value, sizeof(_Tp));
 692   __a->__unlock();
 693   return __ret;
 694 }
 695
 696 template <typename _Tp, typename _Td>
 697 _LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Td __delta, memory_order) {
 698   __a->__lock();
 699   _Tp __old;
 700   __cxx_atomic_assign_volatile(__old, __a->__a_value);
 701   __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old + __delta));
 702   __a->__unlock();
 703   return __old;
 704 }
 705 template <typename _Tp, typename _Td>
 706 _LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp>* __a, _Td __delta, memory_order) {
 707   __a->__lock();
 708   _Tp __old = __a->__a_value;
 709   __a->__a_value += __delta;
 710   __a->__unlock();
 711   return __old;
 712 }
 713
 714 template <typename _Tp, typename _Td>
 715 _LIBCPP_HIDE_FROM_ABI _Tp*
 716 __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp*>* __a, ptrdiff_t __delta, memory_order) {
 717   __a->__lock();
 718   _Tp* __old;
 719   __cxx_atomic_assign_volatile(__old, __a->__a_value);
 720   __cxx_atomic_assign_volatile(__a->__a_value, __old + __delta);
 721   __a->__unlock();
 722   return __old;
 723 }
 724 template <typename _Tp, typename _Td>
 725 _LIBCPP_HIDE_FROM_ABI _Tp* __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp*>* __a, ptrdiff_t __delta, memory_order) {
 726   __a->__lock();
 727   _Tp* __old = __a->__a_value;
 728   __a->__a_value += __delta;
 729   __a->__unlock();
 730   return __old;
 731 }
 732
 733 template <typename _Tp, typename _Td>
 734 _LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Td __delta, memory_order) {
 735   __a->__lock();
 736   _Tp __old;
 737   __cxx_atomic_assign_volatile(__old, __a->__a_value);
 738   __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old - __delta));
 739   __a->__unlock();
 740   return __old;
 741 }
 742 template <typename _Tp, typename _Td>
 743 _LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_sub(__cxx_atomic_lock_impl<_Tp>* __a, _Td __delta, memory_order) {
 744   __a->__lock();
 745   _Tp __old = __a->__a_value;
 746   __a->__a_value -= __delta;
 747   __a->__unlock();
 748   return __old;
 749 }
 750
 751 template <typename _Tp>
 752 _LIBCPP_HIDE_FROM_ABI _Tp
 753 __cxx_atomic_fetch_and(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __pattern, memory_order) {
 754   __a->__lock();
 755   _Tp __old;
 756   __cxx_atomic_assign_volatile(__old, __a->__a_value);
 757   __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old & __pattern));
 758   __a->__unlock();
 759   return __old;
 760 }
 761 template <typename _Tp>
 762 _LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_and(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __pattern, memory_order) {
 763   __a->__lock();
 764   _Tp __old = __a->__a_value;
 765   __a->__a_value &= __pattern;
 766   __a->__unlock();
 767   return __old;
 768 }
 769
 770 template <typename _Tp>
 771 _LIBCPP_HIDE_FROM_ABI _Tp
 772 __cxx_atomic_fetch_or(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __pattern, memory_order) {
 773   __a->__lock();
 774   _Tp __old;
 775   __cxx_atomic_assign_volatile(__old, __a->__a_value);
 776   __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old | __pattern));
 777   __a->__unlock();
 778   return __old;
 779 }
 780 template <typename _Tp>
 781 _LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_or(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __pattern, memory_order) {
 782   __a->__lock();
 783   _Tp __old = __a->__a_value;
 784   __a->__a_value |= __pattern;
 785   __a->__unlock();
 786   return __old;
 787 }
 788
 789 template <typename _Tp>
 790 _LIBCPP_HIDE_FROM_ABI _Tp
 791 __cxx_atomic_fetch_xor(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __pattern, memory_order) {
 792   __a->__lock();
 793   _Tp __old;
 794   __cxx_atomic_assign_volatile(__old, __a->__a_value);
 795   __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old ^ __pattern));
 796   __a->__unlock();
 797   return __old;
 798 }
 799 template <typename _Tp>
 800 _LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_xor(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __pattern, memory_order) {
 801   __a->__lock();
 802   _Tp __old = __a->__a_value;
 803   __a->__a_value ^= __pattern;
 804   __a->__unlock();
 805   return __old;
 806 }
 807
 808 template <typename _Tp,
 809           typename _Base = typename conditional<__libcpp_is_always_lock_free<_Tp>::__value,
 810                                                 __cxx_atomic_base_impl<_Tp>,
 811                                                 __cxx_atomic_lock_impl<_Tp> >::type>
 812 #else
 813 template <typename _Tp, typename _Base = __cxx_atomic_base_impl<_Tp> >
 814 #endif //_LIBCPP_ATOMIC_ONLY_USE_BUILTINS
 815 struct __cxx_atomic_impl : public _Base {
 816   static_assert(is_trivially_copyable<_Tp>::value, "std::atomic<T> requires that 'T' be a trivially copyable type");
 817
 818   _LIBCPP_HIDE_FROM_ABI __cxx_atomic_impl() _NOEXCEPT = default;
 819   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __cxx_atomic_impl(_Tp __value) _NOEXCEPT : _Base(__value) {}
 820 };
 821
 822 _LIBCPP_END_NAMESPACE_STD
 823
 824 #endif // _LIBCPP___ATOMIC_CXX_ATOMIC_IMPL_H