1// -*- C++ -*- 2//===----------------------------------------------------------------------===// 3// 4// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 5// See https://llvm.org/LICENSE.txt for license information. 6// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 7// 8//===----------------------------------------------------------------------===// 9 10#ifndef _LIBCPP_ATOMIC 11#define _LIBCPP_ATOMIC 12 13/* 14 atomic synopsis 15 16namespace std 17{ 18 19// feature test macro [version.syn] 20 21#define __cpp_lib_atomic_is_always_lock_free 22#define __cpp_lib_atomic_flag_test 23#define __cpp_lib_atomic_lock_free_type_aliases 24#define __cpp_lib_atomic_wait 25 26 // order and consistency 27 28 enum memory_order: unspecified // enum class in C++20 29 { 30 relaxed, 31 consume, // load-consume 32 acquire, // load-acquire 33 release, // store-release 34 acq_rel, // store-release load-acquire 35 seq_cst // store-release load-acquire 36 }; 37 38 inline constexpr auto memory_order_relaxed = memory_order::relaxed; 39 inline constexpr auto memory_order_consume = memory_order::consume; 40 inline constexpr auto memory_order_acquire = memory_order::acquire; 41 inline constexpr auto memory_order_release = memory_order::release; 42 inline constexpr auto memory_order_acq_rel = memory_order::acq_rel; 43 inline constexpr auto memory_order_seq_cst = memory_order::seq_cst; 44 45template <class T> T kill_dependency(T y) noexcept; 46 47// lock-free property 48 49#define ATOMIC_BOOL_LOCK_FREE unspecified 50#define ATOMIC_CHAR_LOCK_FREE unspecified 51#define ATOMIC_CHAR8_T_LOCK_FREE unspecified // C++20 52#define ATOMIC_CHAR16_T_LOCK_FREE unspecified 53#define ATOMIC_CHAR32_T_LOCK_FREE unspecified 54#define ATOMIC_WCHAR_T_LOCK_FREE unspecified 55#define ATOMIC_SHORT_LOCK_FREE unspecified 56#define ATOMIC_INT_LOCK_FREE unspecified 57#define ATOMIC_LONG_LOCK_FREE unspecified 58#define ATOMIC_LLONG_LOCK_FREE unspecified 59#define ATOMIC_POINTER_LOCK_FREE unspecified 60 61template <class T> 62struct atomic 63{ 64 using value_type = T; 65 66 static constexpr bool is_always_lock_free; 67 bool is_lock_free() const volatile noexcept; 68 bool is_lock_free() const noexcept; 69 70 atomic() noexcept = default; // until C++20 71 constexpr atomic() noexcept(is_nothrow_default_constructible_v<T>); // since C++20 72 constexpr atomic(T desr) noexcept; 73 atomic(const atomic&) = delete; 74 atomic& operator=(const atomic&) = delete; 75 atomic& operator=(const atomic&) volatile = delete; 76 77 T load(memory_order m = memory_order_seq_cst) const volatile noexcept; 78 T load(memory_order m = memory_order_seq_cst) const noexcept; 79 operator T() const volatile noexcept; 80 operator T() const noexcept; 81 void store(T desr, memory_order m = memory_order_seq_cst) volatile noexcept; 82 void store(T desr, memory_order m = memory_order_seq_cst) noexcept; 83 T operator=(T) volatile noexcept; 84 T operator=(T) noexcept; 85 86 T exchange(T desr, memory_order m = memory_order_seq_cst) volatile noexcept; 87 T exchange(T desr, memory_order m = memory_order_seq_cst) noexcept; 88 bool compare_exchange_weak(T& expc, T desr, 89 memory_order s, memory_order f) volatile noexcept; 90 bool compare_exchange_weak(T& expc, T desr, memory_order s, memory_order f) noexcept; 91 bool compare_exchange_strong(T& expc, T desr, 92 memory_order s, memory_order f) volatile noexcept; 93 bool compare_exchange_strong(T& expc, T desr, 94 memory_order s, memory_order f) noexcept; 95 bool compare_exchange_weak(T& expc, T desr, 96 memory_order m = memory_order_seq_cst) volatile noexcept; 97 bool compare_exchange_weak(T& expc, T desr, 98 memory_order m = memory_order_seq_cst) noexcept; 99 bool compare_exchange_strong(T& expc, T desr, 100 memory_order m = memory_order_seq_cst) volatile noexcept; 101 bool compare_exchange_strong(T& expc, T desr, 102 memory_order m = memory_order_seq_cst) noexcept; 103 104 void wait(T, memory_order = memory_order::seq_cst) const volatile noexcept; 105 void wait(T, memory_order = memory_order::seq_cst) const noexcept; 106 void notify_one() volatile noexcept; 107 void notify_one() noexcept; 108 void notify_all() volatile noexcept; 109 void notify_all() noexcept; 110}; 111 112template <> 113struct atomic<integral> 114{ 115 using value_type = integral; 116 using difference_type = value_type; 117 118 static constexpr bool is_always_lock_free; 119 bool is_lock_free() const volatile noexcept; 120 bool is_lock_free() const noexcept; 121 122 atomic() noexcept = default; 123 constexpr atomic(integral desr) noexcept; 124 atomic(const atomic&) = delete; 125 atomic& operator=(const atomic&) = delete; 126 atomic& operator=(const atomic&) volatile = delete; 127 128 integral load(memory_order m = memory_order_seq_cst) const volatile noexcept; 129 integral load(memory_order m = memory_order_seq_cst) const noexcept; 130 operator integral() const volatile noexcept; 131 operator integral() const noexcept; 132 void store(integral desr, memory_order m = memory_order_seq_cst) volatile noexcept; 133 void store(integral desr, memory_order m = memory_order_seq_cst) noexcept; 134 integral operator=(integral desr) volatile noexcept; 135 integral operator=(integral desr) noexcept; 136 137 integral exchange(integral desr, 138 memory_order m = memory_order_seq_cst) volatile noexcept; 139 integral exchange(integral desr, memory_order m = memory_order_seq_cst) noexcept; 140 bool compare_exchange_weak(integral& expc, integral desr, 141 memory_order s, memory_order f) volatile noexcept; 142 bool compare_exchange_weak(integral& expc, integral desr, 143 memory_order s, memory_order f) noexcept; 144 bool compare_exchange_strong(integral& expc, integral desr, 145 memory_order s, memory_order f) volatile noexcept; 146 bool compare_exchange_strong(integral& expc, integral desr, 147 memory_order s, memory_order f) noexcept; 148 bool compare_exchange_weak(integral& expc, integral desr, 149 memory_order m = memory_order_seq_cst) volatile noexcept; 150 bool compare_exchange_weak(integral& expc, integral desr, 151 memory_order m = memory_order_seq_cst) noexcept; 152 bool compare_exchange_strong(integral& expc, integral desr, 153 memory_order m = memory_order_seq_cst) volatile noexcept; 154 bool compare_exchange_strong(integral& expc, integral desr, 155 memory_order m = memory_order_seq_cst) noexcept; 156 157 integral fetch_add(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 158 integral fetch_add(integral op, memory_order m = memory_order_seq_cst) noexcept; 159 integral fetch_sub(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 160 integral fetch_sub(integral op, memory_order m = memory_order_seq_cst) noexcept; 161 integral fetch_and(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 162 integral fetch_and(integral op, memory_order m = memory_order_seq_cst) noexcept; 163 integral fetch_or(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 164 integral fetch_or(integral op, memory_order m = memory_order_seq_cst) noexcept; 165 integral fetch_xor(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 166 integral fetch_xor(integral op, memory_order m = memory_order_seq_cst) noexcept; 167 168 integral operator++(int) volatile noexcept; 169 integral operator++(int) noexcept; 170 integral operator--(int) volatile noexcept; 171 integral operator--(int) noexcept; 172 integral operator++() volatile noexcept; 173 integral operator++() noexcept; 174 integral operator--() volatile noexcept; 175 integral operator--() noexcept; 176 integral operator+=(integral op) volatile noexcept; 177 integral operator+=(integral op) noexcept; 178 integral operator-=(integral op) volatile noexcept; 179 integral operator-=(integral op) noexcept; 180 integral operator&=(integral op) volatile noexcept; 181 integral operator&=(integral op) noexcept; 182 integral operator|=(integral op) volatile noexcept; 183 integral operator|=(integral op) noexcept; 184 integral operator^=(integral op) volatile noexcept; 185 integral operator^=(integral op) noexcept; 186 187 void wait(integral, memory_order = memory_order::seq_cst) const volatile noexcept; 188 void wait(integral, memory_order = memory_order::seq_cst) const noexcept; 189 void notify_one() volatile noexcept; 190 void notify_one() noexcept; 191 void notify_all() volatile noexcept; 192 void notify_all() noexcept; 193}; 194 195template <class T> 196struct atomic<T*> 197{ 198 using value_type = T*; 199 using difference_type = ptrdiff_t; 200 201 static constexpr bool is_always_lock_free; 202 bool is_lock_free() const volatile noexcept; 203 bool is_lock_free() const noexcept; 204 205 atomic() noexcept = default; // until C++20 206 constexpr atomic() noexcept; // since C++20 207 constexpr atomic(T* desr) noexcept; 208 atomic(const atomic&) = delete; 209 atomic& operator=(const atomic&) = delete; 210 atomic& operator=(const atomic&) volatile = delete; 211 212 T* load(memory_order m = memory_order_seq_cst) const volatile noexcept; 213 T* load(memory_order m = memory_order_seq_cst) const noexcept; 214 operator T*() const volatile noexcept; 215 operator T*() const noexcept; 216 void store(T* desr, memory_order m = memory_order_seq_cst) volatile noexcept; 217 void store(T* desr, memory_order m = memory_order_seq_cst) noexcept; 218 T* operator=(T*) volatile noexcept; 219 T* operator=(T*) noexcept; 220 221 T* exchange(T* desr, memory_order m = memory_order_seq_cst) volatile noexcept; 222 T* exchange(T* desr, memory_order m = memory_order_seq_cst) noexcept; 223 bool compare_exchange_weak(T*& expc, T* desr, 224 memory_order s, memory_order f) volatile noexcept; 225 bool compare_exchange_weak(T*& expc, T* desr, 226 memory_order s, memory_order f) noexcept; 227 bool compare_exchange_strong(T*& expc, T* desr, 228 memory_order s, memory_order f) volatile noexcept; 229 bool compare_exchange_strong(T*& expc, T* desr, 230 memory_order s, memory_order f) noexcept; 231 bool compare_exchange_weak(T*& expc, T* desr, 232 memory_order m = memory_order_seq_cst) volatile noexcept; 233 bool compare_exchange_weak(T*& expc, T* desr, 234 memory_order m = memory_order_seq_cst) noexcept; 235 bool compare_exchange_strong(T*& expc, T* desr, 236 memory_order m = memory_order_seq_cst) volatile noexcept; 237 bool compare_exchange_strong(T*& expc, T* desr, 238 memory_order m = memory_order_seq_cst) noexcept; 239 T* fetch_add(ptrdiff_t op, memory_order m = memory_order_seq_cst) volatile noexcept; 240 T* fetch_add(ptrdiff_t op, memory_order m = memory_order_seq_cst) noexcept; 241 T* fetch_sub(ptrdiff_t op, memory_order m = memory_order_seq_cst) volatile noexcept; 242 T* fetch_sub(ptrdiff_t op, memory_order m = memory_order_seq_cst) noexcept; 243 244 T* operator++(int) volatile noexcept; 245 T* operator++(int) noexcept; 246 T* operator--(int) volatile noexcept; 247 T* operator--(int) noexcept; 248 T* operator++() volatile noexcept; 249 T* operator++() noexcept; 250 T* operator--() volatile noexcept; 251 T* operator--() noexcept; 252 T* operator+=(ptrdiff_t op) volatile noexcept; 253 T* operator+=(ptrdiff_t op) noexcept; 254 T* operator-=(ptrdiff_t op) volatile noexcept; 255 T* operator-=(ptrdiff_t op) noexcept; 256 257 void wait(T*, memory_order = memory_order::seq_cst) const volatile noexcept; 258 void wait(T*, memory_order = memory_order::seq_cst) const noexcept; 259 void notify_one() volatile noexcept; 260 void notify_one() noexcept; 261 void notify_all() volatile noexcept; 262 void notify_all() noexcept; 263}; 264 265 266// [atomics.nonmembers], non-member functions 267template<class T> 268 bool atomic_is_lock_free(const volatile atomic<T>*) noexcept; 269template<class T> 270 bool atomic_is_lock_free(const atomic<T>*) noexcept; 271template<class T> 272 void atomic_store(volatile atomic<T>*, atomic<T>::value_type) noexcept; 273template<class T> 274 void atomic_store(atomic<T>*, atomic<T>::value_type) noexcept; 275template<class T> 276 void atomic_store_explicit(volatile atomic<T>*, atomic<T>::value_type, 277 memory_order) noexcept; 278template<class T> 279 void atomic_store_explicit(atomic<T>*, atomic<T>::value_type, 280 memory_order) noexcept; 281template<class T> 282 T atomic_load(const volatile atomic<T>*) noexcept; 283template<class T> 284 T atomic_load(const atomic<T>*) noexcept; 285template<class T> 286 T atomic_load_explicit(const volatile atomic<T>*, memory_order) noexcept; 287template<class T> 288 T atomic_load_explicit(const atomic<T>*, memory_order) noexcept; 289template<class T> 290 T atomic_exchange(volatile atomic<T>*, atomic<T>::value_type) noexcept; 291template<class T> 292 T atomic_exchange(atomic<T>*, atomic<T>::value_type) noexcept; 293template<class T> 294 T atomic_exchange_explicit(volatile atomic<T>*, atomic<T>::value_type, 295 memory_order) noexcept; 296template<class T> 297 T atomic_exchange_explicit(atomic<T>*, atomic<T>::value_type, 298 memory_order) noexcept; 299template<class T> 300 bool atomic_compare_exchange_weak(volatile atomic<T>*, atomic<T>::value_type*, 301 atomic<T>::value_type) noexcept; 302template<class T> 303 bool atomic_compare_exchange_weak(atomic<T>*, atomic<T>::value_type*, 304 atomic<T>::value_type) noexcept; 305template<class T> 306 bool atomic_compare_exchange_strong(volatile atomic<T>*, atomic<T>::value_type*, 307 atomic<T>::value_type) noexcept; 308template<class T> 309 bool atomic_compare_exchange_strong(atomic<T>*, atomic<T>::value_type*, 310 atomic<T>::value_type) noexcept; 311template<class T> 312 bool atomic_compare_exchange_weak_explicit(volatile atomic<T>*, atomic<T>::value_type*, 313 atomic<T>::value_type, 314 memory_order, memory_order) noexcept; 315template<class T> 316 bool atomic_compare_exchange_weak_explicit(atomic<T>*, atomic<T>::value_type*, 317 atomic<T>::value_type, 318 memory_order, memory_order) noexcept; 319template<class T> 320 bool atomic_compare_exchange_strong_explicit(volatile atomic<T>*, atomic<T>::value_type*, 321 atomic<T>::value_type, 322 memory_order, memory_order) noexcept; 323template<class T> 324 bool atomic_compare_exchange_strong_explicit(atomic<T>*, atomic<T>::value_type*, 325 atomic<T>::value_type, 326 memory_order, memory_order) noexcept; 327 328template<class T> 329 T atomic_fetch_add(volatile atomic<T>*, atomic<T>::difference_type) noexcept; 330template<class T> 331 T atomic_fetch_add(atomic<T>*, atomic<T>::difference_type) noexcept; 332template<class T> 333 T atomic_fetch_add_explicit(volatile atomic<T>*, atomic<T>::difference_type, 334 memory_order) noexcept; 335template<class T> 336 T atomic_fetch_add_explicit(atomic<T>*, atomic<T>::difference_type, 337 memory_order) noexcept; 338template<class T> 339 T atomic_fetch_sub(volatile atomic<T>*, atomic<T>::difference_type) noexcept; 340template<class T> 341 T atomic_fetch_sub(atomic<T>*, atomic<T>::difference_type) noexcept; 342template<class T> 343 T atomic_fetch_sub_explicit(volatile atomic<T>*, atomic<T>::difference_type, 344 memory_order) noexcept; 345template<class T> 346 T atomic_fetch_sub_explicit(atomic<T>*, atomic<T>::difference_type, 347 memory_order) noexcept; 348template<class T> 349 T atomic_fetch_and(volatile atomic<T>*, atomic<T>::value_type) noexcept; 350template<class T> 351 T atomic_fetch_and(atomic<T>*, atomic<T>::value_type) noexcept; 352template<class T> 353 T atomic_fetch_and_explicit(volatile atomic<T>*, atomic<T>::value_type, 354 memory_order) noexcept; 355template<class T> 356 T atomic_fetch_and_explicit(atomic<T>*, atomic<T>::value_type, 357 memory_order) noexcept; 358template<class T> 359 T atomic_fetch_or(volatile atomic<T>*, atomic<T>::value_type) noexcept; 360template<class T> 361 T atomic_fetch_or(atomic<T>*, atomic<T>::value_type) noexcept; 362template<class T> 363 T atomic_fetch_or_explicit(volatile atomic<T>*, atomic<T>::value_type, 364 memory_order) noexcept; 365template<class T> 366 T atomic_fetch_or_explicit(atomic<T>*, atomic<T>::value_type, 367 memory_order) noexcept; 368template<class T> 369 T atomic_fetch_xor(volatile atomic<T>*, atomic<T>::value_type) noexcept; 370template<class T> 371 T atomic_fetch_xor(atomic<T>*, atomic<T>::value_type) noexcept; 372template<class T> 373 T atomic_fetch_xor_explicit(volatile atomic<T>*, atomic<T>::value_type, 374 memory_order) noexcept; 375template<class T> 376 T atomic_fetch_xor_explicit(atomic<T>*, atomic<T>::value_type, 377 memory_order) noexcept; 378 379template<class T> 380 void atomic_wait(const volatile atomic<T>*, atomic<T>::value_type); 381template<class T> 382 void atomic_wait(const atomic<T>*, atomic<T>::value_type); 383template<class T> 384 void atomic_wait_explicit(const volatile atomic<T>*, atomic<T>::value_type, 385 memory_order); 386template<class T> 387 void atomic_wait_explicit(const atomic<T>*, atomic<T>::value_type, 388 memory_order); 389template<class T> 390 void atomic_notify_one(volatile atomic<T>*); 391template<class T> 392 void atomic_notify_one(atomic<T>*); 393template<class T> 394 void atomic_notify_all(volatile atomic<T>*); 395template<class T> 396 void atomic_notify_all(atomic<T>*); 397 398// Atomics for standard typedef types 399 400typedef atomic<bool> atomic_bool; 401typedef atomic<char> atomic_char; 402typedef atomic<signed char> atomic_schar; 403typedef atomic<unsigned char> atomic_uchar; 404typedef atomic<short> atomic_short; 405typedef atomic<unsigned short> atomic_ushort; 406typedef atomic<int> atomic_int; 407typedef atomic<unsigned int> atomic_uint; 408typedef atomic<long> atomic_long; 409typedef atomic<unsigned long> atomic_ulong; 410typedef atomic<long long> atomic_llong; 411typedef atomic<unsigned long long> atomic_ullong; 412typedef atomic<char8_t> atomic_char8_t; // C++20 413typedef atomic<char16_t> atomic_char16_t; 414typedef atomic<char32_t> atomic_char32_t; 415typedef atomic<wchar_t> atomic_wchar_t; 416 417typedef atomic<int_least8_t> atomic_int_least8_t; 418typedef atomic<uint_least8_t> atomic_uint_least8_t; 419typedef atomic<int_least16_t> atomic_int_least16_t; 420typedef atomic<uint_least16_t> atomic_uint_least16_t; 421typedef atomic<int_least32_t> atomic_int_least32_t; 422typedef atomic<uint_least32_t> atomic_uint_least32_t; 423typedef atomic<int_least64_t> atomic_int_least64_t; 424typedef atomic<uint_least64_t> atomic_uint_least64_t; 425 426typedef atomic<int_fast8_t> atomic_int_fast8_t; 427typedef atomic<uint_fast8_t> atomic_uint_fast8_t; 428typedef atomic<int_fast16_t> atomic_int_fast16_t; 429typedef atomic<uint_fast16_t> atomic_uint_fast16_t; 430typedef atomic<int_fast32_t> atomic_int_fast32_t; 431typedef atomic<uint_fast32_t> atomic_uint_fast32_t; 432typedef atomic<int_fast64_t> atomic_int_fast64_t; 433typedef atomic<uint_fast64_t> atomic_uint_fast64_t; 434 435typedef atomic<int8_t> atomic_int8_t; 436typedef atomic<uint8_t> atomic_uint8_t; 437typedef atomic<int16_t> atomic_int16_t; 438typedef atomic<uint16_t> atomic_uint16_t; 439typedef atomic<int32_t> atomic_int32_t; 440typedef atomic<uint32_t> atomic_uint32_t; 441typedef atomic<int64_t> atomic_int64_t; 442typedef atomic<uint64_t> atomic_uint64_t; 443 444typedef atomic<intptr_t> atomic_intptr_t; 445typedef atomic<uintptr_t> atomic_uintptr_t; 446typedef atomic<size_t> atomic_size_t; 447typedef atomic<ptrdiff_t> atomic_ptrdiff_t; 448typedef atomic<intmax_t> atomic_intmax_t; 449typedef atomic<uintmax_t> atomic_uintmax_t; 450 451// flag type and operations 452 453typedef struct atomic_flag 454{ 455 atomic_flag() noexcept = default; // until C++20 456 constexpr atomic_flag() noexcept; // since C++20 457 atomic_flag(const atomic_flag&) = delete; 458 atomic_flag& operator=(const atomic_flag&) = delete; 459 atomic_flag& operator=(const atomic_flag&) volatile = delete; 460 461 bool test(memory_order m = memory_order_seq_cst) volatile noexcept; 462 bool test(memory_order m = memory_order_seq_cst) noexcept; 463 bool test_and_set(memory_order m = memory_order_seq_cst) volatile noexcept; 464 bool test_and_set(memory_order m = memory_order_seq_cst) noexcept; 465 void clear(memory_order m = memory_order_seq_cst) volatile noexcept; 466 void clear(memory_order m = memory_order_seq_cst) noexcept; 467 468 void wait(bool, memory_order = memory_order::seq_cst) const volatile noexcept; 469 void wait(bool, memory_order = memory_order::seq_cst) const noexcept; 470 void notify_one() volatile noexcept; 471 void notify_one() noexcept; 472 void notify_all() volatile noexcept; 473 void notify_all() noexcept; 474} atomic_flag; 475 476bool atomic_flag_test(volatile atomic_flag* obj) noexcept; 477bool atomic_flag_test(atomic_flag* obj) noexcept; 478bool atomic_flag_test_explicit(volatile atomic_flag* obj, 479 memory_order m) noexcept; 480bool atomic_flag_test_explicit(atomic_flag* obj, memory_order m) noexcept; 481bool atomic_flag_test_and_set(volatile atomic_flag* obj) noexcept; 482bool atomic_flag_test_and_set(atomic_flag* obj) noexcept; 483bool atomic_flag_test_and_set_explicit(volatile atomic_flag* obj, 484 memory_order m) noexcept; 485bool atomic_flag_test_and_set_explicit(atomic_flag* obj, memory_order m) noexcept; 486void atomic_flag_clear(volatile atomic_flag* obj) noexcept; 487void atomic_flag_clear(atomic_flag* obj) noexcept; 488void atomic_flag_clear_explicit(volatile atomic_flag* obj, memory_order m) noexcept; 489void atomic_flag_clear_explicit(atomic_flag* obj, memory_order m) noexcept; 490 491void atomic_wait(const volatile atomic_flag* obj, T old) noexcept; 492void atomic_wait(const atomic_flag* obj, T old) noexcept; 493void atomic_wait_explicit(const volatile atomic_flag* obj, T old, memory_order m) noexcept; 494void atomic_wait_explicit(const atomic_flag* obj, T old, memory_order m) noexcept; 495void atomic_one(volatile atomic_flag* obj) noexcept; 496void atomic_one(atomic_flag* obj) noexcept; 497void atomic_all(volatile atomic_flag* obj) noexcept; 498void atomic_all(atomic_flag* obj) noexcept; 499 500// fences 501 502void atomic_thread_fence(memory_order m) noexcept; 503void atomic_signal_fence(memory_order m) noexcept; 504 505// deprecated 506 507template <class T> 508 void atomic_init(volatile atomic<T>* obj, atomic<T>::value_type desr) noexcept; 509 510template <class T> 511 void atomic_init(atomic<T>* obj, atomic<T>::value_type desr) noexcept; 512 513#define ATOMIC_VAR_INIT(value) see below 514 515#define ATOMIC_FLAG_INIT see below 516 517} // std 518 519*/ 520 521#include <__availability> 522#include <__chrono/duration.h> 523#include <__config> 524#include <__thread/poll_with_backoff.h> 525#include <__thread/timed_backoff_policy.h> 526#include <cstddef> 527#include <cstdint> 528#include <cstring> 529#include <type_traits> 530#include <version> 531 532#ifndef _LIBCPP_HAS_NO_THREADS 533# include <__threading_support> 534#endif 535 536#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) 537# pragma GCC system_header 538#endif 539 540#ifdef _LIBCPP_HAS_NO_ATOMIC_HEADER 541# error <atomic> is not implemented 542#endif 543#ifdef kill_dependency 544# error C++ standard library is incompatible with <stdatomic.h> 545#endif 546 547#define _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) \ 548 _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_consume || \ 549 __m == memory_order_acquire || \ 550 __m == memory_order_acq_rel, \ 551 "memory order argument to atomic operation is invalid") 552 553#define _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) \ 554 _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_release || \ 555 __m == memory_order_acq_rel, \ 556 "memory order argument to atomic operation is invalid") 557 558#define _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__m, __f) \ 559 _LIBCPP_DIAGNOSE_WARNING(__f == memory_order_release || \ 560 __f == memory_order_acq_rel, \ 561 "memory order argument to atomic operation is invalid") 562 563_LIBCPP_BEGIN_NAMESPACE_STD 564 565// Figure out what the underlying type for `memory_order` would be if it were 566// declared as an unscoped enum (accounting for -fshort-enums). Use this result 567// to pin the underlying type in C++20. 568enum __legacy_memory_order { 569 __mo_relaxed, 570 __mo_consume, 571 __mo_acquire, 572 __mo_release, 573 __mo_acq_rel, 574 __mo_seq_cst 575}; 576 577typedef underlying_type<__legacy_memory_order>::type __memory_order_underlying_t; 578 579#if _LIBCPP_STD_VER > 17 580 581enum class memory_order : __memory_order_underlying_t { 582 relaxed = __mo_relaxed, 583 consume = __mo_consume, 584 acquire = __mo_acquire, 585 release = __mo_release, 586 acq_rel = __mo_acq_rel, 587 seq_cst = __mo_seq_cst 588}; 589 590inline constexpr auto memory_order_relaxed = memory_order::relaxed; 591inline constexpr auto memory_order_consume = memory_order::consume; 592inline constexpr auto memory_order_acquire = memory_order::acquire; 593inline constexpr auto memory_order_release = memory_order::release; 594inline constexpr auto memory_order_acq_rel = memory_order::acq_rel; 595inline constexpr auto memory_order_seq_cst = memory_order::seq_cst; 596 597#else 598 599typedef enum memory_order { 600 memory_order_relaxed = __mo_relaxed, 601 memory_order_consume = __mo_consume, 602 memory_order_acquire = __mo_acquire, 603 memory_order_release = __mo_release, 604 memory_order_acq_rel = __mo_acq_rel, 605 memory_order_seq_cst = __mo_seq_cst, 606} memory_order; 607 608#endif // _LIBCPP_STD_VER > 17 609 610template <typename _Tp> _LIBCPP_INLINE_VISIBILITY 611bool __cxx_nonatomic_compare_equal(_Tp const& __lhs, _Tp const& __rhs) { 612 return _VSTD::memcmp(&__lhs, &__rhs, sizeof(_Tp)) == 0; 613} 614 615static_assert((is_same<underlying_type<memory_order>::type, __memory_order_underlying_t>::value), 616 "unexpected underlying type for std::memory_order"); 617 618#if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) || \ 619 defined(_LIBCPP_ATOMIC_ONLY_USE_BUILTINS) 620 621// [atomics.types.generic]p1 guarantees _Tp is trivially copyable. Because 622// the default operator= in an object is not volatile, a byte-by-byte copy 623// is required. 624template <typename _Tp, typename _Tv> _LIBCPP_INLINE_VISIBILITY 625typename enable_if<is_assignable<_Tp&, _Tv>::value>::type 626__cxx_atomic_assign_volatile(_Tp& __a_value, _Tv const& __val) { 627 __a_value = __val; 628} 629template <typename _Tp, typename _Tv> _LIBCPP_INLINE_VISIBILITY 630typename enable_if<is_assignable<_Tp&, _Tv>::value>::type 631__cxx_atomic_assign_volatile(_Tp volatile& __a_value, _Tv volatile const& __val) { 632 volatile char* __to = reinterpret_cast<volatile char*>(&__a_value); 633 volatile char* __end = __to + sizeof(_Tp); 634 volatile const char* __from = reinterpret_cast<volatile const char*>(&__val); 635 while (__to != __end) 636 *__to++ = *__from++; 637} 638 639#endif 640 641#if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) 642 643template <typename _Tp> 644struct __cxx_atomic_base_impl { 645 646 _LIBCPP_INLINE_VISIBILITY 647#ifndef _LIBCPP_CXX03_LANG 648 __cxx_atomic_base_impl() _NOEXCEPT = default; 649#else 650 __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {} 651#endif // _LIBCPP_CXX03_LANG 652 _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT 653 : __a_value(value) {} 654 _Tp __a_value; 655}; 656 657_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR int __to_gcc_order(memory_order __order) { 658 // Avoid switch statement to make this a constexpr. 659 return __order == memory_order_relaxed ? __ATOMIC_RELAXED: 660 (__order == memory_order_acquire ? __ATOMIC_ACQUIRE: 661 (__order == memory_order_release ? __ATOMIC_RELEASE: 662 (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST: 663 (__order == memory_order_acq_rel ? __ATOMIC_ACQ_REL: 664 __ATOMIC_CONSUME)))); 665} 666 667_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR int __to_gcc_failure_order(memory_order __order) { 668 // Avoid switch statement to make this a constexpr. 669 return __order == memory_order_relaxed ? __ATOMIC_RELAXED: 670 (__order == memory_order_acquire ? __ATOMIC_ACQUIRE: 671 (__order == memory_order_release ? __ATOMIC_RELAXED: 672 (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST: 673 (__order == memory_order_acq_rel ? __ATOMIC_ACQUIRE: 674 __ATOMIC_CONSUME)))); 675} 676 677template <typename _Tp> 678_LIBCPP_INLINE_VISIBILITY 679void __cxx_atomic_init(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val) { 680 __cxx_atomic_assign_volatile(__a->__a_value, __val); 681} 682 683template <typename _Tp> 684_LIBCPP_INLINE_VISIBILITY 685void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) { 686 __a->__a_value = __val; 687} 688 689_LIBCPP_INLINE_VISIBILITY inline 690void __cxx_atomic_thread_fence(memory_order __order) { 691 __atomic_thread_fence(__to_gcc_order(__order)); 692} 693 694_LIBCPP_INLINE_VISIBILITY inline 695void __cxx_atomic_signal_fence(memory_order __order) { 696 __atomic_signal_fence(__to_gcc_order(__order)); 697} 698 699template <typename _Tp> 700_LIBCPP_INLINE_VISIBILITY 701void __cxx_atomic_store(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val, 702 memory_order __order) { 703 __atomic_store(&__a->__a_value, &__val, 704 __to_gcc_order(__order)); 705} 706 707template <typename _Tp> 708_LIBCPP_INLINE_VISIBILITY 709void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, 710 memory_order __order) { 711 __atomic_store(&__a->__a_value, &__val, 712 __to_gcc_order(__order)); 713} 714 715template <typename _Tp> 716_LIBCPP_INLINE_VISIBILITY 717_Tp __cxx_atomic_load(const volatile __cxx_atomic_base_impl<_Tp>* __a, 718 memory_order __order) { 719 _Tp __ret; 720 __atomic_load(&__a->__a_value, &__ret, 721 __to_gcc_order(__order)); 722 return __ret; 723} 724 725template <typename _Tp> 726_LIBCPP_INLINE_VISIBILITY 727_Tp __cxx_atomic_load(const __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) { 728 _Tp __ret; 729 __atomic_load(&__a->__a_value, &__ret, 730 __to_gcc_order(__order)); 731 return __ret; 732} 733 734template <typename _Tp> 735_LIBCPP_INLINE_VISIBILITY 736_Tp __cxx_atomic_exchange(volatile __cxx_atomic_base_impl<_Tp>* __a, 737 _Tp __value, memory_order __order) { 738 _Tp __ret; 739 __atomic_exchange(&__a->__a_value, &__value, &__ret, 740 __to_gcc_order(__order)); 741 return __ret; 742} 743 744template <typename _Tp> 745_LIBCPP_INLINE_VISIBILITY 746_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, 747 memory_order __order) { 748 _Tp __ret; 749 __atomic_exchange(&__a->__a_value, &__value, &__ret, 750 __to_gcc_order(__order)); 751 return __ret; 752} 753 754template <typename _Tp> 755_LIBCPP_INLINE_VISIBILITY 756bool __cxx_atomic_compare_exchange_strong( 757 volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, 758 memory_order __success, memory_order __failure) { 759 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 760 false, 761 __to_gcc_order(__success), 762 __to_gcc_failure_order(__failure)); 763} 764 765template <typename _Tp> 766_LIBCPP_INLINE_VISIBILITY 767bool __cxx_atomic_compare_exchange_strong( 768 __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, 769 memory_order __failure) { 770 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 771 false, 772 __to_gcc_order(__success), 773 __to_gcc_failure_order(__failure)); 774} 775 776template <typename _Tp> 777_LIBCPP_INLINE_VISIBILITY 778bool __cxx_atomic_compare_exchange_weak( 779 volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, 780 memory_order __success, memory_order __failure) { 781 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 782 true, 783 __to_gcc_order(__success), 784 __to_gcc_failure_order(__failure)); 785} 786 787template <typename _Tp> 788_LIBCPP_INLINE_VISIBILITY 789bool __cxx_atomic_compare_exchange_weak( 790 __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, 791 memory_order __failure) { 792 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 793 true, 794 __to_gcc_order(__success), 795 __to_gcc_failure_order(__failure)); 796} 797 798template <typename _Tp> 799struct __skip_amt { enum {value = 1}; }; 800 801template <typename _Tp> 802struct __skip_amt<_Tp*> { enum {value = sizeof(_Tp)}; }; 803 804// FIXME: Haven't figured out what the spec says about using arrays with 805// atomic_fetch_add. Force a failure rather than creating bad behavior. 806template <typename _Tp> 807struct __skip_amt<_Tp[]> { }; 808template <typename _Tp, int n> 809struct __skip_amt<_Tp[n]> { }; 810 811template <typename _Tp, typename _Td> 812_LIBCPP_INLINE_VISIBILITY 813_Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_base_impl<_Tp>* __a, 814 _Td __delta, memory_order __order) { 815 return __atomic_fetch_add(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 816 __to_gcc_order(__order)); 817} 818 819template <typename _Tp, typename _Td> 820_LIBCPP_INLINE_VISIBILITY 821_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, 822 memory_order __order) { 823 return __atomic_fetch_add(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 824 __to_gcc_order(__order)); 825} 826 827template <typename _Tp, typename _Td> 828_LIBCPP_INLINE_VISIBILITY 829_Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_base_impl<_Tp>* __a, 830 _Td __delta, memory_order __order) { 831 return __atomic_fetch_sub(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 832 __to_gcc_order(__order)); 833} 834 835template <typename _Tp, typename _Td> 836_LIBCPP_INLINE_VISIBILITY 837_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, 838 memory_order __order) { 839 return __atomic_fetch_sub(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 840 __to_gcc_order(__order)); 841} 842 843template <typename _Tp> 844_LIBCPP_INLINE_VISIBILITY 845_Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_base_impl<_Tp>* __a, 846 _Tp __pattern, memory_order __order) { 847 return __atomic_fetch_and(&__a->__a_value, __pattern, 848 __to_gcc_order(__order)); 849} 850 851template <typename _Tp> 852_LIBCPP_INLINE_VISIBILITY 853_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, 854 _Tp __pattern, memory_order __order) { 855 return __atomic_fetch_and(&__a->__a_value, __pattern, 856 __to_gcc_order(__order)); 857} 858 859template <typename _Tp> 860_LIBCPP_INLINE_VISIBILITY 861_Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_base_impl<_Tp>* __a, 862 _Tp __pattern, memory_order __order) { 863 return __atomic_fetch_or(&__a->__a_value, __pattern, 864 __to_gcc_order(__order)); 865} 866 867template <typename _Tp> 868_LIBCPP_INLINE_VISIBILITY 869_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, 870 memory_order __order) { 871 return __atomic_fetch_or(&__a->__a_value, __pattern, 872 __to_gcc_order(__order)); 873} 874 875template <typename _Tp> 876_LIBCPP_INLINE_VISIBILITY 877_Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_base_impl<_Tp>* __a, 878 _Tp __pattern, memory_order __order) { 879 return __atomic_fetch_xor(&__a->__a_value, __pattern, 880 __to_gcc_order(__order)); 881} 882 883template <typename _Tp> 884_LIBCPP_INLINE_VISIBILITY 885_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, 886 memory_order __order) { 887 return __atomic_fetch_xor(&__a->__a_value, __pattern, 888 __to_gcc_order(__order)); 889} 890 891#define __cxx_atomic_is_lock_free(__s) __atomic_is_lock_free(__s, 0) 892 893#elif defined(_LIBCPP_HAS_C_ATOMIC_IMP) 894 895template <typename _Tp> 896struct __cxx_atomic_base_impl { 897 898 _LIBCPP_INLINE_VISIBILITY 899#ifndef _LIBCPP_CXX03_LANG 900 __cxx_atomic_base_impl() _NOEXCEPT = default; 901#else 902 __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {} 903#endif // _LIBCPP_CXX03_LANG 904 _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT 905 : __a_value(value) {} 906 _LIBCPP_DISABLE_EXTENSION_WARNING _Atomic(_Tp) __a_value; 907}; 908 909#define __cxx_atomic_is_lock_free(__s) __c11_atomic_is_lock_free(__s) 910 911_LIBCPP_INLINE_VISIBILITY inline 912void __cxx_atomic_thread_fence(memory_order __order) _NOEXCEPT { 913 __c11_atomic_thread_fence(static_cast<__memory_order_underlying_t>(__order)); 914} 915 916_LIBCPP_INLINE_VISIBILITY inline 917void __cxx_atomic_signal_fence(memory_order __order) _NOEXCEPT { 918 __c11_atomic_signal_fence(static_cast<__memory_order_underlying_t>(__order)); 919} 920 921template<class _Tp> 922_LIBCPP_INLINE_VISIBILITY 923void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val) _NOEXCEPT { 924 __c11_atomic_init(&__a->__a_value, __val); 925} 926template<class _Tp> 927_LIBCPP_INLINE_VISIBILITY 928void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val) _NOEXCEPT { 929 __c11_atomic_init(&__a->__a_value, __val); 930} 931 932template<class _Tp> 933_LIBCPP_INLINE_VISIBILITY 934void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT { 935 __c11_atomic_store(&__a->__a_value, __val, static_cast<__memory_order_underlying_t>(__order)); 936} 937template<class _Tp> 938_LIBCPP_INLINE_VISIBILITY 939void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val, memory_order __order) _NOEXCEPT { 940 __c11_atomic_store(&__a->__a_value, __val, static_cast<__memory_order_underlying_t>(__order)); 941} 942 943template<class _Tp> 944_LIBCPP_INLINE_VISIBILITY 945_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const volatile* __a, memory_order __order) _NOEXCEPT { 946 using __ptr_type = typename remove_const<decltype(__a->__a_value)>::type*; 947 return __c11_atomic_load(const_cast<__ptr_type>(&__a->__a_value), static_cast<__memory_order_underlying_t>(__order)); 948} 949template<class _Tp> 950_LIBCPP_INLINE_VISIBILITY 951_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const* __a, memory_order __order) _NOEXCEPT { 952 using __ptr_type = typename remove_const<decltype(__a->__a_value)>::type*; 953 return __c11_atomic_load(const_cast<__ptr_type>(&__a->__a_value), static_cast<__memory_order_underlying_t>(__order)); 954} 955 956template<class _Tp> 957_LIBCPP_INLINE_VISIBILITY 958_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT { 959 return __c11_atomic_exchange(&__a->__a_value, __value, static_cast<__memory_order_underlying_t>(__order)); 960} 961template<class _Tp> 962_LIBCPP_INLINE_VISIBILITY 963_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> * __a, _Tp __value, memory_order __order) _NOEXCEPT { 964 return __c11_atomic_exchange(&__a->__a_value, __value, static_cast<__memory_order_underlying_t>(__order)); 965} 966 967_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR memory_order __to_failure_order(memory_order __order) { 968 // Avoid switch statement to make this a constexpr. 969 return __order == memory_order_release ? memory_order_relaxed: 970 (__order == memory_order_acq_rel ? memory_order_acquire: 971 __order); 972} 973 974template<class _Tp> 975_LIBCPP_INLINE_VISIBILITY 976bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 977 return __c11_atomic_compare_exchange_strong(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); 978} 979template<class _Tp> 980_LIBCPP_INLINE_VISIBILITY 981bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 982 return __c11_atomic_compare_exchange_strong(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); 983} 984 985template<class _Tp> 986_LIBCPP_INLINE_VISIBILITY 987bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 988 return __c11_atomic_compare_exchange_weak(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); 989} 990template<class _Tp> 991_LIBCPP_INLINE_VISIBILITY 992bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 993 return __c11_atomic_compare_exchange_weak(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); 994} 995 996template<class _Tp> 997_LIBCPP_INLINE_VISIBILITY 998_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT { 999 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1000} 1001template<class _Tp> 1002_LIBCPP_INLINE_VISIBILITY 1003_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1004 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1005} 1006 1007template<class _Tp> 1008_LIBCPP_INLINE_VISIBILITY 1009_Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1010 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1011} 1012template<class _Tp> 1013_LIBCPP_INLINE_VISIBILITY 1014_Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1015 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1016} 1017 1018template<class _Tp> 1019_LIBCPP_INLINE_VISIBILITY 1020_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1021 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1022} 1023template<class _Tp> 1024_LIBCPP_INLINE_VISIBILITY 1025_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1026 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1027} 1028template<class _Tp> 1029_LIBCPP_INLINE_VISIBILITY 1030_Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1031 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1032} 1033template<class _Tp> 1034_LIBCPP_INLINE_VISIBILITY 1035_Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1036 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1037} 1038 1039template<class _Tp> 1040_LIBCPP_INLINE_VISIBILITY 1041_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1042 return __c11_atomic_fetch_and(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1043} 1044template<class _Tp> 1045_LIBCPP_INLINE_VISIBILITY 1046_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1047 return __c11_atomic_fetch_and(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1048} 1049 1050template<class _Tp> 1051_LIBCPP_INLINE_VISIBILITY 1052_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1053 return __c11_atomic_fetch_or(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1054} 1055template<class _Tp> 1056_LIBCPP_INLINE_VISIBILITY 1057_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1058 return __c11_atomic_fetch_or(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1059} 1060 1061template<class _Tp> 1062_LIBCPP_INLINE_VISIBILITY 1063_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1064 return __c11_atomic_fetch_xor(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1065} 1066template<class _Tp> 1067_LIBCPP_INLINE_VISIBILITY 1068_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1069 return __c11_atomic_fetch_xor(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1070} 1071 1072#endif // _LIBCPP_HAS_GCC_ATOMIC_IMP, _LIBCPP_HAS_C_ATOMIC_IMP 1073 1074template <class _Tp> 1075_LIBCPP_INLINE_VISIBILITY 1076_Tp kill_dependency(_Tp __y) _NOEXCEPT 1077{ 1078 return __y; 1079} 1080 1081#if defined(__CLANG_ATOMIC_BOOL_LOCK_FREE) 1082# define ATOMIC_BOOL_LOCK_FREE __CLANG_ATOMIC_BOOL_LOCK_FREE 1083# define ATOMIC_CHAR_LOCK_FREE __CLANG_ATOMIC_CHAR_LOCK_FREE 1084#ifndef _LIBCPP_HAS_NO_CHAR8_T 1085# define ATOMIC_CHAR8_T_LOCK_FREE __CLANG_ATOMIC_CHAR8_T_LOCK_FREE 1086#endif 1087# define ATOMIC_CHAR16_T_LOCK_FREE __CLANG_ATOMIC_CHAR16_T_LOCK_FREE 1088# define ATOMIC_CHAR32_T_LOCK_FREE __CLANG_ATOMIC_CHAR32_T_LOCK_FREE 1089# define ATOMIC_WCHAR_T_LOCK_FREE __CLANG_ATOMIC_WCHAR_T_LOCK_FREE 1090# define ATOMIC_SHORT_LOCK_FREE __CLANG_ATOMIC_SHORT_LOCK_FREE 1091# define ATOMIC_INT_LOCK_FREE __CLANG_ATOMIC_INT_LOCK_FREE 1092# define ATOMIC_LONG_LOCK_FREE __CLANG_ATOMIC_LONG_LOCK_FREE 1093# define ATOMIC_LLONG_LOCK_FREE __CLANG_ATOMIC_LLONG_LOCK_FREE 1094# define ATOMIC_POINTER_LOCK_FREE __CLANG_ATOMIC_POINTER_LOCK_FREE 1095#elif defined(__GCC_ATOMIC_BOOL_LOCK_FREE) 1096# define ATOMIC_BOOL_LOCK_FREE __GCC_ATOMIC_BOOL_LOCK_FREE 1097# define ATOMIC_CHAR_LOCK_FREE __GCC_ATOMIC_CHAR_LOCK_FREE 1098#ifndef _LIBCPP_HAS_NO_CHAR8_T 1099# define ATOMIC_CHAR8_T_LOCK_FREE __GCC_ATOMIC_CHAR8_T_LOCK_FREE 1100#endif 1101# define ATOMIC_CHAR16_T_LOCK_FREE __GCC_ATOMIC_CHAR16_T_LOCK_FREE 1102# define ATOMIC_CHAR32_T_LOCK_FREE __GCC_ATOMIC_CHAR32_T_LOCK_FREE 1103# define ATOMIC_WCHAR_T_LOCK_FREE __GCC_ATOMIC_WCHAR_T_LOCK_FREE 1104# define ATOMIC_SHORT_LOCK_FREE __GCC_ATOMIC_SHORT_LOCK_FREE 1105# define ATOMIC_INT_LOCK_FREE __GCC_ATOMIC_INT_LOCK_FREE 1106# define ATOMIC_LONG_LOCK_FREE __GCC_ATOMIC_LONG_LOCK_FREE 1107# define ATOMIC_LLONG_LOCK_FREE __GCC_ATOMIC_LLONG_LOCK_FREE 1108# define ATOMIC_POINTER_LOCK_FREE __GCC_ATOMIC_POINTER_LOCK_FREE 1109#endif 1110 1111#ifdef _LIBCPP_ATOMIC_ONLY_USE_BUILTINS 1112 1113template<typename _Tp> 1114struct __cxx_atomic_lock_impl { 1115 1116 _LIBCPP_INLINE_VISIBILITY 1117 __cxx_atomic_lock_impl() _NOEXCEPT 1118 : __a_value(), __a_lock(0) {} 1119 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR explicit 1120 __cxx_atomic_lock_impl(_Tp value) _NOEXCEPT 1121 : __a_value(value), __a_lock(0) {} 1122 1123 _Tp __a_value; 1124 mutable __cxx_atomic_base_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_lock; 1125 1126 _LIBCPP_INLINE_VISIBILITY void __lock() const volatile { 1127 while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire)) 1128 /*spin*/; 1129 } 1130 _LIBCPP_INLINE_VISIBILITY void __lock() const { 1131 while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire)) 1132 /*spin*/; 1133 } 1134 _LIBCPP_INLINE_VISIBILITY void __unlock() const volatile { 1135 __cxx_atomic_store(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(false), memory_order_release); 1136 } 1137 _LIBCPP_INLINE_VISIBILITY void __unlock() const { 1138 __cxx_atomic_store(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(false), memory_order_release); 1139 } 1140 _LIBCPP_INLINE_VISIBILITY _Tp __read() const volatile { 1141 __lock(); 1142 _Tp __old; 1143 __cxx_atomic_assign_volatile(__old, __a_value); 1144 __unlock(); 1145 return __old; 1146 } 1147 _LIBCPP_INLINE_VISIBILITY _Tp __read() const { 1148 __lock(); 1149 _Tp __old = __a_value; 1150 __unlock(); 1151 return __old; 1152 } 1153}; 1154 1155template <typename _Tp> 1156_LIBCPP_INLINE_VISIBILITY 1157void __cxx_atomic_init(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __val) { 1158 __cxx_atomic_assign_volatile(__a->__a_value, __val); 1159} 1160template <typename _Tp> 1161_LIBCPP_INLINE_VISIBILITY 1162void __cxx_atomic_init(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __val) { 1163 __a->__a_value = __val; 1164} 1165 1166template <typename _Tp> 1167_LIBCPP_INLINE_VISIBILITY 1168void __cxx_atomic_store(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __val, memory_order) { 1169 __a->__lock(); 1170 __cxx_atomic_assign_volatile(__a->__a_value, __val); 1171 __a->__unlock(); 1172} 1173template <typename _Tp> 1174_LIBCPP_INLINE_VISIBILITY 1175void __cxx_atomic_store(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __val, memory_order) { 1176 __a->__lock(); 1177 __a->__a_value = __val; 1178 __a->__unlock(); 1179} 1180 1181template <typename _Tp> 1182_LIBCPP_INLINE_VISIBILITY 1183_Tp __cxx_atomic_load(const volatile __cxx_atomic_lock_impl<_Tp>* __a, memory_order) { 1184 return __a->__read(); 1185} 1186template <typename _Tp> 1187_LIBCPP_INLINE_VISIBILITY 1188_Tp __cxx_atomic_load(const __cxx_atomic_lock_impl<_Tp>* __a, memory_order) { 1189 return __a->__read(); 1190} 1191 1192template <typename _Tp> 1193_LIBCPP_INLINE_VISIBILITY 1194_Tp __cxx_atomic_exchange(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) { 1195 __a->__lock(); 1196 _Tp __old; 1197 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1198 __cxx_atomic_assign_volatile(__a->__a_value, __value); 1199 __a->__unlock(); 1200 return __old; 1201} 1202template <typename _Tp> 1203_LIBCPP_INLINE_VISIBILITY 1204_Tp __cxx_atomic_exchange(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) { 1205 __a->__lock(); 1206 _Tp __old = __a->__a_value; 1207 __a->__a_value = __value; 1208 __a->__unlock(); 1209 return __old; 1210} 1211 1212template <typename _Tp> 1213_LIBCPP_INLINE_VISIBILITY 1214bool __cxx_atomic_compare_exchange_strong(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1215 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1216 _Tp __temp; 1217 __a->__lock(); 1218 __cxx_atomic_assign_volatile(__temp, __a->__a_value); 1219 bool __ret = (_VSTD::memcmp(&__temp, __expected, sizeof(_Tp)) == 0); 1220 if(__ret) 1221 __cxx_atomic_assign_volatile(__a->__a_value, __value); 1222 else 1223 __cxx_atomic_assign_volatile(*__expected, __a->__a_value); 1224 __a->__unlock(); 1225 return __ret; 1226} 1227template <typename _Tp> 1228_LIBCPP_INLINE_VISIBILITY 1229bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_lock_impl<_Tp>* __a, 1230 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1231 __a->__lock(); 1232 bool __ret = (_VSTD::memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0); 1233 if(__ret) 1234 _VSTD::memcpy(&__a->__a_value, &__value, sizeof(_Tp)); 1235 else 1236 _VSTD::memcpy(__expected, &__a->__a_value, sizeof(_Tp)); 1237 __a->__unlock(); 1238 return __ret; 1239} 1240 1241template <typename _Tp> 1242_LIBCPP_INLINE_VISIBILITY 1243bool __cxx_atomic_compare_exchange_weak(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1244 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1245 _Tp __temp; 1246 __a->__lock(); 1247 __cxx_atomic_assign_volatile(__temp, __a->__a_value); 1248 bool __ret = (_VSTD::memcmp(&__temp, __expected, sizeof(_Tp)) == 0); 1249 if(__ret) 1250 __cxx_atomic_assign_volatile(__a->__a_value, __value); 1251 else 1252 __cxx_atomic_assign_volatile(*__expected, __a->__a_value); 1253 __a->__unlock(); 1254 return __ret; 1255} 1256template <typename _Tp> 1257_LIBCPP_INLINE_VISIBILITY 1258bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_lock_impl<_Tp>* __a, 1259 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1260 __a->__lock(); 1261 bool __ret = (_VSTD::memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0); 1262 if(__ret) 1263 _VSTD::memcpy(&__a->__a_value, &__value, sizeof(_Tp)); 1264 else 1265 _VSTD::memcpy(__expected, &__a->__a_value, sizeof(_Tp)); 1266 __a->__unlock(); 1267 return __ret; 1268} 1269 1270template <typename _Tp, typename _Td> 1271_LIBCPP_INLINE_VISIBILITY 1272_Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1273 _Td __delta, memory_order) { 1274 __a->__lock(); 1275 _Tp __old; 1276 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1277 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old + __delta)); 1278 __a->__unlock(); 1279 return __old; 1280} 1281template <typename _Tp, typename _Td> 1282_LIBCPP_INLINE_VISIBILITY 1283_Tp __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp>* __a, 1284 _Td __delta, memory_order) { 1285 __a->__lock(); 1286 _Tp __old = __a->__a_value; 1287 __a->__a_value += __delta; 1288 __a->__unlock(); 1289 return __old; 1290} 1291 1292template <typename _Tp, typename _Td> 1293_LIBCPP_INLINE_VISIBILITY 1294_Tp* __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp*>* __a, 1295 ptrdiff_t __delta, memory_order) { 1296 __a->__lock(); 1297 _Tp* __old; 1298 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1299 __cxx_atomic_assign_volatile(__a->__a_value, __old + __delta); 1300 __a->__unlock(); 1301 return __old; 1302} 1303template <typename _Tp, typename _Td> 1304_LIBCPP_INLINE_VISIBILITY 1305_Tp* __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp*>* __a, 1306 ptrdiff_t __delta, memory_order) { 1307 __a->__lock(); 1308 _Tp* __old = __a->__a_value; 1309 __a->__a_value += __delta; 1310 __a->__unlock(); 1311 return __old; 1312} 1313 1314template <typename _Tp, typename _Td> 1315_LIBCPP_INLINE_VISIBILITY 1316_Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1317 _Td __delta, memory_order) { 1318 __a->__lock(); 1319 _Tp __old; 1320 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1321 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old - __delta)); 1322 __a->__unlock(); 1323 return __old; 1324} 1325template <typename _Tp, typename _Td> 1326_LIBCPP_INLINE_VISIBILITY 1327_Tp __cxx_atomic_fetch_sub(__cxx_atomic_lock_impl<_Tp>* __a, 1328 _Td __delta, memory_order) { 1329 __a->__lock(); 1330 _Tp __old = __a->__a_value; 1331 __a->__a_value -= __delta; 1332 __a->__unlock(); 1333 return __old; 1334} 1335 1336template <typename _Tp> 1337_LIBCPP_INLINE_VISIBILITY 1338_Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1339 _Tp __pattern, memory_order) { 1340 __a->__lock(); 1341 _Tp __old; 1342 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1343 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old & __pattern)); 1344 __a->__unlock(); 1345 return __old; 1346} 1347template <typename _Tp> 1348_LIBCPP_INLINE_VISIBILITY 1349_Tp __cxx_atomic_fetch_and(__cxx_atomic_lock_impl<_Tp>* __a, 1350 _Tp __pattern, memory_order) { 1351 __a->__lock(); 1352 _Tp __old = __a->__a_value; 1353 __a->__a_value &= __pattern; 1354 __a->__unlock(); 1355 return __old; 1356} 1357 1358template <typename _Tp> 1359_LIBCPP_INLINE_VISIBILITY 1360_Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1361 _Tp __pattern, memory_order) { 1362 __a->__lock(); 1363 _Tp __old; 1364 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1365 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old | __pattern)); 1366 __a->__unlock(); 1367 return __old; 1368} 1369template <typename _Tp> 1370_LIBCPP_INLINE_VISIBILITY 1371_Tp __cxx_atomic_fetch_or(__cxx_atomic_lock_impl<_Tp>* __a, 1372 _Tp __pattern, memory_order) { 1373 __a->__lock(); 1374 _Tp __old = __a->__a_value; 1375 __a->__a_value |= __pattern; 1376 __a->__unlock(); 1377 return __old; 1378} 1379 1380template <typename _Tp> 1381_LIBCPP_INLINE_VISIBILITY 1382_Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1383 _Tp __pattern, memory_order) { 1384 __a->__lock(); 1385 _Tp __old; 1386 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1387 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old ^ __pattern)); 1388 __a->__unlock(); 1389 return __old; 1390} 1391template <typename _Tp> 1392_LIBCPP_INLINE_VISIBILITY 1393_Tp __cxx_atomic_fetch_xor(__cxx_atomic_lock_impl<_Tp>* __a, 1394 _Tp __pattern, memory_order) { 1395 __a->__lock(); 1396 _Tp __old = __a->__a_value; 1397 __a->__a_value ^= __pattern; 1398 __a->__unlock(); 1399 return __old; 1400} 1401 1402#ifdef __cpp_lib_atomic_is_always_lock_free 1403 1404template<typename _Tp> struct __cxx_is_always_lock_free { 1405 enum { __value = __atomic_always_lock_free(sizeof(_Tp), 0) }; }; 1406 1407#else 1408 1409template<typename _Tp> struct __cxx_is_always_lock_free { enum { __value = false }; }; 1410// Implementations must match the C ATOMIC_*_LOCK_FREE macro values. 1411template<> struct __cxx_is_always_lock_free<bool> { enum { __value = 2 == ATOMIC_BOOL_LOCK_FREE }; }; 1412template<> struct __cxx_is_always_lock_free<char> { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; }; 1413template<> struct __cxx_is_always_lock_free<signed char> { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; }; 1414template<> struct __cxx_is_always_lock_free<unsigned char> { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; }; 1415#ifndef _LIBCPP_HAS_NO_CHAR8_T 1416template<> struct __cxx_is_always_lock_free<char8_t> { enum { __value = 2 == ATOMIC_CHAR8_T_LOCK_FREE }; }; 1417#endif 1418template<> struct __cxx_is_always_lock_free<char16_t> { enum { __value = 2 == ATOMIC_CHAR16_T_LOCK_FREE }; }; 1419template<> struct __cxx_is_always_lock_free<char32_t> { enum { __value = 2 == ATOMIC_CHAR32_T_LOCK_FREE }; }; 1420#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS 1421template<> struct __cxx_is_always_lock_free<wchar_t> { enum { __value = 2 == ATOMIC_WCHAR_T_LOCK_FREE }; }; 1422#endif 1423template<> struct __cxx_is_always_lock_free<short> { enum { __value = 2 == ATOMIC_SHORT_LOCK_FREE }; }; 1424template<> struct __cxx_is_always_lock_free<unsigned short> { enum { __value = 2 == ATOMIC_SHORT_LOCK_FREE }; }; 1425template<> struct __cxx_is_always_lock_free<int> { enum { __value = 2 == ATOMIC_INT_LOCK_FREE }; }; 1426template<> struct __cxx_is_always_lock_free<unsigned int> { enum { __value = 2 == ATOMIC_INT_LOCK_FREE }; }; 1427template<> struct __cxx_is_always_lock_free<long> { enum { __value = 2 == ATOMIC_LONG_LOCK_FREE }; }; 1428template<> struct __cxx_is_always_lock_free<unsigned long> { enum { __value = 2 == ATOMIC_LONG_LOCK_FREE }; }; 1429template<> struct __cxx_is_always_lock_free<long long> { enum { __value = 2 == ATOMIC_LLONG_LOCK_FREE }; }; 1430template<> struct __cxx_is_always_lock_free<unsigned long long> { enum { __value = 2 == ATOMIC_LLONG_LOCK_FREE }; }; 1431template<typename _Tp> struct __cxx_is_always_lock_free<_Tp*> { enum { __value = 2 == ATOMIC_POINTER_LOCK_FREE }; }; 1432template<> struct __cxx_is_always_lock_free<std::nullptr_t> { enum { __value = 2 == ATOMIC_POINTER_LOCK_FREE }; }; 1433 1434#endif //__cpp_lib_atomic_is_always_lock_free 1435 1436template <typename _Tp, 1437 typename _Base = typename conditional<__cxx_is_always_lock_free<_Tp>::__value, 1438 __cxx_atomic_base_impl<_Tp>, 1439 __cxx_atomic_lock_impl<_Tp> >::type> 1440#else 1441template <typename _Tp, 1442 typename _Base = __cxx_atomic_base_impl<_Tp> > 1443#endif //_LIBCPP_ATOMIC_ONLY_USE_BUILTINS 1444struct __cxx_atomic_impl : public _Base { 1445 static_assert(is_trivially_copyable<_Tp>::value, 1446 "std::atomic<T> requires that 'T' be a trivially copyable type"); 1447 1448 _LIBCPP_INLINE_VISIBILITY __cxx_atomic_impl() _NOEXCEPT = default; 1449 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR explicit __cxx_atomic_impl(_Tp value) _NOEXCEPT 1450 : _Base(value) {} 1451}; 1452 1453#ifdef __linux__ 1454 using __cxx_contention_t = int32_t; 1455#else 1456 using __cxx_contention_t = int64_t; 1457#endif //__linux__ 1458 1459using __cxx_atomic_contention_t = __cxx_atomic_impl<__cxx_contention_t>; 1460 1461#if defined(_LIBCPP_HAS_NO_THREADS) 1462# define _LIBCPP_HAS_NO_PLATFORM_WAIT 1463#endif 1464 1465// TODO: 1466// _LIBCPP_HAS_NO_PLATFORM_WAIT is currently a "dead" macro, in the sense that 1467// it is not tied anywhere into the build system or even documented. We should 1468// clean it up because it is technically never defined except when threads are 1469// disabled. We should clean it up in its own changeset in case we break "bad" 1470// users. 1471#ifndef _LIBCPP_HAS_NO_PLATFORM_WAIT 1472 1473_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(void const volatile*); 1474_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(void const volatile*); 1475_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(void const volatile*); 1476_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(void const volatile*, __cxx_contention_t); 1477 1478_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(__cxx_atomic_contention_t const volatile*); 1479_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(__cxx_atomic_contention_t const volatile*); 1480_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile*); 1481_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(__cxx_atomic_contention_t const volatile*, __cxx_contention_t); 1482 1483template <class _Atp, class _Fn> 1484struct __libcpp_atomic_wait_backoff_impl { 1485 _Atp* __a; 1486 _Fn __test_fn; 1487 _LIBCPP_AVAILABILITY_SYNC 1488 _LIBCPP_INLINE_VISIBILITY bool operator()(chrono::nanoseconds __elapsed) const 1489 { 1490 if(__elapsed > chrono::microseconds(64)) 1491 { 1492 auto const __monitor = __libcpp_atomic_monitor(__a); 1493 if(__test_fn()) 1494 return true; 1495 __libcpp_atomic_wait(__a, __monitor); 1496 } 1497 else if(__elapsed > chrono::microseconds(4)) 1498 __libcpp_thread_yield(); 1499 else 1500 {} // poll 1501 return false; 1502 } 1503}; 1504 1505template <class _Atp, class _Fn> 1506_LIBCPP_AVAILABILITY_SYNC 1507_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp* __a, _Fn && __test_fn) 1508{ 1509 __libcpp_atomic_wait_backoff_impl<_Atp, typename decay<_Fn>::type> __backoff_fn = {__a, __test_fn}; 1510 return __libcpp_thread_poll_with_backoff(__test_fn, __backoff_fn); 1511} 1512 1513#else // _LIBCPP_HAS_NO_PLATFORM_WAIT 1514 1515template <class _Tp> 1516_LIBCPP_INLINE_VISIBILITY void __cxx_atomic_notify_all(__cxx_atomic_impl<_Tp> const volatile*) { } 1517template <class _Tp> 1518_LIBCPP_INLINE_VISIBILITY void __cxx_atomic_notify_one(__cxx_atomic_impl<_Tp> const volatile*) { } 1519template <class _Atp, class _Fn> 1520_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp*, _Fn && __test_fn) 1521{ 1522#if defined(_LIBCPP_HAS_NO_THREADS) 1523 using _Policy = __spinning_backoff_policy; 1524#else 1525 using _Policy = __libcpp_timed_backoff_policy; 1526#endif 1527 return __libcpp_thread_poll_with_backoff(__test_fn, _Policy()); 1528} 1529 1530#endif // _LIBCPP_HAS_NO_PLATFORM_WAIT 1531 1532template <class _Atp, class _Tp> 1533struct __cxx_atomic_wait_test_fn_impl { 1534 _Atp* __a; 1535 _Tp __val; 1536 memory_order __order; 1537 _LIBCPP_INLINE_VISIBILITY bool operator()() const 1538 { 1539 return !__cxx_nonatomic_compare_equal(__cxx_atomic_load(__a, __order), __val); 1540 } 1541}; 1542 1543template <class _Atp, class _Tp> 1544_LIBCPP_AVAILABILITY_SYNC 1545_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp* __a, _Tp const __val, memory_order __order) 1546{ 1547 __cxx_atomic_wait_test_fn_impl<_Atp, _Tp> __test_fn = {__a, __val, __order}; 1548 return __cxx_atomic_wait(__a, __test_fn); 1549} 1550 1551// general atomic<T> 1552 1553template <class _Tp, bool = is_integral<_Tp>::value && !is_same<_Tp, bool>::value> 1554struct __atomic_base // false 1555{ 1556 mutable __cxx_atomic_impl<_Tp> __a_; 1557 1558#if defined(__cpp_lib_atomic_is_always_lock_free) 1559 static _LIBCPP_CONSTEXPR bool is_always_lock_free = __atomic_always_lock_free(sizeof(__a_), 0); 1560#endif 1561 1562 _LIBCPP_INLINE_VISIBILITY 1563 bool is_lock_free() const volatile _NOEXCEPT 1564 {return __cxx_atomic_is_lock_free(sizeof(_Tp));} 1565 _LIBCPP_INLINE_VISIBILITY 1566 bool is_lock_free() const _NOEXCEPT 1567 {return static_cast<__atomic_base const volatile*>(this)->is_lock_free();} 1568 _LIBCPP_INLINE_VISIBILITY 1569 void store(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1570 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1571 {__cxx_atomic_store(&__a_, __d, __m);} 1572 _LIBCPP_INLINE_VISIBILITY 1573 void store(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1574 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1575 {__cxx_atomic_store(&__a_, __d, __m);} 1576 _LIBCPP_INLINE_VISIBILITY 1577 _Tp load(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 1578 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 1579 {return __cxx_atomic_load(&__a_, __m);} 1580 _LIBCPP_INLINE_VISIBILITY 1581 _Tp load(memory_order __m = memory_order_seq_cst) const _NOEXCEPT 1582 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 1583 {return __cxx_atomic_load(&__a_, __m);} 1584 _LIBCPP_INLINE_VISIBILITY 1585 operator _Tp() const volatile _NOEXCEPT {return load();} 1586 _LIBCPP_INLINE_VISIBILITY 1587 operator _Tp() const _NOEXCEPT {return load();} 1588 _LIBCPP_INLINE_VISIBILITY 1589 _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1590 {return __cxx_atomic_exchange(&__a_, __d, __m);} 1591 _LIBCPP_INLINE_VISIBILITY 1592 _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1593 {return __cxx_atomic_exchange(&__a_, __d, __m);} 1594 _LIBCPP_INLINE_VISIBILITY 1595 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1596 memory_order __s, memory_order __f) volatile _NOEXCEPT 1597 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1598 {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);} 1599 _LIBCPP_INLINE_VISIBILITY 1600 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1601 memory_order __s, memory_order __f) _NOEXCEPT 1602 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1603 {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);} 1604 _LIBCPP_INLINE_VISIBILITY 1605 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1606 memory_order __s, memory_order __f) volatile _NOEXCEPT 1607 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1608 {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);} 1609 _LIBCPP_INLINE_VISIBILITY 1610 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1611 memory_order __s, memory_order __f) _NOEXCEPT 1612 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1613 {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);} 1614 _LIBCPP_INLINE_VISIBILITY 1615 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1616 memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1617 {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);} 1618 _LIBCPP_INLINE_VISIBILITY 1619 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1620 memory_order __m = memory_order_seq_cst) _NOEXCEPT 1621 {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);} 1622 _LIBCPP_INLINE_VISIBILITY 1623 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1624 memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1625 {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);} 1626 _LIBCPP_INLINE_VISIBILITY 1627 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1628 memory_order __m = memory_order_seq_cst) _NOEXCEPT 1629 {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);} 1630 1631 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 1632 {__cxx_atomic_wait(&__a_, __v, __m);} 1633 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT 1634 {__cxx_atomic_wait(&__a_, __v, __m);} 1635 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_one() volatile _NOEXCEPT 1636 {__cxx_atomic_notify_one(&__a_);} 1637 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_one() _NOEXCEPT 1638 {__cxx_atomic_notify_one(&__a_);} 1639 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_all() volatile _NOEXCEPT 1640 {__cxx_atomic_notify_all(&__a_);} 1641 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_all() _NOEXCEPT 1642 {__cxx_atomic_notify_all(&__a_);} 1643 1644#if _LIBCPP_STD_VER > 17 1645 _LIBCPP_INLINE_VISIBILITY constexpr 1646 __atomic_base() noexcept(is_nothrow_default_constructible_v<_Tp>) : __a_(_Tp()) {} 1647#else 1648 _LIBCPP_INLINE_VISIBILITY 1649 __atomic_base() _NOEXCEPT = default; 1650#endif 1651 1652 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR 1653 __atomic_base(_Tp __d) _NOEXCEPT : __a_(__d) {} 1654 1655 __atomic_base(const __atomic_base&) = delete; 1656}; 1657 1658#if defined(__cpp_lib_atomic_is_always_lock_free) 1659template <class _Tp, bool __b> 1660_LIBCPP_CONSTEXPR bool __atomic_base<_Tp, __b>::is_always_lock_free; 1661#endif 1662 1663// atomic<Integral> 1664 1665template <class _Tp> 1666struct __atomic_base<_Tp, true> 1667 : public __atomic_base<_Tp, false> 1668{ 1669 typedef __atomic_base<_Tp, false> __base; 1670 1671 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 1672 __atomic_base() _NOEXCEPT = default; 1673 1674 _LIBCPP_INLINE_VISIBILITY 1675 _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __base(__d) {} 1676 1677 _LIBCPP_INLINE_VISIBILITY 1678 _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1679 {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);} 1680 _LIBCPP_INLINE_VISIBILITY 1681 _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1682 {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);} 1683 _LIBCPP_INLINE_VISIBILITY 1684 _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1685 {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);} 1686 _LIBCPP_INLINE_VISIBILITY 1687 _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1688 {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);} 1689 _LIBCPP_INLINE_VISIBILITY 1690 _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1691 {return __cxx_atomic_fetch_and(&this->__a_, __op, __m);} 1692 _LIBCPP_INLINE_VISIBILITY 1693 _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1694 {return __cxx_atomic_fetch_and(&this->__a_, __op, __m);} 1695 _LIBCPP_INLINE_VISIBILITY 1696 _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1697 {return __cxx_atomic_fetch_or(&this->__a_, __op, __m);} 1698 _LIBCPP_INLINE_VISIBILITY 1699 _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1700 {return __cxx_atomic_fetch_or(&this->__a_, __op, __m);} 1701 _LIBCPP_INLINE_VISIBILITY 1702 _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1703 {return __cxx_atomic_fetch_xor(&this->__a_, __op, __m);} 1704 _LIBCPP_INLINE_VISIBILITY 1705 _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1706 {return __cxx_atomic_fetch_xor(&this->__a_, __op, __m);} 1707 1708 _LIBCPP_INLINE_VISIBILITY 1709 _Tp operator++(int) volatile _NOEXCEPT {return fetch_add(_Tp(1));} 1710 _LIBCPP_INLINE_VISIBILITY 1711 _Tp operator++(int) _NOEXCEPT {return fetch_add(_Tp(1));} 1712 _LIBCPP_INLINE_VISIBILITY 1713 _Tp operator--(int) volatile _NOEXCEPT {return fetch_sub(_Tp(1));} 1714 _LIBCPP_INLINE_VISIBILITY 1715 _Tp operator--(int) _NOEXCEPT {return fetch_sub(_Tp(1));} 1716 _LIBCPP_INLINE_VISIBILITY 1717 _Tp operator++() volatile _NOEXCEPT {return fetch_add(_Tp(1)) + _Tp(1);} 1718 _LIBCPP_INLINE_VISIBILITY 1719 _Tp operator++() _NOEXCEPT {return fetch_add(_Tp(1)) + _Tp(1);} 1720 _LIBCPP_INLINE_VISIBILITY 1721 _Tp operator--() volatile _NOEXCEPT {return fetch_sub(_Tp(1)) - _Tp(1);} 1722 _LIBCPP_INLINE_VISIBILITY 1723 _Tp operator--() _NOEXCEPT {return fetch_sub(_Tp(1)) - _Tp(1);} 1724 _LIBCPP_INLINE_VISIBILITY 1725 _Tp operator+=(_Tp __op) volatile _NOEXCEPT {return fetch_add(__op) + __op;} 1726 _LIBCPP_INLINE_VISIBILITY 1727 _Tp operator+=(_Tp __op) _NOEXCEPT {return fetch_add(__op) + __op;} 1728 _LIBCPP_INLINE_VISIBILITY 1729 _Tp operator-=(_Tp __op) volatile _NOEXCEPT {return fetch_sub(__op) - __op;} 1730 _LIBCPP_INLINE_VISIBILITY 1731 _Tp operator-=(_Tp __op) _NOEXCEPT {return fetch_sub(__op) - __op;} 1732 _LIBCPP_INLINE_VISIBILITY 1733 _Tp operator&=(_Tp __op) volatile _NOEXCEPT {return fetch_and(__op) & __op;} 1734 _LIBCPP_INLINE_VISIBILITY 1735 _Tp operator&=(_Tp __op) _NOEXCEPT {return fetch_and(__op) & __op;} 1736 _LIBCPP_INLINE_VISIBILITY 1737 _Tp operator|=(_Tp __op) volatile _NOEXCEPT {return fetch_or(__op) | __op;} 1738 _LIBCPP_INLINE_VISIBILITY 1739 _Tp operator|=(_Tp __op) _NOEXCEPT {return fetch_or(__op) | __op;} 1740 _LIBCPP_INLINE_VISIBILITY 1741 _Tp operator^=(_Tp __op) volatile _NOEXCEPT {return fetch_xor(__op) ^ __op;} 1742 _LIBCPP_INLINE_VISIBILITY 1743 _Tp operator^=(_Tp __op) _NOEXCEPT {return fetch_xor(__op) ^ __op;} 1744}; 1745 1746// atomic<T> 1747 1748template <class _Tp> 1749struct atomic 1750 : public __atomic_base<_Tp> 1751{ 1752 typedef __atomic_base<_Tp> __base; 1753 typedef _Tp value_type; 1754 typedef value_type difference_type; 1755 1756#if _LIBCPP_STD_VER > 17 1757 _LIBCPP_INLINE_VISIBILITY 1758 atomic() = default; 1759#else 1760 _LIBCPP_INLINE_VISIBILITY 1761 atomic() _NOEXCEPT = default; 1762#endif 1763 1764 _LIBCPP_INLINE_VISIBILITY 1765 _LIBCPP_CONSTEXPR atomic(_Tp __d) _NOEXCEPT : __base(__d) {} 1766 1767 _LIBCPP_INLINE_VISIBILITY 1768 _Tp operator=(_Tp __d) volatile _NOEXCEPT 1769 {__base::store(__d); return __d;} 1770 _LIBCPP_INLINE_VISIBILITY 1771 _Tp operator=(_Tp __d) _NOEXCEPT 1772 {__base::store(__d); return __d;} 1773 1774 atomic& operator=(const atomic&) = delete; 1775 atomic& operator=(const atomic&) volatile = delete; 1776}; 1777 1778// atomic<T*> 1779 1780template <class _Tp> 1781struct atomic<_Tp*> 1782 : public __atomic_base<_Tp*> 1783{ 1784 typedef __atomic_base<_Tp*> __base; 1785 typedef _Tp* value_type; 1786 typedef ptrdiff_t difference_type; 1787 1788 _LIBCPP_INLINE_VISIBILITY 1789 atomic() _NOEXCEPT = default; 1790 1791 _LIBCPP_INLINE_VISIBILITY 1792 _LIBCPP_CONSTEXPR atomic(_Tp* __d) _NOEXCEPT : __base(__d) {} 1793 1794 _LIBCPP_INLINE_VISIBILITY 1795 _Tp* operator=(_Tp* __d) volatile _NOEXCEPT 1796 {__base::store(__d); return __d;} 1797 _LIBCPP_INLINE_VISIBILITY 1798 _Tp* operator=(_Tp* __d) _NOEXCEPT 1799 {__base::store(__d); return __d;} 1800 1801 _LIBCPP_INLINE_VISIBILITY 1802 _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { 1803 // __atomic_fetch_add accepts function pointers, guard against them. 1804 static_assert(!is_function<typename remove_pointer<_Tp>::type>::value, "Pointer to function isn't allowed"); 1805 return __cxx_atomic_fetch_add(&this->__a_, __op, __m); 1806 } 1807 1808 _LIBCPP_INLINE_VISIBILITY 1809 _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { 1810 // __atomic_fetch_add accepts function pointers, guard against them. 1811 static_assert(!is_function<typename remove_pointer<_Tp>::type>::value, "Pointer to function isn't allowed"); 1812 return __cxx_atomic_fetch_add(&this->__a_, __op, __m); 1813 } 1814 1815 _LIBCPP_INLINE_VISIBILITY 1816 _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { 1817 // __atomic_fetch_add accepts function pointers, guard against them. 1818 static_assert(!is_function<typename remove_pointer<_Tp>::type>::value, "Pointer to function isn't allowed"); 1819 return __cxx_atomic_fetch_sub(&this->__a_, __op, __m); 1820 } 1821 1822 _LIBCPP_INLINE_VISIBILITY 1823 _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { 1824 // __atomic_fetch_add accepts function pointers, guard against them. 1825 static_assert(!is_function<typename remove_pointer<_Tp>::type>::value, "Pointer to function isn't allowed"); 1826 return __cxx_atomic_fetch_sub(&this->__a_, __op, __m); 1827 } 1828 1829 _LIBCPP_INLINE_VISIBILITY 1830 _Tp* operator++(int) volatile _NOEXCEPT {return fetch_add(1);} 1831 _LIBCPP_INLINE_VISIBILITY 1832 _Tp* operator++(int) _NOEXCEPT {return fetch_add(1);} 1833 _LIBCPP_INLINE_VISIBILITY 1834 _Tp* operator--(int) volatile _NOEXCEPT {return fetch_sub(1);} 1835 _LIBCPP_INLINE_VISIBILITY 1836 _Tp* operator--(int) _NOEXCEPT {return fetch_sub(1);} 1837 _LIBCPP_INLINE_VISIBILITY 1838 _Tp* operator++() volatile _NOEXCEPT {return fetch_add(1) + 1;} 1839 _LIBCPP_INLINE_VISIBILITY 1840 _Tp* operator++() _NOEXCEPT {return fetch_add(1) + 1;} 1841 _LIBCPP_INLINE_VISIBILITY 1842 _Tp* operator--() volatile _NOEXCEPT {return fetch_sub(1) - 1;} 1843 _LIBCPP_INLINE_VISIBILITY 1844 _Tp* operator--() _NOEXCEPT {return fetch_sub(1) - 1;} 1845 _LIBCPP_INLINE_VISIBILITY 1846 _Tp* operator+=(ptrdiff_t __op) volatile _NOEXCEPT {return fetch_add(__op) + __op;} 1847 _LIBCPP_INLINE_VISIBILITY 1848 _Tp* operator+=(ptrdiff_t __op) _NOEXCEPT {return fetch_add(__op) + __op;} 1849 _LIBCPP_INLINE_VISIBILITY 1850 _Tp* operator-=(ptrdiff_t __op) volatile _NOEXCEPT {return fetch_sub(__op) - __op;} 1851 _LIBCPP_INLINE_VISIBILITY 1852 _Tp* operator-=(ptrdiff_t __op) _NOEXCEPT {return fetch_sub(__op) - __op;} 1853 1854 atomic& operator=(const atomic&) = delete; 1855 atomic& operator=(const atomic&) volatile = delete; 1856}; 1857 1858// atomic_is_lock_free 1859 1860template <class _Tp> 1861_LIBCPP_INLINE_VISIBILITY 1862bool 1863atomic_is_lock_free(const volatile atomic<_Tp>* __o) _NOEXCEPT 1864{ 1865 return __o->is_lock_free(); 1866} 1867 1868template <class _Tp> 1869_LIBCPP_INLINE_VISIBILITY 1870bool 1871atomic_is_lock_free(const atomic<_Tp>* __o) _NOEXCEPT 1872{ 1873 return __o->is_lock_free(); 1874} 1875 1876// atomic_init 1877 1878template <class _Tp> 1879_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_INLINE_VISIBILITY 1880void 1881atomic_init(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1882{ 1883 __cxx_atomic_init(&__o->__a_, __d); 1884} 1885 1886template <class _Tp> 1887_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_INLINE_VISIBILITY 1888void 1889atomic_init(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1890{ 1891 __cxx_atomic_init(&__o->__a_, __d); 1892} 1893 1894// atomic_store 1895 1896template <class _Tp> 1897_LIBCPP_INLINE_VISIBILITY 1898void 1899atomic_store(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1900{ 1901 __o->store(__d); 1902} 1903 1904template <class _Tp> 1905_LIBCPP_INLINE_VISIBILITY 1906void 1907atomic_store(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1908{ 1909 __o->store(__d); 1910} 1911 1912// atomic_store_explicit 1913 1914template <class _Tp> 1915_LIBCPP_INLINE_VISIBILITY 1916void 1917atomic_store_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 1918 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1919{ 1920 __o->store(__d, __m); 1921} 1922 1923template <class _Tp> 1924_LIBCPP_INLINE_VISIBILITY 1925void 1926atomic_store_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 1927 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1928{ 1929 __o->store(__d, __m); 1930} 1931 1932// atomic_load 1933 1934template <class _Tp> 1935_LIBCPP_INLINE_VISIBILITY 1936_Tp 1937atomic_load(const volatile atomic<_Tp>* __o) _NOEXCEPT 1938{ 1939 return __o->load(); 1940} 1941 1942template <class _Tp> 1943_LIBCPP_INLINE_VISIBILITY 1944_Tp 1945atomic_load(const atomic<_Tp>* __o) _NOEXCEPT 1946{ 1947 return __o->load(); 1948} 1949 1950// atomic_load_explicit 1951 1952template <class _Tp> 1953_LIBCPP_INLINE_VISIBILITY 1954_Tp 1955atomic_load_explicit(const volatile atomic<_Tp>* __o, memory_order __m) _NOEXCEPT 1956 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 1957{ 1958 return __o->load(__m); 1959} 1960 1961template <class _Tp> 1962_LIBCPP_INLINE_VISIBILITY 1963_Tp 1964atomic_load_explicit(const atomic<_Tp>* __o, memory_order __m) _NOEXCEPT 1965 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 1966{ 1967 return __o->load(__m); 1968} 1969 1970// atomic_exchange 1971 1972template <class _Tp> 1973_LIBCPP_INLINE_VISIBILITY 1974_Tp 1975atomic_exchange(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1976{ 1977 return __o->exchange(__d); 1978} 1979 1980template <class _Tp> 1981_LIBCPP_INLINE_VISIBILITY 1982_Tp 1983atomic_exchange(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1984{ 1985 return __o->exchange(__d); 1986} 1987 1988// atomic_exchange_explicit 1989 1990template <class _Tp> 1991_LIBCPP_INLINE_VISIBILITY 1992_Tp 1993atomic_exchange_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 1994{ 1995 return __o->exchange(__d, __m); 1996} 1997 1998template <class _Tp> 1999_LIBCPP_INLINE_VISIBILITY 2000_Tp 2001atomic_exchange_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 2002{ 2003 return __o->exchange(__d, __m); 2004} 2005 2006// atomic_compare_exchange_weak 2007 2008template <class _Tp> 2009_LIBCPP_INLINE_VISIBILITY 2010bool 2011atomic_compare_exchange_weak(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2012{ 2013 return __o->compare_exchange_weak(*__e, __d); 2014} 2015 2016template <class _Tp> 2017_LIBCPP_INLINE_VISIBILITY 2018bool 2019atomic_compare_exchange_weak(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2020{ 2021 return __o->compare_exchange_weak(*__e, __d); 2022} 2023 2024// atomic_compare_exchange_strong 2025 2026template <class _Tp> 2027_LIBCPP_INLINE_VISIBILITY 2028bool 2029atomic_compare_exchange_strong(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2030{ 2031 return __o->compare_exchange_strong(*__e, __d); 2032} 2033 2034template <class _Tp> 2035_LIBCPP_INLINE_VISIBILITY 2036bool 2037atomic_compare_exchange_strong(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2038{ 2039 return __o->compare_exchange_strong(*__e, __d); 2040} 2041 2042// atomic_compare_exchange_weak_explicit 2043 2044template <class _Tp> 2045_LIBCPP_INLINE_VISIBILITY 2046bool 2047atomic_compare_exchange_weak_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, 2048 typename atomic<_Tp>::value_type __d, 2049 memory_order __s, memory_order __f) _NOEXCEPT 2050 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2051{ 2052 return __o->compare_exchange_weak(*__e, __d, __s, __f); 2053} 2054 2055template <class _Tp> 2056_LIBCPP_INLINE_VISIBILITY 2057bool 2058atomic_compare_exchange_weak_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d, 2059 memory_order __s, memory_order __f) _NOEXCEPT 2060 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2061{ 2062 return __o->compare_exchange_weak(*__e, __d, __s, __f); 2063} 2064 2065// atomic_compare_exchange_strong_explicit 2066 2067template <class _Tp> 2068_LIBCPP_INLINE_VISIBILITY 2069bool 2070atomic_compare_exchange_strong_explicit(volatile atomic<_Tp>* __o, 2071 typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d, 2072 memory_order __s, memory_order __f) _NOEXCEPT 2073 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2074{ 2075 return __o->compare_exchange_strong(*__e, __d, __s, __f); 2076} 2077 2078template <class _Tp> 2079_LIBCPP_INLINE_VISIBILITY 2080bool 2081atomic_compare_exchange_strong_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, 2082 typename atomic<_Tp>::value_type __d, 2083 memory_order __s, memory_order __f) _NOEXCEPT 2084 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2085{ 2086 return __o->compare_exchange_strong(*__e, __d, __s, __f); 2087} 2088 2089// atomic_wait 2090 2091template <class _Tp> 2092_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2093void atomic_wait(const volatile atomic<_Tp>* __o, 2094 typename atomic<_Tp>::value_type __v) _NOEXCEPT 2095{ 2096 return __o->wait(__v); 2097} 2098 2099template <class _Tp> 2100_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2101void atomic_wait(const atomic<_Tp>* __o, 2102 typename atomic<_Tp>::value_type __v) _NOEXCEPT 2103{ 2104 return __o->wait(__v); 2105} 2106 2107// atomic_wait_explicit 2108 2109template <class _Tp> 2110_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2111void atomic_wait_explicit(const volatile atomic<_Tp>* __o, 2112 typename atomic<_Tp>::value_type __v, 2113 memory_order __m) _NOEXCEPT 2114 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 2115{ 2116 return __o->wait(__v, __m); 2117} 2118 2119template <class _Tp> 2120_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2121void atomic_wait_explicit(const atomic<_Tp>* __o, 2122 typename atomic<_Tp>::value_type __v, 2123 memory_order __m) _NOEXCEPT 2124 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 2125{ 2126 return __o->wait(__v, __m); 2127} 2128 2129// atomic_notify_one 2130 2131template <class _Tp> 2132_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2133void atomic_notify_one(volatile atomic<_Tp>* __o) _NOEXCEPT 2134{ 2135 __o->notify_one(); 2136} 2137template <class _Tp> 2138_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2139void atomic_notify_one(atomic<_Tp>* __o) _NOEXCEPT 2140{ 2141 __o->notify_one(); 2142} 2143 2144// atomic_notify_one 2145 2146template <class _Tp> 2147_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2148void atomic_notify_all(volatile atomic<_Tp>* __o) _NOEXCEPT 2149{ 2150 __o->notify_all(); 2151} 2152template <class _Tp> 2153_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2154void atomic_notify_all(atomic<_Tp>* __o) _NOEXCEPT 2155{ 2156 __o->notify_all(); 2157} 2158 2159// atomic_fetch_add 2160 2161template <class _Tp> 2162_LIBCPP_INLINE_VISIBILITY 2163_Tp 2164atomic_fetch_add(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT 2165{ 2166 return __o->fetch_add(__op); 2167} 2168 2169template <class _Tp> 2170_LIBCPP_INLINE_VISIBILITY 2171_Tp 2172atomic_fetch_add(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT 2173{ 2174 return __o->fetch_add(__op); 2175} 2176 2177// atomic_fetch_add_explicit 2178 2179template <class _Tp> 2180_LIBCPP_INLINE_VISIBILITY 2181_Tp atomic_fetch_add_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT 2182{ 2183 return __o->fetch_add(__op, __m); 2184} 2185 2186template <class _Tp> 2187_LIBCPP_INLINE_VISIBILITY 2188_Tp atomic_fetch_add_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT 2189{ 2190 return __o->fetch_add(__op, __m); 2191} 2192 2193// atomic_fetch_sub 2194 2195template <class _Tp> 2196_LIBCPP_INLINE_VISIBILITY 2197_Tp atomic_fetch_sub(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT 2198{ 2199 return __o->fetch_sub(__op); 2200} 2201 2202template <class _Tp> 2203_LIBCPP_INLINE_VISIBILITY 2204_Tp atomic_fetch_sub(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT 2205{ 2206 return __o->fetch_sub(__op); 2207} 2208 2209// atomic_fetch_sub_explicit 2210 2211template <class _Tp> 2212_LIBCPP_INLINE_VISIBILITY 2213_Tp atomic_fetch_sub_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT 2214{ 2215 return __o->fetch_sub(__op, __m); 2216} 2217 2218template <class _Tp> 2219_LIBCPP_INLINE_VISIBILITY 2220_Tp atomic_fetch_sub_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT 2221{ 2222 return __o->fetch_sub(__op, __m); 2223} 2224 2225// atomic_fetch_and 2226 2227template <class _Tp> 2228_LIBCPP_INLINE_VISIBILITY 2229typename enable_if 2230< 2231 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2232 _Tp 2233>::type 2234atomic_fetch_and(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2235{ 2236 return __o->fetch_and(__op); 2237} 2238 2239template <class _Tp> 2240_LIBCPP_INLINE_VISIBILITY 2241typename enable_if 2242< 2243 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2244 _Tp 2245>::type 2246atomic_fetch_and(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2247{ 2248 return __o->fetch_and(__op); 2249} 2250 2251// atomic_fetch_and_explicit 2252 2253template <class _Tp> 2254_LIBCPP_INLINE_VISIBILITY 2255typename enable_if 2256< 2257 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2258 _Tp 2259>::type 2260atomic_fetch_and_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2261{ 2262 return __o->fetch_and(__op, __m); 2263} 2264 2265template <class _Tp> 2266_LIBCPP_INLINE_VISIBILITY 2267typename enable_if 2268< 2269 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2270 _Tp 2271>::type 2272atomic_fetch_and_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2273{ 2274 return __o->fetch_and(__op, __m); 2275} 2276 2277// atomic_fetch_or 2278 2279template <class _Tp> 2280_LIBCPP_INLINE_VISIBILITY 2281typename enable_if 2282< 2283 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2284 _Tp 2285>::type 2286atomic_fetch_or(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2287{ 2288 return __o->fetch_or(__op); 2289} 2290 2291template <class _Tp> 2292_LIBCPP_INLINE_VISIBILITY 2293typename enable_if 2294< 2295 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2296 _Tp 2297>::type 2298atomic_fetch_or(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2299{ 2300 return __o->fetch_or(__op); 2301} 2302 2303// atomic_fetch_or_explicit 2304 2305template <class _Tp> 2306_LIBCPP_INLINE_VISIBILITY 2307typename enable_if 2308< 2309 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2310 _Tp 2311>::type 2312atomic_fetch_or_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2313{ 2314 return __o->fetch_or(__op, __m); 2315} 2316 2317template <class _Tp> 2318_LIBCPP_INLINE_VISIBILITY 2319typename enable_if 2320< 2321 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2322 _Tp 2323>::type 2324atomic_fetch_or_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2325{ 2326 return __o->fetch_or(__op, __m); 2327} 2328 2329// atomic_fetch_xor 2330 2331template <class _Tp> 2332_LIBCPP_INLINE_VISIBILITY 2333typename enable_if 2334< 2335 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2336 _Tp 2337>::type 2338atomic_fetch_xor(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2339{ 2340 return __o->fetch_xor(__op); 2341} 2342 2343template <class _Tp> 2344_LIBCPP_INLINE_VISIBILITY 2345typename enable_if 2346< 2347 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2348 _Tp 2349>::type 2350atomic_fetch_xor(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2351{ 2352 return __o->fetch_xor(__op); 2353} 2354 2355// atomic_fetch_xor_explicit 2356 2357template <class _Tp> 2358_LIBCPP_INLINE_VISIBILITY 2359typename enable_if 2360< 2361 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2362 _Tp 2363>::type 2364atomic_fetch_xor_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2365{ 2366 return __o->fetch_xor(__op, __m); 2367} 2368 2369template <class _Tp> 2370_LIBCPP_INLINE_VISIBILITY 2371typename enable_if 2372< 2373 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2374 _Tp 2375>::type 2376atomic_fetch_xor_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2377{ 2378 return __o->fetch_xor(__op, __m); 2379} 2380 2381// flag type and operations 2382 2383typedef struct atomic_flag 2384{ 2385 __cxx_atomic_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_; 2386 2387 _LIBCPP_INLINE_VISIBILITY 2388 bool test(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 2389 {return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m);} 2390 _LIBCPP_INLINE_VISIBILITY 2391 bool test(memory_order __m = memory_order_seq_cst) const _NOEXCEPT 2392 {return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m);} 2393 2394 _LIBCPP_INLINE_VISIBILITY 2395 bool test_and_set(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 2396 {return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m);} 2397 _LIBCPP_INLINE_VISIBILITY 2398 bool test_and_set(memory_order __m = memory_order_seq_cst) _NOEXCEPT 2399 {return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m);} 2400 _LIBCPP_INLINE_VISIBILITY 2401 void clear(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 2402 {__cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m);} 2403 _LIBCPP_INLINE_VISIBILITY 2404 void clear(memory_order __m = memory_order_seq_cst) _NOEXCEPT 2405 {__cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m);} 2406 2407 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2408 void wait(bool __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 2409 {__cxx_atomic_wait(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);} 2410 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2411 void wait(bool __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT 2412 {__cxx_atomic_wait(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);} 2413 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2414 void notify_one() volatile _NOEXCEPT 2415 {__cxx_atomic_notify_one(&__a_);} 2416 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2417 void notify_one() _NOEXCEPT 2418 {__cxx_atomic_notify_one(&__a_);} 2419 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2420 void notify_all() volatile _NOEXCEPT 2421 {__cxx_atomic_notify_all(&__a_);} 2422 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2423 void notify_all() _NOEXCEPT 2424 {__cxx_atomic_notify_all(&__a_);} 2425 2426#if _LIBCPP_STD_VER > 17 2427 _LIBCPP_INLINE_VISIBILITY constexpr 2428 atomic_flag() _NOEXCEPT : __a_(false) {} 2429#else 2430 _LIBCPP_INLINE_VISIBILITY 2431 atomic_flag() _NOEXCEPT = default; 2432#endif 2433 2434 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR 2435 atomic_flag(bool __b) _NOEXCEPT : __a_(__b) {} // EXTENSION 2436 2437 atomic_flag(const atomic_flag&) = delete; 2438 atomic_flag& operator=(const atomic_flag&) = delete; 2439 atomic_flag& operator=(const atomic_flag&) volatile = delete; 2440 2441} atomic_flag; 2442 2443 2444inline _LIBCPP_INLINE_VISIBILITY 2445bool 2446atomic_flag_test(const volatile atomic_flag* __o) _NOEXCEPT 2447{ 2448 return __o->test(); 2449} 2450 2451inline _LIBCPP_INLINE_VISIBILITY 2452bool 2453atomic_flag_test(const atomic_flag* __o) _NOEXCEPT 2454{ 2455 return __o->test(); 2456} 2457 2458inline _LIBCPP_INLINE_VISIBILITY 2459bool 2460atomic_flag_test_explicit(const volatile atomic_flag* __o, memory_order __m) _NOEXCEPT 2461{ 2462 return __o->test(__m); 2463} 2464 2465inline _LIBCPP_INLINE_VISIBILITY 2466bool 2467atomic_flag_test_explicit(const atomic_flag* __o, memory_order __m) _NOEXCEPT 2468{ 2469 return __o->test(__m); 2470} 2471 2472inline _LIBCPP_INLINE_VISIBILITY 2473bool 2474atomic_flag_test_and_set(volatile atomic_flag* __o) _NOEXCEPT 2475{ 2476 return __o->test_and_set(); 2477} 2478 2479inline _LIBCPP_INLINE_VISIBILITY 2480bool 2481atomic_flag_test_and_set(atomic_flag* __o) _NOEXCEPT 2482{ 2483 return __o->test_and_set(); 2484} 2485 2486inline _LIBCPP_INLINE_VISIBILITY 2487bool 2488atomic_flag_test_and_set_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT 2489{ 2490 return __o->test_and_set(__m); 2491} 2492 2493inline _LIBCPP_INLINE_VISIBILITY 2494bool 2495atomic_flag_test_and_set_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT 2496{ 2497 return __o->test_and_set(__m); 2498} 2499 2500inline _LIBCPP_INLINE_VISIBILITY 2501void 2502atomic_flag_clear(volatile atomic_flag* __o) _NOEXCEPT 2503{ 2504 __o->clear(); 2505} 2506 2507inline _LIBCPP_INLINE_VISIBILITY 2508void 2509atomic_flag_clear(atomic_flag* __o) _NOEXCEPT 2510{ 2511 __o->clear(); 2512} 2513 2514inline _LIBCPP_INLINE_VISIBILITY 2515void 2516atomic_flag_clear_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT 2517{ 2518 __o->clear(__m); 2519} 2520 2521inline _LIBCPP_INLINE_VISIBILITY 2522void 2523atomic_flag_clear_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT 2524{ 2525 __o->clear(__m); 2526} 2527 2528inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2529void 2530atomic_flag_wait(const volatile atomic_flag* __o, bool __v) _NOEXCEPT 2531{ 2532 __o->wait(__v); 2533} 2534 2535inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2536void 2537atomic_flag_wait(const atomic_flag* __o, bool __v) _NOEXCEPT 2538{ 2539 __o->wait(__v); 2540} 2541 2542inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2543void 2544atomic_flag_wait_explicit(const volatile atomic_flag* __o, 2545 bool __v, memory_order __m) _NOEXCEPT 2546{ 2547 __o->wait(__v, __m); 2548} 2549 2550inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2551void 2552atomic_flag_wait_explicit(const atomic_flag* __o, 2553 bool __v, memory_order __m) _NOEXCEPT 2554{ 2555 __o->wait(__v, __m); 2556} 2557 2558inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2559void 2560atomic_flag_notify_one(volatile atomic_flag* __o) _NOEXCEPT 2561{ 2562 __o->notify_one(); 2563} 2564 2565inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2566void 2567atomic_flag_notify_one(atomic_flag* __o) _NOEXCEPT 2568{ 2569 __o->notify_one(); 2570} 2571 2572inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2573void 2574atomic_flag_notify_all(volatile atomic_flag* __o) _NOEXCEPT 2575{ 2576 __o->notify_all(); 2577} 2578 2579inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2580void 2581atomic_flag_notify_all(atomic_flag* __o) _NOEXCEPT 2582{ 2583 __o->notify_all(); 2584} 2585 2586// fences 2587 2588inline _LIBCPP_INLINE_VISIBILITY 2589void 2590atomic_thread_fence(memory_order __m) _NOEXCEPT 2591{ 2592 __cxx_atomic_thread_fence(__m); 2593} 2594 2595inline _LIBCPP_INLINE_VISIBILITY 2596void 2597atomic_signal_fence(memory_order __m) _NOEXCEPT 2598{ 2599 __cxx_atomic_signal_fence(__m); 2600} 2601 2602// Atomics for standard typedef types 2603 2604typedef atomic<bool> atomic_bool; 2605typedef atomic<char> atomic_char; 2606typedef atomic<signed char> atomic_schar; 2607typedef atomic<unsigned char> atomic_uchar; 2608typedef atomic<short> atomic_short; 2609typedef atomic<unsigned short> atomic_ushort; 2610typedef atomic<int> atomic_int; 2611typedef atomic<unsigned int> atomic_uint; 2612typedef atomic<long> atomic_long; 2613typedef atomic<unsigned long> atomic_ulong; 2614typedef atomic<long long> atomic_llong; 2615typedef atomic<unsigned long long> atomic_ullong; 2616#ifndef _LIBCPP_HAS_NO_CHAR8_T 2617typedef atomic<char8_t> atomic_char8_t; 2618#endif 2619typedef atomic<char16_t> atomic_char16_t; 2620typedef atomic<char32_t> atomic_char32_t; 2621#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS 2622typedef atomic<wchar_t> atomic_wchar_t; 2623#endif 2624 2625typedef atomic<int_least8_t> atomic_int_least8_t; 2626typedef atomic<uint_least8_t> atomic_uint_least8_t; 2627typedef atomic<int_least16_t> atomic_int_least16_t; 2628typedef atomic<uint_least16_t> atomic_uint_least16_t; 2629typedef atomic<int_least32_t> atomic_int_least32_t; 2630typedef atomic<uint_least32_t> atomic_uint_least32_t; 2631typedef atomic<int_least64_t> atomic_int_least64_t; 2632typedef atomic<uint_least64_t> atomic_uint_least64_t; 2633 2634typedef atomic<int_fast8_t> atomic_int_fast8_t; 2635typedef atomic<uint_fast8_t> atomic_uint_fast8_t; 2636typedef atomic<int_fast16_t> atomic_int_fast16_t; 2637typedef atomic<uint_fast16_t> atomic_uint_fast16_t; 2638typedef atomic<int_fast32_t> atomic_int_fast32_t; 2639typedef atomic<uint_fast32_t> atomic_uint_fast32_t; 2640typedef atomic<int_fast64_t> atomic_int_fast64_t; 2641typedef atomic<uint_fast64_t> atomic_uint_fast64_t; 2642 2643typedef atomic< int8_t> atomic_int8_t; 2644typedef atomic<uint8_t> atomic_uint8_t; 2645typedef atomic< int16_t> atomic_int16_t; 2646typedef atomic<uint16_t> atomic_uint16_t; 2647typedef atomic< int32_t> atomic_int32_t; 2648typedef atomic<uint32_t> atomic_uint32_t; 2649typedef atomic< int64_t> atomic_int64_t; 2650typedef atomic<uint64_t> atomic_uint64_t; 2651 2652typedef atomic<intptr_t> atomic_intptr_t; 2653typedef atomic<uintptr_t> atomic_uintptr_t; 2654typedef atomic<size_t> atomic_size_t; 2655typedef atomic<ptrdiff_t> atomic_ptrdiff_t; 2656typedef atomic<intmax_t> atomic_intmax_t; 2657typedef atomic<uintmax_t> atomic_uintmax_t; 2658 2659// atomic_*_lock_free : prefer the contention type most highly, then the largest lock-free type 2660 2661#ifdef __cpp_lib_atomic_is_always_lock_free 2662# define _LIBCPP_CONTENTION_LOCK_FREE __atomic_always_lock_free(sizeof(__cxx_contention_t), 0) 2663#else 2664# define _LIBCPP_CONTENTION_LOCK_FREE false 2665#endif 2666 2667#if ATOMIC_LLONG_LOCK_FREE == 2 2668typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, long long>::type __libcpp_signed_lock_free; 2669typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned long long>::type __libcpp_unsigned_lock_free; 2670#elif ATOMIC_INT_LOCK_FREE == 2 2671typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, int>::type __libcpp_signed_lock_free; 2672typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned int>::type __libcpp_unsigned_lock_free; 2673#elif ATOMIC_SHORT_LOCK_FREE == 2 2674typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, short>::type __libcpp_signed_lock_free; 2675typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned short>::type __libcpp_unsigned_lock_free; 2676#elif ATOMIC_CHAR_LOCK_FREE == 2 2677typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, char>::type __libcpp_signed_lock_free; 2678typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned char>::type __libcpp_unsigned_lock_free; 2679#else 2680 // No signed/unsigned lock-free types 2681#define _LIBCPP_NO_LOCK_FREE_TYPES 2682#endif 2683 2684#if !defined(_LIBCPP_NO_LOCK_FREE_TYPES) 2685typedef atomic<__libcpp_signed_lock_free> atomic_signed_lock_free; 2686typedef atomic<__libcpp_unsigned_lock_free> atomic_unsigned_lock_free; 2687#endif 2688 2689#define ATOMIC_FLAG_INIT {false} 2690#define ATOMIC_VAR_INIT(__v) {__v} 2691 2692#if _LIBCPP_STD_VER > 17 && !defined(_LIBCPP_DISABLE_DEPRECATION_WARNINGS) 2693# if defined(_LIBCPP_CLANG_VER) && _LIBCPP_CLANG_VER >= 1400 2694# pragma clang deprecated(ATOMIC_FLAG_INIT) 2695# pragma clang deprecated(ATOMIC_VAR_INIT) 2696# endif 2697#endif // _LIBCPP_STD_VER > 17 && !defined(_LIBCPP_DISABLE_DEPRECATION_WARNINGS) 2698 2699_LIBCPP_END_NAMESPACE_STD 2700 2701#endif // _LIBCPP_ATOMIC 2702