1// -*- C++ -*- 2//===--------------------------- atomic -----------------------------------===// 3// 4// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 5// See https://llvm.org/LICENSE.txt for license information. 6// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 7// 8//===----------------------------------------------------------------------===// 9 10#ifndef _LIBCPP_ATOMIC 11#define _LIBCPP_ATOMIC 12 13/* 14 atomic synopsis 15 16namespace std 17{ 18 19// feature test macro [version.syn] 20 21#define __cpp_lib_atomic_is_always_lock_free 22#define __cpp_lib_atomic_flag_test 23#define __cpp_lib_atomic_lock_free_type_aliases 24#define __cpp_lib_atomic_wait 25 26 // order and consistency 27 28 enum memory_order: unspecified // enum class in C++20 29 { 30 relaxed, 31 consume, // load-consume 32 acquire, // load-acquire 33 release, // store-release 34 acq_rel, // store-release load-acquire 35 seq_cst // store-release load-acquire 36 }; 37 38 inline constexpr auto memory_order_relaxed = memory_order::relaxed; 39 inline constexpr auto memory_order_consume = memory_order::consume; 40 inline constexpr auto memory_order_acquire = memory_order::acquire; 41 inline constexpr auto memory_order_release = memory_order::release; 42 inline constexpr auto memory_order_acq_rel = memory_order::acq_rel; 43 inline constexpr auto memory_order_seq_cst = memory_order::seq_cst; 44 45template <class T> T kill_dependency(T y) noexcept; 46 47// lock-free property 48 49#define ATOMIC_BOOL_LOCK_FREE unspecified 50#define ATOMIC_CHAR_LOCK_FREE unspecified 51#define ATOMIC_CHAR8_T_LOCK_FREE unspecified // C++20 52#define ATOMIC_CHAR16_T_LOCK_FREE unspecified 53#define ATOMIC_CHAR32_T_LOCK_FREE unspecified 54#define ATOMIC_WCHAR_T_LOCK_FREE unspecified 55#define ATOMIC_SHORT_LOCK_FREE unspecified 56#define ATOMIC_INT_LOCK_FREE unspecified 57#define ATOMIC_LONG_LOCK_FREE unspecified 58#define ATOMIC_LLONG_LOCK_FREE unspecified 59#define ATOMIC_POINTER_LOCK_FREE unspecified 60 61template <class T> 62struct atomic 63{ 64 using value_type = T; 65 66 static constexpr bool is_always_lock_free; 67 bool is_lock_free() const volatile noexcept; 68 bool is_lock_free() const noexcept; 69 70 atomic() noexcept = default; // until C++20 71 constexpr atomic() noexcept(is_nothrow_default_constructible_v<T>); // since C++20 72 constexpr atomic(T desr) noexcept; 73 atomic(const atomic&) = delete; 74 atomic& operator=(const atomic&) = delete; 75 atomic& operator=(const atomic&) volatile = delete; 76 77 T load(memory_order m = memory_order_seq_cst) const volatile noexcept; 78 T load(memory_order m = memory_order_seq_cst) const noexcept; 79 operator T() const volatile noexcept; 80 operator T() const noexcept; 81 void store(T desr, memory_order m = memory_order_seq_cst) volatile noexcept; 82 void store(T desr, memory_order m = memory_order_seq_cst) noexcept; 83 T operator=(T) volatile noexcept; 84 T operator=(T) noexcept; 85 86 T exchange(T desr, memory_order m = memory_order_seq_cst) volatile noexcept; 87 T exchange(T desr, memory_order m = memory_order_seq_cst) noexcept; 88 bool compare_exchange_weak(T& expc, T desr, 89 memory_order s, memory_order f) volatile noexcept; 90 bool compare_exchange_weak(T& expc, T desr, memory_order s, memory_order f) noexcept; 91 bool compare_exchange_strong(T& expc, T desr, 92 memory_order s, memory_order f) volatile noexcept; 93 bool compare_exchange_strong(T& expc, T desr, 94 memory_order s, memory_order f) noexcept; 95 bool compare_exchange_weak(T& expc, T desr, 96 memory_order m = memory_order_seq_cst) volatile noexcept; 97 bool compare_exchange_weak(T& expc, T desr, 98 memory_order m = memory_order_seq_cst) noexcept; 99 bool compare_exchange_strong(T& expc, T desr, 100 memory_order m = memory_order_seq_cst) volatile noexcept; 101 bool compare_exchange_strong(T& expc, T desr, 102 memory_order m = memory_order_seq_cst) noexcept; 103 104 void wait(T, memory_order = memory_order::seq_cst) const volatile noexcept; 105 void wait(T, memory_order = memory_order::seq_cst) const noexcept; 106 void notify_one() volatile noexcept; 107 void notify_one() noexcept; 108 void notify_all() volatile noexcept; 109 void notify_all() noexcept; 110}; 111 112template <> 113struct atomic<integral> 114{ 115 using value_type = integral; 116 using difference_type = value_type; 117 118 static constexpr bool is_always_lock_free; 119 bool is_lock_free() const volatile noexcept; 120 bool is_lock_free() const noexcept; 121 122 atomic() noexcept = default; 123 constexpr atomic(integral desr) noexcept; 124 atomic(const atomic&) = delete; 125 atomic& operator=(const atomic&) = delete; 126 atomic& operator=(const atomic&) volatile = delete; 127 128 integral load(memory_order m = memory_order_seq_cst) const volatile noexcept; 129 integral load(memory_order m = memory_order_seq_cst) const noexcept; 130 operator integral() const volatile noexcept; 131 operator integral() const noexcept; 132 void store(integral desr, memory_order m = memory_order_seq_cst) volatile noexcept; 133 void store(integral desr, memory_order m = memory_order_seq_cst) noexcept; 134 integral operator=(integral desr) volatile noexcept; 135 integral operator=(integral desr) noexcept; 136 137 integral exchange(integral desr, 138 memory_order m = memory_order_seq_cst) volatile noexcept; 139 integral exchange(integral desr, memory_order m = memory_order_seq_cst) noexcept; 140 bool compare_exchange_weak(integral& expc, integral desr, 141 memory_order s, memory_order f) volatile noexcept; 142 bool compare_exchange_weak(integral& expc, integral desr, 143 memory_order s, memory_order f) noexcept; 144 bool compare_exchange_strong(integral& expc, integral desr, 145 memory_order s, memory_order f) volatile noexcept; 146 bool compare_exchange_strong(integral& expc, integral desr, 147 memory_order s, memory_order f) noexcept; 148 bool compare_exchange_weak(integral& expc, integral desr, 149 memory_order m = memory_order_seq_cst) volatile noexcept; 150 bool compare_exchange_weak(integral& expc, integral desr, 151 memory_order m = memory_order_seq_cst) noexcept; 152 bool compare_exchange_strong(integral& expc, integral desr, 153 memory_order m = memory_order_seq_cst) volatile noexcept; 154 bool compare_exchange_strong(integral& expc, integral desr, 155 memory_order m = memory_order_seq_cst) noexcept; 156 157 integral fetch_add(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 158 integral fetch_add(integral op, memory_order m = memory_order_seq_cst) noexcept; 159 integral fetch_sub(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 160 integral fetch_sub(integral op, memory_order m = memory_order_seq_cst) noexcept; 161 integral fetch_and(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 162 integral fetch_and(integral op, memory_order m = memory_order_seq_cst) noexcept; 163 integral fetch_or(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 164 integral fetch_or(integral op, memory_order m = memory_order_seq_cst) noexcept; 165 integral fetch_xor(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 166 integral fetch_xor(integral op, memory_order m = memory_order_seq_cst) noexcept; 167 168 integral operator++(int) volatile noexcept; 169 integral operator++(int) noexcept; 170 integral operator--(int) volatile noexcept; 171 integral operator--(int) noexcept; 172 integral operator++() volatile noexcept; 173 integral operator++() noexcept; 174 integral operator--() volatile noexcept; 175 integral operator--() noexcept; 176 integral operator+=(integral op) volatile noexcept; 177 integral operator+=(integral op) noexcept; 178 integral operator-=(integral op) volatile noexcept; 179 integral operator-=(integral op) noexcept; 180 integral operator&=(integral op) volatile noexcept; 181 integral operator&=(integral op) noexcept; 182 integral operator|=(integral op) volatile noexcept; 183 integral operator|=(integral op) noexcept; 184 integral operator^=(integral op) volatile noexcept; 185 integral operator^=(integral op) noexcept; 186 187 void wait(integral, memory_order = memory_order::seq_cst) const volatile noexcept; 188 void wait(integral, memory_order = memory_order::seq_cst) const noexcept; 189 void notify_one() volatile noexcept; 190 void notify_one() noexcept; 191 void notify_all() volatile noexcept; 192 void notify_all() noexcept; 193}; 194 195template <class T> 196struct atomic<T*> 197{ 198 using value_type = T*; 199 using difference_type = ptrdiff_t; 200 201 static constexpr bool is_always_lock_free; 202 bool is_lock_free() const volatile noexcept; 203 bool is_lock_free() const noexcept; 204 205 atomic() noexcept = default; // until C++20 206 constexpr atomic() noexcept; // since C++20 207 constexpr atomic(T* desr) noexcept; 208 atomic(const atomic&) = delete; 209 atomic& operator=(const atomic&) = delete; 210 atomic& operator=(const atomic&) volatile = delete; 211 212 T* load(memory_order m = memory_order_seq_cst) const volatile noexcept; 213 T* load(memory_order m = memory_order_seq_cst) const noexcept; 214 operator T*() const volatile noexcept; 215 operator T*() const noexcept; 216 void store(T* desr, memory_order m = memory_order_seq_cst) volatile noexcept; 217 void store(T* desr, memory_order m = memory_order_seq_cst) noexcept; 218 T* operator=(T*) volatile noexcept; 219 T* operator=(T*) noexcept; 220 221 T* exchange(T* desr, memory_order m = memory_order_seq_cst) volatile noexcept; 222 T* exchange(T* desr, memory_order m = memory_order_seq_cst) noexcept; 223 bool compare_exchange_weak(T*& expc, T* desr, 224 memory_order s, memory_order f) volatile noexcept; 225 bool compare_exchange_weak(T*& expc, T* desr, 226 memory_order s, memory_order f) noexcept; 227 bool compare_exchange_strong(T*& expc, T* desr, 228 memory_order s, memory_order f) volatile noexcept; 229 bool compare_exchange_strong(T*& expc, T* desr, 230 memory_order s, memory_order f) noexcept; 231 bool compare_exchange_weak(T*& expc, T* desr, 232 memory_order m = memory_order_seq_cst) volatile noexcept; 233 bool compare_exchange_weak(T*& expc, T* desr, 234 memory_order m = memory_order_seq_cst) noexcept; 235 bool compare_exchange_strong(T*& expc, T* desr, 236 memory_order m = memory_order_seq_cst) volatile noexcept; 237 bool compare_exchange_strong(T*& expc, T* desr, 238 memory_order m = memory_order_seq_cst) noexcept; 239 T* fetch_add(ptrdiff_t op, memory_order m = memory_order_seq_cst) volatile noexcept; 240 T* fetch_add(ptrdiff_t op, memory_order m = memory_order_seq_cst) noexcept; 241 T* fetch_sub(ptrdiff_t op, memory_order m = memory_order_seq_cst) volatile noexcept; 242 T* fetch_sub(ptrdiff_t op, memory_order m = memory_order_seq_cst) noexcept; 243 244 T* operator++(int) volatile noexcept; 245 T* operator++(int) noexcept; 246 T* operator--(int) volatile noexcept; 247 T* operator--(int) noexcept; 248 T* operator++() volatile noexcept; 249 T* operator++() noexcept; 250 T* operator--() volatile noexcept; 251 T* operator--() noexcept; 252 T* operator+=(ptrdiff_t op) volatile noexcept; 253 T* operator+=(ptrdiff_t op) noexcept; 254 T* operator-=(ptrdiff_t op) volatile noexcept; 255 T* operator-=(ptrdiff_t op) noexcept; 256 257 void wait(T*, memory_order = memory_order::seq_cst) const volatile noexcept; 258 void wait(T*, memory_order = memory_order::seq_cst) const noexcept; 259 void notify_one() volatile noexcept; 260 void notify_one() noexcept; 261 void notify_all() volatile noexcept; 262 void notify_all() noexcept; 263}; 264 265 266// [atomics.nonmembers], non-member functions 267template<class T> 268 bool atomic_is_lock_free(const volatile atomic<T>*) noexcept; 269template<class T> 270 bool atomic_is_lock_free(const atomic<T>*) noexcept; 271template<class T> 272 void atomic_store(volatile atomic<T>*, atomic<T>::value_type) noexcept; 273template<class T> 274 void atomic_store(atomic<T>*, atomic<T>::value_type) noexcept; 275template<class T> 276 void atomic_store_explicit(volatile atomic<T>*, atomic<T>::value_type, 277 memory_order) noexcept; 278template<class T> 279 void atomic_store_explicit(atomic<T>*, atomic<T>::value_type, 280 memory_order) noexcept; 281template<class T> 282 T atomic_load(const volatile atomic<T>*) noexcept; 283template<class T> 284 T atomic_load(const atomic<T>*) noexcept; 285template<class T> 286 T atomic_load_explicit(const volatile atomic<T>*, memory_order) noexcept; 287template<class T> 288 T atomic_load_explicit(const atomic<T>*, memory_order) noexcept; 289template<class T> 290 T atomic_exchange(volatile atomic<T>*, atomic<T>::value_type) noexcept; 291template<class T> 292 T atomic_exchange(atomic<T>*, atomic<T>::value_type) noexcept; 293template<class T> 294 T atomic_exchange_explicit(volatile atomic<T>*, atomic<T>::value_type, 295 memory_order) noexcept; 296template<class T> 297 T atomic_exchange_explicit(atomic<T>*, atomic<T>::value_type, 298 memory_order) noexcept; 299template<class T> 300 bool atomic_compare_exchange_weak(volatile atomic<T>*, atomic<T>::value_type*, 301 atomic<T>::value_type) noexcept; 302template<class T> 303 bool atomic_compare_exchange_weak(atomic<T>*, atomic<T>::value_type*, 304 atomic<T>::value_type) noexcept; 305template<class T> 306 bool atomic_compare_exchange_strong(volatile atomic<T>*, atomic<T>::value_type*, 307 atomic<T>::value_type) noexcept; 308template<class T> 309 bool atomic_compare_exchange_strong(atomic<T>*, atomic<T>::value_type*, 310 atomic<T>::value_type) noexcept; 311template<class T> 312 bool atomic_compare_exchange_weak_explicit(volatile atomic<T>*, atomic<T>::value_type*, 313 atomic<T>::value_type, 314 memory_order, memory_order) noexcept; 315template<class T> 316 bool atomic_compare_exchange_weak_explicit(atomic<T>*, atomic<T>::value_type*, 317 atomic<T>::value_type, 318 memory_order, memory_order) noexcept; 319template<class T> 320 bool atomic_compare_exchange_strong_explicit(volatile atomic<T>*, atomic<T>::value_type*, 321 atomic<T>::value_type, 322 memory_order, memory_order) noexcept; 323template<class T> 324 bool atomic_compare_exchange_strong_explicit(atomic<T>*, atomic<T>::value_type*, 325 atomic<T>::value_type, 326 memory_order, memory_order) noexcept; 327 328template<class T> 329 T atomic_fetch_add(volatile atomic<T>*, atomic<T>::difference_type) noexcept; 330template<class T> 331 T atomic_fetch_add(atomic<T>*, atomic<T>::difference_type) noexcept; 332template<class T> 333 T atomic_fetch_add_explicit(volatile atomic<T>*, atomic<T>::difference_type, 334 memory_order) noexcept; 335template<class T> 336 T atomic_fetch_add_explicit(atomic<T>*, atomic<T>::difference_type, 337 memory_order) noexcept; 338template<class T> 339 T atomic_fetch_sub(volatile atomic<T>*, atomic<T>::difference_type) noexcept; 340template<class T> 341 T atomic_fetch_sub(atomic<T>*, atomic<T>::difference_type) noexcept; 342template<class T> 343 T atomic_fetch_sub_explicit(volatile atomic<T>*, atomic<T>::difference_type, 344 memory_order) noexcept; 345template<class T> 346 T atomic_fetch_sub_explicit(atomic<T>*, atomic<T>::difference_type, 347 memory_order) noexcept; 348template<class T> 349 T atomic_fetch_and(volatile atomic<T>*, atomic<T>::value_type) noexcept; 350template<class T> 351 T atomic_fetch_and(atomic<T>*, atomic<T>::value_type) noexcept; 352template<class T> 353 T atomic_fetch_and_explicit(volatile atomic<T>*, atomic<T>::value_type, 354 memory_order) noexcept; 355template<class T> 356 T atomic_fetch_and_explicit(atomic<T>*, atomic<T>::value_type, 357 memory_order) noexcept; 358template<class T> 359 T atomic_fetch_or(volatile atomic<T>*, atomic<T>::value_type) noexcept; 360template<class T> 361 T atomic_fetch_or(atomic<T>*, atomic<T>::value_type) noexcept; 362template<class T> 363 T atomic_fetch_or_explicit(volatile atomic<T>*, atomic<T>::value_type, 364 memory_order) noexcept; 365template<class T> 366 T atomic_fetch_or_explicit(atomic<T>*, atomic<T>::value_type, 367 memory_order) noexcept; 368template<class T> 369 T atomic_fetch_xor(volatile atomic<T>*, atomic<T>::value_type) noexcept; 370template<class T> 371 T atomic_fetch_xor(atomic<T>*, atomic<T>::value_type) noexcept; 372template<class T> 373 T atomic_fetch_xor_explicit(volatile atomic<T>*, atomic<T>::value_type, 374 memory_order) noexcept; 375template<class T> 376 T atomic_fetch_xor_explicit(atomic<T>*, atomic<T>::value_type, 377 memory_order) noexcept; 378 379template<class T> 380 void atomic_wait(const volatile atomic<T>*, atomic<T>::value_type); 381template<class T> 382 void atomic_wait(const atomic<T>*, atomic<T>::value_type); 383template<class T> 384 void atomic_wait_explicit(const volatile atomic<T>*, atomic<T>::value_type, 385 memory_order); 386template<class T> 387 void atomic_wait_explicit(const atomic<T>*, atomic<T>::value_type, 388 memory_order); 389template<class T> 390 void atomic_notify_one(volatile atomic<T>*); 391template<class T> 392 void atomic_notify_one(atomic<T>*); 393template<class T> 394 void atomic_notify_all(volatile atomic<T>*); 395template<class T> 396 void atomic_notify_all(atomic<T>*); 397 398// Atomics for standard typedef types 399 400typedef atomic<bool> atomic_bool; 401typedef atomic<char> atomic_char; 402typedef atomic<signed char> atomic_schar; 403typedef atomic<unsigned char> atomic_uchar; 404typedef atomic<short> atomic_short; 405typedef atomic<unsigned short> atomic_ushort; 406typedef atomic<int> atomic_int; 407typedef atomic<unsigned int> atomic_uint; 408typedef atomic<long> atomic_long; 409typedef atomic<unsigned long> atomic_ulong; 410typedef atomic<long long> atomic_llong; 411typedef atomic<unsigned long long> atomic_ullong; 412typedef atomic<char8_t> atomic_char8_t; // C++20 413typedef atomic<char16_t> atomic_char16_t; 414typedef atomic<char32_t> atomic_char32_t; 415typedef atomic<wchar_t> atomic_wchar_t; 416 417typedef atomic<int_least8_t> atomic_int_least8_t; 418typedef atomic<uint_least8_t> atomic_uint_least8_t; 419typedef atomic<int_least16_t> atomic_int_least16_t; 420typedef atomic<uint_least16_t> atomic_uint_least16_t; 421typedef atomic<int_least32_t> atomic_int_least32_t; 422typedef atomic<uint_least32_t> atomic_uint_least32_t; 423typedef atomic<int_least64_t> atomic_int_least64_t; 424typedef atomic<uint_least64_t> atomic_uint_least64_t; 425 426typedef atomic<int_fast8_t> atomic_int_fast8_t; 427typedef atomic<uint_fast8_t> atomic_uint_fast8_t; 428typedef atomic<int_fast16_t> atomic_int_fast16_t; 429typedef atomic<uint_fast16_t> atomic_uint_fast16_t; 430typedef atomic<int_fast32_t> atomic_int_fast32_t; 431typedef atomic<uint_fast32_t> atomic_uint_fast32_t; 432typedef atomic<int_fast64_t> atomic_int_fast64_t; 433typedef atomic<uint_fast64_t> atomic_uint_fast64_t; 434 435typedef atomic<int8_t> atomic_int8_t; 436typedef atomic<uint8_t> atomic_uint8_t; 437typedef atomic<int16_t> atomic_int16_t; 438typedef atomic<uint16_t> atomic_uint16_t; 439typedef atomic<int32_t> atomic_int32_t; 440typedef atomic<uint32_t> atomic_uint32_t; 441typedef atomic<int64_t> atomic_int64_t; 442typedef atomic<uint64_t> atomic_uint64_t; 443 444typedef atomic<intptr_t> atomic_intptr_t; 445typedef atomic<uintptr_t> atomic_uintptr_t; 446typedef atomic<size_t> atomic_size_t; 447typedef atomic<ptrdiff_t> atomic_ptrdiff_t; 448typedef atomic<intmax_t> atomic_intmax_t; 449typedef atomic<uintmax_t> atomic_uintmax_t; 450 451// flag type and operations 452 453typedef struct atomic_flag 454{ 455 atomic_flag() noexcept = default; // until C++20 456 constexpr atomic_flag() noexcept; // since C++20 457 atomic_flag(const atomic_flag&) = delete; 458 atomic_flag& operator=(const atomic_flag&) = delete; 459 atomic_flag& operator=(const atomic_flag&) volatile = delete; 460 461 bool test(memory_order m = memory_order_seq_cst) volatile noexcept; 462 bool test(memory_order m = memory_order_seq_cst) noexcept; 463 bool test_and_set(memory_order m = memory_order_seq_cst) volatile noexcept; 464 bool test_and_set(memory_order m = memory_order_seq_cst) noexcept; 465 void clear(memory_order m = memory_order_seq_cst) volatile noexcept; 466 void clear(memory_order m = memory_order_seq_cst) noexcept; 467 468 void wait(bool, memory_order = memory_order::seq_cst) const volatile noexcept; 469 void wait(bool, memory_order = memory_order::seq_cst) const noexcept; 470 void notify_one() volatile noexcept; 471 void notify_one() noexcept; 472 void notify_all() volatile noexcept; 473 void notify_all() noexcept; 474} atomic_flag; 475 476bool atomic_flag_test(volatile atomic_flag* obj) noexcept; 477bool atomic_flag_test(atomic_flag* obj) noexcept; 478bool atomic_flag_test_explicit(volatile atomic_flag* obj, 479 memory_order m) noexcept; 480bool atomic_flag_test_explicit(atomic_flag* obj, memory_order m) noexcept; 481bool atomic_flag_test_and_set(volatile atomic_flag* obj) noexcept; 482bool atomic_flag_test_and_set(atomic_flag* obj) noexcept; 483bool atomic_flag_test_and_set_explicit(volatile atomic_flag* obj, 484 memory_order m) noexcept; 485bool atomic_flag_test_and_set_explicit(atomic_flag* obj, memory_order m) noexcept; 486void atomic_flag_clear(volatile atomic_flag* obj) noexcept; 487void atomic_flag_clear(atomic_flag* obj) noexcept; 488void atomic_flag_clear_explicit(volatile atomic_flag* obj, memory_order m) noexcept; 489void atomic_flag_clear_explicit(atomic_flag* obj, memory_order m) noexcept; 490 491void atomic_wait(const volatile atomic_flag* obj, T old) noexcept; 492void atomic_wait(const atomic_flag* obj, T old) noexcept; 493void atomic_wait_explicit(const volatile atomic_flag* obj, T old, memory_order m) noexcept; 494void atomic_wait_explicit(const atomic_flag* obj, T old, memory_order m) noexcept; 495void atomic_one(volatile atomic_flag* obj) noexcept; 496void atomic_one(atomic_flag* obj) noexcept; 497void atomic_all(volatile atomic_flag* obj) noexcept; 498void atomic_all(atomic_flag* obj) noexcept; 499 500// fences 501 502void atomic_thread_fence(memory_order m) noexcept; 503void atomic_signal_fence(memory_order m) noexcept; 504 505// deprecated 506 507template <class T> 508 void atomic_init(volatile atomic<T>* obj, atomic<T>::value_type desr) noexcept; 509 510template <class T> 511 void atomic_init(atomic<T>* obj, atomic<T>::value_type desr) noexcept; 512 513#define ATOMIC_VAR_INIT(value) see below 514 515#define ATOMIC_FLAG_INIT see below 516 517} // std 518 519*/ 520 521#include <__availability> 522#include <__config> 523#include <__threading_support> 524#include <cstddef> 525#include <cstdint> 526#include <cstring> 527#include <type_traits> 528#include <version> 529 530#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) 531#pragma GCC system_header 532#endif 533 534#ifdef _LIBCPP_HAS_NO_THREADS 535# error <atomic> is not supported on this single threaded system 536#endif 537#ifdef _LIBCPP_HAS_NO_ATOMIC_HEADER 538# error <atomic> is not implemented 539#endif 540#ifdef kill_dependency 541# error C++ standard library is incompatible with <stdatomic.h> 542#endif 543 544#define _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) \ 545 _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_consume || \ 546 __m == memory_order_acquire || \ 547 __m == memory_order_acq_rel, \ 548 "memory order argument to atomic operation is invalid") 549 550#define _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) \ 551 _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_release || \ 552 __m == memory_order_acq_rel, \ 553 "memory order argument to atomic operation is invalid") 554 555#define _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__m, __f) \ 556 _LIBCPP_DIAGNOSE_WARNING(__f == memory_order_release || \ 557 __f == memory_order_acq_rel, \ 558 "memory order argument to atomic operation is invalid") 559 560_LIBCPP_BEGIN_NAMESPACE_STD 561 562// Figure out what the underlying type for `memory_order` would be if it were 563// declared as an unscoped enum (accounting for -fshort-enums). Use this result 564// to pin the underlying type in C++20. 565enum __legacy_memory_order { 566 __mo_relaxed, 567 __mo_consume, 568 __mo_acquire, 569 __mo_release, 570 __mo_acq_rel, 571 __mo_seq_cst 572}; 573 574typedef underlying_type<__legacy_memory_order>::type __memory_order_underlying_t; 575 576#if _LIBCPP_STD_VER > 17 577 578enum class memory_order : __memory_order_underlying_t { 579 relaxed = __mo_relaxed, 580 consume = __mo_consume, 581 acquire = __mo_acquire, 582 release = __mo_release, 583 acq_rel = __mo_acq_rel, 584 seq_cst = __mo_seq_cst 585}; 586 587inline constexpr auto memory_order_relaxed = memory_order::relaxed; 588inline constexpr auto memory_order_consume = memory_order::consume; 589inline constexpr auto memory_order_acquire = memory_order::acquire; 590inline constexpr auto memory_order_release = memory_order::release; 591inline constexpr auto memory_order_acq_rel = memory_order::acq_rel; 592inline constexpr auto memory_order_seq_cst = memory_order::seq_cst; 593 594#else 595 596typedef enum memory_order { 597 memory_order_relaxed = __mo_relaxed, 598 memory_order_consume = __mo_consume, 599 memory_order_acquire = __mo_acquire, 600 memory_order_release = __mo_release, 601 memory_order_acq_rel = __mo_acq_rel, 602 memory_order_seq_cst = __mo_seq_cst, 603} memory_order; 604 605#endif // _LIBCPP_STD_VER > 17 606 607template <typename _Tp> _LIBCPP_INLINE_VISIBILITY 608bool __cxx_nonatomic_compare_equal(_Tp const& __lhs, _Tp const& __rhs) { 609 return _VSTD::memcmp(&__lhs, &__rhs, sizeof(_Tp)) == 0; 610} 611 612static_assert((is_same<underlying_type<memory_order>::type, __memory_order_underlying_t>::value), 613 "unexpected underlying type for std::memory_order"); 614 615#if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) || \ 616 defined(_LIBCPP_ATOMIC_ONLY_USE_BUILTINS) 617 618// [atomics.types.generic]p1 guarantees _Tp is trivially copyable. Because 619// the default operator= in an object is not volatile, a byte-by-byte copy 620// is required. 621template <typename _Tp, typename _Tv> _LIBCPP_INLINE_VISIBILITY 622typename enable_if<is_assignable<_Tp&, _Tv>::value>::type 623__cxx_atomic_assign_volatile(_Tp& __a_value, _Tv const& __val) { 624 __a_value = __val; 625} 626template <typename _Tp, typename _Tv> _LIBCPP_INLINE_VISIBILITY 627typename enable_if<is_assignable<_Tp&, _Tv>::value>::type 628__cxx_atomic_assign_volatile(_Tp volatile& __a_value, _Tv volatile const& __val) { 629 volatile char* __to = reinterpret_cast<volatile char*>(&__a_value); 630 volatile char* __end = __to + sizeof(_Tp); 631 volatile const char* __from = reinterpret_cast<volatile const char*>(&__val); 632 while (__to != __end) 633 *__to++ = *__from++; 634} 635 636#endif 637 638#if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) 639 640template <typename _Tp> 641struct __cxx_atomic_base_impl { 642 643 _LIBCPP_INLINE_VISIBILITY 644#ifndef _LIBCPP_CXX03_LANG 645 __cxx_atomic_base_impl() _NOEXCEPT = default; 646#else 647 __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {} 648#endif // _LIBCPP_CXX03_LANG 649 _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT 650 : __a_value(value) {} 651 _Tp __a_value; 652}; 653 654_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR int __to_gcc_order(memory_order __order) { 655 // Avoid switch statement to make this a constexpr. 656 return __order == memory_order_relaxed ? __ATOMIC_RELAXED: 657 (__order == memory_order_acquire ? __ATOMIC_ACQUIRE: 658 (__order == memory_order_release ? __ATOMIC_RELEASE: 659 (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST: 660 (__order == memory_order_acq_rel ? __ATOMIC_ACQ_REL: 661 __ATOMIC_CONSUME)))); 662} 663 664_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR int __to_gcc_failure_order(memory_order __order) { 665 // Avoid switch statement to make this a constexpr. 666 return __order == memory_order_relaxed ? __ATOMIC_RELAXED: 667 (__order == memory_order_acquire ? __ATOMIC_ACQUIRE: 668 (__order == memory_order_release ? __ATOMIC_RELAXED: 669 (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST: 670 (__order == memory_order_acq_rel ? __ATOMIC_ACQUIRE: 671 __ATOMIC_CONSUME)))); 672} 673 674template <typename _Tp> 675_LIBCPP_INLINE_VISIBILITY 676void __cxx_atomic_init(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val) { 677 __cxx_atomic_assign_volatile(__a->__a_value, __val); 678} 679 680template <typename _Tp> 681_LIBCPP_INLINE_VISIBILITY 682void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) { 683 __a->__a_value = __val; 684} 685 686_LIBCPP_INLINE_VISIBILITY inline 687void __cxx_atomic_thread_fence(memory_order __order) { 688 __atomic_thread_fence(__to_gcc_order(__order)); 689} 690 691_LIBCPP_INLINE_VISIBILITY inline 692void __cxx_atomic_signal_fence(memory_order __order) { 693 __atomic_signal_fence(__to_gcc_order(__order)); 694} 695 696template <typename _Tp> 697_LIBCPP_INLINE_VISIBILITY 698void __cxx_atomic_store(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val, 699 memory_order __order) { 700 __atomic_store(&__a->__a_value, &__val, 701 __to_gcc_order(__order)); 702} 703 704template <typename _Tp> 705_LIBCPP_INLINE_VISIBILITY 706void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, 707 memory_order __order) { 708 __atomic_store(&__a->__a_value, &__val, 709 __to_gcc_order(__order)); 710} 711 712template <typename _Tp> 713_LIBCPP_INLINE_VISIBILITY 714_Tp __cxx_atomic_load(const volatile __cxx_atomic_base_impl<_Tp>* __a, 715 memory_order __order) { 716 _Tp __ret; 717 __atomic_load(&__a->__a_value, &__ret, 718 __to_gcc_order(__order)); 719 return __ret; 720} 721 722template <typename _Tp> 723_LIBCPP_INLINE_VISIBILITY 724_Tp __cxx_atomic_load(const __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) { 725 _Tp __ret; 726 __atomic_load(&__a->__a_value, &__ret, 727 __to_gcc_order(__order)); 728 return __ret; 729} 730 731template <typename _Tp> 732_LIBCPP_INLINE_VISIBILITY 733_Tp __cxx_atomic_exchange(volatile __cxx_atomic_base_impl<_Tp>* __a, 734 _Tp __value, memory_order __order) { 735 _Tp __ret; 736 __atomic_exchange(&__a->__a_value, &__value, &__ret, 737 __to_gcc_order(__order)); 738 return __ret; 739} 740 741template <typename _Tp> 742_LIBCPP_INLINE_VISIBILITY 743_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, 744 memory_order __order) { 745 _Tp __ret; 746 __atomic_exchange(&__a->__a_value, &__value, &__ret, 747 __to_gcc_order(__order)); 748 return __ret; 749} 750 751template <typename _Tp> 752_LIBCPP_INLINE_VISIBILITY 753bool __cxx_atomic_compare_exchange_strong( 754 volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, 755 memory_order __success, memory_order __failure) { 756 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 757 false, 758 __to_gcc_order(__success), 759 __to_gcc_failure_order(__failure)); 760} 761 762template <typename _Tp> 763_LIBCPP_INLINE_VISIBILITY 764bool __cxx_atomic_compare_exchange_strong( 765 __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, 766 memory_order __failure) { 767 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 768 false, 769 __to_gcc_order(__success), 770 __to_gcc_failure_order(__failure)); 771} 772 773template <typename _Tp> 774_LIBCPP_INLINE_VISIBILITY 775bool __cxx_atomic_compare_exchange_weak( 776 volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, 777 memory_order __success, memory_order __failure) { 778 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 779 true, 780 __to_gcc_order(__success), 781 __to_gcc_failure_order(__failure)); 782} 783 784template <typename _Tp> 785_LIBCPP_INLINE_VISIBILITY 786bool __cxx_atomic_compare_exchange_weak( 787 __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, 788 memory_order __failure) { 789 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 790 true, 791 __to_gcc_order(__success), 792 __to_gcc_failure_order(__failure)); 793} 794 795template <typename _Tp> 796struct __skip_amt { enum {value = 1}; }; 797 798template <typename _Tp> 799struct __skip_amt<_Tp*> { enum {value = sizeof(_Tp)}; }; 800 801// FIXME: Haven't figured out what the spec says about using arrays with 802// atomic_fetch_add. Force a failure rather than creating bad behavior. 803template <typename _Tp> 804struct __skip_amt<_Tp[]> { }; 805template <typename _Tp, int n> 806struct __skip_amt<_Tp[n]> { }; 807 808template <typename _Tp, typename _Td> 809_LIBCPP_INLINE_VISIBILITY 810_Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_base_impl<_Tp>* __a, 811 _Td __delta, memory_order __order) { 812 return __atomic_fetch_add(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 813 __to_gcc_order(__order)); 814} 815 816template <typename _Tp, typename _Td> 817_LIBCPP_INLINE_VISIBILITY 818_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, 819 memory_order __order) { 820 return __atomic_fetch_add(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 821 __to_gcc_order(__order)); 822} 823 824template <typename _Tp, typename _Td> 825_LIBCPP_INLINE_VISIBILITY 826_Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_base_impl<_Tp>* __a, 827 _Td __delta, memory_order __order) { 828 return __atomic_fetch_sub(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 829 __to_gcc_order(__order)); 830} 831 832template <typename _Tp, typename _Td> 833_LIBCPP_INLINE_VISIBILITY 834_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, 835 memory_order __order) { 836 return __atomic_fetch_sub(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 837 __to_gcc_order(__order)); 838} 839 840template <typename _Tp> 841_LIBCPP_INLINE_VISIBILITY 842_Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_base_impl<_Tp>* __a, 843 _Tp __pattern, memory_order __order) { 844 return __atomic_fetch_and(&__a->__a_value, __pattern, 845 __to_gcc_order(__order)); 846} 847 848template <typename _Tp> 849_LIBCPP_INLINE_VISIBILITY 850_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, 851 _Tp __pattern, memory_order __order) { 852 return __atomic_fetch_and(&__a->__a_value, __pattern, 853 __to_gcc_order(__order)); 854} 855 856template <typename _Tp> 857_LIBCPP_INLINE_VISIBILITY 858_Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_base_impl<_Tp>* __a, 859 _Tp __pattern, memory_order __order) { 860 return __atomic_fetch_or(&__a->__a_value, __pattern, 861 __to_gcc_order(__order)); 862} 863 864template <typename _Tp> 865_LIBCPP_INLINE_VISIBILITY 866_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, 867 memory_order __order) { 868 return __atomic_fetch_or(&__a->__a_value, __pattern, 869 __to_gcc_order(__order)); 870} 871 872template <typename _Tp> 873_LIBCPP_INLINE_VISIBILITY 874_Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_base_impl<_Tp>* __a, 875 _Tp __pattern, memory_order __order) { 876 return __atomic_fetch_xor(&__a->__a_value, __pattern, 877 __to_gcc_order(__order)); 878} 879 880template <typename _Tp> 881_LIBCPP_INLINE_VISIBILITY 882_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, 883 memory_order __order) { 884 return __atomic_fetch_xor(&__a->__a_value, __pattern, 885 __to_gcc_order(__order)); 886} 887 888#define __cxx_atomic_is_lock_free(__s) __atomic_is_lock_free(__s, 0) 889 890#elif defined(_LIBCPP_HAS_C_ATOMIC_IMP) 891 892template <typename _Tp> 893struct __cxx_atomic_base_impl { 894 895 _LIBCPP_INLINE_VISIBILITY 896#ifndef _LIBCPP_CXX03_LANG 897 __cxx_atomic_base_impl() _NOEXCEPT = default; 898#else 899 __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {} 900#endif // _LIBCPP_CXX03_LANG 901 _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT 902 : __a_value(value) {} 903 _LIBCPP_DISABLE_EXTENSION_WARNING _Atomic(_Tp) __a_value; 904}; 905 906#define __cxx_atomic_is_lock_free(__s) __c11_atomic_is_lock_free(__s) 907 908_LIBCPP_INLINE_VISIBILITY inline 909void __cxx_atomic_thread_fence(memory_order __order) _NOEXCEPT { 910 __c11_atomic_thread_fence(static_cast<__memory_order_underlying_t>(__order)); 911} 912 913_LIBCPP_INLINE_VISIBILITY inline 914void __cxx_atomic_signal_fence(memory_order __order) _NOEXCEPT { 915 __c11_atomic_signal_fence(static_cast<__memory_order_underlying_t>(__order)); 916} 917 918template<class _Tp> 919_LIBCPP_INLINE_VISIBILITY 920void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val) _NOEXCEPT { 921 __c11_atomic_init(&__a->__a_value, __val); 922} 923template<class _Tp> 924_LIBCPP_INLINE_VISIBILITY 925void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val) _NOEXCEPT { 926 __c11_atomic_init(&__a->__a_value, __val); 927} 928 929template<class _Tp> 930_LIBCPP_INLINE_VISIBILITY 931void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT { 932 __c11_atomic_store(&__a->__a_value, __val, static_cast<__memory_order_underlying_t>(__order)); 933} 934template<class _Tp> 935_LIBCPP_INLINE_VISIBILITY 936void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val, memory_order __order) _NOEXCEPT { 937 __c11_atomic_store(&__a->__a_value, __val, static_cast<__memory_order_underlying_t>(__order)); 938} 939 940template<class _Tp> 941_LIBCPP_INLINE_VISIBILITY 942_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const volatile* __a, memory_order __order) _NOEXCEPT { 943 using __ptr_type = typename remove_const<decltype(__a->__a_value)>::type*; 944 return __c11_atomic_load(const_cast<__ptr_type>(&__a->__a_value), static_cast<__memory_order_underlying_t>(__order)); 945} 946template<class _Tp> 947_LIBCPP_INLINE_VISIBILITY 948_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const* __a, memory_order __order) _NOEXCEPT { 949 using __ptr_type = typename remove_const<decltype(__a->__a_value)>::type*; 950 return __c11_atomic_load(const_cast<__ptr_type>(&__a->__a_value), static_cast<__memory_order_underlying_t>(__order)); 951} 952 953template<class _Tp> 954_LIBCPP_INLINE_VISIBILITY 955_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT { 956 return __c11_atomic_exchange(&__a->__a_value, __value, static_cast<__memory_order_underlying_t>(__order)); 957} 958template<class _Tp> 959_LIBCPP_INLINE_VISIBILITY 960_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> * __a, _Tp __value, memory_order __order) _NOEXCEPT { 961 return __c11_atomic_exchange(&__a->__a_value, __value, static_cast<__memory_order_underlying_t>(__order)); 962} 963 964_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR memory_order __to_failure_order(memory_order __order) { 965 // Avoid switch statement to make this a constexpr. 966 return __order == memory_order_release ? memory_order_relaxed: 967 (__order == memory_order_acq_rel ? memory_order_acquire: 968 __order); 969} 970 971template<class _Tp> 972_LIBCPP_INLINE_VISIBILITY 973bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 974 return __c11_atomic_compare_exchange_strong(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); 975} 976template<class _Tp> 977_LIBCPP_INLINE_VISIBILITY 978bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 979 return __c11_atomic_compare_exchange_strong(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); 980} 981 982template<class _Tp> 983_LIBCPP_INLINE_VISIBILITY 984bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 985 return __c11_atomic_compare_exchange_weak(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); 986} 987template<class _Tp> 988_LIBCPP_INLINE_VISIBILITY 989bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 990 return __c11_atomic_compare_exchange_weak(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); 991} 992 993template<class _Tp> 994_LIBCPP_INLINE_VISIBILITY 995_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT { 996 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 997} 998template<class _Tp> 999_LIBCPP_INLINE_VISIBILITY 1000_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1001 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1002} 1003 1004template<class _Tp> 1005_LIBCPP_INLINE_VISIBILITY 1006_Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1007 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1008} 1009template<class _Tp> 1010_LIBCPP_INLINE_VISIBILITY 1011_Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1012 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1013} 1014 1015template<class _Tp> 1016_LIBCPP_INLINE_VISIBILITY 1017_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1018 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1019} 1020template<class _Tp> 1021_LIBCPP_INLINE_VISIBILITY 1022_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1023 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1024} 1025template<class _Tp> 1026_LIBCPP_INLINE_VISIBILITY 1027_Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1028 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1029} 1030template<class _Tp> 1031_LIBCPP_INLINE_VISIBILITY 1032_Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1033 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1034} 1035 1036template<class _Tp> 1037_LIBCPP_INLINE_VISIBILITY 1038_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1039 return __c11_atomic_fetch_and(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1040} 1041template<class _Tp> 1042_LIBCPP_INLINE_VISIBILITY 1043_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1044 return __c11_atomic_fetch_and(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1045} 1046 1047template<class _Tp> 1048_LIBCPP_INLINE_VISIBILITY 1049_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1050 return __c11_atomic_fetch_or(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1051} 1052template<class _Tp> 1053_LIBCPP_INLINE_VISIBILITY 1054_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1055 return __c11_atomic_fetch_or(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1056} 1057 1058template<class _Tp> 1059_LIBCPP_INLINE_VISIBILITY 1060_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1061 return __c11_atomic_fetch_xor(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1062} 1063template<class _Tp> 1064_LIBCPP_INLINE_VISIBILITY 1065_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1066 return __c11_atomic_fetch_xor(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1067} 1068 1069#endif // _LIBCPP_HAS_GCC_ATOMIC_IMP, _LIBCPP_HAS_C_ATOMIC_IMP 1070 1071template <class _Tp> 1072_LIBCPP_INLINE_VISIBILITY 1073_Tp kill_dependency(_Tp __y) _NOEXCEPT 1074{ 1075 return __y; 1076} 1077 1078#if defined(__CLANG_ATOMIC_BOOL_LOCK_FREE) 1079# define ATOMIC_BOOL_LOCK_FREE __CLANG_ATOMIC_BOOL_LOCK_FREE 1080# define ATOMIC_CHAR_LOCK_FREE __CLANG_ATOMIC_CHAR_LOCK_FREE 1081#ifndef _LIBCPP_HAS_NO_CHAR8_T 1082# define ATOMIC_CHAR8_T_LOCK_FREE __CLANG_ATOMIC_CHAR8_T_LOCK_FREE 1083#endif 1084# define ATOMIC_CHAR16_T_LOCK_FREE __CLANG_ATOMIC_CHAR16_T_LOCK_FREE 1085# define ATOMIC_CHAR32_T_LOCK_FREE __CLANG_ATOMIC_CHAR32_T_LOCK_FREE 1086# define ATOMIC_WCHAR_T_LOCK_FREE __CLANG_ATOMIC_WCHAR_T_LOCK_FREE 1087# define ATOMIC_SHORT_LOCK_FREE __CLANG_ATOMIC_SHORT_LOCK_FREE 1088# define ATOMIC_INT_LOCK_FREE __CLANG_ATOMIC_INT_LOCK_FREE 1089# define ATOMIC_LONG_LOCK_FREE __CLANG_ATOMIC_LONG_LOCK_FREE 1090# define ATOMIC_LLONG_LOCK_FREE __CLANG_ATOMIC_LLONG_LOCK_FREE 1091# define ATOMIC_POINTER_LOCK_FREE __CLANG_ATOMIC_POINTER_LOCK_FREE 1092#elif defined(__GCC_ATOMIC_BOOL_LOCK_FREE) 1093# define ATOMIC_BOOL_LOCK_FREE __GCC_ATOMIC_BOOL_LOCK_FREE 1094# define ATOMIC_CHAR_LOCK_FREE __GCC_ATOMIC_CHAR_LOCK_FREE 1095#ifndef _LIBCPP_HAS_NO_CHAR8_T 1096# define ATOMIC_CHAR8_T_LOCK_FREE __GCC_ATOMIC_CHAR8_T_LOCK_FREE 1097#endif 1098# define ATOMIC_CHAR16_T_LOCK_FREE __GCC_ATOMIC_CHAR16_T_LOCK_FREE 1099# define ATOMIC_CHAR32_T_LOCK_FREE __GCC_ATOMIC_CHAR32_T_LOCK_FREE 1100# define ATOMIC_WCHAR_T_LOCK_FREE __GCC_ATOMIC_WCHAR_T_LOCK_FREE 1101# define ATOMIC_SHORT_LOCK_FREE __GCC_ATOMIC_SHORT_LOCK_FREE 1102# define ATOMIC_INT_LOCK_FREE __GCC_ATOMIC_INT_LOCK_FREE 1103# define ATOMIC_LONG_LOCK_FREE __GCC_ATOMIC_LONG_LOCK_FREE 1104# define ATOMIC_LLONG_LOCK_FREE __GCC_ATOMIC_LLONG_LOCK_FREE 1105# define ATOMIC_POINTER_LOCK_FREE __GCC_ATOMIC_POINTER_LOCK_FREE 1106#endif 1107 1108#ifdef _LIBCPP_ATOMIC_ONLY_USE_BUILTINS 1109 1110template<typename _Tp> 1111struct __cxx_atomic_lock_impl { 1112 1113 _LIBCPP_INLINE_VISIBILITY 1114 __cxx_atomic_lock_impl() _NOEXCEPT 1115 : __a_value(), __a_lock(0) {} 1116 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR explicit 1117 __cxx_atomic_lock_impl(_Tp value) _NOEXCEPT 1118 : __a_value(value), __a_lock(0) {} 1119 1120 _Tp __a_value; 1121 mutable __cxx_atomic_base_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_lock; 1122 1123 _LIBCPP_INLINE_VISIBILITY void __lock() const volatile { 1124 while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire)) 1125 /*spin*/; 1126 } 1127 _LIBCPP_INLINE_VISIBILITY void __lock() const { 1128 while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire)) 1129 /*spin*/; 1130 } 1131 _LIBCPP_INLINE_VISIBILITY void __unlock() const volatile { 1132 __cxx_atomic_store(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(false), memory_order_release); 1133 } 1134 _LIBCPP_INLINE_VISIBILITY void __unlock() const { 1135 __cxx_atomic_store(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(false), memory_order_release); 1136 } 1137 _LIBCPP_INLINE_VISIBILITY _Tp __read() const volatile { 1138 __lock(); 1139 _Tp __old; 1140 __cxx_atomic_assign_volatile(__old, __a_value); 1141 __unlock(); 1142 return __old; 1143 } 1144 _LIBCPP_INLINE_VISIBILITY _Tp __read() const { 1145 __lock(); 1146 _Tp __old = __a_value; 1147 __unlock(); 1148 return __old; 1149 } 1150}; 1151 1152template <typename _Tp> 1153_LIBCPP_INLINE_VISIBILITY 1154void __cxx_atomic_init(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __val) { 1155 __cxx_atomic_assign_volatile(__a->__a_value, __val); 1156} 1157template <typename _Tp> 1158_LIBCPP_INLINE_VISIBILITY 1159void __cxx_atomic_init(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __val) { 1160 __a->__a_value = __val; 1161} 1162 1163template <typename _Tp> 1164_LIBCPP_INLINE_VISIBILITY 1165void __cxx_atomic_store(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __val, memory_order) { 1166 __a->__lock(); 1167 __cxx_atomic_assign_volatile(__a->__a_value, __val); 1168 __a->__unlock(); 1169} 1170template <typename _Tp> 1171_LIBCPP_INLINE_VISIBILITY 1172void __cxx_atomic_store(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __val, memory_order) { 1173 __a->__lock(); 1174 __a->__a_value = __val; 1175 __a->__unlock(); 1176} 1177 1178template <typename _Tp> 1179_LIBCPP_INLINE_VISIBILITY 1180_Tp __cxx_atomic_load(const volatile __cxx_atomic_lock_impl<_Tp>* __a, memory_order) { 1181 return __a->__read(); 1182} 1183template <typename _Tp> 1184_LIBCPP_INLINE_VISIBILITY 1185_Tp __cxx_atomic_load(const __cxx_atomic_lock_impl<_Tp>* __a, memory_order) { 1186 return __a->__read(); 1187} 1188 1189template <typename _Tp> 1190_LIBCPP_INLINE_VISIBILITY 1191_Tp __cxx_atomic_exchange(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) { 1192 __a->__lock(); 1193 _Tp __old; 1194 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1195 __cxx_atomic_assign_volatile(__a->__a_value, __value); 1196 __a->__unlock(); 1197 return __old; 1198} 1199template <typename _Tp> 1200_LIBCPP_INLINE_VISIBILITY 1201_Tp __cxx_atomic_exchange(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) { 1202 __a->__lock(); 1203 _Tp __old = __a->__a_value; 1204 __a->__a_value = __value; 1205 __a->__unlock(); 1206 return __old; 1207} 1208 1209template <typename _Tp> 1210_LIBCPP_INLINE_VISIBILITY 1211bool __cxx_atomic_compare_exchange_strong(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1212 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1213 _Tp __temp; 1214 __a->__lock(); 1215 __cxx_atomic_assign_volatile(__temp, __a->__a_value); 1216 bool __ret = (_VSTD::memcmp(&__temp, __expected, sizeof(_Tp)) == 0); 1217 if(__ret) 1218 __cxx_atomic_assign_volatile(__a->__a_value, __value); 1219 else 1220 __cxx_atomic_assign_volatile(*__expected, __a->__a_value); 1221 __a->__unlock(); 1222 return __ret; 1223} 1224template <typename _Tp> 1225_LIBCPP_INLINE_VISIBILITY 1226bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_lock_impl<_Tp>* __a, 1227 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1228 __a->__lock(); 1229 bool __ret = (_VSTD::memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0); 1230 if(__ret) 1231 _VSTD::memcpy(&__a->__a_value, &__value, sizeof(_Tp)); 1232 else 1233 _VSTD::memcpy(__expected, &__a->__a_value, sizeof(_Tp)); 1234 __a->__unlock(); 1235 return __ret; 1236} 1237 1238template <typename _Tp> 1239_LIBCPP_INLINE_VISIBILITY 1240bool __cxx_atomic_compare_exchange_weak(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1241 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1242 _Tp __temp; 1243 __a->__lock(); 1244 __cxx_atomic_assign_volatile(__temp, __a->__a_value); 1245 bool __ret = (_VSTD::memcmp(&__temp, __expected, sizeof(_Tp)) == 0); 1246 if(__ret) 1247 __cxx_atomic_assign_volatile(__a->__a_value, __value); 1248 else 1249 __cxx_atomic_assign_volatile(*__expected, __a->__a_value); 1250 __a->__unlock(); 1251 return __ret; 1252} 1253template <typename _Tp> 1254_LIBCPP_INLINE_VISIBILITY 1255bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_lock_impl<_Tp>* __a, 1256 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1257 __a->__lock(); 1258 bool __ret = (_VSTD::memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0); 1259 if(__ret) 1260 _VSTD::memcpy(&__a->__a_value, &__value, sizeof(_Tp)); 1261 else 1262 _VSTD::memcpy(__expected, &__a->__a_value, sizeof(_Tp)); 1263 __a->__unlock(); 1264 return __ret; 1265} 1266 1267template <typename _Tp, typename _Td> 1268_LIBCPP_INLINE_VISIBILITY 1269_Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1270 _Td __delta, memory_order) { 1271 __a->__lock(); 1272 _Tp __old; 1273 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1274 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old + __delta)); 1275 __a->__unlock(); 1276 return __old; 1277} 1278template <typename _Tp, typename _Td> 1279_LIBCPP_INLINE_VISIBILITY 1280_Tp __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp>* __a, 1281 _Td __delta, memory_order) { 1282 __a->__lock(); 1283 _Tp __old = __a->__a_value; 1284 __a->__a_value += __delta; 1285 __a->__unlock(); 1286 return __old; 1287} 1288 1289template <typename _Tp, typename _Td> 1290_LIBCPP_INLINE_VISIBILITY 1291_Tp* __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp*>* __a, 1292 ptrdiff_t __delta, memory_order) { 1293 __a->__lock(); 1294 _Tp* __old; 1295 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1296 __cxx_atomic_assign_volatile(__a->__a_value, __old + __delta); 1297 __a->__unlock(); 1298 return __old; 1299} 1300template <typename _Tp, typename _Td> 1301_LIBCPP_INLINE_VISIBILITY 1302_Tp* __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp*>* __a, 1303 ptrdiff_t __delta, memory_order) { 1304 __a->__lock(); 1305 _Tp* __old = __a->__a_value; 1306 __a->__a_value += __delta; 1307 __a->__unlock(); 1308 return __old; 1309} 1310 1311template <typename _Tp, typename _Td> 1312_LIBCPP_INLINE_VISIBILITY 1313_Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1314 _Td __delta, memory_order) { 1315 __a->__lock(); 1316 _Tp __old; 1317 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1318 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old - __delta)); 1319 __a->__unlock(); 1320 return __old; 1321} 1322template <typename _Tp, typename _Td> 1323_LIBCPP_INLINE_VISIBILITY 1324_Tp __cxx_atomic_fetch_sub(__cxx_atomic_lock_impl<_Tp>* __a, 1325 _Td __delta, memory_order) { 1326 __a->__lock(); 1327 _Tp __old = __a->__a_value; 1328 __a->__a_value -= __delta; 1329 __a->__unlock(); 1330 return __old; 1331} 1332 1333template <typename _Tp> 1334_LIBCPP_INLINE_VISIBILITY 1335_Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1336 _Tp __pattern, memory_order) { 1337 __a->__lock(); 1338 _Tp __old; 1339 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1340 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old & __pattern)); 1341 __a->__unlock(); 1342 return __old; 1343} 1344template <typename _Tp> 1345_LIBCPP_INLINE_VISIBILITY 1346_Tp __cxx_atomic_fetch_and(__cxx_atomic_lock_impl<_Tp>* __a, 1347 _Tp __pattern, memory_order) { 1348 __a->__lock(); 1349 _Tp __old = __a->__a_value; 1350 __a->__a_value &= __pattern; 1351 __a->__unlock(); 1352 return __old; 1353} 1354 1355template <typename _Tp> 1356_LIBCPP_INLINE_VISIBILITY 1357_Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1358 _Tp __pattern, memory_order) { 1359 __a->__lock(); 1360 _Tp __old; 1361 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1362 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old | __pattern)); 1363 __a->__unlock(); 1364 return __old; 1365} 1366template <typename _Tp> 1367_LIBCPP_INLINE_VISIBILITY 1368_Tp __cxx_atomic_fetch_or(__cxx_atomic_lock_impl<_Tp>* __a, 1369 _Tp __pattern, memory_order) { 1370 __a->__lock(); 1371 _Tp __old = __a->__a_value; 1372 __a->__a_value |= __pattern; 1373 __a->__unlock(); 1374 return __old; 1375} 1376 1377template <typename _Tp> 1378_LIBCPP_INLINE_VISIBILITY 1379_Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1380 _Tp __pattern, memory_order) { 1381 __a->__lock(); 1382 _Tp __old; 1383 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1384 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old ^ __pattern)); 1385 __a->__unlock(); 1386 return __old; 1387} 1388template <typename _Tp> 1389_LIBCPP_INLINE_VISIBILITY 1390_Tp __cxx_atomic_fetch_xor(__cxx_atomic_lock_impl<_Tp>* __a, 1391 _Tp __pattern, memory_order) { 1392 __a->__lock(); 1393 _Tp __old = __a->__a_value; 1394 __a->__a_value ^= __pattern; 1395 __a->__unlock(); 1396 return __old; 1397} 1398 1399#ifdef __cpp_lib_atomic_is_always_lock_free 1400 1401template<typename _Tp> struct __cxx_is_always_lock_free { 1402 enum { __value = __atomic_always_lock_free(sizeof(_Tp), 0) }; }; 1403 1404#else 1405 1406template<typename _Tp> struct __cxx_is_always_lock_free { enum { __value = false }; }; 1407// Implementations must match the C ATOMIC_*_LOCK_FREE macro values. 1408template<> struct __cxx_is_always_lock_free<bool> { enum { __value = 2 == ATOMIC_BOOL_LOCK_FREE }; }; 1409template<> struct __cxx_is_always_lock_free<char> { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; }; 1410template<> struct __cxx_is_always_lock_free<signed char> { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; }; 1411template<> struct __cxx_is_always_lock_free<unsigned char> { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; }; 1412#ifndef _LIBCPP_HAS_NO_CHAR8_T 1413template<> struct __cxx_is_always_lock_free<char8_t> { enum { __value = 2 == ATOMIC_CHAR8_T_LOCK_FREE }; }; 1414#endif 1415template<> struct __cxx_is_always_lock_free<char16_t> { enum { __value = 2 == ATOMIC_CHAR16_T_LOCK_FREE }; }; 1416template<> struct __cxx_is_always_lock_free<char32_t> { enum { __value = 2 == ATOMIC_CHAR32_T_LOCK_FREE }; }; 1417#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS 1418template<> struct __cxx_is_always_lock_free<wchar_t> { enum { __value = 2 == ATOMIC_WCHAR_T_LOCK_FREE }; }; 1419#endif 1420template<> struct __cxx_is_always_lock_free<short> { enum { __value = 2 == ATOMIC_SHORT_LOCK_FREE }; }; 1421template<> struct __cxx_is_always_lock_free<unsigned short> { enum { __value = 2 == ATOMIC_SHORT_LOCK_FREE }; }; 1422template<> struct __cxx_is_always_lock_free<int> { enum { __value = 2 == ATOMIC_INT_LOCK_FREE }; }; 1423template<> struct __cxx_is_always_lock_free<unsigned int> { enum { __value = 2 == ATOMIC_INT_LOCK_FREE }; }; 1424template<> struct __cxx_is_always_lock_free<long> { enum { __value = 2 == ATOMIC_LONG_LOCK_FREE }; }; 1425template<> struct __cxx_is_always_lock_free<unsigned long> { enum { __value = 2 == ATOMIC_LONG_LOCK_FREE }; }; 1426template<> struct __cxx_is_always_lock_free<long long> { enum { __value = 2 == ATOMIC_LLONG_LOCK_FREE }; }; 1427template<> struct __cxx_is_always_lock_free<unsigned long long> { enum { __value = 2 == ATOMIC_LLONG_LOCK_FREE }; }; 1428template<typename _Tp> struct __cxx_is_always_lock_free<_Tp*> { enum { __value = 2 == ATOMIC_POINTER_LOCK_FREE }; }; 1429template<> struct __cxx_is_always_lock_free<std::nullptr_t> { enum { __value = 2 == ATOMIC_POINTER_LOCK_FREE }; }; 1430 1431#endif //__cpp_lib_atomic_is_always_lock_free 1432 1433template <typename _Tp, 1434 typename _Base = typename conditional<__cxx_is_always_lock_free<_Tp>::__value, 1435 __cxx_atomic_base_impl<_Tp>, 1436 __cxx_atomic_lock_impl<_Tp> >::type> 1437#else 1438template <typename _Tp, 1439 typename _Base = __cxx_atomic_base_impl<_Tp> > 1440#endif //_LIBCPP_ATOMIC_ONLY_USE_BUILTINS 1441struct __cxx_atomic_impl : public _Base { 1442 static_assert(is_trivially_copyable<_Tp>::value, 1443 "std::atomic<Tp> requires that 'Tp' be a trivially copyable type"); 1444 1445 _LIBCPP_INLINE_VISIBILITY __cxx_atomic_impl() _NOEXCEPT _LIBCPP_DEFAULT 1446 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR explicit __cxx_atomic_impl(_Tp value) _NOEXCEPT 1447 : _Base(value) {} 1448}; 1449 1450#ifdef __linux__ 1451 using __cxx_contention_t = int32_t; 1452#else 1453 using __cxx_contention_t = int64_t; 1454#endif //__linux__ 1455 1456using __cxx_atomic_contention_t = __cxx_atomic_impl<__cxx_contention_t>; 1457 1458#ifndef _LIBCPP_HAS_NO_PLATFORM_WAIT 1459 1460_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(void const volatile*); 1461_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(void const volatile*); 1462_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(void const volatile*); 1463_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(void const volatile*, __cxx_contention_t); 1464 1465_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(__cxx_atomic_contention_t const volatile*); 1466_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(__cxx_atomic_contention_t const volatile*); 1467_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile*); 1468_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(__cxx_atomic_contention_t const volatile*, __cxx_contention_t); 1469 1470template <class _Atp, class _Fn> 1471struct __libcpp_atomic_wait_backoff_impl { 1472 _Atp* __a; 1473 _Fn __test_fn; 1474 _LIBCPP_AVAILABILITY_SYNC 1475 _LIBCPP_INLINE_VISIBILITY bool operator()(chrono::nanoseconds __elapsed) const 1476 { 1477 if(__elapsed > chrono::microseconds(64)) 1478 { 1479 auto const __monitor = __libcpp_atomic_monitor(__a); 1480 if(__test_fn()) 1481 return true; 1482 __libcpp_atomic_wait(__a, __monitor); 1483 } 1484 else if(__elapsed > chrono::microseconds(4)) 1485 __libcpp_thread_yield(); 1486 else 1487 {} // poll 1488 return false; 1489 } 1490}; 1491 1492template <class _Atp, class _Fn> 1493_LIBCPP_AVAILABILITY_SYNC 1494_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp* __a, _Fn && __test_fn) 1495{ 1496 __libcpp_atomic_wait_backoff_impl<_Atp, typename decay<_Fn>::type> __backoff_fn = {__a, __test_fn}; 1497 return __libcpp_thread_poll_with_backoff(__test_fn, __backoff_fn); 1498} 1499 1500#else // _LIBCPP_HAS_NO_PLATFORM_WAIT 1501 1502template <class _Tp> 1503_LIBCPP_INLINE_VISIBILITY void __cxx_atomic_notify_all(__cxx_atomic_impl<_Tp> const volatile*) { } 1504template <class _Tp> 1505_LIBCPP_INLINE_VISIBILITY void __cxx_atomic_notify_one(__cxx_atomic_impl<_Tp> const volatile*) { } 1506template <class _Atp, class _Fn> 1507_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp*, _Fn && __test_fn) 1508{ 1509 return __libcpp_thread_poll_with_backoff(__test_fn, __libcpp_timed_backoff_policy()); 1510} 1511 1512#endif // _LIBCPP_HAS_NO_PLATFORM_WAIT 1513 1514template <class _Atp, class _Tp> 1515struct __cxx_atomic_wait_test_fn_impl { 1516 _Atp* __a; 1517 _Tp __val; 1518 memory_order __order; 1519 _LIBCPP_INLINE_VISIBILITY bool operator()() const 1520 { 1521 return !__cxx_nonatomic_compare_equal(__cxx_atomic_load(__a, __order), __val); 1522 } 1523}; 1524 1525template <class _Atp, class _Tp> 1526_LIBCPP_AVAILABILITY_SYNC 1527_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp* __a, _Tp const __val, memory_order __order) 1528{ 1529 __cxx_atomic_wait_test_fn_impl<_Atp, _Tp> __test_fn = {__a, __val, __order}; 1530 return __cxx_atomic_wait(__a, __test_fn); 1531} 1532 1533// general atomic<T> 1534 1535template <class _Tp, bool = is_integral<_Tp>::value && !is_same<_Tp, bool>::value> 1536struct __atomic_base // false 1537{ 1538 mutable __cxx_atomic_impl<_Tp> __a_; 1539 1540#if defined(__cpp_lib_atomic_is_always_lock_free) 1541 static _LIBCPP_CONSTEXPR bool is_always_lock_free = __atomic_always_lock_free(sizeof(__a_), 0); 1542#endif 1543 1544 _LIBCPP_INLINE_VISIBILITY 1545 bool is_lock_free() const volatile _NOEXCEPT 1546 {return __cxx_atomic_is_lock_free(sizeof(_Tp));} 1547 _LIBCPP_INLINE_VISIBILITY 1548 bool is_lock_free() const _NOEXCEPT 1549 {return static_cast<__atomic_base const volatile*>(this)->is_lock_free();} 1550 _LIBCPP_INLINE_VISIBILITY 1551 void store(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1552 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1553 {__cxx_atomic_store(&__a_, __d, __m);} 1554 _LIBCPP_INLINE_VISIBILITY 1555 void store(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1556 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1557 {__cxx_atomic_store(&__a_, __d, __m);} 1558 _LIBCPP_INLINE_VISIBILITY 1559 _Tp load(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 1560 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 1561 {return __cxx_atomic_load(&__a_, __m);} 1562 _LIBCPP_INLINE_VISIBILITY 1563 _Tp load(memory_order __m = memory_order_seq_cst) const _NOEXCEPT 1564 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 1565 {return __cxx_atomic_load(&__a_, __m);} 1566 _LIBCPP_INLINE_VISIBILITY 1567 operator _Tp() const volatile _NOEXCEPT {return load();} 1568 _LIBCPP_INLINE_VISIBILITY 1569 operator _Tp() const _NOEXCEPT {return load();} 1570 _LIBCPP_INLINE_VISIBILITY 1571 _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1572 {return __cxx_atomic_exchange(&__a_, __d, __m);} 1573 _LIBCPP_INLINE_VISIBILITY 1574 _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1575 {return __cxx_atomic_exchange(&__a_, __d, __m);} 1576 _LIBCPP_INLINE_VISIBILITY 1577 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1578 memory_order __s, memory_order __f) volatile _NOEXCEPT 1579 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1580 {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);} 1581 _LIBCPP_INLINE_VISIBILITY 1582 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1583 memory_order __s, memory_order __f) _NOEXCEPT 1584 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1585 {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);} 1586 _LIBCPP_INLINE_VISIBILITY 1587 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1588 memory_order __s, memory_order __f) volatile _NOEXCEPT 1589 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1590 {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);} 1591 _LIBCPP_INLINE_VISIBILITY 1592 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1593 memory_order __s, memory_order __f) _NOEXCEPT 1594 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1595 {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);} 1596 _LIBCPP_INLINE_VISIBILITY 1597 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1598 memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1599 {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);} 1600 _LIBCPP_INLINE_VISIBILITY 1601 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1602 memory_order __m = memory_order_seq_cst) _NOEXCEPT 1603 {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);} 1604 _LIBCPP_INLINE_VISIBILITY 1605 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1606 memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1607 {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);} 1608 _LIBCPP_INLINE_VISIBILITY 1609 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1610 memory_order __m = memory_order_seq_cst) _NOEXCEPT 1611 {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);} 1612 1613 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 1614 {__cxx_atomic_wait(&__a_, __v, __m);} 1615 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT 1616 {__cxx_atomic_wait(&__a_, __v, __m);} 1617 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_one() volatile _NOEXCEPT 1618 {__cxx_atomic_notify_one(&__a_);} 1619 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_one() _NOEXCEPT 1620 {__cxx_atomic_notify_one(&__a_);} 1621 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_all() volatile _NOEXCEPT 1622 {__cxx_atomic_notify_all(&__a_);} 1623 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_all() _NOEXCEPT 1624 {__cxx_atomic_notify_all(&__a_);} 1625 1626#if _LIBCPP_STD_VER > 17 1627 _LIBCPP_INLINE_VISIBILITY constexpr 1628 __atomic_base() noexcept(is_nothrow_default_constructible_v<_Tp>) : __a_(_Tp()) {} 1629#else 1630 _LIBCPP_INLINE_VISIBILITY 1631 __atomic_base() _NOEXCEPT _LIBCPP_DEFAULT 1632#endif 1633 1634 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR 1635 __atomic_base(_Tp __d) _NOEXCEPT : __a_(__d) {} 1636 1637#ifndef _LIBCPP_CXX03_LANG 1638 __atomic_base(const __atomic_base&) = delete; 1639#else 1640private: 1641 _LIBCPP_INLINE_VISIBILITY 1642 __atomic_base(const __atomic_base&); 1643#endif 1644}; 1645 1646#if defined(__cpp_lib_atomic_is_always_lock_free) 1647template <class _Tp, bool __b> 1648_LIBCPP_CONSTEXPR bool __atomic_base<_Tp, __b>::is_always_lock_free; 1649#endif 1650 1651// atomic<Integral> 1652 1653template <class _Tp> 1654struct __atomic_base<_Tp, true> 1655 : public __atomic_base<_Tp, false> 1656{ 1657 typedef __atomic_base<_Tp, false> __base; 1658 1659 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 1660 __atomic_base() _NOEXCEPT _LIBCPP_DEFAULT 1661 1662 _LIBCPP_INLINE_VISIBILITY 1663 _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __base(__d) {} 1664 1665 _LIBCPP_INLINE_VISIBILITY 1666 _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1667 {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);} 1668 _LIBCPP_INLINE_VISIBILITY 1669 _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1670 {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);} 1671 _LIBCPP_INLINE_VISIBILITY 1672 _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1673 {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);} 1674 _LIBCPP_INLINE_VISIBILITY 1675 _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1676 {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);} 1677 _LIBCPP_INLINE_VISIBILITY 1678 _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1679 {return __cxx_atomic_fetch_and(&this->__a_, __op, __m);} 1680 _LIBCPP_INLINE_VISIBILITY 1681 _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1682 {return __cxx_atomic_fetch_and(&this->__a_, __op, __m);} 1683 _LIBCPP_INLINE_VISIBILITY 1684 _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1685 {return __cxx_atomic_fetch_or(&this->__a_, __op, __m);} 1686 _LIBCPP_INLINE_VISIBILITY 1687 _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1688 {return __cxx_atomic_fetch_or(&this->__a_, __op, __m);} 1689 _LIBCPP_INLINE_VISIBILITY 1690 _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1691 {return __cxx_atomic_fetch_xor(&this->__a_, __op, __m);} 1692 _LIBCPP_INLINE_VISIBILITY 1693 _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1694 {return __cxx_atomic_fetch_xor(&this->__a_, __op, __m);} 1695 1696 _LIBCPP_INLINE_VISIBILITY 1697 _Tp operator++(int) volatile _NOEXCEPT {return fetch_add(_Tp(1));} 1698 _LIBCPP_INLINE_VISIBILITY 1699 _Tp operator++(int) _NOEXCEPT {return fetch_add(_Tp(1));} 1700 _LIBCPP_INLINE_VISIBILITY 1701 _Tp operator--(int) volatile _NOEXCEPT {return fetch_sub(_Tp(1));} 1702 _LIBCPP_INLINE_VISIBILITY 1703 _Tp operator--(int) _NOEXCEPT {return fetch_sub(_Tp(1));} 1704 _LIBCPP_INLINE_VISIBILITY 1705 _Tp operator++() volatile _NOEXCEPT {return fetch_add(_Tp(1)) + _Tp(1);} 1706 _LIBCPP_INLINE_VISIBILITY 1707 _Tp operator++() _NOEXCEPT {return fetch_add(_Tp(1)) + _Tp(1);} 1708 _LIBCPP_INLINE_VISIBILITY 1709 _Tp operator--() volatile _NOEXCEPT {return fetch_sub(_Tp(1)) - _Tp(1);} 1710 _LIBCPP_INLINE_VISIBILITY 1711 _Tp operator--() _NOEXCEPT {return fetch_sub(_Tp(1)) - _Tp(1);} 1712 _LIBCPP_INLINE_VISIBILITY 1713 _Tp operator+=(_Tp __op) volatile _NOEXCEPT {return fetch_add(__op) + __op;} 1714 _LIBCPP_INLINE_VISIBILITY 1715 _Tp operator+=(_Tp __op) _NOEXCEPT {return fetch_add(__op) + __op;} 1716 _LIBCPP_INLINE_VISIBILITY 1717 _Tp operator-=(_Tp __op) volatile _NOEXCEPT {return fetch_sub(__op) - __op;} 1718 _LIBCPP_INLINE_VISIBILITY 1719 _Tp operator-=(_Tp __op) _NOEXCEPT {return fetch_sub(__op) - __op;} 1720 _LIBCPP_INLINE_VISIBILITY 1721 _Tp operator&=(_Tp __op) volatile _NOEXCEPT {return fetch_and(__op) & __op;} 1722 _LIBCPP_INLINE_VISIBILITY 1723 _Tp operator&=(_Tp __op) _NOEXCEPT {return fetch_and(__op) & __op;} 1724 _LIBCPP_INLINE_VISIBILITY 1725 _Tp operator|=(_Tp __op) volatile _NOEXCEPT {return fetch_or(__op) | __op;} 1726 _LIBCPP_INLINE_VISIBILITY 1727 _Tp operator|=(_Tp __op) _NOEXCEPT {return fetch_or(__op) | __op;} 1728 _LIBCPP_INLINE_VISIBILITY 1729 _Tp operator^=(_Tp __op) volatile _NOEXCEPT {return fetch_xor(__op) ^ __op;} 1730 _LIBCPP_INLINE_VISIBILITY 1731 _Tp operator^=(_Tp __op) _NOEXCEPT {return fetch_xor(__op) ^ __op;} 1732}; 1733 1734// atomic<T> 1735 1736template <class _Tp> 1737struct atomic 1738 : public __atomic_base<_Tp> 1739{ 1740 typedef __atomic_base<_Tp> __base; 1741 typedef _Tp value_type; 1742 typedef value_type difference_type; 1743 1744#if _LIBCPP_STD_VER > 17 1745 _LIBCPP_INLINE_VISIBILITY 1746 atomic() = default; 1747#else 1748 _LIBCPP_INLINE_VISIBILITY 1749 atomic() _NOEXCEPT _LIBCPP_DEFAULT 1750#endif 1751 1752 _LIBCPP_INLINE_VISIBILITY 1753 _LIBCPP_CONSTEXPR atomic(_Tp __d) _NOEXCEPT : __base(__d) {} 1754 1755 _LIBCPP_INLINE_VISIBILITY 1756 _Tp operator=(_Tp __d) volatile _NOEXCEPT 1757 {__base::store(__d); return __d;} 1758 _LIBCPP_INLINE_VISIBILITY 1759 _Tp operator=(_Tp __d) _NOEXCEPT 1760 {__base::store(__d); return __d;} 1761 1762 atomic& operator=(const atomic&) = delete; 1763 atomic& operator=(const atomic&) volatile = delete; 1764}; 1765 1766// atomic<T*> 1767 1768template <class _Tp> 1769struct atomic<_Tp*> 1770 : public __atomic_base<_Tp*> 1771{ 1772 typedef __atomic_base<_Tp*> __base; 1773 typedef _Tp* value_type; 1774 typedef ptrdiff_t difference_type; 1775 1776 _LIBCPP_INLINE_VISIBILITY 1777 atomic() _NOEXCEPT _LIBCPP_DEFAULT 1778 1779 _LIBCPP_INLINE_VISIBILITY 1780 _LIBCPP_CONSTEXPR atomic(_Tp* __d) _NOEXCEPT : __base(__d) {} 1781 1782 _LIBCPP_INLINE_VISIBILITY 1783 _Tp* operator=(_Tp* __d) volatile _NOEXCEPT 1784 {__base::store(__d); return __d;} 1785 _LIBCPP_INLINE_VISIBILITY 1786 _Tp* operator=(_Tp* __d) _NOEXCEPT 1787 {__base::store(__d); return __d;} 1788 1789 _LIBCPP_INLINE_VISIBILITY 1790 _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { 1791 // __atomic_fetch_add accepts function pointers, guard against them. 1792 static_assert(!is_function<typename remove_pointer<_Tp>::type>::value, "Pointer to function isn't allowed"); 1793 return __cxx_atomic_fetch_add(&this->__a_, __op, __m); 1794 } 1795 1796 _LIBCPP_INLINE_VISIBILITY 1797 _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { 1798 // __atomic_fetch_add accepts function pointers, guard against them. 1799 static_assert(!is_function<typename remove_pointer<_Tp>::type>::value, "Pointer to function isn't allowed"); 1800 return __cxx_atomic_fetch_add(&this->__a_, __op, __m); 1801 } 1802 1803 _LIBCPP_INLINE_VISIBILITY 1804 _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { 1805 // __atomic_fetch_add accepts function pointers, guard against them. 1806 static_assert(!is_function<typename remove_pointer<_Tp>::type>::value, "Pointer to function isn't allowed"); 1807 return __cxx_atomic_fetch_sub(&this->__a_, __op, __m); 1808 } 1809 1810 _LIBCPP_INLINE_VISIBILITY 1811 _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { 1812 // __atomic_fetch_add accepts function pointers, guard against them. 1813 static_assert(!is_function<typename remove_pointer<_Tp>::type>::value, "Pointer to function isn't allowed"); 1814 return __cxx_atomic_fetch_sub(&this->__a_, __op, __m); 1815 } 1816 1817 _LIBCPP_INLINE_VISIBILITY 1818 _Tp* operator++(int) volatile _NOEXCEPT {return fetch_add(1);} 1819 _LIBCPP_INLINE_VISIBILITY 1820 _Tp* operator++(int) _NOEXCEPT {return fetch_add(1);} 1821 _LIBCPP_INLINE_VISIBILITY 1822 _Tp* operator--(int) volatile _NOEXCEPT {return fetch_sub(1);} 1823 _LIBCPP_INLINE_VISIBILITY 1824 _Tp* operator--(int) _NOEXCEPT {return fetch_sub(1);} 1825 _LIBCPP_INLINE_VISIBILITY 1826 _Tp* operator++() volatile _NOEXCEPT {return fetch_add(1) + 1;} 1827 _LIBCPP_INLINE_VISIBILITY 1828 _Tp* operator++() _NOEXCEPT {return fetch_add(1) + 1;} 1829 _LIBCPP_INLINE_VISIBILITY 1830 _Tp* operator--() volatile _NOEXCEPT {return fetch_sub(1) - 1;} 1831 _LIBCPP_INLINE_VISIBILITY 1832 _Tp* operator--() _NOEXCEPT {return fetch_sub(1) - 1;} 1833 _LIBCPP_INLINE_VISIBILITY 1834 _Tp* operator+=(ptrdiff_t __op) volatile _NOEXCEPT {return fetch_add(__op) + __op;} 1835 _LIBCPP_INLINE_VISIBILITY 1836 _Tp* operator+=(ptrdiff_t __op) _NOEXCEPT {return fetch_add(__op) + __op;} 1837 _LIBCPP_INLINE_VISIBILITY 1838 _Tp* operator-=(ptrdiff_t __op) volatile _NOEXCEPT {return fetch_sub(__op) - __op;} 1839 _LIBCPP_INLINE_VISIBILITY 1840 _Tp* operator-=(ptrdiff_t __op) _NOEXCEPT {return fetch_sub(__op) - __op;} 1841 1842 atomic& operator=(const atomic&) = delete; 1843 atomic& operator=(const atomic&) volatile = delete; 1844}; 1845 1846// atomic_is_lock_free 1847 1848template <class _Tp> 1849_LIBCPP_INLINE_VISIBILITY 1850bool 1851atomic_is_lock_free(const volatile atomic<_Tp>* __o) _NOEXCEPT 1852{ 1853 return __o->is_lock_free(); 1854} 1855 1856template <class _Tp> 1857_LIBCPP_INLINE_VISIBILITY 1858bool 1859atomic_is_lock_free(const atomic<_Tp>* __o) _NOEXCEPT 1860{ 1861 return __o->is_lock_free(); 1862} 1863 1864// atomic_init 1865 1866template <class _Tp> 1867_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_INLINE_VISIBILITY 1868void 1869atomic_init(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1870{ 1871 __cxx_atomic_init(&__o->__a_, __d); 1872} 1873 1874template <class _Tp> 1875_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_INLINE_VISIBILITY 1876void 1877atomic_init(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1878{ 1879 __cxx_atomic_init(&__o->__a_, __d); 1880} 1881 1882// atomic_store 1883 1884template <class _Tp> 1885_LIBCPP_INLINE_VISIBILITY 1886void 1887atomic_store(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1888{ 1889 __o->store(__d); 1890} 1891 1892template <class _Tp> 1893_LIBCPP_INLINE_VISIBILITY 1894void 1895atomic_store(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1896{ 1897 __o->store(__d); 1898} 1899 1900// atomic_store_explicit 1901 1902template <class _Tp> 1903_LIBCPP_INLINE_VISIBILITY 1904void 1905atomic_store_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 1906 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1907{ 1908 __o->store(__d, __m); 1909} 1910 1911template <class _Tp> 1912_LIBCPP_INLINE_VISIBILITY 1913void 1914atomic_store_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 1915 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1916{ 1917 __o->store(__d, __m); 1918} 1919 1920// atomic_load 1921 1922template <class _Tp> 1923_LIBCPP_INLINE_VISIBILITY 1924_Tp 1925atomic_load(const volatile atomic<_Tp>* __o) _NOEXCEPT 1926{ 1927 return __o->load(); 1928} 1929 1930template <class _Tp> 1931_LIBCPP_INLINE_VISIBILITY 1932_Tp 1933atomic_load(const atomic<_Tp>* __o) _NOEXCEPT 1934{ 1935 return __o->load(); 1936} 1937 1938// atomic_load_explicit 1939 1940template <class _Tp> 1941_LIBCPP_INLINE_VISIBILITY 1942_Tp 1943atomic_load_explicit(const volatile atomic<_Tp>* __o, memory_order __m) _NOEXCEPT 1944 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 1945{ 1946 return __o->load(__m); 1947} 1948 1949template <class _Tp> 1950_LIBCPP_INLINE_VISIBILITY 1951_Tp 1952atomic_load_explicit(const atomic<_Tp>* __o, memory_order __m) _NOEXCEPT 1953 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 1954{ 1955 return __o->load(__m); 1956} 1957 1958// atomic_exchange 1959 1960template <class _Tp> 1961_LIBCPP_INLINE_VISIBILITY 1962_Tp 1963atomic_exchange(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1964{ 1965 return __o->exchange(__d); 1966} 1967 1968template <class _Tp> 1969_LIBCPP_INLINE_VISIBILITY 1970_Tp 1971atomic_exchange(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1972{ 1973 return __o->exchange(__d); 1974} 1975 1976// atomic_exchange_explicit 1977 1978template <class _Tp> 1979_LIBCPP_INLINE_VISIBILITY 1980_Tp 1981atomic_exchange_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 1982{ 1983 return __o->exchange(__d, __m); 1984} 1985 1986template <class _Tp> 1987_LIBCPP_INLINE_VISIBILITY 1988_Tp 1989atomic_exchange_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 1990{ 1991 return __o->exchange(__d, __m); 1992} 1993 1994// atomic_compare_exchange_weak 1995 1996template <class _Tp> 1997_LIBCPP_INLINE_VISIBILITY 1998bool 1999atomic_compare_exchange_weak(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2000{ 2001 return __o->compare_exchange_weak(*__e, __d); 2002} 2003 2004template <class _Tp> 2005_LIBCPP_INLINE_VISIBILITY 2006bool 2007atomic_compare_exchange_weak(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2008{ 2009 return __o->compare_exchange_weak(*__e, __d); 2010} 2011 2012// atomic_compare_exchange_strong 2013 2014template <class _Tp> 2015_LIBCPP_INLINE_VISIBILITY 2016bool 2017atomic_compare_exchange_strong(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2018{ 2019 return __o->compare_exchange_strong(*__e, __d); 2020} 2021 2022template <class _Tp> 2023_LIBCPP_INLINE_VISIBILITY 2024bool 2025atomic_compare_exchange_strong(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2026{ 2027 return __o->compare_exchange_strong(*__e, __d); 2028} 2029 2030// atomic_compare_exchange_weak_explicit 2031 2032template <class _Tp> 2033_LIBCPP_INLINE_VISIBILITY 2034bool 2035atomic_compare_exchange_weak_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, 2036 typename atomic<_Tp>::value_type __d, 2037 memory_order __s, memory_order __f) _NOEXCEPT 2038 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2039{ 2040 return __o->compare_exchange_weak(*__e, __d, __s, __f); 2041} 2042 2043template <class _Tp> 2044_LIBCPP_INLINE_VISIBILITY 2045bool 2046atomic_compare_exchange_weak_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d, 2047 memory_order __s, memory_order __f) _NOEXCEPT 2048 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2049{ 2050 return __o->compare_exchange_weak(*__e, __d, __s, __f); 2051} 2052 2053// atomic_compare_exchange_strong_explicit 2054 2055template <class _Tp> 2056_LIBCPP_INLINE_VISIBILITY 2057bool 2058atomic_compare_exchange_strong_explicit(volatile atomic<_Tp>* __o, 2059 typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d, 2060 memory_order __s, memory_order __f) _NOEXCEPT 2061 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2062{ 2063 return __o->compare_exchange_strong(*__e, __d, __s, __f); 2064} 2065 2066template <class _Tp> 2067_LIBCPP_INLINE_VISIBILITY 2068bool 2069atomic_compare_exchange_strong_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, 2070 typename atomic<_Tp>::value_type __d, 2071 memory_order __s, memory_order __f) _NOEXCEPT 2072 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2073{ 2074 return __o->compare_exchange_strong(*__e, __d, __s, __f); 2075} 2076 2077// atomic_wait 2078 2079template <class _Tp> 2080_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2081void atomic_wait(const volatile atomic<_Tp>* __o, 2082 typename atomic<_Tp>::value_type __v) _NOEXCEPT 2083{ 2084 return __o->wait(__v); 2085} 2086 2087template <class _Tp> 2088_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2089void atomic_wait(const atomic<_Tp>* __o, 2090 typename atomic<_Tp>::value_type __v) _NOEXCEPT 2091{ 2092 return __o->wait(__v); 2093} 2094 2095// atomic_wait_explicit 2096 2097template <class _Tp> 2098_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2099void atomic_wait_explicit(const volatile atomic<_Tp>* __o, 2100 typename atomic<_Tp>::value_type __v, 2101 memory_order __m) _NOEXCEPT 2102 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 2103{ 2104 return __o->wait(__v, __m); 2105} 2106 2107template <class _Tp> 2108_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2109void atomic_wait_explicit(const atomic<_Tp>* __o, 2110 typename atomic<_Tp>::value_type __v, 2111 memory_order __m) _NOEXCEPT 2112 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 2113{ 2114 return __o->wait(__v, __m); 2115} 2116 2117// atomic_notify_one 2118 2119template <class _Tp> 2120_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2121void atomic_notify_one(volatile atomic<_Tp>* __o) _NOEXCEPT 2122{ 2123 __o->notify_one(); 2124} 2125template <class _Tp> 2126_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2127void atomic_notify_one(atomic<_Tp>* __o) _NOEXCEPT 2128{ 2129 __o->notify_one(); 2130} 2131 2132// atomic_notify_one 2133 2134template <class _Tp> 2135_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2136void atomic_notify_all(volatile atomic<_Tp>* __o) _NOEXCEPT 2137{ 2138 __o->notify_all(); 2139} 2140template <class _Tp> 2141_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2142void atomic_notify_all(atomic<_Tp>* __o) _NOEXCEPT 2143{ 2144 __o->notify_all(); 2145} 2146 2147// atomic_fetch_add 2148 2149template <class _Tp> 2150_LIBCPP_INLINE_VISIBILITY 2151_Tp 2152atomic_fetch_add(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT 2153{ 2154 return __o->fetch_add(__op); 2155} 2156 2157template <class _Tp> 2158_LIBCPP_INLINE_VISIBILITY 2159_Tp 2160atomic_fetch_add(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT 2161{ 2162 return __o->fetch_add(__op); 2163} 2164 2165// atomic_fetch_add_explicit 2166 2167template <class _Tp> 2168_LIBCPP_INLINE_VISIBILITY 2169_Tp atomic_fetch_add_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT 2170{ 2171 return __o->fetch_add(__op, __m); 2172} 2173 2174template <class _Tp> 2175_LIBCPP_INLINE_VISIBILITY 2176_Tp atomic_fetch_add_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT 2177{ 2178 return __o->fetch_add(__op, __m); 2179} 2180 2181// atomic_fetch_sub 2182 2183template <class _Tp> 2184_LIBCPP_INLINE_VISIBILITY 2185_Tp atomic_fetch_sub(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT 2186{ 2187 return __o->fetch_sub(__op); 2188} 2189 2190template <class _Tp> 2191_LIBCPP_INLINE_VISIBILITY 2192_Tp atomic_fetch_sub(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT 2193{ 2194 return __o->fetch_sub(__op); 2195} 2196 2197// atomic_fetch_sub_explicit 2198 2199template <class _Tp> 2200_LIBCPP_INLINE_VISIBILITY 2201_Tp atomic_fetch_sub_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT 2202{ 2203 return __o->fetch_sub(__op, __m); 2204} 2205 2206template <class _Tp> 2207_LIBCPP_INLINE_VISIBILITY 2208_Tp atomic_fetch_sub_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT 2209{ 2210 return __o->fetch_sub(__op, __m); 2211} 2212 2213// atomic_fetch_and 2214 2215template <class _Tp> 2216_LIBCPP_INLINE_VISIBILITY 2217typename enable_if 2218< 2219 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2220 _Tp 2221>::type 2222atomic_fetch_and(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2223{ 2224 return __o->fetch_and(__op); 2225} 2226 2227template <class _Tp> 2228_LIBCPP_INLINE_VISIBILITY 2229typename enable_if 2230< 2231 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2232 _Tp 2233>::type 2234atomic_fetch_and(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2235{ 2236 return __o->fetch_and(__op); 2237} 2238 2239// atomic_fetch_and_explicit 2240 2241template <class _Tp> 2242_LIBCPP_INLINE_VISIBILITY 2243typename enable_if 2244< 2245 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2246 _Tp 2247>::type 2248atomic_fetch_and_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2249{ 2250 return __o->fetch_and(__op, __m); 2251} 2252 2253template <class _Tp> 2254_LIBCPP_INLINE_VISIBILITY 2255typename enable_if 2256< 2257 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2258 _Tp 2259>::type 2260atomic_fetch_and_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2261{ 2262 return __o->fetch_and(__op, __m); 2263} 2264 2265// atomic_fetch_or 2266 2267template <class _Tp> 2268_LIBCPP_INLINE_VISIBILITY 2269typename enable_if 2270< 2271 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2272 _Tp 2273>::type 2274atomic_fetch_or(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2275{ 2276 return __o->fetch_or(__op); 2277} 2278 2279template <class _Tp> 2280_LIBCPP_INLINE_VISIBILITY 2281typename enable_if 2282< 2283 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2284 _Tp 2285>::type 2286atomic_fetch_or(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2287{ 2288 return __o->fetch_or(__op); 2289} 2290 2291// atomic_fetch_or_explicit 2292 2293template <class _Tp> 2294_LIBCPP_INLINE_VISIBILITY 2295typename enable_if 2296< 2297 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2298 _Tp 2299>::type 2300atomic_fetch_or_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2301{ 2302 return __o->fetch_or(__op, __m); 2303} 2304 2305template <class _Tp> 2306_LIBCPP_INLINE_VISIBILITY 2307typename enable_if 2308< 2309 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2310 _Tp 2311>::type 2312atomic_fetch_or_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2313{ 2314 return __o->fetch_or(__op, __m); 2315} 2316 2317// atomic_fetch_xor 2318 2319template <class _Tp> 2320_LIBCPP_INLINE_VISIBILITY 2321typename enable_if 2322< 2323 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2324 _Tp 2325>::type 2326atomic_fetch_xor(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2327{ 2328 return __o->fetch_xor(__op); 2329} 2330 2331template <class _Tp> 2332_LIBCPP_INLINE_VISIBILITY 2333typename enable_if 2334< 2335 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2336 _Tp 2337>::type 2338atomic_fetch_xor(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2339{ 2340 return __o->fetch_xor(__op); 2341} 2342 2343// atomic_fetch_xor_explicit 2344 2345template <class _Tp> 2346_LIBCPP_INLINE_VISIBILITY 2347typename enable_if 2348< 2349 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2350 _Tp 2351>::type 2352atomic_fetch_xor_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2353{ 2354 return __o->fetch_xor(__op, __m); 2355} 2356 2357template <class _Tp> 2358_LIBCPP_INLINE_VISIBILITY 2359typename enable_if 2360< 2361 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2362 _Tp 2363>::type 2364atomic_fetch_xor_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2365{ 2366 return __o->fetch_xor(__op, __m); 2367} 2368 2369// flag type and operations 2370 2371typedef struct atomic_flag 2372{ 2373 __cxx_atomic_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_; 2374 2375 _LIBCPP_INLINE_VISIBILITY 2376 bool test(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 2377 {return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m);} 2378 _LIBCPP_INLINE_VISIBILITY 2379 bool test(memory_order __m = memory_order_seq_cst) const _NOEXCEPT 2380 {return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m);} 2381 2382 _LIBCPP_INLINE_VISIBILITY 2383 bool test_and_set(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 2384 {return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m);} 2385 _LIBCPP_INLINE_VISIBILITY 2386 bool test_and_set(memory_order __m = memory_order_seq_cst) _NOEXCEPT 2387 {return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m);} 2388 _LIBCPP_INLINE_VISIBILITY 2389 void clear(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 2390 {__cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m);} 2391 _LIBCPP_INLINE_VISIBILITY 2392 void clear(memory_order __m = memory_order_seq_cst) _NOEXCEPT 2393 {__cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m);} 2394 2395 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2396 void wait(bool __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 2397 {__cxx_atomic_wait(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);} 2398 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2399 void wait(bool __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT 2400 {__cxx_atomic_wait(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);} 2401 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2402 void notify_one() volatile _NOEXCEPT 2403 {__cxx_atomic_notify_one(&__a_);} 2404 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2405 void notify_one() _NOEXCEPT 2406 {__cxx_atomic_notify_one(&__a_);} 2407 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2408 void notify_all() volatile _NOEXCEPT 2409 {__cxx_atomic_notify_all(&__a_);} 2410 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2411 void notify_all() _NOEXCEPT 2412 {__cxx_atomic_notify_all(&__a_);} 2413 2414#if _LIBCPP_STD_VER > 17 2415 _LIBCPP_INLINE_VISIBILITY constexpr 2416 atomic_flag() _NOEXCEPT : __a_(false) {} 2417#else 2418 _LIBCPP_INLINE_VISIBILITY 2419 atomic_flag() _NOEXCEPT _LIBCPP_DEFAULT 2420#endif 2421 2422 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR 2423 atomic_flag(bool __b) _NOEXCEPT : __a_(__b) {} // EXTENSION 2424 2425#ifndef _LIBCPP_CXX03_LANG 2426 atomic_flag(const atomic_flag&) = delete; 2427 atomic_flag& operator=(const atomic_flag&) = delete; 2428 atomic_flag& operator=(const atomic_flag&) volatile = delete; 2429#else 2430private: 2431 _LIBCPP_INLINE_VISIBILITY 2432 atomic_flag(const atomic_flag&); 2433 _LIBCPP_INLINE_VISIBILITY 2434 atomic_flag& operator=(const atomic_flag&); 2435 _LIBCPP_INLINE_VISIBILITY 2436 atomic_flag& operator=(const atomic_flag&) volatile; 2437#endif 2438} atomic_flag; 2439 2440 2441inline _LIBCPP_INLINE_VISIBILITY 2442bool 2443atomic_flag_test(const volatile atomic_flag* __o) _NOEXCEPT 2444{ 2445 return __o->test(); 2446} 2447 2448inline _LIBCPP_INLINE_VISIBILITY 2449bool 2450atomic_flag_test(const atomic_flag* __o) _NOEXCEPT 2451{ 2452 return __o->test(); 2453} 2454 2455inline _LIBCPP_INLINE_VISIBILITY 2456bool 2457atomic_flag_test_explicit(const volatile atomic_flag* __o, memory_order __m) _NOEXCEPT 2458{ 2459 return __o->test(__m); 2460} 2461 2462inline _LIBCPP_INLINE_VISIBILITY 2463bool 2464atomic_flag_test_explicit(const atomic_flag* __o, memory_order __m) _NOEXCEPT 2465{ 2466 return __o->test(__m); 2467} 2468 2469inline _LIBCPP_INLINE_VISIBILITY 2470bool 2471atomic_flag_test_and_set(volatile atomic_flag* __o) _NOEXCEPT 2472{ 2473 return __o->test_and_set(); 2474} 2475 2476inline _LIBCPP_INLINE_VISIBILITY 2477bool 2478atomic_flag_test_and_set(atomic_flag* __o) _NOEXCEPT 2479{ 2480 return __o->test_and_set(); 2481} 2482 2483inline _LIBCPP_INLINE_VISIBILITY 2484bool 2485atomic_flag_test_and_set_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT 2486{ 2487 return __o->test_and_set(__m); 2488} 2489 2490inline _LIBCPP_INLINE_VISIBILITY 2491bool 2492atomic_flag_test_and_set_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT 2493{ 2494 return __o->test_and_set(__m); 2495} 2496 2497inline _LIBCPP_INLINE_VISIBILITY 2498void 2499atomic_flag_clear(volatile atomic_flag* __o) _NOEXCEPT 2500{ 2501 __o->clear(); 2502} 2503 2504inline _LIBCPP_INLINE_VISIBILITY 2505void 2506atomic_flag_clear(atomic_flag* __o) _NOEXCEPT 2507{ 2508 __o->clear(); 2509} 2510 2511inline _LIBCPP_INLINE_VISIBILITY 2512void 2513atomic_flag_clear_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT 2514{ 2515 __o->clear(__m); 2516} 2517 2518inline _LIBCPP_INLINE_VISIBILITY 2519void 2520atomic_flag_clear_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT 2521{ 2522 __o->clear(__m); 2523} 2524 2525inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2526void 2527atomic_flag_wait(const volatile atomic_flag* __o, bool __v) _NOEXCEPT 2528{ 2529 __o->wait(__v); 2530} 2531 2532inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2533void 2534atomic_flag_wait(const atomic_flag* __o, bool __v) _NOEXCEPT 2535{ 2536 __o->wait(__v); 2537} 2538 2539inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2540void 2541atomic_flag_wait_explicit(const volatile atomic_flag* __o, 2542 bool __v, memory_order __m) _NOEXCEPT 2543{ 2544 __o->wait(__v, __m); 2545} 2546 2547inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2548void 2549atomic_flag_wait_explicit(const atomic_flag* __o, 2550 bool __v, memory_order __m) _NOEXCEPT 2551{ 2552 __o->wait(__v, __m); 2553} 2554 2555inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2556void 2557atomic_flag_notify_one(volatile atomic_flag* __o) _NOEXCEPT 2558{ 2559 __o->notify_one(); 2560} 2561 2562inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2563void 2564atomic_flag_notify_one(atomic_flag* __o) _NOEXCEPT 2565{ 2566 __o->notify_one(); 2567} 2568 2569inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2570void 2571atomic_flag_notify_all(volatile atomic_flag* __o) _NOEXCEPT 2572{ 2573 __o->notify_all(); 2574} 2575 2576inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2577void 2578atomic_flag_notify_all(atomic_flag* __o) _NOEXCEPT 2579{ 2580 __o->notify_all(); 2581} 2582 2583// fences 2584 2585inline _LIBCPP_INLINE_VISIBILITY 2586void 2587atomic_thread_fence(memory_order __m) _NOEXCEPT 2588{ 2589 __cxx_atomic_thread_fence(__m); 2590} 2591 2592inline _LIBCPP_INLINE_VISIBILITY 2593void 2594atomic_signal_fence(memory_order __m) _NOEXCEPT 2595{ 2596 __cxx_atomic_signal_fence(__m); 2597} 2598 2599// Atomics for standard typedef types 2600 2601typedef atomic<bool> atomic_bool; 2602typedef atomic<char> atomic_char; 2603typedef atomic<signed char> atomic_schar; 2604typedef atomic<unsigned char> atomic_uchar; 2605typedef atomic<short> atomic_short; 2606typedef atomic<unsigned short> atomic_ushort; 2607typedef atomic<int> atomic_int; 2608typedef atomic<unsigned int> atomic_uint; 2609typedef atomic<long> atomic_long; 2610typedef atomic<unsigned long> atomic_ulong; 2611typedef atomic<long long> atomic_llong; 2612typedef atomic<unsigned long long> atomic_ullong; 2613#ifndef _LIBCPP_HAS_NO_CHAR8_T 2614typedef atomic<char8_t> atomic_char8_t; 2615#endif 2616typedef atomic<char16_t> atomic_char16_t; 2617typedef atomic<char32_t> atomic_char32_t; 2618#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS 2619typedef atomic<wchar_t> atomic_wchar_t; 2620#endif 2621 2622typedef atomic<int_least8_t> atomic_int_least8_t; 2623typedef atomic<uint_least8_t> atomic_uint_least8_t; 2624typedef atomic<int_least16_t> atomic_int_least16_t; 2625typedef atomic<uint_least16_t> atomic_uint_least16_t; 2626typedef atomic<int_least32_t> atomic_int_least32_t; 2627typedef atomic<uint_least32_t> atomic_uint_least32_t; 2628typedef atomic<int_least64_t> atomic_int_least64_t; 2629typedef atomic<uint_least64_t> atomic_uint_least64_t; 2630 2631typedef atomic<int_fast8_t> atomic_int_fast8_t; 2632typedef atomic<uint_fast8_t> atomic_uint_fast8_t; 2633typedef atomic<int_fast16_t> atomic_int_fast16_t; 2634typedef atomic<uint_fast16_t> atomic_uint_fast16_t; 2635typedef atomic<int_fast32_t> atomic_int_fast32_t; 2636typedef atomic<uint_fast32_t> atomic_uint_fast32_t; 2637typedef atomic<int_fast64_t> atomic_int_fast64_t; 2638typedef atomic<uint_fast64_t> atomic_uint_fast64_t; 2639 2640typedef atomic< int8_t> atomic_int8_t; 2641typedef atomic<uint8_t> atomic_uint8_t; 2642typedef atomic< int16_t> atomic_int16_t; 2643typedef atomic<uint16_t> atomic_uint16_t; 2644typedef atomic< int32_t> atomic_int32_t; 2645typedef atomic<uint32_t> atomic_uint32_t; 2646typedef atomic< int64_t> atomic_int64_t; 2647typedef atomic<uint64_t> atomic_uint64_t; 2648 2649typedef atomic<intptr_t> atomic_intptr_t; 2650typedef atomic<uintptr_t> atomic_uintptr_t; 2651typedef atomic<size_t> atomic_size_t; 2652typedef atomic<ptrdiff_t> atomic_ptrdiff_t; 2653typedef atomic<intmax_t> atomic_intmax_t; 2654typedef atomic<uintmax_t> atomic_uintmax_t; 2655 2656// atomic_*_lock_free : prefer the contention type most highly, then the largest lock-free type 2657 2658#ifdef __cpp_lib_atomic_is_always_lock_free 2659# define _LIBCPP_CONTENTION_LOCK_FREE __atomic_always_lock_free(sizeof(__cxx_contention_t), 0) 2660#else 2661# define _LIBCPP_CONTENTION_LOCK_FREE false 2662#endif 2663 2664#if ATOMIC_LLONG_LOCK_FREE == 2 2665typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, long long>::type __libcpp_signed_lock_free; 2666typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned long long>::type __libcpp_unsigned_lock_free; 2667#elif ATOMIC_INT_LOCK_FREE == 2 2668typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, int>::type __libcpp_signed_lock_free; 2669typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned int>::type __libcpp_unsigned_lock_free; 2670#elif ATOMIC_SHORT_LOCK_FREE == 2 2671typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, short>::type __libcpp_signed_lock_free; 2672typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned short>::type __libcpp_unsigned_lock_free; 2673#elif ATOMIC_CHAR_LOCK_FREE == 2 2674typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, char>::type __libcpp_signed_lock_free; 2675typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned char>::type __libcpp_unsigned_lock_free; 2676#else 2677 // No signed/unsigned lock-free types 2678#endif 2679 2680typedef atomic<__libcpp_signed_lock_free> atomic_signed_lock_free; 2681typedef atomic<__libcpp_unsigned_lock_free> atomic_unsigned_lock_free; 2682 2683#define ATOMIC_FLAG_INIT {false} 2684#define ATOMIC_VAR_INIT(__v) {__v} 2685 2686_LIBCPP_END_NAMESPACE_STD 2687 2688#endif // _LIBCPP_ATOMIC 2689