1// -*- C++ -*- 2//===--------------------------- atomic -----------------------------------===// 3// 4// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 5// See https://llvm.org/LICENSE.txt for license information. 6// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 7// 8//===----------------------------------------------------------------------===// 9 10#ifndef _LIBCPP_ATOMIC 11#define _LIBCPP_ATOMIC 12 13/* 14 atomic synopsis 15 16namespace std 17{ 18 19// feature test macro [version.syn] 20 21#define __cpp_lib_atomic_is_always_lock_free 22#define __cpp_lib_atomic_flag_test 23#define __cpp_lib_atomic_lock_free_type_aliases 24#define __cpp_lib_atomic_wait 25 26 // order and consistency 27 28 enum memory_order: unspecified // enum class in C++20 29 { 30 relaxed, 31 consume, // load-consume 32 acquire, // load-acquire 33 release, // store-release 34 acq_rel, // store-release load-acquire 35 seq_cst // store-release load-acquire 36 }; 37 38 inline constexpr auto memory_order_relaxed = memory_order::relaxed; 39 inline constexpr auto memory_order_consume = memory_order::consume; 40 inline constexpr auto memory_order_acquire = memory_order::acquire; 41 inline constexpr auto memory_order_release = memory_order::release; 42 inline constexpr auto memory_order_acq_rel = memory_order::acq_rel; 43 inline constexpr auto memory_order_seq_cst = memory_order::seq_cst; 44 45template <class T> T kill_dependency(T y) noexcept; 46 47// lock-free property 48 49#define ATOMIC_BOOL_LOCK_FREE unspecified 50#define ATOMIC_CHAR_LOCK_FREE unspecified 51#define ATOMIC_CHAR16_T_LOCK_FREE unspecified 52#define ATOMIC_CHAR32_T_LOCK_FREE unspecified 53#define ATOMIC_WCHAR_T_LOCK_FREE unspecified 54#define ATOMIC_SHORT_LOCK_FREE unspecified 55#define ATOMIC_INT_LOCK_FREE unspecified 56#define ATOMIC_LONG_LOCK_FREE unspecified 57#define ATOMIC_LLONG_LOCK_FREE unspecified 58#define ATOMIC_POINTER_LOCK_FREE unspecified 59 60template <class T> 61struct atomic 62{ 63 using value_type = T; 64 65 static constexpr bool is_always_lock_free; 66 bool is_lock_free() const volatile noexcept; 67 bool is_lock_free() const noexcept; 68 69 atomic() noexcept = default; 70 constexpr atomic(T desr) noexcept; 71 atomic(const atomic&) = delete; 72 atomic& operator=(const atomic&) = delete; 73 atomic& operator=(const atomic&) volatile = delete; 74 75 T load(memory_order m = memory_order_seq_cst) const volatile noexcept; 76 T load(memory_order m = memory_order_seq_cst) const noexcept; 77 operator T() const volatile noexcept; 78 operator T() const noexcept; 79 void store(T desr, memory_order m = memory_order_seq_cst) volatile noexcept; 80 void store(T desr, memory_order m = memory_order_seq_cst) noexcept; 81 T operator=(T) volatile noexcept; 82 T operator=(T) noexcept; 83 84 T exchange(T desr, memory_order m = memory_order_seq_cst) volatile noexcept; 85 T exchange(T desr, memory_order m = memory_order_seq_cst) noexcept; 86 bool compare_exchange_weak(T& expc, T desr, 87 memory_order s, memory_order f) volatile noexcept; 88 bool compare_exchange_weak(T& expc, T desr, memory_order s, memory_order f) noexcept; 89 bool compare_exchange_strong(T& expc, T desr, 90 memory_order s, memory_order f) volatile noexcept; 91 bool compare_exchange_strong(T& expc, T desr, 92 memory_order s, memory_order f) noexcept; 93 bool compare_exchange_weak(T& expc, T desr, 94 memory_order m = memory_order_seq_cst) volatile noexcept; 95 bool compare_exchange_weak(T& expc, T desr, 96 memory_order m = memory_order_seq_cst) noexcept; 97 bool compare_exchange_strong(T& expc, T desr, 98 memory_order m = memory_order_seq_cst) volatile noexcept; 99 bool compare_exchange_strong(T& expc, T desr, 100 memory_order m = memory_order_seq_cst) noexcept; 101 102 void wait(T, memory_order = memory_order::seq_cst) const volatile noexcept; 103 void wait(T, memory_order = memory_order::seq_cst) const noexcept; 104 void notify_one() volatile noexcept; 105 void notify_one() noexcept; 106 void notify_all() volatile noexcept; 107 void notify_all() noexcept; 108}; 109 110template <> 111struct atomic<integral> 112{ 113 using value_type = integral; 114 using difference_type = value_type; 115 116 static constexpr bool is_always_lock_free; 117 bool is_lock_free() const volatile noexcept; 118 bool is_lock_free() const noexcept; 119 120 atomic() noexcept = default; 121 constexpr atomic(integral desr) noexcept; 122 atomic(const atomic&) = delete; 123 atomic& operator=(const atomic&) = delete; 124 atomic& operator=(const atomic&) volatile = delete; 125 126 integral load(memory_order m = memory_order_seq_cst) const volatile noexcept; 127 integral load(memory_order m = memory_order_seq_cst) const noexcept; 128 operator integral() const volatile noexcept; 129 operator integral() const noexcept; 130 void store(integral desr, memory_order m = memory_order_seq_cst) volatile noexcept; 131 void store(integral desr, memory_order m = memory_order_seq_cst) noexcept; 132 integral operator=(integral desr) volatile noexcept; 133 integral operator=(integral desr) noexcept; 134 135 integral exchange(integral desr, 136 memory_order m = memory_order_seq_cst) volatile noexcept; 137 integral exchange(integral desr, memory_order m = memory_order_seq_cst) noexcept; 138 bool compare_exchange_weak(integral& expc, integral desr, 139 memory_order s, memory_order f) volatile noexcept; 140 bool compare_exchange_weak(integral& expc, integral desr, 141 memory_order s, memory_order f) noexcept; 142 bool compare_exchange_strong(integral& expc, integral desr, 143 memory_order s, memory_order f) volatile noexcept; 144 bool compare_exchange_strong(integral& expc, integral desr, 145 memory_order s, memory_order f) noexcept; 146 bool compare_exchange_weak(integral& expc, integral desr, 147 memory_order m = memory_order_seq_cst) volatile noexcept; 148 bool compare_exchange_weak(integral& expc, integral desr, 149 memory_order m = memory_order_seq_cst) noexcept; 150 bool compare_exchange_strong(integral& expc, integral desr, 151 memory_order m = memory_order_seq_cst) volatile noexcept; 152 bool compare_exchange_strong(integral& expc, integral desr, 153 memory_order m = memory_order_seq_cst) noexcept; 154 155 integral fetch_add(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 156 integral fetch_add(integral op, memory_order m = memory_order_seq_cst) noexcept; 157 integral fetch_sub(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 158 integral fetch_sub(integral op, memory_order m = memory_order_seq_cst) noexcept; 159 integral fetch_and(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 160 integral fetch_and(integral op, memory_order m = memory_order_seq_cst) noexcept; 161 integral fetch_or(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 162 integral fetch_or(integral op, memory_order m = memory_order_seq_cst) noexcept; 163 integral fetch_xor(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 164 integral fetch_xor(integral op, memory_order m = memory_order_seq_cst) noexcept; 165 166 integral operator++(int) volatile noexcept; 167 integral operator++(int) noexcept; 168 integral operator--(int) volatile noexcept; 169 integral operator--(int) noexcept; 170 integral operator++() volatile noexcept; 171 integral operator++() noexcept; 172 integral operator--() volatile noexcept; 173 integral operator--() noexcept; 174 integral operator+=(integral op) volatile noexcept; 175 integral operator+=(integral op) noexcept; 176 integral operator-=(integral op) volatile noexcept; 177 integral operator-=(integral op) noexcept; 178 integral operator&=(integral op) volatile noexcept; 179 integral operator&=(integral op) noexcept; 180 integral operator|=(integral op) volatile noexcept; 181 integral operator|=(integral op) noexcept; 182 integral operator^=(integral op) volatile noexcept; 183 integral operator^=(integral op) noexcept; 184 185 void wait(integral, memory_order = memory_order::seq_cst) const volatile noexcept; 186 void wait(integral, memory_order = memory_order::seq_cst) const noexcept; 187 void notify_one() volatile noexcept; 188 void notify_one() noexcept; 189 void notify_all() volatile noexcept; 190 void notify_all() noexcept; 191}; 192 193template <class T> 194struct atomic<T*> 195{ 196 using value_type = T*; 197 using difference_type = ptrdiff_t; 198 199 static constexpr bool is_always_lock_free; 200 bool is_lock_free() const volatile noexcept; 201 bool is_lock_free() const noexcept; 202 203 atomic() noexcept = default; 204 constexpr atomic(T* desr) noexcept; 205 atomic(const atomic&) = delete; 206 atomic& operator=(const atomic&) = delete; 207 atomic& operator=(const atomic&) volatile = delete; 208 209 T* load(memory_order m = memory_order_seq_cst) const volatile noexcept; 210 T* load(memory_order m = memory_order_seq_cst) const noexcept; 211 operator T*() const volatile noexcept; 212 operator T*() const noexcept; 213 void store(T* desr, memory_order m = memory_order_seq_cst) volatile noexcept; 214 void store(T* desr, memory_order m = memory_order_seq_cst) noexcept; 215 T* operator=(T*) volatile noexcept; 216 T* operator=(T*) noexcept; 217 218 T* exchange(T* desr, memory_order m = memory_order_seq_cst) volatile noexcept; 219 T* exchange(T* desr, memory_order m = memory_order_seq_cst) noexcept; 220 bool compare_exchange_weak(T*& expc, T* desr, 221 memory_order s, memory_order f) volatile noexcept; 222 bool compare_exchange_weak(T*& expc, T* desr, 223 memory_order s, memory_order f) noexcept; 224 bool compare_exchange_strong(T*& expc, T* desr, 225 memory_order s, memory_order f) volatile noexcept; 226 bool compare_exchange_strong(T*& expc, T* desr, 227 memory_order s, memory_order f) noexcept; 228 bool compare_exchange_weak(T*& expc, T* desr, 229 memory_order m = memory_order_seq_cst) volatile noexcept; 230 bool compare_exchange_weak(T*& expc, T* desr, 231 memory_order m = memory_order_seq_cst) noexcept; 232 bool compare_exchange_strong(T*& expc, T* desr, 233 memory_order m = memory_order_seq_cst) volatile noexcept; 234 bool compare_exchange_strong(T*& expc, T* desr, 235 memory_order m = memory_order_seq_cst) noexcept; 236 T* fetch_add(ptrdiff_t op, memory_order m = memory_order_seq_cst) volatile noexcept; 237 T* fetch_add(ptrdiff_t op, memory_order m = memory_order_seq_cst) noexcept; 238 T* fetch_sub(ptrdiff_t op, memory_order m = memory_order_seq_cst) volatile noexcept; 239 T* fetch_sub(ptrdiff_t op, memory_order m = memory_order_seq_cst) noexcept; 240 241 T* operator++(int) volatile noexcept; 242 T* operator++(int) noexcept; 243 T* operator--(int) volatile noexcept; 244 T* operator--(int) noexcept; 245 T* operator++() volatile noexcept; 246 T* operator++() noexcept; 247 T* operator--() volatile noexcept; 248 T* operator--() noexcept; 249 T* operator+=(ptrdiff_t op) volatile noexcept; 250 T* operator+=(ptrdiff_t op) noexcept; 251 T* operator-=(ptrdiff_t op) volatile noexcept; 252 T* operator-=(ptrdiff_t op) noexcept; 253 254 void wait(T*, memory_order = memory_order::seq_cst) const volatile noexcept; 255 void wait(T*, memory_order = memory_order::seq_cst) const noexcept; 256 void notify_one() volatile noexcept; 257 void notify_one() noexcept; 258 void notify_all() volatile noexcept; 259 void notify_all() noexcept; 260}; 261 262 263template <class T> 264 bool atomic_is_lock_free(const volatile atomic<T>* obj) noexcept; 265 266template <class T> 267 bool atomic_is_lock_free(const atomic<T>* obj) noexcept; 268 269template <class T> 270 void atomic_store(volatile atomic<T>* obj, T desr) noexcept; 271 272template <class T> 273 void atomic_store(atomic<T>* obj, T desr) noexcept; 274 275template <class T> 276 void atomic_store_explicit(volatile atomic<T>* obj, T desr, memory_order m) noexcept; 277 278template <class T> 279 void atomic_store_explicit(atomic<T>* obj, T desr, memory_order m) noexcept; 280 281template <class T> 282 T atomic_load(const volatile atomic<T>* obj) noexcept; 283 284template <class T> 285 T atomic_load(const atomic<T>* obj) noexcept; 286 287template <class T> 288 T atomic_load_explicit(const volatile atomic<T>* obj, memory_order m) noexcept; 289 290template <class T> 291 T atomic_load_explicit(const atomic<T>* obj, memory_order m) noexcept; 292 293template <class T> 294 T atomic_exchange(volatile atomic<T>* obj, T desr) noexcept; 295 296template <class T> 297 T atomic_exchange(atomic<T>* obj, T desr) noexcept; 298 299template <class T> 300 T atomic_exchange_explicit(volatile atomic<T>* obj, T desr, memory_order m) noexcept; 301 302template <class T> 303 T atomic_exchange_explicit(atomic<T>* obj, T desr, memory_order m) noexcept; 304 305template <class T> 306 bool atomic_compare_exchange_weak(volatile atomic<T>* obj, T* expc, T desr) noexcept; 307 308template <class T> 309 bool atomic_compare_exchange_weak(atomic<T>* obj, T* expc, T desr) noexcept; 310 311template <class T> 312 bool atomic_compare_exchange_strong(volatile atomic<T>* obj, T* expc, T desr) noexcept; 313 314template <class T> 315 bool atomic_compare_exchange_strong(atomic<T>* obj, T* expc, T desr) noexcept; 316 317template <class T> 318 bool atomic_compare_exchange_weak_explicit(volatile atomic<T>* obj, T* expc, 319 T desr, 320 memory_order s, memory_order f) noexcept; 321 322template <class T> 323 bool atomic_compare_exchange_weak_explicit(atomic<T>* obj, T* expc, T desr, 324 memory_order s, memory_order f) noexcept; 325 326template <class T> 327 bool atomic_compare_exchange_strong_explicit(volatile atomic<T>* obj, 328 T* expc, T desr, 329 memory_order s, memory_order f) noexcept; 330 331template <class T> 332 bool atomic_compare_exchange_strong_explicit(atomic<T>* obj, T* expc, 333 T desr, 334 memory_order s, memory_order f) noexcept; 335 336template <class T> 337 void atomic_wait(const volatile atomic<T>* obj, T old) noexcept; 338 339template <class T> 340 void atomic_wait(const atomic<T>* obj, T old) noexcept; 341 342template <class T> 343 void atomic_wait_explicit(const volatile atomic<T>* obj, T old, memory_order m) noexcept; 344 345template <class T> 346 void atomic_wait_explicit(const atomic<T>* obj, T old, memory_order m) noexcept; 347 348template <class T> 349 void atomic_one(volatile atomic<T>* obj) noexcept; 350 351template <class T> 352 void atomic_one(atomic<T>* obj) noexcept; 353 354template <class T> 355 void atomic_all(volatile atomic<T>* obj) noexcept; 356 357template <class T> 358 void atomic_all(atomic<T>* obj) noexcept; 359 360template <class Integral> 361 Integral atomic_fetch_add(volatile atomic<Integral>* obj, Integral op) noexcept; 362 363template <class Integral> 364 Integral atomic_fetch_add(atomic<Integral>* obj, Integral op) noexcept; 365 366template <class Integral> 367 Integral atomic_fetch_add_explicit(volatile atomic<Integral>* obj, Integral op, 368 memory_order m) noexcept; 369template <class Integral> 370 Integral atomic_fetch_add_explicit(atomic<Integral>* obj, Integral op, 371 memory_order m) noexcept; 372template <class Integral> 373 Integral atomic_fetch_sub(volatile atomic<Integral>* obj, Integral op) noexcept; 374 375template <class Integral> 376 Integral atomic_fetch_sub(atomic<Integral>* obj, Integral op) noexcept; 377 378template <class Integral> 379 Integral atomic_fetch_sub_explicit(volatile atomic<Integral>* obj, Integral op, 380 memory_order m) noexcept; 381 382template <class Integral> 383 Integral atomic_fetch_sub_explicit(atomic<Integral>* obj, Integral op, 384 memory_order m) noexcept; 385 386template <class Integral> 387 Integral atomic_fetch_and(volatile atomic<Integral>* obj, Integral op) noexcept; 388 389template <class Integral> 390 Integral atomic_fetch_and(atomic<Integral>* obj, Integral op) noexcept; 391 392template <class Integral> 393 Integral atomic_fetch_and_explicit(volatile atomic<Integral>* obj, Integral op, 394 memory_order m) noexcept; 395 396template <class Integral> 397 Integral atomic_fetch_and_explicit(atomic<Integral>* obj, Integral op, 398 memory_order m) noexcept; 399 400template <class Integral> 401 Integral atomic_fetch_or(volatile atomic<Integral>* obj, Integral op) noexcept; 402 403template <class Integral> 404 Integral atomic_fetch_or(atomic<Integral>* obj, Integral op) noexcept; 405 406template <class Integral> 407 Integral atomic_fetch_or_explicit(volatile atomic<Integral>* obj, Integral op, 408 memory_order m) noexcept; 409 410template <class Integral> 411 Integral atomic_fetch_or_explicit(atomic<Integral>* obj, Integral op, 412 memory_order m) noexcept; 413 414template <class Integral> 415 Integral atomic_fetch_xor(volatile atomic<Integral>* obj, Integral op) noexcept; 416 417template <class Integral> 418 Integral atomic_fetch_xor(atomic<Integral>* obj, Integral op) noexcept; 419 420template <class Integral> 421 Integral atomic_fetch_xor_explicit(volatile atomic<Integral>* obj, Integral op, 422 memory_order m) noexcept; 423 424template <class Integral> 425 Integral atomic_fetch_xor_explicit(atomic<Integral>* obj, Integral op, 426 memory_order m) noexcept; 427 428template <class T> 429 T* atomic_fetch_add(volatile atomic<T*>* obj, ptrdiff_t op) noexcept; 430 431template <class T> 432 T* atomic_fetch_add(atomic<T*>* obj, ptrdiff_t op) noexcept; 433 434template <class T> 435 T* atomic_fetch_add_explicit(volatile atomic<T*>* obj, ptrdiff_t op, 436 memory_order m) noexcept; 437 438template <class T> 439 T* atomic_fetch_add_explicit(atomic<T*>* obj, ptrdiff_t op, memory_order m) noexcept; 440 441template <class T> 442 T* atomic_fetch_sub(volatile atomic<T*>* obj, ptrdiff_t op) noexcept; 443 444template <class T> 445 T* atomic_fetch_sub(atomic<T*>* obj, ptrdiff_t op) noexcept; 446 447template <class T> 448 T* atomic_fetch_sub_explicit(volatile atomic<T*>* obj, ptrdiff_t op, 449 memory_order m) noexcept; 450 451template <class T> 452 T* atomic_fetch_sub_explicit(atomic<T*>* obj, ptrdiff_t op, memory_order m) noexcept; 453 454// Atomics for standard typedef types 455 456typedef atomic<bool> atomic_bool; 457typedef atomic<char> atomic_char; 458typedef atomic<signed char> atomic_schar; 459typedef atomic<unsigned char> atomic_uchar; 460typedef atomic<short> atomic_short; 461typedef atomic<unsigned short> atomic_ushort; 462typedef atomic<int> atomic_int; 463typedef atomic<unsigned int> atomic_uint; 464typedef atomic<long> atomic_long; 465typedef atomic<unsigned long> atomic_ulong; 466typedef atomic<long long> atomic_llong; 467typedef atomic<unsigned long long> atomic_ullong; 468typedef atomic<char16_t> atomic_char16_t; 469typedef atomic<char32_t> atomic_char32_t; 470typedef atomic<wchar_t> atomic_wchar_t; 471 472typedef atomic<int_least8_t> atomic_int_least8_t; 473typedef atomic<uint_least8_t> atomic_uint_least8_t; 474typedef atomic<int_least16_t> atomic_int_least16_t; 475typedef atomic<uint_least16_t> atomic_uint_least16_t; 476typedef atomic<int_least32_t> atomic_int_least32_t; 477typedef atomic<uint_least32_t> atomic_uint_least32_t; 478typedef atomic<int_least64_t> atomic_int_least64_t; 479typedef atomic<uint_least64_t> atomic_uint_least64_t; 480 481typedef atomic<int_fast8_t> atomic_int_fast8_t; 482typedef atomic<uint_fast8_t> atomic_uint_fast8_t; 483typedef atomic<int_fast16_t> atomic_int_fast16_t; 484typedef atomic<uint_fast16_t> atomic_uint_fast16_t; 485typedef atomic<int_fast32_t> atomic_int_fast32_t; 486typedef atomic<uint_fast32_t> atomic_uint_fast32_t; 487typedef atomic<int_fast64_t> atomic_int_fast64_t; 488typedef atomic<uint_fast64_t> atomic_uint_fast64_t; 489 490typedef atomic<int8_t> atomic_int8_t; 491typedef atomic<uint8_t> atomic_uint8_t; 492typedef atomic<int16_t> atomic_int16_t; 493typedef atomic<uint16_t> atomic_uint16_t; 494typedef atomic<int32_t> atomic_int32_t; 495typedef atomic<uint32_t> atomic_uint32_t; 496typedef atomic<int64_t> atomic_int64_t; 497typedef atomic<uint64_t> atomic_uint64_t; 498 499typedef atomic<intptr_t> atomic_intptr_t; 500typedef atomic<uintptr_t> atomic_uintptr_t; 501typedef atomic<size_t> atomic_size_t; 502typedef atomic<ptrdiff_t> atomic_ptrdiff_t; 503typedef atomic<intmax_t> atomic_intmax_t; 504typedef atomic<uintmax_t> atomic_uintmax_t; 505 506// flag type and operations 507 508typedef struct atomic_flag 509{ 510 atomic_flag() noexcept = default; 511 atomic_flag(const atomic_flag&) = delete; 512 atomic_flag& operator=(const atomic_flag&) = delete; 513 atomic_flag& operator=(const atomic_flag&) volatile = delete; 514 515 bool test(memory_order m = memory_order_seq_cst) volatile noexcept; 516 bool test(memory_order m = memory_order_seq_cst) noexcept; 517 bool test_and_set(memory_order m = memory_order_seq_cst) volatile noexcept; 518 bool test_and_set(memory_order m = memory_order_seq_cst) noexcept; 519 void clear(memory_order m = memory_order_seq_cst) volatile noexcept; 520 void clear(memory_order m = memory_order_seq_cst) noexcept; 521 522 void wait(bool, memory_order = memory_order::seq_cst) const volatile noexcept; 523 void wait(bool, memory_order = memory_order::seq_cst) const noexcept; 524 void notify_one() volatile noexcept; 525 void notify_one() noexcept; 526 void notify_all() volatile noexcept; 527 void notify_all() noexcept; 528} atomic_flag; 529 530bool atomic_flag_test(volatile atomic_flag* obj) noexcept; 531bool atomic_flag_test(atomic_flag* obj) noexcept; 532bool atomic_flag_test_explicit(volatile atomic_flag* obj, 533 memory_order m) noexcept; 534bool atomic_flag_test_explicit(atomic_flag* obj, memory_order m) noexcept; 535bool atomic_flag_test_and_set(volatile atomic_flag* obj) noexcept; 536bool atomic_flag_test_and_set(atomic_flag* obj) noexcept; 537bool atomic_flag_test_and_set_explicit(volatile atomic_flag* obj, 538 memory_order m) noexcept; 539bool atomic_flag_test_and_set_explicit(atomic_flag* obj, memory_order m) noexcept; 540void atomic_flag_clear(volatile atomic_flag* obj) noexcept; 541void atomic_flag_clear(atomic_flag* obj) noexcept; 542void atomic_flag_clear_explicit(volatile atomic_flag* obj, memory_order m) noexcept; 543void atomic_flag_clear_explicit(atomic_flag* obj, memory_order m) noexcept; 544 545void atomic_wait(const volatile atomic_flag* obj, T old) noexcept; 546void atomic_wait(const atomic_flag* obj, T old) noexcept; 547void atomic_wait_explicit(const volatile atomic_flag* obj, T old, memory_order m) noexcept; 548void atomic_wait_explicit(const atomic_flag* obj, T old, memory_order m) noexcept; 549void atomic_one(volatile atomic_flag* obj) noexcept; 550void atomic_one(atomic_flag* obj) noexcept; 551void atomic_all(volatile atomic_flag* obj) noexcept; 552void atomic_all(atomic_flag* obj) noexcept; 553 554// fences 555 556void atomic_thread_fence(memory_order m) noexcept; 557void atomic_signal_fence(memory_order m) noexcept; 558 559// deprecated 560 561template <class T> 562 void atomic_init(volatile atomic<T>* obj, typename atomic<T>::value_type desr) noexcept; 563 564template <class T> 565 void atomic_init(atomic<T>* obj, typename atomic<T>::value_type desr) noexcept; 566 567#define ATOMIC_VAR_INIT(value) see below 568 569#define ATOMIC_FLAG_INIT see below 570 571} // std 572 573*/ 574 575#include <__config> 576#include <__threading_support> 577#include <cstddef> 578#include <cstdint> 579#include <cstring> 580#include <type_traits> 581#include <version> 582 583#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) 584#pragma GCC system_header 585#endif 586 587#ifdef _LIBCPP_HAS_NO_THREADS 588# error <atomic> is not supported on this single threaded system 589#endif 590#ifdef _LIBCPP_HAS_NO_ATOMIC_HEADER 591# error <atomic> is not implemented 592#endif 593#ifdef kill_dependency 594# error C++ standard library is incompatible with <stdatomic.h> 595#endif 596 597#define _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) \ 598 _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_consume || \ 599 __m == memory_order_acquire || \ 600 __m == memory_order_acq_rel, \ 601 "memory order argument to atomic operation is invalid") 602 603#define _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) \ 604 _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_release || \ 605 __m == memory_order_acq_rel, \ 606 "memory order argument to atomic operation is invalid") 607 608#define _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__m, __f) \ 609 _LIBCPP_DIAGNOSE_WARNING(__f == memory_order_release || \ 610 __f == memory_order_acq_rel, \ 611 "memory order argument to atomic operation is invalid") 612 613_LIBCPP_BEGIN_NAMESPACE_STD 614 615// Figure out what the underlying type for `memory_order` would be if it were 616// declared as an unscoped enum (accounting for -fshort-enums). Use this result 617// to pin the underlying type in C++20. 618enum __legacy_memory_order { 619 __mo_relaxed, 620 __mo_consume, 621 __mo_acquire, 622 __mo_release, 623 __mo_acq_rel, 624 __mo_seq_cst 625}; 626 627typedef underlying_type<__legacy_memory_order>::type __memory_order_underlying_t; 628 629#if _LIBCPP_STD_VER > 17 630 631enum class memory_order : __memory_order_underlying_t { 632 relaxed = __mo_relaxed, 633 consume = __mo_consume, 634 acquire = __mo_acquire, 635 release = __mo_release, 636 acq_rel = __mo_acq_rel, 637 seq_cst = __mo_seq_cst 638}; 639 640inline constexpr auto memory_order_relaxed = memory_order::relaxed; 641inline constexpr auto memory_order_consume = memory_order::consume; 642inline constexpr auto memory_order_acquire = memory_order::acquire; 643inline constexpr auto memory_order_release = memory_order::release; 644inline constexpr auto memory_order_acq_rel = memory_order::acq_rel; 645inline constexpr auto memory_order_seq_cst = memory_order::seq_cst; 646 647#else 648 649typedef enum memory_order { 650 memory_order_relaxed = __mo_relaxed, 651 memory_order_consume = __mo_consume, 652 memory_order_acquire = __mo_acquire, 653 memory_order_release = __mo_release, 654 memory_order_acq_rel = __mo_acq_rel, 655 memory_order_seq_cst = __mo_seq_cst, 656} memory_order; 657 658#endif // _LIBCPP_STD_VER > 17 659 660template <typename _Tp> _LIBCPP_INLINE_VISIBILITY 661bool __cxx_nonatomic_compare_equal(_Tp const& __lhs, _Tp const& __rhs) { 662 return memcmp(&__lhs, &__rhs, sizeof(_Tp)) == 0; 663} 664 665static_assert((is_same<underlying_type<memory_order>::type, __memory_order_underlying_t>::value), 666 "unexpected underlying type for std::memory_order"); 667 668#if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) || \ 669 defined(_LIBCPP_ATOMIC_ONLY_USE_BUILTINS) 670 671// [atomics.types.generic]p1 guarantees _Tp is trivially copyable. Because 672// the default operator= in an object is not volatile, a byte-by-byte copy 673// is required. 674template <typename _Tp, typename _Tv> _LIBCPP_INLINE_VISIBILITY 675typename enable_if<is_assignable<_Tp&, _Tv>::value>::type 676__cxx_atomic_assign_volatile(_Tp& __a_value, _Tv const& __val) { 677 __a_value = __val; 678} 679template <typename _Tp, typename _Tv> _LIBCPP_INLINE_VISIBILITY 680typename enable_if<is_assignable<_Tp&, _Tv>::value>::type 681__cxx_atomic_assign_volatile(_Tp volatile& __a_value, _Tv volatile const& __val) { 682 volatile char* __to = reinterpret_cast<volatile char*>(&__a_value); 683 volatile char* __end = __to + sizeof(_Tp); 684 volatile const char* __from = reinterpret_cast<volatile const char*>(&__val); 685 while (__to != __end) 686 *__to++ = *__from++; 687} 688 689#endif 690 691#if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) 692 693template <typename _Tp> 694struct __cxx_atomic_base_impl { 695 696 _LIBCPP_INLINE_VISIBILITY 697#ifndef _LIBCPP_CXX03_LANG 698 __cxx_atomic_base_impl() _NOEXCEPT = default; 699#else 700 __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {} 701#endif // _LIBCPP_CXX03_LANG 702 _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT 703 : __a_value(value) {} 704 _Tp __a_value; 705}; 706 707_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR int __to_gcc_order(memory_order __order) { 708 // Avoid switch statement to make this a constexpr. 709 return __order == memory_order_relaxed ? __ATOMIC_RELAXED: 710 (__order == memory_order_acquire ? __ATOMIC_ACQUIRE: 711 (__order == memory_order_release ? __ATOMIC_RELEASE: 712 (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST: 713 (__order == memory_order_acq_rel ? __ATOMIC_ACQ_REL: 714 __ATOMIC_CONSUME)))); 715} 716 717_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR int __to_gcc_failure_order(memory_order __order) { 718 // Avoid switch statement to make this a constexpr. 719 return __order == memory_order_relaxed ? __ATOMIC_RELAXED: 720 (__order == memory_order_acquire ? __ATOMIC_ACQUIRE: 721 (__order == memory_order_release ? __ATOMIC_RELAXED: 722 (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST: 723 (__order == memory_order_acq_rel ? __ATOMIC_ACQUIRE: 724 __ATOMIC_CONSUME)))); 725} 726 727template <typename _Tp> 728_LIBCPP_INLINE_VISIBILITY 729void __cxx_atomic_init(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val) { 730 __cxx_atomic_assign_volatile(__a->__a_value, __val); 731} 732 733template <typename _Tp> 734_LIBCPP_INLINE_VISIBILITY 735void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) { 736 __a->__a_value = __val; 737} 738 739_LIBCPP_INLINE_VISIBILITY inline 740void __cxx_atomic_thread_fence(memory_order __order) { 741 __atomic_thread_fence(__to_gcc_order(__order)); 742} 743 744_LIBCPP_INLINE_VISIBILITY inline 745void __cxx_atomic_signal_fence(memory_order __order) { 746 __atomic_signal_fence(__to_gcc_order(__order)); 747} 748 749template <typename _Tp> 750_LIBCPP_INLINE_VISIBILITY 751void __cxx_atomic_store(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val, 752 memory_order __order) { 753 __atomic_store(&__a->__a_value, &__val, 754 __to_gcc_order(__order)); 755} 756 757template <typename _Tp> 758_LIBCPP_INLINE_VISIBILITY 759void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, 760 memory_order __order) { 761 __atomic_store(&__a->__a_value, &__val, 762 __to_gcc_order(__order)); 763} 764 765template <typename _Tp> 766_LIBCPP_INLINE_VISIBILITY 767_Tp __cxx_atomic_load(const volatile __cxx_atomic_base_impl<_Tp>* __a, 768 memory_order __order) { 769 _Tp __ret; 770 __atomic_load(&__a->__a_value, &__ret, 771 __to_gcc_order(__order)); 772 return __ret; 773} 774 775template <typename _Tp> 776_LIBCPP_INLINE_VISIBILITY 777_Tp __cxx_atomic_load(const __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) { 778 _Tp __ret; 779 __atomic_load(&__a->__a_value, &__ret, 780 __to_gcc_order(__order)); 781 return __ret; 782} 783 784template <typename _Tp> 785_LIBCPP_INLINE_VISIBILITY 786_Tp __cxx_atomic_exchange(volatile __cxx_atomic_base_impl<_Tp>* __a, 787 _Tp __value, memory_order __order) { 788 _Tp __ret; 789 __atomic_exchange(&__a->__a_value, &__value, &__ret, 790 __to_gcc_order(__order)); 791 return __ret; 792} 793 794template <typename _Tp> 795_LIBCPP_INLINE_VISIBILITY 796_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, 797 memory_order __order) { 798 _Tp __ret; 799 __atomic_exchange(&__a->__a_value, &__value, &__ret, 800 __to_gcc_order(__order)); 801 return __ret; 802} 803 804template <typename _Tp> 805_LIBCPP_INLINE_VISIBILITY 806bool __cxx_atomic_compare_exchange_strong( 807 volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, 808 memory_order __success, memory_order __failure) { 809 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 810 false, 811 __to_gcc_order(__success), 812 __to_gcc_failure_order(__failure)); 813} 814 815template <typename _Tp> 816_LIBCPP_INLINE_VISIBILITY 817bool __cxx_atomic_compare_exchange_strong( 818 __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, 819 memory_order __failure) { 820 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 821 false, 822 __to_gcc_order(__success), 823 __to_gcc_failure_order(__failure)); 824} 825 826template <typename _Tp> 827_LIBCPP_INLINE_VISIBILITY 828bool __cxx_atomic_compare_exchange_weak( 829 volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, 830 memory_order __success, memory_order __failure) { 831 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 832 true, 833 __to_gcc_order(__success), 834 __to_gcc_failure_order(__failure)); 835} 836 837template <typename _Tp> 838_LIBCPP_INLINE_VISIBILITY 839bool __cxx_atomic_compare_exchange_weak( 840 __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, 841 memory_order __failure) { 842 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 843 true, 844 __to_gcc_order(__success), 845 __to_gcc_failure_order(__failure)); 846} 847 848template <typename _Tp> 849struct __skip_amt { enum {value = 1}; }; 850 851template <typename _Tp> 852struct __skip_amt<_Tp*> { enum {value = sizeof(_Tp)}; }; 853 854// FIXME: Haven't figured out what the spec says about using arrays with 855// atomic_fetch_add. Force a failure rather than creating bad behavior. 856template <typename _Tp> 857struct __skip_amt<_Tp[]> { }; 858template <typename _Tp, int n> 859struct __skip_amt<_Tp[n]> { }; 860 861template <typename _Tp, typename _Td> 862_LIBCPP_INLINE_VISIBILITY 863_Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_base_impl<_Tp>* __a, 864 _Td __delta, memory_order __order) { 865 return __atomic_fetch_add(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 866 __to_gcc_order(__order)); 867} 868 869template <typename _Tp, typename _Td> 870_LIBCPP_INLINE_VISIBILITY 871_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, 872 memory_order __order) { 873 return __atomic_fetch_add(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 874 __to_gcc_order(__order)); 875} 876 877template <typename _Tp, typename _Td> 878_LIBCPP_INLINE_VISIBILITY 879_Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_base_impl<_Tp>* __a, 880 _Td __delta, memory_order __order) { 881 return __atomic_fetch_sub(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 882 __to_gcc_order(__order)); 883} 884 885template <typename _Tp, typename _Td> 886_LIBCPP_INLINE_VISIBILITY 887_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, 888 memory_order __order) { 889 return __atomic_fetch_sub(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 890 __to_gcc_order(__order)); 891} 892 893template <typename _Tp> 894_LIBCPP_INLINE_VISIBILITY 895_Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_base_impl<_Tp>* __a, 896 _Tp __pattern, memory_order __order) { 897 return __atomic_fetch_and(&__a->__a_value, __pattern, 898 __to_gcc_order(__order)); 899} 900 901template <typename _Tp> 902_LIBCPP_INLINE_VISIBILITY 903_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, 904 _Tp __pattern, memory_order __order) { 905 return __atomic_fetch_and(&__a->__a_value, __pattern, 906 __to_gcc_order(__order)); 907} 908 909template <typename _Tp> 910_LIBCPP_INLINE_VISIBILITY 911_Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_base_impl<_Tp>* __a, 912 _Tp __pattern, memory_order __order) { 913 return __atomic_fetch_or(&__a->__a_value, __pattern, 914 __to_gcc_order(__order)); 915} 916 917template <typename _Tp> 918_LIBCPP_INLINE_VISIBILITY 919_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, 920 memory_order __order) { 921 return __atomic_fetch_or(&__a->__a_value, __pattern, 922 __to_gcc_order(__order)); 923} 924 925template <typename _Tp> 926_LIBCPP_INLINE_VISIBILITY 927_Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_base_impl<_Tp>* __a, 928 _Tp __pattern, memory_order __order) { 929 return __atomic_fetch_xor(&__a->__a_value, __pattern, 930 __to_gcc_order(__order)); 931} 932 933template <typename _Tp> 934_LIBCPP_INLINE_VISIBILITY 935_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, 936 memory_order __order) { 937 return __atomic_fetch_xor(&__a->__a_value, __pattern, 938 __to_gcc_order(__order)); 939} 940 941#define __cxx_atomic_is_lock_free(__s) __atomic_is_lock_free(__s, 0) 942 943#elif defined(_LIBCPP_HAS_C_ATOMIC_IMP) 944 945template <typename _Tp> 946struct __cxx_atomic_base_impl { 947 948 _LIBCPP_INLINE_VISIBILITY 949#ifndef _LIBCPP_CXX03_LANG 950 __cxx_atomic_base_impl() _NOEXCEPT = default; 951#else 952 __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {} 953#endif // _LIBCPP_CXX03_LANG 954 _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT 955 : __a_value(value) {} 956 _LIBCPP_DISABLE_EXTENSION_WARNING _Atomic(_Tp) __a_value; 957}; 958 959#define __cxx_atomic_is_lock_free(__s) __c11_atomic_is_lock_free(__s) 960 961_LIBCPP_INLINE_VISIBILITY inline 962void __cxx_atomic_thread_fence(memory_order __order) _NOEXCEPT { 963 __c11_atomic_thread_fence(static_cast<__memory_order_underlying_t>(__order)); 964} 965 966_LIBCPP_INLINE_VISIBILITY inline 967void __cxx_atomic_signal_fence(memory_order __order) _NOEXCEPT { 968 __c11_atomic_signal_fence(static_cast<__memory_order_underlying_t>(__order)); 969} 970 971template<class _Tp> 972_LIBCPP_INLINE_VISIBILITY 973void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val) _NOEXCEPT { 974 __c11_atomic_init(&__a->__a_value, __val); 975} 976template<class _Tp> 977_LIBCPP_INLINE_VISIBILITY 978void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val) _NOEXCEPT { 979 __c11_atomic_init(&__a->__a_value, __val); 980} 981 982template<class _Tp> 983_LIBCPP_INLINE_VISIBILITY 984void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT { 985 __c11_atomic_store(&__a->__a_value, __val, static_cast<__memory_order_underlying_t>(__order)); 986} 987template<class _Tp> 988_LIBCPP_INLINE_VISIBILITY 989void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val, memory_order __order) _NOEXCEPT { 990 __c11_atomic_store(&__a->__a_value, __val, static_cast<__memory_order_underlying_t>(__order)); 991} 992 993template<class _Tp> 994_LIBCPP_INLINE_VISIBILITY 995_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const volatile* __a, memory_order __order) _NOEXCEPT { 996 using __ptr_type = typename remove_const<decltype(__a->__a_value)>::type*; 997 return __c11_atomic_load(const_cast<__ptr_type>(&__a->__a_value), static_cast<__memory_order_underlying_t>(__order)); 998} 999template<class _Tp> 1000_LIBCPP_INLINE_VISIBILITY 1001_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const* __a, memory_order __order) _NOEXCEPT { 1002 using __ptr_type = typename remove_const<decltype(__a->__a_value)>::type*; 1003 return __c11_atomic_load(const_cast<__ptr_type>(&__a->__a_value), static_cast<__memory_order_underlying_t>(__order)); 1004} 1005 1006template<class _Tp> 1007_LIBCPP_INLINE_VISIBILITY 1008_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT { 1009 return __c11_atomic_exchange(&__a->__a_value, __value, static_cast<__memory_order_underlying_t>(__order)); 1010} 1011template<class _Tp> 1012_LIBCPP_INLINE_VISIBILITY 1013_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> * __a, _Tp __value, memory_order __order) _NOEXCEPT { 1014 return __c11_atomic_exchange(&__a->__a_value, __value, static_cast<__memory_order_underlying_t>(__order)); 1015} 1016 1017template<class _Tp> 1018_LIBCPP_INLINE_VISIBILITY 1019bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 1020 return __c11_atomic_compare_exchange_strong(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__failure)); 1021} 1022template<class _Tp> 1023_LIBCPP_INLINE_VISIBILITY 1024bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 1025 return __c11_atomic_compare_exchange_strong(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__failure)); 1026} 1027 1028template<class _Tp> 1029_LIBCPP_INLINE_VISIBILITY 1030bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 1031 return __c11_atomic_compare_exchange_weak(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__failure)); 1032} 1033template<class _Tp> 1034_LIBCPP_INLINE_VISIBILITY 1035bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 1036 return __c11_atomic_compare_exchange_weak(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__failure)); 1037} 1038 1039template<class _Tp> 1040_LIBCPP_INLINE_VISIBILITY 1041_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1042 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1043} 1044template<class _Tp> 1045_LIBCPP_INLINE_VISIBILITY 1046_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1047 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1048} 1049 1050template<class _Tp> 1051_LIBCPP_INLINE_VISIBILITY 1052_Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1053 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1054} 1055template<class _Tp> 1056_LIBCPP_INLINE_VISIBILITY 1057_Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1058 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1059} 1060 1061template<class _Tp> 1062_LIBCPP_INLINE_VISIBILITY 1063_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1064 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1065} 1066template<class _Tp> 1067_LIBCPP_INLINE_VISIBILITY 1068_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1069 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1070} 1071template<class _Tp> 1072_LIBCPP_INLINE_VISIBILITY 1073_Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1074 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1075} 1076template<class _Tp> 1077_LIBCPP_INLINE_VISIBILITY 1078_Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1079 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1080} 1081 1082template<class _Tp> 1083_LIBCPP_INLINE_VISIBILITY 1084_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1085 return __c11_atomic_fetch_and(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1086} 1087template<class _Tp> 1088_LIBCPP_INLINE_VISIBILITY 1089_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1090 return __c11_atomic_fetch_and(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1091} 1092 1093template<class _Tp> 1094_LIBCPP_INLINE_VISIBILITY 1095_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1096 return __c11_atomic_fetch_or(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1097} 1098template<class _Tp> 1099_LIBCPP_INLINE_VISIBILITY 1100_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1101 return __c11_atomic_fetch_or(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1102} 1103 1104template<class _Tp> 1105_LIBCPP_INLINE_VISIBILITY 1106_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1107 return __c11_atomic_fetch_xor(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1108} 1109template<class _Tp> 1110_LIBCPP_INLINE_VISIBILITY 1111_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1112 return __c11_atomic_fetch_xor(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1113} 1114 1115#endif // _LIBCPP_HAS_GCC_ATOMIC_IMP, _LIBCPP_HAS_C_ATOMIC_IMP 1116 1117template <class _Tp> 1118_LIBCPP_INLINE_VISIBILITY 1119_Tp kill_dependency(_Tp __y) _NOEXCEPT 1120{ 1121 return __y; 1122} 1123 1124#if defined(__CLANG_ATOMIC_BOOL_LOCK_FREE) 1125# define ATOMIC_BOOL_LOCK_FREE __CLANG_ATOMIC_BOOL_LOCK_FREE 1126# define ATOMIC_CHAR_LOCK_FREE __CLANG_ATOMIC_CHAR_LOCK_FREE 1127# define ATOMIC_CHAR16_T_LOCK_FREE __CLANG_ATOMIC_CHAR16_T_LOCK_FREE 1128# define ATOMIC_CHAR32_T_LOCK_FREE __CLANG_ATOMIC_CHAR32_T_LOCK_FREE 1129# define ATOMIC_WCHAR_T_LOCK_FREE __CLANG_ATOMIC_WCHAR_T_LOCK_FREE 1130# define ATOMIC_SHORT_LOCK_FREE __CLANG_ATOMIC_SHORT_LOCK_FREE 1131# define ATOMIC_INT_LOCK_FREE __CLANG_ATOMIC_INT_LOCK_FREE 1132# define ATOMIC_LONG_LOCK_FREE __CLANG_ATOMIC_LONG_LOCK_FREE 1133# define ATOMIC_LLONG_LOCK_FREE __CLANG_ATOMIC_LLONG_LOCK_FREE 1134# define ATOMIC_POINTER_LOCK_FREE __CLANG_ATOMIC_POINTER_LOCK_FREE 1135#elif defined(__GCC_ATOMIC_BOOL_LOCK_FREE) 1136# define ATOMIC_BOOL_LOCK_FREE __GCC_ATOMIC_BOOL_LOCK_FREE 1137# define ATOMIC_CHAR_LOCK_FREE __GCC_ATOMIC_CHAR_LOCK_FREE 1138# define ATOMIC_CHAR16_T_LOCK_FREE __GCC_ATOMIC_CHAR16_T_LOCK_FREE 1139# define ATOMIC_CHAR32_T_LOCK_FREE __GCC_ATOMIC_CHAR32_T_LOCK_FREE 1140# define ATOMIC_WCHAR_T_LOCK_FREE __GCC_ATOMIC_WCHAR_T_LOCK_FREE 1141# define ATOMIC_SHORT_LOCK_FREE __GCC_ATOMIC_SHORT_LOCK_FREE 1142# define ATOMIC_INT_LOCK_FREE __GCC_ATOMIC_INT_LOCK_FREE 1143# define ATOMIC_LONG_LOCK_FREE __GCC_ATOMIC_LONG_LOCK_FREE 1144# define ATOMIC_LLONG_LOCK_FREE __GCC_ATOMIC_LLONG_LOCK_FREE 1145# define ATOMIC_POINTER_LOCK_FREE __GCC_ATOMIC_POINTER_LOCK_FREE 1146#endif 1147 1148#ifdef _LIBCPP_ATOMIC_ONLY_USE_BUILTINS 1149 1150template<typename _Tp> 1151struct __cxx_atomic_lock_impl { 1152 1153 _LIBCPP_INLINE_VISIBILITY 1154 __cxx_atomic_lock_impl() _NOEXCEPT 1155 : __a_value(), __a_lock(0) {} 1156 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR explicit 1157 __cxx_atomic_lock_impl(_Tp value) _NOEXCEPT 1158 : __a_value(value), __a_lock(0) {} 1159 1160 _Tp __a_value; 1161 mutable __cxx_atomic_base_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_lock; 1162 1163 _LIBCPP_INLINE_VISIBILITY void __lock() const volatile { 1164 while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire)) 1165 /*spin*/; 1166 } 1167 _LIBCPP_INLINE_VISIBILITY void __lock() const { 1168 while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire)) 1169 /*spin*/; 1170 } 1171 _LIBCPP_INLINE_VISIBILITY void __unlock() const volatile { 1172 __cxx_atomic_store(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(false), memory_order_release); 1173 } 1174 _LIBCPP_INLINE_VISIBILITY void __unlock() const { 1175 __cxx_atomic_store(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(false), memory_order_release); 1176 } 1177 _LIBCPP_INLINE_VISIBILITY _Tp __read() const volatile { 1178 __lock(); 1179 _Tp __old; 1180 __cxx_atomic_assign_volatile(__old, __a_value); 1181 __unlock(); 1182 return __old; 1183 } 1184 _LIBCPP_INLINE_VISIBILITY _Tp __read() const { 1185 __lock(); 1186 _Tp __old = __a_value; 1187 __unlock(); 1188 return __old; 1189 } 1190}; 1191 1192template <typename _Tp> 1193_LIBCPP_INLINE_VISIBILITY 1194void __cxx_atomic_init(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __val) { 1195 __cxx_atomic_assign_volatile(__a->__a_value, __val); 1196} 1197template <typename _Tp> 1198_LIBCPP_INLINE_VISIBILITY 1199void __cxx_atomic_init(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __val) { 1200 __a->__a_value = __val; 1201} 1202 1203template <typename _Tp> 1204_LIBCPP_INLINE_VISIBILITY 1205void __cxx_atomic_store(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __val, memory_order) { 1206 __a->__lock(); 1207 __cxx_atomic_assign_volatile(__a->__a_value, __val); 1208 __a->__unlock(); 1209} 1210template <typename _Tp> 1211_LIBCPP_INLINE_VISIBILITY 1212void __cxx_atomic_store(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __val, memory_order) { 1213 __a->__lock(); 1214 __a->__a_value = __val; 1215 __a->__unlock(); 1216} 1217 1218template <typename _Tp> 1219_LIBCPP_INLINE_VISIBILITY 1220_Tp __cxx_atomic_load(const volatile __cxx_atomic_lock_impl<_Tp>* __a, memory_order) { 1221 return __a->__read(); 1222} 1223template <typename _Tp> 1224_LIBCPP_INLINE_VISIBILITY 1225_Tp __cxx_atomic_load(const __cxx_atomic_lock_impl<_Tp>* __a, memory_order) { 1226 return __a->__read(); 1227} 1228 1229template <typename _Tp> 1230_LIBCPP_INLINE_VISIBILITY 1231_Tp __cxx_atomic_exchange(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) { 1232 __a->__lock(); 1233 _Tp __old; 1234 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1235 __cxx_atomic_assign_volatile(__a->__a_value, __value); 1236 __a->__unlock(); 1237 return __old; 1238} 1239template <typename _Tp> 1240_LIBCPP_INLINE_VISIBILITY 1241_Tp __cxx_atomic_exchange(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) { 1242 __a->__lock(); 1243 _Tp __old = __a->__a_value; 1244 __a->__a_value = __value; 1245 __a->__unlock(); 1246 return __old; 1247} 1248 1249template <typename _Tp> 1250_LIBCPP_INLINE_VISIBILITY 1251bool __cxx_atomic_compare_exchange_strong(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1252 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1253 _Tp __temp; 1254 __a->__lock(); 1255 __cxx_atomic_assign_volatile(__temp, __a->__a_value); 1256 bool __ret = (memcmp(&__temp, __expected, sizeof(_Tp)) == 0); 1257 if(__ret) 1258 __cxx_atomic_assign_volatile(__a->__a_value, __value); 1259 else 1260 __cxx_atomic_assign_volatile(*__expected, __a->__a_value); 1261 __a->__unlock(); 1262 return __ret; 1263} 1264template <typename _Tp> 1265_LIBCPP_INLINE_VISIBILITY 1266bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_lock_impl<_Tp>* __a, 1267 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1268 __a->__lock(); 1269 bool __ret = (memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0); 1270 if(__ret) 1271 memcpy(&__a->__a_value, &__value, sizeof(_Tp)); 1272 else 1273 memcpy(__expected, &__a->__a_value, sizeof(_Tp)); 1274 __a->__unlock(); 1275 return __ret; 1276} 1277 1278template <typename _Tp> 1279_LIBCPP_INLINE_VISIBILITY 1280bool __cxx_atomic_compare_exchange_weak(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1281 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1282 _Tp __temp; 1283 __a->__lock(); 1284 __cxx_atomic_assign_volatile(__temp, __a->__a_value); 1285 bool __ret = (memcmp(&__temp, __expected, sizeof(_Tp)) == 0); 1286 if(__ret) 1287 __cxx_atomic_assign_volatile(__a->__a_value, __value); 1288 else 1289 __cxx_atomic_assign_volatile(*__expected, __a->__a_value); 1290 __a->__unlock(); 1291 return __ret; 1292} 1293template <typename _Tp> 1294_LIBCPP_INLINE_VISIBILITY 1295bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_lock_impl<_Tp>* __a, 1296 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1297 __a->__lock(); 1298 bool __ret = (memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0); 1299 if(__ret) 1300 memcpy(&__a->__a_value, &__value, sizeof(_Tp)); 1301 else 1302 memcpy(__expected, &__a->__a_value, sizeof(_Tp)); 1303 __a->__unlock(); 1304 return __ret; 1305} 1306 1307template <typename _Tp, typename _Td> 1308_LIBCPP_INLINE_VISIBILITY 1309_Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1310 _Td __delta, memory_order) { 1311 __a->__lock(); 1312 _Tp __old; 1313 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1314 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old + __delta)); 1315 __a->__unlock(); 1316 return __old; 1317} 1318template <typename _Tp, typename _Td> 1319_LIBCPP_INLINE_VISIBILITY 1320_Tp __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp>* __a, 1321 _Td __delta, memory_order) { 1322 __a->__lock(); 1323 _Tp __old = __a->__a_value; 1324 __a->__a_value += __delta; 1325 __a->__unlock(); 1326 return __old; 1327} 1328 1329template <typename _Tp, typename _Td> 1330_LIBCPP_INLINE_VISIBILITY 1331_Tp* __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp*>* __a, 1332 ptrdiff_t __delta, memory_order) { 1333 __a->__lock(); 1334 _Tp* __old; 1335 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1336 __cxx_atomic_assign_volatile(__a->__a_value, __old + __delta); 1337 __a->__unlock(); 1338 return __old; 1339} 1340template <typename _Tp, typename _Td> 1341_LIBCPP_INLINE_VISIBILITY 1342_Tp* __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp*>* __a, 1343 ptrdiff_t __delta, memory_order) { 1344 __a->__lock(); 1345 _Tp* __old = __a->__a_value; 1346 __a->__a_value += __delta; 1347 __a->__unlock(); 1348 return __old; 1349} 1350 1351template <typename _Tp, typename _Td> 1352_LIBCPP_INLINE_VISIBILITY 1353_Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1354 _Td __delta, memory_order) { 1355 __a->__lock(); 1356 _Tp __old; 1357 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1358 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old - __delta)); 1359 __a->__unlock(); 1360 return __old; 1361} 1362template <typename _Tp, typename _Td> 1363_LIBCPP_INLINE_VISIBILITY 1364_Tp __cxx_atomic_fetch_sub(__cxx_atomic_lock_impl<_Tp>* __a, 1365 _Td __delta, memory_order) { 1366 __a->__lock(); 1367 _Tp __old = __a->__a_value; 1368 __a->__a_value -= __delta; 1369 __a->__unlock(); 1370 return __old; 1371} 1372 1373template <typename _Tp> 1374_LIBCPP_INLINE_VISIBILITY 1375_Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1376 _Tp __pattern, memory_order) { 1377 __a->__lock(); 1378 _Tp __old; 1379 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1380 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old & __pattern)); 1381 __a->__unlock(); 1382 return __old; 1383} 1384template <typename _Tp> 1385_LIBCPP_INLINE_VISIBILITY 1386_Tp __cxx_atomic_fetch_and(__cxx_atomic_lock_impl<_Tp>* __a, 1387 _Tp __pattern, memory_order) { 1388 __a->__lock(); 1389 _Tp __old = __a->__a_value; 1390 __a->__a_value &= __pattern; 1391 __a->__unlock(); 1392 return __old; 1393} 1394 1395template <typename _Tp> 1396_LIBCPP_INLINE_VISIBILITY 1397_Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1398 _Tp __pattern, memory_order) { 1399 __a->__lock(); 1400 _Tp __old; 1401 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1402 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old | __pattern)); 1403 __a->__unlock(); 1404 return __old; 1405} 1406template <typename _Tp> 1407_LIBCPP_INLINE_VISIBILITY 1408_Tp __cxx_atomic_fetch_or(__cxx_atomic_lock_impl<_Tp>* __a, 1409 _Tp __pattern, memory_order) { 1410 __a->__lock(); 1411 _Tp __old = __a->__a_value; 1412 __a->__a_value |= __pattern; 1413 __a->__unlock(); 1414 return __old; 1415} 1416 1417template <typename _Tp> 1418_LIBCPP_INLINE_VISIBILITY 1419_Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1420 _Tp __pattern, memory_order) { 1421 __a->__lock(); 1422 _Tp __old; 1423 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1424 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old ^ __pattern)); 1425 __a->__unlock(); 1426 return __old; 1427} 1428template <typename _Tp> 1429_LIBCPP_INLINE_VISIBILITY 1430_Tp __cxx_atomic_fetch_xor(__cxx_atomic_lock_impl<_Tp>* __a, 1431 _Tp __pattern, memory_order) { 1432 __a->__lock(); 1433 _Tp __old = __a->__a_value; 1434 __a->__a_value ^= __pattern; 1435 __a->__unlock(); 1436 return __old; 1437} 1438 1439#ifdef __cpp_lib_atomic_is_always_lock_free 1440 1441template<typename _Tp> struct __cxx_is_always_lock_free { 1442 enum { __value = __atomic_always_lock_free(sizeof(_Tp), 0) }; }; 1443 1444#else 1445 1446template<typename _Tp> struct __cxx_is_always_lock_free { enum { __value = false }; }; 1447// Implementations must match the C ATOMIC_*_LOCK_FREE macro values. 1448template<> struct __cxx_is_always_lock_free<bool> { enum { __value = 2 == ATOMIC_BOOL_LOCK_FREE }; }; 1449template<> struct __cxx_is_always_lock_free<char> { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; }; 1450template<> struct __cxx_is_always_lock_free<signed char> { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; }; 1451template<> struct __cxx_is_always_lock_free<unsigned char> { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; }; 1452template<> struct __cxx_is_always_lock_free<char16_t> { enum { __value = 2 == ATOMIC_CHAR16_T_LOCK_FREE }; }; 1453template<> struct __cxx_is_always_lock_free<char32_t> { enum { __value = 2 == ATOMIC_CHAR32_T_LOCK_FREE }; }; 1454template<> struct __cxx_is_always_lock_free<wchar_t> { enum { __value = 2 == ATOMIC_WCHAR_T_LOCK_FREE }; }; 1455template<> struct __cxx_is_always_lock_free<short> { enum { __value = 2 == ATOMIC_SHORT_LOCK_FREE }; }; 1456template<> struct __cxx_is_always_lock_free<unsigned short> { enum { __value = 2 == ATOMIC_SHORT_LOCK_FREE }; }; 1457template<> struct __cxx_is_always_lock_free<int> { enum { __value = 2 == ATOMIC_INT_LOCK_FREE }; }; 1458template<> struct __cxx_is_always_lock_free<unsigned int> { enum { __value = 2 == ATOMIC_INT_LOCK_FREE }; }; 1459template<> struct __cxx_is_always_lock_free<long> { enum { __value = 2 == ATOMIC_LONG_LOCK_FREE }; }; 1460template<> struct __cxx_is_always_lock_free<unsigned long> { enum { __value = 2 == ATOMIC_LONG_LOCK_FREE }; }; 1461template<> struct __cxx_is_always_lock_free<long long> { enum { __value = 2 == ATOMIC_LLONG_LOCK_FREE }; }; 1462template<> struct __cxx_is_always_lock_free<unsigned long long> { enum { __value = 2 == ATOMIC_LLONG_LOCK_FREE }; }; 1463template<typename _Tp> struct __cxx_is_always_lock_free<_Tp*> { enum { __value = 2 == ATOMIC_POINTER_LOCK_FREE }; }; 1464template<> struct __cxx_is_always_lock_free<std::nullptr_t> { enum { __value = 2 == ATOMIC_POINTER_LOCK_FREE }; }; 1465 1466#endif //__cpp_lib_atomic_is_always_lock_free 1467 1468template <typename _Tp, 1469 typename _Base = typename conditional<__cxx_is_always_lock_free<_Tp>::__value, 1470 __cxx_atomic_base_impl<_Tp>, 1471 __cxx_atomic_lock_impl<_Tp> >::type> 1472#else 1473template <typename _Tp, 1474 typename _Base = __cxx_atomic_base_impl<_Tp> > 1475#endif //_LIBCPP_ATOMIC_ONLY_USE_BUILTINS 1476struct __cxx_atomic_impl : public _Base { 1477 1478#if _GNUC_VER >= 501 1479 static_assert(is_trivially_copyable<_Tp>::value, 1480 "std::atomic<Tp> requires that 'Tp' be a trivially copyable type"); 1481#endif 1482 1483 _LIBCPP_INLINE_VISIBILITY __cxx_atomic_impl() _NOEXCEPT _LIBCPP_DEFAULT 1484 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR explicit __cxx_atomic_impl(_Tp value) _NOEXCEPT 1485 : _Base(value) {} 1486}; 1487 1488#ifdef __linux__ 1489 using __cxx_contention_t = int32_t; 1490#else 1491 using __cxx_contention_t = int64_t; 1492#endif //__linux__ 1493 1494#if _LIBCPP_STD_VER >= 11 1495 1496using __cxx_atomic_contention_t = __cxx_atomic_impl<__cxx_contention_t>; 1497 1498#ifndef _LIBCPP_HAS_NO_PLATFORM_WAIT 1499 1500_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(void const volatile*); 1501_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(void const volatile*); 1502_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(void const volatile*); 1503_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(void const volatile*, __cxx_contention_t); 1504 1505_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(__cxx_atomic_contention_t const volatile*); 1506_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(__cxx_atomic_contention_t const volatile*); 1507_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile*); 1508_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(__cxx_atomic_contention_t const volatile*, __cxx_contention_t); 1509 1510template <class _Atp, class _Fn> 1511struct __libcpp_atomic_wait_backoff_impl { 1512 _Atp* __a; 1513 _Fn __test_fn; 1514 _LIBCPP_AVAILABILITY_SYNC 1515 _LIBCPP_INLINE_VISIBILITY bool operator()(chrono::nanoseconds __elapsed) const 1516 { 1517 if(__elapsed > chrono::microseconds(64)) 1518 { 1519 auto const __monitor = __libcpp_atomic_monitor(__a); 1520 if(__test_fn()) 1521 return true; 1522 __libcpp_atomic_wait(__a, __monitor); 1523 } 1524 else if(__elapsed > chrono::microseconds(4)) 1525 __libcpp_thread_yield(); 1526 else 1527 ; // poll 1528 return false; 1529 } 1530}; 1531 1532template <class _Atp, class _Fn> 1533_LIBCPP_AVAILABILITY_SYNC 1534_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp* __a, _Fn && __test_fn) 1535{ 1536 __libcpp_atomic_wait_backoff_impl<_Atp, typename decay<_Fn>::type> __backoff_fn = {__a, __test_fn}; 1537 return __libcpp_thread_poll_with_backoff(__test_fn, __backoff_fn); 1538} 1539 1540#else // _LIBCPP_HAS_NO_PLATFORM_WAIT 1541 1542template <class _Tp> 1543_LIBCPP_INLINE_VISIBILITY void __cxx_atomic_notify_all(__cxx_atomic_impl<_Tp> const volatile*) { } 1544template <class _Tp> 1545_LIBCPP_INLINE_VISIBILITY void __cxx_atomic_notify_one(__cxx_atomic_impl<_Tp> const volatile*) { } 1546template <class _Atp, class _Fn> 1547_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp*, _Fn && __test_fn) 1548{ 1549 return __libcpp_thread_poll_with_backoff(__test_fn, __libcpp_timed_backoff_policy()); 1550} 1551 1552#endif // _LIBCPP_HAS_NO_PLATFORM_WAIT 1553 1554template <class _Atp, class _Tp> 1555struct __cxx_atomic_wait_test_fn_impl { 1556 _Atp* __a; 1557 _Tp __val; 1558 memory_order __order; 1559 _LIBCPP_INLINE_VISIBILITY bool operator()() const 1560 { 1561 return !__cxx_nonatomic_compare_equal(__cxx_atomic_load(__a, __order), __val); 1562 } 1563}; 1564 1565template <class _Atp, class _Tp> 1566_LIBCPP_AVAILABILITY_SYNC 1567_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp* __a, _Tp const __val, memory_order __order) 1568{ 1569 __cxx_atomic_wait_test_fn_impl<_Atp, _Tp> __test_fn = {__a, __val, __order}; 1570 return __cxx_atomic_wait(__a, __test_fn); 1571} 1572 1573#endif //_LIBCPP_STD_VER >= 11 1574 1575// general atomic<T> 1576 1577template <class _Tp, bool = is_integral<_Tp>::value && !is_same<_Tp, bool>::value> 1578struct __atomic_base // false 1579{ 1580 mutable __cxx_atomic_impl<_Tp> __a_; 1581 1582#if defined(__cpp_lib_atomic_is_always_lock_free) 1583 static _LIBCPP_CONSTEXPR bool is_always_lock_free = __atomic_always_lock_free(sizeof(__a_), 0); 1584#endif 1585 1586 _LIBCPP_INLINE_VISIBILITY 1587 bool is_lock_free() const volatile _NOEXCEPT 1588 {return __cxx_atomic_is_lock_free(sizeof(_Tp));} 1589 _LIBCPP_INLINE_VISIBILITY 1590 bool is_lock_free() const _NOEXCEPT 1591 {return static_cast<__atomic_base const volatile*>(this)->is_lock_free();} 1592 _LIBCPP_INLINE_VISIBILITY 1593 void store(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1594 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1595 {__cxx_atomic_store(&__a_, __d, __m);} 1596 _LIBCPP_INLINE_VISIBILITY 1597 void store(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1598 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1599 {__cxx_atomic_store(&__a_, __d, __m);} 1600 _LIBCPP_INLINE_VISIBILITY 1601 _Tp load(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 1602 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 1603 {return __cxx_atomic_load(&__a_, __m);} 1604 _LIBCPP_INLINE_VISIBILITY 1605 _Tp load(memory_order __m = memory_order_seq_cst) const _NOEXCEPT 1606 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 1607 {return __cxx_atomic_load(&__a_, __m);} 1608 _LIBCPP_INLINE_VISIBILITY 1609 operator _Tp() const volatile _NOEXCEPT {return load();} 1610 _LIBCPP_INLINE_VISIBILITY 1611 operator _Tp() const _NOEXCEPT {return load();} 1612 _LIBCPP_INLINE_VISIBILITY 1613 _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1614 {return __cxx_atomic_exchange(&__a_, __d, __m);} 1615 _LIBCPP_INLINE_VISIBILITY 1616 _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1617 {return __cxx_atomic_exchange(&__a_, __d, __m);} 1618 _LIBCPP_INLINE_VISIBILITY 1619 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1620 memory_order __s, memory_order __f) volatile _NOEXCEPT 1621 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1622 {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);} 1623 _LIBCPP_INLINE_VISIBILITY 1624 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1625 memory_order __s, memory_order __f) _NOEXCEPT 1626 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1627 {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);} 1628 _LIBCPP_INLINE_VISIBILITY 1629 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1630 memory_order __s, memory_order __f) volatile _NOEXCEPT 1631 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1632 {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);} 1633 _LIBCPP_INLINE_VISIBILITY 1634 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1635 memory_order __s, memory_order __f) _NOEXCEPT 1636 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1637 {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);} 1638 _LIBCPP_INLINE_VISIBILITY 1639 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1640 memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1641 {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);} 1642 _LIBCPP_INLINE_VISIBILITY 1643 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1644 memory_order __m = memory_order_seq_cst) _NOEXCEPT 1645 {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);} 1646 _LIBCPP_INLINE_VISIBILITY 1647 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1648 memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1649 {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);} 1650 _LIBCPP_INLINE_VISIBILITY 1651 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1652 memory_order __m = memory_order_seq_cst) _NOEXCEPT 1653 {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);} 1654 1655 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 1656 {__cxx_atomic_wait(&__a_, __v, __m);} 1657 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT 1658 {__cxx_atomic_wait(&__a_, __v, __m);} 1659 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_one() volatile _NOEXCEPT 1660 {__cxx_atomic_notify_one(&__a_);} 1661 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_one() _NOEXCEPT 1662 {__cxx_atomic_notify_one(&__a_);} 1663 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_all() volatile _NOEXCEPT 1664 {__cxx_atomic_notify_all(&__a_);} 1665 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_all() _NOEXCEPT 1666 {__cxx_atomic_notify_all(&__a_);} 1667 1668 _LIBCPP_INLINE_VISIBILITY 1669 __atomic_base() _NOEXCEPT _LIBCPP_DEFAULT 1670 1671 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR 1672 __atomic_base(_Tp __d) _NOEXCEPT : __a_(__d) {} 1673 1674#ifndef _LIBCPP_CXX03_LANG 1675 __atomic_base(const __atomic_base&) = delete; 1676 __atomic_base& operator=(const __atomic_base&) = delete; 1677 __atomic_base& operator=(const __atomic_base&) volatile = delete; 1678#else 1679private: 1680 _LIBCPP_INLINE_VISIBILITY 1681 __atomic_base(const __atomic_base&); 1682 _LIBCPP_INLINE_VISIBILITY 1683 __atomic_base& operator=(const __atomic_base&); 1684 _LIBCPP_INLINE_VISIBILITY 1685 __atomic_base& operator=(const __atomic_base&) volatile; 1686#endif 1687}; 1688 1689#if defined(__cpp_lib_atomic_is_always_lock_free) 1690template <class _Tp, bool __b> 1691_LIBCPP_CONSTEXPR bool __atomic_base<_Tp, __b>::is_always_lock_free; 1692#endif 1693 1694// atomic<Integral> 1695 1696template <class _Tp> 1697struct __atomic_base<_Tp, true> 1698 : public __atomic_base<_Tp, false> 1699{ 1700 typedef __atomic_base<_Tp, false> __base; 1701 _LIBCPP_INLINE_VISIBILITY 1702 __atomic_base() _NOEXCEPT _LIBCPP_DEFAULT 1703 _LIBCPP_INLINE_VISIBILITY 1704 _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __base(__d) {} 1705 1706 _LIBCPP_INLINE_VISIBILITY 1707 _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1708 {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);} 1709 _LIBCPP_INLINE_VISIBILITY 1710 _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1711 {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);} 1712 _LIBCPP_INLINE_VISIBILITY 1713 _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1714 {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);} 1715 _LIBCPP_INLINE_VISIBILITY 1716 _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1717 {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);} 1718 _LIBCPP_INLINE_VISIBILITY 1719 _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1720 {return __cxx_atomic_fetch_and(&this->__a_, __op, __m);} 1721 _LIBCPP_INLINE_VISIBILITY 1722 _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1723 {return __cxx_atomic_fetch_and(&this->__a_, __op, __m);} 1724 _LIBCPP_INLINE_VISIBILITY 1725 _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1726 {return __cxx_atomic_fetch_or(&this->__a_, __op, __m);} 1727 _LIBCPP_INLINE_VISIBILITY 1728 _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1729 {return __cxx_atomic_fetch_or(&this->__a_, __op, __m);} 1730 _LIBCPP_INLINE_VISIBILITY 1731 _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1732 {return __cxx_atomic_fetch_xor(&this->__a_, __op, __m);} 1733 _LIBCPP_INLINE_VISIBILITY 1734 _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1735 {return __cxx_atomic_fetch_xor(&this->__a_, __op, __m);} 1736 1737 _LIBCPP_INLINE_VISIBILITY 1738 _Tp operator++(int) volatile _NOEXCEPT {return fetch_add(_Tp(1));} 1739 _LIBCPP_INLINE_VISIBILITY 1740 _Tp operator++(int) _NOEXCEPT {return fetch_add(_Tp(1));} 1741 _LIBCPP_INLINE_VISIBILITY 1742 _Tp operator--(int) volatile _NOEXCEPT {return fetch_sub(_Tp(1));} 1743 _LIBCPP_INLINE_VISIBILITY 1744 _Tp operator--(int) _NOEXCEPT {return fetch_sub(_Tp(1));} 1745 _LIBCPP_INLINE_VISIBILITY 1746 _Tp operator++() volatile _NOEXCEPT {return fetch_add(_Tp(1)) + _Tp(1);} 1747 _LIBCPP_INLINE_VISIBILITY 1748 _Tp operator++() _NOEXCEPT {return fetch_add(_Tp(1)) + _Tp(1);} 1749 _LIBCPP_INLINE_VISIBILITY 1750 _Tp operator--() volatile _NOEXCEPT {return fetch_sub(_Tp(1)) - _Tp(1);} 1751 _LIBCPP_INLINE_VISIBILITY 1752 _Tp operator--() _NOEXCEPT {return fetch_sub(_Tp(1)) - _Tp(1);} 1753 _LIBCPP_INLINE_VISIBILITY 1754 _Tp operator+=(_Tp __op) volatile _NOEXCEPT {return fetch_add(__op) + __op;} 1755 _LIBCPP_INLINE_VISIBILITY 1756 _Tp operator+=(_Tp __op) _NOEXCEPT {return fetch_add(__op) + __op;} 1757 _LIBCPP_INLINE_VISIBILITY 1758 _Tp operator-=(_Tp __op) volatile _NOEXCEPT {return fetch_sub(__op) - __op;} 1759 _LIBCPP_INLINE_VISIBILITY 1760 _Tp operator-=(_Tp __op) _NOEXCEPT {return fetch_sub(__op) - __op;} 1761 _LIBCPP_INLINE_VISIBILITY 1762 _Tp operator&=(_Tp __op) volatile _NOEXCEPT {return fetch_and(__op) & __op;} 1763 _LIBCPP_INLINE_VISIBILITY 1764 _Tp operator&=(_Tp __op) _NOEXCEPT {return fetch_and(__op) & __op;} 1765 _LIBCPP_INLINE_VISIBILITY 1766 _Tp operator|=(_Tp __op) volatile _NOEXCEPT {return fetch_or(__op) | __op;} 1767 _LIBCPP_INLINE_VISIBILITY 1768 _Tp operator|=(_Tp __op) _NOEXCEPT {return fetch_or(__op) | __op;} 1769 _LIBCPP_INLINE_VISIBILITY 1770 _Tp operator^=(_Tp __op) volatile _NOEXCEPT {return fetch_xor(__op) ^ __op;} 1771 _LIBCPP_INLINE_VISIBILITY 1772 _Tp operator^=(_Tp __op) _NOEXCEPT {return fetch_xor(__op) ^ __op;} 1773}; 1774 1775// atomic<T> 1776 1777template <class _Tp> 1778struct atomic 1779 : public __atomic_base<_Tp> 1780{ 1781 typedef __atomic_base<_Tp> __base; 1782 typedef _Tp value_type; 1783 typedef value_type difference_type; 1784 _LIBCPP_INLINE_VISIBILITY 1785 atomic() _NOEXCEPT _LIBCPP_DEFAULT 1786 _LIBCPP_INLINE_VISIBILITY 1787 _LIBCPP_CONSTEXPR atomic(_Tp __d) _NOEXCEPT : __base(__d) {} 1788 1789 _LIBCPP_INLINE_VISIBILITY 1790 _Tp operator=(_Tp __d) volatile _NOEXCEPT 1791 {__base::store(__d); return __d;} 1792 _LIBCPP_INLINE_VISIBILITY 1793 _Tp operator=(_Tp __d) _NOEXCEPT 1794 {__base::store(__d); return __d;} 1795}; 1796 1797// atomic<T*> 1798 1799template <class _Tp> 1800struct atomic<_Tp*> 1801 : public __atomic_base<_Tp*> 1802{ 1803 typedef __atomic_base<_Tp*> __base; 1804 typedef _Tp* value_type; 1805 typedef ptrdiff_t difference_type; 1806 _LIBCPP_INLINE_VISIBILITY 1807 atomic() _NOEXCEPT _LIBCPP_DEFAULT 1808 _LIBCPP_INLINE_VISIBILITY 1809 _LIBCPP_CONSTEXPR atomic(_Tp* __d) _NOEXCEPT : __base(__d) {} 1810 1811 _LIBCPP_INLINE_VISIBILITY 1812 _Tp* operator=(_Tp* __d) volatile _NOEXCEPT 1813 {__base::store(__d); return __d;} 1814 _LIBCPP_INLINE_VISIBILITY 1815 _Tp* operator=(_Tp* __d) _NOEXCEPT 1816 {__base::store(__d); return __d;} 1817 1818 _LIBCPP_INLINE_VISIBILITY 1819 _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) 1820 volatile _NOEXCEPT 1821 {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);} 1822 _LIBCPP_INLINE_VISIBILITY 1823 _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1824 {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);} 1825 _LIBCPP_INLINE_VISIBILITY 1826 _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) 1827 volatile _NOEXCEPT 1828 {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);} 1829 _LIBCPP_INLINE_VISIBILITY 1830 _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1831 {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);} 1832 1833 _LIBCPP_INLINE_VISIBILITY 1834 _Tp* operator++(int) volatile _NOEXCEPT {return fetch_add(1);} 1835 _LIBCPP_INLINE_VISIBILITY 1836 _Tp* operator++(int) _NOEXCEPT {return fetch_add(1);} 1837 _LIBCPP_INLINE_VISIBILITY 1838 _Tp* operator--(int) volatile _NOEXCEPT {return fetch_sub(1);} 1839 _LIBCPP_INLINE_VISIBILITY 1840 _Tp* operator--(int) _NOEXCEPT {return fetch_sub(1);} 1841 _LIBCPP_INLINE_VISIBILITY 1842 _Tp* operator++() volatile _NOEXCEPT {return fetch_add(1) + 1;} 1843 _LIBCPP_INLINE_VISIBILITY 1844 _Tp* operator++() _NOEXCEPT {return fetch_add(1) + 1;} 1845 _LIBCPP_INLINE_VISIBILITY 1846 _Tp* operator--() volatile _NOEXCEPT {return fetch_sub(1) - 1;} 1847 _LIBCPP_INLINE_VISIBILITY 1848 _Tp* operator--() _NOEXCEPT {return fetch_sub(1) - 1;} 1849 _LIBCPP_INLINE_VISIBILITY 1850 _Tp* operator+=(ptrdiff_t __op) volatile _NOEXCEPT {return fetch_add(__op) + __op;} 1851 _LIBCPP_INLINE_VISIBILITY 1852 _Tp* operator+=(ptrdiff_t __op) _NOEXCEPT {return fetch_add(__op) + __op;} 1853 _LIBCPP_INLINE_VISIBILITY 1854 _Tp* operator-=(ptrdiff_t __op) volatile _NOEXCEPT {return fetch_sub(__op) - __op;} 1855 _LIBCPP_INLINE_VISIBILITY 1856 _Tp* operator-=(ptrdiff_t __op) _NOEXCEPT {return fetch_sub(__op) - __op;} 1857}; 1858 1859// atomic_is_lock_free 1860 1861template <class _Tp> 1862_LIBCPP_INLINE_VISIBILITY 1863bool 1864atomic_is_lock_free(const volatile atomic<_Tp>* __o) _NOEXCEPT 1865{ 1866 return __o->is_lock_free(); 1867} 1868 1869template <class _Tp> 1870_LIBCPP_INLINE_VISIBILITY 1871bool 1872atomic_is_lock_free(const atomic<_Tp>* __o) _NOEXCEPT 1873{ 1874 return __o->is_lock_free(); 1875} 1876 1877// atomic_init 1878 1879template <class _Tp> 1880_LIBCPP_INLINE_VISIBILITY 1881void 1882atomic_init(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1883{ 1884 __cxx_atomic_init(&__o->__a_, __d); 1885} 1886 1887template <class _Tp> 1888_LIBCPP_INLINE_VISIBILITY 1889void 1890atomic_init(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1891{ 1892 __cxx_atomic_init(&__o->__a_, __d); 1893} 1894 1895// atomic_store 1896 1897template <class _Tp> 1898_LIBCPP_INLINE_VISIBILITY 1899void 1900atomic_store(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1901{ 1902 __o->store(__d); 1903} 1904 1905template <class _Tp> 1906_LIBCPP_INLINE_VISIBILITY 1907void 1908atomic_store(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1909{ 1910 __o->store(__d); 1911} 1912 1913// atomic_store_explicit 1914 1915template <class _Tp> 1916_LIBCPP_INLINE_VISIBILITY 1917void 1918atomic_store_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 1919 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1920{ 1921 __o->store(__d, __m); 1922} 1923 1924template <class _Tp> 1925_LIBCPP_INLINE_VISIBILITY 1926void 1927atomic_store_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 1928 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1929{ 1930 __o->store(__d, __m); 1931} 1932 1933// atomic_load 1934 1935template <class _Tp> 1936_LIBCPP_INLINE_VISIBILITY 1937_Tp 1938atomic_load(const volatile atomic<_Tp>* __o) _NOEXCEPT 1939{ 1940 return __o->load(); 1941} 1942 1943template <class _Tp> 1944_LIBCPP_INLINE_VISIBILITY 1945_Tp 1946atomic_load(const atomic<_Tp>* __o) _NOEXCEPT 1947{ 1948 return __o->load(); 1949} 1950 1951// atomic_load_explicit 1952 1953template <class _Tp> 1954_LIBCPP_INLINE_VISIBILITY 1955_Tp 1956atomic_load_explicit(const volatile atomic<_Tp>* __o, memory_order __m) _NOEXCEPT 1957 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 1958{ 1959 return __o->load(__m); 1960} 1961 1962template <class _Tp> 1963_LIBCPP_INLINE_VISIBILITY 1964_Tp 1965atomic_load_explicit(const atomic<_Tp>* __o, memory_order __m) _NOEXCEPT 1966 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 1967{ 1968 return __o->load(__m); 1969} 1970 1971// atomic_exchange 1972 1973template <class _Tp> 1974_LIBCPP_INLINE_VISIBILITY 1975_Tp 1976atomic_exchange(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1977{ 1978 return __o->exchange(__d); 1979} 1980 1981template <class _Tp> 1982_LIBCPP_INLINE_VISIBILITY 1983_Tp 1984atomic_exchange(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1985{ 1986 return __o->exchange(__d); 1987} 1988 1989// atomic_exchange_explicit 1990 1991template <class _Tp> 1992_LIBCPP_INLINE_VISIBILITY 1993_Tp 1994atomic_exchange_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 1995{ 1996 return __o->exchange(__d, __m); 1997} 1998 1999template <class _Tp> 2000_LIBCPP_INLINE_VISIBILITY 2001_Tp 2002atomic_exchange_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 2003{ 2004 return __o->exchange(__d, __m); 2005} 2006 2007// atomic_compare_exchange_weak 2008 2009template <class _Tp> 2010_LIBCPP_INLINE_VISIBILITY 2011bool 2012atomic_compare_exchange_weak(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2013{ 2014 return __o->compare_exchange_weak(*__e, __d); 2015} 2016 2017template <class _Tp> 2018_LIBCPP_INLINE_VISIBILITY 2019bool 2020atomic_compare_exchange_weak(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2021{ 2022 return __o->compare_exchange_weak(*__e, __d); 2023} 2024 2025// atomic_compare_exchange_strong 2026 2027template <class _Tp> 2028_LIBCPP_INLINE_VISIBILITY 2029bool 2030atomic_compare_exchange_strong(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2031{ 2032 return __o->compare_exchange_strong(*__e, __d); 2033} 2034 2035template <class _Tp> 2036_LIBCPP_INLINE_VISIBILITY 2037bool 2038atomic_compare_exchange_strong(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2039{ 2040 return __o->compare_exchange_strong(*__e, __d); 2041} 2042 2043// atomic_compare_exchange_weak_explicit 2044 2045template <class _Tp> 2046_LIBCPP_INLINE_VISIBILITY 2047bool 2048atomic_compare_exchange_weak_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, 2049 typename atomic<_Tp>::value_type __d, 2050 memory_order __s, memory_order __f) _NOEXCEPT 2051 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2052{ 2053 return __o->compare_exchange_weak(*__e, __d, __s, __f); 2054} 2055 2056template <class _Tp> 2057_LIBCPP_INLINE_VISIBILITY 2058bool 2059atomic_compare_exchange_weak_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d, 2060 memory_order __s, memory_order __f) _NOEXCEPT 2061 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2062{ 2063 return __o->compare_exchange_weak(*__e, __d, __s, __f); 2064} 2065 2066// atomic_compare_exchange_strong_explicit 2067 2068template <class _Tp> 2069_LIBCPP_INLINE_VISIBILITY 2070bool 2071atomic_compare_exchange_strong_explicit(volatile atomic<_Tp>* __o, 2072 typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d, 2073 memory_order __s, memory_order __f) _NOEXCEPT 2074 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2075{ 2076 return __o->compare_exchange_strong(*__e, __d, __s, __f); 2077} 2078 2079template <class _Tp> 2080_LIBCPP_INLINE_VISIBILITY 2081bool 2082atomic_compare_exchange_strong_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, 2083 typename atomic<_Tp>::value_type __d, 2084 memory_order __s, memory_order __f) _NOEXCEPT 2085 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2086{ 2087 return __o->compare_exchange_strong(*__e, __d, __s, __f); 2088} 2089 2090// atomic_wait 2091 2092template <class _Tp> 2093_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2094void atomic_wait(const volatile atomic<_Tp>* __o, 2095 typename atomic<_Tp>::value_type __v) _NOEXCEPT 2096{ 2097 return __o->wait(__v); 2098} 2099 2100template <class _Tp> 2101_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2102void atomic_wait(const atomic<_Tp>* __o, 2103 typename atomic<_Tp>::value_type __v) _NOEXCEPT 2104{ 2105 return __o->wait(__v); 2106} 2107 2108// atomic_wait_explicit 2109 2110template <class _Tp> 2111_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2112void atomic_wait_explicit(const volatile atomic<_Tp>* __o, 2113 typename atomic<_Tp>::value_type __v, 2114 memory_order __m) _NOEXCEPT 2115 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 2116{ 2117 return __o->wait(__v, __m); 2118} 2119 2120template <class _Tp> 2121_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2122void atomic_wait_explicit(const atomic<_Tp>* __o, 2123 typename atomic<_Tp>::value_type __v, 2124 memory_order __m) _NOEXCEPT 2125 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 2126{ 2127 return __o->wait(__v, __m); 2128} 2129 2130// atomic_notify_one 2131 2132template <class _Tp> 2133_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2134void atomic_notify_one(volatile atomic<_Tp>* __o) _NOEXCEPT 2135{ 2136 __o->notify_one(); 2137} 2138template <class _Tp> 2139_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2140void atomic_notify_one(atomic<_Tp>* __o) _NOEXCEPT 2141{ 2142 __o->notify_one(); 2143} 2144 2145// atomic_notify_one 2146 2147template <class _Tp> 2148_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2149void atomic_notify_all(volatile atomic<_Tp>* __o) _NOEXCEPT 2150{ 2151 __o->notify_all(); 2152} 2153template <class _Tp> 2154_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2155void atomic_notify_all(atomic<_Tp>* __o) _NOEXCEPT 2156{ 2157 __o->notify_all(); 2158} 2159 2160// atomic_fetch_add 2161 2162template <class _Tp> 2163_LIBCPP_INLINE_VISIBILITY 2164typename enable_if 2165< 2166 is_integral<_Tp>::value && !is_same<_Tp, bool>::value && !is_const<_Tp>::value, 2167 _Tp 2168>::type 2169atomic_fetch_add(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT 2170{ 2171 return __o->fetch_add(__op); 2172} 2173 2174template <class _Tp> 2175_LIBCPP_INLINE_VISIBILITY 2176typename enable_if 2177< 2178 is_integral<_Tp>::value && !is_same<_Tp, bool>::value && !is_const<_Tp>::value, 2179 _Tp 2180>::type 2181atomic_fetch_add(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT 2182{ 2183 return __o->fetch_add(__op); 2184} 2185 2186template <class _Tp> 2187_LIBCPP_INLINE_VISIBILITY 2188_Tp* 2189atomic_fetch_add(volatile atomic<_Tp*>* __o, typename atomic<_Tp*>::difference_type __op) _NOEXCEPT 2190{ 2191 return __o->fetch_add(__op); 2192} 2193 2194template <class _Tp> 2195_LIBCPP_INLINE_VISIBILITY 2196_Tp* 2197atomic_fetch_add(atomic<_Tp*>* __o, typename atomic<_Tp*>::difference_type __op) _NOEXCEPT 2198{ 2199 return __o->fetch_add(__op); 2200} 2201 2202// atomic_fetch_add_explicit 2203 2204template <class _Tp> 2205_LIBCPP_INLINE_VISIBILITY 2206typename enable_if 2207< 2208 is_integral<_Tp>::value && !is_same<_Tp, bool>::value && !is_const<_Tp>::value, 2209 _Tp 2210>::type 2211atomic_fetch_add_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT 2212{ 2213 return __o->fetch_add(__op, __m); 2214} 2215 2216template <class _Tp> 2217_LIBCPP_INLINE_VISIBILITY 2218typename enable_if 2219< 2220 is_integral<_Tp>::value && !is_same<_Tp, bool>::value && !is_const<_Tp>::value, 2221 _Tp 2222>::type 2223atomic_fetch_add_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT 2224{ 2225 return __o->fetch_add(__op, __m); 2226} 2227 2228template <class _Tp> 2229_LIBCPP_INLINE_VISIBILITY 2230_Tp* 2231atomic_fetch_add_explicit(volatile atomic<_Tp*>* __o, typename atomic<_Tp*>::difference_type __op, memory_order __m) _NOEXCEPT 2232{ 2233 return __o->fetch_add(__op, __m); 2234} 2235 2236template <class _Tp> 2237_LIBCPP_INLINE_VISIBILITY 2238_Tp* 2239atomic_fetch_add_explicit(atomic<_Tp*>* __o, typename atomic<_Tp*>::difference_type __op, memory_order __m) _NOEXCEPT 2240{ 2241 return __o->fetch_add(__op, __m); 2242} 2243 2244// atomic_fetch_sub 2245 2246template <class _Tp> 2247_LIBCPP_INLINE_VISIBILITY 2248typename enable_if 2249< 2250 is_integral<_Tp>::value && !is_same<_Tp, bool>::value && !is_const<_Tp>::value, 2251 _Tp 2252>::type 2253atomic_fetch_sub(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT 2254{ 2255 return __o->fetch_sub(__op); 2256} 2257 2258template <class _Tp> 2259_LIBCPP_INLINE_VISIBILITY 2260typename enable_if 2261< 2262 is_integral<_Tp>::value && !is_same<_Tp, bool>::value && !is_const<_Tp>::value, 2263 _Tp 2264>::type 2265atomic_fetch_sub(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT 2266{ 2267 return __o->fetch_sub(__op); 2268} 2269 2270template <class _Tp> 2271_LIBCPP_INLINE_VISIBILITY 2272_Tp* 2273atomic_fetch_sub(volatile atomic<_Tp*>* __o, typename atomic<_Tp*>::difference_type __op) _NOEXCEPT 2274{ 2275 return __o->fetch_sub(__op); 2276} 2277 2278template <class _Tp> 2279_LIBCPP_INLINE_VISIBILITY 2280_Tp* 2281atomic_fetch_sub(atomic<_Tp*>* __o, typename atomic<_Tp*>::difference_type __op) _NOEXCEPT 2282{ 2283 return __o->fetch_sub(__op); 2284} 2285 2286// atomic_fetch_sub_explicit 2287 2288template <class _Tp> 2289_LIBCPP_INLINE_VISIBILITY 2290typename enable_if 2291< 2292 is_integral<_Tp>::value && !is_same<_Tp, bool>::value && !is_const<_Tp>::value, 2293 _Tp 2294>::type 2295atomic_fetch_sub_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT 2296{ 2297 return __o->fetch_sub(__op, __m); 2298} 2299 2300template <class _Tp> 2301_LIBCPP_INLINE_VISIBILITY 2302typename enable_if 2303< 2304 is_integral<_Tp>::value && !is_same<_Tp, bool>::value && !is_const<_Tp>::value, 2305 _Tp 2306>::type 2307atomic_fetch_sub_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT 2308{ 2309 return __o->fetch_sub(__op, __m); 2310} 2311 2312template <class _Tp> 2313_LIBCPP_INLINE_VISIBILITY 2314_Tp* 2315atomic_fetch_sub_explicit(volatile atomic<_Tp*>* __o, typename atomic<_Tp*>::difference_type __op, memory_order __m) _NOEXCEPT 2316{ 2317 return __o->fetch_sub(__op, __m); 2318} 2319 2320template <class _Tp> 2321_LIBCPP_INLINE_VISIBILITY 2322_Tp* 2323atomic_fetch_sub_explicit(atomic<_Tp*>* __o, typename atomic<_Tp*>::difference_type __op, memory_order __m) _NOEXCEPT 2324{ 2325 return __o->fetch_sub(__op, __m); 2326} 2327 2328// atomic_fetch_and 2329 2330template <class _Tp> 2331_LIBCPP_INLINE_VISIBILITY 2332typename enable_if 2333< 2334 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2335 _Tp 2336>::type 2337atomic_fetch_and(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2338{ 2339 return __o->fetch_and(__op); 2340} 2341 2342template <class _Tp> 2343_LIBCPP_INLINE_VISIBILITY 2344typename enable_if 2345< 2346 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2347 _Tp 2348>::type 2349atomic_fetch_and(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2350{ 2351 return __o->fetch_and(__op); 2352} 2353 2354// atomic_fetch_and_explicit 2355 2356template <class _Tp> 2357_LIBCPP_INLINE_VISIBILITY 2358typename enable_if 2359< 2360 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2361 _Tp 2362>::type 2363atomic_fetch_and_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2364{ 2365 return __o->fetch_and(__op, __m); 2366} 2367 2368template <class _Tp> 2369_LIBCPP_INLINE_VISIBILITY 2370typename enable_if 2371< 2372 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2373 _Tp 2374>::type 2375atomic_fetch_and_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2376{ 2377 return __o->fetch_and(__op, __m); 2378} 2379 2380// atomic_fetch_or 2381 2382template <class _Tp> 2383_LIBCPP_INLINE_VISIBILITY 2384typename enable_if 2385< 2386 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2387 _Tp 2388>::type 2389atomic_fetch_or(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2390{ 2391 return __o->fetch_or(__op); 2392} 2393 2394template <class _Tp> 2395_LIBCPP_INLINE_VISIBILITY 2396typename enable_if 2397< 2398 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2399 _Tp 2400>::type 2401atomic_fetch_or(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2402{ 2403 return __o->fetch_or(__op); 2404} 2405 2406// atomic_fetch_or_explicit 2407 2408template <class _Tp> 2409_LIBCPP_INLINE_VISIBILITY 2410typename enable_if 2411< 2412 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2413 _Tp 2414>::type 2415atomic_fetch_or_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2416{ 2417 return __o->fetch_or(__op, __m); 2418} 2419 2420template <class _Tp> 2421_LIBCPP_INLINE_VISIBILITY 2422typename enable_if 2423< 2424 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2425 _Tp 2426>::type 2427atomic_fetch_or_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2428{ 2429 return __o->fetch_or(__op, __m); 2430} 2431 2432// atomic_fetch_xor 2433 2434template <class _Tp> 2435_LIBCPP_INLINE_VISIBILITY 2436typename enable_if 2437< 2438 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2439 _Tp 2440>::type 2441atomic_fetch_xor(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2442{ 2443 return __o->fetch_xor(__op); 2444} 2445 2446template <class _Tp> 2447_LIBCPP_INLINE_VISIBILITY 2448typename enable_if 2449< 2450 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2451 _Tp 2452>::type 2453atomic_fetch_xor(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2454{ 2455 return __o->fetch_xor(__op); 2456} 2457 2458// atomic_fetch_xor_explicit 2459 2460template <class _Tp> 2461_LIBCPP_INLINE_VISIBILITY 2462typename enable_if 2463< 2464 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2465 _Tp 2466>::type 2467atomic_fetch_xor_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2468{ 2469 return __o->fetch_xor(__op, __m); 2470} 2471 2472template <class _Tp> 2473_LIBCPP_INLINE_VISIBILITY 2474typename enable_if 2475< 2476 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2477 _Tp 2478>::type 2479atomic_fetch_xor_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2480{ 2481 return __o->fetch_xor(__op, __m); 2482} 2483 2484// flag type and operations 2485 2486typedef struct atomic_flag 2487{ 2488 __cxx_atomic_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_; 2489 2490 _LIBCPP_INLINE_VISIBILITY 2491 bool test(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 2492 {return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m);} 2493 _LIBCPP_INLINE_VISIBILITY 2494 bool test(memory_order __m = memory_order_seq_cst) const _NOEXCEPT 2495 {return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m);} 2496 2497 _LIBCPP_INLINE_VISIBILITY 2498 bool test_and_set(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 2499 {return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m);} 2500 _LIBCPP_INLINE_VISIBILITY 2501 bool test_and_set(memory_order __m = memory_order_seq_cst) _NOEXCEPT 2502 {return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m);} 2503 _LIBCPP_INLINE_VISIBILITY 2504 void clear(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 2505 {__cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m);} 2506 _LIBCPP_INLINE_VISIBILITY 2507 void clear(memory_order __m = memory_order_seq_cst) _NOEXCEPT 2508 {__cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m);} 2509 2510 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2511 void wait(bool __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 2512 {__cxx_atomic_wait(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);} 2513 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2514 void wait(bool __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT 2515 {__cxx_atomic_wait(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);} 2516 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2517 void notify_one() volatile _NOEXCEPT 2518 {__cxx_atomic_notify_one(&__a_);} 2519 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2520 void notify_one() _NOEXCEPT 2521 {__cxx_atomic_notify_one(&__a_);} 2522 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2523 void notify_all() volatile _NOEXCEPT 2524 {__cxx_atomic_notify_all(&__a_);} 2525 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2526 void notify_all() _NOEXCEPT 2527 {__cxx_atomic_notify_all(&__a_);} 2528 2529 _LIBCPP_INLINE_VISIBILITY 2530 atomic_flag() _NOEXCEPT _LIBCPP_DEFAULT 2531 2532 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR 2533 atomic_flag(bool __b) _NOEXCEPT : __a_(__b) {} // EXTENSION 2534 2535#ifndef _LIBCPP_CXX03_LANG 2536 atomic_flag(const atomic_flag&) = delete; 2537 atomic_flag& operator=(const atomic_flag&) = delete; 2538 atomic_flag& operator=(const atomic_flag&) volatile = delete; 2539#else 2540private: 2541 _LIBCPP_INLINE_VISIBILITY 2542 atomic_flag(const atomic_flag&); 2543 _LIBCPP_INLINE_VISIBILITY 2544 atomic_flag& operator=(const atomic_flag&); 2545 _LIBCPP_INLINE_VISIBILITY 2546 atomic_flag& operator=(const atomic_flag&) volatile; 2547#endif 2548} atomic_flag; 2549 2550 2551inline _LIBCPP_INLINE_VISIBILITY 2552bool 2553atomic_flag_test(const volatile atomic_flag* __o) _NOEXCEPT 2554{ 2555 return __o->test(); 2556} 2557 2558inline _LIBCPP_INLINE_VISIBILITY 2559bool 2560atomic_flag_test(const atomic_flag* __o) _NOEXCEPT 2561{ 2562 return __o->test(); 2563} 2564 2565inline _LIBCPP_INLINE_VISIBILITY 2566bool 2567atomic_flag_test_explicit(const volatile atomic_flag* __o, memory_order __m) _NOEXCEPT 2568{ 2569 return __o->test(__m); 2570} 2571 2572inline _LIBCPP_INLINE_VISIBILITY 2573bool 2574atomic_flag_test_explicit(const atomic_flag* __o, memory_order __m) _NOEXCEPT 2575{ 2576 return __o->test(__m); 2577} 2578 2579inline _LIBCPP_INLINE_VISIBILITY 2580bool 2581atomic_flag_test_and_set(volatile atomic_flag* __o) _NOEXCEPT 2582{ 2583 return __o->test_and_set(); 2584} 2585 2586inline _LIBCPP_INLINE_VISIBILITY 2587bool 2588atomic_flag_test_and_set(atomic_flag* __o) _NOEXCEPT 2589{ 2590 return __o->test_and_set(); 2591} 2592 2593inline _LIBCPP_INLINE_VISIBILITY 2594bool 2595atomic_flag_test_and_set_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT 2596{ 2597 return __o->test_and_set(__m); 2598} 2599 2600inline _LIBCPP_INLINE_VISIBILITY 2601bool 2602atomic_flag_test_and_set_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT 2603{ 2604 return __o->test_and_set(__m); 2605} 2606 2607inline _LIBCPP_INLINE_VISIBILITY 2608void 2609atomic_flag_clear(volatile atomic_flag* __o) _NOEXCEPT 2610{ 2611 __o->clear(); 2612} 2613 2614inline _LIBCPP_INLINE_VISIBILITY 2615void 2616atomic_flag_clear(atomic_flag* __o) _NOEXCEPT 2617{ 2618 __o->clear(); 2619} 2620 2621inline _LIBCPP_INLINE_VISIBILITY 2622void 2623atomic_flag_clear_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT 2624{ 2625 __o->clear(__m); 2626} 2627 2628inline _LIBCPP_INLINE_VISIBILITY 2629void 2630atomic_flag_clear_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT 2631{ 2632 __o->clear(__m); 2633} 2634 2635inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2636void 2637atomic_flag_wait(const volatile atomic_flag* __o, bool __v) _NOEXCEPT 2638{ 2639 __o->wait(__v); 2640} 2641 2642inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2643void 2644atomic_flag_wait(const atomic_flag* __o, bool __v) _NOEXCEPT 2645{ 2646 __o->wait(__v); 2647} 2648 2649inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2650void 2651atomic_flag_wait_explicit(const volatile atomic_flag* __o, 2652 bool __v, memory_order __m) _NOEXCEPT 2653{ 2654 __o->wait(__v, __m); 2655} 2656 2657inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2658void 2659atomic_flag_wait_explicit(const atomic_flag* __o, 2660 bool __v, memory_order __m) _NOEXCEPT 2661{ 2662 __o->wait(__v, __m); 2663} 2664 2665inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2666void 2667atomic_flag_notify_one(volatile atomic_flag* __o) _NOEXCEPT 2668{ 2669 __o->notify_one(); 2670} 2671 2672inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2673void 2674atomic_flag_notify_one(atomic_flag* __o) _NOEXCEPT 2675{ 2676 __o->notify_one(); 2677} 2678 2679inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2680void 2681atomic_flag_notify_all(volatile atomic_flag* __o) _NOEXCEPT 2682{ 2683 __o->notify_all(); 2684} 2685 2686inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2687void 2688atomic_flag_notify_all(atomic_flag* __o) _NOEXCEPT 2689{ 2690 __o->notify_all(); 2691} 2692 2693// fences 2694 2695inline _LIBCPP_INLINE_VISIBILITY 2696void 2697atomic_thread_fence(memory_order __m) _NOEXCEPT 2698{ 2699 __cxx_atomic_thread_fence(__m); 2700} 2701 2702inline _LIBCPP_INLINE_VISIBILITY 2703void 2704atomic_signal_fence(memory_order __m) _NOEXCEPT 2705{ 2706 __cxx_atomic_signal_fence(__m); 2707} 2708 2709// Atomics for standard typedef types 2710 2711typedef atomic<bool> atomic_bool; 2712typedef atomic<char> atomic_char; 2713typedef atomic<signed char> atomic_schar; 2714typedef atomic<unsigned char> atomic_uchar; 2715typedef atomic<short> atomic_short; 2716typedef atomic<unsigned short> atomic_ushort; 2717typedef atomic<int> atomic_int; 2718typedef atomic<unsigned int> atomic_uint; 2719typedef atomic<long> atomic_long; 2720typedef atomic<unsigned long> atomic_ulong; 2721typedef atomic<long long> atomic_llong; 2722typedef atomic<unsigned long long> atomic_ullong; 2723typedef atomic<char16_t> atomic_char16_t; 2724typedef atomic<char32_t> atomic_char32_t; 2725typedef atomic<wchar_t> atomic_wchar_t; 2726 2727typedef atomic<int_least8_t> atomic_int_least8_t; 2728typedef atomic<uint_least8_t> atomic_uint_least8_t; 2729typedef atomic<int_least16_t> atomic_int_least16_t; 2730typedef atomic<uint_least16_t> atomic_uint_least16_t; 2731typedef atomic<int_least32_t> atomic_int_least32_t; 2732typedef atomic<uint_least32_t> atomic_uint_least32_t; 2733typedef atomic<int_least64_t> atomic_int_least64_t; 2734typedef atomic<uint_least64_t> atomic_uint_least64_t; 2735 2736typedef atomic<int_fast8_t> atomic_int_fast8_t; 2737typedef atomic<uint_fast8_t> atomic_uint_fast8_t; 2738typedef atomic<int_fast16_t> atomic_int_fast16_t; 2739typedef atomic<uint_fast16_t> atomic_uint_fast16_t; 2740typedef atomic<int_fast32_t> atomic_int_fast32_t; 2741typedef atomic<uint_fast32_t> atomic_uint_fast32_t; 2742typedef atomic<int_fast64_t> atomic_int_fast64_t; 2743typedef atomic<uint_fast64_t> atomic_uint_fast64_t; 2744 2745typedef atomic< int8_t> atomic_int8_t; 2746typedef atomic<uint8_t> atomic_uint8_t; 2747typedef atomic< int16_t> atomic_int16_t; 2748typedef atomic<uint16_t> atomic_uint16_t; 2749typedef atomic< int32_t> atomic_int32_t; 2750typedef atomic<uint32_t> atomic_uint32_t; 2751typedef atomic< int64_t> atomic_int64_t; 2752typedef atomic<uint64_t> atomic_uint64_t; 2753 2754typedef atomic<intptr_t> atomic_intptr_t; 2755typedef atomic<uintptr_t> atomic_uintptr_t; 2756typedef atomic<size_t> atomic_size_t; 2757typedef atomic<ptrdiff_t> atomic_ptrdiff_t; 2758typedef atomic<intmax_t> atomic_intmax_t; 2759typedef atomic<uintmax_t> atomic_uintmax_t; 2760 2761// atomic_*_lock_free : prefer the contention type most highly, then the largest lock-free type 2762 2763#ifdef __cpp_lib_atomic_is_always_lock_free 2764# define _LIBCPP_CONTENTION_LOCK_FREE __atomic_always_lock_free(sizeof(__cxx_contention_t), 0) 2765#else 2766# define _LIBCPP_CONTENTION_LOCK_FREE false 2767#endif 2768 2769#if ATOMIC_LLONG_LOCK_FREE == 2 2770typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, long long>::type __libcpp_signed_lock_free; 2771typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned long long>::type __libcpp_unsigned_lock_free; 2772#elif ATOMIC_INT_LOCK_FREE == 2 2773typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, int>::type __libcpp_signed_lock_free; 2774typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned int>::type __libcpp_unsigned_lock_free; 2775#elif ATOMIC_SHORT_LOCK_FREE == 2 2776typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, short>::type __libcpp_signed_lock_free; 2777typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned short>::type __libcpp_unsigned_lock_free; 2778#elif ATOMIC_CHAR_LOCK_FREE == 2 2779typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, char>::type __libcpp_signed_lock_free; 2780typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned char>::type __libcpp_unsigned_lock_free; 2781#else 2782 // No signed/unsigned lock-free types 2783#endif 2784 2785typedef atomic<__libcpp_signed_lock_free> atomic_signed_lock_free; 2786typedef atomic<__libcpp_unsigned_lock_free> atomic_unsigned_lock_free; 2787 2788#define ATOMIC_FLAG_INIT {false} 2789#define ATOMIC_VAR_INIT(__v) {__v} 2790 2791_LIBCPP_END_NAMESPACE_STD 2792 2793#endif // _LIBCPP_ATOMIC 2794