1// -*- C++ -*- 2//===--------------------------- atomic -----------------------------------===// 3// 4// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 5// See https://llvm.org/LICENSE.txt for license information. 6// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 7// 8//===----------------------------------------------------------------------===// 9 10#ifndef _LIBCPP_ATOMIC 11#define _LIBCPP_ATOMIC 12 13/* 14 atomic synopsis 15 16namespace std 17{ 18 19// feature test macro [version.syn] 20 21#define __cpp_lib_atomic_is_always_lock_free 22#define __cpp_lib_atomic_flag_test 23#define __cpp_lib_atomic_lock_free_type_aliases 24#define __cpp_lib_atomic_wait 25 26 // order and consistency 27 28 enum memory_order: unspecified // enum class in C++20 29 { 30 relaxed, 31 consume, // load-consume 32 acquire, // load-acquire 33 release, // store-release 34 acq_rel, // store-release load-acquire 35 seq_cst // store-release load-acquire 36 }; 37 38 inline constexpr auto memory_order_relaxed = memory_order::relaxed; 39 inline constexpr auto memory_order_consume = memory_order::consume; 40 inline constexpr auto memory_order_acquire = memory_order::acquire; 41 inline constexpr auto memory_order_release = memory_order::release; 42 inline constexpr auto memory_order_acq_rel = memory_order::acq_rel; 43 inline constexpr auto memory_order_seq_cst = memory_order::seq_cst; 44 45template <class T> T kill_dependency(T y) noexcept; 46 47// lock-free property 48 49#define ATOMIC_BOOL_LOCK_FREE unspecified 50#define ATOMIC_CHAR_LOCK_FREE unspecified 51#define ATOMIC_CHAR16_T_LOCK_FREE unspecified 52#define ATOMIC_CHAR32_T_LOCK_FREE unspecified 53#define ATOMIC_WCHAR_T_LOCK_FREE unspecified 54#define ATOMIC_SHORT_LOCK_FREE unspecified 55#define ATOMIC_INT_LOCK_FREE unspecified 56#define ATOMIC_LONG_LOCK_FREE unspecified 57#define ATOMIC_LLONG_LOCK_FREE unspecified 58#define ATOMIC_POINTER_LOCK_FREE unspecified 59 60template <class T> 61struct atomic 62{ 63 using value_type = T; 64 65 static constexpr bool is_always_lock_free; 66 bool is_lock_free() const volatile noexcept; 67 bool is_lock_free() const noexcept; 68 69 atomic() noexcept = default; 70 constexpr atomic(T desr) noexcept; 71 atomic(const atomic&) = delete; 72 atomic& operator=(const atomic&) = delete; 73 atomic& operator=(const atomic&) volatile = delete; 74 75 T load(memory_order m = memory_order_seq_cst) const volatile noexcept; 76 T load(memory_order m = memory_order_seq_cst) const noexcept; 77 operator T() const volatile noexcept; 78 operator T() const noexcept; 79 void store(T desr, memory_order m = memory_order_seq_cst) volatile noexcept; 80 void store(T desr, memory_order m = memory_order_seq_cst) noexcept; 81 T operator=(T) volatile noexcept; 82 T operator=(T) noexcept; 83 84 T exchange(T desr, memory_order m = memory_order_seq_cst) volatile noexcept; 85 T exchange(T desr, memory_order m = memory_order_seq_cst) noexcept; 86 bool compare_exchange_weak(T& expc, T desr, 87 memory_order s, memory_order f) volatile noexcept; 88 bool compare_exchange_weak(T& expc, T desr, memory_order s, memory_order f) noexcept; 89 bool compare_exchange_strong(T& expc, T desr, 90 memory_order s, memory_order f) volatile noexcept; 91 bool compare_exchange_strong(T& expc, T desr, 92 memory_order s, memory_order f) noexcept; 93 bool compare_exchange_weak(T& expc, T desr, 94 memory_order m = memory_order_seq_cst) volatile noexcept; 95 bool compare_exchange_weak(T& expc, T desr, 96 memory_order m = memory_order_seq_cst) noexcept; 97 bool compare_exchange_strong(T& expc, T desr, 98 memory_order m = memory_order_seq_cst) volatile noexcept; 99 bool compare_exchange_strong(T& expc, T desr, 100 memory_order m = memory_order_seq_cst) noexcept; 101 102 void wait(T, memory_order = memory_order::seq_cst) const volatile noexcept; 103 void wait(T, memory_order = memory_order::seq_cst) const noexcept; 104 void notify_one() volatile noexcept; 105 void notify_one() noexcept; 106 void notify_all() volatile noexcept; 107 void notify_all() noexcept; 108}; 109 110template <> 111struct atomic<integral> 112{ 113 using value_type = integral; 114 using difference_type = value_type; 115 116 static constexpr bool is_always_lock_free; 117 bool is_lock_free() const volatile noexcept; 118 bool is_lock_free() const noexcept; 119 120 atomic() noexcept = default; 121 constexpr atomic(integral desr) noexcept; 122 atomic(const atomic&) = delete; 123 atomic& operator=(const atomic&) = delete; 124 atomic& operator=(const atomic&) volatile = delete; 125 126 integral load(memory_order m = memory_order_seq_cst) const volatile noexcept; 127 integral load(memory_order m = memory_order_seq_cst) const noexcept; 128 operator integral() const volatile noexcept; 129 operator integral() const noexcept; 130 void store(integral desr, memory_order m = memory_order_seq_cst) volatile noexcept; 131 void store(integral desr, memory_order m = memory_order_seq_cst) noexcept; 132 integral operator=(integral desr) volatile noexcept; 133 integral operator=(integral desr) noexcept; 134 135 integral exchange(integral desr, 136 memory_order m = memory_order_seq_cst) volatile noexcept; 137 integral exchange(integral desr, memory_order m = memory_order_seq_cst) noexcept; 138 bool compare_exchange_weak(integral& expc, integral desr, 139 memory_order s, memory_order f) volatile noexcept; 140 bool compare_exchange_weak(integral& expc, integral desr, 141 memory_order s, memory_order f) noexcept; 142 bool compare_exchange_strong(integral& expc, integral desr, 143 memory_order s, memory_order f) volatile noexcept; 144 bool compare_exchange_strong(integral& expc, integral desr, 145 memory_order s, memory_order f) noexcept; 146 bool compare_exchange_weak(integral& expc, integral desr, 147 memory_order m = memory_order_seq_cst) volatile noexcept; 148 bool compare_exchange_weak(integral& expc, integral desr, 149 memory_order m = memory_order_seq_cst) noexcept; 150 bool compare_exchange_strong(integral& expc, integral desr, 151 memory_order m = memory_order_seq_cst) volatile noexcept; 152 bool compare_exchange_strong(integral& expc, integral desr, 153 memory_order m = memory_order_seq_cst) noexcept; 154 155 integral fetch_add(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 156 integral fetch_add(integral op, memory_order m = memory_order_seq_cst) noexcept; 157 integral fetch_sub(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 158 integral fetch_sub(integral op, memory_order m = memory_order_seq_cst) noexcept; 159 integral fetch_and(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 160 integral fetch_and(integral op, memory_order m = memory_order_seq_cst) noexcept; 161 integral fetch_or(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 162 integral fetch_or(integral op, memory_order m = memory_order_seq_cst) noexcept; 163 integral fetch_xor(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 164 integral fetch_xor(integral op, memory_order m = memory_order_seq_cst) noexcept; 165 166 integral operator++(int) volatile noexcept; 167 integral operator++(int) noexcept; 168 integral operator--(int) volatile noexcept; 169 integral operator--(int) noexcept; 170 integral operator++() volatile noexcept; 171 integral operator++() noexcept; 172 integral operator--() volatile noexcept; 173 integral operator--() noexcept; 174 integral operator+=(integral op) volatile noexcept; 175 integral operator+=(integral op) noexcept; 176 integral operator-=(integral op) volatile noexcept; 177 integral operator-=(integral op) noexcept; 178 integral operator&=(integral op) volatile noexcept; 179 integral operator&=(integral op) noexcept; 180 integral operator|=(integral op) volatile noexcept; 181 integral operator|=(integral op) noexcept; 182 integral operator^=(integral op) volatile noexcept; 183 integral operator^=(integral op) noexcept; 184 185 void wait(integral, memory_order = memory_order::seq_cst) const volatile noexcept; 186 void wait(integral, memory_order = memory_order::seq_cst) const noexcept; 187 void notify_one() volatile noexcept; 188 void notify_one() noexcept; 189 void notify_all() volatile noexcept; 190 void notify_all() noexcept; 191}; 192 193template <class T> 194struct atomic<T*> 195{ 196 using value_type = T*; 197 using difference_type = ptrdiff_t; 198 199 static constexpr bool is_always_lock_free; 200 bool is_lock_free() const volatile noexcept; 201 bool is_lock_free() const noexcept; 202 203 atomic() noexcept = default; 204 constexpr atomic(T* desr) noexcept; 205 atomic(const atomic&) = delete; 206 atomic& operator=(const atomic&) = delete; 207 atomic& operator=(const atomic&) volatile = delete; 208 209 T* load(memory_order m = memory_order_seq_cst) const volatile noexcept; 210 T* load(memory_order m = memory_order_seq_cst) const noexcept; 211 operator T*() const volatile noexcept; 212 operator T*() const noexcept; 213 void store(T* desr, memory_order m = memory_order_seq_cst) volatile noexcept; 214 void store(T* desr, memory_order m = memory_order_seq_cst) noexcept; 215 T* operator=(T*) volatile noexcept; 216 T* operator=(T*) noexcept; 217 218 T* exchange(T* desr, memory_order m = memory_order_seq_cst) volatile noexcept; 219 T* exchange(T* desr, memory_order m = memory_order_seq_cst) noexcept; 220 bool compare_exchange_weak(T*& expc, T* desr, 221 memory_order s, memory_order f) volatile noexcept; 222 bool compare_exchange_weak(T*& expc, T* desr, 223 memory_order s, memory_order f) noexcept; 224 bool compare_exchange_strong(T*& expc, T* desr, 225 memory_order s, memory_order f) volatile noexcept; 226 bool compare_exchange_strong(T*& expc, T* desr, 227 memory_order s, memory_order f) noexcept; 228 bool compare_exchange_weak(T*& expc, T* desr, 229 memory_order m = memory_order_seq_cst) volatile noexcept; 230 bool compare_exchange_weak(T*& expc, T* desr, 231 memory_order m = memory_order_seq_cst) noexcept; 232 bool compare_exchange_strong(T*& expc, T* desr, 233 memory_order m = memory_order_seq_cst) volatile noexcept; 234 bool compare_exchange_strong(T*& expc, T* desr, 235 memory_order m = memory_order_seq_cst) noexcept; 236 T* fetch_add(ptrdiff_t op, memory_order m = memory_order_seq_cst) volatile noexcept; 237 T* fetch_add(ptrdiff_t op, memory_order m = memory_order_seq_cst) noexcept; 238 T* fetch_sub(ptrdiff_t op, memory_order m = memory_order_seq_cst) volatile noexcept; 239 T* fetch_sub(ptrdiff_t op, memory_order m = memory_order_seq_cst) noexcept; 240 241 T* operator++(int) volatile noexcept; 242 T* operator++(int) noexcept; 243 T* operator--(int) volatile noexcept; 244 T* operator--(int) noexcept; 245 T* operator++() volatile noexcept; 246 T* operator++() noexcept; 247 T* operator--() volatile noexcept; 248 T* operator--() noexcept; 249 T* operator+=(ptrdiff_t op) volatile noexcept; 250 T* operator+=(ptrdiff_t op) noexcept; 251 T* operator-=(ptrdiff_t op) volatile noexcept; 252 T* operator-=(ptrdiff_t op) noexcept; 253 254 void wait(T*, memory_order = memory_order::seq_cst) const volatile noexcept; 255 void wait(T*, memory_order = memory_order::seq_cst) const noexcept; 256 void notify_one() volatile noexcept; 257 void notify_one() noexcept; 258 void notify_all() volatile noexcept; 259 void notify_all() noexcept; 260}; 261 262 263template <class T> 264 bool atomic_is_lock_free(const volatile atomic<T>* obj) noexcept; 265 266template <class T> 267 bool atomic_is_lock_free(const atomic<T>* obj) noexcept; 268 269template <class T> 270 void atomic_store(volatile atomic<T>* obj, T desr) noexcept; 271 272template <class T> 273 void atomic_store(atomic<T>* obj, T desr) noexcept; 274 275template <class T> 276 void atomic_store_explicit(volatile atomic<T>* obj, T desr, memory_order m) noexcept; 277 278template <class T> 279 void atomic_store_explicit(atomic<T>* obj, T desr, memory_order m) noexcept; 280 281template <class T> 282 T atomic_load(const volatile atomic<T>* obj) noexcept; 283 284template <class T> 285 T atomic_load(const atomic<T>* obj) noexcept; 286 287template <class T> 288 T atomic_load_explicit(const volatile atomic<T>* obj, memory_order m) noexcept; 289 290template <class T> 291 T atomic_load_explicit(const atomic<T>* obj, memory_order m) noexcept; 292 293template <class T> 294 T atomic_exchange(volatile atomic<T>* obj, T desr) noexcept; 295 296template <class T> 297 T atomic_exchange(atomic<T>* obj, T desr) noexcept; 298 299template <class T> 300 T atomic_exchange_explicit(volatile atomic<T>* obj, T desr, memory_order m) noexcept; 301 302template <class T> 303 T atomic_exchange_explicit(atomic<T>* obj, T desr, memory_order m) noexcept; 304 305template <class T> 306 bool atomic_compare_exchange_weak(volatile atomic<T>* obj, T* expc, T desr) noexcept; 307 308template <class T> 309 bool atomic_compare_exchange_weak(atomic<T>* obj, T* expc, T desr) noexcept; 310 311template <class T> 312 bool atomic_compare_exchange_strong(volatile atomic<T>* obj, T* expc, T desr) noexcept; 313 314template <class T> 315 bool atomic_compare_exchange_strong(atomic<T>* obj, T* expc, T desr) noexcept; 316 317template <class T> 318 bool atomic_compare_exchange_weak_explicit(volatile atomic<T>* obj, T* expc, 319 T desr, 320 memory_order s, memory_order f) noexcept; 321 322template <class T> 323 bool atomic_compare_exchange_weak_explicit(atomic<T>* obj, T* expc, T desr, 324 memory_order s, memory_order f) noexcept; 325 326template <class T> 327 bool atomic_compare_exchange_strong_explicit(volatile atomic<T>* obj, 328 T* expc, T desr, 329 memory_order s, memory_order f) noexcept; 330 331template <class T> 332 bool atomic_compare_exchange_strong_explicit(atomic<T>* obj, T* expc, 333 T desr, 334 memory_order s, memory_order f) noexcept; 335 336template <class T> 337 void atomic_wait(const volatile atomic<T>* obj, T old) noexcept; 338 339template <class T> 340 void atomic_wait(const atomic<T>* obj, T old) noexcept; 341 342template <class T> 343 void atomic_wait_explicit(const volatile atomic<T>* obj, T old, memory_order m) noexcept; 344 345template <class T> 346 void atomic_wait_explicit(const atomic<T>* obj, T old, memory_order m) noexcept; 347 348template <class T> 349 void atomic_one(volatile atomic<T>* obj) noexcept; 350 351template <class T> 352 void atomic_one(atomic<T>* obj) noexcept; 353 354template <class T> 355 void atomic_all(volatile atomic<T>* obj) noexcept; 356 357template <class T> 358 void atomic_all(atomic<T>* obj) noexcept; 359 360template <class Integral> 361 Integral atomic_fetch_add(volatile atomic<Integral>* obj, Integral op) noexcept; 362 363template <class Integral> 364 Integral atomic_fetch_add(atomic<Integral>* obj, Integral op) noexcept; 365 366template <class Integral> 367 Integral atomic_fetch_add_explicit(volatile atomic<Integral>* obj, Integral op, 368 memory_order m) noexcept; 369template <class Integral> 370 Integral atomic_fetch_add_explicit(atomic<Integral>* obj, Integral op, 371 memory_order m) noexcept; 372template <class Integral> 373 Integral atomic_fetch_sub(volatile atomic<Integral>* obj, Integral op) noexcept; 374 375template <class Integral> 376 Integral atomic_fetch_sub(atomic<Integral>* obj, Integral op) noexcept; 377 378template <class Integral> 379 Integral atomic_fetch_sub_explicit(volatile atomic<Integral>* obj, Integral op, 380 memory_order m) noexcept; 381 382template <class Integral> 383 Integral atomic_fetch_sub_explicit(atomic<Integral>* obj, Integral op, 384 memory_order m) noexcept; 385 386template <class Integral> 387 Integral atomic_fetch_and(volatile atomic<Integral>* obj, Integral op) noexcept; 388 389template <class Integral> 390 Integral atomic_fetch_and(atomic<Integral>* obj, Integral op) noexcept; 391 392template <class Integral> 393 Integral atomic_fetch_and_explicit(volatile atomic<Integral>* obj, Integral op, 394 memory_order m) noexcept; 395 396template <class Integral> 397 Integral atomic_fetch_and_explicit(atomic<Integral>* obj, Integral op, 398 memory_order m) noexcept; 399 400template <class Integral> 401 Integral atomic_fetch_or(volatile atomic<Integral>* obj, Integral op) noexcept; 402 403template <class Integral> 404 Integral atomic_fetch_or(atomic<Integral>* obj, Integral op) noexcept; 405 406template <class Integral> 407 Integral atomic_fetch_or_explicit(volatile atomic<Integral>* obj, Integral op, 408 memory_order m) noexcept; 409 410template <class Integral> 411 Integral atomic_fetch_or_explicit(atomic<Integral>* obj, Integral op, 412 memory_order m) noexcept; 413 414template <class Integral> 415 Integral atomic_fetch_xor(volatile atomic<Integral>* obj, Integral op) noexcept; 416 417template <class Integral> 418 Integral atomic_fetch_xor(atomic<Integral>* obj, Integral op) noexcept; 419 420template <class Integral> 421 Integral atomic_fetch_xor_explicit(volatile atomic<Integral>* obj, Integral op, 422 memory_order m) noexcept; 423 424template <class Integral> 425 Integral atomic_fetch_xor_explicit(atomic<Integral>* obj, Integral op, 426 memory_order m) noexcept; 427 428template <class T> 429 T* atomic_fetch_add(volatile atomic<T*>* obj, ptrdiff_t op) noexcept; 430 431template <class T> 432 T* atomic_fetch_add(atomic<T*>* obj, ptrdiff_t op) noexcept; 433 434template <class T> 435 T* atomic_fetch_add_explicit(volatile atomic<T*>* obj, ptrdiff_t op, 436 memory_order m) noexcept; 437 438template <class T> 439 T* atomic_fetch_add_explicit(atomic<T*>* obj, ptrdiff_t op, memory_order m) noexcept; 440 441template <class T> 442 T* atomic_fetch_sub(volatile atomic<T*>* obj, ptrdiff_t op) noexcept; 443 444template <class T> 445 T* atomic_fetch_sub(atomic<T*>* obj, ptrdiff_t op) noexcept; 446 447template <class T> 448 T* atomic_fetch_sub_explicit(volatile atomic<T*>* obj, ptrdiff_t op, 449 memory_order m) noexcept; 450 451template <class T> 452 T* atomic_fetch_sub_explicit(atomic<T*>* obj, ptrdiff_t op, memory_order m) noexcept; 453 454// Atomics for standard typedef types 455 456typedef atomic<bool> atomic_bool; 457typedef atomic<char> atomic_char; 458typedef atomic<signed char> atomic_schar; 459typedef atomic<unsigned char> atomic_uchar; 460typedef atomic<short> atomic_short; 461typedef atomic<unsigned short> atomic_ushort; 462typedef atomic<int> atomic_int; 463typedef atomic<unsigned int> atomic_uint; 464typedef atomic<long> atomic_long; 465typedef atomic<unsigned long> atomic_ulong; 466typedef atomic<long long> atomic_llong; 467typedef atomic<unsigned long long> atomic_ullong; 468typedef atomic<char16_t> atomic_char16_t; 469typedef atomic<char32_t> atomic_char32_t; 470typedef atomic<wchar_t> atomic_wchar_t; 471 472typedef atomic<int_least8_t> atomic_int_least8_t; 473typedef atomic<uint_least8_t> atomic_uint_least8_t; 474typedef atomic<int_least16_t> atomic_int_least16_t; 475typedef atomic<uint_least16_t> atomic_uint_least16_t; 476typedef atomic<int_least32_t> atomic_int_least32_t; 477typedef atomic<uint_least32_t> atomic_uint_least32_t; 478typedef atomic<int_least64_t> atomic_int_least64_t; 479typedef atomic<uint_least64_t> atomic_uint_least64_t; 480 481typedef atomic<int_fast8_t> atomic_int_fast8_t; 482typedef atomic<uint_fast8_t> atomic_uint_fast8_t; 483typedef atomic<int_fast16_t> atomic_int_fast16_t; 484typedef atomic<uint_fast16_t> atomic_uint_fast16_t; 485typedef atomic<int_fast32_t> atomic_int_fast32_t; 486typedef atomic<uint_fast32_t> atomic_uint_fast32_t; 487typedef atomic<int_fast64_t> atomic_int_fast64_t; 488typedef atomic<uint_fast64_t> atomic_uint_fast64_t; 489 490typedef atomic<int8_t> atomic_int8_t; 491typedef atomic<uint8_t> atomic_uint8_t; 492typedef atomic<int16_t> atomic_int16_t; 493typedef atomic<uint16_t> atomic_uint16_t; 494typedef atomic<int32_t> atomic_int32_t; 495typedef atomic<uint32_t> atomic_uint32_t; 496typedef atomic<int64_t> atomic_int64_t; 497typedef atomic<uint64_t> atomic_uint64_t; 498 499typedef atomic<intptr_t> atomic_intptr_t; 500typedef atomic<uintptr_t> atomic_uintptr_t; 501typedef atomic<size_t> atomic_size_t; 502typedef atomic<ptrdiff_t> atomic_ptrdiff_t; 503typedef atomic<intmax_t> atomic_intmax_t; 504typedef atomic<uintmax_t> atomic_uintmax_t; 505 506// flag type and operations 507 508typedef struct atomic_flag 509{ 510 atomic_flag() noexcept = default; 511 atomic_flag(const atomic_flag&) = delete; 512 atomic_flag& operator=(const atomic_flag&) = delete; 513 atomic_flag& operator=(const atomic_flag&) volatile = delete; 514 515 bool test(memory_order m = memory_order_seq_cst) volatile noexcept; 516 bool test(memory_order m = memory_order_seq_cst) noexcept; 517 bool test_and_set(memory_order m = memory_order_seq_cst) volatile noexcept; 518 bool test_and_set(memory_order m = memory_order_seq_cst) noexcept; 519 void clear(memory_order m = memory_order_seq_cst) volatile noexcept; 520 void clear(memory_order m = memory_order_seq_cst) noexcept; 521 522 void wait(bool, memory_order = memory_order::seq_cst) const volatile noexcept; 523 void wait(bool, memory_order = memory_order::seq_cst) const noexcept; 524 void notify_one() volatile noexcept; 525 void notify_one() noexcept; 526 void notify_all() volatile noexcept; 527 void notify_all() noexcept; 528} atomic_flag; 529 530bool atomic_flag_test(volatile atomic_flag* obj) noexcept; 531bool atomic_flag_test(atomic_flag* obj) noexcept; 532bool atomic_flag_test_explicit(volatile atomic_flag* obj, 533 memory_order m) noexcept; 534bool atomic_flag_test_explicit(atomic_flag* obj, memory_order m) noexcept; 535bool atomic_flag_test_and_set(volatile atomic_flag* obj) noexcept; 536bool atomic_flag_test_and_set(atomic_flag* obj) noexcept; 537bool atomic_flag_test_and_set_explicit(volatile atomic_flag* obj, 538 memory_order m) noexcept; 539bool atomic_flag_test_and_set_explicit(atomic_flag* obj, memory_order m) noexcept; 540void atomic_flag_clear(volatile atomic_flag* obj) noexcept; 541void atomic_flag_clear(atomic_flag* obj) noexcept; 542void atomic_flag_clear_explicit(volatile atomic_flag* obj, memory_order m) noexcept; 543void atomic_flag_clear_explicit(atomic_flag* obj, memory_order m) noexcept; 544 545void atomic_wait(const volatile atomic_flag* obj, T old) noexcept; 546void atomic_wait(const atomic_flag* obj, T old) noexcept; 547void atomic_wait_explicit(const volatile atomic_flag* obj, T old, memory_order m) noexcept; 548void atomic_wait_explicit(const atomic_flag* obj, T old, memory_order m) noexcept; 549void atomic_one(volatile atomic_flag* obj) noexcept; 550void atomic_one(atomic_flag* obj) noexcept; 551void atomic_all(volatile atomic_flag* obj) noexcept; 552void atomic_all(atomic_flag* obj) noexcept; 553 554// fences 555 556void atomic_thread_fence(memory_order m) noexcept; 557void atomic_signal_fence(memory_order m) noexcept; 558 559// deprecated 560 561template <class T> 562 void atomic_init(volatile atomic<T>* obj, typename atomic<T>::value_type desr) noexcept; 563 564template <class T> 565 void atomic_init(atomic<T>* obj, typename atomic<T>::value_type desr) noexcept; 566 567#define ATOMIC_VAR_INIT(value) see below 568 569#define ATOMIC_FLAG_INIT see below 570 571} // std 572 573*/ 574 575#include <__config> 576#include <__availability> 577#include <__threading_support> 578#include <cstddef> 579#include <cstdint> 580#include <cstring> 581#include <type_traits> 582#include <version> 583 584#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) 585#pragma GCC system_header 586#endif 587 588#ifdef _LIBCPP_HAS_NO_THREADS 589# error <atomic> is not supported on this single threaded system 590#endif 591#ifdef _LIBCPP_HAS_NO_ATOMIC_HEADER 592# error <atomic> is not implemented 593#endif 594#ifdef kill_dependency 595# error C++ standard library is incompatible with <stdatomic.h> 596#endif 597 598#define _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) \ 599 _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_consume || \ 600 __m == memory_order_acquire || \ 601 __m == memory_order_acq_rel, \ 602 "memory order argument to atomic operation is invalid") 603 604#define _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) \ 605 _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_release || \ 606 __m == memory_order_acq_rel, \ 607 "memory order argument to atomic operation is invalid") 608 609#define _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__m, __f) \ 610 _LIBCPP_DIAGNOSE_WARNING(__f == memory_order_release || \ 611 __f == memory_order_acq_rel, \ 612 "memory order argument to atomic operation is invalid") 613 614_LIBCPP_BEGIN_NAMESPACE_STD 615 616// Figure out what the underlying type for `memory_order` would be if it were 617// declared as an unscoped enum (accounting for -fshort-enums). Use this result 618// to pin the underlying type in C++20. 619enum __legacy_memory_order { 620 __mo_relaxed, 621 __mo_consume, 622 __mo_acquire, 623 __mo_release, 624 __mo_acq_rel, 625 __mo_seq_cst 626}; 627 628typedef underlying_type<__legacy_memory_order>::type __memory_order_underlying_t; 629 630#if _LIBCPP_STD_VER > 17 631 632enum class memory_order : __memory_order_underlying_t { 633 relaxed = __mo_relaxed, 634 consume = __mo_consume, 635 acquire = __mo_acquire, 636 release = __mo_release, 637 acq_rel = __mo_acq_rel, 638 seq_cst = __mo_seq_cst 639}; 640 641inline constexpr auto memory_order_relaxed = memory_order::relaxed; 642inline constexpr auto memory_order_consume = memory_order::consume; 643inline constexpr auto memory_order_acquire = memory_order::acquire; 644inline constexpr auto memory_order_release = memory_order::release; 645inline constexpr auto memory_order_acq_rel = memory_order::acq_rel; 646inline constexpr auto memory_order_seq_cst = memory_order::seq_cst; 647 648#else 649 650typedef enum memory_order { 651 memory_order_relaxed = __mo_relaxed, 652 memory_order_consume = __mo_consume, 653 memory_order_acquire = __mo_acquire, 654 memory_order_release = __mo_release, 655 memory_order_acq_rel = __mo_acq_rel, 656 memory_order_seq_cst = __mo_seq_cst, 657} memory_order; 658 659#endif // _LIBCPP_STD_VER > 17 660 661template <typename _Tp> _LIBCPP_INLINE_VISIBILITY 662bool __cxx_nonatomic_compare_equal(_Tp const& __lhs, _Tp const& __rhs) { 663 return memcmp(&__lhs, &__rhs, sizeof(_Tp)) == 0; 664} 665 666static_assert((is_same<underlying_type<memory_order>::type, __memory_order_underlying_t>::value), 667 "unexpected underlying type for std::memory_order"); 668 669#if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) || \ 670 defined(_LIBCPP_ATOMIC_ONLY_USE_BUILTINS) 671 672// [atomics.types.generic]p1 guarantees _Tp is trivially copyable. Because 673// the default operator= in an object is not volatile, a byte-by-byte copy 674// is required. 675template <typename _Tp, typename _Tv> _LIBCPP_INLINE_VISIBILITY 676typename enable_if<is_assignable<_Tp&, _Tv>::value>::type 677__cxx_atomic_assign_volatile(_Tp& __a_value, _Tv const& __val) { 678 __a_value = __val; 679} 680template <typename _Tp, typename _Tv> _LIBCPP_INLINE_VISIBILITY 681typename enable_if<is_assignable<_Tp&, _Tv>::value>::type 682__cxx_atomic_assign_volatile(_Tp volatile& __a_value, _Tv volatile const& __val) { 683 volatile char* __to = reinterpret_cast<volatile char*>(&__a_value); 684 volatile char* __end = __to + sizeof(_Tp); 685 volatile const char* __from = reinterpret_cast<volatile const char*>(&__val); 686 while (__to != __end) 687 *__to++ = *__from++; 688} 689 690#endif 691 692#if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) 693 694template <typename _Tp> 695struct __cxx_atomic_base_impl { 696 697 _LIBCPP_INLINE_VISIBILITY 698#ifndef _LIBCPP_CXX03_LANG 699 __cxx_atomic_base_impl() _NOEXCEPT = default; 700#else 701 __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {} 702#endif // _LIBCPP_CXX03_LANG 703 _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT 704 : __a_value(value) {} 705 _Tp __a_value; 706}; 707 708_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR int __to_gcc_order(memory_order __order) { 709 // Avoid switch statement to make this a constexpr. 710 return __order == memory_order_relaxed ? __ATOMIC_RELAXED: 711 (__order == memory_order_acquire ? __ATOMIC_ACQUIRE: 712 (__order == memory_order_release ? __ATOMIC_RELEASE: 713 (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST: 714 (__order == memory_order_acq_rel ? __ATOMIC_ACQ_REL: 715 __ATOMIC_CONSUME)))); 716} 717 718_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR int __to_gcc_failure_order(memory_order __order) { 719 // Avoid switch statement to make this a constexpr. 720 return __order == memory_order_relaxed ? __ATOMIC_RELAXED: 721 (__order == memory_order_acquire ? __ATOMIC_ACQUIRE: 722 (__order == memory_order_release ? __ATOMIC_RELAXED: 723 (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST: 724 (__order == memory_order_acq_rel ? __ATOMIC_ACQUIRE: 725 __ATOMIC_CONSUME)))); 726} 727 728template <typename _Tp> 729_LIBCPP_INLINE_VISIBILITY 730void __cxx_atomic_init(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val) { 731 __cxx_atomic_assign_volatile(__a->__a_value, __val); 732} 733 734template <typename _Tp> 735_LIBCPP_INLINE_VISIBILITY 736void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) { 737 __a->__a_value = __val; 738} 739 740_LIBCPP_INLINE_VISIBILITY inline 741void __cxx_atomic_thread_fence(memory_order __order) { 742 __atomic_thread_fence(__to_gcc_order(__order)); 743} 744 745_LIBCPP_INLINE_VISIBILITY inline 746void __cxx_atomic_signal_fence(memory_order __order) { 747 __atomic_signal_fence(__to_gcc_order(__order)); 748} 749 750template <typename _Tp> 751_LIBCPP_INLINE_VISIBILITY 752void __cxx_atomic_store(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val, 753 memory_order __order) { 754 __atomic_store(&__a->__a_value, &__val, 755 __to_gcc_order(__order)); 756} 757 758template <typename _Tp> 759_LIBCPP_INLINE_VISIBILITY 760void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, 761 memory_order __order) { 762 __atomic_store(&__a->__a_value, &__val, 763 __to_gcc_order(__order)); 764} 765 766template <typename _Tp> 767_LIBCPP_INLINE_VISIBILITY 768_Tp __cxx_atomic_load(const volatile __cxx_atomic_base_impl<_Tp>* __a, 769 memory_order __order) { 770 _Tp __ret; 771 __atomic_load(&__a->__a_value, &__ret, 772 __to_gcc_order(__order)); 773 return __ret; 774} 775 776template <typename _Tp> 777_LIBCPP_INLINE_VISIBILITY 778_Tp __cxx_atomic_load(const __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) { 779 _Tp __ret; 780 __atomic_load(&__a->__a_value, &__ret, 781 __to_gcc_order(__order)); 782 return __ret; 783} 784 785template <typename _Tp> 786_LIBCPP_INLINE_VISIBILITY 787_Tp __cxx_atomic_exchange(volatile __cxx_atomic_base_impl<_Tp>* __a, 788 _Tp __value, memory_order __order) { 789 _Tp __ret; 790 __atomic_exchange(&__a->__a_value, &__value, &__ret, 791 __to_gcc_order(__order)); 792 return __ret; 793} 794 795template <typename _Tp> 796_LIBCPP_INLINE_VISIBILITY 797_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, 798 memory_order __order) { 799 _Tp __ret; 800 __atomic_exchange(&__a->__a_value, &__value, &__ret, 801 __to_gcc_order(__order)); 802 return __ret; 803} 804 805template <typename _Tp> 806_LIBCPP_INLINE_VISIBILITY 807bool __cxx_atomic_compare_exchange_strong( 808 volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, 809 memory_order __success, memory_order __failure) { 810 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 811 false, 812 __to_gcc_order(__success), 813 __to_gcc_failure_order(__failure)); 814} 815 816template <typename _Tp> 817_LIBCPP_INLINE_VISIBILITY 818bool __cxx_atomic_compare_exchange_strong( 819 __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, 820 memory_order __failure) { 821 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 822 false, 823 __to_gcc_order(__success), 824 __to_gcc_failure_order(__failure)); 825} 826 827template <typename _Tp> 828_LIBCPP_INLINE_VISIBILITY 829bool __cxx_atomic_compare_exchange_weak( 830 volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, 831 memory_order __success, memory_order __failure) { 832 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 833 true, 834 __to_gcc_order(__success), 835 __to_gcc_failure_order(__failure)); 836} 837 838template <typename _Tp> 839_LIBCPP_INLINE_VISIBILITY 840bool __cxx_atomic_compare_exchange_weak( 841 __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, 842 memory_order __failure) { 843 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 844 true, 845 __to_gcc_order(__success), 846 __to_gcc_failure_order(__failure)); 847} 848 849template <typename _Tp> 850struct __skip_amt { enum {value = 1}; }; 851 852template <typename _Tp> 853struct __skip_amt<_Tp*> { enum {value = sizeof(_Tp)}; }; 854 855// FIXME: Haven't figured out what the spec says about using arrays with 856// atomic_fetch_add. Force a failure rather than creating bad behavior. 857template <typename _Tp> 858struct __skip_amt<_Tp[]> { }; 859template <typename _Tp, int n> 860struct __skip_amt<_Tp[n]> { }; 861 862template <typename _Tp, typename _Td> 863_LIBCPP_INLINE_VISIBILITY 864_Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_base_impl<_Tp>* __a, 865 _Td __delta, memory_order __order) { 866 return __atomic_fetch_add(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 867 __to_gcc_order(__order)); 868} 869 870template <typename _Tp, typename _Td> 871_LIBCPP_INLINE_VISIBILITY 872_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, 873 memory_order __order) { 874 return __atomic_fetch_add(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 875 __to_gcc_order(__order)); 876} 877 878template <typename _Tp, typename _Td> 879_LIBCPP_INLINE_VISIBILITY 880_Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_base_impl<_Tp>* __a, 881 _Td __delta, memory_order __order) { 882 return __atomic_fetch_sub(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 883 __to_gcc_order(__order)); 884} 885 886template <typename _Tp, typename _Td> 887_LIBCPP_INLINE_VISIBILITY 888_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, 889 memory_order __order) { 890 return __atomic_fetch_sub(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 891 __to_gcc_order(__order)); 892} 893 894template <typename _Tp> 895_LIBCPP_INLINE_VISIBILITY 896_Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_base_impl<_Tp>* __a, 897 _Tp __pattern, memory_order __order) { 898 return __atomic_fetch_and(&__a->__a_value, __pattern, 899 __to_gcc_order(__order)); 900} 901 902template <typename _Tp> 903_LIBCPP_INLINE_VISIBILITY 904_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, 905 _Tp __pattern, memory_order __order) { 906 return __atomic_fetch_and(&__a->__a_value, __pattern, 907 __to_gcc_order(__order)); 908} 909 910template <typename _Tp> 911_LIBCPP_INLINE_VISIBILITY 912_Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_base_impl<_Tp>* __a, 913 _Tp __pattern, memory_order __order) { 914 return __atomic_fetch_or(&__a->__a_value, __pattern, 915 __to_gcc_order(__order)); 916} 917 918template <typename _Tp> 919_LIBCPP_INLINE_VISIBILITY 920_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, 921 memory_order __order) { 922 return __atomic_fetch_or(&__a->__a_value, __pattern, 923 __to_gcc_order(__order)); 924} 925 926template <typename _Tp> 927_LIBCPP_INLINE_VISIBILITY 928_Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_base_impl<_Tp>* __a, 929 _Tp __pattern, memory_order __order) { 930 return __atomic_fetch_xor(&__a->__a_value, __pattern, 931 __to_gcc_order(__order)); 932} 933 934template <typename _Tp> 935_LIBCPP_INLINE_VISIBILITY 936_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, 937 memory_order __order) { 938 return __atomic_fetch_xor(&__a->__a_value, __pattern, 939 __to_gcc_order(__order)); 940} 941 942#define __cxx_atomic_is_lock_free(__s) __atomic_is_lock_free(__s, 0) 943 944#elif defined(_LIBCPP_HAS_C_ATOMIC_IMP) 945 946template <typename _Tp> 947struct __cxx_atomic_base_impl { 948 949 _LIBCPP_INLINE_VISIBILITY 950#ifndef _LIBCPP_CXX03_LANG 951 __cxx_atomic_base_impl() _NOEXCEPT = default; 952#else 953 __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {} 954#endif // _LIBCPP_CXX03_LANG 955 _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT 956 : __a_value(value) {} 957 _LIBCPP_DISABLE_EXTENSION_WARNING _Atomic(_Tp) __a_value; 958}; 959 960#define __cxx_atomic_is_lock_free(__s) __c11_atomic_is_lock_free(__s) 961 962_LIBCPP_INLINE_VISIBILITY inline 963void __cxx_atomic_thread_fence(memory_order __order) _NOEXCEPT { 964 __c11_atomic_thread_fence(static_cast<__memory_order_underlying_t>(__order)); 965} 966 967_LIBCPP_INLINE_VISIBILITY inline 968void __cxx_atomic_signal_fence(memory_order __order) _NOEXCEPT { 969 __c11_atomic_signal_fence(static_cast<__memory_order_underlying_t>(__order)); 970} 971 972template<class _Tp> 973_LIBCPP_INLINE_VISIBILITY 974void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val) _NOEXCEPT { 975 __c11_atomic_init(&__a->__a_value, __val); 976} 977template<class _Tp> 978_LIBCPP_INLINE_VISIBILITY 979void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val) _NOEXCEPT { 980 __c11_atomic_init(&__a->__a_value, __val); 981} 982 983template<class _Tp> 984_LIBCPP_INLINE_VISIBILITY 985void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT { 986 __c11_atomic_store(&__a->__a_value, __val, static_cast<__memory_order_underlying_t>(__order)); 987} 988template<class _Tp> 989_LIBCPP_INLINE_VISIBILITY 990void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val, memory_order __order) _NOEXCEPT { 991 __c11_atomic_store(&__a->__a_value, __val, static_cast<__memory_order_underlying_t>(__order)); 992} 993 994template<class _Tp> 995_LIBCPP_INLINE_VISIBILITY 996_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const volatile* __a, memory_order __order) _NOEXCEPT { 997 using __ptr_type = typename remove_const<decltype(__a->__a_value)>::type*; 998 return __c11_atomic_load(const_cast<__ptr_type>(&__a->__a_value), static_cast<__memory_order_underlying_t>(__order)); 999} 1000template<class _Tp> 1001_LIBCPP_INLINE_VISIBILITY 1002_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const* __a, memory_order __order) _NOEXCEPT { 1003 using __ptr_type = typename remove_const<decltype(__a->__a_value)>::type*; 1004 return __c11_atomic_load(const_cast<__ptr_type>(&__a->__a_value), static_cast<__memory_order_underlying_t>(__order)); 1005} 1006 1007template<class _Tp> 1008_LIBCPP_INLINE_VISIBILITY 1009_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT { 1010 return __c11_atomic_exchange(&__a->__a_value, __value, static_cast<__memory_order_underlying_t>(__order)); 1011} 1012template<class _Tp> 1013_LIBCPP_INLINE_VISIBILITY 1014_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> * __a, _Tp __value, memory_order __order) _NOEXCEPT { 1015 return __c11_atomic_exchange(&__a->__a_value, __value, static_cast<__memory_order_underlying_t>(__order)); 1016} 1017 1018template<class _Tp> 1019_LIBCPP_INLINE_VISIBILITY 1020bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 1021 return __c11_atomic_compare_exchange_strong(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__failure)); 1022} 1023template<class _Tp> 1024_LIBCPP_INLINE_VISIBILITY 1025bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 1026 return __c11_atomic_compare_exchange_strong(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__failure)); 1027} 1028 1029template<class _Tp> 1030_LIBCPP_INLINE_VISIBILITY 1031bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 1032 return __c11_atomic_compare_exchange_weak(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__failure)); 1033} 1034template<class _Tp> 1035_LIBCPP_INLINE_VISIBILITY 1036bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 1037 return __c11_atomic_compare_exchange_weak(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__failure)); 1038} 1039 1040template<class _Tp> 1041_LIBCPP_INLINE_VISIBILITY 1042_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1043 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1044} 1045template<class _Tp> 1046_LIBCPP_INLINE_VISIBILITY 1047_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1048 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1049} 1050 1051template<class _Tp> 1052_LIBCPP_INLINE_VISIBILITY 1053_Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1054 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1055} 1056template<class _Tp> 1057_LIBCPP_INLINE_VISIBILITY 1058_Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1059 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1060} 1061 1062template<class _Tp> 1063_LIBCPP_INLINE_VISIBILITY 1064_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1065 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1066} 1067template<class _Tp> 1068_LIBCPP_INLINE_VISIBILITY 1069_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1070 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1071} 1072template<class _Tp> 1073_LIBCPP_INLINE_VISIBILITY 1074_Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1075 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1076} 1077template<class _Tp> 1078_LIBCPP_INLINE_VISIBILITY 1079_Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1080 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1081} 1082 1083template<class _Tp> 1084_LIBCPP_INLINE_VISIBILITY 1085_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1086 return __c11_atomic_fetch_and(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1087} 1088template<class _Tp> 1089_LIBCPP_INLINE_VISIBILITY 1090_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1091 return __c11_atomic_fetch_and(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1092} 1093 1094template<class _Tp> 1095_LIBCPP_INLINE_VISIBILITY 1096_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1097 return __c11_atomic_fetch_or(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1098} 1099template<class _Tp> 1100_LIBCPP_INLINE_VISIBILITY 1101_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1102 return __c11_atomic_fetch_or(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1103} 1104 1105template<class _Tp> 1106_LIBCPP_INLINE_VISIBILITY 1107_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1108 return __c11_atomic_fetch_xor(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1109} 1110template<class _Tp> 1111_LIBCPP_INLINE_VISIBILITY 1112_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1113 return __c11_atomic_fetch_xor(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1114} 1115 1116#endif // _LIBCPP_HAS_GCC_ATOMIC_IMP, _LIBCPP_HAS_C_ATOMIC_IMP 1117 1118template <class _Tp> 1119_LIBCPP_INLINE_VISIBILITY 1120_Tp kill_dependency(_Tp __y) _NOEXCEPT 1121{ 1122 return __y; 1123} 1124 1125#if defined(__CLANG_ATOMIC_BOOL_LOCK_FREE) 1126# define ATOMIC_BOOL_LOCK_FREE __CLANG_ATOMIC_BOOL_LOCK_FREE 1127# define ATOMIC_CHAR_LOCK_FREE __CLANG_ATOMIC_CHAR_LOCK_FREE 1128# define ATOMIC_CHAR16_T_LOCK_FREE __CLANG_ATOMIC_CHAR16_T_LOCK_FREE 1129# define ATOMIC_CHAR32_T_LOCK_FREE __CLANG_ATOMIC_CHAR32_T_LOCK_FREE 1130# define ATOMIC_WCHAR_T_LOCK_FREE __CLANG_ATOMIC_WCHAR_T_LOCK_FREE 1131# define ATOMIC_SHORT_LOCK_FREE __CLANG_ATOMIC_SHORT_LOCK_FREE 1132# define ATOMIC_INT_LOCK_FREE __CLANG_ATOMIC_INT_LOCK_FREE 1133# define ATOMIC_LONG_LOCK_FREE __CLANG_ATOMIC_LONG_LOCK_FREE 1134# define ATOMIC_LLONG_LOCK_FREE __CLANG_ATOMIC_LLONG_LOCK_FREE 1135# define ATOMIC_POINTER_LOCK_FREE __CLANG_ATOMIC_POINTER_LOCK_FREE 1136#elif defined(__GCC_ATOMIC_BOOL_LOCK_FREE) 1137# define ATOMIC_BOOL_LOCK_FREE __GCC_ATOMIC_BOOL_LOCK_FREE 1138# define ATOMIC_CHAR_LOCK_FREE __GCC_ATOMIC_CHAR_LOCK_FREE 1139# define ATOMIC_CHAR16_T_LOCK_FREE __GCC_ATOMIC_CHAR16_T_LOCK_FREE 1140# define ATOMIC_CHAR32_T_LOCK_FREE __GCC_ATOMIC_CHAR32_T_LOCK_FREE 1141# define ATOMIC_WCHAR_T_LOCK_FREE __GCC_ATOMIC_WCHAR_T_LOCK_FREE 1142# define ATOMIC_SHORT_LOCK_FREE __GCC_ATOMIC_SHORT_LOCK_FREE 1143# define ATOMIC_INT_LOCK_FREE __GCC_ATOMIC_INT_LOCK_FREE 1144# define ATOMIC_LONG_LOCK_FREE __GCC_ATOMIC_LONG_LOCK_FREE 1145# define ATOMIC_LLONG_LOCK_FREE __GCC_ATOMIC_LLONG_LOCK_FREE 1146# define ATOMIC_POINTER_LOCK_FREE __GCC_ATOMIC_POINTER_LOCK_FREE 1147#endif 1148 1149#ifdef _LIBCPP_ATOMIC_ONLY_USE_BUILTINS 1150 1151template<typename _Tp> 1152struct __cxx_atomic_lock_impl { 1153 1154 _LIBCPP_INLINE_VISIBILITY 1155 __cxx_atomic_lock_impl() _NOEXCEPT 1156 : __a_value(), __a_lock(0) {} 1157 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR explicit 1158 __cxx_atomic_lock_impl(_Tp value) _NOEXCEPT 1159 : __a_value(value), __a_lock(0) {} 1160 1161 _Tp __a_value; 1162 mutable __cxx_atomic_base_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_lock; 1163 1164 _LIBCPP_INLINE_VISIBILITY void __lock() const volatile { 1165 while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire)) 1166 /*spin*/; 1167 } 1168 _LIBCPP_INLINE_VISIBILITY void __lock() const { 1169 while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire)) 1170 /*spin*/; 1171 } 1172 _LIBCPP_INLINE_VISIBILITY void __unlock() const volatile { 1173 __cxx_atomic_store(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(false), memory_order_release); 1174 } 1175 _LIBCPP_INLINE_VISIBILITY void __unlock() const { 1176 __cxx_atomic_store(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(false), memory_order_release); 1177 } 1178 _LIBCPP_INLINE_VISIBILITY _Tp __read() const volatile { 1179 __lock(); 1180 _Tp __old; 1181 __cxx_atomic_assign_volatile(__old, __a_value); 1182 __unlock(); 1183 return __old; 1184 } 1185 _LIBCPP_INLINE_VISIBILITY _Tp __read() const { 1186 __lock(); 1187 _Tp __old = __a_value; 1188 __unlock(); 1189 return __old; 1190 } 1191}; 1192 1193template <typename _Tp> 1194_LIBCPP_INLINE_VISIBILITY 1195void __cxx_atomic_init(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __val) { 1196 __cxx_atomic_assign_volatile(__a->__a_value, __val); 1197} 1198template <typename _Tp> 1199_LIBCPP_INLINE_VISIBILITY 1200void __cxx_atomic_init(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __val) { 1201 __a->__a_value = __val; 1202} 1203 1204template <typename _Tp> 1205_LIBCPP_INLINE_VISIBILITY 1206void __cxx_atomic_store(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __val, memory_order) { 1207 __a->__lock(); 1208 __cxx_atomic_assign_volatile(__a->__a_value, __val); 1209 __a->__unlock(); 1210} 1211template <typename _Tp> 1212_LIBCPP_INLINE_VISIBILITY 1213void __cxx_atomic_store(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __val, memory_order) { 1214 __a->__lock(); 1215 __a->__a_value = __val; 1216 __a->__unlock(); 1217} 1218 1219template <typename _Tp> 1220_LIBCPP_INLINE_VISIBILITY 1221_Tp __cxx_atomic_load(const volatile __cxx_atomic_lock_impl<_Tp>* __a, memory_order) { 1222 return __a->__read(); 1223} 1224template <typename _Tp> 1225_LIBCPP_INLINE_VISIBILITY 1226_Tp __cxx_atomic_load(const __cxx_atomic_lock_impl<_Tp>* __a, memory_order) { 1227 return __a->__read(); 1228} 1229 1230template <typename _Tp> 1231_LIBCPP_INLINE_VISIBILITY 1232_Tp __cxx_atomic_exchange(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) { 1233 __a->__lock(); 1234 _Tp __old; 1235 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1236 __cxx_atomic_assign_volatile(__a->__a_value, __value); 1237 __a->__unlock(); 1238 return __old; 1239} 1240template <typename _Tp> 1241_LIBCPP_INLINE_VISIBILITY 1242_Tp __cxx_atomic_exchange(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) { 1243 __a->__lock(); 1244 _Tp __old = __a->__a_value; 1245 __a->__a_value = __value; 1246 __a->__unlock(); 1247 return __old; 1248} 1249 1250template <typename _Tp> 1251_LIBCPP_INLINE_VISIBILITY 1252bool __cxx_atomic_compare_exchange_strong(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1253 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1254 _Tp __temp; 1255 __a->__lock(); 1256 __cxx_atomic_assign_volatile(__temp, __a->__a_value); 1257 bool __ret = (memcmp(&__temp, __expected, sizeof(_Tp)) == 0); 1258 if(__ret) 1259 __cxx_atomic_assign_volatile(__a->__a_value, __value); 1260 else 1261 __cxx_atomic_assign_volatile(*__expected, __a->__a_value); 1262 __a->__unlock(); 1263 return __ret; 1264} 1265template <typename _Tp> 1266_LIBCPP_INLINE_VISIBILITY 1267bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_lock_impl<_Tp>* __a, 1268 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1269 __a->__lock(); 1270 bool __ret = (memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0); 1271 if(__ret) 1272 memcpy(&__a->__a_value, &__value, sizeof(_Tp)); 1273 else 1274 memcpy(__expected, &__a->__a_value, sizeof(_Tp)); 1275 __a->__unlock(); 1276 return __ret; 1277} 1278 1279template <typename _Tp> 1280_LIBCPP_INLINE_VISIBILITY 1281bool __cxx_atomic_compare_exchange_weak(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1282 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1283 _Tp __temp; 1284 __a->__lock(); 1285 __cxx_atomic_assign_volatile(__temp, __a->__a_value); 1286 bool __ret = (memcmp(&__temp, __expected, sizeof(_Tp)) == 0); 1287 if(__ret) 1288 __cxx_atomic_assign_volatile(__a->__a_value, __value); 1289 else 1290 __cxx_atomic_assign_volatile(*__expected, __a->__a_value); 1291 __a->__unlock(); 1292 return __ret; 1293} 1294template <typename _Tp> 1295_LIBCPP_INLINE_VISIBILITY 1296bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_lock_impl<_Tp>* __a, 1297 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1298 __a->__lock(); 1299 bool __ret = (memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0); 1300 if(__ret) 1301 memcpy(&__a->__a_value, &__value, sizeof(_Tp)); 1302 else 1303 memcpy(__expected, &__a->__a_value, sizeof(_Tp)); 1304 __a->__unlock(); 1305 return __ret; 1306} 1307 1308template <typename _Tp, typename _Td> 1309_LIBCPP_INLINE_VISIBILITY 1310_Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1311 _Td __delta, memory_order) { 1312 __a->__lock(); 1313 _Tp __old; 1314 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1315 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old + __delta)); 1316 __a->__unlock(); 1317 return __old; 1318} 1319template <typename _Tp, typename _Td> 1320_LIBCPP_INLINE_VISIBILITY 1321_Tp __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp>* __a, 1322 _Td __delta, memory_order) { 1323 __a->__lock(); 1324 _Tp __old = __a->__a_value; 1325 __a->__a_value += __delta; 1326 __a->__unlock(); 1327 return __old; 1328} 1329 1330template <typename _Tp, typename _Td> 1331_LIBCPP_INLINE_VISIBILITY 1332_Tp* __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp*>* __a, 1333 ptrdiff_t __delta, memory_order) { 1334 __a->__lock(); 1335 _Tp* __old; 1336 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1337 __cxx_atomic_assign_volatile(__a->__a_value, __old + __delta); 1338 __a->__unlock(); 1339 return __old; 1340} 1341template <typename _Tp, typename _Td> 1342_LIBCPP_INLINE_VISIBILITY 1343_Tp* __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp*>* __a, 1344 ptrdiff_t __delta, memory_order) { 1345 __a->__lock(); 1346 _Tp* __old = __a->__a_value; 1347 __a->__a_value += __delta; 1348 __a->__unlock(); 1349 return __old; 1350} 1351 1352template <typename _Tp, typename _Td> 1353_LIBCPP_INLINE_VISIBILITY 1354_Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1355 _Td __delta, memory_order) { 1356 __a->__lock(); 1357 _Tp __old; 1358 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1359 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old - __delta)); 1360 __a->__unlock(); 1361 return __old; 1362} 1363template <typename _Tp, typename _Td> 1364_LIBCPP_INLINE_VISIBILITY 1365_Tp __cxx_atomic_fetch_sub(__cxx_atomic_lock_impl<_Tp>* __a, 1366 _Td __delta, memory_order) { 1367 __a->__lock(); 1368 _Tp __old = __a->__a_value; 1369 __a->__a_value -= __delta; 1370 __a->__unlock(); 1371 return __old; 1372} 1373 1374template <typename _Tp> 1375_LIBCPP_INLINE_VISIBILITY 1376_Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1377 _Tp __pattern, memory_order) { 1378 __a->__lock(); 1379 _Tp __old; 1380 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1381 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old & __pattern)); 1382 __a->__unlock(); 1383 return __old; 1384} 1385template <typename _Tp> 1386_LIBCPP_INLINE_VISIBILITY 1387_Tp __cxx_atomic_fetch_and(__cxx_atomic_lock_impl<_Tp>* __a, 1388 _Tp __pattern, memory_order) { 1389 __a->__lock(); 1390 _Tp __old = __a->__a_value; 1391 __a->__a_value &= __pattern; 1392 __a->__unlock(); 1393 return __old; 1394} 1395 1396template <typename _Tp> 1397_LIBCPP_INLINE_VISIBILITY 1398_Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1399 _Tp __pattern, memory_order) { 1400 __a->__lock(); 1401 _Tp __old; 1402 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1403 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old | __pattern)); 1404 __a->__unlock(); 1405 return __old; 1406} 1407template <typename _Tp> 1408_LIBCPP_INLINE_VISIBILITY 1409_Tp __cxx_atomic_fetch_or(__cxx_atomic_lock_impl<_Tp>* __a, 1410 _Tp __pattern, memory_order) { 1411 __a->__lock(); 1412 _Tp __old = __a->__a_value; 1413 __a->__a_value |= __pattern; 1414 __a->__unlock(); 1415 return __old; 1416} 1417 1418template <typename _Tp> 1419_LIBCPP_INLINE_VISIBILITY 1420_Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1421 _Tp __pattern, memory_order) { 1422 __a->__lock(); 1423 _Tp __old; 1424 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1425 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old ^ __pattern)); 1426 __a->__unlock(); 1427 return __old; 1428} 1429template <typename _Tp> 1430_LIBCPP_INLINE_VISIBILITY 1431_Tp __cxx_atomic_fetch_xor(__cxx_atomic_lock_impl<_Tp>* __a, 1432 _Tp __pattern, memory_order) { 1433 __a->__lock(); 1434 _Tp __old = __a->__a_value; 1435 __a->__a_value ^= __pattern; 1436 __a->__unlock(); 1437 return __old; 1438} 1439 1440#ifdef __cpp_lib_atomic_is_always_lock_free 1441 1442template<typename _Tp> struct __cxx_is_always_lock_free { 1443 enum { __value = __atomic_always_lock_free(sizeof(_Tp), 0) }; }; 1444 1445#else 1446 1447template<typename _Tp> struct __cxx_is_always_lock_free { enum { __value = false }; }; 1448// Implementations must match the C ATOMIC_*_LOCK_FREE macro values. 1449template<> struct __cxx_is_always_lock_free<bool> { enum { __value = 2 == ATOMIC_BOOL_LOCK_FREE }; }; 1450template<> struct __cxx_is_always_lock_free<char> { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; }; 1451template<> struct __cxx_is_always_lock_free<signed char> { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; }; 1452template<> struct __cxx_is_always_lock_free<unsigned char> { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; }; 1453template<> struct __cxx_is_always_lock_free<char16_t> { enum { __value = 2 == ATOMIC_CHAR16_T_LOCK_FREE }; }; 1454template<> struct __cxx_is_always_lock_free<char32_t> { enum { __value = 2 == ATOMIC_CHAR32_T_LOCK_FREE }; }; 1455template<> struct __cxx_is_always_lock_free<wchar_t> { enum { __value = 2 == ATOMIC_WCHAR_T_LOCK_FREE }; }; 1456template<> struct __cxx_is_always_lock_free<short> { enum { __value = 2 == ATOMIC_SHORT_LOCK_FREE }; }; 1457template<> struct __cxx_is_always_lock_free<unsigned short> { enum { __value = 2 == ATOMIC_SHORT_LOCK_FREE }; }; 1458template<> struct __cxx_is_always_lock_free<int> { enum { __value = 2 == ATOMIC_INT_LOCK_FREE }; }; 1459template<> struct __cxx_is_always_lock_free<unsigned int> { enum { __value = 2 == ATOMIC_INT_LOCK_FREE }; }; 1460template<> struct __cxx_is_always_lock_free<long> { enum { __value = 2 == ATOMIC_LONG_LOCK_FREE }; }; 1461template<> struct __cxx_is_always_lock_free<unsigned long> { enum { __value = 2 == ATOMIC_LONG_LOCK_FREE }; }; 1462template<> struct __cxx_is_always_lock_free<long long> { enum { __value = 2 == ATOMIC_LLONG_LOCK_FREE }; }; 1463template<> struct __cxx_is_always_lock_free<unsigned long long> { enum { __value = 2 == ATOMIC_LLONG_LOCK_FREE }; }; 1464template<typename _Tp> struct __cxx_is_always_lock_free<_Tp*> { enum { __value = 2 == ATOMIC_POINTER_LOCK_FREE }; }; 1465template<> struct __cxx_is_always_lock_free<std::nullptr_t> { enum { __value = 2 == ATOMIC_POINTER_LOCK_FREE }; }; 1466 1467#endif //__cpp_lib_atomic_is_always_lock_free 1468 1469template <typename _Tp, 1470 typename _Base = typename conditional<__cxx_is_always_lock_free<_Tp>::__value, 1471 __cxx_atomic_base_impl<_Tp>, 1472 __cxx_atomic_lock_impl<_Tp> >::type> 1473#else 1474template <typename _Tp, 1475 typename _Base = __cxx_atomic_base_impl<_Tp> > 1476#endif //_LIBCPP_ATOMIC_ONLY_USE_BUILTINS 1477struct __cxx_atomic_impl : public _Base { 1478 1479#if _GNUC_VER >= 501 1480 static_assert(is_trivially_copyable<_Tp>::value, 1481 "std::atomic<Tp> requires that 'Tp' be a trivially copyable type"); 1482#endif 1483 1484 _LIBCPP_INLINE_VISIBILITY __cxx_atomic_impl() _NOEXCEPT _LIBCPP_DEFAULT 1485 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR explicit __cxx_atomic_impl(_Tp value) _NOEXCEPT 1486 : _Base(value) {} 1487}; 1488 1489#ifdef __linux__ 1490 using __cxx_contention_t = int32_t; 1491#else 1492 using __cxx_contention_t = int64_t; 1493#endif //__linux__ 1494 1495#if _LIBCPP_STD_VER >= 11 1496 1497using __cxx_atomic_contention_t = __cxx_atomic_impl<__cxx_contention_t>; 1498 1499#ifndef _LIBCPP_HAS_NO_PLATFORM_WAIT 1500 1501_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(void const volatile*); 1502_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(void const volatile*); 1503_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(void const volatile*); 1504_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(void const volatile*, __cxx_contention_t); 1505 1506_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(__cxx_atomic_contention_t const volatile*); 1507_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(__cxx_atomic_contention_t const volatile*); 1508_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile*); 1509_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(__cxx_atomic_contention_t const volatile*, __cxx_contention_t); 1510 1511template <class _Atp, class _Fn> 1512struct __libcpp_atomic_wait_backoff_impl { 1513 _Atp* __a; 1514 _Fn __test_fn; 1515 _LIBCPP_AVAILABILITY_SYNC 1516 _LIBCPP_INLINE_VISIBILITY bool operator()(chrono::nanoseconds __elapsed) const 1517 { 1518 if(__elapsed > chrono::microseconds(64)) 1519 { 1520 auto const __monitor = __libcpp_atomic_monitor(__a); 1521 if(__test_fn()) 1522 return true; 1523 __libcpp_atomic_wait(__a, __monitor); 1524 } 1525 else if(__elapsed > chrono::microseconds(4)) 1526 __libcpp_thread_yield(); 1527 else 1528 ; // poll 1529 return false; 1530 } 1531}; 1532 1533template <class _Atp, class _Fn> 1534_LIBCPP_AVAILABILITY_SYNC 1535_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp* __a, _Fn && __test_fn) 1536{ 1537 __libcpp_atomic_wait_backoff_impl<_Atp, typename decay<_Fn>::type> __backoff_fn = {__a, __test_fn}; 1538 return __libcpp_thread_poll_with_backoff(__test_fn, __backoff_fn); 1539} 1540 1541#else // _LIBCPP_HAS_NO_PLATFORM_WAIT 1542 1543template <class _Tp> 1544_LIBCPP_INLINE_VISIBILITY void __cxx_atomic_notify_all(__cxx_atomic_impl<_Tp> const volatile*) { } 1545template <class _Tp> 1546_LIBCPP_INLINE_VISIBILITY void __cxx_atomic_notify_one(__cxx_atomic_impl<_Tp> const volatile*) { } 1547template <class _Atp, class _Fn> 1548_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp*, _Fn && __test_fn) 1549{ 1550 return __libcpp_thread_poll_with_backoff(__test_fn, __libcpp_timed_backoff_policy()); 1551} 1552 1553#endif // _LIBCPP_HAS_NO_PLATFORM_WAIT 1554 1555template <class _Atp, class _Tp> 1556struct __cxx_atomic_wait_test_fn_impl { 1557 _Atp* __a; 1558 _Tp __val; 1559 memory_order __order; 1560 _LIBCPP_INLINE_VISIBILITY bool operator()() const 1561 { 1562 return !__cxx_nonatomic_compare_equal(__cxx_atomic_load(__a, __order), __val); 1563 } 1564}; 1565 1566template <class _Atp, class _Tp> 1567_LIBCPP_AVAILABILITY_SYNC 1568_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp* __a, _Tp const __val, memory_order __order) 1569{ 1570 __cxx_atomic_wait_test_fn_impl<_Atp, _Tp> __test_fn = {__a, __val, __order}; 1571 return __cxx_atomic_wait(__a, __test_fn); 1572} 1573 1574#endif //_LIBCPP_STD_VER >= 11 1575 1576// general atomic<T> 1577 1578template <class _Tp, bool = is_integral<_Tp>::value && !is_same<_Tp, bool>::value> 1579struct __atomic_base // false 1580{ 1581 mutable __cxx_atomic_impl<_Tp> __a_; 1582 1583#if defined(__cpp_lib_atomic_is_always_lock_free) 1584 static _LIBCPP_CONSTEXPR bool is_always_lock_free = __atomic_always_lock_free(sizeof(__a_), 0); 1585#endif 1586 1587 _LIBCPP_INLINE_VISIBILITY 1588 bool is_lock_free() const volatile _NOEXCEPT 1589 {return __cxx_atomic_is_lock_free(sizeof(_Tp));} 1590 _LIBCPP_INLINE_VISIBILITY 1591 bool is_lock_free() const _NOEXCEPT 1592 {return static_cast<__atomic_base const volatile*>(this)->is_lock_free();} 1593 _LIBCPP_INLINE_VISIBILITY 1594 void store(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1595 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1596 {__cxx_atomic_store(&__a_, __d, __m);} 1597 _LIBCPP_INLINE_VISIBILITY 1598 void store(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1599 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1600 {__cxx_atomic_store(&__a_, __d, __m);} 1601 _LIBCPP_INLINE_VISIBILITY 1602 _Tp load(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 1603 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 1604 {return __cxx_atomic_load(&__a_, __m);} 1605 _LIBCPP_INLINE_VISIBILITY 1606 _Tp load(memory_order __m = memory_order_seq_cst) const _NOEXCEPT 1607 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 1608 {return __cxx_atomic_load(&__a_, __m);} 1609 _LIBCPP_INLINE_VISIBILITY 1610 operator _Tp() const volatile _NOEXCEPT {return load();} 1611 _LIBCPP_INLINE_VISIBILITY 1612 operator _Tp() const _NOEXCEPT {return load();} 1613 _LIBCPP_INLINE_VISIBILITY 1614 _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1615 {return __cxx_atomic_exchange(&__a_, __d, __m);} 1616 _LIBCPP_INLINE_VISIBILITY 1617 _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1618 {return __cxx_atomic_exchange(&__a_, __d, __m);} 1619 _LIBCPP_INLINE_VISIBILITY 1620 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1621 memory_order __s, memory_order __f) volatile _NOEXCEPT 1622 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1623 {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);} 1624 _LIBCPP_INLINE_VISIBILITY 1625 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1626 memory_order __s, memory_order __f) _NOEXCEPT 1627 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1628 {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);} 1629 _LIBCPP_INLINE_VISIBILITY 1630 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1631 memory_order __s, memory_order __f) volatile _NOEXCEPT 1632 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1633 {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);} 1634 _LIBCPP_INLINE_VISIBILITY 1635 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1636 memory_order __s, memory_order __f) _NOEXCEPT 1637 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1638 {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);} 1639 _LIBCPP_INLINE_VISIBILITY 1640 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1641 memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1642 {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);} 1643 _LIBCPP_INLINE_VISIBILITY 1644 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1645 memory_order __m = memory_order_seq_cst) _NOEXCEPT 1646 {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);} 1647 _LIBCPP_INLINE_VISIBILITY 1648 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1649 memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1650 {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);} 1651 _LIBCPP_INLINE_VISIBILITY 1652 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1653 memory_order __m = memory_order_seq_cst) _NOEXCEPT 1654 {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);} 1655 1656 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 1657 {__cxx_atomic_wait(&__a_, __v, __m);} 1658 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT 1659 {__cxx_atomic_wait(&__a_, __v, __m);} 1660 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_one() volatile _NOEXCEPT 1661 {__cxx_atomic_notify_one(&__a_);} 1662 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_one() _NOEXCEPT 1663 {__cxx_atomic_notify_one(&__a_);} 1664 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_all() volatile _NOEXCEPT 1665 {__cxx_atomic_notify_all(&__a_);} 1666 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_all() _NOEXCEPT 1667 {__cxx_atomic_notify_all(&__a_);} 1668 1669 _LIBCPP_INLINE_VISIBILITY 1670 __atomic_base() _NOEXCEPT _LIBCPP_DEFAULT 1671 1672 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR 1673 __atomic_base(_Tp __d) _NOEXCEPT : __a_(__d) {} 1674 1675#ifndef _LIBCPP_CXX03_LANG 1676 __atomic_base(const __atomic_base&) = delete; 1677 __atomic_base& operator=(const __atomic_base&) = delete; 1678 __atomic_base& operator=(const __atomic_base&) volatile = delete; 1679#else 1680private: 1681 _LIBCPP_INLINE_VISIBILITY 1682 __atomic_base(const __atomic_base&); 1683 _LIBCPP_INLINE_VISIBILITY 1684 __atomic_base& operator=(const __atomic_base&); 1685 _LIBCPP_INLINE_VISIBILITY 1686 __atomic_base& operator=(const __atomic_base&) volatile; 1687#endif 1688}; 1689 1690#if defined(__cpp_lib_atomic_is_always_lock_free) 1691template <class _Tp, bool __b> 1692_LIBCPP_CONSTEXPR bool __atomic_base<_Tp, __b>::is_always_lock_free; 1693#endif 1694 1695// atomic<Integral> 1696 1697template <class _Tp> 1698struct __atomic_base<_Tp, true> 1699 : public __atomic_base<_Tp, false> 1700{ 1701 typedef __atomic_base<_Tp, false> __base; 1702 _LIBCPP_INLINE_VISIBILITY 1703 __atomic_base() _NOEXCEPT _LIBCPP_DEFAULT 1704 _LIBCPP_INLINE_VISIBILITY 1705 _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __base(__d) {} 1706 1707 _LIBCPP_INLINE_VISIBILITY 1708 _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1709 {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);} 1710 _LIBCPP_INLINE_VISIBILITY 1711 _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1712 {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);} 1713 _LIBCPP_INLINE_VISIBILITY 1714 _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1715 {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);} 1716 _LIBCPP_INLINE_VISIBILITY 1717 _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1718 {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);} 1719 _LIBCPP_INLINE_VISIBILITY 1720 _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1721 {return __cxx_atomic_fetch_and(&this->__a_, __op, __m);} 1722 _LIBCPP_INLINE_VISIBILITY 1723 _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1724 {return __cxx_atomic_fetch_and(&this->__a_, __op, __m);} 1725 _LIBCPP_INLINE_VISIBILITY 1726 _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1727 {return __cxx_atomic_fetch_or(&this->__a_, __op, __m);} 1728 _LIBCPP_INLINE_VISIBILITY 1729 _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1730 {return __cxx_atomic_fetch_or(&this->__a_, __op, __m);} 1731 _LIBCPP_INLINE_VISIBILITY 1732 _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1733 {return __cxx_atomic_fetch_xor(&this->__a_, __op, __m);} 1734 _LIBCPP_INLINE_VISIBILITY 1735 _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1736 {return __cxx_atomic_fetch_xor(&this->__a_, __op, __m);} 1737 1738 _LIBCPP_INLINE_VISIBILITY 1739 _Tp operator++(int) volatile _NOEXCEPT {return fetch_add(_Tp(1));} 1740 _LIBCPP_INLINE_VISIBILITY 1741 _Tp operator++(int) _NOEXCEPT {return fetch_add(_Tp(1));} 1742 _LIBCPP_INLINE_VISIBILITY 1743 _Tp operator--(int) volatile _NOEXCEPT {return fetch_sub(_Tp(1));} 1744 _LIBCPP_INLINE_VISIBILITY 1745 _Tp operator--(int) _NOEXCEPT {return fetch_sub(_Tp(1));} 1746 _LIBCPP_INLINE_VISIBILITY 1747 _Tp operator++() volatile _NOEXCEPT {return fetch_add(_Tp(1)) + _Tp(1);} 1748 _LIBCPP_INLINE_VISIBILITY 1749 _Tp operator++() _NOEXCEPT {return fetch_add(_Tp(1)) + _Tp(1);} 1750 _LIBCPP_INLINE_VISIBILITY 1751 _Tp operator--() volatile _NOEXCEPT {return fetch_sub(_Tp(1)) - _Tp(1);} 1752 _LIBCPP_INLINE_VISIBILITY 1753 _Tp operator--() _NOEXCEPT {return fetch_sub(_Tp(1)) - _Tp(1);} 1754 _LIBCPP_INLINE_VISIBILITY 1755 _Tp operator+=(_Tp __op) volatile _NOEXCEPT {return fetch_add(__op) + __op;} 1756 _LIBCPP_INLINE_VISIBILITY 1757 _Tp operator+=(_Tp __op) _NOEXCEPT {return fetch_add(__op) + __op;} 1758 _LIBCPP_INLINE_VISIBILITY 1759 _Tp operator-=(_Tp __op) volatile _NOEXCEPT {return fetch_sub(__op) - __op;} 1760 _LIBCPP_INLINE_VISIBILITY 1761 _Tp operator-=(_Tp __op) _NOEXCEPT {return fetch_sub(__op) - __op;} 1762 _LIBCPP_INLINE_VISIBILITY 1763 _Tp operator&=(_Tp __op) volatile _NOEXCEPT {return fetch_and(__op) & __op;} 1764 _LIBCPP_INLINE_VISIBILITY 1765 _Tp operator&=(_Tp __op) _NOEXCEPT {return fetch_and(__op) & __op;} 1766 _LIBCPP_INLINE_VISIBILITY 1767 _Tp operator|=(_Tp __op) volatile _NOEXCEPT {return fetch_or(__op) | __op;} 1768 _LIBCPP_INLINE_VISIBILITY 1769 _Tp operator|=(_Tp __op) _NOEXCEPT {return fetch_or(__op) | __op;} 1770 _LIBCPP_INLINE_VISIBILITY 1771 _Tp operator^=(_Tp __op) volatile _NOEXCEPT {return fetch_xor(__op) ^ __op;} 1772 _LIBCPP_INLINE_VISIBILITY 1773 _Tp operator^=(_Tp __op) _NOEXCEPT {return fetch_xor(__op) ^ __op;} 1774}; 1775 1776// atomic<T> 1777 1778template <class _Tp> 1779struct atomic 1780 : public __atomic_base<_Tp> 1781{ 1782 typedef __atomic_base<_Tp> __base; 1783 typedef _Tp value_type; 1784 typedef value_type difference_type; 1785 _LIBCPP_INLINE_VISIBILITY 1786 atomic() _NOEXCEPT _LIBCPP_DEFAULT 1787 _LIBCPP_INLINE_VISIBILITY 1788 _LIBCPP_CONSTEXPR atomic(_Tp __d) _NOEXCEPT : __base(__d) {} 1789 1790 _LIBCPP_INLINE_VISIBILITY 1791 _Tp operator=(_Tp __d) volatile _NOEXCEPT 1792 {__base::store(__d); return __d;} 1793 _LIBCPP_INLINE_VISIBILITY 1794 _Tp operator=(_Tp __d) _NOEXCEPT 1795 {__base::store(__d); return __d;} 1796}; 1797 1798// atomic<T*> 1799 1800template <class _Tp> 1801struct atomic<_Tp*> 1802 : public __atomic_base<_Tp*> 1803{ 1804 typedef __atomic_base<_Tp*> __base; 1805 typedef _Tp* value_type; 1806 typedef ptrdiff_t difference_type; 1807 _LIBCPP_INLINE_VISIBILITY 1808 atomic() _NOEXCEPT _LIBCPP_DEFAULT 1809 _LIBCPP_INLINE_VISIBILITY 1810 _LIBCPP_CONSTEXPR atomic(_Tp* __d) _NOEXCEPT : __base(__d) {} 1811 1812 _LIBCPP_INLINE_VISIBILITY 1813 _Tp* operator=(_Tp* __d) volatile _NOEXCEPT 1814 {__base::store(__d); return __d;} 1815 _LIBCPP_INLINE_VISIBILITY 1816 _Tp* operator=(_Tp* __d) _NOEXCEPT 1817 {__base::store(__d); return __d;} 1818 1819 _LIBCPP_INLINE_VISIBILITY 1820 _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) 1821 volatile _NOEXCEPT 1822 {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);} 1823 _LIBCPP_INLINE_VISIBILITY 1824 _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1825 {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);} 1826 _LIBCPP_INLINE_VISIBILITY 1827 _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) 1828 volatile _NOEXCEPT 1829 {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);} 1830 _LIBCPP_INLINE_VISIBILITY 1831 _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1832 {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);} 1833 1834 _LIBCPP_INLINE_VISIBILITY 1835 _Tp* operator++(int) volatile _NOEXCEPT {return fetch_add(1);} 1836 _LIBCPP_INLINE_VISIBILITY 1837 _Tp* operator++(int) _NOEXCEPT {return fetch_add(1);} 1838 _LIBCPP_INLINE_VISIBILITY 1839 _Tp* operator--(int) volatile _NOEXCEPT {return fetch_sub(1);} 1840 _LIBCPP_INLINE_VISIBILITY 1841 _Tp* operator--(int) _NOEXCEPT {return fetch_sub(1);} 1842 _LIBCPP_INLINE_VISIBILITY 1843 _Tp* operator++() volatile _NOEXCEPT {return fetch_add(1) + 1;} 1844 _LIBCPP_INLINE_VISIBILITY 1845 _Tp* operator++() _NOEXCEPT {return fetch_add(1) + 1;} 1846 _LIBCPP_INLINE_VISIBILITY 1847 _Tp* operator--() volatile _NOEXCEPT {return fetch_sub(1) - 1;} 1848 _LIBCPP_INLINE_VISIBILITY 1849 _Tp* operator--() _NOEXCEPT {return fetch_sub(1) - 1;} 1850 _LIBCPP_INLINE_VISIBILITY 1851 _Tp* operator+=(ptrdiff_t __op) volatile _NOEXCEPT {return fetch_add(__op) + __op;} 1852 _LIBCPP_INLINE_VISIBILITY 1853 _Tp* operator+=(ptrdiff_t __op) _NOEXCEPT {return fetch_add(__op) + __op;} 1854 _LIBCPP_INLINE_VISIBILITY 1855 _Tp* operator-=(ptrdiff_t __op) volatile _NOEXCEPT {return fetch_sub(__op) - __op;} 1856 _LIBCPP_INLINE_VISIBILITY 1857 _Tp* operator-=(ptrdiff_t __op) _NOEXCEPT {return fetch_sub(__op) - __op;} 1858}; 1859 1860// atomic_is_lock_free 1861 1862template <class _Tp> 1863_LIBCPP_INLINE_VISIBILITY 1864bool 1865atomic_is_lock_free(const volatile atomic<_Tp>* __o) _NOEXCEPT 1866{ 1867 return __o->is_lock_free(); 1868} 1869 1870template <class _Tp> 1871_LIBCPP_INLINE_VISIBILITY 1872bool 1873atomic_is_lock_free(const atomic<_Tp>* __o) _NOEXCEPT 1874{ 1875 return __o->is_lock_free(); 1876} 1877 1878// atomic_init 1879 1880template <class _Tp> 1881_LIBCPP_INLINE_VISIBILITY 1882void 1883atomic_init(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1884{ 1885 __cxx_atomic_init(&__o->__a_, __d); 1886} 1887 1888template <class _Tp> 1889_LIBCPP_INLINE_VISIBILITY 1890void 1891atomic_init(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1892{ 1893 __cxx_atomic_init(&__o->__a_, __d); 1894} 1895 1896// atomic_store 1897 1898template <class _Tp> 1899_LIBCPP_INLINE_VISIBILITY 1900void 1901atomic_store(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1902{ 1903 __o->store(__d); 1904} 1905 1906template <class _Tp> 1907_LIBCPP_INLINE_VISIBILITY 1908void 1909atomic_store(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1910{ 1911 __o->store(__d); 1912} 1913 1914// atomic_store_explicit 1915 1916template <class _Tp> 1917_LIBCPP_INLINE_VISIBILITY 1918void 1919atomic_store_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 1920 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1921{ 1922 __o->store(__d, __m); 1923} 1924 1925template <class _Tp> 1926_LIBCPP_INLINE_VISIBILITY 1927void 1928atomic_store_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 1929 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1930{ 1931 __o->store(__d, __m); 1932} 1933 1934// atomic_load 1935 1936template <class _Tp> 1937_LIBCPP_INLINE_VISIBILITY 1938_Tp 1939atomic_load(const volatile atomic<_Tp>* __o) _NOEXCEPT 1940{ 1941 return __o->load(); 1942} 1943 1944template <class _Tp> 1945_LIBCPP_INLINE_VISIBILITY 1946_Tp 1947atomic_load(const atomic<_Tp>* __o) _NOEXCEPT 1948{ 1949 return __o->load(); 1950} 1951 1952// atomic_load_explicit 1953 1954template <class _Tp> 1955_LIBCPP_INLINE_VISIBILITY 1956_Tp 1957atomic_load_explicit(const volatile atomic<_Tp>* __o, memory_order __m) _NOEXCEPT 1958 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 1959{ 1960 return __o->load(__m); 1961} 1962 1963template <class _Tp> 1964_LIBCPP_INLINE_VISIBILITY 1965_Tp 1966atomic_load_explicit(const atomic<_Tp>* __o, memory_order __m) _NOEXCEPT 1967 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 1968{ 1969 return __o->load(__m); 1970} 1971 1972// atomic_exchange 1973 1974template <class _Tp> 1975_LIBCPP_INLINE_VISIBILITY 1976_Tp 1977atomic_exchange(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1978{ 1979 return __o->exchange(__d); 1980} 1981 1982template <class _Tp> 1983_LIBCPP_INLINE_VISIBILITY 1984_Tp 1985atomic_exchange(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1986{ 1987 return __o->exchange(__d); 1988} 1989 1990// atomic_exchange_explicit 1991 1992template <class _Tp> 1993_LIBCPP_INLINE_VISIBILITY 1994_Tp 1995atomic_exchange_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 1996{ 1997 return __o->exchange(__d, __m); 1998} 1999 2000template <class _Tp> 2001_LIBCPP_INLINE_VISIBILITY 2002_Tp 2003atomic_exchange_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 2004{ 2005 return __o->exchange(__d, __m); 2006} 2007 2008// atomic_compare_exchange_weak 2009 2010template <class _Tp> 2011_LIBCPP_INLINE_VISIBILITY 2012bool 2013atomic_compare_exchange_weak(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2014{ 2015 return __o->compare_exchange_weak(*__e, __d); 2016} 2017 2018template <class _Tp> 2019_LIBCPP_INLINE_VISIBILITY 2020bool 2021atomic_compare_exchange_weak(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2022{ 2023 return __o->compare_exchange_weak(*__e, __d); 2024} 2025 2026// atomic_compare_exchange_strong 2027 2028template <class _Tp> 2029_LIBCPP_INLINE_VISIBILITY 2030bool 2031atomic_compare_exchange_strong(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2032{ 2033 return __o->compare_exchange_strong(*__e, __d); 2034} 2035 2036template <class _Tp> 2037_LIBCPP_INLINE_VISIBILITY 2038bool 2039atomic_compare_exchange_strong(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2040{ 2041 return __o->compare_exchange_strong(*__e, __d); 2042} 2043 2044// atomic_compare_exchange_weak_explicit 2045 2046template <class _Tp> 2047_LIBCPP_INLINE_VISIBILITY 2048bool 2049atomic_compare_exchange_weak_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, 2050 typename atomic<_Tp>::value_type __d, 2051 memory_order __s, memory_order __f) _NOEXCEPT 2052 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2053{ 2054 return __o->compare_exchange_weak(*__e, __d, __s, __f); 2055} 2056 2057template <class _Tp> 2058_LIBCPP_INLINE_VISIBILITY 2059bool 2060atomic_compare_exchange_weak_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d, 2061 memory_order __s, memory_order __f) _NOEXCEPT 2062 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2063{ 2064 return __o->compare_exchange_weak(*__e, __d, __s, __f); 2065} 2066 2067// atomic_compare_exchange_strong_explicit 2068 2069template <class _Tp> 2070_LIBCPP_INLINE_VISIBILITY 2071bool 2072atomic_compare_exchange_strong_explicit(volatile atomic<_Tp>* __o, 2073 typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d, 2074 memory_order __s, memory_order __f) _NOEXCEPT 2075 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2076{ 2077 return __o->compare_exchange_strong(*__e, __d, __s, __f); 2078} 2079 2080template <class _Tp> 2081_LIBCPP_INLINE_VISIBILITY 2082bool 2083atomic_compare_exchange_strong_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, 2084 typename atomic<_Tp>::value_type __d, 2085 memory_order __s, memory_order __f) _NOEXCEPT 2086 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2087{ 2088 return __o->compare_exchange_strong(*__e, __d, __s, __f); 2089} 2090 2091// atomic_wait 2092 2093template <class _Tp> 2094_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2095void atomic_wait(const volatile atomic<_Tp>* __o, 2096 typename atomic<_Tp>::value_type __v) _NOEXCEPT 2097{ 2098 return __o->wait(__v); 2099} 2100 2101template <class _Tp> 2102_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2103void atomic_wait(const atomic<_Tp>* __o, 2104 typename atomic<_Tp>::value_type __v) _NOEXCEPT 2105{ 2106 return __o->wait(__v); 2107} 2108 2109// atomic_wait_explicit 2110 2111template <class _Tp> 2112_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2113void atomic_wait_explicit(const volatile atomic<_Tp>* __o, 2114 typename atomic<_Tp>::value_type __v, 2115 memory_order __m) _NOEXCEPT 2116 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 2117{ 2118 return __o->wait(__v, __m); 2119} 2120 2121template <class _Tp> 2122_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2123void atomic_wait_explicit(const atomic<_Tp>* __o, 2124 typename atomic<_Tp>::value_type __v, 2125 memory_order __m) _NOEXCEPT 2126 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 2127{ 2128 return __o->wait(__v, __m); 2129} 2130 2131// atomic_notify_one 2132 2133template <class _Tp> 2134_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2135void atomic_notify_one(volatile atomic<_Tp>* __o) _NOEXCEPT 2136{ 2137 __o->notify_one(); 2138} 2139template <class _Tp> 2140_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2141void atomic_notify_one(atomic<_Tp>* __o) _NOEXCEPT 2142{ 2143 __o->notify_one(); 2144} 2145 2146// atomic_notify_one 2147 2148template <class _Tp> 2149_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2150void atomic_notify_all(volatile atomic<_Tp>* __o) _NOEXCEPT 2151{ 2152 __o->notify_all(); 2153} 2154template <class _Tp> 2155_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2156void atomic_notify_all(atomic<_Tp>* __o) _NOEXCEPT 2157{ 2158 __o->notify_all(); 2159} 2160 2161// atomic_fetch_add 2162 2163template <class _Tp> 2164_LIBCPP_INLINE_VISIBILITY 2165typename enable_if 2166< 2167 is_integral<_Tp>::value && !is_same<_Tp, bool>::value && !is_const<_Tp>::value, 2168 _Tp 2169>::type 2170atomic_fetch_add(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT 2171{ 2172 return __o->fetch_add(__op); 2173} 2174 2175template <class _Tp> 2176_LIBCPP_INLINE_VISIBILITY 2177typename enable_if 2178< 2179 is_integral<_Tp>::value && !is_same<_Tp, bool>::value && !is_const<_Tp>::value, 2180 _Tp 2181>::type 2182atomic_fetch_add(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT 2183{ 2184 return __o->fetch_add(__op); 2185} 2186 2187template <class _Tp> 2188_LIBCPP_INLINE_VISIBILITY 2189_Tp* 2190atomic_fetch_add(volatile atomic<_Tp*>* __o, typename atomic<_Tp*>::difference_type __op) _NOEXCEPT 2191{ 2192 return __o->fetch_add(__op); 2193} 2194 2195template <class _Tp> 2196_LIBCPP_INLINE_VISIBILITY 2197_Tp* 2198atomic_fetch_add(atomic<_Tp*>* __o, typename atomic<_Tp*>::difference_type __op) _NOEXCEPT 2199{ 2200 return __o->fetch_add(__op); 2201} 2202 2203// atomic_fetch_add_explicit 2204 2205template <class _Tp> 2206_LIBCPP_INLINE_VISIBILITY 2207typename enable_if 2208< 2209 is_integral<_Tp>::value && !is_same<_Tp, bool>::value && !is_const<_Tp>::value, 2210 _Tp 2211>::type 2212atomic_fetch_add_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT 2213{ 2214 return __o->fetch_add(__op, __m); 2215} 2216 2217template <class _Tp> 2218_LIBCPP_INLINE_VISIBILITY 2219typename enable_if 2220< 2221 is_integral<_Tp>::value && !is_same<_Tp, bool>::value && !is_const<_Tp>::value, 2222 _Tp 2223>::type 2224atomic_fetch_add_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT 2225{ 2226 return __o->fetch_add(__op, __m); 2227} 2228 2229template <class _Tp> 2230_LIBCPP_INLINE_VISIBILITY 2231_Tp* 2232atomic_fetch_add_explicit(volatile atomic<_Tp*>* __o, typename atomic<_Tp*>::difference_type __op, memory_order __m) _NOEXCEPT 2233{ 2234 return __o->fetch_add(__op, __m); 2235} 2236 2237template <class _Tp> 2238_LIBCPP_INLINE_VISIBILITY 2239_Tp* 2240atomic_fetch_add_explicit(atomic<_Tp*>* __o, typename atomic<_Tp*>::difference_type __op, memory_order __m) _NOEXCEPT 2241{ 2242 return __o->fetch_add(__op, __m); 2243} 2244 2245// atomic_fetch_sub 2246 2247template <class _Tp> 2248_LIBCPP_INLINE_VISIBILITY 2249typename enable_if 2250< 2251 is_integral<_Tp>::value && !is_same<_Tp, bool>::value && !is_const<_Tp>::value, 2252 _Tp 2253>::type 2254atomic_fetch_sub(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT 2255{ 2256 return __o->fetch_sub(__op); 2257} 2258 2259template <class _Tp> 2260_LIBCPP_INLINE_VISIBILITY 2261typename enable_if 2262< 2263 is_integral<_Tp>::value && !is_same<_Tp, bool>::value && !is_const<_Tp>::value, 2264 _Tp 2265>::type 2266atomic_fetch_sub(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT 2267{ 2268 return __o->fetch_sub(__op); 2269} 2270 2271template <class _Tp> 2272_LIBCPP_INLINE_VISIBILITY 2273_Tp* 2274atomic_fetch_sub(volatile atomic<_Tp*>* __o, typename atomic<_Tp*>::difference_type __op) _NOEXCEPT 2275{ 2276 return __o->fetch_sub(__op); 2277} 2278 2279template <class _Tp> 2280_LIBCPP_INLINE_VISIBILITY 2281_Tp* 2282atomic_fetch_sub(atomic<_Tp*>* __o, typename atomic<_Tp*>::difference_type __op) _NOEXCEPT 2283{ 2284 return __o->fetch_sub(__op); 2285} 2286 2287// atomic_fetch_sub_explicit 2288 2289template <class _Tp> 2290_LIBCPP_INLINE_VISIBILITY 2291typename enable_if 2292< 2293 is_integral<_Tp>::value && !is_same<_Tp, bool>::value && !is_const<_Tp>::value, 2294 _Tp 2295>::type 2296atomic_fetch_sub_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT 2297{ 2298 return __o->fetch_sub(__op, __m); 2299} 2300 2301template <class _Tp> 2302_LIBCPP_INLINE_VISIBILITY 2303typename enable_if 2304< 2305 is_integral<_Tp>::value && !is_same<_Tp, bool>::value && !is_const<_Tp>::value, 2306 _Tp 2307>::type 2308atomic_fetch_sub_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT 2309{ 2310 return __o->fetch_sub(__op, __m); 2311} 2312 2313template <class _Tp> 2314_LIBCPP_INLINE_VISIBILITY 2315_Tp* 2316atomic_fetch_sub_explicit(volatile atomic<_Tp*>* __o, typename atomic<_Tp*>::difference_type __op, memory_order __m) _NOEXCEPT 2317{ 2318 return __o->fetch_sub(__op, __m); 2319} 2320 2321template <class _Tp> 2322_LIBCPP_INLINE_VISIBILITY 2323_Tp* 2324atomic_fetch_sub_explicit(atomic<_Tp*>* __o, typename atomic<_Tp*>::difference_type __op, memory_order __m) _NOEXCEPT 2325{ 2326 return __o->fetch_sub(__op, __m); 2327} 2328 2329// atomic_fetch_and 2330 2331template <class _Tp> 2332_LIBCPP_INLINE_VISIBILITY 2333typename enable_if 2334< 2335 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2336 _Tp 2337>::type 2338atomic_fetch_and(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2339{ 2340 return __o->fetch_and(__op); 2341} 2342 2343template <class _Tp> 2344_LIBCPP_INLINE_VISIBILITY 2345typename enable_if 2346< 2347 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2348 _Tp 2349>::type 2350atomic_fetch_and(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2351{ 2352 return __o->fetch_and(__op); 2353} 2354 2355// atomic_fetch_and_explicit 2356 2357template <class _Tp> 2358_LIBCPP_INLINE_VISIBILITY 2359typename enable_if 2360< 2361 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2362 _Tp 2363>::type 2364atomic_fetch_and_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2365{ 2366 return __o->fetch_and(__op, __m); 2367} 2368 2369template <class _Tp> 2370_LIBCPP_INLINE_VISIBILITY 2371typename enable_if 2372< 2373 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2374 _Tp 2375>::type 2376atomic_fetch_and_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2377{ 2378 return __o->fetch_and(__op, __m); 2379} 2380 2381// atomic_fetch_or 2382 2383template <class _Tp> 2384_LIBCPP_INLINE_VISIBILITY 2385typename enable_if 2386< 2387 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2388 _Tp 2389>::type 2390atomic_fetch_or(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2391{ 2392 return __o->fetch_or(__op); 2393} 2394 2395template <class _Tp> 2396_LIBCPP_INLINE_VISIBILITY 2397typename enable_if 2398< 2399 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2400 _Tp 2401>::type 2402atomic_fetch_or(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2403{ 2404 return __o->fetch_or(__op); 2405} 2406 2407// atomic_fetch_or_explicit 2408 2409template <class _Tp> 2410_LIBCPP_INLINE_VISIBILITY 2411typename enable_if 2412< 2413 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2414 _Tp 2415>::type 2416atomic_fetch_or_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2417{ 2418 return __o->fetch_or(__op, __m); 2419} 2420 2421template <class _Tp> 2422_LIBCPP_INLINE_VISIBILITY 2423typename enable_if 2424< 2425 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2426 _Tp 2427>::type 2428atomic_fetch_or_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2429{ 2430 return __o->fetch_or(__op, __m); 2431} 2432 2433// atomic_fetch_xor 2434 2435template <class _Tp> 2436_LIBCPP_INLINE_VISIBILITY 2437typename enable_if 2438< 2439 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2440 _Tp 2441>::type 2442atomic_fetch_xor(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2443{ 2444 return __o->fetch_xor(__op); 2445} 2446 2447template <class _Tp> 2448_LIBCPP_INLINE_VISIBILITY 2449typename enable_if 2450< 2451 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2452 _Tp 2453>::type 2454atomic_fetch_xor(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2455{ 2456 return __o->fetch_xor(__op); 2457} 2458 2459// atomic_fetch_xor_explicit 2460 2461template <class _Tp> 2462_LIBCPP_INLINE_VISIBILITY 2463typename enable_if 2464< 2465 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2466 _Tp 2467>::type 2468atomic_fetch_xor_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2469{ 2470 return __o->fetch_xor(__op, __m); 2471} 2472 2473template <class _Tp> 2474_LIBCPP_INLINE_VISIBILITY 2475typename enable_if 2476< 2477 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2478 _Tp 2479>::type 2480atomic_fetch_xor_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2481{ 2482 return __o->fetch_xor(__op, __m); 2483} 2484 2485// flag type and operations 2486 2487typedef struct atomic_flag 2488{ 2489 __cxx_atomic_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_; 2490 2491 _LIBCPP_INLINE_VISIBILITY 2492 bool test(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 2493 {return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m);} 2494 _LIBCPP_INLINE_VISIBILITY 2495 bool test(memory_order __m = memory_order_seq_cst) const _NOEXCEPT 2496 {return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m);} 2497 2498 _LIBCPP_INLINE_VISIBILITY 2499 bool test_and_set(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 2500 {return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m);} 2501 _LIBCPP_INLINE_VISIBILITY 2502 bool test_and_set(memory_order __m = memory_order_seq_cst) _NOEXCEPT 2503 {return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m);} 2504 _LIBCPP_INLINE_VISIBILITY 2505 void clear(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 2506 {__cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m);} 2507 _LIBCPP_INLINE_VISIBILITY 2508 void clear(memory_order __m = memory_order_seq_cst) _NOEXCEPT 2509 {__cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m);} 2510 2511 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2512 void wait(bool __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 2513 {__cxx_atomic_wait(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);} 2514 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2515 void wait(bool __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT 2516 {__cxx_atomic_wait(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);} 2517 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2518 void notify_one() volatile _NOEXCEPT 2519 {__cxx_atomic_notify_one(&__a_);} 2520 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2521 void notify_one() _NOEXCEPT 2522 {__cxx_atomic_notify_one(&__a_);} 2523 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2524 void notify_all() volatile _NOEXCEPT 2525 {__cxx_atomic_notify_all(&__a_);} 2526 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2527 void notify_all() _NOEXCEPT 2528 {__cxx_atomic_notify_all(&__a_);} 2529 2530 _LIBCPP_INLINE_VISIBILITY 2531 atomic_flag() _NOEXCEPT _LIBCPP_DEFAULT 2532 2533 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR 2534 atomic_flag(bool __b) _NOEXCEPT : __a_(__b) {} // EXTENSION 2535 2536#ifndef _LIBCPP_CXX03_LANG 2537 atomic_flag(const atomic_flag&) = delete; 2538 atomic_flag& operator=(const atomic_flag&) = delete; 2539 atomic_flag& operator=(const atomic_flag&) volatile = delete; 2540#else 2541private: 2542 _LIBCPP_INLINE_VISIBILITY 2543 atomic_flag(const atomic_flag&); 2544 _LIBCPP_INLINE_VISIBILITY 2545 atomic_flag& operator=(const atomic_flag&); 2546 _LIBCPP_INLINE_VISIBILITY 2547 atomic_flag& operator=(const atomic_flag&) volatile; 2548#endif 2549} atomic_flag; 2550 2551 2552inline _LIBCPP_INLINE_VISIBILITY 2553bool 2554atomic_flag_test(const volatile atomic_flag* __o) _NOEXCEPT 2555{ 2556 return __o->test(); 2557} 2558 2559inline _LIBCPP_INLINE_VISIBILITY 2560bool 2561atomic_flag_test(const atomic_flag* __o) _NOEXCEPT 2562{ 2563 return __o->test(); 2564} 2565 2566inline _LIBCPP_INLINE_VISIBILITY 2567bool 2568atomic_flag_test_explicit(const volatile atomic_flag* __o, memory_order __m) _NOEXCEPT 2569{ 2570 return __o->test(__m); 2571} 2572 2573inline _LIBCPP_INLINE_VISIBILITY 2574bool 2575atomic_flag_test_explicit(const atomic_flag* __o, memory_order __m) _NOEXCEPT 2576{ 2577 return __o->test(__m); 2578} 2579 2580inline _LIBCPP_INLINE_VISIBILITY 2581bool 2582atomic_flag_test_and_set(volatile atomic_flag* __o) _NOEXCEPT 2583{ 2584 return __o->test_and_set(); 2585} 2586 2587inline _LIBCPP_INLINE_VISIBILITY 2588bool 2589atomic_flag_test_and_set(atomic_flag* __o) _NOEXCEPT 2590{ 2591 return __o->test_and_set(); 2592} 2593 2594inline _LIBCPP_INLINE_VISIBILITY 2595bool 2596atomic_flag_test_and_set_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT 2597{ 2598 return __o->test_and_set(__m); 2599} 2600 2601inline _LIBCPP_INLINE_VISIBILITY 2602bool 2603atomic_flag_test_and_set_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT 2604{ 2605 return __o->test_and_set(__m); 2606} 2607 2608inline _LIBCPP_INLINE_VISIBILITY 2609void 2610atomic_flag_clear(volatile atomic_flag* __o) _NOEXCEPT 2611{ 2612 __o->clear(); 2613} 2614 2615inline _LIBCPP_INLINE_VISIBILITY 2616void 2617atomic_flag_clear(atomic_flag* __o) _NOEXCEPT 2618{ 2619 __o->clear(); 2620} 2621 2622inline _LIBCPP_INLINE_VISIBILITY 2623void 2624atomic_flag_clear_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT 2625{ 2626 __o->clear(__m); 2627} 2628 2629inline _LIBCPP_INLINE_VISIBILITY 2630void 2631atomic_flag_clear_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT 2632{ 2633 __o->clear(__m); 2634} 2635 2636inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2637void 2638atomic_flag_wait(const volatile atomic_flag* __o, bool __v) _NOEXCEPT 2639{ 2640 __o->wait(__v); 2641} 2642 2643inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2644void 2645atomic_flag_wait(const atomic_flag* __o, bool __v) _NOEXCEPT 2646{ 2647 __o->wait(__v); 2648} 2649 2650inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2651void 2652atomic_flag_wait_explicit(const volatile atomic_flag* __o, 2653 bool __v, memory_order __m) _NOEXCEPT 2654{ 2655 __o->wait(__v, __m); 2656} 2657 2658inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2659void 2660atomic_flag_wait_explicit(const atomic_flag* __o, 2661 bool __v, memory_order __m) _NOEXCEPT 2662{ 2663 __o->wait(__v, __m); 2664} 2665 2666inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2667void 2668atomic_flag_notify_one(volatile atomic_flag* __o) _NOEXCEPT 2669{ 2670 __o->notify_one(); 2671} 2672 2673inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2674void 2675atomic_flag_notify_one(atomic_flag* __o) _NOEXCEPT 2676{ 2677 __o->notify_one(); 2678} 2679 2680inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2681void 2682atomic_flag_notify_all(volatile atomic_flag* __o) _NOEXCEPT 2683{ 2684 __o->notify_all(); 2685} 2686 2687inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2688void 2689atomic_flag_notify_all(atomic_flag* __o) _NOEXCEPT 2690{ 2691 __o->notify_all(); 2692} 2693 2694// fences 2695 2696inline _LIBCPP_INLINE_VISIBILITY 2697void 2698atomic_thread_fence(memory_order __m) _NOEXCEPT 2699{ 2700 __cxx_atomic_thread_fence(__m); 2701} 2702 2703inline _LIBCPP_INLINE_VISIBILITY 2704void 2705atomic_signal_fence(memory_order __m) _NOEXCEPT 2706{ 2707 __cxx_atomic_signal_fence(__m); 2708} 2709 2710// Atomics for standard typedef types 2711 2712typedef atomic<bool> atomic_bool; 2713typedef atomic<char> atomic_char; 2714typedef atomic<signed char> atomic_schar; 2715typedef atomic<unsigned char> atomic_uchar; 2716typedef atomic<short> atomic_short; 2717typedef atomic<unsigned short> atomic_ushort; 2718typedef atomic<int> atomic_int; 2719typedef atomic<unsigned int> atomic_uint; 2720typedef atomic<long> atomic_long; 2721typedef atomic<unsigned long> atomic_ulong; 2722typedef atomic<long long> atomic_llong; 2723typedef atomic<unsigned long long> atomic_ullong; 2724typedef atomic<char16_t> atomic_char16_t; 2725typedef atomic<char32_t> atomic_char32_t; 2726typedef atomic<wchar_t> atomic_wchar_t; 2727 2728typedef atomic<int_least8_t> atomic_int_least8_t; 2729typedef atomic<uint_least8_t> atomic_uint_least8_t; 2730typedef atomic<int_least16_t> atomic_int_least16_t; 2731typedef atomic<uint_least16_t> atomic_uint_least16_t; 2732typedef atomic<int_least32_t> atomic_int_least32_t; 2733typedef atomic<uint_least32_t> atomic_uint_least32_t; 2734typedef atomic<int_least64_t> atomic_int_least64_t; 2735typedef atomic<uint_least64_t> atomic_uint_least64_t; 2736 2737typedef atomic<int_fast8_t> atomic_int_fast8_t; 2738typedef atomic<uint_fast8_t> atomic_uint_fast8_t; 2739typedef atomic<int_fast16_t> atomic_int_fast16_t; 2740typedef atomic<uint_fast16_t> atomic_uint_fast16_t; 2741typedef atomic<int_fast32_t> atomic_int_fast32_t; 2742typedef atomic<uint_fast32_t> atomic_uint_fast32_t; 2743typedef atomic<int_fast64_t> atomic_int_fast64_t; 2744typedef atomic<uint_fast64_t> atomic_uint_fast64_t; 2745 2746typedef atomic< int8_t> atomic_int8_t; 2747typedef atomic<uint8_t> atomic_uint8_t; 2748typedef atomic< int16_t> atomic_int16_t; 2749typedef atomic<uint16_t> atomic_uint16_t; 2750typedef atomic< int32_t> atomic_int32_t; 2751typedef atomic<uint32_t> atomic_uint32_t; 2752typedef atomic< int64_t> atomic_int64_t; 2753typedef atomic<uint64_t> atomic_uint64_t; 2754 2755typedef atomic<intptr_t> atomic_intptr_t; 2756typedef atomic<uintptr_t> atomic_uintptr_t; 2757typedef atomic<size_t> atomic_size_t; 2758typedef atomic<ptrdiff_t> atomic_ptrdiff_t; 2759typedef atomic<intmax_t> atomic_intmax_t; 2760typedef atomic<uintmax_t> atomic_uintmax_t; 2761 2762// atomic_*_lock_free : prefer the contention type most highly, then the largest lock-free type 2763 2764#ifdef __cpp_lib_atomic_is_always_lock_free 2765# define _LIBCPP_CONTENTION_LOCK_FREE __atomic_always_lock_free(sizeof(__cxx_contention_t), 0) 2766#else 2767# define _LIBCPP_CONTENTION_LOCK_FREE false 2768#endif 2769 2770#if ATOMIC_LLONG_LOCK_FREE == 2 2771typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, long long>::type __libcpp_signed_lock_free; 2772typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned long long>::type __libcpp_unsigned_lock_free; 2773#elif ATOMIC_INT_LOCK_FREE == 2 2774typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, int>::type __libcpp_signed_lock_free; 2775typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned int>::type __libcpp_unsigned_lock_free; 2776#elif ATOMIC_SHORT_LOCK_FREE == 2 2777typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, short>::type __libcpp_signed_lock_free; 2778typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned short>::type __libcpp_unsigned_lock_free; 2779#elif ATOMIC_CHAR_LOCK_FREE == 2 2780typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, char>::type __libcpp_signed_lock_free; 2781typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned char>::type __libcpp_unsigned_lock_free; 2782#else 2783 // No signed/unsigned lock-free types 2784#endif 2785 2786typedef atomic<__libcpp_signed_lock_free> atomic_signed_lock_free; 2787typedef atomic<__libcpp_unsigned_lock_free> atomic_unsigned_lock_free; 2788 2789#define ATOMIC_FLAG_INIT {false} 2790#define ATOMIC_VAR_INIT(__v) {__v} 2791 2792_LIBCPP_END_NAMESPACE_STD 2793 2794#endif // _LIBCPP_ATOMIC 2795