1// -*- C++ -*- 2//===--------------------------- atomic -----------------------------------===// 3// 4// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 5// See https://llvm.org/LICENSE.txt for license information. 6// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 7// 8//===----------------------------------------------------------------------===// 9 10#ifndef _LIBCPP_ATOMIC 11#define _LIBCPP_ATOMIC 12 13/* 14 atomic synopsis 15 16namespace std 17{ 18 19// feature test macro [version.syn] 20 21#define __cpp_lib_atomic_is_always_lock_free 22#define __cpp_lib_atomic_flag_test 23#define __cpp_lib_atomic_lock_free_type_aliases 24#define __cpp_lib_atomic_wait 25 26 // order and consistency 27 28 enum memory_order: unspecified // enum class in C++20 29 { 30 relaxed, 31 consume, // load-consume 32 acquire, // load-acquire 33 release, // store-release 34 acq_rel, // store-release load-acquire 35 seq_cst // store-release load-acquire 36 }; 37 38 inline constexpr auto memory_order_relaxed = memory_order::relaxed; 39 inline constexpr auto memory_order_consume = memory_order::consume; 40 inline constexpr auto memory_order_acquire = memory_order::acquire; 41 inline constexpr auto memory_order_release = memory_order::release; 42 inline constexpr auto memory_order_acq_rel = memory_order::acq_rel; 43 inline constexpr auto memory_order_seq_cst = memory_order::seq_cst; 44 45template <class T> T kill_dependency(T y) noexcept; 46 47// lock-free property 48 49#define ATOMIC_BOOL_LOCK_FREE unspecified 50#define ATOMIC_CHAR_LOCK_FREE unspecified 51#define ATOMIC_CHAR8_T_LOCK_FREE unspecified // C++20 52#define ATOMIC_CHAR16_T_LOCK_FREE unspecified 53#define ATOMIC_CHAR32_T_LOCK_FREE unspecified 54#define ATOMIC_WCHAR_T_LOCK_FREE unspecified 55#define ATOMIC_SHORT_LOCK_FREE unspecified 56#define ATOMIC_INT_LOCK_FREE unspecified 57#define ATOMIC_LONG_LOCK_FREE unspecified 58#define ATOMIC_LLONG_LOCK_FREE unspecified 59#define ATOMIC_POINTER_LOCK_FREE unspecified 60 61template <class T> 62struct atomic 63{ 64 using value_type = T; 65 66 static constexpr bool is_always_lock_free; 67 bool is_lock_free() const volatile noexcept; 68 bool is_lock_free() const noexcept; 69 70 atomic() noexcept = default; 71 constexpr atomic(T desr) noexcept; 72 atomic(const atomic&) = delete; 73 atomic& operator=(const atomic&) = delete; 74 atomic& operator=(const atomic&) volatile = delete; 75 76 T load(memory_order m = memory_order_seq_cst) const volatile noexcept; 77 T load(memory_order m = memory_order_seq_cst) const noexcept; 78 operator T() const volatile noexcept; 79 operator T() const noexcept; 80 void store(T desr, memory_order m = memory_order_seq_cst) volatile noexcept; 81 void store(T desr, memory_order m = memory_order_seq_cst) noexcept; 82 T operator=(T) volatile noexcept; 83 T operator=(T) noexcept; 84 85 T exchange(T desr, memory_order m = memory_order_seq_cst) volatile noexcept; 86 T exchange(T desr, memory_order m = memory_order_seq_cst) noexcept; 87 bool compare_exchange_weak(T& expc, T desr, 88 memory_order s, memory_order f) volatile noexcept; 89 bool compare_exchange_weak(T& expc, T desr, memory_order s, memory_order f) noexcept; 90 bool compare_exchange_strong(T& expc, T desr, 91 memory_order s, memory_order f) volatile noexcept; 92 bool compare_exchange_strong(T& expc, T desr, 93 memory_order s, memory_order f) noexcept; 94 bool compare_exchange_weak(T& expc, T desr, 95 memory_order m = memory_order_seq_cst) volatile noexcept; 96 bool compare_exchange_weak(T& expc, T desr, 97 memory_order m = memory_order_seq_cst) noexcept; 98 bool compare_exchange_strong(T& expc, T desr, 99 memory_order m = memory_order_seq_cst) volatile noexcept; 100 bool compare_exchange_strong(T& expc, T desr, 101 memory_order m = memory_order_seq_cst) noexcept; 102 103 void wait(T, memory_order = memory_order::seq_cst) const volatile noexcept; 104 void wait(T, memory_order = memory_order::seq_cst) const noexcept; 105 void notify_one() volatile noexcept; 106 void notify_one() noexcept; 107 void notify_all() volatile noexcept; 108 void notify_all() noexcept; 109}; 110 111template <> 112struct atomic<integral> 113{ 114 using value_type = integral; 115 using difference_type = value_type; 116 117 static constexpr bool is_always_lock_free; 118 bool is_lock_free() const volatile noexcept; 119 bool is_lock_free() const noexcept; 120 121 atomic() noexcept = default; 122 constexpr atomic(integral desr) noexcept; 123 atomic(const atomic&) = delete; 124 atomic& operator=(const atomic&) = delete; 125 atomic& operator=(const atomic&) volatile = delete; 126 127 integral load(memory_order m = memory_order_seq_cst) const volatile noexcept; 128 integral load(memory_order m = memory_order_seq_cst) const noexcept; 129 operator integral() const volatile noexcept; 130 operator integral() const noexcept; 131 void store(integral desr, memory_order m = memory_order_seq_cst) volatile noexcept; 132 void store(integral desr, memory_order m = memory_order_seq_cst) noexcept; 133 integral operator=(integral desr) volatile noexcept; 134 integral operator=(integral desr) noexcept; 135 136 integral exchange(integral desr, 137 memory_order m = memory_order_seq_cst) volatile noexcept; 138 integral exchange(integral desr, memory_order m = memory_order_seq_cst) noexcept; 139 bool compare_exchange_weak(integral& expc, integral desr, 140 memory_order s, memory_order f) volatile noexcept; 141 bool compare_exchange_weak(integral& expc, integral desr, 142 memory_order s, memory_order f) noexcept; 143 bool compare_exchange_strong(integral& expc, integral desr, 144 memory_order s, memory_order f) volatile noexcept; 145 bool compare_exchange_strong(integral& expc, integral desr, 146 memory_order s, memory_order f) noexcept; 147 bool compare_exchange_weak(integral& expc, integral desr, 148 memory_order m = memory_order_seq_cst) volatile noexcept; 149 bool compare_exchange_weak(integral& expc, integral desr, 150 memory_order m = memory_order_seq_cst) noexcept; 151 bool compare_exchange_strong(integral& expc, integral desr, 152 memory_order m = memory_order_seq_cst) volatile noexcept; 153 bool compare_exchange_strong(integral& expc, integral desr, 154 memory_order m = memory_order_seq_cst) noexcept; 155 156 integral fetch_add(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 157 integral fetch_add(integral op, memory_order m = memory_order_seq_cst) noexcept; 158 integral fetch_sub(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 159 integral fetch_sub(integral op, memory_order m = memory_order_seq_cst) noexcept; 160 integral fetch_and(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 161 integral fetch_and(integral op, memory_order m = memory_order_seq_cst) noexcept; 162 integral fetch_or(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 163 integral fetch_or(integral op, memory_order m = memory_order_seq_cst) noexcept; 164 integral fetch_xor(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 165 integral fetch_xor(integral op, memory_order m = memory_order_seq_cst) noexcept; 166 167 integral operator++(int) volatile noexcept; 168 integral operator++(int) noexcept; 169 integral operator--(int) volatile noexcept; 170 integral operator--(int) noexcept; 171 integral operator++() volatile noexcept; 172 integral operator++() noexcept; 173 integral operator--() volatile noexcept; 174 integral operator--() noexcept; 175 integral operator+=(integral op) volatile noexcept; 176 integral operator+=(integral op) noexcept; 177 integral operator-=(integral op) volatile noexcept; 178 integral operator-=(integral op) noexcept; 179 integral operator&=(integral op) volatile noexcept; 180 integral operator&=(integral op) noexcept; 181 integral operator|=(integral op) volatile noexcept; 182 integral operator|=(integral op) noexcept; 183 integral operator^=(integral op) volatile noexcept; 184 integral operator^=(integral op) noexcept; 185 186 void wait(integral, memory_order = memory_order::seq_cst) const volatile noexcept; 187 void wait(integral, memory_order = memory_order::seq_cst) const noexcept; 188 void notify_one() volatile noexcept; 189 void notify_one() noexcept; 190 void notify_all() volatile noexcept; 191 void notify_all() noexcept; 192}; 193 194template <class T> 195struct atomic<T*> 196{ 197 using value_type = T*; 198 using difference_type = ptrdiff_t; 199 200 static constexpr bool is_always_lock_free; 201 bool is_lock_free() const volatile noexcept; 202 bool is_lock_free() const noexcept; 203 204 atomic() noexcept = default; 205 constexpr atomic(T* desr) noexcept; 206 atomic(const atomic&) = delete; 207 atomic& operator=(const atomic&) = delete; 208 atomic& operator=(const atomic&) volatile = delete; 209 210 T* load(memory_order m = memory_order_seq_cst) const volatile noexcept; 211 T* load(memory_order m = memory_order_seq_cst) const noexcept; 212 operator T*() const volatile noexcept; 213 operator T*() const noexcept; 214 void store(T* desr, memory_order m = memory_order_seq_cst) volatile noexcept; 215 void store(T* desr, memory_order m = memory_order_seq_cst) noexcept; 216 T* operator=(T*) volatile noexcept; 217 T* operator=(T*) noexcept; 218 219 T* exchange(T* desr, memory_order m = memory_order_seq_cst) volatile noexcept; 220 T* exchange(T* desr, memory_order m = memory_order_seq_cst) noexcept; 221 bool compare_exchange_weak(T*& expc, T* desr, 222 memory_order s, memory_order f) volatile noexcept; 223 bool compare_exchange_weak(T*& expc, T* desr, 224 memory_order s, memory_order f) noexcept; 225 bool compare_exchange_strong(T*& expc, T* desr, 226 memory_order s, memory_order f) volatile noexcept; 227 bool compare_exchange_strong(T*& expc, T* desr, 228 memory_order s, memory_order f) noexcept; 229 bool compare_exchange_weak(T*& expc, T* desr, 230 memory_order m = memory_order_seq_cst) volatile noexcept; 231 bool compare_exchange_weak(T*& expc, T* desr, 232 memory_order m = memory_order_seq_cst) noexcept; 233 bool compare_exchange_strong(T*& expc, T* desr, 234 memory_order m = memory_order_seq_cst) volatile noexcept; 235 bool compare_exchange_strong(T*& expc, T* desr, 236 memory_order m = memory_order_seq_cst) noexcept; 237 T* fetch_add(ptrdiff_t op, memory_order m = memory_order_seq_cst) volatile noexcept; 238 T* fetch_add(ptrdiff_t op, memory_order m = memory_order_seq_cst) noexcept; 239 T* fetch_sub(ptrdiff_t op, memory_order m = memory_order_seq_cst) volatile noexcept; 240 T* fetch_sub(ptrdiff_t op, memory_order m = memory_order_seq_cst) noexcept; 241 242 T* operator++(int) volatile noexcept; 243 T* operator++(int) noexcept; 244 T* operator--(int) volatile noexcept; 245 T* operator--(int) noexcept; 246 T* operator++() volatile noexcept; 247 T* operator++() noexcept; 248 T* operator--() volatile noexcept; 249 T* operator--() noexcept; 250 T* operator+=(ptrdiff_t op) volatile noexcept; 251 T* operator+=(ptrdiff_t op) noexcept; 252 T* operator-=(ptrdiff_t op) volatile noexcept; 253 T* operator-=(ptrdiff_t op) noexcept; 254 255 void wait(T*, memory_order = memory_order::seq_cst) const volatile noexcept; 256 void wait(T*, memory_order = memory_order::seq_cst) const noexcept; 257 void notify_one() volatile noexcept; 258 void notify_one() noexcept; 259 void notify_all() volatile noexcept; 260 void notify_all() noexcept; 261}; 262 263 264template <class T> 265 bool atomic_is_lock_free(const volatile atomic<T>* obj) noexcept; 266 267template <class T> 268 bool atomic_is_lock_free(const atomic<T>* obj) noexcept; 269 270template <class T> 271 void atomic_store(volatile atomic<T>* obj, T desr) noexcept; 272 273template <class T> 274 void atomic_store(atomic<T>* obj, T desr) noexcept; 275 276template <class T> 277 void atomic_store_explicit(volatile atomic<T>* obj, T desr, memory_order m) noexcept; 278 279template <class T> 280 void atomic_store_explicit(atomic<T>* obj, T desr, memory_order m) noexcept; 281 282template <class T> 283 T atomic_load(const volatile atomic<T>* obj) noexcept; 284 285template <class T> 286 T atomic_load(const atomic<T>* obj) noexcept; 287 288template <class T> 289 T atomic_load_explicit(const volatile atomic<T>* obj, memory_order m) noexcept; 290 291template <class T> 292 T atomic_load_explicit(const atomic<T>* obj, memory_order m) noexcept; 293 294template <class T> 295 T atomic_exchange(volatile atomic<T>* obj, T desr) noexcept; 296 297template <class T> 298 T atomic_exchange(atomic<T>* obj, T desr) noexcept; 299 300template <class T> 301 T atomic_exchange_explicit(volatile atomic<T>* obj, T desr, memory_order m) noexcept; 302 303template <class T> 304 T atomic_exchange_explicit(atomic<T>* obj, T desr, memory_order m) noexcept; 305 306template <class T> 307 bool atomic_compare_exchange_weak(volatile atomic<T>* obj, T* expc, T desr) noexcept; 308 309template <class T> 310 bool atomic_compare_exchange_weak(atomic<T>* obj, T* expc, T desr) noexcept; 311 312template <class T> 313 bool atomic_compare_exchange_strong(volatile atomic<T>* obj, T* expc, T desr) noexcept; 314 315template <class T> 316 bool atomic_compare_exchange_strong(atomic<T>* obj, T* expc, T desr) noexcept; 317 318template <class T> 319 bool atomic_compare_exchange_weak_explicit(volatile atomic<T>* obj, T* expc, 320 T desr, 321 memory_order s, memory_order f) noexcept; 322 323template <class T> 324 bool atomic_compare_exchange_weak_explicit(atomic<T>* obj, T* expc, T desr, 325 memory_order s, memory_order f) noexcept; 326 327template <class T> 328 bool atomic_compare_exchange_strong_explicit(volatile atomic<T>* obj, 329 T* expc, T desr, 330 memory_order s, memory_order f) noexcept; 331 332template <class T> 333 bool atomic_compare_exchange_strong_explicit(atomic<T>* obj, T* expc, 334 T desr, 335 memory_order s, memory_order f) noexcept; 336 337template <class T> 338 void atomic_wait(const volatile atomic<T>* obj, T old) noexcept; 339 340template <class T> 341 void atomic_wait(const atomic<T>* obj, T old) noexcept; 342 343template <class T> 344 void atomic_wait_explicit(const volatile atomic<T>* obj, T old, memory_order m) noexcept; 345 346template <class T> 347 void atomic_wait_explicit(const atomic<T>* obj, T old, memory_order m) noexcept; 348 349template <class T> 350 void atomic_one(volatile atomic<T>* obj) noexcept; 351 352template <class T> 353 void atomic_one(atomic<T>* obj) noexcept; 354 355template <class T> 356 void atomic_all(volatile atomic<T>* obj) noexcept; 357 358template <class T> 359 void atomic_all(atomic<T>* obj) noexcept; 360 361template <class Integral> 362 Integral atomic_fetch_add(volatile atomic<Integral>* obj, Integral op) noexcept; 363 364template <class Integral> 365 Integral atomic_fetch_add(atomic<Integral>* obj, Integral op) noexcept; 366 367template <class Integral> 368 Integral atomic_fetch_add_explicit(volatile atomic<Integral>* obj, Integral op, 369 memory_order m) noexcept; 370template <class Integral> 371 Integral atomic_fetch_add_explicit(atomic<Integral>* obj, Integral op, 372 memory_order m) noexcept; 373template <class Integral> 374 Integral atomic_fetch_sub(volatile atomic<Integral>* obj, Integral op) noexcept; 375 376template <class Integral> 377 Integral atomic_fetch_sub(atomic<Integral>* obj, Integral op) noexcept; 378 379template <class Integral> 380 Integral atomic_fetch_sub_explicit(volatile atomic<Integral>* obj, Integral op, 381 memory_order m) noexcept; 382 383template <class Integral> 384 Integral atomic_fetch_sub_explicit(atomic<Integral>* obj, Integral op, 385 memory_order m) noexcept; 386 387template <class Integral> 388 Integral atomic_fetch_and(volatile atomic<Integral>* obj, Integral op) noexcept; 389 390template <class Integral> 391 Integral atomic_fetch_and(atomic<Integral>* obj, Integral op) noexcept; 392 393template <class Integral> 394 Integral atomic_fetch_and_explicit(volatile atomic<Integral>* obj, Integral op, 395 memory_order m) noexcept; 396 397template <class Integral> 398 Integral atomic_fetch_and_explicit(atomic<Integral>* obj, Integral op, 399 memory_order m) noexcept; 400 401template <class Integral> 402 Integral atomic_fetch_or(volatile atomic<Integral>* obj, Integral op) noexcept; 403 404template <class Integral> 405 Integral atomic_fetch_or(atomic<Integral>* obj, Integral op) noexcept; 406 407template <class Integral> 408 Integral atomic_fetch_or_explicit(volatile atomic<Integral>* obj, Integral op, 409 memory_order m) noexcept; 410 411template <class Integral> 412 Integral atomic_fetch_or_explicit(atomic<Integral>* obj, Integral op, 413 memory_order m) noexcept; 414 415template <class Integral> 416 Integral atomic_fetch_xor(volatile atomic<Integral>* obj, Integral op) noexcept; 417 418template <class Integral> 419 Integral atomic_fetch_xor(atomic<Integral>* obj, Integral op) noexcept; 420 421template <class Integral> 422 Integral atomic_fetch_xor_explicit(volatile atomic<Integral>* obj, Integral op, 423 memory_order m) noexcept; 424 425template <class Integral> 426 Integral atomic_fetch_xor_explicit(atomic<Integral>* obj, Integral op, 427 memory_order m) noexcept; 428 429template <class T> 430 T* atomic_fetch_add(volatile atomic<T*>* obj, ptrdiff_t op) noexcept; 431 432template <class T> 433 T* atomic_fetch_add(atomic<T*>* obj, ptrdiff_t op) noexcept; 434 435template <class T> 436 T* atomic_fetch_add_explicit(volatile atomic<T*>* obj, ptrdiff_t op, 437 memory_order m) noexcept; 438 439template <class T> 440 T* atomic_fetch_add_explicit(atomic<T*>* obj, ptrdiff_t op, memory_order m) noexcept; 441 442template <class T> 443 T* atomic_fetch_sub(volatile atomic<T*>* obj, ptrdiff_t op) noexcept; 444 445template <class T> 446 T* atomic_fetch_sub(atomic<T*>* obj, ptrdiff_t op) noexcept; 447 448template <class T> 449 T* atomic_fetch_sub_explicit(volatile atomic<T*>* obj, ptrdiff_t op, 450 memory_order m) noexcept; 451 452template <class T> 453 T* atomic_fetch_sub_explicit(atomic<T*>* obj, ptrdiff_t op, memory_order m) noexcept; 454 455// Atomics for standard typedef types 456 457typedef atomic<bool> atomic_bool; 458typedef atomic<char> atomic_char; 459typedef atomic<signed char> atomic_schar; 460typedef atomic<unsigned char> atomic_uchar; 461typedef atomic<short> atomic_short; 462typedef atomic<unsigned short> atomic_ushort; 463typedef atomic<int> atomic_int; 464typedef atomic<unsigned int> atomic_uint; 465typedef atomic<long> atomic_long; 466typedef atomic<unsigned long> atomic_ulong; 467typedef atomic<long long> atomic_llong; 468typedef atomic<unsigned long long> atomic_ullong; 469typedef atomic<char8_t> atomic_char8_t; // C++20 470typedef atomic<char16_t> atomic_char16_t; 471typedef atomic<char32_t> atomic_char32_t; 472typedef atomic<wchar_t> atomic_wchar_t; 473 474typedef atomic<int_least8_t> atomic_int_least8_t; 475typedef atomic<uint_least8_t> atomic_uint_least8_t; 476typedef atomic<int_least16_t> atomic_int_least16_t; 477typedef atomic<uint_least16_t> atomic_uint_least16_t; 478typedef atomic<int_least32_t> atomic_int_least32_t; 479typedef atomic<uint_least32_t> atomic_uint_least32_t; 480typedef atomic<int_least64_t> atomic_int_least64_t; 481typedef atomic<uint_least64_t> atomic_uint_least64_t; 482 483typedef atomic<int_fast8_t> atomic_int_fast8_t; 484typedef atomic<uint_fast8_t> atomic_uint_fast8_t; 485typedef atomic<int_fast16_t> atomic_int_fast16_t; 486typedef atomic<uint_fast16_t> atomic_uint_fast16_t; 487typedef atomic<int_fast32_t> atomic_int_fast32_t; 488typedef atomic<uint_fast32_t> atomic_uint_fast32_t; 489typedef atomic<int_fast64_t> atomic_int_fast64_t; 490typedef atomic<uint_fast64_t> atomic_uint_fast64_t; 491 492typedef atomic<int8_t> atomic_int8_t; 493typedef atomic<uint8_t> atomic_uint8_t; 494typedef atomic<int16_t> atomic_int16_t; 495typedef atomic<uint16_t> atomic_uint16_t; 496typedef atomic<int32_t> atomic_int32_t; 497typedef atomic<uint32_t> atomic_uint32_t; 498typedef atomic<int64_t> atomic_int64_t; 499typedef atomic<uint64_t> atomic_uint64_t; 500 501typedef atomic<intptr_t> atomic_intptr_t; 502typedef atomic<uintptr_t> atomic_uintptr_t; 503typedef atomic<size_t> atomic_size_t; 504typedef atomic<ptrdiff_t> atomic_ptrdiff_t; 505typedef atomic<intmax_t> atomic_intmax_t; 506typedef atomic<uintmax_t> atomic_uintmax_t; 507 508// flag type and operations 509 510typedef struct atomic_flag 511{ 512 atomic_flag() noexcept = default; 513 atomic_flag(const atomic_flag&) = delete; 514 atomic_flag& operator=(const atomic_flag&) = delete; 515 atomic_flag& operator=(const atomic_flag&) volatile = delete; 516 517 bool test(memory_order m = memory_order_seq_cst) volatile noexcept; 518 bool test(memory_order m = memory_order_seq_cst) noexcept; 519 bool test_and_set(memory_order m = memory_order_seq_cst) volatile noexcept; 520 bool test_and_set(memory_order m = memory_order_seq_cst) noexcept; 521 void clear(memory_order m = memory_order_seq_cst) volatile noexcept; 522 void clear(memory_order m = memory_order_seq_cst) noexcept; 523 524 void wait(bool, memory_order = memory_order::seq_cst) const volatile noexcept; 525 void wait(bool, memory_order = memory_order::seq_cst) const noexcept; 526 void notify_one() volatile noexcept; 527 void notify_one() noexcept; 528 void notify_all() volatile noexcept; 529 void notify_all() noexcept; 530} atomic_flag; 531 532bool atomic_flag_test(volatile atomic_flag* obj) noexcept; 533bool atomic_flag_test(atomic_flag* obj) noexcept; 534bool atomic_flag_test_explicit(volatile atomic_flag* obj, 535 memory_order m) noexcept; 536bool atomic_flag_test_explicit(atomic_flag* obj, memory_order m) noexcept; 537bool atomic_flag_test_and_set(volatile atomic_flag* obj) noexcept; 538bool atomic_flag_test_and_set(atomic_flag* obj) noexcept; 539bool atomic_flag_test_and_set_explicit(volatile atomic_flag* obj, 540 memory_order m) noexcept; 541bool atomic_flag_test_and_set_explicit(atomic_flag* obj, memory_order m) noexcept; 542void atomic_flag_clear(volatile atomic_flag* obj) noexcept; 543void atomic_flag_clear(atomic_flag* obj) noexcept; 544void atomic_flag_clear_explicit(volatile atomic_flag* obj, memory_order m) noexcept; 545void atomic_flag_clear_explicit(atomic_flag* obj, memory_order m) noexcept; 546 547void atomic_wait(const volatile atomic_flag* obj, T old) noexcept; 548void atomic_wait(const atomic_flag* obj, T old) noexcept; 549void atomic_wait_explicit(const volatile atomic_flag* obj, T old, memory_order m) noexcept; 550void atomic_wait_explicit(const atomic_flag* obj, T old, memory_order m) noexcept; 551void atomic_one(volatile atomic_flag* obj) noexcept; 552void atomic_one(atomic_flag* obj) noexcept; 553void atomic_all(volatile atomic_flag* obj) noexcept; 554void atomic_all(atomic_flag* obj) noexcept; 555 556// fences 557 558void atomic_thread_fence(memory_order m) noexcept; 559void atomic_signal_fence(memory_order m) noexcept; 560 561// deprecated 562 563template <class T> 564 void atomic_init(volatile atomic<T>* obj, typename atomic<T>::value_type desr) noexcept; 565 566template <class T> 567 void atomic_init(atomic<T>* obj, typename atomic<T>::value_type desr) noexcept; 568 569#define ATOMIC_VAR_INIT(value) see below 570 571#define ATOMIC_FLAG_INIT see below 572 573} // std 574 575*/ 576 577#include <__availability> 578#include <__config> 579#include <__threading_support> 580#include <cstddef> 581#include <cstdint> 582#include <cstring> 583#include <type_traits> 584#include <version> 585 586#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) 587#pragma GCC system_header 588#endif 589 590#ifdef _LIBCPP_HAS_NO_THREADS 591# error <atomic> is not supported on this single threaded system 592#endif 593#ifdef _LIBCPP_HAS_NO_ATOMIC_HEADER 594# error <atomic> is not implemented 595#endif 596#ifdef kill_dependency 597# error C++ standard library is incompatible with <stdatomic.h> 598#endif 599 600#define _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) \ 601 _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_consume || \ 602 __m == memory_order_acquire || \ 603 __m == memory_order_acq_rel, \ 604 "memory order argument to atomic operation is invalid") 605 606#define _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) \ 607 _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_release || \ 608 __m == memory_order_acq_rel, \ 609 "memory order argument to atomic operation is invalid") 610 611#define _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__m, __f) \ 612 _LIBCPP_DIAGNOSE_WARNING(__f == memory_order_release || \ 613 __f == memory_order_acq_rel, \ 614 "memory order argument to atomic operation is invalid") 615 616_LIBCPP_BEGIN_NAMESPACE_STD 617 618// Figure out what the underlying type for `memory_order` would be if it were 619// declared as an unscoped enum (accounting for -fshort-enums). Use this result 620// to pin the underlying type in C++20. 621enum __legacy_memory_order { 622 __mo_relaxed, 623 __mo_consume, 624 __mo_acquire, 625 __mo_release, 626 __mo_acq_rel, 627 __mo_seq_cst 628}; 629 630typedef underlying_type<__legacy_memory_order>::type __memory_order_underlying_t; 631 632#if _LIBCPP_STD_VER > 17 633 634enum class memory_order : __memory_order_underlying_t { 635 relaxed = __mo_relaxed, 636 consume = __mo_consume, 637 acquire = __mo_acquire, 638 release = __mo_release, 639 acq_rel = __mo_acq_rel, 640 seq_cst = __mo_seq_cst 641}; 642 643inline constexpr auto memory_order_relaxed = memory_order::relaxed; 644inline constexpr auto memory_order_consume = memory_order::consume; 645inline constexpr auto memory_order_acquire = memory_order::acquire; 646inline constexpr auto memory_order_release = memory_order::release; 647inline constexpr auto memory_order_acq_rel = memory_order::acq_rel; 648inline constexpr auto memory_order_seq_cst = memory_order::seq_cst; 649 650#else 651 652typedef enum memory_order { 653 memory_order_relaxed = __mo_relaxed, 654 memory_order_consume = __mo_consume, 655 memory_order_acquire = __mo_acquire, 656 memory_order_release = __mo_release, 657 memory_order_acq_rel = __mo_acq_rel, 658 memory_order_seq_cst = __mo_seq_cst, 659} memory_order; 660 661#endif // _LIBCPP_STD_VER > 17 662 663template <typename _Tp> _LIBCPP_INLINE_VISIBILITY 664bool __cxx_nonatomic_compare_equal(_Tp const& __lhs, _Tp const& __rhs) { 665 return _VSTD::memcmp(&__lhs, &__rhs, sizeof(_Tp)) == 0; 666} 667 668static_assert((is_same<underlying_type<memory_order>::type, __memory_order_underlying_t>::value), 669 "unexpected underlying type for std::memory_order"); 670 671#if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) || \ 672 defined(_LIBCPP_ATOMIC_ONLY_USE_BUILTINS) 673 674// [atomics.types.generic]p1 guarantees _Tp is trivially copyable. Because 675// the default operator= in an object is not volatile, a byte-by-byte copy 676// is required. 677template <typename _Tp, typename _Tv> _LIBCPP_INLINE_VISIBILITY 678typename enable_if<is_assignable<_Tp&, _Tv>::value>::type 679__cxx_atomic_assign_volatile(_Tp& __a_value, _Tv const& __val) { 680 __a_value = __val; 681} 682template <typename _Tp, typename _Tv> _LIBCPP_INLINE_VISIBILITY 683typename enable_if<is_assignable<_Tp&, _Tv>::value>::type 684__cxx_atomic_assign_volatile(_Tp volatile& __a_value, _Tv volatile const& __val) { 685 volatile char* __to = reinterpret_cast<volatile char*>(&__a_value); 686 volatile char* __end = __to + sizeof(_Tp); 687 volatile const char* __from = reinterpret_cast<volatile const char*>(&__val); 688 while (__to != __end) 689 *__to++ = *__from++; 690} 691 692#endif 693 694#if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) 695 696template <typename _Tp> 697struct __cxx_atomic_base_impl { 698 699 _LIBCPP_INLINE_VISIBILITY 700#ifndef _LIBCPP_CXX03_LANG 701 __cxx_atomic_base_impl() _NOEXCEPT = default; 702#else 703 __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {} 704#endif // _LIBCPP_CXX03_LANG 705 _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT 706 : __a_value(value) {} 707 _Tp __a_value; 708}; 709 710_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR int __to_gcc_order(memory_order __order) { 711 // Avoid switch statement to make this a constexpr. 712 return __order == memory_order_relaxed ? __ATOMIC_RELAXED: 713 (__order == memory_order_acquire ? __ATOMIC_ACQUIRE: 714 (__order == memory_order_release ? __ATOMIC_RELEASE: 715 (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST: 716 (__order == memory_order_acq_rel ? __ATOMIC_ACQ_REL: 717 __ATOMIC_CONSUME)))); 718} 719 720_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR int __to_gcc_failure_order(memory_order __order) { 721 // Avoid switch statement to make this a constexpr. 722 return __order == memory_order_relaxed ? __ATOMIC_RELAXED: 723 (__order == memory_order_acquire ? __ATOMIC_ACQUIRE: 724 (__order == memory_order_release ? __ATOMIC_RELAXED: 725 (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST: 726 (__order == memory_order_acq_rel ? __ATOMIC_ACQUIRE: 727 __ATOMIC_CONSUME)))); 728} 729 730template <typename _Tp> 731_LIBCPP_INLINE_VISIBILITY 732void __cxx_atomic_init(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val) { 733 __cxx_atomic_assign_volatile(__a->__a_value, __val); 734} 735 736template <typename _Tp> 737_LIBCPP_INLINE_VISIBILITY 738void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) { 739 __a->__a_value = __val; 740} 741 742_LIBCPP_INLINE_VISIBILITY inline 743void __cxx_atomic_thread_fence(memory_order __order) { 744 __atomic_thread_fence(__to_gcc_order(__order)); 745} 746 747_LIBCPP_INLINE_VISIBILITY inline 748void __cxx_atomic_signal_fence(memory_order __order) { 749 __atomic_signal_fence(__to_gcc_order(__order)); 750} 751 752template <typename _Tp> 753_LIBCPP_INLINE_VISIBILITY 754void __cxx_atomic_store(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val, 755 memory_order __order) { 756 __atomic_store(&__a->__a_value, &__val, 757 __to_gcc_order(__order)); 758} 759 760template <typename _Tp> 761_LIBCPP_INLINE_VISIBILITY 762void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, 763 memory_order __order) { 764 __atomic_store(&__a->__a_value, &__val, 765 __to_gcc_order(__order)); 766} 767 768template <typename _Tp> 769_LIBCPP_INLINE_VISIBILITY 770_Tp __cxx_atomic_load(const volatile __cxx_atomic_base_impl<_Tp>* __a, 771 memory_order __order) { 772 _Tp __ret; 773 __atomic_load(&__a->__a_value, &__ret, 774 __to_gcc_order(__order)); 775 return __ret; 776} 777 778template <typename _Tp> 779_LIBCPP_INLINE_VISIBILITY 780_Tp __cxx_atomic_load(const __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) { 781 _Tp __ret; 782 __atomic_load(&__a->__a_value, &__ret, 783 __to_gcc_order(__order)); 784 return __ret; 785} 786 787template <typename _Tp> 788_LIBCPP_INLINE_VISIBILITY 789_Tp __cxx_atomic_exchange(volatile __cxx_atomic_base_impl<_Tp>* __a, 790 _Tp __value, memory_order __order) { 791 _Tp __ret; 792 __atomic_exchange(&__a->__a_value, &__value, &__ret, 793 __to_gcc_order(__order)); 794 return __ret; 795} 796 797template <typename _Tp> 798_LIBCPP_INLINE_VISIBILITY 799_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, 800 memory_order __order) { 801 _Tp __ret; 802 __atomic_exchange(&__a->__a_value, &__value, &__ret, 803 __to_gcc_order(__order)); 804 return __ret; 805} 806 807template <typename _Tp> 808_LIBCPP_INLINE_VISIBILITY 809bool __cxx_atomic_compare_exchange_strong( 810 volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, 811 memory_order __success, memory_order __failure) { 812 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 813 false, 814 __to_gcc_order(__success), 815 __to_gcc_failure_order(__failure)); 816} 817 818template <typename _Tp> 819_LIBCPP_INLINE_VISIBILITY 820bool __cxx_atomic_compare_exchange_strong( 821 __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, 822 memory_order __failure) { 823 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 824 false, 825 __to_gcc_order(__success), 826 __to_gcc_failure_order(__failure)); 827} 828 829template <typename _Tp> 830_LIBCPP_INLINE_VISIBILITY 831bool __cxx_atomic_compare_exchange_weak( 832 volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, 833 memory_order __success, memory_order __failure) { 834 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 835 true, 836 __to_gcc_order(__success), 837 __to_gcc_failure_order(__failure)); 838} 839 840template <typename _Tp> 841_LIBCPP_INLINE_VISIBILITY 842bool __cxx_atomic_compare_exchange_weak( 843 __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, 844 memory_order __failure) { 845 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 846 true, 847 __to_gcc_order(__success), 848 __to_gcc_failure_order(__failure)); 849} 850 851template <typename _Tp> 852struct __skip_amt { enum {value = 1}; }; 853 854template <typename _Tp> 855struct __skip_amt<_Tp*> { enum {value = sizeof(_Tp)}; }; 856 857// FIXME: Haven't figured out what the spec says about using arrays with 858// atomic_fetch_add. Force a failure rather than creating bad behavior. 859template <typename _Tp> 860struct __skip_amt<_Tp[]> { }; 861template <typename _Tp, int n> 862struct __skip_amt<_Tp[n]> { }; 863 864template <typename _Tp, typename _Td> 865_LIBCPP_INLINE_VISIBILITY 866_Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_base_impl<_Tp>* __a, 867 _Td __delta, memory_order __order) { 868 return __atomic_fetch_add(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 869 __to_gcc_order(__order)); 870} 871 872template <typename _Tp, typename _Td> 873_LIBCPP_INLINE_VISIBILITY 874_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, 875 memory_order __order) { 876 return __atomic_fetch_add(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 877 __to_gcc_order(__order)); 878} 879 880template <typename _Tp, typename _Td> 881_LIBCPP_INLINE_VISIBILITY 882_Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_base_impl<_Tp>* __a, 883 _Td __delta, memory_order __order) { 884 return __atomic_fetch_sub(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 885 __to_gcc_order(__order)); 886} 887 888template <typename _Tp, typename _Td> 889_LIBCPP_INLINE_VISIBILITY 890_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, 891 memory_order __order) { 892 return __atomic_fetch_sub(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 893 __to_gcc_order(__order)); 894} 895 896template <typename _Tp> 897_LIBCPP_INLINE_VISIBILITY 898_Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_base_impl<_Tp>* __a, 899 _Tp __pattern, memory_order __order) { 900 return __atomic_fetch_and(&__a->__a_value, __pattern, 901 __to_gcc_order(__order)); 902} 903 904template <typename _Tp> 905_LIBCPP_INLINE_VISIBILITY 906_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, 907 _Tp __pattern, memory_order __order) { 908 return __atomic_fetch_and(&__a->__a_value, __pattern, 909 __to_gcc_order(__order)); 910} 911 912template <typename _Tp> 913_LIBCPP_INLINE_VISIBILITY 914_Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_base_impl<_Tp>* __a, 915 _Tp __pattern, memory_order __order) { 916 return __atomic_fetch_or(&__a->__a_value, __pattern, 917 __to_gcc_order(__order)); 918} 919 920template <typename _Tp> 921_LIBCPP_INLINE_VISIBILITY 922_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, 923 memory_order __order) { 924 return __atomic_fetch_or(&__a->__a_value, __pattern, 925 __to_gcc_order(__order)); 926} 927 928template <typename _Tp> 929_LIBCPP_INLINE_VISIBILITY 930_Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_base_impl<_Tp>* __a, 931 _Tp __pattern, memory_order __order) { 932 return __atomic_fetch_xor(&__a->__a_value, __pattern, 933 __to_gcc_order(__order)); 934} 935 936template <typename _Tp> 937_LIBCPP_INLINE_VISIBILITY 938_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, 939 memory_order __order) { 940 return __atomic_fetch_xor(&__a->__a_value, __pattern, 941 __to_gcc_order(__order)); 942} 943 944#define __cxx_atomic_is_lock_free(__s) __atomic_is_lock_free(__s, 0) 945 946#elif defined(_LIBCPP_HAS_C_ATOMIC_IMP) 947 948template <typename _Tp> 949struct __cxx_atomic_base_impl { 950 951 _LIBCPP_INLINE_VISIBILITY 952#ifndef _LIBCPP_CXX03_LANG 953 __cxx_atomic_base_impl() _NOEXCEPT = default; 954#else 955 __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {} 956#endif // _LIBCPP_CXX03_LANG 957 _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT 958 : __a_value(value) {} 959 _LIBCPP_DISABLE_EXTENSION_WARNING _Atomic(_Tp) __a_value; 960}; 961 962#define __cxx_atomic_is_lock_free(__s) __c11_atomic_is_lock_free(__s) 963 964_LIBCPP_INLINE_VISIBILITY inline 965void __cxx_atomic_thread_fence(memory_order __order) _NOEXCEPT { 966 __c11_atomic_thread_fence(static_cast<__memory_order_underlying_t>(__order)); 967} 968 969_LIBCPP_INLINE_VISIBILITY inline 970void __cxx_atomic_signal_fence(memory_order __order) _NOEXCEPT { 971 __c11_atomic_signal_fence(static_cast<__memory_order_underlying_t>(__order)); 972} 973 974template<class _Tp> 975_LIBCPP_INLINE_VISIBILITY 976void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val) _NOEXCEPT { 977 __c11_atomic_init(&__a->__a_value, __val); 978} 979template<class _Tp> 980_LIBCPP_INLINE_VISIBILITY 981void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val) _NOEXCEPT { 982 __c11_atomic_init(&__a->__a_value, __val); 983} 984 985template<class _Tp> 986_LIBCPP_INLINE_VISIBILITY 987void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT { 988 __c11_atomic_store(&__a->__a_value, __val, static_cast<__memory_order_underlying_t>(__order)); 989} 990template<class _Tp> 991_LIBCPP_INLINE_VISIBILITY 992void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val, memory_order __order) _NOEXCEPT { 993 __c11_atomic_store(&__a->__a_value, __val, static_cast<__memory_order_underlying_t>(__order)); 994} 995 996template<class _Tp> 997_LIBCPP_INLINE_VISIBILITY 998_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const volatile* __a, memory_order __order) _NOEXCEPT { 999 using __ptr_type = typename remove_const<decltype(__a->__a_value)>::type*; 1000 return __c11_atomic_load(const_cast<__ptr_type>(&__a->__a_value), static_cast<__memory_order_underlying_t>(__order)); 1001} 1002template<class _Tp> 1003_LIBCPP_INLINE_VISIBILITY 1004_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const* __a, memory_order __order) _NOEXCEPT { 1005 using __ptr_type = typename remove_const<decltype(__a->__a_value)>::type*; 1006 return __c11_atomic_load(const_cast<__ptr_type>(&__a->__a_value), static_cast<__memory_order_underlying_t>(__order)); 1007} 1008 1009template<class _Tp> 1010_LIBCPP_INLINE_VISIBILITY 1011_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT { 1012 return __c11_atomic_exchange(&__a->__a_value, __value, static_cast<__memory_order_underlying_t>(__order)); 1013} 1014template<class _Tp> 1015_LIBCPP_INLINE_VISIBILITY 1016_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> * __a, _Tp __value, memory_order __order) _NOEXCEPT { 1017 return __c11_atomic_exchange(&__a->__a_value, __value, static_cast<__memory_order_underlying_t>(__order)); 1018} 1019 1020_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR memory_order __to_failure_order(memory_order __order) { 1021 // Avoid switch statement to make this a constexpr. 1022 return __order == memory_order_release ? memory_order_relaxed: 1023 (__order == memory_order_acq_rel ? memory_order_acquire: 1024 __order); 1025} 1026 1027template<class _Tp> 1028_LIBCPP_INLINE_VISIBILITY 1029bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 1030 return __c11_atomic_compare_exchange_strong(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); 1031} 1032template<class _Tp> 1033_LIBCPP_INLINE_VISIBILITY 1034bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 1035 return __c11_atomic_compare_exchange_strong(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); 1036} 1037 1038template<class _Tp> 1039_LIBCPP_INLINE_VISIBILITY 1040bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 1041 return __c11_atomic_compare_exchange_weak(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); 1042} 1043template<class _Tp> 1044_LIBCPP_INLINE_VISIBILITY 1045bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 1046 return __c11_atomic_compare_exchange_weak(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); 1047} 1048 1049template<class _Tp> 1050_LIBCPP_INLINE_VISIBILITY 1051_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1052 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1053} 1054template<class _Tp> 1055_LIBCPP_INLINE_VISIBILITY 1056_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1057 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1058} 1059 1060template<class _Tp> 1061_LIBCPP_INLINE_VISIBILITY 1062_Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1063 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1064} 1065template<class _Tp> 1066_LIBCPP_INLINE_VISIBILITY 1067_Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1068 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1069} 1070 1071template<class _Tp> 1072_LIBCPP_INLINE_VISIBILITY 1073_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1074 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1075} 1076template<class _Tp> 1077_LIBCPP_INLINE_VISIBILITY 1078_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1079 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1080} 1081template<class _Tp> 1082_LIBCPP_INLINE_VISIBILITY 1083_Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1084 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1085} 1086template<class _Tp> 1087_LIBCPP_INLINE_VISIBILITY 1088_Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1089 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1090} 1091 1092template<class _Tp> 1093_LIBCPP_INLINE_VISIBILITY 1094_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1095 return __c11_atomic_fetch_and(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1096} 1097template<class _Tp> 1098_LIBCPP_INLINE_VISIBILITY 1099_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1100 return __c11_atomic_fetch_and(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1101} 1102 1103template<class _Tp> 1104_LIBCPP_INLINE_VISIBILITY 1105_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1106 return __c11_atomic_fetch_or(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1107} 1108template<class _Tp> 1109_LIBCPP_INLINE_VISIBILITY 1110_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1111 return __c11_atomic_fetch_or(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1112} 1113 1114template<class _Tp> 1115_LIBCPP_INLINE_VISIBILITY 1116_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1117 return __c11_atomic_fetch_xor(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1118} 1119template<class _Tp> 1120_LIBCPP_INLINE_VISIBILITY 1121_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1122 return __c11_atomic_fetch_xor(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1123} 1124 1125#endif // _LIBCPP_HAS_GCC_ATOMIC_IMP, _LIBCPP_HAS_C_ATOMIC_IMP 1126 1127template <class _Tp> 1128_LIBCPP_INLINE_VISIBILITY 1129_Tp kill_dependency(_Tp __y) _NOEXCEPT 1130{ 1131 return __y; 1132} 1133 1134#if defined(__CLANG_ATOMIC_BOOL_LOCK_FREE) 1135# define ATOMIC_BOOL_LOCK_FREE __CLANG_ATOMIC_BOOL_LOCK_FREE 1136# define ATOMIC_CHAR_LOCK_FREE __CLANG_ATOMIC_CHAR_LOCK_FREE 1137#ifndef _LIBCPP_HAS_NO_CHAR8_T 1138# define ATOMIC_CHAR8_T_LOCK_FREE __CLANG_ATOMIC_CHAR8_T_LOCK_FREE 1139#endif 1140# define ATOMIC_CHAR16_T_LOCK_FREE __CLANG_ATOMIC_CHAR16_T_LOCK_FREE 1141# define ATOMIC_CHAR32_T_LOCK_FREE __CLANG_ATOMIC_CHAR32_T_LOCK_FREE 1142# define ATOMIC_WCHAR_T_LOCK_FREE __CLANG_ATOMIC_WCHAR_T_LOCK_FREE 1143# define ATOMIC_SHORT_LOCK_FREE __CLANG_ATOMIC_SHORT_LOCK_FREE 1144# define ATOMIC_INT_LOCK_FREE __CLANG_ATOMIC_INT_LOCK_FREE 1145# define ATOMIC_LONG_LOCK_FREE __CLANG_ATOMIC_LONG_LOCK_FREE 1146# define ATOMIC_LLONG_LOCK_FREE __CLANG_ATOMIC_LLONG_LOCK_FREE 1147# define ATOMIC_POINTER_LOCK_FREE __CLANG_ATOMIC_POINTER_LOCK_FREE 1148#elif defined(__GCC_ATOMIC_BOOL_LOCK_FREE) 1149# define ATOMIC_BOOL_LOCK_FREE __GCC_ATOMIC_BOOL_LOCK_FREE 1150# define ATOMIC_CHAR_LOCK_FREE __GCC_ATOMIC_CHAR_LOCK_FREE 1151#ifndef _LIBCPP_HAS_NO_CHAR8_T 1152# define ATOMIC_CHAR8_T_LOCK_FREE __GCC_ATOMIC_CHAR8_T_LOCK_FREE 1153#endif 1154# define ATOMIC_CHAR16_T_LOCK_FREE __GCC_ATOMIC_CHAR16_T_LOCK_FREE 1155# define ATOMIC_CHAR32_T_LOCK_FREE __GCC_ATOMIC_CHAR32_T_LOCK_FREE 1156# define ATOMIC_WCHAR_T_LOCK_FREE __GCC_ATOMIC_WCHAR_T_LOCK_FREE 1157# define ATOMIC_SHORT_LOCK_FREE __GCC_ATOMIC_SHORT_LOCK_FREE 1158# define ATOMIC_INT_LOCK_FREE __GCC_ATOMIC_INT_LOCK_FREE 1159# define ATOMIC_LONG_LOCK_FREE __GCC_ATOMIC_LONG_LOCK_FREE 1160# define ATOMIC_LLONG_LOCK_FREE __GCC_ATOMIC_LLONG_LOCK_FREE 1161# define ATOMIC_POINTER_LOCK_FREE __GCC_ATOMIC_POINTER_LOCK_FREE 1162#endif 1163 1164#ifdef _LIBCPP_ATOMIC_ONLY_USE_BUILTINS 1165 1166template<typename _Tp> 1167struct __cxx_atomic_lock_impl { 1168 1169 _LIBCPP_INLINE_VISIBILITY 1170 __cxx_atomic_lock_impl() _NOEXCEPT 1171 : __a_value(), __a_lock(0) {} 1172 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR explicit 1173 __cxx_atomic_lock_impl(_Tp value) _NOEXCEPT 1174 : __a_value(value), __a_lock(0) {} 1175 1176 _Tp __a_value; 1177 mutable __cxx_atomic_base_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_lock; 1178 1179 _LIBCPP_INLINE_VISIBILITY void __lock() const volatile { 1180 while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire)) 1181 /*spin*/; 1182 } 1183 _LIBCPP_INLINE_VISIBILITY void __lock() const { 1184 while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire)) 1185 /*spin*/; 1186 } 1187 _LIBCPP_INLINE_VISIBILITY void __unlock() const volatile { 1188 __cxx_atomic_store(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(false), memory_order_release); 1189 } 1190 _LIBCPP_INLINE_VISIBILITY void __unlock() const { 1191 __cxx_atomic_store(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(false), memory_order_release); 1192 } 1193 _LIBCPP_INLINE_VISIBILITY _Tp __read() const volatile { 1194 __lock(); 1195 _Tp __old; 1196 __cxx_atomic_assign_volatile(__old, __a_value); 1197 __unlock(); 1198 return __old; 1199 } 1200 _LIBCPP_INLINE_VISIBILITY _Tp __read() const { 1201 __lock(); 1202 _Tp __old = __a_value; 1203 __unlock(); 1204 return __old; 1205 } 1206}; 1207 1208template <typename _Tp> 1209_LIBCPP_INLINE_VISIBILITY 1210void __cxx_atomic_init(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __val) { 1211 __cxx_atomic_assign_volatile(__a->__a_value, __val); 1212} 1213template <typename _Tp> 1214_LIBCPP_INLINE_VISIBILITY 1215void __cxx_atomic_init(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __val) { 1216 __a->__a_value = __val; 1217} 1218 1219template <typename _Tp> 1220_LIBCPP_INLINE_VISIBILITY 1221void __cxx_atomic_store(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __val, memory_order) { 1222 __a->__lock(); 1223 __cxx_atomic_assign_volatile(__a->__a_value, __val); 1224 __a->__unlock(); 1225} 1226template <typename _Tp> 1227_LIBCPP_INLINE_VISIBILITY 1228void __cxx_atomic_store(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __val, memory_order) { 1229 __a->__lock(); 1230 __a->__a_value = __val; 1231 __a->__unlock(); 1232} 1233 1234template <typename _Tp> 1235_LIBCPP_INLINE_VISIBILITY 1236_Tp __cxx_atomic_load(const volatile __cxx_atomic_lock_impl<_Tp>* __a, memory_order) { 1237 return __a->__read(); 1238} 1239template <typename _Tp> 1240_LIBCPP_INLINE_VISIBILITY 1241_Tp __cxx_atomic_load(const __cxx_atomic_lock_impl<_Tp>* __a, memory_order) { 1242 return __a->__read(); 1243} 1244 1245template <typename _Tp> 1246_LIBCPP_INLINE_VISIBILITY 1247_Tp __cxx_atomic_exchange(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) { 1248 __a->__lock(); 1249 _Tp __old; 1250 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1251 __cxx_atomic_assign_volatile(__a->__a_value, __value); 1252 __a->__unlock(); 1253 return __old; 1254} 1255template <typename _Tp> 1256_LIBCPP_INLINE_VISIBILITY 1257_Tp __cxx_atomic_exchange(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) { 1258 __a->__lock(); 1259 _Tp __old = __a->__a_value; 1260 __a->__a_value = __value; 1261 __a->__unlock(); 1262 return __old; 1263} 1264 1265template <typename _Tp> 1266_LIBCPP_INLINE_VISIBILITY 1267bool __cxx_atomic_compare_exchange_strong(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1268 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1269 _Tp __temp; 1270 __a->__lock(); 1271 __cxx_atomic_assign_volatile(__temp, __a->__a_value); 1272 bool __ret = (_VSTD::memcmp(&__temp, __expected, sizeof(_Tp)) == 0); 1273 if(__ret) 1274 __cxx_atomic_assign_volatile(__a->__a_value, __value); 1275 else 1276 __cxx_atomic_assign_volatile(*__expected, __a->__a_value); 1277 __a->__unlock(); 1278 return __ret; 1279} 1280template <typename _Tp> 1281_LIBCPP_INLINE_VISIBILITY 1282bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_lock_impl<_Tp>* __a, 1283 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1284 __a->__lock(); 1285 bool __ret = (_VSTD::memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0); 1286 if(__ret) 1287 _VSTD::memcpy(&__a->__a_value, &__value, sizeof(_Tp)); 1288 else 1289 _VSTD::memcpy(__expected, &__a->__a_value, sizeof(_Tp)); 1290 __a->__unlock(); 1291 return __ret; 1292} 1293 1294template <typename _Tp> 1295_LIBCPP_INLINE_VISIBILITY 1296bool __cxx_atomic_compare_exchange_weak(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1297 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1298 _Tp __temp; 1299 __a->__lock(); 1300 __cxx_atomic_assign_volatile(__temp, __a->__a_value); 1301 bool __ret = (_VSTD::memcmp(&__temp, __expected, sizeof(_Tp)) == 0); 1302 if(__ret) 1303 __cxx_atomic_assign_volatile(__a->__a_value, __value); 1304 else 1305 __cxx_atomic_assign_volatile(*__expected, __a->__a_value); 1306 __a->__unlock(); 1307 return __ret; 1308} 1309template <typename _Tp> 1310_LIBCPP_INLINE_VISIBILITY 1311bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_lock_impl<_Tp>* __a, 1312 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1313 __a->__lock(); 1314 bool __ret = (_VSTD::memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0); 1315 if(__ret) 1316 _VSTD::memcpy(&__a->__a_value, &__value, sizeof(_Tp)); 1317 else 1318 _VSTD::memcpy(__expected, &__a->__a_value, sizeof(_Tp)); 1319 __a->__unlock(); 1320 return __ret; 1321} 1322 1323template <typename _Tp, typename _Td> 1324_LIBCPP_INLINE_VISIBILITY 1325_Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1326 _Td __delta, memory_order) { 1327 __a->__lock(); 1328 _Tp __old; 1329 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1330 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old + __delta)); 1331 __a->__unlock(); 1332 return __old; 1333} 1334template <typename _Tp, typename _Td> 1335_LIBCPP_INLINE_VISIBILITY 1336_Tp __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp>* __a, 1337 _Td __delta, memory_order) { 1338 __a->__lock(); 1339 _Tp __old = __a->__a_value; 1340 __a->__a_value += __delta; 1341 __a->__unlock(); 1342 return __old; 1343} 1344 1345template <typename _Tp, typename _Td> 1346_LIBCPP_INLINE_VISIBILITY 1347_Tp* __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp*>* __a, 1348 ptrdiff_t __delta, memory_order) { 1349 __a->__lock(); 1350 _Tp* __old; 1351 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1352 __cxx_atomic_assign_volatile(__a->__a_value, __old + __delta); 1353 __a->__unlock(); 1354 return __old; 1355} 1356template <typename _Tp, typename _Td> 1357_LIBCPP_INLINE_VISIBILITY 1358_Tp* __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp*>* __a, 1359 ptrdiff_t __delta, memory_order) { 1360 __a->__lock(); 1361 _Tp* __old = __a->__a_value; 1362 __a->__a_value += __delta; 1363 __a->__unlock(); 1364 return __old; 1365} 1366 1367template <typename _Tp, typename _Td> 1368_LIBCPP_INLINE_VISIBILITY 1369_Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1370 _Td __delta, memory_order) { 1371 __a->__lock(); 1372 _Tp __old; 1373 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1374 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old - __delta)); 1375 __a->__unlock(); 1376 return __old; 1377} 1378template <typename _Tp, typename _Td> 1379_LIBCPP_INLINE_VISIBILITY 1380_Tp __cxx_atomic_fetch_sub(__cxx_atomic_lock_impl<_Tp>* __a, 1381 _Td __delta, memory_order) { 1382 __a->__lock(); 1383 _Tp __old = __a->__a_value; 1384 __a->__a_value -= __delta; 1385 __a->__unlock(); 1386 return __old; 1387} 1388 1389template <typename _Tp> 1390_LIBCPP_INLINE_VISIBILITY 1391_Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1392 _Tp __pattern, memory_order) { 1393 __a->__lock(); 1394 _Tp __old; 1395 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1396 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old & __pattern)); 1397 __a->__unlock(); 1398 return __old; 1399} 1400template <typename _Tp> 1401_LIBCPP_INLINE_VISIBILITY 1402_Tp __cxx_atomic_fetch_and(__cxx_atomic_lock_impl<_Tp>* __a, 1403 _Tp __pattern, memory_order) { 1404 __a->__lock(); 1405 _Tp __old = __a->__a_value; 1406 __a->__a_value &= __pattern; 1407 __a->__unlock(); 1408 return __old; 1409} 1410 1411template <typename _Tp> 1412_LIBCPP_INLINE_VISIBILITY 1413_Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1414 _Tp __pattern, memory_order) { 1415 __a->__lock(); 1416 _Tp __old; 1417 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1418 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old | __pattern)); 1419 __a->__unlock(); 1420 return __old; 1421} 1422template <typename _Tp> 1423_LIBCPP_INLINE_VISIBILITY 1424_Tp __cxx_atomic_fetch_or(__cxx_atomic_lock_impl<_Tp>* __a, 1425 _Tp __pattern, memory_order) { 1426 __a->__lock(); 1427 _Tp __old = __a->__a_value; 1428 __a->__a_value |= __pattern; 1429 __a->__unlock(); 1430 return __old; 1431} 1432 1433template <typename _Tp> 1434_LIBCPP_INLINE_VISIBILITY 1435_Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1436 _Tp __pattern, memory_order) { 1437 __a->__lock(); 1438 _Tp __old; 1439 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1440 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old ^ __pattern)); 1441 __a->__unlock(); 1442 return __old; 1443} 1444template <typename _Tp> 1445_LIBCPP_INLINE_VISIBILITY 1446_Tp __cxx_atomic_fetch_xor(__cxx_atomic_lock_impl<_Tp>* __a, 1447 _Tp __pattern, memory_order) { 1448 __a->__lock(); 1449 _Tp __old = __a->__a_value; 1450 __a->__a_value ^= __pattern; 1451 __a->__unlock(); 1452 return __old; 1453} 1454 1455#ifdef __cpp_lib_atomic_is_always_lock_free 1456 1457template<typename _Tp> struct __cxx_is_always_lock_free { 1458 enum { __value = __atomic_always_lock_free(sizeof(_Tp), 0) }; }; 1459 1460#else 1461 1462template<typename _Tp> struct __cxx_is_always_lock_free { enum { __value = false }; }; 1463// Implementations must match the C ATOMIC_*_LOCK_FREE macro values. 1464template<> struct __cxx_is_always_lock_free<bool> { enum { __value = 2 == ATOMIC_BOOL_LOCK_FREE }; }; 1465template<> struct __cxx_is_always_lock_free<char> { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; }; 1466template<> struct __cxx_is_always_lock_free<signed char> { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; }; 1467template<> struct __cxx_is_always_lock_free<unsigned char> { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; }; 1468#ifndef _LIBCPP_HAS_NO_CHAR8_T 1469template<> struct __cxx_is_always_lock_free<char8_t> { enum { __value = 2 == ATOMIC_CHAR8_T_LOCK_FREE }; }; 1470#endif 1471template<> struct __cxx_is_always_lock_free<char16_t> { enum { __value = 2 == ATOMIC_CHAR16_T_LOCK_FREE }; }; 1472template<> struct __cxx_is_always_lock_free<char32_t> { enum { __value = 2 == ATOMIC_CHAR32_T_LOCK_FREE }; }; 1473template<> struct __cxx_is_always_lock_free<wchar_t> { enum { __value = 2 == ATOMIC_WCHAR_T_LOCK_FREE }; }; 1474template<> struct __cxx_is_always_lock_free<short> { enum { __value = 2 == ATOMIC_SHORT_LOCK_FREE }; }; 1475template<> struct __cxx_is_always_lock_free<unsigned short> { enum { __value = 2 == ATOMIC_SHORT_LOCK_FREE }; }; 1476template<> struct __cxx_is_always_lock_free<int> { enum { __value = 2 == ATOMIC_INT_LOCK_FREE }; }; 1477template<> struct __cxx_is_always_lock_free<unsigned int> { enum { __value = 2 == ATOMIC_INT_LOCK_FREE }; }; 1478template<> struct __cxx_is_always_lock_free<long> { enum { __value = 2 == ATOMIC_LONG_LOCK_FREE }; }; 1479template<> struct __cxx_is_always_lock_free<unsigned long> { enum { __value = 2 == ATOMIC_LONG_LOCK_FREE }; }; 1480template<> struct __cxx_is_always_lock_free<long long> { enum { __value = 2 == ATOMIC_LLONG_LOCK_FREE }; }; 1481template<> struct __cxx_is_always_lock_free<unsigned long long> { enum { __value = 2 == ATOMIC_LLONG_LOCK_FREE }; }; 1482template<typename _Tp> struct __cxx_is_always_lock_free<_Tp*> { enum { __value = 2 == ATOMIC_POINTER_LOCK_FREE }; }; 1483template<> struct __cxx_is_always_lock_free<std::nullptr_t> { enum { __value = 2 == ATOMIC_POINTER_LOCK_FREE }; }; 1484 1485#endif //__cpp_lib_atomic_is_always_lock_free 1486 1487template <typename _Tp, 1488 typename _Base = typename conditional<__cxx_is_always_lock_free<_Tp>::__value, 1489 __cxx_atomic_base_impl<_Tp>, 1490 __cxx_atomic_lock_impl<_Tp> >::type> 1491#else 1492template <typename _Tp, 1493 typename _Base = __cxx_atomic_base_impl<_Tp> > 1494#endif //_LIBCPP_ATOMIC_ONLY_USE_BUILTINS 1495struct __cxx_atomic_impl : public _Base { 1496 1497#if _GNUC_VER >= 501 1498 static_assert(is_trivially_copyable<_Tp>::value, 1499 "std::atomic<Tp> requires that 'Tp' be a trivially copyable type"); 1500#endif 1501 1502 _LIBCPP_INLINE_VISIBILITY __cxx_atomic_impl() _NOEXCEPT _LIBCPP_DEFAULT 1503 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR explicit __cxx_atomic_impl(_Tp value) _NOEXCEPT 1504 : _Base(value) {} 1505}; 1506 1507#ifdef __linux__ 1508 using __cxx_contention_t = int32_t; 1509#else 1510 using __cxx_contention_t = int64_t; 1511#endif //__linux__ 1512 1513using __cxx_atomic_contention_t = __cxx_atomic_impl<__cxx_contention_t>; 1514 1515#ifndef _LIBCPP_HAS_NO_PLATFORM_WAIT 1516 1517_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(void const volatile*); 1518_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(void const volatile*); 1519_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(void const volatile*); 1520_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(void const volatile*, __cxx_contention_t); 1521 1522_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(__cxx_atomic_contention_t const volatile*); 1523_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(__cxx_atomic_contention_t const volatile*); 1524_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile*); 1525_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(__cxx_atomic_contention_t const volatile*, __cxx_contention_t); 1526 1527template <class _Atp, class _Fn> 1528struct __libcpp_atomic_wait_backoff_impl { 1529 _Atp* __a; 1530 _Fn __test_fn; 1531 _LIBCPP_AVAILABILITY_SYNC 1532 _LIBCPP_INLINE_VISIBILITY bool operator()(chrono::nanoseconds __elapsed) const 1533 { 1534 if(__elapsed > chrono::microseconds(64)) 1535 { 1536 auto const __monitor = __libcpp_atomic_monitor(__a); 1537 if(__test_fn()) 1538 return true; 1539 __libcpp_atomic_wait(__a, __monitor); 1540 } 1541 else if(__elapsed > chrono::microseconds(4)) 1542 __libcpp_thread_yield(); 1543 else 1544 {} // poll 1545 return false; 1546 } 1547}; 1548 1549template <class _Atp, class _Fn> 1550_LIBCPP_AVAILABILITY_SYNC 1551_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp* __a, _Fn && __test_fn) 1552{ 1553 __libcpp_atomic_wait_backoff_impl<_Atp, typename decay<_Fn>::type> __backoff_fn = {__a, __test_fn}; 1554 return __libcpp_thread_poll_with_backoff(__test_fn, __backoff_fn); 1555} 1556 1557#else // _LIBCPP_HAS_NO_PLATFORM_WAIT 1558 1559template <class _Tp> 1560_LIBCPP_INLINE_VISIBILITY void __cxx_atomic_notify_all(__cxx_atomic_impl<_Tp> const volatile*) { } 1561template <class _Tp> 1562_LIBCPP_INLINE_VISIBILITY void __cxx_atomic_notify_one(__cxx_atomic_impl<_Tp> const volatile*) { } 1563template <class _Atp, class _Fn> 1564_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp*, _Fn && __test_fn) 1565{ 1566 return __libcpp_thread_poll_with_backoff(__test_fn, __libcpp_timed_backoff_policy()); 1567} 1568 1569#endif // _LIBCPP_HAS_NO_PLATFORM_WAIT 1570 1571template <class _Atp, class _Tp> 1572struct __cxx_atomic_wait_test_fn_impl { 1573 _Atp* __a; 1574 _Tp __val; 1575 memory_order __order; 1576 _LIBCPP_INLINE_VISIBILITY bool operator()() const 1577 { 1578 return !__cxx_nonatomic_compare_equal(__cxx_atomic_load(__a, __order), __val); 1579 } 1580}; 1581 1582template <class _Atp, class _Tp> 1583_LIBCPP_AVAILABILITY_SYNC 1584_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp* __a, _Tp const __val, memory_order __order) 1585{ 1586 __cxx_atomic_wait_test_fn_impl<_Atp, _Tp> __test_fn = {__a, __val, __order}; 1587 return __cxx_atomic_wait(__a, __test_fn); 1588} 1589 1590// general atomic<T> 1591 1592template <class _Tp, bool = is_integral<_Tp>::value && !is_same<_Tp, bool>::value> 1593struct __atomic_base // false 1594{ 1595 mutable __cxx_atomic_impl<_Tp> __a_; 1596 1597#if defined(__cpp_lib_atomic_is_always_lock_free) 1598 static _LIBCPP_CONSTEXPR bool is_always_lock_free = __atomic_always_lock_free(sizeof(__a_), 0); 1599#endif 1600 1601 _LIBCPP_INLINE_VISIBILITY 1602 bool is_lock_free() const volatile _NOEXCEPT 1603 {return __cxx_atomic_is_lock_free(sizeof(_Tp));} 1604 _LIBCPP_INLINE_VISIBILITY 1605 bool is_lock_free() const _NOEXCEPT 1606 {return static_cast<__atomic_base const volatile*>(this)->is_lock_free();} 1607 _LIBCPP_INLINE_VISIBILITY 1608 void store(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1609 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1610 {__cxx_atomic_store(&__a_, __d, __m);} 1611 _LIBCPP_INLINE_VISIBILITY 1612 void store(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1613 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1614 {__cxx_atomic_store(&__a_, __d, __m);} 1615 _LIBCPP_INLINE_VISIBILITY 1616 _Tp load(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 1617 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 1618 {return __cxx_atomic_load(&__a_, __m);} 1619 _LIBCPP_INLINE_VISIBILITY 1620 _Tp load(memory_order __m = memory_order_seq_cst) const _NOEXCEPT 1621 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 1622 {return __cxx_atomic_load(&__a_, __m);} 1623 _LIBCPP_INLINE_VISIBILITY 1624 operator _Tp() const volatile _NOEXCEPT {return load();} 1625 _LIBCPP_INLINE_VISIBILITY 1626 operator _Tp() const _NOEXCEPT {return load();} 1627 _LIBCPP_INLINE_VISIBILITY 1628 _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1629 {return __cxx_atomic_exchange(&__a_, __d, __m);} 1630 _LIBCPP_INLINE_VISIBILITY 1631 _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1632 {return __cxx_atomic_exchange(&__a_, __d, __m);} 1633 _LIBCPP_INLINE_VISIBILITY 1634 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1635 memory_order __s, memory_order __f) volatile _NOEXCEPT 1636 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1637 {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);} 1638 _LIBCPP_INLINE_VISIBILITY 1639 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1640 memory_order __s, memory_order __f) _NOEXCEPT 1641 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1642 {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);} 1643 _LIBCPP_INLINE_VISIBILITY 1644 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1645 memory_order __s, memory_order __f) volatile _NOEXCEPT 1646 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1647 {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);} 1648 _LIBCPP_INLINE_VISIBILITY 1649 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1650 memory_order __s, memory_order __f) _NOEXCEPT 1651 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1652 {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);} 1653 _LIBCPP_INLINE_VISIBILITY 1654 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1655 memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1656 {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);} 1657 _LIBCPP_INLINE_VISIBILITY 1658 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1659 memory_order __m = memory_order_seq_cst) _NOEXCEPT 1660 {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);} 1661 _LIBCPP_INLINE_VISIBILITY 1662 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1663 memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1664 {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);} 1665 _LIBCPP_INLINE_VISIBILITY 1666 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1667 memory_order __m = memory_order_seq_cst) _NOEXCEPT 1668 {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);} 1669 1670 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 1671 {__cxx_atomic_wait(&__a_, __v, __m);} 1672 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT 1673 {__cxx_atomic_wait(&__a_, __v, __m);} 1674 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_one() volatile _NOEXCEPT 1675 {__cxx_atomic_notify_one(&__a_);} 1676 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_one() _NOEXCEPT 1677 {__cxx_atomic_notify_one(&__a_);} 1678 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_all() volatile _NOEXCEPT 1679 {__cxx_atomic_notify_all(&__a_);} 1680 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_all() _NOEXCEPT 1681 {__cxx_atomic_notify_all(&__a_);} 1682 1683 _LIBCPP_INLINE_VISIBILITY 1684 __atomic_base() _NOEXCEPT _LIBCPP_DEFAULT 1685 1686 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR 1687 __atomic_base(_Tp __d) _NOEXCEPT : __a_(__d) {} 1688 1689#ifndef _LIBCPP_CXX03_LANG 1690 __atomic_base(const __atomic_base&) = delete; 1691#else 1692private: 1693 _LIBCPP_INLINE_VISIBILITY 1694 __atomic_base(const __atomic_base&); 1695#endif 1696}; 1697 1698#if defined(__cpp_lib_atomic_is_always_lock_free) 1699template <class _Tp, bool __b> 1700_LIBCPP_CONSTEXPR bool __atomic_base<_Tp, __b>::is_always_lock_free; 1701#endif 1702 1703// atomic<Integral> 1704 1705template <class _Tp> 1706struct __atomic_base<_Tp, true> 1707 : public __atomic_base<_Tp, false> 1708{ 1709 typedef __atomic_base<_Tp, false> __base; 1710 _LIBCPP_INLINE_VISIBILITY 1711 __atomic_base() _NOEXCEPT _LIBCPP_DEFAULT 1712 _LIBCPP_INLINE_VISIBILITY 1713 _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __base(__d) {} 1714 1715 _LIBCPP_INLINE_VISIBILITY 1716 _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1717 {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);} 1718 _LIBCPP_INLINE_VISIBILITY 1719 _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1720 {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);} 1721 _LIBCPP_INLINE_VISIBILITY 1722 _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1723 {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);} 1724 _LIBCPP_INLINE_VISIBILITY 1725 _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1726 {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);} 1727 _LIBCPP_INLINE_VISIBILITY 1728 _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1729 {return __cxx_atomic_fetch_and(&this->__a_, __op, __m);} 1730 _LIBCPP_INLINE_VISIBILITY 1731 _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1732 {return __cxx_atomic_fetch_and(&this->__a_, __op, __m);} 1733 _LIBCPP_INLINE_VISIBILITY 1734 _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1735 {return __cxx_atomic_fetch_or(&this->__a_, __op, __m);} 1736 _LIBCPP_INLINE_VISIBILITY 1737 _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1738 {return __cxx_atomic_fetch_or(&this->__a_, __op, __m);} 1739 _LIBCPP_INLINE_VISIBILITY 1740 _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1741 {return __cxx_atomic_fetch_xor(&this->__a_, __op, __m);} 1742 _LIBCPP_INLINE_VISIBILITY 1743 _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1744 {return __cxx_atomic_fetch_xor(&this->__a_, __op, __m);} 1745 1746 _LIBCPP_INLINE_VISIBILITY 1747 _Tp operator++(int) volatile _NOEXCEPT {return fetch_add(_Tp(1));} 1748 _LIBCPP_INLINE_VISIBILITY 1749 _Tp operator++(int) _NOEXCEPT {return fetch_add(_Tp(1));} 1750 _LIBCPP_INLINE_VISIBILITY 1751 _Tp operator--(int) volatile _NOEXCEPT {return fetch_sub(_Tp(1));} 1752 _LIBCPP_INLINE_VISIBILITY 1753 _Tp operator--(int) _NOEXCEPT {return fetch_sub(_Tp(1));} 1754 _LIBCPP_INLINE_VISIBILITY 1755 _Tp operator++() volatile _NOEXCEPT {return fetch_add(_Tp(1)) + _Tp(1);} 1756 _LIBCPP_INLINE_VISIBILITY 1757 _Tp operator++() _NOEXCEPT {return fetch_add(_Tp(1)) + _Tp(1);} 1758 _LIBCPP_INLINE_VISIBILITY 1759 _Tp operator--() volatile _NOEXCEPT {return fetch_sub(_Tp(1)) - _Tp(1);} 1760 _LIBCPP_INLINE_VISIBILITY 1761 _Tp operator--() _NOEXCEPT {return fetch_sub(_Tp(1)) - _Tp(1);} 1762 _LIBCPP_INLINE_VISIBILITY 1763 _Tp operator+=(_Tp __op) volatile _NOEXCEPT {return fetch_add(__op) + __op;} 1764 _LIBCPP_INLINE_VISIBILITY 1765 _Tp operator+=(_Tp __op) _NOEXCEPT {return fetch_add(__op) + __op;} 1766 _LIBCPP_INLINE_VISIBILITY 1767 _Tp operator-=(_Tp __op) volatile _NOEXCEPT {return fetch_sub(__op) - __op;} 1768 _LIBCPP_INLINE_VISIBILITY 1769 _Tp operator-=(_Tp __op) _NOEXCEPT {return fetch_sub(__op) - __op;} 1770 _LIBCPP_INLINE_VISIBILITY 1771 _Tp operator&=(_Tp __op) volatile _NOEXCEPT {return fetch_and(__op) & __op;} 1772 _LIBCPP_INLINE_VISIBILITY 1773 _Tp operator&=(_Tp __op) _NOEXCEPT {return fetch_and(__op) & __op;} 1774 _LIBCPP_INLINE_VISIBILITY 1775 _Tp operator|=(_Tp __op) volatile _NOEXCEPT {return fetch_or(__op) | __op;} 1776 _LIBCPP_INLINE_VISIBILITY 1777 _Tp operator|=(_Tp __op) _NOEXCEPT {return fetch_or(__op) | __op;} 1778 _LIBCPP_INLINE_VISIBILITY 1779 _Tp operator^=(_Tp __op) volatile _NOEXCEPT {return fetch_xor(__op) ^ __op;} 1780 _LIBCPP_INLINE_VISIBILITY 1781 _Tp operator^=(_Tp __op) _NOEXCEPT {return fetch_xor(__op) ^ __op;} 1782}; 1783 1784// atomic<T> 1785 1786template <class _Tp> 1787struct atomic 1788 : public __atomic_base<_Tp> 1789{ 1790 typedef __atomic_base<_Tp> __base; 1791 typedef _Tp value_type; 1792 typedef value_type difference_type; 1793 _LIBCPP_INLINE_VISIBILITY 1794 atomic() _NOEXCEPT _LIBCPP_DEFAULT 1795 _LIBCPP_INLINE_VISIBILITY 1796 _LIBCPP_CONSTEXPR atomic(_Tp __d) _NOEXCEPT : __base(__d) {} 1797 1798 _LIBCPP_INLINE_VISIBILITY 1799 _Tp operator=(_Tp __d) volatile _NOEXCEPT 1800 {__base::store(__d); return __d;} 1801 _LIBCPP_INLINE_VISIBILITY 1802 _Tp operator=(_Tp __d) _NOEXCEPT 1803 {__base::store(__d); return __d;} 1804 1805 atomic& operator=(const atomic&) = delete; 1806 atomic& operator=(const atomic&) volatile = delete; 1807}; 1808 1809// atomic<T*> 1810 1811template <class _Tp> 1812struct atomic<_Tp*> 1813 : public __atomic_base<_Tp*> 1814{ 1815 typedef __atomic_base<_Tp*> __base; 1816 typedef _Tp* value_type; 1817 typedef ptrdiff_t difference_type; 1818 _LIBCPP_INLINE_VISIBILITY 1819 atomic() _NOEXCEPT _LIBCPP_DEFAULT 1820 _LIBCPP_INLINE_VISIBILITY 1821 _LIBCPP_CONSTEXPR atomic(_Tp* __d) _NOEXCEPT : __base(__d) {} 1822 1823 _LIBCPP_INLINE_VISIBILITY 1824 _Tp* operator=(_Tp* __d) volatile _NOEXCEPT 1825 {__base::store(__d); return __d;} 1826 _LIBCPP_INLINE_VISIBILITY 1827 _Tp* operator=(_Tp* __d) _NOEXCEPT 1828 {__base::store(__d); return __d;} 1829 1830 _LIBCPP_INLINE_VISIBILITY 1831 _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) 1832 volatile _NOEXCEPT 1833 {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);} 1834 _LIBCPP_INLINE_VISIBILITY 1835 _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1836 {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);} 1837 _LIBCPP_INLINE_VISIBILITY 1838 _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) 1839 volatile _NOEXCEPT 1840 {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);} 1841 _LIBCPP_INLINE_VISIBILITY 1842 _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1843 {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);} 1844 1845 _LIBCPP_INLINE_VISIBILITY 1846 _Tp* operator++(int) volatile _NOEXCEPT {return fetch_add(1);} 1847 _LIBCPP_INLINE_VISIBILITY 1848 _Tp* operator++(int) _NOEXCEPT {return fetch_add(1);} 1849 _LIBCPP_INLINE_VISIBILITY 1850 _Tp* operator--(int) volatile _NOEXCEPT {return fetch_sub(1);} 1851 _LIBCPP_INLINE_VISIBILITY 1852 _Tp* operator--(int) _NOEXCEPT {return fetch_sub(1);} 1853 _LIBCPP_INLINE_VISIBILITY 1854 _Tp* operator++() volatile _NOEXCEPT {return fetch_add(1) + 1;} 1855 _LIBCPP_INLINE_VISIBILITY 1856 _Tp* operator++() _NOEXCEPT {return fetch_add(1) + 1;} 1857 _LIBCPP_INLINE_VISIBILITY 1858 _Tp* operator--() volatile _NOEXCEPT {return fetch_sub(1) - 1;} 1859 _LIBCPP_INLINE_VISIBILITY 1860 _Tp* operator--() _NOEXCEPT {return fetch_sub(1) - 1;} 1861 _LIBCPP_INLINE_VISIBILITY 1862 _Tp* operator+=(ptrdiff_t __op) volatile _NOEXCEPT {return fetch_add(__op) + __op;} 1863 _LIBCPP_INLINE_VISIBILITY 1864 _Tp* operator+=(ptrdiff_t __op) _NOEXCEPT {return fetch_add(__op) + __op;} 1865 _LIBCPP_INLINE_VISIBILITY 1866 _Tp* operator-=(ptrdiff_t __op) volatile _NOEXCEPT {return fetch_sub(__op) - __op;} 1867 _LIBCPP_INLINE_VISIBILITY 1868 _Tp* operator-=(ptrdiff_t __op) _NOEXCEPT {return fetch_sub(__op) - __op;} 1869 1870 atomic& operator=(const atomic&) = delete; 1871 atomic& operator=(const atomic&) volatile = delete; 1872}; 1873 1874// atomic_is_lock_free 1875 1876template <class _Tp> 1877_LIBCPP_INLINE_VISIBILITY 1878bool 1879atomic_is_lock_free(const volatile atomic<_Tp>* __o) _NOEXCEPT 1880{ 1881 return __o->is_lock_free(); 1882} 1883 1884template <class _Tp> 1885_LIBCPP_INLINE_VISIBILITY 1886bool 1887atomic_is_lock_free(const atomic<_Tp>* __o) _NOEXCEPT 1888{ 1889 return __o->is_lock_free(); 1890} 1891 1892// atomic_init 1893 1894template <class _Tp> 1895_LIBCPP_INLINE_VISIBILITY 1896void 1897atomic_init(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1898{ 1899 __cxx_atomic_init(&__o->__a_, __d); 1900} 1901 1902template <class _Tp> 1903_LIBCPP_INLINE_VISIBILITY 1904void 1905atomic_init(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1906{ 1907 __cxx_atomic_init(&__o->__a_, __d); 1908} 1909 1910// atomic_store 1911 1912template <class _Tp> 1913_LIBCPP_INLINE_VISIBILITY 1914void 1915atomic_store(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1916{ 1917 __o->store(__d); 1918} 1919 1920template <class _Tp> 1921_LIBCPP_INLINE_VISIBILITY 1922void 1923atomic_store(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1924{ 1925 __o->store(__d); 1926} 1927 1928// atomic_store_explicit 1929 1930template <class _Tp> 1931_LIBCPP_INLINE_VISIBILITY 1932void 1933atomic_store_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 1934 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1935{ 1936 __o->store(__d, __m); 1937} 1938 1939template <class _Tp> 1940_LIBCPP_INLINE_VISIBILITY 1941void 1942atomic_store_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 1943 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1944{ 1945 __o->store(__d, __m); 1946} 1947 1948// atomic_load 1949 1950template <class _Tp> 1951_LIBCPP_INLINE_VISIBILITY 1952_Tp 1953atomic_load(const volatile atomic<_Tp>* __o) _NOEXCEPT 1954{ 1955 return __o->load(); 1956} 1957 1958template <class _Tp> 1959_LIBCPP_INLINE_VISIBILITY 1960_Tp 1961atomic_load(const atomic<_Tp>* __o) _NOEXCEPT 1962{ 1963 return __o->load(); 1964} 1965 1966// atomic_load_explicit 1967 1968template <class _Tp> 1969_LIBCPP_INLINE_VISIBILITY 1970_Tp 1971atomic_load_explicit(const volatile atomic<_Tp>* __o, memory_order __m) _NOEXCEPT 1972 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 1973{ 1974 return __o->load(__m); 1975} 1976 1977template <class _Tp> 1978_LIBCPP_INLINE_VISIBILITY 1979_Tp 1980atomic_load_explicit(const atomic<_Tp>* __o, memory_order __m) _NOEXCEPT 1981 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 1982{ 1983 return __o->load(__m); 1984} 1985 1986// atomic_exchange 1987 1988template <class _Tp> 1989_LIBCPP_INLINE_VISIBILITY 1990_Tp 1991atomic_exchange(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1992{ 1993 return __o->exchange(__d); 1994} 1995 1996template <class _Tp> 1997_LIBCPP_INLINE_VISIBILITY 1998_Tp 1999atomic_exchange(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2000{ 2001 return __o->exchange(__d); 2002} 2003 2004// atomic_exchange_explicit 2005 2006template <class _Tp> 2007_LIBCPP_INLINE_VISIBILITY 2008_Tp 2009atomic_exchange_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 2010{ 2011 return __o->exchange(__d, __m); 2012} 2013 2014template <class _Tp> 2015_LIBCPP_INLINE_VISIBILITY 2016_Tp 2017atomic_exchange_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 2018{ 2019 return __o->exchange(__d, __m); 2020} 2021 2022// atomic_compare_exchange_weak 2023 2024template <class _Tp> 2025_LIBCPP_INLINE_VISIBILITY 2026bool 2027atomic_compare_exchange_weak(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2028{ 2029 return __o->compare_exchange_weak(*__e, __d); 2030} 2031 2032template <class _Tp> 2033_LIBCPP_INLINE_VISIBILITY 2034bool 2035atomic_compare_exchange_weak(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2036{ 2037 return __o->compare_exchange_weak(*__e, __d); 2038} 2039 2040// atomic_compare_exchange_strong 2041 2042template <class _Tp> 2043_LIBCPP_INLINE_VISIBILITY 2044bool 2045atomic_compare_exchange_strong(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2046{ 2047 return __o->compare_exchange_strong(*__e, __d); 2048} 2049 2050template <class _Tp> 2051_LIBCPP_INLINE_VISIBILITY 2052bool 2053atomic_compare_exchange_strong(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2054{ 2055 return __o->compare_exchange_strong(*__e, __d); 2056} 2057 2058// atomic_compare_exchange_weak_explicit 2059 2060template <class _Tp> 2061_LIBCPP_INLINE_VISIBILITY 2062bool 2063atomic_compare_exchange_weak_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, 2064 typename atomic<_Tp>::value_type __d, 2065 memory_order __s, memory_order __f) _NOEXCEPT 2066 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2067{ 2068 return __o->compare_exchange_weak(*__e, __d, __s, __f); 2069} 2070 2071template <class _Tp> 2072_LIBCPP_INLINE_VISIBILITY 2073bool 2074atomic_compare_exchange_weak_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d, 2075 memory_order __s, memory_order __f) _NOEXCEPT 2076 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2077{ 2078 return __o->compare_exchange_weak(*__e, __d, __s, __f); 2079} 2080 2081// atomic_compare_exchange_strong_explicit 2082 2083template <class _Tp> 2084_LIBCPP_INLINE_VISIBILITY 2085bool 2086atomic_compare_exchange_strong_explicit(volatile atomic<_Tp>* __o, 2087 typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d, 2088 memory_order __s, memory_order __f) _NOEXCEPT 2089 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2090{ 2091 return __o->compare_exchange_strong(*__e, __d, __s, __f); 2092} 2093 2094template <class _Tp> 2095_LIBCPP_INLINE_VISIBILITY 2096bool 2097atomic_compare_exchange_strong_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, 2098 typename atomic<_Tp>::value_type __d, 2099 memory_order __s, memory_order __f) _NOEXCEPT 2100 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2101{ 2102 return __o->compare_exchange_strong(*__e, __d, __s, __f); 2103} 2104 2105// atomic_wait 2106 2107template <class _Tp> 2108_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2109void atomic_wait(const volatile atomic<_Tp>* __o, 2110 typename atomic<_Tp>::value_type __v) _NOEXCEPT 2111{ 2112 return __o->wait(__v); 2113} 2114 2115template <class _Tp> 2116_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2117void atomic_wait(const atomic<_Tp>* __o, 2118 typename atomic<_Tp>::value_type __v) _NOEXCEPT 2119{ 2120 return __o->wait(__v); 2121} 2122 2123// atomic_wait_explicit 2124 2125template <class _Tp> 2126_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2127void atomic_wait_explicit(const volatile atomic<_Tp>* __o, 2128 typename atomic<_Tp>::value_type __v, 2129 memory_order __m) _NOEXCEPT 2130 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 2131{ 2132 return __o->wait(__v, __m); 2133} 2134 2135template <class _Tp> 2136_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2137void atomic_wait_explicit(const atomic<_Tp>* __o, 2138 typename atomic<_Tp>::value_type __v, 2139 memory_order __m) _NOEXCEPT 2140 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 2141{ 2142 return __o->wait(__v, __m); 2143} 2144 2145// atomic_notify_one 2146 2147template <class _Tp> 2148_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2149void atomic_notify_one(volatile atomic<_Tp>* __o) _NOEXCEPT 2150{ 2151 __o->notify_one(); 2152} 2153template <class _Tp> 2154_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2155void atomic_notify_one(atomic<_Tp>* __o) _NOEXCEPT 2156{ 2157 __o->notify_one(); 2158} 2159 2160// atomic_notify_one 2161 2162template <class _Tp> 2163_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2164void atomic_notify_all(volatile atomic<_Tp>* __o) _NOEXCEPT 2165{ 2166 __o->notify_all(); 2167} 2168template <class _Tp> 2169_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2170void atomic_notify_all(atomic<_Tp>* __o) _NOEXCEPT 2171{ 2172 __o->notify_all(); 2173} 2174 2175// atomic_fetch_add 2176 2177template <class _Tp> 2178_LIBCPP_INLINE_VISIBILITY 2179typename enable_if 2180< 2181 is_integral<_Tp>::value && !is_same<_Tp, bool>::value && !is_const<_Tp>::value, 2182 _Tp 2183>::type 2184atomic_fetch_add(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT 2185{ 2186 return __o->fetch_add(__op); 2187} 2188 2189template <class _Tp> 2190_LIBCPP_INLINE_VISIBILITY 2191typename enable_if 2192< 2193 is_integral<_Tp>::value && !is_same<_Tp, bool>::value && !is_const<_Tp>::value, 2194 _Tp 2195>::type 2196atomic_fetch_add(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT 2197{ 2198 return __o->fetch_add(__op); 2199} 2200 2201template <class _Tp> 2202_LIBCPP_INLINE_VISIBILITY 2203_Tp* 2204atomic_fetch_add(volatile atomic<_Tp*>* __o, typename atomic<_Tp*>::difference_type __op) _NOEXCEPT 2205{ 2206 return __o->fetch_add(__op); 2207} 2208 2209template <class _Tp> 2210_LIBCPP_INLINE_VISIBILITY 2211_Tp* 2212atomic_fetch_add(atomic<_Tp*>* __o, typename atomic<_Tp*>::difference_type __op) _NOEXCEPT 2213{ 2214 return __o->fetch_add(__op); 2215} 2216 2217// atomic_fetch_add_explicit 2218 2219template <class _Tp> 2220_LIBCPP_INLINE_VISIBILITY 2221typename enable_if 2222< 2223 is_integral<_Tp>::value && !is_same<_Tp, bool>::value && !is_const<_Tp>::value, 2224 _Tp 2225>::type 2226atomic_fetch_add_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT 2227{ 2228 return __o->fetch_add(__op, __m); 2229} 2230 2231template <class _Tp> 2232_LIBCPP_INLINE_VISIBILITY 2233typename enable_if 2234< 2235 is_integral<_Tp>::value && !is_same<_Tp, bool>::value && !is_const<_Tp>::value, 2236 _Tp 2237>::type 2238atomic_fetch_add_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT 2239{ 2240 return __o->fetch_add(__op, __m); 2241} 2242 2243template <class _Tp> 2244_LIBCPP_INLINE_VISIBILITY 2245_Tp* 2246atomic_fetch_add_explicit(volatile atomic<_Tp*>* __o, typename atomic<_Tp*>::difference_type __op, memory_order __m) _NOEXCEPT 2247{ 2248 return __o->fetch_add(__op, __m); 2249} 2250 2251template <class _Tp> 2252_LIBCPP_INLINE_VISIBILITY 2253_Tp* 2254atomic_fetch_add_explicit(atomic<_Tp*>* __o, typename atomic<_Tp*>::difference_type __op, memory_order __m) _NOEXCEPT 2255{ 2256 return __o->fetch_add(__op, __m); 2257} 2258 2259// atomic_fetch_sub 2260 2261template <class _Tp> 2262_LIBCPP_INLINE_VISIBILITY 2263typename enable_if 2264< 2265 is_integral<_Tp>::value && !is_same<_Tp, bool>::value && !is_const<_Tp>::value, 2266 _Tp 2267>::type 2268atomic_fetch_sub(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT 2269{ 2270 return __o->fetch_sub(__op); 2271} 2272 2273template <class _Tp> 2274_LIBCPP_INLINE_VISIBILITY 2275typename enable_if 2276< 2277 is_integral<_Tp>::value && !is_same<_Tp, bool>::value && !is_const<_Tp>::value, 2278 _Tp 2279>::type 2280atomic_fetch_sub(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT 2281{ 2282 return __o->fetch_sub(__op); 2283} 2284 2285template <class _Tp> 2286_LIBCPP_INLINE_VISIBILITY 2287_Tp* 2288atomic_fetch_sub(volatile atomic<_Tp*>* __o, typename atomic<_Tp*>::difference_type __op) _NOEXCEPT 2289{ 2290 return __o->fetch_sub(__op); 2291} 2292 2293template <class _Tp> 2294_LIBCPP_INLINE_VISIBILITY 2295_Tp* 2296atomic_fetch_sub(atomic<_Tp*>* __o, typename atomic<_Tp*>::difference_type __op) _NOEXCEPT 2297{ 2298 return __o->fetch_sub(__op); 2299} 2300 2301// atomic_fetch_sub_explicit 2302 2303template <class _Tp> 2304_LIBCPP_INLINE_VISIBILITY 2305typename enable_if 2306< 2307 is_integral<_Tp>::value && !is_same<_Tp, bool>::value && !is_const<_Tp>::value, 2308 _Tp 2309>::type 2310atomic_fetch_sub_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT 2311{ 2312 return __o->fetch_sub(__op, __m); 2313} 2314 2315template <class _Tp> 2316_LIBCPP_INLINE_VISIBILITY 2317typename enable_if 2318< 2319 is_integral<_Tp>::value && !is_same<_Tp, bool>::value && !is_const<_Tp>::value, 2320 _Tp 2321>::type 2322atomic_fetch_sub_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT 2323{ 2324 return __o->fetch_sub(__op, __m); 2325} 2326 2327template <class _Tp> 2328_LIBCPP_INLINE_VISIBILITY 2329_Tp* 2330atomic_fetch_sub_explicit(volatile atomic<_Tp*>* __o, typename atomic<_Tp*>::difference_type __op, memory_order __m) _NOEXCEPT 2331{ 2332 return __o->fetch_sub(__op, __m); 2333} 2334 2335template <class _Tp> 2336_LIBCPP_INLINE_VISIBILITY 2337_Tp* 2338atomic_fetch_sub_explicit(atomic<_Tp*>* __o, typename atomic<_Tp*>::difference_type __op, memory_order __m) _NOEXCEPT 2339{ 2340 return __o->fetch_sub(__op, __m); 2341} 2342 2343// atomic_fetch_and 2344 2345template <class _Tp> 2346_LIBCPP_INLINE_VISIBILITY 2347typename enable_if 2348< 2349 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2350 _Tp 2351>::type 2352atomic_fetch_and(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2353{ 2354 return __o->fetch_and(__op); 2355} 2356 2357template <class _Tp> 2358_LIBCPP_INLINE_VISIBILITY 2359typename enable_if 2360< 2361 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2362 _Tp 2363>::type 2364atomic_fetch_and(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2365{ 2366 return __o->fetch_and(__op); 2367} 2368 2369// atomic_fetch_and_explicit 2370 2371template <class _Tp> 2372_LIBCPP_INLINE_VISIBILITY 2373typename enable_if 2374< 2375 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2376 _Tp 2377>::type 2378atomic_fetch_and_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2379{ 2380 return __o->fetch_and(__op, __m); 2381} 2382 2383template <class _Tp> 2384_LIBCPP_INLINE_VISIBILITY 2385typename enable_if 2386< 2387 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2388 _Tp 2389>::type 2390atomic_fetch_and_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2391{ 2392 return __o->fetch_and(__op, __m); 2393} 2394 2395// atomic_fetch_or 2396 2397template <class _Tp> 2398_LIBCPP_INLINE_VISIBILITY 2399typename enable_if 2400< 2401 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2402 _Tp 2403>::type 2404atomic_fetch_or(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2405{ 2406 return __o->fetch_or(__op); 2407} 2408 2409template <class _Tp> 2410_LIBCPP_INLINE_VISIBILITY 2411typename enable_if 2412< 2413 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2414 _Tp 2415>::type 2416atomic_fetch_or(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2417{ 2418 return __o->fetch_or(__op); 2419} 2420 2421// atomic_fetch_or_explicit 2422 2423template <class _Tp> 2424_LIBCPP_INLINE_VISIBILITY 2425typename enable_if 2426< 2427 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2428 _Tp 2429>::type 2430atomic_fetch_or_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2431{ 2432 return __o->fetch_or(__op, __m); 2433} 2434 2435template <class _Tp> 2436_LIBCPP_INLINE_VISIBILITY 2437typename enable_if 2438< 2439 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2440 _Tp 2441>::type 2442atomic_fetch_or_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2443{ 2444 return __o->fetch_or(__op, __m); 2445} 2446 2447// atomic_fetch_xor 2448 2449template <class _Tp> 2450_LIBCPP_INLINE_VISIBILITY 2451typename enable_if 2452< 2453 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2454 _Tp 2455>::type 2456atomic_fetch_xor(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2457{ 2458 return __o->fetch_xor(__op); 2459} 2460 2461template <class _Tp> 2462_LIBCPP_INLINE_VISIBILITY 2463typename enable_if 2464< 2465 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2466 _Tp 2467>::type 2468atomic_fetch_xor(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2469{ 2470 return __o->fetch_xor(__op); 2471} 2472 2473// atomic_fetch_xor_explicit 2474 2475template <class _Tp> 2476_LIBCPP_INLINE_VISIBILITY 2477typename enable_if 2478< 2479 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2480 _Tp 2481>::type 2482atomic_fetch_xor_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2483{ 2484 return __o->fetch_xor(__op, __m); 2485} 2486 2487template <class _Tp> 2488_LIBCPP_INLINE_VISIBILITY 2489typename enable_if 2490< 2491 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2492 _Tp 2493>::type 2494atomic_fetch_xor_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2495{ 2496 return __o->fetch_xor(__op, __m); 2497} 2498 2499// flag type and operations 2500 2501typedef struct atomic_flag 2502{ 2503 __cxx_atomic_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_; 2504 2505 _LIBCPP_INLINE_VISIBILITY 2506 bool test(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 2507 {return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m);} 2508 _LIBCPP_INLINE_VISIBILITY 2509 bool test(memory_order __m = memory_order_seq_cst) const _NOEXCEPT 2510 {return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m);} 2511 2512 _LIBCPP_INLINE_VISIBILITY 2513 bool test_and_set(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 2514 {return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m);} 2515 _LIBCPP_INLINE_VISIBILITY 2516 bool test_and_set(memory_order __m = memory_order_seq_cst) _NOEXCEPT 2517 {return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m);} 2518 _LIBCPP_INLINE_VISIBILITY 2519 void clear(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 2520 {__cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m);} 2521 _LIBCPP_INLINE_VISIBILITY 2522 void clear(memory_order __m = memory_order_seq_cst) _NOEXCEPT 2523 {__cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m);} 2524 2525 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2526 void wait(bool __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 2527 {__cxx_atomic_wait(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);} 2528 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2529 void wait(bool __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT 2530 {__cxx_atomic_wait(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);} 2531 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2532 void notify_one() volatile _NOEXCEPT 2533 {__cxx_atomic_notify_one(&__a_);} 2534 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2535 void notify_one() _NOEXCEPT 2536 {__cxx_atomic_notify_one(&__a_);} 2537 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2538 void notify_all() volatile _NOEXCEPT 2539 {__cxx_atomic_notify_all(&__a_);} 2540 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2541 void notify_all() _NOEXCEPT 2542 {__cxx_atomic_notify_all(&__a_);} 2543 2544 _LIBCPP_INLINE_VISIBILITY 2545 atomic_flag() _NOEXCEPT _LIBCPP_DEFAULT 2546 2547 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR 2548 atomic_flag(bool __b) _NOEXCEPT : __a_(__b) {} // EXTENSION 2549 2550#ifndef _LIBCPP_CXX03_LANG 2551 atomic_flag(const atomic_flag&) = delete; 2552 atomic_flag& operator=(const atomic_flag&) = delete; 2553 atomic_flag& operator=(const atomic_flag&) volatile = delete; 2554#else 2555private: 2556 _LIBCPP_INLINE_VISIBILITY 2557 atomic_flag(const atomic_flag&); 2558 _LIBCPP_INLINE_VISIBILITY 2559 atomic_flag& operator=(const atomic_flag&); 2560 _LIBCPP_INLINE_VISIBILITY 2561 atomic_flag& operator=(const atomic_flag&) volatile; 2562#endif 2563} atomic_flag; 2564 2565 2566inline _LIBCPP_INLINE_VISIBILITY 2567bool 2568atomic_flag_test(const volatile atomic_flag* __o) _NOEXCEPT 2569{ 2570 return __o->test(); 2571} 2572 2573inline _LIBCPP_INLINE_VISIBILITY 2574bool 2575atomic_flag_test(const atomic_flag* __o) _NOEXCEPT 2576{ 2577 return __o->test(); 2578} 2579 2580inline _LIBCPP_INLINE_VISIBILITY 2581bool 2582atomic_flag_test_explicit(const volatile atomic_flag* __o, memory_order __m) _NOEXCEPT 2583{ 2584 return __o->test(__m); 2585} 2586 2587inline _LIBCPP_INLINE_VISIBILITY 2588bool 2589atomic_flag_test_explicit(const atomic_flag* __o, memory_order __m) _NOEXCEPT 2590{ 2591 return __o->test(__m); 2592} 2593 2594inline _LIBCPP_INLINE_VISIBILITY 2595bool 2596atomic_flag_test_and_set(volatile atomic_flag* __o) _NOEXCEPT 2597{ 2598 return __o->test_and_set(); 2599} 2600 2601inline _LIBCPP_INLINE_VISIBILITY 2602bool 2603atomic_flag_test_and_set(atomic_flag* __o) _NOEXCEPT 2604{ 2605 return __o->test_and_set(); 2606} 2607 2608inline _LIBCPP_INLINE_VISIBILITY 2609bool 2610atomic_flag_test_and_set_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT 2611{ 2612 return __o->test_and_set(__m); 2613} 2614 2615inline _LIBCPP_INLINE_VISIBILITY 2616bool 2617atomic_flag_test_and_set_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT 2618{ 2619 return __o->test_and_set(__m); 2620} 2621 2622inline _LIBCPP_INLINE_VISIBILITY 2623void 2624atomic_flag_clear(volatile atomic_flag* __o) _NOEXCEPT 2625{ 2626 __o->clear(); 2627} 2628 2629inline _LIBCPP_INLINE_VISIBILITY 2630void 2631atomic_flag_clear(atomic_flag* __o) _NOEXCEPT 2632{ 2633 __o->clear(); 2634} 2635 2636inline _LIBCPP_INLINE_VISIBILITY 2637void 2638atomic_flag_clear_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT 2639{ 2640 __o->clear(__m); 2641} 2642 2643inline _LIBCPP_INLINE_VISIBILITY 2644void 2645atomic_flag_clear_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT 2646{ 2647 __o->clear(__m); 2648} 2649 2650inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2651void 2652atomic_flag_wait(const volatile atomic_flag* __o, bool __v) _NOEXCEPT 2653{ 2654 __o->wait(__v); 2655} 2656 2657inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2658void 2659atomic_flag_wait(const atomic_flag* __o, bool __v) _NOEXCEPT 2660{ 2661 __o->wait(__v); 2662} 2663 2664inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2665void 2666atomic_flag_wait_explicit(const volatile atomic_flag* __o, 2667 bool __v, memory_order __m) _NOEXCEPT 2668{ 2669 __o->wait(__v, __m); 2670} 2671 2672inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2673void 2674atomic_flag_wait_explicit(const atomic_flag* __o, 2675 bool __v, memory_order __m) _NOEXCEPT 2676{ 2677 __o->wait(__v, __m); 2678} 2679 2680inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2681void 2682atomic_flag_notify_one(volatile atomic_flag* __o) _NOEXCEPT 2683{ 2684 __o->notify_one(); 2685} 2686 2687inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2688void 2689atomic_flag_notify_one(atomic_flag* __o) _NOEXCEPT 2690{ 2691 __o->notify_one(); 2692} 2693 2694inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2695void 2696atomic_flag_notify_all(volatile atomic_flag* __o) _NOEXCEPT 2697{ 2698 __o->notify_all(); 2699} 2700 2701inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2702void 2703atomic_flag_notify_all(atomic_flag* __o) _NOEXCEPT 2704{ 2705 __o->notify_all(); 2706} 2707 2708// fences 2709 2710inline _LIBCPP_INLINE_VISIBILITY 2711void 2712atomic_thread_fence(memory_order __m) _NOEXCEPT 2713{ 2714 __cxx_atomic_thread_fence(__m); 2715} 2716 2717inline _LIBCPP_INLINE_VISIBILITY 2718void 2719atomic_signal_fence(memory_order __m) _NOEXCEPT 2720{ 2721 __cxx_atomic_signal_fence(__m); 2722} 2723 2724// Atomics for standard typedef types 2725 2726typedef atomic<bool> atomic_bool; 2727typedef atomic<char> atomic_char; 2728typedef atomic<signed char> atomic_schar; 2729typedef atomic<unsigned char> atomic_uchar; 2730typedef atomic<short> atomic_short; 2731typedef atomic<unsigned short> atomic_ushort; 2732typedef atomic<int> atomic_int; 2733typedef atomic<unsigned int> atomic_uint; 2734typedef atomic<long> atomic_long; 2735typedef atomic<unsigned long> atomic_ulong; 2736typedef atomic<long long> atomic_llong; 2737typedef atomic<unsigned long long> atomic_ullong; 2738#ifndef _LIBCPP_HAS_NO_CHAR8_T 2739typedef atomic<char8_t> atomic_char8_t; 2740#endif 2741typedef atomic<char16_t> atomic_char16_t; 2742typedef atomic<char32_t> atomic_char32_t; 2743typedef atomic<wchar_t> atomic_wchar_t; 2744 2745typedef atomic<int_least8_t> atomic_int_least8_t; 2746typedef atomic<uint_least8_t> atomic_uint_least8_t; 2747typedef atomic<int_least16_t> atomic_int_least16_t; 2748typedef atomic<uint_least16_t> atomic_uint_least16_t; 2749typedef atomic<int_least32_t> atomic_int_least32_t; 2750typedef atomic<uint_least32_t> atomic_uint_least32_t; 2751typedef atomic<int_least64_t> atomic_int_least64_t; 2752typedef atomic<uint_least64_t> atomic_uint_least64_t; 2753 2754typedef atomic<int_fast8_t> atomic_int_fast8_t; 2755typedef atomic<uint_fast8_t> atomic_uint_fast8_t; 2756typedef atomic<int_fast16_t> atomic_int_fast16_t; 2757typedef atomic<uint_fast16_t> atomic_uint_fast16_t; 2758typedef atomic<int_fast32_t> atomic_int_fast32_t; 2759typedef atomic<uint_fast32_t> atomic_uint_fast32_t; 2760typedef atomic<int_fast64_t> atomic_int_fast64_t; 2761typedef atomic<uint_fast64_t> atomic_uint_fast64_t; 2762 2763typedef atomic< int8_t> atomic_int8_t; 2764typedef atomic<uint8_t> atomic_uint8_t; 2765typedef atomic< int16_t> atomic_int16_t; 2766typedef atomic<uint16_t> atomic_uint16_t; 2767typedef atomic< int32_t> atomic_int32_t; 2768typedef atomic<uint32_t> atomic_uint32_t; 2769typedef atomic< int64_t> atomic_int64_t; 2770typedef atomic<uint64_t> atomic_uint64_t; 2771 2772typedef atomic<intptr_t> atomic_intptr_t; 2773typedef atomic<uintptr_t> atomic_uintptr_t; 2774typedef atomic<size_t> atomic_size_t; 2775typedef atomic<ptrdiff_t> atomic_ptrdiff_t; 2776typedef atomic<intmax_t> atomic_intmax_t; 2777typedef atomic<uintmax_t> atomic_uintmax_t; 2778 2779// atomic_*_lock_free : prefer the contention type most highly, then the largest lock-free type 2780 2781#ifdef __cpp_lib_atomic_is_always_lock_free 2782# define _LIBCPP_CONTENTION_LOCK_FREE __atomic_always_lock_free(sizeof(__cxx_contention_t), 0) 2783#else 2784# define _LIBCPP_CONTENTION_LOCK_FREE false 2785#endif 2786 2787#if ATOMIC_LLONG_LOCK_FREE == 2 2788typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, long long>::type __libcpp_signed_lock_free; 2789typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned long long>::type __libcpp_unsigned_lock_free; 2790#elif ATOMIC_INT_LOCK_FREE == 2 2791typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, int>::type __libcpp_signed_lock_free; 2792typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned int>::type __libcpp_unsigned_lock_free; 2793#elif ATOMIC_SHORT_LOCK_FREE == 2 2794typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, short>::type __libcpp_signed_lock_free; 2795typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned short>::type __libcpp_unsigned_lock_free; 2796#elif ATOMIC_CHAR_LOCK_FREE == 2 2797typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, char>::type __libcpp_signed_lock_free; 2798typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned char>::type __libcpp_unsigned_lock_free; 2799#else 2800 // No signed/unsigned lock-free types 2801#endif 2802 2803typedef atomic<__libcpp_signed_lock_free> atomic_signed_lock_free; 2804typedef atomic<__libcpp_unsigned_lock_free> atomic_unsigned_lock_free; 2805 2806#define ATOMIC_FLAG_INIT {false} 2807#define ATOMIC_VAR_INIT(__v) {__v} 2808 2809_LIBCPP_END_NAMESPACE_STD 2810 2811#endif // _LIBCPP_ATOMIC 2812