1// -*- C++ -*- 2//===--------------------------- atomic -----------------------------------===// 3// 4// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 5// See https://llvm.org/LICENSE.txt for license information. 6// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 7// 8//===----------------------------------------------------------------------===// 9 10#ifndef _LIBCPP_ATOMIC 11#define _LIBCPP_ATOMIC 12 13/* 14 atomic synopsis 15 16namespace std 17{ 18 19// feature test macro [version.syn] 20 21#define __cpp_lib_atomic_is_always_lock_free 22#define __cpp_lib_atomic_flag_test 23#define __cpp_lib_atomic_lock_free_type_aliases 24#define __cpp_lib_atomic_wait 25 26 // order and consistency 27 28 enum memory_order: unspecified // enum class in C++20 29 { 30 relaxed, 31 consume, // load-consume 32 acquire, // load-acquire 33 release, // store-release 34 acq_rel, // store-release load-acquire 35 seq_cst // store-release load-acquire 36 }; 37 38 inline constexpr auto memory_order_relaxed = memory_order::relaxed; 39 inline constexpr auto memory_order_consume = memory_order::consume; 40 inline constexpr auto memory_order_acquire = memory_order::acquire; 41 inline constexpr auto memory_order_release = memory_order::release; 42 inline constexpr auto memory_order_acq_rel = memory_order::acq_rel; 43 inline constexpr auto memory_order_seq_cst = memory_order::seq_cst; 44 45template <class T> T kill_dependency(T y) noexcept; 46 47// lock-free property 48 49#define ATOMIC_BOOL_LOCK_FREE unspecified 50#define ATOMIC_CHAR_LOCK_FREE unspecified 51#define ATOMIC_CHAR8_T_LOCK_FREE unspecified // C++20 52#define ATOMIC_CHAR16_T_LOCK_FREE unspecified 53#define ATOMIC_CHAR32_T_LOCK_FREE unspecified 54#define ATOMIC_WCHAR_T_LOCK_FREE unspecified 55#define ATOMIC_SHORT_LOCK_FREE unspecified 56#define ATOMIC_INT_LOCK_FREE unspecified 57#define ATOMIC_LONG_LOCK_FREE unspecified 58#define ATOMIC_LLONG_LOCK_FREE unspecified 59#define ATOMIC_POINTER_LOCK_FREE unspecified 60 61template <class T> 62struct atomic 63{ 64 using value_type = T; 65 66 static constexpr bool is_always_lock_free; 67 bool is_lock_free() const volatile noexcept; 68 bool is_lock_free() const noexcept; 69 70 atomic() noexcept = default; // until C++20 71 constexpr atomic() noexcept(is_nothrow_default_constructible_v<T>); // since C++20 72 constexpr atomic(T desr) noexcept; 73 atomic(const atomic&) = delete; 74 atomic& operator=(const atomic&) = delete; 75 atomic& operator=(const atomic&) volatile = delete; 76 77 T load(memory_order m = memory_order_seq_cst) const volatile noexcept; 78 T load(memory_order m = memory_order_seq_cst) const noexcept; 79 operator T() const volatile noexcept; 80 operator T() const noexcept; 81 void store(T desr, memory_order m = memory_order_seq_cst) volatile noexcept; 82 void store(T desr, memory_order m = memory_order_seq_cst) noexcept; 83 T operator=(T) volatile noexcept; 84 T operator=(T) noexcept; 85 86 T exchange(T desr, memory_order m = memory_order_seq_cst) volatile noexcept; 87 T exchange(T desr, memory_order m = memory_order_seq_cst) noexcept; 88 bool compare_exchange_weak(T& expc, T desr, 89 memory_order s, memory_order f) volatile noexcept; 90 bool compare_exchange_weak(T& expc, T desr, memory_order s, memory_order f) noexcept; 91 bool compare_exchange_strong(T& expc, T desr, 92 memory_order s, memory_order f) volatile noexcept; 93 bool compare_exchange_strong(T& expc, T desr, 94 memory_order s, memory_order f) noexcept; 95 bool compare_exchange_weak(T& expc, T desr, 96 memory_order m = memory_order_seq_cst) volatile noexcept; 97 bool compare_exchange_weak(T& expc, T desr, 98 memory_order m = memory_order_seq_cst) noexcept; 99 bool compare_exchange_strong(T& expc, T desr, 100 memory_order m = memory_order_seq_cst) volatile noexcept; 101 bool compare_exchange_strong(T& expc, T desr, 102 memory_order m = memory_order_seq_cst) noexcept; 103 104 void wait(T, memory_order = memory_order::seq_cst) const volatile noexcept; 105 void wait(T, memory_order = memory_order::seq_cst) const noexcept; 106 void notify_one() volatile noexcept; 107 void notify_one() noexcept; 108 void notify_all() volatile noexcept; 109 void notify_all() noexcept; 110}; 111 112template <> 113struct atomic<integral> 114{ 115 using value_type = integral; 116 using difference_type = value_type; 117 118 static constexpr bool is_always_lock_free; 119 bool is_lock_free() const volatile noexcept; 120 bool is_lock_free() const noexcept; 121 122 atomic() noexcept = default; 123 constexpr atomic(integral desr) noexcept; 124 atomic(const atomic&) = delete; 125 atomic& operator=(const atomic&) = delete; 126 atomic& operator=(const atomic&) volatile = delete; 127 128 integral load(memory_order m = memory_order_seq_cst) const volatile noexcept; 129 integral load(memory_order m = memory_order_seq_cst) const noexcept; 130 operator integral() const volatile noexcept; 131 operator integral() const noexcept; 132 void store(integral desr, memory_order m = memory_order_seq_cst) volatile noexcept; 133 void store(integral desr, memory_order m = memory_order_seq_cst) noexcept; 134 integral operator=(integral desr) volatile noexcept; 135 integral operator=(integral desr) noexcept; 136 137 integral exchange(integral desr, 138 memory_order m = memory_order_seq_cst) volatile noexcept; 139 integral exchange(integral desr, memory_order m = memory_order_seq_cst) noexcept; 140 bool compare_exchange_weak(integral& expc, integral desr, 141 memory_order s, memory_order f) volatile noexcept; 142 bool compare_exchange_weak(integral& expc, integral desr, 143 memory_order s, memory_order f) noexcept; 144 bool compare_exchange_strong(integral& expc, integral desr, 145 memory_order s, memory_order f) volatile noexcept; 146 bool compare_exchange_strong(integral& expc, integral desr, 147 memory_order s, memory_order f) noexcept; 148 bool compare_exchange_weak(integral& expc, integral desr, 149 memory_order m = memory_order_seq_cst) volatile noexcept; 150 bool compare_exchange_weak(integral& expc, integral desr, 151 memory_order m = memory_order_seq_cst) noexcept; 152 bool compare_exchange_strong(integral& expc, integral desr, 153 memory_order m = memory_order_seq_cst) volatile noexcept; 154 bool compare_exchange_strong(integral& expc, integral desr, 155 memory_order m = memory_order_seq_cst) noexcept; 156 157 integral fetch_add(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 158 integral fetch_add(integral op, memory_order m = memory_order_seq_cst) noexcept; 159 integral fetch_sub(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 160 integral fetch_sub(integral op, memory_order m = memory_order_seq_cst) noexcept; 161 integral fetch_and(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 162 integral fetch_and(integral op, memory_order m = memory_order_seq_cst) noexcept; 163 integral fetch_or(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 164 integral fetch_or(integral op, memory_order m = memory_order_seq_cst) noexcept; 165 integral fetch_xor(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 166 integral fetch_xor(integral op, memory_order m = memory_order_seq_cst) noexcept; 167 168 integral operator++(int) volatile noexcept; 169 integral operator++(int) noexcept; 170 integral operator--(int) volatile noexcept; 171 integral operator--(int) noexcept; 172 integral operator++() volatile noexcept; 173 integral operator++() noexcept; 174 integral operator--() volatile noexcept; 175 integral operator--() noexcept; 176 integral operator+=(integral op) volatile noexcept; 177 integral operator+=(integral op) noexcept; 178 integral operator-=(integral op) volatile noexcept; 179 integral operator-=(integral op) noexcept; 180 integral operator&=(integral op) volatile noexcept; 181 integral operator&=(integral op) noexcept; 182 integral operator|=(integral op) volatile noexcept; 183 integral operator|=(integral op) noexcept; 184 integral operator^=(integral op) volatile noexcept; 185 integral operator^=(integral op) noexcept; 186 187 void wait(integral, memory_order = memory_order::seq_cst) const volatile noexcept; 188 void wait(integral, memory_order = memory_order::seq_cst) const noexcept; 189 void notify_one() volatile noexcept; 190 void notify_one() noexcept; 191 void notify_all() volatile noexcept; 192 void notify_all() noexcept; 193}; 194 195template <class T> 196struct atomic<T*> 197{ 198 using value_type = T*; 199 using difference_type = ptrdiff_t; 200 201 static constexpr bool is_always_lock_free; 202 bool is_lock_free() const volatile noexcept; 203 bool is_lock_free() const noexcept; 204 205 atomic() noexcept = default; // until C++20 206 constexpr atomic() noexcept; // since C++20 207 constexpr atomic(T* desr) noexcept; 208 atomic(const atomic&) = delete; 209 atomic& operator=(const atomic&) = delete; 210 atomic& operator=(const atomic&) volatile = delete; 211 212 T* load(memory_order m = memory_order_seq_cst) const volatile noexcept; 213 T* load(memory_order m = memory_order_seq_cst) const noexcept; 214 operator T*() const volatile noexcept; 215 operator T*() const noexcept; 216 void store(T* desr, memory_order m = memory_order_seq_cst) volatile noexcept; 217 void store(T* desr, memory_order m = memory_order_seq_cst) noexcept; 218 T* operator=(T*) volatile noexcept; 219 T* operator=(T*) noexcept; 220 221 T* exchange(T* desr, memory_order m = memory_order_seq_cst) volatile noexcept; 222 T* exchange(T* desr, memory_order m = memory_order_seq_cst) noexcept; 223 bool compare_exchange_weak(T*& expc, T* desr, 224 memory_order s, memory_order f) volatile noexcept; 225 bool compare_exchange_weak(T*& expc, T* desr, 226 memory_order s, memory_order f) noexcept; 227 bool compare_exchange_strong(T*& expc, T* desr, 228 memory_order s, memory_order f) volatile noexcept; 229 bool compare_exchange_strong(T*& expc, T* desr, 230 memory_order s, memory_order f) noexcept; 231 bool compare_exchange_weak(T*& expc, T* desr, 232 memory_order m = memory_order_seq_cst) volatile noexcept; 233 bool compare_exchange_weak(T*& expc, T* desr, 234 memory_order m = memory_order_seq_cst) noexcept; 235 bool compare_exchange_strong(T*& expc, T* desr, 236 memory_order m = memory_order_seq_cst) volatile noexcept; 237 bool compare_exchange_strong(T*& expc, T* desr, 238 memory_order m = memory_order_seq_cst) noexcept; 239 T* fetch_add(ptrdiff_t op, memory_order m = memory_order_seq_cst) volatile noexcept; 240 T* fetch_add(ptrdiff_t op, memory_order m = memory_order_seq_cst) noexcept; 241 T* fetch_sub(ptrdiff_t op, memory_order m = memory_order_seq_cst) volatile noexcept; 242 T* fetch_sub(ptrdiff_t op, memory_order m = memory_order_seq_cst) noexcept; 243 244 T* operator++(int) volatile noexcept; 245 T* operator++(int) noexcept; 246 T* operator--(int) volatile noexcept; 247 T* operator--(int) noexcept; 248 T* operator++() volatile noexcept; 249 T* operator++() noexcept; 250 T* operator--() volatile noexcept; 251 T* operator--() noexcept; 252 T* operator+=(ptrdiff_t op) volatile noexcept; 253 T* operator+=(ptrdiff_t op) noexcept; 254 T* operator-=(ptrdiff_t op) volatile noexcept; 255 T* operator-=(ptrdiff_t op) noexcept; 256 257 void wait(T*, memory_order = memory_order::seq_cst) const volatile noexcept; 258 void wait(T*, memory_order = memory_order::seq_cst) const noexcept; 259 void notify_one() volatile noexcept; 260 void notify_one() noexcept; 261 void notify_all() volatile noexcept; 262 void notify_all() noexcept; 263}; 264 265 266template <class T> 267 bool atomic_is_lock_free(const volatile atomic<T>* obj) noexcept; 268 269template <class T> 270 bool atomic_is_lock_free(const atomic<T>* obj) noexcept; 271 272template <class T> 273 void atomic_store(volatile atomic<T>* obj, T desr) noexcept; 274 275template <class T> 276 void atomic_store(atomic<T>* obj, T desr) noexcept; 277 278template <class T> 279 void atomic_store_explicit(volatile atomic<T>* obj, T desr, memory_order m) noexcept; 280 281template <class T> 282 void atomic_store_explicit(atomic<T>* obj, T desr, memory_order m) noexcept; 283 284template <class T> 285 T atomic_load(const volatile atomic<T>* obj) noexcept; 286 287template <class T> 288 T atomic_load(const atomic<T>* obj) noexcept; 289 290template <class T> 291 T atomic_load_explicit(const volatile atomic<T>* obj, memory_order m) noexcept; 292 293template <class T> 294 T atomic_load_explicit(const atomic<T>* obj, memory_order m) noexcept; 295 296template <class T> 297 T atomic_exchange(volatile atomic<T>* obj, T desr) noexcept; 298 299template <class T> 300 T atomic_exchange(atomic<T>* obj, T desr) noexcept; 301 302template <class T> 303 T atomic_exchange_explicit(volatile atomic<T>* obj, T desr, memory_order m) noexcept; 304 305template <class T> 306 T atomic_exchange_explicit(atomic<T>* obj, T desr, memory_order m) noexcept; 307 308template <class T> 309 bool atomic_compare_exchange_weak(volatile atomic<T>* obj, T* expc, T desr) noexcept; 310 311template <class T> 312 bool atomic_compare_exchange_weak(atomic<T>* obj, T* expc, T desr) noexcept; 313 314template <class T> 315 bool atomic_compare_exchange_strong(volatile atomic<T>* obj, T* expc, T desr) noexcept; 316 317template <class T> 318 bool atomic_compare_exchange_strong(atomic<T>* obj, T* expc, T desr) noexcept; 319 320template <class T> 321 bool atomic_compare_exchange_weak_explicit(volatile atomic<T>* obj, T* expc, 322 T desr, 323 memory_order s, memory_order f) noexcept; 324 325template <class T> 326 bool atomic_compare_exchange_weak_explicit(atomic<T>* obj, T* expc, T desr, 327 memory_order s, memory_order f) noexcept; 328 329template <class T> 330 bool atomic_compare_exchange_strong_explicit(volatile atomic<T>* obj, 331 T* expc, T desr, 332 memory_order s, memory_order f) noexcept; 333 334template <class T> 335 bool atomic_compare_exchange_strong_explicit(atomic<T>* obj, T* expc, 336 T desr, 337 memory_order s, memory_order f) noexcept; 338 339template <class T> 340 void atomic_wait(const volatile atomic<T>* obj, T old) noexcept; 341 342template <class T> 343 void atomic_wait(const atomic<T>* obj, T old) noexcept; 344 345template <class T> 346 void atomic_wait_explicit(const volatile atomic<T>* obj, T old, memory_order m) noexcept; 347 348template <class T> 349 void atomic_wait_explicit(const atomic<T>* obj, T old, memory_order m) noexcept; 350 351template <class T> 352 void atomic_one(volatile atomic<T>* obj) noexcept; 353 354template <class T> 355 void atomic_one(atomic<T>* obj) noexcept; 356 357template <class T> 358 void atomic_all(volatile atomic<T>* obj) noexcept; 359 360template <class T> 361 void atomic_all(atomic<T>* obj) noexcept; 362 363template <class Integral> 364 Integral atomic_fetch_add(volatile atomic<Integral>* obj, Integral op) noexcept; 365 366template <class Integral> 367 Integral atomic_fetch_add(atomic<Integral>* obj, Integral op) noexcept; 368 369template <class Integral> 370 Integral atomic_fetch_add_explicit(volatile atomic<Integral>* obj, Integral op, 371 memory_order m) noexcept; 372template <class Integral> 373 Integral atomic_fetch_add_explicit(atomic<Integral>* obj, Integral op, 374 memory_order m) noexcept; 375template <class Integral> 376 Integral atomic_fetch_sub(volatile atomic<Integral>* obj, Integral op) noexcept; 377 378template <class Integral> 379 Integral atomic_fetch_sub(atomic<Integral>* obj, Integral op) noexcept; 380 381template <class Integral> 382 Integral atomic_fetch_sub_explicit(volatile atomic<Integral>* obj, Integral op, 383 memory_order m) noexcept; 384 385template <class Integral> 386 Integral atomic_fetch_sub_explicit(atomic<Integral>* obj, Integral op, 387 memory_order m) noexcept; 388 389template <class Integral> 390 Integral atomic_fetch_and(volatile atomic<Integral>* obj, Integral op) noexcept; 391 392template <class Integral> 393 Integral atomic_fetch_and(atomic<Integral>* obj, Integral op) noexcept; 394 395template <class Integral> 396 Integral atomic_fetch_and_explicit(volatile atomic<Integral>* obj, Integral op, 397 memory_order m) noexcept; 398 399template <class Integral> 400 Integral atomic_fetch_and_explicit(atomic<Integral>* obj, Integral op, 401 memory_order m) noexcept; 402 403template <class Integral> 404 Integral atomic_fetch_or(volatile atomic<Integral>* obj, Integral op) noexcept; 405 406template <class Integral> 407 Integral atomic_fetch_or(atomic<Integral>* obj, Integral op) noexcept; 408 409template <class Integral> 410 Integral atomic_fetch_or_explicit(volatile atomic<Integral>* obj, Integral op, 411 memory_order m) noexcept; 412 413template <class Integral> 414 Integral atomic_fetch_or_explicit(atomic<Integral>* obj, Integral op, 415 memory_order m) noexcept; 416 417template <class Integral> 418 Integral atomic_fetch_xor(volatile atomic<Integral>* obj, Integral op) noexcept; 419 420template <class Integral> 421 Integral atomic_fetch_xor(atomic<Integral>* obj, Integral op) noexcept; 422 423template <class Integral> 424 Integral atomic_fetch_xor_explicit(volatile atomic<Integral>* obj, Integral op, 425 memory_order m) noexcept; 426 427template <class Integral> 428 Integral atomic_fetch_xor_explicit(atomic<Integral>* obj, Integral op, 429 memory_order m) noexcept; 430 431template <class T> 432 T* atomic_fetch_add(volatile atomic<T*>* obj, ptrdiff_t op) noexcept; 433 434template <class T> 435 T* atomic_fetch_add(atomic<T*>* obj, ptrdiff_t op) noexcept; 436 437template <class T> 438 T* atomic_fetch_add_explicit(volatile atomic<T*>* obj, ptrdiff_t op, 439 memory_order m) noexcept; 440 441template <class T> 442 T* atomic_fetch_add_explicit(atomic<T*>* obj, ptrdiff_t op, memory_order m) noexcept; 443 444template <class T> 445 T* atomic_fetch_sub(volatile atomic<T*>* obj, ptrdiff_t op) noexcept; 446 447template <class T> 448 T* atomic_fetch_sub(atomic<T*>* obj, ptrdiff_t op) noexcept; 449 450template <class T> 451 T* atomic_fetch_sub_explicit(volatile atomic<T*>* obj, ptrdiff_t op, 452 memory_order m) noexcept; 453 454template <class T> 455 T* atomic_fetch_sub_explicit(atomic<T*>* obj, ptrdiff_t op, memory_order m) noexcept; 456 457// Atomics for standard typedef types 458 459typedef atomic<bool> atomic_bool; 460typedef atomic<char> atomic_char; 461typedef atomic<signed char> atomic_schar; 462typedef atomic<unsigned char> atomic_uchar; 463typedef atomic<short> atomic_short; 464typedef atomic<unsigned short> atomic_ushort; 465typedef atomic<int> atomic_int; 466typedef atomic<unsigned int> atomic_uint; 467typedef atomic<long> atomic_long; 468typedef atomic<unsigned long> atomic_ulong; 469typedef atomic<long long> atomic_llong; 470typedef atomic<unsigned long long> atomic_ullong; 471typedef atomic<char8_t> atomic_char8_t; // C++20 472typedef atomic<char16_t> atomic_char16_t; 473typedef atomic<char32_t> atomic_char32_t; 474typedef atomic<wchar_t> atomic_wchar_t; 475 476typedef atomic<int_least8_t> atomic_int_least8_t; 477typedef atomic<uint_least8_t> atomic_uint_least8_t; 478typedef atomic<int_least16_t> atomic_int_least16_t; 479typedef atomic<uint_least16_t> atomic_uint_least16_t; 480typedef atomic<int_least32_t> atomic_int_least32_t; 481typedef atomic<uint_least32_t> atomic_uint_least32_t; 482typedef atomic<int_least64_t> atomic_int_least64_t; 483typedef atomic<uint_least64_t> atomic_uint_least64_t; 484 485typedef atomic<int_fast8_t> atomic_int_fast8_t; 486typedef atomic<uint_fast8_t> atomic_uint_fast8_t; 487typedef atomic<int_fast16_t> atomic_int_fast16_t; 488typedef atomic<uint_fast16_t> atomic_uint_fast16_t; 489typedef atomic<int_fast32_t> atomic_int_fast32_t; 490typedef atomic<uint_fast32_t> atomic_uint_fast32_t; 491typedef atomic<int_fast64_t> atomic_int_fast64_t; 492typedef atomic<uint_fast64_t> atomic_uint_fast64_t; 493 494typedef atomic<int8_t> atomic_int8_t; 495typedef atomic<uint8_t> atomic_uint8_t; 496typedef atomic<int16_t> atomic_int16_t; 497typedef atomic<uint16_t> atomic_uint16_t; 498typedef atomic<int32_t> atomic_int32_t; 499typedef atomic<uint32_t> atomic_uint32_t; 500typedef atomic<int64_t> atomic_int64_t; 501typedef atomic<uint64_t> atomic_uint64_t; 502 503typedef atomic<intptr_t> atomic_intptr_t; 504typedef atomic<uintptr_t> atomic_uintptr_t; 505typedef atomic<size_t> atomic_size_t; 506typedef atomic<ptrdiff_t> atomic_ptrdiff_t; 507typedef atomic<intmax_t> atomic_intmax_t; 508typedef atomic<uintmax_t> atomic_uintmax_t; 509 510// flag type and operations 511 512typedef struct atomic_flag 513{ 514 atomic_flag() noexcept = default; // until C++20 515 constexpr atomic_flag() noexcept; // since C++20 516 atomic_flag(const atomic_flag&) = delete; 517 atomic_flag& operator=(const atomic_flag&) = delete; 518 atomic_flag& operator=(const atomic_flag&) volatile = delete; 519 520 bool test(memory_order m = memory_order_seq_cst) volatile noexcept; 521 bool test(memory_order m = memory_order_seq_cst) noexcept; 522 bool test_and_set(memory_order m = memory_order_seq_cst) volatile noexcept; 523 bool test_and_set(memory_order m = memory_order_seq_cst) noexcept; 524 void clear(memory_order m = memory_order_seq_cst) volatile noexcept; 525 void clear(memory_order m = memory_order_seq_cst) noexcept; 526 527 void wait(bool, memory_order = memory_order::seq_cst) const volatile noexcept; 528 void wait(bool, memory_order = memory_order::seq_cst) const noexcept; 529 void notify_one() volatile noexcept; 530 void notify_one() noexcept; 531 void notify_all() volatile noexcept; 532 void notify_all() noexcept; 533} atomic_flag; 534 535bool atomic_flag_test(volatile atomic_flag* obj) noexcept; 536bool atomic_flag_test(atomic_flag* obj) noexcept; 537bool atomic_flag_test_explicit(volatile atomic_flag* obj, 538 memory_order m) noexcept; 539bool atomic_flag_test_explicit(atomic_flag* obj, memory_order m) noexcept; 540bool atomic_flag_test_and_set(volatile atomic_flag* obj) noexcept; 541bool atomic_flag_test_and_set(atomic_flag* obj) noexcept; 542bool atomic_flag_test_and_set_explicit(volatile atomic_flag* obj, 543 memory_order m) noexcept; 544bool atomic_flag_test_and_set_explicit(atomic_flag* obj, memory_order m) noexcept; 545void atomic_flag_clear(volatile atomic_flag* obj) noexcept; 546void atomic_flag_clear(atomic_flag* obj) noexcept; 547void atomic_flag_clear_explicit(volatile atomic_flag* obj, memory_order m) noexcept; 548void atomic_flag_clear_explicit(atomic_flag* obj, memory_order m) noexcept; 549 550void atomic_wait(const volatile atomic_flag* obj, T old) noexcept; 551void atomic_wait(const atomic_flag* obj, T old) noexcept; 552void atomic_wait_explicit(const volatile atomic_flag* obj, T old, memory_order m) noexcept; 553void atomic_wait_explicit(const atomic_flag* obj, T old, memory_order m) noexcept; 554void atomic_one(volatile atomic_flag* obj) noexcept; 555void atomic_one(atomic_flag* obj) noexcept; 556void atomic_all(volatile atomic_flag* obj) noexcept; 557void atomic_all(atomic_flag* obj) noexcept; 558 559// fences 560 561void atomic_thread_fence(memory_order m) noexcept; 562void atomic_signal_fence(memory_order m) noexcept; 563 564// deprecated 565 566template <class T> 567 void atomic_init(volatile atomic<T>* obj, typename atomic<T>::value_type desr) noexcept; 568 569template <class T> 570 void atomic_init(atomic<T>* obj, typename atomic<T>::value_type desr) noexcept; 571 572#define ATOMIC_VAR_INIT(value) see below 573 574#define ATOMIC_FLAG_INIT see below 575 576} // std 577 578*/ 579 580#include <__availability> 581#include <__config> 582#include <__threading_support> 583#include <cstddef> 584#include <cstdint> 585#include <cstring> 586#include <type_traits> 587#include <version> 588 589#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) 590#pragma GCC system_header 591#endif 592 593#ifdef _LIBCPP_HAS_NO_THREADS 594# error <atomic> is not supported on this single threaded system 595#endif 596#ifdef _LIBCPP_HAS_NO_ATOMIC_HEADER 597# error <atomic> is not implemented 598#endif 599#ifdef kill_dependency 600# error C++ standard library is incompatible with <stdatomic.h> 601#endif 602 603#define _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) \ 604 _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_consume || \ 605 __m == memory_order_acquire || \ 606 __m == memory_order_acq_rel, \ 607 "memory order argument to atomic operation is invalid") 608 609#define _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) \ 610 _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_release || \ 611 __m == memory_order_acq_rel, \ 612 "memory order argument to atomic operation is invalid") 613 614#define _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__m, __f) \ 615 _LIBCPP_DIAGNOSE_WARNING(__f == memory_order_release || \ 616 __f == memory_order_acq_rel, \ 617 "memory order argument to atomic operation is invalid") 618 619_LIBCPP_BEGIN_NAMESPACE_STD 620 621// Figure out what the underlying type for `memory_order` would be if it were 622// declared as an unscoped enum (accounting for -fshort-enums). Use this result 623// to pin the underlying type in C++20. 624enum __legacy_memory_order { 625 __mo_relaxed, 626 __mo_consume, 627 __mo_acquire, 628 __mo_release, 629 __mo_acq_rel, 630 __mo_seq_cst 631}; 632 633typedef underlying_type<__legacy_memory_order>::type __memory_order_underlying_t; 634 635#if _LIBCPP_STD_VER > 17 636 637enum class memory_order : __memory_order_underlying_t { 638 relaxed = __mo_relaxed, 639 consume = __mo_consume, 640 acquire = __mo_acquire, 641 release = __mo_release, 642 acq_rel = __mo_acq_rel, 643 seq_cst = __mo_seq_cst 644}; 645 646inline constexpr auto memory_order_relaxed = memory_order::relaxed; 647inline constexpr auto memory_order_consume = memory_order::consume; 648inline constexpr auto memory_order_acquire = memory_order::acquire; 649inline constexpr auto memory_order_release = memory_order::release; 650inline constexpr auto memory_order_acq_rel = memory_order::acq_rel; 651inline constexpr auto memory_order_seq_cst = memory_order::seq_cst; 652 653#else 654 655typedef enum memory_order { 656 memory_order_relaxed = __mo_relaxed, 657 memory_order_consume = __mo_consume, 658 memory_order_acquire = __mo_acquire, 659 memory_order_release = __mo_release, 660 memory_order_acq_rel = __mo_acq_rel, 661 memory_order_seq_cst = __mo_seq_cst, 662} memory_order; 663 664#endif // _LIBCPP_STD_VER > 17 665 666template <typename _Tp> _LIBCPP_INLINE_VISIBILITY 667bool __cxx_nonatomic_compare_equal(_Tp const& __lhs, _Tp const& __rhs) { 668 return _VSTD::memcmp(&__lhs, &__rhs, sizeof(_Tp)) == 0; 669} 670 671static_assert((is_same<underlying_type<memory_order>::type, __memory_order_underlying_t>::value), 672 "unexpected underlying type for std::memory_order"); 673 674#if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) || \ 675 defined(_LIBCPP_ATOMIC_ONLY_USE_BUILTINS) 676 677// [atomics.types.generic]p1 guarantees _Tp is trivially copyable. Because 678// the default operator= in an object is not volatile, a byte-by-byte copy 679// is required. 680template <typename _Tp, typename _Tv> _LIBCPP_INLINE_VISIBILITY 681typename enable_if<is_assignable<_Tp&, _Tv>::value>::type 682__cxx_atomic_assign_volatile(_Tp& __a_value, _Tv const& __val) { 683 __a_value = __val; 684} 685template <typename _Tp, typename _Tv> _LIBCPP_INLINE_VISIBILITY 686typename enable_if<is_assignable<_Tp&, _Tv>::value>::type 687__cxx_atomic_assign_volatile(_Tp volatile& __a_value, _Tv volatile const& __val) { 688 volatile char* __to = reinterpret_cast<volatile char*>(&__a_value); 689 volatile char* __end = __to + sizeof(_Tp); 690 volatile const char* __from = reinterpret_cast<volatile const char*>(&__val); 691 while (__to != __end) 692 *__to++ = *__from++; 693} 694 695#endif 696 697#if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) 698 699template <typename _Tp> 700struct __cxx_atomic_base_impl { 701 702 _LIBCPP_INLINE_VISIBILITY 703#ifndef _LIBCPP_CXX03_LANG 704 __cxx_atomic_base_impl() _NOEXCEPT = default; 705#else 706 __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {} 707#endif // _LIBCPP_CXX03_LANG 708 _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT 709 : __a_value(value) {} 710 _Tp __a_value; 711}; 712 713_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR int __to_gcc_order(memory_order __order) { 714 // Avoid switch statement to make this a constexpr. 715 return __order == memory_order_relaxed ? __ATOMIC_RELAXED: 716 (__order == memory_order_acquire ? __ATOMIC_ACQUIRE: 717 (__order == memory_order_release ? __ATOMIC_RELEASE: 718 (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST: 719 (__order == memory_order_acq_rel ? __ATOMIC_ACQ_REL: 720 __ATOMIC_CONSUME)))); 721} 722 723_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR int __to_gcc_failure_order(memory_order __order) { 724 // Avoid switch statement to make this a constexpr. 725 return __order == memory_order_relaxed ? __ATOMIC_RELAXED: 726 (__order == memory_order_acquire ? __ATOMIC_ACQUIRE: 727 (__order == memory_order_release ? __ATOMIC_RELAXED: 728 (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST: 729 (__order == memory_order_acq_rel ? __ATOMIC_ACQUIRE: 730 __ATOMIC_CONSUME)))); 731} 732 733template <typename _Tp> 734_LIBCPP_INLINE_VISIBILITY 735void __cxx_atomic_init(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val) { 736 __cxx_atomic_assign_volatile(__a->__a_value, __val); 737} 738 739template <typename _Tp> 740_LIBCPP_INLINE_VISIBILITY 741void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) { 742 __a->__a_value = __val; 743} 744 745_LIBCPP_INLINE_VISIBILITY inline 746void __cxx_atomic_thread_fence(memory_order __order) { 747 __atomic_thread_fence(__to_gcc_order(__order)); 748} 749 750_LIBCPP_INLINE_VISIBILITY inline 751void __cxx_atomic_signal_fence(memory_order __order) { 752 __atomic_signal_fence(__to_gcc_order(__order)); 753} 754 755template <typename _Tp> 756_LIBCPP_INLINE_VISIBILITY 757void __cxx_atomic_store(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val, 758 memory_order __order) { 759 __atomic_store(&__a->__a_value, &__val, 760 __to_gcc_order(__order)); 761} 762 763template <typename _Tp> 764_LIBCPP_INLINE_VISIBILITY 765void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, 766 memory_order __order) { 767 __atomic_store(&__a->__a_value, &__val, 768 __to_gcc_order(__order)); 769} 770 771template <typename _Tp> 772_LIBCPP_INLINE_VISIBILITY 773_Tp __cxx_atomic_load(const volatile __cxx_atomic_base_impl<_Tp>* __a, 774 memory_order __order) { 775 _Tp __ret; 776 __atomic_load(&__a->__a_value, &__ret, 777 __to_gcc_order(__order)); 778 return __ret; 779} 780 781template <typename _Tp> 782_LIBCPP_INLINE_VISIBILITY 783_Tp __cxx_atomic_load(const __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) { 784 _Tp __ret; 785 __atomic_load(&__a->__a_value, &__ret, 786 __to_gcc_order(__order)); 787 return __ret; 788} 789 790template <typename _Tp> 791_LIBCPP_INLINE_VISIBILITY 792_Tp __cxx_atomic_exchange(volatile __cxx_atomic_base_impl<_Tp>* __a, 793 _Tp __value, memory_order __order) { 794 _Tp __ret; 795 __atomic_exchange(&__a->__a_value, &__value, &__ret, 796 __to_gcc_order(__order)); 797 return __ret; 798} 799 800template <typename _Tp> 801_LIBCPP_INLINE_VISIBILITY 802_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, 803 memory_order __order) { 804 _Tp __ret; 805 __atomic_exchange(&__a->__a_value, &__value, &__ret, 806 __to_gcc_order(__order)); 807 return __ret; 808} 809 810template <typename _Tp> 811_LIBCPP_INLINE_VISIBILITY 812bool __cxx_atomic_compare_exchange_strong( 813 volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, 814 memory_order __success, memory_order __failure) { 815 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 816 false, 817 __to_gcc_order(__success), 818 __to_gcc_failure_order(__failure)); 819} 820 821template <typename _Tp> 822_LIBCPP_INLINE_VISIBILITY 823bool __cxx_atomic_compare_exchange_strong( 824 __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, 825 memory_order __failure) { 826 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 827 false, 828 __to_gcc_order(__success), 829 __to_gcc_failure_order(__failure)); 830} 831 832template <typename _Tp> 833_LIBCPP_INLINE_VISIBILITY 834bool __cxx_atomic_compare_exchange_weak( 835 volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, 836 memory_order __success, memory_order __failure) { 837 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 838 true, 839 __to_gcc_order(__success), 840 __to_gcc_failure_order(__failure)); 841} 842 843template <typename _Tp> 844_LIBCPP_INLINE_VISIBILITY 845bool __cxx_atomic_compare_exchange_weak( 846 __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, 847 memory_order __failure) { 848 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 849 true, 850 __to_gcc_order(__success), 851 __to_gcc_failure_order(__failure)); 852} 853 854template <typename _Tp> 855struct __skip_amt { enum {value = 1}; }; 856 857template <typename _Tp> 858struct __skip_amt<_Tp*> { enum {value = sizeof(_Tp)}; }; 859 860// FIXME: Haven't figured out what the spec says about using arrays with 861// atomic_fetch_add. Force a failure rather than creating bad behavior. 862template <typename _Tp> 863struct __skip_amt<_Tp[]> { }; 864template <typename _Tp, int n> 865struct __skip_amt<_Tp[n]> { }; 866 867template <typename _Tp, typename _Td> 868_LIBCPP_INLINE_VISIBILITY 869_Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_base_impl<_Tp>* __a, 870 _Td __delta, memory_order __order) { 871 return __atomic_fetch_add(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 872 __to_gcc_order(__order)); 873} 874 875template <typename _Tp, typename _Td> 876_LIBCPP_INLINE_VISIBILITY 877_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, 878 memory_order __order) { 879 return __atomic_fetch_add(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 880 __to_gcc_order(__order)); 881} 882 883template <typename _Tp, typename _Td> 884_LIBCPP_INLINE_VISIBILITY 885_Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_base_impl<_Tp>* __a, 886 _Td __delta, memory_order __order) { 887 return __atomic_fetch_sub(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 888 __to_gcc_order(__order)); 889} 890 891template <typename _Tp, typename _Td> 892_LIBCPP_INLINE_VISIBILITY 893_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, 894 memory_order __order) { 895 return __atomic_fetch_sub(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 896 __to_gcc_order(__order)); 897} 898 899template <typename _Tp> 900_LIBCPP_INLINE_VISIBILITY 901_Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_base_impl<_Tp>* __a, 902 _Tp __pattern, memory_order __order) { 903 return __atomic_fetch_and(&__a->__a_value, __pattern, 904 __to_gcc_order(__order)); 905} 906 907template <typename _Tp> 908_LIBCPP_INLINE_VISIBILITY 909_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, 910 _Tp __pattern, memory_order __order) { 911 return __atomic_fetch_and(&__a->__a_value, __pattern, 912 __to_gcc_order(__order)); 913} 914 915template <typename _Tp> 916_LIBCPP_INLINE_VISIBILITY 917_Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_base_impl<_Tp>* __a, 918 _Tp __pattern, memory_order __order) { 919 return __atomic_fetch_or(&__a->__a_value, __pattern, 920 __to_gcc_order(__order)); 921} 922 923template <typename _Tp> 924_LIBCPP_INLINE_VISIBILITY 925_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, 926 memory_order __order) { 927 return __atomic_fetch_or(&__a->__a_value, __pattern, 928 __to_gcc_order(__order)); 929} 930 931template <typename _Tp> 932_LIBCPP_INLINE_VISIBILITY 933_Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_base_impl<_Tp>* __a, 934 _Tp __pattern, memory_order __order) { 935 return __atomic_fetch_xor(&__a->__a_value, __pattern, 936 __to_gcc_order(__order)); 937} 938 939template <typename _Tp> 940_LIBCPP_INLINE_VISIBILITY 941_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, 942 memory_order __order) { 943 return __atomic_fetch_xor(&__a->__a_value, __pattern, 944 __to_gcc_order(__order)); 945} 946 947#define __cxx_atomic_is_lock_free(__s) __atomic_is_lock_free(__s, 0) 948 949#elif defined(_LIBCPP_HAS_C_ATOMIC_IMP) 950 951template <typename _Tp> 952struct __cxx_atomic_base_impl { 953 954 _LIBCPP_INLINE_VISIBILITY 955#ifndef _LIBCPP_CXX03_LANG 956 __cxx_atomic_base_impl() _NOEXCEPT = default; 957#else 958 __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {} 959#endif // _LIBCPP_CXX03_LANG 960 _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT 961 : __a_value(value) {} 962 _LIBCPP_DISABLE_EXTENSION_WARNING _Atomic(_Tp) __a_value; 963}; 964 965#define __cxx_atomic_is_lock_free(__s) __c11_atomic_is_lock_free(__s) 966 967_LIBCPP_INLINE_VISIBILITY inline 968void __cxx_atomic_thread_fence(memory_order __order) _NOEXCEPT { 969 __c11_atomic_thread_fence(static_cast<__memory_order_underlying_t>(__order)); 970} 971 972_LIBCPP_INLINE_VISIBILITY inline 973void __cxx_atomic_signal_fence(memory_order __order) _NOEXCEPT { 974 __c11_atomic_signal_fence(static_cast<__memory_order_underlying_t>(__order)); 975} 976 977template<class _Tp> 978_LIBCPP_INLINE_VISIBILITY 979void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val) _NOEXCEPT { 980 __c11_atomic_init(&__a->__a_value, __val); 981} 982template<class _Tp> 983_LIBCPP_INLINE_VISIBILITY 984void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val) _NOEXCEPT { 985 __c11_atomic_init(&__a->__a_value, __val); 986} 987 988template<class _Tp> 989_LIBCPP_INLINE_VISIBILITY 990void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT { 991 __c11_atomic_store(&__a->__a_value, __val, static_cast<__memory_order_underlying_t>(__order)); 992} 993template<class _Tp> 994_LIBCPP_INLINE_VISIBILITY 995void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val, memory_order __order) _NOEXCEPT { 996 __c11_atomic_store(&__a->__a_value, __val, static_cast<__memory_order_underlying_t>(__order)); 997} 998 999template<class _Tp> 1000_LIBCPP_INLINE_VISIBILITY 1001_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const volatile* __a, memory_order __order) _NOEXCEPT { 1002 using __ptr_type = typename remove_const<decltype(__a->__a_value)>::type*; 1003 return __c11_atomic_load(const_cast<__ptr_type>(&__a->__a_value), static_cast<__memory_order_underlying_t>(__order)); 1004} 1005template<class _Tp> 1006_LIBCPP_INLINE_VISIBILITY 1007_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const* __a, memory_order __order) _NOEXCEPT { 1008 using __ptr_type = typename remove_const<decltype(__a->__a_value)>::type*; 1009 return __c11_atomic_load(const_cast<__ptr_type>(&__a->__a_value), static_cast<__memory_order_underlying_t>(__order)); 1010} 1011 1012template<class _Tp> 1013_LIBCPP_INLINE_VISIBILITY 1014_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT { 1015 return __c11_atomic_exchange(&__a->__a_value, __value, static_cast<__memory_order_underlying_t>(__order)); 1016} 1017template<class _Tp> 1018_LIBCPP_INLINE_VISIBILITY 1019_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> * __a, _Tp __value, memory_order __order) _NOEXCEPT { 1020 return __c11_atomic_exchange(&__a->__a_value, __value, static_cast<__memory_order_underlying_t>(__order)); 1021} 1022 1023_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR memory_order __to_failure_order(memory_order __order) { 1024 // Avoid switch statement to make this a constexpr. 1025 return __order == memory_order_release ? memory_order_relaxed: 1026 (__order == memory_order_acq_rel ? memory_order_acquire: 1027 __order); 1028} 1029 1030template<class _Tp> 1031_LIBCPP_INLINE_VISIBILITY 1032bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 1033 return __c11_atomic_compare_exchange_strong(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); 1034} 1035template<class _Tp> 1036_LIBCPP_INLINE_VISIBILITY 1037bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 1038 return __c11_atomic_compare_exchange_strong(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); 1039} 1040 1041template<class _Tp> 1042_LIBCPP_INLINE_VISIBILITY 1043bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 1044 return __c11_atomic_compare_exchange_weak(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); 1045} 1046template<class _Tp> 1047_LIBCPP_INLINE_VISIBILITY 1048bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 1049 return __c11_atomic_compare_exchange_weak(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); 1050} 1051 1052template<class _Tp> 1053_LIBCPP_INLINE_VISIBILITY 1054_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1055 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1056} 1057template<class _Tp> 1058_LIBCPP_INLINE_VISIBILITY 1059_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1060 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1061} 1062 1063template<class _Tp> 1064_LIBCPP_INLINE_VISIBILITY 1065_Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1066 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1067} 1068template<class _Tp> 1069_LIBCPP_INLINE_VISIBILITY 1070_Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1071 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1072} 1073 1074template<class _Tp> 1075_LIBCPP_INLINE_VISIBILITY 1076_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1077 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1078} 1079template<class _Tp> 1080_LIBCPP_INLINE_VISIBILITY 1081_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1082 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1083} 1084template<class _Tp> 1085_LIBCPP_INLINE_VISIBILITY 1086_Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1087 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1088} 1089template<class _Tp> 1090_LIBCPP_INLINE_VISIBILITY 1091_Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1092 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1093} 1094 1095template<class _Tp> 1096_LIBCPP_INLINE_VISIBILITY 1097_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1098 return __c11_atomic_fetch_and(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1099} 1100template<class _Tp> 1101_LIBCPP_INLINE_VISIBILITY 1102_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1103 return __c11_atomic_fetch_and(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1104} 1105 1106template<class _Tp> 1107_LIBCPP_INLINE_VISIBILITY 1108_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1109 return __c11_atomic_fetch_or(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1110} 1111template<class _Tp> 1112_LIBCPP_INLINE_VISIBILITY 1113_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1114 return __c11_atomic_fetch_or(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1115} 1116 1117template<class _Tp> 1118_LIBCPP_INLINE_VISIBILITY 1119_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1120 return __c11_atomic_fetch_xor(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1121} 1122template<class _Tp> 1123_LIBCPP_INLINE_VISIBILITY 1124_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1125 return __c11_atomic_fetch_xor(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1126} 1127 1128#endif // _LIBCPP_HAS_GCC_ATOMIC_IMP, _LIBCPP_HAS_C_ATOMIC_IMP 1129 1130template <class _Tp> 1131_LIBCPP_INLINE_VISIBILITY 1132_Tp kill_dependency(_Tp __y) _NOEXCEPT 1133{ 1134 return __y; 1135} 1136 1137#if defined(__CLANG_ATOMIC_BOOL_LOCK_FREE) 1138# define ATOMIC_BOOL_LOCK_FREE __CLANG_ATOMIC_BOOL_LOCK_FREE 1139# define ATOMIC_CHAR_LOCK_FREE __CLANG_ATOMIC_CHAR_LOCK_FREE 1140#ifndef _LIBCPP_HAS_NO_CHAR8_T 1141# define ATOMIC_CHAR8_T_LOCK_FREE __CLANG_ATOMIC_CHAR8_T_LOCK_FREE 1142#endif 1143# define ATOMIC_CHAR16_T_LOCK_FREE __CLANG_ATOMIC_CHAR16_T_LOCK_FREE 1144# define ATOMIC_CHAR32_T_LOCK_FREE __CLANG_ATOMIC_CHAR32_T_LOCK_FREE 1145# define ATOMIC_WCHAR_T_LOCK_FREE __CLANG_ATOMIC_WCHAR_T_LOCK_FREE 1146# define ATOMIC_SHORT_LOCK_FREE __CLANG_ATOMIC_SHORT_LOCK_FREE 1147# define ATOMIC_INT_LOCK_FREE __CLANG_ATOMIC_INT_LOCK_FREE 1148# define ATOMIC_LONG_LOCK_FREE __CLANG_ATOMIC_LONG_LOCK_FREE 1149# define ATOMIC_LLONG_LOCK_FREE __CLANG_ATOMIC_LLONG_LOCK_FREE 1150# define ATOMIC_POINTER_LOCK_FREE __CLANG_ATOMIC_POINTER_LOCK_FREE 1151#elif defined(__GCC_ATOMIC_BOOL_LOCK_FREE) 1152# define ATOMIC_BOOL_LOCK_FREE __GCC_ATOMIC_BOOL_LOCK_FREE 1153# define ATOMIC_CHAR_LOCK_FREE __GCC_ATOMIC_CHAR_LOCK_FREE 1154#ifndef _LIBCPP_HAS_NO_CHAR8_T 1155# define ATOMIC_CHAR8_T_LOCK_FREE __GCC_ATOMIC_CHAR8_T_LOCK_FREE 1156#endif 1157# define ATOMIC_CHAR16_T_LOCK_FREE __GCC_ATOMIC_CHAR16_T_LOCK_FREE 1158# define ATOMIC_CHAR32_T_LOCK_FREE __GCC_ATOMIC_CHAR32_T_LOCK_FREE 1159# define ATOMIC_WCHAR_T_LOCK_FREE __GCC_ATOMIC_WCHAR_T_LOCK_FREE 1160# define ATOMIC_SHORT_LOCK_FREE __GCC_ATOMIC_SHORT_LOCK_FREE 1161# define ATOMIC_INT_LOCK_FREE __GCC_ATOMIC_INT_LOCK_FREE 1162# define ATOMIC_LONG_LOCK_FREE __GCC_ATOMIC_LONG_LOCK_FREE 1163# define ATOMIC_LLONG_LOCK_FREE __GCC_ATOMIC_LLONG_LOCK_FREE 1164# define ATOMIC_POINTER_LOCK_FREE __GCC_ATOMIC_POINTER_LOCK_FREE 1165#endif 1166 1167#ifdef _LIBCPP_ATOMIC_ONLY_USE_BUILTINS 1168 1169template<typename _Tp> 1170struct __cxx_atomic_lock_impl { 1171 1172 _LIBCPP_INLINE_VISIBILITY 1173 __cxx_atomic_lock_impl() _NOEXCEPT 1174 : __a_value(), __a_lock(0) {} 1175 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR explicit 1176 __cxx_atomic_lock_impl(_Tp value) _NOEXCEPT 1177 : __a_value(value), __a_lock(0) {} 1178 1179 _Tp __a_value; 1180 mutable __cxx_atomic_base_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_lock; 1181 1182 _LIBCPP_INLINE_VISIBILITY void __lock() const volatile { 1183 while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire)) 1184 /*spin*/; 1185 } 1186 _LIBCPP_INLINE_VISIBILITY void __lock() const { 1187 while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire)) 1188 /*spin*/; 1189 } 1190 _LIBCPP_INLINE_VISIBILITY void __unlock() const volatile { 1191 __cxx_atomic_store(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(false), memory_order_release); 1192 } 1193 _LIBCPP_INLINE_VISIBILITY void __unlock() const { 1194 __cxx_atomic_store(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(false), memory_order_release); 1195 } 1196 _LIBCPP_INLINE_VISIBILITY _Tp __read() const volatile { 1197 __lock(); 1198 _Tp __old; 1199 __cxx_atomic_assign_volatile(__old, __a_value); 1200 __unlock(); 1201 return __old; 1202 } 1203 _LIBCPP_INLINE_VISIBILITY _Tp __read() const { 1204 __lock(); 1205 _Tp __old = __a_value; 1206 __unlock(); 1207 return __old; 1208 } 1209}; 1210 1211template <typename _Tp> 1212_LIBCPP_INLINE_VISIBILITY 1213void __cxx_atomic_init(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __val) { 1214 __cxx_atomic_assign_volatile(__a->__a_value, __val); 1215} 1216template <typename _Tp> 1217_LIBCPP_INLINE_VISIBILITY 1218void __cxx_atomic_init(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __val) { 1219 __a->__a_value = __val; 1220} 1221 1222template <typename _Tp> 1223_LIBCPP_INLINE_VISIBILITY 1224void __cxx_atomic_store(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __val, memory_order) { 1225 __a->__lock(); 1226 __cxx_atomic_assign_volatile(__a->__a_value, __val); 1227 __a->__unlock(); 1228} 1229template <typename _Tp> 1230_LIBCPP_INLINE_VISIBILITY 1231void __cxx_atomic_store(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __val, memory_order) { 1232 __a->__lock(); 1233 __a->__a_value = __val; 1234 __a->__unlock(); 1235} 1236 1237template <typename _Tp> 1238_LIBCPP_INLINE_VISIBILITY 1239_Tp __cxx_atomic_load(const volatile __cxx_atomic_lock_impl<_Tp>* __a, memory_order) { 1240 return __a->__read(); 1241} 1242template <typename _Tp> 1243_LIBCPP_INLINE_VISIBILITY 1244_Tp __cxx_atomic_load(const __cxx_atomic_lock_impl<_Tp>* __a, memory_order) { 1245 return __a->__read(); 1246} 1247 1248template <typename _Tp> 1249_LIBCPP_INLINE_VISIBILITY 1250_Tp __cxx_atomic_exchange(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) { 1251 __a->__lock(); 1252 _Tp __old; 1253 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1254 __cxx_atomic_assign_volatile(__a->__a_value, __value); 1255 __a->__unlock(); 1256 return __old; 1257} 1258template <typename _Tp> 1259_LIBCPP_INLINE_VISIBILITY 1260_Tp __cxx_atomic_exchange(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) { 1261 __a->__lock(); 1262 _Tp __old = __a->__a_value; 1263 __a->__a_value = __value; 1264 __a->__unlock(); 1265 return __old; 1266} 1267 1268template <typename _Tp> 1269_LIBCPP_INLINE_VISIBILITY 1270bool __cxx_atomic_compare_exchange_strong(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1271 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1272 _Tp __temp; 1273 __a->__lock(); 1274 __cxx_atomic_assign_volatile(__temp, __a->__a_value); 1275 bool __ret = (_VSTD::memcmp(&__temp, __expected, sizeof(_Tp)) == 0); 1276 if(__ret) 1277 __cxx_atomic_assign_volatile(__a->__a_value, __value); 1278 else 1279 __cxx_atomic_assign_volatile(*__expected, __a->__a_value); 1280 __a->__unlock(); 1281 return __ret; 1282} 1283template <typename _Tp> 1284_LIBCPP_INLINE_VISIBILITY 1285bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_lock_impl<_Tp>* __a, 1286 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1287 __a->__lock(); 1288 bool __ret = (_VSTD::memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0); 1289 if(__ret) 1290 _VSTD::memcpy(&__a->__a_value, &__value, sizeof(_Tp)); 1291 else 1292 _VSTD::memcpy(__expected, &__a->__a_value, sizeof(_Tp)); 1293 __a->__unlock(); 1294 return __ret; 1295} 1296 1297template <typename _Tp> 1298_LIBCPP_INLINE_VISIBILITY 1299bool __cxx_atomic_compare_exchange_weak(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1300 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1301 _Tp __temp; 1302 __a->__lock(); 1303 __cxx_atomic_assign_volatile(__temp, __a->__a_value); 1304 bool __ret = (_VSTD::memcmp(&__temp, __expected, sizeof(_Tp)) == 0); 1305 if(__ret) 1306 __cxx_atomic_assign_volatile(__a->__a_value, __value); 1307 else 1308 __cxx_atomic_assign_volatile(*__expected, __a->__a_value); 1309 __a->__unlock(); 1310 return __ret; 1311} 1312template <typename _Tp> 1313_LIBCPP_INLINE_VISIBILITY 1314bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_lock_impl<_Tp>* __a, 1315 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1316 __a->__lock(); 1317 bool __ret = (_VSTD::memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0); 1318 if(__ret) 1319 _VSTD::memcpy(&__a->__a_value, &__value, sizeof(_Tp)); 1320 else 1321 _VSTD::memcpy(__expected, &__a->__a_value, sizeof(_Tp)); 1322 __a->__unlock(); 1323 return __ret; 1324} 1325 1326template <typename _Tp, typename _Td> 1327_LIBCPP_INLINE_VISIBILITY 1328_Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1329 _Td __delta, memory_order) { 1330 __a->__lock(); 1331 _Tp __old; 1332 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1333 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old + __delta)); 1334 __a->__unlock(); 1335 return __old; 1336} 1337template <typename _Tp, typename _Td> 1338_LIBCPP_INLINE_VISIBILITY 1339_Tp __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp>* __a, 1340 _Td __delta, memory_order) { 1341 __a->__lock(); 1342 _Tp __old = __a->__a_value; 1343 __a->__a_value += __delta; 1344 __a->__unlock(); 1345 return __old; 1346} 1347 1348template <typename _Tp, typename _Td> 1349_LIBCPP_INLINE_VISIBILITY 1350_Tp* __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp*>* __a, 1351 ptrdiff_t __delta, memory_order) { 1352 __a->__lock(); 1353 _Tp* __old; 1354 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1355 __cxx_atomic_assign_volatile(__a->__a_value, __old + __delta); 1356 __a->__unlock(); 1357 return __old; 1358} 1359template <typename _Tp, typename _Td> 1360_LIBCPP_INLINE_VISIBILITY 1361_Tp* __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp*>* __a, 1362 ptrdiff_t __delta, memory_order) { 1363 __a->__lock(); 1364 _Tp* __old = __a->__a_value; 1365 __a->__a_value += __delta; 1366 __a->__unlock(); 1367 return __old; 1368} 1369 1370template <typename _Tp, typename _Td> 1371_LIBCPP_INLINE_VISIBILITY 1372_Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1373 _Td __delta, memory_order) { 1374 __a->__lock(); 1375 _Tp __old; 1376 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1377 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old - __delta)); 1378 __a->__unlock(); 1379 return __old; 1380} 1381template <typename _Tp, typename _Td> 1382_LIBCPP_INLINE_VISIBILITY 1383_Tp __cxx_atomic_fetch_sub(__cxx_atomic_lock_impl<_Tp>* __a, 1384 _Td __delta, memory_order) { 1385 __a->__lock(); 1386 _Tp __old = __a->__a_value; 1387 __a->__a_value -= __delta; 1388 __a->__unlock(); 1389 return __old; 1390} 1391 1392template <typename _Tp> 1393_LIBCPP_INLINE_VISIBILITY 1394_Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1395 _Tp __pattern, memory_order) { 1396 __a->__lock(); 1397 _Tp __old; 1398 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1399 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old & __pattern)); 1400 __a->__unlock(); 1401 return __old; 1402} 1403template <typename _Tp> 1404_LIBCPP_INLINE_VISIBILITY 1405_Tp __cxx_atomic_fetch_and(__cxx_atomic_lock_impl<_Tp>* __a, 1406 _Tp __pattern, memory_order) { 1407 __a->__lock(); 1408 _Tp __old = __a->__a_value; 1409 __a->__a_value &= __pattern; 1410 __a->__unlock(); 1411 return __old; 1412} 1413 1414template <typename _Tp> 1415_LIBCPP_INLINE_VISIBILITY 1416_Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1417 _Tp __pattern, memory_order) { 1418 __a->__lock(); 1419 _Tp __old; 1420 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1421 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old | __pattern)); 1422 __a->__unlock(); 1423 return __old; 1424} 1425template <typename _Tp> 1426_LIBCPP_INLINE_VISIBILITY 1427_Tp __cxx_atomic_fetch_or(__cxx_atomic_lock_impl<_Tp>* __a, 1428 _Tp __pattern, memory_order) { 1429 __a->__lock(); 1430 _Tp __old = __a->__a_value; 1431 __a->__a_value |= __pattern; 1432 __a->__unlock(); 1433 return __old; 1434} 1435 1436template <typename _Tp> 1437_LIBCPP_INLINE_VISIBILITY 1438_Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1439 _Tp __pattern, memory_order) { 1440 __a->__lock(); 1441 _Tp __old; 1442 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1443 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old ^ __pattern)); 1444 __a->__unlock(); 1445 return __old; 1446} 1447template <typename _Tp> 1448_LIBCPP_INLINE_VISIBILITY 1449_Tp __cxx_atomic_fetch_xor(__cxx_atomic_lock_impl<_Tp>* __a, 1450 _Tp __pattern, memory_order) { 1451 __a->__lock(); 1452 _Tp __old = __a->__a_value; 1453 __a->__a_value ^= __pattern; 1454 __a->__unlock(); 1455 return __old; 1456} 1457 1458#ifdef __cpp_lib_atomic_is_always_lock_free 1459 1460template<typename _Tp> struct __cxx_is_always_lock_free { 1461 enum { __value = __atomic_always_lock_free(sizeof(_Tp), 0) }; }; 1462 1463#else 1464 1465template<typename _Tp> struct __cxx_is_always_lock_free { enum { __value = false }; }; 1466// Implementations must match the C ATOMIC_*_LOCK_FREE macro values. 1467template<> struct __cxx_is_always_lock_free<bool> { enum { __value = 2 == ATOMIC_BOOL_LOCK_FREE }; }; 1468template<> struct __cxx_is_always_lock_free<char> { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; }; 1469template<> struct __cxx_is_always_lock_free<signed char> { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; }; 1470template<> struct __cxx_is_always_lock_free<unsigned char> { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; }; 1471#ifndef _LIBCPP_HAS_NO_CHAR8_T 1472template<> struct __cxx_is_always_lock_free<char8_t> { enum { __value = 2 == ATOMIC_CHAR8_T_LOCK_FREE }; }; 1473#endif 1474template<> struct __cxx_is_always_lock_free<char16_t> { enum { __value = 2 == ATOMIC_CHAR16_T_LOCK_FREE }; }; 1475template<> struct __cxx_is_always_lock_free<char32_t> { enum { __value = 2 == ATOMIC_CHAR32_T_LOCK_FREE }; }; 1476template<> struct __cxx_is_always_lock_free<wchar_t> { enum { __value = 2 == ATOMIC_WCHAR_T_LOCK_FREE }; }; 1477template<> struct __cxx_is_always_lock_free<short> { enum { __value = 2 == ATOMIC_SHORT_LOCK_FREE }; }; 1478template<> struct __cxx_is_always_lock_free<unsigned short> { enum { __value = 2 == ATOMIC_SHORT_LOCK_FREE }; }; 1479template<> struct __cxx_is_always_lock_free<int> { enum { __value = 2 == ATOMIC_INT_LOCK_FREE }; }; 1480template<> struct __cxx_is_always_lock_free<unsigned int> { enum { __value = 2 == ATOMIC_INT_LOCK_FREE }; }; 1481template<> struct __cxx_is_always_lock_free<long> { enum { __value = 2 == ATOMIC_LONG_LOCK_FREE }; }; 1482template<> struct __cxx_is_always_lock_free<unsigned long> { enum { __value = 2 == ATOMIC_LONG_LOCK_FREE }; }; 1483template<> struct __cxx_is_always_lock_free<long long> { enum { __value = 2 == ATOMIC_LLONG_LOCK_FREE }; }; 1484template<> struct __cxx_is_always_lock_free<unsigned long long> { enum { __value = 2 == ATOMIC_LLONG_LOCK_FREE }; }; 1485template<typename _Tp> struct __cxx_is_always_lock_free<_Tp*> { enum { __value = 2 == ATOMIC_POINTER_LOCK_FREE }; }; 1486template<> struct __cxx_is_always_lock_free<std::nullptr_t> { enum { __value = 2 == ATOMIC_POINTER_LOCK_FREE }; }; 1487 1488#endif //__cpp_lib_atomic_is_always_lock_free 1489 1490template <typename _Tp, 1491 typename _Base = typename conditional<__cxx_is_always_lock_free<_Tp>::__value, 1492 __cxx_atomic_base_impl<_Tp>, 1493 __cxx_atomic_lock_impl<_Tp> >::type> 1494#else 1495template <typename _Tp, 1496 typename _Base = __cxx_atomic_base_impl<_Tp> > 1497#endif //_LIBCPP_ATOMIC_ONLY_USE_BUILTINS 1498struct __cxx_atomic_impl : public _Base { 1499 static_assert(is_trivially_copyable<_Tp>::value, 1500 "std::atomic<Tp> requires that 'Tp' be a trivially copyable type"); 1501 1502 _LIBCPP_INLINE_VISIBILITY __cxx_atomic_impl() _NOEXCEPT _LIBCPP_DEFAULT 1503 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR explicit __cxx_atomic_impl(_Tp value) _NOEXCEPT 1504 : _Base(value) {} 1505}; 1506 1507#ifdef __linux__ 1508 using __cxx_contention_t = int32_t; 1509#else 1510 using __cxx_contention_t = int64_t; 1511#endif //__linux__ 1512 1513using __cxx_atomic_contention_t = __cxx_atomic_impl<__cxx_contention_t>; 1514 1515#ifndef _LIBCPP_HAS_NO_PLATFORM_WAIT 1516 1517_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(void const volatile*); 1518_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(void const volatile*); 1519_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(void const volatile*); 1520_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(void const volatile*, __cxx_contention_t); 1521 1522_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(__cxx_atomic_contention_t const volatile*); 1523_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(__cxx_atomic_contention_t const volatile*); 1524_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile*); 1525_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(__cxx_atomic_contention_t const volatile*, __cxx_contention_t); 1526 1527template <class _Atp, class _Fn> 1528struct __libcpp_atomic_wait_backoff_impl { 1529 _Atp* __a; 1530 _Fn __test_fn; 1531 _LIBCPP_AVAILABILITY_SYNC 1532 _LIBCPP_INLINE_VISIBILITY bool operator()(chrono::nanoseconds __elapsed) const 1533 { 1534 if(__elapsed > chrono::microseconds(64)) 1535 { 1536 auto const __monitor = __libcpp_atomic_monitor(__a); 1537 if(__test_fn()) 1538 return true; 1539 __libcpp_atomic_wait(__a, __monitor); 1540 } 1541 else if(__elapsed > chrono::microseconds(4)) 1542 __libcpp_thread_yield(); 1543 else 1544 {} // poll 1545 return false; 1546 } 1547}; 1548 1549template <class _Atp, class _Fn> 1550_LIBCPP_AVAILABILITY_SYNC 1551_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp* __a, _Fn && __test_fn) 1552{ 1553 __libcpp_atomic_wait_backoff_impl<_Atp, typename decay<_Fn>::type> __backoff_fn = {__a, __test_fn}; 1554 return __libcpp_thread_poll_with_backoff(__test_fn, __backoff_fn); 1555} 1556 1557#else // _LIBCPP_HAS_NO_PLATFORM_WAIT 1558 1559template <class _Tp> 1560_LIBCPP_INLINE_VISIBILITY void __cxx_atomic_notify_all(__cxx_atomic_impl<_Tp> const volatile*) { } 1561template <class _Tp> 1562_LIBCPP_INLINE_VISIBILITY void __cxx_atomic_notify_one(__cxx_atomic_impl<_Tp> const volatile*) { } 1563template <class _Atp, class _Fn> 1564_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp*, _Fn && __test_fn) 1565{ 1566 return __libcpp_thread_poll_with_backoff(__test_fn, __libcpp_timed_backoff_policy()); 1567} 1568 1569#endif // _LIBCPP_HAS_NO_PLATFORM_WAIT 1570 1571template <class _Atp, class _Tp> 1572struct __cxx_atomic_wait_test_fn_impl { 1573 _Atp* __a; 1574 _Tp __val; 1575 memory_order __order; 1576 _LIBCPP_INLINE_VISIBILITY bool operator()() const 1577 { 1578 return !__cxx_nonatomic_compare_equal(__cxx_atomic_load(__a, __order), __val); 1579 } 1580}; 1581 1582template <class _Atp, class _Tp> 1583_LIBCPP_AVAILABILITY_SYNC 1584_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp* __a, _Tp const __val, memory_order __order) 1585{ 1586 __cxx_atomic_wait_test_fn_impl<_Atp, _Tp> __test_fn = {__a, __val, __order}; 1587 return __cxx_atomic_wait(__a, __test_fn); 1588} 1589 1590// general atomic<T> 1591 1592template <class _Tp, bool = is_integral<_Tp>::value && !is_same<_Tp, bool>::value> 1593struct __atomic_base // false 1594{ 1595 mutable __cxx_atomic_impl<_Tp> __a_; 1596 1597#if defined(__cpp_lib_atomic_is_always_lock_free) 1598 static _LIBCPP_CONSTEXPR bool is_always_lock_free = __atomic_always_lock_free(sizeof(__a_), 0); 1599#endif 1600 1601 _LIBCPP_INLINE_VISIBILITY 1602 bool is_lock_free() const volatile _NOEXCEPT 1603 {return __cxx_atomic_is_lock_free(sizeof(_Tp));} 1604 _LIBCPP_INLINE_VISIBILITY 1605 bool is_lock_free() const _NOEXCEPT 1606 {return static_cast<__atomic_base const volatile*>(this)->is_lock_free();} 1607 _LIBCPP_INLINE_VISIBILITY 1608 void store(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1609 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1610 {__cxx_atomic_store(&__a_, __d, __m);} 1611 _LIBCPP_INLINE_VISIBILITY 1612 void store(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1613 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1614 {__cxx_atomic_store(&__a_, __d, __m);} 1615 _LIBCPP_INLINE_VISIBILITY 1616 _Tp load(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 1617 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 1618 {return __cxx_atomic_load(&__a_, __m);} 1619 _LIBCPP_INLINE_VISIBILITY 1620 _Tp load(memory_order __m = memory_order_seq_cst) const _NOEXCEPT 1621 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 1622 {return __cxx_atomic_load(&__a_, __m);} 1623 _LIBCPP_INLINE_VISIBILITY 1624 operator _Tp() const volatile _NOEXCEPT {return load();} 1625 _LIBCPP_INLINE_VISIBILITY 1626 operator _Tp() const _NOEXCEPT {return load();} 1627 _LIBCPP_INLINE_VISIBILITY 1628 _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1629 {return __cxx_atomic_exchange(&__a_, __d, __m);} 1630 _LIBCPP_INLINE_VISIBILITY 1631 _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1632 {return __cxx_atomic_exchange(&__a_, __d, __m);} 1633 _LIBCPP_INLINE_VISIBILITY 1634 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1635 memory_order __s, memory_order __f) volatile _NOEXCEPT 1636 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1637 {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);} 1638 _LIBCPP_INLINE_VISIBILITY 1639 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1640 memory_order __s, memory_order __f) _NOEXCEPT 1641 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1642 {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);} 1643 _LIBCPP_INLINE_VISIBILITY 1644 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1645 memory_order __s, memory_order __f) volatile _NOEXCEPT 1646 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1647 {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);} 1648 _LIBCPP_INLINE_VISIBILITY 1649 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1650 memory_order __s, memory_order __f) _NOEXCEPT 1651 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1652 {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);} 1653 _LIBCPP_INLINE_VISIBILITY 1654 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1655 memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1656 {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);} 1657 _LIBCPP_INLINE_VISIBILITY 1658 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1659 memory_order __m = memory_order_seq_cst) _NOEXCEPT 1660 {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);} 1661 _LIBCPP_INLINE_VISIBILITY 1662 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1663 memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1664 {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);} 1665 _LIBCPP_INLINE_VISIBILITY 1666 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1667 memory_order __m = memory_order_seq_cst) _NOEXCEPT 1668 {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);} 1669 1670 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 1671 {__cxx_atomic_wait(&__a_, __v, __m);} 1672 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT 1673 {__cxx_atomic_wait(&__a_, __v, __m);} 1674 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_one() volatile _NOEXCEPT 1675 {__cxx_atomic_notify_one(&__a_);} 1676 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_one() _NOEXCEPT 1677 {__cxx_atomic_notify_one(&__a_);} 1678 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_all() volatile _NOEXCEPT 1679 {__cxx_atomic_notify_all(&__a_);} 1680 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_all() _NOEXCEPT 1681 {__cxx_atomic_notify_all(&__a_);} 1682 1683#if _LIBCPP_STD_VER > 17 1684 _LIBCPP_INLINE_VISIBILITY constexpr 1685 __atomic_base() noexcept(is_nothrow_default_constructible_v<_Tp>) : __a_(_Tp()) {} 1686#else 1687 _LIBCPP_INLINE_VISIBILITY 1688 __atomic_base() _NOEXCEPT _LIBCPP_DEFAULT 1689#endif 1690 1691 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR 1692 __atomic_base(_Tp __d) _NOEXCEPT : __a_(__d) {} 1693 1694#ifndef _LIBCPP_CXX03_LANG 1695 __atomic_base(const __atomic_base&) = delete; 1696#else 1697private: 1698 _LIBCPP_INLINE_VISIBILITY 1699 __atomic_base(const __atomic_base&); 1700#endif 1701}; 1702 1703#if defined(__cpp_lib_atomic_is_always_lock_free) 1704template <class _Tp, bool __b> 1705_LIBCPP_CONSTEXPR bool __atomic_base<_Tp, __b>::is_always_lock_free; 1706#endif 1707 1708// atomic<Integral> 1709 1710template <class _Tp> 1711struct __atomic_base<_Tp, true> 1712 : public __atomic_base<_Tp, false> 1713{ 1714 typedef __atomic_base<_Tp, false> __base; 1715 1716 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 1717 __atomic_base() _NOEXCEPT _LIBCPP_DEFAULT 1718 1719 _LIBCPP_INLINE_VISIBILITY 1720 _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __base(__d) {} 1721 1722 _LIBCPP_INLINE_VISIBILITY 1723 _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1724 {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);} 1725 _LIBCPP_INLINE_VISIBILITY 1726 _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1727 {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);} 1728 _LIBCPP_INLINE_VISIBILITY 1729 _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1730 {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);} 1731 _LIBCPP_INLINE_VISIBILITY 1732 _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1733 {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);} 1734 _LIBCPP_INLINE_VISIBILITY 1735 _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1736 {return __cxx_atomic_fetch_and(&this->__a_, __op, __m);} 1737 _LIBCPP_INLINE_VISIBILITY 1738 _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1739 {return __cxx_atomic_fetch_and(&this->__a_, __op, __m);} 1740 _LIBCPP_INLINE_VISIBILITY 1741 _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1742 {return __cxx_atomic_fetch_or(&this->__a_, __op, __m);} 1743 _LIBCPP_INLINE_VISIBILITY 1744 _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1745 {return __cxx_atomic_fetch_or(&this->__a_, __op, __m);} 1746 _LIBCPP_INLINE_VISIBILITY 1747 _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1748 {return __cxx_atomic_fetch_xor(&this->__a_, __op, __m);} 1749 _LIBCPP_INLINE_VISIBILITY 1750 _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1751 {return __cxx_atomic_fetch_xor(&this->__a_, __op, __m);} 1752 1753 _LIBCPP_INLINE_VISIBILITY 1754 _Tp operator++(int) volatile _NOEXCEPT {return fetch_add(_Tp(1));} 1755 _LIBCPP_INLINE_VISIBILITY 1756 _Tp operator++(int) _NOEXCEPT {return fetch_add(_Tp(1));} 1757 _LIBCPP_INLINE_VISIBILITY 1758 _Tp operator--(int) volatile _NOEXCEPT {return fetch_sub(_Tp(1));} 1759 _LIBCPP_INLINE_VISIBILITY 1760 _Tp operator--(int) _NOEXCEPT {return fetch_sub(_Tp(1));} 1761 _LIBCPP_INLINE_VISIBILITY 1762 _Tp operator++() volatile _NOEXCEPT {return fetch_add(_Tp(1)) + _Tp(1);} 1763 _LIBCPP_INLINE_VISIBILITY 1764 _Tp operator++() _NOEXCEPT {return fetch_add(_Tp(1)) + _Tp(1);} 1765 _LIBCPP_INLINE_VISIBILITY 1766 _Tp operator--() volatile _NOEXCEPT {return fetch_sub(_Tp(1)) - _Tp(1);} 1767 _LIBCPP_INLINE_VISIBILITY 1768 _Tp operator--() _NOEXCEPT {return fetch_sub(_Tp(1)) - _Tp(1);} 1769 _LIBCPP_INLINE_VISIBILITY 1770 _Tp operator+=(_Tp __op) volatile _NOEXCEPT {return fetch_add(__op) + __op;} 1771 _LIBCPP_INLINE_VISIBILITY 1772 _Tp operator+=(_Tp __op) _NOEXCEPT {return fetch_add(__op) + __op;} 1773 _LIBCPP_INLINE_VISIBILITY 1774 _Tp operator-=(_Tp __op) volatile _NOEXCEPT {return fetch_sub(__op) - __op;} 1775 _LIBCPP_INLINE_VISIBILITY 1776 _Tp operator-=(_Tp __op) _NOEXCEPT {return fetch_sub(__op) - __op;} 1777 _LIBCPP_INLINE_VISIBILITY 1778 _Tp operator&=(_Tp __op) volatile _NOEXCEPT {return fetch_and(__op) & __op;} 1779 _LIBCPP_INLINE_VISIBILITY 1780 _Tp operator&=(_Tp __op) _NOEXCEPT {return fetch_and(__op) & __op;} 1781 _LIBCPP_INLINE_VISIBILITY 1782 _Tp operator|=(_Tp __op) volatile _NOEXCEPT {return fetch_or(__op) | __op;} 1783 _LIBCPP_INLINE_VISIBILITY 1784 _Tp operator|=(_Tp __op) _NOEXCEPT {return fetch_or(__op) | __op;} 1785 _LIBCPP_INLINE_VISIBILITY 1786 _Tp operator^=(_Tp __op) volatile _NOEXCEPT {return fetch_xor(__op) ^ __op;} 1787 _LIBCPP_INLINE_VISIBILITY 1788 _Tp operator^=(_Tp __op) _NOEXCEPT {return fetch_xor(__op) ^ __op;} 1789}; 1790 1791// atomic<T> 1792 1793template <class _Tp> 1794struct atomic 1795 : public __atomic_base<_Tp> 1796{ 1797 typedef __atomic_base<_Tp> __base; 1798 typedef _Tp value_type; 1799 typedef value_type difference_type; 1800 1801#if _LIBCPP_STD_VER > 17 1802 _LIBCPP_INLINE_VISIBILITY 1803 atomic() = default; 1804#else 1805 _LIBCPP_INLINE_VISIBILITY 1806 atomic() _NOEXCEPT _LIBCPP_DEFAULT 1807#endif 1808 1809 _LIBCPP_INLINE_VISIBILITY 1810 _LIBCPP_CONSTEXPR atomic(_Tp __d) _NOEXCEPT : __base(__d) {} 1811 1812 _LIBCPP_INLINE_VISIBILITY 1813 _Tp operator=(_Tp __d) volatile _NOEXCEPT 1814 {__base::store(__d); return __d;} 1815 _LIBCPP_INLINE_VISIBILITY 1816 _Tp operator=(_Tp __d) _NOEXCEPT 1817 {__base::store(__d); return __d;} 1818 1819 atomic& operator=(const atomic&) = delete; 1820 atomic& operator=(const atomic&) volatile = delete; 1821}; 1822 1823// atomic<T*> 1824 1825template <class _Tp> 1826struct atomic<_Tp*> 1827 : public __atomic_base<_Tp*> 1828{ 1829 typedef __atomic_base<_Tp*> __base; 1830 typedef _Tp* value_type; 1831 typedef ptrdiff_t difference_type; 1832 1833 _LIBCPP_INLINE_VISIBILITY 1834 atomic() _NOEXCEPT _LIBCPP_DEFAULT 1835 1836 _LIBCPP_INLINE_VISIBILITY 1837 _LIBCPP_CONSTEXPR atomic(_Tp* __d) _NOEXCEPT : __base(__d) {} 1838 1839 _LIBCPP_INLINE_VISIBILITY 1840 _Tp* operator=(_Tp* __d) volatile _NOEXCEPT 1841 {__base::store(__d); return __d;} 1842 _LIBCPP_INLINE_VISIBILITY 1843 _Tp* operator=(_Tp* __d) _NOEXCEPT 1844 {__base::store(__d); return __d;} 1845 1846 _LIBCPP_INLINE_VISIBILITY 1847 _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { 1848 // __atomic_fetch_add accepts function pointers, guard against them. 1849 static_assert(!is_function<typename remove_pointer<_Tp>::type>::value, "Pointer to function isn't allowed"); 1850 return __cxx_atomic_fetch_add(&this->__a_, __op, __m); 1851 } 1852 1853 _LIBCPP_INLINE_VISIBILITY 1854 _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { 1855 // __atomic_fetch_add accepts function pointers, guard against them. 1856 static_assert(!is_function<typename remove_pointer<_Tp>::type>::value, "Pointer to function isn't allowed"); 1857 return __cxx_atomic_fetch_add(&this->__a_, __op, __m); 1858 } 1859 1860 _LIBCPP_INLINE_VISIBILITY 1861 _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { 1862 // __atomic_fetch_add accepts function pointers, guard against them. 1863 static_assert(!is_function<typename remove_pointer<_Tp>::type>::value, "Pointer to function isn't allowed"); 1864 return __cxx_atomic_fetch_sub(&this->__a_, __op, __m); 1865 } 1866 1867 _LIBCPP_INLINE_VISIBILITY 1868 _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { 1869 // __atomic_fetch_add accepts function pointers, guard against them. 1870 static_assert(!is_function<typename remove_pointer<_Tp>::type>::value, "Pointer to function isn't allowed"); 1871 return __cxx_atomic_fetch_sub(&this->__a_, __op, __m); 1872 } 1873 1874 _LIBCPP_INLINE_VISIBILITY 1875 _Tp* operator++(int) volatile _NOEXCEPT {return fetch_add(1);} 1876 _LIBCPP_INLINE_VISIBILITY 1877 _Tp* operator++(int) _NOEXCEPT {return fetch_add(1);} 1878 _LIBCPP_INLINE_VISIBILITY 1879 _Tp* operator--(int) volatile _NOEXCEPT {return fetch_sub(1);} 1880 _LIBCPP_INLINE_VISIBILITY 1881 _Tp* operator--(int) _NOEXCEPT {return fetch_sub(1);} 1882 _LIBCPP_INLINE_VISIBILITY 1883 _Tp* operator++() volatile _NOEXCEPT {return fetch_add(1) + 1;} 1884 _LIBCPP_INLINE_VISIBILITY 1885 _Tp* operator++() _NOEXCEPT {return fetch_add(1) + 1;} 1886 _LIBCPP_INLINE_VISIBILITY 1887 _Tp* operator--() volatile _NOEXCEPT {return fetch_sub(1) - 1;} 1888 _LIBCPP_INLINE_VISIBILITY 1889 _Tp* operator--() _NOEXCEPT {return fetch_sub(1) - 1;} 1890 _LIBCPP_INLINE_VISIBILITY 1891 _Tp* operator+=(ptrdiff_t __op) volatile _NOEXCEPT {return fetch_add(__op) + __op;} 1892 _LIBCPP_INLINE_VISIBILITY 1893 _Tp* operator+=(ptrdiff_t __op) _NOEXCEPT {return fetch_add(__op) + __op;} 1894 _LIBCPP_INLINE_VISIBILITY 1895 _Tp* operator-=(ptrdiff_t __op) volatile _NOEXCEPT {return fetch_sub(__op) - __op;} 1896 _LIBCPP_INLINE_VISIBILITY 1897 _Tp* operator-=(ptrdiff_t __op) _NOEXCEPT {return fetch_sub(__op) - __op;} 1898 1899 atomic& operator=(const atomic&) = delete; 1900 atomic& operator=(const atomic&) volatile = delete; 1901}; 1902 1903// atomic_is_lock_free 1904 1905template <class _Tp> 1906_LIBCPP_INLINE_VISIBILITY 1907bool 1908atomic_is_lock_free(const volatile atomic<_Tp>* __o) _NOEXCEPT 1909{ 1910 return __o->is_lock_free(); 1911} 1912 1913template <class _Tp> 1914_LIBCPP_INLINE_VISIBILITY 1915bool 1916atomic_is_lock_free(const atomic<_Tp>* __o) _NOEXCEPT 1917{ 1918 return __o->is_lock_free(); 1919} 1920 1921// atomic_init 1922 1923template <class _Tp> 1924_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_INLINE_VISIBILITY 1925void 1926atomic_init(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1927{ 1928 __cxx_atomic_init(&__o->__a_, __d); 1929} 1930 1931template <class _Tp> 1932_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_INLINE_VISIBILITY 1933void 1934atomic_init(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1935{ 1936 __cxx_atomic_init(&__o->__a_, __d); 1937} 1938 1939// atomic_store 1940 1941template <class _Tp> 1942_LIBCPP_INLINE_VISIBILITY 1943void 1944atomic_store(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1945{ 1946 __o->store(__d); 1947} 1948 1949template <class _Tp> 1950_LIBCPP_INLINE_VISIBILITY 1951void 1952atomic_store(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1953{ 1954 __o->store(__d); 1955} 1956 1957// atomic_store_explicit 1958 1959template <class _Tp> 1960_LIBCPP_INLINE_VISIBILITY 1961void 1962atomic_store_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 1963 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1964{ 1965 __o->store(__d, __m); 1966} 1967 1968template <class _Tp> 1969_LIBCPP_INLINE_VISIBILITY 1970void 1971atomic_store_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 1972 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1973{ 1974 __o->store(__d, __m); 1975} 1976 1977// atomic_load 1978 1979template <class _Tp> 1980_LIBCPP_INLINE_VISIBILITY 1981_Tp 1982atomic_load(const volatile atomic<_Tp>* __o) _NOEXCEPT 1983{ 1984 return __o->load(); 1985} 1986 1987template <class _Tp> 1988_LIBCPP_INLINE_VISIBILITY 1989_Tp 1990atomic_load(const atomic<_Tp>* __o) _NOEXCEPT 1991{ 1992 return __o->load(); 1993} 1994 1995// atomic_load_explicit 1996 1997template <class _Tp> 1998_LIBCPP_INLINE_VISIBILITY 1999_Tp 2000atomic_load_explicit(const volatile atomic<_Tp>* __o, memory_order __m) _NOEXCEPT 2001 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 2002{ 2003 return __o->load(__m); 2004} 2005 2006template <class _Tp> 2007_LIBCPP_INLINE_VISIBILITY 2008_Tp 2009atomic_load_explicit(const atomic<_Tp>* __o, memory_order __m) _NOEXCEPT 2010 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 2011{ 2012 return __o->load(__m); 2013} 2014 2015// atomic_exchange 2016 2017template <class _Tp> 2018_LIBCPP_INLINE_VISIBILITY 2019_Tp 2020atomic_exchange(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2021{ 2022 return __o->exchange(__d); 2023} 2024 2025template <class _Tp> 2026_LIBCPP_INLINE_VISIBILITY 2027_Tp 2028atomic_exchange(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2029{ 2030 return __o->exchange(__d); 2031} 2032 2033// atomic_exchange_explicit 2034 2035template <class _Tp> 2036_LIBCPP_INLINE_VISIBILITY 2037_Tp 2038atomic_exchange_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 2039{ 2040 return __o->exchange(__d, __m); 2041} 2042 2043template <class _Tp> 2044_LIBCPP_INLINE_VISIBILITY 2045_Tp 2046atomic_exchange_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 2047{ 2048 return __o->exchange(__d, __m); 2049} 2050 2051// atomic_compare_exchange_weak 2052 2053template <class _Tp> 2054_LIBCPP_INLINE_VISIBILITY 2055bool 2056atomic_compare_exchange_weak(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2057{ 2058 return __o->compare_exchange_weak(*__e, __d); 2059} 2060 2061template <class _Tp> 2062_LIBCPP_INLINE_VISIBILITY 2063bool 2064atomic_compare_exchange_weak(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2065{ 2066 return __o->compare_exchange_weak(*__e, __d); 2067} 2068 2069// atomic_compare_exchange_strong 2070 2071template <class _Tp> 2072_LIBCPP_INLINE_VISIBILITY 2073bool 2074atomic_compare_exchange_strong(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2075{ 2076 return __o->compare_exchange_strong(*__e, __d); 2077} 2078 2079template <class _Tp> 2080_LIBCPP_INLINE_VISIBILITY 2081bool 2082atomic_compare_exchange_strong(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2083{ 2084 return __o->compare_exchange_strong(*__e, __d); 2085} 2086 2087// atomic_compare_exchange_weak_explicit 2088 2089template <class _Tp> 2090_LIBCPP_INLINE_VISIBILITY 2091bool 2092atomic_compare_exchange_weak_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, 2093 typename atomic<_Tp>::value_type __d, 2094 memory_order __s, memory_order __f) _NOEXCEPT 2095 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2096{ 2097 return __o->compare_exchange_weak(*__e, __d, __s, __f); 2098} 2099 2100template <class _Tp> 2101_LIBCPP_INLINE_VISIBILITY 2102bool 2103atomic_compare_exchange_weak_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d, 2104 memory_order __s, memory_order __f) _NOEXCEPT 2105 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2106{ 2107 return __o->compare_exchange_weak(*__e, __d, __s, __f); 2108} 2109 2110// atomic_compare_exchange_strong_explicit 2111 2112template <class _Tp> 2113_LIBCPP_INLINE_VISIBILITY 2114bool 2115atomic_compare_exchange_strong_explicit(volatile atomic<_Tp>* __o, 2116 typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d, 2117 memory_order __s, memory_order __f) _NOEXCEPT 2118 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2119{ 2120 return __o->compare_exchange_strong(*__e, __d, __s, __f); 2121} 2122 2123template <class _Tp> 2124_LIBCPP_INLINE_VISIBILITY 2125bool 2126atomic_compare_exchange_strong_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, 2127 typename atomic<_Tp>::value_type __d, 2128 memory_order __s, memory_order __f) _NOEXCEPT 2129 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2130{ 2131 return __o->compare_exchange_strong(*__e, __d, __s, __f); 2132} 2133 2134// atomic_wait 2135 2136template <class _Tp> 2137_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2138void atomic_wait(const volatile atomic<_Tp>* __o, 2139 typename atomic<_Tp>::value_type __v) _NOEXCEPT 2140{ 2141 return __o->wait(__v); 2142} 2143 2144template <class _Tp> 2145_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2146void atomic_wait(const atomic<_Tp>* __o, 2147 typename atomic<_Tp>::value_type __v) _NOEXCEPT 2148{ 2149 return __o->wait(__v); 2150} 2151 2152// atomic_wait_explicit 2153 2154template <class _Tp> 2155_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2156void atomic_wait_explicit(const volatile atomic<_Tp>* __o, 2157 typename atomic<_Tp>::value_type __v, 2158 memory_order __m) _NOEXCEPT 2159 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 2160{ 2161 return __o->wait(__v, __m); 2162} 2163 2164template <class _Tp> 2165_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2166void atomic_wait_explicit(const atomic<_Tp>* __o, 2167 typename atomic<_Tp>::value_type __v, 2168 memory_order __m) _NOEXCEPT 2169 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 2170{ 2171 return __o->wait(__v, __m); 2172} 2173 2174// atomic_notify_one 2175 2176template <class _Tp> 2177_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2178void atomic_notify_one(volatile atomic<_Tp>* __o) _NOEXCEPT 2179{ 2180 __o->notify_one(); 2181} 2182template <class _Tp> 2183_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2184void atomic_notify_one(atomic<_Tp>* __o) _NOEXCEPT 2185{ 2186 __o->notify_one(); 2187} 2188 2189// atomic_notify_one 2190 2191template <class _Tp> 2192_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2193void atomic_notify_all(volatile atomic<_Tp>* __o) _NOEXCEPT 2194{ 2195 __o->notify_all(); 2196} 2197template <class _Tp> 2198_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2199void atomic_notify_all(atomic<_Tp>* __o) _NOEXCEPT 2200{ 2201 __o->notify_all(); 2202} 2203 2204// atomic_fetch_add 2205 2206template <class _Tp> 2207_LIBCPP_INLINE_VISIBILITY 2208_Tp 2209atomic_fetch_add(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT 2210{ 2211 return __o->fetch_add(__op); 2212} 2213 2214template <class _Tp> 2215_LIBCPP_INLINE_VISIBILITY 2216_Tp 2217atomic_fetch_add(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT 2218{ 2219 return __o->fetch_add(__op); 2220} 2221 2222// atomic_fetch_add_explicit 2223 2224template <class _Tp> 2225_LIBCPP_INLINE_VISIBILITY 2226_Tp atomic_fetch_add_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT 2227{ 2228 return __o->fetch_add(__op, __m); 2229} 2230 2231template <class _Tp> 2232_LIBCPP_INLINE_VISIBILITY 2233_Tp atomic_fetch_add_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT 2234{ 2235 return __o->fetch_add(__op, __m); 2236} 2237 2238// atomic_fetch_sub 2239 2240template <class _Tp> 2241_LIBCPP_INLINE_VISIBILITY 2242_Tp atomic_fetch_sub(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT 2243{ 2244 return __o->fetch_sub(__op); 2245} 2246 2247template <class _Tp> 2248_LIBCPP_INLINE_VISIBILITY 2249_Tp atomic_fetch_sub(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT 2250{ 2251 return __o->fetch_sub(__op); 2252} 2253 2254// atomic_fetch_sub_explicit 2255 2256template <class _Tp> 2257_LIBCPP_INLINE_VISIBILITY 2258_Tp atomic_fetch_sub_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT 2259{ 2260 return __o->fetch_sub(__op, __m); 2261} 2262 2263template <class _Tp> 2264_LIBCPP_INLINE_VISIBILITY 2265_Tp atomic_fetch_sub_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT 2266{ 2267 return __o->fetch_sub(__op, __m); 2268} 2269 2270// atomic_fetch_and 2271 2272template <class _Tp> 2273_LIBCPP_INLINE_VISIBILITY 2274typename enable_if 2275< 2276 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2277 _Tp 2278>::type 2279atomic_fetch_and(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2280{ 2281 return __o->fetch_and(__op); 2282} 2283 2284template <class _Tp> 2285_LIBCPP_INLINE_VISIBILITY 2286typename enable_if 2287< 2288 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2289 _Tp 2290>::type 2291atomic_fetch_and(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2292{ 2293 return __o->fetch_and(__op); 2294} 2295 2296// atomic_fetch_and_explicit 2297 2298template <class _Tp> 2299_LIBCPP_INLINE_VISIBILITY 2300typename enable_if 2301< 2302 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2303 _Tp 2304>::type 2305atomic_fetch_and_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2306{ 2307 return __o->fetch_and(__op, __m); 2308} 2309 2310template <class _Tp> 2311_LIBCPP_INLINE_VISIBILITY 2312typename enable_if 2313< 2314 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2315 _Tp 2316>::type 2317atomic_fetch_and_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2318{ 2319 return __o->fetch_and(__op, __m); 2320} 2321 2322// atomic_fetch_or 2323 2324template <class _Tp> 2325_LIBCPP_INLINE_VISIBILITY 2326typename enable_if 2327< 2328 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2329 _Tp 2330>::type 2331atomic_fetch_or(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2332{ 2333 return __o->fetch_or(__op); 2334} 2335 2336template <class _Tp> 2337_LIBCPP_INLINE_VISIBILITY 2338typename enable_if 2339< 2340 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2341 _Tp 2342>::type 2343atomic_fetch_or(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2344{ 2345 return __o->fetch_or(__op); 2346} 2347 2348// atomic_fetch_or_explicit 2349 2350template <class _Tp> 2351_LIBCPP_INLINE_VISIBILITY 2352typename enable_if 2353< 2354 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2355 _Tp 2356>::type 2357atomic_fetch_or_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2358{ 2359 return __o->fetch_or(__op, __m); 2360} 2361 2362template <class _Tp> 2363_LIBCPP_INLINE_VISIBILITY 2364typename enable_if 2365< 2366 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2367 _Tp 2368>::type 2369atomic_fetch_or_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2370{ 2371 return __o->fetch_or(__op, __m); 2372} 2373 2374// atomic_fetch_xor 2375 2376template <class _Tp> 2377_LIBCPP_INLINE_VISIBILITY 2378typename enable_if 2379< 2380 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2381 _Tp 2382>::type 2383atomic_fetch_xor(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2384{ 2385 return __o->fetch_xor(__op); 2386} 2387 2388template <class _Tp> 2389_LIBCPP_INLINE_VISIBILITY 2390typename enable_if 2391< 2392 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2393 _Tp 2394>::type 2395atomic_fetch_xor(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2396{ 2397 return __o->fetch_xor(__op); 2398} 2399 2400// atomic_fetch_xor_explicit 2401 2402template <class _Tp> 2403_LIBCPP_INLINE_VISIBILITY 2404typename enable_if 2405< 2406 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2407 _Tp 2408>::type 2409atomic_fetch_xor_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2410{ 2411 return __o->fetch_xor(__op, __m); 2412} 2413 2414template <class _Tp> 2415_LIBCPP_INLINE_VISIBILITY 2416typename enable_if 2417< 2418 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2419 _Tp 2420>::type 2421atomic_fetch_xor_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2422{ 2423 return __o->fetch_xor(__op, __m); 2424} 2425 2426// flag type and operations 2427 2428typedef struct atomic_flag 2429{ 2430 __cxx_atomic_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_; 2431 2432 _LIBCPP_INLINE_VISIBILITY 2433 bool test(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 2434 {return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m);} 2435 _LIBCPP_INLINE_VISIBILITY 2436 bool test(memory_order __m = memory_order_seq_cst) const _NOEXCEPT 2437 {return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m);} 2438 2439 _LIBCPP_INLINE_VISIBILITY 2440 bool test_and_set(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 2441 {return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m);} 2442 _LIBCPP_INLINE_VISIBILITY 2443 bool test_and_set(memory_order __m = memory_order_seq_cst) _NOEXCEPT 2444 {return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m);} 2445 _LIBCPP_INLINE_VISIBILITY 2446 void clear(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 2447 {__cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m);} 2448 _LIBCPP_INLINE_VISIBILITY 2449 void clear(memory_order __m = memory_order_seq_cst) _NOEXCEPT 2450 {__cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m);} 2451 2452 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2453 void wait(bool __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 2454 {__cxx_atomic_wait(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);} 2455 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2456 void wait(bool __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT 2457 {__cxx_atomic_wait(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);} 2458 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2459 void notify_one() volatile _NOEXCEPT 2460 {__cxx_atomic_notify_one(&__a_);} 2461 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2462 void notify_one() _NOEXCEPT 2463 {__cxx_atomic_notify_one(&__a_);} 2464 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2465 void notify_all() volatile _NOEXCEPT 2466 {__cxx_atomic_notify_all(&__a_);} 2467 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2468 void notify_all() _NOEXCEPT 2469 {__cxx_atomic_notify_all(&__a_);} 2470 2471#if _LIBCPP_STD_VER > 17 2472 _LIBCPP_INLINE_VISIBILITY constexpr 2473 atomic_flag() _NOEXCEPT : __a_(false) {} 2474#else 2475 _LIBCPP_INLINE_VISIBILITY 2476 atomic_flag() _NOEXCEPT _LIBCPP_DEFAULT 2477#endif 2478 2479 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR 2480 atomic_flag(bool __b) _NOEXCEPT : __a_(__b) {} // EXTENSION 2481 2482#ifndef _LIBCPP_CXX03_LANG 2483 atomic_flag(const atomic_flag&) = delete; 2484 atomic_flag& operator=(const atomic_flag&) = delete; 2485 atomic_flag& operator=(const atomic_flag&) volatile = delete; 2486#else 2487private: 2488 _LIBCPP_INLINE_VISIBILITY 2489 atomic_flag(const atomic_flag&); 2490 _LIBCPP_INLINE_VISIBILITY 2491 atomic_flag& operator=(const atomic_flag&); 2492 _LIBCPP_INLINE_VISIBILITY 2493 atomic_flag& operator=(const atomic_flag&) volatile; 2494#endif 2495} atomic_flag; 2496 2497 2498inline _LIBCPP_INLINE_VISIBILITY 2499bool 2500atomic_flag_test(const volatile atomic_flag* __o) _NOEXCEPT 2501{ 2502 return __o->test(); 2503} 2504 2505inline _LIBCPP_INLINE_VISIBILITY 2506bool 2507atomic_flag_test(const atomic_flag* __o) _NOEXCEPT 2508{ 2509 return __o->test(); 2510} 2511 2512inline _LIBCPP_INLINE_VISIBILITY 2513bool 2514atomic_flag_test_explicit(const volatile atomic_flag* __o, memory_order __m) _NOEXCEPT 2515{ 2516 return __o->test(__m); 2517} 2518 2519inline _LIBCPP_INLINE_VISIBILITY 2520bool 2521atomic_flag_test_explicit(const atomic_flag* __o, memory_order __m) _NOEXCEPT 2522{ 2523 return __o->test(__m); 2524} 2525 2526inline _LIBCPP_INLINE_VISIBILITY 2527bool 2528atomic_flag_test_and_set(volatile atomic_flag* __o) _NOEXCEPT 2529{ 2530 return __o->test_and_set(); 2531} 2532 2533inline _LIBCPP_INLINE_VISIBILITY 2534bool 2535atomic_flag_test_and_set(atomic_flag* __o) _NOEXCEPT 2536{ 2537 return __o->test_and_set(); 2538} 2539 2540inline _LIBCPP_INLINE_VISIBILITY 2541bool 2542atomic_flag_test_and_set_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT 2543{ 2544 return __o->test_and_set(__m); 2545} 2546 2547inline _LIBCPP_INLINE_VISIBILITY 2548bool 2549atomic_flag_test_and_set_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT 2550{ 2551 return __o->test_and_set(__m); 2552} 2553 2554inline _LIBCPP_INLINE_VISIBILITY 2555void 2556atomic_flag_clear(volatile atomic_flag* __o) _NOEXCEPT 2557{ 2558 __o->clear(); 2559} 2560 2561inline _LIBCPP_INLINE_VISIBILITY 2562void 2563atomic_flag_clear(atomic_flag* __o) _NOEXCEPT 2564{ 2565 __o->clear(); 2566} 2567 2568inline _LIBCPP_INLINE_VISIBILITY 2569void 2570atomic_flag_clear_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT 2571{ 2572 __o->clear(__m); 2573} 2574 2575inline _LIBCPP_INLINE_VISIBILITY 2576void 2577atomic_flag_clear_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT 2578{ 2579 __o->clear(__m); 2580} 2581 2582inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2583void 2584atomic_flag_wait(const volatile atomic_flag* __o, bool __v) _NOEXCEPT 2585{ 2586 __o->wait(__v); 2587} 2588 2589inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2590void 2591atomic_flag_wait(const atomic_flag* __o, bool __v) _NOEXCEPT 2592{ 2593 __o->wait(__v); 2594} 2595 2596inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2597void 2598atomic_flag_wait_explicit(const volatile atomic_flag* __o, 2599 bool __v, memory_order __m) _NOEXCEPT 2600{ 2601 __o->wait(__v, __m); 2602} 2603 2604inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2605void 2606atomic_flag_wait_explicit(const atomic_flag* __o, 2607 bool __v, memory_order __m) _NOEXCEPT 2608{ 2609 __o->wait(__v, __m); 2610} 2611 2612inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2613void 2614atomic_flag_notify_one(volatile atomic_flag* __o) _NOEXCEPT 2615{ 2616 __o->notify_one(); 2617} 2618 2619inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2620void 2621atomic_flag_notify_one(atomic_flag* __o) _NOEXCEPT 2622{ 2623 __o->notify_one(); 2624} 2625 2626inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2627void 2628atomic_flag_notify_all(volatile atomic_flag* __o) _NOEXCEPT 2629{ 2630 __o->notify_all(); 2631} 2632 2633inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2634void 2635atomic_flag_notify_all(atomic_flag* __o) _NOEXCEPT 2636{ 2637 __o->notify_all(); 2638} 2639 2640// fences 2641 2642inline _LIBCPP_INLINE_VISIBILITY 2643void 2644atomic_thread_fence(memory_order __m) _NOEXCEPT 2645{ 2646 __cxx_atomic_thread_fence(__m); 2647} 2648 2649inline _LIBCPP_INLINE_VISIBILITY 2650void 2651atomic_signal_fence(memory_order __m) _NOEXCEPT 2652{ 2653 __cxx_atomic_signal_fence(__m); 2654} 2655 2656// Atomics for standard typedef types 2657 2658typedef atomic<bool> atomic_bool; 2659typedef atomic<char> atomic_char; 2660typedef atomic<signed char> atomic_schar; 2661typedef atomic<unsigned char> atomic_uchar; 2662typedef atomic<short> atomic_short; 2663typedef atomic<unsigned short> atomic_ushort; 2664typedef atomic<int> atomic_int; 2665typedef atomic<unsigned int> atomic_uint; 2666typedef atomic<long> atomic_long; 2667typedef atomic<unsigned long> atomic_ulong; 2668typedef atomic<long long> atomic_llong; 2669typedef atomic<unsigned long long> atomic_ullong; 2670#ifndef _LIBCPP_HAS_NO_CHAR8_T 2671typedef atomic<char8_t> atomic_char8_t; 2672#endif 2673typedef atomic<char16_t> atomic_char16_t; 2674typedef atomic<char32_t> atomic_char32_t; 2675typedef atomic<wchar_t> atomic_wchar_t; 2676 2677typedef atomic<int_least8_t> atomic_int_least8_t; 2678typedef atomic<uint_least8_t> atomic_uint_least8_t; 2679typedef atomic<int_least16_t> atomic_int_least16_t; 2680typedef atomic<uint_least16_t> atomic_uint_least16_t; 2681typedef atomic<int_least32_t> atomic_int_least32_t; 2682typedef atomic<uint_least32_t> atomic_uint_least32_t; 2683typedef atomic<int_least64_t> atomic_int_least64_t; 2684typedef atomic<uint_least64_t> atomic_uint_least64_t; 2685 2686typedef atomic<int_fast8_t> atomic_int_fast8_t; 2687typedef atomic<uint_fast8_t> atomic_uint_fast8_t; 2688typedef atomic<int_fast16_t> atomic_int_fast16_t; 2689typedef atomic<uint_fast16_t> atomic_uint_fast16_t; 2690typedef atomic<int_fast32_t> atomic_int_fast32_t; 2691typedef atomic<uint_fast32_t> atomic_uint_fast32_t; 2692typedef atomic<int_fast64_t> atomic_int_fast64_t; 2693typedef atomic<uint_fast64_t> atomic_uint_fast64_t; 2694 2695typedef atomic< int8_t> atomic_int8_t; 2696typedef atomic<uint8_t> atomic_uint8_t; 2697typedef atomic< int16_t> atomic_int16_t; 2698typedef atomic<uint16_t> atomic_uint16_t; 2699typedef atomic< int32_t> atomic_int32_t; 2700typedef atomic<uint32_t> atomic_uint32_t; 2701typedef atomic< int64_t> atomic_int64_t; 2702typedef atomic<uint64_t> atomic_uint64_t; 2703 2704typedef atomic<intptr_t> atomic_intptr_t; 2705typedef atomic<uintptr_t> atomic_uintptr_t; 2706typedef atomic<size_t> atomic_size_t; 2707typedef atomic<ptrdiff_t> atomic_ptrdiff_t; 2708typedef atomic<intmax_t> atomic_intmax_t; 2709typedef atomic<uintmax_t> atomic_uintmax_t; 2710 2711// atomic_*_lock_free : prefer the contention type most highly, then the largest lock-free type 2712 2713#ifdef __cpp_lib_atomic_is_always_lock_free 2714# define _LIBCPP_CONTENTION_LOCK_FREE __atomic_always_lock_free(sizeof(__cxx_contention_t), 0) 2715#else 2716# define _LIBCPP_CONTENTION_LOCK_FREE false 2717#endif 2718 2719#if ATOMIC_LLONG_LOCK_FREE == 2 2720typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, long long>::type __libcpp_signed_lock_free; 2721typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned long long>::type __libcpp_unsigned_lock_free; 2722#elif ATOMIC_INT_LOCK_FREE == 2 2723typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, int>::type __libcpp_signed_lock_free; 2724typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned int>::type __libcpp_unsigned_lock_free; 2725#elif ATOMIC_SHORT_LOCK_FREE == 2 2726typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, short>::type __libcpp_signed_lock_free; 2727typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned short>::type __libcpp_unsigned_lock_free; 2728#elif ATOMIC_CHAR_LOCK_FREE == 2 2729typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, char>::type __libcpp_signed_lock_free; 2730typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned char>::type __libcpp_unsigned_lock_free; 2731#else 2732 // No signed/unsigned lock-free types 2733#endif 2734 2735typedef atomic<__libcpp_signed_lock_free> atomic_signed_lock_free; 2736typedef atomic<__libcpp_unsigned_lock_free> atomic_unsigned_lock_free; 2737 2738#define ATOMIC_FLAG_INIT {false} 2739#define ATOMIC_VAR_INIT(__v) {__v} 2740 2741_LIBCPP_END_NAMESPACE_STD 2742 2743#endif // _LIBCPP_ATOMIC 2744