1// -*- C++ -*- 2//===--------------------------- atomic -----------------------------------===// 3// 4// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 5// See https://llvm.org/LICENSE.txt for license information. 6// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 7// 8//===----------------------------------------------------------------------===// 9 10#ifndef _LIBCPP_ATOMIC 11#define _LIBCPP_ATOMIC 12 13/* 14 atomic synopsis 15 16namespace std 17{ 18 19// feature test macro [version.syn] 20 21#define __cpp_lib_atomic_is_always_lock_free 22#define __cpp_lib_atomic_flag_test 23#define __cpp_lib_atomic_lock_free_type_aliases 24#define __cpp_lib_atomic_wait 25 26 // order and consistency 27 28 enum memory_order: unspecified // enum class in C++20 29 { 30 relaxed, 31 consume, // load-consume 32 acquire, // load-acquire 33 release, // store-release 34 acq_rel, // store-release load-acquire 35 seq_cst // store-release load-acquire 36 }; 37 38 inline constexpr auto memory_order_relaxed = memory_order::relaxed; 39 inline constexpr auto memory_order_consume = memory_order::consume; 40 inline constexpr auto memory_order_acquire = memory_order::acquire; 41 inline constexpr auto memory_order_release = memory_order::release; 42 inline constexpr auto memory_order_acq_rel = memory_order::acq_rel; 43 inline constexpr auto memory_order_seq_cst = memory_order::seq_cst; 44 45template <class T> T kill_dependency(T y) noexcept; 46 47// lock-free property 48 49#define ATOMIC_BOOL_LOCK_FREE unspecified 50#define ATOMIC_CHAR_LOCK_FREE unspecified 51#define ATOMIC_CHAR8_T_LOCK_FREE unspecified // C++20 52#define ATOMIC_CHAR16_T_LOCK_FREE unspecified 53#define ATOMIC_CHAR32_T_LOCK_FREE unspecified 54#define ATOMIC_WCHAR_T_LOCK_FREE unspecified 55#define ATOMIC_SHORT_LOCK_FREE unspecified 56#define ATOMIC_INT_LOCK_FREE unspecified 57#define ATOMIC_LONG_LOCK_FREE unspecified 58#define ATOMIC_LLONG_LOCK_FREE unspecified 59#define ATOMIC_POINTER_LOCK_FREE unspecified 60 61template <class T> 62struct atomic 63{ 64 using value_type = T; 65 66 static constexpr bool is_always_lock_free; 67 bool is_lock_free() const volatile noexcept; 68 bool is_lock_free() const noexcept; 69 70 atomic() noexcept = default; // until C++20 71 constexpr atomic() noexcept(is_nothrow_default_constructible_v<T>); // since C++20 72 constexpr atomic(T desr) noexcept; 73 atomic(const atomic&) = delete; 74 atomic& operator=(const atomic&) = delete; 75 atomic& operator=(const atomic&) volatile = delete; 76 77 T load(memory_order m = memory_order_seq_cst) const volatile noexcept; 78 T load(memory_order m = memory_order_seq_cst) const noexcept; 79 operator T() const volatile noexcept; 80 operator T() const noexcept; 81 void store(T desr, memory_order m = memory_order_seq_cst) volatile noexcept; 82 void store(T desr, memory_order m = memory_order_seq_cst) noexcept; 83 T operator=(T) volatile noexcept; 84 T operator=(T) noexcept; 85 86 T exchange(T desr, memory_order m = memory_order_seq_cst) volatile noexcept; 87 T exchange(T desr, memory_order m = memory_order_seq_cst) noexcept; 88 bool compare_exchange_weak(T& expc, T desr, 89 memory_order s, memory_order f) volatile noexcept; 90 bool compare_exchange_weak(T& expc, T desr, memory_order s, memory_order f) noexcept; 91 bool compare_exchange_strong(T& expc, T desr, 92 memory_order s, memory_order f) volatile noexcept; 93 bool compare_exchange_strong(T& expc, T desr, 94 memory_order s, memory_order f) noexcept; 95 bool compare_exchange_weak(T& expc, T desr, 96 memory_order m = memory_order_seq_cst) volatile noexcept; 97 bool compare_exchange_weak(T& expc, T desr, 98 memory_order m = memory_order_seq_cst) noexcept; 99 bool compare_exchange_strong(T& expc, T desr, 100 memory_order m = memory_order_seq_cst) volatile noexcept; 101 bool compare_exchange_strong(T& expc, T desr, 102 memory_order m = memory_order_seq_cst) noexcept; 103 104 void wait(T, memory_order = memory_order::seq_cst) const volatile noexcept; 105 void wait(T, memory_order = memory_order::seq_cst) const noexcept; 106 void notify_one() volatile noexcept; 107 void notify_one() noexcept; 108 void notify_all() volatile noexcept; 109 void notify_all() noexcept; 110}; 111 112template <> 113struct atomic<integral> 114{ 115 using value_type = integral; 116 using difference_type = value_type; 117 118 static constexpr bool is_always_lock_free; 119 bool is_lock_free() const volatile noexcept; 120 bool is_lock_free() const noexcept; 121 122 atomic() noexcept = default; 123 constexpr atomic(integral desr) noexcept; 124 atomic(const atomic&) = delete; 125 atomic& operator=(const atomic&) = delete; 126 atomic& operator=(const atomic&) volatile = delete; 127 128 integral load(memory_order m = memory_order_seq_cst) const volatile noexcept; 129 integral load(memory_order m = memory_order_seq_cst) const noexcept; 130 operator integral() const volatile noexcept; 131 operator integral() const noexcept; 132 void store(integral desr, memory_order m = memory_order_seq_cst) volatile noexcept; 133 void store(integral desr, memory_order m = memory_order_seq_cst) noexcept; 134 integral operator=(integral desr) volatile noexcept; 135 integral operator=(integral desr) noexcept; 136 137 integral exchange(integral desr, 138 memory_order m = memory_order_seq_cst) volatile noexcept; 139 integral exchange(integral desr, memory_order m = memory_order_seq_cst) noexcept; 140 bool compare_exchange_weak(integral& expc, integral desr, 141 memory_order s, memory_order f) volatile noexcept; 142 bool compare_exchange_weak(integral& expc, integral desr, 143 memory_order s, memory_order f) noexcept; 144 bool compare_exchange_strong(integral& expc, integral desr, 145 memory_order s, memory_order f) volatile noexcept; 146 bool compare_exchange_strong(integral& expc, integral desr, 147 memory_order s, memory_order f) noexcept; 148 bool compare_exchange_weak(integral& expc, integral desr, 149 memory_order m = memory_order_seq_cst) volatile noexcept; 150 bool compare_exchange_weak(integral& expc, integral desr, 151 memory_order m = memory_order_seq_cst) noexcept; 152 bool compare_exchange_strong(integral& expc, integral desr, 153 memory_order m = memory_order_seq_cst) volatile noexcept; 154 bool compare_exchange_strong(integral& expc, integral desr, 155 memory_order m = memory_order_seq_cst) noexcept; 156 157 integral fetch_add(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 158 integral fetch_add(integral op, memory_order m = memory_order_seq_cst) noexcept; 159 integral fetch_sub(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 160 integral fetch_sub(integral op, memory_order m = memory_order_seq_cst) noexcept; 161 integral fetch_and(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 162 integral fetch_and(integral op, memory_order m = memory_order_seq_cst) noexcept; 163 integral fetch_or(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 164 integral fetch_or(integral op, memory_order m = memory_order_seq_cst) noexcept; 165 integral fetch_xor(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 166 integral fetch_xor(integral op, memory_order m = memory_order_seq_cst) noexcept; 167 168 integral operator++(int) volatile noexcept; 169 integral operator++(int) noexcept; 170 integral operator--(int) volatile noexcept; 171 integral operator--(int) noexcept; 172 integral operator++() volatile noexcept; 173 integral operator++() noexcept; 174 integral operator--() volatile noexcept; 175 integral operator--() noexcept; 176 integral operator+=(integral op) volatile noexcept; 177 integral operator+=(integral op) noexcept; 178 integral operator-=(integral op) volatile noexcept; 179 integral operator-=(integral op) noexcept; 180 integral operator&=(integral op) volatile noexcept; 181 integral operator&=(integral op) noexcept; 182 integral operator|=(integral op) volatile noexcept; 183 integral operator|=(integral op) noexcept; 184 integral operator^=(integral op) volatile noexcept; 185 integral operator^=(integral op) noexcept; 186 187 void wait(integral, memory_order = memory_order::seq_cst) const volatile noexcept; 188 void wait(integral, memory_order = memory_order::seq_cst) const noexcept; 189 void notify_one() volatile noexcept; 190 void notify_one() noexcept; 191 void notify_all() volatile noexcept; 192 void notify_all() noexcept; 193}; 194 195template <class T> 196struct atomic<T*> 197{ 198 using value_type = T*; 199 using difference_type = ptrdiff_t; 200 201 static constexpr bool is_always_lock_free; 202 bool is_lock_free() const volatile noexcept; 203 bool is_lock_free() const noexcept; 204 205 atomic() noexcept = default; // until C++20 206 constexpr atomic() noexcept; // since C++20 207 constexpr atomic(T* desr) noexcept; 208 atomic(const atomic&) = delete; 209 atomic& operator=(const atomic&) = delete; 210 atomic& operator=(const atomic&) volatile = delete; 211 212 T* load(memory_order m = memory_order_seq_cst) const volatile noexcept; 213 T* load(memory_order m = memory_order_seq_cst) const noexcept; 214 operator T*() const volatile noexcept; 215 operator T*() const noexcept; 216 void store(T* desr, memory_order m = memory_order_seq_cst) volatile noexcept; 217 void store(T* desr, memory_order m = memory_order_seq_cst) noexcept; 218 T* operator=(T*) volatile noexcept; 219 T* operator=(T*) noexcept; 220 221 T* exchange(T* desr, memory_order m = memory_order_seq_cst) volatile noexcept; 222 T* exchange(T* desr, memory_order m = memory_order_seq_cst) noexcept; 223 bool compare_exchange_weak(T*& expc, T* desr, 224 memory_order s, memory_order f) volatile noexcept; 225 bool compare_exchange_weak(T*& expc, T* desr, 226 memory_order s, memory_order f) noexcept; 227 bool compare_exchange_strong(T*& expc, T* desr, 228 memory_order s, memory_order f) volatile noexcept; 229 bool compare_exchange_strong(T*& expc, T* desr, 230 memory_order s, memory_order f) noexcept; 231 bool compare_exchange_weak(T*& expc, T* desr, 232 memory_order m = memory_order_seq_cst) volatile noexcept; 233 bool compare_exchange_weak(T*& expc, T* desr, 234 memory_order m = memory_order_seq_cst) noexcept; 235 bool compare_exchange_strong(T*& expc, T* desr, 236 memory_order m = memory_order_seq_cst) volatile noexcept; 237 bool compare_exchange_strong(T*& expc, T* desr, 238 memory_order m = memory_order_seq_cst) noexcept; 239 T* fetch_add(ptrdiff_t op, memory_order m = memory_order_seq_cst) volatile noexcept; 240 T* fetch_add(ptrdiff_t op, memory_order m = memory_order_seq_cst) noexcept; 241 T* fetch_sub(ptrdiff_t op, memory_order m = memory_order_seq_cst) volatile noexcept; 242 T* fetch_sub(ptrdiff_t op, memory_order m = memory_order_seq_cst) noexcept; 243 244 T* operator++(int) volatile noexcept; 245 T* operator++(int) noexcept; 246 T* operator--(int) volatile noexcept; 247 T* operator--(int) noexcept; 248 T* operator++() volatile noexcept; 249 T* operator++() noexcept; 250 T* operator--() volatile noexcept; 251 T* operator--() noexcept; 252 T* operator+=(ptrdiff_t op) volatile noexcept; 253 T* operator+=(ptrdiff_t op) noexcept; 254 T* operator-=(ptrdiff_t op) volatile noexcept; 255 T* operator-=(ptrdiff_t op) noexcept; 256 257 void wait(T*, memory_order = memory_order::seq_cst) const volatile noexcept; 258 void wait(T*, memory_order = memory_order::seq_cst) const noexcept; 259 void notify_one() volatile noexcept; 260 void notify_one() noexcept; 261 void notify_all() volatile noexcept; 262 void notify_all() noexcept; 263}; 264 265 266template <class T> 267 bool atomic_is_lock_free(const volatile atomic<T>* obj) noexcept; 268 269template <class T> 270 bool atomic_is_lock_free(const atomic<T>* obj) noexcept; 271 272template <class T> 273 void atomic_store(volatile atomic<T>* obj, T desr) noexcept; 274 275template <class T> 276 void atomic_store(atomic<T>* obj, T desr) noexcept; 277 278template <class T> 279 void atomic_store_explicit(volatile atomic<T>* obj, T desr, memory_order m) noexcept; 280 281template <class T> 282 void atomic_store_explicit(atomic<T>* obj, T desr, memory_order m) noexcept; 283 284template <class T> 285 T atomic_load(const volatile atomic<T>* obj) noexcept; 286 287template <class T> 288 T atomic_load(const atomic<T>* obj) noexcept; 289 290template <class T> 291 T atomic_load_explicit(const volatile atomic<T>* obj, memory_order m) noexcept; 292 293template <class T> 294 T atomic_load_explicit(const atomic<T>* obj, memory_order m) noexcept; 295 296template <class T> 297 T atomic_exchange(volatile atomic<T>* obj, T desr) noexcept; 298 299template <class T> 300 T atomic_exchange(atomic<T>* obj, T desr) noexcept; 301 302template <class T> 303 T atomic_exchange_explicit(volatile atomic<T>* obj, T desr, memory_order m) noexcept; 304 305template <class T> 306 T atomic_exchange_explicit(atomic<T>* obj, T desr, memory_order m) noexcept; 307 308template <class T> 309 bool atomic_compare_exchange_weak(volatile atomic<T>* obj, T* expc, T desr) noexcept; 310 311template <class T> 312 bool atomic_compare_exchange_weak(atomic<T>* obj, T* expc, T desr) noexcept; 313 314template <class T> 315 bool atomic_compare_exchange_strong(volatile atomic<T>* obj, T* expc, T desr) noexcept; 316 317template <class T> 318 bool atomic_compare_exchange_strong(atomic<T>* obj, T* expc, T desr) noexcept; 319 320template <class T> 321 bool atomic_compare_exchange_weak_explicit(volatile atomic<T>* obj, T* expc, 322 T desr, 323 memory_order s, memory_order f) noexcept; 324 325template <class T> 326 bool atomic_compare_exchange_weak_explicit(atomic<T>* obj, T* expc, T desr, 327 memory_order s, memory_order f) noexcept; 328 329template <class T> 330 bool atomic_compare_exchange_strong_explicit(volatile atomic<T>* obj, 331 T* expc, T desr, 332 memory_order s, memory_order f) noexcept; 333 334template <class T> 335 bool atomic_compare_exchange_strong_explicit(atomic<T>* obj, T* expc, 336 T desr, 337 memory_order s, memory_order f) noexcept; 338 339template <class T> 340 void atomic_wait(const volatile atomic<T>* obj, T old) noexcept; 341 342template <class T> 343 void atomic_wait(const atomic<T>* obj, T old) noexcept; 344 345template <class T> 346 void atomic_wait_explicit(const volatile atomic<T>* obj, T old, memory_order m) noexcept; 347 348template <class T> 349 void atomic_wait_explicit(const atomic<T>* obj, T old, memory_order m) noexcept; 350 351template <class T> 352 void atomic_one(volatile atomic<T>* obj) noexcept; 353 354template <class T> 355 void atomic_one(atomic<T>* obj) noexcept; 356 357template <class T> 358 void atomic_all(volatile atomic<T>* obj) noexcept; 359 360template <class T> 361 void atomic_all(atomic<T>* obj) noexcept; 362 363template <class Integral> 364 Integral atomic_fetch_add(volatile atomic<Integral>* obj, Integral op) noexcept; 365 366template <class Integral> 367 Integral atomic_fetch_add(atomic<Integral>* obj, Integral op) noexcept; 368 369template <class Integral> 370 Integral atomic_fetch_add_explicit(volatile atomic<Integral>* obj, Integral op, 371 memory_order m) noexcept; 372template <class Integral> 373 Integral atomic_fetch_add_explicit(atomic<Integral>* obj, Integral op, 374 memory_order m) noexcept; 375template <class Integral> 376 Integral atomic_fetch_sub(volatile atomic<Integral>* obj, Integral op) noexcept; 377 378template <class Integral> 379 Integral atomic_fetch_sub(atomic<Integral>* obj, Integral op) noexcept; 380 381template <class Integral> 382 Integral atomic_fetch_sub_explicit(volatile atomic<Integral>* obj, Integral op, 383 memory_order m) noexcept; 384 385template <class Integral> 386 Integral atomic_fetch_sub_explicit(atomic<Integral>* obj, Integral op, 387 memory_order m) noexcept; 388 389template <class Integral> 390 Integral atomic_fetch_and(volatile atomic<Integral>* obj, Integral op) noexcept; 391 392template <class Integral> 393 Integral atomic_fetch_and(atomic<Integral>* obj, Integral op) noexcept; 394 395template <class Integral> 396 Integral atomic_fetch_and_explicit(volatile atomic<Integral>* obj, Integral op, 397 memory_order m) noexcept; 398 399template <class Integral> 400 Integral atomic_fetch_and_explicit(atomic<Integral>* obj, Integral op, 401 memory_order m) noexcept; 402 403template <class Integral> 404 Integral atomic_fetch_or(volatile atomic<Integral>* obj, Integral op) noexcept; 405 406template <class Integral> 407 Integral atomic_fetch_or(atomic<Integral>* obj, Integral op) noexcept; 408 409template <class Integral> 410 Integral atomic_fetch_or_explicit(volatile atomic<Integral>* obj, Integral op, 411 memory_order m) noexcept; 412 413template <class Integral> 414 Integral atomic_fetch_or_explicit(atomic<Integral>* obj, Integral op, 415 memory_order m) noexcept; 416 417template <class Integral> 418 Integral atomic_fetch_xor(volatile atomic<Integral>* obj, Integral op) noexcept; 419 420template <class Integral> 421 Integral atomic_fetch_xor(atomic<Integral>* obj, Integral op) noexcept; 422 423template <class Integral> 424 Integral atomic_fetch_xor_explicit(volatile atomic<Integral>* obj, Integral op, 425 memory_order m) noexcept; 426 427template <class Integral> 428 Integral atomic_fetch_xor_explicit(atomic<Integral>* obj, Integral op, 429 memory_order m) noexcept; 430 431template <class T> 432 T* atomic_fetch_add(volatile atomic<T*>* obj, ptrdiff_t op) noexcept; 433 434template <class T> 435 T* atomic_fetch_add(atomic<T*>* obj, ptrdiff_t op) noexcept; 436 437template <class T> 438 T* atomic_fetch_add_explicit(volatile atomic<T*>* obj, ptrdiff_t op, 439 memory_order m) noexcept; 440 441template <class T> 442 T* atomic_fetch_add_explicit(atomic<T*>* obj, ptrdiff_t op, memory_order m) noexcept; 443 444template <class T> 445 T* atomic_fetch_sub(volatile atomic<T*>* obj, ptrdiff_t op) noexcept; 446 447template <class T> 448 T* atomic_fetch_sub(atomic<T*>* obj, ptrdiff_t op) noexcept; 449 450template <class T> 451 T* atomic_fetch_sub_explicit(volatile atomic<T*>* obj, ptrdiff_t op, 452 memory_order m) noexcept; 453 454template <class T> 455 T* atomic_fetch_sub_explicit(atomic<T*>* obj, ptrdiff_t op, memory_order m) noexcept; 456 457// Atomics for standard typedef types 458 459typedef atomic<bool> atomic_bool; 460typedef atomic<char> atomic_char; 461typedef atomic<signed char> atomic_schar; 462typedef atomic<unsigned char> atomic_uchar; 463typedef atomic<short> atomic_short; 464typedef atomic<unsigned short> atomic_ushort; 465typedef atomic<int> atomic_int; 466typedef atomic<unsigned int> atomic_uint; 467typedef atomic<long> atomic_long; 468typedef atomic<unsigned long> atomic_ulong; 469typedef atomic<long long> atomic_llong; 470typedef atomic<unsigned long long> atomic_ullong; 471typedef atomic<char8_t> atomic_char8_t; // C++20 472typedef atomic<char16_t> atomic_char16_t; 473typedef atomic<char32_t> atomic_char32_t; 474typedef atomic<wchar_t> atomic_wchar_t; 475 476typedef atomic<int_least8_t> atomic_int_least8_t; 477typedef atomic<uint_least8_t> atomic_uint_least8_t; 478typedef atomic<int_least16_t> atomic_int_least16_t; 479typedef atomic<uint_least16_t> atomic_uint_least16_t; 480typedef atomic<int_least32_t> atomic_int_least32_t; 481typedef atomic<uint_least32_t> atomic_uint_least32_t; 482typedef atomic<int_least64_t> atomic_int_least64_t; 483typedef atomic<uint_least64_t> atomic_uint_least64_t; 484 485typedef atomic<int_fast8_t> atomic_int_fast8_t; 486typedef atomic<uint_fast8_t> atomic_uint_fast8_t; 487typedef atomic<int_fast16_t> atomic_int_fast16_t; 488typedef atomic<uint_fast16_t> atomic_uint_fast16_t; 489typedef atomic<int_fast32_t> atomic_int_fast32_t; 490typedef atomic<uint_fast32_t> atomic_uint_fast32_t; 491typedef atomic<int_fast64_t> atomic_int_fast64_t; 492typedef atomic<uint_fast64_t> atomic_uint_fast64_t; 493 494typedef atomic<int8_t> atomic_int8_t; 495typedef atomic<uint8_t> atomic_uint8_t; 496typedef atomic<int16_t> atomic_int16_t; 497typedef atomic<uint16_t> atomic_uint16_t; 498typedef atomic<int32_t> atomic_int32_t; 499typedef atomic<uint32_t> atomic_uint32_t; 500typedef atomic<int64_t> atomic_int64_t; 501typedef atomic<uint64_t> atomic_uint64_t; 502 503typedef atomic<intptr_t> atomic_intptr_t; 504typedef atomic<uintptr_t> atomic_uintptr_t; 505typedef atomic<size_t> atomic_size_t; 506typedef atomic<ptrdiff_t> atomic_ptrdiff_t; 507typedef atomic<intmax_t> atomic_intmax_t; 508typedef atomic<uintmax_t> atomic_uintmax_t; 509 510// flag type and operations 511 512typedef struct atomic_flag 513{ 514 atomic_flag() noexcept = default; // until C++20 515 constexpr atomic_flag() noexcept; // since C++20 516 atomic_flag(const atomic_flag&) = delete; 517 atomic_flag& operator=(const atomic_flag&) = delete; 518 atomic_flag& operator=(const atomic_flag&) volatile = delete; 519 520 bool test(memory_order m = memory_order_seq_cst) volatile noexcept; 521 bool test(memory_order m = memory_order_seq_cst) noexcept; 522 bool test_and_set(memory_order m = memory_order_seq_cst) volatile noexcept; 523 bool test_and_set(memory_order m = memory_order_seq_cst) noexcept; 524 void clear(memory_order m = memory_order_seq_cst) volatile noexcept; 525 void clear(memory_order m = memory_order_seq_cst) noexcept; 526 527 void wait(bool, memory_order = memory_order::seq_cst) const volatile noexcept; 528 void wait(bool, memory_order = memory_order::seq_cst) const noexcept; 529 void notify_one() volatile noexcept; 530 void notify_one() noexcept; 531 void notify_all() volatile noexcept; 532 void notify_all() noexcept; 533} atomic_flag; 534 535bool atomic_flag_test(volatile atomic_flag* obj) noexcept; 536bool atomic_flag_test(atomic_flag* obj) noexcept; 537bool atomic_flag_test_explicit(volatile atomic_flag* obj, 538 memory_order m) noexcept; 539bool atomic_flag_test_explicit(atomic_flag* obj, memory_order m) noexcept; 540bool atomic_flag_test_and_set(volatile atomic_flag* obj) noexcept; 541bool atomic_flag_test_and_set(atomic_flag* obj) noexcept; 542bool atomic_flag_test_and_set_explicit(volatile atomic_flag* obj, 543 memory_order m) noexcept; 544bool atomic_flag_test_and_set_explicit(atomic_flag* obj, memory_order m) noexcept; 545void atomic_flag_clear(volatile atomic_flag* obj) noexcept; 546void atomic_flag_clear(atomic_flag* obj) noexcept; 547void atomic_flag_clear_explicit(volatile atomic_flag* obj, memory_order m) noexcept; 548void atomic_flag_clear_explicit(atomic_flag* obj, memory_order m) noexcept; 549 550void atomic_wait(const volatile atomic_flag* obj, T old) noexcept; 551void atomic_wait(const atomic_flag* obj, T old) noexcept; 552void atomic_wait_explicit(const volatile atomic_flag* obj, T old, memory_order m) noexcept; 553void atomic_wait_explicit(const atomic_flag* obj, T old, memory_order m) noexcept; 554void atomic_one(volatile atomic_flag* obj) noexcept; 555void atomic_one(atomic_flag* obj) noexcept; 556void atomic_all(volatile atomic_flag* obj) noexcept; 557void atomic_all(atomic_flag* obj) noexcept; 558 559// fences 560 561void atomic_thread_fence(memory_order m) noexcept; 562void atomic_signal_fence(memory_order m) noexcept; 563 564// deprecated 565 566template <class T> 567 void atomic_init(volatile atomic<T>* obj, typename atomic<T>::value_type desr) noexcept; 568 569template <class T> 570 void atomic_init(atomic<T>* obj, typename atomic<T>::value_type desr) noexcept; 571 572#define ATOMIC_VAR_INIT(value) see below 573 574#define ATOMIC_FLAG_INIT see below 575 576} // std 577 578*/ 579 580#include <__availability> 581#include <__config> 582#include <__threading_support> 583#include <cstddef> 584#include <cstdint> 585#include <cstring> 586#include <type_traits> 587#include <version> 588 589#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) 590#pragma GCC system_header 591#endif 592 593#ifdef _LIBCPP_HAS_NO_THREADS 594# error <atomic> is not supported on this single threaded system 595#endif 596#ifdef _LIBCPP_HAS_NO_ATOMIC_HEADER 597# error <atomic> is not implemented 598#endif 599#ifdef kill_dependency 600# error C++ standard library is incompatible with <stdatomic.h> 601#endif 602 603#define _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) \ 604 _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_consume || \ 605 __m == memory_order_acquire || \ 606 __m == memory_order_acq_rel, \ 607 "memory order argument to atomic operation is invalid") 608 609#define _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) \ 610 _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_release || \ 611 __m == memory_order_acq_rel, \ 612 "memory order argument to atomic operation is invalid") 613 614#define _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__m, __f) \ 615 _LIBCPP_DIAGNOSE_WARNING(__f == memory_order_release || \ 616 __f == memory_order_acq_rel, \ 617 "memory order argument to atomic operation is invalid") 618 619_LIBCPP_BEGIN_NAMESPACE_STD 620 621// Figure out what the underlying type for `memory_order` would be if it were 622// declared as an unscoped enum (accounting for -fshort-enums). Use this result 623// to pin the underlying type in C++20. 624enum __legacy_memory_order { 625 __mo_relaxed, 626 __mo_consume, 627 __mo_acquire, 628 __mo_release, 629 __mo_acq_rel, 630 __mo_seq_cst 631}; 632 633typedef underlying_type<__legacy_memory_order>::type __memory_order_underlying_t; 634 635#if _LIBCPP_STD_VER > 17 636 637enum class memory_order : __memory_order_underlying_t { 638 relaxed = __mo_relaxed, 639 consume = __mo_consume, 640 acquire = __mo_acquire, 641 release = __mo_release, 642 acq_rel = __mo_acq_rel, 643 seq_cst = __mo_seq_cst 644}; 645 646inline constexpr auto memory_order_relaxed = memory_order::relaxed; 647inline constexpr auto memory_order_consume = memory_order::consume; 648inline constexpr auto memory_order_acquire = memory_order::acquire; 649inline constexpr auto memory_order_release = memory_order::release; 650inline constexpr auto memory_order_acq_rel = memory_order::acq_rel; 651inline constexpr auto memory_order_seq_cst = memory_order::seq_cst; 652 653#else 654 655typedef enum memory_order { 656 memory_order_relaxed = __mo_relaxed, 657 memory_order_consume = __mo_consume, 658 memory_order_acquire = __mo_acquire, 659 memory_order_release = __mo_release, 660 memory_order_acq_rel = __mo_acq_rel, 661 memory_order_seq_cst = __mo_seq_cst, 662} memory_order; 663 664#endif // _LIBCPP_STD_VER > 17 665 666template <typename _Tp> _LIBCPP_INLINE_VISIBILITY 667bool __cxx_nonatomic_compare_equal(_Tp const& __lhs, _Tp const& __rhs) { 668 return _VSTD::memcmp(&__lhs, &__rhs, sizeof(_Tp)) == 0; 669} 670 671static_assert((is_same<underlying_type<memory_order>::type, __memory_order_underlying_t>::value), 672 "unexpected underlying type for std::memory_order"); 673 674#if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) || \ 675 defined(_LIBCPP_ATOMIC_ONLY_USE_BUILTINS) 676 677// [atomics.types.generic]p1 guarantees _Tp is trivially copyable. Because 678// the default operator= in an object is not volatile, a byte-by-byte copy 679// is required. 680template <typename _Tp, typename _Tv> _LIBCPP_INLINE_VISIBILITY 681typename enable_if<is_assignable<_Tp&, _Tv>::value>::type 682__cxx_atomic_assign_volatile(_Tp& __a_value, _Tv const& __val) { 683 __a_value = __val; 684} 685template <typename _Tp, typename _Tv> _LIBCPP_INLINE_VISIBILITY 686typename enable_if<is_assignable<_Tp&, _Tv>::value>::type 687__cxx_atomic_assign_volatile(_Tp volatile& __a_value, _Tv volatile const& __val) { 688 volatile char* __to = reinterpret_cast<volatile char*>(&__a_value); 689 volatile char* __end = __to + sizeof(_Tp); 690 volatile const char* __from = reinterpret_cast<volatile const char*>(&__val); 691 while (__to != __end) 692 *__to++ = *__from++; 693} 694 695#endif 696 697#if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) 698 699template <typename _Tp> 700struct __cxx_atomic_base_impl { 701 702 _LIBCPP_INLINE_VISIBILITY 703#ifndef _LIBCPP_CXX03_LANG 704 __cxx_atomic_base_impl() _NOEXCEPT = default; 705#else 706 __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {} 707#endif // _LIBCPP_CXX03_LANG 708 _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT 709 : __a_value(value) {} 710 _Tp __a_value; 711}; 712 713_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR int __to_gcc_order(memory_order __order) { 714 // Avoid switch statement to make this a constexpr. 715 return __order == memory_order_relaxed ? __ATOMIC_RELAXED: 716 (__order == memory_order_acquire ? __ATOMIC_ACQUIRE: 717 (__order == memory_order_release ? __ATOMIC_RELEASE: 718 (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST: 719 (__order == memory_order_acq_rel ? __ATOMIC_ACQ_REL: 720 __ATOMIC_CONSUME)))); 721} 722 723_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR int __to_gcc_failure_order(memory_order __order) { 724 // Avoid switch statement to make this a constexpr. 725 return __order == memory_order_relaxed ? __ATOMIC_RELAXED: 726 (__order == memory_order_acquire ? __ATOMIC_ACQUIRE: 727 (__order == memory_order_release ? __ATOMIC_RELAXED: 728 (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST: 729 (__order == memory_order_acq_rel ? __ATOMIC_ACQUIRE: 730 __ATOMIC_CONSUME)))); 731} 732 733template <typename _Tp> 734_LIBCPP_INLINE_VISIBILITY 735void __cxx_atomic_init(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val) { 736 __cxx_atomic_assign_volatile(__a->__a_value, __val); 737} 738 739template <typename _Tp> 740_LIBCPP_INLINE_VISIBILITY 741void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) { 742 __a->__a_value = __val; 743} 744 745_LIBCPP_INLINE_VISIBILITY inline 746void __cxx_atomic_thread_fence(memory_order __order) { 747 __atomic_thread_fence(__to_gcc_order(__order)); 748} 749 750_LIBCPP_INLINE_VISIBILITY inline 751void __cxx_atomic_signal_fence(memory_order __order) { 752 __atomic_signal_fence(__to_gcc_order(__order)); 753} 754 755template <typename _Tp> 756_LIBCPP_INLINE_VISIBILITY 757void __cxx_atomic_store(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val, 758 memory_order __order) { 759 __atomic_store(&__a->__a_value, &__val, 760 __to_gcc_order(__order)); 761} 762 763template <typename _Tp> 764_LIBCPP_INLINE_VISIBILITY 765void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, 766 memory_order __order) { 767 __atomic_store(&__a->__a_value, &__val, 768 __to_gcc_order(__order)); 769} 770 771template <typename _Tp> 772_LIBCPP_INLINE_VISIBILITY 773_Tp __cxx_atomic_load(const volatile __cxx_atomic_base_impl<_Tp>* __a, 774 memory_order __order) { 775 _Tp __ret; 776 __atomic_load(&__a->__a_value, &__ret, 777 __to_gcc_order(__order)); 778 return __ret; 779} 780 781template <typename _Tp> 782_LIBCPP_INLINE_VISIBILITY 783_Tp __cxx_atomic_load(const __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) { 784 _Tp __ret; 785 __atomic_load(&__a->__a_value, &__ret, 786 __to_gcc_order(__order)); 787 return __ret; 788} 789 790template <typename _Tp> 791_LIBCPP_INLINE_VISIBILITY 792_Tp __cxx_atomic_exchange(volatile __cxx_atomic_base_impl<_Tp>* __a, 793 _Tp __value, memory_order __order) { 794 _Tp __ret; 795 __atomic_exchange(&__a->__a_value, &__value, &__ret, 796 __to_gcc_order(__order)); 797 return __ret; 798} 799 800template <typename _Tp> 801_LIBCPP_INLINE_VISIBILITY 802_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, 803 memory_order __order) { 804 _Tp __ret; 805 __atomic_exchange(&__a->__a_value, &__value, &__ret, 806 __to_gcc_order(__order)); 807 return __ret; 808} 809 810template <typename _Tp> 811_LIBCPP_INLINE_VISIBILITY 812bool __cxx_atomic_compare_exchange_strong( 813 volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, 814 memory_order __success, memory_order __failure) { 815 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 816 false, 817 __to_gcc_order(__success), 818 __to_gcc_failure_order(__failure)); 819} 820 821template <typename _Tp> 822_LIBCPP_INLINE_VISIBILITY 823bool __cxx_atomic_compare_exchange_strong( 824 __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, 825 memory_order __failure) { 826 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 827 false, 828 __to_gcc_order(__success), 829 __to_gcc_failure_order(__failure)); 830} 831 832template <typename _Tp> 833_LIBCPP_INLINE_VISIBILITY 834bool __cxx_atomic_compare_exchange_weak( 835 volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, 836 memory_order __success, memory_order __failure) { 837 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 838 true, 839 __to_gcc_order(__success), 840 __to_gcc_failure_order(__failure)); 841} 842 843template <typename _Tp> 844_LIBCPP_INLINE_VISIBILITY 845bool __cxx_atomic_compare_exchange_weak( 846 __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, 847 memory_order __failure) { 848 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 849 true, 850 __to_gcc_order(__success), 851 __to_gcc_failure_order(__failure)); 852} 853 854template <typename _Tp> 855struct __skip_amt { enum {value = 1}; }; 856 857template <typename _Tp> 858struct __skip_amt<_Tp*> { enum {value = sizeof(_Tp)}; }; 859 860// FIXME: Haven't figured out what the spec says about using arrays with 861// atomic_fetch_add. Force a failure rather than creating bad behavior. 862template <typename _Tp> 863struct __skip_amt<_Tp[]> { }; 864template <typename _Tp, int n> 865struct __skip_amt<_Tp[n]> { }; 866 867template <typename _Tp, typename _Td> 868_LIBCPP_INLINE_VISIBILITY 869_Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_base_impl<_Tp>* __a, 870 _Td __delta, memory_order __order) { 871 return __atomic_fetch_add(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 872 __to_gcc_order(__order)); 873} 874 875template <typename _Tp, typename _Td> 876_LIBCPP_INLINE_VISIBILITY 877_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, 878 memory_order __order) { 879 return __atomic_fetch_add(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 880 __to_gcc_order(__order)); 881} 882 883template <typename _Tp, typename _Td> 884_LIBCPP_INLINE_VISIBILITY 885_Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_base_impl<_Tp>* __a, 886 _Td __delta, memory_order __order) { 887 return __atomic_fetch_sub(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 888 __to_gcc_order(__order)); 889} 890 891template <typename _Tp, typename _Td> 892_LIBCPP_INLINE_VISIBILITY 893_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, 894 memory_order __order) { 895 return __atomic_fetch_sub(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 896 __to_gcc_order(__order)); 897} 898 899template <typename _Tp> 900_LIBCPP_INLINE_VISIBILITY 901_Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_base_impl<_Tp>* __a, 902 _Tp __pattern, memory_order __order) { 903 return __atomic_fetch_and(&__a->__a_value, __pattern, 904 __to_gcc_order(__order)); 905} 906 907template <typename _Tp> 908_LIBCPP_INLINE_VISIBILITY 909_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, 910 _Tp __pattern, memory_order __order) { 911 return __atomic_fetch_and(&__a->__a_value, __pattern, 912 __to_gcc_order(__order)); 913} 914 915template <typename _Tp> 916_LIBCPP_INLINE_VISIBILITY 917_Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_base_impl<_Tp>* __a, 918 _Tp __pattern, memory_order __order) { 919 return __atomic_fetch_or(&__a->__a_value, __pattern, 920 __to_gcc_order(__order)); 921} 922 923template <typename _Tp> 924_LIBCPP_INLINE_VISIBILITY 925_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, 926 memory_order __order) { 927 return __atomic_fetch_or(&__a->__a_value, __pattern, 928 __to_gcc_order(__order)); 929} 930 931template <typename _Tp> 932_LIBCPP_INLINE_VISIBILITY 933_Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_base_impl<_Tp>* __a, 934 _Tp __pattern, memory_order __order) { 935 return __atomic_fetch_xor(&__a->__a_value, __pattern, 936 __to_gcc_order(__order)); 937} 938 939template <typename _Tp> 940_LIBCPP_INLINE_VISIBILITY 941_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, 942 memory_order __order) { 943 return __atomic_fetch_xor(&__a->__a_value, __pattern, 944 __to_gcc_order(__order)); 945} 946 947#define __cxx_atomic_is_lock_free(__s) __atomic_is_lock_free(__s, 0) 948 949#elif defined(_LIBCPP_HAS_C_ATOMIC_IMP) 950 951template <typename _Tp> 952struct __cxx_atomic_base_impl { 953 954 _LIBCPP_INLINE_VISIBILITY 955#ifndef _LIBCPP_CXX03_LANG 956 __cxx_atomic_base_impl() _NOEXCEPT = default; 957#else 958 __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {} 959#endif // _LIBCPP_CXX03_LANG 960 _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT 961 : __a_value(value) {} 962 _LIBCPP_DISABLE_EXTENSION_WARNING _Atomic(_Tp) __a_value; 963}; 964 965#define __cxx_atomic_is_lock_free(__s) __c11_atomic_is_lock_free(__s) 966 967_LIBCPP_INLINE_VISIBILITY inline 968void __cxx_atomic_thread_fence(memory_order __order) _NOEXCEPT { 969 __c11_atomic_thread_fence(static_cast<__memory_order_underlying_t>(__order)); 970} 971 972_LIBCPP_INLINE_VISIBILITY inline 973void __cxx_atomic_signal_fence(memory_order __order) _NOEXCEPT { 974 __c11_atomic_signal_fence(static_cast<__memory_order_underlying_t>(__order)); 975} 976 977template<class _Tp> 978_LIBCPP_INLINE_VISIBILITY 979void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val) _NOEXCEPT { 980 __c11_atomic_init(&__a->__a_value, __val); 981} 982template<class _Tp> 983_LIBCPP_INLINE_VISIBILITY 984void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val) _NOEXCEPT { 985 __c11_atomic_init(&__a->__a_value, __val); 986} 987 988template<class _Tp> 989_LIBCPP_INLINE_VISIBILITY 990void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT { 991 __c11_atomic_store(&__a->__a_value, __val, static_cast<__memory_order_underlying_t>(__order)); 992} 993template<class _Tp> 994_LIBCPP_INLINE_VISIBILITY 995void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val, memory_order __order) _NOEXCEPT { 996 __c11_atomic_store(&__a->__a_value, __val, static_cast<__memory_order_underlying_t>(__order)); 997} 998 999template<class _Tp> 1000_LIBCPP_INLINE_VISIBILITY 1001_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const volatile* __a, memory_order __order) _NOEXCEPT { 1002 using __ptr_type = typename remove_const<decltype(__a->__a_value)>::type*; 1003 return __c11_atomic_load(const_cast<__ptr_type>(&__a->__a_value), static_cast<__memory_order_underlying_t>(__order)); 1004} 1005template<class _Tp> 1006_LIBCPP_INLINE_VISIBILITY 1007_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const* __a, memory_order __order) _NOEXCEPT { 1008 using __ptr_type = typename remove_const<decltype(__a->__a_value)>::type*; 1009 return __c11_atomic_load(const_cast<__ptr_type>(&__a->__a_value), static_cast<__memory_order_underlying_t>(__order)); 1010} 1011 1012template<class _Tp> 1013_LIBCPP_INLINE_VISIBILITY 1014_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT { 1015 return __c11_atomic_exchange(&__a->__a_value, __value, static_cast<__memory_order_underlying_t>(__order)); 1016} 1017template<class _Tp> 1018_LIBCPP_INLINE_VISIBILITY 1019_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> * __a, _Tp __value, memory_order __order) _NOEXCEPT { 1020 return __c11_atomic_exchange(&__a->__a_value, __value, static_cast<__memory_order_underlying_t>(__order)); 1021} 1022 1023_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR memory_order __to_failure_order(memory_order __order) { 1024 // Avoid switch statement to make this a constexpr. 1025 return __order == memory_order_release ? memory_order_relaxed: 1026 (__order == memory_order_acq_rel ? memory_order_acquire: 1027 __order); 1028} 1029 1030template<class _Tp> 1031_LIBCPP_INLINE_VISIBILITY 1032bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 1033 return __c11_atomic_compare_exchange_strong(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); 1034} 1035template<class _Tp> 1036_LIBCPP_INLINE_VISIBILITY 1037bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 1038 return __c11_atomic_compare_exchange_strong(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); 1039} 1040 1041template<class _Tp> 1042_LIBCPP_INLINE_VISIBILITY 1043bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 1044 return __c11_atomic_compare_exchange_weak(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); 1045} 1046template<class _Tp> 1047_LIBCPP_INLINE_VISIBILITY 1048bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 1049 return __c11_atomic_compare_exchange_weak(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); 1050} 1051 1052template<class _Tp> 1053_LIBCPP_INLINE_VISIBILITY 1054_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1055 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1056} 1057template<class _Tp> 1058_LIBCPP_INLINE_VISIBILITY 1059_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1060 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1061} 1062 1063template<class _Tp> 1064_LIBCPP_INLINE_VISIBILITY 1065_Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1066 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1067} 1068template<class _Tp> 1069_LIBCPP_INLINE_VISIBILITY 1070_Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1071 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1072} 1073 1074template<class _Tp> 1075_LIBCPP_INLINE_VISIBILITY 1076_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1077 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1078} 1079template<class _Tp> 1080_LIBCPP_INLINE_VISIBILITY 1081_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1082 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1083} 1084template<class _Tp> 1085_LIBCPP_INLINE_VISIBILITY 1086_Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1087 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1088} 1089template<class _Tp> 1090_LIBCPP_INLINE_VISIBILITY 1091_Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1092 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1093} 1094 1095template<class _Tp> 1096_LIBCPP_INLINE_VISIBILITY 1097_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1098 return __c11_atomic_fetch_and(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1099} 1100template<class _Tp> 1101_LIBCPP_INLINE_VISIBILITY 1102_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1103 return __c11_atomic_fetch_and(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1104} 1105 1106template<class _Tp> 1107_LIBCPP_INLINE_VISIBILITY 1108_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1109 return __c11_atomic_fetch_or(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1110} 1111template<class _Tp> 1112_LIBCPP_INLINE_VISIBILITY 1113_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1114 return __c11_atomic_fetch_or(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1115} 1116 1117template<class _Tp> 1118_LIBCPP_INLINE_VISIBILITY 1119_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1120 return __c11_atomic_fetch_xor(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1121} 1122template<class _Tp> 1123_LIBCPP_INLINE_VISIBILITY 1124_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1125 return __c11_atomic_fetch_xor(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1126} 1127 1128#endif // _LIBCPP_HAS_GCC_ATOMIC_IMP, _LIBCPP_HAS_C_ATOMIC_IMP 1129 1130template <class _Tp> 1131_LIBCPP_INLINE_VISIBILITY 1132_Tp kill_dependency(_Tp __y) _NOEXCEPT 1133{ 1134 return __y; 1135} 1136 1137#if defined(__CLANG_ATOMIC_BOOL_LOCK_FREE) 1138# define ATOMIC_BOOL_LOCK_FREE __CLANG_ATOMIC_BOOL_LOCK_FREE 1139# define ATOMIC_CHAR_LOCK_FREE __CLANG_ATOMIC_CHAR_LOCK_FREE 1140#ifndef _LIBCPP_HAS_NO_CHAR8_T 1141# define ATOMIC_CHAR8_T_LOCK_FREE __CLANG_ATOMIC_CHAR8_T_LOCK_FREE 1142#endif 1143# define ATOMIC_CHAR16_T_LOCK_FREE __CLANG_ATOMIC_CHAR16_T_LOCK_FREE 1144# define ATOMIC_CHAR32_T_LOCK_FREE __CLANG_ATOMIC_CHAR32_T_LOCK_FREE 1145# define ATOMIC_WCHAR_T_LOCK_FREE __CLANG_ATOMIC_WCHAR_T_LOCK_FREE 1146# define ATOMIC_SHORT_LOCK_FREE __CLANG_ATOMIC_SHORT_LOCK_FREE 1147# define ATOMIC_INT_LOCK_FREE __CLANG_ATOMIC_INT_LOCK_FREE 1148# define ATOMIC_LONG_LOCK_FREE __CLANG_ATOMIC_LONG_LOCK_FREE 1149# define ATOMIC_LLONG_LOCK_FREE __CLANG_ATOMIC_LLONG_LOCK_FREE 1150# define ATOMIC_POINTER_LOCK_FREE __CLANG_ATOMIC_POINTER_LOCK_FREE 1151#elif defined(__GCC_ATOMIC_BOOL_LOCK_FREE) 1152# define ATOMIC_BOOL_LOCK_FREE __GCC_ATOMIC_BOOL_LOCK_FREE 1153# define ATOMIC_CHAR_LOCK_FREE __GCC_ATOMIC_CHAR_LOCK_FREE 1154#ifndef _LIBCPP_HAS_NO_CHAR8_T 1155# define ATOMIC_CHAR8_T_LOCK_FREE __GCC_ATOMIC_CHAR8_T_LOCK_FREE 1156#endif 1157# define ATOMIC_CHAR16_T_LOCK_FREE __GCC_ATOMIC_CHAR16_T_LOCK_FREE 1158# define ATOMIC_CHAR32_T_LOCK_FREE __GCC_ATOMIC_CHAR32_T_LOCK_FREE 1159# define ATOMIC_WCHAR_T_LOCK_FREE __GCC_ATOMIC_WCHAR_T_LOCK_FREE 1160# define ATOMIC_SHORT_LOCK_FREE __GCC_ATOMIC_SHORT_LOCK_FREE 1161# define ATOMIC_INT_LOCK_FREE __GCC_ATOMIC_INT_LOCK_FREE 1162# define ATOMIC_LONG_LOCK_FREE __GCC_ATOMIC_LONG_LOCK_FREE 1163# define ATOMIC_LLONG_LOCK_FREE __GCC_ATOMIC_LLONG_LOCK_FREE 1164# define ATOMIC_POINTER_LOCK_FREE __GCC_ATOMIC_POINTER_LOCK_FREE 1165#endif 1166 1167#ifdef _LIBCPP_ATOMIC_ONLY_USE_BUILTINS 1168 1169template<typename _Tp> 1170struct __cxx_atomic_lock_impl { 1171 1172 _LIBCPP_INLINE_VISIBILITY 1173 __cxx_atomic_lock_impl() _NOEXCEPT 1174 : __a_value(), __a_lock(0) {} 1175 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR explicit 1176 __cxx_atomic_lock_impl(_Tp value) _NOEXCEPT 1177 : __a_value(value), __a_lock(0) {} 1178 1179 _Tp __a_value; 1180 mutable __cxx_atomic_base_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_lock; 1181 1182 _LIBCPP_INLINE_VISIBILITY void __lock() const volatile { 1183 while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire)) 1184 /*spin*/; 1185 } 1186 _LIBCPP_INLINE_VISIBILITY void __lock() const { 1187 while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire)) 1188 /*spin*/; 1189 } 1190 _LIBCPP_INLINE_VISIBILITY void __unlock() const volatile { 1191 __cxx_atomic_store(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(false), memory_order_release); 1192 } 1193 _LIBCPP_INLINE_VISIBILITY void __unlock() const { 1194 __cxx_atomic_store(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(false), memory_order_release); 1195 } 1196 _LIBCPP_INLINE_VISIBILITY _Tp __read() const volatile { 1197 __lock(); 1198 _Tp __old; 1199 __cxx_atomic_assign_volatile(__old, __a_value); 1200 __unlock(); 1201 return __old; 1202 } 1203 _LIBCPP_INLINE_VISIBILITY _Tp __read() const { 1204 __lock(); 1205 _Tp __old = __a_value; 1206 __unlock(); 1207 return __old; 1208 } 1209}; 1210 1211template <typename _Tp> 1212_LIBCPP_INLINE_VISIBILITY 1213void __cxx_atomic_init(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __val) { 1214 __cxx_atomic_assign_volatile(__a->__a_value, __val); 1215} 1216template <typename _Tp> 1217_LIBCPP_INLINE_VISIBILITY 1218void __cxx_atomic_init(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __val) { 1219 __a->__a_value = __val; 1220} 1221 1222template <typename _Tp> 1223_LIBCPP_INLINE_VISIBILITY 1224void __cxx_atomic_store(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __val, memory_order) { 1225 __a->__lock(); 1226 __cxx_atomic_assign_volatile(__a->__a_value, __val); 1227 __a->__unlock(); 1228} 1229template <typename _Tp> 1230_LIBCPP_INLINE_VISIBILITY 1231void __cxx_atomic_store(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __val, memory_order) { 1232 __a->__lock(); 1233 __a->__a_value = __val; 1234 __a->__unlock(); 1235} 1236 1237template <typename _Tp> 1238_LIBCPP_INLINE_VISIBILITY 1239_Tp __cxx_atomic_load(const volatile __cxx_atomic_lock_impl<_Tp>* __a, memory_order) { 1240 return __a->__read(); 1241} 1242template <typename _Tp> 1243_LIBCPP_INLINE_VISIBILITY 1244_Tp __cxx_atomic_load(const __cxx_atomic_lock_impl<_Tp>* __a, memory_order) { 1245 return __a->__read(); 1246} 1247 1248template <typename _Tp> 1249_LIBCPP_INLINE_VISIBILITY 1250_Tp __cxx_atomic_exchange(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) { 1251 __a->__lock(); 1252 _Tp __old; 1253 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1254 __cxx_atomic_assign_volatile(__a->__a_value, __value); 1255 __a->__unlock(); 1256 return __old; 1257} 1258template <typename _Tp> 1259_LIBCPP_INLINE_VISIBILITY 1260_Tp __cxx_atomic_exchange(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) { 1261 __a->__lock(); 1262 _Tp __old = __a->__a_value; 1263 __a->__a_value = __value; 1264 __a->__unlock(); 1265 return __old; 1266} 1267 1268template <typename _Tp> 1269_LIBCPP_INLINE_VISIBILITY 1270bool __cxx_atomic_compare_exchange_strong(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1271 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1272 _Tp __temp; 1273 __a->__lock(); 1274 __cxx_atomic_assign_volatile(__temp, __a->__a_value); 1275 bool __ret = (_VSTD::memcmp(&__temp, __expected, sizeof(_Tp)) == 0); 1276 if(__ret) 1277 __cxx_atomic_assign_volatile(__a->__a_value, __value); 1278 else 1279 __cxx_atomic_assign_volatile(*__expected, __a->__a_value); 1280 __a->__unlock(); 1281 return __ret; 1282} 1283template <typename _Tp> 1284_LIBCPP_INLINE_VISIBILITY 1285bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_lock_impl<_Tp>* __a, 1286 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1287 __a->__lock(); 1288 bool __ret = (_VSTD::memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0); 1289 if(__ret) 1290 _VSTD::memcpy(&__a->__a_value, &__value, sizeof(_Tp)); 1291 else 1292 _VSTD::memcpy(__expected, &__a->__a_value, sizeof(_Tp)); 1293 __a->__unlock(); 1294 return __ret; 1295} 1296 1297template <typename _Tp> 1298_LIBCPP_INLINE_VISIBILITY 1299bool __cxx_atomic_compare_exchange_weak(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1300 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1301 _Tp __temp; 1302 __a->__lock(); 1303 __cxx_atomic_assign_volatile(__temp, __a->__a_value); 1304 bool __ret = (_VSTD::memcmp(&__temp, __expected, sizeof(_Tp)) == 0); 1305 if(__ret) 1306 __cxx_atomic_assign_volatile(__a->__a_value, __value); 1307 else 1308 __cxx_atomic_assign_volatile(*__expected, __a->__a_value); 1309 __a->__unlock(); 1310 return __ret; 1311} 1312template <typename _Tp> 1313_LIBCPP_INLINE_VISIBILITY 1314bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_lock_impl<_Tp>* __a, 1315 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1316 __a->__lock(); 1317 bool __ret = (_VSTD::memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0); 1318 if(__ret) 1319 _VSTD::memcpy(&__a->__a_value, &__value, sizeof(_Tp)); 1320 else 1321 _VSTD::memcpy(__expected, &__a->__a_value, sizeof(_Tp)); 1322 __a->__unlock(); 1323 return __ret; 1324} 1325 1326template <typename _Tp, typename _Td> 1327_LIBCPP_INLINE_VISIBILITY 1328_Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1329 _Td __delta, memory_order) { 1330 __a->__lock(); 1331 _Tp __old; 1332 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1333 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old + __delta)); 1334 __a->__unlock(); 1335 return __old; 1336} 1337template <typename _Tp, typename _Td> 1338_LIBCPP_INLINE_VISIBILITY 1339_Tp __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp>* __a, 1340 _Td __delta, memory_order) { 1341 __a->__lock(); 1342 _Tp __old = __a->__a_value; 1343 __a->__a_value += __delta; 1344 __a->__unlock(); 1345 return __old; 1346} 1347 1348template <typename _Tp, typename _Td> 1349_LIBCPP_INLINE_VISIBILITY 1350_Tp* __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp*>* __a, 1351 ptrdiff_t __delta, memory_order) { 1352 __a->__lock(); 1353 _Tp* __old; 1354 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1355 __cxx_atomic_assign_volatile(__a->__a_value, __old + __delta); 1356 __a->__unlock(); 1357 return __old; 1358} 1359template <typename _Tp, typename _Td> 1360_LIBCPP_INLINE_VISIBILITY 1361_Tp* __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp*>* __a, 1362 ptrdiff_t __delta, memory_order) { 1363 __a->__lock(); 1364 _Tp* __old = __a->__a_value; 1365 __a->__a_value += __delta; 1366 __a->__unlock(); 1367 return __old; 1368} 1369 1370template <typename _Tp, typename _Td> 1371_LIBCPP_INLINE_VISIBILITY 1372_Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1373 _Td __delta, memory_order) { 1374 __a->__lock(); 1375 _Tp __old; 1376 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1377 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old - __delta)); 1378 __a->__unlock(); 1379 return __old; 1380} 1381template <typename _Tp, typename _Td> 1382_LIBCPP_INLINE_VISIBILITY 1383_Tp __cxx_atomic_fetch_sub(__cxx_atomic_lock_impl<_Tp>* __a, 1384 _Td __delta, memory_order) { 1385 __a->__lock(); 1386 _Tp __old = __a->__a_value; 1387 __a->__a_value -= __delta; 1388 __a->__unlock(); 1389 return __old; 1390} 1391 1392template <typename _Tp> 1393_LIBCPP_INLINE_VISIBILITY 1394_Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1395 _Tp __pattern, memory_order) { 1396 __a->__lock(); 1397 _Tp __old; 1398 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1399 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old & __pattern)); 1400 __a->__unlock(); 1401 return __old; 1402} 1403template <typename _Tp> 1404_LIBCPP_INLINE_VISIBILITY 1405_Tp __cxx_atomic_fetch_and(__cxx_atomic_lock_impl<_Tp>* __a, 1406 _Tp __pattern, memory_order) { 1407 __a->__lock(); 1408 _Tp __old = __a->__a_value; 1409 __a->__a_value &= __pattern; 1410 __a->__unlock(); 1411 return __old; 1412} 1413 1414template <typename _Tp> 1415_LIBCPP_INLINE_VISIBILITY 1416_Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1417 _Tp __pattern, memory_order) { 1418 __a->__lock(); 1419 _Tp __old; 1420 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1421 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old | __pattern)); 1422 __a->__unlock(); 1423 return __old; 1424} 1425template <typename _Tp> 1426_LIBCPP_INLINE_VISIBILITY 1427_Tp __cxx_atomic_fetch_or(__cxx_atomic_lock_impl<_Tp>* __a, 1428 _Tp __pattern, memory_order) { 1429 __a->__lock(); 1430 _Tp __old = __a->__a_value; 1431 __a->__a_value |= __pattern; 1432 __a->__unlock(); 1433 return __old; 1434} 1435 1436template <typename _Tp> 1437_LIBCPP_INLINE_VISIBILITY 1438_Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1439 _Tp __pattern, memory_order) { 1440 __a->__lock(); 1441 _Tp __old; 1442 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1443 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old ^ __pattern)); 1444 __a->__unlock(); 1445 return __old; 1446} 1447template <typename _Tp> 1448_LIBCPP_INLINE_VISIBILITY 1449_Tp __cxx_atomic_fetch_xor(__cxx_atomic_lock_impl<_Tp>* __a, 1450 _Tp __pattern, memory_order) { 1451 __a->__lock(); 1452 _Tp __old = __a->__a_value; 1453 __a->__a_value ^= __pattern; 1454 __a->__unlock(); 1455 return __old; 1456} 1457 1458#ifdef __cpp_lib_atomic_is_always_lock_free 1459 1460template<typename _Tp> struct __cxx_is_always_lock_free { 1461 enum { __value = __atomic_always_lock_free(sizeof(_Tp), 0) }; }; 1462 1463#else 1464 1465template<typename _Tp> struct __cxx_is_always_lock_free { enum { __value = false }; }; 1466// Implementations must match the C ATOMIC_*_LOCK_FREE macro values. 1467template<> struct __cxx_is_always_lock_free<bool> { enum { __value = 2 == ATOMIC_BOOL_LOCK_FREE }; }; 1468template<> struct __cxx_is_always_lock_free<char> { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; }; 1469template<> struct __cxx_is_always_lock_free<signed char> { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; }; 1470template<> struct __cxx_is_always_lock_free<unsigned char> { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; }; 1471#ifndef _LIBCPP_HAS_NO_CHAR8_T 1472template<> struct __cxx_is_always_lock_free<char8_t> { enum { __value = 2 == ATOMIC_CHAR8_T_LOCK_FREE }; }; 1473#endif 1474template<> struct __cxx_is_always_lock_free<char16_t> { enum { __value = 2 == ATOMIC_CHAR16_T_LOCK_FREE }; }; 1475template<> struct __cxx_is_always_lock_free<char32_t> { enum { __value = 2 == ATOMIC_CHAR32_T_LOCK_FREE }; }; 1476#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS 1477template<> struct __cxx_is_always_lock_free<wchar_t> { enum { __value = 2 == ATOMIC_WCHAR_T_LOCK_FREE }; }; 1478#endif 1479template<> struct __cxx_is_always_lock_free<short> { enum { __value = 2 == ATOMIC_SHORT_LOCK_FREE }; }; 1480template<> struct __cxx_is_always_lock_free<unsigned short> { enum { __value = 2 == ATOMIC_SHORT_LOCK_FREE }; }; 1481template<> struct __cxx_is_always_lock_free<int> { enum { __value = 2 == ATOMIC_INT_LOCK_FREE }; }; 1482template<> struct __cxx_is_always_lock_free<unsigned int> { enum { __value = 2 == ATOMIC_INT_LOCK_FREE }; }; 1483template<> struct __cxx_is_always_lock_free<long> { enum { __value = 2 == ATOMIC_LONG_LOCK_FREE }; }; 1484template<> struct __cxx_is_always_lock_free<unsigned long> { enum { __value = 2 == ATOMIC_LONG_LOCK_FREE }; }; 1485template<> struct __cxx_is_always_lock_free<long long> { enum { __value = 2 == ATOMIC_LLONG_LOCK_FREE }; }; 1486template<> struct __cxx_is_always_lock_free<unsigned long long> { enum { __value = 2 == ATOMIC_LLONG_LOCK_FREE }; }; 1487template<typename _Tp> struct __cxx_is_always_lock_free<_Tp*> { enum { __value = 2 == ATOMIC_POINTER_LOCK_FREE }; }; 1488template<> struct __cxx_is_always_lock_free<std::nullptr_t> { enum { __value = 2 == ATOMIC_POINTER_LOCK_FREE }; }; 1489 1490#endif //__cpp_lib_atomic_is_always_lock_free 1491 1492template <typename _Tp, 1493 typename _Base = typename conditional<__cxx_is_always_lock_free<_Tp>::__value, 1494 __cxx_atomic_base_impl<_Tp>, 1495 __cxx_atomic_lock_impl<_Tp> >::type> 1496#else 1497template <typename _Tp, 1498 typename _Base = __cxx_atomic_base_impl<_Tp> > 1499#endif //_LIBCPP_ATOMIC_ONLY_USE_BUILTINS 1500struct __cxx_atomic_impl : public _Base { 1501 static_assert(is_trivially_copyable<_Tp>::value, 1502 "std::atomic<Tp> requires that 'Tp' be a trivially copyable type"); 1503 1504 _LIBCPP_INLINE_VISIBILITY __cxx_atomic_impl() _NOEXCEPT _LIBCPP_DEFAULT 1505 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR explicit __cxx_atomic_impl(_Tp value) _NOEXCEPT 1506 : _Base(value) {} 1507}; 1508 1509#ifdef __linux__ 1510 using __cxx_contention_t = int32_t; 1511#else 1512 using __cxx_contention_t = int64_t; 1513#endif //__linux__ 1514 1515using __cxx_atomic_contention_t = __cxx_atomic_impl<__cxx_contention_t>; 1516 1517#ifndef _LIBCPP_HAS_NO_PLATFORM_WAIT 1518 1519_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(void const volatile*); 1520_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(void const volatile*); 1521_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(void const volatile*); 1522_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(void const volatile*, __cxx_contention_t); 1523 1524_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(__cxx_atomic_contention_t const volatile*); 1525_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(__cxx_atomic_contention_t const volatile*); 1526_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile*); 1527_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(__cxx_atomic_contention_t const volatile*, __cxx_contention_t); 1528 1529template <class _Atp, class _Fn> 1530struct __libcpp_atomic_wait_backoff_impl { 1531 _Atp* __a; 1532 _Fn __test_fn; 1533 _LIBCPP_AVAILABILITY_SYNC 1534 _LIBCPP_INLINE_VISIBILITY bool operator()(chrono::nanoseconds __elapsed) const 1535 { 1536 if(__elapsed > chrono::microseconds(64)) 1537 { 1538 auto const __monitor = __libcpp_atomic_monitor(__a); 1539 if(__test_fn()) 1540 return true; 1541 __libcpp_atomic_wait(__a, __monitor); 1542 } 1543 else if(__elapsed > chrono::microseconds(4)) 1544 __libcpp_thread_yield(); 1545 else 1546 {} // poll 1547 return false; 1548 } 1549}; 1550 1551template <class _Atp, class _Fn> 1552_LIBCPP_AVAILABILITY_SYNC 1553_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp* __a, _Fn && __test_fn) 1554{ 1555 __libcpp_atomic_wait_backoff_impl<_Atp, typename decay<_Fn>::type> __backoff_fn = {__a, __test_fn}; 1556 return __libcpp_thread_poll_with_backoff(__test_fn, __backoff_fn); 1557} 1558 1559#else // _LIBCPP_HAS_NO_PLATFORM_WAIT 1560 1561template <class _Tp> 1562_LIBCPP_INLINE_VISIBILITY void __cxx_atomic_notify_all(__cxx_atomic_impl<_Tp> const volatile*) { } 1563template <class _Tp> 1564_LIBCPP_INLINE_VISIBILITY void __cxx_atomic_notify_one(__cxx_atomic_impl<_Tp> const volatile*) { } 1565template <class _Atp, class _Fn> 1566_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp*, _Fn && __test_fn) 1567{ 1568 return __libcpp_thread_poll_with_backoff(__test_fn, __libcpp_timed_backoff_policy()); 1569} 1570 1571#endif // _LIBCPP_HAS_NO_PLATFORM_WAIT 1572 1573template <class _Atp, class _Tp> 1574struct __cxx_atomic_wait_test_fn_impl { 1575 _Atp* __a; 1576 _Tp __val; 1577 memory_order __order; 1578 _LIBCPP_INLINE_VISIBILITY bool operator()() const 1579 { 1580 return !__cxx_nonatomic_compare_equal(__cxx_atomic_load(__a, __order), __val); 1581 } 1582}; 1583 1584template <class _Atp, class _Tp> 1585_LIBCPP_AVAILABILITY_SYNC 1586_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp* __a, _Tp const __val, memory_order __order) 1587{ 1588 __cxx_atomic_wait_test_fn_impl<_Atp, _Tp> __test_fn = {__a, __val, __order}; 1589 return __cxx_atomic_wait(__a, __test_fn); 1590} 1591 1592// general atomic<T> 1593 1594template <class _Tp, bool = is_integral<_Tp>::value && !is_same<_Tp, bool>::value> 1595struct __atomic_base // false 1596{ 1597 mutable __cxx_atomic_impl<_Tp> __a_; 1598 1599#if defined(__cpp_lib_atomic_is_always_lock_free) 1600 static _LIBCPP_CONSTEXPR bool is_always_lock_free = __atomic_always_lock_free(sizeof(__a_), 0); 1601#endif 1602 1603 _LIBCPP_INLINE_VISIBILITY 1604 bool is_lock_free() const volatile _NOEXCEPT 1605 {return __cxx_atomic_is_lock_free(sizeof(_Tp));} 1606 _LIBCPP_INLINE_VISIBILITY 1607 bool is_lock_free() const _NOEXCEPT 1608 {return static_cast<__atomic_base const volatile*>(this)->is_lock_free();} 1609 _LIBCPP_INLINE_VISIBILITY 1610 void store(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1611 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1612 {__cxx_atomic_store(&__a_, __d, __m);} 1613 _LIBCPP_INLINE_VISIBILITY 1614 void store(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1615 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1616 {__cxx_atomic_store(&__a_, __d, __m);} 1617 _LIBCPP_INLINE_VISIBILITY 1618 _Tp load(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 1619 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 1620 {return __cxx_atomic_load(&__a_, __m);} 1621 _LIBCPP_INLINE_VISIBILITY 1622 _Tp load(memory_order __m = memory_order_seq_cst) const _NOEXCEPT 1623 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 1624 {return __cxx_atomic_load(&__a_, __m);} 1625 _LIBCPP_INLINE_VISIBILITY 1626 operator _Tp() const volatile _NOEXCEPT {return load();} 1627 _LIBCPP_INLINE_VISIBILITY 1628 operator _Tp() const _NOEXCEPT {return load();} 1629 _LIBCPP_INLINE_VISIBILITY 1630 _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1631 {return __cxx_atomic_exchange(&__a_, __d, __m);} 1632 _LIBCPP_INLINE_VISIBILITY 1633 _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1634 {return __cxx_atomic_exchange(&__a_, __d, __m);} 1635 _LIBCPP_INLINE_VISIBILITY 1636 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1637 memory_order __s, memory_order __f) volatile _NOEXCEPT 1638 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1639 {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);} 1640 _LIBCPP_INLINE_VISIBILITY 1641 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1642 memory_order __s, memory_order __f) _NOEXCEPT 1643 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1644 {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);} 1645 _LIBCPP_INLINE_VISIBILITY 1646 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1647 memory_order __s, memory_order __f) volatile _NOEXCEPT 1648 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1649 {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);} 1650 _LIBCPP_INLINE_VISIBILITY 1651 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1652 memory_order __s, memory_order __f) _NOEXCEPT 1653 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1654 {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);} 1655 _LIBCPP_INLINE_VISIBILITY 1656 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1657 memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1658 {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);} 1659 _LIBCPP_INLINE_VISIBILITY 1660 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1661 memory_order __m = memory_order_seq_cst) _NOEXCEPT 1662 {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);} 1663 _LIBCPP_INLINE_VISIBILITY 1664 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1665 memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1666 {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);} 1667 _LIBCPP_INLINE_VISIBILITY 1668 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1669 memory_order __m = memory_order_seq_cst) _NOEXCEPT 1670 {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);} 1671 1672 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 1673 {__cxx_atomic_wait(&__a_, __v, __m);} 1674 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT 1675 {__cxx_atomic_wait(&__a_, __v, __m);} 1676 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_one() volatile _NOEXCEPT 1677 {__cxx_atomic_notify_one(&__a_);} 1678 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_one() _NOEXCEPT 1679 {__cxx_atomic_notify_one(&__a_);} 1680 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_all() volatile _NOEXCEPT 1681 {__cxx_atomic_notify_all(&__a_);} 1682 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_all() _NOEXCEPT 1683 {__cxx_atomic_notify_all(&__a_);} 1684 1685#if _LIBCPP_STD_VER > 17 1686 _LIBCPP_INLINE_VISIBILITY constexpr 1687 __atomic_base() noexcept(is_nothrow_default_constructible_v<_Tp>) : __a_(_Tp()) {} 1688#else 1689 _LIBCPP_INLINE_VISIBILITY 1690 __atomic_base() _NOEXCEPT _LIBCPP_DEFAULT 1691#endif 1692 1693 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR 1694 __atomic_base(_Tp __d) _NOEXCEPT : __a_(__d) {} 1695 1696#ifndef _LIBCPP_CXX03_LANG 1697 __atomic_base(const __atomic_base&) = delete; 1698#else 1699private: 1700 _LIBCPP_INLINE_VISIBILITY 1701 __atomic_base(const __atomic_base&); 1702#endif 1703}; 1704 1705#if defined(__cpp_lib_atomic_is_always_lock_free) 1706template <class _Tp, bool __b> 1707_LIBCPP_CONSTEXPR bool __atomic_base<_Tp, __b>::is_always_lock_free; 1708#endif 1709 1710// atomic<Integral> 1711 1712template <class _Tp> 1713struct __atomic_base<_Tp, true> 1714 : public __atomic_base<_Tp, false> 1715{ 1716 typedef __atomic_base<_Tp, false> __base; 1717 1718 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 1719 __atomic_base() _NOEXCEPT _LIBCPP_DEFAULT 1720 1721 _LIBCPP_INLINE_VISIBILITY 1722 _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __base(__d) {} 1723 1724 _LIBCPP_INLINE_VISIBILITY 1725 _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1726 {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);} 1727 _LIBCPP_INLINE_VISIBILITY 1728 _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1729 {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);} 1730 _LIBCPP_INLINE_VISIBILITY 1731 _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1732 {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);} 1733 _LIBCPP_INLINE_VISIBILITY 1734 _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1735 {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);} 1736 _LIBCPP_INLINE_VISIBILITY 1737 _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1738 {return __cxx_atomic_fetch_and(&this->__a_, __op, __m);} 1739 _LIBCPP_INLINE_VISIBILITY 1740 _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1741 {return __cxx_atomic_fetch_and(&this->__a_, __op, __m);} 1742 _LIBCPP_INLINE_VISIBILITY 1743 _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1744 {return __cxx_atomic_fetch_or(&this->__a_, __op, __m);} 1745 _LIBCPP_INLINE_VISIBILITY 1746 _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1747 {return __cxx_atomic_fetch_or(&this->__a_, __op, __m);} 1748 _LIBCPP_INLINE_VISIBILITY 1749 _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1750 {return __cxx_atomic_fetch_xor(&this->__a_, __op, __m);} 1751 _LIBCPP_INLINE_VISIBILITY 1752 _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1753 {return __cxx_atomic_fetch_xor(&this->__a_, __op, __m);} 1754 1755 _LIBCPP_INLINE_VISIBILITY 1756 _Tp operator++(int) volatile _NOEXCEPT {return fetch_add(_Tp(1));} 1757 _LIBCPP_INLINE_VISIBILITY 1758 _Tp operator++(int) _NOEXCEPT {return fetch_add(_Tp(1));} 1759 _LIBCPP_INLINE_VISIBILITY 1760 _Tp operator--(int) volatile _NOEXCEPT {return fetch_sub(_Tp(1));} 1761 _LIBCPP_INLINE_VISIBILITY 1762 _Tp operator--(int) _NOEXCEPT {return fetch_sub(_Tp(1));} 1763 _LIBCPP_INLINE_VISIBILITY 1764 _Tp operator++() volatile _NOEXCEPT {return fetch_add(_Tp(1)) + _Tp(1);} 1765 _LIBCPP_INLINE_VISIBILITY 1766 _Tp operator++() _NOEXCEPT {return fetch_add(_Tp(1)) + _Tp(1);} 1767 _LIBCPP_INLINE_VISIBILITY 1768 _Tp operator--() volatile _NOEXCEPT {return fetch_sub(_Tp(1)) - _Tp(1);} 1769 _LIBCPP_INLINE_VISIBILITY 1770 _Tp operator--() _NOEXCEPT {return fetch_sub(_Tp(1)) - _Tp(1);} 1771 _LIBCPP_INLINE_VISIBILITY 1772 _Tp operator+=(_Tp __op) volatile _NOEXCEPT {return fetch_add(__op) + __op;} 1773 _LIBCPP_INLINE_VISIBILITY 1774 _Tp operator+=(_Tp __op) _NOEXCEPT {return fetch_add(__op) + __op;} 1775 _LIBCPP_INLINE_VISIBILITY 1776 _Tp operator-=(_Tp __op) volatile _NOEXCEPT {return fetch_sub(__op) - __op;} 1777 _LIBCPP_INLINE_VISIBILITY 1778 _Tp operator-=(_Tp __op) _NOEXCEPT {return fetch_sub(__op) - __op;} 1779 _LIBCPP_INLINE_VISIBILITY 1780 _Tp operator&=(_Tp __op) volatile _NOEXCEPT {return fetch_and(__op) & __op;} 1781 _LIBCPP_INLINE_VISIBILITY 1782 _Tp operator&=(_Tp __op) _NOEXCEPT {return fetch_and(__op) & __op;} 1783 _LIBCPP_INLINE_VISIBILITY 1784 _Tp operator|=(_Tp __op) volatile _NOEXCEPT {return fetch_or(__op) | __op;} 1785 _LIBCPP_INLINE_VISIBILITY 1786 _Tp operator|=(_Tp __op) _NOEXCEPT {return fetch_or(__op) | __op;} 1787 _LIBCPP_INLINE_VISIBILITY 1788 _Tp operator^=(_Tp __op) volatile _NOEXCEPT {return fetch_xor(__op) ^ __op;} 1789 _LIBCPP_INLINE_VISIBILITY 1790 _Tp operator^=(_Tp __op) _NOEXCEPT {return fetch_xor(__op) ^ __op;} 1791}; 1792 1793// atomic<T> 1794 1795template <class _Tp> 1796struct atomic 1797 : public __atomic_base<_Tp> 1798{ 1799 typedef __atomic_base<_Tp> __base; 1800 typedef _Tp value_type; 1801 typedef value_type difference_type; 1802 1803#if _LIBCPP_STD_VER > 17 1804 _LIBCPP_INLINE_VISIBILITY 1805 atomic() = default; 1806#else 1807 _LIBCPP_INLINE_VISIBILITY 1808 atomic() _NOEXCEPT _LIBCPP_DEFAULT 1809#endif 1810 1811 _LIBCPP_INLINE_VISIBILITY 1812 _LIBCPP_CONSTEXPR atomic(_Tp __d) _NOEXCEPT : __base(__d) {} 1813 1814 _LIBCPP_INLINE_VISIBILITY 1815 _Tp operator=(_Tp __d) volatile _NOEXCEPT 1816 {__base::store(__d); return __d;} 1817 _LIBCPP_INLINE_VISIBILITY 1818 _Tp operator=(_Tp __d) _NOEXCEPT 1819 {__base::store(__d); return __d;} 1820 1821 atomic& operator=(const atomic&) = delete; 1822 atomic& operator=(const atomic&) volatile = delete; 1823}; 1824 1825// atomic<T*> 1826 1827template <class _Tp> 1828struct atomic<_Tp*> 1829 : public __atomic_base<_Tp*> 1830{ 1831 typedef __atomic_base<_Tp*> __base; 1832 typedef _Tp* value_type; 1833 typedef ptrdiff_t difference_type; 1834 1835 _LIBCPP_INLINE_VISIBILITY 1836 atomic() _NOEXCEPT _LIBCPP_DEFAULT 1837 1838 _LIBCPP_INLINE_VISIBILITY 1839 _LIBCPP_CONSTEXPR atomic(_Tp* __d) _NOEXCEPT : __base(__d) {} 1840 1841 _LIBCPP_INLINE_VISIBILITY 1842 _Tp* operator=(_Tp* __d) volatile _NOEXCEPT 1843 {__base::store(__d); return __d;} 1844 _LIBCPP_INLINE_VISIBILITY 1845 _Tp* operator=(_Tp* __d) _NOEXCEPT 1846 {__base::store(__d); return __d;} 1847 1848 _LIBCPP_INLINE_VISIBILITY 1849 _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { 1850 // __atomic_fetch_add accepts function pointers, guard against them. 1851 static_assert(!is_function<typename remove_pointer<_Tp>::type>::value, "Pointer to function isn't allowed"); 1852 return __cxx_atomic_fetch_add(&this->__a_, __op, __m); 1853 } 1854 1855 _LIBCPP_INLINE_VISIBILITY 1856 _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { 1857 // __atomic_fetch_add accepts function pointers, guard against them. 1858 static_assert(!is_function<typename remove_pointer<_Tp>::type>::value, "Pointer to function isn't allowed"); 1859 return __cxx_atomic_fetch_add(&this->__a_, __op, __m); 1860 } 1861 1862 _LIBCPP_INLINE_VISIBILITY 1863 _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { 1864 // __atomic_fetch_add accepts function pointers, guard against them. 1865 static_assert(!is_function<typename remove_pointer<_Tp>::type>::value, "Pointer to function isn't allowed"); 1866 return __cxx_atomic_fetch_sub(&this->__a_, __op, __m); 1867 } 1868 1869 _LIBCPP_INLINE_VISIBILITY 1870 _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { 1871 // __atomic_fetch_add accepts function pointers, guard against them. 1872 static_assert(!is_function<typename remove_pointer<_Tp>::type>::value, "Pointer to function isn't allowed"); 1873 return __cxx_atomic_fetch_sub(&this->__a_, __op, __m); 1874 } 1875 1876 _LIBCPP_INLINE_VISIBILITY 1877 _Tp* operator++(int) volatile _NOEXCEPT {return fetch_add(1);} 1878 _LIBCPP_INLINE_VISIBILITY 1879 _Tp* operator++(int) _NOEXCEPT {return fetch_add(1);} 1880 _LIBCPP_INLINE_VISIBILITY 1881 _Tp* operator--(int) volatile _NOEXCEPT {return fetch_sub(1);} 1882 _LIBCPP_INLINE_VISIBILITY 1883 _Tp* operator--(int) _NOEXCEPT {return fetch_sub(1);} 1884 _LIBCPP_INLINE_VISIBILITY 1885 _Tp* operator++() volatile _NOEXCEPT {return fetch_add(1) + 1;} 1886 _LIBCPP_INLINE_VISIBILITY 1887 _Tp* operator++() _NOEXCEPT {return fetch_add(1) + 1;} 1888 _LIBCPP_INLINE_VISIBILITY 1889 _Tp* operator--() volatile _NOEXCEPT {return fetch_sub(1) - 1;} 1890 _LIBCPP_INLINE_VISIBILITY 1891 _Tp* operator--() _NOEXCEPT {return fetch_sub(1) - 1;} 1892 _LIBCPP_INLINE_VISIBILITY 1893 _Tp* operator+=(ptrdiff_t __op) volatile _NOEXCEPT {return fetch_add(__op) + __op;} 1894 _LIBCPP_INLINE_VISIBILITY 1895 _Tp* operator+=(ptrdiff_t __op) _NOEXCEPT {return fetch_add(__op) + __op;} 1896 _LIBCPP_INLINE_VISIBILITY 1897 _Tp* operator-=(ptrdiff_t __op) volatile _NOEXCEPT {return fetch_sub(__op) - __op;} 1898 _LIBCPP_INLINE_VISIBILITY 1899 _Tp* operator-=(ptrdiff_t __op) _NOEXCEPT {return fetch_sub(__op) - __op;} 1900 1901 atomic& operator=(const atomic&) = delete; 1902 atomic& operator=(const atomic&) volatile = delete; 1903}; 1904 1905// atomic_is_lock_free 1906 1907template <class _Tp> 1908_LIBCPP_INLINE_VISIBILITY 1909bool 1910atomic_is_lock_free(const volatile atomic<_Tp>* __o) _NOEXCEPT 1911{ 1912 return __o->is_lock_free(); 1913} 1914 1915template <class _Tp> 1916_LIBCPP_INLINE_VISIBILITY 1917bool 1918atomic_is_lock_free(const atomic<_Tp>* __o) _NOEXCEPT 1919{ 1920 return __o->is_lock_free(); 1921} 1922 1923// atomic_init 1924 1925template <class _Tp> 1926_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_INLINE_VISIBILITY 1927void 1928atomic_init(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1929{ 1930 __cxx_atomic_init(&__o->__a_, __d); 1931} 1932 1933template <class _Tp> 1934_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_INLINE_VISIBILITY 1935void 1936atomic_init(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1937{ 1938 __cxx_atomic_init(&__o->__a_, __d); 1939} 1940 1941// atomic_store 1942 1943template <class _Tp> 1944_LIBCPP_INLINE_VISIBILITY 1945void 1946atomic_store(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1947{ 1948 __o->store(__d); 1949} 1950 1951template <class _Tp> 1952_LIBCPP_INLINE_VISIBILITY 1953void 1954atomic_store(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1955{ 1956 __o->store(__d); 1957} 1958 1959// atomic_store_explicit 1960 1961template <class _Tp> 1962_LIBCPP_INLINE_VISIBILITY 1963void 1964atomic_store_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 1965 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1966{ 1967 __o->store(__d, __m); 1968} 1969 1970template <class _Tp> 1971_LIBCPP_INLINE_VISIBILITY 1972void 1973atomic_store_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 1974 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1975{ 1976 __o->store(__d, __m); 1977} 1978 1979// atomic_load 1980 1981template <class _Tp> 1982_LIBCPP_INLINE_VISIBILITY 1983_Tp 1984atomic_load(const volatile atomic<_Tp>* __o) _NOEXCEPT 1985{ 1986 return __o->load(); 1987} 1988 1989template <class _Tp> 1990_LIBCPP_INLINE_VISIBILITY 1991_Tp 1992atomic_load(const atomic<_Tp>* __o) _NOEXCEPT 1993{ 1994 return __o->load(); 1995} 1996 1997// atomic_load_explicit 1998 1999template <class _Tp> 2000_LIBCPP_INLINE_VISIBILITY 2001_Tp 2002atomic_load_explicit(const volatile atomic<_Tp>* __o, memory_order __m) _NOEXCEPT 2003 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 2004{ 2005 return __o->load(__m); 2006} 2007 2008template <class _Tp> 2009_LIBCPP_INLINE_VISIBILITY 2010_Tp 2011atomic_load_explicit(const atomic<_Tp>* __o, memory_order __m) _NOEXCEPT 2012 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 2013{ 2014 return __o->load(__m); 2015} 2016 2017// atomic_exchange 2018 2019template <class _Tp> 2020_LIBCPP_INLINE_VISIBILITY 2021_Tp 2022atomic_exchange(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2023{ 2024 return __o->exchange(__d); 2025} 2026 2027template <class _Tp> 2028_LIBCPP_INLINE_VISIBILITY 2029_Tp 2030atomic_exchange(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2031{ 2032 return __o->exchange(__d); 2033} 2034 2035// atomic_exchange_explicit 2036 2037template <class _Tp> 2038_LIBCPP_INLINE_VISIBILITY 2039_Tp 2040atomic_exchange_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 2041{ 2042 return __o->exchange(__d, __m); 2043} 2044 2045template <class _Tp> 2046_LIBCPP_INLINE_VISIBILITY 2047_Tp 2048atomic_exchange_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 2049{ 2050 return __o->exchange(__d, __m); 2051} 2052 2053// atomic_compare_exchange_weak 2054 2055template <class _Tp> 2056_LIBCPP_INLINE_VISIBILITY 2057bool 2058atomic_compare_exchange_weak(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2059{ 2060 return __o->compare_exchange_weak(*__e, __d); 2061} 2062 2063template <class _Tp> 2064_LIBCPP_INLINE_VISIBILITY 2065bool 2066atomic_compare_exchange_weak(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2067{ 2068 return __o->compare_exchange_weak(*__e, __d); 2069} 2070 2071// atomic_compare_exchange_strong 2072 2073template <class _Tp> 2074_LIBCPP_INLINE_VISIBILITY 2075bool 2076atomic_compare_exchange_strong(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2077{ 2078 return __o->compare_exchange_strong(*__e, __d); 2079} 2080 2081template <class _Tp> 2082_LIBCPP_INLINE_VISIBILITY 2083bool 2084atomic_compare_exchange_strong(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2085{ 2086 return __o->compare_exchange_strong(*__e, __d); 2087} 2088 2089// atomic_compare_exchange_weak_explicit 2090 2091template <class _Tp> 2092_LIBCPP_INLINE_VISIBILITY 2093bool 2094atomic_compare_exchange_weak_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, 2095 typename atomic<_Tp>::value_type __d, 2096 memory_order __s, memory_order __f) _NOEXCEPT 2097 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2098{ 2099 return __o->compare_exchange_weak(*__e, __d, __s, __f); 2100} 2101 2102template <class _Tp> 2103_LIBCPP_INLINE_VISIBILITY 2104bool 2105atomic_compare_exchange_weak_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d, 2106 memory_order __s, memory_order __f) _NOEXCEPT 2107 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2108{ 2109 return __o->compare_exchange_weak(*__e, __d, __s, __f); 2110} 2111 2112// atomic_compare_exchange_strong_explicit 2113 2114template <class _Tp> 2115_LIBCPP_INLINE_VISIBILITY 2116bool 2117atomic_compare_exchange_strong_explicit(volatile atomic<_Tp>* __o, 2118 typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d, 2119 memory_order __s, memory_order __f) _NOEXCEPT 2120 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2121{ 2122 return __o->compare_exchange_strong(*__e, __d, __s, __f); 2123} 2124 2125template <class _Tp> 2126_LIBCPP_INLINE_VISIBILITY 2127bool 2128atomic_compare_exchange_strong_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, 2129 typename atomic<_Tp>::value_type __d, 2130 memory_order __s, memory_order __f) _NOEXCEPT 2131 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2132{ 2133 return __o->compare_exchange_strong(*__e, __d, __s, __f); 2134} 2135 2136// atomic_wait 2137 2138template <class _Tp> 2139_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2140void atomic_wait(const volatile atomic<_Tp>* __o, 2141 typename atomic<_Tp>::value_type __v) _NOEXCEPT 2142{ 2143 return __o->wait(__v); 2144} 2145 2146template <class _Tp> 2147_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2148void atomic_wait(const atomic<_Tp>* __o, 2149 typename atomic<_Tp>::value_type __v) _NOEXCEPT 2150{ 2151 return __o->wait(__v); 2152} 2153 2154// atomic_wait_explicit 2155 2156template <class _Tp> 2157_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2158void atomic_wait_explicit(const volatile atomic<_Tp>* __o, 2159 typename atomic<_Tp>::value_type __v, 2160 memory_order __m) _NOEXCEPT 2161 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 2162{ 2163 return __o->wait(__v, __m); 2164} 2165 2166template <class _Tp> 2167_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2168void atomic_wait_explicit(const atomic<_Tp>* __o, 2169 typename atomic<_Tp>::value_type __v, 2170 memory_order __m) _NOEXCEPT 2171 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 2172{ 2173 return __o->wait(__v, __m); 2174} 2175 2176// atomic_notify_one 2177 2178template <class _Tp> 2179_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2180void atomic_notify_one(volatile atomic<_Tp>* __o) _NOEXCEPT 2181{ 2182 __o->notify_one(); 2183} 2184template <class _Tp> 2185_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2186void atomic_notify_one(atomic<_Tp>* __o) _NOEXCEPT 2187{ 2188 __o->notify_one(); 2189} 2190 2191// atomic_notify_one 2192 2193template <class _Tp> 2194_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2195void atomic_notify_all(volatile atomic<_Tp>* __o) _NOEXCEPT 2196{ 2197 __o->notify_all(); 2198} 2199template <class _Tp> 2200_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2201void atomic_notify_all(atomic<_Tp>* __o) _NOEXCEPT 2202{ 2203 __o->notify_all(); 2204} 2205 2206// atomic_fetch_add 2207 2208template <class _Tp> 2209_LIBCPP_INLINE_VISIBILITY 2210_Tp 2211atomic_fetch_add(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT 2212{ 2213 return __o->fetch_add(__op); 2214} 2215 2216template <class _Tp> 2217_LIBCPP_INLINE_VISIBILITY 2218_Tp 2219atomic_fetch_add(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT 2220{ 2221 return __o->fetch_add(__op); 2222} 2223 2224// atomic_fetch_add_explicit 2225 2226template <class _Tp> 2227_LIBCPP_INLINE_VISIBILITY 2228_Tp atomic_fetch_add_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT 2229{ 2230 return __o->fetch_add(__op, __m); 2231} 2232 2233template <class _Tp> 2234_LIBCPP_INLINE_VISIBILITY 2235_Tp atomic_fetch_add_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT 2236{ 2237 return __o->fetch_add(__op, __m); 2238} 2239 2240// atomic_fetch_sub 2241 2242template <class _Tp> 2243_LIBCPP_INLINE_VISIBILITY 2244_Tp atomic_fetch_sub(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT 2245{ 2246 return __o->fetch_sub(__op); 2247} 2248 2249template <class _Tp> 2250_LIBCPP_INLINE_VISIBILITY 2251_Tp atomic_fetch_sub(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT 2252{ 2253 return __o->fetch_sub(__op); 2254} 2255 2256// atomic_fetch_sub_explicit 2257 2258template <class _Tp> 2259_LIBCPP_INLINE_VISIBILITY 2260_Tp atomic_fetch_sub_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT 2261{ 2262 return __o->fetch_sub(__op, __m); 2263} 2264 2265template <class _Tp> 2266_LIBCPP_INLINE_VISIBILITY 2267_Tp atomic_fetch_sub_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT 2268{ 2269 return __o->fetch_sub(__op, __m); 2270} 2271 2272// atomic_fetch_and 2273 2274template <class _Tp> 2275_LIBCPP_INLINE_VISIBILITY 2276typename enable_if 2277< 2278 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2279 _Tp 2280>::type 2281atomic_fetch_and(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2282{ 2283 return __o->fetch_and(__op); 2284} 2285 2286template <class _Tp> 2287_LIBCPP_INLINE_VISIBILITY 2288typename enable_if 2289< 2290 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2291 _Tp 2292>::type 2293atomic_fetch_and(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2294{ 2295 return __o->fetch_and(__op); 2296} 2297 2298// atomic_fetch_and_explicit 2299 2300template <class _Tp> 2301_LIBCPP_INLINE_VISIBILITY 2302typename enable_if 2303< 2304 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2305 _Tp 2306>::type 2307atomic_fetch_and_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2308{ 2309 return __o->fetch_and(__op, __m); 2310} 2311 2312template <class _Tp> 2313_LIBCPP_INLINE_VISIBILITY 2314typename enable_if 2315< 2316 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2317 _Tp 2318>::type 2319atomic_fetch_and_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2320{ 2321 return __o->fetch_and(__op, __m); 2322} 2323 2324// atomic_fetch_or 2325 2326template <class _Tp> 2327_LIBCPP_INLINE_VISIBILITY 2328typename enable_if 2329< 2330 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2331 _Tp 2332>::type 2333atomic_fetch_or(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2334{ 2335 return __o->fetch_or(__op); 2336} 2337 2338template <class _Tp> 2339_LIBCPP_INLINE_VISIBILITY 2340typename enable_if 2341< 2342 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2343 _Tp 2344>::type 2345atomic_fetch_or(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2346{ 2347 return __o->fetch_or(__op); 2348} 2349 2350// atomic_fetch_or_explicit 2351 2352template <class _Tp> 2353_LIBCPP_INLINE_VISIBILITY 2354typename enable_if 2355< 2356 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2357 _Tp 2358>::type 2359atomic_fetch_or_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2360{ 2361 return __o->fetch_or(__op, __m); 2362} 2363 2364template <class _Tp> 2365_LIBCPP_INLINE_VISIBILITY 2366typename enable_if 2367< 2368 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2369 _Tp 2370>::type 2371atomic_fetch_or_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2372{ 2373 return __o->fetch_or(__op, __m); 2374} 2375 2376// atomic_fetch_xor 2377 2378template <class _Tp> 2379_LIBCPP_INLINE_VISIBILITY 2380typename enable_if 2381< 2382 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2383 _Tp 2384>::type 2385atomic_fetch_xor(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2386{ 2387 return __o->fetch_xor(__op); 2388} 2389 2390template <class _Tp> 2391_LIBCPP_INLINE_VISIBILITY 2392typename enable_if 2393< 2394 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2395 _Tp 2396>::type 2397atomic_fetch_xor(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2398{ 2399 return __o->fetch_xor(__op); 2400} 2401 2402// atomic_fetch_xor_explicit 2403 2404template <class _Tp> 2405_LIBCPP_INLINE_VISIBILITY 2406typename enable_if 2407< 2408 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2409 _Tp 2410>::type 2411atomic_fetch_xor_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2412{ 2413 return __o->fetch_xor(__op, __m); 2414} 2415 2416template <class _Tp> 2417_LIBCPP_INLINE_VISIBILITY 2418typename enable_if 2419< 2420 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2421 _Tp 2422>::type 2423atomic_fetch_xor_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2424{ 2425 return __o->fetch_xor(__op, __m); 2426} 2427 2428// flag type and operations 2429 2430typedef struct atomic_flag 2431{ 2432 __cxx_atomic_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_; 2433 2434 _LIBCPP_INLINE_VISIBILITY 2435 bool test(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 2436 {return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m);} 2437 _LIBCPP_INLINE_VISIBILITY 2438 bool test(memory_order __m = memory_order_seq_cst) const _NOEXCEPT 2439 {return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m);} 2440 2441 _LIBCPP_INLINE_VISIBILITY 2442 bool test_and_set(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 2443 {return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m);} 2444 _LIBCPP_INLINE_VISIBILITY 2445 bool test_and_set(memory_order __m = memory_order_seq_cst) _NOEXCEPT 2446 {return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m);} 2447 _LIBCPP_INLINE_VISIBILITY 2448 void clear(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 2449 {__cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m);} 2450 _LIBCPP_INLINE_VISIBILITY 2451 void clear(memory_order __m = memory_order_seq_cst) _NOEXCEPT 2452 {__cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m);} 2453 2454 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2455 void wait(bool __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 2456 {__cxx_atomic_wait(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);} 2457 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2458 void wait(bool __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT 2459 {__cxx_atomic_wait(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);} 2460 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2461 void notify_one() volatile _NOEXCEPT 2462 {__cxx_atomic_notify_one(&__a_);} 2463 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2464 void notify_one() _NOEXCEPT 2465 {__cxx_atomic_notify_one(&__a_);} 2466 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2467 void notify_all() volatile _NOEXCEPT 2468 {__cxx_atomic_notify_all(&__a_);} 2469 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2470 void notify_all() _NOEXCEPT 2471 {__cxx_atomic_notify_all(&__a_);} 2472 2473#if _LIBCPP_STD_VER > 17 2474 _LIBCPP_INLINE_VISIBILITY constexpr 2475 atomic_flag() _NOEXCEPT : __a_(false) {} 2476#else 2477 _LIBCPP_INLINE_VISIBILITY 2478 atomic_flag() _NOEXCEPT _LIBCPP_DEFAULT 2479#endif 2480 2481 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR 2482 atomic_flag(bool __b) _NOEXCEPT : __a_(__b) {} // EXTENSION 2483 2484#ifndef _LIBCPP_CXX03_LANG 2485 atomic_flag(const atomic_flag&) = delete; 2486 atomic_flag& operator=(const atomic_flag&) = delete; 2487 atomic_flag& operator=(const atomic_flag&) volatile = delete; 2488#else 2489private: 2490 _LIBCPP_INLINE_VISIBILITY 2491 atomic_flag(const atomic_flag&); 2492 _LIBCPP_INLINE_VISIBILITY 2493 atomic_flag& operator=(const atomic_flag&); 2494 _LIBCPP_INLINE_VISIBILITY 2495 atomic_flag& operator=(const atomic_flag&) volatile; 2496#endif 2497} atomic_flag; 2498 2499 2500inline _LIBCPP_INLINE_VISIBILITY 2501bool 2502atomic_flag_test(const volatile atomic_flag* __o) _NOEXCEPT 2503{ 2504 return __o->test(); 2505} 2506 2507inline _LIBCPP_INLINE_VISIBILITY 2508bool 2509atomic_flag_test(const atomic_flag* __o) _NOEXCEPT 2510{ 2511 return __o->test(); 2512} 2513 2514inline _LIBCPP_INLINE_VISIBILITY 2515bool 2516atomic_flag_test_explicit(const volatile atomic_flag* __o, memory_order __m) _NOEXCEPT 2517{ 2518 return __o->test(__m); 2519} 2520 2521inline _LIBCPP_INLINE_VISIBILITY 2522bool 2523atomic_flag_test_explicit(const atomic_flag* __o, memory_order __m) _NOEXCEPT 2524{ 2525 return __o->test(__m); 2526} 2527 2528inline _LIBCPP_INLINE_VISIBILITY 2529bool 2530atomic_flag_test_and_set(volatile atomic_flag* __o) _NOEXCEPT 2531{ 2532 return __o->test_and_set(); 2533} 2534 2535inline _LIBCPP_INLINE_VISIBILITY 2536bool 2537atomic_flag_test_and_set(atomic_flag* __o) _NOEXCEPT 2538{ 2539 return __o->test_and_set(); 2540} 2541 2542inline _LIBCPP_INLINE_VISIBILITY 2543bool 2544atomic_flag_test_and_set_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT 2545{ 2546 return __o->test_and_set(__m); 2547} 2548 2549inline _LIBCPP_INLINE_VISIBILITY 2550bool 2551atomic_flag_test_and_set_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT 2552{ 2553 return __o->test_and_set(__m); 2554} 2555 2556inline _LIBCPP_INLINE_VISIBILITY 2557void 2558atomic_flag_clear(volatile atomic_flag* __o) _NOEXCEPT 2559{ 2560 __o->clear(); 2561} 2562 2563inline _LIBCPP_INLINE_VISIBILITY 2564void 2565atomic_flag_clear(atomic_flag* __o) _NOEXCEPT 2566{ 2567 __o->clear(); 2568} 2569 2570inline _LIBCPP_INLINE_VISIBILITY 2571void 2572atomic_flag_clear_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT 2573{ 2574 __o->clear(__m); 2575} 2576 2577inline _LIBCPP_INLINE_VISIBILITY 2578void 2579atomic_flag_clear_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT 2580{ 2581 __o->clear(__m); 2582} 2583 2584inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2585void 2586atomic_flag_wait(const volatile atomic_flag* __o, bool __v) _NOEXCEPT 2587{ 2588 __o->wait(__v); 2589} 2590 2591inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2592void 2593atomic_flag_wait(const atomic_flag* __o, bool __v) _NOEXCEPT 2594{ 2595 __o->wait(__v); 2596} 2597 2598inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2599void 2600atomic_flag_wait_explicit(const volatile atomic_flag* __o, 2601 bool __v, memory_order __m) _NOEXCEPT 2602{ 2603 __o->wait(__v, __m); 2604} 2605 2606inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2607void 2608atomic_flag_wait_explicit(const atomic_flag* __o, 2609 bool __v, memory_order __m) _NOEXCEPT 2610{ 2611 __o->wait(__v, __m); 2612} 2613 2614inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2615void 2616atomic_flag_notify_one(volatile atomic_flag* __o) _NOEXCEPT 2617{ 2618 __o->notify_one(); 2619} 2620 2621inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2622void 2623atomic_flag_notify_one(atomic_flag* __o) _NOEXCEPT 2624{ 2625 __o->notify_one(); 2626} 2627 2628inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2629void 2630atomic_flag_notify_all(volatile atomic_flag* __o) _NOEXCEPT 2631{ 2632 __o->notify_all(); 2633} 2634 2635inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2636void 2637atomic_flag_notify_all(atomic_flag* __o) _NOEXCEPT 2638{ 2639 __o->notify_all(); 2640} 2641 2642// fences 2643 2644inline _LIBCPP_INLINE_VISIBILITY 2645void 2646atomic_thread_fence(memory_order __m) _NOEXCEPT 2647{ 2648 __cxx_atomic_thread_fence(__m); 2649} 2650 2651inline _LIBCPP_INLINE_VISIBILITY 2652void 2653atomic_signal_fence(memory_order __m) _NOEXCEPT 2654{ 2655 __cxx_atomic_signal_fence(__m); 2656} 2657 2658// Atomics for standard typedef types 2659 2660typedef atomic<bool> atomic_bool; 2661typedef atomic<char> atomic_char; 2662typedef atomic<signed char> atomic_schar; 2663typedef atomic<unsigned char> atomic_uchar; 2664typedef atomic<short> atomic_short; 2665typedef atomic<unsigned short> atomic_ushort; 2666typedef atomic<int> atomic_int; 2667typedef atomic<unsigned int> atomic_uint; 2668typedef atomic<long> atomic_long; 2669typedef atomic<unsigned long> atomic_ulong; 2670typedef atomic<long long> atomic_llong; 2671typedef atomic<unsigned long long> atomic_ullong; 2672#ifndef _LIBCPP_HAS_NO_CHAR8_T 2673typedef atomic<char8_t> atomic_char8_t; 2674#endif 2675typedef atomic<char16_t> atomic_char16_t; 2676typedef atomic<char32_t> atomic_char32_t; 2677#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS 2678typedef atomic<wchar_t> atomic_wchar_t; 2679#endif 2680 2681typedef atomic<int_least8_t> atomic_int_least8_t; 2682typedef atomic<uint_least8_t> atomic_uint_least8_t; 2683typedef atomic<int_least16_t> atomic_int_least16_t; 2684typedef atomic<uint_least16_t> atomic_uint_least16_t; 2685typedef atomic<int_least32_t> atomic_int_least32_t; 2686typedef atomic<uint_least32_t> atomic_uint_least32_t; 2687typedef atomic<int_least64_t> atomic_int_least64_t; 2688typedef atomic<uint_least64_t> atomic_uint_least64_t; 2689 2690typedef atomic<int_fast8_t> atomic_int_fast8_t; 2691typedef atomic<uint_fast8_t> atomic_uint_fast8_t; 2692typedef atomic<int_fast16_t> atomic_int_fast16_t; 2693typedef atomic<uint_fast16_t> atomic_uint_fast16_t; 2694typedef atomic<int_fast32_t> atomic_int_fast32_t; 2695typedef atomic<uint_fast32_t> atomic_uint_fast32_t; 2696typedef atomic<int_fast64_t> atomic_int_fast64_t; 2697typedef atomic<uint_fast64_t> atomic_uint_fast64_t; 2698 2699typedef atomic< int8_t> atomic_int8_t; 2700typedef atomic<uint8_t> atomic_uint8_t; 2701typedef atomic< int16_t> atomic_int16_t; 2702typedef atomic<uint16_t> atomic_uint16_t; 2703typedef atomic< int32_t> atomic_int32_t; 2704typedef atomic<uint32_t> atomic_uint32_t; 2705typedef atomic< int64_t> atomic_int64_t; 2706typedef atomic<uint64_t> atomic_uint64_t; 2707 2708typedef atomic<intptr_t> atomic_intptr_t; 2709typedef atomic<uintptr_t> atomic_uintptr_t; 2710typedef atomic<size_t> atomic_size_t; 2711typedef atomic<ptrdiff_t> atomic_ptrdiff_t; 2712typedef atomic<intmax_t> atomic_intmax_t; 2713typedef atomic<uintmax_t> atomic_uintmax_t; 2714 2715// atomic_*_lock_free : prefer the contention type most highly, then the largest lock-free type 2716 2717#ifdef __cpp_lib_atomic_is_always_lock_free 2718# define _LIBCPP_CONTENTION_LOCK_FREE __atomic_always_lock_free(sizeof(__cxx_contention_t), 0) 2719#else 2720# define _LIBCPP_CONTENTION_LOCK_FREE false 2721#endif 2722 2723#if ATOMIC_LLONG_LOCK_FREE == 2 2724typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, long long>::type __libcpp_signed_lock_free; 2725typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned long long>::type __libcpp_unsigned_lock_free; 2726#elif ATOMIC_INT_LOCK_FREE == 2 2727typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, int>::type __libcpp_signed_lock_free; 2728typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned int>::type __libcpp_unsigned_lock_free; 2729#elif ATOMIC_SHORT_LOCK_FREE == 2 2730typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, short>::type __libcpp_signed_lock_free; 2731typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned short>::type __libcpp_unsigned_lock_free; 2732#elif ATOMIC_CHAR_LOCK_FREE == 2 2733typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, char>::type __libcpp_signed_lock_free; 2734typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned char>::type __libcpp_unsigned_lock_free; 2735#else 2736 // No signed/unsigned lock-free types 2737#endif 2738 2739typedef atomic<__libcpp_signed_lock_free> atomic_signed_lock_free; 2740typedef atomic<__libcpp_unsigned_lock_free> atomic_unsigned_lock_free; 2741 2742#define ATOMIC_FLAG_INIT {false} 2743#define ATOMIC_VAR_INIT(__v) {__v} 2744 2745_LIBCPP_END_NAMESPACE_STD 2746 2747#endif // _LIBCPP_ATOMIC 2748