1// -*- C++ -*- 2//===--------------------------- atomic -----------------------------------===// 3// 4// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 5// See https://llvm.org/LICENSE.txt for license information. 6// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 7// 8//===----------------------------------------------------------------------===// 9 10#ifndef _LIBCPP_ATOMIC 11#define _LIBCPP_ATOMIC 12 13/* 14 atomic synopsis 15 16namespace std 17{ 18 19// feature test macro 20 21#define __cpp_lib_atomic_is_always_lock_free // as specified by SG10 22 23 // order and consistency 24 25 enum memory_order: unspecified // enum class in C++20 26 { 27 relaxed, 28 consume, // load-consume 29 acquire, // load-acquire 30 release, // store-release 31 acq_rel, // store-release load-acquire 32 seq_cst // store-release load-acquire 33 }; 34 35 inline constexpr auto memory_order_relaxed = memory_order::relaxed; 36 inline constexpr auto memory_order_consume = memory_order::consume; 37 inline constexpr auto memory_order_acquire = memory_order::acquire; 38 inline constexpr auto memory_order_release = memory_order::release; 39 inline constexpr auto memory_order_acq_rel = memory_order::acq_rel; 40 inline constexpr auto memory_order_seq_cst = memory_order::seq_cst; 41 42template <class T> T kill_dependency(T y) noexcept; 43 44// lock-free property 45 46#define ATOMIC_BOOL_LOCK_FREE unspecified 47#define ATOMIC_CHAR_LOCK_FREE unspecified 48#define ATOMIC_CHAR16_T_LOCK_FREE unspecified 49#define ATOMIC_CHAR32_T_LOCK_FREE unspecified 50#define ATOMIC_WCHAR_T_LOCK_FREE unspecified 51#define ATOMIC_SHORT_LOCK_FREE unspecified 52#define ATOMIC_INT_LOCK_FREE unspecified 53#define ATOMIC_LONG_LOCK_FREE unspecified 54#define ATOMIC_LLONG_LOCK_FREE unspecified 55#define ATOMIC_POINTER_LOCK_FREE unspecified 56 57// flag type and operations 58 59typedef struct atomic_flag 60{ 61 bool test_and_set(memory_order m = memory_order_seq_cst) volatile noexcept; 62 bool test_and_set(memory_order m = memory_order_seq_cst) noexcept; 63 void clear(memory_order m = memory_order_seq_cst) volatile noexcept; 64 void clear(memory_order m = memory_order_seq_cst) noexcept; 65 atomic_flag() noexcept = default; 66 atomic_flag(const atomic_flag&) = delete; 67 atomic_flag& operator=(const atomic_flag&) = delete; 68 atomic_flag& operator=(const atomic_flag&) volatile = delete; 69} atomic_flag; 70 71bool 72 atomic_flag_test_and_set(volatile atomic_flag* obj) noexcept; 73 74bool 75 atomic_flag_test_and_set(atomic_flag* obj) noexcept; 76 77bool 78 atomic_flag_test_and_set_explicit(volatile atomic_flag* obj, 79 memory_order m) noexcept; 80 81bool 82 atomic_flag_test_and_set_explicit(atomic_flag* obj, memory_order m) noexcept; 83 84void 85 atomic_flag_clear(volatile atomic_flag* obj) noexcept; 86 87void 88 atomic_flag_clear(atomic_flag* obj) noexcept; 89 90void 91 atomic_flag_clear_explicit(volatile atomic_flag* obj, memory_order m) noexcept; 92 93void 94 atomic_flag_clear_explicit(atomic_flag* obj, memory_order m) noexcept; 95 96#define ATOMIC_FLAG_INIT see below 97#define ATOMIC_VAR_INIT(value) see below 98 99template <class T> 100struct atomic 101{ 102 static constexpr bool is_always_lock_free; 103 bool is_lock_free() const volatile noexcept; 104 bool is_lock_free() const noexcept; 105 void store(T desr, memory_order m = memory_order_seq_cst) volatile noexcept; 106 void store(T desr, memory_order m = memory_order_seq_cst) noexcept; 107 T load(memory_order m = memory_order_seq_cst) const volatile noexcept; 108 T load(memory_order m = memory_order_seq_cst) const noexcept; 109 operator T() const volatile noexcept; 110 operator T() const noexcept; 111 T exchange(T desr, memory_order m = memory_order_seq_cst) volatile noexcept; 112 T exchange(T desr, memory_order m = memory_order_seq_cst) noexcept; 113 bool compare_exchange_weak(T& expc, T desr, 114 memory_order s, memory_order f) volatile noexcept; 115 bool compare_exchange_weak(T& expc, T desr, memory_order s, memory_order f) noexcept; 116 bool compare_exchange_strong(T& expc, T desr, 117 memory_order s, memory_order f) volatile noexcept; 118 bool compare_exchange_strong(T& expc, T desr, 119 memory_order s, memory_order f) noexcept; 120 bool compare_exchange_weak(T& expc, T desr, 121 memory_order m = memory_order_seq_cst) volatile noexcept; 122 bool compare_exchange_weak(T& expc, T desr, 123 memory_order m = memory_order_seq_cst) noexcept; 124 bool compare_exchange_strong(T& expc, T desr, 125 memory_order m = memory_order_seq_cst) volatile noexcept; 126 bool compare_exchange_strong(T& expc, T desr, 127 memory_order m = memory_order_seq_cst) noexcept; 128 129 atomic() noexcept = default; 130 constexpr atomic(T desr) noexcept; 131 atomic(const atomic&) = delete; 132 atomic& operator=(const atomic&) = delete; 133 atomic& operator=(const atomic&) volatile = delete; 134 T operator=(T) volatile noexcept; 135 T operator=(T) noexcept; 136}; 137 138template <> 139struct atomic<integral> 140{ 141 static constexpr bool is_always_lock_free; 142 bool is_lock_free() const volatile noexcept; 143 bool is_lock_free() const noexcept; 144 void store(integral desr, memory_order m = memory_order_seq_cst) volatile noexcept; 145 void store(integral desr, memory_order m = memory_order_seq_cst) noexcept; 146 integral load(memory_order m = memory_order_seq_cst) const volatile noexcept; 147 integral load(memory_order m = memory_order_seq_cst) const noexcept; 148 operator integral() const volatile noexcept; 149 operator integral() const noexcept; 150 integral exchange(integral desr, 151 memory_order m = memory_order_seq_cst) volatile noexcept; 152 integral exchange(integral desr, memory_order m = memory_order_seq_cst) noexcept; 153 bool compare_exchange_weak(integral& expc, integral desr, 154 memory_order s, memory_order f) volatile noexcept; 155 bool compare_exchange_weak(integral& expc, integral desr, 156 memory_order s, memory_order f) noexcept; 157 bool compare_exchange_strong(integral& expc, integral desr, 158 memory_order s, memory_order f) volatile noexcept; 159 bool compare_exchange_strong(integral& expc, integral desr, 160 memory_order s, memory_order f) noexcept; 161 bool compare_exchange_weak(integral& expc, integral desr, 162 memory_order m = memory_order_seq_cst) volatile noexcept; 163 bool compare_exchange_weak(integral& expc, integral desr, 164 memory_order m = memory_order_seq_cst) noexcept; 165 bool compare_exchange_strong(integral& expc, integral desr, 166 memory_order m = memory_order_seq_cst) volatile noexcept; 167 bool compare_exchange_strong(integral& expc, integral desr, 168 memory_order m = memory_order_seq_cst) noexcept; 169 170 integral 171 fetch_add(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 172 integral fetch_add(integral op, memory_order m = memory_order_seq_cst) noexcept; 173 integral 174 fetch_sub(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 175 integral fetch_sub(integral op, memory_order m = memory_order_seq_cst) noexcept; 176 integral 177 fetch_and(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 178 integral fetch_and(integral op, memory_order m = memory_order_seq_cst) noexcept; 179 integral 180 fetch_or(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 181 integral fetch_or(integral op, memory_order m = memory_order_seq_cst) noexcept; 182 integral 183 fetch_xor(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 184 integral fetch_xor(integral op, memory_order m = memory_order_seq_cst) noexcept; 185 186 atomic() noexcept = default; 187 constexpr atomic(integral desr) noexcept; 188 atomic(const atomic&) = delete; 189 atomic& operator=(const atomic&) = delete; 190 atomic& operator=(const atomic&) volatile = delete; 191 integral operator=(integral desr) volatile noexcept; 192 integral operator=(integral desr) noexcept; 193 194 integral operator++(int) volatile noexcept; 195 integral operator++(int) noexcept; 196 integral operator--(int) volatile noexcept; 197 integral operator--(int) noexcept; 198 integral operator++() volatile noexcept; 199 integral operator++() noexcept; 200 integral operator--() volatile noexcept; 201 integral operator--() noexcept; 202 integral operator+=(integral op) volatile noexcept; 203 integral operator+=(integral op) noexcept; 204 integral operator-=(integral op) volatile noexcept; 205 integral operator-=(integral op) noexcept; 206 integral operator&=(integral op) volatile noexcept; 207 integral operator&=(integral op) noexcept; 208 integral operator|=(integral op) volatile noexcept; 209 integral operator|=(integral op) noexcept; 210 integral operator^=(integral op) volatile noexcept; 211 integral operator^=(integral op) noexcept; 212}; 213 214template <class T> 215struct atomic<T*> 216{ 217 static constexpr bool is_always_lock_free; 218 bool is_lock_free() const volatile noexcept; 219 bool is_lock_free() const noexcept; 220 void store(T* desr, memory_order m = memory_order_seq_cst) volatile noexcept; 221 void store(T* desr, memory_order m = memory_order_seq_cst) noexcept; 222 T* load(memory_order m = memory_order_seq_cst) const volatile noexcept; 223 T* load(memory_order m = memory_order_seq_cst) const noexcept; 224 operator T*() const volatile noexcept; 225 operator T*() const noexcept; 226 T* exchange(T* desr, memory_order m = memory_order_seq_cst) volatile noexcept; 227 T* exchange(T* desr, memory_order m = memory_order_seq_cst) noexcept; 228 bool compare_exchange_weak(T*& expc, T* desr, 229 memory_order s, memory_order f) volatile noexcept; 230 bool compare_exchange_weak(T*& expc, T* desr, 231 memory_order s, memory_order f) noexcept; 232 bool compare_exchange_strong(T*& expc, T* desr, 233 memory_order s, memory_order f) volatile noexcept; 234 bool compare_exchange_strong(T*& expc, T* desr, 235 memory_order s, memory_order f) noexcept; 236 bool compare_exchange_weak(T*& expc, T* desr, 237 memory_order m = memory_order_seq_cst) volatile noexcept; 238 bool compare_exchange_weak(T*& expc, T* desr, 239 memory_order m = memory_order_seq_cst) noexcept; 240 bool compare_exchange_strong(T*& expc, T* desr, 241 memory_order m = memory_order_seq_cst) volatile noexcept; 242 bool compare_exchange_strong(T*& expc, T* desr, 243 memory_order m = memory_order_seq_cst) noexcept; 244 T* fetch_add(ptrdiff_t op, memory_order m = memory_order_seq_cst) volatile noexcept; 245 T* fetch_add(ptrdiff_t op, memory_order m = memory_order_seq_cst) noexcept; 246 T* fetch_sub(ptrdiff_t op, memory_order m = memory_order_seq_cst) volatile noexcept; 247 T* fetch_sub(ptrdiff_t op, memory_order m = memory_order_seq_cst) noexcept; 248 249 atomic() noexcept = default; 250 constexpr atomic(T* desr) noexcept; 251 atomic(const atomic&) = delete; 252 atomic& operator=(const atomic&) = delete; 253 atomic& operator=(const atomic&) volatile = delete; 254 255 T* operator=(T*) volatile noexcept; 256 T* operator=(T*) noexcept; 257 T* operator++(int) volatile noexcept; 258 T* operator++(int) noexcept; 259 T* operator--(int) volatile noexcept; 260 T* operator--(int) noexcept; 261 T* operator++() volatile noexcept; 262 T* operator++() noexcept; 263 T* operator--() volatile noexcept; 264 T* operator--() noexcept; 265 T* operator+=(ptrdiff_t op) volatile noexcept; 266 T* operator+=(ptrdiff_t op) noexcept; 267 T* operator-=(ptrdiff_t op) volatile noexcept; 268 T* operator-=(ptrdiff_t op) noexcept; 269}; 270 271 272template <class T> 273 bool 274 atomic_is_lock_free(const volatile atomic<T>* obj) noexcept; 275 276template <class T> 277 bool 278 atomic_is_lock_free(const atomic<T>* obj) noexcept; 279 280template <class T> 281 void 282 atomic_init(volatile atomic<T>* obj, T desr) noexcept; 283 284template <class T> 285 void 286 atomic_init(atomic<T>* obj, T desr) noexcept; 287 288template <class T> 289 void 290 atomic_store(volatile atomic<T>* obj, T desr) noexcept; 291 292template <class T> 293 void 294 atomic_store(atomic<T>* obj, T desr) noexcept; 295 296template <class T> 297 void 298 atomic_store_explicit(volatile atomic<T>* obj, T desr, memory_order m) noexcept; 299 300template <class T> 301 void 302 atomic_store_explicit(atomic<T>* obj, T desr, memory_order m) noexcept; 303 304template <class T> 305 T 306 atomic_load(const volatile atomic<T>* obj) noexcept; 307 308template <class T> 309 T 310 atomic_load(const atomic<T>* obj) noexcept; 311 312template <class T> 313 T 314 atomic_load_explicit(const volatile atomic<T>* obj, memory_order m) noexcept; 315 316template <class T> 317 T 318 atomic_load_explicit(const atomic<T>* obj, memory_order m) noexcept; 319 320template <class T> 321 T 322 atomic_exchange(volatile atomic<T>* obj, T desr) noexcept; 323 324template <class T> 325 T 326 atomic_exchange(atomic<T>* obj, T desr) noexcept; 327 328template <class T> 329 T 330 atomic_exchange_explicit(volatile atomic<T>* obj, T desr, memory_order m) noexcept; 331 332template <class T> 333 T 334 atomic_exchange_explicit(atomic<T>* obj, T desr, memory_order m) noexcept; 335 336template <class T> 337 bool 338 atomic_compare_exchange_weak(volatile atomic<T>* obj, T* expc, T desr) noexcept; 339 340template <class T> 341 bool 342 atomic_compare_exchange_weak(atomic<T>* obj, T* expc, T desr) noexcept; 343 344template <class T> 345 bool 346 atomic_compare_exchange_strong(volatile atomic<T>* obj, T* expc, T desr) noexcept; 347 348template <class T> 349 bool 350 atomic_compare_exchange_strong(atomic<T>* obj, T* expc, T desr) noexcept; 351 352template <class T> 353 bool 354 atomic_compare_exchange_weak_explicit(volatile atomic<T>* obj, T* expc, 355 T desr, 356 memory_order s, memory_order f) noexcept; 357 358template <class T> 359 bool 360 atomic_compare_exchange_weak_explicit(atomic<T>* obj, T* expc, T desr, 361 memory_order s, memory_order f) noexcept; 362 363template <class T> 364 bool 365 atomic_compare_exchange_strong_explicit(volatile atomic<T>* obj, 366 T* expc, T desr, 367 memory_order s, memory_order f) noexcept; 368 369template <class T> 370 bool 371 atomic_compare_exchange_strong_explicit(atomic<T>* obj, T* expc, 372 T desr, 373 memory_order s, memory_order f) noexcept; 374 375template <class Integral> 376 Integral 377 atomic_fetch_add(volatile atomic<Integral>* obj, Integral op) noexcept; 378 379template <class Integral> 380 Integral 381 atomic_fetch_add(atomic<Integral>* obj, Integral op) noexcept; 382 383template <class Integral> 384 Integral 385 atomic_fetch_add_explicit(volatile atomic<Integral>* obj, Integral op, 386 memory_order m) noexcept; 387template <class Integral> 388 Integral 389 atomic_fetch_add_explicit(atomic<Integral>* obj, Integral op, 390 memory_order m) noexcept; 391template <class Integral> 392 Integral 393 atomic_fetch_sub(volatile atomic<Integral>* obj, Integral op) noexcept; 394 395template <class Integral> 396 Integral 397 atomic_fetch_sub(atomic<Integral>* obj, Integral op) noexcept; 398 399template <class Integral> 400 Integral 401 atomic_fetch_sub_explicit(volatile atomic<Integral>* obj, Integral op, 402 memory_order m) noexcept; 403template <class Integral> 404 Integral 405 atomic_fetch_sub_explicit(atomic<Integral>* obj, Integral op, 406 memory_order m) noexcept; 407template <class Integral> 408 Integral 409 atomic_fetch_and(volatile atomic<Integral>* obj, Integral op) noexcept; 410 411template <class Integral> 412 Integral 413 atomic_fetch_and(atomic<Integral>* obj, Integral op) noexcept; 414 415template <class Integral> 416 Integral 417 atomic_fetch_and_explicit(volatile atomic<Integral>* obj, Integral op, 418 memory_order m) noexcept; 419template <class Integral> 420 Integral 421 atomic_fetch_and_explicit(atomic<Integral>* obj, Integral op, 422 memory_order m) noexcept; 423template <class Integral> 424 Integral 425 atomic_fetch_or(volatile atomic<Integral>* obj, Integral op) noexcept; 426 427template <class Integral> 428 Integral 429 atomic_fetch_or(atomic<Integral>* obj, Integral op) noexcept; 430 431template <class Integral> 432 Integral 433 atomic_fetch_or_explicit(volatile atomic<Integral>* obj, Integral op, 434 memory_order m) noexcept; 435template <class Integral> 436 Integral 437 atomic_fetch_or_explicit(atomic<Integral>* obj, Integral op, 438 memory_order m) noexcept; 439template <class Integral> 440 Integral 441 atomic_fetch_xor(volatile atomic<Integral>* obj, Integral op) noexcept; 442 443template <class Integral> 444 Integral 445 atomic_fetch_xor(atomic<Integral>* obj, Integral op) noexcept; 446 447template <class Integral> 448 Integral 449 atomic_fetch_xor_explicit(volatile atomic<Integral>* obj, Integral op, 450 memory_order m) noexcept; 451template <class Integral> 452 Integral 453 atomic_fetch_xor_explicit(atomic<Integral>* obj, Integral op, 454 memory_order m) noexcept; 455 456template <class T> 457 T* 458 atomic_fetch_add(volatile atomic<T*>* obj, ptrdiff_t op) noexcept; 459 460template <class T> 461 T* 462 atomic_fetch_add(atomic<T*>* obj, ptrdiff_t op) noexcept; 463 464template <class T> 465 T* 466 atomic_fetch_add_explicit(volatile atomic<T*>* obj, ptrdiff_t op, 467 memory_order m) noexcept; 468template <class T> 469 T* 470 atomic_fetch_add_explicit(atomic<T*>* obj, ptrdiff_t op, memory_order m) noexcept; 471 472template <class T> 473 T* 474 atomic_fetch_sub(volatile atomic<T*>* obj, ptrdiff_t op) noexcept; 475 476template <class T> 477 T* 478 atomic_fetch_sub(atomic<T*>* obj, ptrdiff_t op) noexcept; 479 480template <class T> 481 T* 482 atomic_fetch_sub_explicit(volatile atomic<T*>* obj, ptrdiff_t op, 483 memory_order m) noexcept; 484template <class T> 485 T* 486 atomic_fetch_sub_explicit(atomic<T*>* obj, ptrdiff_t op, memory_order m) noexcept; 487 488// Atomics for standard typedef types 489 490typedef atomic<bool> atomic_bool; 491typedef atomic<char> atomic_char; 492typedef atomic<signed char> atomic_schar; 493typedef atomic<unsigned char> atomic_uchar; 494typedef atomic<short> atomic_short; 495typedef atomic<unsigned short> atomic_ushort; 496typedef atomic<int> atomic_int; 497typedef atomic<unsigned int> atomic_uint; 498typedef atomic<long> atomic_long; 499typedef atomic<unsigned long> atomic_ulong; 500typedef atomic<long long> atomic_llong; 501typedef atomic<unsigned long long> atomic_ullong; 502typedef atomic<char16_t> atomic_char16_t; 503typedef atomic<char32_t> atomic_char32_t; 504typedef atomic<wchar_t> atomic_wchar_t; 505 506typedef atomic<int_least8_t> atomic_int_least8_t; 507typedef atomic<uint_least8_t> atomic_uint_least8_t; 508typedef atomic<int_least16_t> atomic_int_least16_t; 509typedef atomic<uint_least16_t> atomic_uint_least16_t; 510typedef atomic<int_least32_t> atomic_int_least32_t; 511typedef atomic<uint_least32_t> atomic_uint_least32_t; 512typedef atomic<int_least64_t> atomic_int_least64_t; 513typedef atomic<uint_least64_t> atomic_uint_least64_t; 514 515typedef atomic<int_fast8_t> atomic_int_fast8_t; 516typedef atomic<uint_fast8_t> atomic_uint_fast8_t; 517typedef atomic<int_fast16_t> atomic_int_fast16_t; 518typedef atomic<uint_fast16_t> atomic_uint_fast16_t; 519typedef atomic<int_fast32_t> atomic_int_fast32_t; 520typedef atomic<uint_fast32_t> atomic_uint_fast32_t; 521typedef atomic<int_fast64_t> atomic_int_fast64_t; 522typedef atomic<uint_fast64_t> atomic_uint_fast64_t; 523 524typedef atomic<int8_t> atomic_int8_t; 525typedef atomic<uint8_t> atomic_uint8_t; 526typedef atomic<int16_t> atomic_int16_t; 527typedef atomic<uint16_t> atomic_uint16_t; 528typedef atomic<int32_t> atomic_int32_t; 529typedef atomic<uint32_t> atomic_uint32_t; 530typedef atomic<int64_t> atomic_int64_t; 531typedef atomic<uint64_t> atomic_uint64_t; 532 533typedef atomic<intptr_t> atomic_intptr_t; 534typedef atomic<uintptr_t> atomic_uintptr_t; 535typedef atomic<size_t> atomic_size_t; 536typedef atomic<ptrdiff_t> atomic_ptrdiff_t; 537typedef atomic<intmax_t> atomic_intmax_t; 538typedef atomic<uintmax_t> atomic_uintmax_t; 539 540// fences 541 542void atomic_thread_fence(memory_order m) noexcept; 543void atomic_signal_fence(memory_order m) noexcept; 544 545} // std 546 547*/ 548 549#include <__config> 550#include <__threading_support> 551#include <cstddef> 552#include <cstdint> 553#include <cstring> 554#include <type_traits> 555#include <version> 556 557#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) 558#pragma GCC system_header 559#endif 560 561#ifdef _LIBCPP_HAS_NO_THREADS 562# error <atomic> is not supported on this single threaded system 563#endif 564#ifdef _LIBCPP_HAS_NO_ATOMIC_HEADER 565# error <atomic> is not implemented 566#endif 567#ifdef kill_dependency 568# error C++ standard library is incompatible with <stdatomic.h> 569#endif 570 571#define _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) \ 572 _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_consume || \ 573 __m == memory_order_acquire || \ 574 __m == memory_order_acq_rel, \ 575 "memory order argument to atomic operation is invalid") 576 577#define _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) \ 578 _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_release || \ 579 __m == memory_order_acq_rel, \ 580 "memory order argument to atomic operation is invalid") 581 582#define _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__m, __f) \ 583 _LIBCPP_DIAGNOSE_WARNING(__f == memory_order_release || \ 584 __f == memory_order_acq_rel, \ 585 "memory order argument to atomic operation is invalid") 586 587_LIBCPP_BEGIN_NAMESPACE_STD 588 589// Figure out what the underlying type for `memory_order` would be if it were 590// declared as an unscoped enum (accounting for -fshort-enums). Use this result 591// to pin the underlying type in C++20. 592enum __legacy_memory_order { 593 __mo_relaxed, 594 __mo_consume, 595 __mo_acquire, 596 __mo_release, 597 __mo_acq_rel, 598 __mo_seq_cst 599}; 600 601typedef underlying_type<__legacy_memory_order>::type __memory_order_underlying_t; 602 603#if _LIBCPP_STD_VER > 17 604 605enum class memory_order : __memory_order_underlying_t { 606 relaxed = __mo_relaxed, 607 consume = __mo_consume, 608 acquire = __mo_acquire, 609 release = __mo_release, 610 acq_rel = __mo_acq_rel, 611 seq_cst = __mo_seq_cst 612}; 613 614inline constexpr auto memory_order_relaxed = memory_order::relaxed; 615inline constexpr auto memory_order_consume = memory_order::consume; 616inline constexpr auto memory_order_acquire = memory_order::acquire; 617inline constexpr auto memory_order_release = memory_order::release; 618inline constexpr auto memory_order_acq_rel = memory_order::acq_rel; 619inline constexpr auto memory_order_seq_cst = memory_order::seq_cst; 620 621#else 622 623typedef enum memory_order { 624 memory_order_relaxed = __mo_relaxed, 625 memory_order_consume = __mo_consume, 626 memory_order_acquire = __mo_acquire, 627 memory_order_release = __mo_release, 628 memory_order_acq_rel = __mo_acq_rel, 629 memory_order_seq_cst = __mo_seq_cst, 630} memory_order; 631 632#endif // _LIBCPP_STD_VER > 17 633 634template <typename _Tp> _LIBCPP_INLINE_VISIBILITY 635bool __cxx_nonatomic_compare_equal(_Tp const& __lhs, _Tp const& __rhs) { 636 return memcmp(&__lhs, &__rhs, sizeof(_Tp)) == 0; 637} 638 639static_assert((is_same<underlying_type<memory_order>::type, __memory_order_underlying_t>::value), 640 "unexpected underlying type for std::memory_order"); 641 642#if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) || \ 643 defined(_LIBCPP_ATOMIC_ONLY_USE_BUILTINS) 644 645// [atomics.types.generic]p1 guarantees _Tp is trivially copyable. Because 646// the default operator= in an object is not volatile, a byte-by-byte copy 647// is required. 648template <typename _Tp, typename _Tv> _LIBCPP_INLINE_VISIBILITY 649typename enable_if<is_assignable<_Tp&, _Tv>::value>::type 650__cxx_atomic_assign_volatile(_Tp& __a_value, _Tv const& __val) { 651 __a_value = __val; 652} 653template <typename _Tp, typename _Tv> _LIBCPP_INLINE_VISIBILITY 654typename enable_if<is_assignable<_Tp&, _Tv>::value>::type 655__cxx_atomic_assign_volatile(_Tp volatile& __a_value, _Tv volatile const& __val) { 656 volatile char* __to = reinterpret_cast<volatile char*>(&__a_value); 657 volatile char* __end = __to + sizeof(_Tp); 658 volatile const char* __from = reinterpret_cast<volatile const char*>(&__val); 659 while (__to != __end) 660 *__to++ = *__from++; 661} 662 663#endif 664 665#if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) 666 667template <typename _Tp> 668struct __cxx_atomic_base_impl { 669 670 _LIBCPP_INLINE_VISIBILITY 671#ifndef _LIBCPP_CXX03_LANG 672 __cxx_atomic_base_impl() _NOEXCEPT = default; 673#else 674 __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {} 675#endif // _LIBCPP_CXX03_LANG 676 _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT 677 : __a_value(value) {} 678 _Tp __a_value; 679}; 680 681_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR int __to_gcc_order(memory_order __order) { 682 // Avoid switch statement to make this a constexpr. 683 return __order == memory_order_relaxed ? __ATOMIC_RELAXED: 684 (__order == memory_order_acquire ? __ATOMIC_ACQUIRE: 685 (__order == memory_order_release ? __ATOMIC_RELEASE: 686 (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST: 687 (__order == memory_order_acq_rel ? __ATOMIC_ACQ_REL: 688 __ATOMIC_CONSUME)))); 689} 690 691_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR int __to_gcc_failure_order(memory_order __order) { 692 // Avoid switch statement to make this a constexpr. 693 return __order == memory_order_relaxed ? __ATOMIC_RELAXED: 694 (__order == memory_order_acquire ? __ATOMIC_ACQUIRE: 695 (__order == memory_order_release ? __ATOMIC_RELAXED: 696 (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST: 697 (__order == memory_order_acq_rel ? __ATOMIC_ACQUIRE: 698 __ATOMIC_CONSUME)))); 699} 700 701template <typename _Tp> 702_LIBCPP_INLINE_VISIBILITY 703void __cxx_atomic_init(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val) { 704 __cxx_atomic_assign_volatile(__a->__a_value, __val); 705} 706 707template <typename _Tp> 708_LIBCPP_INLINE_VISIBILITY 709void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) { 710 __a->__a_value = __val; 711} 712 713_LIBCPP_INLINE_VISIBILITY inline 714void __cxx_atomic_thread_fence(memory_order __order) { 715 __atomic_thread_fence(__to_gcc_order(__order)); 716} 717 718_LIBCPP_INLINE_VISIBILITY inline 719void __cxx_atomic_signal_fence(memory_order __order) { 720 __atomic_signal_fence(__to_gcc_order(__order)); 721} 722 723template <typename _Tp> 724_LIBCPP_INLINE_VISIBILITY 725void __cxx_atomic_store(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val, 726 memory_order __order) { 727 __atomic_store(&__a->__a_value, &__val, 728 __to_gcc_order(__order)); 729} 730 731template <typename _Tp> 732_LIBCPP_INLINE_VISIBILITY 733void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, 734 memory_order __order) { 735 __atomic_store(&__a->__a_value, &__val, 736 __to_gcc_order(__order)); 737} 738 739template <typename _Tp> 740_LIBCPP_INLINE_VISIBILITY 741_Tp __cxx_atomic_load(const volatile __cxx_atomic_base_impl<_Tp>* __a, 742 memory_order __order) { 743 _Tp __ret; 744 __atomic_load(&__a->__a_value, &__ret, 745 __to_gcc_order(__order)); 746 return __ret; 747} 748 749template <typename _Tp> 750_LIBCPP_INLINE_VISIBILITY 751_Tp __cxx_atomic_load(const __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) { 752 _Tp __ret; 753 __atomic_load(&__a->__a_value, &__ret, 754 __to_gcc_order(__order)); 755 return __ret; 756} 757 758template <typename _Tp> 759_LIBCPP_INLINE_VISIBILITY 760_Tp __cxx_atomic_exchange(volatile __cxx_atomic_base_impl<_Tp>* __a, 761 _Tp __value, memory_order __order) { 762 _Tp __ret; 763 __atomic_exchange(&__a->__a_value, &__value, &__ret, 764 __to_gcc_order(__order)); 765 return __ret; 766} 767 768template <typename _Tp> 769_LIBCPP_INLINE_VISIBILITY 770_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, 771 memory_order __order) { 772 _Tp __ret; 773 __atomic_exchange(&__a->__a_value, &__value, &__ret, 774 __to_gcc_order(__order)); 775 return __ret; 776} 777 778template <typename _Tp> 779_LIBCPP_INLINE_VISIBILITY 780bool __cxx_atomic_compare_exchange_strong( 781 volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, 782 memory_order __success, memory_order __failure) { 783 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 784 false, 785 __to_gcc_order(__success), 786 __to_gcc_failure_order(__failure)); 787} 788 789template <typename _Tp> 790_LIBCPP_INLINE_VISIBILITY 791bool __cxx_atomic_compare_exchange_strong( 792 __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, 793 memory_order __failure) { 794 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 795 false, 796 __to_gcc_order(__success), 797 __to_gcc_failure_order(__failure)); 798} 799 800template <typename _Tp> 801_LIBCPP_INLINE_VISIBILITY 802bool __cxx_atomic_compare_exchange_weak( 803 volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, 804 memory_order __success, memory_order __failure) { 805 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 806 true, 807 __to_gcc_order(__success), 808 __to_gcc_failure_order(__failure)); 809} 810 811template <typename _Tp> 812_LIBCPP_INLINE_VISIBILITY 813bool __cxx_atomic_compare_exchange_weak( 814 __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, 815 memory_order __failure) { 816 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 817 true, 818 __to_gcc_order(__success), 819 __to_gcc_failure_order(__failure)); 820} 821 822template <typename _Tp> 823struct __skip_amt { enum {value = 1}; }; 824 825template <typename _Tp> 826struct __skip_amt<_Tp*> { enum {value = sizeof(_Tp)}; }; 827 828// FIXME: Haven't figured out what the spec says about using arrays with 829// atomic_fetch_add. Force a failure rather than creating bad behavior. 830template <typename _Tp> 831struct __skip_amt<_Tp[]> { }; 832template <typename _Tp, int n> 833struct __skip_amt<_Tp[n]> { }; 834 835template <typename _Tp, typename _Td> 836_LIBCPP_INLINE_VISIBILITY 837_Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_base_impl<_Tp>* __a, 838 _Td __delta, memory_order __order) { 839 return __atomic_fetch_add(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 840 __to_gcc_order(__order)); 841} 842 843template <typename _Tp, typename _Td> 844_LIBCPP_INLINE_VISIBILITY 845_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, 846 memory_order __order) { 847 return __atomic_fetch_add(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 848 __to_gcc_order(__order)); 849} 850 851template <typename _Tp, typename _Td> 852_LIBCPP_INLINE_VISIBILITY 853_Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_base_impl<_Tp>* __a, 854 _Td __delta, memory_order __order) { 855 return __atomic_fetch_sub(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 856 __to_gcc_order(__order)); 857} 858 859template <typename _Tp, typename _Td> 860_LIBCPP_INLINE_VISIBILITY 861_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, 862 memory_order __order) { 863 return __atomic_fetch_sub(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 864 __to_gcc_order(__order)); 865} 866 867template <typename _Tp> 868_LIBCPP_INLINE_VISIBILITY 869_Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_base_impl<_Tp>* __a, 870 _Tp __pattern, memory_order __order) { 871 return __atomic_fetch_and(&__a->__a_value, __pattern, 872 __to_gcc_order(__order)); 873} 874 875template <typename _Tp> 876_LIBCPP_INLINE_VISIBILITY 877_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, 878 _Tp __pattern, memory_order __order) { 879 return __atomic_fetch_and(&__a->__a_value, __pattern, 880 __to_gcc_order(__order)); 881} 882 883template <typename _Tp> 884_LIBCPP_INLINE_VISIBILITY 885_Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_base_impl<_Tp>* __a, 886 _Tp __pattern, memory_order __order) { 887 return __atomic_fetch_or(&__a->__a_value, __pattern, 888 __to_gcc_order(__order)); 889} 890 891template <typename _Tp> 892_LIBCPP_INLINE_VISIBILITY 893_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, 894 memory_order __order) { 895 return __atomic_fetch_or(&__a->__a_value, __pattern, 896 __to_gcc_order(__order)); 897} 898 899template <typename _Tp> 900_LIBCPP_INLINE_VISIBILITY 901_Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_base_impl<_Tp>* __a, 902 _Tp __pattern, memory_order __order) { 903 return __atomic_fetch_xor(&__a->__a_value, __pattern, 904 __to_gcc_order(__order)); 905} 906 907template <typename _Tp> 908_LIBCPP_INLINE_VISIBILITY 909_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, 910 memory_order __order) { 911 return __atomic_fetch_xor(&__a->__a_value, __pattern, 912 __to_gcc_order(__order)); 913} 914 915#define __cxx_atomic_is_lock_free(__s) __atomic_is_lock_free(__s, 0) 916 917#elif defined(_LIBCPP_HAS_C_ATOMIC_IMP) 918 919template <typename _Tp> 920struct __cxx_atomic_base_impl { 921 922 _LIBCPP_INLINE_VISIBILITY 923#ifndef _LIBCPP_CXX03_LANG 924 __cxx_atomic_base_impl() _NOEXCEPT = default; 925#else 926 __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {} 927#endif // _LIBCPP_CXX03_LANG 928 _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT 929 : __a_value(value) {} 930 _LIBCPP_DISABLE_EXTENSION_WARNING _Atomic(_Tp) __a_value; 931}; 932 933#define __cxx_atomic_is_lock_free(__s) __c11_atomic_is_lock_free(__s) 934 935_LIBCPP_INLINE_VISIBILITY inline 936void __cxx_atomic_thread_fence(memory_order __order) _NOEXCEPT { 937 __c11_atomic_thread_fence(static_cast<__memory_order_underlying_t>(__order)); 938} 939 940_LIBCPP_INLINE_VISIBILITY inline 941void __cxx_atomic_signal_fence(memory_order __order) _NOEXCEPT { 942 __c11_atomic_signal_fence(static_cast<__memory_order_underlying_t>(__order)); 943} 944 945template<class _Tp> 946_LIBCPP_INLINE_VISIBILITY 947void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val) _NOEXCEPT { 948 __c11_atomic_init(&__a->__a_value, __val); 949} 950template<class _Tp> 951_LIBCPP_INLINE_VISIBILITY 952void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val) _NOEXCEPT { 953 __c11_atomic_init(&__a->__a_value, __val); 954} 955 956template<class _Tp> 957_LIBCPP_INLINE_VISIBILITY 958void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT { 959 __c11_atomic_store(&__a->__a_value, __val, static_cast<__memory_order_underlying_t>(__order)); 960} 961template<class _Tp> 962_LIBCPP_INLINE_VISIBILITY 963void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val, memory_order __order) _NOEXCEPT { 964 __c11_atomic_store(&__a->__a_value, __val, static_cast<__memory_order_underlying_t>(__order)); 965} 966 967template<class _Tp> 968_LIBCPP_INLINE_VISIBILITY 969_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const volatile* __a, memory_order __order) _NOEXCEPT { 970 using __ptr_type = typename remove_const<decltype(__a->__a_value)>::type*; 971 return __c11_atomic_load(const_cast<__ptr_type>(&__a->__a_value), static_cast<__memory_order_underlying_t>(__order)); 972} 973template<class _Tp> 974_LIBCPP_INLINE_VISIBILITY 975_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const* __a, memory_order __order) _NOEXCEPT { 976 using __ptr_type = typename remove_const<decltype(__a->__a_value)>::type*; 977 return __c11_atomic_load(const_cast<__ptr_type>(&__a->__a_value), static_cast<__memory_order_underlying_t>(__order)); 978} 979 980template<class _Tp> 981_LIBCPP_INLINE_VISIBILITY 982_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT { 983 return __c11_atomic_exchange(&__a->__a_value, __value, static_cast<__memory_order_underlying_t>(__order)); 984} 985template<class _Tp> 986_LIBCPP_INLINE_VISIBILITY 987_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> * __a, _Tp __value, memory_order __order) _NOEXCEPT { 988 return __c11_atomic_exchange(&__a->__a_value, __value, static_cast<__memory_order_underlying_t>(__order)); 989} 990 991template<class _Tp> 992_LIBCPP_INLINE_VISIBILITY 993bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 994 return __c11_atomic_compare_exchange_strong(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__failure)); 995} 996template<class _Tp> 997_LIBCPP_INLINE_VISIBILITY 998bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 999 return __c11_atomic_compare_exchange_strong(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__failure)); 1000} 1001 1002template<class _Tp> 1003_LIBCPP_INLINE_VISIBILITY 1004bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 1005 return __c11_atomic_compare_exchange_weak(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__failure)); 1006} 1007template<class _Tp> 1008_LIBCPP_INLINE_VISIBILITY 1009bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 1010 return __c11_atomic_compare_exchange_weak(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__failure)); 1011} 1012 1013template<class _Tp> 1014_LIBCPP_INLINE_VISIBILITY 1015_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1016 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1017} 1018template<class _Tp> 1019_LIBCPP_INLINE_VISIBILITY 1020_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1021 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1022} 1023 1024template<class _Tp> 1025_LIBCPP_INLINE_VISIBILITY 1026_Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1027 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1028} 1029template<class _Tp> 1030_LIBCPP_INLINE_VISIBILITY 1031_Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1032 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1033} 1034 1035template<class _Tp> 1036_LIBCPP_INLINE_VISIBILITY 1037_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1038 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1039} 1040template<class _Tp> 1041_LIBCPP_INLINE_VISIBILITY 1042_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1043 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1044} 1045template<class _Tp> 1046_LIBCPP_INLINE_VISIBILITY 1047_Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1048 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1049} 1050template<class _Tp> 1051_LIBCPP_INLINE_VISIBILITY 1052_Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1053 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1054} 1055 1056template<class _Tp> 1057_LIBCPP_INLINE_VISIBILITY 1058_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1059 return __c11_atomic_fetch_and(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1060} 1061template<class _Tp> 1062_LIBCPP_INLINE_VISIBILITY 1063_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1064 return __c11_atomic_fetch_and(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1065} 1066 1067template<class _Tp> 1068_LIBCPP_INLINE_VISIBILITY 1069_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1070 return __c11_atomic_fetch_or(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1071} 1072template<class _Tp> 1073_LIBCPP_INLINE_VISIBILITY 1074_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1075 return __c11_atomic_fetch_or(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1076} 1077 1078template<class _Tp> 1079_LIBCPP_INLINE_VISIBILITY 1080_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1081 return __c11_atomic_fetch_xor(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1082} 1083template<class _Tp> 1084_LIBCPP_INLINE_VISIBILITY 1085_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1086 return __c11_atomic_fetch_xor(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1087} 1088 1089#endif // _LIBCPP_HAS_GCC_ATOMIC_IMP, _LIBCPP_HAS_C_ATOMIC_IMP 1090 1091template <class _Tp> 1092_LIBCPP_INLINE_VISIBILITY 1093_Tp kill_dependency(_Tp __y) _NOEXCEPT 1094{ 1095 return __y; 1096} 1097 1098#if defined(__CLANG_ATOMIC_BOOL_LOCK_FREE) 1099# define ATOMIC_BOOL_LOCK_FREE __CLANG_ATOMIC_BOOL_LOCK_FREE 1100# define ATOMIC_CHAR_LOCK_FREE __CLANG_ATOMIC_CHAR_LOCK_FREE 1101# define ATOMIC_CHAR16_T_LOCK_FREE __CLANG_ATOMIC_CHAR16_T_LOCK_FREE 1102# define ATOMIC_CHAR32_T_LOCK_FREE __CLANG_ATOMIC_CHAR32_T_LOCK_FREE 1103# define ATOMIC_WCHAR_T_LOCK_FREE __CLANG_ATOMIC_WCHAR_T_LOCK_FREE 1104# define ATOMIC_SHORT_LOCK_FREE __CLANG_ATOMIC_SHORT_LOCK_FREE 1105# define ATOMIC_INT_LOCK_FREE __CLANG_ATOMIC_INT_LOCK_FREE 1106# define ATOMIC_LONG_LOCK_FREE __CLANG_ATOMIC_LONG_LOCK_FREE 1107# define ATOMIC_LLONG_LOCK_FREE __CLANG_ATOMIC_LLONG_LOCK_FREE 1108# define ATOMIC_POINTER_LOCK_FREE __CLANG_ATOMIC_POINTER_LOCK_FREE 1109#elif defined(__GCC_ATOMIC_BOOL_LOCK_FREE) 1110# define ATOMIC_BOOL_LOCK_FREE __GCC_ATOMIC_BOOL_LOCK_FREE 1111# define ATOMIC_CHAR_LOCK_FREE __GCC_ATOMIC_CHAR_LOCK_FREE 1112# define ATOMIC_CHAR16_T_LOCK_FREE __GCC_ATOMIC_CHAR16_T_LOCK_FREE 1113# define ATOMIC_CHAR32_T_LOCK_FREE __GCC_ATOMIC_CHAR32_T_LOCK_FREE 1114# define ATOMIC_WCHAR_T_LOCK_FREE __GCC_ATOMIC_WCHAR_T_LOCK_FREE 1115# define ATOMIC_SHORT_LOCK_FREE __GCC_ATOMIC_SHORT_LOCK_FREE 1116# define ATOMIC_INT_LOCK_FREE __GCC_ATOMIC_INT_LOCK_FREE 1117# define ATOMIC_LONG_LOCK_FREE __GCC_ATOMIC_LONG_LOCK_FREE 1118# define ATOMIC_LLONG_LOCK_FREE __GCC_ATOMIC_LLONG_LOCK_FREE 1119# define ATOMIC_POINTER_LOCK_FREE __GCC_ATOMIC_POINTER_LOCK_FREE 1120#endif 1121 1122#ifdef _LIBCPP_ATOMIC_ONLY_USE_BUILTINS 1123 1124template<typename _Tp> 1125struct __cxx_atomic_lock_impl { 1126 1127 _LIBCPP_INLINE_VISIBILITY 1128 __cxx_atomic_lock_impl() _NOEXCEPT 1129 : __a_value(), __a_lock(0) {} 1130 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR explicit 1131 __cxx_atomic_lock_impl(_Tp value) _NOEXCEPT 1132 : __a_value(value), __a_lock(0) {} 1133 1134 _Tp __a_value; 1135 mutable __cxx_atomic_base_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_lock; 1136 1137 _LIBCPP_INLINE_VISIBILITY void __lock() const volatile { 1138 while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire)) 1139 /*spin*/; 1140 } 1141 _LIBCPP_INLINE_VISIBILITY void __lock() const { 1142 while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire)) 1143 /*spin*/; 1144 } 1145 _LIBCPP_INLINE_VISIBILITY void __unlock() const volatile { 1146 __cxx_atomic_store(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(false), memory_order_release); 1147 } 1148 _LIBCPP_INLINE_VISIBILITY void __unlock() const { 1149 __cxx_atomic_store(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(false), memory_order_release); 1150 } 1151 _LIBCPP_INLINE_VISIBILITY _Tp __read() const volatile { 1152 __lock(); 1153 _Tp __old; 1154 __cxx_atomic_assign_volatile(__old, __a_value); 1155 __unlock(); 1156 return __old; 1157 } 1158 _LIBCPP_INLINE_VISIBILITY _Tp __read() const { 1159 __lock(); 1160 _Tp __old = __a_value; 1161 __unlock(); 1162 return __old; 1163 } 1164}; 1165 1166template <typename _Tp> 1167_LIBCPP_INLINE_VISIBILITY 1168void __cxx_atomic_init(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __val) { 1169 __cxx_atomic_assign_volatile(__a->__a_value, __val); 1170} 1171template <typename _Tp> 1172_LIBCPP_INLINE_VISIBILITY 1173void __cxx_atomic_init(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __val) { 1174 __a->__a_value = __val; 1175} 1176 1177template <typename _Tp> 1178_LIBCPP_INLINE_VISIBILITY 1179void __cxx_atomic_store(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __val, memory_order) { 1180 __a->__lock(); 1181 __cxx_atomic_assign_volatile(__a->__a_value, __val); 1182 __a->__unlock(); 1183} 1184template <typename _Tp> 1185_LIBCPP_INLINE_VISIBILITY 1186void __cxx_atomic_store(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __val, memory_order) { 1187 __a->__lock(); 1188 __a->__a_value = __val; 1189 __a->__unlock(); 1190} 1191 1192template <typename _Tp> 1193_LIBCPP_INLINE_VISIBILITY 1194_Tp __cxx_atomic_load(const volatile __cxx_atomic_lock_impl<_Tp>* __a, memory_order) { 1195 return __a->__read(); 1196} 1197template <typename _Tp> 1198_LIBCPP_INLINE_VISIBILITY 1199_Tp __cxx_atomic_load(const __cxx_atomic_lock_impl<_Tp>* __a, memory_order) { 1200 return __a->__read(); 1201} 1202 1203template <typename _Tp> 1204_LIBCPP_INLINE_VISIBILITY 1205_Tp __cxx_atomic_exchange(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) { 1206 __a->__lock(); 1207 _Tp __old; 1208 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1209 __cxx_atomic_assign_volatile(__a->__a_value, __value); 1210 __a->__unlock(); 1211 return __old; 1212} 1213template <typename _Tp> 1214_LIBCPP_INLINE_VISIBILITY 1215_Tp __cxx_atomic_exchange(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) { 1216 __a->__lock(); 1217 _Tp __old = __a->__a_value; 1218 __a->__a_value = __value; 1219 __a->__unlock(); 1220 return __old; 1221} 1222 1223template <typename _Tp> 1224_LIBCPP_INLINE_VISIBILITY 1225bool __cxx_atomic_compare_exchange_strong(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1226 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1227 __a->__lock(); 1228 _Tp __temp; 1229 __cxx_atomic_assign_volatile(__temp, __a->__a_value); 1230 bool __ret = __temp == *__expected; 1231 if(__ret) 1232 __cxx_atomic_assign_volatile(__a->__a_value, __value); 1233 else 1234 __cxx_atomic_assign_volatile(*__expected, __a->__a_value); 1235 __a->__unlock(); 1236 return __ret; 1237} 1238template <typename _Tp> 1239_LIBCPP_INLINE_VISIBILITY 1240bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_lock_impl<_Tp>* __a, 1241 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1242 __a->__lock(); 1243 bool __ret = __a->__a_value == *__expected; 1244 if(__ret) 1245 __a->__a_value = __value; 1246 else 1247 *__expected = __a->__a_value; 1248 __a->__unlock(); 1249 return __ret; 1250} 1251 1252template <typename _Tp> 1253_LIBCPP_INLINE_VISIBILITY 1254bool __cxx_atomic_compare_exchange_weak(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1255 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1256 __a->__lock(); 1257 _Tp __temp; 1258 __cxx_atomic_assign_volatile(__temp, __a->__a_value); 1259 bool __ret = __temp == *__expected; 1260 if(__ret) 1261 __cxx_atomic_assign_volatile(__a->__a_value, __value); 1262 else 1263 __cxx_atomic_assign_volatile(*__expected, __a->__a_value); 1264 __a->__unlock(); 1265 return __ret; 1266} 1267template <typename _Tp> 1268_LIBCPP_INLINE_VISIBILITY 1269bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_lock_impl<_Tp>* __a, 1270 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1271 __a->__lock(); 1272 bool __ret = __a->__a_value == *__expected; 1273 if(__ret) 1274 __a->__a_value = __value; 1275 else 1276 *__expected = __a->__a_value; 1277 __a->__unlock(); 1278 return __ret; 1279} 1280 1281template <typename _Tp, typename _Td> 1282_LIBCPP_INLINE_VISIBILITY 1283_Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1284 _Td __delta, memory_order) { 1285 __a->__lock(); 1286 _Tp __old; 1287 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1288 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old + __delta)); 1289 __a->__unlock(); 1290 return __old; 1291} 1292template <typename _Tp, typename _Td> 1293_LIBCPP_INLINE_VISIBILITY 1294_Tp __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp>* __a, 1295 _Td __delta, memory_order) { 1296 __a->__lock(); 1297 _Tp __old = __a->__a_value; 1298 __a->__a_value += __delta; 1299 __a->__unlock(); 1300 return __old; 1301} 1302 1303template <typename _Tp, typename _Td> 1304_LIBCPP_INLINE_VISIBILITY 1305_Tp* __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp*>* __a, 1306 ptrdiff_t __delta, memory_order) { 1307 __a->__lock(); 1308 _Tp* __old; 1309 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1310 __cxx_atomic_assign_volatile(__a->__a_value, __old + __delta); 1311 __a->__unlock(); 1312 return __old; 1313} 1314template <typename _Tp, typename _Td> 1315_LIBCPP_INLINE_VISIBILITY 1316_Tp* __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp*>* __a, 1317 ptrdiff_t __delta, memory_order) { 1318 __a->__lock(); 1319 _Tp* __old = __a->__a_value; 1320 __a->__a_value += __delta; 1321 __a->__unlock(); 1322 return __old; 1323} 1324 1325template <typename _Tp, typename _Td> 1326_LIBCPP_INLINE_VISIBILITY 1327_Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1328 _Td __delta, memory_order) { 1329 __a->__lock(); 1330 _Tp __old; 1331 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1332 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old - __delta)); 1333 __a->__unlock(); 1334 return __old; 1335} 1336template <typename _Tp, typename _Td> 1337_LIBCPP_INLINE_VISIBILITY 1338_Tp __cxx_atomic_fetch_sub(__cxx_atomic_lock_impl<_Tp>* __a, 1339 _Td __delta, memory_order) { 1340 __a->__lock(); 1341 _Tp __old = __a->__a_value; 1342 __a->__a_value -= __delta; 1343 __a->__unlock(); 1344 return __old; 1345} 1346 1347template <typename _Tp> 1348_LIBCPP_INLINE_VISIBILITY 1349_Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1350 _Tp __pattern, memory_order) { 1351 __a->__lock(); 1352 _Tp __old; 1353 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1354 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old & __pattern)); 1355 __a->__unlock(); 1356 return __old; 1357} 1358template <typename _Tp> 1359_LIBCPP_INLINE_VISIBILITY 1360_Tp __cxx_atomic_fetch_and(__cxx_atomic_lock_impl<_Tp>* __a, 1361 _Tp __pattern, memory_order) { 1362 __a->__lock(); 1363 _Tp __old = __a->__a_value; 1364 __a->__a_value &= __pattern; 1365 __a->__unlock(); 1366 return __old; 1367} 1368 1369template <typename _Tp> 1370_LIBCPP_INLINE_VISIBILITY 1371_Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1372 _Tp __pattern, memory_order) { 1373 __a->__lock(); 1374 _Tp __old; 1375 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1376 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old | __pattern)); 1377 __a->__unlock(); 1378 return __old; 1379} 1380template <typename _Tp> 1381_LIBCPP_INLINE_VISIBILITY 1382_Tp __cxx_atomic_fetch_or(__cxx_atomic_lock_impl<_Tp>* __a, 1383 _Tp __pattern, memory_order) { 1384 __a->__lock(); 1385 _Tp __old = __a->__a_value; 1386 __a->__a_value |= __pattern; 1387 __a->__unlock(); 1388 return __old; 1389} 1390 1391template <typename _Tp> 1392_LIBCPP_INLINE_VISIBILITY 1393_Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1394 _Tp __pattern, memory_order) { 1395 __a->__lock(); 1396 _Tp __old; 1397 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1398 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old ^ __pattern)); 1399 __a->__unlock(); 1400 return __old; 1401} 1402template <typename _Tp> 1403_LIBCPP_INLINE_VISIBILITY 1404_Tp __cxx_atomic_fetch_xor(__cxx_atomic_lock_impl<_Tp>* __a, 1405 _Tp __pattern, memory_order) { 1406 __a->__lock(); 1407 _Tp __old = __a->__a_value; 1408 __a->__a_value ^= __pattern; 1409 __a->__unlock(); 1410 return __old; 1411} 1412 1413#ifdef __cpp_lib_atomic_is_always_lock_free 1414 1415template<typename _Tp> struct __cxx_is_always_lock_free { 1416 enum { __value = __atomic_always_lock_free(sizeof(_Tp), 0) }; }; 1417 1418#else 1419 1420template<typename _Tp> struct __cxx_is_always_lock_free { enum { __value = false }; }; 1421// Implementations must match the C ATOMIC_*_LOCK_FREE macro values. 1422template<> struct __cxx_is_always_lock_free<bool> { enum { __value = 2 == ATOMIC_BOOL_LOCK_FREE }; }; 1423template<> struct __cxx_is_always_lock_free<char> { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; }; 1424template<> struct __cxx_is_always_lock_free<signed char> { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; }; 1425template<> struct __cxx_is_always_lock_free<unsigned char> { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; }; 1426template<> struct __cxx_is_always_lock_free<char16_t> { enum { __value = 2 == ATOMIC_CHAR16_T_LOCK_FREE }; }; 1427template<> struct __cxx_is_always_lock_free<char32_t> { enum { __value = 2 == ATOMIC_CHAR32_T_LOCK_FREE }; }; 1428template<> struct __cxx_is_always_lock_free<wchar_t> { enum { __value = 2 == ATOMIC_WCHAR_T_LOCK_FREE }; }; 1429template<> struct __cxx_is_always_lock_free<short> { enum { __value = 2 == ATOMIC_SHORT_LOCK_FREE }; }; 1430template<> struct __cxx_is_always_lock_free<unsigned short> { enum { __value = 2 == ATOMIC_SHORT_LOCK_FREE }; }; 1431template<> struct __cxx_is_always_lock_free<int> { enum { __value = 2 == ATOMIC_INT_LOCK_FREE }; }; 1432template<> struct __cxx_is_always_lock_free<unsigned int> { enum { __value = 2 == ATOMIC_INT_LOCK_FREE }; }; 1433template<> struct __cxx_is_always_lock_free<long> { enum { __value = 2 == ATOMIC_LONG_LOCK_FREE }; }; 1434template<> struct __cxx_is_always_lock_free<unsigned long> { enum { __value = 2 == ATOMIC_LONG_LOCK_FREE }; }; 1435template<> struct __cxx_is_always_lock_free<long long> { enum { __value = 2 == ATOMIC_LLONG_LOCK_FREE }; }; 1436template<> struct __cxx_is_always_lock_free<unsigned long long> { enum { __value = 2 == ATOMIC_LLONG_LOCK_FREE }; }; 1437template<typename _Tp> struct __cxx_is_always_lock_free<_Tp*> { enum { __value = 2 == ATOMIC_POINTER_LOCK_FREE }; }; 1438template<> struct __cxx_is_always_lock_free<std::nullptr_t> { enum { __value = 2 == ATOMIC_POINTER_LOCK_FREE }; }; 1439 1440#endif //__cpp_lib_atomic_is_always_lock_free 1441 1442template <typename _Tp, 1443 typename _Base = typename conditional<__cxx_is_always_lock_free<_Tp>::__value, 1444 __cxx_atomic_base_impl<_Tp>, 1445 __cxx_atomic_lock_impl<_Tp> >::type> 1446#else 1447template <typename _Tp, 1448 typename _Base = __cxx_atomic_base_impl<_Tp> > 1449#endif //_LIBCPP_ATOMIC_ONLY_USE_BUILTINS 1450struct __cxx_atomic_impl : public _Base { 1451 1452#if _GNUC_VER >= 501 1453 static_assert(is_trivially_copyable<_Tp>::value, 1454 "std::atomic<Tp> requires that 'Tp' be a trivially copyable type"); 1455#endif 1456 1457 _LIBCPP_INLINE_VISIBILITY __cxx_atomic_impl() _NOEXCEPT _LIBCPP_DEFAULT 1458 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR explicit __cxx_atomic_impl(_Tp value) _NOEXCEPT 1459 : _Base(value) {} 1460}; 1461 1462#ifdef __linux__ 1463 using __cxx_contention_t = int32_t; 1464#else 1465 using __cxx_contention_t = int64_t; 1466#endif //__linux__ 1467 1468#if _LIBCPP_STD_VER >= 11 1469 1470using __cxx_atomic_contention_t = __cxx_atomic_impl<__cxx_contention_t>; 1471 1472#ifndef _LIBCPP_HAS_NO_PLATFORM_WAIT 1473 1474_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(void const volatile*); 1475_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(void const volatile*); 1476_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(void const volatile*); 1477_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(void const volatile*, __cxx_contention_t); 1478 1479_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(__cxx_atomic_contention_t const volatile*); 1480_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(__cxx_atomic_contention_t const volatile*); 1481_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile*); 1482_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(__cxx_atomic_contention_t const volatile*, __cxx_contention_t); 1483 1484template <class _Atp, class _Fn> 1485struct __libcpp_atomic_wait_backoff_impl { 1486 _Atp* __a; 1487 _Fn __test_fn; 1488 _LIBCPP_AVAILABILITY_SYNC 1489 _LIBCPP_INLINE_VISIBILITY bool operator()(chrono::nanoseconds __elapsed) const 1490 { 1491 if(__elapsed > chrono::microseconds(64)) 1492 { 1493 auto const __monitor = __libcpp_atomic_monitor(__a); 1494 if(__test_fn()) 1495 return true; 1496 __libcpp_atomic_wait(__a, __monitor); 1497 } 1498 else if(__elapsed > chrono::microseconds(4)) 1499 __libcpp_thread_yield(); 1500 else 1501 ; // poll 1502 return false; 1503 } 1504}; 1505 1506template <class _Atp, class _Fn> 1507_LIBCPP_AVAILABILITY_SYNC 1508_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp* __a, _Fn && __test_fn) 1509{ 1510 __libcpp_atomic_wait_backoff_impl<_Atp, typename decay<_Fn>::type> __backoff_fn = {__a, __test_fn}; 1511 return __libcpp_thread_poll_with_backoff(__test_fn, __backoff_fn); 1512} 1513 1514#else // _LIBCPP_HAS_NO_PLATFORM_WAIT 1515 1516template <class _Tp> 1517_LIBCPP_INLINE_VISIBILITY void __cxx_atomic_notify_all(__cxx_atomic_impl<_Tp> const volatile*) { } 1518template <class _Tp> 1519_LIBCPP_INLINE_VISIBILITY void __cxx_atomic_notify_one(__cxx_atomic_impl<_Tp> const volatile*) { } 1520template <class _Atp, class _Fn> 1521_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp*, _Fn && __test_fn) 1522{ 1523 return __libcpp_thread_poll_with_backoff(__test_fn, __libcpp_timed_backoff_policy()); 1524} 1525 1526#endif // _LIBCPP_HAS_NO_PLATFORM_WAIT 1527 1528template <class _Atp, class _Tp> 1529struct __cxx_atomic_wait_test_fn_impl { 1530 _Atp* __a; 1531 _Tp __val; 1532 memory_order __order; 1533 _LIBCPP_INLINE_VISIBILITY bool operator()() const 1534 { 1535 return !__cxx_nonatomic_compare_equal(__cxx_atomic_load(__a, __order), __val); 1536 } 1537}; 1538 1539template <class _Atp, class _Tp> 1540_LIBCPP_AVAILABILITY_SYNC 1541_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp* __a, _Tp const __val, memory_order __order) 1542{ 1543 __cxx_atomic_wait_test_fn_impl<_Atp, _Tp> __test_fn = {__a, __val, __order}; 1544 return __cxx_atomic_wait(__a, __test_fn); 1545} 1546 1547#endif //_LIBCPP_STD_VER >= 11 1548 1549// general atomic<T> 1550 1551template <class _Tp, bool = is_integral<_Tp>::value && !is_same<_Tp, bool>::value> 1552struct __atomic_base // false 1553{ 1554 mutable __cxx_atomic_impl<_Tp> __a_; 1555 1556#if defined(__cpp_lib_atomic_is_always_lock_free) 1557 static _LIBCPP_CONSTEXPR bool is_always_lock_free = __atomic_always_lock_free(sizeof(__a_), 0); 1558#endif 1559 1560 _LIBCPP_INLINE_VISIBILITY 1561 bool is_lock_free() const volatile _NOEXCEPT 1562 {return __cxx_atomic_is_lock_free(sizeof(_Tp));} 1563 _LIBCPP_INLINE_VISIBILITY 1564 bool is_lock_free() const _NOEXCEPT 1565 {return static_cast<__atomic_base const volatile*>(this)->is_lock_free();} 1566 _LIBCPP_INLINE_VISIBILITY 1567 void store(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1568 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1569 {__cxx_atomic_store(&__a_, __d, __m);} 1570 _LIBCPP_INLINE_VISIBILITY 1571 void store(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1572 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1573 {__cxx_atomic_store(&__a_, __d, __m);} 1574 _LIBCPP_INLINE_VISIBILITY 1575 _Tp load(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 1576 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 1577 {return __cxx_atomic_load(&__a_, __m);} 1578 _LIBCPP_INLINE_VISIBILITY 1579 _Tp load(memory_order __m = memory_order_seq_cst) const _NOEXCEPT 1580 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 1581 {return __cxx_atomic_load(&__a_, __m);} 1582 _LIBCPP_INLINE_VISIBILITY 1583 operator _Tp() const volatile _NOEXCEPT {return load();} 1584 _LIBCPP_INLINE_VISIBILITY 1585 operator _Tp() const _NOEXCEPT {return load();} 1586 _LIBCPP_INLINE_VISIBILITY 1587 _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1588 {return __cxx_atomic_exchange(&__a_, __d, __m);} 1589 _LIBCPP_INLINE_VISIBILITY 1590 _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1591 {return __cxx_atomic_exchange(&__a_, __d, __m);} 1592 _LIBCPP_INLINE_VISIBILITY 1593 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1594 memory_order __s, memory_order __f) volatile _NOEXCEPT 1595 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1596 {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);} 1597 _LIBCPP_INLINE_VISIBILITY 1598 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1599 memory_order __s, memory_order __f) _NOEXCEPT 1600 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1601 {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);} 1602 _LIBCPP_INLINE_VISIBILITY 1603 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1604 memory_order __s, memory_order __f) volatile _NOEXCEPT 1605 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1606 {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);} 1607 _LIBCPP_INLINE_VISIBILITY 1608 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1609 memory_order __s, memory_order __f) _NOEXCEPT 1610 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1611 {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);} 1612 _LIBCPP_INLINE_VISIBILITY 1613 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1614 memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1615 {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);} 1616 _LIBCPP_INLINE_VISIBILITY 1617 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1618 memory_order __m = memory_order_seq_cst) _NOEXCEPT 1619 {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);} 1620 _LIBCPP_INLINE_VISIBILITY 1621 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1622 memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1623 {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);} 1624 _LIBCPP_INLINE_VISIBILITY 1625 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1626 memory_order __m = memory_order_seq_cst) _NOEXCEPT 1627 {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);} 1628 1629 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 1630 {__cxx_atomic_wait(&__a_, __v, __m);} 1631 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT 1632 {__cxx_atomic_wait(&__a_, __v, __m);} 1633 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_one() volatile _NOEXCEPT 1634 {__cxx_atomic_notify_one(&__a_);} 1635 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_one() _NOEXCEPT 1636 {__cxx_atomic_notify_one(&__a_);} 1637 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_all() volatile _NOEXCEPT 1638 {__cxx_atomic_notify_all(&__a_);} 1639 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_all() _NOEXCEPT 1640 {__cxx_atomic_notify_all(&__a_);} 1641 1642 _LIBCPP_INLINE_VISIBILITY 1643 __atomic_base() _NOEXCEPT _LIBCPP_DEFAULT 1644 1645 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR 1646 __atomic_base(_Tp __d) _NOEXCEPT : __a_(__d) {} 1647 1648#ifndef _LIBCPP_CXX03_LANG 1649 __atomic_base(const __atomic_base&) = delete; 1650 __atomic_base& operator=(const __atomic_base&) = delete; 1651 __atomic_base& operator=(const __atomic_base&) volatile = delete; 1652#else 1653private: 1654 _LIBCPP_INLINE_VISIBILITY 1655 __atomic_base(const __atomic_base&); 1656 _LIBCPP_INLINE_VISIBILITY 1657 __atomic_base& operator=(const __atomic_base&); 1658 _LIBCPP_INLINE_VISIBILITY 1659 __atomic_base& operator=(const __atomic_base&) volatile; 1660#endif 1661}; 1662 1663#if defined(__cpp_lib_atomic_is_always_lock_free) 1664template <class _Tp, bool __b> 1665_LIBCPP_CONSTEXPR bool __atomic_base<_Tp, __b>::is_always_lock_free; 1666#endif 1667 1668// atomic<Integral> 1669 1670template <class _Tp> 1671struct __atomic_base<_Tp, true> 1672 : public __atomic_base<_Tp, false> 1673{ 1674 typedef __atomic_base<_Tp, false> __base; 1675 _LIBCPP_INLINE_VISIBILITY 1676 __atomic_base() _NOEXCEPT _LIBCPP_DEFAULT 1677 _LIBCPP_INLINE_VISIBILITY 1678 _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __base(__d) {} 1679 1680 _LIBCPP_INLINE_VISIBILITY 1681 _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1682 {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);} 1683 _LIBCPP_INLINE_VISIBILITY 1684 _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1685 {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);} 1686 _LIBCPP_INLINE_VISIBILITY 1687 _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1688 {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);} 1689 _LIBCPP_INLINE_VISIBILITY 1690 _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1691 {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);} 1692 _LIBCPP_INLINE_VISIBILITY 1693 _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1694 {return __cxx_atomic_fetch_and(&this->__a_, __op, __m);} 1695 _LIBCPP_INLINE_VISIBILITY 1696 _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1697 {return __cxx_atomic_fetch_and(&this->__a_, __op, __m);} 1698 _LIBCPP_INLINE_VISIBILITY 1699 _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1700 {return __cxx_atomic_fetch_or(&this->__a_, __op, __m);} 1701 _LIBCPP_INLINE_VISIBILITY 1702 _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1703 {return __cxx_atomic_fetch_or(&this->__a_, __op, __m);} 1704 _LIBCPP_INLINE_VISIBILITY 1705 _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1706 {return __cxx_atomic_fetch_xor(&this->__a_, __op, __m);} 1707 _LIBCPP_INLINE_VISIBILITY 1708 _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1709 {return __cxx_atomic_fetch_xor(&this->__a_, __op, __m);} 1710 1711 _LIBCPP_INLINE_VISIBILITY 1712 _Tp operator++(int) volatile _NOEXCEPT {return fetch_add(_Tp(1));} 1713 _LIBCPP_INLINE_VISIBILITY 1714 _Tp operator++(int) _NOEXCEPT {return fetch_add(_Tp(1));} 1715 _LIBCPP_INLINE_VISIBILITY 1716 _Tp operator--(int) volatile _NOEXCEPT {return fetch_sub(_Tp(1));} 1717 _LIBCPP_INLINE_VISIBILITY 1718 _Tp operator--(int) _NOEXCEPT {return fetch_sub(_Tp(1));} 1719 _LIBCPP_INLINE_VISIBILITY 1720 _Tp operator++() volatile _NOEXCEPT {return fetch_add(_Tp(1)) + _Tp(1);} 1721 _LIBCPP_INLINE_VISIBILITY 1722 _Tp operator++() _NOEXCEPT {return fetch_add(_Tp(1)) + _Tp(1);} 1723 _LIBCPP_INLINE_VISIBILITY 1724 _Tp operator--() volatile _NOEXCEPT {return fetch_sub(_Tp(1)) - _Tp(1);} 1725 _LIBCPP_INLINE_VISIBILITY 1726 _Tp operator--() _NOEXCEPT {return fetch_sub(_Tp(1)) - _Tp(1);} 1727 _LIBCPP_INLINE_VISIBILITY 1728 _Tp operator+=(_Tp __op) volatile _NOEXCEPT {return fetch_add(__op) + __op;} 1729 _LIBCPP_INLINE_VISIBILITY 1730 _Tp operator+=(_Tp __op) _NOEXCEPT {return fetch_add(__op) + __op;} 1731 _LIBCPP_INLINE_VISIBILITY 1732 _Tp operator-=(_Tp __op) volatile _NOEXCEPT {return fetch_sub(__op) - __op;} 1733 _LIBCPP_INLINE_VISIBILITY 1734 _Tp operator-=(_Tp __op) _NOEXCEPT {return fetch_sub(__op) - __op;} 1735 _LIBCPP_INLINE_VISIBILITY 1736 _Tp operator&=(_Tp __op) volatile _NOEXCEPT {return fetch_and(__op) & __op;} 1737 _LIBCPP_INLINE_VISIBILITY 1738 _Tp operator&=(_Tp __op) _NOEXCEPT {return fetch_and(__op) & __op;} 1739 _LIBCPP_INLINE_VISIBILITY 1740 _Tp operator|=(_Tp __op) volatile _NOEXCEPT {return fetch_or(__op) | __op;} 1741 _LIBCPP_INLINE_VISIBILITY 1742 _Tp operator|=(_Tp __op) _NOEXCEPT {return fetch_or(__op) | __op;} 1743 _LIBCPP_INLINE_VISIBILITY 1744 _Tp operator^=(_Tp __op) volatile _NOEXCEPT {return fetch_xor(__op) ^ __op;} 1745 _LIBCPP_INLINE_VISIBILITY 1746 _Tp operator^=(_Tp __op) _NOEXCEPT {return fetch_xor(__op) ^ __op;} 1747}; 1748 1749// atomic<T> 1750 1751template <class _Tp> 1752struct atomic 1753 : public __atomic_base<_Tp> 1754{ 1755 typedef __atomic_base<_Tp> __base; 1756 typedef _Tp value_type; 1757 _LIBCPP_INLINE_VISIBILITY 1758 atomic() _NOEXCEPT _LIBCPP_DEFAULT 1759 _LIBCPP_INLINE_VISIBILITY 1760 _LIBCPP_CONSTEXPR atomic(_Tp __d) _NOEXCEPT : __base(__d) {} 1761 1762 _LIBCPP_INLINE_VISIBILITY 1763 _Tp operator=(_Tp __d) volatile _NOEXCEPT 1764 {__base::store(__d); return __d;} 1765 _LIBCPP_INLINE_VISIBILITY 1766 _Tp operator=(_Tp __d) _NOEXCEPT 1767 {__base::store(__d); return __d;} 1768}; 1769 1770// atomic<T*> 1771 1772template <class _Tp> 1773struct atomic<_Tp*> 1774 : public __atomic_base<_Tp*> 1775{ 1776 typedef __atomic_base<_Tp*> __base; 1777 typedef _Tp* value_type; 1778 _LIBCPP_INLINE_VISIBILITY 1779 atomic() _NOEXCEPT _LIBCPP_DEFAULT 1780 _LIBCPP_INLINE_VISIBILITY 1781 _LIBCPP_CONSTEXPR atomic(_Tp* __d) _NOEXCEPT : __base(__d) {} 1782 1783 _LIBCPP_INLINE_VISIBILITY 1784 _Tp* operator=(_Tp* __d) volatile _NOEXCEPT 1785 {__base::store(__d); return __d;} 1786 _LIBCPP_INLINE_VISIBILITY 1787 _Tp* operator=(_Tp* __d) _NOEXCEPT 1788 {__base::store(__d); return __d;} 1789 1790 _LIBCPP_INLINE_VISIBILITY 1791 _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) 1792 volatile _NOEXCEPT 1793 {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);} 1794 _LIBCPP_INLINE_VISIBILITY 1795 _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1796 {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);} 1797 _LIBCPP_INLINE_VISIBILITY 1798 _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) 1799 volatile _NOEXCEPT 1800 {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);} 1801 _LIBCPP_INLINE_VISIBILITY 1802 _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1803 {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);} 1804 1805 _LIBCPP_INLINE_VISIBILITY 1806 _Tp* operator++(int) volatile _NOEXCEPT {return fetch_add(1);} 1807 _LIBCPP_INLINE_VISIBILITY 1808 _Tp* operator++(int) _NOEXCEPT {return fetch_add(1);} 1809 _LIBCPP_INLINE_VISIBILITY 1810 _Tp* operator--(int) volatile _NOEXCEPT {return fetch_sub(1);} 1811 _LIBCPP_INLINE_VISIBILITY 1812 _Tp* operator--(int) _NOEXCEPT {return fetch_sub(1);} 1813 _LIBCPP_INLINE_VISIBILITY 1814 _Tp* operator++() volatile _NOEXCEPT {return fetch_add(1) + 1;} 1815 _LIBCPP_INLINE_VISIBILITY 1816 _Tp* operator++() _NOEXCEPT {return fetch_add(1) + 1;} 1817 _LIBCPP_INLINE_VISIBILITY 1818 _Tp* operator--() volatile _NOEXCEPT {return fetch_sub(1) - 1;} 1819 _LIBCPP_INLINE_VISIBILITY 1820 _Tp* operator--() _NOEXCEPT {return fetch_sub(1) - 1;} 1821 _LIBCPP_INLINE_VISIBILITY 1822 _Tp* operator+=(ptrdiff_t __op) volatile _NOEXCEPT {return fetch_add(__op) + __op;} 1823 _LIBCPP_INLINE_VISIBILITY 1824 _Tp* operator+=(ptrdiff_t __op) _NOEXCEPT {return fetch_add(__op) + __op;} 1825 _LIBCPP_INLINE_VISIBILITY 1826 _Tp* operator-=(ptrdiff_t __op) volatile _NOEXCEPT {return fetch_sub(__op) - __op;} 1827 _LIBCPP_INLINE_VISIBILITY 1828 _Tp* operator-=(ptrdiff_t __op) _NOEXCEPT {return fetch_sub(__op) - __op;} 1829}; 1830 1831// atomic_is_lock_free 1832 1833template <class _Tp> 1834_LIBCPP_INLINE_VISIBILITY 1835bool 1836atomic_is_lock_free(const volatile atomic<_Tp>* __o) _NOEXCEPT 1837{ 1838 return __o->is_lock_free(); 1839} 1840 1841template <class _Tp> 1842_LIBCPP_INLINE_VISIBILITY 1843bool 1844atomic_is_lock_free(const atomic<_Tp>* __o) _NOEXCEPT 1845{ 1846 return __o->is_lock_free(); 1847} 1848 1849// atomic_init 1850 1851template <class _Tp> 1852_LIBCPP_INLINE_VISIBILITY 1853void 1854atomic_init(volatile atomic<_Tp>* __o, _Tp __d) _NOEXCEPT 1855{ 1856 __cxx_atomic_init(&__o->__a_, __d); 1857} 1858 1859template <class _Tp> 1860_LIBCPP_INLINE_VISIBILITY 1861void 1862atomic_init(atomic<_Tp>* __o, _Tp __d) _NOEXCEPT 1863{ 1864 __cxx_atomic_init(&__o->__a_, __d); 1865} 1866 1867// atomic_store 1868 1869template <class _Tp> 1870_LIBCPP_INLINE_VISIBILITY 1871void 1872atomic_store(volatile atomic<_Tp>* __o, _Tp __d) _NOEXCEPT 1873{ 1874 __o->store(__d); 1875} 1876 1877template <class _Tp> 1878_LIBCPP_INLINE_VISIBILITY 1879void 1880atomic_store(atomic<_Tp>* __o, _Tp __d) _NOEXCEPT 1881{ 1882 __o->store(__d); 1883} 1884 1885// atomic_store_explicit 1886 1887template <class _Tp> 1888_LIBCPP_INLINE_VISIBILITY 1889void 1890atomic_store_explicit(volatile atomic<_Tp>* __o, _Tp __d, memory_order __m) _NOEXCEPT 1891 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1892{ 1893 __o->store(__d, __m); 1894} 1895 1896template <class _Tp> 1897_LIBCPP_INLINE_VISIBILITY 1898void 1899atomic_store_explicit(atomic<_Tp>* __o, _Tp __d, memory_order __m) _NOEXCEPT 1900 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1901{ 1902 __o->store(__d, __m); 1903} 1904 1905// atomic_load 1906 1907template <class _Tp> 1908_LIBCPP_INLINE_VISIBILITY 1909_Tp 1910atomic_load(const volatile atomic<_Tp>* __o) _NOEXCEPT 1911{ 1912 return __o->load(); 1913} 1914 1915template <class _Tp> 1916_LIBCPP_INLINE_VISIBILITY 1917_Tp 1918atomic_load(const atomic<_Tp>* __o) _NOEXCEPT 1919{ 1920 return __o->load(); 1921} 1922 1923// atomic_load_explicit 1924 1925template <class _Tp> 1926_LIBCPP_INLINE_VISIBILITY 1927_Tp 1928atomic_load_explicit(const volatile atomic<_Tp>* __o, memory_order __m) _NOEXCEPT 1929 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 1930{ 1931 return __o->load(__m); 1932} 1933 1934template <class _Tp> 1935_LIBCPP_INLINE_VISIBILITY 1936_Tp 1937atomic_load_explicit(const atomic<_Tp>* __o, memory_order __m) _NOEXCEPT 1938 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 1939{ 1940 return __o->load(__m); 1941} 1942 1943// atomic_exchange 1944 1945template <class _Tp> 1946_LIBCPP_INLINE_VISIBILITY 1947_Tp 1948atomic_exchange(volatile atomic<_Tp>* __o, _Tp __d) _NOEXCEPT 1949{ 1950 return __o->exchange(__d); 1951} 1952 1953template <class _Tp> 1954_LIBCPP_INLINE_VISIBILITY 1955_Tp 1956atomic_exchange(atomic<_Tp>* __o, _Tp __d) _NOEXCEPT 1957{ 1958 return __o->exchange(__d); 1959} 1960 1961// atomic_exchange_explicit 1962 1963template <class _Tp> 1964_LIBCPP_INLINE_VISIBILITY 1965_Tp 1966atomic_exchange_explicit(volatile atomic<_Tp>* __o, _Tp __d, memory_order __m) _NOEXCEPT 1967{ 1968 return __o->exchange(__d, __m); 1969} 1970 1971template <class _Tp> 1972_LIBCPP_INLINE_VISIBILITY 1973_Tp 1974atomic_exchange_explicit(atomic<_Tp>* __o, _Tp __d, memory_order __m) _NOEXCEPT 1975{ 1976 return __o->exchange(__d, __m); 1977} 1978 1979// atomic_compare_exchange_weak 1980 1981template <class _Tp> 1982_LIBCPP_INLINE_VISIBILITY 1983bool 1984atomic_compare_exchange_weak(volatile atomic<_Tp>* __o, _Tp* __e, _Tp __d) _NOEXCEPT 1985{ 1986 return __o->compare_exchange_weak(*__e, __d); 1987} 1988 1989template <class _Tp> 1990_LIBCPP_INLINE_VISIBILITY 1991bool 1992atomic_compare_exchange_weak(atomic<_Tp>* __o, _Tp* __e, _Tp __d) _NOEXCEPT 1993{ 1994 return __o->compare_exchange_weak(*__e, __d); 1995} 1996 1997// atomic_compare_exchange_strong 1998 1999template <class _Tp> 2000_LIBCPP_INLINE_VISIBILITY 2001bool 2002atomic_compare_exchange_strong(volatile atomic<_Tp>* __o, _Tp* __e, _Tp __d) _NOEXCEPT 2003{ 2004 return __o->compare_exchange_strong(*__e, __d); 2005} 2006 2007template <class _Tp> 2008_LIBCPP_INLINE_VISIBILITY 2009bool 2010atomic_compare_exchange_strong(atomic<_Tp>* __o, _Tp* __e, _Tp __d) _NOEXCEPT 2011{ 2012 return __o->compare_exchange_strong(*__e, __d); 2013} 2014 2015// atomic_compare_exchange_weak_explicit 2016 2017template <class _Tp> 2018_LIBCPP_INLINE_VISIBILITY 2019bool 2020atomic_compare_exchange_weak_explicit(volatile atomic<_Tp>* __o, _Tp* __e, 2021 _Tp __d, 2022 memory_order __s, memory_order __f) _NOEXCEPT 2023 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2024{ 2025 return __o->compare_exchange_weak(*__e, __d, __s, __f); 2026} 2027 2028template <class _Tp> 2029_LIBCPP_INLINE_VISIBILITY 2030bool 2031atomic_compare_exchange_weak_explicit(atomic<_Tp>* __o, _Tp* __e, _Tp __d, 2032 memory_order __s, memory_order __f) _NOEXCEPT 2033 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2034{ 2035 return __o->compare_exchange_weak(*__e, __d, __s, __f); 2036} 2037 2038// atomic_compare_exchange_strong_explicit 2039 2040template <class _Tp> 2041_LIBCPP_INLINE_VISIBILITY 2042bool 2043atomic_compare_exchange_strong_explicit(volatile atomic<_Tp>* __o, 2044 _Tp* __e, _Tp __d, 2045 memory_order __s, memory_order __f) _NOEXCEPT 2046 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2047{ 2048 return __o->compare_exchange_strong(*__e, __d, __s, __f); 2049} 2050 2051template <class _Tp> 2052_LIBCPP_INLINE_VISIBILITY 2053bool 2054atomic_compare_exchange_strong_explicit(atomic<_Tp>* __o, _Tp* __e, 2055 _Tp __d, 2056 memory_order __s, memory_order __f) _NOEXCEPT 2057 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2058{ 2059 return __o->compare_exchange_strong(*__e, __d, __s, __f); 2060} 2061 2062// atomic_wait 2063 2064template <class _Tp> 2065_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2066void atomic_wait(const volatile atomic<_Tp>* __o, 2067 typename atomic<_Tp>::value_type __v) _NOEXCEPT 2068{ 2069 return __o->wait(__v); 2070} 2071 2072template <class _Tp> 2073_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2074void atomic_wait(const atomic<_Tp>* __o, 2075 typename atomic<_Tp>::value_type __v) _NOEXCEPT 2076{ 2077 return __o->wait(__v); 2078} 2079 2080// atomic_wait_explicit 2081 2082template <class _Tp> 2083_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2084void atomic_wait_explicit(const volatile atomic<_Tp>* __o, 2085 typename atomic<_Tp>::value_type __v, 2086 memory_order __m) _NOEXCEPT 2087 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 2088{ 2089 return __o->wait(__v, __m); 2090} 2091 2092template <class _Tp> 2093_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2094void atomic_wait_explicit(const atomic<_Tp>* __o, 2095 typename atomic<_Tp>::value_type __v, 2096 memory_order __m) _NOEXCEPT 2097 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 2098{ 2099 return __o->wait(__v, __m); 2100} 2101 2102// atomic_notify_one 2103 2104template <class _Tp> 2105_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2106void atomic_notify_one(volatile atomic<_Tp>* __o) _NOEXCEPT 2107{ 2108 __o->notify_one(); 2109} 2110template <class _Tp> 2111_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2112void atomic_notify_one(atomic<_Tp>* __o) _NOEXCEPT 2113{ 2114 __o->notify_one(); 2115} 2116 2117// atomic_notify_one 2118 2119template <class _Tp> 2120_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2121void atomic_notify_all(volatile atomic<_Tp>* __o) _NOEXCEPT 2122{ 2123 __o->notify_all(); 2124} 2125template <class _Tp> 2126_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2127void atomic_notify_all(atomic<_Tp>* __o) _NOEXCEPT 2128{ 2129 __o->notify_all(); 2130} 2131 2132// atomic_fetch_add 2133 2134template <class _Tp> 2135_LIBCPP_INLINE_VISIBILITY 2136typename enable_if 2137< 2138 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2139 _Tp 2140>::type 2141atomic_fetch_add(volatile atomic<_Tp>* __o, _Tp __op) _NOEXCEPT 2142{ 2143 return __o->fetch_add(__op); 2144} 2145 2146template <class _Tp> 2147_LIBCPP_INLINE_VISIBILITY 2148typename enable_if 2149< 2150 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2151 _Tp 2152>::type 2153atomic_fetch_add(atomic<_Tp>* __o, _Tp __op) _NOEXCEPT 2154{ 2155 return __o->fetch_add(__op); 2156} 2157 2158template <class _Tp> 2159_LIBCPP_INLINE_VISIBILITY 2160_Tp* 2161atomic_fetch_add(volatile atomic<_Tp*>* __o, ptrdiff_t __op) _NOEXCEPT 2162{ 2163 return __o->fetch_add(__op); 2164} 2165 2166template <class _Tp> 2167_LIBCPP_INLINE_VISIBILITY 2168_Tp* 2169atomic_fetch_add(atomic<_Tp*>* __o, ptrdiff_t __op) _NOEXCEPT 2170{ 2171 return __o->fetch_add(__op); 2172} 2173 2174// atomic_fetch_add_explicit 2175 2176template <class _Tp> 2177_LIBCPP_INLINE_VISIBILITY 2178typename enable_if 2179< 2180 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2181 _Tp 2182>::type 2183atomic_fetch_add_explicit(volatile atomic<_Tp>* __o, _Tp __op, memory_order __m) _NOEXCEPT 2184{ 2185 return __o->fetch_add(__op, __m); 2186} 2187 2188template <class _Tp> 2189_LIBCPP_INLINE_VISIBILITY 2190typename enable_if 2191< 2192 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2193 _Tp 2194>::type 2195atomic_fetch_add_explicit(atomic<_Tp>* __o, _Tp __op, memory_order __m) _NOEXCEPT 2196{ 2197 return __o->fetch_add(__op, __m); 2198} 2199 2200template <class _Tp> 2201_LIBCPP_INLINE_VISIBILITY 2202_Tp* 2203atomic_fetch_add_explicit(volatile atomic<_Tp*>* __o, ptrdiff_t __op, 2204 memory_order __m) _NOEXCEPT 2205{ 2206 return __o->fetch_add(__op, __m); 2207} 2208 2209template <class _Tp> 2210_LIBCPP_INLINE_VISIBILITY 2211_Tp* 2212atomic_fetch_add_explicit(atomic<_Tp*>* __o, ptrdiff_t __op, memory_order __m) _NOEXCEPT 2213{ 2214 return __o->fetch_add(__op, __m); 2215} 2216 2217// atomic_fetch_sub 2218 2219template <class _Tp> 2220_LIBCPP_INLINE_VISIBILITY 2221typename enable_if 2222< 2223 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2224 _Tp 2225>::type 2226atomic_fetch_sub(volatile atomic<_Tp>* __o, _Tp __op) _NOEXCEPT 2227{ 2228 return __o->fetch_sub(__op); 2229} 2230 2231template <class _Tp> 2232_LIBCPP_INLINE_VISIBILITY 2233typename enable_if 2234< 2235 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2236 _Tp 2237>::type 2238atomic_fetch_sub(atomic<_Tp>* __o, _Tp __op) _NOEXCEPT 2239{ 2240 return __o->fetch_sub(__op); 2241} 2242 2243template <class _Tp> 2244_LIBCPP_INLINE_VISIBILITY 2245_Tp* 2246atomic_fetch_sub(volatile atomic<_Tp*>* __o, ptrdiff_t __op) _NOEXCEPT 2247{ 2248 return __o->fetch_sub(__op); 2249} 2250 2251template <class _Tp> 2252_LIBCPP_INLINE_VISIBILITY 2253_Tp* 2254atomic_fetch_sub(atomic<_Tp*>* __o, ptrdiff_t __op) _NOEXCEPT 2255{ 2256 return __o->fetch_sub(__op); 2257} 2258 2259// atomic_fetch_sub_explicit 2260 2261template <class _Tp> 2262_LIBCPP_INLINE_VISIBILITY 2263typename enable_if 2264< 2265 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2266 _Tp 2267>::type 2268atomic_fetch_sub_explicit(volatile atomic<_Tp>* __o, _Tp __op, memory_order __m) _NOEXCEPT 2269{ 2270 return __o->fetch_sub(__op, __m); 2271} 2272 2273template <class _Tp> 2274_LIBCPP_INLINE_VISIBILITY 2275typename enable_if 2276< 2277 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2278 _Tp 2279>::type 2280atomic_fetch_sub_explicit(atomic<_Tp>* __o, _Tp __op, memory_order __m) _NOEXCEPT 2281{ 2282 return __o->fetch_sub(__op, __m); 2283} 2284 2285template <class _Tp> 2286_LIBCPP_INLINE_VISIBILITY 2287_Tp* 2288atomic_fetch_sub_explicit(volatile atomic<_Tp*>* __o, ptrdiff_t __op, 2289 memory_order __m) _NOEXCEPT 2290{ 2291 return __o->fetch_sub(__op, __m); 2292} 2293 2294template <class _Tp> 2295_LIBCPP_INLINE_VISIBILITY 2296_Tp* 2297atomic_fetch_sub_explicit(atomic<_Tp*>* __o, ptrdiff_t __op, memory_order __m) _NOEXCEPT 2298{ 2299 return __o->fetch_sub(__op, __m); 2300} 2301 2302// atomic_fetch_and 2303 2304template <class _Tp> 2305_LIBCPP_INLINE_VISIBILITY 2306typename enable_if 2307< 2308 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2309 _Tp 2310>::type 2311atomic_fetch_and(volatile atomic<_Tp>* __o, _Tp __op) _NOEXCEPT 2312{ 2313 return __o->fetch_and(__op); 2314} 2315 2316template <class _Tp> 2317_LIBCPP_INLINE_VISIBILITY 2318typename enable_if 2319< 2320 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2321 _Tp 2322>::type 2323atomic_fetch_and(atomic<_Tp>* __o, _Tp __op) _NOEXCEPT 2324{ 2325 return __o->fetch_and(__op); 2326} 2327 2328// atomic_fetch_and_explicit 2329 2330template <class _Tp> 2331_LIBCPP_INLINE_VISIBILITY 2332typename enable_if 2333< 2334 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2335 _Tp 2336>::type 2337atomic_fetch_and_explicit(volatile atomic<_Tp>* __o, _Tp __op, memory_order __m) _NOEXCEPT 2338{ 2339 return __o->fetch_and(__op, __m); 2340} 2341 2342template <class _Tp> 2343_LIBCPP_INLINE_VISIBILITY 2344typename enable_if 2345< 2346 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2347 _Tp 2348>::type 2349atomic_fetch_and_explicit(atomic<_Tp>* __o, _Tp __op, memory_order __m) _NOEXCEPT 2350{ 2351 return __o->fetch_and(__op, __m); 2352} 2353 2354// atomic_fetch_or 2355 2356template <class _Tp> 2357_LIBCPP_INLINE_VISIBILITY 2358typename enable_if 2359< 2360 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2361 _Tp 2362>::type 2363atomic_fetch_or(volatile atomic<_Tp>* __o, _Tp __op) _NOEXCEPT 2364{ 2365 return __o->fetch_or(__op); 2366} 2367 2368template <class _Tp> 2369_LIBCPP_INLINE_VISIBILITY 2370typename enable_if 2371< 2372 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2373 _Tp 2374>::type 2375atomic_fetch_or(atomic<_Tp>* __o, _Tp __op) _NOEXCEPT 2376{ 2377 return __o->fetch_or(__op); 2378} 2379 2380// atomic_fetch_or_explicit 2381 2382template <class _Tp> 2383_LIBCPP_INLINE_VISIBILITY 2384typename enable_if 2385< 2386 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2387 _Tp 2388>::type 2389atomic_fetch_or_explicit(volatile atomic<_Tp>* __o, _Tp __op, memory_order __m) _NOEXCEPT 2390{ 2391 return __o->fetch_or(__op, __m); 2392} 2393 2394template <class _Tp> 2395_LIBCPP_INLINE_VISIBILITY 2396typename enable_if 2397< 2398 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2399 _Tp 2400>::type 2401atomic_fetch_or_explicit(atomic<_Tp>* __o, _Tp __op, memory_order __m) _NOEXCEPT 2402{ 2403 return __o->fetch_or(__op, __m); 2404} 2405 2406// atomic_fetch_xor 2407 2408template <class _Tp> 2409_LIBCPP_INLINE_VISIBILITY 2410typename enable_if 2411< 2412 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2413 _Tp 2414>::type 2415atomic_fetch_xor(volatile atomic<_Tp>* __o, _Tp __op) _NOEXCEPT 2416{ 2417 return __o->fetch_xor(__op); 2418} 2419 2420template <class _Tp> 2421_LIBCPP_INLINE_VISIBILITY 2422typename enable_if 2423< 2424 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2425 _Tp 2426>::type 2427atomic_fetch_xor(atomic<_Tp>* __o, _Tp __op) _NOEXCEPT 2428{ 2429 return __o->fetch_xor(__op); 2430} 2431 2432// atomic_fetch_xor_explicit 2433 2434template <class _Tp> 2435_LIBCPP_INLINE_VISIBILITY 2436typename enable_if 2437< 2438 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2439 _Tp 2440>::type 2441atomic_fetch_xor_explicit(volatile atomic<_Tp>* __o, _Tp __op, memory_order __m) _NOEXCEPT 2442{ 2443 return __o->fetch_xor(__op, __m); 2444} 2445 2446template <class _Tp> 2447_LIBCPP_INLINE_VISIBILITY 2448typename enable_if 2449< 2450 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2451 _Tp 2452>::type 2453atomic_fetch_xor_explicit(atomic<_Tp>* __o, _Tp __op, memory_order __m) _NOEXCEPT 2454{ 2455 return __o->fetch_xor(__op, __m); 2456} 2457 2458// flag type and operations 2459 2460typedef struct atomic_flag 2461{ 2462 __cxx_atomic_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_; 2463 2464 _LIBCPP_INLINE_VISIBILITY 2465 bool test(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 2466 {return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m);} 2467 _LIBCPP_INLINE_VISIBILITY 2468 bool test(memory_order __m = memory_order_seq_cst) const _NOEXCEPT 2469 {return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m);} 2470 2471 _LIBCPP_INLINE_VISIBILITY 2472 bool test_and_set(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 2473 {return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m);} 2474 _LIBCPP_INLINE_VISIBILITY 2475 bool test_and_set(memory_order __m = memory_order_seq_cst) _NOEXCEPT 2476 {return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m);} 2477 _LIBCPP_INLINE_VISIBILITY 2478 void clear(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 2479 {__cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m);} 2480 _LIBCPP_INLINE_VISIBILITY 2481 void clear(memory_order __m = memory_order_seq_cst) _NOEXCEPT 2482 {__cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m);} 2483 2484 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2485 void wait(bool __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 2486 {__cxx_atomic_wait(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);} 2487 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2488 void wait(bool __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT 2489 {__cxx_atomic_wait(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);} 2490 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2491 void notify_one() volatile _NOEXCEPT 2492 {__cxx_atomic_notify_one(&__a_);} 2493 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2494 void notify_one() _NOEXCEPT 2495 {__cxx_atomic_notify_one(&__a_);} 2496 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2497 void notify_all() volatile _NOEXCEPT 2498 {__cxx_atomic_notify_all(&__a_);} 2499 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2500 void notify_all() _NOEXCEPT 2501 {__cxx_atomic_notify_all(&__a_);} 2502 2503 _LIBCPP_INLINE_VISIBILITY 2504 atomic_flag() _NOEXCEPT _LIBCPP_DEFAULT 2505 2506 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR 2507 atomic_flag(bool __b) _NOEXCEPT : __a_(__b) {} // EXTENSION 2508 2509#ifndef _LIBCPP_CXX03_LANG 2510 atomic_flag(const atomic_flag&) = delete; 2511 atomic_flag& operator=(const atomic_flag&) = delete; 2512 atomic_flag& operator=(const atomic_flag&) volatile = delete; 2513#else 2514private: 2515 _LIBCPP_INLINE_VISIBILITY 2516 atomic_flag(const atomic_flag&); 2517 _LIBCPP_INLINE_VISIBILITY 2518 atomic_flag& operator=(const atomic_flag&); 2519 _LIBCPP_INLINE_VISIBILITY 2520 atomic_flag& operator=(const atomic_flag&) volatile; 2521#endif 2522} atomic_flag; 2523 2524 2525inline _LIBCPP_INLINE_VISIBILITY 2526bool 2527atomic_flag_test(const volatile atomic_flag* __o) _NOEXCEPT 2528{ 2529 return __o->test(); 2530} 2531 2532inline _LIBCPP_INLINE_VISIBILITY 2533bool 2534atomic_flag_test(const atomic_flag* __o) _NOEXCEPT 2535{ 2536 return __o->test(); 2537} 2538 2539inline _LIBCPP_INLINE_VISIBILITY 2540bool 2541atomic_flag_test_explicit(const volatile atomic_flag* __o, memory_order __m) _NOEXCEPT 2542{ 2543 return __o->test(__m); 2544} 2545 2546inline _LIBCPP_INLINE_VISIBILITY 2547bool 2548atomic_flag_test_explicit(const atomic_flag* __o, memory_order __m) _NOEXCEPT 2549{ 2550 return __o->test(__m); 2551} 2552 2553inline _LIBCPP_INLINE_VISIBILITY 2554bool 2555atomic_flag_test_and_set(volatile atomic_flag* __o) _NOEXCEPT 2556{ 2557 return __o->test_and_set(); 2558} 2559 2560inline _LIBCPP_INLINE_VISIBILITY 2561bool 2562atomic_flag_test_and_set(atomic_flag* __o) _NOEXCEPT 2563{ 2564 return __o->test_and_set(); 2565} 2566 2567inline _LIBCPP_INLINE_VISIBILITY 2568bool 2569atomic_flag_test_and_set_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT 2570{ 2571 return __o->test_and_set(__m); 2572} 2573 2574inline _LIBCPP_INLINE_VISIBILITY 2575bool 2576atomic_flag_test_and_set_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT 2577{ 2578 return __o->test_and_set(__m); 2579} 2580 2581inline _LIBCPP_INLINE_VISIBILITY 2582void 2583atomic_flag_clear(volatile atomic_flag* __o) _NOEXCEPT 2584{ 2585 __o->clear(); 2586} 2587 2588inline _LIBCPP_INLINE_VISIBILITY 2589void 2590atomic_flag_clear(atomic_flag* __o) _NOEXCEPT 2591{ 2592 __o->clear(); 2593} 2594 2595inline _LIBCPP_INLINE_VISIBILITY 2596void 2597atomic_flag_clear_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT 2598{ 2599 __o->clear(__m); 2600} 2601 2602inline _LIBCPP_INLINE_VISIBILITY 2603void 2604atomic_flag_clear_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT 2605{ 2606 __o->clear(__m); 2607} 2608 2609inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2610void 2611atomic_flag_wait(const volatile atomic_flag* __o, bool __v) _NOEXCEPT 2612{ 2613 __o->wait(__v); 2614} 2615 2616inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2617void 2618atomic_flag_wait(const atomic_flag* __o, bool __v) _NOEXCEPT 2619{ 2620 __o->wait(__v); 2621} 2622 2623inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2624void 2625atomic_flag_wait_explicit(const volatile atomic_flag* __o, 2626 bool __v, memory_order __m) _NOEXCEPT 2627{ 2628 __o->wait(__v, __m); 2629} 2630 2631inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2632void 2633atomic_flag_wait_explicit(const atomic_flag* __o, 2634 bool __v, memory_order __m) _NOEXCEPT 2635{ 2636 __o->wait(__v, __m); 2637} 2638 2639inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2640void 2641atomic_flag_notify_one(volatile atomic_flag* __o) _NOEXCEPT 2642{ 2643 __o->notify_one(); 2644} 2645 2646inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2647void 2648atomic_flag_notify_one(atomic_flag* __o) _NOEXCEPT 2649{ 2650 __o->notify_one(); 2651} 2652 2653inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2654void 2655atomic_flag_notify_all(volatile atomic_flag* __o) _NOEXCEPT 2656{ 2657 __o->notify_all(); 2658} 2659 2660inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2661void 2662atomic_flag_notify_all(atomic_flag* __o) _NOEXCEPT 2663{ 2664 __o->notify_all(); 2665} 2666 2667// fences 2668 2669inline _LIBCPP_INLINE_VISIBILITY 2670void 2671atomic_thread_fence(memory_order __m) _NOEXCEPT 2672{ 2673 __cxx_atomic_thread_fence(__m); 2674} 2675 2676inline _LIBCPP_INLINE_VISIBILITY 2677void 2678atomic_signal_fence(memory_order __m) _NOEXCEPT 2679{ 2680 __cxx_atomic_signal_fence(__m); 2681} 2682 2683// Atomics for standard typedef types 2684 2685typedef atomic<bool> atomic_bool; 2686typedef atomic<char> atomic_char; 2687typedef atomic<signed char> atomic_schar; 2688typedef atomic<unsigned char> atomic_uchar; 2689typedef atomic<short> atomic_short; 2690typedef atomic<unsigned short> atomic_ushort; 2691typedef atomic<int> atomic_int; 2692typedef atomic<unsigned int> atomic_uint; 2693typedef atomic<long> atomic_long; 2694typedef atomic<unsigned long> atomic_ulong; 2695typedef atomic<long long> atomic_llong; 2696typedef atomic<unsigned long long> atomic_ullong; 2697typedef atomic<char16_t> atomic_char16_t; 2698typedef atomic<char32_t> atomic_char32_t; 2699typedef atomic<wchar_t> atomic_wchar_t; 2700 2701typedef atomic<int_least8_t> atomic_int_least8_t; 2702typedef atomic<uint_least8_t> atomic_uint_least8_t; 2703typedef atomic<int_least16_t> atomic_int_least16_t; 2704typedef atomic<uint_least16_t> atomic_uint_least16_t; 2705typedef atomic<int_least32_t> atomic_int_least32_t; 2706typedef atomic<uint_least32_t> atomic_uint_least32_t; 2707typedef atomic<int_least64_t> atomic_int_least64_t; 2708typedef atomic<uint_least64_t> atomic_uint_least64_t; 2709 2710typedef atomic<int_fast8_t> atomic_int_fast8_t; 2711typedef atomic<uint_fast8_t> atomic_uint_fast8_t; 2712typedef atomic<int_fast16_t> atomic_int_fast16_t; 2713typedef atomic<uint_fast16_t> atomic_uint_fast16_t; 2714typedef atomic<int_fast32_t> atomic_int_fast32_t; 2715typedef atomic<uint_fast32_t> atomic_uint_fast32_t; 2716typedef atomic<int_fast64_t> atomic_int_fast64_t; 2717typedef atomic<uint_fast64_t> atomic_uint_fast64_t; 2718 2719typedef atomic< int8_t> atomic_int8_t; 2720typedef atomic<uint8_t> atomic_uint8_t; 2721typedef atomic< int16_t> atomic_int16_t; 2722typedef atomic<uint16_t> atomic_uint16_t; 2723typedef atomic< int32_t> atomic_int32_t; 2724typedef atomic<uint32_t> atomic_uint32_t; 2725typedef atomic< int64_t> atomic_int64_t; 2726typedef atomic<uint64_t> atomic_uint64_t; 2727 2728typedef atomic<intptr_t> atomic_intptr_t; 2729typedef atomic<uintptr_t> atomic_uintptr_t; 2730typedef atomic<size_t> atomic_size_t; 2731typedef atomic<ptrdiff_t> atomic_ptrdiff_t; 2732typedef atomic<intmax_t> atomic_intmax_t; 2733typedef atomic<uintmax_t> atomic_uintmax_t; 2734 2735// atomic_*_lock_free : prefer the contention type most highly, then the largest lock-free type 2736 2737#ifdef __cpp_lib_atomic_is_always_lock_free 2738# define _LIBCPP_CONTENTION_LOCK_FREE __atomic_always_lock_free(sizeof(__cxx_contention_t), 0) 2739#else 2740# define _LIBCPP_CONTENTION_LOCK_FREE false 2741#endif 2742 2743#if ATOMIC_LLONG_LOCK_FREE == 2 2744typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, long long>::type __libcpp_signed_lock_free; 2745typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned long long>::type __libcpp_unsigned_lock_free; 2746#elif ATOMIC_INT_LOCK_FREE == 2 2747typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, int>::type __libcpp_signed_lock_free; 2748typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned int>::type __libcpp_unsigned_lock_free; 2749#elif ATOMIC_SHORT_LOCK_FREE == 2 2750typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, short>::type __libcpp_signed_lock_free; 2751typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned short>::type __libcpp_unsigned_lock_free; 2752#elif ATOMIC_CHAR_LOCK_FREE == 2 2753typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, char>::type __libcpp_signed_lock_free; 2754typedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned char>::type __libcpp_unsigned_lock_free; 2755#else 2756 // No signed/unsigned lock-free types 2757#endif 2758 2759typedef atomic<__libcpp_signed_lock_free> atomic_signed_lock_free; 2760typedef atomic<__libcpp_unsigned_lock_free> atomic_unsigned_lock_free; 2761 2762#define ATOMIC_FLAG_INIT {false} 2763#define ATOMIC_VAR_INIT(__v) {__v} 2764 2765_LIBCPP_END_NAMESPACE_STD 2766 2767#endif // _LIBCPP_ATOMIC 2768