Lines Matching refs:mo
35 static bool IsLoadOrder(morder mo) { in IsLoadOrder() argument
36 return mo == mo_relaxed || mo == mo_consume in IsLoadOrder()
37 || mo == mo_acquire || mo == mo_seq_cst; in IsLoadOrder()
40 static bool IsStoreOrder(morder mo) { in IsStoreOrder() argument
41 return mo == mo_relaxed || mo == mo_release || mo == mo_seq_cst; in IsStoreOrder()
44 static bool IsReleaseOrder(morder mo) { in IsReleaseOrder() argument
45 return mo == mo_release || mo == mo_acq_rel || mo == mo_seq_cst; in IsReleaseOrder()
48 static bool IsAcquireOrder(morder mo) { in IsAcquireOrder() argument
49 return mo == mo_consume || mo == mo_acquire in IsAcquireOrder()
50 || mo == mo_acq_rel || mo == mo_seq_cst; in IsAcquireOrder()
53 static bool IsAcqRelOrder(morder mo) { in IsAcqRelOrder() argument
54 return mo == mo_acq_rel || mo == mo_seq_cst; in IsAcqRelOrder()
196 static memory_order to_mo(morder mo) { in to_mo() argument
197 switch (mo) { in to_mo()
210 static T NoTsanAtomicLoad(const volatile T *a, morder mo) { in NoTsanAtomicLoad() argument
211 return atomic_load(to_atomic(a), to_mo(mo)); in NoTsanAtomicLoad()
215 static a128 NoTsanAtomicLoad(const volatile a128 *a, morder mo) { in NoTsanAtomicLoad() argument
223 morder mo) NO_THREAD_SAFETY_ANALYSIS { in AtomicLoad() argument
224 CHECK(IsLoadOrder(mo)); in AtomicLoad()
227 if (!IsAcquireOrder(mo)) { in AtomicLoad()
229 return NoTsanAtomicLoad(a, mo); in AtomicLoad()
233 T v = NoTsanAtomicLoad(a, mo); in AtomicLoad()
239 v = NoTsanAtomicLoad(a, mo); in AtomicLoad()
247 static void NoTsanAtomicStore(volatile T *a, T v, morder mo) { in NoTsanAtomicStore() argument
248 atomic_store(to_atomic(a), v, to_mo(mo)); in NoTsanAtomicStore()
252 static void NoTsanAtomicStore(volatile a128 *a, a128 v, morder mo) { in NoTsanAtomicStore() argument
260 morder mo) NO_THREAD_SAFETY_ANALYSIS { in AtomicStore() argument
261 CHECK(IsStoreOrder(mo)); in AtomicStore()
267 if (!IsReleaseOrder(mo)) { in AtomicStore()
268 NoTsanAtomicStore(a, v, mo); in AtomicStore()
277 NoTsanAtomicStore(a, v, mo); in AtomicStore()
283 morder mo) NO_THREAD_SAFETY_ANALYSIS { in AtomicRMW() argument
286 if (mo != mo_relaxed) { in AtomicRMW()
291 if (IsAcqRelOrder(mo)) in AtomicRMW()
293 else if (IsReleaseOrder(mo)) in AtomicRMW()
295 else if (IsAcquireOrder(mo)) in AtomicRMW()
305 static T NoTsanAtomicExchange(volatile T *a, T v, morder mo) { in NoTsanAtomicExchange() argument
310 static T NoTsanAtomicFetchAdd(volatile T *a, T v, morder mo) { in NoTsanAtomicFetchAdd() argument
315 static T NoTsanAtomicFetchSub(volatile T *a, T v, morder mo) { in NoTsanAtomicFetchSub() argument
320 static T NoTsanAtomicFetchAnd(volatile T *a, T v, morder mo) { in NoTsanAtomicFetchAnd() argument
325 static T NoTsanAtomicFetchOr(volatile T *a, T v, morder mo) { in NoTsanAtomicFetchOr() argument
330 static T NoTsanAtomicFetchXor(volatile T *a, T v, morder mo) { in NoTsanAtomicFetchXor() argument
335 static T NoTsanAtomicFetchNand(volatile T *a, T v, morder mo) { in NoTsanAtomicFetchNand() argument
341 morder mo) { in AtomicExchange() argument
342 return AtomicRMW<T, func_xchg>(thr, pc, a, v, mo); in AtomicExchange()
347 morder mo) { in AtomicFetchAdd() argument
348 return AtomicRMW<T, func_add>(thr, pc, a, v, mo); in AtomicFetchAdd()
353 morder mo) { in AtomicFetchSub() argument
354 return AtomicRMW<T, func_sub>(thr, pc, a, v, mo); in AtomicFetchSub()
359 morder mo) { in AtomicFetchAnd() argument
360 return AtomicRMW<T, func_and>(thr, pc, a, v, mo); in AtomicFetchAnd()
365 morder mo) { in AtomicFetchOr() argument
366 return AtomicRMW<T, func_or>(thr, pc, a, v, mo); in AtomicFetchOr()
371 morder mo) { in AtomicFetchXor() argument
372 return AtomicRMW<T, func_xor>(thr, pc, a, v, mo); in AtomicFetchXor()
377 morder mo) { in AtomicFetchNand() argument
378 return AtomicRMW<T, func_nand>(thr, pc, a, v, mo); in AtomicFetchNand()
382 static bool NoTsanAtomicCAS(volatile T *a, T *c, T v, morder mo, morder fmo) { in NoTsanAtomicCAS() argument
383 return atomic_compare_exchange_strong(to_atomic(a), c, v, to_mo(mo)); in NoTsanAtomicCAS()
388 morder mo, morder fmo) { in NoTsanAtomicCAS() argument
399 static T NoTsanAtomicCAS(volatile T *a, T c, T v, morder mo, morder fmo) { in NoTsanAtomicCAS() argument
400 NoTsanAtomicCAS(a, &c, v, mo, fmo); in NoTsanAtomicCAS()
405 static bool AtomicCAS(ThreadState *thr, uptr pc, volatile T *a, T *c, T v, morder mo, in AtomicCAS() argument
414 bool write_lock = IsReleaseOrder(mo); in AtomicCAS()
416 if (mo != mo_relaxed || fmo != mo_relaxed) in AtomicCAS()
424 mo = fmo; in AtomicCAS()
432 if (success && IsAcqRelOrder(mo)) in AtomicCAS()
434 else if (success && IsReleaseOrder(mo)) in AtomicCAS()
436 else if (IsAcquireOrder(mo)) in AtomicCAS()
450 volatile T *a, T c, T v, morder mo, morder fmo) { in AtomicCAS() argument
451 AtomicCAS(thr, pc, a, &c, v, mo, fmo); in AtomicCAS()
456 static void NoTsanAtomicFence(morder mo) { in NoTsanAtomicFence() argument
460 static void AtomicFence(ThreadState *thr, uptr pc, morder mo) { in AtomicFence() argument
471 static morder convert_morder(morder mo) { in convert_morder() argument
485 return (morder)(mo & 0x7fff); in convert_morder()
496 mo = convert_morder(mo); \
497 ScopedAtomic sa(thr, callpc, a, mo, __func__); \
504 morder mo, const char *func) in ScopedAtomic() argument
507 DPrintf("#%d: %s(%p, %d)\n", thr_->tid, func, a, mo); in ScopedAtomic()
519 a8 __tsan_atomic8_load(const volatile a8 *a, morder mo) { in __tsan_atomic8_load() argument
520 SCOPED_ATOMIC(Load, a, mo); in __tsan_atomic8_load()
524 a16 __tsan_atomic16_load(const volatile a16 *a, morder mo) { in __tsan_atomic16_load() argument
525 SCOPED_ATOMIC(Load, a, mo); in __tsan_atomic16_load()
529 a32 __tsan_atomic32_load(const volatile a32 *a, morder mo) { in __tsan_atomic32_load() argument
530 SCOPED_ATOMIC(Load, a, mo); in __tsan_atomic32_load()
534 a64 __tsan_atomic64_load(const volatile a64 *a, morder mo) { in __tsan_atomic64_load() argument
535 SCOPED_ATOMIC(Load, a, mo); in __tsan_atomic64_load()
540 a128 __tsan_atomic128_load(const volatile a128 *a, morder mo) { in __tsan_atomic128_load() argument
541 SCOPED_ATOMIC(Load, a, mo); in __tsan_atomic128_load()
546 void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo) { in __tsan_atomic8_store() argument
547 SCOPED_ATOMIC(Store, a, v, mo); in __tsan_atomic8_store()
551 void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo) { in __tsan_atomic16_store() argument
552 SCOPED_ATOMIC(Store, a, v, mo); in __tsan_atomic16_store()
556 void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo) { in __tsan_atomic32_store() argument
557 SCOPED_ATOMIC(Store, a, v, mo); in __tsan_atomic32_store()
561 void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo) { in __tsan_atomic64_store() argument
562 SCOPED_ATOMIC(Store, a, v, mo); in __tsan_atomic64_store()
567 void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo) { in __tsan_atomic128_store() argument
568 SCOPED_ATOMIC(Store, a, v, mo); in __tsan_atomic128_store()
573 a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo) { in __tsan_atomic8_exchange() argument
574 SCOPED_ATOMIC(Exchange, a, v, mo); in __tsan_atomic8_exchange()
578 a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo) { in __tsan_atomic16_exchange() argument
579 SCOPED_ATOMIC(Exchange, a, v, mo); in __tsan_atomic16_exchange()
583 a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo) { in __tsan_atomic32_exchange() argument
584 SCOPED_ATOMIC(Exchange, a, v, mo); in __tsan_atomic32_exchange()
588 a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo) { in __tsan_atomic64_exchange() argument
589 SCOPED_ATOMIC(Exchange, a, v, mo); in __tsan_atomic64_exchange()
594 a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo) { in __tsan_atomic128_exchange() argument
595 SCOPED_ATOMIC(Exchange, a, v, mo); in __tsan_atomic128_exchange()
600 a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo) { in __tsan_atomic8_fetch_add() argument
601 SCOPED_ATOMIC(FetchAdd, a, v, mo); in __tsan_atomic8_fetch_add()
605 a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo) { in __tsan_atomic16_fetch_add() argument
606 SCOPED_ATOMIC(FetchAdd, a, v, mo); in __tsan_atomic16_fetch_add()
610 a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo) { in __tsan_atomic32_fetch_add() argument
611 SCOPED_ATOMIC(FetchAdd, a, v, mo); in __tsan_atomic32_fetch_add()
615 a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo) { in __tsan_atomic64_fetch_add() argument
616 SCOPED_ATOMIC(FetchAdd, a, v, mo); in __tsan_atomic64_fetch_add()
621 a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo) { in __tsan_atomic128_fetch_add() argument
622 SCOPED_ATOMIC(FetchAdd, a, v, mo); in __tsan_atomic128_fetch_add()
627 a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo) { in __tsan_atomic8_fetch_sub() argument
628 SCOPED_ATOMIC(FetchSub, a, v, mo); in __tsan_atomic8_fetch_sub()
632 a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo) { in __tsan_atomic16_fetch_sub() argument
633 SCOPED_ATOMIC(FetchSub, a, v, mo); in __tsan_atomic16_fetch_sub()
637 a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo) { in __tsan_atomic32_fetch_sub() argument
638 SCOPED_ATOMIC(FetchSub, a, v, mo); in __tsan_atomic32_fetch_sub()
642 a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo) { in __tsan_atomic64_fetch_sub() argument
643 SCOPED_ATOMIC(FetchSub, a, v, mo); in __tsan_atomic64_fetch_sub()
648 a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo) { in __tsan_atomic128_fetch_sub() argument
649 SCOPED_ATOMIC(FetchSub, a, v, mo); in __tsan_atomic128_fetch_sub()
654 a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo) { in __tsan_atomic8_fetch_and() argument
655 SCOPED_ATOMIC(FetchAnd, a, v, mo); in __tsan_atomic8_fetch_and()
659 a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo) { in __tsan_atomic16_fetch_and() argument
660 SCOPED_ATOMIC(FetchAnd, a, v, mo); in __tsan_atomic16_fetch_and()
664 a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo) { in __tsan_atomic32_fetch_and() argument
665 SCOPED_ATOMIC(FetchAnd, a, v, mo); in __tsan_atomic32_fetch_and()
669 a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo) { in __tsan_atomic64_fetch_and() argument
670 SCOPED_ATOMIC(FetchAnd, a, v, mo); in __tsan_atomic64_fetch_and()
675 a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo) { in __tsan_atomic128_fetch_and() argument
676 SCOPED_ATOMIC(FetchAnd, a, v, mo); in __tsan_atomic128_fetch_and()
681 a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo) { in __tsan_atomic8_fetch_or() argument
682 SCOPED_ATOMIC(FetchOr, a, v, mo); in __tsan_atomic8_fetch_or()
686 a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo) { in __tsan_atomic16_fetch_or() argument
687 SCOPED_ATOMIC(FetchOr, a, v, mo); in __tsan_atomic16_fetch_or()
691 a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo) { in __tsan_atomic32_fetch_or() argument
692 SCOPED_ATOMIC(FetchOr, a, v, mo); in __tsan_atomic32_fetch_or()
696 a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo) { in __tsan_atomic64_fetch_or() argument
697 SCOPED_ATOMIC(FetchOr, a, v, mo); in __tsan_atomic64_fetch_or()
702 a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo) { in __tsan_atomic128_fetch_or() argument
703 SCOPED_ATOMIC(FetchOr, a, v, mo); in __tsan_atomic128_fetch_or()
708 a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo) { in __tsan_atomic8_fetch_xor() argument
709 SCOPED_ATOMIC(FetchXor, a, v, mo); in __tsan_atomic8_fetch_xor()
713 a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo) { in __tsan_atomic16_fetch_xor() argument
714 SCOPED_ATOMIC(FetchXor, a, v, mo); in __tsan_atomic16_fetch_xor()
718 a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo) { in __tsan_atomic32_fetch_xor() argument
719 SCOPED_ATOMIC(FetchXor, a, v, mo); in __tsan_atomic32_fetch_xor()
723 a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo) { in __tsan_atomic64_fetch_xor() argument
724 SCOPED_ATOMIC(FetchXor, a, v, mo); in __tsan_atomic64_fetch_xor()
729 a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo) { in __tsan_atomic128_fetch_xor() argument
730 SCOPED_ATOMIC(FetchXor, a, v, mo); in __tsan_atomic128_fetch_xor()
735 a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo) { in __tsan_atomic8_fetch_nand() argument
736 SCOPED_ATOMIC(FetchNand, a, v, mo); in __tsan_atomic8_fetch_nand()
740 a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo) { in __tsan_atomic16_fetch_nand() argument
741 SCOPED_ATOMIC(FetchNand, a, v, mo); in __tsan_atomic16_fetch_nand()
745 a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo) { in __tsan_atomic32_fetch_nand() argument
746 SCOPED_ATOMIC(FetchNand, a, v, mo); in __tsan_atomic32_fetch_nand()
750 a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo) { in __tsan_atomic64_fetch_nand() argument
751 SCOPED_ATOMIC(FetchNand, a, v, mo); in __tsan_atomic64_fetch_nand()
756 a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo) { in __tsan_atomic128_fetch_nand() argument
757 SCOPED_ATOMIC(FetchNand, a, v, mo); in __tsan_atomic128_fetch_nand()
763 morder mo, morder fmo) { in __tsan_atomic8_compare_exchange_strong() argument
764 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); in __tsan_atomic8_compare_exchange_strong()
769 morder mo, morder fmo) { in __tsan_atomic16_compare_exchange_strong() argument
770 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); in __tsan_atomic16_compare_exchange_strong()
775 morder mo, morder fmo) { in __tsan_atomic32_compare_exchange_strong() argument
776 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); in __tsan_atomic32_compare_exchange_strong()
781 morder mo, morder fmo) { in __tsan_atomic64_compare_exchange_strong() argument
782 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); in __tsan_atomic64_compare_exchange_strong()
788 morder mo, morder fmo) { in __tsan_atomic128_compare_exchange_strong() argument
789 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); in __tsan_atomic128_compare_exchange_strong()
795 morder mo, morder fmo) { in __tsan_atomic8_compare_exchange_weak() argument
796 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); in __tsan_atomic8_compare_exchange_weak()
801 morder mo, morder fmo) { in __tsan_atomic16_compare_exchange_weak() argument
802 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); in __tsan_atomic16_compare_exchange_weak()
807 morder mo, morder fmo) { in __tsan_atomic32_compare_exchange_weak() argument
808 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); in __tsan_atomic32_compare_exchange_weak()
813 morder mo, morder fmo) { in __tsan_atomic64_compare_exchange_weak() argument
814 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); in __tsan_atomic64_compare_exchange_weak()
820 morder mo, morder fmo) { in __tsan_atomic128_compare_exchange_weak() argument
821 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); in __tsan_atomic128_compare_exchange_weak()
827 morder mo, morder fmo) { in __tsan_atomic8_compare_exchange_val() argument
828 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); in __tsan_atomic8_compare_exchange_val()
833 morder mo, morder fmo) { in __tsan_atomic16_compare_exchange_val() argument
834 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); in __tsan_atomic16_compare_exchange_val()
839 morder mo, morder fmo) { in __tsan_atomic32_compare_exchange_val() argument
840 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); in __tsan_atomic32_compare_exchange_val()
845 morder mo, morder fmo) { in __tsan_atomic64_compare_exchange_val() argument
846 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); in __tsan_atomic64_compare_exchange_val()
852 morder mo, morder fmo) { in __tsan_atomic128_compare_exchange_val() argument
853 SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); in __tsan_atomic128_compare_exchange_val()
858 void __tsan_atomic_thread_fence(morder mo) { in __tsan_atomic_thread_fence() argument
860 SCOPED_ATOMIC(Fence, mo); in __tsan_atomic_thread_fence()
864 void __tsan_atomic_signal_fence(morder mo) { in __tsan_atomic_signal_fence() argument