Lines Matching refs:mo
36 static bool IsLoadOrder(morder mo) { in IsLoadOrder() argument
37 return mo == mo_relaxed || mo == mo_consume in IsLoadOrder()
38 || mo == mo_acquire || mo == mo_seq_cst; in IsLoadOrder()
41 static bool IsStoreOrder(morder mo) { in IsStoreOrder() argument
42 return mo == mo_relaxed || mo == mo_release || mo == mo_seq_cst; in IsStoreOrder()
46 static bool IsReleaseOrder(morder mo) { in IsReleaseOrder() argument
47 return mo == mo_release || mo == mo_acq_rel || mo == mo_seq_cst; in IsReleaseOrder()
50 static bool IsAcquireOrder(morder mo) { in IsAcquireOrder() argument
51 return mo == mo_consume || mo == mo_acquire in IsAcquireOrder()
52 || mo == mo_acq_rel || mo == mo_seq_cst; in IsAcquireOrder()
55 static bool IsAcqRelOrder(morder mo) { in IsAcqRelOrder() argument
56 return mo == mo_acq_rel || mo == mo_seq_cst; in IsAcqRelOrder()
198 static memory_order to_mo(morder mo) { in to_mo() argument
199 switch (mo) { in to_mo()
212 static T NoTsanAtomicLoad(const volatile T *a, morder mo) { in NoTsanAtomicLoad() argument
213 return atomic_load(to_atomic(a), to_mo(mo)); in NoTsanAtomicLoad()
217 static a128 NoTsanAtomicLoad(const volatile a128 *a, morder mo) { in NoTsanAtomicLoad() argument
224 static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a, morder mo) { in AtomicLoad() argument
225 DCHECK(IsLoadOrder(mo)); in AtomicLoad()
228 if (!IsAcquireOrder(mo)) { in AtomicLoad()
231 return NoTsanAtomicLoad(a, mo); in AtomicLoad()
235 T v = NoTsanAtomicLoad(a, mo); in AtomicLoad()
243 v = NoTsanAtomicLoad(a, mo); in AtomicLoad()
250 static void NoTsanAtomicStore(volatile T *a, T v, morder mo) { in NoTsanAtomicStore() argument
251 atomic_store(to_atomic(a), v, to_mo(mo)); in NoTsanAtomicStore()
255 static void NoTsanAtomicStore(volatile a128 *a, a128 v, morder mo) { in NoTsanAtomicStore() argument
263 morder mo) { in AtomicStore() argument
264 DCHECK(IsStoreOrder(mo)); in AtomicStore()
270 if (!IsReleaseOrder(mo)) { in AtomicStore()
271 NoTsanAtomicStore(a, v, mo); in AtomicStore()
279 NoTsanAtomicStore(a, v, mo); in AtomicStore()
285 static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { in AtomicRMW() argument
287 if (LIKELY(mo == mo_relaxed)) in AtomicRMW()
292 RWLock lock(&s->mtx, IsReleaseOrder(mo)); in AtomicRMW()
293 if (IsAcqRelOrder(mo)) in AtomicRMW()
295 else if (IsReleaseOrder(mo)) in AtomicRMW()
297 else if (IsAcquireOrder(mo)) in AtomicRMW()
301 if (IsReleaseOrder(mo)) in AtomicRMW()
307 static T NoTsanAtomicExchange(volatile T *a, T v, morder mo) { in NoTsanAtomicExchange() argument
312 static T NoTsanAtomicFetchAdd(volatile T *a, T v, morder mo) { in NoTsanAtomicFetchAdd() argument
317 static T NoTsanAtomicFetchSub(volatile T *a, T v, morder mo) { in NoTsanAtomicFetchSub() argument
322 static T NoTsanAtomicFetchAnd(volatile T *a, T v, morder mo) { in NoTsanAtomicFetchAnd() argument
327 static T NoTsanAtomicFetchOr(volatile T *a, T v, morder mo) { in NoTsanAtomicFetchOr() argument
332 static T NoTsanAtomicFetchXor(volatile T *a, T v, morder mo) { in NoTsanAtomicFetchXor() argument
337 static T NoTsanAtomicFetchNand(volatile T *a, T v, morder mo) { in NoTsanAtomicFetchNand() argument
343 morder mo) { in AtomicExchange() argument
344 return AtomicRMW<T, func_xchg>(thr, pc, a, v, mo); in AtomicExchange()
349 morder mo) { in AtomicFetchAdd() argument
350 return AtomicRMW<T, func_add>(thr, pc, a, v, mo); in AtomicFetchAdd()
355 morder mo) { in AtomicFetchSub() argument
356 return AtomicRMW<T, func_sub>(thr, pc, a, v, mo); in AtomicFetchSub()
361 morder mo) { in AtomicFetchAnd() argument
362 return AtomicRMW<T, func_and>(thr, pc, a, v, mo); in AtomicFetchAnd()
367 morder mo) { in AtomicFetchOr() argument
368 return AtomicRMW<T, func_or>(thr, pc, a, v, mo); in AtomicFetchOr()
373 morder mo) { in AtomicFetchXor() argument
374 return AtomicRMW<T, func_xor>(thr, pc, a, v, mo); in AtomicFetchXor()
379 morder mo) { in AtomicFetchNand() argument
380 return AtomicRMW<T, func_nand>(thr, pc, a, v, mo); in AtomicFetchNand()
384 static bool NoTsanAtomicCAS(volatile T *a, T *c, T v, morder mo, morder fmo) { in NoTsanAtomicCAS() argument
385 return atomic_compare_exchange_strong(to_atomic(a), c, v, to_mo(mo)); in NoTsanAtomicCAS()
390 morder mo, morder fmo) { in NoTsanAtomicCAS() argument
401 static T NoTsanAtomicCAS(volatile T *a, T c, T v, morder mo, morder fmo) { in NoTsanAtomicCAS() argument
402 NoTsanAtomicCAS(a, &c, v, mo, fmo); in NoTsanAtomicCAS()
408 morder mo, morder fmo) { in AtomicCAS() argument
415 if (LIKELY(mo == mo_relaxed && fmo == mo_relaxed)) { in AtomicCAS()
424 bool release = IsReleaseOrder(mo); in AtomicCAS()
434 mo = fmo; in AtomicCAS()
436 if (success && IsAcqRelOrder(mo)) in AtomicCAS()
438 else if (success && IsReleaseOrder(mo)) in AtomicCAS()
440 else if (IsAcquireOrder(mo)) in AtomicCAS()
450 volatile T *a, T c, T v, morder mo, morder fmo) { in AtomicCAS() argument
451 AtomicCAS(thr, pc, a, &c, v, mo, fmo); in AtomicCAS()
456 static void NoTsanAtomicFence(morder mo) { in NoTsanAtomicFence() argument
460 static void AtomicFence(ThreadState *thr, uptr pc, morder mo) { in AtomicFence() argument
471 static morder convert_morder(morder mo) { in convert_morder() argument
485 return (morder)(mo & 0x7fff); in convert_morder()
493 mo = convert_morder(mo); \
498 a8 __tsan_atomic8_load(const volatile a8 *a, morder mo) { in __tsan_atomic8_load() argument
499 ATOMIC_IMPL(Load, a, mo); in __tsan_atomic8_load()
503 a16 __tsan_atomic16_load(const volatile a16 *a, morder mo) { in __tsan_atomic16_load() argument
504 ATOMIC_IMPL(Load, a, mo); in __tsan_atomic16_load()
508 a32 __tsan_atomic32_load(const volatile a32 *a, morder mo) { in __tsan_atomic32_load() argument
509 ATOMIC_IMPL(Load, a, mo); in __tsan_atomic32_load()
513 a64 __tsan_atomic64_load(const volatile a64 *a, morder mo) { in __tsan_atomic64_load() argument
514 ATOMIC_IMPL(Load, a, mo); in __tsan_atomic64_load()
519 a128 __tsan_atomic128_load(const volatile a128 *a, morder mo) { in __tsan_atomic128_load() argument
520 ATOMIC_IMPL(Load, a, mo); in __tsan_atomic128_load()
525 void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo) { in __tsan_atomic8_store() argument
526 ATOMIC_IMPL(Store, a, v, mo); in __tsan_atomic8_store()
530 void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo) { in __tsan_atomic16_store() argument
531 ATOMIC_IMPL(Store, a, v, mo); in __tsan_atomic16_store()
535 void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo) { in __tsan_atomic32_store() argument
536 ATOMIC_IMPL(Store, a, v, mo); in __tsan_atomic32_store()
540 void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo) { in __tsan_atomic64_store() argument
541 ATOMIC_IMPL(Store, a, v, mo); in __tsan_atomic64_store()
546 void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo) { in __tsan_atomic128_store() argument
547 ATOMIC_IMPL(Store, a, v, mo); in __tsan_atomic128_store()
552 a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo) { in __tsan_atomic8_exchange() argument
553 ATOMIC_IMPL(Exchange, a, v, mo); in __tsan_atomic8_exchange()
557 a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo) { in __tsan_atomic16_exchange() argument
558 ATOMIC_IMPL(Exchange, a, v, mo); in __tsan_atomic16_exchange()
562 a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo) { in __tsan_atomic32_exchange() argument
563 ATOMIC_IMPL(Exchange, a, v, mo); in __tsan_atomic32_exchange()
567 a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo) { in __tsan_atomic64_exchange() argument
568 ATOMIC_IMPL(Exchange, a, v, mo); in __tsan_atomic64_exchange()
573 a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo) { in __tsan_atomic128_exchange() argument
574 ATOMIC_IMPL(Exchange, a, v, mo); in __tsan_atomic128_exchange()
579 a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo) { in __tsan_atomic8_fetch_add() argument
580 ATOMIC_IMPL(FetchAdd, a, v, mo); in __tsan_atomic8_fetch_add()
584 a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo) { in __tsan_atomic16_fetch_add() argument
585 ATOMIC_IMPL(FetchAdd, a, v, mo); in __tsan_atomic16_fetch_add()
589 a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo) { in __tsan_atomic32_fetch_add() argument
590 ATOMIC_IMPL(FetchAdd, a, v, mo); in __tsan_atomic32_fetch_add()
594 a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo) { in __tsan_atomic64_fetch_add() argument
595 ATOMIC_IMPL(FetchAdd, a, v, mo); in __tsan_atomic64_fetch_add()
600 a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo) { in __tsan_atomic128_fetch_add() argument
601 ATOMIC_IMPL(FetchAdd, a, v, mo); in __tsan_atomic128_fetch_add()
606 a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo) { in __tsan_atomic8_fetch_sub() argument
607 ATOMIC_IMPL(FetchSub, a, v, mo); in __tsan_atomic8_fetch_sub()
611 a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo) { in __tsan_atomic16_fetch_sub() argument
612 ATOMIC_IMPL(FetchSub, a, v, mo); in __tsan_atomic16_fetch_sub()
616 a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo) { in __tsan_atomic32_fetch_sub() argument
617 ATOMIC_IMPL(FetchSub, a, v, mo); in __tsan_atomic32_fetch_sub()
621 a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo) { in __tsan_atomic64_fetch_sub() argument
622 ATOMIC_IMPL(FetchSub, a, v, mo); in __tsan_atomic64_fetch_sub()
627 a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo) { in __tsan_atomic128_fetch_sub() argument
628 ATOMIC_IMPL(FetchSub, a, v, mo); in __tsan_atomic128_fetch_sub()
633 a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo) { in __tsan_atomic8_fetch_and() argument
634 ATOMIC_IMPL(FetchAnd, a, v, mo); in __tsan_atomic8_fetch_and()
638 a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo) { in __tsan_atomic16_fetch_and() argument
639 ATOMIC_IMPL(FetchAnd, a, v, mo); in __tsan_atomic16_fetch_and()
643 a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo) { in __tsan_atomic32_fetch_and() argument
644 ATOMIC_IMPL(FetchAnd, a, v, mo); in __tsan_atomic32_fetch_and()
648 a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo) { in __tsan_atomic64_fetch_and() argument
649 ATOMIC_IMPL(FetchAnd, a, v, mo); in __tsan_atomic64_fetch_and()
654 a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo) { in __tsan_atomic128_fetch_and() argument
655 ATOMIC_IMPL(FetchAnd, a, v, mo); in __tsan_atomic128_fetch_and()
660 a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo) { in __tsan_atomic8_fetch_or() argument
661 ATOMIC_IMPL(FetchOr, a, v, mo); in __tsan_atomic8_fetch_or()
665 a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo) { in __tsan_atomic16_fetch_or() argument
666 ATOMIC_IMPL(FetchOr, a, v, mo); in __tsan_atomic16_fetch_or()
670 a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo) { in __tsan_atomic32_fetch_or() argument
671 ATOMIC_IMPL(FetchOr, a, v, mo); in __tsan_atomic32_fetch_or()
675 a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo) { in __tsan_atomic64_fetch_or() argument
676 ATOMIC_IMPL(FetchOr, a, v, mo); in __tsan_atomic64_fetch_or()
681 a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo) { in __tsan_atomic128_fetch_or() argument
682 ATOMIC_IMPL(FetchOr, a, v, mo); in __tsan_atomic128_fetch_or()
687 a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo) { in __tsan_atomic8_fetch_xor() argument
688 ATOMIC_IMPL(FetchXor, a, v, mo); in __tsan_atomic8_fetch_xor()
692 a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo) { in __tsan_atomic16_fetch_xor() argument
693 ATOMIC_IMPL(FetchXor, a, v, mo); in __tsan_atomic16_fetch_xor()
697 a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo) { in __tsan_atomic32_fetch_xor() argument
698 ATOMIC_IMPL(FetchXor, a, v, mo); in __tsan_atomic32_fetch_xor()
702 a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo) { in __tsan_atomic64_fetch_xor() argument
703 ATOMIC_IMPL(FetchXor, a, v, mo); in __tsan_atomic64_fetch_xor()
708 a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo) { in __tsan_atomic128_fetch_xor() argument
709 ATOMIC_IMPL(FetchXor, a, v, mo); in __tsan_atomic128_fetch_xor()
714 a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo) { in __tsan_atomic8_fetch_nand() argument
715 ATOMIC_IMPL(FetchNand, a, v, mo); in __tsan_atomic8_fetch_nand()
719 a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo) { in __tsan_atomic16_fetch_nand() argument
720 ATOMIC_IMPL(FetchNand, a, v, mo); in __tsan_atomic16_fetch_nand()
724 a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo) { in __tsan_atomic32_fetch_nand() argument
725 ATOMIC_IMPL(FetchNand, a, v, mo); in __tsan_atomic32_fetch_nand()
729 a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo) { in __tsan_atomic64_fetch_nand() argument
730 ATOMIC_IMPL(FetchNand, a, v, mo); in __tsan_atomic64_fetch_nand()
735 a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo) { in __tsan_atomic128_fetch_nand() argument
736 ATOMIC_IMPL(FetchNand, a, v, mo); in __tsan_atomic128_fetch_nand()
742 morder mo, morder fmo) { in __tsan_atomic8_compare_exchange_strong() argument
743 ATOMIC_IMPL(CAS, a, c, v, mo, fmo); in __tsan_atomic8_compare_exchange_strong()
748 morder mo, morder fmo) { in __tsan_atomic16_compare_exchange_strong() argument
749 ATOMIC_IMPL(CAS, a, c, v, mo, fmo); in __tsan_atomic16_compare_exchange_strong()
754 morder mo, morder fmo) { in __tsan_atomic32_compare_exchange_strong() argument
755 ATOMIC_IMPL(CAS, a, c, v, mo, fmo); in __tsan_atomic32_compare_exchange_strong()
760 morder mo, morder fmo) { in __tsan_atomic64_compare_exchange_strong() argument
761 ATOMIC_IMPL(CAS, a, c, v, mo, fmo); in __tsan_atomic64_compare_exchange_strong()
767 morder mo, morder fmo) { in __tsan_atomic128_compare_exchange_strong() argument
768 ATOMIC_IMPL(CAS, a, c, v, mo, fmo); in __tsan_atomic128_compare_exchange_strong()
774 morder mo, morder fmo) { in __tsan_atomic8_compare_exchange_weak() argument
775 ATOMIC_IMPL(CAS, a, c, v, mo, fmo); in __tsan_atomic8_compare_exchange_weak()
780 morder mo, morder fmo) { in __tsan_atomic16_compare_exchange_weak() argument
781 ATOMIC_IMPL(CAS, a, c, v, mo, fmo); in __tsan_atomic16_compare_exchange_weak()
786 morder mo, morder fmo) { in __tsan_atomic32_compare_exchange_weak() argument
787 ATOMIC_IMPL(CAS, a, c, v, mo, fmo); in __tsan_atomic32_compare_exchange_weak()
792 morder mo, morder fmo) { in __tsan_atomic64_compare_exchange_weak() argument
793 ATOMIC_IMPL(CAS, a, c, v, mo, fmo); in __tsan_atomic64_compare_exchange_weak()
799 morder mo, morder fmo) { in __tsan_atomic128_compare_exchange_weak() argument
800 ATOMIC_IMPL(CAS, a, c, v, mo, fmo); in __tsan_atomic128_compare_exchange_weak()
806 morder mo, morder fmo) { in __tsan_atomic8_compare_exchange_val() argument
807 ATOMIC_IMPL(CAS, a, c, v, mo, fmo); in __tsan_atomic8_compare_exchange_val()
812 morder mo, morder fmo) { in __tsan_atomic16_compare_exchange_val() argument
813 ATOMIC_IMPL(CAS, a, c, v, mo, fmo); in __tsan_atomic16_compare_exchange_val()
818 morder mo, morder fmo) { in __tsan_atomic32_compare_exchange_val() argument
819 ATOMIC_IMPL(CAS, a, c, v, mo, fmo); in __tsan_atomic32_compare_exchange_val()
824 morder mo, morder fmo) { in __tsan_atomic64_compare_exchange_val() argument
825 ATOMIC_IMPL(CAS, a, c, v, mo, fmo); in __tsan_atomic64_compare_exchange_val()
831 morder mo, morder fmo) { in __tsan_atomic128_compare_exchange_val() argument
832 ATOMIC_IMPL(CAS, a, c, v, mo, fmo); in __tsan_atomic128_compare_exchange_val()
837 void __tsan_atomic_thread_fence(morder mo) { ATOMIC_IMPL(Fence, mo); } in __tsan_atomic_thread_fence() argument
840 void __tsan_atomic_signal_fence(morder mo) { in __tsan_atomic_signal_fence() argument