148 NOINLINE void DoReportRace(ThreadState* thr, RawShadow* shadow_mem, Shadow cur,  in DoReportRace()  argument
166   ReportRace(thr, shadow_mem, cur, Shadow(old), typ);  in DoReportRace()
173 bool ContainsSameAccess(RawShadow* s, Shadow cur, int unused0, int unused1,  in ContainsSameAccess()  argument
178       if (old == cur.raw())  in ContainsSameAccess()
184     if (masked == cur.raw())  in ContainsSameAccess()
195 bool CheckRaces(ThreadState* thr, RawShadow* shadow_mem, Shadow cur,  in CheckRaces()  argument
203         StoreShadow(sp, cur.raw());  in CheckRaces()
206     if (LIKELY(!(cur.access() & old.access())))  in CheckRaces()
208     if (LIKELY(cur.sid() == old.sid())) {  in CheckRaces()
210           LIKELY(cur.access() == old.access() && old.IsRWWeakerOrEqual(typ))) {  in CheckRaces()
211         StoreShadow(sp, cur.raw());  in CheckRaces()
220     DoReportRace(thr, shadow_mem, cur, old, typ);  in CheckRaces()
230   StoreShadow(&shadow_mem[index], cur.raw());  in CheckRaces()
234 #  define LOAD_CURRENT_SHADOW(cur, shadow_mem) UNUSED int access = 0, shadow = 0  argument
274 NOINLINE void DoReportRaceV(ThreadState* thr, RawShadow* shadow_mem, Shadow cur,  in DoReportRaceV()  argument
301   DoReportRace(thr, shadow_mem, cur, prev, typ);  in DoReportRaceV()
305 bool CheckRaces(ThreadState* thr, RawShadow* shadow_mem, Shadow cur,  in CheckRaces()  argument
351   StoreShadow(&shadow_mem[index / 4], cur.raw());  in CheckRaces()
380   DoReportRaceV(thr, shadow_mem, cur, concurrent_mask, shadow, typ);  in CheckRaces()
384 #  define LOAD_CURRENT_SHADOW(cur, shadow_mem)                         \  argument
385     const m128 access = _mm_set1_epi32(static_cast<u32>((cur).raw())); \
433   Shadow cur(fast_state, addr, size, typ);  in MemoryAccess()  local
435   LOAD_CURRENT_SHADOW(cur, shadow_mem);  in MemoryAccess()
436   if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))  in MemoryAccess()
442   CheckRaces(thr, shadow_mem, cur, shadow, access, typ);  in MemoryAccess()
460   Shadow cur(fast_state, 0, 8, typ);  in MemoryAccess16()  local
464     LOAD_CURRENT_SHADOW(cur, shadow_mem);  in MemoryAccess16()
465     if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))  in MemoryAccess16()
470     if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ)))  in MemoryAccess16()
475   LOAD_CURRENT_SHADOW(cur, shadow_mem);  in MemoryAccess16()
476   if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))  in MemoryAccess16()
480   CheckRaces(thr, shadow_mem, cur, shadow, access, typ);  in MemoryAccess16()
501     Shadow cur(fast_state, addr, size1, typ);  in UnalignedMemoryAccess()  local
502     LOAD_CURRENT_SHADOW(cur, shadow_mem);  in UnalignedMemoryAccess()
503     if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))  in UnalignedMemoryAccess()
508     if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ)))  in UnalignedMemoryAccess()
516   Shadow cur(fast_state, 0, size2, typ);  in UnalignedMemoryAccess()  local
517   LOAD_CURRENT_SHADOW(cur, shadow_mem);  in UnalignedMemoryAccess()
518   if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))  in UnalignedMemoryAccess()
522   CheckRaces(thr, shadow_mem, cur, shadow, access, typ);  in UnalignedMemoryAccess()
609   Shadow cur(thr->fast_state, 0, kShadowCell, typ);  in MemoryRangeFreed()  local
611   const m128 access = _mm_set1_epi32(static_cast<u32>(cur.raw()));  in MemoryRangeFreed()
614       static_cast<u32>(Shadow::FreedInfo(cur.sid(), cur.epoch())), 0, 0);  in MemoryRangeFreed()
617     if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ)))  in MemoryRangeFreed()
623     if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, 0, 0, typ)))  in MemoryRangeFreed()
626     StoreShadow(&shadow_mem[1], Shadow::FreedInfo(cur.sid(), cur.epoch()));  in MemoryRangeFreed()
637   Shadow cur(thr->fast_state, 0, 8, kAccessWrite);  in MemoryRangeImitateWrite()  local
638   MemoryRangeSet(addr, size, cur.raw());  in MemoryRangeImitateWrite()
650 bool MemoryAccessRangeOne(ThreadState* thr, RawShadow* shadow_mem, Shadow cur,  in MemoryAccessRangeOne()  argument
652   LOAD_CURRENT_SHADOW(cur, shadow_mem);  in MemoryAccessRangeOne()
653   if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))  in MemoryAccessRangeOne()
655   return CheckRaces(thr, shadow_mem, cur, shadow, access, typ);  in MemoryAccessRangeOne()
715     Shadow cur(fast_state, addr, size1, typ);  in MemoryAccessRangeT()  local
716     if (UNLIKELY(MemoryAccessRangeOne(thr, shadow_mem, cur, typ)))  in MemoryAccessRangeT()
721   Shadow cur(fast_state, 0, kShadowCell, typ);  in MemoryAccessRangeT()  local
723     if (UNLIKELY(MemoryAccessRangeOne(thr, shadow_mem, cur, typ)))  in MemoryAccessRangeT()
728     Shadow cur(fast_state, 0, size, typ);  in MemoryAccessRangeT()  local
729     if (UNLIKELY(MemoryAccessRangeOne(thr, shadow_mem, cur, typ)))  in MemoryAccessRangeT()