234 #  define LOAD_CURRENT_SHADOW(cur, shadow_mem) UNUSED int access = 0, shadow = 0
239 bool ContainsSameAccess(RawShadow* unused0, Shadow unused1, m128 shadow,  in ContainsSameAccess()  argument
248     const m128 same = _mm_cmpeq_epi32(shadow, access);  in ContainsSameAccess()
262   const m128 masked_shadow = _mm_or_si128(shadow, read_mask);  in ContainsSameAccess()
268     const m128 ro = _mm_cmpeq_epi32(shadow, read_mask);  in ContainsSameAccess()
275                             u32 race_mask, m128 shadow, AccessType typ) {  in DoReportRaceV()  argument
283       old = _mm_extract_epi32(shadow, 0);  in DoReportRaceV()
286       old = _mm_extract_epi32(shadow, 1);  in DoReportRaceV()
289       old = _mm_extract_epi32(shadow, 2);  in DoReportRaceV()
292       old = _mm_extract_epi32(shadow, 3);  in DoReportRaceV()
300     prev = Shadow(static_cast<RawShadow>(_mm_extract_epi32(shadow, 1)));  in DoReportRaceV()
306                 m128 shadow, m128 access, AccessType typ) {  in CheckRaces()  argument
312   const m128 access_and = _mm_and_si128(access, shadow);  in CheckRaces()
313   const m128 access_xor = _mm_xor_si128(access, shadow);  in CheckRaces()
340       _mm_cmpeq_epi32(_mm_max_epu32(shadow, access_read_atomic), shadow);  in CheckRaces()
345     const m128 empty = _mm_cmpeq_epi32(shadow, zero);  in CheckRaces()
364       u8 sid = _mm_extract_epi8(shadow, idx * 4 + 1);                         \  in CheckRaces()
374   const m128 shadow_epochs = _mm_and_si128(shadow, mask_epoch);  in CheckRaces()
380   DoReportRaceV(thr, shadow_mem, cur, concurrent_mask, shadow, typ);  in CheckRaces()
386     const m128 shadow = _mm_load_si128(reinterpret_cast<m128*>(shadow_mem))
436   if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))  in MemoryAccess()
442   CheckRaces(thr, shadow_mem, cur, shadow, access, typ);  in MemoryAccess()
465     if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))  in MemoryAccess16()
470     if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ)))  in MemoryAccess16()
476   if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))  in MemoryAccess16()
480   CheckRaces(thr, shadow_mem, cur, shadow, access, typ);  in MemoryAccess16()
503     if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))  in UnalignedMemoryAccess()
508     if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ)))  in UnalignedMemoryAccess()
518   if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))  in UnalignedMemoryAccess()
522   CheckRaces(thr, shadow_mem, cur, shadow, access, typ);  in UnalignedMemoryAccess()
616     const m128 shadow = _mm_load_si128((m128*)shadow_mem);  in MemoryRangeFreed()  local
617     if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ)))  in MemoryRangeFreed()
653   if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))  in MemoryAccessRangeOne()
655   return CheckRaces(thr, shadow_mem, cur, shadow, access, typ);  in MemoryAccessRangeOne()