Lines Matching refs:shadow_mem
148 NOINLINE void DoReportRace(ThreadState* thr, RawShadow* shadow_mem, Shadow cur, in DoReportRace() argument
155 old = Shadow(LoadShadow(&shadow_mem[1])); in DoReportRace()
158 StoreShadow(&shadow_mem[i], i == 0 ? Shadow::kRodata : Shadow::kEmpty); in DoReportRace()
166 ReportRace(thr, shadow_mem, cur, Shadow(old), typ); in DoReportRace()
195 bool CheckRaces(ThreadState* thr, RawShadow* shadow_mem, Shadow cur, in CheckRaces() argument
199 RawShadow* sp = &shadow_mem[idx]; in CheckRaces()
220 DoReportRace(thr, shadow_mem, cur, old, typ); in CheckRaces()
230 StoreShadow(&shadow_mem[index], cur.raw()); in CheckRaces()
234 # define LOAD_CURRENT_SHADOW(cur, shadow_mem) UNUSED int access = 0, shadow = 0 argument
274 NOINLINE void DoReportRaceV(ThreadState* thr, RawShadow* shadow_mem, Shadow cur, in DoReportRaceV() argument
301 DoReportRace(thr, shadow_mem, cur, prev, typ); in DoReportRaceV()
305 bool CheckRaces(ThreadState* thr, RawShadow* shadow_mem, Shadow cur, in CheckRaces() argument
351 StoreShadow(&shadow_mem[index / 4], cur.raw()); in CheckRaces()
380 DoReportRaceV(thr, shadow_mem, cur, concurrent_mask, shadow, typ); in CheckRaces()
384 # define LOAD_CURRENT_SHADOW(cur, shadow_mem) \ argument
386 const m128 shadow = _mm_load_si128(reinterpret_cast<m128*>(shadow_mem))
422 RawShadow* shadow_mem = MemToShadow(addr); in MemoryAccess() local
427 static_cast<int>(typ), DumpShadow(memBuf[0], shadow_mem[0]), in MemoryAccess()
428 DumpShadow(memBuf[1], shadow_mem[1]), in MemoryAccess()
429 DumpShadow(memBuf[2], shadow_mem[2]), in MemoryAccess()
430 DumpShadow(memBuf[3], shadow_mem[3])); in MemoryAccess()
435 LOAD_CURRENT_SHADOW(cur, shadow_mem); in MemoryAccess()
436 if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ))) in MemoryAccess()
442 CheckRaces(thr, shadow_mem, cur, shadow, access, typ); in MemoryAccess()
461 RawShadow* shadow_mem = MemToShadow(addr); in MemoryAccess16() local
464 LOAD_CURRENT_SHADOW(cur, shadow_mem); in MemoryAccess16()
465 if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ))) in MemoryAccess16()
470 if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ))) in MemoryAccess16()
474 shadow_mem += kShadowCnt; in MemoryAccess16()
475 LOAD_CURRENT_SHADOW(cur, shadow_mem); in MemoryAccess16()
476 if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ))) in MemoryAccess16()
480 CheckRaces(thr, shadow_mem, cur, shadow, access, typ); in MemoryAccess16()
497 RawShadow* shadow_mem = MemToShadow(addr); in UnalignedMemoryAccess() local
502 LOAD_CURRENT_SHADOW(cur, shadow_mem); in UnalignedMemoryAccess()
503 if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ))) in UnalignedMemoryAccess()
508 if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ))) in UnalignedMemoryAccess()
515 shadow_mem += kShadowCnt; in UnalignedMemoryAccess()
517 LOAD_CURRENT_SHADOW(cur, shadow_mem); in UnalignedMemoryAccess()
518 if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ))) in UnalignedMemoryAccess()
522 CheckRaces(thr, shadow_mem, cur, shadow, access, typ); in UnalignedMemoryAccess()
608 RawShadow* shadow_mem = MemToShadow(addr); in MemoryRangeFreed() local
615 for (; size; size -= kShadowCell, shadow_mem += kShadowCnt) { in MemoryRangeFreed()
616 const m128 shadow = _mm_load_si128((m128*)shadow_mem); in MemoryRangeFreed()
617 if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ))) in MemoryRangeFreed()
619 _mm_store_si128((m128*)shadow_mem, freed); in MemoryRangeFreed()
622 for (; size; size -= kShadowCell, shadow_mem += kShadowCnt) { in MemoryRangeFreed()
623 if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, 0, 0, typ))) in MemoryRangeFreed()
625 StoreShadow(&shadow_mem[0], Shadow::FreedMarker()); in MemoryRangeFreed()
626 StoreShadow(&shadow_mem[1], Shadow::FreedInfo(cur.sid(), cur.epoch())); in MemoryRangeFreed()
627 StoreShadow(&shadow_mem[2], Shadow::kEmpty); in MemoryRangeFreed()
628 StoreShadow(&shadow_mem[3], Shadow::kEmpty); in MemoryRangeFreed()
650 bool MemoryAccessRangeOne(ThreadState* thr, RawShadow* shadow_mem, Shadow cur, in MemoryAccessRangeOne() argument
652 LOAD_CURRENT_SHADOW(cur, shadow_mem); in MemoryAccessRangeOne()
653 if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ))) in MemoryAccessRangeOne()
655 return CheckRaces(thr, shadow_mem, cur, shadow, access, typ); in MemoryAccessRangeOne()
669 RawShadow* shadow_mem = MemToShadow(addr); in MemoryAccessRangeT() local
682 if (!IsShadowMem(shadow_mem)) { in MemoryAccessRangeT()
683 Printf("Bad shadow addr %p (%zx)\n", static_cast<void*>(shadow_mem), addr); in MemoryAccessRangeT()
684 DCHECK(IsShadowMem(shadow_mem)); in MemoryAccessRangeT()
686 if (!IsShadowMem(shadow_mem + size * kShadowCnt - 1)) { in MemoryAccessRangeT()
688 static_cast<void*>(shadow_mem + size * kShadowCnt - 1), in MemoryAccessRangeT()
690 DCHECK(IsShadowMem(shadow_mem + size * kShadowCnt - 1)); in MemoryAccessRangeT()
701 if (*shadow_mem == Shadow::kRodata) in MemoryAccessRangeT()
716 if (UNLIKELY(MemoryAccessRangeOne(thr, shadow_mem, cur, typ))) in MemoryAccessRangeT()
718 shadow_mem += kShadowCnt; in MemoryAccessRangeT()
722 for (; size >= kShadowCell; size -= kShadowCell, shadow_mem += kShadowCnt) { in MemoryAccessRangeT()
723 if (UNLIKELY(MemoryAccessRangeOne(thr, shadow_mem, cur, typ))) in MemoryAccessRangeT()
729 if (UNLIKELY(MemoryAccessRangeOne(thr, shadow_mem, cur, typ))) in MemoryAccessRangeT()