Lines Matching refs:IRB
775 IRBuilder<> IRB(*C); in createKernelApi() local
788 IRB.getVoidTy(), IRB.getInt32Ty()); in createKernelApi()
793 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), in createKernelApi()
794 ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8), in createKernelApi()
795 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), in createKernelApi()
796 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), /* va_arg_origin */ in createKernelApi()
797 IRB.getInt64Ty(), ArrayType::get(OriginTy, kParamTLSSize / 4), OriginTy, in createKernelApi()
802 MsanMetadata = StructType::get(PointerType::get(IRB.getInt8Ty(), 0), in createKernelApi()
803 PointerType::get(IRB.getInt32Ty(), 0)); in createKernelApi()
811 M, name_load, PointerType::get(IRB.getInt8Ty(), 0)); in createKernelApi()
813 M, name_store, PointerType::get(IRB.getInt8Ty(), 0)); in createKernelApi()
817 M, "__msan_metadata_ptr_for_load_n", PointerType::get(IRB.getInt8Ty(), 0), in createKernelApi()
818 IRB.getInt64Ty()); in createKernelApi()
821 PointerType::get(IRB.getInt8Ty(), 0), IRB.getInt64Ty()); in createKernelApi()
825 "__msan_poison_alloca", IRB.getVoidTy(), PtrTy, IntptrTy, PtrTy); in createKernelApi()
827 "__msan_unpoison_alloca", IRB.getVoidTy(), PtrTy, IntptrTy); in createKernelApi()
840 IRBuilder<> IRB(*C); in createUserspaceApi() local
850 IRB.getVoidTy(), IRB.getInt32Ty()); in createUserspaceApi()
854 WarningFn = M.getOrInsertFunction(WarningFnName, IRB.getVoidTy()); in createUserspaceApi()
860 ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8)); in createUserspaceApi()
866 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8)); in createUserspaceApi()
874 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8)); in createUserspaceApi()
881 getOrInsertGlobal(M, "__msan_va_arg_overflow_size_tls", IRB.getInt64Ty()); in createUserspaceApi()
889 IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), IRB.getInt32Ty()); in createUserspaceApi()
894 IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), PtrTy, in createUserspaceApi()
895 IRB.getInt32Ty()); in createUserspaceApi()
900 IRB.getVoidTy(), PtrTy, IntptrTy, PtrTy, PtrTy); in createUserspaceApi()
903 IRB.getVoidTy(), PtrTy, IntptrTy, PtrTy); in createUserspaceApi()
905 IRB.getVoidTy(), PtrTy, IntptrTy); in createUserspaceApi()
914 IRBuilder<> IRB(*C); in initializeCallbacks() local
919 TLI.getAttrList(C, {0}, /*Signed=*/false, /*Ret=*/true), IRB.getInt32Ty(), in initializeCallbacks()
920 IRB.getInt32Ty()); in initializeCallbacks()
923 IRB.getVoidTy(), PtrTy, IntptrTy, IRB.getInt32Ty()); in initializeCallbacks()
930 PtrTy, PtrTy, IRB.getInt32Ty(), IntptrTy); in initializeCallbacks()
933 M.getOrInsertFunction("__msan_instrument_asm_store", IRB.getVoidTy(), in initializeCallbacks()
934 PointerType::get(IRB.getInt8Ty(), 0), IntptrTy); in initializeCallbacks()
1041 IRBuilder<> IRB(*C); in initializeModule() local
1042 IntptrTy = IRB.getIntPtrTy(DL); in initializeModule()
1043 OriginTy = IRB.getInt32Ty(); in initializeModule()
1044 PtrTy = IRB.getPtrTy(); in initializeModule()
1051 M.getOrInsertGlobal("__msan_track_origins", IRB.getInt32Ty(), [&] { in initializeModule()
1053 M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage, in initializeModule()
1054 IRB.getInt32(TrackOrigins), "__msan_track_origins"); in initializeModule()
1058 M.getOrInsertGlobal("__msan_keep_going", IRB.getInt32Ty(), [&] { in initializeModule()
1059 return new GlobalVariable(M, IRB.getInt32Ty(), true, in initializeModule()
1061 IRB.getInt32(Recover), "__msan_keep_going"); in initializeModule()
1080 virtual void visitCallBase(CallBase &CB, IRBuilder<> &IRB) = 0;
1182 IRBuilder<> IRB(FnPrologueEnd); in MemorySanitizerVisitor() local
1183 insertKmsanPrologue(IRB); in MemorySanitizerVisitor()
1210 Value *updateOrigin(Value *V, IRBuilder<> &IRB) { in updateOrigin()
1213 return IRB.CreateCall(MS.MsanChainOriginFn, V); in updateOrigin()
1216 Value *originToIntptr(IRBuilder<> &IRB, Value *Origin) { in originToIntptr()
1222 Origin = IRB.CreateIntCast(Origin, MS.IntptrTy, /* isSigned */ false); in originToIntptr()
1223 return IRB.CreateOr(Origin, IRB.CreateShl(Origin, kOriginSize * 8)); in originToIntptr()
1227 void paintOrigin(IRBuilder<> &IRB, Value *Origin, Value *OriginPtr, in paintOrigin()
1238 Value *Size = IRB.CreateTypeSize(IRB.getInt32Ty(), TS); in paintOrigin()
1239 Value *RoundUp = IRB.CreateAdd(Size, IRB.getInt32(kOriginSize - 1)); in paintOrigin()
1240 Value *End = IRB.CreateUDiv(RoundUp, IRB.getInt32(kOriginSize)); in paintOrigin()
1242 SplitBlockAndInsertSimpleForLoop(End, &*IRB.GetInsertPoint()); in paintOrigin()
1243 IRB.SetInsertPoint(InsertPt); in paintOrigin()
1245 Value *GEP = IRB.CreateGEP(MS.OriginTy, OriginPtr, Index); in paintOrigin()
1246 IRB.CreateAlignedStore(Origin, GEP, kMinOriginAlignment); in paintOrigin()
1255 Value *IntptrOrigin = originToIntptr(IRB, Origin); in paintOrigin()
1257 IRB.CreatePointerCast(OriginPtr, PointerType::get(MS.IntptrTy, 0)); in paintOrigin()
1259 Value *Ptr = i ? IRB.CreateConstGEP1_32(MS.IntptrTy, IntptrOriginPtr, i) in paintOrigin()
1261 IRB.CreateAlignedStore(IntptrOrigin, Ptr, CurrentAlignment); in paintOrigin()
1269 i ? IRB.CreateConstGEP1_32(MS.OriginTy, OriginPtr, i) : OriginPtr; in paintOrigin()
1270 IRB.CreateAlignedStore(Origin, GEP, CurrentAlignment); in paintOrigin()
1275 void storeOrigin(IRBuilder<> &IRB, Value *Addr, Value *Shadow, Value *Origin, in storeOrigin()
1280 Value *ConvertedShadow = convertShadowToScalar(Shadow, IRB); in storeOrigin()
1289 paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize, in storeOrigin()
1302 IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex))); in storeOrigin()
1303 CallBase *CB = IRB.CreateCall(Fn, {ConvertedShadow2, Addr, Origin}); in storeOrigin()
1307 Value *Cmp = convertToBool(ConvertedShadow, IRB, "_mscmp"); in storeOrigin()
1309 Cmp, &*IRB.GetInsertPoint(), false, MS.OriginStoreWeights); in storeOrigin()
1318 IRBuilder<> IRB(SI); in materializeStores() local
1327 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ true); in materializeStores()
1329 StoreInst *NewSI = IRB.CreateAlignedStore(Shadow, ShadowPtr, Alignment); in materializeStores()
1337 storeOrigin(IRB, Addr, Shadow, getOrigin(Val), OriginPtr, in materializeStores()
1355 void insertWarningFn(IRBuilder<> &IRB, Value *Origin) { in insertWarningFn()
1357 Origin = (Value *)IRB.getInt32(0); in insertWarningFn()
1360 if (shouldDisambiguateWarningLocation(IRB.getCurrentDebugLocation())) { in insertWarningFn()
1368 if (NewDebugLoc && NewDebugLoc != IRB.getCurrentDebugLocation()) { in insertWarningFn()
1371 IRBuilder<> IRBOrigin(&*IRB.GetInsertPoint()); in insertWarningFn()
1379 IRB.CreateCall(MS.WarningFn, Origin)->setCannotMerge(); in insertWarningFn()
1381 IRB.CreateCall(MS.WarningFn)->setCannotMerge(); in insertWarningFn()
1387 void materializeOneCheck(IRBuilder<> &IRB, Value *ConvertedShadow, in materializeOneCheck()
1396 IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex))); in materializeOneCheck()
1397 CallBase *CB = IRB.CreateCall( in materializeOneCheck()
1399 MS.TrackOrigins && Origin ? Origin : (Value *)IRB.getInt32(0)}); in materializeOneCheck()
1403 Value *Cmp = convertToBool(ConvertedShadow, IRB, "_mscmp"); in materializeOneCheck()
1405 Cmp, &*IRB.GetInsertPoint(), in materializeOneCheck()
1408 IRB.SetInsertPoint(CheckTerm); in materializeOneCheck()
1409 insertWarningFn(IRB, Origin); in materializeOneCheck()
1424 IRBuilder<> IRB(Instruction); in materializeInstructionChecks() local
1435 insertWarningFn(IRB, ShadowData.Origin); in materializeInstructionChecks()
1445 materializeOneCheck(IRB, ConvertedShadow, ShadowData.Origin); in materializeInstructionChecks()
1454 Shadow = convertToBool(Shadow, IRB, "_mscmp"); in materializeInstructionChecks()
1455 ConvertedShadow = convertToBool(ConvertedShadow, IRB, "_mscmp"); in materializeInstructionChecks()
1456 Shadow = IRB.CreateOr(Shadow, ConvertedShadow, "_msor"); in materializeInstructionChecks()
1461 IRBuilder<> IRB(Instruction); in materializeInstructionChecks() local
1462 materializeOneCheck(IRB, Shadow, nullptr); in materializeInstructionChecks()
1489 void insertKmsanPrologue(IRBuilder<> &IRB) { in insertKmsanPrologue()
1490 Value *ContextState = IRB.CreateCall(MS.MsanGetContextStateFn, {}); in insertKmsanPrologue()
1491 Constant *Zero = IRB.getInt32(0); in insertKmsanPrologue()
1492 MS.ParamTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState, in insertKmsanPrologue()
1493 {Zero, IRB.getInt32(0)}, "param_shadow"); in insertKmsanPrologue()
1494 MS.RetvalTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState, in insertKmsanPrologue()
1495 {Zero, IRB.getInt32(1)}, "retval_shadow"); in insertKmsanPrologue()
1496 MS.VAArgTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState, in insertKmsanPrologue()
1497 {Zero, IRB.getInt32(2)}, "va_arg_shadow"); in insertKmsanPrologue()
1498 MS.VAArgOriginTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState, in insertKmsanPrologue()
1499 {Zero, IRB.getInt32(3)}, "va_arg_origin"); in insertKmsanPrologue()
1501 IRB.CreateGEP(MS.MsanContextStateTy, ContextState, in insertKmsanPrologue()
1502 {Zero, IRB.getInt32(4)}, "va_arg_overflow_size"); in insertKmsanPrologue()
1503 MS.ParamOriginTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState, in insertKmsanPrologue()
1504 {Zero, IRB.getInt32(5)}, "param_origin"); in insertKmsanPrologue()
1506 IRB.CreateGEP(MS.MsanContextStateTy, ContextState, in insertKmsanPrologue()
1507 {Zero, IRB.getInt32(6)}, "retval_origin"); in insertKmsanPrologue()
1509 MS.MsanMetadataAlloca = IRB.CreateAlloca(MS.MsanMetadata, 0u); in insertKmsanPrologue()
1593 IRBuilder<> &IRB) { in collapseStructShadow()
1594 Value *FalseVal = IRB.getIntN(/* width */ 1, /* value */ 0); in collapseStructShadow()
1599 Value *ShadowItem = IRB.CreateExtractValue(Shadow, Idx); in collapseStructShadow()
1600 Value *ShadowBool = convertToBool(ShadowItem, IRB); in collapseStructShadow()
1603 Aggregator = IRB.CreateOr(Aggregator, ShadowBool); in collapseStructShadow()
1613 IRBuilder<> &IRB) { in collapseArrayShadow()
1615 return IRB.getIntN(/* width */ 1, /* value */ 0); in collapseArrayShadow()
1617 Value *FirstItem = IRB.CreateExtractValue(Shadow, 0); in collapseArrayShadow()
1618 Value *Aggregator = convertShadowToScalar(FirstItem, IRB); in collapseArrayShadow()
1621 Value *ShadowItem = IRB.CreateExtractValue(Shadow, Idx); in collapseArrayShadow()
1622 Value *ShadowInner = convertShadowToScalar(ShadowItem, IRB); in collapseArrayShadow()
1623 Aggregator = IRB.CreateOr(Aggregator, ShadowInner); in collapseArrayShadow()
1631 Value *convertShadowToScalar(Value *V, IRBuilder<> &IRB) { in convertShadowToScalar()
1633 return collapseStructShadow(Struct, V, IRB); in convertShadowToScalar()
1635 return collapseArrayShadow(Array, V, IRB); in convertShadowToScalar()
1638 return convertShadowToScalar(IRB.CreateOrReduce(V), IRB); in convertShadowToScalar()
1641 return IRB.CreateBitCast(V, IntegerType::get(*MS.C, BitWidth)); in convertShadowToScalar()
1647 Value *convertToBool(Value *V, IRBuilder<> &IRB, const Twine &name = "") { in convertToBool()
1650 return convertToBool(convertShadowToScalar(V, IRB), IRB, name); in convertToBool()
1654 return IRB.CreateICmpNE(V, ConstantInt::get(VTy, 0), name); in convertToBool()
1692 Value *getShadowPtrOffset(Value *Addr, IRBuilder<> &IRB) { in getShadowPtrOffset()
1694 Value *OffsetLong = IRB.CreatePointerCast(Addr, IntptrTy); in getShadowPtrOffset()
1697 OffsetLong = IRB.CreateAnd(OffsetLong, constToIntPtr(IntptrTy, ~AndMask)); in getShadowPtrOffset()
1700 OffsetLong = IRB.CreateXor(OffsetLong, constToIntPtr(IntptrTy, XorMask)); in getShadowPtrOffset()
1713 getShadowOriginPtrUserspace(Value *Addr, IRBuilder<> &IRB, Type *ShadowTy, in getShadowOriginPtrUserspace()
1722 Value *ShadowOffset = getShadowPtrOffset(Addr, IRB); in getShadowOriginPtrUserspace()
1726 IRB.CreateAdd(ShadowLong, constToIntPtr(IntptrTy, ShadowBase)); in getShadowOriginPtrUserspace()
1728 Value *ShadowPtr = IRB.CreateIntToPtr( in getShadowOriginPtrUserspace()
1737 IRB.CreateAdd(OriginLong, constToIntPtr(IntptrTy, OriginBase)); in getShadowOriginPtrUserspace()
1740 OriginLong = IRB.CreateAnd(OriginLong, constToIntPtr(IntptrTy, ~Mask)); in getShadowOriginPtrUserspace()
1742 OriginPtr = IRB.CreateIntToPtr( in getShadowOriginPtrUserspace()
1749 Value *createMetadataCall(IRBuilder<> &IRB, FunctionCallee Callee, in createMetadataCall()
1752 IRB.CreateCall(Callee, in createMetadataCall()
1754 return IRB.CreateLoad(MS.MsanMetadata, MS.MsanMetadataAlloca); in createMetadataCall()
1757 return IRB.CreateCall(Callee, {std::forward<ArgsTy>(Args)...}); in createMetadataCall()
1761 IRBuilder<> &IRB, in getShadowOriginPtrKernelNoVec()
1770 IRB.CreatePointerCast(Addr, PointerType::get(IRB.getInt8Ty(), 0)); in getShadowOriginPtrKernelNoVec()
1772 ShadowOriginPtrs = createMetadataCall(IRB, Getter, AddrCast); in getShadowOriginPtrKernelNoVec()
1776 IRB, in getShadowOriginPtrKernelNoVec()
1780 Value *ShadowPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 0); in getShadowOriginPtrKernelNoVec()
1781 ShadowPtr = IRB.CreatePointerCast(ShadowPtr, PointerType::get(ShadowTy, 0)); in getShadowOriginPtrKernelNoVec()
1782 Value *OriginPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 1); in getShadowOriginPtrKernelNoVec()
1791 IRBuilder<> &IRB, in getShadowOriginPtrKernel()
1797 return getShadowOriginPtrKernelNoVec(Addr, IRB, ShadowTy, isStore); in getShadowOriginPtrKernel()
1803 FixedVectorType::get(IRB.getPtrTy(), NumElements)); in getShadowOriginPtrKernel()
1807 FixedVectorType::get(IRB.getPtrTy(), NumElements)); in getShadowOriginPtrKernel()
1810 IRB.CreateExtractElement(Addr, ConstantInt::get(IRB.getInt32Ty(), i)); in getShadowOriginPtrKernel()
1812 getShadowOriginPtrKernelNoVec(OneAddr, IRB, ShadowTy, isStore); in getShadowOriginPtrKernel()
1814 ShadowPtrs = IRB.CreateInsertElement( in getShadowOriginPtrKernel()
1815 ShadowPtrs, ShadowPtr, ConstantInt::get(IRB.getInt32Ty(), i)); in getShadowOriginPtrKernel()
1817 OriginPtrs = IRB.CreateInsertElement( in getShadowOriginPtrKernel()
1818 OriginPtrs, OriginPtr, ConstantInt::get(IRB.getInt32Ty(), i)); in getShadowOriginPtrKernel()
1823 std::pair<Value *, Value *> getShadowOriginPtr(Value *Addr, IRBuilder<> &IRB, in getShadowOriginPtr()
1828 return getShadowOriginPtrKernel(Addr, IRB, ShadowTy, isStore); in getShadowOriginPtr()
1829 return getShadowOriginPtrUserspace(Addr, IRB, ShadowTy, Alignment); in getShadowOriginPtr()
1835 Value *getShadowPtrForArgument(IRBuilder<> &IRB, int ArgOffset) { in getShadowPtrForArgument()
1836 Value *Base = IRB.CreatePointerCast(MS.ParamTLS, MS.IntptrTy); in getShadowPtrForArgument()
1838 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset)); in getShadowPtrForArgument()
1839 return IRB.CreateIntToPtr(Base, IRB.getPtrTy(0), "_msarg"); in getShadowPtrForArgument()
1843 Value *getOriginPtrForArgument(IRBuilder<> &IRB, int ArgOffset) { in getOriginPtrForArgument()
1846 Value *Base = IRB.CreatePointerCast(MS.ParamOriginTLS, MS.IntptrTy); in getOriginPtrForArgument()
1848 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset)); in getOriginPtrForArgument()
1849 return IRB.CreateIntToPtr(Base, IRB.getPtrTy(0), "_msarg_o"); in getOriginPtrForArgument()
1853 Value *getShadowPtrForRetval(IRBuilder<> &IRB) { in getShadowPtrForRetval()
1854 return IRB.CreatePointerCast(MS.RetvalTLS, IRB.getPtrTy(0), "_msret"); in getShadowPtrForRetval()
2123 Value *makeAddReleaseOrderingTable(IRBuilder<> &IRB) { in makeAddReleaseOrderingTable()
2137 return ConstantDataVector::get(IRB.getContext(), in makeAddReleaseOrderingTable()
2158 Value *makeAddAcquireOrderingTable(IRBuilder<> &IRB) { in makeAddAcquireOrderingTable()
2172 return ConstantDataVector::get(IRB.getContext(), in makeAddAcquireOrderingTable()
2194 NextNodeIRBuilder IRB(&I); in visitLoadInst() local
2201 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false); in visitLoadInst()
2203 IRB.CreateAlignedLoad(ShadowTy, ShadowPtr, Alignment, "_msld")); in visitLoadInst()
2218 &I, IRB.CreateAlignedLoad(MS.OriginTy, OriginPtr, OriginAlignment)); in visitLoadInst()
2238 IRBuilder<> IRB(&I); in handleCASOrRMW() local
2241 Value *ShadowPtr = getShadowOriginPtr(Addr, IRB, getShadowTy(Val), Align(1), in handleCASOrRMW()
2254 IRB.CreateStore(getCleanShadow(Val), ShadowPtr); in handleCASOrRMW()
2273 IRBuilder<> IRB(&I); in visitExtractElementInst() local
2274 setShadow(&I, IRB.CreateExtractElement(getShadow(&I, 0), I.getOperand(1), in visitExtractElementInst()
2281 IRBuilder<> IRB(&I); in visitInsertElementInst() local
2284 setShadow(&I, IRB.CreateInsertElement(Shadow0, Shadow1, I.getOperand(2), in visitInsertElementInst()
2290 IRBuilder<> IRB(&I); in visitShuffleVectorInst() local
2293 setShadow(&I, IRB.CreateShuffleVector(Shadow0, Shadow1, I.getShuffleMask(), in visitShuffleVectorInst()
2300 IRBuilder<> IRB(&I); in visitSExtInst() local
2301 setShadow(&I, IRB.CreateSExt(getShadow(&I, 0), I.getType(), "_msprop")); in visitSExtInst()
2306 IRBuilder<> IRB(&I); in visitZExtInst() local
2307 setShadow(&I, IRB.CreateZExt(getShadow(&I, 0), I.getType(), "_msprop")); in visitZExtInst()
2312 IRBuilder<> IRB(&I); in visitTruncInst() local
2313 setShadow(&I, IRB.CreateTrunc(getShadow(&I, 0), I.getType(), "_msprop")); in visitTruncInst()
2324 IRBuilder<> IRB(&I); in visitBitCastInst() local
2325 setShadow(&I, IRB.CreateBitCast(getShadow(&I, 0), getShadowTy(&I))); in visitBitCastInst()
2330 IRBuilder<> IRB(&I); in visitPtrToIntInst() local
2331 setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false, in visitPtrToIntInst()
2337 IRBuilder<> IRB(&I); in visitIntToPtrInst() local
2338 setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false, in visitIntToPtrInst()
2356 IRBuilder<> IRB(&I); in visitAnd() local
2367 V1 = IRB.CreateIntCast(V1, S1->getType(), false); in visitAnd()
2368 V2 = IRB.CreateIntCast(V2, S2->getType(), false); in visitAnd()
2370 Value *S1S2 = IRB.CreateAnd(S1, S2); in visitAnd()
2371 Value *V1S2 = IRB.CreateAnd(V1, S2); in visitAnd()
2372 Value *S1V2 = IRB.CreateAnd(S1, V2); in visitAnd()
2373 setShadow(&I, IRB.CreateOr({S1S2, V1S2, S1V2})); in visitAnd()
2378 IRBuilder<> IRB(&I); in visitOr() local
2386 Value *V1 = IRB.CreateNot(I.getOperand(0)); in visitOr()
2387 Value *V2 = IRB.CreateNot(I.getOperand(1)); in visitOr()
2389 V1 = IRB.CreateIntCast(V1, S1->getType(), false); in visitOr()
2390 V2 = IRB.CreateIntCast(V2, S2->getType(), false); in visitOr()
2392 Value *S1S2 = IRB.CreateAnd(S1, S2); in visitOr()
2393 Value *V1S2 = IRB.CreateAnd(V1, S2); in visitOr()
2394 Value *S1V2 = IRB.CreateAnd(S1, V2); in visitOr()
2395 setShadow(&I, IRB.CreateOr({S1S2, V1S2, S1V2})); in visitOr()
2417 IRBuilder<> &IRB; member in __anonf93d87790811::MemorySanitizerVisitor::Combiner
2421 Combiner(MemorySanitizerVisitor *MSV, IRBuilder<> &IRB) in Combiner() argument
2422 : IRB(IRB), MSV(MSV) {} in Combiner()
2431 OpShadow = MSV->CreateShadowCast(IRB, OpShadow, Shadow->getType()); in Add()
2432 Shadow = IRB.CreateOr(Shadow, OpShadow, "_msprop"); in Add()
2444 Value *Cond = MSV->convertToBool(OpShadow, IRB); in Add()
2445 Origin = IRB.CreateSelect(Cond, OpOrigin, Origin); in Add()
2464 Shadow = MSV->CreateShadowCast(IRB, Shadow, MSV->getShadowTy(I)); in Done()
2481 IRBuilder<> IRB(&I); in setOriginForNaryOp() local
2482 OriginCombiner OC(this, IRB); in setOriginForNaryOp()
2498 Value *CreateShadowCast(IRBuilder<> &IRB, Value *V, Type *dstTy, in CreateShadowCast()
2504 return IRB.CreateICmpNE(V, getCleanShadow(V)); in CreateShadowCast()
2507 return IRB.CreateIntCast(V, dstTy, Signed); in CreateShadowCast()
2511 return IRB.CreateIntCast(V, dstTy, Signed); in CreateShadowCast()
2512 Value *V1 = IRB.CreateBitCast(V, Type::getIntNTy(*MS.C, srcSizeInBits)); in CreateShadowCast()
2514 IRB.CreateIntCast(V1, Type::getIntNTy(*MS.C, dstSizeInBits), Signed); in CreateShadowCast()
2515 return IRB.CreateBitCast(V2, dstTy); in CreateShadowCast()
2520 Value *CreateAppToShadowCast(IRBuilder<> &IRB, Value *V) { in CreateAppToShadowCast()
2525 return IRB.CreatePtrToInt(V, ShadowTy); in CreateAppToShadowCast()
2527 return IRB.CreateBitCast(V, ShadowTy); in CreateAppToShadowCast()
2532 IRBuilder<> IRB(&I); in handleShadowOr() local
2533 ShadowAndOriginCombiner SC(this, IRB); in handleShadowOr()
2579 IRBuilder<> IRB(&I); in handleMulByConstant() local
2581 IRB.CreateMul(getShadow(OtherArg), ShadowMul, "msprop_mul_cst")); in handleMulByConstant()
2604 IRBuilder<> IRB(&I); in handleIntegerDiv() local
2626 IRBuilder<> IRB(&I); in handleEqualityComparison() local
2635 A = IRB.CreatePointerCast(A, Sa->getType()); in handleEqualityComparison()
2636 B = IRB.CreatePointerCast(B, Sb->getType()); in handleEqualityComparison()
2641 Value *C = IRB.CreateXor(A, B); in handleEqualityComparison()
2642 Value *Sc = IRB.CreateOr(Sa, Sb); in handleEqualityComparison()
2650 Value *LHS = IRB.CreateICmpNE(Sc, Zero); in handleEqualityComparison()
2652 IRB.CreateICmpEQ(IRB.CreateAnd(IRB.CreateXor(Sc, MinusOne), C), Zero); in handleEqualityComparison()
2653 Value *Si = IRB.CreateAnd(LHS, RHS); in handleEqualityComparison()
2661 Value *getLowestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa, in getLowestPossibleValue()
2665 Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1); in getLowestPossibleValue()
2666 Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits); in getLowestPossibleValue()
2668 return IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaOtherBits)), in getLowestPossibleValue()
2672 return IRB.CreateAnd(A, IRB.CreateNot(Sa)); in getLowestPossibleValue()
2678 Value *getHighestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa, in getHighestPossibleValue()
2682 Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1); in getHighestPossibleValue()
2683 Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits); in getHighestPossibleValue()
2685 return IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaSignBit)), in getHighestPossibleValue()
2689 return IRB.CreateOr(A, Sa); in getHighestPossibleValue()
2699 IRBuilder<> IRB(&I); in handleRelationalComparisonExact() local
2708 A = IRB.CreatePointerCast(A, Sa->getType()); in handleRelationalComparisonExact()
2709 B = IRB.CreatePointerCast(B, Sb->getType()); in handleRelationalComparisonExact()
2715 Value *S1 = IRB.CreateICmp(I.getPredicate(), in handleRelationalComparisonExact()
2716 getLowestPossibleValue(IRB, A, Sa, IsSigned), in handleRelationalComparisonExact()
2717 getHighestPossibleValue(IRB, B, Sb, IsSigned)); in handleRelationalComparisonExact()
2718 Value *S2 = IRB.CreateICmp(I.getPredicate(), in handleRelationalComparisonExact()
2719 getHighestPossibleValue(IRB, A, Sa, IsSigned), in handleRelationalComparisonExact()
2720 getLowestPossibleValue(IRB, B, Sb, IsSigned)); in handleRelationalComparisonExact()
2721 Value *Si = IRB.CreateXor(S1, S2); in handleRelationalComparisonExact()
2749 IRBuilder<> IRB(&I); in handleSignedRelationalComparison() local
2750 Value *Shadow = IRB.CreateICmpSLT(getShadow(op), getCleanShadow(op), in handleSignedRelationalComparison()
2791 IRBuilder<> IRB(&I); in handleShift() local
2797 IRB.CreateSExt(IRB.CreateICmpNE(S2, getCleanShadow(S2)), S2->getType()); in handleShift()
2799 Value *Shift = IRB.CreateBinOp(I.getOpcode(), S1, V2); in handleShift()
2800 setShadow(&I, IRB.CreateOr(Shift, S2Conv)); in handleShift()
2809 IRBuilder<> IRB(&I); in handleFunnelShift() local
2816 IRB.CreateSExt(IRB.CreateICmpNE(S2, getCleanShadow(S2)), S2->getType()); in handleFunnelShift()
2820 Value *Shift = IRB.CreateCall(Intrin, {S0, S1, V2}); in handleFunnelShift()
2821 setShadow(&I, IRB.CreateOr(Shift, S2Conv)); in handleFunnelShift()
2837 IRBuilder<> IRB(&I); in visitMemMoveInst() local
2838 IRB.CreateCall(MS.MemmoveFn, in visitMemMoveInst()
2840 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)}); in visitMemMoveInst()
2860 IRBuilder<> IRB(&I); in visitMemCpyInst() local
2861 IRB.CreateCall(MS.MemcpyFn, in visitMemCpyInst()
2863 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)}); in visitMemCpyInst()
2869 IRBuilder<> IRB(&I); in visitMemSetInst() local
2870 IRB.CreateCall( in visitMemSetInst()
2873 IRB.CreateIntCast(I.getArgOperand(1), IRB.getInt32Ty(), false), in visitMemSetInst()
2874 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)}); in visitMemSetInst()
2887 IRBuilder<> IRB(&I); in handleVectorStoreIntrinsic() local
2895 Addr, IRB, Shadow->getType(), Align(1), /*isStore*/ true); in handleVectorStoreIntrinsic()
2896 IRB.CreateAlignedStore(Shadow, ShadowPtr, Align(1)); in handleVectorStoreIntrinsic()
2903 IRB.CreateStore(getOrigin(&I, 1), OriginPtr); in handleVectorStoreIntrinsic()
2912 IRBuilder<> IRB(&I); in handleVectorLoadIntrinsic() local
2922 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false); in handleVectorLoadIntrinsic()
2924 IRB.CreateAlignedLoad(ShadowTy, ShadowPtr, Alignment, "_msld")); in handleVectorLoadIntrinsic()
2934 setOrigin(&I, IRB.CreateLoad(MS.OriginTy, OriginPtr)); in handleVectorLoadIntrinsic()
2960 IRBuilder<> IRB(&I); in maybeHandleSimpleNomemIntrinsic() local
2961 ShadowAndOriginCombiner SC(this, IRB); in maybeHandleSimpleNomemIntrinsic()
3020 IRBuilder<> IRB(&I); in handleBswap() local
3025 setShadow(&I, IRB.CreateCall(BswapFunc, getShadow(Op))); in handleBswap()
3030 IRBuilder<> IRB(&I); in handleCountZeroes() local
3034 Value *BoolShadow = IRB.CreateIsNotNull(getShadow(Src), "_mscz_bs"); in handleCountZeroes()
3039 Value *BoolZeroPoison = IRB.CreateIsNull(Src, "_mscz_bzp"); in handleCountZeroes()
3040 BoolShadow = IRB.CreateOr(BoolShadow, BoolZeroPoison, "_mscz_bs"); in handleCountZeroes()
3044 IRB.CreateSExt(BoolShadow, getShadowTy(Src), "_mscz_os"); in handleCountZeroes()
3067 IRBuilder<> IRB(&I); in handleVectorConvertIntrinsic() local
3097 AggShadow = IRB.CreateExtractElement( in handleVectorConvertIntrinsic()
3098 ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), 0)); in handleVectorConvertIntrinsic()
3100 Value *MoreShadow = IRB.CreateExtractElement( in handleVectorConvertIntrinsic()
3101 ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), i)); in handleVectorConvertIntrinsic()
3102 AggShadow = IRB.CreateOr(AggShadow, MoreShadow); in handleVectorConvertIntrinsic()
3118 ResultShadow = IRB.CreateInsertElement( in handleVectorConvertIntrinsic()
3120 ConstantInt::get(IRB.getInt32Ty(), i)); in handleVectorConvertIntrinsic()
3132 Value *Lower64ShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) { in Lower64ShadowExtend()
3134 S = CreateShadowCast(IRB, S, IRB.getInt64Ty(), /* Signed */ true); in Lower64ShadowExtend()
3136 Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S)); in Lower64ShadowExtend()
3137 return CreateShadowCast(IRB, S2, T, /* Signed */ true); in Lower64ShadowExtend()
3142 Value *LowerElementShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) { in LowerElementShadowExtend()
3143 Value *S1 = IRB.CreateExtractElement(S, (uint64_t)0); in LowerElementShadowExtend()
3144 Value *S2 = IRB.CreateICmpNE(S1, getCleanShadow(S1)); in LowerElementShadowExtend()
3145 return CreateShadowCast(IRB, S2, T, /* Signed */ true); in LowerElementShadowExtend()
3148 Value *VariableShadowExtend(IRBuilder<> &IRB, Value *S) { in VariableShadowExtend()
3151 Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S)); in VariableShadowExtend()
3152 return IRB.CreateSExt(S2, T); in VariableShadowExtend()
3164 IRBuilder<> IRB(&I); in handleVectorShiftIntrinsic() local
3169 Value *S2Conv = Variable ? VariableShadowExtend(IRB, S2) in handleVectorShiftIntrinsic()
3170 : Lower64ShadowExtend(IRB, S2, getShadowTy(&I)); in handleVectorShiftIntrinsic()
3173 Value *Shift = IRB.CreateCall(I.getFunctionType(), I.getCalledOperand(), in handleVectorShiftIntrinsic()
3174 {IRB.CreateBitCast(S1, V1->getType()), V2}); in handleVectorShiftIntrinsic()
3175 Shift = IRB.CreateBitCast(Shift, getShadowTy(&I)); in handleVectorShiftIntrinsic()
3176 setShadow(&I, IRB.CreateOr(Shift, S2Conv)); in handleVectorShiftIntrinsic()
3230 IRBuilder<> IRB(&I); in handleVectorPackIntrinsic() local
3240 S1 = IRB.CreateBitCast(S1, T); in handleVectorPackIntrinsic()
3241 S2 = IRB.CreateBitCast(S2, T); in handleVectorPackIntrinsic()
3244 IRB.CreateSExt(IRB.CreateICmpNE(S1, Constant::getNullValue(T)), T); in handleVectorPackIntrinsic()
3246 IRB.CreateSExt(IRB.CreateICmpNE(S2, Constant::getNullValue(T)), T); in handleVectorPackIntrinsic()
3249 S1_ext = IRB.CreateBitCast(S1_ext, X86_MMXTy); in handleVectorPackIntrinsic()
3250 S2_ext = IRB.CreateBitCast(S2_ext, X86_MMXTy); in handleVectorPackIntrinsic()
3257 IRB.CreateCall(ShadowFn, {S1_ext, S2_ext}, "_msprop_vector_pack"); in handleVectorPackIntrinsic()
3259 S = IRB.CreateBitCast(S, getShadowTy(&I)); in handleVectorPackIntrinsic()
3272 IRBuilder<> IRB(&I); in handleVectorSadIntrinsic() local
3275 Value *S = IRB.CreateOr(Shadow0, Shadow1); in handleVectorSadIntrinsic()
3276 S = IRB.CreateBitCast(S, ResTy); in handleVectorSadIntrinsic()
3277 S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)), in handleVectorSadIntrinsic()
3279 S = IRB.CreateLShr(S, ZeroBitsPerResultElement); in handleVectorSadIntrinsic()
3280 S = IRB.CreateBitCast(S, getShadowTy(&I)); in handleVectorSadIntrinsic()
3290 IRBuilder<> IRB(&I); in handleVectorPmaddIntrinsic() local
3293 Value *S = IRB.CreateOr(Shadow0, Shadow1); in handleVectorPmaddIntrinsic()
3294 S = IRB.CreateBitCast(S, ResTy); in handleVectorPmaddIntrinsic()
3295 S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)), in handleVectorPmaddIntrinsic()
3297 S = IRB.CreateBitCast(S, getShadowTy(&I)); in handleVectorPmaddIntrinsic()
3306 IRBuilder<> IRB(&I); in handleVectorComparePackedIntrinsic() local
3310 Value *S0 = IRB.CreateOr(Shadow0, Shadow1); in handleVectorComparePackedIntrinsic()
3311 Value *S = IRB.CreateSExt( in handleVectorComparePackedIntrinsic()
3312 IRB.CreateICmpNE(S0, Constant::getNullValue(ResTy)), ResTy); in handleVectorComparePackedIntrinsic()
3321 IRBuilder<> IRB(&I); in handleVectorCompareScalarIntrinsic() local
3324 Value *S0 = IRB.CreateOr(Shadow0, Shadow1); in handleVectorCompareScalarIntrinsic()
3325 Value *S = LowerElementShadowExtend(IRB, S0, getShadowTy(&I)); in handleVectorCompareScalarIntrinsic()
3333 IRBuilder<> IRB(&I); in handleVectorReduceIntrinsic() local
3334 Value *S = IRB.CreateOrReduce(getShadow(&I, 0)); in handleVectorReduceIntrinsic()
3343 IRBuilder<> IRB(&I); in handleVectorReduceOrIntrinsic() local
3345 Value *OperandUnsetBits = IRB.CreateNot(I.getOperand(0)); in handleVectorReduceOrIntrinsic()
3346 Value *OperandUnsetOrPoison = IRB.CreateOr(OperandUnsetBits, OperandShadow); in handleVectorReduceOrIntrinsic()
3348 Value *OutShadowMask = IRB.CreateAndReduce(OperandUnsetOrPoison); in handleVectorReduceOrIntrinsic()
3350 Value *OrShadow = IRB.CreateOrReduce(OperandShadow); in handleVectorReduceOrIntrinsic()
3351 Value *S = IRB.CreateAnd(OutShadowMask, OrShadow); in handleVectorReduceOrIntrinsic()
3361 IRBuilder<> IRB(&I); in handleVectorReduceAndIntrinsic() local
3363 Value *OperandSetOrPoison = IRB.CreateOr(I.getOperand(0), OperandShadow); in handleVectorReduceAndIntrinsic()
3365 Value *OutShadowMask = IRB.CreateAndReduce(OperandSetOrPoison); in handleVectorReduceAndIntrinsic()
3367 Value *OrShadow = IRB.CreateOrReduce(OperandShadow); in handleVectorReduceAndIntrinsic()
3368 Value *S = IRB.CreateAnd(OutShadowMask, OrShadow); in handleVectorReduceAndIntrinsic()
3375 IRBuilder<> IRB(&I); in handleStmxcsr() local
3377 Type *Ty = IRB.getInt32Ty(); in handleStmxcsr()
3379 getShadowOriginPtr(Addr, IRB, Ty, Align(1), /*isStore*/ true).first; in handleStmxcsr()
3381 IRB.CreateStore(getCleanShadow(Ty), ShadowPtr); in handleStmxcsr()
3391 IRBuilder<> IRB(&I); in handleLdmxcsr() local
3393 Type *Ty = IRB.getInt32Ty(); in handleLdmxcsr()
3397 getShadowOriginPtr(Addr, IRB, Ty, Alignment, /*isStore*/ false); in handleLdmxcsr()
3402 Value *Shadow = IRB.CreateAlignedLoad(Ty, ShadowPtr, Alignment, "_ldmxcsr"); in handleLdmxcsr()
3403 Value *Origin = MS.TrackOrigins ? IRB.CreateLoad(MS.OriginTy, OriginPtr) in handleLdmxcsr()
3409 IRBuilder<> IRB(&I); in handleMaskedExpandLoad() local
3428 getShadowOriginPtr(Ptr, IRB, ElementShadowTy, {}, /*isStore*/ false); in handleMaskedExpandLoad()
3430 Value *Shadow = IRB.CreateMaskedExpandLoad( in handleMaskedExpandLoad()
3440 IRBuilder<> IRB(&I); in handleMaskedCompressStore() local
3454 getShadowOriginPtr(Ptr, IRB, ElementShadowTy, {}, /*isStore*/ true); in handleMaskedCompressStore()
3456 IRB.CreateMaskedCompressStore(Shadow, ShadowPtr, Mask); in handleMaskedCompressStore()
3462 IRBuilder<> IRB(&I); in handleMaskedGather() local
3472 Value *MaskedPtrShadow = IRB.CreateSelect( in handleMaskedGather()
3487 Ptrs, IRB, ElementShadowTy, Alignment, /*isStore*/ false); in handleMaskedGather()
3490 IRB.CreateMaskedGather(ShadowTy, ShadowPtrs, Alignment, Mask, in handleMaskedGather()
3500 IRBuilder<> IRB(&I); in handleMaskedScatter() local
3510 Value *MaskedPtrShadow = IRB.CreateSelect( in handleMaskedScatter()
3520 Ptrs, IRB, ElementShadowTy, Alignment, /*isStore*/ true); in handleMaskedScatter()
3522 IRB.CreateMaskedScatter(Shadow, ShadowPtrs, Alignment, Mask); in handleMaskedScatter()
3528 IRBuilder<> IRB(&I); in handleMaskedStore() local
3544 Ptr, IRB, Shadow->getType(), Alignment, /*isStore*/ true); in handleMaskedStore()
3546 IRB.CreateMaskedStore(Shadow, ShadowPtr, Alignment, Mask); in handleMaskedStore()
3552 paintOrigin(IRB, getOrigin(V), OriginPtr, in handleMaskedStore()
3558 IRBuilder<> IRB(&I); in handleMaskedLoad() local
3579 getShadowOriginPtr(Ptr, IRB, ShadowTy, Alignment, /*isStore*/ false); in handleMaskedLoad()
3580 setShadow(&I, IRB.CreateMaskedLoad(ShadowTy, ShadowPtr, Alignment, Mask, in handleMaskedLoad()
3587 Value *MaskedPassThruShadow = IRB.CreateAnd( in handleMaskedLoad()
3588 getShadow(PassThru), IRB.CreateSExt(IRB.CreateNeg(Mask), ShadowTy)); in handleMaskedLoad()
3590 Value *NotNull = convertToBool(MaskedPassThruShadow, IRB, "_mscmp"); in handleMaskedLoad()
3592 Value *PtrOrigin = IRB.CreateLoad(MS.OriginTy, OriginPtr); in handleMaskedLoad()
3593 Value *Origin = IRB.CreateSelect(NotNull, getOrigin(PassThru), PtrOrigin); in handleMaskedLoad()
3604 IRBuilder<> IRB(&I); in handleBmiIntrinsic() local
3609 SMask = IRB.CreateSExt(IRB.CreateICmpNE(SMask, getCleanShadow(ShadowTy)), in handleBmiIntrinsic()
3612 Value *S = IRB.CreateCall(I.getCalledFunction(), in handleBmiIntrinsic()
3614 S = IRB.CreateOr(SMask, S); in handleBmiIntrinsic()
3636 IRBuilder<> IRB(&I); in handlePclmulIntrinsic() local
3642 Value *Shuf0 = IRB.CreateShuffleVector(getShadow(&I, 0), in handlePclmulIntrinsic()
3644 Value *Shuf1 = IRB.CreateShuffleVector(getShadow(&I, 1), in handlePclmulIntrinsic()
3646 ShadowAndOriginCombiner SOC(this, IRB); in handlePclmulIntrinsic()
3654 IRBuilder<> IRB(&I); in handleUnarySdSsIntrinsic() local
3664 Value *Shadow = IRB.CreateShuffleVector(First, Second, Mask); in handleUnarySdSsIntrinsic()
3671 IRBuilder<> IRB(&I); in handleVtestIntrinsic() local
3674 Value *Or = IRB.CreateOr(Shadow0, Shadow1); in handleVtestIntrinsic()
3675 Value *NZ = IRB.CreateICmpNE(Or, Constant::getNullValue(Or->getType())); in handleVtestIntrinsic()
3676 Value *Scalar = convertShadowToScalar(NZ, IRB); in handleVtestIntrinsic()
3677 Value *Shadow = IRB.CreateZExt(Scalar, getShadowTy(&I)); in handleVtestIntrinsic()
3684 IRBuilder<> IRB(&I); in handleBinarySdSsIntrinsic() local
3689 Value *OrShadow = IRB.CreateOr(First, Second); in handleBinarySdSsIntrinsic()
3695 Value *Shadow = IRB.CreateShuffleVector(First, OrShadow, Mask); in handleBinarySdSsIntrinsic()
3709 IRBuilder<> IRB(&I); in handleAbsIntrinsic() local
3715 IRBuilder<> IRB(&I); in handleIsFpClass() local
3717 setShadow(&I, IRB.CreateICmpNE(Shadow, getCleanShadow(Shadow))); in handleIsFpClass()
4048 IRBuilder<> IRB(&CB); in visitLibAtomicLoad() local
4056 IRB.CreateExtractElement(makeAddAcquireOrderingTable(IRB), Ordering); in visitLibAtomicLoad()
4079 IRBuilder<> IRB(&CB); in visitLibAtomicStore() local
4086 IRB.CreateExtractElement(makeAddReleaseOrderingTable(IRB), Ordering); in visitLibAtomicStore()
4090 getShadowOriginPtr(DstPtr, IRB, IRB.getInt8Ty(), Align(1), in visitLibAtomicStore()
4095 IRB.CreateMemSet(DstShadowPtr, getCleanShadow(IRB.getInt8Ty()), Size, in visitLibAtomicStore()
4156 IRBuilder<> IRB(&CB); in visitCallBase() local
4187 Value *ArgShadowBase = getShadowPtrForArgument(IRB, ArgOffset); in visitCallBase()
4204 getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), Alignment, in visitCallBase()
4207 Store = IRB.CreateMemSet(ArgShadowBase, in visitCallBase()
4208 Constant::getNullValue(IRB.getInt8Ty()), in visitCallBase()
4211 Store = IRB.CreateMemCpy(ArgShadowBase, Alignment, AShadowPtr, in visitCallBase()
4214 Value *ArgOriginBase = getOriginPtrForArgument(IRB, ArgOffset); in visitCallBase()
4218 IRB.CreateMemCpy( in visitCallBase()
4231 Store = IRB.CreateAlignedStore(ArgShadow, ArgShadowBase, in visitCallBase()
4235 IRB.CreateStore(getOrigin(A), in visitCallBase()
4236 getOriginPtrForArgument(IRB, ArgOffset)); in visitCallBase()
4250 VAHelper->visitCallBase(CB, IRB); in visitCallBase()
4312 IRBuilder<> IRB(&I); in visitReturnInst() local
4319 Value *ShadowPtr = getShadowPtrForRetval(IRB); in visitReturnInst()
4337 IRB.CreateAlignedStore(Shadow, ShadowPtr, kShadowTLSAlignment); in visitReturnInst()
4339 IRB.CreateStore(getOrigin(RetVal), getOriginPtrForRetval()); in visitReturnInst()
4344 IRBuilder<> IRB(&I); in visitPHINode() local
4352 setShadow(&I, IRB.CreatePHI(getShadowTy(&I), I.getNumIncomingValues(), in visitPHINode()
4356 &I, IRB.CreatePHI(MS.OriginTy, I.getNumIncomingValues(), "_msphi_o")); in visitPHINode()
4371 void poisonAllocaUserspace(AllocaInst &I, IRBuilder<> &IRB, Value *Len) { in poisonAllocaUserspace()
4373 IRB.CreateCall(MS.MsanPoisonStackFn, {&I, Len}); in poisonAllocaUserspace()
4377 &I, IRB, IRB.getInt8Ty(), Align(1), /*isStore*/ true); in poisonAllocaUserspace()
4379 Value *PoisonValue = IRB.getInt8(PoisonStack ? ClPoisonStackPattern : 0); in poisonAllocaUserspace()
4380 IRB.CreateMemSet(ShadowBase, PoisonValue, Len, I.getAlign()); in poisonAllocaUserspace()
4387 IRB.CreateCall(MS.MsanSetAllocaOriginWithDescriptionFn, in poisonAllocaUserspace()
4390 IRB.CreateCall(MS.MsanSetAllocaOriginNoDescriptionFn, {&I, Len, Idptr}); in poisonAllocaUserspace()
4395 void poisonAllocaKmsan(AllocaInst &I, IRBuilder<> &IRB, Value *Len) { in poisonAllocaKmsan()
4398 IRB.CreateCall(MS.MsanPoisonAllocaFn, {&I, Len, Descr}); in poisonAllocaKmsan()
4400 IRB.CreateCall(MS.MsanUnpoisonAllocaFn, {&I, Len}); in poisonAllocaKmsan()
4407 NextNodeIRBuilder IRB(InsPoint); in instrumentAlloca() local
4412 Len = IRB.CreateMul(Len, in instrumentAlloca()
4413 IRB.CreateZExtOrTrunc(I.getArraySize(), MS.IntptrTy)); in instrumentAlloca()
4416 poisonAllocaKmsan(I, IRB, Len); in instrumentAlloca()
4418 poisonAllocaUserspace(I, IRB, Len); in instrumentAlloca()
4430 IRBuilder<> IRB(&I); in visitSelectInst() local
4440 Value *Sa0 = IRB.CreateSelect(B, Sc, Sd); in visitSelectInst()
4454 C = CreateAppToShadowCast(IRB, C); in visitSelectInst()
4455 D = CreateAppToShadowCast(IRB, D); in visitSelectInst()
4458 Sa1 = IRB.CreateOr({IRB.CreateXor(C, D), Sc, Sd}); in visitSelectInst()
4460 Value *Sa = IRB.CreateSelect(Sb, Sa1, Sa0, "_msprop_select"); in visitSelectInst()
4466 B = convertToBool(B, IRB); in visitSelectInst()
4467 Sb = convertToBool(Sb, IRB); in visitSelectInst()
4472 &I, IRB.CreateSelect(Sb, getOrigin(I.getCondition()), in visitSelectInst()
4473 IRB.CreateSelect(B, getOrigin(I.getTrueValue()), in visitSelectInst()
4498 IRBuilder<> IRB(&I); in visitExtractValueInst() local
4503 Value *ResShadow = IRB.CreateExtractValue(AggShadow, I.getIndices()); in visitExtractValueInst()
4510 IRBuilder<> IRB(&I); in visitInsertValueInst() local
4516 Value *Res = IRB.CreateInsertValue(AggShadow, InsShadow, I.getIndices()); in visitInsertValueInst()
4547 IRBuilder<> &IRB, const DataLayout &DL, in instrumentAsmArgument()
4563 IRB.CreateTypeSize(MS.IntptrTy, DL.getTypeStoreSize(ElemTy)); in instrumentAsmArgument()
4565 IRB.CreateCall(MS.MsanInstrumentAsmStoreFn, {Operand, SizeVal}); in instrumentAsmArgument()
4570 getShadowOriginPtrUserspace(Operand, IRB, IRB.getInt8Ty(), Align(1)); in instrumentAsmArgument()
4571 IRB.CreateAlignedStore(getCleanShadow(ElemTy), ShadowPtr, Align(1)); in instrumentAsmArgument()
4619 IRBuilder<> IRB(&I); in visitAsmInstruction() local
4629 instrumentAsmArgument(Operand, CB->getParamElementType(i), I, IRB, DL, in visitAsmInstruction()
4637 instrumentAsmArgument(Operand, CB->getParamElementType(i), I, IRB, DL, in visitAsmInstruction()
4677 Value *getShadowAddrForVAArgument(IRBuilder<> &IRB, unsigned ArgOffset) { in getShadowAddrForVAArgument()
4678 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy); in getShadowAddrForVAArgument()
4679 return IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset)); in getShadowAddrForVAArgument()
4683 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB, in getShadowPtrForVAArgument()
4685 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy); in getShadowPtrForVAArgument()
4686 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset)); in getShadowPtrForVAArgument()
4687 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0), in getShadowPtrForVAArgument()
4692 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB, in getShadowPtrForVAArgument()
4697 return getShadowPtrForVAArgument(Ty, IRB, ArgOffset); in getShadowPtrForVAArgument()
4701 Value *getOriginPtrForVAArgument(IRBuilder<> &IRB, int ArgOffset) { in getOriginPtrForVAArgument()
4702 Value *Base = IRB.CreatePointerCast(MS.VAArgOriginTLS, MS.IntptrTy); in getOriginPtrForVAArgument()
4706 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset)); in getOriginPtrForVAArgument()
4707 return IRB.CreateIntToPtr(Base, PointerType::get(MS.OriginTy, 0), in getOriginPtrForVAArgument()
4711 void CleanUnusedTLS(IRBuilder<> &IRB, Value *ShadowBase, in CleanUnusedTLS()
4719 ConstantInt::getSigned(IRB.getInt32Ty(), kParamTLSSize - BaseOffset); in CleanUnusedTLS()
4720 IRB.CreateMemSet(ShadowBase, ConstantInt::getNullValue(IRB.getInt8Ty()), in CleanUnusedTLS()
4725 IRBuilder<> IRB(&I); in unpoisonVAListTagForInst() local
4729 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true); in unpoisonVAListTagForInst()
4731 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()), in unpoisonVAListTagForInst()
4801 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override { in visitCallBase()
4822 getShadowPtrForVAArgument(RealTy, IRB, OverflowOffset); in visitCallBase()
4825 OriginBase = getOriginPtrForVAArgument(IRB, OverflowOffset); in visitCallBase()
4829 CleanUnusedTLS(IRB, ShadowBase, BaseOffset); in visitCallBase()
4835 MSV.getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), kShadowTLSAlignment, in visitCallBase()
4837 IRB.CreateMemCpy(ShadowBase, kShadowTLSAlignment, ShadowPtr, in visitCallBase()
4840 IRB.CreateMemCpy(OriginBase, kShadowTLSAlignment, OriginPtr, in visitCallBase()
4851 ShadowBase = getShadowPtrForVAArgument(A->getType(), IRB, GpOffset); in visitCallBase()
4853 OriginBase = getOriginPtrForVAArgument(IRB, GpOffset); in visitCallBase()
4858 ShadowBase = getShadowPtrForVAArgument(A->getType(), IRB, FpOffset); in visitCallBase()
4860 OriginBase = getOriginPtrForVAArgument(IRB, FpOffset); in visitCallBase()
4871 getShadowPtrForVAArgument(A->getType(), IRB, OverflowOffset); in visitCallBase()
4873 OriginBase = getOriginPtrForVAArgument(IRB, OverflowOffset); in visitCallBase()
4878 CleanUnusedTLS(IRB, ShadowBase, BaseOffset); in visitCallBase()
4888 IRB.CreateAlignedStore(Shadow, ShadowBase, kShadowTLSAlignment); in visitCallBase()
4892 MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize, in visitCallBase()
4898 ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AMD64FpEndOffset); in visitCallBase()
4899 IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS); in visitCallBase()
4908 IRBuilder<> IRB(MSV.FnPrologueEnd); in finalizeInstrumentation() local
4910 IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS); in finalizeInstrumentation()
4911 Value *CopySize = IRB.CreateAdd( in finalizeInstrumentation()
4913 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize); in finalizeInstrumentation()
4915 IRB.CreateMemSet(VAArgTLSCopy, Constant::getNullValue(IRB.getInt8Ty()), in finalizeInstrumentation()
4918 Value *SrcSize = IRB.CreateBinaryIntrinsic( in finalizeInstrumentation()
4921 IRB.CreateMemCpy(VAArgTLSCopy, kShadowTLSAlignment, MS.VAArgTLS, in finalizeInstrumentation()
4924 VAArgTLSOriginCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize); in finalizeInstrumentation()
4926 IRB.CreateMemCpy(VAArgTLSOriginCopy, kShadowTLSAlignment, in finalizeInstrumentation()
4935 NextNodeIRBuilder IRB(OrigInst); in finalizeInstrumentation() local
4939 Value *RegSaveAreaPtrPtr = IRB.CreateIntToPtr( in finalizeInstrumentation()
4940 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy), in finalizeInstrumentation()
4944 IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr); in finalizeInstrumentation()
4948 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(), in finalizeInstrumentation()
4950 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment, in finalizeInstrumentation()
4953 IRB.CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy, in finalizeInstrumentation()
4956 Value *OverflowArgAreaPtrPtr = IRB.CreateIntToPtr( in finalizeInstrumentation()
4957 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy), in finalizeInstrumentation()
4961 IRB.CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr); in finalizeInstrumentation()
4964 MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.getInt8Ty(), in finalizeInstrumentation()
4966 Value *SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSCopy, in finalizeInstrumentation()
4968 IRB.CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment, in finalizeInstrumentation()
4971 SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSOriginCopy, in finalizeInstrumentation()
4973 IRB.CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment, in finalizeInstrumentation()
4990 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override { in visitCallBase()
5004 Base = getShadowPtrForVAArgument(A->getType(), IRB, VAArgOffset, ArgSize); in visitCallBase()
5009 IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment); in visitCallBase()
5012 Constant *TotalVAArgSize = ConstantInt::get(IRB.getInt64Ty(), VAArgOffset); in visitCallBase()
5015 IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS); in visitCallBase()
5021 IRBuilder<> IRB(MSV.FnPrologueEnd); in finalizeInstrumentation() local
5022 VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS); in finalizeInstrumentation()
5024 IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0), VAArgSize); in finalizeInstrumentation()
5029 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize); in finalizeInstrumentation()
5031 IRB.CreateMemSet(VAArgTLSCopy, Constant::getNullValue(IRB.getInt8Ty()), in finalizeInstrumentation()
5034 Value *SrcSize = IRB.CreateBinaryIntrinsic( in finalizeInstrumentation()
5037 IRB.CreateMemCpy(VAArgTLSCopy, kShadowTLSAlignment, MS.VAArgTLS, in finalizeInstrumentation()
5045 NextNodeIRBuilder IRB(OrigInst); in finalizeInstrumentation() local
5049 IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy), in finalizeInstrumentation()
5052 IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr); in finalizeInstrumentation()
5056 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(), in finalizeInstrumentation()
5058 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment, in finalizeInstrumentation()
5118 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override { in visitCallBase()
5136 Base = getShadowPtrForVAArgument(A->getType(), IRB, GrOffset); in visitCallBase()
5140 Base = getShadowPtrForVAArgument(A->getType(), IRB, VrOffset); in visitCallBase()
5151 Base = getShadowPtrForVAArgument(A->getType(), IRB, BaseOffset); in visitCallBase()
5155 CleanUnusedTLS(IRB, Base, BaseOffset); in visitCallBase()
5164 IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment); in visitCallBase()
5167 ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AArch64VAEndOffset); in visitCallBase()
5168 IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS); in visitCallBase()
5172 Value *getVAField64(IRBuilder<> &IRB, Value *VAListTag, int offset) { in getVAField64()
5173 Value *SaveAreaPtrPtr = IRB.CreateIntToPtr( in getVAField64()
5174 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy), in getVAField64()
5177 return IRB.CreateLoad(Type::getInt64Ty(*MS.C), SaveAreaPtrPtr); in getVAField64()
5181 Value *getVAField32(IRBuilder<> &IRB, Value *VAListTag, int offset) { in getVAField32()
5182 Value *SaveAreaPtr = IRB.CreateIntToPtr( in getVAField32()
5183 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy), in getVAField32()
5186 Value *SaveArea32 = IRB.CreateLoad(IRB.getInt32Ty(), SaveAreaPtr); in getVAField32()
5187 return IRB.CreateSExt(SaveArea32, MS.IntptrTy); in getVAField32()
5196 IRBuilder<> IRB(MSV.FnPrologueEnd); in finalizeInstrumentation() local
5198 IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS); in finalizeInstrumentation()
5199 Value *CopySize = IRB.CreateAdd( in finalizeInstrumentation()
5201 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize); in finalizeInstrumentation()
5203 IRB.CreateMemSet(VAArgTLSCopy, Constant::getNullValue(IRB.getInt8Ty()), in finalizeInstrumentation()
5206 Value *SrcSize = IRB.CreateBinaryIntrinsic( in finalizeInstrumentation()
5209 IRB.CreateMemCpy(VAArgTLSCopy, kShadowTLSAlignment, MS.VAArgTLS, in finalizeInstrumentation()
5220 NextNodeIRBuilder IRB(OrigInst); in finalizeInstrumentation() local
5236 Type *RegSaveAreaPtrTy = IRB.getPtrTy(); in finalizeInstrumentation()
5240 IRB.CreateIntToPtr(getVAField64(IRB, VAListTag, 0), RegSaveAreaPtrTy); in finalizeInstrumentation()
5243 Value *GrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 8); in finalizeInstrumentation()
5244 Value *GrOffSaveArea = getVAField32(IRB, VAListTag, 24); in finalizeInstrumentation()
5246 Value *GrRegSaveAreaPtr = IRB.CreateIntToPtr( in finalizeInstrumentation()
5247 IRB.CreateAdd(GrTopSaveAreaPtr, GrOffSaveArea), RegSaveAreaPtrTy); in finalizeInstrumentation()
5250 Value *VrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 16); in finalizeInstrumentation()
5251 Value *VrOffSaveArea = getVAField32(IRB, VAListTag, 28); in finalizeInstrumentation()
5253 Value *VrRegSaveAreaPtr = IRB.CreateIntToPtr( in finalizeInstrumentation()
5254 IRB.CreateAdd(VrTopSaveAreaPtr, VrOffSaveArea), RegSaveAreaPtrTy); in finalizeInstrumentation()
5261 IRB.CreateAdd(GrArgSize, GrOffSaveArea); in finalizeInstrumentation()
5264 MSV.getShadowOriginPtr(GrRegSaveAreaPtr, IRB, IRB.getInt8Ty(), in finalizeInstrumentation()
5269 IRB.CreateInBoundsPtrAdd(VAArgTLSCopy, GrRegSaveAreaShadowPtrOff); in finalizeInstrumentation()
5270 Value *GrCopySize = IRB.CreateSub(GrArgSize, GrRegSaveAreaShadowPtrOff); in finalizeInstrumentation()
5272 IRB.CreateMemCpy(GrRegSaveAreaShadowPtr, Align(8), GrSrcPtr, Align(8), in finalizeInstrumentation()
5277 IRB.CreateAdd(VrArgSize, VrOffSaveArea); in finalizeInstrumentation()
5280 MSV.getShadowOriginPtr(VrRegSaveAreaPtr, IRB, IRB.getInt8Ty(), in finalizeInstrumentation()
5284 Value *VrSrcPtr = IRB.CreateInBoundsPtrAdd( in finalizeInstrumentation()
5285 IRB.CreateInBoundsPtrAdd(VAArgTLSCopy, in finalizeInstrumentation()
5286 IRB.getInt32(AArch64VrBegOffset)), in finalizeInstrumentation()
5288 Value *VrCopySize = IRB.CreateSub(VrArgSize, VrRegSaveAreaShadowPtrOff); in finalizeInstrumentation()
5290 IRB.CreateMemCpy(VrRegSaveAreaShadowPtr, Align(8), VrSrcPtr, Align(8), in finalizeInstrumentation()
5295 MSV.getShadowOriginPtr(StackSaveAreaPtr, IRB, IRB.getInt8Ty(), in finalizeInstrumentation()
5299 Value *StackSrcPtr = IRB.CreateInBoundsPtrAdd( in finalizeInstrumentation()
5300 VAArgTLSCopy, IRB.getInt32(AArch64VAEndOffset)); in finalizeInstrumentation()
5302 IRB.CreateMemCpy(StackSaveAreaShadowPtr, Align(16), StackSrcPtr, in finalizeInstrumentation()
5317 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override { in visitCallBase()
5348 RealTy, IRB, VAArgOffset - VAArgBase, ArgSize); in visitCallBase()
5352 MSV.getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), in visitCallBase()
5355 IRB.CreateMemCpy(Base, kShadowTLSAlignment, AShadowPtr, in visitCallBase()
5384 Base = getShadowPtrForVAArgument(A->getType(), IRB, in visitCallBase()
5387 IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment); in visitCallBase()
5397 ConstantInt::get(IRB.getInt64Ty(), VAArgOffset - VAArgBase); in visitCallBase()
5400 IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS); in visitCallBase()
5406 IRBuilder<> IRB(MSV.FnPrologueEnd); in finalizeInstrumentation() local
5407 VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS); in finalizeInstrumentation()
5409 IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0), VAArgSize); in finalizeInstrumentation()
5415 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize); in finalizeInstrumentation()
5417 IRB.CreateMemSet(VAArgTLSCopy, Constant::getNullValue(IRB.getInt8Ty()), in finalizeInstrumentation()
5420 Value *SrcSize = IRB.CreateBinaryIntrinsic( in finalizeInstrumentation()
5423 IRB.CreateMemCpy(VAArgTLSCopy, kShadowTLSAlignment, MS.VAArgTLS, in finalizeInstrumentation()
5431 NextNodeIRBuilder IRB(OrigInst); in finalizeInstrumentation() local
5435 IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy), in finalizeInstrumentation()
5438 IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr); in finalizeInstrumentation()
5442 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(), in finalizeInstrumentation()
5444 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment, in finalizeInstrumentation()
5520 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override { in visitCallBase()
5558 ShadowBase = getShadowAddrForVAArgument(IRB, GpOffset + GapSize); in visitCallBase()
5560 OriginBase = getOriginPtrForVAArgument(IRB, GpOffset + GapSize); in visitCallBase()
5577 ShadowBase = getShadowAddrForVAArgument(IRB, FpOffset); in visitCallBase()
5579 OriginBase = getOriginPtrForVAArgument(IRB, FpOffset); in visitCallBase()
5606 getShadowAddrForVAArgument(IRB, OverflowOffset + GapSize); in visitCallBase()
5609 getOriginPtrForVAArgument(IRB, OverflowOffset + GapSize); in visitCallBase()
5624 Shadow = MSV.CreateShadowCast(IRB, Shadow, IRB.getInt64Ty(), in visitCallBase()
5626 ShadowBase = IRB.CreateIntToPtr( in visitCallBase()
5628 IRB.CreateStore(Shadow, ShadowBase); in visitCallBase()
5632 MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize, in visitCallBase()
5637 IRB.getInt64Ty(), OverflowOffset - SystemZOverflowOffset); in visitCallBase()
5638 IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS); in visitCallBase()
5641 void copyRegSaveArea(IRBuilder<> &IRB, Value *VAListTag) { in copyRegSaveArea()
5643 Value *RegSaveAreaPtrPtr = IRB.CreateIntToPtr( in copyRegSaveArea()
5644 IRB.CreateAdd( in copyRegSaveArea()
5645 IRB.CreatePtrToInt(VAListTag, MS.IntptrTy), in copyRegSaveArea()
5648 Value *RegSaveAreaPtr = IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr); in copyRegSaveArea()
5652 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(), Alignment, in copyRegSaveArea()
5659 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment, in copyRegSaveArea()
5662 IRB.CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy, in copyRegSaveArea()
5668 void copyOverflowArea(IRBuilder<> &IRB, Value *VAListTag) { in copyOverflowArea()
5670 Value *OverflowArgAreaPtrPtr = IRB.CreateIntToPtr( in copyOverflowArea()
5671 IRB.CreateAdd( in copyOverflowArea()
5672 IRB.CreatePtrToInt(VAListTag, MS.IntptrTy), in copyOverflowArea()
5676 IRB.CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr); in copyOverflowArea()
5680 MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.getInt8Ty(), in copyOverflowArea()
5682 Value *SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSCopy, in copyOverflowArea()
5684 IRB.CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment, in copyOverflowArea()
5687 SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSOriginCopy, in copyOverflowArea()
5689 IRB.CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment, in copyOverflowArea()
5700 IRBuilder<> IRB(MSV.FnPrologueEnd); in finalizeInstrumentation() local
5702 IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS); in finalizeInstrumentation()
5704 IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, SystemZOverflowOffset), in finalizeInstrumentation()
5706 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize); in finalizeInstrumentation()
5708 IRB.CreateMemSet(VAArgTLSCopy, Constant::getNullValue(IRB.getInt8Ty()), in finalizeInstrumentation()
5711 Value *SrcSize = IRB.CreateBinaryIntrinsic( in finalizeInstrumentation()
5714 IRB.CreateMemCpy(VAArgTLSCopy, kShadowTLSAlignment, MS.VAArgTLS, in finalizeInstrumentation()
5717 VAArgTLSOriginCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize); in finalizeInstrumentation()
5719 IRB.CreateMemCpy(VAArgTLSOriginCopy, kShadowTLSAlignment, in finalizeInstrumentation()
5729 NextNodeIRBuilder IRB(OrigInst); in finalizeInstrumentation() local
5731 copyRegSaveArea(IRB, VAListTag); in finalizeInstrumentation()
5732 copyOverflowArea(IRB, VAListTag); in finalizeInstrumentation()
5746 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {} in visitCallBase()