300   Value *getOpaqueNoopCast(IRBuilder<> &IRB, Value *Val);
302   Value *getDynamicShadowIfunc(IRBuilder<> &IRB);
303   Value *getShadowNonTls(IRBuilder<> &IRB);
306   Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
328   void tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag, size_t Size);
329   Value *tagPointer(IRBuilder<> &IRB, Type *Ty, Value *PtrLong, Value *Tag);
330   Value *untagPointer(IRBuilder<> &IRB, Value *PtrLong);
334   Value *readRegister(IRBuilder<> &IRB, StringRef Name);
336   Value *getNextTagWithCall(IRBuilder<> &IRB);
337   Value *getStackBaseTag(IRBuilder<> &IRB);
338   Value *getAllocaTag(IRBuilder<> &IRB, Value *StackTag, unsigned AllocaNo);
339   Value *getUARTag(IRBuilder<> &IRB);
341   Value *getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty);
342   Value *applyTagMask(IRBuilder<> &IRB, Value *OldTag);
345   void emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord);
350   Value *getPC(IRBuilder<> &IRB);
351   Value *getSP(IRBuilder<> &IRB);
352   Value *getFrameRecordInfo(IRBuilder<> &IRB);
588   IRBuilder<> IRB(*C);  in initializeModule()  local
589   IntptrTy = IRB.getIntPtrTy(DL);  in initializeModule()
590   PtrTy = IRB.getPtrTy();  in initializeModule()
591   Int8Ty = IRB.getInt8Ty();  in initializeModule()
592   Int32Ty = IRB.getInt32Ty();  in initializeModule()
660   IRBuilder<> IRB(*C);  in initializeCallbacks()  local
730 Value *HWAddressSanitizer::getOpaqueNoopCast(IRBuilder<> &IRB, Value *Val) {  in getOpaqueNoopCast()  argument
739   return IRB.CreateCall(Asm, {Val}, ".hwasan.shadow");  in getOpaqueNoopCast()
742 Value *HWAddressSanitizer::getDynamicShadowIfunc(IRBuilder<> &IRB) {  in getDynamicShadowIfunc()  argument
743   return getOpaqueNoopCast(IRB, ShadowGlobal);  in getDynamicShadowIfunc()
746 Value *HWAddressSanitizer::getShadowNonTls(IRBuilder<> &IRB) {  in getShadowNonTls()  argument
749         IRB, ConstantExpr::getIntToPtr(  in getShadowNonTls()
753     return getDynamicShadowIfunc(IRB);  in getShadowNonTls()
756       IRB.GetInsertBlock()->getParent()->getParent()->getOrInsertGlobal(  in getShadowNonTls()
758   return IRB.CreateLoad(PtrTy, GlobalDynamicAddress);  in getShadowNonTls()
852   IRBuilder<> IRB(I);  in untagPointerOperand()  local
853   Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);  in untagPointerOperand()
855       IRB.CreateIntToPtr(untagPointer(IRB, AddrLong), Addr->getType());  in untagPointerOperand()
859 Value *HWAddressSanitizer::memToShadow(Value *Mem, IRBuilder<> &IRB) {  in memToShadow()  argument
861   Value *Shadow = IRB.CreateLShr(Mem, Mapping.Scale);  in memToShadow()
863     return IRB.CreateIntToPtr(Shadow, PtrTy);  in memToShadow()
865   return IRB.CreatePtrAdd(ShadowBase, Shadow);  in memToShadow()
883   IRBuilder<> IRB(InsertBefore);  in insertShadowTagCheck()  local
885   R.PtrLong = IRB.CreatePointerCast(Ptr, IntptrTy);  in insertShadowTagCheck()
887       IRB.CreateTrunc(IRB.CreateLShr(R.PtrLong, PointerTagShift), Int8Ty);  in insertShadowTagCheck()
888   R.AddrLong = untagPointer(IRB, R.PtrLong);  in insertShadowTagCheck()
889   Value *Shadow = memToShadow(R.AddrLong, IRB);  in insertShadowTagCheck()
890   R.MemTag = IRB.CreateLoad(Int8Ty, Shadow);  in insertShadowTagCheck()
891   Value *TagMismatch = IRB.CreateICmpNE(R.PtrTag, R.MemTag);  in insertShadowTagCheck()
894     Value *TagNotIgnored = IRB.CreateICmpNE(  in insertShadowTagCheck()
896     TagMismatch = IRB.CreateAnd(TagMismatch, TagNotIgnored);  in insertShadowTagCheck()
918   IRBuilder<> IRB(InsertBefore);  in instrumentMemAccessOutline()  local
919   Module *M = IRB.GetInsertBlock()->getParent()->getParent();  in instrumentMemAccessOutline()
920   IRB.CreateCall(Intrinsic::getDeclaration(  in instrumentMemAccessOutline()
937   IRBuilder<> IRB(TCI.TagMismatchTerm);  in instrumentMemAccessInline()  local
939       IRB.CreateICmpUGT(TCI.MemTag, ConstantInt::get(Int8Ty, 15));  in instrumentMemAccessInline()
944   IRB.SetInsertPoint(TCI.TagMismatchTerm);  in instrumentMemAccessInline()
945   Value *PtrLowBits = IRB.CreateTrunc(IRB.CreateAnd(TCI.PtrLong, 15), Int8Ty);  in instrumentMemAccessInline()
946   PtrLowBits = IRB.CreateAdd(  in instrumentMemAccessInline()
948   Value *PtrLowBitsOOB = IRB.CreateICmpUGE(PtrLowBits, TCI.MemTag);  in instrumentMemAccessInline()
953   IRB.SetInsertPoint(TCI.TagMismatchTerm);  in instrumentMemAccessInline()
954   Value *InlineTagAddr = IRB.CreateOr(TCI.AddrLong, 15);  in instrumentMemAccessInline()
955   InlineTagAddr = IRB.CreateIntToPtr(InlineTagAddr, PtrTy);  in instrumentMemAccessInline()
956   Value *InlineTag = IRB.CreateLoad(Int8Ty, InlineTagAddr);  in instrumentMemAccessInline()
957   Value *InlineTagMismatch = IRB.CreateICmpNE(TCI.PtrTag, InlineTag);  in instrumentMemAccessInline()
962   IRB.SetInsertPoint(CheckFailTerm);  in instrumentMemAccessInline()
996   IRB.CreateCall(Asm, TCI.PtrLong);  in instrumentMemAccessInline()
1013   IRBuilder<> IRB(MI);  in instrumentMemIntrinsic()  local
1017         IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)};  in instrumentMemIntrinsic()
1021     IRB.CreateCall(isa<MemMoveInst>(MI) ? HwasanMemmove : HwasanMemcpy, Args);  in instrumentMemIntrinsic()
1025         IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),  in instrumentMemIntrinsic()
1026         IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)};  in instrumentMemIntrinsic()
1029     IRB.CreateCall(HwasanMemset, Args);  in instrumentMemIntrinsic()
1044   IRBuilder<> IRB(O.getInsn());  in instrumentMemAccess()  local
1051       SmallVector<Value *, 2> Args{IRB.CreatePointerCast(Addr, IntptrTy)};  in instrumentMemAccess()
1054       IRB.CreateCall(HwasanMemoryAccessCallback[O.IsWrite][AccessSizeIndex],  in instrumentMemAccess()
1065         IRB.CreatePointerCast(Addr, IntptrTy),  in instrumentMemAccess()
1066         IRB.CreateUDiv(IRB.CreateTypeSize(IntptrTy, O.TypeStoreSize),  in instrumentMemAccess()
1070     IRB.CreateCall(HwasanMemoryAccessCallbackSized[O.IsWrite], Args);  in instrumentMemAccess()
1077 void HWAddressSanitizer::tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag,  in tagAlloca()  argument
1083   Tag = IRB.CreateTrunc(Tag, Int8Ty);  in tagAlloca()
1085     IRB.CreateCall(HwasanTagMemoryFunc,  in tagAlloca()
1086                    {IRB.CreatePointerCast(AI, PtrTy), Tag,  in tagAlloca()
1090     Value *AddrLong = untagPointer(IRB, IRB.CreatePointerCast(AI, IntptrTy));  in tagAlloca()
1091     Value *ShadowPtr = memToShadow(AddrLong, IRB);  in tagAlloca()
1099       IRB.CreateMemSet(ShadowPtr, Tag, ShadowSize, Align(1));  in tagAlloca()
1102       IRB.CreateStore(ConstantInt::get(Int8Ty, SizeRemainder),  in tagAlloca()
1103                       IRB.CreateConstGEP1_32(Int8Ty, ShadowPtr, ShadowSize));  in tagAlloca()
1104       IRB.CreateStore(  in tagAlloca()
1105           Tag, IRB.CreateConstGEP1_32(Int8Ty, IRB.CreatePointerCast(AI, PtrTy),  in tagAlloca()
1132 Value *HWAddressSanitizer::applyTagMask(IRBuilder<> &IRB, Value *OldTag) {  in applyTagMask()  argument
1135   return IRB.CreateAnd(OldTag,  in applyTagMask()
1139 Value *HWAddressSanitizer::getNextTagWithCall(IRBuilder<> &IRB) {  in getNextTagWithCall()  argument
1140   return IRB.CreateZExt(IRB.CreateCall(HwasanGenerateTagFunc), IntptrTy);  in getNextTagWithCall()
1143 Value *HWAddressSanitizer::getStackBaseTag(IRBuilder<> &IRB) {  in getStackBaseTag()  argument
1151   Value *StackPointerLong = getSP(IRB);  in getStackBaseTag()
1153       applyTagMask(IRB, IRB.CreateXor(StackPointerLong,  in getStackBaseTag()
1154                                       IRB.CreateLShr(StackPointerLong, 20)));  in getStackBaseTag()
1159 Value *HWAddressSanitizer::getAllocaTag(IRBuilder<> &IRB, Value *StackTag,  in getAllocaTag()  argument
1162     return getNextTagWithCall(IRB);  in getAllocaTag()
1163   return IRB.CreateXor(  in getAllocaTag()
1167 Value *HWAddressSanitizer::getUARTag(IRBuilder<> &IRB) {  in getUARTag()  argument
1168   Value *StackPointerLong = getSP(IRB);  in getUARTag()
1170       applyTagMask(IRB, IRB.CreateLShr(StackPointerLong, PointerTagShift));  in getUARTag()
1177 Value *HWAddressSanitizer::tagPointer(IRBuilder<> &IRB, Type *Ty,  in tagPointer()  argument
1184         IRB.CreateOr(IRB.CreateShl(Tag, PointerTagShift),  in tagPointer()
1186     TaggedPtrLong = IRB.CreateAnd(PtrLong, ShiftedTag);  in tagPointer()
1189     Value *ShiftedTag = IRB.CreateShl(Tag, PointerTagShift);  in tagPointer()
1190     TaggedPtrLong = IRB.CreateOr(PtrLong, ShiftedTag);  in tagPointer()
1192   return IRB.CreateIntToPtr(TaggedPtrLong, Ty);  in tagPointer()
1196 Value *HWAddressSanitizer::untagPointer(IRBuilder<> &IRB, Value *PtrLong) {  in untagPointer()  argument
1202         IRB.CreateOr(PtrLong, ConstantInt::get(PtrLong->getType(),  in untagPointer()
1206     UntaggedPtrLong = IRB.CreateAnd(  in untagPointer()
1213 Value *HWAddressSanitizer::getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty) {  in getHwasanThreadSlotPtr()  argument
1214   Module *M = IRB.GetInsertBlock()->getParent()->getParent();  in getHwasanThreadSlotPtr()
1220     return IRB.CreateConstGEP1_32(Int8Ty, IRB.CreateCall(ThreadPointerFunc),  in getHwasanThreadSlotPtr()
1229 Value *HWAddressSanitizer::getPC(IRBuilder<> &IRB) {  in getPC()  argument
1231     return readRegister(IRB, "pc");  in getPC()
1232   return IRB.CreatePtrToInt(IRB.GetInsertBlock()->getParent(), IntptrTy);  in getPC()
1235 Value *HWAddressSanitizer::getSP(IRBuilder<> &IRB) {  in getSP()  argument
1239     Function *F = IRB.GetInsertBlock()->getParent();  in getSP()
1243         IRB.getPtrTy(M->getDataLayout().getAllocaAddrSpace()));  in getSP()
1244     CachedSP = IRB.CreatePtrToInt(  in getSP()
1245         IRB.CreateCall(GetStackPointerFn, {Constant::getNullValue(Int32Ty)}),  in getSP()
1251 Value *HWAddressSanitizer::getFrameRecordInfo(IRBuilder<> &IRB) {  in getFrameRecordInfo()  argument
1253   Value *PC = getPC(IRB);  in getFrameRecordInfo()
1254   Value *SP = getSP(IRB);  in getFrameRecordInfo()
1262   SP = IRB.CreateShl(SP, 44);  in getFrameRecordInfo()
1263   return IRB.CreateOr(PC, SP);  in getFrameRecordInfo()
1266 void HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord) {  in emitPrologue()  argument
1268     ShadowBase = getShadowNonTls(IRB);  in emitPrologue()
1270     ShadowBase = getDynamicShadowIfunc(IRB);  in emitPrologue()
1281       SlotPtr = getHwasanThreadSlotPtr(IRB, IntptrTy);  in emitPrologue()
1283       ThreadLong = IRB.CreateLoad(IntptrTy, SlotPtr);  in emitPrologue()
1287                                     : untagPointer(IRB, ThreadLong);  in emitPrologue()
1295       Value *FrameRecordInfo = getFrameRecordInfo(IRB);  in emitPrologue()
1296       IRB.CreateCall(HwasanRecordFrameRecordFunc, {FrameRecordInfo});  in emitPrologue()
1302       StackBaseTag = IRB.CreateAShr(ThreadLong, 3);  in emitPrologue()
1305       Value *FrameRecordInfo = getFrameRecordInfo(IRB);  in emitPrologue()
1307           IRB.CreateIntToPtr(ThreadLongMaybeUntagged, IRB.getPtrTy(0));  in emitPrologue()
1308       IRB.CreateStore(FrameRecordInfo, RecordPtr);  in emitPrologue()
1317       Value *WrapMask = IRB.CreateXor(  in emitPrologue()
1318           IRB.CreateShl(IRB.CreateAShr(ThreadLong, 56), 12, "", true, true),  in emitPrologue()
1320       Value *ThreadLongNew = IRB.CreateAnd(  in emitPrologue()
1321           IRB.CreateAdd(ThreadLong, ConstantInt::get(IntptrTy, 8)), WrapMask);  in emitPrologue()
1322       IRB.CreateStore(ThreadLongNew, SlotPtr);  in emitPrologue()
1339     ShadowBase = IRB.CreateAdd(  in emitPrologue()
1340         IRB.CreateOr(  in emitPrologue()
1344     ShadowBase = IRB.CreateIntToPtr(ShadowBase, PtrTy);  in emitPrologue()
1348 Value *HWAddressSanitizer::readRegister(IRBuilder<> &IRB, StringRef Name) {  in readRegister()  argument
1349   Module *M = IRB.GetInsertBlock()->getParent()->getParent();  in readRegister()
1354   return IRB.CreateCall(ReadRegister, Args);  in readRegister()
1360     IRBuilder<> IRB(LP->getNextNode());  in instrumentLandingPads()  local
1361     IRB.CreateCall(  in instrumentLandingPads()
1363         {readRegister(IRB, (TargetTriple.getArch() == Triple::x86_64) ? "rsp"  in instrumentLandingPads()
1390     IRBuilder<> IRB(AI->getNextNode());  in instrumentStack()  local
1393     Value *Tag = getAllocaTag(IRB, StackTag, N);  in instrumentStack()
1394     Value *AILong = IRB.CreatePointerCast(AI, IntptrTy);  in instrumentStack()
1395     Value *AINoTagLong = untagPointer(IRB, AILong);  in instrumentStack()
1396     Value *Replacement = tagPointer(IRB, AI->getType(), AINoTagLong, Tag);  in instrumentStack()
1404     Value *AICast = IRB.CreatePointerCast(AI, PtrTy);  in instrumentStack()
1441       IRB.SetInsertPoint(Node);  in instrumentStack()
1446       tagAlloca(IRB, AI, UARTag, AlignedSize);  in instrumentStack()
1459       IRB.SetInsertPoint(Start->getNextNode());  in instrumentStack()
1460       tagAlloca(IRB, AI, Tag, Size);  in instrumentStack()
1467       tagAlloca(IRB, AI, Tag, Size);  in instrumentStack()
1746     IRBuilder<> IRB(BB);  in instrumentPersonalityFunctions()  local
1747     CallInst *WrapperCall = IRB.CreateCall(  in instrumentPersonalityFunctions()
1754     IRB.CreateRet(WrapperCall);  in instrumentPersonalityFunctions()