Lines Matching refs:SI
235 isSafeSelectToSpeculate(SelectInst &SI, bool PreserveCFG);
744 iterator SI, SJ; member in __anonc3d49dbe0411::Partition
751 Partition(iterator SI) : SI(SI), SJ(SI) {} in Partition() argument
774 bool empty() const { return SI == SJ; } in empty()
785 iterator begin() const { return SI; } in begin()
826 partition_iterator(AllocaSlices::iterator SI, AllocaSlices::iterator SE) in partition_iterator() argument
827 : P(SI), SE(SE) { in partition_iterator()
830 if (SI != SE) in partition_iterator()
838 assert((P.SI != SE || !P.SplitTails.empty()) && in advance()
868 if (P.SI == SE) { in advance()
875 if (P.SI != P.SJ) { in advance()
886 P.SI = P.SJ; in advance()
889 if (P.SI == SE) { in advance()
898 if (!P.SplitTails.empty() && P.SI->beginOffset() != P.EndOffset && in advance()
899 !P.SI->isSplittable()) { in advance()
901 P.EndOffset = P.SI->beginOffset(); in advance()
911 P.BeginOffset = P.SplitTails.empty() ? P.SI->beginOffset() : P.EndOffset; in advance()
912 P.EndOffset = P.SI->endOffset(); in advance()
917 if (!P.SI->isSplittable()) { in advance()
920 assert(P.BeginOffset == P.SI->beginOffset()); in advance()
938 assert(P.SI->isSplittable() && "Forming a splittable partition!"); in advance()
966 if (P.SI == RHS.P.SI && P.SplitTails.empty() == RHS.P.SplitTails.empty()) { in operator ==()
997 static Value *foldSelectInst(SelectInst &SI) { in foldSelectInst() argument
1001 if (ConstantInt *CI = dyn_cast<ConstantInt>(SI.getCondition())) in foldSelectInst()
1002 return SI.getOperand(1 + CI->isZero()); in foldSelectInst()
1003 if (SI.getOperand(1) == SI.getOperand(2)) in foldSelectInst()
1004 return SI.getOperand(1); in foldSelectInst()
1171 void visitStoreInst(StoreInst &SI) { in visitStoreInst() argument
1172 Value *ValOp = SI.getValueOperand(); in visitStoreInst()
1174 return PI.setEscapedAndAborted(&SI); in visitStoreInst()
1176 return PI.setAborted(&SI); in visitStoreInst()
1180 return PI.setAborted(&SI); in visitStoreInst()
1196 << " use: " << SI << "\n"); in visitStoreInst()
1197 return markAsDead(SI); in visitStoreInst()
1200 assert((!SI.isSimple() || ValOp->getType()->isSingleValueType()) && in visitStoreInst()
1202 handleLoadOrStore(ValOp->getType(), SI, Offset, Size, SI.isVolatile()); in visitStoreInst()
1347 if (StoreInst *SI = dyn_cast<StoreInst>(I)) { in hasUnsafePHIOrSelectUse() local
1348 Value *Op = SI->getOperand(0); in hasUnsafePHIOrSelectUse()
1350 return SI; in hasUnsafePHIOrSelectUse()
1353 PI.setAborted(SI); in hasUnsafePHIOrSelectUse()
1436 void visitSelectInst(SelectInst &SI) { visitPHINodeOrSelectInst(SI); } in visitSelectInst() argument
1528 } else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) { in findCommonType() local
1529 UserTy = SI->getValueOperand()->getType(); in findCommonType()
1731 isSafeLoadOfSelectToSpeculate(LoadInst &LI, SelectInst &SI, bool PreserveCFG) { in isSafeLoadOfSelectToSpeculate() argument
1735 const DataLayout &DL = SI.getModule()->getDataLayout(); in isSafeLoadOfSelectToSpeculate()
1736 for (Value *Value : {SI.getTrueValue(), SI.getFalseValue()}) in isSafeLoadOfSelectToSpeculate()
1739 Spec.setAsSpeculatable(/*isTrueVal=*/Value == SI.getTrueValue()); in isSafeLoadOfSelectToSpeculate()
1747 SROA::isSafeSelectToSpeculate(SelectInst &SI, bool PreserveCFG) { in isSafeSelectToSpeculate() argument
1750 for (User *U : SI.users()) { in isSafeSelectToSpeculate()
1782 isSafeLoadOfSelectToSpeculate(*LI, SI, PreserveCFG); in isSafeSelectToSpeculate()
1793 static void speculateSelectInstLoads(SelectInst &SI, LoadInst &LI, in speculateSelectInstLoads() argument
1795 LLVM_DEBUG(dbgs() << " original load: " << SI << "\n"); in speculateSelectInstLoads()
1797 Value *TV = SI.getTrueValue(); in speculateSelectInstLoads()
1798 Value *FV = SI.getFalseValue(); in speculateSelectInstLoads()
1823 Value *V = IRB.CreateSelect(SI.getCondition(), TL, FL, in speculateSelectInstLoads()
1831 static void rewriteMemOpOfSelect(SelectInst &SI, T &I, in rewriteMemOpOfSelect() argument
1840 SplitBlockAndInsertIfThenElse(SI.getCondition(), &I, &ThenTerm, &ElseTerm, in rewriteMemOpOfSelect()
1841 SI.getMetadata(LLVMContext::MD_prof), &DTU); in rewriteMemOpOfSelect()
1843 SplitBlockAndInsertIfThen(SI.getCondition(), &I, /*Unreachable=*/false, in rewriteMemOpOfSelect()
1844 SI.getMetadata(LLVMContext::MD_prof), &DTU, in rewriteMemOpOfSelect()
1872 Value *Ptr = SI.getOperand(1 + SuccIdx); in rewriteMemOpOfSelect()
1892 else if (auto *SI = dyn_cast<StoreInst>(&I)) in rewriteMemOpOfSelect() local
1893 rewriteMemOpOfSelect(SelInst, *SI, Spec, DTU); in rewriteMemOpOfSelect()
1898 static bool rewriteSelectInstMemOps(SelectInst &SI, in rewriteSelectInstMemOps() argument
1902 LLVM_DEBUG(dbgs() << " original select: " << SI << "\n"); in rewriteSelectInstMemOps()
1915 speculateSelectInstLoads(SI, cast<LoadInst>(*I), IRB); in rewriteSelectInstMemOps()
1918 rewriteMemOpOfSelect(SI, *I, Spec, *DTU); in rewriteSelectInstMemOps()
1924 for (User *U : make_early_inc_range(SI.users())) in rewriteSelectInstMemOps()
1926 SI.eraseFromParent(); in rewriteSelectInstMemOps()
2119 } else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) { in isVectorPromotionViableForSlice() local
2120 if (SI->isVolatile()) in isVectorPromotionViableForSlice()
2122 Type *STy = SI->getValueOperand()->getType(); in isVectorPromotionViableForSlice()
2349 else if (auto *SI = dyn_cast<StoreInst>(S.getUse()->getUser())) in isVectorPromotionViable() local
2350 Ty = SI->getValueOperand()->getType(); in isVectorPromotionViable()
2435 } else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) { in isIntegerWideningViableForSlice() local
2436 Type *ValueTy = SI->getValueOperand()->getType(); in isIntegerWideningViableForSlice()
2437 if (SI->isVolatile()) in isIntegerWideningViableForSlice()
3008 bool rewriteVectorizedStoreInst(Value *V, StoreInst &SI, Value *OldOp, in rewriteVectorizedStoreInst() argument
3032 Store->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access, in rewriteVectorizedStoreInst()
3036 Pass.DeadInsts.push_back(&SI); in rewriteVectorizedStoreInst()
3039 migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, &SI, in rewriteVectorizedStoreInst()
3045 bool rewriteIntegerStore(Value *V, StoreInst &SI, AAMDNodes AATags) { in rewriteIntegerStore() argument
3047 assert(!SI.isVolatile()); in rewriteIntegerStore()
3055 V = insertInteger(DL, IRB, Old, SI.getValueOperand(), Offset, "insert"); in rewriteIntegerStore()
3059 Store->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access, in rewriteIntegerStore()
3064 migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, &SI, in rewriteIntegerStore()
3068 Pass.DeadInsts.push_back(&SI); in rewriteIntegerStore()
3073 bool visitStoreInst(StoreInst &SI) { in visitStoreInst() argument
3074 LLVM_DEBUG(dbgs() << " original: " << SI << "\n"); in visitStoreInst()
3075 Value *OldOp = SI.getOperand(1); in visitStoreInst()
3078 AAMDNodes AATags = SI.getAAMetadata(); in visitStoreInst()
3079 Value *V = SI.getValueOperand(); in visitStoreInst()
3088 assert(!SI.isVolatile()); in visitStoreInst()
3093 IntegerType *NarrowTy = Type::getIntNTy(SI.getContext(), SliceSize * 8); in visitStoreInst()
3099 return rewriteVectorizedStoreInst(V, SI, OldOp, AATags); in visitStoreInst()
3101 return rewriteIntegerStore(V, SI, AATags); in visitStoreInst()
3109 getPtrToNewAI(SI.getPointerAddressSpace(), SI.isVolatile()); in visitStoreInst()
3112 IRB.CreateAlignedStore(V, NewPtr, NewAI.getAlign(), SI.isVolatile()); in visitStoreInst()
3114 unsigned AS = SI.getPointerAddressSpace(); in visitStoreInst()
3117 IRB.CreateAlignedStore(V, NewPtr, getSliceAlign(), SI.isVolatile()); in visitStoreInst()
3119 NewSI->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access, in visitStoreInst()
3123 if (SI.isVolatile()) in visitStoreInst()
3124 NewSI->setAtomic(SI.getOrdering(), SI.getSyncScopeID()); in visitStoreInst()
3126 NewSI->setAlignment(SI.getAlign()); in visitStoreInst()
3128 migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, &SI, in visitStoreInst()
3132 Pass.DeadInsts.push_back(&SI); in visitStoreInst()
3138 !SI.isVolatile(); in visitStoreInst()
3615 if (StoreInst *SI = dyn_cast<StoreInst>(I)) { in fixLoadStoreAlign() local
3616 SI->setAlignment(std::min(SI->getAlign(), getSliceAlign())); in fixLoadStoreAlign()
3663 bool visitSelectInst(SelectInst &SI) { in visitSelectInst() argument
3664 LLVM_DEBUG(dbgs() << " original: " << SI << "\n"); in visitSelectInst()
3665 assert((SI.getTrueValue() == OldPtr || SI.getFalseValue() == OldPtr) && in visitSelectInst()
3672 if (SI.getOperand(1) == OldPtr) in visitSelectInst()
3673 SI.setOperand(1, NewPtr); in visitSelectInst()
3674 if (SI.getOperand(2) == OldPtr) in visitSelectInst()
3675 SI.setOperand(2, NewPtr); in visitSelectInst()
3677 LLVM_DEBUG(dbgs() << " to: " << SI << "\n"); in visitSelectInst()
3681 fixLoadStoreAlign(SI); in visitSelectInst()
3686 SelectUsers.insert(&SI); in visitSelectInst()
3936 bool visitStoreInst(StoreInst &SI) { in visitStoreInst() argument
3937 if (!SI.isSimple() || SI.getPointerOperand() != *U) in visitStoreInst()
3939 Value *V = SI.getValueOperand(); in visitStoreInst()
3944 LLVM_DEBUG(dbgs() << " original: " << SI << "\n"); in visitStoreInst()
3945 StoreOpSplitter Splitter(&SI, *U, V->getType(), SI.getAAMetadata(), &SI, in visitStoreInst()
3946 getAdjustedAlignment(&SI, 0), DL, IRB); in visitStoreInst()
3948 Visited.erase(&SI); in visitStoreInst()
3951 at::deleteAssignmentMarkers(&SI); in visitStoreInst()
3952 SI.eraseFromParent(); in visitStoreInst()
4080 bool visitSelectInst(SelectInst &SI) { in visitSelectInst() argument
4081 enqueueUsers(SI); in visitSelectInst()
4307 else if (auto *SI = dyn_cast<StoreInst>(I)) in presplitLoadsAndStores() local
4308 if (auto *LI = dyn_cast<LoadInst>(SI->getValueOperand())) in presplitLoadsAndStores()
4324 auto *SI = dyn_cast<StoreInst>(LU); in presplitLoadsAndStores() local
4325 if (!SI || !SI->isSimple()) in presplitLoadsAndStores()
4336 } else if (auto *SI = dyn_cast<StoreInst>(I)) { in presplitLoadsAndStores() local
4337 if (S.getUse() != &SI->getOperandUse(SI->getPointerOperandIndex())) in presplitLoadsAndStores()
4340 auto *StoredLoad = dyn_cast<LoadInst>(SI->getValueOperand()); in presplitLoadsAndStores()
4343 assert(!SI->isVolatile() && "Cannot split volatile stores!"); in presplitLoadsAndStores()
4345 Stores.push_back(SI); in presplitLoadsAndStores()
4387 llvm::erase_if(Stores, [&UnsplittableLoads, &SplitOffsetsMap](StoreInst *SI) { in presplitLoadsAndStores() argument
4390 auto *LI = cast<LoadInst>(SI->getValueOperand()); in presplitLoadsAndStores()
4402 auto &StoreOffsets = SplitOffsetsMap[SI]; in presplitLoadsAndStores()
4412 << " " << *SI << "\n"); in presplitLoadsAndStores()
4425 llvm::erase_if(Stores, [&UnsplittableLoads](StoreInst *SI) { in presplitLoadsAndStores() argument
4426 auto *LI = cast<LoadInst>(SI->getValueOperand()); in presplitLoadsAndStores()
4525 StoreInst *SI = cast<StoreInst>(LU); in presplitLoadsAndStores() local
4526 if (!Stores.empty() && SplitOffsetsMap.count(SI)) { in presplitLoadsAndStores()
4528 LLVM_DEBUG(dbgs() << " Deferred splitting of store: " << *SI in presplitLoadsAndStores()
4533 Value *StoreBasePtr = SI->getPointerOperand(); in presplitLoadsAndStores()
4534 IRB.SetInsertPoint(SI); in presplitLoadsAndStores()
4536 LLVM_DEBUG(dbgs() << " Splitting store of load: " << *SI << "\n"); in presplitLoadsAndStores()
4541 auto *PartPtrTy = SI->getPointerOperandType(); in presplitLoadsAndStores()
4543 auto AS = SI->getPointerAddressSpace(); in presplitLoadsAndStores()
4549 getAdjustedAlignment(SI, PartOffset), in presplitLoadsAndStores()
4551 PStore->copyMetadata(*SI, {LLVMContext::MD_mem_parallel_loop_access, in presplitLoadsAndStores()
4570 DeadInsts.push_back(SI); in presplitLoadsAndStores()
4587 for (StoreInst *SI : Stores) { in presplitLoadsAndStores()
4588 auto *LI = cast<LoadInst>(SI->getValueOperand()); in presplitLoadsAndStores()
4594 auto &Offsets = SplitOffsetsMap[SI]; in presplitLoadsAndStores()
4602 Instruction *StoreBasePtr = cast<Instruction>(SI->getPointerOperand()); in presplitLoadsAndStores()
4604 LLVM_DEBUG(dbgs() << " Splitting store: " << *SI << "\n"); in presplitLoadsAndStores()
4622 auto *StorePartPtrTy = SI->getPointerOperandType(); in presplitLoadsAndStores()
4643 IRB.SetInsertPoint(SI); in presplitLoadsAndStores()
4644 auto AS = SI->getPointerAddressSpace(); in presplitLoadsAndStores()
4650 getAdjustedAlignment(SI, PartOffset), in presplitLoadsAndStores()
4652 PStore->copyMetadata(*SI, {LLVMContext::MD_mem_parallel_loop_access, in presplitLoadsAndStores()
4704 assert(*LI->user_begin() == SI && "Single use isn't this store!"); in presplitLoadsAndStores()
4707 DeadInsts.push_back(SI); in presplitLoadsAndStores()