Lines Matching refs:State
132 void VPLiveOut::fixPhi(VPlan &Plan, VPTransformState &State) { in fixPhi() argument
133 auto Lane = VPLane::getLastLaneForVF(State.VF); in fixPhi()
137 Phi->addIncoming(State.get(ExitValue, VPIteration(State.UF - 1, Lane)), in fixPhi()
138 State.Builder.GetInsertBlock()); in fixPhi()
187 void VPInstruction::generateInstruction(VPTransformState &State, in generateInstruction() argument
189 IRBuilderBase &Builder = State.Builder; in generateInstruction()
193 Value *A = State.get(getOperand(0), Part); in generateInstruction()
194 Value *B = State.get(getOperand(1), Part); in generateInstruction()
197 State.set(this, V, Part); in generateInstruction()
203 Value *A = State.get(getOperand(0), Part); in generateInstruction()
205 State.set(this, V, Part); in generateInstruction()
209 Value *IV = State.get(getOperand(0), Part); in generateInstruction()
210 Value *TC = State.get(getOperand(1), Part); in generateInstruction()
212 State.set(this, V, Part); in generateInstruction()
216 Value *Cond = State.get(getOperand(0), Part); in generateInstruction()
217 Value *Op1 = State.get(getOperand(1), Part); in generateInstruction()
218 Value *Op2 = State.get(getOperand(2), Part); in generateInstruction()
220 State.set(this, V, Part); in generateInstruction()
225 Value *VIVElem0 = State.get(getOperand(0), VPIteration(Part, 0)); in generateInstruction()
227 Value *ScalarTC = State.get(getOperand(1), Part); in generateInstruction()
230 auto *PredTy = VectorType::get(Int1Ty, State.VF); in generateInstruction()
234 State.set(this, Call, Part); in generateInstruction()
251 auto *V1 = State.get(getOperand(0), 0); in generateInstruction()
252 Value *PartMinus1 = Part == 0 ? V1 : State.get(getOperand(1), Part - 1); in generateInstruction()
254 State.set(this, PartMinus1, Part); in generateInstruction()
256 Value *V2 = State.get(getOperand(1), Part); in generateInstruction()
257 State.set(this, Builder.CreateVectorSplice(PartMinus1, V2, -1, Name), in generateInstruction()
267 auto *Phi = State.get(getOperand(0), 0); in generateInstruction()
271 createStepForVF(Builder, Phi->getType(), State.VF, State.UF); in generateInstruction()
274 Next = State.get(this, 0); in generateInstruction()
277 State.set(this, Next, Part); in generateInstruction()
284 auto *IV = State.get(getOperand(0), VPIteration(0, 0)); in generateInstruction()
286 State.set(this, IV, Part); in generateInstruction()
292 Value *Step = createStepForVF(Builder, IV->getType(), State.VF, Part); in generateInstruction()
294 State.set(this, Next, Part); in generateInstruction()
301 Value *Cond = State.get(getOperand(0), VPIteration(Part, 0)); in generateInstruction()
312 CondBr->setSuccessor(1, State.CFG.VPBB2IRBB[Header]); in generateInstruction()
322 Value *IV = State.get(getOperand(0), Part); in generateInstruction()
323 Value *TC = State.get(getOperand(1), Part); in generateInstruction()
337 State.CFG.VPBB2IRBB[Header]); in generateInstruction()
347 void VPInstruction::execute(VPTransformState &State) { in execute() argument
348 assert(!State.Instance && "VPInstruction executing an Instance"); in execute()
349 IRBuilderBase::FastMathFlagGuard FMFGuard(State.Builder); in execute()
350 State.Builder.setFastMathFlags(FMF); in execute()
351 for (unsigned Part = 0; Part < State.UF; ++Part) in execute()
352 generateInstruction(State, Part); in execute()
467 void VPWidenSelectRecipe::execute(VPTransformState &State) { in execute() argument
469 State.setDebugLocFromInst(&I); in execute()
476 InvariantCond ? State.get(getOperand(0), VPIteration(0, 0)) : nullptr; in execute()
478 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
479 Value *Cond = InvarCond ? InvarCond : State.get(getOperand(0), Part); in execute()
480 Value *Op0 = State.get(getOperand(1), Part); in execute()
481 Value *Op1 = State.get(getOperand(2), Part); in execute()
482 Value *Sel = State.Builder.CreateSelect(Cond, Op0, Op1); in execute()
483 State.set(this, Sel, Part); in execute()
484 State.addMetadata(Sel, &I); in execute()
488 void VPWidenRecipe::execute(VPTransformState &State) { in execute() argument
490 auto &Builder = State.Builder; in execute()
518 State.setDebugLocFromInst(&I); in execute()
520 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
523 Ops.push_back(State.get(VPOp, Part)); in execute()
535 if (State.MayGeneratePoisonRecipes.contains(this)) in execute()
540 State.set(this, V, Part); in execute()
541 State.addMetadata(V, &I); in execute()
547 State.setDebugLocFromInst(&I); in execute()
549 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
550 Value *Op = State.get(getOperand(0), Part); in execute()
553 State.set(this, Freeze, Part); in execute()
562 State.setDebugLocFromInst(Cmp); in execute()
563 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
564 Value *A = State.get(getOperand(0), Part); in execute()
565 Value *B = State.get(getOperand(1), Part); in execute()
575 State.set(this, C, Part); in execute()
576 State.addMetadata(C, &I); in execute()
595 State.setDebugLocFromInst(CI); in execute()
598 Type *DestTy = (State.VF.isScalar()) in execute()
600 : VectorType::get(CI->getType(), State.VF); in execute()
602 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
603 Value *A = State.get(getOperand(0), Part); in execute()
605 State.set(this, Cast, Part); in execute()
606 State.addMetadata(Cast, &I); in execute()
674 void VPWidenGEPRecipe::execute(VPTransformState &State) { in execute() argument
682 if (State.VF.isVector() && IsPtrLoopInvariant && IsIndexLoopInvariant.all()) { in execute()
695 auto *Clone = State.Builder.Insert(GEP->clone()); in execute()
696 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
697 Value *EntryPart = State.Builder.CreateVectorSplat(State.VF, Clone); in execute()
698 State.set(this, EntryPart, Part); in execute()
699 State.addMetadata(EntryPart, GEP); in execute()
709 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
713 ? State.get(getOperand(0), VPIteration(0, 0)) in execute()
714 : State.get(getOperand(0), Part); in execute()
722 Indices.push_back(State.get(Operand, VPIteration(0, 0))); in execute()
724 Indices.push_back(State.get(Operand, Part)); in execute()
733 GEP->isInBounds() && State.MayGeneratePoisonRecipes.count(this) == 0; in execute()
737 auto *NewGEP = State.Builder.CreateGEP(GEP->getSourceElementType(), Ptr, in execute()
739 assert((State.VF.isScalar() || NewGEP->getType()->isVectorTy()) && in execute()
741 State.set(this, NewGEP, Part); in execute()
742 State.addMetadata(NewGEP, GEP); in execute()
763 void VPBlendRecipe::execute(VPTransformState &State) { in execute() argument
764 State.setDebugLocFromInst(Phi); in execute()
781 VectorParts Entry(State.UF); in execute()
783 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
786 Value *In0 = State.get(getIncomingValue(In), Part); in execute()
792 Value *Cond = State.get(getMask(In), Part); in execute()
794 State.Builder.CreateSelect(Cond, In0, Entry[Part], "predphi"); in execute()
798 for (unsigned Part = 0; Part < State.UF; ++Part) in execute()
799 State.set(this, Entry[Part], Part); in execute()
869 void VPBranchOnMaskRecipe::execute(VPTransformState &State) { in execute() argument
870 assert(State.Instance && "Branch on Mask works only on single instance."); in execute()
872 unsigned Part = State.Instance->Part; in execute()
873 unsigned Lane = State.Instance->Lane.getKnownLane(); in execute()
878 ConditionBit = State.get(BlockInMask, Part); in execute()
880 ConditionBit = State.Builder.CreateExtractElement( in execute()
881 ConditionBit, State.Builder.getInt32(Lane)); in execute()
883 ConditionBit = State.Builder.getTrue(); in execute()
887 auto *CurrentTerminator = State.CFG.PrevBB->getTerminator(); in execute()
890 auto *CondBr = BranchInst::Create(State.CFG.PrevBB, nullptr, ConditionBit); in execute()
895 void VPPredInstPHIRecipe::execute(VPTransformState &State) { in execute() argument
896 assert(State.Instance && "Predicated instruction PHI works per instance."); in execute()
898 cast<Instruction>(State.get(getOperand(0), *State.Instance)); in execute()
911 unsigned Part = State.Instance->Part; in execute()
912 if (State.hasVectorValue(getOperand(0), Part)) { in execute()
913 Value *VectorValue = State.get(getOperand(0), Part); in execute()
915 PHINode *VPhi = State.Builder.CreatePHI(IEI->getType(), 2); in execute()
918 if (State.hasVectorValue(this, Part)) in execute()
919 State.reset(this, VPhi, Part); in execute()
921 State.set(this, VPhi, Part); in execute()
924 State.reset(getOperand(0), VPhi, Part); in execute()
927 PHINode *Phi = State.Builder.CreatePHI(PredInstType, 2); in execute()
931 if (State.hasScalarValue(this, *State.Instance)) in execute()
932 State.reset(this, Phi, *State.Instance); in execute()
934 State.set(this, Phi, *State.Instance); in execute()
937 State.reset(getOperand(0), Phi, *State.Instance); in execute()
964 void VPCanonicalIVPHIRecipe::execute(VPTransformState &State) { in execute() argument
967 Start->getType(), 2, "index", &*State.CFG.PrevBB->getFirstInsertionPt()); in execute()
969 BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this); in execute()
972 for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) in execute()
973 State.set(this, EntryPart, Part); in execute()
1001 void VPExpandSCEVRecipe::execute(VPTransformState &State) { in execute() argument
1002 assert(!State.Instance && "cannot be used in per-lane"); in execute()
1003 const DataLayout &DL = State.CFG.PrevBB->getModule()->getDataLayout(); in execute()
1007 &*State.Builder.GetInsertPoint()); in execute()
1009 for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) in execute()
1010 State.set(this, Res, Part); in execute()
1022 void VPWidenCanonicalIVRecipe::execute(VPTransformState &State) { in execute() argument
1023 Value *CanonicalIV = State.get(getOperand(0), 0); in execute()
1025 IRBuilder<> Builder(State.CFG.PrevBB->getTerminator()); in execute()
1026 ElementCount VF = State.VF; in execute()
1030 for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) { in execute()
1038 State.set(this, CanonicalVectorIV, Part); in execute()
1052 void VPFirstOrderRecurrencePHIRecipe::execute(VPTransformState &State) { in execute() argument
1053 auto &Builder = State.Builder; in execute()
1057 Type *VecTy = State.VF.isScalar() in execute()
1059 : VectorType::get(VectorInit->getType(), State.VF); in execute()
1061 BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this); in execute()
1062 if (State.VF.isVector()) { in execute()
1067 auto *RuntimeVF = getRuntimeVF(Builder, IdxTy, State.VF); in execute()
1075 VecTy, 2, "vector.recur", &*State.CFG.PrevBB->getFirstInsertionPt()); in execute()
1077 State.set(this, EntryPart, 0); in execute()
1090 void VPReductionPHIRecipe::execute(VPTransformState &State) { in execute() argument
1092 auto &Builder = State.Builder; in execute()
1098 bool ScalarPHI = State.VF.isScalar() || IsInLoop; in execute()
1100 ScalarPHI ? PN->getType() : VectorType::get(PN->getType(), State.VF); in execute()
1102 BasicBlock *HeaderBB = State.CFG.PrevBB; in execute()
1103 assert(State.CurrentVectorLoop->getHeader() == HeaderBB && in execute()
1105 unsigned LastPartForNewPhi = isOrdered() ? 1 : State.UF; in execute()
1109 State.set(this, EntryPart, Part); in execute()
1112 BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this); in execute()
1130 Builder.CreateVectorSplat(State.VF, StartV, "minmax.ident"); in execute()
1137 Iden = Builder.CreateVectorSplat(State.VF, Iden); in execute()
1146 Value *EntryPart = State.get(this, Part); in execute()
1165 void VPWidenPHIRecipe::execute(VPTransformState &State) { in execute() argument
1185 Value *Op0 = State.get(getOperand(StartIdx), 0); in execute()
1187 Value *VecPhi = State.Builder.CreatePHI(VecTy, 2, "vec.phi"); in execute()
1188 State.set(this, VecPhi, 0); in execute()
1214 void VPActiveLaneMaskPHIRecipe::execute(VPTransformState &State) { in execute() argument
1215 BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this); in execute()
1216 for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) { in execute()
1217 Value *StartMask = State.get(getOperand(0), Part); in execute()
1219 State.Builder.CreatePHI(StartMask->getType(), 2, "active.lane.mask"); in execute()
1222 State.set(this, EntryPart, Part); in execute()