Lines Matching refs:State
172 void VPLiveOut::fixPhi(VPlan &Plan, VPTransformState &State) { in fixPhi() argument
173 auto Lane = VPLane::getLastLaneForVF(State.VF); in fixPhi()
181 BasicBlock *MiddleBB = State.CFG.VPBB2IRBB[MiddleVPBB]; in fixPhi()
182 Phi->addIncoming(State.get(ExitValue, VPIteration(State.UF - 1, Lane)), in fixPhi()
275 Value *VPInstruction::generateInstruction(VPTransformState &State, in generateInstruction() argument
277 IRBuilderBase &Builder = State.Builder; in generateInstruction()
282 return State.get(this, 0); in generateInstruction()
284 Value *A = State.get(getOperand(0), Part); in generateInstruction()
285 Value *B = State.get(getOperand(1), Part); in generateInstruction()
295 Value *A = State.get(getOperand(0), Part); in generateInstruction()
299 Value *A = State.get(getOperand(0), Part); in generateInstruction()
300 Value *B = State.get(getOperand(1), Part); in generateInstruction()
304 Value *Cond = State.get(getOperand(0), Part); in generateInstruction()
305 Value *Op1 = State.get(getOperand(1), Part); in generateInstruction()
306 Value *Op2 = State.get(getOperand(2), Part); in generateInstruction()
311 Value *VIVElem0 = State.get(getOperand(0), VPIteration(Part, 0)); in generateInstruction()
313 Value *ScalarTC = State.get(getOperand(1), VPIteration(Part, 0)); in generateInstruction()
316 auto *PredTy = VectorType::get(Int1Ty, State.VF); in generateInstruction()
335 auto *V1 = State.get(getOperand(0), 0); in generateInstruction()
336 Value *PartMinus1 = Part == 0 ? V1 : State.get(getOperand(1), Part - 1); in generateInstruction()
339 Value *V2 = State.get(getOperand(1), Part); in generateInstruction()
343 Value *ScalarTC = State.get(getOperand(0), {0, 0}); in generateInstruction()
345 createStepForVF(Builder, ScalarTC->getType(), State.VF, State.UF); in generateInstruction()
352 auto *IV = State.get(getOperand(0), VPIteration(0, 0)); in generateInstruction()
358 Value *Step = createStepForVF(Builder, IV->getType(), State.VF, Part); in generateInstruction()
366 Value *Cond = State.get(getOperand(0), VPIteration(Part, 0)); in generateInstruction()
377 CondBr->setSuccessor(1, State.CFG.VPBB2IRBB[Header]); in generateInstruction()
387 Value *IV = State.get(getOperand(0), Part); in generateInstruction()
388 Value *TC = State.get(getOperand(1), Part); in generateInstruction()
402 State.CFG.VPBB2IRBB[Header]); in generateInstruction()
409 return State.get(this, 0); in generateInstruction()
420 State.setDebugLocFrom(getDebugLoc()); in generateInstruction()
424 VectorParts RdxParts(State.UF); in generateInstruction()
425 for (unsigned Part = 0; Part < State.UF; ++Part) in generateInstruction()
426 RdxParts[Part] = State.get(LoopExitingDef, Part); in generateInstruction()
432 if (State.VF.isVector() && PhiTy != RdxDesc.getRecurrenceType()) { in generateInstruction()
433 Type *RdxVecTy = VectorType::get(RdxDesc.getRecurrenceType(), State.VF); in generateInstruction()
434 for (unsigned Part = 0; Part < State.UF; ++Part) in generateInstruction()
442 ReducedPartRdx = RdxParts[State.UF - 1]; in generateInstruction()
447 for (unsigned Part = 1; Part < State.UF; ++Part) { in generateInstruction()
464 if (State.VF.isVector() && !PhiR->isInLoop()) { in generateInstruction()
501 void VPInstruction::execute(VPTransformState &State) { in execute() argument
502 assert(!State.Instance && "VPInstruction executing an Instance"); in execute()
503 IRBuilderBase::FastMathFlagGuard FMFGuard(State.Builder); in execute()
508 State.Builder.setFastMathFlags(getFastMathFlags()); in execute()
509 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
510 Value *GeneratedValue = generateInstruction(State, Part); in execute()
514 State.set(this, GeneratedValue, Part); in execute()
578 void VPWidenCallRecipe::execute(VPTransformState &State) { in execute() argument
579 assert(State.VF.isVector() && "not widening"); in execute()
583 State.setDebugLocFrom(getDebugLoc()); in execute()
589 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
595 VectorType::get(CI.getType()->getScalarType(), State.VF)); in execute()
603 Arg = State.get(I.value(), VPIteration(0, 0)); in execute()
608 Arg = State.get(I.value(), VPIteration(Part, 0)); in execute()
610 Arg = State.get(I.value(), Part); in execute()
620 Module *M = State.Builder.GetInsertBlock()->getModule(); in execute()
632 CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles); in execute()
637 State.set(this, V, Part); in execute()
638 State.addMetadata(V, &CI); in execute()
683 void VPWidenSelectRecipe::execute(VPTransformState &State) { in execute() argument
684 State.setDebugLocFrom(getDebugLoc()); in execute()
691 isInvariantCond() ? State.get(getCond(), VPIteration(0, 0)) : nullptr; in execute()
693 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
694 Value *Cond = InvarCond ? InvarCond : State.get(getCond(), Part); in execute()
695 Value *Op0 = State.get(getOperand(1), Part); in execute()
696 Value *Op1 = State.get(getOperand(2), Part); in execute()
697 Value *Sel = State.Builder.CreateSelect(Cond, Op0, Op1); in execute()
698 State.set(this, Sel, Part); in execute()
699 State.addMetadata(Sel, dyn_cast_or_null<Instruction>(getUnderlyingValue())); in execute()
753 void VPWidenRecipe::execute(VPTransformState &State) { in execute() argument
754 State.setDebugLocFrom(getDebugLoc()); in execute()
755 auto &Builder = State.Builder; in execute()
783 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
786 Ops.push_back(State.get(VPOp, Part)); in execute()
794 State.set(this, V, Part); in execute()
795 State.addMetadata(V, dyn_cast_or_null<Instruction>(getUnderlyingValue())); in execute()
801 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
802 Value *Op = State.get(getOperand(0), Part); in execute()
805 State.set(this, Freeze, Part); in execute()
813 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
814 Value *A = State.get(getOperand(0), Part); in execute()
815 Value *B = State.get(getOperand(1), Part); in execute()
826 State.set(this, C, Part); in execute()
827 State.addMetadata(C, dyn_cast_or_null<Instruction>(getUnderlyingValue())); in execute()
842 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
843 assert(VectorType::get(State.TypeAnalysis.inferScalarType(this), in execute()
844 State.VF) == State.get(this, Part)->getType() && in execute()
861 void VPWidenCastRecipe::execute(VPTransformState &State) { in execute() argument
862 State.setDebugLocFrom(getDebugLoc()); in execute()
863 auto &Builder = State.Builder; in execute()
865 assert(State.VF.isVector() && "Not vectorizing?"); in execute()
866 Type *DestTy = VectorType::get(getResultType(), State.VF); in execute()
868 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
871 State.set(this, State.get(this, 0), Part); in execute()
874 Value *A = State.get(Op, Part); in execute()
876 State.set(this, Cast, Part); in execute()
877 State.addMetadata(Cast, cast_or_null<Instruction>(getUnderlyingValue())); in execute()
961 void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) { in execute() argument
962 assert(!State.Instance && "Int or FP induction being replicated."); in execute()
967 IRBuilderBase &Builder = State.Builder; in execute()
969 assert(State.VF.isVector() && "must have vector VF"); in execute()
981 Value *Step = State.get(getStepValue(), VPIteration(0, 0)); in execute()
988 BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this); in execute()
999 Value *SplatStart = Builder.CreateVectorSplat(State.VF, Start); in execute()
1001 SplatStart, Zero, Step, ID.getInductionOpcode(), State.VF, State.Builder); in execute()
1020 RuntimeVF = getRuntimeVFAsFloat(Builder, StepType, State.VF); in execute()
1022 RuntimeVF = getRuntimeVF(Builder, StepType, State.VF); in execute()
1031 ? ConstantVector::getSplat(State.VF, cast<Constant>(Mul)) in execute()
1032 : Builder.CreateVectorSplat(State.VF, Mul); in execute()
1038 VecInd->insertBefore(State.CFG.PrevBB->getFirstInsertionPt()); in execute()
1041 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
1042 State.set(this, LastInduction, Part); in execute()
1045 State.addMetadata(LastInduction, EntryVal); in execute()
1107 void VPScalarIVStepsRecipe::execute(VPTransformState &State) { in execute() argument
1109 IRBuilder<>::FastMathFlagGuard FMFG(State.Builder); in execute()
1111 State.Builder.setFastMathFlags(getFastMathFlags()); in execute()
1116 Value *BaseIV = State.get(getOperand(0), VPIteration(0, 0)); in execute()
1117 Value *Step = State.get(getStepValue(), VPIteration(0, 0)); in execute()
1118 IRBuilderBase &Builder = State.Builder; in execute()
1127 Step = State.Builder.CreateTrunc(Step, BaseIVTy); in execute()
1150 if (!FirstLaneOnly && State.VF.isScalable()) { in execute()
1151 VecIVTy = VectorType::get(BaseIVTy, State.VF); in execute()
1153 Builder.CreateStepVector(VectorType::get(IntStepTy, State.VF)); in execute()
1154 SplatStep = Builder.CreateVectorSplat(State.VF, Step); in execute()
1155 SplatIV = Builder.CreateVectorSplat(State.VF, BaseIV); in execute()
1159 unsigned EndPart = State.UF; in execute()
1161 unsigned EndLane = FirstLaneOnly ? 1 : State.VF.getKnownMinValue(); in execute()
1162 if (State.Instance) { in execute()
1163 StartPart = State.Instance->Part; in execute()
1165 StartLane = State.Instance->Lane.getKnownLane(); in execute()
1169 Value *StartIdx0 = createStepForVF(Builder, IntStepTy, State.VF, Part); in execute()
1171 if (!FirstLaneOnly && State.VF.isScalable()) { in execute()
1172 auto *SplatStartIdx = Builder.CreateVectorSplat(State.VF, StartIdx0); in execute()
1178 State.set(this, Add, Part); in execute()
1192 assert((State.VF.isScalable() || isa<Constant>(StartIdx)) && in execute()
1197 State.set(this, Add, VPIteration(Part, Lane)); in execute()
1212 void VPWidenGEPRecipe::execute(VPTransformState &State) { in execute() argument
1213 assert(State.VF.isVector() && "not widening"); in execute()
1236 Ops.push_back(State.get(getOperand(I), VPIteration(0, 0))); in execute()
1239 State.Builder.CreateGEP(GEP->getSourceElementType(), Ops[0], in execute()
1241 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
1242 Value *EntryPart = State.Builder.CreateVectorSplat(State.VF, NewGEP); in execute()
1243 State.set(this, EntryPart, Part); in execute()
1244 State.addMetadata(EntryPart, GEP); in execute()
1254 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
1258 ? State.get(getOperand(0), VPIteration(0, 0)) in execute()
1259 : State.get(getOperand(0), Part); in execute()
1267 Indices.push_back(State.get(Operand, VPIteration(0, 0))); in execute()
1269 Indices.push_back(State.get(Operand, Part)); in execute()
1274 auto *NewGEP = State.Builder.CreateGEP(GEP->getSourceElementType(), Ptr, in execute()
1276 assert((State.VF.isScalar() || NewGEP->getType()->isVectorTy()) && in execute()
1278 State.set(this, NewGEP, Part); in execute()
1279 State.addMetadata(NewGEP, GEP); in execute()
1300 void VPVectorPointerRecipe ::execute(VPTransformState &State) { in execute() argument
1301 auto &Builder = State.Builder; in execute()
1302 State.setDebugLocFrom(getDebugLoc()); in execute()
1303 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
1310 Type *IndexTy = State.VF.isScalable() && (IsReverse || Part > 0) in execute()
1313 Value *Ptr = State.get(getOperand(0), VPIteration(0, 0)); in execute()
1320 Value *RunTimeVF = getRuntimeVF(Builder, IndexTy, State.VF); in execute()
1330 Value *Increment = createStepForVF(Builder, IndexTy, State.VF, Part); in execute()
1334 State.set(this, PartPtr, Part); in execute()
1351 void VPBlendRecipe::execute(VPTransformState &State) { in execute() argument
1352 State.setDebugLocFrom(getDebugLoc()); in execute()
1369 VectorParts Entry(State.UF); in execute()
1371 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
1374 Value *In0 = State.get(getIncomingValue(In), Part); in execute()
1380 Value *Cond = State.get(getMask(In), Part); in execute()
1382 State.Builder.CreateSelect(Cond, In0, Entry[Part], "predphi"); in execute()
1386 for (unsigned Part = 0; Part < State.UF; ++Part) in execute()
1387 State.set(this, Entry[Part], Part); in execute()
1474 void VPBranchOnMaskRecipe::execute(VPTransformState &State) { in execute() argument
1475 assert(State.Instance && "Branch on Mask works only on single instance."); in execute()
1477 unsigned Part = State.Instance->Part; in execute()
1478 unsigned Lane = State.Instance->Lane.getKnownLane(); in execute()
1483 ConditionBit = State.get(BlockInMask, Part); in execute()
1485 ConditionBit = State.Builder.CreateExtractElement( in execute()
1486 ConditionBit, State.Builder.getInt32(Lane)); in execute()
1488 ConditionBit = State.Builder.getTrue(); in execute()
1492 auto *CurrentTerminator = State.CFG.PrevBB->getTerminator(); in execute()
1495 auto *CondBr = BranchInst::Create(State.CFG.PrevBB, nullptr, ConditionBit); in execute()
1500 void VPPredInstPHIRecipe::execute(VPTransformState &State) { in execute() argument
1501 assert(State.Instance && "Predicated instruction PHI works per instance."); in execute()
1503 cast<Instruction>(State.get(getOperand(0), *State.Instance)); in execute()
1516 unsigned Part = State.Instance->Part; in execute()
1517 if (State.hasVectorValue(getOperand(0), Part)) { in execute()
1518 Value *VectorValue = State.get(getOperand(0), Part); in execute()
1520 PHINode *VPhi = State.Builder.CreatePHI(IEI->getType(), 2); in execute()
1523 if (State.hasVectorValue(this, Part)) in execute()
1524 State.reset(this, VPhi, Part); in execute()
1526 State.set(this, VPhi, Part); in execute()
1529 State.reset(getOperand(0), VPhi, Part); in execute()
1532 PHINode *Phi = State.Builder.CreatePHI(PredInstType, 2); in execute()
1536 if (State.hasScalarValue(this, *State.Instance)) in execute()
1537 State.reset(this, Phi, *State.Instance); in execute()
1539 State.set(this, Phi, *State.Instance); in execute()
1542 State.reset(getOperand(0), Phi, *State.Instance); in execute()
1569 void VPCanonicalIVPHIRecipe::execute(VPTransformState &State) { in execute() argument
1572 EntryPart->insertBefore(State.CFG.PrevBB->getFirstInsertionPt()); in execute()
1574 BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this); in execute()
1577 for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) in execute()
1578 State.set(this, EntryPart, Part); in execute()
1625 void VPExpandSCEVRecipe::execute(VPTransformState &State) { in execute() argument
1626 assert(!State.Instance && "cannot be used in per-lane"); in execute()
1627 const DataLayout &DL = State.CFG.PrevBB->getModule()->getDataLayout(); in execute()
1631 &*State.Builder.GetInsertPoint()); in execute()
1632 assert(!State.ExpandedSCEVs.contains(Expr) && in execute()
1634 State.ExpandedSCEVs[Expr] = Res; in execute()
1635 for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) in execute()
1636 State.set(this, Res, {Part, 0}); in execute()
1648 void VPWidenCanonicalIVRecipe::execute(VPTransformState &State) { in execute() argument
1649 Value *CanonicalIV = State.get(getOperand(0), 0); in execute()
1651 IRBuilder<> Builder(State.CFG.PrevBB->getTerminator()); in execute()
1652 ElementCount VF = State.VF; in execute()
1656 for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) { in execute()
1664 State.set(this, CanonicalVectorIV, Part); in execute()
1678 void VPFirstOrderRecurrencePHIRecipe::execute(VPTransformState &State) { in execute() argument
1679 auto &Builder = State.Builder; in execute()
1683 Type *VecTy = State.VF.isScalar() in execute()
1685 : VectorType::get(VectorInit->getType(), State.VF); in execute()
1687 BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this); in execute()
1688 if (State.VF.isVector()) { in execute()
1693 auto *RuntimeVF = getRuntimeVF(Builder, IdxTy, State.VF); in execute()
1701 EntryPart->insertBefore(State.CFG.PrevBB->getFirstInsertionPt()); in execute()
1703 State.set(this, EntryPart, 0); in execute()
1716 void VPReductionPHIRecipe::execute(VPTransformState &State) { in execute() argument
1717 auto &Builder = State.Builder; in execute()
1728 bool ScalarPHI = State.VF.isScalar() || IsInLoop; in execute()
1730 : VectorType::get(StartV->getType(), State.VF); in execute()
1732 BasicBlock *HeaderBB = State.CFG.PrevBB; in execute()
1733 assert(State.CurrentVectorLoop->getHeader() == HeaderBB && in execute()
1735 unsigned LastPartForNewPhi = isOrdered() ? 1 : State.UF; in execute()
1739 State.set(this, EntryPart, Part); in execute()
1742 BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this); in execute()
1755 Builder.CreateVectorSplat(State.VF, StartV, "minmax.ident"); in execute()
1762 Iden = Builder.CreateVectorSplat(State.VF, Iden); in execute()
1771 Value *EntryPart = State.get(this, Part); in execute()
1790 void VPWidenPHIRecipe::execute(VPTransformState &State) { in execute() argument
1794 Value *Op0 = State.get(getOperand(0), 0); in execute()
1796 Value *VecPhi = State.Builder.CreatePHI(VecTy, 2, "vec.phi"); in execute()
1797 State.set(this, VecPhi, 0); in execute()
1823 void VPActiveLaneMaskPHIRecipe::execute(VPTransformState &State) { in execute() argument
1824 BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this); in execute()
1825 for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) { in execute()
1826 Value *StartMask = State.get(getOperand(0), Part); in execute()
1828 State.Builder.CreatePHI(StartMask->getType(), 2, "active.lane.mask"); in execute()
1831 State.set(this, EntryPart, Part); in execute()