Lines Matching refs:ElementCount
444 OptimizationRemarkEmitter *ORE, ElementCount VecWidth, in InnerLoopVectorizer()
445 ElementCount MinProfitableTripCount, in InnerLoopVectorizer()
663 ElementCount VF;
665 ElementCount MinProfitableTripCount;
746 ElementCount::getFixed(1), in InnerLoopUnroller()
747 ElementCount::getFixed(1), UnrollFactor, LVL, CM, in InnerLoopUnroller()
758 ElementCount MainLoopVF = ElementCount::getFixed(0);
760 ElementCount EpilogueVF = ElementCount::getFixed(0);
769 EpilogueLoopVectorizationInfo(ElementCount MVF, unsigned MUF, in EpilogueLoopVectorizationInfo()
770 ElementCount EVF, unsigned EUF) in EpilogueLoopVectorizationInfo()
941 Value *createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, in createStepForVF()
949 Value *getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF) { in getRuntimeVF()
955 ElementCount VF) { in getRuntimeVFAsFloat()
1119 bool operator()(const ElementCount &LHS, const ElementCount &RHS) const { in operator ()()
1124 using ElementCountSet = SmallSet<ElementCount, 16, ElementCountComparator>;
1151 FixedScalableVFPair computeMaxVF(ElementCount UserVF, unsigned UserIC);
1165 selectEpilogueVectorizationFactor(const ElementCount MaxVF,
1170 bool selectUserVectorizationFactor(ElementCount UserVF) { in selectUserVectorizationFactor()
1185 unsigned selectInterleaveCount(ElementCount VF, unsigned LoopCost);
1194 void setCostBasedWideningDecision(ElementCount VF);
1210 calculateRegisterUsage(ArrayRef<ElementCount> VFs);
1239 bool isProfitableToScalarize(Instruction *I, ElementCount VF) const { in isProfitableToScalarize()
1255 bool isUniformAfterVectorization(Instruction *I, ElementCount VF) const { in isUniformAfterVectorization()
1271 bool isScalarAfterVectorization(Instruction *I, ElementCount VF) const { in isScalarAfterVectorization()
1288 bool canTruncateToMinimalBitwidth(Instruction *I, ElementCount VF) const { in canTruncateToMinimalBitwidth()
1306 void setWideningDecision(Instruction *I, ElementCount VF, InstWidening W, in setWideningDecision()
1315 ElementCount VF, InstWidening W, in setWideningDecision()
1333 InstWidening getWideningDecision(Instruction *I, ElementCount VF) const { in getWideningDecision()
1340 std::pair<Instruction *, ElementCount> InstOnVF = std::make_pair(I, VF); in getWideningDecision()
1349 InstructionCost getWideningCost(Instruction *I, ElementCount VF) { in getWideningCost()
1351 std::pair<Instruction *, ElementCount> InstOnVF = std::make_pair(I, VF); in getWideningCost()
1360 bool isOptimizableIVTruncate(Instruction *I, ElementCount VF) { in isOptimizableIVTruncate()
1385 void collectInstsToScalarize(ElementCount VF);
1390 void collectUniformsAndScalars(ElementCount VF) { in collectUniformsAndScalars()
1416 ElementCount VF = ElementCount::getFixed(1)) { in isLegalGatherOrScatter()
1431 bool canVectorizeReductions(ElementCount VF) const { in canVectorizeReductions()
1442 bool isScalarWithPredication(Instruction *I, ElementCount VF) const;
1448 bool isPredicatedInst(Instruction *I, ElementCount VF) { in isPredicatedInst()
1471 ElementCount VF = ElementCount::getFixed(1));
1478 ElementCount VF = ElementCount::getFixed(1));
1493 bool requiresScalarEpilogue(ElementCount VF) const { in requiresScalarEpilogue()
1545 InstructionCost getVectorIntrinsicCost(CallInst *CI, ElementCount VF) const;
1552 InstructionCost getVectorCallCost(CallInst *CI, ElementCount VF,
1581 ElementCount UserVF,
1587 ElementCount getMaximizedVFForTarget(unsigned ConstTripCount,
1590 ElementCount MaxSafeVF,
1595 ElementCount getMaxLegalScalableVF(unsigned MaxSafeElements);
1610 using InstructionVFPair = std::pair<Instruction *, ElementCount>;
1612 expectedCost(ElementCount VF,
1617 VectorizationCostTy getInstructionCost(Instruction *I, ElementCount VF);
1621 InstructionCost getInstructionCost(Instruction *I, ElementCount VF,
1627 getReductionPatternCost(Instruction *I, ElementCount VF, Type *VectorTy,
1631 InstructionCost getMemoryInstructionCost(Instruction *I, ElementCount VF);
1634 InstructionCost getMemInstScalarizationCost(Instruction *I, ElementCount VF);
1637 InstructionCost getInterleaveGroupCost(Instruction *I, ElementCount VF);
1640 InstructionCost getGatherScatterCost(Instruction *I, ElementCount VF);
1644 InstructionCost getConsecutiveMemOpCost(Instruction *I, ElementCount VF);
1650 InstructionCost getUniformMemOpCost(Instruction *I, ElementCount VF);
1655 ElementCount VF) const;
1659 bool useEmulatedMaskMemRefHack(Instruction *I, ElementCount VF);
1673 DenseMap<ElementCount, SmallPtrSet<BasicBlock *, 4>>
1692 DenseMap<ElementCount, ScalarCostsTy> InstsToScalarize;
1696 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Uniforms;
1700 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Scalars;
1704 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> ForcedScalars;
1723 ElementCount VF);
1734 void collectLoopUniforms(ElementCount VF);
1743 void collectLoopScalars(ElementCount VF);
1747 using DecisionList = DenseMap<std::pair<Instruction *, ElementCount>,
1754 bool needsExtract(Value *V, ElementCount VF) const { in needsExtract()
1772 ElementCount VF) const { in filterExtractingOperands()
1780 const ElementCount VF) const;
1785 bool isEpilogueVectorizationProfitable(const ElementCount VF) const;
1881 const SCEVPredicate &UnionPred, ElementCount VF, unsigned IC) { in Create()
2270 Instruction::BinaryOps BinOp, ElementCount VF, in getStepVector()
2276 ElementCount VLen = ValVTy->getElementCount(); in getStepVector()
3420 LoopVectorizationCostModel::getVectorCallCost(CallInst *CI, ElementCount VF, in getVectorCallCost()
3468 static Type *MaybeVectorizeType(Type *Elt, ElementCount VF) { in MaybeVectorizeType()
3476 ElementCount VF) const { in getVectorIntrinsicCost()
4245 void LoopVectorizationCostModel::collectLoopScalars(ElementCount VF) { in collectLoopScalars()
4431 Instruction *I, ElementCount VF) const { in isScalarWithPredication()
4464 Instruction *I, ElementCount VF) { in interleavedAccessCanBeWidened()
4533 Instruction *I, ElementCount VF) { in memoryInstructionCanBeWidened()
4558 void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) { in collectLoopUniforms()
4613 auto isUniformDecision = [&](Instruction *I, ElementCount VF) { in collectLoopUniforms()
4802 ElementCount
4805 return ElementCount::getScalable(0); in getMaxLegalScalableVF()
4810 return ElementCount::getScalable(0); in getMaxLegalScalableVF()
4815 auto MaxScalableVF = ElementCount::getScalable( in getMaxLegalScalableVF()
4816 std::numeric_limits<ElementCount::ScalarTy>::max()); in getMaxLegalScalableVF()
4830 return ElementCount::getScalable(0); in getMaxLegalScalableVF()
4842 return ElementCount::getScalable(0); in getMaxLegalScalableVF()
4853 MaxScalableVF = ElementCount::getScalable( in getMaxLegalScalableVF()
4865 unsigned ConstTripCount, ElementCount UserVF, bool FoldTailByMasking) { in computeFeasibleMaxVF()
4877 auto MaxSafeFixedVF = ElementCount::getFixed(MaxSafeElements); in computeFeasibleMaxVF()
4890 if (ElementCount::isKnownLE(UserVF, MaxSafeUserVF)) { in computeFeasibleMaxVF()
4894 ElementCount::getFixed(UserVF.getKnownMinValue()), UserVF); in computeFeasibleMaxVF()
4899 assert(ElementCount::isKnownGT(UserVF, MaxSafeUserVF)); in computeFeasibleMaxVF()
4950 FixedScalableVFPair Result(ElementCount::getFixed(1), in computeFeasibleMaxVF()
4951 ElementCount::getScalable(0)); in computeFeasibleMaxVF()
4970 LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) { in computeMaxVF()
5053 ElementCount MaxFixedVF = MaxFactors.FixedVF; in computeMaxVF()
5112 ElementCount LoopVectorizationCostModel::getMaximizedVFForTarget( in getMaximizedVFForTarget()
5114 ElementCount MaxSafeVF, bool FoldTailByMasking) { in getMaximizedVFForTarget()
5121 auto MinVF = [](const ElementCount &LHS, const ElementCount &RHS) { in getMaximizedVFForTarget()
5124 return ElementCount::isKnownLT(LHS, RHS) ? LHS : RHS; in getMaximizedVFForTarget()
5129 auto MaxVectorElementCount = ElementCount::get( in getMaximizedVFForTarget()
5140 return ElementCount::getFixed(1); in getMaximizedVFForTarget()
5143 const auto TripCountEC = ElementCount::getFixed(ConstTripCount); in getMaximizedVFForTarget()
5145 ElementCount::isKnownLE(TripCountEC, MaxVectorElementCount) && in getMaximizedVFForTarget()
5156 return ElementCount::getFixed(ClampedConstTripCount); in getMaximizedVFForTarget()
5162 ElementCount MaxVF = MaxVectorElementCount; in getMaximizedVFForTarget()
5165 auto MaxVectorElementCountMaxBW = ElementCount::get( in getMaximizedVFForTarget()
5172 SmallVector<ElementCount, 8> VFs; in getMaximizedVFForTarget()
5173 for (ElementCount VS = MaxVectorElementCount * 2; in getMaximizedVFForTarget()
5174 ElementCount::isKnownLE(VS, MaxVectorElementCountMaxBW); VS *= 2) in getMaximizedVFForTarget()
5194 if (ElementCount MinVF = in getMaximizedVFForTarget()
5196 if (ElementCount::isKnownLT(MaxVF, MinVF)) { in getMaximizedVFForTarget()
5268 InstructionCost ExpectedCost = expectedCost(ElementCount::getFixed(1)).first; in selectVectorizationFactor()
5271 assert(VFCandidates.count(ElementCount::getFixed(1)) && in selectVectorizationFactor()
5274 const VectorizationFactor ScalarCost(ElementCount::getFixed(1), ExpectedCost, in selectVectorizationFactor()
5402 const Loop &L, ElementCount VF) const { in isCandidateForEpilogueVectorization()
5441 const ElementCount VF) const { in isEpilogueVectorizationProfitable()
5459 const ElementCount MainLoopVF, const LoopVectorizationPlanner &LVP) { in selectEpilogueVectorizationFactor()
5484 ElementCount ForcedEC = ElementCount::getFixed(EpilogueVectorizationForceVF); in selectEpilogueVectorizationFactor()
5512 ElementCount EstimatedRuntimeVF = MainLoopVF; in selectEpilogueVectorizationFactor()
5514 EstimatedRuntimeVF = ElementCount::getFixed(MainLoopVF.getKnownMinValue()); in selectEpilogueVectorizationFactor()
5521 ElementCount::isKnownLT(NextVF.Width, EstimatedRuntimeVF)) || in selectEpilogueVectorizationFactor()
5522 ElementCount::isKnownLT(NextVF.Width, MainLoopVF)) && in selectEpilogueVectorizationFactor()
5608 unsigned LoopVectorizationCostModel::selectInterleaveCount(ElementCount VF, in selectInterleaveCount()
5857 LoopVectorizationCostModel::calculateRegisterUsage(ArrayRef<ElementCount> VFs) { in calculateRegisterUsage()
5935 auto GetRegUsage = [&TTICapture](Type *Ty, ElementCount VF) -> unsigned { in calculateRegisterUsage()
6048 ElementCount VF) { in useEmulatedMaskMemRefHack()
6064 void LoopVectorizationCostModel::collectInstsToScalarize(ElementCount VF) { in collectInstsToScalarize()
6103 Instruction *PredInst, ScalarCostsTy &ScalarCosts, ElementCount VF) { in computePredInstDiscount()
6172 getInstructionCost(I, ElementCount::getFixed(1)).first; in computePredInstDiscount()
6216 ElementCount VF, SmallVectorImpl<InstructionVFPair> *Invalid) { in expectedCost()
6302 ElementCount VF) { in getMemInstScalarizationCost()
6361 ElementCount VF) { in getConsecutiveMemOpCost()
6389 ElementCount VF) { in getUniformMemOpCost()
6417 ElementCount VF) { in getGatherScatterCost()
6431 ElementCount VF) { in getInterleaveGroupCost()
6473 Instruction *I, ElementCount VF, Type *Ty, TTI::TargetCostKind CostKind) { in getReductionPatternCost()
6650 ElementCount VF) { in getMemoryInstructionCost()
6667 ElementCount VF) { in getInstructionCost()
6671 VF = ElementCount::getFixed(1); in getInstructionCost()
6682 (getInstructionCost(I, ElementCount::getFixed(1)).first * in getInstructionCost()
6710 ElementCount VF) const { in getScalarizationOverhead()
6749 void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) { in setCostBasedWideningDecision()
6899 getMemoryInstructionCost(I, ElementCount::getFixed(1)))); in setCostBasedWideningDecision()
6907 getMemoryInstructionCost(Member, ElementCount::getFixed(1)))); in setCostBasedWideningDecision()
6918 LoopVectorizationCostModel::getInstructionCost(Instruction *I, ElementCount VF, in getInstructionCost()
6927 ElementCount VF) -> bool { in getInstructionCost()
7140 ElementCount Width = VF; in getInstructionCost()
7148 Width = ElementCount::getFixed(1); in getInstructionCost()
7384 LoopVectorizationPlanner::planInVPlanNativePath(ElementCount UserVF) { in planInVPlanNativePath()
7386 ElementCount VF = UserVF; in planInVPlanNativePath()
7395 VF = ElementCount::getFixed(determineVPlanVF( in planInVPlanNativePath()
7405 VF = ElementCount::getFixed(4); in planInVPlanNativePath()
7429 LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) { in plan()
7449 ElementCount MaxUserVF = in plan()
7451 bool UserVFIsLegal = ElementCount::isKnownLE(UserVF, MaxUserVF); in plan()
7470 for (auto VF = ElementCount::getFixed(1); in plan()
7471 ElementCount::isKnownLE(VF, MaxFactors.FixedVF); VF *= 2) in plan()
7473 for (auto VF = ElementCount::getScalable(1); in plan()
7474 ElementCount::isKnownLE(VF, MaxFactors.ScalableVF); VF *= 2) in plan()
7488 buildVPlansWithVPRecipes(ElementCount::getFixed(1), MaxFactors.FixedVF); in plan()
7489 buildVPlansWithVPRecipes(ElementCount::getScalable(1), MaxFactors.ScalableVF); in plan()
7501 VPlan &LoopVectorizationPlanner::getBestPlanFor(ElementCount VF) const { in getBestPlanFor()
7548 void LoopVectorizationPlanner::executePlan(ElementCount BestVF, unsigned BestUF, in executePlan()
7722 ElementCount VFactor = ForEpilogue ? EPI.EpilogueVF : VF; in emitIterationCountCheck()
7925 const std::function<bool(ElementCount)> &Predicate, VFRange &Range) { in getDecisionAndClampRange()
7929 for (ElementCount TmpVF = Range.Start * 2; in getDecisionAndClampRange()
7930 ElementCount::isKnownLT(TmpVF, Range.End); TmpVF *= 2) in getDecisionAndClampRange()
7944 void LoopVectorizationPlanner::buildVPlans(ElementCount MinVF, in buildVPlans()
7945 ElementCount MaxVF) { in buildVPlans()
7947 for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFPlusOne);) { in buildVPlans()
8071 auto willWiden = [&](ElementCount VF) -> bool { in tryToWidenMemory()
8116 auto ShouldScalarizeInstruction = [&CM](Instruction *I, ElementCount VF) { in createWidenInductionRecipes()
8122 [&](ElementCount VF) { in createWidenInductionRecipes()
8156 [&](ElementCount VF) { in tryToOptimizeInductionPHI()
8174 [&](Instruction *K) -> std::function<bool(ElementCount)> { in tryToOptimizeInductionTruncate()
8175 return [=](ElementCount VF) -> bool { in tryToOptimizeInductionTruncate()
8246 [this, CI](ElementCount VF) { in tryToWidenCall()
8261 auto willWiden = [&](ElementCount VF) -> bool { in tryToWidenCall()
8286 auto WillScalarize = [this, I](ElementCount VF) -> bool { in shouldWiden()
8360 [&](ElementCount VF) { return CM.isUniformAfterVectorization(I, VF); }, in handleReplication()
8364 [&](ElementCount VF) { return CM.isPredicatedInst(I, VF); }, in handleReplication()
8520 [&](ElementCount VF) { return VF.isScalar(); }, Range)) in tryToCreateWidenRecipe()
8546 void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF, in buildVPlansWithVPRecipes()
8547 ElementCount MaxVF) { in buildVPlansWithVPRecipes()
8582 for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFPlusOne);) { in buildVPlansWithVPRecipes()
8732 auto applyIG = [IG, this](ElementCount VF) -> bool { in buildVPlanWithVPRecipes()
9022 ElementCount VF = Range.Start; in buildVPlanWithVPRecipes()
9025 for (VF *= 2; ElementCount::isKnownLT(VF, Range.End); VF *= 2) { in buildVPlanWithVPRecipes()
9067 for (ElementCount VF = Range.Start; ElementCount::isKnownLT(VF, Range.End); in buildVPlan()
9095 ElementCount MinVF) { in adjustRecipesForReductions()
9858 ElementCount UserVF = Hints.getWidth(); in processLoopInVPlanNativePath()
10011 VF.MinProfitableTripCount = ElementCount::getFixed(alignTo(MinTC, IntVF)); in areRuntimeChecksProfitable()
10020 if (ElementCount::isKnownLT(ElementCount::getFixed(*ExpectedTC), in areRuntimeChecksProfitable()
10204 ElementCount UserVF = Hints.getWidth(); in processLoop()