Lines Matching refs:Legal
451 Builder(PSE.getSE()->getContext()), Legal(LVL), Cost(CM), BFI(BFI), in InnerLoopVectorizer()
705 LoopVectorizationLegality *Legal; member in llvm::InnerLoopVectorizer
1056 Legal->blockNeedsPredication(UnderlyingInstr.getParent())) in collectPoisonGeneratingRecipes()
1071 Legal->blockNeedsPredication(Member->getParent()); in collectPoisonGeneratingRecipes()
1137 LoopVectorizationLegality *Legal, in LoopVectorizationCostModel() argument
1144 : ScalarEpilogueStatus(SEL), TheLoop(L), PSE(PSE), LI(LI), Legal(Legal), in LoopVectorizationCostModel()
1376 if (Op != Legal->getPrimaryInduction() && TTI.isTruncateFree(SrcTy, DestTy)) in isOptimizableIVTruncate()
1380 return Legal->isInductionPhi(Op); in isOptimizableIVTruncate()
1402 return Legal->isConsecutivePtr(DataType, Ptr) && in isLegalMaskedStore()
1409 return Legal->isConsecutivePtr(DataType, Ptr) && in isLegalMaskedLoad()
1432 return (all_of(Legal->getReductionVars(), [&](auto &Reduction) -> bool { in canVectorizeReductions()
1455 if (Legal->isUniformMemOp(*I) && isa<LoadInst>(I) && in isPredicatedInst()
1456 !Legal->blockNeedsPredication(I->getParent())) in isPredicatedInst()
1463 return Legal->isMaskRequired(I); in isPredicatedInst()
1523 return foldTailByMasking() || Legal->blockNeedsPredication(BB); in blockNeedsPredicationForAnyReason()
1798 LoopVectorizationLegality *Legal; member in llvm::LoopVectorizationCostModel
2811 Type *IdxTy = Legal->getWidestInductionType(); in getOrCreateTripCount()
3119 Instruction *OldInduction = Legal->getPrimaryInduction(); in createInductionResumeValues()
3120 for (auto &InductionEntry : Legal->getInductionVars()) { in createInductionResumeValues()
3675 for (auto &Entry : Legal->getInductionVars()) in fixVectorizedLoop()
3846 assert(Legal->isReductionVariable(OrigPhi) && in fixReduction()
4379 for (auto &Induction : Legal->getInductionVars()) { in collectLoopScalars()
4385 if (Ind == Legal->getPrimaryInduction() && foldTailByMasking()) in collectLoopScalars()
4439 if (!Legal->isMaskRequired(I)) in isScalarWithPredication()
4506 Legal->isMaskRequired(I); in interleavedAccessCanBeWidened()
4541 if (!Legal->isConsecutivePtr(ScalarTy, Ptr)) in memoryInstructionCanBeWidened()
4620 if (isa<LoadInst>(I) && Legal->isUniformMemOp(*I)) { in collectLoopUniforms()
4679 if (isa<LoadInst>(I) && Legal->isUniformMemOp(I)) in collectLoopUniforms()
4717 if (OP && Legal->isFirstOrderRecurrence(OP)) in collectLoopUniforms()
4736 for (auto &Induction : Legal->getInductionVars()) { in collectLoopUniforms()
4772 if (Legal->getRuntimePointerChecking()->Need) { in runtimeChecksRequired()
4791 if (!Legal->getLAI()->getSymbolicStrides().empty()) { in runtimeChecksRequired()
4845 if (Legal->isSafeForAnyVectorWidth()) in getMaxLegalScalableVF()
4875 PowerOf2Floor(Legal->getMaxSafeVectorWidthInBits() / WidestType); in computeFeasibleMaxVF()
4971 if (Legal->getRuntimePointerChecking()->Need && TTI.hasBranchDivergence()) { in computeMaxVF()
5076 if (Legal->prepareToFoldTailByMasking()) { in computeMaxVF()
5406 [&](PHINode &Phi) { return Legal->isFirstOrderRecurrence(&Phi); })) in isCandidateForEpilogueVectorization()
5411 for (auto &Entry : Legal->getInductionVars()) { in isCandidateForEpilogueVectorization()
5425 if (any_of(Legal->getInductionVars(), [&](auto &Entry) { in isCandidateForEpilogueVectorization()
5541 if (ElementTypesInLoop.empty() && !Legal->getReductionVars().empty()) { in getSmallestAndWidestTypes()
5545 for (auto &PhiDescriptorPair : Legal->getReductionVars()) { in getSmallestAndWidestTypes()
5584 if (!Legal->isReductionVariable(PN)) in collectElementTypesForWidening()
5587 Legal->getReductionVars().find(PN)->second; in collectElementTypesForWidening()
5628 if (Legal->getMaxSafeDepDistBytes() != -1U) in selectInterleaveCount()
5632 const bool HasReductions = !Legal->getReductionVars().empty(); in selectInterleaveCount()
5758 return Legal->blockNeedsPredication(BB); in selectInterleaveCount()
5761 (VF.isScalar() && Legal->getRuntimePointerChecking()->Need); in selectInterleaveCount()
5780 unsigned NumStores = Legal->getNumStores(); in selectInterleaveCount()
5781 unsigned NumLoads = Legal->getNumLoads(); in selectInterleaveCount()
5791 any_of(Legal->getReductionVars(), [&](auto &Reduction) -> bool { in selectInterleaveCount()
5808 any_of(Legal->getReductionVars(), [&](auto &Reduction) -> bool { in selectInterleaveCount()
6057 assert((isPredicatedInst(I, VF) || Legal->isUniformMemOp(*I)) && in useEmulatedMaskMemRefHack()
6255 if (VF.isScalar() && Legal->blockNeedsPredication(BB)) in expectedCost()
6272 LoopVectorizationLegality *Legal, in getAddressAccessSCEV() argument
6287 !Legal->isInductionVariable(Opd)) in getAddressAccessSCEV()
6295 static bool isStrideMul(Instruction *I, LoopVectorizationLegality *Legal) { in isStrideMul() argument
6296 return Legal->hasStride(I->getOperand(0)) || in isStrideMul()
6297 Legal->hasStride(I->getOperand(1)); in isStrideMul()
6319 const SCEV *PtrSCEV = getAddressAccessSCEV(Ptr, Legal, PSE, TheLoop); in getMemInstScalarizationCost()
6366 int ConsecutiveStride = Legal->isConsecutivePtr(ValTy, Ptr); in getConsecutiveMemOpCost()
6373 if (Legal->isMaskRequired(I)) in getConsecutiveMemOpCost()
6390 assert(Legal->isUniformMemOp(*I)); in getUniformMemOpCost()
6405 bool isLoopInvariantStoreValue = Legal->isUniform(SI->getValueOperand()); in getUniformMemOpCost()
6425 I->getOpcode(), VectorTy, Ptr, Legal->isMaskRequired(I), Alignment, in getGatherScatterCost()
6459 AS, TTI::TCK_RecipThroughput, Legal->isMaskRequired(I), UseMaskForGaps); in getInterleaveGroupCost()
6463 assert(!Legal->isMaskRequired(I) && in getInterleaveGroupCost()
6518 Legal->getReductionVars().find(cast<PHINode>(ReductionPhi))->second; in getReductionPatternCost()
6767 if (Legal->isUniformMemOp(I)) { in setCostBasedWideningDecision()
6797 int ConsecutiveStride = Legal->isConsecutivePtr( in setCostBasedWideningDecision()
7002 if (VF.isVector() && Legal->isFirstOrderRecurrence(Phi)) in getInstructionCost()
7066 if (I->getOpcode() == Instruction::Mul && isStrideMul(I, Legal)) in getInstructionCost()
7079 if (Op2VK == TargetTransformInfo::OK_AnyValue && Legal->isUniform(Op2)) in getInstructionCost()
7183 return Legal->isMaskRequired(I) ? TTI::CastContextHint::Masked in getInstructionCost()
7314 Legal->isInvariantAddressOfReduction(SI->getPointerOperand())) in collectValuesToIgnore()
7320 for (auto &Reduction : Legal->getReductionVars()) { in collectValuesToIgnore()
7327 for (auto &Induction : Legal->getInductionVars()) { in collectValuesToIgnore()
7335 for (auto &Reduction : Legal->getReductionVars()) { in collectInLoopReductions()
7567 const LoopAccessInfo *LAI = ILV.Legal->getLAI(); in executePlan()
7857 Type *IdxTy = Legal->getWidestInductionType(); in createEpilogueVectorizedLoopSkeleton()
8088 if (Legal->isMaskRequired(I)) in tryToWidenMemory()
8147 if (auto *II = Legal->getIntOrFpInductionDescriptor(Phi)) in tryToOptimizeInductionPHI()
8152 if (auto *II = Legal->getPointerInductionDescriptor(Phi)) { in tryToOptimizeInductionPHI()
8184 const InductionDescriptor &II = *Legal->getIntOrFpInductionDescriptor(Phi); in tryToOptimizeInductionTruncate()
8489 assert((Legal->isReductionVariable(Phi) || in tryToCreateWidenRecipe()
8490 Legal->isFirstOrderRecurrence(Phi)) && in tryToCreateWidenRecipe()
8493 if (Legal->isReductionVariable(Phi)) { in tryToCreateWidenRecipe()
8495 Legal->getReductionVars().find(Phi)->second; in tryToCreateWidenRecipe()
8555 auto &ConditionalAssumes = Legal->getConditionalAssumes(); in buildVPlansWithVPRecipes()
8558 MapVector<Instruction *, Instruction *> &SinkAfter = Legal->getSinkAfter(); in buildVPlansWithVPRecipes()
8565 for (auto &P : Legal->getSinkAfter()) { in buildVPlansWithVPRecipes()
8696 VPRecipeBuilder RecipeBuilder(OrigLoop, TLI, Legal, CM, PSE, Builder); in buildVPlanWithVPRecipes()
8712 Legal->getReductionVars().find(Phi)->second.getRecurrenceKind(); in buildVPlanWithVPRecipes()
8765 getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()); in buildVPlanWithVPRecipes()
8766 addCanonicalIVRecipes(*Plan, Legal->getWidestInductionType(), in buildVPlanWithVPRecipes()
8810 Legal->isInvariantAddressOfReduction(SI->getPointerOperand())) in buildVPlanWithVPRecipes()
9074 [this](PHINode *P) { return Legal->getIntOrFpInductionDescriptor(P); }, in buildVPlan()
9083 addCanonicalIVRecipes(*Plan, Legal->getWidestInductionType(), DebugLoc(), in buildVPlan()
9099 Legal->getReductionVars().find(Phi)->second; in adjustRecipesForReductions()