Lines Matching refs:RdxDesc
519 bool useOrderedReductions(const RecurrenceDescriptor &RdxDesc);
530 PHINode *getReductionResumeValue(const RecurrenceDescriptor &RdxDesc);
1084 const RecurrenceDescriptor &RdxDesc) { in getReductionResumeValue() argument
1085 auto It = ReductionResumeValues.find(&RdxDesc); in getReductionResumeValue()
1226 bool useOrderedReductions(const RecurrenceDescriptor &RdxDesc) const { in useOrderedReductions()
1227 return !Hints->allowReordering() && RdxDesc.isOrdered(); in useOrderedReductions()
1433 const RecurrenceDescriptor &RdxDesc = Reduction.second; in canVectorizeReductions() local
1434 return TTI.isLegalToVectorizeReduction(RdxDesc, VF); in canVectorizeReductions()
3848 const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor(); in fixReduction() local
3850 RecurKind RK = RdxDesc.getRecurrenceKind(); in fixReduction()
3851 TrackingVH<Value> ReductionStartValue = RdxDesc.getRecurrenceStartValue(); in fixReduction()
3852 Instruction *LoopExitInst = RdxDesc.getLoopExitInstr(); in fixReduction()
3894 Sel->setFastMathFlags(RdxDesc.getFastMathFlags()); in fixReduction()
3903 RdxDesc.getOpcode(), PhiTy, in fixReduction()
3915 if (VF.isVector() && PhiTy != RdxDesc.getRecurrenceType()) { in fixReduction()
3917 Type *RdxVecTy = VectorType::get(RdxDesc.getRecurrenceType(), VF); in fixReduction()
3923 Value *Extnd = RdxDesc.isSigned() ? Builder.CreateSExt(Trunc, VecTy) in fixReduction()
3955 Builder.setFastMathFlags(RdxDesc.getFastMathFlags()); in fixReduction()
3973 createTargetReduction(Builder, TTI, RdxDesc, ReducedPartRdx, OrigPhi); in fixReduction()
3976 if (PhiTy != RdxDesc.getRecurrenceType()) in fixReduction()
3977 ReducedPartRdx = RdxDesc.isSigned() in fixReduction()
4004 ReductionResumeValues.insert({&RdxDesc, BCBlockPhi}); in fixReduction()
4008 if (StoreInst *SI = RdxDesc.IntermediateStore) { in fixReduction()
4043 const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor(); in clearReductionWrapFlags() local
4044 RecurKind RK = RdxDesc.getRecurrenceKind(); in clearReductionWrapFlags()
4169 const RecurrenceDescriptor &RdxDesc) { in useOrderedReductions() argument
4170 return Cost->useOrderedReductions(RdxDesc); in useOrderedReductions()
5546 const RecurrenceDescriptor &RdxDesc = PhiDescriptorPair.second; in getSmallestAndWidestTypes() local
5551 RdxDesc.getMinWidthCastToRecurrenceTypeInBits(), in getSmallestAndWidestTypes()
5552 RdxDesc.getRecurrenceType()->getScalarSizeInBits())); in getSmallestAndWidestTypes()
5586 const RecurrenceDescriptor &RdxDesc = in collectElementTypesForWidening() local
5588 if (PreferInLoopReductions || useOrderedReductions(RdxDesc) || in collectElementTypesForWidening()
5589 TTI.preferInLoopReduction(RdxDesc.getOpcode(), in collectElementTypesForWidening()
5590 RdxDesc.getRecurrenceType(), in collectElementTypesForWidening()
5593 T = RdxDesc.getRecurrenceType(); in collectElementTypesForWidening()
5792 const RecurrenceDescriptor &RdxDesc = Reduction.second; in selectInterleaveCount() local
5794 RdxDesc.getRecurrenceKind()); in selectInterleaveCount()
5809 const RecurrenceDescriptor &RdxDesc = Reduction.second; in selectInterleaveCount() local
5810 return RdxDesc.isOrdered(); in selectInterleaveCount()
6517 const RecurrenceDescriptor &RdxDesc = in getReductionPatternCost() local
6521 RdxDesc.getOpcode(), VectorTy, RdxDesc.getFastMathFlags(), CostKind); in getReductionPatternCost()
6525 if (RdxDesc.getRecurrenceKind() == RecurKind::FMulAdd) in getReductionPatternCost()
6532 if (useOrderedReductions(RdxDesc)) in getReductionPatternCost()
6571 /*IsMLA=*/true, IsUnsigned, RdxDesc.getRecurrenceType(), ExtType, in getReductionPatternCost()
6583 /*IsMLA=*/false, IsUnsigned, RdxDesc.getRecurrenceType(), ExtType, in getReductionPatternCost()
6617 /*IsMLA=*/true, IsUnsigned, RdxDesc.getRecurrenceType(), ExtType, in getReductionPatternCost()
6637 /*IsMLA=*/true, true, RdxDesc.getRecurrenceType(), VectorTy, in getReductionPatternCost()
7337 const RecurrenceDescriptor &RdxDesc = Reduction.second; in collectInLoopReductions() local
7340 if (RdxDesc.getRecurrenceType() != Phi->getType()) in collectInLoopReductions()
7345 unsigned Opcode = RdxDesc.getOpcode(); in collectInLoopReductions()
7346 if (!PreferInLoopReductions && !useOrderedReductions(RdxDesc) && in collectInLoopReductions()
7355 RdxDesc.getReductionOpChain(Phi, TheLoop); in collectInLoopReductions()
8494 const RecurrenceDescriptor &RdxDesc = in tryToCreateWidenRecipe() local
8496 assert(RdxDesc.getRecurrenceStartValue() == in tryToCreateWidenRecipe()
8498 PhiRecipe = new VPReductionPHIRecipe(Phi, RdxDesc, *StartV, in tryToCreateWidenRecipe()
8500 CM.useOrderedReductions(RdxDesc)); in tryToCreateWidenRecipe()
9098 const RecurrenceDescriptor &RdxDesc = in adjustRecipesForReductions() local
9102 if (MinVF.isScalar() && !CM.useOrderedReductions(RdxDesc)) in adjustRecipesForReductions()
9112 RecurKind Kind = RdxDesc.getRecurrenceKind(); in adjustRecipesForReductions()
9152 new VPReductionRecipe(&RdxDesc, R, ChainOp, VecOp, CondOp, TTI); in adjustRecipesForReductions()
9498 RecurKind Kind = RdxDesc->getRecurrenceKind(); in execute()
9499 bool IsOrdered = State.ILV->useOrderedReductions(*RdxDesc); in execute()
9502 State.Builder.setFastMathFlags(RdxDesc->getFastMathFlags()); in execute()
9508 Value *Iden = RdxDesc->getRecurrenceIdentity( in execute()
9509 Kind, VecTy->getElementType(), RdxDesc->getFastMathFlags()); in execute()
9519 NewRed = createOrderedReduction(State.Builder, *RdxDesc, NewVecOp, in execute()
9523 (Instruction::BinaryOps)RdxDesc->getOpcode(Kind), PrevInChain, in execute()
9528 NewRed = createTargetReduction(State.Builder, TTI, *RdxDesc, NewVecOp); in execute()
9532 createMinMaxOp(State.Builder, RdxDesc->getRecurrenceKind(), in execute()
9538 (Instruction::BinaryOps)RdxDesc->getOpcode(Kind), NewRed, in execute()