Lines Matching refs:VL
231 static bool allSameBlock(ArrayRef<Value *> VL) { in allSameBlock() argument
232 Instruction *I0 = dyn_cast<Instruction>(VL[0]); in allSameBlock()
235 if (all_of(VL, isVectorLikeInstWithConstOps)) in allSameBlock()
239 for (int I = 1, E = VL.size(); I < E; I++) { in allSameBlock()
240 auto *II = dyn_cast<Instruction>(VL[I]); in allSameBlock()
252 static bool allConstant(ArrayRef<Value *> VL) { in allConstant() argument
255 return all_of(VL, isConstant); in allConstant()
260 static bool isSplat(ArrayRef<Value *> VL) { in isSplat() argument
262 for (Value *V : VL) { in isSplat()
350 isFixedVectorShuffle(ArrayRef<Value *> VL, SmallVectorImpl<int> &Mask) { in isFixedVectorShuffle() argument
352 find_if(VL, [](Value *V) { return isa<ExtractElementInst>(V); }); in isFixedVectorShuffle()
353 if (It == VL.end()) in isFixedVectorShuffle()
364 Mask.assign(VL.size(), UndefMaskElem); in isFixedVectorShuffle()
365 for (unsigned I = 0, E = VL.size(); I < E; ++I) { in isFixedVectorShuffle()
367 if (isa<UndefValue>(VL[I])) in isFixedVectorShuffle()
369 auto *EI = cast<ExtractElementInst>(VL[I]); in isFixedVectorShuffle()
475 static InstructionsState getSameOpcode(ArrayRef<Value *> VL,
493 static InstructionsState getSameOpcode(ArrayRef<Value *> VL, in getSameOpcode() argument
496 if (llvm::any_of(VL, [](Value *V) { return !isa<Instruction>(V); })) in getSameOpcode()
497 return InstructionsState(VL[BaseIndex], nullptr, nullptr); in getSameOpcode()
499 bool IsCastOp = isa<CastInst>(VL[BaseIndex]); in getSameOpcode()
500 bool IsBinOp = isa<BinaryOperator>(VL[BaseIndex]); in getSameOpcode()
501 bool IsCmpOp = isa<CmpInst>(VL[BaseIndex]); in getSameOpcode()
503 IsCmpOp ? cast<CmpInst>(VL[BaseIndex])->getPredicate() in getSameOpcode()
505 unsigned Opcode = cast<Instruction>(VL[BaseIndex])->getOpcode(); in getSameOpcode()
511 for (int Cnt = 0, E = VL.size(); Cnt < E; Cnt++) { in getSameOpcode()
512 unsigned InstOpcode = cast<Instruction>(VL[Cnt])->getOpcode(); in getSameOpcode()
513 if (IsBinOp && isa<BinaryOperator>(VL[Cnt])) { in getSameOpcode()
522 } else if (IsCastOp && isa<CastInst>(VL[Cnt])) { in getSameOpcode()
523 Type *Ty0 = cast<Instruction>(VL[BaseIndex])->getOperand(0)->getType(); in getSameOpcode()
524 Type *Ty1 = cast<Instruction>(VL[Cnt])->getOperand(0)->getType(); in getSameOpcode()
537 } else if (IsCmpOp && isa<CmpInst>(VL[Cnt])) { in getSameOpcode()
538 auto *BaseInst = cast<Instruction>(VL[BaseIndex]); in getSameOpcode()
539 auto *Inst = cast<Instruction>(VL[Cnt]); in getSameOpcode()
548 cast<CmpInst>(VL[Cnt])->getPredicate(); in getSameOpcode()
563 auto *AltInst = cast<CmpInst>(VL[AltIndex]); in getSameOpcode()
582 auto *AltInst = cast<CmpInst>(VL[AltIndex]); in getSameOpcode()
590 return InstructionsState(VL[BaseIndex], nullptr, nullptr); in getSameOpcode()
593 return InstructionsState(VL[BaseIndex], cast<Instruction>(VL[BaseIndex]), in getSameOpcode()
594 cast<Instruction>(VL[AltIndex])); in getSameOpcode()
599 static bool allSameType(ArrayRef<Value *> VL) { in allSameType() argument
600 Type *Ty = VL[0]->getType(); in allSameType()
601 for (int i = 1, e = VL.size(); i < e; i++) in allSameType()
602 if (VL[i]->getType() != Ty) in allSameType()
831 static bool doesNotNeedToSchedule(ArrayRef<Value *> VL) { in doesNotNeedToSchedule() argument
832 return !VL.empty() && in doesNotNeedToSchedule()
833 (all_of(VL, isUsedOutsideBlock) || all_of(VL, areAllOperandsNonInsts)); in doesNotNeedToSchedule()
1734 void appendOperandsOfVL(ArrayRef<Value *> VL) { in appendOperandsOfVL() argument
1735 assert(!VL.empty() && "Bad VL"); in appendOperandsOfVL()
1736 assert((empty() || VL.size() == getNumLanes()) && in appendOperandsOfVL()
1738 assert(isa<Instruction>(VL[0]) && "Expected instruction"); in appendOperandsOfVL()
1739 unsigned NumOperands = cast<Instruction>(VL[0])->getNumOperands(); in appendOperandsOfVL()
1741 unsigned NumLanes = VL.size(); in appendOperandsOfVL()
1745 assert(isa<Instruction>(VL[Lane]) && "Expected instruction"); in appendOperandsOfVL()
1756 bool IsInverseOperation = !isCommutative(cast<Instruction>(VL[Lane])); in appendOperandsOfVL()
1758 OpsVec[OpIdx][Lane] = {cast<Instruction>(VL[Lane])->getOperand(OpIdx), in appendOperandsOfVL()
2066 bool areAnalyzedReductionVals(ArrayRef<Value *> VL) { in areAnalyzedReductionVals() argument
2067 return AnalyzedReductionVals.contains(hash_value(VL)); in areAnalyzedReductionVals()
2071 void analyzedReductionVals(ArrayRef<Value *> VL) { in analyzedReductionVals() argument
2072 AnalyzedReductionVals.insert(hash_value(VL)); in analyzedReductionVals()
2103 ArrayRef<Value *> VL = UserTE->getOperand(OpIdx); in getVectorizedOperand() local
2105 const auto *It = find_if(VL, [this, &TE](Value *V) { in getVectorizedOperand()
2109 if (It != VL.end() && TE->isSame(VL)) in getVectorizedOperand()
2139 bool canReuseExtract(ArrayRef<Value *> VL, Value *OpValue,
2146 Value *vectorizeTree(ArrayRef<Value *> VL);
2151 Value *createBuildVector(ArrayRef<Value *> VL);
2172 InstructionCost getGatherCost(ArrayRef<Value *> VL) const;
2179 Value *gather(ArrayRef<Value *> VL);
2187 static void reorderInputsAccordingToOpcode(ArrayRef<Value *> VL,
2221 bool isSame(ArrayRef<Value *> VL) const { in isSame()
2222 auto &&IsSame = [VL](ArrayRef<Value *> Scalars, ArrayRef<int> Mask) { in isSame()
2223 if (Mask.size() != VL.size() && VL.size() == Scalars.size()) in isSame()
2224 return std::equal(VL.begin(), VL.end(), Scalars.begin()); in isSame()
2225 return VL.size() == Mask.size() && in isSame()
2226 std::equal(VL.begin(), VL.end(), Mask.begin(), in isSame()
2239 if (VL.size() == Scalars.size()) in isSame()
2241 if (VL.size() == ReuseShuffleIndices.size()) { in isSame()
2511 TreeEntry *newTreeEntry(ArrayRef<Value *> VL, Optional<ScheduleData *> Bundle, in newTreeEntry() argument
2518 return newTreeEntry(VL, EntryState, Bundle, S, UserTreeIdx, in newTreeEntry()
2522 TreeEntry *newTreeEntry(ArrayRef<Value *> VL, in newTreeEntry() argument
2539 Last->Scalars.assign(VL.begin(), VL.end()); in newTreeEntry()
2543 Last->Scalars.assign(VL.size(), nullptr); in newTreeEntry()
2545 [VL](unsigned Idx) -> Value * { in newTreeEntry()
2546 if (Idx >= VL.size()) in newTreeEntry()
2547 return UndefValue::get(VL.front()->getType()); in newTreeEntry()
2548 return VL[Idx]; in newTreeEntry()
2555 for (Value *V : VL) { in newTreeEntry()
2563 doesNotNeedToSchedule(VL)) && in newTreeEntry()
2566 for (Value *V : VL) { in newTreeEntry()
2576 MustGather.insert(VL.begin(), VL.end()); in newTreeEntry()
3100 ScheduleData *buildBundle(ArrayRef<Value *> VL);
3108 tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
3112 void cancelScheduling(ArrayRef<Value *> VL, Value *OpValue);
3483 static LoadsState canVectorizeLoads(ArrayRef<Value *> VL, const Value *VL0, in canVectorizeLoads() argument
3503 PointerOps.resize(VL.size()); in canVectorizeLoads()
3505 for (Value *V : VL) { in canVectorizeLoads()
3542 if (static_cast<unsigned>(*Diff) == VL.size() - 1) in canVectorizeLoads()
3552 })) <= VL.size() / 2 && VL.size() > 2; in canVectorizeLoads()
3559 for (Value *V : VL) in canVectorizeLoads()
3562 auto *VecTy = FixedVectorType::get(ScalarTy, VL.size()); in canVectorizeLoads()
3572 bool clusterSortPtrAccesses(ArrayRef<Value *> VL, Type *ElemTy, in clusterSortPtrAccesses() argument
3576 VL, [](const Value *V) { return V->getType()->isPointerTy(); }) && in clusterSortPtrAccesses()
3582 Bases[VL[0]].push_back(std::make_tuple(VL[0], 0U, 0U)); in clusterSortPtrAccesses()
3585 for (Value *Ptr : VL.drop_front()) { in clusterSortPtrAccesses()
3599 if (Bases.size() > VL.size() / 2 - 1) in clusterSortPtrAccesses()
3634 assert(SortedIndices.size() == VL.size() && in clusterSortPtrAccesses()
4448 static bool needToScheduleSingleInstruction(ArrayRef<Value *> VL) { in needToScheduleSingleInstruction() argument
4450 for (Value *V : VL) { in needToScheduleSingleInstruction()
4550 void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth, in buildTree_rec() argument
4552 assert((allConstant(VL) || allSameType(VL)) && "Invalid types!"); in buildTree_rec()
4556 auto &&TryToFindDuplicates = [&VL, &ReuseShuffleIndicies, &UniqueValues, in buildTree_rec()
4561 for (Value *V : VL) { in buildTree_rec()
4574 if (NumUniqueScalarValues == VL.size()) { in buildTree_rec()
4586 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx); in buildTree_rec()
4589 VL = UniqueValues; in buildTree_rec()
4594 InstructionsState S = getSameOpcode(VL); in buildTree_rec()
4601 VL.size() >= 4 && in buildTree_rec()
4602 (match(S.MainOp, m_Load(m_Value())) || all_of(VL, [&S](const Value *I) { in buildTree_rec()
4610 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, in buildTree_rec()
4621 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, in buildTree_rec()
4630 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx); in buildTree_rec()
4637 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx); in buildTree_rec()
4647 Depth](ArrayRef<Value *> VL) { in buildTree_rec() argument
4648 if (!S.getOpcode() || !S.isAltShuffle() || VL.size() > 2) in buildTree_rec()
4656 SmallVector<unsigned, 2> InstsCount(VL.size(), 0); in buildTree_rec()
4657 for (Value *V : VL) { in buildTree_rec()
4669 assert(VL.size() == 2 && "Expected only 2 alternate op instructions."); in buildTree_rec()
4671 auto *I1 = cast<Instruction>(VL.front()); in buildTree_rec()
4672 auto *I2 = cast<Instruction>(VL.back()); in buildTree_rec()
4703 (S.getOpcode() && allSameBlock(VL)) || in buildTree_rec()
4705 VL.size() > 2 && in buildTree_rec()
4706 all_of(VL, in buildTree_rec()
4716 sortPtrAccesses(VL, UserTreeIdx.UserTE->getMainOp()->getType(), *DL, *SE, in buildTree_rec()
4718 if (allConstant(VL) || isSplat(VL) || !AreAllSameInsts || in buildTree_rec()
4721 !all_of(VL, isVectorLikeInstWithConstOps)) || in buildTree_rec()
4722 NotProfitableForVectorization(VL)) { in buildTree_rec()
4725 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, in buildTree_rec()
4735 for (Value *V : VL) { in buildTree_rec()
4739 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx); in buildTree_rec()
4748 if (!E->isSame(VL)) { in buildTree_rec()
4751 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, in buildTree_rec()
4764 for (Value *V : VL) { in buildTree_rec()
4771 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, in buildTree_rec()
4779 for (Value *V : VL) { in buildTree_rec()
4783 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, in buildTree_rec()
4792 if (AreAllSameInsts && !(S.getOpcode() && allSameBlock(VL)) && in buildTree_rec()
4796 count_if(VL, [](Value *V) { return isa<GetElementPtrInst>(V); }) >= in buildTree_rec()
4800 const auto *It = find_if(VL, [](Value *V) { return isa<GetElementPtrInst>(V); }); in buildTree_rec()
4801 assert(It != VL.end() && "Expected at least one GEP."); in buildTree_rec()
4814 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx); in buildTree_rec()
4823 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx); in buildTree_rec()
4837 Optional<ScheduleData *> Bundle = BS.tryScheduleBundle(VL, this, S); in buildTree_rec()
4847 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, in buildTree_rec()
4860 for (Value *V : VL) in buildTree_rec()
4866 BS.cancelScheduling(VL, VL0); in buildTree_rec()
4867 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, in buildTree_rec()
4874 newTreeEntry(VL, Bundle, S, UserTreeIdx, ReuseShuffleIndicies); in buildTree_rec()
4881 ValueList Operands(VL.size(), PoisonValue::get(PH->getType())); in buildTree_rec()
4888 for (Value *V : VL) in buildTree_rec()
4901 bool Reuse = canReuseExtract(VL, VL0, CurrentOrder); in buildTree_rec()
4904 newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, in buildTree_rec()
4909 Op0.assign(VL.size(), VL0->getOperand(0)); in buildTree_rec()
4924 newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, in buildTree_rec()
4929 Op0.assign(VL.size(), VL0->getOperand(0)); in buildTree_rec()
4934 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, in buildTree_rec()
4936 BS.cancelScheduling(VL, VL0); in buildTree_rec()
4945 for (Value *V : VL) { in buildTree_rec()
4950 if (count_if(VL, [&SourceVectors](Value *V) { in buildTree_rec()
4956 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx); in buildTree_rec()
4957 BS.cancelScheduling(VL, VL0); in buildTree_rec()
4968 for (int I = 0, E = VL.size(); I < E; ++I) { in buildTree_rec()
4969 unsigned Idx = *getInsertIndex(VL[I]); in buildTree_rec()
4972 OrdersType CurrentOrder(VL.size(), VL.size()); in buildTree_rec()
4974 for (int I = 0, E = VL.size(); I < E; ++I) { in buildTree_rec()
4981 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, in buildTree_rec()
4988 for (Value *V : VL) in buildTree_rec()
5006 switch (canVectorizeLoads(VL, VL0, *TTI, *DL, *SE, *LI, CurrentOrder, in buildTree_rec()
5011 TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, in buildTree_rec()
5017 TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, in buildTree_rec()
5025 TE = newTreeEntry(VL, TreeEntry::ScatterVectorize, Bundle, S, in buildTree_rec()
5032 BS.cancelScheduling(VL, VL0); in buildTree_rec()
5033 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, in buildTree_rec()
5040 else if (any_of(VL, [](Value *V) { in buildTree_rec()
5064 for (Value *V : VL) { in buildTree_rec()
5067 BS.cancelScheduling(VL, VL0); in buildTree_rec()
5068 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, in buildTree_rec()
5075 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, in buildTree_rec()
5083 for (Value *V : VL) in buildTree_rec()
5096 for (Value *V : VL) { in buildTree_rec()
5100 BS.cancelScheduling(VL, VL0); in buildTree_rec()
5101 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, in buildTree_rec()
5109 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, in buildTree_rec()
5118 reorderInputsAccordingToOpcode(VL, Left, Right, *DL, *SE, *this); in buildTree_rec()
5121 for (Value *V : VL) { in buildTree_rec()
5157 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, in buildTree_rec()
5165 reorderInputsAccordingToOpcode(VL, Left, Right, *DL, *SE, *this); in buildTree_rec()
5177 for (Value *V : VL) in buildTree_rec()
5186 for (Value *V : VL) { in buildTree_rec()
5192 BS.cancelScheduling(VL, VL0); in buildTree_rec()
5193 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, in buildTree_rec()
5202 for (Value *V : VL) { in buildTree_rec()
5210 BS.cancelScheduling(VL, VL0); in buildTree_rec()
5211 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, in buildTree_rec()
5219 for (Value *V : VL) { in buildTree_rec()
5232 BS.cancelScheduling(VL, VL0); in buildTree_rec()
5233 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, in buildTree_rec()
5239 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, in buildTree_rec()
5244 for (Value *V : VL) { in buildTree_rec()
5260 Type *Ty = all_of(VL, in buildTree_rec()
5272 for (Value *V : VL) { in buildTree_rec()
5300 BS.cancelScheduling(VL, VL0); in buildTree_rec()
5301 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, in buildTree_rec()
5308 SmallVector<Value *, 4> PointerOps(VL.size()); in buildTree_rec()
5309 ValueList Operands(VL.size()); in buildTree_rec()
5312 for (Value *V : VL) { in buildTree_rec()
5315 BS.cancelScheduling(VL, VL0); in buildTree_rec()
5316 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, in buildTree_rec()
5342 if (static_cast<unsigned>(*Dist) == VL.size() - 1) { in buildTree_rec()
5345 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, in buildTree_rec()
5353 newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, in buildTree_rec()
5363 BS.cancelScheduling(VL, VL0); in buildTree_rec()
5364 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, in buildTree_rec()
5376 *CI, ElementCount::getFixed(static_cast<unsigned int>(VL.size())), in buildTree_rec()
5381 BS.cancelScheduling(VL, VL0); in buildTree_rec()
5382 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, in buildTree_rec()
5393 for (Value *V : VL) { in buildTree_rec()
5400 BS.cancelScheduling(VL, VL0); in buildTree_rec()
5401 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, in buildTree_rec()
5413 BS.cancelScheduling(VL, VL0); in buildTree_rec()
5414 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, in buildTree_rec()
5428 BS.cancelScheduling(VL, VL0); in buildTree_rec()
5429 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, in buildTree_rec()
5437 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, in buildTree_rec()
5447 for (Value *V : VL) { in buildTree_rec()
5459 BS.cancelScheduling(VL, VL0); in buildTree_rec()
5460 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, in buildTree_rec()
5465 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, in buildTree_rec()
5473 if (!CI || all_of(VL, [](Value *V) { in buildTree_rec()
5476 reorderInputsAccordingToOpcode(VL, Left, Right, *DL, *SE, *this); in buildTree_rec()
5487 for (Value *V : VL) { in buildTree_rec()
5517 for (Value *V : VL) in buildTree_rec()
5525 BS.cancelScheduling(VL, VL0); in buildTree_rec()
5526 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, in buildTree_rec()
5564 bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL, Value *OpValue, in canReuseExtract() argument
5566 const auto *It = find_if(VL, [](Value *V) { in canReuseExtract()
5569 assert(It != VL.end() && "Expected at least one extract instruction."); in canReuseExtract()
5571 assert(all_of(VL, in canReuseExtract()
5592 if (!LI || !LI->isSimple() || !LI->hasNUses(VL.size())) in canReuseExtract()
5598 if (NElts != VL.size()) in canReuseExtract()
5603 unsigned E = VL.size(); in canReuseExtract()
5612 auto *Inst = dyn_cast<Instruction>(VL[I]); in canReuseExtract()
5691 computeExtractCost(ArrayRef<Value *> VL, FixedVectorType *VecTy, in computeExtractCost() argument
5709 for (auto *V : VL) { in computeExtractCost()
5726 if (!isa<UndefValue>(VL[Idx - 1]) && Mask[Idx - 1] != UndefMaskElem) { in computeExtractCost()
5727 unsigned PrevIdx = *getExtractIndex(cast<Instruction>(VL[Idx - 1])); in computeExtractCost()
5737 if ((Idx + 1) % EltsPerVector != 0 && Idx + 1 != VL.size()) in computeExtractCost()
5753 buildShuffleEntryMask(ArrayRef<Value *> VL, ArrayRef<unsigned> ReorderIndices, in buildShuffleEntryMask() argument
5759 unsigned Sz = VL.size(); in buildShuffleEntryMask()
5768 auto *OpInst = cast<Instruction>(VL[Idx]); in buildShuffleEntryMask()
5813 ArrayRef<Value*> VL = E->Scalars; in getEntryCost() local
5815 Type *ScalarTy = VL[0]->getType(); in getEntryCost()
5816 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0])) in getEntryCost()
5818 else if (CmpInst *CI = dyn_cast<CmpInst>(VL[0])) in getEntryCost()
5820 else if (auto *IE = dyn_cast<InsertElementInst>(VL[0])) in getEntryCost()
5822 auto *VecTy = FixedVectorType::get(ScalarTy, VL.size()); in getEntryCost()
5827 if (MinBWs.count(VL[0])) in getEntryCost()
5829 IntegerType::get(F->getContext(), MinBWs[VL[0]].first), VL.size()); in getEntryCost()
5836 auto &&AdjustExtractsCost = [this, &TTIRef, CostKind, VL, VecTy, in getEntryCost()
5840 for (auto *V : VL) { in getEntryCost()
5916 if (allConstant(VL)) in getEntryCost()
5918 if (isa<InsertElementInst>(VL[0])) in getEntryCost()
5932 << *VL.front() << ".\n"); in getEntryCost()
5940 << *VL.front() << ".\n"); in getEntryCost()
5954 allSameType(VL)) { in getEntryCost()
5959 isFixedVectorShuffle(VL, Mask); in getEntryCost()
5965 computeExtractCost(VL, VecTy, *ShuffleKind, Mask, *TTI); in getEntryCost()
5973 if (isSplat(VL)) { in getEntryCost()
5980 /*SubTp=*/nullptr, /*Args=*/VL[0]); in getEntryCost()
5988 if (VL.size() > 2 && E->getOpcode() == Instruction::Load && in getEntryCost()
5992 unsigned VF = VL.size() / 2; in getEntryCost()
5997 for (unsigned Cnt = StartIdx, End = VL.size(); Cnt + VF <= End; in getEntryCost()
5999 ArrayRef<Value *> Slice = VL.slice(Cnt, VF); in getEntryCost()
6028 if (StartIdx >= VL.size()) in getEntryCost()
6038 !NumParts || (VL.size() / VF) > NumParts; in getEntryCost()
6040 for (unsigned I = 0, End = VL.size(); I < End; I += VF) { in getEntryCost()
6041 if (VectorizedLoads.contains(VL[I])) in getEntryCost()
6043 GatherCost += getGatherCost(VL.slice(I, VF)); in getEntryCost()
6066 for (int I = VF, E = VL.size(); I < E; I += VF) in getEntryCost()
6073 return ReuseShuffleCost + getGatherCost(VL); in getEntryCost()
6097 ((allSameType(VL) && allSameBlock(VL)) || in getEntryCost()
6116 auto *EE = cast<ExtractElementInst>(VL[I]); in getEntryCost()
6127 for (Value *V : VL) { in getEntryCost()
6141 for (unsigned I = 0, E = VL.size(); I < E; ++I) { in getEntryCost()
6142 auto *EI = cast<Instruction>(VL[I]); in getEntryCost()
6173 unsigned const NumScalars = VL.size(); in getEntryCost()
6177 unsigned OffsetBeg = *getInsertIndex(VL.front()); in getEntryCost()
6179 for (Value *V : VL.drop_front()) { in getEntryCost()
6222 unsigned InsertIdx = *getInsertIndex(VL[PrevMask[I]]); in getEntryCost()
6287 CommonCost -= (EntryVF - VL.size()) * ScalarEltCost; in getEntryCost()
6291 InstructionCost ScalarCost = VL.size() * ScalarEltCost; in getEntryCost()
6293 auto *SrcVecTy = FixedVectorType::get(SrcTy, VL.size()); in getEntryCost()
6312 CommonCost -= (EntryVF - VL.size()) * ScalarEltCost; in getEntryCost()
6314 auto *MaskTy = FixedVectorType::get(Builder.getInt1Ty(), VL.size()); in getEntryCost()
6321 for (auto *V : VL) { in getEntryCost()
6339 auto IntrinsicAndUse = canConvertToMinOrMaxIntrinsic(VL); in getEntryCost()
6391 for (unsigned i = 0, e = VL.size(); i < e; ++i) { in getEntryCost()
6392 const Instruction *I = cast<Instruction>(VL[i]); in getEntryCost()
6416 CommonCost -= (EntryVF - VL.size()) * ScalarEltCost; in getEntryCost()
6429 any_of(VL, in getEntryCost()
6441 CommonCost -= (EntryVF - VL.size()) * ScalarEltCost; in getEntryCost()
6455 CommonCost -= (EntryVF - VL.size()) * ScalarEltCost; in getEntryCost()
6465 for (Value *V : VL) in getEntryCost()
6479 cast<StoreInst>(IsReorder ? VL[E->ReorderIndices.front()] : VL0); in getEntryCost()
6498 CommonCost -= (EntryVF - VL.size()) * ScalarEltCost; in getEntryCost()
6523 Instruction *I = cast<Instruction>(VL[Idx]); in getEntryCost()
6526 for (Value *V : VL) { in getEntryCost()
6531 for (Value *V : VL) { in getEntryCost()
6577 auto *Src0Ty = FixedVectorType::get(Src0SclTy, VL.size()); in getEntryCost()
6578 auto *Src1Ty = FixedVectorType::get(Src1SclTy, VL.size()); in getEntryCost()
7396 InstructionCost BoUpSLP::getGatherCost(ArrayRef<Value *> VL) const { in getGatherCost()
7398 Type *ScalarTy = VL[0]->getType(); in getGatherCost()
7399 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0])) in getGatherCost()
7401 auto *VecTy = FixedVectorType::get(ScalarTy, VL.size()); in getGatherCost()
7406 APInt ShuffledElements = APInt::getZero(VL.size()); in getGatherCost()
7409 for (unsigned I = VL.size(); I > 0; --I) { in getGatherCost()
7412 if (isConstant(VL[Idx])) { in getGatherCost()
7416 if (!UniqueElements.insert(VL[Idx]).second) { in getGatherCost()
7426 void BoUpSLP::reorderInputsAccordingToOpcode(ArrayRef<Value *> VL, in reorderInputsAccordingToOpcode() argument
7432 if (VL.empty()) in reorderInputsAccordingToOpcode()
7434 VLOperands Ops(VL, DL, SE, R); in reorderInputsAccordingToOpcode()
7570 Value *BoUpSLP::gather(ArrayRef<Value *> VL) { in gather() argument
7584 for (int I = 0, E = VL.size(); I < E; ++I) { in gather()
7585 if (auto *Inst = dyn_cast<Instruction>(VL[I])) in gather()
7608 isa<StoreInst>(VL[0]) ? cast<StoreInst>(VL[0])->getValueOperand() : VL[0]; in gather()
7609 FixedVectorType *VecTy = FixedVectorType::get(Val0->getType(), VL.size()); in gather()
7613 for (int I = 0, E = VL.size(); I < E; ++I) { in gather()
7616 if (!isConstant(VL[I])) { in gather()
7620 Vec = CreateInsertElement(Vec, VL[I], I); in gather()
7624 Vec = CreateInsertElement(Vec, VL[I], I); in gather()
7695 Value *BoUpSLP::vectorizeTree(ArrayRef<Value *> VL) { in vectorizeTree() argument
7696 const unsigned VF = VL.size(); in vectorizeTree()
7697 InstructionsState S = getSameOpcode(VL); in vectorizeTree()
7699 if (!S.getOpcode() && VL.front()->getType()->isPointerTy()) { in vectorizeTree()
7701 find_if(VL, [](Value *V) { return isa<GetElementPtrInst>(V); }); in vectorizeTree()
7702 if (It != VL.end()) in vectorizeTree()
7707 if (E->isSame(VL)) { in vectorizeTree()
7731 int Sz = VL.size(); in vectorizeTree()
7761 return createBuildVector(VL); in vectorizeTree()
7763 Value *BoUpSLP::createBuildVector(ArrayRef<Value *> VL) { in createBuildVector() argument
7765 [VL](const std::unique_ptr<TreeEntry> &TE) { in createBuildVector()
7766 return TE->State == TreeEntry::NeedToGather && TE->isSame(VL); in createBuildVector()
7769 unsigned VF = VL.size(); in createBuildVector()
7773 if (VL.size() > 2) { in createBuildVector()
7776 std::distance(VL.begin(), find_if(reverse(VL), [](Value *V) { in createBuildVector()
7781 for (Value *V : VL.drop_back(VL.size() - VF)) { in createBuildVector()
7804 assert(all_of(VL, UndefValue::classof) && "Expected list of undefs."); in createBuildVector()
7809 UniqueValues.append(VL.begin(), std::next(VL.begin(), NumValues)); in createBuildVector()
7812 PoisonValue::get(VL[0]->getType())); in createBuildVector()
7813 VL = UniqueValues; in createBuildVector()
7818 Value *Vec = gather(VL); in createBuildVector()
9050 BoUpSLP::BlockScheduling::buildBundle(ArrayRef<Value *> VL) { in buildBundle() argument
9053 for (Value *V : VL) { in buildBundle()
9079 BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP, in tryScheduleBundle() argument
9084 doesNotNeedToSchedule(VL)) in tryScheduleBundle()
9129 for (Value *V : VL) { in tryScheduleBundle()
9145 for (Value *V : VL) { in tryScheduleBundle()
9166 auto *Bundle = buildBundle(VL); in tryScheduleBundle()
9169 cancelScheduling(VL, S.OpValue); in tryScheduleBundle()
9175 void BoUpSLP::BlockScheduling::cancelScheduling(ArrayRef<Value *> VL, in cancelScheduling() argument
9178 doesNotNeedToSchedule(VL)) in cancelScheduling()
9182 OpValue = *find_if_not(VL, doesNotNeedToBeScheduled); in cancelScheduling()
9188 (Bundle->isPartOfBundle() || needToScheduleSingleInstruction(VL)) && in cancelScheduling()
10284 Value *VL[] = {A, B}; in tryToVectorizePair() local
10285 return tryToVectorizeList(VL, R); in tryToVectorizePair()
10288 bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R, in tryToVectorizeList() argument
10290 if (VL.size() < 2) in tryToVectorizeList()
10294 << VL.size() << ".\n"); in tryToVectorizeList()
10298 InstructionsState S = getSameOpcode(VL); in tryToVectorizeList()
10305 for (Value *V : VL) { in tryToVectorizeList()
10324 unsigned MaxVF = std::max<unsigned>(PowerOf2Floor(VL.size()), MinVF); in tryToVectorizeList()
10338 Type *ScalarTy = VL[0]->getType(); in tryToVectorizeList()
10339 if (auto *IE = dyn_cast<InsertElementInst>(VL[0])) in tryToVectorizeList()
10342 unsigned NextInst = 0, MaxInst = VL.size(); in tryToVectorizeList()
10365 ArrayRef<Value *> Ops = VL.slice(I, OpsWidth); in tryToVectorizeList()
11139 ArrayRef<Value *> VL(std::next(Candidates.begin(), Pos), ReduxWidth); in tryToReduce() local
11141 if (V.areAnalyzedReductionVals(VL)) { in tryToReduce()
11147 if (any_of(VL, [&V](Value *RedVal) { in tryToReduce()
11154 V.buildTree(VL, IgnoreList); in tryToReduce()
11157 V.analyzedReductionVals(VL); in tryToReduce()
11162 V.analyzedReductionVals(VL); in tryToReduce()
11187 NumUses[V] = std::count(VL.begin(), VL.end(), V); in tryToReduce()
11193 NumUses[V] = std::count(VL.begin(), VL.end(), V); in tryToReduce()
11224 InstructionCost TreeCost = V.getTreeCost(VL); in tryToReduce()
11226 getReductionCost(TTI, VL, ReduxWidth, RdxFMF); in tryToReduce()
11236 ReducedValsToOps.find(VL[0])->second.front()) in tryToReduce()
11243 V.analyzedReductionVals(VL); in tryToReduce()
11252 ReducedValsToOps.find(VL[0])->second.front()) in tryToReduce()
11291 for (Value *V : VL) in tryToReduce()