1 //===----- CodeGen/ExpandVectorPredication.cpp - Expand VP intrinsics -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass implements IR expansion for vector predication intrinsics, allowing
10 // targets to enable vector predication until just before codegen.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/CodeGen/ExpandVectorPredication.h"
15 #include "llvm/ADT/Statistic.h"
16 #include "llvm/Analysis/TargetTransformInfo.h"
17 #include "llvm/Analysis/ValueTracking.h"
18 #include "llvm/CodeGen/Passes.h"
19 #include "llvm/IR/Constants.h"
20 #include "llvm/IR/Function.h"
21 #include "llvm/IR/IRBuilder.h"
22 #include "llvm/IR/InstIterator.h"
23 #include "llvm/IR/Instructions.h"
24 #include "llvm/IR/IntrinsicInst.h"
25 #include "llvm/IR/Intrinsics.h"
26 #include "llvm/InitializePasses.h"
27 #include "llvm/Pass.h"
28 #include "llvm/Support/CommandLine.h"
29 #include "llvm/Support/Compiler.h"
30 #include "llvm/Support/Debug.h"
31 
32 using namespace llvm;
33 
34 using VPLegalization = TargetTransformInfo::VPLegalization;
35 using VPTransform = TargetTransformInfo::VPLegalization::VPTransform;
36 
37 // Keep this in sync with TargetTransformInfo::VPLegalization.
38 #define VPINTERNAL_VPLEGAL_CASES                                               \
39   VPINTERNAL_CASE(Legal)                                                       \
40   VPINTERNAL_CASE(Discard)                                                     \
41   VPINTERNAL_CASE(Convert)
42 
43 #define VPINTERNAL_CASE(X) "|" #X
44 
45 // Override options.
46 static cl::opt<std::string> EVLTransformOverride(
47     "expandvp-override-evl-transform", cl::init(""), cl::Hidden,
48     cl::desc("Options: <empty>" VPINTERNAL_VPLEGAL_CASES
49              ". If non-empty, ignore "
50              "TargetTransformInfo and "
51              "always use this transformation for the %evl parameter (Used in "
52              "testing)."));
53 
54 static cl::opt<std::string> MaskTransformOverride(
55     "expandvp-override-mask-transform", cl::init(""), cl::Hidden,
56     cl::desc("Options: <empty>" VPINTERNAL_VPLEGAL_CASES
57              ". If non-empty, Ignore "
58              "TargetTransformInfo and "
59              "always use this transformation for the %mask parameter (Used in "
60              "testing)."));
61 
62 #undef VPINTERNAL_CASE
63 #define VPINTERNAL_CASE(X) .Case(#X, VPLegalization::X)
64 
65 static VPTransform parseOverrideOption(const std::string &TextOpt) {
66   return StringSwitch<VPTransform>(TextOpt) VPINTERNAL_VPLEGAL_CASES;
67 }
68 
69 #undef VPINTERNAL_VPLEGAL_CASES
70 
71 // Whether any override options are set.
72 static bool anyExpandVPOverridesSet() {
73   return !EVLTransformOverride.empty() || !MaskTransformOverride.empty();
74 }
75 
76 #define DEBUG_TYPE "expandvp"
77 
78 STATISTIC(NumFoldedVL, "Number of folded vector length params");
79 STATISTIC(NumLoweredVPOps, "Number of folded vector predication operations");
80 
81 ///// Helpers {
82 
83 /// \returns Whether the vector mask \p MaskVal has all lane bits set.
84 static bool isAllTrueMask(Value *MaskVal) {
85   auto *ConstVec = dyn_cast<ConstantVector>(MaskVal);
86   return ConstVec && ConstVec->isAllOnesValue();
87 }
88 
89 /// \returns A non-excepting divisor constant for this type.
90 static Constant *getSafeDivisor(Type *DivTy) {
91   assert(DivTy->isIntOrIntVectorTy() && "Unsupported divisor type");
92   return ConstantInt::get(DivTy, 1u, false);
93 }
94 
95 /// Transfer operation properties from \p OldVPI to \p NewVal.
96 static void transferDecorations(Value &NewVal, VPIntrinsic &VPI) {
97   auto *NewInst = dyn_cast<Instruction>(&NewVal);
98   if (!NewInst || !isa<FPMathOperator>(NewVal))
99     return;
100 
101   auto *OldFMOp = dyn_cast<FPMathOperator>(&VPI);
102   if (!OldFMOp)
103     return;
104 
105   NewInst->setFastMathFlags(OldFMOp->getFastMathFlags());
106 }
107 
108 /// Transfer all properties from \p OldOp to \p NewOp and replace all uses.
109 /// OldVP gets erased.
110 static void replaceOperation(Value &NewOp, VPIntrinsic &OldOp) {
111   transferDecorations(NewOp, OldOp);
112   OldOp.replaceAllUsesWith(&NewOp);
113   OldOp.eraseFromParent();
114 }
115 
116 static bool maySpeculateLanes(VPIntrinsic &VPI) {
117   // The result of VP reductions depends on the mask and evl.
118   if (isa<VPReductionIntrinsic>(VPI))
119     return false;
120   // Fallback to whether the intrinsic is speculatable.
121   Optional<unsigned> OpcOpt = VPI.getFunctionalOpcode();
122   unsigned FunctionalOpc = OpcOpt.value_or((unsigned)Instruction::Call);
123   return isSafeToSpeculativelyExecuteWithOpcode(FunctionalOpc, &VPI);
124 }
125 
126 //// } Helpers
127 
128 namespace {
129 
130 // Expansion pass state at function scope.
131 struct CachingVPExpander {
132   Function &F;
133   const TargetTransformInfo &TTI;
134 
135   /// \returns A (fixed length) vector with ascending integer indices
136   /// (<0, 1, ..., NumElems-1>).
137   /// \p Builder
138   ///    Used for instruction creation.
139   /// \p LaneTy
140   ///    Integer element type of the result vector.
141   /// \p NumElems
142   ///    Number of vector elements.
143   Value *createStepVector(IRBuilder<> &Builder, Type *LaneTy,
144                           unsigned NumElems);
145 
146   /// \returns A bitmask that is true where the lane position is less-than \p
147   /// EVLParam
148   ///
149   /// \p Builder
150   ///    Used for instruction creation.
151   /// \p VLParam
152   ///    The explicit vector length parameter to test against the lane
153   ///    positions.
154   /// \p ElemCount
155   ///    Static (potentially scalable) number of vector elements.
156   Value *convertEVLToMask(IRBuilder<> &Builder, Value *EVLParam,
157                           ElementCount ElemCount);
158 
159   Value *foldEVLIntoMask(VPIntrinsic &VPI);
160 
161   /// "Remove" the %evl parameter of \p PI by setting it to the static vector
162   /// length of the operation.
163   void discardEVLParameter(VPIntrinsic &PI);
164 
165   /// \brief Lower this VP binary operator to a unpredicated binary operator.
166   Value *expandPredicationInBinaryOperator(IRBuilder<> &Builder,
167                                            VPIntrinsic &PI);
168 
169   /// \brief Lower this VP reduction to a call to an unpredicated reduction
170   /// intrinsic.
171   Value *expandPredicationInReduction(IRBuilder<> &Builder,
172                                       VPReductionIntrinsic &PI);
173 
174   /// \brief Query TTI and expand the vector predication in \p P accordingly.
175   Value *expandPredication(VPIntrinsic &PI);
176 
177   /// \brief  Determine how and whether the VPIntrinsic \p VPI shall be
178   /// expanded. This overrides TTI with the cl::opts listed at the top of this
179   /// file.
180   VPLegalization getVPLegalizationStrategy(const VPIntrinsic &VPI) const;
181   bool UsingTTIOverrides;
182 
183 public:
184   CachingVPExpander(Function &F, const TargetTransformInfo &TTI)
185       : F(F), TTI(TTI), UsingTTIOverrides(anyExpandVPOverridesSet()) {}
186 
187   bool expandVectorPredication();
188 };
189 
190 //// CachingVPExpander {
191 
192 Value *CachingVPExpander::createStepVector(IRBuilder<> &Builder, Type *LaneTy,
193                                            unsigned NumElems) {
194   // TODO add caching
195   SmallVector<Constant *, 16> ConstElems;
196 
197   for (unsigned Idx = 0; Idx < NumElems; ++Idx)
198     ConstElems.push_back(ConstantInt::get(LaneTy, Idx, false));
199 
200   return ConstantVector::get(ConstElems);
201 }
202 
203 Value *CachingVPExpander::convertEVLToMask(IRBuilder<> &Builder,
204                                            Value *EVLParam,
205                                            ElementCount ElemCount) {
206   // TODO add caching
207   // Scalable vector %evl conversion.
208   if (ElemCount.isScalable()) {
209     auto *M = Builder.GetInsertBlock()->getModule();
210     Type *BoolVecTy = VectorType::get(Builder.getInt1Ty(), ElemCount);
211     Function *ActiveMaskFunc = Intrinsic::getDeclaration(
212         M, Intrinsic::get_active_lane_mask, {BoolVecTy, EVLParam->getType()});
213     // `get_active_lane_mask` performs an implicit less-than comparison.
214     Value *ConstZero = Builder.getInt32(0);
215     return Builder.CreateCall(ActiveMaskFunc, {ConstZero, EVLParam});
216   }
217 
218   // Fixed vector %evl conversion.
219   Type *LaneTy = EVLParam->getType();
220   unsigned NumElems = ElemCount.getFixedValue();
221   Value *VLSplat = Builder.CreateVectorSplat(NumElems, EVLParam);
222   Value *IdxVec = createStepVector(Builder, LaneTy, NumElems);
223   return Builder.CreateICmp(CmpInst::ICMP_ULT, IdxVec, VLSplat);
224 }
225 
226 Value *
227 CachingVPExpander::expandPredicationInBinaryOperator(IRBuilder<> &Builder,
228                                                      VPIntrinsic &VPI) {
229   assert((maySpeculateLanes(VPI) || VPI.canIgnoreVectorLengthParam()) &&
230          "Implicitly dropping %evl in non-speculatable operator!");
231 
232   auto OC = static_cast<Instruction::BinaryOps>(*VPI.getFunctionalOpcode());
233   assert(Instruction::isBinaryOp(OC));
234 
235   Value *Op0 = VPI.getOperand(0);
236   Value *Op1 = VPI.getOperand(1);
237   Value *Mask = VPI.getMaskParam();
238 
239   // Blend in safe operands.
240   if (Mask && !isAllTrueMask(Mask)) {
241     switch (OC) {
242     default:
243       // Can safely ignore the predicate.
244       break;
245 
246     // Division operators need a safe divisor on masked-off lanes (1).
247     case Instruction::UDiv:
248     case Instruction::SDiv:
249     case Instruction::URem:
250     case Instruction::SRem:
251       // 2nd operand must not be zero.
252       Value *SafeDivisor = getSafeDivisor(VPI.getType());
253       Op1 = Builder.CreateSelect(Mask, Op1, SafeDivisor);
254     }
255   }
256 
257   Value *NewBinOp = Builder.CreateBinOp(OC, Op0, Op1, VPI.getName());
258 
259   replaceOperation(*NewBinOp, VPI);
260   return NewBinOp;
261 }
262 
263 static Value *getNeutralReductionElement(const VPReductionIntrinsic &VPI,
264                                          Type *EltTy) {
265   bool Negative = false;
266   unsigned EltBits = EltTy->getScalarSizeInBits();
267   switch (VPI.getIntrinsicID()) {
268   default:
269     llvm_unreachable("Expecting a VP reduction intrinsic");
270   case Intrinsic::vp_reduce_add:
271   case Intrinsic::vp_reduce_or:
272   case Intrinsic::vp_reduce_xor:
273   case Intrinsic::vp_reduce_umax:
274     return Constant::getNullValue(EltTy);
275   case Intrinsic::vp_reduce_mul:
276     return ConstantInt::get(EltTy, 1, /*IsSigned*/ false);
277   case Intrinsic::vp_reduce_and:
278   case Intrinsic::vp_reduce_umin:
279     return ConstantInt::getAllOnesValue(EltTy);
280   case Intrinsic::vp_reduce_smin:
281     return ConstantInt::get(EltTy->getContext(),
282                             APInt::getSignedMaxValue(EltBits));
283   case Intrinsic::vp_reduce_smax:
284     return ConstantInt::get(EltTy->getContext(),
285                             APInt::getSignedMinValue(EltBits));
286   case Intrinsic::vp_reduce_fmax:
287     Negative = true;
288     LLVM_FALLTHROUGH;
289   case Intrinsic::vp_reduce_fmin: {
290     FastMathFlags Flags = VPI.getFastMathFlags();
291     const fltSemantics &Semantics = EltTy->getFltSemantics();
292     return !Flags.noNaNs() ? ConstantFP::getQNaN(EltTy, Negative)
293            : !Flags.noInfs()
294                ? ConstantFP::getInfinity(EltTy, Negative)
295                : ConstantFP::get(EltTy,
296                                  APFloat::getLargest(Semantics, Negative));
297   }
298   case Intrinsic::vp_reduce_fadd:
299     return ConstantFP::getNegativeZero(EltTy);
300   case Intrinsic::vp_reduce_fmul:
301     return ConstantFP::get(EltTy, 1.0);
302   }
303 }
304 
305 Value *
306 CachingVPExpander::expandPredicationInReduction(IRBuilder<> &Builder,
307                                                 VPReductionIntrinsic &VPI) {
308   assert((maySpeculateLanes(VPI) || VPI.canIgnoreVectorLengthParam()) &&
309          "Implicitly dropping %evl in non-speculatable operator!");
310 
311   Value *Mask = VPI.getMaskParam();
312   Value *RedOp = VPI.getOperand(VPI.getVectorParamPos());
313 
314   // Insert neutral element in masked-out positions
315   if (Mask && !isAllTrueMask(Mask)) {
316     auto *NeutralElt = getNeutralReductionElement(VPI, VPI.getType());
317     auto *NeutralVector = Builder.CreateVectorSplat(
318         cast<VectorType>(RedOp->getType())->getElementCount(), NeutralElt);
319     RedOp = Builder.CreateSelect(Mask, RedOp, NeutralVector);
320   }
321 
322   Value *Reduction;
323   Value *Start = VPI.getOperand(VPI.getStartParamPos());
324 
325   switch (VPI.getIntrinsicID()) {
326   default:
327     llvm_unreachable("Impossible reduction kind");
328   case Intrinsic::vp_reduce_add:
329     Reduction = Builder.CreateAddReduce(RedOp);
330     Reduction = Builder.CreateAdd(Reduction, Start);
331     break;
332   case Intrinsic::vp_reduce_mul:
333     Reduction = Builder.CreateMulReduce(RedOp);
334     Reduction = Builder.CreateMul(Reduction, Start);
335     break;
336   case Intrinsic::vp_reduce_and:
337     Reduction = Builder.CreateAndReduce(RedOp);
338     Reduction = Builder.CreateAnd(Reduction, Start);
339     break;
340   case Intrinsic::vp_reduce_or:
341     Reduction = Builder.CreateOrReduce(RedOp);
342     Reduction = Builder.CreateOr(Reduction, Start);
343     break;
344   case Intrinsic::vp_reduce_xor:
345     Reduction = Builder.CreateXorReduce(RedOp);
346     Reduction = Builder.CreateXor(Reduction, Start);
347     break;
348   case Intrinsic::vp_reduce_smax:
349     Reduction = Builder.CreateIntMaxReduce(RedOp, /*IsSigned*/ true);
350     Reduction =
351         Builder.CreateBinaryIntrinsic(Intrinsic::smax, Reduction, Start);
352     break;
353   case Intrinsic::vp_reduce_smin:
354     Reduction = Builder.CreateIntMinReduce(RedOp, /*IsSigned*/ true);
355     Reduction =
356         Builder.CreateBinaryIntrinsic(Intrinsic::smin, Reduction, Start);
357     break;
358   case Intrinsic::vp_reduce_umax:
359     Reduction = Builder.CreateIntMaxReduce(RedOp, /*IsSigned*/ false);
360     Reduction =
361         Builder.CreateBinaryIntrinsic(Intrinsic::umax, Reduction, Start);
362     break;
363   case Intrinsic::vp_reduce_umin:
364     Reduction = Builder.CreateIntMinReduce(RedOp, /*IsSigned*/ false);
365     Reduction =
366         Builder.CreateBinaryIntrinsic(Intrinsic::umin, Reduction, Start);
367     break;
368   case Intrinsic::vp_reduce_fmax:
369     Reduction = Builder.CreateFPMaxReduce(RedOp);
370     transferDecorations(*Reduction, VPI);
371     Reduction =
372         Builder.CreateBinaryIntrinsic(Intrinsic::maxnum, Reduction, Start);
373     break;
374   case Intrinsic::vp_reduce_fmin:
375     Reduction = Builder.CreateFPMinReduce(RedOp);
376     transferDecorations(*Reduction, VPI);
377     Reduction =
378         Builder.CreateBinaryIntrinsic(Intrinsic::minnum, Reduction, Start);
379     break;
380   case Intrinsic::vp_reduce_fadd:
381     Reduction = Builder.CreateFAddReduce(Start, RedOp);
382     break;
383   case Intrinsic::vp_reduce_fmul:
384     Reduction = Builder.CreateFMulReduce(Start, RedOp);
385     break;
386   }
387 
388   replaceOperation(*Reduction, VPI);
389   return Reduction;
390 }
391 
392 void CachingVPExpander::discardEVLParameter(VPIntrinsic &VPI) {
393   LLVM_DEBUG(dbgs() << "Discard EVL parameter in " << VPI << "\n");
394 
395   if (VPI.canIgnoreVectorLengthParam())
396     return;
397 
398   Value *EVLParam = VPI.getVectorLengthParam();
399   if (!EVLParam)
400     return;
401 
402   ElementCount StaticElemCount = VPI.getStaticVectorLength();
403   Value *MaxEVL = nullptr;
404   Type *Int32Ty = Type::getInt32Ty(VPI.getContext());
405   if (StaticElemCount.isScalable()) {
406     // TODO add caching
407     auto *M = VPI.getModule();
408     Function *VScaleFunc =
409         Intrinsic::getDeclaration(M, Intrinsic::vscale, Int32Ty);
410     IRBuilder<> Builder(VPI.getParent(), VPI.getIterator());
411     Value *FactorConst = Builder.getInt32(StaticElemCount.getKnownMinValue());
412     Value *VScale = Builder.CreateCall(VScaleFunc, {}, "vscale");
413     MaxEVL = Builder.CreateMul(VScale, FactorConst, "scalable_size",
414                                /*NUW*/ true, /*NSW*/ false);
415   } else {
416     MaxEVL = ConstantInt::get(Int32Ty, StaticElemCount.getFixedValue(), false);
417   }
418   VPI.setVectorLengthParam(MaxEVL);
419 }
420 
421 Value *CachingVPExpander::foldEVLIntoMask(VPIntrinsic &VPI) {
422   LLVM_DEBUG(dbgs() << "Folding vlen for " << VPI << '\n');
423 
424   IRBuilder<> Builder(&VPI);
425 
426   // Ineffective %evl parameter and so nothing to do here.
427   if (VPI.canIgnoreVectorLengthParam())
428     return &VPI;
429 
430   // Only VP intrinsics can have an %evl parameter.
431   Value *OldMaskParam = VPI.getMaskParam();
432   Value *OldEVLParam = VPI.getVectorLengthParam();
433   assert(OldMaskParam && "no mask param to fold the vl param into");
434   assert(OldEVLParam && "no EVL param to fold away");
435 
436   LLVM_DEBUG(dbgs() << "OLD evl: " << *OldEVLParam << '\n');
437   LLVM_DEBUG(dbgs() << "OLD mask: " << *OldMaskParam << '\n');
438 
439   // Convert the %evl predication into vector mask predication.
440   ElementCount ElemCount = VPI.getStaticVectorLength();
441   Value *VLMask = convertEVLToMask(Builder, OldEVLParam, ElemCount);
442   Value *NewMaskParam = Builder.CreateAnd(VLMask, OldMaskParam);
443   VPI.setMaskParam(NewMaskParam);
444 
445   // Drop the %evl parameter.
446   discardEVLParameter(VPI);
447   assert(VPI.canIgnoreVectorLengthParam() &&
448          "transformation did not render the evl param ineffective!");
449 
450   // Reassess the modified instruction.
451   return &VPI;
452 }
453 
454 Value *CachingVPExpander::expandPredication(VPIntrinsic &VPI) {
455   LLVM_DEBUG(dbgs() << "Lowering to unpredicated op: " << VPI << '\n');
456 
457   IRBuilder<> Builder(&VPI);
458 
459   // Try lowering to a LLVM instruction first.
460   auto OC = VPI.getFunctionalOpcode();
461 
462   if (OC && Instruction::isBinaryOp(*OC))
463     return expandPredicationInBinaryOperator(Builder, VPI);
464 
465   if (auto *VPRI = dyn_cast<VPReductionIntrinsic>(&VPI))
466     return expandPredicationInReduction(Builder, *VPRI);
467 
468   return &VPI;
469 }
470 
471 //// } CachingVPExpander
472 
473 struct TransformJob {
474   VPIntrinsic *PI;
475   TargetTransformInfo::VPLegalization Strategy;
476   TransformJob(VPIntrinsic *PI, TargetTransformInfo::VPLegalization InitStrat)
477       : PI(PI), Strategy(InitStrat) {}
478 
479   bool isDone() const { return Strategy.shouldDoNothing(); }
480 };
481 
482 void sanitizeStrategy(VPIntrinsic &VPI, VPLegalization &LegalizeStrat) {
483   // Operations with speculatable lanes do not strictly need predication.
484   if (maySpeculateLanes(VPI)) {
485     // Converting a speculatable VP intrinsic means dropping %mask and %evl.
486     // No need to expand %evl into the %mask only to ignore that code.
487     if (LegalizeStrat.OpStrategy == VPLegalization::Convert)
488       LegalizeStrat.EVLParamStrategy = VPLegalization::Discard;
489     return;
490   }
491 
492   // We have to preserve the predicating effect of %evl for this
493   // non-speculatable VP intrinsic.
494   // 1) Never discard %evl.
495   // 2) If this VP intrinsic will be expanded to non-VP code, make sure that
496   //    %evl gets folded into %mask.
497   if ((LegalizeStrat.EVLParamStrategy == VPLegalization::Discard) ||
498       (LegalizeStrat.OpStrategy == VPLegalization::Convert)) {
499     LegalizeStrat.EVLParamStrategy = VPLegalization::Convert;
500   }
501 }
502 
503 VPLegalization
504 CachingVPExpander::getVPLegalizationStrategy(const VPIntrinsic &VPI) const {
505   auto VPStrat = TTI.getVPLegalizationStrategy(VPI);
506   if (LLVM_LIKELY(!UsingTTIOverrides)) {
507     // No overrides - we are in production.
508     return VPStrat;
509   }
510 
511   // Overrides set - we are in testing, the following does not need to be
512   // efficient.
513   VPStrat.EVLParamStrategy = parseOverrideOption(EVLTransformOverride);
514   VPStrat.OpStrategy = parseOverrideOption(MaskTransformOverride);
515   return VPStrat;
516 }
517 
518 /// \brief Expand llvm.vp.* intrinsics as requested by \p TTI.
519 bool CachingVPExpander::expandVectorPredication() {
520   SmallVector<TransformJob, 16> Worklist;
521 
522   // Collect all VPIntrinsics that need expansion and determine their expansion
523   // strategy.
524   for (auto &I : instructions(F)) {
525     auto *VPI = dyn_cast<VPIntrinsic>(&I);
526     if (!VPI)
527       continue;
528     auto VPStrat = getVPLegalizationStrategy(*VPI);
529     sanitizeStrategy(*VPI, VPStrat);
530     if (!VPStrat.shouldDoNothing())
531       Worklist.emplace_back(VPI, VPStrat);
532   }
533   if (Worklist.empty())
534     return false;
535 
536   // Transform all VPIntrinsics on the worklist.
537   LLVM_DEBUG(dbgs() << "\n:::: Transforming " << Worklist.size()
538                     << " instructions ::::\n");
539   for (TransformJob Job : Worklist) {
540     // Transform the EVL parameter.
541     switch (Job.Strategy.EVLParamStrategy) {
542     case VPLegalization::Legal:
543       break;
544     case VPLegalization::Discard:
545       discardEVLParameter(*Job.PI);
546       break;
547     case VPLegalization::Convert:
548       if (foldEVLIntoMask(*Job.PI))
549         ++NumFoldedVL;
550       break;
551     }
552     Job.Strategy.EVLParamStrategy = VPLegalization::Legal;
553 
554     // Replace with a non-predicated operation.
555     switch (Job.Strategy.OpStrategy) {
556     case VPLegalization::Legal:
557       break;
558     case VPLegalization::Discard:
559       llvm_unreachable("Invalid strategy for operators.");
560     case VPLegalization::Convert:
561       expandPredication(*Job.PI);
562       ++NumLoweredVPOps;
563       break;
564     }
565     Job.Strategy.OpStrategy = VPLegalization::Legal;
566 
567     assert(Job.isDone() && "incomplete transformation");
568   }
569 
570   return true;
571 }
572 class ExpandVectorPredication : public FunctionPass {
573 public:
574   static char ID;
575   ExpandVectorPredication() : FunctionPass(ID) {
576     initializeExpandVectorPredicationPass(*PassRegistry::getPassRegistry());
577   }
578 
579   bool runOnFunction(Function &F) override {
580     const auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
581     CachingVPExpander VPExpander(F, *TTI);
582     return VPExpander.expandVectorPredication();
583   }
584 
585   void getAnalysisUsage(AnalysisUsage &AU) const override {
586     AU.addRequired<TargetTransformInfoWrapperPass>();
587     AU.setPreservesCFG();
588   }
589 };
590 } // namespace
591 
592 char ExpandVectorPredication::ID;
593 INITIALIZE_PASS_BEGIN(ExpandVectorPredication, "expandvp",
594                       "Expand vector predication intrinsics", false, false)
595 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
596 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
597 INITIALIZE_PASS_END(ExpandVectorPredication, "expandvp",
598                     "Expand vector predication intrinsics", false, false)
599 
600 FunctionPass *llvm::createExpandVectorPredicationPass() {
601   return new ExpandVectorPredication();
602 }
603 
604 PreservedAnalyses
605 ExpandVectorPredicationPass::run(Function &F, FunctionAnalysisManager &AM) {
606   const auto &TTI = AM.getResult<TargetIRAnalysis>(F);
607   CachingVPExpander VPExpander(F, TTI);
608   if (!VPExpander.expandVectorPredication())
609     return PreservedAnalyses::all();
610   PreservedAnalyses PA;
611   PA.preserveSet<CFGAnalyses>();
612   return PA;
613 }
614