1 //===-- VPlanTransforms.cpp - Utility VPlan to VPlan transforms -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// This file implements a set of utility VPlan to VPlan transformations.
11 ///
12 //===----------------------------------------------------------------------===//
13 
14 #include "VPlanTransforms.h"
15 #include "llvm/ADT/PostOrderIterator.h"
16 
17 using namespace llvm;
18 
19 void VPlanTransforms::VPInstructionsToVPRecipes(
20     Loop *OrigLoop, VPlanPtr &Plan,
21     LoopVectorizationLegality::InductionList &Inductions,
22     SmallPtrSetImpl<Instruction *> &DeadInstructions, ScalarEvolution &SE) {
23 
24   auto *TopRegion = cast<VPRegionBlock>(Plan->getEntry());
25   ReversePostOrderTraversal<VPBlockBase *> RPOT(TopRegion->getEntry());
26 
27   for (VPBlockBase *Base : RPOT) {
28     // Do not widen instructions in pre-header and exit blocks.
29     if (Base->getNumPredecessors() == 0 || Base->getNumSuccessors() == 0)
30       continue;
31 
32     VPBasicBlock *VPBB = Base->getEntryBasicBlock();
33     // Introduce each ingredient into VPlan.
34     for (auto I = VPBB->begin(), E = VPBB->end(); I != E;) {
35       VPRecipeBase *Ingredient = &*I++;
36       VPValue *VPV = Ingredient->getVPSingleValue();
37       Instruction *Inst = cast<Instruction>(VPV->getUnderlyingValue());
38       if (DeadInstructions.count(Inst)) {
39         VPValue DummyValue;
40         VPV->replaceAllUsesWith(&DummyValue);
41         Ingredient->eraseFromParent();
42         continue;
43       }
44 
45       VPRecipeBase *NewRecipe = nullptr;
46       if (auto *VPPhi = dyn_cast<VPWidenPHIRecipe>(Ingredient)) {
47         auto *Phi = cast<PHINode>(VPPhi->getUnderlyingValue());
48         InductionDescriptor II = Inductions.lookup(Phi);
49         if (II.getKind() == InductionDescriptor::IK_IntInduction ||
50             II.getKind() == InductionDescriptor::IK_FpInduction) {
51           VPValue *Start = Plan->getOrAddVPValue(II.getStartValue());
52           NewRecipe = new VPWidenIntOrFpInductionRecipe(Phi, Start, nullptr);
53         } else {
54           Plan->addVPValue(Phi, VPPhi);
55           continue;
56         }
57       } else {
58         assert(isa<VPInstruction>(Ingredient) &&
59                "only VPInstructions expected here");
60         assert(!isa<PHINode>(Inst) && "phis should be handled above");
61         // Create VPWidenMemoryInstructionRecipe for loads and stores.
62         if (LoadInst *Load = dyn_cast<LoadInst>(Inst)) {
63           NewRecipe = new VPWidenMemoryInstructionRecipe(
64               *Load, Plan->getOrAddVPValue(getLoadStorePointerOperand(Inst)),
65               nullptr /*Mask*/);
66         } else if (StoreInst *Store = dyn_cast<StoreInst>(Inst)) {
67           NewRecipe = new VPWidenMemoryInstructionRecipe(
68               *Store, Plan->getOrAddVPValue(getLoadStorePointerOperand(Inst)),
69               Plan->getOrAddVPValue(Store->getValueOperand()),
70               nullptr /*Mask*/);
71         } else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Inst)) {
72           NewRecipe = new VPWidenGEPRecipe(
73               GEP, Plan->mapToVPValues(GEP->operands()), OrigLoop);
74         } else if (CallInst *CI = dyn_cast<CallInst>(Inst)) {
75           NewRecipe = new VPWidenCallRecipe(
76               *CI, Plan->mapToVPValues(CI->arg_operands()));
77         } else if (SelectInst *SI = dyn_cast<SelectInst>(Inst)) {
78           bool InvariantCond =
79               SE.isLoopInvariant(SE.getSCEV(SI->getOperand(0)), OrigLoop);
80           NewRecipe = new VPWidenSelectRecipe(
81               *SI, Plan->mapToVPValues(SI->operands()), InvariantCond);
82         } else {
83           NewRecipe =
84               new VPWidenRecipe(*Inst, Plan->mapToVPValues(Inst->operands()));
85         }
86       }
87 
88       NewRecipe->insertBefore(Ingredient);
89       if (NewRecipe->getNumDefinedValues() == 1)
90         VPV->replaceAllUsesWith(NewRecipe->getVPSingleValue());
91       else
92         assert(NewRecipe->getNumDefinedValues() == 0 &&
93                "Only recpies with zero or one defined values expected");
94       Ingredient->eraseFromParent();
95       Plan->removeVPValueFor(Inst);
96       for (auto *Def : NewRecipe->definedValues()) {
97         Plan->addVPValue(Inst, Def);
98       }
99     }
100   }
101 }
102 
103 bool VPlanTransforms::sinkScalarOperands(VPlan &Plan) {
104   auto Iter = depth_first(
105       VPBlockRecursiveTraversalWrapper<VPBlockBase *>(Plan.getEntry()));
106   bool Changed = false;
107   // First, collect the operands of all predicated replicate recipes as seeds
108   // for sinking.
109   SetVector<std::pair<VPBasicBlock *, VPValue *>> WorkList;
110   for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) {
111     for (auto &Recipe : *VPBB) {
112       auto *RepR = dyn_cast<VPReplicateRecipe>(&Recipe);
113       if (!RepR || !RepR->isPredicated())
114         continue;
115       for (VPValue *Op : RepR->operands())
116         WorkList.insert(std::make_pair(RepR->getParent(), Op));
117     }
118   }
119 
120   // Try to sink each replicate recipe in the worklist.
121   while (!WorkList.empty()) {
122     VPBasicBlock *SinkTo;
123     VPValue *C;
124     std::tie(SinkTo, C) = WorkList.pop_back_val();
125     auto *SinkCandidate = dyn_cast_or_null<VPReplicateRecipe>(C->Def);
126     if (!SinkCandidate || SinkCandidate->isUniform() ||
127         SinkCandidate->getParent() == SinkTo ||
128         SinkCandidate->mayHaveSideEffects() ||
129         SinkCandidate->mayReadOrWriteMemory())
130       continue;
131 
132     // All recipe users of the sink candidate must be in the same block SinkTo.
133     if (any_of(SinkCandidate->users(), [SinkTo](VPUser *U) {
134           auto *UI = dyn_cast<VPRecipeBase>(U);
135           return !UI || UI->getParent() != SinkTo;
136         }))
137       continue;
138 
139     SinkCandidate->moveBefore(*SinkTo, SinkTo->getFirstNonPhi());
140     for (VPValue *Op : SinkCandidate->operands())
141       WorkList.insert(std::make_pair(SinkTo, Op));
142     Changed = true;
143   }
144   return Changed;
145 }
146 
147 /// If \p R is a region with a VPBranchOnMaskRecipe in the entry block, return
148 /// the mask.
149 VPValue *getPredicatedMask(VPRegionBlock *R) {
150   auto *EntryBB = dyn_cast<VPBasicBlock>(R->getEntry());
151   if (!EntryBB || EntryBB->size() != 1 ||
152       !isa<VPBranchOnMaskRecipe>(EntryBB->begin()))
153     return nullptr;
154 
155   return cast<VPBranchOnMaskRecipe>(&*EntryBB->begin())->getOperand(0);
156 }
157 
158 /// If \p R is a triangle region, return the 'then' block of the triangle.
159 static VPBasicBlock *getPredicatedThenBlock(VPRegionBlock *R) {
160   auto *EntryBB = cast<VPBasicBlock>(R->getEntry());
161   if (EntryBB->getNumSuccessors() != 2)
162     return nullptr;
163 
164   auto *Succ0 = dyn_cast<VPBasicBlock>(EntryBB->getSuccessors()[0]);
165   auto *Succ1 = dyn_cast<VPBasicBlock>(EntryBB->getSuccessors()[1]);
166   if (!Succ0 || !Succ1)
167     return nullptr;
168 
169   if (Succ0->getNumSuccessors() + Succ1->getNumSuccessors() != 1)
170     return nullptr;
171   if (Succ0->getSingleSuccessor() == Succ1)
172     return Succ0;
173   if (Succ1->getSingleSuccessor() == Succ0)
174     return Succ1;
175   return nullptr;
176 }
177 
178 bool VPlanTransforms::mergeReplicateRegions(VPlan &Plan) {
179   SetVector<VPRegionBlock *> DeletedRegions;
180   bool Changed = false;
181 
182   // Collect region blocks to process up-front, to avoid iterator invalidation
183   // issues while merging regions.
184   SmallVector<VPRegionBlock *, 8> CandidateRegions(
185       VPBlockUtils::blocksOnly<VPRegionBlock>(depth_first(
186           VPBlockRecursiveTraversalWrapper<VPBlockBase *>(Plan.getEntry()))));
187 
188   // Check if Base is a predicated triangle, followed by an empty block,
189   // followed by another predicate triangle. If that's the case, move the
190   // recipes from the first to the second triangle.
191   for (VPRegionBlock *Region1 : CandidateRegions) {
192     if (DeletedRegions.contains(Region1))
193       continue;
194     auto *MiddleBasicBlock =
195         dyn_cast_or_null<VPBasicBlock>(Region1->getSingleSuccessor());
196     if (!MiddleBasicBlock || !MiddleBasicBlock->empty())
197       continue;
198 
199     auto *Region2 =
200         dyn_cast_or_null<VPRegionBlock>(MiddleBasicBlock->getSingleSuccessor());
201     if (!Region2)
202       continue;
203 
204     VPValue *Mask1 = getPredicatedMask(Region1);
205     VPValue *Mask2 = getPredicatedMask(Region2);
206     if (!Mask1 || Mask1 != Mask2)
207       continue;
208     VPBasicBlock *Then1 = getPredicatedThenBlock(Region1);
209     VPBasicBlock *Then2 = getPredicatedThenBlock(Region2);
210     if (!Then1 || !Then2)
211       continue;
212 
213     assert(Mask1 && Mask2 && "both region must have conditions");
214 
215     // Note: No fusion-preventing memory dependencies are expected in either
216     // region. Such dependencies should be rejected during earlier dependence
217     // checks, which guarantee accesses can be re-ordered for vectorization.
218     //
219     // Move recipes to the successor region.
220     for (VPRecipeBase &ToMove : make_early_inc_range(reverse(*Then1)))
221       ToMove.moveBefore(*Then2, Then2->getFirstNonPhi());
222 
223     auto *Merge1 = cast<VPBasicBlock>(Then1->getSingleSuccessor());
224     auto *Merge2 = cast<VPBasicBlock>(Then2->getSingleSuccessor());
225 
226     // Move VPPredInstPHIRecipes from the merge block to the successor region's
227     // merge block. Update all users inside the successor region to use the
228     // original values.
229     for (VPRecipeBase &Phi1ToMove : make_early_inc_range(reverse(*Merge1))) {
230       VPValue *PredInst1 =
231           cast<VPPredInstPHIRecipe>(&Phi1ToMove)->getOperand(0);
232       VPValue *Phi1ToMoveV = Phi1ToMove.getVPSingleValue();
233       SmallVector<VPUser *> Users(Phi1ToMoveV->user_begin(),
234                                   Phi1ToMoveV->user_end());
235       for (VPUser *U : Users) {
236         auto *UI = dyn_cast<VPRecipeBase>(U);
237         if (!UI || UI->getParent() != Then2)
238           continue;
239         for (unsigned I = 0, E = U->getNumOperands(); I != E; ++I) {
240           if (Phi1ToMoveV != U->getOperand(I))
241             continue;
242           U->setOperand(I, PredInst1);
243         }
244       }
245 
246       Phi1ToMove.moveBefore(*Merge2, Merge2->begin());
247     }
248 
249     // Finally, remove the first region.
250     for (VPBlockBase *Pred : make_early_inc_range(Region1->getPredecessors())) {
251       VPBlockUtils::disconnectBlocks(Pred, Region1);
252       VPBlockUtils::connectBlocks(Pred, MiddleBasicBlock);
253     }
254     VPBlockUtils::disconnectBlocks(Region1, MiddleBasicBlock);
255     DeletedRegions.insert(Region1);
256   }
257 
258   for (VPRegionBlock *ToDelete : DeletedRegions)
259     delete ToDelete;
260   return Changed;
261 }
262