1 //===- MergedLoadStoreMotion.cpp - merge and hoist/sink load/stores -------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 //! \file
11 //! \brief This pass performs merges of loads and stores on both sides of a
12 //  diamond (hammock). It hoists the loads and sinks the stores.
13 //
14 // The algorithm iteratively hoists two loads to the same address out of a
15 // diamond (hammock) and merges them into a single load in the header. Similar
16 // it sinks and merges two stores to the tail block (footer). The algorithm
17 // iterates over the instructions of one side of the diamond and attempts to
18 // find a matching load/store on the other side. It hoists / sinks when it
19 // thinks it safe to do so.  This optimization helps with eg. hiding load
20 // latencies, triggering if-conversion, and reducing static code size.
21 //
22 //===----------------------------------------------------------------------===//
23 //
24 //
25 // Example:
26 // Diamond shaped code before merge:
27 //
28 //            header:
29 //                     br %cond, label %if.then, label %if.else
30 //                        +                    +
31 //                       +                      +
32 //                      +                        +
33 //            if.then:                         if.else:
34 //               %lt = load %addr_l               %le = load %addr_l
35 //               <use %lt>                        <use %le>
36 //               <...>                            <...>
37 //               store %st, %addr_s               store %se, %addr_s
38 //               br label %if.end                 br label %if.end
39 //                     +                         +
40 //                      +                       +
41 //                       +                     +
42 //            if.end ("footer"):
43 //                     <...>
44 //
45 // Diamond shaped code after merge:
46 //
47 //            header:
48 //                     %l = load %addr_l
49 //                     br %cond, label %if.then, label %if.else
50 //                        +                    +
51 //                       +                      +
52 //                      +                        +
53 //            if.then:                         if.else:
54 //               <use %l>                         <use %l>
55 //               <...>                            <...>
56 //               br label %if.end                 br label %if.end
57 //                      +                        +
58 //                       +                      +
59 //                        +                    +
60 //            if.end ("footer"):
61 //                     %s.sink = phi [%st, if.then], [%se, if.else]
62 //                     <...>
63 //                     store %s.sink, %addr_s
64 //                     <...>
65 //
66 //
67 //===----------------------- TODO -----------------------------------------===//
68 //
69 // 1) Generalize to regions other than diamonds
70 // 2) Be more aggressive merging memory operations
71 // Note that both changes require register pressure control
72 //
73 //===----------------------------------------------------------------------===//
74 
75 #include "llvm/Transforms/Scalar.h"
76 #include "llvm/ADT/SetVector.h"
77 #include "llvm/ADT/SmallPtrSet.h"
78 #include "llvm/ADT/Statistic.h"
79 #include "llvm/Analysis/AliasAnalysis.h"
80 #include "llvm/Analysis/CFG.h"
81 #include "llvm/Analysis/GlobalsModRef.h"
82 #include "llvm/Analysis/Loads.h"
83 #include "llvm/Analysis/MemoryBuiltins.h"
84 #include "llvm/Analysis/MemoryDependenceAnalysis.h"
85 #include "llvm/Analysis/TargetLibraryInfo.h"
86 #include "llvm/IR/Metadata.h"
87 #include "llvm/IR/PatternMatch.h"
88 #include "llvm/Support/Allocator.h"
89 #include "llvm/Support/CommandLine.h"
90 #include "llvm/Support/Debug.h"
91 #include "llvm/Support/raw_ostream.h"
92 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
93 #include "llvm/Transforms/Utils/SSAUpdater.h"
94 #include <vector>
95 
96 using namespace llvm;
97 
98 #define DEBUG_TYPE "mldst-motion"
99 
100 //===----------------------------------------------------------------------===//
101 //                         MergedLoadStoreMotion Pass
102 //===----------------------------------------------------------------------===//
103 
104 namespace {
105 class MergedLoadStoreMotion : public FunctionPass {
106   AliasAnalysis *AA;
107   MemoryDependenceAnalysis *MD;
108 
109 public:
110   static char ID; // Pass identification, replacement for typeid
111   explicit MergedLoadStoreMotion(void)
112       : FunctionPass(ID), MD(nullptr), MagicCompileTimeControl(250) {
113     initializeMergedLoadStoreMotionPass(*PassRegistry::getPassRegistry());
114   }
115 
116   bool runOnFunction(Function &F) override;
117 
118 private:
119   // This transformation requires dominator postdominator info
120   void getAnalysisUsage(AnalysisUsage &AU) const override {
121     AU.addRequired<TargetLibraryInfoWrapperPass>();
122     AU.addRequired<AAResultsWrapperPass>();
123     AU.addPreserved<GlobalsAAWrapperPass>();
124     AU.addPreserved<MemoryDependenceAnalysis>();
125   }
126 
127   // Helper routines
128 
129   ///
130   /// \brief Remove instruction from parent and update memory dependence
131   /// analysis.
132   ///
133   void removeInstruction(Instruction *Inst);
134   BasicBlock *getDiamondTail(BasicBlock *BB);
135   bool isDiamondHead(BasicBlock *BB);
136   // Routines for hoisting loads
137   bool isLoadHoistBarrierInRange(const Instruction& Start,
138                                  const Instruction& End,
139                                  LoadInst* LI);
140   LoadInst *canHoistFromBlock(BasicBlock *BB, LoadInst *LI);
141   void hoistInstruction(BasicBlock *BB, Instruction *HoistCand,
142                         Instruction *ElseInst);
143   bool isSafeToHoist(Instruction *I) const;
144   bool hoistLoad(BasicBlock *BB, LoadInst *HoistCand, LoadInst *ElseInst);
145   bool mergeLoads(BasicBlock *BB);
146   // Routines for sinking stores
147   StoreInst *canSinkFromBlock(BasicBlock *BB, StoreInst *SI);
148   PHINode *getPHIOperand(BasicBlock *BB, StoreInst *S0, StoreInst *S1);
149   bool isStoreSinkBarrierInRange(const Instruction &Start,
150                                  const Instruction &End, MemoryLocation Loc);
151   bool sinkStore(BasicBlock *BB, StoreInst *SinkCand, StoreInst *ElseInst);
152   bool mergeStores(BasicBlock *BB);
153   // The mergeLoad/Store algorithms could have Size0 * Size1 complexity,
154   // where Size0 and Size1 are the #instructions on the two sides of
155   // the diamond. The constant chosen here is arbitrary. Compiler Time
156   // Control is enforced by the check Size0 * Size1 < MagicCompileTimeControl.
157   const int MagicCompileTimeControl;
158 };
159 
160 char MergedLoadStoreMotion::ID = 0;
161 }
162 
163 ///
164 /// \brief createMergedLoadStoreMotionPass - The public interface to this file.
165 ///
166 FunctionPass *llvm::createMergedLoadStoreMotionPass() {
167   return new MergedLoadStoreMotion();
168 }
169 
170 INITIALIZE_PASS_BEGIN(MergedLoadStoreMotion, "mldst-motion",
171                       "MergedLoadStoreMotion", false, false)
172 INITIALIZE_PASS_DEPENDENCY(MemoryDependenceAnalysis)
173 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
174 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
175 INITIALIZE_PASS_DEPENDENCY(GlobalsAAWrapperPass)
176 INITIALIZE_PASS_END(MergedLoadStoreMotion, "mldst-motion",
177                     "MergedLoadStoreMotion", false, false)
178 
179 ///
180 /// \brief Remove instruction from parent and update memory dependence analysis.
181 ///
182 void MergedLoadStoreMotion::removeInstruction(Instruction *Inst) {
183   // Notify the memory dependence analysis.
184   if (MD) {
185     MD->removeInstruction(Inst);
186     if (LoadInst *LI = dyn_cast<LoadInst>(Inst))
187       MD->invalidateCachedPointerInfo(LI->getPointerOperand());
188     if (Inst->getType()->getScalarType()->isPointerTy()) {
189       MD->invalidateCachedPointerInfo(Inst);
190     }
191   }
192   Inst->eraseFromParent();
193 }
194 
195 ///
196 /// \brief Return tail block of a diamond.
197 ///
198 BasicBlock *MergedLoadStoreMotion::getDiamondTail(BasicBlock *BB) {
199   assert(isDiamondHead(BB) && "Basic block is not head of a diamond");
200   BranchInst *BI = (BranchInst *)(BB->getTerminator());
201   BasicBlock *Succ0 = BI->getSuccessor(0);
202   BasicBlock *Tail = Succ0->getTerminator()->getSuccessor(0);
203   return Tail;
204 }
205 
206 ///
207 /// \brief True when BB is the head of a diamond (hammock)
208 ///
209 bool MergedLoadStoreMotion::isDiamondHead(BasicBlock *BB) {
210   if (!BB)
211     return false;
212   if (!isa<BranchInst>(BB->getTerminator()))
213     return false;
214   if (BB->getTerminator()->getNumSuccessors() != 2)
215     return false;
216 
217   BranchInst *BI = (BranchInst *)(BB->getTerminator());
218   BasicBlock *Succ0 = BI->getSuccessor(0);
219   BasicBlock *Succ1 = BI->getSuccessor(1);
220 
221   if (!Succ0->getSinglePredecessor() ||
222       Succ0->getTerminator()->getNumSuccessors() != 1)
223     return false;
224   if (!Succ1->getSinglePredecessor() ||
225       Succ1->getTerminator()->getNumSuccessors() != 1)
226     return false;
227 
228   BasicBlock *Tail = Succ0->getTerminator()->getSuccessor(0);
229   // Ignore triangles.
230   if (Succ1->getTerminator()->getSuccessor(0) != Tail)
231     return false;
232   return true;
233 }
234 
235 ///
236 /// \brief True when instruction is a hoist barrier for a load
237 ///
238 /// Whenever an instruction could possibly modify the value
239 /// being loaded or protect against the load from happening
240 /// it is considered a hoist barrier.
241 ///
242 bool MergedLoadStoreMotion::isLoadHoistBarrierInRange(const Instruction& Start,
243                                                       const Instruction& End,
244                                                       LoadInst* LI) {
245   MemoryLocation Loc = MemoryLocation::get(LI);
246   return AA->canInstructionRangeModRef(Start, End, Loc, MRI_Mod);
247 }
248 
249 ///
250 /// \brief Decide if a load can be hoisted
251 ///
252 /// When there is a load in \p BB to the same address as \p LI
253 /// and it can be hoisted from \p BB, return that load.
254 /// Otherwise return Null.
255 ///
256 LoadInst *MergedLoadStoreMotion::canHoistFromBlock(BasicBlock *BB1,
257                                                    LoadInst *Load0) {
258 
259   for (BasicBlock::iterator BBI = BB1->begin(), BBE = BB1->end(); BBI != BBE;
260        ++BBI) {
261     Instruction *Inst = &*BBI;
262 
263     // Only merge and hoist loads when their result in used only in BB
264     if (!isa<LoadInst>(Inst) || Inst->isUsedOutsideOfBlock(BB1))
265       continue;
266 
267     LoadInst *Load1 = dyn_cast<LoadInst>(Inst);
268     BasicBlock *BB0 = Load0->getParent();
269 
270     MemoryLocation Loc0 = MemoryLocation::get(Load0);
271     MemoryLocation Loc1 = MemoryLocation::get(Load1);
272     if (AA->isMustAlias(Loc0, Loc1) && Load0->isSameOperationAs(Load1) &&
273         !isLoadHoistBarrierInRange(BB1->front(), *Load1, Load1) &&
274         !isLoadHoistBarrierInRange(BB0->front(), *Load0, Load0)) {
275       return Load1;
276     }
277   }
278   return nullptr;
279 }
280 
281 ///
282 /// \brief Merge two equivalent instructions \p HoistCand and \p ElseInst into
283 /// \p BB
284 ///
285 /// BB is the head of a diamond
286 ///
287 void MergedLoadStoreMotion::hoistInstruction(BasicBlock *BB,
288                                              Instruction *HoistCand,
289                                              Instruction *ElseInst) {
290   DEBUG(dbgs() << " Hoist Instruction into BB \n"; BB->dump();
291         dbgs() << "Instruction Left\n"; HoistCand->dump(); dbgs() << "\n";
292         dbgs() << "Instruction Right\n"; ElseInst->dump(); dbgs() << "\n");
293   // Hoist the instruction.
294   assert(HoistCand->getParent() != BB);
295 
296   // Intersect optional metadata.
297   HoistCand->intersectOptionalDataWith(ElseInst);
298   HoistCand->dropUnknownNonDebugMetadata();
299 
300   // Prepend point for instruction insert
301   Instruction *HoistPt = BB->getTerminator();
302 
303   // Merged instruction
304   Instruction *HoistedInst = HoistCand->clone();
305 
306   // Hoist instruction.
307   HoistedInst->insertBefore(HoistPt);
308 
309   HoistCand->replaceAllUsesWith(HoistedInst);
310   removeInstruction(HoistCand);
311   // Replace the else block instruction.
312   ElseInst->replaceAllUsesWith(HoistedInst);
313   removeInstruction(ElseInst);
314 }
315 
316 ///
317 /// \brief Return true if no operand of \p I is defined in I's parent block
318 ///
319 bool MergedLoadStoreMotion::isSafeToHoist(Instruction *I) const {
320   BasicBlock *Parent = I->getParent();
321   for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
322     Instruction *Instr = dyn_cast<Instruction>(I->getOperand(i));
323     if (Instr && Instr->getParent() == Parent)
324       return false;
325   }
326   return true;
327 }
328 
329 ///
330 /// \brief Merge two equivalent loads and GEPs and hoist into diamond head
331 ///
332 bool MergedLoadStoreMotion::hoistLoad(BasicBlock *BB, LoadInst *L0,
333                                       LoadInst *L1) {
334   // Only one definition?
335   Instruction *A0 = dyn_cast<Instruction>(L0->getPointerOperand());
336   Instruction *A1 = dyn_cast<Instruction>(L1->getPointerOperand());
337   if (A0 && A1 && A0->isIdenticalTo(A1) && isSafeToHoist(A0) &&
338       A0->hasOneUse() && (A0->getParent() == L0->getParent()) &&
339       A1->hasOneUse() && (A1->getParent() == L1->getParent()) &&
340       isa<GetElementPtrInst>(A0)) {
341     DEBUG(dbgs() << "Hoist Instruction into BB \n"; BB->dump();
342           dbgs() << "Instruction Left\n"; L0->dump(); dbgs() << "\n";
343           dbgs() << "Instruction Right\n"; L1->dump(); dbgs() << "\n");
344     hoistInstruction(BB, A0, A1);
345     hoistInstruction(BB, L0, L1);
346     return true;
347   } else
348     return false;
349 }
350 
351 ///
352 /// \brief Try to hoist two loads to same address into diamond header
353 ///
354 /// Starting from a diamond head block, iterate over the instructions in one
355 /// successor block and try to match a load in the second successor.
356 ///
357 bool MergedLoadStoreMotion::mergeLoads(BasicBlock *BB) {
358   bool MergedLoads = false;
359   assert(isDiamondHead(BB));
360   BranchInst *BI = dyn_cast<BranchInst>(BB->getTerminator());
361   BasicBlock *Succ0 = BI->getSuccessor(0);
362   BasicBlock *Succ1 = BI->getSuccessor(1);
363   // #Instructions in Succ1 for Compile Time Control
364   int Size1 = Succ1->size();
365   int NLoads = 0;
366   for (BasicBlock::iterator BBI = Succ0->begin(), BBE = Succ0->end();
367        BBI != BBE;) {
368     Instruction *I = &*BBI;
369     ++BBI;
370 
371     // Only move non-simple (atomic, volatile) loads.
372     LoadInst *L0 = dyn_cast<LoadInst>(I);
373     if (!L0 || !L0->isSimple() || L0->isUsedOutsideOfBlock(Succ0))
374       continue;
375 
376     ++NLoads;
377     if (NLoads * Size1 >= MagicCompileTimeControl)
378       break;
379     if (LoadInst *L1 = canHoistFromBlock(Succ1, L0)) {
380       bool Res = hoistLoad(BB, L0, L1);
381       MergedLoads |= Res;
382       // Don't attempt to hoist above loads that had not been hoisted.
383       if (!Res)
384         break;
385     }
386   }
387   return MergedLoads;
388 }
389 
390 ///
391 /// \brief True when instruction is a sink barrier for a store
392 /// located in Loc
393 ///
394 /// Whenever an instruction could possibly read or modify the
395 /// value being stored or protect against the store from
396 /// happening it is considered a sink barrier.
397 ///
398 bool MergedLoadStoreMotion::isStoreSinkBarrierInRange(const Instruction &Start,
399                                                       const Instruction &End,
400                                                       MemoryLocation Loc) {
401   return AA->canInstructionRangeModRef(Start, End, Loc, MRI_ModRef);
402 }
403 
404 ///
405 /// \brief Check if \p BB contains a store to the same address as \p SI
406 ///
407 /// \return The store in \p  when it is safe to sink. Otherwise return Null.
408 ///
409 StoreInst *MergedLoadStoreMotion::canSinkFromBlock(BasicBlock *BB1,
410                                                    StoreInst *Store0) {
411   DEBUG(dbgs() << "can Sink? : "; Store0->dump(); dbgs() << "\n");
412   BasicBlock *BB0 = Store0->getParent();
413   for (BasicBlock::reverse_iterator RBI = BB1->rbegin(), RBE = BB1->rend();
414        RBI != RBE; ++RBI) {
415     Instruction *Inst = &*RBI;
416 
417     if (!isa<StoreInst>(Inst))
418        continue;
419 
420     StoreInst *Store1 = cast<StoreInst>(Inst);
421 
422     MemoryLocation Loc0 = MemoryLocation::get(Store0);
423     MemoryLocation Loc1 = MemoryLocation::get(Store1);
424     if (AA->isMustAlias(Loc0, Loc1) && Store0->isSameOperationAs(Store1) &&
425       !isStoreSinkBarrierInRange(*(std::next(BasicBlock::iterator(Store1))),
426                                  BB1->back(), Loc1) &&
427       !isStoreSinkBarrierInRange(*(std::next(BasicBlock::iterator(Store0))),
428                                  BB0->back(), Loc0)) {
429       return Store1;
430     }
431   }
432   return nullptr;
433 }
434 
435 ///
436 /// \brief Create a PHI node in BB for the operands of S0 and S1
437 ///
438 PHINode *MergedLoadStoreMotion::getPHIOperand(BasicBlock *BB, StoreInst *S0,
439                                               StoreInst *S1) {
440   // Create a phi if the values mismatch.
441   PHINode *NewPN = nullptr;
442   Value *Opd1 = S0->getValueOperand();
443   Value *Opd2 = S1->getValueOperand();
444   if (Opd1 != Opd2) {
445     NewPN = PHINode::Create(Opd1->getType(), 2, Opd2->getName() + ".sink",
446                             &BB->front());
447     NewPN->addIncoming(Opd1, S0->getParent());
448     NewPN->addIncoming(Opd2, S1->getParent());
449     if (MD && NewPN->getType()->getScalarType()->isPointerTy())
450       MD->invalidateCachedPointerInfo(NewPN);
451   }
452   return NewPN;
453 }
454 
455 ///
456 /// \brief Merge two stores to same address and sink into \p BB
457 ///
458 /// Also sinks GEP instruction computing the store address
459 ///
460 bool MergedLoadStoreMotion::sinkStore(BasicBlock *BB, StoreInst *S0,
461                                       StoreInst *S1) {
462   // Only one definition?
463   Instruction *A0 = dyn_cast<Instruction>(S0->getPointerOperand());
464   Instruction *A1 = dyn_cast<Instruction>(S1->getPointerOperand());
465   if (A0 && A1 && A0->isIdenticalTo(A1) && A0->hasOneUse() &&
466       (A0->getParent() == S0->getParent()) && A1->hasOneUse() &&
467       (A1->getParent() == S1->getParent()) && isa<GetElementPtrInst>(A0)) {
468     DEBUG(dbgs() << "Sink Instruction into BB \n"; BB->dump();
469           dbgs() << "Instruction Left\n"; S0->dump(); dbgs() << "\n";
470           dbgs() << "Instruction Right\n"; S1->dump(); dbgs() << "\n");
471     // Hoist the instruction.
472     BasicBlock::iterator InsertPt = BB->getFirstInsertionPt();
473     // Intersect optional metadata.
474     S0->intersectOptionalDataWith(S1);
475     S0->dropUnknownNonDebugMetadata();
476 
477     // Create the new store to be inserted at the join point.
478     StoreInst *SNew = (StoreInst *)(S0->clone());
479     Instruction *ANew = A0->clone();
480     SNew->insertBefore(&*InsertPt);
481     ANew->insertBefore(SNew);
482 
483     assert(S0->getParent() == A0->getParent());
484     assert(S1->getParent() == A1->getParent());
485 
486     PHINode *NewPN = getPHIOperand(BB, S0, S1);
487     // New PHI operand? Use it.
488     if (NewPN)
489       SNew->setOperand(0, NewPN);
490     removeInstruction(S0);
491     removeInstruction(S1);
492     A0->replaceAllUsesWith(ANew);
493     removeInstruction(A0);
494     A1->replaceAllUsesWith(ANew);
495     removeInstruction(A1);
496     return true;
497   }
498   return false;
499 }
500 
501 ///
502 /// \brief True when two stores are equivalent and can sink into the footer
503 ///
504 /// Starting from a diamond tail block, iterate over the instructions in one
505 /// predecessor block and try to match a store in the second predecessor.
506 ///
507 bool MergedLoadStoreMotion::mergeStores(BasicBlock *T) {
508 
509   bool MergedStores = false;
510   assert(T && "Footer of a diamond cannot be empty");
511 
512   pred_iterator PI = pred_begin(T), E = pred_end(T);
513   assert(PI != E);
514   BasicBlock *Pred0 = *PI;
515   ++PI;
516   BasicBlock *Pred1 = *PI;
517   ++PI;
518   // tail block  of a diamond/hammock?
519   if (Pred0 == Pred1)
520     return false; // No.
521   if (PI != E)
522     return false; // No. More than 2 predecessors.
523 
524   // #Instructions in Succ1 for Compile Time Control
525   int Size1 = Pred1->size();
526   int NStores = 0;
527 
528   for (BasicBlock::reverse_iterator RBI = Pred0->rbegin(), RBE = Pred0->rend();
529        RBI != RBE;) {
530 
531     Instruction *I = &*RBI;
532     ++RBI;
533 
534     // Sink move non-simple (atomic, volatile) stores
535     if (!isa<StoreInst>(I))
536       continue;
537     StoreInst *S0 = (StoreInst *)I;
538     if (!S0->isSimple())
539       continue;
540 
541     ++NStores;
542     if (NStores * Size1 >= MagicCompileTimeControl)
543       break;
544     if (StoreInst *S1 = canSinkFromBlock(Pred1, S0)) {
545       bool Res = sinkStore(T, S0, S1);
546       MergedStores |= Res;
547       // Don't attempt to sink below stores that had to stick around
548       // But after removal of a store and some of its feeding
549       // instruction search again from the beginning since the iterator
550       // is likely stale at this point.
551       if (!Res)
552         break;
553       else {
554         RBI = Pred0->rbegin();
555         RBE = Pred0->rend();
556         DEBUG(dbgs() << "Search again\n"; Instruction *I = &*RBI; I->dump());
557       }
558     }
559   }
560   return MergedStores;
561 }
562 
563 ///
564 /// \brief Run the transformation for each function
565 ///
566 bool MergedLoadStoreMotion::runOnFunction(Function &F) {
567   MD = getAnalysisIfAvailable<MemoryDependenceAnalysis>();
568   AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
569 
570   bool Changed = false;
571   DEBUG(dbgs() << "Instruction Merger\n");
572 
573   // Merge unconditional branches, allowing PRE to catch more
574   // optimization opportunities.
575   for (Function::iterator FI = F.begin(), FE = F.end(); FI != FE;) {
576     BasicBlock *BB = &*FI++;
577 
578     // Hoist equivalent loads and sink stores
579     // outside diamonds when possible
580     if (isDiamondHead(BB)) {
581       Changed |= mergeLoads(BB);
582       Changed |= mergeStores(getDiamondTail(BB));
583     }
584   }
585   return Changed;
586 }
587