1 //===- ScheduleDAG.cpp - Implement the ScheduleDAG class ------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file Implements the ScheduleDAG class, which is a base class used by
10 /// scheduling implementation classes.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/CodeGen/ScheduleDAG.h"
15 #include "llvm/ADT/STLExtras.h"
16 #include "llvm/ADT/SmallVector.h"
17 #include "llvm/ADT/Statistic.h"
18 #include "llvm/ADT/iterator_range.h"
19 #include "llvm/CodeGen/MachineFunction.h"
20 #include "llvm/CodeGen/ScheduleHazardRecognizer.h"
21 #include "llvm/CodeGen/SelectionDAGNodes.h"
22 #include "llvm/CodeGen/TargetInstrInfo.h"
23 #include "llvm/CodeGen/TargetRegisterInfo.h"
24 #include "llvm/CodeGen/TargetSubtargetInfo.h"
25 #include "llvm/Config/llvm-config.h"
26 #include "llvm/Support/CommandLine.h"
27 #include "llvm/Support/Compiler.h"
28 #include "llvm/Support/Debug.h"
29 #include "llvm/Support/raw_ostream.h"
30 #include <algorithm>
31 #include <cassert>
32 #include <iterator>
33 #include <limits>
34 #include <utility>
35 #include <vector>
36 
37 using namespace llvm;
38 
39 #define DEBUG_TYPE "pre-RA-sched"
40 
41 STATISTIC(NumNewPredsAdded, "Number of times a  single predecessor was added");
42 STATISTIC(NumTopoInits,
43           "Number of times the topological order has been recomputed");
44 
45 #ifndef NDEBUG
46 static cl::opt<bool> StressSchedOpt(
47   "stress-sched", cl::Hidden, cl::init(false),
48   cl::desc("Stress test instruction scheduling"));
49 #endif
50 
51 void SchedulingPriorityQueue::anchor() {}
52 
53 ScheduleDAG::ScheduleDAG(MachineFunction &mf)
54     : TM(mf.getTarget()), TII(mf.getSubtarget().getInstrInfo()),
55       TRI(mf.getSubtarget().getRegisterInfo()), MF(mf),
56       MRI(mf.getRegInfo()) {
57 #ifndef NDEBUG
58   StressSched = StressSchedOpt;
59 #endif
60 }
61 
62 ScheduleDAG::~ScheduleDAG() = default;
63 
64 void ScheduleDAG::clearDAG() {
65   SUnits.clear();
66   ExitSU = SUnit();
67 }
68 
69 const MCInstrDesc *ScheduleDAG::getNodeDesc(const SDNode *Node) const {
70   if (!Node || !Node->isMachineOpcode()) return nullptr;
71   return &TII->get(Node->getMachineOpcode());
72 }
73 
74 LLVM_DUMP_METHOD void SDep::dump(const TargetRegisterInfo *TRI) const {
75   switch (getKind()) {
76   case Data:   dbgs() << "Data"; break;
77   case Anti:   dbgs() << "Anti"; break;
78   case Output: dbgs() << "Out "; break;
79   case Order:  dbgs() << "Ord "; break;
80   }
81 
82   switch (getKind()) {
83   case Data:
84     dbgs() << " Latency=" << getLatency();
85     if (TRI && isAssignedRegDep())
86       dbgs() << " Reg=" << printReg(getReg(), TRI);
87     break;
88   case Anti:
89   case Output:
90     dbgs() << " Latency=" << getLatency();
91     break;
92   case Order:
93     dbgs() << " Latency=" << getLatency();
94     switch(Contents.OrdKind) {
95     case Barrier:      dbgs() << " Barrier"; break;
96     case MayAliasMem:
97     case MustAliasMem: dbgs() << " Memory"; break;
98     case Artificial:   dbgs() << " Artificial"; break;
99     case Weak:         dbgs() << " Weak"; break;
100     case Cluster:      dbgs() << " Cluster"; break;
101     }
102     break;
103   }
104 }
105 
106 bool SUnit::addPred(const SDep &D, bool Required) {
107   // If this node already has this dependence, don't add a redundant one.
108   for (SDep &PredDep : Preds) {
109     // Zero-latency weak edges may be added purely for heuristic ordering. Don't
110     // add them if another kind of edge already exists.
111     if (!Required && PredDep.getSUnit() == D.getSUnit())
112       return false;
113     if (PredDep.overlaps(D)) {
114       // Extend the latency if needed. Equivalent to
115       // removePred(PredDep) + addPred(D).
116       if (PredDep.getLatency() < D.getLatency()) {
117         SUnit *PredSU = PredDep.getSUnit();
118         // Find the corresponding successor in N.
119         SDep ForwardD = PredDep;
120         ForwardD.setSUnit(this);
121         for (SDep &SuccDep : PredSU->Succs) {
122           if (SuccDep == ForwardD) {
123             SuccDep.setLatency(D.getLatency());
124             break;
125           }
126         }
127         PredDep.setLatency(D.getLatency());
128       }
129       return false;
130     }
131   }
132   // Now add a corresponding succ to N.
133   SDep P = D;
134   P.setSUnit(this);
135   SUnit *N = D.getSUnit();
136   // Update the bookkeeping.
137   if (D.getKind() == SDep::Data) {
138     assert(NumPreds < std::numeric_limits<unsigned>::max() &&
139            "NumPreds will overflow!");
140     assert(N->NumSuccs < std::numeric_limits<unsigned>::max() &&
141            "NumSuccs will overflow!");
142     ++NumPreds;
143     ++N->NumSuccs;
144   }
145   if (!N->isScheduled) {
146     if (D.isWeak()) {
147       ++WeakPredsLeft;
148     }
149     else {
150       assert(NumPredsLeft < std::numeric_limits<unsigned>::max() &&
151              "NumPredsLeft will overflow!");
152       ++NumPredsLeft;
153     }
154   }
155   if (!isScheduled) {
156     if (D.isWeak()) {
157       ++N->WeakSuccsLeft;
158     }
159     else {
160       assert(N->NumSuccsLeft < std::numeric_limits<unsigned>::max() &&
161              "NumSuccsLeft will overflow!");
162       ++N->NumSuccsLeft;
163     }
164   }
165   Preds.push_back(D);
166   N->Succs.push_back(P);
167   if (P.getLatency() != 0) {
168     this->setDepthDirty();
169     N->setHeightDirty();
170   }
171   return true;
172 }
173 
174 void SUnit::removePred(const SDep &D) {
175   // Find the matching predecessor.
176   SmallVectorImpl<SDep>::iterator I = llvm::find(Preds, D);
177   if (I == Preds.end())
178     return;
179   // Find the corresponding successor in N.
180   SDep P = D;
181   P.setSUnit(this);
182   SUnit *N = D.getSUnit();
183   SmallVectorImpl<SDep>::iterator Succ = llvm::find(N->Succs, P);
184   assert(Succ != N->Succs.end() && "Mismatching preds / succs lists!");
185   N->Succs.erase(Succ);
186   Preds.erase(I);
187   // Update the bookkeeping.
188   if (P.getKind() == SDep::Data) {
189     assert(NumPreds > 0 && "NumPreds will underflow!");
190     assert(N->NumSuccs > 0 && "NumSuccs will underflow!");
191     --NumPreds;
192     --N->NumSuccs;
193   }
194   if (!N->isScheduled) {
195     if (D.isWeak())
196       --WeakPredsLeft;
197     else {
198       assert(NumPredsLeft > 0 && "NumPredsLeft will underflow!");
199       --NumPredsLeft;
200     }
201   }
202   if (!isScheduled) {
203     if (D.isWeak())
204       --N->WeakSuccsLeft;
205     else {
206       assert(N->NumSuccsLeft > 0 && "NumSuccsLeft will underflow!");
207       --N->NumSuccsLeft;
208     }
209   }
210   if (P.getLatency() != 0) {
211     this->setDepthDirty();
212     N->setHeightDirty();
213   }
214 }
215 
216 void SUnit::setDepthDirty() {
217   if (!isDepthCurrent) return;
218   SmallVector<SUnit*, 8> WorkList;
219   WorkList.push_back(this);
220   do {
221     SUnit *SU = WorkList.pop_back_val();
222     SU->isDepthCurrent = false;
223     for (SDep &SuccDep : SU->Succs) {
224       SUnit *SuccSU = SuccDep.getSUnit();
225       if (SuccSU->isDepthCurrent)
226         WorkList.push_back(SuccSU);
227     }
228   } while (!WorkList.empty());
229 }
230 
231 void SUnit::setHeightDirty() {
232   if (!isHeightCurrent) return;
233   SmallVector<SUnit*, 8> WorkList;
234   WorkList.push_back(this);
235   do {
236     SUnit *SU = WorkList.pop_back_val();
237     SU->isHeightCurrent = false;
238     for (SDep &PredDep : SU->Preds) {
239       SUnit *PredSU = PredDep.getSUnit();
240       if (PredSU->isHeightCurrent)
241         WorkList.push_back(PredSU);
242     }
243   } while (!WorkList.empty());
244 }
245 
246 void SUnit::setDepthToAtLeast(unsigned NewDepth) {
247   if (NewDepth <= getDepth())
248     return;
249   setDepthDirty();
250   Depth = NewDepth;
251   isDepthCurrent = true;
252 }
253 
254 void SUnit::setHeightToAtLeast(unsigned NewHeight) {
255   if (NewHeight <= getHeight())
256     return;
257   setHeightDirty();
258   Height = NewHeight;
259   isHeightCurrent = true;
260 }
261 
262 /// Calculates the maximal path from the node to the exit.
263 void SUnit::ComputeDepth() {
264   SmallVector<SUnit*, 8> WorkList;
265   WorkList.push_back(this);
266   do {
267     SUnit *Cur = WorkList.back();
268 
269     bool Done = true;
270     unsigned MaxPredDepth = 0;
271     for (const SDep &PredDep : Cur->Preds) {
272       SUnit *PredSU = PredDep.getSUnit();
273       if (PredSU->isDepthCurrent)
274         MaxPredDepth = std::max(MaxPredDepth,
275                                 PredSU->Depth + PredDep.getLatency());
276       else {
277         Done = false;
278         WorkList.push_back(PredSU);
279       }
280     }
281 
282     if (Done) {
283       WorkList.pop_back();
284       if (MaxPredDepth != Cur->Depth) {
285         Cur->setDepthDirty();
286         Cur->Depth = MaxPredDepth;
287       }
288       Cur->isDepthCurrent = true;
289     }
290   } while (!WorkList.empty());
291 }
292 
293 /// Calculates the maximal path from the node to the entry.
294 void SUnit::ComputeHeight() {
295   SmallVector<SUnit*, 8> WorkList;
296   WorkList.push_back(this);
297   do {
298     SUnit *Cur = WorkList.back();
299 
300     bool Done = true;
301     unsigned MaxSuccHeight = 0;
302     for (const SDep &SuccDep : Cur->Succs) {
303       SUnit *SuccSU = SuccDep.getSUnit();
304       if (SuccSU->isHeightCurrent)
305         MaxSuccHeight = std::max(MaxSuccHeight,
306                                  SuccSU->Height + SuccDep.getLatency());
307       else {
308         Done = false;
309         WorkList.push_back(SuccSU);
310       }
311     }
312 
313     if (Done) {
314       WorkList.pop_back();
315       if (MaxSuccHeight != Cur->Height) {
316         Cur->setHeightDirty();
317         Cur->Height = MaxSuccHeight;
318       }
319       Cur->isHeightCurrent = true;
320     }
321   } while (!WorkList.empty());
322 }
323 
324 void SUnit::biasCriticalPath() {
325   if (NumPreds < 2)
326     return;
327 
328   SUnit::pred_iterator BestI = Preds.begin();
329   unsigned MaxDepth = BestI->getSUnit()->getDepth();
330   for (SUnit::pred_iterator I = std::next(BestI), E = Preds.end(); I != E;
331        ++I) {
332     if (I->getKind() == SDep::Data && I->getSUnit()->getDepth() > MaxDepth)
333       BestI = I;
334   }
335   if (BestI != Preds.begin())
336     std::swap(*Preds.begin(), *BestI);
337 }
338 
339 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
340 LLVM_DUMP_METHOD void SUnit::dumpAttributes() const {
341   dbgs() << "  # preds left       : " << NumPredsLeft << "\n";
342   dbgs() << "  # succs left       : " << NumSuccsLeft << "\n";
343   if (WeakPredsLeft)
344     dbgs() << "  # weak preds left  : " << WeakPredsLeft << "\n";
345   if (WeakSuccsLeft)
346     dbgs() << "  # weak succs left  : " << WeakSuccsLeft << "\n";
347   dbgs() << "  # rdefs left       : " << NumRegDefsLeft << "\n";
348   dbgs() << "  Latency            : " << Latency << "\n";
349   dbgs() << "  Depth              : " << getDepth() << "\n";
350   dbgs() << "  Height             : " << getHeight() << "\n";
351 }
352 
353 LLVM_DUMP_METHOD void ScheduleDAG::dumpNodeName(const SUnit &SU) const {
354   if (&SU == &ExitSU)
355     dbgs() << "ExitSU";
356   else
357     dbgs() << "SU(" << SU.NodeNum << ")";
358 }
359 
360 LLVM_DUMP_METHOD void ScheduleDAG::dumpNodeAll(const SUnit &SU) const {
361   dumpNode(SU);
362   SU.dumpAttributes();
363   if (SU.Preds.size() > 0) {
364     dbgs() << "  Predecessors:\n";
365     for (const SDep &Dep : SU.Preds) {
366       dbgs() << "    ";
367       dumpNodeName(*Dep.getSUnit());
368       dbgs() << ": ";
369       Dep.dump(TRI);
370       dbgs() << '\n';
371     }
372   }
373   if (SU.Succs.size() > 0) {
374     dbgs() << "  Successors:\n";
375     for (const SDep &Dep : SU.Succs) {
376       dbgs() << "    ";
377       dumpNodeName(*Dep.getSUnit());
378       dbgs() << ": ";
379       Dep.dump(TRI);
380       dbgs() << '\n';
381     }
382   }
383 }
384 #endif
385 
386 #ifndef NDEBUG
387 unsigned ScheduleDAG::VerifyScheduledDAG(bool isBottomUp) {
388   bool AnyNotSched = false;
389   unsigned DeadNodes = 0;
390   for (const SUnit &SUnit : SUnits) {
391     if (!SUnit.isScheduled) {
392       if (SUnit.NumPreds == 0 && SUnit.NumSuccs == 0) {
393         ++DeadNodes;
394         continue;
395       }
396       if (!AnyNotSched)
397         dbgs() << "*** Scheduling failed! ***\n";
398       dumpNode(SUnit);
399       dbgs() << "has not been scheduled!\n";
400       AnyNotSched = true;
401     }
402     if (SUnit.isScheduled &&
403         (isBottomUp ? SUnit.getHeight() : SUnit.getDepth()) >
404           unsigned(std::numeric_limits<int>::max())) {
405       if (!AnyNotSched)
406         dbgs() << "*** Scheduling failed! ***\n";
407       dumpNode(SUnit);
408       dbgs() << "has an unexpected "
409            << (isBottomUp ? "Height" : "Depth") << " value!\n";
410       AnyNotSched = true;
411     }
412     if (isBottomUp) {
413       if (SUnit.NumSuccsLeft != 0) {
414         if (!AnyNotSched)
415           dbgs() << "*** Scheduling failed! ***\n";
416         dumpNode(SUnit);
417         dbgs() << "has successors left!\n";
418         AnyNotSched = true;
419       }
420     } else {
421       if (SUnit.NumPredsLeft != 0) {
422         if (!AnyNotSched)
423           dbgs() << "*** Scheduling failed! ***\n";
424         dumpNode(SUnit);
425         dbgs() << "has predecessors left!\n";
426         AnyNotSched = true;
427       }
428     }
429   }
430   assert(!AnyNotSched);
431   return SUnits.size() - DeadNodes;
432 }
433 #endif
434 
435 void ScheduleDAGTopologicalSort::InitDAGTopologicalSorting() {
436   // The idea of the algorithm is taken from
437   // "Online algorithms for managing the topological order of
438   // a directed acyclic graph" by David J. Pearce and Paul H.J. Kelly
439   // This is the MNR algorithm, which was first introduced by
440   // A. Marchetti-Spaccamela, U. Nanni and H. Rohnert in
441   // "Maintaining a topological order under edge insertions".
442   //
443   // Short description of the algorithm:
444   //
445   // Topological ordering, ord, of a DAG maps each node to a topological
446   // index so that for all edges X->Y it is the case that ord(X) < ord(Y).
447   //
448   // This means that if there is a path from the node X to the node Z,
449   // then ord(X) < ord(Z).
450   //
451   // This property can be used to check for reachability of nodes:
452   // if Z is reachable from X, then an insertion of the edge Z->X would
453   // create a cycle.
454   //
455   // The algorithm first computes a topological ordering for the DAG by
456   // initializing the Index2Node and Node2Index arrays and then tries to keep
457   // the ordering up-to-date after edge insertions by reordering the DAG.
458   //
459   // On insertion of the edge X->Y, the algorithm first marks by calling DFS
460   // the nodes reachable from Y, and then shifts them using Shift to lie
461   // immediately after X in Index2Node.
462 
463   // Cancel pending updates, mark as valid.
464   Dirty = false;
465   Updates.clear();
466 
467   unsigned DAGSize = SUnits.size();
468   std::vector<SUnit*> WorkList;
469   WorkList.reserve(DAGSize);
470 
471   Index2Node.resize(DAGSize);
472   Node2Index.resize(DAGSize);
473 
474   // Initialize the data structures.
475   if (ExitSU)
476     WorkList.push_back(ExitSU);
477   for (SUnit &SU : SUnits) {
478     int NodeNum = SU.NodeNum;
479     unsigned Degree = SU.Succs.size();
480     // Temporarily use the Node2Index array as scratch space for degree counts.
481     Node2Index[NodeNum] = Degree;
482 
483     // Is it a node without dependencies?
484     if (Degree == 0) {
485       assert(SU.Succs.empty() && "SUnit should have no successors");
486       // Collect leaf nodes.
487       WorkList.push_back(&SU);
488     }
489   }
490 
491   int Id = DAGSize;
492   while (!WorkList.empty()) {
493     SUnit *SU = WorkList.back();
494     WorkList.pop_back();
495     if (SU->NodeNum < DAGSize)
496       Allocate(SU->NodeNum, --Id);
497     for (const SDep &PredDep : SU->Preds) {
498       SUnit *SU = PredDep.getSUnit();
499       if (SU->NodeNum < DAGSize && !--Node2Index[SU->NodeNum])
500         // If all dependencies of the node are processed already,
501         // then the node can be computed now.
502         WorkList.push_back(SU);
503     }
504   }
505 
506   Visited.resize(DAGSize);
507   NumTopoInits++;
508 
509 #ifndef NDEBUG
510   // Check correctness of the ordering
511   for (SUnit &SU : SUnits)  {
512     for (const SDep &PD : SU.Preds) {
513       assert(Node2Index[SU.NodeNum] > Node2Index[PD.getSUnit()->NodeNum] &&
514       "Wrong topological sorting");
515     }
516   }
517 #endif
518 }
519 
520 void ScheduleDAGTopologicalSort::FixOrder() {
521   // Recompute from scratch after new nodes have been added.
522   if (Dirty) {
523     InitDAGTopologicalSorting();
524     return;
525   }
526 
527   // Otherwise apply updates one-by-one.
528   for (auto &U : Updates)
529     AddPred(U.first, U.second);
530   Updates.clear();
531 }
532 
533 void ScheduleDAGTopologicalSort::AddPredQueued(SUnit *Y, SUnit *X) {
534   // Recomputing the order from scratch is likely more efficient than applying
535   // updates one-by-one for too many updates. The current cut-off is arbitrarily
536   // chosen.
537   Dirty = Dirty || Updates.size() > 10;
538 
539   if (Dirty)
540     return;
541 
542   Updates.emplace_back(Y, X);
543 }
544 
545 void ScheduleDAGTopologicalSort::AddPred(SUnit *Y, SUnit *X) {
546   int UpperBound, LowerBound;
547   LowerBound = Node2Index[Y->NodeNum];
548   UpperBound = Node2Index[X->NodeNum];
549   bool HasLoop = false;
550   // Is Ord(X) < Ord(Y) ?
551   if (LowerBound < UpperBound) {
552     // Update the topological order.
553     Visited.reset();
554     DFS(Y, UpperBound, HasLoop);
555     assert(!HasLoop && "Inserted edge creates a loop!");
556     // Recompute topological indexes.
557     Shift(Visited, LowerBound, UpperBound);
558   }
559 
560   NumNewPredsAdded++;
561 }
562 
563 void ScheduleDAGTopologicalSort::RemovePred(SUnit *M, SUnit *N) {
564   // InitDAGTopologicalSorting();
565 }
566 
567 void ScheduleDAGTopologicalSort::DFS(const SUnit *SU, int UpperBound,
568                                      bool &HasLoop) {
569   std::vector<const SUnit*> WorkList;
570   WorkList.reserve(SUnits.size());
571 
572   WorkList.push_back(SU);
573   do {
574     SU = WorkList.back();
575     WorkList.pop_back();
576     Visited.set(SU->NodeNum);
577     for (const SDep &SuccDep
578          : make_range(SU->Succs.rbegin(), SU->Succs.rend())) {
579       unsigned s = SuccDep.getSUnit()->NodeNum;
580       // Edges to non-SUnits are allowed but ignored (e.g. ExitSU).
581       if (s >= Node2Index.size())
582         continue;
583       if (Node2Index[s] == UpperBound) {
584         HasLoop = true;
585         return;
586       }
587       // Visit successors if not already and in affected region.
588       if (!Visited.test(s) && Node2Index[s] < UpperBound) {
589         WorkList.push_back(SuccDep.getSUnit());
590       }
591     }
592   } while (!WorkList.empty());
593 }
594 
595 std::vector<int> ScheduleDAGTopologicalSort::GetSubGraph(const SUnit &StartSU,
596                                                          const SUnit &TargetSU,
597                                                          bool &Success) {
598   std::vector<const SUnit*> WorkList;
599   int LowerBound = Node2Index[StartSU.NodeNum];
600   int UpperBound = Node2Index[TargetSU.NodeNum];
601   bool Found = false;
602   BitVector VisitedBack;
603   std::vector<int> Nodes;
604 
605   if (LowerBound > UpperBound) {
606     Success = false;
607     return Nodes;
608   }
609 
610   WorkList.reserve(SUnits.size());
611   Visited.reset();
612 
613   // Starting from StartSU, visit all successors up
614   // to UpperBound.
615   WorkList.push_back(&StartSU);
616   do {
617     const SUnit *SU = WorkList.back();
618     WorkList.pop_back();
619     for (int I = SU->Succs.size()-1; I >= 0; --I) {
620       const SUnit *Succ = SU->Succs[I].getSUnit();
621       unsigned s = Succ->NodeNum;
622       // Edges to non-SUnits are allowed but ignored (e.g. ExitSU).
623       if (Succ->isBoundaryNode())
624         continue;
625       if (Node2Index[s] == UpperBound) {
626         Found = true;
627         continue;
628       }
629       // Visit successors if not already and in affected region.
630       if (!Visited.test(s) && Node2Index[s] < UpperBound) {
631         Visited.set(s);
632         WorkList.push_back(Succ);
633       }
634     }
635   } while (!WorkList.empty());
636 
637   if (!Found) {
638     Success = false;
639     return Nodes;
640   }
641 
642   WorkList.clear();
643   VisitedBack.resize(SUnits.size());
644   Found = false;
645 
646   // Starting from TargetSU, visit all predecessors up
647   // to LowerBound. SUs that are visited by the two
648   // passes are added to Nodes.
649   WorkList.push_back(&TargetSU);
650   do {
651     const SUnit *SU = WorkList.back();
652     WorkList.pop_back();
653     for (int I = SU->Preds.size()-1; I >= 0; --I) {
654       const SUnit *Pred = SU->Preds[I].getSUnit();
655       unsigned s = Pred->NodeNum;
656       // Edges to non-SUnits are allowed but ignored (e.g. ExitSU).
657       if (Pred->isBoundaryNode())
658         continue;
659       if (Node2Index[s] == LowerBound) {
660         Found = true;
661         continue;
662       }
663       if (!VisitedBack.test(s) && Visited.test(s)) {
664         VisitedBack.set(s);
665         WorkList.push_back(Pred);
666         Nodes.push_back(s);
667       }
668     }
669   } while (!WorkList.empty());
670 
671   assert(Found && "Error in SUnit Graph!");
672   Success = true;
673   return Nodes;
674 }
675 
676 void ScheduleDAGTopologicalSort::Shift(BitVector& Visited, int LowerBound,
677                                        int UpperBound) {
678   std::vector<int> L;
679   int shift = 0;
680   int i;
681 
682   for (i = LowerBound; i <= UpperBound; ++i) {
683     // w is node at topological index i.
684     int w = Index2Node[i];
685     if (Visited.test(w)) {
686       // Unmark.
687       Visited.reset(w);
688       L.push_back(w);
689       shift = shift + 1;
690     } else {
691       Allocate(w, i - shift);
692     }
693   }
694 
695   for (unsigned LI : L) {
696     Allocate(LI, i - shift);
697     i = i + 1;
698   }
699 }
700 
701 bool ScheduleDAGTopologicalSort::WillCreateCycle(SUnit *TargetSU, SUnit *SU) {
702   FixOrder();
703   // Is SU reachable from TargetSU via successor edges?
704   if (IsReachable(SU, TargetSU))
705     return true;
706   for (const SDep &PredDep : TargetSU->Preds)
707     if (PredDep.isAssignedRegDep() &&
708         IsReachable(SU, PredDep.getSUnit()))
709       return true;
710   return false;
711 }
712 
713 void ScheduleDAGTopologicalSort::AddSUnitWithoutPredecessors(const SUnit *SU) {
714   assert(SU->NodeNum == Index2Node.size() && "Node cannot be added at the end");
715   assert(SU->NumPreds == 0 && "Can only add SU's with no predecessors");
716   Node2Index.push_back(Index2Node.size());
717   Index2Node.push_back(SU->NodeNum);
718   Visited.resize(Node2Index.size());
719 }
720 
721 bool ScheduleDAGTopologicalSort::IsReachable(const SUnit *SU,
722                                              const SUnit *TargetSU) {
723   FixOrder();
724   // If insertion of the edge SU->TargetSU would create a cycle
725   // then there is a path from TargetSU to SU.
726   int UpperBound, LowerBound;
727   LowerBound = Node2Index[TargetSU->NodeNum];
728   UpperBound = Node2Index[SU->NodeNum];
729   bool HasLoop = false;
730   // Is Ord(TargetSU) < Ord(SU) ?
731   if (LowerBound < UpperBound) {
732     Visited.reset();
733     // There may be a path from TargetSU to SU. Check for it.
734     DFS(TargetSU, UpperBound, HasLoop);
735   }
736   return HasLoop;
737 }
738 
739 void ScheduleDAGTopologicalSort::Allocate(int n, int index) {
740   Node2Index[n] = index;
741   Index2Node[index] = n;
742 }
743 
744 ScheduleDAGTopologicalSort::
745 ScheduleDAGTopologicalSort(std::vector<SUnit> &sunits, SUnit *exitsu)
746   : SUnits(sunits), ExitSU(exitsu) {}
747 
748 ScheduleHazardRecognizer::~ScheduleHazardRecognizer() = default;
749