1 //===-- GCNSchedStrategy.h - GCN Scheduler Strategy -*- C++ -*-------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #ifndef LLVM_LIB_TARGET_AMDGPU_GCNSCHEDSTRATEGY_H
14 #define LLVM_LIB_TARGET_AMDGPU_GCNSCHEDSTRATEGY_H
15 
16 #include "GCNRegPressure.h"
17 #include "llvm/ADT/MapVector.h"
18 #include "llvm/CodeGen/MachineScheduler.h"
19 
20 namespace llvm {
21 
22 class SIMachineFunctionInfo;
23 class SIRegisterInfo;
24 class GCNSubtarget;
25 
26 /// This is a minimal scheduler strategy.  The main difference between this
27 /// and the GenericScheduler is that GCNSchedStrategy uses different
28 /// heuristics to determine excess/critical pressure sets.  Its goal is to
29 /// maximize kernel occupancy (i.e. maximum number of waves per simd).
30 class GCNMaxOccupancySchedStrategy final : public GenericScheduler {
31   friend class GCNScheduleDAGMILive;
32 
33   SUnit *pickNodeBidirectional(bool &IsTopNode);
34 
35   void pickNodeFromQueue(SchedBoundary &Zone, const CandPolicy &ZonePolicy,
36                          const RegPressureTracker &RPTracker,
37                          SchedCandidate &Cand);
38 
39   void initCandidate(SchedCandidate &Cand, SUnit *SU,
40                      bool AtTop, const RegPressureTracker &RPTracker,
41                      const SIRegisterInfo *SRI,
42                      unsigned SGPRPressure, unsigned VGPRPressure);
43 
44   std::vector<unsigned> Pressure;
45   std::vector<unsigned> MaxPressure;
46 
47   unsigned SGPRExcessLimit;
48   unsigned VGPRExcessLimit;
49   unsigned SGPRCriticalLimit;
50   unsigned VGPRCriticalLimit;
51 
52   unsigned TargetOccupancy;
53 
54   // schedule() have seen a clustered memory operation. Set it to false
55   // before a region scheduling to know if the region had such clusters.
56   bool HasClusteredNodes;
57 
58   // schedule() have seen an excess register pressure and had to track
59   // register pressure for actual scheduling heuristics.
60   bool HasExcessPressure;
61 
62   MachineFunction *MF;
63 
64 public:
65   GCNMaxOccupancySchedStrategy(const MachineSchedContext *C);
66 
67   SUnit *pickNode(bool &IsTopNode) override;
68 
69   void initialize(ScheduleDAGMI *DAG) override;
70 
71   void setTargetOccupancy(unsigned Occ) { TargetOccupancy = Occ; }
72 };
73 
74 class GCNScheduleDAGMILive final : public ScheduleDAGMILive {
75 
76   enum : unsigned {
77     Collect,
78     InitialSchedule,
79     UnclusteredReschedule,
80     ClusteredLowOccupancyReschedule,
81     PreRARematerialize,
82     LastStage = PreRARematerialize
83   };
84 
85   const GCNSubtarget &ST;
86 
87   SIMachineFunctionInfo &MFI;
88 
89   // Occupancy target at the beginning of function scheduling cycle.
90   unsigned StartingOccupancy;
91 
92   // Minimal real occupancy recorder for the function.
93   unsigned MinOccupancy;
94 
95   // Scheduling stage number.
96   unsigned Stage;
97 
98   // Current region index.
99   size_t RegionIdx;
100 
101   // Vector of regions recorder for later rescheduling
102   SmallVector<std::pair<MachineBasicBlock::iterator,
103                         MachineBasicBlock::iterator>, 32> Regions;
104 
105   // Records if a region is not yet scheduled, or schedule has been reverted,
106   // or we generally desire to reschedule it.
107   BitVector RescheduleRegions;
108 
109   // Record regions which use clustered loads/stores.
110   BitVector RegionsWithClusters;
111 
112   // Record regions with high register pressure.
113   BitVector RegionsWithHighRP;
114 
115   // Regions that has the same occupancy as the latest MinOccupancy
116   BitVector RegionsWithMinOcc;
117 
118   // Region live-in cache.
119   SmallVector<GCNRPTracker::LiveRegSet, 32> LiveIns;
120 
121   // Region pressure cache.
122   SmallVector<GCNRegPressure, 32> Pressure;
123 
124   // Each region at MinOccupancy will have their own list of trivially
125   // rematerializable instructions we can remat to reduce RP. The list maps an
126   // instruction to the position we should remat before, usually the MI using
127   // the rematerializable instruction.
128   MapVector<unsigned, MapVector<MachineInstr *, MachineInstr *>>
129       RematerializableInsts;
130 
131   // Map a trivially remateriazable def to a list of regions at MinOccupancy
132   // that has the defined reg as a live-in.
133   DenseMap<MachineInstr *, SmallVector<unsigned, 4>> RematDefToLiveInRegions;
134 
135   // Temporary basic block live-in cache.
136   DenseMap<const MachineBasicBlock*, GCNRPTracker::LiveRegSet> MBBLiveIns;
137 
138   DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet> BBLiveInMap;
139   DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet> getBBLiveInMap() const;
140 
141   // Collect all trivially rematerializable VGPR instructions with a single def
142   // and single use outside the defining block into RematerializableInsts.
143   void collectRematerializableInstructions();
144 
145   bool isTriviallyReMaterializable(const MachineInstr &MI);
146 
147   // TODO: Should also attempt to reduce RP of SGPRs and AGPRs
148   // Attempt to reduce RP of VGPR by sinking trivially rematerializable
149   // instructions. Returns true if we were able to sink instruction(s).
150   bool sinkTriviallyRematInsts(const GCNSubtarget &ST,
151                                const TargetInstrInfo *TII);
152 
153   // Return current region pressure.
154   GCNRegPressure getRealRegPressure() const;
155 
156   // Compute and cache live-ins and pressure for all regions in block.
157   void computeBlockPressure(const MachineBasicBlock *MBB);
158 
159   // Update region boundaries when removing MI or inserting NewMI before MI.
160   void updateRegionBoundaries(
161       SmallVectorImpl<std::pair<MachineBasicBlock::iterator,
162                                 MachineBasicBlock::iterator>> &RegionBoundaries,
163       MachineBasicBlock::iterator MI, MachineInstr *NewMI,
164       bool Removing = false);
165 
166 public:
167   GCNScheduleDAGMILive(MachineSchedContext *C,
168                        std::unique_ptr<MachineSchedStrategy> S);
169 
170   void schedule() override;
171 
172   void finalizeSchedule() override;
173 };
174 
175 } // End namespace llvm
176 
177 #endif // LLVM_LIB_TARGET_AMDGPU_GCNSCHEDSTRATEGY_H
178