1 //===- SIPreAllocateWWMRegs.cpp - WWM Register Pre-allocation -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// Pass to pre-allocated WWM registers
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "AMDGPU.h"
15 #include "AMDGPUSubtarget.h"
16 #include "SIMachineFunctionInfo.h"
17 #include "llvm/ADT/PostOrderIterator.h"
18 #include "llvm/CodeGen/LiveIntervals.h"
19 #include "llvm/CodeGen/LiveRegMatrix.h"
20 #include "llvm/CodeGen/MachineFunctionPass.h"
21 #include "llvm/InitializePasses.h"
22 
23 using namespace llvm;
24 
25 #define DEBUG_TYPE "si-pre-allocate-wwm-regs"
26 
27 namespace {
28 
29 class SIPreAllocateWWMRegs : public MachineFunctionPass {
30 private:
31   const SIInstrInfo *TII;
32   const SIRegisterInfo *TRI;
33   MachineRegisterInfo *MRI;
34   LiveIntervals *LIS;
35   LiveRegMatrix *Matrix;
36   VirtRegMap *VRM;
37   RegisterClassInfo RegClassInfo;
38 
39   std::vector<unsigned> RegsToRewrite;
40 
41 public:
42   static char ID;
43 
44   SIPreAllocateWWMRegs() : MachineFunctionPass(ID) {
45     initializeSIPreAllocateWWMRegsPass(*PassRegistry::getPassRegistry());
46   }
47 
48   bool runOnMachineFunction(MachineFunction &MF) override;
49 
50   void getAnalysisUsage(AnalysisUsage &AU) const override {
51     AU.addRequired<LiveIntervals>();
52     AU.addPreserved<LiveIntervals>();
53     AU.addRequired<VirtRegMap>();
54     AU.addRequired<LiveRegMatrix>();
55     AU.addPreserved<SlotIndexes>();
56     AU.setPreservesCFG();
57     MachineFunctionPass::getAnalysisUsage(AU);
58   }
59 
60 private:
61   bool processDef(MachineOperand &MO);
62   void rewriteRegs(MachineFunction &MF);
63 };
64 
65 } // End anonymous namespace.
66 
67 INITIALIZE_PASS_BEGIN(SIPreAllocateWWMRegs, DEBUG_TYPE,
68                 "SI Pre-allocate WWM Registers", false, false)
69 INITIALIZE_PASS_DEPENDENCY(LiveIntervals)
70 INITIALIZE_PASS_DEPENDENCY(VirtRegMap)
71 INITIALIZE_PASS_DEPENDENCY(LiveRegMatrix)
72 INITIALIZE_PASS_END(SIPreAllocateWWMRegs, DEBUG_TYPE,
73                 "SI Pre-allocate WWM Registers", false, false)
74 
75 char SIPreAllocateWWMRegs::ID = 0;
76 
77 char &llvm::SIPreAllocateWWMRegsID = SIPreAllocateWWMRegs::ID;
78 
79 FunctionPass *llvm::createSIPreAllocateWWMRegsPass() {
80   return new SIPreAllocateWWMRegs();
81 }
82 
83 bool SIPreAllocateWWMRegs::processDef(MachineOperand &MO) {
84   if (!MO.isReg())
85     return false;
86 
87   Register Reg = MO.getReg();
88   if (Reg.isPhysical())
89     return false;
90 
91   if (!TRI->isVGPR(*MRI, Reg))
92     return false;
93 
94   if (VRM->hasPhys(Reg))
95     return false;
96 
97   LiveInterval &LI = LIS->getInterval(Reg);
98 
99   for (MCRegister PhysReg : RegClassInfo.getOrder(MRI->getRegClass(Reg))) {
100     if (!MRI->isPhysRegUsed(PhysReg) &&
101         Matrix->checkInterference(LI, PhysReg) == LiveRegMatrix::IK_Free) {
102       Matrix->assign(LI, PhysReg);
103       assert(PhysReg != 0);
104       RegsToRewrite.push_back(Reg);
105       return true;
106     }
107   }
108 
109   llvm_unreachable("physreg not found for WWM expression");
110   return false;
111 }
112 
113 void SIPreAllocateWWMRegs::rewriteRegs(MachineFunction &MF) {
114   for (MachineBasicBlock &MBB : MF) {
115     for (MachineInstr &MI : MBB) {
116       for (MachineOperand &MO : MI.operands()) {
117         if (!MO.isReg())
118           continue;
119 
120         const Register VirtReg = MO.getReg();
121         if (VirtReg.isPhysical())
122           continue;
123 
124         if (!VRM->hasPhys(VirtReg))
125           continue;
126 
127         Register PhysReg = VRM->getPhys(VirtReg);
128         const unsigned SubReg = MO.getSubReg();
129         if (SubReg != 0) {
130           PhysReg = TRI->getSubReg(PhysReg, SubReg);
131           MO.setSubReg(0);
132         }
133 
134         MO.setReg(PhysReg);
135         MO.setIsRenamable(false);
136       }
137     }
138   }
139 
140   SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
141 
142   for (unsigned Reg : RegsToRewrite) {
143     LIS->removeInterval(Reg);
144 
145     const Register PhysReg = VRM->getPhys(Reg);
146     assert(PhysReg != 0);
147     MFI->ReserveWWMRegister(PhysReg);
148   }
149 
150   RegsToRewrite.clear();
151 
152   // Update the set of reserved registers to include WWM ones.
153   MRI->freezeReservedRegs(MF);
154 }
155 
156 bool SIPreAllocateWWMRegs::runOnMachineFunction(MachineFunction &MF) {
157   LLVM_DEBUG(dbgs() << "SIPreAllocateWWMRegs: function " << MF.getName() << "\n");
158 
159   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
160 
161   TII = ST.getInstrInfo();
162   TRI = &TII->getRegisterInfo();
163   MRI = &MF.getRegInfo();
164 
165   LIS = &getAnalysis<LiveIntervals>();
166   Matrix = &getAnalysis<LiveRegMatrix>();
167   VRM = &getAnalysis<VirtRegMap>();
168 
169   RegClassInfo.runOnMachineFunction(MF);
170 
171   bool RegsAssigned = false;
172 
173   // We use a reverse post-order traversal of the control-flow graph to
174   // guarantee that we visit definitions in dominance order. Since WWM
175   // expressions are guaranteed to never involve phi nodes, and we can only
176   // escape WWM through the special WWM instruction, this means that this is a
177   // perfect elimination order, so we can never do any better.
178   ReversePostOrderTraversal<MachineFunction*> RPOT(&MF);
179 
180   for (MachineBasicBlock *MBB : RPOT) {
181     bool InWWM = false;
182     for (MachineInstr &MI : *MBB) {
183       if (MI.getOpcode() == AMDGPU::V_SET_INACTIVE_B32 ||
184           MI.getOpcode() == AMDGPU::V_SET_INACTIVE_B64)
185         RegsAssigned |= processDef(MI.getOperand(0));
186 
187       if (MI.getOpcode() == AMDGPU::ENTER_WWM) {
188         LLVM_DEBUG(dbgs() << "entering WWM region: " << MI << "\n");
189         InWWM = true;
190         continue;
191       }
192 
193       if (MI.getOpcode() == AMDGPU::EXIT_WWM) {
194         LLVM_DEBUG(dbgs() << "exiting WWM region: " << MI << "\n");
195         InWWM = false;
196       }
197 
198       if (!InWWM)
199         continue;
200 
201       LLVM_DEBUG(dbgs() << "processing " << MI << "\n");
202 
203       for (MachineOperand &DefOpnd : MI.defs()) {
204         RegsAssigned |= processDef(DefOpnd);
205       }
206     }
207   }
208 
209   if (!RegsAssigned)
210     return false;
211 
212   rewriteRegs(MF);
213   return true;
214 }
215