1 //=======- GCNDPPCombine.cpp - optimization for DPP instructions ---==========//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 // The pass combines V_MOV_B32_dpp instruction with its VALU uses as a DPP src0
9 // operand.If any of the use instruction cannot be combined with the mov the
10 // whole sequence is reverted.
11 //
12 // $old = ...
13 // $dpp_value = V_MOV_B32_dpp $old, $vgpr_to_be_read_from_other_lane,
14 //                            dpp_controls..., $bound_ctrl
15 // $res = VALU $dpp_value, ...
16 //
17 // to
18 //
19 // $res = VALU_DPP $folded_old, $vgpr_to_be_read_from_other_lane, ...,
20 //                 dpp_controls..., $folded_bound_ctrl
21 //
22 // Combining rules :
23 //
24 // $bound_ctrl is DPP_BOUND_ZERO, $old is any
25 // $bound_ctrl is DPP_BOUND_OFF, $old is 0
26 //
27 // ->$folded_old = undef, $folded_bound_ctrl = DPP_BOUND_ZERO
28 // $bound_ctrl is DPP_BOUND_OFF, $old is undef
29 //
30 // ->$folded_old = undef, $folded_bound_ctrl = DPP_BOUND_OFF
31 // $bound_ctrl is DPP_BOUND_OFF, $old is foldable
32 //
33 // ->$folded_old = folded value, $folded_bound_ctrl = DPP_BOUND_OFF
34 //===----------------------------------------------------------------------===//
35 
36 #include "AMDGPU.h"
37 #include "AMDGPUSubtarget.h"
38 #include "SIInstrInfo.h"
39 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
40 #include "llvm/ADT/SmallVector.h"
41 #include "llvm/ADT/Statistic.h"
42 #include "llvm/CodeGen/MachineBasicBlock.h"
43 #include "llvm/CodeGen/MachineFunction.h"
44 #include "llvm/CodeGen/MachineFunctionPass.h"
45 #include "llvm/CodeGen/MachineInstr.h"
46 #include "llvm/CodeGen/MachineInstrBuilder.h"
47 #include "llvm/CodeGen/MachineOperand.h"
48 #include "llvm/CodeGen/MachineRegisterInfo.h"
49 #include "llvm/CodeGen/TargetRegisterInfo.h"
50 #include "llvm/Pass.h"
51 #include <cassert>
52 
53 using namespace llvm;
54 
55 #define DEBUG_TYPE "gcn-dpp-combine"
56 
57 STATISTIC(NumDPPMovsCombined, "Number of DPP moves combined.");
58 
59 namespace {
60 
61 class GCNDPPCombine : public MachineFunctionPass {
62   MachineRegisterInfo *MRI;
63   const SIInstrInfo *TII;
64 
65   using RegSubRegPair = TargetInstrInfo::RegSubRegPair;
66 
67   MachineOperand *getOldOpndValue(MachineOperand &OldOpnd) const;
68 
69   RegSubRegPair foldOldOpnd(MachineInstr &OrigMI,
70                             RegSubRegPair OldOpndVGPR,
71                             MachineOperand &OldOpndValue) const;
72 
73   MachineInstr *createDPPInst(MachineInstr &OrigMI,
74                               MachineInstr &MovMI,
75                               RegSubRegPair OldOpndVGPR,
76                               MachineOperand *OldOpnd,
77                               bool BoundCtrlZero) const;
78 
79   MachineInstr *createDPPInst(MachineInstr &OrigMI,
80                               MachineInstr &MovMI,
81                               RegSubRegPair OldOpndVGPR,
82                               bool BoundCtrlZero) const;
83 
84   bool hasNoImmOrEqual(MachineInstr &MI,
85                        unsigned OpndName,
86                        int64_t Value,
87                        int64_t Mask = -1) const;
88 
89   bool combineDPPMov(MachineInstr &MI) const;
90 
91 public:
92   static char ID;
93 
94   GCNDPPCombine() : MachineFunctionPass(ID) {
95     initializeGCNDPPCombinePass(*PassRegistry::getPassRegistry());
96   }
97 
98   bool runOnMachineFunction(MachineFunction &MF) override;
99 
100   StringRef getPassName() const override { return "GCN DPP Combine"; }
101 
102   void getAnalysisUsage(AnalysisUsage &AU) const override {
103     AU.setPreservesCFG();
104     MachineFunctionPass::getAnalysisUsage(AU);
105   }
106 };
107 
108 } // end anonymous namespace
109 
110 INITIALIZE_PASS(GCNDPPCombine, DEBUG_TYPE, "GCN DPP Combine", false, false)
111 
112 char GCNDPPCombine::ID = 0;
113 
114 char &llvm::GCNDPPCombineID = GCNDPPCombine::ID;
115 
116 FunctionPass *llvm::createGCNDPPCombinePass() {
117   return new GCNDPPCombine();
118 }
119 
120 static int getDPPOp(unsigned Op) {
121   auto DPP32 = AMDGPU::getDPPOp32(Op);
122   if (DPP32 != -1)
123     return DPP32;
124 
125   auto E32 = AMDGPU::getVOPe32(Op);
126   return E32 != -1 ? AMDGPU::getDPPOp32(E32) : -1;
127 }
128 
129 // tracks the register operand definition and returns:
130 //   1. immediate operand used to initialize the register if found
131 //   2. nullptr if the register operand is undef
132 //   3. the operand itself otherwise
133 MachineOperand *GCNDPPCombine::getOldOpndValue(MachineOperand &OldOpnd) const {
134   auto *Def = getVRegSubRegDef(getRegSubRegPair(OldOpnd), *MRI);
135   if (!Def)
136     return nullptr;
137 
138   switch(Def->getOpcode()) {
139   default: break;
140   case AMDGPU::IMPLICIT_DEF:
141     return nullptr;
142   case AMDGPU::COPY:
143   case AMDGPU::V_MOV_B32_e32: {
144     auto &Op1 = Def->getOperand(1);
145     if (Op1.isImm())
146       return &Op1;
147     break;
148   }
149   }
150   return &OldOpnd;
151 }
152 
153 MachineInstr *GCNDPPCombine::createDPPInst(MachineInstr &OrigMI,
154                                            MachineInstr &MovMI,
155                                            RegSubRegPair OldOpndVGPR,
156                                            bool BoundCtrlZero) const {
157   assert(MovMI.getOpcode() == AMDGPU::V_MOV_B32_dpp);
158   assert(TII->getNamedOperand(MovMI, AMDGPU::OpName::vdst)->getReg() ==
159          TII->getNamedOperand(OrigMI, AMDGPU::OpName::src0)->getReg());
160 
161   auto OrigOp = OrigMI.getOpcode();
162   auto DPPOp = getDPPOp(OrigOp);
163   if (DPPOp == -1) {
164     LLVM_DEBUG(dbgs() << "  failed: no DPP opcode\n");
165     return nullptr;
166   }
167 
168   auto DPPInst = BuildMI(*OrigMI.getParent(), OrigMI,
169                          OrigMI.getDebugLoc(), TII->get(DPPOp));
170   bool Fail = false;
171   do {
172     auto *Dst = TII->getNamedOperand(OrigMI, AMDGPU::OpName::vdst);
173     assert(Dst);
174     DPPInst.add(*Dst);
175     int NumOperands = 1;
176 
177     const int OldIdx = AMDGPU::getNamedOperandIdx(DPPOp, AMDGPU::OpName::old);
178     if (OldIdx != -1) {
179       assert(OldIdx == NumOperands);
180       assert(isOfRegClass(OldOpndVGPR, AMDGPU::VGPR_32RegClass, *MRI));
181       DPPInst.addReg(OldOpndVGPR.Reg, 0, OldOpndVGPR.SubReg);
182       ++NumOperands;
183     }
184 
185     if (auto *Mod0 = TII->getNamedOperand(OrigMI,
186                                           AMDGPU::OpName::src0_modifiers)) {
187       assert(NumOperands == AMDGPU::getNamedOperandIdx(DPPOp,
188                                           AMDGPU::OpName::src0_modifiers));
189       assert(0LL == (Mod0->getImm() & ~(SISrcMods::ABS | SISrcMods::NEG)));
190       DPPInst.addImm(Mod0->getImm());
191       ++NumOperands;
192     }
193     auto *Src0 = TII->getNamedOperand(MovMI, AMDGPU::OpName::src0);
194     assert(Src0);
195     if (!TII->isOperandLegal(*DPPInst.getInstr(), NumOperands, Src0)) {
196       LLVM_DEBUG(dbgs() << "  failed: src0 is illegal\n");
197       Fail = true;
198       break;
199     }
200     DPPInst.add(*Src0);
201     ++NumOperands;
202 
203     if (auto *Mod1 = TII->getNamedOperand(OrigMI,
204                                           AMDGPU::OpName::src1_modifiers)) {
205       assert(NumOperands == AMDGPU::getNamedOperandIdx(DPPOp,
206                                           AMDGPU::OpName::src1_modifiers));
207       assert(0LL == (Mod1->getImm() & ~(SISrcMods::ABS | SISrcMods::NEG)));
208       DPPInst.addImm(Mod1->getImm());
209       ++NumOperands;
210     }
211     if (auto *Src1 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src1)) {
212       if (!TII->isOperandLegal(*DPPInst.getInstr(), NumOperands, Src1)) {
213         LLVM_DEBUG(dbgs() << "  failed: src1 is illegal\n");
214         Fail = true;
215         break;
216       }
217       DPPInst.add(*Src1);
218       ++NumOperands;
219     }
220 
221     if (auto *Src2 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src2)) {
222       if (!TII->isOperandLegal(*DPPInst.getInstr(), NumOperands, Src2)) {
223         LLVM_DEBUG(dbgs() << "  failed: src2 is illegal\n");
224         Fail = true;
225         break;
226       }
227       DPPInst.add(*Src2);
228     }
229 
230     DPPInst.add(*TII->getNamedOperand(MovMI, AMDGPU::OpName::dpp_ctrl));
231     DPPInst.add(*TII->getNamedOperand(MovMI, AMDGPU::OpName::row_mask));
232     DPPInst.add(*TII->getNamedOperand(MovMI, AMDGPU::OpName::bank_mask));
233     DPPInst.addImm(BoundCtrlZero ? 1 : 0);
234   } while (false);
235 
236   if (Fail) {
237     DPPInst.getInstr()->eraseFromParent();
238     return nullptr;
239   }
240   LLVM_DEBUG(dbgs() << "  combined:  " << *DPPInst.getInstr());
241   return DPPInst.getInstr();
242 }
243 
244 GCNDPPCombine::RegSubRegPair
245 GCNDPPCombine::foldOldOpnd(MachineInstr &OrigMI,
246                            RegSubRegPair OldOpndVGPR,
247                            MachineOperand &OldOpndValue) const {
248   assert(OldOpndValue.isImm());
249   switch (OrigMI.getOpcode()) {
250   default: break;
251   case AMDGPU::V_MAX_U32_e32:
252     if (OldOpndValue.getImm() == std::numeric_limits<uint32_t>::max())
253       return OldOpndVGPR;
254     break;
255   case AMDGPU::V_MAX_I32_e32:
256     if (OldOpndValue.getImm() == std::numeric_limits<int32_t>::max())
257       return OldOpndVGPR;
258     break;
259   case AMDGPU::V_MIN_I32_e32:
260     if (OldOpndValue.getImm() == std::numeric_limits<int32_t>::min())
261       return OldOpndVGPR;
262     break;
263 
264   case AMDGPU::V_MUL_I32_I24_e32:
265   case AMDGPU::V_MUL_U32_U24_e32:
266     if (OldOpndValue.getImm() == 1) {
267       auto *Src1 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src1);
268       assert(Src1 && Src1->isReg());
269       return getRegSubRegPair(*Src1);
270     }
271     break;
272   }
273   return RegSubRegPair();
274 }
275 
276 // Cases to combine:
277 //  $bound_ctrl is DPP_BOUND_ZERO, $old is any
278 //  $bound_ctrl is DPP_BOUND_OFF, $old is 0
279 //  -> $old = undef, $bound_ctrl = DPP_BOUND_ZERO
280 
281 //  $bound_ctrl is DPP_BOUND_OFF, $old is undef
282 //  -> $old = undef, $bound_ctrl = DPP_BOUND_OFF
283 
284 //  $bound_ctrl is DPP_BOUND_OFF, $old is foldable
285 //  -> $old = folded value, $bound_ctrl = DPP_BOUND_OFF
286 
287 MachineInstr *GCNDPPCombine::createDPPInst(MachineInstr &OrigMI,
288                                            MachineInstr &MovMI,
289                                            RegSubRegPair OldOpndVGPR,
290                                            MachineOperand *OldOpndValue,
291                                            bool BoundCtrlZero) const {
292   assert(OldOpndVGPR.Reg);
293   if (!BoundCtrlZero && OldOpndValue) {
294     assert(OldOpndValue->isImm());
295     OldOpndVGPR = foldOldOpnd(OrigMI, OldOpndVGPR, *OldOpndValue);
296     if (!OldOpndVGPR.Reg) {
297       LLVM_DEBUG(dbgs() << "  failed: old immediate cannot be folded\n");
298       return nullptr;
299     }
300   }
301   return createDPPInst(OrigMI, MovMI, OldOpndVGPR, BoundCtrlZero);
302 }
303 
304 // returns true if MI doesn't have OpndName immediate operand or the
305 // operand has Value
306 bool GCNDPPCombine::hasNoImmOrEqual(MachineInstr &MI, unsigned OpndName,
307                                     int64_t Value, int64_t Mask) const {
308   auto *Imm = TII->getNamedOperand(MI, OpndName);
309   if (!Imm)
310     return true;
311 
312   assert(Imm->isImm());
313   return (Imm->getImm() & Mask) == Value;
314 }
315 
316 bool GCNDPPCombine::combineDPPMov(MachineInstr &MovMI) const {
317   assert(MovMI.getOpcode() == AMDGPU::V_MOV_B32_dpp);
318   auto *BCZOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::bound_ctrl);
319   assert(BCZOpnd && BCZOpnd->isImm());
320   bool BoundCtrlZero = 0 != BCZOpnd->getImm();
321 
322   LLVM_DEBUG(dbgs() << "\nDPP combine: " << MovMI);
323 
324   auto *OldOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::old);
325   assert(OldOpnd && OldOpnd->isReg());
326   auto OldOpndVGPR = getRegSubRegPair(*OldOpnd);
327   auto *OldOpndValue = getOldOpndValue(*OldOpnd);
328   assert(!OldOpndValue || OldOpndValue->isImm() || OldOpndValue == OldOpnd);
329   if (OldOpndValue) {
330     if (BoundCtrlZero) {
331       OldOpndVGPR.Reg = AMDGPU::NoRegister; // should be undef, ignore old opnd
332       OldOpndValue = nullptr;
333     } else {
334       if (!OldOpndValue->isImm()) {
335         LLVM_DEBUG(dbgs() << "  failed: old operand isn't an imm or undef\n");
336         return false;
337       }
338       if (OldOpndValue->getImm() == 0) {
339         OldOpndVGPR.Reg = AMDGPU::NoRegister; // should be undef
340         OldOpndValue = nullptr;
341         BoundCtrlZero = true;
342       }
343     }
344   }
345 
346   LLVM_DEBUG(dbgs() << "  old=";
347     if (!OldOpndValue)
348       dbgs() << "undef";
349     else
350       dbgs() << OldOpndValue->getImm();
351     dbgs() << ", bound_ctrl=" << BoundCtrlZero << '\n');
352 
353   std::vector<MachineInstr*> OrigMIs, DPPMIs;
354   if (!OldOpndVGPR.Reg) { // OldOpndVGPR = undef
355     OldOpndVGPR = RegSubRegPair(
356       MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass));
357     auto UndefInst = BuildMI(*MovMI.getParent(), MovMI, MovMI.getDebugLoc(),
358                              TII->get(AMDGPU::IMPLICIT_DEF), OldOpndVGPR.Reg);
359     DPPMIs.push_back(UndefInst.getInstr());
360   }
361 
362   OrigMIs.push_back(&MovMI);
363   bool Rollback = true;
364   for (auto &Use : MRI->use_nodbg_operands(
365        TII->getNamedOperand(MovMI, AMDGPU::OpName::vdst)->getReg())) {
366     Rollback = true;
367 
368     auto &OrigMI = *Use.getParent();
369     auto OrigOp = OrigMI.getOpcode();
370     if (TII->isVOP3(OrigOp)) {
371       if (!TII->hasVALU32BitEncoding(OrigOp)) {
372         LLVM_DEBUG(dbgs() << "  failed: VOP3 hasn't e32 equivalent\n");
373         break;
374       }
375       // check if other than abs|neg modifiers are set (opsel for example)
376       const int64_t Mask = ~(SISrcMods::ABS | SISrcMods::NEG);
377       if (!hasNoImmOrEqual(OrigMI, AMDGPU::OpName::src0_modifiers, 0, Mask) ||
378           !hasNoImmOrEqual(OrigMI, AMDGPU::OpName::src1_modifiers, 0, Mask) ||
379           !hasNoImmOrEqual(OrigMI, AMDGPU::OpName::clamp, 0) ||
380           !hasNoImmOrEqual(OrigMI, AMDGPU::OpName::omod, 0)) {
381         LLVM_DEBUG(dbgs() << "  failed: VOP3 has non-default modifiers\n");
382         break;
383       }
384     } else if (!TII->isVOP1(OrigOp) && !TII->isVOP2(OrigOp)) {
385       LLVM_DEBUG(dbgs() << "  failed: not VOP1/2/3\n");
386       break;
387     }
388 
389     LLVM_DEBUG(dbgs() << "  combining: " << OrigMI);
390     if (&Use == TII->getNamedOperand(OrigMI, AMDGPU::OpName::src0)) {
391       if (auto *DPPInst = createDPPInst(OrigMI, MovMI, OldOpndVGPR,
392                                         OldOpndValue, BoundCtrlZero)) {
393         DPPMIs.push_back(DPPInst);
394         Rollback = false;
395       }
396     } else if (OrigMI.isCommutable() &&
397                &Use == TII->getNamedOperand(OrigMI, AMDGPU::OpName::src1)) {
398       auto *BB = OrigMI.getParent();
399       auto *NewMI = BB->getParent()->CloneMachineInstr(&OrigMI);
400       BB->insert(OrigMI, NewMI);
401       if (TII->commuteInstruction(*NewMI)) {
402         LLVM_DEBUG(dbgs() << "  commuted:  " << *NewMI);
403         if (auto *DPPInst = createDPPInst(*NewMI, MovMI, OldOpndVGPR,
404                                           OldOpndValue, BoundCtrlZero)) {
405           DPPMIs.push_back(DPPInst);
406           Rollback = false;
407         }
408       } else
409         LLVM_DEBUG(dbgs() << "  failed: cannot be commuted\n");
410       NewMI->eraseFromParent();
411     } else
412       LLVM_DEBUG(dbgs() << "  failed: no suitable operands\n");
413     if (Rollback)
414       break;
415     OrigMIs.push_back(&OrigMI);
416   }
417 
418   for (auto *MI : *(Rollback? &DPPMIs : &OrigMIs))
419     MI->eraseFromParent();
420 
421   return !Rollback;
422 }
423 
424 bool GCNDPPCombine::runOnMachineFunction(MachineFunction &MF) {
425   auto &ST = MF.getSubtarget<GCNSubtarget>();
426   if (!ST.hasDPP() || skipFunction(MF.getFunction()))
427     return false;
428 
429   MRI = &MF.getRegInfo();
430   TII = ST.getInstrInfo();
431 
432   assert(MRI->isSSA() && "Must be run on SSA");
433 
434   bool Changed = false;
435   for (auto &MBB : MF) {
436     for (auto I = MBB.rbegin(), E = MBB.rend(); I != E;) {
437       auto &MI = *I++;
438       if (MI.getOpcode() == AMDGPU::V_MOV_B32_dpp && combineDPPMov(MI)) {
439         Changed = true;
440         ++NumDPPMovsCombined;
441       }
442     }
443   }
444   return Changed;
445 }
446