1 //=======- GCNDPPCombine.cpp - optimization for DPP instructions ---==========//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 // The pass combines V_MOV_B32_dpp instruction with its VALU uses as a DPP src0
9 // operand. If any of the use instruction cannot be combined with the mov the
10 // whole sequence is reverted.
11 //
12 // $old = ...
13 // $dpp_value = V_MOV_B32_dpp $old, $vgpr_to_be_read_from_other_lane,
14 //                            dpp_controls..., $row_mask, $bank_mask, $bound_ctrl
15 // $res = VALU $dpp_value [, src1]
16 //
17 // to
18 //
19 // $res = VALU_DPP $combined_old, $vgpr_to_be_read_from_other_lane, [src1,]
20 //                 dpp_controls..., $row_mask, $bank_mask, $combined_bound_ctrl
21 //
22 // Combining rules :
23 //
24 // if $row_mask and $bank_mask are fully enabled (0xF) and
25 //    $bound_ctrl==DPP_BOUND_ZERO or $old==0
26 // -> $combined_old = undef,
27 //    $combined_bound_ctrl = DPP_BOUND_ZERO
28 //
29 // if the VALU op is binary and
30 //    $bound_ctrl==DPP_BOUND_OFF and
31 //    $old==identity value (immediate) for the VALU op
32 // -> $combined_old = src1,
33 //    $combined_bound_ctrl = DPP_BOUND_OFF
34 //
35 // Otherwise cancel.
36 //
37 // The mov_dpp instruction should reside in the same BB as all its uses
38 //===----------------------------------------------------------------------===//
39 
40 #include "AMDGPU.h"
41 #include "GCNSubtarget.h"
42 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
43 #include "llvm/ADT/Statistic.h"
44 #include "llvm/CodeGen/MachineFunctionPass.h"
45 
46 using namespace llvm;
47 
48 #define DEBUG_TYPE "gcn-dpp-combine"
49 
50 STATISTIC(NumDPPMovsCombined, "Number of DPP moves combined.");
51 
52 namespace {
53 
54 class GCNDPPCombine : public MachineFunctionPass {
55   MachineRegisterInfo *MRI;
56   const SIInstrInfo *TII;
57   const GCNSubtarget *ST;
58 
59   using RegSubRegPair = TargetInstrInfo::RegSubRegPair;
60 
61   MachineOperand *getOldOpndValue(MachineOperand &OldOpnd) const;
62 
63   MachineInstr *createDPPInst(MachineInstr &OrigMI, MachineInstr &MovMI,
64                               RegSubRegPair CombOldVGPR,
65                               MachineOperand *OldOpnd, bool CombBCZ,
66                               bool IsShrinkable) const;
67 
68   MachineInstr *createDPPInst(MachineInstr &OrigMI, MachineInstr &MovMI,
69                               RegSubRegPair CombOldVGPR, bool CombBCZ,
70                               bool IsShrinkable) const;
71 
72   bool hasNoImmOrEqual(MachineInstr &MI,
73                        unsigned OpndName,
74                        int64_t Value,
75                        int64_t Mask = -1) const;
76 
77   bool combineDPPMov(MachineInstr &MI) const;
78 
79 public:
80   static char ID;
81 
82   GCNDPPCombine() : MachineFunctionPass(ID) {
83     initializeGCNDPPCombinePass(*PassRegistry::getPassRegistry());
84   }
85 
86   bool runOnMachineFunction(MachineFunction &MF) override;
87 
88   StringRef getPassName() const override { return "GCN DPP Combine"; }
89 
90   void getAnalysisUsage(AnalysisUsage &AU) const override {
91     AU.setPreservesCFG();
92     MachineFunctionPass::getAnalysisUsage(AU);
93   }
94 
95   MachineFunctionProperties getRequiredProperties() const override {
96     return MachineFunctionProperties()
97       .set(MachineFunctionProperties::Property::IsSSA);
98   }
99 
100 private:
101   int getDPPOp(unsigned Op, bool IsShrinkable) const;
102   bool isShrinkable(MachineInstr &OrigMI, unsigned OrigOp) const;
103 };
104 
105 } // end anonymous namespace
106 
107 INITIALIZE_PASS(GCNDPPCombine, DEBUG_TYPE, "GCN DPP Combine", false, false)
108 
109 char GCNDPPCombine::ID = 0;
110 
111 char &llvm::GCNDPPCombineID = GCNDPPCombine::ID;
112 
113 FunctionPass *llvm::createGCNDPPCombinePass() {
114   return new GCNDPPCombine();
115 }
116 
117 bool GCNDPPCombine::isShrinkable(MachineInstr &OrigMI, unsigned OrigOp) const {
118   if (!TII->isVOP3(OrigOp)) {
119     return false;
120   }
121   if (!TII->hasVALU32BitEncoding(OrigOp)) {
122     LLVM_DEBUG(dbgs() << "  Inst hasn't e32 equivalent\n");
123     return false;
124   }
125   // check if other than abs|neg modifiers are set (opsel for example)
126   const int64_t Mask = ~(SISrcMods::ABS | SISrcMods::NEG);
127   if (!hasNoImmOrEqual(OrigMI, AMDGPU::OpName::src0_modifiers, 0, Mask) ||
128       !hasNoImmOrEqual(OrigMI, AMDGPU::OpName::src1_modifiers, 0, Mask) ||
129       !hasNoImmOrEqual(OrigMI, AMDGPU::OpName::clamp, 0) ||
130       !hasNoImmOrEqual(OrigMI, AMDGPU::OpName::omod, 0)) {
131     LLVM_DEBUG(dbgs() << "  Inst has non-default modifiers\n");
132     return false;
133   }
134   return true;
135 }
136 
137 int GCNDPPCombine::getDPPOp(unsigned Op, bool IsShrinkable) const {
138   auto DPP32 = AMDGPU::getDPPOp32(Op);
139   if (IsShrinkable) {
140     assert(DPP32 == -1);
141     auto E32 = AMDGPU::getVOPe32(Op);
142     DPP32 = (E32 == -1) ? -1 : AMDGPU::getDPPOp32(E32);
143   }
144   return (DPP32 == -1 || TII->pseudoToMCOpcode(DPP32) == -1) ? -1 : DPP32;
145 }
146 
147 // tracks the register operand definition and returns:
148 //   1. immediate operand used to initialize the register if found
149 //   2. nullptr if the register operand is undef
150 //   3. the operand itself otherwise
151 MachineOperand *GCNDPPCombine::getOldOpndValue(MachineOperand &OldOpnd) const {
152   auto *Def = getVRegSubRegDef(getRegSubRegPair(OldOpnd), *MRI);
153   if (!Def)
154     return nullptr;
155 
156   switch(Def->getOpcode()) {
157   default: break;
158   case AMDGPU::IMPLICIT_DEF:
159     return nullptr;
160   case AMDGPU::COPY:
161   case AMDGPU::V_MOV_B32_e32:
162   case AMDGPU::V_MOV_B64_PSEUDO: {
163     auto &Op1 = Def->getOperand(1);
164     if (Op1.isImm())
165       return &Op1;
166     break;
167   }
168   }
169   return &OldOpnd;
170 }
171 
172 MachineInstr *GCNDPPCombine::createDPPInst(MachineInstr &OrigMI,
173                                            MachineInstr &MovMI,
174                                            RegSubRegPair CombOldVGPR,
175                                            bool CombBCZ,
176                                            bool IsShrinkable) const {
177   assert(MovMI.getOpcode() == AMDGPU::V_MOV_B32_dpp ||
178          MovMI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);
179 
180   auto OrigOp = OrigMI.getOpcode();
181   auto DPPOp = getDPPOp(OrigOp, IsShrinkable);
182   if (DPPOp == -1) {
183     LLVM_DEBUG(dbgs() << "  failed: no DPP opcode\n");
184     return nullptr;
185   }
186 
187   auto DPPInst = BuildMI(*OrigMI.getParent(), OrigMI,
188                          OrigMI.getDebugLoc(), TII->get(DPPOp))
189     .setMIFlags(OrigMI.getFlags());
190 
191   bool Fail = false;
192   do {
193     auto *Dst = TII->getNamedOperand(OrigMI, AMDGPU::OpName::vdst);
194     assert(Dst);
195     DPPInst.add(*Dst);
196     int NumOperands = 1;
197 
198     const int OldIdx = AMDGPU::getNamedOperandIdx(DPPOp, AMDGPU::OpName::old);
199     if (OldIdx != -1) {
200       assert(OldIdx == NumOperands);
201       assert(isOfRegClass(
202           CombOldVGPR,
203           *MRI->getRegClass(
204               TII->getNamedOperand(MovMI, AMDGPU::OpName::vdst)->getReg()),
205           *MRI));
206       auto *Def = getVRegSubRegDef(CombOldVGPR, *MRI);
207       DPPInst.addReg(CombOldVGPR.Reg, Def ? 0 : RegState::Undef,
208                      CombOldVGPR.SubReg);
209       ++NumOperands;
210     } else {
211       // TODO: this discards MAC/FMA instructions for now, let's add it later
212       LLVM_DEBUG(dbgs() << "  failed: no old operand in DPP instruction,"
213                            " TBD\n");
214       Fail = true;
215       break;
216     }
217 
218     if (auto *Mod0 = TII->getNamedOperand(OrigMI,
219                                           AMDGPU::OpName::src0_modifiers)) {
220       assert(NumOperands == AMDGPU::getNamedOperandIdx(DPPOp,
221                                           AMDGPU::OpName::src0_modifiers));
222       assert(0LL == (Mod0->getImm() & ~(SISrcMods::ABS | SISrcMods::NEG)));
223       DPPInst.addImm(Mod0->getImm());
224       ++NumOperands;
225     } else if (AMDGPU::getNamedOperandIdx(DPPOp,
226                    AMDGPU::OpName::src0_modifiers) != -1) {
227       DPPInst.addImm(0);
228       ++NumOperands;
229     }
230     auto *Src0 = TII->getNamedOperand(MovMI, AMDGPU::OpName::src0);
231     assert(Src0);
232     if (!TII->isOperandLegal(*DPPInst.getInstr(), NumOperands, Src0)) {
233       LLVM_DEBUG(dbgs() << "  failed: src0 is illegal\n");
234       Fail = true;
235       break;
236     }
237     DPPInst.add(*Src0);
238     DPPInst->getOperand(NumOperands).setIsKill(false);
239     ++NumOperands;
240 
241     if (auto *Mod1 = TII->getNamedOperand(OrigMI,
242                                           AMDGPU::OpName::src1_modifiers)) {
243       assert(NumOperands == AMDGPU::getNamedOperandIdx(DPPOp,
244                                           AMDGPU::OpName::src1_modifiers));
245       assert(0LL == (Mod1->getImm() & ~(SISrcMods::ABS | SISrcMods::NEG)));
246       DPPInst.addImm(Mod1->getImm());
247       ++NumOperands;
248     } else if (AMDGPU::getNamedOperandIdx(DPPOp,
249                    AMDGPU::OpName::src1_modifiers) != -1) {
250       DPPInst.addImm(0);
251       ++NumOperands;
252     }
253     if (auto *Src1 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src1)) {
254       if (!TII->isOperandLegal(*DPPInst.getInstr(), NumOperands, Src1)) {
255         LLVM_DEBUG(dbgs() << "  failed: src1 is illegal\n");
256         Fail = true;
257         break;
258       }
259       DPPInst.add(*Src1);
260       ++NumOperands;
261     }
262 
263     if (auto *Src2 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src2)) {
264       if (!TII->getNamedOperand(*DPPInst.getInstr(), AMDGPU::OpName::src2) ||
265           !TII->isOperandLegal(*DPPInst.getInstr(), NumOperands, Src2)) {
266         LLVM_DEBUG(dbgs() << "  failed: src2 is illegal\n");
267         Fail = true;
268         break;
269       }
270       DPPInst.add(*Src2);
271     }
272 
273     DPPInst.add(*TII->getNamedOperand(MovMI, AMDGPU::OpName::dpp_ctrl));
274     DPPInst.add(*TII->getNamedOperand(MovMI, AMDGPU::OpName::row_mask));
275     DPPInst.add(*TII->getNamedOperand(MovMI, AMDGPU::OpName::bank_mask));
276     DPPInst.addImm(CombBCZ ? 1 : 0);
277   } while (false);
278 
279   if (Fail) {
280     DPPInst.getInstr()->eraseFromParent();
281     return nullptr;
282   }
283   LLVM_DEBUG(dbgs() << "  combined:  " << *DPPInst.getInstr());
284   return DPPInst.getInstr();
285 }
286 
287 static bool isIdentityValue(unsigned OrigMIOp, MachineOperand *OldOpnd) {
288   assert(OldOpnd->isImm());
289   switch (OrigMIOp) {
290   default: break;
291   case AMDGPU::V_ADD_U32_e32:
292   case AMDGPU::V_ADD_U32_e64:
293   case AMDGPU::V_ADD_CO_U32_e32:
294   case AMDGPU::V_ADD_CO_U32_e64:
295   case AMDGPU::V_OR_B32_e32:
296   case AMDGPU::V_OR_B32_e64:
297   case AMDGPU::V_SUBREV_U32_e32:
298   case AMDGPU::V_SUBREV_U32_e64:
299   case AMDGPU::V_SUBREV_CO_U32_e32:
300   case AMDGPU::V_SUBREV_CO_U32_e64:
301   case AMDGPU::V_MAX_U32_e32:
302   case AMDGPU::V_MAX_U32_e64:
303   case AMDGPU::V_XOR_B32_e32:
304   case AMDGPU::V_XOR_B32_e64:
305     if (OldOpnd->getImm() == 0)
306       return true;
307     break;
308   case AMDGPU::V_AND_B32_e32:
309   case AMDGPU::V_AND_B32_e64:
310   case AMDGPU::V_MIN_U32_e32:
311   case AMDGPU::V_MIN_U32_e64:
312     if (static_cast<uint32_t>(OldOpnd->getImm()) ==
313         std::numeric_limits<uint32_t>::max())
314       return true;
315     break;
316   case AMDGPU::V_MIN_I32_e32:
317   case AMDGPU::V_MIN_I32_e64:
318     if (static_cast<int32_t>(OldOpnd->getImm()) ==
319         std::numeric_limits<int32_t>::max())
320       return true;
321     break;
322   case AMDGPU::V_MAX_I32_e32:
323   case AMDGPU::V_MAX_I32_e64:
324     if (static_cast<int32_t>(OldOpnd->getImm()) ==
325         std::numeric_limits<int32_t>::min())
326       return true;
327     break;
328   case AMDGPU::V_MUL_I32_I24_e32:
329   case AMDGPU::V_MUL_I32_I24_e64:
330   case AMDGPU::V_MUL_U32_U24_e32:
331   case AMDGPU::V_MUL_U32_U24_e64:
332     if (OldOpnd->getImm() == 1)
333       return true;
334     break;
335   }
336   return false;
337 }
338 
339 MachineInstr *GCNDPPCombine::createDPPInst(
340     MachineInstr &OrigMI, MachineInstr &MovMI, RegSubRegPair CombOldVGPR,
341     MachineOperand *OldOpndValue, bool CombBCZ, bool IsShrinkable) const {
342   assert(CombOldVGPR.Reg);
343   if (!CombBCZ && OldOpndValue && OldOpndValue->isImm()) {
344     auto *Src1 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src1);
345     if (!Src1 || !Src1->isReg()) {
346       LLVM_DEBUG(dbgs() << "  failed: no src1 or it isn't a register\n");
347       return nullptr;
348     }
349     if (!isIdentityValue(OrigMI.getOpcode(), OldOpndValue)) {
350       LLVM_DEBUG(dbgs() << "  failed: old immediate isn't an identity\n");
351       return nullptr;
352     }
353     CombOldVGPR = getRegSubRegPair(*Src1);
354     auto MovDst = TII->getNamedOperand(MovMI, AMDGPU::OpName::vdst);
355     const TargetRegisterClass *RC = MRI->getRegClass(MovDst->getReg());
356     if (!isOfRegClass(CombOldVGPR, *RC, *MRI)) {
357       LLVM_DEBUG(dbgs() << "  failed: src1 has wrong register class\n");
358       return nullptr;
359     }
360   }
361   return createDPPInst(OrigMI, MovMI, CombOldVGPR, CombBCZ, IsShrinkable);
362 }
363 
364 // returns true if MI doesn't have OpndName immediate operand or the
365 // operand has Value
366 bool GCNDPPCombine::hasNoImmOrEqual(MachineInstr &MI, unsigned OpndName,
367                                     int64_t Value, int64_t Mask) const {
368   auto *Imm = TII->getNamedOperand(MI, OpndName);
369   if (!Imm)
370     return true;
371 
372   assert(Imm->isImm());
373   return (Imm->getImm() & Mask) == Value;
374 }
375 
376 bool GCNDPPCombine::combineDPPMov(MachineInstr &MovMI) const {
377   assert(MovMI.getOpcode() == AMDGPU::V_MOV_B32_dpp ||
378          MovMI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);
379   LLVM_DEBUG(dbgs() << "\nDPP combine: " << MovMI);
380 
381   auto *DstOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::vdst);
382   assert(DstOpnd && DstOpnd->isReg());
383   auto DPPMovReg = DstOpnd->getReg();
384   if (DPPMovReg.isPhysical()) {
385     LLVM_DEBUG(dbgs() << "  failed: dpp move writes physreg\n");
386     return false;
387   }
388   if (execMayBeModifiedBeforeAnyUse(*MRI, DPPMovReg, MovMI)) {
389     LLVM_DEBUG(dbgs() << "  failed: EXEC mask should remain the same"
390                          " for all uses\n");
391     return false;
392   }
393 
394   if (MovMI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO) {
395     auto *DppCtrl = TII->getNamedOperand(MovMI, AMDGPU::OpName::dpp_ctrl);
396     assert(DppCtrl && DppCtrl->isImm());
397     if (!AMDGPU::isLegal64BitDPPControl(DppCtrl->getImm())) {
398       LLVM_DEBUG(dbgs() << "  failed: 64 bit dpp move uses unsupported"
399                            " control value\n");
400       // Let it split, then control may become legal.
401       return false;
402     }
403   }
404 
405   auto *RowMaskOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::row_mask);
406   assert(RowMaskOpnd && RowMaskOpnd->isImm());
407   auto *BankMaskOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::bank_mask);
408   assert(BankMaskOpnd && BankMaskOpnd->isImm());
409   const bool MaskAllLanes = RowMaskOpnd->getImm() == 0xF &&
410                             BankMaskOpnd->getImm() == 0xF;
411 
412   auto *BCZOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::bound_ctrl);
413   assert(BCZOpnd && BCZOpnd->isImm());
414   bool BoundCtrlZero = BCZOpnd->getImm();
415 
416   auto *OldOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::old);
417   auto *SrcOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::src0);
418   assert(OldOpnd && OldOpnd->isReg());
419   assert(SrcOpnd && SrcOpnd->isReg());
420   if (OldOpnd->getReg().isPhysical() || SrcOpnd->getReg().isPhysical()) {
421     LLVM_DEBUG(dbgs() << "  failed: dpp move reads physreg\n");
422     return false;
423   }
424 
425   auto * const OldOpndValue = getOldOpndValue(*OldOpnd);
426   // OldOpndValue is either undef (IMPLICIT_DEF) or immediate or something else
427   // We could use: assert(!OldOpndValue || OldOpndValue->isImm())
428   // but the third option is used to distinguish undef from non-immediate
429   // to reuse IMPLICIT_DEF instruction later
430   assert(!OldOpndValue || OldOpndValue->isImm() || OldOpndValue == OldOpnd);
431 
432   bool CombBCZ = false;
433 
434   if (MaskAllLanes && BoundCtrlZero) { // [1]
435     CombBCZ = true;
436   } else {
437     if (!OldOpndValue || !OldOpndValue->isImm()) {
438       LLVM_DEBUG(dbgs() << "  failed: the DPP mov isn't combinable\n");
439       return false;
440     }
441 
442     if (OldOpndValue->getParent()->getParent() != MovMI.getParent()) {
443       LLVM_DEBUG(dbgs() <<
444         "  failed: old reg def and mov should be in the same BB\n");
445       return false;
446     }
447 
448     if (OldOpndValue->getImm() == 0) {
449       if (MaskAllLanes) {
450         assert(!BoundCtrlZero); // by check [1]
451         CombBCZ = true;
452       }
453     } else if (BoundCtrlZero) {
454       assert(!MaskAllLanes); // by check [1]
455       LLVM_DEBUG(dbgs() <<
456         "  failed: old!=0 and bctrl:0 and not all lanes isn't combinable\n");
457       return false;
458     }
459   }
460 
461   LLVM_DEBUG(dbgs() << "  old=";
462     if (!OldOpndValue)
463       dbgs() << "undef";
464     else
465       dbgs() << *OldOpndValue;
466     dbgs() << ", bound_ctrl=" << CombBCZ << '\n');
467 
468   SmallVector<MachineInstr*, 4> OrigMIs, DPPMIs;
469   DenseMap<MachineInstr*, SmallVector<unsigned, 4>> RegSeqWithOpNos;
470   auto CombOldVGPR = getRegSubRegPair(*OldOpnd);
471   // try to reuse previous old reg if its undefined (IMPLICIT_DEF)
472   if (CombBCZ && OldOpndValue) { // CombOldVGPR should be undef
473     const TargetRegisterClass *RC = MRI->getRegClass(DPPMovReg);
474     CombOldVGPR = RegSubRegPair(
475       MRI->createVirtualRegister(RC));
476     auto UndefInst = BuildMI(*MovMI.getParent(), MovMI, MovMI.getDebugLoc(),
477                              TII->get(AMDGPU::IMPLICIT_DEF), CombOldVGPR.Reg);
478     DPPMIs.push_back(UndefInst.getInstr());
479   }
480 
481   OrigMIs.push_back(&MovMI);
482   bool Rollback = true;
483   SmallVector<MachineOperand*, 16> Uses;
484 
485   for (auto &Use : MRI->use_nodbg_operands(DPPMovReg)) {
486     Uses.push_back(&Use);
487   }
488 
489   while (!Uses.empty()) {
490     MachineOperand *Use = Uses.pop_back_val();
491     Rollback = true;
492 
493     auto &OrigMI = *Use->getParent();
494     LLVM_DEBUG(dbgs() << "  try: " << OrigMI);
495 
496     auto OrigOp = OrigMI.getOpcode();
497     if (OrigOp == AMDGPU::REG_SEQUENCE) {
498       Register FwdReg = OrigMI.getOperand(0).getReg();
499       unsigned FwdSubReg = 0;
500 
501       if (execMayBeModifiedBeforeAnyUse(*MRI, FwdReg, OrigMI)) {
502         LLVM_DEBUG(dbgs() << "  failed: EXEC mask should remain the same"
503                              " for all uses\n");
504         break;
505       }
506 
507       unsigned OpNo, E = OrigMI.getNumOperands();
508       for (OpNo = 1; OpNo < E; OpNo += 2) {
509         if (OrigMI.getOperand(OpNo).getReg() == DPPMovReg) {
510           FwdSubReg = OrigMI.getOperand(OpNo + 1).getImm();
511           break;
512         }
513       }
514 
515       if (!FwdSubReg)
516         break;
517 
518       for (auto &Op : MRI->use_nodbg_operands(FwdReg)) {
519         if (Op.getSubReg() == FwdSubReg)
520           Uses.push_back(&Op);
521       }
522       RegSeqWithOpNos[&OrigMI].push_back(OpNo);
523       continue;
524     }
525 
526     bool IsShrinkable = isShrinkable(OrigMI, OrigOp);
527     if (!(IsShrinkable || TII->isVOP1(OrigOp) || TII->isVOP2(OrigOp))) {
528       LLVM_DEBUG(dbgs() << "  failed: not VOP1/2/3\n");
529       break;
530     }
531 
532     auto *Src0 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src0);
533     auto *Src1 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src1);
534     if (Use != Src0 && !(Use == Src1 && OrigMI.isCommutable())) { // [1]
535       LLVM_DEBUG(dbgs() << "  failed: no suitable operands\n");
536       break;
537     }
538 
539     assert(Src0 && "Src1 without Src0?");
540     if (Src1 && Src1->isIdenticalTo(*Src0)) {
541       assert(Src1->isReg());
542       LLVM_DEBUG(
543           dbgs()
544           << "  " << OrigMI
545           << "  failed: DPP register is used more than once per instruction\n");
546       break;
547     }
548 
549     LLVM_DEBUG(dbgs() << "  combining: " << OrigMI);
550     if (Use == Src0) {
551       if (auto *DPPInst = createDPPInst(OrigMI, MovMI, CombOldVGPR,
552                                         OldOpndValue, CombBCZ, IsShrinkable)) {
553         DPPMIs.push_back(DPPInst);
554         Rollback = false;
555       }
556     } else {
557       assert(Use == Src1 && OrigMI.isCommutable()); // by check [1]
558       auto *BB = OrigMI.getParent();
559       auto *NewMI = BB->getParent()->CloneMachineInstr(&OrigMI);
560       BB->insert(OrigMI, NewMI);
561       if (TII->commuteInstruction(*NewMI)) {
562         LLVM_DEBUG(dbgs() << "  commuted:  " << *NewMI);
563         if (auto *DPPInst =
564                 createDPPInst(*NewMI, MovMI, CombOldVGPR, OldOpndValue, CombBCZ,
565                               IsShrinkable)) {
566           DPPMIs.push_back(DPPInst);
567           Rollback = false;
568         }
569       } else
570         LLVM_DEBUG(dbgs() << "  failed: cannot be commuted\n");
571       NewMI->eraseFromParent();
572     }
573     if (Rollback)
574       break;
575     OrigMIs.push_back(&OrigMI);
576   }
577 
578   Rollback |= !Uses.empty();
579 
580   for (auto *MI : *(Rollback? &DPPMIs : &OrigMIs))
581     MI->eraseFromParent();
582 
583   if (!Rollback) {
584     for (auto &S : RegSeqWithOpNos) {
585       if (MRI->use_nodbg_empty(S.first->getOperand(0).getReg())) {
586         S.first->eraseFromParent();
587         continue;
588       }
589       while (!S.second.empty())
590         S.first->getOperand(S.second.pop_back_val()).setIsUndef(true);
591     }
592   }
593 
594   return !Rollback;
595 }
596 
597 bool GCNDPPCombine::runOnMachineFunction(MachineFunction &MF) {
598   ST = &MF.getSubtarget<GCNSubtarget>();
599   if (!ST->hasDPP() || skipFunction(MF.getFunction()))
600     return false;
601 
602   MRI = &MF.getRegInfo();
603   TII = ST->getInstrInfo();
604 
605   bool Changed = false;
606   for (auto &MBB : MF) {
607     for (auto I = MBB.rbegin(), E = MBB.rend(); I != E;) {
608       auto &MI = *I++;
609       if (MI.getOpcode() == AMDGPU::V_MOV_B32_dpp && combineDPPMov(MI)) {
610         Changed = true;
611         ++NumDPPMovsCombined;
612       } else if (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO) {
613         if (ST->has64BitDPP() && combineDPPMov(MI)) {
614           Changed = true;
615           ++NumDPPMovsCombined;
616         } else {
617           auto Split = TII->expandMovDPP64(MI);
618           for (auto M : { Split.first, Split.second }) {
619             if (M && combineDPPMov(*M))
620               ++NumDPPMovsCombined;
621           }
622           Changed = true;
623         }
624       }
625     }
626   }
627   return Changed;
628 }
629