1 //===-- GCNHazardRecognizers.cpp - GCN Hazard Recognizer Impls ------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements hazard recognizers for scheduling on GCN processors.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "GCNHazardRecognizer.h"
14 #include "GCNSubtarget.h"
15 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
16 #include "llvm/CodeGen/MachineFunction.h"
17 #include "llvm/CodeGen/ScheduleDAG.h"
18 #include "llvm/Support/TargetParser.h"
19 
20 using namespace llvm;
21 
22 //===----------------------------------------------------------------------===//
23 // Hazard Recoginizer Implementation
24 //===----------------------------------------------------------------------===//
25 
26 GCNHazardRecognizer::GCNHazardRecognizer(const MachineFunction &MF) :
27   IsHazardRecognizerMode(false),
28   CurrCycleInstr(nullptr),
29   MF(MF),
30   ST(MF.getSubtarget<GCNSubtarget>()),
31   TII(*ST.getInstrInfo()),
32   TRI(TII.getRegisterInfo()),
33   ClauseUses(TRI.getNumRegUnits()),
34   ClauseDefs(TRI.getNumRegUnits()) {
35   MaxLookAhead = MF.getRegInfo().isPhysRegUsed(AMDGPU::AGPR0) ? 19 : 5;
36   TSchedModel.init(&ST);
37 }
38 
39 void GCNHazardRecognizer::Reset() {
40   EmittedInstrs.clear();
41 }
42 
43 void GCNHazardRecognizer::EmitInstruction(SUnit *SU) {
44   EmitInstruction(SU->getInstr());
45 }
46 
47 void GCNHazardRecognizer::EmitInstruction(MachineInstr *MI) {
48   CurrCycleInstr = MI;
49 }
50 
51 static bool isDivFMas(unsigned Opcode) {
52   return Opcode == AMDGPU::V_DIV_FMAS_F32_e64 || Opcode == AMDGPU::V_DIV_FMAS_F64_e64;
53 }
54 
55 static bool isSGetReg(unsigned Opcode) {
56   return Opcode == AMDGPU::S_GETREG_B32;
57 }
58 
59 static bool isSSetReg(unsigned Opcode) {
60   switch (Opcode) {
61   case AMDGPU::S_SETREG_B32:
62   case AMDGPU::S_SETREG_B32_mode:
63   case AMDGPU::S_SETREG_IMM32_B32:
64   case AMDGPU::S_SETREG_IMM32_B32_mode:
65     return true;
66   }
67   return false;
68 }
69 
70 static bool isRWLane(unsigned Opcode) {
71   return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32;
72 }
73 
74 static bool isRFE(unsigned Opcode) {
75   return Opcode == AMDGPU::S_RFE_B64;
76 }
77 
78 static bool isSMovRel(unsigned Opcode) {
79   switch (Opcode) {
80   case AMDGPU::S_MOVRELS_B32:
81   case AMDGPU::S_MOVRELS_B64:
82   case AMDGPU::S_MOVRELD_B32:
83   case AMDGPU::S_MOVRELD_B64:
84     return true;
85   default:
86     return false;
87   }
88 }
89 
90 static bool isDGEMM(unsigned Opcode) {
91   return Opcode == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
92          Opcode == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64 ||
93          Opcode == AMDGPU::V_MFMA_F64_16X16X4F64_e64 ||
94          Opcode == AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64;
95 }
96 
97 static bool isXDL(const GCNSubtarget &ST, const MachineInstr &MI) {
98   unsigned Opcode = MI.getOpcode();
99 
100   if (!SIInstrInfo::isMAI(MI) ||
101       isDGEMM(Opcode) ||
102       Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
103       Opcode == AMDGPU::V_ACCVGPR_READ_B32_e64)
104     return false;
105 
106   return true;
107 }
108 
109 static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII,
110                                     const MachineInstr &MI) {
111   if (TII.isAlwaysGDS(MI.getOpcode()))
112     return true;
113 
114   switch (MI.getOpcode()) {
115   case AMDGPU::S_SENDMSG:
116   case AMDGPU::S_SENDMSGHALT:
117   case AMDGPU::S_TTRACEDATA:
118     return true;
119   // These DS opcodes don't support GDS.
120   case AMDGPU::DS_NOP:
121   case AMDGPU::DS_PERMUTE_B32:
122   case AMDGPU::DS_BPERMUTE_B32:
123     return false;
124   default:
125     if (TII.isDS(MI.getOpcode())) {
126       int GDS = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
127                                            AMDGPU::OpName::gds);
128       if (MI.getOperand(GDS).getImm())
129         return true;
130     }
131     return false;
132   }
133 }
134 
135 static bool isPermlane(const MachineInstr &MI) {
136   unsigned Opcode = MI.getOpcode();
137   return Opcode == AMDGPU::V_PERMLANE16_B32_e64 ||
138          Opcode == AMDGPU::V_PERMLANEX16_B32_e64;
139 }
140 
141 static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr) {
142   const MachineOperand *RegOp = TII->getNamedOperand(RegInstr,
143                                                      AMDGPU::OpName::simm16);
144   return RegOp->getImm() & AMDGPU::Hwreg::ID_MASK_;
145 }
146 
147 ScheduleHazardRecognizer::HazardType
148 GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
149   MachineInstr *MI = SU->getInstr();
150   // If we are not in "HazardRecognizerMode" and therefore not being run from
151   // the scheduler, track possible stalls from hazards but don't insert noops.
152   auto HazardType = IsHazardRecognizerMode ? NoopHazard : Hazard;
153 
154   if (MI->isBundle())
155    return NoHazard;
156 
157   if (SIInstrInfo::isSMRD(*MI) && checkSMRDHazards(MI) > 0)
158     return HazardType;
159 
160   if (ST.hasNSAtoVMEMBug() && checkNSAtoVMEMHazard(MI) > 0)
161     return HazardType;
162 
163   if (checkFPAtomicToDenormModeHazard(MI) > 0)
164     return HazardType;
165 
166   if (ST.hasNoDataDepHazard())
167     return NoHazard;
168 
169   // FIXME: Should flat be considered vmem?
170   if ((SIInstrInfo::isVMEM(*MI) ||
171        SIInstrInfo::isFLAT(*MI))
172       && checkVMEMHazards(MI) > 0)
173     return HazardType;
174 
175   if (SIInstrInfo::isVALU(*MI) && checkVALUHazards(MI) > 0)
176     return HazardType;
177 
178   if (SIInstrInfo::isDPP(*MI) && checkDPPHazards(MI) > 0)
179     return HazardType;
180 
181   if (isDivFMas(MI->getOpcode()) && checkDivFMasHazards(MI) > 0)
182     return HazardType;
183 
184   if (isRWLane(MI->getOpcode()) && checkRWLaneHazards(MI) > 0)
185     return HazardType;
186 
187   if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) ||
188        SIInstrInfo::isFLAT(*MI) || SIInstrInfo::isDS(*MI) ||
189        SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0)
190     return HazardType;
191 
192   if (isSGetReg(MI->getOpcode()) && checkGetRegHazards(MI) > 0)
193     return HazardType;
194 
195   if (isSSetReg(MI->getOpcode()) && checkSetRegHazards(MI) > 0)
196     return HazardType;
197 
198   if (isRFE(MI->getOpcode()) && checkRFEHazards(MI) > 0)
199     return HazardType;
200 
201   if (ST.hasReadM0MovRelInterpHazard() &&
202       (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode())) &&
203       checkReadM0Hazards(MI) > 0)
204     return HazardType;
205 
206   if (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI) &&
207       checkReadM0Hazards(MI) > 0)
208     return HazardType;
209 
210   if (SIInstrInfo::isMAI(*MI) && checkMAIHazards(MI) > 0)
211     return HazardType;
212 
213   if ((SIInstrInfo::isVMEM(*MI) ||
214        SIInstrInfo::isFLAT(*MI) ||
215        SIInstrInfo::isDS(*MI)) && checkMAILdStHazards(MI) > 0)
216     return HazardType;
217 
218   if (MI->isInlineAsm() && checkInlineAsmHazards(MI) > 0)
219     return HazardType;
220 
221   return NoHazard;
222 }
223 
224 static void insertNoopsInBundle(MachineInstr *MI, const SIInstrInfo &TII,
225                                 unsigned Quantity) {
226   while (Quantity > 0) {
227     unsigned Arg = std::min(Quantity, 8u);
228     Quantity -= Arg;
229     BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII.get(AMDGPU::S_NOP))
230         .addImm(Arg - 1);
231   }
232 }
233 
234 void GCNHazardRecognizer::processBundle() {
235   MachineBasicBlock::instr_iterator MI = std::next(CurrCycleInstr->getIterator());
236   MachineBasicBlock::instr_iterator E = CurrCycleInstr->getParent()->instr_end();
237   // Check bundled MachineInstr's for hazards.
238   for (; MI != E && MI->isInsideBundle(); ++MI) {
239     CurrCycleInstr = &*MI;
240     unsigned WaitStates = PreEmitNoopsCommon(CurrCycleInstr);
241 
242     if (IsHazardRecognizerMode) {
243       fixHazards(CurrCycleInstr);
244 
245       insertNoopsInBundle(CurrCycleInstr, TII, WaitStates);
246     }
247 
248     // It’s unnecessary to track more than MaxLookAhead instructions. Since we
249     // include the bundled MI directly after, only add a maximum of
250     // (MaxLookAhead - 1) noops to EmittedInstrs.
251     for (unsigned i = 0, e = std::min(WaitStates, MaxLookAhead - 1); i < e; ++i)
252       EmittedInstrs.push_front(nullptr);
253 
254     EmittedInstrs.push_front(CurrCycleInstr);
255     EmittedInstrs.resize(MaxLookAhead);
256   }
257   CurrCycleInstr = nullptr;
258 }
259 
260 unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) {
261   IsHazardRecognizerMode = true;
262   CurrCycleInstr = MI;
263   unsigned W = PreEmitNoopsCommon(MI);
264   fixHazards(MI);
265   CurrCycleInstr = nullptr;
266   return W;
267 }
268 
269 unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) {
270   if (MI->isBundle())
271     return 0;
272 
273   int WaitStates = 0;
274 
275   if (SIInstrInfo::isSMRD(*MI))
276     return std::max(WaitStates, checkSMRDHazards(MI));
277 
278   if (ST.hasNSAtoVMEMBug())
279     WaitStates = std::max(WaitStates, checkNSAtoVMEMHazard(MI));
280 
281   WaitStates = std::max(WaitStates, checkFPAtomicToDenormModeHazard(MI));
282 
283   if (ST.hasNoDataDepHazard())
284     return WaitStates;
285 
286   if (SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isFLAT(*MI))
287     WaitStates = std::max(WaitStates, checkVMEMHazards(MI));
288 
289   if (SIInstrInfo::isVALU(*MI))
290     WaitStates = std::max(WaitStates, checkVALUHazards(MI));
291 
292   if (SIInstrInfo::isDPP(*MI))
293     WaitStates = std::max(WaitStates, checkDPPHazards(MI));
294 
295   if (isDivFMas(MI->getOpcode()))
296     WaitStates = std::max(WaitStates, checkDivFMasHazards(MI));
297 
298   if (isRWLane(MI->getOpcode()))
299     WaitStates = std::max(WaitStates, checkRWLaneHazards(MI));
300 
301   if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) ||
302        SIInstrInfo::isFLAT(*MI) || SIInstrInfo::isDS(*MI) ||
303        SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0)
304     WaitStates = std::max(WaitStates, checkMAIVALUHazards(MI));
305 
306   if (MI->isInlineAsm())
307     return std::max(WaitStates, checkInlineAsmHazards(MI));
308 
309   if (isSGetReg(MI->getOpcode()))
310     return std::max(WaitStates, checkGetRegHazards(MI));
311 
312   if (isSSetReg(MI->getOpcode()))
313     return std::max(WaitStates, checkSetRegHazards(MI));
314 
315   if (isRFE(MI->getOpcode()))
316     return std::max(WaitStates, checkRFEHazards(MI));
317 
318   if (ST.hasReadM0MovRelInterpHazard() && (TII.isVINTRP(*MI) ||
319                                            isSMovRel(MI->getOpcode())))
320     return std::max(WaitStates, checkReadM0Hazards(MI));
321 
322   if (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI))
323     return std::max(WaitStates, checkReadM0Hazards(MI));
324 
325   if (SIInstrInfo::isMAI(*MI))
326     return std::max(WaitStates, checkMAIHazards(MI));
327 
328   if (SIInstrInfo::isVMEM(*MI) ||
329       SIInstrInfo::isFLAT(*MI) ||
330       SIInstrInfo::isDS(*MI))
331     return std::max(WaitStates, checkMAILdStHazards(MI));
332 
333   return WaitStates;
334 }
335 
336 void GCNHazardRecognizer::EmitNoop() {
337   EmittedInstrs.push_front(nullptr);
338 }
339 
340 void GCNHazardRecognizer::AdvanceCycle() {
341   // When the scheduler detects a stall, it will call AdvanceCycle() without
342   // emitting any instructions.
343   if (!CurrCycleInstr) {
344     EmittedInstrs.push_front(nullptr);
345     return;
346   }
347 
348   // Do not track non-instructions which do not affect the wait states.
349   // If included, these instructions can lead to buffer overflow such that
350   // detectable hazards are missed.
351   if (CurrCycleInstr->isMetaInstruction()) {
352     CurrCycleInstr = nullptr;
353     return;
354   }
355 
356   if (CurrCycleInstr->isBundle()) {
357     processBundle();
358     return;
359   }
360 
361   unsigned NumWaitStates = TII.getNumWaitStates(*CurrCycleInstr);
362 
363   // Keep track of emitted instructions
364   EmittedInstrs.push_front(CurrCycleInstr);
365 
366   // Add a nullptr for each additional wait state after the first.  Make sure
367   // not to add more than getMaxLookAhead() items to the list, since we
368   // truncate the list to that size right after this loop.
369   for (unsigned i = 1, e = std::min(NumWaitStates, getMaxLookAhead());
370        i < e; ++i) {
371     EmittedInstrs.push_front(nullptr);
372   }
373 
374   // getMaxLookahead() is the largest number of wait states we will ever need
375   // to insert, so there is no point in keeping track of more than that many
376   // wait states.
377   EmittedInstrs.resize(getMaxLookAhead());
378 
379   CurrCycleInstr = nullptr;
380 }
381 
382 void GCNHazardRecognizer::RecedeCycle() {
383   llvm_unreachable("hazard recognizer does not support bottom-up scheduling.");
384 }
385 
386 //===----------------------------------------------------------------------===//
387 // Helper Functions
388 //===----------------------------------------------------------------------===//
389 
390 typedef function_ref<bool(MachineInstr *, int WaitStates)> IsExpiredFn;
391 
392 // Returns a minimum wait states since \p I walking all predecessors.
393 // Only scans until \p IsExpired does not return true.
394 // Can only be run in a hazard recognizer mode.
395 static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard,
396                               MachineBasicBlock *MBB,
397                               MachineBasicBlock::reverse_instr_iterator I,
398                               int WaitStates,
399                               IsExpiredFn IsExpired,
400                               DenseSet<const MachineBasicBlock *> &Visited) {
401   for (auto E = MBB->instr_rend(); I != E; ++I) {
402     // Don't add WaitStates for parent BUNDLE instructions.
403     if (I->isBundle())
404       continue;
405 
406     if (IsHazard(&*I))
407       return WaitStates;
408 
409     if (I->isInlineAsm() || I->isMetaInstruction())
410       continue;
411 
412     WaitStates += SIInstrInfo::getNumWaitStates(*I);
413 
414     if (IsExpired(&*I, WaitStates))
415       return std::numeric_limits<int>::max();
416   }
417 
418   int MinWaitStates = WaitStates;
419   bool Found = false;
420   for (MachineBasicBlock *Pred : MBB->predecessors()) {
421     if (!Visited.insert(Pred).second)
422       continue;
423 
424     int W = getWaitStatesSince(IsHazard, Pred, Pred->instr_rbegin(),
425                                WaitStates, IsExpired, Visited);
426 
427     if (W == std::numeric_limits<int>::max())
428       continue;
429 
430     MinWaitStates = Found ? std::min(MinWaitStates, W) : W;
431     if (IsExpired(nullptr, MinWaitStates))
432       return MinWaitStates;
433 
434     Found = true;
435   }
436 
437   if (Found)
438     return MinWaitStates;
439 
440   return std::numeric_limits<int>::max();
441 }
442 
443 static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard,
444                               MachineInstr *MI,
445                               IsExpiredFn IsExpired) {
446   DenseSet<const MachineBasicBlock *> Visited;
447   return getWaitStatesSince(IsHazard, MI->getParent(),
448                             std::next(MI->getReverseIterator()),
449                             0, IsExpired, Visited);
450 }
451 
452 int GCNHazardRecognizer::getWaitStatesSince(IsHazardFn IsHazard, int Limit) {
453   if (IsHazardRecognizerMode) {
454     auto IsExpiredFn = [Limit] (MachineInstr *, int WaitStates) {
455       return WaitStates >= Limit;
456     };
457     return ::getWaitStatesSince(IsHazard, CurrCycleInstr, IsExpiredFn);
458   }
459 
460   int WaitStates = 0;
461   for (MachineInstr *MI : EmittedInstrs) {
462     if (MI) {
463       if (IsHazard(MI))
464         return WaitStates;
465 
466       if (MI->isInlineAsm())
467         continue;
468     }
469     ++WaitStates;
470 
471     if (WaitStates >= Limit)
472       break;
473   }
474   return std::numeric_limits<int>::max();
475 }
476 
477 int GCNHazardRecognizer::getWaitStatesSinceDef(unsigned Reg,
478                                                IsHazardFn IsHazardDef,
479                                                int Limit) {
480   const SIRegisterInfo *TRI = ST.getRegisterInfo();
481 
482   auto IsHazardFn = [IsHazardDef, TRI, Reg] (MachineInstr *MI) {
483     return IsHazardDef(MI) && MI->modifiesRegister(Reg, TRI);
484   };
485 
486   return getWaitStatesSince(IsHazardFn, Limit);
487 }
488 
489 int GCNHazardRecognizer::getWaitStatesSinceSetReg(IsHazardFn IsHazard,
490                                                   int Limit) {
491   auto IsHazardFn = [IsHazard] (MachineInstr *MI) {
492     return isSSetReg(MI->getOpcode()) && IsHazard(MI);
493   };
494 
495   return getWaitStatesSince(IsHazardFn, Limit);
496 }
497 
498 //===----------------------------------------------------------------------===//
499 // No-op Hazard Detection
500 //===----------------------------------------------------------------------===//
501 
502 static void addRegUnits(const SIRegisterInfo &TRI, BitVector &BV,
503                         MCRegister Reg) {
504   for (MCRegUnitIterator RUI(Reg, &TRI); RUI.isValid(); ++RUI)
505     BV.set(*RUI);
506 }
507 
508 static void addRegsToSet(const SIRegisterInfo &TRI,
509                          iterator_range<MachineInstr::const_mop_iterator> Ops,
510                          BitVector &Set) {
511   for (const MachineOperand &Op : Ops) {
512     if (Op.isReg())
513       addRegUnits(TRI, Set, Op.getReg().asMCReg());
514   }
515 }
516 
517 void GCNHazardRecognizer::addClauseInst(const MachineInstr &MI) {
518   // XXX: Do we need to worry about implicit operands
519   addRegsToSet(TRI, MI.defs(), ClauseDefs);
520   addRegsToSet(TRI, MI.uses(), ClauseUses);
521 }
522 
523 static bool breaksSMEMSoftClause(MachineInstr *MI) {
524   return !SIInstrInfo::isSMRD(*MI);
525 }
526 
527 static bool breaksVMEMSoftClause(MachineInstr *MI) {
528   return !SIInstrInfo::isVMEM(*MI) && !SIInstrInfo::isFLAT(*MI);
529 }
530 
531 int GCNHazardRecognizer::checkSoftClauseHazards(MachineInstr *MEM) {
532   // SMEM soft clause are only present on VI+, and only matter if xnack is
533   // enabled.
534   if (!ST.isXNACKEnabled())
535     return 0;
536 
537   bool IsSMRD = TII.isSMRD(*MEM);
538 
539   resetClause();
540 
541   // A soft-clause is any group of consecutive SMEM instructions.  The
542   // instructions in this group may return out of order and/or may be
543   // replayed (i.e. the same instruction issued more than once).
544   //
545   // In order to handle these situations correctly we need to make sure that
546   // when a clause has more than one instruction, no instruction in the clause
547   // writes to a register that is read by another instruction in the clause
548   // (including itself). If we encounter this situaion, we need to break the
549   // clause by inserting a non SMEM instruction.
550 
551   for (MachineInstr *MI : EmittedInstrs) {
552     // When we hit a non-SMEM instruction then we have passed the start of the
553     // clause and we can stop.
554     if (!MI)
555       break;
556 
557     if (IsSMRD ? breaksSMEMSoftClause(MI) : breaksVMEMSoftClause(MI))
558       break;
559 
560     addClauseInst(*MI);
561   }
562 
563   if (ClauseDefs.none())
564     return 0;
565 
566   // We need to make sure not to put loads and stores in the same clause if they
567   // use the same address. For now, just start a new clause whenever we see a
568   // store.
569   if (MEM->mayStore())
570     return 1;
571 
572   addClauseInst(*MEM);
573 
574   // If the set of defs and uses intersect then we cannot add this instruction
575   // to the clause, so we have a hazard.
576   return ClauseDefs.anyCommon(ClauseUses) ? 1 : 0;
577 }
578 
579 int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) {
580   int WaitStatesNeeded = 0;
581 
582   WaitStatesNeeded = checkSoftClauseHazards(SMRD);
583 
584   // This SMRD hazard only affects SI.
585   if (!ST.hasSMRDReadVALUDefHazard())
586     return WaitStatesNeeded;
587 
588   // A read of an SGPR by SMRD instruction requires 4 wait states when the
589   // SGPR was written by a VALU instruction.
590   int SmrdSgprWaitStates = 4;
591   auto IsHazardDefFn = [this] (MachineInstr *MI) { return TII.isVALU(*MI); };
592   auto IsBufferHazardDefFn = [this] (MachineInstr *MI) { return TII.isSALU(*MI); };
593 
594   bool IsBufferSMRD = TII.isBufferSMRD(*SMRD);
595 
596   for (const MachineOperand &Use : SMRD->uses()) {
597     if (!Use.isReg())
598       continue;
599     int WaitStatesNeededForUse =
600         SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn,
601                                                    SmrdSgprWaitStates);
602     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
603 
604     // This fixes what appears to be undocumented hardware behavior in SI where
605     // s_mov writing a descriptor and s_buffer_load_dword reading the descriptor
606     // needs some number of nops in between. We don't know how many we need, but
607     // let's use 4. This wasn't discovered before probably because the only
608     // case when this happens is when we expand a 64-bit pointer into a full
609     // descriptor and use s_buffer_load_dword instead of s_load_dword, which was
610     // probably never encountered in the closed-source land.
611     if (IsBufferSMRD) {
612       int WaitStatesNeededForUse =
613         SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(),
614                                                    IsBufferHazardDefFn,
615                                                    SmrdSgprWaitStates);
616       WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
617     }
618   }
619 
620   return WaitStatesNeeded;
621 }
622 
623 int GCNHazardRecognizer::checkVMEMHazards(MachineInstr* VMEM) {
624   if (!ST.hasVMEMReadSGPRVALUDefHazard())
625     return 0;
626 
627   int WaitStatesNeeded = checkSoftClauseHazards(VMEM);
628 
629   // A read of an SGPR by a VMEM instruction requires 5 wait states when the
630   // SGPR was written by a VALU Instruction.
631   const int VmemSgprWaitStates = 5;
632   auto IsHazardDefFn = [this] (MachineInstr *MI) { return TII.isVALU(*MI); };
633   for (const MachineOperand &Use : VMEM->uses()) {
634     if (!Use.isReg() || TRI.isVectorRegister(MF.getRegInfo(), Use.getReg()))
635       continue;
636 
637     int WaitStatesNeededForUse =
638         VmemSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn,
639                                                    VmemSgprWaitStates);
640     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
641   }
642   return WaitStatesNeeded;
643 }
644 
645 int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) {
646   const SIRegisterInfo *TRI = ST.getRegisterInfo();
647   const SIInstrInfo *TII = ST.getInstrInfo();
648 
649   // Check for DPP VGPR read after VALU VGPR write and EXEC write.
650   int DppVgprWaitStates = 2;
651   int DppExecWaitStates = 5;
652   int WaitStatesNeeded = 0;
653   auto IsHazardDefFn = [TII] (MachineInstr *MI) { return TII->isVALU(*MI); };
654 
655   for (const MachineOperand &Use : DPP->uses()) {
656     if (!Use.isReg() || !TRI->isVGPR(MF.getRegInfo(), Use.getReg()))
657       continue;
658     int WaitStatesNeededForUse =
659         DppVgprWaitStates - getWaitStatesSinceDef(Use.getReg(),
660                               [](MachineInstr *) { return true; },
661                               DppVgprWaitStates);
662     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
663   }
664 
665   WaitStatesNeeded = std::max(
666       WaitStatesNeeded,
667       DppExecWaitStates - getWaitStatesSinceDef(AMDGPU::EXEC, IsHazardDefFn,
668                                                 DppExecWaitStates));
669 
670   return WaitStatesNeeded;
671 }
672 
673 int GCNHazardRecognizer::checkDivFMasHazards(MachineInstr *DivFMas) {
674   const SIInstrInfo *TII = ST.getInstrInfo();
675 
676   // v_div_fmas requires 4 wait states after a write to vcc from a VALU
677   // instruction.
678   const int DivFMasWaitStates = 4;
679   auto IsHazardDefFn = [TII] (MachineInstr *MI) { return TII->isVALU(*MI); };
680   int WaitStatesNeeded = getWaitStatesSinceDef(AMDGPU::VCC, IsHazardDefFn,
681                                                DivFMasWaitStates);
682 
683   return DivFMasWaitStates - WaitStatesNeeded;
684 }
685 
686 int GCNHazardRecognizer::checkGetRegHazards(MachineInstr *GetRegInstr) {
687   const SIInstrInfo *TII = ST.getInstrInfo();
688   unsigned GetRegHWReg = getHWReg(TII, *GetRegInstr);
689 
690   const int GetRegWaitStates = 2;
691   auto IsHazardFn = [TII, GetRegHWReg] (MachineInstr *MI) {
692     return GetRegHWReg == getHWReg(TII, *MI);
693   };
694   int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, GetRegWaitStates);
695 
696   return GetRegWaitStates - WaitStatesNeeded;
697 }
698 
699 int GCNHazardRecognizer::checkSetRegHazards(MachineInstr *SetRegInstr) {
700   const SIInstrInfo *TII = ST.getInstrInfo();
701   unsigned HWReg = getHWReg(TII, *SetRegInstr);
702 
703   const int SetRegWaitStates = ST.getSetRegWaitStates();
704   auto IsHazardFn = [TII, HWReg] (MachineInstr *MI) {
705     return HWReg == getHWReg(TII, *MI);
706   };
707   int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, SetRegWaitStates);
708   return SetRegWaitStates - WaitStatesNeeded;
709 }
710 
711 int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) {
712   if (!MI.mayStore())
713     return -1;
714 
715   const SIInstrInfo *TII = ST.getInstrInfo();
716   unsigned Opcode = MI.getOpcode();
717   const MCInstrDesc &Desc = MI.getDesc();
718 
719   int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);
720   int VDataRCID = -1;
721   if (VDataIdx != -1)
722     VDataRCID = Desc.OpInfo[VDataIdx].RegClass;
723 
724   if (TII->isMUBUF(MI) || TII->isMTBUF(MI)) {
725     // There is no hazard if the instruction does not use vector regs
726     // (like wbinvl1)
727     if (VDataIdx == -1)
728       return -1;
729     // For MUBUF/MTBUF instructions this hazard only exists if the
730     // instruction is not using a register in the soffset field.
731     const MachineOperand *SOffset =
732         TII->getNamedOperand(MI, AMDGPU::OpName::soffset);
733     // If we have no soffset operand, then assume this field has been
734     // hardcoded to zero.
735     if (AMDGPU::getRegBitWidth(VDataRCID) > 64 &&
736         (!SOffset || !SOffset->isReg()))
737       return VDataIdx;
738   }
739 
740   // MIMG instructions create a hazard if they don't use a 256-bit T# and
741   // the store size is greater than 8 bytes and they have more than two bits
742   // of their dmask set.
743   // All our MIMG definitions use a 256-bit T#, so we can skip checking for them.
744   if (TII->isMIMG(MI)) {
745     int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc);
746     assert(SRsrcIdx != -1 &&
747            AMDGPU::getRegBitWidth(Desc.OpInfo[SRsrcIdx].RegClass) == 256);
748     (void)SRsrcIdx;
749   }
750 
751   if (TII->isFLAT(MI)) {
752     int DataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);
753     if (AMDGPU::getRegBitWidth(Desc.OpInfo[DataIdx].RegClass) > 64)
754       return DataIdx;
755   }
756 
757   return -1;
758 }
759 
760 int
761 GCNHazardRecognizer::checkVALUHazardsHelper(const MachineOperand &Def,
762                                             const MachineRegisterInfo &MRI) {
763   // Helper to check for the hazard where VMEM instructions that store more than
764   // 8 bytes can have there store data over written by the next instruction.
765   const SIRegisterInfo *TRI = ST.getRegisterInfo();
766 
767   const int VALUWaitStates = 1;
768   int WaitStatesNeeded = 0;
769 
770   if (!TRI->isVectorRegister(MRI, Def.getReg()))
771     return WaitStatesNeeded;
772   Register Reg = Def.getReg();
773   auto IsHazardFn = [this, Reg, TRI] (MachineInstr *MI) {
774     int DataIdx = createsVALUHazard(*MI);
775     return DataIdx >= 0 &&
776     TRI->regsOverlap(MI->getOperand(DataIdx).getReg(), Reg);
777   };
778   int WaitStatesNeededForDef =
779     VALUWaitStates - getWaitStatesSince(IsHazardFn, VALUWaitStates);
780   WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
781 
782   return WaitStatesNeeded;
783 }
784 
785 int GCNHazardRecognizer::checkVALUHazards(MachineInstr *VALU) {
786   // This checks for the hazard where VMEM instructions that store more than
787   // 8 bytes can have there store data over written by the next instruction.
788   if (!ST.has12DWordStoreHazard())
789     return 0;
790 
791   const MachineRegisterInfo &MRI = MF.getRegInfo();
792   int WaitStatesNeeded = 0;
793 
794   for (const MachineOperand &Def : VALU->defs()) {
795     WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Def, MRI));
796   }
797 
798   return WaitStatesNeeded;
799 }
800 
801 int GCNHazardRecognizer::checkInlineAsmHazards(MachineInstr *IA) {
802   // This checks for hazards associated with inline asm statements.
803   // Since inline asms can contain just about anything, we use this
804   // to call/leverage other check*Hazard routines. Note that
805   // this function doesn't attempt to address all possible inline asm
806   // hazards (good luck), but is a collection of what has been
807   // problematic thus far.
808 
809   // see checkVALUHazards()
810   if (!ST.has12DWordStoreHazard())
811     return 0;
812 
813   const MachineRegisterInfo &MRI = MF.getRegInfo();
814   int WaitStatesNeeded = 0;
815 
816   for (unsigned I = InlineAsm::MIOp_FirstOperand, E = IA->getNumOperands();
817        I != E; ++I) {
818     const MachineOperand &Op = IA->getOperand(I);
819     if (Op.isReg() && Op.isDef()) {
820       WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Op, MRI));
821     }
822   }
823 
824   return WaitStatesNeeded;
825 }
826 
827 int GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) {
828   const SIInstrInfo *TII = ST.getInstrInfo();
829   const SIRegisterInfo *TRI = ST.getRegisterInfo();
830   const MachineRegisterInfo &MRI = MF.getRegInfo();
831 
832   const MachineOperand *LaneSelectOp =
833       TII->getNamedOperand(*RWLane, AMDGPU::OpName::src1);
834 
835   if (!LaneSelectOp->isReg() || !TRI->isSGPRReg(MRI, LaneSelectOp->getReg()))
836     return 0;
837 
838   Register LaneSelectReg = LaneSelectOp->getReg();
839   auto IsHazardFn = [TII] (MachineInstr *MI) {
840     return TII->isVALU(*MI);
841   };
842 
843   const int RWLaneWaitStates = 4;
844   int WaitStatesSince = getWaitStatesSinceDef(LaneSelectReg, IsHazardFn,
845                                               RWLaneWaitStates);
846   return RWLaneWaitStates - WaitStatesSince;
847 }
848 
849 int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) {
850   if (!ST.hasRFEHazards())
851     return 0;
852 
853   const SIInstrInfo *TII = ST.getInstrInfo();
854 
855   const int RFEWaitStates = 1;
856 
857   auto IsHazardFn = [TII] (MachineInstr *MI) {
858     return getHWReg(TII, *MI) == AMDGPU::Hwreg::ID_TRAPSTS;
859   };
860   int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, RFEWaitStates);
861   return RFEWaitStates - WaitStatesNeeded;
862 }
863 
864 int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) {
865   const SIInstrInfo *TII = ST.getInstrInfo();
866   const int SMovRelWaitStates = 1;
867   auto IsHazardFn = [TII] (MachineInstr *MI) {
868     return TII->isSALU(*MI);
869   };
870   return SMovRelWaitStates - getWaitStatesSinceDef(AMDGPU::M0, IsHazardFn,
871                                                    SMovRelWaitStates);
872 }
873 
874 void GCNHazardRecognizer::fixHazards(MachineInstr *MI) {
875   fixVMEMtoScalarWriteHazards(MI);
876   fixVcmpxPermlaneHazards(MI);
877   fixSMEMtoVectorWriteHazards(MI);
878   fixVcmpxExecWARHazard(MI);
879   fixLdsBranchVmemWARHazard(MI);
880 }
881 
882 bool GCNHazardRecognizer::fixVcmpxPermlaneHazards(MachineInstr *MI) {
883   if (!ST.hasVcmpxPermlaneHazard() || !isPermlane(*MI))
884     return false;
885 
886   const SIInstrInfo *TII = ST.getInstrInfo();
887   auto IsHazardFn = [TII] (MachineInstr *MI) {
888     return TII->isVOPC(*MI);
889   };
890 
891   auto IsExpiredFn = [] (MachineInstr *MI, int) {
892     if (!MI)
893       return false;
894     unsigned Opc = MI->getOpcode();
895     return SIInstrInfo::isVALU(*MI) &&
896            Opc != AMDGPU::V_NOP_e32 &&
897            Opc != AMDGPU::V_NOP_e64 &&
898            Opc != AMDGPU::V_NOP_sdwa;
899   };
900 
901   if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
902       std::numeric_limits<int>::max())
903     return false;
904 
905   // V_NOP will be discarded by SQ.
906   // Use V_MOB_B32 v?, v?. Register must be alive so use src0 of V_PERMLANE*
907   // which is always a VGPR and available.
908   auto *Src0 = TII->getNamedOperand(*MI, AMDGPU::OpName::src0);
909   Register Reg = Src0->getReg();
910   bool IsUndef = Src0->isUndef();
911   BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
912           TII->get(AMDGPU::V_MOV_B32_e32))
913     .addReg(Reg, RegState::Define | (IsUndef ? RegState::Dead : 0))
914     .addReg(Reg, IsUndef ? RegState::Undef : RegState::Kill);
915 
916   return true;
917 }
918 
919 bool GCNHazardRecognizer::fixVMEMtoScalarWriteHazards(MachineInstr *MI) {
920   if (!ST.hasVMEMtoScalarWriteHazard())
921     return false;
922 
923   if (!SIInstrInfo::isSALU(*MI) && !SIInstrInfo::isSMRD(*MI))
924     return false;
925 
926   if (MI->getNumDefs() == 0)
927     return false;
928 
929   const SIRegisterInfo *TRI = ST.getRegisterInfo();
930 
931   auto IsHazardFn = [TRI, MI] (MachineInstr *I) {
932     if (!SIInstrInfo::isVMEM(*I) && !SIInstrInfo::isDS(*I) &&
933         !SIInstrInfo::isFLAT(*I))
934       return false;
935 
936     for (const MachineOperand &Def : MI->defs()) {
937       MachineOperand *Op = I->findRegisterUseOperand(Def.getReg(), false, TRI);
938       if (!Op)
939         continue;
940       return true;
941     }
942     return false;
943   };
944 
945   auto IsExpiredFn = [](MachineInstr *MI, int) {
946     return MI && (SIInstrInfo::isVALU(*MI) ||
947                   (MI->getOpcode() == AMDGPU::S_WAITCNT &&
948                    !MI->getOperand(0).getImm()) ||
949                   (MI->getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
950                    MI->getOperand(0).getImm() == 0xffe3));
951   };
952 
953   if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
954       std::numeric_limits<int>::max())
955     return false;
956 
957   const SIInstrInfo *TII = ST.getInstrInfo();
958   BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
959           TII->get(AMDGPU::S_WAITCNT_DEPCTR))
960       .addImm(0xffe3);
961   return true;
962 }
963 
964 bool GCNHazardRecognizer::fixSMEMtoVectorWriteHazards(MachineInstr *MI) {
965   if (!ST.hasSMEMtoVectorWriteHazard())
966     return false;
967 
968   if (!SIInstrInfo::isVALU(*MI))
969     return false;
970 
971   unsigned SDSTName;
972   switch (MI->getOpcode()) {
973   case AMDGPU::V_READLANE_B32:
974   case AMDGPU::V_READFIRSTLANE_B32:
975     SDSTName = AMDGPU::OpName::vdst;
976     break;
977   default:
978     SDSTName = AMDGPU::OpName::sdst;
979     break;
980   }
981 
982   const SIInstrInfo *TII = ST.getInstrInfo();
983   const SIRegisterInfo *TRI = ST.getRegisterInfo();
984   const AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(ST.getCPU());
985   const MachineOperand *SDST = TII->getNamedOperand(*MI, SDSTName);
986   if (!SDST) {
987     for (const auto &MO : MI->implicit_operands()) {
988       if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegClass(MO.getReg()))) {
989         SDST = &MO;
990         break;
991       }
992     }
993   }
994 
995   if (!SDST)
996     return false;
997 
998   const Register SDSTReg = SDST->getReg();
999   auto IsHazardFn = [SDSTReg, TRI] (MachineInstr *I) {
1000     return SIInstrInfo::isSMRD(*I) && I->readsRegister(SDSTReg, TRI);
1001   };
1002 
1003   auto IsExpiredFn = [TII, IV] (MachineInstr *MI, int) {
1004     if (MI) {
1005       if (TII->isSALU(*MI)) {
1006         switch (MI->getOpcode()) {
1007         case AMDGPU::S_SETVSKIP:
1008         case AMDGPU::S_VERSION:
1009         case AMDGPU::S_WAITCNT_VSCNT:
1010         case AMDGPU::S_WAITCNT_VMCNT:
1011         case AMDGPU::S_WAITCNT_EXPCNT:
1012           // These instructions cannot not mitigate the hazard.
1013           return false;
1014         case AMDGPU::S_WAITCNT_LGKMCNT:
1015           // Reducing lgkmcnt count to 0 always mitigates the hazard.
1016           return (MI->getOperand(1).getImm() == 0) &&
1017                  (MI->getOperand(0).getReg() == AMDGPU::SGPR_NULL);
1018         case AMDGPU::S_WAITCNT: {
1019           const int64_t Imm = MI->getOperand(0).getImm();
1020           AMDGPU::Waitcnt Decoded = AMDGPU::decodeWaitcnt(IV, Imm);
1021           return (Decoded.LgkmCnt == 0);
1022         }
1023         default:
1024           // SOPP instructions cannot mitigate the hazard.
1025           if (TII->isSOPP(*MI))
1026             return false;
1027           // At this point the SALU can be assumed to mitigate the hazard
1028           // because either:
1029           // (a) it is independent of the at risk SMEM (breaking chain),
1030           // or
1031           // (b) it is dependent on the SMEM, in which case an appropriate
1032           //     s_waitcnt lgkmcnt _must_ exist between it and the at risk
1033           //     SMEM instruction.
1034           return true;
1035         }
1036       }
1037     }
1038     return false;
1039   };
1040 
1041   if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1042       std::numeric_limits<int>::max())
1043     return false;
1044 
1045   BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1046           TII->get(AMDGPU::S_MOV_B32), AMDGPU::SGPR_NULL)
1047       .addImm(0);
1048   return true;
1049 }
1050 
1051 bool GCNHazardRecognizer::fixVcmpxExecWARHazard(MachineInstr *MI) {
1052   if (!ST.hasVcmpxExecWARHazard() || !SIInstrInfo::isVALU(*MI))
1053     return false;
1054 
1055   const SIRegisterInfo *TRI = ST.getRegisterInfo();
1056   if (!MI->modifiesRegister(AMDGPU::EXEC, TRI))
1057     return false;
1058 
1059   auto IsHazardFn = [TRI] (MachineInstr *I) {
1060     if (SIInstrInfo::isVALU(*I))
1061       return false;
1062     return I->readsRegister(AMDGPU::EXEC, TRI);
1063   };
1064 
1065   const SIInstrInfo *TII = ST.getInstrInfo();
1066   auto IsExpiredFn = [TII, TRI] (MachineInstr *MI, int) {
1067     if (!MI)
1068       return false;
1069     if (SIInstrInfo::isVALU(*MI)) {
1070       if (TII->getNamedOperand(*MI, AMDGPU::OpName::sdst))
1071         return true;
1072       for (auto MO : MI->implicit_operands())
1073         if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegClass(MO.getReg())))
1074           return true;
1075     }
1076     if (MI->getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
1077         (MI->getOperand(0).getImm() & 0xfffe) == 0xfffe)
1078       return true;
1079     return false;
1080   };
1081 
1082   if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1083       std::numeric_limits<int>::max())
1084     return false;
1085 
1086   BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1087           TII->get(AMDGPU::S_WAITCNT_DEPCTR))
1088     .addImm(0xfffe);
1089   return true;
1090 }
1091 
1092 bool GCNHazardRecognizer::fixLdsBranchVmemWARHazard(MachineInstr *MI) {
1093   if (!ST.hasLdsBranchVmemWARHazard())
1094     return false;
1095 
1096   auto IsHazardInst = [] (const MachineInstr *MI) {
1097     if (SIInstrInfo::isDS(*MI))
1098       return 1;
1099     if (SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isSegmentSpecificFLAT(*MI))
1100       return 2;
1101     return 0;
1102   };
1103 
1104   auto InstType = IsHazardInst(MI);
1105   if (!InstType)
1106     return false;
1107 
1108   auto IsExpiredFn = [&IsHazardInst] (MachineInstr *I, int) {
1109     return I && (IsHazardInst(I) ||
1110                  (I->getOpcode() == AMDGPU::S_WAITCNT_VSCNT &&
1111                   I->getOperand(0).getReg() == AMDGPU::SGPR_NULL &&
1112                   !I->getOperand(1).getImm()));
1113   };
1114 
1115   auto IsHazardFn = [InstType, &IsHazardInst] (MachineInstr *I) {
1116     if (!I->isBranch())
1117       return false;
1118 
1119     auto IsHazardFn = [InstType, IsHazardInst] (MachineInstr *I) {
1120       auto InstType2 = IsHazardInst(I);
1121       return InstType2 && InstType != InstType2;
1122     };
1123 
1124     auto IsExpiredFn = [InstType, &IsHazardInst] (MachineInstr *I, int) {
1125       if (!I)
1126         return false;
1127 
1128       auto InstType2 = IsHazardInst(I);
1129       if (InstType == InstType2)
1130         return true;
1131 
1132       return I->getOpcode() == AMDGPU::S_WAITCNT_VSCNT &&
1133              I->getOperand(0).getReg() == AMDGPU::SGPR_NULL &&
1134              !I->getOperand(1).getImm();
1135     };
1136 
1137     return ::getWaitStatesSince(IsHazardFn, I, IsExpiredFn) !=
1138            std::numeric_limits<int>::max();
1139   };
1140 
1141   if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1142       std::numeric_limits<int>::max())
1143     return false;
1144 
1145   const SIInstrInfo *TII = ST.getInstrInfo();
1146   BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1147           TII->get(AMDGPU::S_WAITCNT_VSCNT))
1148     .addReg(AMDGPU::SGPR_NULL, RegState::Undef)
1149     .addImm(0);
1150 
1151   return true;
1152 }
1153 
1154 int GCNHazardRecognizer::checkNSAtoVMEMHazard(MachineInstr *MI) {
1155   int NSAtoVMEMWaitStates = 1;
1156 
1157   if (!ST.hasNSAtoVMEMBug())
1158     return 0;
1159 
1160   if (!SIInstrInfo::isMUBUF(*MI) && !SIInstrInfo::isMTBUF(*MI))
1161     return 0;
1162 
1163   const SIInstrInfo *TII = ST.getInstrInfo();
1164   const auto *Offset = TII->getNamedOperand(*MI, AMDGPU::OpName::offset);
1165   if (!Offset || (Offset->getImm() & 6) == 0)
1166     return 0;
1167 
1168   auto IsHazardFn = [TII] (MachineInstr *I) {
1169     if (!SIInstrInfo::isMIMG(*I))
1170       return false;
1171     const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(I->getOpcode());
1172     return Info->MIMGEncoding == AMDGPU::MIMGEncGfx10NSA &&
1173            TII->getInstSizeInBytes(*I) >= 16;
1174   };
1175 
1176   return NSAtoVMEMWaitStates - getWaitStatesSince(IsHazardFn, 1);
1177 }
1178 
1179 int GCNHazardRecognizer::checkFPAtomicToDenormModeHazard(MachineInstr *MI) {
1180   int FPAtomicToDenormModeWaitStates = 3;
1181 
1182   if (MI->getOpcode() != AMDGPU::S_DENORM_MODE)
1183     return 0;
1184 
1185   auto IsHazardFn = [] (MachineInstr *I) {
1186     if (!SIInstrInfo::isVMEM(*I) && !SIInstrInfo::isFLAT(*I))
1187       return false;
1188     return SIInstrInfo::isFPAtomic(*I);
1189   };
1190 
1191   auto IsExpiredFn = [] (MachineInstr *MI, int WaitStates) {
1192     if (WaitStates >= 3 || SIInstrInfo::isVALU(*MI))
1193       return true;
1194 
1195     switch (MI->getOpcode()) {
1196     case AMDGPU::S_WAITCNT:
1197     case AMDGPU::S_WAITCNT_VSCNT:
1198     case AMDGPU::S_WAITCNT_VMCNT:
1199     case AMDGPU::S_WAITCNT_EXPCNT:
1200     case AMDGPU::S_WAITCNT_LGKMCNT:
1201     case AMDGPU::S_WAIT_IDLE:
1202       return true;
1203     default:
1204       break;
1205     }
1206 
1207     return false;
1208   };
1209 
1210 
1211   return FPAtomicToDenormModeWaitStates -
1212          ::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn);
1213 }
1214 
1215 int GCNHazardRecognizer::checkMAIHazards(MachineInstr *MI) {
1216   assert(SIInstrInfo::isMAI(*MI));
1217 
1218   return ST.hasGFX90AInsts() ? checkMAIHazards90A(MI) : checkMAIHazards908(MI);
1219 }
1220 
1221 int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) {
1222   int WaitStatesNeeded = 0;
1223   unsigned Opc = MI->getOpcode();
1224 
1225   auto IsVALUFn = [] (MachineInstr *MI) {
1226     return SIInstrInfo::isVALU(*MI);
1227   };
1228 
1229   if (Opc != AMDGPU::V_ACCVGPR_READ_B32_e64) { // MFMA or v_accvgpr_write
1230     const int LegacyVALUWritesVGPRWaitStates = 2;
1231     const int VALUWritesExecWaitStates = 4;
1232     const int MaxWaitStates = 4;
1233 
1234     int WaitStatesNeededForUse = VALUWritesExecWaitStates -
1235       getWaitStatesSinceDef(AMDGPU::EXEC, IsVALUFn, MaxWaitStates);
1236     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1237 
1238     if (WaitStatesNeeded < MaxWaitStates) {
1239       for (const MachineOperand &Use : MI->explicit_uses()) {
1240         const int MaxWaitStates = 2;
1241 
1242         if (!Use.isReg() || !TRI.isVGPR(MF.getRegInfo(), Use.getReg()))
1243           continue;
1244 
1245         int WaitStatesNeededForUse = LegacyVALUWritesVGPRWaitStates -
1246           getWaitStatesSinceDef(Use.getReg(), IsVALUFn, MaxWaitStates);
1247         WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1248 
1249         if (WaitStatesNeeded == MaxWaitStates)
1250           break;
1251       }
1252     }
1253   }
1254 
1255   auto IsMFMAFn = [] (MachineInstr *MI) {
1256     return SIInstrInfo::isMAI(*MI) &&
1257            MI->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
1258            MI->getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64;
1259   };
1260 
1261   for (const MachineOperand &Op : MI->explicit_operands()) {
1262     if (!Op.isReg() || !TRI.isAGPR(MF.getRegInfo(), Op.getReg()))
1263       continue;
1264 
1265     if (Op.isDef() && Opc != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
1266       continue;
1267 
1268     const int MFMAWritesAGPROverlappedSrcABWaitStates = 4;
1269     const int MFMAWritesAGPROverlappedSrcCWaitStates = 2;
1270     const int MFMA4x4WritesAGPRAccVgprReadWaitStates = 4;
1271     const int MFMA16x16WritesAGPRAccVgprReadWaitStates = 10;
1272     const int MFMA32x32WritesAGPRAccVgprReadWaitStates = 18;
1273     const int MFMA4x4WritesAGPRAccVgprWriteWaitStates = 1;
1274     const int MFMA16x16WritesAGPRAccVgprWriteWaitStates = 7;
1275     const int MFMA32x32WritesAGPRAccVgprWriteWaitStates = 15;
1276     const int MaxWaitStates = 18;
1277     Register Reg = Op.getReg();
1278     unsigned HazardDefLatency = 0;
1279 
1280     auto IsOverlappedMFMAFn = [Reg, &IsMFMAFn, &HazardDefLatency, this]
1281                               (MachineInstr *MI) {
1282       if (!IsMFMAFn(MI))
1283         return false;
1284       Register DstReg = MI->getOperand(0).getReg();
1285       if (DstReg == Reg)
1286         return false;
1287       HazardDefLatency = std::max(HazardDefLatency,
1288                                   TSchedModel.computeInstrLatency(MI));
1289       return TRI.regsOverlap(DstReg, Reg);
1290     };
1291 
1292     int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsOverlappedMFMAFn,
1293                                                    MaxWaitStates);
1294     int NeedWaitStates = MFMAWritesAGPROverlappedSrcABWaitStates;
1295     int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
1296     int OpNo = MI->getOperandNo(&Op);
1297     if (OpNo == SrcCIdx) {
1298       NeedWaitStates = MFMAWritesAGPROverlappedSrcCWaitStates;
1299     } else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) {
1300       switch (HazardDefLatency) {
1301       case 2:  NeedWaitStates = MFMA4x4WritesAGPRAccVgprReadWaitStates;
1302                break;
1303       case 8:  NeedWaitStates = MFMA16x16WritesAGPRAccVgprReadWaitStates;
1304                break;
1305       case 16: LLVM_FALLTHROUGH;
1306       default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprReadWaitStates;
1307                break;
1308       }
1309     } else if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
1310       switch (HazardDefLatency) {
1311       case 2:  NeedWaitStates = MFMA4x4WritesAGPRAccVgprWriteWaitStates;
1312                break;
1313       case 8:  NeedWaitStates = MFMA16x16WritesAGPRAccVgprWriteWaitStates;
1314                break;
1315       case 16: LLVM_FALLTHROUGH;
1316       default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprWriteWaitStates;
1317                break;
1318       }
1319     }
1320 
1321     int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
1322     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1323 
1324     if (WaitStatesNeeded == MaxWaitStates)
1325       return WaitStatesNeeded; // Early exit.
1326 
1327     auto IsAccVgprWriteFn = [Reg, this] (MachineInstr *MI) {
1328       if (MI->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
1329         return false;
1330       Register DstReg = MI->getOperand(0).getReg();
1331       return TRI.regsOverlap(Reg, DstReg);
1332     };
1333 
1334     const int AccVGPRWriteMFMAReadSrcCWaitStates = 1;
1335     const int AccVGPRWriteMFMAReadSrcABWaitStates = 3;
1336     const int AccVGPRWriteAccVgprReadWaitStates = 3;
1337     NeedWaitStates = AccVGPRWriteMFMAReadSrcABWaitStates;
1338     if (OpNo == SrcCIdx)
1339       NeedWaitStates = AccVGPRWriteMFMAReadSrcCWaitStates;
1340     else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64)
1341       NeedWaitStates = AccVGPRWriteAccVgprReadWaitStates;
1342 
1343     WaitStatesNeededForUse = NeedWaitStates -
1344       getWaitStatesSinceDef(Reg, IsAccVgprWriteFn, MaxWaitStates);
1345     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1346 
1347     if (WaitStatesNeeded == MaxWaitStates)
1348       return WaitStatesNeeded; // Early exit.
1349   }
1350 
1351   if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
1352     const int MFMA4x4ReadSrcCAccVgprWriteWaitStates = 0;
1353     const int MFMA16x16ReadSrcCAccVgprWriteWaitStates = 5;
1354     const int MFMA32x32ReadSrcCAccVgprWriteWaitStates = 13;
1355     const int MaxWaitStates = 13;
1356     Register DstReg = MI->getOperand(0).getReg();
1357     unsigned HazardDefLatency = 0;
1358 
1359     auto IsSrcCMFMAFn = [DstReg, &IsMFMAFn, &HazardDefLatency, this]
1360                          (MachineInstr *MI) {
1361       if (!IsMFMAFn(MI))
1362         return false;
1363       Register Reg = TII.getNamedOperand(*MI, AMDGPU::OpName::src2)->getReg();
1364       HazardDefLatency = std::max(HazardDefLatency,
1365                                   TSchedModel.computeInstrLatency(MI));
1366       return TRI.regsOverlap(Reg, DstReg);
1367     };
1368 
1369     int WaitStatesSince = getWaitStatesSince(IsSrcCMFMAFn, MaxWaitStates);
1370     int NeedWaitStates;
1371     switch (HazardDefLatency) {
1372     case 2:  NeedWaitStates = MFMA4x4ReadSrcCAccVgprWriteWaitStates;
1373              break;
1374     case 8:  NeedWaitStates = MFMA16x16ReadSrcCAccVgprWriteWaitStates;
1375              break;
1376     case 16: LLVM_FALLTHROUGH;
1377     default: NeedWaitStates = MFMA32x32ReadSrcCAccVgprWriteWaitStates;
1378              break;
1379     }
1380 
1381     int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSince;
1382     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1383   }
1384 
1385   return WaitStatesNeeded;
1386 }
1387 
1388 int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) {
1389   int WaitStatesNeeded = 0;
1390   unsigned Opc = MI->getOpcode();
1391 
1392   auto IsMFMAFn = [] (MachineInstr *MI) {
1393     return SIInstrInfo::isMAI(*MI) &&
1394            MI->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
1395            MI->getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64;
1396   };
1397 
1398   auto IsLegacyVALUFn = [&IsMFMAFn] (MachineInstr *MI) {
1399     return SIInstrInfo::isVALU(*MI) && !IsMFMAFn(MI);
1400   };
1401 
1402   auto IsLegacyVALUNotDotFn = [&IsMFMAFn] (MachineInstr *MI) {
1403     return SIInstrInfo::isVALU(*MI) &&
1404            !IsMFMAFn(MI) && !SIInstrInfo::isDOT(*MI);
1405   };
1406 
1407   if (!IsMFMAFn(MI))
1408     return WaitStatesNeeded;
1409 
1410   const int VALUWritesExecWaitStates = 4;
1411   int WaitStatesNeededForUse = VALUWritesExecWaitStates -
1412     getWaitStatesSinceDef(AMDGPU::EXEC, IsLegacyVALUFn,
1413                           VALUWritesExecWaitStates);
1414   WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1415 
1416   int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
1417 
1418   // Loop for both DGEMM and S/HGEMM 2nd instruction.
1419   for (const MachineOperand &Use : MI->explicit_uses()) {
1420     const int LegacyVALUNotDotWritesVGPRWaitStates = 2;
1421     const int SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates = 2;
1422     const int SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates = 8;
1423     const int SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates = 16;
1424     const int SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates = 3;
1425     const int SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates = 9;
1426     const int SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates = 17;
1427     const int DMFMA16x16WritesVGPROverlappedSrcCWaitStates = 9;
1428     const int DMFMA4x4WritesVGPROverlappedSrcCWaitStates = 4;
1429     const int SMFMA4x4WritesVGPROverlappedSrcABWaitStates = 5;
1430     const int SMFMA16x16WritesVGPROverlappedSrcABWaitStates = 11;
1431     const int SMFMA32x32WritesVGPROverlappedSrcABWaitStates = 19;
1432     const int DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates = 6;
1433     const int DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates = 11;
1434     const int DMFMA4x4WritesVGPRFullSrcCWaitStates = 4;
1435     const int MaxWaitStates = 19;
1436 
1437     if (!Use.isReg())
1438       continue;
1439     unsigned Reg = Use.getReg();
1440     bool FullReg;
1441     MachineInstr *MI1;
1442 
1443     auto IsOverlappedDGEMMorXDLFn = [Reg, &IsMFMAFn, &FullReg, &MI1, this]
1444                                     (MachineInstr *MI) {
1445       if (!IsMFMAFn(MI))
1446         return false;
1447       if (!isDGEMM(MI->getOpcode()) && !isXDL(ST, *MI))
1448         return false;
1449       Register DstReg = MI->getOperand(0).getReg();
1450       FullReg = (DstReg == Reg);
1451       MI1 = MI;
1452       return TRI.regsOverlap(DstReg, Reg);
1453     };
1454 
1455     WaitStatesNeededForUse = LegacyVALUNotDotWritesVGPRWaitStates -
1456       getWaitStatesSinceDef(Reg, IsLegacyVALUNotDotFn, MaxWaitStates);
1457     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1458 
1459     int NumWaitStates = getWaitStatesSinceDef(Reg, IsOverlappedDGEMMorXDLFn,
1460                                               MaxWaitStates);
1461     if (NumWaitStates == std::numeric_limits<int>::max())
1462       continue;
1463 
1464     int OpNo = MI->getOperandNo(&Use);
1465     unsigned Opc1 = MI1->getOpcode();
1466     int NeedWaitStates = 0;
1467     if (OpNo == SrcCIdx) {
1468       if (!isDGEMM(Opc) && isDGEMM(Opc1)) {
1469         NeedWaitStates = 0;
1470       } else if (FullReg) {
1471         if ((Opc == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
1472              Opc == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64) &&
1473             (Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
1474              Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64))
1475           NeedWaitStates = DMFMA4x4WritesVGPRFullSrcCWaitStates;
1476       } else {
1477         switch (Opc1) {
1478         case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
1479         case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
1480           if (!isXDL(ST, *MI))
1481             NeedWaitStates = DMFMA16x16WritesVGPROverlappedSrcCWaitStates;
1482           break;
1483         case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
1484         case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
1485           if (!isXDL(ST, *MI))
1486             NeedWaitStates = DMFMA4x4WritesVGPROverlappedSrcCWaitStates;
1487           break;
1488         default:
1489           switch (TSchedModel.computeInstrLatency(MI1)) {
1490           case 2:
1491             NeedWaitStates = isDGEMM(Opc)
1492               ? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates
1493               : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates;
1494             break;
1495           case 8:
1496             NeedWaitStates = isDGEMM(Opc)
1497               ? SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates
1498               : SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates;
1499             break;
1500           case 16: LLVM_FALLTHROUGH;
1501           default:
1502             NeedWaitStates = isDGEMM(Opc)
1503               ? SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates
1504               : SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates;
1505           }
1506         }
1507       }
1508     } else {
1509       switch (Opc1) {
1510       case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
1511       case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
1512         NeedWaitStates = DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates;
1513         break;
1514       case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
1515       case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
1516         NeedWaitStates = DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates;
1517         break;
1518       default:
1519         switch (TSchedModel.computeInstrLatency(MI1)) {
1520         case 2:
1521           NeedWaitStates = SMFMA4x4WritesVGPROverlappedSrcABWaitStates;
1522           break;
1523         case 8:
1524           NeedWaitStates = SMFMA16x16WritesVGPROverlappedSrcABWaitStates;
1525           break;
1526         case 16: LLVM_FALLTHROUGH;
1527         default:
1528           NeedWaitStates = SMFMA32x32WritesVGPROverlappedSrcABWaitStates;
1529         }
1530       }
1531     }
1532     if (WaitStatesNeeded >= NeedWaitStates)
1533       continue;
1534 
1535     WaitStatesNeededForUse = NeedWaitStates - NumWaitStates;
1536     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1537 
1538     if (WaitStatesNeeded == MaxWaitStates)
1539       break;
1540   }
1541 
1542   return WaitStatesNeeded;
1543 }
1544 
1545 int GCNHazardRecognizer::checkMAILdStHazards(MachineInstr *MI) {
1546   // On gfx90a+ releveant hazards are checked in checkMAIVALUHazards()
1547   if (!ST.hasMAIInsts() || ST.hasGFX90AInsts())
1548     return 0;
1549 
1550   int WaitStatesNeeded = 0;
1551 
1552   auto IsAccVgprReadFn = [] (MachineInstr *MI) {
1553     return MI->getOpcode() == AMDGPU::V_ACCVGPR_READ_B32_e64;
1554   };
1555 
1556   for (const MachineOperand &Op : MI->explicit_uses()) {
1557     if (!Op.isReg() || !TRI.isVGPR(MF.getRegInfo(), Op.getReg()))
1558       continue;
1559 
1560     Register Reg = Op.getReg();
1561 
1562     const int AccVgprReadLdStWaitStates = 2;
1563     const int VALUWriteAccVgprRdWrLdStDepVALUWaitStates = 1;
1564     const int MaxWaitStates = 2;
1565 
1566     int WaitStatesNeededForUse = AccVgprReadLdStWaitStates -
1567       getWaitStatesSinceDef(Reg, IsAccVgprReadFn, MaxWaitStates);
1568     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1569 
1570     if (WaitStatesNeeded == MaxWaitStates)
1571       return WaitStatesNeeded; // Early exit.
1572 
1573     auto IsVALUAccVgprRdWrCheckFn = [Reg, this](MachineInstr *MI) {
1574       if (MI->getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64 &&
1575           MI->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
1576         return false;
1577       auto IsVALUFn = [] (MachineInstr *MI) {
1578         return SIInstrInfo::isVALU(*MI) && !SIInstrInfo::isMAI(*MI);
1579       };
1580       return getWaitStatesSinceDef(Reg, IsVALUFn, 2 /*MaxWaitStates*/) <
1581              std::numeric_limits<int>::max();
1582     };
1583 
1584     WaitStatesNeededForUse = VALUWriteAccVgprRdWrLdStDepVALUWaitStates -
1585       getWaitStatesSince(IsVALUAccVgprRdWrCheckFn, MaxWaitStates);
1586     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1587   }
1588 
1589   return WaitStatesNeeded;
1590 }
1591 
1592 int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) {
1593   if (!ST.hasGFX90AInsts())
1594     return 0;
1595 
1596   auto IsMFMAFn = [] (MachineInstr *MI) -> bool {
1597     return SIInstrInfo::isMAI(*MI) &&
1598            MI->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
1599            MI->getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64;
1600   };
1601 
1602   auto IsDGEMMFn = [] (MachineInstr *MI) -> bool {
1603     return isDGEMM(MI->getOpcode());
1604   };
1605 
1606   // This is checked in checkMAIHazards90A()
1607   if (IsMFMAFn(MI))
1608     return 0;
1609 
1610   int WaitStatesNeeded = 0;
1611 
1612   bool IsMemOrExport = SIInstrInfo::isVMEM(*MI) ||
1613                        SIInstrInfo::isFLAT(*MI) ||
1614                        SIInstrInfo::isDS(*MI) ||
1615                        SIInstrInfo::isEXP(*MI);
1616   bool IsVALU = SIInstrInfo::isVALU(*MI);
1617 
1618   MachineInstr *MFMA = nullptr;
1619   unsigned Reg;
1620   auto IsDGEMMorXDLWriteFn = [&Reg, &IsMFMAFn, &MFMA, this] (MachineInstr *MI) {
1621     if (!IsMFMAFn(MI) || !TRI.regsOverlap(MI->getOperand(0).getReg(), Reg))
1622       return false;
1623     if (!isDGEMM(MI->getOpcode()) && !isXDL(ST, *MI))
1624       return false;
1625     MFMA = MI;
1626     return true;
1627   };
1628 
1629   MachineInstr *DOT = nullptr;
1630   auto IsDotWriteFn = [&Reg, &DOT, this] (MachineInstr *MI) {
1631     if (!SIInstrInfo::isDOT(*MI) ||
1632         !TRI.regsOverlap(MI->getOperand(0).getReg(), Reg))
1633       return false;
1634     DOT = MI;
1635     return true;
1636   };
1637 
1638   int SrcCIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(),
1639                                            AMDGPU::OpName::src2);
1640 
1641   if (IsMemOrExport || IsVALU) {
1642     const int SMFMA4x4WriteVgprVALUMemExpReadWaitStates = 5;
1643     const int SMFMA16x16WriteVgprVALUMemExpReadWaitStates = 11;
1644     const int SMFMA32x32WriteVgprVALUMemExpReadWaitStates = 19;
1645     const int DMFMA4x4WriteVgprMemExpReadWaitStates = 9;
1646     const int DMFMA16x16WriteVgprMemExpReadWaitStates = 18;
1647     const int DMFMA4x4WriteVgprVALUReadWaitStates = 6;
1648     const int DMFMA16x16WriteVgprVALUReadWaitStates = 11;
1649     const int DotWriteSameDotReadSrcAB = 3;
1650     const int DotWriteDifferentVALURead = 3;
1651     const int MaxWaitStates = 19;
1652 
1653     for (const MachineOperand &Use : MI->explicit_uses()) {
1654       if (!Use.isReg())
1655         continue;
1656       Reg = Use.getReg();
1657 
1658       DOT = nullptr;
1659       int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn,
1660                                                      MaxWaitStates);
1661       if (DOT) {
1662         int NeedWaitStates = 0;
1663         if (DOT->getOpcode() == MI->getOpcode()) {
1664           if (&Use - &MI->getOperand(0) != SrcCIdx)
1665             NeedWaitStates = DotWriteSameDotReadSrcAB;
1666         } else {
1667           NeedWaitStates = DotWriteDifferentVALURead;
1668         }
1669 
1670         int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
1671         WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1672       }
1673 
1674       MFMA = nullptr;
1675       WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDGEMMorXDLWriteFn,
1676                                                  MaxWaitStates);
1677       if (!MFMA)
1678         continue;
1679 
1680       unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA);
1681       int NeedWaitStates = MaxWaitStates;
1682       switch (HazardDefLatency) {
1683       case 2:
1684         NeedWaitStates = SMFMA4x4WriteVgprVALUMemExpReadWaitStates;
1685         break;
1686       case 4:
1687         assert(isDGEMM(MFMA->getOpcode()));
1688         NeedWaitStates =
1689             IsMemOrExport ? DMFMA4x4WriteVgprMemExpReadWaitStates
1690                           : DMFMA4x4WriteVgprVALUReadWaitStates;
1691         break;
1692       case 8:
1693         NeedWaitStates = SMFMA16x16WriteVgprVALUMemExpReadWaitStates;
1694         break;
1695       case 16: LLVM_FALLTHROUGH;
1696       default:
1697         NeedWaitStates =
1698           isDGEMM(MFMA->getOpcode())
1699             ? IsMemOrExport ? DMFMA16x16WriteVgprMemExpReadWaitStates
1700                             : DMFMA16x16WriteVgprVALUReadWaitStates
1701             : SMFMA32x32WriteVgprVALUMemExpReadWaitStates;
1702         break;
1703       }
1704 
1705       int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
1706       WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1707 
1708       if (WaitStatesNeeded == MaxWaitStates)
1709         break;
1710     }
1711   }
1712 
1713   unsigned Opc = MI->getOpcode();
1714   const int DMFMAToFMA64WaitStates = 2;
1715   if ((Opc == AMDGPU::V_FMA_F64_e64 ||
1716        Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64 ||
1717        Opc == AMDGPU::V_FMAC_F64_dpp) &&
1718       WaitStatesNeeded < DMFMAToFMA64WaitStates) {
1719     int WaitStatesNeededForUse = DMFMAToFMA64WaitStates -
1720       getWaitStatesSince(IsDGEMMFn, DMFMAToFMA64WaitStates);
1721     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1722   }
1723 
1724   if (!IsVALU && !IsMemOrExport)
1725     return WaitStatesNeeded;
1726 
1727   for (const MachineOperand &Def : MI->defs()) {
1728     const int SMFMA4x4WriteVgprVALUWawWaitStates = 5;
1729     const int SMFMA16x16WriteVgprVALUWawWaitStates = 11;
1730     const int SMFMA32x32WriteVgprVALUWawWaitStates = 19;
1731     const int SMFMA4x4ReadVgprVALUWarWaitStates = 1;
1732     const int SMFMA16x16ReadVgprVALUWarWaitStates = 7;
1733     const int SMFMA32x32ReadVgprVALUWarWaitStates = 15;
1734     const int DMFMA4x4WriteVgprVALUWriteWaitStates = 6;
1735     const int DMFMA16x16WriteVgprVALUWriteWaitStates = 11;
1736     const int DotWriteDifferentVALUWrite = 3;
1737     const int MaxWaitStates = 19;
1738     const int MaxWarWaitStates = 15;
1739 
1740     Reg = Def.getReg();
1741 
1742     DOT = nullptr;
1743     int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn,
1744                                                    MaxWaitStates);
1745     if (DOT && DOT->getOpcode() != MI->getOpcode())
1746       WaitStatesNeeded = std::max(WaitStatesNeeded, DotWriteDifferentVALUWrite -
1747                                                     WaitStatesSinceDef);
1748 
1749     MFMA = nullptr;
1750     WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDGEMMorXDLWriteFn,
1751                                                MaxWaitStates);
1752     if (MFMA) {
1753       int NeedWaitStates = MaxWaitStates;
1754       switch (TSchedModel.computeInstrLatency(MFMA)) {
1755       case 2:
1756         NeedWaitStates = SMFMA4x4WriteVgprVALUWawWaitStates;
1757         break;
1758       case 4:
1759         assert(isDGEMM(MFMA->getOpcode()));
1760         NeedWaitStates = DMFMA4x4WriteVgprVALUWriteWaitStates;
1761         break;
1762       case 8:
1763         NeedWaitStates = SMFMA16x16WriteVgprVALUWawWaitStates;
1764         break;
1765       case 16: LLVM_FALLTHROUGH;
1766       default:
1767         NeedWaitStates = isDGEMM(MFMA->getOpcode())
1768                    ? DMFMA16x16WriteVgprVALUWriteWaitStates
1769                    : SMFMA32x32WriteVgprVALUWawWaitStates;
1770         break;
1771       }
1772 
1773       int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
1774       WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1775 
1776       if (WaitStatesNeeded == MaxWaitStates)
1777         break;
1778     }
1779 
1780     auto IsSMFMAReadAsCFn = [&Reg, &IsMFMAFn, &MFMA, this]
1781                                (MachineInstr *MI) {
1782       if (!IsMFMAFn(MI) || isDGEMM(MI->getOpcode()) ||
1783           !MI->readsRegister(Reg, &TRI))
1784         return false;
1785 
1786       MachineOperand *SrcC = TII.getNamedOperand(*MI, AMDGPU::OpName::src2);
1787       assert(SrcC);
1788       if (!SrcC->isReg() || !TRI.regsOverlap(SrcC->getReg(), Reg))
1789         return false;
1790 
1791       MFMA = MI;
1792       return true;
1793     };
1794 
1795     MFMA = nullptr;
1796     int WaitStatesSinceUse = getWaitStatesSince(IsSMFMAReadAsCFn,
1797                                                 MaxWarWaitStates);
1798     if (!MFMA)
1799       continue;
1800 
1801     unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA);
1802     int NeedWaitStates = MaxWaitStates;
1803     switch (HazardDefLatency) {
1804     case 2:  NeedWaitStates = SMFMA4x4ReadVgprVALUWarWaitStates;
1805              break;
1806     case 8:  NeedWaitStates = SMFMA16x16ReadVgprVALUWarWaitStates;
1807              break;
1808     case 16: LLVM_FALLTHROUGH;
1809     default: NeedWaitStates = SMFMA32x32ReadVgprVALUWarWaitStates;
1810              break;
1811     }
1812 
1813     int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceUse;
1814     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1815   }
1816 
1817   return WaitStatesNeeded;
1818 }
1819 
1820 bool GCNHazardRecognizer::ShouldPreferAnother(SUnit *SU) {
1821   if (!SU->isInstr())
1822     return false;
1823 
1824   MachineInstr *MAI = nullptr;
1825   auto IsMFMAFn = [&MAI] (MachineInstr *MI) {
1826     MAI = nullptr;
1827     if (SIInstrInfo::isMAI(*MI) &&
1828         MI->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
1829         MI->getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64)
1830       MAI = MI;
1831     return MAI != nullptr;
1832   };
1833 
1834   MachineInstr *MI = SU->getInstr();
1835   if (IsMFMAFn(MI)) {
1836     int W = getWaitStatesSince(IsMFMAFn, 16);
1837     if (MAI)
1838       return W < (int)TSchedModel.computeInstrLatency(MAI);
1839   }
1840 
1841   return false;
1842 }
1843