Lines Matching refs:MI
75 void GCNHazardRecognizer::EmitInstruction(MachineInstr *MI) { in EmitInstruction() argument
76 CurrCycleInstr = MI; in EmitInstruction()
122 static bool isXDL(const GCNSubtarget &ST, const MachineInstr &MI) { in isXDL() argument
123 unsigned Opcode = MI.getOpcode(); in isXDL()
125 if (!SIInstrInfo::isMAI(MI) || in isXDL()
138 const MachineInstr &MI) { in isSendMsgTraceDataOrGDS() argument
139 if (TII.isAlwaysGDS(MI.getOpcode())) in isSendMsgTraceDataOrGDS()
142 switch (MI.getOpcode()) { in isSendMsgTraceDataOrGDS()
153 if (TII.isDS(MI.getOpcode())) { in isSendMsgTraceDataOrGDS()
154 int GDS = AMDGPU::getNamedOperandIdx(MI.getOpcode(), in isSendMsgTraceDataOrGDS()
156 if (MI.getOperand(GDS).getImm()) in isSendMsgTraceDataOrGDS()
163 static bool isPermlane(const MachineInstr &MI) { in isPermlane() argument
164 unsigned Opcode = MI.getOpcode(); in isPermlane()
169 static bool isLdsDma(const MachineInstr &MI) { in isLdsDma() argument
170 return SIInstrInfo::isVALU(MI) && in isLdsDma()
171 (SIInstrInfo::isMUBUF(MI) || SIInstrInfo::isFLAT(MI)); in isLdsDma()
182 MachineInstr *MI = SU->getInstr(); in getHazardType() local
187 if (MI->isBundle()) in getHazardType()
190 if (SIInstrInfo::isSMRD(*MI) && checkSMRDHazards(MI) > 0) in getHazardType()
193 if (ST.hasNSAtoVMEMBug() && checkNSAtoVMEMHazard(MI) > 0) in getHazardType()
196 if (checkFPAtomicToDenormModeHazard(MI) > 0) in getHazardType()
203 if ((SIInstrInfo::isVMEM(*MI) || in getHazardType()
204 SIInstrInfo::isFLAT(*MI)) in getHazardType()
205 && checkVMEMHazards(MI) > 0) in getHazardType()
208 if (SIInstrInfo::isVALU(*MI) && checkVALUHazards(MI) > 0) in getHazardType()
211 if (SIInstrInfo::isDPP(*MI) && checkDPPHazards(MI) > 0) in getHazardType()
214 if (isDivFMas(MI->getOpcode()) && checkDivFMasHazards(MI) > 0) in getHazardType()
217 if (isRWLane(MI->getOpcode()) && checkRWLaneHazards(MI) > 0) in getHazardType()
220 if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) || in getHazardType()
221 SIInstrInfo::isFLAT(*MI) || SIInstrInfo::isDS(*MI) || in getHazardType()
222 SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0) in getHazardType()
225 if (isSGetReg(MI->getOpcode()) && checkGetRegHazards(MI) > 0) in getHazardType()
228 if (isSSetReg(MI->getOpcode()) && checkSetRegHazards(MI) > 0) in getHazardType()
231 if (isRFE(MI->getOpcode()) && checkRFEHazards(MI) > 0) in getHazardType()
235 (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode()))) || in getHazardType()
236 (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI)) || in getHazardType()
237 (ST.hasReadM0LdsDmaHazard() && isLdsDma(*MI)) || in getHazardType()
239 MI->readsRegister(AMDGPU::LDS_DIRECT))) && in getHazardType()
240 checkReadM0Hazards(MI) > 0) in getHazardType()
243 if (SIInstrInfo::isMAI(*MI) && checkMAIHazards(MI) > 0) in getHazardType()
246 if ((SIInstrInfo::isVMEM(*MI) || in getHazardType()
247 SIInstrInfo::isFLAT(*MI) || in getHazardType()
248 SIInstrInfo::isDS(*MI)) && checkMAILdStHazards(MI) > 0) in getHazardType()
251 if (MI->isInlineAsm() && checkInlineAsmHazards(MI) > 0) in getHazardType()
257 static void insertNoopsInBundle(MachineInstr *MI, const SIInstrInfo &TII, in insertNoopsInBundle() argument
262 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII.get(AMDGPU::S_NOP)) in insertNoopsInBundle()
268 GCNHazardRecognizer::getMFMAPipelineWaitStates(const MachineInstr &MI) const { in getMFMAPipelineWaitStates()
269 const MCSchedClassDesc *SC = TSchedModel.resolveSchedClass(&MI); in getMFMAPipelineWaitStates()
276 MachineBasicBlock::instr_iterator MI = std::next(CurrCycleInstr->getIterator()); in processBundle() local
279 for (; MI != E && MI->isInsideBundle(); ++MI) { in processBundle()
280 CurrCycleInstr = &*MI; in processBundle()
301 unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) { in PreEmitNoops() argument
303 CurrCycleInstr = MI; in PreEmitNoops()
304 unsigned W = PreEmitNoopsCommon(MI); in PreEmitNoops()
305 fixHazards(MI); in PreEmitNoops()
310 unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) { in PreEmitNoopsCommon() argument
311 if (MI->isBundle()) in PreEmitNoopsCommon()
316 if (SIInstrInfo::isSMRD(*MI)) in PreEmitNoopsCommon()
317 return std::max(WaitStates, checkSMRDHazards(MI)); in PreEmitNoopsCommon()
320 WaitStates = std::max(WaitStates, checkNSAtoVMEMHazard(MI)); in PreEmitNoopsCommon()
322 WaitStates = std::max(WaitStates, checkFPAtomicToDenormModeHazard(MI)); in PreEmitNoopsCommon()
327 if (SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isFLAT(*MI)) in PreEmitNoopsCommon()
328 WaitStates = std::max(WaitStates, checkVMEMHazards(MI)); in PreEmitNoopsCommon()
330 if (SIInstrInfo::isVALU(*MI)) in PreEmitNoopsCommon()
331 WaitStates = std::max(WaitStates, checkVALUHazards(MI)); in PreEmitNoopsCommon()
333 if (SIInstrInfo::isDPP(*MI)) in PreEmitNoopsCommon()
334 WaitStates = std::max(WaitStates, checkDPPHazards(MI)); in PreEmitNoopsCommon()
336 if (isDivFMas(MI->getOpcode())) in PreEmitNoopsCommon()
337 WaitStates = std::max(WaitStates, checkDivFMasHazards(MI)); in PreEmitNoopsCommon()
339 if (isRWLane(MI->getOpcode())) in PreEmitNoopsCommon()
340 WaitStates = std::max(WaitStates, checkRWLaneHazards(MI)); in PreEmitNoopsCommon()
342 if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) || in PreEmitNoopsCommon()
343 SIInstrInfo::isFLAT(*MI) || SIInstrInfo::isDS(*MI) || in PreEmitNoopsCommon()
344 SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0) in PreEmitNoopsCommon()
345 WaitStates = std::max(WaitStates, checkMAIVALUHazards(MI)); in PreEmitNoopsCommon()
347 if (MI->isInlineAsm()) in PreEmitNoopsCommon()
348 return std::max(WaitStates, checkInlineAsmHazards(MI)); in PreEmitNoopsCommon()
350 if (isSGetReg(MI->getOpcode())) in PreEmitNoopsCommon()
351 return std::max(WaitStates, checkGetRegHazards(MI)); in PreEmitNoopsCommon()
353 if (isSSetReg(MI->getOpcode())) in PreEmitNoopsCommon()
354 return std::max(WaitStates, checkSetRegHazards(MI)); in PreEmitNoopsCommon()
356 if (isRFE(MI->getOpcode())) in PreEmitNoopsCommon()
357 return std::max(WaitStates, checkRFEHazards(MI)); in PreEmitNoopsCommon()
360 (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode()))) || in PreEmitNoopsCommon()
361 (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI)) || in PreEmitNoopsCommon()
362 (ST.hasReadM0LdsDmaHazard() && isLdsDma(*MI)) || in PreEmitNoopsCommon()
363 (ST.hasReadM0LdsDirectHazard() && MI->readsRegister(AMDGPU::LDS_DIRECT))) in PreEmitNoopsCommon()
364 return std::max(WaitStates, checkReadM0Hazards(MI)); in PreEmitNoopsCommon()
366 if (SIInstrInfo::isMAI(*MI)) in PreEmitNoopsCommon()
367 return std::max(WaitStates, checkMAIHazards(MI)); in PreEmitNoopsCommon()
369 if (SIInstrInfo::isVMEM(*MI) || in PreEmitNoopsCommon()
370 SIInstrInfo::isFLAT(*MI) || in PreEmitNoopsCommon()
371 SIInstrInfo::isDS(*MI)) in PreEmitNoopsCommon()
372 return std::max(WaitStates, checkMAILdStHazards(MI)); in PreEmitNoopsCommon()
514 const MachineInstr *MI, IsExpiredFn IsExpired) { in getWaitStatesSince() argument
516 return getWaitStatesSince(IsHazard, MI->getParent(), in getWaitStatesSince()
517 std::next(MI->getReverseIterator()), in getWaitStatesSince()
530 for (MachineInstr *MI : EmittedInstrs) { in getWaitStatesSince()
531 if (MI) { in getWaitStatesSince()
532 if (IsHazard(*MI)) in getWaitStatesSince()
535 if (MI->isInlineAsm()) in getWaitStatesSince()
551 auto IsHazardFn = [IsHazardDef, TRI, Reg](const MachineInstr &MI) { in getWaitStatesSinceDef() argument
552 return IsHazardDef(MI) && MI.modifiesRegister(Reg, TRI); in getWaitStatesSinceDef()
560 auto IsHazardFn = [IsHazard](const MachineInstr &MI) { in getWaitStatesSinceSetReg() argument
561 return isSSetReg(MI.getOpcode()) && IsHazard(MI); in getWaitStatesSinceSetReg()
586 void GCNHazardRecognizer::addClauseInst(const MachineInstr &MI) { in addClauseInst() argument
588 addRegsToSet(TRI, MI.defs(), ClauseDefs); in addClauseInst()
589 addRegsToSet(TRI, MI.uses(), ClauseUses); in addClauseInst()
592 static bool breaksSMEMSoftClause(MachineInstr *MI) { in breaksSMEMSoftClause() argument
593 return !SIInstrInfo::isSMRD(*MI); in breaksSMEMSoftClause()
596 static bool breaksVMEMSoftClause(MachineInstr *MI) { in breaksVMEMSoftClause() argument
597 return !SIInstrInfo::isVMEM(*MI) && !SIInstrInfo::isFLAT(*MI); in breaksVMEMSoftClause()
620 for (MachineInstr *MI : EmittedInstrs) { in checkSoftClauseHazards()
623 if (!MI) in checkSoftClauseHazards()
626 if (IsSMRD ? breaksSMEMSoftClause(MI) : breaksVMEMSoftClause(MI)) in checkSoftClauseHazards()
629 addClauseInst(*MI); in checkSoftClauseHazards()
660 auto IsHazardDefFn = [this](const MachineInstr &MI) { in checkSMRDHazards() argument
661 return TII.isVALU(MI); in checkSMRDHazards()
663 auto IsBufferHazardDefFn = [this](const MachineInstr &MI) { in checkSMRDHazards() argument
664 return TII.isSALU(MI); in checkSMRDHazards()
705 auto IsHazardDefFn = [this](const MachineInstr &MI) { in checkVMEMHazards() argument
706 return TII.isVALU(MI); in checkVMEMHazards()
728 auto IsHazardDefFn = [TII](const MachineInstr &MI) { in checkDPPHazards() argument
729 return TII->isVALU(MI); in checkDPPHazards()
757 auto IsHazardDefFn = [TII](const MachineInstr &MI) { in checkDivFMasHazards() argument
758 return TII->isVALU(MI); in checkDivFMasHazards()
771 auto IsHazardFn = [TII, GetRegHWReg](const MachineInstr &MI) { in checkGetRegHazards() argument
772 return GetRegHWReg == getHWReg(TII, MI); in checkGetRegHazards()
784 auto IsHazardFn = [TII, HWReg](const MachineInstr &MI) { in checkSetRegHazards() argument
785 return HWReg == getHWReg(TII, MI); in checkSetRegHazards()
791 int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) { in createsVALUHazard() argument
792 if (!MI.mayStore()) in createsVALUHazard()
796 unsigned Opcode = MI.getOpcode(); in createsVALUHazard()
797 const MCInstrDesc &Desc = MI.getDesc(); in createsVALUHazard()
804 if (TII->isMUBUF(MI) || TII->isMTBUF(MI)) { in createsVALUHazard()
812 TII->getNamedOperand(MI, AMDGPU::OpName::soffset); in createsVALUHazard()
824 if (TII->isMIMG(MI)) { in createsVALUHazard()
831 if (TII->isFLAT(MI)) { in createsVALUHazard()
853 auto IsHazardFn = [this, Reg, TRI](const MachineInstr &MI) { in checkVALUHazardsHelper() argument
854 int DataIdx = createsVALUHazard(MI); in checkVALUHazardsHelper()
856 TRI->regsOverlap(MI.getOperand(DataIdx).getReg(), Reg); in checkVALUHazardsHelper()
871 auto IsTransDefFn = [this, VALU](const MachineInstr &MI) { in checkVALUHazards() argument
872 if (!SIInstrInfo::isTRANS(MI)) in checkVALUHazards()
876 Register Def = TII->getNamedOperand(MI, AMDGPU::OpName::vdst)->getReg(); in checkVALUHazards()
895 auto IsShift16BitDefFn = [this, VALU](const MachineInstr &MI) { in checkVALUHazards() argument
896 if (!SIInstrInfo::isVALU(MI)) in checkVALUHazards()
899 if (SIInstrInfo::isSDWA(MI)) { in checkVALUHazards()
900 if (auto *DstSel = TII->getNamedOperand(MI, AMDGPU::OpName::dst_sel)) in checkVALUHazards()
904 if ((AMDGPU::getNamedOperandIdx(MI.getOpcode(), in checkVALUHazards()
906 !(TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers) in checkVALUHazards()
912 if (auto *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst)) { in checkVALUHazards()
938 auto IsVALUDefSGPRFn = [&UseReg, TRI](const MachineInstr &MI) { in checkVALUHazards() argument
939 if (!SIInstrInfo::isVALU(MI)) in checkVALUHazards()
941 return MI.modifiesRegister(UseReg, TRI); in checkVALUHazards()
1042 auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isVALU(MI); }; in checkRWLaneHazards() argument
1058 auto IsHazardFn = [TII](const MachineInstr &MI) { in checkRFEHazards() argument
1059 return getHWReg(TII, MI) == AMDGPU::Hwreg::ID_TRAPSTS; in checkRFEHazards()
1065 int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) { in checkReadM0Hazards() argument
1068 auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isSALU(MI); }; in checkReadM0Hazards() argument
1073 void GCNHazardRecognizer::fixHazards(MachineInstr *MI) { in fixHazards() argument
1074 fixVMEMtoScalarWriteHazards(MI); in fixHazards()
1075 fixVcmpxPermlaneHazards(MI); in fixHazards()
1076 fixSMEMtoVectorWriteHazards(MI); in fixHazards()
1077 fixVcmpxExecWARHazard(MI); in fixHazards()
1078 fixLdsBranchVmemWARHazard(MI); in fixHazards()
1080 fixLdsDirectVALUHazard(MI); in fixHazards()
1081 fixLdsDirectVMEMHazard(MI); in fixHazards()
1083 fixVALUPartialForwardingHazard(MI); in fixHazards()
1084 fixVALUTransUseHazard(MI); in fixHazards()
1085 fixWMMAHazards(MI); in fixHazards()
1088 bool GCNHazardRecognizer::fixVcmpxPermlaneHazards(MachineInstr *MI) { in fixVcmpxPermlaneHazards() argument
1089 if (!ST.hasVcmpxPermlaneHazard() || !isPermlane(*MI)) in fixVcmpxPermlaneHazards()
1094 auto IsHazardFn = [TII, TRI](const MachineInstr &MI) { in fixVcmpxPermlaneHazards() argument
1095 return (TII->isVOPC(MI) || in fixVcmpxPermlaneHazards()
1096 ((TII->isVOP3(MI) || TII->isSDWA(MI)) && MI.isCompare())) && in fixVcmpxPermlaneHazards()
1097 MI.modifiesRegister(AMDGPU::EXEC, TRI); in fixVcmpxPermlaneHazards()
1100 auto IsExpiredFn = [](const MachineInstr &MI, int) { in fixVcmpxPermlaneHazards() argument
1101 unsigned Opc = MI.getOpcode(); in fixVcmpxPermlaneHazards()
1102 return SIInstrInfo::isVALU(MI) && Opc != AMDGPU::V_NOP_e32 && in fixVcmpxPermlaneHazards()
1106 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == in fixVcmpxPermlaneHazards()
1113 auto *Src0 = TII->getNamedOperand(*MI, AMDGPU::OpName::src0); in fixVcmpxPermlaneHazards()
1116 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), in fixVcmpxPermlaneHazards()
1124 bool GCNHazardRecognizer::fixVMEMtoScalarWriteHazards(MachineInstr *MI) { in fixVMEMtoScalarWriteHazards() argument
1128 if (!SIInstrInfo::isSALU(*MI) && !SIInstrInfo::isSMRD(*MI)) in fixVMEMtoScalarWriteHazards()
1131 if (MI->getNumDefs() == 0) in fixVMEMtoScalarWriteHazards()
1136 auto IsHazardFn = [TRI, MI](const MachineInstr &I) { in fixVMEMtoScalarWriteHazards()
1141 for (const MachineOperand &Def : MI->defs()) { in fixVMEMtoScalarWriteHazards()
1151 auto IsExpiredFn = [](const MachineInstr &MI, int) { in fixVMEMtoScalarWriteHazards() argument
1152 return SIInstrInfo::isVALU(MI) || in fixVMEMtoScalarWriteHazards()
1153 (MI.getOpcode() == AMDGPU::S_WAITCNT && in fixVMEMtoScalarWriteHazards()
1154 !MI.getOperand(0).getImm()) || in fixVMEMtoScalarWriteHazards()
1155 (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && in fixVMEMtoScalarWriteHazards()
1156 MI.getOperand(0).getImm() == 0xffe3); in fixVMEMtoScalarWriteHazards()
1159 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == in fixVMEMtoScalarWriteHazards()
1164 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), in fixVMEMtoScalarWriteHazards()
1170 bool GCNHazardRecognizer::fixSMEMtoVectorWriteHazards(MachineInstr *MI) { in fixSMEMtoVectorWriteHazards() argument
1174 if (!SIInstrInfo::isVALU(*MI)) in fixSMEMtoVectorWriteHazards()
1178 switch (MI->getOpcode()) { in fixSMEMtoVectorWriteHazards()
1191 const MachineOperand *SDST = TII->getNamedOperand(*MI, SDSTName); in fixSMEMtoVectorWriteHazards()
1193 for (const auto &MO : MI->implicit_operands()) { in fixSMEMtoVectorWriteHazards()
1209 auto IsExpiredFn = [TII, IV](const MachineInstr &MI, int) { in fixSMEMtoVectorWriteHazards() argument
1210 if (TII->isSALU(MI)) { in fixSMEMtoVectorWriteHazards()
1211 switch (MI.getOpcode()) { in fixSMEMtoVectorWriteHazards()
1221 return (MI.getOperand(1).getImm() == 0) && in fixSMEMtoVectorWriteHazards()
1222 (MI.getOperand(0).getReg() == AMDGPU::SGPR_NULL); in fixSMEMtoVectorWriteHazards()
1224 const int64_t Imm = MI.getOperand(0).getImm(); in fixSMEMtoVectorWriteHazards()
1230 if (TII->isSOPP(MI)) in fixSMEMtoVectorWriteHazards()
1245 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == in fixSMEMtoVectorWriteHazards()
1249 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), in fixSMEMtoVectorWriteHazards()
1255 bool GCNHazardRecognizer::fixVcmpxExecWARHazard(MachineInstr *MI) { in fixVcmpxExecWARHazard() argument
1256 if (!ST.hasVcmpxExecWARHazard() || !SIInstrInfo::isVALU(*MI)) in fixVcmpxExecWARHazard()
1260 if (!MI->modifiesRegister(AMDGPU::EXEC, TRI)) in fixVcmpxExecWARHazard()
1270 auto IsExpiredFn = [TII, TRI](const MachineInstr &MI, int) { in fixVcmpxExecWARHazard() argument
1271 if (SIInstrInfo::isVALU(MI)) { in fixVcmpxExecWARHazard()
1272 if (TII->getNamedOperand(MI, AMDGPU::OpName::sdst)) in fixVcmpxExecWARHazard()
1274 for (auto MO : MI.implicit_operands()) in fixVcmpxExecWARHazard()
1278 if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && in fixVcmpxExecWARHazard()
1279 (MI.getOperand(0).getImm() & 0xfffe) == 0xfffe) in fixVcmpxExecWARHazard()
1284 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == in fixVcmpxExecWARHazard()
1288 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), in fixVcmpxExecWARHazard()
1304 for (auto &MI : MBB) { in shouldRunLdsBranchVmemWARHazardFixup() local
1305 HasLds |= SIInstrInfo::isDS(MI); in shouldRunLdsBranchVmemWARHazardFixup()
1307 SIInstrInfo::isVMEM(MI) || SIInstrInfo::isSegmentSpecificFLAT(MI); in shouldRunLdsBranchVmemWARHazardFixup()
1315 bool GCNHazardRecognizer::fixLdsBranchVmemWARHazard(MachineInstr *MI) { in fixLdsBranchVmemWARHazard() argument
1321 auto IsHazardInst = [](const MachineInstr &MI) { in fixLdsBranchVmemWARHazard() argument
1322 if (SIInstrInfo::isDS(MI)) in fixLdsBranchVmemWARHazard()
1324 if (SIInstrInfo::isVMEM(MI) || SIInstrInfo::isSegmentSpecificFLAT(MI)) in fixLdsBranchVmemWARHazard()
1329 auto InstType = IsHazardInst(*MI); in fixLdsBranchVmemWARHazard()
1362 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == in fixLdsBranchVmemWARHazard()
1367 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), in fixLdsBranchVmemWARHazard()
1375 bool GCNHazardRecognizer::fixLdsDirectVALUHazard(MachineInstr *MI) { in fixLdsDirectVALUHazard() argument
1376 if (!SIInstrInfo::isLDSDIR(*MI)) in fixLdsDirectVALUHazard()
1380 const MachineOperand *VDST = TII.getNamedOperand(*MI, AMDGPU::OpName::vdst); in fixLdsDirectVALUHazard()
1398 auto GetWaitStatesFn = [](const MachineInstr &MI) { in fixLdsDirectVALUHazard() argument
1399 return SIInstrInfo::isVALU(MI) ? 1 : 0; in fixLdsDirectVALUHazard()
1403 auto Count = ::getWaitStatesSince(IsHazardFn, MI->getParent(), in fixLdsDirectVALUHazard()
1404 std::next(MI->getReverseIterator()), 0, in fixLdsDirectVALUHazard()
1413 TII.getNamedOperand(*MI, AMDGPU::OpName::waitvdst); in fixLdsDirectVALUHazard()
1419 bool GCNHazardRecognizer::fixLdsDirectVMEMHazard(MachineInstr *MI) { in fixLdsDirectVMEMHazard() argument
1420 if (!SIInstrInfo::isLDSDIR(*MI)) in fixLdsDirectVMEMHazard()
1423 const MachineOperand *VDST = TII.getNamedOperand(*MI, AMDGPU::OpName::vdst); in fixLdsDirectVMEMHazard()
1439 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == in fixLdsDirectVMEMHazard()
1443 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), in fixLdsDirectVMEMHazard()
1450 bool GCNHazardRecognizer::fixVALUPartialForwardingHazard(MachineInstr *MI) { in fixVALUPartialForwardingHazard() argument
1455 if (!SIInstrInfo::isVALU(*MI)) in fixVALUPartialForwardingHazard()
1460 for (const MachineOperand &Use : MI->explicit_uses()) { in fixVALUPartialForwardingHazard()
1582 auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) { in fixVALUPartialForwardingHazard() argument
1583 if (SIInstrInfo::isVALU(MI)) in fixVALUPartialForwardingHazard()
1588 if (!hasHazard<StateType>(State, IsHazardFn, UpdateStateFn, MI->getParent(), in fixVALUPartialForwardingHazard()
1589 std::next(MI->getReverseIterator()), Visited)) in fixVALUPartialForwardingHazard()
1592 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), in fixVALUPartialForwardingHazard()
1599 bool GCNHazardRecognizer::fixVALUTransUseHazard(MachineInstr *MI) { in fixVALUTransUseHazard() argument
1602 if (!SIInstrInfo::isVALU(*MI)) in fixVALUTransUseHazard()
1607 for (const MachineOperand &Use : MI->explicit_uses()) { in fixVALUTransUseHazard()
1656 auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) { in fixVALUTransUseHazard() argument
1657 if (SIInstrInfo::isVALU(MI)) in fixVALUTransUseHazard()
1659 if (SIInstrInfo::isTRANS(MI)) in fixVALUTransUseHazard()
1664 if (!hasHazard<StateType>(State, IsHazardFn, UpdateStateFn, MI->getParent(), in fixVALUTransUseHazard()
1665 std::next(MI->getReverseIterator()), Visited)) in fixVALUTransUseHazard()
1670 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), in fixVALUTransUseHazard()
1677 bool GCNHazardRecognizer::fixWMMAHazards(MachineInstr *MI) { in fixWMMAHazards() argument
1678 if (!SIInstrInfo::isWMMA(*MI)) in fixWMMAHazards()
1684 auto IsHazardFn = [MI, TII, TRI](const MachineInstr &I) { in fixWMMAHazards()
1691 TII->getNamedOperand(*MI, AMDGPU::OpName::src0)->getReg(); in fixWMMAHazards()
1693 TII->getNamedOperand(*MI, AMDGPU::OpName::src1)->getReg(); in fixWMMAHazards()
1706 TII->getNamedOperand(*MI, AMDGPU::OpName::src2); in fixWMMAHazards()
1713 TII->getNamedOperand(*MI, AMDGPU::OpName::src2_modifiers); in fixWMMAHazards()
1719 TII->pseudoToMCOpcode(MI->getOpcode()))); in fixWMMAHazards()
1729 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == in fixWMMAHazards()
1733 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII->get(AMDGPU::V_NOP_e32)); in fixWMMAHazards()
1738 int GCNHazardRecognizer::checkNSAtoVMEMHazard(MachineInstr *MI) { in checkNSAtoVMEMHazard() argument
1744 if (!SIInstrInfo::isMUBUF(*MI) && !SIInstrInfo::isMTBUF(*MI)) in checkNSAtoVMEMHazard()
1748 const auto *Offset = TII->getNamedOperand(*MI, AMDGPU::OpName::offset); in checkNSAtoVMEMHazard()
1763 int GCNHazardRecognizer::checkFPAtomicToDenormModeHazard(MachineInstr *MI) { in checkFPAtomicToDenormModeHazard() argument
1766 if (MI->getOpcode() != AMDGPU::S_DENORM_MODE) in checkFPAtomicToDenormModeHazard()
1775 auto IsExpiredFn = [](const MachineInstr &MI, int WaitStates) { in checkFPAtomicToDenormModeHazard() argument
1776 if (WaitStates >= 3 || SIInstrInfo::isVALU(MI)) in checkFPAtomicToDenormModeHazard()
1779 switch (MI.getOpcode()) { in checkFPAtomicToDenormModeHazard()
1795 ::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn); in checkFPAtomicToDenormModeHazard()
1798 int GCNHazardRecognizer::checkMAIHazards(MachineInstr *MI) { in checkMAIHazards() argument
1799 assert(SIInstrInfo::isMAI(*MI)); in checkMAIHazards()
1801 return ST.hasGFX90AInsts() ? checkMAIHazards90A(MI) : checkMAIHazards908(MI); in checkMAIHazards()
1804 int GCNHazardRecognizer::checkMFMAPadding(MachineInstr *MI) { in checkMFMAPadding() argument
1810 if (!SIInstrInfo::isMFMA(*MI) || MFI->getOccupancy() < 2) in checkMFMAPadding()
1815 this](const MachineInstr &MI) { in checkMFMAPadding() argument
1816 if (!SIInstrInfo::isMFMA(MI)) in checkMFMAPadding()
1819 NeighborMFMALatency = this->getMFMAPipelineWaitStates(MI); in checkMFMAPadding()
1834 int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) { in checkMAIHazards908() argument
1836 unsigned Opc = MI->getOpcode(); in checkMAIHazards908()
1838 auto IsVALUFn = [](const MachineInstr &MI) { in checkMAIHazards908() argument
1839 return SIInstrInfo::isVALU(MI); in checkMAIHazards908()
1852 for (const MachineOperand &Use : MI->explicit_uses()) { in checkMAIHazards908()
1868 for (const MachineOperand &Op : MI->explicit_operands()) { in checkMAIHazards908()
1888 this](const MachineInstr &MI) { in checkMAIHazards908() argument
1889 if (!SIInstrInfo::isMFMA(MI)) in checkMAIHazards908()
1891 Register DstReg = MI.getOperand(0).getReg(); in checkMAIHazards908()
1895 std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI)); in checkMAIHazards908()
1903 int OpNo = MI->getOperandNo(&Op); in checkMAIHazards908()
1934 auto IsAccVgprWriteFn = [Reg, this](const MachineInstr &MI) { in checkMAIHazards908() argument
1935 if (MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64) in checkMAIHazards908()
1937 Register DstReg = MI.getOperand(0).getReg(); in checkMAIHazards908()
1963 Register DstReg = MI->getOperand(0).getReg(); in checkMAIHazards908()
1967 this](const MachineInstr &MI) { in checkMAIHazards908() argument
1968 if (!SIInstrInfo::isMFMA(MI)) in checkMAIHazards908()
1970 Register Reg = TII.getNamedOperand(MI, AMDGPU::OpName::src2)->getReg(); in checkMAIHazards908()
1972 std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI)); in checkMAIHazards908()
1993 WaitStatesNeeded = std::max(WaitStatesNeeded, checkMFMAPadding(MI)); in checkMAIHazards908()
1998 int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) { in checkMAIHazards90A() argument
2000 unsigned Opc = MI->getOpcode(); in checkMAIHazards90A()
2002 auto IsLegacyVALUFn = [](const MachineInstr &MI) { in checkMAIHazards90A() argument
2003 return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMFMA(MI); in checkMAIHazards90A()
2006 auto IsLegacyVALUNotDotFn = [](const MachineInstr &MI) { in checkMAIHazards90A() argument
2007 return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMFMA(MI) && in checkMAIHazards90A()
2008 !SIInstrInfo::isDOT(MI); in checkMAIHazards90A()
2011 if (!SIInstrInfo::isMFMA(*MI)) in checkMAIHazards90A()
2023 for (const MachineOperand &Use : MI->explicit_uses()) { in checkMAIHazards90A()
2064 this](const MachineInstr &MI) { in checkMAIHazards90A() argument
2065 if (!SIInstrInfo::isMFMA(MI)) in checkMAIHazards90A()
2067 Register DstReg = MI.getOperand(0).getReg(); in checkMAIHazards90A()
2069 MI1 = &MI; in checkMAIHazards90A()
2082 int OpNo = MI->getOperandNo(&Use); in checkMAIHazards90A()
2103 if (!isXDL(ST, *MI)) in checkMAIHazards90A()
2108 if (!isXDL(ST, *MI)) in checkMAIHazards90A()
2112 if (ST.hasGFX940Insts() && isXDL(ST, *MI) && !isXDL(ST, *MI1)) in checkMAIHazards90A()
2208 int GCNHazardRecognizer::checkMAILdStHazards(MachineInstr *MI) { in checkMAILdStHazards() argument
2215 auto IsAccVgprReadFn = [](const MachineInstr &MI) { in checkMAILdStHazards() argument
2216 return MI.getOpcode() == AMDGPU::V_ACCVGPR_READ_B32_e64; in checkMAILdStHazards()
2219 for (const MachineOperand &Op : MI->explicit_uses()) { in checkMAILdStHazards()
2236 auto IsVALUAccVgprRdWrCheckFn = [Reg, this](const MachineInstr &MI) { in checkMAILdStHazards() argument
2237 if (MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64 && in checkMAILdStHazards()
2238 MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64) in checkMAILdStHazards()
2240 auto IsVALUFn = [](const MachineInstr &MI) { in checkMAILdStHazards() argument
2241 return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMAI(MI); in checkMAILdStHazards()
2255 int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) { in checkMAIVALUHazards() argument
2259 auto IsDGEMMFn = [](const MachineInstr &MI) -> bool { in checkMAIVALUHazards() argument
2260 return isDGEMM(MI.getOpcode()); in checkMAIVALUHazards()
2264 if (SIInstrInfo::isMFMA(*MI)) in checkMAIVALUHazards()
2269 bool IsMemOrExport = SIInstrInfo::isVMEM(*MI) || in checkMAIVALUHazards()
2270 SIInstrInfo::isFLAT(*MI) || in checkMAIVALUHazards()
2271 SIInstrInfo::isDS(*MI) || in checkMAIVALUHazards()
2272 SIInstrInfo::isEXP(*MI); in checkMAIVALUHazards()
2273 bool IsVALU = SIInstrInfo::isVALU(*MI); in checkMAIVALUHazards()
2277 auto IsMFMAWriteFn = [&Reg, &MFMA, this](const MachineInstr &MI) { in checkMAIVALUHazards() argument
2278 if (!SIInstrInfo::isMFMA(MI) || in checkMAIVALUHazards()
2279 !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg)) in checkMAIVALUHazards()
2281 MFMA = &MI; in checkMAIVALUHazards()
2286 auto IsDotWriteFn = [&Reg, &DOT, this](const MachineInstr &MI) { in checkMAIVALUHazards() argument
2287 if (!SIInstrInfo::isDOT(MI) || in checkMAIVALUHazards()
2288 !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg)) in checkMAIVALUHazards()
2290 DOT = &MI; in checkMAIVALUHazards()
2294 int SrcCIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), in checkMAIVALUHazards()
2317 for (const MachineOperand &Use : MI->explicit_uses()) { in checkMAIVALUHazards()
2327 if (DOT->getOpcode() == MI->getOpcode()) { in checkMAIVALUHazards()
2328 if (&Use - &MI->getOperand(0) != SrcCIdx) in checkMAIVALUHazards()
2395 unsigned Opc = MI->getOpcode(); in checkMAIVALUHazards()
2409 for (const MachineOperand &Def : MI->defs()) { in checkMAIVALUHazards()
2436 if (DOT && DOT->getOpcode() != MI->getOpcode()) in checkMAIVALUHazards()
2487 auto IsSMFMAReadAsCFn = [&Reg, &MFMA, this](const MachineInstr &MI) { in checkMAIVALUHazards() argument
2488 if (!SIInstrInfo::isMFMA(MI) || isDGEMM(MI.getOpcode()) || in checkMAIVALUHazards()
2489 !MI.readsRegister(Reg, &TRI)) in checkMAIVALUHazards()
2492 if (ST.hasGFX940Insts() && !isXDL(ST, MI)) in checkMAIVALUHazards()
2496 TII.getNamedOperand(MI, AMDGPU::OpName::src2); in checkMAIVALUHazards()
2501 MFMA = &MI; in checkMAIVALUHazards()
2539 auto IsMFMAFn = [&MAI](const MachineInstr &MI) { in ShouldPreferAnother() argument
2541 if (SIInstrInfo::isMFMA(MI)) in ShouldPreferAnother()
2542 MAI = &MI; in ShouldPreferAnother()
2546 MachineInstr *MI = SU->getInstr(); in ShouldPreferAnother() local
2547 if (IsMFMAFn(*MI)) { in ShouldPreferAnother()