Lines Matching refs:MI
75 void GCNHazardRecognizer::EmitInstruction(MachineInstr *MI) { in EmitInstruction() argument
76 CurrCycleInstr = MI; in EmitInstruction()
122 static bool isXDL(const GCNSubtarget &ST, const MachineInstr &MI) { in isXDL() argument
123 unsigned Opcode = MI.getOpcode(); in isXDL()
125 if (!SIInstrInfo::isMAI(MI) || in isXDL()
138 const MachineInstr &MI) { in isSendMsgTraceDataOrGDS() argument
139 if (TII.isAlwaysGDS(MI.getOpcode())) in isSendMsgTraceDataOrGDS()
142 switch (MI.getOpcode()) { in isSendMsgTraceDataOrGDS()
153 if (TII.isDS(MI.getOpcode())) { in isSendMsgTraceDataOrGDS()
154 int GDS = AMDGPU::getNamedOperandIdx(MI.getOpcode(), in isSendMsgTraceDataOrGDS()
156 if (MI.getOperand(GDS).getImm()) in isSendMsgTraceDataOrGDS()
163 static bool isPermlane(const MachineInstr &MI) { in isPermlane() argument
164 unsigned Opcode = MI.getOpcode(); in isPermlane()
172 static bool isLdsDma(const MachineInstr &MI) { in isLdsDma() argument
173 return SIInstrInfo::isVALU(MI) && in isLdsDma()
174 (SIInstrInfo::isMUBUF(MI) || SIInstrInfo::isFLAT(MI)); in isLdsDma()
185 MachineInstr *MI = SU->getInstr(); in getHazardType() local
190 if (MI->isBundle()) in getHazardType()
193 if (SIInstrInfo::isSMRD(*MI) && checkSMRDHazards(MI) > 0) in getHazardType()
196 if (ST.hasNSAtoVMEMBug() && checkNSAtoVMEMHazard(MI) > 0) in getHazardType()
199 if (checkFPAtomicToDenormModeHazard(MI) > 0) in getHazardType()
206 if ((SIInstrInfo::isVMEM(*MI) || in getHazardType()
207 SIInstrInfo::isFLAT(*MI)) in getHazardType()
208 && checkVMEMHazards(MI) > 0) in getHazardType()
211 if (SIInstrInfo::isVALU(*MI) && checkVALUHazards(MI) > 0) in getHazardType()
214 if (SIInstrInfo::isDPP(*MI) && checkDPPHazards(MI) > 0) in getHazardType()
217 if (isDivFMas(MI->getOpcode()) && checkDivFMasHazards(MI) > 0) in getHazardType()
220 if (isRWLane(MI->getOpcode()) && checkRWLaneHazards(MI) > 0) in getHazardType()
223 if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) || in getHazardType()
224 SIInstrInfo::isFLAT(*MI) || SIInstrInfo::isDS(*MI) || in getHazardType()
225 SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0) in getHazardType()
228 if (isSGetReg(MI->getOpcode()) && checkGetRegHazards(MI) > 0) in getHazardType()
231 if (isSSetReg(MI->getOpcode()) && checkSetRegHazards(MI) > 0) in getHazardType()
234 if (isRFE(MI->getOpcode()) && checkRFEHazards(MI) > 0) in getHazardType()
238 (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode()) || in getHazardType()
239 MI->getOpcode() == AMDGPU::DS_WRITE_ADDTID_B32 || in getHazardType()
240 MI->getOpcode() == AMDGPU::DS_READ_ADDTID_B32)) || in getHazardType()
241 (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI)) || in getHazardType()
242 (ST.hasReadM0LdsDmaHazard() && isLdsDma(*MI)) || in getHazardType()
244 MI->readsRegister(AMDGPU::LDS_DIRECT))) && in getHazardType()
245 checkReadM0Hazards(MI) > 0) in getHazardType()
248 if (SIInstrInfo::isMAI(*MI) && checkMAIHazards(MI) > 0) in getHazardType()
251 if ((SIInstrInfo::isVMEM(*MI) || in getHazardType()
252 SIInstrInfo::isFLAT(*MI) || in getHazardType()
253 SIInstrInfo::isDS(*MI)) && checkMAILdStHazards(MI) > 0) in getHazardType()
256 if (MI->isInlineAsm() && checkInlineAsmHazards(MI) > 0) in getHazardType()
262 static void insertNoopsInBundle(MachineInstr *MI, const SIInstrInfo &TII, in insertNoopsInBundle() argument
267 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII.get(AMDGPU::S_NOP)) in insertNoopsInBundle()
273 GCNHazardRecognizer::getMFMAPipelineWaitStates(const MachineInstr &MI) const { in getMFMAPipelineWaitStates()
274 const MCSchedClassDesc *SC = TSchedModel.resolveSchedClass(&MI); in getMFMAPipelineWaitStates()
281 MachineBasicBlock::instr_iterator MI = std::next(CurrCycleInstr->getIterator()); in processBundle() local
284 for (; MI != E && MI->isInsideBundle(); ++MI) { in processBundle()
285 CurrCycleInstr = &*MI; in processBundle()
306 void GCNHazardRecognizer::runOnInstruction(MachineInstr *MI) { in runOnInstruction() argument
309 unsigned NumPreNoops = PreEmitNoops(MI); in runOnInstruction()
311 if (MI->isInsideBundle()) in runOnInstruction()
312 insertNoopsInBundle(MI, TII, NumPreNoops); in runOnInstruction()
314 TII.insertNoops(*MI->getParent(), MachineBasicBlock::iterator(MI), in runOnInstruction()
316 EmitInstruction(MI); in runOnInstruction()
320 unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) { in PreEmitNoops() argument
322 CurrCycleInstr = MI; in PreEmitNoops()
323 unsigned W = PreEmitNoopsCommon(MI); in PreEmitNoops()
324 fixHazards(MI); in PreEmitNoops()
329 unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) { in PreEmitNoopsCommon() argument
330 if (MI->isBundle()) in PreEmitNoopsCommon()
335 if (SIInstrInfo::isSMRD(*MI)) in PreEmitNoopsCommon()
336 return std::max(WaitStates, checkSMRDHazards(MI)); in PreEmitNoopsCommon()
339 WaitStates = std::max(WaitStates, checkNSAtoVMEMHazard(MI)); in PreEmitNoopsCommon()
341 WaitStates = std::max(WaitStates, checkFPAtomicToDenormModeHazard(MI)); in PreEmitNoopsCommon()
346 if (SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isFLAT(*MI)) in PreEmitNoopsCommon()
347 WaitStates = std::max(WaitStates, checkVMEMHazards(MI)); in PreEmitNoopsCommon()
349 if (SIInstrInfo::isVALU(*MI)) in PreEmitNoopsCommon()
350 WaitStates = std::max(WaitStates, checkVALUHazards(MI)); in PreEmitNoopsCommon()
352 if (SIInstrInfo::isDPP(*MI)) in PreEmitNoopsCommon()
353 WaitStates = std::max(WaitStates, checkDPPHazards(MI)); in PreEmitNoopsCommon()
355 if (isDivFMas(MI->getOpcode())) in PreEmitNoopsCommon()
356 WaitStates = std::max(WaitStates, checkDivFMasHazards(MI)); in PreEmitNoopsCommon()
358 if (isRWLane(MI->getOpcode())) in PreEmitNoopsCommon()
359 WaitStates = std::max(WaitStates, checkRWLaneHazards(MI)); in PreEmitNoopsCommon()
361 if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) || in PreEmitNoopsCommon()
362 SIInstrInfo::isFLAT(*MI) || SIInstrInfo::isDS(*MI) || in PreEmitNoopsCommon()
363 SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0) in PreEmitNoopsCommon()
364 WaitStates = std::max(WaitStates, checkMAIVALUHazards(MI)); in PreEmitNoopsCommon()
366 if (MI->isInlineAsm()) in PreEmitNoopsCommon()
367 return std::max(WaitStates, checkInlineAsmHazards(MI)); in PreEmitNoopsCommon()
369 if (isSGetReg(MI->getOpcode())) in PreEmitNoopsCommon()
370 return std::max(WaitStates, checkGetRegHazards(MI)); in PreEmitNoopsCommon()
372 if (isSSetReg(MI->getOpcode())) in PreEmitNoopsCommon()
373 return std::max(WaitStates, checkSetRegHazards(MI)); in PreEmitNoopsCommon()
375 if (isRFE(MI->getOpcode())) in PreEmitNoopsCommon()
376 return std::max(WaitStates, checkRFEHazards(MI)); in PreEmitNoopsCommon()
379 (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode()) || in PreEmitNoopsCommon()
380 MI->getOpcode() == AMDGPU::DS_WRITE_ADDTID_B32 || in PreEmitNoopsCommon()
381 MI->getOpcode() == AMDGPU::DS_READ_ADDTID_B32)) || in PreEmitNoopsCommon()
382 (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI)) || in PreEmitNoopsCommon()
383 (ST.hasReadM0LdsDmaHazard() && isLdsDma(*MI)) || in PreEmitNoopsCommon()
384 (ST.hasReadM0LdsDirectHazard() && MI->readsRegister(AMDGPU::LDS_DIRECT))) in PreEmitNoopsCommon()
385 return std::max(WaitStates, checkReadM0Hazards(MI)); in PreEmitNoopsCommon()
387 if (SIInstrInfo::isMAI(*MI)) in PreEmitNoopsCommon()
388 return std::max(WaitStates, checkMAIHazards(MI)); in PreEmitNoopsCommon()
390 if (SIInstrInfo::isVMEM(*MI) || in PreEmitNoopsCommon()
391 SIInstrInfo::isFLAT(*MI) || in PreEmitNoopsCommon()
392 SIInstrInfo::isDS(*MI)) in PreEmitNoopsCommon()
393 return std::max(WaitStates, checkMAILdStHazards(MI)); in PreEmitNoopsCommon()
535 const MachineInstr *MI, IsExpiredFn IsExpired) { in getWaitStatesSince() argument
537 return getWaitStatesSince(IsHazard, MI->getParent(), in getWaitStatesSince()
538 std::next(MI->getReverseIterator()), in getWaitStatesSince()
551 for (MachineInstr *MI : EmittedInstrs) { in getWaitStatesSince()
552 if (MI) { in getWaitStatesSince()
553 if (IsHazard(*MI)) in getWaitStatesSince()
556 if (MI->isInlineAsm()) in getWaitStatesSince()
572 auto IsHazardFn = [IsHazardDef, TRI, Reg](const MachineInstr &MI) { in getWaitStatesSinceDef() argument
573 return IsHazardDef(MI) && MI.modifiesRegister(Reg, TRI); in getWaitStatesSinceDef()
581 auto IsHazardFn = [IsHazard](const MachineInstr &MI) { in getWaitStatesSinceSetReg() argument
582 return isSSetReg(MI.getOpcode()) && IsHazard(MI); in getWaitStatesSinceSetReg()
607 void GCNHazardRecognizer::addClauseInst(const MachineInstr &MI) { in addClauseInst() argument
608 addRegsToSet(TRI, MI.operands(), ClauseDefs, ClauseUses); in addClauseInst()
611 static bool breaksSMEMSoftClause(MachineInstr *MI) { in breaksSMEMSoftClause() argument
612 return !SIInstrInfo::isSMRD(*MI); in breaksSMEMSoftClause()
615 static bool breaksVMEMSoftClause(MachineInstr *MI) { in breaksVMEMSoftClause() argument
616 return !SIInstrInfo::isVMEM(*MI) && !SIInstrInfo::isFLAT(*MI); in breaksVMEMSoftClause()
639 for (MachineInstr *MI : EmittedInstrs) { in checkSoftClauseHazards()
642 if (!MI) in checkSoftClauseHazards()
645 if (IsSMRD ? breaksSMEMSoftClause(MI) : breaksVMEMSoftClause(MI)) in checkSoftClauseHazards()
648 addClauseInst(*MI); in checkSoftClauseHazards()
679 auto IsHazardDefFn = [this](const MachineInstr &MI) { in checkSMRDHazards() argument
680 return TII.isVALU(MI); in checkSMRDHazards()
682 auto IsBufferHazardDefFn = [this](const MachineInstr &MI) { in checkSMRDHazards() argument
683 return TII.isSALU(MI); in checkSMRDHazards()
724 auto IsHazardDefFn = [this](const MachineInstr &MI) { in checkVMEMHazards() argument
725 return TII.isVALU(MI); in checkVMEMHazards()
747 auto IsHazardDefFn = [TII](const MachineInstr &MI) { in checkDPPHazards() argument
748 return TII->isVALU(MI); in checkDPPHazards()
776 auto IsHazardDefFn = [TII](const MachineInstr &MI) { in checkDivFMasHazards() argument
777 return TII->isVALU(MI); in checkDivFMasHazards()
790 auto IsHazardFn = [TII, GetRegHWReg](const MachineInstr &MI) { in checkGetRegHazards() argument
791 return GetRegHWReg == getHWReg(TII, MI); in checkGetRegHazards()
803 auto IsHazardFn = [TII, HWReg](const MachineInstr &MI) { in checkSetRegHazards() argument
804 return HWReg == getHWReg(TII, MI); in checkSetRegHazards()
810 int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) { in createsVALUHazard() argument
811 if (!MI.mayStore()) in createsVALUHazard()
815 unsigned Opcode = MI.getOpcode(); in createsVALUHazard()
816 const MCInstrDesc &Desc = MI.getDesc(); in createsVALUHazard()
823 if (TII->isMUBUF(MI) || TII->isMTBUF(MI)) { in createsVALUHazard()
831 TII->getNamedOperand(MI, AMDGPU::OpName::soffset); in createsVALUHazard()
843 if (TII->isMIMG(MI)) { in createsVALUHazard()
850 if (TII->isFLAT(MI)) { in createsVALUHazard()
872 auto IsHazardFn = [this, Reg, TRI](const MachineInstr &MI) { in checkVALUHazardsHelper() argument
873 int DataIdx = createsVALUHazard(MI); in checkVALUHazardsHelper()
875 TRI->regsOverlap(MI.getOperand(DataIdx).getReg(), Reg); in checkVALUHazardsHelper()
890 auto IsTransDefFn = [this, VALU](const MachineInstr &MI) { in checkVALUHazards() argument
891 if (!SIInstrInfo::isTRANS(MI)) in checkVALUHazards()
895 Register Def = TII->getNamedOperand(MI, AMDGPU::OpName::vdst)->getReg(); in checkVALUHazards()
914 auto IsShift16BitDefFn = [this, VALU](const MachineInstr &MI) { in checkVALUHazards() argument
915 if (!SIInstrInfo::isVALU(MI)) in checkVALUHazards()
918 if (SIInstrInfo::isSDWA(MI)) { in checkVALUHazards()
919 if (auto *DstSel = TII->getNamedOperand(MI, AMDGPU::OpName::dst_sel)) in checkVALUHazards()
923 if (!AMDGPU::hasNamedOperand(MI.getOpcode(), AMDGPU::OpName::op_sel) || in checkVALUHazards()
924 !(TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers) in checkVALUHazards()
930 if (auto *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst)) { in checkVALUHazards()
956 auto IsVALUDefSGPRFn = [&UseReg, TRI](const MachineInstr &MI) { in checkVALUHazards() argument
957 if (!SIInstrInfo::isVALU(MI)) in checkVALUHazards()
959 return MI.modifiesRegister(UseReg, TRI); in checkVALUHazards()
1060 auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isVALU(MI); }; in checkRWLaneHazards() argument
1076 auto IsHazardFn = [TII](const MachineInstr &MI) { in checkRFEHazards() argument
1077 return getHWReg(TII, MI) == AMDGPU::Hwreg::ID_TRAPSTS; in checkRFEHazards()
1083 int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) { in checkReadM0Hazards() argument
1086 auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isSALU(MI); }; in checkReadM0Hazards() argument
1091 void GCNHazardRecognizer::fixHazards(MachineInstr *MI) { in fixHazards() argument
1092 fixVMEMtoScalarWriteHazards(MI); in fixHazards()
1093 fixVcmpxPermlaneHazards(MI); in fixHazards()
1094 fixSMEMtoVectorWriteHazards(MI); in fixHazards()
1095 fixVcmpxExecWARHazard(MI); in fixHazards()
1096 fixLdsBranchVmemWARHazard(MI); in fixHazards()
1098 fixLdsDirectVALUHazard(MI); in fixHazards()
1099 fixLdsDirectVMEMHazard(MI); in fixHazards()
1101 fixVALUPartialForwardingHazard(MI); in fixHazards()
1102 fixVALUTransUseHazard(MI); in fixHazards()
1103 fixWMMAHazards(MI); in fixHazards()
1104 fixShift64HighRegBug(MI); in fixHazards()
1105 fixVALUMaskWriteHazard(MI); in fixHazards()
1108 bool GCNHazardRecognizer::fixVcmpxPermlaneHazards(MachineInstr *MI) { in fixVcmpxPermlaneHazards() argument
1109 if (!ST.hasVcmpxPermlaneHazard() || !isPermlane(*MI)) in fixVcmpxPermlaneHazards()
1114 auto IsHazardFn = [TII, TRI](const MachineInstr &MI) { in fixVcmpxPermlaneHazards() argument
1115 return (TII->isVOPC(MI) || in fixVcmpxPermlaneHazards()
1116 ((TII->isVOP3(MI) || TII->isSDWA(MI)) && MI.isCompare())) && in fixVcmpxPermlaneHazards()
1117 MI.modifiesRegister(AMDGPU::EXEC, TRI); in fixVcmpxPermlaneHazards()
1120 auto IsExpiredFn = [](const MachineInstr &MI, int) { in fixVcmpxPermlaneHazards() argument
1121 unsigned Opc = MI.getOpcode(); in fixVcmpxPermlaneHazards()
1122 return SIInstrInfo::isVALU(MI) && Opc != AMDGPU::V_NOP_e32 && in fixVcmpxPermlaneHazards()
1126 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == in fixVcmpxPermlaneHazards()
1133 auto *Src0 = TII->getNamedOperand(*MI, AMDGPU::OpName::src0); in fixVcmpxPermlaneHazards()
1136 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), in fixVcmpxPermlaneHazards()
1144 bool GCNHazardRecognizer::fixVMEMtoScalarWriteHazards(MachineInstr *MI) { in fixVMEMtoScalarWriteHazards() argument
1149 if (!SIInstrInfo::isSALU(*MI) && !SIInstrInfo::isSMRD(*MI)) in fixVMEMtoScalarWriteHazards()
1152 if (MI->getNumDefs() == 0) in fixVMEMtoScalarWriteHazards()
1157 auto IsHazardFn = [TRI, MI](const MachineInstr &I) { in fixVMEMtoScalarWriteHazards()
1162 for (const MachineOperand &Def : MI->defs()) { in fixVMEMtoScalarWriteHazards()
1172 auto IsExpiredFn = [](const MachineInstr &MI, int) { in fixVMEMtoScalarWriteHazards() argument
1173 return SIInstrInfo::isVALU(MI) || in fixVMEMtoScalarWriteHazards()
1174 (MI.getOpcode() == AMDGPU::S_WAITCNT && in fixVMEMtoScalarWriteHazards()
1175 !MI.getOperand(0).getImm()) || in fixVMEMtoScalarWriteHazards()
1176 (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && in fixVMEMtoScalarWriteHazards()
1177 AMDGPU::DepCtr::decodeFieldVmVsrc(MI.getOperand(0).getImm()) == 0); in fixVMEMtoScalarWriteHazards()
1180 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == in fixVMEMtoScalarWriteHazards()
1185 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), in fixVMEMtoScalarWriteHazards()
1191 bool GCNHazardRecognizer::fixSMEMtoVectorWriteHazards(MachineInstr *MI) { in fixSMEMtoVectorWriteHazards() argument
1196 if (!SIInstrInfo::isVALU(*MI)) in fixSMEMtoVectorWriteHazards()
1200 switch (MI->getOpcode()) { in fixSMEMtoVectorWriteHazards()
1213 const MachineOperand *SDST = TII->getNamedOperand(*MI, SDSTName); in fixSMEMtoVectorWriteHazards()
1215 for (const auto &MO : MI->implicit_operands()) { in fixSMEMtoVectorWriteHazards()
1231 auto IsExpiredFn = [TII, IV](const MachineInstr &MI, int) { in fixSMEMtoVectorWriteHazards() argument
1232 if (TII->isSALU(MI)) { in fixSMEMtoVectorWriteHazards()
1233 switch (MI.getOpcode()) { in fixSMEMtoVectorWriteHazards()
1243 return (MI.getOperand(1).getImm() == 0) && in fixSMEMtoVectorWriteHazards()
1244 (MI.getOperand(0).getReg() == AMDGPU::SGPR_NULL); in fixSMEMtoVectorWriteHazards()
1246 const int64_t Imm = MI.getOperand(0).getImm(); in fixSMEMtoVectorWriteHazards()
1253 if (TII->isSOPP(MI)) in fixSMEMtoVectorWriteHazards()
1268 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == in fixSMEMtoVectorWriteHazards()
1272 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), in fixSMEMtoVectorWriteHazards()
1278 bool GCNHazardRecognizer::fixVcmpxExecWARHazard(MachineInstr *MI) { in fixVcmpxExecWARHazard() argument
1283 if (!SIInstrInfo::isVALU(*MI)) in fixVcmpxExecWARHazard()
1287 if (!MI->modifiesRegister(AMDGPU::EXEC, TRI)) in fixVcmpxExecWARHazard()
1297 auto IsExpiredFn = [TII, TRI](const MachineInstr &MI, int) { in fixVcmpxExecWARHazard() argument
1298 if (SIInstrInfo::isVALU(MI)) { in fixVcmpxExecWARHazard()
1299 if (TII->getNamedOperand(MI, AMDGPU::OpName::sdst)) in fixVcmpxExecWARHazard()
1301 for (auto MO : MI.implicit_operands()) in fixVcmpxExecWARHazard()
1305 if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && in fixVcmpxExecWARHazard()
1306 AMDGPU::DepCtr::decodeFieldSaSdst(MI.getOperand(0).getImm()) == 0) in fixVcmpxExecWARHazard()
1311 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == in fixVcmpxExecWARHazard()
1315 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), in fixVcmpxExecWARHazard()
1331 for (auto &MI : MBB) { in shouldRunLdsBranchVmemWARHazardFixup() local
1332 HasLds |= SIInstrInfo::isDS(MI); in shouldRunLdsBranchVmemWARHazardFixup()
1334 SIInstrInfo::isVMEM(MI) || SIInstrInfo::isSegmentSpecificFLAT(MI); in shouldRunLdsBranchVmemWARHazardFixup()
1348 bool GCNHazardRecognizer::fixLdsBranchVmemWARHazard(MachineInstr *MI) { in fixLdsBranchVmemWARHazard() argument
1355 auto IsHazardInst = [](const MachineInstr &MI) { in fixLdsBranchVmemWARHazard() argument
1356 if (SIInstrInfo::isDS(MI)) in fixLdsBranchVmemWARHazard()
1358 if (SIInstrInfo::isVMEM(MI) || SIInstrInfo::isSegmentSpecificFLAT(MI)) in fixLdsBranchVmemWARHazard()
1363 auto InstType = IsHazardInst(*MI); in fixLdsBranchVmemWARHazard()
1392 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == in fixLdsBranchVmemWARHazard()
1397 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), in fixLdsBranchVmemWARHazard()
1405 bool GCNHazardRecognizer::fixLdsDirectVALUHazard(MachineInstr *MI) { in fixLdsDirectVALUHazard() argument
1406 if (!SIInstrInfo::isLDSDIR(*MI)) in fixLdsDirectVALUHazard()
1410 const MachineOperand *VDST = TII.getNamedOperand(*MI, AMDGPU::OpName::vdst); in fixLdsDirectVALUHazard()
1428 auto GetWaitStatesFn = [](const MachineInstr &MI) { in fixLdsDirectVALUHazard() argument
1429 return SIInstrInfo::isVALU(MI) ? 1 : 0; in fixLdsDirectVALUHazard()
1433 auto Count = ::getWaitStatesSince(IsHazardFn, MI->getParent(), in fixLdsDirectVALUHazard()
1434 std::next(MI->getReverseIterator()), 0, in fixLdsDirectVALUHazard()
1443 TII.getNamedOperand(*MI, AMDGPU::OpName::waitvdst); in fixLdsDirectVALUHazard()
1449 bool GCNHazardRecognizer::fixLdsDirectVMEMHazard(MachineInstr *MI) { in fixLdsDirectVMEMHazard() argument
1450 if (!SIInstrInfo::isLDSDIR(*MI)) in fixLdsDirectVMEMHazard()
1453 const MachineOperand *VDST = TII.getNamedOperand(*MI, AMDGPU::OpName::vdst); in fixLdsDirectVMEMHazard()
1474 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == in fixLdsDirectVMEMHazard()
1479 TII.getNamedOperand(*MI, AMDGPU::OpName::waitvsrc)->setImm(0); in fixLdsDirectVMEMHazard()
1481 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), in fixLdsDirectVMEMHazard()
1489 bool GCNHazardRecognizer::fixVALUPartialForwardingHazard(MachineInstr *MI) { in fixVALUPartialForwardingHazard() argument
1494 if (!ST.isWave64() || !SIInstrInfo::isVALU(*MI)) in fixVALUPartialForwardingHazard()
1499 for (const MachineOperand &Use : MI->explicit_uses()) { in fixVALUPartialForwardingHazard()
1621 auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) { in fixVALUPartialForwardingHazard() argument
1622 if (SIInstrInfo::isVALU(MI)) in fixVALUPartialForwardingHazard()
1627 if (!hasHazard<StateType>(State, IsHazardFn, UpdateStateFn, MI->getParent(), in fixVALUPartialForwardingHazard()
1628 std::next(MI->getReverseIterator()), Visited)) in fixVALUPartialForwardingHazard()
1631 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), in fixVALUPartialForwardingHazard()
1638 bool GCNHazardRecognizer::fixVALUTransUseHazard(MachineInstr *MI) { in fixVALUTransUseHazard() argument
1643 if (!SIInstrInfo::isVALU(*MI)) in fixVALUTransUseHazard()
1648 for (const MachineOperand &Use : MI->explicit_uses()) { in fixVALUTransUseHazard()
1697 auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) { in fixVALUTransUseHazard() argument
1698 if (SIInstrInfo::isVALU(MI)) in fixVALUTransUseHazard()
1700 if (SIInstrInfo::isTRANS(MI)) in fixVALUTransUseHazard()
1705 if (!hasHazard<StateType>(State, IsHazardFn, UpdateStateFn, MI->getParent(), in fixVALUTransUseHazard()
1706 std::next(MI->getReverseIterator()), Visited)) in fixVALUTransUseHazard()
1711 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), in fixVALUTransUseHazard()
1718 bool GCNHazardRecognizer::fixWMMAHazards(MachineInstr *MI) { in fixWMMAHazards() argument
1719 if (!SIInstrInfo::isWMMA(*MI) && !SIInstrInfo::isSWMMAC(*MI)) in fixWMMAHazards()
1725 auto IsHazardFn = [MI, TII, TRI, this](const MachineInstr &I) { in fixWMMAHazards()
1732 TII->getNamedOperand(*MI, AMDGPU::OpName::src0)->getReg(); in fixWMMAHazards()
1734 TII->getNamedOperand(*MI, AMDGPU::OpName::src1)->getReg(); in fixWMMAHazards()
1747 TII->getNamedOperand(*MI, AMDGPU::OpName::src2); in fixWMMAHazards()
1754 TII->getNamedOperand(*MI, AMDGPU::OpName::src2_modifiers); in fixWMMAHazards()
1761 TII->pseudoToMCOpcode(MI->getOpcode()))); in fixWMMAHazards()
1767 if (SIInstrInfo::isSWMMAC(*MI)) { in fixWMMAHazards()
1769 TII->getNamedOperand(*MI, AMDGPU::OpName::src2)->getReg(); in fixWMMAHazards()
1783 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == in fixWMMAHazards()
1787 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII->get(AMDGPU::V_NOP_e32)); in fixWMMAHazards()
1792 bool GCNHazardRecognizer::fixShift64HighRegBug(MachineInstr *MI) { in fixShift64HighRegBug() argument
1797 switch (MI->getOpcode()) { in fixShift64HighRegBug()
1806 MachineOperand *Amt = TII.getNamedOperand(*MI, AMDGPU::OpName::src0); in fixShift64HighRegBug()
1819 MachineOperand *Src1 = TII.getNamedOperand(*MI, AMDGPU::OpName::src1); in fixShift64HighRegBug()
1821 bool OverlappedDst = MI->modifiesRegister(AmtReg, &TRI); in fixShift64HighRegBug()
1825 Src1->getReg() == MI->getOperand(0).getReg()); in fixShift64HighRegBug()
1832 if (!MI->modifiesRegister(Reg, &TRI) && !MI->readsRegister(Reg, &TRI)) { in fixShift64HighRegBug()
1845 DebugLoc DL = MI->getDebugLoc(); in fixShift64HighRegBug()
1846 MachineBasicBlock *MBB = MI->getParent(); in fixShift64HighRegBug()
1848 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::S_WAITCNT)) in fixShift64HighRegBug()
1854 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_SWAP_B32), NewAmtLo) in fixShift64HighRegBug()
1858 runOnInstruction(BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_SWAP_B32), NewAmt) in fixShift64HighRegBug()
1865 BuildMI(*MBB, std::next(MI->getIterator()), DL, TII.get(AMDGPU::V_SWAP_B32), in fixShift64HighRegBug()
1871 BuildMI(*MBB, std::next(MI->getIterator()), DL, TII.get(AMDGPU::V_SWAP_B32), in fixShift64HighRegBug()
1885 MI->getOperand(0).setReg(NewReg); in fixShift64HighRegBug()
1895 int GCNHazardRecognizer::checkNSAtoVMEMHazard(MachineInstr *MI) { in checkNSAtoVMEMHazard() argument
1901 if (!SIInstrInfo::isMUBUF(*MI) && !SIInstrInfo::isMTBUF(*MI)) in checkNSAtoVMEMHazard()
1905 const auto *Offset = TII->getNamedOperand(*MI, AMDGPU::OpName::offset); in checkNSAtoVMEMHazard()
1920 int GCNHazardRecognizer::checkFPAtomicToDenormModeHazard(MachineInstr *MI) { in checkFPAtomicToDenormModeHazard() argument
1927 if (MI->getOpcode() != AMDGPU::S_DENORM_MODE) in checkFPAtomicToDenormModeHazard()
1936 auto IsExpiredFn = [](const MachineInstr &MI, int WaitStates) { in checkFPAtomicToDenormModeHazard() argument
1937 if (WaitStates >= 3 || SIInstrInfo::isVALU(MI)) in checkFPAtomicToDenormModeHazard()
1940 switch (MI.getOpcode()) { in checkFPAtomicToDenormModeHazard()
1956 ::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn); in checkFPAtomicToDenormModeHazard()
1959 int GCNHazardRecognizer::checkMAIHazards(MachineInstr *MI) { in checkMAIHazards() argument
1960 assert(SIInstrInfo::isMAI(*MI)); in checkMAIHazards()
1962 return ST.hasGFX90AInsts() ? checkMAIHazards90A(MI) : checkMAIHazards908(MI); in checkMAIHazards()
1965 int GCNHazardRecognizer::checkMFMAPadding(MachineInstr *MI) { in checkMFMAPadding() argument
1971 if (!SIInstrInfo::isMFMA(*MI) || MFI->getOccupancy() < 2) in checkMFMAPadding()
1976 this](const MachineInstr &MI) { in checkMFMAPadding() argument
1977 if (!SIInstrInfo::isMFMA(MI)) in checkMFMAPadding()
1980 NeighborMFMALatency = this->getMFMAPipelineWaitStates(MI); in checkMFMAPadding()
1995 int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) { in checkMAIHazards908() argument
1997 unsigned Opc = MI->getOpcode(); in checkMAIHazards908()
1999 auto IsVALUFn = [](const MachineInstr &MI) { in checkMAIHazards908() argument
2000 return SIInstrInfo::isVALU(MI) || MI.isInlineAsm(); in checkMAIHazards908()
2013 for (const MachineOperand &Use : MI->explicit_uses()) { in checkMAIHazards908()
2029 for (const MachineOperand &Op : MI->explicit_operands()) { in checkMAIHazards908()
2049 this](const MachineInstr &MI) { in checkMAIHazards908() argument
2050 if (!SIInstrInfo::isMFMA(MI)) in checkMAIHazards908()
2052 Register DstReg = MI.getOperand(0).getReg(); in checkMAIHazards908()
2056 std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI)); in checkMAIHazards908()
2095 auto IsAccVgprWriteFn = [Reg, this](const MachineInstr &MI) { in checkMAIHazards908() argument
2096 if (MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64) in checkMAIHazards908()
2098 Register DstReg = MI.getOperand(0).getReg(); in checkMAIHazards908()
2124 Register DstReg = MI->getOperand(0).getReg(); in checkMAIHazards908()
2128 this](const MachineInstr &MI) { in checkMAIHazards908() argument
2129 if (!SIInstrInfo::isMFMA(MI)) in checkMAIHazards908()
2131 Register Reg = TII.getNamedOperand(MI, AMDGPU::OpName::src2)->getReg(); in checkMAIHazards908()
2133 std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI)); in checkMAIHazards908()
2154 WaitStatesNeeded = std::max(WaitStatesNeeded, checkMFMAPadding(MI)); in checkMAIHazards908()
2159 int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) { in checkMAIHazards90A() argument
2161 unsigned Opc = MI->getOpcode(); in checkMAIHazards90A()
2163 auto IsLegacyVALUFn = [](const MachineInstr &MI) { in checkMAIHazards90A() argument
2164 return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMFMA(MI); in checkMAIHazards90A()
2167 auto IsLegacyVALUNotDotFn = [](const MachineInstr &MI) { in checkMAIHazards90A() argument
2168 return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMFMA(MI) && in checkMAIHazards90A()
2169 !SIInstrInfo::isDOT(MI); in checkMAIHazards90A()
2172 if (!SIInstrInfo::isMFMA(*MI)) in checkMAIHazards90A()
2184 for (const MachineOperand &Use : MI->explicit_uses()) { in checkMAIHazards90A()
2225 this](const MachineInstr &MI) { in checkMAIHazards90A() argument
2226 if (!SIInstrInfo::isMFMA(MI)) in checkMAIHazards90A()
2228 Register DstReg = MI.getOperand(0).getReg(); in checkMAIHazards90A()
2230 MI1 = &MI; in checkMAIHazards90A()
2264 if (!isXDL(ST, *MI)) in checkMAIHazards90A()
2269 if (!isXDL(ST, *MI)) in checkMAIHazards90A()
2273 if (ST.hasGFX940Insts() && isXDL(ST, *MI) && !isXDL(ST, *MI1)) in checkMAIHazards90A()
2369 int GCNHazardRecognizer::checkMAILdStHazards(MachineInstr *MI) { in checkMAILdStHazards() argument
2376 auto IsAccVgprReadFn = [](const MachineInstr &MI) { in checkMAILdStHazards() argument
2377 return MI.getOpcode() == AMDGPU::V_ACCVGPR_READ_B32_e64; in checkMAILdStHazards()
2380 for (const MachineOperand &Op : MI->explicit_uses()) { in checkMAILdStHazards()
2397 auto IsVALUAccVgprRdWrCheckFn = [Reg, this](const MachineInstr &MI) { in checkMAILdStHazards() argument
2398 if (MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64 && in checkMAILdStHazards()
2399 MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64) in checkMAILdStHazards()
2401 auto IsVALUFn = [](const MachineInstr &MI) { in checkMAILdStHazards() argument
2402 return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMAI(MI); in checkMAILdStHazards()
2416 int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) { in checkMAIVALUHazards() argument
2420 auto IsDGEMMFn = [](const MachineInstr &MI) -> bool { in checkMAIVALUHazards() argument
2421 return isDGEMM(MI.getOpcode()); in checkMAIVALUHazards()
2425 if (SIInstrInfo::isMFMA(*MI)) in checkMAIVALUHazards()
2432 bool IsMem = SIInstrInfo::isVMEM(*MI) || in checkMAIVALUHazards()
2433 SIInstrInfo::isFLAT(*MI) || in checkMAIVALUHazards()
2434 SIInstrInfo::isDS(*MI); in checkMAIVALUHazards()
2435 bool IsMemOrExport = IsMem || SIInstrInfo::isEXP(*MI); in checkMAIVALUHazards()
2436 bool IsVALU = SIInstrInfo::isVALU(*MI); in checkMAIVALUHazards()
2440 auto IsMFMAWriteFn = [&Reg, &MFMA, this](const MachineInstr &MI) { in checkMAIVALUHazards() argument
2441 if (!SIInstrInfo::isMFMA(MI) || in checkMAIVALUHazards()
2442 !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg)) in checkMAIVALUHazards()
2444 MFMA = &MI; in checkMAIVALUHazards()
2449 auto IsDotWriteFn = [&Reg, &DOT, this](const MachineInstr &MI) { in checkMAIVALUHazards() argument
2450 if (!SIInstrInfo::isDOT(MI) || in checkMAIVALUHazards()
2451 !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg)) in checkMAIVALUHazards()
2453 DOT = &MI; in checkMAIVALUHazards()
2458 auto IsDGEMMHazard = [&DGEMMAfterVALUWrite, this](const MachineInstr &MI) { in checkMAIVALUHazards() argument
2460 if (isDGEMM(MI.getOpcode())) in checkMAIVALUHazards()
2465 if (!TII.isVALU(MI) || !DGEMMAfterVALUWrite) in checkMAIVALUHazards()
2471 int SrcCIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), in checkMAIVALUHazards()
2495 for (const MachineOperand &Use : MI->explicit_uses()) { in checkMAIVALUHazards()
2505 if (DOT->getOpcode() == MI->getOpcode()) { in checkMAIVALUHazards()
2506 if (&Use - &MI->getOperand(0) != SrcCIdx) in checkMAIVALUHazards()
2589 unsigned Opc = MI->getOpcode(); in checkMAIVALUHazards()
2603 for (const MachineOperand &Def : MI->defs()) { in checkMAIVALUHazards()
2630 if (DOT && DOT->getOpcode() != MI->getOpcode()) in checkMAIVALUHazards()
2681 auto IsSMFMAReadAsCFn = [&Reg, &MFMA, this](const MachineInstr &MI) { in checkMAIVALUHazards() argument
2682 if (!SIInstrInfo::isMFMA(MI) || isDGEMM(MI.getOpcode()) || in checkMAIVALUHazards()
2683 !MI.readsRegister(Reg, &TRI)) in checkMAIVALUHazards()
2686 if (ST.hasGFX940Insts() && !isXDL(ST, MI)) in checkMAIVALUHazards()
2690 TII.getNamedOperand(MI, AMDGPU::OpName::src2); in checkMAIVALUHazards()
2695 MFMA = &MI; in checkMAIVALUHazards()
2733 auto IsMFMAFn = [&MAI](const MachineInstr &MI) { in ShouldPreferAnother() argument
2735 if (SIInstrInfo::isMFMA(MI)) in ShouldPreferAnother()
2736 MAI = &MI; in ShouldPreferAnother()
2740 MachineInstr *MI = SU->getInstr(); in ShouldPreferAnother() local
2741 if (IsMFMAFn(*MI)) { in ShouldPreferAnother()
2750 bool GCNHazardRecognizer::fixVALUMaskWriteHazard(MachineInstr *MI) { in fixVALUMaskWriteHazard() argument
2755 if (!ST.isWave64() || !SIInstrInfo::isSALU(*MI)) in fixVALUMaskWriteHazard()
2766 const MachineOperand *SDSTOp = TII.getNamedOperand(*MI, AMDGPU::OpName::sdst); in fixVALUMaskWriteHazard()
2860 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == in fixVALUMaskWriteHazard()
2864 auto NextMI = std::next(MI->getIterator()); in fixVALUMaskWriteHazard()
2867 BuildMI(*MI->getParent(), NextMI, MI->getDebugLoc(), in fixVALUMaskWriteHazard()
2872 if (MI->getOpcode() == AMDGPU::S_GETPC_B64) { in fixVALUMaskWriteHazard()
2874 while (NextMI != MI->getParent()->end() && in fixVALUMaskWriteHazard()