Lines Matching refs:ST
51 const GCNSubtarget &ST);
57 ST(MF.getSubtarget<GCNSubtarget>()), in GCNHazardRecognizer()
58 TII(*ST.getInstrInfo()), in GCNHazardRecognizer()
63 TSchedModel.init(&ST); in GCNHazardRecognizer()
64 RunLdsBranchVmemWARHazardFixup = shouldRunLdsBranchVmemWARHazardFixup(MF, ST); in GCNHazardRecognizer()
122 static bool isXDL(const GCNSubtarget &ST, const MachineInstr &MI) { in isXDL() argument
131 if (!ST.hasGFX940Insts()) in isXDL()
196 if (ST.hasNSAtoVMEMBug() && checkNSAtoVMEMHazard(MI) > 0) in getHazardType()
202 if (ST.hasNoDataDepHazard()) in getHazardType()
237 if (((ST.hasReadM0MovRelInterpHazard() && in getHazardType()
241 (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI)) || in getHazardType()
242 (ST.hasReadM0LdsDmaHazard() && isLdsDma(*MI)) || in getHazardType()
243 (ST.hasReadM0LdsDirectHazard() && in getHazardType()
338 if (ST.hasNSAtoVMEMBug()) in PreEmitNoopsCommon()
343 if (ST.hasNoDataDepHazard()) in PreEmitNoopsCommon()
378 if ((ST.hasReadM0MovRelInterpHazard() && in PreEmitNoopsCommon()
382 (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI)) || in PreEmitNoopsCommon()
383 (ST.hasReadM0LdsDmaHazard() && isLdsDma(*MI)) || in PreEmitNoopsCommon()
384 (ST.hasReadM0LdsDirectHazard() && MI->readsRegister(AMDGPU::LDS_DIRECT))) in PreEmitNoopsCommon()
570 const SIRegisterInfo *TRI = ST.getRegisterInfo(); in getWaitStatesSinceDef()
622 if (!ST.isXNACKEnabled()) in checkSoftClauseHazards()
673 if (!ST.hasSMRDReadVALUDefHazard()) in checkSMRDHazards()
716 if (!ST.hasVMEMReadSGPRVALUDefHazard()) in checkVMEMHazards()
740 const SIRegisterInfo *TRI = ST.getRegisterInfo(); in checkDPPHazards()
741 const SIInstrInfo *TII = ST.getInstrInfo(); in checkDPPHazards()
771 const SIInstrInfo *TII = ST.getInstrInfo(); in checkDivFMasHazards()
786 const SIInstrInfo *TII = ST.getInstrInfo(); in checkGetRegHazards()
799 const SIInstrInfo *TII = ST.getInstrInfo(); in checkSetRegHazards()
802 const int SetRegWaitStates = ST.getSetRegWaitStates(); in checkSetRegHazards()
814 const SIInstrInfo *TII = ST.getInstrInfo(); in createsVALUHazard()
864 const SIRegisterInfo *TRI = ST.getRegisterInfo(); in checkVALUHazardsHelper()
866 const int VALUWaitStates = ST.hasGFX940Insts() ? 2 : 1; in checkVALUHazardsHelper()
887 if (ST.hasTransForwardingHazard() && !SIInstrInfo::isTRANS(*VALU)) { in checkVALUHazards()
893 const SIRegisterInfo *TRI = ST.getRegisterInfo(); in checkVALUHazards()
894 const SIInstrInfo *TII = ST.getInstrInfo(); in checkVALUHazards()
911 if (ST.hasDstSelForwardingHazard()) { in checkVALUHazards()
917 const SIInstrInfo *TII = ST.getInstrInfo(); in checkVALUHazards()
929 const SIRegisterInfo *TRI = ST.getRegisterInfo(); in checkVALUHazards()
948 if (ST.hasVDecCoExecHazard()) { in checkVALUHazards()
953 const SIRegisterInfo *TRI = ST.getRegisterInfo(); in checkVALUHazards()
1010 if (!ST.has12DWordStoreHazard()) in checkVALUHazards()
1031 if (!ST.has12DWordStoreHazard()) in checkInlineAsmHazards()
1049 const SIInstrInfo *TII = ST.getInstrInfo(); in checkRWLaneHazards()
1050 const SIRegisterInfo *TRI = ST.getRegisterInfo(); in checkRWLaneHazards()
1069 if (!ST.hasRFEHazards()) in checkRFEHazards()
1072 const SIInstrInfo *TII = ST.getInstrInfo(); in checkRFEHazards()
1084 const SIInstrInfo *TII = ST.getInstrInfo(); in checkReadM0Hazards()
1097 if (ST.hasLdsDirect()) { in fixHazards()
1109 if (!ST.hasVcmpxPermlaneHazard() || !isPermlane(*MI)) in fixVcmpxPermlaneHazards()
1112 const SIInstrInfo *TII = ST.getInstrInfo(); in fixVcmpxPermlaneHazards()
1113 const SIRegisterInfo *TRI = ST.getRegisterInfo(); in fixVcmpxPermlaneHazards()
1145 if (!ST.hasVMEMtoScalarWriteHazard()) in fixVMEMtoScalarWriteHazards()
1147 assert(!ST.hasExtendedWaitCounts()); in fixVMEMtoScalarWriteHazards()
1155 const SIRegisterInfo *TRI = ST.getRegisterInfo(); in fixVMEMtoScalarWriteHazards()
1184 const SIInstrInfo *TII = ST.getInstrInfo(); in fixVMEMtoScalarWriteHazards()
1192 if (!ST.hasSMEMtoVectorWriteHazard()) in fixSMEMtoVectorWriteHazards()
1194 assert(!ST.hasExtendedWaitCounts()); in fixSMEMtoVectorWriteHazards()
1210 const SIInstrInfo *TII = ST.getInstrInfo(); in fixSMEMtoVectorWriteHazards()
1211 const SIRegisterInfo *TRI = ST.getRegisterInfo(); in fixSMEMtoVectorWriteHazards()
1212 const AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(ST.getCPU()); in fixSMEMtoVectorWriteHazards()
1279 if (!ST.hasVcmpxExecWARHazard()) in fixVcmpxExecWARHazard()
1281 assert(!ST.hasExtendedWaitCounts()); in fixVcmpxExecWARHazard()
1286 const SIRegisterInfo *TRI = ST.getRegisterInfo(); in fixVcmpxExecWARHazard()
1296 const SIInstrInfo *TII = ST.getInstrInfo(); in fixVcmpxExecWARHazard()
1322 const GCNSubtarget &ST) { in shouldRunLdsBranchVmemWARHazardFixup() argument
1323 if (!ST.hasLdsBranchVmemWARHazard()) in shouldRunLdsBranchVmemWARHazardFixup()
1352 assert(ST.hasLdsBranchVmemWARHazard()); in fixLdsBranchVmemWARHazard()
1353 assert(!ST.hasExtendedWaitCounts()); in fixLdsBranchVmemWARHazard()
1396 const SIInstrInfo *TII = ST.getInstrInfo(); in fixLdsBranchVmemWARHazard()
1462 bool LdsdirCanWait = ST.hasLdsWaitVMSRC(); in fixLdsDirectVMEMHazard()
1490 if (!ST.hasVALUPartialForwardingHazard()) in fixVALUPartialForwardingHazard()
1492 assert(!ST.hasExtendedWaitCounts()); in fixVALUPartialForwardingHazard()
1494 if (!ST.isWave64() || !SIInstrInfo::isVALU(*MI)) in fixVALUPartialForwardingHazard()
1639 if (!ST.hasVALUTransUseHazard()) in fixVALUTransUseHazard()
1641 assert(!ST.hasExtendedWaitCounts()); in fixVALUTransUseHazard()
1722 const SIInstrInfo *TII = ST.getInstrInfo(); in fixWMMAHazards()
1723 const SIRegisterInfo *TRI = ST.getRegisterInfo(); in fixWMMAHazards()
1766 if (AMDGPU::isGFX12Plus(ST)) { in fixWMMAHazards()
1793 if (!ST.hasShift64HighRegBug()) in fixShift64HighRegBug()
1795 assert(!ST.hasExtendedWaitCounts()); in fixShift64HighRegBug()
1826 assert(ST.needsAlignedVGPRs()); in fixShift64HighRegBug()
1898 if (!ST.hasNSAtoVMEMBug()) in checkNSAtoVMEMHazard()
1904 const SIInstrInfo *TII = ST.getInstrInfo(); in checkNSAtoVMEMHazard()
1923 if (!ST.hasFPAtomicToDenormModeHazard()) in checkFPAtomicToDenormModeHazard()
1925 assert(!ST.hasExtendedWaitCounts()); in checkFPAtomicToDenormModeHazard()
1962 return ST.hasGFX90AInsts() ? checkMAIHazards90A(MI) : checkMAIHazards908(MI); in checkMAIHazards()
2247 if (!isDGEMM(Opc) && (!ST.hasGFX940Insts() && isDGEMM(Opc1))) { in checkMAIHazards90A()
2255 else if (ST.hasGFX940Insts() && in checkMAIHazards90A()
2264 if (!isXDL(ST, *MI)) in checkMAIHazards90A()
2269 if (!isXDL(ST, *MI)) in checkMAIHazards90A()
2273 if (ST.hasGFX940Insts() && isXDL(ST, *MI) && !isXDL(ST, *MI1)) in checkMAIHazards90A()
2277 NeedWaitStates = ST.hasGFX940Insts() in checkMAIHazards90A()
2278 ? isXDL(ST, *MI1) in checkMAIHazards90A()
2286 assert(ST.hasGFX940Insts()); in checkMAIHazards90A()
2287 NeedWaitStates = isXDL(ST, *MI1) in checkMAIHazards90A()
2292 NeedWaitStates = ST.hasGFX940Insts() in checkMAIHazards90A()
2293 ? isXDL(ST, *MI1) in checkMAIHazards90A()
2302 NeedWaitStates = ST.hasGFX940Insts() in checkMAIHazards90A()
2303 ? isXDL(ST, *MI1) in checkMAIHazards90A()
2327 NeedWaitStates = ST.hasGFX940Insts() in checkMAIHazards90A()
2328 ? isXDL(ST, *MI1) in checkMAIHazards90A()
2334 assert(ST.hasGFX940Insts()); in checkMAIHazards90A()
2335 NeedWaitStates = isXDL(ST, *MI1) in checkMAIHazards90A()
2340 NeedWaitStates = ST.hasGFX940Insts() in checkMAIHazards90A()
2341 ? isXDL(ST, *MI1) in checkMAIHazards90A()
2348 NeedWaitStates = ST.hasGFX940Insts() in checkMAIHazards90A()
2349 ? isXDL(ST, *MI1) in checkMAIHazards90A()
2371 if (!ST.hasMAIInsts() || ST.hasGFX90AInsts()) in checkMAILdStHazards()
2417 if (!ST.hasGFX90AInsts()) in checkMAIVALUHazards()
2520 if (IsMem && ST.hasGFX90AInsts() && !ST.hasGFX940Insts()) { in checkMAIVALUHazards()
2543 ST.hasGFX940Insts() in checkMAIVALUHazards()
2544 ? isXDL(ST, *MFMA) in checkMAIVALUHazards()
2550 assert(isDGEMM(MFMA->getOpcode()) || ST.hasGFX940Insts()); in checkMAIVALUHazards()
2555 : isXDL(ST, *MFMA) in checkMAIVALUHazards()
2561 ST.hasGFX940Insts() in checkMAIVALUHazards()
2562 ? isXDL(ST, *MFMA) in checkMAIVALUHazards()
2573 : ST.hasGFX940Insts() in checkMAIVALUHazards()
2574 ? isXDL(ST, *MFMA) in checkMAIVALUHazards()
2641 NeedWaitStates = ST.hasGFX940Insts() in checkMAIVALUHazards()
2642 ? isXDL(ST, *MFMA) in checkMAIVALUHazards()
2648 assert(isDGEMM(MFMA->getOpcode()) || ST.hasGFX940Insts()); in checkMAIVALUHazards()
2651 : isXDL(ST, *MFMA) in checkMAIVALUHazards()
2656 NeedWaitStates = ST.hasGFX940Insts() in checkMAIVALUHazards()
2657 ? isXDL(ST, *MFMA) in checkMAIVALUHazards()
2666 : ST.hasGFX940Insts() in checkMAIVALUHazards()
2667 ? isXDL(ST, *MFMA) in checkMAIVALUHazards()
2686 if (ST.hasGFX940Insts() && !isXDL(ST, MI)) in checkMAIVALUHazards()
2710 case 4: assert(ST.hasGFX940Insts()); in checkMAIVALUHazards()
2751 if (!ST.hasVALUMaskWriteHazard()) in fixVALUMaskWriteHazard()
2753 assert(!ST.hasExtendedWaitCounts()); in fixVALUMaskWriteHazard()
2755 if (!ST.isWave64() || !SIInstrInfo::isSALU(*MI)) in fixVALUMaskWriteHazard()