Lines Matching refs:AMDGPU
62 MaxLookAhead = MF.getRegInfo().isPhysRegUsed(AMDGPU::AGPR0) ? 19 : 5; in GCNHazardRecognizer()
80 return Opcode == AMDGPU::V_DIV_FMAS_F32_e64 || Opcode == AMDGPU::V_DIV_FMAS_F64_e64; in isDivFMas()
84 return Opcode == AMDGPU::S_GETREG_B32; in isSGetReg()
89 case AMDGPU::S_SETREG_B32: in isSSetReg()
90 case AMDGPU::S_SETREG_B32_mode: in isSSetReg()
91 case AMDGPU::S_SETREG_IMM32_B32: in isSSetReg()
92 case AMDGPU::S_SETREG_IMM32_B32_mode: in isSSetReg()
99 return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32; in isRWLane()
103 return Opcode == AMDGPU::S_RFE_B64; in isRFE()
108 case AMDGPU::S_MOVRELS_B32: in isSMovRel()
109 case AMDGPU::S_MOVRELS_B64: in isSMovRel()
110 case AMDGPU::S_MOVRELD_B32: in isSMovRel()
111 case AMDGPU::S_MOVRELD_B64: in isSMovRel()
119 return AMDGPU::getMAIIsDGEMM(Opcode); in isDGEMM()
127 Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 || in isXDL()
128 Opcode == AMDGPU::V_ACCVGPR_READ_B32_e64) in isXDL()
134 return AMDGPU::getMAIIsGFX940XDL(Opcode); in isXDL()
143 case AMDGPU::S_SENDMSG: in isSendMsgTraceDataOrGDS()
144 case AMDGPU::S_SENDMSGHALT: in isSendMsgTraceDataOrGDS()
145 case AMDGPU::S_TTRACEDATA: in isSendMsgTraceDataOrGDS()
148 case AMDGPU::DS_NOP: in isSendMsgTraceDataOrGDS()
149 case AMDGPU::DS_PERMUTE_B32: in isSendMsgTraceDataOrGDS()
150 case AMDGPU::DS_BPERMUTE_B32: in isSendMsgTraceDataOrGDS()
154 int GDS = AMDGPU::getNamedOperandIdx(MI.getOpcode(), in isSendMsgTraceDataOrGDS()
155 AMDGPU::OpName::gds); in isSendMsgTraceDataOrGDS()
165 return Opcode == AMDGPU::V_PERMLANE16_B32_e64 || in isPermlane()
166 Opcode == AMDGPU::V_PERMLANEX16_B32_e64; in isPermlane()
176 AMDGPU::OpName::simm16); in getHWReg()
177 return RegOp->getImm() & AMDGPU::Hwreg::ID_MASK_; in getHWReg()
239 MI->readsRegister(AMDGPU::LDS_DIRECT))) && in getHazardType()
262 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII.get(AMDGPU::S_NOP)) in insertNoopsInBundle()
363 (ST.hasReadM0LdsDirectHazard() && MI->readsRegister(AMDGPU::LDS_DIRECT))) in PreEmitNoopsCommon()
745 DppExecWaitStates - getWaitStatesSinceDef(AMDGPU::EXEC, IsHazardDefFn, in checkDPPHazards()
760 int WaitStatesNeeded = getWaitStatesSinceDef(AMDGPU::VCC, IsHazardDefFn, in checkDivFMasHazards()
799 int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata); in createsVALUHazard()
812 TII->getNamedOperand(MI, AMDGPU::OpName::soffset); in createsVALUHazard()
815 if (AMDGPU::getRegBitWidth(VDataRCID) > 64 && in createsVALUHazard()
825 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc); in createsVALUHazard()
827 AMDGPU::getRegBitWidth(Desc.OpInfo[SRsrcIdx].RegClass) == 256); in createsVALUHazard()
832 int DataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata); in createsVALUHazard()
833 if (AMDGPU::getRegBitWidth(Desc.OpInfo[DataIdx].RegClass) > 64) in createsVALUHazard()
876 Register Def = TII->getNamedOperand(MI, AMDGPU::OpName::vdst)->getReg(); in checkVALUHazards()
900 if (auto *DstSel = TII->getNamedOperand(MI, AMDGPU::OpName::dst_sel)) in checkVALUHazards()
901 if (DstSel->getImm() == AMDGPU::SDWA::DWORD) in checkVALUHazards()
904 if ((AMDGPU::getNamedOperandIdx(MI.getOpcode(), in checkVALUHazards()
905 AMDGPU::OpName::op_sel) == -1) || in checkVALUHazards()
906 !(TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers) in checkVALUHazards()
912 if (auto *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst)) { in checkVALUHazards()
958 if (VALU->readsRegister(AMDGPU::VCC, TRI)) { in checkVALUHazards()
959 UseReg = AMDGPU::VCC; in checkVALUHazards()
967 case AMDGPU::V_READLANE_B32: in checkVALUHazards()
968 case AMDGPU::V_READFIRSTLANE_B32: { in checkVALUHazards()
969 MachineOperand *Src = TII.getNamedOperand(*VALU, AMDGPU::OpName::src0); in checkVALUHazards()
977 case AMDGPU::V_WRITELANE_B32: { in checkVALUHazards()
978 UseReg = AMDGPU::EXEC; in checkVALUHazards()
1036 TII->getNamedOperand(*RWLane, AMDGPU::OpName::src1); in checkRWLaneHazards()
1059 return getHWReg(TII, MI) == AMDGPU::Hwreg::ID_TRAPSTS; in checkRFEHazards()
1070 getWaitStatesSinceDef(AMDGPU::M0, IsHazardFn, ReadM0WaitStates); in checkReadM0Hazards()
1097 MI.modifiesRegister(AMDGPU::EXEC, TRI); in fixVcmpxPermlaneHazards()
1102 return SIInstrInfo::isVALU(MI) && Opc != AMDGPU::V_NOP_e32 && in fixVcmpxPermlaneHazards()
1103 Opc != AMDGPU::V_NOP_e64 && Opc != AMDGPU::V_NOP_sdwa; in fixVcmpxPermlaneHazards()
1113 auto *Src0 = TII->getNamedOperand(*MI, AMDGPU::OpName::src0); in fixVcmpxPermlaneHazards()
1117 TII->get(AMDGPU::V_MOV_B32_e32)) in fixVcmpxPermlaneHazards()
1153 (MI.getOpcode() == AMDGPU::S_WAITCNT && in fixVMEMtoScalarWriteHazards()
1155 (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && in fixVMEMtoScalarWriteHazards()
1165 TII->get(AMDGPU::S_WAITCNT_DEPCTR)) in fixVMEMtoScalarWriteHazards()
1179 case AMDGPU::V_READLANE_B32: in fixSMEMtoVectorWriteHazards()
1180 case AMDGPU::V_READFIRSTLANE_B32: in fixSMEMtoVectorWriteHazards()
1181 SDSTName = AMDGPU::OpName::vdst; in fixSMEMtoVectorWriteHazards()
1184 SDSTName = AMDGPU::OpName::sdst; in fixSMEMtoVectorWriteHazards()
1190 const AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(ST.getCPU()); in fixSMEMtoVectorWriteHazards()
1212 case AMDGPU::S_SETVSKIP: in fixSMEMtoVectorWriteHazards()
1213 case AMDGPU::S_VERSION: in fixSMEMtoVectorWriteHazards()
1214 case AMDGPU::S_WAITCNT_VSCNT: in fixSMEMtoVectorWriteHazards()
1215 case AMDGPU::S_WAITCNT_VMCNT: in fixSMEMtoVectorWriteHazards()
1216 case AMDGPU::S_WAITCNT_EXPCNT: in fixSMEMtoVectorWriteHazards()
1219 case AMDGPU::S_WAITCNT_LGKMCNT: in fixSMEMtoVectorWriteHazards()
1222 (MI.getOperand(0).getReg() == AMDGPU::SGPR_NULL); in fixSMEMtoVectorWriteHazards()
1223 case AMDGPU::S_WAITCNT: { in fixSMEMtoVectorWriteHazards()
1225 AMDGPU::Waitcnt Decoded = AMDGPU::decodeWaitcnt(IV, Imm); in fixSMEMtoVectorWriteHazards()
1250 TII->get(AMDGPU::S_MOV_B32), AMDGPU::SGPR_NULL) in fixSMEMtoVectorWriteHazards()
1260 if (!MI->modifiesRegister(AMDGPU::EXEC, TRI)) in fixVcmpxExecWARHazard()
1266 return I.readsRegister(AMDGPU::EXEC, TRI); in fixVcmpxExecWARHazard()
1272 if (TII->getNamedOperand(MI, AMDGPU::OpName::sdst)) in fixVcmpxExecWARHazard()
1278 if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && in fixVcmpxExecWARHazard()
1289 TII->get(AMDGPU::S_WAITCNT_DEPCTR)) in fixVcmpxExecWARHazard()
1334 return IsHazardInst(I) || (I.getOpcode() == AMDGPU::S_WAITCNT_VSCNT && in fixLdsBranchVmemWARHazard()
1335 I.getOperand(0).getReg() == AMDGPU::SGPR_NULL && in fixLdsBranchVmemWARHazard()
1353 return I.getOpcode() == AMDGPU::S_WAITCNT_VSCNT && in fixLdsBranchVmemWARHazard()
1354 I.getOperand(0).getReg() == AMDGPU::SGPR_NULL && in fixLdsBranchVmemWARHazard()
1368 TII->get(AMDGPU::S_WAITCNT_VSCNT)) in fixLdsBranchVmemWARHazard()
1369 .addReg(AMDGPU::SGPR_NULL, RegState::Undef) in fixLdsBranchVmemWARHazard()
1380 const MachineOperand *VDST = TII.getNamedOperand(*MI, AMDGPU::OpName::vdst); in fixLdsDirectVALUHazard()
1413 TII.getNamedOperand(*MI, AMDGPU::OpName::waitvdst); in fixLdsDirectVALUHazard()
1423 const MachineOperand *VDST = TII.getNamedOperand(*MI, AMDGPU::OpName::vdst); in fixLdsDirectVMEMHazard()
1434 (I.getOpcode() == AMDGPU::S_WAITCNT && !I.getOperand(0).getImm()) || in fixLdsDirectVMEMHazard()
1435 (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && in fixLdsDirectVMEMHazard()
1444 TII.get(AMDGPU::S_WAITCNT_DEPCTR)) in fixLdsDirectVMEMHazard()
1506 (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && in fixVALUPartialForwardingHazard()
1521 if (!State.DefPos.empty() && I.modifiesRegister(AMDGPU::EXEC, &TRI)) { in fixVALUPartialForwardingHazard()
1593 TII.get(AMDGPU::S_WAITCNT_DEPCTR)) in fixVALUPartialForwardingHazard()
1641 (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && in fixVALUTransUseHazard()
1671 TII.get(AMDGPU::S_WAITCNT_DEPCTR)) in fixVALUTransUseHazard()
1691 TII->getNamedOperand(*MI, AMDGPU::OpName::src0)->getReg(); in fixWMMAHazards()
1693 TII->getNamedOperand(*MI, AMDGPU::OpName::src1)->getReg(); in fixWMMAHazards()
1696 TII->getNamedOperand(I, AMDGPU::OpName::vdst)->getReg(); in fixWMMAHazards()
1706 TII->getNamedOperand(*MI, AMDGPU::OpName::src2); in fixWMMAHazards()
1709 if (CurSrc2Reg != AMDGPU::NoRegister && in fixWMMAHazards()
1713 TII->getNamedOperand(*MI, AMDGPU::OpName::src2_modifiers); in fixWMMAHazards()
1733 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII->get(AMDGPU::V_NOP_e32)); in fixWMMAHazards()
1748 const auto *Offset = TII->getNamedOperand(*MI, AMDGPU::OpName::offset); in checkNSAtoVMEMHazard()
1755 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(I.getOpcode()); in checkNSAtoVMEMHazard()
1756 return Info->MIMGEncoding == AMDGPU::MIMGEncGfx10NSA && in checkNSAtoVMEMHazard()
1766 if (MI->getOpcode() != AMDGPU::S_DENORM_MODE) in checkFPAtomicToDenormModeHazard()
1780 case AMDGPU::S_WAITCNT: in checkFPAtomicToDenormModeHazard()
1781 case AMDGPU::S_WAITCNT_VSCNT: in checkFPAtomicToDenormModeHazard()
1782 case AMDGPU::S_WAITCNT_VMCNT: in checkFPAtomicToDenormModeHazard()
1783 case AMDGPU::S_WAITCNT_EXPCNT: in checkFPAtomicToDenormModeHazard()
1784 case AMDGPU::S_WAITCNT_LGKMCNT: in checkFPAtomicToDenormModeHazard()
1785 case AMDGPU::S_WAIT_IDLE: in checkFPAtomicToDenormModeHazard()
1842 if (Opc != AMDGPU::V_ACCVGPR_READ_B32_e64) { // MFMA or v_accvgpr_write in checkMAIHazards908()
1848 getWaitStatesSinceDef(AMDGPU::EXEC, IsVALUFn, MaxWaitStates); in checkMAIHazards908()
1872 if (Op.isDef() && Opc != AMDGPU::V_ACCVGPR_WRITE_B32_e64) in checkMAIHazards908()
1902 int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2); in checkMAIHazards908()
1906 } else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) { in checkMAIHazards908()
1916 } else if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) { in checkMAIHazards908()
1935 if (MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64) in checkMAIHazards908()
1947 else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) in checkMAIHazards908()
1958 if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) { in checkMAIHazards908()
1970 Register Reg = TII.getNamedOperand(MI, AMDGPU::OpName::src2)->getReg(); in checkMAIHazards908()
2016 getWaitStatesSinceDef(AMDGPU::EXEC, IsLegacyVALUFn, in checkMAIHazards90A()
2020 int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2); in checkMAIHazards90A()
2089 if ((Opc == AMDGPU::V_MFMA_F64_4X4X4F64_e64 || in checkMAIHazards90A()
2090 Opc == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64) && in checkMAIHazards90A()
2091 (Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_e64 || in checkMAIHazards90A()
2092 Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64)) in checkMAIHazards90A()
2099 case AMDGPU::V_MFMA_F64_16X16X4F64_e64: in checkMAIHazards90A()
2100 case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64: in checkMAIHazards90A()
2101 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64: in checkMAIHazards90A()
2102 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64: in checkMAIHazards90A()
2106 case AMDGPU::V_MFMA_F64_4X4X4F64_e64: in checkMAIHazards90A()
2107 case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64: in checkMAIHazards90A()
2153 case AMDGPU::V_MFMA_F64_16X16X4F64_e64: in checkMAIHazards90A()
2154 case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64: in checkMAIHazards90A()
2155 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64: in checkMAIHazards90A()
2156 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64: in checkMAIHazards90A()
2159 case AMDGPU::V_MFMA_F64_4X4X4F64_e64: in checkMAIHazards90A()
2160 case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64: in checkMAIHazards90A()
2216 return MI.getOpcode() == AMDGPU::V_ACCVGPR_READ_B32_e64; in checkMAILdStHazards()
2237 if (MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64 && in checkMAILdStHazards()
2238 MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64) in checkMAILdStHazards()
2294 int SrcCIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), in checkMAIVALUHazards()
2295 AMDGPU::OpName::src2); in checkMAIVALUHazards()
2397 if ((Opc == AMDGPU::V_FMA_F64_e64 || in checkMAIVALUHazards()
2398 Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64 || in checkMAIVALUHazards()
2399 Opc == AMDGPU::V_FMAC_F64_dpp) && in checkMAIVALUHazards()
2496 TII.getNamedOperand(MI, AMDGPU::OpName::src2); in checkMAIVALUHazards()