Lines Matching refs:AMDGPU
38 MaxLookAhead = MF.getRegInfo().isPhysRegUsed(AMDGPU::AGPR0) ? 19 : 5; in GCNHazardRecognizer()
56 return Opcode == AMDGPU::V_DIV_FMAS_F32_e64 || Opcode == AMDGPU::V_DIV_FMAS_F64_e64; in isDivFMas()
60 return Opcode == AMDGPU::S_GETREG_B32; in isSGetReg()
65 case AMDGPU::S_SETREG_B32: in isSSetReg()
66 case AMDGPU::S_SETREG_B32_mode: in isSSetReg()
67 case AMDGPU::S_SETREG_IMM32_B32: in isSSetReg()
68 case AMDGPU::S_SETREG_IMM32_B32_mode: in isSSetReg()
75 return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32; in isRWLane()
79 return Opcode == AMDGPU::S_RFE_B64; in isRFE()
84 case AMDGPU::S_MOVRELS_B32: in isSMovRel()
85 case AMDGPU::S_MOVRELS_B64: in isSMovRel()
86 case AMDGPU::S_MOVRELD_B32: in isSMovRel()
87 case AMDGPU::S_MOVRELD_B64: in isSMovRel()
95 return Opcode == AMDGPU::V_MFMA_F64_4X4X4F64_e64 || in isDGEMM()
96 Opcode == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64 || in isDGEMM()
97 Opcode == AMDGPU::V_MFMA_F64_16X16X4F64_e64 || in isDGEMM()
98 Opcode == AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64; in isDGEMM()
106 Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 || in isXDL()
107 Opcode == AMDGPU::V_ACCVGPR_READ_B32_e64) in isXDL()
119 case AMDGPU::S_SENDMSG: in isSendMsgTraceDataOrGDS()
120 case AMDGPU::S_SENDMSGHALT: in isSendMsgTraceDataOrGDS()
121 case AMDGPU::S_TTRACEDATA: in isSendMsgTraceDataOrGDS()
124 case AMDGPU::DS_NOP: in isSendMsgTraceDataOrGDS()
125 case AMDGPU::DS_PERMUTE_B32: in isSendMsgTraceDataOrGDS()
126 case AMDGPU::DS_BPERMUTE_B32: in isSendMsgTraceDataOrGDS()
130 int GDS = AMDGPU::getNamedOperandIdx(MI.getOpcode(), in isSendMsgTraceDataOrGDS()
131 AMDGPU::OpName::gds); in isSendMsgTraceDataOrGDS()
141 return Opcode == AMDGPU::V_PERMLANE16_B32_e64 || in isPermlane()
142 Opcode == AMDGPU::V_PERMLANEX16_B32_e64; in isPermlane()
147 AMDGPU::OpName::simm16); in getHWReg()
148 return RegOp->getImm() & AMDGPU::Hwreg::ID_MASK_; in getHWReg()
233 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII.get(AMDGPU::S_NOP)) in insertNoopsInBundle()
667 DppExecWaitStates - getWaitStatesSinceDef(AMDGPU::EXEC, IsHazardDefFn, in checkDPPHazards()
682 int WaitStatesNeeded = getWaitStatesSinceDef(AMDGPU::VCC, IsHazardDefFn, in checkDivFMasHazards()
721 int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata); in createsVALUHazard()
734 TII->getNamedOperand(MI, AMDGPU::OpName::soffset); in createsVALUHazard()
737 if (AMDGPU::getRegBitWidth(VDataRCID) > 64 && in createsVALUHazard()
747 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc); in createsVALUHazard()
749 AMDGPU::getRegBitWidth(Desc.OpInfo[SRsrcIdx].RegClass) == 256); in createsVALUHazard()
754 int DataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata); in createsVALUHazard()
755 if (AMDGPU::getRegBitWidth(Desc.OpInfo[DataIdx].RegClass) > 64) in createsVALUHazard()
835 TII->getNamedOperand(*RWLane, AMDGPU::OpName::src1); in checkRWLaneHazards()
858 return getHWReg(TII, MI) == AMDGPU::Hwreg::ID_TRAPSTS; in checkRFEHazards()
868 return SMovRelWaitStates - getWaitStatesSinceDef(AMDGPU::M0, IsHazardFn, in checkReadM0Hazards()
889 return SIInstrInfo::isVALU(MI) && Opc != AMDGPU::V_NOP_e32 && in fixVcmpxPermlaneHazards()
890 Opc != AMDGPU::V_NOP_e64 && Opc != AMDGPU::V_NOP_sdwa; in fixVcmpxPermlaneHazards()
900 auto *Src0 = TII->getNamedOperand(*MI, AMDGPU::OpName::src0); in fixVcmpxPermlaneHazards()
904 TII->get(AMDGPU::V_MOV_B32_e32)) in fixVcmpxPermlaneHazards()
940 (MI.getOpcode() == AMDGPU::S_WAITCNT && in fixVMEMtoScalarWriteHazards()
942 (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && in fixVMEMtoScalarWriteHazards()
952 TII->get(AMDGPU::S_WAITCNT_DEPCTR)) in fixVMEMtoScalarWriteHazards()
966 case AMDGPU::V_READLANE_B32: in fixSMEMtoVectorWriteHazards()
967 case AMDGPU::V_READFIRSTLANE_B32: in fixSMEMtoVectorWriteHazards()
968 SDSTName = AMDGPU::OpName::vdst; in fixSMEMtoVectorWriteHazards()
971 SDSTName = AMDGPU::OpName::sdst; in fixSMEMtoVectorWriteHazards()
977 const AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(ST.getCPU()); in fixSMEMtoVectorWriteHazards()
999 case AMDGPU::S_SETVSKIP: in fixSMEMtoVectorWriteHazards()
1000 case AMDGPU::S_VERSION: in fixSMEMtoVectorWriteHazards()
1001 case AMDGPU::S_WAITCNT_VSCNT: in fixSMEMtoVectorWriteHazards()
1002 case AMDGPU::S_WAITCNT_VMCNT: in fixSMEMtoVectorWriteHazards()
1003 case AMDGPU::S_WAITCNT_EXPCNT: in fixSMEMtoVectorWriteHazards()
1006 case AMDGPU::S_WAITCNT_LGKMCNT: in fixSMEMtoVectorWriteHazards()
1009 (MI.getOperand(0).getReg() == AMDGPU::SGPR_NULL); in fixSMEMtoVectorWriteHazards()
1010 case AMDGPU::S_WAITCNT: { in fixSMEMtoVectorWriteHazards()
1012 AMDGPU::Waitcnt Decoded = AMDGPU::decodeWaitcnt(IV, Imm); in fixSMEMtoVectorWriteHazards()
1037 TII->get(AMDGPU::S_MOV_B32), AMDGPU::SGPR_NULL) in fixSMEMtoVectorWriteHazards()
1047 if (!MI->modifiesRegister(AMDGPU::EXEC, TRI)) in fixVcmpxExecWARHazard()
1053 return I.readsRegister(AMDGPU::EXEC, TRI); in fixVcmpxExecWARHazard()
1059 if (TII->getNamedOperand(MI, AMDGPU::OpName::sdst)) in fixVcmpxExecWARHazard()
1065 if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && in fixVcmpxExecWARHazard()
1076 TII->get(AMDGPU::S_WAITCNT_DEPCTR)) in fixVcmpxExecWARHazard()
1121 return IsHazardInst(I) || (I.getOpcode() == AMDGPU::S_WAITCNT_VSCNT && in fixLdsBranchVmemWARHazard()
1122 I.getOperand(0).getReg() == AMDGPU::SGPR_NULL && in fixLdsBranchVmemWARHazard()
1140 return I.getOpcode() == AMDGPU::S_WAITCNT_VSCNT && in fixLdsBranchVmemWARHazard()
1141 I.getOperand(0).getReg() == AMDGPU::SGPR_NULL && in fixLdsBranchVmemWARHazard()
1155 TII->get(AMDGPU::S_WAITCNT_VSCNT)) in fixLdsBranchVmemWARHazard()
1156 .addReg(AMDGPU::SGPR_NULL, RegState::Undef) in fixLdsBranchVmemWARHazard()
1172 const auto *Offset = TII->getNamedOperand(*MI, AMDGPU::OpName::offset); in checkNSAtoVMEMHazard()
1179 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(I.getOpcode()); in checkNSAtoVMEMHazard()
1180 return Info->MIMGEncoding == AMDGPU::MIMGEncGfx10NSA && in checkNSAtoVMEMHazard()
1190 if (MI->getOpcode() != AMDGPU::S_DENORM_MODE) in checkFPAtomicToDenormModeHazard()
1204 case AMDGPU::S_WAITCNT: in checkFPAtomicToDenormModeHazard()
1205 case AMDGPU::S_WAITCNT_VSCNT: in checkFPAtomicToDenormModeHazard()
1206 case AMDGPU::S_WAITCNT_VMCNT: in checkFPAtomicToDenormModeHazard()
1207 case AMDGPU::S_WAITCNT_EXPCNT: in checkFPAtomicToDenormModeHazard()
1208 case AMDGPU::S_WAITCNT_LGKMCNT: in checkFPAtomicToDenormModeHazard()
1209 case AMDGPU::S_WAIT_IDLE: in checkFPAtomicToDenormModeHazard()
1236 if (Opc != AMDGPU::V_ACCVGPR_READ_B32_e64) { // MFMA or v_accvgpr_write in checkMAIHazards908()
1242 getWaitStatesSinceDef(AMDGPU::EXEC, IsVALUFn, MaxWaitStates); in checkMAIHazards908()
1264 MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 && in checkMAIHazards908()
1265 MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64; in checkMAIHazards908()
1272 if (Op.isDef() && Opc != AMDGPU::V_ACCVGPR_WRITE_B32_e64) in checkMAIHazards908()
1302 int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2); in checkMAIHazards908()
1306 } else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) { in checkMAIHazards908()
1316 } else if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) { in checkMAIHazards908()
1335 if (MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64) in checkMAIHazards908()
1347 else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) in checkMAIHazards908()
1358 if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) { in checkMAIHazards908()
1370 Register Reg = TII.getNamedOperand(MI, AMDGPU::OpName::src2)->getReg(); in checkMAIHazards908()
1401 MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 && in checkMAIHazards90A()
1402 MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64; in checkMAIHazards90A()
1418 getWaitStatesSinceDef(AMDGPU::EXEC, IsLegacyVALUFn, in checkMAIHazards90A()
1422 int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2); in checkMAIHazards90A()
1477 if ((Opc == AMDGPU::V_MFMA_F64_4X4X4F64_e64 || in checkMAIHazards90A()
1478 Opc == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64) && in checkMAIHazards90A()
1479 (Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_e64 || in checkMAIHazards90A()
1480 Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64)) in checkMAIHazards90A()
1484 case AMDGPU::V_MFMA_F64_16X16X4F64_e64: in checkMAIHazards90A()
1485 case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64: in checkMAIHazards90A()
1489 case AMDGPU::V_MFMA_F64_4X4X4F64_e64: in checkMAIHazards90A()
1490 case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64: in checkMAIHazards90A()
1516 case AMDGPU::V_MFMA_F64_16X16X4F64_e64: in checkMAIHazards90A()
1517 case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64: in checkMAIHazards90A()
1520 case AMDGPU::V_MFMA_F64_4X4X4F64_e64: in checkMAIHazards90A()
1521 case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64: in checkMAIHazards90A()
1559 return MI.getOpcode() == AMDGPU::V_ACCVGPR_READ_B32_e64; in checkMAILdStHazards()
1580 if (MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64 && in checkMAILdStHazards()
1581 MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64) in checkMAILdStHazards()
1604 MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 && in checkMAIVALUHazards()
1605 MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64; in checkMAIVALUHazards()
1645 int SrcCIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), in checkMAIVALUHazards()
1646 AMDGPU::OpName::src2); in checkMAIVALUHazards()
1722 if ((Opc == AMDGPU::V_FMA_F64_e64 || in checkMAIVALUHazards()
1723 Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64 || in checkMAIVALUHazards()
1724 Opc == AMDGPU::V_FMAC_F64_dpp) && in checkMAIVALUHazards()
1794 TII.getNamedOperand(MI, AMDGPU::OpName::src2); in checkMAIVALUHazards()
1836 MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 && in ShouldPreferAnother()
1837 MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64) in ShouldPreferAnother()