Lines Matching refs:AMDGPU
44 namespace AMDGPU { namespace
67 : AMDGPUGenInstrInfo(AMDGPU::ADJCALLSTACKUP, AMDGPU::ADJCALLSTACKDOWN), in SIInstrInfo()
89 int Op0Idx = AMDGPU::getNamedOperandIdx(Opc0, OpName); in nodesHaveSameOperandValue()
90 int Op1Idx = AMDGPU::getNamedOperandIdx(Opc1, OpName); in nodesHaveSameOperandValue()
162 case AMDGPU::S_AND_SAVEEXEC_B32: in resultDependsOnExec()
163 case AMDGPU::S_AND_SAVEEXEC_B64: in resultDependsOnExec()
165 case AMDGPU::S_AND_B32: in resultDependsOnExec()
166 case AMDGPU::S_AND_B64: in resultDependsOnExec()
167 if (!Use.readsRegister(AMDGPU::EXEC)) in resultDependsOnExec()
180 case AMDGPU::V_READFIRSTLANE_B32: in resultDependsOnExec()
189 return MO.getReg() == AMDGPU::EXEC && MO.isImplicit() && in isIgnorableUse()
197 if (MI.getOpcode() == AMDGPU::SI_IF_BREAK) in isSafeToSink()
265 int Offset0Idx = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset); in areLoadsFromSameBasePtr()
266 int Offset1Idx = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset); in areLoadsFromSameBasePtr()
283 if (!AMDGPU::hasNamedOperand(Opc0, AMDGPU::OpName::sbase) || in areLoadsFromSameBasePtr()
284 !AMDGPU::hasNamedOperand(Opc1, AMDGPU::OpName::sbase)) in areLoadsFromSameBasePtr()
317 if (!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::soffset) || in areLoadsFromSameBasePtr()
318 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::vaddr) || in areLoadsFromSameBasePtr()
319 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::srsrc)) in areLoadsFromSameBasePtr()
322 int OffIdx0 = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset); in areLoadsFromSameBasePtr()
323 int OffIdx1 = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset); in areLoadsFromSameBasePtr()
351 case AMDGPU::DS_READ2ST64_B32: in isStride64()
352 case AMDGPU::DS_READ2ST64_B64: in isStride64()
353 case AMDGPU::DS_WRITE2ST64_B32: in isStride64()
354 case AMDGPU::DS_WRITE2ST64_B64: in isStride64()
374 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr); in getMemOperandsWithOffsetWidth()
375 OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset); in getMemOperandsWithOffsetWidth()
386 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); in getMemOperandsWithOffsetWidth()
388 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0); in getMemOperandsWithOffsetWidth()
395 getNamedOperand(LdSt, AMDGPU::OpName::offset0); in getMemOperandsWithOffsetWidth()
397 getNamedOperand(LdSt, AMDGPU::OpName::offset1); in getMemOperandsWithOffsetWidth()
412 int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0); in getMemOperandsWithOffsetWidth()
422 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); in getMemOperandsWithOffsetWidth()
424 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0); in getMemOperandsWithOffsetWidth()
426 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data1); in getMemOperandsWithOffsetWidth()
436 const MachineOperand *RSrc = getNamedOperand(LdSt, AMDGPU::OpName::srsrc); in getMemOperandsWithOffsetWidth()
440 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr); in getMemOperandsWithOffsetWidth()
444 getNamedOperand(LdSt, AMDGPU::OpName::offset); in getMemOperandsWithOffsetWidth()
447 getNamedOperand(LdSt, AMDGPU::OpName::soffset); in getMemOperandsWithOffsetWidth()
455 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); in getMemOperandsWithOffsetWidth()
457 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata); in getMemOperandsWithOffsetWidth()
465 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc); in getMemOperandsWithOffsetWidth()
467 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0); in getMemOperandsWithOffsetWidth()
473 BaseOps.push_back(getNamedOperand(LdSt, AMDGPU::OpName::vaddr)); in getMemOperandsWithOffsetWidth()
477 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata); in getMemOperandsWithOffsetWidth()
483 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::sbase); in getMemOperandsWithOffsetWidth()
487 OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset); in getMemOperandsWithOffsetWidth()
490 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::sdst); in getMemOperandsWithOffsetWidth()
499 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr); in getMemOperandsWithOffsetWidth()
502 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::saddr); in getMemOperandsWithOffsetWidth()
505 Offset = getNamedOperand(LdSt, AMDGPU::OpName::offset)->getImm(); in getMemOperandsWithOffsetWidth()
507 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); in getMemOperandsWithOffsetWidth()
509 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata); in getMemOperandsWithOffsetWidth()
617 BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_ILLEGAL_COPY), DestReg) in reportIllegalCopy()
636 assert((AMDGPU::SReg_32RegClass.contains(SrcReg) || in indirectCopyToAGPR()
637 AMDGPU::AGPR_32RegClass.contains(SrcReg)) && in indirectCopyToAGPR()
640 assert(AMDGPU::AGPR_32RegClass.contains(DestReg) && in indirectCopyToAGPR()
656 if (Def->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 || in indirectCopyToAGPR()
678 BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg) in indirectCopyToAGPR()
697 unsigned MaxVGPRs = RI.getRegPressureLimit(&AMDGPU::VGPR_32RegClass, in indirectCopyToAGPR()
702 unsigned RegNo = (DestReg - AMDGPU::AGPR0) % 3; in indirectCopyToAGPR()
711 Register Tmp2 = RS.scavengeRegisterBackwards(AMDGPU::VGPR_32RegClass, MI, in indirectCopyToAGPR()
721 unsigned TmpCopyOp = AMDGPU::V_MOV_B32_e32; in indirectCopyToAGPR()
722 if (AMDGPU::AGPR_32RegClass.contains(SrcReg)) { in indirectCopyToAGPR()
723 TmpCopyOp = AMDGPU::V_ACCVGPR_READ_B32_e64; in indirectCopyToAGPR()
725 assert(AMDGPU::SReg_32RegClass.contains(SrcReg)); in indirectCopyToAGPR()
736 = BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg) in indirectCopyToAGPR()
757 unsigned Opcode = AMDGPU::S_MOV_B32; in expandSGPRCopy()
760 bool AlignedDest = ((DestSubReg - AMDGPU::SGPR0) % 2) == 0; in expandSGPRCopy()
761 bool AlignedSrc = ((SrcSubReg - AMDGPU::SGPR0) % 2) == 0; in expandSGPRCopy()
769 Opcode = AMDGPU::S_MOV_B64; in expandSGPRCopy()
812 MCRegister SubReg = RI.getSubReg(RegToFix, AMDGPU::lo16); in copyPhysReg()
818 BuildMI(MBB, MI, DL, get(AMDGPU::BUNDLE)); in copyPhysReg()
828 if (RC == &AMDGPU::VGPR_32RegClass) { in copyPhysReg()
829 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) || in copyPhysReg()
830 AMDGPU::SReg_32RegClass.contains(SrcReg) || in copyPhysReg()
831 AMDGPU::AGPR_32RegClass.contains(SrcReg)); in copyPhysReg()
832 unsigned Opc = AMDGPU::AGPR_32RegClass.contains(SrcReg) ? in copyPhysReg()
833 AMDGPU::V_ACCVGPR_READ_B32_e64 : AMDGPU::V_MOV_B32_e32; in copyPhysReg()
839 if (RC == &AMDGPU::SReg_32_XM0RegClass || in copyPhysReg()
840 RC == &AMDGPU::SReg_32RegClass) { in copyPhysReg()
841 if (SrcReg == AMDGPU::SCC) { in copyPhysReg()
842 BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B32), DestReg) in copyPhysReg()
848 if (DestReg == AMDGPU::VCC_LO) { in copyPhysReg()
849 if (AMDGPU::SReg_32RegClass.contains(SrcReg)) { in copyPhysReg()
850 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), AMDGPU::VCC_LO) in copyPhysReg()
854 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg)); in copyPhysReg()
855 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32)) in copyPhysReg()
863 if (!AMDGPU::SReg_32RegClass.contains(SrcReg)) { in copyPhysReg()
868 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg) in copyPhysReg()
873 if (RC == &AMDGPU::SReg_64RegClass) { in copyPhysReg()
874 if (SrcReg == AMDGPU::SCC) { in copyPhysReg()
875 BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B64), DestReg) in copyPhysReg()
881 if (DestReg == AMDGPU::VCC) { in copyPhysReg()
882 if (AMDGPU::SReg_64RegClass.contains(SrcReg)) { in copyPhysReg()
883 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), AMDGPU::VCC) in copyPhysReg()
887 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg)); in copyPhysReg()
888 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32)) in copyPhysReg()
896 if (!AMDGPU::SReg_64RegClass.contains(SrcReg)) { in copyPhysReg()
901 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg) in copyPhysReg()
906 if (DestReg == AMDGPU::SCC) { in copyPhysReg()
909 if (AMDGPU::SReg_64RegClass.contains(SrcReg)) { in copyPhysReg()
914 BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U64)) in copyPhysReg()
918 assert(AMDGPU::SReg_32RegClass.contains(SrcReg)); in copyPhysReg()
919 BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U32)) in copyPhysReg()
927 if (RC == &AMDGPU::AGPR_32RegClass) { in copyPhysReg()
928 if (AMDGPU::VGPR_32RegClass.contains(SrcReg) || in copyPhysReg()
929 (ST.hasGFX90AInsts() && AMDGPU::SReg_32RegClass.contains(SrcReg))) { in copyPhysReg()
930 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg) in copyPhysReg()
935 if (AMDGPU::AGPR_32RegClass.contains(SrcReg) && ST.hasGFX90AInsts()) { in copyPhysReg()
936 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_MOV_B32), DestReg) in copyPhysReg()
950 assert(AMDGPU::VGPR_16RegClass.contains(SrcReg) || in copyPhysReg()
951 AMDGPU::SReg_LO16RegClass.contains(SrcReg) || in copyPhysReg()
952 AMDGPU::AGPR_LO16RegClass.contains(SrcReg)); in copyPhysReg()
954 bool IsSGPRDst = AMDGPU::SReg_LO16RegClass.contains(DestReg); in copyPhysReg()
955 bool IsSGPRSrc = AMDGPU::SReg_LO16RegClass.contains(SrcReg); in copyPhysReg()
956 bool IsAGPRDst = AMDGPU::AGPR_LO16RegClass.contains(DestReg); in copyPhysReg()
957 bool IsAGPRSrc = AMDGPU::AGPR_LO16RegClass.contains(SrcReg); in copyPhysReg()
958 bool DstLow = !AMDGPU::isHi(DestReg, RI); in copyPhysReg()
959 bool SrcLow = !AMDGPU::isHi(SrcReg, RI); in copyPhysReg()
969 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), NewDestReg) in copyPhysReg()
990 if (AMDGPU::VGPR_16_Lo128RegClass.contains(DestReg) && in copyPhysReg()
991 (IsSGPRSrc || AMDGPU::VGPR_16_Lo128RegClass.contains(SrcReg))) { in copyPhysReg()
992 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B16_t16_e32), DestReg) in copyPhysReg()
995 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B16_t16_e64), DestReg) in copyPhysReg()
1009 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), NewDestReg) in copyPhysReg()
1014 auto MIB = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_sdwa), NewDestReg) in copyPhysReg()
1018 .addImm(DstLow ? AMDGPU::SDWA::SdwaSel::WORD_0 in copyPhysReg()
1019 : AMDGPU::SDWA::SdwaSel::WORD_1) in copyPhysReg()
1020 .addImm(AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE) in copyPhysReg()
1021 .addImm(SrcLow ? AMDGPU::SDWA::SdwaSel::WORD_0 in copyPhysReg()
1022 : AMDGPU::SDWA::SdwaSel::WORD_1) in copyPhysReg()
1031 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_e32), DestReg) in copyPhysReg()
1036 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), DestReg) in copyPhysReg()
1064 unsigned Opcode = AMDGPU::V_MOV_B32_e32; in copyPhysReg()
1067 Opcode = AMDGPU::V_ACCVGPR_MOV_B32; in copyPhysReg()
1070 Opcode = AMDGPU::V_ACCVGPR_WRITE_B32_e64; in copyPhysReg()
1072 Opcode = AMDGPU::INSTRUCTION_LIST_END; in copyPhysReg()
1074 Opcode = AMDGPU::V_ACCVGPR_READ_B32_e64; in copyPhysReg()
1080 Opcode = AMDGPU::V_MOV_B64_e32; in copyPhysReg()
1083 Opcode = AMDGPU::V_PK_MOV_B32; in copyPhysReg()
1094 if (Opcode == AMDGPU::INSTRUCTION_LIST_END) in copyPhysReg()
1117 if (Opcode == AMDGPU::INSTRUCTION_LIST_END) { in copyPhysReg()
1122 } else if (Opcode == AMDGPU::V_PK_MOV_B32) { in copyPhysReg()
1124 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), DestSubReg) in copyPhysReg()
1152 NewOpc = AMDGPU::getCommuteRev(Opcode); in commuteOpcode()
1158 NewOpc = AMDGPU::getCommuteOrig(Opcode); in commuteOpcode()
1172 if (RegClass == &AMDGPU::SReg_32RegClass || in materializeImmediate()
1173 RegClass == &AMDGPU::SGPR_32RegClass || in materializeImmediate()
1174 RegClass == &AMDGPU::SReg_32_XM0RegClass || in materializeImmediate()
1175 RegClass == &AMDGPU::SReg_32_XM0_XEXECRegClass) { in materializeImmediate()
1176 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg) in materializeImmediate()
1181 if (RegClass == &AMDGPU::SReg_64RegClass || in materializeImmediate()
1182 RegClass == &AMDGPU::SGPR_64RegClass || in materializeImmediate()
1183 RegClass == &AMDGPU::SReg_64_XEXECRegClass) { in materializeImmediate()
1184 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg) in materializeImmediate()
1189 if (RegClass == &AMDGPU::VGPR_32RegClass) { in materializeImmediate()
1190 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg) in materializeImmediate()
1194 if (RegClass->hasSuperClassEq(&AMDGPU::VReg_64RegClass)) { in materializeImmediate()
1195 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), DestReg) in materializeImmediate()
1201 unsigned Opcode = AMDGPU::V_MOV_B32_e32; in materializeImmediate()
1204 Opcode = AMDGPU::S_MOV_B64; in materializeImmediate()
1207 Opcode = AMDGPU::S_MOV_B32; in materializeImmediate()
1224 return &AMDGPU::VGPR_32RegClass; in getPreferredSelectRegClass()
1235 RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); in insertVectorSelect()
1236 assert(MRI.getRegClass(DstReg) == &AMDGPU::VGPR_32RegClass && in insertVectorSelect()
1241 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg) in insertVectorSelect()
1243 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
1254 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 in insertVectorSelect()
1255 : AMDGPU::S_CSELECT_B64), SReg) in insertVectorSelect()
1258 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
1268 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 in insertVectorSelect()
1269 : AMDGPU::S_CSELECT_B64), SReg) in insertVectorSelect()
1272 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
1284 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg) in insertVectorSelect()
1286 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
1298 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg) in insertVectorSelect()
1300 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
1311 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32 in insertVectorSelect()
1312 : AMDGPU::S_OR_SAVEEXEC_B64), SReg2) in insertVectorSelect()
1314 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 in insertVectorSelect()
1315 : AMDGPU::S_CSELECT_B64), SReg) in insertVectorSelect()
1318 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
1329 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32 in insertVectorSelect()
1330 : AMDGPU::S_OR_SAVEEXEC_B64), SReg2) in insertVectorSelect()
1332 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 in insertVectorSelect()
1333 : AMDGPU::S_CSELECT_B64), SReg) in insertVectorSelect()
1336 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
1359 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_EQ_I32_e64), Reg) in insertEQ()
1372 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_NE_I32_e64), Reg) in insertNE()
1382 return AMDGPU::COPY; in getMovOpcode()
1386 return RI.isSGPRClass(DstRC) ? AMDGPU::COPY : AMDGPU::V_MOV_B16_t16_e64; in getMovOpcode()
1388 return RI.isSGPRClass(DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32; in getMovOpcode()
1390 return AMDGPU::S_MOV_B64; in getMovOpcode()
1392 return AMDGPU::V_MOV_B64_PSEUDO; in getMovOpcode()
1394 return AMDGPU::COPY; in getMovOpcode()
1402 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1); in getIndirectGPRIDXPseudo()
1404 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2); in getIndirectGPRIDXPseudo()
1406 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3); in getIndirectGPRIDXPseudo()
1408 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4); in getIndirectGPRIDXPseudo()
1410 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5); in getIndirectGPRIDXPseudo()
1412 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8); in getIndirectGPRIDXPseudo()
1414 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V9); in getIndirectGPRIDXPseudo()
1416 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V10); in getIndirectGPRIDXPseudo()
1418 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V11); in getIndirectGPRIDXPseudo()
1420 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V12); in getIndirectGPRIDXPseudo()
1422 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16); in getIndirectGPRIDXPseudo()
1424 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32); in getIndirectGPRIDXPseudo()
1430 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1); in getIndirectGPRIDXPseudo()
1432 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2); in getIndirectGPRIDXPseudo()
1434 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3); in getIndirectGPRIDXPseudo()
1436 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4); in getIndirectGPRIDXPseudo()
1438 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5); in getIndirectGPRIDXPseudo()
1440 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8); in getIndirectGPRIDXPseudo()
1442 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V9); in getIndirectGPRIDXPseudo()
1444 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V10); in getIndirectGPRIDXPseudo()
1446 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V11); in getIndirectGPRIDXPseudo()
1448 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V12); in getIndirectGPRIDXPseudo()
1450 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16); in getIndirectGPRIDXPseudo()
1452 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32); in getIndirectGPRIDXPseudo()
1459 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1; in getIndirectVGPRWriteMovRelPseudoOpc()
1461 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2; in getIndirectVGPRWriteMovRelPseudoOpc()
1463 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3; in getIndirectVGPRWriteMovRelPseudoOpc()
1465 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4; in getIndirectVGPRWriteMovRelPseudoOpc()
1467 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5; in getIndirectVGPRWriteMovRelPseudoOpc()
1469 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8; in getIndirectVGPRWriteMovRelPseudoOpc()
1471 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V9; in getIndirectVGPRWriteMovRelPseudoOpc()
1473 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V10; in getIndirectVGPRWriteMovRelPseudoOpc()
1475 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V11; in getIndirectVGPRWriteMovRelPseudoOpc()
1477 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V12; in getIndirectVGPRWriteMovRelPseudoOpc()
1479 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16; in getIndirectVGPRWriteMovRelPseudoOpc()
1481 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32; in getIndirectVGPRWriteMovRelPseudoOpc()
1488 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1; in getIndirectSGPRWriteMovRelPseudo32()
1490 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2; in getIndirectSGPRWriteMovRelPseudo32()
1492 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3; in getIndirectSGPRWriteMovRelPseudo32()
1494 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4; in getIndirectSGPRWriteMovRelPseudo32()
1496 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5; in getIndirectSGPRWriteMovRelPseudo32()
1498 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8; in getIndirectSGPRWriteMovRelPseudo32()
1500 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V9; in getIndirectSGPRWriteMovRelPseudo32()
1502 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V10; in getIndirectSGPRWriteMovRelPseudo32()
1504 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V11; in getIndirectSGPRWriteMovRelPseudo32()
1506 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V12; in getIndirectSGPRWriteMovRelPseudo32()
1508 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16; in getIndirectSGPRWriteMovRelPseudo32()
1510 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32; in getIndirectSGPRWriteMovRelPseudo32()
1517 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1; in getIndirectSGPRWriteMovRelPseudo64()
1519 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2; in getIndirectSGPRWriteMovRelPseudo64()
1521 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4; in getIndirectSGPRWriteMovRelPseudo64()
1523 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8; in getIndirectSGPRWriteMovRelPseudo64()
1525 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16; in getIndirectSGPRWriteMovRelPseudo64()
1551 return AMDGPU::SI_SPILL_S32_SAVE; in getSGPRSpillSaveOpcode()
1553 return AMDGPU::SI_SPILL_S64_SAVE; in getSGPRSpillSaveOpcode()
1555 return AMDGPU::SI_SPILL_S96_SAVE; in getSGPRSpillSaveOpcode()
1557 return AMDGPU::SI_SPILL_S128_SAVE; in getSGPRSpillSaveOpcode()
1559 return AMDGPU::SI_SPILL_S160_SAVE; in getSGPRSpillSaveOpcode()
1561 return AMDGPU::SI_SPILL_S192_SAVE; in getSGPRSpillSaveOpcode()
1563 return AMDGPU::SI_SPILL_S224_SAVE; in getSGPRSpillSaveOpcode()
1565 return AMDGPU::SI_SPILL_S256_SAVE; in getSGPRSpillSaveOpcode()
1567 return AMDGPU::SI_SPILL_S288_SAVE; in getSGPRSpillSaveOpcode()
1569 return AMDGPU::SI_SPILL_S320_SAVE; in getSGPRSpillSaveOpcode()
1571 return AMDGPU::SI_SPILL_S352_SAVE; in getSGPRSpillSaveOpcode()
1573 return AMDGPU::SI_SPILL_S384_SAVE; in getSGPRSpillSaveOpcode()
1575 return AMDGPU::SI_SPILL_S512_SAVE; in getSGPRSpillSaveOpcode()
1577 return AMDGPU::SI_SPILL_S1024_SAVE; in getSGPRSpillSaveOpcode()
1586 return AMDGPU::SI_SPILL_V32_SAVE; in getVGPRSpillSaveOpcode()
1588 return AMDGPU::SI_SPILL_V64_SAVE; in getVGPRSpillSaveOpcode()
1590 return AMDGPU::SI_SPILL_V96_SAVE; in getVGPRSpillSaveOpcode()
1592 return AMDGPU::SI_SPILL_V128_SAVE; in getVGPRSpillSaveOpcode()
1594 return AMDGPU::SI_SPILL_V160_SAVE; in getVGPRSpillSaveOpcode()
1596 return AMDGPU::SI_SPILL_V192_SAVE; in getVGPRSpillSaveOpcode()
1598 return AMDGPU::SI_SPILL_V224_SAVE; in getVGPRSpillSaveOpcode()
1600 return AMDGPU::SI_SPILL_V256_SAVE; in getVGPRSpillSaveOpcode()
1602 return AMDGPU::SI_SPILL_V288_SAVE; in getVGPRSpillSaveOpcode()
1604 return AMDGPU::SI_SPILL_V320_SAVE; in getVGPRSpillSaveOpcode()
1606 return AMDGPU::SI_SPILL_V352_SAVE; in getVGPRSpillSaveOpcode()
1608 return AMDGPU::SI_SPILL_V384_SAVE; in getVGPRSpillSaveOpcode()
1610 return AMDGPU::SI_SPILL_V512_SAVE; in getVGPRSpillSaveOpcode()
1612 return AMDGPU::SI_SPILL_V1024_SAVE; in getVGPRSpillSaveOpcode()
1621 return AMDGPU::SI_SPILL_A32_SAVE; in getAGPRSpillSaveOpcode()
1623 return AMDGPU::SI_SPILL_A64_SAVE; in getAGPRSpillSaveOpcode()
1625 return AMDGPU::SI_SPILL_A96_SAVE; in getAGPRSpillSaveOpcode()
1627 return AMDGPU::SI_SPILL_A128_SAVE; in getAGPRSpillSaveOpcode()
1629 return AMDGPU::SI_SPILL_A160_SAVE; in getAGPRSpillSaveOpcode()
1631 return AMDGPU::SI_SPILL_A192_SAVE; in getAGPRSpillSaveOpcode()
1633 return AMDGPU::SI_SPILL_A224_SAVE; in getAGPRSpillSaveOpcode()
1635 return AMDGPU::SI_SPILL_A256_SAVE; in getAGPRSpillSaveOpcode()
1637 return AMDGPU::SI_SPILL_A288_SAVE; in getAGPRSpillSaveOpcode()
1639 return AMDGPU::SI_SPILL_A320_SAVE; in getAGPRSpillSaveOpcode()
1641 return AMDGPU::SI_SPILL_A352_SAVE; in getAGPRSpillSaveOpcode()
1643 return AMDGPU::SI_SPILL_A384_SAVE; in getAGPRSpillSaveOpcode()
1645 return AMDGPU::SI_SPILL_A512_SAVE; in getAGPRSpillSaveOpcode()
1647 return AMDGPU::SI_SPILL_A1024_SAVE; in getAGPRSpillSaveOpcode()
1656 return AMDGPU::SI_SPILL_AV32_SAVE; in getAVSpillSaveOpcode()
1658 return AMDGPU::SI_SPILL_AV64_SAVE; in getAVSpillSaveOpcode()
1660 return AMDGPU::SI_SPILL_AV96_SAVE; in getAVSpillSaveOpcode()
1662 return AMDGPU::SI_SPILL_AV128_SAVE; in getAVSpillSaveOpcode()
1664 return AMDGPU::SI_SPILL_AV160_SAVE; in getAVSpillSaveOpcode()
1666 return AMDGPU::SI_SPILL_AV192_SAVE; in getAVSpillSaveOpcode()
1668 return AMDGPU::SI_SPILL_AV224_SAVE; in getAVSpillSaveOpcode()
1670 return AMDGPU::SI_SPILL_AV256_SAVE; in getAVSpillSaveOpcode()
1672 return AMDGPU::SI_SPILL_AV288_SAVE; in getAVSpillSaveOpcode()
1674 return AMDGPU::SI_SPILL_AV320_SAVE; in getAVSpillSaveOpcode()
1676 return AMDGPU::SI_SPILL_AV352_SAVE; in getAVSpillSaveOpcode()
1678 return AMDGPU::SI_SPILL_AV384_SAVE; in getAVSpillSaveOpcode()
1680 return AMDGPU::SI_SPILL_AV512_SAVE; in getAVSpillSaveOpcode()
1682 return AMDGPU::SI_SPILL_AV1024_SAVE; in getAVSpillSaveOpcode()
1695 return AMDGPU::SI_SPILL_WWM_AV32_SAVE; in getWWMRegSpillSaveOpcode()
1697 return AMDGPU::SI_SPILL_WWM_V32_SAVE; in getWWMRegSpillSaveOpcode()
1708 if (MFI.checkFlag(Reg, AMDGPU::VirtRegFlag::WWM_REG)) in getVectorRegSpillSaveOpcode()
1737 assert(SrcReg != AMDGPU::M0 && "m0 should not be spilled"); in storeRegToStackSlot()
1738 assert(SrcReg != AMDGPU::EXEC_LO && SrcReg != AMDGPU::EXEC_HI && in storeRegToStackSlot()
1739 SrcReg != AMDGPU::EXEC && "exec should not be spilled"); in storeRegToStackSlot()
1748 MRI.constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0_XEXECRegClass); in storeRegToStackSlot()
1777 return AMDGPU::SI_SPILL_S32_RESTORE; in getSGPRSpillRestoreOpcode()
1779 return AMDGPU::SI_SPILL_S64_RESTORE; in getSGPRSpillRestoreOpcode()
1781 return AMDGPU::SI_SPILL_S96_RESTORE; in getSGPRSpillRestoreOpcode()
1783 return AMDGPU::SI_SPILL_S128_RESTORE; in getSGPRSpillRestoreOpcode()
1785 return AMDGPU::SI_SPILL_S160_RESTORE; in getSGPRSpillRestoreOpcode()
1787 return AMDGPU::SI_SPILL_S192_RESTORE; in getSGPRSpillRestoreOpcode()
1789 return AMDGPU::SI_SPILL_S224_RESTORE; in getSGPRSpillRestoreOpcode()
1791 return AMDGPU::SI_SPILL_S256_RESTORE; in getSGPRSpillRestoreOpcode()
1793 return AMDGPU::SI_SPILL_S288_RESTORE; in getSGPRSpillRestoreOpcode()
1795 return AMDGPU::SI_SPILL_S320_RESTORE; in getSGPRSpillRestoreOpcode()
1797 return AMDGPU::SI_SPILL_S352_RESTORE; in getSGPRSpillRestoreOpcode()
1799 return AMDGPU::SI_SPILL_S384_RESTORE; in getSGPRSpillRestoreOpcode()
1801 return AMDGPU::SI_SPILL_S512_RESTORE; in getSGPRSpillRestoreOpcode()
1803 return AMDGPU::SI_SPILL_S1024_RESTORE; in getSGPRSpillRestoreOpcode()
1812 return AMDGPU::SI_SPILL_V32_RESTORE; in getVGPRSpillRestoreOpcode()
1814 return AMDGPU::SI_SPILL_V64_RESTORE; in getVGPRSpillRestoreOpcode()
1816 return AMDGPU::SI_SPILL_V96_RESTORE; in getVGPRSpillRestoreOpcode()
1818 return AMDGPU::SI_SPILL_V128_RESTORE; in getVGPRSpillRestoreOpcode()
1820 return AMDGPU::SI_SPILL_V160_RESTORE; in getVGPRSpillRestoreOpcode()
1822 return AMDGPU::SI_SPILL_V192_RESTORE; in getVGPRSpillRestoreOpcode()
1824 return AMDGPU::SI_SPILL_V224_RESTORE; in getVGPRSpillRestoreOpcode()
1826 return AMDGPU::SI_SPILL_V256_RESTORE; in getVGPRSpillRestoreOpcode()
1828 return AMDGPU::SI_SPILL_V288_RESTORE; in getVGPRSpillRestoreOpcode()
1830 return AMDGPU::SI_SPILL_V320_RESTORE; in getVGPRSpillRestoreOpcode()
1832 return AMDGPU::SI_SPILL_V352_RESTORE; in getVGPRSpillRestoreOpcode()
1834 return AMDGPU::SI_SPILL_V384_RESTORE; in getVGPRSpillRestoreOpcode()
1836 return AMDGPU::SI_SPILL_V512_RESTORE; in getVGPRSpillRestoreOpcode()
1838 return AMDGPU::SI_SPILL_V1024_RESTORE; in getVGPRSpillRestoreOpcode()
1847 return AMDGPU::SI_SPILL_A32_RESTORE; in getAGPRSpillRestoreOpcode()
1849 return AMDGPU::SI_SPILL_A64_RESTORE; in getAGPRSpillRestoreOpcode()
1851 return AMDGPU::SI_SPILL_A96_RESTORE; in getAGPRSpillRestoreOpcode()
1853 return AMDGPU::SI_SPILL_A128_RESTORE; in getAGPRSpillRestoreOpcode()
1855 return AMDGPU::SI_SPILL_A160_RESTORE; in getAGPRSpillRestoreOpcode()
1857 return AMDGPU::SI_SPILL_A192_RESTORE; in getAGPRSpillRestoreOpcode()
1859 return AMDGPU::SI_SPILL_A224_RESTORE; in getAGPRSpillRestoreOpcode()
1861 return AMDGPU::SI_SPILL_A256_RESTORE; in getAGPRSpillRestoreOpcode()
1863 return AMDGPU::SI_SPILL_A288_RESTORE; in getAGPRSpillRestoreOpcode()
1865 return AMDGPU::SI_SPILL_A320_RESTORE; in getAGPRSpillRestoreOpcode()
1867 return AMDGPU::SI_SPILL_A352_RESTORE; in getAGPRSpillRestoreOpcode()
1869 return AMDGPU::SI_SPILL_A384_RESTORE; in getAGPRSpillRestoreOpcode()
1871 return AMDGPU::SI_SPILL_A512_RESTORE; in getAGPRSpillRestoreOpcode()
1873 return AMDGPU::SI_SPILL_A1024_RESTORE; in getAGPRSpillRestoreOpcode()
1882 return AMDGPU::SI_SPILL_AV32_RESTORE; in getAVSpillRestoreOpcode()
1884 return AMDGPU::SI_SPILL_AV64_RESTORE; in getAVSpillRestoreOpcode()
1886 return AMDGPU::SI_SPILL_AV96_RESTORE; in getAVSpillRestoreOpcode()
1888 return AMDGPU::SI_SPILL_AV128_RESTORE; in getAVSpillRestoreOpcode()
1890 return AMDGPU::SI_SPILL_AV160_RESTORE; in getAVSpillRestoreOpcode()
1892 return AMDGPU::SI_SPILL_AV192_RESTORE; in getAVSpillRestoreOpcode()
1894 return AMDGPU::SI_SPILL_AV224_RESTORE; in getAVSpillRestoreOpcode()
1896 return AMDGPU::SI_SPILL_AV256_RESTORE; in getAVSpillRestoreOpcode()
1898 return AMDGPU::SI_SPILL_AV288_RESTORE; in getAVSpillRestoreOpcode()
1900 return AMDGPU::SI_SPILL_AV320_RESTORE; in getAVSpillRestoreOpcode()
1902 return AMDGPU::SI_SPILL_AV352_RESTORE; in getAVSpillRestoreOpcode()
1904 return AMDGPU::SI_SPILL_AV384_RESTORE; in getAVSpillRestoreOpcode()
1906 return AMDGPU::SI_SPILL_AV512_RESTORE; in getAVSpillRestoreOpcode()
1908 return AMDGPU::SI_SPILL_AV1024_RESTORE; in getAVSpillRestoreOpcode()
1921 return AMDGPU::SI_SPILL_WWM_AV32_RESTORE; in getWWMRegSpillRestoreOpcode()
1923 return AMDGPU::SI_SPILL_WWM_V32_RESTORE; in getWWMRegSpillRestoreOpcode()
1933 if (MFI.checkFlag(Reg, AMDGPU::VirtRegFlag::WWM_REG)) in getVectorRegSpillRestoreOpcode()
1964 assert(DestReg != AMDGPU::M0 && "m0 should not be reloaded into"); in loadRegFromStackSlot()
1965 assert(DestReg != AMDGPU::EXEC_LO && DestReg != AMDGPU::EXEC_HI && in loadRegFromStackSlot()
1966 DestReg != AMDGPU::EXEC && "exec should not be spilled"); in loadRegFromStackSlot()
1973 MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0_XEXECRegClass); in loadRegFromStackSlot()
2007 BuildMI(MBB, MI, DL, get(AMDGPU::S_NOP)).addImm(Arg - 1); in insertNoops()
2021 BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::S_ENDPGM)).addImm(0); in insertReturn()
2023 BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::SI_RETURN_TO_EPILOG)); in insertReturn()
2036 case AMDGPU::S_NOP: in getNumWaitStates()
2049 case AMDGPU::S_MOV_B64_term: in expandPostRAPseudo()
2052 MI.setDesc(get(AMDGPU::S_MOV_B64)); in expandPostRAPseudo()
2055 case AMDGPU::S_MOV_B32_term: in expandPostRAPseudo()
2058 MI.setDesc(get(AMDGPU::S_MOV_B32)); in expandPostRAPseudo()
2061 case AMDGPU::S_XOR_B64_term: in expandPostRAPseudo()
2064 MI.setDesc(get(AMDGPU::S_XOR_B64)); in expandPostRAPseudo()
2067 case AMDGPU::S_XOR_B32_term: in expandPostRAPseudo()
2070 MI.setDesc(get(AMDGPU::S_XOR_B32)); in expandPostRAPseudo()
2072 case AMDGPU::S_OR_B64_term: in expandPostRAPseudo()
2075 MI.setDesc(get(AMDGPU::S_OR_B64)); in expandPostRAPseudo()
2077 case AMDGPU::S_OR_B32_term: in expandPostRAPseudo()
2080 MI.setDesc(get(AMDGPU::S_OR_B32)); in expandPostRAPseudo()
2083 case AMDGPU::S_ANDN2_B64_term: in expandPostRAPseudo()
2086 MI.setDesc(get(AMDGPU::S_ANDN2_B64)); in expandPostRAPseudo()
2089 case AMDGPU::S_ANDN2_B32_term: in expandPostRAPseudo()
2092 MI.setDesc(get(AMDGPU::S_ANDN2_B32)); in expandPostRAPseudo()
2095 case AMDGPU::S_AND_B64_term: in expandPostRAPseudo()
2098 MI.setDesc(get(AMDGPU::S_AND_B64)); in expandPostRAPseudo()
2101 case AMDGPU::S_AND_B32_term: in expandPostRAPseudo()
2104 MI.setDesc(get(AMDGPU::S_AND_B32)); in expandPostRAPseudo()
2107 case AMDGPU::S_AND_SAVEEXEC_B64_term: in expandPostRAPseudo()
2110 MI.setDesc(get(AMDGPU::S_AND_SAVEEXEC_B64)); in expandPostRAPseudo()
2113 case AMDGPU::S_AND_SAVEEXEC_B32_term: in expandPostRAPseudo()
2116 MI.setDesc(get(AMDGPU::S_AND_SAVEEXEC_B32)); in expandPostRAPseudo()
2119 case AMDGPU::SI_SPILL_S32_TO_VGPR: in expandPostRAPseudo()
2120 MI.setDesc(get(AMDGPU::V_WRITELANE_B32)); in expandPostRAPseudo()
2123 case AMDGPU::SI_RESTORE_S32_FROM_VGPR: in expandPostRAPseudo()
2124 MI.setDesc(get(AMDGPU::V_READLANE_B32)); in expandPostRAPseudo()
2127 case AMDGPU::V_MOV_B64_PSEUDO: { in expandPostRAPseudo()
2129 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0); in expandPostRAPseudo()
2130 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1); in expandPostRAPseudo()
2136 MI.setDesc(get(AMDGPU::V_MOV_B64_e32)); in expandPostRAPseudo()
2146 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), Dst) in expandPostRAPseudo()
2157 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo) in expandPostRAPseudo()
2160 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi) in expandPostRAPseudo()
2168 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), Dst) in expandPostRAPseudo()
2179 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo) in expandPostRAPseudo()
2180 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub0)) in expandPostRAPseudo()
2182 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi) in expandPostRAPseudo()
2183 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub1)) in expandPostRAPseudo()
2190 case AMDGPU::V_MOV_B64_DPP_PSEUDO: { in expandPostRAPseudo()
2194 case AMDGPU::S_MOV_B64_IMM_PSEUDO: { in expandPostRAPseudo()
2199 MI.setDesc(get(AMDGPU::S_MOV_B64)); in expandPostRAPseudo()
2204 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0); in expandPostRAPseudo()
2205 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1); in expandPostRAPseudo()
2209 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DstLo) in expandPostRAPseudo()
2212 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DstHi) in expandPostRAPseudo()
2218 case AMDGPU::V_SET_INACTIVE_B32: { in expandPostRAPseudo()
2219 unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64; in expandPostRAPseudo()
2220 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; in expandPostRAPseudo()
2223 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg()) in expandPostRAPseudo()
2226 FirstNot->addRegisterDead(AMDGPU::SCC, TRI); // SCC is overwritten in expandPostRAPseudo()
2227 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg()) in expandPostRAPseudo()
2234 case AMDGPU::V_SET_INACTIVE_B64: { in expandPostRAPseudo()
2235 unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64; in expandPostRAPseudo()
2236 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; in expandPostRAPseudo()
2237 MachineInstr *Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), in expandPostRAPseudo()
2242 FirstNot->addRegisterDead(AMDGPU::SCC, TRI); // SCC is overwritten in expandPostRAPseudo()
2243 Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), in expandPostRAPseudo()
2252 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1: in expandPostRAPseudo()
2253 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2: in expandPostRAPseudo()
2254 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3: in expandPostRAPseudo()
2255 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4: in expandPostRAPseudo()
2256 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5: in expandPostRAPseudo()
2257 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8: in expandPostRAPseudo()
2258 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V9: in expandPostRAPseudo()
2259 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V10: in expandPostRAPseudo()
2260 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V11: in expandPostRAPseudo()
2261 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V12: in expandPostRAPseudo()
2262 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16: in expandPostRAPseudo()
2263 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32: in expandPostRAPseudo()
2264 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1: in expandPostRAPseudo()
2265 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2: in expandPostRAPseudo()
2266 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3: in expandPostRAPseudo()
2267 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4: in expandPostRAPseudo()
2268 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5: in expandPostRAPseudo()
2269 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8: in expandPostRAPseudo()
2270 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V9: in expandPostRAPseudo()
2271 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V10: in expandPostRAPseudo()
2272 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V11: in expandPostRAPseudo()
2273 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V12: in expandPostRAPseudo()
2274 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16: in expandPostRAPseudo()
2275 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32: in expandPostRAPseudo()
2276 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1: in expandPostRAPseudo()
2277 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2: in expandPostRAPseudo()
2278 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4: in expandPostRAPseudo()
2279 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8: in expandPostRAPseudo()
2280 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16: { in expandPostRAPseudo()
2285 Opc = AMDGPU::V_MOVRELD_B32_e32; in expandPostRAPseudo()
2287 Opc = RI.getRegSizeInBits(*EltRC) == 64 ? AMDGPU::S_MOVRELD_B64 in expandPostRAPseudo()
2288 : AMDGPU::S_MOVRELD_B32; in expandPostRAPseudo()
2311 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1: in expandPostRAPseudo()
2312 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2: in expandPostRAPseudo()
2313 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3: in expandPostRAPseudo()
2314 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4: in expandPostRAPseudo()
2315 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5: in expandPostRAPseudo()
2316 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8: in expandPostRAPseudo()
2317 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V9: in expandPostRAPseudo()
2318 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V10: in expandPostRAPseudo()
2319 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V11: in expandPostRAPseudo()
2320 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V12: in expandPostRAPseudo()
2321 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16: in expandPostRAPseudo()
2322 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32: { in expandPostRAPseudo()
2329 MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON)) in expandPostRAPseudo()
2331 .addImm(AMDGPU::VGPRIndexMode::DST_ENABLE); in expandPostRAPseudo()
2334 const MCInstrDesc &OpDesc = get(AMDGPU::V_MOV_B32_indirect_write); in expandPostRAPseudo()
2348 MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF)); in expandPostRAPseudo()
2355 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1: in expandPostRAPseudo()
2356 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2: in expandPostRAPseudo()
2357 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3: in expandPostRAPseudo()
2358 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4: in expandPostRAPseudo()
2359 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5: in expandPostRAPseudo()
2360 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8: in expandPostRAPseudo()
2361 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V9: in expandPostRAPseudo()
2362 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V10: in expandPostRAPseudo()
2363 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V11: in expandPostRAPseudo()
2364 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V12: in expandPostRAPseudo()
2365 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16: in expandPostRAPseudo()
2366 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32: { in expandPostRAPseudo()
2374 MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON)) in expandPostRAPseudo()
2376 .addImm(AMDGPU::VGPRIndexMode::SRC0_ENABLE); in expandPostRAPseudo()
2379 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_indirect_read)) in expandPostRAPseudo()
2384 MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF)); in expandPostRAPseudo()
2391 case AMDGPU::SI_PC_ADD_REL_OFFSET: { in expandPostRAPseudo()
2394 Register RegLo = RI.getSubReg(Reg, AMDGPU::sub0); in expandPostRAPseudo()
2395 Register RegHi = RI.getSubReg(Reg, AMDGPU::sub1); in expandPostRAPseudo()
2402 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_GETPC_B64), Reg)); in expandPostRAPseudo()
2418 BuildMI(MF, DL, get(AMDGPU::S_SEXT_I32_I16), RegHi).addReg(RegHi)); in expandPostRAPseudo()
2425 BuildMI(MF, DL, get(AMDGPU::S_ADD_U32), RegLo).addReg(RegLo).add(OpLo)); in expandPostRAPseudo()
2429 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_ADDC_U32), RegHi) in expandPostRAPseudo()
2438 case AMDGPU::ENTER_STRICT_WWM: { in expandPostRAPseudo()
2441 MI.setDesc(get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32 in expandPostRAPseudo()
2442 : AMDGPU::S_OR_SAVEEXEC_B64)); in expandPostRAPseudo()
2445 case AMDGPU::ENTER_STRICT_WQM: { in expandPostRAPseudo()
2448 const unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; in expandPostRAPseudo()
2449 const unsigned WQMOp = ST.isWave32() ? AMDGPU::S_WQM_B32 : AMDGPU::S_WQM_B64; in expandPostRAPseudo()
2450 const unsigned MovOp = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; in expandPostRAPseudo()
2457 case AMDGPU::EXIT_STRICT_WWM: in expandPostRAPseudo()
2458 case AMDGPU::EXIT_STRICT_WQM: { in expandPostRAPseudo()
2461 MI.setDesc(get(ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64)); in expandPostRAPseudo()
2464 case AMDGPU::ENTER_PSEUDO_WM: in expandPostRAPseudo()
2465 case AMDGPU::EXIT_PSEUDO_WM: { in expandPostRAPseudo()
2470 case AMDGPU::SI_RETURN: { in expandPostRAPseudo()
2480 BuildMI(MBB, MI, DL, get(AMDGPU::S_SETPC_B64_return)) in expandPostRAPseudo()
2488 case AMDGPU::S_MUL_U64_U32_PSEUDO: in expandPostRAPseudo()
2489 case AMDGPU::S_MUL_I64_I32_PSEUDO: in expandPostRAPseudo()
2490 MI.setDesc(get(AMDGPU::S_MUL_U64)); in expandPostRAPseudo()
2493 case AMDGPU::S_GETPC_B64_pseudo: in expandPostRAPseudo()
2494 MI.setDesc(get(AMDGPU::S_GETPC_B64)); in expandPostRAPseudo()
2497 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1); in expandPostRAPseudo()
2500 BuildMI(MBB, std::next(MI.getIterator()), DL, get(AMDGPU::S_SEXT_I32_I16), in expandPostRAPseudo()
2519 case AMDGPU::S_LOAD_DWORDX16_IMM: in reMaterialize()
2520 case AMDGPU::S_LOAD_DWORDX8_IMM: { in reMaterialize()
2542 if (!UseMO || UseMO->getSubReg() == AMDGPU::NoSubRegister) in reMaterialize()
2554 NewOpcode = AMDGPU::S_LOAD_DWORDX8_IMM; in reMaterialize()
2556 NewOpcode = AMDGPU::S_LOAD_DWORDX4_IMM; in reMaterialize()
2566 UseMO->setSubReg(AMDGPU::NoSubRegister); in reMaterialize()
2572 MI->getOperand(0).setSubReg(AMDGPU::NoSubRegister); in reMaterialize()
2574 MachineOperand *OffsetMO = getNamedOperand(*MI, AMDGPU::OpName::offset); in reMaterialize()
2597 assert (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO); in expandMovDPP64()
2600 AMDGPU::isLegalDPALU_DPPControl( in expandMovDPP64()
2601 getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl)->getImm())) { in expandMovDPP64()
2602 MI.setDesc(get(AMDGPU::V_MOV_B64_dpp)); in expandMovDPP64()
2614 for (auto Sub : { AMDGPU::sub0, AMDGPU::sub1 }) { in expandMovDPP64()
2615 auto MovDPP = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_dpp)); in expandMovDPP64()
2620 auto Tmp = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in expandMovDPP64()
2649 BuildMI(MBB, MI, DL, get(AMDGPU::REG_SEQUENCE), Dst) in expandMovDPP64()
2651 .addImm(AMDGPU::sub0) in expandMovDPP64()
2653 .addImm(AMDGPU::sub1); in expandMovDPP64()
2661 if (MI.getOpcode() == AMDGPU::WWM_COPY) in isCopyInstrImpl()
2730 assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) == in commuteInstructionImpl()
2732 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) == in commuteInstructionImpl()
2760 swapSourceModifiers(MI, Src0, AMDGPU::OpName::src0_modifiers, in commuteInstructionImpl()
2761 Src1, AMDGPU::OpName::src1_modifiers); in commuteInstructionImpl()
2785 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); in findCommutedOpIndices()
2789 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); in findCommutedOpIndices()
2800 assert(BranchOp != AMDGPU::S_SETPC_B64); in isBranchOffsetInRange()
2819 if (MI.getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO || in hasDivergentBranch()
2820 MI.getOpcode() == AMDGPU::SI_IF || MI.getOpcode() == AMDGPU::SI_ELSE || in hasDivergentBranch()
2821 MI.getOpcode() == AMDGPU::SI_LOOP) in hasDivergentBranch()
2845 Register PCReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in insertIndirectBranch()
2851 MachineInstr *GetPC = BuildMI(MBB, I, DL, get(AMDGPU::S_GETPC_B64), PCReg); in insertIndirectBranch()
2862 BuildMI(MBB, I, DL, get(AMDGPU::S_ADD_U32)) in insertIndirectBranch()
2863 .addReg(PCReg, RegState::Define, AMDGPU::sub0) in insertIndirectBranch()
2864 .addReg(PCReg, 0, AMDGPU::sub0) in insertIndirectBranch()
2866 BuildMI(MBB, I, DL, get(AMDGPU::S_ADDC_U32)) in insertIndirectBranch()
2867 .addReg(PCReg, RegState::Define, AMDGPU::sub1) in insertIndirectBranch()
2868 .addReg(PCReg, 0, AMDGPU::sub1) in insertIndirectBranch()
2872 BuildMI(&MBB, DL, get(AMDGPU::S_SETPC_B64)) in insertIndirectBranch()
2916 AMDGPU::SReg_64RegClass, MachineBasicBlock::iterator(GetPC), in insertIndirectBranch()
2928 TRI->spillEmergencySGPR(GetPC, RestoreBB, AMDGPU::SGPR0_SGPR1, RS); in insertIndirectBranch()
2929 MRI.replaceRegWith(PCReg, AMDGPU::SGPR0_SGPR1); in insertIndirectBranch()
2948 return AMDGPU::S_CBRANCH_SCC1; in getBranchOpcode()
2950 return AMDGPU::S_CBRANCH_SCC0; in getBranchOpcode()
2952 return AMDGPU::S_CBRANCH_VCCNZ; in getBranchOpcode()
2954 return AMDGPU::S_CBRANCH_VCCZ; in getBranchOpcode()
2956 return AMDGPU::S_CBRANCH_EXECNZ; in getBranchOpcode()
2958 return AMDGPU::S_CBRANCH_EXECZ; in getBranchOpcode()
2966 case AMDGPU::S_CBRANCH_SCC0: in getBranchPredicate()
2968 case AMDGPU::S_CBRANCH_SCC1: in getBranchPredicate()
2970 case AMDGPU::S_CBRANCH_VCCNZ: in getBranchPredicate()
2972 case AMDGPU::S_CBRANCH_VCCZ: in getBranchPredicate()
2974 case AMDGPU::S_CBRANCH_EXECNZ: in getBranchPredicate()
2976 case AMDGPU::S_CBRANCH_EXECZ: in getBranchPredicate()
2989 if (I->getOpcode() == AMDGPU::S_BRANCH) { in analyzeBranchImpl()
2997 if (I->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) { in analyzeBranchImpl()
3017 if (I->getOpcode() == AMDGPU::S_BRANCH) { in analyzeBranchImpl()
3039 case AMDGPU::S_MOV_B64_term: in analyzeBranch()
3040 case AMDGPU::S_XOR_B64_term: in analyzeBranch()
3041 case AMDGPU::S_OR_B64_term: in analyzeBranch()
3042 case AMDGPU::S_ANDN2_B64_term: in analyzeBranch()
3043 case AMDGPU::S_AND_B64_term: in analyzeBranch()
3044 case AMDGPU::S_AND_SAVEEXEC_B64_term: in analyzeBranch()
3045 case AMDGPU::S_MOV_B32_term: in analyzeBranch()
3046 case AMDGPU::S_XOR_B32_term: in analyzeBranch()
3047 case AMDGPU::S_OR_B32_term: in analyzeBranch()
3048 case AMDGPU::S_ANDN2_B32_term: in analyzeBranch()
3049 case AMDGPU::S_AND_B32_term: in analyzeBranch()
3050 case AMDGPU::S_AND_SAVEEXEC_B32_term: in analyzeBranch()
3052 case AMDGPU::SI_IF: in analyzeBranch()
3053 case AMDGPU::SI_ELSE: in analyzeBranch()
3054 case AMDGPU::SI_KILL_I1_TERMINATOR: in analyzeBranch()
3055 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR: in analyzeBranch()
3104 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH)) in insertBranch()
3112 BuildMI(&MBB, DL, get(AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO)) in insertBranch()
3143 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH)) in insertBranch()
3183 int NumInsts = AMDGPU::getRegBitWidth(*RC) / 32; in canInsertSelect()
3198 int NumInsts = AMDGPU::getRegBitWidth(*RC) / 32; in canInsertSelect()
3229 Select = BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B32), DstReg) in insertSelect()
3234 Select = BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e32), DstReg) in insertSelect()
3245 BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), DstReg) in insertSelect()
3254 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3, in insertSelect()
3255 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7, in insertSelect()
3256 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11, in insertSelect()
3257 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15, in insertSelect()
3261 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3, in insertSelect()
3262 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7, in insertSelect()
3263 AMDGPU::sub8_sub9, AMDGPU::sub10_sub11, in insertSelect()
3264 AMDGPU::sub12_sub13, AMDGPU::sub14_sub15, in insertSelect()
3267 unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32; in insertSelect()
3268 const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass; in insertSelect()
3276 SelOp = AMDGPU::S_CSELECT_B32; in insertSelect()
3277 EltRC = &AMDGPU::SGPR_32RegClass; in insertSelect()
3279 SelOp = AMDGPU::S_CSELECT_B64; in insertSelect()
3280 EltRC = &AMDGPU::SGPR_64RegClass; in insertSelect()
3287 MBB, I, DL, get(AMDGPU::REG_SEQUENCE), DstReg); in insertSelect()
3299 if (SelOp == AMDGPU::V_CNDMASK_B32_e32) { in insertSelect()
3321 case AMDGPU::V_MOV_B32_e32: in isFoldableCopy()
3322 case AMDGPU::V_MOV_B32_e64: in isFoldableCopy()
3323 case AMDGPU::V_MOV_B64_PSEUDO: in isFoldableCopy()
3324 case AMDGPU::V_MOV_B64_e32: in isFoldableCopy()
3325 case AMDGPU::V_MOV_B64_e64: in isFoldableCopy()
3326 case AMDGPU::S_MOV_B32: in isFoldableCopy()
3327 case AMDGPU::S_MOV_B64: in isFoldableCopy()
3328 case AMDGPU::S_MOV_B64_IMM_PSEUDO: in isFoldableCopy()
3329 case AMDGPU::COPY: in isFoldableCopy()
3330 case AMDGPU::WWM_COPY: in isFoldableCopy()
3331 case AMDGPU::V_ACCVGPR_WRITE_B32_e64: in isFoldableCopy()
3332 case AMDGPU::V_ACCVGPR_READ_B32_e64: in isFoldableCopy()
3333 case AMDGPU::V_ACCVGPR_MOV_B32: in isFoldableCopy()
3341 AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src1_modifiers,
3342 AMDGPU::OpName::src2_modifiers, AMDGPU::OpName::clamp,
3343 AMDGPU::OpName::omod, AMDGPU::OpName::op_sel};
3348 int Idx = AMDGPU::getNamedOperandIdx(Opc, Name); in removeModOperands()
3362 case AMDGPU::V_MOV_B64_e32: in FoldImmediate()
3363 case AMDGPU::S_MOV_B64: in FoldImmediate()
3364 case AMDGPU::V_MOV_B64_PSEUDO: in FoldImmediate()
3365 case AMDGPU::S_MOV_B64_IMM_PSEUDO: in FoldImmediate()
3366 case AMDGPU::V_MOV_B32_e32: in FoldImmediate()
3367 case AMDGPU::S_MOV_B32: in FoldImmediate()
3368 case AMDGPU::V_ACCVGPR_WRITE_B32_e64: in FoldImmediate()
3372 const MachineOperand *ImmOp = getNamedOperand(DefMI, AMDGPU::OpName::src0); in FoldImmediate()
3383 case AMDGPU::sub0: in FoldImmediate()
3385 case AMDGPU::sub1: in FoldImmediate()
3387 case AMDGPU::lo16: in FoldImmediate()
3389 case AMDGPU::hi16: in FoldImmediate()
3391 case AMDGPU::sub1_lo16: in FoldImmediate()
3393 case AMDGPU::sub1_hi16: in FoldImmediate()
3401 if (Opc == AMDGPU::COPY) { in FoldImmediate()
3409 unsigned NewOpc = isVGPRCopy ? Is64Bit ? AMDGPU::V_MOV_B64_PSEUDO in FoldImmediate()
3410 : AMDGPU::V_MOV_B32_e32 in FoldImmediate()
3411 : Is64Bit ? AMDGPU::S_MOV_B64_IMM_PSEUDO in FoldImmediate()
3412 : AMDGPU::S_MOV_B32; in FoldImmediate()
3418 NewOpc = AMDGPU::V_ACCVGPR_WRITE_B32_e64; in FoldImmediate()
3425 if (DstReg.isVirtual() && UseMI.getOperand(0).getSubReg() != AMDGPU::lo16) in FoldImmediate()
3447 if (Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 || in FoldImmediate()
3448 Opc == AMDGPU::V_MAD_F16_e64 || Opc == AMDGPU::V_MAC_F16_e64 || in FoldImmediate()
3449 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 || in FoldImmediate()
3450 Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64 || in FoldImmediate()
3451 Opc == AMDGPU::V_FMAC_F16_t16_e64) { in FoldImmediate()
3460 MachineOperand *Src0 = getNamedOperand(UseMI, AMDGPU::OpName::src0); in FoldImmediate()
3466 bool IsF32 = Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 || in FoldImmediate()
3467 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64; in FoldImmediate()
3469 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 || in FoldImmediate()
3470 Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64 || in FoldImmediate()
3471 Opc == AMDGPU::V_FMAC_F16_t16_e64; in FoldImmediate()
3472 MachineOperand *Src1 = getNamedOperand(UseMI, AMDGPU::OpName::src1); in FoldImmediate()
3473 MachineOperand *Src2 = getNamedOperand(UseMI, AMDGPU::OpName::src2); in FoldImmediate()
3503 IsFMA ? (IsF32 ? AMDGPU::V_FMAMK_F32 in FoldImmediate()
3504 : ST.hasTrue16BitInsts() ? AMDGPU::V_FMAMK_F16_t16 in FoldImmediate()
3505 : AMDGPU::V_FMAMK_F16) in FoldImmediate()
3506 : (IsF32 ? AMDGPU::V_MADMK_F32 : AMDGPU::V_MADMK_F16); in FoldImmediate()
3513 if (NewOpc == AMDGPU::V_FMAMK_F16_t16) in FoldImmediate()
3527 if (Opc == AMDGPU::V_MAC_F32_e64 || Opc == AMDGPU::V_MAC_F16_e64 || in FoldImmediate()
3528 Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_t16_e64 || in FoldImmediate()
3529 Opc == AMDGPU::V_FMAC_F16_e64) in FoldImmediate()
3531 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)); in FoldImmediate()
3582 IsFMA ? (IsF32 ? AMDGPU::V_FMAAK_F32 in FoldImmediate()
3583 : ST.hasTrue16BitInsts() ? AMDGPU::V_FMAAK_F16_t16 in FoldImmediate()
3584 : AMDGPU::V_FMAAK_F16) in FoldImmediate()
3585 : (IsF32 ? AMDGPU::V_MADAK_F32 : AMDGPU::V_MADAK_F16); in FoldImmediate()
3592 if (NewOpc == AMDGPU::V_FMAAK_F16_t16) in FoldImmediate()
3598 if (Opc == AMDGPU::V_MAC_F32_e64 || Opc == AMDGPU::V_MAC_F16_e64 || in FoldImmediate()
3599 Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_t16_e64 || in FoldImmediate()
3600 Opc == AMDGPU::V_FMAC_F16_e64) in FoldImmediate()
3602 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)); in FoldImmediate()
3776 int NewMFMAOpc = AMDGPU::getMFMAEarlyClobberOp(Opc); in convertToThreeAddress()
3789 unsigned NewOpc = AMDGPU::mapWMMA2AddrTo3AddrOpcode(MI.getOpcode()); in convertToThreeAddress()
3802 assert(Opc != AMDGPU::V_FMAC_F16_t16_e32 && in convertToThreeAddress()
3807 bool IsF16 = Opc == AMDGPU::V_MAC_F16_e32 || Opc == AMDGPU::V_MAC_F16_e64 || in convertToThreeAddress()
3808 Opc == AMDGPU::V_FMAC_F16_e32 || Opc == AMDGPU::V_FMAC_F16_e64 || in convertToThreeAddress()
3809 Opc == AMDGPU::V_FMAC_F16_t16_e64; in convertToThreeAddress()
3810 bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F32_e64 || in convertToThreeAddress()
3811 Opc == AMDGPU::V_FMAC_LEGACY_F32_e32 || in convertToThreeAddress()
3812 Opc == AMDGPU::V_FMAC_LEGACY_F32_e64 || in convertToThreeAddress()
3813 Opc == AMDGPU::V_FMAC_F16_e32 || Opc == AMDGPU::V_FMAC_F16_e64 || in convertToThreeAddress()
3814 Opc == AMDGPU::V_FMAC_F16_t16_e64 || in convertToThreeAddress()
3815 Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64; in convertToThreeAddress()
3816 bool IsF64 = Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64; in convertToThreeAddress()
3817 bool IsLegacy = Opc == AMDGPU::V_MAC_LEGACY_F32_e32 || in convertToThreeAddress()
3818 Opc == AMDGPU::V_MAC_LEGACY_F32_e64 || in convertToThreeAddress()
3819 Opc == AMDGPU::V_FMAC_LEGACY_F32_e32 || in convertToThreeAddress()
3820 Opc == AMDGPU::V_FMAC_LEGACY_F32_e64; in convertToThreeAddress()
3826 case AMDGPU::V_MAC_F16_e64: in convertToThreeAddress()
3827 case AMDGPU::V_FMAC_F16_e64: in convertToThreeAddress()
3828 case AMDGPU::V_FMAC_F16_t16_e64: in convertToThreeAddress()
3829 case AMDGPU::V_MAC_F32_e64: in convertToThreeAddress()
3830 case AMDGPU::V_MAC_LEGACY_F32_e64: in convertToThreeAddress()
3831 case AMDGPU::V_FMAC_F32_e64: in convertToThreeAddress()
3832 case AMDGPU::V_FMAC_LEGACY_F32_e64: in convertToThreeAddress()
3833 case AMDGPU::V_FMAC_F64_e64: in convertToThreeAddress()
3835 case AMDGPU::V_MAC_F16_e32: in convertToThreeAddress()
3836 case AMDGPU::V_FMAC_F16_e32: in convertToThreeAddress()
3837 case AMDGPU::V_MAC_F32_e32: in convertToThreeAddress()
3838 case AMDGPU::V_MAC_LEGACY_F32_e32: in convertToThreeAddress()
3839 case AMDGPU::V_FMAC_F32_e32: in convertToThreeAddress()
3840 case AMDGPU::V_FMAC_LEGACY_F32_e32: in convertToThreeAddress()
3841 case AMDGPU::V_FMAC_F64_e32: { in convertToThreeAddress()
3842 int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), in convertToThreeAddress()
3843 AMDGPU::OpName::src0); in convertToThreeAddress()
3856 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst); in convertToThreeAddress()
3857 const MachineOperand *Src0 = getNamedOperand(MI, AMDGPU::OpName::src0); in convertToThreeAddress()
3859 getNamedOperand(MI, AMDGPU::OpName::src0_modifiers); in convertToThreeAddress()
3860 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1); in convertToThreeAddress()
3862 getNamedOperand(MI, AMDGPU::OpName::src1_modifiers); in convertToThreeAddress()
3863 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); in convertToThreeAddress()
3865 getNamedOperand(MI, AMDGPU::OpName::src2_modifiers); in convertToThreeAddress()
3866 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp); in convertToThreeAddress()
3867 const MachineOperand *Omod = getNamedOperand(MI, AMDGPU::OpName::omod); in convertToThreeAddress()
3868 const MachineOperand *OpSel = getNamedOperand(MI, AMDGPU::OpName::op_sel); in convertToThreeAddress()
3883 DefMI->setDesc(get(AMDGPU::IMPLICIT_DEF)); in convertToThreeAddress()
3893 IsFMA ? (IsF16 ? (ST.hasTrue16BitInsts() ? AMDGPU::V_FMAAK_F16_t16 in convertToThreeAddress()
3894 : AMDGPU::V_FMAAK_F16) in convertToThreeAddress()
3895 : AMDGPU::V_FMAAK_F32) in convertToThreeAddress()
3896 : (IsF16 ? AMDGPU::V_MADAK_F16 : AMDGPU::V_MADAK_F32); in convertToThreeAddress()
3911 IsFMA ? (IsF16 ? (ST.hasTrue16BitInsts() ? AMDGPU::V_FMAMK_F16_t16 in convertToThreeAddress()
3912 : AMDGPU::V_FMAMK_F16) in convertToThreeAddress()
3913 : AMDGPU::V_FMAMK_F32) in convertToThreeAddress()
3914 : (IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32); in convertToThreeAddress()
3936 MI, AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::src0), in convertToThreeAddress()
3958 unsigned NewOpc = IsFMA ? IsF16 ? AMDGPU::V_FMA_F16_gfx9_e64 in convertToThreeAddress()
3959 : IsF64 ? AMDGPU::V_FMA_F64_e64 in convertToThreeAddress()
3961 ? AMDGPU::V_FMA_LEGACY_F32_e64 in convertToThreeAddress()
3962 : AMDGPU::V_FMA_F32_e64 in convertToThreeAddress()
3963 : IsF16 ? AMDGPU::V_MAD_F16_e64 in convertToThreeAddress()
3964 : IsLegacy ? AMDGPU::V_MAD_LEGACY_F32_e64 in convertToThreeAddress()
3965 : AMDGPU::V_MAD_F32_e64; in convertToThreeAddress()
3979 if (AMDGPU::hasNamedOperand(NewOpc, AMDGPU::OpName::op_sel)) in convertToThreeAddress()
3992 case AMDGPU::S_SET_GPR_IDX_ON: in changesVGPRIndexingMode()
3993 case AMDGPU::S_SET_GPR_IDX_MODE: in changesVGPRIndexingMode()
3994 case AMDGPU::S_SET_GPR_IDX_OFF: in changesVGPRIndexingMode()
4019 if (MI.getOpcode() == AMDGPU::SCHED_BARRIER && MI.getOperand(0).getImm() == 0) in isSchedulingBoundary()
4025 return MI.modifiesRegister(AMDGPU::EXEC, &RI) || in isSchedulingBoundary()
4026 MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 || in isSchedulingBoundary()
4027 MI.getOpcode() == AMDGPU::S_SETREG_B32 || in isSchedulingBoundary()
4028 MI.getOpcode() == AMDGPU::S_SETPRIO || in isSchedulingBoundary()
4033 return Opcode == AMDGPU::DS_ORDERED_COUNT || isGWS(Opcode); in isAlwaysGDS()
4040 return is_contained(MI.getDesc().implicit_defs(), AMDGPU::MODE); in modifiesModeRegister()
4059 if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT || in hasUnwantedEffectsWhenEXECEmpty()
4061 Opcode == AMDGPU::DS_ORDERED_COUNT || Opcode == AMDGPU::S_TRAP || in hasUnwantedEffectsWhenEXECEmpty()
4062 Opcode == AMDGPU::DS_GWS_INIT || Opcode == AMDGPU::DS_GWS_BARRIER) in hasUnwantedEffectsWhenEXECEmpty()
4077 if (Opcode == AMDGPU::V_READFIRSTLANE_B32 || in hasUnwantedEffectsWhenEXECEmpty()
4078 Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32 || in hasUnwantedEffectsWhenEXECEmpty()
4079 Opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR || in hasUnwantedEffectsWhenEXECEmpty()
4080 Opcode == AMDGPU::SI_SPILL_S32_TO_VGPR) in hasUnwantedEffectsWhenEXECEmpty()
4097 return MI.readsRegister(AMDGPU::EXEC, &RI); in mayReadEXEC()
4108 return !isSALU(MI) || MI.readsRegister(AMDGPU::EXEC, &RI); in mayReadEXEC()
4117 return AMDGPU::isInlinableLiteral32(Imm.getSExtValue(), in isInlineConstant()
4120 return AMDGPU::isInlinableLiteral64(Imm.getSExtValue(), in isInlineConstant()
4124 AMDGPU::isInlinableLiteral16(Imm.getSExtValue(), in isInlineConstant()
4144 case AMDGPU::OPERAND_REG_IMM_INT32: in isInlineConstant()
4145 case AMDGPU::OPERAND_REG_IMM_FP32: in isInlineConstant()
4146 case AMDGPU::OPERAND_REG_IMM_FP32_DEFERRED: in isInlineConstant()
4147 case AMDGPU::OPERAND_REG_INLINE_C_INT32: in isInlineConstant()
4148 case AMDGPU::OPERAND_REG_INLINE_C_FP32: in isInlineConstant()
4149 case AMDGPU::OPERAND_REG_IMM_V2FP32: in isInlineConstant()
4150 case AMDGPU::OPERAND_REG_INLINE_C_V2FP32: in isInlineConstant()
4151 case AMDGPU::OPERAND_REG_IMM_V2INT32: in isInlineConstant()
4152 case AMDGPU::OPERAND_REG_INLINE_C_V2INT32: in isInlineConstant()
4153 case AMDGPU::OPERAND_REG_INLINE_AC_INT32: in isInlineConstant()
4154 case AMDGPU::OPERAND_REG_INLINE_AC_FP32: in isInlineConstant()
4155 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32: { in isInlineConstant()
4157 return AMDGPU::isInlinableLiteral32(Trunc, ST.hasInv2PiInlineImm()); in isInlineConstant()
4159 case AMDGPU::OPERAND_REG_IMM_INT64: in isInlineConstant()
4160 case AMDGPU::OPERAND_REG_IMM_FP64: in isInlineConstant()
4161 case AMDGPU::OPERAND_REG_INLINE_C_INT64: in isInlineConstant()
4162 case AMDGPU::OPERAND_REG_INLINE_C_FP64: in isInlineConstant()
4163 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: in isInlineConstant()
4164 return AMDGPU::isInlinableLiteral64(MO.getImm(), in isInlineConstant()
4166 case AMDGPU::OPERAND_REG_IMM_INT16: in isInlineConstant()
4167 case AMDGPU::OPERAND_REG_INLINE_C_INT16: in isInlineConstant()
4168 case AMDGPU::OPERAND_REG_INLINE_AC_INT16: in isInlineConstant()
4179 return AMDGPU::isInlinableIntLiteral(Imm); in isInlineConstant()
4180 case AMDGPU::OPERAND_REG_IMM_V2INT16: in isInlineConstant()
4181 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16: in isInlineConstant()
4182 case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16: in isInlineConstant()
4183 return AMDGPU::isInlinableLiteralV2I16(Imm); in isInlineConstant()
4184 case AMDGPU::OPERAND_REG_IMM_V2FP16: in isInlineConstant()
4185 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: in isInlineConstant()
4186 case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16: in isInlineConstant()
4187 return AMDGPU::isInlinableLiteralV2F16(Imm); in isInlineConstant()
4188 case AMDGPU::OPERAND_REG_IMM_FP16: in isInlineConstant()
4189 case AMDGPU::OPERAND_REG_IMM_FP16_DEFERRED: in isInlineConstant()
4190 case AMDGPU::OPERAND_REG_INLINE_C_FP16: in isInlineConstant()
4191 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: { in isInlineConstant()
4199 AMDGPU::isInlinableLiteral16(Trunc, ST.hasInv2PiInlineImm()); in isInlineConstant()
4204 case AMDGPU::OPERAND_KIMM32: in isInlineConstant()
4205 case AMDGPU::OPERAND_KIMM16: in isInlineConstant()
4207 case AMDGPU::OPERAND_INPUT_MODS: in isInlineConstant()
4257 OpNo ==(unsigned)AMDGPU::getNamedOperandIdx(MI.getOpcode(), in isImmOperandLegal()
4258 AMDGPU::OpName::src2)) in isImmOperandLegal()
4266 if (!isVOP3(MI) || !AMDGPU::isSISrcOperand(InstDesc, OpNo)) in isImmOperandLegal()
4274 if (Opcode == AMDGPU::V_MUL_LEGACY_F32_e64 && ST.hasGFX90AInsts()) in hasVALU32BitEncoding()
4277 int Op32 = AMDGPU::getVOPe32(Opcode); in hasVALU32BitEncoding()
4288 return AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::src0_modifiers); in hasModifiers()
4304 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); in canShrink()
4310 case AMDGPU::V_ADDC_U32_e64: in canShrink()
4311 case AMDGPU::V_SUBB_U32_e64: in canShrink()
4312 case AMDGPU::V_SUBBREV_U32_e64: { in canShrink()
4314 = getNamedOperand(MI, AMDGPU::OpName::src1); in canShrink()
4320 case AMDGPU::V_MAC_F16_e64: in canShrink()
4321 case AMDGPU::V_MAC_F32_e64: in canShrink()
4322 case AMDGPU::V_MAC_LEGACY_F32_e64: in canShrink()
4323 case AMDGPU::V_FMAC_F16_e64: in canShrink()
4324 case AMDGPU::V_FMAC_F16_t16_e64: in canShrink()
4325 case AMDGPU::V_FMAC_F32_e64: in canShrink()
4326 case AMDGPU::V_FMAC_F64_e64: in canShrink()
4327 case AMDGPU::V_FMAC_LEGACY_F32_e64: in canShrink()
4329 hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers)) in canShrink()
4333 case AMDGPU::V_CNDMASK_B32_e64: in canShrink()
4338 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1); in canShrink()
4340 hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers))) in canShrink()
4345 if (hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers)) in canShrink()
4353 return !hasModifiersSet(MI, AMDGPU::OpName::omod) && in canShrink()
4354 !hasModifiersSet(MI, AMDGPU::OpName::clamp); in canShrink()
4364 (Use.getReg() == AMDGPU::VCC || Use.getReg() == AMDGPU::VCC_LO)) { in copyFlagsToImplicitVCC()
4381 if (AMDGPU::hasNamedOperand(Op32, AMDGPU::OpName::vdst)) { in buildShrunkInst()
4384 } else if (AMDGPU::hasNamedOperand(Op32, AMDGPU::OpName::sdst)) { in buildShrunkInst()
4387 assert(((MI.getOperand(0).getReg() == AMDGPU::VCC) || in buildShrunkInst()
4388 (MI.getOperand(0).getReg() == AMDGPU::VCC_LO)) && in buildShrunkInst()
4392 Inst32.add(*getNamedOperand(MI, AMDGPU::OpName::src0)); in buildShrunkInst()
4394 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1); in buildShrunkInst()
4398 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); in buildShrunkInst()
4401 int Op32Src2Idx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::src2); in buildShrunkInst()
4432 if (MO.getReg() == AMDGPU::SGPR_NULL || MO.getReg() == AMDGPU::SGPR_NULL64) in usesConstantBus()
4437 return MO.getReg() == AMDGPU::M0 || in usesConstantBus()
4438 MO.getReg() == AMDGPU::VCC || in usesConstantBus()
4439 MO.getReg() == AMDGPU::VCC_LO; in usesConstantBus()
4441 return AMDGPU::SReg_32RegClass.contains(MO.getReg()) || in usesConstantBus()
4442 AMDGPU::SReg_64RegClass.contains(MO.getReg()); in usesConstantBus()
4453 case AMDGPU::VCC: in findImplicitSGPRRead()
4454 case AMDGPU::VCC_LO: in findImplicitSGPRRead()
4455 case AMDGPU::VCC_HI: in findImplicitSGPRRead()
4456 case AMDGPU::M0: in findImplicitSGPRRead()
4457 case AMDGPU::FLAT_SCR: in findImplicitSGPRRead()
4471 case AMDGPU::V_READLANE_B32: in shouldReadExec()
4472 case AMDGPU::SI_RESTORE_S32_FROM_VGPR: in shouldReadExec()
4473 case AMDGPU::V_WRITELANE_B32: in shouldReadExec()
4474 case AMDGPU::SI_SPILL_S32_TO_VGPR: in shouldReadExec()
4496 return SubReg.getSubReg() != AMDGPU::NoSubRegister && in isSubRegOf()
4509 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0); in verifyInstruction()
4510 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1); in verifyInstruction()
4511 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2); in verifyInstruction()
4515 Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0X); in verifyInstruction()
4516 Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vsrc1X); in verifyInstruction()
4517 Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0Y); in verifyInstruction()
4518 Src3Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vsrc1Y); in verifyInstruction()
4574 case AMDGPU::OPERAND_REG_IMM_INT32: in verifyInstruction()
4575 case AMDGPU::OPERAND_REG_IMM_FP32: in verifyInstruction()
4576 case AMDGPU::OPERAND_REG_IMM_FP32_DEFERRED: in verifyInstruction()
4577 case AMDGPU::OPERAND_REG_IMM_V2FP32: in verifyInstruction()
4579 case AMDGPU::OPERAND_REG_INLINE_C_INT32: in verifyInstruction()
4580 case AMDGPU::OPERAND_REG_INLINE_C_FP32: in verifyInstruction()
4581 case AMDGPU::OPERAND_REG_INLINE_C_INT64: in verifyInstruction()
4582 case AMDGPU::OPERAND_REG_INLINE_C_FP64: in verifyInstruction()
4583 case AMDGPU::OPERAND_REG_INLINE_C_INT16: in verifyInstruction()
4584 case AMDGPU::OPERAND_REG_INLINE_C_FP16: in verifyInstruction()
4585 case AMDGPU::OPERAND_REG_INLINE_AC_INT32: in verifyInstruction()
4586 case AMDGPU::OPERAND_REG_INLINE_AC_FP32: in verifyInstruction()
4587 case AMDGPU::OPERAND_REG_INLINE_AC_INT16: in verifyInstruction()
4588 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: in verifyInstruction()
4589 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: { in verifyInstruction()
4596 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32: in verifyInstruction()
4603 case AMDGPU::OPERAND_KIMM32: in verifyInstruction()
4662 int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst); in verifyInstruction()
4687 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod); in verifyInstruction()
4695 uint16_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode); in verifyInstruction()
4700 if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) { in verifyInstruction()
4706 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp); in verifyInstruction()
4713 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod); in verifyInstruction()
4721 const MachineOperand *DstUnused = getNamedOperand(MI, AMDGPU::OpName::dst_unused); in verifyInstruction()
4723 DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE) { in verifyInstruction()
4748 const MachineOperand *DMask = getNamedOperand(MI, AMDGPU::OpName::dmask); in verifyInstruction()
4753 const MachineOperand *TFE = getNamedOperand(MI, AMDGPU::OpName::tfe); in verifyInstruction()
4754 const MachineOperand *LWE = getNamedOperand(MI, AMDGPU::OpName::lwe); in verifyInstruction()
4755 const MachineOperand *D16 = getNamedOperand(MI, AMDGPU::OpName::d16); in verifyInstruction()
4766 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata); in verifyInstruction()
4781 if (isVALU(MI) && Desc.getOpcode() != AMDGPU::V_WRITELANE_B32) { in verifyInstruction()
4786 int ImmIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm); in verifyInstruction()
4838 Opcode != AMDGPU::V_WRITELANE_B32) { in verifyInstruction()
4851 if (Desc.getOpcode() == AMDGPU::V_WRITELANE_B32) { in verifyInstruction()
4862 if (MO.isReg() && MO.getReg() != AMDGPU::M0) { in verifyInstruction()
4876 if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32_e64 || in verifyInstruction()
4877 Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64_e64) { in verifyInstruction()
4888 if ((getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)->getImm() & in verifyInstruction()
4890 (getNamedOperand(MI, AMDGPU::OpName::src1_modifiers)->getImm() & in verifyInstruction()
4892 (getNamedOperand(MI, AMDGPU::OpName::src2_modifiers)->getImm() & in verifyInstruction()
4913 auto Op = getNamedOperand(MI, AMDGPU::OpName::simm16); in verifyInstruction()
4935 if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 || in verifyInstruction()
4936 Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 || in verifyInstruction()
4937 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 || in verifyInstruction()
4938 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) { in verifyInstruction()
4939 const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 || in verifyInstruction()
4940 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64; in verifyInstruction()
4954 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst); in verifyInstruction()
4982 if (!MI.hasRegisterImplicitUseOperand(AMDGPU::EXEC)) { in verifyInstruction()
4993 const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soffset); in verifyInstruction()
4994 if (Soff && Soff->getReg() != AMDGPU::M0) { in verifyInstruction()
5002 const MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset); in verifyInstruction()
5010 const MachineOperand *GDSOp = getNamedOperand(MI, AMDGPU::OpName::gds); in verifyInstruction()
5018 const MachineOperand *DimOp = getNamedOperand(MI, AMDGPU::OpName::dim); in verifyInstruction()
5020 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode, in verifyInstruction()
5021 AMDGPU::OpName::vaddr0); in verifyInstruction()
5023 isMIMG(MI) ? AMDGPU::OpName::srsrc : AMDGPU::OpName::rsrc; in verifyInstruction()
5024 int RsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, RSrcOpName); in verifyInstruction()
5025 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opcode); in verifyInstruction()
5026 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode = in verifyInstruction()
5027 AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode); in verifyInstruction()
5028 const AMDGPU::MIMGDimInfo *Dim = in verifyInstruction()
5029 AMDGPU::getMIMGDimInfoByEncoding(DimOp->getImm()); in verifyInstruction()
5038 const MachineOperand *R128A16 = getNamedOperand(MI, AMDGPU::OpName::r128); in verifyInstruction()
5041 const MachineOperand *A16 = getNamedOperand(MI, AMDGPU::OpName::a16); in verifyInstruction()
5048 AMDGPU::getAddrSizeMIMGOp(BaseOpcode, Dim, IsA16, ST.hasG16()); in verifyInstruction()
5073 const MachineOperand *DppCt = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl); in verifyInstruction()
5075 using namespace AMDGPU::DPP; in verifyInstruction()
5116 if (Opcode != AMDGPU::V_MOV_B64_DPP_PSEUDO && in verifyInstruction()
5117 !AMDGPU::isLegalDPALU_DPPControl(DC) && AMDGPU::isDPALU_DPP(Desc)) { in verifyInstruction()
5125 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst); in verifyInstruction()
5126 uint16_t DataNameIdx = isDS(Opcode) ? AMDGPU::OpName::data0 in verifyInstruction()
5127 : AMDGPU::OpName::vdata; in verifyInstruction()
5129 const MachineOperand *Data2 = getNamedOperand(MI, AMDGPU::OpName::data1); in verifyInstruction()
5170 if (MI.getOpcode() == AMDGPU::DS_GWS_INIT || in verifyInstruction()
5171 MI.getOpcode() == AMDGPU::DS_GWS_SEMA_BR || in verifyInstruction()
5172 MI.getOpcode() == AMDGPU::DS_GWS_BARRIER) { in verifyInstruction()
5174 if (!isAlignedReg(AMDGPU::OpName::data0)) { in verifyInstruction()
5182 if (!isAlignedReg(AMDGPU::OpName::vaddr)) { in verifyInstruction()
5190 if (MI.getOpcode() == AMDGPU::V_ACCVGPR_WRITE_B32_e64 && in verifyInstruction()
5192 const MachineOperand *Src = getNamedOperand(MI, AMDGPU::OpName::src0); in verifyInstruction()
5200 if (Desc.getOpcode() == AMDGPU::G_AMDGPU_WAVE_ADDRESS) { in verifyInstruction()
5216 default: return AMDGPU::INSTRUCTION_LIST_END; in getVALUOp()
5217 case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE; in getVALUOp()
5218 case AMDGPU::COPY: return AMDGPU::COPY; in getVALUOp()
5219 case AMDGPU::PHI: return AMDGPU::PHI; in getVALUOp()
5220 case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG; in getVALUOp()
5221 case AMDGPU::WQM: return AMDGPU::WQM; in getVALUOp()
5222 case AMDGPU::SOFT_WQM: return AMDGPU::SOFT_WQM; in getVALUOp()
5223 case AMDGPU::STRICT_WWM: return AMDGPU::STRICT_WWM; in getVALUOp()
5224 case AMDGPU::STRICT_WQM: return AMDGPU::STRICT_WQM; in getVALUOp()
5225 case AMDGPU::S_MOV_B32: { in getVALUOp()
5229 AMDGPU::COPY : AMDGPU::V_MOV_B32_e32; in getVALUOp()
5231 case AMDGPU::S_ADD_I32: in getVALUOp()
5232 return ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_CO_U32_e32; in getVALUOp()
5233 case AMDGPU::S_ADDC_U32: in getVALUOp()
5234 return AMDGPU::V_ADDC_U32_e32; in getVALUOp()
5235 case AMDGPU::S_SUB_I32: in getVALUOp()
5236 return ST.hasAddNoCarry() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_CO_U32_e32; in getVALUOp()
5239 case AMDGPU::S_ADD_U32: in getVALUOp()
5240 return AMDGPU::V_ADD_CO_U32_e32; in getVALUOp()
5241 case AMDGPU::S_SUB_U32: in getVALUOp()
5242 return AMDGPU::V_SUB_CO_U32_e32; in getVALUOp()
5243 case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32; in getVALUOp()
5244 case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_U32_e64; in getVALUOp()
5245 case AMDGPU::S_MUL_HI_U32: return AMDGPU::V_MUL_HI_U32_e64; in getVALUOp()
5246 case AMDGPU::S_MUL_HI_I32: return AMDGPU::V_MUL_HI_I32_e64; in getVALUOp()
5247 case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64; in getVALUOp()
5248 case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64; in getVALUOp()
5249 case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64; in getVALUOp()
5250 case AMDGPU::S_XNOR_B32: in getVALUOp()
5251 return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END; in getVALUOp()
5252 case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64; in getVALUOp()
5253 case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64; in getVALUOp()
5254 case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64; in getVALUOp()
5255 case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64; in getVALUOp()
5256 case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32; in getVALUOp()
5257 case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64_e64; in getVALUOp()
5258 case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32; in getVALUOp()
5259 case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64_e64; in getVALUOp()
5260 case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32; in getVALUOp()
5261 case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64_e64; in getVALUOp()
5262 case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32_e64; in getVALUOp()
5263 case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32_e64; in getVALUOp()
5264 case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32_e64; in getVALUOp()
5265 case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32_e64; in getVALUOp()
5266 case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64; in getVALUOp()
5267 case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32; in getVALUOp()
5268 case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32; in getVALUOp()
5269 case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32; in getVALUOp()
5270 case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e64; in getVALUOp()
5271 case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e64; in getVALUOp()
5272 case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e64; in getVALUOp()
5273 case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e64; in getVALUOp()
5274 case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e64; in getVALUOp()
5275 case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e64; in getVALUOp()
5276 case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e64; in getVALUOp()
5277 case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e64; in getVALUOp()
5278 case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e64; in getVALUOp()
5279 case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e64; in getVALUOp()
5280 case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e64; in getVALUOp()
5281 case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e64; in getVALUOp()
5282 case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e64; in getVALUOp()
5283 case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e64; in getVALUOp()
5284 case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64; in getVALUOp()
5285 case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32; in getVALUOp()
5286 case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32; in getVALUOp()
5287 case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64; in getVALUOp()
5288 case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ; in getVALUOp()
5289 case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ; in getVALUOp()
5290 case AMDGPU::S_CVT_F32_I32: return AMDGPU::V_CVT_F32_I32_e64; in getVALUOp()
5291 case AMDGPU::S_CVT_F32_U32: return AMDGPU::V_CVT_F32_U32_e64; in getVALUOp()
5292 case AMDGPU::S_CVT_I32_F32: return AMDGPU::V_CVT_I32_F32_e64; in getVALUOp()
5293 case AMDGPU::S_CVT_U32_F32: return AMDGPU::V_CVT_U32_F32_e64; in getVALUOp()
5294 case AMDGPU::S_CVT_F32_F16: return AMDGPU::V_CVT_F32_F16_t16_e64; in getVALUOp()
5295 case AMDGPU::S_CVT_HI_F32_F16: return AMDGPU::V_CVT_F32_F16_t16_e64; in getVALUOp()
5296 case AMDGPU::S_CVT_F16_F32: return AMDGPU::V_CVT_F16_F32_t16_e64; in getVALUOp()
5297 case AMDGPU::S_CEIL_F32: return AMDGPU::V_CEIL_F32_e64; in getVALUOp()
5298 case AMDGPU::S_FLOOR_F32: return AMDGPU::V_FLOOR_F32_e64; in getVALUOp()
5299 case AMDGPU::S_TRUNC_F32: return AMDGPU::V_TRUNC_F32_e64; in getVALUOp()
5300 case AMDGPU::S_RNDNE_F32: return AMDGPU::V_RNDNE_F32_e64; in getVALUOp()
5301 case AMDGPU::S_CEIL_F16: in getVALUOp()
5302 return ST.useRealTrue16Insts() ? AMDGPU::V_CEIL_F16_t16_e64 in getVALUOp()
5303 : AMDGPU::V_CEIL_F16_fake16_e64; in getVALUOp()
5304 case AMDGPU::S_FLOOR_F16: in getVALUOp()
5305 return ST.useRealTrue16Insts() ? AMDGPU::V_FLOOR_F16_t16_e64 in getVALUOp()
5306 : AMDGPU::V_FLOOR_F16_fake16_e64; in getVALUOp()
5307 case AMDGPU::S_TRUNC_F16: in getVALUOp()
5308 return AMDGPU::V_TRUNC_F16_fake16_e64; in getVALUOp()
5309 case AMDGPU::S_RNDNE_F16: in getVALUOp()
5310 return AMDGPU::V_RNDNE_F16_fake16_e64; in getVALUOp()
5311 case AMDGPU::S_ADD_F32: return AMDGPU::V_ADD_F32_e64; in getVALUOp()
5312 case AMDGPU::S_SUB_F32: return AMDGPU::V_SUB_F32_e64; in getVALUOp()
5313 case AMDGPU::S_MIN_F32: return AMDGPU::V_MIN_F32_e64; in getVALUOp()
5314 case AMDGPU::S_MAX_F32: return AMDGPU::V_MAX_F32_e64; in getVALUOp()
5315 case AMDGPU::S_MINIMUM_F32: return AMDGPU::V_MINIMUM_F32_e64; in getVALUOp()
5316 case AMDGPU::S_MAXIMUM_F32: return AMDGPU::V_MAXIMUM_F32_e64; in getVALUOp()
5317 case AMDGPU::S_MUL_F32: return AMDGPU::V_MUL_F32_e64; in getVALUOp()
5318 case AMDGPU::S_ADD_F16: return AMDGPU::V_ADD_F16_fake16_e64; in getVALUOp()
5319 case AMDGPU::S_SUB_F16: return AMDGPU::V_SUB_F16_fake16_e64; in getVALUOp()
5320 case AMDGPU::S_MIN_F16: return AMDGPU::V_MIN_F16_fake16_e64; in getVALUOp()
5321 case AMDGPU::S_MAX_F16: return AMDGPU::V_MAX_F16_fake16_e64; in getVALUOp()
5322 case AMDGPU::S_MINIMUM_F16: return AMDGPU::V_MINIMUM_F16_e64; in getVALUOp()
5323 case AMDGPU::S_MAXIMUM_F16: return AMDGPU::V_MAXIMUM_F16_e64; in getVALUOp()
5324 case AMDGPU::S_MUL_F16: return AMDGPU::V_MUL_F16_fake16_e64; in getVALUOp()
5325 case AMDGPU::S_CVT_PK_RTZ_F16_F32: return AMDGPU::V_CVT_PKRTZ_F16_F32_e64; in getVALUOp()
5326 case AMDGPU::S_FMAC_F32: return AMDGPU::V_FMAC_F32_e64; in getVALUOp()
5327 case AMDGPU::S_FMAC_F16: return AMDGPU::V_FMAC_F16_t16_e64; in getVALUOp()
5328 case AMDGPU::S_FMAMK_F32: return AMDGPU::V_FMAMK_F32; in getVALUOp()
5329 case AMDGPU::S_FMAAK_F32: return AMDGPU::V_FMAAK_F32; in getVALUOp()
5330 case AMDGPU::S_CMP_LT_F32: return AMDGPU::V_CMP_LT_F32_e64; in getVALUOp()
5331 case AMDGPU::S_CMP_EQ_F32: return AMDGPU::V_CMP_EQ_F32_e64; in getVALUOp()
5332 case AMDGPU::S_CMP_LE_F32: return AMDGPU::V_CMP_LE_F32_e64; in getVALUOp()
5333 case AMDGPU::S_CMP_GT_F32: return AMDGPU::V_CMP_GT_F32_e64; in getVALUOp()
5334 case AMDGPU::S_CMP_LG_F32: return AMDGPU::V_CMP_LG_F32_e64; in getVALUOp()
5335 case AMDGPU::S_CMP_GE_F32: return AMDGPU::V_CMP_GE_F32_e64; in getVALUOp()
5336 case AMDGPU::S_CMP_O_F32: return AMDGPU::V_CMP_O_F32_e64; in getVALUOp()
5337 case AMDGPU::S_CMP_U_F32: return AMDGPU::V_CMP_U_F32_e64; in getVALUOp()
5338 case AMDGPU::S_CMP_NGE_F32: return AMDGPU::V_CMP_NGE_F32_e64; in getVALUOp()
5339 case AMDGPU::S_CMP_NLG_F32: return AMDGPU::V_CMP_NLG_F32_e64; in getVALUOp()
5340 case AMDGPU::S_CMP_NGT_F32: return AMDGPU::V_CMP_NGT_F32_e64; in getVALUOp()
5341 case AMDGPU::S_CMP_NLE_F32: return AMDGPU::V_CMP_NLE_F32_e64; in getVALUOp()
5342 case AMDGPU::S_CMP_NEQ_F32: return AMDGPU::V_CMP_NEQ_F32_e64; in getVALUOp()
5343 case AMDGPU::S_CMP_NLT_F32: return AMDGPU::V_CMP_NLT_F32_e64; in getVALUOp()
5344 case AMDGPU::S_CMP_LT_F16: return AMDGPU::V_CMP_LT_F16_t16_e64; in getVALUOp()
5345 case AMDGPU::S_CMP_EQ_F16: return AMDGPU::V_CMP_EQ_F16_t16_e64; in getVALUOp()
5346 case AMDGPU::S_CMP_LE_F16: return AMDGPU::V_CMP_LE_F16_t16_e64; in getVALUOp()
5347 case AMDGPU::S_CMP_GT_F16: return AMDGPU::V_CMP_GT_F16_t16_e64; in getVALUOp()
5348 case AMDGPU::S_CMP_LG_F16: return AMDGPU::V_CMP_LG_F16_t16_e64; in getVALUOp()
5349 case AMDGPU::S_CMP_GE_F16: return AMDGPU::V_CMP_GE_F16_t16_e64; in getVALUOp()
5350 case AMDGPU::S_CMP_O_F16: return AMDGPU::V_CMP_O_F16_t16_e64; in getVALUOp()
5351 case AMDGPU::S_CMP_U_F16: return AMDGPU::V_CMP_U_F16_t16_e64; in getVALUOp()
5352 case AMDGPU::S_CMP_NGE_F16: return AMDGPU::V_CMP_NGE_F16_t16_e64; in getVALUOp()
5353 case AMDGPU::S_CMP_NLG_F16: return AMDGPU::V_CMP_NLG_F16_t16_e64; in getVALUOp()
5354 case AMDGPU::S_CMP_NGT_F16: return AMDGPU::V_CMP_NGT_F16_t16_e64; in getVALUOp()
5355 case AMDGPU::S_CMP_NLE_F16: return AMDGPU::V_CMP_NLE_F16_t16_e64; in getVALUOp()
5356 case AMDGPU::S_CMP_NEQ_F16: return AMDGPU::V_CMP_NEQ_F16_t16_e64; in getVALUOp()
5357 case AMDGPU::S_CMP_NLT_F16: return AMDGPU::V_CMP_NLT_F16_t16_e64; in getVALUOp()
5358 case AMDGPU::V_S_EXP_F32_e64: return AMDGPU::V_EXP_F32_e64; in getVALUOp()
5359 case AMDGPU::V_S_EXP_F16_e64: return AMDGPU::V_EXP_F16_fake16_e64; in getVALUOp()
5360 case AMDGPU::V_S_LOG_F32_e64: return AMDGPU::V_LOG_F32_e64; in getVALUOp()
5361 case AMDGPU::V_S_LOG_F16_e64: return AMDGPU::V_LOG_F16_fake16_e64; in getVALUOp()
5362 case AMDGPU::V_S_RCP_F32_e64: return AMDGPU::V_RCP_F32_e64; in getVALUOp()
5363 case AMDGPU::V_S_RCP_F16_e64: return AMDGPU::V_RCP_F16_fake16_e64; in getVALUOp()
5364 case AMDGPU::V_S_RSQ_F32_e64: return AMDGPU::V_RSQ_F32_e64; in getVALUOp()
5365 case AMDGPU::V_S_RSQ_F16_e64: return AMDGPU::V_RSQ_F16_fake16_e64; in getVALUOp()
5366 case AMDGPU::V_S_SQRT_F32_e64: return AMDGPU::V_SQRT_F32_e64; in getVALUOp()
5367 case AMDGPU::V_S_SQRT_F16_e64: return AMDGPU::V_SQRT_F16_fake16_e64; in getVALUOp()
5388 unsigned MovOpc = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; in insertScratchExecCopy()
5389 MCRegister Exec = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC; in insertScratchExecCopy()
5399 IsWave32 ? AMDGPU::S_OR_SAVEEXEC_B32 : AMDGPU::S_OR_SAVEEXEC_B64; in insertScratchExecCopy()
5412 unsigned ExecMov = isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; in restoreExec()
5413 MCRegister Exec = isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; in restoreExec()
5430 case AMDGPU::AV_32RegClassID: in adjustAllocatableRegClass()
5431 RCID = AMDGPU::VGPR_32RegClassID; in adjustAllocatableRegClass()
5433 case AMDGPU::AV_64RegClassID: in adjustAllocatableRegClass()
5434 RCID = AMDGPU::VReg_64RegClassID; in adjustAllocatableRegClass()
5436 case AMDGPU::AV_96RegClassID: in adjustAllocatableRegClass()
5437 RCID = AMDGPU::VReg_96RegClassID; in adjustAllocatableRegClass()
5439 case AMDGPU::AV_128RegClassID: in adjustAllocatableRegClass()
5440 RCID = AMDGPU::VReg_128RegClassID; in adjustAllocatableRegClass()
5442 case AMDGPU::AV_160RegClassID: in adjustAllocatableRegClass()
5443 RCID = AMDGPU::VReg_160RegClassID; in adjustAllocatableRegClass()
5445 case AMDGPU::AV_512RegClassID: in adjustAllocatableRegClass()
5446 RCID = AMDGPU::VReg_512RegClassID; in adjustAllocatableRegClass()
5472 const int VDstIdx = AMDGPU::getNamedOperandIdx(TID.Opcode, in getRegClass()
5473 AMDGPU::OpName::vdst); in getRegClass()
5474 const int DataIdx = AMDGPU::getNamedOperandIdx(TID.Opcode, in getRegClass()
5475 (TID.TSFlags & SIInstrFlags::DS) ? AMDGPU::OpName::data0 in getRegClass()
5476 : AMDGPU::OpName::vdata); in getRegClass()
5478 IsAllocatable = VDstIdx != -1 || AMDGPU::hasNamedOperand( in getRegClass()
5479 TID.Opcode, AMDGPU::OpName::data1); in getRegClass()
5511 unsigned Opcode = (Size == 64) ? AMDGPU::V_MOV_B64_PSEUDO : AMDGPU::V_MOV_B32_e32; in legalizeOpWithMove()
5513 Opcode = AMDGPU::COPY; in legalizeOpWithMove()
5515 Opcode = (Size == 64) ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32; in legalizeOpWithMove()
5532 if (SuperReg.getSubReg() == AMDGPU::NoSubRegister) { in buildExtractSubReg()
5558 if (SubIdx == AMDGPU::sub0) in buildExtractSubRegOrImm()
5560 if (SubIdx == AMDGPU::sub1) in buildExtractSubRegOrImm()
5651 } else if (AMDGPU::isSISrcOperand(InstDesc, i) && in isOperandLegal()
5675 const int VDstIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); in isOperandLegal()
5676 const int DataIdx = AMDGPU::getNamedOperandIdx(Opc, in isOperandLegal()
5677 isDS(Opc) ? AMDGPU::OpName::data0 : AMDGPU::OpName::vdata); in isOperandLegal()
5687 const int Data1Idx = AMDGPU::getNamedOperandIdx(Opc, in isOperandLegal()
5688 AMDGPU::OpName::data1); in isOperandLegal()
5693 if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64 && !ST.hasGFX90AInsts() && in isOperandLegal()
5694 (int)OpIdx == AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) && in isOperandLegal()
5702 bool Is64BitFPOp = OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_FP64; in isOperandLegal()
5704 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_INT64 || in isOperandLegal()
5705 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2INT32 || in isOperandLegal()
5706 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2FP32; in isOperandLegal()
5708 !AMDGPU::isInlinableLiteral64(Imm, ST.hasInv2PiInlineImm())) { in isOperandLegal()
5709 if (!AMDGPU::isValid32BitLiteral(Imm, Is64BitFPOp)) in isOperandLegal()
5737 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); in legalizeOperandsVOP2()
5740 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); in legalizeOperandsVOP2()
5753 if (Opc == AMDGPU::V_WRITELANE_B32) { in legalizeOperandsVOP2()
5756 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in legalizeOperandsVOP2()
5757 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) in legalizeOperandsVOP2()
5762 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in legalizeOperandsVOP2()
5764 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) in legalizeOperandsVOP2()
5779 if (Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F16_e32) { in legalizeOperandsVOP2()
5780 int Src2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2); in legalizeOperandsVOP2()
5793 if (Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() && in legalizeOperandsVOP2()
5795 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in legalizeOperandsVOP2()
5797 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) in legalizeOperandsVOP2()
5855 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0), in legalizeOperandsVOP3()
5856 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1), in legalizeOperandsVOP3()
5857 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2) in legalizeOperandsVOP3()
5860 if (Opc == AMDGPU::V_PERMLANE16_B32_e64 || in legalizeOperandsVOP3()
5861 Opc == AMDGPU::V_PERMLANEX16_B32_e64) { in legalizeOperandsVOP3()
5867 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in legalizeOperandsVOP3()
5868 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) in legalizeOperandsVOP3()
5873 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in legalizeOperandsVOP3()
5874 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) in legalizeOperandsVOP3()
5936 if ((Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_e64) && in legalizeOperandsVOP3()
5959 get(AMDGPU::V_READFIRSTLANE_B32), DstReg) in readlaneVGPRToSGPR()
5966 Register SGPR = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); in readlaneVGPRToSGPR()
5968 get(AMDGPU::V_READFIRSTLANE_B32), SGPR) in readlaneVGPRToSGPR()
5975 get(AMDGPU::REG_SEQUENCE), DstReg); in readlaneVGPRToSGPR()
5990 MachineOperand *SBase = getNamedOperand(MI, AMDGPU::OpName::sbase); in legalizeOperandsSMRD()
5995 MachineOperand *SOff = getNamedOperand(MI, AMDGPU::OpName::soffset); in legalizeOperandsSMRD()
6004 int OldSAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr); in moveFlatAddrToVGPR()
6010 int NewOpc = AMDGPU::getGlobalVaddrOp(Opc); in moveFlatAddrToVGPR()
6012 NewOpc = AMDGPU::getFlatScratchInstSVfromSS(Opc); in moveFlatAddrToVGPR()
6021 int NewVAddrIdx = AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vaddr); in moveFlatAddrToVGPR()
6025 int OldVAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr); in moveFlatAddrToVGPR()
6032 if (!VAddrDef || VAddrDef->getOpcode() != AMDGPU::V_MOV_B32_e32 || in moveFlatAddrToVGPR()
6057 int NewVDstIn = AMDGPU::getNamedOperandIdx(NewOpc, in moveFlatAddrToVGPR()
6058 AMDGPU::OpName::vdst_in); in moveFlatAddrToVGPR()
6063 int OldVDstIn = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst_in); in moveFlatAddrToVGPR()
6070 int NewVDst = AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vdst); in moveFlatAddrToVGPR()
6090 MachineOperand *SAddr = getNamedOperand(MI, AMDGPU::OpName::saddr); in legalizeOperandsFLAT()
6118 auto Copy = BuildMI(InsertMBB, I, DL, get(AMDGPU::COPY), DstReg).add(Op); in legalizeGenericOperand()
6128 if (Def->isMoveImmediate() && DstRC != &AMDGPU::VReg_1RegClass) in legalizeGenericOperand()
6138 if (!RI.isSGPRClass(DstRC) && !Copy->readsRegister(AMDGPU::EXEC, &RI) && in legalizeGenericOperand()
6140 Copy.addReg(AMDGPU::EXEC, RegState::Implicit); in legalizeGenericOperand()
6153 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; in emitLoadScalarOpsFromVGPRLoop()
6155 ST.isWave32() ? AMDGPU::S_AND_SAVEEXEC_B32 : AMDGPU::S_AND_SAVEEXEC_B64; in emitLoadScalarOpsFromVGPRLoop()
6157 ST.isWave32() ? AMDGPU::S_XOR_B32_term : AMDGPU::S_XOR_B64_term; in emitLoadScalarOpsFromVGPRLoop()
6159 ST.isWave32() ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64; in emitLoadScalarOpsFromVGPRLoop()
6160 const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID); in emitLoadScalarOpsFromVGPRLoop()
6173 Register CurReg = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); in emitLoadScalarOpsFromVGPRLoop()
6175 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurReg) in emitLoadScalarOpsFromVGPRLoop()
6180 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U32_e64), NewCondReg) in emitLoadScalarOpsFromVGPRLoop()
6204 Register CurRegLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); in emitLoadScalarOpsFromVGPRLoop()
6205 Register CurRegHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); in emitLoadScalarOpsFromVGPRLoop()
6208 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegLo) in emitLoadScalarOpsFromVGPRLoop()
6212 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegHi) in emitLoadScalarOpsFromVGPRLoop()
6220 Register CurReg = MRI.createVirtualRegister(&AMDGPU::SGPR_64RegClass); in emitLoadScalarOpsFromVGPRLoop()
6221 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), CurReg) in emitLoadScalarOpsFromVGPRLoop()
6223 .addImm(AMDGPU::sub0) in emitLoadScalarOpsFromVGPRLoop()
6225 .addImm(AMDGPU::sub1); in emitLoadScalarOpsFromVGPRLoop()
6228 auto Cmp = BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), in emitLoadScalarOpsFromVGPRLoop()
6255 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), SScalarOp); in emitLoadScalarOpsFromVGPRLoop()
6282 BuildMI(BodyBB, I, DL, TII.get(AMDGPU::SI_WATERFALL_LOOP)).addMBB(&LoopBB); in emitLoadScalarOpsFromVGPRLoop()
6306 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; in loadMBUFScalarOperandsFromVGPR()
6307 unsigned MovExecOpc = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; in loadMBUFScalarOperandsFromVGPR()
6308 const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID); in loadMBUFScalarOperandsFromVGPR()
6312 bool SCCNotDead = (MBB.computeRegisterLiveness(TRI, AMDGPU::SCC, MI, 30) != in loadMBUFScalarOperandsFromVGPR()
6315 SaveSCCReg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); in loadMBUFScalarOperandsFromVGPR()
6316 BuildMI(MBB, Begin, DL, TII.get(AMDGPU::S_CSELECT_B32), SaveSCCReg) in loadMBUFScalarOperandsFromVGPR()
6379 BuildMI(*RemainderBB, First, DL, TII.get(AMDGPU::S_CMP_LG_U32)) in loadMBUFScalarOperandsFromVGPR()
6398 TII.buildExtractSubReg(MI, MRI, Rsrc, &AMDGPU::VReg_128RegClass, in extractRsrcPtr()
6399 AMDGPU::sub0_sub1, &AMDGPU::VReg_64RegClass); in extractRsrcPtr()
6402 Register Zero64 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in extractRsrcPtr()
6403 Register SRsrcFormatLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); in extractRsrcPtr()
6404 Register SRsrcFormatHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); in extractRsrcPtr()
6405 Register NewSRsrc = MRI.createVirtualRegister(&AMDGPU::SGPR_128RegClass); in extractRsrcPtr()
6409 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B64), Zero64) in extractRsrcPtr()
6413 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatLo) in extractRsrcPtr()
6417 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatHi) in extractRsrcPtr()
6421 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::REG_SEQUENCE), NewSRsrc) in extractRsrcPtr()
6423 .addImm(AMDGPU::sub0_sub1) in extractRsrcPtr()
6425 .addImm(AMDGPU::sub2) in extractRsrcPtr()
6427 .addImm(AMDGPU::sub3); in extractRsrcPtr()
6466 if (MI.getOpcode() == AMDGPU::PHI) { in legalizeOperands()
6486 if (getOpRegClass(MI, 0) == &AMDGPU::VReg_1RegClass) { in legalizeOperands()
6487 VRC = &AMDGPU::VReg_1RegClass; in legalizeOperands()
6521 if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) { in legalizeOperands()
6548 if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) { in legalizeOperands()
6562 if (MI.getOpcode() == AMDGPU::SI_INIT_M0) { in legalizeOperands()
6570 if (MI.getOpcode() == AMDGPU::S_BITREPLICATE_B64_B32 || in legalizeOperands()
6571 MI.getOpcode() == AMDGPU::S_QUADMASK_B32 || in legalizeOperands()
6572 MI.getOpcode() == AMDGPU::S_QUADMASK_B64 || in legalizeOperands()
6573 MI.getOpcode() == AMDGPU::S_WQM_B32 || in legalizeOperands()
6574 MI.getOpcode() == AMDGPU::S_WQM_B64) { in legalizeOperands()
6586 if (isImage(MI) || (AMDGPU::isGraphics(MF.getFunction().getCallingConv()) && in legalizeOperands()
6588 int RSrcOpName = (isVIMAGE(MI) || isVSAMPLE(MI)) ? AMDGPU::OpName::rsrc in legalizeOperands()
6589 : AMDGPU::OpName::srsrc; in legalizeOperands()
6594 int SampOpName = isMIMG(MI) ? AMDGPU::OpName::ssamp : AMDGPU::OpName::samp; in legalizeOperands()
6603 if (MI.getOpcode() == AMDGPU::SI_CALL_ISEL) { in legalizeOperands()
6631 if (MI.getOpcode() == AMDGPU::S_SLEEP_VAR) { in legalizeOperands()
6633 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in legalizeOperands()
6635 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0); in legalizeOperands()
6637 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) in legalizeOperands()
6646 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::soffset); in legalizeOperands()
6657 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::srsrc); in legalizeOperands()
6686 MachineOperand *VAddr = getNamedOperand(MI, AMDGPU::OpName::vaddr); in legalizeOperands()
6687 if (VAddr && AMDGPU::getIfAddr64Inst(MI.getOpcode()) != -1) { in legalizeOperands()
6690 Register NewVAddrLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in legalizeOperands()
6691 Register NewVAddrHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in legalizeOperands()
6692 Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); in legalizeOperands()
6694 const auto *BoolXExecRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); in legalizeOperands()
6703 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADD_CO_U32_e64), NewVAddrLo) in legalizeOperands()
6705 .addReg(RsrcPtr, 0, AMDGPU::sub0) in legalizeOperands()
6706 .addReg(VAddr->getReg(), 0, AMDGPU::sub0) in legalizeOperands()
6710 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADDC_U32_e64), NewVAddrHi) in legalizeOperands()
6712 .addReg(RsrcPtr, 0, AMDGPU::sub1) in legalizeOperands()
6713 .addReg(VAddr->getReg(), 0, AMDGPU::sub1) in legalizeOperands()
6718 BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewVAddr) in legalizeOperands()
6720 .addImm(AMDGPU::sub0) in legalizeOperands()
6722 .addImm(AMDGPU::sub1); in legalizeOperands()
6735 Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); in legalizeOperands()
6736 MachineOperand *VData = getNamedOperand(MI, AMDGPU::OpName::vdata); in legalizeOperands()
6737 MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset); in legalizeOperands()
6738 MachineOperand *SOffset = getNamedOperand(MI, AMDGPU::OpName::soffset); in legalizeOperands()
6739 unsigned Addr64Opcode = AMDGPU::getAddr64Inst(MI.getOpcode()); in legalizeOperands()
6743 MachineOperand *VDataIn = getNamedOperand(MI, AMDGPU::OpName::vdata_in); in legalizeOperands()
6757 getNamedOperand(MI, AMDGPU::OpName::cpol)) { in legalizeOperands()
6762 getNamedOperand(MI, AMDGPU::OpName::tfe)) { in legalizeOperands()
6766 MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::swz)); in legalizeOperands()
6779 .addImm(getNamedImmOperand(MI, AMDGPU::OpName::cpol)) in legalizeOperands()
6786 BuildMI(MBB, Addr64, Addr64->getDebugLoc(), get(AMDGPU::REG_SEQUENCE), in legalizeOperands()
6788 .addReg(RsrcPtr, 0, AMDGPU::sub0) in legalizeOperands()
6789 .addImm(AMDGPU::sub0) in legalizeOperands()
6790 .addReg(RsrcPtr, 0, AMDGPU::sub1) in legalizeOperands()
6791 .addImm(AMDGPU::sub1); in legalizeOperands()
6795 MachineOperand *Soffset = getNamedOperand(MI, AMDGPU::OpName::soffset); in legalizeOperands()
6807 MachineOperand *Soffset = getNamedOperand(MI, AMDGPU::OpName::soffset); in legalizeOperands()
6818 AMDGPU::getNamedOperandIdx(MI->getOpcode(), AMDGPU::OpName::srsrc); in insert()
6863 case AMDGPU::S_ADD_U64_PSEUDO: in moveToVALUImpl()
6864 NewOpcode = AMDGPU::V_ADD_U64_PSEUDO; in moveToVALUImpl()
6866 case AMDGPU::S_SUB_U64_PSEUDO: in moveToVALUImpl()
6867 NewOpcode = AMDGPU::V_SUB_U64_PSEUDO; in moveToVALUImpl()
6869 case AMDGPU::S_ADD_I32: in moveToVALUImpl()
6870 case AMDGPU::S_SUB_I32: { in moveToVALUImpl()
6882 case AMDGPU::S_MUL_U64: in moveToVALUImpl()
6888 case AMDGPU::S_MUL_U64_U32_PSEUDO: in moveToVALUImpl()
6889 case AMDGPU::S_MUL_I64_I32_PSEUDO: in moveToVALUImpl()
6896 case AMDGPU::S_AND_B64: in moveToVALUImpl()
6897 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_AND_B32, MDT); in moveToVALUImpl()
6901 case AMDGPU::S_OR_B64: in moveToVALUImpl()
6902 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_OR_B32, MDT); in moveToVALUImpl()
6906 case AMDGPU::S_XOR_B64: in moveToVALUImpl()
6907 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XOR_B32, MDT); in moveToVALUImpl()
6911 case AMDGPU::S_NAND_B64: in moveToVALUImpl()
6912 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NAND_B32, MDT); in moveToVALUImpl()
6916 case AMDGPU::S_NOR_B64: in moveToVALUImpl()
6917 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NOR_B32, MDT); in moveToVALUImpl()
6921 case AMDGPU::S_XNOR_B64: in moveToVALUImpl()
6923 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XNOR_B32, MDT); in moveToVALUImpl()
6929 case AMDGPU::S_ANDN2_B64: in moveToVALUImpl()
6930 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ANDN2_B32, MDT); in moveToVALUImpl()
6934 case AMDGPU::S_ORN2_B64: in moveToVALUImpl()
6935 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ORN2_B32, MDT); in moveToVALUImpl()
6939 case AMDGPU::S_BREV_B64: in moveToVALUImpl()
6940 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_BREV_B32, true); in moveToVALUImpl()
6944 case AMDGPU::S_NOT_B64: in moveToVALUImpl()
6945 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_NOT_B32); in moveToVALUImpl()
6949 case AMDGPU::S_BCNT1_I32_B64: in moveToVALUImpl()
6954 case AMDGPU::S_BFE_I64: in moveToVALUImpl()
6959 case AMDGPU::S_FLBIT_I32_B64: in moveToVALUImpl()
6960 splitScalar64BitCountOp(Worklist, Inst, AMDGPU::V_FFBH_U32_e32); in moveToVALUImpl()
6963 case AMDGPU::S_FF1_I32_B64: in moveToVALUImpl()
6964 splitScalar64BitCountOp(Worklist, Inst, AMDGPU::V_FFBL_B32_e32); in moveToVALUImpl()
6968 case AMDGPU::S_LSHL_B32: in moveToVALUImpl()
6970 NewOpcode = AMDGPU::V_LSHLREV_B32_e64; in moveToVALUImpl()
6974 case AMDGPU::S_ASHR_I32: in moveToVALUImpl()
6976 NewOpcode = AMDGPU::V_ASHRREV_I32_e64; in moveToVALUImpl()
6980 case AMDGPU::S_LSHR_B32: in moveToVALUImpl()
6982 NewOpcode = AMDGPU::V_LSHRREV_B32_e64; in moveToVALUImpl()
6986 case AMDGPU::S_LSHL_B64: in moveToVALUImpl()
6989 ? AMDGPU::V_LSHLREV_B64_pseudo_e64 in moveToVALUImpl()
6990 : AMDGPU::V_LSHLREV_B64_e64; in moveToVALUImpl()
6994 case AMDGPU::S_ASHR_I64: in moveToVALUImpl()
6996 NewOpcode = AMDGPU::V_ASHRREV_I64_e64; in moveToVALUImpl()
7000 case AMDGPU::S_LSHR_B64: in moveToVALUImpl()
7002 NewOpcode = AMDGPU::V_LSHRREV_B64_e64; in moveToVALUImpl()
7007 case AMDGPU::S_ABS_I32: in moveToVALUImpl()
7012 case AMDGPU::S_CBRANCH_SCC0: in moveToVALUImpl()
7013 case AMDGPU::S_CBRANCH_SCC1: { in moveToVALUImpl()
7016 bool IsSCC = CondReg == AMDGPU::SCC; in moveToVALUImpl()
7018 Register EXEC = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; in moveToVALUImpl()
7019 unsigned Opc = ST.isWave32() ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64; in moveToVALUImpl()
7026 case AMDGPU::S_BFE_U64: in moveToVALUImpl()
7027 case AMDGPU::S_BFM_B64: in moveToVALUImpl()
7030 case AMDGPU::S_PACK_LL_B32_B16: in moveToVALUImpl()
7031 case AMDGPU::S_PACK_LH_B32_B16: in moveToVALUImpl()
7032 case AMDGPU::S_PACK_HL_B32_B16: in moveToVALUImpl()
7033 case AMDGPU::S_PACK_HH_B32_B16: in moveToVALUImpl()
7038 case AMDGPU::S_XNOR_B32: in moveToVALUImpl()
7043 case AMDGPU::S_NAND_B32: in moveToVALUImpl()
7044 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_AND_B32); in moveToVALUImpl()
7048 case AMDGPU::S_NOR_B32: in moveToVALUImpl()
7049 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_OR_B32); in moveToVALUImpl()
7053 case AMDGPU::S_ANDN2_B32: in moveToVALUImpl()
7054 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_AND_B32); in moveToVALUImpl()
7058 case AMDGPU::S_ORN2_B32: in moveToVALUImpl()
7059 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_OR_B32); in moveToVALUImpl()
7067 case AMDGPU::S_ADD_CO_PSEUDO: in moveToVALUImpl()
7068 case AMDGPU::S_SUB_CO_PSEUDO: { in moveToVALUImpl()
7069 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO) in moveToVALUImpl()
7070 ? AMDGPU::V_ADDC_U32_e64 in moveToVALUImpl()
7071 : AMDGPU::V_SUBB_U32_e64; in moveToVALUImpl()
7072 const auto *CarryRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); in moveToVALUImpl()
7077 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::COPY), NewCarryReg) in moveToVALUImpl()
7098 case AMDGPU::S_UADDO_PSEUDO: in moveToVALUImpl()
7099 case AMDGPU::S_USUBO_PSEUDO: { in moveToVALUImpl()
7106 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_UADDO_PSEUDO) in moveToVALUImpl()
7107 ? AMDGPU::V_ADD_CO_U32_e64 in moveToVALUImpl()
7108 : AMDGPU::V_SUB_CO_U32_e64; in moveToVALUImpl()
7126 case AMDGPU::S_CSELECT_B32: in moveToVALUImpl()
7127 case AMDGPU::S_CSELECT_B64: in moveToVALUImpl()
7131 case AMDGPU::S_CMP_EQ_I32: in moveToVALUImpl()
7132 case AMDGPU::S_CMP_LG_I32: in moveToVALUImpl()
7133 case AMDGPU::S_CMP_GT_I32: in moveToVALUImpl()
7134 case AMDGPU::S_CMP_GE_I32: in moveToVALUImpl()
7135 case AMDGPU::S_CMP_LT_I32: in moveToVALUImpl()
7136 case AMDGPU::S_CMP_LE_I32: in moveToVALUImpl()
7137 case AMDGPU::S_CMP_EQ_U32: in moveToVALUImpl()
7138 case AMDGPU::S_CMP_LG_U32: in moveToVALUImpl()
7139 case AMDGPU::S_CMP_GT_U32: in moveToVALUImpl()
7140 case AMDGPU::S_CMP_GE_U32: in moveToVALUImpl()
7141 case AMDGPU::S_CMP_LT_U32: in moveToVALUImpl()
7142 case AMDGPU::S_CMP_LE_U32: in moveToVALUImpl()
7143 case AMDGPU::S_CMP_EQ_U64: in moveToVALUImpl()
7144 case AMDGPU::S_CMP_LG_U64: in moveToVALUImpl()
7145 case AMDGPU::S_CMP_LT_F32: in moveToVALUImpl()
7146 case AMDGPU::S_CMP_EQ_F32: in moveToVALUImpl()
7147 case AMDGPU::S_CMP_LE_F32: in moveToVALUImpl()
7148 case AMDGPU::S_CMP_GT_F32: in moveToVALUImpl()
7149 case AMDGPU::S_CMP_LG_F32: in moveToVALUImpl()
7150 case AMDGPU::S_CMP_GE_F32: in moveToVALUImpl()
7151 case AMDGPU::S_CMP_O_F32: in moveToVALUImpl()
7152 case AMDGPU::S_CMP_U_F32: in moveToVALUImpl()
7153 case AMDGPU::S_CMP_NGE_F32: in moveToVALUImpl()
7154 case AMDGPU::S_CMP_NLG_F32: in moveToVALUImpl()
7155 case AMDGPU::S_CMP_NGT_F32: in moveToVALUImpl()
7156 case AMDGPU::S_CMP_NLE_F32: in moveToVALUImpl()
7157 case AMDGPU::S_CMP_NEQ_F32: in moveToVALUImpl()
7158 case AMDGPU::S_CMP_NLT_F32: in moveToVALUImpl()
7159 case AMDGPU::S_CMP_LT_F16: in moveToVALUImpl()
7160 case AMDGPU::S_CMP_EQ_F16: in moveToVALUImpl()
7161 case AMDGPU::S_CMP_LE_F16: in moveToVALUImpl()
7162 case AMDGPU::S_CMP_GT_F16: in moveToVALUImpl()
7163 case AMDGPU::S_CMP_LG_F16: in moveToVALUImpl()
7164 case AMDGPU::S_CMP_GE_F16: in moveToVALUImpl()
7165 case AMDGPU::S_CMP_O_F16: in moveToVALUImpl()
7166 case AMDGPU::S_CMP_U_F16: in moveToVALUImpl()
7167 case AMDGPU::S_CMP_NGE_F16: in moveToVALUImpl()
7168 case AMDGPU::S_CMP_NLG_F16: in moveToVALUImpl()
7169 case AMDGPU::S_CMP_NGT_F16: in moveToVALUImpl()
7170 case AMDGPU::S_CMP_NLE_F16: in moveToVALUImpl()
7171 case AMDGPU::S_CMP_NEQ_F16: in moveToVALUImpl()
7172 case AMDGPU::S_CMP_NLT_F16: { in moveToVALUImpl()
7177 if (AMDGPU::getNamedOperandIdx(NewOpcode, in moveToVALUImpl()
7178 AMDGPU::OpName::src0_modifiers) >= 0) { in moveToVALUImpl()
7191 int SCCIdx = Inst.findRegisterDefOperandIdx(AMDGPU::SCC); in moveToVALUImpl()
7197 case AMDGPU::S_CVT_HI_F32_F16: { in moveToVALUImpl()
7199 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in moveToVALUImpl()
7200 Register NewDst = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in moveToVALUImpl()
7201 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg) in moveToVALUImpl()
7215 case AMDGPU::S_MINIMUM_F32: in moveToVALUImpl()
7216 case AMDGPU::S_MAXIMUM_F32: in moveToVALUImpl()
7217 case AMDGPU::S_MINIMUM_F16: in moveToVALUImpl()
7218 case AMDGPU::S_MAXIMUM_F16: { in moveToVALUImpl()
7220 Register NewDst = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in moveToVALUImpl()
7237 if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) { in moveToVALUImpl()
7255 get(AMDGPU::V_READFIRSTLANE_B32), Inst.getOperand(0).getReg()) in moveToVALUImpl()
7278 Inst.setDesc(get(AMDGPU::IMPLICIT_DEF)); in moveToVALUImpl()
7294 if (AMDGPU::getNamedOperandIdx(NewOpcode, in moveToVALUImpl()
7295 AMDGPU::OpName::src0_modifiers) >= 0) in moveToVALUImpl()
7297 if (AMDGPU::hasNamedOperand(NewOpcode, AMDGPU::OpName::src0)) { in moveToVALUImpl()
7299 if (AMDGPU::isTrue16Inst(NewOpcode) && ST.useRealTrue16Insts() && in moveToVALUImpl()
7301 NewInstr.addReg(Src.getReg(), 0, AMDGPU::lo16); in moveToVALUImpl()
7306 if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) { in moveToVALUImpl()
7309 unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16; in moveToVALUImpl()
7312 } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) { in moveToVALUImpl()
7316 } else if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) { in moveToVALUImpl()
7329 if (AMDGPU::getNamedOperandIdx(NewOpcode, in moveToVALUImpl()
7330 AMDGPU::OpName::src1_modifiers) >= 0) in moveToVALUImpl()
7332 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::src1) >= 0) in moveToVALUImpl()
7334 if (AMDGPU::getNamedOperandIdx(NewOpcode, in moveToVALUImpl()
7335 AMDGPU::OpName::src2_modifiers) >= 0) in moveToVALUImpl()
7337 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::src2) >= 0) in moveToVALUImpl()
7339 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::clamp) >= 0) in moveToVALUImpl()
7341 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::omod) >= 0) in moveToVALUImpl()
7343 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::op_sel) >= 0) in moveToVALUImpl()
7356 if (Op.getReg() == AMDGPU::SCC) { in moveToVALUImpl()
7395 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in moveScalarAddSub()
7398 assert(Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32); in moveScalarAddSub()
7400 unsigned NewOpc = Opc == AMDGPU::S_ADD_I32 ? in moveScalarAddSub()
7401 AMDGPU::V_ADD_U32_e64 : AMDGPU::V_SUB_U32_e64; in moveScalarAddSub()
7403 assert(Inst.getOperand(3).getReg() == AMDGPU::SCC); in moveScalarAddSub()
7433 bool IsSCC = (CondReg == AMDGPU::SCC); in lowerSelect()
7448 RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); in lowerSelect()
7457 if (CandI.findRegisterDefOperandIdx(AMDGPU::SCC, false, false, &RI) != in lowerSelect()
7459 if (CandI.isCopy() && CandI.getOperand(0).getReg() == AMDGPU::SCC) { in lowerSelect()
7460 BuildMI(MBB, MII, DL, get(AMDGPU::COPY), NewCondReg) in lowerSelect()
7472 unsigned Opcode = (ST.getWavefrontSize() == 64) ? AMDGPU::S_CSELECT_B64 in lowerSelect()
7473 : AMDGPU::S_CSELECT_B32; in lowerSelect()
7483 if (Inst.getOpcode() == AMDGPU::S_CSELECT_B32) { in lowerSelect()
7484 NewInst = BuildMI(MBB, MII, DL, get(AMDGPU::V_CNDMASK_B32_e64), NewDestReg) in lowerSelect()
7492 BuildMI(MBB, MII, DL, get(AMDGPU::V_CNDMASK_B64_PSEUDO), NewDestReg) in lowerSelect()
7511 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in lowerScalarAbs()
7512 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in lowerScalarAbs()
7515 AMDGPU::V_SUB_U32_e32 : AMDGPU::V_SUB_CO_U32_e32; in lowerScalarAbs()
7521 BuildMI(MBB, MII, DL, get(AMDGPU::V_MAX_I32_e64), ResultReg) in lowerScalarAbs()
7541 Register NewDest = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in lowerScalarXnor()
7542 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src0, MRI, DL); in lowerScalarXnor()
7543 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src1, MRI, DL); in lowerScalarXnor()
7545 BuildMI(MBB, MII, DL, get(AMDGPU::V_XNOR_B32_e64), NewDest) in lowerScalarXnor()
7561 Register Temp = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); in lowerScalarXnor()
7562 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); in lowerScalarXnor()
7568 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src0); in lowerScalarXnor()
7569 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest) in lowerScalarXnor()
7573 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src1); in lowerScalarXnor()
7574 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest) in lowerScalarXnor()
7578 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), Temp) in lowerScalarXnor()
7582 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest).addReg(Temp); in lowerScalarXnor()
7606 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); in splitScalarNotBinop()
7607 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); in splitScalarNotBinop()
7613 MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest) in splitScalarNotBinop()
7635 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in splitScalarBinOpN2()
7636 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in splitScalarBinOpN2()
7638 MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Interm) in splitScalarBinOpN2()
7667 &AMDGPU::SGPR_32RegClass; in splitScalar64BitUnaryOp()
7670 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0); in splitScalar64BitUnaryOp()
7673 AMDGPU::sub0, Src0SubRC); in splitScalar64BitUnaryOp()
7678 RI.getSubRegisterClass(NewDestRC, AMDGPU::sub0); in splitScalar64BitUnaryOp()
7684 AMDGPU::sub1, Src0SubRC); in splitScalar64BitUnaryOp()
7695 .addImm(AMDGPU::sub0) in splitScalar64BitUnaryOp()
7697 .addImm(AMDGPU::sub1); in splitScalar64BitUnaryOp()
7719 Register FullDestReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); in splitScalarSMulU64()
7720 Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalarSMulU64()
7721 Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalarSMulU64()
7732 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0); in splitScalarSMulU64()
7736 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0); in splitScalarSMulU64()
7743 buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, AMDGPU::sub0, Src0SubRC); in splitScalarSMulU64()
7745 buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, AMDGPU::sub0, Src1SubRC); in splitScalarSMulU64()
7747 buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, AMDGPU::sub1, Src0SubRC); in splitScalarSMulU64()
7749 buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, AMDGPU::sub1, Src1SubRC); in splitScalarSMulU64()
7766 Register Op1L_Op0H_Reg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalarSMulU64()
7768 BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_LO_U32_e64), Op1L_Op0H_Reg) in splitScalarSMulU64()
7772 Register Op1H_Op0L_Reg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalarSMulU64()
7774 BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_LO_U32_e64), Op1H_Op0L_Reg) in splitScalarSMulU64()
7778 Register CarryReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalarSMulU64()
7780 BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_HI_U32_e64), CarryReg) in splitScalarSMulU64()
7785 BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_LO_U32_e64), DestSub0) in splitScalarSMulU64()
7789 Register AddReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalarSMulU64()
7790 MachineInstr *Add = BuildMI(MBB, MII, DL, get(AMDGPU::V_ADD_U32_e32), AddReg) in splitScalarSMulU64()
7795 BuildMI(MBB, MII, DL, get(AMDGPU::V_ADD_U32_e32), DestSub1) in splitScalarSMulU64()
7801 .addImm(AMDGPU::sub0) in splitScalarSMulU64()
7803 .addImm(AMDGPU::sub1); in splitScalarSMulU64()
7828 Register FullDestReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); in splitScalarSMulPseudo()
7829 Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalarSMulPseudo()
7830 Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalarSMulPseudo()
7841 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0); in splitScalarSMulPseudo()
7845 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0); in splitScalarSMulPseudo()
7852 buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, AMDGPU::sub0, Src0SubRC); in splitScalarSMulPseudo()
7854 buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, AMDGPU::sub0, Src1SubRC); in splitScalarSMulPseudo()
7857 unsigned NewOpc = Opc == AMDGPU::S_MUL_U64_U32_PSEUDO in splitScalarSMulPseudo()
7858 ? AMDGPU::V_MUL_HI_U32_e64 in splitScalarSMulPseudo()
7859 : AMDGPU::V_MUL_HI_I32_e64; in splitScalarSMulPseudo()
7864 BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_LO_U32_e64), DestSub0) in splitScalarSMulPseudo()
7870 .addImm(AMDGPU::sub0) in splitScalarSMulPseudo()
7872 .addImm(AMDGPU::sub1); in splitScalarSMulPseudo()
7901 &AMDGPU::SGPR_32RegClass; in splitScalar64BitBinaryOp()
7904 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0); in splitScalar64BitBinaryOp()
7907 &AMDGPU::SGPR_32RegClass; in splitScalar64BitBinaryOp()
7910 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0); in splitScalar64BitBinaryOp()
7913 AMDGPU::sub0, Src0SubRC); in splitScalar64BitBinaryOp()
7915 AMDGPU::sub0, Src1SubRC); in splitScalar64BitBinaryOp()
7917 AMDGPU::sub1, Src0SubRC); in splitScalar64BitBinaryOp()
7919 AMDGPU::sub1, Src1SubRC); in splitScalar64BitBinaryOp()
7924 RI.getSubRegisterClass(NewDestRC, AMDGPU::sub0); in splitScalar64BitBinaryOp()
7939 .addImm(AMDGPU::sub0) in splitScalar64BitBinaryOp()
7941 .addImm(AMDGPU::sub1); in splitScalar64BitBinaryOp()
7967 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in splitScalar64BitXnor()
7980 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B64), Interm) in splitScalar64BitXnor()
7985 MachineInstr &Xor = *BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B64), NewDest) in splitScalar64BitXnor()
8005 const MCInstrDesc &InstDesc = get(AMDGPU::V_BCNT_U32_B32_e64); in splitScalar64BitBCNT()
8008 &AMDGPU::SGPR_32RegClass; in splitScalar64BitBCNT()
8010 Register MidReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitBCNT()
8011 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitBCNT()
8014 RI.getSubRegisterClass(SrcRC, AMDGPU::sub0); in splitScalar64BitBCNT()
8017 AMDGPU::sub0, SrcSubRC); in splitScalar64BitBCNT()
8019 AMDGPU::sub1, SrcSubRC); in splitScalar64BitBCNT()
8047 assert(Inst.getOpcode() == AMDGPU::S_BFE_I64 && BitWidth <= 32 && in splitScalar64BitBFE()
8051 Register MidRegLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitBFE()
8052 Register MidRegHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitBFE()
8053 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); in splitScalar64BitBFE()
8055 BuildMI(MBB, MII, DL, get(AMDGPU::V_BFE_I32_e64), MidRegLo) in splitScalar64BitBFE()
8056 .addReg(Inst.getOperand(1).getReg(), 0, AMDGPU::sub0) in splitScalar64BitBFE()
8060 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e32), MidRegHi) in splitScalar64BitBFE()
8066 .addImm(AMDGPU::sub0) in splitScalar64BitBFE()
8068 .addImm(AMDGPU::sub1); in splitScalar64BitBFE()
8076 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitBFE()
8077 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); in splitScalar64BitBFE()
8079 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e64), TmpReg) in splitScalar64BitBFE()
8081 .addReg(Src.getReg(), 0, AMDGPU::sub0); in splitScalar64BitBFE()
8084 .addReg(Src.getReg(), 0, AMDGPU::sub0) in splitScalar64BitBFE()
8085 .addImm(AMDGPU::sub0) in splitScalar64BitBFE()
8087 .addImm(AMDGPU::sub1); in splitScalar64BitBFE()
8111 bool IsCtlz = Opcode == AMDGPU::V_FFBH_U32_e32; in splitScalar64BitCountOp()
8113 ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_CO_U32_e32; in splitScalar64BitCountOp()
8116 Src.isReg() ? MRI.getRegClass(Src.getReg()) : &AMDGPU::SGPR_32RegClass; in splitScalar64BitCountOp()
8118 RI.getSubRegisterClass(SrcRC, AMDGPU::sub0); in splitScalar64BitCountOp()
8121 buildExtractSubRegOrImm(MII, MRI, Src, SrcRC, AMDGPU::sub0, SrcSubRC); in splitScalar64BitCountOp()
8123 buildExtractSubRegOrImm(MII, MRI, Src, SrcRC, AMDGPU::sub1, SrcSubRC); in splitScalar64BitCountOp()
8125 Register MidReg1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitCountOp()
8126 Register MidReg2 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitCountOp()
8127 Register MidReg3 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitCountOp()
8128 Register MidReg4 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitCountOp()
8139 BuildMI(MBB, MII, DL, get(AMDGPU::V_MIN_U32_e64), MidReg4) in splitScalar64BitCountOp()
8158 case AMDGPU::COPY: in addUsersToMoveToVALUWorklist()
8159 case AMDGPU::WQM: in addUsersToMoveToVALUWorklist()
8160 case AMDGPU::SOFT_WQM: in addUsersToMoveToVALUWorklist()
8161 case AMDGPU::STRICT_WWM: in addUsersToMoveToVALUWorklist()
8162 case AMDGPU::STRICT_WQM: in addUsersToMoveToVALUWorklist()
8163 case AMDGPU::REG_SEQUENCE: in addUsersToMoveToVALUWorklist()
8164 case AMDGPU::PHI: in addUsersToMoveToVALUWorklist()
8165 case AMDGPU::INSERT_SUBREG: in addUsersToMoveToVALUWorklist()
8187 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in movePackToVALU()
8194 case AMDGPU::S_PACK_LL_B32_B16: { in movePackToVALU()
8195 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in movePackToVALU()
8196 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in movePackToVALU()
8200 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg) in movePackToVALU()
8203 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_B32_e64), TmpReg) in movePackToVALU()
8207 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHL_OR_B32_e64), ResultReg) in movePackToVALU()
8213 case AMDGPU::S_PACK_LH_B32_B16: { in movePackToVALU()
8214 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in movePackToVALU()
8215 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg) in movePackToVALU()
8217 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_BFI_B32_e64), ResultReg) in movePackToVALU()
8223 case AMDGPU::S_PACK_HL_B32_B16: { in movePackToVALU()
8224 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in movePackToVALU()
8225 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg) in movePackToVALU()
8228 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHL_OR_B32_e64), ResultReg) in movePackToVALU()
8234 case AMDGPU::S_PACK_HH_B32_B16: { in movePackToVALU()
8235 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in movePackToVALU()
8236 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in movePackToVALU()
8237 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg) in movePackToVALU()
8240 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg) in movePackToVALU()
8242 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_OR_B32_e64), ResultReg) in movePackToVALU()
8263 assert(Op.isReg() && Op.getReg() == AMDGPU::SCC && Op.isDef() && in addSCCDefUsersToVALUWorklist()
8272 int SCCIdx = MI.findRegisterUseOperandIdx(AMDGPU::SCC, false, &RI); in addSCCDefUsersToVALUWorklist()
8289 if (MI.findRegisterDefOperandIdx(AMDGPU::SCC, false, false, &RI) != -1) in addSCCDefUsersToVALUWorklist()
8310 if (MI.modifiesRegister(AMDGPU::VCC, &RI)) in addSCCDefsToVALUWorklist()
8312 if (MI.definesRegister(AMDGPU::SCC, &RI)) { in addSCCDefsToVALUWorklist()
8327 case AMDGPU::COPY: in getDestEquivalentVGPRClass()
8328 case AMDGPU::PHI: in getDestEquivalentVGPRClass()
8329 case AMDGPU::REG_SEQUENCE: in getDestEquivalentVGPRClass()
8330 case AMDGPU::INSERT_SUBREG: in getDestEquivalentVGPRClass()
8331 case AMDGPU::WQM: in getDestEquivalentVGPRClass()
8332 case AMDGPU::SOFT_WQM: in getDestEquivalentVGPRClass()
8333 case AMDGPU::STRICT_WWM: in getDestEquivalentVGPRClass()
8334 case AMDGPU::STRICT_WQM: { in getDestEquivalentVGPRClass()
8341 case AMDGPU::PHI: in getDestEquivalentVGPRClass()
8342 case AMDGPU::REG_SEQUENCE: in getDestEquivalentVGPRClass()
8343 case AMDGPU::INSERT_SUBREG: in getDestEquivalentVGPRClass()
8353 if (RI.isVGPRClass(NewDstRC) || NewDstRC == &AMDGPU::VReg_1RegClass) in getDestEquivalentVGPRClass()
8441 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OperandName); in getNamedOperand()
8451 ? (int64_t)AMDGPU::UfmtGFX11::UFMT_32_FLOAT in getDefaultRsrcDataFormat()
8452 : (int64_t)AMDGPU::UfmtGFX10::UFMT_32_FLOAT; in getDefaultRsrcDataFormat()
8458 uint64_t RsrcDataFormat = AMDGPU::RSRC_DATA_FORMAT; in getDefaultRsrcDataFormat()
8475 AMDGPU::RSRC_TID_ENABLE | in getScratchRsrcWords23()
8481 Rsrc23 |= EltSizeValue << AMDGPU::RSRC_ELEMENT_SIZE_SHIFT; in getScratchRsrcWords23()
8486 Rsrc23 |= IndexStride << AMDGPU::RSRC_INDEX_STRIDE_SHIFT; in getScratchRsrcWords23()
8492 Rsrc23 &= ~AMDGPU::RSRC_DATA_FORMAT; in getScratchRsrcWords23()
8510 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::vaddr); in isStackAccess()
8518 return getNamedOperand(MI, AMDGPU::OpName::vdata)->getReg(); in isStackAccess()
8523 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::addr); in isSGPRStackAccess()
8526 return getNamedOperand(MI, AMDGPU::OpName::data)->getReg(); in isSGPRStackAccess()
8606 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0); in getInstSizeInBytes()
8610 int RSrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc); in getInstSizeInBytes()
8645 return Branch.getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO; in isNonUniformBranchInstr()
8657 if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) { in convertNonUniformIfRegion()
8660 BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_IF), DstReg) in convertNonUniformIfRegion()
8664 BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_END_CF)) in convertNonUniformIfRegion()
8683 if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) { in convertNonUniformLoopRegion()
8702 get(AMDGPU::SI_IF_BREAK), BackEdgeReg) in convertNonUniformLoopRegion()
8706 BuildMI(*(MF), Branch->getDebugLoc(), get(AMDGPU::SI_LOOP)) in convertNonUniformLoopRegion()
8720 {AMDGPU::TI_CONSTDATA_START, "amdgpu-constdata-start"}, in getSerializableTargetIndices()
8721 {AMDGPU::TI_SCRATCH_RSRC_DWORD0, "amdgpu-scratch-rsrc-dword0"}, in getSerializableTargetIndices()
8722 {AMDGPU::TI_SCRATCH_RSRC_DWORD1, "amdgpu-scratch-rsrc-dword1"}, in getSerializableTargetIndices()
8723 {AMDGPU::TI_SCRATCH_RSRC_DWORD2, "amdgpu-scratch-rsrc-dword2"}, in getSerializableTargetIndices()
8724 {AMDGPU::TI_SCRATCH_RSRC_DWORD3, "amdgpu-scratch-rsrc-dword3"}}; in getSerializableTargetIndices()
8792 if (MFI->checkFlag(SrcReg, AMDGPU::VirtRegFlag::WWM_REG)) in getLiveRangeSplitOpcode()
8793 return AMDGPU::WWM_COPY; in getLiveRangeSplitOpcode()
8795 return AMDGPU::COPY; in getLiveRangeSplitOpcode()
8816 (isSpillOpcode(Opc) || (!MI.isTerminator() && Opc != AMDGPU::COPY && in isBasicBlockPrologue()
8817 MI.modifiesRegister(AMDGPU::EXEC, &RI))); in isBasicBlockPrologue()
8826 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e64), DestReg); in getAddNoCarry()
8832 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_CO_U32_e64), DestReg) in getAddNoCarry()
8842 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e32), DestReg); in getAddNoCarry()
8845 Register UnusedCarry = !RS.isRegUsed(AMDGPU::VCC) in getAddNoCarry()
8855 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_CO_U32_e64), DestReg) in getAddNoCarry()
8861 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR: in isKillTerminator()
8862 case AMDGPU::SI_KILL_I1_TERMINATOR: in isKillTerminator()
8871 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO: in getKillTerminatorFromPseudo()
8872 return get(AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR); in getKillTerminatorFromPseudo()
8873 case AMDGPU::SI_KILL_I1_PSEUDO: in getKillTerminatorFromPseudo()
8874 return get(AMDGPU::SI_KILL_I1_TERMINATOR); in getKillTerminatorFromPseudo()
8899 if (Op.isReg() && Op.getReg() == AMDGPU::VCC) in fixImplicitOperands()
8900 Op.setReg(AMDGPU::VCC_LO); in fixImplicitOperands()
8909 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::sbase); in isBufferSMRD()
8914 return RI.getRegClass(RCID)->hasSubClassEq(&AMDGPU::SGPR_128RegClass); in isBufferSMRD()
9018 unsigned N = AMDGPU::getNumFlatOffsetBits(ST); in isLegalFLATOffset()
9030 const unsigned NumBits = AMDGPU::getNumFlatOffsetBits(ST) - 1; in splitFlatOffset()
9060 return FlatVariant != SIInstrFlags::FLAT || AMDGPU::isGFX12Plus(ST); in allowNegativeFlatOffset()
9089 case AMDGPU::V_MOVRELS_B32_dpp_gfx10: in isAsmOnlyOpcode()
9090 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10: in isAsmOnlyOpcode()
9091 case AMDGPU::V_MOVRELD_B32_dpp_gfx10: in isAsmOnlyOpcode()
9092 case AMDGPU::V_MOVRELD_B32_sdwa_gfx10: in isAsmOnlyOpcode()
9093 case AMDGPU::V_MOVRELSD_B32_dpp_gfx10: in isAsmOnlyOpcode()
9094 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10: in isAsmOnlyOpcode()
9095 case AMDGPU::V_MOVRELSD_2_B32_dpp_gfx10: in isAsmOnlyOpcode()
9096 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10: in isAsmOnlyOpcode()
9133 int MFMAOp = AMDGPU::getMFMAEarlyClobberOp(Opcode); in pseudoToMCOpcode()
9138 int MCOp = AMDGPU::getMCOpcode(Opcode, Gen); in pseudoToMCOpcode()
9147 NMCOp = AMDGPU::getMCOpcode(Opcode, SIEncodingFamily::GFX940); in pseudoToMCOpcode()
9149 NMCOp = AMDGPU::getMCOpcode(Opcode, SIEncodingFamily::GFX90A); in pseudoToMCOpcode()
9151 NMCOp = AMDGPU::getMCOpcode(Opcode, SIEncodingFamily::GFX9); in pseudoToMCOpcode()
9193 case AMDGPU::REG_SEQUENCE: in followSubRegDef()
9197 case AMDGPU::INSERT_SUBREG: in followSubRegDef()
9223 case AMDGPU::COPY: in getVRegSubRegDef()
9224 case AMDGPU::V_MOV_B32_e32: { in getVRegSubRegDef()
9273 if (I->modifiesRegister(AMDGPU::EXEC, TRI)) in execMayBeModifiedBeforeUse()
9330 } else if (TRI->regsOverlap(Reg, AMDGPU::EXEC)) in execMayBeModifiedBeforeAnyUse()
9355 (InsPt->getOpcode() == AMDGPU::SI_IF || in createPHISourceCopy()
9356 InsPt->getOpcode() == AMDGPU::SI_ELSE || in createPHISourceCopy()
9357 InsPt->getOpcode() == AMDGPU::SI_IF_BREAK) && in createPHISourceCopy()
9361 get(ST.isWave32() ? AMDGPU::S_MOV_B32_term in createPHISourceCopy()
9362 : AMDGPU::S_MOV_B64_term), in createPHISourceCopy()
9365 .addReg(AMDGPU::EXEC, RegState::Implicit); in createPHISourceCopy()
9397 if (RC->hasSuperClassEq(&AMDGPU::SReg_32RegClass)) { in foldMemoryOperandImpl()
9398 MRI.constrainRegClass(VirtReg, &AMDGPU::SReg_32_XM0_XEXECRegClass); in foldMemoryOperandImpl()
9400 } else if (RC->hasSuperClassEq(&AMDGPU::SReg_64RegClass)) { in foldMemoryOperandImpl()
9401 MRI.constrainRegClass(VirtReg, &AMDGPU::SReg_64_XEXECRegClass); in foldMemoryOperandImpl()
9432 if (AMDGPU::isIntrinsicSourceOfDivergence(IID)) in getGenericInstructionUniformity()
9434 if (AMDGPU::isIntrinsicAlwaysUniform(IID)) in getGenericInstructionUniformity()
9453 if (opcode == AMDGPU::G_LOAD) { in getGenericInstructionUniformity()
9468 opcode == AMDGPU::G_ATOMIC_CMPXCHG || in getGenericInstructionUniformity()
9469 opcode == AMDGPU::G_ATOMIC_CMPXCHG_WITH_SUCCESS || in getGenericInstructionUniformity()
9470 AMDGPU::isGenericAtomic(opcode)) { in getGenericInstructionUniformity()
9483 if (opcode == AMDGPU::V_READLANE_B32 || in getInstructionUniformity()
9484 opcode == AMDGPU::V_READFIRSTLANE_B32 || in getInstructionUniformity()
9485 opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR) in getInstructionUniformity()
9547 if (RegBank && RegBank->getID() != AMDGPU::SGPRRegBankID) in getInstructionUniformity()
9592 case AMDGPU::S_CMP_EQ_U32: in analyzeCompare()
9593 case AMDGPU::S_CMP_EQ_I32: in analyzeCompare()
9594 case AMDGPU::S_CMP_LG_U32: in analyzeCompare()
9595 case AMDGPU::S_CMP_LG_I32: in analyzeCompare()
9596 case AMDGPU::S_CMP_LT_U32: in analyzeCompare()
9597 case AMDGPU::S_CMP_LT_I32: in analyzeCompare()
9598 case AMDGPU::S_CMP_GT_U32: in analyzeCompare()
9599 case AMDGPU::S_CMP_GT_I32: in analyzeCompare()
9600 case AMDGPU::S_CMP_LE_U32: in analyzeCompare()
9601 case AMDGPU::S_CMP_LE_I32: in analyzeCompare()
9602 case AMDGPU::S_CMP_GE_U32: in analyzeCompare()
9603 case AMDGPU::S_CMP_GE_I32: in analyzeCompare()
9604 case AMDGPU::S_CMP_EQ_U64: in analyzeCompare()
9605 case AMDGPU::S_CMP_LG_U64: in analyzeCompare()
9620 case AMDGPU::S_CMPK_EQ_U32: in analyzeCompare()
9621 case AMDGPU::S_CMPK_EQ_I32: in analyzeCompare()
9622 case AMDGPU::S_CMPK_LG_U32: in analyzeCompare()
9623 case AMDGPU::S_CMPK_LG_I32: in analyzeCompare()
9624 case AMDGPU::S_CMPK_LT_U32: in analyzeCompare()
9625 case AMDGPU::S_CMPK_LT_I32: in analyzeCompare()
9626 case AMDGPU::S_CMPK_GT_U32: in analyzeCompare()
9627 case AMDGPU::S_CMPK_GT_I32: in analyzeCompare()
9628 case AMDGPU::S_CMPK_LE_U32: in analyzeCompare()
9629 case AMDGPU::S_CMPK_LE_I32: in analyzeCompare()
9630 case AMDGPU::S_CMPK_GE_U32: in analyzeCompare()
9631 case AMDGPU::S_CMPK_GE_I32: in analyzeCompare()
9682 if (Def->getOpcode() != AMDGPU::S_AND_B32 && in optimizeCompareInstr()
9683 Def->getOpcode() != AMDGPU::S_AND_B64) in optimizeCompareInstr()
9725 if (I->modifiesRegister(AMDGPU::SCC, &RI) || in optimizeCompareInstr()
9726 I->killsRegister(AMDGPU::SCC, &RI)) in optimizeCompareInstr()
9730 MachineOperand *SccDef = Def->findRegisterDefOperand(AMDGPU::SCC); in optimizeCompareInstr()
9742 unsigned NewOpc = (SrcSize == 32) ? IsReversedCC ? AMDGPU::S_BITCMP0_B32 in optimizeCompareInstr()
9743 : AMDGPU::S_BITCMP1_B32 in optimizeCompareInstr()
9744 : IsReversedCC ? AMDGPU::S_BITCMP0_B64 in optimizeCompareInstr()
9745 : AMDGPU::S_BITCMP1_B64; in optimizeCompareInstr()
9758 case AMDGPU::S_CMP_EQ_U32: in optimizeCompareInstr()
9759 case AMDGPU::S_CMP_EQ_I32: in optimizeCompareInstr()
9760 case AMDGPU::S_CMPK_EQ_U32: in optimizeCompareInstr()
9761 case AMDGPU::S_CMPK_EQ_I32: in optimizeCompareInstr()
9763 case AMDGPU::S_CMP_GE_U32: in optimizeCompareInstr()
9764 case AMDGPU::S_CMPK_GE_U32: in optimizeCompareInstr()
9766 case AMDGPU::S_CMP_GE_I32: in optimizeCompareInstr()
9767 case AMDGPU::S_CMPK_GE_I32: in optimizeCompareInstr()
9769 case AMDGPU::S_CMP_EQ_U64: in optimizeCompareInstr()
9771 case AMDGPU::S_CMP_LG_U32: in optimizeCompareInstr()
9772 case AMDGPU::S_CMP_LG_I32: in optimizeCompareInstr()
9773 case AMDGPU::S_CMPK_LG_U32: in optimizeCompareInstr()
9774 case AMDGPU::S_CMPK_LG_I32: in optimizeCompareInstr()
9776 case AMDGPU::S_CMP_GT_U32: in optimizeCompareInstr()
9777 case AMDGPU::S_CMPK_GT_U32: in optimizeCompareInstr()
9779 case AMDGPU::S_CMP_GT_I32: in optimizeCompareInstr()
9780 case AMDGPU::S_CMPK_GT_I32: in optimizeCompareInstr()
9782 case AMDGPU::S_CMP_LG_U64: in optimizeCompareInstr()
9794 int OpNo = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OpName); in enforceOperandRCAlignment()
9808 IsAGPR ? &AMDGPU::AGPR_32RegClass : &AMDGPU::VGPR_32RegClass); in enforceOperandRCAlignment()
9809 BuildMI(*BB, MI, DL, get(AMDGPU::IMPLICIT_DEF), Undef); in enforceOperandRCAlignment()
9811 MRI.createVirtualRegister(IsAGPR ? &AMDGPU::AReg_64_Align2RegClass in enforceOperandRCAlignment()
9812 : &AMDGPU::VReg_64_Align2RegClass); in enforceOperandRCAlignment()
9813 BuildMI(*BB, MI, DL, get(AMDGPU::REG_SEQUENCE), NewVR) in enforceOperandRCAlignment()
9815 .addImm(AMDGPU::sub0) in enforceOperandRCAlignment()
9817 .addImm(AMDGPU::sub1); in enforceOperandRCAlignment()
9819 Op.setSubReg(AMDGPU::sub0); in enforceOperandRCAlignment()