Lines Matching refs:AMDGPU
45 namespace AMDGPU { namespace
68 : AMDGPUGenInstrInfo(AMDGPU::ADJCALLSTACKUP, AMDGPU::ADJCALLSTACKDOWN), in SIInstrInfo()
90 int Op0Idx = AMDGPU::getNamedOperandIdx(Opc0, OpName); in nodesHaveSameOperandValue()
91 int Op1Idx = AMDGPU::getNamedOperandIdx(Opc1, OpName); in nodesHaveSameOperandValue()
144 case AMDGPU::S_AND_SAVEEXEC_B32: in resultDependsOnExec()
145 case AMDGPU::S_AND_SAVEEXEC_B64: in resultDependsOnExec()
147 case AMDGPU::S_AND_B32: in resultDependsOnExec()
148 case AMDGPU::S_AND_B64: in resultDependsOnExec()
149 if (!Use.readsRegister(AMDGPU::EXEC)) in resultDependsOnExec()
162 case AMDGPU::V_READFIRSTLANE_B32: in resultDependsOnExec()
171 return MO.getReg() == AMDGPU::EXEC && MO.isImplicit() && in isIgnorableUse()
201 int Offset0Idx = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset); in areLoadsFromSameBasePtr()
202 int Offset1Idx = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset); in areLoadsFromSameBasePtr()
219 if (AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::sbase) == -1 || in areLoadsFromSameBasePtr()
220 AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::sbase) == -1) in areLoadsFromSameBasePtr()
253 if (!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::soffset) || in areLoadsFromSameBasePtr()
254 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::vaddr) || in areLoadsFromSameBasePtr()
255 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::srsrc)) in areLoadsFromSameBasePtr()
258 int OffIdx0 = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset); in areLoadsFromSameBasePtr()
259 int OffIdx1 = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset); in areLoadsFromSameBasePtr()
287 case AMDGPU::DS_READ2ST64_B32: in isStride64()
288 case AMDGPU::DS_READ2ST64_B64: in isStride64()
289 case AMDGPU::DS_WRITE2ST64_B32: in isStride64()
290 case AMDGPU::DS_WRITE2ST64_B64: in isStride64()
310 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr); in getMemOperandsWithOffsetWidth()
311 OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset); in getMemOperandsWithOffsetWidth()
322 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); in getMemOperandsWithOffsetWidth()
324 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0); in getMemOperandsWithOffsetWidth()
331 getNamedOperand(LdSt, AMDGPU::OpName::offset0); in getMemOperandsWithOffsetWidth()
333 getNamedOperand(LdSt, AMDGPU::OpName::offset1); in getMemOperandsWithOffsetWidth()
348 int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0); in getMemOperandsWithOffsetWidth()
358 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); in getMemOperandsWithOffsetWidth()
360 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0); in getMemOperandsWithOffsetWidth()
362 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data1); in getMemOperandsWithOffsetWidth()
372 const MachineOperand *RSrc = getNamedOperand(LdSt, AMDGPU::OpName::srsrc); in getMemOperandsWithOffsetWidth()
376 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr); in getMemOperandsWithOffsetWidth()
380 getNamedOperand(LdSt, AMDGPU::OpName::offset); in getMemOperandsWithOffsetWidth()
383 getNamedOperand(LdSt, AMDGPU::OpName::soffset); in getMemOperandsWithOffsetWidth()
391 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); in getMemOperandsWithOffsetWidth()
393 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata); in getMemOperandsWithOffsetWidth()
401 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc); in getMemOperandsWithOffsetWidth()
403 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0); in getMemOperandsWithOffsetWidth()
409 BaseOps.push_back(getNamedOperand(LdSt, AMDGPU::OpName::vaddr)); in getMemOperandsWithOffsetWidth()
413 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata); in getMemOperandsWithOffsetWidth()
419 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::sbase); in getMemOperandsWithOffsetWidth()
423 OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset); in getMemOperandsWithOffsetWidth()
426 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::sdst); in getMemOperandsWithOffsetWidth()
433 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr); in getMemOperandsWithOffsetWidth()
436 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::saddr); in getMemOperandsWithOffsetWidth()
439 Offset = getNamedOperand(LdSt, AMDGPU::OpName::offset)->getImm(); in getMemOperandsWithOffsetWidth()
441 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); in getMemOperandsWithOffsetWidth()
443 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata); in getMemOperandsWithOffsetWidth()
549 BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_ILLEGAL_COPY), DestReg) in reportIllegalCopy()
568 assert((AMDGPU::SReg_32RegClass.contains(SrcReg) || in indirectCopyToAGPR()
569 AMDGPU::AGPR_32RegClass.contains(SrcReg)) && in indirectCopyToAGPR()
572 assert(AMDGPU::AGPR_32RegClass.contains(DestReg) && in indirectCopyToAGPR()
582 if (Def->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64) in indirectCopyToAGPR()
603 BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg) in indirectCopyToAGPR()
621 unsigned MaxVGPRs = RI.getRegPressureLimit(&AMDGPU::VGPR_32RegClass, in indirectCopyToAGPR()
626 unsigned RegNo = (DestReg - AMDGPU::AGPR0) % 3; in indirectCopyToAGPR()
635 while (RegNo-- && RS.FindUnusedReg(&AMDGPU::VGPR_32RegClass)) { in indirectCopyToAGPR()
636 Register Tmp2 = RS.scavengeRegister(&AMDGPU::VGPR_32RegClass, 0); in indirectCopyToAGPR()
644 unsigned TmpCopyOp = AMDGPU::V_MOV_B32_e32; in indirectCopyToAGPR()
645 if (AMDGPU::AGPR_32RegClass.contains(SrcReg)) { in indirectCopyToAGPR()
646 TmpCopyOp = AMDGPU::V_ACCVGPR_READ_B32_e64; in indirectCopyToAGPR()
648 assert(AMDGPU::SReg_32RegClass.contains(SrcReg)); in indirectCopyToAGPR()
659 = BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg) in indirectCopyToAGPR()
678 unsigned Opcode = AMDGPU::S_MOV_B32; in expandSGPRCopy()
682 bool AlignedDest = ((Reg - AMDGPU::SGPR0) % 2) == 0; in expandSGPRCopy()
683 bool AlignedSrc = ((Src - AMDGPU::SGPR0) % 2) == 0; in expandSGPRCopy()
688 Opcode = AMDGPU::S_MOV_B64; in expandSGPRCopy()
727 assert(RI.getSubReg(Super, AMDGPU::lo16) == RegToFix); in copyPhysReg()
732 BuildMI(MBB, MI, DL, get(AMDGPU::BUNDLE)); in copyPhysReg()
739 if (RC == &AMDGPU::VGPR_32RegClass) { in copyPhysReg()
740 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) || in copyPhysReg()
741 AMDGPU::SReg_32RegClass.contains(SrcReg) || in copyPhysReg()
742 AMDGPU::AGPR_32RegClass.contains(SrcReg)); in copyPhysReg()
743 unsigned Opc = AMDGPU::AGPR_32RegClass.contains(SrcReg) ? in copyPhysReg()
744 AMDGPU::V_ACCVGPR_READ_B32_e64 : AMDGPU::V_MOV_B32_e32; in copyPhysReg()
750 if (RC == &AMDGPU::SReg_32_XM0RegClass || in copyPhysReg()
751 RC == &AMDGPU::SReg_32RegClass) { in copyPhysReg()
752 if (SrcReg == AMDGPU::SCC) { in copyPhysReg()
753 BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B32), DestReg) in copyPhysReg()
759 if (DestReg == AMDGPU::VCC_LO) { in copyPhysReg()
760 if (AMDGPU::SReg_32RegClass.contains(SrcReg)) { in copyPhysReg()
761 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), AMDGPU::VCC_LO) in copyPhysReg()
765 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg)); in copyPhysReg()
766 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32)) in copyPhysReg()
774 if (!AMDGPU::SReg_32RegClass.contains(SrcReg)) { in copyPhysReg()
779 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg) in copyPhysReg()
784 if (RC == &AMDGPU::SReg_64RegClass) { in copyPhysReg()
785 if (SrcReg == AMDGPU::SCC) { in copyPhysReg()
786 BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B64), DestReg) in copyPhysReg()
792 if (DestReg == AMDGPU::VCC) { in copyPhysReg()
793 if (AMDGPU::SReg_64RegClass.contains(SrcReg)) { in copyPhysReg()
794 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), AMDGPU::VCC) in copyPhysReg()
798 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg)); in copyPhysReg()
799 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32)) in copyPhysReg()
807 if (!AMDGPU::SReg_64RegClass.contains(SrcReg)) { in copyPhysReg()
812 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg) in copyPhysReg()
817 if (DestReg == AMDGPU::SCC) { in copyPhysReg()
820 if (AMDGPU::SReg_64RegClass.contains(SrcReg)) { in copyPhysReg()
825 BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U64)) in copyPhysReg()
829 assert(AMDGPU::SReg_32RegClass.contains(SrcReg)); in copyPhysReg()
830 BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U32)) in copyPhysReg()
838 if (RC == &AMDGPU::AGPR_32RegClass) { in copyPhysReg()
839 if (AMDGPU::VGPR_32RegClass.contains(SrcReg) || in copyPhysReg()
840 (ST.hasGFX90AInsts() && AMDGPU::SReg_32RegClass.contains(SrcReg))) { in copyPhysReg()
841 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg) in copyPhysReg()
846 if (AMDGPU::AGPR_32RegClass.contains(SrcReg) && ST.hasGFX90AInsts()) { in copyPhysReg()
847 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_MOV_B32), DestReg) in copyPhysReg()
861 assert(AMDGPU::VGPR_LO16RegClass.contains(SrcReg) || in copyPhysReg()
862 AMDGPU::VGPR_HI16RegClass.contains(SrcReg) || in copyPhysReg()
863 AMDGPU::SReg_LO16RegClass.contains(SrcReg) || in copyPhysReg()
864 AMDGPU::AGPR_LO16RegClass.contains(SrcReg)); in copyPhysReg()
866 bool IsSGPRDst = AMDGPU::SReg_LO16RegClass.contains(DestReg); in copyPhysReg()
867 bool IsSGPRSrc = AMDGPU::SReg_LO16RegClass.contains(SrcReg); in copyPhysReg()
868 bool IsAGPRDst = AMDGPU::AGPR_LO16RegClass.contains(DestReg); in copyPhysReg()
869 bool IsAGPRSrc = AMDGPU::AGPR_LO16RegClass.contains(SrcReg); in copyPhysReg()
870 bool DstLow = AMDGPU::VGPR_LO16RegClass.contains(DestReg) || in copyPhysReg()
871 AMDGPU::SReg_LO16RegClass.contains(DestReg) || in copyPhysReg()
872 AMDGPU::AGPR_LO16RegClass.contains(DestReg); in copyPhysReg()
873 bool SrcLow = AMDGPU::VGPR_LO16RegClass.contains(SrcReg) || in copyPhysReg()
874 AMDGPU::SReg_LO16RegClass.contains(SrcReg) || in copyPhysReg()
875 AMDGPU::AGPR_LO16RegClass.contains(SrcReg); in copyPhysReg()
885 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), NewDestReg) in copyPhysReg()
906 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), NewDestReg) in copyPhysReg()
911 auto MIB = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_sdwa), NewDestReg) in copyPhysReg()
915 .addImm(DstLow ? AMDGPU::SDWA::SdwaSel::WORD_0 in copyPhysReg()
916 : AMDGPU::SDWA::SdwaSel::WORD_1) in copyPhysReg()
917 .addImm(AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE) in copyPhysReg()
918 .addImm(SrcLow ? AMDGPU::SDWA::SdwaSel::WORD_0 in copyPhysReg()
919 : AMDGPU::SDWA::SdwaSel::WORD_1) in copyPhysReg()
929 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_e32), DestReg) in copyPhysReg()
934 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), DestReg) in copyPhysReg()
962 unsigned Opcode = AMDGPU::V_MOV_B32_e32; in copyPhysReg()
965 Opcode = AMDGPU::V_ACCVGPR_MOV_B32; in copyPhysReg()
968 Opcode = AMDGPU::V_ACCVGPR_WRITE_B32_e64; in copyPhysReg()
970 Opcode = AMDGPU::INSTRUCTION_LIST_END; in copyPhysReg()
972 Opcode = AMDGPU::V_ACCVGPR_READ_B32_e64; in copyPhysReg()
978 Opcode = AMDGPU::V_MOV_B64_e32; in copyPhysReg()
981 Opcode = AMDGPU::V_PK_MOV_B32; in copyPhysReg()
992 if (Opcode == AMDGPU::INSTRUCTION_LIST_END) in copyPhysReg()
1010 if (Opcode == AMDGPU::INSTRUCTION_LIST_END) { in copyPhysReg()
1016 } else if (Opcode == AMDGPU::V_PK_MOV_B32) { in copyPhysReg()
1020 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), DstSubReg) in copyPhysReg()
1049 NewOpc = AMDGPU::getCommuteRev(Opcode); in commuteOpcode()
1055 NewOpc = AMDGPU::getCommuteOrig(Opcode); in commuteOpcode()
1069 if (RegClass == &AMDGPU::SReg_32RegClass || in materializeImmediate()
1070 RegClass == &AMDGPU::SGPR_32RegClass || in materializeImmediate()
1071 RegClass == &AMDGPU::SReg_32_XM0RegClass || in materializeImmediate()
1072 RegClass == &AMDGPU::SReg_32_XM0_XEXECRegClass) { in materializeImmediate()
1073 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg) in materializeImmediate()
1078 if (RegClass == &AMDGPU::SReg_64RegClass || in materializeImmediate()
1079 RegClass == &AMDGPU::SGPR_64RegClass || in materializeImmediate()
1080 RegClass == &AMDGPU::SReg_64_XEXECRegClass) { in materializeImmediate()
1081 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg) in materializeImmediate()
1086 if (RegClass == &AMDGPU::VGPR_32RegClass) { in materializeImmediate()
1087 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg) in materializeImmediate()
1091 if (RegClass->hasSuperClassEq(&AMDGPU::VReg_64RegClass)) { in materializeImmediate()
1092 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), DestReg) in materializeImmediate()
1098 unsigned Opcode = AMDGPU::V_MOV_B32_e32; in materializeImmediate()
1101 Opcode = AMDGPU::S_MOV_B64; in materializeImmediate()
1104 Opcode = AMDGPU::S_MOV_B32; in materializeImmediate()
1121 return &AMDGPU::VGPR_32RegClass; in getPreferredSelectRegClass()
1132 RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); in insertVectorSelect()
1133 assert(MRI.getRegClass(DstReg) == &AMDGPU::VGPR_32RegClass && in insertVectorSelect()
1138 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg) in insertVectorSelect()
1140 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
1151 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 in insertVectorSelect()
1152 : AMDGPU::S_CSELECT_B64), SReg) in insertVectorSelect()
1155 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
1165 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 in insertVectorSelect()
1166 : AMDGPU::S_CSELECT_B64), SReg) in insertVectorSelect()
1169 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
1181 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg) in insertVectorSelect()
1183 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
1195 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg) in insertVectorSelect()
1197 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
1208 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32 in insertVectorSelect()
1209 : AMDGPU::S_OR_SAVEEXEC_B64), SReg2) in insertVectorSelect()
1211 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 in insertVectorSelect()
1212 : AMDGPU::S_CSELECT_B64), SReg) in insertVectorSelect()
1215 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
1226 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32 in insertVectorSelect()
1227 : AMDGPU::S_OR_SAVEEXEC_B64), SReg2) in insertVectorSelect()
1229 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 in insertVectorSelect()
1230 : AMDGPU::S_CSELECT_B64), SReg) in insertVectorSelect()
1233 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
1256 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_EQ_I32_e64), Reg) in insertEQ()
1269 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_NE_I32_e64), Reg) in insertNE()
1279 return AMDGPU::COPY; in getMovOpcode()
1281 return RI.isSGPRClass(DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32; in getMovOpcode()
1283 return AMDGPU::S_MOV_B64; in getMovOpcode()
1285 return AMDGPU::V_MOV_B64_PSEUDO; in getMovOpcode()
1287 return AMDGPU::COPY; in getMovOpcode()
1295 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1); in getIndirectGPRIDXPseudo()
1297 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2); in getIndirectGPRIDXPseudo()
1299 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3); in getIndirectGPRIDXPseudo()
1301 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4); in getIndirectGPRIDXPseudo()
1303 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5); in getIndirectGPRIDXPseudo()
1305 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8); in getIndirectGPRIDXPseudo()
1307 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16); in getIndirectGPRIDXPseudo()
1309 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32); in getIndirectGPRIDXPseudo()
1315 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1); in getIndirectGPRIDXPseudo()
1317 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2); in getIndirectGPRIDXPseudo()
1319 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3); in getIndirectGPRIDXPseudo()
1321 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4); in getIndirectGPRIDXPseudo()
1323 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5); in getIndirectGPRIDXPseudo()
1325 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8); in getIndirectGPRIDXPseudo()
1327 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16); in getIndirectGPRIDXPseudo()
1329 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32); in getIndirectGPRIDXPseudo()
1336 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1; in getIndirectVGPRWriteMovRelPseudoOpc()
1338 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2; in getIndirectVGPRWriteMovRelPseudoOpc()
1340 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3; in getIndirectVGPRWriteMovRelPseudoOpc()
1342 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4; in getIndirectVGPRWriteMovRelPseudoOpc()
1344 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5; in getIndirectVGPRWriteMovRelPseudoOpc()
1346 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8; in getIndirectVGPRWriteMovRelPseudoOpc()
1348 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16; in getIndirectVGPRWriteMovRelPseudoOpc()
1350 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32; in getIndirectVGPRWriteMovRelPseudoOpc()
1357 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1; in getIndirectSGPRWriteMovRelPseudo32()
1359 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2; in getIndirectSGPRWriteMovRelPseudo32()
1361 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3; in getIndirectSGPRWriteMovRelPseudo32()
1363 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4; in getIndirectSGPRWriteMovRelPseudo32()
1365 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5; in getIndirectSGPRWriteMovRelPseudo32()
1367 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8; in getIndirectSGPRWriteMovRelPseudo32()
1369 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16; in getIndirectSGPRWriteMovRelPseudo32()
1371 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32; in getIndirectSGPRWriteMovRelPseudo32()
1378 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1; in getIndirectSGPRWriteMovRelPseudo64()
1380 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2; in getIndirectSGPRWriteMovRelPseudo64()
1382 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4; in getIndirectSGPRWriteMovRelPseudo64()
1384 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8; in getIndirectSGPRWriteMovRelPseudo64()
1386 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16; in getIndirectSGPRWriteMovRelPseudo64()
1412 return AMDGPU::SI_SPILL_S32_SAVE; in getSGPRSpillSaveOpcode()
1414 return AMDGPU::SI_SPILL_S64_SAVE; in getSGPRSpillSaveOpcode()
1416 return AMDGPU::SI_SPILL_S96_SAVE; in getSGPRSpillSaveOpcode()
1418 return AMDGPU::SI_SPILL_S128_SAVE; in getSGPRSpillSaveOpcode()
1420 return AMDGPU::SI_SPILL_S160_SAVE; in getSGPRSpillSaveOpcode()
1422 return AMDGPU::SI_SPILL_S192_SAVE; in getSGPRSpillSaveOpcode()
1424 return AMDGPU::SI_SPILL_S224_SAVE; in getSGPRSpillSaveOpcode()
1426 return AMDGPU::SI_SPILL_S256_SAVE; in getSGPRSpillSaveOpcode()
1428 return AMDGPU::SI_SPILL_S512_SAVE; in getSGPRSpillSaveOpcode()
1430 return AMDGPU::SI_SPILL_S1024_SAVE; in getSGPRSpillSaveOpcode()
1439 return AMDGPU::SI_SPILL_V32_SAVE; in getVGPRSpillSaveOpcode()
1441 return AMDGPU::SI_SPILL_V64_SAVE; in getVGPRSpillSaveOpcode()
1443 return AMDGPU::SI_SPILL_V96_SAVE; in getVGPRSpillSaveOpcode()
1445 return AMDGPU::SI_SPILL_V128_SAVE; in getVGPRSpillSaveOpcode()
1447 return AMDGPU::SI_SPILL_V160_SAVE; in getVGPRSpillSaveOpcode()
1449 return AMDGPU::SI_SPILL_V192_SAVE; in getVGPRSpillSaveOpcode()
1451 return AMDGPU::SI_SPILL_V224_SAVE; in getVGPRSpillSaveOpcode()
1453 return AMDGPU::SI_SPILL_V256_SAVE; in getVGPRSpillSaveOpcode()
1455 return AMDGPU::SI_SPILL_V512_SAVE; in getVGPRSpillSaveOpcode()
1457 return AMDGPU::SI_SPILL_V1024_SAVE; in getVGPRSpillSaveOpcode()
1466 return AMDGPU::SI_SPILL_A32_SAVE; in getAGPRSpillSaveOpcode()
1468 return AMDGPU::SI_SPILL_A64_SAVE; in getAGPRSpillSaveOpcode()
1470 return AMDGPU::SI_SPILL_A96_SAVE; in getAGPRSpillSaveOpcode()
1472 return AMDGPU::SI_SPILL_A128_SAVE; in getAGPRSpillSaveOpcode()
1474 return AMDGPU::SI_SPILL_A160_SAVE; in getAGPRSpillSaveOpcode()
1476 return AMDGPU::SI_SPILL_A192_SAVE; in getAGPRSpillSaveOpcode()
1478 return AMDGPU::SI_SPILL_A224_SAVE; in getAGPRSpillSaveOpcode()
1480 return AMDGPU::SI_SPILL_A256_SAVE; in getAGPRSpillSaveOpcode()
1482 return AMDGPU::SI_SPILL_A512_SAVE; in getAGPRSpillSaveOpcode()
1484 return AMDGPU::SI_SPILL_A1024_SAVE; in getAGPRSpillSaveOpcode()
1493 return AMDGPU::SI_SPILL_AV32_SAVE; in getAVSpillSaveOpcode()
1495 return AMDGPU::SI_SPILL_AV64_SAVE; in getAVSpillSaveOpcode()
1497 return AMDGPU::SI_SPILL_AV96_SAVE; in getAVSpillSaveOpcode()
1499 return AMDGPU::SI_SPILL_AV128_SAVE; in getAVSpillSaveOpcode()
1501 return AMDGPU::SI_SPILL_AV160_SAVE; in getAVSpillSaveOpcode()
1503 return AMDGPU::SI_SPILL_AV192_SAVE; in getAVSpillSaveOpcode()
1505 return AMDGPU::SI_SPILL_AV224_SAVE; in getAVSpillSaveOpcode()
1507 return AMDGPU::SI_SPILL_AV256_SAVE; in getAVSpillSaveOpcode()
1509 return AMDGPU::SI_SPILL_AV512_SAVE; in getAVSpillSaveOpcode()
1511 return AMDGPU::SI_SPILL_AV1024_SAVE; in getAVSpillSaveOpcode()
1538 assert(SrcReg != AMDGPU::M0 && "m0 should not be spilled"); in storeRegToStackSlot()
1539 assert(SrcReg != AMDGPU::EXEC_LO && SrcReg != AMDGPU::EXEC_HI && in storeRegToStackSlot()
1540 SrcReg != AMDGPU::EXEC && "exec should not be spilled"); in storeRegToStackSlot()
1549 MRI.constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0_XEXECRegClass); in storeRegToStackSlot()
1579 return AMDGPU::SI_SPILL_S32_RESTORE; in getSGPRSpillRestoreOpcode()
1581 return AMDGPU::SI_SPILL_S64_RESTORE; in getSGPRSpillRestoreOpcode()
1583 return AMDGPU::SI_SPILL_S96_RESTORE; in getSGPRSpillRestoreOpcode()
1585 return AMDGPU::SI_SPILL_S128_RESTORE; in getSGPRSpillRestoreOpcode()
1587 return AMDGPU::SI_SPILL_S160_RESTORE; in getSGPRSpillRestoreOpcode()
1589 return AMDGPU::SI_SPILL_S192_RESTORE; in getSGPRSpillRestoreOpcode()
1591 return AMDGPU::SI_SPILL_S224_RESTORE; in getSGPRSpillRestoreOpcode()
1593 return AMDGPU::SI_SPILL_S256_RESTORE; in getSGPRSpillRestoreOpcode()
1595 return AMDGPU::SI_SPILL_S512_RESTORE; in getSGPRSpillRestoreOpcode()
1597 return AMDGPU::SI_SPILL_S1024_RESTORE; in getSGPRSpillRestoreOpcode()
1606 return AMDGPU::SI_SPILL_V32_RESTORE; in getVGPRSpillRestoreOpcode()
1608 return AMDGPU::SI_SPILL_V64_RESTORE; in getVGPRSpillRestoreOpcode()
1610 return AMDGPU::SI_SPILL_V96_RESTORE; in getVGPRSpillRestoreOpcode()
1612 return AMDGPU::SI_SPILL_V128_RESTORE; in getVGPRSpillRestoreOpcode()
1614 return AMDGPU::SI_SPILL_V160_RESTORE; in getVGPRSpillRestoreOpcode()
1616 return AMDGPU::SI_SPILL_V192_RESTORE; in getVGPRSpillRestoreOpcode()
1618 return AMDGPU::SI_SPILL_V224_RESTORE; in getVGPRSpillRestoreOpcode()
1620 return AMDGPU::SI_SPILL_V256_RESTORE; in getVGPRSpillRestoreOpcode()
1622 return AMDGPU::SI_SPILL_V512_RESTORE; in getVGPRSpillRestoreOpcode()
1624 return AMDGPU::SI_SPILL_V1024_RESTORE; in getVGPRSpillRestoreOpcode()
1633 return AMDGPU::SI_SPILL_A32_RESTORE; in getAGPRSpillRestoreOpcode()
1635 return AMDGPU::SI_SPILL_A64_RESTORE; in getAGPRSpillRestoreOpcode()
1637 return AMDGPU::SI_SPILL_A96_RESTORE; in getAGPRSpillRestoreOpcode()
1639 return AMDGPU::SI_SPILL_A128_RESTORE; in getAGPRSpillRestoreOpcode()
1641 return AMDGPU::SI_SPILL_A160_RESTORE; in getAGPRSpillRestoreOpcode()
1643 return AMDGPU::SI_SPILL_A192_RESTORE; in getAGPRSpillRestoreOpcode()
1645 return AMDGPU::SI_SPILL_A224_RESTORE; in getAGPRSpillRestoreOpcode()
1647 return AMDGPU::SI_SPILL_A256_RESTORE; in getAGPRSpillRestoreOpcode()
1649 return AMDGPU::SI_SPILL_A512_RESTORE; in getAGPRSpillRestoreOpcode()
1651 return AMDGPU::SI_SPILL_A1024_RESTORE; in getAGPRSpillRestoreOpcode()
1660 return AMDGPU::SI_SPILL_AV32_RESTORE; in getAVSpillRestoreOpcode()
1662 return AMDGPU::SI_SPILL_AV64_RESTORE; in getAVSpillRestoreOpcode()
1664 return AMDGPU::SI_SPILL_AV96_RESTORE; in getAVSpillRestoreOpcode()
1666 return AMDGPU::SI_SPILL_AV128_RESTORE; in getAVSpillRestoreOpcode()
1668 return AMDGPU::SI_SPILL_AV160_RESTORE; in getAVSpillRestoreOpcode()
1670 return AMDGPU::SI_SPILL_AV192_RESTORE; in getAVSpillRestoreOpcode()
1672 return AMDGPU::SI_SPILL_AV224_RESTORE; in getAVSpillRestoreOpcode()
1674 return AMDGPU::SI_SPILL_AV256_RESTORE; in getAVSpillRestoreOpcode()
1676 return AMDGPU::SI_SPILL_AV512_RESTORE; in getAVSpillRestoreOpcode()
1678 return AMDGPU::SI_SPILL_AV1024_RESTORE; in getAVSpillRestoreOpcode()
1704 assert(DestReg != AMDGPU::M0 && "m0 should not be reloaded into"); in loadRegFromStackSlot()
1705 assert(DestReg != AMDGPU::EXEC_LO && DestReg != AMDGPU::EXEC_HI && in loadRegFromStackSlot()
1706 DestReg != AMDGPU::EXEC && "exec should not be spilled"); in loadRegFromStackSlot()
1713 MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0_XEXECRegClass); in loadRegFromStackSlot()
1749 BuildMI(MBB, MI, DL, get(AMDGPU::S_NOP)).addImm(Arg - 1); in insertNoops()
1763 BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::S_ENDPGM)).addImm(0); in insertReturn()
1765 BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::SI_RETURN_TO_EPILOG)); in insertReturn()
1778 case AMDGPU::S_NOP: in getNumWaitStates()
1791 case AMDGPU::S_MOV_B64_term: in expandPostRAPseudo()
1794 MI.setDesc(get(AMDGPU::S_MOV_B64)); in expandPostRAPseudo()
1797 case AMDGPU::S_MOV_B32_term: in expandPostRAPseudo()
1800 MI.setDesc(get(AMDGPU::S_MOV_B32)); in expandPostRAPseudo()
1803 case AMDGPU::S_XOR_B64_term: in expandPostRAPseudo()
1806 MI.setDesc(get(AMDGPU::S_XOR_B64)); in expandPostRAPseudo()
1809 case AMDGPU::S_XOR_B32_term: in expandPostRAPseudo()
1812 MI.setDesc(get(AMDGPU::S_XOR_B32)); in expandPostRAPseudo()
1814 case AMDGPU::S_OR_B64_term: in expandPostRAPseudo()
1817 MI.setDesc(get(AMDGPU::S_OR_B64)); in expandPostRAPseudo()
1819 case AMDGPU::S_OR_B32_term: in expandPostRAPseudo()
1822 MI.setDesc(get(AMDGPU::S_OR_B32)); in expandPostRAPseudo()
1825 case AMDGPU::S_ANDN2_B64_term: in expandPostRAPseudo()
1828 MI.setDesc(get(AMDGPU::S_ANDN2_B64)); in expandPostRAPseudo()
1831 case AMDGPU::S_ANDN2_B32_term: in expandPostRAPseudo()
1834 MI.setDesc(get(AMDGPU::S_ANDN2_B32)); in expandPostRAPseudo()
1837 case AMDGPU::S_AND_B64_term: in expandPostRAPseudo()
1840 MI.setDesc(get(AMDGPU::S_AND_B64)); in expandPostRAPseudo()
1843 case AMDGPU::S_AND_B32_term: in expandPostRAPseudo()
1846 MI.setDesc(get(AMDGPU::S_AND_B32)); in expandPostRAPseudo()
1849 case AMDGPU::V_MOV_B64_PSEUDO: { in expandPostRAPseudo()
1851 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0); in expandPostRAPseudo()
1852 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1); in expandPostRAPseudo()
1858 MI.setDesc(get(AMDGPU::V_MOV_B64_e32)); in expandPostRAPseudo()
1867 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), Dst) in expandPostRAPseudo()
1878 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo) in expandPostRAPseudo()
1881 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi) in expandPostRAPseudo()
1889 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), Dst) in expandPostRAPseudo()
1900 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo) in expandPostRAPseudo()
1901 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub0)) in expandPostRAPseudo()
1903 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi) in expandPostRAPseudo()
1904 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub1)) in expandPostRAPseudo()
1911 case AMDGPU::V_MOV_B64_DPP_PSEUDO: { in expandPostRAPseudo()
1915 case AMDGPU::S_MOV_B64_IMM_PSEUDO: { in expandPostRAPseudo()
1920 MI.setDesc(get(AMDGPU::S_MOV_B64)); in expandPostRAPseudo()
1925 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0); in expandPostRAPseudo()
1926 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1); in expandPostRAPseudo()
1930 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DstLo) in expandPostRAPseudo()
1933 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DstHi) in expandPostRAPseudo()
1939 case AMDGPU::V_SET_INACTIVE_B32: { in expandPostRAPseudo()
1940 unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64; in expandPostRAPseudo()
1941 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; in expandPostRAPseudo()
1944 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg()) in expandPostRAPseudo()
1947 FirstNot->addRegisterDead(AMDGPU::SCC, TRI); // SCC is overwritten in expandPostRAPseudo()
1948 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg()) in expandPostRAPseudo()
1955 case AMDGPU::V_SET_INACTIVE_B64: { in expandPostRAPseudo()
1956 unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64; in expandPostRAPseudo()
1957 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; in expandPostRAPseudo()
1958 MachineInstr *Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), in expandPostRAPseudo()
1963 FirstNot->addRegisterDead(AMDGPU::SCC, TRI); // SCC is overwritten in expandPostRAPseudo()
1964 Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), in expandPostRAPseudo()
1973 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1: in expandPostRAPseudo()
1974 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2: in expandPostRAPseudo()
1975 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3: in expandPostRAPseudo()
1976 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4: in expandPostRAPseudo()
1977 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5: in expandPostRAPseudo()
1978 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8: in expandPostRAPseudo()
1979 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16: in expandPostRAPseudo()
1980 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32: in expandPostRAPseudo()
1981 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1: in expandPostRAPseudo()
1982 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2: in expandPostRAPseudo()
1983 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3: in expandPostRAPseudo()
1984 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4: in expandPostRAPseudo()
1985 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5: in expandPostRAPseudo()
1986 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8: in expandPostRAPseudo()
1987 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16: in expandPostRAPseudo()
1988 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32: in expandPostRAPseudo()
1989 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1: in expandPostRAPseudo()
1990 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2: in expandPostRAPseudo()
1991 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4: in expandPostRAPseudo()
1992 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8: in expandPostRAPseudo()
1993 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16: { in expandPostRAPseudo()
1998 Opc = AMDGPU::V_MOVRELD_B32_e32; in expandPostRAPseudo()
2000 Opc = RI.getRegSizeInBits(*EltRC) == 64 ? AMDGPU::S_MOVRELD_B64 in expandPostRAPseudo()
2001 : AMDGPU::S_MOVRELD_B32; in expandPostRAPseudo()
2024 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1: in expandPostRAPseudo()
2025 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2: in expandPostRAPseudo()
2026 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3: in expandPostRAPseudo()
2027 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4: in expandPostRAPseudo()
2028 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5: in expandPostRAPseudo()
2029 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8: in expandPostRAPseudo()
2030 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16: in expandPostRAPseudo()
2031 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32: { in expandPostRAPseudo()
2038 MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON)) in expandPostRAPseudo()
2040 .addImm(AMDGPU::VGPRIndexMode::DST_ENABLE); in expandPostRAPseudo()
2043 const MCInstrDesc &OpDesc = get(AMDGPU::V_MOV_B32_indirect_write); in expandPostRAPseudo()
2056 MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF)); in expandPostRAPseudo()
2063 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1: in expandPostRAPseudo()
2064 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2: in expandPostRAPseudo()
2065 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3: in expandPostRAPseudo()
2066 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4: in expandPostRAPseudo()
2067 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5: in expandPostRAPseudo()
2068 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8: in expandPostRAPseudo()
2069 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16: in expandPostRAPseudo()
2070 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32: { in expandPostRAPseudo()
2078 MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON)) in expandPostRAPseudo()
2080 .addImm(AMDGPU::VGPRIndexMode::SRC0_ENABLE); in expandPostRAPseudo()
2083 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_indirect_read)) in expandPostRAPseudo()
2088 MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF)); in expandPostRAPseudo()
2095 case AMDGPU::SI_PC_ADD_REL_OFFSET: { in expandPostRAPseudo()
2098 Register RegLo = RI.getSubReg(Reg, AMDGPU::sub0); in expandPostRAPseudo()
2099 Register RegHi = RI.getSubReg(Reg, AMDGPU::sub1); in expandPostRAPseudo()
2104 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_GETPC_B64), Reg)); in expandPostRAPseudo()
2108 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_ADD_U32), RegLo) in expandPostRAPseudo()
2112 MachineInstrBuilder MIB = BuildMI(MF, DL, get(AMDGPU::S_ADDC_U32), RegHi) in expandPostRAPseudo()
2122 case AMDGPU::ENTER_STRICT_WWM: { in expandPostRAPseudo()
2125 MI.setDesc(get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32 in expandPostRAPseudo()
2126 : AMDGPU::S_OR_SAVEEXEC_B64)); in expandPostRAPseudo()
2129 case AMDGPU::ENTER_STRICT_WQM: { in expandPostRAPseudo()
2132 const unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; in expandPostRAPseudo()
2133 const unsigned WQMOp = ST.isWave32() ? AMDGPU::S_WQM_B32 : AMDGPU::S_WQM_B64; in expandPostRAPseudo()
2134 const unsigned MovOp = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; in expandPostRAPseudo()
2141 case AMDGPU::EXIT_STRICT_WWM: in expandPostRAPseudo()
2142 case AMDGPU::EXIT_STRICT_WQM: { in expandPostRAPseudo()
2145 MI.setDesc(get(ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64)); in expandPostRAPseudo()
2148 case AMDGPU::SI_RETURN: { in expandPostRAPseudo()
2158 BuildMI(MBB, MI, DL, get(AMDGPU::S_SETPC_B64_return)) in expandPostRAPseudo()
2171 assert (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO); in expandMovDPP64()
2174 AMDGPU::isLegal64BitDPPControl( in expandMovDPP64()
2175 getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl)->getImm())) { in expandMovDPP64()
2176 MI.setDesc(get(AMDGPU::V_MOV_B64_dpp)); in expandMovDPP64()
2188 for (auto Sub : { AMDGPU::sub0, AMDGPU::sub1 }) { in expandMovDPP64()
2189 auto MovDPP = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_dpp)); in expandMovDPP64()
2194 auto Tmp = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in expandMovDPP64()
2223 BuildMI(MBB, MI, DL, get(AMDGPU::REG_SEQUENCE), Dst) in expandMovDPP64()
2225 .addImm(AMDGPU::sub0) in expandMovDPP64()
2227 .addImm(AMDGPU::sub1); in expandMovDPP64()
2293 assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) == in commuteInstructionImpl()
2295 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) == in commuteInstructionImpl()
2323 swapSourceModifiers(MI, Src0, AMDGPU::OpName::src0_modifiers, in commuteInstructionImpl()
2324 Src1, AMDGPU::OpName::src1_modifiers); in commuteInstructionImpl()
2347 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); in findCommutedOpIndices()
2351 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); in findCommutedOpIndices()
2362 assert(BranchOp != AMDGPU::S_SETPC_B64); in isBranchOffsetInRange()
2376 if (MI.getOpcode() == AMDGPU::S_SETPC_B64) { in getBranchDestBlock()
2402 Register PCReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in insertIndirectBranch()
2408 MachineInstr *GetPC = BuildMI(MBB, I, DL, get(AMDGPU::S_GETPC_B64), PCReg); in insertIndirectBranch()
2419 BuildMI(MBB, I, DL, get(AMDGPU::S_ADD_U32)) in insertIndirectBranch()
2420 .addReg(PCReg, RegState::Define, AMDGPU::sub0) in insertIndirectBranch()
2421 .addReg(PCReg, 0, AMDGPU::sub0) in insertIndirectBranch()
2423 BuildMI(MBB, I, DL, get(AMDGPU::S_ADDC_U32)) in insertIndirectBranch()
2424 .addReg(PCReg, RegState::Define, AMDGPU::sub1) in insertIndirectBranch()
2425 .addReg(PCReg, 0, AMDGPU::sub1) in insertIndirectBranch()
2429 BuildMI(&MBB, DL, get(AMDGPU::S_SETPC_B64)) in insertIndirectBranch()
2470 AMDGPU::SReg_64RegClass, MachineBasicBlock::iterator(GetPC), in insertIndirectBranch()
2481 TRI->spillEmergencySGPR(GetPC, RestoreBB, AMDGPU::SGPR0_SGPR1, RS); in insertIndirectBranch()
2482 MRI.replaceRegWith(PCReg, AMDGPU::SGPR0_SGPR1); in insertIndirectBranch()
2501 return AMDGPU::S_CBRANCH_SCC1; in getBranchOpcode()
2503 return AMDGPU::S_CBRANCH_SCC0; in getBranchOpcode()
2505 return AMDGPU::S_CBRANCH_VCCNZ; in getBranchOpcode()
2507 return AMDGPU::S_CBRANCH_VCCZ; in getBranchOpcode()
2509 return AMDGPU::S_CBRANCH_EXECNZ; in getBranchOpcode()
2511 return AMDGPU::S_CBRANCH_EXECZ; in getBranchOpcode()
2519 case AMDGPU::S_CBRANCH_SCC0: in getBranchPredicate()
2521 case AMDGPU::S_CBRANCH_SCC1: in getBranchPredicate()
2523 case AMDGPU::S_CBRANCH_VCCNZ: in getBranchPredicate()
2525 case AMDGPU::S_CBRANCH_VCCZ: in getBranchPredicate()
2527 case AMDGPU::S_CBRANCH_EXECNZ: in getBranchPredicate()
2529 case AMDGPU::S_CBRANCH_EXECZ: in getBranchPredicate()
2542 if (I->getOpcode() == AMDGPU::S_BRANCH) { in analyzeBranchImpl()
2550 if (I->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) { in analyzeBranchImpl()
2570 if (I->getOpcode() == AMDGPU::S_BRANCH) { in analyzeBranchImpl()
2592 case AMDGPU::S_MOV_B64_term: in analyzeBranch()
2593 case AMDGPU::S_XOR_B64_term: in analyzeBranch()
2594 case AMDGPU::S_OR_B64_term: in analyzeBranch()
2595 case AMDGPU::S_ANDN2_B64_term: in analyzeBranch()
2596 case AMDGPU::S_AND_B64_term: in analyzeBranch()
2597 case AMDGPU::S_MOV_B32_term: in analyzeBranch()
2598 case AMDGPU::S_XOR_B32_term: in analyzeBranch()
2599 case AMDGPU::S_OR_B32_term: in analyzeBranch()
2600 case AMDGPU::S_ANDN2_B32_term: in analyzeBranch()
2601 case AMDGPU::S_AND_B32_term: in analyzeBranch()
2603 case AMDGPU::SI_IF: in analyzeBranch()
2604 case AMDGPU::SI_ELSE: in analyzeBranch()
2605 case AMDGPU::SI_KILL_I1_TERMINATOR: in analyzeBranch()
2606 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR: in analyzeBranch()
2655 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH)) in insertBranch()
2663 BuildMI(&MBB, DL, get(AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO)) in insertBranch()
2695 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH)) in insertBranch()
2735 int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32; in canInsertSelect()
2750 int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32; in canInsertSelect()
2781 Select = BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B32), DstReg) in insertSelect()
2786 Select = BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e32), DstReg) in insertSelect()
2797 BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), DstReg) in insertSelect()
2806 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3, in insertSelect()
2807 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7, in insertSelect()
2808 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11, in insertSelect()
2809 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15, in insertSelect()
2813 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3, in insertSelect()
2814 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7, in insertSelect()
2815 AMDGPU::sub8_sub9, AMDGPU::sub10_sub11, in insertSelect()
2816 AMDGPU::sub12_sub13, AMDGPU::sub14_sub15, in insertSelect()
2819 unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32; in insertSelect()
2820 const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass; in insertSelect()
2828 SelOp = AMDGPU::S_CSELECT_B32; in insertSelect()
2829 EltRC = &AMDGPU::SGPR_32RegClass; in insertSelect()
2831 SelOp = AMDGPU::S_CSELECT_B64; in insertSelect()
2832 EltRC = &AMDGPU::SGPR_64RegClass; in insertSelect()
2839 MBB, I, DL, get(AMDGPU::REG_SEQUENCE), DstReg); in insertSelect()
2851 if (SelOp == AMDGPU::V_CNDMASK_B32_e32) { in insertSelect()
2873 case AMDGPU::V_MOV_B32_e32: in isFoldableCopy()
2874 case AMDGPU::V_MOV_B32_e64: in isFoldableCopy()
2875 case AMDGPU::V_MOV_B64_PSEUDO: in isFoldableCopy()
2876 case AMDGPU::V_MOV_B64_e32: in isFoldableCopy()
2877 case AMDGPU::V_MOV_B64_e64: in isFoldableCopy()
2878 case AMDGPU::S_MOV_B32: in isFoldableCopy()
2879 case AMDGPU::S_MOV_B64: in isFoldableCopy()
2880 case AMDGPU::COPY: in isFoldableCopy()
2881 case AMDGPU::V_ACCVGPR_WRITE_B32_e64: in isFoldableCopy()
2882 case AMDGPU::V_ACCVGPR_READ_B32_e64: in isFoldableCopy()
2883 case AMDGPU::V_ACCVGPR_MOV_B32: in isFoldableCopy()
2891 AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src1_modifiers,
2892 AMDGPU::OpName::src2_modifiers, AMDGPU::OpName::clamp,
2893 AMDGPU::OpName::omod};
2898 MI.removeOperand(AMDGPU::getNamedOperandIdx(Opc, Name)); in removeModOperands()
2909 case AMDGPU::S_MOV_B64: in FoldImmediate()
2914 case AMDGPU::V_MOV_B32_e32: in FoldImmediate()
2915 case AMDGPU::S_MOV_B32: in FoldImmediate()
2916 case AMDGPU::V_ACCVGPR_WRITE_B32_e64: in FoldImmediate()
2920 const MachineOperand *ImmOp = getNamedOperand(DefMI, AMDGPU::OpName::src0); in FoldImmediate()
2927 if (Opc == AMDGPU::COPY) { in FoldImmediate()
2931 unsigned NewOpc = isVGPRCopy ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32; in FoldImmediate()
2934 if (UseMI.getOperand(1).getSubReg() == AMDGPU::hi16) in FoldImmediate()
2940 NewOpc = AMDGPU::V_ACCVGPR_WRITE_B32_e64; in FoldImmediate()
2947 if (DstReg.isVirtual() && UseMI.getOperand(0).getSubReg() != AMDGPU::lo16) in FoldImmediate()
2964 if (Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 || in FoldImmediate()
2965 Opc == AMDGPU::V_MAD_F16_e64 || Opc == AMDGPU::V_MAC_F16_e64 || in FoldImmediate()
2966 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 || in FoldImmediate()
2967 Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64) { in FoldImmediate()
2976 MachineOperand *Src0 = getNamedOperand(UseMI, AMDGPU::OpName::src0); in FoldImmediate()
2982 bool IsF32 = Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 || in FoldImmediate()
2983 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64; in FoldImmediate()
2984 bool IsFMA = Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 || in FoldImmediate()
2985 Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64; in FoldImmediate()
2986 MachineOperand *Src1 = getNamedOperand(UseMI, AMDGPU::OpName::src1); in FoldImmediate()
2987 MachineOperand *Src2 = getNamedOperand(UseMI, AMDGPU::OpName::src2); in FoldImmediate()
2999 IsFMA ? (IsF32 ? AMDGPU::V_FMAMK_F32 : AMDGPU::V_FMAMK_F16) in FoldImmediate()
3000 : (IsF32 ? AMDGPU::V_MADMK_F32 : AMDGPU::V_MADMK_F16); in FoldImmediate()
3017 if (Opc == AMDGPU::V_MAC_F32_e64 || in FoldImmediate()
3018 Opc == AMDGPU::V_MAC_F16_e64 || in FoldImmediate()
3019 Opc == AMDGPU::V_FMAC_F32_e64 || in FoldImmediate()
3020 Opc == AMDGPU::V_FMAC_F16_e64) in FoldImmediate()
3022 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)); in FoldImmediate()
3078 IsFMA ? (IsF32 ? AMDGPU::V_FMAAK_F32 : AMDGPU::V_FMAAK_F16) in FoldImmediate()
3079 : (IsF32 ? AMDGPU::V_MADAK_F32 : AMDGPU::V_MADAK_F16); in FoldImmediate()
3088 if (Opc == AMDGPU::V_MAC_F32_e64 || in FoldImmediate()
3089 Opc == AMDGPU::V_MAC_F16_e64 || in FoldImmediate()
3090 Opc == AMDGPU::V_FMAC_F32_e64 || in FoldImmediate()
3091 Opc == AMDGPU::V_FMAC_F16_e64) in FoldImmediate()
3093 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)); in FoldImmediate()
3253 int NewMFMAOpc = AMDGPU::getMFMAEarlyClobberOp(Opc); in convertToThreeAddress()
3266 unsigned NewOpc = AMDGPU::mapWMMA2AddrTo3AddrOpcode(MI.getOpcode()); in convertToThreeAddress()
3280 bool IsF16 = Opc == AMDGPU::V_MAC_F16_e32 || Opc == AMDGPU::V_MAC_F16_e64 || in convertToThreeAddress()
3281 Opc == AMDGPU::V_FMAC_F16_e32 || Opc == AMDGPU::V_FMAC_F16_e64; in convertToThreeAddress()
3282 bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F32_e64 || in convertToThreeAddress()
3283 Opc == AMDGPU::V_FMAC_LEGACY_F32_e32 || in convertToThreeAddress()
3284 Opc == AMDGPU::V_FMAC_LEGACY_F32_e64 || in convertToThreeAddress()
3285 Opc == AMDGPU::V_FMAC_F16_e32 || Opc == AMDGPU::V_FMAC_F16_e64 || in convertToThreeAddress()
3286 Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64; in convertToThreeAddress()
3287 bool IsF64 = Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64; in convertToThreeAddress()
3288 bool IsLegacy = Opc == AMDGPU::V_MAC_LEGACY_F32_e32 || in convertToThreeAddress()
3289 Opc == AMDGPU::V_MAC_LEGACY_F32_e64 || in convertToThreeAddress()
3290 Opc == AMDGPU::V_FMAC_LEGACY_F32_e32 || in convertToThreeAddress()
3291 Opc == AMDGPU::V_FMAC_LEGACY_F32_e64; in convertToThreeAddress()
3297 case AMDGPU::V_MAC_F16_e64: in convertToThreeAddress()
3298 case AMDGPU::V_FMAC_F16_e64: in convertToThreeAddress()
3299 case AMDGPU::V_MAC_F32_e64: in convertToThreeAddress()
3300 case AMDGPU::V_MAC_LEGACY_F32_e64: in convertToThreeAddress()
3301 case AMDGPU::V_FMAC_F32_e64: in convertToThreeAddress()
3302 case AMDGPU::V_FMAC_LEGACY_F32_e64: in convertToThreeAddress()
3303 case AMDGPU::V_FMAC_F64_e64: in convertToThreeAddress()
3305 case AMDGPU::V_MAC_F16_e32: in convertToThreeAddress()
3306 case AMDGPU::V_FMAC_F16_e32: in convertToThreeAddress()
3307 case AMDGPU::V_MAC_F32_e32: in convertToThreeAddress()
3308 case AMDGPU::V_MAC_LEGACY_F32_e32: in convertToThreeAddress()
3309 case AMDGPU::V_FMAC_F32_e32: in convertToThreeAddress()
3310 case AMDGPU::V_FMAC_LEGACY_F32_e32: in convertToThreeAddress()
3311 case AMDGPU::V_FMAC_F64_e32: { in convertToThreeAddress()
3312 int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), in convertToThreeAddress()
3313 AMDGPU::OpName::src0); in convertToThreeAddress()
3326 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst); in convertToThreeAddress()
3327 const MachineOperand *Src0 = getNamedOperand(MI, AMDGPU::OpName::src0); in convertToThreeAddress()
3329 getNamedOperand(MI, AMDGPU::OpName::src0_modifiers); in convertToThreeAddress()
3330 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1); in convertToThreeAddress()
3332 getNamedOperand(MI, AMDGPU::OpName::src1_modifiers); in convertToThreeAddress()
3333 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); in convertToThreeAddress()
3335 getNamedOperand(MI, AMDGPU::OpName::src2_modifiers); in convertToThreeAddress()
3336 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp); in convertToThreeAddress()
3337 const MachineOperand *Omod = getNamedOperand(MI, AMDGPU::OpName::omod); in convertToThreeAddress()
3352 DefMI->setDesc(get(AMDGPU::IMPLICIT_DEF)); in convertToThreeAddress()
3362 IsFMA ? (IsF16 ? AMDGPU::V_FMAAK_F16 : AMDGPU::V_FMAAK_F32) in convertToThreeAddress()
3363 : (IsF16 ? AMDGPU::V_MADAK_F16 : AMDGPU::V_MADAK_F32); in convertToThreeAddress()
3378 ? (IsF16 ? AMDGPU::V_FMAMK_F16 : AMDGPU::V_FMAMK_F32) in convertToThreeAddress()
3379 : (IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32); in convertToThreeAddress()
3401 MI, AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::src0), in convertToThreeAddress()
3424 unsigned NewOpc = IsFMA ? IsF16 ? AMDGPU::V_FMA_F16_gfx9_e64 in convertToThreeAddress()
3425 : IsF64 ? AMDGPU::V_FMA_F64_e64 in convertToThreeAddress()
3427 ? AMDGPU::V_FMA_LEGACY_F32_e64 in convertToThreeAddress()
3428 : AMDGPU::V_FMA_F32_e64 in convertToThreeAddress()
3429 : IsF16 ? AMDGPU::V_MAD_F16_e64 in convertToThreeAddress()
3430 : IsLegacy ? AMDGPU::V_MAD_LEGACY_F32_e64 in convertToThreeAddress()
3431 : AMDGPU::V_MAD_F32_e64; in convertToThreeAddress()
3456 case AMDGPU::S_SET_GPR_IDX_ON: in changesVGPRIndexingMode()
3457 case AMDGPU::S_SET_GPR_IDX_MODE: in changesVGPRIndexingMode()
3458 case AMDGPU::S_SET_GPR_IDX_OFF: in changesVGPRIndexingMode()
3483 if (MI.getOpcode() == AMDGPU::SCHED_BARRIER && MI.getOperand(0).getImm() == 0) in isSchedulingBoundary()
3489 return MI.modifiesRegister(AMDGPU::EXEC, &RI) || in isSchedulingBoundary()
3490 MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 || in isSchedulingBoundary()
3491 MI.getOpcode() == AMDGPU::S_SETREG_B32 || in isSchedulingBoundary()
3496 return Opcode == AMDGPU::DS_ORDERED_COUNT || in isAlwaysGDS()
3497 Opcode == AMDGPU::DS_GWS_INIT || in isAlwaysGDS()
3498 Opcode == AMDGPU::DS_GWS_SEMA_V || in isAlwaysGDS()
3499 Opcode == AMDGPU::DS_GWS_SEMA_BR || in isAlwaysGDS()
3500 Opcode == AMDGPU::DS_GWS_SEMA_P || in isAlwaysGDS()
3501 Opcode == AMDGPU::DS_GWS_SEMA_RELEASE_ALL || in isAlwaysGDS()
3502 Opcode == AMDGPU::DS_GWS_BARRIER; in isAlwaysGDS()
3511 if (*ImpDef == AMDGPU::MODE) in modifiesModeRegister()
3535 if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT || in hasUnwantedEffectsWhenEXECEmpty()
3537 Opcode == AMDGPU::DS_ORDERED_COUNT || Opcode == AMDGPU::S_TRAP || in hasUnwantedEffectsWhenEXECEmpty()
3538 Opcode == AMDGPU::DS_GWS_INIT || Opcode == AMDGPU::DS_GWS_BARRIER) in hasUnwantedEffectsWhenEXECEmpty()
3553 if (Opcode == AMDGPU::V_READFIRSTLANE_B32 || in hasUnwantedEffectsWhenEXECEmpty()
3554 Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32) in hasUnwantedEffectsWhenEXECEmpty()
3571 return MI.readsRegister(AMDGPU::EXEC, &RI); in mayReadEXEC()
3582 return !isSALU(MI) || MI.readsRegister(AMDGPU::EXEC, &RI); in mayReadEXEC()
3591 return AMDGPU::isInlinableLiteral32(Imm.getSExtValue(), in isInlineConstant()
3594 return AMDGPU::isInlinableLiteral64(Imm.getSExtValue(), in isInlineConstant()
3598 AMDGPU::isInlinableLiteral16(Imm.getSExtValue(), in isInlineConstant()
3608 OperandType < AMDGPU::OPERAND_SRC_FIRST || in isInlineConstant()
3609 OperandType > AMDGPU::OPERAND_SRC_LAST) in isInlineConstant()
3619 case AMDGPU::OPERAND_REG_IMM_INT32: in isInlineConstant()
3620 case AMDGPU::OPERAND_REG_IMM_FP32: in isInlineConstant()
3621 case AMDGPU::OPERAND_REG_IMM_FP32_DEFERRED: in isInlineConstant()
3622 case AMDGPU::OPERAND_REG_INLINE_C_INT32: in isInlineConstant()
3623 case AMDGPU::OPERAND_REG_INLINE_C_FP32: in isInlineConstant()
3624 case AMDGPU::OPERAND_REG_IMM_V2FP32: in isInlineConstant()
3625 case AMDGPU::OPERAND_REG_INLINE_C_V2FP32: in isInlineConstant()
3626 case AMDGPU::OPERAND_REG_IMM_V2INT32: in isInlineConstant()
3627 case AMDGPU::OPERAND_REG_INLINE_C_V2INT32: in isInlineConstant()
3628 case AMDGPU::OPERAND_REG_INLINE_AC_INT32: in isInlineConstant()
3629 case AMDGPU::OPERAND_REG_INLINE_AC_FP32: { in isInlineConstant()
3631 return AMDGPU::isInlinableLiteral32(Trunc, ST.hasInv2PiInlineImm()); in isInlineConstant()
3633 case AMDGPU::OPERAND_REG_IMM_INT64: in isInlineConstant()
3634 case AMDGPU::OPERAND_REG_IMM_FP64: in isInlineConstant()
3635 case AMDGPU::OPERAND_REG_INLINE_C_INT64: in isInlineConstant()
3636 case AMDGPU::OPERAND_REG_INLINE_C_FP64: in isInlineConstant()
3637 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: in isInlineConstant()
3638 return AMDGPU::isInlinableLiteral64(MO.getImm(), in isInlineConstant()
3640 case AMDGPU::OPERAND_REG_IMM_INT16: in isInlineConstant()
3641 case AMDGPU::OPERAND_REG_INLINE_C_INT16: in isInlineConstant()
3642 case AMDGPU::OPERAND_REG_INLINE_AC_INT16: in isInlineConstant()
3653 return AMDGPU::isInlinableIntLiteral(Imm); in isInlineConstant()
3654 case AMDGPU::OPERAND_REG_IMM_V2INT16: in isInlineConstant()
3655 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16: in isInlineConstant()
3656 case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16: in isInlineConstant()
3658 return AMDGPU::isInlinableIntLiteralV216(Imm); in isInlineConstant()
3659 case AMDGPU::OPERAND_REG_IMM_FP16: in isInlineConstant()
3660 case AMDGPU::OPERAND_REG_IMM_FP16_DEFERRED: in isInlineConstant()
3661 case AMDGPU::OPERAND_REG_INLINE_C_FP16: in isInlineConstant()
3662 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: { in isInlineConstant()
3670 AMDGPU::isInlinableLiteral16(Trunc, ST.hasInv2PiInlineImm()); in isInlineConstant()
3675 case AMDGPU::OPERAND_REG_IMM_V2FP16: in isInlineConstant()
3676 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: in isInlineConstant()
3677 case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16: { in isInlineConstant()
3679 return AMDGPU::isInlinableLiteralV216(Trunc, ST.hasInv2PiInlineImm()); in isInlineConstant()
3681 case AMDGPU::OPERAND_KIMM32: in isInlineConstant()
3682 case AMDGPU::OPERAND_KIMM16: in isInlineConstant()
3737 OpNo ==(unsigned)AMDGPU::getNamedOperandIdx(MI.getOpcode(), in isImmOperandLegal()
3738 AMDGPU::OpName::src2)) in isImmOperandLegal()
3746 if (!isVOP3(MI) || !AMDGPU::isSISrcOperand(InstDesc, OpNo)) in isImmOperandLegal()
3754 if (Opcode == AMDGPU::V_MUL_LEGACY_F32_e64 && ST.hasGFX90AInsts()) in hasVALU32BitEncoding()
3757 int Op32 = AMDGPU::getVOPe32(Opcode); in hasVALU32BitEncoding()
3768 return AMDGPU::getNamedOperandIdx(Opcode, in hasModifiers()
3769 AMDGPU::OpName::src0_modifiers) != -1; in hasModifiers()
3785 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); in canShrink()
3791 case AMDGPU::V_ADDC_U32_e64: in canShrink()
3792 case AMDGPU::V_SUBB_U32_e64: in canShrink()
3793 case AMDGPU::V_SUBBREV_U32_e64: { in canShrink()
3795 = getNamedOperand(MI, AMDGPU::OpName::src1); in canShrink()
3801 case AMDGPU::V_MAC_F16_e64: in canShrink()
3802 case AMDGPU::V_MAC_F32_e64: in canShrink()
3803 case AMDGPU::V_MAC_LEGACY_F32_e64: in canShrink()
3804 case AMDGPU::V_FMAC_F16_e64: in canShrink()
3805 case AMDGPU::V_FMAC_F32_e64: in canShrink()
3806 case AMDGPU::V_FMAC_F64_e64: in canShrink()
3807 case AMDGPU::V_FMAC_LEGACY_F32_e64: in canShrink()
3809 hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers)) in canShrink()
3813 case AMDGPU::V_CNDMASK_B32_e64: in canShrink()
3818 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1); in canShrink()
3820 hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers))) in canShrink()
3825 if (hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers)) in canShrink()
3833 return !hasModifiersSet(MI, AMDGPU::OpName::omod) && in canShrink()
3834 !hasModifiersSet(MI, AMDGPU::OpName::clamp); in canShrink()
3844 (Use.getReg() == AMDGPU::VCC || Use.getReg() == AMDGPU::VCC_LO)) { in copyFlagsToImplicitVCC()
3861 if (AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::vdst) != -1) { in buildShrunkInst()
3864 } else if (AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::sdst) != -1) { in buildShrunkInst()
3867 assert(((MI.getOperand(0).getReg() == AMDGPU::VCC) || in buildShrunkInst()
3868 (MI.getOperand(0).getReg() == AMDGPU::VCC_LO)) && in buildShrunkInst()
3872 Inst32.add(*getNamedOperand(MI, AMDGPU::OpName::src0)); in buildShrunkInst()
3874 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1); in buildShrunkInst()
3878 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); in buildShrunkInst()
3881 int Op32Src2Idx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::src2); in buildShrunkInst()
3917 if (MO.getReg() == AMDGPU::SGPR_NULL || MO.getReg() == AMDGPU::SGPR_NULL64) in usesConstantBus()
3922 return MO.getReg() == AMDGPU::M0 || in usesConstantBus()
3923 MO.getReg() == AMDGPU::VCC || in usesConstantBus()
3924 MO.getReg() == AMDGPU::VCC_LO; in usesConstantBus()
3926 return AMDGPU::SReg_32RegClass.contains(MO.getReg()) || in usesConstantBus()
3927 AMDGPU::SReg_64RegClass.contains(MO.getReg()); in usesConstantBus()
3938 case AMDGPU::VCC: in findImplicitSGPRRead()
3939 case AMDGPU::VCC_LO: in findImplicitSGPRRead()
3940 case AMDGPU::VCC_HI: in findImplicitSGPRRead()
3941 case AMDGPU::M0: in findImplicitSGPRRead()
3942 case AMDGPU::FLAT_SCR: in findImplicitSGPRRead()
3950 return AMDGPU::NoRegister; in findImplicitSGPRRead()
3956 case AMDGPU::V_READLANE_B32: in shouldReadExec()
3957 case AMDGPU::V_WRITELANE_B32: in shouldReadExec()
3979 return SubReg.getSubReg() != AMDGPU::NoSubRegister && in isSubRegOf()
3992 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0); in verifyInstruction()
3993 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1); in verifyInstruction()
3994 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2); in verifyInstruction()
3998 Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0X); in verifyInstruction()
3999 Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vsrc1X); in verifyInstruction()
4000 Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0Y); in verifyInstruction()
4001 Src3Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vsrc1Y); in verifyInstruction()
4057 case AMDGPU::OPERAND_REG_IMM_INT32: in verifyInstruction()
4058 case AMDGPU::OPERAND_REG_IMM_FP32: in verifyInstruction()
4059 case AMDGPU::OPERAND_REG_IMM_FP32_DEFERRED: in verifyInstruction()
4060 case AMDGPU::OPERAND_REG_IMM_V2FP32: in verifyInstruction()
4062 case AMDGPU::OPERAND_REG_INLINE_C_INT32: in verifyInstruction()
4063 case AMDGPU::OPERAND_REG_INLINE_C_FP32: in verifyInstruction()
4064 case AMDGPU::OPERAND_REG_INLINE_C_INT64: in verifyInstruction()
4065 case AMDGPU::OPERAND_REG_INLINE_C_FP64: in verifyInstruction()
4066 case AMDGPU::OPERAND_REG_INLINE_C_INT16: in verifyInstruction()
4067 case AMDGPU::OPERAND_REG_INLINE_C_FP16: in verifyInstruction()
4068 case AMDGPU::OPERAND_REG_INLINE_AC_INT32: in verifyInstruction()
4069 case AMDGPU::OPERAND_REG_INLINE_AC_FP32: in verifyInstruction()
4070 case AMDGPU::OPERAND_REG_INLINE_AC_INT16: in verifyInstruction()
4071 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: in verifyInstruction()
4072 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: { in verifyInstruction()
4080 case AMDGPU::OPERAND_KIMM32: in verifyInstruction()
4139 int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst); in verifyInstruction()
4164 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod); in verifyInstruction()
4172 uint16_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode); in verifyInstruction()
4177 if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) { in verifyInstruction()
4183 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp); in verifyInstruction()
4190 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod); in verifyInstruction()
4198 const MachineOperand *DstUnused = getNamedOperand(MI, AMDGPU::OpName::dst_unused); in verifyInstruction()
4200 DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE) { in verifyInstruction()
4225 const MachineOperand *DMask = getNamedOperand(MI, AMDGPU::OpName::dmask); in verifyInstruction()
4230 const MachineOperand *TFE = getNamedOperand(MI, AMDGPU::OpName::tfe); in verifyInstruction()
4231 const MachineOperand *LWE = getNamedOperand(MI, AMDGPU::OpName::lwe); in verifyInstruction()
4232 const MachineOperand *D16 = getNamedOperand(MI, AMDGPU::OpName::d16); in verifyInstruction()
4243 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata); in verifyInstruction()
4258 if (isVALU(MI) && Desc.getOpcode() != AMDGPU::V_WRITELANE_B32) { in verifyInstruction()
4263 int ImmIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm); in verifyInstruction()
4304 if (SGPRUsed != AMDGPU::NoRegister) { in verifyInstruction()
4317 Opcode != AMDGPU::V_WRITELANE_B32) { in verifyInstruction()
4330 if (Desc.getOpcode() == AMDGPU::V_WRITELANE_B32) { in verifyInstruction()
4332 Register SGPRUsed = AMDGPU::NoRegister; in verifyInstruction()
4341 if (MO.isReg() && MO.getReg() != AMDGPU::M0) { in verifyInstruction()
4355 if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32_e64 || in verifyInstruction()
4356 Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64_e64) { in verifyInstruction()
4367 if ((getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)->getImm() & in verifyInstruction()
4369 (getNamedOperand(MI, AMDGPU::OpName::src1_modifiers)->getImm() & in verifyInstruction()
4371 (getNamedOperand(MI, AMDGPU::OpName::src2_modifiers)->getImm() & in verifyInstruction()
4392 auto Op = getNamedOperand(MI, AMDGPU::OpName::simm16); in verifyInstruction()
4414 if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 || in verifyInstruction()
4415 Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 || in verifyInstruction()
4416 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 || in verifyInstruction()
4417 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) { in verifyInstruction()
4418 const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 || in verifyInstruction()
4419 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64; in verifyInstruction()
4433 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst); in verifyInstruction()
4461 if (!MI.hasRegisterImplicitUseOperand(AMDGPU::EXEC)) { in verifyInstruction()
4472 const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soffset); in verifyInstruction()
4473 if (Soff && Soff->getReg() != AMDGPU::M0) { in verifyInstruction()
4481 const MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset); in verifyInstruction()
4489 const MachineOperand *DimOp = getNamedOperand(MI, AMDGPU::OpName::dim); in verifyInstruction()
4491 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode, in verifyInstruction()
4492 AMDGPU::OpName::vaddr0); in verifyInstruction()
4493 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc); in verifyInstruction()
4494 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opcode); in verifyInstruction()
4495 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode = in verifyInstruction()
4496 AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode); in verifyInstruction()
4497 const AMDGPU::MIMGDimInfo *Dim = in verifyInstruction()
4498 AMDGPU::getMIMGDimInfoByEncoding(DimOp->getImm()); in verifyInstruction()
4507 const MachineOperand *R128A16 = getNamedOperand(MI, AMDGPU::OpName::r128); in verifyInstruction()
4510 const MachineOperand *A16 = getNamedOperand(MI, AMDGPU::OpName::a16); in verifyInstruction()
4517 AMDGPU::getAddrSizeMIMGOp(BaseOpcode, Dim, IsA16, ST.hasG16()); in verifyInstruction()
4538 const MachineOperand *DppCt = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl); in verifyInstruction()
4540 using namespace AMDGPU::DPP; in verifyInstruction()
4581 int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst); in verifyInstruction()
4583 if (Opcode != AMDGPU::V_MOV_B64_DPP_PSEUDO && in verifyInstruction()
4585 (Desc.OpInfo[DstIdx].RegClass == AMDGPU::VReg_64RegClassID || in verifyInstruction()
4586 Desc.OpInfo[DstIdx].RegClass == AMDGPU::VReg_64_Align2RegClassID)) || in verifyInstruction()
4588 (Desc.OpInfo[Src0Idx].RegClass == AMDGPU::VReg_64RegClassID || in verifyInstruction()
4590 AMDGPU::VReg_64_Align2RegClassID)))) && in verifyInstruction()
4591 !AMDGPU::isLegal64BitDPPControl(DC)) { in verifyInstruction()
4599 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst); in verifyInstruction()
4600 uint16_t DataNameIdx = isDS(Opcode) ? AMDGPU::OpName::data0 in verifyInstruction()
4601 : AMDGPU::OpName::vdata; in verifyInstruction()
4603 const MachineOperand *Data2 = getNamedOperand(MI, AMDGPU::OpName::data1); in verifyInstruction()
4644 if (MI.getOpcode() == AMDGPU::DS_GWS_INIT || in verifyInstruction()
4645 MI.getOpcode() == AMDGPU::DS_GWS_SEMA_BR || in verifyInstruction()
4646 MI.getOpcode() == AMDGPU::DS_GWS_BARRIER) { in verifyInstruction()
4648 if (!isAlignedReg(AMDGPU::OpName::data0)) { in verifyInstruction()
4656 if (!isAlignedReg(AMDGPU::OpName::vaddr)) { in verifyInstruction()
4664 if (MI.getOpcode() == AMDGPU::V_ACCVGPR_WRITE_B32_e64 && in verifyInstruction()
4666 const MachineOperand *Src = getNamedOperand(MI, AMDGPU::OpName::src0); in verifyInstruction()
4674 if (Desc.getOpcode() == AMDGPU::G_AMDGPU_WAVE_ADDRESS) { in verifyInstruction()
4687 default: return AMDGPU::INSTRUCTION_LIST_END; in getVALUOp()
4688 case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE; in getVALUOp()
4689 case AMDGPU::COPY: return AMDGPU::COPY; in getVALUOp()
4690 case AMDGPU::PHI: return AMDGPU::PHI; in getVALUOp()
4691 case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG; in getVALUOp()
4692 case AMDGPU::WQM: return AMDGPU::WQM; in getVALUOp()
4693 case AMDGPU::SOFT_WQM: return AMDGPU::SOFT_WQM; in getVALUOp()
4694 case AMDGPU::STRICT_WWM: return AMDGPU::STRICT_WWM; in getVALUOp()
4695 case AMDGPU::STRICT_WQM: return AMDGPU::STRICT_WQM; in getVALUOp()
4696 case AMDGPU::S_MOV_B32: { in getVALUOp()
4700 AMDGPU::COPY : AMDGPU::V_MOV_B32_e32; in getVALUOp()
4702 case AMDGPU::S_ADD_I32: in getVALUOp()
4703 return ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_CO_U32_e32; in getVALUOp()
4704 case AMDGPU::S_ADDC_U32: in getVALUOp()
4705 return AMDGPU::V_ADDC_U32_e32; in getVALUOp()
4706 case AMDGPU::S_SUB_I32: in getVALUOp()
4707 return ST.hasAddNoCarry() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_CO_U32_e32; in getVALUOp()
4710 case AMDGPU::S_ADD_U32: in getVALUOp()
4711 return AMDGPU::V_ADD_CO_U32_e32; in getVALUOp()
4712 case AMDGPU::S_SUB_U32: in getVALUOp()
4713 return AMDGPU::V_SUB_CO_U32_e32; in getVALUOp()
4714 case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32; in getVALUOp()
4715 case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_U32_e64; in getVALUOp()
4716 case AMDGPU::S_MUL_HI_U32: return AMDGPU::V_MUL_HI_U32_e64; in getVALUOp()
4717 case AMDGPU::S_MUL_HI_I32: return AMDGPU::V_MUL_HI_I32_e64; in getVALUOp()
4718 case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64; in getVALUOp()
4719 case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64; in getVALUOp()
4720 case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64; in getVALUOp()
4721 case AMDGPU::S_XNOR_B32: in getVALUOp()
4722 return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END; in getVALUOp()
4723 case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64; in getVALUOp()
4724 case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64; in getVALUOp()
4725 case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64; in getVALUOp()
4726 case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64; in getVALUOp()
4727 case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32; in getVALUOp()
4728 case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64_e64; in getVALUOp()
4729 case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32; in getVALUOp()
4730 case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64_e64; in getVALUOp()
4731 case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32; in getVALUOp()
4732 case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64_e64; in getVALUOp()
4733 case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32_e64; in getVALUOp()
4734 case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32_e64; in getVALUOp()
4735 case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32_e64; in getVALUOp()
4736 case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32_e64; in getVALUOp()
4737 case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64; in getVALUOp()
4738 case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32; in getVALUOp()
4739 case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32; in getVALUOp()
4740 case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32; in getVALUOp()
4741 case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e64; in getVALUOp()
4742 case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e64; in getVALUOp()
4743 case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e64; in getVALUOp()
4744 case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e64; in getVALUOp()
4745 case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e64; in getVALUOp()
4746 case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e64; in getVALUOp()
4747 case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e64; in getVALUOp()
4748 case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e64; in getVALUOp()
4749 case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e64; in getVALUOp()
4750 case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e64; in getVALUOp()
4751 case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e64; in getVALUOp()
4752 case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e64; in getVALUOp()
4753 case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e64; in getVALUOp()
4754 case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e64; in getVALUOp()
4755 case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64; in getVALUOp()
4756 case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32; in getVALUOp()
4757 case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32; in getVALUOp()
4758 case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64; in getVALUOp()
4759 case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ; in getVALUOp()
4760 case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ; in getVALUOp()
4776 case AMDGPU::AV_32RegClassID: in adjustAllocatableRegClass()
4777 RCID = AMDGPU::VGPR_32RegClassID; in adjustAllocatableRegClass()
4779 case AMDGPU::AV_64RegClassID: in adjustAllocatableRegClass()
4780 RCID = AMDGPU::VReg_64RegClassID; in adjustAllocatableRegClass()
4782 case AMDGPU::AV_96RegClassID: in adjustAllocatableRegClass()
4783 RCID = AMDGPU::VReg_96RegClassID; in adjustAllocatableRegClass()
4785 case AMDGPU::AV_128RegClassID: in adjustAllocatableRegClass()
4786 RCID = AMDGPU::VReg_128RegClassID; in adjustAllocatableRegClass()
4788 case AMDGPU::AV_160RegClassID: in adjustAllocatableRegClass()
4789 RCID = AMDGPU::VReg_160RegClassID; in adjustAllocatableRegClass()
4791 case AMDGPU::AV_512RegClassID: in adjustAllocatableRegClass()
4792 RCID = AMDGPU::VReg_512RegClassID; in adjustAllocatableRegClass()
4818 const int VDstIdx = AMDGPU::getNamedOperandIdx(TID.Opcode, in getRegClass()
4819 AMDGPU::OpName::vdst); in getRegClass()
4820 const int DataIdx = AMDGPU::getNamedOperandIdx(TID.Opcode, in getRegClass()
4821 (TID.TSFlags & SIInstrFlags::DS) ? AMDGPU::OpName::data0 in getRegClass()
4822 : AMDGPU::OpName::vdata); in getRegClass()
4825 AMDGPU::getNamedOperandIdx(TID.Opcode, in getRegClass()
4826 AMDGPU::OpName::data1) != -1; in getRegClass()
4858 unsigned Opcode = (Size == 64) ? AMDGPU::V_MOV_B64_PSEUDO : AMDGPU::V_MOV_B32_e32; in legalizeOpWithMove()
4860 Opcode = AMDGPU::COPY; in legalizeOpWithMove()
4862 Opcode = (Size == 64) ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32; in legalizeOpWithMove()
4869 VRC = &AMDGPU::VGPR_32RegClass; in legalizeOpWithMove()
4888 if (SuperReg.getSubReg() == AMDGPU::NoSubRegister) { in buildExtractSubReg()
4917 if (SubIdx == AMDGPU::sub0) in buildExtractSubRegOrImm()
4919 if (SubIdx == AMDGPU::sub1) in buildExtractSubRegOrImm()
5009 } else if (InstDesc.OpInfo[i].OperandType == AMDGPU::OPERAND_KIMM32 || in isOperandLegal()
5010 (AMDGPU::isSISrcOperand(InstDesc, i) && in isOperandLegal()
5034 const int VDstIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); in isOperandLegal()
5035 const int DataIdx = AMDGPU::getNamedOperandIdx(Opc, in isOperandLegal()
5036 isDS(Opc) ? AMDGPU::OpName::data0 : AMDGPU::OpName::vdata); in isOperandLegal()
5046 const int Data1Idx = AMDGPU::getNamedOperandIdx(Opc, in isOperandLegal()
5047 AMDGPU::OpName::data1); in isOperandLegal()
5052 if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64 && !ST.hasGFX90AInsts() && in isOperandLegal()
5053 (int)OpIdx == AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) && in isOperandLegal()
5075 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); in legalizeOperandsVOP2()
5078 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); in legalizeOperandsVOP2()
5083 bool HasImplicitSGPR = findImplicitSGPRRead(MI) != AMDGPU::NoRegister; in legalizeOperandsVOP2()
5092 if (Opc == AMDGPU::V_WRITELANE_B32) { in legalizeOperandsVOP2()
5095 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in legalizeOperandsVOP2()
5096 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) in legalizeOperandsVOP2()
5101 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in legalizeOperandsVOP2()
5103 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) in legalizeOperandsVOP2()
5125 if (Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() && in legalizeOperandsVOP2()
5127 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in legalizeOperandsVOP2()
5129 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) in legalizeOperandsVOP2()
5187 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0), in legalizeOperandsVOP3()
5188 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1), in legalizeOperandsVOP3()
5189 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2) in legalizeOperandsVOP3()
5192 if (Opc == AMDGPU::V_PERMLANE16_B32_e64 || in legalizeOperandsVOP3()
5193 Opc == AMDGPU::V_PERMLANEX16_B32_e64) { in legalizeOperandsVOP3()
5199 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in legalizeOperandsVOP3()
5200 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) in legalizeOperandsVOP3()
5205 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in legalizeOperandsVOP3()
5206 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) in legalizeOperandsVOP3()
5217 if (SGPRReg != AMDGPU::NoRegister) { in legalizeOperandsVOP3()
5286 get(AMDGPU::V_READFIRSTLANE_B32), DstReg) in readlaneVGPRToSGPR()
5293 Register SGPR = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); in readlaneVGPRToSGPR()
5295 get(AMDGPU::V_READFIRSTLANE_B32), SGPR) in readlaneVGPRToSGPR()
5302 get(AMDGPU::REG_SEQUENCE), DstReg); in readlaneVGPRToSGPR()
5317 MachineOperand *SBase = getNamedOperand(MI, AMDGPU::OpName::sbase); in legalizeOperandsSMRD()
5322 MachineOperand *SOff = getNamedOperand(MI, AMDGPU::OpName::soffset); in legalizeOperandsSMRD()
5331 int OldSAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr); in moveFlatAddrToVGPR()
5337 int NewOpc = AMDGPU::getGlobalVaddrOp(Opc); in moveFlatAddrToVGPR()
5339 NewOpc = AMDGPU::getFlatScratchInstSVfromSS(Opc); in moveFlatAddrToVGPR()
5348 int NewVAddrIdx = AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vaddr); in moveFlatAddrToVGPR()
5352 int OldVAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr); in moveFlatAddrToVGPR()
5359 if (!VAddrDef || VAddrDef->getOpcode() != AMDGPU::V_MOV_B32_e32 || in moveFlatAddrToVGPR()
5384 int NewVDstIn = AMDGPU::getNamedOperandIdx(NewOpc, in moveFlatAddrToVGPR()
5385 AMDGPU::OpName::vdst_in); in moveFlatAddrToVGPR()
5390 int OldVDstIn = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst_in); in moveFlatAddrToVGPR()
5397 int NewVDst = AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vdst); in moveFlatAddrToVGPR()
5417 MachineOperand *SAddr = getNamedOperand(MI, AMDGPU::OpName::saddr); in legalizeOperandsFLAT()
5445 auto Copy = BuildMI(InsertMBB, I, DL, get(AMDGPU::COPY), DstReg).add(Op); in legalizeGenericOperand()
5455 if (Def->isMoveImmediate() && DstRC != &AMDGPU::VReg_1RegClass) in legalizeGenericOperand()
5465 if (!RI.isSGPRClass(DstRC) && !Copy->readsRegister(AMDGPU::EXEC, &RI) && in legalizeGenericOperand()
5467 Copy.addReg(AMDGPU::EXEC, RegState::Implicit); in legalizeGenericOperand()
5481 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; in emitLoadSRsrcFromVGPRLoop()
5483 ST.isWave32() ? AMDGPU::S_AND_SAVEEXEC_B32 : AMDGPU::S_AND_SAVEEXEC_B64; in emitLoadSRsrcFromVGPRLoop()
5485 ST.isWave32() ? AMDGPU::S_XOR_B32_term : AMDGPU::S_XOR_B64_term; in emitLoadSRsrcFromVGPRLoop()
5487 ST.isWave32() ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64; in emitLoadSRsrcFromVGPRLoop()
5488 const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID); in emitLoadSRsrcFromVGPRLoop()
5493 Register CondReg = AMDGPU::NoRegister; in emitLoadSRsrcFromVGPRLoop()
5504 Register CurRegLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); in emitLoadSRsrcFromVGPRLoop()
5505 Register CurRegHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); in emitLoadSRsrcFromVGPRLoop()
5508 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegLo) in emitLoadSRsrcFromVGPRLoop()
5512 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegHi) in emitLoadSRsrcFromVGPRLoop()
5519 Register CurReg = MRI.createVirtualRegister(&AMDGPU::SGPR_64RegClass); in emitLoadSRsrcFromVGPRLoop()
5520 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), CurReg) in emitLoadSRsrcFromVGPRLoop()
5522 .addImm(AMDGPU::sub0) in emitLoadSRsrcFromVGPRLoop()
5524 .addImm(AMDGPU::sub1); in emitLoadSRsrcFromVGPRLoop()
5528 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), NewCondReg) in emitLoadSRsrcFromVGPRLoop()
5536 if (CondReg == AMDGPU::NoRegister) // First. in emitLoadSRsrcFromVGPRLoop()
5551 auto Merge = BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), SRsrc); in emitLoadSRsrcFromVGPRLoop()
5577 BuildMI(BodyBB, I, DL, TII.get(AMDGPU::SI_WATERFALL_LOOP)).addMBB(&LoopBB); in emitLoadSRsrcFromVGPRLoop()
5600 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; in loadSRsrcFromVGPR()
5601 unsigned MovExecOpc = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; in loadSRsrcFromVGPR()
5602 const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID); in loadSRsrcFromVGPR()
5677 TII.buildExtractSubReg(MI, MRI, Rsrc, &AMDGPU::VReg_128RegClass, in extractRsrcPtr()
5678 AMDGPU::sub0_sub1, &AMDGPU::VReg_64RegClass); in extractRsrcPtr()
5681 Register Zero64 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in extractRsrcPtr()
5682 Register SRsrcFormatLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); in extractRsrcPtr()
5683 Register SRsrcFormatHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); in extractRsrcPtr()
5684 Register NewSRsrc = MRI.createVirtualRegister(&AMDGPU::SGPR_128RegClass); in extractRsrcPtr()
5688 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B64), Zero64) in extractRsrcPtr()
5692 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatLo) in extractRsrcPtr()
5696 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatHi) in extractRsrcPtr()
5700 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::REG_SEQUENCE), NewSRsrc) in extractRsrcPtr()
5702 .addImm(AMDGPU::sub0_sub1) in extractRsrcPtr()
5704 .addImm(AMDGPU::sub2) in extractRsrcPtr()
5706 .addImm(AMDGPU::sub3); in extractRsrcPtr()
5745 if (MI.getOpcode() == AMDGPU::PHI) { in legalizeOperands()
5765 if (getOpRegClass(MI, 0) == &AMDGPU::VReg_1RegClass) { in legalizeOperands()
5766 VRC = &AMDGPU::VReg_1RegClass; in legalizeOperands()
5800 if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) { in legalizeOperands()
5827 if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) { in legalizeOperands()
5841 if (MI.getOpcode() == AMDGPU::SI_INIT_M0) { in legalizeOperands()
5853 if (isMIMG(MI) || (AMDGPU::isGraphics(MF.getFunction().getCallingConv()) && in legalizeOperands()
5855 MachineOperand *SRsrc = getNamedOperand(MI, AMDGPU::OpName::srsrc); in legalizeOperands()
5859 MachineOperand *SSamp = getNamedOperand(MI, AMDGPU::OpName::ssamp); in legalizeOperands()
5867 if (MI.getOpcode() == AMDGPU::SI_CALL_ISEL) { in legalizeOperands()
5895 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::srsrc); in legalizeOperands()
5922 MachineOperand *VAddr = getNamedOperand(MI, AMDGPU::OpName::vaddr); in legalizeOperands()
5923 if (VAddr && AMDGPU::getIfAddr64Inst(MI.getOpcode()) != -1) { in legalizeOperands()
5926 Register NewVAddrLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in legalizeOperands()
5927 Register NewVAddrHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in legalizeOperands()
5928 Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); in legalizeOperands()
5930 const auto *BoolXExecRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); in legalizeOperands()
5939 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADD_CO_U32_e64), NewVAddrLo) in legalizeOperands()
5941 .addReg(RsrcPtr, 0, AMDGPU::sub0) in legalizeOperands()
5942 .addReg(VAddr->getReg(), 0, AMDGPU::sub0) in legalizeOperands()
5946 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADDC_U32_e64), NewVAddrHi) in legalizeOperands()
5948 .addReg(RsrcPtr, 0, AMDGPU::sub1) in legalizeOperands()
5949 .addReg(VAddr->getReg(), 0, AMDGPU::sub1) in legalizeOperands()
5954 BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewVAddr) in legalizeOperands()
5956 .addImm(AMDGPU::sub0) in legalizeOperands()
5958 .addImm(AMDGPU::sub1); in legalizeOperands()
5971 Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); in legalizeOperands()
5972 MachineOperand *VData = getNamedOperand(MI, AMDGPU::OpName::vdata); in legalizeOperands()
5973 MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset); in legalizeOperands()
5974 MachineOperand *SOffset = getNamedOperand(MI, AMDGPU::OpName::soffset); in legalizeOperands()
5975 unsigned Addr64Opcode = AMDGPU::getAddr64Inst(MI.getOpcode()); in legalizeOperands()
5979 MachineOperand *VDataIn = getNamedOperand(MI, AMDGPU::OpName::vdata_in); in legalizeOperands()
5993 getNamedOperand(MI, AMDGPU::OpName::cpol)) { in legalizeOperands()
5998 getNamedOperand(MI, AMDGPU::OpName::tfe)) { in legalizeOperands()
6002 MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::swz)); in legalizeOperands()
6015 .addImm(getNamedImmOperand(MI, AMDGPU::OpName::cpol)) in legalizeOperands()
6022 BuildMI(MBB, Addr64, Addr64->getDebugLoc(), get(AMDGPU::REG_SEQUENCE), in legalizeOperands()
6024 .addReg(RsrcPtr, 0, AMDGPU::sub0) in legalizeOperands()
6025 .addImm(AMDGPU::sub0) in legalizeOperands()
6026 .addReg(RsrcPtr, 0, AMDGPU::sub1) in legalizeOperands()
6027 .addImm(AMDGPU::sub1); in legalizeOperands()
6057 case AMDGPU::S_ADD_U64_PSEUDO: in moveToVALU()
6058 case AMDGPU::S_SUB_U64_PSEUDO: in moveToVALU()
6062 case AMDGPU::S_ADD_I32: in moveToVALU()
6063 case AMDGPU::S_SUB_I32: { in moveToVALU()
6075 case AMDGPU::S_AND_B64: in moveToVALU()
6076 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_AND_B32, MDT); in moveToVALU()
6080 case AMDGPU::S_OR_B64: in moveToVALU()
6081 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_OR_B32, MDT); in moveToVALU()
6085 case AMDGPU::S_XOR_B64: in moveToVALU()
6086 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XOR_B32, MDT); in moveToVALU()
6090 case AMDGPU::S_NAND_B64: in moveToVALU()
6091 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NAND_B32, MDT); in moveToVALU()
6095 case AMDGPU::S_NOR_B64: in moveToVALU()
6096 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NOR_B32, MDT); in moveToVALU()
6100 case AMDGPU::S_XNOR_B64: in moveToVALU()
6102 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XNOR_B32, MDT); in moveToVALU()
6108 case AMDGPU::S_ANDN2_B64: in moveToVALU()
6109 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ANDN2_B32, MDT); in moveToVALU()
6113 case AMDGPU::S_ORN2_B64: in moveToVALU()
6114 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ORN2_B32, MDT); in moveToVALU()
6118 case AMDGPU::S_BREV_B64: in moveToVALU()
6119 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_BREV_B32, true); in moveToVALU()
6123 case AMDGPU::S_NOT_B64: in moveToVALU()
6124 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_NOT_B32); in moveToVALU()
6128 case AMDGPU::S_BCNT1_I32_B64: in moveToVALU()
6133 case AMDGPU::S_BFE_I64: in moveToVALU()
6138 case AMDGPU::S_LSHL_B32: in moveToVALU()
6140 NewOpcode = AMDGPU::V_LSHLREV_B32_e64; in moveToVALU()
6144 case AMDGPU::S_ASHR_I32: in moveToVALU()
6146 NewOpcode = AMDGPU::V_ASHRREV_I32_e64; in moveToVALU()
6150 case AMDGPU::S_LSHR_B32: in moveToVALU()
6152 NewOpcode = AMDGPU::V_LSHRREV_B32_e64; in moveToVALU()
6156 case AMDGPU::S_LSHL_B64: in moveToVALU()
6158 NewOpcode = AMDGPU::V_LSHLREV_B64_e64; in moveToVALU()
6162 case AMDGPU::S_ASHR_I64: in moveToVALU()
6164 NewOpcode = AMDGPU::V_ASHRREV_I64_e64; in moveToVALU()
6168 case AMDGPU::S_LSHR_B64: in moveToVALU()
6170 NewOpcode = AMDGPU::V_LSHRREV_B64_e64; in moveToVALU()
6175 case AMDGPU::S_ABS_I32: in moveToVALU()
6180 case AMDGPU::S_CBRANCH_SCC0: in moveToVALU()
6181 case AMDGPU::S_CBRANCH_SCC1: { in moveToVALU()
6184 bool IsSCC = CondReg == AMDGPU::SCC; in moveToVALU()
6186 Register EXEC = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; in moveToVALU()
6187 unsigned Opc = ST.isWave32() ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64; in moveToVALU()
6195 case AMDGPU::S_BFE_U64: in moveToVALU()
6196 case AMDGPU::S_BFM_B64: in moveToVALU()
6199 case AMDGPU::S_PACK_LL_B32_B16: in moveToVALU()
6200 case AMDGPU::S_PACK_LH_B32_B16: in moveToVALU()
6201 case AMDGPU::S_PACK_HL_B32_B16: in moveToVALU()
6202 case AMDGPU::S_PACK_HH_B32_B16: in moveToVALU()
6207 case AMDGPU::S_XNOR_B32: in moveToVALU()
6212 case AMDGPU::S_NAND_B32: in moveToVALU()
6213 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_AND_B32); in moveToVALU()
6217 case AMDGPU::S_NOR_B32: in moveToVALU()
6218 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_OR_B32); in moveToVALU()
6222 case AMDGPU::S_ANDN2_B32: in moveToVALU()
6223 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_AND_B32); in moveToVALU()
6227 case AMDGPU::S_ORN2_B32: in moveToVALU()
6228 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_OR_B32); in moveToVALU()
6236 case AMDGPU::S_ADD_CO_PSEUDO: in moveToVALU()
6237 case AMDGPU::S_SUB_CO_PSEUDO: { in moveToVALU()
6238 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO) in moveToVALU()
6239 ? AMDGPU::V_ADDC_U32_e64 in moveToVALU()
6240 : AMDGPU::V_SUBB_U32_e64; in moveToVALU()
6241 const auto *CarryRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); in moveToVALU()
6246 BuildMI(*MBB, &Inst, Inst.getDebugLoc(), get(AMDGPU::COPY), NewCarryReg) in moveToVALU()
6269 case AMDGPU::S_UADDO_PSEUDO: in moveToVALU()
6270 case AMDGPU::S_USUBO_PSEUDO: { in moveToVALU()
6277 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_UADDO_PSEUDO) in moveToVALU()
6278 ? AMDGPU::V_ADD_CO_U32_e64 in moveToVALU()
6279 : AMDGPU::V_SUB_CO_U32_e64; in moveToVALU()
6300 case AMDGPU::S_CSELECT_B32: in moveToVALU()
6301 case AMDGPU::S_CSELECT_B64: in moveToVALU()
6305 case AMDGPU::S_CMP_EQ_I32: in moveToVALU()
6306 case AMDGPU::S_CMP_LG_I32: in moveToVALU()
6307 case AMDGPU::S_CMP_GT_I32: in moveToVALU()
6308 case AMDGPU::S_CMP_GE_I32: in moveToVALU()
6309 case AMDGPU::S_CMP_LT_I32: in moveToVALU()
6310 case AMDGPU::S_CMP_LE_I32: in moveToVALU()
6311 case AMDGPU::S_CMP_EQ_U32: in moveToVALU()
6312 case AMDGPU::S_CMP_LG_U32: in moveToVALU()
6313 case AMDGPU::S_CMP_GT_U32: in moveToVALU()
6314 case AMDGPU::S_CMP_GE_U32: in moveToVALU()
6315 case AMDGPU::S_CMP_LT_U32: in moveToVALU()
6316 case AMDGPU::S_CMP_LE_U32: in moveToVALU()
6317 case AMDGPU::S_CMP_EQ_U64: in moveToVALU()
6318 case AMDGPU::S_CMP_LG_U64: { in moveToVALU()
6326 int SCCIdx = Inst.findRegisterDefOperandIdx(AMDGPU::SCC); in moveToVALU()
6335 if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) { in moveToVALU()
6353 if (Op.isReg() && Op.getReg() == AMDGPU::SCC) { in moveToVALU()
6363 if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) { in moveToVALU()
6366 unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16; in moveToVALU()
6370 } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) { in moveToVALU()
6379 if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) { in moveToVALU()
6395 unsigned NewDstReg = AMDGPU::NoRegister; in moveToVALU()
6424 Inst.setDesc(get(AMDGPU::IMPLICIT_DEF)); in moveToVALU()
6456 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in moveScalarAddSub()
6459 assert(Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32); in moveScalarAddSub()
6461 unsigned NewOpc = Opc == AMDGPU::S_ADD_I32 ? in moveScalarAddSub()
6462 AMDGPU::V_ADD_U32_e64 : AMDGPU::V_SUB_U32_e64; in moveScalarAddSub()
6464 assert(Inst.getOperand(3).getReg() == AMDGPU::SCC); in moveScalarAddSub()
6494 bool IsSCC = (SCCSource == AMDGPU::SCC); in lowerSelect()
6507 RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); in lowerSelect()
6518 if (CandI.findRegisterDefOperandIdx(AMDGPU::SCC, false, false, &RI) != in lowerSelect()
6520 if (CandI.isCopy() && CandI.getOperand(0).getReg() == AMDGPU::SCC) { in lowerSelect()
6521 BuildMI(MBB, MII, DL, get(AMDGPU::COPY), CopySCC) in lowerSelect()
6533 unsigned Opcode = (ST.getWavefrontSize() == 64) ? AMDGPU::S_CSELECT_B64 in lowerSelect()
6534 : AMDGPU::S_CSELECT_B32; in lowerSelect()
6541 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in lowerSelect()
6544 BuildMI(MBB, MII, DL, get(AMDGPU::V_CNDMASK_B32_e64), ResultReg) in lowerSelect()
6565 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in lowerScalarAbs()
6566 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in lowerScalarAbs()
6569 AMDGPU::V_SUB_U32_e32 : AMDGPU::V_SUB_CO_U32_e32; in lowerScalarAbs()
6575 BuildMI(MBB, MII, DL, get(AMDGPU::V_MAX_I32_e64), ResultReg) in lowerScalarAbs()
6595 Register NewDest = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in lowerScalarXnor()
6596 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src0, MRI, DL); in lowerScalarXnor()
6597 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src1, MRI, DL); in lowerScalarXnor()
6599 BuildMI(MBB, MII, DL, get(AMDGPU::V_XNOR_B32_e64), NewDest) in lowerScalarXnor()
6615 Register Temp = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); in lowerScalarXnor()
6616 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); in lowerScalarXnor()
6622 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src0); in lowerScalarXnor()
6623 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest) in lowerScalarXnor()
6627 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src1); in lowerScalarXnor()
6628 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest) in lowerScalarXnor()
6632 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), Temp) in lowerScalarXnor()
6636 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest).addReg(Temp); in lowerScalarXnor()
6660 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); in splitScalarNotBinop()
6661 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); in splitScalarNotBinop()
6667 MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest) in splitScalarNotBinop()
6689 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in splitScalarBinOpN2()
6690 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in splitScalarBinOpN2()
6692 MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Interm) in splitScalarBinOpN2()
6721 &AMDGPU::SGPR_32RegClass; in splitScalar64BitUnaryOp()
6723 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0); in splitScalar64BitUnaryOp()
6726 AMDGPU::sub0, Src0SubRC); in splitScalar64BitUnaryOp()
6730 const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0); in splitScalar64BitUnaryOp()
6736 AMDGPU::sub1, Src0SubRC); in splitScalar64BitUnaryOp()
6747 .addImm(AMDGPU::sub0) in splitScalar64BitUnaryOp()
6749 .addImm(AMDGPU::sub1); in splitScalar64BitUnaryOp()
6766 bool IsAdd = (Inst.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO); in splitScalar64BitAddSub()
6770 const auto *CarryRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); in splitScalar64BitAddSub()
6772 Register FullDestReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); in splitScalar64BitAddSub()
6773 Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitAddSub()
6774 Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitAddSub()
6787 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0); in splitScalar64BitAddSub()
6788 const TargetRegisterClass *Src1SubRC = RI.getSubRegClass(Src1RC, AMDGPU::sub0); in splitScalar64BitAddSub()
6791 AMDGPU::sub0, Src0SubRC); in splitScalar64BitAddSub()
6793 AMDGPU::sub0, Src1SubRC); in splitScalar64BitAddSub()
6797 AMDGPU::sub1, Src0SubRC); in splitScalar64BitAddSub()
6799 AMDGPU::sub1, Src1SubRC); in splitScalar64BitAddSub()
6801 unsigned LoOpc = IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64; in splitScalar64BitAddSub()
6809 unsigned HiOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64; in splitScalar64BitAddSub()
6820 .addImm(AMDGPU::sub0) in splitScalar64BitAddSub()
6822 .addImm(AMDGPU::sub1); in splitScalar64BitAddSub()
6851 &AMDGPU::SGPR_32RegClass; in splitScalar64BitBinaryOp()
6853 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0); in splitScalar64BitBinaryOp()
6856 &AMDGPU::SGPR_32RegClass; in splitScalar64BitBinaryOp()
6858 const TargetRegisterClass *Src1SubRC = RI.getSubRegClass(Src1RC, AMDGPU::sub0); in splitScalar64BitBinaryOp()
6861 AMDGPU::sub0, Src0SubRC); in splitScalar64BitBinaryOp()
6863 AMDGPU::sub0, Src1SubRC); in splitScalar64BitBinaryOp()
6865 AMDGPU::sub1, Src0SubRC); in splitScalar64BitBinaryOp()
6867 AMDGPU::sub1, Src1SubRC); in splitScalar64BitBinaryOp()
6871 const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0); in splitScalar64BitBinaryOp()
6886 .addImm(AMDGPU::sub0) in splitScalar64BitBinaryOp()
6888 .addImm(AMDGPU::sub1); in splitScalar64BitBinaryOp()
6914 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in splitScalar64BitXnor()
6927 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B64), Interm) in splitScalar64BitXnor()
6932 MachineInstr &Xor = *BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B64), NewDest) in splitScalar64BitXnor()
6952 const MCInstrDesc &InstDesc = get(AMDGPU::V_BCNT_U32_B32_e64); in splitScalar64BitBCNT()
6955 &AMDGPU::SGPR_32RegClass; in splitScalar64BitBCNT()
6957 Register MidReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitBCNT()
6958 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitBCNT()
6960 const TargetRegisterClass *SrcSubRC = RI.getSubRegClass(SrcRC, AMDGPU::sub0); in splitScalar64BitBCNT()
6963 AMDGPU::sub0, SrcSubRC); in splitScalar64BitBCNT()
6965 AMDGPU::sub1, SrcSubRC); in splitScalar64BitBCNT()
6993 assert(Inst.getOpcode() == AMDGPU::S_BFE_I64 && BitWidth <= 32 && in splitScalar64BitBFE()
6997 Register MidRegLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitBFE()
6998 Register MidRegHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitBFE()
6999 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); in splitScalar64BitBFE()
7001 BuildMI(MBB, MII, DL, get(AMDGPU::V_BFE_I32_e64), MidRegLo) in splitScalar64BitBFE()
7002 .addReg(Inst.getOperand(1).getReg(), 0, AMDGPU::sub0) in splitScalar64BitBFE()
7006 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e32), MidRegHi) in splitScalar64BitBFE()
7012 .addImm(AMDGPU::sub0) in splitScalar64BitBFE()
7014 .addImm(AMDGPU::sub1); in splitScalar64BitBFE()
7022 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitBFE()
7023 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); in splitScalar64BitBFE()
7025 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e64), TmpReg) in splitScalar64BitBFE()
7027 .addReg(Src.getReg(), 0, AMDGPU::sub0); in splitScalar64BitBFE()
7030 .addReg(Src.getReg(), 0, AMDGPU::sub0) in splitScalar64BitBFE()
7031 .addImm(AMDGPU::sub0) in splitScalar64BitBFE()
7033 .addImm(AMDGPU::sub1); in splitScalar64BitBFE()
7050 case AMDGPU::COPY: in addUsersToMoveToVALUWorklist()
7051 case AMDGPU::WQM: in addUsersToMoveToVALUWorklist()
7052 case AMDGPU::SOFT_WQM: in addUsersToMoveToVALUWorklist()
7053 case AMDGPU::STRICT_WWM: in addUsersToMoveToVALUWorklist()
7054 case AMDGPU::STRICT_WQM: in addUsersToMoveToVALUWorklist()
7055 case AMDGPU::REG_SEQUENCE: in addUsersToMoveToVALUWorklist()
7056 case AMDGPU::PHI: in addUsersToMoveToVALUWorklist()
7057 case AMDGPU::INSERT_SUBREG: in addUsersToMoveToVALUWorklist()
7079 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in movePackToVALU()
7086 case AMDGPU::S_PACK_LL_B32_B16: { in movePackToVALU()
7087 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in movePackToVALU()
7088 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in movePackToVALU()
7092 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg) in movePackToVALU()
7095 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_B32_e64), TmpReg) in movePackToVALU()
7099 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHL_OR_B32_e64), ResultReg) in movePackToVALU()
7105 case AMDGPU::S_PACK_LH_B32_B16: { in movePackToVALU()
7106 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in movePackToVALU()
7107 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg) in movePackToVALU()
7109 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_BFI_B32_e64), ResultReg) in movePackToVALU()
7115 case AMDGPU::S_PACK_HL_B32_B16: { in movePackToVALU()
7116 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in movePackToVALU()
7117 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg) in movePackToVALU()
7120 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHL_OR_B32_e64), ResultReg) in movePackToVALU()
7126 case AMDGPU::S_PACK_HH_B32_B16: { in movePackToVALU()
7127 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in movePackToVALU()
7128 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in movePackToVALU()
7129 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg) in movePackToVALU()
7132 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg) in movePackToVALU()
7134 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_OR_B32_e64), ResultReg) in movePackToVALU()
7155 assert(Op.isReg() && Op.getReg() == AMDGPU::SCC && Op.isDef() && in addSCCDefUsersToVALUWorklist()
7164 int SCCIdx = MI.findRegisterUseOperandIdx(AMDGPU::SCC, false, &RI); in addSCCDefUsersToVALUWorklist()
7181 if (MI.findRegisterDefOperandIdx(AMDGPU::SCC, false, false, &RI) != -1) in addSCCDefUsersToVALUWorklist()
7195 assert(Op.isReg() && Op.getReg() == AMDGPU::SCC && Op.isUse()); in addSCCDefsToVALUWorklist()
7205 if (MI.modifiesRegister(AMDGPU::VCC, &RI)) in addSCCDefsToVALUWorklist()
7207 if (MI.definesRegister(AMDGPU::SCC, &RI)) { in addSCCDefsToVALUWorklist()
7222 case AMDGPU::COPY: in getDestEquivalentVGPRClass()
7223 case AMDGPU::PHI: in getDestEquivalentVGPRClass()
7224 case AMDGPU::REG_SEQUENCE: in getDestEquivalentVGPRClass()
7225 case AMDGPU::INSERT_SUBREG: in getDestEquivalentVGPRClass()
7226 case AMDGPU::WQM: in getDestEquivalentVGPRClass()
7227 case AMDGPU::SOFT_WQM: in getDestEquivalentVGPRClass()
7228 case AMDGPU::STRICT_WWM: in getDestEquivalentVGPRClass()
7229 case AMDGPU::STRICT_WQM: { in getDestEquivalentVGPRClass()
7236 case AMDGPU::PHI: in getDestEquivalentVGPRClass()
7237 case AMDGPU::REG_SEQUENCE: in getDestEquivalentVGPRClass()
7238 case AMDGPU::INSERT_SUBREG: in getDestEquivalentVGPRClass()
7248 if (RI.isVGPRClass(NewDstRC) || NewDstRC == &AMDGPU::VReg_1RegClass) in getDestEquivalentVGPRClass()
7278 if (SGPRReg != AMDGPU::NoRegister) in findUsedSGPR()
7281 Register UsedSGPRs[3] = { AMDGPU::NoRegister }; in findUsedSGPR()
7320 if (UsedSGPRs[0] != AMDGPU::NoRegister) { in findUsedSGPR()
7325 if (SGPRReg == AMDGPU::NoRegister && UsedSGPRs[1] != AMDGPU::NoRegister) { in findUsedSGPR()
7335 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OperandName); in getNamedOperand()
7345 AMDGPU::UfmtGFX11::UFMT_32_FLOAT : in getDefaultRsrcDataFormat()
7346 AMDGPU::UfmtGFX10::UFMT_32_FLOAT; in getDefaultRsrcDataFormat()
7352 uint64_t RsrcDataFormat = AMDGPU::RSRC_DATA_FORMAT; in getDefaultRsrcDataFormat()
7369 AMDGPU::RSRC_TID_ENABLE | in getScratchRsrcWords23()
7375 Rsrc23 |= EltSizeValue << AMDGPU::RSRC_ELEMENT_SIZE_SHIFT; in getScratchRsrcWords23()
7380 Rsrc23 |= IndexStride << AMDGPU::RSRC_INDEX_STRIDE_SHIFT; in getScratchRsrcWords23()
7386 Rsrc23 &= ~AMDGPU::RSRC_DATA_FORMAT; in getScratchRsrcWords23()
7404 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::vaddr); in isStackAccess()
7406 return AMDGPU::NoRegister; in isStackAccess()
7412 return getNamedOperand(MI, AMDGPU::OpName::vdata)->getReg(); in isStackAccess()
7417 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::addr); in isSGPRStackAccess()
7420 return getNamedOperand(MI, AMDGPU::OpName::data)->getReg(); in isSGPRStackAccess()
7426 return AMDGPU::NoRegister; in isLoadFromStackSlot()
7434 return AMDGPU::NoRegister; in isLoadFromStackSlot()
7440 return AMDGPU::NoRegister; in isStoreToStackSlot()
7448 return AMDGPU::NoRegister; in isStoreToStackSlot()
7500 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0); in getInstSizeInBytes()
7504 int RSrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc); in getInstSizeInBytes()
7539 return Branch.getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO; in isNonUniformBranchInstr()
7551 if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) { in convertNonUniformIfRegion()
7554 BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_IF), DstReg) in convertNonUniformIfRegion()
7558 BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_END_CF)) in convertNonUniformIfRegion()
7577 if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) { in convertNonUniformLoopRegion()
7596 get(AMDGPU::SI_IF_BREAK), BackEdgeReg) in convertNonUniformLoopRegion()
7600 BuildMI(*(MF), Branch->getDebugLoc(), get(AMDGPU::SI_LOOP)) in convertNonUniformLoopRegion()
7614 {AMDGPU::TI_CONSTDATA_START, "amdgpu-constdata-start"}, in getSerializableTargetIndices()
7615 {AMDGPU::TI_SCRATCH_RSRC_DWORD0, "amdgpu-scratch-rsrc-dword0"}, in getSerializableTargetIndices()
7616 {AMDGPU::TI_SCRATCH_RSRC_DWORD1, "amdgpu-scratch-rsrc-dword1"}, in getSerializableTargetIndices()
7617 {AMDGPU::TI_SCRATCH_RSRC_DWORD2, "amdgpu-scratch-rsrc-dword2"}, in getSerializableTargetIndices()
7618 {AMDGPU::TI_SCRATCH_RSRC_DWORD3, "amdgpu-scratch-rsrc-dword3"}}; in getSerializableTargetIndices()
7682 return !MI.isTerminator() && MI.getOpcode() != AMDGPU::COPY && in isBasicBlockPrologue()
7683 MI.modifiesRegister(AMDGPU::EXEC, &RI); in isBasicBlockPrologue()
7692 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e64), DestReg); in getAddNoCarry()
7698 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_CO_U32_e64), DestReg) in getAddNoCarry()
7708 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e32), DestReg); in getAddNoCarry()
7711 Register UnusedCarry = !RS.isRegUsed(AMDGPU::VCC) in getAddNoCarry()
7719 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_CO_U32_e64), DestReg) in getAddNoCarry()
7725 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR: in isKillTerminator()
7726 case AMDGPU::SI_KILL_I1_TERMINATOR: in isKillTerminator()
7735 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO: in getKillTerminatorFromPseudo()
7736 return get(AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR); in getKillTerminatorFromPseudo()
7737 case AMDGPU::SI_KILL_I1_PSEUDO: in getKillTerminatorFromPseudo()
7738 return get(AMDGPU::SI_KILL_I1_TERMINATOR); in getKillTerminatorFromPseudo()
7749 if (Op.isReg() && Op.getReg() == AMDGPU::VCC) in fixImplicitOperands()
7750 Op.setReg(AMDGPU::VCC_LO); in fixImplicitOperands()
7759 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::sbase); in isBufferSMRD()
7764 return RI.getRegClass(RCID)->hasSubClassEq(&AMDGPU::SGPR_128RegClass); in isBufferSMRD()
7818 unsigned N = AMDGPU::getNumFlatOffsetBits(ST, Signed); in isLegalFLATOffset()
7833 const unsigned NumBits = AMDGPU::getNumFlatOffsetBits(ST, Signed); in splitFlatOffset()
7897 case AMDGPU::V_MOVRELS_B32_dpp_gfx10: in isAsmOnlyOpcode()
7898 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10: in isAsmOnlyOpcode()
7899 case AMDGPU::V_MOVRELD_B32_dpp_gfx10: in isAsmOnlyOpcode()
7900 case AMDGPU::V_MOVRELD_B32_sdwa_gfx10: in isAsmOnlyOpcode()
7901 case AMDGPU::V_MOVRELSD_B32_dpp_gfx10: in isAsmOnlyOpcode()
7902 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10: in isAsmOnlyOpcode()
7903 case AMDGPU::V_MOVRELSD_2_B32_dpp_gfx10: in isAsmOnlyOpcode()
7904 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10: in isAsmOnlyOpcode()
7939 int MFMAOp = AMDGPU::getMFMAEarlyClobberOp(Opcode); in pseudoToMCOpcode()
7944 int MCOp = AMDGPU::getMCOpcode(Opcode, Gen); in pseudoToMCOpcode()
7953 NMCOp = AMDGPU::getMCOpcode(Opcode, SIEncodingFamily::GFX940); in pseudoToMCOpcode()
7955 NMCOp = AMDGPU::getMCOpcode(Opcode, SIEncodingFamily::GFX90A); in pseudoToMCOpcode()
7957 NMCOp = AMDGPU::getMCOpcode(Opcode, SIEncodingFamily::GFX9); in pseudoToMCOpcode()
7999 case AMDGPU::REG_SEQUENCE: in followSubRegDef()
8003 case AMDGPU::INSERT_SUBREG: in followSubRegDef()
8029 case AMDGPU::COPY: in getVRegSubRegDef()
8030 case AMDGPU::V_MOV_B32_e32: { in getVRegSubRegDef()
8079 if (I->modifiesRegister(AMDGPU::EXEC, TRI)) in execMayBeModifiedBeforeUse()
8136 } else if (TRI->regsOverlap(Reg, AMDGPU::EXEC)) in execMayBeModifiedBeforeAnyUse()
8161 (InsPt->getOpcode() == AMDGPU::SI_IF || in createPHISourceCopy()
8162 InsPt->getOpcode() == AMDGPU::SI_ELSE || in createPHISourceCopy()
8163 InsPt->getOpcode() == AMDGPU::SI_IF_BREAK) && in createPHISourceCopy()
8167 get(ST.isWave32() ? AMDGPU::S_MOV_B32_term in createPHISourceCopy()
8168 : AMDGPU::S_MOV_B64_term), in createPHISourceCopy()
8171 .addReg(AMDGPU::EXEC, RegState::Implicit); in createPHISourceCopy()
8203 if (RC->hasSuperClassEq(&AMDGPU::SReg_32RegClass)) { in foldMemoryOperandImpl()
8204 MRI.constrainRegClass(VirtReg, &AMDGPU::SReg_32_XM0_XEXECRegClass); in foldMemoryOperandImpl()
8206 } else if (RC->hasSuperClassEq(&AMDGPU::SReg_64RegClass)) { in foldMemoryOperandImpl()
8207 MRI.constrainRegClass(VirtReg, &AMDGPU::SReg_64_XEXECRegClass); in foldMemoryOperandImpl()
8264 case AMDGPU::S_CMP_EQ_U32: in analyzeCompare()
8265 case AMDGPU::S_CMP_EQ_I32: in analyzeCompare()
8266 case AMDGPU::S_CMP_LG_U32: in analyzeCompare()
8267 case AMDGPU::S_CMP_LG_I32: in analyzeCompare()
8268 case AMDGPU::S_CMP_LT_U32: in analyzeCompare()
8269 case AMDGPU::S_CMP_LT_I32: in analyzeCompare()
8270 case AMDGPU::S_CMP_GT_U32: in analyzeCompare()
8271 case AMDGPU::S_CMP_GT_I32: in analyzeCompare()
8272 case AMDGPU::S_CMP_LE_U32: in analyzeCompare()
8273 case AMDGPU::S_CMP_LE_I32: in analyzeCompare()
8274 case AMDGPU::S_CMP_GE_U32: in analyzeCompare()
8275 case AMDGPU::S_CMP_GE_I32: in analyzeCompare()
8276 case AMDGPU::S_CMP_EQ_U64: in analyzeCompare()
8277 case AMDGPU::S_CMP_LG_U64: in analyzeCompare()
8292 case AMDGPU::S_CMPK_EQ_U32: in analyzeCompare()
8293 case AMDGPU::S_CMPK_EQ_I32: in analyzeCompare()
8294 case AMDGPU::S_CMPK_LG_U32: in analyzeCompare()
8295 case AMDGPU::S_CMPK_LG_I32: in analyzeCompare()
8296 case AMDGPU::S_CMPK_LT_U32: in analyzeCompare()
8297 case AMDGPU::S_CMPK_LT_I32: in analyzeCompare()
8298 case AMDGPU::S_CMPK_GT_U32: in analyzeCompare()
8299 case AMDGPU::S_CMPK_GT_I32: in analyzeCompare()
8300 case AMDGPU::S_CMPK_LE_U32: in analyzeCompare()
8301 case AMDGPU::S_CMPK_LE_I32: in analyzeCompare()
8302 case AMDGPU::S_CMPK_GE_U32: in analyzeCompare()
8303 case AMDGPU::S_CMPK_GE_I32: in analyzeCompare()
8354 if (Def->getOpcode() != AMDGPU::S_AND_B32 && in optimizeCompareInstr()
8355 Def->getOpcode() != AMDGPU::S_AND_B64) in optimizeCompareInstr()
8397 if (I->modifiesRegister(AMDGPU::SCC, &RI) || in optimizeCompareInstr()
8398 I->killsRegister(AMDGPU::SCC, &RI)) in optimizeCompareInstr()
8402 MachineOperand *SccDef = Def->findRegisterDefOperand(AMDGPU::SCC); in optimizeCompareInstr()
8414 unsigned NewOpc = (SrcSize == 32) ? IsReversedCC ? AMDGPU::S_BITCMP0_B32 in optimizeCompareInstr()
8415 : AMDGPU::S_BITCMP1_B32 in optimizeCompareInstr()
8416 : IsReversedCC ? AMDGPU::S_BITCMP0_B64 in optimizeCompareInstr()
8417 : AMDGPU::S_BITCMP1_B64; in optimizeCompareInstr()
8430 case AMDGPU::S_CMP_EQ_U32: in optimizeCompareInstr()
8431 case AMDGPU::S_CMP_EQ_I32: in optimizeCompareInstr()
8432 case AMDGPU::S_CMPK_EQ_U32: in optimizeCompareInstr()
8433 case AMDGPU::S_CMPK_EQ_I32: in optimizeCompareInstr()
8435 case AMDGPU::S_CMP_GE_U32: in optimizeCompareInstr()
8436 case AMDGPU::S_CMPK_GE_U32: in optimizeCompareInstr()
8438 case AMDGPU::S_CMP_GE_I32: in optimizeCompareInstr()
8439 case AMDGPU::S_CMPK_GE_I32: in optimizeCompareInstr()
8441 case AMDGPU::S_CMP_EQ_U64: in optimizeCompareInstr()
8443 case AMDGPU::S_CMP_LG_U32: in optimizeCompareInstr()
8444 case AMDGPU::S_CMP_LG_I32: in optimizeCompareInstr()
8445 case AMDGPU::S_CMPK_LG_U32: in optimizeCompareInstr()
8446 case AMDGPU::S_CMPK_LG_I32: in optimizeCompareInstr()
8448 case AMDGPU::S_CMP_GT_U32: in optimizeCompareInstr()
8449 case AMDGPU::S_CMPK_GT_U32: in optimizeCompareInstr()
8451 case AMDGPU::S_CMP_GT_I32: in optimizeCompareInstr()
8452 case AMDGPU::S_CMPK_GT_I32: in optimizeCompareInstr()
8454 case AMDGPU::S_CMP_LG_U64: in optimizeCompareInstr()
8466 int OpNo = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OpName); in enforceOperandRCAlignment()
8480 IsAGPR ? &AMDGPU::AGPR_32RegClass : &AMDGPU::VGPR_32RegClass); in enforceOperandRCAlignment()
8481 BuildMI(*BB, MI, DL, get(AMDGPU::IMPLICIT_DEF), Undef); in enforceOperandRCAlignment()
8483 MRI.createVirtualRegister(IsAGPR ? &AMDGPU::AReg_64_Align2RegClass in enforceOperandRCAlignment()
8484 : &AMDGPU::VReg_64_Align2RegClass); in enforceOperandRCAlignment()
8485 BuildMI(*BB, MI, DL, get(AMDGPU::REG_SEQUENCE), NewVR) in enforceOperandRCAlignment()
8487 .addImm(AMDGPU::sub0) in enforceOperandRCAlignment()
8489 .addImm(AMDGPU::sub1); in enforceOperandRCAlignment()
8491 Op.setSubReg(AMDGPU::sub0); in enforceOperandRCAlignment()