Lines Matching refs:AMDGPU
43 namespace AMDGPU { namespace
66 : AMDGPUGenInstrInfo(AMDGPU::ADJCALLSTACKUP, AMDGPU::ADJCALLSTACKDOWN), in SIInstrInfo()
88 int Op0Idx = AMDGPU::getNamedOperandIdx(Opc0, OpName); in nodesHaveSameOperandValue()
89 int Op1Idx = AMDGPU::getNamedOperandIdx(Opc1, OpName); in nodesHaveSameOperandValue()
129 return MO.getReg() == AMDGPU::EXEC && MO.isImplicit() && in isIgnorableUse()
159 int Offset0Idx = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset); in areLoadsFromSameBasePtr()
160 int Offset1Idx = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset); in areLoadsFromSameBasePtr()
177 if (AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::sbase) == -1 || in areLoadsFromSameBasePtr()
178 AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::sbase) == -1) in areLoadsFromSameBasePtr()
204 if (!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::soffset) || in areLoadsFromSameBasePtr()
205 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::vaddr) || in areLoadsFromSameBasePtr()
206 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::srsrc)) in areLoadsFromSameBasePtr()
209 int OffIdx0 = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset); in areLoadsFromSameBasePtr()
210 int OffIdx1 = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset); in areLoadsFromSameBasePtr()
238 case AMDGPU::DS_READ2ST64_B32: in isStride64()
239 case AMDGPU::DS_READ2ST64_B64: in isStride64()
240 case AMDGPU::DS_WRITE2ST64_B32: in isStride64()
241 case AMDGPU::DS_WRITE2ST64_B64: in isStride64()
261 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr); in getMemOperandsWithOffsetWidth()
262 OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset); in getMemOperandsWithOffsetWidth()
273 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); in getMemOperandsWithOffsetWidth()
275 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0); in getMemOperandsWithOffsetWidth()
282 getNamedOperand(LdSt, AMDGPU::OpName::offset0); in getMemOperandsWithOffsetWidth()
284 getNamedOperand(LdSt, AMDGPU::OpName::offset1); in getMemOperandsWithOffsetWidth()
299 int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0); in getMemOperandsWithOffsetWidth()
309 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); in getMemOperandsWithOffsetWidth()
311 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0); in getMemOperandsWithOffsetWidth()
313 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data1); in getMemOperandsWithOffsetWidth()
323 const MachineOperand *RSrc = getNamedOperand(LdSt, AMDGPU::OpName::srsrc); in getMemOperandsWithOffsetWidth()
327 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr); in getMemOperandsWithOffsetWidth()
331 getNamedOperand(LdSt, AMDGPU::OpName::offset); in getMemOperandsWithOffsetWidth()
334 getNamedOperand(LdSt, AMDGPU::OpName::soffset); in getMemOperandsWithOffsetWidth()
342 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); in getMemOperandsWithOffsetWidth()
344 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata); in getMemOperandsWithOffsetWidth()
350 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc); in getMemOperandsWithOffsetWidth()
352 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0); in getMemOperandsWithOffsetWidth()
358 BaseOps.push_back(getNamedOperand(LdSt, AMDGPU::OpName::vaddr)); in getMemOperandsWithOffsetWidth()
362 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata); in getMemOperandsWithOffsetWidth()
368 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::sbase); in getMemOperandsWithOffsetWidth()
372 OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset); in getMemOperandsWithOffsetWidth()
375 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::sdst); in getMemOperandsWithOffsetWidth()
382 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr); in getMemOperandsWithOffsetWidth()
385 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::saddr); in getMemOperandsWithOffsetWidth()
388 Offset = getNamedOperand(LdSt, AMDGPU::OpName::offset)->getImm(); in getMemOperandsWithOffsetWidth()
390 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); in getMemOperandsWithOffsetWidth()
392 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata); in getMemOperandsWithOffsetWidth()
496 BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_ILLEGAL_COPY), DestReg) in reportIllegalCopy()
512 assert(AMDGPU::SReg_32RegClass.contains(SrcReg) || in indirectCopyToAGPR()
513 AMDGPU::AGPR_32RegClass.contains(SrcReg)); in indirectCopyToAGPR()
520 if (Def->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64) in indirectCopyToAGPR()
541 BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg) in indirectCopyToAGPR()
559 unsigned MaxVGPRs = RI.getRegPressureLimit(&AMDGPU::VGPR_32RegClass, in indirectCopyToAGPR()
565 Register Tmp = RS.scavengeRegister(&AMDGPU::VGPR_32RegClass, 0); in indirectCopyToAGPR()
574 while (RegNo-- && RS.FindUnusedReg(&AMDGPU::VGPR_32RegClass)) { in indirectCopyToAGPR()
575 Register Tmp2 = RS.scavengeRegister(&AMDGPU::VGPR_32RegClass, 0); in indirectCopyToAGPR()
584 unsigned TmpCopyOp = AMDGPU::V_MOV_B32_e32; in indirectCopyToAGPR()
585 if (AMDGPU::AGPR_32RegClass.contains(SrcReg)) { in indirectCopyToAGPR()
586 TmpCopyOp = AMDGPU::V_ACCVGPR_READ_B32_e64; in indirectCopyToAGPR()
588 assert(AMDGPU::SReg_32RegClass.contains(SrcReg)); in indirectCopyToAGPR()
599 = BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg) in indirectCopyToAGPR()
618 unsigned Opcode = AMDGPU::S_MOV_B32; in expandSGPRCopy()
622 bool AlignedDest = ((Reg - AMDGPU::SGPR0) % 2) == 0; in expandSGPRCopy()
623 bool AlignedSrc = ((Src - AMDGPU::SGPR0) % 2) == 0; in expandSGPRCopy()
628 Opcode = AMDGPU::S_MOV_B64; in expandSGPRCopy()
667 assert(RI.getSubReg(Super, AMDGPU::lo16) == RegToFix); in copyPhysReg()
672 BuildMI(MBB, MI, DL, get(AMDGPU::BUNDLE)); in copyPhysReg()
679 if (RC == &AMDGPU::VGPR_32RegClass) { in copyPhysReg()
680 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) || in copyPhysReg()
681 AMDGPU::SReg_32RegClass.contains(SrcReg) || in copyPhysReg()
682 AMDGPU::AGPR_32RegClass.contains(SrcReg)); in copyPhysReg()
683 unsigned Opc = AMDGPU::AGPR_32RegClass.contains(SrcReg) ? in copyPhysReg()
684 AMDGPU::V_ACCVGPR_READ_B32_e64 : AMDGPU::V_MOV_B32_e32; in copyPhysReg()
690 if (RC == &AMDGPU::SReg_32_XM0RegClass || in copyPhysReg()
691 RC == &AMDGPU::SReg_32RegClass) { in copyPhysReg()
692 if (SrcReg == AMDGPU::SCC) { in copyPhysReg()
693 BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B32), DestReg) in copyPhysReg()
699 if (DestReg == AMDGPU::VCC_LO) { in copyPhysReg()
700 if (AMDGPU::SReg_32RegClass.contains(SrcReg)) { in copyPhysReg()
701 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), AMDGPU::VCC_LO) in copyPhysReg()
705 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg)); in copyPhysReg()
706 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32)) in copyPhysReg()
714 if (!AMDGPU::SReg_32RegClass.contains(SrcReg)) { in copyPhysReg()
719 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg) in copyPhysReg()
724 if (RC == &AMDGPU::SReg_64RegClass) { in copyPhysReg()
725 if (SrcReg == AMDGPU::SCC) { in copyPhysReg()
726 BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B64), DestReg) in copyPhysReg()
732 if (DestReg == AMDGPU::VCC) { in copyPhysReg()
733 if (AMDGPU::SReg_64RegClass.contains(SrcReg)) { in copyPhysReg()
734 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), AMDGPU::VCC) in copyPhysReg()
738 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg)); in copyPhysReg()
739 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32)) in copyPhysReg()
747 if (!AMDGPU::SReg_64RegClass.contains(SrcReg)) { in copyPhysReg()
752 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg) in copyPhysReg()
757 if (DestReg == AMDGPU::SCC) { in copyPhysReg()
760 if (AMDGPU::SReg_64RegClass.contains(SrcReg)) { in copyPhysReg()
765 BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U64)) in copyPhysReg()
769 assert(AMDGPU::SReg_32RegClass.contains(SrcReg)); in copyPhysReg()
770 BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U32)) in copyPhysReg()
778 if (RC == &AMDGPU::AGPR_32RegClass) { in copyPhysReg()
779 if (AMDGPU::VGPR_32RegClass.contains(SrcReg)) { in copyPhysReg()
780 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg) in copyPhysReg()
785 if (AMDGPU::AGPR_32RegClass.contains(SrcReg) && ST.hasGFX90AInsts()) { in copyPhysReg()
786 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_MOV_B32), DestReg) in copyPhysReg()
800 assert(AMDGPU::VGPR_LO16RegClass.contains(SrcReg) || in copyPhysReg()
801 AMDGPU::VGPR_HI16RegClass.contains(SrcReg) || in copyPhysReg()
802 AMDGPU::SReg_LO16RegClass.contains(SrcReg) || in copyPhysReg()
803 AMDGPU::AGPR_LO16RegClass.contains(SrcReg)); in copyPhysReg()
805 bool IsSGPRDst = AMDGPU::SReg_LO16RegClass.contains(DestReg); in copyPhysReg()
806 bool IsSGPRSrc = AMDGPU::SReg_LO16RegClass.contains(SrcReg); in copyPhysReg()
807 bool IsAGPRDst = AMDGPU::AGPR_LO16RegClass.contains(DestReg); in copyPhysReg()
808 bool IsAGPRSrc = AMDGPU::AGPR_LO16RegClass.contains(SrcReg); in copyPhysReg()
809 bool DstLow = AMDGPU::VGPR_LO16RegClass.contains(DestReg) || in copyPhysReg()
810 AMDGPU::SReg_LO16RegClass.contains(DestReg) || in copyPhysReg()
811 AMDGPU::AGPR_LO16RegClass.contains(DestReg); in copyPhysReg()
812 bool SrcLow = AMDGPU::VGPR_LO16RegClass.contains(SrcReg) || in copyPhysReg()
813 AMDGPU::SReg_LO16RegClass.contains(SrcReg) || in copyPhysReg()
814 AMDGPU::AGPR_LO16RegClass.contains(SrcReg); in copyPhysReg()
824 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), NewDestReg) in copyPhysReg()
845 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), NewDestReg) in copyPhysReg()
850 auto MIB = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_sdwa), NewDestReg) in copyPhysReg()
854 .addImm(DstLow ? AMDGPU::SDWA::SdwaSel::WORD_0 in copyPhysReg()
855 : AMDGPU::SDWA::SdwaSel::WORD_1) in copyPhysReg()
856 .addImm(AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE) in copyPhysReg()
857 .addImm(SrcLow ? AMDGPU::SDWA::SdwaSel::WORD_0 in copyPhysReg()
858 : AMDGPU::SDWA::SdwaSel::WORD_1) in copyPhysReg()
868 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), DestReg) in copyPhysReg()
894 unsigned Opcode = AMDGPU::V_MOV_B32_e32; in copyPhysReg()
897 AMDGPU::V_ACCVGPR_WRITE_B32_e64 : AMDGPU::INSTRUCTION_LIST_END; in copyPhysReg()
899 Opcode = AMDGPU::V_ACCVGPR_READ_B32_e64; in copyPhysReg()
905 Opcode = AMDGPU::V_PK_MOV_B32; in copyPhysReg()
916 if (Opcode == AMDGPU::INSTRUCTION_LIST_END) in copyPhysReg()
934 if (Opcode == AMDGPU::INSTRUCTION_LIST_END) { in copyPhysReg()
940 } else if (Opcode == AMDGPU::V_PK_MOV_B32) { in copyPhysReg()
944 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), DstSubReg) in copyPhysReg()
973 NewOpc = AMDGPU::getCommuteRev(Opcode); in commuteOpcode()
979 NewOpc = AMDGPU::getCommuteOrig(Opcode); in commuteOpcode()
993 if (RegClass == &AMDGPU::SReg_32RegClass || in materializeImmediate()
994 RegClass == &AMDGPU::SGPR_32RegClass || in materializeImmediate()
995 RegClass == &AMDGPU::SReg_32_XM0RegClass || in materializeImmediate()
996 RegClass == &AMDGPU::SReg_32_XM0_XEXECRegClass) { in materializeImmediate()
997 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg) in materializeImmediate()
1002 if (RegClass == &AMDGPU::SReg_64RegClass || in materializeImmediate()
1003 RegClass == &AMDGPU::SGPR_64RegClass || in materializeImmediate()
1004 RegClass == &AMDGPU::SReg_64_XEXECRegClass) { in materializeImmediate()
1005 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg) in materializeImmediate()
1010 if (RegClass == &AMDGPU::VGPR_32RegClass) { in materializeImmediate()
1011 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg) in materializeImmediate()
1015 if (RegClass->hasSuperClassEq(&AMDGPU::VReg_64RegClass)) { in materializeImmediate()
1016 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), DestReg) in materializeImmediate()
1022 unsigned Opcode = AMDGPU::V_MOV_B32_e32; in materializeImmediate()
1025 Opcode = AMDGPU::S_MOV_B64; in materializeImmediate()
1028 Opcode = AMDGPU::S_MOV_B32; in materializeImmediate()
1045 return &AMDGPU::VGPR_32RegClass; in getPreferredSelectRegClass()
1056 RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); in insertVectorSelect()
1057 assert(MRI.getRegClass(DstReg) == &AMDGPU::VGPR_32RegClass && in insertVectorSelect()
1062 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg) in insertVectorSelect()
1064 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
1075 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 in insertVectorSelect()
1076 : AMDGPU::S_CSELECT_B64), SReg) in insertVectorSelect()
1079 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
1089 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 in insertVectorSelect()
1090 : AMDGPU::S_CSELECT_B64), SReg) in insertVectorSelect()
1093 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
1105 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg) in insertVectorSelect()
1107 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
1119 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg) in insertVectorSelect()
1121 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
1132 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32 in insertVectorSelect()
1133 : AMDGPU::S_OR_SAVEEXEC_B64), SReg2) in insertVectorSelect()
1135 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 in insertVectorSelect()
1136 : AMDGPU::S_CSELECT_B64), SReg) in insertVectorSelect()
1139 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
1150 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32 in insertVectorSelect()
1151 : AMDGPU::S_OR_SAVEEXEC_B64), SReg2) in insertVectorSelect()
1153 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 in insertVectorSelect()
1154 : AMDGPU::S_CSELECT_B64), SReg) in insertVectorSelect()
1157 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
1180 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_EQ_I32_e64), Reg) in insertEQ()
1193 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_NE_I32_e64), Reg) in insertNE()
1203 return AMDGPU::COPY; in getMovOpcode()
1205 return RI.isSGPRClass(DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32; in getMovOpcode()
1207 return AMDGPU::S_MOV_B64; in getMovOpcode()
1209 return AMDGPU::V_MOV_B64_PSEUDO; in getMovOpcode()
1211 return AMDGPU::COPY; in getMovOpcode()
1219 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1); in getIndirectGPRIDXPseudo()
1221 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2); in getIndirectGPRIDXPseudo()
1223 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3); in getIndirectGPRIDXPseudo()
1225 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4); in getIndirectGPRIDXPseudo()
1227 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5); in getIndirectGPRIDXPseudo()
1229 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8); in getIndirectGPRIDXPseudo()
1231 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16); in getIndirectGPRIDXPseudo()
1233 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32); in getIndirectGPRIDXPseudo()
1239 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1); in getIndirectGPRIDXPseudo()
1241 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2); in getIndirectGPRIDXPseudo()
1243 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3); in getIndirectGPRIDXPseudo()
1245 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4); in getIndirectGPRIDXPseudo()
1247 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5); in getIndirectGPRIDXPseudo()
1249 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8); in getIndirectGPRIDXPseudo()
1251 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16); in getIndirectGPRIDXPseudo()
1253 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32); in getIndirectGPRIDXPseudo()
1260 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1; in getIndirectVGPRWriteMovRelPseudoOpc()
1262 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2; in getIndirectVGPRWriteMovRelPseudoOpc()
1264 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3; in getIndirectVGPRWriteMovRelPseudoOpc()
1266 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4; in getIndirectVGPRWriteMovRelPseudoOpc()
1268 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5; in getIndirectVGPRWriteMovRelPseudoOpc()
1270 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8; in getIndirectVGPRWriteMovRelPseudoOpc()
1272 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16; in getIndirectVGPRWriteMovRelPseudoOpc()
1274 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32; in getIndirectVGPRWriteMovRelPseudoOpc()
1281 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1; in getIndirectSGPRWriteMovRelPseudo32()
1283 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2; in getIndirectSGPRWriteMovRelPseudo32()
1285 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3; in getIndirectSGPRWriteMovRelPseudo32()
1287 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4; in getIndirectSGPRWriteMovRelPseudo32()
1289 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5; in getIndirectSGPRWriteMovRelPseudo32()
1291 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8; in getIndirectSGPRWriteMovRelPseudo32()
1293 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16; in getIndirectSGPRWriteMovRelPseudo32()
1295 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32; in getIndirectSGPRWriteMovRelPseudo32()
1302 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1; in getIndirectSGPRWriteMovRelPseudo64()
1304 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2; in getIndirectSGPRWriteMovRelPseudo64()
1306 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4; in getIndirectSGPRWriteMovRelPseudo64()
1308 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8; in getIndirectSGPRWriteMovRelPseudo64()
1310 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16; in getIndirectSGPRWriteMovRelPseudo64()
1336 return AMDGPU::SI_SPILL_S32_SAVE; in getSGPRSpillSaveOpcode()
1338 return AMDGPU::SI_SPILL_S64_SAVE; in getSGPRSpillSaveOpcode()
1340 return AMDGPU::SI_SPILL_S96_SAVE; in getSGPRSpillSaveOpcode()
1342 return AMDGPU::SI_SPILL_S128_SAVE; in getSGPRSpillSaveOpcode()
1344 return AMDGPU::SI_SPILL_S160_SAVE; in getSGPRSpillSaveOpcode()
1346 return AMDGPU::SI_SPILL_S192_SAVE; in getSGPRSpillSaveOpcode()
1348 return AMDGPU::SI_SPILL_S224_SAVE; in getSGPRSpillSaveOpcode()
1350 return AMDGPU::SI_SPILL_S256_SAVE; in getSGPRSpillSaveOpcode()
1352 return AMDGPU::SI_SPILL_S512_SAVE; in getSGPRSpillSaveOpcode()
1354 return AMDGPU::SI_SPILL_S1024_SAVE; in getSGPRSpillSaveOpcode()
1363 return AMDGPU::SI_SPILL_V32_SAVE; in getVGPRSpillSaveOpcode()
1365 return AMDGPU::SI_SPILL_V64_SAVE; in getVGPRSpillSaveOpcode()
1367 return AMDGPU::SI_SPILL_V96_SAVE; in getVGPRSpillSaveOpcode()
1369 return AMDGPU::SI_SPILL_V128_SAVE; in getVGPRSpillSaveOpcode()
1371 return AMDGPU::SI_SPILL_V160_SAVE; in getVGPRSpillSaveOpcode()
1373 return AMDGPU::SI_SPILL_V192_SAVE; in getVGPRSpillSaveOpcode()
1375 return AMDGPU::SI_SPILL_V224_SAVE; in getVGPRSpillSaveOpcode()
1377 return AMDGPU::SI_SPILL_V256_SAVE; in getVGPRSpillSaveOpcode()
1379 return AMDGPU::SI_SPILL_V512_SAVE; in getVGPRSpillSaveOpcode()
1381 return AMDGPU::SI_SPILL_V1024_SAVE; in getVGPRSpillSaveOpcode()
1390 return AMDGPU::SI_SPILL_A32_SAVE; in getAGPRSpillSaveOpcode()
1392 return AMDGPU::SI_SPILL_A64_SAVE; in getAGPRSpillSaveOpcode()
1394 return AMDGPU::SI_SPILL_A96_SAVE; in getAGPRSpillSaveOpcode()
1396 return AMDGPU::SI_SPILL_A128_SAVE; in getAGPRSpillSaveOpcode()
1398 return AMDGPU::SI_SPILL_A160_SAVE; in getAGPRSpillSaveOpcode()
1400 return AMDGPU::SI_SPILL_A192_SAVE; in getAGPRSpillSaveOpcode()
1402 return AMDGPU::SI_SPILL_A224_SAVE; in getAGPRSpillSaveOpcode()
1404 return AMDGPU::SI_SPILL_A256_SAVE; in getAGPRSpillSaveOpcode()
1406 return AMDGPU::SI_SPILL_A512_SAVE; in getAGPRSpillSaveOpcode()
1408 return AMDGPU::SI_SPILL_A1024_SAVE; in getAGPRSpillSaveOpcode()
1434 assert(SrcReg != AMDGPU::M0 && "m0 should not be spilled"); in storeRegToStackSlot()
1435 assert(SrcReg != AMDGPU::EXEC_LO && SrcReg != AMDGPU::EXEC_HI && in storeRegToStackSlot()
1436 SrcReg != AMDGPU::EXEC && "exec should not be spilled"); in storeRegToStackSlot()
1446 MRI.constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0_XEXECRegClass); in storeRegToStackSlot()
1475 return AMDGPU::SI_SPILL_S32_RESTORE; in getSGPRSpillRestoreOpcode()
1477 return AMDGPU::SI_SPILL_S64_RESTORE; in getSGPRSpillRestoreOpcode()
1479 return AMDGPU::SI_SPILL_S96_RESTORE; in getSGPRSpillRestoreOpcode()
1481 return AMDGPU::SI_SPILL_S128_RESTORE; in getSGPRSpillRestoreOpcode()
1483 return AMDGPU::SI_SPILL_S160_RESTORE; in getSGPRSpillRestoreOpcode()
1485 return AMDGPU::SI_SPILL_S192_RESTORE; in getSGPRSpillRestoreOpcode()
1487 return AMDGPU::SI_SPILL_S224_RESTORE; in getSGPRSpillRestoreOpcode()
1489 return AMDGPU::SI_SPILL_S256_RESTORE; in getSGPRSpillRestoreOpcode()
1491 return AMDGPU::SI_SPILL_S512_RESTORE; in getSGPRSpillRestoreOpcode()
1493 return AMDGPU::SI_SPILL_S1024_RESTORE; in getSGPRSpillRestoreOpcode()
1502 return AMDGPU::SI_SPILL_V32_RESTORE; in getVGPRSpillRestoreOpcode()
1504 return AMDGPU::SI_SPILL_V64_RESTORE; in getVGPRSpillRestoreOpcode()
1506 return AMDGPU::SI_SPILL_V96_RESTORE; in getVGPRSpillRestoreOpcode()
1508 return AMDGPU::SI_SPILL_V128_RESTORE; in getVGPRSpillRestoreOpcode()
1510 return AMDGPU::SI_SPILL_V160_RESTORE; in getVGPRSpillRestoreOpcode()
1512 return AMDGPU::SI_SPILL_V192_RESTORE; in getVGPRSpillRestoreOpcode()
1514 return AMDGPU::SI_SPILL_V224_RESTORE; in getVGPRSpillRestoreOpcode()
1516 return AMDGPU::SI_SPILL_V256_RESTORE; in getVGPRSpillRestoreOpcode()
1518 return AMDGPU::SI_SPILL_V512_RESTORE; in getVGPRSpillRestoreOpcode()
1520 return AMDGPU::SI_SPILL_V1024_RESTORE; in getVGPRSpillRestoreOpcode()
1529 return AMDGPU::SI_SPILL_A32_RESTORE; in getAGPRSpillRestoreOpcode()
1531 return AMDGPU::SI_SPILL_A64_RESTORE; in getAGPRSpillRestoreOpcode()
1533 return AMDGPU::SI_SPILL_A96_RESTORE; in getAGPRSpillRestoreOpcode()
1535 return AMDGPU::SI_SPILL_A128_RESTORE; in getAGPRSpillRestoreOpcode()
1537 return AMDGPU::SI_SPILL_A160_RESTORE; in getAGPRSpillRestoreOpcode()
1539 return AMDGPU::SI_SPILL_A192_RESTORE; in getAGPRSpillRestoreOpcode()
1541 return AMDGPU::SI_SPILL_A224_RESTORE; in getAGPRSpillRestoreOpcode()
1543 return AMDGPU::SI_SPILL_A256_RESTORE; in getAGPRSpillRestoreOpcode()
1545 return AMDGPU::SI_SPILL_A512_RESTORE; in getAGPRSpillRestoreOpcode()
1547 return AMDGPU::SI_SPILL_A1024_RESTORE; in getAGPRSpillRestoreOpcode()
1573 assert(DestReg != AMDGPU::M0 && "m0 should not be reloaded into"); in loadRegFromStackSlot()
1574 assert(DestReg != AMDGPU::EXEC_LO && DestReg != AMDGPU::EXEC_HI && in loadRegFromStackSlot()
1575 DestReg != AMDGPU::EXEC && "exec should not be spilled"); in loadRegFromStackSlot()
1582 MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0_XEXECRegClass); in loadRegFromStackSlot()
1616 BuildMI(MBB, MI, DL, get(AMDGPU::S_NOP)).addImm(Arg - 1); in insertNoops()
1630 BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::S_ENDPGM)).addImm(0); in insertReturn()
1632 BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::SI_RETURN_TO_EPILOG)); in insertReturn()
1642 case AMDGPU::S_NOP: in getNumWaitStates()
1653 case AMDGPU::S_MOV_B64_term: in expandPostRAPseudo()
1656 MI.setDesc(get(AMDGPU::S_MOV_B64)); in expandPostRAPseudo()
1659 case AMDGPU::S_MOV_B32_term: in expandPostRAPseudo()
1662 MI.setDesc(get(AMDGPU::S_MOV_B32)); in expandPostRAPseudo()
1665 case AMDGPU::S_XOR_B64_term: in expandPostRAPseudo()
1668 MI.setDesc(get(AMDGPU::S_XOR_B64)); in expandPostRAPseudo()
1671 case AMDGPU::S_XOR_B32_term: in expandPostRAPseudo()
1674 MI.setDesc(get(AMDGPU::S_XOR_B32)); in expandPostRAPseudo()
1676 case AMDGPU::S_OR_B64_term: in expandPostRAPseudo()
1679 MI.setDesc(get(AMDGPU::S_OR_B64)); in expandPostRAPseudo()
1681 case AMDGPU::S_OR_B32_term: in expandPostRAPseudo()
1684 MI.setDesc(get(AMDGPU::S_OR_B32)); in expandPostRAPseudo()
1687 case AMDGPU::S_ANDN2_B64_term: in expandPostRAPseudo()
1690 MI.setDesc(get(AMDGPU::S_ANDN2_B64)); in expandPostRAPseudo()
1693 case AMDGPU::S_ANDN2_B32_term: in expandPostRAPseudo()
1696 MI.setDesc(get(AMDGPU::S_ANDN2_B32)); in expandPostRAPseudo()
1699 case AMDGPU::S_AND_B64_term: in expandPostRAPseudo()
1702 MI.setDesc(get(AMDGPU::S_AND_B64)); in expandPostRAPseudo()
1705 case AMDGPU::S_AND_B32_term: in expandPostRAPseudo()
1708 MI.setDesc(get(AMDGPU::S_AND_B32)); in expandPostRAPseudo()
1711 case AMDGPU::V_MOV_B64_PSEUDO: { in expandPostRAPseudo()
1713 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0); in expandPostRAPseudo()
1714 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1); in expandPostRAPseudo()
1724 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), Dst) in expandPostRAPseudo()
1735 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo) in expandPostRAPseudo()
1738 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi) in expandPostRAPseudo()
1746 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), Dst) in expandPostRAPseudo()
1757 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo) in expandPostRAPseudo()
1758 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub0)) in expandPostRAPseudo()
1760 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi) in expandPostRAPseudo()
1761 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub1)) in expandPostRAPseudo()
1768 case AMDGPU::V_MOV_B64_DPP_PSEUDO: { in expandPostRAPseudo()
1772 case AMDGPU::S_MOV_B64_IMM_PSEUDO: { in expandPostRAPseudo()
1777 MI.setDesc(get(AMDGPU::S_MOV_B64)); in expandPostRAPseudo()
1782 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0); in expandPostRAPseudo()
1783 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1); in expandPostRAPseudo()
1787 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DstLo) in expandPostRAPseudo()
1790 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DstHi) in expandPostRAPseudo()
1796 case AMDGPU::V_SET_INACTIVE_B32: { in expandPostRAPseudo()
1797 unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64; in expandPostRAPseudo()
1798 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; in expandPostRAPseudo()
1800 FirstNot->addRegisterDead(AMDGPU::SCC, TRI); // SCC is overwritten in expandPostRAPseudo()
1801 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg()) in expandPostRAPseudo()
1808 case AMDGPU::V_SET_INACTIVE_B64: { in expandPostRAPseudo()
1809 unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64; in expandPostRAPseudo()
1810 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; in expandPostRAPseudo()
1812 FirstNot->addRegisterDead(AMDGPU::SCC, TRI); // SCC is overwritten in expandPostRAPseudo()
1813 MachineInstr *Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), in expandPostRAPseudo()
1822 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1: in expandPostRAPseudo()
1823 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2: in expandPostRAPseudo()
1824 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3: in expandPostRAPseudo()
1825 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4: in expandPostRAPseudo()
1826 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5: in expandPostRAPseudo()
1827 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8: in expandPostRAPseudo()
1828 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16: in expandPostRAPseudo()
1829 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32: in expandPostRAPseudo()
1830 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1: in expandPostRAPseudo()
1831 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2: in expandPostRAPseudo()
1832 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3: in expandPostRAPseudo()
1833 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4: in expandPostRAPseudo()
1834 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5: in expandPostRAPseudo()
1835 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8: in expandPostRAPseudo()
1836 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16: in expandPostRAPseudo()
1837 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32: in expandPostRAPseudo()
1838 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1: in expandPostRAPseudo()
1839 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2: in expandPostRAPseudo()
1840 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4: in expandPostRAPseudo()
1841 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8: in expandPostRAPseudo()
1842 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16: { in expandPostRAPseudo()
1847 Opc = AMDGPU::V_MOVRELD_B32_e32; in expandPostRAPseudo()
1849 Opc = RI.getRegSizeInBits(*EltRC) == 64 ? AMDGPU::S_MOVRELD_B64 in expandPostRAPseudo()
1850 : AMDGPU::S_MOVRELD_B32; in expandPostRAPseudo()
1873 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1: in expandPostRAPseudo()
1874 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2: in expandPostRAPseudo()
1875 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3: in expandPostRAPseudo()
1876 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4: in expandPostRAPseudo()
1877 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5: in expandPostRAPseudo()
1878 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8: in expandPostRAPseudo()
1879 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16: in expandPostRAPseudo()
1880 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32: { in expandPostRAPseudo()
1887 MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON)) in expandPostRAPseudo()
1889 .addImm(AMDGPU::VGPRIndexMode::DST_ENABLE); in expandPostRAPseudo()
1892 const MCInstrDesc &OpDesc = get(AMDGPU::V_MOV_B32_indirect); in expandPostRAPseudo()
1905 MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF)); in expandPostRAPseudo()
1912 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1: in expandPostRAPseudo()
1913 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2: in expandPostRAPseudo()
1914 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3: in expandPostRAPseudo()
1915 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4: in expandPostRAPseudo()
1916 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5: in expandPostRAPseudo()
1917 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8: in expandPostRAPseudo()
1918 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16: in expandPostRAPseudo()
1919 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32: { in expandPostRAPseudo()
1927 MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON)) in expandPostRAPseudo()
1929 .addImm(AMDGPU::VGPRIndexMode::SRC0_ENABLE); in expandPostRAPseudo()
1932 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32)) in expandPostRAPseudo()
1936 .addReg(AMDGPU::M0, RegState::Implicit); in expandPostRAPseudo()
1938 MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF)); in expandPostRAPseudo()
1945 case AMDGPU::SI_PC_ADD_REL_OFFSET: { in expandPostRAPseudo()
1948 Register RegLo = RI.getSubReg(Reg, AMDGPU::sub0); in expandPostRAPseudo()
1949 Register RegHi = RI.getSubReg(Reg, AMDGPU::sub1); in expandPostRAPseudo()
1954 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_GETPC_B64), Reg)); in expandPostRAPseudo()
1958 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_ADD_U32), RegLo) in expandPostRAPseudo()
1962 MachineInstrBuilder MIB = BuildMI(MF, DL, get(AMDGPU::S_ADDC_U32), RegHi) in expandPostRAPseudo()
1972 case AMDGPU::ENTER_STRICT_WWM: { in expandPostRAPseudo()
1975 MI.setDesc(get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32 in expandPostRAPseudo()
1976 : AMDGPU::S_OR_SAVEEXEC_B64)); in expandPostRAPseudo()
1979 case AMDGPU::ENTER_STRICT_WQM: { in expandPostRAPseudo()
1982 const unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; in expandPostRAPseudo()
1983 const unsigned WQMOp = ST.isWave32() ? AMDGPU::S_WQM_B32 : AMDGPU::S_WQM_B64; in expandPostRAPseudo()
1984 const unsigned MovOp = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; in expandPostRAPseudo()
1991 case AMDGPU::EXIT_STRICT_WWM: in expandPostRAPseudo()
1992 case AMDGPU::EXIT_STRICT_WQM: { in expandPostRAPseudo()
1995 MI.setDesc(get(ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64)); in expandPostRAPseudo()
2004 assert (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO); in expandMovDPP64()
2014 for (auto Sub : { AMDGPU::sub0, AMDGPU::sub1 }) { in expandMovDPP64()
2015 auto MovDPP = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_dpp)); in expandMovDPP64()
2020 auto Tmp = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in expandMovDPP64()
2049 BuildMI(MBB, MI, DL, get(AMDGPU::REG_SEQUENCE), Dst) in expandMovDPP64()
2051 .addImm(AMDGPU::sub0) in expandMovDPP64()
2053 .addImm(AMDGPU::sub1); in expandMovDPP64()
2119 assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) == in commuteInstructionImpl()
2121 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) == in commuteInstructionImpl()
2149 swapSourceModifiers(MI, Src0, AMDGPU::OpName::src0_modifiers, in commuteInstructionImpl()
2150 Src1, AMDGPU::OpName::src1_modifiers); in commuteInstructionImpl()
2173 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); in findCommutedOpIndices()
2177 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); in findCommutedOpIndices()
2188 assert(BranchOp != AMDGPU::S_SETPC_B64); in isBranchOffsetInRange()
2202 if (MI.getOpcode() == AMDGPU::S_SETPC_B64) { in getBranchDestBlock()
2226 Register PCReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in insertIndirectBranch()
2232 MachineInstr *GetPC = BuildMI(MBB, I, DL, get(AMDGPU::S_GETPC_B64), PCReg); in insertIndirectBranch()
2243 BuildMI(MBB, I, DL, get(AMDGPU::S_ADD_U32)) in insertIndirectBranch()
2244 .addReg(PCReg, RegState::Define, AMDGPU::sub0) in insertIndirectBranch()
2245 .addReg(PCReg, 0, AMDGPU::sub0) in insertIndirectBranch()
2247 BuildMI(MBB, I, DL, get(AMDGPU::S_ADDC_U32)) in insertIndirectBranch()
2248 .addReg(PCReg, RegState::Define, AMDGPU::sub1) in insertIndirectBranch()
2249 .addReg(PCReg, 0, AMDGPU::sub1) in insertIndirectBranch()
2253 BuildMI(&MBB, DL, get(AMDGPU::S_SETPC_B64)) in insertIndirectBranch()
2302 AMDGPU::SReg_64RegClass, in insertIndirectBranch()
2323 return AMDGPU::S_CBRANCH_SCC1; in getBranchOpcode()
2325 return AMDGPU::S_CBRANCH_SCC0; in getBranchOpcode()
2327 return AMDGPU::S_CBRANCH_VCCNZ; in getBranchOpcode()
2329 return AMDGPU::S_CBRANCH_VCCZ; in getBranchOpcode()
2331 return AMDGPU::S_CBRANCH_EXECNZ; in getBranchOpcode()
2333 return AMDGPU::S_CBRANCH_EXECZ; in getBranchOpcode()
2341 case AMDGPU::S_CBRANCH_SCC0: in getBranchPredicate()
2343 case AMDGPU::S_CBRANCH_SCC1: in getBranchPredicate()
2345 case AMDGPU::S_CBRANCH_VCCNZ: in getBranchPredicate()
2347 case AMDGPU::S_CBRANCH_VCCZ: in getBranchPredicate()
2349 case AMDGPU::S_CBRANCH_EXECNZ: in getBranchPredicate()
2351 case AMDGPU::S_CBRANCH_EXECZ: in getBranchPredicate()
2364 if (I->getOpcode() == AMDGPU::S_BRANCH) { in analyzeBranchImpl()
2372 if (I->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) { in analyzeBranchImpl()
2392 if (I->getOpcode() == AMDGPU::S_BRANCH) { in analyzeBranchImpl()
2414 case AMDGPU::S_MOV_B64_term: in analyzeBranch()
2415 case AMDGPU::S_XOR_B64_term: in analyzeBranch()
2416 case AMDGPU::S_OR_B64_term: in analyzeBranch()
2417 case AMDGPU::S_ANDN2_B64_term: in analyzeBranch()
2418 case AMDGPU::S_AND_B64_term: in analyzeBranch()
2419 case AMDGPU::S_MOV_B32_term: in analyzeBranch()
2420 case AMDGPU::S_XOR_B32_term: in analyzeBranch()
2421 case AMDGPU::S_OR_B32_term: in analyzeBranch()
2422 case AMDGPU::S_ANDN2_B32_term: in analyzeBranch()
2423 case AMDGPU::S_AND_B32_term: in analyzeBranch()
2425 case AMDGPU::SI_IF: in analyzeBranch()
2426 case AMDGPU::SI_ELSE: in analyzeBranch()
2427 case AMDGPU::SI_KILL_I1_TERMINATOR: in analyzeBranch()
2428 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR: in analyzeBranch()
2478 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH)) in insertBranch()
2486 BuildMI(&MBB, DL, get(AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO)) in insertBranch()
2518 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH)) in insertBranch()
2558 int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32; in canInsertSelect()
2573 int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32; in canInsertSelect()
2604 Select = BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B32), DstReg) in insertSelect()
2609 Select = BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e32), DstReg) in insertSelect()
2620 BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), DstReg) in insertSelect()
2629 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3, in insertSelect()
2630 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7, in insertSelect()
2631 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11, in insertSelect()
2632 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15, in insertSelect()
2636 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3, in insertSelect()
2637 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7, in insertSelect()
2638 AMDGPU::sub8_sub9, AMDGPU::sub10_sub11, in insertSelect()
2639 AMDGPU::sub12_sub13, AMDGPU::sub14_sub15, in insertSelect()
2642 unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32; in insertSelect()
2643 const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass; in insertSelect()
2651 SelOp = AMDGPU::S_CSELECT_B32; in insertSelect()
2652 EltRC = &AMDGPU::SGPR_32RegClass; in insertSelect()
2654 SelOp = AMDGPU::S_CSELECT_B64; in insertSelect()
2655 EltRC = &AMDGPU::SGPR_64RegClass; in insertSelect()
2662 MBB, I, DL, get(AMDGPU::REG_SEQUENCE), DstReg); in insertSelect()
2674 if (SelOp == AMDGPU::V_CNDMASK_B32_e32) { in insertSelect()
2696 case AMDGPU::V_MOV_B32_e32: in isFoldableCopy()
2697 case AMDGPU::V_MOV_B32_e64: in isFoldableCopy()
2698 case AMDGPU::V_MOV_B64_PSEUDO: { in isFoldableCopy()
2706 case AMDGPU::S_MOV_B32: in isFoldableCopy()
2707 case AMDGPU::S_MOV_B64: in isFoldableCopy()
2708 case AMDGPU::COPY: in isFoldableCopy()
2709 case AMDGPU::V_ACCVGPR_WRITE_B32_e64: in isFoldableCopy()
2710 case AMDGPU::V_ACCVGPR_READ_B32_e64: in isFoldableCopy()
2711 case AMDGPU::V_ACCVGPR_MOV_B32: in isFoldableCopy()
2737 int Src0ModIdx = AMDGPU::getNamedOperandIdx(Opc, in removeModOperands()
2738 AMDGPU::OpName::src0_modifiers); in removeModOperands()
2739 int Src1ModIdx = AMDGPU::getNamedOperandIdx(Opc, in removeModOperands()
2740 AMDGPU::OpName::src1_modifiers); in removeModOperands()
2741 int Src2ModIdx = AMDGPU::getNamedOperandIdx(Opc, in removeModOperands()
2742 AMDGPU::OpName::src2_modifiers); in removeModOperands()
2757 case AMDGPU::S_MOV_B64: in FoldImmediate()
2762 case AMDGPU::V_MOV_B32_e32: in FoldImmediate()
2763 case AMDGPU::S_MOV_B32: in FoldImmediate()
2764 case AMDGPU::V_ACCVGPR_WRITE_B32_e64: in FoldImmediate()
2768 const MachineOperand *ImmOp = getNamedOperand(DefMI, AMDGPU::OpName::src0); in FoldImmediate()
2775 if (Opc == AMDGPU::COPY) { in FoldImmediate()
2779 unsigned NewOpc = isVGPRCopy ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32; in FoldImmediate()
2782 if (UseMI.getOperand(1).getSubReg() == AMDGPU::hi16) in FoldImmediate()
2788 NewOpc = AMDGPU::V_ACCVGPR_WRITE_B32_e64; in FoldImmediate()
2796 UseMI.getOperand(0).getSubReg() != AMDGPU::lo16) in FoldImmediate()
2813 if (Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 || in FoldImmediate()
2814 Opc == AMDGPU::V_MAD_F16_e64 || Opc == AMDGPU::V_MAC_F16_e64 || in FoldImmediate()
2815 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 || in FoldImmediate()
2816 Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64) { in FoldImmediate()
2825 MachineOperand *Src0 = getNamedOperand(UseMI, AMDGPU::OpName::src0); in FoldImmediate()
2831 bool IsF32 = Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 || in FoldImmediate()
2832 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64; in FoldImmediate()
2833 bool IsFMA = Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 || in FoldImmediate()
2834 Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64; in FoldImmediate()
2835 MachineOperand *Src1 = getNamedOperand(UseMI, AMDGPU::OpName::src1); in FoldImmediate()
2836 MachineOperand *Src2 = getNamedOperand(UseMI, AMDGPU::OpName::src2); in FoldImmediate()
2848 IsFMA ? (IsF32 ? AMDGPU::V_FMAMK_F32 : AMDGPU::V_FMAMK_F16) in FoldImmediate()
2849 : (IsF32 ? AMDGPU::V_MADMK_F32 : AMDGPU::V_MADMK_F16); in FoldImmediate()
2862 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod)); in FoldImmediate()
2864 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp)); in FoldImmediate()
2872 if (Opc == AMDGPU::V_MAC_F32_e64 || in FoldImmediate()
2873 Opc == AMDGPU::V_MAC_F16_e64 || in FoldImmediate()
2874 Opc == AMDGPU::V_FMAC_F32_e64 || in FoldImmediate()
2875 Opc == AMDGPU::V_FMAC_F16_e64) in FoldImmediate()
2877 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)); in FoldImmediate()
2933 IsFMA ? (IsF32 ? AMDGPU::V_FMAAK_F32 : AMDGPU::V_FMAAK_F16) in FoldImmediate()
2934 : (IsF32 ? AMDGPU::V_MADAK_F32 : AMDGPU::V_MADAK_F16); in FoldImmediate()
2945 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod)); in FoldImmediate()
2947 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp)); in FoldImmediate()
2949 if (Opc == AMDGPU::V_MAC_F32_e64 || in FoldImmediate()
2950 Opc == AMDGPU::V_MAC_F16_e64 || in FoldImmediate()
2951 Opc == AMDGPU::V_FMAC_F32_e64 || in FoldImmediate()
2952 Opc == AMDGPU::V_FMAC_F16_e64) in FoldImmediate()
2954 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)); in FoldImmediate()
3078 if (Def && Def->getOpcode() == AMDGPU::V_MOV_B32_e32 && in getFoldableImm()
3081 return AMDGPU::NoRegister; in getFoldableImm()
3101 bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F32_e64 || in convertToThreeAddress()
3102 Opc == AMDGPU::V_FMAC_F16_e32 || Opc == AMDGPU::V_FMAC_F16_e64 || in convertToThreeAddress()
3103 Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64; in convertToThreeAddress()
3104 bool IsF64 = Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64; in convertToThreeAddress()
3109 case AMDGPU::V_MAC_F16_e64: in convertToThreeAddress()
3110 case AMDGPU::V_FMAC_F16_e64: in convertToThreeAddress()
3113 case AMDGPU::V_MAC_F32_e64: in convertToThreeAddress()
3114 case AMDGPU::V_FMAC_F32_e64: in convertToThreeAddress()
3115 case AMDGPU::V_FMAC_F64_e64: in convertToThreeAddress()
3117 case AMDGPU::V_MAC_F16_e32: in convertToThreeAddress()
3118 case AMDGPU::V_FMAC_F16_e32: in convertToThreeAddress()
3121 case AMDGPU::V_MAC_F32_e32: in convertToThreeAddress()
3122 case AMDGPU::V_FMAC_F32_e32: in convertToThreeAddress()
3123 case AMDGPU::V_FMAC_F64_e32: { in convertToThreeAddress()
3124 int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), in convertToThreeAddress()
3125 AMDGPU::OpName::src0); in convertToThreeAddress()
3137 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst); in convertToThreeAddress()
3138 const MachineOperand *Src0 = getNamedOperand(MI, AMDGPU::OpName::src0); in convertToThreeAddress()
3140 getNamedOperand(MI, AMDGPU::OpName::src0_modifiers); in convertToThreeAddress()
3141 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1); in convertToThreeAddress()
3143 getNamedOperand(MI, AMDGPU::OpName::src1_modifiers); in convertToThreeAddress()
3144 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); in convertToThreeAddress()
3145 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp); in convertToThreeAddress()
3146 const MachineOperand *Omod = getNamedOperand(MI, AMDGPU::OpName::omod); in convertToThreeAddress()
3155 IsFMA ? (IsF16 ? AMDGPU::V_FMAAK_F16 : AMDGPU::V_FMAAK_F32) in convertToThreeAddress()
3156 : (IsF16 ? AMDGPU::V_MADAK_F16 : AMDGPU::V_MADAK_F32); in convertToThreeAddress()
3168 ? (IsF16 ? AMDGPU::V_FMAMK_F16 : AMDGPU::V_FMAMK_F32) in convertToThreeAddress()
3169 : (IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32); in convertToThreeAddress()
3184 MI, AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::src0), in convertToThreeAddress()
3197 unsigned NewOpc = IsFMA ? (IsF16 ? AMDGPU::V_FMA_F16_e64 in convertToThreeAddress()
3198 : IsF64 ? AMDGPU::V_FMA_F64_e64 in convertToThreeAddress()
3199 : AMDGPU::V_FMA_F32_e64) in convertToThreeAddress()
3200 : (IsF16 ? AMDGPU::V_MAD_F16_e64 : AMDGPU::V_MAD_F32_e64); in convertToThreeAddress()
3223 case AMDGPU::S_SET_GPR_IDX_ON: in changesVGPRIndexingMode()
3224 case AMDGPU::S_SET_GPR_IDX_MODE: in changesVGPRIndexingMode()
3225 case AMDGPU::S_SET_GPR_IDX_OFF: in changesVGPRIndexingMode()
3253 return MI.modifiesRegister(AMDGPU::EXEC, &RI) || in isSchedulingBoundary()
3254 MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 || in isSchedulingBoundary()
3255 MI.getOpcode() == AMDGPU::S_SETREG_B32 || in isSchedulingBoundary()
3260 return Opcode == AMDGPU::DS_ORDERED_COUNT || in isAlwaysGDS()
3261 Opcode == AMDGPU::DS_GWS_INIT || in isAlwaysGDS()
3262 Opcode == AMDGPU::DS_GWS_SEMA_V || in isAlwaysGDS()
3263 Opcode == AMDGPU::DS_GWS_SEMA_BR || in isAlwaysGDS()
3264 Opcode == AMDGPU::DS_GWS_SEMA_P || in isAlwaysGDS()
3265 Opcode == AMDGPU::DS_GWS_SEMA_RELEASE_ALL || in isAlwaysGDS()
3266 Opcode == AMDGPU::DS_GWS_BARRIER; in isAlwaysGDS()
3275 if (*ImpDef == AMDGPU::MODE) in modifiesModeRegister()
3299 if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT || in hasUnwantedEffectsWhenEXECEmpty()
3301 Opcode == AMDGPU::DS_ORDERED_COUNT || Opcode == AMDGPU::S_TRAP || in hasUnwantedEffectsWhenEXECEmpty()
3302 Opcode == AMDGPU::DS_GWS_INIT || Opcode == AMDGPU::DS_GWS_BARRIER) in hasUnwantedEffectsWhenEXECEmpty()
3317 if (Opcode == AMDGPU::V_READFIRSTLANE_B32 || in hasUnwantedEffectsWhenEXECEmpty()
3318 Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32) in hasUnwantedEffectsWhenEXECEmpty()
3335 return MI.readsRegister(AMDGPU::EXEC, &RI); in mayReadEXEC()
3346 return !isSALU(MI) || MI.readsRegister(AMDGPU::EXEC, &RI); in mayReadEXEC()
3355 return AMDGPU::isInlinableLiteral32(Imm.getSExtValue(), in isInlineConstant()
3358 return AMDGPU::isInlinableLiteral64(Imm.getSExtValue(), in isInlineConstant()
3362 AMDGPU::isInlinableLiteral16(Imm.getSExtValue(), in isInlineConstant()
3372 OperandType < AMDGPU::OPERAND_SRC_FIRST || in isInlineConstant()
3373 OperandType > AMDGPU::OPERAND_SRC_LAST) in isInlineConstant()
3383 case AMDGPU::OPERAND_REG_IMM_INT32: in isInlineConstant()
3384 case AMDGPU::OPERAND_REG_IMM_FP32: in isInlineConstant()
3385 case AMDGPU::OPERAND_REG_INLINE_C_INT32: in isInlineConstant()
3386 case AMDGPU::OPERAND_REG_INLINE_C_FP32: in isInlineConstant()
3387 case AMDGPU::OPERAND_REG_IMM_V2FP32: in isInlineConstant()
3388 case AMDGPU::OPERAND_REG_INLINE_C_V2FP32: in isInlineConstant()
3389 case AMDGPU::OPERAND_REG_IMM_V2INT32: in isInlineConstant()
3390 case AMDGPU::OPERAND_REG_INLINE_C_V2INT32: in isInlineConstant()
3391 case AMDGPU::OPERAND_REG_INLINE_AC_INT32: in isInlineConstant()
3392 case AMDGPU::OPERAND_REG_INLINE_AC_FP32: { in isInlineConstant()
3394 return AMDGPU::isInlinableLiteral32(Trunc, ST.hasInv2PiInlineImm()); in isInlineConstant()
3396 case AMDGPU::OPERAND_REG_IMM_INT64: in isInlineConstant()
3397 case AMDGPU::OPERAND_REG_IMM_FP64: in isInlineConstant()
3398 case AMDGPU::OPERAND_REG_INLINE_C_INT64: in isInlineConstant()
3399 case AMDGPU::OPERAND_REG_INLINE_C_FP64: in isInlineConstant()
3400 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: in isInlineConstant()
3401 return AMDGPU::isInlinableLiteral64(MO.getImm(), in isInlineConstant()
3403 case AMDGPU::OPERAND_REG_IMM_INT16: in isInlineConstant()
3404 case AMDGPU::OPERAND_REG_INLINE_C_INT16: in isInlineConstant()
3405 case AMDGPU::OPERAND_REG_INLINE_AC_INT16: in isInlineConstant()
3416 return AMDGPU::isInlinableIntLiteral(Imm); in isInlineConstant()
3417 case AMDGPU::OPERAND_REG_IMM_V2INT16: in isInlineConstant()
3418 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16: in isInlineConstant()
3419 case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16: in isInlineConstant()
3421 return AMDGPU::isInlinableIntLiteralV216(Imm); in isInlineConstant()
3422 case AMDGPU::OPERAND_REG_IMM_FP16: in isInlineConstant()
3423 case AMDGPU::OPERAND_REG_INLINE_C_FP16: in isInlineConstant()
3424 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: { in isInlineConstant()
3432 AMDGPU::isInlinableLiteral16(Trunc, ST.hasInv2PiInlineImm()); in isInlineConstant()
3437 case AMDGPU::OPERAND_REG_IMM_V2FP16: in isInlineConstant()
3438 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: in isInlineConstant()
3439 case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16: { in isInlineConstant()
3441 return AMDGPU::isInlinableLiteralV216(Trunc, ST.hasInv2PiInlineImm()); in isInlineConstant()
3496 OpNo ==(unsigned)AMDGPU::getNamedOperandIdx(MI.getOpcode(), in isImmOperandLegal()
3497 AMDGPU::OpName::src2)) in isImmOperandLegal()
3505 if (!isVOP3(MI) || !AMDGPU::isSISrcOperand(InstDesc, OpNo)) in isImmOperandLegal()
3513 if (Opcode == AMDGPU::V_MUL_LEGACY_F32_e64 && ST.hasGFX90AInsts()) in hasVALU32BitEncoding()
3516 int Op32 = AMDGPU::getVOPe32(Opcode); in hasVALU32BitEncoding()
3527 return AMDGPU::getNamedOperandIdx(Opcode, in hasModifiers()
3528 AMDGPU::OpName::src0_modifiers) != -1; in hasModifiers()
3538 return hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) || in hasAnyModifiersSet()
3539 hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) || in hasAnyModifiersSet()
3540 hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers) || in hasAnyModifiersSet()
3541 hasModifiersSet(MI, AMDGPU::OpName::clamp) || in hasAnyModifiersSet()
3542 hasModifiersSet(MI, AMDGPU::OpName::omod); in hasAnyModifiersSet()
3547 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); in canShrink()
3559 case AMDGPU::V_ADDC_U32_e64: in canShrink()
3560 case AMDGPU::V_SUBB_U32_e64: in canShrink()
3561 case AMDGPU::V_SUBBREV_U32_e64: { in canShrink()
3563 = getNamedOperand(MI, AMDGPU::OpName::src1); in canShrink()
3569 case AMDGPU::V_MAC_F32_e64: in canShrink()
3570 case AMDGPU::V_MAC_F16_e64: in canShrink()
3571 case AMDGPU::V_FMAC_F32_e64: in canShrink()
3572 case AMDGPU::V_FMAC_F16_e64: in canShrink()
3573 case AMDGPU::V_FMAC_F64_e64: in canShrink()
3575 hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers)) in canShrink()
3579 case AMDGPU::V_CNDMASK_B32_e64: in canShrink()
3584 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1); in canShrink()
3586 hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers))) in canShrink()
3591 if (hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers)) in canShrink()
3599 return !hasModifiersSet(MI, AMDGPU::OpName::omod) && in canShrink()
3600 !hasModifiersSet(MI, AMDGPU::OpName::clamp); in canShrink()
3610 (Use.getReg() == AMDGPU::VCC || Use.getReg() == AMDGPU::VCC_LO)) { in copyFlagsToImplicitVCC()
3627 int Op32DstIdx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::vdst); in buildShrunkInst()
3632 assert(((MI.getOperand(0).getReg() == AMDGPU::VCC) || in buildShrunkInst()
3633 (MI.getOperand(0).getReg() == AMDGPU::VCC_LO)) && in buildShrunkInst()
3637 Inst32.add(*getNamedOperand(MI, AMDGPU::OpName::src0)); in buildShrunkInst()
3639 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1); in buildShrunkInst()
3643 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); in buildShrunkInst()
3646 int Op32Src2Idx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::src2); in buildShrunkInst()
3682 if (MO.getReg() == AMDGPU::SGPR_NULL) in usesConstantBus()
3687 return MO.getReg() == AMDGPU::M0 || in usesConstantBus()
3688 MO.getReg() == AMDGPU::VCC || in usesConstantBus()
3689 MO.getReg() == AMDGPU::VCC_LO; in usesConstantBus()
3691 return AMDGPU::SReg_32RegClass.contains(MO.getReg()) || in usesConstantBus()
3692 AMDGPU::SReg_64RegClass.contains(MO.getReg()); in usesConstantBus()
3703 case AMDGPU::VCC: in findImplicitSGPRRead()
3704 case AMDGPU::VCC_LO: in findImplicitSGPRRead()
3705 case AMDGPU::VCC_HI: in findImplicitSGPRRead()
3706 case AMDGPU::M0: in findImplicitSGPRRead()
3707 case AMDGPU::FLAT_SCR: in findImplicitSGPRRead()
3715 return AMDGPU::NoRegister; in findImplicitSGPRRead()
3721 case AMDGPU::V_READLANE_B32: in shouldReadExec()
3722 case AMDGPU::V_WRITELANE_B32: in shouldReadExec()
3744 return SubReg.getSubReg() != AMDGPU::NoSubRegister && in isSubRegOf()
3757 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0); in verifyInstruction()
3758 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1); in verifyInstruction()
3759 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2); in verifyInstruction()
3814 case AMDGPU::OPERAND_REG_IMM_INT32: in verifyInstruction()
3815 case AMDGPU::OPERAND_REG_IMM_FP32: in verifyInstruction()
3817 case AMDGPU::OPERAND_REG_INLINE_C_INT32: in verifyInstruction()
3818 case AMDGPU::OPERAND_REG_INLINE_C_FP32: in verifyInstruction()
3819 case AMDGPU::OPERAND_REG_INLINE_C_INT64: in verifyInstruction()
3820 case AMDGPU::OPERAND_REG_INLINE_C_FP64: in verifyInstruction()
3821 case AMDGPU::OPERAND_REG_INLINE_C_INT16: in verifyInstruction()
3822 case AMDGPU::OPERAND_REG_INLINE_C_FP16: in verifyInstruction()
3823 case AMDGPU::OPERAND_REG_INLINE_AC_INT32: in verifyInstruction()
3824 case AMDGPU::OPERAND_REG_INLINE_AC_FP32: in verifyInstruction()
3825 case AMDGPU::OPERAND_REG_INLINE_AC_INT16: in verifyInstruction()
3826 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: in verifyInstruction()
3827 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: { in verifyInstruction()
3835 case AMDGPU::OPERAND_KIMM32: in verifyInstruction()
3896 int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst); in verifyInstruction()
3923 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod); in verifyInstruction()
3931 uint16_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode); in verifyInstruction()
3936 if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) { in verifyInstruction()
3942 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp); in verifyInstruction()
3949 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod); in verifyInstruction()
3957 const MachineOperand *DstUnused = getNamedOperand(MI, AMDGPU::OpName::dst_unused); in verifyInstruction()
3959 DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE) { in verifyInstruction()
3984 const MachineOperand *DMask = getNamedOperand(MI, AMDGPU::OpName::dmask); in verifyInstruction()
3989 const MachineOperand *TFE = getNamedOperand(MI, AMDGPU::OpName::tfe); in verifyInstruction()
3990 const MachineOperand *LWE = getNamedOperand(MI, AMDGPU::OpName::lwe); in verifyInstruction()
3991 const MachineOperand *D16 = getNamedOperand(MI, AMDGPU::OpName::d16); in verifyInstruction()
4002 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata); in verifyInstruction()
4017 if (Desc.getOpcode() != AMDGPU::V_WRITELANE_B32 in verifyInstruction()
4028 if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1) in verifyInstruction()
4062 if (SGPRUsed != AMDGPU::NoRegister) { in verifyInstruction()
4075 Opcode != AMDGPU::V_WRITELANE_B32) { in verifyInstruction()
4088 if (Desc.getOpcode() == AMDGPU::V_WRITELANE_B32) { in verifyInstruction()
4090 Register SGPRUsed = AMDGPU::NoRegister; in verifyInstruction()
4099 if (MO.isReg() && MO.getReg() != AMDGPU::M0) { in verifyInstruction()
4113 if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32_e64 || in verifyInstruction()
4114 Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64_e64) { in verifyInstruction()
4125 if ((getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)->getImm() & in verifyInstruction()
4127 (getNamedOperand(MI, AMDGPU::OpName::src1_modifiers)->getImm() & in verifyInstruction()
4129 (getNamedOperand(MI, AMDGPU::OpName::src2_modifiers)->getImm() & in verifyInstruction()
4155 auto Op = getNamedOperand(MI, AMDGPU::OpName::simm16); in verifyInstruction()
4177 if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 || in verifyInstruction()
4178 Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 || in verifyInstruction()
4179 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 || in verifyInstruction()
4180 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) { in verifyInstruction()
4181 const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 || in verifyInstruction()
4182 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64; in verifyInstruction()
4196 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst); in verifyInstruction()
4224 if (!MI.hasRegisterImplicitUseOperand(AMDGPU::EXEC)) { in verifyInstruction()
4234 const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soff); in verifyInstruction()
4235 if (Soff && Soff->getReg() != AMDGPU::M0) { in verifyInstruction()
4243 const MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset); in verifyInstruction()
4251 const MachineOperand *DimOp = getNamedOperand(MI, AMDGPU::OpName::dim); in verifyInstruction()
4253 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode, in verifyInstruction()
4254 AMDGPU::OpName::vaddr0); in verifyInstruction()
4255 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc); in verifyInstruction()
4256 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opcode); in verifyInstruction()
4257 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode = in verifyInstruction()
4258 AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode); in verifyInstruction()
4259 const AMDGPU::MIMGDimInfo *Dim = in verifyInstruction()
4260 AMDGPU::getMIMGDimInfoByEncoding(DimOp->getImm()); in verifyInstruction()
4269 const MachineOperand *R128A16 = getNamedOperand(MI, AMDGPU::OpName::r128); in verifyInstruction()
4272 const MachineOperand *A16 = getNamedOperand(MI, AMDGPU::OpName::a16); in verifyInstruction()
4279 AMDGPU::getAddrSizeMIMGOp(BaseOpcode, Dim, IsA16, ST.hasG16()); in verifyInstruction()
4300 const MachineOperand *DppCt = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl); in verifyInstruction()
4302 using namespace AMDGPU::DPP; in verifyInstruction()
4343 int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst); in verifyInstruction()
4344 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0); in verifyInstruction()
4346 if (Opcode != AMDGPU::V_MOV_B64_DPP_PSEUDO && in verifyInstruction()
4348 (Desc.OpInfo[DstIdx].RegClass == AMDGPU::VReg_64RegClassID || in verifyInstruction()
4349 Desc.OpInfo[DstIdx].RegClass == AMDGPU::VReg_64_Align2RegClassID)) || in verifyInstruction()
4351 (Desc.OpInfo[Src0Idx].RegClass == AMDGPU::VReg_64RegClassID || in verifyInstruction()
4353 AMDGPU::VReg_64_Align2RegClassID)))) && in verifyInstruction()
4354 !AMDGPU::isLegal64BitDPPControl(DC)) { in verifyInstruction()
4362 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst); in verifyInstruction()
4363 uint16_t DataNameIdx = isDS(Opcode) ? AMDGPU::OpName::data0 in verifyInstruction()
4364 : AMDGPU::OpName::vdata; in verifyInstruction()
4366 const MachineOperand *Data2 = getNamedOperand(MI, AMDGPU::OpName::data1); in verifyInstruction()
4395 (MI.getOpcode() == AMDGPU::DS_GWS_INIT || in verifyInstruction()
4396 MI.getOpcode() == AMDGPU::DS_GWS_SEMA_BR || in verifyInstruction()
4397 MI.getOpcode() == AMDGPU::DS_GWS_BARRIER)) { in verifyInstruction()
4398 const MachineOperand *Op = getNamedOperand(MI, AMDGPU::OpName::data0); in verifyInstruction()
4421 default: return AMDGPU::INSTRUCTION_LIST_END; in getVALUOp()
4422 case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE; in getVALUOp()
4423 case AMDGPU::COPY: return AMDGPU::COPY; in getVALUOp()
4424 case AMDGPU::PHI: return AMDGPU::PHI; in getVALUOp()
4425 case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG; in getVALUOp()
4426 case AMDGPU::WQM: return AMDGPU::WQM; in getVALUOp()
4427 case AMDGPU::SOFT_WQM: return AMDGPU::SOFT_WQM; in getVALUOp()
4428 case AMDGPU::STRICT_WWM: return AMDGPU::STRICT_WWM; in getVALUOp()
4429 case AMDGPU::STRICT_WQM: return AMDGPU::STRICT_WQM; in getVALUOp()
4430 case AMDGPU::S_MOV_B32: { in getVALUOp()
4434 AMDGPU::COPY : AMDGPU::V_MOV_B32_e32; in getVALUOp()
4436 case AMDGPU::S_ADD_I32: in getVALUOp()
4437 return ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_CO_U32_e32; in getVALUOp()
4438 case AMDGPU::S_ADDC_U32: in getVALUOp()
4439 return AMDGPU::V_ADDC_U32_e32; in getVALUOp()
4440 case AMDGPU::S_SUB_I32: in getVALUOp()
4441 return ST.hasAddNoCarry() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_CO_U32_e32; in getVALUOp()
4444 case AMDGPU::S_ADD_U32: in getVALUOp()
4445 return AMDGPU::V_ADD_CO_U32_e32; in getVALUOp()
4446 case AMDGPU::S_SUB_U32: in getVALUOp()
4447 return AMDGPU::V_SUB_CO_U32_e32; in getVALUOp()
4448 case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32; in getVALUOp()
4449 case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_U32_e64; in getVALUOp()
4450 case AMDGPU::S_MUL_HI_U32: return AMDGPU::V_MUL_HI_U32_e64; in getVALUOp()
4451 case AMDGPU::S_MUL_HI_I32: return AMDGPU::V_MUL_HI_I32_e64; in getVALUOp()
4452 case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64; in getVALUOp()
4453 case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64; in getVALUOp()
4454 case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64; in getVALUOp()
4455 case AMDGPU::S_XNOR_B32: in getVALUOp()
4456 return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END; in getVALUOp()
4457 case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64; in getVALUOp()
4458 case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64; in getVALUOp()
4459 case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64; in getVALUOp()
4460 case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64; in getVALUOp()
4461 case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32; in getVALUOp()
4462 case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64_e64; in getVALUOp()
4463 case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32; in getVALUOp()
4464 case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64_e64; in getVALUOp()
4465 case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32; in getVALUOp()
4466 case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64_e64; in getVALUOp()
4467 case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32_e64; in getVALUOp()
4468 case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32_e64; in getVALUOp()
4469 case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32_e64; in getVALUOp()
4470 case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32_e64; in getVALUOp()
4471 case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64; in getVALUOp()
4472 case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32; in getVALUOp()
4473 case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32; in getVALUOp()
4474 case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32; in getVALUOp()
4475 case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e32; in getVALUOp()
4476 case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e32; in getVALUOp()
4477 case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e32; in getVALUOp()
4478 case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e32; in getVALUOp()
4479 case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e32; in getVALUOp()
4480 case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e32; in getVALUOp()
4481 case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e32; in getVALUOp()
4482 case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e32; in getVALUOp()
4483 case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e32; in getVALUOp()
4484 case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e32; in getVALUOp()
4485 case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e32; in getVALUOp()
4486 case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e32; in getVALUOp()
4487 case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e32; in getVALUOp()
4488 case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e32; in getVALUOp()
4489 case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64; in getVALUOp()
4490 case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32; in getVALUOp()
4491 case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32; in getVALUOp()
4492 case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64; in getVALUOp()
4493 case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ; in getVALUOp()
4494 case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ; in getVALUOp()
4509 case AMDGPU::AV_32RegClassID: return AMDGPU::VGPR_32RegClassID; in adjustAllocatableRegClass()
4510 case AMDGPU::AV_64RegClassID: return AMDGPU::VReg_64RegClassID; in adjustAllocatableRegClass()
4511 case AMDGPU::AV_96RegClassID: return AMDGPU::VReg_96RegClassID; in adjustAllocatableRegClass()
4512 case AMDGPU::AV_128RegClassID: return AMDGPU::VReg_128RegClassID; in adjustAllocatableRegClass()
4513 case AMDGPU::AV_160RegClassID: return AMDGPU::VReg_160RegClassID; in adjustAllocatableRegClass()
4537 const int VDstIdx = AMDGPU::getNamedOperandIdx(TID.Opcode, in getRegClass()
4538 AMDGPU::OpName::vdst); in getRegClass()
4539 const int DataIdx = AMDGPU::getNamedOperandIdx(TID.Opcode, in getRegClass()
4540 (TID.TSFlags & SIInstrFlags::DS) ? AMDGPU::OpName::data0 in getRegClass()
4541 : AMDGPU::OpName::vdata); in getRegClass()
4544 AMDGPU::getNamedOperandIdx(TID.Opcode, in getRegClass()
4545 AMDGPU::OpName::data1) != -1; in getRegClass()
4579 unsigned Opcode = (Size == 64) ? AMDGPU::V_MOV_B64_PSEUDO : AMDGPU::V_MOV_B32_e32; in legalizeOpWithMove()
4581 Opcode = AMDGPU::COPY; in legalizeOpWithMove()
4583 Opcode = (Size == 64) ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32; in legalizeOpWithMove()
4590 VRC = &AMDGPU::VGPR_32RegClass; in legalizeOpWithMove()
4609 if (SuperReg.getSubReg() == AMDGPU::NoSubRegister) { in buildExtractSubReg()
4638 if (SubIdx == AMDGPU::sub0) in buildExtractSubRegOrImm()
4640 if (SubIdx == AMDGPU::sub1) in buildExtractSubRegOrImm()
4730 } else if (InstDesc.OpInfo[i].OperandType == AMDGPU::OPERAND_KIMM32) { in isOperandLegal()
4733 } else if (isVOP3(MI) && AMDGPU::isSISrcOperand(InstDesc, i) && in isOperandLegal()
4756 const int VDstIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); in isOperandLegal()
4757 const int DataIdx = AMDGPU::getNamedOperandIdx(Opc, in isOperandLegal()
4758 isDS(Opc) ? AMDGPU::OpName::data0 : AMDGPU::OpName::vdata); in isOperandLegal()
4768 const int Data1Idx = AMDGPU::getNamedOperandIdx(Opc, in isOperandLegal()
4769 AMDGPU::OpName::data1); in isOperandLegal()
4774 if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64 && in isOperandLegal()
4775 (int)OpIdx == AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) && in isOperandLegal()
4797 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); in legalizeOperandsVOP2()
4800 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); in legalizeOperandsVOP2()
4805 bool HasImplicitSGPR = findImplicitSGPRRead(MI) != AMDGPU::NoRegister; in legalizeOperandsVOP2()
4814 if (Opc == AMDGPU::V_WRITELANE_B32) { in legalizeOperandsVOP2()
4817 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in legalizeOperandsVOP2()
4818 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) in legalizeOperandsVOP2()
4823 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in legalizeOperandsVOP2()
4825 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) in legalizeOperandsVOP2()
4847 if (Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() && in legalizeOperandsVOP2()
4849 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in legalizeOperandsVOP2()
4851 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) in legalizeOperandsVOP2()
4909 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0), in legalizeOperandsVOP3()
4910 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1), in legalizeOperandsVOP3()
4911 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2) in legalizeOperandsVOP3()
4914 if (Opc == AMDGPU::V_PERMLANE16_B32_e64 || in legalizeOperandsVOP3()
4915 Opc == AMDGPU::V_PERMLANEX16_B32_e64) { in legalizeOperandsVOP3()
4921 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in legalizeOperandsVOP3()
4922 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) in legalizeOperandsVOP3()
4927 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in legalizeOperandsVOP3()
4928 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) in legalizeOperandsVOP3()
4939 if (SGPRReg != AMDGPU::NoRegister) { in legalizeOperandsVOP3()
5009 get(AMDGPU::V_READFIRSTLANE_B32), DstReg) in readlaneVGPRToSGPR()
5016 Register SGPR = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); in readlaneVGPRToSGPR()
5018 get(AMDGPU::V_READFIRSTLANE_B32), SGPR) in readlaneVGPRToSGPR()
5025 get(AMDGPU::REG_SEQUENCE), DstReg); in readlaneVGPRToSGPR()
5040 MachineOperand *SBase = getNamedOperand(MI, AMDGPU::OpName::sbase); in legalizeOperandsSMRD()
5045 MachineOperand *SOff = getNamedOperand(MI, AMDGPU::OpName::soff); in legalizeOperandsSMRD()
5054 int OldSAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr); in moveFlatAddrToVGPR()
5060 int NewOpc = AMDGPU::getGlobalVaddrOp(Opc); in moveFlatAddrToVGPR()
5062 NewOpc = AMDGPU::getFlatScratchInstSVfromSS(Opc); in moveFlatAddrToVGPR()
5071 int NewVAddrIdx = AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vaddr); in moveFlatAddrToVGPR()
5075 int OldVAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr); in moveFlatAddrToVGPR()
5082 if (!VAddrDef || VAddrDef->getOpcode() != AMDGPU::V_MOV_B32_e32 || in moveFlatAddrToVGPR()
5107 int NewVDstIn = AMDGPU::getNamedOperandIdx(NewOpc, in moveFlatAddrToVGPR()
5108 AMDGPU::OpName::vdst_in); in moveFlatAddrToVGPR()
5113 int OldVDstIn = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst_in); in moveFlatAddrToVGPR()
5120 int NewVDst = AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vdst); in moveFlatAddrToVGPR()
5140 MachineOperand *SAddr = getNamedOperand(MI, AMDGPU::OpName::saddr); in legalizeOperandsFLAT()
5169 BuildMI(InsertMBB, I, DL, get(AMDGPU::COPY), DstReg).add(Op); in legalizeGenericOperand()
5179 if (Def->isMoveImmediate() && DstRC != &AMDGPU::VReg_1RegClass) in legalizeGenericOperand()
5189 if (!RI.isSGPRClass(DstRC) && !Copy->readsRegister(AMDGPU::EXEC, &RI) && in legalizeGenericOperand()
5191 Copy->addOperand(MachineOperand::CreateReg(AMDGPU::EXEC, false, true)); in legalizeGenericOperand()
5204 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; in emitLoadSRsrcFromVGPRLoop()
5206 ST.isWave32() ? AMDGPU::S_AND_SAVEEXEC_B32 : AMDGPU::S_AND_SAVEEXEC_B64; in emitLoadSRsrcFromVGPRLoop()
5208 ST.isWave32() ? AMDGPU::S_XOR_B32_term : AMDGPU::S_XOR_B64_term; in emitLoadSRsrcFromVGPRLoop()
5210 ST.isWave32() ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64; in emitLoadSRsrcFromVGPRLoop()
5211 const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID); in emitLoadSRsrcFromVGPRLoop()
5216 Register CondReg = AMDGPU::NoRegister; in emitLoadSRsrcFromVGPRLoop()
5227 Register CurRegLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); in emitLoadSRsrcFromVGPRLoop()
5228 Register CurRegHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); in emitLoadSRsrcFromVGPRLoop()
5231 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegLo) in emitLoadSRsrcFromVGPRLoop()
5235 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegHi) in emitLoadSRsrcFromVGPRLoop()
5242 Register CurReg = MRI.createVirtualRegister(&AMDGPU::SGPR_64RegClass); in emitLoadSRsrcFromVGPRLoop()
5243 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), CurReg) in emitLoadSRsrcFromVGPRLoop()
5245 .addImm(AMDGPU::sub0) in emitLoadSRsrcFromVGPRLoop()
5247 .addImm(AMDGPU::sub1); in emitLoadSRsrcFromVGPRLoop()
5251 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), NewCondReg) in emitLoadSRsrcFromVGPRLoop()
5259 if (CondReg == AMDGPU::NoRegister) // First. in emitLoadSRsrcFromVGPRLoop()
5274 auto Merge = BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), SRsrc); in emitLoadSRsrcFromVGPRLoop()
5300 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::SI_WATERFALL_LOOP)).addMBB(&LoopBB); in emitLoadSRsrcFromVGPRLoop()
5323 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; in loadSRsrcFromVGPR()
5324 unsigned MovExecOpc = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; in loadSRsrcFromVGPR()
5325 const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID); in loadSRsrcFromVGPR()
5396 TII.buildExtractSubReg(MI, MRI, Rsrc, &AMDGPU::VReg_128RegClass, in extractRsrcPtr()
5397 AMDGPU::sub0_sub1, &AMDGPU::VReg_64RegClass); in extractRsrcPtr()
5400 Register Zero64 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in extractRsrcPtr()
5401 Register SRsrcFormatLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); in extractRsrcPtr()
5402 Register SRsrcFormatHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); in extractRsrcPtr()
5403 Register NewSRsrc = MRI.createVirtualRegister(&AMDGPU::SGPR_128RegClass); in extractRsrcPtr()
5407 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B64), Zero64) in extractRsrcPtr()
5411 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatLo) in extractRsrcPtr()
5415 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatHi) in extractRsrcPtr()
5419 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::REG_SEQUENCE), NewSRsrc) in extractRsrcPtr()
5421 .addImm(AMDGPU::sub0_sub1) in extractRsrcPtr()
5423 .addImm(AMDGPU::sub2) in extractRsrcPtr()
5425 .addImm(AMDGPU::sub3); in extractRsrcPtr()
5464 if (MI.getOpcode() == AMDGPU::PHI) { in legalizeOperands()
5484 if (getOpRegClass(MI, 0) == &AMDGPU::VReg_1RegClass) { in legalizeOperands()
5485 VRC = &AMDGPU::VReg_1RegClass; in legalizeOperands()
5519 if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) { in legalizeOperands()
5546 if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) { in legalizeOperands()
5560 if (MI.getOpcode() == AMDGPU::SI_INIT_M0) { in legalizeOperands()
5572 if (isMIMG(MI) || (AMDGPU::isGraphics(MF.getFunction().getCallingConv()) && in legalizeOperands()
5574 MachineOperand *SRsrc = getNamedOperand(MI, AMDGPU::OpName::srsrc); in legalizeOperands()
5578 MachineOperand *SSamp = getNamedOperand(MI, AMDGPU::OpName::ssamp); in legalizeOperands()
5586 if (MI.getOpcode() == AMDGPU::SI_CALL_ISEL) { in legalizeOperands()
5614 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::srsrc); in legalizeOperands()
5641 MachineOperand *VAddr = getNamedOperand(MI, AMDGPU::OpName::vaddr); in legalizeOperands()
5642 if (VAddr && AMDGPU::getIfAddr64Inst(MI.getOpcode()) != -1) { in legalizeOperands()
5645 Register NewVAddrLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in legalizeOperands()
5646 Register NewVAddrHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in legalizeOperands()
5647 Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); in legalizeOperands()
5649 const auto *BoolXExecRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); in legalizeOperands()
5658 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADD_CO_U32_e64), NewVAddrLo) in legalizeOperands()
5660 .addReg(RsrcPtr, 0, AMDGPU::sub0) in legalizeOperands()
5661 .addReg(VAddr->getReg(), 0, AMDGPU::sub0) in legalizeOperands()
5665 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADDC_U32_e64), NewVAddrHi) in legalizeOperands()
5667 .addReg(RsrcPtr, 0, AMDGPU::sub1) in legalizeOperands()
5668 .addReg(VAddr->getReg(), 0, AMDGPU::sub1) in legalizeOperands()
5673 BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewVAddr) in legalizeOperands()
5675 .addImm(AMDGPU::sub0) in legalizeOperands()
5677 .addImm(AMDGPU::sub1); in legalizeOperands()
5690 Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); in legalizeOperands()
5691 MachineOperand *VData = getNamedOperand(MI, AMDGPU::OpName::vdata); in legalizeOperands()
5692 MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset); in legalizeOperands()
5693 MachineOperand *SOffset = getNamedOperand(MI, AMDGPU::OpName::soffset); in legalizeOperands()
5694 unsigned Addr64Opcode = AMDGPU::getAddr64Inst(MI.getOpcode()); in legalizeOperands()
5698 MachineOperand *VDataIn = getNamedOperand(MI, AMDGPU::OpName::vdata_in); in legalizeOperands()
5712 getNamedOperand(MI, AMDGPU::OpName::cpol)) { in legalizeOperands()
5717 getNamedOperand(MI, AMDGPU::OpName::tfe)) { in legalizeOperands()
5721 MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::swz)); in legalizeOperands()
5734 .addImm(getNamedImmOperand(MI, AMDGPU::OpName::cpol)) in legalizeOperands()
5741 BuildMI(MBB, Addr64, Addr64->getDebugLoc(), get(AMDGPU::REG_SEQUENCE), in legalizeOperands()
5743 .addReg(RsrcPtr, 0, AMDGPU::sub0) in legalizeOperands()
5744 .addImm(AMDGPU::sub0) in legalizeOperands()
5745 .addReg(RsrcPtr, 0, AMDGPU::sub1) in legalizeOperands()
5746 .addImm(AMDGPU::sub1); in legalizeOperands()
5776 case AMDGPU::S_ADD_U64_PSEUDO: in moveToVALU()
5777 case AMDGPU::S_SUB_U64_PSEUDO: in moveToVALU()
5781 case AMDGPU::S_ADD_I32: in moveToVALU()
5782 case AMDGPU::S_SUB_I32: { in moveToVALU()
5794 case AMDGPU::S_AND_B64: in moveToVALU()
5795 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_AND_B32, MDT); in moveToVALU()
5799 case AMDGPU::S_OR_B64: in moveToVALU()
5800 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_OR_B32, MDT); in moveToVALU()
5804 case AMDGPU::S_XOR_B64: in moveToVALU()
5805 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XOR_B32, MDT); in moveToVALU()
5809 case AMDGPU::S_NAND_B64: in moveToVALU()
5810 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NAND_B32, MDT); in moveToVALU()
5814 case AMDGPU::S_NOR_B64: in moveToVALU()
5815 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NOR_B32, MDT); in moveToVALU()
5819 case AMDGPU::S_XNOR_B64: in moveToVALU()
5821 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XNOR_B32, MDT); in moveToVALU()
5827 case AMDGPU::S_ANDN2_B64: in moveToVALU()
5828 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ANDN2_B32, MDT); in moveToVALU()
5832 case AMDGPU::S_ORN2_B64: in moveToVALU()
5833 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ORN2_B32, MDT); in moveToVALU()
5837 case AMDGPU::S_BREV_B64: in moveToVALU()
5838 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_BREV_B32, true); in moveToVALU()
5842 case AMDGPU::S_NOT_B64: in moveToVALU()
5843 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_NOT_B32); in moveToVALU()
5847 case AMDGPU::S_BCNT1_I32_B64: in moveToVALU()
5852 case AMDGPU::S_BFE_I64: in moveToVALU()
5857 case AMDGPU::S_LSHL_B32: in moveToVALU()
5859 NewOpcode = AMDGPU::V_LSHLREV_B32_e64; in moveToVALU()
5863 case AMDGPU::S_ASHR_I32: in moveToVALU()
5865 NewOpcode = AMDGPU::V_ASHRREV_I32_e64; in moveToVALU()
5869 case AMDGPU::S_LSHR_B32: in moveToVALU()
5871 NewOpcode = AMDGPU::V_LSHRREV_B32_e64; in moveToVALU()
5875 case AMDGPU::S_LSHL_B64: in moveToVALU()
5877 NewOpcode = AMDGPU::V_LSHLREV_B64_e64; in moveToVALU()
5881 case AMDGPU::S_ASHR_I64: in moveToVALU()
5883 NewOpcode = AMDGPU::V_ASHRREV_I64_e64; in moveToVALU()
5887 case AMDGPU::S_LSHR_B64: in moveToVALU()
5889 NewOpcode = AMDGPU::V_LSHRREV_B64_e64; in moveToVALU()
5894 case AMDGPU::S_ABS_I32: in moveToVALU()
5899 case AMDGPU::S_CBRANCH_SCC0: in moveToVALU()
5900 case AMDGPU::S_CBRANCH_SCC1: in moveToVALU()
5903 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B32), in moveToVALU()
5904 AMDGPU::VCC_LO) in moveToVALU()
5905 .addReg(AMDGPU::EXEC_LO) in moveToVALU()
5906 .addReg(AMDGPU::VCC_LO); in moveToVALU()
5908 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B64), in moveToVALU()
5909 AMDGPU::VCC) in moveToVALU()
5910 .addReg(AMDGPU::EXEC) in moveToVALU()
5911 .addReg(AMDGPU::VCC); in moveToVALU()
5914 case AMDGPU::S_BFE_U64: in moveToVALU()
5915 case AMDGPU::S_BFM_B64: in moveToVALU()
5918 case AMDGPU::S_PACK_LL_B32_B16: in moveToVALU()
5919 case AMDGPU::S_PACK_LH_B32_B16: in moveToVALU()
5920 case AMDGPU::S_PACK_HH_B32_B16: in moveToVALU()
5925 case AMDGPU::S_XNOR_B32: in moveToVALU()
5930 case AMDGPU::S_NAND_B32: in moveToVALU()
5931 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_AND_B32); in moveToVALU()
5935 case AMDGPU::S_NOR_B32: in moveToVALU()
5936 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_OR_B32); in moveToVALU()
5940 case AMDGPU::S_ANDN2_B32: in moveToVALU()
5941 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_AND_B32); in moveToVALU()
5945 case AMDGPU::S_ORN2_B32: in moveToVALU()
5946 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_OR_B32); in moveToVALU()
5954 case AMDGPU::S_ADD_CO_PSEUDO: in moveToVALU()
5955 case AMDGPU::S_SUB_CO_PSEUDO: { in moveToVALU()
5956 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO) in moveToVALU()
5957 ? AMDGPU::V_ADDC_U32_e64 in moveToVALU()
5958 : AMDGPU::V_SUBB_U32_e64; in moveToVALU()
5959 const auto *CarryRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); in moveToVALU()
5964 BuildMI(*MBB, &Inst, Inst.getDebugLoc(), get(AMDGPU::COPY), NewCarryReg) in moveToVALU()
5987 case AMDGPU::S_UADDO_PSEUDO: in moveToVALU()
5988 case AMDGPU::S_USUBO_PSEUDO: { in moveToVALU()
5995 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_UADDO_PSEUDO) in moveToVALU()
5996 ? AMDGPU::V_ADD_CO_U32_e64 in moveToVALU()
5997 : AMDGPU::V_SUB_CO_U32_e64; in moveToVALU()
6018 case AMDGPU::S_CSELECT_B32: in moveToVALU()
6019 case AMDGPU::S_CSELECT_B64: in moveToVALU()
6025 if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) { in moveToVALU()
6043 if (Op.isReg() && Op.getReg() == AMDGPU::SCC) { in moveToVALU()
6053 if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) { in moveToVALU()
6056 unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16; in moveToVALU()
6060 } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) { in moveToVALU()
6069 if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) { in moveToVALU()
6085 unsigned NewDstReg = AMDGPU::NoRegister; in moveToVALU()
6114 Inst.setDesc(get(AMDGPU::IMPLICIT_DEF)); in moveToVALU()
6146 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in moveScalarAddSub()
6149 assert(Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32); in moveScalarAddSub()
6151 unsigned NewOpc = Opc == AMDGPU::S_ADD_I32 ? in moveScalarAddSub()
6152 AMDGPU::V_ADD_U32_e64 : AMDGPU::V_SUB_U32_e64; in moveScalarAddSub()
6154 assert(Inst.getOperand(3).getReg() == AMDGPU::SCC); in moveScalarAddSub()
6189 if (CandI.findRegisterDefOperandIdx(AMDGPU::SCC, false, false, &RI) != in lowerSelect()
6191 if (CandI.isCopy() && CandI.getOperand(0).getReg() == AMDGPU::SCC) { in lowerSelect()
6203 if ((SCCSource != AMDGPU::SCC) && Src0.isImm() && (Src0.getImm() == -1) && in lowerSelect()
6210 ? &AMDGPU::SReg_64_XEXECRegClass in lowerSelect()
6211 : &AMDGPU::SReg_32_XM0_XEXECRegClass; in lowerSelect()
6214 if (SCCSource == AMDGPU::SCC) { in lowerSelect()
6218 unsigned Opcode = (ST.getWavefrontSize() == 64) ? AMDGPU::S_CSELECT_B64 in lowerSelect()
6219 : AMDGPU::S_CSELECT_B32; in lowerSelect()
6224 BuildMI(MBB, MII, DL, get(AMDGPU::COPY), CopySCC).addReg(SCCSource); in lowerSelect()
6227 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in lowerSelect()
6230 BuildMI(MBB, MII, DL, get(AMDGPU::V_CNDMASK_B32_e64), ResultReg) in lowerSelect()
6251 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in lowerScalarAbs()
6252 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in lowerScalarAbs()
6255 AMDGPU::V_SUB_U32_e32 : AMDGPU::V_SUB_CO_U32_e32; in lowerScalarAbs()
6261 BuildMI(MBB, MII, DL, get(AMDGPU::V_MAX_I32_e64), ResultReg) in lowerScalarAbs()
6281 Register NewDest = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in lowerScalarXnor()
6282 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src0, MRI, DL); in lowerScalarXnor()
6283 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src1, MRI, DL); in lowerScalarXnor()
6285 BuildMI(MBB, MII, DL, get(AMDGPU::V_XNOR_B32_e64), NewDest) in lowerScalarXnor()
6301 Register Temp = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); in lowerScalarXnor()
6302 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); in lowerScalarXnor()
6308 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src0); in lowerScalarXnor()
6309 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest) in lowerScalarXnor()
6313 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src1); in lowerScalarXnor()
6314 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest) in lowerScalarXnor()
6318 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), Temp) in lowerScalarXnor()
6322 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest).addReg(Temp); in lowerScalarXnor()
6346 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); in splitScalarNotBinop()
6347 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); in splitScalarNotBinop()
6353 MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest) in splitScalarNotBinop()
6375 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in splitScalarBinOpN2()
6376 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in splitScalarBinOpN2()
6378 MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Interm) in splitScalarBinOpN2()
6407 &AMDGPU::SGPR_32RegClass; in splitScalar64BitUnaryOp()
6409 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0); in splitScalar64BitUnaryOp()
6412 AMDGPU::sub0, Src0SubRC); in splitScalar64BitUnaryOp()
6416 const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0); in splitScalar64BitUnaryOp()
6422 AMDGPU::sub1, Src0SubRC); in splitScalar64BitUnaryOp()
6433 .addImm(AMDGPU::sub0) in splitScalar64BitUnaryOp()
6435 .addImm(AMDGPU::sub1); in splitScalar64BitUnaryOp()
6452 bool IsAdd = (Inst.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO); in splitScalar64BitAddSub()
6456 const auto *CarryRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); in splitScalar64BitAddSub()
6458 Register FullDestReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); in splitScalar64BitAddSub()
6459 Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitAddSub()
6460 Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitAddSub()
6473 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0); in splitScalar64BitAddSub()
6474 const TargetRegisterClass *Src1SubRC = RI.getSubRegClass(Src1RC, AMDGPU::sub0); in splitScalar64BitAddSub()
6477 AMDGPU::sub0, Src0SubRC); in splitScalar64BitAddSub()
6479 AMDGPU::sub0, Src1SubRC); in splitScalar64BitAddSub()
6483 AMDGPU::sub1, Src0SubRC); in splitScalar64BitAddSub()
6485 AMDGPU::sub1, Src1SubRC); in splitScalar64BitAddSub()
6487 unsigned LoOpc = IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64; in splitScalar64BitAddSub()
6495 unsigned HiOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64; in splitScalar64BitAddSub()
6506 .addImm(AMDGPU::sub0) in splitScalar64BitAddSub()
6508 .addImm(AMDGPU::sub1); in splitScalar64BitAddSub()
6537 &AMDGPU::SGPR_32RegClass; in splitScalar64BitBinaryOp()
6539 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0); in splitScalar64BitBinaryOp()
6542 &AMDGPU::SGPR_32RegClass; in splitScalar64BitBinaryOp()
6544 const TargetRegisterClass *Src1SubRC = RI.getSubRegClass(Src1RC, AMDGPU::sub0); in splitScalar64BitBinaryOp()
6547 AMDGPU::sub0, Src0SubRC); in splitScalar64BitBinaryOp()
6549 AMDGPU::sub0, Src1SubRC); in splitScalar64BitBinaryOp()
6551 AMDGPU::sub1, Src0SubRC); in splitScalar64BitBinaryOp()
6553 AMDGPU::sub1, Src1SubRC); in splitScalar64BitBinaryOp()
6557 const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0); in splitScalar64BitBinaryOp()
6572 .addImm(AMDGPU::sub0) in splitScalar64BitBinaryOp()
6574 .addImm(AMDGPU::sub1); in splitScalar64BitBinaryOp()
6600 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in splitScalar64BitXnor()
6613 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B64), Interm) in splitScalar64BitXnor()
6618 MachineInstr &Xor = *BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B64), NewDest) in splitScalar64BitXnor()
6638 const MCInstrDesc &InstDesc = get(AMDGPU::V_BCNT_U32_B32_e64); in splitScalar64BitBCNT()
6641 &AMDGPU::SGPR_32RegClass; in splitScalar64BitBCNT()
6643 Register MidReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitBCNT()
6644 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitBCNT()
6646 const TargetRegisterClass *SrcSubRC = RI.getSubRegClass(SrcRC, AMDGPU::sub0); in splitScalar64BitBCNT()
6649 AMDGPU::sub0, SrcSubRC); in splitScalar64BitBCNT()
6651 AMDGPU::sub1, SrcSubRC); in splitScalar64BitBCNT()
6679 assert(Inst.getOpcode() == AMDGPU::S_BFE_I64 && BitWidth <= 32 && in splitScalar64BitBFE()
6683 Register MidRegLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitBFE()
6684 Register MidRegHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitBFE()
6685 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); in splitScalar64BitBFE()
6687 BuildMI(MBB, MII, DL, get(AMDGPU::V_BFE_I32_e64), MidRegLo) in splitScalar64BitBFE()
6688 .addReg(Inst.getOperand(1).getReg(), 0, AMDGPU::sub0) in splitScalar64BitBFE()
6692 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e32), MidRegHi) in splitScalar64BitBFE()
6698 .addImm(AMDGPU::sub0) in splitScalar64BitBFE()
6700 .addImm(AMDGPU::sub1); in splitScalar64BitBFE()
6708 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitBFE()
6709 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); in splitScalar64BitBFE()
6711 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e64), TmpReg) in splitScalar64BitBFE()
6713 .addReg(Src.getReg(), 0, AMDGPU::sub0); in splitScalar64BitBFE()
6716 .addReg(Src.getReg(), 0, AMDGPU::sub0) in splitScalar64BitBFE()
6717 .addImm(AMDGPU::sub0) in splitScalar64BitBFE()
6719 .addImm(AMDGPU::sub1); in splitScalar64BitBFE()
6736 case AMDGPU::COPY: in addUsersToMoveToVALUWorklist()
6737 case AMDGPU::WQM: in addUsersToMoveToVALUWorklist()
6738 case AMDGPU::SOFT_WQM: in addUsersToMoveToVALUWorklist()
6739 case AMDGPU::STRICT_WWM: in addUsersToMoveToVALUWorklist()
6740 case AMDGPU::STRICT_WQM: in addUsersToMoveToVALUWorklist()
6741 case AMDGPU::REG_SEQUENCE: in addUsersToMoveToVALUWorklist()
6742 case AMDGPU::PHI: in addUsersToMoveToVALUWorklist()
6743 case AMDGPU::INSERT_SUBREG: in addUsersToMoveToVALUWorklist()
6765 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in movePackToVALU()
6772 case AMDGPU::S_PACK_LL_B32_B16: { in movePackToVALU()
6773 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in movePackToVALU()
6774 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in movePackToVALU()
6778 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg) in movePackToVALU()
6781 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_B32_e64), TmpReg) in movePackToVALU()
6785 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHL_OR_B32_e64), ResultReg) in movePackToVALU()
6791 case AMDGPU::S_PACK_LH_B32_B16: { in movePackToVALU()
6792 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in movePackToVALU()
6793 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg) in movePackToVALU()
6795 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_BFI_B32_e64), ResultReg) in movePackToVALU()
6801 case AMDGPU::S_PACK_HH_B32_B16: { in movePackToVALU()
6802 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in movePackToVALU()
6803 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in movePackToVALU()
6804 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg) in movePackToVALU()
6807 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg) in movePackToVALU()
6809 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_OR_B32_e64), ResultReg) in movePackToVALU()
6830 assert(Op.isReg() && Op.getReg() == AMDGPU::SCC && Op.isDef() && in addSCCDefUsersToVALUWorklist()
6839 if (MI.findRegisterUseOperandIdx(AMDGPU::SCC, false, &RI) != -1) { in addSCCDefUsersToVALUWorklist()
6845 if ((User.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO) || in addSCCDefUsersToVALUWorklist()
6846 (User.getOpcode() == AMDGPU::S_SUB_CO_PSEUDO)) { in addSCCDefUsersToVALUWorklist()
6849 } else if (User.getOpcode() == AMDGPU::V_CNDMASK_B32_e64) { in addSCCDefUsersToVALUWorklist()
6856 if (MI.getOpcode() == AMDGPU::S_CSELECT_B32 || in addSCCDefUsersToVALUWorklist()
6857 MI.getOpcode() == AMDGPU::S_CSELECT_B64) { in addSCCDefUsersToVALUWorklist()
6871 if (MI.findRegisterDefOperandIdx(AMDGPU::SCC, false, false, &RI) != -1) in addSCCDefUsersToVALUWorklist()
6879 SCCDefInst.getDebugLoc(), get(AMDGPU::COPY), AMDGPU::SCC) in addSCCDefUsersToVALUWorklist()
6891 assert(Op.isReg() && Op.getReg() == AMDGPU::SCC && Op.isUse()); in addSCCDefsToVALUWorklist()
6901 if (MI.modifiesRegister(AMDGPU::VCC, &RI)) in addSCCDefsToVALUWorklist()
6903 if (MI.definesRegister(AMDGPU::SCC, &RI)) { in addSCCDefsToVALUWorklist()
6918 case AMDGPU::COPY: in getDestEquivalentVGPRClass()
6919 case AMDGPU::PHI: in getDestEquivalentVGPRClass()
6920 case AMDGPU::REG_SEQUENCE: in getDestEquivalentVGPRClass()
6921 case AMDGPU::INSERT_SUBREG: in getDestEquivalentVGPRClass()
6922 case AMDGPU::WQM: in getDestEquivalentVGPRClass()
6923 case AMDGPU::SOFT_WQM: in getDestEquivalentVGPRClass()
6924 case AMDGPU::STRICT_WWM: in getDestEquivalentVGPRClass()
6925 case AMDGPU::STRICT_WQM: { in getDestEquivalentVGPRClass()
6932 case AMDGPU::PHI: in getDestEquivalentVGPRClass()
6933 case AMDGPU::REG_SEQUENCE: in getDestEquivalentVGPRClass()
6934 case AMDGPU::INSERT_SUBREG: in getDestEquivalentVGPRClass()
6944 if (RI.hasVGPRs(NewDstRC) || NewDstRC == &AMDGPU::VReg_1RegClass) in getDestEquivalentVGPRClass()
6974 if (SGPRReg != AMDGPU::NoRegister) in findUsedSGPR()
6977 Register UsedSGPRs[3] = { AMDGPU::NoRegister }; in findUsedSGPR()
7016 if (UsedSGPRs[0] != AMDGPU::NoRegister) { in findUsedSGPR()
7021 if (SGPRReg == AMDGPU::NoRegister && UsedSGPRs[1] != AMDGPU::NoRegister) { in findUsedSGPR()
7031 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OperandName); in getNamedOperand()
7040 return (AMDGPU::MTBUFFormat::UFMT_32_FLOAT << 44) | in getDefaultRsrcDataFormat()
7045 uint64_t RsrcDataFormat = AMDGPU::RSRC_DATA_FORMAT; in getDefaultRsrcDataFormat()
7062 AMDGPU::RSRC_TID_ENABLE | in getScratchRsrcWords23()
7068 Rsrc23 |= EltSizeValue << AMDGPU::RSRC_ELEMENT_SIZE_SHIFT; in getScratchRsrcWords23()
7073 Rsrc23 |= IndexStride << AMDGPU::RSRC_INDEX_STRIDE_SHIFT; in getScratchRsrcWords23()
7079 Rsrc23 &= ~AMDGPU::RSRC_DATA_FORMAT; in getScratchRsrcWords23()
7097 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::vaddr); in isStackAccess()
7099 return AMDGPU::NoRegister; in isStackAccess()
7105 return getNamedOperand(MI, AMDGPU::OpName::vdata)->getReg(); in isStackAccess()
7110 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::addr); in isSGPRStackAccess()
7113 return getNamedOperand(MI, AMDGPU::OpName::data)->getReg(); in isSGPRStackAccess()
7119 return AMDGPU::NoRegister; in isLoadFromStackSlot()
7127 return AMDGPU::NoRegister; in isLoadFromStackSlot()
7133 return AMDGPU::NoRegister; in isStoreToStackSlot()
7141 return AMDGPU::NoRegister; in isStoreToStackSlot()
7177 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); in getInstSizeInBytes()
7184 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); in getInstSizeInBytes()
7191 int Src2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2); in getInstSizeInBytes()
7203 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0); in getInstSizeInBytes()
7207 int RSrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc); in getInstSizeInBytes()
7242 return Branch.getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO; in isNonUniformBranchInstr()
7254 if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) { in convertNonUniformIfRegion()
7257 BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_IF), DstReg) in convertNonUniformIfRegion()
7261 BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_END_CF)) in convertNonUniformIfRegion()
7280 if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) { in convertNonUniformLoopRegion()
7302 get(AMDGPU::SI_IF_BREAK), BackEdgeReg) in convertNonUniformLoopRegion()
7306 BuildMI(*(MF), Branch->getDebugLoc(), get(AMDGPU::SI_LOOP)) in convertNonUniformLoopRegion()
7320 {AMDGPU::TI_CONSTDATA_START, "amdgpu-constdata-start"}, in getSerializableTargetIndices()
7321 {AMDGPU::TI_SCRATCH_RSRC_DWORD0, "amdgpu-scratch-rsrc-dword0"}, in getSerializableTargetIndices()
7322 {AMDGPU::TI_SCRATCH_RSRC_DWORD1, "amdgpu-scratch-rsrc-dword1"}, in getSerializableTargetIndices()
7323 {AMDGPU::TI_SCRATCH_RSRC_DWORD2, "amdgpu-scratch-rsrc-dword2"}, in getSerializableTargetIndices()
7324 {AMDGPU::TI_SCRATCH_RSRC_DWORD3, "amdgpu-scratch-rsrc-dword3"}}; in getSerializableTargetIndices()
7364 return !MI.isTerminator() && MI.getOpcode() != AMDGPU::COPY && in isBasicBlockPrologue()
7365 MI.modifiesRegister(AMDGPU::EXEC, &RI); in isBasicBlockPrologue()
7374 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e64), DestReg); in getAddNoCarry()
7380 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_CO_U32_e64), DestReg) in getAddNoCarry()
7390 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e32), DestReg); in getAddNoCarry()
7393 Register UnusedCarry = !RS.isRegUsed(AMDGPU::VCC) in getAddNoCarry()
7401 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_CO_U32_e64), DestReg) in getAddNoCarry()
7407 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR: in isKillTerminator()
7408 case AMDGPU::SI_KILL_I1_TERMINATOR: in isKillTerminator()
7417 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO: in getKillTerminatorFromPseudo()
7418 return get(AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR); in getKillTerminatorFromPseudo()
7419 case AMDGPU::SI_KILL_I1_PSEUDO: in getKillTerminatorFromPseudo()
7420 return get(AMDGPU::SI_KILL_I1_TERMINATOR); in getKillTerminatorFromPseudo()
7431 if (Op.isReg() && Op.getReg() == AMDGPU::VCC) in fixImplicitOperands()
7432 Op.setReg(AMDGPU::VCC_LO); in fixImplicitOperands()
7441 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::sbase); in isBufferSMRD()
7446 return RI.getRegClass(RCID)->hasSubClassEq(&AMDGPU::SGPR_128RegClass); in isBufferSMRD()
7500 unsigned N = AMDGPU::getNumFlatOffsetBits(ST, Signed); in isLegalFLATOffset()
7515 const unsigned NumBits = AMDGPU::getNumFlatOffsetBits(ST, Signed); in splitFlatOffset()
7574 case AMDGPU::V_MOVRELS_B32_dpp_gfx10: in isAsmOnlyOpcode()
7575 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10: in isAsmOnlyOpcode()
7576 case AMDGPU::V_MOVRELD_B32_dpp_gfx10: in isAsmOnlyOpcode()
7577 case AMDGPU::V_MOVRELD_B32_sdwa_gfx10: in isAsmOnlyOpcode()
7578 case AMDGPU::V_MOVRELSD_B32_dpp_gfx10: in isAsmOnlyOpcode()
7579 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10: in isAsmOnlyOpcode()
7580 case AMDGPU::V_MOVRELSD_2_B32_dpp_gfx10: in isAsmOnlyOpcode()
7581 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10: in isAsmOnlyOpcode()
7615 int MCOp = AMDGPU::getMCOpcode(Opcode, Gen); in pseudoToMCOpcode()
7623 NMCOp = AMDGPU::getMCOpcode(Opcode, SIEncodingFamily::GFX90A); in pseudoToMCOpcode()
7625 NMCOp = AMDGPU::getMCOpcode(Opcode, SIEncodingFamily::GFX9); in pseudoToMCOpcode()
7667 case AMDGPU::REG_SEQUENCE: in followSubRegDef()
7671 case AMDGPU::INSERT_SUBREG: in followSubRegDef()
7697 case AMDGPU::COPY: in getVRegSubRegDef()
7698 case AMDGPU::V_MOV_B32_e32: { in getVRegSubRegDef()
7747 if (I->modifiesRegister(AMDGPU::EXEC, TRI)) in execMayBeModifiedBeforeUse()
7804 } else if (TRI->regsOverlap(Reg, AMDGPU::EXEC)) in execMayBeModifiedBeforeAnyUse()
7829 (InsPt->getOpcode() == AMDGPU::SI_IF || in createPHISourceCopy()
7830 InsPt->getOpcode() == AMDGPU::SI_ELSE || in createPHISourceCopy()
7831 InsPt->getOpcode() == AMDGPU::SI_IF_BREAK) && in createPHISourceCopy()
7835 get(ST.isWave32() ? AMDGPU::S_MOV_B32_term in createPHISourceCopy()
7836 : AMDGPU::S_MOV_B64_term), in createPHISourceCopy()
7839 .addReg(AMDGPU::EXEC, RegState::Implicit); in createPHISourceCopy()
7871 if (RC->hasSuperClassEq(&AMDGPU::SReg_32RegClass)) { in foldMemoryOperandImpl()
7872 MRI.constrainRegClass(VirtReg, &AMDGPU::SReg_32_XM0_XEXECRegClass); in foldMemoryOperandImpl()
7874 } else if (RC->hasSuperClassEq(&AMDGPU::SReg_64RegClass)) { in foldMemoryOperandImpl()
7875 MRI.constrainRegClass(VirtReg, &AMDGPU::SReg_64_XEXECRegClass); in foldMemoryOperandImpl()