Lines Matching refs:AMDGPU
72 namespace AMDGPU { namespace
89 : AMDGPUGenInstrInfo(AMDGPU::ADJCALLSTACKUP, AMDGPU::ADJCALLSTACKDOWN), in SIInstrInfo()
115 int Op0Idx = AMDGPU::getNamedOperandIdx(Opc0, OpName); in nodesHaveSameOperandValue()
116 int Op1Idx = AMDGPU::getNamedOperandIdx(Opc1, OpName); in nodesHaveSameOperandValue()
142 case AMDGPU::V_MOV_B32_e32: in isReallyTriviallyReMaterializable()
143 case AMDGPU::V_MOV_B32_e64: in isReallyTriviallyReMaterializable()
144 case AMDGPU::V_MOV_B64_PSEUDO: in isReallyTriviallyReMaterializable()
181 if (AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::data1) != -1 || in areLoadsFromSameBasePtr()
182 AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::data1) != -1) in areLoadsFromSameBasePtr()
192 if (AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::sbase) == -1 || in areLoadsFromSameBasePtr()
193 AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::sbase) == -1) in areLoadsFromSameBasePtr()
223 if (!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::soffset) || in areLoadsFromSameBasePtr()
225 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::vaddr) || in areLoadsFromSameBasePtr()
226 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::srsrc)) in areLoadsFromSameBasePtr()
229 int OffIdx0 = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset); in areLoadsFromSameBasePtr()
230 int OffIdx1 = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset); in areLoadsFromSameBasePtr()
258 case AMDGPU::DS_READ2ST64_B32: in isStride64()
259 case AMDGPU::DS_READ2ST64_B64: in isStride64()
260 case AMDGPU::DS_WRITE2ST64_B32: in isStride64()
261 case AMDGPU::DS_WRITE2ST64_B64: in isStride64()
276 getNamedOperand(LdSt, AMDGPU::OpName::offset); in getMemOperandWithOffset()
279 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr); in getMemOperandWithOffset()
290 getNamedOperand(LdSt, AMDGPU::OpName::offset0); in getMemOperandWithOffset()
292 getNamedOperand(LdSt, AMDGPU::OpName::offset1); in getMemOperandWithOffset()
306 int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0); in getMemOperandWithOffset()
313 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr); in getMemOperandWithOffset()
324 const MachineOperand *SOffset = getNamedOperand(LdSt, AMDGPU::OpName::soffset); in getMemOperandWithOffset()
328 MachineOperand *AddrReg = getNamedOperand(LdSt, AMDGPU::OpName::vaddr); in getMemOperandWithOffset()
333 getNamedOperand(LdSt, AMDGPU::OpName::offset); in getMemOperandWithOffset()
347 getNamedOperand(LdSt, AMDGPU::OpName::offset); in getMemOperandWithOffset()
351 MachineOperand *SBaseReg = getNamedOperand(LdSt, AMDGPU::OpName::sbase); in getMemOperandWithOffset()
360 MachineOperand *VAddr = getNamedOperand(LdSt, AMDGPU::OpName::vaddr); in getMemOperandWithOffset()
363 if (getNamedOperand(LdSt, AMDGPU::OpName::saddr)) in getMemOperandWithOffset()
369 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::saddr); in getMemOperandWithOffset()
372 Offset = getNamedOperand(LdSt, AMDGPU::OpName::offset)->getImm(); in getMemOperandWithOffset()
435 FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdata); in shouldClusterMemOps()
437 FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdst); in shouldClusterMemOps()
438 SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdata); in shouldClusterMemOps()
440 SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdst); in shouldClusterMemOps()
442 FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::sdst); in shouldClusterMemOps()
443 SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::sdst); in shouldClusterMemOps()
445 FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdst); in shouldClusterMemOps()
446 SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdst); in shouldClusterMemOps()
502 BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_ILLEGAL_COPY), DestReg) in reportIllegalCopy()
512 if (RC == &AMDGPU::VGPR_32RegClass) { in copyPhysReg()
513 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) || in copyPhysReg()
514 AMDGPU::SReg_32RegClass.contains(SrcReg)); in copyPhysReg()
515 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg) in copyPhysReg()
520 if (RC == &AMDGPU::SReg_32_XM0RegClass || in copyPhysReg()
521 RC == &AMDGPU::SReg_32RegClass) { in copyPhysReg()
522 if (SrcReg == AMDGPU::SCC) { in copyPhysReg()
523 BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B32), DestReg) in copyPhysReg()
529 if (!AMDGPU::SReg_32RegClass.contains(SrcReg)) { in copyPhysReg()
534 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg) in copyPhysReg()
539 if (RC == &AMDGPU::SReg_64RegClass) { in copyPhysReg()
540 if (DestReg == AMDGPU::VCC) { in copyPhysReg()
541 if (AMDGPU::SReg_64RegClass.contains(SrcReg)) { in copyPhysReg()
542 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), AMDGPU::VCC) in copyPhysReg()
546 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg)); in copyPhysReg()
547 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32)) in copyPhysReg()
555 if (!AMDGPU::SReg_64RegClass.contains(SrcReg)) { in copyPhysReg()
560 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg) in copyPhysReg()
565 if (DestReg == AMDGPU::SCC) { in copyPhysReg()
566 assert(AMDGPU::SReg_32RegClass.contains(SrcReg)); in copyPhysReg()
567 BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U32)) in copyPhysReg()
574 unsigned Opcode = AMDGPU::V_MOV_B32_e32; in copyPhysReg()
577 Opcode = AMDGPU::S_MOV_B64; in copyPhysReg()
580 Opcode = AMDGPU::S_MOV_B32; in copyPhysReg()
617 NewOpc = AMDGPU::getCommuteRev(Opcode); in commuteOpcode()
623 NewOpc = AMDGPU::getCommuteOrig(Opcode); in commuteOpcode()
637 if (RegClass == &AMDGPU::SReg_32RegClass || in materializeImmediate()
638 RegClass == &AMDGPU::SGPR_32RegClass || in materializeImmediate()
639 RegClass == &AMDGPU::SReg_32_XM0RegClass || in materializeImmediate()
640 RegClass == &AMDGPU::SReg_32_XM0_XEXECRegClass) { in materializeImmediate()
641 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg) in materializeImmediate()
646 if (RegClass == &AMDGPU::SReg_64RegClass || in materializeImmediate()
647 RegClass == &AMDGPU::SGPR_64RegClass || in materializeImmediate()
648 RegClass == &AMDGPU::SReg_64_XEXECRegClass) { in materializeImmediate()
649 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg) in materializeImmediate()
654 if (RegClass == &AMDGPU::VGPR_32RegClass) { in materializeImmediate()
655 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg) in materializeImmediate()
659 if (RegClass == &AMDGPU::VReg_64RegClass) { in materializeImmediate()
660 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), DestReg) in materializeImmediate()
666 unsigned Opcode = AMDGPU::V_MOV_B32_e32; in materializeImmediate()
669 Opcode = AMDGPU::S_MOV_B64; in materializeImmediate()
672 Opcode = AMDGPU::S_MOV_B32; in materializeImmediate()
689 return &AMDGPU::VGPR_32RegClass; in getPreferredSelectRegClass()
699 assert(MRI.getRegClass(DstReg) == &AMDGPU::VGPR_32RegClass && in insertVectorSelect()
703 unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass); in insertVectorSelect()
704 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg) in insertVectorSelect()
706 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
714 unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass); in insertVectorSelect()
715 BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), SReg) in insertVectorSelect()
718 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
725 unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass); in insertVectorSelect()
726 BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), SReg) in insertVectorSelect()
729 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
738 unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass); in insertVectorSelect()
739 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg) in insertVectorSelect()
741 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
750 unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass); in insertVectorSelect()
751 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg) in insertVectorSelect()
753 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
760 unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass); in insertVectorSelect()
761 unsigned SReg2 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in insertVectorSelect()
762 BuildMI(MBB, I, DL, get(AMDGPU::S_OR_SAVEEXEC_B64), SReg2) in insertVectorSelect()
764 BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), SReg) in insertVectorSelect()
767 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
774 unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass); in insertVectorSelect()
775 unsigned SReg2 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in insertVectorSelect()
776 BuildMI(MBB, I, DL, get(AMDGPU::S_OR_SAVEEXEC_B64), SReg2) in insertVectorSelect()
778 BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), SReg) in insertVectorSelect()
781 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in insertVectorSelect()
801 unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in insertEQ()
802 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_EQ_I32_e64), Reg) in insertEQ()
814 unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in insertNE()
815 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_NE_I32_e64), Reg) in insertNE()
825 return RI.isSGPRClass(DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32; in getMovOpcode()
827 return AMDGPU::S_MOV_B64; in getMovOpcode()
829 return AMDGPU::V_MOV_B64_PSEUDO; in getMovOpcode()
831 return AMDGPU::COPY; in getMovOpcode()
837 return AMDGPU::SI_SPILL_S32_SAVE; in getSGPRSpillSaveOpcode()
839 return AMDGPU::SI_SPILL_S64_SAVE; in getSGPRSpillSaveOpcode()
841 return AMDGPU::SI_SPILL_S128_SAVE; in getSGPRSpillSaveOpcode()
843 return AMDGPU::SI_SPILL_S256_SAVE; in getSGPRSpillSaveOpcode()
845 return AMDGPU::SI_SPILL_S512_SAVE; in getSGPRSpillSaveOpcode()
854 return AMDGPU::SI_SPILL_V32_SAVE; in getVGPRSpillSaveOpcode()
856 return AMDGPU::SI_SPILL_V64_SAVE; in getVGPRSpillSaveOpcode()
858 return AMDGPU::SI_SPILL_V96_SAVE; in getVGPRSpillSaveOpcode()
860 return AMDGPU::SI_SPILL_V128_SAVE; in getVGPRSpillSaveOpcode()
862 return AMDGPU::SI_SPILL_V256_SAVE; in getVGPRSpillSaveOpcode()
864 return AMDGPU::SI_SPILL_V512_SAVE; in getVGPRSpillSaveOpcode()
901 MRI.constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0RegClass); in storeRegToStackSlot()
917 Spill.addReg(AMDGPU::M0, RegState::ImplicitDefine | RegState::Dead); in storeRegToStackSlot()
939 return AMDGPU::SI_SPILL_S32_RESTORE; in getSGPRSpillRestoreOpcode()
941 return AMDGPU::SI_SPILL_S64_RESTORE; in getSGPRSpillRestoreOpcode()
943 return AMDGPU::SI_SPILL_S128_RESTORE; in getSGPRSpillRestoreOpcode()
945 return AMDGPU::SI_SPILL_S256_RESTORE; in getSGPRSpillRestoreOpcode()
947 return AMDGPU::SI_SPILL_S512_RESTORE; in getSGPRSpillRestoreOpcode()
956 return AMDGPU::SI_SPILL_V32_RESTORE; in getVGPRSpillRestoreOpcode()
958 return AMDGPU::SI_SPILL_V64_RESTORE; in getVGPRSpillRestoreOpcode()
960 return AMDGPU::SI_SPILL_V96_RESTORE; in getVGPRSpillRestoreOpcode()
962 return AMDGPU::SI_SPILL_V128_RESTORE; in getVGPRSpillRestoreOpcode()
964 return AMDGPU::SI_SPILL_V256_RESTORE; in getVGPRSpillRestoreOpcode()
966 return AMDGPU::SI_SPILL_V512_RESTORE; in getVGPRSpillRestoreOpcode()
999 MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0RegClass); in loadRegFromStackSlot()
1011 Spill.addReg(AMDGPU::M0, RegState::ImplicitDefine | RegState::Dead); in loadRegFromStackSlot()
1045 TIDReg = RI.findUnusedRegister(MF->getRegInfo(), &AMDGPU::VGPR_32RegClass, in calculateLDSSpillAddress()
1047 if (TIDReg == AMDGPU::NoRegister) in calculateLDSSpillAddress()
1050 if (!AMDGPU::isShader(MF->getFunction().getCallingConv()) && in calculateLDSSpillAddress()
1067 unsigned STmp0 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0); in calculateLDSSpillAddress()
1068 unsigned STmp1 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0); in calculateLDSSpillAddress()
1069 BuildMI(Entry, Insert, DL, get(AMDGPU::S_LOAD_DWORD_IMM), STmp0) in calculateLDSSpillAddress()
1072 BuildMI(Entry, Insert, DL, get(AMDGPU::S_LOAD_DWORD_IMM), STmp1) in calculateLDSSpillAddress()
1077 BuildMI(Entry, Insert, DL, get(AMDGPU::S_MUL_I32), STmp1) in calculateLDSSpillAddress()
1081 BuildMI(Entry, Insert, DL, get(AMDGPU::V_MUL_U32_U24_e32), TIDReg) in calculateLDSSpillAddress()
1085 BuildMI(Entry, Insert, DL, get(AMDGPU::V_MAD_U32_U24), TIDReg) in calculateLDSSpillAddress()
1095 BuildMI(Entry, Insert, DL, get(AMDGPU::V_MBCNT_LO_U32_B32_e64), in calculateLDSSpillAddress()
1100 BuildMI(Entry, Insert, DL, get(AMDGPU::V_MBCNT_HI_U32_B32_e64), in calculateLDSSpillAddress()
1106 BuildMI(Entry, Insert, DL, get(AMDGPU::V_LSHLREV_B32_e32), in calculateLDSSpillAddress()
1133 BuildMI(MBB, MI, DL, get(AMDGPU::S_NOP)) in insertWaitStates()
1153 get(Info->returnsVoid() ? AMDGPU::S_ENDPGM : AMDGPU::SI_RETURN_TO_EPILOG)); in insertReturn()
1161 case AMDGPU::S_NOP: in getNumWaitStates()
1171 case AMDGPU::S_MOV_B64_term: in expandPostRAPseudo()
1174 MI.setDesc(get(AMDGPU::S_MOV_B64)); in expandPostRAPseudo()
1177 case AMDGPU::S_XOR_B64_term: in expandPostRAPseudo()
1180 MI.setDesc(get(AMDGPU::S_XOR_B64)); in expandPostRAPseudo()
1183 case AMDGPU::S_ANDN2_B64_term: in expandPostRAPseudo()
1186 MI.setDesc(get(AMDGPU::S_ANDN2_B64)); in expandPostRAPseudo()
1189 case AMDGPU::V_MOV_B64_PSEUDO: { in expandPostRAPseudo()
1191 unsigned DstLo = RI.getSubReg(Dst, AMDGPU::sub0); in expandPostRAPseudo()
1192 unsigned DstHi = RI.getSubReg(Dst, AMDGPU::sub1); in expandPostRAPseudo()
1199 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo) in expandPostRAPseudo()
1202 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi) in expandPostRAPseudo()
1207 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo) in expandPostRAPseudo()
1208 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub0)) in expandPostRAPseudo()
1210 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi) in expandPostRAPseudo()
1211 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub1)) in expandPostRAPseudo()
1217 case AMDGPU::V_SET_INACTIVE_B32: { in expandPostRAPseudo()
1218 BuildMI(MBB, MI, DL, get(AMDGPU::S_NOT_B64), AMDGPU::EXEC) in expandPostRAPseudo()
1219 .addReg(AMDGPU::EXEC); in expandPostRAPseudo()
1220 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg()) in expandPostRAPseudo()
1222 BuildMI(MBB, MI, DL, get(AMDGPU::S_NOT_B64), AMDGPU::EXEC) in expandPostRAPseudo()
1223 .addReg(AMDGPU::EXEC); in expandPostRAPseudo()
1227 case AMDGPU::V_SET_INACTIVE_B64: { in expandPostRAPseudo()
1228 BuildMI(MBB, MI, DL, get(AMDGPU::S_NOT_B64), AMDGPU::EXEC) in expandPostRAPseudo()
1229 .addReg(AMDGPU::EXEC); in expandPostRAPseudo()
1230 MachineInstr *Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), in expandPostRAPseudo()
1234 BuildMI(MBB, MI, DL, get(AMDGPU::S_NOT_B64), AMDGPU::EXEC) in expandPostRAPseudo()
1235 .addReg(AMDGPU::EXEC); in expandPostRAPseudo()
1239 case AMDGPU::V_MOVRELD_B32_V1: in expandPostRAPseudo()
1240 case AMDGPU::V_MOVRELD_B32_V2: in expandPostRAPseudo()
1241 case AMDGPU::V_MOVRELD_B32_V4: in expandPostRAPseudo()
1242 case AMDGPU::V_MOVRELD_B32_V8: in expandPostRAPseudo()
1243 case AMDGPU::V_MOVRELD_B32_V16: { in expandPostRAPseudo()
1244 const MCInstrDesc &MovRelDesc = get(AMDGPU::V_MOVRELD_B32_e32); in expandPostRAPseudo()
1247 unsigned SubReg = AMDGPU::sub0 + MI.getOperand(3).getImm(); in expandPostRAPseudo()
1266 case AMDGPU::SI_PC_ADD_REL_OFFSET: { in expandPostRAPseudo()
1269 unsigned RegLo = RI.getSubReg(Reg, AMDGPU::sub0); in expandPostRAPseudo()
1270 unsigned RegHi = RI.getSubReg(Reg, AMDGPU::sub1); in expandPostRAPseudo()
1275 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_GETPC_B64), Reg)); in expandPostRAPseudo()
1279 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_ADD_U32), RegLo) in expandPostRAPseudo()
1283 MachineInstrBuilder MIB = BuildMI(MF, DL, get(AMDGPU::S_ADDC_U32), RegHi) in expandPostRAPseudo()
1296 case AMDGPU::EXIT_WWM: { in expandPostRAPseudo()
1299 MI.setDesc(get(AMDGPU::S_MOV_B64)); in expandPostRAPseudo()
1376 assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) == in commuteInstructionImpl()
1378 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) == in commuteInstructionImpl()
1406 swapSourceModifiers(MI, Src0, AMDGPU::OpName::src0_modifiers, in commuteInstructionImpl()
1407 Src1, AMDGPU::OpName::src1_modifiers); in commuteInstructionImpl()
1429 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); in findCommutedOpIndices()
1433 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); in findCommutedOpIndices()
1444 assert(BranchOp != AMDGPU::S_SETPC_B64); in isBranchOffsetInRange()
1458 if (MI.getOpcode() == AMDGPU::S_SETPC_B64) { in getBranchDestBlock()
1482 unsigned PCReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in insertIndirectBranch()
1488 MachineInstr *GetPC = BuildMI(MBB, I, DL, get(AMDGPU::S_GETPC_B64), PCReg); in insertIndirectBranch()
1492 BuildMI(MBB, I, DL, get(AMDGPU::S_ADD_U32)) in insertIndirectBranch()
1493 .addReg(PCReg, RegState::Define, AMDGPU::sub0) in insertIndirectBranch()
1494 .addReg(PCReg, 0, AMDGPU::sub0) in insertIndirectBranch()
1495 .addMBB(&DestBB, AMDGPU::TF_LONG_BRANCH_FORWARD); in insertIndirectBranch()
1496 BuildMI(MBB, I, DL, get(AMDGPU::S_ADDC_U32)) in insertIndirectBranch()
1497 .addReg(PCReg, RegState::Define, AMDGPU::sub1) in insertIndirectBranch()
1498 .addReg(PCReg, 0, AMDGPU::sub1) in insertIndirectBranch()
1502 BuildMI(MBB, I, DL, get(AMDGPU::S_SUB_U32)) in insertIndirectBranch()
1503 .addReg(PCReg, RegState::Define, AMDGPU::sub0) in insertIndirectBranch()
1504 .addReg(PCReg, 0, AMDGPU::sub0) in insertIndirectBranch()
1505 .addMBB(&DestBB, AMDGPU::TF_LONG_BRANCH_BACKWARD); in insertIndirectBranch()
1506 BuildMI(MBB, I, DL, get(AMDGPU::S_SUBB_U32)) in insertIndirectBranch()
1507 .addReg(PCReg, RegState::Define, AMDGPU::sub1) in insertIndirectBranch()
1508 .addReg(PCReg, 0, AMDGPU::sub1) in insertIndirectBranch()
1513 BuildMI(&MBB, DL, get(AMDGPU::S_SETPC_B64)) in insertIndirectBranch()
1554 AMDGPU::SReg_64RegClass, in insertIndirectBranch()
1566 return AMDGPU::S_CBRANCH_SCC1; in getBranchOpcode()
1568 return AMDGPU::S_CBRANCH_SCC0; in getBranchOpcode()
1570 return AMDGPU::S_CBRANCH_VCCNZ; in getBranchOpcode()
1572 return AMDGPU::S_CBRANCH_VCCZ; in getBranchOpcode()
1574 return AMDGPU::S_CBRANCH_EXECNZ; in getBranchOpcode()
1576 return AMDGPU::S_CBRANCH_EXECZ; in getBranchOpcode()
1584 case AMDGPU::S_CBRANCH_SCC0: in getBranchPredicate()
1586 case AMDGPU::S_CBRANCH_SCC1: in getBranchPredicate()
1588 case AMDGPU::S_CBRANCH_VCCNZ: in getBranchPredicate()
1590 case AMDGPU::S_CBRANCH_VCCZ: in getBranchPredicate()
1592 case AMDGPU::S_CBRANCH_EXECNZ: in getBranchPredicate()
1594 case AMDGPU::S_CBRANCH_EXECZ: in getBranchPredicate()
1607 if (I->getOpcode() == AMDGPU::S_BRANCH) { in analyzeBranchImpl()
1615 if (I->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) { in analyzeBranchImpl()
1635 if (I->getOpcode() == AMDGPU::S_BRANCH) { in analyzeBranchImpl()
1656 I->getOpcode() != AMDGPU::SI_MASK_BRANCH) { in analyzeBranch()
1658 case AMDGPU::SI_MASK_BRANCH: in analyzeBranch()
1659 case AMDGPU::S_MOV_B64_term: in analyzeBranch()
1660 case AMDGPU::S_XOR_B64_term: in analyzeBranch()
1661 case AMDGPU::S_ANDN2_B64_term: in analyzeBranch()
1663 case AMDGPU::SI_IF: in analyzeBranch()
1664 case AMDGPU::SI_ELSE: in analyzeBranch()
1665 case AMDGPU::SI_KILL_I1_TERMINATOR: in analyzeBranch()
1666 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR: in analyzeBranch()
1679 if (I->getOpcode() != AMDGPU::SI_MASK_BRANCH) in analyzeBranch()
1717 if (I->getOpcode() == AMDGPU::SI_MASK_BRANCH) { in removeBranch()
1748 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH)) in insertBranch()
1756 BuildMI(&MBB, DL, get(AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO)) in insertBranch()
1786 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH)) in insertBranch()
1825 int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32; in canInsertSelect()
1839 int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32; in canInsertSelect()
1869 AMDGPU::S_CSELECT_B32 : AMDGPU::V_CNDMASK_B32_e32; in insertSelect()
1883 BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), DstReg) in insertSelect()
1892 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3, in insertSelect()
1893 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7, in insertSelect()
1894 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11, in insertSelect()
1895 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15, in insertSelect()
1899 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3, in insertSelect()
1900 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7, in insertSelect()
1901 AMDGPU::sub8_sub9, AMDGPU::sub10_sub11, in insertSelect()
1902 AMDGPU::sub12_sub13, AMDGPU::sub14_sub15, in insertSelect()
1905 unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32; in insertSelect()
1906 const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass; in insertSelect()
1912 SelOp = AMDGPU::S_CSELECT_B64; in insertSelect()
1913 EltRC = &AMDGPU::SGPR_64RegClass; in insertSelect()
1921 MBB, I, DL, get(AMDGPU::REG_SEQUENCE), DstReg); in insertSelect()
1945 case AMDGPU::V_MOV_B32_e32: in isFoldableCopy()
1946 case AMDGPU::V_MOV_B32_e64: in isFoldableCopy()
1947 case AMDGPU::V_MOV_B64_PSEUDO: { in isFoldableCopy()
1955 case AMDGPU::S_MOV_B32: in isFoldableCopy()
1956 case AMDGPU::S_MOV_B64: in isFoldableCopy()
1957 case AMDGPU::COPY: in isFoldableCopy()
1983 int Src0ModIdx = AMDGPU::getNamedOperandIdx(Opc, in removeModOperands()
1984 AMDGPU::OpName::src0_modifiers); in removeModOperands()
1985 int Src1ModIdx = AMDGPU::getNamedOperandIdx(Opc, in removeModOperands()
1986 AMDGPU::OpName::src1_modifiers); in removeModOperands()
1987 int Src2ModIdx = AMDGPU::getNamedOperandIdx(Opc, in removeModOperands()
1988 AMDGPU::OpName::src2_modifiers); in removeModOperands()
2003 case AMDGPU::S_MOV_B64: in FoldImmediate()
2008 case AMDGPU::V_MOV_B32_e32: in FoldImmediate()
2009 case AMDGPU::S_MOV_B32: in FoldImmediate()
2013 const MachineOperand *ImmOp = getNamedOperand(DefMI, AMDGPU::OpName::src0); in FoldImmediate()
2020 if (Opc == AMDGPU::COPY) { in FoldImmediate()
2022 unsigned NewOpc = isVGPRCopy ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32; in FoldImmediate()
2029 if (Opc == AMDGPU::V_MAD_F32 || Opc == AMDGPU::V_MAC_F32_e64 || in FoldImmediate()
2030 Opc == AMDGPU::V_MAD_F16 || Opc == AMDGPU::V_MAC_F16_e64) { in FoldImmediate()
2039 MachineOperand *Src0 = getNamedOperand(UseMI, AMDGPU::OpName::src0); in FoldImmediate()
2045 bool IsF32 = Opc == AMDGPU::V_MAD_F32 || Opc == AMDGPU::V_MAC_F32_e64; in FoldImmediate()
2046 MachineOperand *Src1 = getNamedOperand(UseMI, AMDGPU::OpName::src1); in FoldImmediate()
2047 MachineOperand *Src2 = getNamedOperand(UseMI, AMDGPU::OpName::src2); in FoldImmediate()
2067 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod)); in FoldImmediate()
2069 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp)); in FoldImmediate()
2077 if (Opc == AMDGPU::V_MAC_F32_e64 || in FoldImmediate()
2078 Opc == AMDGPU::V_MAC_F16_e64) in FoldImmediate()
2080 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)); in FoldImmediate()
2085 UseMI.setDesc(get(IsF32 ? AMDGPU::V_MADMK_F32 : AMDGPU::V_MADMK_F16)); in FoldImmediate()
2140 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod)); in FoldImmediate()
2142 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp)); in FoldImmediate()
2144 if (Opc == AMDGPU::V_MAC_F32_e64 || in FoldImmediate()
2145 Opc == AMDGPU::V_MAC_F16_e64) in FoldImmediate()
2147 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)); in FoldImmediate()
2154 UseMI.setDesc(get(IsF32 ? AMDGPU::V_MADAK_F32 : AMDGPU::V_MADAK_F16)); in FoldImmediate()
2267 if (Def && Def->getOpcode() == AMDGPU::V_MOV_B32_e32 && in getFoldableImm()
2270 return AMDGPU::NoRegister; in getFoldableImm()
2278 bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F32_e64; in convertToThreeAddress()
2283 case AMDGPU::V_MAC_F16_e64: in convertToThreeAddress()
2286 case AMDGPU::V_MAC_F32_e64: in convertToThreeAddress()
2287 case AMDGPU::V_FMAC_F32_e64: in convertToThreeAddress()
2289 case AMDGPU::V_MAC_F16_e32: in convertToThreeAddress()
2292 case AMDGPU::V_MAC_F32_e32: in convertToThreeAddress()
2293 case AMDGPU::V_FMAC_F32_e32: { in convertToThreeAddress()
2294 int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), in convertToThreeAddress()
2295 AMDGPU::OpName::src0); in convertToThreeAddress()
2307 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst); in convertToThreeAddress()
2308 const MachineOperand *Src0 = getNamedOperand(MI, AMDGPU::OpName::src0); in convertToThreeAddress()
2310 getNamedOperand(MI, AMDGPU::OpName::src0_modifiers); in convertToThreeAddress()
2311 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1); in convertToThreeAddress()
2313 getNamedOperand(MI, AMDGPU::OpName::src1_modifiers); in convertToThreeAddress()
2314 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); in convertToThreeAddress()
2315 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp); in convertToThreeAddress()
2316 const MachineOperand *Omod = getNamedOperand(MI, AMDGPU::OpName::omod); in convertToThreeAddress()
2323 get(IsF16 ? AMDGPU::V_MADAK_F16 : AMDGPU::V_MADAK_F32)) in convertToThreeAddress()
2331 get(IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32)) in convertToThreeAddress()
2338 if (isOperandLegal(MI, AMDGPU::getNamedOperandIdx(AMDGPU::V_MADMK_F32, in convertToThreeAddress()
2339 AMDGPU::OpName::src0), Src1)) in convertToThreeAddress()
2341 get(IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32)) in convertToThreeAddress()
2350 unsigned NewOpc = IsFMA ? AMDGPU::V_FMA_F32 : in convertToThreeAddress()
2351 (IsF16 ? AMDGPU::V_MAD_F16 : AMDGPU::V_MAD_F32); in convertToThreeAddress()
2369 case AMDGPU::S_SET_GPR_IDX_ON: in changesVGPRIndexingMode()
2370 case AMDGPU::S_SET_GPR_IDX_MODE: in changesVGPRIndexingMode()
2371 case AMDGPU::S_SET_GPR_IDX_OFF: in changesVGPRIndexingMode()
2387 MI.modifiesRegister(AMDGPU::EXEC, &RI) || in isSchedulingBoundary()
2388 MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 || in isSchedulingBoundary()
2389 MI.getOpcode() == AMDGPU::S_SETREG_B32 || in isSchedulingBoundary()
2394 return Opcode == AMDGPU::DS_ORDERED_COUNT || in isAlwaysGDS()
2395 Opcode == AMDGPU::DS_GWS_INIT || in isAlwaysGDS()
2396 Opcode == AMDGPU::DS_GWS_SEMA_V || in isAlwaysGDS()
2397 Opcode == AMDGPU::DS_GWS_SEMA_BR || in isAlwaysGDS()
2398 Opcode == AMDGPU::DS_GWS_SEMA_P || in isAlwaysGDS()
2399 Opcode == AMDGPU::DS_GWS_SEMA_RELEASE_ALL || in isAlwaysGDS()
2400 Opcode == AMDGPU::DS_GWS_BARRIER; in isAlwaysGDS()
2415 if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT || in hasUnwantedEffectsWhenEXECEmpty()
2416 Opcode == AMDGPU::EXP || Opcode == AMDGPU::EXP_DONE || in hasUnwantedEffectsWhenEXECEmpty()
2417 Opcode == AMDGPU::DS_ORDERED_COUNT) in hasUnwantedEffectsWhenEXECEmpty()
2428 if (Opcode == AMDGPU::V_READFIRSTLANE_B32 || Opcode == AMDGPU::V_READLANE_B32) in hasUnwantedEffectsWhenEXECEmpty()
2437 return AMDGPU::isInlinableLiteral32(Imm.getSExtValue(), in isInlineConstant()
2440 return AMDGPU::isInlinableLiteral64(Imm.getSExtValue(), in isInlineConstant()
2444 AMDGPU::isInlinableLiteral16(Imm.getSExtValue(), in isInlineConstant()
2454 OperandType < AMDGPU::OPERAND_SRC_FIRST || in isInlineConstant()
2455 OperandType > AMDGPU::OPERAND_SRC_LAST) in isInlineConstant()
2465 case AMDGPU::OPERAND_REG_IMM_INT32: in isInlineConstant()
2466 case AMDGPU::OPERAND_REG_IMM_FP32: in isInlineConstant()
2467 case AMDGPU::OPERAND_REG_INLINE_C_INT32: in isInlineConstant()
2468 case AMDGPU::OPERAND_REG_INLINE_C_FP32: { in isInlineConstant()
2470 return AMDGPU::isInlinableLiteral32(Trunc, ST.hasInv2PiInlineImm()); in isInlineConstant()
2472 case AMDGPU::OPERAND_REG_IMM_INT64: in isInlineConstant()
2473 case AMDGPU::OPERAND_REG_IMM_FP64: in isInlineConstant()
2474 case AMDGPU::OPERAND_REG_INLINE_C_INT64: in isInlineConstant()
2475 case AMDGPU::OPERAND_REG_INLINE_C_FP64: in isInlineConstant()
2476 return AMDGPU::isInlinableLiteral64(MO.getImm(), in isInlineConstant()
2478 case AMDGPU::OPERAND_REG_IMM_INT16: in isInlineConstant()
2479 case AMDGPU::OPERAND_REG_IMM_FP16: in isInlineConstant()
2480 case AMDGPU::OPERAND_REG_INLINE_C_INT16: in isInlineConstant()
2481 case AMDGPU::OPERAND_REG_INLINE_C_FP16: { in isInlineConstant()
2489 AMDGPU::isInlinableLiteral16(Trunc, ST.hasInv2PiInlineImm()); in isInlineConstant()
2494 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16: in isInlineConstant()
2495 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: { in isInlineConstant()
2499 AMDGPU::isInlinableLiteral16(Trunc, ST.hasInv2PiInlineImm()); in isInlineConstant()
2503 AMDGPU::isInlinableLiteral16(Imm >> 16, ST.hasInv2PiInlineImm()); in isInlineConstant()
2506 return AMDGPU::isInlinableLiteralV216(Trunc, ST.hasInv2PiInlineImm()); in isInlineConstant()
2565 int Op32 = AMDGPU::getVOPe32(Opcode); in hasVALU32BitEncoding()
2576 return AMDGPU::getNamedOperandIdx(Opcode, in hasModifiers()
2577 AMDGPU::OpName::src0_modifiers) != -1; in hasModifiers()
2587 return hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) || in hasAnyModifiersSet()
2588 hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) || in hasAnyModifiersSet()
2589 hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers) || in hasAnyModifiersSet()
2590 hasModifiersSet(MI, AMDGPU::OpName::clamp) || in hasAnyModifiersSet()
2591 hasModifiersSet(MI, AMDGPU::OpName::omod); in hasAnyModifiersSet()
2596 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); in canShrink()
2607 case AMDGPU::V_ADDC_U32_e64: in canShrink()
2608 case AMDGPU::V_SUBB_U32_e64: in canShrink()
2609 case AMDGPU::V_SUBBREV_U32_e64: { in canShrink()
2611 = getNamedOperand(MI, AMDGPU::OpName::src1); in canShrink()
2617 case AMDGPU::V_MAC_F32_e64: in canShrink()
2618 case AMDGPU::V_MAC_F16_e64: in canShrink()
2619 case AMDGPU::V_FMAC_F32_e64: in canShrink()
2621 hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers)) in canShrink()
2625 case AMDGPU::V_CNDMASK_B32_e64: in canShrink()
2630 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1); in canShrink()
2632 hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers))) in canShrink()
2637 if (hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers)) in canShrink()
2645 return !hasModifiersSet(MI, AMDGPU::OpName::omod) && in canShrink()
2646 !hasModifiersSet(MI, AMDGPU::OpName::clamp); in canShrink()
2655 if (Use.isUse() && Use.getReg() == AMDGPU::VCC) { in copyFlagsToImplicitVCC()
2671 int Op32DstIdx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::vdst); in buildShrunkInst()
2676 assert(MI.getOperand(0).getReg() == AMDGPU::VCC && in buildShrunkInst()
2680 Inst32.add(*getNamedOperand(MI, AMDGPU::OpName::src0)); in buildShrunkInst()
2682 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1); in buildShrunkInst()
2686 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); in buildShrunkInst()
2689 int Op32Src2Idx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::src2); in buildShrunkInst()
2722 if (!MO.isImplicit() && (MO.getReg() == AMDGPU::FLAT_SCR)) in usesConstantBus()
2726 if (!MO.isImplicit() && MO.getReg() == AMDGPU::EXEC) in usesConstantBus()
2730 return (MO.getReg() == AMDGPU::VCC || MO.getReg() == AMDGPU::M0 || in usesConstantBus()
2732 (AMDGPU::SGPR_32RegClass.contains(MO.getReg()) || in usesConstantBus()
2733 AMDGPU::SGPR_64RegClass.contains(MO.getReg())))); in usesConstantBus()
2743 case AMDGPU::VCC: in findImplicitSGPRRead()
2744 case AMDGPU::M0: in findImplicitSGPRRead()
2745 case AMDGPU::FLAT_SCR: in findImplicitSGPRRead()
2753 return AMDGPU::NoRegister; in findImplicitSGPRRead()
2759 case AMDGPU::V_READLANE_B32: in shouldReadExec()
2760 case AMDGPU::V_READLANE_B32_si: in shouldReadExec()
2761 case AMDGPU::V_READLANE_B32_vi: in shouldReadExec()
2762 case AMDGPU::V_WRITELANE_B32: in shouldReadExec()
2763 case AMDGPU::V_WRITELANE_B32_si: in shouldReadExec()
2764 case AMDGPU::V_WRITELANE_B32_vi: in shouldReadExec()
2785 return SubReg.getSubReg() != AMDGPU::NoSubRegister && in isSubRegOf()
2798 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0); in verifyInstruction()
2799 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1); in verifyInstruction()
2800 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2); in verifyInstruction()
2849 case AMDGPU::OPERAND_REG_IMM_INT32: in verifyInstruction()
2850 case AMDGPU::OPERAND_REG_IMM_FP32: in verifyInstruction()
2852 case AMDGPU::OPERAND_REG_INLINE_C_INT32: in verifyInstruction()
2853 case AMDGPU::OPERAND_REG_INLINE_C_FP32: in verifyInstruction()
2854 case AMDGPU::OPERAND_REG_INLINE_C_INT64: in verifyInstruction()
2855 case AMDGPU::OPERAND_REG_INLINE_C_FP64: in verifyInstruction()
2856 case AMDGPU::OPERAND_REG_INLINE_C_INT16: in verifyInstruction()
2857 case AMDGPU::OPERAND_REG_INLINE_C_FP16: { in verifyInstruction()
2866 case AMDGPU::OPERAND_KIMM32: in verifyInstruction()
2884 if (Reg == AMDGPU::NoRegister || in verifyInstruction()
2903 int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst); in verifyInstruction()
2929 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod); in verifyInstruction()
2937 uint16_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode); in verifyInstruction()
2942 if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) { in verifyInstruction()
2948 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp); in verifyInstruction()
2955 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod); in verifyInstruction()
2963 const MachineOperand *DstUnused = getNamedOperand(MI, AMDGPU::OpName::dst_unused); in verifyInstruction()
2965 DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE) { in verifyInstruction()
2990 const MachineOperand *DMask = getNamedOperand(MI, AMDGPU::OpName::dmask); in verifyInstruction()
2995 const MachineOperand *TFE = getNamedOperand(MI, AMDGPU::OpName::tfe); in verifyInstruction()
2996 const MachineOperand *LWE = getNamedOperand(MI, AMDGPU::OpName::lwe); in verifyInstruction()
2997 const MachineOperand *D16 = getNamedOperand(MI, AMDGPU::OpName::d16); in verifyInstruction()
3008 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata); in verifyInstruction()
3023 if (Desc.getOpcode() != AMDGPU::V_WRITELANE_B32 in verifyInstruction()
3033 if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1) in verifyInstruction()
3037 if (SGPRUsed != AMDGPU::NoRegister) in verifyInstruction()
3067 if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32 || in verifyInstruction()
3068 Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64) { in verifyInstruction()
3082 int64_t Imm = getNamedOperand(MI, AMDGPU::OpName::simm16)->getImm(); in verifyInstruction()
3096 if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 || in verifyInstruction()
3097 Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 || in verifyInstruction()
3098 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 || in verifyInstruction()
3099 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) { in verifyInstruction()
3100 const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 || in verifyInstruction()
3101 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64; in verifyInstruction()
3115 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst); in verifyInstruction()
3143 if (!MI.hasRegisterImplicitUseOperand(AMDGPU::EXEC)) { in verifyInstruction()
3153 const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soff); in verifyInstruction()
3154 if (Soff && Soff->getReg() != AMDGPU::M0) { in verifyInstruction()
3162 const MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset); in verifyInstruction()
3169 const MachineOperand *DppCt = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl); in verifyInstruction()
3171 using namespace AMDGPU::DPP; in verifyInstruction()
3190 default: return AMDGPU::INSTRUCTION_LIST_END; in getVALUOp()
3191 case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE; in getVALUOp()
3192 case AMDGPU::COPY: return AMDGPU::COPY; in getVALUOp()
3193 case AMDGPU::PHI: return AMDGPU::PHI; in getVALUOp()
3194 case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG; in getVALUOp()
3195 case AMDGPU::WQM: return AMDGPU::WQM; in getVALUOp()
3196 case AMDGPU::WWM: return AMDGPU::WWM; in getVALUOp()
3197 case AMDGPU::S_MOV_B32: in getVALUOp()
3199 AMDGPU::COPY : AMDGPU::V_MOV_B32_e32; in getVALUOp()
3200 case AMDGPU::S_ADD_I32: in getVALUOp()
3201 return ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_I32_e32; in getVALUOp()
3202 case AMDGPU::S_ADDC_U32: in getVALUOp()
3203 return AMDGPU::V_ADDC_U32_e32; in getVALUOp()
3204 case AMDGPU::S_SUB_I32: in getVALUOp()
3205 return ST.hasAddNoCarry() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_I32_e32; in getVALUOp()
3208 case AMDGPU::S_ADD_U32: in getVALUOp()
3209 return AMDGPU::V_ADD_I32_e32; in getVALUOp()
3210 case AMDGPU::S_SUB_U32: in getVALUOp()
3211 return AMDGPU::V_SUB_I32_e32; in getVALUOp()
3212 case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32; in getVALUOp()
3213 case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_I32; in getVALUOp()
3214 case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64; in getVALUOp()
3215 case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64; in getVALUOp()
3216 case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64; in getVALUOp()
3217 case AMDGPU::S_XNOR_B32: in getVALUOp()
3218 return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END; in getVALUOp()
3219 case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64; in getVALUOp()
3220 case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64; in getVALUOp()
3221 case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64; in getVALUOp()
3222 case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64; in getVALUOp()
3223 case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32; in getVALUOp()
3224 case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64; in getVALUOp()
3225 case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32; in getVALUOp()
3226 case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64; in getVALUOp()
3227 case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32; in getVALUOp()
3228 case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64; in getVALUOp()
3229 case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32; in getVALUOp()
3230 case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32; in getVALUOp()
3231 case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32; in getVALUOp()
3232 case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32; in getVALUOp()
3233 case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64; in getVALUOp()
3234 case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32; in getVALUOp()
3235 case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32; in getVALUOp()
3236 case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32; in getVALUOp()
3237 case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e32; in getVALUOp()
3238 case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e32; in getVALUOp()
3239 case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e32; in getVALUOp()
3240 case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e32; in getVALUOp()
3241 case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e32; in getVALUOp()
3242 case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e32; in getVALUOp()
3243 case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e32; in getVALUOp()
3244 case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e32; in getVALUOp()
3245 case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e32; in getVALUOp()
3246 case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e32; in getVALUOp()
3247 case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e32; in getVALUOp()
3248 case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e32; in getVALUOp()
3249 case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e32; in getVALUOp()
3250 case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e32; in getVALUOp()
3251 case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64; in getVALUOp()
3252 case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32; in getVALUOp()
3253 case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32; in getVALUOp()
3254 case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64; in getVALUOp()
3255 case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ; in getVALUOp()
3256 case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ; in getVALUOp()
3279 case AMDGPU::COPY: in canReadVGPR()
3280 case AMDGPU::REG_SEQUENCE: in canReadVGPR()
3281 case AMDGPU::PHI: in canReadVGPR()
3282 case AMDGPU::INSERT_SUBREG: in canReadVGPR()
3296 unsigned Opcode = AMDGPU::V_MOV_B32_e32; in legalizeOpWithMove()
3298 Opcode = AMDGPU::COPY; in legalizeOpWithMove()
3300 Opcode = AMDGPU::S_MOV_B32; in legalizeOpWithMove()
3303 if (RI.getCommonSubClass(&AMDGPU::VReg_64RegClass, VRC)) in legalizeOpWithMove()
3304 VRC = &AMDGPU::VReg_64RegClass; in legalizeOpWithMove()
3306 VRC = &AMDGPU::VGPR_32RegClass; in legalizeOpWithMove()
3325 if (SuperReg.getSubReg() == AMDGPU::NoSubRegister) { in buildExtractSubReg()
3354 if (SubIdx == AMDGPU::sub0) in buildExtractSubRegOrImm()
3356 if (SubIdx == AMDGPU::sub1) in buildExtractSubRegOrImm()
3439 } else if (InstDesc.OpInfo[i].OperandType == AMDGPU::OPERAND_KIMM32) { in isOperandLegal()
3466 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); in legalizeOperandsVOP2()
3475 bool HasImplicitSGPR = findImplicitSGPRRead(MI) != AMDGPU::NoRegister; in legalizeOperandsVOP2()
3477 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); in legalizeOperandsVOP2()
3487 if (Opc == AMDGPU::V_WRITELANE_B32) { in legalizeOperandsVOP2()
3488 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); in legalizeOperandsVOP2()
3492 unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in legalizeOperandsVOP2()
3493 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) in legalizeOperandsVOP2()
3498 unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in legalizeOperandsVOP2()
3500 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) in legalizeOperandsVOP2()
3515 if (Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() && in legalizeOperandsVOP2()
3517 unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in legalizeOperandsVOP2()
3519 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) in legalizeOperandsVOP2()
3534 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); in legalizeOperandsVOP2()
3581 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0), in legalizeOperandsVOP3()
3582 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1), in legalizeOperandsVOP3()
3583 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2) in legalizeOperandsVOP3()
3602 if (SGPRReg == AMDGPU::NoRegister || SGPRReg == MO.getReg()) { in legalizeOperandsVOP3()
3623 get(AMDGPU::V_READFIRSTLANE_B32), DstReg) in readlaneVGPRToSGPR()
3630 unsigned SGPR = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); in readlaneVGPRToSGPR()
3632 get(AMDGPU::V_READFIRSTLANE_B32), SGPR) in readlaneVGPRToSGPR()
3639 get(AMDGPU::REG_SEQUENCE), DstReg); in readlaneVGPRToSGPR()
3654 MachineOperand *SBase = getNamedOperand(MI, AMDGPU::OpName::sbase); in legalizeOperandsSMRD()
3659 MachineOperand *SOff = getNamedOperand(MI, AMDGPU::OpName::soff); in legalizeOperandsSMRD()
3684 BuildMI(InsertMBB, I, DL, get(AMDGPU::COPY), DstReg).add(Op); in legalizeGenericOperand()
3710 unsigned SaveExec = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in emitLoadSRsrcFromVGPRLoop()
3711 unsigned CondReg0 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in emitLoadSRsrcFromVGPRLoop()
3712 unsigned CondReg1 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in emitLoadSRsrcFromVGPRLoop()
3713 unsigned AndCond = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in emitLoadSRsrcFromVGPRLoop()
3714 unsigned SRsrcSub0 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); in emitLoadSRsrcFromVGPRLoop()
3715 unsigned SRsrcSub1 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); in emitLoadSRsrcFromVGPRLoop()
3716 unsigned SRsrcSub2 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); in emitLoadSRsrcFromVGPRLoop()
3717 unsigned SRsrcSub3 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); in emitLoadSRsrcFromVGPRLoop()
3718 unsigned SRsrc = MRI.createVirtualRegister(&AMDGPU::SReg_128RegClass); in emitLoadSRsrcFromVGPRLoop()
3721 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub0) in emitLoadSRsrcFromVGPRLoop()
3722 .addReg(VRsrc, VRsrcUndef, AMDGPU::sub0); in emitLoadSRsrcFromVGPRLoop()
3723 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub1) in emitLoadSRsrcFromVGPRLoop()
3724 .addReg(VRsrc, VRsrcUndef, AMDGPU::sub1); in emitLoadSRsrcFromVGPRLoop()
3725 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub2) in emitLoadSRsrcFromVGPRLoop()
3726 .addReg(VRsrc, VRsrcUndef, AMDGPU::sub2); in emitLoadSRsrcFromVGPRLoop()
3727 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub3) in emitLoadSRsrcFromVGPRLoop()
3728 .addReg(VRsrc, VRsrcUndef, AMDGPU::sub3); in emitLoadSRsrcFromVGPRLoop()
3730 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), SRsrc) in emitLoadSRsrcFromVGPRLoop()
3732 .addImm(AMDGPU::sub0) in emitLoadSRsrcFromVGPRLoop()
3734 .addImm(AMDGPU::sub1) in emitLoadSRsrcFromVGPRLoop()
3736 .addImm(AMDGPU::sub2) in emitLoadSRsrcFromVGPRLoop()
3738 .addImm(AMDGPU::sub3); in emitLoadSRsrcFromVGPRLoop()
3745 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), CondReg0) in emitLoadSRsrcFromVGPRLoop()
3746 .addReg(SRsrc, 0, AMDGPU::sub0_sub1) in emitLoadSRsrcFromVGPRLoop()
3747 .addReg(VRsrc, 0, AMDGPU::sub0_sub1); in emitLoadSRsrcFromVGPRLoop()
3748 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), CondReg1) in emitLoadSRsrcFromVGPRLoop()
3749 .addReg(SRsrc, 0, AMDGPU::sub2_sub3) in emitLoadSRsrcFromVGPRLoop()
3750 .addReg(VRsrc, 0, AMDGPU::sub2_sub3); in emitLoadSRsrcFromVGPRLoop()
3751 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::S_AND_B64), AndCond) in emitLoadSRsrcFromVGPRLoop()
3758 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::S_AND_SAVEEXEC_B64), SaveExec) in emitLoadSRsrcFromVGPRLoop()
3765 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::S_XOR_B64_term), AMDGPU::EXEC) in emitLoadSRsrcFromVGPRLoop()
3766 .addReg(AMDGPU::EXEC) in emitLoadSRsrcFromVGPRLoop()
3768 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::S_CBRANCH_EXECNZ)).addMBB(&LoopBB); in emitLoadSRsrcFromVGPRLoop()
3781 unsigned SaveExec = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass); in loadSRsrcFromVGPR()
3784 BuildMI(MBB, I, DL, TII.get(AMDGPU::S_MOV_B64), SaveExec) in loadSRsrcFromVGPR()
3785 .addReg(AMDGPU::EXEC); in loadSRsrcFromVGPR()
3834 BuildMI(*RemainderBB, First, DL, TII.get(AMDGPU::S_MOV_B64), AMDGPU::EXEC) in loadSRsrcFromVGPR()
3847 TII.buildExtractSubReg(MI, MRI, Rsrc, &AMDGPU::VReg_128RegClass, in extractRsrcPtr()
3848 AMDGPU::sub0_sub1, &AMDGPU::VReg_64RegClass); in extractRsrcPtr()
3851 unsigned Zero64 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in extractRsrcPtr()
3852 unsigned SRsrcFormatLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); in extractRsrcPtr()
3853 unsigned SRsrcFormatHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); in extractRsrcPtr()
3854 unsigned NewSRsrc = MRI.createVirtualRegister(&AMDGPU::SReg_128RegClass); in extractRsrcPtr()
3858 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B64), Zero64) in extractRsrcPtr()
3862 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatLo) in extractRsrcPtr()
3866 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatHi) in extractRsrcPtr()
3870 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::REG_SEQUENCE), NewSRsrc) in extractRsrcPtr()
3872 .addImm(AMDGPU::sub0_sub1) in extractRsrcPtr()
3874 .addImm(AMDGPU::sub2) in extractRsrcPtr()
3876 .addImm(AMDGPU::sub3); in extractRsrcPtr()
3907 if (MI.getOpcode() == AMDGPU::PHI) { in legalizeOperands()
3954 if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) { in legalizeOperands()
3981 if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) { in legalizeOperands()
3995 if (MI.getOpcode() == AMDGPU::SI_INIT_M0) { in legalizeOperands()
4008 (AMDGPU::isShader(MF.getFunction().getCallingConv()) && in legalizeOperands()
4010 MachineOperand *SRsrc = getNamedOperand(MI, AMDGPU::OpName::srsrc); in legalizeOperands()
4016 MachineOperand *SSamp = getNamedOperand(MI, AMDGPU::OpName::ssamp); in legalizeOperands()
4026 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::srsrc); in legalizeOperands()
4053 MachineOperand *VAddr = getNamedOperand(MI, AMDGPU::OpName::vaddr); in legalizeOperands()
4054 if (VAddr && AMDGPU::getIfAddr64Inst(MI.getOpcode()) != -1) { in legalizeOperands()
4057 unsigned NewVAddrLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in legalizeOperands()
4058 unsigned NewVAddrHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in legalizeOperands()
4059 unsigned NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); in legalizeOperands()
4066 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADD_I32_e32), NewVAddrLo) in legalizeOperands()
4067 .addReg(RsrcPtr, 0, AMDGPU::sub0) in legalizeOperands()
4068 .addReg(VAddr->getReg(), 0, AMDGPU::sub0); in legalizeOperands()
4071 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADDC_U32_e32), NewVAddrHi) in legalizeOperands()
4072 .addReg(RsrcPtr, 0, AMDGPU::sub1) in legalizeOperands()
4073 .addReg(VAddr->getReg(), 0, AMDGPU::sub1); in legalizeOperands()
4076 BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewVAddr) in legalizeOperands()
4078 .addImm(AMDGPU::sub0) in legalizeOperands()
4080 .addImm(AMDGPU::sub1); in legalizeOperands()
4094 unsigned NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); in legalizeOperands()
4095 MachineOperand *VData = getNamedOperand(MI, AMDGPU::OpName::vdata); in legalizeOperands()
4096 MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset); in legalizeOperands()
4097 MachineOperand *SOffset = getNamedOperand(MI, AMDGPU::OpName::soffset); in legalizeOperands()
4098 unsigned Addr64Opcode = AMDGPU::getAddr64Inst(MI.getOpcode()); in legalizeOperands()
4102 MachineOperand *VDataIn = getNamedOperand(MI, AMDGPU::OpName::vdata_in); in legalizeOperands()
4117 getNamedOperand(MI, AMDGPU::OpName::glc)) { in legalizeOperands()
4121 MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::slc)); in legalizeOperands()
4124 getNamedOperand(MI, AMDGPU::OpName::tfe)) { in legalizeOperands()
4139 .addImm(getNamedImmOperand(MI, AMDGPU::OpName::slc)) in legalizeOperands()
4146 BuildMI(MBB, Addr64, Addr64->getDebugLoc(), get(AMDGPU::REG_SEQUENCE), in legalizeOperands()
4148 .addReg(RsrcPtr, 0, AMDGPU::sub0) in legalizeOperands()
4149 .addImm(AMDGPU::sub0) in legalizeOperands()
4150 .addReg(RsrcPtr, 0, AMDGPU::sub1) in legalizeOperands()
4151 .addImm(AMDGPU::sub1); in legalizeOperands()
4177 case AMDGPU::S_ADD_U64_PSEUDO: in moveToVALU()
4178 case AMDGPU::S_SUB_U64_PSEUDO: in moveToVALU()
4182 case AMDGPU::S_ADD_I32: in moveToVALU()
4183 case AMDGPU::S_SUB_I32: in moveToVALU()
4190 case AMDGPU::S_AND_B64: in moveToVALU()
4191 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_AND_B32, MDT); in moveToVALU()
4195 case AMDGPU::S_OR_B64: in moveToVALU()
4196 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_OR_B32, MDT); in moveToVALU()
4200 case AMDGPU::S_XOR_B64: in moveToVALU()
4201 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XOR_B32, MDT); in moveToVALU()
4205 case AMDGPU::S_NAND_B64: in moveToVALU()
4206 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NAND_B32, MDT); in moveToVALU()
4210 case AMDGPU::S_NOR_B64: in moveToVALU()
4211 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NOR_B32, MDT); in moveToVALU()
4215 case AMDGPU::S_XNOR_B64: in moveToVALU()
4217 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XNOR_B32, MDT); in moveToVALU()
4223 case AMDGPU::S_ANDN2_B64: in moveToVALU()
4224 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ANDN2_B32, MDT); in moveToVALU()
4228 case AMDGPU::S_ORN2_B64: in moveToVALU()
4229 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ORN2_B32, MDT); in moveToVALU()
4233 case AMDGPU::S_NOT_B64: in moveToVALU()
4234 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_NOT_B32); in moveToVALU()
4238 case AMDGPU::S_BCNT1_I32_B64: in moveToVALU()
4243 case AMDGPU::S_BFE_I64: in moveToVALU()
4248 case AMDGPU::S_LSHL_B32: in moveToVALU()
4250 NewOpcode = AMDGPU::V_LSHLREV_B32_e64; in moveToVALU()
4254 case AMDGPU::S_ASHR_I32: in moveToVALU()
4256 NewOpcode = AMDGPU::V_ASHRREV_I32_e64; in moveToVALU()
4260 case AMDGPU::S_LSHR_B32: in moveToVALU()
4262 NewOpcode = AMDGPU::V_LSHRREV_B32_e64; in moveToVALU()
4266 case AMDGPU::S_LSHL_B64: in moveToVALU()
4268 NewOpcode = AMDGPU::V_LSHLREV_B64; in moveToVALU()
4272 case AMDGPU::S_ASHR_I64: in moveToVALU()
4274 NewOpcode = AMDGPU::V_ASHRREV_I64; in moveToVALU()
4278 case AMDGPU::S_LSHR_B64: in moveToVALU()
4280 NewOpcode = AMDGPU::V_LSHRREV_B64; in moveToVALU()
4285 case AMDGPU::S_ABS_I32: in moveToVALU()
4290 case AMDGPU::S_CBRANCH_SCC0: in moveToVALU()
4291 case AMDGPU::S_CBRANCH_SCC1: in moveToVALU()
4293 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B64), in moveToVALU()
4294 AMDGPU::VCC) in moveToVALU()
4295 .addReg(AMDGPU::EXEC) in moveToVALU()
4296 .addReg(AMDGPU::VCC); in moveToVALU()
4299 case AMDGPU::S_BFE_U64: in moveToVALU()
4300 case AMDGPU::S_BFM_B64: in moveToVALU()
4303 case AMDGPU::S_PACK_LL_B32_B16: in moveToVALU()
4304 case AMDGPU::S_PACK_LH_B32_B16: in moveToVALU()
4305 case AMDGPU::S_PACK_HH_B32_B16: in moveToVALU()
4310 case AMDGPU::S_XNOR_B32: in moveToVALU()
4315 case AMDGPU::S_NAND_B32: in moveToVALU()
4316 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_AND_B32); in moveToVALU()
4320 case AMDGPU::S_NOR_B32: in moveToVALU()
4321 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_OR_B32); in moveToVALU()
4325 case AMDGPU::S_ANDN2_B32: in moveToVALU()
4326 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_AND_B32); in moveToVALU()
4330 case AMDGPU::S_ORN2_B32: in moveToVALU()
4331 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_OR_B32); in moveToVALU()
4336 if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) { in moveToVALU()
4352 if (Op.isReg() && Op.getReg() == AMDGPU::SCC) { in moveToVALU()
4358 if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) { in moveToVALU()
4361 unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16; in moveToVALU()
4365 } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) { in moveToVALU()
4373 if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) { in moveToVALU()
4389 unsigned NewDstReg = AMDGPU::NoRegister; in moveToVALU()
4419 Inst.setDesc(get(AMDGPU::IMPLICIT_DEF)); in moveToVALU()
4447 unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in moveScalarAddSub()
4450 assert(Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32); in moveScalarAddSub()
4452 unsigned NewOpc = Opc == AMDGPU::S_ADD_I32 ? in moveScalarAddSub()
4453 AMDGPU::V_ADD_U32_e64 : AMDGPU::V_SUB_U32_e64; in moveScalarAddSub()
4455 assert(Inst.getOperand(3).getReg() == AMDGPU::SCC); in moveScalarAddSub()
4479 unsigned TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in lowerScalarAbs()
4480 unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in lowerScalarAbs()
4483 AMDGPU::V_SUB_U32_e32 : AMDGPU::V_SUB_I32_e32; in lowerScalarAbs()
4489 BuildMI(MBB, MII, DL, get(AMDGPU::V_MAX_I32_e64), ResultReg) in lowerScalarAbs()
4509 unsigned NewDest = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in lowerScalarXnor()
4510 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src0, MRI, DL); in lowerScalarXnor()
4511 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src1, MRI, DL); in lowerScalarXnor()
4513 BuildMI(MBB, MII, DL, get(AMDGPU::V_XNOR_B32_e64), NewDest) in lowerScalarXnor()
4530 unsigned Temp = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in lowerScalarXnor()
4531 unsigned NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in lowerScalarXnor()
4537 Not = BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp) in lowerScalarXnor()
4539 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest) in lowerScalarXnor()
4543 Not = BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp) in lowerScalarXnor()
4545 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest) in lowerScalarXnor()
4549 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), Temp) in lowerScalarXnor()
4552 Not = BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest) in lowerScalarXnor()
4577 unsigned NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in splitScalarNotBinop()
4578 unsigned Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in splitScalarNotBinop()
4584 MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest) in splitScalarNotBinop()
4606 unsigned NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in splitScalarBinOpN2()
4607 unsigned Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in splitScalarBinOpN2()
4609 MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Interm) in splitScalarBinOpN2()
4638 &AMDGPU::SGPR_32RegClass; in splitScalar64BitUnaryOp()
4640 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0); in splitScalar64BitUnaryOp()
4643 AMDGPU::sub0, Src0SubRC); in splitScalar64BitUnaryOp()
4647 const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0); in splitScalar64BitUnaryOp()
4653 AMDGPU::sub1, Src0SubRC); in splitScalar64BitUnaryOp()
4661 .addImm(AMDGPU::sub0) in splitScalar64BitUnaryOp()
4663 .addImm(AMDGPU::sub1); in splitScalar64BitUnaryOp()
4680 bool IsAdd = (Inst.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO); in splitScalar64BitAddSub()
4685 unsigned FullDestReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); in splitScalar64BitAddSub()
4686 unsigned DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitAddSub()
4687 unsigned DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitAddSub()
4689 unsigned CarryReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass); in splitScalar64BitAddSub()
4690 unsigned DeadCarryReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass); in splitScalar64BitAddSub()
4700 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0); in splitScalar64BitAddSub()
4701 const TargetRegisterClass *Src1SubRC = RI.getSubRegClass(Src1RC, AMDGPU::sub0); in splitScalar64BitAddSub()
4704 AMDGPU::sub0, Src0SubRC); in splitScalar64BitAddSub()
4706 AMDGPU::sub0, Src1SubRC); in splitScalar64BitAddSub()
4710 AMDGPU::sub1, Src0SubRC); in splitScalar64BitAddSub()
4712 AMDGPU::sub1, Src1SubRC); in splitScalar64BitAddSub()
4714 unsigned LoOpc = IsAdd ? AMDGPU::V_ADD_I32_e64 : AMDGPU::V_SUB_I32_e64; in splitScalar64BitAddSub()
4721 unsigned HiOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64; in splitScalar64BitAddSub()
4731 .addImm(AMDGPU::sub0) in splitScalar64BitAddSub()
4733 .addImm(AMDGPU::sub1); in splitScalar64BitAddSub()
4762 &AMDGPU::SGPR_32RegClass; in splitScalar64BitBinaryOp()
4764 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0); in splitScalar64BitBinaryOp()
4767 &AMDGPU::SGPR_32RegClass; in splitScalar64BitBinaryOp()
4769 const TargetRegisterClass *Src1SubRC = RI.getSubRegClass(Src1RC, AMDGPU::sub0); in splitScalar64BitBinaryOp()
4772 AMDGPU::sub0, Src0SubRC); in splitScalar64BitBinaryOp()
4774 AMDGPU::sub0, Src1SubRC); in splitScalar64BitBinaryOp()
4776 AMDGPU::sub1, Src0SubRC); in splitScalar64BitBinaryOp()
4778 AMDGPU::sub1, Src1SubRC); in splitScalar64BitBinaryOp()
4782 const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0); in splitScalar64BitBinaryOp()
4797 .addImm(AMDGPU::sub0) in splitScalar64BitBinaryOp()
4799 .addImm(AMDGPU::sub1); in splitScalar64BitBinaryOp()
4825 unsigned Interm = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in splitScalar64BitXnor()
4838 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B64), Interm) in splitScalar64BitXnor()
4843 MachineInstr &Xor = *BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B64), NewDest) in splitScalar64BitXnor()
4863 const MCInstrDesc &InstDesc = get(AMDGPU::V_BCNT_U32_B32_e64); in splitScalar64BitBCNT()
4866 &AMDGPU::SGPR_32RegClass; in splitScalar64BitBCNT()
4868 unsigned MidReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitBCNT()
4869 unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitBCNT()
4871 const TargetRegisterClass *SrcSubRC = RI.getSubRegClass(SrcRC, AMDGPU::sub0); in splitScalar64BitBCNT()
4874 AMDGPU::sub0, SrcSubRC); in splitScalar64BitBCNT()
4876 AMDGPU::sub1, SrcSubRC); in splitScalar64BitBCNT()
4904 assert(Inst.getOpcode() == AMDGPU::S_BFE_I64 && BitWidth <= 32 && in splitScalar64BitBFE()
4908 unsigned MidRegLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitBFE()
4909 unsigned MidRegHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitBFE()
4910 unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); in splitScalar64BitBFE()
4912 BuildMI(MBB, MII, DL, get(AMDGPU::V_BFE_I32), MidRegLo) in splitScalar64BitBFE()
4913 .addReg(Inst.getOperand(1).getReg(), 0, AMDGPU::sub0) in splitScalar64BitBFE()
4917 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e32), MidRegHi) in splitScalar64BitBFE()
4923 .addImm(AMDGPU::sub0) in splitScalar64BitBFE()
4925 .addImm(AMDGPU::sub1); in splitScalar64BitBFE()
4933 unsigned TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in splitScalar64BitBFE()
4934 unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); in splitScalar64BitBFE()
4936 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e64), TmpReg) in splitScalar64BitBFE()
4938 .addReg(Src.getReg(), 0, AMDGPU::sub0); in splitScalar64BitBFE()
4941 .addReg(Src.getReg(), 0, AMDGPU::sub0) in splitScalar64BitBFE()
4942 .addImm(AMDGPU::sub0) in splitScalar64BitBFE()
4944 .addImm(AMDGPU::sub1); in splitScalar64BitBFE()
4972 unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in movePackToVALU()
4979 case AMDGPU::S_PACK_LL_B32_B16: { in movePackToVALU()
4980 unsigned ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in movePackToVALU()
4981 unsigned TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in movePackToVALU()
4985 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg) in movePackToVALU()
4988 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_B32_e64), TmpReg) in movePackToVALU()
4992 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHL_OR_B32), ResultReg) in movePackToVALU()
4998 case AMDGPU::S_PACK_LH_B32_B16: { in movePackToVALU()
4999 unsigned ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in movePackToVALU()
5000 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg) in movePackToVALU()
5002 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_BFI_B32), ResultReg) in movePackToVALU()
5008 case AMDGPU::S_PACK_HH_B32_B16: { in movePackToVALU()
5009 unsigned ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in movePackToVALU()
5010 unsigned TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in movePackToVALU()
5011 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg) in movePackToVALU()
5014 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg) in movePackToVALU()
5016 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_OR_B32), ResultReg) in movePackToVALU()
5039 if (MI.findRegisterDefOperandIdx(AMDGPU::SCC, false, false, &RI) != -1) in addSCCDefUsersToVALUWorklist()
5042 if (MI.findRegisterUseOperandIdx(AMDGPU::SCC, false, &RI) != -1) in addSCCDefUsersToVALUWorklist()
5055 case AMDGPU::COPY: in getDestEquivalentVGPRClass()
5056 case AMDGPU::PHI: in getDestEquivalentVGPRClass()
5057 case AMDGPU::REG_SEQUENCE: in getDestEquivalentVGPRClass()
5058 case AMDGPU::INSERT_SUBREG: in getDestEquivalentVGPRClass()
5059 case AMDGPU::WQM: in getDestEquivalentVGPRClass()
5060 case AMDGPU::WWM: in getDestEquivalentVGPRClass()
5088 if (SGPRReg != AMDGPU::NoRegister) in findUsedSGPR()
5091 unsigned UsedSGPRs[3] = { AMDGPU::NoRegister }; in findUsedSGPR()
5130 if (UsedSGPRs[0] != AMDGPU::NoRegister) { in findUsedSGPR()
5135 if (SGPRReg == AMDGPU::NoRegister && UsedSGPRs[1] != AMDGPU::NoRegister) { in findUsedSGPR()
5145 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OperandName); in getNamedOperand()
5153 uint64_t RsrcDataFormat = AMDGPU::RSRC_DATA_FORMAT; in getDefaultRsrcDataFormat()
5170 AMDGPU::RSRC_TID_ENABLE | in getScratchRsrcWords23()
5176 Rsrc23 |= EltSizeValue << AMDGPU::RSRC_ELEMENT_SIZE_SHIFT; in getScratchRsrcWords23()
5180 Rsrc23 |= UINT64_C(3) << AMDGPU::RSRC_INDEX_STRIDE_SHIFT; in getScratchRsrcWords23()
5185 Rsrc23 &= ~AMDGPU::RSRC_DATA_FORMAT; in getScratchRsrcWords23()
5204 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::vaddr); in isStackAccess()
5206 return AMDGPU::NoRegister; in isStackAccess()
5212 return getNamedOperand(MI, AMDGPU::OpName::vdata)->getReg(); in isStackAccess()
5217 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::addr); in isSGPRStackAccess()
5220 return getNamedOperand(MI, AMDGPU::OpName::data)->getReg(); in isSGPRStackAccess()
5226 return AMDGPU::NoRegister; in isLoadFromStackSlot()
5234 return AMDGPU::NoRegister; in isLoadFromStackSlot()
5240 return AMDGPU::NoRegister; in isStoreToStackSlot()
5248 return AMDGPU::NoRegister; in isStoreToStackSlot()
5276 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); in getInstSizeInBytes()
5283 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); in getInstSizeInBytes()
5290 int Src2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2); in getInstSizeInBytes()
5333 return Branch.getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO; in isNonUniformBranchInstr()
5345 if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) { in convertNonUniformIfRegion()
5346 unsigned DstReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in convertNonUniformIfRegion()
5348 BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_IF), DstReg) in convertNonUniformIfRegion()
5352 BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_END_CF)) in convertNonUniformIfRegion()
5371 if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) { in convertNonUniformLoopRegion()
5373 unsigned DstReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in convertNonUniformLoopRegion()
5374 unsigned BackEdgeReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in convertNonUniformLoopRegion()
5384 unsigned ZeroReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in convertNonUniformLoopRegion()
5393 get(AMDGPU::SI_IF_BREAK), BackEdgeReg) in convertNonUniformLoopRegion()
5397 BuildMI(*(MF), Branch->getDebugLoc(), get(AMDGPU::SI_LOOP)) in convertNonUniformLoopRegion()
5411 {AMDGPU::TI_CONSTDATA_START, "amdgpu-constdata-start"}, in getSerializableTargetIndices()
5412 {AMDGPU::TI_SCRATCH_RSRC_DWORD0, "amdgpu-scratch-rsrc-dword0"}, in getSerializableTargetIndices()
5413 {AMDGPU::TI_SCRATCH_RSRC_DWORD1, "amdgpu-scratch-rsrc-dword1"}, in getSerializableTargetIndices()
5414 {AMDGPU::TI_SCRATCH_RSRC_DWORD2, "amdgpu-scratch-rsrc-dword2"}, in getSerializableTargetIndices()
5415 {AMDGPU::TI_SCRATCH_RSRC_DWORD3, "amdgpu-scratch-rsrc-dword3"}}; in getSerializableTargetIndices()
5453 return !MI.isTerminator() && MI.getOpcode() != AMDGPU::COPY && in isBasicBlockPrologue()
5454 MI.modifiesRegister(AMDGPU::EXEC, &RI); in isBasicBlockPrologue()
5463 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e64), DestReg); in getAddNoCarry()
5466 unsigned UnusedCarry = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in getAddNoCarry()
5467 MRI.setRegAllocationHint(UnusedCarry, 0, AMDGPU::VCC); in getAddNoCarry()
5469 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_I32_e64), DestReg) in getAddNoCarry()
5475 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR: in isKillTerminator()
5476 case AMDGPU::SI_KILL_I1_TERMINATOR: in isKillTerminator()
5485 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO: in getKillTerminatorFromPseudo()
5486 return get(AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR); in getKillTerminatorFromPseudo()
5487 case AMDGPU::SI_KILL_I1_PSEUDO: in getKillTerminatorFromPseudo()
5488 return get(AMDGPU::SI_KILL_I1_TERMINATOR); in getKillTerminatorFromPseudo()
5499 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::sbase); in isBufferSMRD()
5504 return RCID == AMDGPU::SReg_128RegClassID; in isBufferSMRD()
5547 int MCOp = AMDGPU::getMCOpcode(Opcode, Gen); in pseudoToMCOpcode()
5587 case AMDGPU::REG_SEQUENCE: in followSubRegDef()
5591 case AMDGPU::INSERT_SUBREG: in followSubRegDef()
5617 case AMDGPU::COPY: in getVRegSubRegDef()
5618 case AMDGPU::V_MOV_B32_e32: { in getVRegSubRegDef()