Lines Matching refs:AMDGPU
77 return Def->getOpcode() == AMDGPU::G_AMDGPU_WAVE_ADDRESS in getWaveAddress()
96 return MRI.getVRegDef(Reg)->getOpcode() != AMDGPU::G_TRUNC && in isVCC()
101 return RB->getID() == AMDGPU::VCCRegBankID; in isVCC()
108 MI.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true)); in constrainCopyLikeIntrin()
139 if (SrcReg == AMDGPU::SCC) { in selectCOPY()
159 STI.isWave64() ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32; in selectCOPY()
171 IsSGPR ? AMDGPU::S_AND_B32 : AMDGPU::V_AND_B32_e32; in selectCOPY()
178 BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_CMP_NE_U32_e64), DstReg) in selectCOPY()
260 BuildMI(*BB, MI, MI->getDebugLoc(), TII.get(AMDGPU::COPY), DstReg) in getSubOperand64()
276 case AMDGPU::sub0: in getSubOperand64()
278 case AMDGPU::sub1: in getSubOperand64()
285 case AMDGPU::G_AND: in getLogicalBitOpcode()
286 return Is64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32; in getLogicalBitOpcode()
287 case AMDGPU::G_OR: in getLogicalBitOpcode()
288 return Is64 ? AMDGPU::S_OR_B64 : AMDGPU::S_OR_B32; in getLogicalBitOpcode()
289 case AMDGPU::G_XOR: in getLogicalBitOpcode()
290 return Is64 ? AMDGPU::S_XOR_B64 : AMDGPU::S_XOR_B32; in getLogicalBitOpcode()
301 if (DstRB->getID() != AMDGPU::SGPRRegBankID && in selectG_AND_OR_XOR()
302 DstRB->getID() != AMDGPU::VCCRegBankID) in selectG_AND_OR_XOR()
305 bool Is64 = Size > 32 || (DstRB->getID() == AMDGPU::VCCRegBankID && in selectG_AND_OR_XOR()
310 I.addOperand(MachineOperand::CreateReg(AMDGPU::SCC, true, // isDef in selectG_AND_OR_XOR()
328 const bool IsSALU = DstRB->getID() == AMDGPU::SGPRRegBankID; in selectG_ADD_SUB()
333 const unsigned Opc = Sub ? AMDGPU::S_SUB_U32 : AMDGPU::S_ADD_U32; in selectG_ADD_SUB()
344 const unsigned Opc = Sub ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_ADD_U32_e64; in selectG_ADD_SUB()
347 I.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true)); in selectG_ADD_SUB()
351 const unsigned Opc = Sub ? AMDGPU::V_SUB_CO_U32_e64 : AMDGPU::V_ADD_CO_U32_e64; in selectG_ADD_SUB()
367 = IsSALU ? AMDGPU::SReg_64_XEXECRegClass : AMDGPU::VReg_64RegClass; in selectG_ADD_SUB()
369 = IsSALU ? AMDGPU::SReg_32RegClass : AMDGPU::VGPR_32RegClass; in selectG_ADD_SUB()
371 MachineOperand Lo1(getSubOperand64(I.getOperand(1), HalfRC, AMDGPU::sub0)); in selectG_ADD_SUB()
372 MachineOperand Lo2(getSubOperand64(I.getOperand(2), HalfRC, AMDGPU::sub0)); in selectG_ADD_SUB()
373 MachineOperand Hi1(getSubOperand64(I.getOperand(1), HalfRC, AMDGPU::sub1)); in selectG_ADD_SUB()
374 MachineOperand Hi2(getSubOperand64(I.getOperand(2), HalfRC, AMDGPU::sub1)); in selectG_ADD_SUB()
380 BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_ADD_U32), DstLo) in selectG_ADD_SUB()
383 BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_ADDC_U32), DstHi) in selectG_ADD_SUB()
390 BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_ADD_CO_U32_e64), DstLo) in selectG_ADD_SUB()
395 MachineInstr *Addc = BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_ADDC_U32_e64), DstHi) in selectG_ADD_SUB()
406 BuildMI(*BB, &I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg) in selectG_ADD_SUB()
408 .addImm(AMDGPU::sub0) in selectG_ADD_SUB()
410 .addImm(AMDGPU::sub1); in selectG_ADD_SUB()
427 const bool IsAdd = I.getOpcode() == AMDGPU::G_UADDO || in selectG_UADDO_USUBO_UADDE_USUBE()
428 I.getOpcode() == AMDGPU::G_UADDE; in selectG_UADDO_USUBO_UADDE_USUBE()
429 const bool HasCarryIn = I.getOpcode() == AMDGPU::G_UADDE || in selectG_UADDO_USUBO_UADDE_USUBE()
430 I.getOpcode() == AMDGPU::G_USUBE; in selectG_UADDO_USUBO_UADDE_USUBE()
434 IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64; in selectG_UADDO_USUBO_UADDE_USUBE()
435 unsigned CarryOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64; in selectG_UADDO_USUBO_UADDE_USUBE()
437 I.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true)); in selectG_UADDO_USUBO_UADDE_USUBE()
446 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), AMDGPU::SCC) in selectG_UADDO_USUBO_UADDE_USUBE()
450 unsigned NoCarryOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32; in selectG_UADDO_USUBO_UADDE_USUBE()
451 unsigned CarryOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32; in selectG_UADDO_USUBO_UADDE_USUBE()
460 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), Dst1Reg) in selectG_UADDO_USUBO_UADDE_USUBE()
461 .addReg(AMDGPU::SCC); in selectG_UADDO_USUBO_UADDE_USUBE()
463 MRI->setRegClass(Dst1Reg, &AMDGPU::SReg_32RegClass); in selectG_UADDO_USUBO_UADDE_USUBE()
466 if (!RBI.constrainGenericRegister(Dst0Reg, AMDGPU::SReg_32RegClass, *MRI) || in selectG_UADDO_USUBO_UADDE_USUBE()
467 !RBI.constrainGenericRegister(Src0Reg, AMDGPU::SReg_32RegClass, *MRI) || in selectG_UADDO_USUBO_UADDE_USUBE()
468 !RBI.constrainGenericRegister(Src1Reg, AMDGPU::SReg_32RegClass, *MRI)) in selectG_UADDO_USUBO_UADDE_USUBE()
473 AMDGPU::SReg_32RegClass, *MRI)) in selectG_UADDO_USUBO_UADDE_USUBE()
484 const bool IsUnsigned = I.getOpcode() == AMDGPU::G_AMDGPU_MAD_U64_U32; in selectG_AMDGPU_MAD_64_32()
488 Opc = IsUnsigned ? AMDGPU::V_MAD_U64_U32_gfx11_e64 in selectG_AMDGPU_MAD_64_32()
489 : AMDGPU::V_MAD_I64_I32_gfx11_e64; in selectG_AMDGPU_MAD_64_32()
491 Opc = IsUnsigned ? AMDGPU::V_MAD_U64_U32_e64 : AMDGPU::V_MAD_I64_I32_e64; in selectG_AMDGPU_MAD_64_32()
629 assert(MI.getOpcode() == AMDGPU::G_BUILD_VECTOR_TRUNC || in selectG_BUILD_VECTOR()
630 MI.getOpcode() == AMDGPU::G_BUILD_VECTOR); in selectG_BUILD_VECTOR()
638 if (MI.getOpcode() == AMDGPU::G_BUILD_VECTOR && SrcSize >= 32) { in selectG_BUILD_VECTOR()
646 (MI.getOpcode() == AMDGPU::G_BUILD_VECTOR_TRUNC && in selectG_BUILD_VECTOR()
651 if (DstBank->getID() == AMDGPU::AGPRRegBankID) in selectG_BUILD_VECTOR()
654 assert(DstBank->getID() == AMDGPU::SGPRRegBankID || in selectG_BUILD_VECTOR()
655 DstBank->getID() == AMDGPU::VGPRRegBankID); in selectG_BUILD_VECTOR()
656 const bool IsVector = DstBank->getID() == AMDGPU::VGPRRegBankID; in selectG_BUILD_VECTOR()
677 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::V_MOV_B32_e32), Dst).addImm(Imm); in selectG_BUILD_VECTOR()
679 return RBI.constrainGenericRegister(Dst, AMDGPU::VGPR_32RegClass, *MRI); in selectG_BUILD_VECTOR()
683 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_MOV_B32), Dst).addImm(Imm); in selectG_BUILD_VECTOR()
685 return RBI.constrainGenericRegister(Dst, AMDGPU::SReg_32RegClass, *MRI); in selectG_BUILD_VECTOR()
696 if (Src1Def->getOpcode() == AMDGPU::G_IMPLICIT_DEF) { in selectG_BUILD_VECTOR()
697 MI.setDesc(TII.get(AMDGPU::COPY)); in selectG_BUILD_VECTOR()
700 IsVector ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass; in selectG_BUILD_VECTOR()
707 Register TmpReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectG_BUILD_VECTOR()
708 auto MIB = BuildMI(*BB, MI, DL, TII.get(AMDGPU::V_AND_B32_e32), TmpReg) in selectG_BUILD_VECTOR()
714 MIB = BuildMI(*BB, MI, DL, TII.get(AMDGPU::V_LSHL_OR_B32_e64), Dst) in selectG_BUILD_VECTOR()
746 unsigned Opc = AMDGPU::S_PACK_LL_B32_B16; in selectG_BUILD_VECTOR()
748 Opc = AMDGPU::S_PACK_HH_B32_B16; in selectG_BUILD_VECTOR()
752 Opc = AMDGPU::S_PACK_LH_B32_B16; in selectG_BUILD_VECTOR()
759 auto MIB = BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_LSHR_B32), Dst) in selectG_BUILD_VECTOR()
768 Opc = AMDGPU::S_PACK_HL_B32_B16; in selectG_BUILD_VECTOR()
818 if (SubReg == AMDGPU::NoSubRegister) in selectG_INSERT()
861 assert(RBI.getRegBank(DstReg, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID && in selectG_SBFX_UBFX()
870 unsigned Opc = IsSigned ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64; in selectG_SBFX_UBFX()
886 if (!RBI.constrainGenericRegister(M0Val, AMDGPU::SReg_32RegClass, *MRI) || in selectInterpP1F16()
887 !RBI.constrainGenericRegister(Dst, AMDGPU::VGPR_32RegClass, *MRI) || in selectInterpP1F16()
888 !RBI.constrainGenericRegister(Src0, AMDGPU::VGPR_32RegClass, *MRI)) in selectInterpP1F16()
898 Register InterpMov = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectInterpP1F16()
902 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectInterpP1F16()
904 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::V_INTERP_MOV_F32), InterpMov) in selectInterpP1F16()
909 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::V_INTERP_P1LV_F16), Dst) in selectInterpP1F16()
931 if (STI.getConstantBusLimit(AMDGPU::V_WRITELANE_B32) > 1) in selectWritelane()
941 auto MIB = BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::V_WRITELANE_B32), VDst); in selectWritelane()
957 if (ConstVal && AMDGPU::isInlinableLiteral32(ConstVal->Value.getSExtValue(), in selectWritelane()
967 RBI.constrainGenericRegister(LaneSelect, AMDGPU::SReg_32_XM0RegClass, *MRI); in selectWritelane()
969 BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectWritelane()
971 MIB.addReg(AMDGPU::M0); in selectWritelane()
990 Opc = AMDGPU::V_DIV_SCALE_F32_e64; in selectDivScale()
992 Opc = AMDGPU::V_DIV_SCALE_F64_e64; in selectDivScale()
1030 BuildMI(*BB, &I, I.getDebugLoc(), TII.get(AMDGPU::SI_IF_BREAK)) in selectG_INTRINSIC()
1049 return constrainCopyLikeIntrin(I, AMDGPU::WQM); in selectG_INTRINSIC()
1051 return constrainCopyLikeIntrin(I, AMDGPU::SOFT_WQM); in selectG_INTRINSIC()
1054 return constrainCopyLikeIntrin(I, AMDGPU::STRICT_WWM); in selectG_INTRINSIC()
1056 return constrainCopyLikeIntrin(I, AMDGPU::STRICT_WQM); in selectG_INTRINSIC()
1117 return Select(AMDGPU::V_CMP_NE_U16_e64, AMDGPU::V_CMP_NE_U16_t16_e64, in getV_CMPOpcode()
1118 AMDGPU::V_CMP_NE_U32_e64, AMDGPU::V_CMP_NE_U64_e64); in getV_CMPOpcode()
1120 return Select(AMDGPU::V_CMP_EQ_U16_e64, AMDGPU::V_CMP_EQ_U16_t16_e64, in getV_CMPOpcode()
1121 AMDGPU::V_CMP_EQ_U32_e64, AMDGPU::V_CMP_EQ_U64_e64); in getV_CMPOpcode()
1123 return Select(AMDGPU::V_CMP_GT_I16_e64, AMDGPU::V_CMP_GT_I16_t16_e64, in getV_CMPOpcode()
1124 AMDGPU::V_CMP_GT_I32_e64, AMDGPU::V_CMP_GT_I64_e64); in getV_CMPOpcode()
1126 return Select(AMDGPU::V_CMP_GE_I16_e64, AMDGPU::V_CMP_GE_I16_t16_e64, in getV_CMPOpcode()
1127 AMDGPU::V_CMP_GE_I32_e64, AMDGPU::V_CMP_GE_I64_e64); in getV_CMPOpcode()
1129 return Select(AMDGPU::V_CMP_LT_I16_e64, AMDGPU::V_CMP_LT_I16_t16_e64, in getV_CMPOpcode()
1130 AMDGPU::V_CMP_LT_I32_e64, AMDGPU::V_CMP_LT_I64_e64); in getV_CMPOpcode()
1132 return Select(AMDGPU::V_CMP_LE_I16_e64, AMDGPU::V_CMP_LE_I16_t16_e64, in getV_CMPOpcode()
1133 AMDGPU::V_CMP_LE_I32_e64, AMDGPU::V_CMP_LE_I64_e64); in getV_CMPOpcode()
1135 return Select(AMDGPU::V_CMP_GT_U16_e64, AMDGPU::V_CMP_GT_U16_t16_e64, in getV_CMPOpcode()
1136 AMDGPU::V_CMP_GT_U32_e64, AMDGPU::V_CMP_GT_U64_e64); in getV_CMPOpcode()
1138 return Select(AMDGPU::V_CMP_GE_U16_e64, AMDGPU::V_CMP_GE_U16_t16_e64, in getV_CMPOpcode()
1139 AMDGPU::V_CMP_GE_U32_e64, AMDGPU::V_CMP_GE_U64_e64); in getV_CMPOpcode()
1141 return Select(AMDGPU::V_CMP_LT_U16_e64, AMDGPU::V_CMP_LT_U16_t16_e64, in getV_CMPOpcode()
1142 AMDGPU::V_CMP_LT_U32_e64, AMDGPU::V_CMP_LT_U64_e64); in getV_CMPOpcode()
1144 return Select(AMDGPU::V_CMP_LE_U16_e64, AMDGPU::V_CMP_LE_U16_t16_e64, in getV_CMPOpcode()
1145 AMDGPU::V_CMP_LE_U32_e64, AMDGPU::V_CMP_LE_U64_e64); in getV_CMPOpcode()
1148 return Select(AMDGPU::V_CMP_EQ_F16_e64, AMDGPU::V_CMP_EQ_F16_t16_e64, in getV_CMPOpcode()
1149 AMDGPU::V_CMP_EQ_F32_e64, AMDGPU::V_CMP_EQ_F64_e64); in getV_CMPOpcode()
1151 return Select(AMDGPU::V_CMP_GT_F16_e64, AMDGPU::V_CMP_GT_F16_t16_e64, in getV_CMPOpcode()
1152 AMDGPU::V_CMP_GT_F32_e64, AMDGPU::V_CMP_GT_F64_e64); in getV_CMPOpcode()
1154 return Select(AMDGPU::V_CMP_GE_F16_e64, AMDGPU::V_CMP_GE_F16_t16_e64, in getV_CMPOpcode()
1155 AMDGPU::V_CMP_GE_F32_e64, AMDGPU::V_CMP_GE_F64_e64); in getV_CMPOpcode()
1157 return Select(AMDGPU::V_CMP_LT_F16_e64, AMDGPU::V_CMP_LT_F16_t16_e64, in getV_CMPOpcode()
1158 AMDGPU::V_CMP_LT_F32_e64, AMDGPU::V_CMP_LT_F64_e64); in getV_CMPOpcode()
1160 return Select(AMDGPU::V_CMP_LE_F16_e64, AMDGPU::V_CMP_LE_F16_t16_e64, in getV_CMPOpcode()
1161 AMDGPU::V_CMP_LE_F32_e64, AMDGPU::V_CMP_LE_F64_e64); in getV_CMPOpcode()
1163 return Select(AMDGPU::V_CMP_NEQ_F16_e64, AMDGPU::V_CMP_NEQ_F16_t16_e64, in getV_CMPOpcode()
1164 AMDGPU::V_CMP_NEQ_F32_e64, AMDGPU::V_CMP_NEQ_F64_e64); in getV_CMPOpcode()
1166 return Select(AMDGPU::V_CMP_O_F16_e64, AMDGPU::V_CMP_O_F16_t16_e64, in getV_CMPOpcode()
1167 AMDGPU::V_CMP_O_F32_e64, AMDGPU::V_CMP_O_F64_e64); in getV_CMPOpcode()
1169 return Select(AMDGPU::V_CMP_U_F16_e64, AMDGPU::V_CMP_U_F16_t16_e64, in getV_CMPOpcode()
1170 AMDGPU::V_CMP_U_F32_e64, AMDGPU::V_CMP_U_F64_e64); in getV_CMPOpcode()
1172 return Select(AMDGPU::V_CMP_NLG_F16_e64, AMDGPU::V_CMP_NLG_F16_t16_e64, in getV_CMPOpcode()
1173 AMDGPU::V_CMP_NLG_F32_e64, AMDGPU::V_CMP_NLG_F64_e64); in getV_CMPOpcode()
1175 return Select(AMDGPU::V_CMP_NLE_F16_e64, AMDGPU::V_CMP_NLE_F16_t16_e64, in getV_CMPOpcode()
1176 AMDGPU::V_CMP_NLE_F32_e64, AMDGPU::V_CMP_NLE_F64_e64); in getV_CMPOpcode()
1178 return Select(AMDGPU::V_CMP_NLT_F16_e64, AMDGPU::V_CMP_NLT_F16_t16_e64, in getV_CMPOpcode()
1179 AMDGPU::V_CMP_NLT_F32_e64, AMDGPU::V_CMP_NLT_F64_e64); in getV_CMPOpcode()
1181 return Select(AMDGPU::V_CMP_NGE_F16_e64, AMDGPU::V_CMP_NGE_F16_t16_e64, in getV_CMPOpcode()
1182 AMDGPU::V_CMP_NGE_F32_e64, AMDGPU::V_CMP_NGE_F64_e64); in getV_CMPOpcode()
1184 return Select(AMDGPU::V_CMP_NGT_F16_e64, AMDGPU::V_CMP_NGT_F16_t16_e64, in getV_CMPOpcode()
1185 AMDGPU::V_CMP_NGT_F32_e64, AMDGPU::V_CMP_NGT_F64_e64); in getV_CMPOpcode()
1187 return Select(AMDGPU::V_CMP_NEQ_F16_e64, AMDGPU::V_CMP_NEQ_F16_t16_e64, in getV_CMPOpcode()
1188 AMDGPU::V_CMP_NEQ_F32_e64, AMDGPU::V_CMP_NEQ_F64_e64); in getV_CMPOpcode()
1190 return Select(AMDGPU::V_CMP_TRU_F16_e64, AMDGPU::V_CMP_TRU_F16_t16_e64, in getV_CMPOpcode()
1191 AMDGPU::V_CMP_TRU_F32_e64, AMDGPU::V_CMP_TRU_F64_e64); in getV_CMPOpcode()
1193 return Select(AMDGPU::V_CMP_F_F16_e64, AMDGPU::V_CMP_F_F16_t16_e64, in getV_CMPOpcode()
1194 AMDGPU::V_CMP_F_F32_e64, AMDGPU::V_CMP_F_F64_e64); in getV_CMPOpcode()
1206 return AMDGPU::S_CMP_LG_U64; in getS_CMPOpcode()
1208 return AMDGPU::S_CMP_EQ_U64; in getS_CMPOpcode()
1217 return AMDGPU::S_CMP_LG_U32; in getS_CMPOpcode()
1219 return AMDGPU::S_CMP_EQ_U32; in getS_CMPOpcode()
1221 return AMDGPU::S_CMP_GT_I32; in getS_CMPOpcode()
1223 return AMDGPU::S_CMP_GE_I32; in getS_CMPOpcode()
1225 return AMDGPU::S_CMP_LT_I32; in getS_CMPOpcode()
1227 return AMDGPU::S_CMP_LE_I32; in getS_CMPOpcode()
1229 return AMDGPU::S_CMP_GT_U32; in getS_CMPOpcode()
1231 return AMDGPU::S_CMP_GE_U32; in getS_CMPOpcode()
1233 return AMDGPU::S_CMP_LT_U32; in getS_CMPOpcode()
1235 return AMDGPU::S_CMP_LE_U32; in getS_CMPOpcode()
1237 return AMDGPU::S_CMP_EQ_F32; in getS_CMPOpcode()
1239 return AMDGPU::S_CMP_GT_F32; in getS_CMPOpcode()
1241 return AMDGPU::S_CMP_GE_F32; in getS_CMPOpcode()
1243 return AMDGPU::S_CMP_LT_F32; in getS_CMPOpcode()
1245 return AMDGPU::S_CMP_LE_F32; in getS_CMPOpcode()
1247 return AMDGPU::S_CMP_LG_F32; in getS_CMPOpcode()
1249 return AMDGPU::S_CMP_O_F32; in getS_CMPOpcode()
1251 return AMDGPU::S_CMP_U_F32; in getS_CMPOpcode()
1253 return AMDGPU::S_CMP_NLG_F32; in getS_CMPOpcode()
1255 return AMDGPU::S_CMP_NLE_F32; in getS_CMPOpcode()
1257 return AMDGPU::S_CMP_NLT_F32; in getS_CMPOpcode()
1259 return AMDGPU::S_CMP_NGE_F32; in getS_CMPOpcode()
1261 return AMDGPU::S_CMP_NGT_F32; in getS_CMPOpcode()
1263 return AMDGPU::S_CMP_NEQ_F32; in getS_CMPOpcode()
1275 return AMDGPU::S_CMP_EQ_F16; in getS_CMPOpcode()
1277 return AMDGPU::S_CMP_GT_F16; in getS_CMPOpcode()
1279 return AMDGPU::S_CMP_GE_F16; in getS_CMPOpcode()
1281 return AMDGPU::S_CMP_LT_F16; in getS_CMPOpcode()
1283 return AMDGPU::S_CMP_LE_F16; in getS_CMPOpcode()
1285 return AMDGPU::S_CMP_LG_F16; in getS_CMPOpcode()
1287 return AMDGPU::S_CMP_O_F16; in getS_CMPOpcode()
1289 return AMDGPU::S_CMP_U_F16; in getS_CMPOpcode()
1291 return AMDGPU::S_CMP_NLG_F16; in getS_CMPOpcode()
1293 return AMDGPU::S_CMP_NLE_F16; in getS_CMPOpcode()
1295 return AMDGPU::S_CMP_NLT_F16; in getS_CMPOpcode()
1297 return AMDGPU::S_CMP_NGE_F16; in getS_CMPOpcode()
1299 return AMDGPU::S_CMP_NGT_F16; in getS_CMPOpcode()
1301 return AMDGPU::S_CMP_NEQ_F16; in getS_CMPOpcode()
1328 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), CCReg) in selectG_ICMP_or_FCMP()
1329 .addReg(AMDGPU::SCC); in selectG_ICMP_or_FCMP()
1332 RBI.constrainGenericRegister(CCReg, AMDGPU::SReg_32RegClass, *MRI); in selectG_ICMP_or_FCMP()
1337 if (I.getOpcode() == AMDGPU::G_FCMP) in selectG_ICMP_or_FCMP()
1375 BuildMI(*BB, &I, DL, TII.get(AMDGPU::IMPLICIT_DEF), Dst); in selectIntrinsicCmp()
1394 if (AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::src0_modifiers)) in selectIntrinsicCmp()
1397 if (AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::src1_modifiers)) in selectIntrinsicCmp()
1400 if (AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::clamp)) in selectIntrinsicCmp()
1402 if (AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::op_sel)) in selectIntrinsicCmp()
1431 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), DstReg) in selectBallot()
1437 Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectBallot()
1438 BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_MOV_B32), HiReg).addImm(0); in selectBallot()
1439 BuildMI(*BB, &I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg) in selectBallot()
1441 .addImm(AMDGPU::sub0) in selectBallot()
1443 .addImm(AMDGPU::sub1); in selectBallot()
1449 unsigned Opcode = Is64 ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32; in selectBallot()
1452 BuildCopy(IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC); in selectBallot()
1468 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), DstReg).addReg(MaskReg); in selectInverseBallot()
1480 const bool IsVALU = DstBank->getID() == AMDGPU::VGPRRegBankID; in selectRelocConstant()
1490 TII.get(IsVALU ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32), DstReg) in selectRelocConstant()
1502 unsigned Mov = DstRB->getID() == AMDGPU::SGPRRegBankID ? in selectGroupStaticSize()
1503 AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32; in selectGroupStaticSize()
1535 if (!RC->hasSubClassEq(&AMDGPU::SGPR_64RegClass) || in selectReturnAddress()
1542 BuildMI(*MBB, &I, DL, TII.get(AMDGPU::S_MOV_B64), DstReg) in selectReturnAddress()
1555 AMDGPU::SReg_64RegClass, DL); in selectReturnAddress()
1556 BuildMI(*MBB, &I, DL, TII.get(AMDGPU::COPY), DstReg) in selectReturnAddress()
1566 BuildMI(*BB, &MI, MI.getDebugLoc(), TII.get(AMDGPU::SI_END_CF)) in selectEndCfIntrinsic()
1622 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectDSOrderedIntrinsic()
1628 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::DS_ORDERED_COUNT), DstReg) in selectDSOrderedIntrinsic()
1633 if (!RBI.constrainGenericRegister(M0Val, AMDGPU::SReg_32RegClass, *MRI)) in selectDSOrderedIntrinsic()
1644 return AMDGPU::DS_GWS_INIT; in gwsIntrinToOpcode()
1646 return AMDGPU::DS_GWS_BARRIER; in gwsIntrinToOpcode()
1648 return AMDGPU::DS_GWS_SEMA_V; in gwsIntrinToOpcode()
1650 return AMDGPU::DS_GWS_SEMA_BR; in gwsIntrinToOpcode()
1652 return AMDGPU::DS_GWS_SEMA_P; in gwsIntrinToOpcode()
1654 return AMDGPU::DS_GWS_SEMA_RELEASE_ALL; in gwsIntrinToOpcode()
1672 if (OffsetRB->getID() != AMDGPU::SGPRRegBankID) in selectDSGWSIntrinsic()
1686 if (OffsetDef->getOpcode() == AMDGPU::V_READFIRSTLANE_B32) { in selectDSGWSIntrinsic()
1692 if (OffsetDef->getOpcode() == AMDGPU::G_CONSTANT) { in selectDSGWSIntrinsic()
1699 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::S_MOV_B32), AMDGPU::M0) in selectDSGWSIntrinsic()
1703 AMDGPU::getBaseWithConstantOffset(*MRI, BaseOffset, KB); in selectDSGWSIntrinsic()
1708 if (!RBI.constrainGenericRegister(BaseOffset, AMDGPU::VGPR_32RegClass, *MRI)) in selectDSGWSIntrinsic()
1715 AMDGPU::SReg_32RegClass, *MRI)) in selectDSGWSIntrinsic()
1719 Register M0Base = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectDSGWSIntrinsic()
1720 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::S_LSHL_B32), M0Base) in selectDSGWSIntrinsic()
1725 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectDSGWSIntrinsic()
1738 if (!RBI.constrainGenericRegister(VSrc, AMDGPU::VGPR_32RegClass, *MRI)) in selectDSGWSIntrinsic()
1745 TII.enforceOperandRCAlignment(*MIB, AMDGPU::OpName::data0); in selectDSGWSIntrinsic()
1768 const unsigned Opc = IsAppend ? AMDGPU::DS_APPEND : AMDGPU::DS_CONSUME; in selectDSAppendConsume()
1770 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectDSAppendConsume()
1772 if (!RBI.constrainGenericRegister(PtrBase, AMDGPU::SReg_32RegClass, *MRI)) in selectDSAppendConsume()
1789 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::WAVE_BARRIER)); in selectSBarrier()
1799 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::S_BARRIER_SIGNAL_IMM)) in selectSBarrier()
1800 .addImm(AMDGPU::Barrier::WORKGROUP); in selectSBarrier()
1801 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::S_BARRIER_WAIT)) in selectSBarrier()
1802 .addImm(AMDGPU::Barrier::WORKGROUP); in selectSBarrier()
1824 MachineInstr &MI, const AMDGPU::ImageDimIntrinsicInfo *Intr) const { in selectImageIntrinsic()
1828 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode = in selectImageIntrinsic()
1829 AMDGPU::getMIMGBaseOpcodeInfo(Intr->BaseOpcode); in selectImageIntrinsic()
1831 const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfo(Intr->Dim); in selectImageIntrinsic()
1833 const bool IsGFX10Plus = AMDGPU::isGFX10Plus(STI); in selectImageIntrinsic()
1834 const bool IsGFX11Plus = AMDGPU::isGFX11Plus(STI); in selectImageIntrinsic()
1835 const bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI); in selectImageIntrinsic()
1842 bool IsD16 = MI.getOpcode() == AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16 || in selectImageIntrinsic()
1843 MI.getOpcode() == AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16; in selectImageIntrinsic()
1880 assert(MI.getOperand(3).getReg() == AMDGPU::NoRegister); in selectImageIntrinsic()
1908 const AMDGPU::MIMGG16MappingInfo *G16MappingInfo = in selectImageIntrinsic()
1909 AMDGPU::getMIMGG16MappingInfo(Intr->BaseOpcode); in selectImageIntrinsic()
1919 CPol |= AMDGPU::CPol::GLC; // TODO no-return optimization in selectImageIntrinsic()
1920 if (CPol & ~((IsGFX12Plus ? AMDGPU::CPol::ALL : AMDGPU::CPol::ALL_pregfx12) | in selectImageIntrinsic()
1921 AMDGPU::CPol::VOLATILE)) in selectImageIntrinsic()
1947 if (UseNSA && !STI.hasFeature(AMDGPU::FeatureNSAEncoding)) { in selectImageIntrinsic()
1957 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx12, in selectImageIntrinsic()
1960 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, in selectImageIntrinsic()
1961 UseNSA ? AMDGPU::MIMGEncGfx11NSA in selectImageIntrinsic()
1962 : AMDGPU::MIMGEncGfx11Default, in selectImageIntrinsic()
1965 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, in selectImageIntrinsic()
1966 UseNSA ? AMDGPU::MIMGEncGfx10NSA in selectImageIntrinsic()
1967 : AMDGPU::MIMGEncGfx10Default, in selectImageIntrinsic()
1971 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx90a, in selectImageIntrinsic()
1982 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx8, in selectImageIntrinsic()
1985 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx6, in selectImageIntrinsic()
1999 Is64 ? &AMDGPU::VReg_128RegClass : &AMDGPU::VReg_64RegClass); in selectImageIntrinsic()
2000 unsigned SubReg = Is64 ? AMDGPU::sub0_sub1 : AMDGPU::sub0; in selectImageIntrinsic()
2004 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), VDataOut) in selectImageIntrinsic()
2032 if (AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::unorm)) in selectImageIntrinsic()
2037 STI.hasFeature(AMDGPU::FeatureR128A16) ? -1 : 0); in selectImageIntrinsic()
2048 if (AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::lwe)) in selectImageIntrinsic()
2061 Register Zero = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectImageIntrinsic()
2062 BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::V_MOV_B32_e32), Zero) in selectImageIntrinsic()
2069 BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::REG_SEQUENCE), Tied); in selectImageIntrinsic()
2075 Register Undef = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectImageIntrinsic()
2076 BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::IMPLICIT_DEF), Undef); in selectImageIntrinsic()
2078 BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::REG_SEQUENCE), Tied); in selectImageIntrinsic()
2089 TII.enforceOperandRCAlignment(*MIB, AMDGPU::OpName::vaddr); in selectImageIntrinsic()
2108 auto MIB = BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::DS_BVH_STACK_RTN_B32), Dst0) in selectDSBvhStackIntrinsic()
2187 unsigned SelectOpcode = Size == 64 ? AMDGPU::S_CSELECT_B64 : in selectG_SELECT()
2188 AMDGPU::S_CSELECT_B32; in selectG_SELECT()
2189 MachineInstr *CopySCC = BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), AMDGPU::SCC) in selectG_SELECT()
2213 BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in selectG_SELECT()
2228 return AMDGPU::sub0; in sizeToSubRegIndex()
2230 return AMDGPU::sub0_sub1; in sizeToSubRegIndex()
2232 return AMDGPU::sub0_sub1_sub2; in sizeToSubRegIndex()
2234 return AMDGPU::sub0_sub1_sub2_sub3; in sizeToSubRegIndex()
2236 return AMDGPU::sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7; in sizeToSubRegIndex()
2239 return AMDGPU::sub0; in sizeToSubRegIndex()
2265 const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID; in selectG_TRUNC()
2289 BuildMI(*MBB, I, DL, TII.get(AMDGPU::COPY), LoReg) in selectG_TRUNC()
2290 .addReg(SrcReg, 0, AMDGPU::sub0); in selectG_TRUNC()
2291 BuildMI(*MBB, I, DL, TII.get(AMDGPU::COPY), HiReg) in selectG_TRUNC()
2292 .addReg(SrcReg, 0, AMDGPU::sub1); in selectG_TRUNC()
2298 BuildMI(*MBB, I, DL, TII.get(AMDGPU::V_MOV_B32_sdwa), DstReg) in selectG_TRUNC()
2302 .addImm(AMDGPU::SDWA::WORD_1) // $dst_sel in selectG_TRUNC()
2303 .addImm(AMDGPU::SDWA::UNUSED_PRESERVE) // $dst_unused in selectG_TRUNC()
2304 .addImm(AMDGPU::SDWA::WORD_0) // $src0_sel in selectG_TRUNC()
2312 BuildMI(*MBB, I, DL, TII.get(AMDGPU::V_LSHLREV_B32_e64), TmpReg0) in selectG_TRUNC()
2316 BuildMI(*MBB, I, DL, TII.get(AMDGPU::S_LSHL_B32), TmpReg0) in selectG_TRUNC()
2322 unsigned MovOpc = IsVALU ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32; in selectG_TRUNC()
2323 unsigned AndOpc = IsVALU ? AMDGPU::V_AND_B32_e64 : AMDGPU::S_AND_B32; in selectG_TRUNC()
2324 unsigned OrOpc = IsVALU ? AMDGPU::V_OR_B32_e64 : AMDGPU::S_OR_B32; in selectG_TRUNC()
2394 bool InReg = I.getOpcode() == AMDGPU::G_SEXT_INREG; in selectG_SZA_EXT()
2395 bool Signed = I.getOpcode() == AMDGPU::G_SEXT || InReg; in selectG_SZA_EXT()
2403 const unsigned SrcSize = I.getOpcode() == AMDGPU::G_SEXT_INREG ? in selectG_SZA_EXT()
2413 if (I.getOpcode() == AMDGPU::G_ANYEXT) { in selectG_SZA_EXT()
2424 BuildMI(MBB, I, DL, TII.get(AMDGPU::IMPLICIT_DEF), UndefReg); in selectG_SZA_EXT()
2425 BuildMI(MBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg) in selectG_SZA_EXT()
2427 .addImm(AMDGPU::sub0) in selectG_SZA_EXT()
2429 .addImm(AMDGPU::sub1); in selectG_SZA_EXT()
2436 if (SrcBank->getID() == AMDGPU::VGPRRegBankID && DstSize <= 32) { in selectG_SZA_EXT()
2443 BuildMI(MBB, I, DL, TII.get(AMDGPU::V_AND_B32_e32), DstReg) in selectG_SZA_EXT()
2450 const unsigned BFE = Signed ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64; in selectG_SZA_EXT()
2460 if (SrcBank->getID() == AMDGPU::SGPRRegBankID && DstSize <= 64) { in selectG_SZA_EXT()
2462 AMDGPU::SReg_64RegClass : AMDGPU::SReg_32RegClass; in selectG_SZA_EXT()
2468 AMDGPU::S_SEXT_I32_I8 : AMDGPU::S_SEXT_I32_I16; in selectG_SZA_EXT()
2472 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_32RegClass, *MRI); in selectG_SZA_EXT()
2478 Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_SZA_EXT()
2479 unsigned SubReg = InReg ? AMDGPU::sub0 : AMDGPU::NoSubRegister; in selectG_SZA_EXT()
2481 BuildMI(MBB, I, DL, TII.get(AMDGPU::S_ASHR_I32), HiReg) in selectG_SZA_EXT()
2486 BuildMI(MBB, I, DL, TII.get(AMDGPU::S_MOV_B32), HiReg) in selectG_SZA_EXT()
2489 BuildMI(MBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg) in selectG_SZA_EXT()
2491 .addImm(AMDGPU::sub0) in selectG_SZA_EXT()
2493 .addImm(AMDGPU::sub1); in selectG_SZA_EXT()
2495 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_64RegClass, in selectG_SZA_EXT()
2499 const unsigned BFE64 = Signed ? AMDGPU::S_BFE_I64 : AMDGPU::S_BFE_U64; in selectG_SZA_EXT()
2500 const unsigned BFE32 = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32; in selectG_SZA_EXT()
2505 Register ExtReg = MRI->createVirtualRegister(&AMDGPU::SReg_64RegClass); in selectG_SZA_EXT()
2506 Register UndefReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_SZA_EXT()
2507 unsigned SubReg = InReg ? AMDGPU::sub0 : AMDGPU::NoSubRegister; in selectG_SZA_EXT()
2509 BuildMI(MBB, I, DL, TII.get(AMDGPU::IMPLICIT_DEF), UndefReg); in selectG_SZA_EXT()
2510 BuildMI(MBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), ExtReg) in selectG_SZA_EXT()
2512 .addImm(AMDGPU::sub0) in selectG_SZA_EXT()
2514 .addImm(AMDGPU::sub1); in selectG_SZA_EXT()
2521 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_64RegClass, *MRI); in selectG_SZA_EXT()
2526 BuildMI(MBB, I, DL, TII.get(AMDGPU::S_AND_B32), DstReg) in selectG_SZA_EXT()
2537 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_32RegClass, *MRI); in selectG_SZA_EXT()
2560 if (DstRB->getID() != AMDGPU::SGPRRegBankID) in selectG_FPEXT()
2569 BuildMI(*BB, &I, I.getDebugLoc(), TII.get(AMDGPU::S_CVT_HI_F32_F16), Dst) in selectG_FPEXT()
2572 return RBI.constrainGenericRegister(Dst, AMDGPU::SReg_32RegClass, *MRI); in selectG_FPEXT()
2598 const bool IsSgpr = DstRB->getID() == AMDGPU::SGPRRegBankID; in selectG_CONSTANT()
2601 if (DstRB->getID() == AMDGPU::VCCRegBankID) { in selectG_CONSTANT()
2602 Opcode = STI.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; in selectG_CONSTANT()
2604 AMDGPU::isValid32BitLiteral(I.getOperand(1).getImm(), IsFP)) { in selectG_CONSTANT()
2605 Opcode = IsSgpr ? AMDGPU::S_MOV_B64_IMM_PSEUDO : AMDGPU::V_MOV_B64_PSEUDO; in selectG_CONSTANT()
2610 Opcode = IsSgpr ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32; in selectG_CONSTANT()
2631 ResInst = BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_MOV_B64), DstReg) in selectG_CONSTANT()
2635 &AMDGPU::SReg_32RegClass : &AMDGPU::VGPR_32RegClass; in selectG_CONSTANT()
2645 ResInst = BuildMI(*BB, &I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg) in selectG_CONSTANT()
2647 .addImm(AMDGPU::sub0) in selectG_CONSTANT()
2649 .addImm(AMDGPU::sub1); in selectG_CONSTANT()
2676 if (DstRB->getID() != AMDGPU::SGPRRegBankID || in selectG_FNEG()
2685 if (!RBI.constrainGenericRegister(Src, AMDGPU::SReg_64RegClass, *MRI) || in selectG_FNEG()
2686 !RBI.constrainGenericRegister(Dst, AMDGPU::SReg_64RegClass, *MRI)) in selectG_FNEG()
2691 Register LoReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FNEG()
2692 Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FNEG()
2693 Register ConstReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FNEG()
2694 Register OpReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FNEG()
2696 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), LoReg) in selectG_FNEG()
2697 .addReg(Src, 0, AMDGPU::sub0); in selectG_FNEG()
2698 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), HiReg) in selectG_FNEG()
2699 .addReg(Src, 0, AMDGPU::sub1); in selectG_FNEG()
2700 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_MOV_B32), ConstReg) in selectG_FNEG()
2704 unsigned Opc = Fabs ? AMDGPU::S_OR_B32 : AMDGPU::S_XOR_B32; in selectG_FNEG()
2709 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::REG_SEQUENCE), Dst) in selectG_FNEG()
2711 .addImm(AMDGPU::sub0) in selectG_FNEG()
2713 .addImm(AMDGPU::sub1); in selectG_FNEG()
2722 if (DstRB->getID() != AMDGPU::SGPRRegBankID || in selectG_FABS()
2729 Register LoReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FABS()
2730 Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FABS()
2731 Register ConstReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FABS()
2732 Register OpReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FABS()
2734 if (!RBI.constrainGenericRegister(Src, AMDGPU::SReg_64RegClass, *MRI) || in selectG_FABS()
2735 !RBI.constrainGenericRegister(Dst, AMDGPU::SReg_64RegClass, *MRI)) in selectG_FABS()
2738 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), LoReg) in selectG_FABS()
2739 .addReg(Src, 0, AMDGPU::sub0); in selectG_FABS()
2740 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), HiReg) in selectG_FABS()
2741 .addReg(Src, 0, AMDGPU::sub1); in selectG_FABS()
2742 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_MOV_B32), ConstReg) in selectG_FABS()
2747 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_AND_B32), OpReg) in selectG_FABS()
2751 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::REG_SEQUENCE), Dst) in selectG_FABS()
2753 .addImm(AMDGPU::sub0) in selectG_FABS()
2755 .addImm(AMDGPU::sub1); in selectG_FABS()
2768 unsigned OpNo = Load.getOpcode() == AMDGPU::G_PREFETCH ? 0 : 1; in getAddrModeInfo()
2791 if (OpBank->getID() == AMDGPU::SGPRRegBankID) in getAddrModeInfo()
2802 return RBI.getRegBank(Reg, *MRI, TRI)->getID() == AMDGPU::SGPRRegBankID; in isSGPR()
2823 if (MI.getOpcode() == AMDGPU::G_PREFETCH) in isInstrUniform()
2825 AMDGPU::SGPRRegBankID; in isInstrUniform()
2847 BuildMI(*BB, &I, I.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), AMDGPU::M0) in initM0()
2865 if (Opcode == AMDGPU::COPY) in isVCmpResult()
2868 if (Opcode == AMDGPU::G_AND || Opcode == AMDGPU::G_OR || in isVCmpResult()
2869 Opcode == AMDGPU::G_XOR) in isVCmpResult()
2876 return Opcode == AMDGPU::G_ICMP || Opcode == AMDGPU::G_FCMP; in isVCmpResult()
2898 CondPhysReg = AMDGPU::SCC; in selectG_BRCOND()
2899 BrOpcode = AMDGPU::S_CBRANCH_SCC1; in selectG_BRCOND()
2900 ConstrainRC = &AMDGPU::SReg_32RegClass; in selectG_BRCOND()
2908 const unsigned Opcode = Is64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32; in selectG_BRCOND()
2909 const Register Exec = Is64 ? AMDGPU::EXEC : AMDGPU::EXEC_LO; in selectG_BRCOND()
2920 BrOpcode = AMDGPU::S_CBRANCH_VCCNZ; in selectG_BRCOND()
2927 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), CondPhysReg) in selectG_BRCOND()
2940 const bool IsVGPR = DstRB->getID() == AMDGPU::VGPRRegBankID; in selectG_GLOBAL_VALUE()
2941 I.setDesc(TII.get(IsVGPR ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32)); in selectG_GLOBAL_VALUE()
2943 I.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true)); in selectG_GLOBAL_VALUE()
2946 DstReg, IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass, *MRI); in selectG_GLOBAL_VALUE()
2961 const bool IsVGPR = DstRB->getID() == AMDGPU::VGPRRegBankID; in selectG_PTRMASK()
2976 auto MIB = BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_AND_B64), DstReg) in selectG_PTRMASK()
2984 unsigned NewOpc = IsVGPR ? AMDGPU::V_AND_B32_e64 : AMDGPU::S_AND_B32; in selectG_PTRMASK()
2986 = IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass; in selectG_PTRMASK()
3016 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), LoReg) in selectG_PTRMASK()
3017 .addReg(SrcReg, 0, AMDGPU::sub0); in selectG_PTRMASK()
3018 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), HiReg) in selectG_PTRMASK()
3019 .addReg(SrcReg, 0, AMDGPU::sub1); in selectG_PTRMASK()
3031 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), MaskLo) in selectG_PTRMASK()
3032 .addReg(MaskReg, 0, AMDGPU::sub0); in selectG_PTRMASK()
3045 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), MaskHi) in selectG_PTRMASK()
3046 .addReg(MaskReg, 0, AMDGPU::sub1); in selectG_PTRMASK()
3052 BuildMI(*BB, &I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg) in selectG_PTRMASK()
3054 .addImm(AMDGPU::sub0) in selectG_PTRMASK()
3056 .addImm(AMDGPU::sub1); in selectG_PTRMASK()
3071 AMDGPU::getBaseWithConstantOffset(MRI, IdxReg, &KnownBits); in computeIndirectRegIndex()
3072 if (IdxBaseReg == AMDGPU::NoRegister) { in computeIndirectRegIndex()
3103 if (IdxRB->getID() != AMDGPU::SGPRRegBankID) in selectG_EXTRACT_VECTOR_ELT()
3114 !RBI.constrainGenericRegister(IdxReg, AMDGPU::SReg_32RegClass, *MRI)) in selectG_EXTRACT_VECTOR_ELT()
3125 if (SrcRB->getID() == AMDGPU::SGPRRegBankID) { in selectG_EXTRACT_VECTOR_ELT()
3129 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectG_EXTRACT_VECTOR_ELT()
3132 unsigned Opc = Is64 ? AMDGPU::S_MOVRELS_B64 : AMDGPU::S_MOVRELS_B32; in selectG_EXTRACT_VECTOR_ELT()
3140 if (SrcRB->getID() != AMDGPU::VGPRRegBankID || DstTy.getSizeInBits() != 32) in selectG_EXTRACT_VECTOR_ELT()
3144 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectG_EXTRACT_VECTOR_ELT()
3146 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::V_MOVRELS_B32_e32), DstReg) in selectG_EXTRACT_VECTOR_ELT()
3185 if (IdxRB->getID() != AMDGPU::SGPRRegBankID) in selectG_INSERT_VECTOR_ELT()
3196 !RBI.constrainGenericRegister(IdxReg, AMDGPU::SReg_32RegClass, *MRI)) in selectG_INSERT_VECTOR_ELT()
3199 if (VecRB->getID() == AMDGPU::VGPRRegBankID && ValSize != 32) in selectG_INSERT_VECTOR_ELT()
3206 const bool IndexMode = VecRB->getID() == AMDGPU::VGPRRegBankID && in selectG_INSERT_VECTOR_ELT()
3213 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectG_INSERT_VECTOR_ELT()
3217 VecSize, ValSize, VecRB->getID() == AMDGPU::SGPRRegBankID); in selectG_INSERT_VECTOR_ELT()
3239 assert(!AMDGPU::isGFX12Plus(STI)); in selectBufferLoadLds()
3261 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_BOTHEN in selectBufferLoadLds()
3262 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_IDXEN in selectBufferLoadLds()
3263 : HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFEN in selectBufferLoadLds()
3264 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFSET; in selectBufferLoadLds()
3267 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_BOTHEN in selectBufferLoadLds()
3268 : AMDGPU::BUFFER_LOAD_USHORT_LDS_IDXEN in selectBufferLoadLds()
3269 : HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFEN in selectBufferLoadLds()
3270 : AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFSET; in selectBufferLoadLds()
3273 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_BOTHEN in selectBufferLoadLds()
3274 : AMDGPU::BUFFER_LOAD_DWORD_LDS_IDXEN in selectBufferLoadLds()
3275 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFEN in selectBufferLoadLds()
3276 : AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFSET; in selectBufferLoadLds()
3282 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectBufferLoadLds()
3289 BuildMI(*MBB, &*MIB, DL, TII.get(AMDGPU::REG_SEQUENCE), IdxReg) in selectBufferLoadLds()
3291 .addImm(AMDGPU::sub0) in selectBufferLoadLds()
3293 .addImm(AMDGPU::sub1); in selectBufferLoadLds()
3306 MIB.addImm(Aux & AMDGPU::CPol::ALL); // cpol in selectBufferLoadLds()
3307 MIB.addImm(Aux & AMDGPU::CPol::SWZ_pregfx12 ? 1 : 0); // swz in selectBufferLoadLds()
3339 if (Def->getOpcode() != AMDGPU::G_MERGE_VALUES) in matchZeroExtendFromS32()
3359 Opc = AMDGPU::GLOBAL_LOAD_LDS_UBYTE; in selectGlobalLoadLds()
3362 Opc = AMDGPU::GLOBAL_LOAD_LDS_USHORT; in selectGlobalLoadLds()
3365 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORD; in selectGlobalLoadLds()
3371 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectGlobalLoadLds()
3382 } else if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) { in selectGlobalLoadLds()
3396 Opc = AMDGPU::getGlobalSaddrOp(Opc); in selectGlobalLoadLds()
3398 VOffset = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectGlobalLoadLds()
3399 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::V_MOV_B32_e32), VOffset) in selectGlobalLoadLds()
3444 Opc = AMDGPU::V_SMFMAC_F32_16X16X32_F16_e64; in selectSMFMACIntrin()
3447 Opc = AMDGPU::V_SMFMAC_F32_32X32X16_F16_e64; in selectSMFMACIntrin()
3450 Opc = AMDGPU::V_SMFMAC_F32_16X16X32_BF16_e64; in selectSMFMACIntrin()
3453 Opc = AMDGPU::V_SMFMAC_F32_32X32X16_BF16_e64; in selectSMFMACIntrin()
3456 Opc = AMDGPU::V_SMFMAC_I32_16X16X64_I8_e64; in selectSMFMACIntrin()
3459 Opc = AMDGPU::V_SMFMAC_I32_32X32X32_I8_e64; in selectSMFMACIntrin()
3462 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_BF8_BF8_e64; in selectSMFMACIntrin()
3465 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_BF8_FP8_e64; in selectSMFMACIntrin()
3468 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_FP8_BF8_e64; in selectSMFMACIntrin()
3471 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_FP8_FP8_e64; in selectSMFMACIntrin()
3474 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_BF8_BF8_e64; in selectSMFMACIntrin()
3477 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_BF8_FP8_e64; in selectSMFMACIntrin()
3480 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_FP8_BF8_e64; in selectSMFMACIntrin()
3483 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_FP8_FP8_e64; in selectSMFMACIntrin()
3503 const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID; in selectWaveAddress()
3508 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_LSHRREV_B32_e64), DstReg) in selectWaveAddress()
3512 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::S_LSHR_B32), DstReg) in selectWaveAddress()
3519 IsVALU ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass; in selectWaveAddress()
3529 if (!RBI.constrainGenericRegister(SrcReg, AMDGPU::SReg_32RegClass, *MRI)) in selectStackRestore()
3540 WaveAddr = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectStackRestore()
3541 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::S_LSHR_B32), WaveAddr) in selectStackRestore()
3547 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), SP) in selectStackRestore()
3581 case AMDGPU::G_AMDGPU_MAD_U64_U32: in select()
3582 case AMDGPU::G_AMDGPU_MAD_I64_I32: in select()
3646 case AMDGPU::G_AMDGPU_ATOMIC_FMIN: in select()
3647 case AMDGPU::G_AMDGPU_ATOMIC_FMAX: in select()
3678 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD: in select()
3679 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16: in select()
3680 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE: in select()
3681 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16: { in select()
3682 const AMDGPU::ImageDimIntrinsicInfo *Intr = in select()
3683 AMDGPU::getImageDimIntrinsicInfo(AMDGPU::getIntrinsicID(I)); in select()
3687 case AMDGPU::G_AMDGPU_INTRIN_BVH_INTERSECT_RAY: in select()
3689 case AMDGPU::G_SBFX: in select()
3690 case AMDGPU::G_UBFX: in select()
3692 case AMDGPU::G_SI_CALL: in select()
3693 I.setDesc(TII.get(AMDGPU::SI_CALL)); in select()
3695 case AMDGPU::G_AMDGPU_WAVE_ADDRESS: in select()
3697 case AMDGPU::G_STACKRESTORE: in select()
3721 if (MI->getOpcode() == AMDGPU::G_FNEG) { in selectVOP3ModsImpl()
3725 } else if (MI->getOpcode() == AMDGPU::G_FSUB && IsCanonicalizing) { in selectVOP3ModsImpl()
3736 if (AllowAbs && MI->getOpcode() == AMDGPU::G_FABS) { in selectVOP3ModsImpl()
3751 RBI.getRegBank(Src, *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID) { in copyToVGPRIfSrcFolded()
3758 TII.get(AMDGPU::COPY), VGPRSrc) in copyToVGPRIfSrcFolded()
3867 if (Def->getOpcode() == AMDGPU::G_FNEG || Def->getOpcode() == AMDGPU::G_FABS) in selectVOP3NoMods()
3880 if (MI && MI->getOpcode() == AMDGPU::G_FNEG && in selectVOP3PModsImpl()
3965 DstRegClass = &AMDGPU::VReg_256RegClass; in buildRegSequence()
3968 DstRegClass = &AMDGPU::VReg_128RegClass; in buildRegSequence()
3971 DstRegClass = &AMDGPU::VReg_64RegClass; in buildRegSequence()
3978 auto MIB = B.buildInstr(AMDGPU::REG_SEQUENCE) in buildRegSequence()
4029 ModOpcode = (ElF32->getOpcode() == AMDGPU::G_FNEG) ? AMDGPU::G_FNEG in selectWMMAModsF32NegAbs()
4030 : AMDGPU::G_FABS; in selectWMMAModsF32NegAbs()
4085 ModOpcode = (ElV2F16->getOpcode() == AMDGPU::G_FNEG) ? AMDGPU::G_FNEG in selectWMMAModsF16NegAbs()
4086 : AMDGPU::G_FABS; in selectWMMAModsF16NegAbs()
4238 AMDGPU::getSMRDEncodedOffset(STI, GEPI.Imm, false); in selectSmrdOffset()
4271 *SOffset = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectSmrdOffset()
4272 BuildMI(*MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), *SOffset) in selectSmrdOffset()
4310 AMDGPU::getSMRDEncodedLiteralOffset32(STI, GEPInfo.Imm); in selectSmrdImm32()
4432 MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectGlobalSAddr()
4434 BuildMI(*MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::V_MOV_B32_e32), in selectGlobalSAddr()
4456 if (STI.getConstantBusLimit(AMDGPU::V_ADD_U32_e64) > NumLiterals) in selectGlobalSAddr()
4464 if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) { in selectGlobalSAddr()
4490 if (AddrDef->MI->getOpcode() == AMDGPU::G_IMPLICIT_DEF || in selectGlobalSAddr()
4491 AddrDef->MI->getOpcode() == AMDGPU::G_CONSTANT || !isSGPR(AddrDef->Reg)) in selectGlobalSAddr()
4498 Register VOffset = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectGlobalSAddr()
4500 BuildMI(*MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::V_MOV_B32_e32), VOffset) in selectGlobalSAddr()
4529 if (AddrDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX) { in selectScratchSAddr()
4539 if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) { in selectScratchSAddr()
4545 if (LHSDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX && in selectScratchSAddr()
4551 SAddr = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectScratchSAddr()
4553 BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_ADD_I32), SAddr) in selectScratchSAddr()
4606 if (AddrDef->MI->getOpcode() != AMDGPU::G_PTR_ADD) in selectScratchSVAddr()
4610 if (RBI.getRegBank(RHS, *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID) in selectScratchSVAddr()
4627 if (LHSDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX) { in selectScratchSVAddr()
4656 Register HighBits = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectMUBUFScratchOffen()
4661 BuildMI(*MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::V_MOV_B32_e32), in selectMUBUFScratchOffen()
4696 if (PtrBaseDef->getOpcode() == AMDGPU::G_FRAME_INDEX) in selectMUBUFScratchOffen()
4702 } else if (RootDef->getOpcode() == AMDGPU::G_FRAME_INDEX) { in selectMUBUFScratchOffen()
4941 } else if (RootDef->getOpcode() == AMDGPU::G_SUB) { in selectDS1Addr1OffsetImpl()
5008 } else if (RootDef->getOpcode() == AMDGPU::G_SUB) { in selectDSReadWrite2Impl()
5047 Register RSrc2 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); in buildRSRC()
5048 Register RSrc3 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); in buildRSRC()
5049 Register RSrcHi = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in buildRSRC()
5050 Register RSrc = MRI.createVirtualRegister(&AMDGPU::SGPR_128RegClass); in buildRSRC()
5052 B.buildInstr(AMDGPU::S_MOV_B32) in buildRSRC()
5055 B.buildInstr(AMDGPU::S_MOV_B32) in buildRSRC()
5062 B.buildInstr(AMDGPU::REG_SEQUENCE) in buildRSRC()
5065 .addImm(AMDGPU::sub0) in buildRSRC()
5067 .addImm(AMDGPU::sub1); in buildRSRC()
5071 RSrcLo = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in buildRSRC()
5072 B.buildInstr(AMDGPU::S_MOV_B64) in buildRSRC()
5077 B.buildInstr(AMDGPU::REG_SEQUENCE) in buildRSRC()
5080 .addImm(AMDGPU::sub0_sub1) in buildRSRC()
5082 .addImm(AMDGPU::sub2_sub3); in buildRSRC()
5144 return N0Bank->getID() == AMDGPU::VGPRRegBankID; in shouldUseAddr64()
5156 SOffset = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in splitIllegalMUBUFOffset()
5157 B.buildInstr(AMDGPU::S_MOV_B32) in splitIllegalMUBUFOffset()
5184 if (RBI.getRegBank(N2, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) { in selectMUBUFAddr64Impl()
5186 if (RBI.getRegBank(N3, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) { in selectMUBUFAddr64Impl()
5199 } else if (RBI.getRegBank(N0, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) { in selectMUBUFAddr64Impl()
5262 MIB.addReg(AMDGPU::SGPR_NULL); in selectMUBUFAddr64()
5292 MIB.addReg(AMDGPU::SGPR_NULL); in selectMUBUFOffset()
5309 SOffset = AMDGPU::SGPR_NULL; in selectBUFSOffset()
5331 AMDGPU::getSMRDEncodedOffset(STI, *OffsetVal, true); in selectSMRDBufferImm()
5347 AMDGPU::getSMRDEncodedLiteralOffset32(STI, *OffsetVal); in selectSMRDBufferImm32()
5360 std::tie(SOffset, Offset) = AMDGPU::getBaseWithConstantOffset( in selectSMRDBufferSgprImm()
5366 AMDGPU::getSMRDEncodedOffset(STI, Offset, /* IsBuffer */ true); in selectSMRDBufferSgprImm()
5378 if (MI->getOpcode() == AMDGPU::G_BITCAST) in stripBitCast()
5389 if (Inst->getOpcode() != AMDGPU::G_TRUNC) in isExtractHiElt()
5397 if (TruncOp->getOpcode() == AMDGPU::G_LSHR) { in isExtractHiElt()
5411 if (TruncOp->getOpcode() == AMDGPU::G_SHUFFLE_VECTOR) { in isExtractHiElt()
5438 if (MI->getOpcode() == AMDGPU::G_FPEXT) { in selectVOP3PMadMixModsImpl()
5447 if (MI->getOpcode() == AMDGPU::G_BITCAST) { in selectVOP3PMadMixModsImpl()
5531 auto CopyMIB = BuildMI(*MBB, &I, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectSBarrierSignalIsfirst()
5533 BuildMI(*MBB, &I, DL, TII.get(AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0)); in selectSBarrierSignalIsfirst()
5537 BuildMI(*MBB, &I, DL, TII.get(AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM)) in selectSBarrierSignalIsfirst()
5541 BuildMI(*MBB, &I, DL, TII.get(AMDGPU::COPY), CCReg).addReg(AMDGPU::SCC); in selectSBarrierSignalIsfirst()
5544 return RBI.constrainGenericRegister(CCReg, AMDGPU::SReg_32_XM0_XEXECRegClass, in selectSBarrierSignalIsfirst()
5554 return AMDGPU::S_BARRIER_INIT_IMM; in getNamedBarrierOp()
5556 return AMDGPU::S_BARRIER_JOIN_IMM; in getNamedBarrierOp()
5558 return AMDGPU::S_WAKEUP_BARRIER_IMM; in getNamedBarrierOp()
5560 return AMDGPU::S_GET_BARRIER_STATE_IMM; in getNamedBarrierOp()
5567 return AMDGPU::S_BARRIER_INIT_M0; in getNamedBarrierOp()
5569 return AMDGPU::S_BARRIER_JOIN_M0; in getNamedBarrierOp()
5571 return AMDGPU::S_WAKEUP_BARRIER_M0; in getNamedBarrierOp()
5573 return AMDGPU::S_GET_BARRIER_STATE_M0; in getNamedBarrierOp()
5593 TmpReg0 = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectNamedBarrierInst()
5596 BuildMI(*MBB, &I, DL, TII.get(AMDGPU::S_LSHL_B32), TmpReg0) in selectNamedBarrierInst()
5608 Register TmpReg1 = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectNamedBarrierInst()
5609 BuildMI(*MBB, &I, DL, TII.get(AMDGPU::S_OR_B32), TmpReg1) in selectNamedBarrierInst()
5621 BuildMI(*MBB, &I, DL, TII.get(AMDGPU::COPY), AMDGPU::M0).addReg(M0Val); in selectNamedBarrierInst()
5644 BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_BARRIER_LEAVE)); in selectSBarrierLeave()
5645 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), CCReg).addReg(AMDGPU::SCC); in selectSBarrierLeave()
5648 return RBI.constrainGenericRegister(CCReg, AMDGPU::SReg_32_XM0_XEXECRegClass, in selectSBarrierLeave()
5710 (AMDGPU::isGFX12Plus(STI) ? AMDGPU::CPol::ALL in renderExtractCPol()
5711 : AMDGPU::CPol::ALL_pregfx12)); in renderExtractCPol()
5719 (AMDGPU::isGFX12Plus(STI) ? AMDGPU::CPol::SWZ in renderExtractSWZ()
5720 : AMDGPU::CPol::SWZ_pregfx12); in renderExtractSWZ()
5728 (AMDGPU::isGFX12Plus(STI) ? AMDGPU::CPol::ALL in renderExtractCpolSetGLC()
5729 : AMDGPU::CPol::ALL_pregfx12); in renderExtractCpolSetGLC()
5730 MIB.addImm(Cpol | AMDGPU::CPol::GLC); in renderExtractCpolSetGLC()
5749 return AMDGPU::isInlinableLiteral16(Imm, STI.hasInv2PiInlineImm()); in isInlineImmediate16()
5753 return AMDGPU::isInlinableLiteral32(Imm, STI.hasInv2PiInlineImm()); in isInlineImmediate32()
5757 return AMDGPU::isInlinableLiteral64(Imm, STI.hasInv2PiInlineImm()); in isInlineImmediate64()