Lines Matching refs:AMDGPU
87 return MRI.getVRegDef(Reg)->getOpcode() != AMDGPU::G_TRUNC && in isVCC()
92 return RB->getID() == AMDGPU::VCCRegBankID; in isVCC()
99 MI.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true)); in constrainCopyLikeIntrin()
130 if (SrcReg == AMDGPU::SCC) { in selectCOPY()
150 STI.isWave64() ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32; in selectCOPY()
161 TRI.isSGPRClass(SrcRC) ? AMDGPU::S_AND_B32 : AMDGPU::V_AND_B32_e32; in selectCOPY()
165 BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_CMP_NE_U32_e64), DstReg) in selectCOPY()
247 BuildMI(*BB, MI, MI->getDebugLoc(), TII.get(AMDGPU::COPY), DstReg) in getSubOperand64()
263 case AMDGPU::sub0: in getSubOperand64()
265 case AMDGPU::sub1: in getSubOperand64()
272 case AMDGPU::G_AND: in getLogicalBitOpcode()
273 return Is64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32; in getLogicalBitOpcode()
274 case AMDGPU::G_OR: in getLogicalBitOpcode()
275 return Is64 ? AMDGPU::S_OR_B64 : AMDGPU::S_OR_B32; in getLogicalBitOpcode()
276 case AMDGPU::G_XOR: in getLogicalBitOpcode()
277 return Is64 ? AMDGPU::S_XOR_B64 : AMDGPU::S_XOR_B32; in getLogicalBitOpcode()
288 if (DstRB->getID() != AMDGPU::SGPRRegBankID && in selectG_AND_OR_XOR()
289 DstRB->getID() != AMDGPU::VCCRegBankID) in selectG_AND_OR_XOR()
292 bool Is64 = Size > 32 || (DstRB->getID() == AMDGPU::VCCRegBankID && in selectG_AND_OR_XOR()
297 I.addOperand(MachineOperand::CreateReg(AMDGPU::SCC, true, // isDef in selectG_AND_OR_XOR()
315 const bool IsSALU = DstRB->getID() == AMDGPU::SGPRRegBankID; in selectG_ADD_SUB()
320 const unsigned Opc = Sub ? AMDGPU::S_SUB_U32 : AMDGPU::S_ADD_U32; in selectG_ADD_SUB()
330 const unsigned Opc = Sub ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_ADD_U32_e64; in selectG_ADD_SUB()
333 I.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true)); in selectG_ADD_SUB()
337 const unsigned Opc = Sub ? AMDGPU::V_SUB_CO_U32_e64 : AMDGPU::V_ADD_CO_U32_e64; in selectG_ADD_SUB()
353 = IsSALU ? AMDGPU::SReg_64_XEXECRegClass : AMDGPU::VReg_64RegClass; in selectG_ADD_SUB()
355 = IsSALU ? AMDGPU::SReg_32RegClass : AMDGPU::VGPR_32RegClass; in selectG_ADD_SUB()
357 MachineOperand Lo1(getSubOperand64(I.getOperand(1), HalfRC, AMDGPU::sub0)); in selectG_ADD_SUB()
358 MachineOperand Lo2(getSubOperand64(I.getOperand(2), HalfRC, AMDGPU::sub0)); in selectG_ADD_SUB()
359 MachineOperand Hi1(getSubOperand64(I.getOperand(1), HalfRC, AMDGPU::sub1)); in selectG_ADD_SUB()
360 MachineOperand Hi2(getSubOperand64(I.getOperand(2), HalfRC, AMDGPU::sub1)); in selectG_ADD_SUB()
366 BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_ADD_U32), DstLo) in selectG_ADD_SUB()
369 BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_ADDC_U32), DstHi) in selectG_ADD_SUB()
375 BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_ADD_CO_U32_e64), DstLo) in selectG_ADD_SUB()
380 MachineInstr *Addc = BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_ADDC_U32_e64), DstHi) in selectG_ADD_SUB()
391 BuildMI(*BB, &I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg) in selectG_ADD_SUB()
393 .addImm(AMDGPU::sub0) in selectG_ADD_SUB()
395 .addImm(AMDGPU::sub1); in selectG_ADD_SUB()
412 const bool IsAdd = I.getOpcode() == AMDGPU::G_UADDO || in selectG_UADDO_USUBO_UADDE_USUBE()
413 I.getOpcode() == AMDGPU::G_UADDE; in selectG_UADDO_USUBO_UADDE_USUBE()
414 const bool HasCarryIn = I.getOpcode() == AMDGPU::G_UADDE || in selectG_UADDO_USUBO_UADDE_USUBE()
415 I.getOpcode() == AMDGPU::G_USUBE; in selectG_UADDO_USUBO_UADDE_USUBE()
419 IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64; in selectG_UADDO_USUBO_UADDE_USUBE()
420 unsigned CarryOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64; in selectG_UADDO_USUBO_UADDE_USUBE()
422 I.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true)); in selectG_UADDO_USUBO_UADDE_USUBE()
431 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), AMDGPU::SCC) in selectG_UADDO_USUBO_UADDE_USUBE()
435 unsigned NoCarryOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32; in selectG_UADDO_USUBO_UADDE_USUBE()
436 unsigned CarryOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32; in selectG_UADDO_USUBO_UADDE_USUBE()
441 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), Dst1Reg) in selectG_UADDO_USUBO_UADDE_USUBE()
442 .addReg(AMDGPU::SCC); in selectG_UADDO_USUBO_UADDE_USUBE()
445 MRI->setRegClass(Dst1Reg, &AMDGPU::SReg_32RegClass); in selectG_UADDO_USUBO_UADDE_USUBE()
447 if (!RBI.constrainGenericRegister(Dst0Reg, AMDGPU::SReg_32RegClass, *MRI) || in selectG_UADDO_USUBO_UADDE_USUBE()
448 !RBI.constrainGenericRegister(Src0Reg, AMDGPU::SReg_32RegClass, *MRI) || in selectG_UADDO_USUBO_UADDE_USUBE()
449 !RBI.constrainGenericRegister(Src1Reg, AMDGPU::SReg_32RegClass, *MRI)) in selectG_UADDO_USUBO_UADDE_USUBE()
454 AMDGPU::SReg_32RegClass, *MRI)) in selectG_UADDO_USUBO_UADDE_USUBE()
465 const bool IsUnsigned = I.getOpcode() == AMDGPU::G_AMDGPU_MAD_U64_U32; in selectG_AMDGPU_MAD_64_32()
469 Opc = IsUnsigned ? AMDGPU::V_MAD_U64_U32_gfx11_e64 in selectG_AMDGPU_MAD_64_32()
470 : AMDGPU::V_MAD_I64_I32_gfx11_e64; in selectG_AMDGPU_MAD_64_32()
472 Opc = IsUnsigned ? AMDGPU::V_MAD_U64_U32_e64 : AMDGPU::V_MAD_I64_I32_e64; in selectG_AMDGPU_MAD_64_32()
622 if (DstBank->getID() != AMDGPU::SGPRRegBankID) in selectG_BUILD_VECTOR_TRUNC()
643 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_MOV_B32), Dst) in selectG_BUILD_VECTOR_TRUNC()
646 return RBI.constrainGenericRegister(Dst, AMDGPU::SReg_32RegClass, *MRI); in selectG_BUILD_VECTOR_TRUNC()
653 if (Src1Def && Src1Def->getOpcode() == AMDGPU::G_IMPLICIT_DEF) { in selectG_BUILD_VECTOR_TRUNC()
654 MI.setDesc(TII.get(AMDGPU::COPY)); in selectG_BUILD_VECTOR_TRUNC()
656 return RBI.constrainGenericRegister(Dst, AMDGPU::SReg_32RegClass, *MRI) && in selectG_BUILD_VECTOR_TRUNC()
657 RBI.constrainGenericRegister(Src0, AMDGPU::SReg_32RegClass, *MRI); in selectG_BUILD_VECTOR_TRUNC()
681 unsigned Opc = AMDGPU::S_PACK_LL_B32_B16; in selectG_BUILD_VECTOR_TRUNC()
683 Opc = AMDGPU::S_PACK_HH_B32_B16; in selectG_BUILD_VECTOR_TRUNC()
687 Opc = AMDGPU::S_PACK_LH_B32_B16; in selectG_BUILD_VECTOR_TRUNC()
692 auto MIB = BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_LSHR_B32), Dst) in selectG_BUILD_VECTOR_TRUNC()
700 Opc = AMDGPU::S_PACK_HL_B32_B16; in selectG_BUILD_VECTOR_TRUNC()
750 if (SubReg == AMDGPU::NoSubRegister) in selectG_INSERT()
793 assert(RBI.getRegBank(DstReg, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID && in selectG_SBFX_UBFX()
802 unsigned Opc = IsSigned ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64; in selectG_SBFX_UBFX()
818 if (!RBI.constrainGenericRegister(M0Val, AMDGPU::SReg_32RegClass, *MRI) || in selectInterpP1F16()
819 !RBI.constrainGenericRegister(Dst, AMDGPU::VGPR_32RegClass, *MRI) || in selectInterpP1F16()
820 !RBI.constrainGenericRegister(Src0, AMDGPU::VGPR_32RegClass, *MRI)) in selectInterpP1F16()
830 Register InterpMov = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectInterpP1F16()
834 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectInterpP1F16()
836 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::V_INTERP_MOV_F32), InterpMov) in selectInterpP1F16()
841 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::V_INTERP_P1LV_F16), Dst) in selectInterpP1F16()
863 if (STI.getConstantBusLimit(AMDGPU::V_WRITELANE_B32) > 1) in selectWritelane()
873 auto MIB = BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::V_WRITELANE_B32), VDst); in selectWritelane()
889 if (ConstVal && AMDGPU::isInlinableLiteral32(ConstVal->Value.getSExtValue(), in selectWritelane()
899 RBI.constrainGenericRegister(LaneSelect, AMDGPU::SReg_32_XM0RegClass, *MRI); in selectWritelane()
901 BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectWritelane()
903 MIB.addReg(AMDGPU::M0); in selectWritelane()
922 Opc = AMDGPU::V_DIV_SCALE_F32_e64; in selectDivScale()
924 Opc = AMDGPU::V_DIV_SCALE_F64_e64; in selectDivScale()
962 BuildMI(*BB, &I, I.getDebugLoc(), TII.get(AMDGPU::SI_IF_BREAK)) in selectG_INTRINSIC()
981 return constrainCopyLikeIntrin(I, AMDGPU::WQM); in selectG_INTRINSIC()
983 return constrainCopyLikeIntrin(I, AMDGPU::SOFT_WQM); in selectG_INTRINSIC()
986 return constrainCopyLikeIntrin(I, AMDGPU::STRICT_WWM); in selectG_INTRINSIC()
988 return constrainCopyLikeIntrin(I, AMDGPU::STRICT_WQM); in selectG_INTRINSIC()
1030 return Size == 32 ? AMDGPU::V_CMP_NE_U32_e64 : AMDGPU::V_CMP_NE_U64_e64; in getV_CMPOpcode()
1032 return Size == 32 ? AMDGPU::V_CMP_EQ_U32_e64 : AMDGPU::V_CMP_EQ_U64_e64; in getV_CMPOpcode()
1034 return Size == 32 ? AMDGPU::V_CMP_GT_I32_e64 : AMDGPU::V_CMP_GT_I64_e64; in getV_CMPOpcode()
1036 return Size == 32 ? AMDGPU::V_CMP_GE_I32_e64 : AMDGPU::V_CMP_GE_I64_e64; in getV_CMPOpcode()
1038 return Size == 32 ? AMDGPU::V_CMP_LT_I32_e64 : AMDGPU::V_CMP_LT_I64_e64; in getV_CMPOpcode()
1040 return Size == 32 ? AMDGPU::V_CMP_LE_I32_e64 : AMDGPU::V_CMP_LE_I64_e64; in getV_CMPOpcode()
1042 return Size == 32 ? AMDGPU::V_CMP_GT_U32_e64 : AMDGPU::V_CMP_GT_U64_e64; in getV_CMPOpcode()
1044 return Size == 32 ? AMDGPU::V_CMP_GE_U32_e64 : AMDGPU::V_CMP_GE_U64_e64; in getV_CMPOpcode()
1046 return Size == 32 ? AMDGPU::V_CMP_LT_U32_e64 : AMDGPU::V_CMP_LT_U64_e64; in getV_CMPOpcode()
1048 return Size == 32 ? AMDGPU::V_CMP_LE_U32_e64 : AMDGPU::V_CMP_LE_U64_e64; in getV_CMPOpcode()
1060 return AMDGPU::S_CMP_LG_U64; in getS_CMPOpcode()
1062 return AMDGPU::S_CMP_EQ_U64; in getS_CMPOpcode()
1073 return AMDGPU::S_CMP_LG_U32; in getS_CMPOpcode()
1075 return AMDGPU::S_CMP_EQ_U32; in getS_CMPOpcode()
1077 return AMDGPU::S_CMP_GT_I32; in getS_CMPOpcode()
1079 return AMDGPU::S_CMP_GE_I32; in getS_CMPOpcode()
1081 return AMDGPU::S_CMP_LT_I32; in getS_CMPOpcode()
1083 return AMDGPU::S_CMP_LE_I32; in getS_CMPOpcode()
1085 return AMDGPU::S_CMP_GT_U32; in getS_CMPOpcode()
1087 return AMDGPU::S_CMP_GE_U32; in getS_CMPOpcode()
1089 return AMDGPU::S_CMP_LT_U32; in getS_CMPOpcode()
1091 return AMDGPU::S_CMP_LE_U32; in getS_CMPOpcode()
1114 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), CCReg) in selectG_ICMP()
1115 .addReg(AMDGPU::SCC); in selectG_ICMP()
1118 RBI.constrainGenericRegister(CCReg, AMDGPU::SReg_32RegClass, *MRI); in selectG_ICMP()
1154 BuildMI(*BB, &I, DL, TII.get(AMDGPU::IMPLICIT_DEF), Dst); in selectIntrinsicIcmp()
1193 unsigned Opcode = Is64 ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32; in selectBallot()
1196 Register SrcReg = Is64 ? AMDGPU::EXEC : AMDGPU::EXEC_LO; in selectBallot()
1197 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), DstReg).addReg(SrcReg); in selectBallot()
1202 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), DstReg).addReg(SrcReg); in selectBallot()
1216 const bool IsVALU = DstBank->getID() == AMDGPU::VGPRRegBankID; in selectRelocConstant()
1226 TII.get(IsVALU ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32), DstReg) in selectRelocConstant()
1238 unsigned Mov = DstRB->getID() == AMDGPU::SGPRRegBankID ? in selectGroupStaticSize()
1239 AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32; in selectGroupStaticSize()
1271 if (!RC->hasSubClassEq(&AMDGPU::SGPR_64RegClass) || in selectReturnAddress()
1278 BuildMI(*MBB, &I, DL, TII.get(AMDGPU::S_MOV_B64), DstReg) in selectReturnAddress()
1291 AMDGPU::SReg_64RegClass, DL); in selectReturnAddress()
1292 BuildMI(*MBB, &I, DL, TII.get(AMDGPU::COPY), DstReg) in selectReturnAddress()
1302 BuildMI(*BB, &MI, MI.getDebugLoc(), TII.get(AMDGPU::SI_END_CF)) in selectEndCfIntrinsic()
1358 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectDSOrderedIntrinsic()
1364 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::DS_ORDERED_COUNT), DstReg) in selectDSOrderedIntrinsic()
1369 if (!RBI.constrainGenericRegister(M0Val, AMDGPU::SReg_32RegClass, *MRI)) in selectDSOrderedIntrinsic()
1380 return AMDGPU::DS_GWS_INIT; in gwsIntrinToOpcode()
1382 return AMDGPU::DS_GWS_BARRIER; in gwsIntrinToOpcode()
1384 return AMDGPU::DS_GWS_SEMA_V; in gwsIntrinToOpcode()
1386 return AMDGPU::DS_GWS_SEMA_BR; in gwsIntrinToOpcode()
1388 return AMDGPU::DS_GWS_SEMA_P; in gwsIntrinToOpcode()
1390 return AMDGPU::DS_GWS_SEMA_RELEASE_ALL; in gwsIntrinToOpcode()
1408 if (OffsetRB->getID() != AMDGPU::SGPRRegBankID) in selectDSGWSIntrinsic()
1424 if (OffsetDef->getOpcode() == AMDGPU::V_READFIRSTLANE_B32) { in selectDSGWSIntrinsic()
1430 if (OffsetDef->getOpcode() == AMDGPU::G_CONSTANT) { in selectDSGWSIntrinsic()
1437 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::S_MOV_B32), AMDGPU::M0) in selectDSGWSIntrinsic()
1441 AMDGPU::getBaseWithConstantOffset(*MRI, BaseOffset); in selectDSGWSIntrinsic()
1446 if (!RBI.constrainGenericRegister(BaseOffset, AMDGPU::VGPR_32RegClass, *MRI)) in selectDSGWSIntrinsic()
1453 AMDGPU::SReg_32RegClass, *MRI)) in selectDSGWSIntrinsic()
1457 Register M0Base = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectDSGWSIntrinsic()
1458 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::S_LSHL_B32), M0Base) in selectDSGWSIntrinsic()
1462 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectDSGWSIntrinsic()
1475 if (!RBI.constrainGenericRegister(VSrc, AMDGPU::VGPR_32RegClass, *MRI)) in selectDSGWSIntrinsic()
1482 TII.enforceOperandRCAlignment(*MIB, AMDGPU::OpName::data0); in selectDSGWSIntrinsic()
1505 const unsigned Opc = IsAppend ? AMDGPU::DS_APPEND : AMDGPU::DS_CONSUME; in selectDSAppendConsume()
1507 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectDSAppendConsume()
1509 if (!RBI.constrainGenericRegister(PtrBase, AMDGPU::SReg_32RegClass, *MRI)) in selectDSAppendConsume()
1526 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::WAVE_BARRIER)); in selectSBarrier()
1548 MachineInstr &MI, const AMDGPU::ImageDimIntrinsicInfo *Intr) const { in selectImageIntrinsic()
1552 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode = in selectImageIntrinsic()
1553 AMDGPU::getMIMGBaseOpcodeInfo(Intr->BaseOpcode); in selectImageIntrinsic()
1555 const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfo(Intr->Dim); in selectImageIntrinsic()
1557 const bool IsGFX10Plus = AMDGPU::isGFX10Plus(STI); in selectImageIntrinsic()
1558 const bool IsGFX11Plus = AMDGPU::isGFX11Plus(STI); in selectImageIntrinsic()
1565 bool IsD16 = MI.getOpcode() == AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16 || in selectImageIntrinsic()
1566 MI.getOpcode() == AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16; in selectImageIntrinsic()
1603 assert(MI.getOperand(3).getReg() == AMDGPU::NoRegister); in selectImageIntrinsic()
1631 const AMDGPU::MIMGG16MappingInfo *G16MappingInfo = in selectImageIntrinsic()
1632 AMDGPU::getMIMGG16MappingInfo(Intr->BaseOpcode); in selectImageIntrinsic()
1642 CPol |= AMDGPU::CPol::GLC; // TODO no-return optimization in selectImageIntrinsic()
1643 if (CPol & ~AMDGPU::CPol::ALL) in selectImageIntrinsic()
1666 if (UseNSA && !STI.hasFeature(AMDGPU::FeatureNSAEncoding)) { in selectImageIntrinsic()
1676 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, in selectImageIntrinsic()
1677 UseNSA ? AMDGPU::MIMGEncGfx11NSA in selectImageIntrinsic()
1678 : AMDGPU::MIMGEncGfx11Default, in selectImageIntrinsic()
1681 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, in selectImageIntrinsic()
1682 UseNSA ? AMDGPU::MIMGEncGfx10NSA in selectImageIntrinsic()
1683 : AMDGPU::MIMGEncGfx10Default, in selectImageIntrinsic()
1687 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx90a, in selectImageIntrinsic()
1698 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx8, in selectImageIntrinsic()
1701 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx6, in selectImageIntrinsic()
1714 Is64 ? &AMDGPU::VReg_128RegClass : &AMDGPU::VReg_64RegClass); in selectImageIntrinsic()
1715 unsigned SubReg = Is64 ? AMDGPU::sub0_sub1 : AMDGPU::sub0; in selectImageIntrinsic()
1719 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), VDataOut) in selectImageIntrinsic()
1751 STI.hasFeature(AMDGPU::FeatureR128A16) ? -1 : 0); in selectImageIntrinsic()
1774 Register Zero = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectImageIntrinsic()
1775 BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::V_MOV_B32_e32), Zero) in selectImageIntrinsic()
1782 BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::REG_SEQUENCE), Tied); in selectImageIntrinsic()
1788 Register Undef = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectImageIntrinsic()
1789 BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::IMPLICIT_DEF), Undef); in selectImageIntrinsic()
1791 BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::REG_SEQUENCE), Tied); in selectImageIntrinsic()
1802 TII.enforceOperandRCAlignment(*MIB, AMDGPU::OpName::vaddr); in selectImageIntrinsic()
1861 unsigned SelectOpcode = Size == 64 ? AMDGPU::S_CSELECT_B64 : in selectG_SELECT()
1862 AMDGPU::S_CSELECT_B32; in selectG_SELECT()
1863 MachineInstr *CopySCC = BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), AMDGPU::SCC) in selectG_SELECT()
1887 BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in selectG_SELECT()
1902 return AMDGPU::sub0; in sizeToSubRegIndex()
1904 return AMDGPU::sub0_sub1; in sizeToSubRegIndex()
1906 return AMDGPU::sub0_sub1_sub2; in sizeToSubRegIndex()
1908 return AMDGPU::sub0_sub1_sub2_sub3; in sizeToSubRegIndex()
1910 return AMDGPU::sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7; in sizeToSubRegIndex()
1913 return AMDGPU::sub0; in sizeToSubRegIndex()
1939 const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID; in selectG_TRUNC()
1963 BuildMI(*MBB, I, DL, TII.get(AMDGPU::COPY), LoReg) in selectG_TRUNC()
1964 .addReg(SrcReg, 0, AMDGPU::sub0); in selectG_TRUNC()
1965 BuildMI(*MBB, I, DL, TII.get(AMDGPU::COPY), HiReg) in selectG_TRUNC()
1966 .addReg(SrcReg, 0, AMDGPU::sub1); in selectG_TRUNC()
1972 BuildMI(*MBB, I, DL, TII.get(AMDGPU::V_MOV_B32_sdwa), DstReg) in selectG_TRUNC()
1976 .addImm(AMDGPU::SDWA::WORD_1) // $dst_sel in selectG_TRUNC()
1977 .addImm(AMDGPU::SDWA::UNUSED_PRESERVE) // $dst_unused in selectG_TRUNC()
1978 .addImm(AMDGPU::SDWA::WORD_0) // $src0_sel in selectG_TRUNC()
1986 BuildMI(*MBB, I, DL, TII.get(AMDGPU::V_LSHLREV_B32_e64), TmpReg0) in selectG_TRUNC()
1990 BuildMI(*MBB, I, DL, TII.get(AMDGPU::S_LSHL_B32), TmpReg0) in selectG_TRUNC()
1995 unsigned MovOpc = IsVALU ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32; in selectG_TRUNC()
1996 unsigned AndOpc = IsVALU ? AMDGPU::V_AND_B32_e64 : AMDGPU::S_AND_B32; in selectG_TRUNC()
1997 unsigned OrOpc = IsVALU ? AMDGPU::V_OR_B32_e64 : AMDGPU::S_OR_B32; in selectG_TRUNC()
2062 bool InReg = I.getOpcode() == AMDGPU::G_SEXT_INREG; in selectG_SZA_EXT()
2063 bool Signed = I.getOpcode() == AMDGPU::G_SEXT || InReg; in selectG_SZA_EXT()
2071 const unsigned SrcSize = I.getOpcode() == AMDGPU::G_SEXT_INREG ? in selectG_SZA_EXT()
2081 if (I.getOpcode() == AMDGPU::G_ANYEXT) { in selectG_SZA_EXT()
2092 BuildMI(MBB, I, DL, TII.get(AMDGPU::IMPLICIT_DEF), UndefReg); in selectG_SZA_EXT()
2093 BuildMI(MBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg) in selectG_SZA_EXT()
2095 .addImm(AMDGPU::sub0) in selectG_SZA_EXT()
2097 .addImm(AMDGPU::sub1); in selectG_SZA_EXT()
2104 if (SrcBank->getID() == AMDGPU::VGPRRegBankID && DstSize <= 32) { in selectG_SZA_EXT()
2111 BuildMI(MBB, I, DL, TII.get(AMDGPU::V_AND_B32_e32), DstReg) in selectG_SZA_EXT()
2118 const unsigned BFE = Signed ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64; in selectG_SZA_EXT()
2128 if (SrcBank->getID() == AMDGPU::SGPRRegBankID && DstSize <= 64) { in selectG_SZA_EXT()
2130 AMDGPU::SReg_64RegClass : AMDGPU::SReg_32RegClass; in selectG_SZA_EXT()
2136 AMDGPU::S_SEXT_I32_I8 : AMDGPU::S_SEXT_I32_I16; in selectG_SZA_EXT()
2140 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_32RegClass, *MRI); in selectG_SZA_EXT()
2143 const unsigned BFE64 = Signed ? AMDGPU::S_BFE_I64 : AMDGPU::S_BFE_U64; in selectG_SZA_EXT()
2144 const unsigned BFE32 = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32; in selectG_SZA_EXT()
2149 Register ExtReg = MRI->createVirtualRegister(&AMDGPU::SReg_64RegClass); in selectG_SZA_EXT()
2150 Register UndefReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_SZA_EXT()
2151 unsigned SubReg = InReg ? AMDGPU::sub0 : 0; in selectG_SZA_EXT()
2153 BuildMI(MBB, I, DL, TII.get(AMDGPU::IMPLICIT_DEF), UndefReg); in selectG_SZA_EXT()
2154 BuildMI(MBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), ExtReg) in selectG_SZA_EXT()
2156 .addImm(AMDGPU::sub0) in selectG_SZA_EXT()
2158 .addImm(AMDGPU::sub1); in selectG_SZA_EXT()
2165 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_64RegClass, *MRI); in selectG_SZA_EXT()
2170 BuildMI(MBB, I, DL, TII.get(AMDGPU::S_AND_B32), DstReg) in selectG_SZA_EXT()
2180 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_32RegClass, *MRI); in selectG_SZA_EXT()
2203 const bool IsSgpr = DstRB->getID() == AMDGPU::SGPRRegBankID; in selectG_CONSTANT()
2206 if (DstRB->getID() == AMDGPU::VCCRegBankID) { in selectG_CONSTANT()
2207 Opcode = STI.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; in selectG_CONSTANT()
2209 Opcode = IsSgpr ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32; in selectG_CONSTANT()
2230 ResInst = BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_MOV_B64), DstReg) in selectG_CONSTANT()
2234 &AMDGPU::SReg_32RegClass : &AMDGPU::VGPR_32RegClass; in selectG_CONSTANT()
2244 ResInst = BuildMI(*BB, &I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg) in selectG_CONSTANT()
2246 .addImm(AMDGPU::sub0) in selectG_CONSTANT()
2248 .addImm(AMDGPU::sub1); in selectG_CONSTANT()
2275 if (DstRB->getID() != AMDGPU::SGPRRegBankID || in selectG_FNEG()
2284 if (!RBI.constrainGenericRegister(Src, AMDGPU::SReg_64RegClass, *MRI) || in selectG_FNEG()
2285 !RBI.constrainGenericRegister(Dst, AMDGPU::SReg_64RegClass, *MRI)) in selectG_FNEG()
2290 Register LoReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FNEG()
2291 Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FNEG()
2292 Register ConstReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FNEG()
2293 Register OpReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FNEG()
2295 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), LoReg) in selectG_FNEG()
2296 .addReg(Src, 0, AMDGPU::sub0); in selectG_FNEG()
2297 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), HiReg) in selectG_FNEG()
2298 .addReg(Src, 0, AMDGPU::sub1); in selectG_FNEG()
2299 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_MOV_B32), ConstReg) in selectG_FNEG()
2303 unsigned Opc = Fabs ? AMDGPU::S_OR_B32 : AMDGPU::S_XOR_B32; in selectG_FNEG()
2307 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::REG_SEQUENCE), Dst) in selectG_FNEG()
2309 .addImm(AMDGPU::sub0) in selectG_FNEG()
2311 .addImm(AMDGPU::sub1); in selectG_FNEG()
2320 if (DstRB->getID() != AMDGPU::SGPRRegBankID || in selectG_FABS()
2327 Register LoReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FABS()
2328 Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FABS()
2329 Register ConstReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FABS()
2330 Register OpReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FABS()
2332 if (!RBI.constrainGenericRegister(Src, AMDGPU::SReg_64RegClass, *MRI) || in selectG_FABS()
2333 !RBI.constrainGenericRegister(Dst, AMDGPU::SReg_64RegClass, *MRI)) in selectG_FABS()
2336 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), LoReg) in selectG_FABS()
2337 .addReg(Src, 0, AMDGPU::sub0); in selectG_FABS()
2338 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), HiReg) in selectG_FABS()
2339 .addReg(Src, 0, AMDGPU::sub1); in selectG_FABS()
2340 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_MOV_B32), ConstReg) in selectG_FABS()
2345 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_AND_B32), OpReg) in selectG_FABS()
2348 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::REG_SEQUENCE), Dst) in selectG_FABS()
2350 .addImm(AMDGPU::sub0) in selectG_FABS()
2352 .addImm(AMDGPU::sub1); in selectG_FABS()
2386 if (OpBank->getID() == AMDGPU::SGPRRegBankID) in getAddrModeInfo()
2397 return RBI.getRegBank(Reg, *MRI, TRI)->getID() == AMDGPU::SGPRRegBankID; in isSGPR()
2438 BuildMI(*BB, &I, I.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), AMDGPU::M0) in initM0()
2463 if (Opcode == AMDGPU::COPY) in isVCmpResult()
2466 if (Opcode == AMDGPU::G_AND || Opcode == AMDGPU::G_OR || in isVCmpResult()
2467 Opcode == AMDGPU::G_XOR) in isVCmpResult()
2474 return Opcode == AMDGPU::G_ICMP || Opcode == AMDGPU::G_FCMP; in isVCmpResult()
2496 CondPhysReg = AMDGPU::SCC; in selectG_BRCOND()
2497 BrOpcode = AMDGPU::S_CBRANCH_SCC1; in selectG_BRCOND()
2498 ConstrainRC = &AMDGPU::SReg_32RegClass; in selectG_BRCOND()
2506 const unsigned Opcode = Is64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32; in selectG_BRCOND()
2507 const Register Exec = Is64 ? AMDGPU::EXEC : AMDGPU::EXEC_LO; in selectG_BRCOND()
2517 BrOpcode = AMDGPU::S_CBRANCH_VCCNZ; in selectG_BRCOND()
2524 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), CondPhysReg) in selectG_BRCOND()
2537 const bool IsVGPR = DstRB->getID() == AMDGPU::VGPRRegBankID; in selectG_GLOBAL_VALUE()
2538 I.setDesc(TII.get(IsVGPR ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32)); in selectG_GLOBAL_VALUE()
2540 I.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true)); in selectG_GLOBAL_VALUE()
2543 DstReg, IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass, *MRI); in selectG_GLOBAL_VALUE()
2558 const bool IsVGPR = DstRB->getID() == AMDGPU::VGPRRegBankID; in selectG_PTRMASK()
2573 auto MIB = BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_AND_B64), DstReg) in selectG_PTRMASK()
2580 unsigned NewOpc = IsVGPR ? AMDGPU::V_AND_B32_e64 : AMDGPU::S_AND_B32; in selectG_PTRMASK()
2582 = IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass; in selectG_PTRMASK()
2609 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), LoReg) in selectG_PTRMASK()
2610 .addReg(SrcReg, 0, AMDGPU::sub0); in selectG_PTRMASK()
2611 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), HiReg) in selectG_PTRMASK()
2612 .addReg(SrcReg, 0, AMDGPU::sub1); in selectG_PTRMASK()
2624 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), MaskLo) in selectG_PTRMASK()
2625 .addReg(MaskReg, 0, AMDGPU::sub0); in selectG_PTRMASK()
2638 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), MaskHi) in selectG_PTRMASK()
2639 .addReg(MaskReg, 0, AMDGPU::sub1); in selectG_PTRMASK()
2645 BuildMI(*BB, &I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg) in selectG_PTRMASK()
2647 .addImm(AMDGPU::sub0) in selectG_PTRMASK()
2649 .addImm(AMDGPU::sub1); in selectG_PTRMASK()
2665 std::tie(IdxBaseReg, Offset) = AMDGPU::getBaseWithConstantOffset(MRI, IdxReg); in computeIndirectRegIndex()
2666 if (IdxBaseReg == AMDGPU::NoRegister) { in computeIndirectRegIndex()
2697 if (IdxRB->getID() != AMDGPU::SGPRRegBankID) in selectG_EXTRACT_VECTOR_ELT()
2708 !RBI.constrainGenericRegister(IdxReg, AMDGPU::SReg_32RegClass, *MRI)) in selectG_EXTRACT_VECTOR_ELT()
2719 if (SrcRB->getID() == AMDGPU::SGPRRegBankID) { in selectG_EXTRACT_VECTOR_ELT()
2723 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectG_EXTRACT_VECTOR_ELT()
2726 unsigned Opc = Is64 ? AMDGPU::S_MOVRELS_B64 : AMDGPU::S_MOVRELS_B32; in selectG_EXTRACT_VECTOR_ELT()
2734 if (SrcRB->getID() != AMDGPU::VGPRRegBankID || DstTy.getSizeInBits() != 32) in selectG_EXTRACT_VECTOR_ELT()
2738 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectG_EXTRACT_VECTOR_ELT()
2740 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::V_MOVRELS_B32_e32), DstReg) in selectG_EXTRACT_VECTOR_ELT()
2779 if (IdxRB->getID() != AMDGPU::SGPRRegBankID) in selectG_INSERT_VECTOR_ELT()
2790 !RBI.constrainGenericRegister(IdxReg, AMDGPU::SReg_32RegClass, *MRI)) in selectG_INSERT_VECTOR_ELT()
2793 if (VecRB->getID() == AMDGPU::VGPRRegBankID && ValSize != 32) in selectG_INSERT_VECTOR_ELT()
2800 const bool IndexMode = VecRB->getID() == AMDGPU::VGPRRegBankID && in selectG_INSERT_VECTOR_ELT()
2807 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectG_INSERT_VECTOR_ELT()
2811 VecSize, ValSize, VecRB->getID() == AMDGPU::SGPRRegBankID); in selectG_INSERT_VECTOR_ELT()
2874 if (!AMDGPU::isLegalVOP3PShuffleMask(ShufMask)) in selectG_SHUFFLE_VECTOR()
2883 const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID; in selectG_SHUFFLE_VECTOR()
2885 AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass; in selectG_SHUFFLE_VECTOR()
2889 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::IMPLICIT_DEF), DstReg); in selectG_SHUFFLE_VECTOR()
2905 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::COPY), DstReg) in selectG_SHUFFLE_VECTOR()
2914 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_LSHRREV_B32_e64), DstReg) in selectG_SHUFFLE_VECTOR()
2918 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::S_LSHR_B32), DstReg) in selectG_SHUFFLE_VECTOR()
2924 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_LSHLREV_B32_e64), DstReg) in selectG_SHUFFLE_VECTOR()
2928 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::S_LSHL_B32), DstReg) in selectG_SHUFFLE_VECTOR()
2937 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_MOV_B32_sdwa), DstReg) in selectG_SHUFFLE_VECTOR()
2941 .addImm(AMDGPU::SDWA::WORD_1) // $dst_sel in selectG_SHUFFLE_VECTOR()
2942 .addImm(AMDGPU::SDWA::UNUSED_PRESERVE) // $dst_unused in selectG_SHUFFLE_VECTOR()
2943 .addImm(AMDGPU::SDWA::WORD_0) // $src0_sel in selectG_SHUFFLE_VECTOR()
2947 Register TmpReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectG_SHUFFLE_VECTOR()
2948 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_AND_B32_e32), TmpReg) in selectG_SHUFFLE_VECTOR()
2951 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_LSHL_OR_B32_e64), DstReg) in selectG_SHUFFLE_VECTOR()
2957 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::S_PACK_LL_B32_B16), DstReg) in selectG_SHUFFLE_VECTOR()
2966 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_MOV_B32_sdwa), DstReg) in selectG_SHUFFLE_VECTOR()
2970 .addImm(AMDGPU::SDWA::WORD_0) // $dst_sel in selectG_SHUFFLE_VECTOR()
2971 .addImm(AMDGPU::SDWA::UNUSED_PRESERVE) // $dst_unused in selectG_SHUFFLE_VECTOR()
2972 .addImm(AMDGPU::SDWA::WORD_1) // $src0_sel in selectG_SHUFFLE_VECTOR()
2976 Register TmpReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectG_SHUFFLE_VECTOR()
2977 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_LSHRREV_B32_e64), TmpReg) in selectG_SHUFFLE_VECTOR()
2980 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_LSHL_OR_B32_e64), DstReg) in selectG_SHUFFLE_VECTOR()
2986 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::S_PACK_HH_B32_B16), DstReg) in selectG_SHUFFLE_VECTOR()
2992 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_ALIGNBIT_B32_e64), DstReg) in selectG_SHUFFLE_VECTOR()
2998 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::S_PACK_HL_B32_B16), DstReg) in selectG_SHUFFLE_VECTOR()
3002 Register TmpReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_SHUFFLE_VECTOR()
3003 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::S_LSHR_B32), TmpReg) in selectG_SHUFFLE_VECTOR()
3006 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::S_PACK_LL_B32_B16), DstReg) in selectG_SHUFFLE_VECTOR()
3022 if (AMDGPU::hasAtomicFaddRtnForTy(STI, DefTy)) in selectAMDGPU_BUFFER_ATOMIC_FADD()
3051 Opcode = HasVIndex ? AMDGPU::BUFFER_ATOMIC_ADD_F32_BOTHEN in selectAMDGPU_BUFFER_ATOMIC_FADD()
3052 : AMDGPU::BUFFER_ATOMIC_ADD_F32_OFFEN; in selectAMDGPU_BUFFER_ATOMIC_FADD()
3054 Opcode = HasVIndex ? AMDGPU::BUFFER_ATOMIC_ADD_F32_IDXEN in selectAMDGPU_BUFFER_ATOMIC_FADD()
3055 : AMDGPU::BUFFER_ATOMIC_ADD_F32_OFFSET; in selectAMDGPU_BUFFER_ATOMIC_FADD()
3060 case AMDGPU::BUFFER_ATOMIC_ADD_F32_BOTHEN: in selectAMDGPU_BUFFER_ATOMIC_FADD()
3061 Opcode = AMDGPU::BUFFER_ATOMIC_PK_ADD_F16_BOTHEN; in selectAMDGPU_BUFFER_ATOMIC_FADD()
3063 case AMDGPU::BUFFER_ATOMIC_ADD_F32_OFFEN: in selectAMDGPU_BUFFER_ATOMIC_FADD()
3064 Opcode = AMDGPU::BUFFER_ATOMIC_PK_ADD_F16_OFFEN; in selectAMDGPU_BUFFER_ATOMIC_FADD()
3066 case AMDGPU::BUFFER_ATOMIC_ADD_F32_IDXEN: in selectAMDGPU_BUFFER_ATOMIC_FADD()
3067 Opcode = AMDGPU::BUFFER_ATOMIC_PK_ADD_F16_IDXEN; in selectAMDGPU_BUFFER_ATOMIC_FADD()
3069 case AMDGPU::BUFFER_ATOMIC_ADD_F32_OFFSET: in selectAMDGPU_BUFFER_ATOMIC_FADD()
3070 Opcode = AMDGPU::BUFFER_ATOMIC_PK_ADD_F16_OFFSET; in selectAMDGPU_BUFFER_ATOMIC_FADD()
3078 if (Opcode == AMDGPU::BUFFER_ATOMIC_ADD_F32_BOTHEN || in selectAMDGPU_BUFFER_ATOMIC_FADD()
3079 Opcode == AMDGPU::BUFFER_ATOMIC_PK_ADD_F16_BOTHEN) { in selectAMDGPU_BUFFER_ATOMIC_FADD()
3081 BuildMI(*MBB, &*I, DL, TII.get(AMDGPU::REG_SEQUENCE), IdxReg) in selectAMDGPU_BUFFER_ATOMIC_FADD()
3083 .addImm(AMDGPU::sub0) in selectAMDGPU_BUFFER_ATOMIC_FADD()
3085 .addImm(AMDGPU::sub1); in selectAMDGPU_BUFFER_ATOMIC_FADD()
3133 AMDGPU::GLOBAL_ATOMIC_PK_ADD_F16 : AMDGPU::GLOBAL_ATOMIC_ADD_F32; in selectGlobalAtomicFadd()
3167 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_BOTHEN in selectBufferLoadLds()
3168 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_IDXEN in selectBufferLoadLds()
3169 : HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFEN in selectBufferLoadLds()
3170 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFSET; in selectBufferLoadLds()
3173 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_BOTHEN in selectBufferLoadLds()
3174 : AMDGPU::BUFFER_LOAD_USHORT_LDS_IDXEN in selectBufferLoadLds()
3175 : HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFEN in selectBufferLoadLds()
3176 : AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFSET; in selectBufferLoadLds()
3179 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_BOTHEN in selectBufferLoadLds()
3180 : AMDGPU::BUFFER_LOAD_DWORD_LDS_IDXEN in selectBufferLoadLds()
3181 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFEN in selectBufferLoadLds()
3182 : AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFSET; in selectBufferLoadLds()
3188 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectBufferLoadLds()
3195 BuildMI(*MBB, &*MIB, DL, TII.get(AMDGPU::REG_SEQUENCE), IdxReg) in selectBufferLoadLds()
3197 .addImm(AMDGPU::sub0) in selectBufferLoadLds()
3199 .addImm(AMDGPU::sub1); in selectBufferLoadLds()
3212 MIB.addImm(Aux & AMDGPU::CPol::ALL); // cpol in selectBufferLoadLds()
3245 if (Def->getOpcode() != AMDGPU::G_MERGE_VALUES) in matchZeroExtendFromS32()
3265 Opc = AMDGPU::GLOBAL_LOAD_LDS_UBYTE; in selectGlobalLoadLds()
3268 Opc = AMDGPU::GLOBAL_LOAD_LDS_USHORT; in selectGlobalLoadLds()
3271 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORD; in selectGlobalLoadLds()
3277 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectGlobalLoadLds()
3288 } else if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) { in selectGlobalLoadLds()
3302 Opc = AMDGPU::getGlobalSaddrOp(Opc); in selectGlobalLoadLds()
3304 VOffset = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectGlobalLoadLds()
3305 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::V_MOV_B32_e32), VOffset) in selectGlobalLoadLds()
3350 Opc = AMDGPU::V_SMFMAC_F32_16X16X32_F16_e64; in selectSMFMACIntrin()
3353 Opc = AMDGPU::V_SMFMAC_F32_32X32X16_F16_e64; in selectSMFMACIntrin()
3356 Opc = AMDGPU::V_SMFMAC_F32_16X16X32_BF16_e64; in selectSMFMACIntrin()
3359 Opc = AMDGPU::V_SMFMAC_F32_32X32X16_BF16_e64; in selectSMFMACIntrin()
3362 Opc = AMDGPU::V_SMFMAC_I32_16X16X64_I8_e64; in selectSMFMACIntrin()
3365 Opc = AMDGPU::V_SMFMAC_I32_32X32X32_I8_e64; in selectSMFMACIntrin()
3368 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_BF8_BF8_e64; in selectSMFMACIntrin()
3371 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_BF8_FP8_e64; in selectSMFMACIntrin()
3374 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_FP8_BF8_e64; in selectSMFMACIntrin()
3377 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_FP8_FP8_e64; in selectSMFMACIntrin()
3380 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_BF8_BF8_e64; in selectSMFMACIntrin()
3383 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_BF8_FP8_e64; in selectSMFMACIntrin()
3386 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_FP8_BF8_e64; in selectSMFMACIntrin()
3389 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_FP8_FP8_e64; in selectSMFMACIntrin()
3409 const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID; in selectWaveAddress()
3414 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_LSHRREV_B32_e64), DstReg) in selectWaveAddress()
3418 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::S_LSHR_B32), DstReg) in selectWaveAddress()
3424 IsVALU ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass; in selectWaveAddress()
3459 case AMDGPU::G_AMDGPU_MAD_U64_U32: in select()
3460 case AMDGPU::G_AMDGPU_MAD_I64_I32: in select()
3517 case AMDGPU::G_AMDGPU_ATOMIC_INC: in select()
3518 case AMDGPU::G_AMDGPU_ATOMIC_DEC: in select()
3519 case AMDGPU::G_AMDGPU_ATOMIC_FMIN: in select()
3520 case AMDGPU::G_AMDGPU_ATOMIC_FMAX: in select()
3545 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD: in select()
3546 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16: in select()
3547 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE: in select()
3548 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16: { in select()
3549 const AMDGPU::ImageDimIntrinsicInfo *Intr in select()
3550 = AMDGPU::getImageDimIntrinsicInfo(I.getIntrinsicID()); in select()
3554 case AMDGPU::G_AMDGPU_INTRIN_BVH_INTERSECT_RAY: in select()
3556 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FADD: in select()
3558 case AMDGPU::G_SBFX: in select()
3559 case AMDGPU::G_UBFX: in select()
3561 case AMDGPU::G_SI_CALL: in select()
3562 I.setDesc(TII.get(AMDGPU::SI_CALL)); in select()
3564 case AMDGPU::G_AMDGPU_WAVE_ADDRESS: in select()
3587 if (MI && MI->getOpcode() == AMDGPU::G_FNEG) { in selectVOP3ModsImpl()
3593 if (AllowAbs && MI && MI->getOpcode() == AMDGPU::G_FABS) { in selectVOP3ModsImpl()
3602 RBI.getRegBank(Src, *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID) { in selectVOP3ModsImpl()
3610 TII.get(AMDGPU::COPY), VGPRSrc) in selectVOP3ModsImpl()
3693 if (Def && (Def->getOpcode() == AMDGPU::G_FNEG || in selectVOP3NoMods()
3694 Def->getOpcode() == AMDGPU::G_FABS)) in selectVOP3NoMods()
3707 if (MI && MI->getOpcode() == AMDGPU::G_FNEG && in selectVOP3PModsImpl()
3854 AMDGPU::getSMRDEncodedOffset(STI, GEPI.Imm, false); in selectSmrdOffset()
3887 *SOffset = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectSmrdOffset()
3888 BuildMI(*MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), *SOffset) in selectSmrdOffset()
3926 AMDGPU::getSMRDEncodedLiteralOffset32(STI, GEPInfo.Imm); in selectSmrdImm32()
4046 MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectGlobalSAddr()
4048 BuildMI(*MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::V_MOV_B32_e32), in selectGlobalSAddr()
4070 if (STI.getConstantBusLimit(AMDGPU::V_ADD_U32_e64) > NumLiterals) in selectGlobalSAddr()
4078 if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) { in selectGlobalSAddr()
4104 if (AddrDef->MI->getOpcode() == AMDGPU::G_IMPLICIT_DEF || in selectGlobalSAddr()
4105 AddrDef->MI->getOpcode() == AMDGPU::G_CONSTANT || !isSGPR(AddrDef->Reg)) in selectGlobalSAddr()
4112 Register VOffset = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectGlobalSAddr()
4114 BuildMI(*MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::V_MOV_B32_e32), VOffset) in selectGlobalSAddr()
4143 if (AddrDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX) { in selectScratchSAddr()
4153 if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) { in selectScratchSAddr()
4159 if (LHSDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX && in selectScratchSAddr()
4165 SAddr = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectScratchSAddr()
4167 BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_ADD_I32), SAddr) in selectScratchSAddr()
4218 if (AddrDef->MI->getOpcode() != AMDGPU::G_PTR_ADD) in selectScratchSVAddr()
4222 if (RBI.getRegBank(RHS, *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID) in selectScratchSVAddr()
4231 if (LHSDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX) { in selectScratchSVAddr()
4260 Register HighBits = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectMUBUFScratchOffen()
4264 BuildMI(*MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::V_MOV_B32_e32), in selectMUBUFScratchOffen()
4299 if (PtrBaseDef->getOpcode() == AMDGPU::G_FRAME_INDEX) in selectMUBUFScratchOffen()
4305 } else if (RootDef->getOpcode() == AMDGPU::G_FRAME_INDEX) { in selectMUBUFScratchOffen()
4376 return Def->getOpcode() == AMDGPU::G_AMDGPU_WAVE_ADDRESS in getWaveAddress()
4456 } else if (RootDef->getOpcode() == AMDGPU::G_SUB) { in selectDS1Addr1OffsetImpl()
4523 } else if (RootDef->getOpcode() == AMDGPU::G_SUB) { in selectDSReadWrite2Impl()
4562 Register RSrc2 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); in buildRSRC()
4563 Register RSrc3 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); in buildRSRC()
4564 Register RSrcHi = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in buildRSRC()
4565 Register RSrc = MRI.createVirtualRegister(&AMDGPU::SGPR_128RegClass); in buildRSRC()
4567 B.buildInstr(AMDGPU::S_MOV_B32) in buildRSRC()
4570 B.buildInstr(AMDGPU::S_MOV_B32) in buildRSRC()
4577 B.buildInstr(AMDGPU::REG_SEQUENCE) in buildRSRC()
4580 .addImm(AMDGPU::sub0) in buildRSRC()
4582 .addImm(AMDGPU::sub1); in buildRSRC()
4586 RSrcLo = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in buildRSRC()
4587 B.buildInstr(AMDGPU::S_MOV_B64) in buildRSRC()
4592 B.buildInstr(AMDGPU::REG_SEQUENCE) in buildRSRC()
4595 .addImm(AMDGPU::sub0_sub1) in buildRSRC()
4597 .addImm(AMDGPU::sub2_sub3); in buildRSRC()
4659 return N0Bank->getID() == AMDGPU::VGPRRegBankID; in shouldUseAddr64()
4671 SOffset = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in splitIllegalMUBUFOffset()
4672 B.buildInstr(AMDGPU::S_MOV_B32) in splitIllegalMUBUFOffset()
4699 if (RBI.getRegBank(N2, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) { in selectMUBUFAddr64Impl()
4701 if (RBI.getRegBank(N3, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) { in selectMUBUFAddr64Impl()
4714 } else if (RBI.getRegBank(N0, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) { in selectMUBUFAddr64Impl()
4843 MIB.addImm(AMDGPU::CPol::GLC); // cpol in selectMUBUFAddr64Atomic()
4868 [=](MachineInstrBuilder &MIB) { MIB.addImm(AMDGPU::CPol::GLC); } // cpol in selectMUBUFOffsetAtomic()
4889 AMDGPU::getSMRDEncodedOffset(STI, *OffsetVal, true); in selectSMRDBufferImm()
4905 = AMDGPU::getSMRDEncodedLiteralOffset32(STI, *OffsetVal); in selectSMRDBufferImm32()
4962 MIB.addImm(MI.getOperand(OpIdx).getImm() & AMDGPU::CPol::ALL); in renderExtractCPol()
4976 MIB.addImm(MI.getOperand(OpIdx).getImm() | AMDGPU::CPol::GLC); in renderSetGLC()
4986 return AMDGPU::isInlinableLiteral16(Imm, STI.hasInv2PiInlineImm()); in isInlineImmediate16()
4990 return AMDGPU::isInlinableLiteral32(Imm, STI.hasInv2PiInlineImm()); in isInlineImmediate32()
4994 return AMDGPU::isInlinableLiteral64(Imm, STI.hasInv2PiInlineImm()); in isInlineImmediate64()