Lines Matching refs:AMDGPU
88 return RB->getID() == AMDGPU::VCCRegBankID; in isVCC()
95 MI.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true)); in constrainCopyLikeIntrin()
126 if (SrcReg == AMDGPU::SCC) { in selectCOPY()
146 STI.isWave64() ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32; in selectCOPY()
157 TRI.isSGPRClass(SrcRC) ? AMDGPU::S_AND_B32 : AMDGPU::V_AND_B32_e32; in selectCOPY()
161 BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_CMP_NE_U32_e64), DstReg) in selectCOPY()
243 BuildMI(*BB, MI, MI->getDebugLoc(), TII.get(AMDGPU::COPY), DstReg) in getSubOperand64()
259 case AMDGPU::sub0: in getSubOperand64()
261 case AMDGPU::sub1: in getSubOperand64()
268 case AMDGPU::G_AND: in getLogicalBitOpcode()
269 return Is64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32; in getLogicalBitOpcode()
270 case AMDGPU::G_OR: in getLogicalBitOpcode()
271 return Is64 ? AMDGPU::S_OR_B64 : AMDGPU::S_OR_B32; in getLogicalBitOpcode()
272 case AMDGPU::G_XOR: in getLogicalBitOpcode()
273 return Is64 ? AMDGPU::S_XOR_B64 : AMDGPU::S_XOR_B32; in getLogicalBitOpcode()
284 if (DstRB->getID() != AMDGPU::SGPRRegBankID && in selectG_AND_OR_XOR()
285 DstRB->getID() != AMDGPU::VCCRegBankID) in selectG_AND_OR_XOR()
288 bool Is64 = Size > 32 || (DstRB->getID() == AMDGPU::VCCRegBankID && in selectG_AND_OR_XOR()
293 I.addOperand(MachineOperand::CreateReg(AMDGPU::SCC, true, // isDef in selectG_AND_OR_XOR()
311 const bool IsSALU = DstRB->getID() == AMDGPU::SGPRRegBankID; in selectG_ADD_SUB()
316 const unsigned Opc = Sub ? AMDGPU::S_SUB_U32 : AMDGPU::S_ADD_U32; in selectG_ADD_SUB()
326 const unsigned Opc = Sub ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_ADD_U32_e64; in selectG_ADD_SUB()
329 I.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true)); in selectG_ADD_SUB()
333 const unsigned Opc = Sub ? AMDGPU::V_SUB_CO_U32_e64 : AMDGPU::V_ADD_CO_U32_e64; in selectG_ADD_SUB()
349 = IsSALU ? AMDGPU::SReg_64_XEXECRegClass : AMDGPU::VReg_64RegClass; in selectG_ADD_SUB()
351 = IsSALU ? AMDGPU::SReg_32RegClass : AMDGPU::VGPR_32RegClass; in selectG_ADD_SUB()
353 MachineOperand Lo1(getSubOperand64(I.getOperand(1), HalfRC, AMDGPU::sub0)); in selectG_ADD_SUB()
354 MachineOperand Lo2(getSubOperand64(I.getOperand(2), HalfRC, AMDGPU::sub0)); in selectG_ADD_SUB()
355 MachineOperand Hi1(getSubOperand64(I.getOperand(1), HalfRC, AMDGPU::sub1)); in selectG_ADD_SUB()
356 MachineOperand Hi2(getSubOperand64(I.getOperand(2), HalfRC, AMDGPU::sub1)); in selectG_ADD_SUB()
362 BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_ADD_U32), DstLo) in selectG_ADD_SUB()
365 BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_ADDC_U32), DstHi) in selectG_ADD_SUB()
371 BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_ADD_CO_U32_e64), DstLo) in selectG_ADD_SUB()
376 MachineInstr *Addc = BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_ADDC_U32_e64), DstHi) in selectG_ADD_SUB()
387 BuildMI(*BB, &I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg) in selectG_ADD_SUB()
389 .addImm(AMDGPU::sub0) in selectG_ADD_SUB()
391 .addImm(AMDGPU::sub1); in selectG_ADD_SUB()
408 const bool IsAdd = I.getOpcode() == AMDGPU::G_UADDO || in selectG_UADDO_USUBO_UADDE_USUBE()
409 I.getOpcode() == AMDGPU::G_UADDE; in selectG_UADDO_USUBO_UADDE_USUBE()
410 const bool HasCarryIn = I.getOpcode() == AMDGPU::G_UADDE || in selectG_UADDO_USUBO_UADDE_USUBE()
411 I.getOpcode() == AMDGPU::G_USUBE; in selectG_UADDO_USUBO_UADDE_USUBE()
415 IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64; in selectG_UADDO_USUBO_UADDE_USUBE()
416 unsigned CarryOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64; in selectG_UADDO_USUBO_UADDE_USUBE()
418 I.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true)); in selectG_UADDO_USUBO_UADDE_USUBE()
427 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), AMDGPU::SCC) in selectG_UADDO_USUBO_UADDE_USUBE()
431 unsigned NoCarryOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32; in selectG_UADDO_USUBO_UADDE_USUBE()
432 unsigned CarryOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32; in selectG_UADDO_USUBO_UADDE_USUBE()
437 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), Dst1Reg) in selectG_UADDO_USUBO_UADDE_USUBE()
438 .addReg(AMDGPU::SCC); in selectG_UADDO_USUBO_UADDE_USUBE()
441 MRI->setRegClass(Dst1Reg, &AMDGPU::SReg_32RegClass); in selectG_UADDO_USUBO_UADDE_USUBE()
443 if (!RBI.constrainGenericRegister(Dst0Reg, AMDGPU::SReg_32RegClass, *MRI) || in selectG_UADDO_USUBO_UADDE_USUBE()
444 !RBI.constrainGenericRegister(Src0Reg, AMDGPU::SReg_32RegClass, *MRI) || in selectG_UADDO_USUBO_UADDE_USUBE()
445 !RBI.constrainGenericRegister(Src1Reg, AMDGPU::SReg_32RegClass, *MRI)) in selectG_UADDO_USUBO_UADDE_USUBE()
450 AMDGPU::SReg_32RegClass, *MRI)) in selectG_UADDO_USUBO_UADDE_USUBE()
600 if (DstBank->getID() != AMDGPU::SGPRRegBankID) in selectG_BUILD_VECTOR_TRUNC()
622 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_MOV_B32), Dst) in selectG_BUILD_VECTOR_TRUNC()
625 return RBI.constrainGenericRegister(Dst, AMDGPU::SReg_32RegClass, *MRI); in selectG_BUILD_VECTOR_TRUNC()
632 if (Src1Def && Src1Def->getOpcode() == AMDGPU::G_IMPLICIT_DEF) { in selectG_BUILD_VECTOR_TRUNC()
633 MI.setDesc(TII.get(AMDGPU::COPY)); in selectG_BUILD_VECTOR_TRUNC()
635 return RBI.constrainGenericRegister(Dst, AMDGPU::SReg_32RegClass, *MRI) && in selectG_BUILD_VECTOR_TRUNC()
636 RBI.constrainGenericRegister(Src0, AMDGPU::SReg_32RegClass, *MRI); in selectG_BUILD_VECTOR_TRUNC()
658 unsigned Opc = AMDGPU::S_PACK_LL_B32_B16; in selectG_BUILD_VECTOR_TRUNC()
660 Opc = AMDGPU::S_PACK_HH_B32_B16; in selectG_BUILD_VECTOR_TRUNC()
664 Opc = AMDGPU::S_PACK_LH_B32_B16; in selectG_BUILD_VECTOR_TRUNC()
668 auto MIB = BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_LSHR_B32), Dst) in selectG_BUILD_VECTOR_TRUNC()
721 if (SubReg == AMDGPU::NoSubRegister) in selectG_INSERT()
764 assert(RBI.getRegBank(DstReg, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID && in selectG_SBFX_UBFX()
773 unsigned Opc = IsSigned ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64; in selectG_SBFX_UBFX()
789 if (!RBI.constrainGenericRegister(M0Val, AMDGPU::SReg_32RegClass, *MRI) || in selectInterpP1F16()
790 !RBI.constrainGenericRegister(Dst, AMDGPU::VGPR_32RegClass, *MRI) || in selectInterpP1F16()
791 !RBI.constrainGenericRegister(Src0, AMDGPU::VGPR_32RegClass, *MRI)) in selectInterpP1F16()
801 Register InterpMov = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectInterpP1F16()
805 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectInterpP1F16()
807 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::V_INTERP_MOV_F32), InterpMov) in selectInterpP1F16()
812 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::V_INTERP_P1LV_F16), Dst) in selectInterpP1F16()
834 if (STI.getConstantBusLimit(AMDGPU::V_WRITELANE_B32) > 1) in selectWritelane()
844 auto MIB = BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::V_WRITELANE_B32), VDst); in selectWritelane()
860 if (ConstVal && AMDGPU::isInlinableLiteral32(ConstVal->Value.getSExtValue(), in selectWritelane()
870 RBI.constrainGenericRegister(LaneSelect, AMDGPU::SReg_32_XM0RegClass, *MRI); in selectWritelane()
872 BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectWritelane()
874 MIB.addReg(AMDGPU::M0); in selectWritelane()
893 Opc = AMDGPU::V_DIV_SCALE_F32_e64; in selectDivScale()
895 Opc = AMDGPU::V_DIV_SCALE_F64_e64; in selectDivScale()
933 BuildMI(*BB, &I, I.getDebugLoc(), TII.get(AMDGPU::SI_IF_BREAK)) in selectG_INTRINSIC()
952 return constrainCopyLikeIntrin(I, AMDGPU::WQM); in selectG_INTRINSIC()
954 return constrainCopyLikeIntrin(I, AMDGPU::SOFT_WQM); in selectG_INTRINSIC()
957 return constrainCopyLikeIntrin(I, AMDGPU::STRICT_WWM); in selectG_INTRINSIC()
959 return constrainCopyLikeIntrin(I, AMDGPU::STRICT_WQM); in selectG_INTRINSIC()
986 return Size == 32 ? AMDGPU::V_CMP_NE_U32_e64 : AMDGPU::V_CMP_NE_U64_e64; in getV_CMPOpcode()
988 return Size == 32 ? AMDGPU::V_CMP_EQ_U32_e64 : AMDGPU::V_CMP_EQ_U64_e64; in getV_CMPOpcode()
990 return Size == 32 ? AMDGPU::V_CMP_GT_I32_e64 : AMDGPU::V_CMP_GT_I64_e64; in getV_CMPOpcode()
992 return Size == 32 ? AMDGPU::V_CMP_GE_I32_e64 : AMDGPU::V_CMP_GE_I64_e64; in getV_CMPOpcode()
994 return Size == 32 ? AMDGPU::V_CMP_LT_I32_e64 : AMDGPU::V_CMP_LT_I64_e64; in getV_CMPOpcode()
996 return Size == 32 ? AMDGPU::V_CMP_LE_I32_e64 : AMDGPU::V_CMP_LE_I64_e64; in getV_CMPOpcode()
998 return Size == 32 ? AMDGPU::V_CMP_GT_U32_e64 : AMDGPU::V_CMP_GT_U64_e64; in getV_CMPOpcode()
1000 return Size == 32 ? AMDGPU::V_CMP_GE_U32_e64 : AMDGPU::V_CMP_GE_U64_e64; in getV_CMPOpcode()
1002 return Size == 32 ? AMDGPU::V_CMP_LT_U32_e64 : AMDGPU::V_CMP_LT_U64_e64; in getV_CMPOpcode()
1004 return Size == 32 ? AMDGPU::V_CMP_LE_U32_e64 : AMDGPU::V_CMP_LE_U64_e64; in getV_CMPOpcode()
1016 return AMDGPU::S_CMP_LG_U64; in getS_CMPOpcode()
1018 return AMDGPU::S_CMP_EQ_U64; in getS_CMPOpcode()
1029 return AMDGPU::S_CMP_LG_U32; in getS_CMPOpcode()
1031 return AMDGPU::S_CMP_EQ_U32; in getS_CMPOpcode()
1033 return AMDGPU::S_CMP_GT_I32; in getS_CMPOpcode()
1035 return AMDGPU::S_CMP_GE_I32; in getS_CMPOpcode()
1037 return AMDGPU::S_CMP_LT_I32; in getS_CMPOpcode()
1039 return AMDGPU::S_CMP_LE_I32; in getS_CMPOpcode()
1041 return AMDGPU::S_CMP_GT_U32; in getS_CMPOpcode()
1043 return AMDGPU::S_CMP_GE_U32; in getS_CMPOpcode()
1045 return AMDGPU::S_CMP_LT_U32; in getS_CMPOpcode()
1047 return AMDGPU::S_CMP_LE_U32; in getS_CMPOpcode()
1070 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), CCReg) in selectG_ICMP()
1071 .addReg(AMDGPU::SCC); in selectG_ICMP()
1074 RBI.constrainGenericRegister(CCReg, AMDGPU::SReg_32RegClass, *MRI); in selectG_ICMP()
1138 unsigned Opcode = Is64 ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32; in selectBallot()
1141 Register SrcReg = Is64 ? AMDGPU::EXEC : AMDGPU::EXEC_LO; in selectBallot()
1142 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), DstReg).addReg(SrcReg); in selectBallot()
1147 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), DstReg).addReg(SrcReg); in selectBallot()
1162 const bool IsVALU = DstBank->getID() == AMDGPU::VGPRRegBankID; in selectRelocConstant()
1172 TII.get(IsVALU ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32), DstReg) in selectRelocConstant()
1184 unsigned Mov = DstRB->getID() == AMDGPU::SGPRRegBankID ? in selectGroupStaticSize()
1185 AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32; in selectGroupStaticSize()
1217 if (!RC->hasSubClassEq(&AMDGPU::SGPR_64RegClass) || in selectReturnAddress()
1224 BuildMI(*MBB, &I, DL, TII.get(AMDGPU::S_MOV_B64), DstReg) in selectReturnAddress()
1237 AMDGPU::SReg_64RegClass); in selectReturnAddress()
1238 BuildMI(*MBB, &I, DL, TII.get(AMDGPU::COPY), DstReg) in selectReturnAddress()
1248 BuildMI(*BB, &MI, MI.getDebugLoc(), TII.get(AMDGPU::SI_END_CF)) in selectEndCfIntrinsic()
1302 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectDSOrderedIntrinsic()
1308 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::DS_ORDERED_COUNT), DstReg) in selectDSOrderedIntrinsic()
1313 if (!RBI.constrainGenericRegister(M0Val, AMDGPU::SReg_32RegClass, *MRI)) in selectDSOrderedIntrinsic()
1324 return AMDGPU::DS_GWS_INIT; in gwsIntrinToOpcode()
1326 return AMDGPU::DS_GWS_BARRIER; in gwsIntrinToOpcode()
1328 return AMDGPU::DS_GWS_SEMA_V; in gwsIntrinToOpcode()
1330 return AMDGPU::DS_GWS_SEMA_BR; in gwsIntrinToOpcode()
1332 return AMDGPU::DS_GWS_SEMA_P; in gwsIntrinToOpcode()
1334 return AMDGPU::DS_GWS_SEMA_RELEASE_ALL; in gwsIntrinToOpcode()
1352 if (OffsetRB->getID() != AMDGPU::SGPRRegBankID) in selectDSGWSIntrinsic()
1368 if (OffsetDef->getOpcode() == AMDGPU::V_READFIRSTLANE_B32) { in selectDSGWSIntrinsic()
1374 if (OffsetDef->getOpcode() == AMDGPU::G_CONSTANT) { in selectDSGWSIntrinsic()
1381 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::S_MOV_B32), AMDGPU::M0) in selectDSGWSIntrinsic()
1385 AMDGPU::getBaseWithConstantOffset(*MRI, BaseOffset); in selectDSGWSIntrinsic()
1390 if (!RBI.constrainGenericRegister(BaseOffset, AMDGPU::VGPR_32RegClass, *MRI)) in selectDSGWSIntrinsic()
1397 AMDGPU::SReg_32RegClass, *MRI)) in selectDSGWSIntrinsic()
1401 Register M0Base = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectDSGWSIntrinsic()
1402 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::S_LSHL_B32), M0Base) in selectDSGWSIntrinsic()
1406 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectDSGWSIntrinsic()
1420 Register Undef = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectDSGWSIntrinsic()
1421 BuildMI(*MBB, &*MIB, DL, TII.get(AMDGPU::IMPLICIT_DEF), Undef); in selectDSGWSIntrinsic()
1423 MRI->createVirtualRegister(&AMDGPU::VReg_64_Align2RegClass); in selectDSGWSIntrinsic()
1424 BuildMI(*MBB, &*MIB, DL, TII.get(AMDGPU::REG_SEQUENCE), NewVR) in selectDSGWSIntrinsic()
1426 .addImm(AMDGPU::sub0) in selectDSGWSIntrinsic()
1428 .addImm(AMDGPU::sub1); in selectDSGWSIntrinsic()
1429 MIB.addReg(NewVR, 0, AMDGPU::sub0); in selectDSGWSIntrinsic()
1435 if (!RBI.constrainGenericRegister(VSrc, AMDGPU::VGPR_32RegClass, *MRI)) in selectDSGWSIntrinsic()
1463 const unsigned Opc = IsAppend ? AMDGPU::DS_APPEND : AMDGPU::DS_CONSUME; in selectDSAppendConsume()
1465 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectDSAppendConsume()
1467 if (!RBI.constrainGenericRegister(PtrBase, AMDGPU::SReg_32RegClass, *MRI)) in selectDSAppendConsume()
1484 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::WAVE_BARRIER)); in selectSBarrier()
1506 MachineInstr &MI, const AMDGPU::ImageDimIntrinsicInfo *Intr) const { in selectImageIntrinsic()
1510 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode = in selectImageIntrinsic()
1511 AMDGPU::getMIMGBaseOpcodeInfo(Intr->BaseOpcode); in selectImageIntrinsic()
1513 const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfo(Intr->Dim); in selectImageIntrinsic()
1514 const AMDGPU::MIMGLZMappingInfo *LZMappingInfo = in selectImageIntrinsic()
1515 AMDGPU::getMIMGLZMappingInfo(Intr->BaseOpcode); in selectImageIntrinsic()
1516 const AMDGPU::MIMGMIPMappingInfo *MIPMappingInfo = in selectImageIntrinsic()
1517 AMDGPU::getMIMGMIPMappingInfo(Intr->BaseOpcode); in selectImageIntrinsic()
1519 const bool IsGFX10Plus = AMDGPU::isGFX10Plus(STI); in selectImageIntrinsic()
1563 assert(MI.getOperand(3).getReg() == AMDGPU::NoRegister); in selectImageIntrinsic()
1621 const AMDGPU::MIMGG16MappingInfo *G16MappingInfo = in selectImageIntrinsic()
1622 AMDGPU::getMIMGG16MappingInfo(Intr->BaseOpcode); in selectImageIntrinsic()
1632 CPol |= AMDGPU::CPol::GLC; // TODO no-return optimization in selectImageIntrinsic()
1633 if (CPol & ~AMDGPU::CPol::ALL) in selectImageIntrinsic()
1656 if (UseNSA && !STI.hasFeature(AMDGPU::FeatureNSAEncoding)) { in selectImageIntrinsic()
1666 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, in selectImageIntrinsic()
1667 UseNSA ? AMDGPU::MIMGEncGfx10NSA in selectImageIntrinsic()
1668 : AMDGPU::MIMGEncGfx10Default, in selectImageIntrinsic()
1672 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx8, in selectImageIntrinsic()
1675 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx6, in selectImageIntrinsic()
1688 Is64 ? &AMDGPU::VReg_128RegClass : &AMDGPU::VReg_64RegClass); in selectImageIntrinsic()
1689 unsigned SubReg = Is64 ? AMDGPU::sub0_sub1 : AMDGPU::sub0; in selectImageIntrinsic()
1693 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), VDataOut) in selectImageIntrinsic()
1725 STI.hasFeature(AMDGPU::FeatureR128A16) ? -1 : 0); in selectImageIntrinsic()
1742 Register Zero = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectImageIntrinsic()
1743 BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::V_MOV_B32_e32), Zero) in selectImageIntrinsic()
1750 BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::REG_SEQUENCE), Tied); in selectImageIntrinsic()
1756 Register Undef = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectImageIntrinsic()
1757 BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::IMPLICIT_DEF), Undef); in selectImageIntrinsic()
1759 BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::REG_SEQUENCE), Tied); in selectImageIntrinsic()
1815 unsigned SelectOpcode = Size == 64 ? AMDGPU::S_CSELECT_B64 : in selectG_SELECT()
1816 AMDGPU::S_CSELECT_B32; in selectG_SELECT()
1817 MachineInstr *CopySCC = BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), AMDGPU::SCC) in selectG_SELECT()
1840 BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in selectG_SELECT()
1855 return AMDGPU::sub0; in sizeToSubRegIndex()
1857 return AMDGPU::sub0_sub1; in sizeToSubRegIndex()
1859 return AMDGPU::sub0_sub1_sub2; in sizeToSubRegIndex()
1861 return AMDGPU::sub0_sub1_sub2_sub3; in sizeToSubRegIndex()
1863 return AMDGPU::sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7; in sizeToSubRegIndex()
1866 return AMDGPU::sub0; in sizeToSubRegIndex()
1892 const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID; in selectG_TRUNC()
1916 BuildMI(*MBB, I, DL, TII.get(AMDGPU::COPY), LoReg) in selectG_TRUNC()
1917 .addReg(SrcReg, 0, AMDGPU::sub0); in selectG_TRUNC()
1918 BuildMI(*MBB, I, DL, TII.get(AMDGPU::COPY), HiReg) in selectG_TRUNC()
1919 .addReg(SrcReg, 0, AMDGPU::sub1); in selectG_TRUNC()
1925 BuildMI(*MBB, I, DL, TII.get(AMDGPU::V_MOV_B32_sdwa), DstReg) in selectG_TRUNC()
1929 .addImm(AMDGPU::SDWA::WORD_1) // $dst_sel in selectG_TRUNC()
1930 .addImm(AMDGPU::SDWA::UNUSED_PRESERVE) // $dst_unused in selectG_TRUNC()
1931 .addImm(AMDGPU::SDWA::WORD_0) // $src0_sel in selectG_TRUNC()
1939 BuildMI(*MBB, I, DL, TII.get(AMDGPU::V_LSHLREV_B32_e64), TmpReg0) in selectG_TRUNC()
1943 BuildMI(*MBB, I, DL, TII.get(AMDGPU::S_LSHL_B32), TmpReg0) in selectG_TRUNC()
1948 unsigned MovOpc = IsVALU ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32; in selectG_TRUNC()
1949 unsigned AndOpc = IsVALU ? AMDGPU::V_AND_B32_e64 : AMDGPU::S_AND_B32; in selectG_TRUNC()
1950 unsigned OrOpc = IsVALU ? AMDGPU::V_OR_B32_e64 : AMDGPU::S_OR_B32; in selectG_TRUNC()
2015 bool InReg = I.getOpcode() == AMDGPU::G_SEXT_INREG; in selectG_SZA_EXT()
2016 bool Signed = I.getOpcode() == AMDGPU::G_SEXT || InReg; in selectG_SZA_EXT()
2024 const unsigned SrcSize = I.getOpcode() == AMDGPU::G_SEXT_INREG ? in selectG_SZA_EXT()
2034 if (I.getOpcode() == AMDGPU::G_ANYEXT) { in selectG_SZA_EXT()
2045 BuildMI(MBB, I, DL, TII.get(AMDGPU::IMPLICIT_DEF), UndefReg); in selectG_SZA_EXT()
2046 BuildMI(MBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg) in selectG_SZA_EXT()
2048 .addImm(AMDGPU::sub0) in selectG_SZA_EXT()
2050 .addImm(AMDGPU::sub1); in selectG_SZA_EXT()
2057 if (SrcBank->getID() == AMDGPU::VGPRRegBankID && DstSize <= 32) { in selectG_SZA_EXT()
2064 BuildMI(MBB, I, DL, TII.get(AMDGPU::V_AND_B32_e32), DstReg) in selectG_SZA_EXT()
2071 const unsigned BFE = Signed ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64; in selectG_SZA_EXT()
2081 if (SrcBank->getID() == AMDGPU::SGPRRegBankID && DstSize <= 64) { in selectG_SZA_EXT()
2083 AMDGPU::SReg_64RegClass : AMDGPU::SReg_32RegClass; in selectG_SZA_EXT()
2089 AMDGPU::S_SEXT_I32_I8 : AMDGPU::S_SEXT_I32_I16; in selectG_SZA_EXT()
2093 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_32RegClass, *MRI); in selectG_SZA_EXT()
2096 const unsigned BFE64 = Signed ? AMDGPU::S_BFE_I64 : AMDGPU::S_BFE_U64; in selectG_SZA_EXT()
2097 const unsigned BFE32 = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32; in selectG_SZA_EXT()
2102 Register ExtReg = MRI->createVirtualRegister(&AMDGPU::SReg_64RegClass); in selectG_SZA_EXT()
2103 Register UndefReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_SZA_EXT()
2104 unsigned SubReg = InReg ? AMDGPU::sub0 : 0; in selectG_SZA_EXT()
2106 BuildMI(MBB, I, DL, TII.get(AMDGPU::IMPLICIT_DEF), UndefReg); in selectG_SZA_EXT()
2107 BuildMI(MBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), ExtReg) in selectG_SZA_EXT()
2109 .addImm(AMDGPU::sub0) in selectG_SZA_EXT()
2111 .addImm(AMDGPU::sub1); in selectG_SZA_EXT()
2118 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_64RegClass, *MRI); in selectG_SZA_EXT()
2123 BuildMI(MBB, I, DL, TII.get(AMDGPU::S_AND_B32), DstReg) in selectG_SZA_EXT()
2133 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_32RegClass, *MRI); in selectG_SZA_EXT()
2156 const bool IsSgpr = DstRB->getID() == AMDGPU::SGPRRegBankID; in selectG_CONSTANT()
2159 if (DstRB->getID() == AMDGPU::VCCRegBankID) { in selectG_CONSTANT()
2160 Opcode = STI.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; in selectG_CONSTANT()
2162 Opcode = IsSgpr ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32; in selectG_CONSTANT()
2183 ResInst = BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_MOV_B64), DstReg) in selectG_CONSTANT()
2187 &AMDGPU::SReg_32RegClass : &AMDGPU::VGPR_32RegClass; in selectG_CONSTANT()
2197 ResInst = BuildMI(*BB, &I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg) in selectG_CONSTANT()
2199 .addImm(AMDGPU::sub0) in selectG_CONSTANT()
2201 .addImm(AMDGPU::sub1); in selectG_CONSTANT()
2228 if (DstRB->getID() != AMDGPU::SGPRRegBankID || in selectG_FNEG()
2237 if (!RBI.constrainGenericRegister(Src, AMDGPU::SReg_64RegClass, *MRI) || in selectG_FNEG()
2238 !RBI.constrainGenericRegister(Dst, AMDGPU::SReg_64RegClass, *MRI)) in selectG_FNEG()
2243 Register LoReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FNEG()
2244 Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FNEG()
2245 Register ConstReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FNEG()
2246 Register OpReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FNEG()
2248 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), LoReg) in selectG_FNEG()
2249 .addReg(Src, 0, AMDGPU::sub0); in selectG_FNEG()
2250 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), HiReg) in selectG_FNEG()
2251 .addReg(Src, 0, AMDGPU::sub1); in selectG_FNEG()
2252 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_MOV_B32), ConstReg) in selectG_FNEG()
2256 unsigned Opc = Fabs ? AMDGPU::S_OR_B32 : AMDGPU::S_XOR_B32; in selectG_FNEG()
2260 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::REG_SEQUENCE), Dst) in selectG_FNEG()
2262 .addImm(AMDGPU::sub0) in selectG_FNEG()
2264 .addImm(AMDGPU::sub1); in selectG_FNEG()
2273 if (DstRB->getID() != AMDGPU::SGPRRegBankID || in selectG_FABS()
2280 Register LoReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FABS()
2281 Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FABS()
2282 Register ConstReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FABS()
2283 Register OpReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FABS()
2285 if (!RBI.constrainGenericRegister(Src, AMDGPU::SReg_64RegClass, *MRI) || in selectG_FABS()
2286 !RBI.constrainGenericRegister(Dst, AMDGPU::SReg_64RegClass, *MRI)) in selectG_FABS()
2289 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), LoReg) in selectG_FABS()
2290 .addReg(Src, 0, AMDGPU::sub0); in selectG_FABS()
2291 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), HiReg) in selectG_FABS()
2292 .addReg(Src, 0, AMDGPU::sub1); in selectG_FABS()
2293 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_MOV_B32), ConstReg) in selectG_FABS()
2298 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_AND_B32), OpReg) in selectG_FABS()
2301 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::REG_SEQUENCE), Dst) in selectG_FABS()
2303 .addImm(AMDGPU::sub0) in selectG_FABS()
2305 .addImm(AMDGPU::sub1); in selectG_FABS()
2339 if (OpBank->getID() == AMDGPU::SGPRRegBankID) in getAddrModeInfo()
2350 return RBI.getRegBank(Reg, *MRI, TRI)->getID() == AMDGPU::SGPRRegBankID; in isSGPR()
2391 BuildMI(*BB, &I, I.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), AMDGPU::M0) in initM0()
2421 const unsigned SubReg = Is64 ? AMDGPU::sub0_sub1 : AMDGPU::sub0; in selectG_AMDGPU_ATOMIC_CMPXCHG()
2423 Is64 ? &AMDGPU::VReg_128RegClass : &AMDGPU::VReg_64RegClass); in selectG_AMDGPU_ATOMIC_CMPXCHG()
2433 Opcode = Is64 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_OFFSET_RTN : in selectG_AMDGPU_ATOMIC_CMPXCHG()
2434 AMDGPU::BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN; in selectG_AMDGPU_ATOMIC_CMPXCHG()
2437 Opcode = Is64 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_ADDR64_RTN : in selectG_AMDGPU_ATOMIC_CMPXCHG()
2438 AMDGPU::BUFFER_ATOMIC_CMPSWAP_ADDR64_RTN; in selectG_AMDGPU_ATOMIC_CMPXCHG()
2455 MIB.addImm(AMDGPU::CPol::GLC); in selectG_AMDGPU_ATOMIC_CMPXCHG()
2458 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), DstReg) in selectG_AMDGPU_ATOMIC_CMPXCHG()
2464 DstReg, Is64 ? &AMDGPU::VReg_64RegClass : &AMDGPU::VGPR_32RegClass); in selectG_AMDGPU_ATOMIC_CMPXCHG()
2487 CondPhysReg = AMDGPU::SCC; in selectG_BRCOND()
2488 BrOpcode = AMDGPU::S_CBRANCH_SCC1; in selectG_BRCOND()
2489 ConstrainRC = &AMDGPU::SReg_32RegClass; in selectG_BRCOND()
2497 BrOpcode = AMDGPU::S_CBRANCH_VCCNZ; in selectG_BRCOND()
2504 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), CondPhysReg) in selectG_BRCOND()
2517 const bool IsVGPR = DstRB->getID() == AMDGPU::VGPRRegBankID; in selectG_GLOBAL_VALUE()
2518 I.setDesc(TII.get(IsVGPR ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32)); in selectG_GLOBAL_VALUE()
2520 I.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true)); in selectG_GLOBAL_VALUE()
2523 DstReg, IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass, *MRI); in selectG_GLOBAL_VALUE()
2536 const bool IsVGPR = DstRB->getID() == AMDGPU::VGPRRegBankID; in selectG_PTRMASK()
2540 unsigned NewOpc = IsVGPR ? AMDGPU::V_AND_B32_e64 : AMDGPU::S_AND_B32; in selectG_PTRMASK()
2542 = IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass; in selectG_PTRMASK()
2573 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), LoReg) in selectG_PTRMASK()
2574 .addReg(SrcReg, 0, AMDGPU::sub0); in selectG_PTRMASK()
2575 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), HiReg) in selectG_PTRMASK()
2576 .addReg(SrcReg, 0, AMDGPU::sub1); in selectG_PTRMASK()
2594 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), MaskLo) in selectG_PTRMASK()
2595 .addReg(MaskReg, 0, AMDGPU::sub0); in selectG_PTRMASK()
2608 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), MaskHi) in selectG_PTRMASK()
2609 .addReg(MaskReg, 0, AMDGPU::sub1); in selectG_PTRMASK()
2615 BuildMI(*BB, &I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg) in selectG_PTRMASK()
2617 .addImm(AMDGPU::sub0) in selectG_PTRMASK()
2619 .addImm(AMDGPU::sub1); in selectG_PTRMASK()
2635 std::tie(IdxBaseReg, Offset) = AMDGPU::getBaseWithConstantOffset(MRI, IdxReg); in computeIndirectRegIndex()
2636 if (IdxBaseReg == AMDGPU::NoRegister) { in computeIndirectRegIndex()
2667 if (IdxRB->getID() != AMDGPU::SGPRRegBankID) in selectG_EXTRACT_VECTOR_ELT()
2678 !RBI.constrainGenericRegister(IdxReg, AMDGPU::SReg_32RegClass, *MRI)) in selectG_EXTRACT_VECTOR_ELT()
2689 if (SrcRB->getID() == AMDGPU::SGPRRegBankID) { in selectG_EXTRACT_VECTOR_ELT()
2693 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectG_EXTRACT_VECTOR_ELT()
2696 unsigned Opc = Is64 ? AMDGPU::S_MOVRELS_B64 : AMDGPU::S_MOVRELS_B32; in selectG_EXTRACT_VECTOR_ELT()
2704 if (SrcRB->getID() != AMDGPU::VGPRRegBankID || DstTy.getSizeInBits() != 32) in selectG_EXTRACT_VECTOR_ELT()
2708 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectG_EXTRACT_VECTOR_ELT()
2710 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::V_MOVRELS_B32_e32), DstReg) in selectG_EXTRACT_VECTOR_ELT()
2749 if (IdxRB->getID() != AMDGPU::SGPRRegBankID) in selectG_INSERT_VECTOR_ELT()
2760 !RBI.constrainGenericRegister(IdxReg, AMDGPU::SReg_32RegClass, *MRI)) in selectG_INSERT_VECTOR_ELT()
2763 if (VecRB->getID() == AMDGPU::VGPRRegBankID && ValSize != 32) in selectG_INSERT_VECTOR_ELT()
2770 const bool IndexMode = VecRB->getID() == AMDGPU::VGPRRegBankID && in selectG_INSERT_VECTOR_ELT()
2777 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectG_INSERT_VECTOR_ELT()
2781 VecSize, ValSize, VecRB->getID() == AMDGPU::SGPRRegBankID); in selectG_INSERT_VECTOR_ELT()
2844 if (!AMDGPU::isLegalVOP3PShuffleMask(ShufMask)) in selectG_SHUFFLE_VECTOR()
2854 const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID; in selectG_SHUFFLE_VECTOR()
2856 AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass; in selectG_SHUFFLE_VECTOR()
2860 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::IMPLICIT_DEF), DstReg); in selectG_SHUFFLE_VECTOR()
2876 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::COPY), DstReg) in selectG_SHUFFLE_VECTOR()
2885 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_LSHRREV_B32_e64), DstReg) in selectG_SHUFFLE_VECTOR()
2889 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::S_LSHR_B32), DstReg) in selectG_SHUFFLE_VECTOR()
2895 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_LSHLREV_B32_e64), DstReg) in selectG_SHUFFLE_VECTOR()
2899 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::S_LSHL_B32), DstReg) in selectG_SHUFFLE_VECTOR()
2907 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_MOV_B32_sdwa), DstReg) in selectG_SHUFFLE_VECTOR()
2911 .addImm(AMDGPU::SDWA::WORD_1) // $dst_sel in selectG_SHUFFLE_VECTOR()
2912 .addImm(AMDGPU::SDWA::UNUSED_PRESERVE) // $dst_unused in selectG_SHUFFLE_VECTOR()
2913 .addImm(AMDGPU::SDWA::WORD_0) // $src0_sel in selectG_SHUFFLE_VECTOR()
2917 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::S_PACK_LL_B32_B16), DstReg) in selectG_SHUFFLE_VECTOR()
2925 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_MOV_B32_sdwa), DstReg) in selectG_SHUFFLE_VECTOR()
2929 .addImm(AMDGPU::SDWA::WORD_0) // $dst_sel in selectG_SHUFFLE_VECTOR()
2930 .addImm(AMDGPU::SDWA::UNUSED_PRESERVE) // $dst_unused in selectG_SHUFFLE_VECTOR()
2931 .addImm(AMDGPU::SDWA::WORD_1) // $src0_sel in selectG_SHUFFLE_VECTOR()
2935 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::S_PACK_HH_B32_B16), DstReg) in selectG_SHUFFLE_VECTOR()
2941 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_ALIGNBIT_B32_e64), DstReg) in selectG_SHUFFLE_VECTOR()
2946 Register TmpReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_SHUFFLE_VECTOR()
2947 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::S_LSHR_B32), TmpReg) in selectG_SHUFFLE_VECTOR()
2950 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::S_PACK_LL_B32_B16), DstReg) in selectG_SHUFFLE_VECTOR()
2992 Opcode = HasVIndex ? AMDGPU::BUFFER_ATOMIC_ADD_F32_BOTHEN in selectAMDGPU_BUFFER_ATOMIC_FADD()
2993 : AMDGPU::BUFFER_ATOMIC_ADD_F32_OFFEN; in selectAMDGPU_BUFFER_ATOMIC_FADD()
2995 Opcode = HasVIndex ? AMDGPU::BUFFER_ATOMIC_ADD_F32_IDXEN in selectAMDGPU_BUFFER_ATOMIC_FADD()
2996 : AMDGPU::BUFFER_ATOMIC_ADD_F32_OFFSET; in selectAMDGPU_BUFFER_ATOMIC_FADD()
3001 case AMDGPU::BUFFER_ATOMIC_ADD_F32_BOTHEN: in selectAMDGPU_BUFFER_ATOMIC_FADD()
3002 Opcode = AMDGPU::BUFFER_ATOMIC_PK_ADD_F16_BOTHEN; in selectAMDGPU_BUFFER_ATOMIC_FADD()
3004 case AMDGPU::BUFFER_ATOMIC_ADD_F32_OFFEN: in selectAMDGPU_BUFFER_ATOMIC_FADD()
3005 Opcode = AMDGPU::BUFFER_ATOMIC_PK_ADD_F16_OFFEN; in selectAMDGPU_BUFFER_ATOMIC_FADD()
3007 case AMDGPU::BUFFER_ATOMIC_ADD_F32_IDXEN: in selectAMDGPU_BUFFER_ATOMIC_FADD()
3008 Opcode = AMDGPU::BUFFER_ATOMIC_PK_ADD_F16_IDXEN; in selectAMDGPU_BUFFER_ATOMIC_FADD()
3010 case AMDGPU::BUFFER_ATOMIC_ADD_F32_OFFSET: in selectAMDGPU_BUFFER_ATOMIC_FADD()
3011 Opcode = AMDGPU::BUFFER_ATOMIC_PK_ADD_F16_OFFSET; in selectAMDGPU_BUFFER_ATOMIC_FADD()
3019 if (Opcode == AMDGPU::BUFFER_ATOMIC_ADD_F32_BOTHEN || in selectAMDGPU_BUFFER_ATOMIC_FADD()
3020 Opcode == AMDGPU::BUFFER_ATOMIC_PK_ADD_F16_BOTHEN) { in selectAMDGPU_BUFFER_ATOMIC_FADD()
3022 BuildMI(*MBB, &*I, DL, TII.get(AMDGPU::REG_SEQUENCE), IdxReg) in selectAMDGPU_BUFFER_ATOMIC_FADD()
3024 .addImm(AMDGPU::sub0) in selectAMDGPU_BUFFER_ATOMIC_FADD()
3026 .addImm(AMDGPU::sub1); in selectAMDGPU_BUFFER_ATOMIC_FADD()
3074 AMDGPU::GLOBAL_ATOMIC_PK_ADD_F16 : AMDGPU::GLOBAL_ATOMIC_ADD_F32; in selectGlobalAtomicFadd()
3175 case AMDGPU::G_AMDGPU_ATOMIC_INC: in select()
3176 case AMDGPU::G_AMDGPU_ATOMIC_DEC: in select()
3177 case AMDGPU::G_AMDGPU_ATOMIC_FMIN: in select()
3178 case AMDGPU::G_AMDGPU_ATOMIC_FMAX: in select()
3180 case AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG: in select()
3205 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD: in select()
3206 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE: { in select()
3207 const AMDGPU::ImageDimIntrinsicInfo *Intr in select()
3208 = AMDGPU::getImageDimIntrinsicInfo(I.getIntrinsicID()); in select()
3212 case AMDGPU::G_AMDGPU_INTRIN_BVH_INTERSECT_RAY: in select()
3214 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FADD: in select()
3216 case AMDGPU::G_SBFX: in select()
3217 case AMDGPU::G_UBFX: in select()
3241 if (MI && MI->getOpcode() == AMDGPU::G_FNEG) { in selectVOP3ModsImpl()
3247 if (AllowAbs && MI && MI->getOpcode() == AMDGPU::G_FABS) { in selectVOP3ModsImpl()
3253 RBI.getRegBank(Src, *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID) { in selectVOP3ModsImpl()
3261 TII.get(AMDGPU::COPY), VGPRSrc) in selectVOP3ModsImpl()
3344 if (Def && (Def->getOpcode() == AMDGPU::G_FNEG || in selectVOP3NoMods()
3345 Def->getOpcode() == AMDGPU::G_FABS)) in selectVOP3NoMods()
3358 if (MI && MI->getOpcode() == AMDGPU::G_FNEG && in selectVOP3PModsImpl()
3423 AMDGPU::getSMRDEncodedOffset(STI, GEPInfo.Imm, false); in selectSmrdImm()
3445 AMDGPU::getSMRDEncodedLiteralOffset32(STI, GEPInfo.Imm); in selectSmrdImm32()
3478 Register OffsetReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectSmrdSgpr()
3479 BuildMI(*MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), OffsetReg) in selectSmrdSgpr()
3549 if (Def->getOpcode() != AMDGPU::G_MERGE_VALUES) in matchZeroExtendFromS32()
3596 MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectGlobalSAddr()
3598 BuildMI(*MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::V_MOV_B32_e32), in selectGlobalSAddr()
3620 if (STI.getConstantBusLimit(AMDGPU::V_ADD_U32_e64) > NumLiterals) in selectGlobalSAddr()
3631 if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) { in selectGlobalSAddr()
3657 if (AddrDef->MI->getOpcode() == AMDGPU::G_IMPLICIT_DEF || in selectGlobalSAddr()
3658 AddrDef->MI->getOpcode() == AMDGPU::G_CONSTANT || !isSGPR(AddrDef->Reg)) in selectGlobalSAddr()
3665 Register VOffset = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectGlobalSAddr()
3667 BuildMI(*MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::V_MOV_B32_e32), VOffset) in selectGlobalSAddr()
3699 if (AddrDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX) { in selectScratchSAddr()
3709 if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) { in selectScratchSAddr()
3716 LHSDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX && in selectScratchSAddr()
3722 SAddr = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectScratchSAddr()
3724 BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_ADD_I32), SAddr) in selectScratchSAddr()
3749 Register HighBits = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectMUBUFScratchOffen()
3753 BuildMI(*MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::V_MOV_B32_e32), in selectMUBUFScratchOffen()
3788 if (PtrBaseDef->getOpcode() == AMDGPU::G_FRAME_INDEX) in selectMUBUFScratchOffen()
3794 } else if (RootDef->getOpcode() == AMDGPU::G_FRAME_INDEX) { in selectMUBUFScratchOffen()
3890 } else if (RootDef->getOpcode() == AMDGPU::G_SUB) { in selectDS1Addr1OffsetImpl()
3957 } else if (RootDef->getOpcode() == AMDGPU::G_SUB) { in selectDSReadWrite2Impl()
3996 Register RSrc2 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); in buildRSRC()
3997 Register RSrc3 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); in buildRSRC()
3998 Register RSrcHi = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in buildRSRC()
3999 Register RSrc = MRI.createVirtualRegister(&AMDGPU::SGPR_128RegClass); in buildRSRC()
4001 B.buildInstr(AMDGPU::S_MOV_B32) in buildRSRC()
4004 B.buildInstr(AMDGPU::S_MOV_B32) in buildRSRC()
4011 B.buildInstr(AMDGPU::REG_SEQUENCE) in buildRSRC()
4014 .addImm(AMDGPU::sub0) in buildRSRC()
4016 .addImm(AMDGPU::sub1); in buildRSRC()
4020 RSrcLo = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in buildRSRC()
4021 B.buildInstr(AMDGPU::S_MOV_B64) in buildRSRC()
4026 B.buildInstr(AMDGPU::REG_SEQUENCE) in buildRSRC()
4029 .addImm(AMDGPU::sub0_sub1) in buildRSRC()
4031 .addImm(AMDGPU::sub2_sub3); in buildRSRC()
4093 return N0Bank->getID() == AMDGPU::VGPRRegBankID; in shouldUseAddr64()
4105 SOffset = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in splitIllegalMUBUFOffset()
4106 B.buildInstr(AMDGPU::S_MOV_B32) in splitIllegalMUBUFOffset()
4133 if (RBI.getRegBank(N2, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) { in selectMUBUFAddr64Impl()
4135 if (RBI.getRegBank(N3, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) { in selectMUBUFAddr64Impl()
4148 } else if (RBI.getRegBank(N0, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) { in selectMUBUFAddr64Impl()
4277 MIB.addImm(AMDGPU::CPol::GLC); // cpol in selectMUBUFAddr64Atomic()
4302 [=](MachineInstrBuilder &MIB) { MIB.addImm(AMDGPU::CPol::GLC); } // cpol in selectMUBUFOffsetAtomic()
4323 AMDGPU::getSMRDEncodedOffset(STI, *OffsetVal, true); in selectSMRDBufferImm()
4339 = AMDGPU::getSMRDEncodedLiteralOffset32(STI, *OffsetVal); in selectSMRDBufferImm32()
4396 MIB.addImm(MI.getOperand(OpIdx).getImm() & AMDGPU::CPol::ALL); in renderExtractCPol()
4410 MIB.addImm(MI.getOperand(OpIdx).getImm() | AMDGPU::CPol::GLC); in renderSetGLC()
4420 return AMDGPU::isInlinableLiteral16(Imm, STI.hasInv2PiInlineImm()); in isInlineImmediate16()
4424 return AMDGPU::isInlinableLiteral32(Imm, STI.hasInv2PiInlineImm()); in isInlineImmediate32()
4428 return AMDGPU::isInlinableLiteral64(Imm, STI.hasInv2PiInlineImm()); in isInlineImmediate64()