Lines Matching refs:MRI
68 MRI = &MF.getRegInfo(); in setupMF()
74 const MachineRegisterInfo &MRI) const { in isVCC()
79 auto &RegClassOrBank = MRI.getRegClassOrRegBank(Reg); in isVCC()
83 const LLT Ty = MRI.getType(Reg); in isVCC()
87 return MRI.getVRegDef(Reg)->getOpcode() != AMDGPU::G_TRUNC && in isVCC()
105 if (MRI->getType(Dst.getReg()) == LLT::scalar(1)) in constrainCopyLikeIntrin()
109 = TRI.getConstrainedRegClassForOperand(Dst, *MRI); in constrainCopyLikeIntrin()
111 = TRI.getConstrainedRegClassForOperand(Src, *MRI); in constrainCopyLikeIntrin()
115 return RBI.constrainGenericRegister(Dst.getReg(), *DstRC, *MRI) && in constrainCopyLikeIntrin()
116 RBI.constrainGenericRegister(Src.getReg(), *SrcRC, *MRI); in constrainCopyLikeIntrin()
129 if (isVCC(DstReg, *MRI)) { in selectCOPY()
132 = TRI.getConstrainedRegClassForOperand(Dst, *MRI); in selectCOPY()
135 return RBI.constrainGenericRegister(DstReg, *RC, *MRI); in selectCOPY()
138 if (!isVCC(SrcReg, *MRI)) { in selectCOPY()
140 if (!RBI.constrainGenericRegister(DstReg, *TRI.getBoolRC(), *MRI)) in selectCOPY()
144 = TRI.getConstrainedRegClassForOperand(Src, *MRI); in selectCOPY()
147 getIConstantVRegValWithLookThrough(SrcReg, *MRI, true); in selectCOPY()
154 Register MaskedReg = MRI->createVirtualRegister(SrcRC); in selectCOPY()
170 if (!MRI->getRegClassOrNull(SrcReg)) in selectCOPY()
171 MRI->setRegClass(SrcReg, SrcRC); in selectCOPY()
177 TRI.getConstrainedRegClassForOperand(Dst, *MRI); in selectCOPY()
178 if (RC && !RBI.constrainGenericRegister(DstReg, *RC, *MRI)) in selectCOPY()
189 TRI.getConstrainedRegClassForOperand(MO, *MRI); in selectCOPY()
192 RBI.constrainGenericRegister(MO.getReg(), *RC, *MRI); in selectCOPY()
199 const LLT DefTy = MRI->getType(DefReg); in selectPHI()
212 MRI->getRegClassOrRegBank(DefReg); in selectPHI()
232 return RBI.constrainGenericRegister(DefReg, *DefRC, *MRI); in selectPHI()
242 Register DstReg = MRI->createVirtualRegister(&SubRC); in getSubOperand64()
285 unsigned Size = RBI.getSizeInBits(DstReg, *MRI, TRI); in selectG_AND_OR_XOR()
287 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI); in selectG_AND_OR_XOR()
309 LLT Ty = MRI->getType(DstReg); in selectG_ADD_SUB()
314 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI); in selectG_ADD_SUB()
339 Register UnusedCarry = MRI->createVirtualRegister(TRI.getWaveMaskRegClass()); in selectG_ADD_SUB()
362 Register DstLo = MRI->createVirtualRegister(&HalfRC); in selectG_ADD_SUB()
363 Register DstHi = MRI->createVirtualRegister(&HalfRC); in selectG_ADD_SUB()
374 Register CarryReg = MRI->createVirtualRegister(CarryRC); in selectG_ADD_SUB()
381 .addDef(MRI->createVirtualRegister(CarryRC), RegState::Dead) in selectG_ADD_SUB()
398 if (!RBI.constrainGenericRegister(DstReg, RC, *MRI)) in selectG_ADD_SUB()
417 if (isVCC(Dst1Reg, *MRI)) { in selectG_UADDO_USUBO_UADDE_USUBE()
444 if (!MRI->getRegClassOrNull(Dst1Reg)) in selectG_UADDO_USUBO_UADDE_USUBE()
445 MRI->setRegClass(Dst1Reg, &AMDGPU::SReg_32RegClass); in selectG_UADDO_USUBO_UADDE_USUBE()
447 if (!RBI.constrainGenericRegister(Dst0Reg, AMDGPU::SReg_32RegClass, *MRI) || in selectG_UADDO_USUBO_UADDE_USUBE()
448 !RBI.constrainGenericRegister(Src0Reg, AMDGPU::SReg_32RegClass, *MRI) || in selectG_UADDO_USUBO_UADDE_USUBE()
449 !RBI.constrainGenericRegister(Src1Reg, AMDGPU::SReg_32RegClass, *MRI)) in selectG_UADDO_USUBO_UADDE_USUBE()
454 AMDGPU::SReg_32RegClass, *MRI)) in selectG_UADDO_USUBO_UADDE_USUBE()
484 LLT DstTy = MRI->getType(DstReg); in selectG_EXTRACT()
485 LLT SrcTy = MRI->getType(SrcReg); in selectG_EXTRACT()
500 TRI.getConstrainedRegClassForOperand(I.getOperand(0), *MRI); in selectG_EXTRACT()
501 if (!DstRC || !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) in selectG_EXTRACT()
504 const RegisterBank *SrcBank = RBI.getRegBank(SrcReg, *MRI, TRI); in selectG_EXTRACT()
515 SrcReg = constrainOperandRegClass(*MF, TRI, *MRI, TII, RBI, I, in selectG_EXTRACT()
528 LLT DstTy = MRI->getType(DstReg); in selectG_MERGE_VALUES()
529 LLT SrcTy = MRI->getType(MI.getOperand(1).getReg()); in selectG_MERGE_VALUES()
536 const RegisterBank *DstBank = RBI.getRegBank(DstReg, *MRI, TRI); in selectG_MERGE_VALUES()
552 = TRI.getConstrainedRegClassForOperand(Src, *MRI); in selectG_MERGE_VALUES()
553 if (SrcRC && !RBI.constrainGenericRegister(Src.getReg(), *SrcRC, *MRI)) in selectG_MERGE_VALUES()
557 if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) in selectG_MERGE_VALUES()
572 LLT DstTy = MRI->getType(DstReg0); in selectG_UNMERGE_VALUES()
573 LLT SrcTy = MRI->getType(SrcReg); in selectG_UNMERGE_VALUES()
578 const RegisterBank *SrcBank = RBI.getRegBank(SrcReg, *MRI, TRI); in selectG_UNMERGE_VALUES()
582 if (!SrcRC || !RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI)) in selectG_UNMERGE_VALUES()
596 if (!SrcRC || !RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI)) in selectG_UNMERGE_VALUES()
600 TRI.getConstrainedRegClassForOperand(Dst, *MRI); in selectG_UNMERGE_VALUES()
601 if (DstRC && !RBI.constrainGenericRegister(Dst.getReg(), *DstRC, *MRI)) in selectG_UNMERGE_VALUES()
618 if (MRI->getType(Dst) != V2S16) in selectG_BUILD_VECTOR_TRUNC()
621 const RegisterBank *DstBank = RBI.getRegBank(Dst, *MRI, TRI); in selectG_BUILD_VECTOR_TRUNC()
627 if (MRI->getType(Src0) != S32) in selectG_BUILD_VECTOR_TRUNC()
633 auto ConstSrc1 = getAnyConstantVRegValWithLookThrough(Src1, *MRI, true, true); in selectG_BUILD_VECTOR_TRUNC()
636 getAnyConstantVRegValWithLookThrough(Src0, *MRI, true, true); in selectG_BUILD_VECTOR_TRUNC()
646 return RBI.constrainGenericRegister(Dst, AMDGPU::SReg_32RegClass, *MRI); in selectG_BUILD_VECTOR_TRUNC()
652 MachineInstr *Src1Def = getDefIgnoringCopies(Src1, *MRI); in selectG_BUILD_VECTOR_TRUNC()
656 return RBI.constrainGenericRegister(Dst, AMDGPU::SReg_32RegClass, *MRI) && in selectG_BUILD_VECTOR_TRUNC()
657 RBI.constrainGenericRegister(Src0, AMDGPU::SReg_32RegClass, *MRI); in selectG_BUILD_VECTOR_TRUNC()
676 Src0, *MRI, m_OneUse(m_GLShr(m_Reg(ShiftSrc0), m_SpecificICst(16)))); in selectG_BUILD_VECTOR_TRUNC()
679 Src1, *MRI, m_OneUse(m_GLShr(m_Reg(ShiftSrc1), m_SpecificICst(16)))); in selectG_BUILD_VECTOR_TRUNC()
718 const TargetRegisterClass *RC = TRI.getConstrainedRegClassForOperand(MO, *MRI); in selectG_IMPLICIT_DEF()
719 if ((!RC && !MRI->getRegBankOrNull(MO.getReg())) || in selectG_IMPLICIT_DEF()
720 (RC && RBI.constrainGenericRegister(MO.getReg(), *RC, *MRI))) { in selectG_IMPLICIT_DEF()
734 LLT Src1Ty = MRI->getType(Src1Reg); in selectG_INSERT()
736 unsigned DstSize = MRI->getType(DstReg).getSizeInBits(); in selectG_INSERT()
753 const RegisterBank *DstBank = RBI.getRegBank(DstReg, *MRI, TRI); in selectG_INSERT()
759 const RegisterBank *Src0Bank = RBI.getRegBank(Src0Reg, *MRI, TRI); in selectG_INSERT()
760 const RegisterBank *Src1Bank = RBI.getRegBank(Src1Reg, *MRI, TRI); in selectG_INSERT()
772 if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI) || in selectG_INSERT()
773 !RBI.constrainGenericRegister(Src0Reg, *Src0RC, *MRI) || in selectG_INSERT()
774 !RBI.constrainGenericRegister(Src1Reg, *Src1RC, *MRI)) in selectG_INSERT()
793 assert(RBI.getRegBank(DstReg, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID && in selectG_SBFX_UBFX()
795 assert(MRI->getType(MI.getOperand(0).getReg()).getSizeInBits() == 32 && in selectG_SBFX_UBFX()
818 if (!RBI.constrainGenericRegister(M0Val, AMDGPU::SReg_32RegClass, *MRI) || in selectInterpP1F16()
819 !RBI.constrainGenericRegister(Dst, AMDGPU::VGPR_32RegClass, *MRI) || in selectInterpP1F16()
820 !RBI.constrainGenericRegister(Src0, AMDGPU::VGPR_32RegClass, *MRI)) in selectInterpP1F16()
830 Register InterpMov = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectInterpP1F16()
876 getIConstantVRegValWithLookThrough(LaneSelect, *MRI); in selectWritelane()
885 getIConstantVRegValWithLookThrough(Val, *MRI); in selectWritelane()
899 RBI.constrainGenericRegister(LaneSelect, AMDGPU::SReg_32_XM0RegClass, *MRI); in selectWritelane()
919 LLT Ty = MRI->getType(Dst0); in selectDivScale()
974 MRI->setRegClass(Reg, TRI.getWaveMaskRegClass()); in selectG_INTRINSIC()
1102 unsigned Size = RBI.getSizeInBits(SrcReg, *MRI, TRI); in selectG_ICMP()
1107 if (!isVCC(CCReg, *MRI)) { in selectG_ICMP()
1118 RBI.constrainGenericRegister(CCReg, AMDGPU::SReg_32RegClass, *MRI); in selectG_ICMP()
1132 *TRI.getBoolRC(), *MRI); in selectG_ICMP()
1140 if (isVCC(Dst, *MRI)) in selectIntrinsicIcmp()
1143 if (MRI->getType(Dst).getSizeInBits() != STI.getWavefrontSize()) in selectIntrinsicIcmp()
1149 unsigned Size = RBI.getSizeInBits(SrcReg, *MRI, TRI); in selectIntrinsicIcmp()
1157 *TRI.getBoolRC(), *MRI)) in selectIntrinsicIcmp()
1171 *MRI); in selectIntrinsicIcmp()
1181 const unsigned Size = MRI->getType(DstReg).getSizeInBits(); in selectBallot()
1188 getIConstantVRegValWithLookThrough(I.getOperand(2).getReg(), *MRI); in selectBallot()
1211 const RegisterBank *DstBank = RBI.getRegBank(DstReg, *MRI, TRI); in selectRelocConstant()
1213 if (!DstRC || !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) in selectRelocConstant()
1237 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI); in selectGroupStaticSize()
1270 = TRI.getConstrainedRegClassForOperand(Dst, *MRI); in selectReturnAddress()
1272 !RBI.constrainGenericRegister(DstReg, *RC, *MRI)) in selectReturnAddress()
1308 if (!MRI->getRegClassOrNull(Reg)) in selectEndCfIntrinsic()
1309 MRI->setRegClass(Reg, TRI.getWaveMaskRegClass()); in selectEndCfIntrinsic()
1369 if (!RBI.constrainGenericRegister(M0Val, AMDGPU::SReg_32RegClass, *MRI)) in selectDSOrderedIntrinsic()
1407 const RegisterBank *OffsetRB = RBI.getRegBank(BaseOffset, *MRI, TRI); in selectDSGWSIntrinsic()
1411 MachineInstr *OffsetDef = getDefIgnoringCopies(BaseOffset, *MRI); in selectDSGWSIntrinsic()
1427 OffsetDef = getDefIgnoringCopies(BaseOffset, *MRI); in selectDSGWSIntrinsic()
1441 AMDGPU::getBaseWithConstantOffset(*MRI, BaseOffset); in selectDSGWSIntrinsic()
1446 if (!RBI.constrainGenericRegister(BaseOffset, AMDGPU::VGPR_32RegClass, *MRI)) in selectDSGWSIntrinsic()
1453 AMDGPU::SReg_32RegClass, *MRI)) in selectDSGWSIntrinsic()
1457 Register M0Base = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectDSGWSIntrinsic()
1475 if (!RBI.constrainGenericRegister(VSrc, AMDGPU::VGPR_32RegClass, *MRI)) in selectDSGWSIntrinsic()
1491 LLT PtrTy = MRI->getType(PtrBase); in selectDSAppendConsume()
1509 if (!RBI.constrainGenericRegister(PtrBase, AMDGPU::SReg_32RegClass, *MRI)) in selectDSAppendConsume()
1595 LLT Ty = MRI->getType(VDataIn); in selectImageIntrinsic()
1617 VDataTy = MRI->getType(VDataIn); in selectImageIntrinsic()
1621 VDataTy = MRI->getType(VDataOut); in selectImageIntrinsic()
1659 NumVAddrDwords += (MRI->getType(Addr).getSizeInBits() + 31) / 32; in selectImageIntrinsic()
1711 const bool Is64 = MRI->getType(VDataOut).getSizeInBits() == 64; in selectImageIntrinsic()
1713 Register TmpReg = MRI->createVirtualRegister( in selectImageIntrinsic()
1718 if (!MRI->use_empty(VDataOut)) { in selectImageIntrinsic()
1773 Register Tied = MRI->cloneVirtualRegister(VDataOut); in selectImageIntrinsic()
1774 Register Zero = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectImageIntrinsic()
1777 auto Parts = TRI.getRegSplitParts(MRI->getRegClass(Tied), 4); in selectImageIntrinsic()
1788 Register Undef = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectImageIntrinsic()
1856 unsigned Size = RBI.getSizeInBits(DstReg, *MRI, TRI); in selectG_SELECT()
1860 if (!isVCC(CCReg, *MRI)) { in selectG_SELECT()
1869 if (!MRI->getRegClassOrNull(CCReg)) in selectG_SELECT()
1870 MRI->setRegClass(CCReg, TRI.getConstrainedRegClassForOperand(CCOp, *MRI)); in selectG_SELECT()
1923 const LLT DstTy = MRI->getType(DstReg); in selectG_TRUNC()
1924 const LLT SrcTy = MRI->getType(SrcReg); in selectG_TRUNC()
1927 const RegisterBank *SrcRB = RBI.getRegBank(SrcReg, *MRI, TRI); in selectG_TRUNC()
1934 DstRB = RBI.getRegBank(DstReg, *MRI, TRI); in selectG_TRUNC()
1951 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI) || in selectG_TRUNC()
1952 !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) { in selectG_TRUNC()
1961 Register LoReg = MRI->createVirtualRegister(DstRC); in selectG_TRUNC()
1962 Register HiReg = MRI->createVirtualRegister(DstRC); in selectG_TRUNC()
1982 Register TmpReg0 = MRI->createVirtualRegister(DstRC); in selectG_TRUNC()
1983 Register TmpReg1 = MRI->createVirtualRegister(DstRC); in selectG_TRUNC()
1984 Register ImmReg = MRI->createVirtualRegister(DstRC); in selectG_TRUNC()
2029 if (!RBI.constrainGenericRegister(SrcReg, *SrcWithSubRC, *MRI)) in selectG_TRUNC()
2049 Register Reg, const MachineRegisterInfo &MRI, in getArtifactRegBank() argument
2051 const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(Reg); in getArtifactRegBank()
2069 const LLT DstTy = MRI->getType(DstReg); in selectG_SZA_EXT()
2070 const LLT SrcTy = MRI->getType(SrcReg); in selectG_SZA_EXT()
2078 const RegisterBank *SrcBank = getArtifactRegBank(SrcReg, *MRI, TRI); in selectG_SZA_EXT()
2087 const RegisterBank *DstBank = RBI.getRegBank(DstReg, *MRI, TRI); in selectG_SZA_EXT()
2091 Register UndefReg = MRI->createVirtualRegister(SrcRC); in selectG_SZA_EXT()
2100 return RBI.constrainGenericRegister(DstReg, *DstRC, *MRI) && in selectG_SZA_EXT()
2101 RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI); in selectG_SZA_EXT()
2131 if (!RBI.constrainGenericRegister(SrcReg, SrcRC, *MRI)) in selectG_SZA_EXT()
2140 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_32RegClass, *MRI); in selectG_SZA_EXT()
2149 Register ExtReg = MRI->createVirtualRegister(&AMDGPU::SReg_64RegClass); in selectG_SZA_EXT()
2150 Register UndefReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_SZA_EXT()
2165 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_64RegClass, *MRI); in selectG_SZA_EXT()
2180 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_32RegClass, *MRI); in selectG_SZA_EXT()
2190 unsigned Size = MRI->getType(DstReg).getSizeInBits(); in selectG_CONSTANT()
2202 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI); in selectG_CONSTANT()
2235 Register LoReg = MRI->createVirtualRegister(RC); in selectG_CONSTANT()
2236 Register HiReg = MRI->createVirtualRegister(RC); in selectG_CONSTANT()
2255 TRI.getConstrainedRegClassForOperand(ResInst->getOperand(0), *MRI); in selectG_CONSTANT()
2258 return RBI.constrainGenericRegister(DstReg, *DstRC, *MRI); in selectG_CONSTANT()
2274 const RegisterBank *DstRB = RBI.getRegBank(Dst, *MRI, TRI); in selectG_FNEG()
2276 MRI->getType(Dst) != LLT::scalar(64)) in selectG_FNEG()
2280 MachineInstr *Fabs = getOpcodeDef(TargetOpcode::G_FABS, Src, *MRI); in selectG_FNEG()
2284 if (!RBI.constrainGenericRegister(Src, AMDGPU::SReg_64RegClass, *MRI) || in selectG_FNEG()
2285 !RBI.constrainGenericRegister(Dst, AMDGPU::SReg_64RegClass, *MRI)) in selectG_FNEG()
2290 Register LoReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FNEG()
2291 Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FNEG()
2292 Register ConstReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FNEG()
2293 Register OpReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FNEG()
2319 const RegisterBank *DstRB = RBI.getRegBank(Dst, *MRI, TRI); in selectG_FABS()
2321 MRI->getType(Dst) != LLT::scalar(64)) in selectG_FABS()
2327 Register LoReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FABS()
2328 Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FABS()
2329 Register ConstReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FABS()
2330 Register OpReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FABS()
2332 if (!RBI.constrainGenericRegister(Src, AMDGPU::SReg_64RegClass, *MRI) || in selectG_FABS()
2333 !RBI.constrainGenericRegister(Dst, AMDGPU::SReg_64RegClass, *MRI)) in selectG_FABS()
2363 const MachineRegisterInfo &MRI, SmallVectorImpl<GEPInfo> &AddrInfo) const { in getAddrModeInfo() argument
2365 const MachineInstr *PtrMI = MRI.getUniqueVRegDef(Load.getOperand(1).getReg()); in getAddrModeInfo()
2376 const MachineInstr *OpDef = MRI.getUniqueVRegDef(GEPOp.getReg()); in getAddrModeInfo()
2385 const RegisterBank *OpBank = RBI.getRegBank(GEPOp.getReg(), MRI, TRI); in getAddrModeInfo()
2393 getAddrModeInfo(*PtrMI, MRI, AddrInfo); in getAddrModeInfo()
2397 return RBI.getRegBank(Reg, *MRI, TRI)->getID() == AMDGPU::SGPRRegBankID; in isSGPR()
2431 const LLT PtrTy = MRI->getType(I.getOperand(1).getReg()); in initM0()
2446 const LLT PtrTy = MRI->getType(I.getOperand(1).getReg()); in selectG_LOAD_STORE_ATOMICRMW()
2456 static bool isVCmpResult(Register Reg, MachineRegisterInfo &MRI) { in isVCmpResult() argument
2460 MachineInstr &MI = *MRI.getUniqueVRegDef(Reg); in isVCmpResult()
2464 return isVCmpResult(MI.getOperand(1).getReg(), MRI); in isVCmpResult()
2468 return isVCmpResult(MI.getOperand(1).getReg(), MRI) && in isVCmpResult()
2469 isVCmpResult(MI.getOperand(2).getReg(), MRI); in isVCmpResult()
2492 if (!isVCC(CondReg, *MRI)) { in selectG_BRCOND()
2493 if (MRI->getType(CondReg) != LLT::scalar(32)) in selectG_BRCOND()
2504 if (!isVCmpResult(CondReg, *MRI)) { in selectG_BRCOND()
2509 Register TmpReg = MRI->createVirtualRegister(TRI.getBoolRC()); in selectG_BRCOND()
2521 if (!MRI->getRegClassOrNull(CondReg)) in selectG_BRCOND()
2522 MRI->setRegClass(CondReg, ConstrainRC); in selectG_BRCOND()
2536 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI); in selectG_GLOBAL_VALUE()
2543 DstReg, IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass, *MRI); in selectG_GLOBAL_VALUE()
2550 LLT Ty = MRI->getType(DstReg); in selectG_PTRMASK()
2551 LLT MaskTy = MRI->getType(MaskReg); in selectG_PTRMASK()
2555 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI); in selectG_PTRMASK()
2556 const RegisterBank *SrcRB = RBI.getRegBank(SrcReg, *MRI, TRI); in selectG_PTRMASK()
2557 const RegisterBank *MaskRB = RBI.getRegBank(MaskReg, *MRI, TRI); in selectG_PTRMASK()
2589 if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI) || in selectG_PTRMASK()
2590 !RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI) || in selectG_PTRMASK()
2591 !RBI.constrainGenericRegister(MaskReg, *MaskRC, *MRI)) in selectG_PTRMASK()
2605 Register HiReg = MRI->createVirtualRegister(&RegRC); in selectG_PTRMASK()
2606 Register LoReg = MRI->createVirtualRegister(&RegRC); in selectG_PTRMASK()
2621 Register MaskLo = MRI->createVirtualRegister(&RegRC); in selectG_PTRMASK()
2622 MaskedLo = MRI->createVirtualRegister(&RegRC); in selectG_PTRMASK()
2635 Register MaskHi = MRI->createVirtualRegister(&RegRC); in selectG_PTRMASK()
2636 MaskedHi = MRI->createVirtualRegister(&RegRC); in selectG_PTRMASK()
2657 computeIndirectRegIndex(MachineRegisterInfo &MRI, in computeIndirectRegIndex() argument
2665 std::tie(IdxBaseReg, Offset) = AMDGPU::getBaseWithConstantOffset(MRI, IdxReg); in computeIndirectRegIndex()
2688 LLT DstTy = MRI->getType(DstReg); in selectG_EXTRACT_VECTOR_ELT()
2689 LLT SrcTy = MRI->getType(SrcReg); in selectG_EXTRACT_VECTOR_ELT()
2691 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI); in selectG_EXTRACT_VECTOR_ELT()
2692 const RegisterBank *SrcRB = RBI.getRegBank(SrcReg, *MRI, TRI); in selectG_EXTRACT_VECTOR_ELT()
2693 const RegisterBank *IdxRB = RBI.getRegBank(IdxReg, *MRI, TRI); in selectG_EXTRACT_VECTOR_ELT()
2706 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI) || in selectG_EXTRACT_VECTOR_ELT()
2707 !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI) || in selectG_EXTRACT_VECTOR_ELT()
2708 !RBI.constrainGenericRegister(IdxReg, AMDGPU::SReg_32RegClass, *MRI)) in selectG_EXTRACT_VECTOR_ELT()
2716 std::tie(IdxReg, SubReg) = computeIndirectRegIndex(*MRI, TRI, SrcRC, IdxReg, in selectG_EXTRACT_VECTOR_ELT()
2766 LLT VecTy = MRI->getType(DstReg); in selectG_INSERT_VECTOR_ELT()
2767 LLT ValTy = MRI->getType(ValReg); in selectG_INSERT_VECTOR_ELT()
2771 const RegisterBank *VecRB = RBI.getRegBank(VecReg, *MRI, TRI); in selectG_INSERT_VECTOR_ELT()
2772 const RegisterBank *ValRB = RBI.getRegBank(ValReg, *MRI, TRI); in selectG_INSERT_VECTOR_ELT()
2773 const RegisterBank *IdxRB = RBI.getRegBank(IdxReg, *MRI, TRI); in selectG_INSERT_VECTOR_ELT()
2787 if (!RBI.constrainGenericRegister(VecReg, *VecRC, *MRI) || in selectG_INSERT_VECTOR_ELT()
2788 !RBI.constrainGenericRegister(DstReg, *VecRC, *MRI) || in selectG_INSERT_VECTOR_ELT()
2789 !RBI.constrainGenericRegister(ValReg, *ValRC, *MRI) || in selectG_INSERT_VECTOR_ELT()
2790 !RBI.constrainGenericRegister(IdxReg, AMDGPU::SReg_32RegClass, *MRI)) in selectG_INSERT_VECTOR_ELT()
2797 std::tie(IdxReg, SubReg) = computeIndirectRegIndex(*MRI, TRI, VecRC, IdxReg, in selectG_INSERT_VECTOR_ELT()
2871 if (MRI->getType(DstReg) != V2S16 || MRI->getType(Src0Reg) != V2S16) in selectG_SHUFFLE_VECTOR()
2882 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI); in selectG_SHUFFLE_VECTOR()
2892 return RBI.constrainGenericRegister(DstReg, RC, *MRI); in selectG_SHUFFLE_VECTOR()
2899 if (!RBI.constrainGenericRegister(DstReg, RC, *MRI) || in selectG_SHUFFLE_VECTOR()
2900 !RBI.constrainGenericRegister(SrcVec, RC, *MRI)) in selectG_SHUFFLE_VECTOR()
2947 Register TmpReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectG_SHUFFLE_VECTOR()
2976 Register TmpReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectG_SHUFFLE_VECTOR()
3002 Register TmpReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_SHUFFLE_VECTOR()
3021 LLT DefTy = MRI->getType(DefReg); in selectAMDGPU_BUFFER_ATOMIC_FADD()
3028 if (!MRI->use_nodbg_empty(DefReg)) { in selectAMDGPU_BUFFER_ATOMIC_FADD()
3046 bool HasVOffset = !isOperandImmEqual(VOffset, 0, *MRI); in selectAMDGPU_BUFFER_ATOMIC_FADD()
3047 bool HasVIndex = !isOperandImmEqual(VIndex, 0, *MRI); in selectAMDGPU_BUFFER_ATOMIC_FADD()
3058 if (MRI->getType(VDataIn.getReg()).isVector()) { in selectAMDGPU_BUFFER_ATOMIC_FADD()
3080 Register IdxReg = MRI->createVirtualRegister(TRI.getVGPR64Class()); in selectAMDGPU_BUFFER_ATOMIC_FADD()
3117 if (!MRI->use_nodbg_empty(MI.getOperand(0).getReg())) { in selectGlobalAtomicFadd()
3132 const unsigned Opc = MRI->getType(Data).isVector() ? in selectGlobalAtomicFadd()
3160 getIConstantVRegValWithLookThrough(VOffset, *MRI); in selectBufferLoadLds()
3194 Register IdxReg = MRI->createVirtualRegister(TRI.getVGPR64Class()); in selectBufferLoadLds()
3238 static Register matchZeroExtendFromS32(MachineRegisterInfo &MRI, Register Reg) { in matchZeroExtendFromS32() argument
3240 if (mi_match(Reg, MRI, m_GZExt(m_Reg(ZExtSrc)))) in matchZeroExtendFromS32()
3241 return MRI.getType(ZExtSrc) == LLT::scalar(32) ? ZExtSrc : Register(); in matchZeroExtendFromS32()
3244 const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI); in matchZeroExtendFromS32()
3249 MRI.getType(Def->getOperand(0).getReg()) == LLT::scalar(64)); in matchZeroExtendFromS32()
3250 if (mi_match(Def->getOperand(2).getReg(), MRI, m_ZeroInt())) { in matchZeroExtendFromS32()
3285 auto AddrDef = getDefSrcRegIgnoringCopies(Addr, *MRI); in selectGlobalLoadLds()
3290 getSrcRegIgnoringCopies(AddrDef->MI->getOperand(1).getReg(), *MRI); in selectGlobalLoadLds()
3293 if (Register Off = matchZeroExtendFromS32(*MRI, PtrBaseOffset)) { in selectGlobalLoadLds()
3304 VOffset = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectGlobalLoadLds()
3408 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI); in selectWaveAddress()
3425 if (!RBI.constrainGenericRegister(DstReg, RC, *MRI)) in selectWaveAddress()
3585 MachineInstr *MI = getDefIgnoringCopies(Src, *MRI); in selectVOP3ModsImpl()
3590 MI = getDefIgnoringCopies(Src, *MRI); in selectVOP3ModsImpl()
3602 RBI.getRegBank(Src, *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID) { in selectVOP3ModsImpl()
3608 Register VGPRSrc = MRI->cloneVirtualRegister(OrigSrc); in selectVOP3ModsImpl()
3692 const MachineInstr *Def = getDefIgnoringCopies(Reg, *MRI); in selectVOP3NoMods()
3703 Register Src, const MachineRegisterInfo &MRI, bool IsDOT) const { in selectVOP3PModsImpl() argument
3705 MachineInstr *MI = MRI.getVRegDef(Src); in selectVOP3PModsImpl()
3710 MRI.getType(Src) == LLT::fixed_vector(2, 16)) { in selectVOP3PModsImpl()
3713 MI = MRI.getVRegDef(Src); in selectVOP3PModsImpl()
3727 MachineRegisterInfo &MRI in selectVOP3PMods() local
3732 std::tie(Src, Mods) = selectVOP3PModsImpl(Root.getReg(), MRI); in selectVOP3PMods()
3742 MachineRegisterInfo &MRI in selectVOP3PModsDOT() local
3747 std::tie(Src, Mods) = selectVOP3PModsImpl(Root.getReg(), MRI, true); in selectVOP3PModsDOT()
3789 if (!isKnownNeverNaN(Src, *MRI)) in selectVOP3Mods_nnan()
3847 getAddrModeInfo(*MI, *MRI, AddrInfo); in selectSmrdOffset()
3862 matchZeroExtendFromS32(*MRI, GEPI2.SgprParts[1])) { in selectSmrdOffset()
3887 *SOffset = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectSmrdOffset()
3894 if (Register OffsetReg = matchZeroExtendFromS32(*MRI, GEPI.SgprParts[1])) { in selectSmrdOffset()
3918 getAddrModeInfo(*Root.getParent(), *MRI, AddrInfo); in selectSmrdImm32()
3971 getPtrBaseWithConstantOffset(Root.getReg(), *MRI); in selectFlatOffsetImpl()
4022 std::tie(PtrBase, ConstOffset) = getPtrBaseWithConstantOffset(Addr, *MRI); in selectGlobalSAddr()
4030 auto PtrBaseDef = getDefSrcRegIgnoringCopies(PtrBase, *MRI); in selectGlobalSAddr()
4046 MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectGlobalSAddr()
4077 auto AddrDef = getDefSrcRegIgnoringCopies(Addr, *MRI); in selectGlobalSAddr()
4081 getSrcRegIgnoringCopies(AddrDef->MI->getOperand(1).getReg(), *MRI); in selectGlobalSAddr()
4088 if (Register VOffset = matchZeroExtendFromS32(*MRI, PtrBaseOffset)) { in selectGlobalSAddr()
4112 Register VOffset = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectGlobalSAddr()
4133 std::tie(PtrBase, ConstOffset) = getPtrBaseWithConstantOffset(Addr, *MRI); in selectScratchSAddr()
4142 auto AddrDef = getDefSrcRegIgnoringCopies(Addr, *MRI); in selectScratchSAddr()
4156 auto LHSDef = getDefSrcRegIgnoringCopies(LHS, *MRI); in selectScratchSAddr()
4157 auto RHSDef = getDefSrcRegIgnoringCopies(RHS, *MRI); in selectScratchSAddr()
4165 SAddr = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectScratchSAddr()
4209 std::tie(PtrBase, ConstOffset) = getPtrBaseWithConstantOffset(Addr, *MRI); in selectScratchSVAddr()
4217 auto AddrDef = getDefSrcRegIgnoringCopies(Addr, *MRI); in selectScratchSVAddr()
4222 if (RBI.getRegBank(RHS, *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID) in selectScratchSVAddr()
4226 auto LHSDef = getDefSrcRegIgnoringCopies(LHS, *MRI); in selectScratchSVAddr()
4258 if (mi_match(Root.getReg(), *MRI, m_ICst(Offset)) && in selectMUBUFScratchOffen()
4260 Register HighBits = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectMUBUFScratchOffen()
4290 if (const MachineInstr *RootDef = MRI->getVRegDef(Root.getReg())) { in selectMUBUFScratchOffen()
4293 std::tie(PtrBase, ConstOffset) = getPtrBaseWithConstantOffset(VAddr, *MRI); in selectMUBUFScratchOffen()
4298 const MachineInstr *PtrBaseDef = MRI->getVRegDef(PtrBase); in selectMUBUFScratchOffen()
4362 Optional<APInt> RHS = getIConstantVRegVal(MI.getOperand(2).getReg(), *MRI); in isUnneededShiftMask()
4387 const MachineInstr *Def = MRI->getVRegDef(Reg); in selectMUBUFScratchOffset()
4404 if (mi_match(Reg, *MRI, m_GPtrAdd(m_Reg(BasePtr), m_Copy(m_ICst(Offset))))) { in selectMUBUFScratchOffset()
4407 const MachineInstr *BasePtrDef = MRI->getVRegDef(BasePtr); in selectMUBUFScratchOffset()
4423 if (!mi_match(Root.getReg(), *MRI, m_ICst(Offset)) || in selectMUBUFScratchOffset()
4440 const MachineInstr *RootDef = MRI->getVRegDef(Root.getReg()); in selectDS1Addr1OffsetImpl()
4449 getPtrBaseWithConstantOffset(Root.getReg(), *MRI); in selectDS1Addr1OffsetImpl()
4460 } else if (mi_match(Root.getReg(), *MRI, m_ICst(ConstAddr))) { in selectDS1Addr1OffsetImpl()
4505 const MachineInstr *RootDef = MRI->getVRegDef(Root.getReg()); in selectDSReadWrite2Impl()
4514 getPtrBaseWithConstantOffset(Root.getReg(), *MRI); in selectDSReadWrite2Impl()
4526 } else if (mi_match(Root.getReg(), *MRI, m_ICst(ConstAddr))) { in selectDSReadWrite2Impl()
4540 Register Root, const MachineRegisterInfo &MRI) const { in getPtrBaseWithConstantOffset()
4541 MachineInstr *RootI = getDefIgnoringCopies(Root, MRI); in getPtrBaseWithConstantOffset()
4547 getIConstantVRegValWithLookThrough(RHS.getReg(), MRI); in getPtrBaseWithConstantOffset()
4559 static Register buildRSRC(MachineIRBuilder &B, MachineRegisterInfo &MRI, in buildRSRC() argument
4562 Register RSrc2 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); in buildRSRC()
4563 Register RSrc3 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); in buildRSRC()
4564 Register RSrcHi = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in buildRSRC()
4565 Register RSrc = MRI.createVirtualRegister(&AMDGPU::SGPR_128RegClass); in buildRSRC()
4586 RSrcLo = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in buildRSRC()
4602 static Register buildAddr64RSrc(MachineIRBuilder &B, MachineRegisterInfo &MRI, in buildAddr64RSrc() argument
4608 return buildRSRC(B, MRI, 0, Hi_32(DefaultFormat), BasePtr); in buildAddr64RSrc()
4611 static Register buildOffsetSrc(MachineIRBuilder &B, MachineRegisterInfo &MRI, in buildOffsetSrc() argument
4617 return buildRSRC(B, MRI, -1, Hi_32(DefaultFormat), BasePtr); in buildOffsetSrc()
4628 std::tie(PtrBase, Offset) = getPtrBaseWithConstantOffset(Src, *MRI); in parseMUBUFAddress()
4635 = getOpcodeDef(TargetOpcode::G_PTR_ADD, Data.N0, *MRI)) { in parseMUBUFAddress()
4644 Data.N2 = getDefIgnoringCopies(Data.N2, *MRI)->getOperand(0).getReg(); in parseMUBUFAddress()
4645 Data.N3 = getDefIgnoringCopies(Data.N3, *MRI)->getOperand(0).getReg(); in parseMUBUFAddress()
4658 const RegisterBank *N0Bank = RBI.getRegBank(Addr.N0, *MRI, TRI); in shouldUseAddr64()
4671 SOffset = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in splitIllegalMUBUFOffset()
4699 if (RBI.getRegBank(N2, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) { in selectMUBUFAddr64Impl()
4701 if (RBI.getRegBank(N3, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) { in selectMUBUFAddr64Impl()
4714 } else if (RBI.getRegBank(N0, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) { in selectMUBUFAddr64Impl()
4724 RSrcReg = buildAddr64RSrc(B, *MRI, TII, SRDPtr); in selectMUBUFAddr64Impl()
4749 RSrcReg = buildOffsetSrc(B, *MRI, TII, SRDPtr); in selectMUBUFOffsetImpl()
4874 const MachineRegisterInfo &MRI) { in getConstantZext32Val() argument
4876 Optional<int64_t> OffsetVal = getIConstantVRegSExtVal(Reg, MRI); in getConstantZext32Val()
4884 Optional<uint64_t> OffsetVal = getConstantZext32Val(Root.getReg(), *MRI); in selectSMRDBufferImm()
4900 Optional<uint64_t> OffsetVal = getConstantZext32Val(Root.getReg(), *MRI); in selectSMRDBufferImm32()