Lines Matching refs:MRI
67 MRI = &MF.getRegInfo(); in setupMF()
73 const MachineRegisterInfo &MRI) const { in isVCC()
78 auto &RegClassOrBank = MRI.getRegClassOrRegBank(Reg); in isVCC()
82 const LLT Ty = MRI.getType(Reg); in isVCC()
101 if (MRI->getType(Dst.getReg()) == LLT::scalar(1)) in constrainCopyLikeIntrin()
105 = TRI.getConstrainedRegClassForOperand(Dst, *MRI); in constrainCopyLikeIntrin()
107 = TRI.getConstrainedRegClassForOperand(Src, *MRI); in constrainCopyLikeIntrin()
111 return RBI.constrainGenericRegister(Dst.getReg(), *DstRC, *MRI) && in constrainCopyLikeIntrin()
112 RBI.constrainGenericRegister(Src.getReg(), *SrcRC, *MRI); in constrainCopyLikeIntrin()
125 if (isVCC(DstReg, *MRI)) { in selectCOPY()
128 = TRI.getConstrainedRegClassForOperand(Dst, *MRI); in selectCOPY()
131 return RBI.constrainGenericRegister(DstReg, *RC, *MRI); in selectCOPY()
134 if (!isVCC(SrcReg, *MRI)) { in selectCOPY()
136 if (!RBI.constrainGenericRegister(DstReg, *TRI.getBoolRC(), *MRI)) in selectCOPY()
140 = TRI.getConstrainedRegClassForOperand(Src, *MRI); in selectCOPY()
143 getConstantVRegValWithLookThrough(SrcReg, *MRI, true, true); in selectCOPY()
150 Register MaskedReg = MRI->createVirtualRegister(SrcRC); in selectCOPY()
166 if (!MRI->getRegClassOrNull(SrcReg)) in selectCOPY()
167 MRI->setRegClass(SrcReg, SrcRC); in selectCOPY()
173 TRI.getConstrainedRegClassForOperand(Dst, *MRI); in selectCOPY()
174 if (RC && !RBI.constrainGenericRegister(DstReg, *RC, *MRI)) in selectCOPY()
185 TRI.getConstrainedRegClassForOperand(MO, *MRI); in selectCOPY()
188 RBI.constrainGenericRegister(MO.getReg(), *RC, *MRI); in selectCOPY()
195 const LLT DefTy = MRI->getType(DefReg); in selectPHI()
208 MRI->getRegClassOrRegBank(DefReg); in selectPHI()
219 DefRC = TRI.getRegClassForTypeOnBank(DefTy, RB, *MRI); in selectPHI()
228 return RBI.constrainGenericRegister(DefReg, *DefRC, *MRI); in selectPHI()
238 Register DstReg = MRI->createVirtualRegister(&SubRC); in getSubOperand64()
281 unsigned Size = RBI.getSizeInBits(DstReg, *MRI, TRI); in selectG_AND_OR_XOR()
283 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI); in selectG_AND_OR_XOR()
305 LLT Ty = MRI->getType(DstReg); in selectG_ADD_SUB()
310 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI); in selectG_ADD_SUB()
335 Register UnusedCarry = MRI->createVirtualRegister(TRI.getWaveMaskRegClass()); in selectG_ADD_SUB()
358 Register DstLo = MRI->createVirtualRegister(&HalfRC); in selectG_ADD_SUB()
359 Register DstHi = MRI->createVirtualRegister(&HalfRC); in selectG_ADD_SUB()
370 Register CarryReg = MRI->createVirtualRegister(CarryRC); in selectG_ADD_SUB()
377 .addDef(MRI->createVirtualRegister(CarryRC), RegState::Dead) in selectG_ADD_SUB()
394 if (!RBI.constrainGenericRegister(DstReg, RC, *MRI)) in selectG_ADD_SUB()
413 if (isVCC(Dst1Reg, *MRI)) { in selectG_UADDO_USUBO_UADDE_USUBE()
440 if (!MRI->getRegClassOrNull(Dst1Reg)) in selectG_UADDO_USUBO_UADDE_USUBE()
441 MRI->setRegClass(Dst1Reg, &AMDGPU::SReg_32RegClass); in selectG_UADDO_USUBO_UADDE_USUBE()
443 if (!RBI.constrainGenericRegister(Dst0Reg, AMDGPU::SReg_32RegClass, *MRI) || in selectG_UADDO_USUBO_UADDE_USUBE()
444 !RBI.constrainGenericRegister(Src0Reg, AMDGPU::SReg_32RegClass, *MRI) || in selectG_UADDO_USUBO_UADDE_USUBE()
445 !RBI.constrainGenericRegister(Src1Reg, AMDGPU::SReg_32RegClass, *MRI)) in selectG_UADDO_USUBO_UADDE_USUBE()
450 AMDGPU::SReg_32RegClass, *MRI)) in selectG_UADDO_USUBO_UADDE_USUBE()
462 LLT DstTy = MRI->getType(DstReg); in selectG_EXTRACT()
463 LLT SrcTy = MRI->getType(SrcReg); in selectG_EXTRACT()
478 TRI.getConstrainedRegClassForOperand(I.getOperand(0), *MRI); in selectG_EXTRACT()
479 if (!DstRC || !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) in selectG_EXTRACT()
482 const RegisterBank *SrcBank = RBI.getRegBank(SrcReg, *MRI, TRI); in selectG_EXTRACT()
484 TRI.getRegClassForSizeOnBank(SrcSize, *SrcBank, *MRI); in selectG_EXTRACT()
493 SrcReg = constrainOperandRegClass(*MF, TRI, *MRI, TII, RBI, I, in selectG_EXTRACT()
506 LLT DstTy = MRI->getType(DstReg); in selectG_MERGE_VALUES()
507 LLT SrcTy = MRI->getType(MI.getOperand(1).getReg()); in selectG_MERGE_VALUES()
514 const RegisterBank *DstBank = RBI.getRegBank(DstReg, *MRI, TRI); in selectG_MERGE_VALUES()
517 TRI.getRegClassForSizeOnBank(DstSize, *DstBank, *MRI); in selectG_MERGE_VALUES()
530 = TRI.getConstrainedRegClassForOperand(Src, *MRI); in selectG_MERGE_VALUES()
531 if (SrcRC && !RBI.constrainGenericRegister(Src.getReg(), *SrcRC, *MRI)) in selectG_MERGE_VALUES()
535 if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) in selectG_MERGE_VALUES()
550 LLT DstTy = MRI->getType(DstReg0); in selectG_UNMERGE_VALUES()
551 LLT SrcTy = MRI->getType(SrcReg); in selectG_UNMERGE_VALUES()
556 const RegisterBank *SrcBank = RBI.getRegBank(SrcReg, *MRI, TRI); in selectG_UNMERGE_VALUES()
559 TRI.getRegClassForSizeOnBank(SrcSize, *SrcBank, *MRI); in selectG_UNMERGE_VALUES()
560 if (!SrcRC || !RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI)) in selectG_UNMERGE_VALUES()
574 if (!SrcRC || !RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI)) in selectG_UNMERGE_VALUES()
578 TRI.getConstrainedRegClassForOperand(Dst, *MRI); in selectG_UNMERGE_VALUES()
579 if (DstRC && !RBI.constrainGenericRegister(Dst.getReg(), *DstRC, *MRI)) in selectG_UNMERGE_VALUES()
596 if (MRI->getType(Dst) != V2S16) in selectG_BUILD_VECTOR_TRUNC()
599 const RegisterBank *DstBank = RBI.getRegBank(Dst, *MRI, TRI); in selectG_BUILD_VECTOR_TRUNC()
605 if (MRI->getType(Src0) != S32) in selectG_BUILD_VECTOR_TRUNC()
612 getConstantVRegValWithLookThrough(Src1, *MRI, true, true, true); in selectG_BUILD_VECTOR_TRUNC()
615 getConstantVRegValWithLookThrough(Src0, *MRI, true, true, true); in selectG_BUILD_VECTOR_TRUNC()
625 return RBI.constrainGenericRegister(Dst, AMDGPU::SReg_32RegClass, *MRI); in selectG_BUILD_VECTOR_TRUNC()
631 MachineInstr *Src1Def = getDefIgnoringCopies(Src1, *MRI); in selectG_BUILD_VECTOR_TRUNC()
635 return RBI.constrainGenericRegister(Dst, AMDGPU::SReg_32RegClass, *MRI) && in selectG_BUILD_VECTOR_TRUNC()
636 RBI.constrainGenericRegister(Src0, AMDGPU::SReg_32RegClass, *MRI); in selectG_BUILD_VECTOR_TRUNC()
653 Src0, *MRI, m_OneUse(m_GLShr(m_Reg(ShiftSrc0), m_SpecificICst(16)))); in selectG_BUILD_VECTOR_TRUNC()
656 Src1, *MRI, m_OneUse(m_GLShr(m_Reg(ShiftSrc1), m_SpecificICst(16)))); in selectG_BUILD_VECTOR_TRUNC()
689 const TargetRegisterClass *RC = TRI.getConstrainedRegClassForOperand(MO, *MRI); in selectG_IMPLICIT_DEF()
690 if ((!RC && !MRI->getRegBankOrNull(MO.getReg())) || in selectG_IMPLICIT_DEF()
691 (RC && RBI.constrainGenericRegister(MO.getReg(), *RC, *MRI))) { in selectG_IMPLICIT_DEF()
705 LLT Src1Ty = MRI->getType(Src1Reg); in selectG_INSERT()
707 unsigned DstSize = MRI->getType(DstReg).getSizeInBits(); in selectG_INSERT()
724 const RegisterBank *DstBank = RBI.getRegBank(DstReg, *MRI, TRI); in selectG_INSERT()
726 TRI.getRegClassForSizeOnBank(DstSize, *DstBank, *MRI); in selectG_INSERT()
730 const RegisterBank *Src0Bank = RBI.getRegBank(Src0Reg, *MRI, TRI); in selectG_INSERT()
731 const RegisterBank *Src1Bank = RBI.getRegBank(Src1Reg, *MRI, TRI); in selectG_INSERT()
733 TRI.getRegClassForSizeOnBank(DstSize, *Src0Bank, *MRI); in selectG_INSERT()
735 TRI.getRegClassForSizeOnBank(InsSize, *Src1Bank, *MRI); in selectG_INSERT()
743 if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI) || in selectG_INSERT()
744 !RBI.constrainGenericRegister(Src0Reg, *Src0RC, *MRI) || in selectG_INSERT()
745 !RBI.constrainGenericRegister(Src1Reg, *Src1RC, *MRI)) in selectG_INSERT()
764 assert(RBI.getRegBank(DstReg, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID && in selectG_SBFX_UBFX()
766 assert(MRI->getType(MI.getOperand(0).getReg()).getSizeInBits() == 32 && in selectG_SBFX_UBFX()
789 if (!RBI.constrainGenericRegister(M0Val, AMDGPU::SReg_32RegClass, *MRI) || in selectInterpP1F16()
790 !RBI.constrainGenericRegister(Dst, AMDGPU::VGPR_32RegClass, *MRI) || in selectInterpP1F16()
791 !RBI.constrainGenericRegister(Src0, AMDGPU::VGPR_32RegClass, *MRI)) in selectInterpP1F16()
801 Register InterpMov = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectInterpP1F16()
847 getConstantVRegValWithLookThrough(LaneSelect, *MRI, true, true); in selectWritelane()
856 getConstantVRegValWithLookThrough(Val, *MRI, true, true); in selectWritelane()
870 RBI.constrainGenericRegister(LaneSelect, AMDGPU::SReg_32_XM0RegClass, *MRI); in selectWritelane()
890 LLT Ty = MRI->getType(Dst0); in selectDivScale()
945 MRI->setRegClass(Reg, TRI.getWaveMaskRegClass()); in selectG_INTRINSIC()
1058 unsigned Size = RBI.getSizeInBits(SrcReg, *MRI, TRI); in selectG_ICMP()
1063 if (!isVCC(CCReg, *MRI)) { in selectG_ICMP()
1074 RBI.constrainGenericRegister(CCReg, AMDGPU::SReg_32RegClass, *MRI); in selectG_ICMP()
1088 *TRI.getBoolRC(), *MRI); in selectG_ICMP()
1096 if (isVCC(Dst, *MRI)) in selectIntrinsicIcmp()
1099 if (MRI->getType(Dst).getSizeInBits() != STI.getWavefrontSize()) in selectIntrinsicIcmp()
1105 unsigned Size = RBI.getSizeInBits(SrcReg, *MRI, TRI); in selectIntrinsicIcmp()
1116 *MRI); in selectIntrinsicIcmp()
1126 const unsigned Size = MRI->getType(DstReg).getSizeInBits(); in selectBallot()
1133 getConstantVRegValWithLookThrough(I.getOperand(2).getReg(), *MRI, true); in selectBallot()
1156 const RegisterBank *DstBank = RBI.getRegBank(DstReg, *MRI, TRI); in selectRelocConstant()
1158 TRI.getRegClassForSizeOnBank(32, *DstBank, *MRI); in selectRelocConstant()
1159 if (!DstRC || !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) in selectRelocConstant()
1183 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI); in selectGroupStaticSize()
1216 = TRI.getConstrainedRegClassForOperand(Dst, *MRI); in selectReturnAddress()
1218 !RBI.constrainGenericRegister(DstReg, *RC, *MRI)) in selectReturnAddress()
1254 if (!MRI->getRegClassOrNull(Reg)) in selectEndCfIntrinsic()
1255 MRI->setRegClass(Reg, TRI.getWaveMaskRegClass()); in selectEndCfIntrinsic()
1313 if (!RBI.constrainGenericRegister(M0Val, AMDGPU::SReg_32RegClass, *MRI)) in selectDSOrderedIntrinsic()
1351 const RegisterBank *OffsetRB = RBI.getRegBank(BaseOffset, *MRI, TRI); in selectDSGWSIntrinsic()
1355 MachineInstr *OffsetDef = getDefIgnoringCopies(BaseOffset, *MRI); in selectDSGWSIntrinsic()
1371 OffsetDef = getDefIgnoringCopies(BaseOffset, *MRI); in selectDSGWSIntrinsic()
1385 AMDGPU::getBaseWithConstantOffset(*MRI, BaseOffset); in selectDSGWSIntrinsic()
1390 if (!RBI.constrainGenericRegister(BaseOffset, AMDGPU::VGPR_32RegClass, *MRI)) in selectDSGWSIntrinsic()
1397 AMDGPU::SReg_32RegClass, *MRI)) in selectDSGWSIntrinsic()
1401 Register M0Base = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectDSGWSIntrinsic()
1420 Register Undef = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectDSGWSIntrinsic()
1423 MRI->createVirtualRegister(&AMDGPU::VReg_64_Align2RegClass); in selectDSGWSIntrinsic()
1435 if (!RBI.constrainGenericRegister(VSrc, AMDGPU::VGPR_32RegClass, *MRI)) in selectDSGWSIntrinsic()
1449 LLT PtrTy = MRI->getType(PtrBase); in selectDSAppendConsume()
1467 if (!RBI.constrainGenericRegister(PtrBase, AMDGPU::SReg_32RegClass, *MRI)) in selectDSAppendConsume()
1555 LLT Ty = MRI->getType(VDataIn); in selectImageIntrinsic()
1587 VDataTy = MRI->getType(VDataIn); in selectImageIntrinsic()
1591 VDataTy = MRI->getType(VDataOut); in selectImageIntrinsic()
1649 NumVAddrDwords += (MRI->getType(Addr).getSizeInBits() + 31) / 32; in selectImageIntrinsic()
1685 const bool Is64 = MRI->getType(VDataOut).getSizeInBits() == 64; in selectImageIntrinsic()
1687 Register TmpReg = MRI->createVirtualRegister( in selectImageIntrinsic()
1692 if (!MRI->use_empty(VDataOut)) { in selectImageIntrinsic()
1741 Register Tied = MRI->cloneVirtualRegister(VDataOut); in selectImageIntrinsic()
1742 Register Zero = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectImageIntrinsic()
1745 auto Parts = TRI.getRegSplitParts(MRI->getRegClass(Tied), 4); in selectImageIntrinsic()
1756 Register Undef = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectImageIntrinsic()
1810 unsigned Size = RBI.getSizeInBits(DstReg, *MRI, TRI); in selectG_SELECT()
1814 if (!isVCC(CCReg, *MRI)) { in selectG_SELECT()
1823 if (!MRI->getRegClassOrNull(CCReg)) in selectG_SELECT()
1824 MRI->setRegClass(CCReg, TRI.getConstrainedRegClassForOperand(CCOp, *MRI)); in selectG_SELECT()
1876 const LLT DstTy = MRI->getType(DstReg); in selectG_TRUNC()
1877 const LLT SrcTy = MRI->getType(SrcReg); in selectG_TRUNC()
1880 const RegisterBank *SrcRB = RBI.getRegBank(SrcReg, *MRI, TRI); in selectG_TRUNC()
1887 DstRB = RBI.getRegBank(DstReg, *MRI, TRI); in selectG_TRUNC()
1898 = TRI.getRegClassForSizeOnBank(SrcSize, *SrcRB, *MRI); in selectG_TRUNC()
1900 = TRI.getRegClassForSizeOnBank(DstSize, *DstRB, *MRI); in selectG_TRUNC()
1904 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI) || in selectG_TRUNC()
1905 !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) { in selectG_TRUNC()
1914 Register LoReg = MRI->createVirtualRegister(DstRC); in selectG_TRUNC()
1915 Register HiReg = MRI->createVirtualRegister(DstRC); in selectG_TRUNC()
1935 Register TmpReg0 = MRI->createVirtualRegister(DstRC); in selectG_TRUNC()
1936 Register TmpReg1 = MRI->createVirtualRegister(DstRC); in selectG_TRUNC()
1937 Register ImmReg = MRI->createVirtualRegister(DstRC); in selectG_TRUNC()
1982 if (!RBI.constrainGenericRegister(SrcReg, *SrcWithSubRC, *MRI)) in selectG_TRUNC()
2002 Register Reg, const MachineRegisterInfo &MRI, in getArtifactRegBank() argument
2004 const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(Reg); in getArtifactRegBank()
2022 const LLT DstTy = MRI->getType(DstReg); in selectG_SZA_EXT()
2023 const LLT SrcTy = MRI->getType(SrcReg); in selectG_SZA_EXT()
2031 const RegisterBank *SrcBank = getArtifactRegBank(SrcReg, *MRI, TRI); in selectG_SZA_EXT()
2039 TRI.getRegClassForTypeOnBank(SrcTy, *SrcBank, *MRI); in selectG_SZA_EXT()
2040 const RegisterBank *DstBank = RBI.getRegBank(DstReg, *MRI, TRI); in selectG_SZA_EXT()
2042 TRI.getRegClassForSizeOnBank(DstSize, *DstBank, *MRI); in selectG_SZA_EXT()
2044 Register UndefReg = MRI->createVirtualRegister(SrcRC); in selectG_SZA_EXT()
2053 return RBI.constrainGenericRegister(DstReg, *DstRC, *MRI) && in selectG_SZA_EXT()
2054 RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI); in selectG_SZA_EXT()
2084 if (!RBI.constrainGenericRegister(SrcReg, SrcRC, *MRI)) in selectG_SZA_EXT()
2093 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_32RegClass, *MRI); in selectG_SZA_EXT()
2102 Register ExtReg = MRI->createVirtualRegister(&AMDGPU::SReg_64RegClass); in selectG_SZA_EXT()
2103 Register UndefReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_SZA_EXT()
2118 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_64RegClass, *MRI); in selectG_SZA_EXT()
2133 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_32RegClass, *MRI); in selectG_SZA_EXT()
2143 unsigned Size = MRI->getType(DstReg).getSizeInBits(); in selectG_CONSTANT()
2155 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI); in selectG_CONSTANT()
2188 Register LoReg = MRI->createVirtualRegister(RC); in selectG_CONSTANT()
2189 Register HiReg = MRI->createVirtualRegister(RC); in selectG_CONSTANT()
2208 TRI.getConstrainedRegClassForOperand(ResInst->getOperand(0), *MRI); in selectG_CONSTANT()
2211 return RBI.constrainGenericRegister(DstReg, *DstRC, *MRI); in selectG_CONSTANT()
2227 const RegisterBank *DstRB = RBI.getRegBank(Dst, *MRI, TRI); in selectG_FNEG()
2229 MRI->getType(Dst) != LLT::scalar(64)) in selectG_FNEG()
2233 MachineInstr *Fabs = getOpcodeDef(TargetOpcode::G_FABS, Src, *MRI); in selectG_FNEG()
2237 if (!RBI.constrainGenericRegister(Src, AMDGPU::SReg_64RegClass, *MRI) || in selectG_FNEG()
2238 !RBI.constrainGenericRegister(Dst, AMDGPU::SReg_64RegClass, *MRI)) in selectG_FNEG()
2243 Register LoReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FNEG()
2244 Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FNEG()
2245 Register ConstReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FNEG()
2246 Register OpReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FNEG()
2272 const RegisterBank *DstRB = RBI.getRegBank(Dst, *MRI, TRI); in selectG_FABS()
2274 MRI->getType(Dst) != LLT::scalar(64)) in selectG_FABS()
2280 Register LoReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FABS()
2281 Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FABS()
2282 Register ConstReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FABS()
2283 Register OpReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FABS()
2285 if (!RBI.constrainGenericRegister(Src, AMDGPU::SReg_64RegClass, *MRI) || in selectG_FABS()
2286 !RBI.constrainGenericRegister(Dst, AMDGPU::SReg_64RegClass, *MRI)) in selectG_FABS()
2316 const MachineRegisterInfo &MRI, SmallVectorImpl<GEPInfo> &AddrInfo) const { in getAddrModeInfo() argument
2318 const MachineInstr *PtrMI = MRI.getUniqueVRegDef(Load.getOperand(1).getReg()); in getAddrModeInfo()
2329 const MachineInstr *OpDef = MRI.getUniqueVRegDef(GEPOp.getReg()); in getAddrModeInfo()
2338 const RegisterBank *OpBank = RBI.getRegBank(GEPOp.getReg(), MRI, TRI); in getAddrModeInfo()
2346 getAddrModeInfo(*PtrMI, MRI, AddrInfo); in getAddrModeInfo()
2350 return RBI.getRegBank(Reg, *MRI, TRI)->getID() == AMDGPU::SGPRRegBankID; in isSGPR()
2384 const LLT PtrTy = MRI->getType(I.getOperand(1).getReg()); in initM0()
2399 const LLT PtrTy = MRI->getType(I.getOperand(1).getReg()); in selectG_LOAD_STORE_ATOMICRMW()
2413 const LLT PtrTy = MRI->getType(PtrReg); in selectG_AMDGPU_ATOMIC_CMPXCHG()
2419 const LLT Ty = MRI->getType(DstReg); in selectG_AMDGPU_ATOMIC_CMPXCHG()
2422 Register TmpReg = MRI->createVirtualRegister( in selectG_AMDGPU_ATOMIC_CMPXCHG()
2463 MRI->setRegClass( in selectG_AMDGPU_ATOMIC_CMPXCHG()
2483 if (!isVCC(CondReg, *MRI)) { in selectG_BRCOND()
2484 if (MRI->getType(CondReg) != LLT::scalar(32)) in selectG_BRCOND()
2501 if (!MRI->getRegClassOrNull(CondReg)) in selectG_BRCOND()
2502 MRI->setRegClass(CondReg, ConstrainRC); in selectG_BRCOND()
2516 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI); in selectG_GLOBAL_VALUE()
2523 DstReg, IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass, *MRI); in selectG_GLOBAL_VALUE()
2530 LLT Ty = MRI->getType(DstReg); in selectG_PTRMASK()
2531 LLT MaskTy = MRI->getType(MaskReg); in selectG_PTRMASK()
2533 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI); in selectG_PTRMASK()
2534 const RegisterBank *SrcRB = RBI.getRegBank(SrcReg, *MRI, TRI); in selectG_PTRMASK()
2535 const RegisterBank *MaskRB = RBI.getRegBank(MaskReg, *MRI, TRI); in selectG_PTRMASK()
2545 *MRI); in selectG_PTRMASK()
2547 *MRI); in selectG_PTRMASK()
2549 TRI.getRegClassForTypeOnBank(MaskTy, *MaskRB, *MRI); in selectG_PTRMASK()
2551 if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI) || in selectG_PTRMASK()
2552 !RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI) || in selectG_PTRMASK()
2553 !RBI.constrainGenericRegister(MaskReg, *MaskRC, *MRI)) in selectG_PTRMASK()
2569 Register HiReg = MRI->createVirtualRegister(&RegRC); in selectG_PTRMASK()
2570 Register LoReg = MRI->createVirtualRegister(&RegRC); in selectG_PTRMASK()
2591 Register MaskLo = MRI->createVirtualRegister(&RegRC); in selectG_PTRMASK()
2592 MaskedLo = MRI->createVirtualRegister(&RegRC); in selectG_PTRMASK()
2605 Register MaskHi = MRI->createVirtualRegister(&RegRC); in selectG_PTRMASK()
2606 MaskedHi = MRI->createVirtualRegister(&RegRC); in selectG_PTRMASK()
2627 computeIndirectRegIndex(MachineRegisterInfo &MRI, in computeIndirectRegIndex() argument
2635 std::tie(IdxBaseReg, Offset) = AMDGPU::getBaseWithConstantOffset(MRI, IdxReg); in computeIndirectRegIndex()
2658 LLT DstTy = MRI->getType(DstReg); in selectG_EXTRACT_VECTOR_ELT()
2659 LLT SrcTy = MRI->getType(SrcReg); in selectG_EXTRACT_VECTOR_ELT()
2661 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI); in selectG_EXTRACT_VECTOR_ELT()
2662 const RegisterBank *SrcRB = RBI.getRegBank(SrcReg, *MRI, TRI); in selectG_EXTRACT_VECTOR_ELT()
2663 const RegisterBank *IdxRB = RBI.getRegBank(IdxReg, *MRI, TRI); in selectG_EXTRACT_VECTOR_ELT()
2671 *MRI); in selectG_EXTRACT_VECTOR_ELT()
2673 *MRI); in selectG_EXTRACT_VECTOR_ELT()
2676 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI) || in selectG_EXTRACT_VECTOR_ELT()
2677 !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI) || in selectG_EXTRACT_VECTOR_ELT()
2678 !RBI.constrainGenericRegister(IdxReg, AMDGPU::SReg_32RegClass, *MRI)) in selectG_EXTRACT_VECTOR_ELT()
2686 std::tie(IdxReg, SubReg) = computeIndirectRegIndex(*MRI, TRI, SrcRC, IdxReg, in selectG_EXTRACT_VECTOR_ELT()
2736 LLT VecTy = MRI->getType(DstReg); in selectG_INSERT_VECTOR_ELT()
2737 LLT ValTy = MRI->getType(ValReg); in selectG_INSERT_VECTOR_ELT()
2741 const RegisterBank *VecRB = RBI.getRegBank(VecReg, *MRI, TRI); in selectG_INSERT_VECTOR_ELT()
2742 const RegisterBank *ValRB = RBI.getRegBank(ValReg, *MRI, TRI); in selectG_INSERT_VECTOR_ELT()
2743 const RegisterBank *IdxRB = RBI.getRegBank(IdxReg, *MRI, TRI); in selectG_INSERT_VECTOR_ELT()
2753 *MRI); in selectG_INSERT_VECTOR_ELT()
2755 *MRI); in selectG_INSERT_VECTOR_ELT()
2757 if (!RBI.constrainGenericRegister(VecReg, *VecRC, *MRI) || in selectG_INSERT_VECTOR_ELT()
2758 !RBI.constrainGenericRegister(DstReg, *VecRC, *MRI) || in selectG_INSERT_VECTOR_ELT()
2759 !RBI.constrainGenericRegister(ValReg, *ValRC, *MRI) || in selectG_INSERT_VECTOR_ELT()
2760 !RBI.constrainGenericRegister(IdxReg, AMDGPU::SReg_32RegClass, *MRI)) in selectG_INSERT_VECTOR_ELT()
2767 std::tie(IdxReg, SubReg) = computeIndirectRegIndex(*MRI, TRI, VecRC, IdxReg, in selectG_INSERT_VECTOR_ELT()
2841 if (MRI->getType(DstReg) != V2S16 || MRI->getType(Src0Reg) != V2S16) in selectG_SHUFFLE_VECTOR()
2853 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI); in selectG_SHUFFLE_VECTOR()
2863 return RBI.constrainGenericRegister(DstReg, RC, *MRI); in selectG_SHUFFLE_VECTOR()
2870 if (!RBI.constrainGenericRegister(DstReg, RC, *MRI) || in selectG_SHUFFLE_VECTOR()
2871 !RBI.constrainGenericRegister(SrcVec, RC, *MRI)) in selectG_SHUFFLE_VECTOR()
2946 Register TmpReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_SHUFFLE_VECTOR()
2969 if (!MRI->use_nodbg_empty(MI.getOperand(0).getReg())) { in selectAMDGPU_BUFFER_ATOMIC_FADD()
2987 bool HasVOffset = !isOperandImmEqual(VOffset, 0, *MRI); in selectAMDGPU_BUFFER_ATOMIC_FADD()
2988 bool HasVIndex = !isOperandImmEqual(VIndex, 0, *MRI); in selectAMDGPU_BUFFER_ATOMIC_FADD()
2999 if (MRI->getType(VDataIn.getReg()).isVector()) { in selectAMDGPU_BUFFER_ATOMIC_FADD()
3021 Register IdxReg = MRI->createVirtualRegister(TRI.getVGPR64Class()); in selectAMDGPU_BUFFER_ATOMIC_FADD()
3058 if (!MRI->use_nodbg_empty(MI.getOperand(0).getReg())) { in selectGlobalAtomicFadd()
3073 const unsigned Opc = MRI->getType(Data).isVector() ? in selectGlobalAtomicFadd()
3239 MachineInstr *MI = getDefIgnoringCopies(Src, *MRI); in selectVOP3ModsImpl()
3244 MI = getDefIgnoringCopies(Src, *MRI); in selectVOP3ModsImpl()
3253 RBI.getRegBank(Src, *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID) { in selectVOP3ModsImpl()
3259 Register VGPRSrc = MRI->cloneVirtualRegister(OrigSrc); in selectVOP3ModsImpl()
3343 const MachineInstr *Def = getDefIgnoringCopies(Reg, *MRI); in selectVOP3NoMods()
3354 Register Src, const MachineRegisterInfo &MRI) const { in selectVOP3PModsImpl()
3356 MachineInstr *MI = MRI.getVRegDef(Src); in selectVOP3PModsImpl()
3361 MRI.getType(Src) == LLT::fixed_vector(2, 16)) { in selectVOP3PModsImpl()
3364 MI = MRI.getVRegDef(Src); in selectVOP3PModsImpl()
3377 MachineRegisterInfo &MRI in selectVOP3PMods() local
3382 std::tie(Src, Mods) = selectVOP3PModsImpl(Root.getReg(), MRI); in selectVOP3PMods()
3395 if (!isKnownNeverNaN(Src, *MRI)) in selectVOP3Mods_nnan()
3416 getAddrModeInfo(*Root.getParent(), *MRI, AddrInfo); in selectSmrdImm()
3437 getAddrModeInfo(*Root.getParent(), *MRI, AddrInfo); in selectSmrdImm32()
3461 getAddrModeInfo(*MI, *MRI, AddrInfo); in selectSmrdSgpr()
3478 Register OffsetReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectSmrdSgpr()
3500 getPtrBaseWithConstantOffset(Root.getReg(), *MRI); in selectFlatOffsetImpl()
3542 static Register matchZeroExtendFromS32(MachineRegisterInfo &MRI, Register Reg) { in matchZeroExtendFromS32() argument
3544 if (mi_match(Reg, MRI, m_GZExt(m_Reg(ZExtSrc)))) in matchZeroExtendFromS32()
3545 return MRI.getType(ZExtSrc) == LLT::scalar(32) ? ZExtSrc : Register(); in matchZeroExtendFromS32()
3548 const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI); in matchZeroExtendFromS32()
3552 if (mi_match(Def->getOperand(2).getReg(), MRI, m_ZeroInt())) { in matchZeroExtendFromS32()
3569 std::tie(PtrBase, ConstOffset) = getPtrBaseWithConstantOffset(Addr, *MRI); in selectGlobalSAddr()
3577 auto PtrBaseDef = getDefSrcRegIgnoringCopies(PtrBase, *MRI); in selectGlobalSAddr()
3596 MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectGlobalSAddr()
3626 auto AddrDef = getDefSrcRegIgnoringCopies(Addr, *MRI); in selectGlobalSAddr()
3634 getSrcRegIgnoringCopies(AddrDef->MI->getOperand(1).getReg(), *MRI); in selectGlobalSAddr()
3641 if (Register VOffset = matchZeroExtendFromS32(*MRI, PtrBaseOffset)) { in selectGlobalSAddr()
3665 Register VOffset = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectGlobalSAddr()
3686 std::tie(PtrBase, ConstOffset) = getPtrBaseWithConstantOffset(Addr, *MRI); in selectScratchSAddr()
3695 auto AddrDef = getDefSrcRegIgnoringCopies(Addr, *MRI); in selectScratchSAddr()
3712 auto LHSDef = getDefSrcRegIgnoringCopies(LHS, *MRI); in selectScratchSAddr()
3713 auto RHSDef = getDefSrcRegIgnoringCopies(RHS, *MRI); in selectScratchSAddr()
3722 SAddr = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectScratchSAddr()
3747 if (mi_match(Root.getReg(), *MRI, m_ICst(Offset)) && in selectMUBUFScratchOffen()
3749 Register HighBits = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectMUBUFScratchOffen()
3779 if (const MachineInstr *RootDef = MRI->getVRegDef(Root.getReg())) { in selectMUBUFScratchOffen()
3782 std::tie(PtrBase, ConstOffset) = getPtrBaseWithConstantOffset(VAddr, *MRI); in selectMUBUFScratchOffen()
3787 const MachineInstr *PtrBaseDef = MRI->getVRegDef(PtrBase); in selectMUBUFScratchOffen()
3854 if (!mi_match(Root.getReg(), *MRI, m_ICst(Offset)) || in selectMUBUFScratchOffset()
3874 const MachineInstr *RootDef = MRI->getVRegDef(Root.getReg()); in selectDS1Addr1OffsetImpl()
3883 getPtrBaseWithConstantOffset(Root.getReg(), *MRI); in selectDS1Addr1OffsetImpl()
3894 } else if (mi_match(Root.getReg(), *MRI, m_ICst(ConstAddr))) { in selectDS1Addr1OffsetImpl()
3939 const MachineInstr *RootDef = MRI->getVRegDef(Root.getReg()); in selectDSReadWrite2Impl()
3948 getPtrBaseWithConstantOffset(Root.getReg(), *MRI); in selectDSReadWrite2Impl()
3960 } else if (mi_match(Root.getReg(), *MRI, m_ICst(ConstAddr))) { in selectDSReadWrite2Impl()
3974 Register Root, const MachineRegisterInfo &MRI) const { in getPtrBaseWithConstantOffset()
3975 MachineInstr *RootI = getDefIgnoringCopies(Root, MRI); in getPtrBaseWithConstantOffset()
3981 = getConstantVRegValWithLookThrough(RHS.getReg(), MRI, true); in getPtrBaseWithConstantOffset()
3993 static Register buildRSRC(MachineIRBuilder &B, MachineRegisterInfo &MRI, in buildRSRC() argument
3996 Register RSrc2 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); in buildRSRC()
3997 Register RSrc3 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); in buildRSRC()
3998 Register RSrcHi = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in buildRSRC()
3999 Register RSrc = MRI.createVirtualRegister(&AMDGPU::SGPR_128RegClass); in buildRSRC()
4020 RSrcLo = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in buildRSRC()
4036 static Register buildAddr64RSrc(MachineIRBuilder &B, MachineRegisterInfo &MRI, in buildAddr64RSrc() argument
4042 return buildRSRC(B, MRI, 0, Hi_32(DefaultFormat), BasePtr); in buildAddr64RSrc()
4045 static Register buildOffsetSrc(MachineIRBuilder &B, MachineRegisterInfo &MRI, in buildOffsetSrc() argument
4051 return buildRSRC(B, MRI, -1, Hi_32(DefaultFormat), BasePtr); in buildOffsetSrc()
4062 std::tie(PtrBase, Offset) = getPtrBaseWithConstantOffset(Src, *MRI); in parseMUBUFAddress()
4069 = getOpcodeDef(TargetOpcode::G_PTR_ADD, Data.N0, *MRI)) { in parseMUBUFAddress()
4078 Data.N2 = getDefIgnoringCopies(Data.N2, *MRI)->getOperand(0).getReg(); in parseMUBUFAddress()
4079 Data.N3 = getDefIgnoringCopies(Data.N3, *MRI)->getOperand(0).getReg(); in parseMUBUFAddress()
4092 const RegisterBank *N0Bank = RBI.getRegBank(Addr.N0, *MRI, TRI); in shouldUseAddr64()
4105 SOffset = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in splitIllegalMUBUFOffset()
4133 if (RBI.getRegBank(N2, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) { in selectMUBUFAddr64Impl()
4135 if (RBI.getRegBank(N3, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) { in selectMUBUFAddr64Impl()
4148 } else if (RBI.getRegBank(N0, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) { in selectMUBUFAddr64Impl()
4158 RSrcReg = buildAddr64RSrc(B, *MRI, TII, SRDPtr); in selectMUBUFAddr64Impl()
4183 RSrcReg = buildOffsetSrc(B, *MRI, TII, SRDPtr); in selectMUBUFOffsetImpl()
4308 const MachineRegisterInfo &MRI) { in getConstantZext32Val() argument
4310 Optional<int64_t> OffsetVal = getConstantVRegSExtVal(Reg, MRI); in getConstantZext32Val()
4318 Optional<uint64_t> OffsetVal = getConstantZext32Val(Root.getReg(), *MRI); in selectSMRDBufferImm()
4334 Optional<uint64_t> OffsetVal = getConstantZext32Val(Root.getReg(), *MRI); in selectSMRDBufferImm32()