Lines Matching refs:MRI

57     : Builder(B), MRI(Builder.getMF().getRegInfo()), Observer(Observer), KB(KB),  in CombinerHelper()
80 auto &MRI = *MIB.getMRI(); in buildLogBase2() local
81 LLT Ty = MRI.getType(V); in buildLogBase2()
161 void CombinerHelper::replaceRegWith(MachineRegisterInfo &MRI, Register FromReg, in replaceRegWith() argument
163 Observer.changingAllUsesOfReg(MRI, FromReg); in replaceRegWith()
165 if (MRI.constrainRegAttrs(ToReg, FromReg)) in replaceRegWith()
166 MRI.replaceRegWith(FromReg, ToReg); in replaceRegWith()
173 void CombinerHelper::replaceRegOpWith(MachineRegisterInfo &MRI, in replaceRegOpWith() argument
194 return RBI->getRegBank(Reg, MRI, *TRI); in getRegBank()
199 MRI.setRegBank(Reg, *RegBank); in setRegBank()
214 return canReplaceReg(DstReg, SrcReg, MRI); in matchCombineCopy()
220 replaceRegWith(MRI, DstReg, SrcReg); in applyCombineCopy()
245 MachineInstr *Def = MRI.getVRegDef(Reg); in matchCombineConcatVectors()
256 LLT OpType = MRI.getType(Reg); in matchCombineConcatVectors()
262 assert(MRI.getType(Undef->getOperand(0).getReg()) == in matchCombineConcatVectors()
284 Register NewDstReg = MRI.cloneVirtualRegister(DstReg); in applyCombineConcatVectors()
297 replaceRegWith(MRI, DstReg, NewDstReg); in applyCombineConcatVectors()
313 LLT DstType = MRI.getType(MI.getOperand(0).getReg()); in matchCombineShuffleVector()
315 LLT SrcType = MRI.getType(Src1); in matchCombineShuffleVector()
389 Register NewDstReg = MRI.cloneVirtualRegister(DstReg); in applyCombineShuffleVector()
397 replaceRegWith(MRI, DstReg, NewDstReg); in applyCombineShuffleVector()
414 LLT Src1Ty = MRI.getType(Src1); in applyShuffleToExtract()
425 else if (!MRI.getType(SrcReg).isVector()) in applyShuffleToExtract()
562 LLT LoadValueTy = MRI.getType(LoadReg); in matchCombineExtendingLoads()
589 for (auto &UseMI : MRI.use_nodbg_instructions(LoadReg)) { in matchCombineExtendingLoads()
601 LLT UseTy = MRI.getType(UseMI.getOperand(0).getReg()); in matchCombineExtendingLoads()
602 LLT SrcTy = MRI.getType(LoadMI->getPointerReg()); in matchCombineExtendingLoads()
608 MRI.getType(UseMI.getOperand(0).getReg()), in matchCombineExtendingLoads()
644 Register NewDstReg = MRI.cloneVirtualRegister(MI.getOperand(0).getReg()); in applyCombineExtendingLoads()
647 replaceRegOpWith(MRI, UseMO, NewDstReg); in applyCombineExtendingLoads()
657 for (auto &UseMO : MRI.use_operands(LoadValue.getReg())) in applyCombineExtendingLoads()
669 const LLT UseDstTy = MRI.getType(UseDstReg); in applyCombineExtendingLoads()
681 replaceRegWith(MRI, UseDstReg, ChosenDstReg); in applyCombineExtendingLoads()
695 replaceRegOpWith(MRI, UseSrcMO, ChosenDstReg); in applyCombineExtendingLoads()
743 if (MRI.getType(Dst).isVector()) in matchCombineLoadWithAndMask()
747 getIConstantVRegValWithLookThrough(MI.getOperand(2).getReg(), MRI); in matchCombineLoadWithAndMask()
759 GAnyLoad *LoadMI = dyn_cast<GAnyLoad>(MRI.getVRegDef(SrcReg)); in matchCombineLoadWithAndMask()
760 if (!LoadMI || !MRI.hasOneNonDBGUse(LoadMI->getDstReg())) in matchCombineLoadWithAndMask()
764 LLT RegTy = MRI.getType(LoadReg); in matchCombineLoadWithAndMask()
797 {TargetOpcode::G_ZEXTLOAD, {RegTy, MRI.getType(PtrReg)}, {MemDesc}})) in matchCombineLoadWithAndMask()
844 if (MRI.getType(SrcReg).isVector()) in matchSextTruncSextLoad()
848 if (mi_match(SrcReg, MRI, m_GTrunc(m_Reg(TruncSrc)))) in matchSextTruncSextLoad()
854 if (auto *LoadMI = getOpcodeDef<GSExtLoad>(LoadUser, MRI)) { in matchSextTruncSextLoad()
857 if (TruncSrc && MRI.getType(TruncSrc).getSizeInBits() < LoadSizeBits) in matchSextTruncSextLoad()
877 LLT RegTy = MRI.getType(DstReg); in matchSextInRegOfLoad()
884 auto *LoadDef = getOpcodeDef<GLoad>(SrcReg, MRI); in matchSextInRegOfLoad()
885 if (!LoadDef || !MRI.hasOneNonDBGUse(DstReg)) in matchSextInRegOfLoad()
915 {MRI.getType(LoadDef->getDstReg()), in matchSextInRegOfLoad()
916 MRI.getType(LoadDef->getPointerReg())}, in matchSextInRegOfLoad()
930 GLoad *LoadDef = cast<GLoad>(MRI.getVRegDef(LoadReg)); in applySextInRegOfLoad()
958 MachineRegisterInfo &MRI) { in canFoldInAddressingMode() argument
961 auto *Addr = getOpcodeDef<GPtrAdd>(MI->getPointerReg(), MRI); in canFoldInAddressingMode()
966 if (auto CstOff = getIConstantVRegVal(Addr->getOffsetReg(), MRI)) in canFoldInAddressingMode()
995 LLT PtrTy = MRI.getType(LdSt.getPointerReg()); in isIndexedLoadStoreLegal()
996 LLT Ty = MRI.getType(LdSt.getReg(0)); in isIndexedLoadStoreLegal()
1028 if (MRI.hasOneNonDBGUse(Ptr)) in findPostIndexCandidate()
1034 if (getOpcodeDef(TargetOpcode::G_FRAME_INDEX, Ptr, MRI)) in findPostIndexCandidate()
1037 MachineInstr *StoredValDef = getDefIgnoringCopies(LdSt.getReg(0), MRI); in findPostIndexCandidate()
1038 auto *PtrDef = MRI.getVRegDef(Ptr); in findPostIndexCandidate()
1041 for (auto &Use : MRI.use_nodbg_instructions(Ptr)) { in findPostIndexCandidate()
1048 if (!PtrAdd || MRI.use_nodbg_empty(PtrAdd->getReg(0))) in findPostIndexCandidate()
1059 /*IsPre*/ false, MRI)) in findPostIndexCandidate()
1063 MachineInstr *OffsetDef = MRI.getVRegDef(Offset); in findPostIndexCandidate()
1073 for (auto &BasePtrUse : MRI.use_nodbg_instructions(PtrAdd->getBaseReg())) { in findPostIndexCandidate()
1089 for (auto &BaseUseUse : MRI.use_nodbg_instructions(PtrAddDefReg)) { in findPostIndexCandidate()
1096 if (canFoldInAddressingMode(UseUseLdSt, TLI, MRI)) in findPostIndexCandidate()
1118 if (!mi_match(Addr, MRI, m_GPtrAdd(m_Reg(Base), m_Reg(Offset))) || in findPreIndexCandidate()
1119 MRI.hasOneNonDBGUse(Addr)) in findPreIndexCandidate()
1123 !TLI.isIndexingLegal(LdSt, Base, Offset, /*IsPre*/ true, MRI)) in findPreIndexCandidate()
1129 MachineInstr *BaseDef = getDefIgnoringCopies(Base, MRI); in findPreIndexCandidate()
1145 for (auto &AddrUse : MRI.use_nodbg_instructions(Addr)) in findPreIndexCandidate()
1152 for (auto &AddrUse : MRI.use_nodbg_instructions(Addr)) { in findPreIndexCandidate()
1159 if (!canFoldInAddressingMode(UseLdSt, TLI, MRI)) in findPreIndexCandidate()
1173 auto *LoadMI = getOpcodeDef<GLoad>(MI.getOperand(1).getReg(), MRI); in matchCombineExtractedVectorLoad()
1178 LLT VecEltTy = MRI.getType(Vector).getElementType(); in matchCombineExtractedVectorLoad()
1180 assert(MRI.getType(MI.getOperand(0).getReg()) == VecEltTy); in matchCombineExtractedVectorLoad()
1183 if (!MRI.hasOneNonDBGUse(Vector)) in matchCombineExtractedVectorLoad()
1207 if (auto CVal = getIConstantVRegVal(Vector, MRI)) { in matchCombineExtractedVectorLoad()
1222 LLT PtrTy = MRI.getType(VecPtr); in matchCombineExtractedVectorLoad()
1251 LoadMI->getPointerReg(), MRI.getType(LoadMI->getOperand(0).getReg()), in matchCombineExtractedVectorLoad()
1281 MachineInstr &AddrDef = *MRI.getUniqueVRegDef(MatchInfo.Addr); in applyCombineIndexedLoadStore()
1290 auto *OldCst = MRI.getVRegDef(MatchInfo.Offset); in applyCombineIndexedLoadStore()
1291 auto NewCst = Builder.buildConstant(MRI.getType(MatchInfo.Offset), in applyCombineIndexedLoadStore()
1349 if (!isLegalOrBeforeLegalizer({DivremOpcode, {MRI.getType(Src1)}})) in matchCombineDivRem()
1364 for (auto &UseMI : MRI.use_nodbg_instructions(Src1)) { in matchCombineDivRem()
1454 LLT Ty = MRI.getType(BrCond->getOperand(0).getReg()); in applyOptBrCondByInvertingCond()
1493 const MachineRegisterInfo &MRI, in constantFoldFpUnary() argument
1509 LLT DstTy = MRI.getType(MI.getOperand(0).getReg()); in constantFoldFpUnary()
1540 APFloat Folded = constantFoldFpUnary(MI, MRI, Cst->getValue()); in applyCombineConstantFoldFpUnary()
1559 auto MaybeImmVal = getIConstantVRegValWithLookThrough(Imm1, MRI); in matchPtrAddImmedChain()
1563 MachineInstr *Add2Def = MRI.getVRegDef(Add2); in matchPtrAddImmedChain()
1569 auto MaybeImm2Val = getIConstantVRegValWithLookThrough(Imm2, MRI); in matchPtrAddImmedChain()
1579 for (auto &UseMI : MRI.use_nodbg_instructions(MI.getOperand(0).getReg())) { in matchPtrAddImmedChain()
1581 AccessTy = getTypeForLLT(MRI.getType(LdSt->getReg(0)), in matchPtrAddImmedChain()
1594 unsigned AS = MRI.getType(Add2).getAddressSpace(); in matchPtrAddImmedChain()
1612 LLT OffsetTy = MRI.getType(MI.getOperand(2).getReg()); in applyPtrAddImmedChain()
1638 auto MaybeImmVal = getIConstantVRegValWithLookThrough(Imm1, MRI); in matchShiftImmedChain()
1642 MachineInstr *Shl2Def = MRI.getUniqueVRegDef(Shl2); in matchShiftImmedChain()
1648 auto MaybeImm2Val = getIConstantVRegValWithLookThrough(Imm2, MRI); in matchShiftImmedChain()
1660 MatchInfo.Imm >= MRI.getType(Shl2).getScalarSizeInBits()) in matchShiftImmedChain()
1675 LLT Ty = MRI.getType(MI.getOperand(1).getReg()); in applyShiftImmedChain()
1691 LLT ImmTy = MRI.getType(MI.getOperand(2).getReg()); in applyShiftImmedChain()
1721 if (!MRI.hasOneNonDBGUse(LogicDest)) in matchShiftOfShiftedLogic()
1724 MachineInstr *LogicMI = MRI.getUniqueVRegDef(LogicDest); in matchShiftOfShiftedLogic()
1732 auto MaybeImmVal = getIConstantVRegValWithLookThrough(C1, MRI); in matchShiftOfShiftedLogic()
1741 !MRI.hasOneNonDBGUse(MI->getOperand(0).getReg())) in matchShiftOfShiftedLogic()
1746 getIConstantVRegValWithLookThrough(MI->getOperand(2).getReg(), MRI); in matchShiftOfShiftedLogic()
1756 MachineInstr *LogicMIOp1 = MRI.getUniqueVRegDef(LogicMIReg1); in matchShiftOfShiftedLogic()
1758 MachineInstr *LogicMIOp2 = MRI.getUniqueVRegDef(LogicMIReg2); in matchShiftOfShiftedLogic()
1773 if (MatchInfo.ValSum >= MRI.getType(LogicDest).getScalarSizeInBits()) in matchShiftOfShiftedLogic()
1788 LLT ShlType = MRI.getType(MI.getOperand(2).getReg()); in applyShiftOfShiftedLogic()
1789 LLT DestType = MRI.getType(MI.getOperand(0).getReg()); in applyShiftOfShiftedLogic()
1833 if (!mi_match(SrcReg, MRI, in matchCommuteShift()
1839 if (!mi_match(C1, MRI, m_ICstOrSplat(C1Val)) || in matchCommuteShift()
1840 !mi_match(ShiftReg, MRI, m_ICstOrSplat(C2Val))) in matchCommuteShift()
1843 auto *SrcDef = MRI.getVRegDef(SrcReg); in matchCommuteShift()
1846 LLT SrcTy = MRI.getType(SrcReg); in matchCommuteShift()
1859 getIConstantVRegValWithLookThrough(MI.getOperand(2).getReg(), MRI); in matchCombineMulToShl()
1871 LLT ShiftTy = MRI.getType(MI.getOperand(0).getReg()); in applyCombineMulToShl()
1889 if (!mi_match(LHS, MRI, m_GAnyExt(m_Reg(ExtSrc))) && in matchCombineShlOfExtend()
1890 !mi_match(LHS, MRI, m_GZExt(m_Reg(ExtSrc))) && in matchCombineShlOfExtend()
1891 !mi_match(LHS, MRI, m_GSExt(m_Reg(ExtSrc)))) in matchCombineShlOfExtend()
1895 MachineInstr *MIShiftAmt = MRI.getVRegDef(RHS); in matchCombineShlOfExtend()
1896 auto MaybeShiftAmtVal = isConstantOrConstantSplatVector(*MIShiftAmt, MRI); in matchCombineShlOfExtend()
1901 LLT SrcTy = MRI.getType(ExtSrc); in matchCombineShlOfExtend()
1916 unsigned SrcTySize = MRI.getType(ExtSrc).getScalarSizeInBits(); in matchCombineShlOfExtend()
1925 LLT ExtSrcTy = MRI.getType(ExtSrcReg); in applyCombineShlOfExtend()
1941 auto *Unmerge = getOpcodeDef<GUnmerge>(MergedValues[0], MRI); in matchCombineMergeUnmerge()
1954 const MachineRegisterInfo &MRI) { in peekThroughBitcast() argument
1955 while (mi_match(Reg, MRI, m_GBitcast(m_Reg(Reg)))) in peekThroughBitcast()
1966 Register SrcReg = peekThroughBitcast(Unmerge.getSourceReg(), MRI); in matchCombineUnmergeMergeToPlainValues()
1968 auto *SrcInstr = getOpcodeDef<GMergeLikeInstr>(SrcReg, MRI); in matchCombineUnmergeMergeToPlainValues()
1973 LLT SrcMergeTy = MRI.getType(SrcInstr->getSourceReg(0)); in matchCombineUnmergeMergeToPlainValues()
1974 LLT Dst0Ty = MRI.getType(Unmerge.getReg(0)); in matchCombineUnmergeMergeToPlainValues()
1993 LLT SrcTy = MRI.getType(Operands[0]); in applyCombineUnmergeMergeToPlainValues()
1994 LLT DstTy = MRI.getType(MI.getOperand(0).getReg()); in applyCombineUnmergeMergeToPlainValues()
2003 const auto &DstCB = MRI.getRegClassOrRegBank(DstReg); in applyCombineUnmergeMergeToPlainValues()
2004 if (!DstCB.isNull() && DstCB != MRI.getRegClassOrRegBank(SrcReg)) { in applyCombineUnmergeMergeToPlainValues()
2005 SrcReg = Builder.buildCopy(MRI.getType(SrcReg), SrcReg).getReg(0); in applyCombineUnmergeMergeToPlainValues()
2006 MRI.setRegClassOrRegBank(SrcReg, DstCB); in applyCombineUnmergeMergeToPlainValues()
2010 replaceRegWith(MRI, DstReg, SrcReg); in applyCombineUnmergeMergeToPlainValues()
2021 MachineInstr *SrcInstr = MRI.getVRegDef(SrcReg); in matchCombineUnmergeConstant()
2031 LLT Dst0Ty = MRI.getType(MI.getOperand(0).getReg()); in matchCombineUnmergeConstant()
2069 return isa<GImplicitDef>(MRI.getVRegDef(SrcReg)); in matchCombineUnmergeUndef()
2077 if (!MRI.use_nodbg_empty(MI.getOperand(Idx).getReg())) in matchCombineUnmergeWithDeadLanesToTrunc()
2089 LLT SrcTy = MRI.getType(SrcReg); in applyCombineUnmergeWithDeadLanesToTrunc()
2095 LLT Dst0Ty = MRI.getType(Dst0Reg); in applyCombineUnmergeWithDeadLanesToTrunc()
2108 LLT Dst0Ty = MRI.getType(Dst0Reg); in matchCombineUnmergeZExtToZExt()
2115 LLT SrcTy = MRI.getType(SrcReg); in matchCombineUnmergeZExtToZExt()
2120 if (!mi_match(SrcReg, MRI, m_GZExt(m_Reg(ZExtSrcReg)))) in matchCombineUnmergeZExtToZExt()
2126 LLT ZExtSrcTy = MRI.getType(ZExtSrcReg); in matchCombineUnmergeZExtToZExt()
2137 MRI.getVRegDef(MI.getOperand(MI.getNumDefs()).getReg()); in applyCombineUnmergeZExtToZExt()
2142 LLT Dst0Ty = MRI.getType(Dst0Reg); in applyCombineUnmergeZExtToZExt()
2143 LLT ZExtSrcTy = MRI.getType(ZExtSrcReg); in applyCombineUnmergeZExtToZExt()
2152 replaceRegWith(MRI, Dst0Reg, ZExtSrcReg); in applyCombineUnmergeZExtToZExt()
2159 replaceRegWith(MRI, MI.getOperand(Idx).getReg(), ZeroReg); in applyCombineUnmergeZExtToZExt()
2171 LLT Ty = MRI.getType(MI.getOperand(0).getReg()); in matchCombineShiftToUnmerge()
2181 getIConstantVRegValWithLookThrough(MI.getOperand(2).getReg(), MRI); in matchCombineShiftToUnmerge()
2193 LLT Ty = MRI.getType(SrcReg); in applyCombineShiftToUnmerge()
2276 LLT DstTy = MRI.getType(DstReg); in matchCombineI2PToP2I()
2278 return mi_match(SrcReg, MRI, in matchCombineI2PToP2I()
2303 LLT IntTy = MRI.getType(LHS); in matchCombineAddP2IToPtrAdd()
2309 if (mi_match(SrcReg, MRI, m_GPtrToInt(m_Reg(PtrReg.first)))) { in matchCombineAddP2IToPtrAdd()
2312 LLT PtrTy = MRI.getType(PtrReg.first); in matchCombineAddP2IToPtrAdd()
2334 LLT PtrTy = MRI.getType(LHS); in applyCombineAddP2IToPtrAdd()
2347 MachineRegisterInfo &MRI = Builder.getMF().getRegInfo(); in matchCombineConstPtrAddToI2P() local
2349 if (auto RHSCst = getIConstantVRegVal(RHS, MRI)) { in matchCombineConstPtrAddToI2P()
2351 if (mi_match(LHS, MRI, m_GIntToPtr(m_ICst(Cst)))) { in matchCombineConstPtrAddToI2P()
2352 auto DstTy = MRI.getType(PtrAdd.getReg(0)); in matchCombineConstPtrAddToI2P()
2377 Register OriginalSrcReg = getSrcRegIgnoringCopies(SrcReg, MRI); in matchCombineAnyExtTrunc()
2380 LLT DstTy = MRI.getType(DstReg); in matchCombineAnyExtTrunc()
2381 return mi_match(SrcReg, MRI, in matchCombineAnyExtTrunc()
2389 LLT DstTy = MRI.getType(DstReg); in matchCombineZextTrunc()
2390 if (mi_match(SrcReg, MRI, in matchCombineZextTrunc()
2393 unsigned SrcSize = MRI.getType(SrcReg).getScalarSizeInBits(); in matchCombineZextTrunc()
2406 Register OriginalSrcReg = getSrcRegIgnoringCopies(SrcReg, MRI); in matchCombineExtOfExt()
2409 MachineInstr *SrcMI = MRI.getVRegDef(SrcReg); in matchCombineExtOfExt()
2458 MachineInstr *SrcMI = MRI.getVRegDef(SrcReg); in matchCombineTruncOfExt()
2474 LLT SrcTy = MRI.getType(SrcReg); in applyCombineTruncOfExt()
2475 LLT DstTy = MRI.getType(DstReg); in applyCombineTruncOfExt()
2478 replaceRegWith(MRI, DstReg, SrcReg); in applyCombineTruncOfExt()
2512 if (!MRI.hasOneNonDBGUse(SrcReg)) in matchCombineTruncOfShift()
2515 LLT SrcTy = MRI.getType(SrcReg); in matchCombineTruncOfShift()
2516 LLT DstTy = MRI.getType(DstReg); in matchCombineTruncOfShift()
2518 MachineInstr *SrcMI = getDefIgnoringCopies(SrcReg, MRI); in matchCombineTruncOfShift()
2541 for (auto &User : MRI.use_instructions(DstReg)) in matchCombineTruncOfShift()
2575 LLT DstTy = MRI.getType(Dst); in applyCombineTruncOfShift()
2587 replaceRegWith(MRI, Dst, NewShift); in applyCombineTruncOfShift()
2597 getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, MO.getReg(), MRI); in matchAnyExplicitUseIsUndef()
2604 getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, MO.getReg(), MRI); in matchAllExplicitUsesAreUndef()
2617 MRI); in matchUndefStore()
2623 MRI); in matchUndefSelectCmp()
2630 LLT VecTy = MRI.getType(MI.getOperand(1).getReg()); in matchInsertExtractVecEltOutOfBounds()
2633 auto Idx = getIConstantVRegVal(MI.getOperand(IdxIdx).getReg(), MRI); in matchInsertExtractVecEltOutOfBounds()
2642 isConstantOrConstantSplatVector(*MRI.getVRegDef(SelMI.getCondReg()), MRI); in matchConstantSelectCmp()
2655 auto InstAndDef1 = getDefSrcRegIgnoringCopies(MOP1.getReg(), MRI); in matchEqualDefs()
2658 auto InstAndDef2 = getDefSrcRegIgnoringCopies(MOP2.getReg(), MRI); in matchEqualDefs()
2735 if (Builder.getTII().produceSameValue(*I1, *I2, &MRI)) { in matchEqualDefs()
2751 auto *MI = MRI.getVRegDef(MOP.getReg()); in matchConstantOp()
2752 auto MaybeCst = isConstantOrConstantSplatVector(*MI, MRI); in matchConstantOp()
2761 if (!mi_match(MOP.getReg(), MRI, m_GFCstOrSplat(MaybeCst))) in matchConstantFPOp()
2772 assert(canReplaceReg(OldReg, Replacement, MRI) && "Cannot replace register?"); in replaceSingleDefInstWithOperand()
2774 replaceRegWith(MRI, OldReg, Replacement); in replaceSingleDefInstWithOperand()
2781 assert(canReplaceReg(OldReg, Replacement, MRI) && "Cannot replace register?"); in replaceSingleDefInstWithReg()
2783 replaceRegWith(MRI, OldReg, Replacement); in replaceSingleDefInstWithReg()
2789 LLT DstTy = MRI.getType(MI.getOperand(0).getReg()); in matchConstantLargerBitWidth()
2792 auto VRegAndVal = getIConstantVRegValWithLookThrough(ConstReg, MRI); in matchConstantLargerBitWidth()
2806 LLT ConstTy = MRI.getType(ConstReg); in applyFunnelShiftConstantModulo()
2807 LLT DstTy = MRI.getType(MI.getOperand(0).getReg()); in applyFunnelShiftConstantModulo()
2809 auto VRegAndVal = getIConstantVRegValWithLookThrough(ConstReg, MRI); in applyFunnelShiftConstantModulo()
2830 MRI); in matchSelectSameVal()
2836 MRI); in matchBinOpSameVal()
2842 MRI); in matchOperandIsZero()
2848 getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, MO.getReg(), MRI); in matchOperandIsUndef()
2854 return isKnownToBeAPowerOfTwo(MO.getReg(), MRI, KB); in matchOperandIsKnownToBeAPowerOfTwo()
2903 if (!mi_match(MaybeSub, MRI, m_Neg(m_Reg(NewRHS)))) in matchSimplifyAddToSub()
2917 LLT DstTy = MRI.getType(DstReg); in matchCombineInsertVecElts()
2922 if (MRI.hasOneUse(DstReg) && MRI.use_instr_begin(DstReg)->getOpcode() == in matchCombineInsertVecElts()
2931 CurrInst->getOperand(0).getReg(), MRI, in matchCombineInsertVecElts()
2960 LLT DstTy = MRI.getType(MI.getOperand(0).getReg()); in applyCombineInsertVecElts()
2999 if (!MRI.hasOneNonDBGUse(LHSReg) || !MRI.hasOneNonDBGUse(RHSReg)) in matchHoistLogicOpWithSameOpcodeHands()
3003 MachineInstr *LeftHandInst = getDefIgnoringCopies(LHSReg, MRI); in matchHoistLogicOpWithSameOpcodeHands()
3004 MachineInstr *RightHandInst = getDefIgnoringCopies(RHSReg, MRI); in matchHoistLogicOpWithSameOpcodeHands()
3018 LLT XTy = MRI.getType(X); in matchHoistLogicOpWithSameOpcodeHands()
3019 LLT YTy = MRI.getType(Y); in matchHoistLogicOpWithSameOpcodeHands()
3053 auto NewLogicDst = MRI.createGenericVirtualRegister(XTy); in matchHoistLogicOpWithSameOpcodeHands()
3093 if (!mi_match(MI.getOperand(0).getReg(), MRI, in matchAshrShlToSextInreg()
3100 {TargetOpcode::G_SEXT_INREG, {MRI.getType(Src)}})) in matchAshrShlToSextInreg()
3112 unsigned Size = MRI.getType(Src).getScalarSizeInBits(); in applyAshShlToSextInreg()
3124 LLT Ty = MRI.getType(Dst); in matchOverlappingAnd()
3130 Dst, MRI, in matchOverlappingAnd()
3140 replaceRegWith(MRI, Dst, Zero->getOperand(0).getReg()); in matchOverlappingAnd()
3178 if (canReplaceReg(AndDst, LHS, MRI) && in matchRedundantAnd()
3185 if (canReplaceReg(AndDst, RHS, MRI) && in matchRedundantAnd()
3218 if (canReplaceReg(OrDst, LHS, MRI) && in matchRedundantOr()
3225 if (canReplaceReg(OrDst, RHS, MRI) && in matchRedundantOr()
3238 unsigned TypeSize = MRI.getType(Src).getScalarSizeInBits(); in matchRedundantSExtInReg()
3252 LLT Ty = MRI.getType(MI.getOperand(0).getReg()); in matchNotCmp()
3257 if (!mi_match(MI.getOperand(0).getReg(), MRI, in matchNotCmp()
3261 if (!MRI.hasOneNonDBGUse(XorSrc)) in matchNotCmp()
3273 if (!MRI.hasOneNonDBGUse(Reg)) in matchNotCmp()
3275 MachineInstr *Def = MRI.getVRegDef(Reg); in matchNotCmp()
3310 MachineInstr *CstDef = MRI.getVRegDef(CstReg); in matchNotCmp()
3311 auto MaybeCst = getIConstantSplatSExtVal(*CstDef, MRI); in matchNotCmp()
3317 if (!mi_match(CstReg, MRI, m_ICst(Cst))) in matchNotCmp()
3329 MachineInstr *Def = MRI.getVRegDef(Reg); in applyNotCmp()
3354 replaceRegWith(MRI, MI.getOperand(0).getReg(), MI.getOperand(1).getReg()); in applyNotCmp()
3372 if (!mi_match(AndReg, MRI, m_GAnd(m_Reg(X), m_Reg(Y)))) { in matchXorOfAndWithSameReg()
3374 if (!mi_match(AndReg, MRI, m_GAnd(m_Reg(X), m_Reg(Y)))) in matchXorOfAndWithSameReg()
3379 if (!MRI.hasOneNonDBGUse(AndReg)) in matchXorOfAndWithSameReg()
3395 auto Not = Builder.buildNot(MRI.getType(X), X); in applyXorOfAndWithSameReg()
3406 LLT Ty = MRI.getType(DstReg); in matchPtrAddZero()
3413 auto ConstVal = getIConstantVRegVal(PtrAdd.getBaseReg(), MRI); in matchPtrAddZero()
3418 const MachineInstr *VecMI = MRI.getVRegDef(PtrAdd.getBaseReg()); in matchPtrAddZero()
3419 return isBuildVectorAllZeros(*VecMI, MRI); in matchPtrAddZero()
3434 LLT Ty = MRI.getType(DstReg); in applySimplifyURemByPow2()
3451 MachineInstr *Select = MRI.getVRegDef(LHS); in matchFoldBinOpIntoSelect()
3456 !MRI.hasOneNonDBGUse(LHS)) { in matchFoldBinOpIntoSelect()
3459 Select = MRI.getVRegDef(RHS); in matchFoldBinOpIntoSelect()
3461 !MRI.hasOneNonDBGUse(RHS)) in matchFoldBinOpIntoSelect()
3465 MachineInstr *SelectLHS = MRI.getVRegDef(Select->getOperand(2).getReg()); in matchFoldBinOpIntoSelect()
3466 MachineInstr *SelectRHS = MRI.getVRegDef(Select->getOperand(3).getReg()); in matchFoldBinOpIntoSelect()
3468 if (!isConstantOrConstantVector(*SelectLHS, MRI, in matchFoldBinOpIntoSelect()
3472 if (!isConstantOrConstantVector(*SelectRHS, MRI, in matchFoldBinOpIntoSelect()
3484 (isNullOrNullSplat(*SelectLHS, MRI) || in matchFoldBinOpIntoSelect()
3485 isAllOnesOrAllOnesSplat(*SelectLHS, MRI)) && in matchFoldBinOpIntoSelect()
3486 (isNullOrNullSplat(*SelectRHS, MRI) || in matchFoldBinOpIntoSelect()
3487 isAllOnesOrAllOnesSplat(*SelectRHS, MRI)); in matchFoldBinOpIntoSelect()
3491 return isConstantOrConstantVector(*MRI.getVRegDef(OtherOperandReg), MRI, in matchFoldBinOpIntoSelect()
3505 MachineInstr *Select = MRI.getVRegDef(MI.getOperand(SelectOperand).getReg()); in applyFoldBinOpIntoSelect()
3511 LLT Ty = MRI.getType(Dst); in applyFoldBinOpIntoSelect()
3568 MRI.getType(Root->getOperand(0).getReg()).getSizeInBytes() - 1; in findCandidatesForLoadOrCombine()
3577 if (!MRI.hasOneNonDBGUse(OrLHS) || !MRI.hasOneNonDBGUse(OrRHS)) in findCandidatesForLoadOrCombine()
3582 if (const MachineInstr *Or = getOpcodeDef(TargetOpcode::G_OR, OrLHS, MRI)) in findCandidatesForLoadOrCombine()
3586 if (const MachineInstr *Or = getOpcodeDef(TargetOpcode::G_OR, OrRHS, MRI)) in findCandidatesForLoadOrCombine()
3609 const MachineRegisterInfo &MRI) { in matchLoadAndBytePosition() argument
3610 assert(MRI.hasOneNonDBGUse(Reg) && in matchLoadAndBytePosition()
3614 if (!mi_match(Reg, MRI, in matchLoadAndBytePosition()
3624 auto *Load = getOpcodeDef<GZExtLoad>(MaybeLoad, MRI); in matchLoadAndBytePosition()
3672 auto LoadAndPos = matchLoadAndBytePosition(Reg, MemSizeInBits, MRI); in findLoadOffsetsForLoadOrCombine()
3697 if (!mi_match(Load->getOperand(1).getReg(), MRI, in findLoadOffsetsForLoadOrCombine()
3783 LLT Ty = MRI.getType(Dst); in matchLoadOrCombine()
3858 {TargetOpcode::G_LOAD, {Ty, MRI.getType(Ptr)}, {MMDesc}})) in matchLoadOrCombine()
3873 Register LoadDst = NeedsBSwap ? MRI.cloneVirtualRegister(Dst) : Dst; in matchLoadOrCombine()
3888 if (MRI.getType(DstReg).isVector()) in matchExtendThroughPhis()
3892 if (!MRI.hasOneNonDBGUse(DstReg)) in matchExtendThroughPhis()
3894 ExtMI = &*MRI.use_instr_nodbg_begin(DstReg); in matchExtendThroughPhis()
3906 if (Builder.getTII().isExtendLikelyToBeFolded(*ExtMI, MRI)) in matchExtendThroughPhis()
3914 auto *DefMI = getDefIgnoringCopies(PHI.getIncomingValue(I), MRI); in matchExtendThroughPhis()
3939 LLT ExtTy = MRI.getType(DstReg); in applyExtendThroughPhis()
3948 auto *SrcMI = MRI.getVRegDef(SrcReg); in applyExtendThroughPhis()
3973 auto *NewSrc = OldToNewSrcMap[MRI.getVRegDef(MO.getReg())]; in applyExtendThroughPhis()
3986 LLT SrcTy = MRI.getType(SrcVec); in matchExtractVecEltBuildVec()
3988 auto Cst = getIConstantVRegValWithLookThrough(MI.getOperand(2).getReg(), MRI); in matchExtractVecEltBuildVec()
3996 MachineInstr *SrcVecMI = MRI.getVRegDef(SrcVec); in matchExtractVecEltBuildVec()
3998 SrcVecMI = MRI.getVRegDef(SrcVecMI->getOperand(1).getReg()); in matchExtractVecEltBuildVec()
4006 if (!MRI.hasOneNonDBGUse(SrcVec) && in matchExtractVecEltBuildVec()
4018 LLT ScalarTy = MRI.getType(Reg); in applyExtractVecEltBuildVec()
4020 LLT DstTy = MRI.getType(DstReg); in applyExtractVecEltBuildVec()
4053 LLT DstTy = MRI.getType(DstReg); in matchExtractAllEltsFromBuildVector()
4057 for (MachineInstr &II : MRI.use_nodbg_instructions(DstReg)) { in matchExtractAllEltsFromBuildVector()
4060 auto Cst = getIConstantVRegVal(II.getOperand(2).getReg(), MRI); in matchExtractAllEltsFromBuildVector()
4080 replaceRegWith(MRI, ExtMI->getOperand(0).getReg(), Pair.first); in applyExtractAllEltsFromBuildVector()
4104 LLT Ty = MRI.getType(Dst); in matchOrShiftToFunnelShift()
4111 if (!mi_match(Dst, MRI, in matchOrShiftToFunnelShift()
4120 if (mi_match(ShlAmt, MRI, m_ICstOrSplat(CstShlAmt)) && in matchOrShiftToFunnelShift()
4121 mi_match(LShrAmt, MRI, m_ICstOrSplat(CstLShrAmt)) && in matchOrShiftToFunnelShift()
4126 } else if (mi_match(LShrAmt, MRI, in matchOrShiftToFunnelShift()
4132 } else if (mi_match(ShlAmt, MRI, in matchOrShiftToFunnelShift()
4142 LLT AmtTy = MRI.getType(Amt); in matchOrShiftToFunnelShift()
4162 return isLegalOrBeforeLegalizer({RotateOpc, {MRI.getType(X), MRI.getType(Y)}}); in matchFunnelShiftToRotate()
4181 MRI.getType(MI.getOperand(0).getReg()).getScalarSizeInBits(); in matchRotateOutOfRange()
4189 return matchUnaryPredicate(MRI, AmtReg, MatchOutOfRange) && OutOfRange; in matchRotateOutOfRange()
4196 MRI.getType(MI.getOperand(0).getReg()).getScalarSizeInBits(); in applyRotateOutOfRange()
4199 LLT AmtTy = MRI.getType(Amt); in applyRotateOutOfRange()
4254 MRI.getType(MI.getOperand(0).getReg()).isVector(), in matchICmpToTrueFalseKnownBits()
4278 LLT DstTy = MRI.getType(Dst); in matchICmpToLHSKnownBits()
4283 if (!mi_match(MI.getOperand(3).getReg(), MRI, m_SpecificICst(OneOrZero))) in matchICmpToLHSKnownBits()
4290 LLT LHSTy = MRI.getType(LHS); in matchICmpToLHSKnownBits()
4309 LLT Ty = MRI.getType(MI.getOperand(0).getReg()); in matchAndOrDisjointMask()
4317 if (!mi_match(MI, MRI, in matchAndOrDisjointMask()
4343 LLT Ty = MRI.getType(Src); in matchBitfieldExtractFromSExtInReg()
4351 Src, MRI, in matchBitfieldExtractFromSExtInReg()
4371 LLT Ty = MRI.getType(Dst); in matchBitfieldExtractFromAnd()
4379 if (!mi_match(MI.getOperand(0).getReg(), MRI, in matchBitfieldExtractFromAnd()
4414 LLT Ty = MRI.getType(Dst); in matchBitfieldExtractFromShr()
4425 if (!mi_match(Dst, MRI, in matchBitfieldExtractFromShr()
4457 LLT Ty = MRI.getType(Dst); in matchBitfieldExtractFromShrAnd()
4466 if (!mi_match(Dst, MRI, in matchBitfieldExtractFromShrAnd()
4513 auto *Src1Def = getOpcodeDef<GPtrAdd>(Src1Reg, MRI); in reassociationCanBreakAddressingModePattern()
4519 if (MRI.hasOneNonDBGUse(Src1Reg)) in reassociationCanBreakAddressingModePattern()
4522 auto C1 = getIConstantVRegVal(Src1Def->getOffsetReg(), MRI); in reassociationCanBreakAddressingModePattern()
4525 auto C2 = getIConstantVRegVal(Src2Reg, MRI); in reassociationCanBreakAddressingModePattern()
4533 for (auto &UseMI : MRI.use_nodbg_instructions(PtrAdd.getReg(0))) { in reassociationCanBreakAddressingModePattern()
4541 if (!MRI.hasOneNonDBGUse(DefReg)) in reassociationCanBreakAddressingModePattern()
4543 ConvUseMI = &*MRI.use_instr_nodbg_begin(DefReg); in reassociationCanBreakAddressingModePattern()
4555 unsigned AS = MRI.getType(LdStMI->getPointerReg()).getAddressSpace(); in reassociationCanBreakAddressingModePattern()
4580 auto C2 = getIConstantVRegVal(RHS->getOperand(2).getReg(), MRI); in matchReassocConstantInnerRHS()
4585 LLT PtrTy = MRI.getType(MI.getOperand(0).getReg()); in matchReassocConstantInnerRHS()
4605 if (!mi_match(MI.getBaseReg(), MRI, in matchReassocConstantInnerLHS()
4617 auto NewCst = B.buildConstant(MRI.getType(RHSReg), LHSCstOff->Value); in matchReassocConstantInnerLHS()
4640 auto C1 = getIConstantVRegVal(LHSSrc2, MRI); in matchReassocFoldConstantsInSubTree()
4643 auto C2 = getIConstantVRegVal(Src2Reg, MRI); in matchReassocFoldConstantsInSubTree()
4648 auto NewCst = B.buildConstant(MRI.getType(Src2Reg), *C1 + *C2); in matchReassocFoldConstantsInSubTree()
4672 MachineInstr *LHS = MRI.getVRegDef(PtrAdd.getBaseReg()); in matchReassocPtrAdd()
4673 MachineInstr *RHS = MRI.getVRegDef(PtrAdd.getOffsetReg()); in matchReassocPtrAdd()
4692 LLT OpRHSTy = MRI.getType(OpRHS); in tryReassocBinOp()
4693 MachineInstr *OpLHSDef = MRI.getVRegDef(OpLHS); in tryReassocBinOp()
4698 MachineInstr *OpRHSDef = MRI.getVRegDef(OpRHS); in tryReassocBinOp()
4706 if (isConstantOrConstantSplatVector(*MRI.getVRegDef(OpLHSRHS), MRI) && in tryReassocBinOp()
4707 !isConstantOrConstantSplatVector(*MRI.getVRegDef(OpLHSLHS), MRI)) { in tryReassocBinOp()
4708 if (isConstantOrConstantSplatVector(*OpRHSDef, MRI)) { in tryReassocBinOp()
4716 if (getTargetLowering().isReassocProfitable(MRI, OpLHS, OpRHS)) { in tryReassocBinOp()
4747 LLT DstTy = MRI.getType(MI.getOperand(0).getReg()); in matchConstantFoldCastOp()
4750 if (auto MaybeCst = ConstantFoldCastOp(MI.getOpcode(), DstTy, SrcOp, MRI)) { in matchConstantFoldCastOp()
4761 auto MaybeCst = ConstantFoldBinOp(MI.getOpcode(), Op1, Op2, MRI); in matchConstantFoldBinOp()
4771 auto MaybeCst = ConstantFoldFPBinOp(MI.getOpcode(), Op1, Op2, MRI); in matchConstantFoldFPBinOp()
4785 const ConstantFP *Op3Cst = getConstantFPVRegVal(Op3, MRI); in matchConstantFoldFMA()
4789 const ConstantFP *Op2Cst = getConstantFPVRegVal(Op2, MRI); in matchConstantFoldFMA()
4793 const ConstantFP *Op1Cst = getConstantFPVRegVal(Op1, MRI); in matchConstantFoldFMA()
4826 LLT WideTy = MRI.getType(Dst); in matchNarrowBinopFeedingAnd()
4830 if (!WideTy.isScalar() || !MRI.hasOneNonDBGUse(AndLHS)) in matchNarrowBinopFeedingAnd()
4839 MachineInstr *LHSInst = getDefIgnoringCopies(AndLHS, MRI); in matchNarrowBinopFeedingAnd()
4856 auto Cst = getIConstantVRegValWithLookThrough(AndRHS, MRI); in matchNarrowBinopFeedingAnd()
4899 if (!mi_match(MI.getOperand(3).getReg(), MRI, m_SpecificICstOrSplat(2))) in matchMulOBy2()
4917 if (!mi_match(MI.getOperand(3).getReg(), MRI, m_SpecificICstOrSplat(0))) in matchMulOBy0()
4921 if (!isConstantLegalOrBeforeLegalizer(MRI.getType(Dst)) || in matchMulOBy0()
4922 !isConstantLegalOrBeforeLegalizer(MRI.getType(Carry))) in matchMulOBy0()
4935 if (!mi_match(MI.getOperand(3).getReg(), MRI, m_SpecificICstOrSplat(0))) in matchAddOBy0()
4938 if (!isConstantLegalOrBeforeLegalizer(MRI.getType(Carry))) in matchAddOBy0()
4956 if (!mi_match(MI.getOperand(4).getReg(), MRI, m_SpecificICstOrSplat(0))) in matchAddEToAddO()
4989 if (mi_match(Dst, MRI, m_GSub(m_GAdd(m_Reg(X), m_Reg(Y)), m_Reg(Z)))) { in matchSubAddSameReg()
4992 if (Y == Z || (mi_match(Y, MRI, m_ICstOrSplat(CstY)) && in matchSubAddSameReg()
4993 mi_match(Z, MRI, m_SpecificICstOrSplat(CstY)))) in matchSubAddSameReg()
4995 else if (X == Z || (mi_match(X, MRI, m_ICstOrSplat(CstX)) && in matchSubAddSameReg()
4996 mi_match(Z, MRI, m_SpecificICstOrSplat(CstX)))) in matchSubAddSameReg()
5006 if (mi_match(Dst, MRI, m_GSub(m_Reg(X), m_GAdd(m_Reg(Y), m_Reg(Z))))) { in matchSubAddSameReg()
5009 if (X == Z || (mi_match(X, MRI, m_ICstOrSplat(CstX)) && in matchSubAddSameReg()
5010 mi_match(Z, MRI, m_SpecificICstOrSplat(CstX)))) in matchSubAddSameReg()
5012 else if (X == Y || (mi_match(X, MRI, m_ICstOrSplat(CstX)) && in matchSubAddSameReg()
5013 mi_match(Y, MRI, m_SpecificICstOrSplat(CstX)))) in matchSubAddSameReg()
5017 auto Zero = B.buildConstant(MRI.getType(Dst), 0); in matchSubAddSameReg()
5032 LLT Ty = MRI.getType(Dst); in buildUDivUsingMul()
5085 bool Matched = matchUnaryPredicate(MRI, RHS, BuildUDIVPattern); in buildUDivUsingMul()
5090 auto *RHSDef = getOpcodeDef<GBuildVector>(RHS, MRI); in buildUDivUsingMul()
5097 assert(MRI.getType(RHS).isScalar() && in buildUDivUsingMul()
5135 LLT DstTy = MRI.getType(Dst); in matchUDivByConst()
5136 auto *RHSDef = MRI.getVRegDef(RHS); in matchUDivByConst()
5137 if (!isConstantOrConstantVector(*RHSDef, MRI)) in matchUDivByConst()
5171 return matchUnaryPredicate(MRI, RHS, CheckEltValue); in matchUDivByConst()
5183 LLT DstTy = MRI.getType(Dst); in matchSDivByConst()
5201 MRI, RHS, [](const Constant *C) { return C && !C->isZeroValue(); }); in matchSDivByConst()
5219 LLT Ty = MRI.getType(Dst); in buildSDivUsingMul()
5229 auto *RHSDef = cast<GenericMachineInstr>(getDefIgnoringCopies(RHS, MRI)); in buildSDivUsingMul()
5230 bool IsSplat = getIConstantSplatVal(*RHSDef, MRI).has_value(); in buildSDivUsingMul()
5260 bool Matched = matchUnaryPredicate(MRI, RHS, BuildSDIVPattern); in buildSDivUsingMul()
5285 LLT Ty = MRI.getType(Dst); in matchUMulHToLShr()
5292 if (!matchUnaryPredicate(MRI, RHS, MatchPow2ExceptOne, false)) in matchUMulHToLShr()
5301 LLT Ty = MRI.getType(Dst); in applyUMulHToLShr()
5324 LLT Type = MRI.getType(Dst); in matchRedundantNegOperands()
5329 if (mi_match(Dst, MRI, m_GFAdd(m_Reg(X), m_GFNeg(m_Reg(Y)))) && in matchRedundantNegOperands()
5334 else if (mi_match(Dst, MRI, m_GFSub(m_Reg(X), m_GFNeg(m_Reg(Y)))) && in matchRedundantNegOperands()
5344 mi_match(X, MRI, m_GFNeg(m_Reg(X))) && in matchRedundantNegOperands()
5345 mi_match(Y, MRI, m_GFNeg(m_Reg(Y)))) { in matchRedundantNegOperands()
5365 LLT Ty = MRI.getType(MI.getOperand(0).getReg()); in matchFsubToFneg()
5368 ? getFConstantSplat(LHS, MRI, /* allowUndef */ true) in matchFsubToFneg()
5369 : getFConstantVRegValWithLookThrough(LHS, MRI); in matchFsubToFneg()
5388 Dst, Builder.buildFCanonicalize(MRI.getType(Dst), MatchInfo).getReg(0)); in applyFsubToFneg()
5401 const MachineRegisterInfo &MRI) { in hasMoreUses() argument
5402 return std::distance(MRI.use_instr_nodbg_begin(MI0.getOperand(0).getReg()), in hasMoreUses()
5403 MRI.use_instr_nodbg_end()) > in hasMoreUses()
5404 std::distance(MRI.use_instr_nodbg_begin(MI1.getOperand(0).getReg()), in hasMoreUses()
5405 MRI.use_instr_nodbg_end()); in hasMoreUses()
5416 LLT DstType = MRI.getType(MI.getOperand(0).getReg()); in canCombineFMadOrFMA()
5451 DefinitionAndSourceRegister LHS = {MRI.getVRegDef(Op1), Op1}; in matchCombineFAddFMulToFMadOrFMA()
5452 DefinitionAndSourceRegister RHS = {MRI.getVRegDef(Op2), Op2}; in matchCombineFAddFMulToFMadOrFMA()
5460 if (hasMoreUses(*LHS.MI, *RHS.MI, MRI)) in matchCombineFAddFMulToFMadOrFMA()
5466 (Aggressive || MRI.hasOneNonDBGUse(LHS.Reg))) { in matchCombineFAddFMulToFMadOrFMA()
5477 (Aggressive || MRI.hasOneNonDBGUse(RHS.Reg))) { in matchCombineFAddFMulToFMadOrFMA()
5500 DefinitionAndSourceRegister LHS = {MRI.getVRegDef(Op1), Op1}; in matchCombineFAddFpExtFMulToFMadOrFMA()
5501 DefinitionAndSourceRegister RHS = {MRI.getVRegDef(Op2), Op2}; in matchCombineFAddFpExtFMulToFMadOrFMA()
5502 LLT DstType = MRI.getType(MI.getOperand(0).getReg()); in matchCombineFAddFpExtFMulToFMadOrFMA()
5511 if (hasMoreUses(*LHS.MI, *RHS.MI, MRI)) in matchCombineFAddFpExtFMulToFMadOrFMA()
5517 if (mi_match(LHS.Reg, MRI, m_GFPExt(m_MInstr(FpExtSrc))) && in matchCombineFAddFpExtFMulToFMadOrFMA()
5520 MRI.getType(FpExtSrc->getOperand(1).getReg()))) { in matchCombineFAddFpExtFMulToFMadOrFMA()
5532 if (mi_match(RHS.Reg, MRI, m_GFPExt(m_MInstr(FpExtSrc))) && in matchCombineFAddFpExtFMulToFMadOrFMA()
5535 MRI.getType(FpExtSrc->getOperand(1).getReg()))) { in matchCombineFAddFpExtFMulToFMadOrFMA()
5558 DefinitionAndSourceRegister LHS = {MRI.getVRegDef(Op1), Op1}; in matchCombineFAddFMAFMulToFMadOrFMA()
5559 DefinitionAndSourceRegister RHS = {MRI.getVRegDef(Op2), Op2}; in matchCombineFAddFMAFMulToFMadOrFMA()
5560 LLT DstTy = MRI.getType(MI.getOperand(0).getReg()); in matchCombineFAddFMAFMulToFMadOrFMA()
5569 if (hasMoreUses(*LHS.MI, *RHS.MI, MRI)) in matchCombineFAddFMAFMulToFMadOrFMA()
5577 (MRI.getVRegDef(LHS.MI->getOperand(3).getReg())->getOpcode() == in matchCombineFAddFMAFMulToFMadOrFMA()
5579 MRI.hasOneNonDBGUse(LHS.MI->getOperand(0).getReg()) && in matchCombineFAddFMAFMulToFMadOrFMA()
5580 MRI.hasOneNonDBGUse(LHS.MI->getOperand(3).getReg())) { in matchCombineFAddFMAFMulToFMadOrFMA()
5586 (MRI.getVRegDef(RHS.MI->getOperand(3).getReg())->getOpcode() == in matchCombineFAddFMAFMulToFMadOrFMA()
5588 MRI.hasOneNonDBGUse(RHS.MI->getOperand(0).getReg()) && in matchCombineFAddFMAFMulToFMadOrFMA()
5589 MRI.hasOneNonDBGUse(RHS.MI->getOperand(3).getReg())) { in matchCombineFAddFMAFMulToFMadOrFMA()
5595 MachineInstr *FMulMI = MRI.getVRegDef(FMA->getOperand(3).getReg()); in matchCombineFAddFMAFMulToFMadOrFMA()
5602 Register InnerFMA = MRI.createGenericVirtualRegister(DstTy); in matchCombineFAddFMAFMulToFMadOrFMA()
5625 LLT DstType = MRI.getType(MI.getOperand(0).getReg()); in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5628 DefinitionAndSourceRegister LHS = {MRI.getVRegDef(Op1), Op1}; in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5629 DefinitionAndSourceRegister RHS = {MRI.getVRegDef(Op2), Op2}; in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5638 if (hasMoreUses(*LHS.MI, *RHS.MI, MRI)) in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5658 mi_match(LHS.MI->getOperand(3).getReg(), MRI, in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5662 MRI.getType(FMulMI->getOperand(0).getReg()))) { in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5677 if (mi_match(LHS.Reg, MRI, m_GFPExt(m_MInstr(FMAMI))) && in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5679 MachineInstr *FMulMI = MRI.getVRegDef(FMAMI->getOperand(3).getReg()); in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5682 MRI.getType(FMAMI->getOperand(0).getReg()))) { in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5699 mi_match(RHS.MI->getOperand(3).getReg(), MRI, in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5703 MRI.getType(FMulMI->getOperand(0).getReg()))) { in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5718 if (mi_match(RHS.Reg, MRI, m_GFPExt(m_MInstr(FMAMI))) && in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5720 MachineInstr *FMulMI = MRI.getVRegDef(FMAMI->getOperand(3).getReg()); in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5723 MRI.getType(FMAMI->getOperand(0).getReg()))) { in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5749 DefinitionAndSourceRegister LHS = {MRI.getVRegDef(Op1), Op1}; in matchCombineFSubFMulToFMadOrFMA()
5750 DefinitionAndSourceRegister RHS = {MRI.getVRegDef(Op2), Op2}; in matchCombineFSubFMulToFMadOrFMA()
5751 LLT DstTy = MRI.getType(MI.getOperand(0).getReg()); in matchCombineFSubFMulToFMadOrFMA()
5758 hasMoreUses(*LHS.MI, *RHS.MI, MRI)) in matchCombineFSubFMulToFMadOrFMA()
5767 (Aggressive || MRI.hasOneNonDBGUse(LHS.Reg)))) { in matchCombineFSubFMulToFMadOrFMA()
5778 (Aggressive || MRI.hasOneNonDBGUse(RHS.Reg)))) { in matchCombineFSubFMulToFMadOrFMA()
5801 LLT DstTy = MRI.getType(MI.getOperand(0).getReg()); in matchCombineFSubFNegFMulToFMadOrFMA()
5808 if (mi_match(LHSReg, MRI, m_GFNeg(m_MInstr(FMulMI))) && in matchCombineFSubFNegFMulToFMadOrFMA()
5809 (Aggressive || (MRI.hasOneNonDBGUse(LHSReg) && in matchCombineFSubFNegFMulToFMadOrFMA()
5810 MRI.hasOneNonDBGUse(FMulMI->getOperand(0).getReg()))) && in matchCombineFSubFNegFMulToFMadOrFMA()
5823 if (mi_match(RHSReg, MRI, m_GFNeg(m_MInstr(FMulMI))) && in matchCombineFSubFNegFMulToFMadOrFMA()
5824 (Aggressive || (MRI.hasOneNonDBGUse(RHSReg) && in matchCombineFSubFNegFMulToFMadOrFMA()
5825 MRI.hasOneNonDBGUse(FMulMI->getOperand(0).getReg()))) && in matchCombineFSubFNegFMulToFMadOrFMA()
5848 LLT DstTy = MRI.getType(MI.getOperand(0).getReg()); in matchCombineFSubFpExtFMulToFMadOrFMA()
5855 if (mi_match(LHSReg, MRI, m_GFPExt(m_MInstr(FMulMI))) && in matchCombineFSubFpExtFMulToFMadOrFMA()
5857 (Aggressive || MRI.hasOneNonDBGUse(LHSReg))) { in matchCombineFSubFpExtFMulToFMadOrFMA()
5871 if (mi_match(RHSReg, MRI, m_GFPExt(m_MInstr(FMulMI))) && in matchCombineFSubFpExtFMulToFMadOrFMA()
5873 (Aggressive || MRI.hasOneNonDBGUse(RHSReg))) { in matchCombineFSubFpExtFMulToFMadOrFMA()
5898 LLT DstTy = MRI.getType(MI.getOperand(0).getReg()); in matchCombineFSubFpExtFNegFMulToFMadOrFMA()
5917 if ((mi_match(LHSReg, MRI, m_GFPExt(m_GFNeg(m_MInstr(FMulMI)))) || in matchCombineFSubFpExtFNegFMulToFMadOrFMA()
5918 mi_match(LHSReg, MRI, m_GFNeg(m_GFPExt(m_MInstr(FMulMI))))) && in matchCombineFSubFpExtFNegFMulToFMadOrFMA()
5921 MRI.getType(FMulMI->getOperand(0).getReg()))) { in matchCombineFSubFpExtFNegFMulToFMadOrFMA()
5923 Register FMAReg = MRI.createGenericVirtualRegister(DstTy); in matchCombineFSubFpExtFNegFMulToFMadOrFMA()
5933 if ((mi_match(RHSReg, MRI, m_GFPExt(m_GFNeg(m_MInstr(FMulMI)))) || in matchCombineFSubFpExtFNegFMulToFMadOrFMA()
5934 mi_match(RHSReg, MRI, m_GFNeg(m_GFPExt(m_MInstr(FMulMI))))) && in matchCombineFSubFpExtFNegFMulToFMadOrFMA()
5937 MRI.getType(FMulMI->getOperand(0).getReg()))) { in matchCombineFSubFpExtFNegFMulToFMadOrFMA()
5966 const ConstantFP *MaybeCst = getConstantFPVRegVal(MaybeNaNReg, MRI); in matchCombineFMinMaxNaN()
5986 return mi_match(MaybeSub, MRI, m_GSub(m_Reg(Src), m_Reg(Reg))) && in matchAddSubSameReg()
6010 LLT DstVecTy = MRI.getType(MI.getOperand(0).getReg()); in matchBuildVectorIdentityFold()
6016 MI, MRI, in matchBuildVectorIdentityFold()
6019 return MRI.getType(MatchInfo) == DstVecTy; in matchBuildVectorIdentityFold()
6026 MI, MRI, in matchBuildVectorIdentityFold()
6031 return MRI.getType(MatchInfo) == DstVecTy; in matchBuildVectorIdentityFold()
6042 if (!mi_match(MI.getOperand(1).getReg(), MRI, in matchTruncBuildVectorFold()
6046 return MRI.getType(MatchInfo) == MRI.getType(MI.getOperand(0).getReg()); in matchTruncBuildVectorFold()
6054 if (!mi_match(MI.getOperand(1).getReg(), MRI, in matchTruncLshrBuildVectorFold()
6059 LLT MatchTy = MRI.getType(MatchInfo); in matchTruncLshrBuildVectorFold()
6061 MatchTy == MRI.getType(MI.getOperand(0).getReg()); in matchTruncLshrBuildVectorFold()
6106 bool LHSSafe = isKnownNeverNaN(LHS, MRI); in computeRetValAgainstNaN()
6107 bool RHSSafe = isKnownNeverNaN(RHS, MRI); in computeRetValAgainstNaN()
6130 LLT DstTy = MRI.getType(Dst); in matchFPSelectToMinMax()
6138 if (!mi_match(Cond, MRI, in matchFPSelectToMinMax()
6167 auto KnownNonZeroSide = getFConstantVRegValWithLookThrough(CmpLHS, MRI); in matchFPSelectToMinMax()
6169 KnownNonZeroSide = getFConstantVRegValWithLookThrough(CmpRHS, MRI); in matchFPSelectToMinMax()
6187 if (mi_match(Cond, MRI, m_OneNonDBGUse(m_GTrunc(m_Reg(MaybeTrunc))))) in matchSimplifySelectToMinMax()
6208 Dst, MRI, in matchRedundantBinOpInEquality()
6213 if (!mi_match(Dst, MRI, in matchRedundantBinOpInEquality()
6221 auto Zero = B.buildConstant(MRI.getType(Y), 0); in matchRedundantBinOpInEquality()
6229 LLT ResTy = MRI.getType(MI.getOperand(0).getReg()); in matchShiftsTooBig()
6234 return matchUnaryPredicate(MRI, ShiftReg, IsShiftTooBig); in matchShiftsTooBig()
6240 auto *LHSDef = MRI.getVRegDef(LHS); in matchCommuteConstantToRHS()
6241 if (getIConstantVRegVal(LHS, MRI).has_value()) in matchCommuteConstantToRHS()
6248 return MRI.getVRegDef(RHS)->getOpcode() != in matchCommuteConstantToRHS()
6250 !getIConstantVRegVal(RHS, MRI); in matchCommuteConstantToRHS()
6257 if (!mi_match(LHS, MRI, m_GFCstOrSplat(ValAndVReg))) in matchCommuteFPConstantToRHS()
6259 return !mi_match(RHS, MRI, m_GFCstOrSplat(ValAndVReg)); in matchCommuteFPConstantToRHS()
6272 LLT SrcTy = MRI.getType(Src); in isOneOrOneSplat()
6276 if (AllowUndefs && getOpcodeDef<GImplicitDef>(Src, MRI) != nullptr) in isOneOrOneSplat()
6278 auto IConstant = getIConstantVRegValWithLookThrough(Src, MRI); in isOneOrOneSplat()
6285 LLT SrcTy = MRI.getType(Src); in isZeroOrZeroSplat()
6289 if (AllowUndefs && getOpcodeDef<GImplicitDef>(Src, MRI) != nullptr) in isZeroOrZeroSplat()
6291 auto IConstant = getIConstantVRegValWithLookThrough(Src, MRI); in isZeroOrZeroSplat()
6301 GBuildVector *BuildVector = getOpcodeDef<GBuildVector>(Src, MRI); in isConstantSplatVector()
6308 getOpcodeDef<GImplicitDef>(BuildVector->getSourceReg(I), MRI); in isConstantSplatVector()
6314 getIConstantVRegValWithLookThrough(BuildVector->getSourceReg(I), MRI); in isConstantSplatVector()
6326 auto IConstant = getIConstantVRegValWithLookThrough(Src, MRI); in getConstantOrConstantSplatVector()
6330 GBuildVector *BuildVector = getOpcodeDef<GBuildVector>(Src, MRI); in getConstantOrConstantSplatVector()
6338 getIConstantVRegValWithLookThrough(BuildVector->getSourceReg(I), MRI); in getConstantOrConstantSplatVector()
6357 LLT CondTy = MRI.getType(Select->getCondReg()); in tryFoldSelectOfConstants()
6358 LLT TrueTy = MRI.getType(Select->getTrueReg()); in tryFoldSelectOfConstants()
6366 getIConstantVRegValWithLookThrough(True, MRI); in tryFoldSelectOfConstants()
6368 getIConstantVRegValWithLookThrough(False, MRI); in tryFoldSelectOfConstants()
6398 Register Inner = MRI.createGenericVirtualRegister(CondTy); in tryFoldSelectOfConstants()
6409 Register Inner = MRI.createGenericVirtualRegister(CondTy); in tryFoldSelectOfConstants()
6420 Register Inner = MRI.createGenericVirtualRegister(TrueTy); in tryFoldSelectOfConstants()
6431 Register Inner = MRI.createGenericVirtualRegister(TrueTy); in tryFoldSelectOfConstants()
6442 Register Inner = MRI.createGenericVirtualRegister(TrueTy); in tryFoldSelectOfConstants()
6455 Register Inner = MRI.createGenericVirtualRegister(TrueTy); in tryFoldSelectOfConstants()
6466 Register Not = MRI.createGenericVirtualRegister(CondTy); in tryFoldSelectOfConstants()
6468 Register Inner = MRI.createGenericVirtualRegister(TrueTy); in tryFoldSelectOfConstants()
6486 LLT CondTy = MRI.getType(Select->getCondReg()); in tryFoldBoolSelectToLogic()
6487 LLT TrueTy = MRI.getType(Select->getTrueReg()); in tryFoldBoolSelectToLogic()
6504 Register Ext = MRI.createGenericVirtualRegister(TrueTy); in tryFoldBoolSelectToLogic()
6516 Register Ext = MRI.createGenericVirtualRegister(TrueTy); in tryFoldBoolSelectToLogic()
6528 Register Inner = MRI.createGenericVirtualRegister(CondTy); in tryFoldBoolSelectToLogic()
6531 Register Ext = MRI.createGenericVirtualRegister(TrueTy); in tryFoldBoolSelectToLogic()
6543 Register Inner = MRI.createGenericVirtualRegister(CondTy); in tryFoldBoolSelectToLogic()
6546 Register Ext = MRI.createGenericVirtualRegister(TrueTy); in tryFoldBoolSelectToLogic()
6562 LLT DstTy = MRI.getType(DstReg); in tryFoldSelectToIntMinMax()
6568 GICmp *Cmp = getOpcodeDef<GICmp>(Cond, MRI); in tryFoldSelectToIntMinMax()
6573 if (!MRI.hasOneNonDBGUse(Cmp->getReg(0))) in tryFoldSelectToIntMinMax()