Lines Matching refs:Root
3706 AMDGPUInstructionSelector::selectVCSRC(MachineOperand &Root) const { in selectVCSRC()
3708 [=](MachineInstrBuilder &MIB) { MIB.add(Root); } in selectVCSRC()
3714 AMDGPUInstructionSelector::selectVOP3ModsImpl(MachineOperand &Root, in selectVOP3ModsImpl() argument
3717 Register Src = Root.getReg(); in selectVOP3ModsImpl()
3748 Register Src, unsigned Mods, MachineOperand Root, MachineInstr *InsertPt, in copyToVGPRIfSrcFolded() argument
3756 Register VGPRSrc = MRI->cloneVirtualRegister(Root.getReg()); in copyToVGPRIfSrcFolded()
3770 AMDGPUInstructionSelector::selectVSRC0(MachineOperand &Root) const { in selectVSRC0()
3772 [=](MachineInstrBuilder &MIB) { MIB.add(Root); } in selectVSRC0()
3777 AMDGPUInstructionSelector::selectVOP3Mods0(MachineOperand &Root) const { in selectVOP3Mods0()
3780 std::tie(Src, Mods) = selectVOP3ModsImpl(Root); in selectVOP3Mods0()
3784 MIB.addReg(copyToVGPRIfSrcFolded(Src, Mods, Root, MIB)); in selectVOP3Mods0()
3793 AMDGPUInstructionSelector::selectVOP3BMods0(MachineOperand &Root) const { in selectVOP3BMods0()
3796 std::tie(Src, Mods) = selectVOP3ModsImpl(Root, in selectVOP3BMods0()
3802 MIB.addReg(copyToVGPRIfSrcFolded(Src, Mods, Root, MIB)); in selectVOP3BMods0()
3811 AMDGPUInstructionSelector::selectVOP3OMods(MachineOperand &Root) const { in selectVOP3OMods()
3813 [=](MachineInstrBuilder &MIB) { MIB.add(Root); }, in selectVOP3OMods()
3820 AMDGPUInstructionSelector::selectVOP3Mods(MachineOperand &Root) const { in selectVOP3Mods()
3823 std::tie(Src, Mods) = selectVOP3ModsImpl(Root); in selectVOP3Mods()
3827 MIB.addReg(copyToVGPRIfSrcFolded(Src, Mods, Root, MIB)); in selectVOP3Mods()
3835 MachineOperand &Root) const { in selectVOP3ModsNonCanonicalizing()
3838 std::tie(Src, Mods) = selectVOP3ModsImpl(Root, /*IsCanonicalizing=*/false); in selectVOP3ModsNonCanonicalizing()
3842 MIB.addReg(copyToVGPRIfSrcFolded(Src, Mods, Root, MIB)); in selectVOP3ModsNonCanonicalizing()
3849 AMDGPUInstructionSelector::selectVOP3BMods(MachineOperand &Root) const { in selectVOP3BMods()
3852 std::tie(Src, Mods) = selectVOP3ModsImpl(Root, /*IsCanonicalizing=*/true, in selectVOP3BMods()
3857 MIB.addReg(copyToVGPRIfSrcFolded(Src, Mods, Root, MIB)); in selectVOP3BMods()
3864 AMDGPUInstructionSelector::selectVOP3NoMods(MachineOperand &Root) const { in selectVOP3NoMods()
3865 Register Reg = Root.getReg(); in selectVOP3NoMods()
3901 AMDGPUInstructionSelector::selectVOP3PMods(MachineOperand &Root) const { in selectVOP3PMods()
3903 = Root.getParent()->getParent()->getParent()->getRegInfo(); in selectVOP3PMods()
3907 std::tie(Src, Mods) = selectVOP3PModsImpl(Root.getReg(), MRI); in selectVOP3PMods()
3916 AMDGPUInstructionSelector::selectVOP3PModsDOT(MachineOperand &Root) const { in selectVOP3PModsDOT()
3918 = Root.getParent()->getParent()->getParent()->getRegInfo(); in selectVOP3PModsDOT()
3922 std::tie(Src, Mods) = selectVOP3PModsImpl(Root.getReg(), MRI, true); in selectVOP3PModsDOT()
3931 AMDGPUInstructionSelector::selectVOP3PModsNeg(MachineOperand &Root) const { in selectVOP3PModsNeg()
3935 assert((Root.isImm() && (Root.getImm() == -1 || Root.getImm() == 0)) && in selectVOP3PModsNeg()
3938 if (Root.getImm() == -1) in selectVOP3PModsNeg()
3947 MachineOperand &Root) const { in selectWMMAOpSelVOP3PMods()
3948 assert((Root.isImm() && (Root.getImm() == -1 || Root.getImm() == 0)) && in selectWMMAOpSelVOP3PMods()
3951 if (Root.getImm() != 0) in selectWMMAOpSelVOP3PMods()
4018 AMDGPUInstructionSelector::selectWMMAModsF32NegAbs(MachineOperand &Root) const { in selectWMMAModsF32NegAbs()
4019 Register Src = Root.getReg(); in selectWMMAModsF32NegAbs()
4038 selectWMMAModsNegAbs(ModOpcode, Mods, EltsF32, Src, Root.getParent(), in selectWMMAModsF32NegAbs()
4048 AMDGPUInstructionSelector::selectWMMAModsF16Neg(MachineOperand &Root) const { in selectWMMAModsF16Neg()
4049 Register Src = Root.getReg(); in selectWMMAModsF16Neg()
4065 Src = buildRegSequence(EltsV2F16, Root.getParent(), *MRI); in selectWMMAModsF16Neg()
4074 AMDGPUInstructionSelector::selectWMMAModsF16NegAbs(MachineOperand &Root) const { in selectWMMAModsF16NegAbs()
4075 Register Src = Root.getReg(); in selectWMMAModsF16NegAbs()
4094 MachineIRBuilder B(*Root.getParent()); in selectWMMAModsF16NegAbs()
4095 selectWMMAModsNegAbs(ModOpcode, Mods, EltsV2F16, Src, Root.getParent(), in selectWMMAModsF16NegAbs()
4105 AMDGPUInstructionSelector::selectWMMAVISrc(MachineOperand &Root) const { in selectWMMAVISrc()
4107 if (mi_match(Root.getReg(), *MRI, m_GFCstOrSplat(FPValReg))) { in selectWMMAVISrc()
4119 if (mi_match(Root.getReg(), *MRI, m_ICstOrSplat(ICst))) { in selectWMMAVISrc()
4130 AMDGPUInstructionSelector::selectSWMMACIndex8(MachineOperand &Root) const { in selectSWMMACIndex8()
4132 getDefIgnoringCopies(Root.getReg(), *MRI)->getOperand(0).getReg(); in selectSWMMACIndex8()
4151 AMDGPUInstructionSelector::selectSWMMACIndex16(MachineOperand &Root) const { in selectSWMMACIndex16()
4154 getDefIgnoringCopies(Root.getReg(), *MRI)->getOperand(0).getReg(); in selectSWMMACIndex16()
4173 AMDGPUInstructionSelector::selectVOP3OpSelMods(MachineOperand &Root) const { in selectVOP3OpSelMods()
4176 std::tie(Src, Mods) = selectVOP3ModsImpl(Root); in selectVOP3OpSelMods()
4186 AMDGPUInstructionSelector::selectVINTERPMods(MachineOperand &Root) const { in selectVINTERPMods()
4189 std::tie(Src, Mods) = selectVOP3ModsImpl(Root, in selectVINTERPMods()
4197 copyToVGPRIfSrcFolded(Src, Mods, Root, MIB, /* ForceVGPR */ true)); in selectVINTERPMods()
4204 AMDGPUInstructionSelector::selectVINTERPModsHi(MachineOperand &Root) const { in selectVINTERPModsHi()
4207 std::tie(Src, Mods) = selectVOP3ModsImpl(Root, in selectVINTERPModsHi()
4215 copyToVGPRIfSrcFolded(Src, Mods, Root, MIB, /* ForceVGPR */ true)); in selectVINTERPModsHi()
4221 bool AMDGPUInstructionSelector::selectSmrdOffset(MachineOperand &Root, in selectSmrdOffset() argument
4225 MachineInstr *MI = Root.getParent(); in selectSmrdOffset()
4289 AMDGPUInstructionSelector::selectSmrdImm(MachineOperand &Root) const { in selectSmrdImm()
4292 if (!selectSmrdOffset(Root, Base, /* SOffset= */ nullptr, &Offset)) in selectSmrdImm()
4300 AMDGPUInstructionSelector::selectSmrdImm32(MachineOperand &Root) const { in selectSmrdImm32()
4302 getAddrModeInfo(*Root.getParent(), *MRI, AddrInfo); in selectSmrdImm32()
4321 AMDGPUInstructionSelector::selectSmrdSgpr(MachineOperand &Root) const { in selectSmrdSgpr()
4323 if (!selectSmrdOffset(Root, Base, &SOffset, /* Offset= */ nullptr)) in selectSmrdSgpr()
4331 AMDGPUInstructionSelector::selectSmrdSgprImm(MachineOperand &Root) const { in selectSmrdSgprImm()
4334 if (!selectSmrdOffset(Root, Base, &SOffset, &Offset)) in selectSmrdSgprImm()
4343 AMDGPUInstructionSelector::selectFlatOffsetImpl(MachineOperand &Root, in selectFlatOffsetImpl() argument
4345 MachineInstr *MI = Root.getParent(); in selectFlatOffsetImpl()
4347 auto Default = std::pair(Root.getReg(), 0); in selectFlatOffsetImpl()
4355 getPtrBaseWithConstantOffset(Root.getReg(), *MRI); in selectFlatOffsetImpl()
4358 !isFlatScratchBaseLegal(Root.getReg()))) in selectFlatOffsetImpl()
4369 AMDGPUInstructionSelector::selectFlatOffset(MachineOperand &Root) const { in selectFlatOffset()
4370 auto PtrWithOffset = selectFlatOffsetImpl(Root, SIInstrFlags::FLAT); in selectFlatOffset()
4379 AMDGPUInstructionSelector::selectGlobalOffset(MachineOperand &Root) const { in selectGlobalOffset()
4380 auto PtrWithOffset = selectFlatOffsetImpl(Root, SIInstrFlags::FlatGlobal); in selectGlobalOffset()
4389 AMDGPUInstructionSelector::selectScratchOffset(MachineOperand &Root) const { in selectScratchOffset()
4390 auto PtrWithOffset = selectFlatOffsetImpl(Root, SIInstrFlags::FlatScratch); in selectScratchOffset()
4400 AMDGPUInstructionSelector::selectGlobalSAddr(MachineOperand &Root) const { in selectGlobalSAddr()
4401 Register Addr = Root.getReg(); in selectGlobalSAddr()
4429 MachineInstr *MI = Root.getParent(); in selectGlobalSAddr()
4496 MachineInstr *MI = Root.getParent(); in selectGlobalSAddr()
4511 AMDGPUInstructionSelector::selectScratchSAddr(MachineOperand &Root) const { in selectScratchSAddr()
4512 Register Addr = Root.getReg(); in selectScratchSAddr()
4548 MachineInstr &I = *Root.getParent(); in selectScratchSAddr()
4588 AMDGPUInstructionSelector::selectScratchSVAddr(MachineOperand &Root) const { in selectScratchSVAddr()
4589 Register Addr = Root.getReg(); in selectScratchSVAddr()
4647 AMDGPUInstructionSelector::selectMUBUFScratchOffen(MachineOperand &Root) const { in selectMUBUFScratchOffen()
4648 MachineInstr *MI = Root.getParent(); in selectMUBUFScratchOffen()
4654 if (mi_match(Root.getReg(), *MRI, m_ICst(Offset)) && in selectMUBUFScratchOffen()
4686 Register VAddr = Root.getReg(); in selectMUBUFScratchOffen()
4687 if (const MachineInstr *RootDef = MRI->getVRegDef(Root.getReg())) { in selectMUBUFScratchOffen()
4862 MachineOperand &Root) const { in selectMUBUFScratchOffset()
4863 Register Reg = Root.getReg(); in selectMUBUFScratchOffset()
4908 if (!mi_match(Root.getReg(), *MRI, m_ICst(Offset)) || in selectMUBUFScratchOffset()
4924 AMDGPUInstructionSelector::selectDS1Addr1OffsetImpl(MachineOperand &Root) const { in selectDS1Addr1OffsetImpl()
4925 const MachineInstr *RootDef = MRI->getVRegDef(Root.getReg()); in selectDS1Addr1OffsetImpl()
4927 return std::pair(Root.getReg(), 0); in selectDS1Addr1OffsetImpl()
4934 getPtrBaseWithConstantOffset(Root.getReg(), *MRI); in selectDS1Addr1OffsetImpl()
4945 } else if (mi_match(Root.getReg(), *MRI, m_ICst(ConstAddr))) { in selectDS1Addr1OffsetImpl()
4950 return std::pair(Root.getReg(), 0); in selectDS1Addr1OffsetImpl()
4954 AMDGPUInstructionSelector::selectDS1Addr1Offset(MachineOperand &Root) const { in selectDS1Addr1Offset()
4957 std::tie(Reg, Offset) = selectDS1Addr1OffsetImpl(Root); in selectDS1Addr1Offset()
4965 AMDGPUInstructionSelector::selectDS64Bit4ByteAligned(MachineOperand &Root) const { in selectDS64Bit4ByteAligned()
4966 return selectDSReadWrite2(Root, 4); in selectDS64Bit4ByteAligned()
4970 AMDGPUInstructionSelector::selectDS128Bit8ByteAligned(MachineOperand &Root) const { in selectDS128Bit8ByteAligned()
4971 return selectDSReadWrite2(Root, 8); in selectDS128Bit8ByteAligned()
4975 AMDGPUInstructionSelector::selectDSReadWrite2(MachineOperand &Root, in selectDSReadWrite2() argument
4979 std::tie(Reg, Offset) = selectDSReadWrite2Impl(Root, Size); in selectDSReadWrite2()
4988 AMDGPUInstructionSelector::selectDSReadWrite2Impl(MachineOperand &Root, in selectDSReadWrite2Impl() argument
4990 const MachineInstr *RootDef = MRI->getVRegDef(Root.getReg()); in selectDSReadWrite2Impl()
4992 return std::pair(Root.getReg(), 0); in selectDSReadWrite2Impl()
4999 getPtrBaseWithConstantOffset(Root.getReg(), *MRI); in selectDSReadWrite2Impl()
5011 } else if (mi_match(Root.getReg(), *MRI, m_ICst(ConstAddr))) { in selectDSReadWrite2Impl()
5016 return std::pair(Root.getReg(), 0); in selectDSReadWrite2Impl()
5025 Register Root, const MachineRegisterInfo &MRI) const { in getPtrBaseWithConstantOffset() argument
5026 MachineInstr *RootI = getDefIgnoringCopies(Root, MRI); in getPtrBaseWithConstantOffset()
5028 return {Root, 0}; in getPtrBaseWithConstantOffset()
5034 return {Root, 0}; in getPtrBaseWithConstantOffset()
5164 MachineOperand &Root, Register &VAddr, Register &RSrcReg, in selectMUBUFAddr64Impl() argument
5171 MUBUFAddressData AddrData = parseMUBUFAddress(Root.getReg()); in selectMUBUFAddr64Impl()
5208 MachineIRBuilder B(*Root.getParent()); in selectMUBUFAddr64Impl()
5215 MachineOperand &Root, Register &RSrcReg, Register &SOffset, in selectMUBUFOffsetImpl() argument
5222 MUBUFAddressData AddrData = parseMUBUFAddress(Root.getReg()); in selectMUBUFOffsetImpl()
5232 MachineIRBuilder B(*Root.getParent()); in selectMUBUFOffsetImpl()
5240 AMDGPUInstructionSelector::selectMUBUFAddr64(MachineOperand &Root) const { in selectMUBUFAddr64()
5246 if (!selectMUBUFAddr64Impl(Root, VAddr, RSrcReg, SOffset, Offset)) in selectMUBUFAddr64()
5276 AMDGPUInstructionSelector::selectMUBUFOffset(MachineOperand &Root) const { in selectMUBUFOffset()
5281 if (!selectMUBUFOffsetImpl(Root, RSrcReg, SOffset, Offset)) in selectMUBUFOffset()
5304 AMDGPUInstructionSelector::selectBUFSOffset(MachineOperand &Root) const { in selectBUFSOffset()
5306 Register SOffset = Root.getReg(); in selectBUFSOffset()
5325 AMDGPUInstructionSelector::selectSMRDBufferImm(MachineOperand &Root) const { in selectSMRDBufferImm()
5326 std::optional<uint64_t> OffsetVal = getConstantZext32Val(Root.getReg(), *MRI); in selectSMRDBufferImm()
5339 AMDGPUInstructionSelector::selectSMRDBufferImm32(MachineOperand &Root) const { in selectSMRDBufferImm32()
5342 std::optional<uint64_t> OffsetVal = getConstantZext32Val(Root.getReg(), *MRI); in selectSMRDBufferImm32()
5355 AMDGPUInstructionSelector::selectSMRDBufferSgprImm(MachineOperand &Root) const { in selectSMRDBufferSgprImm()
5361 *MRI, Root.getReg(), KB, /*CheckNUW*/ true); in selectSMRDBufferSgprImm()
5429 AMDGPUInstructionSelector::selectVOP3PMadMixModsImpl(MachineOperand &Root, in selectVOP3PMadMixModsImpl() argument
5435 std::tie(Src, Mods) = selectVOP3ModsImpl(Root); in selectVOP3PMadMixModsImpl()
5495 MachineOperand &Root) const { in selectVOP3PMadMixModsExt()
5499 std::tie(Src, Mods) = selectVOP3PMadMixModsImpl(Root, Matched); in selectVOP3PMadMixModsExt()
5510 AMDGPUInstructionSelector::selectVOP3PMadMixMods(MachineOperand &Root) const { in selectVOP3PMadMixMods()
5514 std::tie(Src, Mods) = selectVOP3PMadMixModsImpl(Root, Matched); in selectVOP3PMadMixMods()