Lines Matching refs:CI
204 static bool dmasksCanBeCombined(const CombineInfo &CI,
207 static bool offsetsCanBeCombined(CombineInfo &CI, const GCNSubtarget &STI,
209 static bool widthsFit(const GCNSubtarget &STI, const CombineInfo &CI,
211 static unsigned getNewOpcode(const CombineInfo &CI, const CombineInfo &Paired);
212 static std::pair<unsigned, unsigned> getSubRegIdxs(const CombineInfo &CI,
214 const TargetRegisterClass *getTargetRegisterClass(const CombineInfo &CI,
218 CombineInfo *checkAndPrepareMerge(CombineInfo &CI, CombineInfo &Paired);
223 mergeRead2Pair(CombineInfo &CI, CombineInfo &Paired,
229 mergeWrite2Pair(CombineInfo &CI, CombineInfo &Paired,
232 mergeImagePair(CombineInfo &CI, CombineInfo &Paired,
235 mergeSBufferLoadImmPair(CombineInfo &CI, CombineInfo &Paired,
238 mergeBufferLoadPair(CombineInfo &CI, CombineInfo &Paired,
241 mergeBufferStorePair(CombineInfo &CI, CombineInfo &Paired,
244 mergeTBufferLoadPair(CombineInfo &CI, CombineInfo &Paired,
247 mergeTBufferStorePair(CombineInfo &CI, CombineInfo &Paired,
250 mergeFlatLoadPair(CombineInfo &CI, CombineInfo &Paired,
253 mergeFlatStorePair(CombineInfo &CI, CombineInfo &Paired,
265 bool promoteConstantOffsetToImm(MachineInstr &CI,
268 void addInstToMergeableList(const CombineInfo &CI,
276 static MachineMemOperand *combineKnownAdjacentMMOs(const CombineInfo &CI,
279 static InstClassEnum getCommonInstClass(const CombineInfo &CI,
540 SILoadStoreOptimizer::getCommonInstClass(const CombineInfo &CI, in getCommonInstClass() argument
542 assert(CI.InstClass == Paired.InstClass); in getCommonInstClass()
544 if ((CI.InstClass == FLAT_LOAD || CI.InstClass == FLAT_STORE) && in getCommonInstClass()
545 SIInstrInfo::isFLATGlobal(*CI.I) && SIInstrInfo::isFLATGlobal(*Paired.I)) in getCommonInstClass()
546 return (CI.InstClass == FLAT_STORE) ? GLOBAL_STORE : GLOBAL_LOAD; in getCommonInstClass()
548 return CI.InstClass; in getCommonInstClass()
773 SILoadStoreOptimizer::combineKnownAdjacentMMOs(const CombineInfo &CI, in combineKnownAdjacentMMOs() argument
775 const MachineMemOperand *MMOa = *CI.I->memoperands_begin(); in combineKnownAdjacentMMOs()
782 if (Paired < CI) in combineKnownAdjacentMMOs()
790 MachineFunction *MF = CI.I->getMF(); in combineKnownAdjacentMMOs()
794 bool SILoadStoreOptimizer::dmasksCanBeCombined(const CombineInfo &CI, in dmasksCanBeCombined() argument
797 assert(CI.InstClass == MIMG); in dmasksCanBeCombined()
800 const auto *TFEOp = TII.getNamedOperand(*CI.I, AMDGPU::OpName::tfe); in dmasksCanBeCombined()
801 const auto *LWEOp = TII.getNamedOperand(*CI.I, AMDGPU::OpName::lwe); in dmasksCanBeCombined()
812 int Idx = AMDGPU::getNamedOperandIdx(CI.I->getOpcode(), op); in dmasksCanBeCombined()
816 CI.I->getOperand(Idx).getImm() != Paired.I->getOperand(Idx).getImm()) in dmasksCanBeCombined()
821 unsigned MaxMask = std::max(CI.DMask, Paired.DMask); in dmasksCanBeCombined()
822 unsigned MinMask = std::min(CI.DMask, Paired.DMask); in dmasksCanBeCombined()
866 bool SILoadStoreOptimizer::offsetsCanBeCombined(CombineInfo &CI, in offsetsCanBeCombined() argument
870 assert(CI.InstClass != MIMG); in offsetsCanBeCombined()
874 if (CI.Offset == Paired.Offset) in offsetsCanBeCombined()
878 if ((CI.Offset % CI.EltSize != 0) || (Paired.Offset % CI.EltSize != 0)) in offsetsCanBeCombined()
881 if (CI.InstClass == TBUFFER_LOAD || CI.InstClass == TBUFFER_STORE) { in offsetsCanBeCombined()
884 llvm::AMDGPU::getGcnBufferFormatInfo(CI.Format, STI); in offsetsCanBeCombined()
901 if (getBufferFormatWithCompCount(CI.Format, CI.Width + Paired.Width, STI) == 0) in offsetsCanBeCombined()
905 uint32_t EltOffset0 = CI.Offset / CI.EltSize; in offsetsCanBeCombined()
906 uint32_t EltOffset1 = Paired.Offset / CI.EltSize; in offsetsCanBeCombined()
907 CI.UseST64 = false; in offsetsCanBeCombined()
908 CI.BaseOff = 0; in offsetsCanBeCombined()
911 if ((CI.InstClass != DS_READ) && (CI.InstClass != DS_WRITE)) { in offsetsCanBeCombined()
912 return (EltOffset0 + CI.Width == EltOffset1 || in offsetsCanBeCombined()
914 CI.CPol == Paired.CPol; in offsetsCanBeCombined()
922 CI.Offset = EltOffset0 / 64; in offsetsCanBeCombined()
924 CI.UseST64 = true; in offsetsCanBeCombined()
932 CI.Offset = EltOffset0; in offsetsCanBeCombined()
952 CI.BaseOff = BaseOff * CI.EltSize; in offsetsCanBeCombined()
953 CI.Offset = (EltOffset0 - BaseOff) / 64; in offsetsCanBeCombined()
955 CI.UseST64 = true; in offsetsCanBeCombined()
966 CI.BaseOff = BaseOff * CI.EltSize; in offsetsCanBeCombined()
967 CI.Offset = EltOffset0 - BaseOff; in offsetsCanBeCombined()
977 const CombineInfo &CI, in widthsFit() argument
979 const unsigned Width = (CI.Width + Paired.Width); in widthsFit()
980 switch (CI.InstClass) { in widthsFit()
1018 SILoadStoreOptimizer::checkAndPrepareMerge(CombineInfo &CI, in checkAndPrepareMerge() argument
1022 if (CI.InstClass == UNKNOWN || Paired.InstClass == UNKNOWN) in checkAndPrepareMerge()
1024 assert(CI.InstClass == Paired.InstClass); in checkAndPrepareMerge()
1026 if (getInstSubclass(CI.I->getOpcode(), *TII) != in checkAndPrepareMerge()
1032 if (CI.InstClass == MIMG) { in checkAndPrepareMerge()
1033 if (!dmasksCanBeCombined(CI, *TII, Paired)) in checkAndPrepareMerge()
1036 if (!widthsFit(*STM, CI, Paired) || !offsetsCanBeCombined(CI, *STM, Paired)) in checkAndPrepareMerge()
1043 if (CI.I->mayLoad()) { in checkAndPrepareMerge()
1046 for (MachineBasicBlock::iterator MBBI = Paired.I; --MBBI != CI.I;) { in checkAndPrepareMerge()
1050 Where = &CI; in checkAndPrepareMerge()
1053 addDefsUsesToList(*CI.I, RegDefs, RegUses); in checkAndPrepareMerge()
1054 for (MachineBasicBlock::iterator MBBI = CI.I; ++MBBI != Paired.I;) { in checkAndPrepareMerge()
1055 if (!canSwapInstructions(RegDefs, RegUses, *CI.I, *MBBI)) in checkAndPrepareMerge()
1065 if (CI.InstClass == DS_READ || CI.InstClass == DS_WRITE) in checkAndPrepareMerge()
1066 offsetsCanBeCombined(CI, *STM, Paired, true); in checkAndPrepareMerge()
1085 SILoadStoreOptimizer::mergeRead2Pair(CombineInfo &CI, CombineInfo &Paired, in mergeRead2Pair() argument
1087 MachineBasicBlock *MBB = CI.I->getParent(); in mergeRead2Pair()
1091 const auto *AddrReg = TII->getNamedOperand(*CI.I, AMDGPU::OpName::addr); in mergeRead2Pair()
1093 const auto *Dest0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::vdst); in mergeRead2Pair()
1096 unsigned NewOffset0 = CI.Offset; in mergeRead2Pair()
1099 CI.UseST64 ? read2ST64Opcode(CI.EltSize) : read2Opcode(CI.EltSize); in mergeRead2Pair()
1101 unsigned SubRegIdx0 = (CI.EltSize == 4) ? AMDGPU::sub0 : AMDGPU::sub0_sub1; in mergeRead2Pair()
1102 unsigned SubRegIdx1 = (CI.EltSize == 4) ? AMDGPU::sub1 : AMDGPU::sub2_sub3; in mergeRead2Pair()
1115 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeRead2Pair()
1118 DebugLoc DL = CI.I->getDebugLoc(); in mergeRead2Pair()
1123 if (CI.BaseOff) { in mergeRead2Pair()
1126 .addImm(CI.BaseOff); in mergeRead2Pair()
1144 .cloneMergedMemRefs({&*CI.I, &*Paired.I}); in mergeRead2Pair()
1158 CI.I->eraseFromParent(); in mergeRead2Pair()
1182 CombineInfo &CI, CombineInfo &Paired, in mergeWrite2Pair() argument
1184 MachineBasicBlock *MBB = CI.I->getParent(); in mergeWrite2Pair()
1189 TII->getNamedOperand(*CI.I, AMDGPU::OpName::addr); in mergeWrite2Pair()
1191 TII->getNamedOperand(*CI.I, AMDGPU::OpName::data0); in mergeWrite2Pair()
1195 unsigned NewOffset0 = CI.Offset; in mergeWrite2Pair()
1198 CI.UseST64 ? write2ST64Opcode(CI.EltSize) : write2Opcode(CI.EltSize); in mergeWrite2Pair()
1210 DebugLoc DL = CI.I->getDebugLoc(); in mergeWrite2Pair()
1215 if (CI.BaseOff) { in mergeWrite2Pair()
1218 .addImm(CI.BaseOff); in mergeWrite2Pair()
1238 .cloneMergedMemRefs({&*CI.I, &*Paired.I}); in mergeWrite2Pair()
1240 CI.I->eraseFromParent(); in mergeWrite2Pair()
1248 SILoadStoreOptimizer::mergeImagePair(CombineInfo &CI, CombineInfo &Paired, in mergeImagePair() argument
1250 MachineBasicBlock *MBB = CI.I->getParent(); in mergeImagePair()
1251 DebugLoc DL = CI.I->getDebugLoc(); in mergeImagePair()
1252 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeImagePair()
1254 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeImagePair()
1257 unsigned MergedDMask = CI.DMask | Paired.DMask; in mergeImagePair()
1259 AMDGPU::getNamedOperandIdx(CI.I->getOpcode(), AMDGPU::OpName::dmask); in mergeImagePair()
1262 for (unsigned I = 1, E = (*CI.I).getNumOperands(); I != E; ++I) { in mergeImagePair()
1266 MIB.add((*CI.I).getOperand(I)); in mergeImagePair()
1272 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeImagePair()
1274 MachineInstr *New = MIB.addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeImagePair()
1277 std::tie(SubRegIdx0, SubRegIdx1) = getSubRegIdxs(CI, Paired); in mergeImagePair()
1281 const auto *Dest0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::vdata); in mergeImagePair()
1291 CI.I->eraseFromParent(); in mergeImagePair()
1297 CombineInfo &CI, CombineInfo &Paired, in mergeSBufferLoadImmPair() argument
1299 MachineBasicBlock *MBB = CI.I->getParent(); in mergeSBufferLoadImmPair()
1300 DebugLoc DL = CI.I->getDebugLoc(); in mergeSBufferLoadImmPair()
1301 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeSBufferLoadImmPair()
1303 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeSBufferLoadImmPair()
1306 unsigned MergedOffset = std::min(CI.Offset, Paired.Offset); in mergeSBufferLoadImmPair()
1311 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeSBufferLoadImmPair()
1315 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::sbase)) in mergeSBufferLoadImmPair()
1317 .addImm(CI.CPol) // cpol in mergeSBufferLoadImmPair()
1318 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeSBufferLoadImmPair()
1320 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); in mergeSBufferLoadImmPair()
1326 const auto *Dest0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::sdst); in mergeSBufferLoadImmPair()
1336 CI.I->eraseFromParent(); in mergeSBufferLoadImmPair()
1342 CombineInfo &CI, CombineInfo &Paired, in mergeBufferLoadPair() argument
1344 MachineBasicBlock *MBB = CI.I->getParent(); in mergeBufferLoadPair()
1345 DebugLoc DL = CI.I->getDebugLoc(); in mergeBufferLoadPair()
1347 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeBufferLoadPair()
1349 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeBufferLoadPair()
1353 unsigned MergedOffset = std::min(CI.Offset, Paired.Offset); in mergeBufferLoadPair()
1360 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr)); in mergeBufferLoadPair()
1365 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeBufferLoadPair()
1368 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::srsrc)) in mergeBufferLoadPair()
1369 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::soffset)) in mergeBufferLoadPair()
1371 .addImm(CI.CPol) // cpol in mergeBufferLoadPair()
1374 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeBufferLoadPair()
1376 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); in mergeBufferLoadPair()
1382 const auto *Dest0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::vdata); in mergeBufferLoadPair()
1392 CI.I->eraseFromParent(); in mergeBufferLoadPair()
1398 CombineInfo &CI, CombineInfo &Paired, in mergeTBufferLoadPair() argument
1400 MachineBasicBlock *MBB = CI.I->getParent(); in mergeTBufferLoadPair()
1401 DebugLoc DL = CI.I->getDebugLoc(); in mergeTBufferLoadPair()
1403 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeTBufferLoadPair()
1405 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeTBufferLoadPair()
1409 unsigned MergedOffset = std::min(CI.Offset, Paired.Offset); in mergeTBufferLoadPair()
1416 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr)); in mergeTBufferLoadPair()
1419 getBufferFormatWithCompCount(CI.Format, CI.Width + Paired.Width, *STM); in mergeTBufferLoadPair()
1424 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeTBufferLoadPair()
1427 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::srsrc)) in mergeTBufferLoadPair()
1428 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::soffset)) in mergeTBufferLoadPair()
1431 .addImm(CI.CPol) // cpol in mergeTBufferLoadPair()
1434 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeTBufferLoadPair()
1436 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); in mergeTBufferLoadPair()
1442 const auto *Dest0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::vdata); in mergeTBufferLoadPair()
1452 CI.I->eraseFromParent(); in mergeTBufferLoadPair()
1458 CombineInfo &CI, CombineInfo &Paired, in mergeTBufferStorePair() argument
1460 MachineBasicBlock *MBB = CI.I->getParent(); in mergeTBufferStorePair()
1461 DebugLoc DL = CI.I->getDebugLoc(); in mergeTBufferStorePair()
1463 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeTBufferStorePair()
1465 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); in mergeTBufferStorePair()
1470 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeTBufferStorePair()
1473 const auto *Src0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::vdata); in mergeTBufferStorePair()
1488 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr)); in mergeTBufferStorePair()
1491 getBufferFormatWithCompCount(CI.Format, CI.Width + Paired.Width, *STM); in mergeTBufferStorePair()
1496 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeTBufferStorePair()
1499 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::srsrc)) in mergeTBufferStorePair()
1500 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::soffset)) in mergeTBufferStorePair()
1501 .addImm(std::min(CI.Offset, Paired.Offset)) // offset in mergeTBufferStorePair()
1503 .addImm(CI.CPol) // cpol in mergeTBufferStorePair()
1506 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeTBufferStorePair()
1508 CI.I->eraseFromParent(); in mergeTBufferStorePair()
1514 CombineInfo &CI, CombineInfo &Paired, in mergeFlatLoadPair() argument
1516 MachineBasicBlock *MBB = CI.I->getParent(); in mergeFlatLoadPair()
1517 DebugLoc DL = CI.I->getDebugLoc(); in mergeFlatLoadPair()
1519 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeFlatLoadPair()
1521 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeFlatLoadPair()
1526 if (auto *SAddr = TII->getNamedOperand(*CI.I, AMDGPU::OpName::saddr)) in mergeFlatLoadPair()
1530 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr)) in mergeFlatLoadPair()
1531 .addImm(std::min(CI.Offset, Paired.Offset)) in mergeFlatLoadPair()
1532 .addImm(CI.CPol) in mergeFlatLoadPair()
1533 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeFlatLoadPair()
1535 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); in mergeFlatLoadPair()
1541 const auto *Dest0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::vdst); in mergeFlatLoadPair()
1551 CI.I->eraseFromParent(); in mergeFlatLoadPair()
1557 CombineInfo &CI, CombineInfo &Paired, in mergeFlatStorePair() argument
1559 MachineBasicBlock *MBB = CI.I->getParent(); in mergeFlatStorePair()
1560 DebugLoc DL = CI.I->getDebugLoc(); in mergeFlatStorePair()
1562 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeFlatStorePair()
1564 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); in mergeFlatStorePair()
1569 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeFlatStorePair()
1572 const auto *Src0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::vdata); in mergeFlatStorePair()
1582 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr)) in mergeFlatStorePair()
1585 if (auto *SAddr = TII->getNamedOperand(*CI.I, AMDGPU::OpName::saddr)) in mergeFlatStorePair()
1589 MIB.addImm(std::min(CI.Offset, Paired.Offset)) in mergeFlatStorePair()
1590 .addImm(CI.CPol) in mergeFlatStorePair()
1591 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeFlatStorePair()
1593 CI.I->eraseFromParent(); in mergeFlatStorePair()
1598 unsigned SILoadStoreOptimizer::getNewOpcode(const CombineInfo &CI, in getNewOpcode() argument
1600 const unsigned Width = CI.Width + Paired.Width; in getNewOpcode()
1602 switch (getCommonInstClass(CI, Paired)) { in getNewOpcode()
1604 assert(CI.InstClass == BUFFER_LOAD || CI.InstClass == BUFFER_STORE); in getNewOpcode()
1606 return AMDGPU::getMUBUFOpcode(AMDGPU::getMUBUFBaseOpcode(CI.I->getOpcode()), in getNewOpcode()
1610 return AMDGPU::getMTBUFOpcode(AMDGPU::getMTBUFBaseOpcode(CI.I->getOpcode()), in getNewOpcode()
1693 assert((countPopulation(CI.DMask | Paired.DMask) == Width) && in getNewOpcode()
1695 return AMDGPU::getMaskedMIMGOp(CI.I->getOpcode(), Width); in getNewOpcode()
1700 SILoadStoreOptimizer::getSubRegIdxs(const CombineInfo &CI, in getSubRegIdxs() argument
1702 assert((CI.InstClass != MIMG || (countPopulation(CI.DMask | Paired.DMask) == in getSubRegIdxs()
1703 CI.Width + Paired.Width)) && in getSubRegIdxs()
1717 assert(CI.Width >= 1 && CI.Width <= 4); in getSubRegIdxs()
1720 if (Paired < CI) { in getSubRegIdxs()
1722 Idx0 = Idxs[Paired.Width][CI.Width - 1]; in getSubRegIdxs()
1724 Idx0 = Idxs[0][CI.Width - 1]; in getSubRegIdxs()
1725 Idx1 = Idxs[CI.Width][Paired.Width - 1]; in getSubRegIdxs()
1732 SILoadStoreOptimizer::getTargetRegisterClass(const CombineInfo &CI, in getTargetRegisterClass() argument
1734 if (CI.InstClass == S_BUFFER_LOAD_IMM) { in getTargetRegisterClass()
1735 switch (CI.Width + Paired.Width) { in getTargetRegisterClass()
1749 unsigned BitWidth = 32 * (CI.Width + Paired.Width); in getTargetRegisterClass()
1750 return TRI->isAGPRClass(getDataRegClass(*CI.I)) in getTargetRegisterClass()
1756 CombineInfo &CI, CombineInfo &Paired, in mergeBufferStorePair() argument
1758 MachineBasicBlock *MBB = CI.I->getParent(); in mergeBufferStorePair()
1759 DebugLoc DL = CI.I->getDebugLoc(); in mergeBufferStorePair()
1761 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeBufferStorePair()
1763 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); in mergeBufferStorePair()
1768 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeBufferStorePair()
1771 const auto *Src0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::vdata); in mergeBufferStorePair()
1786 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr)); in mergeBufferStorePair()
1792 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeBufferStorePair()
1795 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::srsrc)) in mergeBufferStorePair()
1796 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::soffset)) in mergeBufferStorePair()
1797 .addImm(std::min(CI.Offset, Paired.Offset)) // offset in mergeBufferStorePair()
1798 .addImm(CI.CPol) // cpol in mergeBufferStorePair()
1801 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeBufferStorePair()
1803 CI.I->eraseFromParent(); in mergeBufferStorePair()
2121 void SILoadStoreOptimizer::addInstToMergeableList(const CombineInfo &CI, in addInstToMergeableList() argument
2124 if (AddrList.front().InstClass == CI.InstClass && in addInstToMergeableList()
2125 AddrList.front().IsAGPR == CI.IsAGPR && in addInstToMergeableList()
2126 AddrList.front().hasSameBaseAddress(*CI.I)) { in addInstToMergeableList()
2127 AddrList.emplace_back(CI); in addInstToMergeableList()
2133 MergeableInsts.emplace_back(1, CI); in addInstToMergeableList()
2174 CombineInfo CI; in collectMergeableInsts() local
2175 CI.setMI(MI, *this); in collectMergeableInsts()
2176 CI.Order = Order++; in collectMergeableInsts()
2178 if (!CI.hasMergeableAddress(*MRI)) in collectMergeableInsts()
2181 if (CI.InstClass == DS_WRITE && CI.IsAGPR) { in collectMergeableInsts()
2193 addInstToMergeableList(CI, MergeableInsts); in collectMergeableInsts()
2278 CombineInfo &CI = *First; in optimizeInstsWithSameBaseAddr() local
2281 CombineInfo *Where = checkAndPrepareMerge(CI, Paired); in optimizeInstsWithSameBaseAddr()
2289 LLVM_DEBUG(dbgs() << "Merging: " << *CI.I << " with: " << *Paired.I); in optimizeInstsWithSameBaseAddr()
2292 switch (CI.InstClass) { in optimizeInstsWithSameBaseAddr()
2297 NewMI = mergeRead2Pair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2300 NewMI = mergeWrite2Pair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2303 NewMI = mergeSBufferLoadImmPair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2304 OptimizeListAgain |= CI.Width + Paired.Width < 8; in optimizeInstsWithSameBaseAddr()
2307 NewMI = mergeBufferLoadPair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2308 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2311 NewMI = mergeBufferStorePair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2312 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2315 NewMI = mergeImagePair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2316 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2319 NewMI = mergeTBufferLoadPair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2320 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2323 NewMI = mergeTBufferStorePair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2324 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2329 NewMI = mergeFlatLoadPair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2330 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2335 NewMI = mergeFlatStorePair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2336 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2339 CI.setMI(NewMI, *this); in optimizeInstsWithSameBaseAddr()
2340 CI.Order = Where->Order; in optimizeInstsWithSameBaseAddr()