Lines Matching refs:CI
125 bool hasSameBaseAddress(const CombineInfo &CI) { in hasSameBaseAddress()
126 if (NumAddresses != CI.NumAddresses) in hasSameBaseAddress()
129 const MachineInstr &MI = *CI.I; in hasSameBaseAddress()
212 static bool dmasksCanBeCombined(const CombineInfo &CI,
215 static bool offsetsCanBeCombined(CombineInfo &CI, const GCNSubtarget &STI,
217 static bool widthsFit(const GCNSubtarget &STI, const CombineInfo &CI,
219 static unsigned getNewOpcode(const CombineInfo &CI, const CombineInfo &Paired);
220 static std::pair<unsigned, unsigned> getSubRegIdxs(const CombineInfo &CI,
222 const TargetRegisterClass *getTargetRegisterClass(const CombineInfo &CI,
226 CombineInfo *checkAndPrepareMerge(CombineInfo &CI, CombineInfo &Paired);
231 mergeRead2Pair(CombineInfo &CI, CombineInfo &Paired,
237 mergeWrite2Pair(CombineInfo &CI, CombineInfo &Paired,
240 mergeImagePair(CombineInfo &CI, CombineInfo &Paired,
243 mergeSMemLoadImmPair(CombineInfo &CI, CombineInfo &Paired,
246 mergeBufferLoadPair(CombineInfo &CI, CombineInfo &Paired,
249 mergeBufferStorePair(CombineInfo &CI, CombineInfo &Paired,
252 mergeTBufferLoadPair(CombineInfo &CI, CombineInfo &Paired,
255 mergeTBufferStorePair(CombineInfo &CI, CombineInfo &Paired,
258 mergeFlatLoadPair(CombineInfo &CI, CombineInfo &Paired,
261 mergeFlatStorePair(CombineInfo &CI, CombineInfo &Paired,
273 bool promoteConstantOffsetToImm(MachineInstr &CI,
276 void addInstToMergeableList(const CombineInfo &CI,
284 static MachineMemOperand *combineKnownAdjacentMMOs(const CombineInfo &CI,
287 static InstClassEnum getCommonInstClass(const CombineInfo &CI,
609 SILoadStoreOptimizer::getCommonInstClass(const CombineInfo &CI, in getCommonInstClass() argument
611 assert(CI.InstClass == Paired.InstClass); in getCommonInstClass()
613 if ((CI.InstClass == FLAT_LOAD || CI.InstClass == FLAT_STORE) && in getCommonInstClass()
614 SIInstrInfo::isFLATGlobal(*CI.I) && SIInstrInfo::isFLATGlobal(*Paired.I)) in getCommonInstClass()
615 return (CI.InstClass == FLAT_STORE) ? GLOBAL_STORE : GLOBAL_LOAD; in getCommonInstClass()
617 return CI.InstClass; in getCommonInstClass()
860 SILoadStoreOptimizer::combineKnownAdjacentMMOs(const CombineInfo &CI, in combineKnownAdjacentMMOs() argument
862 const MachineMemOperand *MMOa = *CI.I->memoperands_begin(); in combineKnownAdjacentMMOs()
869 if (Paired < CI) in combineKnownAdjacentMMOs()
877 MachineFunction *MF = CI.I->getMF(); in combineKnownAdjacentMMOs()
881 bool SILoadStoreOptimizer::dmasksCanBeCombined(const CombineInfo &CI, in dmasksCanBeCombined() argument
884 assert(CI.InstClass == MIMG); in dmasksCanBeCombined()
887 const auto *TFEOp = TII.getNamedOperand(*CI.I, AMDGPU::OpName::tfe); in dmasksCanBeCombined()
888 const auto *LWEOp = TII.getNamedOperand(*CI.I, AMDGPU::OpName::lwe); in dmasksCanBeCombined()
899 int Idx = AMDGPU::getNamedOperandIdx(CI.I->getOpcode(), op); in dmasksCanBeCombined()
903 CI.I->getOperand(Idx).getImm() != Paired.I->getOperand(Idx).getImm()) in dmasksCanBeCombined()
908 unsigned MaxMask = std::max(CI.DMask, Paired.DMask); in dmasksCanBeCombined()
909 unsigned MinMask = std::min(CI.DMask, Paired.DMask); in dmasksCanBeCombined()
956 bool SILoadStoreOptimizer::offsetsCanBeCombined(CombineInfo &CI, in offsetsCanBeCombined() argument
960 assert(CI.InstClass != MIMG); in offsetsCanBeCombined()
964 if (CI.Offset == Paired.Offset) in offsetsCanBeCombined()
968 if ((CI.Offset % CI.EltSize != 0) || (Paired.Offset % CI.EltSize != 0)) in offsetsCanBeCombined()
971 if (CI.InstClass == TBUFFER_LOAD || CI.InstClass == TBUFFER_STORE) { in offsetsCanBeCombined()
974 llvm::AMDGPU::getGcnBufferFormatInfo(CI.Format, STI); in offsetsCanBeCombined()
991 if (getBufferFormatWithCompCount(CI.Format, CI.Width + Paired.Width, STI) == 0) in offsetsCanBeCombined()
995 uint32_t EltOffset0 = CI.Offset / CI.EltSize; in offsetsCanBeCombined()
996 uint32_t EltOffset1 = Paired.Offset / CI.EltSize; in offsetsCanBeCombined()
997 CI.UseST64 = false; in offsetsCanBeCombined()
998 CI.BaseOff = 0; in offsetsCanBeCombined()
1001 if ((CI.InstClass != DS_READ) && (CI.InstClass != DS_WRITE)) { in offsetsCanBeCombined()
1002 if (EltOffset0 + CI.Width != EltOffset1 && in offsetsCanBeCombined()
1005 if (CI.CPol != Paired.CPol) in offsetsCanBeCombined()
1007 if (CI.InstClass == S_LOAD_IMM || CI.InstClass == S_BUFFER_LOAD_IMM || in offsetsCanBeCombined()
1008 CI.InstClass == S_BUFFER_LOAD_SGPR_IMM) { in offsetsCanBeCombined()
1014 if (CI.Width != Paired.Width && in offsetsCanBeCombined()
1015 (CI.Width < Paired.Width) == (CI.Offset < Paired.Offset)) in offsetsCanBeCombined()
1026 CI.Offset = EltOffset0 / 64; in offsetsCanBeCombined()
1028 CI.UseST64 = true; in offsetsCanBeCombined()
1036 CI.Offset = EltOffset0; in offsetsCanBeCombined()
1056 CI.BaseOff = BaseOff * CI.EltSize; in offsetsCanBeCombined()
1057 CI.Offset = (EltOffset0 - BaseOff) / 64; in offsetsCanBeCombined()
1059 CI.UseST64 = true; in offsetsCanBeCombined()
1070 CI.BaseOff = BaseOff * CI.EltSize; in offsetsCanBeCombined()
1071 CI.Offset = EltOffset0 - BaseOff; in offsetsCanBeCombined()
1081 const CombineInfo &CI, in widthsFit() argument
1083 const unsigned Width = (CI.Width + Paired.Width); in widthsFit()
1084 switch (CI.InstClass) { in widthsFit()
1126 SILoadStoreOptimizer::checkAndPrepareMerge(CombineInfo &CI, in checkAndPrepareMerge() argument
1130 if (CI.InstClass == UNKNOWN || Paired.InstClass == UNKNOWN) in checkAndPrepareMerge()
1132 assert(CI.InstClass == Paired.InstClass); in checkAndPrepareMerge()
1134 if (getInstSubclass(CI.I->getOpcode(), *TII) != in checkAndPrepareMerge()
1140 if (CI.InstClass == MIMG) { in checkAndPrepareMerge()
1141 if (!dmasksCanBeCombined(CI, *TII, Paired)) in checkAndPrepareMerge()
1144 if (!widthsFit(*STM, CI, Paired) || !offsetsCanBeCombined(CI, *STM, Paired)) in checkAndPrepareMerge()
1151 if (CI.I->mayLoad()) { in checkAndPrepareMerge()
1154 for (MachineBasicBlock::iterator MBBI = Paired.I; --MBBI != CI.I;) { in checkAndPrepareMerge()
1158 Where = &CI; in checkAndPrepareMerge()
1161 addDefsUsesToList(*CI.I, RegDefs, RegUses); in checkAndPrepareMerge()
1162 for (MachineBasicBlock::iterator MBBI = CI.I; ++MBBI != Paired.I;) { in checkAndPrepareMerge()
1163 if (!canSwapInstructions(RegDefs, RegUses, *CI.I, *MBBI)) in checkAndPrepareMerge()
1173 if (CI.InstClass == DS_READ || CI.InstClass == DS_WRITE) in checkAndPrepareMerge()
1174 offsetsCanBeCombined(CI, *STM, Paired, true); in checkAndPrepareMerge()
1193 SILoadStoreOptimizer::mergeRead2Pair(CombineInfo &CI, CombineInfo &Paired, in mergeRead2Pair() argument
1195 MachineBasicBlock *MBB = CI.I->getParent(); in mergeRead2Pair()
1199 const auto *AddrReg = TII->getNamedOperand(*CI.I, AMDGPU::OpName::addr); in mergeRead2Pair()
1201 const auto *Dest0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::vdst); in mergeRead2Pair()
1204 unsigned NewOffset0 = CI.Offset; in mergeRead2Pair()
1207 CI.UseST64 ? read2ST64Opcode(CI.EltSize) : read2Opcode(CI.EltSize); in mergeRead2Pair()
1209 unsigned SubRegIdx0 = (CI.EltSize == 4) ? AMDGPU::sub0 : AMDGPU::sub0_sub1; in mergeRead2Pair()
1210 unsigned SubRegIdx1 = (CI.EltSize == 4) ? AMDGPU::sub1 : AMDGPU::sub2_sub3; in mergeRead2Pair()
1223 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeRead2Pair()
1226 DebugLoc DL = CI.I->getDebugLoc(); in mergeRead2Pair()
1231 if (CI.BaseOff) { in mergeRead2Pair()
1234 .addImm(CI.BaseOff); in mergeRead2Pair()
1252 .cloneMergedMemRefs({&*CI.I, &*Paired.I}); in mergeRead2Pair()
1266 CI.I->eraseFromParent(); in mergeRead2Pair()
1290 CombineInfo &CI, CombineInfo &Paired, in mergeWrite2Pair() argument
1292 MachineBasicBlock *MBB = CI.I->getParent(); in mergeWrite2Pair()
1297 TII->getNamedOperand(*CI.I, AMDGPU::OpName::addr); in mergeWrite2Pair()
1299 TII->getNamedOperand(*CI.I, AMDGPU::OpName::data0); in mergeWrite2Pair()
1303 unsigned NewOffset0 = CI.Offset; in mergeWrite2Pair()
1306 CI.UseST64 ? write2ST64Opcode(CI.EltSize) : write2Opcode(CI.EltSize); in mergeWrite2Pair()
1318 DebugLoc DL = CI.I->getDebugLoc(); in mergeWrite2Pair()
1323 if (CI.BaseOff) { in mergeWrite2Pair()
1326 .addImm(CI.BaseOff); in mergeWrite2Pair()
1346 .cloneMergedMemRefs({&*CI.I, &*Paired.I}); in mergeWrite2Pair()
1348 CI.I->eraseFromParent(); in mergeWrite2Pair()
1356 SILoadStoreOptimizer::mergeImagePair(CombineInfo &CI, CombineInfo &Paired, in mergeImagePair() argument
1358 MachineBasicBlock *MBB = CI.I->getParent(); in mergeImagePair()
1359 DebugLoc DL = CI.I->getDebugLoc(); in mergeImagePair()
1360 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeImagePair()
1362 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeImagePair()
1365 unsigned MergedDMask = CI.DMask | Paired.DMask; in mergeImagePair()
1367 AMDGPU::getNamedOperandIdx(CI.I->getOpcode(), AMDGPU::OpName::dmask); in mergeImagePair()
1370 for (unsigned I = 1, E = (*CI.I).getNumOperands(); I != E; ++I) { in mergeImagePair()
1374 MIB.add((*CI.I).getOperand(I)); in mergeImagePair()
1380 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeImagePair()
1382 MachineInstr *New = MIB.addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeImagePair()
1385 std::tie(SubRegIdx0, SubRegIdx1) = getSubRegIdxs(CI, Paired); in mergeImagePair()
1389 const auto *Dest0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::vdata); in mergeImagePair()
1399 CI.I->eraseFromParent(); in mergeImagePair()
1405 CombineInfo &CI, CombineInfo &Paired, in mergeSMemLoadImmPair() argument
1407 MachineBasicBlock *MBB = CI.I->getParent(); in mergeSMemLoadImmPair()
1408 DebugLoc DL = CI.I->getDebugLoc(); in mergeSMemLoadImmPair()
1409 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeSMemLoadImmPair()
1411 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeSMemLoadImmPair()
1414 unsigned MergedOffset = std::min(CI.Offset, Paired.Offset); in mergeSMemLoadImmPair()
1419 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeSMemLoadImmPair()
1423 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::sbase)); in mergeSMemLoadImmPair()
1424 if (CI.InstClass == S_BUFFER_LOAD_SGPR_IMM) in mergeSMemLoadImmPair()
1425 New.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::soffset)); in mergeSMemLoadImmPair()
1427 New.addImm(CI.CPol).addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeSMemLoadImmPair()
1429 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); in mergeSMemLoadImmPair()
1435 const auto *Dest0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::sdst); in mergeSMemLoadImmPair()
1445 CI.I->eraseFromParent(); in mergeSMemLoadImmPair()
1451 CombineInfo &CI, CombineInfo &Paired, in mergeBufferLoadPair() argument
1453 MachineBasicBlock *MBB = CI.I->getParent(); in mergeBufferLoadPair()
1454 DebugLoc DL = CI.I->getDebugLoc(); in mergeBufferLoadPair()
1456 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeBufferLoadPair()
1458 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeBufferLoadPair()
1462 unsigned MergedOffset = std::min(CI.Offset, Paired.Offset); in mergeBufferLoadPair()
1469 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr)); in mergeBufferLoadPair()
1474 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeBufferLoadPair()
1477 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::srsrc)) in mergeBufferLoadPair()
1478 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::soffset)) in mergeBufferLoadPair()
1480 .addImm(CI.CPol) // cpol in mergeBufferLoadPair()
1482 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeBufferLoadPair()
1484 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); in mergeBufferLoadPair()
1490 const auto *Dest0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::vdata); in mergeBufferLoadPair()
1500 CI.I->eraseFromParent(); in mergeBufferLoadPair()
1506 CombineInfo &CI, CombineInfo &Paired, in mergeTBufferLoadPair() argument
1508 MachineBasicBlock *MBB = CI.I->getParent(); in mergeTBufferLoadPair()
1509 DebugLoc DL = CI.I->getDebugLoc(); in mergeTBufferLoadPair()
1511 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeTBufferLoadPair()
1513 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeTBufferLoadPair()
1517 unsigned MergedOffset = std::min(CI.Offset, Paired.Offset); in mergeTBufferLoadPair()
1524 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr)); in mergeTBufferLoadPair()
1527 getBufferFormatWithCompCount(CI.Format, CI.Width + Paired.Width, *STM); in mergeTBufferLoadPair()
1532 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeTBufferLoadPair()
1535 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::srsrc)) in mergeTBufferLoadPair()
1536 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::soffset)) in mergeTBufferLoadPair()
1539 .addImm(CI.CPol) // cpol in mergeTBufferLoadPair()
1541 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeTBufferLoadPair()
1543 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); in mergeTBufferLoadPair()
1549 const auto *Dest0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::vdata); in mergeTBufferLoadPair()
1559 CI.I->eraseFromParent(); in mergeTBufferLoadPair()
1565 CombineInfo &CI, CombineInfo &Paired, in mergeTBufferStorePair() argument
1567 MachineBasicBlock *MBB = CI.I->getParent(); in mergeTBufferStorePair()
1568 DebugLoc DL = CI.I->getDebugLoc(); in mergeTBufferStorePair()
1570 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeTBufferStorePair()
1572 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); in mergeTBufferStorePair()
1577 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeTBufferStorePair()
1580 const auto *Src0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::vdata); in mergeTBufferStorePair()
1595 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr)); in mergeTBufferStorePair()
1598 getBufferFormatWithCompCount(CI.Format, CI.Width + Paired.Width, *STM); in mergeTBufferStorePair()
1603 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeTBufferStorePair()
1606 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::srsrc)) in mergeTBufferStorePair()
1607 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::soffset)) in mergeTBufferStorePair()
1608 .addImm(std::min(CI.Offset, Paired.Offset)) // offset in mergeTBufferStorePair()
1610 .addImm(CI.CPol) // cpol in mergeTBufferStorePair()
1612 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeTBufferStorePair()
1614 CI.I->eraseFromParent(); in mergeTBufferStorePair()
1620 CombineInfo &CI, CombineInfo &Paired, in mergeFlatLoadPair() argument
1622 MachineBasicBlock *MBB = CI.I->getParent(); in mergeFlatLoadPair()
1623 DebugLoc DL = CI.I->getDebugLoc(); in mergeFlatLoadPair()
1625 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeFlatLoadPair()
1627 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeFlatLoadPair()
1632 if (auto *SAddr = TII->getNamedOperand(*CI.I, AMDGPU::OpName::saddr)) in mergeFlatLoadPair()
1636 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr)) in mergeFlatLoadPair()
1637 .addImm(std::min(CI.Offset, Paired.Offset)) in mergeFlatLoadPair()
1638 .addImm(CI.CPol) in mergeFlatLoadPair()
1639 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeFlatLoadPair()
1641 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); in mergeFlatLoadPair()
1647 const auto *Dest0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::vdst); in mergeFlatLoadPair()
1657 CI.I->eraseFromParent(); in mergeFlatLoadPair()
1663 CombineInfo &CI, CombineInfo &Paired, in mergeFlatStorePair() argument
1665 MachineBasicBlock *MBB = CI.I->getParent(); in mergeFlatStorePair()
1666 DebugLoc DL = CI.I->getDebugLoc(); in mergeFlatStorePair()
1668 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeFlatStorePair()
1670 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); in mergeFlatStorePair()
1675 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeFlatStorePair()
1678 const auto *Src0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::vdata); in mergeFlatStorePair()
1688 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr)) in mergeFlatStorePair()
1691 if (auto *SAddr = TII->getNamedOperand(*CI.I, AMDGPU::OpName::saddr)) in mergeFlatStorePair()
1695 MIB.addImm(std::min(CI.Offset, Paired.Offset)) in mergeFlatStorePair()
1696 .addImm(CI.CPol) in mergeFlatStorePair()
1697 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeFlatStorePair()
1699 CI.I->eraseFromParent(); in mergeFlatStorePair()
1704 unsigned SILoadStoreOptimizer::getNewOpcode(const CombineInfo &CI, in getNewOpcode() argument
1706 const unsigned Width = CI.Width + Paired.Width; in getNewOpcode()
1708 switch (getCommonInstClass(CI, Paired)) { in getNewOpcode()
1710 assert(CI.InstClass == BUFFER_LOAD || CI.InstClass == BUFFER_STORE); in getNewOpcode()
1712 return AMDGPU::getMUBUFOpcode(AMDGPU::getMUBUFBaseOpcode(CI.I->getOpcode()), in getNewOpcode()
1716 return AMDGPU::getMTBUFOpcode(AMDGPU::getMTBUFBaseOpcode(CI.I->getOpcode()), in getNewOpcode()
1827 assert(((unsigned)llvm::popcount(CI.DMask | Paired.DMask) == Width) && in getNewOpcode()
1829 return AMDGPU::getMaskedMIMGOp(CI.I->getOpcode(), Width); in getNewOpcode()
1834 SILoadStoreOptimizer::getSubRegIdxs(const CombineInfo &CI, in getSubRegIdxs() argument
1836 assert((CI.InstClass != MIMG || in getSubRegIdxs()
1837 ((unsigned)llvm::popcount(CI.DMask | Paired.DMask) == in getSubRegIdxs()
1838 CI.Width + Paired.Width)) && in getSubRegIdxs()
1852 assert(CI.Width >= 1 && CI.Width <= 4); in getSubRegIdxs()
1855 if (Paired < CI) { in getSubRegIdxs()
1857 Idx0 = Idxs[Paired.Width][CI.Width - 1]; in getSubRegIdxs()
1859 Idx0 = Idxs[0][CI.Width - 1]; in getSubRegIdxs()
1860 Idx1 = Idxs[CI.Width][Paired.Width - 1]; in getSubRegIdxs()
1867 SILoadStoreOptimizer::getTargetRegisterClass(const CombineInfo &CI, in getTargetRegisterClass() argument
1869 if (CI.InstClass == S_BUFFER_LOAD_IMM || in getTargetRegisterClass()
1870 CI.InstClass == S_BUFFER_LOAD_SGPR_IMM || CI.InstClass == S_LOAD_IMM) { in getTargetRegisterClass()
1871 switch (CI.Width + Paired.Width) { in getTargetRegisterClass()
1887 unsigned BitWidth = 32 * (CI.Width + Paired.Width); in getTargetRegisterClass()
1888 return TRI->isAGPRClass(getDataRegClass(*CI.I)) in getTargetRegisterClass()
1894 CombineInfo &CI, CombineInfo &Paired, in mergeBufferStorePair() argument
1896 MachineBasicBlock *MBB = CI.I->getParent(); in mergeBufferStorePair()
1897 DebugLoc DL = CI.I->getDebugLoc(); in mergeBufferStorePair()
1899 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeBufferStorePair()
1901 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); in mergeBufferStorePair()
1906 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeBufferStorePair()
1909 const auto *Src0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::vdata); in mergeBufferStorePair()
1924 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr)); in mergeBufferStorePair()
1930 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeBufferStorePair()
1933 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::srsrc)) in mergeBufferStorePair()
1934 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::soffset)) in mergeBufferStorePair()
1935 .addImm(std::min(CI.Offset, Paired.Offset)) // offset in mergeBufferStorePair()
1936 .addImm(CI.CPol) // cpol in mergeBufferStorePair()
1938 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeBufferStorePair()
1940 CI.I->eraseFromParent(); in mergeBufferStorePair()
2258 void SILoadStoreOptimizer::addInstToMergeableList(const CombineInfo &CI, in addInstToMergeableList() argument
2261 if (AddrList.front().InstClass == CI.InstClass && in addInstToMergeableList()
2262 AddrList.front().IsAGPR == CI.IsAGPR && in addInstToMergeableList()
2263 AddrList.front().hasSameBaseAddress(CI)) { in addInstToMergeableList()
2264 AddrList.emplace_back(CI); in addInstToMergeableList()
2270 MergeableInsts.emplace_back(1, CI); in addInstToMergeableList()
2311 CombineInfo CI; in collectMergeableInsts() local
2312 CI.setMI(MI, *this); in collectMergeableInsts()
2313 CI.Order = Order++; in collectMergeableInsts()
2315 if (!CI.hasMergeableAddress(*MRI)) in collectMergeableInsts()
2318 if (CI.InstClass == DS_WRITE && CI.IsAGPR) { in collectMergeableInsts()
2330 addInstToMergeableList(CI, MergeableInsts); in collectMergeableInsts()
2415 CombineInfo &CI = *First; in optimizeInstsWithSameBaseAddr() local
2418 CombineInfo *Where = checkAndPrepareMerge(CI, Paired); in optimizeInstsWithSameBaseAddr()
2426 LLVM_DEBUG(dbgs() << "Merging: " << *CI.I << " with: " << *Paired.I); in optimizeInstsWithSameBaseAddr()
2429 switch (CI.InstClass) { in optimizeInstsWithSameBaseAddr()
2434 NewMI = mergeRead2Pair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2437 NewMI = mergeWrite2Pair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2442 NewMI = mergeSMemLoadImmPair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2443 OptimizeListAgain |= CI.Width + Paired.Width < 8; in optimizeInstsWithSameBaseAddr()
2446 NewMI = mergeBufferLoadPair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2447 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2450 NewMI = mergeBufferStorePair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2451 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2454 NewMI = mergeImagePair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2455 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2458 NewMI = mergeTBufferLoadPair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2459 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2462 NewMI = mergeTBufferStorePair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2463 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2468 NewMI = mergeFlatLoadPair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2469 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2474 NewMI = mergeFlatStorePair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2475 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2478 CI.setMI(NewMI, *this); in optimizeInstsWithSameBaseAddr()
2479 CI.Order = Where->Order; in optimizeInstsWithSameBaseAddr()