Lines Matching refs:Paired
214 const CombineInfo &Paired);
216 CombineInfo &Paired, bool Modify = false);
218 const CombineInfo &Paired);
219 static unsigned getNewOpcode(const CombineInfo &CI, const CombineInfo &Paired);
221 const CombineInfo &Paired);
223 const CombineInfo &Paired);
226 CombineInfo *checkAndPrepareMerge(CombineInfo &CI, CombineInfo &Paired);
231 mergeRead2Pair(CombineInfo &CI, CombineInfo &Paired,
237 mergeWrite2Pair(CombineInfo &CI, CombineInfo &Paired,
240 mergeImagePair(CombineInfo &CI, CombineInfo &Paired,
243 mergeSMemLoadImmPair(CombineInfo &CI, CombineInfo &Paired,
246 mergeBufferLoadPair(CombineInfo &CI, CombineInfo &Paired,
249 mergeBufferStorePair(CombineInfo &CI, CombineInfo &Paired,
252 mergeTBufferLoadPair(CombineInfo &CI, CombineInfo &Paired,
255 mergeTBufferStorePair(CombineInfo &CI, CombineInfo &Paired,
258 mergeFlatLoadPair(CombineInfo &CI, CombineInfo &Paired,
261 mergeFlatStorePair(CombineInfo &CI, CombineInfo &Paired,
285 const CombineInfo &Paired);
288 const CombineInfo &Paired);
610 const CombineInfo &Paired) { in getCommonInstClass() argument
611 assert(CI.InstClass == Paired.InstClass); in getCommonInstClass()
614 SIInstrInfo::isFLATGlobal(*CI.I) && SIInstrInfo::isFLATGlobal(*Paired.I)) in getCommonInstClass()
861 const CombineInfo &Paired) { in combineKnownAdjacentMMOs() argument
863 const MachineMemOperand *MMOb = *Paired.I->memoperands_begin(); in combineKnownAdjacentMMOs()
869 if (Paired < CI) in combineKnownAdjacentMMOs()
883 const CombineInfo &Paired) { in dmasksCanBeCombined() argument
900 if (AMDGPU::getNamedOperandIdx(Paired.I->getOpcode(), op) != Idx) in dmasksCanBeCombined()
903 CI.I->getOperand(Idx).getImm() != Paired.I->getOperand(Idx).getImm()) in dmasksCanBeCombined()
908 unsigned MaxMask = std::max(CI.DMask, Paired.DMask); in dmasksCanBeCombined()
909 unsigned MinMask = std::min(CI.DMask, Paired.DMask); in dmasksCanBeCombined()
958 CombineInfo &Paired, in offsetsCanBeCombined() argument
964 if (CI.Offset == Paired.Offset) in offsetsCanBeCombined()
968 if ((CI.Offset % CI.EltSize != 0) || (Paired.Offset % CI.EltSize != 0)) in offsetsCanBeCombined()
978 llvm::AMDGPU::getGcnBufferFormatInfo(Paired.Format, STI); in offsetsCanBeCombined()
991 if (getBufferFormatWithCompCount(CI.Format, CI.Width + Paired.Width, STI) == 0) in offsetsCanBeCombined()
996 uint32_t EltOffset1 = Paired.Offset / CI.EltSize; in offsetsCanBeCombined()
1003 EltOffset1 + Paired.Width != EltOffset0) in offsetsCanBeCombined()
1005 if (CI.CPol != Paired.CPol) in offsetsCanBeCombined()
1014 if (CI.Width != Paired.Width && in offsetsCanBeCombined()
1015 (CI.Width < Paired.Width) == (CI.Offset < Paired.Offset)) in offsetsCanBeCombined()
1027 Paired.Offset = EltOffset1 / 64; in offsetsCanBeCombined()
1037 Paired.Offset = EltOffset1; in offsetsCanBeCombined()
1058 Paired.Offset = (EltOffset1 - BaseOff) / 64; in offsetsCanBeCombined()
1072 Paired.Offset = EltOffset1 - BaseOff; in offsetsCanBeCombined()
1082 const CombineInfo &Paired) { in widthsFit() argument
1083 const unsigned Width = (CI.Width + Paired.Width); in widthsFit()
1127 CombineInfo &Paired) { in checkAndPrepareMerge() argument
1130 if (CI.InstClass == UNKNOWN || Paired.InstClass == UNKNOWN) in checkAndPrepareMerge()
1132 assert(CI.InstClass == Paired.InstClass); in checkAndPrepareMerge()
1135 getInstSubclass(Paired.I->getOpcode(), *TII)) in checkAndPrepareMerge()
1141 if (!dmasksCanBeCombined(CI, *TII, Paired)) in checkAndPrepareMerge()
1144 if (!widthsFit(*STM, CI, Paired) || !offsetsCanBeCombined(CI, *STM, Paired)) in checkAndPrepareMerge()
1153 addDefsUsesToList(*Paired.I, RegDefs, RegUses); in checkAndPrepareMerge()
1154 for (MachineBasicBlock::iterator MBBI = Paired.I; --MBBI != CI.I;) { in checkAndPrepareMerge()
1155 if (!canSwapInstructions(RegDefs, RegUses, *Paired.I, *MBBI)) in checkAndPrepareMerge()
1162 for (MachineBasicBlock::iterator MBBI = CI.I; ++MBBI != Paired.I;) { in checkAndPrepareMerge()
1166 Where = &Paired; in checkAndPrepareMerge()
1174 offsetsCanBeCombined(CI, *STM, Paired, true); in checkAndPrepareMerge()
1193 SILoadStoreOptimizer::mergeRead2Pair(CombineInfo &CI, CombineInfo &Paired, in mergeRead2Pair() argument
1202 const auto *Dest1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdst); in mergeRead2Pair()
1205 unsigned NewOffset1 = Paired.Offset; in mergeRead2Pair()
1223 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeRead2Pair()
1252 .cloneMergedMemRefs({&*CI.I, &*Paired.I}); in mergeRead2Pair()
1267 Paired.I->eraseFromParent(); in mergeRead2Pair()
1290 CombineInfo &CI, CombineInfo &Paired, in mergeWrite2Pair() argument
1301 TII->getNamedOperand(*Paired.I, AMDGPU::OpName::data0); in mergeWrite2Pair()
1304 unsigned NewOffset1 = Paired.Offset; in mergeWrite2Pair()
1346 .cloneMergedMemRefs({&*CI.I, &*Paired.I}); in mergeWrite2Pair()
1349 Paired.I->eraseFromParent(); in mergeWrite2Pair()
1356 SILoadStoreOptimizer::mergeImagePair(CombineInfo &CI, CombineInfo &Paired, in mergeImagePair() argument
1360 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeImagePair()
1362 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeImagePair()
1365 unsigned MergedDMask = CI.DMask | Paired.DMask; in mergeImagePair()
1380 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeImagePair()
1382 MachineInstr *New = MIB.addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeImagePair()
1385 std::tie(SubRegIdx0, SubRegIdx1) = getSubRegIdxs(CI, Paired); in mergeImagePair()
1390 const auto *Dest1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdata); in mergeImagePair()
1400 Paired.I->eraseFromParent(); in mergeImagePair()
1405 CombineInfo &CI, CombineInfo &Paired, in mergeSMemLoadImmPair() argument
1409 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeSMemLoadImmPair()
1411 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeSMemLoadImmPair()
1414 unsigned MergedOffset = std::min(CI.Offset, Paired.Offset); in mergeSMemLoadImmPair()
1419 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeSMemLoadImmPair()
1427 New.addImm(CI.CPol).addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeSMemLoadImmPair()
1429 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); in mergeSMemLoadImmPair()
1436 const auto *Dest1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::sdst); in mergeSMemLoadImmPair()
1446 Paired.I->eraseFromParent(); in mergeSMemLoadImmPair()
1451 CombineInfo &CI, CombineInfo &Paired, in mergeBufferLoadPair() argument
1456 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeBufferLoadPair()
1458 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeBufferLoadPair()
1462 unsigned MergedOffset = std::min(CI.Offset, Paired.Offset); in mergeBufferLoadPair()
1474 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeBufferLoadPair()
1482 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeBufferLoadPair()
1484 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); in mergeBufferLoadPair()
1491 const auto *Dest1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdata); in mergeBufferLoadPair()
1501 Paired.I->eraseFromParent(); in mergeBufferLoadPair()
1506 CombineInfo &CI, CombineInfo &Paired, in mergeTBufferLoadPair() argument
1511 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeTBufferLoadPair()
1513 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeTBufferLoadPair()
1517 unsigned MergedOffset = std::min(CI.Offset, Paired.Offset); in mergeTBufferLoadPair()
1527 getBufferFormatWithCompCount(CI.Format, CI.Width + Paired.Width, *STM); in mergeTBufferLoadPair()
1532 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeTBufferLoadPair()
1541 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeTBufferLoadPair()
1543 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); in mergeTBufferLoadPair()
1550 const auto *Dest1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdata); in mergeTBufferLoadPair()
1560 Paired.I->eraseFromParent(); in mergeTBufferLoadPair()
1565 CombineInfo &CI, CombineInfo &Paired, in mergeTBufferStorePair() argument
1570 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeTBufferStorePair()
1572 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); in mergeTBufferStorePair()
1577 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeTBufferStorePair()
1581 const auto *Src1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdata); in mergeTBufferStorePair()
1598 getBufferFormatWithCompCount(CI.Format, CI.Width + Paired.Width, *STM); in mergeTBufferStorePair()
1603 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeTBufferStorePair()
1608 .addImm(std::min(CI.Offset, Paired.Offset)) // offset in mergeTBufferStorePair()
1612 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeTBufferStorePair()
1615 Paired.I->eraseFromParent(); in mergeTBufferStorePair()
1620 CombineInfo &CI, CombineInfo &Paired, in mergeFlatLoadPair() argument
1625 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeFlatLoadPair()
1627 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeFlatLoadPair()
1637 .addImm(std::min(CI.Offset, Paired.Offset)) in mergeFlatLoadPair()
1639 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeFlatLoadPair()
1641 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); in mergeFlatLoadPair()
1648 const auto *Dest1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdst); in mergeFlatLoadPair()
1658 Paired.I->eraseFromParent(); in mergeFlatLoadPair()
1663 CombineInfo &CI, CombineInfo &Paired, in mergeFlatStorePair() argument
1668 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeFlatStorePair()
1670 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); in mergeFlatStorePair()
1675 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeFlatStorePair()
1679 const auto *Src1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdata); in mergeFlatStorePair()
1695 MIB.addImm(std::min(CI.Offset, Paired.Offset)) in mergeFlatStorePair()
1697 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeFlatStorePair()
1700 Paired.I->eraseFromParent(); in mergeFlatStorePair()
1705 const CombineInfo &Paired) { in getNewOpcode() argument
1706 const unsigned Width = CI.Width + Paired.Width; in getNewOpcode()
1708 switch (getCommonInstClass(CI, Paired)) { in getNewOpcode()
1827 assert(((unsigned)llvm::popcount(CI.DMask | Paired.DMask) == Width) && in getNewOpcode()
1835 const CombineInfo &Paired) { in getSubRegIdxs() argument
1837 ((unsigned)llvm::popcount(CI.DMask | Paired.DMask) == in getSubRegIdxs()
1838 CI.Width + Paired.Width)) && in getSubRegIdxs()
1853 assert(Paired.Width >= 1 && Paired.Width <= 4); in getSubRegIdxs()
1855 if (Paired < CI) { in getSubRegIdxs()
1856 Idx1 = Idxs[0][Paired.Width - 1]; in getSubRegIdxs()
1857 Idx0 = Idxs[Paired.Width][CI.Width - 1]; in getSubRegIdxs()
1860 Idx1 = Idxs[CI.Width][Paired.Width - 1]; in getSubRegIdxs()
1868 const CombineInfo &Paired) { in getTargetRegisterClass() argument
1871 switch (CI.Width + Paired.Width) { in getTargetRegisterClass()
1887 unsigned BitWidth = 32 * (CI.Width + Paired.Width); in getTargetRegisterClass()
1894 CombineInfo &CI, CombineInfo &Paired, in mergeBufferStorePair() argument
1899 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeBufferStorePair()
1901 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); in mergeBufferStorePair()
1906 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeBufferStorePair()
1910 const auto *Src1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdata); in mergeBufferStorePair()
1930 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeBufferStorePair()
1935 .addImm(std::min(CI.Offset, Paired.Offset)) // offset in mergeBufferStorePair()
1938 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeBufferStorePair()
1941 Paired.I->eraseFromParent(); in mergeBufferStorePair()
2416 CombineInfo &Paired = *Second; in optimizeInstsWithSameBaseAddr() local
2418 CombineInfo *Where = checkAndPrepareMerge(CI, Paired); in optimizeInstsWithSameBaseAddr()
2426 LLVM_DEBUG(dbgs() << "Merging: " << *CI.I << " with: " << *Paired.I); in optimizeInstsWithSameBaseAddr()
2434 NewMI = mergeRead2Pair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2437 NewMI = mergeWrite2Pair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2442 NewMI = mergeSMemLoadImmPair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2443 OptimizeListAgain |= CI.Width + Paired.Width < 8; in optimizeInstsWithSameBaseAddr()
2446 NewMI = mergeBufferLoadPair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2447 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2450 NewMI = mergeBufferStorePair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2451 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2454 NewMI = mergeImagePair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2455 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2458 NewMI = mergeTBufferLoadPair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2459 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2462 NewMI = mergeTBufferStorePair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2463 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2468 NewMI = mergeFlatLoadPair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2469 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2474 NewMI = mergeFlatStorePair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2475 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()