Lines Matching refs:Paired
206 const CombineInfo &Paired);
208 CombineInfo &Paired, bool Modify = false);
210 const CombineInfo &Paired);
211 static unsigned getNewOpcode(const CombineInfo &CI, const CombineInfo &Paired);
213 const CombineInfo &Paired);
215 const CombineInfo &Paired);
218 CombineInfo *checkAndPrepareMerge(CombineInfo &CI, CombineInfo &Paired);
223 mergeRead2Pair(CombineInfo &CI, CombineInfo &Paired,
229 mergeWrite2Pair(CombineInfo &CI, CombineInfo &Paired,
232 mergeImagePair(CombineInfo &CI, CombineInfo &Paired,
235 mergeSBufferLoadImmPair(CombineInfo &CI, CombineInfo &Paired,
238 mergeBufferLoadPair(CombineInfo &CI, CombineInfo &Paired,
241 mergeBufferStorePair(CombineInfo &CI, CombineInfo &Paired,
244 mergeTBufferLoadPair(CombineInfo &CI, CombineInfo &Paired,
247 mergeTBufferStorePair(CombineInfo &CI, CombineInfo &Paired,
250 mergeFlatLoadPair(CombineInfo &CI, CombineInfo &Paired,
253 mergeFlatStorePair(CombineInfo &CI, CombineInfo &Paired,
277 const CombineInfo &Paired);
280 const CombineInfo &Paired);
541 const CombineInfo &Paired) { in getCommonInstClass() argument
542 assert(CI.InstClass == Paired.InstClass); in getCommonInstClass()
545 SIInstrInfo::isFLATGlobal(*CI.I) && SIInstrInfo::isFLATGlobal(*Paired.I)) in getCommonInstClass()
774 const CombineInfo &Paired) { in combineKnownAdjacentMMOs() argument
776 const MachineMemOperand *MMOb = *Paired.I->memoperands_begin(); in combineKnownAdjacentMMOs()
782 if (Paired < CI) in combineKnownAdjacentMMOs()
796 const CombineInfo &Paired) { in dmasksCanBeCombined() argument
813 if (AMDGPU::getNamedOperandIdx(Paired.I->getOpcode(), op) != Idx) in dmasksCanBeCombined()
816 CI.I->getOperand(Idx).getImm() != Paired.I->getOperand(Idx).getImm()) in dmasksCanBeCombined()
821 unsigned MaxMask = std::max(CI.DMask, Paired.DMask); in dmasksCanBeCombined()
822 unsigned MinMask = std::min(CI.DMask, Paired.DMask); in dmasksCanBeCombined()
868 CombineInfo &Paired, in offsetsCanBeCombined() argument
874 if (CI.Offset == Paired.Offset) in offsetsCanBeCombined()
878 if ((CI.Offset % CI.EltSize != 0) || (Paired.Offset % CI.EltSize != 0)) in offsetsCanBeCombined()
888 llvm::AMDGPU::getGcnBufferFormatInfo(Paired.Format, STI); in offsetsCanBeCombined()
901 if (getBufferFormatWithCompCount(CI.Format, CI.Width + Paired.Width, STI) == 0) in offsetsCanBeCombined()
906 uint32_t EltOffset1 = Paired.Offset / CI.EltSize; in offsetsCanBeCombined()
913 EltOffset1 + Paired.Width == EltOffset0) && in offsetsCanBeCombined()
914 CI.CPol == Paired.CPol; in offsetsCanBeCombined()
923 Paired.Offset = EltOffset1 / 64; in offsetsCanBeCombined()
933 Paired.Offset = EltOffset1; in offsetsCanBeCombined()
954 Paired.Offset = (EltOffset1 - BaseOff) / 64; in offsetsCanBeCombined()
968 Paired.Offset = EltOffset1 - BaseOff; in offsetsCanBeCombined()
978 const CombineInfo &Paired) { in widthsFit() argument
979 const unsigned Width = (CI.Width + Paired.Width); in widthsFit()
1019 CombineInfo &Paired) { in checkAndPrepareMerge() argument
1022 if (CI.InstClass == UNKNOWN || Paired.InstClass == UNKNOWN) in checkAndPrepareMerge()
1024 assert(CI.InstClass == Paired.InstClass); in checkAndPrepareMerge()
1027 getInstSubclass(Paired.I->getOpcode(), *TII)) in checkAndPrepareMerge()
1033 if (!dmasksCanBeCombined(CI, *TII, Paired)) in checkAndPrepareMerge()
1036 if (!widthsFit(*STM, CI, Paired) || !offsetsCanBeCombined(CI, *STM, Paired)) in checkAndPrepareMerge()
1045 addDefsUsesToList(*Paired.I, RegDefs, RegUses); in checkAndPrepareMerge()
1046 for (MachineBasicBlock::iterator MBBI = Paired.I; --MBBI != CI.I;) { in checkAndPrepareMerge()
1047 if (!canSwapInstructions(RegDefs, RegUses, *Paired.I, *MBBI)) in checkAndPrepareMerge()
1054 for (MachineBasicBlock::iterator MBBI = CI.I; ++MBBI != Paired.I;) { in checkAndPrepareMerge()
1058 Where = &Paired; in checkAndPrepareMerge()
1066 offsetsCanBeCombined(CI, *STM, Paired, true); in checkAndPrepareMerge()
1085 SILoadStoreOptimizer::mergeRead2Pair(CombineInfo &CI, CombineInfo &Paired, in mergeRead2Pair() argument
1094 const auto *Dest1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdst); in mergeRead2Pair()
1097 unsigned NewOffset1 = Paired.Offset; in mergeRead2Pair()
1115 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeRead2Pair()
1144 .cloneMergedMemRefs({&*CI.I, &*Paired.I}); in mergeRead2Pair()
1159 Paired.I->eraseFromParent(); in mergeRead2Pair()
1182 CombineInfo &CI, CombineInfo &Paired, in mergeWrite2Pair() argument
1193 TII->getNamedOperand(*Paired.I, AMDGPU::OpName::data0); in mergeWrite2Pair()
1196 unsigned NewOffset1 = Paired.Offset; in mergeWrite2Pair()
1238 .cloneMergedMemRefs({&*CI.I, &*Paired.I}); in mergeWrite2Pair()
1241 Paired.I->eraseFromParent(); in mergeWrite2Pair()
1248 SILoadStoreOptimizer::mergeImagePair(CombineInfo &CI, CombineInfo &Paired, in mergeImagePair() argument
1252 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeImagePair()
1254 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeImagePair()
1257 unsigned MergedDMask = CI.DMask | Paired.DMask; in mergeImagePair()
1272 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeImagePair()
1274 MachineInstr *New = MIB.addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeImagePair()
1277 std::tie(SubRegIdx0, SubRegIdx1) = getSubRegIdxs(CI, Paired); in mergeImagePair()
1282 const auto *Dest1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdata); in mergeImagePair()
1292 Paired.I->eraseFromParent(); in mergeImagePair()
1297 CombineInfo &CI, CombineInfo &Paired, in mergeSBufferLoadImmPair() argument
1301 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeSBufferLoadImmPair()
1303 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeSBufferLoadImmPair()
1306 unsigned MergedOffset = std::min(CI.Offset, Paired.Offset); in mergeSBufferLoadImmPair()
1311 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeSBufferLoadImmPair()
1318 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeSBufferLoadImmPair()
1320 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); in mergeSBufferLoadImmPair()
1327 const auto *Dest1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::sdst); in mergeSBufferLoadImmPair()
1337 Paired.I->eraseFromParent(); in mergeSBufferLoadImmPair()
1342 CombineInfo &CI, CombineInfo &Paired, in mergeBufferLoadPair() argument
1347 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeBufferLoadPair()
1349 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeBufferLoadPair()
1353 unsigned MergedOffset = std::min(CI.Offset, Paired.Offset); in mergeBufferLoadPair()
1365 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeBufferLoadPair()
1374 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeBufferLoadPair()
1376 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); in mergeBufferLoadPair()
1383 const auto *Dest1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdata); in mergeBufferLoadPair()
1393 Paired.I->eraseFromParent(); in mergeBufferLoadPair()
1398 CombineInfo &CI, CombineInfo &Paired, in mergeTBufferLoadPair() argument
1403 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeTBufferLoadPair()
1405 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeTBufferLoadPair()
1409 unsigned MergedOffset = std::min(CI.Offset, Paired.Offset); in mergeTBufferLoadPair()
1419 getBufferFormatWithCompCount(CI.Format, CI.Width + Paired.Width, *STM); in mergeTBufferLoadPair()
1424 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeTBufferLoadPair()
1434 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeTBufferLoadPair()
1436 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); in mergeTBufferLoadPair()
1443 const auto *Dest1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdata); in mergeTBufferLoadPair()
1453 Paired.I->eraseFromParent(); in mergeTBufferLoadPair()
1458 CombineInfo &CI, CombineInfo &Paired, in mergeTBufferStorePair() argument
1463 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeTBufferStorePair()
1465 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); in mergeTBufferStorePair()
1470 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeTBufferStorePair()
1474 const auto *Src1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdata); in mergeTBufferStorePair()
1491 getBufferFormatWithCompCount(CI.Format, CI.Width + Paired.Width, *STM); in mergeTBufferStorePair()
1496 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeTBufferStorePair()
1501 .addImm(std::min(CI.Offset, Paired.Offset)) // offset in mergeTBufferStorePair()
1506 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeTBufferStorePair()
1509 Paired.I->eraseFromParent(); in mergeTBufferStorePair()
1514 CombineInfo &CI, CombineInfo &Paired, in mergeFlatLoadPair() argument
1519 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeFlatLoadPair()
1521 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeFlatLoadPair()
1531 .addImm(std::min(CI.Offset, Paired.Offset)) in mergeFlatLoadPair()
1533 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeFlatLoadPair()
1535 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); in mergeFlatLoadPair()
1542 const auto *Dest1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdst); in mergeFlatLoadPair()
1552 Paired.I->eraseFromParent(); in mergeFlatLoadPair()
1557 CombineInfo &CI, CombineInfo &Paired, in mergeFlatStorePair() argument
1562 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeFlatStorePair()
1564 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); in mergeFlatStorePair()
1569 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeFlatStorePair()
1573 const auto *Src1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdata); in mergeFlatStorePair()
1589 MIB.addImm(std::min(CI.Offset, Paired.Offset)) in mergeFlatStorePair()
1591 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeFlatStorePair()
1594 Paired.I->eraseFromParent(); in mergeFlatStorePair()
1599 const CombineInfo &Paired) { in getNewOpcode() argument
1600 const unsigned Width = CI.Width + Paired.Width; in getNewOpcode()
1602 switch (getCommonInstClass(CI, Paired)) { in getNewOpcode()
1693 assert((countPopulation(CI.DMask | Paired.DMask) == Width) && in getNewOpcode()
1701 const CombineInfo &Paired) { in getSubRegIdxs() argument
1702 assert((CI.InstClass != MIMG || (countPopulation(CI.DMask | Paired.DMask) == in getSubRegIdxs()
1703 CI.Width + Paired.Width)) && in getSubRegIdxs()
1718 assert(Paired.Width >= 1 && Paired.Width <= 4); in getSubRegIdxs()
1720 if (Paired < CI) { in getSubRegIdxs()
1721 Idx1 = Idxs[0][Paired.Width - 1]; in getSubRegIdxs()
1722 Idx0 = Idxs[Paired.Width][CI.Width - 1]; in getSubRegIdxs()
1725 Idx1 = Idxs[CI.Width][Paired.Width - 1]; in getSubRegIdxs()
1733 const CombineInfo &Paired) { in getTargetRegisterClass() argument
1735 switch (CI.Width + Paired.Width) { in getTargetRegisterClass()
1749 unsigned BitWidth = 32 * (CI.Width + Paired.Width); in getTargetRegisterClass()
1756 CombineInfo &CI, CombineInfo &Paired, in mergeBufferStorePair() argument
1761 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeBufferStorePair()
1763 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); in mergeBufferStorePair()
1768 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeBufferStorePair()
1772 const auto *Src1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdata); in mergeBufferStorePair()
1792 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeBufferStorePair()
1797 .addImm(std::min(CI.Offset, Paired.Offset)) // offset in mergeBufferStorePair()
1801 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeBufferStorePair()
1804 Paired.I->eraseFromParent(); in mergeBufferStorePair()
2279 CombineInfo &Paired = *Second; in optimizeInstsWithSameBaseAddr() local
2281 CombineInfo *Where = checkAndPrepareMerge(CI, Paired); in optimizeInstsWithSameBaseAddr()
2289 LLVM_DEBUG(dbgs() << "Merging: " << *CI.I << " with: " << *Paired.I); in optimizeInstsWithSameBaseAddr()
2297 NewMI = mergeRead2Pair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2300 NewMI = mergeWrite2Pair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2303 NewMI = mergeSBufferLoadImmPair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2304 OptimizeListAgain |= CI.Width + Paired.Width < 8; in optimizeInstsWithSameBaseAddr()
2307 NewMI = mergeBufferLoadPair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2308 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2311 NewMI = mergeBufferStorePair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2312 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2315 NewMI = mergeImagePair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2316 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2319 NewMI = mergeTBufferLoadPair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2320 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2323 NewMI = mergeTBufferStorePair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2324 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2329 NewMI = mergeFlatLoadPair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2330 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2335 NewMI = mergeFlatStorePair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2336 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()