Lines Matching refs:Paired
190 const CombineInfo &Paired);
192 CombineInfo &Paired, bool Modify = false);
194 const CombineInfo &Paired);
195 static unsigned getNewOpcode(const CombineInfo &CI, const CombineInfo &Paired);
197 const CombineInfo &Paired);
199 const CombineInfo &Paired);
202 bool checkAndPrepareMerge(CombineInfo &CI, CombineInfo &Paired,
208 CombineInfo &Paired,
214 mergeWrite2Pair(CombineInfo &CI, CombineInfo &Paired,
217 mergeImagePair(CombineInfo &CI, CombineInfo &Paired,
220 mergeSBufferLoadImmPair(CombineInfo &CI, CombineInfo &Paired,
223 mergeBufferLoadPair(CombineInfo &CI, CombineInfo &Paired,
226 mergeBufferStorePair(CombineInfo &CI, CombineInfo &Paired,
229 mergeTBufferLoadPair(CombineInfo &CI, CombineInfo &Paired,
232 mergeTBufferStorePair(CombineInfo &CI, CombineInfo &Paired,
672 const CombineInfo &Paired) { in dmasksCanBeCombined() argument
689 if (AMDGPU::getNamedOperandIdx(Paired.I->getOpcode(), op) != Idx) in dmasksCanBeCombined()
692 CI.I->getOperand(Idx).getImm() != Paired.I->getOperand(Idx).getImm()) in dmasksCanBeCombined()
697 unsigned MaxMask = std::max(CI.DMask, Paired.DMask); in dmasksCanBeCombined()
698 unsigned MinMask = std::min(CI.DMask, Paired.DMask); in dmasksCanBeCombined()
744 CombineInfo &Paired, in offsetsCanBeCombined() argument
750 if (CI.Offset == Paired.Offset) in offsetsCanBeCombined()
754 if ((CI.Offset % CI.EltSize != 0) || (Paired.Offset % CI.EltSize != 0)) in offsetsCanBeCombined()
764 llvm::AMDGPU::getGcnBufferFormatInfo(Paired.Format, STI); in offsetsCanBeCombined()
777 if (getBufferFormatWithCompCount(CI.Format, CI.Width + Paired.Width, STI) == 0) in offsetsCanBeCombined()
782 uint32_t EltOffset1 = Paired.Offset / CI.EltSize; in offsetsCanBeCombined()
789 EltOffset1 + Paired.Width == EltOffset0) && in offsetsCanBeCombined()
790 CI.CPol == Paired.CPol && in offsetsCanBeCombined()
791 (CI.InstClass == S_BUFFER_LOAD_IMM || CI.CPol == Paired.CPol); in offsetsCanBeCombined()
800 Paired.Offset = EltOffset1 / 64; in offsetsCanBeCombined()
810 Paired.Offset = EltOffset1; in offsetsCanBeCombined()
831 Paired.Offset = (EltOffset1 - BaseOff) / 64; in offsetsCanBeCombined()
845 Paired.Offset = EltOffset1 - BaseOff; in offsetsCanBeCombined()
855 const CombineInfo &Paired) { in widthsFit() argument
856 const unsigned Width = (CI.Width + Paired.Width); in widthsFit()
893 CombineInfo &CI, CombineInfo &Paired, in checkAndPrepareMerge() argument
898 if (CI.InstClass == MIMG && !dmasksCanBeCombined(CI, *TII, Paired)) in checkAndPrepareMerge()
902 (!widthsFit(*STM, CI, Paired) || !offsetsCanBeCombined(CI, *STM, Paired))) in checkAndPrepareMerge()
926 MachineBasicBlock::iterator E = std::next(Paired.I); in checkAndPrepareMerge()
993 if (&*MBBI == &*Paired.I) { in checkAndPrepareMerge()
1015 offsetsCanBeCombined(CI, *STM, Paired, true); in checkAndPrepareMerge()
1048 SILoadStoreOptimizer::mergeRead2Pair(CombineInfo &CI, CombineInfo &Paired, in mergeRead2Pair() argument
1057 const auto *Dest1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdst); in mergeRead2Pair()
1060 unsigned NewOffset1 = Paired.Offset; in mergeRead2Pair()
1078 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeRead2Pair()
1088 BuildMI(*MBB, Paired.I, DL, TII->get(AMDGPU::S_MOV_B32), ImmReg) in mergeRead2Pair()
1094 TII->getAddNoCarry(*MBB, Paired.I, DL, BaseReg) in mergeRead2Pair()
1102 BuildMI(*MBB, Paired.I, DL, Read2Desc, DestReg) in mergeRead2Pair()
1107 .cloneMergedMemRefs({&*CI.I, &*Paired.I}); in mergeRead2Pair()
1114 BuildMI(*MBB, Paired.I, DL, CopyDesc) in mergeRead2Pair()
1117 MachineInstr *Copy1 = BuildMI(*MBB, Paired.I, DL, CopyDesc) in mergeRead2Pair()
1124 Paired.I->eraseFromParent(); in mergeRead2Pair()
1147 SILoadStoreOptimizer::mergeWrite2Pair(CombineInfo &CI, CombineInfo &Paired, in mergeWrite2Pair() argument
1158 TII->getNamedOperand(*Paired.I, AMDGPU::OpName::data0); in mergeWrite2Pair()
1161 unsigned NewOffset1 = Paired.Offset; in mergeWrite2Pair()
1182 BuildMI(*MBB, Paired.I, DL, TII->get(AMDGPU::S_MOV_B32), ImmReg) in mergeWrite2Pair()
1188 TII->getAddNoCarry(*MBB, Paired.I, DL, BaseReg) in mergeWrite2Pair()
1196 BuildMI(*MBB, Paired.I, DL, Write2Desc) in mergeWrite2Pair()
1203 .cloneMergedMemRefs({&*CI.I, &*Paired.I}); in mergeWrite2Pair()
1208 Paired.I->eraseFromParent(); in mergeWrite2Pair()
1215 SILoadStoreOptimizer::mergeImagePair(CombineInfo &CI, CombineInfo &Paired, in mergeImagePair() argument
1219 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeImagePair()
1221 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeImagePair()
1224 unsigned MergedDMask = CI.DMask | Paired.DMask; in mergeImagePair()
1228 auto MIB = BuildMI(*MBB, Paired.I, DL, TII->get(Opcode), DestReg); in mergeImagePair()
1239 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeImagePair()
1242 const MachineMemOperand *MMOb = *Paired.I->memoperands_begin(); in mergeImagePair()
1247 std::tie(SubRegIdx0, SubRegIdx1) = getSubRegIdxs(CI, Paired); in mergeImagePair()
1252 const auto *Dest1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdata); in mergeImagePair()
1254 BuildMI(*MBB, Paired.I, DL, CopyDesc) in mergeImagePair()
1257 MachineInstr *Copy1 = BuildMI(*MBB, Paired.I, DL, CopyDesc) in mergeImagePair()
1264 Paired.I->eraseFromParent(); in mergeImagePair()
1269 CombineInfo &CI, CombineInfo &Paired, in mergeSBufferLoadImmPair() argument
1273 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeSBufferLoadImmPair()
1275 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeSBufferLoadImmPair()
1278 unsigned MergedOffset = std::min(CI.Offset, Paired.Offset); in mergeSBufferLoadImmPair()
1283 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeSBufferLoadImmPair()
1286 const MachineMemOperand *MMOb = *Paired.I->memoperands_begin(); in mergeSBufferLoadImmPair()
1289 BuildMI(*MBB, Paired.I, DL, TII->get(Opcode), DestReg) in mergeSBufferLoadImmPair()
1295 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); in mergeSBufferLoadImmPair()
1302 const auto *Dest1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::sdst); in mergeSBufferLoadImmPair()
1304 BuildMI(*MBB, Paired.I, DL, CopyDesc) in mergeSBufferLoadImmPair()
1307 MachineInstr *Copy1 = BuildMI(*MBB, Paired.I, DL, CopyDesc) in mergeSBufferLoadImmPair()
1314 Paired.I->eraseFromParent(); in mergeSBufferLoadImmPair()
1319 CombineInfo &CI, CombineInfo &Paired, in mergeBufferLoadPair() argument
1324 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeBufferLoadPair()
1326 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeBufferLoadPair()
1330 unsigned MergedOffset = std::min(CI.Offset, Paired.Offset); in mergeBufferLoadPair()
1332 auto MIB = BuildMI(*MBB, Paired.I, DL, TII->get(Opcode), DestReg); in mergeBufferLoadPair()
1342 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeBufferLoadPair()
1345 const MachineMemOperand *MMOb = *Paired.I->memoperands_begin(); in mergeBufferLoadPair()
1356 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); in mergeBufferLoadPair()
1363 const auto *Dest1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdata); in mergeBufferLoadPair()
1365 BuildMI(*MBB, Paired.I, DL, CopyDesc) in mergeBufferLoadPair()
1368 MachineInstr *Copy1 = BuildMI(*MBB, Paired.I, DL, CopyDesc) in mergeBufferLoadPair()
1375 Paired.I->eraseFromParent(); in mergeBufferLoadPair()
1380 CombineInfo &CI, CombineInfo &Paired, in mergeTBufferLoadPair() argument
1385 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeTBufferLoadPair()
1387 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeTBufferLoadPair()
1391 unsigned MergedOffset = std::min(CI.Offset, Paired.Offset); in mergeTBufferLoadPair()
1393 auto MIB = BuildMI(*MBB, Paired.I, DL, TII->get(Opcode), DestReg); in mergeTBufferLoadPair()
1401 getBufferFormatWithCompCount(CI.Format, CI.Width + Paired.Width, *STM); in mergeTBufferLoadPair()
1406 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeTBufferLoadPair()
1409 const MachineMemOperand *MMOb = *Paired.I->memoperands_begin(); in mergeTBufferLoadPair()
1422 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); in mergeTBufferLoadPair()
1429 const auto *Dest1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdata); in mergeTBufferLoadPair()
1431 BuildMI(*MBB, Paired.I, DL, CopyDesc) in mergeTBufferLoadPair()
1434 MachineInstr *Copy1 = BuildMI(*MBB, Paired.I, DL, CopyDesc) in mergeTBufferLoadPair()
1441 Paired.I->eraseFromParent(); in mergeTBufferLoadPair()
1446 CombineInfo &CI, CombineInfo &Paired, in mergeTBufferStorePair() argument
1451 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeTBufferStorePair()
1453 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); in mergeTBufferStorePair()
1458 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeTBufferStorePair()
1462 const auto *Src1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdata); in mergeTBufferStorePair()
1464 BuildMI(*MBB, Paired.I, DL, TII->get(AMDGPU::REG_SEQUENCE), SrcReg) in mergeTBufferStorePair()
1470 auto MIB = BuildMI(*MBB, Paired.I, DL, TII->get(Opcode)) in mergeTBufferStorePair()
1479 getBufferFormatWithCompCount(CI.Format, CI.Width + Paired.Width, *STM); in mergeTBufferStorePair()
1484 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeTBufferStorePair()
1487 const MachineMemOperand *MMOb = *Paired.I->memoperands_begin(); in mergeTBufferStorePair()
1492 .addImm(std::min(CI.Offset, Paired.Offset)) // offset in mergeTBufferStorePair()
1503 Paired.I->eraseFromParent(); in mergeTBufferStorePair()
1508 const CombineInfo &Paired) { in getNewOpcode() argument
1509 const unsigned Width = CI.Width + Paired.Width; in getNewOpcode()
1534 assert("No overlaps" && (countPopulation(CI.DMask | Paired.DMask) == Width)); in getNewOpcode()
1540 SILoadStoreOptimizer::getSubRegIdxs(const CombineInfo &CI, const CombineInfo &Paired) { in getSubRegIdxs() argument
1542 if (CI.Width == 0 || Paired.Width == 0 || CI.Width + Paired.Width > 4) in getSubRegIdxs()
1547 assert((countPopulation(CI.DMask | Paired.DMask) == CI.Width + Paired.Width) && in getSubRegIdxs()
1549 ReverseOrder = CI.DMask > Paired.DMask; in getSubRegIdxs()
1551 ReverseOrder = CI.Offset > Paired.Offset; in getSubRegIdxs()
1563 assert(Paired.Width >= 1 && Paired.Width <= 3); in getSubRegIdxs()
1566 Idx1 = Idxs[0][Paired.Width - 1]; in getSubRegIdxs()
1567 Idx0 = Idxs[Paired.Width][CI.Width - 1]; in getSubRegIdxs()
1570 Idx1 = Idxs[CI.Width][Paired.Width - 1]; in getSubRegIdxs()
1578 const CombineInfo &Paired) { in getTargetRegisterClass() argument
1580 switch (CI.Width + Paired.Width) { in getTargetRegisterClass()
1594 unsigned BitWidth = 32 * (CI.Width + Paired.Width); in getTargetRegisterClass()
1601 CombineInfo &CI, CombineInfo &Paired, in mergeBufferStorePair() argument
1606 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeBufferStorePair()
1608 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); in mergeBufferStorePair()
1613 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeBufferStorePair()
1617 const auto *Src1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdata); in mergeBufferStorePair()
1619 BuildMI(*MBB, Paired.I, DL, TII->get(AMDGPU::REG_SEQUENCE), SrcReg) in mergeBufferStorePair()
1625 auto MIB = BuildMI(*MBB, Paired.I, DL, TII->get(Opcode)) in mergeBufferStorePair()
1637 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeBufferStorePair()
1640 const MachineMemOperand *MMOb = *Paired.I->memoperands_begin(); in mergeBufferStorePair()
1645 .addImm(std::min(CI.Offset, Paired.Offset)) // offset in mergeBufferStorePair()
1654 Paired.I->eraseFromParent(); in mergeBufferStorePair()
2111 CombineInfo &Paired = *Second; in optimizeInstsWithSameBaseAddr() local
2114 if (!checkAndPrepareMerge(CI, Paired, InstsToMove)) { in optimizeInstsWithSameBaseAddr()
2121 LLVM_DEBUG(dbgs() << "Merging: " << *CI.I << " with: " << *Paired.I); in optimizeInstsWithSameBaseAddr()
2129 mergeRead2Pair(CI, Paired, InstsToMove); in optimizeInstsWithSameBaseAddr()
2135 mergeWrite2Pair(CI, Paired, InstsToMove); in optimizeInstsWithSameBaseAddr()
2141 mergeSBufferLoadImmPair(CI, Paired, InstsToMove); in optimizeInstsWithSameBaseAddr()
2143 OptimizeListAgain |= (CI.Width + Paired.Width) < 16; in optimizeInstsWithSameBaseAddr()
2148 mergeBufferLoadPair(CI, Paired, InstsToMove); in optimizeInstsWithSameBaseAddr()
2150 OptimizeListAgain |= (CI.Width + Paired.Width) < 4; in optimizeInstsWithSameBaseAddr()
2155 mergeBufferStorePair(CI, Paired, InstsToMove); in optimizeInstsWithSameBaseAddr()
2157 OptimizeListAgain |= (CI.Width + Paired.Width) < 4; in optimizeInstsWithSameBaseAddr()
2162 mergeImagePair(CI, Paired, InstsToMove); in optimizeInstsWithSameBaseAddr()
2164 OptimizeListAgain |= (CI.Width + Paired.Width) < 4; in optimizeInstsWithSameBaseAddr()
2169 mergeTBufferLoadPair(CI, Paired, InstsToMove); in optimizeInstsWithSameBaseAddr()
2171 OptimizeListAgain |= (CI.Width + Paired.Width) < 4; in optimizeInstsWithSameBaseAddr()
2176 mergeTBufferStorePair(CI, Paired, InstsToMove); in optimizeInstsWithSameBaseAddr()
2178 OptimizeListAgain |= (CI.Width + Paired.Width) < 4; in optimizeInstsWithSameBaseAddr()
2182 CI.Order = Paired.Order; in optimizeInstsWithSameBaseAddr()