Lines Matching refs:NarrowTy
56 getNarrowTypeBreakDown(LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy) { in getNarrowTypeBreakDown() argument
60 unsigned NarrowSize = NarrowTy.getSizeInBits(); in getNarrowTypeBreakDown()
68 if (NarrowTy.isVector()) { in getNarrowTypeBreakDown()
250 LLT NarrowTy, Register SrcReg) { in extractGCDType() argument
252 LLT GCDTy = getGCDType(getGCDType(SrcTy, NarrowTy), DstTy); in extractGCDType()
257 LLT LegalizerHelper::buildLCMMergePieces(LLT DstTy, LLT NarrowTy, LLT GCDTy, in buildLCMMergePieces() argument
260 LLT LCMTy = getLCMType(DstTy, NarrowTy); in buildLCMMergePieces()
262 int NumParts = LCMTy.getSizeInBits() / NarrowTy.getSizeInBits(); in buildLCMMergePieces()
263 int NumSubParts = NarrowTy.getSizeInBits() / GCDTy.getSizeInBits(); in buildLCMMergePieces()
319 AllPadReg = MIRBuilder.buildUndef(NarrowTy).getReg(0); in buildLCMMergePieces()
321 AllPadReg = MIRBuilder.buildConstant(NarrowTy, 0).getReg(0); in buildLCMMergePieces()
337 Remerge[I] = MIRBuilder.buildMergeLikeInstr(NarrowTy, SubMerge).getReg(0); in buildLCMMergePieces()
1181 LLT NarrowTy) { in narrowScalar() argument
1183 uint64_t NarrowSize = NarrowTy.getSizeInBits(); in narrowScalar()
1199 LLT ImplicitTy = NarrowTy; in narrowScalar()
1214 DstRegs.push_back(MIRBuilder.buildUndef(NarrowTy).getReg(0)); in narrowScalar()
1227 unsigned NarrowSize = NarrowTy.getSizeInBits(); in narrowScalar()
1233 auto K = MIRBuilder.buildConstant(NarrowTy, in narrowScalar()
1250 Ty, NarrowTy, PartRegs, LeftoverTy, LeftoverRegs); in narrowScalar()
1258 return narrowScalarExt(MI, TypeIdx, NarrowTy); in narrowScalar()
1264 if (NarrowTy.getSizeInBits() * 2 != SizeOp1) { in narrowScalar()
1265 LLVM_DEBUG(dbgs() << "Can't narrow trunc to type " << NarrowTy << "\n"); in narrowScalar()
1269 auto Unmerge = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1)); in narrowScalar()
1281 if (Ty.getSizeInBits() % NarrowTy.getSizeInBits() != 0) in narrowScalar()
1284 auto Unmerge = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1).getReg()); in narrowScalar()
1288 MIRBuilder.buildFreeze(NarrowTy, Unmerge.getReg(i)).getReg(0)); in narrowScalar()
1305 return narrowScalarAddSub(MI, TypeIdx, NarrowTy); in narrowScalar()
1308 return narrowScalarMul(MI, NarrowTy); in narrowScalar()
1310 return narrowScalarExtract(MI, TypeIdx, NarrowTy); in narrowScalar()
1312 return narrowScalarInsert(MI, TypeIdx, NarrowTy); in narrowScalar()
1321 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy); in narrowScalar()
1328 return reduceLoadStoreWidth(LoadMI, TypeIdx, NarrowTy); in narrowScalar()
1336 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy); in narrowScalar()
1366 unsigned HandledSize = NumParts * NarrowTy.getSizeInBits(); in narrowScalar()
1372 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy); in narrowScalar()
1379 return reduceLoadStoreWidth(StoreMI, 0, NarrowTy); in narrowScalar()
1382 return narrowScalarSelect(MI, TypeIdx, NarrowTy); in narrowScalar()
1395 return narrowScalarBasic(MI, TypeIdx, NarrowTy); in narrowScalar()
1400 return narrowScalarShift(MI, TypeIdx, NarrowTy); in narrowScalar()
1410 return narrowScalarCTLZ(MI, TypeIdx, NarrowTy); in narrowScalar()
1413 return narrowScalarCTTZ(MI, TypeIdx, NarrowTy); in narrowScalar()
1415 return narrowScalarCTPOP(MI, TypeIdx, NarrowTy); in narrowScalar()
1421 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT); in narrowScalar()
1429 narrowScalarSrc(MI, NarrowTy, 1); in narrowScalar()
1437 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT); in narrowScalar()
1453 extractParts(MI.getOperand(i).getReg(), NarrowTy, NumParts, in narrowScalar()
1459 DstRegs[i] = MRI.createGenericVirtualRegister(NarrowTy); in narrowScalar()
1478 narrowScalarSrc(MI, NarrowTy, OpIdx); in narrowScalar()
1495 if (!extractParts(LHS, SrcTy, NarrowTy, LeftoverTy, LHSPartRegs, in narrowScalar()
1501 if (!extractParts(MI.getOperand(3).getReg(), SrcTy, NarrowTy, Unused, in narrowScalar()
1513 auto Zero = MIRBuilder.buildConstant(NarrowTy, 0); in narrowScalar()
1518 auto Xor = MIRBuilder.buildXor(NarrowTy, LHS, RHS).getReg(0); in narrowScalar()
1529 LLT GCDTy = extractGCDType(WidenedXors, NarrowTy, LeftoverTy, Xor); in narrowScalar()
1530 buildLCMMergePieces(LeftoverTy, NarrowTy, GCDTy, WidenedXors, in narrowScalar()
1539 auto Or = MIRBuilder.buildOr(NarrowTy, Xors[0], Xors[1]); in narrowScalar()
1541 Or = MIRBuilder.buildOr(NarrowTy, Or, Xors[I]); in narrowScalar()
1569 if (NarrowTy.getScalarSizeInBits() > SizeInBits) { in narrowScalar()
1574 auto TruncMIB = MIRBuilder.buildTrunc(NarrowTy, MO1); in narrowScalar()
1578 Register DstExt = MRI.createGenericVirtualRegister(NarrowTy); in narrowScalar()
1601 Register SrcReg = MRI.createGenericVirtualRegister(NarrowTy); in narrowScalar()
1610 MIRBuilder.buildConstant(NarrowTy, NarrowTy.getScalarSizeInBits() - 1) in narrowScalar()
1617 if ((i + 1) * NarrowTy.getScalarSizeInBits() <= SizeInBits) { in narrowScalar()
1620 } else if (i * NarrowTy.getScalarSizeInBits() >= SizeInBits) { in narrowScalar()
1628 MIRBuilder.buildAShr(NarrowTy, PartialExtensionReg, AshrCstReg) in narrowScalar()
1635 TargetOpcode::G_SEXT_INREG, {NarrowTy}, in narrowScalar()
1636 {SrcRegs[i], SizeInBits % NarrowTy.getScalarSizeInBits()}) in narrowScalar()
1656 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs, in narrowScalar()
1660 auto DstPart = MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy}, in narrowScalar()
1676 narrowScalarSrc(MI, NarrowTy, 2); in narrowScalar()
1682 return narrowScalarFPTOI(MI, TypeIdx, NarrowTy); in narrowScalar()
1687 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_FPEXT); in narrowScalar()
1692 return narrowScalarFLDEXP(MI, TypeIdx, NarrowTy); in narrowScalar()
1725 void LegalizerHelper::narrowScalarSrc(MachineInstr &MI, LLT NarrowTy, in narrowScalarSrc() argument
1728 auto ExtB = MIRBuilder.buildTrunc(NarrowTy, MO); in narrowScalarSrc()
1741 void LegalizerHelper::narrowScalarDst(MachineInstr &MI, LLT NarrowTy, in narrowScalarDst() argument
1744 Register DstTrunc = MRI.createGenericVirtualRegister(NarrowTy); in narrowScalarDst()
4030 LLT NarrowTy = (NumElts == 1) ? EltTy : LLT::fixed_vector(NumElts, EltTy); in makeDstOps() local
4033 getNarrowTypeBreakDown(Ty, NarrowTy, LeftoverTy); in makeDstOps()
4037 DstOps.push_back(NarrowTy); in makeDstOps()
4201 LLT NarrowTy) { in fewerElementsVectorUnmergeValues() argument
4207 if (TypeIdx != 1 || NarrowTy == DstTy) in fewerElementsVectorUnmergeValues()
4214 assert(SrcTy.isVector() && NarrowTy.isVector() && "Expected vector types"); in fewerElementsVectorUnmergeValues()
4215 assert((SrcTy.getScalarType() == NarrowTy.getScalarType()) && "bad type"); in fewerElementsVectorUnmergeValues()
4217 if ((SrcTy.getSizeInBits() % NarrowTy.getSizeInBits() != 0) || in fewerElementsVectorUnmergeValues()
4218 (NarrowTy.getSizeInBits() % DstTy.getSizeInBits() != 0)) in fewerElementsVectorUnmergeValues()
4231 auto Unmerge = MIRBuilder.buildUnmerge(NarrowTy, SrcReg); in fewerElementsVectorUnmergeValues()
4249 LLT NarrowTy) { in fewerElementsVectorMerge() argument
4254 assert(DstTy.isVector() && NarrowTy.isVector() && "Expected vector types"); in fewerElementsVectorMerge()
4255 assert((DstTy.getScalarType() == NarrowTy.getScalarType()) && "bad type"); in fewerElementsVectorMerge()
4256 if (NarrowTy == SrcTy) in fewerElementsVectorMerge()
4265 assert((SrcTy.getScalarType() == NarrowTy.getScalarType()) && "bad type"); in fewerElementsVectorMerge()
4266 if ((DstTy.getSizeInBits() % NarrowTy.getSizeInBits() != 0) || in fewerElementsVectorMerge()
4267 (NarrowTy.getNumElements() >= SrcTy.getNumElements())) in fewerElementsVectorMerge()
4287 unsigned NumNarrowTyElts = NarrowTy.getNumElements(); in fewerElementsVectorMerge()
4293 MIRBuilder.buildMergeLikeInstr(NarrowTy, Pieces).getReg(0)); in fewerElementsVectorMerge()
4302 if ((NarrowTy.getSizeInBits() % SrcTy.getSizeInBits() != 0) || in fewerElementsVectorMerge()
4303 (DstTy.getSizeInBits() % NarrowTy.getSizeInBits() != 0)) in fewerElementsVectorMerge()
4317 unsigned NumParts = DstTy.getNumElements() / NarrowTy.getNumElements(); in fewerElementsVectorMerge()
4319 unsigned NumElts = NarrowTy.getNumElements() / NumSrcElts; in fewerElementsVectorMerge()
4325 MIRBuilder.buildMergeLikeInstr(NarrowTy, Sources).getReg(0)); in fewerElementsVectorMerge()
4409 LLT NarrowTy) { in reduceLoadStoreWidth() argument
4435 std::tie(NumParts, NumLeftover) = getNarrowTypeBreakDown(ValTy, NarrowTy, LeftoverTy); in reduceLoadStoreWidth()
4437 if (extractParts(ValReg, ValTy, NarrowTy, LeftoverTy, NarrowRegs, in reduceLoadStoreWidth()
4485 unsigned Offset = isBigEndian ? TotalSize - NarrowTy.getSizeInBits() : 0; in reduceLoadStoreWidth()
4487 splitTypePieces(NarrowTy, NarrowRegs, NumParts, Offset); in reduceLoadStoreWidth()
4494 insertParts(ValReg, ValTy, NarrowTy, NarrowRegs, in reduceLoadStoreWidth()
4504 LLT NarrowTy) { in fewerElementsVector() argument
4507 unsigned NumElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1; in fewerElementsVector()
4629 return fewerElementsVectorUnmergeValues(MI, TypeIdx, NarrowTy); in fewerElementsVector()
4632 return fewerElementsVectorMerge(MI, TypeIdx, NarrowTy); in fewerElementsVector()
4636 return fewerElementsVectorMerge(MI, TypeIdx, NarrowTy); in fewerElementsVector()
4639 return fewerElementsVectorExtractInsertVectorElt(MI, TypeIdx, NarrowTy); in fewerElementsVector()
4642 return reduceLoadStoreWidth(cast<GLoadStore>(MI), TypeIdx, NarrowTy); in fewerElementsVector()
4646 return fewerElementsVectorReductions(MI, TypeIdx, NarrowTy); in fewerElementsVector()
4649 return fewerElementsVectorSeqReductions(MI, TypeIdx, NarrowTy); in fewerElementsVector()
4651 return fewerElementsVectorShuffle(MI, TypeIdx, NarrowTy); in fewerElementsVector()
4660 MachineInstr &MI, unsigned int TypeIdx, LLT NarrowTy) { in fewerElementsVectorShuffle() argument
4679 NarrowTy = in fewerElementsVectorShuffle()
4681 unsigned NewElts = NarrowTy.getNumElements(); in fewerElementsVectorShuffle()
4684 extractParts(Src1Reg, NarrowTy, 2, SplitSrc1Regs, MIRBuilder, MRI); in fewerElementsVectorShuffle()
4685 extractParts(Src2Reg, NarrowTy, 2, SplitSrc2Regs, MIRBuilder, MRI); in fewerElementsVectorShuffle()
4746 LLT EltTy = NarrowTy.getElementType(); in fewerElementsVectorShuffle()
4775 Output = MIRBuilder.buildBuildVector(NarrowTy, SVOps).getReg(0); in fewerElementsVectorShuffle()
4778 Output = MIRBuilder.buildUndef(NarrowTy).getReg(0); in fewerElementsVectorShuffle()
4783 ? MIRBuilder.buildUndef(NarrowTy).getReg(0) in fewerElementsVectorShuffle()
4786 Output = MIRBuilder.buildShuffleVector(NarrowTy, Op0, Op1, Ops).getReg(0); in fewerElementsVectorShuffle()
4798 MachineInstr &MI, unsigned int TypeIdx, LLT NarrowTy) { in fewerElementsVectorReductions() argument
4808 if (NarrowTy.isVector() && in fewerElementsVectorReductions()
4809 (SrcTy.getNumElements() % NarrowTy.getNumElements() != 0)) in fewerElementsVectorReductions()
4816 NarrowTy.isVector() ? SrcTy.getNumElements() / NarrowTy.getNumElements() in fewerElementsVectorReductions()
4819 extractParts(SrcReg, NarrowTy, NumParts, SplitSrcs, MIRBuilder, MRI); in fewerElementsVectorReductions()
4820 if (NarrowTy.isScalar()) { in fewerElementsVectorReductions()
4821 if (DstTy != NarrowTy) in fewerElementsVectorReductions()
4832 .buildInstr(ScalarOpc, {NarrowTy}, in fewerElementsVectorReductions()
4848 Acc = MIRBuilder.buildInstr(ScalarOpc, {NarrowTy}, {Acc, SplitSrcs[Idx]}) in fewerElementsVectorReductions()
4864 isPowerOf2_32(NarrowTy.getNumElements())) { in fewerElementsVectorReductions()
4865 return tryNarrowPow2Reduction(MI, SrcReg, SrcTy, NarrowTy, ScalarOpc); in fewerElementsVectorReductions()
4886 LLT NarrowTy) { in fewerElementsVectorSeqReductions() argument
4889 if (!NarrowTy.isScalar() || TypeIdx != 2 || DstTy != ScalarTy || in fewerElementsVectorSeqReductions()
4890 DstTy != NarrowTy) in fewerElementsVectorSeqReductions()
4902 extractParts(SrcReg, NarrowTy, NumParts, SplitSrcs, MIRBuilder, MRI); in fewerElementsVectorSeqReductions()
4905 Acc = MIRBuilder.buildInstr(ScalarOpc, {NarrowTy}, {Acc, SplitSrcs[i]}) in fewerElementsVectorSeqReductions()
4915 LLT SrcTy, LLT NarrowTy, in tryNarrowPow2Reduction() argument
4919 extractParts(SrcReg, NarrowTy, in tryNarrowPow2Reduction()
4920 SrcTy.getNumElements() / NarrowTy.getNumElements(), SplitSrcs, in tryNarrowPow2Reduction()
4931 MIRBuilder.buildInstr(ScalarOpc, {NarrowTy}, {LHS, RHS}).getReg(0); in tryNarrowPow2Reduction()
5460 LLT NarrowTy) { in multiplyRegisters() argument
5467 B.buildMul(NarrowTy, Src1Regs[DstIdx], Src2Regs[DstIdx]).getReg(0); in multiplyRegisters()
5478 B.buildMul(NarrowTy, Src1Regs[DstIdx - i], Src2Regs[i]); in multiplyRegisters()
5485 B.buildUMulH(NarrowTy, Src1Regs[DstIdx - 1 - i], Src2Regs[i]); in multiplyRegisters()
5497 B.buildUAddo(NarrowTy, LLT::scalar(1), Factors[0], Factors[1]); in multiplyRegisters()
5499 CarrySum = B.buildZExt(NarrowTy, Uaddo.getReg(1)).getReg(0); in multiplyRegisters()
5502 B.buildUAddo(NarrowTy, LLT::scalar(1), FactorSum, Factors[i]); in multiplyRegisters()
5504 MachineInstrBuilder Carry = B.buildZExt(NarrowTy, Uaddo.getReg(1)); in multiplyRegisters()
5505 CarrySum = B.buildAdd(NarrowTy, CarrySum, Carry).getReg(0); in multiplyRegisters()
5509 FactorSum = B.buildAdd(NarrowTy, Factors[0], Factors[1]).getReg(0); in multiplyRegisters()
5511 FactorSum = B.buildAdd(NarrowTy, FactorSum, Factors[i]).getReg(0); in multiplyRegisters()
5522 LLT NarrowTy) { in narrowScalarAddSub() argument
5574 extractParts(Src1, RegTy, NarrowTy, LeftoverTy, Src1Regs, Src1Left, in narrowScalarAddSub()
5576 extractParts(Src2, RegTy, NarrowTy, DummyTy, Src2Regs, Src2Left, MIRBuilder, in narrowScalarAddSub()
5608 insertParts(MI.getOperand(0).getReg(), RegTy, NarrowTy, in narrowScalarAddSub()
5617 LegalizerHelper::narrowScalarMul(MachineInstr &MI, LLT NarrowTy) { in narrowScalarMul() argument
5625 unsigned NarrowSize = NarrowTy.getSizeInBits(); in narrowScalarMul()
5635 extractParts(Src1, NarrowTy, NumParts, Src1Parts, MIRBuilder, MRI); in narrowScalarMul()
5636 extractParts(Src2, NarrowTy, NumParts, Src2Parts, MIRBuilder, MRI); in narrowScalarMul()
5637 multiplyRegisters(DstTmpRegs, Src1Parts, Src2Parts, NarrowTy); in narrowScalarMul()
5648 LLT NarrowTy) { in narrowScalarFPTOI() argument
5661 NarrowTy.getScalarSizeInBits() < (IsSigned ? 17u : 16u)) in narrowScalarFPTOI()
5665 narrowScalarDst(MI, NarrowTy, 0, in narrowScalarFPTOI()
5673 LLT NarrowTy) { in narrowScalarExtract() argument
5677 uint64_t NarrowSize = NarrowTy.getSizeInBits(); in narrowScalarExtract()
5688 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs, in narrowScalarExtract()
5700 } else if (SrcStart == OpStart && NarrowTy == MRI.getType(OpReg)) { in narrowScalarExtract()
5741 LLT NarrowTy) { in narrowScalarInsert() argument
5750 extractParts(MI.getOperand(1).getReg(), RegTy, NarrowTy, LeftoverTy, SrcRegs, in narrowScalarInsert()
5756 uint64_t NarrowSize = NarrowTy.getSizeInBits(); in narrowScalarInsert()
5763 if (DstStart == OpStart && NarrowTy == MRI.getType(OpReg)) { in narrowScalarInsert()
5773 SrcReg = MRI.createGenericVirtualRegister(NarrowTy); in narrowScalarInsert()
5805 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy); in narrowScalarInsert()
5825 LLT NarrowTy) { in narrowScalarBasic() argument
5835 if (!extractParts(MI.getOperand(1).getReg(), DstTy, NarrowTy, LeftoverTy, in narrowScalarBasic()
5840 if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, Unused, in narrowScalarBasic()
5845 auto Inst = MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy}, in narrowScalarBasic()
5857 insertParts(DstReg, DstTy, NarrowTy, DstRegs, in narrowScalarBasic()
5866 LLT NarrowTy) { in narrowScalarExt() argument
5877 LLT GCDTy = extractGCDType(Parts, DstTy, NarrowTy, SrcReg); in narrowScalarExt()
5878 LLT LCMTy = buildLCMMergePieces(DstTy, NarrowTy, GCDTy, Parts, MI.getOpcode()); in narrowScalarExt()
5887 LLT NarrowTy) { in narrowScalarSelect() argument
5903 if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, LeftoverTy, in narrowScalarSelect()
5908 if (!extractParts(MI.getOperand(3).getReg(), DstTy, NarrowTy, Unused, in narrowScalarSelect()
5913 auto Select = MIRBuilder.buildSelect(NarrowTy, in narrowScalarSelect()
5924 insertParts(DstReg, DstTy, NarrowTy, DstRegs, in narrowScalarSelect()
5933 LLT NarrowTy) { in narrowScalarCTLZ() argument
5938 unsigned NarrowSize = NarrowTy.getSizeInBits(); in narrowScalarCTLZ()
5944 auto UnmergeSrc = B.buildUnmerge(NarrowTy, SrcReg); in narrowScalarCTLZ()
5946 auto C_0 = B.buildConstant(NarrowTy, 0); in narrowScalarCTLZ()
5966 LLT NarrowTy) { in narrowScalarCTTZ() argument
5971 unsigned NarrowSize = NarrowTy.getSizeInBits(); in narrowScalarCTTZ()
5977 auto UnmergeSrc = B.buildUnmerge(NarrowTy, SrcReg); in narrowScalarCTTZ()
5979 auto C_0 = B.buildConstant(NarrowTy, 0); in narrowScalarCTTZ()
5999 LLT NarrowTy) { in narrowScalarCTPOP() argument
6004 unsigned NarrowSize = NarrowTy.getSizeInBits(); in narrowScalarCTPOP()
6007 auto UnmergeSrc = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1)); in narrowScalarCTPOP()
6022 LLT NarrowTy) { in narrowScalarFLDEXP() argument
6030 unsigned ClampSize = NarrowTy.getScalarSizeInBits(); in narrowScalarFLDEXP()
6038 auto Trunc = B.buildTrunc(NarrowTy, Clamp); in narrowScalarFLDEXP()