Lines Matching refs:NarrowTy
51 getNarrowTypeBreakDown(LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy) { in getNarrowTypeBreakDown() argument
55 unsigned NarrowSize = NarrowTy.getSizeInBits(); in getNarrowTypeBreakDown()
63 if (NarrowTy.isVector()) { in getNarrowTypeBreakDown()
217 LLT NarrowTy = (NumElts == 1) ? EltTy : LLT::fixed_vector(NumElts, EltTy); in extractVectorParts() local
224 return extractParts(Reg, NarrowTy, NumNarrowTyPieces, VRegs); in extractVectorParts()
236 VRegs.push_back(MIRBuilder.buildMerge(NarrowTy, Pieces).getReg(0)); in extractVectorParts()
337 LLT NarrowTy, Register SrcReg) { in extractGCDType() argument
339 LLT GCDTy = getGCDType(getGCDType(SrcTy, NarrowTy), DstTy); in extractGCDType()
344 LLT LegalizerHelper::buildLCMMergePieces(LLT DstTy, LLT NarrowTy, LLT GCDTy, in buildLCMMergePieces() argument
347 LLT LCMTy = getLCMType(DstTy, NarrowTy); in buildLCMMergePieces()
349 int NumParts = LCMTy.getSizeInBits() / NarrowTy.getSizeInBits(); in buildLCMMergePieces()
350 int NumSubParts = NarrowTy.getSizeInBits() / GCDTy.getSizeInBits(); in buildLCMMergePieces()
406 AllPadReg = MIRBuilder.buildUndef(NarrowTy).getReg(0); in buildLCMMergePieces()
408 AllPadReg = MIRBuilder.buildConstant(NarrowTy, 0).getReg(0); in buildLCMMergePieces()
424 Remerge[I] = MIRBuilder.buildMerge(NarrowTy, SubMerge).getReg(0); in buildLCMMergePieces()
901 LLT NarrowTy) { in narrowScalar() argument
903 uint64_t NarrowSize = NarrowTy.getSizeInBits(); in narrowScalar()
919 LLT ImplicitTy = NarrowTy; in narrowScalar()
934 DstRegs.push_back(MIRBuilder.buildUndef(NarrowTy).getReg(0)); in narrowScalar()
947 unsigned NarrowSize = NarrowTy.getSizeInBits(); in narrowScalar()
953 auto K = MIRBuilder.buildConstant(NarrowTy, in narrowScalar()
970 Ty, NarrowTy, PartRegs, LeftoverTy, LeftoverRegs); in narrowScalar()
978 return narrowScalarExt(MI, TypeIdx, NarrowTy); in narrowScalar()
984 if (NarrowTy.getSizeInBits() * 2 != SizeOp1) { in narrowScalar()
985 LLVM_DEBUG(dbgs() << "Can't narrow trunc to type " << NarrowTy << "\n"); in narrowScalar()
989 auto Unmerge = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1)); in narrowScalar()
1001 if (Ty.getSizeInBits() % NarrowTy.getSizeInBits() != 0) in narrowScalar()
1004 auto Unmerge = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1).getReg()); in narrowScalar()
1008 MIRBuilder.buildFreeze(NarrowTy, Unmerge.getReg(i)).getReg(0)); in narrowScalar()
1025 return narrowScalarAddSub(MI, TypeIdx, NarrowTy); in narrowScalar()
1028 return narrowScalarMul(MI, NarrowTy); in narrowScalar()
1030 return narrowScalarExtract(MI, TypeIdx, NarrowTy); in narrowScalar()
1032 return narrowScalarInsert(MI, TypeIdx, NarrowTy); in narrowScalar()
1041 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy); in narrowScalar()
1048 return reduceLoadStoreWidth(LoadMI, TypeIdx, NarrowTy); in narrowScalar()
1056 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy); in narrowScalar()
1086 unsigned HandledSize = NumParts * NarrowTy.getSizeInBits(); in narrowScalar()
1092 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy); in narrowScalar()
1099 return reduceLoadStoreWidth(StoreMI, 0, NarrowTy); in narrowScalar()
1102 return narrowScalarSelect(MI, TypeIdx, NarrowTy); in narrowScalar()
1115 return narrowScalarBasic(MI, TypeIdx, NarrowTy); in narrowScalar()
1120 return narrowScalarShift(MI, TypeIdx, NarrowTy); in narrowScalar()
1130 return narrowScalarCTLZ(MI, TypeIdx, NarrowTy); in narrowScalar()
1133 return narrowScalarCTTZ(MI, TypeIdx, NarrowTy); in narrowScalar()
1135 return narrowScalarCTPOP(MI, TypeIdx, NarrowTy); in narrowScalar()
1141 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT); in narrowScalar()
1149 narrowScalarSrc(MI, NarrowTy, 1); in narrowScalar()
1157 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT); in narrowScalar()
1173 extractParts(MI.getOperand(i).getReg(), NarrowTy, NumParts, in narrowScalar()
1179 DstRegs[i] = MRI.createGenericVirtualRegister(NarrowTy); in narrowScalar()
1198 narrowScalarSrc(MI, NarrowTy, OpIdx); in narrowScalar()
1215 if (!extractParts(LHS, SrcTy, NarrowTy, LeftoverTy, LHSPartRegs, in narrowScalar()
1221 if (!extractParts(MI.getOperand(3).getReg(), SrcTy, NarrowTy, Unused, in narrowScalar()
1233 auto Zero = MIRBuilder.buildConstant(NarrowTy, 0); in narrowScalar()
1238 auto Xor = MIRBuilder.buildXor(NarrowTy, LHS, RHS).getReg(0); in narrowScalar()
1249 LLT GCDTy = extractGCDType(WidenedXors, NarrowTy, LeftoverTy, Xor); in narrowScalar()
1250 buildLCMMergePieces(LeftoverTy, NarrowTy, GCDTy, WidenedXors, in narrowScalar()
1259 auto Or = MIRBuilder.buildOr(NarrowTy, Xors[0], Xors[1]); in narrowScalar()
1261 Or = MIRBuilder.buildOr(NarrowTy, Or, Xors[I]); in narrowScalar()
1289 if (NarrowTy.getScalarSizeInBits() >= SizeInBits) { in narrowScalar()
1294 auto TruncMIB = MIRBuilder.buildTrunc(NarrowTy, MO1); in narrowScalar()
1298 Register DstExt = MRI.createGenericVirtualRegister(NarrowTy); in narrowScalar()
1321 Register SrcReg = MRI.createGenericVirtualRegister(NarrowTy); in narrowScalar()
1330 MIRBuilder.buildConstant(NarrowTy, NarrowTy.getScalarSizeInBits() - 1) in narrowScalar()
1337 if ((i + 1) * NarrowTy.getScalarSizeInBits() < SizeInBits) in narrowScalar()
1339 else if (i * NarrowTy.getScalarSizeInBits() > SizeInBits) { in narrowScalar()
1347 MIRBuilder.buildAShr(NarrowTy, PartialExtensionReg, AshrCstReg) in narrowScalar()
1354 TargetOpcode::G_SEXT_INREG, {NarrowTy}, in narrowScalar()
1355 {SrcRegs[i], SizeInBits % NarrowTy.getScalarSizeInBits()}) in narrowScalar()
1375 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs); in narrowScalar()
1378 auto DstPart = MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy}, in narrowScalar()
1394 narrowScalarSrc(MI, NarrowTy, 2); in narrowScalar()
1400 return narrowScalarFPTOI(MI, TypeIdx, NarrowTy); in narrowScalar()
1405 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_FPEXT); in narrowScalar()
1440 void LegalizerHelper::narrowScalarSrc(MachineInstr &MI, LLT NarrowTy, in narrowScalarSrc() argument
1443 auto ExtB = MIRBuilder.buildTrunc(NarrowTy, MO); in narrowScalarSrc()
1456 void LegalizerHelper::narrowScalarDst(MachineInstr &MI, LLT NarrowTy, in narrowScalarDst() argument
1459 Register DstTrunc = MRI.createGenericVirtualRegister(NarrowTy); in narrowScalarDst()
3644 LLT NarrowTy = (NumElts == 1) ? EltTy : LLT::fixed_vector(NumElts, EltTy); in makeDstOps() local
3647 getNarrowTypeBreakDown(Ty, NarrowTy, LeftoverTy); in makeDstOps()
3651 DstOps.push_back(NarrowTy); in makeDstOps()
3809 LLT NarrowTy) { in fewerElementsVectorUnmergeValues() argument
3815 if (TypeIdx != 1 || NarrowTy == DstTy) in fewerElementsVectorUnmergeValues()
3822 assert(SrcTy.isVector() && NarrowTy.isVector() && "Expected vector types"); in fewerElementsVectorUnmergeValues()
3823 assert((SrcTy.getScalarType() == NarrowTy.getScalarType()) && "bad type"); in fewerElementsVectorUnmergeValues()
3825 if ((SrcTy.getSizeInBits() % NarrowTy.getSizeInBits() != 0) || in fewerElementsVectorUnmergeValues()
3826 (NarrowTy.getSizeInBits() % DstTy.getSizeInBits() != 0)) in fewerElementsVectorUnmergeValues()
3839 auto Unmerge = MIRBuilder.buildUnmerge(NarrowTy, SrcReg); in fewerElementsVectorUnmergeValues()
3857 LLT NarrowTy) { in fewerElementsVectorMerge() argument
3864 assert(DstTy.isVector() && NarrowTy.isVector() && "Expected vector types"); in fewerElementsVectorMerge()
3865 assert((DstTy.getScalarType() == NarrowTy.getScalarType()) && "bad type"); in fewerElementsVectorMerge()
3866 if (NarrowTy == SrcTy) in fewerElementsVectorMerge()
3875 assert((SrcTy.getScalarType() == NarrowTy.getScalarType()) && "bad type"); in fewerElementsVectorMerge()
3876 if ((DstTy.getSizeInBits() % NarrowTy.getSizeInBits() != 0) || in fewerElementsVectorMerge()
3877 (NarrowTy.getNumElements() >= SrcTy.getNumElements())) in fewerElementsVectorMerge()
3897 unsigned NumNarrowTyElts = NarrowTy.getNumElements(); in fewerElementsVectorMerge()
3902 NarrowTyElts.push_back(MIRBuilder.buildMerge(NarrowTy, Pieces).getReg(0)); in fewerElementsVectorMerge()
3911 if ((NarrowTy.getSizeInBits() % SrcTy.getSizeInBits() != 0) || in fewerElementsVectorMerge()
3912 (DstTy.getSizeInBits() % NarrowTy.getSizeInBits() != 0)) in fewerElementsVectorMerge()
3926 unsigned NumParts = DstTy.getNumElements() / NarrowTy.getNumElements(); in fewerElementsVectorMerge()
3928 unsigned NumElts = NarrowTy.getNumElements() / NumSrcElts; in fewerElementsVectorMerge()
3933 NarrowTyElts.push_back(MIRBuilder.buildMerge(NarrowTy, Sources).getReg(0)); in fewerElementsVectorMerge()
4018 LLT NarrowTy) { in reduceLoadStoreWidth() argument
4044 std::tie(NumParts, NumLeftover) = getNarrowTypeBreakDown(ValTy, NarrowTy, LeftoverTy); in reduceLoadStoreWidth()
4046 if (extractParts(ValReg, ValTy, NarrowTy, LeftoverTy, NarrowRegs, in reduceLoadStoreWidth()
4094 unsigned Offset = isBigEndian ? TotalSize - NarrowTy.getSizeInBits() : 0; in reduceLoadStoreWidth()
4096 splitTypePieces(NarrowTy, NarrowRegs, NumParts, Offset); in reduceLoadStoreWidth()
4103 insertParts(ValReg, ValTy, NarrowTy, NarrowRegs, in reduceLoadStoreWidth()
4113 LLT NarrowTy) { in fewerElementsVector() argument
4116 unsigned NumElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1; in fewerElementsVector()
4228 return fewerElementsVectorUnmergeValues(MI, TypeIdx, NarrowTy); in fewerElementsVector()
4231 return fewerElementsVectorMerge(MI, TypeIdx, NarrowTy); in fewerElementsVector()
4235 return fewerElementsVectorMerge(MI, TypeIdx, NarrowTy); in fewerElementsVector()
4238 return fewerElementsVectorExtractInsertVectorElt(MI, TypeIdx, NarrowTy); in fewerElementsVector()
4241 return reduceLoadStoreWidth(cast<GLoadStore>(MI), TypeIdx, NarrowTy); in fewerElementsVector()
4245 return fewerElementsVectorReductions(MI, TypeIdx, NarrowTy); in fewerElementsVector()
4247 return fewerElementsVectorShuffle(MI, TypeIdx, NarrowTy); in fewerElementsVector()
4254 MachineInstr &MI, unsigned int TypeIdx, LLT NarrowTy) { in fewerElementsVectorShuffle() argument
4277 NarrowTy = in fewerElementsVectorShuffle()
4279 unsigned NewElts = NarrowTy.getNumElements(); in fewerElementsVectorShuffle()
4282 extractParts(Src1Reg, NarrowTy, 2, SplitSrc1Regs); in fewerElementsVectorShuffle()
4283 extractParts(Src2Reg, NarrowTy, 2, SplitSrc2Regs); in fewerElementsVectorShuffle()
4344 LLT EltTy = NarrowTy.getElementType(); in fewerElementsVectorShuffle()
4373 Output = MIRBuilder.buildBuildVector(NarrowTy, SVOps).getReg(0); in fewerElementsVectorShuffle()
4376 Output = MIRBuilder.buildUndef(NarrowTy).getReg(0); in fewerElementsVectorShuffle()
4381 ? MIRBuilder.buildUndef(NarrowTy).getReg(0) in fewerElementsVectorShuffle()
4384 Output = MIRBuilder.buildShuffleVector(NarrowTy, Op0, Op1, Ops).getReg(0); in fewerElementsVectorShuffle()
4444 MachineInstr &MI, unsigned int TypeIdx, LLT NarrowTy) { in fewerElementsVectorReductions() argument
4460 if (NarrowTy.isVector() && in fewerElementsVectorReductions()
4461 (SrcTy.getNumElements() % NarrowTy.getNumElements() != 0)) in fewerElementsVectorReductions()
4468 NarrowTy.isVector() ? SrcTy.getNumElements() / NarrowTy.getNumElements() in fewerElementsVectorReductions()
4471 extractParts(SrcReg, NarrowTy, NumParts, SplitSrcs); in fewerElementsVectorReductions()
4472 if (NarrowTy.isScalar()) { in fewerElementsVectorReductions()
4473 if (DstTy != NarrowTy) in fewerElementsVectorReductions()
4484 .buildInstr(ScalarOpc, {NarrowTy}, in fewerElementsVectorReductions()
4500 Acc = MIRBuilder.buildInstr(ScalarOpc, {NarrowTy}, {Acc, SplitSrcs[Idx]}) in fewerElementsVectorReductions()
4516 isPowerOf2_32(NarrowTy.getNumElements())) { in fewerElementsVectorReductions()
4517 return tryNarrowPow2Reduction(MI, SrcReg, SrcTy, NarrowTy, ScalarOpc); in fewerElementsVectorReductions()
4537 LLT SrcTy, LLT NarrowTy, in tryNarrowPow2Reduction() argument
4541 extractParts(SrcReg, NarrowTy, in tryNarrowPow2Reduction()
4542 SrcTy.getNumElements() / NarrowTy.getNumElements(), SplitSrcs); in tryNarrowPow2Reduction()
4552 MIRBuilder.buildInstr(ScalarOpc, {NarrowTy}, {LHS, RHS}).getReg(0); in tryNarrowPow2Reduction()
4971 LLT NarrowTy) { in multiplyRegisters() argument
4978 B.buildMul(NarrowTy, Src1Regs[DstIdx], Src2Regs[DstIdx]).getReg(0); in multiplyRegisters()
4989 B.buildMul(NarrowTy, Src1Regs[DstIdx - i], Src2Regs[i]); in multiplyRegisters()
4996 B.buildUMulH(NarrowTy, Src1Regs[DstIdx - 1 - i], Src2Regs[i]); in multiplyRegisters()
5008 B.buildUAddo(NarrowTy, LLT::scalar(1), Factors[0], Factors[1]); in multiplyRegisters()
5010 CarrySum = B.buildZExt(NarrowTy, Uaddo.getReg(1)).getReg(0); in multiplyRegisters()
5013 B.buildUAddo(NarrowTy, LLT::scalar(1), FactorSum, Factors[i]); in multiplyRegisters()
5015 MachineInstrBuilder Carry = B.buildZExt(NarrowTy, Uaddo.getReg(1)); in multiplyRegisters()
5016 CarrySum = B.buildAdd(NarrowTy, CarrySum, Carry).getReg(0); in multiplyRegisters()
5020 FactorSum = B.buildAdd(NarrowTy, Factors[0], Factors[1]).getReg(0); in multiplyRegisters()
5022 FactorSum = B.buildAdd(NarrowTy, FactorSum, Factors[i]).getReg(0); in multiplyRegisters()
5033 LLT NarrowTy) { in narrowScalarAddSub() argument
5085 extractParts(Src1, RegTy, NarrowTy, LeftoverTy, Src1Regs, Src1Left); in narrowScalarAddSub()
5086 extractParts(Src2, RegTy, NarrowTy, DummyTy, Src2Regs, Src2Left); in narrowScalarAddSub()
5117 insertParts(MI.getOperand(0).getReg(), RegTy, NarrowTy, in narrowScalarAddSub()
5126 LegalizerHelper::narrowScalarMul(MachineInstr &MI, LLT NarrowTy) { in narrowScalarMul() argument
5136 unsigned NarrowSize = NarrowTy.getSizeInBits(); in narrowScalarMul()
5146 extractParts(Src1, NarrowTy, NumParts, Src1Parts); in narrowScalarMul()
5147 extractParts(Src2, NarrowTy, NumParts, Src2Parts); in narrowScalarMul()
5148 multiplyRegisters(DstTmpRegs, Src1Parts, Src2Parts, NarrowTy); in narrowScalarMul()
5159 LLT NarrowTy) { in narrowScalarFPTOI() argument
5172 NarrowTy.getScalarSizeInBits() < (IsSigned ? 17u : 16u)) in narrowScalarFPTOI()
5176 narrowScalarDst(MI, NarrowTy, 0, in narrowScalarFPTOI()
5184 LLT NarrowTy) { in narrowScalarExtract() argument
5188 uint64_t NarrowSize = NarrowTy.getSizeInBits(); in narrowScalarExtract()
5199 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs); in narrowScalarExtract()
5210 } else if (SrcStart == OpStart && NarrowTy == MRI.getType(OpReg)) { in narrowScalarExtract()
5251 LLT NarrowTy) { in narrowScalarInsert() argument
5260 extractParts(MI.getOperand(1).getReg(), RegTy, NarrowTy, LeftoverTy, SrcRegs, in narrowScalarInsert()
5266 uint64_t NarrowSize = NarrowTy.getSizeInBits(); in narrowScalarInsert()
5273 if (DstStart == OpStart && NarrowTy == MRI.getType(OpReg)) { in narrowScalarInsert()
5283 SrcReg = MRI.createGenericVirtualRegister(NarrowTy); in narrowScalarInsert()
5315 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy); in narrowScalarInsert()
5335 LLT NarrowTy) { in narrowScalarBasic() argument
5345 if (!extractParts(MI.getOperand(1).getReg(), DstTy, NarrowTy, LeftoverTy, in narrowScalarBasic()
5350 if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, Unused, in narrowScalarBasic()
5355 auto Inst = MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy}, in narrowScalarBasic()
5367 insertParts(DstReg, DstTy, NarrowTy, DstRegs, in narrowScalarBasic()
5376 LLT NarrowTy) { in narrowScalarExt() argument
5388 LLT GCDTy = extractGCDType(Parts, DstTy, NarrowTy, SrcReg); in narrowScalarExt()
5389 LLT LCMTy = buildLCMMergePieces(DstTy, NarrowTy, GCDTy, Parts, MI.getOpcode()); in narrowScalarExt()
5398 LLT NarrowTy) { in narrowScalarSelect() argument
5414 if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, LeftoverTy, in narrowScalarSelect()
5419 if (!extractParts(MI.getOperand(3).getReg(), DstTy, NarrowTy, Unused, in narrowScalarSelect()
5424 auto Select = MIRBuilder.buildSelect(NarrowTy, in narrowScalarSelect()
5435 insertParts(DstReg, DstTy, NarrowTy, DstRegs, in narrowScalarSelect()
5444 LLT NarrowTy) { in narrowScalarCTLZ() argument
5452 unsigned NarrowSize = NarrowTy.getSizeInBits(); in narrowScalarCTLZ()
5458 auto UnmergeSrc = B.buildUnmerge(NarrowTy, SrcReg); in narrowScalarCTLZ()
5460 auto C_0 = B.buildConstant(NarrowTy, 0); in narrowScalarCTLZ()
5480 LLT NarrowTy) { in narrowScalarCTTZ() argument
5488 unsigned NarrowSize = NarrowTy.getSizeInBits(); in narrowScalarCTTZ()
5494 auto UnmergeSrc = B.buildUnmerge(NarrowTy, SrcReg); in narrowScalarCTTZ()
5496 auto C_0 = B.buildConstant(NarrowTy, 0); in narrowScalarCTTZ()
5516 LLT NarrowTy) { in narrowScalarCTPOP() argument
5523 unsigned NarrowSize = NarrowTy.getSizeInBits(); in narrowScalarCTPOP()
5526 auto UnmergeSrc = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1)); in narrowScalarCTPOP()