Lines Matching refs:ContainerVT
214 MVT ContainerVT = getContainerForFixedLengthVector(VT); in RISCVTargetLowering() local
215 unsigned RCID = getRegClassIDForVecVT(ContainerVT); in RISCVTargetLowering()
2596 static SDValue getVLOp(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL, in getVLOp() argument
2602 RISCVTargetLowering::computeVLMAXBounds(ContainerVT, Subtarget); in getVLOp()
2619 getDefaultVLOps(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL, in getDefaultVLOps() argument
2621 assert(ContainerVT.isScalableVector() && "Expecting scalable container type"); in getDefaultVLOps()
2622 SDValue VL = getVLOp(NumElts, ContainerVT, DL, DAG, Subtarget); in getDefaultVLOps()
2623 SDValue Mask = getAllOnesMask(ContainerVT, VL, DL, DAG); in getDefaultVLOps()
2632 getDefaultVLOps(MVT VecVT, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG, in getDefaultVLOps() argument
2635 return getDefaultVLOps(VecVT.getVectorNumElements(), ContainerVT, DL, DAG, in getDefaultVLOps()
2637 assert(ContainerVT.isScalableVector() && "Expecting scalable container type"); in getDefaultVLOps()
2638 return getDefaultScalableVLOps(ContainerVT, DL, DAG, Subtarget); in getDefaultVLOps()
2877 MVT ContainerVT = VT; in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND() local
2879 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2880 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget); in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2887 Mask = convertToScalableVector(getMaskTypeFor(ContainerVT), Mask, DAG, in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2891 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2898 SDValue Abs = DAG.getNode(RISCVISD::FABS_VL, DL, ContainerVT, Src, Mask, VL); in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2903 const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(ContainerVT); in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2909 DAG.getConstantFP(MaxVal, DL, ContainerVT.getVectorElementType()); in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2910 SDValue MaxValSplat = DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, ContainerVT, in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2911 DAG.getUNDEF(ContainerVT), MaxValNode, VL); in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2914 MVT SetccVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount()); in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2921 MVT IntVT = ContainerVT.changeVectorElementTypeToInteger(); in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2953 Truncated = DAG.getNode(RISCVISD::VFROUND_NOEXCEPT_VL, DL, ContainerVT, Src, in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2960 Truncated = DAG.getNode(RISCVISD::SINT_TO_FP_VL, DL, ContainerVT, Truncated, in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2964 Truncated = DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Truncated, in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2984 MVT ContainerVT = VT; in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND() local
2986 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
2987 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget); in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
2990 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3003 DAG.getVTList(ContainerVT, MVT::Other), in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3004 {Chain, Src, Src, DAG.getUNDEF(ContainerVT), Unorder, VL}); in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3008 SDValue Abs = DAG.getNode(RISCVISD::FABS_VL, DL, ContainerVT, Src, Mask, VL); in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3013 const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(ContainerVT); in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3019 DAG.getConstantFP(MaxVal, DL, ContainerVT.getVectorElementType()); in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3020 SDValue MaxValSplat = DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, ContainerVT, in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3021 DAG.getUNDEF(ContainerVT), MaxValNode, VL); in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3029 MVT IntVT = ContainerVT.changeVectorElementTypeToInteger(); in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3054 DAG.getVTList(ContainerVT, MVT::Other), Chain, Src, in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3063 DAG.getVTList(ContainerVT, MVT::Other), Chain, in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3069 Truncated = DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Truncated, in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3113 MVT ContainerVT = VT; in lowerVectorXRINT() local
3116 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerVectorXRINT()
3117 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget); in lowerVectorXRINT()
3120 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerVectorXRINT()
3122 DAG.getNode(RISCVISD::VFCVT_X_F_VL, DL, ContainerVT, Src, Mask, VL); in lowerVectorXRINT()
3319 MVT ContainerVT = VT; in matchSplatAsGather() local
3321 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in matchSplatAsGather()
3322 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in matchSplatAsGather()
3325 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in matchSplatAsGather()
3327 SDValue Gather = DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT, Vec, in matchSplatAsGather()
3328 Idx, DAG.getUNDEF(ContainerVT), Mask, VL); in matchSplatAsGather()
3350 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerBuildVectorViaDominantValues() local
3353 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerBuildVectorViaDominantValues()
3413 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in lowerBuildVectorViaDominantValues()
3418 Vec = DAG.getNode(OpCode, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Vec, in lowerBuildVectorViaDominantValues()
3457 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerBuildVectorOfConstants() local
3460 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerBuildVectorOfConstants()
3467 SDValue VMClr = DAG.getNode(RISCVISD::VMCLR_VL, DL, ContainerVT, VL); in lowerBuildVectorOfConstants()
3472 SDValue VMSet = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL); in lowerBuildVectorOfConstants()
3548 DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Splat, VL); in lowerBuildVectorOfConstants()
3737 Source = convertToScalableVector(ContainerVT.changeVectorElementType(MVT::i8), in lowerBuildVectorOfConstants()
3739 SDValue Res = DAG.getNode(RISCVISD::VSEXT_VL, DL, ContainerVT, Source, Mask, VL); in lowerBuildVectorOfConstants()
3762 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerBUILD_VECTOR() local
3765 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerBUILD_VECTOR()
3801 DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Splat, VL); in lowerBUILD_VECTOR()
3815 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerBUILD_VECTOR() local
3826 SDValue Vec = DAG.getUNDEF(ContainerVT); in lowerBUILD_VECTOR()
3833 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT, Vec, SubBV, in lowerBUILD_VECTOR()
3850 switch (RISCVTargetLowering::getLMUL(ContainerVT)) { in lowerBUILD_VECTOR()
3903 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in lowerBUILD_VECTOR()
3910 Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT), in lowerBUILD_VECTOR()
3918 Vec = DAG.getNode(OpCode, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Vec, in lowerBUILD_VECTOR()
3923 Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT), in lowerBUILD_VECTOR()
4096 static bool isDeinterleaveShuffle(MVT VT, MVT ContainerVT, SDValue V1, in isDeinterleaveShuffle() argument
4256 MVT ContainerVT = VT; in getDeinterleaveViaVNSRL() local
4258 if (ContainerVT.isFixedLengthVector()) { in getDeinterleaveViaVNSRL()
4260 ContainerVT = getContainerForFixedLengthVector(DAG, ContainerVT, Subtarget); in getDeinterleaveViaVNSRL()
4264 MVT::getVectorVT(ContainerVT.getVectorElementType(), in getDeinterleaveViaVNSRL()
4265 ContainerVT.getVectorElementCount() * 2); in getDeinterleaveViaVNSRL()
4269 auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in getDeinterleaveViaVNSRL()
4273 unsigned EltBits = ContainerVT.getScalarSizeInBits(); in getDeinterleaveViaVNSRL()
4275 MVT::getIntegerVT(EltBits * 2), ContainerVT.getVectorElementCount()); in getDeinterleaveViaVNSRL()
4279 MVT IntContainerVT = ContainerVT.changeVectorElementTypeToInteger(); in getDeinterleaveViaVNSRL()
4285 RISCVISD::VMV_V_X_VL, DL, IntContainerVT, DAG.getUNDEF(ContainerVT), in getDeinterleaveViaVNSRL()
4291 Res = DAG.getBitcast(ContainerVT, Res); in getDeinterleaveViaVNSRL()
4363 MVT ContainerVT = getContainerForFixedLengthVector(DAG, SrcVT, Subtarget); in lowerVECTOR_SHUFFLEAsVSlidedown() local
4364 auto [TrueMask, VL] = getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget); in lowerVECTOR_SHUFFLEAsVSlidedown()
4366 getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT), in lowerVECTOR_SHUFFLEAsVSlidedown()
4367 convertToScalableVector(ContainerVT, Src, DAG, Subtarget), in lowerVECTOR_SHUFFLEAsVSlidedown()
4403 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerVECTOR_SHUFFLEAsVSlideup() local
4404 auto TrueMask = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).first; in lowerVECTOR_SHUFFLEAsVSlideup()
4413 InPlace = convertToScalableVector(ContainerVT, InPlace, DAG, Subtarget); in lowerVECTOR_SHUFFLEAsVSlideup()
4414 ToInsert = convertToScalableVector(ContainerVT, ToInsert, DAG, Subtarget); in lowerVECTOR_SHUFFLEAsVSlideup()
4421 Res = DAG.getNode(RISCVISD::VMV_V_V_VL, DL, ContainerVT, InPlace, ToInsert, in lowerVECTOR_SHUFFLEAsVSlideup()
4424 Res = getVSlideup(DAG, Subtarget, DL, ContainerVT, InPlace, ToInsert, in lowerVECTOR_SHUFFLEAsVSlideup()
4468 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerVECTOR_SHUFFLEAsVSlide1() local
4469 auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerVECTOR_SHUFFLEAsVSlide1()
4475 auto Vec = DAG.getNode(OpCode, DL, ContainerVT, in lowerVECTOR_SHUFFLEAsVSlide1()
4476 DAG.getUNDEF(ContainerVT), in lowerVECTOR_SHUFFLEAsVSlide1()
4477 convertToScalableVector(ContainerVT, V2, DAG, Subtarget), in lowerVECTOR_SHUFFLEAsVSlide1()
4707 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerShuffleViaVRegSplitting() local
4712 SDValue Vec = DAG.getUNDEF(ContainerVT); in lowerShuffleViaVRegSplitting()
4716 V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget); in lowerShuffleViaVRegSplitting()
4717 V2 = convertToScalableVector(ContainerVT, V2, DAG, Subtarget); in lowerShuffleViaVRegSplitting()
4730 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT, Vec, SubVec, in lowerShuffleViaVRegSplitting()
4764 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerVECTOR_SHUFFLE() local
4766 auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
4795 SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other}); in lowerVECTOR_SHUFFLE()
4800 DAG.getUNDEF(ContainerVT), in lowerVECTOR_SHUFFLE()
4830 DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT), V, VL); in lowerVECTOR_SHUFFLE()
4834 V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
4836 SDValue Gather = DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT, in lowerVECTOR_SHUFFLE()
4838 DAG.getUNDEF(ContainerVT), TrueMask, VL); in lowerVECTOR_SHUFFLE()
4872 LoV = convertToScalableVector(ContainerVT, LoV, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
4876 HiV = convertToScalableVector(ContainerVT, HiV, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
4883 SDValue Res = DAG.getUNDEF(ContainerVT); in lowerVECTOR_SHUFFLE()
4887 Res = getVSlidedown(DAG, Subtarget, DL, ContainerVT, Res, HiV, in lowerVECTOR_SHUFFLE()
4891 Res = getVSlideup(DAG, Subtarget, DL, ContainerVT, Res, LoV, in lowerVECTOR_SHUFFLE()
4900 if (isDeinterleaveShuffle(VT, ContainerVT, V1, V2, Mask, Subtarget)) { in lowerVECTOR_SHUFFLE()
5036 ContainerVT.changeVectorElementType(IndexVT.getScalarType()); in lowerVECTOR_SHUFFLE()
5042 Gather = lowerScalarSplat(SDValue(), SplatValue, VL, ContainerVT, DL, DAG, in lowerVECTOR_SHUFFLE()
5045 V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
5051 Gather = DAG.getNode(GatherVXOpc, DL, ContainerVT, V1, in lowerVECTOR_SHUFFLE()
5053 DAG.getUNDEF(ContainerVT), TrueMask, VL); in lowerVECTOR_SHUFFLE()
5059 Gather = DAG.getNode(GatherVVOpc, DL, ContainerVT, V1, LHSIndices, in lowerVECTOR_SHUFFLE()
5060 DAG.getUNDEF(ContainerVT), TrueMask, VL); in lowerVECTOR_SHUFFLE()
5067 V2 = convertToScalableVector(ContainerVT, V2, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
5069 MVT MaskContainerVT = ContainerVT.changeVectorElementType(MVT::i1); in lowerVECTOR_SHUFFLE()
5078 Gather = DAG.getNode(GatherVXOpc, DL, ContainerVT, V2, in lowerVECTOR_SHUFFLE()
5085 Gather = DAG.getNode(GatherVVOpc, DL, ContainerVT, V2, RHSIndices, Gather, in lowerVECTOR_SHUFFLE()
5122 MVT ContainerVT = VT; in lowerCTLZ_CTTZ_ZERO_UNDEF() local
5128 Mask = convertToScalableVector(getMaskTypeFor(ContainerVT), Mask, DAG, in lowerCTLZ_CTTZ_ZERO_UNDEF()
5168 ContainerVT = getContainerForFixedLengthVector(VT); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5169 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5172 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5176 MVT::getVectorVT(FloatEltVT, ContainerVT.getVectorElementCount()); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5417 MVT ContainerVT = getContainerForFixedLengthVector(VT); in LowerIS_FPCLASS() local
5436 DAG.getNode(RISCVISD::SETCC_VL, DL, ContainerVT, in LowerIS_FPCLASS()
5438 DAG.getUNDEF(ContainerVT), Mask, VL}); in LowerIS_FPCLASS()
5448 SDValue VMSNE = DAG.getNode(RISCVISD::SETCC_VL, DL, ContainerVT, in LowerIS_FPCLASS()
5450 DAG.getUNDEF(ContainerVT), Mask, VL}); in LowerIS_FPCLASS()
5500 MVT ContainerVT = VT; in lowerFMAXIMUM_FMINIMUM() local
5502 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget); in lowerFMAXIMUM_FMINIMUM()
5503 X = convertToScalableVector(ContainerVT, X, DAG, Subtarget); in lowerFMAXIMUM_FMINIMUM()
5504 Y = convertToScalableVector(ContainerVT, Y, DAG, Subtarget); in lowerFMAXIMUM_FMINIMUM()
5511 Mask = convertToScalableVector(getMaskTypeFor(ContainerVT), Mask, DAG, in lowerFMAXIMUM_FMINIMUM()
5515 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerFMAXIMUM_FMINIMUM()
5522 DAG.getUNDEF(ContainerVT), Mask, VL}); in lowerFMAXIMUM_FMINIMUM()
5523 NewY = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, XIsNonNan, Y, X, in lowerFMAXIMUM_FMINIMUM()
5524 DAG.getUNDEF(ContainerVT), VL); in lowerFMAXIMUM_FMINIMUM()
5531 DAG.getUNDEF(ContainerVT), Mask, VL}); in lowerFMAXIMUM_FMINIMUM()
5532 NewX = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, YIsNonNan, X, Y, in lowerFMAXIMUM_FMINIMUM()
5533 DAG.getUNDEF(ContainerVT), VL); in lowerFMAXIMUM_FMINIMUM()
5540 SDValue Res = DAG.getNode(Opc, DL, ContainerVT, NewX, NewY, in lowerFMAXIMUM_FMINIMUM()
5541 DAG.getUNDEF(ContainerVT), Mask, VL); in lowerFMAXIMUM_FMINIMUM()
6011 MVT ContainerVT = VT; in LowerOperation() local
6013 ContainerVT = getContainerForFixedLengthVector(VT); in LowerOperation()
6014 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in LowerOperation()
6016 SDValue V = DAG.getNode(RISCVISD::VMV_S_X_VL, DL, ContainerVT, in LowerOperation()
6017 DAG.getUNDEF(ContainerVT), Scalar, VL); in LowerOperation()
6273 MVT ContainerVT = getContainerForFixedLengthVector(VT); in LowerOperation() local
6275 assert(ContainerVT.getVectorElementCount() == SrcContainerVT.getVectorElementCount() && in LowerOperation()
6278 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in LowerOperation()
6282 Src = DAG.getNode(RVVOpc, DL, DAG.getVTList(ContainerVT, MVT::Other), in LowerOperation()
6287 Src = DAG.getNode(RVVOpc, DL, ContainerVT, Src, Mask, VL); in LowerOperation()
6400 MVT ContainerVT = getContainerForFixedLengthVector(Op.getSimpleValueType()); in LowerOperation() local
6402 DAG.getUNDEF(ContainerVT), DAG, Subtarget); in LowerOperation()
7557 MVT ContainerVT = VecVT; in lowerSPLAT_VECTOR_PARTS() local
7559 ContainerVT = getContainerForFixedLengthVector(VecVT); in lowerSPLAT_VECTOR_PARTS()
7561 auto VL = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).second; in lowerSPLAT_VECTOR_PARTS()
7564 splatPartsI64WithVL(DL, ContainerVT, SDValue(), Lo, Hi, VL, DAG); in lowerSPLAT_VECTOR_PARTS()
7591 MVT ContainerVT = getContainerForFixedLengthVector(VecVT); in lowerVectorMaskExt() local
7593 MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount()); in lowerVectorMaskExt()
7597 SDValue VL = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).second; in lowerVectorMaskExt()
7603 SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, in lowerVectorMaskExt()
7604 DAG.getUNDEF(ContainerVT), SplatZero, VL); in lowerVectorMaskExt()
7605 SplatTrueVal = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, in lowerVectorMaskExt()
7606 DAG.getUNDEF(ContainerVT), SplatTrueVal, VL); in lowerVectorMaskExt()
7608 DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, CC, SplatTrueVal, in lowerVectorMaskExt()
7609 SplatZero, DAG.getUNDEF(ContainerVT), VL); in lowerVectorMaskExt()
7627 MVT ContainerVT = MVT::getVectorVT(VT.getVectorElementType(), in lowerFixedLengthVectorExtendToRVV() local
7631 convertToScalableVector(ContainerVT, Op.getOperand(0), DAG, Subtarget); in lowerFixedLengthVectorExtendToRVV()
7634 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerFixedLengthVectorExtendToRVV()
7660 MVT ContainerVT = VecVT; in lowerVectorMaskTruncLike() local
7663 ContainerVT = getContainerForFixedLengthVector(VecVT); in lowerVectorMaskTruncLike()
7664 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget); in lowerVectorMaskTruncLike()
7674 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget); in lowerVectorMaskTruncLike()
7680 SplatOne = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, in lowerVectorMaskTruncLike()
7681 DAG.getUNDEF(ContainerVT), SplatOne, VL); in lowerVectorMaskTruncLike()
7682 SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, in lowerVectorMaskTruncLike()
7683 DAG.getUNDEF(ContainerVT), SplatZero, VL); in lowerVectorMaskTruncLike()
7685 MVT MaskContainerVT = ContainerVT.changeVectorElementType(MVT::i1); in lowerVectorMaskTruncLike()
7686 SDValue Trunc = DAG.getNode(RISCVISD::AND_VL, DL, ContainerVT, Src, SplatOne, in lowerVectorMaskTruncLike()
7687 DAG.getUNDEF(ContainerVT), Mask, VL); in lowerVectorMaskTruncLike()
7722 MVT ContainerVT = SrcVT; in lowerVectorTruncLike() local
7729 ContainerVT = getContainerForFixedLengthVector(SrcVT); in lowerVectorTruncLike()
7730 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget); in lowerVectorTruncLike()
7732 MVT MaskVT = getMaskTypeFor(ContainerVT); in lowerVectorTruncLike()
7740 getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget); in lowerVectorTruncLike()
7744 const ElementCount Count = ContainerVT.getVectorElementCount(); in lowerVectorTruncLike()
7766 MVT ContainerVT = VT; in lowerStrictFPExtendOrRoundLike() local
7769 ContainerVT = in lowerStrictFPExtendOrRoundLike()
7774 auto [Mask, VL] = getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget); in lowerStrictFPExtendOrRoundLike()
7785 MVT InterVT = ContainerVT.changeVectorElementType(MVT::f32); in lowerStrictFPExtendOrRoundLike()
7794 SDValue Res = DAG.getNode(ConvOpc, DL, DAG.getVTList(ContainerVT, MVT::Other), in lowerStrictFPExtendOrRoundLike()
7831 MVT ContainerVT = VT; in lowerVectorFPExtendOrRoundLike() local
7839 ContainerVT = in lowerVectorFPExtendOrRoundLike()
7843 MVT MaskVT = getMaskTypeFor(ContainerVT); in lowerVectorFPExtendOrRoundLike()
7850 getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget); in lowerVectorFPExtendOrRoundLike()
7855 Src = DAG.getNode(ConvOpc, DL, ContainerVT, Src, Mask, VL); in lowerVectorFPExtendOrRoundLike()
7864 MVT InterVT = ContainerVT.changeVectorElementType(MVT::f32); in lowerVectorFPExtendOrRoundLike()
7868 DAG.getNode(ConvOpc, DL, ContainerVT, IntermediateConv, Mask, VL); in lowerVectorFPExtendOrRoundLike()
7923 MVT ContainerVT = VecVT; in lowerINSERT_VECTOR_ELT() local
7926 ContainerVT = getContainerForFixedLengthVector(VecVT); in lowerINSERT_VECTOR_ELT()
7927 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in lowerINSERT_VECTOR_ELT()
7932 MVT OrigContainerVT = ContainerVT; in lowerINSERT_VECTOR_ELT()
7938 if (auto ShrunkVT = getSmallestVTForIndex(ContainerVT, OrigIdx, in lowerINSERT_VECTOR_ELT()
7940 ContainerVT = *ShrunkVT; in lowerINSERT_VECTOR_ELT()
7949 const MVT M1VT = getLMUL1VT(ContainerVT); in lowerINSERT_VECTOR_ELT()
7950 if (MinVLen == MaxVLen && ContainerVT.bitsGT(M1VT)) { in lowerINSERT_VECTOR_ELT()
7959 ContainerVT = M1VT; in lowerINSERT_VECTOR_ELT()
7963 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ContainerVT, Vec, in lowerINSERT_VECTOR_ELT()
7982 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget); in lowerINSERT_VECTOR_ELT()
7992 Vec = DAG.getNode(Opc, DL, ContainerVT, Vec, Val, VL); in lowerINSERT_VECTOR_ELT()
8001 ValInVec = lowerScalarInsert(Val, VL, ContainerVT, DL, DAG, Subtarget); in lowerINSERT_VECTOR_ELT()
8010 MVT::getVectorVT(MVT::i32, ContainerVT.getVectorElementCount() * 2); in lowerINSERT_VECTOR_ELT()
8027 ValInVec = DAG.getBitcast(ContainerVT, ValInVec); in lowerINSERT_VECTOR_ELT()
8048 ValInVec = DAG.getBitcast(ContainerVT, ValInVec); in lowerINSERT_VECTOR_ELT()
8060 SDValue Slideup = getVSlideup(DAG, Subtarget, DL, ContainerVT, Vec, ValInVec, in lowerINSERT_VECTOR_ELT()
8087 MVT ContainerVT = VecVT; in lowerEXTRACT_VECTOR_ELT() local
8089 ContainerVT = getContainerForFixedLengthVector(VecVT); in lowerEXTRACT_VECTOR_ELT()
8090 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in lowerEXTRACT_VECTOR_ELT()
8092 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget); in lowerEXTRACT_VECTOR_ELT()
8147 MVT ContainerVT = VecVT; in lowerEXTRACT_VECTOR_ELT() local
8149 ContainerVT = getContainerForFixedLengthVector(VecVT); in lowerEXTRACT_VECTOR_ELT()
8150 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in lowerEXTRACT_VECTOR_ELT()
8162 MVT M1VT = getLMUL1VT(ContainerVT); in lowerEXTRACT_VECTOR_ELT()
8173 ContainerVT = M1VT; in lowerEXTRACT_VECTOR_ELT()
8185 getSmallestVTForIndex(ContainerVT, *MaxIdx, DL, DAG, Subtarget)) { in lowerEXTRACT_VECTOR_ELT()
8186 ContainerVT = *SmallerVT; in lowerEXTRACT_VECTOR_ELT()
8187 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ContainerVT, Vec, in lowerEXTRACT_VECTOR_ELT()
8204 MVT LMUL2VT = getLMUL1VT(ContainerVT).getDoubleNumVectorElementsVT(); in lowerEXTRACT_VECTOR_ELT()
8205 if (ContainerVT.bitsGT(LMUL2VT) && VecVT.isFixedLengthVector()) in lowerEXTRACT_VECTOR_ELT()
8211 auto [Mask, VL] = getDefaultVLOps(1, ContainerVT, DL, DAG, Subtarget); in lowerEXTRACT_VECTOR_ELT()
8212 Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT, in lowerEXTRACT_VECTOR_ELT()
8213 DAG.getUNDEF(ContainerVT), Vec, Idx, Mask, VL); in lowerEXTRACT_VECTOR_ELT()
8746 MVT ContainerVT = VT; in LowerINTRINSIC_W_CHAIN() local
8748 ContainerVT = getContainerForFixedLengthVector(VT); in LowerINTRINSIC_W_CHAIN()
8752 MVT MaskVT = getMaskTypeFor(ContainerVT); in LowerINTRINSIC_W_CHAIN()
8755 PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget); in LowerINTRINSIC_W_CHAIN()
8760 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in LowerINTRINSIC_W_CHAIN()
8768 MVT ScalarVT = ContainerVT.getVectorElementType(); in LowerINTRINSIC_W_CHAIN()
8773 Result = lowerScalarSplat(SDValue(), ScalarLoad, VL, ContainerVT, DL, DAG, in LowerINTRINSIC_W_CHAIN()
8782 Ops.push_back(DAG.getUNDEF(ContainerVT)); in LowerINTRINSIC_W_CHAIN()
8796 SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other}); in LowerINTRINSIC_W_CHAIN()
8823 MVT ContainerVT = getContainerForFixedLengthVector(VT); in LowerINTRINSIC_W_CHAIN() local
8825 SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG, in LowerINTRINSIC_W_CHAIN()
8829 SmallVector<EVT, 9> ContainerVTs(NF, ContainerVT); in LowerINTRINSIC_W_CHAIN()
8833 Ops.insert(Ops.end(), NF, DAG.getUNDEF(ContainerVT)); in LowerINTRINSIC_W_CHAIN()
8911 MVT ContainerVT = VT; in LowerINTRINSIC_VOID() local
8913 ContainerVT = getContainerForFixedLengthVector(VT); in LowerINTRINSIC_VOID()
8914 Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget); in LowerINTRINSIC_VOID()
8917 MVT MaskVT = getMaskTypeFor(ContainerVT); in LowerINTRINSIC_VOID()
8922 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in LowerINTRINSIC_VOID()
8959 MVT ContainerVT = getContainerForFixedLengthVector(VT); in LowerINTRINSIC_VOID() local
8961 SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG, in LowerINTRINSIC_VOID()
8970 ContainerVT, FixedIntrinsic->getOperand(2 + i), DAG, Subtarget)); in LowerINTRINSIC_VOID()
9105 MVT ContainerVT = VecVT; in lowerVectorMaskVecReduction() local
9107 ContainerVT = getContainerForFixedLengthVector(VecVT); in lowerVectorMaskVecReduction()
9108 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in lowerVectorMaskVecReduction()
9117 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget); in lowerVectorMaskVecReduction()
9130 SDValue TrueMask = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL); in lowerVectorMaskVecReduction()
9131 Vec = DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Vec, TrueMask, VL); in lowerVectorMaskVecReduction()
9236 MVT ContainerVT = VecVT; in lowerVECREDUCE() local
9238 ContainerVT = getContainerForFixedLengthVector(VecVT); in lowerVECREDUCE()
9239 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in lowerVECREDUCE()
9242 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget); in lowerVECREDUCE()
9306 MVT ContainerVT = VecVT; in lowerFPVECREDUCE() local
9308 ContainerVT = getContainerForFixedLengthVector(VecVT); in lowerFPVECREDUCE()
9309 VectorVal = convertToScalableVector(ContainerVT, VectorVal, DAG, Subtarget); in lowerFPVECREDUCE()
9312 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget); in lowerFPVECREDUCE()
9332 auto ContainerVT = getContainerForFixedLengthVector(VecVT); in lowerVPREDUCE() local
9333 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in lowerVPREDUCE()
9399 MVT ContainerVT = VecVT; in lowerINSERT_SUBVECTOR() local
9401 ContainerVT = getContainerForFixedLengthVector(VecVT); in lowerINSERT_SUBVECTOR()
9402 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in lowerINSERT_SUBVECTOR()
9406 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT, in lowerINSERT_SUBVECTOR()
9407 DAG.getUNDEF(ContainerVT), SubVec, in lowerINSERT_SUBVECTOR()
9413 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT, in lowerINSERT_SUBVECTOR()
9414 DAG.getUNDEF(ContainerVT), SubVec, in lowerINSERT_SUBVECTOR()
9417 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first; in lowerINSERT_SUBVECTOR()
9421 SDValue VL = getVLOp(EndIndex, ContainerVT, DL, DAG, Subtarget); in lowerINSERT_SUBVECTOR()
9432 DAG.getNode(RISCVISD::VMV_V_V_VL, DL, ContainerVT, Vec, SubVec, VL); in lowerINSERT_SUBVECTOR()
9435 SubVec = getVSlideup(DAG, Subtarget, DL, ContainerVT, Vec, SubVec, in lowerINSERT_SUBVECTOR()
9579 MVT ContainerVT = VecVT; in lowerEXTRACT_SUBVECTOR() local
9581 ContainerVT = getContainerForFixedLengthVector(VecVT); in lowerEXTRACT_SUBVECTOR()
9582 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in lowerEXTRACT_SUBVECTOR()
9588 getSmallestVTForIndex(ContainerVT, LastIdx, DL, DAG, Subtarget)) { in lowerEXTRACT_SUBVECTOR()
9589 ContainerVT = *ShrunkVT; in lowerEXTRACT_SUBVECTOR()
9590 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ContainerVT, Vec, in lowerEXTRACT_SUBVECTOR()
9595 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first; in lowerEXTRACT_SUBVECTOR()
9598 SDValue VL = getVLOp(SubVecVT.getVectorNumElements(), ContainerVT, DL, DAG, in lowerEXTRACT_SUBVECTOR()
9602 getVSlidedown(DAG, Subtarget, DL, ContainerVT, in lowerEXTRACT_SUBVECTOR()
9603 DAG.getUNDEF(ContainerVT), Vec, SlidedownAmt, Mask, VL); in lowerEXTRACT_SUBVECTOR()
10004 MVT ContainerVT = getContainerForFixedLengthVector(VT); in lowerFixedLengthVectorLoadToRVV() local
10009 RISCVTargetLowering::computeVLMAXBounds(ContainerVT, Subtarget); in lowerFixedLengthVectorLoadToRVV()
10011 getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) { in lowerFixedLengthVectorLoadToRVV()
10013 DAG.getLoad(ContainerVT, DL, Load->getChain(), Load->getBasePtr(), in lowerFixedLengthVectorLoadToRVV()
10019 SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG, Subtarget); in lowerFixedLengthVectorLoadToRVV()
10026 Ops.push_back(DAG.getUNDEF(ContainerVT)); in lowerFixedLengthVectorLoadToRVV()
10029 SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other}); in lowerFixedLengthVectorLoadToRVV()
10061 MVT ContainerVT = getContainerForFixedLengthVector(VT); in lowerFixedLengthVectorStoreToRVV() local
10064 convertToScalableVector(ContainerVT, StoreVal, DAG, Subtarget); in lowerFixedLengthVectorStoreToRVV()
10070 RISCVTargetLowering::computeVLMAXBounds(ContainerVT, Subtarget); in lowerFixedLengthVectorStoreToRVV()
10072 getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) in lowerFixedLengthVectorStoreToRVV()
10076 SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG, in lowerFixedLengthVectorStoreToRVV()
10114 MVT ContainerVT = VT; in lowerMaskedLoad() local
10116 ContainerVT = getContainerForFixedLengthVector(VT); in lowerMaskedLoad()
10117 PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget); in lowerMaskedLoad()
10119 MVT MaskVT = getMaskTypeFor(ContainerVT); in lowerMaskedLoad()
10125 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in lowerMaskedLoad()
10131 Ops.push_back(DAG.getUNDEF(ContainerVT)); in lowerMaskedLoad()
10141 SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other}); in lowerMaskedLoad()
10179 MVT ContainerVT = VT; in lowerMaskedStore() local
10181 ContainerVT = getContainerForFixedLengthVector(VT); in lowerMaskedStore()
10183 Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget); in lowerMaskedStore()
10185 MVT MaskVT = getMaskTypeFor(ContainerVT); in lowerMaskedStore()
10191 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in lowerMaskedStore()
10210 MVT ContainerVT = getContainerForFixedLengthVector(InVT); in lowerFixedLengthVectorSetccToRVV() local
10215 convertToScalableVector(ContainerVT, Op.getOperand(0), DAG, Subtarget); in lowerFixedLengthVectorSetccToRVV()
10217 convertToScalableVector(ContainerVT, Op.getOperand(1), DAG, Subtarget); in lowerFixedLengthVectorSetccToRVV()
10220 auto [Mask, VL] = getDefaultVLOps(VT.getVectorNumElements(), ContainerVT, DL, in lowerFixedLengthVectorSetccToRVV()
10222 MVT MaskVT = getMaskTypeFor(ContainerVT); in lowerFixedLengthVectorSetccToRVV()
10328 MVT ContainerVT = VT; in lowerABS() local
10330 ContainerVT = getContainerForFixedLengthVector(VT); in lowerABS()
10331 X = convertToScalableVector(ContainerVT, X, DAG, Subtarget); in lowerABS()
10338 Mask = convertToScalableVector(getMaskTypeFor(ContainerVT), Mask, DAG, in lowerABS()
10342 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerABS()
10345 RISCVISD::VMV_V_X_VL, DL, ContainerVT, DAG.getUNDEF(ContainerVT), in lowerABS()
10347 SDValue NegX = DAG.getNode(RISCVISD::SUB_VL, DL, ContainerVT, SplatZero, X, in lowerABS()
10348 DAG.getUNDEF(ContainerVT), Mask, VL); in lowerABS()
10349 SDValue Max = DAG.getNode(RISCVISD::SMAX_VL, DL, ContainerVT, X, NegX, in lowerABS()
10350 DAG.getUNDEF(ContainerVT), Mask, VL); in lowerABS()
10366 MVT ContainerVT = getContainerForFixedLengthVector(VT); in lowerFixedLengthVectorFCOPYSIGNToRVV() local
10367 Mag = convertToScalableVector(ContainerVT, Mag, DAG, Subtarget); in lowerFixedLengthVectorFCOPYSIGNToRVV()
10368 Sign = convertToScalableVector(ContainerVT, Sign, DAG, Subtarget); in lowerFixedLengthVectorFCOPYSIGNToRVV()
10370 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerFixedLengthVectorFCOPYSIGNToRVV()
10372 SDValue CopySign = DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Mag, in lowerFixedLengthVectorFCOPYSIGNToRVV()
10373 Sign, DAG.getUNDEF(ContainerVT), Mask, VL); in lowerFixedLengthVectorFCOPYSIGNToRVV()
10381 MVT ContainerVT = getContainerForFixedLengthVector(VT); in lowerFixedLengthVectorSelectToRVV() local
10384 MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount()); in lowerFixedLengthVectorSelectToRVV()
10389 convertToScalableVector(ContainerVT, Op.getOperand(1), DAG, Subtarget); in lowerFixedLengthVectorSelectToRVV()
10391 convertToScalableVector(ContainerVT, Op.getOperand(2), DAG, Subtarget); in lowerFixedLengthVectorSelectToRVV()
10394 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in lowerFixedLengthVectorSelectToRVV()
10396 SDValue Select = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, CC, Op1, in lowerFixedLengthVectorSelectToRVV()
10397 Op2, DAG.getUNDEF(ContainerVT), VL); in lowerFixedLengthVectorSelectToRVV()
10409 MVT ContainerVT = getContainerForFixedLengthVector(VT); in lowerToScalableOp() local
10425 Ops.push_back(convertToScalableVector(ContainerVT, V, DAG, Subtarget)); in lowerToScalableOp()
10429 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerToScalableOp()
10431 Ops.push_back(DAG.getUNDEF(ContainerVT)); in lowerToScalableOp()
10440 DAG.getNode(NewOpc, DL, DAG.getVTList(ContainerVT, MVT::Other), Ops, in lowerToScalableOp()
10447 DAG.getNode(NewOpc, DL, ContainerVT, Ops, Op->getFlags()); in lowerToScalableOp()
10464 MVT ContainerVT = VT; in lowerVPOp() local
10466 ContainerVT = getContainerForFixedLengthVector(VT); in lowerVPOp()
10477 Ops.push_back(DAG.getUNDEF(ContainerVT)); in lowerVPOp()
10486 Ops.push_back(DAG.getUNDEF(ContainerVT)); in lowerVPOp()
10497 MVT ContainerVT = getContainerForFixedLengthVector(OpVT); in lowerVPOp() local
10500 Ops.push_back(convertToScalableVector(ContainerVT, V, DAG, Subtarget)); in lowerVPOp()
10506 SDValue VPOp = DAG.getNode(RISCVISDOpc, DL, ContainerVT, Ops, Op->getFlags()); in lowerVPOp()
10520 MVT ContainerVT = VT; in lowerVPExtMaskOp() local
10522 ContainerVT = getContainerForFixedLengthVector(VT); in lowerVPExtMaskOp()
10523 MVT SrcVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount()); in lowerVPExtMaskOp()
10529 SDValue ZeroSplat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, in lowerVPExtMaskOp()
10530 DAG.getUNDEF(ContainerVT), Zero, VL); in lowerVPExtMaskOp()
10534 SDValue Splat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, in lowerVPExtMaskOp()
10535 DAG.getUNDEF(ContainerVT), SplatValue, VL); in lowerVPExtMaskOp()
10537 SDValue Result = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, Src, Splat, in lowerVPExtMaskOp()
10538 ZeroSplat, DAG.getUNDEF(ContainerVT), VL); in lowerVPExtMaskOp()
10555 MVT ContainerVT = VT; in lowerVPSetCCMaskOp() local
10557 ContainerVT = getContainerForFixedLengthVector(VT); in lowerVPSetCCMaskOp()
10558 Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget); in lowerVPSetCCMaskOp()
10559 Op2 = convertToScalableVector(ContainerVT, Op2, DAG, Subtarget); in lowerVPSetCCMaskOp()
10563 SDValue AllOneMask = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL); in lowerVPSetCCMaskOp()
10570 Result = DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op1, Op2, VL); in lowerVPSetCCMaskOp()
10575 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op1, Op2, VL); in lowerVPSetCCMaskOp()
10577 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Temp, AllOneMask, VL); in lowerVPSetCCMaskOp()
10585 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op1, AllOneMask, VL); in lowerVPSetCCMaskOp()
10586 Result = DAG.getNode(RISCVISD::VMAND_VL, DL, ContainerVT, Temp, Op2, VL); in lowerVPSetCCMaskOp()
10594 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op2, AllOneMask, VL); in lowerVPSetCCMaskOp()
10595 Result = DAG.getNode(RISCVISD::VMAND_VL, DL, ContainerVT, Op1, Temp, VL); in lowerVPSetCCMaskOp()
10603 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op1, AllOneMask, VL); in lowerVPSetCCMaskOp()
10604 Result = DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Temp, Op2, VL); in lowerVPSetCCMaskOp()
10612 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op2, AllOneMask, VL); in lowerVPSetCCMaskOp()
10613 Result = DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Temp, Op1, VL); in lowerVPSetCCMaskOp()
10771 MVT ContainerVT = VT; in lowerVPSpliceExperimental() local
10773 ContainerVT = getContainerForFixedLengthVector(VT); in lowerVPSpliceExperimental()
10774 Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget); in lowerVPSpliceExperimental()
10775 Op2 = convertToScalableVector(ContainerVT, Op2, DAG, Subtarget); in lowerVPSpliceExperimental()
10776 MVT MaskVT = getMaskTypeFor(ContainerVT); in lowerVPSpliceExperimental()
10782 ContainerVT = ContainerVT.changeVectorElementType(MVT::i8); in lowerVPSpliceExperimental()
10785 SDValue SplatOneOp1 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, in lowerVPSpliceExperimental()
10786 DAG.getUNDEF(ContainerVT), in lowerVPSpliceExperimental()
10788 SDValue SplatZeroOp1 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, in lowerVPSpliceExperimental()
10789 DAG.getUNDEF(ContainerVT), in lowerVPSpliceExperimental()
10791 Op1 = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, Op1, SplatOneOp1, in lowerVPSpliceExperimental()
10792 SplatZeroOp1, DAG.getUNDEF(ContainerVT), EVL1); in lowerVPSpliceExperimental()
10794 SDValue SplatOneOp2 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, in lowerVPSpliceExperimental()
10795 DAG.getUNDEF(ContainerVT), in lowerVPSpliceExperimental()
10797 SDValue SplatZeroOp2 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, in lowerVPSpliceExperimental()
10798 DAG.getUNDEF(ContainerVT), in lowerVPSpliceExperimental()
10800 Op2 = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, Op2, SplatOneOp2, in lowerVPSpliceExperimental()
10801 SplatZeroOp2, DAG.getUNDEF(ContainerVT), EVL2); in lowerVPSpliceExperimental()
10819 getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT), in lowerVPSpliceExperimental()
10821 SDValue Result = getVSlideup(DAG, Subtarget, DL, ContainerVT, SlideDown, Op2, in lowerVPSpliceExperimental()
10827 RISCVISD::SETCC_VL, DL, ContainerVT.changeVectorElementType(MVT::i1), in lowerVPSpliceExperimental()
10828 {Result, DAG.getConstant(0, DL, ContainerVT), in lowerVPSpliceExperimental()
10829 DAG.getCondCode(ISD::SETNE), DAG.getUNDEF(getMaskTypeFor(ContainerVT)), in lowerVPSpliceExperimental()
10849 MVT ContainerVT = VT; in lowerVPReverseExperimental() local
10851 ContainerVT = getContainerForFixedLengthVector(VT); in lowerVPReverseExperimental()
10852 Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget); in lowerVPReverseExperimental()
10853 MVT MaskVT = getMaskTypeFor(ContainerVT); in lowerVPReverseExperimental()
10857 MVT GatherVT = ContainerVT; in lowerVPReverseExperimental()
10858 MVT IndicesVT = ContainerVT.changeVectorElementTypeToInteger(); in lowerVPReverseExperimental()
10860 bool IsMaskVector = ContainerVT.getVectorElementType() == MVT::i1; in lowerVPReverseExperimental()
10862 GatherVT = IndicesVT = ContainerVT.changeVectorElementType(MVT::i8); in lowerVPReverseExperimental()
10920 DAG.getNode(RISCVISD::SETCC_VL, DL, ContainerVT, in lowerVPReverseExperimental()
10923 DAG.getUNDEF(getMaskTypeFor(ContainerVT)), Mask, EVL}); in lowerVPReverseExperimental()
10949 RISCVISD::SETCC_VL, DL, ContainerVT, in lowerVPReverseExperimental()
10951 DAG.getUNDEF(getMaskTypeFor(ContainerVT)), Mask, EVL}); in lowerVPReverseExperimental()
10970 MVT ContainerVT = VT; in lowerLogicVPOp() local
10973 ContainerVT = getContainerForFixedLengthVector(VT); in lowerLogicVPOp()
10974 Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget); in lowerLogicVPOp()
10975 Op2 = convertToScalableVector(ContainerVT, Op2, DAG, Subtarget); in lowerLogicVPOp()
10979 SDValue Val = DAG.getNode(getRISCVVLOp(Op), DL, ContainerVT, Op1, Op2, VL); in lowerLogicVPOp()
10990 MVT ContainerVT = VT; in lowerVPStridedLoad() local
10992 ContainerVT = getContainerForFixedLengthVector(VT); in lowerVPStridedLoad()
10994 SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other}); in lowerVPStridedLoad()
11005 DAG.getUNDEF(ContainerVT), VPNode->getBasePtr(), in lowerVPStridedLoad()
11009 MVT MaskVT = ContainerVT.changeVectorElementType(MVT::i1); in lowerVPStridedLoad()
11039 MVT ContainerVT = VT; in lowerVPStridedStore() local
11041 ContainerVT = getContainerForFixedLengthVector(VT); in lowerVPStridedStore()
11042 StoreVal = convertToScalableVector(ContainerVT, StoreVal, DAG, Subtarget); in lowerVPStridedStore()
11056 MVT MaskVT = ContainerVT.changeVectorElementType(MVT::i1); in lowerVPStridedStore()
11119 MVT ContainerVT = VT; in lowerMaskedGather() local
11121 ContainerVT = getContainerForFixedLengthVector(VT); in lowerMaskedGather()
11123 ContainerVT.getVectorElementCount()); in lowerMaskedGather()
11128 MVT MaskVT = getMaskTypeFor(ContainerVT); in lowerMaskedGather()
11130 PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget); in lowerMaskedGather()
11135 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in lowerMaskedGather()
11146 Ops.push_back(DAG.getUNDEF(ContainerVT)); in lowerMaskedGather()
11157 SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other}); in lowerMaskedGather()
11218 MVT ContainerVT = VT; in lowerMaskedScatter() local
11220 ContainerVT = getContainerForFixedLengthVector(VT); in lowerMaskedScatter()
11222 ContainerVT.getVectorElementCount()); in lowerMaskedScatter()
11225 Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget); in lowerMaskedScatter()
11228 MVT MaskVT = getMaskTypeFor(ContainerVT); in lowerMaskedScatter()
11234 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in lowerMaskedScatter()
11853 MVT ContainerVT = VecVT; in ReplaceNodeResults() local
11855 ContainerVT = getContainerForFixedLengthVector(VecVT); in ReplaceNodeResults()
11856 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); in ReplaceNodeResults()
11862 auto [Mask, VL] = getDefaultVLOps(1, ContainerVT, DL, DAG, Subtarget); in ReplaceNodeResults()
11867 Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT, in ReplaceNodeResults()
11868 DAG.getUNDEF(ContainerVT), Vec, Idx, Mask, VL); in ReplaceNodeResults()
11876 SDValue ThirtyTwoV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, in ReplaceNodeResults()
11877 DAG.getUNDEF(ContainerVT), in ReplaceNodeResults()
11880 DAG.getNode(RISCVISD::SRL_VL, DL, ContainerVT, Vec, ThirtyTwoV, in ReplaceNodeResults()
11881 DAG.getUNDEF(ContainerVT), Mask, VL); in ReplaceNodeResults()
13874 MVT ContainerVT = VT.getSimpleVT(); in performFP_TO_INTCombine() local
13889 ContainerVT = in performFP_TO_INTCombine()
13890 getContainerForFixedLengthVector(DAG, ContainerVT, Subtarget); in performFP_TO_INTCombine()
13902 FpToInt = DAG.getNode(Opc, DL, ContainerVT, XVal, Mask, VL); in performFP_TO_INTCombine()
13906 FpToInt = DAG.getNode(Opc, DL, ContainerVT, XVal, Mask, VL); in performFP_TO_INTCombine()
13910 FpToInt = DAG.getNode(Opc, DL, ContainerVT, XVal, Mask, in performFP_TO_INTCombine()
19990 MVT ContainerVT = VT.getSimpleVT(); in isLegalInterleavedAccessType() local
20000 ContainerVT = getContainerForFixedLengthVector(VT.getSimpleVT()); in isLegalInterleavedAccessType()
20004 auto [LMUL, Fractional] = RISCVVType::decodeVLMUL(getLMUL(ContainerVT)); in isLegalInterleavedAccessType()