Lines Matching refs:DL
1418 EVT RISCVTargetLowering::getSetCCResultType(const DataLayout &DL, in getSetCCResultType() argument
1422 return getPointerTy(DL); in getSetCCResultType()
1464 auto &DL = I.getModule()->getDataLayout(); in getTgtMemIntrinsic() local
1483 Info.memVT = getValueType(DL, MemTy); in getTgtMemIntrinsic()
1484 Info.align = Align(DL.getTypeSizeInBits(MemTy->getScalarType()) / 8); in getTgtMemIntrinsic()
1724 bool RISCVTargetLowering::isLegalAddressingMode(const DataLayout &DL, in isLegalAddressingMode() argument
2220 static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS, in translateSetCCForBranch() argument
2239 LHS = DAG.getNode(ISD::SHL, DL, LHS.getValueType(), LHS, in translateSetCCForBranch()
2240 DAG.getConstant(ShAmt, DL, LHS.getValueType())); in translateSetCCForBranch()
2252 RHS = DAG.getConstant(0, DL, RHS.getValueType()); in translateSetCCForBranch()
2261 LHS = DAG.getConstant(0, DL, RHS.getValueType()); in translateSetCCForBranch()
2562 SDLoc DL(V); in convertToScalableVector() local
2563 SDValue Zero = DAG.getConstant(0, DL, Subtarget.getXLenVT()); in convertToScalableVector()
2564 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V, Zero); in convertToScalableVector()
2574 SDLoc DL(V); in convertFromScalableVector() local
2575 SDValue Zero = DAG.getConstant(0, DL, Subtarget.getXLenVT()); in convertFromScalableVector()
2576 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V, Zero); in convertFromScalableVector()
2590 static SDValue getAllOnesMask(MVT VecVT, SDValue VL, const SDLoc &DL, in getAllOnesMask() argument
2593 return DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL); in getAllOnesMask()
2596 static SDValue getVLOp(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL, in getVLOp() argument
2606 return DAG.getConstant(NumElts, DL, Subtarget.getXLenVT()); in getVLOp()
2610 getDefaultScalableVLOps(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG, in getDefaultScalableVLOps() argument
2614 SDValue Mask = getAllOnesMask(VecVT, VL, DL, DAG); in getDefaultScalableVLOps()
2619 getDefaultVLOps(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL, in getDefaultVLOps() argument
2622 SDValue VL = getVLOp(NumElts, ContainerVT, DL, DAG, Subtarget); in getDefaultVLOps()
2623 SDValue Mask = getAllOnesMask(ContainerVT, VL, DL, DAG); in getDefaultVLOps()
2632 getDefaultVLOps(MVT VecVT, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG, in getDefaultVLOps() argument
2635 return getDefaultVLOps(VecVT.getVectorNumElements(), ContainerVT, DL, DAG, in getDefaultVLOps()
2638 return getDefaultScalableVLOps(ContainerVT, DL, DAG, Subtarget); in getDefaultVLOps()
2641 SDValue RISCVTargetLowering::computeVLMax(MVT VecVT, const SDLoc &DL, in computeVLMax() argument
2644 return DAG.getElementCount(DL, Subtarget.getXLenVT(), in computeVLMax()
2762 SDLoc DL(Op); in lowerFP_TO_INT_SAT() local
2764 Opc, DL, DstVT, Src, in lowerFP_TO_INT_SAT()
2765 DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, Subtarget.getXLenVT())); in lowerFP_TO_INT_SAT()
2768 FpToInt = DAG.getZeroExtendInReg(FpToInt, DL, MVT::i32); in lowerFP_TO_INT_SAT()
2770 SDValue ZeroInt = DAG.getConstant(0, DL, DstVT); in lowerFP_TO_INT_SAT()
2771 return DAG.getSelectCC(DL, Src, Src, ZeroInt, FpToInt, in lowerFP_TO_INT_SAT()
2802 SDLoc DL(Op); in lowerFP_TO_INT_SAT() local
2804 auto [Mask, VL] = getDefaultVLOps(DstVT, DstContainerVT, DL, DAG, Subtarget); in lowerFP_TO_INT_SAT()
2806 SDValue IsNan = DAG.getNode(RISCVISD::SETCC_VL, DL, Mask.getValueType(), in lowerFP_TO_INT_SAT()
2815 Src = DAG.getNode(RISCVISD::FP_EXTEND_VL, DL, InterVT, Src, Mask, VL); in lowerFP_TO_INT_SAT()
2820 SDValue Res = DAG.getNode(RVVOpc, DL, DstContainerVT, Src, Mask, VL); in lowerFP_TO_INT_SAT()
2823 RISCVISD::VMV_V_X_VL, DL, DstContainerVT, DAG.getUNDEF(DstContainerVT), in lowerFP_TO_INT_SAT()
2824 DAG.getConstant(0, DL, Subtarget.getXLenVT()), VL); in lowerFP_TO_INT_SAT()
2825 Res = DAG.getNode(RISCVISD::VMERGE_VL, DL, DstContainerVT, IsNan, SplatZero, in lowerFP_TO_INT_SAT()
2873 SDLoc DL(Op); in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND() local
2891 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2898 SDValue Abs = DAG.getNode(RISCVISD::FABS_VL, DL, ContainerVT, Src, Mask, VL); in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2909 DAG.getConstantFP(MaxVal, DL, ContainerVT.getVectorElementType()); in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2910 SDValue MaxValSplat = DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, ContainerVT, in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2916 DAG.getNode(RISCVISD::SETCC_VL, DL, SetccVT, in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2939 Truncated = DAG.getNode(RISCVISD::VFCVT_RM_X_F_VL, DL, IntVT, Src, Mask, in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2940 DAG.getTargetConstant(FRM, DL, XLenVT), VL); in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2944 Truncated = DAG.getNode(RISCVISD::VFCVT_RTZ_X_F_VL, DL, IntVT, Src, in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2949 Truncated = DAG.getNode(RISCVISD::VFCVT_X_F_VL, DL, IntVT, Src, Mask, VL); in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2953 Truncated = DAG.getNode(RISCVISD::VFROUND_NOEXCEPT_VL, DL, ContainerVT, Src, in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2960 Truncated = DAG.getNode(RISCVISD::SINT_TO_FP_VL, DL, ContainerVT, Truncated, in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2964 Truncated = DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Truncated, in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2979 SDLoc DL(Op); in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND() local
2990 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
2997 SDValue Unorder = DAG.getNode(RISCVISD::STRICT_FSETCC_VL, DL, in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3002 Src = DAG.getNode(RISCVISD::STRICT_FADD_VL, DL, in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3008 SDValue Abs = DAG.getNode(RISCVISD::FABS_VL, DL, ContainerVT, Src, Mask, VL); in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3019 DAG.getConstantFP(MaxVal, DL, ContainerVT.getVectorElementType()); in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3020 SDValue MaxValSplat = DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, ContainerVT, in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3025 RISCVISD::SETCC_VL, DL, MaskVT, in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3043 RISCVISD::STRICT_VFCVT_RM_X_F_VL, DL, DAG.getVTList(IntVT, MVT::Other), in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3044 {Chain, Src, Mask, DAG.getTargetConstant(FRM, DL, XLenVT), VL}); in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3049 DAG.getNode(RISCVISD::STRICT_VFCVT_RTZ_X_F_VL, DL, in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3053 Truncated = DAG.getNode(RISCVISD::STRICT_VFROUND_NOEXCEPT_VL, DL, in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3062 Truncated = DAG.getNode(RISCVISD::STRICT_SINT_TO_FP_VL, DL, in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3069 Truncated = DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Truncated, in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3074 return DAG.getMergeValues({Truncated, Chain}, DL); in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3087 SDLoc DL(Op); in lowerFTRUNC_FCEIL_FFLOOR_FROUND() local
3098 SDValue MaxValNode = DAG.getConstantFP(MaxVal, DL, VT); in lowerFTRUNC_FCEIL_FFLOOR_FROUND()
3101 return DAG.getNode(RISCVISD::FROUND, DL, VT, Src, MaxValNode, in lowerFTRUNC_FCEIL_FFLOOR_FROUND()
3102 DAG.getTargetConstant(FRM, DL, Subtarget.getXLenVT())); in lowerFTRUNC_FCEIL_FFLOOR_FROUND()
3111 SDLoc DL(Op); in lowerVectorXRINT() local
3120 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerVectorXRINT()
3122 DAG.getNode(RISCVISD::VFCVT_X_F_VL, DL, ContainerVT, Src, Mask, VL); in lowerVectorXRINT()
3132 const SDLoc &DL, EVT VT, SDValue Merge, SDValue Op, in getVSlidedown() argument
3137 SDValue PolicyOp = DAG.getTargetConstant(Policy, DL, Subtarget.getXLenVT()); in getVSlidedown()
3139 return DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, VT, Ops); in getVSlidedown()
3143 getVSlideup(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL, in getVSlideup() argument
3149 SDValue PolicyOp = DAG.getTargetConstant(Policy, DL, Subtarget.getXLenVT()); in getVSlideup()
3151 return DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, VT, Ops); in getVSlideup()
3303 static SDValue matchSplatAsGather(SDValue SplatVal, MVT VT, const SDLoc &DL, in matchSplatAsGather() argument
3325 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in matchSplatAsGather()
3327 SDValue Gather = DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT, Vec, in matchSplatAsGather()
3352 SDLoc DL(Op); in lowerBuildVectorViaDominantValues() local
3353 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerBuildVectorViaDominantValues()
3401 SDValue Vec = DAG.getSplatBuildVector(VT, DL, DominantValue); in lowerBuildVectorViaDominantValues()
3417 LastOp = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, LastOp); in lowerBuildVectorViaDominantValues()
3418 Vec = DAG.getNode(OpCode, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Vec, in lowerBuildVectorViaDominantValues()
3430 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, Vec, V, in lowerBuildVectorViaDominantValues()
3431 DAG.getConstant(OpIdx.index(), DL, XLenVT)); in lowerBuildVectorViaDominantValues()
3438 return DAG.getConstant(V == V1, DL, XLenVT); in lowerBuildVectorViaDominantValues()
3440 Vec = DAG.getNode(ISD::VSELECT, DL, VT, in lowerBuildVectorViaDominantValues()
3441 DAG.getBuildVector(SelMaskTy, DL, Ops), in lowerBuildVectorViaDominantValues()
3442 DAG.getSplatBuildVector(VT, DL, V), Vec); in lowerBuildVectorViaDominantValues()
3459 SDLoc DL(Op); in lowerBuildVectorOfConstants() local
3460 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerBuildVectorOfConstants()
3467 SDValue VMClr = DAG.getNode(RISCVISD::VMCLR_VL, DL, ContainerVT, VL); in lowerBuildVectorOfConstants()
3472 SDValue VMSet = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL); in lowerBuildVectorOfConstants()
3514 SDValue Elt = DAG.getConstant(Bits, DL, XLenVT); in lowerBuildVectorOfConstants()
3522 SDValue Vec = DAG.getBuildVector(IntegerViaVecVT, DL, Elts); in lowerBuildVectorOfConstants()
3530 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Vec, in lowerBuildVectorOfConstants()
3531 DAG.getConstant(0, DL, XLenVT)); in lowerBuildVectorOfConstants()
3546 Splat = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Splat); in lowerBuildVectorOfConstants()
3548 DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Splat, VL); in lowerBuildVectorOfConstants()
3585 SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, VIDContainerVT, Mask, VL); in lowerBuildVectorOfConstants()
3593 SDValue SplatStep = DAG.getConstant(SplatStepVal, DL, VIDVT); in lowerBuildVectorOfConstants()
3594 VID = DAG.getNode(StepOpcode, DL, VIDVT, VID, SplatStep); in lowerBuildVectorOfConstants()
3598 DAG.getConstant(Log2_64(StepDenominator), DL, VIDVT); in lowerBuildVectorOfConstants()
3599 VID = DAG.getNode(ISD::SRL, DL, VIDVT, VID, SplatStep); in lowerBuildVectorOfConstants()
3602 SDValue SplatAddend = DAG.getConstant(Addend, DL, VIDVT); in lowerBuildVectorOfConstants()
3603 VID = DAG.getNode(Negate ? ISD::SUB : ISD::ADD, DL, VIDVT, SplatAddend, in lowerBuildVectorOfConstants()
3608 VID = DAG.getNode(ISD::SINT_TO_FP, DL, VT, VID); in lowerBuildVectorOfConstants()
3644 SDValue Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ViaVecVT, in lowerBuildVectorOfConstants()
3646 DAG.getConstant(SplatValue, DL, XLenVT), in lowerBuildVectorOfConstants()
3647 DAG.getConstant(0, DL, XLenVT)); in lowerBuildVectorOfConstants()
3649 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, in lowerBuildVectorOfConstants()
3651 DAG.getConstant(0, DL, XLenVT)); in lowerBuildVectorOfConstants()
3709 DAG.getConstant(ViaVecVT.getVectorNumElements(), DL, XLenVT); in lowerBuildVectorOfConstants()
3713 DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ViaContainerVT, in lowerBuildVectorOfConstants()
3715 DAG.getConstant(SplatValue, DL, XLenVT), ViaVL); in lowerBuildVectorOfConstants()
3718 Splat = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, in lowerBuildVectorOfConstants()
3720 DAG.getConstant(0, DL, XLenVT)); in lowerBuildVectorOfConstants()
3736 DL, Op->ops()); in lowerBuildVectorOfConstants()
3739 SDValue Res = DAG.getNode(RISCVISD::VSEXT_VL, DL, ContainerVT, Source, Mask, VL); in lowerBuildVectorOfConstants()
3764 SDLoc DL(Op); in lowerBUILD_VECTOR() local
3765 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerBUILD_VECTOR()
3774 SDValue VecZero = DAG.getConstant(0, DL, WideVecVT); in lowerBUILD_VECTOR()
3780 Splat = DAG.getNode(ISD::AND, DL, Splat.getValueType(), Splat, in lowerBUILD_VECTOR()
3781 DAG.getConstant(1, DL, Splat.getValueType())); in lowerBUILD_VECTOR()
3782 WideVec = DAG.getSplatBuildVector(WideVecVT, DL, Splat); in lowerBUILD_VECTOR()
3785 WideVec = DAG.getBuildVector(WideVecVT, DL, Ops); in lowerBUILD_VECTOR()
3786 SDValue VecOne = DAG.getConstant(1, DL, WideVecVT); in lowerBUILD_VECTOR()
3787 WideVec = DAG.getNode(ISD::AND, DL, WideVecVT, WideVec, VecOne); in lowerBUILD_VECTOR()
3790 return DAG.getSetCC(DL, VT, WideVec, VecZero, ISD::SETNE); in lowerBUILD_VECTOR()
3794 if (auto Gather = matchSplatAsGather(Splat, VT, DL, DAG, Subtarget)) in lowerBUILD_VECTOR()
3799 Splat = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Splat); in lowerBUILD_VECTOR()
3801 DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Splat, VL); in lowerBUILD_VECTOR()
3830 DAG.getNode(ISD::BUILD_VECTOR, DL, OneRegVT, OneVRegOfOps); in lowerBUILD_VECTOR()
3833 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT, Vec, SubBV, in lowerBUILD_VECTOR()
3834 DAG.getVectorIdxConstant(InsertIdx, DL)); in lowerBUILD_VECTOR()
3902 Vec = DAG.getSplatVector(VT, DL, V); in lowerBUILD_VECTOR()
3909 const SDValue Offset = DAG.getConstant(UndefCount, DL, Subtarget.getXLenVT()); in lowerBUILD_VECTOR()
3910 Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT), in lowerBUILD_VECTOR()
3917 V = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), V); in lowerBUILD_VECTOR()
3918 Vec = DAG.getNode(OpCode, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Vec, in lowerBUILD_VECTOR()
3922 const SDValue Offset = DAG.getConstant(UndefCount, DL, Subtarget.getXLenVT()); in lowerBUILD_VECTOR()
3923 Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT), in lowerBUILD_VECTOR()
3929 static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru, in splatPartsI64WithVL() argument
3940 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL); in splatPartsI64WithVL()
3954 NewVL = DAG.getNode(ISD::ADD, DL, VL.getValueType(), VL, VL); in splatPartsI64WithVL()
3959 auto InterVec = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, InterVT, in splatPartsI64WithVL()
3962 return DAG.getNode(ISD::BITCAST, DL, VT, InterVec); in splatPartsI64WithVL()
3971 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL); in splatPartsI64WithVL()
3976 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL); in splatPartsI64WithVL()
3979 return DAG.getNode(RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL, DL, VT, Passthru, Lo, in splatPartsI64WithVL()
3986 static SDValue splatSplitI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru, in splatSplitI64WithVL() argument
3991 std::tie(Lo, Hi) = DAG.SplitScalar(Scalar, DL, MVT::i32, MVT::i32); in splatSplitI64WithVL()
3992 return splatPartsI64WithVL(DL, VT, Passthru, Lo, Hi, VL, DAG); in splatSplitI64WithVL()
3999 MVT VT, const SDLoc &DL, SelectionDAG &DAG, in lowerScalarSplat() argument
4005 return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, VT, Passthru, Scalar, VL); in lowerScalarSplat()
4017 Scalar = DAG.getNode(ExtOpc, DL, XLenVT, Scalar); in lowerScalarSplat()
4018 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Scalar, VL); in lowerScalarSplat()
4025 return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, Passthru, in lowerScalarSplat()
4026 DAG.getConstant(0, DL, XLenVT), VL); in lowerScalarSplat()
4029 return splatSplitI64WithVL(DL, VT, Passthru, Scalar, VL, DAG); in lowerScalarSplat()
4037 const SDLoc &DL, SelectionDAG &DAG, in lowerScalarInsert() argument
4059 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Passthru, in lowerScalarInsert()
4060 ExtractedVal, DAG.getConstant(0, DL, XLenVT)); in lowerScalarInsert()
4061 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, ExtractedVal, in lowerScalarInsert()
4062 DAG.getConstant(0, DL, XLenVT)); in lowerScalarInsert()
4068 return DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, VT, in lowerScalarInsert()
4075 DAG.getConstant(1, DL, XLenVT), in lowerScalarInsert()
4076 VT, DL, DAG, Subtarget); in lowerScalarInsert()
4084 Scalar = DAG.getNode(ExtOpc, DL, XLenVT, Scalar); in lowerScalarInsert()
4085 return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, in lowerScalarInsert()
4251 static SDValue getDeinterleaveViaVNSRL(const SDLoc &DL, MVT VT, SDValue Src, in getDeinterleaveViaVNSRL() argument
4269 auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in getDeinterleaveViaVNSRL()
4285 RISCVISD::VMV_V_X_VL, DL, IntContainerVT, DAG.getUNDEF(ContainerVT), in getDeinterleaveViaVNSRL()
4286 DAG.getConstant(Shift, DL, Subtarget.getXLenVT()), VL); in getDeinterleaveViaVNSRL()
4288 DAG.getNode(RISCVISD::VNSRL_VL, DL, IntContainerVT, Src, SplatShift, in getDeinterleaveViaVNSRL()
4310 static SDValue lowerVECTOR_SHUFFLEAsVSlidedown(const SDLoc &DL, MVT VT, in lowerVECTOR_SHUFFLEAsVSlidedown() argument
4364 auto [TrueMask, VL] = getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget); in lowerVECTOR_SHUFFLEAsVSlidedown()
4366 getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT), in lowerVECTOR_SHUFFLEAsVSlidedown()
4368 DAG.getConstant(NewMask[0], DL, XLenVT), TrueMask, VL); in lowerVECTOR_SHUFFLEAsVSlidedown()
4370 ISD::EXTRACT_SUBVECTOR, DL, VT, in lowerVECTOR_SHUFFLEAsVSlidedown()
4372 DAG.getConstant(0, DL, XLenVT)); in lowerVECTOR_SHUFFLEAsVSlidedown()
4387 static SDValue lowerVECTOR_SHUFFLEAsVSlideup(const SDLoc &DL, MVT VT, in lowerVECTOR_SHUFFLEAsVSlideup() argument
4404 auto TrueMask = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).first; in lowerVECTOR_SHUFFLEAsVSlideup()
4415 SDValue VL = DAG.getConstant(NumSubElts + Index, DL, XLenVT); in lowerVECTOR_SHUFFLEAsVSlideup()
4421 Res = DAG.getNode(RISCVISD::VMV_V_V_VL, DL, ContainerVT, InPlace, ToInsert, in lowerVECTOR_SHUFFLEAsVSlideup()
4424 Res = getVSlideup(DAG, Subtarget, DL, ContainerVT, InPlace, ToInsert, in lowerVECTOR_SHUFFLEAsVSlideup()
4425 DAG.getConstant(Index, DL, XLenVT), TrueMask, VL, Policy); in lowerVECTOR_SHUFFLEAsVSlideup()
4431 static SDValue lowerVECTOR_SHUFFLEAsVSlide1(const SDLoc &DL, MVT VT, in lowerVECTOR_SHUFFLEAsVSlide1() argument
4469 auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerVECTOR_SHUFFLEAsVSlide1()
4474 Splat = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), Splat); in lowerVECTOR_SHUFFLEAsVSlide1()
4475 auto Vec = DAG.getNode(OpCode, DL, ContainerVT, in lowerVECTOR_SHUFFLEAsVSlide1()
4486 const SDLoc &DL, SelectionDAG &DAG, in getWideningInterleave() argument
4515 auto [Mask, VL] = getDefaultVLOps(VecVT, VecContainerVT, DL, DAG, Subtarget); in getWideningInterleave()
4522 DAG.getSplatVector(VecContainerVT, DL, in getWideningInterleave()
4523 DAG.getConstant(VecVT.getScalarSizeInBits(), DL, in getWideningInterleave()
4525 Interleaved = DAG.getNode(RISCVISD::VWSLL_VL, DL, WideContainerVT, OddV, in getWideningInterleave()
4527 Interleaved = DAG.getNode(RISCVISD::VWADDU_W_VL, DL, WideContainerVT, in getWideningInterleave()
4532 Interleaved = DAG.getNode(RISCVISD::VWADDU_VL, DL, WideContainerVT, EvenV, in getWideningInterleave()
4537 VecContainerVT, DL, DAG.getAllOnesConstant(DL, Subtarget.getXLenVT())); in getWideningInterleave()
4538 SDValue OddsMul = DAG.getNode(RISCVISD::VWMULU_VL, DL, WideContainerVT, in getWideningInterleave()
4546 Interleaved = DAG.getNode(RISCVISD::ADD_VL, DL, WideContainerVT, in getWideningInterleave()
4572 SDLoc DL(SVN); in lowerBitreverseShuffle() local
4601 V = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ViaBitVT, DAG.getUNDEF(ViaBitVT), in lowerBitreverseShuffle()
4602 V, DAG.getVectorIdxConstant(0, DL)); in lowerBitreverseShuffle()
4605 DAG.getNode(ISD::BITREVERSE, DL, ViaVT, DAG.getBitcast(ViaVT, V)); in lowerBitreverseShuffle()
4610 Res = DAG.getNode(ISD::SRL, DL, ViaVT, Res, in lowerBitreverseShuffle()
4611 DAG.getConstant(ViaEltSize - NumElts, DL, ViaVT)); in lowerBitreverseShuffle()
4616 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res, in lowerBitreverseShuffle()
4617 DAG.getVectorIdxConstant(0, DL)); in lowerBitreverseShuffle()
4627 SDLoc DL(SVN); in lowerVECTOR_SHUFFLEAsRotate() local
4649 Rotate = DAG.getNode(ISD::BSWAP, DL, RotateVT, Op); in lowerVECTOR_SHUFFLEAsRotate()
4651 Rotate = DAG.getNode(ISD::ROTL, DL, RotateVT, Op, in lowerVECTOR_SHUFFLEAsRotate()
4652 DAG.getConstant(RotateAmt, DL, RotateVT)); in lowerVECTOR_SHUFFLEAsRotate()
4663 SDLoc DL(SVN); in lowerShuffleViaVRegSplitting() local
4724 SDValue SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, M1VT, SrcVec, in lowerShuffleViaVRegSplitting()
4725 DAG.getVectorIdxConstant(ExtractIdx, DL)); in lowerShuffleViaVRegSplitting()
4727 SubVec = DAG.getVectorShuffle(OneRegVT, DL, SubVec, SubVec, SrcSubMask); in lowerShuffleViaVRegSplitting()
4730 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT, Vec, SubVec, in lowerShuffleViaVRegSplitting()
4731 DAG.getVectorIdxConstant(InsertIdx, DL)); in lowerShuffleViaVRegSplitting()
4740 SDLoc DL(Op); in lowerVECTOR_SHUFFLE() local
4756 V1 = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenVT, V1); in lowerVECTOR_SHUFFLE()
4758 : DAG.getNode(ISD::ZERO_EXTEND, DL, WidenVT, V2); in lowerVECTOR_SHUFFLE()
4759 SDValue Shuffled = DAG.getVectorShuffle(WidenVT, DL, V1, V2, SVN->getMask()); in lowerVECTOR_SHUFFLE()
4760 return DAG.getSetCC(DL, VT, Shuffled, DAG.getConstant(0, DL, WidenVT), in lowerVECTOR_SHUFFLE()
4766 auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
4791 Ld->getBasePtr(), TypeSize::getFixed(Offset), DL); in lowerVECTOR_SHUFFLE()
4797 DAG.getTargetConstant(Intrinsic::riscv_vlse, DL, XLenVT); in lowerVECTOR_SHUFFLE()
4805 ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, SVT, in lowerVECTOR_SHUFFLE()
4816 V = DAG.getLoad(SVT, DL, Ld->getChain(), NewAddr, in lowerVECTOR_SHUFFLE()
4821 V = DAG.getExtLoad(ISD::SEXTLOAD, DL, XLenVT, Ld->getChain(), NewAddr, in lowerVECTOR_SHUFFLE()
4830 DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT), V, VL); in lowerVECTOR_SHUFFLE()
4836 SDValue Gather = DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT, in lowerVECTOR_SHUFFLE()
4837 V1, DAG.getConstant(Lane, DL, XLenVT), in lowerVECTOR_SHUFFLE()
4851 lowerVECTOR_SHUFFLEAsVSlide1(DL, VT, V1, V2, Mask, Subtarget, DAG)) in lowerVECTOR_SHUFFLE()
4855 lowerVECTOR_SHUFFLEAsVSlidedown(DL, VT, V1, V2, Mask, Subtarget, DAG)) in lowerVECTOR_SHUFFLE()
4887 Res = getVSlidedown(DAG, Subtarget, DL, ContainerVT, Res, HiV, in lowerVECTOR_SHUFFLE()
4888 DAG.getConstant(Rotation, DL, XLenVT), TrueMask, VL); in lowerVECTOR_SHUFFLE()
4891 Res = getVSlideup(DAG, Subtarget, DL, ContainerVT, Res, LoV, in lowerVECTOR_SHUFFLE()
4892 DAG.getConstant(InvRotate, DL, XLenVT), TrueMask, VL, in lowerVECTOR_SHUFFLE()
4901 return getDeinterleaveViaVNSRL(DL, VT, V1.getOperand(0), Mask[0] == 0, in lowerVECTOR_SHUFFLE()
4906 lowerVECTOR_SHUFFLEAsVSlideup(DL, VT, V1, V2, Mask, Subtarget, DAG)) in lowerVECTOR_SHUFFLE()
4920 EvenV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, EvenV, in lowerVECTOR_SHUFFLE()
4921 DAG.getConstant(EvenSrc % Size, DL, XLenVT)); in lowerVECTOR_SHUFFLE()
4925 OddV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, OddV, in lowerVECTOR_SHUFFLE()
4926 DAG.getConstant(OddSrc % Size, DL, XLenVT)); in lowerVECTOR_SHUFFLE()
4928 return getWideningInterleave(EvenV, OddV, DL, DAG, Subtarget); in lowerVECTOR_SHUFFLE()
4952 MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT)); in lowerVECTOR_SHUFFLE()
4960 SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals); in lowerVECTOR_SHUFFLE()
4961 return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, V1, V2); in lowerVECTOR_SHUFFLE()
4992 MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT)); in lowerVECTOR_SHUFFLE()
4995 ? DAG.getConstant(MaskIndex, DL, XLenVT) in lowerVECTOR_SHUFFLE()
4999 : DAG.getConstant(MaskIndex - NumElts, DL, XLenVT)); in lowerVECTOR_SHUFFLE()
5013 SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals); in lowerVECTOR_SHUFFLE()
5042 Gather = lowerScalarSplat(SDValue(), SplatValue, VL, ContainerVT, DL, DAG, in lowerVECTOR_SHUFFLE()
5051 Gather = DAG.getNode(GatherVXOpc, DL, ContainerVT, V1, in lowerVECTOR_SHUFFLE()
5052 DAG.getConstant(SplatIndex, DL, XLenVT), in lowerVECTOR_SHUFFLE()
5055 SDValue LHSIndices = DAG.getBuildVector(IndexVT, DL, GatherIndicesLHS); in lowerVECTOR_SHUFFLE()
5059 Gather = DAG.getNode(GatherVVOpc, DL, ContainerVT, V1, LHSIndices, in lowerVECTOR_SHUFFLE()
5078 Gather = DAG.getNode(GatherVXOpc, DL, ContainerVT, V2, in lowerVECTOR_SHUFFLE()
5079 DAG.getConstant(SplatIndex, DL, XLenVT), Gather, in lowerVECTOR_SHUFFLE()
5082 SDValue RHSIndices = DAG.getBuildVector(IndexVT, DL, GatherIndicesRHS); in lowerVECTOR_SHUFFLE()
5085 Gather = DAG.getNode(GatherVVOpc, DL, ContainerVT, V2, RHSIndices, Gather, in lowerVECTOR_SHUFFLE()
5121 SDLoc DL(Op); in lowerCTLZ_CTTZ_ZERO_UNDEF() local
5150 SDValue Neg = DAG.getNegative(Src, DL, VT); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5151 Src = DAG.getNode(ISD::AND, DL, VT, Src, Neg); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5153 SDValue Neg = DAG.getNode(ISD::VP_SUB, DL, VT, DAG.getConstant(0, DL, VT), in lowerCTLZ_CTTZ_ZERO_UNDEF()
5155 Src = DAG.getNode(ISD::VP_AND, DL, VT, Src, Neg, Mask, VL); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5162 FloatVal = DAG.getNode(ISD::VP_UINT_TO_FP, DL, FloatVT, Src, Mask, VL); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5164 FloatVal = DAG.getNode(ISD::UINT_TO_FP, DL, FloatVT, Src); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5172 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5174 DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, Subtarget.getXLenVT()); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5177 FloatVal = DAG.getNode(RISCVISD::VFCVT_RM_F_XU_VL, DL, ContainerFloatVT, in lowerCTLZ_CTTZ_ZERO_UNDEF()
5190 Exp = DAG.getNode(ISD::VP_LSHR, DL, IntVT, Bitcast, in lowerCTLZ_CTTZ_ZERO_UNDEF()
5191 DAG.getConstant(ShiftAmt, DL, IntVT), Mask, VL); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5192 Exp = DAG.getVPZExtOrTrunc(DL, VT, Exp, Mask, VL); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5194 Exp = DAG.getNode(ISD::SRL, DL, IntVT, Bitcast, in lowerCTLZ_CTTZ_ZERO_UNDEF()
5195 DAG.getConstant(ShiftAmt, DL, IntVT)); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5197 Exp = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Exp); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5199 Exp = DAG.getNode(ISD::TRUNCATE, DL, VT, Exp); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5206 return DAG.getNode(ISD::SUB, DL, VT, Exp, in lowerCTLZ_CTTZ_ZERO_UNDEF()
5207 DAG.getConstant(ExponentBias, DL, VT)); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5209 return DAG.getNode(ISD::VP_SUB, DL, VT, Exp, in lowerCTLZ_CTTZ_ZERO_UNDEF()
5210 DAG.getConstant(ExponentBias, DL, VT), Mask, VL); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5217 Res = DAG.getNode(ISD::VP_SUB, DL, VT, DAG.getConstant(Adjust, DL, VT), Exp, in lowerCTLZ_CTTZ_ZERO_UNDEF()
5220 Res = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(Adjust, DL, VT), Exp); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5225 Res = DAG.getNode(ISD::UMIN, DL, VT, Res, DAG.getConstant(EltSize, DL, VT)); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5227 Res = DAG.getNode(ISD::VP_UMIN, DL, VT, Res, in lowerCTLZ_CTTZ_ZERO_UNDEF()
5228 DAG.getConstant(EltSize, DL, VT), Mask, VL); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5246 SDLoc DL(Op); in expandUnalignedRVVLoad() local
5255 SDValue L = DAG.getLoad(NewVT, DL, Load->getChain(), Load->getBasePtr(), in expandUnalignedRVVLoad()
5258 return DAG.getMergeValues({DAG.getBitcast(VT, L), L.getValue(1)}, DL); in expandUnalignedRVVLoad()
5276 SDLoc DL(Op); in expandUnalignedRVVStore() local
5287 return DAG.getStore(Store->getChain(), DL, StoredVal, Store->getBasePtr(), in expandUnalignedRVVStore()
5363 SDLoc DL(Op); in LowerIS_FPCLASS() local
5391 SDValue TDCMaskV = DAG.getConstant(TDCMask, DL, XLenVT); in LowerIS_FPCLASS()
5399 auto [Mask, VL] = getDefaultScalableVLOps(VT0, DL, DAG, Subtarget); in LowerIS_FPCLASS()
5404 SDValue FPCLASS = DAG.getNode(RISCVISD::FCLASS_VL, DL, DstVT, Op0, Mask, in LowerIS_FPCLASS()
5407 return DAG.getSetCC(DL, VT, FPCLASS, in LowerIS_FPCLASS()
5408 DAG.getConstant(TDCMask, DL, DstVT), in LowerIS_FPCLASS()
5410 SDValue AND = DAG.getNode(ISD::AND, DL, DstVT, FPCLASS, in LowerIS_FPCLASS()
5411 DAG.getConstant(TDCMask, DL, DstVT)); in LowerIS_FPCLASS()
5412 return DAG.getSetCC(DL, VT, AND, DAG.getConstant(0, DL, DstVT), in LowerIS_FPCLASS()
5419 auto [Mask, VL] = getDefaultVLOps(VT0, ContainerVT0, DL, DAG, Subtarget); in LowerIS_FPCLASS()
5429 SDValue FPCLASS = DAG.getNode(RISCVISD::FCLASS_VL, DL, ContainerDstVT, Op0, in LowerIS_FPCLASS()
5432 TDCMaskV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerDstVT, in LowerIS_FPCLASS()
5436 DAG.getNode(RISCVISD::SETCC_VL, DL, ContainerVT, in LowerIS_FPCLASS()
5441 SDValue AND = DAG.getNode(RISCVISD::AND_VL, DL, ContainerDstVT, FPCLASS, in LowerIS_FPCLASS()
5444 SDValue SplatZero = DAG.getConstant(0, DL, XLenVT); in LowerIS_FPCLASS()
5445 SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerDstVT, in LowerIS_FPCLASS()
5448 SDValue VMSNE = DAG.getNode(RISCVISD::SETCC_VL, DL, ContainerVT, in LowerIS_FPCLASS()
5454 SDValue FCLASS = DAG.getNode(RISCVISD::FCLASS, DL, XLenVT, Op.getOperand(0)); in LowerIS_FPCLASS()
5455 SDValue AND = DAG.getNode(ISD::AND, DL, XLenVT, FCLASS, TDCMaskV); in LowerIS_FPCLASS()
5456 SDValue Res = DAG.getSetCC(DL, XLenVT, AND, DAG.getConstant(0, DL, XLenVT), in LowerIS_FPCLASS()
5458 return DAG.getNode(ISD::TRUNCATE, DL, VT, Res); in LowerIS_FPCLASS()
5465 SDLoc DL(Op); in lowerFMAXIMUM_FMINIMUM() local
5481 SDValue XIsNonNan = DAG.getSetCC(DL, XLenVT, X, X, ISD::SETOEQ); in lowerFMAXIMUM_FMINIMUM()
5482 NewY = DAG.getSelect(DL, VT, XIsNonNan, Y, X); in lowerFMAXIMUM_FMINIMUM()
5487 SDValue YIsNonNan = DAG.getSetCC(DL, XLenVT, Y, Y, ISD::SETOEQ); in lowerFMAXIMUM_FMINIMUM()
5488 NewX = DAG.getSelect(DL, VT, YIsNonNan, X, Y); in lowerFMAXIMUM_FMINIMUM()
5493 return DAG.getNode(Opc, DL, VT, NewX, NewY); in lowerFMAXIMUM_FMINIMUM()
5515 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerFMAXIMUM_FMINIMUM()
5520 SDValue XIsNonNan = DAG.getNode(RISCVISD::SETCC_VL, DL, Mask.getValueType(), in lowerFMAXIMUM_FMINIMUM()
5523 NewY = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, XIsNonNan, Y, X, in lowerFMAXIMUM_FMINIMUM()
5529 SDValue YIsNonNan = DAG.getNode(RISCVISD::SETCC_VL, DL, Mask.getValueType(), in lowerFMAXIMUM_FMINIMUM()
5532 NewX = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, YIsNonNan, X, Y, in lowerFMAXIMUM_FMINIMUM()
5540 SDValue Res = DAG.getNode(Opc, DL, ContainerVT, NewX, NewY, in lowerFMAXIMUM_FMINIMUM()
5731 SDLoc DL(Op); in SplitVectorOp() local
5743 DAG.SplitVector(Op.getOperand(j), DL); in SplitVectorOp()
5747 DAG.getNode(Op.getOpcode(), DL, LoVT, LoOperands, Op->getFlags()); in SplitVectorOp()
5749 DAG.getNode(Op.getOpcode(), DL, HiVT, HiOperands, Op->getFlags()); in SplitVectorOp()
5751 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Op.getValueType(), LoRes, HiRes); in SplitVectorOp()
5757 SDLoc DL(Op); in SplitVPOp() local
5765 DAG.SplitEVL(Op.getOperand(j), Op.getValueType(), DL); in SplitVPOp()
5774 DAG.SplitVector(Op.getOperand(j), DL); in SplitVPOp()
5778 DAG.getNode(Op.getOpcode(), DL, LoVT, LoOperands, Op->getFlags()); in SplitVPOp()
5780 DAG.getNode(Op.getOpcode(), DL, HiVT, HiOperands, Op->getFlags()); in SplitVPOp()
5782 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Op.getValueType(), LoRes, HiRes); in SplitVPOp()
5786 SDLoc DL(Op); in SplitVectorReductionOp() local
5788 auto [Lo, Hi] = DAG.SplitVector(Op.getOperand(1), DL); in SplitVectorReductionOp()
5789 auto [MaskLo, MaskHi] = DAG.SplitVector(Op.getOperand(2), DL); in SplitVectorReductionOp()
5791 DAG.SplitEVL(Op.getOperand(3), Op.getOperand(1).getValueType(), DL); in SplitVectorReductionOp()
5794 DAG.getNode(Op.getOpcode(), DL, Op.getValueType(), in SplitVectorReductionOp()
5796 return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(), in SplitVectorReductionOp()
5809 SDLoc DL(Op); in SplitStrictFPVectorOp() local
5821 DAG.SplitVector(Op.getOperand(j), DL); in SplitStrictFPVectorOp()
5825 DAG.getNode(Op.getOpcode(), DL, LoVTs, LoOperands, Op->getFlags()); in SplitStrictFPVectorOp()
5828 DAG.getNode(Op.getOpcode(), DL, HiVTs, HiOperands, Op->getFlags()); in SplitStrictFPVectorOp()
5830 SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, DL, Op->getValueType(0), in SplitStrictFPVectorOp()
5832 return DAG.getMergeValues({V, HiRes.getValue(1)}, DL); in SplitStrictFPVectorOp()
5884 SDLoc DL(Op); in LowerOperation() local
5891 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Op0); in LowerOperation()
5892 SDValue FPConv = DAG.getNode(RISCVISD::FMV_H_X, DL, MVT::f16, NewOp0); in LowerOperation()
5897 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Op0); in LowerOperation()
5898 SDValue FPConv = DAG.getNode(RISCVISD::FMV_H_X, DL, MVT::bf16, NewOp0); in LowerOperation()
5903 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0); in LowerOperation()
5905 DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, NewOp0); in LowerOperation()
5911 std::tie(Lo, Hi) = DAG.SplitScalar(Op0, DL, MVT::i32, MVT::i32); in LowerOperation()
5913 DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, Lo, Hi); in LowerOperation()
5940 return DAG.getBitcast(VT, DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, BVT, in LowerOperation()
5942 DAG.getConstant(0, DL, XLenVT))); in LowerOperation()
5954 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, BVec, in LowerOperation()
5955 DAG.getConstant(0, DL, XLenVT)); in LowerOperation()
5973 SDLoc DL(Op); in LowerOperation() local
5977 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, Op.getOperand(0)); in LowerOperation()
5978 return DAG.getNode(RISCVISD::BREV8, DL, VT, BSwap); in LowerOperation()
6004 SDLoc DL(Op); in LowerOperation() local
6008 SDValue V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, WideVT, Scalar); in LowerOperation()
6009 return DAG.getNode(ISD::TRUNCATE, DL, VT, V); in LowerOperation()
6014 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in LowerOperation()
6015 Scalar = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), Scalar); in LowerOperation()
6016 SDValue V = DAG.getNode(RISCVISD::VMV_S_X_VL, DL, ContainerVT, in LowerOperation()
6025 SDLoc DL(Op); in LowerOperation() local
6026 SDValue Res = DAG.getNode(RISCVISD::READ_VLENB, DL, XLenVT); in LowerOperation()
6039 Res = DAG.getNode(ISD::SRL, DL, XLenVT, Res, in LowerOperation()
6040 DAG.getConstant(3 - Log2, DL, VT)); in LowerOperation()
6042 Res = DAG.getNode(ISD::SHL, DL, XLenVT, Res, in LowerOperation()
6043 DAG.getConstant(Log2 - 3, DL, XLenVT)); in LowerOperation()
6047 Res = DAG.getNode(ISD::MUL, DL, XLenVT, Res, in LowerOperation()
6048 DAG.getConstant(Val / 8, DL, XLenVT)); in LowerOperation()
6050 SDValue VScale = DAG.getNode(ISD::SRL, DL, XLenVT, Res, in LowerOperation()
6051 DAG.getConstant(3, DL, XLenVT)); in LowerOperation()
6052 Res = DAG.getNode(ISD::MUL, DL, XLenVT, VScale, in LowerOperation()
6053 DAG.getConstant(Val, DL, XLenVT)); in LowerOperation()
6055 return DAG.getNode(ISD::TRUNCATE, DL, VT, Res); in LowerOperation()
6062 SDLoc DL(Op); in LowerOperation() local
6063 SDValue Op0 = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op.getOperand(0)); in LowerOperation()
6065 DAG.getNode(ISD::FPOWI, DL, MVT::f32, Op0, Op.getOperand(1)); in LowerOperation()
6066 return DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, Powi, in LowerOperation()
6067 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)); in LowerOperation()
6079 SDLoc DL(Op); in LowerOperation() local
6084 return DAG.getNode(RISCVISD::FP_EXTEND_BF16, DL, MVT::f32, Op0); in LowerOperation()
6087 DAG.getNode(RISCVISD::FP_EXTEND_BF16, DL, MVT::f32, Op0); in LowerOperation()
6088 return DAG.getNode(ISD::FP_EXTEND, DL, MVT::f64, FloatVal); in LowerOperation()
6096 SDLoc DL(Op); in LowerOperation() local
6101 return DAG.getNode(RISCVISD::FP_ROUND_BF16, DL, MVT::bf16, Op0); in LowerOperation()
6105 DAG.getNode(ISD::FP_ROUND, DL, MVT::f32, Op0, in LowerOperation()
6106 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)); in LowerOperation()
6107 return DAG.getNode(RISCVISD::FP_ROUND_BF16, DL, MVT::bf16, FloatVal); in LowerOperation()
6126 SDLoc DL(Op); in LowerOperation() local
6129 SDValue NC = DAG.getNode(Op.getOpcode(), DL, NVT, Op->ops()); in LowerOperation()
6131 return DAG.getNode(ISD::FP_ROUND, DL, Op.getValueType(), NC, in LowerOperation()
6132 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)); in LowerOperation()
6145 SDLoc DL(Op); in LowerOperation() local
6148 SDValue WidenVec = DAG.getNode(ISD::FP_EXTEND, DL, NVT, Op1); in LowerOperation()
6150 return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(), WidenVec); in LowerOperation()
6163 SDLoc DL(Op); in LowerOperation() local
6185 SDValue Ext = DAG.getNode(ExtOpcode, DL, IVecVT, Src); in LowerOperation()
6187 return DAG.getNode(Op.getOpcode(), DL, Op->getVTList(), in LowerOperation()
6189 return DAG.getNode(Op.getOpcode(), DL, VT, Ext); in LowerOperation()
6198 DAG.getStrictFPExtendOrRound(Src, Op.getOperand(0), DL, InterimFVT); in LowerOperation()
6199 return DAG.getNode(Op.getOpcode(), DL, Op->getVTList(), Chain, FExt); in LowerOperation()
6201 SDValue FExt = DAG.getFPExtendOrRound(Src, DL, InterimFVT); in LowerOperation()
6202 return DAG.getNode(Op.getOpcode(), DL, VT, FExt); in LowerOperation()
6212 SDValue Int2FP = DAG.getNode(Op.getOpcode(), DL, in LowerOperation()
6216 return DAG.getStrictFPExtendOrRound(Int2FP, Chain, DL, VT).first; in LowerOperation()
6218 SDValue Int2FP = DAG.getNode(Op.getOpcode(), DL, InterimFVT, Src); in LowerOperation()
6219 return DAG.getFPExtendOrRound(Int2FP, DL, VT); in LowerOperation()
6228 DAG.getNode(Op.getOpcode(), DL, DAG.getVTList(IVecVT, MVT::Other), in LowerOperation()
6230 SDValue Res = DAG.getNode(ISD::TRUNCATE, DL, VT, FP2Int); in LowerOperation()
6231 return DAG.getMergeValues({Res, FP2Int.getValue(1)}, DL); in LowerOperation()
6233 SDValue FP2Int = DAG.getNode(Op.getOpcode(), DL, IVecVT, Src); in LowerOperation()
6234 return DAG.getNode(ISD::TRUNCATE, DL, VT, FP2Int); in LowerOperation()
6278 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in LowerOperation()
6282 Src = DAG.getNode(RVVOpc, DL, DAG.getVTList(ContainerVT, MVT::Other), in LowerOperation()
6285 return DAG.getMergeValues({SubVec, Src.getValue(1)}, DL); in LowerOperation()
6287 Src = DAG.getNode(RVVOpc, DL, ContainerVT, Src, Mask, VL); in LowerOperation()
6297 SDLoc DL(Op); in LowerOperation() local
6302 makeLibCall(DAG, LC, MVT::f32, Op.getOperand(0), CallOptions, DL).first; in LowerOperation()
6304 return DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Res); in LowerOperation()
6310 SDLoc DL(Op); in LowerOperation() local
6312 ISD::SHL, DL, Op.getOperand(0).getValueType(), Op.getOperand(0), in LowerOperation()
6313 DAG.getShiftAmountConstant(16, Op.getOperand(0).getValueType(), DL)); in LowerOperation()
6315 ? DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, Op) in LowerOperation()
6319 return DAG.getNode(ISD::FP_EXTEND, DL, VT, Res); in LowerOperation()
6326 SDLoc DL(Op); in LowerOperation() local
6331 makeLibCall(DAG, LC, MVT::f32, Op.getOperand(0), CallOptions, DL).first; in LowerOperation()
6333 return DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Res); in LowerOperation()
6340 SDLoc DL(Op); in LowerOperation() local
6343 ? DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, in LowerOperation()
6347 makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Arg, CallOptions, DL) in LowerOperation()
6426 SDLoc DL(Op); in LowerOperation() local
6428 DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op.getOperand(0)); in LowerOperation()
6430 ISD::SPLAT_VECTOR, DL, in LowerOperation()
6433 return DAG.getNode(ISD::FP_ROUND, DL, Op.getValueType(), NewSplat, in LowerOperation()
6434 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)); in LowerOperation()
6444 SDLoc DL(Op); in LowerOperation() local
6454 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Vec, SubVec, in LowerOperation()
6455 DAG.getIntPtrConstant(OpIdx.index() * NumOpElts, DL)); in LowerOperation()
6490 SDLoc DL(Op); in LowerOperation() local
6492 DAG.getNode(ISD::SETCC, DL, CCVT, Tmp1, Tmp2, CC, Op->getFlags()); in LowerOperation()
6493 return DAG.getSelect(DL, VT, Cond, True, False); in LowerOperation()
6505 SDLoc DL(Op); in LowerOperation() local
6517 return DAG.getConstant(0, DL, VT); in LowerOperation()
6521 DL, VT, LHS, DAG.getConstant(Imm + 1, DL, OpVT), CCVal); in LowerOperation()
6522 return DAG.getLogicalNOT(DL, SetCC, VT); in LowerOperation()
6529 return DAG.getSetCC(DL, VT, RHS, LHS, CCVal); in LowerOperation()
6697 SDLoc DL(Op); in LowerOperation() local
6700 auto NC = DAG.getNode(Op.getOpcode(), DL, NVT, Op->ops()); in LowerOperation()
6702 return DAG.getNode(ISD::FP_ROUND, DL, Op.getValueType(), NC, in LowerOperation()
6703 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)); in LowerOperation()
6716 SDLoc DL(Op); in LowerOperation() local
6719 SDValue WidenVec = DAG.getNode(ISD::FP_EXTEND, DL, NVT, Op1); in LowerOperation()
6721 return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(), in LowerOperation()
6782 static SDValue getTargetNode(GlobalAddressSDNode *N, const SDLoc &DL, EVT Ty, in getTargetNode() argument
6784 return DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, Flags); in getTargetNode()
6787 static SDValue getTargetNode(BlockAddressSDNode *N, const SDLoc &DL, EVT Ty, in getTargetNode() argument
6793 static SDValue getTargetNode(ConstantPoolSDNode *N, const SDLoc &DL, EVT Ty, in getTargetNode() argument
6799 static SDValue getTargetNode(JumpTableSDNode *N, const SDLoc &DL, EVT Ty, in getTargetNode() argument
6807 SDLoc DL(N); in getAddr() local
6815 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0); in getAddr()
6820 return DAG.getNode(RISCVISD::LLA, DL, Ty, Addr); in getAddr()
6826 SDValue(DAG.getMachineNode(RISCV::PseudoLGA, DL, Ty, Addr), 0); in getAddr()
6843 SDValue AddrHi = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_HI); in getAddr()
6844 SDValue AddrLo = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_LO); in getAddr()
6845 SDValue MNHi = DAG.getNode(RISCVISD::HI, DL, Ty, AddrHi); in getAddr()
6846 return DAG.getNode(RISCVISD::ADD_LO, DL, Ty, MNHi, AddrLo); in getAddr()
6849 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0); in getAddr()
6856 SDValue(DAG.getMachineNode(RISCV::PseudoLGA, DL, Ty, Addr), 0); in getAddr()
6870 return DAG.getNode(RISCVISD::LLA, DL, Ty, Addr); in getAddr()
6907 SDLoc DL(N); in getStaticTLSAddr() local
6917 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0); in getStaticTLSAddr()
6919 SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_IE, DL, Ty, Addr), 0); in getStaticTLSAddr()
6930 return DAG.getNode(ISD::ADD, DL, Ty, Load, TPReg); in getStaticTLSAddr()
6938 DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_HI); in getStaticTLSAddr()
6940 DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_ADD); in getStaticTLSAddr()
6942 DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_LO); in getStaticTLSAddr()
6944 SDValue MNHi = DAG.getNode(RISCVISD::HI, DL, Ty, AddrHi); in getStaticTLSAddr()
6947 DAG.getNode(RISCVISD::ADD_TPREL, DL, Ty, MNHi, TPReg, AddrAdd); in getStaticTLSAddr()
6948 return DAG.getNode(RISCVISD::ADD_LO, DL, Ty, MNAdd, AddrLo); in getStaticTLSAddr()
6953 SDLoc DL(N); in getDynamicTLSAddr() local
6961 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0); in getDynamicTLSAddr()
6963 SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_GD, DL, Ty, Addr), 0); in getDynamicTLSAddr()
6974 CLI.setDebugLoc(DL) in getDynamicTLSAddr()
6985 SDLoc DL(N); in getTLSDescAddr() local
6996 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0); in getTLSDescAddr()
6997 return SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLSDESC, DL, Ty, Addr), 0); in getTLSDescAddr()
7064 SDLoc DL(N); in combineSelectToBinOp() local
7069 SDValue Neg = DAG.getNegative(CondV, DL, VT); in combineSelectToBinOp()
7070 return DAG.getNode(ISD::OR, DL, VT, Neg, FalseV); in combineSelectToBinOp()
7074 SDValue Neg = DAG.getNode(ISD::ADD, DL, VT, CondV, in combineSelectToBinOp()
7075 DAG.getAllOnesConstant(DL, VT)); in combineSelectToBinOp()
7076 return DAG.getNode(ISD::OR, DL, VT, Neg, TrueV); in combineSelectToBinOp()
7081 SDValue Neg = DAG.getNode(ISD::ADD, DL, VT, CondV, in combineSelectToBinOp()
7082 DAG.getAllOnesConstant(DL, VT)); in combineSelectToBinOp()
7083 return DAG.getNode(ISD::AND, DL, VT, Neg, FalseV); in combineSelectToBinOp()
7087 SDValue Neg = DAG.getNegative(CondV, DL, VT); in combineSelectToBinOp()
7088 return DAG.getNode(ISD::AND, DL, VT, Neg, TrueV); in combineSelectToBinOp()
7103 return DAG.getNode(*MatchResult ? ISD::OR : ISD::AND, DL, VT, TrueV, in combineSelectToBinOp()
7109 return DAG.getNode(*MatchResult ? ISD::AND : ISD::OR, DL, VT, TrueV, in combineSelectToBinOp()
7155 SDLoc DL(Sel); in foldBinOpIntoSelectIfProfitable() local
7163 DAG.FoldConstantArithmetic(BO->getOpcode(), DL, VT, NewConstOps); in foldBinOpIntoSelectIfProfitable()
7175 SDValue NewNonConstOp = DAG.getNode(BO->getOpcode(), DL, VT, NewNonConstOps); in foldBinOpIntoSelectIfProfitable()
7179 return DAG.getSelect(DL, VT, Sel.getOperand(0), NewT, NewF); in foldBinOpIntoSelectIfProfitable()
7186 SDLoc DL(Op); in lowerSELECT() local
7193 SDValue CondSplat = DAG.getSplat(SplatCondVT, DL, CondV); in lowerSELECT()
7194 return DAG.getNode(ISD::VSELECT, DL, VT, CondSplat, TrueV, FalseV); in lowerSELECT()
7205 return DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, TrueV, CondV); in lowerSELECT()
7208 return DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, FalseV, CondV); in lowerSELECT()
7214 ISD::OR, DL, VT, TrueV, in lowerSELECT()
7215 DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, FalseV, CondV)); in lowerSELECT()
7220 ISD::OR, DL, VT, FalseV, in lowerSELECT()
7221 DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, TrueV, CondV)); in lowerSELECT()
7231 ISD::OR, DL, VT, in lowerSELECT()
7232 DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, TrueV, CondV), in lowerSELECT()
7233 DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, FalseV, CondV)); in lowerSELECT()
7257 return DAG.getNode(ISD::SINT_TO_FP, DL, VT, CondV); in lowerSELECT()
7259 SDValue XOR = DAG.getNode(ISD::XOR, DL, XLenVT, CondV, in lowerSELECT()
7260 DAG.getConstant(1, DL, XLenVT)); in lowerSELECT()
7261 return DAG.getNode(ISD::SINT_TO_FP, DL, VT, XOR); in lowerSELECT()
7271 SDValue Zero = DAG.getConstant(0, DL, XLenVT); in lowerSELECT()
7276 return DAG.getNode(RISCVISD::SELECT_CC, DL, VT, Ops); in lowerSELECT()
7300 return DAG.getNode(ISD::ADD, DL, VT, CondV, FalseV); in lowerSELECT()
7302 return DAG.getNode(ISD::SUB, DL, VT, FalseV, CondV); in lowerSELECT()
7305 translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG); in lowerSELECT()
7309 LHS = DAG.getConstant(0, DL, VT); in lowerSELECT()
7320 RHS = DAG.getConstant(0, DL, VT); in lowerSELECT()
7333 return DAG.getNode(RISCVISD::SELECT_CC, DL, VT, Ops); in lowerSELECT()
7338 SDLoc DL(Op); in lowerBRCOND() local
7347 translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG); in lowerBRCOND()
7350 return DAG.getNode(RISCVISD::BR_CC, DL, Op.getValueType(), Op.getOperand(0), in lowerBRCOND()
7354 return DAG.getNode(RISCVISD::BR_CC, DL, Op.getValueType(), Op.getOperand(0), in lowerBRCOND()
7355 CondV, DAG.getConstant(0, DL, XLenVT), in lowerBRCOND()
7363 SDLoc DL(Op); in lowerVASTART() local
7370 return DAG.getStore(Op.getOperand(0), DL, FI, Op.getOperand(1), in lowerVASTART()
7384 SDLoc DL(Op); in lowerFRAMEADDR() local
7385 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), DL, FrameReg, VT); in lowerFRAMEADDR()
7389 SDValue Ptr = DAG.getNode(ISD::ADD, DL, VT, FrameAddr, in lowerFRAMEADDR()
7390 DAG.getIntPtrConstant(Offset, DL)); in lowerFRAMEADDR()
7392 DAG.getLoad(VT, DL, DAG.getEntryNode(), Ptr, MachinePointerInfo()); in lowerFRAMEADDR()
7410 SDLoc DL(Op); in lowerRETURNADDR() local
7415 SDValue Offset = DAG.getConstant(Off, DL, VT); in lowerRETURNADDR()
7416 return DAG.getLoad(VT, DL, DAG.getEntryNode(), in lowerRETURNADDR()
7417 DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset), in lowerRETURNADDR()
7424 return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, XLenVT); in lowerRETURNADDR()
7429 SDLoc DL(Op); in lowerShiftLeftParts() local
7442 SDValue Zero = DAG.getConstant(0, DL, VT); in lowerShiftLeftParts()
7443 SDValue One = DAG.getConstant(1, DL, VT); in lowerShiftLeftParts()
7444 SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT); in lowerShiftLeftParts()
7445 SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT); in lowerShiftLeftParts()
7446 SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen); in lowerShiftLeftParts()
7447 SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt); in lowerShiftLeftParts()
7449 SDValue LoTrue = DAG.getNode(ISD::SHL, DL, VT, Lo, Shamt); in lowerShiftLeftParts()
7450 SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, VT, Lo, One); in lowerShiftLeftParts()
7452 DAG.getNode(ISD::SRL, DL, VT, ShiftRight1Lo, XLenMinus1Shamt); in lowerShiftLeftParts()
7453 SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, Hi, Shamt); in lowerShiftLeftParts()
7454 SDValue HiTrue = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo); in lowerShiftLeftParts()
7455 SDValue HiFalse = DAG.getNode(ISD::SHL, DL, VT, Lo, ShamtMinusXLen); in lowerShiftLeftParts()
7457 SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT); in lowerShiftLeftParts()
7459 Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, Zero); in lowerShiftLeftParts()
7460 Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse); in lowerShiftLeftParts()
7463 return DAG.getMergeValues(Parts, DL); in lowerShiftLeftParts()
7468 SDLoc DL(Op); in lowerShiftRightParts() local
7492 SDValue Zero = DAG.getConstant(0, DL, VT); in lowerShiftRightParts()
7493 SDValue One = DAG.getConstant(1, DL, VT); in lowerShiftRightParts()
7494 SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT); in lowerShiftRightParts()
7495 SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT); in lowerShiftRightParts()
7496 SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen); in lowerShiftRightParts()
7497 SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt); in lowerShiftRightParts()
7499 SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, Lo, Shamt); in lowerShiftRightParts()
7500 SDValue ShiftLeftHi1 = DAG.getNode(ISD::SHL, DL, VT, Hi, One); in lowerShiftRightParts()
7502 DAG.getNode(ISD::SHL, DL, VT, ShiftLeftHi1, XLenMinus1Shamt); in lowerShiftRightParts()
7503 SDValue LoTrue = DAG.getNode(ISD::OR, DL, VT, ShiftRightLo, ShiftLeftHi); in lowerShiftRightParts()
7504 SDValue HiTrue = DAG.getNode(ShiftRightOp, DL, VT, Hi, Shamt); in lowerShiftRightParts()
7505 SDValue LoFalse = DAG.getNode(ShiftRightOp, DL, VT, Hi, ShamtMinusXLen); in lowerShiftRightParts()
7507 IsSRA ? DAG.getNode(ISD::SRA, DL, VT, Hi, XLenMinus1) : Zero; in lowerShiftRightParts()
7509 SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT); in lowerShiftRightParts()
7511 Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, LoFalse); in lowerShiftRightParts()
7512 Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse); in lowerShiftRightParts()
7515 return DAG.getMergeValues(Parts, DL); in lowerShiftRightParts()
7522 SDLoc DL(Op); in lowerVectorMaskSplat() local
7527 SDValue VL = getDefaultScalableVLOps(VT, DL, DAG, Subtarget).second; in lowerVectorMaskSplat()
7528 return DAG.getNode(RISCVISD::VMSET_VL, DL, VT, VL); in lowerVectorMaskSplat()
7531 SDValue VL = getDefaultScalableVLOps(VT, DL, DAG, Subtarget).second; in lowerVectorMaskSplat()
7532 return DAG.getNode(RISCVISD::VMCLR_VL, DL, VT, VL); in lowerVectorMaskSplat()
7535 SplatVal = DAG.getNode(ISD::AND, DL, SplatVal.getValueType(), SplatVal, in lowerVectorMaskSplat()
7536 DAG.getConstant(1, DL, SplatVal.getValueType())); in lowerVectorMaskSplat()
7537 SDValue LHS = DAG.getSplatVector(InterVT, DL, SplatVal); in lowerVectorMaskSplat()
7538 SDValue Zero = DAG.getConstant(0, DL, InterVT); in lowerVectorMaskSplat()
7539 return DAG.getSetCC(DL, VT, LHS, Zero, ISD::SETNE); in lowerVectorMaskSplat()
7548 SDLoc DL(Op); in lowerSPLAT_VECTOR_PARTS() local
7561 auto VL = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).second; in lowerSPLAT_VECTOR_PARTS()
7564 splatPartsI64WithVL(DL, ContainerVT, SDValue(), Lo, Hi, VL, DAG); in lowerSPLAT_VECTOR_PARTS()
7578 SDLoc DL(Op); in lowerVectorMaskExt() local
7586 SDValue SplatZero = DAG.getConstant(0, DL, VecVT); in lowerVectorMaskExt()
7587 SDValue SplatTrueVal = DAG.getConstant(ExtTrueVal, DL, VecVT); in lowerVectorMaskExt()
7588 return DAG.getNode(ISD::VSELECT, DL, VecVT, Src, SplatTrueVal, SplatZero); in lowerVectorMaskExt()
7597 SDValue VL = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).second; in lowerVectorMaskExt()
7600 SDValue SplatZero = DAG.getConstant(0, DL, XLenVT); in lowerVectorMaskExt()
7601 SDValue SplatTrueVal = DAG.getConstant(ExtTrueVal, DL, XLenVT); in lowerVectorMaskExt()
7603 SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, in lowerVectorMaskExt()
7605 SplatTrueVal = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, in lowerVectorMaskExt()
7608 DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, CC, SplatTrueVal, in lowerVectorMaskExt()
7633 SDLoc DL(Op); in lowerFixedLengthVectorExtendToRVV() local
7634 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerFixedLengthVectorExtendToRVV()
7636 SDValue Ext = DAG.getNode(ExtendOpc, DL, ContainerExtVT, Op1, Mask, VL); in lowerFixedLengthVectorExtendToRVV()
7647 SDLoc DL(Op); in lowerVectorMaskTruncLike() local
7674 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget); in lowerVectorMaskTruncLike()
7677 SDValue SplatOne = DAG.getConstant(1, DL, Subtarget.getXLenVT()); in lowerVectorMaskTruncLike()
7678 SDValue SplatZero = DAG.getConstant(0, DL, Subtarget.getXLenVT()); in lowerVectorMaskTruncLike()
7680 SplatOne = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, in lowerVectorMaskTruncLike()
7682 SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, in lowerVectorMaskTruncLike()
7686 SDValue Trunc = DAG.getNode(RISCVISD::AND_VL, DL, ContainerVT, Src, SplatOne, in lowerVectorMaskTruncLike()
7688 Trunc = DAG.getNode(RISCVISD::SETCC_VL, DL, MaskContainerVT, in lowerVectorMaskTruncLike()
7699 SDLoc DL(Op); in lowerVectorTruncLike() local
7740 getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget); in lowerVectorTruncLike()
7748 Result = DAG.getNode(RISCVISD::TRUNCATE_VECTOR_VL, DL, ResultVT, Result, in lowerVectorTruncLike()
7761 SDLoc DL(Op); in lowerStrictFPExtendOrRoundLike() local
7774 auto [Mask, VL] = getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget); in lowerStrictFPExtendOrRoundLike()
7786 Src = DAG.getNode(InterConvOpc, DL, DAG.getVTList(InterVT, MVT::Other), in lowerStrictFPExtendOrRoundLike()
7794 SDValue Res = DAG.getNode(ConvOpc, DL, DAG.getVTList(ContainerVT, MVT::Other), in lowerStrictFPExtendOrRoundLike()
7800 Res = DAG.getMergeValues({SubVec, Res.getValue(1)}, DL); in lowerStrictFPExtendOrRoundLike()
7815 SDLoc DL(Op); in lowerVectorFPExtendOrRoundLike() local
7850 getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget); in lowerVectorFPExtendOrRoundLike()
7855 Src = DAG.getNode(ConvOpc, DL, ContainerVT, Src, Mask, VL); in lowerVectorFPExtendOrRoundLike()
7866 DAG.getNode(InterConvOpc, DL, InterVT, Src, Mask, VL); in lowerVectorFPExtendOrRoundLike()
7868 DAG.getNode(ConvOpc, DL, ContainerVT, IntermediateConv, Mask, VL); in lowerVectorFPExtendOrRoundLike()
7880 getSmallestVTForIndex(MVT VecVT, unsigned MaxIdx, SDLoc DL, SelectionDAG &DAG, in getSmallestVTForIndex() argument
7908 SDLoc DL(Op); in lowerINSERT_VECTOR_ELT() local
7918 Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Vec); in lowerINSERT_VECTOR_ELT()
7919 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, WideVT, Vec, Val, Idx); in lowerINSERT_VECTOR_ELT()
7920 return DAG.getNode(ISD::TRUNCATE, DL, VecVT, Vec); in lowerINSERT_VECTOR_ELT()
7939 DL, DAG, Subtarget)) { in lowerINSERT_VECTOR_ELT()
7941 AlignedIdx = DAG.getVectorIdxConstant(0, DL); in lowerINSERT_VECTOR_ELT()
7957 AlignedIdx = DAG.getVectorIdxConstant(ExtractIdx, DL); in lowerINSERT_VECTOR_ELT()
7958 Idx = DAG.getVectorIdxConstant(RemIdx, DL); in lowerINSERT_VECTOR_ELT()
7963 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ContainerVT, Vec, in lowerINSERT_VECTOR_ELT()
7978 Val = DAG.getConstant(CVal->getSExtValue(), DL, MVT::i32); in lowerINSERT_VECTOR_ELT()
7982 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget); in lowerINSERT_VECTOR_ELT()
7991 Val = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Val); in lowerINSERT_VECTOR_ELT()
7992 Vec = DAG.getNode(Opc, DL, ContainerVT, Vec, Val, VL); in lowerINSERT_VECTOR_ELT()
7995 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, OrigContainerVT, OrigVec, in lowerINSERT_VECTOR_ELT()
8001 ValInVec = lowerScalarInsert(Val, VL, ContainerVT, DL, DAG, Subtarget); in lowerINSERT_VECTOR_ELT()
8008 std::tie(ValLo, ValHi) = DAG.SplitScalar(Val, DL, MVT::i32, MVT::i32); in lowerINSERT_VECTOR_ELT()
8012 getDefaultScalableVLOps(I32ContainerVT, DL, DAG, Subtarget).first; in lowerINSERT_VECTOR_ELT()
8014 SDValue InsertI64VL = DAG.getConstant(2, DL, XLenVT); in lowerINSERT_VECTOR_ELT()
8019 ValInVec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32ContainerVT, in lowerINSERT_VECTOR_ELT()
8024 ValInVec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32ContainerVT, in lowerINSERT_VECTOR_ELT()
8031 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, OrigContainerVT, OrigVec, in lowerINSERT_VECTOR_ELT()
8040 ValInVec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32ContainerVT, in lowerINSERT_VECTOR_ELT()
8044 ValInVec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32ContainerVT, in lowerINSERT_VECTOR_ELT()
8053 DAG.getNode(ISD::ADD, DL, XLenVT, Idx, DAG.getConstant(1, DL, XLenVT)); in lowerINSERT_VECTOR_ELT()
8060 SDValue Slideup = getVSlideup(DAG, Subtarget, DL, ContainerVT, Vec, ValInVec, in lowerINSERT_VECTOR_ELT()
8064 Slideup = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, OrigContainerVT, OrigVec, in lowerINSERT_VECTOR_ELT()
8077 SDLoc DL(Op); in lowerEXTRACT_VECTOR_ELT() local
8092 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget); in lowerEXTRACT_VECTOR_ELT()
8094 DAG.getNode(RISCVISD::VFIRST_VL, DL, XLenVT, Vec, Mask, VL); in lowerEXTRACT_VECTOR_ELT()
8095 SDValue Res = DAG.getSetCC(DL, XLenVT, Vfirst, in lowerEXTRACT_VECTOR_ELT()
8096 DAG.getConstant(0, DL, XLenVT), ISD::SETEQ); in lowerEXTRACT_VECTOR_ELT()
8097 return DAG.getNode(ISD::TRUNCATE, DL, EltVT, Res); in lowerEXTRACT_VECTOR_ELT()
8114 ExtractElementIdx = DAG.getConstant(0, DL, XLenVT); in lowerEXTRACT_VECTOR_ELT()
8121 ISD::SRL, DL, XLenVT, Idx, in lowerEXTRACT_VECTOR_ELT()
8122 DAG.getConstant(Log2_64(WideEltVT.getSizeInBits()), DL, XLenVT)); in lowerEXTRACT_VECTOR_ELT()
8125 ISD::AND, DL, XLenVT, Idx, in lowerEXTRACT_VECTOR_ELT()
8126 DAG.getConstant(WideEltVT.getSizeInBits() - 1, DL, XLenVT)); in lowerEXTRACT_VECTOR_ELT()
8129 Vec = DAG.getNode(ISD::BITCAST, DL, WideVT, Vec); in lowerEXTRACT_VECTOR_ELT()
8130 SDValue ExtractElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, XLenVT, in lowerEXTRACT_VECTOR_ELT()
8134 DAG.getNode(ISD::SRL, DL, XLenVT, ExtractElt, ExtractBitIdx); in lowerEXTRACT_VECTOR_ELT()
8135 SDValue Res = DAG.getNode(ISD::AND, DL, XLenVT, ShiftRight, in lowerEXTRACT_VECTOR_ELT()
8136 DAG.getConstant(1, DL, XLenVT)); in lowerEXTRACT_VECTOR_ELT()
8137 return DAG.getNode(ISD::TRUNCATE, DL, EltVT, Res); in lowerEXTRACT_VECTOR_ELT()
8142 Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Vec); in lowerEXTRACT_VECTOR_ELT()
8143 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vec, Idx); in lowerEXTRACT_VECTOR_ELT()
8170 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, M1VT, Vec, in lowerEXTRACT_VECTOR_ELT()
8171 DAG.getVectorIdxConstant(ExtractIdx, DL)); in lowerEXTRACT_VECTOR_ELT()
8172 Idx = DAG.getVectorIdxConstant(RemIdx, DL); in lowerEXTRACT_VECTOR_ELT()
8185 getSmallestVTForIndex(ContainerVT, *MaxIdx, DL, DAG, Subtarget)) { in lowerEXTRACT_VECTOR_ELT()
8187 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ContainerVT, Vec, in lowerEXTRACT_VECTOR_ELT()
8188 DAG.getConstant(0, DL, XLenVT)); in lowerEXTRACT_VECTOR_ELT()
8211 auto [Mask, VL] = getDefaultVLOps(1, ContainerVT, DL, DAG, Subtarget); in lowerEXTRACT_VECTOR_ELT()
8212 Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT, in lowerEXTRACT_VECTOR_ELT()
8218 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vec, in lowerEXTRACT_VECTOR_ELT()
8219 DAG.getConstant(0, DL, XLenVT)); in lowerEXTRACT_VECTOR_ELT()
8222 SDValue Elt0 = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec); in lowerEXTRACT_VECTOR_ELT()
8223 return DAG.getNode(ISD::TRUNCATE, DL, EltVT, Elt0); in lowerEXTRACT_VECTOR_ELT()
8242 SDLoc DL(Op); in lowerVectorIntrinsicScalars() local
8269 ScalarOp = DAG.getNode(ExtOpc, DL, XLenVT, ScalarOp); in lowerVectorIntrinsicScalars()
8270 return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands); in lowerVectorIntrinsicScalars()
8289 ScalarOp = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, ScalarOp); in lowerVectorIntrinsicScalars()
8290 return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands); in lowerVectorIntrinsicScalars()
8307 DAG.SplitScalar(ScalarOp, DL, MVT::i32, MVT::i32); in lowerVectorIntrinsicScalars()
8320 I32VL = DAG.getConstant(2 * AVLInt, DL, XLenVT); in lowerVectorIntrinsicScalars()
8324 SDValue LMUL = DAG.getConstant(Lmul, DL, XLenVT); in lowerVectorIntrinsicScalars()
8326 SDValue SEW = DAG.getConstant(Sew, DL, XLenVT); in lowerVectorIntrinsicScalars()
8328 Intrinsic::riscv_vsetvlimax, DL, MVT::i32); in lowerVectorIntrinsicScalars()
8329 I32VL = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, XLenVT, SETVLMAX, SEW, in lowerVectorIntrinsicScalars()
8339 SDValue LMUL = DAG.getConstant(Lmul, DL, XLenVT); in lowerVectorIntrinsicScalars()
8341 SDValue SEW = DAG.getConstant(Sew, DL, XLenVT); in lowerVectorIntrinsicScalars()
8343 DAG.getTargetConstant(Intrinsic::riscv_vsetvli, DL, MVT::i32); in lowerVectorIntrinsicScalars()
8346 SDValue VL = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, XLenVT, SETVL, AVL, in lowerVectorIntrinsicScalars()
8349 DAG.getNode(ISD::SHL, DL, XLenVT, VL, DAG.getConstant(1, DL, XLenVT)); in lowerVectorIntrinsicScalars()
8352 SDValue I32Mask = getAllOnesMask(I32VT, I32VL, DL, DAG); in lowerVectorIntrinsicScalars()
8364 Vec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32VT, Passthru, Vec, in lowerVectorIntrinsicScalars()
8366 Vec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32VT, Passthru, Vec, in lowerVectorIntrinsicScalars()
8369 Vec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32VT, Passthru, Vec, in lowerVectorIntrinsicScalars()
8371 Vec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32VT, Passthru, Vec, in lowerVectorIntrinsicScalars()
8390 return DAG.getNode(RISCVISD::VMERGE_VL, DL, VT, Mask, Vec, MaskedOff, in lowerVectorIntrinsicScalars()
8394 return DAG.getNode(RISCVISD::VMERGE_VL, DL, VT, Mask, Vec, MaskedOff, in lowerVectorIntrinsicScalars()
8402 ScalarOp = splatSplitI64WithVL(DL, VT, SDValue(), ScalarOp, VL, DAG); in lowerVectorIntrinsicScalars()
8403 return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands); in lowerVectorIntrinsicScalars()
8438 SDLoc DL(N); in lowerGetVectorLength() local
8440 SDValue LMul = DAG.getTargetConstant(VLMUL, DL, XLenVT); in lowerGetVectorLength()
8441 SDValue Sew = DAG.getTargetConstant(VSEW, DL, XLenVT); in lowerGetVectorLength()
8443 SDValue AVL = DAG.getNode(ISD::ZERO_EXTEND, DL, XLenVT, N->getOperand(1)); in lowerGetVectorLength()
8445 SDValue ID = DAG.getTargetConstant(Intrinsic::riscv_vsetvli, DL, XLenVT); in lowerGetVectorLength()
8447 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, XLenVT, ID, AVL, Sew, LMul); in lowerGetVectorLength()
8448 return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), Res); in lowerGetVectorLength()
8453 SDLoc DL(Op); in getVCIXOperands() local
8484 SDLoc DL(Op); in LowerINTRINSIC_WO_CHAIN() local
8516 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(1)); in LowerINTRINSIC_WO_CHAIN()
8517 SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp); in LowerINTRINSIC_WO_CHAIN()
8518 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res); in LowerINTRINSIC_WO_CHAIN()
8521 return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1)); in LowerINTRINSIC_WO_CHAIN()
8530 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(1)); in LowerINTRINSIC_WO_CHAIN()
8532 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(2)); in LowerINTRINSIC_WO_CHAIN()
8534 DAG.getNode(Opc, DL, MVT::i64, NewOp0, NewOp1, Op.getOperand(3)); in LowerINTRINSIC_WO_CHAIN()
8535 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res); in LowerINTRINSIC_WO_CHAIN()
8538 return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1), Op.getOperand(2), in LowerINTRINSIC_WO_CHAIN()
8545 return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1)); in LowerINTRINSIC_WO_CHAIN()
8550 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(1)); in LowerINTRINSIC_WO_CHAIN()
8552 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(2)); in LowerINTRINSIC_WO_CHAIN()
8553 SDValue Res = DAG.getNode(RISCVISD::CLMUL, DL, MVT::i64, NewOp0, NewOp1); in LowerINTRINSIC_WO_CHAIN()
8554 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res); in LowerINTRINSIC_WO_CHAIN()
8556 return DAG.getNode(RISCVISD::CLMUL, DL, XLenVT, Op.getOperand(1), in LowerINTRINSIC_WO_CHAIN()
8564 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(1)); in LowerINTRINSIC_WO_CHAIN()
8566 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(2)); in LowerINTRINSIC_WO_CHAIN()
8567 NewOp0 = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp0, in LowerINTRINSIC_WO_CHAIN()
8568 DAG.getConstant(32, DL, MVT::i64)); in LowerINTRINSIC_WO_CHAIN()
8569 NewOp1 = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp1, in LowerINTRINSIC_WO_CHAIN()
8570 DAG.getConstant(32, DL, MVT::i64)); in LowerINTRINSIC_WO_CHAIN()
8571 SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp0, NewOp1); in LowerINTRINSIC_WO_CHAIN()
8572 Res = DAG.getNode(ISD::SRL, DL, MVT::i64, Res, in LowerINTRINSIC_WO_CHAIN()
8573 DAG.getConstant(32, DL, MVT::i64)); in LowerINTRINSIC_WO_CHAIN()
8574 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res); in LowerINTRINSIC_WO_CHAIN()
8577 return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1), Op.getOperand(2)); in LowerINTRINSIC_WO_CHAIN()
8582 SDValue Res = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Op.getOperand(1)); in LowerINTRINSIC_WO_CHAIN()
8583 return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Res); in LowerINTRINSIC_WO_CHAIN()
8586 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
8587 Op.getOperand(1), DAG.getConstant(0, DL, XLenVT)); in LowerINTRINSIC_WO_CHAIN()
8590 Op.getOperand(3), Op.getSimpleValueType(), DL, DAG, in LowerINTRINSIC_WO_CHAIN()
8593 return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
8599 Scalar = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Scalar); in LowerINTRINSIC_WO_CHAIN()
8600 return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
8625 SDValue SplattedVal = splatSplitI64WithVL(DL, VT, SDValue(), Scalar, VL, DAG); in LowerINTRINSIC_WO_CHAIN()
8629 DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, DAG.getUNDEF(VT), in LowerINTRINSIC_WO_CHAIN()
8630 DAG.getConstant(0, DL, MVT::i32), VL); in LowerINTRINSIC_WO_CHAIN()
8633 SDValue Mask = getAllOnesMask(VT, VL, DL, DAG); in LowerINTRINSIC_WO_CHAIN()
8634 SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, VT, Mask, VL); in LowerINTRINSIC_WO_CHAIN()
8636 DAG.getNode(RISCVISD::SETCC_VL, DL, MaskVT, in LowerINTRINSIC_WO_CHAIN()
8639 return DAG.getNode(RISCVISD::VMERGE_VL, DL, VT, SelectCond, SplattedVal, in LowerINTRINSIC_WO_CHAIN()
8643 return DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, Op.getSimpleValueType(), in LowerINTRINSIC_WO_CHAIN()
8713 SDValue NewNode = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, RetVT, Ops); in LowerINTRINSIC_WO_CHAIN()
8737 SDLoc DL(Op); in LowerINTRINSIC_W_CHAIN() local
8760 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in LowerINTRINSIC_W_CHAIN()
8770 DAG.getExtLoad(ISD::ZEXTLOAD, DL, XLenVT, Load->getChain(), Ptr, in LowerINTRINSIC_W_CHAIN()
8773 Result = lowerScalarSplat(SDValue(), ScalarLoad, VL, ContainerVT, DL, DAG, in LowerINTRINSIC_W_CHAIN()
8777 IsUnmasked ? Intrinsic::riscv_vlse : Intrinsic::riscv_vlse_mask, DL, in LowerINTRINSIC_W_CHAIN()
8792 DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT); in LowerINTRINSIC_W_CHAIN()
8798 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, in LowerINTRINSIC_W_CHAIN()
8804 return DAG.getMergeValues({Result, Chain}, DL); in LowerINTRINSIC_W_CHAIN()
8813 SDLoc DL(Op); in LowerINTRINSIC_W_CHAIN() local
8825 SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG, in LowerINTRINSIC_W_CHAIN()
8827 SDValue IntID = DAG.getTargetConstant(VlsegInts[NF - 2], DL, XLenVT); in LowerINTRINSIC_W_CHAIN()
8837 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, in LowerINTRINSIC_W_CHAIN()
8844 return DAG.getMergeValues(Results, DL); in LowerINTRINSIC_W_CHAIN()
8861 SDLoc DL(Op); in LowerINTRINSIC_W_CHAIN() local
8873 SDValue NewNode = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops); in LowerINTRINSIC_W_CHAIN()
8878 NewNode = DAG.getMergeValues({FixedVector, NewNode.getValue(1)}, DL); in LowerINTRINSIC_W_CHAIN()
8881 NewNode = DAG.getMergeValues({BitCast, NewNode.getValue(1)}, DL); in LowerINTRINSIC_W_CHAIN()
8901 SDLoc DL(Op); in LowerINTRINSIC_VOID() local
8922 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in LowerINTRINSIC_VOID()
8925 IsUnmasked ? Intrinsic::riscv_vsse : Intrinsic::riscv_vsse_mask, DL, in LowerINTRINSIC_VOID()
8937 return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, Store->getVTList(), in LowerINTRINSIC_VOID()
8948 SDLoc DL(Op); in LowerINTRINSIC_VOID() local
8961 SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG, in LowerINTRINSIC_VOID()
8963 SDValue IntID = DAG.getTargetConstant(VssegInts[NF - 2], DL, XLenVT); in LowerINTRINSIC_VOID()
8974 ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Ops, in LowerINTRINSIC_VOID()
9092 SDLoc DL(Op); in lowerVectorMaskVecReduction() local
9117 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget); in lowerVectorMaskVecReduction()
9122 SDValue Zero = DAG.getConstant(0, DL, XLenVT); in lowerVectorMaskVecReduction()
9130 SDValue TrueMask = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL); in lowerVectorMaskVecReduction()
9131 Vec = DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Vec, TrueMask, VL); in lowerVectorMaskVecReduction()
9132 Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL); in lowerVectorMaskVecReduction()
9140 Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL); in lowerVectorMaskVecReduction()
9147 SDValue One = DAG.getConstant(1, DL, XLenVT); in lowerVectorMaskVecReduction()
9148 Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL); in lowerVectorMaskVecReduction()
9149 Vec = DAG.getNode(ISD::AND, DL, XLenVT, Vec, One); in lowerVectorMaskVecReduction()
9156 SDValue SetCC = DAG.getSetCC(DL, XLenVT, Vec, Zero, CC); in lowerVectorMaskVecReduction()
9157 SetCC = DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), SetCC); in lowerVectorMaskVecReduction()
9168 return DAG.getNode(BaseOpc, DL, Op.getValueType(), SetCC, Op.getOperand(0)); in lowerVectorMaskVecReduction()
9182 SDValue VL, const SDLoc &DL, SelectionDAG &DAG, in lowerReductionSeq() argument
9195 auto InnerVL = NonZeroAVL ? VL : DAG.getConstant(1, DL, XLenVT); in lowerReductionSeq()
9196 SDValue InitialValue = lowerScalarInsert(StartValue, InnerVL, InnerVT, DL, in lowerReductionSeq()
9199 InitialValue = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, M1VT, in lowerReductionSeq()
9201 InitialValue, DAG.getConstant(0, DL, XLenVT)); in lowerReductionSeq()
9203 SDValue Policy = DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT); in lowerReductionSeq()
9205 SDValue Reduction = DAG.getNode(RVVOpcode, DL, M1VT, Ops); in lowerReductionSeq()
9206 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT, Reduction, in lowerReductionSeq()
9207 DAG.getConstant(0, DL, XLenVT)); in lowerReductionSeq()
9212 SDLoc DL(Op); in lowerVECREDUCE() local
9222 auto [Lo, Hi] = DAG.SplitVector(Vec, DL); in lowerVECREDUCE()
9224 Vec = DAG.getNode(BaseOpc, DL, VecEVT, Lo, Hi); in lowerVECREDUCE()
9242 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget); in lowerVECREDUCE()
9244 SDValue StartV = DAG.getNeutralElement(BaseOpc, DL, VecEltVT, SDNodeFlags()); in lowerVECREDUCE()
9253 StartV = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VecEltVT, Vec, in lowerVECREDUCE()
9254 DAG.getConstant(0, DL, XLenVT)); in lowerVECREDUCE()
9257 Mask, VL, DL, DAG, Subtarget); in lowerVECREDUCE()
9266 SDLoc DL(Op); in getRVVFPReductionOpAndOperands() local
9275 DAG.getConstantFP(Flags.hasNoSignedZeros() ? 0.0 : -0.0, DL, EltVT); in getRVVFPReductionOpAndOperands()
9285 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op.getOperand(0), in getRVVFPReductionOpAndOperands()
9286 DAG.getConstant(0, DL, XLenVT)); in getRVVFPReductionOpAndOperands()
9297 SDLoc DL(Op); in lowerFPVECREDUCE() local
9312 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget); in lowerFPVECREDUCE()
9314 VectorVal, Mask, VL, DL, DAG, Subtarget); in lowerFPVECREDUCE()
9319 SDLoc DL(Op); in lowerVPREDUCE() local
9339 Vec, Mask, VL, DL, DAG, Subtarget); in lowerVPREDUCE()
9349 SDLoc DL(Op); in lowerINSERT_SUBVECTOR() local
9382 Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVecVT, Vec); in lowerINSERT_SUBVECTOR()
9383 SubVec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtSubVecVT, SubVec); in lowerINSERT_SUBVECTOR()
9384 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ExtVecVT, Vec, SubVec, in lowerINSERT_SUBVECTOR()
9386 SDValue SplatZero = DAG.getConstant(0, DL, ExtVecVT); in lowerINSERT_SUBVECTOR()
9387 return DAG.getSetCC(DL, VecVT, Vec, SplatZero, ISD::SETNE); in lowerINSERT_SUBVECTOR()
9406 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT, in lowerINSERT_SUBVECTOR()
9408 DAG.getConstant(0, DL, XLenVT)); in lowerINSERT_SUBVECTOR()
9413 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT, in lowerINSERT_SUBVECTOR()
9415 DAG.getConstant(0, DL, XLenVT)); in lowerINSERT_SUBVECTOR()
9417 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first; in lowerINSERT_SUBVECTOR()
9421 SDValue VL = getVLOp(EndIndex, ContainerVT, DL, DAG, Subtarget); in lowerINSERT_SUBVECTOR()
9432 DAG.getNode(RISCVISD::VMV_V_V_VL, DL, ContainerVT, Vec, SubVec, VL); in lowerINSERT_SUBVECTOR()
9434 SDValue SlideupAmt = DAG.getConstant(OrigIdx, DL, XLenVT); in lowerINSERT_SUBVECTOR()
9435 SubVec = getVSlideup(DAG, Subtarget, DL, ContainerVT, Vec, SubVec, in lowerINSERT_SUBVECTOR()
9481 AlignedExtract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InterSubVT, Vec, in lowerINSERT_SUBVECTOR()
9482 DAG.getConstant(AlignedIdx, DL, XLenVT)); in lowerINSERT_SUBVECTOR()
9485 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, InterSubVT, in lowerINSERT_SUBVECTOR()
9487 DAG.getConstant(0, DL, XLenVT)); in lowerINSERT_SUBVECTOR()
9489 auto [Mask, VL] = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget); in lowerINSERT_SUBVECTOR()
9491 VL = computeVLMax(SubVecVT, DL, DAG); in lowerINSERT_SUBVECTOR()
9496 SubVec = DAG.getNode(RISCVISD::VMV_V_V_VL, DL, InterSubVT, AlignedExtract, in lowerINSERT_SUBVECTOR()
9500 DAG.getVScale(DL, XLenVT, APInt(XLenVT.getSizeInBits(), RemIdx)); in lowerINSERT_SUBVECTOR()
9503 VL = DAG.getNode(ISD::ADD, DL, XLenVT, SlideupAmt, VL); in lowerINSERT_SUBVECTOR()
9505 SubVec = getVSlideup(DAG, Subtarget, DL, InterSubVT, AlignedExtract, SubVec, in lowerINSERT_SUBVECTOR()
9512 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT, Vec, SubVec, in lowerINSERT_SUBVECTOR()
9513 DAG.getConstant(AlignedIdx, DL, XLenVT)); in lowerINSERT_SUBVECTOR()
9526 SDLoc DL(Op); in lowerEXTRACT_SUBVECTOR() local
9560 Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVecVT, Vec); in lowerEXTRACT_SUBVECTOR()
9561 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtSubVecVT, Vec, in lowerEXTRACT_SUBVECTOR()
9563 SDValue SplatZero = DAG.getConstant(0, DL, ExtSubVecVT); in lowerEXTRACT_SUBVECTOR()
9564 return DAG.getSetCC(DL, SubVecVT, Vec, SplatZero, ISD::SETNE); in lowerEXTRACT_SUBVECTOR()
9588 getSmallestVTForIndex(ContainerVT, LastIdx, DL, DAG, Subtarget)) { in lowerEXTRACT_SUBVECTOR()
9590 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ContainerVT, Vec, in lowerEXTRACT_SUBVECTOR()
9591 DAG.getVectorIdxConstant(0, DL)); in lowerEXTRACT_SUBVECTOR()
9595 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first; in lowerEXTRACT_SUBVECTOR()
9598 SDValue VL = getVLOp(SubVecVT.getVectorNumElements(), ContainerVT, DL, DAG, in lowerEXTRACT_SUBVECTOR()
9600 SDValue SlidedownAmt = DAG.getConstant(OrigIdx, DL, XLenVT); in lowerEXTRACT_SUBVECTOR()
9602 getVSlidedown(DAG, Subtarget, DL, ContainerVT, in lowerEXTRACT_SUBVECTOR()
9605 Slidedown = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVecVT, Slidedown, in lowerEXTRACT_SUBVECTOR()
9606 DAG.getConstant(0, DL, XLenVT)); in lowerEXTRACT_SUBVECTOR()
9632 Vec = DAG.getTargetExtractSubreg(SubRegIdx, DL, InterSubVT, Vec); in lowerEXTRACT_SUBVECTOR()
9638 DAG.getVScale(DL, XLenVT, APInt(XLenVT.getSizeInBits(), RemIdx)); in lowerEXTRACT_SUBVECTOR()
9640 auto [Mask, VL] = getDefaultScalableVLOps(InterSubVT, DL, DAG, Subtarget); in lowerEXTRACT_SUBVECTOR()
9642 getVSlidedown(DAG, Subtarget, DL, InterSubVT, DAG.getUNDEF(InterSubVT), in lowerEXTRACT_SUBVECTOR()
9647 Slidedown = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVecVT, Slidedown, in lowerEXTRACT_SUBVECTOR()
9648 DAG.getConstant(0, DL, XLenVT)); in lowerEXTRACT_SUBVECTOR()
9657 static SDValue widenVectorOpsToi8(SDValue N, const SDLoc &DL, in widenVectorOpsToi8() argument
9665 WideOps.push_back(DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Op)); in widenVectorOpsToi8()
9672 SDValue WideN = DAG.getNode(N.getOpcode(), DL, VTs, WideOps); in widenVectorOpsToi8()
9676 DAG.getSetCC(DL, N->getSimpleValueType(I), WideN.getValue(I), in widenVectorOpsToi8()
9677 DAG.getConstant(0, DL, WideVT), ISD::SETNE)); in widenVectorOpsToi8()
9681 return DAG.getMergeValues(TruncVals, DL); in widenVectorOpsToi8()
9687 SDLoc DL(Op); in lowerVECTOR_DEINTERLEAVE() local
9696 return widenVectorOpsToi8(Op, DL, DAG); in lowerVECTOR_DEINTERLEAVE()
9705 SDValue ResLo = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL, in lowerVECTOR_DEINTERLEAVE()
9707 SDValue ResHi = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL, in lowerVECTOR_DEINTERLEAVE()
9710 SDValue Even = DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, in lowerVECTOR_DEINTERLEAVE()
9712 SDValue Odd = DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, ResLo.getValue(1), in lowerVECTOR_DEINTERLEAVE()
9714 return DAG.getMergeValues({Even, Odd}, DL); in lowerVECTOR_DEINTERLEAVE()
9721 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, in lowerVECTOR_DEINTERLEAVE()
9725 auto [Mask, VL] = getDefaultScalableVLOps(ConcatVT, DL, DAG, Subtarget); in lowerVECTOR_DEINTERLEAVE()
9732 getDeinterleaveViaVNSRL(DL, VecVT, Concat, true, Subtarget, DAG); in lowerVECTOR_DEINTERLEAVE()
9734 getDeinterleaveViaVNSRL(DL, VecVT, Concat, false, Subtarget, DAG); in lowerVECTOR_DEINTERLEAVE()
9735 return DAG.getMergeValues({Even, Odd}, DL); in lowerVECTOR_DEINTERLEAVE()
9742 DAG.getStepVector(DL, IdxVT, APInt(IdxVT.getScalarSizeInBits(), 2)); in lowerVECTOR_DEINTERLEAVE()
9745 DAG.getNode(ISD::ADD, DL, IdxVT, EvenIdx, DAG.getConstant(1, DL, IdxVT)); in lowerVECTOR_DEINTERLEAVE()
9748 SDValue EvenWide = DAG.getNode(RISCVISD::VRGATHER_VV_VL, DL, ConcatVT, in lowerVECTOR_DEINTERLEAVE()
9750 SDValue OddWide = DAG.getNode(RISCVISD::VRGATHER_VV_VL, DL, ConcatVT, in lowerVECTOR_DEINTERLEAVE()
9754 SDValue Even = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VecVT, EvenWide, in lowerVECTOR_DEINTERLEAVE()
9755 DAG.getConstant(0, DL, XLenVT)); in lowerVECTOR_DEINTERLEAVE()
9756 SDValue Odd = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VecVT, OddWide, in lowerVECTOR_DEINTERLEAVE()
9757 DAG.getConstant(0, DL, XLenVT)); in lowerVECTOR_DEINTERLEAVE()
9759 return DAG.getMergeValues({Even, Odd}, DL); in lowerVECTOR_DEINTERLEAVE()
9764 SDLoc DL(Op); in lowerVECTOR_INTERLEAVE() local
9772 return widenVectorOpsToi8(Op, DL, DAG); in lowerVECTOR_INTERLEAVE()
9783 SDValue ResLo = DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, in lowerVECTOR_INTERLEAVE()
9785 SDValue ResHi = DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, in lowerVECTOR_INTERLEAVE()
9788 SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, in lowerVECTOR_INTERLEAVE()
9790 SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, in lowerVECTOR_INTERLEAVE()
9792 return DAG.getMergeValues({Lo, Hi}, DL); in lowerVECTOR_INTERLEAVE()
9800 Interleaved = getWideningInterleave(Op.getOperand(0), Op.getOperand(1), DL, in lowerVECTOR_INTERLEAVE()
9807 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, in lowerVECTOR_INTERLEAVE()
9813 SDValue StepVec = DAG.getStepVector(DL, IdxVT); in lowerVECTOR_INTERLEAVE()
9816 SDValue Ones = DAG.getSplatVector(IdxVT, DL, DAG.getConstant(1, DL, XLenVT)); in lowerVECTOR_INTERLEAVE()
9819 SDValue OddMask = DAG.getNode(ISD::AND, DL, IdxVT, StepVec, Ones); in lowerVECTOR_INTERLEAVE()
9821 DL, IdxVT.changeVectorElementType(MVT::i1), OddMask, in lowerVECTOR_INTERLEAVE()
9822 DAG.getSplatVector(IdxVT, DL, DAG.getConstant(0, DL, XLenVT)), in lowerVECTOR_INTERLEAVE()
9825 SDValue VLMax = DAG.getSplatVector(IdxVT, DL, computeVLMax(VecVT, DL, DAG)); in lowerVECTOR_INTERLEAVE()
9829 SDValue Idx = DAG.getNode(ISD::SRL, DL, IdxVT, StepVec, Ones); in lowerVECTOR_INTERLEAVE()
9832 DAG.getNode(RISCVISD::ADD_VL, DL, IdxVT, Idx, VLMax, Idx, OddMask, VL); in lowerVECTOR_INTERLEAVE()
9836 SDValue TrueMask = getAllOnesMask(IdxVT, VL, DL, DAG); in lowerVECTOR_INTERLEAVE()
9837 Interleaved = DAG.getNode(RISCVISD::VRGATHEREI16_VV_VL, DL, ConcatVT, in lowerVECTOR_INTERLEAVE()
9842 SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VecVT, Interleaved, in lowerVECTOR_INTERLEAVE()
9843 DAG.getVectorIdxConstant(0, DL)); in lowerVECTOR_INTERLEAVE()
9845 ISD::EXTRACT_SUBVECTOR, DL, VecVT, Interleaved, in lowerVECTOR_INTERLEAVE()
9846 DAG.getVectorIdxConstant(VecVT.getVectorMinNumElements(), DL)); in lowerVECTOR_INTERLEAVE()
9848 return DAG.getMergeValues({Lo, Hi}, DL); in lowerVECTOR_INTERLEAVE()
9855 SDLoc DL(Op); in lowerSTEP_VECTOR() local
9859 auto [Mask, VL] = getDefaultScalableVLOps(VT, DL, DAG, Subtarget); in lowerSTEP_VECTOR()
9860 SDValue StepVec = DAG.getNode(RISCVISD::VID_VL, DL, VT, Mask, VL); in lowerSTEP_VECTOR()
9865 DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, DAG.getUNDEF(VT), in lowerSTEP_VECTOR()
9866 DAG.getConstant(Log2_64(StepValImm), DL, XLenVT), VL); in lowerSTEP_VECTOR()
9867 StepVec = DAG.getNode(ISD::SHL, DL, VT, StepVec, StepVal); in lowerSTEP_VECTOR()
9870 SDValue(), DAG.getConstant(StepValImm, DL, VT.getVectorElementType()), in lowerSTEP_VECTOR()
9871 VL, VT, DL, DAG, Subtarget); in lowerSTEP_VECTOR()
9872 StepVec = DAG.getNode(ISD::MUL, DL, VT, StepVec, StepVal); in lowerSTEP_VECTOR()
9885 SDLoc DL(Op); in lowerVECTOR_REVERSE() local
9889 SDValue Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenVT, Op.getOperand(0)); in lowerVECTOR_REVERSE()
9890 SDValue Op2 = DAG.getNode(ISD::VECTOR_REVERSE, DL, WidenVT, Op1); in lowerVECTOR_REVERSE()
9891 return DAG.getNode(ISD::TRUNCATE, DL, VecVT, Op2); in lowerVECTOR_REVERSE()
9914 Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, LoVT, Lo); in lowerVECTOR_REVERSE()
9915 Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, HiVT, Hi); in lowerVECTOR_REVERSE()
9919 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT, DAG.getUNDEF(VecVT), Hi, in lowerVECTOR_REVERSE()
9920 DAG.getIntPtrConstant(0, DL)); in lowerVECTOR_REVERSE()
9922 ISD::INSERT_SUBVECTOR, DL, VecVT, Res, Lo, in lowerVECTOR_REVERSE()
9923 DAG.getIntPtrConstant(LoVT.getVectorMinNumElements(), DL)); in lowerVECTOR_REVERSE()
9932 auto [Mask, VL] = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget); in lowerVECTOR_REVERSE()
9935 SDValue VLMinus1 = DAG.getNode(ISD::SUB, DL, XLenVT, in lowerVECTOR_REVERSE()
9936 computeVLMax(VecVT, DL, DAG), in lowerVECTOR_REVERSE()
9937 DAG.getConstant(1, DL, XLenVT)); in lowerVECTOR_REVERSE()
9944 SplatVL = DAG.getSplatVector(IntVT, DL, VLMinus1); in lowerVECTOR_REVERSE()
9946 SplatVL = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IntVT, DAG.getUNDEF(IntVT), in lowerVECTOR_REVERSE()
9949 SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, IntVT, Mask, VL); in lowerVECTOR_REVERSE()
9950 SDValue Indices = DAG.getNode(RISCVISD::SUB_VL, DL, IntVT, SplatVL, VID, in lowerVECTOR_REVERSE()
9953 return DAG.getNode(GatherOpc, DL, VecVT, Op.getOperand(0), Indices, in lowerVECTOR_REVERSE()
9959 SDLoc DL(Op); in lowerVECTOR_SPLICE() local
9965 SDValue VLMax = computeVLMax(VecVT, DL, DAG); in lowerVECTOR_SPLICE()
9972 DownOffset = DAG.getConstant(ImmValue, DL, XLenVT); in lowerVECTOR_SPLICE()
9973 UpOffset = DAG.getNode(ISD::SUB, DL, XLenVT, VLMax, DownOffset); in lowerVECTOR_SPLICE()
9977 UpOffset = DAG.getConstant(-ImmValue, DL, XLenVT); in lowerVECTOR_SPLICE()
9978 DownOffset = DAG.getNode(ISD::SUB, DL, XLenVT, VLMax, UpOffset); in lowerVECTOR_SPLICE()
9981 SDValue TrueMask = getAllOnesMask(VecVT, VLMax, DL, DAG); in lowerVECTOR_SPLICE()
9984 getVSlidedown(DAG, Subtarget, DL, VecVT, DAG.getUNDEF(VecVT), V1, in lowerVECTOR_SPLICE()
9986 return getVSlideup(DAG, Subtarget, DL, VecVT, SlideDown, V2, UpOffset, in lowerVECTOR_SPLICE()
9994 SDLoc DL(Op); in lowerFixedLengthVectorLoadToRVV() local
10013 DAG.getLoad(ContainerVT, DL, Load->getChain(), Load->getBasePtr(), in lowerFixedLengthVectorLoadToRVV()
10016 return DAG.getMergeValues({Result, NewLoad.getValue(1)}, DL); in lowerFixedLengthVectorLoadToRVV()
10019 SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG, Subtarget); in lowerFixedLengthVectorLoadToRVV()
10023 IsMaskOp ? Intrinsic::riscv_vlm : Intrinsic::riscv_vle, DL, XLenVT); in lowerFixedLengthVectorLoadToRVV()
10031 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, in lowerFixedLengthVectorLoadToRVV()
10035 return DAG.getMergeValues({Result, NewLoad.getValue(1)}, DL); in lowerFixedLengthVectorLoadToRVV()
10041 SDLoc DL(Op); in lowerFixedLengthVectorStoreToRVV() local
10056 StoreVal = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, in lowerFixedLengthVectorStoreToRVV()
10057 DAG.getConstant(0, DL, VT), StoreVal, in lowerFixedLengthVectorStoreToRVV()
10058 DAG.getIntPtrConstant(0, DL)); in lowerFixedLengthVectorStoreToRVV()
10073 return DAG.getStore(Store->getChain(), DL, NewValue, Store->getBasePtr(), in lowerFixedLengthVectorStoreToRVV()
10076 SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG, in lowerFixedLengthVectorStoreToRVV()
10081 IsMaskOp ? Intrinsic::riscv_vsm : Intrinsic::riscv_vse, DL, XLenVT); in lowerFixedLengthVectorStoreToRVV()
10083 ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), in lowerFixedLengthVectorStoreToRVV()
10090 SDLoc DL(Op); in lowerMaskedLoad() local
10125 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in lowerMaskedLoad()
10129 SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)}; in lowerMaskedLoad()
10139 Ops.push_back(DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT)); in lowerMaskedLoad()
10144 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, MMO); in lowerMaskedLoad()
10150 return DAG.getMergeValues({Result, Chain}, DL); in lowerMaskedLoad()
10155 SDLoc DL(Op); in lowerMaskedStore() local
10191 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in lowerMaskedStore()
10195 SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)}; in lowerMaskedStore()
10202 return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, in lowerMaskedStore()
10219 SDLoc DL(Op); in lowerFixedLengthVectorSetccToRVV() local
10220 auto [Mask, VL] = getDefaultVLOps(VT.getVectorNumElements(), ContainerVT, DL, in lowerFixedLengthVectorSetccToRVV()
10225 DAG.getNode(RISCVISD::SETCC_VL, DL, MaskVT, in lowerFixedLengthVectorSetccToRVV()
10234 SDLoc DL(Op); in lowerVectorStrictFSetcc() local
10251 SDValue Tmp1 = DAG.getNode(ISD::STRICT_FSETCCS, DL, VTList, Chain, Op1, in lowerVectorStrictFSetcc()
10253 SDValue Tmp2 = DAG.getNode(ISD::STRICT_FSETCCS, DL, VTList, Chain, Op2, in lowerVectorStrictFSetcc()
10255 SDValue OutChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, in lowerVectorStrictFSetcc()
10259 Tmp1 = DAG.getNode(ISD::AND, DL, VT, Tmp1, Tmp2); in lowerVectorStrictFSetcc()
10260 return DAG.getMergeValues({Tmp1, OutChain}, DL); in lowerVectorStrictFSetcc()
10266 SDValue OEQ = DAG.getNode(ISD::STRICT_FSETCCS, DL, VTList, Chain, Op1, in lowerVectorStrictFSetcc()
10268 SDValue Res = DAG.getNOT(DL, OEQ, VT); in lowerVectorStrictFSetcc()
10269 return DAG.getMergeValues({Res, OEQ.getValue(1)}, DL); in lowerVectorStrictFSetcc()
10281 auto [Mask, VL] = getDefaultVLOps(InVT, ContainerInVT, DL, DAG, Subtarget); in lowerVectorStrictFSetcc()
10289 SDValue True = getAllOnesMask(ContainerInVT, VL, DL, DAG); in lowerVectorStrictFSetcc()
10291 RISCVISD::STRICT_FSETCC_VL, DL, DAG.getVTList(MaskVT, MVT::Other), in lowerVectorStrictFSetcc()
10295 RISCVISD::STRICT_FSETCC_VL, DL, DAG.getVTList(MaskVT, MVT::Other), in lowerVectorStrictFSetcc()
10299 DAG.getNode(RISCVISD::VMAND_VL, DL, MaskVT, OrderMask1, OrderMask2, VL); in lowerVectorStrictFSetcc()
10302 Res = DAG.getNode(RISCVISD::STRICT_FSETCCS_VL, DL, in lowerVectorStrictFSetcc()
10308 Res = DAG.getNode(RVVOpc, DL, DAG.getVTList(MaskVT, MVT::Other), in lowerVectorStrictFSetcc()
10314 return DAG.getMergeValues({SubVec, Res.getValue(1)}, DL); in lowerVectorStrictFSetcc()
10321 SDLoc DL(Op); in lowerABS() local
10342 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerABS()
10345 RISCVISD::VMV_V_X_VL, DL, ContainerVT, DAG.getUNDEF(ContainerVT), in lowerABS()
10346 DAG.getConstant(0, DL, Subtarget.getXLenVT()), VL); in lowerABS()
10347 SDValue NegX = DAG.getNode(RISCVISD::SUB_VL, DL, ContainerVT, SplatZero, X, in lowerABS()
10349 SDValue Max = DAG.getNode(RISCVISD::SMAX_VL, DL, ContainerVT, X, NegX, in lowerABS()
10359 SDLoc DL(Op); in lowerFixedLengthVectorFCOPYSIGNToRVV() local
10370 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerFixedLengthVectorFCOPYSIGNToRVV()
10372 SDValue CopySign = DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Mag, in lowerFixedLengthVectorFCOPYSIGNToRVV()
10393 SDLoc DL(Op); in lowerFixedLengthVectorSelectToRVV() local
10394 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in lowerFixedLengthVectorSelectToRVV()
10396 SDValue Select = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, CC, Op1, in lowerFixedLengthVectorSelectToRVV()
10428 SDLoc DL(Op); in lowerToScalableOp() local
10429 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); in lowerToScalableOp()
10440 DAG.getNode(NewOpc, DL, DAG.getVTList(ContainerVT, MVT::Other), Ops, in lowerToScalableOp()
10443 return DAG.getMergeValues({SubVec, ScalableRes.getValue(1)}, DL); in lowerToScalableOp()
10447 DAG.getNode(NewOpc, DL, ContainerVT, Ops, Op->getFlags()); in lowerToScalableOp()
10460 SDLoc DL(Op); in lowerVPOp() local
10504 return DAG.getNode(RISCVISDOpc, DL, VT, Ops, Op->getFlags()); in lowerVPOp()
10506 SDValue VPOp = DAG.getNode(RISCVISDOpc, DL, ContainerVT, Ops, Op->getFlags()); in lowerVPOp()
10513 SDLoc DL(Op); in lowerVPExtMaskOp() local
10528 SDValue Zero = DAG.getConstant(0, DL, XLenVT); in lowerVPExtMaskOp()
10529 SDValue ZeroSplat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, in lowerVPExtMaskOp()
10533 Op.getOpcode() == ISD::VP_ZERO_EXTEND ? 1 : -1, DL, XLenVT); in lowerVPExtMaskOp()
10534 SDValue Splat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, in lowerVPExtMaskOp()
10537 SDValue Result = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, Src, Splat, in lowerVPExtMaskOp()
10546 SDLoc DL(Op); in lowerVPSetCCMaskOp() local
10563 SDValue AllOneMask = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL); in lowerVPSetCCMaskOp()
10570 Result = DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op1, Op2, VL); in lowerVPSetCCMaskOp()
10575 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op1, Op2, VL); in lowerVPSetCCMaskOp()
10577 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Temp, AllOneMask, VL); in lowerVPSetCCMaskOp()
10585 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op1, AllOneMask, VL); in lowerVPSetCCMaskOp()
10586 Result = DAG.getNode(RISCVISD::VMAND_VL, DL, ContainerVT, Temp, Op2, VL); in lowerVPSetCCMaskOp()
10594 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op2, AllOneMask, VL); in lowerVPSetCCMaskOp()
10595 Result = DAG.getNode(RISCVISD::VMAND_VL, DL, ContainerVT, Op1, Temp, VL); in lowerVPSetCCMaskOp()
10603 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op1, AllOneMask, VL); in lowerVPSetCCMaskOp()
10604 Result = DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Temp, Op2, VL); in lowerVPSetCCMaskOp()
10612 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op2, AllOneMask, VL); in lowerVPSetCCMaskOp()
10613 Result = DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Temp, Op1, VL); in lowerVPSetCCMaskOp()
10626 SDLoc DL(Op); in lowerVPFPIntConvOp() local
10659 SDValue Zero = DAG.getConstant(0, DL, XLenVT); in lowerVPFPIntConvOp()
10660 SDValue ZeroSplat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IntVT, in lowerVPFPIntConvOp()
10663 RISCVISDExtOpc == RISCVISD::VZEXT_VL ? 1 : -1, DL, XLenVT); in lowerVPFPIntConvOp()
10664 SDValue OneSplat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IntVT, in lowerVPFPIntConvOp()
10666 Src = DAG.getNode(RISCVISD::VMERGE_VL, DL, IntVT, Src, OneSplat, in lowerVPFPIntConvOp()
10672 Src = DAG.getNode(RISCVISDExtOpc, DL, IntVT, Src, Mask, VL); in lowerVPFPIntConvOp()
10675 Result = DAG.getNode(RISCVISDOpc, DL, DstVT, Src, Mask, VL); in lowerVPFPIntConvOp()
10686 DAG.getNode(RISCVISD::FP_EXTEND_VL, DL, InterimFVT, Src, Mask, VL); in lowerVPFPIntConvOp()
10689 Result = DAG.getNode(RISCVISDOpc, DL, DstVT, Src, Mask, VL); in lowerVPFPIntConvOp()
10704 Result = DAG.getNode(RISCVISDOpc, DL, InterimFVT, Src, Mask, VL); in lowerVPFPIntConvOp()
10708 Result = DAG.getNode(RISCVISD::FP_ROUND_VL, DL, DstVT, Src, Mask, VL); in lowerVPFPIntConvOp()
10722 Result = DAG.getNode(RISCVISDOpc, DL, InterimIVT, Src, Mask, VL); in lowerVPFPIntConvOp()
10727 SDValue SplatZero = DAG.getConstant(0, DL, XLenVT); in lowerVPFPIntConvOp()
10728 SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, InterimIVT, in lowerVPFPIntConvOp()
10730 Result = DAG.getNode(RISCVISD::SETCC_VL, DL, DstVT, in lowerVPFPIntConvOp()
10737 Result = DAG.getNode(RISCVISDOpc, DL, InterimIVT, Src, Mask, VL); in lowerVPFPIntConvOp()
10744 Result = DAG.getNode(RISCVISD::TRUNCATE_VECTOR_VL, DL, InterimIVT, in lowerVPFPIntConvOp()
10760 SDLoc DL(Op); in lowerVPSpliceExperimental() local
10785 SDValue SplatOneOp1 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, in lowerVPSpliceExperimental()
10787 DAG.getConstant(1, DL, XLenVT), EVL1); in lowerVPSpliceExperimental()
10788 SDValue SplatZeroOp1 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, in lowerVPSpliceExperimental()
10790 DAG.getConstant(0, DL, XLenVT), EVL1); in lowerVPSpliceExperimental()
10791 Op1 = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, Op1, SplatOneOp1, in lowerVPSpliceExperimental()
10794 SDValue SplatOneOp2 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, in lowerVPSpliceExperimental()
10796 DAG.getConstant(1, DL, XLenVT), EVL2); in lowerVPSpliceExperimental()
10797 SDValue SplatZeroOp2 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, in lowerVPSpliceExperimental()
10799 DAG.getConstant(0, DL, XLenVT), EVL2); in lowerVPSpliceExperimental()
10800 Op2 = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, Op2, SplatOneOp2, in lowerVPSpliceExperimental()
10809 DownOffset = DAG.getConstant(ImmValue, DL, XLenVT); in lowerVPSpliceExperimental()
10810 UpOffset = DAG.getNode(ISD::SUB, DL, XLenVT, EVL1, DownOffset); in lowerVPSpliceExperimental()
10814 UpOffset = DAG.getConstant(-ImmValue, DL, XLenVT); in lowerVPSpliceExperimental()
10815 DownOffset = DAG.getNode(ISD::SUB, DL, XLenVT, EVL1, UpOffset); in lowerVPSpliceExperimental()
10819 getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT), in lowerVPSpliceExperimental()
10821 SDValue Result = getVSlideup(DAG, Subtarget, DL, ContainerVT, SlideDown, Op2, in lowerVPSpliceExperimental()
10827 RISCVISD::SETCC_VL, DL, ContainerVT.changeVectorElementType(MVT::i1), in lowerVPSpliceExperimental()
10828 {Result, DAG.getConstant(0, DL, ContainerVT), in lowerVPSpliceExperimental()
10841 SDLoc DL(Op); in lowerVPReverseExperimental() local
10865 SDValue SplatOne = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IndicesVT, in lowerVPReverseExperimental()
10867 DAG.getConstant(1, DL, XLenVT), EVL); in lowerVPReverseExperimental()
10868 SDValue SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IndicesVT, in lowerVPReverseExperimental()
10870 DAG.getConstant(0, DL, XLenVT), EVL); in lowerVPReverseExperimental()
10871 Op1 = DAG.getNode(RISCVISD::VMERGE_VL, DL, IndicesVT, Op1, SplatOne, in lowerVPReverseExperimental()
10895 auto [Lo, Hi] = DAG.SplitVector(Op1, DL); in lowerVPReverseExperimental()
10897 SDValue LoRev = DAG.getNode(ISD::VECTOR_REVERSE, DL, LoVT, Lo); in lowerVPReverseExperimental()
10898 SDValue HiRev = DAG.getNode(ISD::VECTOR_REVERSE, DL, HiVT, Hi); in lowerVPReverseExperimental()
10905 DAG.getNode(ISD::CONCAT_VECTORS, DL, GatherVT, HiRev, LoRev); in lowerVPReverseExperimental()
10910 SDValue VLMax = DAG.getNode(ISD::VSCALE, DL, XLenVT, in lowerVPReverseExperimental()
10911 DAG.getConstant(MinElts, DL, XLenVT)); in lowerVPReverseExperimental()
10912 SDValue Diff = DAG.getNode(ISD::SUB, DL, XLenVT, VLMax, EVL); in lowerVPReverseExperimental()
10914 Result = getVSlidedown(DAG, Subtarget, DL, GatherVT, in lowerVPReverseExperimental()
10920 DAG.getNode(RISCVISD::SETCC_VL, DL, ContainerVT, in lowerVPReverseExperimental()
10921 {Result, DAG.getConstant(0, DL, GatherVT), in lowerVPReverseExperimental()
10936 SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, IndicesVT, Mask, EVL); in lowerVPReverseExperimental()
10938 DAG.getNode(ISD::SUB, DL, XLenVT, EVL, DAG.getConstant(1, DL, XLenVT)); in lowerVPReverseExperimental()
10939 SDValue VecLenSplat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IndicesVT, in lowerVPReverseExperimental()
10941 SDValue VRSUB = DAG.getNode(RISCVISD::SUB_VL, DL, IndicesVT, VecLenSplat, VID, in lowerVPReverseExperimental()
10943 SDValue Result = DAG.getNode(GatherOpc, DL, GatherVT, Op1, VRSUB, in lowerVPReverseExperimental()
10949 RISCVISD::SETCC_VL, DL, ContainerVT, in lowerVPReverseExperimental()
10950 {Result, DAG.getConstant(0, DL, GatherVT), DAG.getCondCode(ISD::SETNE), in lowerVPReverseExperimental()
10978 SDLoc DL(Op); in lowerLogicVPOp() local
10979 SDValue Val = DAG.getNode(getRISCVVLOp(Op), DL, ContainerVT, Op1, Op2, VL); in lowerLogicVPOp()
10987 SDLoc DL(Op); in lowerVPStridedLoad() local
11003 DL, XLenVT); in lowerVPStridedLoad()
11016 SDValue Policy = DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT); in lowerVPStridedLoad()
11021 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, in lowerVPStridedLoad()
11028 return DAG.getMergeValues({Result, Chain}, DL); in lowerVPStridedLoad()
11033 SDLoc DL(Op); in lowerVPStridedStore() local
11051 DL, XLenVT); in lowerVPStridedStore()
11063 return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, VPNode->getVTList(), in lowerVPStridedStore()
11076 SDLoc DL(Op); in lowerMaskedGather() local
11135 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in lowerMaskedGather()
11139 Index = DAG.getNode(ISD::TRUNCATE, DL, IndexVT, Index); in lowerMaskedGather()
11144 SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)}; in lowerMaskedGather()
11155 Ops.push_back(DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT)); in lowerMaskedGather()
11159 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, MMO); in lowerMaskedGather()
11165 return DAG.getMergeValues({Result, Chain}, DL); in lowerMaskedGather()
11176 SDLoc DL(Op); in lowerMaskedScatter() local
11234 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; in lowerMaskedScatter()
11238 Index = DAG.getNode(ISD::TRUNCATE, DL, IndexVT, Index); in lowerMaskedScatter()
11243 SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)}; in lowerMaskedScatter()
11251 return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, in lowerMaskedScatter()
11258 SDLoc DL(Op); in lowerGET_ROUNDING() local
11261 RISCVSysReg::lookupSysRegByName("FRM")->Encoding, DL, XLenVT); in lowerGET_ROUNDING()
11263 SDValue RM = DAG.getNode(RISCVISD::READ_CSR, DL, VTs, Chain, SysRegNo); in lowerGET_ROUNDING()
11277 DAG.getNode(ISD::SHL, DL, XLenVT, RM, DAG.getConstant(2, DL, XLenVT)); in lowerGET_ROUNDING()
11278 SDValue Shifted = DAG.getNode(ISD::SRL, DL, XLenVT, in lowerGET_ROUNDING()
11279 DAG.getConstant(Table, DL, XLenVT), Shift); in lowerGET_ROUNDING()
11280 SDValue Masked = DAG.getNode(ISD::AND, DL, XLenVT, Shifted, in lowerGET_ROUNDING()
11281 DAG.getConstant(7, DL, XLenVT)); in lowerGET_ROUNDING()
11283 return DAG.getMergeValues({Masked, Chain}, DL); in lowerGET_ROUNDING()
11289 SDLoc DL(Op); in lowerSET_ROUNDING() local
11293 RISCVSysReg::lookupSysRegByName("FRM")->Encoding, DL, XLenVT); in lowerSET_ROUNDING()
11306 RMValue = DAG.getNode(ISD::ZERO_EXTEND, DL, XLenVT, RMValue); in lowerSET_ROUNDING()
11308 SDValue Shift = DAG.getNode(ISD::SHL, DL, XLenVT, RMValue, in lowerSET_ROUNDING()
11309 DAG.getConstant(2, DL, XLenVT)); in lowerSET_ROUNDING()
11310 SDValue Shifted = DAG.getNode(ISD::SRL, DL, XLenVT, in lowerSET_ROUNDING()
11311 DAG.getConstant(Table, DL, XLenVT), Shift); in lowerSET_ROUNDING()
11312 RMValue = DAG.getNode(ISD::AND, DL, XLenVT, Shifted, in lowerSET_ROUNDING()
11313 DAG.getConstant(0x7, DL, XLenVT)); in lowerSET_ROUNDING()
11314 return DAG.getNode(RISCVISD::WRITE_CSR, DL, MVT::Other, Chain, SysRegNo, in lowerSET_ROUNDING()
11361 SDLoc DL(N); in customLegalizeToWOp() local
11363 SDValue NewOp0 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(0)); in customLegalizeToWOp()
11364 SDValue NewOp1 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(1)); in customLegalizeToWOp()
11365 SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1); in customLegalizeToWOp()
11367 return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewRes); in customLegalizeToWOp()
11373 SDLoc DL(N); in customLegalizeToWOpWithSExt() local
11374 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0)); in customLegalizeToWOpWithSExt()
11375 SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1)); in customLegalizeToWOpWithSExt()
11376 SDValue NewWOp = DAG.getNode(N->getOpcode(), DL, MVT::i64, NewOp0, NewOp1); in customLegalizeToWOpWithSExt()
11377 SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp, in customLegalizeToWOpWithSExt()
11379 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes); in customLegalizeToWOpWithSExt()
11385 SDLoc DL(N); in ReplaceNodeResults() local
11408 Op0 = DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {MVT::f32, MVT::Other}, in ReplaceNodeResults()
11416 Opc, DL, VTs, Chain, Op0, in ReplaceNodeResults()
11417 DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, MVT::i64)); in ReplaceNodeResults()
11418 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); in ReplaceNodeResults()
11427 Op0 = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op0); in ReplaceNodeResults()
11431 DAG.getNode(Opc, DL, MVT::i64, Op0, in ReplaceNodeResults()
11432 DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, MVT::i64)); in ReplaceNodeResults()
11433 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); in ReplaceNodeResults()
11451 makeLibCall(DAG, LC, N->getValueType(0), Op0, CallOptions, DL, Chain); in ReplaceNodeResults()
11467 Op0 = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op0); in ReplaceNodeResults()
11470 DAG.getNode(RISCVISD::FCVT_W_RV64, DL, MVT::i64, Op0, in ReplaceNodeResults()
11471 DAG.getTargetConstant(RISCVFPRndMode::RMM, DL, MVT::i64)); in ReplaceNodeResults()
11472 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); in ReplaceNodeResults()
11483 SDValue Result = makeLibCall(DAG, LC, MVT::i64, Op0, CallOptions, DL).first; in ReplaceNodeResults()
11484 Result = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Result); in ReplaceNodeResults()
11494 DAG.getNode(RISCVISD::READ_CYCLE_WIDE, DL, VTs, N->getOperand(0)); in ReplaceNodeResults()
11497 DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, RCW, RCW.getValue(1))); in ReplaceNodeResults()
11535 S = DAG.getNode(ISD::TRUNCATE, DL, XLenVT, S); in ReplaceNodeResults()
11536 U = DAG.getNode(ISD::TRUNCATE, DL, XLenVT, U); in ReplaceNodeResults()
11537 SDValue Lo = DAG.getNode(ISD::MUL, DL, XLenVT, S, U); in ReplaceNodeResults()
11538 SDValue Hi = DAG.getNode(RISCVISD::MULHSU, DL, XLenVT, S, U); in ReplaceNodeResults()
11539 return DAG.getNode(ISD::BUILD_PAIR, DL, N->getValueType(0), Lo, Hi); in ReplaceNodeResults()
11580 SDLoc DL(N); in ReplaceNodeResults() local
11582 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0)); in ReplaceNodeResults()
11584 DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N->getOperand(1)); in ReplaceNodeResults()
11585 SDValue NewWOp = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp0, NewOp1); in ReplaceNodeResults()
11586 SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp, in ReplaceNodeResults()
11588 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes)); in ReplaceNodeResults()
11612 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0)); in ReplaceNodeResults()
11616 SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp0); in ReplaceNodeResults()
11617 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); in ReplaceNodeResults()
11654 SDValue LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0)); in ReplaceNodeResults()
11655 SDValue RHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(1)); in ReplaceNodeResults()
11656 SDValue Res = DAG.getNode(ISD::ADD, DL, MVT::i64, LHS, RHS); in ReplaceNodeResults()
11657 Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Res, in ReplaceNodeResults()
11660 SDValue Zero = DAG.getConstant(0, DL, MVT::i64); in ReplaceNodeResults()
11669 SDValue ResultLowerThanLHS = DAG.getSetCC(DL, OType, Res, LHS, ISD::SETLT); in ReplaceNodeResults()
11670 SDValue ConditionRHS = DAG.getSetCC(DL, OType, RHS, Zero, ISD::SETLT); in ReplaceNodeResults()
11673 DAG.getNode(ISD::XOR, DL, OType, ConditionRHS, ResultLowerThanLHS); in ReplaceNodeResults()
11674 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); in ReplaceNodeResults()
11684 SDValue LHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0)); in ReplaceNodeResults()
11685 SDValue RHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1)); in ReplaceNodeResults()
11687 DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, DL, MVT::i64, LHS, RHS); in ReplaceNodeResults()
11688 Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Res, in ReplaceNodeResults()
11698 Overflow = DAG.getSetCC(DL, N->getValueType(1), Res, in ReplaceNodeResults()
11699 DAG.getConstant(0, DL, MVT::i64), ISD::SETEQ); in ReplaceNodeResults()
11702 Overflow = DAG.getSetCC(DL, N->getValueType(1), N->getOperand(0), in ReplaceNodeResults()
11703 DAG.getConstant(0, DL, MVT::i32), ISD::SETNE); in ReplaceNodeResults()
11708 LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0)); in ReplaceNodeResults()
11709 Overflow = DAG.getSetCC(DL, N->getValueType(1), Res, LHS, in ReplaceNodeResults()
11713 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); in ReplaceNodeResults()
11726 DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0)); in ReplaceNodeResults()
11728 DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(1)); in ReplaceNodeResults()
11729 SDValue Res = DAG.getNode(N->getOpcode(), DL, MVT::i64, LHS, RHS); in ReplaceNodeResults()
11730 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); in ReplaceNodeResults()
11747 SDValue Src = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, in ReplaceNodeResults()
11749 SDValue Abs = DAG.getNode(RISCVISD::ABSW, DL, MVT::i64, Src); in ReplaceNodeResults()
11750 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Abs)); in ReplaceNodeResults()
11755 SDValue Src = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0)); in ReplaceNodeResults()
11761 SDValue SignFill = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Src, in ReplaceNodeResults()
11763 SignFill = DAG.getNode(ISD::SRA, DL, MVT::i64, SignFill, in ReplaceNodeResults()
11764 DAG.getConstant(31, DL, MVT::i64)); in ReplaceNodeResults()
11766 SDValue NewRes = DAG.getNode(ISD::XOR, DL, MVT::i64, Src, SignFill); in ReplaceNodeResults()
11767 NewRes = DAG.getNode(ISD::SUB, DL, MVT::i64, NewRes, SignFill); in ReplaceNodeResults()
11771 NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewRes, in ReplaceNodeResults()
11773 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes)); in ReplaceNodeResults()
11784 SDValue FPConv = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Op0); in ReplaceNodeResults()
11785 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FPConv)); in ReplaceNodeResults()
11788 SDValue FPConv = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Op0); in ReplaceNodeResults()
11789 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FPConv)); in ReplaceNodeResults()
11793 DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Op0); in ReplaceNodeResults()
11794 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, FPConv)); in ReplaceNodeResults()
11797 SDValue NewReg = DAG.getNode(RISCVISD::SplitF64, DL, in ReplaceNodeResults()
11799 SDValue RetReg = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, in ReplaceNodeResults()
11811 Results.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, BVec, in ReplaceNodeResults()
11812 DAG.getConstant(0, DL, XLenVT))); in ReplaceNodeResults()
11823 SDValue NewOp = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, N->getOperand(0)); in ReplaceNodeResults()
11824 SDValue NewRes = DAG.getNode(N->getOpcode(), DL, XLenVT, NewOp); in ReplaceNodeResults()
11827 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, NewRes)); in ReplaceNodeResults()
11862 auto [Mask, VL] = getDefaultVLOps(1, ContainerVT, DL, DAG, Subtarget); in ReplaceNodeResults()
11867 Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT, in ReplaceNodeResults()
11872 SDValue EltLo = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec); in ReplaceNodeResults()
11876 SDValue ThirtyTwoV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, in ReplaceNodeResults()
11878 DAG.getConstant(32, DL, XLenVT), VL); in ReplaceNodeResults()
11880 DAG.getNode(RISCVISD::SRL_VL, DL, ContainerVT, Vec, ThirtyTwoV, in ReplaceNodeResults()
11883 SDValue EltHi = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, LShr32); in ReplaceNodeResults()
11885 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, EltLo, EltHi)); in ReplaceNodeResults()
11896 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); in ReplaceNodeResults()
11922 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1)); in ReplaceNodeResults()
11923 SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp); in ReplaceNodeResults()
11924 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); in ReplaceNodeResults()
11932 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1)); in ReplaceNodeResults()
11934 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2)); in ReplaceNodeResults()
11936 DAG.getNode(Opc, DL, MVT::i64, NewOp0, NewOp1, N->getOperand(3)); in ReplaceNodeResults()
11937 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); in ReplaceNodeResults()
11945 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1)); in ReplaceNodeResults()
11947 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2)); in ReplaceNodeResults()
11948 SDValue Res = DAG.getNode(RISCVISD::CLMUL, DL, MVT::i64, NewOp0, NewOp1); in ReplaceNodeResults()
11949 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); in ReplaceNodeResults()
11967 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1)); in ReplaceNodeResults()
11969 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2)); in ReplaceNodeResults()
11970 NewOp0 = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp0, in ReplaceNodeResults()
11971 DAG.getConstant(32, DL, MVT::i64)); in ReplaceNodeResults()
11972 NewOp1 = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp1, in ReplaceNodeResults()
11973 DAG.getConstant(32, DL, MVT::i64)); in ReplaceNodeResults()
11976 SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp0, NewOp1); in ReplaceNodeResults()
11977 Res = DAG.getNode(ISD::SRL, DL, MVT::i64, Res, in ReplaceNodeResults()
11978 DAG.getConstant(32, DL, MVT::i64)); in ReplaceNodeResults()
11979 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); in ReplaceNodeResults()
11987 SDValue Extract = DAG.getNode(RISCVISD::VMV_X_S, DL, in ReplaceNodeResults()
11989 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, Extract)); in ReplaceNodeResults()
12001 SDValue EltLo = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec); in ReplaceNodeResults()
12005 auto [Mask, VL] = getDefaultVLOps(1, VecVT, DL, DAG, Subtarget); in ReplaceNodeResults()
12008 DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VecVT, DAG.getUNDEF(VecVT), in ReplaceNodeResults()
12009 DAG.getConstant(32, DL, XLenVT), VL); in ReplaceNodeResults()
12010 SDValue LShr32 = DAG.getNode(RISCVISD::SRL_VL, DL, VecVT, Vec, ThirtyTwoV, in ReplaceNodeResults()
12012 SDValue EltHi = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, LShr32); in ReplaceNodeResults()
12015 DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, EltLo, EltHi)); in ReplaceNodeResults()
12045 SDValue Res = DAG.getNode(ISD::GET_ROUNDING, DL, VTs, N->getOperand(0)); in ReplaceNodeResults()
12102 const SDLoc DL(N); in combineBinOpOfExtractToReduceTree() local
12150 SDValue Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ReduceVT, SrcVec, in combineBinOpOfExtractToReduceTree()
12151 DAG.getVectorIdxConstant(0, DL)); in combineBinOpOfExtractToReduceTree()
12152 return DAG.getNode(ReduceOpc, DL, VT, Vec, N->getFlags()); in combineBinOpOfExtractToReduceTree()
12172 SDValue Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ReduceVT, SrcVec, in combineBinOpOfExtractToReduceTree()
12173 DAG.getVectorIdxConstant(0, DL)); in combineBinOpOfExtractToReduceTree()
12176 return DAG.getNode(ReduceOpc, DL, VT, Vec, Flags); in combineBinOpOfExtractToReduceTree()
12268 SDLoc DL(N); in combineBinOpToReduce() local
12271 ScalarV.getSimpleValueType(), DL, DAG, Subtarget); in combineBinOpToReduce()
12276 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ScalarVT, DAG.getUNDEF(ScalarVT), in combineBinOpToReduce()
12277 NewScalarV, DAG.getConstant(0, DL, Subtarget.getXLenVT())); in combineBinOpToReduce()
12283 DAG.getNode(Reduce.getOpcode(), DL, Reduce.getValueType(), Ops); in combineBinOpToReduce()
12284 return DAG.getNode(Extract.getOpcode(), DL, Extract.getValueType(), NewReduce, in combineBinOpToReduce()
12325 SDLoc DL(N); in transformAddShlImm() local
12329 DAG.getNode(ISD::SHL, DL, VT, NL, DAG.getConstant(Diff, DL, VT)); in transformAddShlImm()
12330 SDValue NA1 = DAG.getNode(ISD::ADD, DL, VT, NA0, NS); in transformAddShlImm()
12331 return DAG.getNode(ISD::SHL, DL, VT, NA1, DAG.getConstant(Bits, DL, VT)); in transformAddShlImm()
12482 SDLoc DL(N); in transformAddImmMulImm() local
12483 SDValue New0 = DAG.getNode(ISD::ADD, DL, VT, N0->getOperand(0), in transformAddImmMulImm()
12484 DAG.getConstant(CA, DL, VT)); in transformAddImmMulImm()
12486 DAG.getNode(ISD::MUL, DL, VT, New0, DAG.getConstant(C0, DL, VT)); in transformAddImmMulImm()
12487 return DAG.getNode(ISD::ADD, DL, VT, New1, DAG.getConstant(CB, DL, VT)); in transformAddImmMulImm()
12495 SDLoc DL(N); in combineAddOfBooleanXor() local
12511 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), in combineAddOfBooleanXor()
12538 SDLoc DL(N); in combineSubOfBoolean() local
12570 SDValue NewRHS = DAG.getConstant(ImmValMinus1, DL, VT); in combineSubOfBoolean()
12571 return DAG.getNode(ISD::ADD, DL, VT, NewLHS, NewRHS); in combineSubOfBoolean()
12587 SDLoc DL(N); in performSUBCombine() local
12589 return DAG.getNode(ISD::SRA, DL, VT, N1.getOperand(0), in performSUBCombine()
12590 DAG.getConstant(ShAmt, DL, VT)); in performSUBCombine()
12640 SDLoc DL(N); in combineDeMorganOfBoolean() local
12642 SDValue Logic = DAG.getNode(Opc, DL, VT, N00, N10); in combineDeMorganOfBoolean()
12643 return DAG.getNode(ISD::XOR, DL, VT, Logic, DAG.getConstant(1, DL, VT)); in combineDeMorganOfBoolean()
12659 SDLoc DL(N0); in performTRUNCATECombine() local
12660 SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N0.getOperand(0)); in performTRUNCATECombine()
12661 SDValue Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N0.getOperand(1)); in performTRUNCATECombine()
12662 SDValue Srl = DAG.getNode(ISD::SRL, DL, MVT::i64, Op0, Op1); in performTRUNCATECombine()
12687 SDLoc DL(N); in performANDCombine() local
12688 SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N0.getOperand(0)); in performANDCombine()
12689 SDValue Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N0.getOperand(1)); in performANDCombine()
12690 SDValue Srl = DAG.getNode(ISD::SRL, DL, MVT::i64, Op0, Op1); in performANDCombine()
12691 SDValue And = DAG.getNode(ISD::AND, DL, MVT::i64, Srl, in performANDCombine()
12692 DAG.getConstant(1, DL, MVT::i64)); in performANDCombine()
12693 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, And); in performANDCombine()
12736 SDLoc DL(N); in combineOrOfCZERO() local
12738 SDValue NewN0 = DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, TrueV.getOperand(0), in combineOrOfCZERO()
12740 SDValue NewN1 = DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, FalseV.getOperand(0), in combineOrOfCZERO()
12742 SDValue NewOr = DAG.getNode(ISD::OR, DL, VT, NewN0, NewN1); in combineOrOfCZERO()
12743 return DAG.getNode(ISD::XOR, DL, VT, NewOr, TrueV.getOperand(1)); in combineOrOfCZERO()
12785 SDLoc DL(N); in performXORCombine() local
12786 SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N0.getOperand(0)); in performXORCombine()
12787 SDValue Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N0.getOperand(1)); in performXORCombine()
12788 SDValue Shl = DAG.getNode(ISD::SHL, DL, MVT::i64, Op0, Op1); in performXORCombine()
12789 SDValue And = DAG.getNOT(DL, Shl, MVT::i64); in performXORCombine()
12790 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, And); in performXORCombine()
12799 SDLoc DL(N); in performXORCombine() local
12800 return DAG.getNode(RISCVISD::ROLW, DL, MVT::i64, in performXORCombine()
12801 DAG.getConstant(~1, DL, MVT::i64), N0.getOperand(1)); in performXORCombine()
12810 SDLoc DL(N0); in performXORCombine() local
12813 return DAG.getSetCC(DL, VT, N0.getOperand(1), in performXORCombine()
12814 DAG.getConstant(Imm + 1, DL, VT), CC); in performXORCombine()
12833 SDLoc DL(N); in performMULCombine() local
12857 SDValue MulVal = DAG.getNode(ISD::MUL, DL, VT, N1, MulOper); in performMULCombine()
12858 return DAG.getNode(AddSubOpc, DL, VT, N1, MulVal); in performMULCombine()
12862 SDValue MulVal = DAG.getNode(ISD::MUL, DL, VT, N0, MulOper); in performMULCombine()
12863 return DAG.getNode(AddSubOpc, DL, VT, N0, MulVal); in performMULCombine()
12879 SDLoc DL(N); in narrowIndex() local
12894 N = DAG.getNode(ISD::TRUNCATE, DL, in narrowIndex()
12930 SDValue NewExt = DAG.getNode(N0->getOpcode(), DL, NewVT, N0->ops()); in narrowIndex()
12931 SDValue NewShAmtVec = DAG.getConstant(ShAmtV, DL, NewVT); in narrowIndex()
12932 N = DAG.getNode(ISD::SHL, DL, NewVT, NewExt, NewShAmtVec); in narrowIndex()
13088 SDLoc DL(Root); in getOrCreateExtendedOp() local
13095 return DAG.getNode(ExtOpc, DL, NarrowVT, Source, Mask, VL); in getOrCreateExtendedOp()
13097 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, NarrowVT, in getOrCreateExtendedOp()
13214 SDLoc DL(Root); in fillUpExtensionSupport() local
13215 std::tie(Mask, VL) = getDefaultScalableVLOps(VT, DL, DAG, Subtarget); in fillUpExtensionSupport()
13349 SDLoc DL(Root); in getMaskAndVL() local
13351 return getDefaultScalableVLOps(VT, DL, DAG, Subtarget); in getMaskAndVL()
13867 SDLoc DL(N); in performFP_TO_INTCombine() local
13894 getDefaultVLOps(SrcVT, SrcContainerVT, DL, DAG, Subtarget); in performFP_TO_INTCombine()
13902 FpToInt = DAG.getNode(Opc, DL, ContainerVT, XVal, Mask, VL); in performFP_TO_INTCombine()
13906 FpToInt = DAG.getNode(Opc, DL, ContainerVT, XVal, Mask, VL); in performFP_TO_INTCombine()
13910 FpToInt = DAG.getNode(Opc, DL, ContainerVT, XVal, Mask, in performFP_TO_INTCombine()
13911 DAG.getTargetConstant(FRM, DL, XLenVT), VL); in performFP_TO_INTCombine()
13932 SDValue FpToInt = DAG.getNode(Opc, DL, XLenVT, Src.getOperand(0), in performFP_TO_INTCombine()
13933 DAG.getTargetConstant(FRM, DL, XLenVT)); in performFP_TO_INTCombine()
13934 return DAG.getNode(ISD::TRUNCATE, DL, VT, FpToInt); in performFP_TO_INTCombine()
13990 SDLoc DL(N); in performFP_TO_INT_SATCombine() local
13991 SDValue FpToInt = DAG.getNode(Opc, DL, XLenVT, Src, in performFP_TO_INT_SATCombine()
13992 DAG.getTargetConstant(FRM, DL, XLenVT)); in performFP_TO_INT_SATCombine()
13997 FpToInt = DAG.getZeroExtendInReg(FpToInt, DL, MVT::i32); in performFP_TO_INT_SATCombine()
14001 SDValue ZeroInt = DAG.getConstant(0, DL, DstVT); in performFP_TO_INT_SATCombine()
14002 return DAG.getSelectCC(DL, Src, Src, ZeroInt, FpToInt, ISD::CondCode::SETUO); in performFP_TO_INT_SATCombine()
14020 SDLoc DL(N); in performBITREVERSECombine() local
14021 return DAG.getNode(RISCVISD::BREV8, DL, VT, Src.getOperand(0)); in performBITREVERSECombine()
14294 SDLoc DL(N); in performSRACombine() local
14295 return DAG.getNode(ISD::SRA, DL, MVT::i64, Shl, in performSRACombine()
14296 DAG.getConstant(ShAmt + 32, DL, MVT::i64)); in performSRACombine()
14349 SDLoc DL(N); in performSRACombine() local
14356 DAG.getConstant(AddC->getAPIntValue().lshr(32), DL, MVT::i64); in performSRACombine()
14358 In = DAG.getNode(ISD::ADD, DL, MVT::i64, In, ShiftedAddC); in performSRACombine()
14360 In = DAG.getNode(ISD::SUB, DL, MVT::i64, ShiftedAddC, In); in performSRACombine()
14363 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, In, in performSRACombine()
14369 ISD::SHL, DL, MVT::i64, SExt, in performSRACombine()
14370 DAG.getConstant(32 - ShAmt, DL, MVT::i64)); in performSRACombine()
14436 static bool combine_CC(SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL, in combine_CC() argument
14466 translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG); in combine_CC()
14495 DAG.getNode(ISD::SHL, DL, LHS.getValueType(), LHS0.getOperand(0), in combine_CC()
14496 DAG.getConstant(ShAmt, DL, LHS.getValueType())); in combine_CC()
14508 RHS = DAG.getConstant(0, DL, LHS.getValueType()); in combine_CC()
14561 SDLoc DL(N); in tryFoldSelectIntoOp() local
14565 DAG.getNeutralElement(Opc, DL, OtherOpVT, N->getFlags()); in tryFoldSelectIntoOp()
14567 IdentityOperand = DAG.getConstant(0, DL, OtherOpVT); in tryFoldSelectIntoOp()
14573 DAG.getSelect(DL, OtherOpVT, N->getOperand(0), OtherOp, IdentityOperand); in tryFoldSelectIntoOp()
14574 return DAG.getNode(TrueVal.getOpcode(), DL, VT, FalseVal, NewSel); in tryFoldSelectIntoOp()
14643 SDLoc DL(N); in useInversedSetcc() local
14661 return DAG.getSelect(DL, VT, in useInversedSetcc()
14662 DAG.getSetCC(DL, CondVT, LHS, RHS, ISD::SETNE), in useInversedSetcc()
14695 SDLoc DL(N); in performBUILD_VECTORCombine() local
14740 return DAG.getNode(Opcode, DL, VT, DAG.getBuildVector(VT, DL, LHSOps), in performBUILD_VECTORCombine()
14741 DAG.getBuildVector(VT, DL, RHSOps)); in performBUILD_VECTORCombine()
14750 SDLoc DL(N); in performINSERT_VECTOR_ELTCombine() local
14778 SDValue LHS = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, in performINSERT_VECTOR_ELTCombine()
14780 SDValue RHS = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, in performINSERT_VECTOR_ELTCombine()
14782 return DAG.getNode(InVecOpcode, DL, VT, LHS, RHS); in performINSERT_VECTOR_ELTCombine()
14799 SDValue NewIdx = DAG.getConstant(Elt % ConcatNumElts, DL, in performINSERT_VECTOR_ELTCombine()
14804 ConcatOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ConcatVT, in performINSERT_VECTOR_ELTCombine()
14810 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps); in performINSERT_VECTOR_ELTCombine()
14820 SDLoc DL(N); in performCONCAT_VECTORSCombine() local
14907 : DAG.getConstant(std::get<int64_t>(StrideVariant), DL, in performCONCAT_VECTORSCombine()
14910 Stride = DAG.getNegative(Stride, DL, Stride.getValueType()); in performCONCAT_VECTORSCombine()
14914 DAG.getTargetConstant(Intrinsic::riscv_masked_strided_load, DL, in performCONCAT_VECTORSCombine()
14918 DAG.getSplat(WideVecVT.changeVectorElementType(MVT::i1), DL, in performCONCAT_VECTORSCombine()
14919 DAG.getConstant(1, DL, MVT::i1)); in performCONCAT_VECTORSCombine()
14939 SDValue StridedLoad = DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, in performCONCAT_VECTORSCombine()
14989 SDLoc DL(N); in combineToVWMACC() local
14990 return getDefaultScalableVLOps(N->getSimpleValueType(0), DL, DAG, in combineToVWMACC()
15012 SDLoc DL(N); in combineToVWMACC() local
15016 return DAG.getNode(Opc, DL, VT, Ops); in combineToVWMACC()
15019 static bool legalizeScatterGatherIndexType(SDLoc DL, SDValue &Index, in legalizeScatterGatherIndexType() argument
15041 Index = DAG.getNode(ISD::SIGN_EXTEND, DL, in legalizeScatterGatherIndexType()
15133 SDLoc DL(N); in PerformDAGCombine() local
15170 SDValue Lo = DAG.getConstant(V.trunc(32), DL, MVT::i32); in PerformDAGCombine()
15171 SDValue Hi = DAG.getConstant(V.lshr(32).trunc(32), DL, MVT::i32); in PerformDAGCombine()
15183 DAG.getNode(RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32), in PerformDAGCombine()
15189 SDValue NewHi = DAG.getNode(ISD::XOR, DL, MVT::i32, Hi, in PerformDAGCombine()
15190 DAG.getConstant(SignBit, DL, MVT::i32)); in PerformDAGCombine()
15194 SDValue NewHi = DAG.getNode(ISD::AND, DL, MVT::i32, Hi, in PerformDAGCombine()
15195 DAG.getConstant(~SignBit, DL, MVT::i32)); in PerformDAGCombine()
15228 SDLoc DL(N); in PerformDAGCombine() local
15250 SDValue NewFMV = DAG.getNode(N->getOpcode(), DL, VT, Op0.getOperand(0)); in PerformDAGCombine()
15254 return DAG.getNode(ISD::XOR, DL, VT, NewFMV, in PerformDAGCombine()
15255 DAG.getConstant(SignBit, DL, VT)); in PerformDAGCombine()
15258 return DAG.getNode(ISD::AND, DL, VT, NewFMV, in PerformDAGCombine()
15259 DAG.getConstant(~SignBit, DL, VT)); in PerformDAGCombine()
15402 SDLoc DL(N); in PerformDAGCombine() local
15424 DAG.getNode(ISD::SRA, DL, VT, LHS, in PerformDAGCombine()
15425 DAG.getConstant(Subtarget.getXLen() - 1, DL, VT)); in PerformDAGCombine()
15427 DAG.getNode(ISD::AND, DL, VT, SRA, in PerformDAGCombine()
15428 DAG.getConstant(TrueSImm - FalseSImm, DL, VT)); in PerformDAGCombine()
15429 return DAG.getNode(ISD::ADD, DL, VT, AND, FalseV); in PerformDAGCombine()
15436 if (combine_CC(LHS, RHS, CC, DL, DAG, Subtarget)) in PerformDAGCombine()
15437 return DAG.getNode(RISCVISD::SELECT_CC, DL, N->getValueType(0), in PerformDAGCombine()
15443 SDValue C = DAG.getSetCC(DL, VT, LHS, RHS, CCVal); in PerformDAGCombine()
15444 SDValue Neg = DAG.getNegative(C, DL, VT); in PerformDAGCombine()
15445 return DAG.getNode(ISD::OR, DL, VT, Neg, FalseV); in PerformDAGCombine()
15450 DAG.getSetCC(DL, VT, LHS, RHS, ISD::getSetCCInverse(CCVal, VT)); in PerformDAGCombine()
15451 SDValue Neg = DAG.getNegative(C, DL, VT); in PerformDAGCombine()
15452 return DAG.getNode(ISD::OR, DL, VT, Neg, TrueV); in PerformDAGCombine()
15458 DAG.getSetCC(DL, VT, LHS, RHS, ISD::getSetCCInverse(CCVal, VT)); in PerformDAGCombine()
15459 SDValue Neg = DAG.getNegative(C, DL, VT); in PerformDAGCombine()
15460 return DAG.getNode(ISD::AND, DL, VT, Neg, FalseV); in PerformDAGCombine()
15464 SDValue C = DAG.getSetCC(DL, VT, LHS, RHS, CCVal); in PerformDAGCombine()
15465 SDValue Neg = DAG.getNegative(C, DL, VT); in PerformDAGCombine()
15466 return DAG.getNode(ISD::AND, DL, VT, Neg, TrueV); in PerformDAGCombine()
15477 SDValue C = DAG.getSetCC(DL, VT, LHS, RHS, ISD::CondCode::SETEQ); in PerformDAGCombine()
15478 return DAG.getNode(ISD::ADD, DL, VT, LHS, C); in PerformDAGCombine()
15490 SDValue NewSel = DAG.getNode(RISCVISD::SELECT_CC, DL, VT, LHS, RHS, CC, in PerformDAGCombine()
15492 return DAG.getNode(ISD::XOR, DL, VT, NewSel, TrueV.getOperand(1)); in PerformDAGCombine()
15501 SDLoc DL(N); in PerformDAGCombine() local
15503 if (combine_CC(LHS, RHS, CC, DL, DAG, Subtarget)) in PerformDAGCombine()
15504 return DAG.getNode(RISCVISD::BR_CC, DL, N->getValueType(0), in PerformDAGCombine()
15536 SDLoc DL(N); in PerformDAGCombine() local
15537 SDValue NewFPExtRound = DAG.getFPExtendOrRound(In2.getOperand(0), DL, VT); in PerformDAGCombine()
15538 return DAG.getNode(ISD::FCOPYSIGN, DL, VT, N->getOperand(0), in PerformDAGCombine()
15539 DAG.getNode(ISD::FNEG, DL, VT, NewFPExtRound)); in PerformDAGCombine()
15550 SDLoc DL(N); in PerformDAGCombine() local
15551 if (legalizeScatterGatherIndexType(DL, Index, IndexType, DCI)) in PerformDAGCombine()
15553 N->getVTList(), MGN->getMemoryVT(), DL, in PerformDAGCombine()
15560 N->getVTList(), MGN->getMemoryVT(), DL, in PerformDAGCombine()
15582 SDValue BasePtr = DAG.getNode(ISD::ADD, DL, PtrVT, MGN->getBasePtr(), in PerformDAGCombine()
15583 DAG.getConstant(Addend, DL, PtrVT)); in PerformDAGCombine()
15587 DAG.getTargetConstant(Intrinsic::riscv_masked_strided_load, DL, in PerformDAGCombine()
15591 DAG.getConstant(StepNumerator, DL, XLenVT), MGN->getMask()}; in PerformDAGCombine()
15592 return DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, in PerformDAGCombine()
15600 SDValue Load = DAG.getMaskedLoad(VT, DL, MGN->getChain(), in PerformDAGCombine()
15606 DAG.getVectorShuffle(VT, DL, Load, DAG.getUNDEF(VT), ShuffleMask); in PerformDAGCombine()
15607 return DAG.getMergeValues({Shuffle, Load.getValue(1)}, DL); in PerformDAGCombine()
15618 Index = DAG.getBuildVector(IndexVT, DL, NewIndices); in PerformDAGCombine()
15629 SDValue Mask = DAG.getSplat(MaskVT, DL, DAG.getConstant(1, DL, MVT::i1)); in PerformDAGCombine()
15632 DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other), WideVT, DL, in PerformDAGCombine()
15637 return DAG.getMergeValues({Result, Gather.getValue(1)}, DL); in PerformDAGCombine()
15649 SDLoc DL(N); in PerformDAGCombine() local
15650 if (legalizeScatterGatherIndexType(DL, Index, IndexType, DCI)) in PerformDAGCombine()
15652 N->getVTList(), MSN->getMemoryVT(), DL, in PerformDAGCombine()
15659 N->getVTList(), MSN->getMemoryVT(), DL, in PerformDAGCombine()
15668 SDValue Shuffle = DAG.getVectorShuffle(VT, DL, MSN->getValue(), in PerformDAGCombine()
15670 return DAG.getMaskedStore(MSN->getChain(), DL, Shuffle, MSN->getBasePtr(), in PerformDAGCombine()
15685 SDLoc DL(N); in PerformDAGCombine() local
15686 if (legalizeScatterGatherIndexType(DL, Index, IndexType, DCI)) in PerformDAGCombine()
15687 return DAG.getGatherVP(N->getVTList(), VPGN->getMemoryVT(), DL, in PerformDAGCombine()
15694 return DAG.getGatherVP(N->getVTList(), VPGN->getMemoryVT(), DL, in PerformDAGCombine()
15710 SDLoc DL(N); in PerformDAGCombine() local
15711 if (legalizeScatterGatherIndexType(DL, Index, IndexType, DCI)) in PerformDAGCombine()
15712 return DAG.getScatterVP(N->getVTList(), VPSN->getMemoryVT(), DL, in PerformDAGCombine()
15719 return DAG.getScatterVP(N->getVTList(), VPSN->getMemoryVT(), DL, in PerformDAGCombine()
15732 SDLoc DL(N); in PerformDAGCombine() local
15735 ShAmt = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, DAG.getUNDEF(VT), in PerformDAGCombine()
15737 return DAG.getNode(N->getOpcode(), DL, VT, N->getOperand(0), ShAmt, in PerformDAGCombine()
15751 SDLoc DL(N); in PerformDAGCombine() local
15753 ShAmt = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, DAG.getUNDEF(VT), in PerformDAGCombine()
15756 return DAG.getNode(N->getOpcode(), DL, VT, N->getOperand(0), ShAmt); in PerformDAGCombine()
15798 SDLoc DL(N); in PerformDAGCombine() local
15833 SDValue NewV = DAG.getConstant(NewC, DL, NewVT); in PerformDAGCombine()
15834 return DAG.getStore(Chain, DL, NewV, Store->getBasePtr(), in PerformDAGCombine()
15854 SDValue NewL = DAG.getLoad(NewVT, DL, L->getChain(), L->getBasePtr(), in PerformDAGCombine()
15857 return DAG.getStore(Chain, DL, NewL, Store->getBasePtr(), in PerformDAGCombine()
15875 SDLoc DL(N); in PerformDAGCombine() local
15878 Store->getChain(), DL, Src, Store->getBasePtr(), Store->getOffset(), in PerformDAGCombine()
15879 DAG.getConstant(1, DL, MaskVT), in PerformDAGCombine()
15880 DAG.getConstant(1, DL, Subtarget.getXLenVT()), MemVT, in PerformDAGCombine()
15918 return DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, VT, Passthru, Scalar, VL); in PerformDAGCombine()
15941 return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, Passthru, Scalar, VL); in PerformDAGCombine()
15974 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, M1VT, Passthru, in PerformDAGCombine()
15975 DAG.getVectorIdxConstant(0, DL)); in PerformDAGCombine()
15977 DAG.getNode(N->getOpcode(), DL, M1VT, M1Passthru, Scalar, VL); in PerformDAGCombine()
15978 Result = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Passthru, Result, in PerformDAGCombine()
15979 DAG.getConstant(0, DL, XLenVT)); in PerformDAGCombine()
15989 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Scalar, VL); in PerformDAGCombine()
15998 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, M1VT, Vec, in PerformDAGCombine()
15999 DAG.getVectorIdxConstant(0, DL)); in PerformDAGCombine()
16000 return DAG.getNode(RISCVISD::VMV_X_S, DL, N->getSimpleValueType(0), Vec); in PerformDAGCombine()
16026 return DAG.getMaskedLoad(VT, DL, Load->getChain(), Base, in PerformDAGCombine()
16044 return DAG.getMaskedStore(Store->getChain(), DL, Value, Base, in PerformDAGCombine()
16061 SDLoc DL(N); in PerformDAGCombine() local
16065 return DAG.getConstant(-1, DL, VT); in PerformDAGCombine()
16066 return DAG.getConstant(0, DL, VT); in PerformDAGCombine()
16082 SDLoc DL(N); in PerformDAGCombine() local
16083 N0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i1, Ops); in PerformDAGCombine()
16085 return DAG.getNode(ISD::TRUNCATE, DL, VT, N0); in PerformDAGCombine()
16200 SDLoc DL(Op); in targetShrinkDemandedConstant() local
16201 SDValue NewC = TLO.DAG.getConstant(NewMask, DL, Op.getValueType()); in targetShrinkDemandedConstant()
16202 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), DL, Op.getValueType(), in targetShrinkDemandedConstant()
16579 DebugLoc DL = MI.getDebugLoc(); in emitReadCycleWidePseudo() local
16582 BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), HiReg) in emitReadCycleWidePseudo()
16585 BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), LoReg) in emitReadCycleWidePseudo()
16588 BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), ReadAgainReg) in emitReadCycleWidePseudo()
16592 BuildMI(LoopMBB, DL, TII->get(RISCV::BNE)) in emitReadCycleWidePseudo()
16613 DebugLoc DL = MI.getDebugLoc(); in emitSplitF64Pseudo() local
16632 BuildMI(*BB, MI, DL, TII.get(RISCV::LW), LoReg) in emitSplitF64Pseudo()
16636 BuildMI(*BB, MI, DL, TII.get(RISCV::LW), HiReg) in emitSplitF64Pseudo()
16652 DebugLoc DL = MI.getDebugLoc(); in emitBuildPairF64Pseudo() local
16669 BuildMI(*BB, MI, DL, TII.get(RISCV::SW)) in emitBuildPairF64Pseudo()
16674 BuildMI(*BB, MI, DL, TII.get(RISCV::SW)) in emitBuildPairF64Pseudo()
16703 DebugLoc DL = MI.getDebugLoc(); in emitQuietFCMP() local
16712 BuildMI(*BB, MI, DL, TII.get(RISCV::ReadFFLAGS), SavedFFlags); in emitQuietFCMP()
16714 auto MIB = BuildMI(*BB, MI, DL, TII.get(RelOpcode), DstReg) in emitQuietFCMP()
16721 BuildMI(*BB, MI, DL, TII.get(RISCV::WriteFFLAGS)) in emitQuietFCMP()
16725 auto MIB2 = BuildMI(*BB, MI, DL, TII.get(EqOpcode), RISCV::X0) in emitQuietFCMP()
16776 const DebugLoc &DL = First.getDebugLoc(); in EmitLoweredCascadedSelect() local
16806 BuildMI(FirstMBB, DL, TII.getBrCond(FirstCC)) in EmitLoweredCascadedSelect()
16818 BuildMI(ThisMBB, DL, TII.getBrCond(SecondCC)) in EmitLoweredCascadedSelect()
16825 BuildMI(*SinkMBB, SinkMBB->begin(), DL, TII.get(RISCV::PHI), DestReg) in EmitLoweredCascadedSelect()
16916 DebugLoc DL = MI.getDebugLoc(); in emitSelectPseudo() local
16943 BuildMI(HeadMBB, DL, TII.getBrCond(CC)) in emitSelectPseudo()
16978 DebugLoc DL = MI.getDebugLoc(); in emitVFROUND_NOEXCEPT_MASK() local
16986 BuildMI(*BB, MI, DL, TII.get(RISCV::ReadFFLAGS), SavedFFLAGS); in emitVFROUND_NOEXCEPT_MASK()
16995 BuildMI(*BB, MI, DL, TII.get(CVTXOpc), Tmp) in emitVFROUND_NOEXCEPT_MASK()
17008 BuildMI(*BB, MI, DL, TII.get(CVTFOpc)) in emitVFROUND_NOEXCEPT_MASK()
17022 BuildMI(*BB, MI, DL, TII.get(RISCV::WriteFFLAGS)) in emitVFROUND_NOEXCEPT_MASK()
17090 DebugLoc DL = MI.getDebugLoc(); in emitFROUND() local
17118 BuildMI(MBB, DL, TII.get(FSGNJXOpc), FabsReg).addReg(SrcReg).addReg(SrcReg); in emitFROUND()
17123 BuildMI(MBB, DL, TII.get(CmpOpc), CmpReg).addReg(FabsReg).addReg(MaxReg); in emitFROUND()
17128 BuildMI(MBB, DL, TII.get(RISCV::BEQ)) in emitFROUND()
17137 MIB = BuildMI(CvtMBB, DL, TII.get(F2IOpc), F2IReg).addReg(SrcReg).addImm(FRM); in emitFROUND()
17143 MIB = BuildMI(CvtMBB, DL, TII.get(I2FOpc), I2FReg).addReg(F2IReg).addImm(FRM); in emitFROUND()
17149 BuildMI(CvtMBB, DL, TII.get(FSGNJOpc), CvtReg).addReg(I2FReg).addReg(SrcReg); in emitFROUND()
17152 BuildMI(*DoneMBB, DoneMBB->begin(), DL, TII.get(RISCV::PHI), DstReg) in emitFROUND()
17425 bool RISCV::CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo, in CC_RISCV() argument
17430 unsigned XLen = DL.getLargestLegalIntTypeSizeInBits(); in CC_RISCV()
17507 DL.getTypeAllocSize(OrigTy) == TwoXLenInBytes && in CC_RISCV()
17738 const CCValAssign &VA, const SDLoc &DL, in convertLocVTToValVT() argument
17750 Val = DAG.getNode(RISCVISD::FMV_H_X, DL, VA.getValVT(), Val); in convertLocVTToValVT()
17753 Val = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Val); in convertLocVTToValVT()
17754 Val = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Val); in convertLocVTToValVT()
17756 Val = DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, Val); in convertLocVTToValVT()
17759 Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val); in convertLocVTToValVT()
17769 const CCValAssign &VA, const SDLoc &DL, in unpackFromRegLoc() argument
17779 Val = DAG.getCopyFromReg(Chain, DL, VReg, LocVT); in unpackFromRegLoc()
17798 return convertLocVTToValVT(DAG, Val, VA, DL, TLI.getSubtarget()); in unpackFromRegLoc()
17802 const CCValAssign &VA, const SDLoc &DL, in convertValVTToLocVT() argument
17816 Val = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, LocVT, Val); in convertValVTToLocVT()
17819 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Val); in convertValVTToLocVT()
17820 Val = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Val); in convertValVTToLocVT()
17822 Val = DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Val); in convertValVTToLocVT()
17825 Val = DAG.getNode(ISD::BITCAST, DL, LocVT, Val); in convertValVTToLocVT()
17835 const CCValAssign &VA, const SDLoc &DL) { in unpackFromMemLoc() argument
17863 ExtType, DL, LocVT, Chain, FIN, in unpackFromMemLoc()
17871 const SDLoc &DL) { in unpackF64OnRV32DSoftABI() argument
17882 SDValue Lo = DAG.getCopyFromReg(Chain, DL, LoVReg, MVT::i32); in unpackF64OnRV32DSoftABI()
17889 Hi = DAG.getLoad(MVT::i32, DL, Chain, FIN, in unpackF64OnRV32DSoftABI()
17895 Hi = DAG.getCopyFromReg(Chain, DL, HiVReg, MVT::i32); in unpackF64OnRV32DSoftABI()
17897 return DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, Lo, Hi); in unpackF64OnRV32DSoftABI()
17902 bool RISCV::CC_RISCV_FastCC(const DataLayout &DL, RISCVABI::ABI ABI, in CC_RISCV_FastCC() argument
18082 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL, in LowerFormalArguments() argument
18141 ArgValue = unpackF64OnRV32DSoftABI(DAG, Chain, VA, ArgLocs[++i], DL); in LowerFormalArguments()
18143 ArgValue = unpackFromRegLoc(DAG, Chain, VA, DL, Ins[InsIdx], *this); in LowerFormalArguments()
18145 ArgValue = unpackFromMemLoc(DAG, Chain, VA, DL); in LowerFormalArguments()
18153 InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue, in LowerFormalArguments()
18161 SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL); in LowerFormalArguments()
18163 Offset = DAG.getNode(ISD::VSCALE, DL, XLenVT, Offset); in LowerFormalArguments()
18164 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue, Offset); in LowerFormalArguments()
18165 InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address, in LowerFormalArguments()
18217 SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, XLenVT); in LowerFormalArguments()
18219 Chain, DL, ArgValue, FIN, in LowerFormalArguments()
18223 DAG.getMemBasePlusOffset(FIN, TypeSize::getFixed(XLenInBytes), DL); in LowerFormalArguments()
18237 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OutChains); in LowerFormalArguments()
18315 SDLoc &DL = CLI.DL; in LowerCall() local
18369 SDValue SizeNode = DAG.getConstant(Size, DL, XLenVT); in LowerCall()
18371 Chain = DAG.getMemcpy(Chain, DL, FIPtr, Arg, SizeNode, Alignment, in LowerCall()
18379 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, CLI.DL); in LowerCall()
18396 RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32), ArgValue); in LowerCall()
18409 StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT); in LowerCall()
18411 DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, in LowerCall()
18412 DAG.getIntPtrConstant(HiVA.getLocMemOffset(), DL)); in LowerCall()
18415 DAG.getStore(Chain, DL, Hi, Address, MachinePointerInfo())); in LowerCall()
18447 SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL); in LowerCall()
18450 Offset = DAG.getNode(ISD::VSCALE, DL, XLenVT, Offset); in LowerCall()
18460 DAG.getStore(Chain, DL, ArgValue, SpillSlot, in LowerCall()
18466 DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot, PartOffset); in LowerCall()
18468 DAG.getStore(Chain, DL, PartValue, Address, in LowerCall()
18473 ArgValue = convertValVTToLocVT(DAG, ArgValue, VA, DL, Subtarget); in LowerCall()
18490 StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT); in LowerCall()
18492 DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, in LowerCall()
18493 DAG.getIntPtrConstant(VA.getLocMemOffset(), DL)); in LowerCall()
18497 DAG.getStore(Chain, DL, ArgValue, Address, MachinePointerInfo())); in LowerCall()
18503 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains); in LowerCall()
18509 Chain = DAG.getCopyToReg(Chain, DL, Reg.first, Reg.second, Glue); in LowerCall()
18528 Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, RISCVII::MO_CALL); in LowerCall()
18563 SDValue Ret = DAG.getNode(RISCVISD::TAIL, DL, NodeTys, Ops); in LowerCall()
18570 Chain = DAG.getNode(RISCVISD::CALL, DL, NodeTys, Ops); in LowerCall()
18577 Chain = DAG.getCALLSEQ_END(Chain, NumBytes, 0, Glue, DL); in LowerCall()
18590 DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), Glue); in LowerCall()
18597 SDValue RetValue2 = DAG.getCopyFromReg(Chain, DL, RVLocs[++i].getLocReg(), in LowerCall()
18601 RetValue = DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, RetValue, in LowerCall()
18605 RetValue = convertLocVTToValVT(DAG, RetValue, VA, DL, Subtarget); in LowerCall()
18640 const SDLoc &DL, SelectionDAG &DAG) const { in LowerReturn() argument
18670 SDValue SplitF64 = DAG.getNode(RISCVISD::SplitF64, DL, in LowerReturn()
18683 Chain = DAG.getCopyToReg(Chain, DL, RegLo, Lo, Glue); in LowerReturn()
18686 Chain = DAG.getCopyToReg(Chain, DL, RegHi, Hi, Glue); in LowerReturn()
18691 Val = convertValVTToLocVT(DAG, Val, VA, DL, Subtarget); in LowerReturn()
18692 Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Val, Glue); in LowerReturn()
18734 return DAG.getNode(RetOpc, DL, MVT::Other, RetOps); in LowerReturn()
19461 const DataLayout &DL = AI->getModule()->getDataLayout(); in emitMaskedAtomicRMWIntrinsic() local
19463 DL.getTypeStoreSizeInBits(AI->getValOperand()->getType()); in emitMaskedAtomicRMWIntrinsic()
19843 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, in splitValueIntoRegisterParts() argument
19851 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i16, Val); in splitValueIntoRegisterParts()
19852 Val = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Val); in splitValueIntoRegisterParts()
19853 Val = DAG.getNode(ISD::OR, DL, MVT::i32, Val, in splitValueIntoRegisterParts()
19854 DAG.getConstant(0xFFFF0000, DL, MVT::i32)); in splitValueIntoRegisterParts()
19855 Val = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Val); in splitValueIntoRegisterParts()
19880 Val = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SameEltTypeVT, in splitValueIntoRegisterParts()
19882 DAG.getVectorIdxConstant(0, DL)); in splitValueIntoRegisterParts()
19884 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); in splitValueIntoRegisterParts()
19887 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT), in splitValueIntoRegisterParts()
19888 Val, DAG.getVectorIdxConstant(0, DL)); in splitValueIntoRegisterParts()
19898 SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, in joinRegisterPartsIntoValue() argument
19906 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Val); in joinRegisterPartsIntoValue()
19907 Val = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Val); in joinRegisterPartsIntoValue()
19908 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); in joinRegisterPartsIntoValue()
19933 Val = DAG.getNode(ISD::BITCAST, DL, SameEltTypeVT, Val); in joinRegisterPartsIntoValue()
19935 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val, in joinRegisterPartsIntoValue()
19936 DAG.getVectorIdxConstant(0, DL)); in joinRegisterPartsIntoValue()
19979 const DataLayout &DL) const { in isLegalInterleavedAccessType()
19980 EVT VT = getValueType(DL, VTy); in isLegalInterleavedAccessType()
19986 !allowsMemoryAccessForAlignment(VTy->getContext(), DL, VT, AddrSpace, in isLegalInterleavedAccessType()