Lines Matching refs:SL
882 SDLoc SL(Op); in getNegatedExpression() local
887 return DAG.getNode(AMDGPUISD::RCP, SL, VT, NegSrc, Op->getFlags()); in getNegatedExpression()
1469 SDLoc SL(Op); in LowerCONCAT_VECTORS() local
1481 SDValue NewIn = DAG.getNode(ISD::BITCAST, SL, NewEltVT, In); in LowerCONCAT_VECTORS()
1490 SDValue BV = DAG.getBuildVector(NewVT, SL, Args); in LowerCONCAT_VECTORS()
1491 return DAG.getNode(ISD::BITCAST, SL, VT, BV); in LowerCONCAT_VECTORS()
1498 return DAG.getBuildVector(Op.getValueType(), SL, Args); in LowerCONCAT_VECTORS()
1503 SDLoc SL(Op); in LowerEXTRACT_SUBVECTOR() local
1519 SDValue Tmp = DAG.getNode(ISD::BITCAST, SL, NewSrcVT, Op.getOperand(0)); in LowerEXTRACT_SUBVECTOR()
1525 Tmp = DAG.getBuildVector(NewVT, SL, Args); in LowerEXTRACT_SUBVECTOR()
1527 return DAG.getNode(ISD::BITCAST, SL, VT, Tmp); in LowerEXTRACT_SUBVECTOR()
1533 return DAG.getBuildVector(Op.getValueType(), SL, Args); in LowerEXTRACT_SUBVECTOR()
1663 SDLoc SL(Op); in split64BitValue() local
1665 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op); in split64BitValue()
1667 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32); in split64BitValue()
1668 const SDValue One = DAG.getConstant(1, SL, MVT::i32); in split64BitValue()
1670 SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero); in split64BitValue()
1671 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One); in split64BitValue()
1677 SDLoc SL(Op); in getLoHalf64() local
1679 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op); in getLoHalf64()
1680 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32); in getLoHalf64()
1681 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero); in getLoHalf64()
1685 SDLoc SL(Op); in getHiHalf64() local
1687 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op); in getHiHalf64()
1688 const SDValue One = DAG.getConstant(1, SL, MVT::i32); in getHiHalf64()
1689 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One); in getHiHalf64()
1730 SDLoc SL(Op); in SplitVectorLoad() local
1738 return DAG.getMergeValues(Ops, SL); in SplitVectorLoad()
1752 std::tie(Lo, Hi) = splitVector(Op, SL, LoVT, HiVT, DAG); in SplitVectorLoad()
1758 SDValue LoLoad = DAG.getExtLoad(Load->getExtensionType(), SL, LoVT, in SplitVectorLoad()
1761 SDValue HiPtr = DAG.getObjectPtrOffset(SL, BasePtr, TypeSize::getFixed(Size)); in SplitVectorLoad()
1763 DAG.getExtLoad(Load->getExtensionType(), SL, HiVT, Load->getChain(), in SplitVectorLoad()
1770 Join = DAG.getNode(ISD::CONCAT_VECTORS, SL, VT, LoLoad, HiLoad); in SplitVectorLoad()
1772 Join = DAG.getNode(ISD::INSERT_SUBVECTOR, SL, VT, DAG.getUNDEF(VT), LoLoad, in SplitVectorLoad()
1773 DAG.getVectorIdxConstant(0, SL)); in SplitVectorLoad()
1775 HiVT.isVector() ? ISD::INSERT_SUBVECTOR : ISD::INSERT_VECTOR_ELT, SL, in SplitVectorLoad()
1777 DAG.getVectorIdxConstant(LoVT.getVectorNumElements(), SL)); in SplitVectorLoad()
1780 SDValue Ops[] = {Join, DAG.getNode(ISD::TokenFactor, SL, MVT::Other, in SplitVectorLoad()
1783 return DAG.getMergeValues(Ops, SL); in SplitVectorLoad()
1792 SDLoc SL(Op); in WidenOrSplitVectorLoad() local
1811 Load->getExtensionType(), SL, WideVT, Load->getChain(), BasePtr, SrcValue, in WidenOrSplitVectorLoad()
1814 {DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, VT, WideLoad, in WidenOrSplitVectorLoad()
1815 DAG.getVectorIdxConstant(0, SL)), in WidenOrSplitVectorLoad()
1817 SL); in WidenOrSplitVectorLoad()
1834 SDLoc SL(Op); in SplitVectorStore() local
1842 std::tie(Lo, Hi) = splitVector(Val, SL, LoVT, HiVT, DAG); in SplitVectorStore()
1844 SDValue HiPtr = DAG.getObjectPtrOffset(SL, BasePtr, LoMemVT.getStoreSize()); in SplitVectorStore()
1852 DAG.getTruncStore(Chain, SL, Lo, BasePtr, SrcValue, LoMemVT, BaseAlign, in SplitVectorStore()
1855 DAG.getTruncStore(Chain, SL, Hi, HiPtr, SrcValue.getWithOffset(Size), in SplitVectorStore()
1858 return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoStore, HiStore); in SplitVectorStore()
2310 SDLoc SL(Op); in LowerFREM() local
2316 SDValue Div = DAG.getNode(ISD::FDIV, SL, VT, X, Y, Flags); in LowerFREM()
2317 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, VT, Div, Flags); in LowerFREM()
2318 SDValue Neg = DAG.getNode(ISD::FNEG, SL, VT, Trunc, Flags); in LowerFREM()
2320 return DAG.getNode(ISD::FMA, SL, VT, Neg, Y, X, Flags); in LowerFREM()
2324 SDLoc SL(Op); in LowerFCEIL() local
2331 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src); in LowerFCEIL()
2333 const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64); in LowerFCEIL()
2334 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f64); in LowerFCEIL()
2339 SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOGT); in LowerFCEIL()
2340 SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE); in LowerFCEIL()
2341 SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc); in LowerFCEIL()
2343 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, One, Zero); in LowerFCEIL()
2345 return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add); in LowerFCEIL()
2348 static SDValue extractF64Exponent(SDValue Hi, const SDLoc &SL, in extractF64Exponent() argument
2353 SDValue ExpPart = DAG.getNode(AMDGPUISD::BFE_U32, SL, MVT::i32, in extractF64Exponent()
2355 DAG.getConstant(FractBits - 32, SL, MVT::i32), in extractF64Exponent()
2356 DAG.getConstant(ExpBits, SL, MVT::i32)); in extractF64Exponent()
2357 SDValue Exp = DAG.getNode(ISD::SUB, SL, MVT::i32, ExpPart, in extractF64Exponent()
2358 DAG.getConstant(1023, SL, MVT::i32)); in extractF64Exponent()
2364 SDLoc SL(Op); in LowerFTRUNC() local
2369 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32); in LowerFTRUNC()
2375 SDValue Exp = extractF64Exponent(Hi, SL, DAG); in LowerFTRUNC()
2380 const SDValue SignBitMask = DAG.getConstant(UINT32_C(1) << 31, SL, MVT::i32); in LowerFTRUNC()
2381 SDValue SignBit = DAG.getNode(ISD::AND, SL, MVT::i32, Hi, SignBitMask); in LowerFTRUNC()
2384 SDValue SignBit64 = DAG.getBuildVector(MVT::v2i32, SL, {Zero, SignBit}); in LowerFTRUNC()
2385 SignBit64 = DAG.getNode(ISD::BITCAST, SL, MVT::i64, SignBit64); in LowerFTRUNC()
2387 SDValue BcInt = DAG.getNode(ISD::BITCAST, SL, MVT::i64, Src); in LowerFTRUNC()
2389 = DAG.getConstant((UINT64_C(1) << FractBits) - 1, SL, MVT::i64); in LowerFTRUNC()
2391 SDValue Shr = DAG.getNode(ISD::SRA, SL, MVT::i64, FractMask, Exp); in LowerFTRUNC()
2392 SDValue Not = DAG.getNOT(SL, Shr, MVT::i64); in LowerFTRUNC()
2393 SDValue Tmp0 = DAG.getNode(ISD::AND, SL, MVT::i64, BcInt, Not); in LowerFTRUNC()
2398 const SDValue FiftyOne = DAG.getConstant(FractBits - 1, SL, MVT::i32); in LowerFTRUNC()
2400 SDValue ExpLt0 = DAG.getSetCC(SL, SetCCVT, Exp, Zero, ISD::SETLT); in LowerFTRUNC()
2401 SDValue ExpGt51 = DAG.getSetCC(SL, SetCCVT, Exp, FiftyOne, ISD::SETGT); in LowerFTRUNC()
2403 SDValue Tmp1 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpLt0, SignBit64, Tmp0); in LowerFTRUNC()
2404 SDValue Tmp2 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpGt51, BcInt, Tmp1); in LowerFTRUNC()
2406 return DAG.getNode(ISD::BITCAST, SL, MVT::f64, Tmp2); in LowerFTRUNC()
2411 SDLoc SL(Op); in LowerFROUNDEVEN() local
2417 SDValue C1 = DAG.getConstantFP(C1Val, SL, MVT::f64); in LowerFROUNDEVEN()
2418 SDValue CopySign = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f64, C1, Src); in LowerFROUNDEVEN()
2422 SDValue Tmp1 = DAG.getNode(ISD::FADD, SL, MVT::f64, Src, CopySign); in LowerFROUNDEVEN()
2423 SDValue Tmp2 = DAG.getNode(ISD::FSUB, SL, MVT::f64, Tmp1, CopySign); in LowerFROUNDEVEN()
2425 SDValue Fabs = DAG.getNode(ISD::FABS, SL, MVT::f64, Src); in LowerFROUNDEVEN()
2428 SDValue C2 = DAG.getConstantFP(C2Val, SL, MVT::f64); in LowerFROUNDEVEN()
2432 SDValue Cond = DAG.getSetCC(SL, SetCCVT, Fabs, C2, ISD::SETOGT); in LowerFROUNDEVEN()
2434 return DAG.getSelect(SL, MVT::f64, Cond, Src, Tmp2); in LowerFROUNDEVEN()
2458 SDLoc SL(Op); in LowerFROUND() local
2462 SDValue T = DAG.getNode(ISD::FTRUNC, SL, VT, X); in LowerFROUND()
2466 SDValue Diff = DAG.getNode(ISD::FSUB, SL, VT, X, T); in LowerFROUND()
2468 SDValue AbsDiff = DAG.getNode(ISD::FABS, SL, VT, Diff); in LowerFROUND()
2470 const SDValue Zero = DAG.getConstantFP(0.0, SL, VT); in LowerFROUND()
2471 const SDValue One = DAG.getConstantFP(1.0, SL, VT); in LowerFROUND()
2476 const SDValue Half = DAG.getConstantFP(0.5, SL, VT); in LowerFROUND()
2477 SDValue Cmp = DAG.getSetCC(SL, SetCCVT, AbsDiff, Half, ISD::SETOGE); in LowerFROUND()
2478 SDValue OneOrZeroFP = DAG.getNode(ISD::SELECT, SL, VT, Cmp, One, Zero); in LowerFROUND()
2480 SDValue SignedOffset = DAG.getNode(ISD::FCOPYSIGN, SL, VT, OneOrZeroFP, X); in LowerFROUND()
2481 return DAG.getNode(ISD::FADD, SL, VT, T, SignedOffset); in LowerFROUND()
2485 SDLoc SL(Op); in LowerFFLOOR() local
2492 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src); in LowerFFLOOR()
2494 const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64); in LowerFFLOOR()
2495 const SDValue NegOne = DAG.getConstantFP(-1.0, SL, MVT::f64); in LowerFFLOOR()
2500 SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOLT); in LowerFFLOOR()
2501 SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE); in LowerFFLOOR()
2502 SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc); in LowerFFLOOR()
2504 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, NegOne, Zero); in LowerFFLOOR()
2506 return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add); in LowerFFLOOR()
2553 SDLoc SL(Src); in getIsLtSmallestNormal() local
2557 DAG.getConstantFP(APFloat::getSmallestNormalized(Semantics), SL, VT); in getIsLtSmallestNormal()
2562 SL, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), Src, in getIsLtSmallestNormal()
2570 SDLoc SL(Src); in getIsFinite() local
2573 SDValue Inf = DAG.getConstantFP(APFloat::getInf(Semantics), SL, VT); in getIsFinite()
2575 SDValue Fabs = DAG.getNode(ISD::FABS, SL, VT, Src, Flags); in getIsFinite()
2577 SL, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), Fabs, in getIsFinite()
2585 AMDGPUTargetLowering::getScaledLogInput(SelectionDAG &DAG, const SDLoc SL, in getScaledLogInput() argument
2593 DAG.getConstantFP(APFloat::getSmallestNormalized(Semantics), SL, VT); in getScaledLogInput()
2596 SL, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), Src, in getScaledLogInput()
2599 SDValue Scale32 = DAG.getConstantFP(0x1.0p+32, SL, VT); in getScaledLogInput()
2600 SDValue One = DAG.getConstantFP(1.0, SL, VT); in getScaledLogInput()
2602 DAG.getNode(ISD::SELECT, SL, VT, IsLtSmallestNormal, Scale32, One, Flags); in getScaledLogInput()
2604 SDValue ScaledInput = DAG.getNode(ISD::FMUL, SL, VT, Src, ScaleFactor, Flags); in getScaledLogInput()
2615 SDLoc SL(Op); in LowerFLOG2() local
2623 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src, Flags); in LowerFLOG2()
2624 SDValue Log = DAG.getNode(AMDGPUISD::LOG, SL, MVT::f32, Ext, Flags); in LowerFLOG2()
2625 return DAG.getNode(ISD::FP_ROUND, SL, VT, Log, in LowerFLOG2()
2626 DAG.getTargetConstant(0, SL, MVT::i32), Flags); in LowerFLOG2()
2630 getScaledLogInput(DAG, SL, Src, Flags); in LowerFLOG2()
2632 return DAG.getNode(AMDGPUISD::LOG, SL, VT, Src, Flags); in LowerFLOG2()
2634 SDValue Log2 = DAG.getNode(AMDGPUISD::LOG, SL, VT, ScaledInput, Flags); in LowerFLOG2()
2636 SDValue ThirtyTwo = DAG.getConstantFP(32.0, SL, VT); in LowerFLOG2()
2637 SDValue Zero = DAG.getConstantFP(0.0, SL, VT); in LowerFLOG2()
2639 DAG.getNode(ISD::SELECT, SL, VT, IsLtSmallestNormal, ThirtyTwo, Zero); in LowerFLOG2()
2640 return DAG.getNode(ISD::FSUB, SL, VT, Log2, ResultOffset, Flags); in LowerFLOG2()
2643 static SDValue getMad(SelectionDAG &DAG, const SDLoc &SL, EVT VT, SDValue X, in getMad() argument
2645 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, X, Y, Flags); in getMad()
2646 return DAG.getNode(ISD::FADD, SL, VT, Mul, C, Flags); in getMad()
2752 SDValue AMDGPUTargetLowering::LowerFLOGUnsafe(SDValue Src, const SDLoc &SL, in LowerFLOGUnsafe() argument
2763 auto [ScaledInput, IsScaled] = getScaledLogInput(DAG, SL, Src, Flags); in LowerFLOGUnsafe()
2765 SDValue LogSrc = DAG.getNode(AMDGPUISD::LOG, SL, VT, ScaledInput, Flags); in LowerFLOGUnsafe()
2767 DAG.getConstantFP(-32.0 * Log2BaseInverted, SL, VT); in LowerFLOGUnsafe()
2769 SDValue Zero = DAG.getConstantFP(0.0f, SL, VT); in LowerFLOGUnsafe()
2771 SDValue ResultOffset = DAG.getNode(ISD::SELECT, SL, VT, IsScaled, in LowerFLOGUnsafe()
2774 SDValue Log2Inv = DAG.getConstantFP(Log2BaseInverted, SL, VT); in LowerFLOGUnsafe()
2777 return DAG.getNode(ISD::FMA, SL, VT, LogSrc, Log2Inv, ResultOffset, in LowerFLOGUnsafe()
2779 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, LogSrc, Log2Inv, Flags); in LowerFLOGUnsafe()
2780 return DAG.getNode(ISD::FADD, SL, VT, Mul, ResultOffset); in LowerFLOGUnsafe()
2784 SDValue Log2Operand = DAG.getNode(LogOp, SL, VT, Src, Flags); in LowerFLOGUnsafe()
2785 SDValue Log2BaseInvertedOperand = DAG.getConstantFP(Log2BaseInverted, SL, VT); in LowerFLOGUnsafe()
2787 return DAG.getNode(ISD::FMUL, SL, VT, Log2Operand, Log2BaseInvertedOperand, in LowerFLOGUnsafe()
2795 SDLoc SL(Op); in lowerFEXP2() local
2803 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src, Flags); in lowerFEXP2()
2804 SDValue Log = DAG.getNode(AMDGPUISD::EXP, SL, MVT::f32, Ext, Flags); in lowerFEXP2()
2805 return DAG.getNode(ISD::FP_ROUND, SL, VT, Log, in lowerFEXP2()
2806 DAG.getTargetConstant(0, SL, MVT::i32), Flags); in lowerFEXP2()
2812 return DAG.getNode(AMDGPUISD::EXP, SL, MVT::f32, Src, Flags); in lowerFEXP2()
2818 SDValue RangeCheckConst = DAG.getConstantFP(-0x1.f80000p+6f, SL, VT); in lowerFEXP2()
2823 DAG.getSetCC(SL, SetCCVT, Src, RangeCheckConst, ISD::SETOLT); in lowerFEXP2()
2825 SDValue SixtyFour = DAG.getConstantFP(0x1.0p+6f, SL, VT); in lowerFEXP2()
2826 SDValue Zero = DAG.getConstantFP(0.0, SL, VT); in lowerFEXP2()
2829 DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, SixtyFour, Zero); in lowerFEXP2()
2831 SDValue AddInput = DAG.getNode(ISD::FADD, SL, VT, Src, AddOffset, Flags); in lowerFEXP2()
2832 SDValue Exp2 = DAG.getNode(AMDGPUISD::EXP, SL, VT, AddInput, Flags); in lowerFEXP2()
2834 SDValue TwoExpNeg64 = DAG.getConstantFP(0x1.0p-64f, SL, VT); in lowerFEXP2()
2835 SDValue One = DAG.getConstantFP(1.0, SL, VT); in lowerFEXP2()
2837 DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, TwoExpNeg64, One); in lowerFEXP2()
2839 return DAG.getNode(ISD::FMUL, SL, VT, Exp2, ResultScale, Flags); in lowerFEXP2()
2842 SDValue AMDGPUTargetLowering::lowerFEXPUnsafe(SDValue X, const SDLoc &SL, in lowerFEXPUnsafe() argument
2846 const SDValue Log2E = DAG.getConstantFP(numbers::log2e, SL, VT); in lowerFEXPUnsafe()
2850 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, X, Log2E, Flags); in lowerFEXPUnsafe()
2853 SL, VT, Mul, Flags); in lowerFEXPUnsafe()
2858 SDValue Threshold = DAG.getConstantFP(-0x1.5d58a0p+6f, SL, VT); in lowerFEXPUnsafe()
2859 SDValue NeedsScaling = DAG.getSetCC(SL, SetCCVT, X, Threshold, ISD::SETOLT); in lowerFEXPUnsafe()
2861 SDValue ScaleOffset = DAG.getConstantFP(0x1.0p+6f, SL, VT); in lowerFEXPUnsafe()
2863 SDValue ScaledX = DAG.getNode(ISD::FADD, SL, VT, X, ScaleOffset, Flags); in lowerFEXPUnsafe()
2866 DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, ScaledX, X); in lowerFEXPUnsafe()
2868 SDValue ExpInput = DAG.getNode(ISD::FMUL, SL, VT, AdjustedX, Log2E, Flags); in lowerFEXPUnsafe()
2870 SDValue Exp2 = DAG.getNode(AMDGPUISD::EXP, SL, VT, ExpInput, Flags); in lowerFEXPUnsafe()
2872 SDValue ResultScaleFactor = DAG.getConstantFP(0x1.969d48p-93f, SL, VT); in lowerFEXPUnsafe()
2874 DAG.getNode(ISD::FMUL, SL, VT, Exp2, ResultScaleFactor, Flags); in lowerFEXPUnsafe()
2876 return DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, AdjustedResult, Exp2, in lowerFEXPUnsafe()
2882 SDValue AMDGPUTargetLowering::lowerFEXP10Unsafe(SDValue X, const SDLoc &SL, in lowerFEXP10Unsafe() argument
2890 SDValue K0 = DAG.getConstantFP(0x1.a92000p+1f, SL, VT); in lowerFEXP10Unsafe()
2891 SDValue K1 = DAG.getConstantFP(0x1.4f0978p-11f, SL, VT); in lowerFEXP10Unsafe()
2893 SDValue Mul0 = DAG.getNode(ISD::FMUL, SL, VT, X, K0, Flags); in lowerFEXP10Unsafe()
2894 SDValue Exp2_0 = DAG.getNode(Exp2Op, SL, VT, Mul0, Flags); in lowerFEXP10Unsafe()
2895 SDValue Mul1 = DAG.getNode(ISD::FMUL, SL, VT, X, K1, Flags); in lowerFEXP10Unsafe()
2896 SDValue Exp2_1 = DAG.getNode(Exp2Op, SL, VT, Mul1, Flags); in lowerFEXP10Unsafe()
2897 return DAG.getNode(ISD::FMUL, SL, VT, Exp2_0, Exp2_1); in lowerFEXP10Unsafe()
2908 SDValue Threshold = DAG.getConstantFP(-0x1.2f7030p+5f, SL, VT); in lowerFEXP10Unsafe()
2909 SDValue NeedsScaling = DAG.getSetCC(SL, SetCCVT, X, Threshold, ISD::SETOLT); in lowerFEXP10Unsafe()
2911 SDValue ScaleOffset = DAG.getConstantFP(0x1.0p+5f, SL, VT); in lowerFEXP10Unsafe()
2912 SDValue ScaledX = DAG.getNode(ISD::FADD, SL, VT, X, ScaleOffset, Flags); in lowerFEXP10Unsafe()
2914 DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, ScaledX, X); in lowerFEXP10Unsafe()
2916 SDValue K0 = DAG.getConstantFP(0x1.a92000p+1f, SL, VT); in lowerFEXP10Unsafe()
2917 SDValue K1 = DAG.getConstantFP(0x1.4f0978p-11f, SL, VT); in lowerFEXP10Unsafe()
2919 SDValue Mul0 = DAG.getNode(ISD::FMUL, SL, VT, AdjustedX, K0, Flags); in lowerFEXP10Unsafe()
2920 SDValue Exp2_0 = DAG.getNode(Exp2Op, SL, VT, Mul0, Flags); in lowerFEXP10Unsafe()
2921 SDValue Mul1 = DAG.getNode(ISD::FMUL, SL, VT, AdjustedX, K1, Flags); in lowerFEXP10Unsafe()
2922 SDValue Exp2_1 = DAG.getNode(Exp2Op, SL, VT, Mul1, Flags); in lowerFEXP10Unsafe()
2924 SDValue MulExps = DAG.getNode(ISD::FMUL, SL, VT, Exp2_0, Exp2_1, Flags); in lowerFEXP10Unsafe()
2926 SDValue ResultScaleFactor = DAG.getConstantFP(0x1.9f623ep-107f, SL, VT); in lowerFEXP10Unsafe()
2928 DAG.getNode(ISD::FMUL, SL, VT, MulExps, ResultScaleFactor, Flags); in lowerFEXP10Unsafe()
2930 return DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, AdjustedResult, MulExps, in lowerFEXP10Unsafe()
2936 SDLoc SL(Op); in lowerFEXP() local
2944 return lowerFEXPUnsafe(X, SL, DAG, Flags); in lowerFEXP()
2953 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, X, Flags); in lowerFEXP()
2954 SDValue Lowered = lowerFEXPUnsafe(Ext, SL, DAG, Flags); in lowerFEXP()
2955 return DAG.getNode(ISD::FP_ROUND, SL, VT, Lowered, in lowerFEXP()
2956 DAG.getTargetConstant(0, SL, MVT::i32), Flags); in lowerFEXP()
2964 return IsExp10 ? lowerFEXP10Unsafe(X, SL, DAG, Flags) in lowerFEXP()
2965 : lowerFEXPUnsafe(X, SL, DAG, Flags); in lowerFEXP()
3002 SDValue C = DAG.getConstantFP(IsExp10 ? c_exp10 : c_exp, SL, VT); in lowerFEXP()
3003 SDValue CC = DAG.getConstantFP(IsExp10 ? cc_exp10 : cc_exp, SL, VT); in lowerFEXP()
3005 PH = DAG.getNode(ISD::FMUL, SL, VT, X, C, Flags); in lowerFEXP()
3006 SDValue NegPH = DAG.getNode(ISD::FNEG, SL, VT, PH, Flags); in lowerFEXP()
3007 SDValue FMA0 = DAG.getNode(ISD::FMA, SL, VT, X, C, NegPH, Flags); in lowerFEXP()
3008 PL = DAG.getNode(ISD::FMA, SL, VT, X, CC, FMA0, Flags); in lowerFEXP()
3016 SDValue CH = DAG.getConstantFP(IsExp10 ? ch_exp10 : ch_exp, SL, VT); in lowerFEXP()
3017 SDValue CL = DAG.getConstantFP(IsExp10 ? cl_exp10 : cl_exp, SL, VT); in lowerFEXP()
3019 SDValue XAsInt = DAG.getNode(ISD::BITCAST, SL, MVT::i32, X); in lowerFEXP()
3020 SDValue MaskConst = DAG.getConstant(0xfffff000, SL, MVT::i32); in lowerFEXP()
3021 SDValue XHAsInt = DAG.getNode(ISD::AND, SL, MVT::i32, XAsInt, MaskConst); in lowerFEXP()
3022 SDValue XH = DAG.getNode(ISD::BITCAST, SL, VT, XHAsInt); in lowerFEXP()
3023 SDValue XL = DAG.getNode(ISD::FSUB, SL, VT, X, XH, Flags); in lowerFEXP()
3025 PH = DAG.getNode(ISD::FMUL, SL, VT, XH, CH, Flags); in lowerFEXP()
3027 SDValue XLCL = DAG.getNode(ISD::FMUL, SL, VT, XL, CL, Flags); in lowerFEXP()
3028 SDValue Mad0 = getMad(DAG, SL, VT, XL, CH, XLCL, Flags); in lowerFEXP()
3029 PL = getMad(DAG, SL, VT, XH, CL, Mad0, Flags); in lowerFEXP()
3032 SDValue E = DAG.getNode(ISD::FROUNDEVEN, SL, VT, PH, Flags); in lowerFEXP()
3035 SDValue PHSubE = DAG.getNode(ISD::FSUB, SL, VT, PH, E, FlagsNoContract); in lowerFEXP()
3037 SDValue A = DAG.getNode(ISD::FADD, SL, VT, PHSubE, PL, Flags); in lowerFEXP()
3038 SDValue IntE = DAG.getNode(ISD::FP_TO_SINT, SL, MVT::i32, E); in lowerFEXP()
3039 SDValue Exp2 = DAG.getNode(AMDGPUISD::EXP, SL, VT, A, Flags); in lowerFEXP()
3041 SDValue R = DAG.getNode(ISD::FLDEXP, SL, VT, Exp2, IntE, Flags); in lowerFEXP()
3044 DAG.getConstantFP(IsExp10 ? -0x1.66d3e8p+5f : -0x1.9d1da0p+6f, SL, VT); in lowerFEXP()
3047 SDValue Zero = DAG.getConstantFP(0.0, SL, VT); in lowerFEXP()
3049 DAG.getSetCC(SL, SetCCVT, X, UnderflowCheckConst, ISD::SETOLT); in lowerFEXP()
3051 R = DAG.getNode(ISD::SELECT, SL, VT, Underflow, Zero, R); in lowerFEXP()
3056 DAG.getConstantFP(IsExp10 ? 0x1.344136p+5f : 0x1.62e430p+6f, SL, VT); in lowerFEXP()
3058 DAG.getSetCC(SL, SetCCVT, X, OverflowCheckConst, ISD::SETOGT); in lowerFEXP()
3060 DAG.getConstantFP(APFloat::getInf(APFloat::IEEEsingle()), SL, VT); in lowerFEXP()
3061 R = DAG.getNode(ISD::SELECT, SL, VT, Overflow, Inf, R); in lowerFEXP()
3077 auto SL = SDLoc(Op); in lowerCTLZResults() local
3088 auto NewOp = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Arg); in lowerCTLZResults()
3089 auto ShiftVal = DAG.getConstant(LeadingZeroes, SL, MVT::i32); in lowerCTLZResults()
3090 NewOp = DAG.getNode(ISD::SHL, SL, MVT::i32, NewOp, ShiftVal); in lowerCTLZResults()
3091 NewOp = DAG.getNode(Op.getOpcode(), SL, MVT::i32, NewOp); in lowerCTLZResults()
3092 return DAG.getNode(ISD::TRUNCATE, SL, ResultVT, NewOp); in lowerCTLZResults()
3096 SDLoc SL(Op); in LowerCTLZ_CTTZ() local
3118 SDValue NewOpr = DAG.getNode(NewOpc, SL, MVT::i32, Src); in LowerCTLZ_CTTZ()
3121 Op.getValueType().getScalarSizeInBits(), SL, MVT::i32); in LowerCTLZ_CTTZ()
3122 NewOpr = DAG.getNode(ISD::UMIN, SL, MVT::i32, NewOpr, ConstVal); in LowerCTLZ_CTTZ()
3124 return DAG.getNode(ISD::ZERO_EXTEND, SL, Src.getValueType(), NewOpr); in LowerCTLZ_CTTZ()
3130 SDValue OprLo = DAG.getNode(NewOpc, SL, MVT::i32, Lo); in LowerCTLZ_CTTZ()
3131 SDValue OprHi = DAG.getNode(NewOpc, SL, MVT::i32, Hi); in LowerCTLZ_CTTZ()
3139 const SDValue Const32 = DAG.getConstant(32, SL, MVT::i32); in LowerCTLZ_CTTZ()
3141 OprLo = DAG.getNode(AddOpc, SL, MVT::i32, OprLo, Const32); in LowerCTLZ_CTTZ()
3143 OprHi = DAG.getNode(AddOpc, SL, MVT::i32, OprHi, Const32); in LowerCTLZ_CTTZ()
3146 NewOpr = DAG.getNode(ISD::UMIN, SL, MVT::i32, OprLo, OprHi); in LowerCTLZ_CTTZ()
3148 const SDValue Const64 = DAG.getConstant(64, SL, MVT::i32); in LowerCTLZ_CTTZ()
3149 NewOpr = DAG.getNode(ISD::UMIN, SL, MVT::i32, NewOpr, Const64); in LowerCTLZ_CTTZ()
3152 return DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i64, NewOpr); in LowerCTLZ_CTTZ()
3183 SDLoc SL(Op); in LowerINT_TO_FP32() local
3213 ISD::SRA, SL, MVT::i32, DAG.getNode(ISD::XOR, SL, MVT::i32, Lo, Hi), in LowerINT_TO_FP32()
3214 DAG.getConstant(31, SL, MVT::i32)); in LowerINT_TO_FP32()
3216 DAG.getNode(ISD::ADD, SL, MVT::i32, DAG.getConstant(32, SL, MVT::i32), in LowerINT_TO_FP32()
3219 ShAmt = DAG.getNode(AMDGPUISD::FFBH_I32, SL, MVT::i32, Hi); in LowerINT_TO_FP32()
3222 ShAmt = DAG.getNode(ISD::SUB, SL, MVT::i32, ShAmt, in LowerINT_TO_FP32()
3223 DAG.getConstant(1, SL, MVT::i32)); in LowerINT_TO_FP32()
3224 ShAmt = DAG.getNode(ISD::UMIN, SL, MVT::i32, ShAmt, MaxShAmt); in LowerINT_TO_FP32()
3229 Sign = DAG.getNode(ISD::SRA, SL, MVT::i64, Src, in LowerINT_TO_FP32()
3230 DAG.getConstant(63, SL, MVT::i64)); in LowerINT_TO_FP32()
3232 DAG.getNode(ISD::XOR, SL, MVT::i64, in LowerINT_TO_FP32()
3233 DAG.getNode(ISD::ADD, SL, MVT::i64, Src, Sign), Sign); in LowerINT_TO_FP32()
3237 ShAmt = DAG.getNode(ISD::CTLZ, SL, MVT::i32, Hi); in LowerINT_TO_FP32()
3241 SDValue Norm = DAG.getNode(ISD::SHL, SL, MVT::i64, Src, ShAmt); in LowerINT_TO_FP32()
3246 SDValue Adjust = DAG.getNode(ISD::UMIN, SL, MVT::i32, in LowerINT_TO_FP32()
3247 DAG.getConstant(1, SL, MVT::i32), Lo); in LowerINT_TO_FP32()
3249 Norm = DAG.getNode(ISD::OR, SL, MVT::i32, Hi, Adjust); in LowerINT_TO_FP32()
3253 SDValue FVal = DAG.getNode(Opc, SL, MVT::f32, Norm); in LowerINT_TO_FP32()
3257 ShAmt = DAG.getNode(ISD::SUB, SL, MVT::i32, DAG.getConstant(32, SL, MVT::i32), in LowerINT_TO_FP32()
3261 return DAG.getNode(ISD::FLDEXP, SL, MVT::f32, FVal, ShAmt); in LowerINT_TO_FP32()
3266 SDValue Exp = DAG.getNode(ISD::SHL, SL, MVT::i32, ShAmt, in LowerINT_TO_FP32()
3267 DAG.getConstant(23, SL, MVT::i32)); in LowerINT_TO_FP32()
3269 DAG.getNode(ISD::ADD, SL, MVT::i32, in LowerINT_TO_FP32()
3270 DAG.getNode(ISD::BITCAST, SL, MVT::i32, FVal), Exp); in LowerINT_TO_FP32()
3273 Sign = DAG.getNode(ISD::SHL, SL, MVT::i32, in LowerINT_TO_FP32()
3274 DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Sign), in LowerINT_TO_FP32()
3275 DAG.getConstant(31, SL, MVT::i32)); in LowerINT_TO_FP32()
3276 IVal = DAG.getNode(ISD::OR, SL, MVT::i32, IVal, Sign); in LowerINT_TO_FP32()
3278 return DAG.getNode(ISD::BITCAST, SL, MVT::f32, IVal); in LowerINT_TO_FP32()
3283 SDLoc SL(Op); in LowerINT_TO_FP64() local
3290 SL, MVT::f64, Hi); in LowerINT_TO_FP64()
3292 SDValue CvtLo = DAG.getNode(ISD::UINT_TO_FP, SL, MVT::f64, Lo); in LowerINT_TO_FP64()
3294 SDValue LdExp = DAG.getNode(ISD::FLDEXP, SL, MVT::f64, CvtHi, in LowerINT_TO_FP64()
3295 DAG.getConstant(32, SL, MVT::i32)); in LowerINT_TO_FP64()
3297 return DAG.getNode(ISD::FADD, SL, MVT::f64, LdExp, CvtLo); in LowerINT_TO_FP64()
3318 SDLoc SL(Op); in LowerUINT_TO_FP() local
3319 SDValue ToF32 = DAG.getNode(ISD::UINT_TO_FP, SL, MVT::f32, Src); in LowerUINT_TO_FP()
3320 SDValue FPRoundFlag = DAG.getIntPtrConstant(0, SL, /*isTarget=*/true); in LowerUINT_TO_FP()
3321 return DAG.getNode(ISD::FP_ROUND, SL, MVT::bf16, ToF32, FPRoundFlag); in LowerUINT_TO_FP()
3364 SDLoc SL(Op); in LowerSINT_TO_FP() local
3365 SDValue ToF32 = DAG.getNode(ISD::SINT_TO_FP, SL, MVT::f32, Src); in LowerSINT_TO_FP()
3366 SDValue FPRoundFlag = DAG.getIntPtrConstant(0, SL, /*isTarget=*/true); in LowerSINT_TO_FP()
3367 return DAG.getNode(ISD::FP_ROUND, SL, MVT::bf16, ToF32, FPRoundFlag); in LowerSINT_TO_FP()
3397 SDLoc SL(Op); in LowerFP_TO_INT64() local
3413 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, SrcVT, Src); in LowerFP_TO_INT64()
3421 Sign = DAG.getNode(ISD::SRA, SL, MVT::i32, in LowerFP_TO_INT64()
3422 DAG.getNode(ISD::BITCAST, SL, MVT::i32, Trunc), in LowerFP_TO_INT64()
3423 DAG.getConstant(31, SL, MVT::i32)); in LowerFP_TO_INT64()
3424 Trunc = DAG.getNode(ISD::FABS, SL, SrcVT, Trunc); in LowerFP_TO_INT64()
3430 llvm::bit_cast<double>(UINT64_C(/*2^-32*/ 0x3df0000000000000)), SL, in LowerFP_TO_INT64()
3433 llvm::bit_cast<double>(UINT64_C(/*-2^32*/ 0xc1f0000000000000)), SL, in LowerFP_TO_INT64()
3437 llvm::bit_cast<float>(UINT32_C(/*2^-32*/ 0x2f800000)), SL, SrcVT); in LowerFP_TO_INT64()
3439 llvm::bit_cast<float>(UINT32_C(/*-2^32*/ 0xcf800000)), SL, SrcVT); in LowerFP_TO_INT64()
3442 SDValue Mul = DAG.getNode(ISD::FMUL, SL, SrcVT, Trunc, K0); in LowerFP_TO_INT64()
3444 SDValue FloorMul = DAG.getNode(ISD::FFLOOR, SL, SrcVT, Mul); in LowerFP_TO_INT64()
3446 SDValue Fma = DAG.getNode(ISD::FMA, SL, SrcVT, FloorMul, K1, Trunc); in LowerFP_TO_INT64()
3450 SL, MVT::i32, FloorMul); in LowerFP_TO_INT64()
3451 SDValue Lo = DAG.getNode(ISD::FP_TO_UINT, SL, MVT::i32, Fma); in LowerFP_TO_INT64()
3453 SDValue Result = DAG.getNode(ISD::BITCAST, SL, MVT::i64, in LowerFP_TO_INT64()
3454 DAG.getBuildVector(MVT::v2i32, SL, {Lo, Hi})); in LowerFP_TO_INT64()
3459 Sign = DAG.getNode(ISD::BITCAST, SL, MVT::i64, in LowerFP_TO_INT64()
3460 DAG.getBuildVector(MVT::v2i32, SL, {Sign, Sign})); in LowerFP_TO_INT64()
3463 DAG.getNode(ISD::SUB, SL, MVT::i64, in LowerFP_TO_INT64()
3464 DAG.getNode(ISD::XOR, SL, MVT::i64, Result, Sign), Sign); in LowerFP_TO_INT64()
3755 SDLoc SL(N); in performLoadCombine() local
3789 = DAG.getLoad(NewVT, SL, LN->getChain(), in performLoadCombine()
3792 SDValue BC = DAG.getNode(ISD::BITCAST, SL, VT, NewLoad); in performLoadCombine()
3811 SDLoc SL(N); in performStoreCombine() local
3843 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NewVT, Val); in performStoreCombine()
3845 SDValue CastBack = DAG.getNode(ISD::BITCAST, SL, VT, CastVal); in performStoreCombine()
3849 return DAG.getStore(SN->getChain(), SL, CastVal, in performStoreCombine()
3866 SDLoc SL(N); in performAssertSZExtCombine() local
3871 SDValue NewInReg = DAG.getNode(N->getOpcode(), SL, SrcVT, Src, N1); in performAssertSZExtCombine()
3872 return DAG.getNode(ISD::TRUNCATE, SL, N->getValueType(0), NewInReg); in performAssertSZExtCombine()
3916 DAGCombinerInfo &DCI, const SDLoc &SL, in splitBinaryBitConstantOpImpl() argument
3923 SDValue LoRHS = DAG.getConstant(ValLo, SL, MVT::i32); in splitBinaryBitConstantOpImpl()
3924 SDValue HiRHS = DAG.getConstant(ValHi, SL, MVT::i32); in splitBinaryBitConstantOpImpl()
3926 SDValue LoAnd = DAG.getNode(Opc, SL, MVT::i32, Lo, LoRHS); in splitBinaryBitConstantOpImpl()
3927 SDValue HiAnd = DAG.getNode(Opc, SL, MVT::i32, Hi, HiRHS); in splitBinaryBitConstantOpImpl()
3934 SDValue Vec = DAG.getBuildVector(MVT::v2i32, SL, {LoAnd, HiAnd}); in splitBinaryBitConstantOpImpl()
3935 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec); in splitBinaryBitConstantOpImpl()
3951 SDLoc SL(N); in performShlCombine() local
3966 SDValue Vec = DAG.getBuildVector(MVT::v2i16, SL, in performShlCombine()
3967 { DAG.getConstant(0, SL, MVT::i16), LHS->getOperand(0) }); in performShlCombine()
3968 return DAG.getNode(ISD::BITCAST, SL, MVT::i32, Vec); in performShlCombine()
3979 SDValue Shl = DAG.getNode(ISD::SHL, SL, XVT, X, SDValue(RHS, 0)); in performShlCombine()
3980 return DAG.getZExtOrTrunc(Shl, SL, VT); in performShlCombine()
3995 SDValue ShiftAmt = DAG.getConstant(RHSVal - 32, SL, MVT::i32); in performShlCombine()
3997 SDValue Lo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, LHS); in performShlCombine()
3998 SDValue NewShift = DAG.getNode(ISD::SHL, SL, MVT::i32, Lo, ShiftAmt); in performShlCombine()
4000 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32); in performShlCombine()
4002 SDValue Vec = DAG.getBuildVector(MVT::v2i32, SL, {Zero, NewShift}); in performShlCombine()
4003 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec); in performShlCombine()
4016 SDLoc SL(N); in performSraCombine() local
4022 SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi, in performSraCombine()
4023 DAG.getConstant(31, SL, MVT::i32)); in performSraCombine()
4025 SDValue BuildVec = DAG.getBuildVector(MVT::v2i32, SL, {Hi, NewShift}); in performSraCombine()
4026 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildVec); in performSraCombine()
4032 SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi, in performSraCombine()
4033 DAG.getConstant(31, SL, MVT::i32)); in performSraCombine()
4034 SDValue BuildVec = DAG.getBuildVector(MVT::v2i32, SL, {NewShift, NewShift}); in performSraCombine()
4035 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildVec); in performSraCombine()
4051 SDLoc SL(N); in performSrlCombine() local
4061 ISD::AND, SL, VT, in performSrlCombine()
4062 DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(0), N->getOperand(1)), in performSrlCombine()
4063 DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(1), N->getOperand(1))); in performSrlCombine()
4077 SDValue Zero = DAG.getConstant(0, SL, MVT::i32); in performSrlCombine()
4081 SDValue NewConst = DAG.getConstant(ShiftAmt - 32, SL, MVT::i32); in performSrlCombine()
4082 SDValue NewShift = DAG.getNode(ISD::SRL, SL, MVT::i32, Hi, NewConst); in performSrlCombine()
4084 SDValue BuildPair = DAG.getBuildVector(MVT::v2i32, SL, {NewShift, Zero}); in performSrlCombine()
4086 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildPair); in performSrlCombine()
4091 SDLoc SL(N); in performTruncateCombine() local
4104 Elt0 = DAG.getNode(ISD::BITCAST, SL, in performTruncateCombine()
4108 return DAG.getNode(ISD::TRUNCATE, SL, VT, Elt0); in performTruncateCombine()
4125 SrcElt = DAG.getNode(ISD::BITCAST, SL, in performTruncateCombine()
4129 return DAG.getNode(ISD::TRUNCATE, SL, VT, SrcElt); in performTruncateCombine()
4160 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, MidVT, in performTruncateCombine()
4165 Amt = DAG.getZExtOrTrunc(Amt, SL, NewShiftVT); in performTruncateCombine()
4169 SDValue ShrunkShift = DAG.getNode(Src.getOpcode(), SL, MidVT, in performTruncateCombine()
4171 return DAG.getNode(ISD::TRUNCATE, SL, VT, ShrunkShift); in performTruncateCombine()
4183 static SDValue getMul24(SelectionDAG &DAG, const SDLoc &SL, in getMul24() argument
4187 return DAG.getNode(MulOpc, SL, MVT::i32, N0, N1); in getMul24()
4193 SDValue MulLo = DAG.getNode(MulLoOpc, SL, MVT::i32, N0, N1); in getMul24()
4194 SDValue MulHi = DAG.getNode(MulHiOpc, SL, MVT::i32, N0, N1); in getMul24()
4196 return DAG.getNode(ISD::BUILD_PAIR, SL, MVT::i64, MulLo, MulHi); in getMul24()
4432 SDValue AMDGPUTargetLowering::performCtlz_CttzCombine(const SDLoc &SL, SDValue Cond, in performCtlz_CttzCombine() argument
4449 return getFFBX_U32(DAG, CmpLHS, SL, Opc); in performCtlz_CttzCombine()
4460 return getFFBX_U32(DAG, CmpLHS, SL, Opc); in performCtlz_CttzCombine()
4468 const SDLoc &SL, in distributeOpThroughSelect() argument
4475 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT, Cond, in distributeOpThroughSelect()
4478 return DAG.getNode(Op, SL, VT, NewSelect); in distributeOpThroughSelect()
4516 SDLoc SL(N); in foldFreeOpFromSelect() local
4551 NewRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in foldFreeOpFromSelect()
4556 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT, in foldFreeOpFromSelect()
4559 return DAG.getNode(LHS.getOpcode(), SL, VT, NewSelect); in foldFreeOpFromSelect()
4591 SDLoc SL(N); in performSelectCombine() local
4595 SDValue NewCond = DAG.getSetCC(SL, Cond.getValueType(), LHS, RHS, NewCC); in performSelectCombine()
4596 return DAG.getNode(ISD::SELECT, SL, VT, NewCond, False, True); in performSelectCombine()
4702 SDLoc SL(N); in performFNegCombine() local
4713 LHS = DAG.getNode(ISD::FNEG, SL, VT, LHS); in performFNegCombine()
4718 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in performFNegCombine()
4722 SDValue Res = DAG.getNode(ISD::FADD, SL, VT, LHS, RHS, N0->getFlags()); in performFNegCombine()
4726 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res)); in performFNegCombine()
4741 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in performFNegCombine()
4743 SDValue Res = DAG.getNode(Opc, SL, VT, LHS, RHS, N0->getFlags()); in performFNegCombine()
4747 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res)); in performFNegCombine()
4766 MHS = DAG.getNode(ISD::FNEG, SL, VT, MHS); in performFNegCombine()
4769 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in performFNegCombine()
4773 SDValue Res = DAG.getNode(Opc, SL, VT, LHS, MHS, RHS); in performFNegCombine()
4777 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res)); in performFNegCombine()
4801 SDValue NegLHS = DAG.getNode(ISD::FNEG, SL, VT, LHS); in performFNegCombine()
4802 SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in performFNegCombine()
4805 SDValue Res = DAG.getNode(Opposite, SL, VT, NegLHS, NegRHS, N0->getFlags()); in performFNegCombine()
4809 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res)); in performFNegCombine()
4815 Ops[I] = DAG.getNode(ISD::FNEG, SL, VT, N0->getOperand(I), N0->getFlags()); in performFNegCombine()
4817 SDValue Res = DAG.getNode(AMDGPUISD::FMED3, SL, VT, Ops, N0->getFlags()); in performFNegCombine()
4822 SDValue Neg = DAG.getNode(ISD::FNEG, SL, VT, Res); in performFNegCombine()
4846 return DAG.getNode(Opc, SL, VT, CvtSrc.getOperand(0)); in performFNegCombine()
4854 SDValue Neg = DAG.getNode(ISD::FNEG, SL, CvtSrc.getValueType(), CvtSrc); in performFNegCombine()
4855 return DAG.getNode(Opc, SL, VT, Neg, N0->getFlags()); in performFNegCombine()
4862 return DAG.getNode(ISD::FP_ROUND, SL, VT, in performFNegCombine()
4870 SDValue Neg = DAG.getNode(ISD::FNEG, SL, CvtSrc.getValueType(), CvtSrc); in performFNegCombine()
4871 return DAG.getNode(ISD::FP_ROUND, SL, VT, Neg, N0.getOperand(1)); in performFNegCombine()
4877 SDLoc SL(N); in performFNegCombine() local
4883 SDValue IntFNeg = DAG.getNode(ISD::XOR, SL, SrcVT, Src, in performFNegCombine()
4884 DAG.getConstant(0x8000, SL, SrcVT)); in performFNegCombine()
4885 return DAG.getNode(ISD::FP16_TO_FP, SL, N->getValueType(0), IntFNeg); in performFNegCombine()
4893 SDLoc SL(N); in performFNegCombine() local
4910 SDValue CastHi = DAG.getNode(ISD::BITCAST, SL, MVT::f32, HighBits); in performFNegCombine()
4911 SDValue NegHi = DAG.getNode(ISD::FNEG, SL, MVT::f32, CastHi); in performFNegCombine()
4913 DAG.getNode(ISD::BITCAST, SL, HighBits.getValueType(), NegHi); in performFNegCombine()
4919 DAG.getNode(ISD::BUILD_VECTOR, SL, BCSrc.getValueType(), Ops); in performFNegCombine()
4920 SDValue Result = DAG.getNode(ISD::BITCAST, SL, VT, Build); in performFNegCombine()
4923 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Result)); in performFNegCombine()
4935 DAG.getNode(ISD::BITCAST, SL, MVT::f32, BCSrc.getOperand(1)); in performFNegCombine()
4937 DAG.getNode(ISD::BITCAST, SL, MVT::f32, BCSrc.getOperand(2)); in performFNegCombine()
4939 SDValue NegLHS = DAG.getNode(ISD::FNEG, SL, MVT::f32, LHS); in performFNegCombine()
4940 SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, MVT::f32, RHS); in performFNegCombine()
4942 return DAG.getNode(ISD::SELECT, SL, MVT::f32, BCSrc.getOperand(0), NegLHS, in performFNegCombine()
4964 SDLoc SL(N); in performFAbsCombine() local
4969 SDValue IntFAbs = DAG.getNode(ISD::AND, SL, SrcVT, Src, in performFAbsCombine()
4970 DAG.getConstant(0x7fff, SL, SrcVT)); in performFAbsCombine()
4971 return DAG.getNode(ISD::FP16_TO_FP, SL, N->getValueType(0), IntFAbs); in performFAbsCombine()
5018 SDLoc SL(N); in PerformDAGCombine() local
5024 return DAG.getBuildVector(DestVT, SL, CastedElts); in PerformDAGCombine()
5038 SDLoc SL(N); in PerformDAGCombine() local
5040 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, in PerformDAGCombine()
5041 DAG.getConstant(Lo_32(CVal), SL, MVT::i32), in PerformDAGCombine()
5042 DAG.getConstant(Hi_32(CVal), SL, MVT::i32)); in PerformDAGCombine()
5043 return DAG.getNode(ISD::BITCAST, SL, DestVT, BV); in PerformDAGCombine()
5048 SDLoc SL(N); in PerformDAGCombine() local
5050 SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, in PerformDAGCombine()
5051 DAG.getConstant(Lo_32(CVal), SL, MVT::i32), in PerformDAGCombine()
5052 DAG.getConstant(Hi_32(CVal), SL, MVT::i32)); in PerformDAGCombine()
5054 return DAG.getNode(ISD::BITCAST, SL, DestVT, Vec); in PerformDAGCombine()
5240 const SDLoc &SL, in CreateLiveInRegister() argument
5256 return DAG.getCopyFromReg(DAG.getEntryNode(), SL, VReg, VT); in CreateLiveInRegister()
5275 const SDLoc &SL, in loadStackInputValue() argument
5284 return DAG.getLoad(VT, SL, DAG.getEntryNode(), Ptr, SrcPtrInfo, Align(4), in loadStackInputValue()
5290 const SDLoc &SL, in storeStackInputValue() argument
5298 SDValue Ptr = DAG.getConstant(Offset, SL, MVT::i32); in storeStackInputValue()
5301 DAG.getCopyFromReg(Chain, SL, Info->getStackPtrOffsetReg(), MVT::i32); in storeStackInputValue()
5302 Ptr = DAG.getNode(ISD::ADD, SL, MVT::i32, SP, Ptr); in storeStackInputValue()
5303 SDValue Store = DAG.getStore(Chain, SL, ArgVal, Ptr, DstInfo, Align(4), in storeStackInputValue()
5310 EVT VT, const SDLoc &SL, in loadInputValue() argument
5315 CreateLiveInRegister(DAG, RC, Arg.getRegister(), VT, SL) : in loadInputValue()
5316 loadStackInputValue(DAG, VT, SL, Arg.getStackOffset()); in loadInputValue()
5323 V = DAG.getNode(ISD::SRL, SL, VT, V, in loadInputValue()
5324 DAG.getShiftAmountConstant(Shift, VT, SL)); in loadInputValue()
5325 return DAG.getNode(ISD::AND, SL, VT, V, in loadInputValue()
5326 DAG.getConstant(Mask >> Shift, SL, VT)); in loadInputValue()