Lines Matching refs:getNode
246 if (ISD::isConstantSplatVectorAllOnes(Op.getNode())) in isZeroingInactiveLanes()
1815 New = TLO.DAG.getNode(Op.getOpcode(), DL, VT, Op.getOperand(0), in optimizeLogicalImm()
2673 N = N->getOperand(0).getNode(); in isZerosVector()
2873 LHS = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {MVT::f32, MVT::Other}, in emitStrictFPComparison()
2875 RHS = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {MVT::f32, MVT::Other}, in emitStrictFPComparison()
2882 return DAG.getNode(Opcode, dl, {VT, MVT::Other}, {Chain, LHS, RHS}); in emitStrictFPComparison()
2893 LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, LHS); in emitComparison()
2894 RHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, RHS); in emitComparison()
2897 return DAG.getNode(AArch64ISD::FCMP, dl, VT, LHS, RHS); in emitComparison()
2920 const SDValue ANDSNode = DAG.getNode(AArch64ISD::ANDS, dl, in emitComparison()
2933 return DAG.getNode(Opcode, dl, DAG.getVTList(VT, MVT_CC), LHS, RHS) in emitComparison()
3001 LHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, LHS); in emitConditionalComparison()
3002 RHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, RHS); in emitConditionalComparison()
3020 return DAG.getNode(Opcode, DL, MVT_CC, LHS, RHS, NZCVOp, Condition, CCOp); in emitConditionalComparison()
3123 if (!CCOp.getNode()) in emitConjunctionRec()
3262 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS.getNode())) { in getAArch64Cmp()
3361 LHS.getNode()->hasNUsesOfValue(1, 0)) { in getAArch64Cmp()
3365 DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, LHS.getValueType(), LHS, in getAArch64Cmp()
3426 LHS = DAG.getNode(ExtendOpc, DL, MVT::i64, LHS); in getAArch64XALUOOp()
3427 RHS = DAG.getNode(ExtendOpc, DL, MVT::i64, RHS); in getAArch64XALUOOp()
3428 SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS); in getAArch64XALUOOp()
3429 Value = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Mul); in getAArch64XALUOOp()
3435 SDValue SExtMul = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Value); in getAArch64XALUOOp()
3437 DAG.getNode(AArch64ISD::SUBS, DL, VTs, Mul, SExtMul).getValue(1); in getAArch64XALUOOp()
3442 DAG.getNode(AArch64ISD::ANDS, DL, VTs, Mul, UpperBits).getValue(1); in getAArch64XALUOOp()
3448 Value = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS); in getAArch64XALUOOp()
3450 SDValue UpperBits = DAG.getNode(ISD::MULHS, DL, MVT::i64, LHS, RHS); in getAArch64XALUOOp()
3451 SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i64, Value, in getAArch64XALUOOp()
3456 Overflow = DAG.getNode(AArch64ISD::SUBS, DL, VTs, UpperBits, LowerBits) in getAArch64XALUOOp()
3459 SDValue UpperBits = DAG.getNode(ISD::MULHU, DL, MVT::i64, LHS, RHS); in getAArch64XALUOOp()
3462 DAG.getNode(AArch64ISD::SUBS, DL, VTs, in getAArch64XALUOOp()
3474 Value = DAG.getNode(Opc, DL, VTs, LHS, RHS); in getAArch64XALUOOp()
3506 return DAG.getNode(AArch64ISD::CSEL, dl, Op.getValueType(), TVal, FVal, in LowerXOR()
3553 TVal = DAG.getNode(ISD::XOR, dl, Other.getValueType(), Other, in LowerXOR()
3556 return DAG.getNode(AArch64ISD::CSEL, dl, Sel.getValueType(), FVal, TVal, in LowerXOR()
3572 DAG.getNode(AArch64ISD::SUBS, DL, DAG.getVTList(VT, MVT::Glue), Op0, Op1); in valueToCarryFlag()
3586 return DAG.getNode(AArch64ISD::CSEL, DL, VT, One, Zero, CC, Flag); in carryFlagToValue()
3596 return DAG.getNode(AArch64ISD::CSEL, DL, VT, One, Zero, CC, Flag); in overflowFlagToValue()
3617 SDValue Sum = DAG.getNode(Opcode, DL, DAG.getVTList(VT0, MVT::Glue), OpLHS, in lowerADDSUBCARRY()
3624 return DAG.getNode(ISD::MERGE_VALUES, DL, VTs, Sum, OutFlag); in lowerADDSUBCARRY()
3646 Overflow = DAG.getNode(AArch64ISD::CSEL, dl, MVT::i32, FVal, TVal, in LowerXALUO()
3650 return DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Value, Overflow); in LowerXALUO()
3680 return DAG.getNode(AArch64ISD::PREFETCH, DL, MVT::Other, Op.getOperand(0), in LowerPREFETCH()
3748 SDValue Ext = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NewVT, MVT::Other}, in LowerVectorFP_TO_INT()
3750 return DAG.getNode(Op.getOpcode(), dl, {VT, MVT::Other}, in LowerVectorFP_TO_INT()
3753 return DAG.getNode( in LowerVectorFP_TO_INT()
3755 DAG.getNode(ISD::FP_EXTEND, dl, NewVT, Op.getOperand(0))); in LowerVectorFP_TO_INT()
3764 SDValue Cv = DAG.getNode(Op.getOpcode(), dl, {InVT, MVT::Other}, in LowerVectorFP_TO_INT()
3766 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, VT, Cv); in LowerVectorFP_TO_INT()
3770 DAG.getNode(Op.getOpcode(), dl, InVT.changeVectorElementTypeToInteger(), in LowerVectorFP_TO_INT()
3772 return DAG.getNode(ISD::TRUNCATE, dl, VT, Cv); in LowerVectorFP_TO_INT()
3781 SDValue Ext = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {ExtVT, MVT::Other}, in LowerVectorFP_TO_INT()
3783 return DAG.getNode(Op.getOpcode(), dl, {VT, MVT::Other}, in LowerVectorFP_TO_INT()
3786 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, dl, ExtVT, Op.getOperand(0)); in LowerVectorFP_TO_INT()
3787 return DAG.getNode(Op.getOpcode(), dl, VT, Ext); in LowerVectorFP_TO_INT()
3794 SDValue Extract = DAG.getNode( in LowerVectorFP_TO_INT()
3799 return DAG.getNode(Op.getOpcode(), dl, {ScalarVT, MVT::Other}, in LowerVectorFP_TO_INT()
3801 return DAG.getNode(Op.getOpcode(), dl, ScalarVT, Extract); in LowerVectorFP_TO_INT()
3821 DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {MVT::f32, MVT::Other}, in LowerFP_TO_INT()
3823 return DAG.getNode(Op.getOpcode(), dl, {Op.getValueType(), MVT::Other}, in LowerFP_TO_INT()
3826 return DAG.getNode( in LowerFP_TO_INT()
3828 DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, SrcVal)); in LowerFP_TO_INT()
3867 SrcVal = DAG.getNode(ISD::FP_EXTEND, SDLoc(Op), F32VT, SrcVal); in LowerVectorFP_TO_INT_SAT()
3878 return DAG.getNode(Op.getOpcode(), DL, DstVT, SrcVal, in LowerVectorFP_TO_INT_SAT()
3889 SDValue NativeCvt = DAG.getNode(Op.getOpcode(), DL, IntVT, SrcVal, in LowerVectorFP_TO_INT_SAT()
3895 SDValue Min = DAG.getNode(ISD::SMIN, DL, IntVT, NativeCvt, MinC); in LowerVectorFP_TO_INT_SAT()
3898 Sat = DAG.getNode(ISD::SMAX, DL, IntVT, Min, MaxC); in LowerVectorFP_TO_INT_SAT()
3902 Sat = DAG.getNode(ISD::UMIN, DL, IntVT, NativeCvt, MinC); in LowerVectorFP_TO_INT_SAT()
3905 return DAG.getNode(ISD::TRUNCATE, DL, DstVT, Sat); in LowerVectorFP_TO_INT_SAT()
3926 SrcVal = DAG.getNode(ISD::FP_EXTEND, SDLoc(Op), MVT::f32, SrcVal); in LowerFP_TO_INT_SAT()
3936 return DAG.getNode(Op.getOpcode(), DL, DstVT, SrcVal, in LowerFP_TO_INT_SAT()
3946 DAG.getNode(Op.getOpcode(), DL, DstVT, SrcVal, DAG.getValueType(DstVT)); in LowerFP_TO_INT_SAT()
3951 SDValue Min = DAG.getNode(ISD::SMIN, DL, DstVT, NativeCvt, MinC); in LowerFP_TO_INT_SAT()
3954 Sat = DAG.getNode(ISD::SMAX, DL, DstVT, Min, MaxC); in LowerFP_TO_INT_SAT()
3958 Sat = DAG.getNode(ISD::UMIN, DL, DstVT, NativeCvt, MinC); in LowerFP_TO_INT_SAT()
3961 return DAG.getNode(ISD::TRUNCATE, DL, DstVT, Sat); in LowerFP_TO_INT_SAT()
3982 In = DAG.getNode(CastOpc, dl, CastVT, In); in LowerVectorINT_TO_FP()
3983 return DAG.getNode(Opc, dl, VT, In); in LowerVectorINT_TO_FP()
4001 In = DAG.getNode(Opc, dl, {CastVT, MVT::Other}, in LowerVectorINT_TO_FP()
4003 return DAG.getNode( in LowerVectorINT_TO_FP()
4007 In = DAG.getNode(Opc, dl, CastVT, In); in LowerVectorINT_TO_FP()
4008 return DAG.getNode(ISD::FP_ROUND, dl, VT, In, DAG.getIntPtrConstant(0, dl)); in LowerVectorINT_TO_FP()
4014 In = DAG.getNode(CastOpc, dl, CastVT, In); in LowerVectorINT_TO_FP()
4016 return DAG.getNode(Opc, dl, {VT, MVT::Other}, {Op.getOperand(0), In}); in LowerVectorINT_TO_FP()
4017 return DAG.getNode(Opc, dl, VT, In); in LowerVectorINT_TO_FP()
4023 SDValue Extract = DAG.getNode( in LowerVectorINT_TO_FP()
4028 return DAG.getNode(Op.getOpcode(), dl, {ScalarVT, MVT::Other}, in LowerVectorINT_TO_FP()
4030 return DAG.getNode(Op.getOpcode(), dl, ScalarVT, Extract); in LowerVectorINT_TO_FP()
4048 SDValue Val = DAG.getNode(Op.getOpcode(), dl, {MVT::f32, MVT::Other}, in LowerINT_TO_FP()
4050 return DAG.getNode( in LowerINT_TO_FP()
4054 return DAG.getNode( in LowerINT_TO_FP()
4056 DAG.getNode(Op.getOpcode(), dl, MVT::f32, SrcVal), in LowerINT_TO_FP()
4128 DAG.getNode(ISD::ANY_EXTEND, SDLoc(Op), getSVEContainerType(ArgVT), in LowerBITCAST()
4145 Op = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op.getOperand(0)); in LowerBITCAST()
4146 Op = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Op); in LowerBITCAST()
4184 return DAG.getNode(ExtOpcode, SDLoc(N), NewVT, N); in addRequiredExtensionForVectorMULL()
4253 SDNode *N0 = N->getOperand(0).getNode(); in isAddSubSExt()
4254 SDNode *N1 = N->getOperand(1).getNode(); in isAddSubSExt()
4264 SDNode *N0 = N->getOperand(0).getNode(); in isAddSubZExt()
4265 SDNode *N1 = N->getOperand(1).getNode(); in isAddSubZExt()
4281 SDValue FPCR_64 = DAG.getNode( in LowerFLT_ROUNDS_()
4285 SDValue FPCR_32 = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, FPCR_64); in LowerFLT_ROUNDS_()
4286 SDValue FltRounds = DAG.getNode(ISD::ADD, dl, MVT::i32, FPCR_32, in LowerFLT_ROUNDS_()
4288 SDValue RMODE = DAG.getNode(ISD::SRL, dl, MVT::i32, FltRounds, in LowerFLT_ROUNDS_()
4290 SDValue AND = DAG.getNode(ISD::AND, dl, MVT::i32, RMODE, in LowerFLT_ROUNDS_()
4311 RMValue = DAG.getNode(ISD::SUB, DL, MVT::i32, RMValue, in LowerSET_ROUNDING()
4313 RMValue = DAG.getNode(ISD::AND, DL, MVT::i32, RMValue, in LowerSET_ROUNDING()
4316 DAG.getNode(ISD::SHL, DL, MVT::i32, RMValue, in LowerSET_ROUNDING()
4318 RMValue = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, RMValue); in LowerSET_ROUNDING()
4324 DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, {MVT::i64, MVT::Other}, Ops); in LowerSET_ROUNDING()
4330 FPCR = DAG.getNode(ISD::AND, DL, MVT::i64, FPCR, in LowerSET_ROUNDING()
4332 FPCR = DAG.getNode(ISD::OR, DL, MVT::i64, FPCR, RMValue); in LowerSET_ROUNDING()
4336 return DAG.getNode(ISD::INTRINSIC_VOID, DL, MVT::Other, Ops2); in LowerSET_ROUNDING()
4352 SDNode *N0 = Op.getOperand(0).getNode(); in LowerMUL()
4353 SDNode *N1 = Op.getOperand(1).getNode(); in LowerMUL()
4400 return DAG.getNode(NewOpc, DL, VT, Op0, Op1); in LowerMUL()
4405 SDValue N00 = skipExtensionForVectorMULL(N0->getOperand(0).getNode(), DAG); in LowerMUL()
4406 SDValue N01 = skipExtensionForVectorMULL(N0->getOperand(1).getNode(), DAG); in LowerMUL()
4408 return DAG.getNode(N0->getOpcode(), DL, VT, in LowerMUL()
4409 DAG.getNode(NewOpc, DL, VT, in LowerMUL()
4410 DAG.getNode(ISD::BITCAST, DL, Op1VT, N00), Op1), in LowerMUL()
4411 DAG.getNode(NewOpc, DL, VT, in LowerMUL()
4412 DAG.getNode(ISD::BITCAST, DL, Op1VT, N01), Op1)); in LowerMUL()
4419 return DAG.getNode(AArch64ISD::PTRUE, DL, VT, in getPTrue()
4442 SDValue Reinterpret = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, VT, Op); in getSVEPredicateBitCast()
4458 Mask = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, VT, Mask); in getSVEPredicateBitCast()
4459 return DAG.getNode(ISD::AND, DL, VT, Reinterpret, Mask); in getSVEPredicateBitCast()
4470 auto Node = cast<MemIntrinsicSDNode>(Op.getNode()); in LowerINTRINSIC_W_CHAIN()
4494 SDValue MRS = DAG.getNode( in LowerINTRINSIC_W_CHAIN()
4498 SDValue And = DAG.getNode(ISD::AND, DL, MVT::i64, MRS, Mask); in LowerINTRINSIC_W_CHAIN()
4512 return DAG.getNode(AArch64ISD::THREAD_POINTER, dl, PtrVT); in LowerINTRINSIC_WO_CHAIN()
4517 SDValue Result = DAG.getNode(ISD::BITCAST, dl, MVT::v1i64, in LowerINTRINSIC_WO_CHAIN()
4519 Result = DAG.getNode(ISD::ABS, dl, MVT::v1i64, Result); in LowerINTRINSIC_WO_CHAIN()
4520 return DAG.getNode(ISD::BITCAST, dl, MVT::i64, Result); in LowerINTRINSIC_WO_CHAIN()
4522 return DAG.getNode(ISD::ABS, dl, Ty, Op.getOperand(1)); in LowerINTRINSIC_WO_CHAIN()
4528 return DAG.getNode(ISD::SMAX, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4531 return DAG.getNode(ISD::UMAX, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4534 return DAG.getNode(ISD::SMIN, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4537 return DAG.getNode(ISD::UMIN, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4541 return DAG.getNode(AArch64ISD::SUNPKHI, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4544 return DAG.getNode(AArch64ISD::SUNPKLO, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4547 return DAG.getNode(AArch64ISD::UUNPKHI, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4550 return DAG.getNode(AArch64ISD::UUNPKLO, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4553 return DAG.getNode(AArch64ISD::CLASTA_N, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4556 return DAG.getNode(AArch64ISD::CLASTB_N, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4559 return DAG.getNode(AArch64ISD::LASTA, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4562 return DAG.getNode(AArch64ISD::LASTB, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4565 return DAG.getNode(ISD::VECTOR_REVERSE, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4568 return DAG.getNode(AArch64ISD::TBL, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4571 return DAG.getNode(AArch64ISD::TRN1, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4574 return DAG.getNode(AArch64ISD::TRN2, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4577 return DAG.getNode(AArch64ISD::UZP1, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4580 return DAG.getNode(AArch64ISD::UZP2, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4583 return DAG.getNode(AArch64ISD::ZIP1, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4586 return DAG.getNode(AArch64ISD::ZIP2, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4589 return DAG.getNode(AArch64ISD::SPLICE, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4595 return DAG.getNode(AArch64ISD::CTLZ_MERGE_PASSTHRU, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4598 return DAG.getNode(AArch64ISD::RDSVL, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4602 SDValue Bytes = DAG.getNode(AArch64ISD::RDSVL, dl, Op.getValueType(), One); in LowerINTRINSIC_WO_CHAIN()
4603 return DAG.getNode(ISD::SRL, dl, Op.getValueType(), Bytes, One); in LowerINTRINSIC_WO_CHAIN()
4606 SDValue Bytes = DAG.getNode(AArch64ISD::RDSVL, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4608 return DAG.getNode(ISD::SRL, dl, Op.getValueType(), Bytes, in LowerINTRINSIC_WO_CHAIN()
4612 SDValue Bytes = DAG.getNode(AArch64ISD::RDSVL, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4614 return DAG.getNode(ISD::SRL, dl, Op.getValueType(), Bytes, in LowerINTRINSIC_WO_CHAIN()
4621 Data = DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Data); in LowerINTRINSIC_WO_CHAIN()
4622 return DAG.getNode(AArch64ISD::CTPOP_MERGE_PASSTHRU, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4632 return DAG.getNode(AArch64ISD::FNEG_MERGE_PASSTHRU, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4635 return DAG.getNode(AArch64ISD::FCEIL_MERGE_PASSTHRU, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4638 return DAG.getNode(AArch64ISD::FFLOOR_MERGE_PASSTHRU, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4641 return DAG.getNode(AArch64ISD::FNEARBYINT_MERGE_PASSTHRU, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4644 return DAG.getNode(AArch64ISD::FRINT_MERGE_PASSTHRU, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4647 return DAG.getNode(AArch64ISD::FROUND_MERGE_PASSTHRU, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4650 return DAG.getNode(AArch64ISD::FROUNDEVEN_MERGE_PASSTHRU, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4653 return DAG.getNode(AArch64ISD::FTRUNC_MERGE_PASSTHRU, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4656 return DAG.getNode(AArch64ISD::UINT_TO_FP_MERGE_PASSTHRU, dl, in LowerINTRINSIC_WO_CHAIN()
4660 return DAG.getNode(AArch64ISD::SINT_TO_FP_MERGE_PASSTHRU, dl, in LowerINTRINSIC_WO_CHAIN()
4664 return DAG.getNode(AArch64ISD::FCVTZU_MERGE_PASSTHRU, dl, in LowerINTRINSIC_WO_CHAIN()
4668 return DAG.getNode(AArch64ISD::FCVTZS_MERGE_PASSTHRU, dl, in LowerINTRINSIC_WO_CHAIN()
4672 return DAG.getNode(AArch64ISD::FSQRT_MERGE_PASSTHRU, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4675 return DAG.getNode(AArch64ISD::FRECPX_MERGE_PASSTHRU, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4678 return DAG.getNode(AArch64ISD::FRECPE, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4681 return DAG.getNode(AArch64ISD::FRECPS, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4684 return DAG.getNode(AArch64ISD::FRSQRTE, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4687 return DAG.getNode(AArch64ISD::FRSQRTS, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4690 return DAG.getNode(AArch64ISD::FABS_MERGE_PASSTHRU, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4693 return DAG.getNode(AArch64ISD::ABS_MERGE_PASSTHRU, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4696 return DAG.getNode(AArch64ISD::NEG_MERGE_PASSTHRU, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4702 Scalar = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Scalar); in LowerINTRINSIC_WO_CHAIN()
4704 return DAG.getNode(AArch64ISD::INSR, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4708 return DAG.getNode(AArch64ISD::BITREVERSE_MERGE_PASSTHRU, dl, in LowerINTRINSIC_WO_CHAIN()
4712 return DAG.getNode(AArch64ISD::BSWAP_MERGE_PASSTHRU, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4715 return DAG.getNode(AArch64ISD::REVH_MERGE_PASSTHRU, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4718 return DAG.getNode(AArch64ISD::REVW_MERGE_PASSTHRU, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4721 return DAG.getNode(AArch64ISD::REVD_MERGE_PASSTHRU, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4724 return DAG.getNode( in LowerINTRINSIC_WO_CHAIN()
4730 return DAG.getNode( in LowerINTRINSIC_WO_CHAIN()
4736 return DAG.getNode( in LowerINTRINSIC_WO_CHAIN()
4742 return DAG.getNode( in LowerINTRINSIC_WO_CHAIN()
4748 return DAG.getNode( in LowerINTRINSIC_WO_CHAIN()
4754 return DAG.getNode( in LowerINTRINSIC_WO_CHAIN()
4792 return DAG.getNode(Opcode, dl, Ty, Op.getOperand(1), Op.getOperand(2), in LowerINTRINSIC_WO_CHAIN()
4807 return DAG.getNode(Opcode, dl, Op.getValueType(), Op.getOperand(1), in LowerINTRINSIC_WO_CHAIN()
4814 return DAG.getNode(Opcode, dl, Op.getValueType(), Op.getOperand(1), in LowerINTRINSIC_WO_CHAIN()
4822 return DAG.getNode(Opcode, dl, Op.getValueType(), Op.getOperand(1)); in LowerINTRINSIC_WO_CHAIN()
4832 return DAG.getNode(Opcode, dl, Op.getValueType(), Op.getOperand(1), in LowerINTRINSIC_WO_CHAIN()
4838 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, Op.getValueType(), ID, in LowerINTRINSIC_WO_CHAIN()
4938 if (!PassThru->isUndef() && !isZerosVector(PassThru.getNode())) { in LowerMGATHER()
4956 Index = DAG.getNode(ISD::SHL, DL, IndexVT, Index, in LowerMGATHER()
4984 Index = DAG.getNode(ExtOpcode, DL, PromotedVT, Index); in LowerMGATHER()
4985 Mask = DAG.getNode(ISD::SIGN_EXTEND, DL, PromotedVT, Mask); in LowerMGATHER()
5008 Result = DAG.getNode(ISD::TRUNCATE, DL, DataVT, Result); in LowerMGATHER()
5010 Result = DAG.getNode(ISD::BITCAST, DL, VT, Result); in LowerMGATHER()
5044 Index = DAG.getNode(ISD::SHL, DL, IndexVT, Index, in LowerMSCATTER()
5062 StoreVal = DAG.getNode(ISD::BITCAST, DL, VT, StoreVal); in LowerMSCATTER()
5074 Index = DAG.getNode(ExtOpcode, DL, PromotedVT, Index); in LowerMSCATTER()
5075 Mask = DAG.getNode(ISD::SIGN_EXTEND, DL, PromotedVT, Mask); in LowerMSCATTER()
5076 StoreVal = DAG.getNode(ISD::ANY_EXTEND, DL, PromotedVT, StoreVal); in LowerMSCATTER()
5114 if (PassThru->isUndef() || isZerosVector(PassThru.getNode())) in LowerMLOAD()
5148 SDValue TruncExt = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i16, in LowerTruncateVectorStore()
5150 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i8, TruncExt); in LowerTruncateVectorStore()
5152 Trunc = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32, Trunc); in LowerTruncateVectorStore()
5153 SDValue ExtractTrunc = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, in LowerTruncateVectorStore()
5204 DAG.getNode(ISD::EXTRACT_SUBVECTOR, Dl, in LowerSTORE()
5208 DAG.getNode(ISD::EXTRACT_SUBVECTOR, Dl, in LowerSTORE()
5227 SDValue Part = DAG.getNode(AArch64ISD::LS64_EXTRACT, Dl, MVT::i64, in LowerSTORE()
5229 SDValue Ptr = DAG.getNode(ISD::ADD, Dl, PtrVT, Base, in LowerSTORE()
5254 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, Value, in LowerStore128()
5256 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, Value, in LowerStore128()
5277 SDValue Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, Base, in LowerLOAD()
5283 Chain = SDValue(Part.getNode(), 1); in LowerLOAD()
5285 SDValue Loaded = DAG.getNode(AArch64ISD::LS64_BUILD, DL, MVT::i64x8, Ops); in LowerLOAD()
5308 SDValue Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f32, Load); in LowerLOAD()
5309 SDValue BC = DAG.getNode(ISD::BITCAST, DL, MVT::v8i8, Vec); in LowerLOAD()
5310 SDValue Ext = DAG.getNode(ExtType, DL, MVT::v8i16, BC); in LowerLOAD()
5311 Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i16, Ext, in LowerLOAD()
5314 Ext = DAG.getNode(ExtType, DL, MVT::v4i32, Ext); in LowerLOAD()
5326 SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), in LowerABS()
5330 DAG.getNode(AArch64ISD::SUBS, DL, DAG.getVTList(VT, MVT::i32), in LowerABS()
5332 return DAG.getNode(AArch64ISD::CSEL, DL, VT, Op.getOperand(0), Neg, in LowerABS()
5346 return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, in LowerBRCOND()
5620 SDValue Ext = DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {MVT::f32, MVT::Other}, in LowerOperation()
5622 return DAG.getNode(Op.getOpcode(), DL, {Op.getValueType(), MVT::Other}, in LowerOperation()
5701 unsigned IID = getIntrinsicID(N1.getNode()); in isReassocProfitable()
5878 ArgValue = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), ArgValue); in LowerFormalArguments()
5885 ArgValue = DAG.getNode(ISD::SRL, DL, RegVT, ArgValue, in LowerFormalArguments()
5966 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, in LowerFormalArguments()
5974 ArgValue = DAG.getNode(ISD::AssertZext, DL, ArgValue.getValueType(), in LowerFormalArguments()
5983 ArgValue = DAG.getNode(AArch64ISD::ASSERT_ZEXT_BOOL, DL, in LowerFormalArguments()
6043 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Copy, Chain); in LowerFormalArguments()
6116 DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getConstant(8, DL, PtrVT)); in saveVarArgRegisters()
6143 FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, in saveVarArgRegisters()
6152 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps); in saveVarArgRegisters()
6194 Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val); in LowerCallResult()
6197 Val = DAG.getNode(ISD::SRL, DL, VA.getLocVT(), Val, in LowerCallResult()
6439 for (SDNode *U : DAG.getEntryNode().getNode()->uses()) in addTokenForArgument()
6453 return DAG.getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains); in addTokenForArgument()
6634 Arg = DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Arg); in LowerCall()
6637 Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg); in LowerCall()
6655 Arg = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Arg); in LowerCall()
6656 Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i8, Arg); in LowerCall()
6659 Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg); in LowerCall()
6663 Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg); in LowerCall()
6664 Arg = DAG.getNode(ISD::SHL, DL, VA.getLocVT(), Arg, in LowerCall()
6674 Arg = DAG.getNode(ISD::FP_EXTEND, DL, VA.getLocVT(), Arg); in LowerCall()
6714 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, in LowerCall()
6745 Bits = DAG.getNode(ISD::OR, DL, Bits.getValueType(), Bits, Arg); in LowerCall()
6784 PtrOff = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, PtrOff); in LowerCall()
6800 DstAddr = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, PtrOff); in LowerCall()
6820 Arg = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Arg); in LowerCall()
6829 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains); in LowerCall()
6849 Callee = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, Callee); in LowerCall()
6859 Callee = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, Callee); in LowerCall()
6915 if (InFlag.getNode()) in LowerCall()
6924 SDValue Ret = DAG.getNode(AArch64ISD::TC_RETURN, DL, NodeTys, Ops); in LowerCall()
6925 DAG.addCallSiteInfo(Ret.getNode(), std::move(CSInfo)); in LowerCall()
6947 Chain = DAG.getNode(CallOpc, DL, NodeTys, Ops); in LowerCall()
6948 DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge); in LowerCall()
6950 DAG.addCallSiteInfo(Chain.getNode(), std::move(CSInfo)); in LowerCall()
7009 Arg = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Arg); in LowerReturn()
7010 Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg); in LowerReturn()
7014 Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg); in LowerReturn()
7023 Arg = DAG.getNode(ISD::SHL, DL, VA.getLocVT(), Arg, in LowerReturn()
7033 Bits = DAG.getNode(ISD::OR, DL, Bits.getValueType(), Bits, Arg); in LowerReturn()
7080 if (Flag.getNode()) in LowerReturn()
7083 return DAG.getNode(AArch64ISD::RET_FLAG, DL, MVT::Other, RetOps); in LowerReturn()
7126 return DAG.getNode(AArch64ISD::LOADgot, DL, Ty, GotAddr); in getGOT()
7137 return DAG.getNode( in getAddrLarge()
7155 SDValue ADRP = DAG.getNode(AArch64ISD::ADRP, DL, Ty, Hi); in getAddr()
7156 return DAG.getNode(AArch64ISD::ADDlow, DL, Ty, ADRP, Lo); in getAddr()
7167 return DAG.getNode(AArch64ISD::ADR, DL, Ty, Sym); in getAddrTiny()
7243 SDValue DescAddr = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, TLVPAddr); in LowerDarwinGlobalTLSAddress()
7274 DAG.getNode(AArch64ISD::CALL, DL, DAG.getVTList(MVT::Other, MVT::Glue), in LowerDarwinGlobalTLSAddress()
7340 return DAG.getNode(ISD::ADD, DL, PtrVT, ThreadBase, TPOff); in LowerELFTLSLocalExec()
7366 return DAG.getNode(ISD::ADD, DL, PtrVT, ThreadBase, TPOff); in LowerELFTLSLocalExec()
7398 DAG.getNode(AArch64ISD::TLSDESC_CALLSEQ, DL, NodeTys, {Chain, SymAddr}); in LowerELFTLSDescCallSeq()
7434 SDValue ThreadBase = DAG.getNode(AArch64ISD::THREAD_POINTER, DL, PtrVT); in LowerELFGlobalTLSAddress()
7440 TPOff = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, TPOff); in LowerELFGlobalTLSAddress()
7488 return DAG.getNode(ISD::ADD, DL, PtrVT, ThreadBase, TPOff); in LowerELFGlobalTLSAddress()
7505 DAG.getNode(ISD::ADD, DL, PtrVT, TEB, DAG.getIntPtrConstant(0x58, DL)); in LowerWindowsGlobalTLSAddress()
7517 SDValue ADRP = DAG.getNode(AArch64ISD::ADRP, DL, PtrVT, TLSIndexHi); in LowerWindowsGlobalTLSAddress()
7519 DAG.getNode(AArch64ISD::ADDlow, DL, PtrVT, ADRP, TLSIndexLo); in LowerWindowsGlobalTLSAddress()
7525 TLSIndex = DAG.getNode(ISD::ZERO_EXTEND, DL, PtrVT, TLSIndex); in LowerWindowsGlobalTLSAddress()
7526 SDValue Slot = DAG.getNode(ISD::SHL, DL, PtrVT, TLSIndex, in LowerWindowsGlobalTLSAddress()
7529 DAG.getNode(ISD::ADD, DL, PtrVT, TLSArray, Slot), in LowerWindowsGlobalTLSAddress()
7546 Addr = DAG.getNode(AArch64ISD::ADDlow, DL, PtrVT, Addr, TGALo); in LowerWindowsGlobalTLSAddress()
7605 if (!RHS.getNode()) { in LowerBR_CC()
7628 return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, in LowerBR_CC()
7650 return DAG.getNode(AArch64ISD::TBZ, dl, MVT::Other, Chain, Test, in LowerBR_CC()
7655 return DAG.getNode(AArch64ISD::CBZ, dl, MVT::Other, Chain, LHS, Dest); in LowerBR_CC()
7666 return DAG.getNode(AArch64ISD::TBNZ, dl, MVT::Other, Chain, Test, in LowerBR_CC()
7671 return DAG.getNode(AArch64ISD::CBNZ, dl, MVT::Other, Chain, LHS, Dest); in LowerBR_CC()
7678 return DAG.getNode(AArch64ISD::TBNZ, dl, MVT::Other, Chain, LHS, in LowerBR_CC()
7689 return DAG.getNode(AArch64ISD::TBZ, dl, MVT::Other, Chain, LHS, in LowerBR_CC()
7695 return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, in LowerBR_CC()
7709 DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CC1Val, Cmp); in LowerBR_CC()
7712 return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, BR1, Dest, CC2Val, in LowerBR_CC()
7733 In2 = DAG.getNode(ISD::FP_EXTEND, DL, VT, In2); in LowerFCOPYSIGN()
7735 In2 = DAG.getNode(ISD::FP_ROUND, DL, VT, In2, DAG.getIntPtrConstant(0, DL)); in LowerFCOPYSIGN()
7788 SignMaskV = DAG.getNode(ISD::BITCAST, DL, MVT::v2f64, SignMaskV); in LowerFCOPYSIGN()
7789 SignMaskV = DAG.getNode(ISD::FNEG, DL, MVT::v2f64, SignMaskV); in LowerFCOPYSIGN()
7790 SignMaskV = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, SignMaskV); in LowerFCOPYSIGN()
7794 DAG.getNode(AArch64ISD::BSP, DL, VecVT, SignMaskV, VecVal1, VecVal2); in LowerFCOPYSIGN()
7830 Val = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Val); in LowerCTPOP_PARITY()
7831 Val = DAG.getNode(ISD::BITCAST, DL, MVT::v8i8, Val); in LowerCTPOP_PARITY()
7833 SDValue CtPop = DAG.getNode(ISD::CTPOP, DL, MVT::v8i8, Val); in LowerCTPOP_PARITY()
7834 SDValue UaddLV = DAG.getNode( in LowerCTPOP_PARITY()
7839 UaddLV = DAG.getNode(ISD::AND, DL, MVT::i32, UaddLV, in LowerCTPOP_PARITY()
7843 UaddLV = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, UaddLV); in LowerCTPOP_PARITY()
7846 Val = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Val); in LowerCTPOP_PARITY()
7848 SDValue CtPop = DAG.getNode(ISD::CTPOP, DL, MVT::v16i8, Val); in LowerCTPOP_PARITY()
7849 SDValue UaddLV = DAG.getNode( in LowerCTPOP_PARITY()
7854 UaddLV = DAG.getNode(ISD::AND, DL, MVT::i32, UaddLV, in LowerCTPOP_PARITY()
7857 return DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i128, UaddLV); in LowerCTPOP_PARITY()
7871 Val = DAG.getNode(ISD::CTPOP, DL, VT8Bit, Val); in LowerCTPOP_PARITY()
7880 Val = DAG.getNode( in LowerCTPOP_PARITY()
7895 SDValue RBIT = DAG.getNode(ISD::BITREVERSE, DL, VT, Op.getOperand(0)); in LowerCTTZ()
7896 return DAG.getNode(ISD::CTLZ, DL, VT, RBIT); in LowerCTTZ()
7965 REVB = DAG.getNode(AArch64ISD::REV32, DL, VST, Op.getOperand(0)); in LowerBitreverse()
7972 REVB = DAG.getNode(AArch64ISD::REV32, DL, VST, Op.getOperand(0)); in LowerBitreverse()
7979 REVB = DAG.getNode(AArch64ISD::REV64, DL, VST, Op.getOperand(0)); in LowerBitreverse()
7986 REVB = DAG.getNode(AArch64ISD::REV64, DL, VST, Op.getOperand(0)); in LowerBitreverse()
7992 return DAG.getNode(AArch64ISD::NVCAST, DL, VT, in LowerBitreverse()
7993 DAG.getNode(ISD::BITREVERSE, DL, VST, REVB)); in LowerBitreverse()
8024 if (!RHS.getNode()) { in LowerSETCC()
8039 SDValue Res = DAG.getNode(AArch64ISD::CSEL, dl, VT, FVal, TVal, CCVal, Cmp); in LowerSETCC()
8066 Res = DAG.getNode(AArch64ISD::CSEL, dl, VT, FVal, TVal, CC1Val, Cmp); in LowerSETCC()
8076 DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp); in LowerSETCC()
8079 Res = DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, CS1, CC2Val, Cmp); in LowerSETCC()
8095 if (!RHS.getNode()) { in LowerSELECT_CC()
8103 LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, LHS); in LowerSELECT_CC()
8104 RHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, RHS); in LowerSELECT_CC()
8123 DAG.getNode(ISD::SRA, dl, VT, LHS, in LowerSELECT_CC()
8125 return DAG.getNode(ISD::OR, dl, VT, Shift, DAG.getConstant(1, dl, VT)); in LowerSELECT_CC()
8238 return DAG.getNode(Opcode, dl, VT, TVal, FVal, CCVal, Cmp); in LowerSELECT_CC()
8273 SDValue CS1 = DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp); in LowerSELECT_CC()
8279 return DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, CS1, CC2Val, Cmp); in LowerSELECT_CC()
8310 Pred = DAG.getNode(ISD::VECTOR_REVERSE, DL, PredVT, Pred); in LowerVECTOR_SPLICE()
8313 return DAG.getNode(AArch64ISD::SPLICE, DL, Ty, Pred, Op.getOperand(0), in LowerVECTOR_SPLICE()
8346 SDValue TruncCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, CCVal); in LowerSELECT()
8348 SDValue SplatPred = DAG.getNode(ISD::SPLAT_VECTOR, DL, PredVT, TruncCC); in LowerSELECT()
8349 return DAG.getNode(ISD::VSELECT, DL, Ty, SplatPred, TVal, FVal); in LowerSELECT()
8359 SDValue SplatPred = DAG.getNode(ISD::SPLAT_VECTOR, DL, PredVT, SplatVal); in LowerSELECT()
8360 return DAG.getNode(ISD::VSELECT, DL, Ty, SplatPred, TVal, FVal); in LowerSELECT()
8375 return DAG.getNode(AArch64ISD::CSEL, DL, Op.getValueType(), TVal, FVal, in LowerSELECT()
8416 int JTI = cast<JumpTableSDNode>(JT.getNode())->getIndex(); in LowerBR_JT()
8424 return DAG.getNode(ISD::BRIND, DL, MVT::Other, Op.getOperand(0), in LowerBR_JT()
8515 GRTopAddr = DAG.getNode(ISD::ADD, DL, PtrVT, VAList, in LowerAAPCS_VASTART()
8519 GRTop = DAG.getNode(ISD::ADD, DL, PtrVT, GRTop, in LowerAAPCS_VASTART()
8533 VRTopAddr = DAG.getNode(ISD::ADD, DL, PtrVT, VAList, in LowerAAPCS_VASTART()
8537 VRTop = DAG.getNode(ISD::ADD, DL, PtrVT, VRTop, in LowerAAPCS_VASTART()
8548 SDValue GROffsAddr = DAG.getNode(ISD::ADD, DL, PtrVT, VAList, in LowerAAPCS_VASTART()
8556 SDValue VROffsAddr = DAG.getNode(ISD::ADD, DL, PtrVT, VAList, in LowerAAPCS_VASTART()
8562 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps); in LowerAAPCS_VASTART()
8619 VAList = DAG.getNode(ISD::ADD, DL, PtrVT, VAList, in LowerVAARG()
8621 VAList = DAG.getNode(ISD::AND, DL, PtrVT, VAList, in LowerVAARG()
8641 SDValue VANext = DAG.getNode(ISD::ADD, DL, PtrVT, VAList, in LowerVAARG()
8655 SDValue NarrowFP = DAG.getNode(ISD::FP_ROUND, DL, VT, WideFP.getValue(0), in LowerVAARG()
8680 FrameAddr = DAG.getNode(ISD::AssertZext, DL, MVT::i64, FrameAddr, in LowerFRAMEADDR()
8727 return DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset); in LowerADDROFRETURNADDR()
8745 DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset), MachinePointerInfo()); in LowerRETURNADDR()
8774 expandShiftParts(Op.getNode(), Lo, Hi, DAG); in LowerShiftParts()
8844 return DAG.getNode(Opcode, SDLoc(Operand), VT, Operand); in getEstimate()
8884 SDValue Step = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Estimate, in getSqrtEstimate()
8886 Step = DAG.getNode(AArch64ISD::FRSQRTS, DL, VT, Operand, Step, Flags); in getSqrtEstimate()
8887 Estimate = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Step, Flags); in getSqrtEstimate()
8890 Estimate = DAG.getNode(ISD::FMUL, DL, VT, Operand, Estimate, Flags); in getSqrtEstimate()
8914 SDValue Step = DAG.getNode(AArch64ISD::FRECPS, DL, VT, Operand, in getRecipEstimate()
8916 Estimate = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Step, Flags); in getRecipEstimate()
9308 if (Result.getNode()) { in LowerAsmOperandForConstraint()
9329 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideTy, DAG.getUNDEF(WideTy), in WidenVector()
9459 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i8, Src, in ReconstructShuffle()
9473 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, in ReconstructShuffle()
9521 DAG.getNode(ISD::CONCAT_VECTORS, dl, DestVT, Src.ShuffleVec, in ReconstructShuffle()
9541 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec, in ReconstructShuffle()
9547 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec, in ReconstructShuffle()
9552 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec, in ReconstructShuffle()
9555 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec, in ReconstructShuffle()
9566 Src.ShuffleVec = DAG.getNode(AArch64ISD::EXT, dl, DestVT, VEXTSrc1, in ReconstructShuffle()
9581 Src.ShuffleVec = DAG.getNode(ISD::BITCAST, dl, ShuffleVT, Src.ShuffleVec); in ReconstructShuffle()
9633 SDValue V = DAG.getNode(ISD::BITCAST, dl, VT, Shuffle); in ReconstructShuffle()
9712 Trunc[I] = DAG.getNode(ISD::TRUNCATE, DL, MVT::v4i16, Trunc[I]); in ReconstructTruncateFromBuildVector()
9714 DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i16, Trunc[0], Trunc[1]); in ReconstructTruncateFromBuildVector()
9716 DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i16, Trunc[2], Trunc[3]); in ReconstructTruncateFromBuildVector()
9717 SDValue Trunc0 = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i8, Concat0); in ReconstructTruncateFromBuildVector()
9718 SDValue Trunc1 = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i8, Concat1); in ReconstructTruncateFromBuildVector()
9719 return DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, Trunc0, Trunc1); in ReconstructTruncateFromBuildVector()
10040 V0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, CastVT, V0, in tryFormConcatFromShuffle()
10044 V1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, CastVT, V1, in tryFormConcatFromShuffle()
10047 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, V0, V1); in tryFormConcatFromShuffle()
10140 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, in GeneratePerfectShuffle()
10144 DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Input.getValueType(), OpLHS, in GeneratePerfectShuffle()
10163 return DAG.getNode(AArch64ISD::REV64, dl, VT, OpLHS); in GeneratePerfectShuffle()
10168 return DAG.getNode(AArch64ISD::REV32, dl, VT, OpLHS); in GeneratePerfectShuffle()
10171 return DAG.getNode(AArch64ISD::REV16, dl, VT, OpLHS); in GeneratePerfectShuffle()
10192 return DAG.getNode(Opcode, dl, VT, OpLHS, Lane); in GeneratePerfectShuffle()
10198 return DAG.getNode(AArch64ISD::EXT, dl, VT, OpLHS, OpRHS, in GeneratePerfectShuffle()
10202 return DAG.getNode(AArch64ISD::UZP1, dl, DAG.getVTList(VT, VT), OpLHS, in GeneratePerfectShuffle()
10205 return DAG.getNode(AArch64ISD::UZP2, dl, DAG.getVTList(VT, VT), OpLHS, in GeneratePerfectShuffle()
10208 return DAG.getNode(AArch64ISD::ZIP1, dl, DAG.getVTList(VT, VT), OpLHS, in GeneratePerfectShuffle()
10211 return DAG.getNode(AArch64ISD::ZIP2, dl, DAG.getVTList(VT, VT), OpLHS, in GeneratePerfectShuffle()
10214 return DAG.getNode(AArch64ISD::TRN1, dl, DAG.getVTList(VT, VT), OpLHS, in GeneratePerfectShuffle()
10217 return DAG.getNode(AArch64ISD::TRN2, dl, DAG.getVTList(VT, VT), OpLHS, in GeneratePerfectShuffle()
10233 if (V1.isUndef() || isZerosVector(V1.getNode())) { in GenerateTBL()
10241 bool IsUndefOrZero = V2.isUndef() || isZerosVector(V2.getNode()); in GenerateTBL()
10261 SDValue V1Cst = DAG.getNode(ISD::BITCAST, DL, IndexVT, V1); in GenerateTBL()
10262 SDValue V2Cst = DAG.getNode(ISD::BITCAST, DL, IndexVT, V2); in GenerateTBL()
10267 V1Cst = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, V1Cst, V1Cst); in GenerateTBL()
10268 Shuffle = DAG.getNode( in GenerateTBL()
10275 V1Cst = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, V1Cst, V2Cst); in GenerateTBL()
10276 Shuffle = DAG.getNode( in GenerateTBL()
10288 Shuffle = DAG.getNode( in GenerateTBL()
10295 return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Shuffle); in GenerateTBL()
10367 return DAG.getNode(Opcode, dl, VT, V, DAG.getConstant(Lane, dl, MVT::i64)); in constructDup()
10455 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode()); in LowerVECTOR_SHUFFLE()
10480 return DAG.getNode(AArch64ISD::DUP, dl, V1.getValueType(), in LowerVECTOR_SHUFFLE()
10486 return DAG.getNode(AArch64ISD::DUP, dl, VT, V1.getOperand(Lane)); in LowerVECTOR_SHUFFLE()
10513 return DAG.getNode(AArch64ISD::REV64, dl, V1.getValueType(), V1, V2); in LowerVECTOR_SHUFFLE()
10515 return DAG.getNode(AArch64ISD::REV32, dl, V1.getValueType(), V1, V2); in LowerVECTOR_SHUFFLE()
10517 return DAG.getNode(AArch64ISD::REV16, dl, V1.getValueType(), V1, V2); in LowerVECTOR_SHUFFLE()
10522 SDValue Rev = DAG.getNode(AArch64ISD::REV64, dl, VT, V1); in LowerVECTOR_SHUFFLE()
10523 return DAG.getNode(AArch64ISD::EXT, dl, VT, Rev, Rev, in LowerVECTOR_SHUFFLE()
10533 return DAG.getNode(AArch64ISD::EXT, dl, V1.getValueType(), V1, V2, in LowerVECTOR_SHUFFLE()
10537 return DAG.getNode(AArch64ISD::EXT, dl, V1.getValueType(), V1, V1, in LowerVECTOR_SHUFFLE()
10544 return DAG.getNode(Opc, dl, V1.getValueType(), V1, V2); in LowerVECTOR_SHUFFLE()
10548 return DAG.getNode(Opc, dl, V1.getValueType(), V1, V2); in LowerVECTOR_SHUFFLE()
10552 return DAG.getNode(Opc, dl, V1.getValueType(), V1, V2); in LowerVECTOR_SHUFFLE()
10557 return DAG.getNode(Opc, dl, V1.getValueType(), V1, V1); in LowerVECTOR_SHUFFLE()
10561 return DAG.getNode(Opc, dl, V1.getValueType(), V1, V1); in LowerVECTOR_SHUFFLE()
10565 return DAG.getNode(Opc, dl, V1.getValueType(), V1, V1); in LowerVECTOR_SHUFFLE()
10591 return DAG.getNode( in LowerVECTOR_SHUFFLE()
10593 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ScalarVT, SrcVec, SrcLaneV), in LowerVECTOR_SHUFFLE()
10641 SplatVal = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, SplatVal, in LowerSPLAT_VECTOR()
10647 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::nxv1i1, in LowerSPLAT_VECTOR()
10648 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::nxv2i1, ID, in LowerSPLAT_VECTOR()
10651 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, ID, Zero, SplatVal); in LowerSPLAT_VECTOR()
10673 return DAG.getNode(AArch64ISD::DUPLANE128, DL, VT, Op.getOperand(1), CI); in LowerDUPQLane()
10676 SDValue V = DAG.getNode(ISD::BITCAST, DL, MVT::nxv2i64, Op.getOperand(1)); in LowerDUPQLane()
10683 SDValue SplatOne = DAG.getNode(ISD::SPLAT_VECTOR, DL, MVT::nxv2i64, One); in LowerDUPQLane()
10687 SV = DAG.getNode(ISD::AND, DL, MVT::nxv2i64, SV, SplatOne); in LowerDUPQLane()
10690 SDValue Idx64 = DAG.getNode(ISD::ADD, DL, MVT::i64, Idx128, Idx128); in LowerDUPQLane()
10691 SDValue SplatIdx64 = DAG.getNode(ISD::SPLAT_VECTOR, DL, MVT::nxv2i64, Idx64); in LowerDUPQLane()
10692 SDValue ShuffleMask = DAG.getNode(ISD::ADD, DL, MVT::nxv2i64, SV, SplatIdx64); in LowerDUPQLane()
10695 SDValue TBL = DAG.getNode(AArch64ISD::TBL, DL, MVT::nxv2i64, V, ShuffleMask); in LowerDUPQLane()
10696 return DAG.getNode(ISD::BITCAST, DL, VT, TBL); in LowerDUPQLane()
10734 SDValue Mov = DAG.getNode(NewOp, dl, MovTy, in tryAdvSIMDModImm64()
10736 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in tryAdvSIMDModImm64()
10776 Mov = DAG.getNode(NewOp, dl, MovTy, *LHS, in tryAdvSIMDModImm32()
10780 Mov = DAG.getNode(NewOp, dl, MovTy, in tryAdvSIMDModImm32()
10784 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in tryAdvSIMDModImm32()
10816 Mov = DAG.getNode(NewOp, dl, MovTy, *LHS, in tryAdvSIMDModImm16()
10820 Mov = DAG.getNode(NewOp, dl, MovTy, in tryAdvSIMDModImm16()
10824 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in tryAdvSIMDModImm16()
10852 SDValue Mov = DAG.getNode(NewOp, dl, MovTy, in tryAdvSIMDModImm321s()
10855 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in tryAdvSIMDModImm321s()
10874 SDValue Mov = DAG.getNode(NewOp, dl, MovTy, in tryAdvSIMDModImm8()
10876 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in tryAdvSIMDModImm8()
10905 SDValue Mov = DAG.getNode(NewOp, dl, MovTy, in tryAdvSIMDModImmFP()
10907 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov); in tryAdvSIMDModImmFP()
11011 SDValue ResultSLI = DAG.getNode(Inst, DL, VT, X, Y, Shift.getOperand(1)); in tryLowerToSLI()
11028 if (SDValue Res = tryLowerToSLI(Op.getNode(), DAG)) in LowerVectorOR()
11035 dyn_cast<BuildVectorSDNode>(Op.getOperand(1).getNode()); in LowerVectorOR()
11039 BVN = dyn_cast<BuildVectorSDNode>(Op.getOperand(0).getNode()); in LowerVectorOR()
11088 } else if (Lane.getNode()->isUndef()) { in NormalizeBuildVector()
11104 BuildVectorSDNode *BVN = cast<BuildVectorSDNode>(Op.getNode()); in ConstantBuildVector()
11150 SDValue Seq = DAG.getNode(ISD::ADD, DL, ContainerVT, Start, Steps); in LowerBUILD_VECTOR()
11165 BuildVectorSDNode *BVN = cast<BuildVectorSDNode>(Op.getNode()); in LowerBUILD_VECTOR()
11218 if (!ConstantValue.getNode()) in LowerBUILD_VECTOR()
11224 if (!Value.getNode()) in LowerBUILD_VECTOR()
11232 if (!Value.getNode()) { in LowerBUILD_VECTOR()
11244 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value); in LowerBUILD_VECTOR()
11255 const SDNode *N = V.getNode(); in LowerBUILD_VECTOR()
11262 Vector = N0.getNode(); in LowerBUILD_VECTOR()
11268 } else if (Vector != N0.getNode()) { in LowerBUILD_VECTOR()
11293 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, SDValue(Vector, 0), in LowerBUILD_VECTOR()
11296 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, SDValue(Vector, 0), in LowerBUILD_VECTOR()
11300 return DAG.getNode(AArch64ISD::UZP1, dl, DAG.getVTList(VT, VT), LHS, in LowerBUILD_VECTOR()
11303 return DAG.getNode(AArch64ISD::UZP2, dl, DAG.getVTList(VT, VT), LHS, in LowerBUILD_VECTOR()
11316 return DAG.getNode(AArch64ISD::DUP, dl, VT, Value); in LowerBUILD_VECTOR()
11331 return DAG.getNode(Opcode, dl, VT, Value, Lane); in LowerBUILD_VECTOR()
11344 Ops.push_back(DAG.getNode(ISD::BITCAST, dl, NewType, Op.getOperand(i))); in LowerBUILD_VECTOR()
11350 if (Val.getNode()) in LowerBUILD_VECTOR()
11351 return DAG.getNode(ISD::BITCAST, dl, VT, Val); in LowerBUILD_VECTOR()
11374 Val = DAG.getNode(AArch64ISD::DUP, dl, VT, ConstantValue); in LowerBUILD_VECTOR()
11375 DAG.ReplaceAllUsesWith(Vec.getNode(), &Val); in LowerBUILD_VECTOR()
11385 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Val, V, LaneIdx); in LowerBUILD_VECTOR()
11418 DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, NewVector, in LowerBUILD_VECTOR()
11451 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op0); in LowerBUILD_VECTOR()
11461 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Vec, V, LaneIdx); in LowerBUILD_VECTOR()
11498 DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Op), PairVT, V1, V2); in LowerCONCAT_VECTORS()
11529 DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VectorVT, ExtendedVector, in LowerINSERT_VECTOR_ELT()
11555 SDValue Node = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, WideTy, WideVec, in LowerINSERT_VECTOR_ELT()
11573 DAG.getNode(ISD::ANY_EXTEND, DL, VectorVT, Op.getOperand(0)); in LowerEXTRACT_VECTOR_ELT()
11575 SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractTy, in LowerEXTRACT_VECTOR_ELT()
11610 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtrTy, WideVec, in LowerEXTRACT_VECTOR_ELT()
11652 SDValue Splice = DAG.getNode(ISD::VECTOR_SPLICE, DL, ContainerVT, NewInVec, in LowerEXTRACT_SUBVECTOR()
11683 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, Vec0, in LowerINSERT_SUBVECTOR()
11685 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, Vec0, in LowerINSERT_SUBVECTOR()
11688 SDValue NewLo = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, HalfVT, Lo, Vec1, in LowerINSERT_SUBVECTOR()
11690 return DAG.getNode(AArch64ISD::UZP1, DL, VT, NewLo, Hi); in LowerINSERT_SUBVECTOR()
11693 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, HalfVT, Hi, Vec1, in LowerINSERT_SUBVECTOR()
11695 return DAG.getNode(AArch64ISD::UZP1, DL, VT, Lo, NewHi); in LowerINSERT_SUBVECTOR()
11715 Vec1 = DAG.getNode(ISD::ANY_EXTEND, DL, WideVT, Vec1); in LowerINSERT_SUBVECTOR()
11723 SDValue HiVec0 = DAG.getNode(AArch64ISD::UUNPKHI, DL, WideVT, Vec0); in LowerINSERT_SUBVECTOR()
11724 Narrow = DAG.getNode(AArch64ISD::UZP1, DL, NarrowVT, Vec1, HiVec0); in LowerINSERT_SUBVECTOR()
11728 SDValue LoVec0 = DAG.getNode(AArch64ISD::UUNPKLO, DL, WideVT, Vec0); in LowerINSERT_SUBVECTOR()
11729 Narrow = DAG.getNode(AArch64ISD::UZP1, DL, NarrowVT, LoVec0, Vec1); in LowerINSERT_SUBVECTOR()
11745 return DAG.getNode(ISD::VSELECT, DL, VT, PTrue, ScalableVec1, Vec0); in LowerINSERT_SUBVECTOR()
11799 DAG.getNode(AArch64ISD::SRAD_MERGE_OP1, dl, VT, Pg, Op->getOperand(0), in LowerDIV()
11802 Res = DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(0, dl, VT), Res); in LowerDIV()
11822 SDValue Op0Lo = DAG.getNode(UnpkLo, dl, WidenedVT, Op.getOperand(0)); in LowerDIV()
11823 SDValue Op1Lo = DAG.getNode(UnpkLo, dl, WidenedVT, Op.getOperand(1)); in LowerDIV()
11824 SDValue Op0Hi = DAG.getNode(UnpkHi, dl, WidenedVT, Op.getOperand(0)); in LowerDIV()
11825 SDValue Op1Hi = DAG.getNode(UnpkHi, dl, WidenedVT, Op.getOperand(1)); in LowerDIV()
11826 SDValue ResultLo = DAG.getNode(Op.getOpcode(), dl, WidenedVT, Op0Lo, Op1Lo); in LowerDIV()
11827 SDValue ResultHi = DAG.getNode(Op.getOpcode(), dl, WidenedVT, Op0Hi, Op1Hi); in LowerDIV()
11828 return DAG.getNode(AArch64ISD::UZP1, dl, VT, ResultLo, ResultHi); in LowerDIV()
11873 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Op.getNode()); in getVShiftImm()
11918 SDValue And = DAG.getNode(ISD::AND, dl, OpVT, Op.getOperand(0), One); in LowerTRUNCATE()
11947 return DAG.getNode(AArch64ISD::VSHL, DL, VT, Op.getOperand(0), in LowerVectorSRA_SRL_SHL()
11949 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, in LowerVectorSRA_SRL_SHL()
11965 return DAG.getNode(Opc, DL, VT, Op.getOperand(0), in LowerVectorSRA_SRL_SHL()
11975 SDValue NegShift = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), in LowerVectorSRA_SRL_SHL()
11978 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, in LowerVectorSRA_SRL_SHL()
11994 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(RHS.getNode()); in EmitVectorComparison()
12007 Fcmeq = DAG.getNode(AArch64ISD::FCMEQz, dl, VT, LHS); in EmitVectorComparison()
12009 Fcmeq = DAG.getNode(AArch64ISD::FCMEQ, dl, VT, LHS, RHS); in EmitVectorComparison()
12014 return DAG.getNode(AArch64ISD::FCMEQz, dl, VT, LHS); in EmitVectorComparison()
12015 return DAG.getNode(AArch64ISD::FCMEQ, dl, VT, LHS, RHS); in EmitVectorComparison()
12018 return DAG.getNode(AArch64ISD::FCMGEz, dl, VT, LHS); in EmitVectorComparison()
12019 return DAG.getNode(AArch64ISD::FCMGE, dl, VT, LHS, RHS); in EmitVectorComparison()
12022 return DAG.getNode(AArch64ISD::FCMGTz, dl, VT, LHS); in EmitVectorComparison()
12023 return DAG.getNode(AArch64ISD::FCMGT, dl, VT, LHS, RHS); in EmitVectorComparison()
12031 return DAG.getNode(AArch64ISD::FCMLEz, dl, VT, LHS); in EmitVectorComparison()
12032 return DAG.getNode(AArch64ISD::FCMGE, dl, VT, RHS, LHS); in EmitVectorComparison()
12040 return DAG.getNode(AArch64ISD::FCMLTz, dl, VT, LHS); in EmitVectorComparison()
12041 return DAG.getNode(AArch64ISD::FCMGT, dl, VT, RHS, LHS); in EmitVectorComparison()
12051 Cmeq = DAG.getNode(AArch64ISD::CMEQz, dl, VT, LHS); in EmitVectorComparison()
12053 Cmeq = DAG.getNode(AArch64ISD::CMEQ, dl, VT, LHS, RHS); in EmitVectorComparison()
12058 return DAG.getNode(AArch64ISD::CMEQz, dl, VT, LHS); in EmitVectorComparison()
12059 return DAG.getNode(AArch64ISD::CMEQ, dl, VT, LHS, RHS); in EmitVectorComparison()
12062 return DAG.getNode(AArch64ISD::CMGEz, dl, VT, LHS); in EmitVectorComparison()
12063 return DAG.getNode(AArch64ISD::CMGE, dl, VT, LHS, RHS); in EmitVectorComparison()
12066 return DAG.getNode(AArch64ISD::CMGTz, dl, VT, LHS); in EmitVectorComparison()
12067 return DAG.getNode(AArch64ISD::CMGT, dl, VT, LHS, RHS); in EmitVectorComparison()
12070 return DAG.getNode(AArch64ISD::CMLEz, dl, VT, LHS); in EmitVectorComparison()
12071 return DAG.getNode(AArch64ISD::CMGE, dl, VT, RHS, LHS); in EmitVectorComparison()
12073 return DAG.getNode(AArch64ISD::CMHS, dl, VT, RHS, LHS); in EmitVectorComparison()
12075 return DAG.getNode(AArch64ISD::CMHI, dl, VT, RHS, LHS); in EmitVectorComparison()
12078 return DAG.getNode(AArch64ISD::CMLTz, dl, VT, LHS); in EmitVectorComparison()
12079 return DAG.getNode(AArch64ISD::CMGT, dl, VT, RHS, LHS); in EmitVectorComparison()
12081 return DAG.getNode(AArch64ISD::CMHI, dl, VT, LHS, RHS); in EmitVectorComparison()
12083 return DAG.getNode(AArch64ISD::CMHS, dl, VT, LHS, RHS); in EmitVectorComparison()
12115 LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::v4f32, LHS); in LowerVSETCC()
12116 RHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::v4f32, RHS); in LowerVSETCC()
12136 if (!Cmp.getNode()) in LowerVSETCC()
12142 if (!Cmp2.getNode()) in LowerVSETCC()
12145 Cmp = DAG.getNode(ISD::OR, dl, CmpVT, Cmp, Cmp2); in LowerVSETCC()
12159 auto Rdx = DAG.getNode(Op, DL, VecOp.getSimpleValueType(), VecOp); in getReductionSDNode()
12160 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarOp.getValueType(), Rdx, in getReductionSDNode()
12225 return DAG.getNode( in LowerVECREDUCE()
12231 return DAG.getNode( in LowerVECREDUCE()
12251 AtomicSDNode *AN = cast<AtomicSDNode>(Op.getNode()); in LowerATOMIC_LOAD_SUB()
12252 RHS = DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(0, dl, VT), RHS); in LowerATOMIC_LOAD_SUB()
12268 AtomicSDNode *AN = cast<AtomicSDNode>(Op.getNode()); in LowerATOMIC_LOAD_AND()
12269 RHS = DAG.getNode(ISD::XOR, dl, VT, DAG.getConstant(-1ULL, dl, VT), RHS); in LowerATOMIC_LOAD_AND()
12286 Size = DAG.getNode(ISD::SRL, dl, MVT::i64, Size, in LowerWindowsDYNAMIC_STACKALLOC()
12290 DAG.getNode(AArch64ISD::CALL, dl, DAG.getVTList(MVT::Other, MVT::Glue), in LowerWindowsDYNAMIC_STACKALLOC()
12298 Size = DAG.getNode(ISD::SHL, dl, MVT::i64, Size, in LowerWindowsDYNAMIC_STACKALLOC()
12310 SDNode *Node = Op.getNode(); in LowerDYNAMIC_STACKALLOC()
12321 SP = DAG.getNode(ISD::SUB, dl, MVT::i64, SP, Size); in LowerDYNAMIC_STACKALLOC()
12323 SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0), in LowerDYNAMIC_STACKALLOC()
12336 SP = DAG.getNode(ISD::SUB, dl, MVT::i64, SP, Size); in LowerDYNAMIC_STACKALLOC()
12338 SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0), in LowerDYNAMIC_STACKALLOC()
13406 SDValue PseudoLoad = DAG.getNode(Opcode, DL, NodeTys, LoadOps); in LowerSVEStructLoad()
13409 PseudoLoadOps.push_back(SDValue(PseudoLoad.getNode(), I)); in LowerSVEStructLoad()
13410 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, PseudoLoadOps); in LowerSVEStructLoad()
13765 !ISD::isBuildVectorAllOnes(Ones.getNode())) in foldVectorXorShiftIntoCmp()
13774 return DAG.getNode(AArch64ISD::CMGEz, SDLoc(N), VT, Shift.getOperand(0)); in foldVectorXorShiftIntoCmp()
13839 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i8, EXT0->getOperand(0), in performVecReduceAddCombineWithUADDLP()
13842 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i8, EXT1->getOperand(0), in performVecReduceAddCombineWithUADDLP()
13844 SDValue UABDHigh8 = DAG.getNode(IsZExt ? ISD::ABDU : ISD::ABDS, DL, MVT::v8i8, in performVecReduceAddCombineWithUADDLP()
13846 SDValue UABDL = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v8i16, UABDHigh8); in performVecReduceAddCombineWithUADDLP()
13850 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i8, EXT0->getOperand(0), in performVecReduceAddCombineWithUADDLP()
13853 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i8, EXT1->getOperand(0), in performVecReduceAddCombineWithUADDLP()
13855 SDValue UABDLo8 = DAG.getNode(IsZExt ? ISD::ABDU : ISD::ABDS, DL, MVT::v8i8, in performVecReduceAddCombineWithUADDLP()
13857 SDValue ZExtUABD = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v8i16, UABDLo8); in performVecReduceAddCombineWithUADDLP()
13858 SDValue UABAL = DAG.getNode(ISD::ADD, DL, MVT::v8i16, UABDL, ZExtUABD); in performVecReduceAddCombineWithUADDLP()
13861 SDValue UADDLP = DAG.getNode(AArch64ISD::UADDLP, DL, MVT::v4i32, UABAL); in performVecReduceAddCombineWithUADDLP()
13864 return DAG.getNode(ISD::VECREDUCE_ADD, DL, MVT::i32, UADDLP); in performVecReduceAddCombineWithUADDLP()
13910 SDValue Dot = DAG.getNode(DotOpcode, DL, Zeros.getValueType(), Zeros, in performVecReduceAddCombine()
13912 return DAG.getNode(ISD::VECREDUCE_ADD, DL, N->getValueType(0), Dot); in performVecReduceAddCombine()
13950 return DAG.getNode(Opcode, SDLoc(A), VT, Ext0.getOperand(0)); in performUADDVCombine()
13955 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), R); in performUADDVCombine()
13958 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), in performUADDVCombine()
13959 DAG.getNode(ISD::ADD, SDLoc(A), A.getValueType(), R, in performUADDVCombine()
13962 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), in performUADDVCombine()
13963 DAG.getNode(ISD::ADD, SDLoc(A), A.getValueType(), R, in performUADDVCombine()
14008 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne); in BuildSDIVPow2()
14009 SDValue CSel = DAG.getNode(AArch64ISD::CSEL, DL, VT, Add, N0, CCVal, Cmp); in BuildSDIVPow2()
14011 Created.push_back(Cmp.getNode()); in BuildSDIVPow2()
14012 Created.push_back(Add.getNode()); in BuildSDIVPow2()
14013 Created.push_back(CSel.getNode()); in BuildSDIVPow2()
14017 DAG.getNode(ISD::SRA, DL, VT, CSel, DAG.getConstant(Lg2, DL, MVT::i64)); in BuildSDIVPow2()
14024 Created.push_back(SRA.getNode()); in BuildSDIVPow2()
14025 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), SRA); in BuildSDIVPow2()
14059 SDValue And = DAG.getNode(ISD::AND, DL, VT, N0, Pow2MinusOne); in BuildSREMPow2()
14060 CSNeg = DAG.getNode(AArch64ISD::CSNEG, DL, VT, And, And, CCVal, Cmp); in BuildSREMPow2()
14062 Created.push_back(Cmp.getNode()); in BuildSREMPow2()
14063 Created.push_back(And.getNode()); in BuildSREMPow2()
14068 SDValue Negs = DAG.getNode(AArch64ISD::SUBS, DL, VTs, Zero, N0); in BuildSREMPow2()
14069 SDValue AndPos = DAG.getNode(ISD::AND, DL, VT, N0, Pow2MinusOne); in BuildSREMPow2()
14070 SDValue AndNeg = DAG.getNode(ISD::AND, DL, VT, Negs, Pow2MinusOne); in BuildSREMPow2()
14071 CSNeg = DAG.getNode(AArch64ISD::CSNEG, DL, VT, AndPos, AndNeg, CCVal, in BuildSREMPow2()
14074 Created.push_back(Negs.getNode()); in BuildSREMPow2()
14075 Created.push_back(AndPos.getNode()); in BuildSREMPow2()
14076 Created.push_back(AndNeg.getNode()); in BuildSREMPow2()
14083 switch(getIntrinsicID(S.getNode())) { in IsSVECntIntrinsic()
14121 dyn_cast<ConstantSDNode>(Extend.getOperand(1).getNode()); in calculatePreExtendType()
14194 NBV = DAG.getNode(ISD::BUILD_VECTOR, DL, PreExtendVT, NewOps); in performBuildShuffleExtendCombine()
14203 return DAG.getNode(IsSExt ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL, VT, NBV); in performBuildShuffleExtendCombine()
14222 return DAG.getNode(Mul->getOpcode(), DL, VT, Op0 ? Op0 : Mul->getOperand(0), in performMulVectorExtendCombine()
14260 SDValue MulVal = DAG.getNode(ISD::MUL, DL, VT, N1, MulOper); in performMulCombine()
14261 return DAG.getNode(AddSubOpc, DL, VT, N1, MulVal); in performMulCombine()
14265 SDValue MulVal = DAG.getNode(ISD::MUL, DL, VT, N0, MulOper); in performMulCombine()
14266 return DAG.getNode(AddSubOpc, DL, VT, N0, MulVal); in performMulCombine()
14301 if (N0->hasOneUse() && (isSignExtended(N0.getNode(), DAG) || in performMulCombine()
14302 isZeroExtended(N0.getNode(), DAG))) in performMulCombine()
14351 SDValue ShiftedVal = DAG.getNode(ISD::SHL, DL, VT, N0, in performMulCombine()
14356 SDValue Res = DAG.getNode(AddSubOpc, DL, VT, AddSubN0, AddSubN1); in performMulCombine()
14361 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Res); in performMulCombine()
14364 return DAG.getNode(ISD::SHL, DL, VT, Res, in performMulCombine()
14403 SDValue SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0)); in performVectorCompareAndMaskUnaryOpCombine()
14405 SDValue MaskConst = DAG.getNode(ISD::BITCAST, DL, IntVT, SourceConst); in performVectorCompareAndMaskUnaryOpCombine()
14406 SDValue NewAnd = DAG.getNode(ISD::AND, DL, IntVT, in performVectorCompareAndMaskUnaryOpCombine()
14408 SDValue Res = DAG.getNode(ISD::BITCAST, DL, VT, NewAnd); in performVectorCompareAndMaskUnaryOpCombine()
14434 if (Subtarget->hasNEON() && ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() && in performIntToFpCombine()
14448 return DAG.getNode(Opcode, SDLoc(N), VT, Load); in performIntToFpCombine()
14515 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, ResTy, in performFpToIntCombine()
14520 FixConv = DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), FixConv); in performFpToIntCombine()
14584 ConvInput = DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL, in performFDivCombine()
14589 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(), in performFDivCombine()
14654 return DAG.getNode(AArch64ISD::EXTR, DL, VT, LHS, RHS, in tryCombineToEXTR()
14703 if (!ISD::isBuildVectorAllZeros(Sub.getOperand(0).getNode())) in tryCombineToBSL()
14707 if (!ISD::isBuildVectorAllOnes(Add.getOperand(1).getNode())) in tryCombineToBSL()
14713 return DAG.getNode(AArch64ISD::BSP, DL, VT, Sub, SubSibling, AddSibling); in tryCombineToBSL()
14741 return DAG.getNode(AArch64ISD::BSP, DL, VT, SDValue(BVN0, 0), in tryCombineToBSL()
14799 CCmp = DAG.getNode(AArch64ISD::CCMP, DL, MVT_CC, Cmp1.getOperand(0), in performANDORCSELCombine()
14807 CCmp = DAG.getNode(AArch64ISD::CCMP, DL, MVT_CC, Cmp1.getOperand(0), in performANDORCSELCombine()
14810 return DAG.getNode(AArch64ISD::CSEL, DL, VT, CSel0.getOperand(0), in performANDORCSELCombine()
14900 Dup = DAG.getNode(ISD::SPLAT_VECTOR, DL, UnpkOp->getValueType(0), in performSVEAndCombine()
14903 SDValue And = DAG.getNode(ISD::AND, DL, in performSVEAndCombine()
14906 return DAG.getNode(Opc, DL, N->getValueType(0), And); in performSVEAndCombine()
14948 if (isConstantSplatVectorMaskForType(Mask.getNode(), MemVT)) in performSVEAndCombine()
14975 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(RHS.getNode()); in performANDCombine()
15153 SDValue Extract1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Other, in performExtractVectorEltCombine()
15155 SDValue Extract2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Other, in performExtractVectorEltCombine()
15158 return DAG.getNode(N0->getOpcode(), DL, VT, Extract1, Extract2); in performExtractVectorEltCombine()
15164 SDValue Ret = DAG.getNode(N0->getOpcode(), DL, in performExtractVectorEltCombine()
15211 return DAG.getNode(ISD::TRUNCATE, dl, VT, in performConcatVectorsCombine()
15214 DAG.getNode(ISD::BITCAST, dl, MidVT, N00), in performConcatVectorsCombine()
15215 DAG.getNode(ISD::BITCAST, dl, MidVT, N10), Mask)); in performConcatVectorsCombine()
15302 return DAG.getNode(N0Opc, dl, VT, N00Source, N01Source); in performConcatVectorsCombine()
15312 return DAG.getNode(AArch64ISD::DUPLANE64, dl, VT, WidenVector(N0, DAG), in performConcatVectorsCombine()
15338 return DAG.getNode(ISD::BITCAST, dl, VT, in performConcatVectorsCombine()
15339 DAG.getNode(ISD::CONCAT_VECTORS, dl, ConcatTy, in performConcatVectorsCombine()
15340 DAG.getNode(ISD::BITCAST, dl, RHSTy, N0), in performConcatVectorsCombine()
15362 return DAG.getNode(ISD::SPLAT_VECTOR, SDLoc(N), VT, V.getOperand(0)); in performExtractSubvectorCombine()
15399 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, Vec, in performInsertSubvectorCombine()
15402 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, Vec, in performInsertSubvectorCombine()
15406 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, Lo, Hi); in performInsertSubvectorCombine()
15453 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VecResTy, IID, Vec, Shift); in tryCombineFixedPointConvert()
15454 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResTy, Convert, Lane); in tryCombineFixedPointConvert()
15506 N = DAG.getNode(N->getOpcode(), DL, NewVT, N->ops()); in tryExtendDUPToExtractHigh()
15509 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, N, in tryExtendDUPToExtractHigh()
15655 LHS = DAG.getNode(ISD::ADD, dl, VT, RHS, DAG.getConstant(1, dl, VT)); in performSetccAddFolding()
15656 return DAG.getNode(AArch64ISD::CSEL, dl, VT, RHS, LHS, CCVal, Cmp); in performSetccAddFolding()
15690 SDValue AddVal = DAG.getNode(ISD::ADD, DL, ValVT, Val1, Val2); in performAddUADDVCombine()
15691 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, in performAddUADDVCombine()
15692 DAG.getNode(AArch64ISD::UADDV, DL, ValVT, AddVal), in performAddUADDVCombine()
15764 SDValue NewNode = DAG.getNode(ISD::ADD, DL, VT, RHS, SDValue(CTVal, 0)); in performAddCSelIntoCSinc()
15768 return DAG.getNode(AArch64ISD::CSINC, DL, VT, NewNode, RHS, CCVal, Cmp); in performAddCSelIntoCSinc()
15783 isZerosVector(Dot.getOperand(0).getNode()); in performAddDotCombine()
15790 return DAG.getNode(Dot.getOpcode(), SDLoc(N), VT, A, Dot.getOperand(1), in performAddDotCombine()
15802 return DAG.getNode(ISD::SUB, DL, VT, Zero, Op); in getNegatedInteger()
15833 return DAG.getNode(AArch64ISD::CSEL, DL, VT, N0N, N1N, CSel.getOperand(2), in performNegCSelCombine()
15875 if (!RHS.getNode()) in performAddSubLongCombine()
15878 RHS = DAG.getNode(ExtType, SDLoc(N), VT, RHS); in performAddSubLongCombine()
15881 if (!LHS.getNode()) in performAddSubLongCombine()
15884 LHS = DAG.getNode(ExtType, SDLoc(N), VT, LHS); in performAddSubLongCombine()
15887 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, LHS, RHS); in performAddSubLongCombine()
15892 !Op.getNode()->hasAnyUseOfValue(0); in isCMP()
15933 return DAG.getNode(Op->getOpcode(), SDLoc(Op), Op->getVTList(), in foldOverflowCheck()
15952 return DAG.getNode(AArch64ISD::CSINC, DL, VT, LHS, LHS, CC, Cond); in foldADCToCINC()
15982 SDValue NewN0 = DAG.getNode(N->getOperand(0).getOpcode(), DL, HalfVT, N0); in performVectorAddSubExtCombine()
15983 SDValue NewN1 = DAG.getNode(N->getOperand(1).getOpcode(), DL, HalfVT, N1); in performVectorAddSubExtCombine()
15984 SDValue NewOp = DAG.getNode(N->getOpcode(), DL, HalfVT, NewN0, NewN1); in performVectorAddSubExtCombine()
15985 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, NewOp); in performVectorAddSubExtCombine()
16025 SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, DL, ExtVT, VecToExtend); in performBuildVectorCombine()
16026 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i32, Ext, in performBuildVectorCombine()
16075 if (!RHS.getNode()) in tryCombineLongOpWithDup()
16079 if (!LHS.getNode()) in tryCombineLongOpWithDup()
16084 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), LHS, RHS); in tryCombineLongOpWithDup()
16086 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SDLoc(N), N->getValueType(0), in tryCombineLongOpWithDup()
16147 return DAG.getNode(Opcode, dl, N->getValueType(0), N->getOperand(1), in tryCombineShiftImm()
16151 return DAG.getNode(Opcode, dl, N->getValueType(0), N->getOperand(1), in tryCombineShiftImm()
16170 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SDLoc(N), MVT::i32, in tryCombineCRC32()
16177 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, N->getValueType(0), in combineAcrossLanesIntrinsic()
16178 DAG.getNode(Opc, dl, in combineAcrossLanesIntrinsic()
16194 SDValue Step = DAG.getNode(ISD::SPLAT_VECTOR, DL, N->getValueType(0), Op2); in LowerSVEIntrinsicIndex()
16195 SDValue Mul = DAG.getNode(ISD::MUL, DL, N->getValueType(0), StepVector, Step); in LowerSVEIntrinsicIndex()
16196 SDValue Base = DAG.getNode(ISD::SPLAT_VECTOR, DL, N->getValueType(0), Op1); in LowerSVEIntrinsicIndex()
16197 return DAG.getNode(ISD::ADD, DL, N->getValueType(0), Mul, Base); in LowerSVEIntrinsicIndex()
16206 Scalar = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Scalar); in LowerSVEIntrinsicDUP()
16210 return DAG.getNode(AArch64ISD::DUP_MERGE_PASSTHRU, dl, N->getValueType(0), in LowerSVEIntrinsicDUP()
16231 SDValue Op0 = DAG.getNode(ISD::BITCAST, dl, ByteVT, N->getOperand(1)); in LowerSVEIntrinsicEXT()
16232 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, ByteVT, N->getOperand(2)); in LowerSVEIntrinsicEXT()
16233 SDValue Op2 = DAG.getNode(ISD::MUL, dl, MVT::i32, N->getOperand(3), in LowerSVEIntrinsicEXT()
16236 SDValue EXT = DAG.getNode(AArch64ISD::EXT, dl, ByteVT, Op0, Op1, Op2); in LowerSVEIntrinsicEXT()
16237 return DAG.getNode(ISD::BITCAST, dl, VT, EXT); in LowerSVEIntrinsicEXT()
16296 SDValue Splat = DAG.getNode(ISD::SPLAT_VECTOR, DL, CmpVT, Imm); in tryConvertSVEWideCompare()
16297 return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, DL, VT, Pred, in tryConvertSVEWideCompare()
16324 Pg = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, MVT::nxv16i1, Pg); in getPTest()
16327 Op = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, MVT::nxv16i1, Op); in getPTest()
16331 SDValue Test = DAG.getNode(AArch64ISD::PTEST, DL, MVT::Other, Pg, Op); in getPTest()
16336 SDValue Res = DAG.getNode(AArch64ISD::CSEL, DL, OutVT, FVal, TVal, CC, Test); in getPTest()
16350 SDValue Reduce = DAG.getNode(Opc, DL, ReduceVT, Pred, VecToReduce); in combineSVEReductionInt()
16355 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, N->getValueType(0), Reduce, in combineSVEReductionInt()
16367 SDValue Reduce = DAG.getNode(Opc, DL, ReduceVT, Pred, VecToReduce); in combineSVEReductionFP()
16372 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, N->getValueType(0), Reduce, in combineSVEReductionFP()
16388 InitVal = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ReduceVT, in combineSVEReductionOrderedFP()
16391 SDValue Reduce = DAG.getNode(Opc, DL, ReduceVT, Pred, InitVal, VecToReduce); in combineSVEReductionOrderedFP()
16395 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, N->getValueType(0), Reduce, in combineSVEReductionOrderedFP()
16404 return ISD::isConstantSplatVectorAllZeros(N.getNode()); in isAllInactivePredicate()
16419 if (ISD::isConstantSplatVectorAllOnes(N.getNode())) in isAllActivePredicate()
16461 return DAG.getNode(Opc, SDLoc(N), N->getValueType(0), Op1, Op2); in convertMergedOpToPredOp()
16463 return DAG.getNode(Opc, SDLoc(N), N->getValueType(0), Pg, Op1, Op2); in convertMergedOpToPredOp()
16502 Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, WhileVT, ID, in performIntrinsicCombine()
16504 Res = DAG.getNode(ISD::SIGN_EXTEND, DL, PromVT, Res); in performIntrinsicCombine()
16505 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtVT, Res, in performIntrinsicCombine()
16507 Res = DAG.getNode(ISD::TRUNCATE, DL, VT, Res); in performIntrinsicCombine()
16527 return DAG.getNode(ISD::FMAXIMUM, SDLoc(N), N->getValueType(0), in performIntrinsicCombine()
16530 return DAG.getNode(ISD::FMINIMUM, SDLoc(N), N->getValueType(0), in performIntrinsicCombine()
16533 return DAG.getNode(ISD::FMAXNUM, SDLoc(N), N->getValueType(0), in performIntrinsicCombine()
16536 return DAG.getNode(ISD::FMINNUM, SDLoc(N), N->getValueType(0), in performIntrinsicCombine()
16539 return DAG.getNode(AArch64ISD::SMULL, SDLoc(N), N->getValueType(0), in performIntrinsicCombine()
16542 return DAG.getNode(AArch64ISD::UMULL, SDLoc(N), N->getValueType(0), in performIntrinsicCombine()
16588 return DAG.getNode(ISD::SPLAT_VECTOR, SDLoc(N), N->getValueType(0), in performIntrinsicCombine()
16645 return DAG.getNode(ISD::SADDSAT, SDLoc(N), N->getValueType(0), in performIntrinsicCombine()
16648 return DAG.getNode(ISD::SSUBSAT, SDLoc(N), N->getValueType(0), in performIntrinsicCombine()
16651 return DAG.getNode(ISD::UADDSAT, SDLoc(N), N->getValueType(0), in performIntrinsicCombine()
16654 return DAG.getNode(ISD::USUBSAT, SDLoc(N), N->getValueType(0), in performIntrinsicCombine()
16657 return DAG.getNode(AArch64ISD::SRAD_MERGE_OP1, SDLoc(N), N->getValueType(0), in performIntrinsicCombine()
16661 return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, SDLoc(N), in performIntrinsicCombine()
16667 return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, SDLoc(N), in performIntrinsicCombine()
16673 return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, SDLoc(N), in performIntrinsicCombine()
16679 return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, SDLoc(N), in performIntrinsicCombine()
16685 return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, SDLoc(N), in performIntrinsicCombine()
16691 return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, SDLoc(N), in performIntrinsicCombine()
16696 return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, SDLoc(N), in performIntrinsicCombine()
16713 return DAG.getNode(ISD::VSELECT, SDLoc(N), N->getValueType(0), in performIntrinsicCombine()
16751 ISD::isConstantSplatVectorAllZeros(N.getNode()); in isCheapToExtend()
16774 cast<CondCodeSDNode>(SetCC->getOperand(2).getNode())->get(); in performSignExtendSetCCCombine()
16782 DAG.getNode(ExtType, SDLoc(N), N->getValueType(0), CCOp0); in performSignExtendSetCCCombine()
16784 DAG.getNode(ExtType, SDLoc(N), N->getValueType(0), CCOp1); in performSignExtendSetCCCombine()
16788 cast<CondCodeSDNode>(SetCC->getOperand(2).getNode())->get()); in performSignExtendSetCCCombine()
16804 SDNode *ABDNode = N->getOperand(0).getNode(); in performExtendCombine()
16807 if (!NewABD.getNode()) in performExtendCombine()
16810 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), N->getValueType(0), NewABD); in performExtendCombine()
16851 DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr, in splitStoreSplat()
16908 SDValue Load = DAG.getNode(Opc, DL, VTs, Ops); in performLD1Combine()
16909 SDValue LoadChain = SDValue(Load.getNode(), 1); in performLD1Combine()
16912 Load = DAG.getNode(ISD::TRUNCATE, DL, VT, Load.getValue(0)); in performLD1Combine()
16935 SDValue Ops[] = { DAG.getNode(ISD::BITCAST, DL, VT, L), L.getValue(1) }; in performLDNT1Combine()
16955 SDValue Load = DAG.getNode(Opcode, DL, {LoadVT, MVT::Other}, Ops); in performLD1ReplicateCombine()
16956 SDValue LoadChain = SDValue(Load.getNode(), 1); in performLD1ReplicateCombine()
16959 Load = DAG.getNode(ISD::BITCAST, DL, VT, Load.getValue(0)); in performLD1ReplicateCombine()
16976 SrcNew = DAG.getNode(ISD::BITCAST, DL, HwSrcVt, Data); in performST1Combine()
16978 SrcNew = DAG.getNode(ISD::ANY_EXTEND, DL, HwSrcVt, Data); in performST1Combine()
16987 return DAG.getNode(AArch64ISD::ST1_PRED, DL, N->getValueType(0), Ops); in performST1Combine()
16998 Data = DAG.getNode(ISD::BITCAST, DL, DataVT.changeTypeToInteger(), Data); in performSTNT1Combine()
17198 SDValue SubVector0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, StVal, in splitStores()
17200 SDValue SubVector1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, StVal, in splitStores()
17206 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr, in splitStores()
17245 isZerosVector(MLD->getPassThru().getNode()))) { in performUnpackCombine()
17299 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, HalfVT, BC); in performUzpCombine()
17300 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Trunc, in performUzpCombine()
17309 return DAG.getNode(AArch64ISD::UZP1, DL, ResVT, X, Op1); in performUzpCombine()
17317 return DAG.getNode(AArch64ISD::UZP1, DL, ResVT, Op0, Z); in performUzpCombine()
17360 VTSDNode *ExtFrom = cast<VTSDNode>(Offset.getOperand(2).getNode()); in performGLD1Combine()
17373 return DAG.getNode(NewOpc, DL, {ResVT, MVT::Other}, in performGLD1Combine()
17413 SDValue Unpk = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), VT, CC, in performSunpkloCombine()
17415 return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), N->getValueType(0), Unpk); in performSunpkloCombine()
17436 SDNode *LD = N->getOperand(LoadIdx).getNode(); in performPostLD1Combine()
17469 for (SDNode::use_iterator UI = Addr.getNode()->use_begin(), UE = in performPostLD1Combine()
17470 Addr.getNode()->use_end(); UI != UE; ++UI) { in performPostLD1Combine()
17478 if (ConstantSDNode *CInc = dyn_cast<ConstantSDNode>(Inc.getNode())) { in performPostLD1Combine()
17490 Visited.insert(Addr.getNode()); in performPostLD1Combine()
17493 Worklist.push_back(Vector.getNode()); in performPostLD1Combine()
17517 SDValue(UpdN.getNode(), 2) // Chain in performPostLD1Combine()
17520 DCI.CombineTo(N, SDValue(UpdN.getNode(), 0)); // Dup/Inserted Result in performPostLD1Combine()
17521 DCI.CombineTo(User, SDValue(UpdN.getNode(), 1)); // Write back register in performPostLD1Combine()
17581 Value.getNode()->hasOneUse() && ST->isUnindexed() && in performSTORECombine()
17662 Offset = DAG.getNode(ISD::MUL, DL, MVT::i64, Offset, Scale); in foldIndexIntoBase()
17663 BasePtr = DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr, Offset); in foldIndexIntoBase()
17682 Offset = DAG.getNode(ISD::SHL, DL, MVT::i64, Offset, Shift); in foldIndexIntoBase()
17683 Offset = DAG.getNode(ISD::MUL, DL, MVT::i64, Offset, Scale); in foldIndexIntoBase()
17684 BasePtr = DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr, Offset); in foldIndexIntoBase()
17685 Index = DAG.getNode(ISD::SHL, DL, Index.getValueType(), in foldIndexIntoBase()
17751 Index = DAG.getNode(ISD::STEP_VECTOR, SDLoc(N), NewIndexVT, in findMoreOptimalIndexType()
17804 for (SDNode::use_iterator UI = Addr.getNode()->use_begin(), in performNEONPostLDSTCombine()
17805 UE = Addr.getNode()->use_end(); UI != UE; ++UI) { in performNEONPostLDSTCombine()
17815 Visited.insert(Addr.getNode()); in performNEONPostLDSTCombine()
17883 if (ConstantSDNode *CInc = dyn_cast<ConstantSDNode>(Inc.getNode())) { in performNEONPostLDSTCombine()
17919 NewResults.push_back(SDValue(UpdN.getNode(), i)); in performNEONPostLDSTCombine()
17921 NewResults.push_back(SDValue(UpdN.getNode(), NumResultVecs + 1)); in performNEONPostLDSTCombine()
17923 DCI.CombineTo(User, SDValue(UpdN.getNode(), NumResultVecs)); in performNEONPostLDSTCombine()
17935 switch(V.getNode()->getOpcode()) { in checkValueWidth()
17939 LoadSDNode *LoadNode = cast<LoadSDNode>(V.getNode()); in checkValueWidth()
17948 VTSDNode *TypeNode = cast<VTSDNode>(V.getNode()->getOperand(1)); in checkValueWidth()
17957 VTSDNode *TypeNode = cast<VTSDNode>(V.getNode()->getOperand(1)); in checkValueWidth()
17967 return std::abs(cast<ConstantSDNode>(V.getNode())->getSExtValue()) < in checkValueWidth()
18117 SDNode *SubsNode = N->getOperand(CmpIndex).getNode(); in performCONDCombine()
18126 SDNode *AndNode = SubsNode->getOperand(0).getNode(); in performCONDCombine()
18150 SDValue AddInputValue1 = AddValue.getNode()->getOperand(0); in performCONDCombine()
18151 SDValue AddInputValue2 = AddValue.getNode()->getOperand(1); in performCONDCombine()
18157 if (!isa<ConstantSDNode>(AddInputValue2.getNode()) || in performCONDCombine()
18158 !isa<ConstantSDNode>(SubsInputValue.getNode())) in performCONDCombine()
18169 cast<ConstantSDNode>(AddInputValue2.getNode())->getSExtValue(), in performCONDCombine()
18170 cast<ConstantSDNode>(SubsInputValue.getNode())->getSExtValue())) in performCONDCombine()
18179 SDValue NewValue = DAG.getNode(CondOpcode, SDLoc(SubsNode), VTs, Ops); in performCONDCombine()
18180 DAG.ReplaceAllUsesWith(SubsNode, NewValue.getNode()); in performCONDCombine()
18197 N = NV.getNode(); in performBRCONDCombine()
18238 BR = DAG.getNode(AArch64ISD::CBZ, SDLoc(N), MVT::Other, Chain, LHS, Dest); in performBRCONDCombine()
18240 BR = DAG.getNode(AArch64ISD::CBNZ, SDLoc(N), MVT::Other, Chain, LHS, Dest); in performBRCONDCombine()
18285 return DAG.getNode(ISD::AND, SDLoc(N), CTTZ.getValueType(), CTTZ, in foldCSELofCTTZ()
18327 if (!ISD::isConstantSplatVector(Op->getOperand(1).getNode(), V)) in tryToWidenSetCCOperands()
18342 Op1ExtV = DAG.getNode(ISD::SIGN_EXTEND, DL, UseMVT, Op->getOperand(1)); in tryToWidenSetCCOperands()
18345 Op1ExtV = DAG.getNode(ISD::ZERO_EXTEND, DL, UseMVT, Op->getOperand(1)); in tryToWidenSetCCOperands()
18349 return DAG.getNode(ISD::SETCC, DL, UseMVT.changeVectorElementType(MVT::i1), in tryToWidenSetCCOperands()
18378 DAG.getNode(AArch64ISD::CSEL, DL, LHS.getValueType(), LHS.getOperand(0), in performSETCCCombine()
18392 SDValue TST = DAG.getNode(ISD::AND, DL, TstVT, LHS->getOperand(0), in performSETCCCombine()
18394 return DAG.getNode(ISD::SETCC, DL, VT, TST, RHS, N->getOperand(2)); in performSETCCCombine()
18407 LHS = DAG.getNode(ISD::VECREDUCE_OR, DL, MVT::i1, LHS->getOperand(0)); in performSETCCCombine()
18408 LHS = DAG.getNode(ISD::ZERO_EXTEND, DL, ToVT, LHS); in performSETCCCombine()
18428 SDValue Res = DCI.DAG.getNode(GenericOpcode, DL, VT, N->ops()); in performFlagSettingCombine()
18450 if (Cond != ISD::SETNE || !isZerosVector(RHS.getNode()) || in performSetCCPunpkCombine()
18493 if (Cond == ISD::SETNE && isZerosVector(RHS.getNode()) && in performSetccMergeZeroCombine()
18515 return DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), in performSetccMergeZeroCombine()
18621 return DAG.getNode(NewOpc, DL, MVT::Other, N->getOperand(0), NewTestSrc, in performTBZCombine()
18659 return DAG.getNode(ISD::VSELECT, SDLoc(N), NTy, in trySwapVSelectOperands()
18689 SDNode *CmpRHS = SetCC.getOperand(1).getNode(); in performVSelectCombine()
18690 SDNode *SplatLHS = N->getOperand(1).getNode(); in performVSelectCombine()
18691 SDNode *SplatRHS = N->getOperand(2).getNode(); in performVSelectCombine()
18708 auto Shift = DAG.getNode(ISD::SRA, SDLoc(N), VT, CmpLHS, Val); in performVSelectCombine()
18709 auto Or = DAG.getNode(ISD::OR, SDLoc(N), VT, Shift, N->getOperand(1)); in performVSelectCombine()
18731 return DAG.getNode(ISD::VSELECT, SDLoc(N), ResVT, SetCC, in performVSelectCombine()
18788 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, SrcVT, N0.getOperand(0)); in performSelectCombine()
18790 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, SrcVT, N0.getOperand(1)); in performSelectCombine()
18791 SDValue SetCC = DAG.getNode(ISD::SETCC, DL, CCVT, LHS, RHS, N0.getOperand(2)); in performSelectCombine()
18796 Mask = DAG.getNode(ISD::BITCAST, DL, in performSelectCombine()
18812 return DCI.DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, SDValue(LN, 0), in performDUPCombine()
18878 return DAG.getNode(ISD::SUB, DL, MVT::i64, Result, in performGlobalAddressCombine()
18890 SDValue SplatShift = DAG.getNode(ISD::SPLAT_VECTOR, DL, MVT::nxv2i64, Shift); in getScaledOffsetForBitWidth()
18892 return DAG.getNode(ISD::SHL, DL, MVT::nxv2i64, Offset, SplatShift); in getScaledOffsetForBitWidth()
18924 ConstantSDNode *OffsetConst = dyn_cast<ConstantSDNode>(Offset.getNode()); in isValidImmForSVEVecImmAddrMode()
19000 Offset = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::nxv2i64, Offset).getValue(0); in performScatterStoreCombine()
19019 SrcNew = DAG.getNode(ISD::BITCAST, DL, HwSrcVt, Src); in performScatterStoreCombine()
19021 SrcNew = DAG.getNode(ISD::ANY_EXTEND, DL, HwSrcVt, Src); in performScatterStoreCombine()
19030 return DAG.getNode(Opcode, DL, VTs, Ops); in performScatterStoreCombine()
19103 Offset = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::nxv2i64, Offset).getValue(0); in performGatherLoadCombine()
19120 SDValue Load = DAG.getNode(Opcode, DL, VTs, Ops); in performGatherLoadCombine()
19121 SDValue LoadChain = SDValue(Load.getNode(), 1); in performGatherLoadCombine()
19124 Load = DAG.getNode(ISD::TRUNCATE, DL, RetVT, Load.getValue(0)); in performGatherLoadCombine()
19129 Load = DAG.getNode(ISD::BITCAST, DL, RetVT, Load.getValue(0)); in performGatherLoadCombine()
19165 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, ExtOp.getValueType(), in performSignExtendInRegCombine()
19168 return DAG.getNode(SOpc, DL, N->getValueType(0), Ext); in performSignExtendInRegCombine()
19256 SDValue ExtLoad = DAG.getNode(NewOpc, SDLoc(N), VTs, Ops); in performSignExtendInRegCombine()
19258 DCI.CombineTo(Src.getNode(), ExtLoad, ExtLoad.getValue(1)); in performSignExtendInRegCombine()
19277 Offset = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::nxv2i64, Offset); in legalizeSVEGatherPrefetchOffsVec()
19282 return DAG.getNode(N->getOpcode(), DL, DAG.getVTList(MVT::Other), Ops); in legalizeSVEGatherPrefetchOffsVec()
19306 return DAG.getNode(N->getOpcode(), DL, DAG.getVTList(MVT::Other), Ops); in combineSVEPrefetchVecBaseImmOff()
19344 if (!ISD::isConstantSplatVectorAllZeros(InsertVec.getNode())) in removeRedundantInsertVectorElt()
19395 SDValue Splice = DAG.getNode(ISD::VECTOR_SPLICE, DL, ExtIntTy, LHS, RHS, Idx); in performSVESpliceCombine()
19413 if (DCI.isBeforeLegalizeOps() && ISD::isNormalLoad(N0.getNode()) && in performFPExtendCombine()
19422 DCI.CombineTo(N0.getNode(), in performFPExtendCombine()
19423 DAG.getNode(ISD::FP_ROUND, SDLoc(N0), N0.getValueType(), in performFPExtendCombine()
19452 SDValue Sel = DAG.getNode(ISD::AND, DL, VT, Mask, In1); in performBSPExpandForSVE()
19453 SDValue SelInv = DAG.getNode(ISD::AND, DL, VT, InvMask, In2); in performBSPExpandForSVE()
19454 return DAG.getNode(ISD::OR, DL, VT, Sel, SelInv); in performBSPExpandForSVE()
19485 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, NewSubvecVT, in performDupLane128Combine()
19487 SDValue NewDuplane128 = DAG.getNode(AArch64ISD::DUPLANE128, DL, NewSubvecVT, in performDupLane128Combine()
19489 return DAG.getNode(ISD::BITCAST, DL, VT, NewDuplane128); in performDupLane128Combine()
19780 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ResVT, Src1, ExtIdx); in PerformDAGCombine()
19808 Opnds.push_back(DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, in PerformDAGCombine()
19813 DAG.getNode(ISD::CONCAT_VECTORS, DL, Tuple.getValueType(), Opnds); in PerformDAGCombine()
19831 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, DestVT, Opnds); in PerformDAGCombine()
19856 SDValue A = DAG.getNode( in PerformDAGCombine()
19859 SDValue B = DAG.getNode( in PerformDAGCombine()
19961 if (!getIndexedAddressParts(Ptr.getNode(), Base, Offset, AM, IsInc, DAG)) in getPreIndexedAddressParts()
20012 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, CastResult)); in ReplaceBITCASTResults()
20024 Op = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Op); in ReplaceBITCASTResults()
20025 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Op)); in ReplaceBITCASTResults()
20059 SDValue Addp = DAG.getNode(AArch64ISD::ADDP, N, LoHi.first.getValueType(), in ReplaceAddWithADDP()
20070 DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Addp, in ReplaceAddWithADDP()
20084 SDValue InterVal = DAG.getNode(InterOp, dl, LoVT, Lo, Hi); in ReplaceReductionResults()
20085 SDValue SplitVal = DAG.getNode(AcrossOp, dl, LoVT, InterVal); in ReplaceReductionResults()
20091 SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i64, N); in splitInt128()
20092 SDValue Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i64, in splitInt128()
20093 DAG.getNode(ISD::SRL, DL, MVT::i128, N, in splitInt128()
20128 SDValue Half = DAG.getNode(Opcode, DL, ExtendedHalfVT, N->getOperand(0)); in ReplaceExtractSubVectorResults()
20129 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, Half)); in ReplaceExtractSubVectorResults()
20134 SDLoc dl(V.getNode()); in createGPRPairNode()
20137 DAG.getNode(ISD::SRL, dl, MVT::i128, V, DAG.getConstant(64, dl, MVT::i64)), in createGPRPairNode()
20199 DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), MVT::i128, Lo, Hi)); in ReplaceCMP_SWAP_128Results()
20232 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), MVT::i128, in ReplaceCMP_SWAP_128Results()
20308 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), MVT::i128, in ReplaceNodeResults()
20334 auto Op2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, N->getOperand(2)); in ReplaceNodeResults()
20335 auto V = DAG.getNode(AArch64ISD::CLASTA_N, DL, MVT::i32, in ReplaceNodeResults()
20337 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V)); in ReplaceNodeResults()
20342 auto Op2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, N->getOperand(2)); in ReplaceNodeResults()
20343 auto V = DAG.getNode(AArch64ISD::CLASTB_N, DL, MVT::i32, in ReplaceNodeResults()
20345 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V)); in ReplaceNodeResults()
20350 auto V = DAG.getNode(AArch64ISD::LASTA, DL, MVT::i32, in ReplaceNodeResults()
20352 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V)); in ReplaceNodeResults()
20357 auto V = DAG.getNode(AArch64ISD::LASTB, DL, MVT::i32, in ReplaceNodeResults()
20359 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V)); in ReplaceNodeResults()
20966 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V, Zero); in convertToScalableVector()
20977 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V, Zero); in convertFromScalableVector()
21009 Result = DAG.getNode(AArch64ISD::FP_EXTEND_MERGE_PASSTHRU, DL, ContainerVT, in LowerFixedLengthVectorLoadToSVE()
21026 if (ISD::isBuildVectorAllOnes(Mask.getNode())) in convertFixedMaskToScalableVector()
21032 return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, DL, Pg.getValueType(), in convertFixedMaskToScalableVector()
21058 if (isZerosVector(Load->getPassThru().getNode())) in LowerFixedLengthVectorMLoadToSVE()
21096 NewValue = DAG.getNode(AArch64ISD::FP_ROUND_MERGE_PASSTHRU, DL, TruncVT, Pg, in LowerFixedLengthVectorStoreToSVE()
21143 SDValue Res = DAG.getNode(AArch64ISD::SRAD_MERGE_OP1, dl, ContainerVT, Pg, Op1, Op2); in LowerFixedLengthVectorIntDivideToSVE()
21145 Res = DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(0, dl, VT), Res); in LowerFixedLengthVectorIntDivideToSVE()
21164 SDValue Op0 = DAG.getNode(ExtendOpcode, dl, WidenedVT, Op.getOperand(0)); in LowerFixedLengthVectorIntDivideToSVE()
21165 SDValue Op1 = DAG.getNode(ExtendOpcode, dl, WidenedVT, Op.getOperand(1)); in LowerFixedLengthVectorIntDivideToSVE()
21166 SDValue Div = DAG.getNode(Op.getOpcode(), dl, WidenedVT, Op0, Op1); in LowerFixedLengthVectorIntDivideToSVE()
21167 return DAG.getNode(ISD::TRUNCATE, dl, VT, Div); in LowerFixedLengthVectorIntDivideToSVE()
21177 SDValue Op0Lo = DAG.getNode(UnpkLo, dl, ScalableWidenedVT, Op0); in LowerFixedLengthVectorIntDivideToSVE()
21178 SDValue Op1Lo = DAG.getNode(UnpkLo, dl, ScalableWidenedVT, Op1); in LowerFixedLengthVectorIntDivideToSVE()
21179 SDValue Op0Hi = DAG.getNode(UnpkHi, dl, ScalableWidenedVT, Op0); in LowerFixedLengthVectorIntDivideToSVE()
21180 SDValue Op1Hi = DAG.getNode(UnpkHi, dl, ScalableWidenedVT, Op1); in LowerFixedLengthVectorIntDivideToSVE()
21187 SDValue ResultLo = DAG.getNode(Op.getOpcode(), dl, FixedWidenedVT, in LowerFixedLengthVectorIntDivideToSVE()
21189 SDValue ResultHi = DAG.getNode(Op.getOpcode(), dl, FixedWidenedVT, in LowerFixedLengthVectorIntDivideToSVE()
21195 SDValue ScalableResult = DAG.getNode(AArch64ISD::UZP1, dl, ContainerVT, in LowerFixedLengthVectorIntDivideToSVE()
21219 Val = DAG.getNode(ExtendOpc, DL, MVT::nxv8i16, Val); in LowerFixedLengthVectorIntExtendToSVE()
21224 Val = DAG.getNode(ExtendOpc, DL, MVT::nxv4i32, Val); in LowerFixedLengthVectorIntExtendToSVE()
21229 Val = DAG.getNode(ExtendOpc, DL, MVT::nxv2i64, Val); in LowerFixedLengthVectorIntExtendToSVE()
21252 Val = DAG.getNode(ISD::BITCAST, DL, MVT::nxv4i32, Val); in LowerFixedLengthVectorTruncateToSVE()
21253 Val = DAG.getNode(AArch64ISD::UZP1, DL, MVT::nxv4i32, Val, Val); in LowerFixedLengthVectorTruncateToSVE()
21258 Val = DAG.getNode(ISD::BITCAST, DL, MVT::nxv8i16, Val); in LowerFixedLengthVectorTruncateToSVE()
21259 Val = DAG.getNode(AArch64ISD::UZP1, DL, MVT::nxv8i16, Val, Val); in LowerFixedLengthVectorTruncateToSVE()
21264 Val = DAG.getNode(ISD::BITCAST, DL, MVT::nxv16i8, Val); in LowerFixedLengthVectorTruncateToSVE()
21265 Val = DAG.getNode(AArch64ISD::UZP1, DL, MVT::nxv16i8, Val, Val); in LowerFixedLengthVectorTruncateToSVE()
21283 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op0, Op.getOperand(1)); in LowerFixedLengthExtractVectorElt()
21296 auto ScalableRes = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ContainerVT, Op0, in LowerFixedLengthInsertVectorElt()
21339 auto ScalableRes = DAG.getNode(NewOp, DL, ContainerVT, Operands); in LowerToPredicatedOp()
21356 return DAG.getNode(NewOp, DL, VT, Operands, Op->getFlags()); in LowerToPredicatedOp()
21386 auto ScalableRes = DAG.getNode(Op.getOpcode(), SDLoc(Op), ContainerVT, Ops); in LowerToScalableOp()
21408 AccOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ContainerVT, in LowerVECREDUCE_SEQ_FADD()
21412 SDValue Rdx = DAG.getNode(AArch64ISD::FADDA_PRED, DL, ContainerVT, in LowerVECREDUCE_SEQ_FADD()
21415 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT, Rdx, Zero); in LowerVECREDUCE_SEQ_FADD()
21441 Op = DAG.getNode(ISD::XOR, DL, OpVT, Op, Pg); in LowerPredReductionToSVE()
21449 Pg = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, MVT::nxv2i1, Pg); in LowerPredReductionToSVE()
21450 Op = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, MVT::nxv2i1, Op); in LowerPredReductionToSVE()
21453 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i64, ID, Pg, Op); in LowerPredReductionToSVE()
21483 SDValue Rdx = DAG.getNode(Opcode, DL, RdxVT, Pg, VecOp); in LowerReductionToSVE()
21484 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT, in LowerReductionToSVE()
21510 Mask = DAG.getNode(ISD::TRUNCATE, DL, in LowerFixedLengthVectorSelectToSVE()
21513 auto ScalableRes = DAG.getNode(ISD::VSELECT, DL, ContainerVT, in LowerFixedLengthVectorSelectToSVE()
21535 auto Cmp = DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, DL, CmpVT, in LowerFixedLengthVectorSetccToSVE()
21554 Op = DAG.getNode(ISD::BITCAST, DL, ContainerDstVT, SrcOp); in LowerFixedLengthBitcastToSVE()
21575 Ops.push_back(DAG.getNode(ISD::CONCAT_VECTORS, DL, PairVT, in LowerFixedLengthConcatVectorsToSVE()
21578 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Ops); in LowerFixedLengthConcatVectorsToSVE()
21587 Op = DAG.getNode(AArch64ISD::SPLICE, DL, ContainerVT, Pg, SrcOp1, SrcOp2); in LowerFixedLengthConcatVectorsToSVE()
21606 Val = DAG.getNode(ISD::BITCAST, DL, SrcVT.changeTypeToInteger(), Val); in LowerFixedLengthFPExtendToSVE()
21607 Val = DAG.getNode(ISD::ANY_EXTEND, DL, VT.changeTypeToInteger(), Val); in LowerFixedLengthFPExtendToSVE()
21611 Val = DAG.getNode(AArch64ISD::FP_EXTEND_MERGE_PASSTHRU, DL, ContainerVT, in LowerFixedLengthFPExtendToSVE()
21632 Val = DAG.getNode(AArch64ISD::FP_ROUND_MERGE_PASSTHRU, DL, RoundVT, Pg, Val, in LowerFixedLengthFPRoundToSVE()
21637 Val = DAG.getNode(ISD::TRUNCATE, DL, VT.changeTypeToInteger(), Val); in LowerFixedLengthFPRoundToSVE()
21638 return DAG.getNode(ISD::BITCAST, DL, VT, Val); in LowerFixedLengthFPRoundToSVE()
21661 Val = DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL, in LowerFixedLengthIntToFPToSVE()
21668 Val = DAG.getNode(Opcode, DL, ContainerDstVT, Pg, Val, in LowerFixedLengthIntToFPToSVE()
21677 Val = DAG.getNode(Opcode, DL, CvtVT, Pg, Val, DAG.getUNDEF(CvtVT)); in LowerFixedLengthIntToFPToSVE()
21681 Val = DAG.getNode(ISD::TRUNCATE, DL, VT.changeTypeToInteger(), Val); in LowerFixedLengthIntToFPToSVE()
21682 return DAG.getNode(ISD::BITCAST, DL, VT, Val); in LowerFixedLengthIntToFPToSVE()
21708 Val = DAG.getNode(ISD::BITCAST, DL, SrcVT.changeTypeToInteger(), Val); in LowerFixedLengthFPToIntToSVE()
21709 Val = DAG.getNode(ISD::ANY_EXTEND, DL, VT, Val); in LowerFixedLengthFPToIntToSVE()
21713 Val = DAG.getNode(Opcode, DL, ContainerDstVT, Pg, Val, in LowerFixedLengthFPToIntToSVE()
21723 Val = DAG.getNode(Opcode, DL, CvtVT, Pg, Val, DAG.getUNDEF(CvtVT)); in LowerFixedLengthFPToIntToSVE()
21726 return DAG.getNode(ISD::TRUNCATE, DL, VT, Val); in LowerFixedLengthFPToIntToSVE()
21735 auto *SVN = cast<ShuffleVectorSDNode>(Op.getNode()); in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21756 SDValue Scalar = DAG.getNode( in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21759 Op = DAG.getNode(AArch64ISD::INSR, DL, ContainerVT, Op2, Scalar); in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21776 Op = DAG.getNode(ISD::BITCAST, DL, NewVT, Op1); in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21778 Op = DAG.getNode(ISD::BITCAST, DL, ContainerVT, Op); in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21786 DAG, VT, DAG.getNode(AArch64ISD::ZIP1, DL, ContainerVT, Op1, Op2)); in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21791 DAG, VT, DAG.getNode(Opc, DL, ContainerVT, Op1, Op2)); in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21796 DAG, VT, DAG.getNode(AArch64ISD::ZIP1, DL, ContainerVT, Op1, Op1)); in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21801 DAG, VT, DAG.getNode(Opc, DL, ContainerVT, Op1, Op1)); in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21826 Op = DAG.getNode(ISD::VECTOR_REVERSE, DL, ContainerVT, Op1); in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21832 DAG, VT, DAG.getNode(AArch64ISD::ZIP2, DL, ContainerVT, Op1, Op2)); in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21837 DAG, VT, DAG.getNode(Opc, DL, ContainerVT, Op1, Op2)); in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21842 DAG, VT, DAG.getNode(AArch64ISD::ZIP2, DL, ContainerVT, Op1, Op1)); in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21847 DAG, VT, DAG.getNode(Opc, DL, ContainerVT, Op1, Op1)); in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21885 Op = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, PackedInVT, Op); in getSVESafeBitCast()
21887 Op = DAG.getNode(ISD::BITCAST, DL, PackedVT, Op); in getSVESafeBitCast()
21891 Op = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, VT, Op); in getSVESafeBitCast()