Lines Matching refs:getNode
2754 Val = DAG.getNode(ISD::BITCAST, DL, MVT::getIntegerVT(ValueBits), Val); in splitValueIntoRegisterParts()
2755 Val = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::getIntegerVT(PartBits), Val); in splitValueIntoRegisterParts()
2756 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); in splitValueIntoRegisterParts()
2772 Val = DAG.getNode(ISD::BITCAST, DL, MVT::getIntegerVT(PartBits), Val); in joinRegisterPartsIntoValue()
2773 Val = DAG.getNode(ISD::TRUNCATE, DL, MVT::getIntegerVT(ValueBits), Val); in joinRegisterPartsIntoValue()
2774 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); in joinRegisterPartsIntoValue()
2829 return DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), in getPICJumpTableRelocBase()
3031 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, Dl, ValLoc, ValArg, in lowerMasksToReg()
3042 ValToCopy = DAG.getNode(ISD::ANY_EXTEND, Dl, ValLoc, ValToCopy); in lowerMasksToReg()
3053 return DAG.getNode(ISD::ANY_EXTEND, Dl, ValLoc, ValArg); in lowerMasksToReg()
3072 Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, Dl, MVT::i32, Arg, in Passv64i1ArgInRegs()
3074 Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, Dl, MVT::i32, Arg, in Passv64i1ArgInRegs()
3119 ValToCopy = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), ValToCopy); in LowerReturn()
3121 ValToCopy = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), ValToCopy); in LowerReturn()
3126 ValToCopy = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), ValToCopy); in LowerReturn()
3155 ValToCopy = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f80, ValToCopy); in LowerReturn()
3167 ValToCopy = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, in LowerReturn()
3277 if (Flag.getNode()) in LowerReturn()
3283 return DAG.getNode(opcode, dl, MVT::Other, RetOps); in LowerReturn()
3394 return DAG.getNode(ISD::CONCAT_VECTORS, Dl, MVT::v64i1, Lo, Hi); in getv64i1Argument()
3406 return DAG.getNode(ISD::SCALAR_TO_VECTOR, Dl, MVT::v1i1, ValReturned); in lowerRegToMasks()
3428 ValReturned = DAG.getNode(ISD::TRUNCATE, Dl, maskLen, ValReturned); in lowerRegToMasks()
3506 Val = DAG.getNode(ISD::FP_ROUND, dl, VA.getValVT(), Val, in LowerCallResult()
3518 Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val); in LowerCallResult()
3722 DAG.getNode(ISD::ADD, dl, PtrVT, DAG.getFrameIndex(FI, PtrVT), in LowerMemArgument()
3753 ? DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VA.getValVT(), Val) in LowerMemArgument()
3754 : DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val)) in LowerMemArgument()
3926 SDValue FIN = DAG.getNode(ISD::ADD, DL, in createVarArgAreaAndStoreRegisters()
3959 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps); in createVarArgAreaAndStoreRegisters()
4125 ArgValue = DAG.getNode(ISD::AssertSext, dl, RegVT, ArgValue, in LowerFormalArguments()
4128 ArgValue = DAG.getNode(ISD::AssertZext, dl, RegVT, ArgValue, in LowerFormalArguments()
4136 ArgValue = DAG.getNode(X86ISD::MOVDQ2Q, dl, VA.getValVT(), ArgValue); in LowerFormalArguments()
4144 ArgValue = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), ArgValue); in LowerFormalArguments()
4171 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, St, Chain); in LowerFormalArguments()
4192 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Copy, Chain); in LowerFormalArguments()
4264 PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()), in LowerMemOpCallTo()
4290 return SDValue(OutRetAddr.getNode(), 1); in EmitTailCallLoadRetAddr()
4504 Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, RegVT, Arg); in LowerCall()
4507 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, RegVT, Arg); in LowerCall()
4516 Arg = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Arg); in LowerCall()
4519 Arg = DAG.getNode(ISD::ANY_EXTEND, dl, RegVT, Arg); in LowerCall()
4577 if (!StackPtr.getNode()) in LowerCall()
4586 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); in LowerCall()
4597 Register(X86::EBX), DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), in LowerCall()
4695 if (!StackPtr.getNode()) in LowerCall()
4698 Source = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()), in LowerCall()
4713 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains2); in LowerCall()
4746 Callee = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, Callee); in LowerCall()
4830 if (InFlag.getNode()) in LowerCall()
4841 SDValue Ret = DAG.getNode(X86ISD::TC_RETURN, dl, NodeTys, Ops); in LowerCall()
4842 DAG.addCallSiteInfo(Ret.getNode(), std::move(CSInfo)); in LowerCall()
4847 Chain = DAG.getNode(X86ISD::NT_CALL, dl, NodeTys, Ops); in LowerCall()
4862 Chain = DAG.getNode(X86ISD::CALL_RVMARKER, dl, NodeTys, Ops); in LowerCall()
4864 Chain = DAG.getNode(X86ISD::CALL, dl, NodeTys, Ops); in LowerCall()
4868 DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge); in LowerCall()
4869 DAG.addCallSiteInfo(Chain.getNode(), std::move(CSInfo)); in LowerCall()
4874 DAG.addHeapAllocSite(Chain.getNode(), HeapAlloc); in LowerCall()
5256 if (!ISD::isNormalLoad(Op.getNode())) in mayFoldLoad()
5260 auto *Ld = cast<LoadSDNode>(Op.getNode()); in mayFoldLoad()
5280 auto *Ld = cast<LoadSDNode>(Op.getNode()); in mayFoldLoadIntoBroadcastFromMem()
5286 return Op.hasOneUse() && ISD::isNormalStore(*Op.getNode()->use_begin()); in mayFoldIntoStore()
5291 unsigned Opcode = Op.getNode()->use_begin()->getOpcode(); in mayFoldIntoZeroExtend()
5495 if (ISD::isNON_EXTLoad(LHS.getNode()) && in TranslateX86CC()
5496 !ISD::isNON_EXTLoad(RHS.getNode())) { in TranslateX86CC()
5756 if (!ISD::isConstantSplatVector(C.getNode(), MulC)) in decomposeMulByConstant()
6399 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResultVT, Vec, VecIdx); in extractSubVector()
6443 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResultVT, Result, Vec, VecIdx); in insertSubVector()
6468 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, VT, Res, Vec, in widenSubVector()
6579 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, in splitVectorOp()
6580 DAG.getNode(Op.getOpcode(), dl, LoVT, LoOps), in splitVectorOp()
6581 DAG.getNode(Op.getOpcode(), dl, HiVT, HiOps)); in splitVectorOp()
6657 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Subs); in SplitOpsAndApply()
6717 SDValue Res = DAG.getNode(Opcode, DL, DstVT, SrcOps); in getAVX512Node()
6753 if (IdxVal == 0 && ISD::isBuildVectorAllZeros(Vec.getNode())) { in insert1BitVector()
6755 Op = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, in insert1BitVector()
6758 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx); in insert1BitVector()
6772 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, Undef, Vec, in insert1BitVector()
6774 Vec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec, ShiftBits); in insert1BitVector()
6775 Vec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec, ShiftBits); in insert1BitVector()
6777 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, in insert1BitVector()
6780 Op = DAG.getNode(ISD::OR, dl, WideOpVT, Vec, SubVec); in insert1BitVector()
6781 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx); in insert1BitVector()
6784 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, in insert1BitVector()
6789 SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec, in insert1BitVector()
6791 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, SubVec, ZeroIdx); in insert1BitVector()
6794 if (ISD::isBuildVectorAllZeros(Vec.getNode())) { in insert1BitVector()
6799 SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec, in insert1BitVector()
6805 SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec, in insert1BitVector()
6808 SubVec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, SubVec, in insert1BitVector()
6811 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, SubVec, ZeroIdx); in insert1BitVector()
6816 SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec, in insert1BitVector()
6821 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVecVT, Vec, ZeroIdx); in insert1BitVector()
6822 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, in insert1BitVector()
6827 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, in insert1BitVector()
6831 Vec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec, ShiftBits); in insert1BitVector()
6832 Vec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec, ShiftBits); in insert1BitVector()
6834 Op = DAG.getNode(ISD::OR, dl, WideOpVT, Vec, SubVec); in insert1BitVector()
6835 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx); in insert1BitVector()
6843 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, Undef, Vec, ZeroIdx); in insert1BitVector()
6853 SDValue VMask0 = DAG.getNode(ISD::BITCAST, dl, WideOpVT, CMask0); in insert1BitVector()
6854 Vec = DAG.getNode(ISD::AND, dl, WideOpVT, Vec, VMask0); in insert1BitVector()
6855 SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec, in insert1BitVector()
6857 SubVec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, SubVec, in insert1BitVector()
6859 Op = DAG.getNode(ISD::OR, dl, WideOpVT, Vec, SubVec); in insert1BitVector()
6862 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx); in insert1BitVector()
6866 SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec, in insert1BitVector()
6868 SubVec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, SubVec, in insert1BitVector()
6873 SDValue Low = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec, in insert1BitVector()
6875 Low = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Low, in insert1BitVector()
6880 SDValue High = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec, in insert1BitVector()
6882 High = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, High, in insert1BitVector()
6886 Vec = DAG.getNode(ISD::OR, dl, WideOpVT, Low, High); in insert1BitVector()
6887 SubVec = DAG.getNode(ISD::OR, dl, WideOpVT, SubVec, Vec); in insert1BitVector()
6890 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, SubVec, ZeroIdx); in insert1BitVector()
6972 return DAG.getNode(Opcode, DL, VT, In); in getEXTEND_VECTOR_INREG()
6981 ISD::isBuildVectorAllOnes(V.getOperand(1).getNode())) in IsNOT()
6987 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Not), V.getValueType(), in IsNOT()
6992 if (collectConcatOps(V.getNode(), CatOps, DAG)) { in IsNOT()
6998 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(V), V.getValueType(), CatOps); in IsNOT()
7037 if ((ISD::isBuildVectorOfConstantSDNodes(V1.getNode()) || V1.isUndef()) && in getVectorShuffle()
7038 (ISD::isBuildVectorOfConstantSDNodes(V2.getNode()) || V2.isUndef())) { in getVectorShuffle()
7107 return DAG.getNode(X86ISD::PACKUS, dl, VT, LHS, RHS); in getPack()
7111 return DAG.getNode(X86ISD::PACKSS, dl, VT, LHS, RHS); in getPack()
7118 LHS = DAG.getNode(X86ISD::VSRLI, dl, OpVT, LHS, Amt); in getPack()
7119 RHS = DAG.getNode(X86ISD::VSRLI, dl, OpVT, RHS, Amt); in getPack()
7122 LHS = DAG.getNode(ISD::AND, dl, OpVT, LHS, Mask); in getPack()
7123 RHS = DAG.getNode(ISD::AND, dl, OpVT, RHS, Mask); in getPack()
7125 return DAG.getNode(X86ISD::PACKUS, dl, VT, LHS, RHS); in getPack()
7129 LHS = DAG.getNode(X86ISD::VSHLI, dl, OpVT, LHS, Amt); in getPack()
7130 RHS = DAG.getNode(X86ISD::VSHLI, dl, OpVT, RHS, Amt); in getPack()
7132 LHS = DAG.getNode(X86ISD::VSRAI, dl, OpVT, LHS, Amt); in getPack()
7133 RHS = DAG.getNode(X86ISD::VSRAI, dl, OpVT, RHS, Amt); in getPack()
7134 return DAG.getNode(X86ISD::PACKSS, dl, VT, LHS, RHS); in getPack()
7950 bool V1IsZero = ISD::isBuildVectorAllZeros(V1.getNode()); in computeZeroableShuffleElements()
7951 bool V2IsZero = ISD::isBuildVectorAllZeros(V2.getNode()); in computeZeroableShuffleElements()
8032 if (!getTargetShuffleMask(N.getNode(), VT, true, Ops, Mask, IsUnary)) in getTargetShuffleAndZeroables()
8291 if (!N->isOnlyUserOf(Sub.getNode())) in getFauxShuffleMask()
8337 if (ISD::isBuildVectorAllZeros(Src.getNode())) in getFauxShuffleMask()
8454 if (N0.getOpcode() == X86ISD::VSRAI && N->isOnlyUserOf(N0.getNode()) && in getFauxShuffleMask()
8459 if (N1.getOpcode() == X86ISD::VSRAI && N->isOnlyUserOf(N1.getNode()) && in getFauxShuffleMask()
8740 if (!getTargetShuffleMask(Op.getNode(), ShufVT, true, ShuffleOps, in getShuffleScalarElt()
8844 V = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, V); in LowerBuildVectorAsInsert()
8849 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, V, Op.getOperand(i), in LowerBuildVectorAsInsert()
8894 NextElt = DAG.getNode(ISD::SHL, dl, MVT::i32, NextElt, in LowerBuildVectorv16i8()
8897 Elt = DAG.getNode(ISD::OR, dl, MVT::i32, NextElt, Elt); in LowerBuildVectorv16i8()
8909 V = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, Elt); in LowerBuildVectorv16i8()
8914 Elt = DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, Elt); in LowerBuildVectorv16i8()
8915 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, V, Elt, in LowerBuildVectorv16i8()
8954 SDValue Dup = DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v2f64, NewBV); in LowerBuildVectorv4x32()
8983 if (!FirstNonZero.getNode()) { in LowerBuildVectorv4x32()
8989 assert(FirstNonZero.getNode() && "Unexpected build vector of all zeros!"); in LowerBuildVectorv4x32()
9037 if (!V1.getNode()) in LowerBuildVectorv4x32()
9045 assert(V1.getNode() && "Expected at least two non-zero elements!"); in LowerBuildVectorv4x32()
9057 SDValue Result = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2, in LowerBuildVectorv4x32()
9072 return DAG.getBitcast(VT, DAG.getNode(Opc, dl, ShVT, SrcOp, ShiftVal)); in getVShift()
9130 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, in LowerAsSplatVectorLoad()
9151 if (ISD::isNON_EXTLoad(Elt.getNode())) { in findEltLoadSrc()
9218 if (!Elt.getNode()) in EltsFromConsecutiveLoads()
9224 if (X86::isZeroNode(Elt) || ISD::isBuildVectorAllZeros(Elt.getNode())) { in EltsFromConsecutiveLoads()
9377 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), in EltsFromConsecutiveLoads()
9465 DAG.getNode(X86ISD::VBROADCAST, DL, BroadcastVT, RepeatLoad); in EltsFromConsecutiveLoads()
9525 if (Opc == X86ISD::VPERMV && U->getOperand(0).getNode() == N) in isFoldableUseOfShuffle()
9527 if (Opc == X86ISD::VPERMV3 && U->getOperand(1).getNode() == N) in isFoldableUseOfShuffle()
9536 if (Opc == X86ISD::VPDPBUSD && U->getOperand(2).getNode() != N) in isFoldableUseOfShuffle()
9607 SDValue Bcst = DAG.getNode(X86ISD::VBROADCASTM, dl, BcstVT, BOperand); in lowerBuildVectorAsBroadcast()
9687 bool IsLoad = ISD::isNormalLoad(Ld.getNode()); in lowerBuildVectorAsBroadcast()
9694 if (!ConstSplatVal && !IsLoad && !BVOp->isOnlyUserOf(Ld.getNode())) in lowerBuildVectorAsBroadcast()
9750 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld); in lowerBuildVectorAsBroadcast()
9787 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld); in lowerBuildVectorAsBroadcast()
9871 if (!VecIn1.getNode()) in buildFromShuffleMostly()
9874 if (!VecIn2.getNode()) in buildFromShuffleMostly()
9887 if (!VecIn1.getNode()) in buildFromShuffleMostly()
9890 VecIn2 = VecIn2.getNode() ? VecIn2 : DAG.getUNDEF(VT); in buildFromShuffleMostly()
9894 NV = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, NV, Op.getOperand(Idx), in buildFromShuffleMostly()
9909 if (ISD::isBuildVectorAllZeros(Op.getNode()) || in LowerBUILD_VECTORvXi1()
9910 ISD::isBuildVectorAllOnes(Op.getNode())) in LowerBUILD_VECTORvXi1()
9943 Cond = DAG.getNode(ISD::AND, dl, MVT::i8, Cond, in LowerBUILD_VECTORvXi1()
9952 return DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, Select, Select); in LowerBUILD_VECTORvXi1()
9960 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Select, in LowerBUILD_VECTORvXi1()
9973 DstVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, ImmL, ImmH); in LowerBUILD_VECTORvXi1()
9979 DstVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, DstVec, in LowerBUILD_VECTORvXi1()
9987 DstVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, DstVec, in LowerBUILD_VECTORvXi1()
10162 LO = DAG.getNode(X86Opcode, DL, NewVT, V0_LO, V0_HI); in ExpandHorizontalBinOp()
10164 HI = DAG.getNode(X86Opcode, DL, NewVT, V1_LO, V1_HI); in ExpandHorizontalBinOp()
10168 LO = DAG.getNode(X86Opcode, DL, NewVT, V0_LO, V1_LO); in ExpandHorizontalBinOp()
10171 HI = DAG.getNode(X86Opcode, DL, NewVT, V0_HI, V1_HI); in ExpandHorizontalBinOp()
10174 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LO, HI); in ExpandHorizontalBinOp()
10354 return DAG.getNode(Opc, DL, VT, Opnd0, Opnd1, Opnd2); in lowerToAddSubOrFMAddSub()
10369 SDValue Sub = DAG.getNode(ISD::FSUB, DL, VT, Opnd0, Opnd1); in lowerToAddSubOrFMAddSub()
10370 SDValue Add = DAG.getNode(ISD::FADD, DL, VT, Opnd0, Opnd1); in lowerToAddSubOrFMAddSub()
10374 return DAG.getNode(X86ISD::ADDSUB, DL, VT, Opnd0, Opnd1); in lowerToAddSubOrFMAddSub()
10494 SDValue Half = DAG.getNode(HOpcode, SDLoc(BV), HalfVT, V0, V1); in getHopForBuildVector()
10498 return DAG.getNode(HOpcode, SDLoc(BV), VT, V0, V1); in getHopForBuildVector()
10691 SDValue Res = DAG.getNode(Opcode, DL, VT, LHS, RHS); in lowerBuildVectorToBitOp()
10710 if (ISD::isBuildVectorAllZeros(Op.getNode())) in materializeVectorConstant()
10716 if (Subtarget.hasSSE2() && ISD::isBuildVectorAllOnes(Op.getNode())) { in materializeVectorConstant()
10750 IndicesVec = DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(IndicesVec), in createVariablePermute()
10793 Idx = DAG.getNode(ISD::MUL, SDLoc(Idx), SrcVT, Idx, in createVariablePermute()
10795 Idx = DAG.getNode(ISD::ADD, SDLoc(Idx), SrcVT, Idx, in createVariablePermute()
10830 IndicesVec = DAG.getNode(ISD::ADD, DL, IndicesVT, IndicesVec, IndicesVec); in createVariablePermute()
10851 return DAG.getNode( in createVariablePermute()
10853 DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, LoSrc, HiSrc, LoIdx), in createVariablePermute()
10854 DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, LoSrc, HiSrc, HiIdx)); in createVariablePermute()
10858 SDValue LoLo = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Lo); in createVariablePermute()
10859 SDValue HiHi = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Hi, Hi); in createVariablePermute()
10868 DAG.getNode(X86ISD::PSHUFB, DL, VT, Ops[1], Idx), in createVariablePermute()
10869 DAG.getNode(X86ISD::PSHUFB, DL, VT, Ops[0], Idx), in createVariablePermute()
10901 VT, DAG.getNode(X86ISD::VPERMIL2, DL, MVT::v8f32, LoLo, HiHi, in createVariablePermute()
10907 DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, HiHi, IndicesVec), in createVariablePermute()
10908 DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, LoLo, IndicesVec), in createVariablePermute()
10934 IndicesVec = DAG.getNode(ISD::ADD, DL, IndicesVT, IndicesVec, IndicesVec); in createVariablePermute()
10937 VT, DAG.getNode(X86ISD::VPERMIL2, DL, MVT::v4f64, LoLo, HiHi, in createVariablePermute()
10943 DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v4f64, HiHi, IndicesVec), in createVariablePermute()
10944 DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v4f64, LoLo, IndicesVec), in createVariablePermute()
10981 ? DAG.getNode(Opcode, DL, ShuffleVT, IndicesVec, SrcVec) in createVariablePermute()
10982 : DAG.getNode(Opcode, DL, ShuffleVT, SrcVec, IndicesVec); in createVariablePermute()
11088 BuildVectorSDNode *BV = cast<BuildVectorSDNode>(Op.getNode()); in LowerBUILD_VECTOR()
11145 assert(!VarElt.getNode() && !InsIndex.getNode() && in LowerBUILD_VECTOR()
11167 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Ld, VarElt, InsIndex); in LowerBUILD_VECTOR()
11177 SDValue S2V = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, VarElt); in LowerBUILD_VECTOR()
11192 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item); in LowerBUILD_VECTOR()
11200 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item); in LowerBUILD_VECTOR()
11209 Item = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Item); in LowerBUILD_VECTOR()
11211 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, ShufVT, Item); in LowerBUILD_VECTOR()
11223 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, in LowerBUILD_VECTOR()
11237 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item); in LowerBUILD_VECTOR()
11251 if (Op.getNode()->isOnlyUserOf(Item.getNode())) in LowerBUILD_VECTOR()
11294 return DAG.getBitcast(VT, DAG.getNode(X86ISD::VBROADCAST, dl, BcastVT, in LowerBUILD_VECTOR()
11319 SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, in LowerBUILD_VECTOR()
11350 Ops[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i)); in LowerBUILD_VECTOR()
11392 Result = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(0)); in LowerBUILD_VECTOR()
11398 Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Result, in LowerBUILD_VECTOR()
11410 Ops[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i)); in LowerBUILD_VECTOR()
11453 if (ISD::isBuildVectorAllZeros(SubVec.getNode())) in LowerAVXCONCAT_VECTORS()
11466 SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT, in LowerAVXCONCAT_VECTORS()
11468 SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT, in LowerAVXCONCAT_VECTORS()
11470 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi); in LowerAVXCONCAT_VECTORS()
11483 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, in LowerAVXCONCAT_VECTORS()
11512 if (ISD::isBuildVectorAllZeros(SubVec.getNode())) in LowerCONCAT_VECTORSvXi1()
11531 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ShiftVT, in LowerCONCAT_VECTORSvXi1()
11534 Op = DAG.getNode(X86ISD::KSHIFTL, dl, ShiftVT, SubVec, in LowerCONCAT_VECTORSvXi1()
11536 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResVT, Op, in LowerCONCAT_VECTORSvXi1()
11548 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, SubVec, in LowerCONCAT_VECTORSvXi1()
11555 SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT, in LowerCONCAT_VECTORSvXi1()
11557 SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT, in LowerCONCAT_VECTORSvXi1()
11559 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi); in LowerCONCAT_VECTORSvXi1()
11567 SDValue Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, in LowerCONCAT_VECTORSvXi1()
11570 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, Op.getOperand(1), in LowerCONCAT_VECTORSvXi1()
12101 VT, DAG.getNode(X86ISD::PSHUFB, DL, I8VT, DAG.getBitcast(I8VT, V), in lowerShuffleWithPSHUFB()
12130 return DAG.getNode(X86ISD::EXPAND, DL, VT, ExpandedVector, ZeroVector, VMask); in lowerShuffleToEXPAND()
12228 return DAG.getNode(X86ISD::UNPCKL, DL, VT, V1, V2); in lowerShuffleWithUNPCK()
12233 return DAG.getNode(X86ISD::UNPCKH, DL, VT, V1, V2); in lowerShuffleWithUNPCK()
12238 return DAG.getNode(X86ISD::UNPCKL, DL, VT, V2, V1); in lowerShuffleWithUNPCK()
12242 return DAG.getNode(X86ISD::UNPCKH, DL, VT, V2, V1); in lowerShuffleWithUNPCK()
12270 return DAG.getNode(UnpackOpcode, DL, VT, V1, V1); in lowerShuffleWithUNPCK256()
12327 return DAG.getNode(ISD::TRUNCATE, DL, DstVT, Src); in getAVX512TruncNode()
12331 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Src); in getAVX512TruncNode()
12337 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Src); in getAVX512TruncNode()
12351 SDValue Trunc = DAG.getNode(X86ISD::VTRUNC, DL, TruncVT, Src); in getAVX512TruncNode()
12455 SDValue Src = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, V1, V2); in lowerShuffleAsVTRUNC()
12648 Res = DAG.getNode(PackOpcode, DL, DstVT, DAG.getBitcast(SrcVT, V1), in lowerShuffleWithPACK()
12709 SDValue And = DAG.getNode(ISD::AND, DL, LogicVT, V, VMask); in lowerShuffleAsBitMask()
12733 V1 = DAG.getNode(ISD::AND, DL, VT, V1, V1Mask); in lowerShuffleAsBitBlend()
12734 V2 = DAG.getNode(X86ISD::ANDNP, DL, VT, V1Mask, V2); in lowerShuffleAsBitBlend()
12735 return DAG.getNode(ISD::OR, DL, VT, V1, V2); in lowerShuffleAsBitBlend()
12748 V1.isUndef() || ISD::isBuildVectorAllZeros(V1.getNode()); in matchShuffleAsBlend()
12750 V2.isUndef() || ISD::isBuildVectorAllZeros(V2.getNode()); in matchShuffleAsBlend()
12841 return DAG.getNode(X86ISD::BLENDI, DL, VT, V1, V2, in lowerShuffleAsBlend()
12853 return DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2, in lowerShuffleAsBlend()
12862 SDValue Lo = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2, in lowerShuffleAsBlend()
12864 SDValue Hi = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2, in lowerShuffleAsBlend()
12906 if (!ISD::isNormalLoad(V1.getNode()) && ISD::isNormalLoad(V2.getNode())) { in lowerShuffleAsBlend()
13063 SDValue Unpck = DAG.getNode(UnpckOp, DL, VT, Ops); in lowerShuffleAsUNPCKAndPermute()
13127 VT, DAG.getNode(X86ISD::PALIGNR, DL, ByteVT, DAG.getBitcast(ByteVT, Hi), in lowerShuffleAsByteRotateAndPermute()
13208 Input = DAG.getNode(X86ISD::VBROADCAST, DL, VT, Input); in lowerShuffleAsDecomposedShuffleMerge()
13350 SDValue SHL = DAG.getNode(X86ISD::VSHLI, DL, RotateVT, V1, in lowerShuffleAsBitRotate()
13352 SDValue SRL = DAG.getNode(X86ISD::VSRLI, DL, RotateVT, V1, in lowerShuffleAsBitRotate()
13354 SDValue Rot = DAG.getNode(ISD::OR, DL, RotateVT, SHL, SRL); in lowerShuffleAsBitRotate()
13359 DAG.getNode(X86ISD::VROTLI, DL, RotateVT, DAG.getBitcast(RotateVT, V1), in lowerShuffleAsBitRotate()
13496 VT, DAG.getNode(X86ISD::PALIGNR, DL, ByteVT, Lo, Hi, in lowerShuffleAsByteRotate()
13512 DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Lo, in lowerShuffleAsByteRotate()
13515 DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Hi, in lowerShuffleAsByteRotate()
13518 DAG.getNode(ISD::OR, DL, MVT::v16i8, LoShift, HiShift)); in lowerShuffleAsByteRotate()
13547 return DAG.getNode(X86ISD::VALIGN, DL, VT, Lo, Hi, in lowerShuffleAsVALIGN()
13588 Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res, in lowerShuffleAsByteShiftMask()
13590 Res = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Res, in lowerShuffleAsByteShiftMask()
13594 Res = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Res, in lowerShuffleAsByteShiftMask()
13596 Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res, in lowerShuffleAsByteShiftMask()
13603 Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res, in lowerShuffleAsByteShiftMask()
13606 Res = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Res, in lowerShuffleAsByteShiftMask()
13608 Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res, in lowerShuffleAsByteShiftMask()
13730 V = DAG.getNode(Opcode, DL, ShiftVT, V, in lowerShuffleAsShift()
13864 return DAG.getNode(X86ISD::EXTRQI, DL, VT, V1, in lowerShuffleWithSSE4A()
13869 return DAG.getNode(X86ISD::INSERTQI, DL, VT, V1 ? V1 : DAG.getUNDEF(VT), in lowerShuffleWithSSE4A()
13942 VT, DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, in lowerShuffleAsSpecificZeroOrAnyExtend()
13949 InputV = DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, in lowerShuffleAsSpecificZeroOrAnyExtend()
13955 VT, DAG.getNode(OddEvenOp, DL, MVT::v8i16, in lowerShuffleAsSpecificZeroOrAnyExtend()
13968 MVT::v2i64, DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV, in lowerShuffleAsSpecificZeroOrAnyExtend()
13977 MVT::v2i64, DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV, in lowerShuffleAsSpecificZeroOrAnyExtend()
13981 DAG.getNode(X86ISD::UNPCKL, DL, MVT::v2i64, Lo, Hi)); in lowerShuffleAsSpecificZeroOrAnyExtend()
14001 VT, DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, InputV, in lowerShuffleAsSpecificZeroOrAnyExtend()
14028 InputV = DAG.getNode(UnpackLoHi, DL, InputVT, InputV, Ext); in lowerShuffleAsSpecificZeroOrAnyExtend()
14156 V = DAG.getNode(X86ISD::VZEXT_MOVL, DL, MVT::v2i64, V); in lowerShuffleAsZeroOrAnyExtend()
14197 ISD::isNON_EXTLoad(peekThroughOneUseBitcasts(V).getNode()); in isShuffleFoldableLoad()
14252 V2S = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, V2S); in lowerShuffleAsElementInsertion()
14254 V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, ExtVT, V2S); in lowerShuffleAsElementInsertion()
14286 return DAG.getNode(MovOpc, DL, ExtVT, V1, V2); in lowerShuffleAsElementInsertion()
14293 V2 = DAG.getNode(X86ISD::VZEXT_MOVL, DL, ExtVT, V2); in lowerShuffleAsElementInsertion()
14308 V2 = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, V2, in lowerShuffleAsElementInsertion()
14363 Scalar = DAG.getNode(ISD::SRL, DL, Scalar.getValueType(), Scalar, in lowerShuffleAsTruncBroadcast()
14366 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, in lowerShuffleAsTruncBroadcast()
14367 DAG.getNode(ISD::TRUNCATE, DL, EltVT, Scalar)); in lowerShuffleAsTruncBroadcast()
14440 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Shuf, in lowerShuffleOfExtractsAsVperm()
14545 } else if (ISD::isNormalLoad(V.getNode()) && in lowerShuffleAsBroadcast()
14608 V = DAG.getNode(X86ISD::VBROADCAST, DL, MVT::v2f64, V); in lowerShuffleAsBroadcast()
14611 V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, V); in lowerShuffleAsBroadcast()
14620 return DAG.getBitcast(VT, DAG.getNode(Opcode, DL, BroadcastVT, V)); in lowerShuffleAsBroadcast()
14633 return DAG.getNode(Opcode, DL, VT, DAG.getBitcast(CastVT, V)); in lowerShuffleAsBroadcast()
14742 return DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2, in lowerShuffleAsInsertPS()
14815 VT, DAG.getNode(UnpackLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, DL, in lowerShuffleAsPermuteAndUnpack()
14828 if (ISD::isBuildVectorAllZeros(V1.getNode()) || in lowerShuffleAsPermuteAndUnpack()
14829 ISD::isBuildVectorAllZeros(V2.getNode())) in lowerShuffleAsPermuteAndUnpack()
14855 VT, DL, DAG.getNode(NumLoInputs == 0 ? X86ISD::UNPCKH : X86ISD::UNPCKL, in lowerShuffleAsPermuteAndUnpack()
14891 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v2f64, V1, in lowerV2F64Shuffle()
14895 return DAG.getNode( in lowerV2F64Shuffle()
14930 return DAG.getNode( in lowerV2F64Shuffle()
14932 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, V1S)); in lowerV2F64Shuffle()
14944 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v2f64, V1, V2, in lowerV2F64Shuffle()
14978 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1, in lowerV2I64Shuffle()
15079 V2 = DAG.getNode(X86ISD::SHUFP, DL, VT, V2, V1, in lowerShuffleWithSHUFPS()
15118 V1 = DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2, in lowerShuffleWithSHUFPS()
15136 return DAG.getNode(X86ISD::SHUFP, DL, VT, LowV, HighV, in lowerShuffleWithSHUFPS()
15164 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v4f32, V1); in lowerV4F32Shuffle()
15166 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v4f32, V1); in lowerV4F32Shuffle()
15172 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f32, V1, in lowerV4F32Shuffle()
15180 return DAG.getNode(X86ISD::MOVLHPS, DL, MVT::v4f32, V1, V1); in lowerV4F32Shuffle()
15182 return DAG.getNode(X86ISD::MOVHLPS, DL, MVT::v4f32, V1, V1); in lowerV4F32Shuffle()
15187 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f32, V1, V1, in lowerV4F32Shuffle()
15224 return DAG.getNode(X86ISD::MOVLHPS, DL, MVT::v4f32, V1, V2); in lowerV4F32Shuffle()
15226 return DAG.getNode(X86ISD::MOVHLPS, DL, MVT::v4f32, V2, V1); in lowerV4F32Shuffle()
15278 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1, in lowerV4I32Shuffle()
15382 return DAG.getNode(X86ISD::PSHUFLW, DL, VT, V, in lowerV8I16GeneralSingleInputShuffle()
15389 return DAG.getNode(X86ISD::PSHUFHW, DL, VT, V, in lowerV8I16GeneralSingleInputShuffle()
15415 V = DAG.getNode(ShufWOp, DL, VT, V, in lowerV8I16GeneralSingleInputShuffle()
15418 V = DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT, V, in lowerV8I16GeneralSingleInputShuffle()
15565 V = DAG.getNode( in lowerV8I16GeneralSingleInputShuffle()
15593 DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT, DAG.getBitcast(PSHUFDVT, V), in lowerV8I16GeneralSingleInputShuffle()
15827 V = DAG.getNode(X86ISD::PSHUFLW, DL, VT, V, in lowerV8I16GeneralSingleInputShuffle()
15830 V = DAG.getNode(X86ISD::PSHUFHW, DL, VT, V, in lowerV8I16GeneralSingleInputShuffle()
15835 DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT, DAG.getBitcast(PSHUFDVT, V), in lowerV8I16GeneralSingleInputShuffle()
15847 V = DAG.getNode(X86ISD::PSHUFLW, DL, VT, V, in lowerV8I16GeneralSingleInputShuffle()
15855 V = DAG.getNode(X86ISD::PSHUFHW, DL, VT, V, in lowerV8I16GeneralSingleInputShuffle()
15897 V1 = DAG.getNode(X86ISD::PSHUFB, DL, ShufVT, DAG.getBitcast(ShufVT, V1), in lowerShuffleAsBlendOfPSHUFBs()
15900 V2 = DAG.getNode(X86ISD::PSHUFB, DL, ShufVT, DAG.getBitcast(ShufVT, V2), in lowerShuffleAsBlendOfPSHUFBs()
15906 V = DAG.getNode(ISD::OR, DL, ShufVT, V1, V2); in lowerShuffleAsBlendOfPSHUFBs()
16055 V1V2 = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1V2, in lowerV8I16Shuffle()
16068 V1 = DAG.getNode(ISD::AND, DL, MVT::v4i32, DAG.getBitcast(MVT::v4i32, V1), in lowerV8I16Shuffle()
16070 V2 = DAG.getNode(ISD::AND, DL, MVT::v4i32, DAG.getBitcast(MVT::v4i32, V2), in lowerV8I16Shuffle()
16074 SDValue Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v8i16, V1, V2); in lowerV8I16Shuffle()
16077 Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v8i16, Result, Result); in lowerV8I16Shuffle()
16086 V1 = DAG.getNode(HasSSE41 ? X86ISD::VSRLI : X86ISD::VSRAI, DL, MVT::v4i32, in lowerV8I16Shuffle()
16089 V2 = DAG.getNode(HasSSE41 ? X86ISD::VSRLI : X86ISD::VSRAI, DL, MVT::v4i32, in lowerV8I16Shuffle()
16092 return DAG.getNode(HasSSE41 ? X86ISD::PACKUS : X86ISD::PACKSS, DL, in lowerV8I16Shuffle()
16174 Result = DAG.getNode(X86ISD::VPERMV, DL, ShuffleVT, MaskNode, V1); in lowerShuffleWithPERMV()
16176 Result = DAG.getNode(X86ISD::VPERMV3, DL, ShuffleVT, V1, MaskNode, V2); in lowerShuffleWithPERMV()
16325 V1 = DAG.getNode(TargetLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, DL, in lowerV16I8Shuffle()
16418 return DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, V1, V2, MaskNode); in lowerV16I8Shuffle()
16459 V1 = DAG.getNode(ISD::AND, DL, MVT::v8i16, DAG.getBitcast(MVT::v8i16, V1), in lowerV16I8Shuffle()
16462 V2 = DAG.getNode(ISD::AND, DL, MVT::v8i16, DAG.getBitcast(MVT::v8i16, V2), in lowerV16I8Shuffle()
16466 SDValue Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, V1, in lowerV16I8Shuffle()
16470 Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, Result, Result); in lowerV16I8Shuffle()
16477 V1 = DAG.getNode(X86ISD::VSRLI, DL, MVT::v8i16, in lowerV16I8Shuffle()
16481 V2 = DAG.getNode(X86ISD::VSRLI, DL, MVT::v8i16, in lowerV16I8Shuffle()
16484 return DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, V1, in lowerV16I8Shuffle()
16512 VLoHalf = DAG.getNode(ISD::AND, DL, MVT::v8i16, VLoHalf, in lowerV16I8Shuffle()
16531 MVT::v8i16, DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i8, V, Zero)); in lowerV16I8Shuffle()
16533 MVT::v8i16, DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i8, V, Zero)); in lowerV16I8Shuffle()
16539 return DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, LoV, HiV); in lowerV16I8Shuffle()
16668 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi); in splitAndLowerShuffle()
16756 return DAG.getNode(X86ISD::SHUFP, DL, VT, LHS, RHS, in lowerShuffleAsLanePermuteAndSHUFP()
16958 bool V2IsZero = !V2.isUndef() && ISD::isBuildVectorAllZeros(V2.getNode()); in lowerV2X128Shuffle()
16970 SDValue LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V1, in lowerV2X128Shuffle()
16972 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, in lowerV2X128Shuffle()
16998 SDValue SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, in lowerV2X128Shuffle()
17001 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, V1, SubVec, in lowerV2X128Shuffle()
17011 return DAG.getNode(X86ISD::SHUF128, DL, VT, V1, V2, in lowerV2X128Shuffle()
17043 return DAG.getNode(X86ISD::VPERM2X128, DL, VT, V1, V2, in lowerV2X128Shuffle()
17290 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V, in getShuffleHalfVectors()
17303 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Op0, Op1); in getShuffleHalfVectors()
17307 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V, in getShuffleHalfVectors()
17334 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V1, in lowerShuffleWithUndefHalf()
17336 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), Hi, in lowerShuffleWithUndefHalf()
17344 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V1, in lowerShuffleWithUndefHalf()
17346 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), Hi, in lowerShuffleWithUndefHalf()
17692 return DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2, in lowerShuffleWithSHUFPD()
17717 V1 = DAG.getNode(X86ISD::VTRUNC, DL, MVT::v16i8, V1); in lowerShuffleAsVTRUNCAndUnpack()
17718 V2 = DAG.getNode(X86ISD::VTRUNC, DL, MVT::v16i8, V2); in lowerShuffleAsVTRUNCAndUnpack()
17726 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v32i8, in lowerShuffleAsVTRUNCAndUnpack()
17756 return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v4f64, V1); in lowerV4F64Shuffle()
17763 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f64, V1, in lowerV4F64Shuffle()
17769 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4f64, V1, in lowerV4F64Shuffle()
17887 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32, in lowerV4I64Shuffle()
17894 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4i64, V1, in lowerV4I64Shuffle()
17987 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v8f32, V1); in lowerV8F32Shuffle()
17989 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v8f32, V1); in lowerV8F32Shuffle()
17992 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f32, V1, in lowerV8F32Shuffle()
18015 return DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, V1, VPermMask); in lowerV8F32Shuffle()
18019 return DAG.getNode(X86ISD::VPERMV, DL, MVT::v8f32, VPermMask, V1); in lowerV8F32Shuffle()
18102 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32, V1, in lowerV8I32Shuffle()
18146 return DAG.getNode(X86ISD::VPERMV, DL, MVT::v8i32, VPermMask, V1); in lowerV8I32Shuffle()
18498 SDValue LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V1, in lowerV4X128Shuffle()
18500 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, in lowerV4X128Shuffle()
18512 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, OnlyUsesV1 ? V1 : V2, in lowerV4X128Shuffle()
18514 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, V1, SubVec, in lowerV4X128Shuffle()
18543 SDValue Subvec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V2, in lowerV4X128Shuffle()
18579 return DAG.getNode(X86ISD::SHUF128, DL, VT, Ops[0], Ops[1], in lowerV4X128Shuffle()
18595 return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v8f64, V1); in lowerV8F64Shuffle()
18604 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f64, V1, in lowerV8F64Shuffle()
18610 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v8f64, V1, in lowerV8F64Shuffle()
18654 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v16f32, V1); in lowerV16F32Shuffle()
18656 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v16f32, V1); in lowerV16F32Shuffle()
18659 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v16f32, V1, in lowerV16F32Shuffle()
18685 return DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v16f32, V1, VPermMask); in lowerV16F32Shuffle()
18715 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v16i32, in lowerV8I64Shuffle()
18722 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v8i64, V1, in lowerV8I64Shuffle()
18786 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v16i32, V1, in lowerV16I32Shuffle()
19103 SDValue Res = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideVT, in lower1BitShuffleAsKSHIFTR()
19106 Res = DAG.getNode(X86ISD::KSHIFTR, DL, WideVT, Res, in lower1BitShuffleAsKSHIFTR()
19108 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res, in lower1BitShuffleAsKSHIFTR()
19184 SDValue Extract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtractVT, in lower1BitShuffle()
19187 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, in lower1BitShuffle()
19206 SDValue Res = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideVT, in lower1BitShuffle()
19213 Res = DAG.getNode(X86ISD::KSHIFTL, DL, WideVT, Res, in lower1BitShuffle()
19219 Res = DAG.getNode(Opcode, DL, WideVT, Res, in lower1BitShuffle()
19221 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res, in lower1BitShuffle()
19275 V1 = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, V1); in lower1BitShuffle()
19276 V2 = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, V2); in lower1BitShuffle()
19286 return DAG.getNode(ISD::TRUNCATE, DL, VT, Shuffle); in lower1BitShuffle()
19420 bool V2IsZero = !V2IsUndef && ISD::isBuildVectorAllZeros(V2.getNode()); in lowerVECTOR_SHUFFLE()
19516 if (ISD::isBuildVectorOfConstantSDNodes(Cond.getNode())) { in lowerVSELECTtoVectorShuffle()
19534 return DAG.getBitcast(VT, DAG.getNode(ISD::VSELECT, dl, NVT, Cond, in LowerVSELECT()
19541 if (ISD::isBuildVectorOfConstantSDNodes(Cond.getNode()) && in LowerVSELECT()
19542 ISD::isBuildVectorOfConstantSDNodes(LHS.getNode()) && in LowerVSELECT()
19543 ISD::isBuildVectorOfConstantSDNodes(RHS.getNode())) in LowerVSELECT()
19591 return DAG.getNode(ISD::VSELECT, dl, VT, Cond, LHS, RHS); in LowerVSELECT()
19616 SDValue Select = DAG.getNode(ISD::VSELECT, dl, CastVT, Cond, LHS, RHS); in LowerVSELECT()
19637 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, in LowerEXTRACT_VECTOR_ELT_SSE4()
19638 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT_SSE4()
19642 SDValue Extract = DAG.getNode(X86ISD::PEXTRB, dl, MVT::i32, Vec, in LowerEXTRACT_VECTOR_ELT_SSE4()
19644 return DAG.getNode(ISD::TRUNCATE, dl, VT, Extract); in LowerEXTRACT_VECTOR_ELT_SSE4()
19655 SDNode *User = *Op.getNode()->use_begin(); in LowerEXTRACT_VECTOR_ELT_SSE4()
19660 SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT_SSE4()
19693 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, dl, ExtVecVT, Vec); in ExtractBitFromMaskVector()
19694 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ExtEltVT, Ext, Idx); in ExtractBitFromMaskVector()
19695 return DAG.getNode(ISD::TRUNCATE, dl, EltVT, Elt); in ExtractBitFromMaskVector()
19707 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVecVT, in ExtractBitFromMaskVector()
19713 Vec = DAG.getNode(X86ISD::KSHIFTR, dl, WideVecVT, Vec, in ExtractBitFromMaskVector()
19716 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op.getValueType(), Vec, in ExtractBitFromMaskVector()
19781 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op.getValueType(), Vec, in LowerEXTRACT_VECTOR_ELT()
19797 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, in LowerEXTRACT_VECTOR_ELT()
19798 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT()
19802 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32, Vec, in LowerEXTRACT_VECTOR_ELT()
19804 return DAG.getNode(ISD::TRUNCATE, dl, VT, Extract); in LowerEXTRACT_VECTOR_ELT()
19814 if (VT.getSizeInBits() == 8 && Op->isOnlyUserOf(Vec.getNode())) { in LowerEXTRACT_VECTOR_ELT()
19818 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT()
19823 Res = DAG.getNode(ISD::SRL, dl, MVT::i32, Res, in LowerEXTRACT_VECTOR_ELT()
19825 return DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in LowerEXTRACT_VECTOR_ELT()
19829 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, in LowerEXTRACT_VECTOR_ELT()
19834 Res = DAG.getNode(ISD::SRL, dl, MVT::i16, Res, in LowerEXTRACT_VECTOR_ELT()
19836 return DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in LowerEXTRACT_VECTOR_ELT()
19847 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec, in LowerEXTRACT_VECTOR_ELT()
19863 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec, in LowerEXTRACT_VECTOR_ELT()
19886 SDValue ExtOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, ExtVecVT, in InsertBitToMaskVector()
19887 DAG.getNode(ISD::SIGN_EXTEND, dl, ExtVecVT, Vec), in InsertBitToMaskVector()
19888 DAG.getNode(ISD::SIGN_EXTEND, dl, ExtEltVT, Elt), Idx); in InsertBitToMaskVector()
19889 return DAG.getNode(ISD::TRUNCATE, dl, VecVT, ExtOp); in InsertBitToMaskVector()
19893 SDValue EltInVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v1i1, Elt); in InsertBitToMaskVector()
19894 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, VecVT, Vec, EltInVec, Idx); in InsertBitToMaskVector()
19959 return DAG.getNode(ISD::OR, dl, VT, N0, CstVector); in LowerINSERT_VECTOR_ELT()
19985 SDValue N1Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, N1); in LowerINSERT_VECTOR_ELT()
19986 return DAG.getNode(X86ISD::BLENDI, dl, VT, N0, N1Vec, in LowerINSERT_VECTOR_ELT()
20016 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, V.getValueType(), V, N1, in LowerINSERT_VECTOR_ELT()
20025 if (IdxVal == 0 && ISD::isBuildVectorAllZeros(N0.getNode())) { in LowerINSERT_VECTOR_ELT()
20028 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, N1); in LowerINSERT_VECTOR_ELT()
20035 N1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, N1); in LowerINSERT_VECTOR_ELT()
20037 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, ShufVT, N1); in LowerINSERT_VECTOR_ELT()
20057 N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1); in LowerINSERT_VECTOR_ELT()
20059 return DAG.getNode(Opc, dl, VT, N0, N1, N2); in LowerINSERT_VECTOR_ELT()
20082 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1); in LowerINSERT_VECTOR_ELT()
20083 return DAG.getNode(X86ISD::BLENDI, dl, VT, N0, N1, in LowerINSERT_VECTOR_ELT()
20087 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1); in LowerINSERT_VECTOR_ELT()
20088 return DAG.getNode(X86ISD::INSERTPS, dl, VT, N0, N1, in LowerINSERT_VECTOR_ELT()
20118 Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT128, Op.getOperand(0)); in LowerSCALAR_TO_VECTOR()
20131 SDValue AnyExt = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Op.getOperand(0)); in LowerSCALAR_TO_VECTOR()
20133 OpVT, DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, AnyExt)); in LowerSCALAR_TO_VECTOR()
20165 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVecVT, in LowerEXTRACT_SUBVECTOR()
20171 Vec = DAG.getNode(X86ISD::KSHIFTR, dl, WideVecVT, Vec, in LowerEXTRACT_SUBVECTOR()
20174 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, Op.getValueType(), Vec, in LowerEXTRACT_SUBVECTOR()
20215 Result = DAG.getNode(getGlobalWrapperKind(), DL, PtrVT, Result); in LowerConstantPool()
20219 DAG.getNode(ISD::ADD, DL, PtrVT, in LowerConstantPool()
20220 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), Result); in LowerConstantPool()
20236 Result = DAG.getNode(getGlobalWrapperKind(), DL, PtrVT, Result); in LowerJumpTable()
20241 DAG.getNode(ISD::ADD, DL, PtrVT, in LowerJumpTable()
20242 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), Result); in LowerJumpTable()
20262 Result = DAG.getNode(getGlobalWrapperKind(), dl, PtrVT, Result); in LowerBlockAddress()
20266 Result = DAG.getNode(ISD::ADD, dl, PtrVT, in LowerBlockAddress()
20267 DAG.getNode(X86ISD::GlobalBaseReg, dl, PtrVT), Result); in LowerBlockAddress()
20326 Result = DAG.getNode(getGlobalWrapperKind(GV, OpFlags), dl, PtrVT, Result); in LowerGlobalOrExternal()
20330 Result = DAG.getNode(ISD::ADD, dl, PtrVT, in LowerGlobalOrExternal()
20331 DAG.getNode(X86ISD::GlobalBaseReg, dl, PtrVT), Result); in LowerGlobalOrExternal()
20343 Result = DAG.getNode(ISD::ADD, dl, PtrVT, Result, in LowerGlobalOrExternal()
20371 Chain = DAG.getNode(CallType, dl, NodeTys, Ops); in GetTLSADDR()
20374 Chain = DAG.getNode(CallType, dl, NodeTys, Ops); in GetTLSADDR()
20392 DAG.getNode(X86ISD::GlobalBaseReg, in LowerToTLSGeneralDynamicModel32()
20433 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), InFlag); in LowerToTLSLocalDynamicModel()
20448 SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA); in LowerToTLSLocalDynamicModel()
20451 return DAG.getNode(ISD::ADD, dl, PtrVT, Offset, Base); in LowerToTLSLocalDynamicModel()
20491 SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA); in LowerToTLSExecModel()
20495 Offset = DAG.getNode(ISD::ADD, dl, PtrVT, in LowerToTLSExecModel()
20496 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), in LowerToTLSExecModel()
20506 return DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, Offset); in LowerToTLSExecModel()
20559 SDValue Offset = DAG.getNode(WrapperKind, DL, PtrVT, Result); in LowerGlobalTLSAddress()
20563 Offset = DAG.getNode(ISD::ADD, DL, PtrVT, in LowerGlobalTLSAddress()
20564 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), in LowerGlobalTLSAddress()
20573 Chain = DAG.getNode(X86ISD::TLSCALL, DL, NodeTys, Args); in LowerGlobalTLSAddress()
20636 IDX = DAG.getNode(ISD::SHL, dl, PtrVT, IDX, Scale); in LowerGlobalTLSAddress()
20638 res = DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, IDX); in LowerGlobalTLSAddress()
20647 SDValue Offset = DAG.getNode(X86ISD::Wrapper, dl, PtrVT, TGA); in LowerGlobalTLSAddress()
20651 return DAG.getNode(ISD::ADD, dl, PtrVT, res, Offset); in LowerGlobalTLSAddress()
20662 DAG.getTargetLoweringInfo().expandShiftParts(Op.getNode(), Lo, Hi, DAG); in LowerShiftParts()
20693 SDValue InVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecInVT, Src); in LowerI64IntToFP_AVX512DQ()
20695 SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, {VecVT, MVT::Other}, in LowerI64IntToFP_AVX512DQ()
20698 SDValue Value = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec, in LowerI64IntToFP_AVX512DQ()
20703 SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, VecVT, InVec); in LowerI64IntToFP_AVX512DQ()
20705 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec, in LowerI64IntToFP_AVX512DQ()
20730 SDValue InVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Src); in LowerI64IntToFP16()
20732 SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, {MVT::v2f16, MVT::Other}, in LowerI64IntToFP16()
20735 SDValue Value = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec, in LowerI64IntToFP16()
20740 SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, MVT::v2f16, InVec); in LowerI64IntToFP16()
20742 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec, in LowerI64IntToFP16()
20805 SDValue VCast = DAG.getNode(Cast.getOpcode(), DL, ToVT, VecOp); in vectorizeExtractedCast()
20806 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, DestVT, VCast, in vectorizeExtractedCast()
20854 SDValue VecX = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecSrcVT, X); in lowerFPToIntToFP()
20855 SDValue VCastToInt = DAG.getNode(ToIntOpcode, DL, VecIntVT, VecX); in lowerFPToIntToFP()
20856 SDValue VCastToFP = DAG.getNode(ToFPOpcode, DL, VecVT, VCastToInt); in lowerFPToIntToFP()
20857 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, VCastToFP, ZeroIdx); in lowerFPToIntToFP()
20883 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v8i64, Tmp, Src, in lowerINT_TO_FP_vXi64()
20887 Res = DAG.getNode(Op.getOpcode(), DL, {WideVT, MVT::Other}, in lowerINT_TO_FP_vXi64()
20891 Res = DAG.getNode(Op.getOpcode(), DL, WideVT, Src); in lowerINT_TO_FP_vXi64()
20894 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res, in lowerINT_TO_FP_vXi64()
20909 SDValue Sign = DAG.getNode(ISD::OR, DL, MVT::v4i64, in lowerINT_TO_FP_vXi64()
20910 DAG.getNode(ISD::SRL, DL, MVT::v4i64, Src, One), in lowerINT_TO_FP_vXi64()
20911 DAG.getNode(ISD::AND, DL, MVT::v4i64, Src, One)); in lowerINT_TO_FP_vXi64()
20917 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i64, SignSrc, in lowerINT_TO_FP_vXi64()
20921 DAG.getNode(ISD::STRICT_SINT_TO_FP, DL, {MVT::f32, MVT::Other}, in lowerINT_TO_FP_vXi64()
20925 SignCvts[i] = DAG.getNode(ISD::SINT_TO_FP, DL, MVT::f32, Elt); in lowerINT_TO_FP_vXi64()
20932 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); in lowerINT_TO_FP_vXi64()
20933 Slow = DAG.getNode(ISD::STRICT_FADD, DL, {MVT::v4f32, MVT::Other}, in lowerINT_TO_FP_vXi64()
20937 Slow = DAG.getNode(ISD::FADD, DL, MVT::v4f32, SignCvt, SignCvt); in lowerINT_TO_FP_vXi64()
20940 IsNeg = DAG.getNode(ISD::TRUNCATE, DL, MVT::v4i32, IsNeg); in lowerINT_TO_FP_vXi64()
20959 return DAG.getNode( in promoteXINT_TO_FP()
20962 DAG.getNode(Op.getOpcode(), dl, {NVT, MVT::Other}, {Chain, Src}), in promoteXINT_TO_FP()
20964 return DAG.getNode(ISD::FP_ROUND, dl, VT, in promoteXINT_TO_FP()
20965 DAG.getNode(Op.getOpcode(), dl, NVT, Src), Rnd); in promoteXINT_TO_FP()
21017 return DAG.getNode( in LowerSINT_TO_FP()
21019 {Chain, DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src, in LowerSINT_TO_FP()
21021 return DAG.getNode(X86ISD::CVTSI2P, dl, VT, in LowerSINT_TO_FP()
21022 DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src, in LowerSINT_TO_FP()
21050 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i32, Src); in LowerSINT_TO_FP()
21052 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other}, in LowerSINT_TO_FP()
21055 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, Ext); in LowerSINT_TO_FP()
21178 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Op.getOperand(0)); in LowerUINT_TO_FP_i64()
21190 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, XR2F, CLod1); in LowerUINT_TO_FP_i64()
21195 Result = DAG.getNode(X86ISD::FHADD, dl, MVT::v2f64, Sub, Sub); in LowerUINT_TO_FP_i64()
21198 Result = DAG.getNode(ISD::FADD, dl, MVT::v2f64, Shuffle, Sub); in LowerUINT_TO_FP_i64()
21200 Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Result, in LowerUINT_TO_FP_i64()
21208 unsigned OpNo = Op.getNode()->isStrictFPOpcode() ? 1 : 0; in LowerUINT_TO_FP_i32()
21216 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, Op.getOperand(OpNo)); in LowerUINT_TO_FP_i32()
21222 SDValue Or = DAG.getNode( in LowerUINT_TO_FP_i32()
21226 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f64, Bias))); in LowerUINT_TO_FP_i32()
21228 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, in LowerUINT_TO_FP_i32()
21231 if (Op.getNode()->isStrictFPOpcode()) { in LowerUINT_TO_FP_i32()
21235 SDValue Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::f64, MVT::Other}, in LowerUINT_TO_FP_i32()
21250 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Or, Bias); in LowerUINT_TO_FP_i32()
21273 N0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i32, N0, in lowerUINT_TO_FP_v2i32()
21275 SDValue Res = DAG.getNode(Op->getOpcode(), DL, {MVT::v4f64, MVT::Other}, in lowerUINT_TO_FP_v2i32()
21278 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2f64, Res, in lowerUINT_TO_FP_v2i32()
21284 N0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i32, N0, in lowerUINT_TO_FP_v2i32()
21287 return DAG.getNode(X86ISD::STRICT_CVTUI2P, DL, {MVT::v2f64, MVT::Other}, in lowerUINT_TO_FP_v2i32()
21289 return DAG.getNode(X86ISD::CVTUI2P, DL, MVT::v2f64, N0); in lowerUINT_TO_FP_v2i32()
21296 SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v2i64, N0); in lowerUINT_TO_FP_v2i32()
21299 SDValue Or = DAG.getNode(ISD::OR, DL, MVT::v2i64, ZExtIn, in lowerUINT_TO_FP_v2i32()
21304 return DAG.getNode(ISD::STRICT_FSUB, DL, {MVT::v2f64, MVT::Other}, in lowerUINT_TO_FP_v2i32()
21306 return DAG.getNode(ISD::FSUB, DL, MVT::v2f64, Or, VBias); in lowerUINT_TO_FP_v2i32()
21335 V = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideIntVT, Tmp, V, in lowerUINT_TO_FP_vXi32()
21339 Res = DAG.getNode(ISD::STRICT_UINT_TO_FP, DL, {WideVT, MVT::Other}, in lowerUINT_TO_FP_vXi32()
21343 Res = DAG.getNode(ISD::UINT_TO_FP, DL, WideVT, V); in lowerUINT_TO_FP_vXi32()
21346 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res, in lowerUINT_TO_FP_vXi32()
21356 SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v4i64, V); in lowerUINT_TO_FP_vXi32()
21369 SDValue Or = DAG.getNode(ISD::OR, DL, MVT::v4i64, ZExtIn, in lowerUINT_TO_FP_vXi32()
21374 return DAG.getNode(ISD::STRICT_FSUB, DL, {MVT::v4f64, MVT::Other}, in lowerUINT_TO_FP_vXi32()
21376 return DAG.getNode(ISD::FSUB, DL, MVT::v4f64, Or, VBias); in lowerUINT_TO_FP_vXi32()
21412 SDValue HighShift = DAG.getNode(ISD::SRL, DL, VecIntVT, V, VecCstShift); in lowerUINT_TO_FP_vXi32()
21422 Low = DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecBitcast, in lowerUINT_TO_FP_vXi32()
21430 High = DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecShiftBitcast, in lowerUINT_TO_FP_vXi32()
21435 SDValue LowAnd = DAG.getNode(ISD::AND, DL, VecIntVT, V, VecCstMask); in lowerUINT_TO_FP_vXi32()
21436 Low = DAG.getNode(ISD::OR, DL, VecIntVT, LowAnd, VecCstLow); in lowerUINT_TO_FP_vXi32()
21439 High = DAG.getNode(ISD::OR, DL, VecIntVT, HighShift, VecCstHigh); in lowerUINT_TO_FP_vXi32()
21456 SDValue FHigh = DAG.getNode(ISD::STRICT_FSUB, DL, {VecFloatVT, MVT::Other}, in lowerUINT_TO_FP_vXi32()
21458 return DAG.getNode(ISD::STRICT_FADD, DL, {VecFloatVT, MVT::Other}, in lowerUINT_TO_FP_vXi32()
21463 DAG.getNode(ISD::FSUB, DL, VecFloatVT, HighBitcast, VecCstFSub); in lowerUINT_TO_FP_vXi32()
21464 return DAG.getNode(ISD::FADD, DL, VecFloatVT, LowBitcast, FHigh); in lowerUINT_TO_FP_vXi32()
21469 unsigned OpNo = Op.getNode()->isStrictFPOpcode() ? 1 : 0; in lowerUINT_TO_FP_vec()
21526 Src = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, Src); in LowerUINT_TO_FP()
21528 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {DstVT, MVT::Other}, in LowerUINT_TO_FP()
21530 return DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Src); in LowerUINT_TO_FP()
21608 FudgePtr = DAG.getNode(ISD::ADD, dl, PtrVT, FudgePtr, Offset); in LowerUINT_TO_FP()
21619 SDValue Add = DAG.getNode(ISD::STRICT_FADD, dl, {MVT::f80, MVT::Other}, in LowerUINT_TO_FP()
21624 return DAG.getNode(ISD::STRICT_FP_ROUND, dl, {DstVT, MVT::Other}, in LowerUINT_TO_FP()
21627 SDValue Add = DAG.getNode(ISD::FADD, dl, MVT::f80, Fild, Fudge); in LowerUINT_TO_FP()
21628 return DAG.getNode(ISD::FP_ROUND, dl, DstVT, Add, in LowerUINT_TO_FP()
21741 SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Cmp); in FP_TO_INTHelper()
21743 Adjust = DAG.getNode(ISD::SHL, DL, MVT::i64, Zext, Const63); in FP_TO_INTHelper()
21749 Value = DAG.getNode(ISD::STRICT_FSUB, DL, { TheVT, MVT::Other}, in FP_TO_INTHelper()
21753 Value = DAG.getNode(ISD::FSUB, DL, TheVT, Value, FltOfs); in FP_TO_INTHelper()
21787 Res = DAG.getNode(ISD::XOR, DL, MVT::i64, Res, Adjust); in FP_TO_INTHelper()
21837 SDValue OpLo = DAG.getNode(ExtendInVecOpc, dl, HalfVT, In); in LowerAVXExtend()
21843 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpLo); in LowerAVXExtend()
21851 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi); in LowerAVXExtend()
21858 SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v8i1, In, in SplitAndExtendv16i1()
21860 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v8i1, In, in SplitAndExtendv16i1()
21862 Lo = DAG.getNode(ExtOpc, dl, MVT::v8i16, Lo); in SplitAndExtendv16i1()
21863 Hi = DAG.getNode(ExtOpc, dl, MVT::v8i16, Hi); in SplitAndExtendv16i1()
21864 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i16, Lo, Hi); in SplitAndExtendv16i1()
21865 return DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in SplitAndExtendv16i1()
21881 SDValue Extend = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, In); in LowerZERO_EXTEND_Mask()
21882 return DAG.getNode(ISD::SRL, DL, VT, Extend, in LowerZERO_EXTEND_Mask()
21901 In = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, InVT, DAG.getUNDEF(InVT), in LowerZERO_EXTEND_Mask()
21915 SelectedVal = DAG.getNode(ISD::TRUNCATE, DL, WideVT, SelectedVal); in LowerZERO_EXTEND_Mask()
21920 SelectedVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, SelectedVal, in LowerZERO_EXTEND_Mask()
21991 SDValue Res = DAG.getNode(Opcode, DL, OutVT, In, DAG.getUNDEF(InVT)); in truncateVectorWithPACK()
22008 SDValue Res = DAG.getNode(Opcode, DL, OutVT, Lo, Hi); in truncateVectorWithPACK()
22017 SDValue Res = DAG.getNode(Opcode, DL, OutVT, Lo, Hi); in truncateVectorWithPACK()
22043 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, PackedVT, Lo, Hi); in truncateVectorWithPACK()
22066 In = DAG.getNode(ISD::SHL, DL, ExtVT, in LowerTruncateVecI1()
22091 Lo = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, MVT::v8i32, In); in LowerTruncateVecI1()
22095 Hi = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, MVT::v8i32, Hi); in LowerTruncateVecI1()
22103 Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i1, Lo); in LowerTruncateVecI1()
22104 Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i1, Hi); in LowerTruncateVecI1()
22105 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi); in LowerTruncateVecI1()
22112 In = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, In); in LowerTruncateVecI1()
22119 In = DAG.getNode(ISD::SHL, DL, InVT, In, in LowerTruncateVecI1()
22154 Lo = DAG.getNode(ISD::TRUNCATE, DL, LoVT, Lo); in LowerTRUNCATE()
22155 Hi = DAG.getNode(ISD::TRUNCATE, DL, HiVT, Hi); in LowerTRUNCATE()
22156 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi); in LowerTRUNCATE()
22210 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, In, in LowerTRUNCATE()
22214 SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In, in LowerTRUNCATE()
22216 SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In, in LowerTRUNCATE()
22237 In = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In, in LowerTRUNCATE()
22242 SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In, in LowerTRUNCATE()
22244 SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In, in LowerTRUNCATE()
22267 In = DAG.getNode(ISD::AND, DL, InVT, In, DAG.getConstant(255, DL, InVT)); in LowerTRUNCATE()
22269 SDValue InLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i16, In, in LowerTRUNCATE()
22271 SDValue InHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i16, In, in LowerTRUNCATE()
22273 return DAG.getNode(X86ISD::PACKUS, DL, VT, InLo, InHi); in LowerTRUNCATE()
22290 SDValue Small = DAG.getNode(X86ISD::CVTTP2SI, dl, VT, Src); in expandFP_TO_UINT_SSE()
22292 DAG.getNode(X86ISD::CVTTP2SI, dl, VT, in expandFP_TO_UINT_SSE()
22293 DAG.getNode(ISD::FSUB, dl, SrcVT, Src, in expandFP_TO_UINT_SSE()
22306 SDValue Overflow = DAG.getNode(ISD::OR, dl, VT, Small, Big); in expandFP_TO_UINT_SSE()
22307 return DAG.getNode(X86ISD::BLENDV, dl, VT, Small, Overflow, Small); in expandFP_TO_UINT_SSE()
22311 DAG.getNode(X86ISD::VSRAI, dl, VT, Small, in expandFP_TO_UINT_SSE()
22313 return DAG.getNode(ISD::OR, dl, VT, Small, in expandFP_TO_UINT_SSE()
22314 DAG.getNode(ISD::AND, dl, VT, Big, IsOverflown)); in expandFP_TO_UINT_SSE()
22331 return DAG.getNode(Op.getOpcode(), dl, {VT, MVT::Other}, in LowerFP_TO_INT()
22332 {Chain, DAG.getNode(ISD::STRICT_FP_EXTEND, dl, in LowerFP_TO_INT()
22334 return DAG.getNode(Op.getOpcode(), dl, VT, in LowerFP_TO_INT()
22335 DAG.getNode(ISD::FP_EXTEND, dl, NVT, Src)); in LowerFP_TO_INT()
22361 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8f64, Tmp, Src, in LowerFP_TO_INT()
22365 Res = DAG.getNode(Opc, dl, {ResVT, MVT::Other}, {Chain, Src}); in LowerFP_TO_INT()
22368 Res = DAG.getNode(Opc, dl, ResVT, Src); in LowerFP_TO_INT()
22371 Res = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, Res); in LowerFP_TO_INT()
22372 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i1, Res, in LowerFP_TO_INT()
22393 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8f16, Ops); in LowerFP_TO_INT()
22397 Res = DAG.getNode(IsSigned ? X86ISD::STRICT_CVTTP2SI in LowerFP_TO_INT()
22402 Res = DAG.getNode(IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI, dl, in LowerFP_TO_INT()
22409 Res = DAG.getNode(ISD::TRUNCATE, dl, ResVT, Res); in LowerFP_TO_INT()
22413 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res, in LowerFP_TO_INT()
22423 Res = DAG.getNode(IsSigned ? ISD::STRICT_FP_TO_SINT in LowerFP_TO_INT()
22428 Res = DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, dl, in LowerFP_TO_INT()
22433 Res = DAG.getNode(ISD::TRUNCATE, dl, MVT::v8i16, Res); in LowerFP_TO_INT()
22460 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVT, Tmp, Src, in LowerFP_TO_INT()
22464 Res = DAG.getNode(ISD::STRICT_FP_TO_UINT, dl, {ResVT, MVT::Other}, in LowerFP_TO_INT()
22468 Res = DAG.getNode(ISD::FP_TO_UINT, dl, ResVT, Src); in LowerFP_TO_INT()
22471 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res, in LowerFP_TO_INT()
22490 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVT, Tmp, Src, in LowerFP_TO_INT()
22494 Res = DAG.getNode(Op.getOpcode(), dl, {MVT::v8i64, MVT::Other}, in LowerFP_TO_INT()
22498 Res = DAG.getNode(Op.getOpcode(), dl, MVT::v8i64, Src); in LowerFP_TO_INT()
22501 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res, in LowerFP_TO_INT()
22517 SDValue Tmp = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8f32, in LowerFP_TO_INT()
22519 Tmp = DAG.getNode(Op.getOpcode(), dl, {MVT::v8i64, MVT::Other}, in LowerFP_TO_INT()
22522 Tmp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i64, Tmp, in LowerFP_TO_INT()
22528 SDValue Tmp = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src, in LowerFP_TO_INT()
22533 return DAG.getNode(Opc, dl, {VT, MVT::Other}, {Op->getOperand(0), Tmp}); in LowerFP_TO_INT()
22536 return DAG.getNode(Opc, dl, VT, Tmp); in LowerFP_TO_INT()
22566 SDValue FloatOffset = DAG.getNode(ISD::UINT_TO_FP, dl, SrcVT, in LowerFP_TO_INT()
22574 DAG.getNode(X86ISD::CVTTS2SI, dl, VT, in LowerFP_TO_INT()
22575 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, SrcVecVT, Src)); in LowerFP_TO_INT()
22576 SDValue Big = DAG.getNode( in LowerFP_TO_INT()
22578 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, SrcVecVT, in LowerFP_TO_INT()
22579 DAG.getNode(ISD::FSUB, dl, SrcVT, Src, FloatOffset))); in LowerFP_TO_INT()
22587 SDValue IsOverflown = DAG.getNode( in LowerFP_TO_INT()
22589 return DAG.getNode(ISD::OR, dl, VT, Small, in LowerFP_TO_INT()
22590 DAG.getNode(ISD::AND, dl, VT, Big, IsOverflown)); in LowerFP_TO_INT()
22604 Res = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, {MVT::i64, MVT::Other}, in LowerFP_TO_INT()
22608 Res = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i64, Src); in LowerFP_TO_INT()
22610 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in LowerFP_TO_INT()
22628 Res = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, {MVT::i32, MVT::Other}, in LowerFP_TO_INT()
22632 Res = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Src); in LowerFP_TO_INT()
22634 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in LowerFP_TO_INT()
22684 return LRINT_LLRINTHelper(Op.getNode(), DAG); in LowerLRINT_LLRINT()
22708 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); in LRINT_LLRINTHelper()
22735 SDNode *Node = Op.getNode(); in LowerFP_TO_INT_SAT()
22808 SDValue MinClamped = DAG.getNode( in LowerFP_TO_INT_SAT()
22811 SDValue BothClamped = DAG.getNode( in LowerFP_TO_INT_SAT()
22814 SDValue FpToInt = DAG.getNode(FpToIntOpcode, dl, TmpVT, BothClamped); in LowerFP_TO_INT_SAT()
22818 return DAG.getNode(ISD::TRUNCATE, dl, DstVT, FpToInt); in LowerFP_TO_INT_SAT()
22822 SDValue MinClamped = DAG.getNode( in LowerFP_TO_INT_SAT()
22825 SDValue BothClamped = DAG.getNode( in LowerFP_TO_INT_SAT()
22828 SDValue FpToInt = DAG.getNode(FpToIntOpcode, dl, DstVT, BothClamped); in LowerFP_TO_INT_SAT()
22846 SDValue FpToInt = DAG.getNode(FpToIntOpcode, dl, TmpVT, Src); in LowerFP_TO_INT_SAT()
22851 FpToInt = DAG.getNode(ISD::TRUNCATE, dl, DstVT, FpToInt); in LowerFP_TO_INT_SAT()
22899 return DAG.getNode( in LowerFP_EXTEND()
22901 {Chain, DAG.getNode(ISD::STRICT_FP_EXTEND, DL, in LowerFP_EXTEND()
22904 return DAG.getNode(ISD::FP_EXTEND, DL, VT, in LowerFP_EXTEND()
22905 DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, In)); in LowerFP_EXTEND()
22912 In = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v8i16, in LowerFP_EXTEND()
22917 Res = DAG.getNode(X86ISD::STRICT_CVTPH2PS, DL, {MVT::v4f32, MVT::Other}, in LowerFP_EXTEND()
22921 Res = DAG.getNode(X86ISD::CVTPH2PS, DL, MVT::v4f32, In, in LowerFP_EXTEND()
22924 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Res, in LowerFP_EXTEND()
22937 In = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4f16, In, in LowerFP_EXTEND()
22939 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8f16, In, in LowerFP_EXTEND()
22942 return DAG.getNode(X86ISD::STRICT_VFPEXT, DL, {VT, MVT::Other}, in LowerFP_EXTEND()
22944 return DAG.getNode(X86ISD::VFPEXT, DL, VT, Res); in LowerFP_EXTEND()
22952 DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4f32, In, DAG.getUNDEF(SVT)); in LowerFP_EXTEND()
22954 return DAG.getNode(X86ISD::STRICT_VFPEXT, DL, {VT, MVT::Other}, in LowerFP_EXTEND()
22956 return DAG.getNode(X86ISD::VFPEXT, DL, VT, Res); in LowerFP_EXTEND()
22982 Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v4f32, in LowerFP_ROUND()
22985 Res = DAG.getNode(X86ISD::STRICT_CVTPS2PH, DL, {MVT::v8i16, MVT::Other}, in LowerFP_ROUND()
22990 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4f32, In); in LowerFP_ROUND()
22991 Res = DAG.getNode(X86ISD::CVTPS2PH, DL, MVT::v8i16, Res, Rnd); in LowerFP_ROUND()
22994 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i16, Res, in LowerFP_ROUND()
23014 SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, in LowerFP16_TO_FP()
23020 Res = DAG.getNode(X86ISD::STRICT_CVTPH2PS, dl, {MVT::v4f32, MVT::Other}, in LowerFP16_TO_FP()
23024 Res = DAG.getNode(X86ISD::CVTPH2PS, dl, MVT::v4f32, Res); in LowerFP16_TO_FP()
23027 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, Res, in LowerFP16_TO_FP()
23045 Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v4f32, in LowerFP_TO_FP16()
23048 Res = DAG.getNode( in LowerFP_TO_FP16()
23054 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, Src); in LowerFP_TO_FP16()
23055 Res = DAG.getNode(X86ISD::CVTPS2PH, dl, MVT::v8i16, Res, in LowerFP_TO_FP16()
23059 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Res, in LowerFP_TO_FP16()
23076 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, in LowerFP_TO_BF16()
23147 SDValue HOp = DAG.getNode(HOpcode, DL, X.getValueType(), X, X); in lowerAddSubToHorizontalOp()
23148 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, Op.getSimpleValueType(), HOp, in lowerAddSubToHorizontalOp()
23176 SDValue Adder = DAG.getNode(ISD::FCOPYSIGN, dl, VT, in LowerFROUND()
23178 N0 = DAG.getNode(ISD::FADD, dl, VT, N0, Adder); in LowerFROUND()
23181 return DAG.getNode(ISD::FTRUNC, dl, VT, N0); in LowerFROUND()
23237 return DAG.getNode(LogicOp, dl, LogicVT, Operand, Mask); in LowerFABSorFNEG()
23241 Operand = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LogicVT, Operand); in LowerFABSorFNEG()
23242 SDValue LogicNode = DAG.getNode(LogicOp, dl, LogicVT, Operand, Mask); in LowerFABSorFNEG()
23243 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, LogicNode, in LowerFABSorFNEG()
23255 Sign = DAG.getNode(ISD::FP_EXTEND, dl, VT, Sign); in LowerFCOPYSIGN()
23260 DAG.getNode(ISD::FP_ROUND, dl, VT, Sign, DAG.getIntPtrConstant(0, dl)); in LowerFCOPYSIGN()
23292 Sign = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LogicVT, Sign); in LowerFCOPYSIGN()
23293 SDValue SignBit = DAG.getNode(X86ISD::FAND, dl, LogicVT, Sign, SignMask); in LowerFCOPYSIGN()
23306 Mag = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LogicVT, Mag); in LowerFCOPYSIGN()
23307 MagBits = DAG.getNode(X86ISD::FAND, dl, LogicVT, Mag, MagMask); in LowerFCOPYSIGN()
23311 SDValue Or = DAG.getNode(X86ISD::FOR, dl, LogicVT, MagBits, SignBit); in LowerFCOPYSIGN()
23312 return !IsFakeVector ? Or : DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Or, in LowerFCOPYSIGN()
23327 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, N0); in LowerFGETSIGN()
23328 Res = DAG.getNode(X86ISD::MOVMSK, dl, MVT::i32, Res); in LowerFGETSIGN()
23330 Res = DAG.getNode(ISD::AND, dl, VT, Res, DAG.getConstant(1, dl, VT)); in LowerFGETSIGN()
23342 Src = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Src); in getBT()
23354 Src = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Src); in getBT()
23363 BitNo = DAG.getNode(ISD::AND, DL, Src.getValueType(), in getBT()
23364 DAG.getNode(ISD::ANY_EXTEND, DL, Src.getValueType(), in getBT()
23366 DAG.getNode(ISD::ANY_EXTEND, DL, Src.getValueType(), in getBT()
23369 BitNo = DAG.getNode(ISD::ANY_EXTEND, DL, Src.getValueType(), BitNo); in getBT()
23372 return DAG.getNode(X86ISD::BT, DL, MVT::i32, Src, BitNo); in getBT()
23378 return DAG.getNode(X86ISD::SETCC, dl, MVT::i8, in getSETCC()
23474 return DAG.getNode(ISD::AND, DL, SrcVT, Src, MaskValue); in LowerVectorAllZero()
23482 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, in LowerVectorAllZero()
23496 V = DAG.getNode(ISD::OR, DL, VT, Split.first, Split.second); in LowerVectorAllZero()
23503 return DAG.getNode(X86ISD::PTEST, DL, MVT::i32, V, V); in LowerVectorAllZero()
23512 V = DAG.getNode(X86ISD::PCMPEQ, DL, MVT::v16i8, V, in LowerVectorAllZero()
23514 V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V); in LowerVectorAllZero()
23515 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, V, in LowerVectorAllZero()
23568 VecIns.push_back(DAG.getNode(ISD::OR, DL, VT, LHS, RHS)); in MatchVectorAllZeroTest()
23582 DAG.matchBinOpReduction(Op.getNode(), BinOp, {ISD::OR})) { in MatchVectorAllZeroTest()
23652 if (Op.getNode()->getFlags().hasNoSignedWrap()) in EmitTest()
23667 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op, in EmitTest()
23710 return SDValue(Op.getNode(), 1); in EmitTest()
23715 return DAG.getNode(X86ISD::SUB, dl, VTs, Op->getOperand(0), in EmitTest()
23724 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op, in EmitTest()
23730 SDValue New = DAG.getNode(Opcode, dl, VTs, Ops); in EmitTest()
23731 DAG.ReplaceAllUsesOfValueWith(SDValue(Op.getNode(), 0), New); in EmitTest()
23732 return SDValue(New.getNode(), 1); in EmitTest()
23772 Op0 = DAG.getNode(ExtendOp, dl, CmpVT, Op0); in EmitCmp()
23773 Op1 = DAG.getNode(ExtendOp, dl, CmpVT, Op1); in EmitCmp()
23784 Op0 = DAG.getNode(ISD::TRUNCATE, dl, CmpVT, Op0); in EmitCmp()
23785 Op1 = DAG.getNode(ISD::TRUNCATE, dl, CmpVT, Op1); in EmitCmp()
23793 SDValue Add = DAG.getNode(X86ISD::ADD, dl, VTs, Op0.getOperand(1), Op1); in EmitCmp()
23802 SDValue Add = DAG.getNode(X86ISD::ADD, dl, VTs, Op0, Op1.getOperand(1)); in EmitCmp()
23808 SDValue Sub = DAG.getNode(X86ISD::SUB, dl, VTs, Op0, Op1); in EmitCmp()
23858 SDValue Estimate = DAG.getNode(Opcode, DL, VT, Op); in getSqrtEstimate()
23860 Estimate = DAG.getNode(ISD::FMUL, DL, VT, Op, Estimate); in getSqrtEstimate()
23873 Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v8f16, Op); in getSqrtEstimate()
23874 Op = DAG.getNode(X86ISD::RSQRT14S, DL, MVT::v8f16, Undef, Op); in getSqrtEstimate()
23875 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f16, Op, Zero); in getSqrtEstimate()
23878 return DAG.getNode(X86ISD::RSQRT14, DL, VT, Op); in getSqrtEstimate()
23913 return DAG.getNode(Opcode, DL, VT, Op); in getRecipEstimate()
23924 Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v8f16, Op); in getRecipEstimate()
23925 Op = DAG.getNode(X86ISD::RCP14S, DL, MVT::v8f16, Undef, Op); in getRecipEstimate()
23926 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f16, Op, Zero); in getRecipEstimate()
23929 return DAG.getNode(X86ISD::RCP14, DL, VT, Op); in getRecipEstimate()
23981 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne); in BuildSDIVPow2()
23982 SDValue CMov = DAG.getNode(ISD::SELECT, DL, VT, Cmp, Add, N0); in BuildSDIVPow2()
23984 Created.push_back(Cmp.getNode()); in BuildSDIVPow2()
23985 Created.push_back(Add.getNode()); in BuildSDIVPow2()
23986 Created.push_back(CMov.getNode()); in BuildSDIVPow2()
23990 DAG.getNode(ISD::SRA, DL, VT, CMov, DAG.getConstant(Lg2, DL, MVT::i8)); in BuildSDIVPow2()
23997 Created.push_back(SRA.getNode()); in BuildSDIVPow2()
23998 return DAG.getNode(ISD::SUB, DL, VT, Zero, SRA); in BuildSDIVPow2()
24052 if (!Src.getNode()) in LowerAndToBT()
24157 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, in splitIntVSETCC()
24158 DAG.getNode(ISD::SETCC, dl, LoVT, LHS1, RHS1, CC), in splitIntVSETCC()
24159 DAG.getNode(ISD::SETCC, dl, HiVT, LHS2, RHS2, CC)); in splitIntVSETCC()
24189 auto *BV = dyn_cast<BuildVectorSDNode>(V.getNode()); in incDecVectorConstant()
24266 SDValue Result = DAG.getNode(ISD::USUBSAT, dl, VT, Op0, Op1); in LowerVSETCCWithSUBUS()
24267 return DAG.getNode(X86ISD::PCMPEQ, dl, VT, Result, in LowerVSETCCWithSUBUS()
24329 SDValue SignalCmp = DAG.getNode( in LowerVSETCC()
24359 Cmp0 = DAG.getNode( in LowerVSETCC()
24362 Cmp1 = DAG.getNode( in LowerVSETCC()
24365 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Cmp0.getValue(1), in LowerVSETCC()
24368 Cmp0 = DAG.getNode( in LowerVSETCC()
24370 Cmp1 = DAG.getNode( in LowerVSETCC()
24373 Cmp = DAG.getNode(CombineOpc, dl, VT, Cmp0, Cmp1); in LowerVSETCC()
24376 Cmp = DAG.getNode( in LowerVSETCC()
24381 Cmp = DAG.getNode( in LowerVSETCC()
24389 Cmp = DAG.getNode( in LowerVSETCC()
24394 Cmp = DAG.getNode( in LowerVSETCC()
24464 return DAG.getNode(Opc, dl, VT, Op0, Op1, in LowerVSETCC()
24470 if (Cond == ISD::SETNE && ISD::isBuildVectorAllZeros(Op1.getNode())) { in LowerVSETCC()
24495 Result = DAG.getNode(ISD::SHL, dl, VT, Result, in LowerVSETCC()
24497 Result = DAG.getNode(ISD::SRA, dl, VT, Result, in LowerVSETCC()
24519 ISD::isConstantSplatVector(Op1.getNode(), ConstValue)) { in LowerVSETCC()
24566 SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1); in LowerVSETCC()
24567 Result = DAG.getNode(X86ISD::PCMPEQ, dl, VT, Op0, Result); in LowerVSETCC()
24603 if (!FlipSigns && !Invert && ISD::isBuildVectorAllZeros(Op0.getNode())) { in LowerVSETCC()
24607 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1); in LowerVSETCC()
24614 if (!FlipSigns && !Invert && ISD::isBuildVectorAllOnes(Op1.getNode())) { in LowerVSETCC()
24618 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1); in LowerVSETCC()
24632 Op0 = DAG.getNode(ISD::XOR, dl, MVT::v2i64, Op0, SB); in LowerVSETCC()
24633 Op1 = DAG.getNode(ISD::XOR, dl, MVT::v2i64, Op1, SB); in LowerVSETCC()
24640 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1); in LowerVSETCC()
24641 SDValue EQ = DAG.getNode(X86ISD::PCMPEQ, dl, MVT::v4i32, Op0, Op1); in LowerVSETCC()
24650 SDValue Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, EQHi, GTLo); in LowerVSETCC()
24651 Result = DAG.getNode(ISD::OR, dl, MVT::v4i32, Result, GTHi); in LowerVSETCC()
24669 SDValue Result = DAG.getNode(Opc, dl, MVT::v4i32, Op0, Op1); in LowerVSETCC()
24674 Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, Result, Shuf); in LowerVSETCC()
24689 Op0 = DAG.getNode(ISD::XOR, dl, VT, Op0, SM); in LowerVSETCC()
24690 Op1 = DAG.getNode(ISD::XOR, dl, VT, Op1, SM); in LowerVSETCC()
24693 SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1); in LowerVSETCC()
24743 return DAG.getNode(X86ISD::KTEST, dl, MVT::i32, LHS, RHS); in EmitAVX512Test()
24755 return DAG.getNode(X86ISD::KORTEST, dl, MVT::i32, LHS, RHS); in EmitAVX512Test()
24815 SDValue New = DAG.getNode(X86ISD::ADD, dl, VTs, Op0.getOperand(0), in emitFlagsForSetcc()
24817 DAG.ReplaceAllUsesOfValueWith(SDValue(Op0.getNode(), 0), New); in emitFlagsForSetcc()
24820 return SDValue(New.getNode(), 1); in emitFlagsForSetcc()
24859 if (!Op1.getNode()) { in LowerSETCC()
24898 SDValue Res = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, X86CC, EFLAGS); in LowerSETCC()
24911 DAG.getNode(IsSignaling ? X86ISD::STRICT_FCMPS : X86ISD::STRICT_FCMP, in LowerSETCC()
24915 EFLAGS = DAG.getNode(X86ISD::FCMP, dl, MVT::i32, Op0, Op1); in LowerSETCC()
24919 SDValue Res = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, X86CC, EFLAGS); in LowerSETCC()
24935 Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32), in LowerSETCCCARRY()
24939 SDValue Cmp = DAG.getNode(X86ISD::SBB, DL, VTs, LHS, RHS, Carry.getValue(1)); in LowerSETCCCARRY()
24986 Value = DAG.getNode(BaseOp, DL, VTs, LHS, RHS); in getX86XALUOOp()
25005 return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), Value, SetCC); in LowerXALUO()
25044 return DAG.getBitcast(VT, DAG.getNode(ISD::SELECT, DL, NVT, Cond, in LowerSELECT()
25062 DAG.getNode(X86ISD::FSETCCM, DL, MVT::v1i1, CondOp0, CondOp1, in LowerSELECT()
25065 return DAG.getNode(X86ISD::SELECTS, DL, VT, Cmp, Op1, Op2); in LowerSELECT()
25069 SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, VT, CondOp0, CondOp1, in LowerSELECT()
25089 SDValue VOp1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Op1); in LowerSELECT()
25090 SDValue VOp2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Op2); in LowerSELECT()
25091 SDValue VCmp = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Cmp); in LowerSELECT()
25098 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, in LowerSELECT()
25101 SDValue AndN = DAG.getNode(X86ISD::FANDN, DL, VT, Cmp, Op2); in LowerSELECT()
25102 SDValue And = DAG.getNode(X86ISD::FAND, DL, VT, Cmp, Op1); in LowerSELECT()
25103 return DAG.getNode(X86ISD::FOR, DL, VT, AndN, And); in LowerSELECT()
25109 SDValue Cmp = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v1i1, Cond); in LowerSELECT()
25110 return DAG.getNode(X86ISD::SELECTS, DL, VT, Cmp, Op1, Op2); in LowerSELECT()
25168 Sub = DAG.getNode(X86ISD::SUB, DL, CmpVTs, Zero, CmpOp0); in LowerSELECT()
25171 Sub = DAG.getNode(X86ISD::SUB, DL, CmpVTs, CmpOp0, One); in LowerSELECT()
25173 SDValue SBB = DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, in LowerSELECT()
25176 return DAG.getNode(ISD::OR, DL, VT, SBB, Y); in LowerSELECT()
25201 Neg = DAG.getNode(ISD::TRUNCATE, DL, VT, CmpOp0); in LowerSELECT()
25203 Neg = DAG.getNode(ISD::AND, DL, VT, in LowerSELECT()
25204 DAG.getNode(ISD::ANY_EXTEND, DL, VT, CmpOp0.getOperand(0)), in LowerSELECT()
25208 SDValue Mask = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), in LowerSELECT()
25210 SDValue And = DAG.getNode(ISD::AND, DL, VT, Mask, Src1); // Mask & z in LowerSELECT()
25211 return DAG.getNode(Op2.getOpcode(), DL, VT, And, Src2); // And Op y in LowerSELECT()
25214 Cmp.getNode()->hasOneUse() && (CmpOp0 == Op1) && in LowerSELECT()
25225 SDValue Shift = DAG.getNode(ISD::SRA, DL, VT, Op1, ShiftAmt); in LowerSELECT()
25228 return DAG.getNode(ISD::AND, DL, VT, Shift, Op1); in LowerSELECT()
25300 DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(), in LowerSELECT()
25317 SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, T1.getValueType(), T2, T1, in LowerSELECT()
25319 return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Cmov); in LowerSELECT()
25332 Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op1); in LowerSELECT()
25333 Op2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op2); in LowerSELECT()
25335 SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, MVT::i32, Ops); in LowerSELECT()
25336 return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Cmov); in LowerSELECT()
25342 return DAG.getNode(X86ISD::CMOV, DL, Op.getValueType(), Ops); in LowerSELECT()
25372 In = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, InVT, DAG.getUNDEF(InVT), in LowerSIGN_EXTEND_Mask()
25381 V = DAG.getNode(Op.getOpcode(), dl, WideVT, In); in LowerSIGN_EXTEND_Mask()
25391 V = DAG.getNode(ISD::TRUNCATE, dl, WideVT, V); in LowerSIGN_EXTEND_Mask()
25396 V = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, V, in LowerSIGN_EXTEND_Mask()
25459 return DAG.getNode(Op.getOpcode(), dl, VT, In); in LowerEXTEND_VECTOR_INREG()
25466 return DAG.getNode(ExtOpc, dl, VT, In); in LowerEXTEND_VECTOR_INREG()
25480 SDValue Lo = DAG.getNode(Opc, dl, HalfVT, In); in LowerEXTEND_VECTOR_INREG()
25482 Hi = DAG.getNode(Opc, dl, HalfVT, Hi); in LowerEXTEND_VECTOR_INREG()
25483 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); in LowerEXTEND_VECTOR_INREG()
25515 SignExt = DAG.getNode(X86ISD::VSRAI, dl, DestVT, Curr, in LowerEXTEND_VECTOR_INREG()
25569 SDValue OpLo = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, HalfVT, In); in LowerSIGN_EXTEND()
25577 OpHi = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, HalfVT, OpHi); in LowerSIGN_EXTEND()
25579 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi); in LowerSIGN_EXTEND()
25612 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Ch0, Ch1); in splitVectorStore()
25640 SDValue Scl = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, StoreSVT, StoredVal, in scalarizeVectorStore()
25648 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Stores); in scalarizeVectorStore()
25653 StoreSDNode *St = cast<StoreSDNode>(Op.getNode()); in LowerStore()
25667 StoredVal = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v16i1, in LowerStore()
25671 StoredVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, StoredVal); in LowerStore()
25695 collectConcatOps(StoredVal.getNode(), CatOps, DAG)) in LowerStore()
25710 StoredVal = DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT, StoredVal, in LowerStore()
25719 StoredVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, StVT, StoredVal, in LowerStore()
25747 LoadSDNode *Ld = cast<LoadSDNode>(Op.getNode()); in LowerLoad()
25764 SDValue Val = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, NewLd); in LowerLoad()
25765 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, RegVT, in LowerLoad()
25814 return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, in LowerBRCOND()
25821 return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, in LowerBRCOND()
25831 if (Op.getNode()->hasOneUse()) { in LowerBRCOND()
25832 SDNode *User = *Op.getNode()->use_begin(); in LowerBRCOND()
25845 DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS); in LowerBRCOND()
25847 Chain = DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, in LowerBRCOND()
25850 return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, in LowerBRCOND()
25858 SDValue Cmp = DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS); in LowerBRCOND()
25861 DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, Cmp); in LowerBRCOND()
25863 return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, in LowerBRCOND()
25868 SDValue Cmp = DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS); in LowerBRCOND()
25870 return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, in LowerBRCOND()
25881 return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, in LowerBRCOND()
25894 DAG.getNode(ISD::AND, dl, CondVT, Cond, DAG.getConstant(1, dl, CondVT)); in LowerBRCOND()
25901 return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, in LowerBRCOND()
25921 SDNode *Node = Op.getNode(); in LowerDYNAMIC_STACKALLOC()
25949 Result = DAG.getNode(X86ISD::PROBED_ALLOCA, dl, SPTy, Chain, in LowerDYNAMIC_STACKALLOC()
25954 Result = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value in LowerDYNAMIC_STACKALLOC()
25958 DAG.getNode(ISD::AND, dl, VT, Result, in LowerDYNAMIC_STACKALLOC()
25978 Result = DAG.getNode(X86ISD::SEG_ALLOCA, dl, SPTy, Chain, in LowerDYNAMIC_STACKALLOC()
25982 Chain = DAG.getNode(X86ISD::DYN_ALLOCA, dl, NodeTys, Chain, Size); in LowerDYNAMIC_STACKALLOC()
25991 SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0), in LowerDYNAMIC_STACKALLOC()
26046 FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getIntPtrConstant(4, DL)); in LowerVASTART()
26053 FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getIntPtrConstant( in LowerVASTART()
26060 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps); in LowerVASTART()
26071 return DAG.expandVAArg(Op.getNode()); in LowerVAARG()
26079 EVT ArgVT = Op.getNode()->getValueType(0); in LowerVAARG()
26129 return DAG.expandVACopy(Op.getNode()); in LowerVACOPY()
26193 if (ISD::isBuildVectorOfConstantSDNodes(SrcOp.getNode())) { in getTargetVShiftByConstNode()
26213 return DAG.getNode(Opc, dl, VT, SrcOp, in getTargetVShiftByConstNode()
26254 ShAmt = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, ShAmt); in getTargetVShiftNode()
26255 ShAmt = DAG.getNode(X86ISD::VZEXT_MOVL, dl, MVT::v4i32, ShAmt); in getTargetVShiftNode()
26269 ShAmt = DAG.getNode(ISD::AND, dl, AmtVT, ShAmt.getOperand(0), Mask); in getTargetVShiftNode()
26286 ShAmt = DAG.getNode(X86ISD::VZEXT_MOVL, SDLoc(ShAmt), MVT::v4i32, ShAmt); in getTargetVShiftNode()
26288 ShAmt = DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(ShAmt), in getTargetVShiftNode()
26294 ShAmt = DAG.getNode(X86ISD::VSHLDQ, SDLoc(ShAmt), MVT::v16i8, ShAmt, in getTargetVShiftNode()
26296 ShAmt = DAG.getNode(X86ISD::VSRLDQ, SDLoc(ShAmt), MVT::v16i8, ShAmt, in getTargetVShiftNode()
26310 return DAG.getNode(Opc, dl, VT, SrcOp, ShAmt); in getTargetVShiftNode()
26331 Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Mask, in getMaskNode()
26333 Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Mask, in getMaskNode()
26339 return DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, Lo, Hi); in getMaskNode()
26345 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MaskVT, in getMaskNode()
26370 return DAG.getNode(OpcodeSelect, dl, VT, VMask, Op, PreservedSrc); in getVectorMaskingNode()
26393 SDValue IMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v1i1, in getScalarMaskingNode()
26399 return DAG.getNode(ISD::AND, dl, VT, Op, IMask); in getScalarMaskingNode()
26403 return DAG.getNode(X86ISD::SELECTS, dl, VT, IMask, Op, PreservedSrc); in getScalarMaskingNode()
26450 DAG.getNode(ISD::LOCAL_RECOVER, dl, PtrVT, OffsetSymVal); in recoverFramePointer()
26456 return DAG.getNode(ISD::ADD, dl, PtrVT, EntryEBP, ParentFrameOffset); in recoverFramePointer()
26461 SDValue RegNodeBase = DAG.getNode(ISD::SUB, dl, PtrVT, EntryEBP, in recoverFramePointer()
26463 return DAG.getNode(ISD::SUB, dl, PtrVT, RegNodeBase, ParentFrameOffset); in recoverFramePointer()
26524 return DAG.getNode(IntrWithRoundingModeOpcode, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
26530 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
26544 return DAG.getNode(Opc, dl, Op.getValueType(), Op.getOperand(1)); in LowerINTRINSIC_WO_CHAIN()
26557 return DAG.getNode(IntrWithRoundingModeOpcode, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
26564 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
26578 return DAG.getNode(Opc, dl, Op.getValueType(), Op.getOperand(1), in LowerINTRINSIC_WO_CHAIN()
26601 return DAG.getNode(IntrWithRoundingModeOpcode, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
26608 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
26619 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
26636 DAG.getNode(IntrWithRoundingModeOpcode, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
26643 DAG.getNode(IntrData->Opc0, dl, VT, Src), Mask, PassThru, in LowerINTRINSIC_WO_CHAIN()
26660 return getVectorMaskingNode(DAG.getNode(Opc, dl, VT, Src), Mask, PassThru, in LowerINTRINSIC_WO_CHAIN()
26679 DAG.getNode(IntrWithRoundingModeOpcode, dl, VT, Src1, Src2, in LowerINTRINSIC_WO_CHAIN()
26685 return getScalarMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, Src1, in LowerINTRINSIC_WO_CHAIN()
26701 return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1, in LowerINTRINSIC_WO_CHAIN()
26715 NewOp = DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2); in LowerINTRINSIC_WO_CHAIN()
26717 NewOp = DAG.getNode(IntrData->Opc1, dl, VT, Src1, Src2, in LowerINTRINSIC_WO_CHAIN()
26738 return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2), in LowerINTRINSIC_WO_CHAIN()
26751 NewOp = DAG.getNode(IntrData->Opc1, dl, VT, Src1, Src2, in LowerINTRINSIC_WO_CHAIN()
26757 NewOp = DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2); in LowerINTRINSIC_WO_CHAIN()
26775 return getVectorMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2), in LowerINTRINSIC_WO_CHAIN()
26793 return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2, Src3), in LowerINTRINSIC_WO_CHAIN()
26811 return getVectorMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2, Src3), in LowerINTRINSIC_WO_CHAIN()
26823 return DAG.getNode(IntrData->Opc0, dl, VT, Src3, Src2, Src1); in LowerINTRINSIC_WO_CHAIN()
26830 return DAG.getNode(IntrData->Opc0, dl, VT,Src2, Src1); in LowerINTRINSIC_WO_CHAIN()
26852 NewOp = DAG.getNode(IntrData->Opc1, dl, VT, Src1, Src2, Src3, in LowerINTRINSIC_WO_CHAIN()
26858 NewOp = DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2, Src3); in LowerINTRINSIC_WO_CHAIN()
26864 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
26870 SDValue FPclass = DAG.getNode(IntrData->Opc0, dl, MVT::v1i1, Src1, Imm); in LowerINTRINSIC_WO_CHAIN()
26875 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8i1, in LowerINTRINSIC_WO_CHAIN()
26891 return DAG.getNode(IntrData->Opc1, dl, MaskVT, Op.getOperand(1), in LowerINTRINSIC_WO_CHAIN()
26897 return DAG.getNode(IntrData->Opc0, dl, MaskVT, in LowerINTRINSIC_WO_CHAIN()
26910 Cmp = DAG.getNode(IntrData->Opc1, dl, MVT::v1i1, Src1, Src2, CC, Sae); in LowerINTRINSIC_WO_CHAIN()
26915 if (!Cmp.getNode()) in LowerINTRINSIC_WO_CHAIN()
26916 Cmp = DAG.getNode(IntrData->Opc0, dl, MVT::v1i1, Src1, Src2, CC); in LowerINTRINSIC_WO_CHAIN()
26922 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8i1, in LowerINTRINSIC_WO_CHAIN()
26935 SDValue Comi = DAG.getNode(IntrData->Opc0, dl, MVT::i32, LHS, RHS); in LowerINTRINSIC_WO_CHAIN()
26941 SetCC = DAG.getNode(ISD::AND, dl, MVT::i8, SetCC, SetNP); in LowerINTRINSIC_WO_CHAIN()
26947 SetCC = DAG.getNode(ISD::OR, dl, MVT::i8, SetCC, SetP); in LowerINTRINSIC_WO_CHAIN()
26962 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); in LowerINTRINSIC_WO_CHAIN()
26972 FCmp = DAG.getNode(X86ISD::FSETCCM, dl, MVT::v1i1, LHS, RHS, in LowerINTRINSIC_WO_CHAIN()
26975 FCmp = DAG.getNode(X86ISD::FSETCCM_SAE, dl, MVT::v1i1, LHS, RHS, in LowerINTRINSIC_WO_CHAIN()
26981 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v16i1, in LowerINTRINSIC_WO_CHAIN()
26984 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, in LowerINTRINSIC_WO_CHAIN()
26999 ShAmt = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, ShAmt); in LowerINTRINSIC_WO_CHAIN()
27007 if (ISD::isBuildVectorAllOnes(Mask.getNode())) // return data as is in LowerINTRINSIC_WO_CHAIN()
27014 return DAG.getNode(IntrData->Opc0, dl, VT, DataToCompress, PassThru, in LowerINTRINSIC_WO_CHAIN()
27037 SDValue FixupImm = DAG.getNode(Opc, dl, VT, Src1, Src2, Src3, Imm); in LowerINTRINSIC_WO_CHAIN()
27051 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
27061 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
27070 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
27082 Res = DAG.getNode(IntrData->Opc1, dl, VTs, Op.getOperand(2), in LowerINTRINSIC_WO_CHAIN()
27085 SDValue GenCF = DAG.getNode(X86ISD::ADD, dl, CFVTs, Op.getOperand(1), in LowerINTRINSIC_WO_CHAIN()
27087 Res = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(2), in LowerINTRINSIC_WO_CHAIN()
27103 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Src); in LowerINTRINSIC_WO_CHAIN()
27108 return DAG.getNode(IntrData->Opc1, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
27118 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Src, Rnd); in LowerINTRINSIC_WO_CHAIN()
27123 return DAG.getNode(IntrData->Opc1, dl, Op.getValueType(), Src, Rnd, in LowerINTRINSIC_WO_CHAIN()
27132 if (ISD::isBuildVectorAllOnes(Mask.getNode())) in LowerINTRINSIC_WO_CHAIN()
27133 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Src); in LowerINTRINSIC_WO_CHAIN()
27139 return DAG.getNode(IntrData->Opc1, dl, Op.getValueType(), Src, PassThru, in LowerINTRINSIC_WO_CHAIN()
27235 SDValue Test = DAG.getNode(TestOpc, dl, MVT::i32, LHS, RHS); in LowerINTRINSIC_WO_CHAIN()
27237 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); in LowerINTRINSIC_WO_CHAIN()
27297 SDValue PCMP = DAG.getNode(Opcode, dl, VTs, NewOps).getValue(2); in LowerINTRINSIC_WO_CHAIN()
27299 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); in LowerINTRINSIC_WO_CHAIN()
27312 return DAG.getNode(Opcode, dl, VTs, NewOps); in LowerINTRINSIC_WO_CHAIN()
27325 return DAG.getNode(Opcode, dl, VTs, NewOps).getValue(1); in LowerINTRINSIC_WO_CHAIN()
27335 return DAG.getNode(getGlobalWrapperKind(), dl, VT, in LowerINTRINSIC_WO_CHAIN()
27350 return DAG.getNode(X86ISD::Wrapper, dl, VT, Result); in LowerINTRINSIC_WO_CHAIN()
27393 DAG.getNode(X86ISD::VP2INTERSECT, DL, VTs, in LowerINTRINSIC_WO_CHAIN()
27420 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
27457 ShAmt = DAG.getNode(X86ISD::MMX_MOVW2D, DL, MVT::x86mmx, ShAmt); in LowerINTRINSIC_WO_CHAIN()
27458 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
27496 if (Src.isUndef() || ISD::isBuildVectorAllOnes(Mask.getNode())) in getAVX2GatherNode()
27537 if (Src.isUndef() || ISD::isBuildVectorAllOnes(Mask.getNode())) in getGatherNode()
27628 TargetOpcode, DL, Tys, ArrayRef<SDValue>(N1Ops, Glue.getNode() ? 2 : 1)); in expandIntrinsicWChainHelper()
27647 SDValue Tmp = DAG.getNode(ISD::SHL, DL, MVT::i64, HI, in expandIntrinsicWChainHelper()
27649 Results.push_back(DAG.getNode(ISD::OR, DL, MVT::i64, LO, Tmp)); in expandIntrinsicWChainHelper()
27656 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops); in expandIntrinsicWChainHelper()
27690 getReadTimeStampCounter(Op.getNode(), DL, X86::RDTSC, DAG, Subtarget, in LowerREADCYCLECOUNTER()
27775 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), Result, in LowerINTRINSIC_W_CHAIN()
27786 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), Result, in LowerINTRINSIC_W_CHAIN()
27799 return DAG.getNode(X86ISD::RDPKRU, dl, VTs, Op.getOperand(0), in LowerINTRINSIC_W_CHAIN()
27806 return DAG.getNode(X86ISD::WRPKRU, dl, MVT::Other, in LowerINTRINSIC_W_CHAIN()
27853 DAG.getNode(Opcode, dl, VTs, Chain, Op->getOperand(2), in LowerINTRINSIC_W_CHAIN()
27856 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), SetCC, in LowerINTRINSIC_W_CHAIN()
27874 SDValue Operation = DAG.getNode(Opcode, dl, VTs, Chain, Op.getOperand(2), in LowerINTRINSIC_W_CHAIN()
27877 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), SetCC, in LowerINTRINSIC_W_CHAIN()
27913 return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), in LowerINTRINSIC_W_CHAIN()
27954 return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), in LowerINTRINSIC_W_CHAIN()
27965 SDValue Operation = DAG.getNode(X86ISD::TESTUI, dl, VTs, Chain); in LowerINTRINSIC_W_CHAIN()
27967 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), SetCC, in LowerINTRINSIC_W_CHAIN()
27990 Res = DAG.getNode(ISD::SHL, DL, VT, Res, in LowerINTRINSIC_W_CHAIN()
27992 return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), Res, Chain); in LowerINTRINSIC_W_CHAIN()
28005 SDValue Result = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(0)); in LowerINTRINSIC_W_CHAIN()
28012 SDValue(Result.getNode(), 1)}; in LowerINTRINSIC_W_CHAIN()
28013 SDValue isValid = DAG.getNode(X86ISD::CMOV, dl, Op->getValueType(1), Ops); in LowerINTRINSIC_W_CHAIN()
28016 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), Result, isValid, in LowerINTRINSIC_W_CHAIN()
28017 SDValue(Result.getNode(), 2)); in LowerINTRINSIC_W_CHAIN()
28067 getReadTimeStampCounter(Op.getNode(), dl, IntrData->Opc0, DAG, Subtarget, in LowerINTRINSIC_W_CHAIN()
28083 expandIntrinsicWChainHelper(Op.getNode(), dl, DAG, IntrData->Opc0, X86::ECX, in LowerINTRINSIC_W_CHAIN()
28090 SDValue InTrans = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(0)); in LowerINTRINSIC_W_CHAIN()
28093 SDValue Ret = DAG.getNode(ISD::ZERO_EXTEND, dl, Op->getValueType(0), SetCC); in LowerINTRINSIC_W_CHAIN()
28094 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), in LowerINTRINSIC_W_CHAIN()
28095 Ret, SDValue(InTrans.getNode(), 1)); in LowerINTRINSIC_W_CHAIN()
28162 DAG.getNode(ISD::ADD, dl, PtrVT, FrameAddr, Offset), in LowerRETURNADDR()
28290 SDValue StoreAddr = DAG.getNode(ISD::ADD, dl, PtrVT, Frame, in LowerEH_RETURN()
28293 StoreAddr = DAG.getNode(ISD::ADD, dl, PtrVT, StoreAddr, Offset); in LowerEH_RETURN()
28297 return DAG.getNode(X86ISD::EH_RETURN, dl, MVT::Other, Chain, in LowerEH_RETURN()
28314 return DAG.getNode(X86ISD::EH_SJLJ_SETJMP, DL, in lowerEH_SJLJ_SETJMP()
28322 return DAG.getNode(X86ISD::EH_SJLJ_LONGJMP, DL, MVT::Other, in lowerEH_SJLJ_LONGJMP()
28329 return DAG.getNode(X86ISD::EH_SJLJ_SETUP_DISPATCH, DL, MVT::Other, in lowerEH_SJLJ_SETUP_DISPATCH()
28366 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
28374 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
28379 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
28386 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
28392 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
28397 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); in LowerINIT_TRAMPOLINE()
28450 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, in LowerINIT_TRAMPOLINE()
28452 Disp = DAG.getNode(ISD::SUB, dl, MVT::i32, FPtr, Addr); in LowerINIT_TRAMPOLINE()
28461 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, in LowerINIT_TRAMPOLINE()
28467 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, in LowerINIT_TRAMPOLINE()
28473 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, in LowerINIT_TRAMPOLINE()
28478 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); in LowerINIT_TRAMPOLINE()
28529 DAG.getNode(ISD::SRL, DL, MVT::i16, in LowerFLT_ROUNDS_()
28530 DAG.getNode(ISD::AND, DL, MVT::i16, in LowerFLT_ROUNDS_()
28533 Shift = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, Shift); in LowerFLT_ROUNDS_()
28537 DAG.getNode(ISD::AND, DL, MVT::i32, in LowerFLT_ROUNDS_()
28538 DAG.getNode(ISD::SRL, DL, MVT::i32, LUT, Shift), in LowerFLT_ROUNDS_()
28550 SDValue Chain = Op.getNode()->getOperand(0); in LowerSET_ROUNDING()
28569 CWD = DAG.getNode(ISD::AND, DL, MVT::i16, CWD.getValue(0), in LowerSET_ROUNDING()
28573 SDValue NewRM = Op.getNode()->getOperand(1); in LowerSET_ROUNDING()
28601 DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, in LowerSET_ROUNDING()
28602 DAG.getNode(ISD::ADD, DL, MVT::i32, in LowerSET_ROUNDING()
28603 DAG.getNode(ISD::SHL, DL, MVT::i32, NewRM, in LowerSET_ROUNDING()
28607 DAG.getNode(ISD::SHL, DL, MVT::i16, DAG.getConstant(0xc9, DL, MVT::i16), in LowerSET_ROUNDING()
28609 RMBits = DAG.getNode(ISD::AND, DL, MVT::i16, Shifted, in LowerSET_ROUNDING()
28614 CWD = DAG.getNode(ISD::OR, DL, MVT::i16, CWD, RMBits); in LowerSET_ROUNDING()
28628 Chain = DAG.getNode( in LowerSET_ROUNDING()
28636 CWD = DAG.getNode(ISD::AND, DL, MVT::i32, CWD.getValue(0), in LowerSET_ROUNDING()
28640 RMBits = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, RMBits); in LowerSET_ROUNDING()
28641 RMBits = DAG.getNode(ISD::SHL, DL, MVT::i32, RMBits, in LowerSET_ROUNDING()
28645 CWD = DAG.getNode(ISD::OR, DL, MVT::i32, CWD, RMBits); in LowerSET_ROUNDING()
28649 Chain = DAG.getNode( in LowerSET_ROUNDING()
28685 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, NewVT, Op.getOperand(0)); in LowerVectorCTLZ_AVX512CDI()
28686 SDValue CtlzNode = DAG.getNode(ISD::CTLZ, dl, NewVT, Op); in LowerVectorCTLZ_AVX512CDI()
28687 SDValue TruncNode = DAG.getNode(ISD::TRUNCATE, dl, VT, CtlzNode); in LowerVectorCTLZ_AVX512CDI()
28690 return DAG.getNode(ISD::SUB, dl, VT, TruncNode, Delta); in LowerVectorCTLZ_AVX512CDI()
28723 SDValue Hi = DAG.getNode(ISD::SRL, DL, CurrVT, Op0, NibbleShift); in LowerVectorCTLZInRegLUT()
28728 HiZ = DAG.getNode(ISD::SIGN_EXTEND, DL, CurrVT, HiZ); in LowerVectorCTLZInRegLUT()
28733 Lo = DAG.getNode(X86ISD::PSHUFB, DL, CurrVT, InRegLUT, Lo); in LowerVectorCTLZInRegLUT()
28734 Hi = DAG.getNode(X86ISD::PSHUFB, DL, CurrVT, InRegLUT, Hi); in LowerVectorCTLZInRegLUT()
28735 Lo = DAG.getNode(ISD::AND, DL, CurrVT, Lo, HiZ); in LowerVectorCTLZInRegLUT()
28736 SDValue Res = DAG.getNode(ISD::ADD, DL, CurrVT, Lo, Hi); in LowerVectorCTLZInRegLUT()
28755 HiZ = DAG.getNode(ISD::SIGN_EXTEND, DL, CurrVT, HiZ); in LowerVectorCTLZInRegLUT()
28766 SDValue R0 = DAG.getNode(ISD::SRL, DL, NextVT, ResNext, Shift); in LowerVectorCTLZInRegLUT()
28767 SDValue R1 = DAG.getNode(ISD::SRL, DL, NextVT, HiZ, Shift); in LowerVectorCTLZInRegLUT()
28768 R1 = DAG.getNode(ISD::AND, DL, NextVT, ResNext, R1); in LowerVectorCTLZInRegLUT()
28769 Res = DAG.getNode(ISD::ADD, DL, NextVT, R0, R1); in LowerVectorCTLZInRegLUT()
28813 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, OpVT, Op); in LowerCTLZ()
28818 Op = DAG.getNode(X86ISD::BSR, dl, VTs, Op); in LowerCTLZ()
28825 Op = DAG.getNode(X86ISD::CMOV, dl, OpVT, Ops); in LowerCTLZ()
28829 Op = DAG.getNode(ISD::XOR, dl, OpVT, Op, in LowerCTLZ()
28833 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op); in LowerCTLZ()
28849 Op = DAG.getNode(X86ISD::BSF, dl, VTs, N0); in LowerCTTZ()
28855 return DAG.getNode(X86ISD::CMOV, dl, VT, Ops); in LowerCTTZ()
28904 SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, X, SignMask); in LowerADDSAT_SUBSAT()
28905 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, X, ShiftAmt); in LowerADDSAT_SUBSAT()
28906 return DAG.getNode(ISD::AND, DL, VT, Xor, Sra); in LowerADDSAT_SUBSAT()
28911 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, X, Y); in LowerADDSAT_SUBSAT()
28916 return DAG.getNode(ISD::AND, DL, VT, Cmp, Sub); in LowerADDSAT_SUBSAT()
28927 DAG.getNode(Opcode == ISD::SADDSAT ? ISD::SADDO : ISD::SSUBO, DL, in LowerADDSAT_SUBSAT()
28951 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, DAG.getVTList(VT, MVT::i32), in LowerABS()
28954 SDValue(Neg.getNode(), 1)}; in LowerABS()
28955 return DAG.getNode(X86ISD::CMOV, DL, VT, Ops); in LowerABS()
28963 DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Src); in LowerABS()
28964 return DAG.getNode(X86ISD::BLENDV, DL, VT, Src, Sub, Src); in LowerABS()
29033 return DAG.getNode( in LowerMUL()
29035 DAG.getNode(ISD::MUL, dl, ExVT, in LowerMUL()
29036 DAG.getNode(ISD::ANY_EXTEND, dl, ExVT, A), in LowerMUL()
29037 DAG.getNode(ISD::ANY_EXTEND, dl, ExVT, B))); in LowerMUL()
29051 if (ISD::isBuildVectorOfConstantSDNodes(B.getNode())) { in LowerMUL()
29071 SDValue RLo = DAG.getNode(ISD::MUL, dl, ExVT, ALo, BLo); in LowerMUL()
29072 SDValue RHi = DAG.getNode(ISD::MUL, dl, ExVT, AHi, BHi); in LowerMUL()
29087 SDValue Evens = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64, in LowerMUL()
29091 SDValue Odds = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64, in LowerMUL()
29133 AloBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, B); in LowerMUL()
29138 AloBhi = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, Bhi); in LowerMUL()
29144 AhiBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, Ahi, B); in LowerMUL()
29147 SDValue Hi = DAG.getNode(ISD::ADD, dl, VT, AloBhi, AhiBlo); in LowerMUL()
29150 return DAG.getNode(ISD::ADD, dl, VT, AloBlo, Hi); in LowerMUL()
29185 if (ISD::isBuildVectorOfConstantSDNodes(B.getNode())) { in LowervXi8MulWithUNPCK()
29196 LoOp = DAG.getNode(ISD::SHL, dl, MVT::i16, LoOp, in LowervXi8MulWithUNPCK()
29198 HiOp = DAG.getNode(ISD::SHL, dl, MVT::i16, HiOp, in LowervXi8MulWithUNPCK()
29223 SDValue RLo = DAG.getNode(MulOpc, dl, ExVT, ALo, BLo); in LowervXi8MulWithUNPCK()
29224 SDValue RHi = DAG.getNode(MulOpc, dl, ExVT, AHi, BHi); in LowervXi8MulWithUNPCK()
29281 SDValue Mul1 = DAG.getBitcast(VT, DAG.getNode(Opcode, dl, MulVT, in LowerMULH()
29286 SDValue Mul2 = DAG.getBitcast(VT, DAG.getNode(Opcode, dl, MulVT, in LowerMULH()
29301 SDValue T1 = DAG.getNode(ISD::AND, dl, VT, in LowerMULH()
29303 SDValue T2 = DAG.getNode(ISD::AND, dl, VT, in LowerMULH()
29306 SDValue Fixup = DAG.getNode(ISD::ADD, dl, VT, T1, T2); in LowerMULH()
29307 Res = DAG.getNode(ISD::SUB, dl, VT, Res, Fixup); in LowerMULH()
29328 SDValue ExA = DAG.getNode(ExAVX, dl, ExVT, A); in LowerMULH()
29329 SDValue ExB = DAG.getNode(ExAVX, dl, ExVT, B); in LowerMULH()
29330 SDValue Mul = DAG.getNode(ISD::MUL, dl, ExVT, ExA, ExB); in LowerMULH()
29332 return DAG.getNode(ISD::TRUNCATE, dl, VT, Mul); in LowerMULH()
29369 SDValue Lo = DAG.getNode(Op.getOpcode(), dl, LoVTs, LHSLo, RHSLo); in LowerMULO()
29370 SDValue Hi = DAG.getNode(Op.getOpcode(), dl, HiVTs, LHSHi, RHSHi); in LowerMULO()
29373 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); in LowerMULO()
29374 SDValue Ovf = DAG.getNode(ISD::CONCAT_VECTORS, dl, OvfVT, Lo.getValue(1), in LowerMULO()
29389 SDValue ExA = DAG.getNode(ExAVX, dl, ExVT, A); in LowerMULO()
29390 SDValue ExB = DAG.getNode(ExAVX, dl, ExVT, B); in LowerMULO()
29391 SDValue Mul = DAG.getNode(ISD::MUL, dl, ExVT, ExA, ExB); in LowerMULO()
29393 SDValue Low = DAG.getNode(ISD::TRUNCATE, dl, VT, Mul); in LowerMULO()
29411 High = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v16i32, High); in LowerMULO()
29412 LowSign = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v16i32, LowSign); in LowerMULO()
29417 High = DAG.getNode(ISD::TRUNCATE, dl, VT, High); in LowerMULO()
29419 DAG.getNode(ISD::SRA, dl, VT, Low, DAG.getConstant(7, dl, VT)); in LowerMULO()
29432 High = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v16i32, High); in LowerMULO()
29436 High = DAG.getNode(ISD::TRUNCATE, dl, VT, High); in LowerMULO()
29457 DAG.getNode(ISD::SRA, dl, VT, Low, DAG.getConstant(7, dl, VT)); in LowerMULO()
29496 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); in LowerWin64_i128OP()
29587 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); in LowerWin64_INT128_TO_FP()
29668 return DAG.getNode(X86ISD::PCMPGT, dl, VT, DAG.getConstant(0, dl, VT), R); in LowerShiftByScalarImmediate()
29730 R = DAG.getNode(ISD::FREEZE, dl, VT, R); in LowerShiftByScalarImmediate()
29731 return DAG.getNode(ISD::ADD, dl, VT, R, R); in LowerShiftByScalarImmediate()
29740 return DAG.getNode(ISD::SIGN_EXTEND, dl, VT, CMP); in LowerShiftByScalarImmediate()
29742 return DAG.getNode(X86ISD::PCMPGT, dl, VT, Zeros, R); in LowerShiftByScalarImmediate()
29756 return DAG.getNode(ISD::AND, dl, VT, SHL, DAG.getConstant(Mask, dl, VT)); in LowerShiftByScalarImmediate()
29765 return DAG.getNode(ISD::AND, dl, VT, SRL, DAG.getConstant(Mask, dl, VT)); in LowerShiftByScalarImmediate()
29769 SDValue Res = DAG.getNode(ISD::SRL, dl, VT, R, Amt); in LowerShiftByScalarImmediate()
29772 Res = DAG.getNode(ISD::XOR, dl, VT, Res, Mask); in LowerShiftByScalarImmediate()
29773 Res = DAG.getNode(ISD::SUB, dl, VT, Res, Mask); in LowerShiftByScalarImmediate()
29824 Res = DAG.getNode(ISD::AND, dl, VT, Res, BitMask); in LowerShiftByScalarVariable()
29834 Res = DAG.getNode(ISD::XOR, dl, VT, Res, SignMask); in LowerShiftByScalarVariable()
29835 Res = DAG.getNode(ISD::SUB, dl, VT, Res, SignMask); in LowerShiftByScalarVariable()
29879 Amt = DAG.getNode(ISD::SHL, dl, VT, Amt, DAG.getConstant(23, dl, VT)); in convertShiftLeftToScale()
29880 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, in convertShiftLeftToScale()
29883 return DAG.getNode(ISD::FP_TO_SINT, dl, VT, Amt); in convertShiftLeftToScale()
29894 return DAG.getNode(X86ISD::PACKUS, dl, VT, Lo, Hi); in convertShiftLeftToScale()
29908 bool ConstantAmt = ISD::isBuildVectorOfConstantSDNodes(Amt.getNode()); in LowerShift()
29933 SDValue M = DAG.getNode(ISD::SRL, dl, VT, S, Amt); in LowerShift()
29934 R = DAG.getNode(ISD::SRL, dl, VT, R, Amt); in LowerShift()
29935 R = DAG.getNode(ISD::XOR, dl, VT, R, M); in LowerShift()
29936 R = DAG.getNode(ISD::SUB, dl, VT, R, M); in LowerShift()
29946 Amt = DAG.getNode(ISD::SUB, dl, VT, Zero, Amt); in LowerShift()
29949 return DAG.getNode(X86ISD::VPSHL, dl, VT, R, Amt); in LowerShift()
29951 return DAG.getNode(X86ISD::VPSHA, dl, VT, R, Amt); in LowerShift()
29960 SDValue R0 = DAG.getNode(Opc, dl, VT, R, Amt0); in LowerShift()
29961 SDValue R1 = DAG.getNode(Opc, dl, VT, R, Amt1); in LowerShift()
30024 return DAG.getNode(ISD::MUL, dl, VT, R, Scale); in LowerShift()
30031 SDValue RAmt = DAG.getNode(ISD::SUB, dl, VT, EltBits, Amt); in LowerShift()
30035 SDValue Res = DAG.getNode(ISD::MULHU, dl, VT, R, Scale); in LowerShift()
30050 SDValue RAmt = DAG.getNode(ISD::SUB, dl, VT, EltBits, Amt); in LowerShift()
30058 SDValue Res = DAG.getNode(ISD::MULHS, dl, VT, R, Scale); in LowerShift()
30093 Amt0 = DAG.getNode(X86ISD::PSHUFLW, dl, MVT::v8i16, Amt01, Msk02); in LowerShift()
30094 Amt1 = DAG.getNode(X86ISD::PSHUFLW, dl, MVT::v8i16, Amt01, Msk13); in LowerShift()
30095 Amt2 = DAG.getNode(X86ISD::PSHUFLW, dl, MVT::v8i16, Amt23, Msk02); in LowerShift()
30096 Amt3 = DAG.getNode(X86ISD::PSHUFLW, dl, MVT::v8i16, Amt23, Msk13); in LowerShift()
30101 SDValue R0 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt0)); in LowerShift()
30102 SDValue R1 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt1)); in LowerShift()
30103 SDValue R2 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt2)); in LowerShift()
30104 SDValue R3 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt3)); in LowerShift()
30132 R = DAG.getNode(ExtOpc, dl, ExtVT, R); in LowerShift()
30133 Amt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, Amt); in LowerShift()
30134 return DAG.getNode(ISD::TRUNCATE, dl, VT, in LowerShift()
30135 DAG.getNode(Opc, dl, ExtVT, R, Amt)); in LowerShift()
30151 Amt = DAG.getNode(ISD::SUB, dl, ExVT, DAG.getConstant(8, dl, ExVT), Amt); in LowerShift()
30152 Amt = DAG.getNode(ISD::SHL, dl, ExVT, DAG.getConstant(1, dl, ExVT), Amt); in LowerShift()
30153 assert(ISD::isBuildVectorOfConstantSDNodes(Amt.getNode()) && in LowerShift()
30159 R = DAG.getNode(ISD::MUL, dl, ExVT, R, Amt); in LowerShift()
30160 R = DAG.getNode(X86ISD::VSRLI, dl, ExVT, R, Cst8); in LowerShift()
30178 LoR = DAG.getNode(X86OpcI, dl, VT16, LoR, Cst8); in LowerShift()
30179 HiR = DAG.getNode(X86OpcI, dl, VT16, HiR, Cst8); in LowerShift()
30180 LoR = DAG.getNode(ISD::MUL, dl, VT16, LoR, LoA); in LowerShift()
30181 HiR = DAG.getNode(ISD::MUL, dl, VT16, HiR, HiA); in LowerShift()
30182 LoR = DAG.getNode(X86ISD::VSRLI, dl, VT16, LoR, Cst8); in LowerShift()
30183 HiR = DAG.getNode(X86ISD::VSRLI, dl, VT16, HiR, Cst8); in LowerShift()
30184 return DAG.getNode(X86ISD::PACKUS, dl, VT, LoR, HiR); in LowerShift()
30211 DAG.getNode(X86ISD::BLENDV, dl, VT, Sel, V0, V1)); in LowerShift()
30217 SDValue C = DAG.getNode(X86ISD::PCMPGT, dl, SelVT, Z, Sel); in LowerShift()
30230 SDValue M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(4, dl, VT)); in LowerShift()
30234 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt); in LowerShift()
30237 M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(2, dl, VT)); in LowerShift()
30241 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt); in LowerShift()
30244 M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(1, dl, VT)); in LowerShift()
30269 ALo = DAG.getNode(ISD::ADD, dl, ExtVT, ALo, ALo); in LowerShift()
30270 AHi = DAG.getNode(ISD::ADD, dl, ExtVT, AHi, AHi); in LowerShift()
30279 ALo = DAG.getNode(ISD::ADD, dl, ExtVT, ALo, ALo); in LowerShift()
30280 AHi = DAG.getNode(ISD::ADD, dl, ExtVT, AHi, AHi); in LowerShift()
30292 return DAG.getNode(X86ISD::PACKUS, dl, VT, RLo, RHi); in LowerShift()
30307 SDValue Lo = DAG.getNode(Opc, dl, ExtVT, RLo, ALo); in LowerShift()
30308 SDValue Hi = DAG.getNode(Opc, dl, ExtVT, RHi, AHi); in LowerShift()
30311 return DAG.getNode(X86ISD::PACKUS, dl, VT, Lo, Hi); in LowerShift()
30318 !ISD::isBuildVectorOfConstantSDNodes(Amt.getNode()); in LowerShift()
30329 VT, DAG.getNode(X86ISD::BLENDV, dl, ExtVT, Sel, V0, V1)); in LowerShift()
30343 Amt = DAG.getNode( in LowerShift()
30356 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt); in LowerShift()
30363 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt); in LowerShift()
30370 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt); in LowerShift()
30429 SDValue AmtMod = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask); in LowerFunnelShift()
30430 bool IsCst = ISD::isBuildVectorOfConstantSDNodes(AmtMod.getNode()); in LowerFunnelShift()
30448 Op = DAG.getNode(Op.getOpcode(), DL, VT, Op0, Op1, AmtMod); in LowerFunnelShift()
30483 Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, WideVT, Op0); in LowerFunnelShift()
30484 Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Op1); in LowerFunnelShift()
30485 AmtMod = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, AmtMod); in LowerFunnelShift()
30488 SDValue Res = DAG.getNode(ISD::OR, DL, WideVT, Op0, Op1); in LowerFunnelShift()
30489 Res = DAG.getNode(ShiftOpc, DL, WideVT, Res, AmtMod); in LowerFunnelShift()
30493 return DAG.getNode(ISD::TRUNCATE, DL, VT, Res); in LowerFunnelShift()
30504 SDValue Lo = DAG.getNode(ShiftOpc, DL, ExtVT, RLo, ALo); in LowerFunnelShift()
30505 SDValue Hi = DAG.getNode(ShiftOpc, DL, ExtVT, RHi, AHi); in LowerFunnelShift()
30528 Amt = DAG.getNode(ISD::AND, DL, Amt.getValueType(), Amt, Mask); in LowerFunnelShift()
30529 SDValue Res = DAG.getNode(ISD::SHL, DL, MVT::i32, Op0, HiShift); in LowerFunnelShift()
30530 Res = DAG.getNode(ISD::OR, DL, MVT::i32, Res, Op1); in LowerFunnelShift()
30532 Res = DAG.getNode(ISD::SRL, DL, MVT::i32, Res, Amt); in LowerFunnelShift()
30534 Res = DAG.getNode(ISD::SHL, DL, MVT::i32, Res, Amt); in LowerFunnelShift()
30535 Res = DAG.getNode(ISD::SRL, DL, MVT::i32, Res, HiShift); in LowerFunnelShift()
30545 Amt = DAG.getNode(ISD::AND, DL, Amt.getValueType(), Amt, in LowerFunnelShift()
30548 return DAG.getNode(FSHOp, DL, VT, Op0, Op1, Amt); in LowerFunnelShift()
30581 return DAG.getNode(RotOpc, DL, VT, R, in LowerRotate()
30592 return DAG.getNode(FunnelOpc, DL, VT, R, R, Amt); in LowerRotate()
30601 return DAG.getNode(ISD::ROTL, DL, VT, R, NegAmt); in LowerRotate()
30605 return DAG.getNode(ISD::ROTL, DL, VT, R, in LowerRotate()
30606 DAG.getNode(ISD::SUB, DL, VT, Z, Amt)); in LowerRotate()
30623 return DAG.getNode(X86ISD::VROTLI, DL, VT, R, in LowerRotate()
30650 SDValue AmtMod = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask); in LowerRotate()
30660 return DAG.getNode(FunnelOpc, DL, VT, R, R, Amt); in LowerRotate()
30677 bool IsConstAmt = ISD::isBuildVectorOfConstantSDNodes(Amt.getNode()); in LowerRotate()
30691 R = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, R); in LowerRotate()
30692 R = DAG.getNode( in LowerRotate()
30695 Amt = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, AmtMod); in LowerRotate()
30696 R = DAG.getNode(ShiftOpc, DL, WideVT, R, Amt); in LowerRotate()
30699 return DAG.getNode(ISD::TRUNCATE, DL, VT, R); in LowerRotate()
30711 SDValue Lo = DAG.getNode(ShiftOpc, DL, ExtVT, RLo, ALo); in LowerRotate()
30712 SDValue Hi = DAG.getNode(ShiftOpc, DL, ExtVT, RHi, AHi); in LowerRotate()
30726 DAG.getNode(X86ISD::BLENDV, DL, VT, Sel, V0, V1)); in LowerRotate()
30732 SDValue C = DAG.getNode(X86ISD::PCMPGT, DL, SelVT, Z, Sel); in LowerRotate()
30738 Amt = DAG.getNode(ISD::SUB, DL, VT, Z, Amt); in LowerRotate()
30749 Amt = DAG.getNode(ISD::SHL, DL, ExtVT, Amt, DAG.getConstant(5, DL, ExtVT)); in LowerRotate()
30754 M = DAG.getNode( in LowerRotate()
30756 DAG.getNode(ShiftLHS, DL, VT, R, DAG.getConstant(4, DL, VT)), in LowerRotate()
30757 DAG.getNode(ShiftRHS, DL, VT, R, DAG.getConstant(4, DL, VT))); in LowerRotate()
30761 Amt = DAG.getNode(ISD::ADD, DL, VT, Amt, Amt); in LowerRotate()
30764 M = DAG.getNode( in LowerRotate()
30766 DAG.getNode(ShiftLHS, DL, VT, R, DAG.getConstant(2, DL, VT)), in LowerRotate()
30767 DAG.getNode(ShiftRHS, DL, VT, R, DAG.getConstant(6, DL, VT))); in LowerRotate()
30771 Amt = DAG.getNode(ISD::ADD, DL, VT, Amt, Amt); in LowerRotate()
30774 M = DAG.getNode( in LowerRotate()
30776 DAG.getNode(ShiftLHS, DL, VT, R, DAG.getConstant(1, DL, VT)), in LowerRotate()
30777 DAG.getNode(ShiftRHS, DL, VT, R, DAG.getConstant(7, DL, VT))); in LowerRotate()
30782 bool ConstantAmt = ISD::isBuildVectorOfConstantSDNodes(Amt.getNode()); in LowerRotate()
30789 Amt = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask); in LowerRotate()
30791 AmtR = DAG.getNode(ISD::SUB, DL, VT, AmtR, Amt); in LowerRotate()
30792 SDValue SHL = DAG.getNode(IsROTL ? ISD::SHL : ISD::SRL, DL, VT, R, Amt); in LowerRotate()
30793 SDValue SRL = DAG.getNode(IsROTL ? ISD::SRL : ISD::SHL, DL, VT, R, AmtR); in LowerRotate()
30794 return DAG.getNode(ISD::OR, DL, VT, SHL, SRL); in LowerRotate()
30799 Amt = DAG.getNode(ISD::SUB, DL, VT, Z, Amt); in LowerRotate()
30804 Amt = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask); in LowerRotate()
30816 SDValue Lo = DAG.getNode(ISD::MUL, DL, VT, R, Scale); in LowerRotate()
30817 SDValue Hi = DAG.getNode(ISD::MULHU, DL, VT, R, Scale); in LowerRotate()
30818 return DAG.getNode(ISD::OR, DL, VT, Lo, Hi); in LowerRotate()
30829 SDValue Res02 = DAG.getNode(X86ISD::PMULUDQ, DL, MVT::v2i64, in LowerRotate()
30832 SDValue Res13 = DAG.getNode(X86ISD::PMULUDQ, DL, MVT::v2i64, in LowerRotate()
30838 return DAG.getNode(ISD::OR, DL, VT, in LowerRotate()
31148 return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0)); in LowerATOMIC_FENCE()
31155 return DAG.getNode(X86ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0)); in LowerATOMIC_FENCE()
31192 return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), in LowerCMP_SWAP()
31206 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Lo); in getPMOVMSKB()
31207 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Hi); in getPMOVMSKB()
31208 Hi = DAG.getNode(ISD::SHL, DL, MVT::i64, Hi, in getPMOVMSKB()
31210 return DAG.getNode(ISD::OR, DL, MVT::i64, Lo, Hi); in getPMOVMSKB()
31215 Lo = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Lo); in getPMOVMSKB()
31216 Hi = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Hi); in getPMOVMSKB()
31217 Hi = DAG.getNode(ISD::SHL, DL, MVT::i32, Hi, in getPMOVMSKB()
31219 return DAG.getNode(ISD::OR, DL, MVT::i32, Lo, Hi); in getPMOVMSKB()
31222 return DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V); in getPMOVMSKB()
31237 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Src, in LowerBITCAST()
31240 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Src, in LowerBITCAST()
31243 return DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, Lo, Hi); in LowerBITCAST()
31271 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewVT, Src, in LowerBITCAST()
31276 Src = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Src); in LowerBITCAST()
31280 Src = DAG.getNode(ISD::BITCAST, dl, V2X64VT, Src); in LowerBITCAST()
31283 return DAG.getNode(X86ISD::MOVDQ2Q, dl, DstVT, Src); in LowerBITCAST()
31285 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, DstVT, Src, in LowerBITCAST()
31313 V = DAG.getNode(X86ISD::PSADBW, DL, SadVecVT, V, Zeros); in LowerHorizontalByteSum()
31331 Low = DAG.getNode(X86ISD::PSADBW, DL, SadVecVT, in LowerHorizontalByteSum()
31333 High = DAG.getNode(X86ISD::PSADBW, DL, SadVecVT, in LowerHorizontalByteSum()
31338 V = DAG.getNode(X86ISD::PACKUS, DL, ByteVecVT, in LowerHorizontalByteSum()
31353 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, DAG.getBitcast(VT, V), ShifterV); in LowerHorizontalByteSum()
31354 V = DAG.getNode(ISD::ADD, DL, ByteVecVT, DAG.getBitcast(ByteVecVT, Shl), in LowerHorizontalByteSum()
31356 return DAG.getNode(ISD::SRL, DL, VT, DAG.getBitcast(VT, V), ShifterV); in LowerHorizontalByteSum()
31391 SDValue HiNibbles = DAG.getNode(ISD::SRL, DL, VT, Op, FourV); in LowerVectorCTPOPInRegLUT()
31394 SDValue LoNibbles = DAG.getNode(ISD::AND, DL, VT, Op, M0F); in LowerVectorCTPOPInRegLUT()
31399 SDValue HiPopCnt = DAG.getNode(X86ISD::PSHUFB, DL, VT, InRegLUT, HiNibbles); in LowerVectorCTPOPInRegLUT()
31400 SDValue LoPopCnt = DAG.getNode(X86ISD::PSHUFB, DL, VT, InRegLUT, LoNibbles); in LowerVectorCTPOPInRegLUT()
31401 return DAG.getNode(ISD::ADD, DL, VT, HiPopCnt, LoPopCnt); in LowerVectorCTPOPInRegLUT()
31411 SDLoc DL(Op.getNode()); in LowerVectorCTPOP()
31421 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, Op0); in LowerVectorCTPOP()
31422 Op = DAG.getNode(ISD::CTPOP, DL, NewVT, Op); in LowerVectorCTPOP()
31423 return DAG.getNode(ISD::TRUNCATE, DL, VT, Op); in LowerVectorCTPOP()
31439 SDValue PopCnt8 = DAG.getNode(ISD::CTPOP, DL, ByteVT, ByteOp); in LowerVectorCTPOP()
31466 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, In); in LowerBITREVERSE_XOP()
31467 Res = DAG.getNode(ISD::BITREVERSE, DL, VecVT, Res); in LowerBITREVERSE_XOP()
31468 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Res, in LowerBITREVERSE_XOP()
31497 Res = DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, DAG.getUNDEF(MVT::v16i8), in LowerBITREVERSE_XOP()
31532 return DAG.getNode(X86ISD::GF2P8AFFINEQB, DL, VT, In, Matrix, in LowerBITREVERSE()
31540 SDValue Lo = DAG.getNode(ISD::AND, DL, VT, In, NibbleMask); in LowerBITREVERSE()
31541 SDValue Hi = DAG.getNode(ISD::SRL, DL, VT, In, DAG.getConstant(4, DL, VT)); in LowerBITREVERSE()
31562 Lo = DAG.getNode(X86ISD::PSHUFB, DL, VT, LoMask, Lo); in LowerBITREVERSE()
31563 Hi = DAG.getNode(X86ISD::PSHUFB, DL, VT, HiMask, Hi); in LowerBITREVERSE()
31564 return DAG.getNode(ISD::OR, DL, VT, Lo, Hi); in LowerBITREVERSE()
31576 X = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, X); in LowerPARITY()
31577 SDValue Flags = DAG.getNode(X86ISD::CMP, DL, MVT::i32, X, in LowerPARITY()
31582 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Setnp); in LowerPARITY()
31591 SDValue Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, in LowerPARITY()
31592 DAG.getNode(ISD::SRL, DL, MVT::i64, X, in LowerPARITY()
31594 SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, X); in LowerPARITY()
31595 X = DAG.getNode(ISD::XOR, DL, MVT::i32, Lo, Hi); in LowerPARITY()
31600 SDValue Hi16 = DAG.getNode(ISD::SRL, DL, MVT::i32, X, in LowerPARITY()
31602 X = DAG.getNode(ISD::XOR, DL, MVT::i32, X, Hi16); in LowerPARITY()
31605 X = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, X); in LowerPARITY()
31610 SDValue Hi = DAG.getNode( in LowerPARITY()
31612 DAG.getNode(ISD::SRL, DL, MVT::i32, X, DAG.getConstant(8, DL, MVT::i8))); in LowerPARITY()
31613 SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, X); in LowerPARITY()
31615 SDValue Flags = DAG.getNode(X86ISD::XOR, DL, VTs, Lo, Hi).getValue(1); in LowerPARITY()
31620 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Setnp); in LowerPARITY()
31657 AtomicSDNode *AN = cast<AtomicSDNode>(N.getNode()); in lowerAtomicArith()
31672 RHS = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), RHS); in lowerAtomicArith()
31700 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), in lowerAtomicArith()
31704 SDValue NewChain = DAG.getNode(X86ISD::MEMBARRIER, DL, MVT::Other, Chain); in lowerAtomicArith()
31707 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), in lowerAtomicArith()
31715 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), in lowerAtomicArith()
31721 auto *Node = cast<AtomicSDNode>(Op.getNode()); in LowerATOMIC_STORE()
31743 SDValue SclToVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, in LowerATOMIC_STORE()
31755 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); in LowerATOMIC_STORE()
31798 SDNode *N = Op.getNode(); in LowerADDSUBCARRY()
31812 Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32), in LowerADDSUBCARRY()
31816 SDValue Sum = DAG.getNode(IsAdd ? X86ISD::ADC : X86ISD::SBB, DL, VTs, in LowerADDSUBCARRY()
31824 SetCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, SetCC); in LowerADDSUBCARRY()
31826 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Sum, SetCC); in LowerADDSUBCARRY()
31875 SDValue SinVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ArgVT, in LowerFSINCOS()
31877 SDValue CosVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ArgVT, in LowerFSINCOS()
31880 return DAG.getNode(ISD::MERGE_VALUES, dl, Tys, SinVal, CosVal); in LowerFSINCOS()
31907 if ((ISD::isBuildVectorAllZeros(N1.getNode()) && FillWithZeroes) || in ExtendToType()
31914 if (ISD::isBuildVectorOfConstantSDNodes(InOp.getNode()) || in ExtendToType()
31915 ISD::isBuildVectorOfConstantFPSDNodes(InOp.getNode())) { in ExtendToType()
31930 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, NVT, FillVal, in ExtendToType()
31939 MaskedScatterSDNode *N = cast<MaskedScatterSDNode>(Op.getNode()); in LowerMSCATTER()
31957 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT, Src, DAG.getUNDEF(VT)); in LowerMSCATTER()
32000 MaskedLoadSDNode *N = cast<MaskedLoadSDNode>(Op.getNode()); in LowerMLOAD()
32011 if (PassThru.isUndef() || ISD::isBuildVectorAllZeros(PassThru.getNode())) in LowerMLOAD()
32020 SDValue Select = DAG.getNode(ISD::VSELECT, dl, VT, Mask, NewLoad, PassThru); in LowerMLOAD()
32057 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, NewLoad.getValue(0), in LowerMLOAD()
32065 MaskedStoreSDNode *N = cast<MaskedStoreSDNode>(Op.getNode()); in LowerMSTORE()
32110 MaskedGatherSDNode *N = cast<MaskedGatherSDNode>(Op.getNode()); in LowerMGATHER()
32153 SDValue Extract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OrigVT, in LowerMGATHER()
32163 AddrSpaceCastSDNode *N = cast<AddrSpaceCastSDNode>(Op.getNode()); in LowerADDRSPACECAST()
32170 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, DstVT, Src); in LowerADDRSPACECAST()
32172 Op = DAG.getNode(ISD::SIGN_EXTEND, dl, DstVT, Src); in LowerADDRSPACECAST()
32174 Op = DAG.getNode(ISD::TRUNCATE, dl, DstVT, Src); in LowerADDRSPACECAST()
32203 std::tie(Lo, Hi) = DAG.SplitVectorOperand(Op.getNode(), 0); in LowerCVTPS2PH()
32207 Lo = DAG.getNode(X86ISD::CVTPS2PH, dl, LoVT, Lo, RC); in LowerCVTPS2PH()
32208 Hi = DAG.getNode(X86ISD::CVTPS2PH, dl, HiVT, Hi, RC); in LowerCVTPS2PH()
32209 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); in LowerCVTPS2PH()
32379 Lo = DAG.getNode(X86ISD::CVTPH2PS, dl, LoVT, Lo); in ReplaceNodeResults()
32380 Hi = DAG.getNode(X86ISD::CVTPH2PS, dl, HiVT, Hi); in ReplaceNodeResults()
32381 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); in ReplaceNodeResults()
32391 Lo = DAG.getNode(X86ISD::STRICT_CVTPH2PS, dl, {LoVT, MVT::Other}, in ReplaceNodeResults()
32393 Hi = DAG.getNode(X86ISD::STRICT_CVTPH2PS, dl, {HiVT, MVT::Other}, in ReplaceNodeResults()
32395 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, in ReplaceNodeResults()
32397 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); in ReplaceNodeResults()
32413 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, N->getOperand(0)); in ReplaceNodeResults()
32414 Wide = DAG.getNode(ISD::CTPOP, dl, MVT::v2i64, Wide); in ReplaceNodeResults()
32417 Wide = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Wide); in ReplaceNodeResults()
32418 Wide = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, Wide, in ReplaceNodeResults()
32420 Wide = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, Wide); in ReplaceNodeResults()
32432 SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, dl, MulVT, N->getOperand(0)); in ReplaceNodeResults()
32433 SDValue Op1 = DAG.getNode(ISD::ANY_EXTEND, dl, MulVT, N->getOperand(1)); in ReplaceNodeResults()
32434 SDValue Res = DAG.getNode(ISD::MUL, dl, MulVT, Op0, Op1); in ReplaceNodeResults()
32435 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in ReplaceNodeResults()
32439 Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i8, ConcatOps); in ReplaceNodeResults()
32450 SDValue Op0 = DAG.getNode(ExtOpc, dl, MVT::v2i64, N->getOperand(0)); in ReplaceNodeResults()
32451 SDValue Op1 = DAG.getNode(ExtOpc, dl, MVT::v2i64, N->getOperand(1)); in ReplaceNodeResults()
32452 SDValue Res = DAG.getNode(ISD::MUL, dl, MVT::v2i64, Op0, Op1); in ReplaceNodeResults()
32457 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Hi, in ReplaceNodeResults()
32461 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in ReplaceNodeResults()
32466 HiCmp = DAG.getNode(ISD::SRA, dl, VT, Res, DAG.getConstant(31, dl, VT)); in ReplaceNodeResults()
32474 Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Res, in ReplaceNodeResults()
32501 SDValue InVec0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWideVT, Ops); in ReplaceNodeResults()
32503 SDValue InVec1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWideVT, Ops); in ReplaceNodeResults()
32505 SDValue Res = DAG.getNode(N->getOpcode(), dl, WideVT, InVec0, InVec1); in ReplaceNodeResults()
32517 SDValue LHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, in ReplaceNodeResults()
32519 SDValue RHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, in ReplaceNodeResults()
32521 Results.push_back(DAG.getNode(N->getOpcode(), dl, MVT::v4f32, LHS, RHS)); in ReplaceNodeResults()
32536 if (ISD::isConstantSplatVector(N->getOperand(1).getNode(), SplatVal)) { in ReplaceNodeResults()
32541 SDValue N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Ops0); in ReplaceNodeResults()
32543 SDValue Res = DAG.getNode(N->getOpcode(), dl, ResVT, N0, N1); in ReplaceNodeResults()
32578 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, In, in ReplaceNodeResults()
32580 Ops[i] = DAG.getNode(ISD::TRUNCATE, dl, EltVT, Val); in ReplaceNodeResults()
32591 Results.push_back(DAG.getNode(X86ISD::VTRUNC, dl, WidenVT, In)); in ReplaceNodeResults()
32596 In = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i64, In, in ReplaceNodeResults()
32598 Results.push_back(DAG.getNode(X86ISD::VTRUNC, dl, WidenVT, In)); in ReplaceNodeResults()
32610 Lo = DAG.getNode(X86ISD::VTRUNC, dl, MVT::v16i8, Lo); in ReplaceNodeResults()
32611 Hi = DAG.getNode(X86ISD::VTRUNC, dl, MVT::v16i8, Hi); in ReplaceNodeResults()
32641 In = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, In); in ReplaceNodeResults()
32651 Lo = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, Lo); in ReplaceNodeResults()
32654 Hi = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, Hi); in ReplaceNodeResults()
32656 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); in ReplaceNodeResults()
32673 In = DAG.getNode(N->getOpcode(), dl, InVT, In); in ReplaceNodeResults()
32694 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); in ReplaceNodeResults()
32716 DAG.getNode(N->getOpcode(), dl, {VT, MVT::Other}, in ReplaceNodeResults()
32717 {Chain, DAG.getNode(ISD::STRICT_FP_EXTEND, dl, in ReplaceNodeResults()
32721 Res = DAG.getNode(N->getOpcode(), dl, VT, in ReplaceNodeResults()
32722 DAG.getNode(ISD::FP_EXTEND, dl, NVT, Src)); in ReplaceNodeResults()
32741 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8f16, Ops); in ReplaceNodeResults()
32748 DAG.getNode(Opc, dl, {ResVT, MVT::Other}, {N->getOperand(0), Src}); in ReplaceNodeResults()
32752 Res = DAG.getNode(Opc, dl, ResVT, Src); in ReplaceNodeResults()
32758 Res = DAG.getNode(ISD::TRUNCATE, dl, TmpVT, Res); in ReplaceNodeResults()
32765 Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, ConcatVT, ConcatOps); in ReplaceNodeResults()
32786 Res = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, {PromoteVT, MVT::Other}, in ReplaceNodeResults()
32790 Res = DAG.getNode(ISD::FP_TO_SINT, dl, PromoteVT, Src); in ReplaceNodeResults()
32795 Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Res, in ReplaceNodeResults()
32798 Res = DAG.getNode(!IsSigned ? ISD::AssertZext : ISD::AssertSext, dl, in ReplaceNodeResults()
32803 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i32, Res, in ReplaceNodeResults()
32807 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in ReplaceNodeResults()
32815 Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, ConcatVT, ConcatOps); in ReplaceNodeResults()
32852 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f64, Src, in ReplaceNodeResults()
32859 Res = DAG.getNode(Opc, dl, {MVT::v4i32, MVT::Other}, in ReplaceNodeResults()
32863 Res = DAG.getNode(Opc, dl, MVT::v4i32, Src); in ReplaceNodeResults()
32874 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src, in ReplaceNodeResults()
32876 SDValue Res = DAG.getNode(N->getOpcode(), dl, {MVT::v4i32, MVT::Other}, in ReplaceNodeResults()
32909 SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VecInVT, in ReplaceNodeResults()
32915 Res = DAG.getNode(Opc, SDLoc(N), Tys, N->getOperand(0), Res); in ReplaceNodeResults()
32918 Res = DAG.getNode(Opc, SDLoc(N), VecVT, Res); in ReplaceNodeResults()
32919 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Res, ZeroIdx); in ReplaceNodeResults()
32964 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src, in ReplaceNodeResults()
32970 SDValue Res = DAG.getNode(Opc, dl, {MVT::v8f16, MVT::Other}, in ReplaceNodeResults()
32976 Results.push_back(DAG.getNode(Opc, dl, MVT::v8f16, Src)); in ReplaceNodeResults()
32987 SDValue Res = DAG.getNode(Opc, dl, {MVT::v4f32, MVT::Other}, in ReplaceNodeResults()
32993 Results.push_back(DAG.getNode(Opc, dl, MVT::v4f32, Src)); in ReplaceNodeResults()
33001 SDValue Sign = DAG.getNode(ISD::OR, dl, SrcVT, in ReplaceNodeResults()
33002 DAG.getNode(ISD::SRL, dl, SrcVT, Src, One), in ReplaceNodeResults()
33003 DAG.getNode(ISD::AND, dl, SrcVT, Src, One)); in ReplaceNodeResults()
33008 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, in ReplaceNodeResults()
33012 DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {MVT::f32, MVT::Other}, in ReplaceNodeResults()
33015 SignCvts[i] = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Elt); in ReplaceNodeResults()
33020 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, in ReplaceNodeResults()
33022 Slow = DAG.getNode(ISD::STRICT_FADD, dl, {MVT::v4f32, MVT::Other}, in ReplaceNodeResults()
33026 Slow = DAG.getNode(ISD::FADD, dl, MVT::v4f32, SignCvt, SignCvt); in ReplaceNodeResults()
33047 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src, in ReplaceNodeResults()
33049 SDValue Res = DAG.getNode(N->getOpcode(), dl, {MVT::v4f32, MVT::Other}, in ReplaceNodeResults()
33057 SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v2i64, Src); in ReplaceNodeResults()
33060 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::v2i64, ZExtIn, in ReplaceNodeResults()
33064 SDValue Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::v2f64, MVT::Other}, in ReplaceNodeResults()
33066 SDValue Res = DAG.getNode(X86ISD::STRICT_VFPROUND, dl, in ReplaceNodeResults()
33073 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, Or, VBias); in ReplaceNodeResults()
33074 Results.push_back(DAG.getNode(X86ISD::VFPROUND, dl, MVT::v4f32, Sub)); in ReplaceNodeResults()
33090 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src, Ext); in ReplaceNodeResults()
33098 V = DAG.getNode(X86ISD::STRICT_CVTPS2PH, dl, {MVT::v8i16, MVT::Other}, in ReplaceNodeResults()
33101 V = DAG.getNode(X86ISD::CVTPS2PH, dl, MVT::v8i16, Src, Rnd); in ReplaceNodeResults()
33112 V = DAG.getNode(X86ISD::STRICT_VFPROUND, dl, {NewVT, MVT::Other}, in ReplaceNodeResults()
33115 V = DAG.getNode(X86ISD::VFPROUND, dl, NewVT, Src); in ReplaceNodeResults()
33133 SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f16, Src, Ext); in ReplaceNodeResults()
33135 V = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {MVT::v4f32, MVT::Other}, in ReplaceNodeResults()
33138 V = DAG.getNode(ISD::FP_EXTEND, dl, MVT::v4f32, V); in ReplaceNodeResults()
33180 cpInL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(2), in ReplaceNodeResults()
33182 cpInH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(2), in ReplaceNodeResults()
33191 swapInL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(3), in ReplaceNodeResults()
33193 swapInH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(3), in ReplaceNodeResults()
33235 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, T, OpsF)); in ReplaceNodeResults()
33256 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Ld, in ReplaceNodeResults()
33265 SDValue Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2f32, Ld, in ReplaceNodeResults()
33287 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); in ReplaceNodeResults()
33337 SDValue Res = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi); in ReplaceNodeResults()
33348 SDValue Res = DAG.getNode(X86ISD::MOVQ2DQ, dl, MVT::v2i64, in ReplaceNodeResults()
33370 SDValue PassThru = DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT, in ReplaceNodeResults()
33376 Mask = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i1, Mask, in ReplaceNodeResults()
33378 Mask = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, Mask); in ReplaceNodeResults()
33409 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, Res); in ReplaceNodeResults()
36680 if (!ISD::isBuildVectorOfConstantSDNodes(V.getNode())) in targetShrinkDemandedConstant()
36702 TLO.DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(Op), VT, in targetShrinkDemandedConstant()
36705 TLO.DAG.getNode(Opcode, SDLoc(Op), VT, Op.getOperand(0), NewC); in targetShrinkDemandedConstant()
36754 SDValue NewOp = TLO.DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), NewC); in targetShrinkDemandedConstant()
37032 if (getTargetShuffleMask(Op.getNode(), VT.getSimpleVT(), true, Ops, Mask)) { in computeKnownBitsForTargetNode()
37183 if (getTargetShuffleMask(Op.getNode(), VT.getSimpleVT(), true, Ops, Mask)) { in ComputeNumSignBitsForTargetNode()
38013 return DAG.getNode(X86ISD::SHUF128, DL, ShuffleVT, in combineX86ShuffleChain()
38080 return DAG.getNode( in combineX86ShuffleChain()
38099 return DAG.getNode(X86ISD::VPERM2X128, DL, RootVT, in combineX86ShuffleChain()
38170 Res = DAG.getNode(X86ISD::VBROADCAST, DL, MaskVT, Res); in combineX86ShuffleChain()
38177 Res = DAG.getNode(X86ISD::VBROADCAST, DL, MaskVT, Res); in combineX86ShuffleChain()
38190 Res = DAG.getNode(Shuffle, DL, ShuffleVT, Res); in combineX86ShuffleChain()
38202 Res = DAG.getNode(Shuffle, DL, ShuffleVT, Res, in combineX86ShuffleChain()
38221 Res = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, in combineX86ShuffleChain()
38235 Res = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, in combineX86ShuffleChain()
38253 Res = DAG.getNode(Shuffle, DL, ShuffleVT, NewV1, NewV2); in combineX86ShuffleChain()
38267 Res = DAG.getNode(Shuffle, DL, ShuffleVT, NewV1, NewV2, in combineX86ShuffleChain()
38284 Res = DAG.getNode(X86ISD::EXTRQI, DL, IntMaskVT, V1, in combineX86ShuffleChain()
38295 Res = DAG.getNode(X86ISD::INSERTQI, DL, IntMaskVT, V1, V2, in combineX86ShuffleChain()
38314 Res = DAG.getNode(Opc, DL, ShuffleVT, V1); in combineX86ShuffleChain()
38338 Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, ShuffleSrcVT, V1, V2); in combineX86ShuffleChain()
38339 Res = DAG.getNode(ISD::TRUNCATE, DL, IntMaskVT, Res); in combineX86ShuffleChain()
38372 Res = DAG.getNode(X86ISD::VPERMV, DL, MaskVT, VPermMask, Res); in combineX86ShuffleChain()
38463 Res = DAG.getNode(AndOpcode, DL, MaskVT, Res, BitMask); in combineX86ShuffleChain()
38481 Res = DAG.getNode(X86ISD::VPERMILPV, DL, MaskVT, Res, VPermMask); in combineX86ShuffleChain()
38515 Res = DAG.getNode(X86ISD::VPERMIL2, DL, MaskVT, V1, V2, VPerm2MaskOp, in combineX86ShuffleChain()
38549 Res = DAG.getNode(X86ISD::PSHUFB, DL, ByteVT, Res, PSHUFBMaskOp); in combineX86ShuffleChain()
38581 Res = DAG.getNode(X86ISD::VPPERM, DL, ByteVT, V1, V2, VPPERMMaskOp); in combineX86ShuffleChain()
38785 if (Src1.getOpcode() == Opcode0 && Src0->isOnlyUserOf(Src1.getNode())) in canonicalizeShuffleMaskWithHorizOp()
38794 SDValue LHS = DAG.getNode(Opcode0, DL, SrcVT, M0, M1); in canonicalizeShuffleMaskWithHorizOp()
38795 SDValue RHS = DAG.getNode(Opcode0, DL, SrcVT, M2, M3); in canonicalizeShuffleMaskWithHorizOp()
38796 return DAG.getNode(Opcode0, DL, VT0, LHS, RHS); in canonicalizeShuffleMaskWithHorizOp()
38826 SDValue Res = DAG.getNode(Opcode0, DL, VT0, LHS, RHS); in canonicalizeShuffleMaskWithHorizOp()
38831 return DAG.getNode(X86ISD::SHUFP, DL, ShuffleVT, Res, Res, in canonicalizeShuffleMaskWithHorizOp()
38908 return DAG.getNode(Opcode0, DL, VT0, Lo, Hi); in canonicalizeShuffleMaskWithHorizOp()
39267 if (Ops.size() == 1 && ISD::isBuildVectorAllOnes(Ops[0].getNode()) && in combineX86ShufflesRecursively()
39277 CombinedNodes.push_back(Op.getNode()); in combineX86ShufflesRecursively()
39295 if (Ops[i].getNode()->hasOneUse() || in combineX86ShufflesRecursively()
39296 SDNode::areOnlyUsersOf(CombinedNodes, Ops[i].getNode())) { in combineX86ShufflesRecursively()
39440 getTargetShuffleMask(N.getNode(), VT, false, Ops, Mask); in getPSHUFShuffleMask()
39536 !V->isOnlyUserOf(V.getOperand(0).getNode())) in combineRedundantDWordShuffle()
39573 V = DAG.getNode(V.getOpcode(), DL, V.getValueType(), V.getOperand(0), in combineRedundantDWordShuffle()
39589 V = DAG.getNode(W.getOpcode(), DL, W.getValueType(), V, V); in combineRedundantDWordShuffle()
39595 V = DAG.getNode(W.getOpcode(), DL, W.getValueType(), V, W.getOperand(1)); in combineRedundantDWordShuffle()
39616 if (V.getOpcode() != X86ISD::SHUFP || !Parent->isOnlyUserOf(V.getNode())) in combineCommutableSHUFP()
39626 return DAG.getNode(X86ISD::SHUFP, DL, VT, N1, N0, in combineCommutableSHUFP()
39634 return DAG.getNode(X86ISD::VPERMILPI, DL, VT, NewSHUFP, in combineCommutableSHUFP()
39644 return DAG.getNode(X86ISD::SHUFP, DL, VT, NewSHUFP, NewSHUFP, in combineCommutableSHUFP()
39647 return DAG.getNode(X86ISD::SHUFP, DL, VT, NewSHUFP, N1, in combineCommutableSHUFP()
39650 return DAG.getNode(X86ISD::SHUFP, DL, VT, N0, NewSHUFP, in combineCommutableSHUFP()
39671 return ISD::isBuildVectorAllOnes(Op.getNode()) || in canonicalizeShuffleWithBinOps()
39672 ISD::isBuildVectorAllZeros(Op.getNode()) || in canonicalizeShuffleWithBinOps()
39673 ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) || in canonicalizeShuffleWithBinOps()
39674 ISD::isBuildVectorOfConstantFPSDNodes(Op.getNode()) || in canonicalizeShuffleWithBinOps()
39694 if (!getTargetShuffleMask(N.getNode(), ShuffleVT.getSimpleVT(), false, Ops, in canonicalizeShuffleWithBinOps()
39707 N->isOnlyUserOf(N.getOperand(0).getNode())) { in canonicalizeShuffleWithBinOps()
39719 LHS = DAG.getNode(Opc, DL, ShuffleVT, Op00, N.getOperand(1)); in canonicalizeShuffleWithBinOps()
39720 RHS = DAG.getNode(Opc, DL, ShuffleVT, Op01, N.getOperand(1)); in canonicalizeShuffleWithBinOps()
39722 LHS = DAG.getNode(Opc, DL, ShuffleVT, Op00); in canonicalizeShuffleWithBinOps()
39723 RHS = DAG.getNode(Opc, DL, ShuffleVT, Op01); in canonicalizeShuffleWithBinOps()
39727 DAG.getNode(SrcOpcode, DL, OpVT, in canonicalizeShuffleWithBinOps()
39750 if (N->isOnlyUserOf(N.getOperand(0).getNode()) && in canonicalizeShuffleWithBinOps()
39751 N->isOnlyUserOf(N.getOperand(1).getNode())) { in canonicalizeShuffleWithBinOps()
39774 LHS = DAG.getNode(Opc, DL, ShuffleVT, Op00, Op10, N.getOperand(2)); in canonicalizeShuffleWithBinOps()
39775 RHS = DAG.getNode(Opc, DL, ShuffleVT, Op01, Op11, N.getOperand(2)); in canonicalizeShuffleWithBinOps()
39777 LHS = DAG.getNode(Opc, DL, ShuffleVT, Op00, Op10); in canonicalizeShuffleWithBinOps()
39778 RHS = DAG.getNode(Opc, DL, ShuffleVT, Op01, Op11); in canonicalizeShuffleWithBinOps()
39782 DAG.getNode(SrcOpcode, DL, OpVT, in canonicalizeShuffleWithBinOps()
39816 DAG.getNode(X86ISD::VPERM2X128, DL, SrcVT0, LHS, RHS, V.getOperand(2)); in canonicalizeLaneShuffleWithRepeatedOps()
39817 Res = DAG.getNode(SrcOpc0, DL, SrcVT0, Res); in canonicalizeLaneShuffleWithRepeatedOps()
39835 SDValue Res = DAG.getNode(X86ISD::VPERM2X128, DL, SrcVT0, LHS, RHS, in canonicalizeLaneShuffleWithRepeatedOps()
39837 Res = DAG.getNode(SrcOpc0, DL, SrcVT0, Res, Src0.getOperand(1)); in canonicalizeLaneShuffleWithRepeatedOps()
39864 ISD::isNormalLoad(Src.getNode())) { in combineTargetShuffle()
39867 SDValue Movddup = DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v2f64, VZLoad); in combineTargetShuffle()
39868 DCI.CombineTo(N.getNode(), Movddup); in combineTargetShuffle()
39897 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, in combineTargetShuffle()
39910 return DAG.getBitcast(VT, DAG.getNode(X86ISD::VBROADCAST, DL, NewVT, BC)); in combineTargetShuffle()
39915 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, in combineTargetShuffle()
39920 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, Src.getOperand(0)); in combineTargetShuffle()
39927 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, Src.getOperand(0)); in combineTargetShuffle()
39932 if (User != N.getNode() && User->getOpcode() == X86ISD::VBROADCAST && in combineTargetShuffle()
39943 ISD::isNormalLoad(Src.getNode())) { in combineTargetShuffle()
39952 DCI.CombineTo(N.getNode(), BcastLd); in combineTargetShuffle()
39957 SDValue Scl = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT, BcastLd, in combineTargetShuffle()
39973 if (ISD::isNormalLoad(TruncIn.getNode())) { in combineTargetShuffle()
39983 DCI.CombineTo(N.getNode(), BcastLd); in combineTargetShuffle()
39985 DCI.recursivelyDeleteUnusedNodes(Src.getNode()); in combineTargetShuffle()
39991 if (ISD::isUNINDEXEDLoad(Src.getOperand(0).getNode()) && in combineTargetShuffle()
39992 ISD::isEXTLoad(Src.getOperand(0).getNode())) { in combineTargetShuffle()
40000 DCI.CombineTo(N.getNode(), BcastLd); in combineTargetShuffle()
40002 DCI.recursivelyDeleteUnusedNodes(Src.getNode()); in combineTargetShuffle()
40012 ISD::isNormalLoad(TruncIn.getOperand(0).getNode())) { in combineTargetShuffle()
40029 DCI.CombineTo(N.getNode(), BcastLd); in combineTargetShuffle()
40031 DCI.recursivelyDeleteUnusedNodes(Src.getNode()); in combineTargetShuffle()
40046 DCI.CombineTo(N.getNode(), BcastLd); in combineTargetShuffle()
40056 Src.hasOneUse() && ISD::isNormalLoad(Src.getNode())) { in combineTargetShuffle()
40066 DCI.CombineTo(N.getNode(), BcastLd); in combineTargetShuffle()
40080 if (N0.hasOneUse() && ISD::isNormalLoad(N0.getNode())) { in combineTargetShuffle()
40084 DCI.CombineTo(N.getNode(), VZLoad); in combineTargetShuffle()
40102 DCI.CombineTo(N.getNode(), VZLoad); in combineTargetShuffle()
40118 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, In); in combineTargetShuffle()
40120 SDValue SclVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Trunc); in combineTargetShuffle()
40121 SDValue Movl = DAG.getNode(X86ISD::VZEXT_MOVL, DL, VecVT, SclVec); in combineTargetShuffle()
40164 SDValue Movl = DAG.getNode(X86ISD::VZEXT_MOVL, DL, SubVT, In); in combineTargetShuffle()
40165 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, in combineTargetShuffle()
40189 VT, DAG.getNode(X86ISD::BLENDI, DL, SrcVT, N0.getOperand(0), in combineTargetShuffle()
40204 if (getTargetShuffleMask(N.getNode(), VT, false, Ops, Mask) && in combineTargetShuffle()
40226 return DAG.getNode(X86ISD::SHUFP, DL, VT, Ops); in combineTargetShuffle()
40241 SDValue Res = DAG.getNode(X86ISD::VPERMI, DL, SrcVT, Src, N1); in combineTargetShuffle()
40254 return DAG.getBitcast(VT, DAG.getNode(X86ISD::VPERM2X128, DL, SrcVT, in combineTargetShuffle()
40272 if (collectConcatOps(Src.getNode(), SubOps, DAG) && SubOps.size() == 2) in combineTargetShuffle()
40288 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, SubLo, SubHi); in combineTargetShuffle()
40319 N10 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SVT, N10, ZeroIdx); in combineTargetShuffle()
40320 N11 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SVT, N11, ZeroIdx); in combineTargetShuffle()
40321 SDValue Scl = DAG.getNode(Opcode1, DL, SVT, N10, N11); in combineTargetShuffle()
40322 SDValue SclVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Scl); in combineTargetShuffle()
40323 return DAG.getNode(Opcode, DL, VT, N0, SclVec); in combineTargetShuffle()
40340 return DAG.getNode(X86ISD::INSERTPS, DL, VT, DAG.getUNDEF(VT), Op1, in combineTargetShuffle()
40345 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, DAG.getUNDEF(VT), in combineTargetShuffle()
40357 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, DAG.getUNDEF(VT), in combineTargetShuffle()
40365 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, Op1, in combineTargetShuffle()
40412 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, Op1, in combineTargetShuffle()
40425 SDValue Insert = DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, in combineTargetShuffle()
40426 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, in combineTargetShuffle()
40463 V = DAG.getNode(X86ISD::PSHUFD, DL, DVT, V, in combineTargetShuffle()
40495 return DAG.getNode(MappedMask[0] == 0 ? X86ISD::UNPCKL in combineTargetShuffle()
40651 return DAG.getNode(Opcode, DL, VT, FMAdd.getOperand(0), FMAdd.getOperand(1), in combineShuffleToFMAddSub()
40675 return DAG.getNode(Opc, DL, VT, Opnd0, Opnd1, Opnd2); in combineShuffleToAddSubOrFMAddSub()
40693 return DAG.getNode(X86ISD::ADDSUB, DL, VT, Opnd0, Opnd1); in combineShuffleToAddSubOrFMAddSub()
40736 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, N0.getOperand(0), in combineShuffleOfConcatUndef()
40974 Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewLHS, NewRHS)); in SimplifyDemandedVectorEltsForTargetNode()
41023 Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc, Op.getOperand(1))); in SimplifyDemandedVectorEltsForTargetNode()
41076 Op, TLO.DAG.getNode(NewOpc, dl, VT, Src.getOperand(0), NewSA)); in SimplifyDemandedVectorEltsForTargetNode()
41115 Op, TLO.DAG.getNode(NewOpc, dl, VT, Src.getOperand(0), NewSA)); in SimplifyDemandedVectorEltsForTargetNode()
41183 Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewLHS, NewRHS)); in SimplifyDemandedVectorEltsForTargetNode()
41229 TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewN0, NewN1)); in SimplifyDemandedVectorEltsForTargetNode()
41266 TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewN0, NewN1)); in SimplifyDemandedVectorEltsForTargetNode()
41336 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc)); in SimplifyDemandedVectorEltsForTargetNode()
41380 SDValue Bcst = TLO.DAG.getNode(X86ISD::VBROADCAST, DL, BcstVT, Src); in SimplifyDemandedVectorEltsForTargetNode()
41439 TLO.DAG.getNode(Opc, DL, Ext0.getValueType(), Ext0, Op.getOperand(1)); in SimplifyDemandedVectorEltsForTargetNode()
41513 SDValue ExtOp = TLO.DAG.getNode(Opc, DL, ExtVT, Ops); in SimplifyDemandedVectorEltsForTargetNode()
41670 Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, DemandedLHS, DemandedRHS)); in SimplifyDemandedBitsForTargetNode()
41695 SDValue NewShift = TLO.DAG.getNode( in SimplifyDemandedBitsForTargetNode()
41783 Op, TLO.DAG.getNode(X86ISD::VSRLI, SDLoc(Op), VT, Op0, Op1)); in SimplifyDemandedBitsForTargetNode()
41807 return TLO.CombineTo(Op, TLO.DAG.getNode(X86ISD::BLENDV, SDLoc(Op), VT, in SimplifyDemandedBitsForTargetNode()
41843 Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, V, Op.getOperand(1))); in SimplifyDemandedBitsForTargetNode()
41906 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, Op0, Op1)); in SimplifyDemandedBitsForTargetNode()
41927 TLO.DAG.getNode(ISD::TRUNCATE, SDLoc(Src), NewSrcVT, Src); in SimplifyDemandedBitsForTargetNode()
41930 TLO.DAG.getNode(X86ISD::VBROADCAST, SDLoc(Op), NewVT, NewSrc); in SimplifyDemandedBitsForTargetNode()
41939 ISD::isBuildVectorAllZeros(Op.getOperand(0).getNode())) in SimplifyDemandedBitsForTargetNode()
41956 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc)); in SimplifyDemandedBitsForTargetNode()
41984 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc)); in SimplifyDemandedBitsForTargetNode()
42000 Op, TLO.DAG.getNode(X86ISD::BEXTR, DL, VT, Op0, in SimplifyDemandedBitsForTargetNode()
42112 ISD::isBuildVectorAllZeros(Op.getOperand(0).getNode())) in SimplifyMultipleUseDemandedBitsForTargetNode()
42222 ISD::isBuildVectorAllZeros(Src.getOperand(1).getNode()) && in adjustBitcastSrcVectorSSE1()
42225 if (ISD::isNormalLoad(Op0.getNode())) in adjustBitcastSrcVectorSSE1()
42238 return DAG.getNode(getAltBitOpcode(Src.getOpcode()), DL, MVT::v4f32, Op0, in adjustBitcastSrcVectorSSE1()
42252 return DAG.getNode(ISD::SIGN_EXTEND, DL, SExtVT, Src); in signExtendBitcastSrcVector()
42256 return DAG.getNode( in signExtendBitcastSrcVector()
42281 V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, in combineBitcastvxi1()
42300 ISD::isBuildVectorAllZeros(Src.getOperand(1).getNode())) { in combineBitcastvxi1()
42382 : DAG.getNode(ISD::SIGN_EXTEND, DL, SExtVT, Src); in combineBitcastvxi1()
42388 V = DAG.getNode(X86ISD::PACKSS, DL, MVT::v16i8, V, in combineBitcastvxi1()
42390 V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V); in combineBitcastvxi1()
42404 assert(ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) && in combinevXi1ConstantToInteger()
42454 return DAG.getNode(Op.getOpcode(), SDLoc(N), DstVT, LHS.getOperand(0), in combineCastedMaskArithmetic()
42459 return DAG.getNode(Op.getOpcode(), SDLoc(N), DstVT, in combineCastedMaskArithmetic()
42464 if (ISD::isBuildVectorOfConstantSDNodes(RHS.getNode())) { in combineCastedMaskArithmetic()
42466 return DAG.getNode(Op.getOpcode(), SDLoc(N), DstVT, in combineCastedMaskArithmetic()
42485 V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4f32, V); in createMMXBuildVector()
42487 return DAG.getNode(X86ISD::MOVDQ2Q, DL, MVT::x86mmx, V); in createMMXBuildVector()
42493 return DAG.getNode(X86ISD::MMX_MOVW2D, DL, MVT::x86mmx, V); in createMMXBuildVector()
42511 Splat = DAG.getNode( in createMMXBuildVector()
42519 return DAG.getNode( in createMMXBuildVector()
42541 Ops[i / 2] = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::x86mmx, Intrin, in createMMXBuildVector()
42575 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, N0, in combineBitcastToBoolVector()
42588 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, in combineBitcastToBoolVector()
42600 return DAG.getNode(Opc, DL, VT, N0, N1); in combineBitcastToBoolVector()
42612 return DAG.getNode( in combineBitcastToBoolVector()
42644 N0 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i8, N0); in combineBitcast()
42646 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, N0, in combineBitcast()
42663 if (ISD::isBuildVectorAllZeros(LastOp.getNode())) { in combineBitcast()
42668 N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i1, Ops); in combineBitcast()
42670 return DAG.getNode(ISD::TRUNCATE, dl, VT, N0); in combineBitcast()
42677 N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i1, Ops); in combineBitcast()
42679 return DAG.getNode(ISD::TRUNCATE, dl, VT, N0); in combineBitcast()
42701 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, in combineBitcast()
42741 return DAG.getNode(X86ISD::MMX_MOVW2D, DL, VT, in combineBitcast()
42764 return DAG.getNode(X86ISD::MMX_MOVW2D, dl, VT, N00); in combineBitcast()
42782 return DAG.getNode(X86ISD::MOVDQ2Q, SDLoc(N00), VT, in combineBitcast()
42789 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i32, N0, in combineBitcast()
42791 return DAG.getNode(X86ISD::MOVDQ2Q, DL, VT, in combineBitcast()
42800 ISD::isBuildVectorOfConstantSDNodes(N0.getNode())) { in combineBitcast()
42850 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Ops); in combineBitcast()
42892 return DAG.getNode(Opcode, DL0, VT, LogicOp0.getOperand(0), CastedOp1); in combineBitcast()
42901 return DAG.getNode(Opcode, DL0, VT, LogicOp1.getOperand(0), CastedOp0); in combineBitcast()
42985 SDValue DpOp0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops); in createVPDPBUSD()
42987 SDValue DpOp1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops); in createVPDPBUSD()
42994 return DAG.getNode(X86ISD::VPDPBUSD, DL, VT, Ops); in createVPDPBUSD()
43018 SDValue SadOp0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops); in createPSADBW()
43020 SDValue SadOp1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops); in createPSADBW()
43026 return DAG.getNode(X86ISD::PSADBW, DL, VT, Ops); in createPSADBW()
43065 MinPos = DAG.getNode(BinOp, DL, SrcVT, Lo, Hi); in combineMinMaxReduction()
43083 MinPos = DAG.getNode(ISD::XOR, DL, SrcVT, Mask, MinPos); in combineMinMaxReduction()
43093 MinPos = DAG.getNode(ISD::UMIN, DL, SrcVT, MinPos, Upper); in combineMinMaxReduction()
43098 MinPos = DAG.getNode(X86ISD::PHMINPOS, DL, MVT::v8i16, MinPos); in combineMinMaxReduction()
43102 MinPos = DAG.getNode(ISD::XOR, DL, SrcVT, Mask, MinPos); in combineMinMaxReduction()
43104 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractVT, MinPos, in combineMinMaxReduction()
43169 Match = DAG.getNode(BinOp, DL, Lo.getValueType(), Lo, Hi); in combinePredicateReduction()
43199 Match = DAG.getNode(BinOp, DL, Lo.getValueType(), Lo, Hi); in combinePredicateReduction()
43221 SDValue Result = DAG.getNode(ISD::PARITY, DL, CmpVT, Movmsk); in combinePredicateReduction()
43245 return DAG.getNode(ISD::SUB, DL, ExtractVT, Zero, Zext); in combinePredicateReduction()
43306 DP = DAG.getNode(ISD::ADD, DL, DpVT, DP, Shuffle); in combineVPDPBUSDPattern()
43315 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractVT, DP, in combineVPDPBUSDPattern()
43380 SAD = DAG.getNode(ISD::ADD, DL, SadVT, SAD, Shuffle); in combineBasicSADPattern()
43389 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractVT, SAD, in combineBasicSADPattern()
43468 Scl = DAG.getNode(ISD::SRL, dl, SclVT, Scl, in combineExtractWithShuffle()
43484 return DAG.getNode(N->getOpcode(), dl, VT, DAG.getBitcast(ExtractVT, Src), in combineExtractWithShuffle()
43507 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VecVT.getScalarType(), in combineExtractWithShuffle()
43514 return DAG.getNode(OpCode, dl, MVT::i32, DAG.getBitcast(VecVT, Vec), in combineExtractWithShuffle()
43616 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpVT, in scalarizeExtEltFP()
43618 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpVT, in scalarizeExtEltFP()
43620 return DAG.getNode(Vec.getOpcode(), DL, VT, Ext0, Ext1, Vec.getOperand(2)); in scalarizeExtEltFP()
43639 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, in scalarizeExtEltFP()
43642 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, in scalarizeExtEltFP()
43644 SDValue Ext2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, in scalarizeExtEltFP()
43646 return DAG.getNode(ISD::SELECT, DL, VT, Ext0, Ext1, Ext2); in scalarizeExtEltFP()
43683 ExtOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op, Index)); in scalarizeExtEltFP()
43684 return DAG.getNode(Vec.getOpcode(), DL, VT, ExtOps); in scalarizeExtEltFP()
43725 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v4i32, in combineArithReduction()
43731 V = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i8, V, in combineArithReduction()
43735 return DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, V, in combineArithReduction()
43749 Rdx = DAG.getNode(Opc, DL, WideVT, Lo, Hi); in combineArithReduction()
43752 Rdx = DAG.getNode(Opc, DL, Lo.getValueType(), Lo, Hi); in combineArithReduction()
43760 Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx, in combineArithReduction()
43763 Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx, in combineArithReduction()
43766 Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx, in combineArithReduction()
43770 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index); in combineArithReduction()
43776 Rdx = DAG.getNode(X86ISD::PSADBW, DL, MVT::v2i64, Rdx, in combineArithReduction()
43779 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index); in combineArithReduction()
43792 Rdx = DAG.getNode(ISD::ADD, DL, VecVT, Lo, Hi); in combineArithReduction()
43799 Rdx = DAG.getNode(ISD::ADD, DL, MVT::v16i8, Rdx, Hi); in combineArithReduction()
43800 Rdx = DAG.getNode(X86ISD::PSADBW, DL, MVT::v2i64, Rdx, in combineArithReduction()
43803 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index); in combineArithReduction()
43815 Rdx = DAG.getNode(ISD::TRUNCATE, DL, ByteVT, Rdx); in combineArithReduction()
43824 return DAG.getNode(X86ISD::PSADBW, DL, VT, Ops[0], Zero); in combineArithReduction()
43834 Rdx = DAG.getNode(ISD::ADD, DL, VecVT, Lo, Hi); in combineArithReduction()
43840 Rdx = DAG.getNode(ISD::ADD, DL, MVT::v2i64, Rdx, RdxHi); in combineArithReduction()
43845 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index); in combineArithReduction()
43863 Rdx = DAG.getNode(HorizOpcode, DL, Lo.getValueType(), Hi, Lo); in combineArithReduction()
43873 Rdx = DAG.getNode(HorizOpcode, DL, VecVT, Rdx, Rdx); in combineArithReduction()
43875 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index); in combineArithReduction()
43929 Scl = DAG.getNode(ISD::TRUNCATE, dl, SrcVT.getScalarType(), Scl); in combineExtractVectorElt()
43956 return DAG.getNode(X86ISD::MMX_MOVD2W, dl, MVT::i32, MMXSrc); in combineExtractVectorElt()
44018 SDValue Mask = DAG.getNode(ISD::SHL, dl, BCVT, MaskBit, MaskIdx); in combineExtractVectorElt()
44019 SDValue Res = DAG.getNode(ISD::AND, dl, BCVT, BC, Mask); in combineExtractVectorElt()
44105 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, BroadcastVT, N00); in combineToExtendBoolVectorInReg()
44121 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, BroadcastVT, N00); in combineToExtendBoolVectorInReg()
44130 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Scl); in combineToExtendBoolVectorInReg()
44143 Vec = DAG.getNode(ISD::AND, DL, VT, Vec, BitMask); in combineToExtendBoolVectorInReg()
44154 return DAG.getNode(ISD::SRL, DL, VT, Vec, in combineToExtendBoolVectorInReg()
44181 bool TValIsAllZeros = ISD::isBuildVectorAllZeros(LHS.getNode()); in combineVSelectWithAllOnesOrZeros()
44182 bool FValIsAllZeros = ISD::isBuildVectorAllZeros(RHS.getNode()); in combineVSelectWithAllOnesOrZeros()
44202 bool TValIsAllOnes = ISD::isBuildVectorAllOnes(LHS.getNode()); in combineVSelectWithAllOnesOrZeros()
44209 bool FValIsAllOnes = ISD::isBuildVectorAllOnes(RHS.getNode()); in combineVSelectWithAllOnesOrZeros()
44237 SDValue Or = DAG.getNode(ISD::OR, DL, CondVT, Cond, CastRHS); in combineVSelectWithAllOnesOrZeros()
44244 SDValue And = DAG.getNode(ISD::AND, DL, CondVT, Cond, CastLHS); in combineVSelectWithAllOnesOrZeros()
44254 AndN = DAG.getNode(ISD::AND, DL, CondVT, DAG.getNOT(DL, Cond, CondVT), in combineVSelectWithAllOnesOrZeros()
44257 AndN = DAG.getNode(X86ISD::ANDNP, DL, CondVT, Cond, CastRHS); in combineVSelectWithAllOnesOrZeros()
44285 !collectConcatOps(TVal.getNode(), CatOpsT, DAG) || in narrowVectorSelect()
44286 !collectConcatOps(FVal.getNode(), CatOpsF, DAG)) in narrowVectorSelect()
44291 return DAG.getNode(Opcode, DL, Ops[1].getValueType(), Ops); in narrowVectorSelect()
44353 SDValue R = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond); in combineSelectOfTwoConstants()
44357 R = DAG.getNode(ISD::MUL, DL, VT, R, DAG.getConstant(AbsDiff, DL, VT)); in combineSelectOfTwoConstants()
44361 R = DAG.getNode(ISD::ADD, DL, VT, R, SDValue(FalseC, 0)); in combineSelectOfTwoConstants()
44380 ISD::isBuildVectorOfConstantSDNodes(Cond.getNode())) in combineVSelectToBLENDV()
44448 SDValue SB = DAG.getNode(X86ISD::BLENDV, SDLoc(U), U->getValueType(0), in combineVSelectToBLENDV()
44459 return DAG.getNode(X86ISD::BLENDV, SDLoc(N), N->getValueType(0), V, in combineVSelectToBLENDV()
44496 ISD::isBuildVectorAllZeros(N->getOperand(0).getNode()); in combineLogicBlendIntoConditionalNegate()
44500 if (IsNegV(Y.getNode(), X)) in combineLogicBlendIntoConditionalNegate()
44502 else if (IsNegV(X.getNode(), Y)) in combineLogicBlendIntoConditionalNegate()
44507 SDValue SubOp1 = DAG.getNode(ISD::XOR, DL, MaskVT, V, Mask); in combineLogicBlendIntoConditionalNegate()
44523 SDValue Res = DAG.getNode(ISD::SUB, DL, MaskVT, SubOp1, SubOp2); in combineLogicBlendIntoConditionalNegate()
44544 bool CondConstantVector = ISD::isBuildVectorOfConstantSDNodes(Cond.getNode()); in combineSelect()
44576 getTargetShuffleMask(LHS.getNode(), SimpleVT, true, LHSOps, LHSMask) && in combineSelect()
44577 getTargetShuffleMask(RHS.getNode(), SimpleVT, true, RHSOps, RHSMask)) { in combineSelect()
44590 LHS = DAG.getNode(X86ISD::PSHUFB, DL, VT, LHS.getOperand(0), in combineSelect()
44592 RHS = DAG.getNode(X86ISD::PSHUFB, DL, VT, RHS.getOperand(0), in combineSelect()
44594 return DAG.getNode(ISD::OR, DL, VT, LHS, RHS); in combineSelect()
44748 return DAG.getNode(Opcode, DL, N->getValueType(0), LHS, RHS); in combineSelect()
44766 return DAG.getNode(ISD::SELECT, DL, VT, AndNode, RHS, LHS); in combineSelect()
44781 Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond); in combineSelect()
44782 return DAG.getNode(N->getOpcode(), DL, VT, Cond, LHS, RHS); in combineSelect()
44801 bool ZeroLHS = ISD::isBuildVectorAllZeros(LHS.getNode()); in combineSelect()
44802 bool ZeroRHS = ISD::isBuildVectorAllZeros(RHS.getNode()); in combineSelect()
44812 Cond = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SrcCondVT, in combineSelect()
44899 ISD::isBuildVectorAllZeros(LHS.getNode()) && in combineSelect()
44900 !ISD::isBuildVectorAllZeros(RHS.getNode())) { in combineSelect()
44915 ExtCond = DAG.getNode(ISD::TRUNCATE, DL, CondVT, ExtCond); in combineSelect()
44936 return DAG.getNode(N->getOpcode(), DL, VT, in combineSelect()
44940 ISD::isBuildVectorAllOnes(Cond.getOperand(1).getNode())) { in combineSelect()
44941 Cond = DAG.getNode(X86ISD::PCMPGT, DL, CondVT, in combineSelect()
44943 return DAG.getNode(N->getOpcode(), DL, VT, Cond, RHS, LHS); in combineSelect()
44956 bool LHSIsConst = ISD::isBuildVectorOfConstantSDNodes(LHS.getNode()); in combineSelect()
44957 bool RHSIsConst = ISD::isBuildVectorOfConstantSDNodes(RHS.getNode()); in combineSelect()
45022 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, And.getOperand(0), ShlAmt); in combineSelect()
45121 auto *AN = cast<AtomicSDNode>(CmpLHS.getNode()); in combineSetCCAtomicArith()
45356 CarryOp1.getNode()->hasOneUse() && in combineCarryThroughADD()
45360 DAG.getNode(X86ISD::SUB, SDLoc(CarryOp1), CarryOp1->getVTList(), in combineCarryThroughADD()
45362 return SDValue(SubCommute.getNode(), CarryOp1.getResNo()); in combineCarryThroughADD()
45437 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, in combinePTESTCC()
45446 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, in combinePTESTCC()
45458 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, in combinePTESTCC()
45466 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, in combinePTESTCC()
45490 Res = DAG.getNode(ISD::AND, DL, MVT::i32, Res, in combinePTESTCC()
45495 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Res, in combinePTESTCC()
45502 if (ISD::isBuildVectorAllOnes(Op0.getNode())) in combinePTESTCC()
45503 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, Op1, Op1); in combinePTESTCC()
45506 if (ISD::isBuildVectorAllOnes(Op1.getNode())) in combinePTESTCC()
45507 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, Op0, Op0); in combinePTESTCC()
45520 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, in combinePTESTCC()
45577 bool IsOneUse = CmpOp.getNode()->hasOneUse(); in combineSetCCMOVMSK()
45597 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, in combineSetCCMOVMSK()
45598 DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, BC), in combineSetCCMOVMSK()
45609 if (collectConcatOps(peekThroughBitcasts(Vec).getNode(), Ops, DAG) && in combineSetCCMOVMSK()
45614 SDValue V = DAG.getNode(IsAnyOf ? ISD::OR : ISD::AND, DL, SubVT, in combineSetCCMOVMSK()
45618 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, in combineSetCCMOVMSK()
45619 DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V), in combineSetCCMOVMSK()
45634 SDValue V = DAG.getNode(ISD::SUB, SDLoc(BC), BC.getValueType(), in combineSetCCMOVMSK()
45637 return DAG.getNode(X86ISD::PTEST, SDLoc(EFLAGS), MVT::i32, V, V); in combineSetCCMOVMSK()
45645 LHS = DAG.getNode(ISD::SUB, SDLoc(LHS), LHS.getValueType(), in combineSetCCMOVMSK()
45647 RHS = DAG.getNode(ISD::SUB, SDLoc(RHS), RHS.getValueType(), in combineSetCCMOVMSK()
45651 SDValue V = DAG.getNode(ISD::OR, SDLoc(EFLAGS), TestVT, LHS, RHS); in combineSetCCMOVMSK()
45652 return DAG.getNode(X86ISD::PTEST, SDLoc(EFLAGS), MVT::i32, V, V); in combineSetCCMOVMSK()
45671 Result = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result); in combineSetCCMOVMSK()
45674 Result = DAG.getNode(ISD::AND, DL, MVT::i16, Result, in combineSetCCMOVMSK()
45677 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Result, in combineSetCCMOVMSK()
45689 SDValue V = DAG.getNode(ISD::SUB, DL, Result.getValueType(), in combineSetCCMOVMSK()
45692 return DAG.getNode(X86ISD::PTEST, SDLoc(EFLAGS), MVT::i32, V, V); in combineSetCCMOVMSK()
45695 Result = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result); in combineSetCCMOVMSK()
45700 Result = DAG.getNode(ISD::AND, DL, MVT::i32, Result, in combineSetCCMOVMSK()
45703 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Result, in combineSetCCMOVMSK()
45726 Result = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result); in combineSetCCMOVMSK()
45729 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Result, in combineSetCCMOVMSK()
45783 return DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops); in combineCMov()
45807 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, TrueC->getValueType(0), Cond); in combineCMov()
45810 Cond = DAG.getNode(ISD::SHL, DL, Cond.getValueType(), Cond, in combineCMov()
45821 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, in combineCMov()
45823 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond, in combineCMov()
45854 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0), in combineCMov()
45858 Cond = DAG.getNode(ISD::MUL, DL, Cond.getValueType(), Cond, in combineCMov()
45863 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond, in combineCMov()
45904 return DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops); in combineCMov()
45939 SDValue LCMOV = DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), LOps); in combineCMov()
45942 SDValue CMOV = DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops); in combineCMov()
45972 SDValue Diff = DAG.getNode(ISD::SUB, DL, VT, Const, Add.getOperand(1)); in combineCMov()
45974 DAG.getNode(X86ISD::CMOV, DL, VT, Diff, Add.getOperand(0), in combineCMov()
45976 return DAG.getNode(ISD::ADD, DL, VT, CMov, Add.getOperand(1)); in combineCMov()
46078 SDValue NewN0 = DAG.getNode(ISD::TRUNCATE, DL, ReducedVT, N0); in reduceVMULWidth()
46079 SDValue NewN1 = DAG.getNode(ISD::TRUNCATE, DL, ReducedVT, N1); in reduceVMULWidth()
46083 SDValue MulLo = DAG.getNode(ISD::MUL, DL, ReducedVT, NewN0, NewN1); in reduceVMULWidth()
46085 return DAG.getNode((Mode == ShrinkMode::MULU8) ? ISD::ZERO_EXTEND in reduceVMULWidth()
46093 DAG.getNode(Mode == ShrinkMode::MULS16 ? ISD::MULHS : ISD::MULHU, DL, in reduceVMULWidth()
46115 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ResLo, ResHi); in reduceVMULWidth()
46122 SDValue Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0), in combineMulSpecial()
46124 Result = DAG.getNode(ISD::SHL, DL, VT, Result, in combineMulSpecial()
46126 Result = DAG.getNode(isAdd ? ISD::ADD : ISD::SUB, DL, VT, Result, in combineMulSpecial()
46132 SDValue Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0), in combineMulSpecial()
46134 Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, Result, in combineMulSpecial()
46136 Result = DAG.getNode(isAdd ? ISD::ADD : ISD::SUB, DL, VT, Result, in combineMulSpecial()
46155 return DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0), in combineMulSpecial()
46180 return DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0), in combineMulSpecial()
46193 SDValue Shift1 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMulSpecial()
46195 SDValue Shift2 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMulSpecial()
46197 return DAG.getNode(ISD::ADD, DL, VT, Shift1, Shift2); in combineMulSpecial()
46268 if (ISD::isBuildVectorOfConstantSDNodes(Op.getNode())) in combineMulToPMADDWD()
46269 return DAG.getNode(ISD::AND, SDLoc(N), VT, Op, in combineMulToPMADDWD()
46271 if (Op.getOpcode() == ISD::SIGN_EXTEND && N->isOnlyUserOf(Op.getNode())) { in combineMulToPMADDWD()
46275 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, Src); in combineMulToPMADDWD()
46280 Src = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), ExtVT, Src); in combineMulToPMADDWD()
46281 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, Src); in combineMulToPMADDWD()
46286 N->isOnlyUserOf(Op.getNode())) { in combineMulToPMADDWD()
46289 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(N), VT, Src); in combineMulToPMADDWD()
46293 N->isOnlyUserOf(Op.getNode())) { in combineMulToPMADDWD()
46294 return DAG.getNode(X86ISD::VSRLI, SDLoc(N), VT, Op.getOperand(0), in combineMulToPMADDWD()
46311 return DAG.getNode(X86ISD::VPMADDWD, DL, ResVT, in combineMulToPMADDWD()
46341 return DAG.getNode(X86ISD::PMULDQ, DL, Ops[0].getValueType(), Ops); in combineMulToPMULDQ()
46352 return DAG.getNode(X86ISD::PMULUDQ, DL, Ops[0].getValueType(), Ops); in combineMulToPMULDQ()
46402 SDValue NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0), in combineMul()
46405 NewMul = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), in combineMul()
46440 NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMul()
46443 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0), in combineMul()
46447 NewMul = DAG.getNode(ISD::SHL, DL, VT, NewMul, in combineMul()
46450 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, NewMul, in combineMul()
46455 NewMul = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), in combineMul()
46467 NewMul = DAG.getNode( in combineMul()
46469 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMul()
46474 NewMul = DAG.getNode(ISD::SUB, DL, VT, in combineMul()
46478 NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMul()
46483 NewMul = DAG.getNode(ISD::SUB, DL, VT, N->getOperand(0), NewMul); in combineMul()
46485 NewMul = DAG.getNode(ISD::SUB, DL, VT, NewMul, N->getOperand(0)); in combineMul()
46488 NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMul()
46491 NewMul = DAG.getNode(ISD::ADD, DL, VT, NewMul, N->getOperand(0)); in combineMul()
46492 NewMul = DAG.getNode(ISD::ADD, DL, VT, NewMul, N->getOperand(0)); in combineMul()
46495 NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMul()
46498 NewMul = DAG.getNode(ISD::SUB, DL, VT, NewMul, N->getOperand(0)); in combineMul()
46499 NewMul = DAG.getNode(ISD::SUB, DL, VT, NewMul, N->getOperand(0)); in combineMul()
46534 if (!ISD::isConstantSplatVector(N->getOperand(1).getNode(), ShiftAmt) || in combineShiftToPMULH()
46556 SDValue Mulh = DAG.getNode(Opc, DL, MulVT, LHS, RHS); in combineShiftToPMULH()
46559 return DAG.getNode(ExtOpc, DL, VT, Mulh); in combineShiftToPMULH()
46600 return DAG.getNode(ISD::AND, DL, VT, N00, DAG.getConstant(Mask, DL, VT)); in combineShiftLeft()
46615 return DAG.getNode(ISD::ADD, SDLoc(N), VT, N0, N0); in combineShiftLeft()
46664 DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, N00, DAG.getValueType(SVT)); in combineShiftRightArithmetic()
46669 return DAG.getNode(ISD::SHL, DL, VT, NN, in combineShiftRightArithmetic()
46671 return DAG.getNode(ISD::SRA, DL, VT, NN, in combineShiftRightArithmetic()
46724 SDValue NewShift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), N1); in combineShiftRightLogical()
46725 return DAG.getNode(ISD::AND, DL, VT, NewShift, NewMask); in combineShiftRightLogical()
46742 N->isOnlyUserOf(N0.getNode()) ? peekThroughOneUseBitcasts(N0) : N0; in combineHorizOpWithShuffle()
46744 N->isOnlyUserOf(N1.getNode()) ? peekThroughOneUseBitcasts(N1) : N1; in combineHorizOpWithShuffle()
46768 SDValue Res = DAG.getNode(Opcode, DL, VT, Lo, Hi); in combineHorizOpWithShuffle()
46826 SDValue Res = DAG.getNode(Opcode, DL, VT, LHS, RHS); in combineHorizOpWithShuffle()
46862 SDValue Res = DAG.getNode(Opcode, DL, VT, DAG.getBitcast(SrcVT, Op00), in combineHorizOpWithShuffle()
46896 if ((N0.isUndef() || N->isOnlyUserOf(N0.getNode())) && in combineVectorPack()
46897 (N1.isUndef() || N->isOnlyUserOf(N1.getNode())) && in combineVectorPack()
46960 return DAG.getNode(X86ISD::VTRUNC, SDLoc(N), VT, N0.getOperand(0)); in combineVectorPack()
46964 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i32, in combineVectorPack()
46966 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Concat); in combineVectorPack()
46988 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Src0, Src1); in combineVectorPack()
47024 N->isOnlyUserOf(LHS.getNode()) && N->isOnlyUserOf(RHS.getNode())) { in combineVectorHADDSUB()
47032 SDValue Res = DAG.getNode(LHS.getOpcode(), DL, LHS.getValueType(), in combineVectorHADDSUB()
47038 DAG.getNode(X86ISD::PSHUFD, DL, ShufVT, Res, in combineVectorHADDSUB()
47041 DAG.getNode(X86ISD::PSHUFD, DL, ShufVT, Res, in combineVectorHADDSUB()
47043 return DAG.getNode(N->getOpcode(), DL, VT, DAG.getBitcast(VT, NewLHS), in combineVectorHADDSUB()
47067 if (ISD::isBuildVectorAllZeros(N0.getNode())) in combineVectorShiftVar()
47121 if (ISD::isBuildVectorAllZeros(N0.getNode())) in combineVectorShiftImm()
47127 if (!LogicalShift && ISD::isBuildVectorAllOnes(N0.getNode())) in combineVectorShiftImm()
47143 return DAG.getNode(Opcode, SDLoc(N), VT, N0.getOperand(0), in combineVectorShiftImm()
47157 if (N->isOnlyUserOf(N0.getNode()) && in combineVectorShiftImm()
47280 DAG.getNode(X86ISD::FSETCCM, DL, MVT::v1i1, CMP00, CMP01, in combineCompareEqual()
47284 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v16i1, in combineCompareEqual()
47291 DAG.getNode(X86ISD::FSETCC, DL, CMP00.getValueType(), CMP00, in combineCompareEqual()
47303 SDValue Vector64 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, in combineCompareEqual()
47306 OnesOrZeroesF = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, in combineCompareEqual()
47312 SDValue ANDed = DAG.getNode(ISD::AND, DL, IntVT, OnesOrZeroesI, in combineCompareEqual()
47314 SDValue OneBitOfTruth = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, in combineCompareEqual()
47347 return DAG.getNode(X86ISD::VBROADCAST, SDLoc(V), VT, in combineAndNotIntoANDNP()
47364 return DAG.getNode(X86ISD::ANDNP, SDLoc(N), VT, X, Y); in combineAndNotIntoANDNP()
47394 if (SDValue NN0 = PromoteMaskArithmetic(N0.getNode(), VT, DAG, Depth + 1)) in PromoteMaskArithmetic()
47408 if (SDValue NN1 = PromoteMaskArithmetic(N1.getNode(), VT, DAG, Depth + 1)) in PromoteMaskArithmetic()
47414 if (!RHSTrunc && !ISD::isBuildVectorOfConstantSDNodes(N1.getNode())) in PromoteMaskArithmetic()
47420 N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N1); in PromoteMaskArithmetic()
47423 return DAG.getNode(N->getOpcode(), DL, VT, N0, N1); in PromoteMaskArithmetic()
47446 SDValue Op = PromoteMaskArithmetic(Narrow.getNode(), VT, DAG, 0); in PromoteMaskArithmetic()
47456 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, in PromoteMaskArithmetic()
47500 SDValue FPLogic = DAG.getNode(FPOpcode, DL, N00Type, N00, N10); in convertIntLogicToFPLogic()
47527 SDValue Vec00 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, N00); in convertIntLogicToFPLogic()
47528 SDValue Vec01 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, N01); in convertIntLogicToFPLogic()
47529 SDValue Vec10 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, N10); in convertIntLogicToFPLogic()
47530 SDValue Vec11 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, N11); in convertIntLogicToFPLogic()
47533 SDValue Logic = DAG.getNode(N->getOpcode(), DL, BoolVecVT, Setcc0, Setcc1); in convertIntLogicToFPLogic()
47534 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Logic, ZeroIndex); in convertIntLogicToFPLogic()
47567 DAG.getNode(VecOpc, DL, VecVT0, Vec0, DAG.getBitcast(VecVT0, Vec1)); in combineBitOpWithMOVMSK()
47568 return DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result); in combineBitOpWithMOVMSK()
47605 DAG.getNode(Opc, DL, BCVT, BC0.getOperand(0), BC1.getOperand(0)); in combineBitOpWithShift()
47606 SDValue Shift = DAG.getNode(BCOpc, DL, BCVT, BitOp, BC0.getOperand(1)); in combineBitOpWithShift()
47651 return DAG.getNode(X86ISD::ANDNP, DL, VT, Sra, Y); in combineAndMaskToShift()
47656 if (!ISD::isConstantSplatVector(Op1.getNode(), SplatVal) || in combineAndMaskToShift()
47674 SDValue Shift = DAG.getNode(X86ISD::VSRLI, DL, VT, Op0, ShAmt); in combineAndMaskToShift()
47731 LoadSDNode *Ld = dyn_cast<LoadSDNode>(N.getNode()); in combineAndLoadToBZHI()
47782 SDValue Sub = DAG.getNode(ISD::SUB, dl, MVT::i32, SizeC, Index); in combineAndLoadToBZHI()
47783 Sub = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Sub); in combineAndLoadToBZHI()
47786 SDValue LShr = DAG.getNode(ISD::SRL, dl, VT, AllOnes, Sub); in combineAndLoadToBZHI()
47788 return DAG.getNode(ISD::AND, dl, VT, Inp, LShr); in combineAndLoadToBZHI()
47855 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, dl, SrcVT, in combineScalarAndWithMaskSetcc()
47872 DAG.getNode(X86ISD::FAND, dl, MVT::v4f32, in combineAnd()
47882 SDValue LHS = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, N0); in combineAnd()
47883 SDValue RHS = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, N1); in combineAnd()
47884 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, in combineAnd()
47885 DAG.getNode(ISD::AND, dl, MVT::i32, LHS, RHS)); in combineAnd()
47905 Mask = DAG.getNode(ISD::AND, dl, MaskVT, Mask, PartialBits); in combineAnd()
47947 SDValue MaskMul = DAG.getNode(ISD::AND, dl, VT, N0.getOperand(1), N1); in combineAnd()
47948 return DAG.getNode(Opc0, dl, VT, N0.getOperand(0), MaskMul); in combineAnd()
48035 return DAG.getNode(ISD::AND, dl, VT, NewN0 ? NewN0 : N0, in combineAnd()
48050 if (VT == SrcVecVT.getScalarType() && N0->isOnlyUserOf(SrcVec.getNode()) && in combineAnd()
48073 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Shuffle, in combineAnd()
48139 DAG.getNode(X86ISD::ANDNP, DL, VT, DAG.getBitcast(VT, N0.getOperand(1)), in canonicalizeBitSelect()
48141 return DAG.getNode(ISD::OR, DL, VT, X, Y); in canonicalizeBitSelect()
48244 SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Cmp->getOperand(0)); in lowerX86CmpEqZeroToCtlzSrl()
48248 SDValue Scc = DAG.getNode(ISD::SRL, dl, MVT::i32, Trunc, in lowerX86CmpEqZeroToCtlzSrl()
48286 SDNode *OR = N->getOperand(0).getNode(); in combineOrCmpEqZeroToCtlzSrl()
48295 OR = (LHS->getOpcode() == ISD::OR) ? LHS.getNode() : RHS.getNode(); in combineOrCmpEqZeroToCtlzSrl()
48313 Ret = DAG.getNode(ISD::OR, SDLoc(OR), MVT::i32, NewLHS, NewRHS); in combineOrCmpEqZeroToCtlzSrl()
48329 Ret = DAG.getNode(ISD::OR, SDLoc(OR), MVT::i32, Ret, NewRHS); in combineOrCmpEqZeroToCtlzSrl()
48332 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), N->getValueType(0), Ret); in combineOrCmpEqZeroToCtlzSrl()
48349 SDValue Freeze_And0_R = DAG.getNode(ISD::FREEZE, SDLoc(), VT, And0_R); in foldMaskedMergeImpl()
48350 SDValue Xor0 = DAG.getNode(ISD::XOR, DL, VT, And1_R, Freeze_And0_R); in foldMaskedMergeImpl()
48351 SDValue And = DAG.getNode(ISD::AND, DL, VT, Xor0, NotOp); in foldMaskedMergeImpl()
48352 SDValue Xor1 = DAG.getNode(ISD::XOR, DL, VT, And, Freeze_And0_R); in foldMaskedMergeImpl()
48400 DAG.getNode(X86ISD::FOR, dl, MVT::v4f32, in combineOr()
48422 Mask = DAG.getNode(ISD::AND, dl, MaskVT, Mask, PartialBits); in combineOr()
48460 return DAG.getNode( in combineOr()
48468 return DAG.getNode( in combineOr()
48561 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, ResultType, Cond); in foldXorTruncShiftIntoCmp()
48595 !ISD::isBuildVectorAllOnes(Ones.getNode())) in foldVectorXorShiftIntoCmp()
48637 ISD::isConstantSplatVector(V.getOperand(1).getNode(), Limit)) in detectUSatPattern()
48658 return DAG.getNode(ISD::SMAX, DL, InVT, SMin, In.getOperand(1)); in detectUSatPattern()
48682 ISD::isConstantSplatVector(V.getOperand(1).getNode(), C) && C == Limit) in detectSSatPattern()
48728 return DAG.getNode(X86ISD::VTRUNCUS, DL, VT, Mid); in combineTruncateWithSat()
48794 SatVal = DAG.getNode(ISD::CONCAT_VECTORS, DL, InVT, ConcatOps); in combineTruncateWithSat()
48800 SDValue Res = DAG.getNode(TruncOpc, DL, TruncVT, SatVal); in combineTruncateWithSat()
48801 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res, in combineTruncateWithSat()
48874 return DAG.getNode(ISD::AVGCEILU, DL, Ops[0].getValueType(), Ops); in detectAVGPattern()
48880 Op = DAG.getNode(ISD::TRUNCATE, DL, VT, Op); in detectAVGPattern()
48890 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarVT, Op, Idx); in detectAVGPattern()
48898 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res, in detectAVGPattern()
48909 Operands[1] = DAG.getNode(ISD::SUB, DL, InVT, Operands[1], VecOnes); in detectAVGPattern()
48999 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, in combineLoad()
49002 SDValue NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, RegVT, Load1, Load2); in combineLoad()
49165 DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, CastVT, PassThru, Load, VecIndex); in reduceMaskedLoadToScalarLoad()
49174 if (!ISD::isBuildVectorOfConstantSDNodes(ML->getMask().getNode())) in combineMaskedLoadConstantMask()
49204 if (ISD::isBuildVectorAllZeros(ML->getPassThru().getNode())) in combineMaskedLoadConstantMask()
49290 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Value, VecIndex); in reduceMaskedStoreToScalarStore()
49334 if (Value.getOpcode() == ISD::TRUNCATE && Value.getNode()->hasOneUse() && in combineMaskedStore()
49389 StoredVal = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i1, Ops); in combineStore()
49398 ISD::isBuildVectorOfConstantSDNodes(StoredVal.getNode())) { in combineStore()
49420 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Ch0, Ch1); in combineStore()
49472 SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::v16i32, in combineStore()
49580 LoadSDNode *Ld = cast<LoadSDNode>(St->getValue().getNode()); in combineStore()
49614 SDValue NewExtract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, in combineStore()
49786 (!A.getNode() && (LIdx < (int)NumElts || RIdx < (int)NumElts)) || in isHorizontalBinOp()
49787 (!B.getNode() && (LIdx >= (int)NumElts || RIdx >= (int)NumElts))) in isHorizontalBinOp()
49811 SDValue NewLHS = A.getNode() ? A : B; // If A is 'UNDEF', use B for it. in isHorizontalBinOp()
49812 SDValue NewRHS = B.getNode() ? B : A; // If B is 'UNDEF', use A for it. in isHorizontalBinOp()
49865 SDValue HorizBinOp = DAG.getNode(HorizOpcode, SDLoc(N), VT, LHS, RHS); in combineToHorizontalAddSub()
49884 return DAG.getNode(HorizOpcode, DL, Ops[0].getValueType(), Ops); in combineToHorizontalAddSub()
49953 SDValue FCMulC = DAG.getNode(CombineOpcode, SDLoc(N), VT, RHS, I2F); in combineFMulcFCMulc()
50027 ((ISD::isBuildVectorAllZeros(Op0->getOperand(2).getNode()) && in combineFaddCFmul()
50029 IsVectorAllNegativeZero(Op0->getOperand(2).getNode()))) { in combineFaddCFmul()
50052 DAG.getNode(NewOp, SDLoc(N), CVT, MulOp0, MulOp1, FAddOp1, N->getFlags()); in combineFaddCFmul()
50100 return ISD::isBuildVectorOfConstantSDNodes(Op.getNode()); in combineTruncatedArithmetic()
50104 SDValue Trunc0 = DAG.getNode(ISD::TRUNCATE, DL, VT, N0); in combineTruncatedArithmetic()
50105 SDValue Trunc1 = DAG.getNode(ISD::TRUNCATE, DL, VT, N1); in combineTruncatedArithmetic()
50106 return DAG.getNode(SrcOpcode, DL, VT, Trunc0, Trunc1); in combineTruncatedArithmetic()
50160 In = DAG.getNode(ISD::AND, DL, InVT, In, DAG.getConstant(Mask, DL, InVT)); in combineVectorTruncationWithPACKUS()
50171 In = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, InVT, In, in combineVectorTruncationWithPACKSS()
50270 !collectConcatOps(In.getNode(), ConcatOps, DAG)) in combineVectorSignBitsTruncation()
50301 if (In.getOpcode() == ISD::SRL && N->isOnlyUserOf(In.getNode())) in combineVectorSignBitsTruncation()
50305 SDValue NewIn = DAG.getNode(ISD::SRA, DL, InVT, In->ops()); in combineVectorSignBitsTruncation()
50343 if (!ISD::isConstantSplatVector(Src.getOperand(1).getNode(), ShiftAmt) || in combinePMULH()
50384 SDValue Res = DAG.getNode(ISD::MULHU, DL, BCVT, DAG.getBitcast(BCVT, LHS), in combinePMULH()
50386 return DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getBitcast(InVT, Res)); in combinePMULH()
50390 LHS = DAG.getNode(ISD::TRUNCATE, DL, VT, LHS); in combinePMULH()
50391 RHS = DAG.getNode(ISD::TRUNCATE, DL, VT, RHS); in combinePMULH()
50394 return DAG.getNode(Opc, DL, VT, LHS, RHS); in combinePMULH()
50530 return DAG.getNode(X86ISD::VPMADDUBSW, DL, ResVT, Ops[0], Ops[1]); in detectPMADDUBSW()
50567 return DAG.getNode(X86ISD::MMX_MOVD2W, DL, MVT::i32, BCSrc); in combineTruncate()
50584 return DAG.getNode(X86ISD::VTRUNCS, DL, VT, SSatVal); in combineVTRUNC()
50586 return DAG.getNode(X86ISD::VTRUNCUS, DL, VT, USatVal); in combineVTRUNC()
50629 if (SDValue NegOp0 = isFNEG(DAG, Op.getOperand(0).getNode(), Depth + 1)) in isFNEG()
50642 if (SDValue NegInsVal = isFNEG(DAG, InsVal.getNode(), Depth + 1)) in isFNEG()
50644 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(Op), VT, InsVector, in isFNEG()
50768 SDValue NewNode = DAG.getNode(X86ISD::FNMSUB, DL, VT, Arg.getOperand(0), in combineFneg()
50788 if (SDValue Arg = isFNEG(DAG, Op.getNode(), Depth)) { in getNegatedExpression()
50796 SDNodeFlags Flags = Op.getNode()->getFlags(); in getNegatedExpression()
50835 return DAG.getNode(NewOpc, SDLoc(Op), VT, NewOps); in getNegatedExpression()
50841 return DAG.getNode(Opc, SDLoc(Op), VT, NegOp0); in getNegatedExpression()
50872 SDValue IntOp = DAG.getNode(IntOpcode, dl, IntVT, Op0, Op1); in lowerX86FPLogicOp()
50902 DAG.getNode(X86ISD::FXOR, SDLoc(N), MVT::v4f32, in combineXor()
50940 if (ISD::isBuildVectorAllOnes(N1.getNode()) && VT.isVector() && in combineXor()
50944 return DAG.getNode( in combineXor()
50962 return DAG.getNode(ISD::XOR, DL, VT, LHS, in combineXor()
50963 DAG.getNode(ISD::XOR, DL, VT, RHS, N1)); in combineXor()
50988 return isNullFPConstant(V) || ISD::isBuildVectorAllZeros(V.getNode()); in isNullFPScalarOrVectorConst()
51023 return ISD::isBuildVectorAllOnes(V.getNode()); in combineFAndFNotToFAndn()
51030 return DAG.getNode(X86ISD::FANDN, DL, VT, N0.getOperand(0), N1); in combineFAndFNotToFAndn()
51034 return DAG.getNode(X86ISD::FANDN, DL, VT, N1.getOperand(0), N0); in combineFAndFNotToFAndn()
51108 return DAG.getNode(NewOp, SDLoc(N), N->getValueType(0), in combineFMinFMax()
51134 return DAG.getNode(MinMaxOp, DL, VT, Op0, Op1, N->getFlags()); in combineFMinNumFMaxNum()
51139 return DAG.getNode(MinMaxOp, DL, VT, Op0, Op1, N->getFlags()); in combineFMinNumFMaxNum()
51141 return DAG.getNode(MinMaxOp, DL, VT, Op1, Op0, N->getFlags()); in combineFMinNumFMaxNum()
51170 SDValue MinOrMax = DAG.getNode(MinMaxOp, DL, VT, Op1, Op0); in combineFMinNumFMaxNum()
51191 ISD::isNormalLoad(In.getNode()) && In.hasOneUse()) { in combineX86INT_TO_FP()
51199 SDValue Convert = DAG.getNode(N->getOpcode(), dl, VT, in combineX86INT_TO_FP()
51220 ISD::isNormalLoad(In.getNode()) && In.hasOneUse()) { in combineCVTP2I_CVTTP2I()
51230 DAG.getNode(N->getOpcode(), dl, {VT, MVT::Other}, in combineCVTP2I_CVTTP2I()
51235 DAG.getNode(N->getOpcode(), dl, VT, DAG.getBitcast(InVT, VZLoad)); in combineCVTP2I_CVTTP2I()
51263 if (ISD::isBuildVectorAllZeros(N0.getNode())) in combineAndnp()
51267 if (ISD::isBuildVectorAllZeros(N1.getNode())) in combineAndnp()
51272 return DAG.getNode(ISD::AND, SDLoc(N), VT, DAG.getBitcast(VT, Not), N1); in combineAndnp()
51370 if (ISD::isNormalLoad(Src.getNode()) && Src.hasOneUse()) { in combineCVTPH2PS()
51375 SDValue Convert = DAG.getNode( in combineCVTPH2PS()
51380 SDValue Convert = DAG.getNode(N->getOpcode(), dl, MVT::v4f32, in combineCVTPH2PS()
51424 if (!isa<ConstantSDNode>(CMovOp0.getNode()) || in combineSextInRegCmov()
51425 !isa<ConstantSDNode>(CMovOp1.getNode())) in combineSextInRegCmov()
51433 CMovOp0 = DAG.getNode(IntermediateOpc, DL, DstVT, CMovOp0); in combineSextInRegCmov()
51434 CMovOp1 = DAG.getNode(IntermediateOpc, DL, DstVT, CMovOp1); in combineSextInRegCmov()
51437 CMovOp0 = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, DstVT, CMovOp0, N1); in combineSextInRegCmov()
51438 CMovOp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, DstVT, CMovOp1, N1); in combineSextInRegCmov()
51444 CMovOp0 = DAG.getNode(ISD::ZERO_EXTEND, DL, CMovVT, CMovOp0); in combineSextInRegCmov()
51445 CMovOp1 = DAG.getNode(ISD::ZERO_EXTEND, DL, CMovVT, CMovOp1); in combineSextInRegCmov()
51448 SDValue CMov = DAG.getNode(X86ISD::CMOV, DL, CMovVT, CMovOp0, CMovOp1, in combineSextInRegCmov()
51452 CMov = DAG.getNode(ISD::TRUNCATE, DL, DstVT, CMov); in combineSextInRegCmov()
51482 if (!ISD::isNormalLoad(N00.getNode())) in combineSignExtendInReg()
51487 if (SDValue Promote = PromoteMaskArithmetic(N0.getNode(), DAG, Subtarget)) in combineSignExtendInReg()
51488 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, VT, Promote, N1); in combineSignExtendInReg()
51492 DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::v4i32, N00, N1); in combineSignExtendInReg()
51493 return DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i64, Tmp); in combineSignExtendInReg()
51553 SDValue NewExt = DAG.getNode(Ext->getOpcode(), SDLoc(Ext), VT, AddOp0); in promoteExtBeforeAdd()
51561 return DAG.getNode(ISD::ADD, SDLoc(Add), VT, NewExt, NewConstant, Flags); in promoteExtBeforeAdd()
51589 if (!isa<ConstantSDNode>(CMovOp0.getNode()) || in combineToExtendCMOV()
51590 !isa<ConstantSDNode>(CMovOp1.getNode())) in combineToExtendCMOV()
51608 CMovOp0 = DAG.getNode(ExtendOpcode, DL, ExtendVT, CMovOp0); in combineToExtendCMOV()
51609 CMovOp1 = DAG.getNode(ExtendOpcode, DL, ExtendVT, CMovOp1); in combineToExtendCMOV()
51611 SDValue Res = DAG.getNode(X86ISD::CMOV, DL, ExtendVT, CMovOp0, CMovOp1, in combineToExtendCMOV()
51616 Res = DAG.getNode(ExtendOpcode, DL, TargetVT, Res); in combineToExtendCMOV()
51677 SDValue Setcc = DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, N0->getOperand(0), in combineSext()
51683 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), in combineSext()
51685 DCI.CombineTo(N0.getNode(), Trunc); in combineSext()
51709 return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0)); in combineSext()
51739 SDValue Fmul = DAG.getNode(ISD::FMUL, dl, VT, A, B, Flags); in combineFMA()
51740 return DAG.getNode(ISD::FADD, dl, VT, Fmul, C, Flags); in combineFMA()
51764 V = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(V), V.getValueType(), in combineFMA()
51789 return DAG.getNode(NewOpcode, dl, {VT, MVT::Other}, in combineFMA()
51793 return DAG.getNode(NewOpcode, dl, VT, A, B, C, N->getOperand(3)); in combineFMA()
51794 return DAG.getNode(NewOpcode, dl, VT, A, B, C); in combineFMA()
51817 return DAG.getNode(NewOpcode, dl, VT, N->getOperand(0), N->getOperand(1), in combineFMADDSUB()
51819 return DAG.getNode(NewOpcode, dl, VT, N->getOperand(0), N->getOperand(1), in combineFMADDSUB()
51834 SDValue Setcc = DAG.getNode(X86ISD::SETCC_CARRY, dl, VT, N0->getOperand(0), in combineZext()
51840 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), in combineZext()
51842 DCI.CombineTo(N0.getNode(), Trunc); in combineZext()
51907 return DAG.getNode(ISD::OR, DL, CmpVT, A, B); in emitOrXorXorTree()
51909 return DAG.getNode(ISD::OR, DL, VecVT, A, B); in emitOrXorXorTree()
51910 return DAG.getNode(ISD::AND, DL, CmpVT, A, B); in emitOrXorXorTree()
51918 return DAG.getNode(ISD::XOR, DL, VecVT, A, B); in emitOrXorXorTree()
52023 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT, in combineVectorSizedSetCCEquality()
52041 Cmp = DAG.getNode(ISD::XOR, DL, VecVT, VecX, VecY); in combineVectorSizedSetCCEquality()
52056 SDValue PT = DAG.getNode(X86ISD::PTEST, DL, MVT::i32, BCCmp, BCCmp); in combineVectorSizedSetCCEquality()
52059 return DAG.getNode(ISD::TRUNCATE, DL, VT, X86SetCC.getValue(0)); in combineVectorSizedSetCCEquality()
52066 SDValue MovMsk = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Cmp); in combineVectorSizedSetCCEquality()
52092 return DAG.getNode(ISD::TRUNCATE, DL, VT, in combineSetCC()
52093 DAG.getNode(X86ISD::SETCC, DL, MVT::i8, X86CC, V)); in combineSetCC()
52102 return DAG.getNode(ISD::AND, DL, OpVT, DAG.getNOT(DL, N1, OpVT), in combineSetCC()
52105 return DAG.getNode(ISD::AND, DL, OpVT, DAG.getNOT(DL, N1, OpVT), in combineSetCC()
52120 return DAG.getNode(ISD::AND, DL, OpVT, N1, in combineSetCC()
52123 return DAG.getNode(ISD::AND, DL, OpVT, N1, in combineSetCC()
52168 bool IsVZero1 = ISD::isBuildVectorAllZeros(Op1.getNode()); in combineSetCC()
52196 return DAG.getNode(ISD::TRUNCATE, DL, VT, Setcc); in combineSetCC()
52245 return DAG.getNode(X86ISD::MOVMSK, SDLoc(N), VT, Src.getOperand(0)); in combineMOVMSK()
52253 return DAG.getNode(ISD::XOR, DL, VT, in combineMOVMSK()
52254 DAG.getNode(X86ISD::MOVMSK, DL, VT, NotSrc), in combineMOVMSK()
52261 ISD::isBuildVectorAllOnes(Src.getOperand(1).getNode())) { in combineMOVMSK()
52264 return DAG.getNode(ISD::XOR, DL, VT, in combineMOVMSK()
52265 DAG.getNode(X86ISD::MOVMSK, DL, VT, Src.getOperand(0)), in combineMOVMSK()
52273 ISD::isBuildVectorAllZeros(Src.getOperand(1).getNode())) { in combineMOVMSK()
52289 return DAG.getNode(X86ISD::MOVMSK, DL, VT, LHS); in combineMOVMSK()
52400 Index = DAG.getNode(ISD::TRUNCATE, DL, NewVT, Index); in combineGatherScatter()
52414 Index = DAG.getNode(ISD::TRUNCATE, DL, NewVT, Index); in combineGatherScatter()
52437 Base = DAG.getNode(ISD::ADD, DL, PtrVT, Base, in combineGatherScatter()
52450 Splat = DAG.getNode(ISD::ADD, DL, Index.getValueType(), in combineGatherScatter()
52453 Index = DAG.getNode(ISD::ADD, DL, Index.getValueType(), in combineGatherScatter()
52514 return DAG.getNode(X86ISD::BRCOND, DL, N->getVTList(), N->getOperand(0), in combineBrCond()
52560 SourceConst = DAG.getNode(N->getOpcode(), DL, {VT, MVT::Other}, in combineVectorCompareAndMaskUnaryOp()
52563 SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0)); in combineVectorCompareAndMaskUnaryOp()
52566 SDValue NewAnd = DAG.getNode(ISD::AND, DL, IntVT, Op0->getOperand(0), in combineVectorCompareAndMaskUnaryOp()
52607 SDValue NewExtElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, TruncVT, in combineToFPTruncExtElt()
52609 return DAG.getNode(N->getOpcode(), DL, N->getValueType(0), NewExtElt); in combineToFPTruncExtElt()
52632 SDValue P = DAG.getNode(ISD::ZERO_EXTEND, dl, DstVT, Op0); in combineUIntToFP()
52634 return DAG.getNode(ISD::STRICT_UINT_TO_FP, dl, {VT, MVT::Other}, in combineUIntToFP()
52636 return DAG.getNode(ISD::UINT_TO_FP, dl, VT, P); in combineUIntToFP()
52646 SDValue P = DAG.getNode(ISD::ZERO_EXTEND, dl, DstVT, Op0); in combineUIntToFP()
52650 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other}, in combineUIntToFP()
52652 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P); in combineUIntToFP()
52660 return DAG.getNode(ISD::STRICT_SINT_TO_FP, SDLoc(N), {VT, MVT::Other}, in combineUIntToFP()
52662 return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, Op0); in combineUIntToFP()
52695 SDValue P = DAG.getNode(ISD::SIGN_EXTEND, dl, DstVT, Op0); in combineSIntToFP()
52697 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other}, in combineSIntToFP()
52699 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P); in combineSIntToFP()
52709 SDValue P = DAG.getNode(ISD::SIGN_EXTEND, dl, DstVT, Op0); in combineSIntToFP()
52711 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other}, in combineSIntToFP()
52713 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P); in combineSIntToFP()
52728 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, Op0); in combineSIntToFP()
52730 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other}, in combineSIntToFP()
52732 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, Trunc); in combineSIntToFP()
52741 return DAG.getNode(X86ISD::STRICT_CVTSI2P, dl, {VT, MVT::Other}, in combineSIntToFP()
52743 return DAG.getNode(X86ISD::CVTSI2P, dl, VT, Shuf); in combineSIntToFP()
52751 LoadSDNode *Ld = cast<LoadSDNode>(Op0.getNode()); in combineSIntToFP()
52762 if (Ld->isSimple() && !VT.isVector() && ISD::isNormalLoad(Op0.getNode()) && in combineSIntToFP()
52879 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, in combineAddOrSubToADCOrSBB()
52892 SDValue NewSub = DAG.getNode( in combineAddOrSubToADCOrSBB()
52893 X86ISD::SUB, SDLoc(EFLAGS), EFLAGS.getNode()->getVTList(), in combineAddOrSubToADCOrSBB()
52895 SDValue NewEFLAGS = SDValue(NewSub.getNode(), EFLAGS.getResNo()); in combineAddOrSubToADCOrSBB()
52896 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, in combineAddOrSubToADCOrSBB()
52906 return DAG.getNode(IsSub ? X86ISD::SBB : X86ISD::ADC, DL, in combineAddOrSubToADCOrSBB()
52921 if (EFLAGS.getOpcode() == X86ISD::SUB && EFLAGS.getNode()->hasOneUse() && in combineAddOrSubToADCOrSBB()
52925 DAG.getNode(X86ISD::SUB, SDLoc(EFLAGS), EFLAGS.getNode()->getVTList(), in combineAddOrSubToADCOrSBB()
52928 return DAG.getNode(IsSub ? X86ISD::SBB : X86ISD::ADC, DL, in combineAddOrSubToADCOrSBB()
52937 return DAG.getNode(IsSub ? X86ISD::ADC : X86ISD::SBB, DL, in combineAddOrSubToADCOrSBB()
52951 if (EFLAGS.getOpcode() == X86ISD::SUB && EFLAGS.getNode()->hasOneUse() && in combineAddOrSubToADCOrSBB()
52955 DAG.getNode(X86ISD::SUB, SDLoc(EFLAGS), EFLAGS.getNode()->getVTList(), in combineAddOrSubToADCOrSBB()
52958 return DAG.getNode(IsSub ? X86ISD::ADC : X86ISD::SBB, DL, in combineAddOrSubToADCOrSBB()
52986 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Zero, Z); in combineAddOrSubToADCOrSBB()
52987 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, in combineAddOrSubToADCOrSBB()
52989 SDValue(Neg.getNode(), 1)); in combineAddOrSubToADCOrSBB()
53000 SDValue Cmp1 = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Z, One); in combineAddOrSubToADCOrSBB()
53001 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, in combineAddOrSubToADCOrSBB()
53010 SDValue Cmp1 = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Z, One); in combineAddOrSubToADCOrSBB()
53018 return DAG.getNode(IsSub ? X86ISD::ADC : X86ISD::SBB, DL, VTs, X, in combineAddOrSubToADCOrSBB()
53023 return DAG.getNode(IsSub ? X86ISD::SBB : X86ISD::ADC, DL, VTs, X, in combineAddOrSubToADCOrSBB()
53044 DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), ADCOrSBB); in combineAddOrSubToADCOrSBB()
53078 Op = DAG.getNode(ISD::AND, dl, VT, Op.getOperand(0), in combineCMP()
53080 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op, in combineCMP()
53092 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Src, in combineCMP()
53111 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op, in combineCMP()
53146 SDValue Op0 = DAG.getNode(ISD::TRUNCATE, dl, VT, Op.getOperand(0)); in combineCMP()
53147 SDValue Op1 = DAG.getNode(ISD::TRUNCATE, dl, VT, Op.getOperand(1)); in combineCMP()
53151 Op = DAG.getNode(NewOpc, dl, VTs, Op0, Op1); in combineCMP()
53155 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op, in combineCMP()
53176 SDValue Res = DAG.getNode(GenericOpc, DL, VT, LHS, RHS); in combineX86AddSub()
53187 Op = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Op); in combineX86AddSub()
53208 return DAG.getNode(X86ISD::SBB, SDLoc(N), VTs, LHS, RHS, Flags); in combineSBB()
53215 return DAG.getNode(X86ISD::SBB, SDLoc(N), N->getVTList(), LHS.getOperand(0), in combineSBB()
53232 return DAG.getNode(X86ISD::ADC, SDLoc(N), N->getVTList(), RHS, LHS, in combineADC()
53245 SDValue Res1 = DAG.getNode( in combineADC()
53247 DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, in combineADC()
53259 return DAG.getNode(X86ISD::ADC, DL, N->getVTList(), in combineADC()
53267 return DAG.getNode(X86ISD::ADC, SDLoc(N), VTs, LHS, RHS, Flags); in combineADC()
53274 return DAG.getNode(X86ISD::ADC, SDLoc(N), N->getVTList(), LHS.getOperand(0), in combineADC()
53365 if (!canReduceVMulWidth(Mul.getNode(), DAG, Mode) || in matchPMADDWD()
53371 SDValue N0 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Mul.getOperand(0)); in matchPMADDWD()
53372 SDValue N1 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Mul.getOperand(1)); in matchPMADDWD()
53380 return DAG.getNode(X86ISD::VPMADDWD, DL, ResVT, Ops[0], Ops[1]); in matchPMADDWD()
53505 return DAG.getNode(X86ISD::VPMADDWD, DL, ResVT, Ops[0], Ops[1]); in matchPMADDWD_2()
53513 In0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT16, In0, in matchPMADDWD_2()
53517 In1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT16, In1, in matchPMADDWD_2()
53565 return DAG.getNode(X86ISD::VPMADDWD, DL, VT, LHS, RHS); in combineAddOfPMADDWD()
53617 return MemNode && MemNode->getBasePtr().getNode() == N; in pushAddIntoCmovOfConsts()
53623 FalseOp = DAG.getNode(ISD::ADD, DL, VT, X, FalseOp); in pushAddIntoCmovOfConsts()
53624 TrueOp = DAG.getNode(ISD::ADD, DL, VT, X, TrueOp); in pushAddIntoCmovOfConsts()
53625 Cmov = DAG.getNode(X86ISD::CMOV, DL, VT, FalseOp, TrueOp, in pushAddIntoCmovOfConsts()
53627 return DAG.getNode(ISD::ADD, DL, VT, Cmov, Y); in pushAddIntoCmovOfConsts()
53631 FalseOp = DAG.getNode(ISD::ADD, DL, VT, OtherOp, FalseOp); in pushAddIntoCmovOfConsts()
53632 TrueOp = DAG.getNode(ISD::ADD, DL, VT, OtherOp, TrueOp); in pushAddIntoCmovOfConsts()
53633 return DAG.getNode(X86ISD::CMOV, DL, VT, FalseOp, TrueOp, Cmov.getOperand(2), in pushAddIntoCmovOfConsts()
53669 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op0.getOperand(0)); in combineAdd()
53670 return DAG.getNode(ISD::SUB, DL, VT, Op1, SExt); in combineAdd()
53676 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op1.getOperand(0)); in combineAdd()
53677 return DAG.getNode(ISD::SUB, DL, VT, Op0, SExt); in combineAdd()
53685 return DAG.getNode(X86ISD::ADC, SDLoc(Op0), Op0->getVTList(), Op1, in combineAdd()
53727 SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, VT, TrueOp, FalseOp, in combineSubABS()
53730 return DAG.getNode(ISD::ADD, DL, VT, N0, Cmov); in combineSubABS()
53757 SDValue NewXor = DAG.getNode(ISD::XOR, SDLoc(Op1), VT, Op1.getOperand(0), in combineSub()
53760 DAG.getNode(ISD::ADD, DL, VT, Op0, DAG.getConstant(1, DL, VT)); in combineSub()
53761 return DAG.getNode(ISD::ADD, DL, VT, NewXor, NewAdd); in combineSub()
53775 return DAG.getNode(X86ISD::SBB, SDLoc(Op1), Op1->getVTList(), Op0, in combineSub()
53784 SDValue ADC = DAG.getNode(X86ISD::ADC, SDLoc(Op1), Op1->getVTList(), Op0, in combineSub()
53786 return DAG.getNode(ISD::SUB, SDLoc(N), Op0.getValueType(), ADC.getValue(0), in combineSub()
53822 return ISD::isBuildVectorAllZeros(Op.getNode()); in combineConcatVectorOps()
53834 return DAG.getNode(Op0.getOpcode(), DL, VT, Op0.getOperand(0)); in combineConcatVectorOps()
53839 if (ISD::isNormalLoad(Op0.getNode()) || in combineConcatVectorOps()
53860 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, in combineConcatVectorOps()
53861 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f64, in combineConcatVectorOps()
53871 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, Op0.getOperand(0)); in combineConcatVectorOps()
53898 return DAG.getNode(X86ISD::VPERM2X128, DL, VT, in combineConcatVectorOps()
53916 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Subs); in combineConcatVectorOps()
53936 return DAG.getNode(X86ISD::MOVDDUP, DL, VT, in combineConcatVectorOps()
53944 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
53954 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
53965 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
53973 Res = DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f32, Res, in combineConcatVectorOps()
53981 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
53989 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
54002 if (!getTargetShuffleMask(Ops[i].getNode(), OpVT, false, SubOps, in combineConcatVectorOps()
54021 return DAG.getNode(X86ISD::VPERMV3, DL, VT, Src0, Mask, Src1); in combineConcatVectorOps()
54055 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
54066 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
54079 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
54095 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
54107 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
54119 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
54204 if ((Vec.isUndef() || ISD::isBuildVectorAllZeros(Vec.getNode())) && in combineINSERT_SUBVECTOR()
54205 (SubVec.isUndef() || ISD::isBuildVectorAllZeros(SubVec.getNode()))) in combineINSERT_SUBVECTOR()
54208 if (ISD::isBuildVectorAllZeros(Vec.getNode())) { in combineINSERT_SUBVECTOR()
54212 ISD::isBuildVectorAllZeros(SubVec.getOperand(0).getNode())) { in combineINSERT_SUBVECTOR()
54214 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT, in combineINSERT_SUBVECTOR()
54229 ISD::isBuildVectorAllZeros(Ins.getOperand(0).getNode()) && in combineINSERT_SUBVECTOR()
54232 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT, in combineINSERT_SUBVECTOR()
54247 !(Vec.isUndef() || ISD::isBuildVectorAllZeros(Vec.getNode())))) { in combineINSERT_SUBVECTOR()
54277 ISD::isBuildVectorAllZeros(SubVectorOps[1].getNode())) in combineINSERT_SUBVECTOR()
54278 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT, in combineINSERT_SUBVECTOR()
54285 return DAG.getNode(X86ISD::VBROADCAST, dl, OpVT, SubVec.getOperand(0)); in combineINSERT_SUBVECTOR()
54328 !collectConcatOps(Sel.getOperand(0).getNode(), CatOps, DAG)) in narrowExtractedVectorSelect()
54416 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), VT, in combineEXTRACT_SUBVECTOR()
54427 if (ISD::isBuildVectorAllZeros(InVec.getNode())) in combineEXTRACT_SUBVECTOR()
54430 if (ISD::isBuildVectorAllOnes(InVec.getNode())) { in combineEXTRACT_SUBVECTOR()
54449 SDValue NewExt = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, in combineEXTRACT_SUBVECTOR()
54452 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, NewExt, in combineEXTRACT_SUBVECTOR()
54502 return DAG.getNode(X86ISD::CVTSI2P, SDLoc(N), VT, InVec.getOperand(0)); in combineEXTRACT_SUBVECTOR()
54507 return DAG.getNode(X86ISD::CVTUI2P, SDLoc(N), VT, InVec.getOperand(0)); in combineEXTRACT_SUBVECTOR()
54512 return DAG.getNode(X86ISD::VFPEXT, SDLoc(N), VT, InVec.getOperand(0)); in combineEXTRACT_SUBVECTOR()
54529 return DAG.getNode(ExtOp, DL, VT, Ext); in combineEXTRACT_SUBVECTOR()
54539 return DAG.getNode(InOpcode, DL, VT, Ext0, Ext1, Ext2); in combineEXTRACT_SUBVECTOR()
54547 return DAG.getNode(InOpcode, DL, VT, Ext); in combineEXTRACT_SUBVECTOR()
54554 return DAG.getNode(InOpcode, DL, VT, Ext0); in combineEXTRACT_SUBVECTOR()
54566 return DAG.getNode(InOpcode, DL, VT, Ext, InVec.getOperand(1)); in combineEXTRACT_SUBVECTOR()
54584 return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v1i1, in combineScalarToVector()
54593 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src.getOperand(0), in combineScalarToVector()
54613 VT, DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i32, in combineScalarToVector()
54620 return DAG.getNode(X86ISD::MOVQ2DQ, DL, VT, Src.getOperand(0)); in combineScalarToVector()
54652 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), RHS, LHS); in combinePMULDQ()
54656 if (ISD::isBuildVectorAllZeros(RHS.getNode())) in combinePMULDQ()
54679 return DAG.getNode(N->getOpcode(), dl, MVT::v2i64, LHS, RHS); in combinePMULDQ()
54689 return DAG.getNode(N->getOpcode(), dl, MVT::v2i64, LHS, RHS); in combinePMULDQ()
54704 if (ISD::isBuildVectorAllZeros(LHS.getNode()) || in combineVPMADD()
54705 ISD::isBuildVectorAllZeros(RHS.getNode())) in combineVPMADD()
54727 if (!DCI.isBeforeLegalizeOps() && ISD::isNormalLoad(In.getNode()) && in combineEXTEND_VECTOR_INREG()
54748 return DAG.getNode(Opcode, DL, VT, In.getOperand(0)); in combineEXTEND_VECTOR_INREG()
54757 return DAG.getNode(Opcode, DL, VT, In.getOperand(0).getOperand(0)); in combineEXTEND_VECTOR_INREG()
54790 if (ISD::isBuildVectorAllZeros(N->getOperand(0).getNode())) in combineKSHIFT()
54817 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, in combineFP16_TO_FP()
54819 Res = DAG.getNode(X86ISD::CVTPS2PH, dl, MVT::v8i16, Res, in combineFP16_TO_FP()
54821 Res = DAG.getNode(X86ISD::CVTPH2PS, dl, MVT::v4f32, Res); in combineFP16_TO_FP()
54822 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, Res, in combineFP16_TO_FP()
54863 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i16, Ops); in combineFP_EXTEND()
54871 Cvt = DAG.getNode(X86ISD::STRICT_CVTPH2PS, dl, {CvtVT, MVT::Other}, in combineFP_EXTEND()
54875 Cvt = DAG.getNode(X86ISD::CVTPH2PS, dl, CvtVT, Src); in combineFP_EXTEND()
54880 Cvt = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2f32, Cvt, in combineFP_EXTEND()
54887 Cvt = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {VT, MVT::Other}, in combineFP_EXTEND()
54895 return DAG.getNode(ISD::FP_EXTEND, dl, VT, Cvt); in combineFP_EXTEND()
54962 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src, in combineFP_ROUND()
54971 Cvt = DAG.getNode(X86ISD::STRICT_CVTPS2PH, dl, {CvtVT, MVT::Other}, in combineFP_ROUND()
54975 Cvt = DAG.getNode(X86ISD::CVTPS2PH, dl, CvtVT, Src, Rnd); in combineFP_ROUND()
54981 Cvt = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, IntVT, Cvt, in combineFP_ROUND()
54997 if (ISD::isNormalLoad(Src.getNode()) && Src.hasOneUse()) { in combineMOVDQ2Q()
54998 LoadSDNode *LN = cast<LoadSDNode>(Src.getNode()); in combineMOVDQ2Q()
55265 return DAG.getNode(X86ISD::NT_BRIND, dl, MVT::Other, Value, Addr); in expandIndirectJTBranch()
55739 if (Flag.getNode()) { in LowerAsmOutputForConstraint()
55747 SDValue Result = DAG.getNode(ISD::ZERO_EXTEND, DL, OpInfo.ConstraintVT, CC); in LowerAsmOutputForConstraint()
55890 if (Result.getNode()) { in LowerAsmOperandForConstraint()